26 Oct 2017 20:39:27.755 +0700  main                  AD.BulkUpdateAPICaller     ERROR   Failed to update for all accounts event type [BrowserRecord], url [http://appdynamicspoc3:9090/v3/events/BrowserRecord/_bulk?timeout=3600000], key [YWQzMm...], status code [500]
26 Oct 2017 20:39:27.756 +0700  main                  AD.BulkUpdateAPICaller     ERROR   {"statusCode":500,"code":"Unexpected.Exception","message":"UnavailableShardsException[[async_executor_status][0] [1] shardIt, [0] active : Timeout waiting for [1m], request: indices:data/write/update]","developerMessage":"com.appdynamics.analytics.shared.rest.exceptions.JobException: UnavailableShardsException[[async_executor_status][0] [1] shardIt, [0] active : Timeout waiting for [1m], request: indices:data/write/update]\n\tat com.appdynamics.analytics.processor.event.asyncjob.AsynchronousEventTypeUpdateService.updateJobIndex(AsynchronousEventTypeUpdateService.java:180)\n\tat com.appdynamics.analytics.processor.event.asyncjob.AsyncJob.updateJobStatus(AsyncJob.java:105)\n\tat com.appdynamics.analytics.processor.event.asyncjob.AsyncJob.updateJobStatus(AsyncJob.java:95)\n\tat com.appdynamics.analytics.processor.event.asyncjob.AsyncJob.moveToPendingState(AsyncJob.java:86)\n\tat com.appdynamics.analytics.processor.event.asyncjob.AsyncJob.<init>(AsyncJob.java:45)\n\tat com.appdynamics.analytics.processor.event.asyncjob.BulkUpdateAsyncJob.<init>(BulkUpdateAsyncJob.java:54)\n\tat com.appdynamics.analytics.processor.event.asyncjob.AsynchronousEventTypeUpdateService.asyncBulkUpdateSubmit(AsynchronousEventTypeUpdateService.java:131)\n\tat com.appdynamics.analytics.processor.event.resource.EventServiceResource.bulkUpdateEventType(EventServiceResource.java:263)\n\tat sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat org.glassfish.jersey.server.model.internal.ResourceMethodInvocationHandlerFactory$1.invoke(ResourceMethodInvocationHandlerFactory.java:81)\n\tat org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher$1.run(AbstractJavaResourceMethodDispatcher.java:144)\n\tat org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.invoke(AbstractJavaResourceMethodDispatcher.java:161)\n\tat org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$ResponseOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.java:160)\n\tat org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.java:99)\n\tat org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:389)\n\tat org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:347)\n\tat org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:102)\n\tat org.glassfish.jersey.server.ServerRuntime$2.run(ServerRuntime.java:326)\n\tat org.glassfish.jersey.internal.Errors$1.call(Errors.java:271)\n\tat org.glassfish.jersey.internal.Errors$1.call(Errors.java:267)\n\tat org.glassfish.jersey.internal.Errors.process(Errors.java:315)\n\tat org.glassfish.jersey.internal.Errors.process(Errors.java:297)\n\tat org.glassfish.jersey.internal.Errors.process(Errors.java:267)\n\tat org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:317)\n\tat org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:305)\n\tat org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:1154)\n\tat org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:471)\n\tat org.glassfish.jersey.servlet.WebComponent.service(WebComponent.java:425)\n\tat org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:383)\n\tat org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:336)\n\tat org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:223)\n\tat io.dropwizard.jetty.NonblockingServletHolder.handle(NonblockingServletHolder.java:49)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1669)\n\tat org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)\n\tat org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)\n\tat io.dropwizard.jetty.BiDiGzipFilter.doFilter(BiDiGzipFilter.java:132)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)\n\tat io.dropwizard.servlets.ThreadNameFilter.doFilter(ThreadNameFilter.java:29)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)\n\tat io.dropwizard.jersey.filter.AllowedMethodsFilter.handle(AllowedMethodsFilter.java:43)\n\tat io.dropwizard.jersey.filter.AllowedMethodsFilter.doFilter(AllowedMethodsFilter.java:38)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)\n\tat com.appdynamics.common.framework.util.ClickJackSecurityFilter.doFilter(ClickJackSecurityFilter.java:107)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)\n\tat com.codahale.metrics.jetty9.InstrumentedHandler.handle(InstrumentedHandler.java:240)\n\tat io.dropwizard.jetty.RoutingHandler.handle(RoutingHandler.java:51)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)\n\tat org.eclipse.jetty.server.handler.RequestLogHandler.handle(RequestLogHandler.java:95)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)\n\tat org.eclipse.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:159)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:499)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)\n\tat org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)\n\tat java.lang.Thread.run(Thread.java:745)\nCaused by: UnavailableShardsException[[async_executor_status][0] [1] shardIt, [0] active : Timeout waiting for [1m], request: indices:data/write/update]\n\tat org.elasticsearch.action.support.single.instance.TransportInstanceSingleOperationAction$AsyncSingleAction.retry(TransportInstanceSingleOperationAction.java:213)\n\tat org.elasticsearch.action.support.single.instance.TransportInstanceSingleOperationAction$AsyncSingleAction.doStart(TransportInstanceSingleOperationAction.java:168)\n\tat org.elasticsearch.action.support.single.instance.TransportInstanceSingleOperationAction$AsyncSingleAction$2.onTimeout(TransportInstanceSingleOperationAction.java:234)\n\tat org.elasticsearch.cluster.ClusterStateObserver$ObserverClusterStateListener.onTimeout(ClusterStateObserver.java:236)\n\tat org.elasticsearch.cluster.service.InternalClusterService$NotifyTimeout.run(InternalClusterService.java:804)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)\n\t... 1 more\n"}
26 Oct 2017 20:39:27.756 +0700  main                  AD.BulkUpdateAPICaller     ERROR   failed bulk update for event type [BrowserRecord], response [statusCode=ERROR, statusMessage=Failed to update for all accounts event type [BrowserRecord], url [http://appdynamicspoc3:9090/v3/events/BrowserRecord/_bulk?timeout=3600000], key [YWQzMm...], status code [500], errorStack=null], url [http://appdynamicspoc3:9090/v3/events/BrowserRecord/_bulk?timeout=3600000]
26 Oct 2017 20:39:27.757 +0700  main                  AD.BulkUpdateAPICaller     INFO    Start updating SessionRecord
26 Oct 2017 20:40:27.775 +0700  main                  AD.BulkUpdateAPICaller     ERROR   Failed to update for all accounts event type [SessionRecord], url [http://appdynamicspoc3:9090/v3/events/SessionRecord/_bulk?timeout=3600000], key [YWQzMm...], status code [500]
26 Oct 2017 20:40:27.776 +0700  main                  AD.BulkUpdateAPICaller     ERROR   {"statusCode":500,"code":"Unexpected.Exception","message":"UnavailableShardsException[[async_executor_status][0] [1] shardIt, [0] active : Timeout waiting for [1m], request: indices:data/write/update]","developerMessage":"com.appdynamics.analytics.shared.rest.exceptions.JobException: UnavailableShardsException[[async_executor_status][0] [1] shardIt, [0] active : Timeout waiting for [1m], request: indices:data/write/update]\n\tat com.appdynamics.analytics.processor.event.asyncjob.AsynchronousEventTypeUpdateService.updateJobIndex(AsynchronousEventTypeUpdateService.java:180)\n\tat com.appdynamics.analytics.processor.event.asyncjob.AsyncJob.updateJobStatus(AsyncJob.java:105)\n\tat com.appdynamics.analytics.processor.event.asyncjob.AsyncJob.updateJobStatus(AsyncJob.java:95)\n\tat com.appdynamics.analytics.processor.event.asyncjob.AsyncJob.moveToPendingState(AsyncJob.java:86)\n\tat com.appdynamics.analytics.processor.event.asyncjob.AsyncJob.<init>(AsyncJob.java:45)\n\tat com.appdynamics.analytics.processor.event.asyncjob.BulkUpdateAsyncJob.<init>(BulkUpdateAsyncJob.java:54)\n\tat com.appdynamics.analytics.processor.event.asyncjob.AsynchronousEventTypeUpdateService.asyncBulkUpdateSubmit(AsynchronousEventTypeUpdateService.java:131)\n\tat com.appdynamics.analytics.processor.event.resource.EventServiceResource.bulkUpdateEventType(EventServiceResource.java:263)\n\tat sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat org.glassfish.jersey.server.model.internal.ResourceMethodInvocationHandlerFactory$1.invoke(ResourceMethodInvocationHandlerFactory.java:81)\n\tat org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher$1.run(AbstractJavaResourceMethodDispatcher.java:144)\n\tat org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.invoke(AbstractJavaResourceMethodDispatcher.java:161)\n\tat org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$ResponseOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.java:160)\n\tat org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.java:99)\n\tat org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:389)\n\tat org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:347)\n\tat org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:102)\n\tat org.glassfish.jersey.server.ServerRuntime$2.run(ServerRuntime.java:326)\n\tat org.glassfish.jersey.internal.Errors$1.call(Errors.java:271)\n\tat org.glassfish.jersey.internal.Errors$1.call(Errors.java:267)\n\tat org.glassfish.jersey.internal.Errors.process(Errors.java:315)\n\tat org.glassfish.jersey.internal.Errors.process(Errors.java:297)\n\tat org.glassfish.jersey.internal.Errors.process(Errors.java:267)\n\tat org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:317)\n\tat org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:305)\n\tat org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:1154)\n\tat org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:471)\n\tat org.glassfish.jersey.servlet.WebComponent.service(WebComponent.java:425)\n\tat org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:383)\n\tat org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:336)\n\tat org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:223)\n\tat io.dropwizard.jetty.NonblockingServletHolder.handle(NonblockingServletHolder.java:49)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1669)\n\tat org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)\n\tat org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)\n\tat io.dropwizard.jetty.BiDiGzipFilter.doFilter(BiDiGzipFilter.java:132)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)\n\tat io.dropwizard.servlets.ThreadNameFilter.doFilter(ThreadNameFilter.java:29)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)\n\tat io.dropwizard.jersey.filter.AllowedMethodsFilter.handle(AllowedMethodsFilter.java:43)\n\tat io.dropwizard.jersey.filter.AllowedMethodsFilter.doFilter(AllowedMethodsFilter.java:38)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)\n\tat com.appdynamics.common.framework.util.ClickJackSecurityFilter.doFilter(ClickJackSecurityFilter.java:107)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)\n\tat com.codahale.metrics.jetty9.InstrumentedHandler.handle(InstrumentedHandler.java:240)\n\tat io.dropwizard.jetty.RoutingHandler.handle(RoutingHandler.java:51)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)\n\tat org.eclipse.jetty.server.handler.RequestLogHandler.handle(RequestLogHandler.java:95)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)\n\tat org.eclipse.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:159)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:499)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)\n\tat org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)\n\tat java.lang.Thread.run(Thread.java:745)\nCaused by: UnavailableShardsException[[async_executor_status][0] [1] shardIt, [0] active : Timeout waiting for [1m], request: indices:data/write/update]\n\tat org.elasticsearch.action.support.single.instance.TransportInstanceSingleOperationAction$AsyncSingleAction.retry(TransportInstanceSingleOperationAction.java:213)\n\tat org.elasticsearch.action.support.single.instance.TransportInstanceSingleOperationAction$AsyncSingleAction.doStart(TransportInstanceSingleOperationAction.java:168)\n\tat org.elasticsearch.action.support.single.instance.TransportInstanceSingleOperationAction$AsyncSingleAction$2.onTimeout(TransportInstanceSingleOperationAction.java:234)\n\tat org.elasticsearch.cluster.ClusterStateObserver$ObserverClusterStateListener.onTimeout(ClusterStateObserver.java:236)\n\tat org.elasticsearch.cluster.service.InternalClusterService$NotifyTimeout.run(InternalClusterService.java:804)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)\n\t... 1 more\n"}
26 Oct 2017 20:40:27.776 +0700  main                  AD.BulkUpdateAPICaller     ERROR   failed bulk update for event type [SessionRecord], response [statusCode=ERROR, statusMessage=Failed to update for all accounts event type [SessionRecord], url [http://appdynamicspoc3:9090/v3/events/SessionRecord/_bulk?timeout=3600000], key [YWQzMm...], status code [500], errorStack=null], url [http://appdynamicspoc3:9090/v3/events/SessionRecord/_bulk?timeout=3600000]
26 Oct 2017 20:40:27.785 +0700  main                  AD.ALL                     ERROR   
 |	Error Starting EUM Processor Server
 |	java.lang.RuntimeException: failed to update Analytics events. Please see log for details of errors!
 |		at com.appdynamics.eum.processor.EUMProcessorServerApplication.syncEumAccountsToAnalyticsAccounts(EUMProcessorServerApplication.java:287)
 |		at com.appdynamics.eum.processor.EUMProcessorServerApplication.run(EUMProcessorServerApplication.java:210)
 |		at com.appdynamics.eum.processor.EUMProcessorServerApplication.run(EUMProcessorServerApplication.java:144)
 |		at io.dropwizard.cli.EnvironmentCommand.run(EnvironmentCommand.java:40)
 |		at io.dropwizard.cli.ConfiguredCommand.run(ConfiguredCommand.java:77)
 |		at io.dropwizard.cli.Cli.run(Cli.java:70)
 |		at io.dropwizard.Application.run(Application.java:80)
 |		at com.appdynamics.eumcloud.EUMProcessorServer.main(EUMProcessorServer.java:40)