I'm struggling to pinpoint the source class of this concurrent modification exception. I've run into these before but normally it'll be pretty easy to tell where the issue is occurring. We're unable to replicate this issue in lower environments. I'm providing the logs. Please let me know if you need any additional information.
INFO c.f.f.c.u.CallerInterceptor - API :/v1/test/crossingTest/ Time Taken (in ms) : 1644
Apr 23, 2020 @ 08:33:08.884 2020-04-23 08:33:08,883 [nio-8080-exec-2] INFO c.f.f.c.u.CallerInterceptor - API :/v1/test/crossingTest/ Time Taken (in ms) : 1683
Apr 23, 2020 @ 08:33:08.886 cbz.com - [2020-04-23T12:33:06.949+0000] "POST /v1/test/crossingTest/ H
Apr 23, 2020 @ 08:33:08.888 at javax.servlet.http.HttpServlet.service(HttpServlet.java:741)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.valves.RemoteIpValve.invoke(RemoteIpValve.java:679)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:200)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:106)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
Apr 23, 2020 @ 08:33:08.888 at javax.servlet.http.HttpServlet.service(HttpServlet.java:660)
Apr 23, 2020 @ 08:33:08.888 at java.util.ArrayList$Itr.checkForComodification(ArrayList.java:909)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
Apr 23, 2020 @ 08:33:08.888 at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:53)
Apr 23, 2020 @ 08:33:08.888 at com.business.xxxx.cbzbusservice.auth.RestApiAuthSecurityFilter.doFilter(RestApiAuthSecurityFilter.java:73)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:99)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.web.filter.FormContentFilter.doFilterInternal(FormContentFilter.java:92)
Apr 23, 2020 @ 08:33:08.888 at com.business.xxxx.cbzbusservice.cache.CacheService.getFromCache(CacheService.java:76)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:644)
Apr 23, 2020 @ 08:33:08.888 at java.lang.String.valueOf(String.java:2994)
Apr 23, 2020 @ 08:33:08.888 at sun.reflect.GeneratedMethodAccessor142.invoke(Unknown Source)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
Apr 23, 2020 @ 08:33:08.888 at com.business.xxxx.cbzbusservice.util.CallerInterceptor.upldateConsumerLog(CallerInterceptor.java:168)
Apr 23, 2020 @ 08:33:08.888 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:139)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:93)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:189)
Apr 23, 2020 @ 08:33:08.888 at java.lang.reflect.Method.invoke(Method.java:498)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
Apr 23, 2020 @ 08:33:08.888 java.util.ConcurrentModificationException: null
Apr 23, 2020 @ 08:33:08.888 at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:626)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:942)
Apr 23, 2020 @ 08:33:08.888 at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:408)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:102)
Apr 23, 2020 @ 08:33:08.888 at sun.reflect.GeneratedMethodAccessor74.invoke(Unknown Source)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
Apr 23, 2020 @ 08:33:08.888 at java.lang.reflect.Method.invoke(Method.java:498)
Apr 23, 2020 @ 08:33:08.888 2020-04-23 08:33:08,887 [nio-8080-exec-8] ERROR .c.C.[.[.[.[dispatcherServlet] - Servlet.service() for servlet [dispatcherServlet] in context with path [] threw exception [Request processing failed; nested exception is java.util.ConcurrentModificationException] with root cause
Apr 23, 2020 @ 08:33:08.888 at java.util.AbstractCollection.toString(AbstractCollection.java:461)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.aop.aspectj.AspectJAfterReturningAdvice.afterReturning(AspectJAfterReturningAdvice.java:66)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1005)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:200)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:490)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.aop.framework.adapter.AfterReturningAdviceInterceptor.invoke(AfterReturningAdviceInterceptor.java:56)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:138)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.web.filter.HiddenHttpMethodFilter.doFilterInternal(HiddenHttpMethodFilter.java:93)
Apr 23, 2020 @ 08:33:08.888 at java.lang.StringBuilder.append(StringBuilder.java:131)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1038)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:800)
Apr 23, 2020 @ 08:33:08.888 at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66)
Apr 23, 2020 @ 08:33:08.888 at com.business.xxxx.cbzbusservice.util.CallerInterceptor.afterExecution(CallerInterceptor.java:140)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:74)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
Apr 23, 2020 @ 08:33:08.888 at com.business.xxxx.cbzbusservice.controller.BusinessServiceController$$EnhancerBySpringCGLIB$$cd31598c.evaluateCrossingTest(<generated>)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.filterAndRecordMetrics(WebMvcMetricsFilter.java:117)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:882)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:688)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:908)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.boot.actuate.web.trace.servlet.HttpTraceFilter.doFilterInternal(HttpTraceFilter.java:90)
Apr 23, 2020 @ 08:33:08.888 at java.util.ArrayList$Itr.next(ArrayList.java:859)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
Apr 23, 2020 @ 08:33:08.888 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
Apr 23, 2020 @ 08:33:08.888 at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:343)
Apr 23, 2020 @ 08:33:08.888 at org.cloudfoundry.router.ClientCertificateMapper.doFilter(ClientCertificateMapper.java:77)
Apr 23, 2020 @ 08:33:08.888 at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
Apr 23, 2020 @ 08:33:08.889 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
Apr 23, 2020 @ 08:33:08.889 at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1415)
Apr 23, 2020 @ 08:33:08.889 at java.lang.Thread.run(Thread.java:748)
Apr 23, 2020 @ 08:33:08.889 at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
Apr 23, 2020 @ 08:33:08.889 at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
Apr 23, 2020 @ 08:33:08.889 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
Apr 23, 2020 @ 08:33:08.889 at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:834)
The source method for the error was Apr 23, 2020 @ 08:33:08.888 at com.business.xxxx.cbzbusservice.cache.CacheService.getFromCache(CacheService.java:76)