目录
1. 说明
线上出现异常，但是查看节点状态正常，因为使用了容器,挂掉了会重启。因为jvm配置了-XX：HeapDumpOnOutOfMemoryError 参数，出现OMM就会将当时线程和jvm内存情况转存起来可参考:《JVM参数配置》
2. 排查
1.将运维发过来的hprof导入到 vm 可参考《JVM监控》
2.我们去当时的线程日志查看
3.搜索13219这个线程id找到线程查看堆栈信息
4.分析堆栈找到业务代码 分析问题原因
具体看标红处 发现是导入功能问题 导致oom
"http-nio-8080-exec-11" daemon prio=5 tid=13219 RUNNABLE
at java.lang.OutOfMemoryError.<init>(OutOfMemoryError.java:48)
at org.apache.xmlbeans.impl.store.CharUtil.allocate(CharUtil.java:397)
local variable: org.apache.xmlbeans.impl.store.CharUtil#2
at org.apache.xmlbeans.impl.store.CharUtil.saveChars(CharUtil.java:506)
local variable: org.apache.xmlbeans.impl.store.CharUtil#2
local variable: java.lang.String#622892
local variable: char[]#2750421
at org.apache.xmlbeans.impl.store.Cur$CurLoadContext.text(Cur.java:2927)
local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1
at org.apache.xmlbeans.impl.store.Cur$CurLoadContext.attr(Cur.java:3048)
local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1
local variable: java.lang.String#622892
local variable: org.apache.xmlbeans.impl.store.Xobj$AttrXobj#2119297
at org.apache.xmlbeans.impl.store.Locale.loadNode(Locale.java:1440)
local variable: org.apache.xmlbeans.impl.store.Locale#3
local variable: com.sun.org.apache.xerces.internal.dom.DeferredElementNSImpl#1062106
local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1
local variable: com.sun.org.apache.xerces.internal.dom.AttributeMap#1061610
at org.apache.xmlbeans.impl.store.Locale.loadNodeChildren(Locale.java:1403)
local variable: org.apache.xmlbeans.impl.store.Locale#3
local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1
local variable: com.sun.org.apache.xerces.internal.dom.DeferredElementNSImpl#1062106
at org.apache.xmlbeans.impl.store.Locale.loadNode(Locale.java:1445)
local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1
at org.apache.xmlbeans.impl.store.Locale.loadNodeChildren(Locale.java:1403)
local variable: org.apache.xmlbeans.impl.store.Locale#3
local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1
local variable: com.sun.org.apache.xerces.internal.dom.DeferredElementNSImpl#243777
at org.apache.xmlbeans.impl.store.Locale.loadNode(Locale.java:1445)
local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1
at org.apache.xmlbeans.impl.store.Locale.loadNodeChildren(Locale.java:1403)
local variable: org.apache.xmlbeans.impl.store.Locale#3
local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1
local variable: com.sun.org.apache.xerces.internal.dom.DeferredElementNSImpl#1
at org.apache.xmlbeans.impl.store.Locale.loadNode(Locale.java:1445)
local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1
at org.apache.xmlbeans.impl.store.Locale.parseToXmlObject(Locale.java:1385)
local variable: org.apache.xmlbeans.impl.store.Locale#3
local variable: com.sun.org.apache.xerces.internal.dom.DeferredElementNSImpl#2
local variable: org.apache.xmlbeans.impl.schema.SchemaTypeImpl#3
local variable: org.apache.xmlbeans.XmlOptions#3
local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1
at org.apache.xmlbeans.impl.store.Locale.parseToXmlObject(Locale.java:1370)
local variable: org.apache.xmlbeans.impl.schema.SchemaTypeLoaderImpl#1
local variable: com.sun.org.apache.xerces.internal.dom.DeferredElementNSImpl#2
local variable: org.apache.xmlbeans.impl.schema.SchemaTypeImpl#3
local variable: org.apache.xmlbeans.XmlOptions#3
local variable: org.apache.xmlbeans.impl.store.Locale#3
local variable: org.apache.xmlbeans.impl.store.Locale#3
at org.apache.xmlbeans.impl.schema.SchemaTypeLoaderBase.parse(SchemaTypeLoaderBase.java:370)
local variable: org.apache.xmlbeans.impl.schema.SchemaTypeLoaderImpl#1
local variable: com.sun.org.apache.xerces.internal.dom.DeferredElementNSImpl#2
local variable: org.apache.xmlbeans.impl.schema.SchemaTypeImpl#3
local variable: org.apache.xmlbeans.XmlOptions#3
at org.apache.poi.POIXMLTypeLoader.parse(POIXMLTypeLoader.java:164)
local variable: java.io.ByteArrayInputStream#1
local variable: org.apache.xmlbeans.impl.schema.SchemaTypeImpl#3
local variable: org.apache.xmlbeans.XmlOptions#3
local variable: com.sun.org.apache.xerces.internal.dom.DeferredDocumentImpl#50
at org.openxmlformats.schemas.spreadsheetml.x2006.main.WorksheetDocument$Factory.parse(<unresolved string 0x0>)
local variable: java.io.ByteArrayInputStream#1
local variable: org.apache.xmlbeans.XmlOptions#3
at org.apache.poi.xssf.usermodel.XSSFSheet.read(XSSFSheet.java:226)
local variable: org.apache.poi.xssf.usermodel.XSSFSheet#1
local variable: java.io.ByteArrayInputStream#1
at org.apache.poi.xssf.usermodel.XSSFSheet.onDocumentRead(XSSFSheet.java:218)
local variable: org.apache.poi.xssf.usermodel.XSSFSheet#1
at org.apache.poi.xssf.usermodel.XSSFWorkbook.parseSheet(XSSFWorkbook.java:443)
local variable: org.apache.poi.xssf.usermodel.XSSFWorkbook#1
local variable: java.util.HashMap#17405
local variable: org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSheetImpl#1
local variable: org.apache.poi.xssf.usermodel.XSSFSheet#1
at org.apache.poi.xssf.usermodel.XSSFWorkbook.onDocumentRead(XSSFWorkbook.java:408)
local variable: org.apache.poi.xssf.usermodel.XSSFWorkbook#1
local variable: org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.WorkbookDocumentImpl#1
local variable: org.apache.poi.xssf.model.ThemesTable#1
local variable: java.util.HashMap#17405
local variable: java.util.HashMap#17406
local variable: org.openxmlformats.schemas.spreadsheetml.x2006.main.CTSheet[]#1
local variable: org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSheetImpl#1
at org.apache.poi.POIXMLDocument.load(POIXMLDocument.java:169)
local variable: java.util.HashMap#17407
at org.apache.poi.xssf.usermodel.XSSFWorkbook.<init>(XSSFWorkbook.java:300)
local variable: org.apache.poi.xssf.usermodel.XSSFWorkbook#1
at com.biaoguoworks.commons.utils.ExcelUtils.readExcel(ExcelUtils.java:204)
local variable: class com.biaoguoworks.scm.logistics.dto.sorting.task.SkuSortingImportDTO
local variable: java.lang.String#254630
local variable: java.util.ArrayList#63983
local variable: java.io.FileInputStream#8
at com.biaoguoworks.scm.logistics.controller.demeter.web.SortingTagController.sortingImport(SortingTagController.java:168)
local variable: com.biaoguoworks.scm.logistics.controller.demeter.web.SortingTagController#1
local variable: com.biaoguoworks.scm.logistics.vo.sorting.SkuSortingImportVO#1
at com.biaoguoworks.scm.logistics.controller.demeter.web.SortingTagController$$FastClassBySpringCGLIB$$9eba9d0d.invoke(<generated>)
at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218)
local variable: org.springframework.cglib.proxy.MethodProxy#4788
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:771)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749)
local variable: org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation#1
at org.springframework.validation.beanvalidation.MethodValidationInterceptor.invoke(MethodValidationInterceptor.java:119)
local variable: org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation#1
local variable: java.lang.Class[]#5445
local variable: org.hibernate.validator.internal.engine.ValidatorImpl#1
local variable: java.lang.reflect.Method#91741
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186)
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749)
local variable: org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation#1
at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:691)
local variable: com.biaoguoworks.scm.logistics.controller.demeter.web.SortingTagController$$EnhancerBySpringCGLIB$$fbb0efe6#1
local variable: java.lang.reflect.Method#91741
local variable: com.biaoguoworks.scm.logistics.controller.demeter.web.SortingTagController#1
local variable: org.springframework.aop.target.SingletonTargetSource#57
at com.biaoguoworks.scm.logistics.controller.demeter.web.SortingTagController$$EnhancerBySpringCGLIB$$fbb0efe6.sortingImport(<generated>)
local variable: com.biaoguoworks.scm.logistics.controller.demeter.web.SortingTagController$$EnhancerBySpringCGLIB$$fbb0efe6#1
local variable: com.biaoguoworks.scm.logistics.vo.sorting.SkuSortingImportVO#1
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:190)
local variable: org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod#1
local variable: java.lang.Object[]#89529
at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:138)
at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:105)
local variable: org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod#1
local variable: org.springframework.web.context.request.ServletWebRequest#1
local variable: org.springframework.web.method.support.ModelAndViewContainer#1
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:878)
local variable: org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter#1
local variable: org.springframework.web.context.request.ServletWebRequest#1
local variable: org.springframework.web.method.annotation.ModelFactory#1
local variable: org.springframework.web.method.support.ModelAndViewContainer#1
local variable: org.springframework.web.context.request.async.WebAsyncManager#1
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:792)
local variable: org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter#1
local variable: org.apache.catalina.connector.ResponseFacade#5
local variable: org.springframework.web.method.HandlerMethod#378
at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1040)
local variable: org.springframework.web.servlet.DispatcherServlet#1
local variable: org.apache.catalina.connector.ResponseFacade#5
local variable: org.springframework.web.multipart.support.StandardMultipartHttpServletRequest#1
local variable: org.springframework.web.servlet.HandlerExecutionChain#1
local variable: org.springframework.web.context.request.async.WebAsyncManager#1
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:943)
local variable: org.springframework.web.servlet.DispatcherServlet#1
local variable: org.apache.catalina.connector.RequestFacade#5
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
local variable: org.springframework.web.servlet.DispatcherServlet#1
local variable: org.apache.catalina.connector.RequestFacade#5
local variable: org.apache.catalina.connector.ResponseFacade#5
local variable: org.springframework.context.i18n.SimpleLocaleContext#1
local variable: org.springframework.web.context.request.ServletRequestAttributes#2
local variable: org.springframework.web.context.request.ServletRequestAttributes#1
local variable: org.springframework.web.context.request.async.WebAsyncManager#1
at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:909)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:652)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:733)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:53)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at com.github.xiaoymin.knife4j.spring.filter.SecurityBasicAuthFilter.doFilter(SecurityBasicAuthFilter.java:90)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at com.github.xiaoymin.knife4j.spring.filter.ProductionSecurityFilter.doFilter(ProductionSecurityFilter.java:53)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
local variable: org.springframework.boot.web.servlet.filter.OrderedRequestContextFilter#1
local variable: org.apache.catalina.connector.RequestFacade#5
local variable: org.springframework.web.context.request.ServletRequestAttributes#2
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
local variable: org.apache.catalina.connector.RequestFacade#5
local variable: java.lang.String#254631
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at org.springframework.web.filter.FormContentFilter.doFilterInternal(FormContentFilter.java:93)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
local variable: org.apache.catalina.connector.RequestFacade#5
local variable: java.lang.String#254632
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
local variable: org.apache.catalina.connector.RequestFacade#5
local variable: java.lang.String#254633
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:202)
local variable: org.apache.catalina.core.StandardWrapperValve#2
local variable: org.apache.catalina.connector.Request#5
local variable: org.apache.catalina.connector.Response#5
local variable: org.apache.catalina.core.StandardWrapper#2
local variable: org.springframework.web.servlet.DispatcherServlet#1
local variable: org.springframework.boot.web.embedded.tomcat.TomcatEmbeddedContext#1
local variable: org.apache.catalina.core.ApplicationFilterChain#5
local variable: org.apache.catalina.core.StandardWrapper#2
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:541)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:139)
local variable: org.apache.catalina.core.StandardHostValve#1
local variable: org.apache.catalina.connector.Request#5
local variable: org.apache.catalina.connector.Response#5
local variable: org.springframework.boot.web.embedded.tomcat.TomcatEmbeddedContext#1
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
local variable: org.apache.catalina.valves.ErrorReportValve#1
local variable: org.apache.catalina.connector.Request#5
local variable: org.apache.catalina.connector.Response#5
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:74)
at org.apache.catalina.valves.RemoteIpValve.invoke(RemoteIpValve.java:747)
local variable: org.apache.catalina.valves.RemoteIpValve#1
local variable: org.apache.catalina.connector.Request#5
local variable: java.lang.String#254634
local variable: java.lang.String#254634
local variable: java.lang.String#18896
local variable: java.lang.String#254635
local variable: java.lang.String#254637
local variable: java.lang.String#254636
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:343)
local variable: org.apache.catalina.connector.CoyoteAdapter#1
local variable: org.apache.coyote.Request#5
local variable: org.apache.coyote.Response#5
local variable: org.apache.catalina.connector.Request#5
local variable: org.apache.catalina.connector.Response#5
at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:373)
local variable: org.apache.coyote.http11.Http11Processor#5
local variable: org.apache.tomcat.util.net.NioEndpoint$NioSocketWrapper#7
local variable: org.apache.coyote.RequestInfo#5
at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)
local variable: org.apache.coyote.http11.Http11Processor#5
local variable: org.apache.tomcat.util.net.NioEndpoint$NioSocketWrapper#7
local variable: org.apache.tomcat.util.net.SocketEvent#6
at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:868)
local variable: org.apache.coyote.AbstractProtocol$ConnectionHandler#1
local variable: org.apache.tomcat.util.net.NioEndpoint$NioSocketWrapper#7
local variable: org.apache.tomcat.util.net.SocketEvent#6
local variable: org.apache.coyote.http11.Http11Processor#5
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1589)
local variable: org.apache.tomcat.util.net.NioEndpoint$SocketProcessor#4
local variable: org.apache.tomcat.util.net.NioChannel#17
local variable: org.apache.tomcat.util.net.NioEndpoint$Poller#1
at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
local variable: org.apache.tomcat.util.net.NioEndpoint$NioSocketWrapper#7
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
local variable: org.apache.tomcat.util.threads.ThreadPoolExecutor#1
local variable: java.util.concurrent.ThreadPoolExecutor$Worker#63
local variable: org.apache.tomcat.util.threads.TaskThread#14
local variable: org.apache.tomcat.util.net.NioEndpoint$SocketProcessor#4
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
local variable: org.apache.tomcat.util.threads.TaskThread$WrappingRunnable#14
at java.lang.Thread.run(Thread.java:748)
