1.通过jstack 出来的线程有巨量的锁死现象
"http://*:3050-8188" daemon prio=10 tid=0x00007fc597d60800 nid=0x9f03 waiting on condition [0x00007fc2d3df9000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x0000000704953160> (a java.util.concurrent.locks.ReentrantLock$NonfairSync)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:811)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:842)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1178)
at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:186)
at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:262)
at org.apache.activemq.transport.MutexTransport.oneway(MutexTransport.java:66)
at org.apache.activemq.transport.ResponseCorrelator.oneway(ResponseCorrelator.java:60)
at org.apache.activemq.ActiveMQConnection.doAsyncSendPacket(ActiveMQConnection.java:1330)
at org.apache.activemq.ActiveMQConnection.asyncSendPacket(ActiveMQConnection.java:1324)
at org.apache.activemq.ActiveMQSession.<init>(ActiveMQSession.java:253)
at org.apache.activemq.ActiveMQConnection.createSession(ActiveMQConnection.java:337)
at org.springframework.jms.connection.SingleConnectionFactory.createSession(SingleConnectionFactory.java:406)
at org.springframework.jms.connection.CachingConnectionFactory.getSession(CachingConnectionFactory.java:225)
at org.springframework.jms.connection.SingleConnectionFactory$SharedConnectionInvocationHandler.invoke(SingleConnectionFactory.java:534)
at $Proxy206.createSession(Unknown Source)
at org.springframework.jms.support.JmsAccessor.createSession(JmsAccessor.java:196)
at org.springframework.jms.core.JmsTemplate$$EnhancerByCGLIB$$48e1e410.CGLIB$createSession$76(<generated>)
at org.springframework.jms.core.JmsTemplate$$EnhancerByCGLIB$$48e1e410$$FastClassByCGLIB$$c7ff63a4.invoke(<generated>)
at net.sf.cglib.proxy.MethodProxy.invokeSuper(MethodProxy.java:228)
at com.qfang.base.logger.monitor.MqProductionMonitor.intercept(MqProductionMonitor.java:86)
at org.springframework.jms.core.JmsTemplate$$EnhancerByCGLIB$$48e1e410.createSession(<generated>)
at org.springframework.jms.core.JmsTemplate.execute(JmsTemplate.java:457)
at org.springframework.jms.core.JmsTemplate$$EnhancerByCGLIB$$48e1e410.CGLIB$execute$6(<generated>)
at org.springframework.jms.core.JmsTemplate$$EnhancerByCGLIB$$48e1e410$$FastClassByCGLIB$$c7ff63a4.invoke(<generated>)
at net.sf.cglib.proxy.MethodProxy.invokeSuper(MethodProxy.java:228)
at com.qfang.base.logger.monitor.MqProductionMonitor.intercept(MqProductionMonitor.java:86)
at org.springframework.jms.core.JmsTemplate$$EnhancerByCGLIB$$48e1e410.execute(<generated>)
at org.springframework.jms.core.JmsTemplate.send(JmsTemplate.java:543)
at org.springframework.jms.core.JmsTemplate$$EnhancerByCGLIB$$48e1e410.CGLIB$send$4(<generated>)
at org.springframework.jms.core.JmsTemplate$$EnhancerByCGLIB$$48e1e410$$FastClassByCGLIB$$c7ff63a4.invoke(<generated>)
at net.sf.cglib.proxy.MethodProxy.invokeSuper(MethodProxy.java:228)
at com.qfang.base.logger.monitor.MqProductionMonitor.intercept(MqProductionMonitor.java:86)
at org.springframework.jms.core.JmsTemplate$$EnhancerByCGLIB$$48e1e410.send(<generated>)
at org.springframework.jms.core.JmsTemplate.convertAndSend(JmsTemplate.java:620)
at org.springframework.jms.core.JmsTemplate$$EnhancerByCGLIB$$48e1e410.CGLIB$convertAndSend$20(<generated>)
at org.springframework.jms.core.JmsTemplate$$EnhancerByCGLIB$$48e1e410$$FastClassByCGLIB$$c7ff63a4.invoke(<generated>)
at net.sf.cglib.proxy.MethodProxy.invokeSuper(MethodProxy.java:228)
at com.qfang.base.logger.monitor.MqProductionMonitor.intercept(MqProductionMonitor.java:86)
at org.springframework.jms.core.JmsTemplate$$EnhancerByCGLIB$$48e1e410.convertAndSend(<generated>)
at com.qfang.service.mq.queue.RoomHandlerSender.send(RoomHandlerSender.java:36)
at com.qfang.service.room.impl.RoomHandlerServiceImpl.refurbish(RoomHandlerServiceImpl.java:383)
at sun.reflect.GeneratedMethodAccessor343.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:318)
at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:91)
at com.qfang.service.aop.room.RoomHandlerAspect.aopExecute(RoomHandlerAspect.java:89)
at com.qfang.service.aop.room.RoomHandlerAspect.aopExecute(RoomHandlerAspect.java:52)
at com.qfang.service.aop.room.RoomHandlerAspect.refurbish(RoomHandlerAspect.java:48)
at sun.reflect.GeneratedMethodAccessor342.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:621)
at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:610)
at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:65)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:80)
at com.qfang.base.logger.access.DiagnosisMonitor.monitor(DiagnosisMonitor.java:42)
at sun.reflect.GeneratedMethodAccessor94.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:621)
at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:610)
at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:65)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:90)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:202)
at $Proxy106.refurbish(Unknown Source)
at com.qfang.web.controller.room.RoomHandlerController.refurbishRoom(RoomHandlerController.java:328)
at com.qfang.web.controller.room.RoomHandlerController$$FastClassByCGLIB$$f020cafa.invoke(<generated>)
at net.sf.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
at org.springframework.aop.framework.Cglib2AopProxy$CglibMethodInvocation.invokeJoinpoint(Cglib2AopProxy.java:689)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:80)
at com.qfang.base.logger.access.DiagnosisMonitor.monitor(DiagnosisMonitor.java:42)
at sun.reflect.GeneratedMethodAccessor94.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:621)
at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:610)
at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:65)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:90)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at org.springframework.aop.framework.Cglib2AopProxy$DynamicAdvisedInterceptor.intercept(Cglib2AopProxy.java:622)
at com.qfang.web.controller.room.RoomHandlerController$$EnhancerByCGLIB$$878bd92d.refurbishRoom(<generated>)
at sun.reflect.GeneratedMethodAccessor339.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.springframework.web.method.support.InvocableHandlerMethod.invoke(InvocableHandlerMethod.java:212)
at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:126)
at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:96)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:617)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:578)
at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:80)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:900)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:827)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:882)
at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:778)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:119)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:96)
at com.caucho.server.dispatch.ServletFilterChain.doFilter(ServletFilterChain.java:109)
at com.qfang.web.session.redis.RedisSessionFilter.doFilter(RedisSessionFilter.java:65)
at com.caucho.server.dispatch.FilterFilterChain.doFilter(FilterFilterChain.java:89)
at com.qfang.web.filter.EncodeFilter.doFilter(EncodeFilter.java:23)
at com.caucho.server.dispatch.FilterFilterChain.doFilter(FilterFilterChain.java:89)
at com.caucho.server.webapp.WebAppFilterChain.doFilter(WebAppFilterChain.java:156)
at com.caucho.server.webapp.AccessLogFilterChain.doFilter(AccessLogFilterChain.java:95)
at com.caucho.server.dispatch.ServletInvocation.service(ServletInvocation.java:289)
at com.caucho.server.http.HttpRequest.handleRequest(HttpRequest.java:811)
at com.caucho.network.listen.TcpSocketLink.dispatchRequest(TcpSocketLink.java:1221)
at com.caucho.network.listen.TcpSocketLink.handleRequest(TcpSocketLink.java:1177)
at com.caucho.network.listen.TcpSocketLink.handleRequestsImpl(TcpSocketLink.java:1161)
at com.caucho.network.listen.TcpSocketLink.handleRequests(TcpSocketLink.java:1084)
at com.caucho.network.listen.TcpSocketLink.handleAcceptTask(TcpSocketLink.java:907)
at com.caucho.network.listen.AcceptTask.doTask(AcceptTask.java:74)
at com.caucho.network.listen.ConnectionTask.runThread(ConnectionTask.java:97)
at com.caucho.network.listen.ConnectionTask.run(ConnectionTask.java:80)
at com.caucho.network.listen.AcceptTask.run(AcceptTask.java:59)
at com.caucho.env.thread.ResinThread.runTasks(ResinThread.java:164)
at com.caucho.env.thread.ResinThread.run(ResinThread.java:130)
原因是activemq连接不能创建,问题聚焦到activemq服务器上。
发现Store percent used 达到100%,导致消息不能处理。
首先在activemq的配置文件中增加文件容量限制,由100g->160g。
接着看/data/kahadb 下面的日志文件,db-*没有清除。
有两个说法:
1.有未消费的消息还存在,导致日志文件不会被自动清理线程cleanup
2.activemq的清理线程调度有问题,
设置http://www.360doc.com/content/14/0328/08/2795334_364337432.shtml