由activemq消息存储满导致的服务器崩溃

    1.通过jstack 出来的线程有巨量的锁死现象

"http://*:3050-8188" daemon prio=10 tid=0x00007fc597d60800 nid=0x9f03 waiting on condition [0x00007fc2d3df9000]

   java.lang.Thread.State: WAITING (parking)

at sun.misc.Unsafe.park(Native Method)

- parking to wait for  <0x0000000704953160> (a java.util.concurrent.locks.ReentrantLock$NonfairSync)

at java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)

at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:811)

at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:842)

at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1178)

at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:186)

at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:262)

at org.apache.activemq.transport.MutexTransport.oneway(MutexTransport.java:66)

at org.apache.activemq.transport.ResponseCorrelator.oneway(ResponseCorrelator.java:60)

at org.apache.activemq.ActiveMQConnection.doAsyncSendPacket(ActiveMQConnection.java:1330)

at org.apache.activemq.ActiveMQConnection.asyncSendPacket(ActiveMQConnection.java:1324)

at org.apache.activemq.ActiveMQSession.<init>(ActiveMQSession.java:253)

at org.apache.activemq.ActiveMQConnection.createSession(ActiveMQConnection.java:337)

at org.springframework.jms.connection.SingleConnectionFactory.createSession(SingleConnectionFactory.java:406)

at org.springframework.jms.connection.CachingConnectionFactory.getSession(CachingConnectionFactory.java:225)

at org.springframework.jms.connection.SingleConnectionFactory$SharedConnectionInvocationHandler.invoke(SingleConnectionFactory.java:534)

at $Proxy206.createSession(Unknown Source)

at org.springframework.jms.support.JmsAccessor.createSession(JmsAccessor.java:196)

at org.springframework.jms.core.JmsTemplate$$EnhancerByCGLIB$$48e1e410.CGLIB$createSession$76(<generated>)

at org.springframework.jms.core.JmsTemplate$$EnhancerByCGLIB$$48e1e410$$FastClassByCGLIB$$c7ff63a4.invoke(<generated>)

at net.sf.cglib.proxy.MethodProxy.invokeSuper(MethodProxy.java:228)

at com.qfang.base.logger.monitor.MqProductionMonitor.intercept(MqProductionMonitor.java:86)

at org.springframework.jms.core.JmsTemplate$$EnhancerByCGLIB$$48e1e410.createSession(<generated>)

at org.springframework.jms.core.JmsTemplate.execute(JmsTemplate.java:457)

at org.springframework.jms.core.JmsTemplate$$EnhancerByCGLIB$$48e1e410.CGLIB$execute$6(<generated>)

at org.springframework.jms.core.JmsTemplate$$EnhancerByCGLIB$$48e1e410$$FastClassByCGLIB$$c7ff63a4.invoke(<generated>)

at net.sf.cglib.proxy.MethodProxy.invokeSuper(MethodProxy.java:228)

at com.qfang.base.logger.monitor.MqProductionMonitor.intercept(MqProductionMonitor.java:86)

at org.springframework.jms.core.JmsTemplate$$EnhancerByCGLIB$$48e1e410.execute(<generated>)

at org.springframework.jms.core.JmsTemplate.send(JmsTemplate.java:543)

at org.springframework.jms.core.JmsTemplate$$EnhancerByCGLIB$$48e1e410.CGLIB$send$4(<generated>)

at org.springframework.jms.core.JmsTemplate$$EnhancerByCGLIB$$48e1e410$$FastClassByCGLIB$$c7ff63a4.invoke(<generated>)

at net.sf.cglib.proxy.MethodProxy.invokeSuper(MethodProxy.java:228)

at com.qfang.base.logger.monitor.MqProductionMonitor.intercept(MqProductionMonitor.java:86)

at org.springframework.jms.core.JmsTemplate$$EnhancerByCGLIB$$48e1e410.send(<generated>)

at org.springframework.jms.core.JmsTemplate.convertAndSend(JmsTemplate.java:620)

at org.springframework.jms.core.JmsTemplate$$EnhancerByCGLIB$$48e1e410.CGLIB$convertAndSend$20(<generated>)

at org.springframework.jms.core.JmsTemplate$$EnhancerByCGLIB$$48e1e410$$FastClassByCGLIB$$c7ff63a4.invoke(<generated>)

at net.sf.cglib.proxy.MethodProxy.invokeSuper(MethodProxy.java:228)

at com.qfang.base.logger.monitor.MqProductionMonitor.intercept(MqProductionMonitor.java:86)

at org.springframework.jms.core.JmsTemplate$$EnhancerByCGLIB$$48e1e410.convertAndSend(<generated>)

at com.qfang.service.mq.queue.RoomHandlerSender.send(RoomHandlerSender.java:36)

at com.qfang.service.room.impl.RoomHandlerServiceImpl.refurbish(RoomHandlerServiceImpl.java:383)

at sun.reflect.GeneratedMethodAccessor343.invoke(Unknown Source)

at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

at java.lang.reflect.Method.invoke(Method.java:597)

at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:318)

at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)

at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)

at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:91)

at com.qfang.service.aop.room.RoomHandlerAspect.aopExecute(RoomHandlerAspect.java:89)

at com.qfang.service.aop.room.RoomHandlerAspect.aopExecute(RoomHandlerAspect.java:52)

at com.qfang.service.aop.room.RoomHandlerAspect.refurbish(RoomHandlerAspect.java:48)

at sun.reflect.GeneratedMethodAccessor342.invoke(Unknown Source)

at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

at java.lang.reflect.Method.invoke(Method.java:597)

at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:621)

at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:610)

at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:65)

at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)

at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:80)

at com.qfang.base.logger.access.DiagnosisMonitor.monitor(DiagnosisMonitor.java:42)

at sun.reflect.GeneratedMethodAccessor94.invoke(Unknown Source)

at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

at java.lang.reflect.Method.invoke(Method.java:597)

at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:621)

at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:610)

at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:65)

at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)

at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:90)

at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)

at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:202)

at $Proxy106.refurbish(Unknown Source)

at com.qfang.web.controller.room.RoomHandlerController.refurbishRoom(RoomHandlerController.java:328)

at com.qfang.web.controller.room.RoomHandlerController$$FastClassByCGLIB$$f020cafa.invoke(<generated>)

at net.sf.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)

at org.springframework.aop.framework.Cglib2AopProxy$CglibMethodInvocation.invokeJoinpoint(Cglib2AopProxy.java:689)

at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)

at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:80)

at com.qfang.base.logger.access.DiagnosisMonitor.monitor(DiagnosisMonitor.java:42)

at sun.reflect.GeneratedMethodAccessor94.invoke(Unknown Source)

at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

at java.lang.reflect.Method.invoke(Method.java:597)

at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:621)

at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:610)

at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:65)

at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)

at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:90)

at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)

at org.springframework.aop.framework.Cglib2AopProxy$DynamicAdvisedInterceptor.intercept(Cglib2AopProxy.java:622)

at com.qfang.web.controller.room.RoomHandlerController$$EnhancerByCGLIB$$878bd92d.refurbishRoom(<generated>)

at sun.reflect.GeneratedMethodAccessor339.invoke(Unknown Source)

at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

at java.lang.reflect.Method.invoke(Method.java:597)

at org.springframework.web.method.support.InvocableHandlerMethod.invoke(InvocableHandlerMethod.java:212)

at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:126)

at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:96)

at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:617)

at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:578)

at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:80)

at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:900)

at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:827)

at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:882)

at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:778)

at javax.servlet.http.HttpServlet.service(HttpServlet.java:119)

at javax.servlet.http.HttpServlet.service(HttpServlet.java:96)

at com.caucho.server.dispatch.ServletFilterChain.doFilter(ServletFilterChain.java:109)

at com.qfang.web.session.redis.RedisSessionFilter.doFilter(RedisSessionFilter.java:65)

at com.caucho.server.dispatch.FilterFilterChain.doFilter(FilterFilterChain.java:89)

at com.qfang.web.filter.EncodeFilter.doFilter(EncodeFilter.java:23)

at com.caucho.server.dispatch.FilterFilterChain.doFilter(FilterFilterChain.java:89)

at com.caucho.server.webapp.WebAppFilterChain.doFilter(WebAppFilterChain.java:156)

at com.caucho.server.webapp.AccessLogFilterChain.doFilter(AccessLogFilterChain.java:95)

at com.caucho.server.dispatch.ServletInvocation.service(ServletInvocation.java:289)

at com.caucho.server.http.HttpRequest.handleRequest(HttpRequest.java:811)

at com.caucho.network.listen.TcpSocketLink.dispatchRequest(TcpSocketLink.java:1221)

at com.caucho.network.listen.TcpSocketLink.handleRequest(TcpSocketLink.java:1177)

at com.caucho.network.listen.TcpSocketLink.handleRequestsImpl(TcpSocketLink.java:1161)

at com.caucho.network.listen.TcpSocketLink.handleRequests(TcpSocketLink.java:1084)

at com.caucho.network.listen.TcpSocketLink.handleAcceptTask(TcpSocketLink.java:907)

at com.caucho.network.listen.AcceptTask.doTask(AcceptTask.java:74)

at com.caucho.network.listen.ConnectionTask.runThread(ConnectionTask.java:97)

at com.caucho.network.listen.ConnectionTask.run(ConnectionTask.java:80)

at com.caucho.network.listen.AcceptTask.run(AcceptTask.java:59)

at com.caucho.env.thread.ResinThread.runTasks(ResinThread.java:164)

at com.caucho.env.thread.ResinThread.run(ResinThread.java:130)

原因是activemq连接不能创建，问题聚焦到activemq服务器上。

发现Store percent used 达到100%，导致消息不能处理。

首先在activemq的配置文件中增加文件容量限制，由100g->160g。

接着看/data/kahadb 下面的日志文件，db-*没有清除。

有两个说法：

1.有未消费的消息还存在，导致日志文件不会被自动清理线程cleanup

2.activemq的清理线程调度有问题，
设置http://www.360doc.com/content/14/0328/08/2795334_364337432.shtml