今天在重现出HashMap cpu占用100%了,只要并发稍微大一点就会出现,我用一个本地HashMap mock一个Memcached的客户端,通过netty暴露结果中招了:下面是打印的jstack:
jstack 写道
Full thread dump Java HotSpot(TM) 64-Bit Server VM (20.5-b03 mixed mode):
"Attach Listener" daemon prio=10 tid=0x0000000054d55800 nid=0x78a0 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"New I/O server worker #1-16" prio=10 tid=0x0000000054d88000 nid=0x785e runnable [0x0000000044425000]
java.lang.Thread.State: RUNNABLE
at java.util.HashMap.get(HashMap.java:303)
at com.duitang.cat.LocalCacheServiceImpl.get(LocalCacheServiceImpl.java:22)
at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at com.duitang.dboss.service.ServiceInvoker.accept(ServiceInvoker.java:52)
at com.duitang.dboss.remote.nio.DbossServerHandler.messageReceived(DbossServerHandler.java:60)
at org.jboss.netty.channel.SimpleChannelHandler.handleUpstream(SimpleChannelHandler.java:98)
at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:560)
at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:796)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:391)
at org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:317)
at org.jboss.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:299)
at org.jboss.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:216)
at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:80)
at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:560)
at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:555)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:345)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:332)
at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:323)
at org.jboss.netty.channel.socket.nio.NioWorker.processSelectedKeys(NioWorker.java:275)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:196)
at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at org.jboss.netty.util.internal.IoWorkerRunnable.run(IoWorkerRunnable.java:46)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
"New I/O server worker #1-14" prio=10 tid=0x0000000054d53800 nid=0x785c runnable [0x0000000044223000]
java.lang.Thread.State: RUNNABLE
at java.util.HashMap.put(HashMap.java:374)
at com.duitang.cat.LocalCacheServiceImpl.set(LocalCacheServiceImpl.java:13)
at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at com.duitang.dboss.service.ServiceInvoker.accept(ServiceInvoker.java:52)
at com.duitang.dboss.remote.nio.DbossServerHandler.messageReceived(DbossServerHandler.java:60)
at org.jboss.netty.channel.SimpleChannelHandler.handleUpstream(SimpleChannelHandler.java:98)
at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:560)
at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:796)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:391)
at org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:317)
at org.jboss.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:299)
at org.jboss.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:214)
at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:80)
at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:560)
at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:555)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:345)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:332)
at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:323)
at org.jboss.netty.channel.socket.nio.NioWorker.processSelectedKeys(NioWorker.java:275)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:196)
at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at org.jboss.netty.util.internal.IoWorkerRunnable.run(IoWorkerRunnable.java:46)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
"Attach Listener" daemon prio=10 tid=0x0000000054d55800 nid=0x78a0 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"New I/O server worker #1-16" prio=10 tid=0x0000000054d88000 nid=0x785e runnable [0x0000000044425000]
java.lang.Thread.State: RUNNABLE
at java.util.HashMap.get(HashMap.java:303)
at com.duitang.cat.LocalCacheServiceImpl.get(LocalCacheServiceImpl.java:22)
at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at com.duitang.dboss.service.ServiceInvoker.accept(ServiceInvoker.java:52)
at com.duitang.dboss.remote.nio.DbossServerHandler.messageReceived(DbossServerHandler.java:60)
at org.jboss.netty.channel.SimpleChannelHandler.handleUpstream(SimpleChannelHandler.java:98)
at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:560)
at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:796)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:391)
at org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:317)
at org.jboss.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:299)
at org.jboss.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:216)
at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:80)
at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:560)
at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:555)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:345)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:332)
at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:323)
at org.jboss.netty.channel.socket.nio.NioWorker.processSelectedKeys(NioWorker.java:275)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:196)
at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at org.jboss.netty.util.internal.IoWorkerRunnable.run(IoWorkerRunnable.java:46)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
"New I/O server worker #1-14" prio=10 tid=0x0000000054d53800 nid=0x785c runnable [0x0000000044223000]
java.lang.Thread.State: RUNNABLE
at java.util.HashMap.put(HashMap.java:374)
at com.duitang.cat.LocalCacheServiceImpl.set(LocalCacheServiceImpl.java:13)
at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at com.duitang.dboss.service.ServiceInvoker.accept(ServiceInvoker.java:52)
at com.duitang.dboss.remote.nio.DbossServerHandler.messageReceived(DbossServerHandler.java:60)
at org.jboss.netty.channel.SimpleChannelHandler.handleUpstream(SimpleChannelHandler.java:98)
at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:560)
at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:796)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:391)
at org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:317)
at org.jboss.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:299)
at org.jboss.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:214)
at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:80)
at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:560)
at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:555)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:345)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:332)
at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:323)
at org.jboss.netty.channel.socket.nio.NioWorker.processSelectedKeys(NioWorker.java:275)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:196)
at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at org.jboss.netty.util.internal.IoWorkerRunnable.run(IoWorkerRunnable.java:46)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
可以发现在:at java.util.HashMap.get(HashMap.java:303) 有一个for循环调用内部的next()方法,而同时,另外一个线程在put操作: java.util.HashMap.put(HashMap.java:374) ,据说这两处在某种情况会导致死循环。结果我的cache服务器也挂了:
connect: Connection refused ('192.168.172.2', 8989)
connect: Connection refused ('192.168.172.2', 8989)
connect: Connection refused ('192.168.172.2', 8989)
connect: Connection refused ('192.168.172.2', 8989)
connect: Connection refused ('192.168.172.2', 8989)
connect: Connection refused ('192.168.172.2', 8989)
connect: Connection refused ('192.168.172.2', 8989)