0 full gc时cpu idle_Java gc 回收导致 cpu 虚高，用了 jstack 分析，一点头绪都没有，希望大家帮忙分析一下...

2020-04-08 15:47:54

Full thread dump Java HotSpot(TM) Server VM (25.152-b16 mixed mode):

"threadDeathWatcher-3-1" #29 daemon prio=1 os_prio=0 tid=0x1cf3e800 nid=0x5d6c sleeping[0x1845c000]

java.lang.Thread.State: TIMED_WAITING (sleeping)

at java.lang.Thread.sleep(Native Method)

at org.apache.hbase.thirdparty.io.netty.util.ThreadDeathWatcher$Watcher.run(ThreadDeathWatcher.java:152)

at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138)

at java.lang.Thread.run(Thread.java:748)

"RpcClient-timer-pool1-t1" #25 daemon prio=5 os_prio=0 tid=0x16d91800 nid=0x5d6b sleeping[0x184ad000]

java.lang.Thread.State: TIMED_WAITING (sleeping)

at java.lang.Thread.sleep(Native Method)

at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:560)

at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:459)

at java.lang.Thread.run(Thread.java:748)

"Idle-Rpc-Conn-Sweeper-pool2-t1" #26 daemon prio=5 os_prio=0 tid=0x1cd51400 nid=0x5d68 waiting on condition [0x193fe000]

java.lang.Thread.State: TIMED_WAITING (parking)

at sun.misc.Unsafe.park(Native Method)

- parking to wait for <0x32b10890> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)

at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)

at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)

at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)

at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)

at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

at java.lang.Thread.run(Thread.java:748)

"ReadOnlyZKClient-linux1:2181,linux2:2181,linux3:[email protected]" #22 daemon prio=10 os_prio=0 tid=0x1faf7000 nid=0x5d65 waiting on condition [0x197fd000]

java.lang.Thread.State: TIMED_WAITING (parking)

at sun.misc.Unsafe.park(Native Method)

- parking to wait for <0x32bf7ff8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)

at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)

at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)

at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259)

at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:303)

at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$27/31456712.run(Unknown Source)

at java.lang.Thread.run(Thread.java:748)

"OkHttp ConnectionPool" #18 daemon prio=5 os_prio=0 tid=0x1b4cf400 nid=0x5d5f in Object.wait() [0x1ddad000]

java.lang.Thread.State: TIMED_WAITING (on object monitor)

at java.lang.Object.wait(Native Method)

at java.lang.Object.wait(Object.java:460)

at com.squareup.okhttp.ConnectionPool.performCleanup(ConnectionPool.java:305)

- locked <0x329bedb8> (a com.squareup.okhttp.ConnectionPool)

at com.squareup.okhttp.ConnectionPool.runCleanupUntilPoolIsEmpty(ConnectionPool.java:242)

at com.squareup.okhttp.ConnectionPool.access$000(ConnectionPool.java:54)

at com.squareup.okhttp.ConnectionPool$1.run(ConnectionPool.java:97)

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

at java.lang.Thread.run(Thread.java:748)

"Thread-3" #16 prio=5 os_prio=0 tid=0x1dc7d000 nid=0x5d57 runnable [0x1ddfe000]

java.lang.Thread.State: RUNNABLE

at java.net.SocketInputStream.socketRead0(Native Method)

at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)

at java.net.SocketInputStream.read(SocketInputStream.java:171)

at java.net.SocketInputStream.read(SocketInputStream.java:141)

at java.net.SocketInputStream.read(SocketInputStream.java:127)

at redis.clients.jedis.util.RedisInputStream.ensureFill(RedisInputStream.java:199)

at redis.clients.jedis.util.RedisInputStream.readByte(RedisInputStream.java:43)

at redis.clients.jedis.Protocol.process(Protocol.java:154)

at redis.clients.jedis.Protocol.read(Protocol.java:219)

at redis.clients.jedis.Connection.readProtocolWithCheckingBroken(Connection.java:309)

at redis.clients.jedis.Connection.getRawObjectMultiBulkReply(Connection.java:276)

at redis.clients.jedis.JedisPubSub.process(JedisPubSub.java:123)

at redis.clients.jedis.JedisPubSub.proceed(JedisPubSub.java:117)

at redis.clients.jedis.Jedis.subscribe(Jedis.java:2779)

at redis.clients.jedis.JedisCluster$153.execute(JedisCluster.java:1693)

at redis.clients.jedis.JedisCluster$153.execute(JedisCluster.java:1690)

at redis.clients.jedis.JedisClusterCommand.runWithAnyNode(JedisClusterCommand.java:76)

at redis.clients.jedis.JedisCluster.subscribe(JedisCluster.java:1696)

at com.flo.recommend.init.RedisSubThread.run(RedisSubThread.java:28)

at java.lang.Thread.run(Thread.java:748)

"commons-pool-evictor-thread" #14 prio=5 os_prio=0 tid=0x1dc4e800 nid=0x5d56 runnable [0x1e3ad000]

java.lang.Thread.State: RUNNABLE

at java.lang.Throwable.fillInStackTrace(Native Method)

at java.lang.Throwable.fillInStackTrace(Throwable.java:783)

- locked <0xcf435f08> (a java.lang.IllegalStateException)

at java.lang.Throwable.(Throwable.java:265)

at java.lang.Exception.(Exception.java:66)

at java.lang.RuntimeException.(RuntimeException.java:62)

at java.lang.IllegalStateException.(IllegalStateException.java:55)

at org.apache.commons.pool2.impl.BaseGenericObjectPool.assertOpen(BaseGenericObjectPool.java:713)

at org.apache.commons.pool2.impl.GenericObjectPool.evict(GenericObjectPool.java:721)

at org.apache.commons.pool2.impl.BaseGenericObjectPool$Evictor.run(BaseGenericObjectPool.java:1077)

at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)

at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)

at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

at java.lang.Thread.run(Thread.java:748)

"Thread-1" #11 prio=5 os_prio=0 tid=0x1eaee400 nid=0x5d54 sleeping[0x1e3fe000]

java.lang.Thread.State: TIMED_WAITING (sleeping)

at java.lang.Thread.sleep(Native Method)

at com.flo.recommend.reduce.PushUrlCache.run(PushUrlCache.java:55)

at java.lang.Thread.run(Thread.java:748)

"Thread-0" #10 prio=5 os_prio=0 tid=0x1e90d400 nid=0x5d53 runnable [0x1e652000]

java.lang.Thread.State: RUNNABLE

at com.flo.recommend.util.MyLevenshtein.levenshtein(MyLevenshtein.java:19)

at com.flo.recommend.mahout.DistancesRedue.apprun(DistancesRedue.java:70)

at com.flo.recommend.init.RedisInit.initAppInfo(RedisInit.java:129)

at com.flo.recommend.init.RedisInit.run(RedisInit.java:37)

at java.lang.Thread.run(Thread.java:748)

"mysql-cj-abandoned-connection-cleanup" #9 daemon prio=5 os_prio=0 tid=0x1fc77800 nid=0x5d52 runnable [0x20d95000]

java.lang.Thread.State: RUNNABLE

at java.security.AccessController.doPrivileged(Native Method)

at java.net.URLClassLoader.findResource(URLClassLoader.java:566)

at java.lang.ClassLoader.getResource(ClassLoader.java:1096)

at java.lang.ClassLoader.getResource(ClassLoader.java:1091)

at java.lang.ClassLoader.getResource(ClassLoader.java:1091)

at com.mysql.jdbc.AbandonedConnectionCleanupThread.checkThreadContextClassLoader(AbandonedConnectionCleanupThread.java:112)

at com.mysql.jdbc.AbandonedConnectionCleanupThread.run(AbandonedConnectionCleanupThread.java:79)

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

at java.lang.Thread.run(Thread.java:748)

"Service Thread" #8 daemon prio=9 os_prio=0 tid=0x216aac00 nid=0x5d50 runnable [0x00000000]

java.lang.Thread.State: RUNNABLE

"C1 CompilerThread2" #7 daemon prio=9 os_prio=0 tid=0x216a7800 nid=0x5d4f waiting on condition [0x00000000]

java.lang.Thread.State: RUNNABLE

"C2 CompilerThread1" #6 daemon prio=9 os_prio=0 tid=0x216a6400 nid=0x5d4e waiting on condition [0x00000000]

java.lang.Thread.State: RUNNABLE

"C2 CompilerThread0" #5 daemon prio=9 os_prio=0 tid=0x216a4400 nid=0x5d4d runnable [0x00000000]

java.lang.Thread.State: RUNNABLE

"Signal Dispatcher" #4 daemon prio=9 os_prio=0 tid=0x216a2c00 nid=0x5d4c runnable [0x00000000]

java.lang.Thread.State: RUNNABLE

"Finalizer" #3 daemon prio=8 os_prio=0 tid=0x21688800 nid=0x5d4b in Object.wait() [0x28a7d000]

java.lang.Thread.State: WAITING (on object monitor)

at java.lang.Object.wait(Native Method)

at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)

- locked <0x29ebc1b0> (a java.lang.ref.ReferenceQueue$Lock)

at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)

at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209)

"Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x21685c00 nid=0x5d4a in Object.wait() [0x2157d000]

java.lang.Thread.State: WAITING (on object monitor)

at java.lang.Object.wait(Native Method)

at java.lang.Object.wait(Object.java:502)

at java.lang.ref.Reference.tryHandlePending(Reference.java:191)

- locked <0x29ebc3b0> (a java.lang.ref.Reference$Lock)

at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153)

"main" #1 prio=5 os_prio=0 tid=0xf7607000 nid=0x5d44 in Object.wait() [0xf7788000]

java.lang.Thread.State: WAITING (on object monitor)

at java.lang.Object.wait(Native Method)

- waiting on <0x29ebc4b0> (a io.netty.channel.AbstractChannel$CloseFuture)

at java.lang.Object.wait(Object.java:502)

at io.netty.util.concurrent.DefaultPromise.await(DefaultPromise.java:231)

- locked <0x29ebc4b0> (a io.netty.channel.AbstractChannel$CloseFuture)

at io.netty.channel.DefaultChannelPromise.await(DefaultChannelPromise.java:131)

at io.netty.channel.DefaultChannelPromise.await(DefaultChannelPromise.java:30)

at io.netty.util.concurrent.DefaultPromise.sync(DefaultPromise.java:337)

at io.netty.channel.DefaultChannelPromise.sync(DefaultChannelPromise.java:119)

at io.netty.channel.DefaultChannelPromise.sync(DefaultChannelPromise.java:30)

at com.flo.recommend.HttpServer.main(HttpServer.java:110)

at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)

at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

at java.lang.reflect.Method.invoke(Method.java:498)

at org.springframework.boot.loader.MainMethodRunner.run(MainMethodRunner.java:48)

at org.springframework.boot.loader.Launcher.launch(Launcher.java:87)

at org.springframework.boot.loader.Launcher.launch(Launcher.java:50)

at org.springframework.boot.loader.JarLauncher.main(JarLauncher.java:51)

"VM Thread" os_prio=0 tid=0x21681000 nid=0x5d49 runnable

"GC task thread#0 (ParallelGC)" os_prio=0 tid=0xf7610800 nid=0x5d45 runnable

"GC task thread#1 (ParallelGC)" os_prio=0 tid=0xf7611c00 nid=0x5d46 runnable

"GC task thread#2 (ParallelGC)" os_prio=0 tid=0xf7613000 nid=0x5d47 runnable

"GC task thread#3 (ParallelGC)" os_prio=0 tid=0xf7614400 nid=0x5d48 runnable

"VM Periodic Task Thread" os_prio=0 tid=0x216bf000 nid=0x5d51 waiting on condition

JNI global references: 431