neo4j问题系列-JVM

某项目的系统又出问题了，一大早工程师发过来问题日志和截图，

这个问题非常怪，查找neo4j的源代码，在代码中：

channelPool = new ResourcePool<Triplet<Channel, ChannelBuffer, ByteBuffer>>( maxUnusedChannels,
new ResourcePool.CheckStrategy.TimeoutCheckStrategy( ResourcePool.DEFAULT_CHECK_INTERVAL,
Clock.REAL_CLOCK ),
new LoggingResourcePoolMonitor( msgLog ) )
{
@Override
protected Triplet<Channel, ChannelBuffer, ByteBuffer> create()
{
ChannelFuture channelFuture = bootstrap.connect( address );
channelFuture.awaitUninterruptibly( 5, TimeUnit.SECONDS );
Triplet<Channel, ChannelBuffer, ByteBuffer> channel = null;
if ( channelFuture.isSuccess() )
{
channel = Triplet.of( channelFuture.getChannel(),
ChannelBuffers.dynamicBuffer(),
ByteBuffer.allocate( 1024 * 1024 ) );
msgLog.logMessage( "Opened a new channel to " + address, true );
return channel;
}

String msg = Client.this.getClass().getSimpleName() + " could not connect to " + address;
msgLog.logMessage( msg, true );
ComException exception = new ComException( msg );
// connectionLostHandler.handle( exception );
throw exception;
}

红字部分是NEO4J集群超时检测的策略，这里写死的代码是5秒钟连接一次，如果得不到结果则报连不到其它节点。

结合JVM的GC日志:

838848.0 838848.0 393256.3 0.0 6710912.0 4034318.3 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4047794.2 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4065917.6 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4096465.2 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4109666.1 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4122958.8 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4140524.4 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4153747.1 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4192433.5 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4209845.7 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4222981.6 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4236453.5 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4249570.1 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4262667.9 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4275803.3 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4302302.3 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4315451.5 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4328642.9 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4341828.1 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4368032.2 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4381214.7 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4394387.5 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4407574.8 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4433785.8 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4446897.6 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4460088.9 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4473219.6 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4486337.2 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4499552.5 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4512696.3 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4512833.1 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4525992.0 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4539110.7 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4552214.6 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4552286.2 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4565467.4 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4578656.7 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4578760.3 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4592102.7 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4614557.1 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4643717.6 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4703662.3 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4781383.2 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4874863.2 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 4956980.7 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 5021316.2 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 5082403.3 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 5186498.6 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 5272571.1 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 5336442.5 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 5435527.4 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 5522161.0 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 5628996.2 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 5671821.6 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 5710974.3 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 5763211.6 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 5824083.4 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 5880336.7 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 5911785.0 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 5924983.1 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 5938083.0 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 5956058.5 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 5970411.9 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 5987960.0 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6001154.8 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6018323.2 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6031721.3 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6049252.8 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6066716.6 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6079821.6 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6079900.0 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6097365.6 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6110549.5 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6123648.3 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6136783.9 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6150179.5 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6163355.0 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6176542.7 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6189712.3 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6202824.4 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6216032.0 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6229178.0 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6242382.4 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6255490.9 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6268600.8 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6281767.4 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6294983.6 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6308106.2 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6321249.9 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6334388.5 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6347509.3 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6360783.6 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6373907.9 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6387005.7 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6400119.1 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6413285.3 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6426462.7 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6439742.0 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6443932.0 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201
838848.0 838848.0 393256.3 0.0 6710912.0 6504376.5 2477168.0 1260112.9 262144.0 93696.5 280 144.570 53 100.630 245.201

基本上可定位为GC回收引起业务线程停止响应导致超时，看来还是jvm中数据太多，回收不过来。问题明确了方向，接下来检查代码吧。