【好记性不如烂笔头】记一次线上问题，系统卡爆了，排查日志发现出现了ownerThread current state is WAITING, current stackTrace

问题描述

  项目部署在linux环境，运行期间页面访问特别卡，功能无法正常使用。

---

原因分析：

  排查后台日志发现出现了"ownerThread current state is WAITING, current stackTrace"
贴出日志：

2022-10-10 11:24:11,496 ERROR [admin-web] [Druid-ConnectionPool-Destroy-1388477779] [c.alibaba.druid.pool.DruidDataSource:2953] -  abandon connection, owner thread: DubboServerHandler-192.168.160.143:28081-thread-188, connected at : 1665372133713, open stackTrace
	at java.lang.Thread.getStackTrace(Thread.java:1559)
	at com.alibaba.druid.pool.DruidDataSource.getConnectionDirect(DruidDataSource.java:1471)
	at com.alibaba.druid.filter.FilterChainImpl.dataSource_connect(FilterChainImpl.java:5059)
	at com.alibaba.druid.filter.logging.LogFilter.dataSource_getConnection(LogFilter.java:886)
	at com.alibaba.druid.filter.FilterChainImpl.dataSource_connect(FilterChainImpl.java:5055)
	at com.alibaba.druid.filter.stat.StatFilter.dataSource_getConnection(StatFilter.java:680)
	at com.alibaba.druid.filter.FilterChainImpl.dataSource_connect(FilterChainImpl.java:5055)
	at com.alibaba.druid.filter.FilterAdapter.dataSource_getConnection(FilterAdapter.java:2756)
	at com.alibaba.druid.filter.FilterChainImpl.dataSource_connect(FilterChainImpl.java:5055)
	at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:1380)
	at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:1372)
	at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:109)
	at org.springframework.jdbc.datasource.DataSourceUtils.fetchConnection(DataSourceUtils.java:151)
	at org.springframework.jdbc.datasource.DataSourceUtils.doGetConnection(DataSourceUtils.java:115)
	at org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:78)
	at org.mybatis.spring.transaction.SpringManagedTransaction.openConnection(SpringManagedTransaction.java:82)
	at org.mybatis.spring.transaction.SpringManagedTransaction.getConnection(SpringManagedTransaction.java:68)
	at org.apache.ibatis.executor.BaseExecutor.getConnection(BaseExecutor.java:336)
	at com.baomidou.mybatisplus.core.executor.MybatisSimpleExecutor.prepareStatement(MybatisSimpleExecutor.java:93)
	at com.baomidou.mybatisplus.core.executor.MybatisSimpleExecutor.doQuery(MybatisSimpleExecutor.java:66)
	at org.apache.ibatis.executor.BaseExecutor.queryFromDatabase(BaseExecutor.java:324)
	at org.apache.ibatis.executor.BaseExecutor.query(BaseExecutor.java:156)
	at org.apache.ibatis.executor.CachingExecutor.query(CachingExecutor.java:109)
	at com.github.pagehelper.PageInterceptor.intercept(PageInterceptor.java:108)
	at org.apache.ibatis.plugin.Plugin.invoke(Plugin.java:61)
	at com.sun.proxy.$Proxy923.query(Unknown Source)
	at org.apache.ibatis.session.defaults.DefaultSqlSession.selectList(DefaultSqlSession.java:147)
	at org.apache.ibatis.session.defaults.DefaultSqlSession.selectList(DefaultSqlSession.java:140)
	at sun.reflect.GeneratedMethodAccessor492.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.mybatis.spring.SqlSessionTemplate$SqlSessionInterceptor.invoke(SqlSessionTemplate.java:433)
	at com.sun.proxy.$Proxy154.selectList(Unknown Source)
	at org.mybatis.spring.SqlSessionTemplate.selectList(SqlSessionTemplate.java:230)
	at com.baomidou.mybatisplus.core.override.MybatisMapperMethod.executeForMany(MybatisMapperMethod.java:158)
	at com.baomidou.mybatisplus.core.override.MybatisMapperMethod.execute(MybatisMapperMethod.java:76)
	at com.baomidou.mybatisplus.core.override.MybatisMapperProxy.invoke(MybatisMapperProxy.java:61)
	at com.sun.proxy.$Proxy190.queryNqaRoleMapLisByIds(Unknown Source)
	at sun.reflect.GeneratedMethodAccessor1673.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:343)
	at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:198)
	at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)
	at org.springframework.dao.support.PersistenceExceptionTranslationInterceptor.invoke(PersistenceExceptionTranslationInterceptor.java:139)
	at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186)
	at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:212)

ownerThread current state is WAITING, current stackTrace
	at java.lang.Throwable.fillInStackTrace(Native Method)
	at java.lang.Throwable.fillInStackTrace(Throwable.java:783)
	at java.lang.Throwable.<init>(Throwable.java:287)
	at java.lang.Exception.<init>(Exception.java:84)
	at java.lang.RuntimeException.<init>(RuntimeException.java:80)
	at org.springframework.core.NestedRuntimeException.<init>(NestedRuntimeException.java:66)
	at org.springframework.dao.DataAccessException.<init>(DataAccessException.java:57)
	at org.springframework.dao.RecoverableDataAccessException.<init>(RecoverableDataAccessException.java:48)
	at org.springframework.jdbc.support.SQLExceptionSubclassTranslator.doTranslate(SQLExceptionSubclassTranslator.java:100)
	at org.springframework.jdbc.support.AbstractFallbackSQLExceptionTranslator.translate(AbstractFallbackSQLExceptionTranslator.java:72)
	at org.springframework.jdbc.support.AbstractFallbackSQLExceptionTranslator.translate(AbstractFallbackSQLExceptionTranslator.java:81)
	at org.mybatis.spring.MyBatisExceptionTranslator.translateExceptionIfPossible(MyBatisExceptionTranslator.java:73)
	at org.mybatis.spring.SqlSessionTemplate$SqlSessionInterceptor.invoke(SqlSessionTemplate.java:446)
	at com.sun.proxy.$Proxy154.selectList(Unknown Source)
	at org.mybatis.spring.SqlSessionTemplate.selectList(SqlSessionTemplate.java:230)
	at com.baomidou.mybatisplus.core.override.MybatisMapperMethod.executeForMany(MybatisMapperMethod.java:158)
	at com.baomidou.mybatisplus.core.override.MybatisMapperMethod.execute(MybatisMapperMethod.java:76)
	at com.baomidou.mybatisplus.core.override.MybatisMapperProxy.invoke(MybatisMapperProxy.java:61)
	at com.sun.proxy.$Proxy190.queryNqaRoleMapLisByIds(Unknown Source)
	at sun.reflect.GeneratedMethodAccessor1673.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:343)
	at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:198)
	at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)
	at org.springframework.dao.support.PersistenceExceptionTranslationInterceptor.invoke(PersistenceExceptionTranslationInterceptor.java:139)
	at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186)
	at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:212)

  无奈之下只好往上翻日志看看有没有啥可用的信息，找到了一条关键日志

2022-10-10 11:24:11,503 ERROR [admin-web] [DubboServerHandler-192.168.160.143:28081-thread-193] [c.b.n.s.i.NurseBlockImportServiceImpl:93] -  importNurseDossier error . error info : {"@type":"java.lang.NullPointerException","stackTrace":[{"className":"com.bozhong.nursestaff.service.impl.NurseBlockImportServiceImpl","fileName":"NurseBlockImportServiceImpl.java","lineNumber":164,"methodName":"importNewTest","nativeMethod":false},{"className":"com.bozhong.nursestaff.service.impl.NurseBlockImportServiceImpl","fileName":"NurseBlockImportServiceImpl.java","lineNumber":79,"methodName":"importDossierBlock","nativeMethod":false},{"className":"com.bozhong.nursestaff.service.impl.NurseBlockImportServiceImpl$$FastClassBySpringCGLIB$$e3f52ec8","fileName":"<generated>","lineNumber":-1,"methodName":"invoke","nativeMethod":false},{"className":"org.springframework.cglib.proxy.MethodProxy","fileName":"MethodProxy.java","lineNumber":218,"methodName":"invoke","nativeMethod":false},{"className":"org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor","fileName":"CglibAopProxy.java","lineNumber":684,"methodName":"intercept","nativeMethod":false},{"className":"com.bozhong.nursestaff.service.impl.NurseBlockImportServiceImpl$$EnhancerBySpringCGLIB$$aa3b72d8","fileName":"<generated>","lineNumber":-1,"methodName":"importDossierBlock","nativeMethod":false},{"className":"com.bozhong.nursestaff.controller.load.ImportNurseBlockController","fileName":"ImportNurseBlockController.java","lineNumber":35,"methodName":"importDossierBlock","nativeMethod":false},{"className":"com.bozhong.nursestaff.controller.load.ImportNurseBlockController$$FastClassBySpringCGLIB$$d7d5e4a4","fileName":"<generated>","lineNumber":-1,"methodName":"invoke","nativeMethod":false},{"className":"org.springframework.cglib.proxy.MethodProxy","fileName":"MethodProxy.java","lineNumber":218,"methodName":"invoke","nativeMethod":false},{"className":"org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation","fileName":"CglibAopProxy.java","lineNumber":749,"methodName":"invokeJoinpoint","nativeMethod":false},{"className":"org.springframework.aop.framework.ReflectiveMethodInvocation","fileName":"ReflectiveMethodInvocation.java","lineNumber":163,"methodName":"proceed","nativeMethod":false},{"className":"org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint","fileName":"MethodInvocationProceedingJoinPoint.java","lineNumber":88,"methodName":"proceed","nativeMethod":false},{"className":"com.bozhong.config.LogAspectControllerConfig","fileName":"LogAspectControllerConfig.java","lineNumber":80,"methodName":"around","nativeMethod":false},{"className":"sun.reflect.GeneratedMethodAccessor1069","lineNumber":-1,"methodName":"invoke","nativeMethod":false},{"className":"sun.reflect.DelegatingMethodAccessorImpl","fileName":"DelegatingMethodAccessorImpl.java","lineNumber":43,"methodName":"invoke","nativeMethod":false},{"className":"java.lang.reflect.Method","fileName":"Method.java","lineNumber":498,"methodName":"invoke","nativeMethod":false},{"className":"org.springframework.aop.aspectj.AbstractAspectJAdvice","fileName":"AbstractAspectJAdvice.java","lineNumber":644,"methodName":"invokeAdviceMethodWithGivenArgs","nativeMethod":false},{"className":"org.springframework.aop.aspectj.AbstractAspectJAdvice","fileName":"AbstractAspectJAdvice.java","lineNumber":633,"methodName":"invokeAdviceMethod","nativeMethod":false},{"className":"org.springframework.aop.aspectj.AspectJAroundAdvice","fileName":"AspectJAroundAdvice.java","lineNumber":70,"methodName":"invoke","nativeMethod":false},{"className":"org.springframework.aop.framework.ReflectiveMethodInvocation","fileName":"ReflectiveMethodInvocation.java","lineNumber":186,"methodName":"proceed","nativeMethod":false},{"className":"org.springframework.aop.interceptor.ExposeInvocationInterceptor","fileName":"ExposeInvocationInterceptor.java","lineNumber":93,"methodName":"invoke","nativeMethod":false},{"className":"org.springframework.aop.framework.ReflectiveMethodInvocation","fileName":"ReflectiveMethodInvocation.java","lineNumber":186,"methodName":"proceed","nativeMethod":false},{"className":"org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor","fileName":"CglibAopProxy.java","lineNumber":688,"methodName":"intercept","nativeMethod":false},{"className":"com.bozhong.nursestaff.controller.load.ImportNurseBlockController$$EnhancerBySpringCGLIB$$b9a8122c","fileName":"<generated>","lineNumber":-1,"methodName":"importDossierBlock","nativeMethod":false},{"className":"com.alibaba.dubbo.common.bytecode.Wrapper356","fileName":"Wrapper356.java","lineNumber":-1,"methodName":"invokeMethod","nativeMethod":false},{"className":"com.alibaba.dubbo.rpc.proxy.javassist.JavassistProxyFactory$1","fileName":"JavassistProxyFactory.java","lineNumber":47,"methodName":"doInvoke","nativeMethod":false},{"className":"com.alibaba.dubbo.rpc.proxy.AbstractProxyInvoker","fileName":"AbstractProxyInvoker.java","lineNumber":76,"methodName":"invoke","nativeMethod":false},{"className":"com.alibaba.dubbo.config.invoker.DelegateProviderMetaDataInvoker","fileName":"DelegateProviderMetaDataInvoker.java","lineNumber":52,"methodName":"invoke","nativeMethod":false},{"className":"com.alibaba.dubbo.rpc.protocol.InvokerWrapper","fileName":"InvokerWrapper.java","lineNumber":56,"methodName":"invoke","nativeMethod":false},{"className":"io.seata.integration.dubbo.alibaba.AlibabaDubboTransactionPropagationFilter","fileName":"AlibabaDubboTransactionPropagationFilter.java","lineNumber":68,"methodName":"invoke","nativeMethod":false},{"className":"com.alibaba.dubbo.rpc.protocol.ProtocolFilterWrapper$1","fileName":"ProtocolFilterWrapper.java","lineNumber":72,"methodName":"invoke","nativeMethod":false},{"className":"co.faao.plugin.filter.ProviderExceptionFilter","fileName":"ProviderExceptionFilter.java","lineNumber":20,"methodName":"invoke","nativeMethod":false},{"className":"com.alibaba.dubbo.rpc.protocol.ProtocolFilterWrapper$1","fileName":"ProtocolFilterWrapper.java","lineNumber":72,"methodName":"invoke","nativeMethod":false},{"className":"com.alibaba.dubbo.rpc.filter.ExceptionFilter","fileName":"ExceptionFilter.java","lineNumber":62,"methodName":"invoke","nativeMethod":false},{"className":"com.alibaba.dubbo.rpc.protocol.ProtocolFilterWrapper$1","fileName":"ProtocolFilterWrapper.java","lineNumber":72,"methodName":"invoke","nativeMethod":false},{"className":"com.alibaba.dubbo.monitor.support.MonitorFilter","fileName":"MonitorFilter.java","lineNumber":75,"methodName":"invoke","nativeMethod":false},{"className":"com.alibaba.dubbo.rpc.protocol.ProtocolFilterWrapper$1","fileName":"ProtocolFilterWrapper.java","lineNumber":72,"methodName":"invoke","nativeMethod":false},{"className":"com.alibaba.dubbo.rpc.filter.TimeoutFilter","fileName":"TimeoutFilter.java","lineNumber":42,"methodName":"invoke","nativeMethod":false},{"className":"com.alibaba.dubbo.rpc.protocol.ProtocolFilterWrapper$1","fileName":"ProtocolFilterWrapper.java","lineNumber":72,"methodName":"invoke","nativeMethod":false},{"className":"com.alibaba.dubbo.rpc.protocol.dubbo.filter.TraceFilter","fileName":"TraceFilter.java","lineNumber":78,"methodName":"invoke","nativeMethod":false},{"className":"com.alibaba.dubbo.rpc.protocol.ProtocolFilterWrapper$1","fileName":"ProtocolFilterWrapper.java","lineNumber":72,"methodName":"invoke","nativeMethod":false},{"className":"co.faao.plugin.starter.jaeger.opentracing.filter.OpenTracingFilter","fileName":"OpenTracingFilter.java","lineNumber":97,"methodName":"processProviderTrace","nativeMethod":false},{"className":"co.faao.plugin.starter.jaeger.opentracing.filter.OpenTracingFilter","fileName":"OpenTracingFilter.java","lineNumber":63,"methodName":"invoke","nativeMethod":false},{"className":"com.alibaba.dubbo.rpc.protocol.ProtocolFilterWrapper$1","fileName":"ProtocolFilterWrapper.java","lineNumber":72,"methodName":"invoke","nativeMethod":false},{"className":"co.faao.plugin.starter.dubbo.filter.UserFilter","fileName":"UserFilter.java","lineNumber":39,"methodName":"invoke","nativeMethod":false},{"className":"com.alibaba.dubbo.rpc.protocol.ProtocolFilterWrapper$1","fileName":"ProtocolFilterWrapper.java","lineNumber":72,"methodName":"invoke","nativeMethod":false},{"className":"com.alibaba.dubbo.rpc.filter.ContextFilter","fileName":"ContextFilter.java","lineNumber":72,"methodName":"invoke","nativeMethod":false},{"className":"com.alibaba.dubbo.rpc.protocol.ProtocolFilterWrapper$1","fileName":"ProtocolFilterWrapper.java","lineNumber":72,"methodName":"invoke","nativeMethod":false},{"className":"com.alibaba.dubbo.rpc.filter.GenericFilter","fileName":"GenericFilter.java","lineNumber":131,"methodName":"invoke","nativeMethod":false},{"className":"com.alibaba.dubbo.rpc.protocol.ProtocolFilterWrapper$1","fileName":"ProtocolFilterWrapper.java","lineNumber":72,"methodName":"invoke","nativeMethod":false},{"className":"com.alibaba.dubbo.rpc.filter.ClassLoaderFilter","fileName":"ClassLoaderFilter.java","lineNumber":38,"methodName":"invoke","nativeMethod":false},{"className":"com.alibaba.dubbo.rpc.protocol.ProtocolFilterWrapper$1","fileName":"ProtocolFilterWrapper.java","lineNumber":72,"methodName":"invoke","nativeMethod":false},{"className":"com.alibaba.dubbo.rpc.filter.EchoFilter","fileName":"EchoFilter.java","lineNumber":38,"methodName":"invoke","nativeMethod":false},{"className":"com.alibaba.dubbo.rpc.protocol.ProtocolFilterWrapper$1","fileName":"ProtocolFilterWrapper.java","lineNumber":72,"methodName":"invoke","nativeMethod":false},{"className":"com.alibaba.dubbo.rpc.protocol.dubbo.DubboProtocol$1","fileName":"DubboProtocol.java","lineNumber":103,"methodName":"reply","nativeMethod":false},{"className":"com.alibaba.dubbo.remoting.exchange.support.header.HeaderExchangeHandler","fileName":"HeaderExchangeHandler.java","lineNumber":96,"methodName":"handleRequest","nativeMethod":false},{"className":"com.alibaba.dubbo.remoting.exchange.support.header.HeaderExchangeHandler","fileName":"HeaderExchangeHandler.java","lineNumber":172,"methodName":"received","nativeMethod":false},{"className":"com.alibaba.dubbo.remoting.transport.DecodeHandler","fileName":"DecodeHandler.java","lineNumber":51,"methodName":"received","nativeMethod":false},{"className":"com.alibaba.dubbo.remoting.transport.dispatcher.ChannelEventRunnable","fileName":"ChannelEventRunnable.java","lineNumber":80,"methodName":"run","nativeMethod":false},{"className":"java.util.concurrent.ThreadPoolExecutor","fileName":"ThreadPoolExecutor.java","lineNumber":1142,"methodName":"runWorker","nativeMethod":false},{"className":"java.util.concurrent.ThreadPoolExecutor$Worker","fileName":"ThreadPoolExecutor.java","lineNumber":617,"methodName":"run","nativeMethod":false},{"className":"java.lang.Thread","fileName":"Thread.java","lineNumber":748,"methodName":"run","nativeMethod":false}]}

  总结就是出现了大量的，非常非常多的NullPointerException，这还得了，报错日志json格式化一下好看一些：

  真的是要吐血了，代码难道丢失了一大堆？赶紧把jar包下下来查看class文件，也都全乎，没有丢失啊。

  用的是Zookeeper注册服务的呀，难不成是Zookeeper也出问题了，果不其然，在Zookeeper日志中发现了大量的
“Error:KeeperErrorCode = NodeExists for“。

---

解决方案：

  我是自己部署的单机环境，虽然使用了Zookeeper，其实只有一台，然后重启了下ZK就好了。

总结

  网上的原因搜出来大致都是相似的，看了两篇就没看了，还是查看日志一步步跟下去，就算原因可能更加深入，难以找到最终的源头，但至少也能弄个方向出来，我这重启ZK确实是好了，但也有可能是瞎猫碰到死耗子，但不管怎么说结局是好的，记录一下这种处理方式也有可能解决这个问题。