记录一次正式环境服务报错排查记录。
某日被通知线上服务告警,错误日志全是 Timeout waiting for connection
首先梳理项目架构,项目很简单,就是一个使用axis2构建的webserice的客户端, Axis2版本为1.5.5
开始从此段报错入手排查,定位到 MultiThreadedHttpConnectionManager 这个类的 doGetConnection 方法
private HttpConnection doGetConnection(HostConfiguration hostConfiguration,
long timeout) throws ConnectionPoolTimeoutException {
HttpConnection connection = null;
int maxHostConnections = this.params.getMaxConnectionsPerHost(hostConfiguration);
int maxTotalConnections = this.params.getMaxTotalConnections();
synchronized (connectionPool) {
// we clone the hostConfiguration
// so that it cannot be changed once the connection has been retrieved
hostConfiguration = new HostConfiguration(hostConfiguration);
HostConnectionPool hostPool = connectionPool.getHostPool(hostConfiguration, true);
WaitingThread waitingThread = null;
boolean useTimeout = (timeout > 0);
long timeToWait = timeout;
long startWait = 0;
long endWait = 0;
while (connection == null) {
if (shutdown) {
throw new IllegalStateException("Connection factory has been shutdown.");
}
// happen to have a free connection with the right specs
//
if (hostPool.freeConnections.size() > 0) {
connection = connectionPool.getFreeConnection(hostConfiguration);
// have room to make more
//
} else if ((hostPool.numConnections < maxHostConnections)
&& (connectionPool.numConnections < maxTotalConnections)) {
connect