目前有一个进程服务脚本是不断查询渠道的接口,但是历史问题是有时订单量大的时候进程会卡死,这次遇到了进行排查一下:
首先获取该进程ID
ps -aux | grep QueryABC.php
sync360 11115 0.0 0.0 6564 864 ? Ss 14:00 0:00 /bin/sh -c /usr/local/bin/php /xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/QueryABC.php BILL99DF 10-8>> /home
sync360 11124 0.0 0.4 361628 17296 ? S 14:00 0:04 /usr/local/bin/php /xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/QueryABC.php BILL99DF 10-8
sync360 25230 0.0 0.0 63384 872 pts/0 S+ 15:28 0:00 grep QueryABC.php
1
2
3
4
5
6
ps-aux|grepQueryABC.php
sync360111150.00.06564864?Ss14:000:00/bin/sh-c/usr/local/bin/php/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/QueryABC.phpBILL99DF10-8>>/home
sync360111240.00.436162817296?S14:000:04/usr/local/bin/php/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/QueryABC.phpBILL99DF10-8
sync360252300.00.063384872pts/0S+15:280:00grepQueryABC.php
strace查看该进程正在持续的状态
sudo strace -T -tt -e trace=all -p 11124
[sudo] password for ancongcong:
Process 11124 attached - interrupt to quit
15:33:07.259044 read(9,
1
2
3
4
5
sudostrace-T-tt-etrace=all-p11124
[sudo]passwordforancongcong:
Process11124attached-interrupttoquit
15:33:07.259044read(9,
lsof查看进程的所使用的文件
lsof -p 11124
....
php 11124 sync360 mem REG 8,1 23736 3211320 /lib64/libnss_dns-2.5.so
php 11124 sync360 0r FIFO 0,6 1522728709 pipe
php 11124 sync360 1w REG 8,1 4088819 1869737 /xxxx/xxxx/xxxx/xxxx/logs/QueryABC.log
php 11124 sync360 2w FIFO 0,6 1522728710 pipe
php 11124 sync360 3w CHR 1,3 982 /dev/null
php 11124 sync360 4u IPv4 1522728838 TCP 211.151.122.234:46004->10.117.128.47:rtmp-port (CLOSE_WAIT)
php 11124 sync360 5wW REG 8,1 0 2704363 /xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/lockfile/QueryABC.php.BILL99DF.10-8
php 11124 sync360 6u IPv4 1522728841 TCP 211.151.122.234:51019->10.117.128.46:rtmp-port (CLOSE_WAIT)
php 11124 sync360 7w REG 8,1 31960384 1869789 /xxxx/xxxx/xxxx/xxxx/logs/XXXX_info.log.20180118
php 11124 sync360 8w REG 8,1 18151722 1869806 /xxxx/xxxx/xxxx/xxxx/logs/XXXX_QRY_info.log.20180118
php 11124 sync360 9u IPv4 1522729884 TCP 211.151.122.234:54976->61.152.114.130:https (ESTABLISHED)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
lsof-p11124
....
php11124sync360memREG8,1237363211320/lib64/libnss_dns-2.5.so
php11124sync3600rFIFO0,61522728709pipe
php11124sync3601wREG8,140888191869737/xxxx/xxxx/xxxx/xxxx/logs/QueryABC.log
php11124sync3602wFIFO0,61522728710pipe
php11124sync3603wCHR1,3982/dev/null
php11124sync3604uIPv41522728838TCP211.151.122.234:46004->10.117.128.47:rtmp-port(CLOSE_WAIT)
php11124sync3605wWREG8,102704363/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/lockfile/QueryABC.php.BILL99DF.10-8
php11124sync3606uIPv41522728841TCP211.151.122.234:51019->10.117.128.46:rtmp-port(CLOSE_WAIT)
php11124sync3607wREG8,1319603841869789/xxxx/xxxx/xxxx/xxxx/logs/XXXX_info.log.20180118
php11124sync3608wREG8,1181517221869806/xxxx/xxxx/xxxx/xxxx/logs/XXXX_QRY_info.log.20180118
php11124sync3609uIPv41522729884TCP211.151.122.234:54976->61.152.114.130:https(ESTABLISHED)
sudo netstat -tunpa | grep 11124
tcp 0 0 211.151.122.234:54976 61.152.114.130:443 ESTABLISHED 11124/php
tcp 1 0 211.151.122.234:51019 10.117.128.46:3500 CLOSE_WAIT 11124/php
tcp 1 0 211.151.122.234:46004 10.117.128.47:3500 CLOSE_WAIT 11124/php
1
2
3
4
5
sudonetstat-tunpa|grep11124
tcp00211.151.122.234:5497661.152.114.130:443ESTABLISHED11124/php
tcp10211.151.122.234:5101910.117.128.46:3500CLOSE_WAIT11124/php
tcp10211.151.122.234:4600410.117.128.47:3500CLOSE_WAIT11124/php
可以发现最终是停留在https的链接建立,等待获取数据,查看此处代码
ini_set('default_socket_timeout',30);
$scOptions = array('connection_timeout' => 30);
$clientObj = new SoapClient( $wsdl , $scOptions);
1
2
3
4
ini_set('default_socket_timeout',30);
$scOptions=array('connection_timeout'=>30);
$clientObj=newSoapClient($wsdl,$scOptions);
当前版本php较老,这里是有个bug的在https链接请求时SOAPClient的超时时间是不生效,最终采取如下方案解决此问题:
复写SOAPClient,在https时候使用curl来完成请求解决问题
class SoapClientTimeout extends SoapClient
{
private $timeout;
public function __setTimeout($timeout)
{
if (!is_int($timeout) && !is_null($timeout))
{
throw new Exception("Invalid timeout value");
}
$this->timeout = $timeout;
}
public function __doRequest($request, $location, $action, $version, $one_way = FALSE)
{
if (!$this->timeout)
{
// Call via parent because we require no timeout
$response = parent::__doRequest($request, $location, $action, $version, $one_way);
}
else
{
// Call via Curl and use the timeout
$curl = curl_init($location);
curl_setopt($curl, CURLOPT_VERBOSE, FALSE);
curl_setopt($curl, CURLOPT_RETURNTRANSFER, TRUE);
curl_setopt($curl, CURLOPT_POST, TRUE);
curl_setopt($curl, CURLOPT_POSTFIELDS, $request);
curl_setopt($curl, CURLOPT_HEADER, FALSE);
curl_setopt($curl, CURLOPT_HTTPHEADER, array("Content-Type: text/xml"));
curl_setopt($curl, CURLOPT_TIMEOUT, $this->timeout);
$response = curl_exec($curl);
if (curl_errno($curl))
{
throw new Exception(curl_error($curl));
}
curl_close($curl);
}
// Return?
if (!$one_way)
{
return ($response);
}
}
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
classSoapClientTimeoutextendsSoapClient
{
private$timeout;
publicfunction__setTimeout($timeout)
{
if(!is_int($timeout)&&!is_null($timeout))
{
thrownewException("Invalid timeout value");
}
$this->timeout=$timeout;
}
publicfunction__doRequest($request,$location,$action,$version,$one_way=FALSE)
{
if(!$this->timeout)
{
// Call via parent because we require no timeout
$response=parent::__doRequest($request,$location,$action,$version,$one_way);
}
else
{
// Call via Curl and use the timeout
$curl=curl_init($location);
curl_setopt($curl,CURLOPT_VERBOSE,FALSE);
curl_setopt($curl,CURLOPT_RETURNTRANSFER,TRUE);
curl_setopt($curl,CURLOPT_POST,TRUE);
curl_setopt($curl,CURLOPT_POSTFIELDS,$request);
curl_setopt($curl,CURLOPT_HEADER,FALSE);
curl_setopt($curl,CURLOPT_HTTPHEADER,array("Content-Type: text/xml"));
curl_setopt($curl,CURLOPT_TIMEOUT,$this->timeout);
$response=curl_exec($curl);
if(curl_errno($curl))
{
thrownewException(curl_error($curl));
}
curl_close($curl);
}
// Return?
if(!$one_way)
{
return($response);
}
}
}