php页面查询卡死的原因分析,利用Strace、lsof排查PHP进程卡死问题

目前有一个进程服务脚本是不断查询渠道的接口，但是历史问题是有时订单量大的时候进程会卡死，这次遇到了进行排查一下：

首先获取该进程ID

ps -aux | grep QueryABC.php

sync360 11115 0.0 0.0 6564 864 ? Ss 14:00 0:00 /bin/sh -c /usr/local/bin/php /xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/QueryABC.php BILL99DF 10-8>> /home

sync360 11124 0.0 0.4 361628 17296 ? S 14:00 0:04 /usr/local/bin/php /xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/QueryABC.php BILL99DF 10-8

sync360 25230 0.0 0.0 63384 872 pts/0 S+ 15:28 0:00 grep QueryABC.php

1

2

3

4

5

6

ps-aux|grepQueryABC.php

sync360111150.00.06564864?Ss14:000:00/bin/sh-c/usr/local/bin/php/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/QueryABC.phpBILL99DF10-8>>/home

sync360111240.00.436162817296?S14:000:04/usr/local/bin/php/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/QueryABC.phpBILL99DF10-8

sync360252300.00.063384872pts/0S+15:280:00grepQueryABC.php

strace查看该进程正在持续的状态

sudo strace -T -tt -e trace=all -p 11124

[sudo] password for ancongcong:

Process 11124 attached - interrupt to quit

15:33:07.259044 read(9,

1

2

3

4

5

sudostrace-T-tt-etrace=all-p11124

[sudo]passwordforancongcong:

Process11124attached-interrupttoquit

15:33:07.259044read(9,

lsof查看进程的所使用的文件

lsof -p 11124

....

php 11124 sync360 mem REG 8,1 23736 3211320 /lib64/libnss_dns-2.5.so

php 11124 sync360 0r FIFO 0,6 1522728709 pipe

php 11124 sync360 1w REG 8,1 4088819 1869737 /xxxx/xxxx/xxxx/xxxx/logs/QueryABC.log

php 11124 sync360 2w FIFO 0,6 1522728710 pipe

php 11124 sync360 3w CHR 1,3 982 /dev/null

php 11124 sync360 4u IPv4 1522728838 TCP 211.151.122.234:46004->10.117.128.47:rtmp-port (CLOSE_WAIT)

php 11124 sync360 5wW REG 8,1 0 2704363 /xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/lockfile/QueryABC.php.BILL99DF.10-8

php 11124 sync360 6u IPv4 1522728841 TCP 211.151.122.234:51019->10.117.128.46:rtmp-port (CLOSE_WAIT)

php 11124 sync360 7w REG 8,1 31960384 1869789 /xxxx/xxxx/xxxx/xxxx/logs/XXXX_info.log.20180118

php 11124 sync360 8w REG 8,1 18151722 1869806 /xxxx/xxxx/xxxx/xxxx/logs/XXXX_QRY_info.log.20180118

php 11124 sync360 9u IPv4 1522729884 TCP 211.151.122.234:54976->61.152.114.130:https (ESTABLISHED)

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

lsof-p11124

....

php11124sync360memREG8,1237363211320/lib64/libnss_dns-2.5.so

php11124sync3600rFIFO0,61522728709pipe

php11124sync3601wREG8,140888191869737/xxxx/xxxx/xxxx/xxxx/logs/QueryABC.log

php11124sync3602wFIFO0,61522728710pipe

php11124sync3603wCHR1,3982/dev/null

php11124sync3604uIPv41522728838TCP211.151.122.234:46004->10.117.128.47:rtmp-port(CLOSE_WAIT)

php11124sync3605wWREG8,102704363/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/xxxx/lockfile/QueryABC.php.BILL99DF.10-8

php11124sync3606uIPv41522728841TCP211.151.122.234:51019->10.117.128.46:rtmp-port(CLOSE_WAIT)

php11124sync3607wREG8,1319603841869789/xxxx/xxxx/xxxx/xxxx/logs/XXXX_info.log.20180118

php11124sync3608wREG8,1181517221869806/xxxx/xxxx/xxxx/xxxx/logs/XXXX_QRY_info.log.20180118

php11124sync3609uIPv41522729884TCP211.151.122.234:54976->61.152.114.130:https(ESTABLISHED)

sudo netstat -tunpa | grep 11124

tcp 0 0 211.151.122.234:54976 61.152.114.130:443 ESTABLISHED 11124/php

tcp 1 0 211.151.122.234:51019 10.117.128.46:3500 CLOSE_WAIT 11124/php

tcp 1 0 211.151.122.234:46004 10.117.128.47:3500 CLOSE_WAIT 11124/php

1

2

3

4

5

sudonetstat-tunpa|grep11124

tcp00211.151.122.234:5497661.152.114.130:443ESTABLISHED11124/php

tcp10211.151.122.234:5101910.117.128.46:3500CLOSE_WAIT11124/php

tcp10211.151.122.234:4600410.117.128.47:3500CLOSE_WAIT11124/php

可以发现最终是停留在https的链接建立，等待获取数据，查看此处代码

ini_set('default_socket_timeout',30);

$scOptions = array('connection_timeout' => 30);

$clientObj = new SoapClient( $wsdl , $scOptions);

1

2

3

4

ini_set('default_socket_timeout',30);

$scOptions=array('connection_timeout'=>30);

$clientObj=newSoapClient($wsdl,$scOptions);

当前版本php较老，这里是有个bug的在https链接请求时SOAPClient的超时时间是不生效，最终采取如下方案解决此问题：

复写SOAPClient，在https时候使用curl来完成请求解决问题

class SoapClientTimeout extends SoapClient

{

private $timeout;

public function __setTimeout($timeout)

{

if (!is_int($timeout) && !is_null($timeout))

{

throw new Exception("Invalid timeout value");

}

$this->timeout = $timeout;

}

public function __doRequest($request, $location, $action, $version, $one_way = FALSE)

{

if (!$this->timeout)

{

// Call via parent because we require no timeout

$response = parent::__doRequest($request, $location, $action, $version, $one_way);

}

else

{

// Call via Curl and use the timeout

$curl = curl_init($location);

curl_setopt($curl, CURLOPT_VERBOSE, FALSE);

curl_setopt($curl, CURLOPT_RETURNTRANSFER, TRUE);

curl_setopt($curl, CURLOPT_POST, TRUE);

curl_setopt($curl, CURLOPT_POSTFIELDS, $request);

curl_setopt($curl, CURLOPT_HEADER, FALSE);

curl_setopt($curl, CURLOPT_HTTPHEADER, array("Content-Type: text/xml"));

curl_setopt($curl, CURLOPT_TIMEOUT, $this->timeout);

$response = curl_exec($curl);

if (curl_errno($curl))

{

throw new Exception(curl_error($curl));

}

curl_close($curl);

}

// Return?

if (!$one_way)

{

return ($response);

}

}

}

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

classSoapClientTimeoutextendsSoapClient

{

private$timeout;

publicfunction__setTimeout($timeout)

{

if(!is_int($timeout)&&!is_null($timeout))

{

thrownewException("Invalid timeout value");

}

$this->timeout=$timeout;

}

publicfunction__doRequest($request,$location,$action,$version,$one_way=FALSE)

{

if(!$this->timeout)

{

// Call via parent because we require no timeout

$response=parent::__doRequest($request,$location,$action,$version,$one_way);

}

else

{

// Call via Curl and use the timeout

$curl=curl_init($location);

curl_setopt($curl,CURLOPT_VERBOSE,FALSE);

curl_setopt($curl,CURLOPT_RETURNTRANSFER,TRUE);

curl_setopt($curl,CURLOPT_POST,TRUE);

curl_setopt($curl,CURLOPT_POSTFIELDS,$request);

curl_setopt($curl,CURLOPT_HEADER,FALSE);

curl_setopt($curl,CURLOPT_HTTPHEADER,array("Content-Type: text/xml"));

curl_setopt($curl,CURLOPT_TIMEOUT,$this->timeout);

$response=curl_exec($curl);

if(curl_errno($curl))

{

thrownewException(curl_error($curl));

}

curl_close($curl);

}

// Return?

if(!$one_way)

{

return($response);

}

}

}