Apache访问resin间歇性出现503错误的解决方法

最近使用apache-2.2.22 和resin-3.1.12 组合做web容器的时候 遇到一些问题，apache和resin都处于正常启动状态，两者的配置都使用默认配置，但是apache的访问日志内会间歇出现503的错误，越来越多，直到最后404错误，resin应用挂起。

 

查看netstat类似如下数据，总共有效连接竟然达到600多个，但是apache的MaxKeepAliveRequests总共配置着300个，

总归原因可能此时resin有效连接数大于apache默认的连接数导致的。

 

参照某哥们的博客，修改resin配置：

http://www.javatang.com/archives/2010/11/13/1228487.html

那为什么会间歇性出现503错误呢？这个是因为resin默认的socket-timeout为65s，时间太短了，有些比较大的请求在这个时间之内还没有进行完。

综合上面所说，参考的配置如下：

1.  <resin ...>
2.      <cluster id="app-tier">
3.          <server-default>
4.              <socket-timeout>180s</socket-timeout>
5.              <keepalive-max>3000</keepalive-max>
6.              <keepalive-timeout>180s</keepalive-timeout>
7.              <load-balance-idle-time>120s</load-balance-idle-time>
8.          </server-default>
9.      </cluster>
10.  </resin>
11. 只修改前三个参数还不管用，最终添加了load-balance-idle-time才最终解决问题。

    参考官网http://caucho.com/resin-3.1/doc/server-tags.xtp#loadbalanceidletime

     

    shell监控access_log.2012.04.03日志如下：

    [k1302262@localhost conf]$ cat  /opt/sys_job/check_apache_access.sh
    #!/bin/sh
    function getmonth {
    mon=$1
     case $mon in
       '01') num=Jan;;
       '02') num=Feb;;  
       '03') num=Mar;; 
       '04') num=Apr;; 
        '05') num=May;;  
        '06') num=June;;
        '07') num=July;;
        '08') num=Aug;;   
        '09') num=Sept;;   
        '10') num=Oct;;
       '11') num=Nov;;   
       '12') num=Dec;;
      esac
    }

    host=`/sbin/ifconfig|grep inet|awk '{print $2}' |head -1`
    echo "check apache acces log a hour ago on $host"

    now=`/bin/date '+%Y.%m.%d'`
    #echo $now
    lastonehour=$(/bin/date -d "1 hour ago" +%H)
    #lastonehour=`/bin/date  +%H '1 hour ago'`
    lastonedate=$(/bin/date +%d)
    lastonemonth=`/bin/date +%m`
    lastoneyear=$(/bin/date +%Y)    
    getmonth "01"
    #echo $num
    #echo $lastonehour
    serchtime="$lastonedate/$num/$lastoneyear:$lastonehour"
    echo "serch time is $serchtime"
    successcount=`grep "$serchtime" /usr/local/apache/logs/access_log.$now | grep -E "503|404" | wc -l`
    #echo $successcount
    content="$now/$host apache access too many 503 error  : count is $successcount"
    #echo $content
    if [ $successcount -gt 3 ]
    then
    echo $content
     sh /opt/sys_job/send2mobile.sh "${content}"
    fi

     

 

Active Internet connections (w/o servers)
Proto Recv-Q Send-Q Local Address               Foreign Address             State     
tcp        0      1 localhost.localdomain:22217 localhost.localdomain:6801  SYN_SENT   
tcp        0      1 localhost.localdomain:22216 localhost.localdomain:6801  SYN_SENT   
tcp        0      1 localhost.localdomain:22219 localhost.localdomain:6801  SYN_SENT   
tcp        0      1 localhost.localdomain:22218 localhost.localdomain:6801  SYN_SENT   
tcp        0     37 localhost.localdomain:22215 localhost.localdomain:6801  ESTABLISHED
tcp        0     37 localhost.localdomain:22211 localhost.localdomain:6801  ESTABLISHED
tcp        0     37 localhost.localdomain:22205 localhost.localdomain:6801  ESTABLISHED
tcp        0   1258 localhost.localdomain:22206 localhost.localdomain:6801  ESTABLISHED
tcp        0     37 localhost.localdomain:22200 localhost.localdomain:6801  ESTABLISHED
tcp        0   1306 localhost.localdomain:22198 localhost.localdomain:6801  ESTABLISHED
tcp        0   1256 localhost.localdomain:22195 localhost.localdomain:6801  ESTABLISHED
tcp        0     37 localhost.localdomain:22188 localhost.localdomain:6801  ESTABLISHED
tcp        0     37 localhost.localdomain:22185 localhost.localdomain:6801  ESTABLISHED
tcp        1      0 localhost.localdomain:18931 localhost.localdomain:6801  CLOSE_WAIT 

tcp       31      0 localhost.localdomain:6801  localhost.localdomain:19443 ESTABLISHED
tcp       37      0 localhost.localdomain:6801  localhost.localdomain:19955 ESTABLISHED
tcp       31      0 localhost.localdomain:6801  localhost.localdomain:19441 ESTABLISHED
tcp       37      0 localhost.localdomain:6801  localhost.localdomain:19953 ESTABLISHED
tcp       37      0 localhost.localdomain:6801  localhost.localdomain:19957 ESTABLISHED
tcp     1261      0 localhost.localdomain:6801  localhost.localdomain:19946 CLOSE_WAIT 
tcp       31      0 localhost.localdomain:6801  localhost.localdomain:19435 ESTABLISHED
tcp        1      0 localhost.localdomain:6801  localhost.localdomain:19432 CLOSE_WAIT 
tcp     1262      0 localhost.localdomain:6801  localhost.localdomain:19944 CLOSE_WAIT 
tcp        0      0 localhost.localdomain:40274 localhost.localdomain:40303 ESTABLISHED
tcp     1357      0 localhost.localdomain:6801  localhost.localdomain:19945 CLOSE_WAIT 
tcp       37      0 localhost.localdomain:6801  localhost.localdomain:19951 ESTABLISHED
tcp       37      0 localhost.localdomain:6801  localhost.localdomain:19948 ESTABLISHED
tcp       31      0 localhost.localdomain:6801  localhost.localdomain:19437 ESTABLISHED
tcp       31      0 localhost.localdomain:6801  localhost.localdomain:19426 ESTABLISHED
tcp       31      0 localhost.localdomain:6801  localhost.localdomain:19427 ESTABLISHED
tcp       31      0 localhost.localdomain:6801  localhost.localdomain:19425 ESTABLISHED
tcp       37      0 localhost.localdomain:6801  localhost.localdomain:19942 ESTABLISHED
tcp       31      0 localhost.localdomain:6801  localhost.localdomain:19430 ESTABLISHED
tcp        1      0 localhost.localdomain:6801  localhost.localdomain:19687 CLOSE_WAIT 
tcp     1299      0 localhost.localdomain:6801  localhost.localdomain:19943 CLOSE_WAIT 
tcp       31      0 localhost.localdomain:6801  localhost.localdomain:19431 ESTABLISHED
tcp       37      0 localhost.localdomain:6801  localhost.localdomain:19940 ESTABLISHED
tcp       31      0 localhost.localdomain:6801  localhost.localdomain:19428 ESTABLISHED
tcp       31      0 localhost.localdomain:6801  localhost.localdomain:19429 ESTABLISHED
tcp        1      0 ::ffff:220.181.130.226:6100 ::ffff:118.85.201.42:10121  CLOSE_WAIT