################################################################################
极限测试4 - 系统能创建的最大tcp连接数
################################################################################
系统参数查询: sysctl -a,
系统参数修改:
修改/etc/sysctl.conf, 然后调用sysctl -p
临时修改:
sysctl -w xxxx=xx,
或直接用echo 命令修改对应文件内容。
遇到的问题及解决办法:
1. 打开文件失败, "Too many open files"
解决办法: 修改用户进程可打开文件数限制
临时修改(本终端有效): ulimit -n xxxxx
永久修改: 例如: hjj 用户20万文件句柄
root@hjj /home/hjj]# cat /etc/security/limits.conf
hjj soft nofile 204800
hjj hard nofile 204800
查看Linux系统级的最大打开文件数限制,使用如下命令:
[root@hjj /home/hjj]# cat /proc/sys/fs/file-max
186381
不满足要求时,可以修改该值。
2. connect()调用返回失败,
不能分配请求地址. "Can't assign requested address"
如果现有的TCP客户端连接已将所有的本地端口号占满,则此时就无法为新的TCP客户端连接分配一个本地端口号了
修改/etc/sysctl.conf文件,添加如下行:
默认:net.ipv4.ip_local_port_range = 32768 61000
修改为:net.ipv4.ip_local_port_range = 1024 65535
请注意,小于1024的端口为系统保留
修改完后调用 sysctl -p
临时修改:
sysctl -w net.ipv4.ip_local_port_range=1024 65535
也可以用echo 命令直接修改内核文件
如果统计发现time_wait 状态过多, 需要修改如下参数,使得可以快速释放端口并重新利用
net.ipv4.tcp_timestamps = 1 开启对于TCP时间戳的支持,
net.ipv4.tcp_tw_recycle = 1 回收
net.ipv4.tcp_tw_reuse = 1 利用
3. "Connection reset by peer"
原因。(主动端(客户端))
NOT Connected, error:Connection reset by peer
客户端不经过close 而直接退出连接时。系统发reset 包
原因。(被动端(服务器))
NOT Connected, error:Connection reset by peer
NOT Connected, error:Connection timed out
NOT Connected, error:Broken pipe
服务器已经不堪重负。调优参数,减轻负载
1. 减少运行负载。(包括优化程序执行代码负载,例如不要加log)
2. 增加内核backlog
net.ipv4.tcp_max_syn_backlog = 4096
3. 增加 net.ipv4.tcp_mem 值
tcp_mem[0]:低于此值,TCP没有内存压力.
tcp_mem[1]:在此值下,进入内存压力阶段.
tcp_mem[2]:高于此值,TCP拒绝分配socket.
net.ipv4.tcp_mem = 176448 235264 352896
关于内存及缓存大小,适当的修改,有效果保留,无效果可以放弃。
还有很多项可如此设置。摘录一些目前我的tcp 内核设置,供参考。
可以满足无负载时10万连接,重负载时3万连接。
机器配置:
1. cpu (双核3G cpu)
[root@hjj /home/hjj]# cat /proc/cpuinfo |grep cpu
cpu family : 6
cpu MHz : 2925.705
cpu cores : 2
cpuid level : 13
cpu family : 6
cpu MHz : 2925.705
cpu cores : 2
cpuid level : 13
2. mem (2G)
[root@hjj /home/hjj]# free
total used free shared buffers cached
Mem: 1903204 1328016 575188 0 143100 768496
-/+ buffers/cache: 416420 1486784
Swap: 4095992 0 4095992
[root@hjj /home/hjj]# sysctl -a |grep ipv4.tcp
net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_sack = 1
net.ipv4.tcp_retrans_collapse = 1
net.ipv4.tcp_syn_retries = 5
net.ipv4.tcp_synack_retries = 5
net.ipv4.tcp_max_orphans = 131072
net.ipv4.tcp_max_tw_buckets = 131072
net.ipv4.tcp_keepalive_time = 7200
net.ipv4.tcp_keepalive_probes = 9
net.ipv4.tcp_keepalive_intvl = 75
net.ipv4.tcp_retries1 = 3
net.ipv4.tcp_retries2 = 15
net.ipv4.tcp_fin_timeout = 2
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_abort_on_overflow = 0
net.ipv4.tcp_stdurg = 0
net.ipv4.tcp_rfc1337 = 0
net.ipv4.tcp_max_syn_backlog = 4096
net.ipv4.tcp_orphan_retries = 0
net.ipv4.tcp_fack = 1
net.ipv4.tcp_reordering = 3
net.ipv4.tcp_ecn = 2
net.ipv4.tcp_dsack = 1
net.ipv4.tcp_mem = 176448 235264 352896
net.ipv4.tcp_wmem = 4096 16384 4194304
net.ipv4.tcp_rmem = 4096 87380 4194304
net.ipv4.tcp_app_win = 31
net.ipv4.tcp_adv_win_scale = 2
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_frto = 2
net.ipv4.tcp_frto_response = 0
net.ipv4.tcp_low_latency = 0
net.ipv4.tcp_no_metrics_save = 0
net.ipv4.tcp_moderate_rcvbuf = 1
net.ipv4.tcp_tso_win_divisor = 3
net.ipv4.tcp_congestion_control = cubic
net.ipv4.tcp_abc = 0
net.ipv4.tcp_mtu_probing = 0
net.ipv4.tcp_base_mss = 512
net.ipv4.tcp_workaround_signed_windows = 0
net.ipv4.tcp_challenge_ack_limit = 100
net.ipv4.tcp_limit_output_bytes = 131072
net.ipv4.tcp_dma_copybreak = 4096
net.ipv4.tcp_slow_start_after_idle = 1
net.ipv4.tcp_available_congestion_control = cubic reno
net.ipv4.tcp_allowed_congestion_control = cubic reno
net.ipv4.tcp_max_ssthresh = 0
net.ipv4.tcp_thin_linear_timeouts = 0
net.ipv4.tcp_thin_dupack = 0
net.ipv4.tcp_min_tso_segs = 2
极限测试4 - 系统能创建的最大tcp连接数
################################################################################
系统参数查询: sysctl -a,
系统参数修改:
修改/etc/sysctl.conf, 然后调用sysctl -p
临时修改:
sysctl -w xxxx=xx,
或直接用echo 命令修改对应文件内容。
遇到的问题及解决办法:
1. 打开文件失败, "Too many open files"
解决办法: 修改用户进程可打开文件数限制
临时修改(本终端有效): ulimit -n xxxxx
永久修改: 例如: hjj 用户20万文件句柄
root@hjj /home/hjj]# cat /etc/security/limits.conf
hjj soft nofile 204800
hjj hard nofile 204800
查看Linux系统级的最大打开文件数限制,使用如下命令:
[root@hjj /home/hjj]# cat /proc/sys/fs/file-max
186381
不满足要求时,可以修改该值。
2. connect()调用返回失败,
不能分配请求地址. "Can't assign requested address"
如果现有的TCP客户端连接已将所有的本地端口号占满,则此时就无法为新的TCP客户端连接分配一个本地端口号了
修改/etc/sysctl.conf文件,添加如下行:
默认:net.ipv4.ip_local_port_range = 32768 61000
修改为:net.ipv4.ip_local_port_range = 1024 65535
请注意,小于1024的端口为系统保留
修改完后调用 sysctl -p
临时修改:
sysctl -w net.ipv4.ip_local_port_range=1024 65535
也可以用echo 命令直接修改内核文件
如果统计发现time_wait 状态过多, 需要修改如下参数,使得可以快速释放端口并重新利用
net.ipv4.tcp_timestamps = 1 开启对于TCP时间戳的支持,
net.ipv4.tcp_tw_recycle = 1 回收
net.ipv4.tcp_tw_reuse = 1 利用
3. "Connection reset by peer"
原因。(主动端(客户端))
NOT Connected, error:Connection reset by peer
客户端不经过close 而直接退出连接时。系统发reset 包
原因。(被动端(服务器))
NOT Connected, error:Connection reset by peer
NOT Connected, error:Connection timed out
NOT Connected, error:Broken pipe
服务器已经不堪重负。调优参数,减轻负载
1. 减少运行负载。(包括优化程序执行代码负载,例如不要加log)
2. 增加内核backlog
net.ipv4.tcp_max_syn_backlog = 4096
3. 增加 net.ipv4.tcp_mem 值
tcp_mem[0]:低于此值,TCP没有内存压力.
tcp_mem[1]:在此值下,进入内存压力阶段.
tcp_mem[2]:高于此值,TCP拒绝分配socket.
net.ipv4.tcp_mem = 176448 235264 352896
关于内存及缓存大小,适当的修改,有效果保留,无效果可以放弃。
还有很多项可如此设置。摘录一些目前我的tcp 内核设置,供参考。
可以满足无负载时10万连接,重负载时3万连接。
机器配置:
1. cpu (双核3G cpu)
[root@hjj /home/hjj]# cat /proc/cpuinfo |grep cpu
cpu family : 6
cpu MHz : 2925.705
cpu cores : 2
cpuid level : 13
cpu family : 6
cpu MHz : 2925.705
cpu cores : 2
cpuid level : 13
2. mem (2G)
[root@hjj /home/hjj]# free
total used free shared buffers cached
Mem: 1903204 1328016 575188 0 143100 768496
-/+ buffers/cache: 416420 1486784
Swap: 4095992 0 4095992
[root@hjj /home/hjj]# sysctl -a |grep ipv4.tcp
net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_sack = 1
net.ipv4.tcp_retrans_collapse = 1
net.ipv4.tcp_syn_retries = 5
net.ipv4.tcp_synack_retries = 5
net.ipv4.tcp_max_orphans = 131072
net.ipv4.tcp_max_tw_buckets = 131072
net.ipv4.tcp_keepalive_time = 7200
net.ipv4.tcp_keepalive_probes = 9
net.ipv4.tcp_keepalive_intvl = 75
net.ipv4.tcp_retries1 = 3
net.ipv4.tcp_retries2 = 15
net.ipv4.tcp_fin_timeout = 2
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_abort_on_overflow = 0
net.ipv4.tcp_stdurg = 0
net.ipv4.tcp_rfc1337 = 0
net.ipv4.tcp_max_syn_backlog = 4096
net.ipv4.tcp_orphan_retries = 0
net.ipv4.tcp_fack = 1
net.ipv4.tcp_reordering = 3
net.ipv4.tcp_ecn = 2
net.ipv4.tcp_dsack = 1
net.ipv4.tcp_mem = 176448 235264 352896
net.ipv4.tcp_wmem = 4096 16384 4194304
net.ipv4.tcp_rmem = 4096 87380 4194304
net.ipv4.tcp_app_win = 31
net.ipv4.tcp_adv_win_scale = 2
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_frto = 2
net.ipv4.tcp_frto_response = 0
net.ipv4.tcp_low_latency = 0
net.ipv4.tcp_no_metrics_save = 0
net.ipv4.tcp_moderate_rcvbuf = 1
net.ipv4.tcp_tso_win_divisor = 3
net.ipv4.tcp_congestion_control = cubic
net.ipv4.tcp_abc = 0
net.ipv4.tcp_mtu_probing = 0
net.ipv4.tcp_base_mss = 512
net.ipv4.tcp_workaround_signed_windows = 0
net.ipv4.tcp_challenge_ack_limit = 100
net.ipv4.tcp_limit_output_bytes = 131072
net.ipv4.tcp_dma_copybreak = 4096
net.ipv4.tcp_slow_start_after_idle = 1
net.ipv4.tcp_available_congestion_control = cubic reno
net.ipv4.tcp_allowed_congestion_control = cubic reno
net.ipv4.tcp_max_ssthresh = 0
net.ipv4.tcp_thin_linear_timeouts = 0
net.ipv4.tcp_thin_dupack = 0
net.ipv4.tcp_min_tso_segs = 2