排查:判断为节点内文件被删除后,资源被其他进程占用未及时释放。
处理方式如下:
#使用lsof获取被删除文件的进程占用状态
[root@node /tmp]$ lsof |grep deleted >/tmp/deleted.txt
#根据被删除文件大小顺序进行排序
[root@node /tmp]$ sort -k7 -rn /tmp/deleted.txt |head -n 10
openresty 243890 root 37w REG 8,20 448899811641 1558786034 /usr/local/openresty/nginx/logs/access.log-20220321 (deleted)
openresty 243852 root 5w REG 8,20 448899810987 1558786034 /usr/local/openresty/nginx/logs/access.log-20220321 (deleted)
openresty 235587 root 3w REG 8,20 221839246380 65719322 /usr/local/openresty/nginx/logs/access.log-20220321 (deleted)
openresty 235531 root 5w REG 8,20 221839245694 65719322 /usr/local/openresty/nginx/logs/access.log-20220321 (deleted)
tail 353795 root 1w REG 8,20 2863407045 1558786007 /nginx/ingress-es-r2x9n/access.log (deleted)
openresty 243890 root 2w REG 8,20 850509873 1558786000 /usr/local/openresty/nginx/logs/error.log-20220321 (deleted)
openresty 243890 root 19w REG 8,20 850509873 1558786000 /usr/local/openresty/nginx/logs/error.log-20220321 (deleted)
openresty 243852 root 4w REG 8,20 850509873 1558786000 /usr/local/openresty/nginx/logs/error.log-20220321 (deleted)
openresty 243852 root 2w REG 8,20 850509873 1558786000 /usr/local/openresty/nginx/logs/error.log-20220321 (deleted)
openresty 235587 root 2w REG 8,20 257939249 65719305 /usr/local/openresty/nginx/logs/error.log-20220321 (deleted)
#获取进程id后,进入/proc/<进程pid>/fd目录查询该进程内对应到此删除文件
[root@node /proc/243890/fd]$ ll |grep /usr/local/openresty/nginx/logs/access.log-20220321
l-wx------ 1 root root 64 Apr 20 11:14 37 -> /usr/local/openresty/nginx/logs/access.log-20220321 (deleted)
[root@node /proc/243890/fd]$ echo >3