文件删除后仍然被进程占用导致分区占满

---

[背景]
docker 容器启动异常，查看日志报错 /var/log/pods 下面存储分区不够，查看 /var 分区未发现异常目录。

[process]
/var/lib/docker 为额外挂载的分区，从 du -sh 的结果来看并没有大目录占用的 /var 分区。

root@10.10.180.70[/root]#df -h | head -n 20
Filesystem                     Size  Used Avail Use% Mounted on
devtmpfs                        24G     0   24G   0% /dev
tmpfs                           24G     0   24G   0% /dev/shm
tmpfs                           24G  2.3G   22G  10% /run
tmpfs                           24G     0   24G   0% /sys/fs/cgroup
/dev/mapper/vg00-lvroot         15G   15G  950M  94% /
tmpfs                           24G  8.0K   24G   1% /tmp
/dev/sda1                     1014M  156M  859M  16% /boot
/dev/mapper/vgdata-lvzpaas      50G   12G   39G  24% /zpaas
/dev/mapper/vgdata-lvzlogs      15G  767M   15G   5% /tmp/zlogs
/dev/mapper/vgdata-lvzcm        15G  531M   15G   4% /zcm
/dev/mapper/vg00-lvvar         8.0G  8.0G   20K 100% /var
/dev/mapper/vg00-lvhome         15G  2.2G   13G  15% /home
/dev/mapper/vgdata-lvdocker     50G  9.3G   41G  19% /var/lib/docker
/dev/mapper/vgdata-lvkubelet    35G  283M   35G   1% /var/lib/kubelet
/dev/mapper/vgdata-lvzpaasssd   15G  2.6G   13G  18% /zpaasssd
/dev/mapper/vgdata-lvzcore      15G  140M   15G   1% /tmp/zcore
tmpfs                          4.7G     0  4.7G   0% /run/user/1803
overlay                         10G  8.0K   10G   1% /var/lib/docker/overlay2/5d649e849b7008a4191dc1d8d1ea91caf26060e63064806fba4f27fe0481f4cb/merged
overlay                         10G   12K   10G   1% /var/lib/docker/overlay2/6eacec6dd46da0545efe991153dc644172bdf6eb1a521afdddd521bd33f4fbd2/merged

root@10.10.180.70[/var/lib]#du -sh ./*
132K    ./alternatives
0       ./bluetooth
8.0K    ./calico
0       ./clamav
156K    ./cni
2.2M    ./containerd
60K     ./containers
0       ./dbus
0       ./dhclient
0       ./dhcpd
4.3M    ./dnf
22G     ./docker
116K    ./dockershim
0       ./dracut
0       ./ebtables
189M    ./etcd
0       ./games
16K     ./hsqldb
0       ./initramfs
608K    ./kubelet
4.0K    ./logrotate
0       ./misc
0       ./mysql
0       ./mysql-files
0       ./mysql-keyring
0       ./net-snmp
16K     ./NetworkManager
0       ./nginx
0       ./ntp
0       ./os-prober
0       ./polkit-1
0       ./portables
0       ./private
0       ./registry
14M     ./rpm
12K     ./rpm-state
0       ./rsyslog
4.0K    ./run
27M     ./selinux
0       ./sntp
4.0K    ./supportinfo
152K    ./systemd
0       ./tpm
0       ./tuned
4.0K    ./unbound
0       ./zookeeper

怀疑是有文件删除后仍然被进程占用导致没有释放。

root@10.10.180.70[/var/lib]#lsof  -w|grep -i deleted
rsyslogd     1555                       root    5w      REG              253,5 8190754816   12583066 /var/log/messages (deleted)
rsyslogd     1555                       root    8w      REG              253,5   26185728   12583067 /var/log/secure (deleted)
rsyslogd     1555    1576 in:imjour     root    5w      REG              253,5 8190754816   12583066 /var/log/messages (deleted)
rsyslogd     1555    1576 in:imjour     root    8w      REG              253,5   26185728   12583067 /var/log/secure (deleted)
rsyslogd     1555    1577 rs:main       root    5w      REG              253,5 8190754816   12583066 /var/log/messages (deleted)
rsyslogd     1555    1577 rs:main       root    8w      REG              253,5   26185728   12583067 /var/log/secure (deleted)
redis-ser    2330                     zcache    8wW     REG              253,3        771   23450691 /home/zcache/data/2102/nodes-2102.conf (deleted)
redis-ser    2330    2333 redis-ser   zcache    8wW     REG              253,3        771   23450691 /home/zcache/data/2102/nodes-2102.conf (deleted)
redis-ser    2330    2335 redis-ser   zcache    8wW     REG              253,3        771   23450691 /home/zcache/data/2102/nodes-2102.conf (deleted)
redis-ser    2330    2336 redis-ser   zcache    8wW     REG              253,3        771   23450691 /home/zcache/data/2102/nodes-2102.conf (deleted)
redis-ser    2330    2337 redis-ser   zcache    8wW     REG              253,3        771   23450691 /home/zcache/data/2102/nodes-2102.conf (deleted)
redis-ser    2330    2338 redis-ser   zcache    8wW     REG              253,3        771   23450691 /home/zcache/data/2102/nodes-2102.conf (deleted)
redis-ser    2330    2339 redis-ser   zcache    8wW     REG              253,3        771   23450691 /home/zcache/data/2102/nodes-2102.conf (deleted)
redis-ser    6214                     zcache    8wW     REG              253,3        771     273180 /home/zcache/data/2100/nodes-2100.conf (deleted)
redis-ser    6214    6215 redis-ser   zcache    8wW     REG              253,3        771     273180 /home/zcache/data/2100/nodes-2100.conf (deleted)
redis-ser    6214    6216 redis-ser   zcache    8wW     REG              253,3        771     273180 /home/zcache/data/2100/nodes-2100.conf (deleted)
redis-ser    6214    6217 redis-ser   zcache    8wW     REG              253,3        771     273180 /home/zcache/data/2100/nodes-2100.conf (deleted)
redis-ser    6214    6218 redis-ser   zcache    8wW     REG              253,3        771     273180 /home/zcache/data/2100/nodes-2100.conf (deleted)
redis-ser    6214    6219 redis-ser   zcache    8wW     REG              253,3        771     273180 /home/zcache/data/2100/nodes-2100.conf (deleted)
redis-ser    6214    6220 redis-ser   zcache    8wW     REG              253,3        771     273180 /home/zcache/data/2100/nodes-2100.conf (deleted)
redis-ser    6277                     zcache    8wW     REG              253,3        771    8541496 /home/zcache/data/2101/nodes-2101.conf (deleted)
redis-ser    6277    6278 redis-ser   zcache    8wW     REG              253,3        771    8541496 /home/zcache/data/2101/nodes-2101.conf (deleted)
redis-ser    6277    6279 redis-ser   zcache    8wW     REG              253,3        771    8541496 /home/zcache/data/2101/nodes-2101.conf (deleted)
redis-ser    6277    6280 redis-ser   zcache    8wW     REG              253,3        771    8541496 /home/zcache/data/2101/nodes-2101.conf (deleted)
redis-ser    6277    6281 redis-ser   zcache    8wW     REG              253,3        771    8541496 /home/zcache/data/2101/nodes-2101.conf (deleted)
redis-ser    6277    6282 redis-ser   zcache    8wW     REG              253,3        771    8541496 /home/zcache/data/2101/nodes-2101.conf (deleted)
redis-ser    6277    6283 redis-ser   zcache    8wW     REG              253,3        771    8541496 /home/zcache/data/2101/nodes-2101.conf (deleted)

root@10.10.180.70[/var/lib]#ps -ef | grep 1555
root        1555       1  0 Apr12 ?        21:14:35 /usr/sbin/rsyslogd -n -i/var/run/rsyslogd.pid
root     3933146 3485470  0 17:23 pts/0    00:00:00 grep --color=auto 1555

root@10.10.180.70[/var/lib]#systemctl status rsyslog
● rsyslog.service - System Logging Service
   Loaded: loaded (/usr/lib/systemd/system/rsyslog.service; enabled; vendor preset: enabled)
   Active: active (running) since Wed 2023-04-12 16:10:34 CST; 2 months 30 days ago
     Docs: man:rsyslogd(8)
           http://www.rsyslog.com/doc/
 Main PID: 1555 (rsyslogd)
    Tasks: 3
   Memory: 6.4G
   CGroup: /system.slice/rsyslog.service
           └─1555 /usr/sbin/rsyslogd -n -i/var/run/rsyslogd.pid

可以看到是 rsyslog 服务的日志占用导致，删除的日志文件句柄没有释放，停止 rsyslog 服务。

root@10.10.180.70[/var/lib]#systemctl stop rsyslog

查看分区是否已释放

root@10.10.180.70[/var/lib]#df -h | head -n 20
Filesystem                     Size  Used Avail Use% Mounted on
devtmpfs                        24G     0   24G   0% /dev
tmpfs                           24G     0   24G   0% /dev/shm
tmpfs                           24G  2.3G   22G  10% /run
tmpfs                           24G     0   24G   0% /sys/fs/cgroup
/dev/mapper/vg00-lvroot         15G   15G  950M  94% /
tmpfs                           24G  8.0K   24G   1% /tmp
/dev/sda1                     1014M  156M  859M  16% /boot
/dev/mapper/vgdata-lvzpaas      50G   12G   39G  24% /zpaas
/dev/mapper/vgdata-lvzlogs      15G  770M   15G   6% /tmp/zlogs
/dev/mapper/vgdata-lvzcm        15G  531M   15G   4% /zcm
/dev/mapper/vg00-lvvar         8.0G  346M  7.7G   5% /var
/dev/mapper/vg00-lvhome         15G  2.2G   13G  15% /home
/dev/mapper/vgdata-lvdocker     50G  9.3G   41G  19% /var/lib/docker
/dev/mapper/vgdata-lvkubelet    35G  283M   35G   1% /var/lib/kubelet
/dev/mapper/vgdata-lvzpaasssd   15G  2.6G   13G  18% /zpaasssd
/dev/mapper/vgdata-lvzcore      15G  140M   15G   1% /tmp/zcore
tmpfs                          4.7G     0  4.7G   0% /run/user/1803
overlay                         10G  8.0K   10G   1% /var/lib/docker/overlay2/5d649e849b7008a4191dc1d8d1ea91caf26060e63064806fba4f27fe0481f4cb/merged
overlay                         10G   12K   10G   1% /var/lib/docker/overlay2/6eacec6dd46da0545efe991153dc644172bdf6eb1a521afdddd521bd33f4fbd2/merged

---