df -h 看到根目录下磁盘使用率很高:
[root@pokepzs01 /]# df -h
Filesystem Size Used Avail Use% Mounted on
/dev/vda1 90G 72G 13G 86% /
devtmpfs 3.9G 0 3.9G 0% /dev
tmpfs 3.9G 0 3.9G 0% /dev/shm
tmpfs 3.9G 409M 3.5G 11% /run
tmpfs 3.9G 0 3.9G 0% /sys/fs/cgroup
/dev/vdb 200G 30G 171G 15% /data
tmpfs 783M 0 783M 0% /run/user/0
du -sh /查看根目录占用情况:
[root@pokepzs01 /]# du -sh *
976K beanstalkd
0 bin
97M boot
0 cache
29G data
0 dev
0 diskcache
4.0K download
39M etc
0 file
20K home
0 lib
0 lib64
16K lost+found
4.0K media
0 memcache
4.0K mnt
1.4G opt
0 play
du: cannot access ‘proc/27646’: No such file or directory
du: cannot access ‘proc/27647’: No such file or directory
du: cannot access ‘proc/27648’: No such file or directory
du: cannot access ‘proc/27654/task/27654/fd/4’: No such file or directory
du: cannot access ‘proc/27654/task/27654/fdinfo/4’: No such file or directory
du: cannot access ‘proc/27654/fd/4’: No such file or directory
du: cannot access ‘proc/27654/fdinfo/4’: No such file or directory
0 proc
8.9G python
0 pztemp
75M redis
2.5G root
409M run
0 sbin
4.0K srv
0 storage
0 sys
100K tmp
4.4G usr
814M var
两者所查到的空间使用情况相差巨大。
分析原因:
可能是系统进程占用的空间没有释放掉,导致的。
解决方法:
通过命令lsof | grep deleted 获取对已删除文件的读写操作的进程id,
通过重启该进程或者kill掉该进程来释放所占磁盘空间,
通过ps -ef |grep id ,查看该进程id是否存在,不存在即可kill掉,存在可重启该应用。
[root@pokepzs01 /]# lsof -n / |grep deleted
systemd-j 363 root txt REG 253,1 278808 3674439 /usr/lib/systemd/systemd-journald;5c4904e8 (deleted)
systemd-l 528 root txt REG 253,1 584560 3674441 /usr/lib/systemd/systemd-logind;5c4904e8 (deleted)
zabbix_ag 1524 zabbix 1w REG 253,1 50142 5242936 /var/log/zabbix/zabbix_agentd.log-20190127 (deleted)
zabbix_ag 1524 zabbix 2w REG 253,1 50142 5242936 /var/log/zabbix/zabbix_agentd.log-20190127 (deleted)
zabbix_ag 1547 zabbix 1w REG 253,1 50142 5242936 /var/log/zabbix/zabbix_agentd.log-20190127 (deleted)
zabbix_ag 1547 zabbix 2w REG 253,1 50142 5242936 /var/log/zabbix/zabbix_agentd.log-20190127 (deleted)
zabbix_ag 1549 zabbix 1w REG 253,1 50142 5242936 /var/log/zabbix/zabbix_agentd.log-20190127 (deleted)
zabbix_ag 1549 zabbix 2w REG 253,1 50142 5242936 /var/log/zabbix/zabbix_agentd.log-20190127 (deleted)
zabbix_ag 1551 zabbix 1w REG 253,1 50142 5242936 /var/log/zabbix/zabbix_agentd.log-20190127 (deleted)
zabbix_ag 1551 zabbix 2w REG 253,1 50142 5242936 /var/log/zabbix/zabbix_agentd.log-20190127 (deleted)
zabbix_ag 1553 zabbix 1w REG 253,1 50142 5242936 /var/log/zabbix/zabbix_agentd.log-20190127 (deleted)
zabbix_ag 1553 zabbix 2w REG 253,1 50142 5242936 /var/log/zabbix/zabbix_agentd.log-20190127 (deleted)
zabbix_ag 1554 zabbix 1w REG 253,1 50142 5242936 /var/log/zabbix/zabbix_agentd.log-20190127 (deleted)
zabbix_ag 1554 zabbix 2w REG 253,1 50142 5242936 /var/log/zabbix/zabbix_agentd.log-20190127 (deleted)
python3.5 1758 root 7u REG 253,1 0 2228259 /tmp/wgunicorn-jylwb129 (deleted)
python3.5 1758 root 11w REG 253,1 0 3145790 /python/iroom_server/log-bak/iroom_server.lock (deleted)
python3.5 1759 root 8u REG 253,1 0 2228260 /tmp/wgunicorn-7wjgoib5 (deleted)
python3.5 1759 root 11w REG 253,1 0 3145790 /python/iroom_server/log-bak/iroom_server.lock (deleted)
python3.5 1760 root 9u REG 253,1 0 2228261 /tmp/wgunicorn-70_89i59 (deleted)
python3.5 1760 root 11w REG 253,1 0 3145790 /python/iroom_server/log-bak/iroom_server.lock (deleted)
python3.5 1761 root 10u REG 253,1 0 2228262 /tmp/wgunicorn-_h09cueh (deleted)
python3.5 1761 root 11w REG 253,1 0 3145790 /python/iroom_server/log-bak/iroom_server.lock (deleted)
python3.5 1762 root 10w REG 253,1 0 3145790 /python/iroom_server/log-bak/iroom_server.lock (deleted)
python3.5 1762 root 11u REG 253,1 0 2228263 /tmp/wgunicorn-0j__38j3 (deleted)
python3.5 1763 root 10w REG 253,1 0 3145790 /python/iroom_server/log-bak/iroom_server.lock (deleted)
python3.5 1763 root 12u REG 253,1 0 2228264 /tmp/wgunicorn-5jxfw18_ (deleted)
python3.5 1764 root 10w REG 253,1 0 3145790 /python/iroom_server/log-bak/iroom_server.lock (deleted)
python3.5 1764 root 13u REG 253,1 0 2228265 /tmp/wgunicorn-4amxt2k_ (deleted)
python3.5 1765 root 10w REG 253,1 0 3145790 /python/iroom_server/log-bak/iroom_server.lock (deleted)
python3.5 1765 root 14u REG 253,1 0 2228266 /tmp/wgunicorn-eq70eh22 (deleted)
python3.5 1766 root 10w REG 253,1 0 3145790 /python/iroom_server/log-bak/iroom_server.lock (deleted)
python3.5 1766 root 15u REG 253,1 0 2228267 /tmp/wgunicorn-7j4b5uoh (deleted)
mysqld 12555 mysql 4u REG 253,1 0 2228232 /tmp/ibE7Yibx (deleted)
mysqld 12555 mysql 5u REG 253,1 0 2228237 /tmp/ibPvkCpI (deleted)
mysqld 12555 mysql 6u REG 253,1 0 2228238 /tmp/ibEsPVDT (deleted)
mysqld 12555 mysql 7u REG 253,1 0 2228239 /tmp/ibnLDt8f (deleted)
mysqld 12555 mysql 11u REG 253,1 0 2228240 /tmp/ibi8iCpr (deleted)
python3.5 13470 root 7u REG 253,1 0 2228259 /tmp/wgunicorn-jylwb129 (deleted)
python3.5 13470 root 8u REG 253,1 0 2228260 /tmp/wgunicorn-7wjgoib5 (deleted)
python3.5 13470 root 9u REG 253,1 0 2228261 /tmp/wgunicorn-70_89i59 (deleted)
python3.5 13470 root 10u REG 253,1 0 2228262 /tmp/wgunicorn-_h09cueh (deleted)
python3.5 13470 root 11u REG 253,1 0 2228263 /tmp/wgunicorn-0j__38j3 (deleted)
python3.5 13470 root 12u REG 253,1 0 2228264 /tmp/wgunicorn-5jxfw18_ (deleted)
python3.5 13470 root 13u REG 253,1 0 2228265 /tmp/wgunicorn-4amxt2k_ (deleted)
python3.5 13470 root 14u REG 253,1 0 2228266 /tmp/wgunicorn-eq70eh22 (deleted)
python3.5 13470 root 15u REG 253,1 0 2228267 /tmp/wgunicorn-7j4b5uoh (deleted)
gunicorn 17141 root 7u REG 253,1 0 2228235 /tmp/wgunicorn-jzok5osz (deleted)
gunicorn 17141 root 8u REG 253,1 0 2228241 /tmp/wgunicorn-ri65ypno (deleted)
gunicorn 17141 root 9u REG 253,1 0 2228242 /tmp/wgunicorn-tzpfqyxa (deleted)
gunicorn 17141 root 10u REG 253,1 0 2228243 /tmp/wgunicorn-2v1tjuob (deleted)
gunicorn 17141 root 11u REG 253,1 0 2228244 /tmp/wgunicorn-wtyo4088 (deleted)
gunicorn 17141 root 12u REG 253,1 0 2228245 /tmp/wgunicorn-pk4arkms (deleted)
gunicorn 17141 root 13u REG 253,1 0 2228246 /tmp/wgunicorn-z0jn8orj (deleted)
gunicorn 17141 root 14u REG 253,1 0 2228247 /tmp/wgunicorn-hikdrrih (deleted)
gunicorn 17141 root 15u REG 253,1 0 2228248 /tmp/wgunicorn-v0ttlh2_ (deleted)
gunicorn 17141 root 16u REG 253,1 0 2228249 /tmp/wgunicorn-2scepzy0 (deleted)
gunicorn 17158 root 7u REG 253,1 0 2228235 /tmp/wgunicorn-jzok5osz (deleted)
gunicorn 17168 root 8u REG 253,1 0 2228241 /tmp/wgunicorn-ri65ypno (deleted)
gunicorn 17171 root 9u REG 253,1 0 2228242 /tmp/wgunicorn-tzpfqyxa (deleted)
gunicorn 17174 root 10u REG 253,1 0 2228243 /tmp/wgunicorn-2v1tjuob (deleted)
gunicorn 17180 root 11u REG 253,1 0 2228244 /tmp/wgunicorn-wtyo4088 (deleted)
gunicorn 17182 root 12u REG 253,1 0 2228245 /tmp/wgunicorn-pk4arkms (deleted)
gunicorn 17184 root 13u REG 253,1 0 2228246 /tmp/wgunicorn-z0jn8orj (deleted)
gunicorn 17185 root 14u REG 253,1 0 2228247 /tmp/wgunicorn-hikdrrih (deleted)
gunicorn 17187 root 15u REG 253,1 0 2228248 /tmp/wgunicorn-v0ttlh2_ (deleted)
gunicorn 17189 root 16u REG 253,1 0 2228249 /tmp/wgunicorn-2scepzy0 (deleted)
java 24363 root 49w REG 253,1 57653101735 263451 /root/resin-4.0.48/log/access.log (deleted)
sz 29622 root cwd DIR 253,1 0 5636106 /storage-bak/new_upload_file/pokematch/2019-01-24 (deleted)
输出上述各个进程的进程号和所占空间的大小:
[root@pokepzs01 ~]# lsof -n / |grep deleted|awk '{print $2,$7,$9}'
363 278808 /usr/lib/systemd/systemd-journald;5c4904e8
528 584560 /usr/lib/systemd/systemd-logind;5c4904e8
1524 50142 /var/log/zabbix/zabbix_agentd.log-20190127
1524 50142 /var/log/zabbix/zabbix_agentd.log-20190127
1547 50142 /var/log/zabbix/zabbix_agentd.log-20190127
1547 50142 /var/log/zabbix/zabbix_agentd.log-20190127
1549 50142 /var/log/zabbix/zabbix_agentd.log-20190127
1549 50142 /var/log/zabbix/zabbix_agentd.log-20190127
1551 50142 /var/log/zabbix/zabbix_agentd.log-20190127
1551 50142 /var/log/zabbix/zabbix_agentd.log-20190127
1553 50142 /var/log/zabbix/zabbix_agentd.log-20190127
1553 50142 /var/log/zabbix/zabbix_agentd.log-20190127
1554 50142 /var/log/zabbix/zabbix_agentd.log-20190127
1554 50142 /var/log/zabbix/zabbix_agentd.log-20190127
1758 0 /tmp/wgunicorn-jylwb129
1758 0 /python/iroom_server/log-bak/iroom_server.lock
1759 0 /tmp/wgunicorn-7wjgoib5
1759 0 /python/iroom_server/log-bak/iroom_server.lock
1760 0 /tmp/wgunicorn-70_89i59
1760 0 /python/iroom_server/log-bak/iroom_server.lock
1761 0 /tmp/wgunicorn-_h09cueh
1761 0 /python/iroom_server/log-bak/iroom_server.lock
1762 0 /python/iroom_server/log-bak/iroom_server.lock
1762 0 /tmp/wgunicorn-0j__38j3
1763 0 /python/iroom_server/log-bak/iroom_server.lock
1763 0 /tmp/wgunicorn-5jxfw18_
1764 0 /python/iroom_server/log-bak/iroom_server.lock
1764 0 /tmp/wgunicorn-4amxt2k_
1765 0 /python/iroom_server/log-bak/iroom_server.lock
1765 0 /tmp/wgunicorn-eq70eh22
1766 0 /python/iroom_server/log-bak/iroom_server.lock
1766 0 /tmp/wgunicorn-7j4b5uoh
12555 0 /tmp/ibE7Yibx
12555 0 /tmp/ibPvkCpI
12555 0 /tmp/ibEsPVDT
12555 0 /tmp/ibnLDt8f
12555 0 /tmp/ibi8iCpr
13470 0 /tmp/wgunicorn-jylwb129
13470 0 /tmp/wgunicorn-7wjgoib5
13470 0 /tmp/wgunicorn-70_89i59
13470 0 /tmp/wgunicorn-_h09cueh
13470 0 /tmp/wgunicorn-0j__38j3
13470 0 /tmp/wgunicorn-5jxfw18_
13470 0 /tmp/wgunicorn-4amxt2k_
13470 0 /tmp/wgunicorn-eq70eh22
13470 0 /tmp/wgunicorn-7j4b5uoh
17141 0 /tmp/wgunicorn-jzok5osz
17141 0 /tmp/wgunicorn-ri65ypno
17141 0 /tmp/wgunicorn-tzpfqyxa
17141 0 /tmp/wgunicorn-2v1tjuob
17141 0 /tmp/wgunicorn-wtyo4088
17141 0 /tmp/wgunicorn-pk4arkms
17141 0 /tmp/wgunicorn-z0jn8orj
17141 0 /tmp/wgunicorn-hikdrrih
17141 0 /tmp/wgunicorn-v0ttlh2_
17141 0 /tmp/wgunicorn-2scepzy0
17158 0 /tmp/wgunicorn-jzok5osz
17168 0 /tmp/wgunicorn-ri65ypno
17171 0 /tmp/wgunicorn-tzpfqyxa
17174 0 /tmp/wgunicorn-2v1tjuob
17180 0 /tmp/wgunicorn-wtyo4088
17182 0 /tmp/wgunicorn-pk4arkms
17184 0 /tmp/wgunicorn-z0jn8orj
17185 0 /tmp/wgunicorn-hikdrrih
17187 0 /tmp/wgunicorn-v0ttlh2_
17189 0 /tmp/wgunicorn-2scepzy0
24363 57655094353 /root/resin-4.0.48/log/access.log
29622 0 /storage-bak/new_upload_file/pokematch/2019-01-24
可以看到“/root/resin-4.0.48/log/access.log”占空间最大,约53G。
重启resin进程即可。