记一次linux磁盘清理

Xshell 6 (Build 0170)
Copyright (c) 2002 NetSarang Computer, Inc. All rights reserved.

Type `help' to learn how to use Xshell prompt.
[C:\~]$ 

Connecting to 47.97.27.199:22...
Connection established.
To escape to local shell, press 'Ctrl+Alt+]'.

WARNING! The remote SSH server rejected X11 forwarding request.
Last login: Thu Apr  9 20:34:13 2020 from 124.79.221.3

Welcome to Alibaba Cloud Elastic Compute Service !

[root@izbp1c527y2uc9f7yez4g4z ~]# mysql
ERROR 2002 (HY000): Can't connect to local MySQL server through socket '/tmp/mysql.sock' (2)
[root@izbp1c527y2uc9f7yez4g4z ~]# df -h
Filesystem      Size  Used Avail Use% Mounted on
devtmpfs        909M     0  909M   0% /dev
tmpfs           919M     0  919M   0% /dev/shm
tmpfs           919M   90M  830M  10% /run
tmpfs           919M     0  919M   0% /sys/fs/cgroup
/dev/vda1        40G   40G     0 100% /
overlay          40G   40G     0 100% /var/lib/docker/overlay/69e1f2f3ad0db7bf298713a823b532cad530c5a89ef9cb5c7fe527ed95e349c3/merged
shm              64M     0   64M   0% /var/lib/docker/containers/0da03464277eff1c2e4b7505394a3c5f5d646c2fc64e225858ff8168bf7c582b/shm
tmpfs           184M     0  184M   0% /run/user/0
[root@izbp1c527y2uc9f7yez4g4z ~]# du -h --max-depth=1   - 未发现占用太大的文件与目录
100K    ./.cache
12K    ./.config
18M    ./.npminstall_tarball
8.0K    ./.docker
8.0K    ./.oracle_jre_usage
4.0K    ./.ssh
32M    ./.bundle
49M    ./.npm
8.0K    ./.pki
8.0K    ./.arthas
8.0K    ./.pip
4.2M    ./.gem
102M    .
[root@izbp1c527y2uc9f7yez4g4z ~]# pwd
/root
[root@izbp1c527y2uc9f7yez4g4z ~]# cd /
[root@izbp1c527y2uc9f7yez4g4z /]# du -h --max-depth=1
0    ./sys
0    ./dev
132K    ./tmp
102M    ./root
4.0K    ./opt
4.0K    ./media
du: cannot access ‘./proc/6775/task/6775/fd/4’: No such file or directory
du: cannot access ‘./proc/6775/task/6775/fdinfo/4’: No such file or directory
du: cannot access ‘./proc/6775/fd/3’: No such file or directory
du: cannot access ‘./proc/6775/fdinfo/3’: No such file or directory
0    ./proc
31G    ./var
90M    ./run
4.0K    ./srv
38M    ./etc
40K    ./home

^C
[root@izbp1c527y2uc9f7yez4g4z /]# ll
total 60
lrwxrwxrwx   1 root root     7 Oct 12  2019 bin -> usr/bin
dr-xr-xr-x.  5 root root  4096 Nov 27 02:13 boot
drwxr-xr-x  19 root root  2960 Nov 27 02:13 dev
drwxr-xr-x. 81 root root  4096 Apr  3 17:54 etc
drwxr-xr-x.  3 root root  4096 Nov 15 20:57 home
lrwxrwxrwx   1 root root     7 Oct 12  2019 lib -> usr/lib
lrwxrwxrwx   1 root root     9 Oct 12  2019 lib64 -> usr/lib64
drwx------.  2 root root 16384 Oct 15  2017 lost+found
drwxr-xr-x.  2 root root  4096 Apr 11  2018 media
drwxr-xr-x.  3 root root  4096 Nov 27 03:36 mnt
drwxr-xr-x.  2 root root  4096 Apr 11  2018 opt
dr-xr-xr-x  96 root root     0 Nov 27 02:12 proc
dr-xr-x---. 14 root root  4096 Apr  7 17:58 root
drwxr-xr-x  27 root root   860 Apr  3 15:52 run
lrwxrwxrwx   1 root root     8 Oct 12  2019 sbin -> usr/sbin
drwxr-xr-x.  2 root root  4096 Apr 11  2018 srv
dr-xr-xr-x  13 root root     0 Nov 27 04:03 sys
drwxrwxrwt.  9 root root  4096 Apr 19 13:27 tmp
drwxr-xr-x. 13 root root  4096 Apr  3 16:11 usr
drwxr-xr-x. 21 root root  4096 Oct 13  2019 var
[root@izbp1c527y2uc9f7yez4g4z /]# cd /var/   -- 进入占用最大的文件夹，进一步定位大文件位置
[root@izbp1c527y2uc9f7yez4g4z var]# du -h --max-depth=1
4.0K    ./yp
590M    ./log
56K    ./zookeeper
4.0K    ./crash
104K    ./spool
12K    ./tmp
4.0K    ./opt
4.0K    ./games
24K    ./db
113M    ./cache
4.0K    ./gopher
4.0K    ./nis
4.0K    ./zkdata
12K    ./kerberos
31G    ./lib
4.0K    ./preserve
4.0K    ./adm
4.0K    ./local
8.0K    ./empty
31G    .       -- 什么鬼？显然这种方式定位不准确，换个方式定位！！！ 使用 du -sh * 命令
[root@izbp1c527y2uc9f7yez4g4z var]# du -sh *
4.0K    adm
113M    cache
4.0K    crash
24K    db
8.0K    empty
4.0K    games
4.0K    gopher
12K    kerberos
31G    lib     -- 发现lib文件夹占用很大空间
4.0K    local
0    lock
590M    log
0    mail
4.0K    nis
4.0K    opt
4.0K    preserve
0    run
104K    spool
12K    tmp
4.0K    yp
4.0K    zkdata
56K    zookeeper
[root@izbp1c527y2uc9f7yez4g4z var]# ll
total 76
drwxr-xr-x.  2 root root 4096 Apr 11  2018 adm
drwxr-xr-x.  5 root root 4096 Apr 11  2018 cache
drwxr-xr-x.  2 root root 4096 Aug  8  2019 crash
drwxr-xr-x.  4 root root 4096 Oct 12  2019 db
drwxr-xr-x.  3 root root 4096 Apr 11  2018 empty
drwxr-xr-x.  2 root root 4096 Apr 11  2018 games
drwxr-xr-x.  2 root root 4096 Apr 11  2018 gopher
drwxr-xr-x.  3 root root 4096 Sep 14  2019 kerberos
drwxr-xr-x. 28 root root 4096 Nov 27 02:53 lib
drwxr-xr-x.  2 root root 4096 Apr 11  2018 local
lrwxrwxrwx.  1 root root   11 Oct 15  2017 lock -> ../run/lock
drwxr-xr-x. 11 root root 4096 Apr  5 03:32 log
lrwxrwxrwx   1 root root   10 Oct 12  2019 mail -> spool/mail
drwxr-xr-x.  2 root root 4096 Apr 11  2018 nis
drwxr-xr-x.  2 root root 4096 Apr 11  2018 opt
drwxr-xr-x.  2 root root 4096 Apr 11  2018 preserve
lrwxrwxrwx.  1 root root    6 Oct 15  2017 run -> ../run
drwxr-xr-x.  9 root root 4096 Apr 11  2018 spool
drwxrwxrwt.  3 root root 4096 Apr 16 02:53 tmp
drwxr-xr-x.  2 root root 4096 Apr 11  2018 yp
drwxr-xr-x   2 root root 4096 Oct 13  2019 zkdata
drwxr-xr-x   3 root root 4096 Oct 19  2019 zookeeper
[root@izbp1c527y2uc9f7yez4g4z var]# cd lib/      -- 继续深入定位
[root@izbp1c527y2uc9f7yez4g4z lib]# du -sh *
24K    alternatives
8.0K    authconfig
8.0K    chrony
236K    cloud
4.0K    dbus
8.0K    dhclient
30G    docker     -- 发现其中docker占用磁盘达到30G空间
4.0K    games
4.0K    initramfs
8.0K    logrotate
4.0K    machines
4.0K    misc
24K    NetworkManager
4.0K    ntp
4.0K    os-prober
8.0K    plymouth
28K    polkit-1
8.0K    postfix
121M    rpm
4.0K    rpm-state
8.0K    rsyslog
4.0K    selinux
12K    stateless
76K    systemd
4.0K    tuned
11M    yum
[root@izbp1c527y2uc9f7yez4g4z lib]# cd docker/   -- 继续定位
[root@izbp1c527y2uc9f7yez4g4z docker]# du -sh *
29G    containers    -- docker 容器文件夹占用达29G ! ! !
2.2M    image
52K    network
1.8G    overlay
20K    plugins
4.0K    swarm
4.0K    tmp
4.0K    trust
328K    volumes
[root@izbp1c527y2uc9f7yez4g4z docker]# cd containers/
[root@izbp1c527y2uc9f7yez4g4z containers]# du -sh *
29G    0da03464277eff1c2e4b7505394a3c5f5d646c2fc64e225858ff8168bf7c582b  --- 该容器占用 29 G磁盘空间
40K    0e2cb468f07e266ade54d49807c83c027d825735be961a05ba89a3887b4dff03
40K    1d957f19855b834fbe09deffa680f348b30d986fe17ebf2477bee8ce1cbf6216
40K    3443d4a679cf2b3dcf63b72f13d0184bd95a414a24ab79496cc462c17415335d
40K    39b0d098e54cea3bc4c82deb750e6c5ccb40b706ca093848b46c55a72543bc5a
40K    568d35d5dd08526f1d6815b6ef2a7c01bc2572f5767619590e71e2b04813b450
44K    779f0adea2f95b2e639eed605a68ad47b987abc770924e09da0cd6b7853f0669
44K    98dcfc668213755d608e724da385da53282fc8f83a52654c085651559b75e25f
40K    a02de99a1da5dd3b0872001b8bcb1c5ec3150c68fd4fa1272b9d948512e4c355
40K    c9b2c49254ec3a65460a9d050f039b5ab74066c982b0275da19e946e4049e26b
28K    cba5352dc77fd2420582fd3e39d74c4ea185ac72520468542b55c3f39e8eaf2f
28K    d21bd9b988a335504cda370e9d683670c8801c9f9c6f7323f7f9fe54002efa5a
44K    dfa7e455c8b6b87613a33e76df77316cdd3a0ad23a9d6545cb87517a1c6efa47
44K    f6cbae71c3dc492c6951260420e89402de7ba70370fd6a314c8b4a3a55467c50
[root@izbp1c527y2uc9f7yez4g4z containers]# docker ps -a   -- 4个月以前的容器
CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS                      PORTS                                            NAMES
0da03464277e        582b08f396ca        "java -Djava.secur..."   4 months ago        Up 4 months                 0.0.0.0:1099->1099/tcp, 0.0.0.0:8082->8082/tcp   kind_wing
0e2cb468f07e        feb71ca4264f        "java $JAVA_OPTS -..."   4 months ago        Exited (1) 4 months ago                                                      goofy_jang
39b0d098e54c        cd9bbfa896bb        "java ${JAVA_OPTS}..."   4 months ago        Exited (1) 4 months ago                                                      brave_noyce
d21bd9b988a3        cd9bbfa896bb        "java ${JAVA_OPTS}..."   4 months ago        Created                                                                      youthful_morse
cba5352dc77f        cd9bbfa896bb        "java ${JAVA_OPTS}..."   4 months ago        Created                                                                      lucid_carson
98dcfc668213        bf675a1279e7        "java -Djava.secur..."   4 months ago        Exited (143) 4 months ago                                                    unruffled_babbage
779f0adea2f9        bf675a1279e7        "java -Djava.secur..."   4 months ago        Exited (143) 4 months ago                                                    nervous_fermi
f6cbae71c3dc        585287856ed3        "java -Djava.secur..."   4 months ago        Exited (143) 4 months ago                                                    nostalgic_shannon
dfa7e455c8b6        37e171ae2021        "java -Djava.secur..."   4 months ago        Exited (143) 4 months ago                                                    sad_aryabhata
c9b2c49254ec        a317e91f144e        "java -Djava.secur..."   4 months ago        Exited (1) 4 months ago                                                      epic_kalam
3443d4a679cf        6a42529b201f        "java -jar '/app.j..."   4 months ago        Exited (1) 4 months ago                                                      elated_kirch
a02de99a1da5        6a42529b201f        "java -jar '/app.j..."   4 months ago        Exited (1) 4 months ago                                                      suspicious_wilson
1d957f19855b        34880a18e97a        "java -Djava.secur..."   4 months ago        Exited (1) 4 months ago                                                      goofy_carson
568d35d5dd08        hello-world         "/hello"                 4 months ago        Exited (0) 4 months ago                                                      angry_mccarthy
[root@izbp1c527y2uc9f7yez4g4z containers]# systemctl status docker  -- 查看docker状态，避免删除正在运行中使用的文件
● docker.service - Docker Application Container Engine
   Loaded: loaded (/etc/systemd/system/docker.service; enabled; vendor preset: disabled)
   Active: active (running) since Wed 2019-11-27 07:37:31 CST; 4 months 22 days ago
     Docs: https://docs.docker.com
 Main PID: 29551 (dockerd)
    Tasks: 43
   Memory: 61.1M
   CGroup: /system.slice/docker.service
           ├─10617 docker-containerd -l unix:///var/run/docker/libcontainerd/docker-containerd.sock --metrics-interval=0 --start-timeout 2m --state-dir /var/run/docker/libcontainerd/containerd...
           ├─29551 /usr/bin/dockerd
           ├─31037 /usr/bin/docker-proxy -proto tcp -host-ip 0.0.0.0 -host-port 8082 -container-ip 172.17.0.2 -container-port 8082
           ├─31047 /usr/bin/docker-proxy -proto tcp -host-ip 0.0.0.0 -host-port 1099 -container-ip 172.17.0.2 -container-port 1099
           └─31051 docker-containerd-shim 0da03464277eff1c2e4b7505394a3c5f5d646c2fc64e225858ff8168bf7c582b /var/run/docker/libcontainerd/0da03464277eff1c2e4b7505394a3c5f5d646c2fc64e225858ff816...

Apr 19 14:03:48 izbp1c527y2uc9f7yez4g4z dockerd[29551]: time="2020-04-19T14:03:48.168703222+08:00" level=error msg="Failed to log msg \"\" for logger json-file: write /var/lib/docker/container...
Apr 19 14:03:48 izbp1c527y2uc9f7yez4g4z dockerd[29551]: time="2020-04-19T14:03:48.168729512+08:00" level=error msg="Failed to log msg \"com.mysql.cj.jdbc.exceptions.CommunicationsException: Co...
Apr 19 14:03:48 izbp1c527y2uc9f7yez4g4z dockerd[29551]: time="2020-04-19T14:03:48.168754458+08:00" level=error msg="Failed to log msg \"\" for logger json-file: write /var/lib/docker/container...
Apr 19 14:03:48 izbp1c527y2uc9f7yez4g4z dockerd[29551]: time="2020-04-19T14:03:48.168780574+08:00" level=error msg="Failed to log msg \"The last packet sent successfully to the server was 0 mi...
Apr 19 14:03:48 izbp1c527y2uc9f7yez4g4z dockerd[29551]: time="2020-04-19T14:03:48.168807526+08:00" level=error msg="Failed to log msg \"\\tat com.mysql.cj.jdbc.exceptions.SQLError.createCommun...
Apr 19 14:03:48 izbp1c527y2uc9f7yez4g4z dockerd[29551]: time="2020-04-19T14:03:48.168835260+08:00" level=error msg="Failed to log msg \"\\tat com.mysql.cj.jdbc.exceptions.SQLExceptionsMapping....
Apr 19 14:03:48 izbp1c527y2uc9f7yez4g4z dockerd[29551]: time="2020-04-19T14:03:48.168862515+08:00" level=error msg="Failed to log msg \"\\tat com.mysql.cj.jdbc.ConnectionImpl.createNewIO(Conne...
Apr 19 14:03:48 izbp1c527y2uc9f7yez4g4z dockerd[29551]: time="2020-04-19T14:03:48.168909924+08:00" level=error msg="Failed to log msg \"\\tat com.mysql.cj.jdbc.ConnectionImpl.<init>(Connection...
Apr 19 14:03:48 izbp1c527y2uc9f7yez4g4z dockerd[29551]: time="2020-04-19T14:03:48.168940986+08:00" level=error msg="Failed to log msg \"\\tat com.mysql.cj.jdbc.ConnectionImpl.getInstance(Conne...
Apr 19 14:03:48 izbp1c527y2uc9f7yez4g4z dockerd[29551]: time="2020-04-19T14:03:48.168971722+08:00" level=error msg="Failed to log msg \"\\tat com.mysql.cj.jdbc.NonRegisteringDriver.connect(Non...
Hint: Some lines were ellipsized, use -l to show in full.
[root@izbp1c527y2uc9f7yez4g4z containers]# docker container prune  --- 清理容器，删除关闭的容器
WARNING! This will remove all stopped containers.
Are you sure you want to continue? [y/N] y
Deleted Containers:
0e2cb468f07e266ade54d49807c83c027d825735be961a05ba89a3887b4dff03
39b0d098e54cea3bc4c82deb750e6c5ccb40b706ca093848b46c55a72543bc5a
d21bd9b988a335504cda370e9d683670c8801c9f9c6f7323f7f9fe54002efa5a
cba5352dc77fd2420582fd3e39d74c4ea185ac72520468542b55c3f39e8eaf2f
98dcfc668213755d608e724da385da53282fc8f83a52654c085651559b75e25f
779f0adea2f95b2e639eed605a68ad47b987abc770924e09da0cd6b7853f0669
f6cbae71c3dc492c6951260420e89402de7ba70370fd6a314c8b4a3a55467c50
dfa7e455c8b6b87613a33e76df77316cdd3a0ad23a9d6545cb87517a1c6efa47
c9b2c49254ec3a65460a9d050f039b5ab74066c982b0275da19e946e4049e26b
3443d4a679cf2b3dcf63b72f13d0184bd95a414a24ab79496cc462c17415335d
a02de99a1da5dd3b0872001b8bcb1c5ec3150c68fd4fa1272b9d948512e4c355
1d957f19855b834fbe09deffa680f348b30d986fe17ebf2477bee8ce1cbf6216
568d35d5dd08526f1d6815b6ef2a7c01bc2572f5767619590e71e2b04813b450

Total reclaimed space: 0 B
[root@izbp1c527y2uc9f7yez4g4z containers]# docker system df  -- 查看docker对磁盘的使用情况
TYPE                TOTAL               ACTIVE              SIZE                RECLAIMABLE
Images              3                   1                   708.6 MB            643.1 MB (90%)
Containers          1                   1                   0 B                 0 B
Local Volumes       13                  1                   32.77 kB            0 B (0%)
[root@izbp1c527y2uc9f7yez4g4z containers]# df -h   -- 查看目前系统磁盘占用情况
Filesystem      Size  Used Avail Use% Mounted on
devtmpfs        909M     0  909M   0% /dev
tmpfs           919M     0  919M   0% /dev/shm
tmpfs           919M   99M  821M  11% /run
tmpfs           919M     0  919M   0% /sys/fs/cgroup
/dev/vda1        40G   39G     0 100% /    --- 仅仅释放了 1G左右的空间，显然还远远达不到清理目标
overlay          40G   39G     0 100% /var/lib/docker/overlay/69e1f2f3ad0db7bf298713a823b532cad530c5a89ef9cb5c7fe527ed95e349c3/merged
shm              64M     0   64M   0% /var/lib/docker/containers/0da03464277eff1c2e4b7505394a3c5f5d646c2fc64e225858ff8168bf7c582b/shm
tmpfs           184M     0  184M   0% /run/user/0
[root@izbp1c527y2uc9f7yez4g4z containers]# docker --version
Docker version 17.03.0-ce, build 3a232c8
[root@izbp1c527y2uc9f7yez4g4z containers]# docker info
Containers: 1
 Running: 1
 Paused: 0
 Stopped: 0
Images: 9
...
[root@izbp1c527y2uc9f7yez4g4z containers]# docker ps
CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS                                            NAMES
0da03464277e        582b08f396ca        "java -Djava.secur..."   4 months ago        Up 4 months         0.0.0.0:1099->1099/tcp, 0.0.0.0:8082->8082/tcp   kind_wing
[root@izbp1c527y2uc9f7yez4g4z containers]# du -sh *
29G    0da03464277eff1c2e4b7505394a3c5f5d646c2fc64e225858ff8168bf7c582b   --- 继续寻找清理目标
[root@izbp1c527y2uc9f7yez4g4z containers]# cd 0da03464277eff1c2e4b7505394a3c5f5d646c2fc64e225858ff8168bf7c582b/
[root@izbp1c527y2uc9f7yez4g4z 0da03464277eff1c2e4b7505394a3c5f5d646c2fc64e225858ff8168bf7c582b]# du -sh *
29G    0da03464277eff1c2e4b7505394a3c5f5d646c2fc64e225858ff8168bf7c582b-json.log  -- 一个占用29G的日志文件
4.0K    checkpoints
4.0K    config.v2.json
4.0K    hostconfig.json
4.0K    hostname
4.0K    hosts
4.0K    resolv.conf
4.0K    resolv.conf.hash
0    shm
[root@izbp1c527y2uc9f7yez4g4z 0da03464277eff1c2e4b7505394a3c5f5d646c2fc64e225858ff8168bf7c582b]# du -sh *
29G    0da03464277eff1c2e4b7505394a3c5f5d646c2fc64e225858ff8168bf7c582b-json.log
4.0K    checkpoints
4.0K    config.v2.json
4.0K    hostconfig.json
4.0K    hostname
4.0K    hosts
4.0K    resolv.conf
4.0K    resolv.conf.hash
0    shm
4.0K    toTemp
[root@izbp1c527y2uc9f7yez4g4z 0da03464277eff1c2e4b7505394a3c5f5d646c2fc64e225858ff8168bf7c582b]# pwd  -- 定位
/var/lib/docker/containers/0da03464277eff1c2e4b7505394a3c5f5d646c2fc64e225858ff8168bf7c582b
[root@izbp1c527y2uc9f7yez4g4z 0da03464277eff1c2e4b7505394a3c5f5d646c2fc64e225858ff8168bf7c582b]# cat /dev/null > /var/lib/docker/containers/0da03464277eff1c2e4b7505394a3c5f5d646c2fc64e225858ff8168bf7c582b/0da03464277eff1c2e4b7505394a3c5f5d646c2fc64e225858ff8168bf7c582b-json.log   -- 删除该占用达29G的日志文件
[root@izbp1c527y2uc9f7yez4g4z 0da03464277eff1c2e4b7505394a3c5f5d646c2fc64e225858ff8168bf7c582b]# 
[root@izbp1c527y2uc9f7yez4g4z 0da03464277eff1c2e4b7505394a3c5f5d646c2fc64e225858ff8168bf7c582b]# df -h
Filesystem      Size  Used Avail Use% Mounted on
devtmpfs        909M     0  909M   0% /dev
tmpfs           919M     0  919M   0% /dev/shm
tmpfs           919M  656K  919M   1% /run
tmpfs           919M     0  919M   0% /sys/fs/cgroup
/dev/vda1        40G   11G   27G  30% /    -- 可用空间达到 27G，清理目标达成 ，可喜可贺 ！！！
overlay          40G   11G   27G  30% /var/lib/docker/overlay/69e1f2f3ad0db7bf298713a823b532cad530c5a89ef9cb5c7fe527ed95e349c3/merged
shm              64M     0   64M   0% /var/lib/docker/containers/0da03464277eff1c2e4b7505394a3c5f5d646c2fc64e225858ff8168bf7c582b/shm
tmpfs           184M     0  184M   0% /run/user/0
[root@izbp1c527y2uc9f7yez4g4z 0da03464277eff1c2e4b7505394a3c5f5d646c2fc64e225858ff8168bf7c582b]# 

 

总结：

        对于docker的使用，需要制定合适的日志策略，否则很容造成磁盘空间的大量占用，甚至占满，从而导致对外应用服务的故障与性能下降等问题。

docker清理问题请参考：http://dockone.io/article/3056