linux df命令无反馈,Linux df 命令长时间没有返回

今天有个报警是：

执行结果=critical : "df -h" process number is more then 5 | 脚本监控告警：请根据提示检查相关项 [contain critica

登录主机查看df-h 发现卡住很久，于是mount一下看到有个nfs卷

ip地址替换成*了，保密

*.*.*.*:/data02 on /mnt type nfs (rw,vers=4,addr=*.*.*.*,clientaddr=*.*.*.*)

于是百度了一波可能是nfs卷卸载异常了。但是同事告诉我用

strace df -h

正常是不会卡主的，异常情况可以看到卡在哪个卷那里

[logmonitor@localhost ~]$ strace df -h

execve("/bin/df", ["df", "-h"], [/* 27 vars */]) = 0

brk(0) = 0x1c52000

mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff0f0c7d000

access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)

open("/etc/ld.so.cache", O_RDONLY) = 3

fstat(3, {st_mode=S_IFREG|0644, st_size=46602, ...}) = 0

mmap(NULL, 46602, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff0f0c71000

close(3) = 0

open("/lib64/libc.so.6", O_RDONLY) = 3

read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0000\356\201\2233\0\0\0"..., 832) = 832

fstat(3, {st_mode=S_IFREG|0755, st_size=1928936, ...}) = 0

mmap(0x3393800000, 3750184, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3393800000

mprotect(0x339398a000, 2097152, PROT_NONE) = 0

mmap(0x3393b8a000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x18a000) = 0x3393b8a000

mmap(0x3393b90000, 14632, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x3393b90000

close(3) = 0

mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff0f0c70000

mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff0f0c6f000

mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff0f0c6e000

arch_prctl(ARCH_SET_FS, 0x7ff0f0c6f700) = 0

mprotect(0x3393b8a000, 16384, PROT_READ) = 0

mprotect(0x339321f000, 4096, PROT_READ) = 0

munmap(0x7ff0f0c71000, 46602) = 0

brk(0) = 0x1c52000

brk(0x1c73000) = 0x1c73000

open("/usr/lib/locale/locale-archive", O_RDONLY) = 3

fstat(3, {st_mode=S_IFREG|0644, st_size=99164480, ...}) = 0

mmap(NULL, 99164480, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff0eaddb000

close(3) = 0

open("/etc/mtab", O_RDONLY) = 3

fstat(3, {st_mode=S_IFREG|0644, st_size=795, ...}) = 0

mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff0f0c7c000

read(3, "/dev/mapper/VolGroup00-LVroot / "..., 4096) = 795

read(3, "", 4096) = 0

close(3) = 0

munmap(0x7ff0f0c7c000, 4096) = 0

statfs("/", {f_type="EXT2_SUPER_MAGIC", f_bsize=4096, f_blocks=7708159, f_bfree=7114459, f_bavail=6721243, f_files=1966080, f_ffree=1883060, f_fsid={-776585995, 1108717655}, f_namelen=255, f_frsize=4096}) = 0

open("/usr/share/locale/locale.alias", O_RDONLY) = 3

fstat(3, {st_mode=S_IFREG|0644, st_size=2512, ...}) = 0

mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff0f0c7c000

read(3, "# Locale name alias data base.\n#"..., 4096) = 2512

read(3, "", 4096) = 0

close(3) = 0

munmap(0x7ff0f0c7c000, 4096) = 0

open("/usr/share/locale/en_US.UTF-8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)

open("/usr/share/locale/en_US.utf8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)

open("/usr/share/locale/en_US/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)

open("/usr/share/locale/en.UTF-8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)

open("/usr/share/locale/en.utf8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)

open("/usr/share/locale/en/LC_MESSAGES/coreutils.mo", O_RDONLY) = 3

fstat(3, {st_mode=S_IFREG|0644, st_size=435, ...}) = 0

mmap(NULL, 435, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff0f0c7c000

close(3) = 0

statfs("/sys", {f_type="SYSFS_MAGIC", f_bsize=4096, f_blocks=0, f_bfree=0, f_bavail=0, f_files=0, f_ffree=0, f_fsid={0, 0}, f_namelen=255, f_frsize=4096}) = 0

statfs("/dev/pts", {f_type="DEVPTS_SUPER_MAGIC", f_bsize=4096, f_blocks=0, f_bfree=0, f_bavail=0, f_files=0, f_ffree=0, f_fsid={0, 0}, f_namelen=255, f_frsize=4096}) = 0

statfs("/dev/shm", {f_type=0x1021994, f_bsize=4096, f_blocks=33052438, f_bfree=33052438, f_bavail=33052438, f_files=33052438, f_ffree=33052437, f_fsid={0, 0}, f_namelen=255, f_frsize=4096}) = 0

statfs("/boot", {f_type="EXT2_SUPER_MAGIC", f_bsize=4096, f_blocks=249830, f_bfree=240021, f_bavail=226914, f_files=65536, f_ffree=65497, f_fsid={-116806949, 885015568}, f_namelen=255, f_frsize=4096}) = 0

statfs("/home", {f_type="EXT2_SUPER_MAGIC", f_bsize=4096, f_blocks=7708159, f_bfree=7696866, f_bavail=7303650, f_files=1966080, f_ffree=1966024, f_fsid={236779102, 105200375}, f_namelen=255, f_frsize=4096}) = 0

statfs("/kdump", {f_type="EXT2_SUPER_MAGIC", f_bsize=4096, f_blocks=12868767, f_bfree=12855499, f_bavail=12200139, f_files=3276800, f_ffree=3276787, f_fsid={-605602565, -1856834956}, f_namelen=255, f_frsize=4096}) = 0

statfs("/tmp", {f_type="EXT2_SUPER_MAGIC", f_bsize=4096, f_blocks=5127839, f_bfree=5116565, f_bavail=4854421, f_files=1310720, f_ffree=1310686, f_fsid={560414392, 1857271414}, f_namelen=255, f_frsize=4096}) = 0

statfs("/var", {f_type="EXT2_SUPER_MAGIC", f_bsize=4096, f_blocks=5127839, f_bfree=4949767, f_bavail=4687623, f_files=1310720, f_ffree=1308530, f_fsid={989462255, 1576435300}, f_namelen=255, f_frsize=4096}) = 0

statfs("/proc/sys/fs/binfmt_misc", {f_type=0x42494e4d, f_bsize=4096, f_blocks=0, f_bfree=0, f_bavail=0, f_files=0, f_ffree=0, f_fsid={0, 0}, f_namelen=255, f_frsize=4096}) = 0

statfs("/dba/app", {f_type="EXT2_SUPER_MAGIC", f_bsize=4096, f_blocks=7708159, f_bfree=6382705, f_bavail=5989489, f_files=1966080, f_ffree=1965611, f_fsid={1799800290, 416388298}, f_namelen=255, f_frsize=4096}) = 0

statfs("/mysql_tmp", {f_type="EXT2_SUPER_MAGIC", f_bsize=4096, f_blocks=77376512, f_bfree=77360426, f_bavail=73428266, f_files=19660800, f_ffree=19660789, f_fsid={-727106197, -1600548577}, f_namelen=255, f_frsize=4096}) = 0

statfs("/mysql_backup", {f_type="EXT2_SUPER_MAGIC", f_bsize=4096, f_blocks=128982638, f_bfree=128964776, f_bavail=122411176, f_files=32768000, f_ffree=32767989, f_fsid={-1985690702, 273485452}, f_namelen=255, f_frsize=4096}) = 0

statfs("/data", {f_type="EXT2_SUPER_MAGIC", f_bsize=4096, f_blocks=1290120324, f_bfree=1274709053, f_bavail=1209173053, f_files=327680000, f_ffree=327678617, f_fsid={-1399205202, -542965053}, f_namelen=255, f_frsize=4096}) = 0

statfs("/var/lib/nfs/rpc_pipefs", {f_type=0x67596969, f_bsize=4096, f_blocks=0, f_bfree=0, f_bavail=0, f_files=0, f_ffree=0, f_fsid={0, 0}, f_namelen=255, f_frsize=4096}) = 0

statfs("/mnt",

如上图显示，卡在了 mnt 目录。通过 mount 命令查看这个目录，是一个网络目录：

所以，相应的解决办法是：

耐心等待统计完成后返回

卸载 NFS 分区的挂载后重新统计

检查 NFS 分区访问情况

检查服务器端NFS服务是否开启