记一次CentOS 内核错误导致无法启动排错

系统是在 yum update 下更新了内核（或许是），什么原因更新的内核已经不知道了。

重启系统后发现卡在启动画面，日志如下

[    4.226327] List of all partitions:
[    4.232005] No filesystem could mount root, tried: 
[    4.239459] Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(0,0)
[    4.251804] CPU: 0 PID: 1 Comm: swapper/0 Not tainted 3.10.0-1160.24.1.el7.x86_64 #1
[    4.264505] Hardware name: Amazon EC2 t3.small/, BIOS 1.0 10/16/2017
[    4.272405] Call Trace:
[    4.277836]  [<ffffffff9ed8308a>] dump_stack+0x19/0x1b
[    4.284558]  [<ffffffff9ed7c5e2>] panic+0xe8/0x21f
[    4.290821]  [<ffffffff9f38b794>] mount_block_root+0x291/0x2a0
[    4.297862]  [<ffffffff9f38b7f6>] mount_root+0x53/0x56
[    4.305231]  [<ffffffff9f38b935>] prepare_namespace+0x13c/0x174
[    4.312348]  [<ffffffff9f38b412>] kernel_init_freeable+0x222/0x249
[    4.320206]  [<ffffffff9f38ab28>] ? initcall_blacklist+0xb0/0xb0
[    4.327237]  [<ffffffff9ed71720>] ? rest_init+0x80/0x80
[    4.333861]  [<ffffffff9ed7172e>] kernel_init+0xe/0x100
[    4.341405]  [<ffffffff9ed95df7>] ret_from_fork_nospec_begin+0x21/0x21
[    4.349275]  [<ffffffff9ed71720>] ? rest_init+0x80/0x80
[    4.356767] Kernel Offset: 0x1d600000 from 0xffffffff81000000 (relocation range: 0xffffffff80000000-0xffffffffbfffffff)

，由日志可以分析由于内核原因导致无法识别文件系统，从而导致找不到根卷无法启动设备。

 

因为该系统已经无法正常启动了，所以我觉得将它挂载到别的linux主机上查看文件系统，以分析哪里配置出现的问题。

 

挂载到别的主机后（挂载方法有很多，根据公有云，私有云，实体机，虚拟机各有不同），第一步检查是否识别到该硬盘。

 

这里我使用lsblk 识别所有硬盘设备。
 

[root@ipxxxxxxx]# lsblk
NAME          MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
nvme1n1       259:0    0  20G  0 disk
└─nvme1n1p1   259:1    0  20G  0 part
nvme0n1       259:2    0   8G  0 disk
├─nvme0n1p1   259:3    0   8G  0 part /
└─nvme0n1p128 259:4    0   1M  0 part


发现已经读到硬盘，然后挂载 并读取

 sudo mkdir -p /mnt/disk001 && sudo mount /dev/nvme1n1p1 /mnt/disk001 && cd /mnt/disk001

打开 /boot/grup2/grup.cfg

#
# DO NOT EDIT THIS FILE
#
# It is automatically generated by grub2-mkconfig using templates
# from /etc/grub.d and settings from /etc/default/grub
#

### BEGIN /etc/grub.d/00_header ###
set pager=1

if [ -s $prefix/grubenv ]; then
  load_env
fi
if [ "${next_entry}" ] ; then
   set default="${next_entry}"
   set next_entry=
   save_env next_entry
   set boot_once=true
else
   set default="${saved_entry}"
fi

if [ x"${feature_menuentry_id}" = xy ]; then
  menuentry_id_option="--id"
else
  menuentry_id_option=""
fi

export menuentry_id_option

if [ "${prev_saved_entry}" ]; then
  set saved_entry="${prev_saved_entry}"
  save_env saved_entry
  set prev_saved_entry=
  save_env prev_saved_entry
  set boot_once=true
fi

function savedefault {
  if [ -z "${boot_once}" ]; then
    saved_entry="${chosen}"
    save_env saved_entry
  fi
}

function load_video {
  if [ x$feature_all_video_module = xy ]; then
    insmod all_video
  else
    insmod efi_gop
    insmod efi_uga
    insmod ieee1275_fb
    insmod vbe
    insmod vga
    insmod video_bochs
    insmod video_cirrus
  fi
}

serial --speed=115200
terminal_input serial console
terminal_output serial console
if [ x$feature_timeout_style = xy ] ; then
  set timeout_style=menu
  set timeout=1
# Fallback normal timeout code in case the timeout_style feature is
# unavailable.
else
  set timeout=1
fi
### END /etc/grub.d/00_header ###

### BEGIN /etc/grub.d/00_tuned ###
set tuned_params=""
set tuned_initrd=""
### END /etc/grub.d/00_tuned ###

### BEGIN /etc/grub.d/01_users ###
if [ -f ${prefix}/user.cfg ]; then
  source ${prefix}/user.cfg
  if [ -n "${GRUB2_PASSWORD}" ]; then
    set superusers="root"
    export superusers
    password_pbkdf2 root ${GRUB2_PASSWORD}
  fi
fi
### END /etc/grub.d/01_users ###

### BEGIN /etc/grub.d/10_linux ###
menuentry 'CentOS Linux (3.10.0-957.1.3.el7.x86_64) 7 (Core)' --class centos --c
lass gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnuli
nux-3.10.0-957.1.3.el7.x86_64-advanced-f41e390f-835b-4223-a9bb-9b45984ddf8d' {
        load_video
        set gfxpayload=keep
        insmod gzio
        insmod part_msdos
        insmod xfs
        set root='hd0,msdos1'
        if [ x$feature_platform_search_hint = xy ]; then
          search --no-floppy --fs-uuid --set=root --hint='hd0,msdos1'  f41e390f-
835b-4223-a9bb-9b45984ddf8d
        else
          search --no-floppy --fs-uuid --set=root f41e390f-835b-4223-a9bb-9b45984ddf8d
        fi
        linux16 /boot/vmlinuz-3.10.0-957.1.3.el7.x86_64 root=UUID=f41e390f-8350-4213-a9bb-9b45314dd452 ro
console=tty0 console=ttyS0,115200n8 crashkernel=auto console=ttyS0,115200 LANG=en_US.UTF-8
        initrd16 /boot/initramfs-3.10.0-957.1.3.el7.x86_64.img
}

menuentry 'CentOS Linux (3.10.0-1160.24.1.el7.x86_64) 7 (Core)' --class centos -
-class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnu
linux-3.10.0-957.1.3.el7.x86_64-advanced-f41e390f-835b-4223-a9bb-9b45984ddf8d' {
        load_video
        set gfxpayload=keep
        insmod gzio
        insmod part_msdos
        insmod xfs
        set root='hd0,msdos1'
        if [ x$feature_platform_search_hint = xy ]; then
          search --no-floppy --fs-uuid --set=root --hint='hd0,msdos1'  f41e390f-8350-4213-a9bb-9b45314dd45
        else
          search --no-floppy --fs-uuid --set=root f41e390f-835b-4223-a9bb-9b4598
4ddf8d
        fi
        linux16 /boot/vmlinuz-3.10.0-1160.24.1.el7.x86_64 root=UUID=f41e390f-835
b-4223-a9bb-9b45984ddf8d ro console=tty0 console=ttyS0,115200n8 crashkernel=auto
 console=ttyS0,115200 LANG=en_US.UTF-8
}
menuentry 'CentOS Linux (3.10.0-1127.13.1.el7.x86_64) 7 (Core)' --class centos -
-class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnu
linux-3.10.0-957.1.3.el7.x86_64-advanced-f41e390f-835b-4223-a9bb-9b45984ddf8d' {
        load_video
        set gfxpayload=keep
        insmod gzio
        insmod part_msdos
        insmod xfs
        set root='hd0,msdos1'
        if [ x$feature_platform_search_hint = xy ]; then
          search --no-floppy --fs-uuid --set=root --hint='hd0,msdos1'  f41e390f-
835b-4223-a9bb-9b45984ddf8d
        else
          search --no-floppy --fs-uuid --set=root f41e390f-835b-4223-a9bb-9b4598
4ddf8d
        fi
        linux16 /boot/vmlinuz-3.10.0-1127.13.1.el7.x86_64 root=UUID=f41e390f-835
b-4223-a9bb-9b45984ddf8d ro console=tty0 console=ttyS0,115200n8 crashkernel=auto
 console=ttyS0,115200 LANG=en_US.UTF-8
        initrd16 /boot/initramfs-3.10.0-1127.13.1.el7.x86_64.img
}
menuentry 'CentOS Linux (3.10.0-1062.18.1.el7.x86_64) 7 (Core)' --class centos -
-class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnu
linux-3.10.0-957.1.3.el7.x86_64-advanced-f41e390f-835b-4223-a9bb-9b45984ddf8d' {
        load_video
        set gfxpayload=keep
        insmod gzio
        insmod part_msdos
        insmod xfs
        set root='hd0,msdos1'
        if [ x$feature_platform_search_hint = xy ]; then
          search --no-floppy --fs-uuid --set=root --hint='hd0,msdos1'  f41e390f-
835b-4223-a9bb-9b45984ddf8d
        else
          search --no-floppy --fs-uuid --set=root f41e390f-835b-4223-a9bb-9b4598
4ddf8d
        fi
        linux16 /boot/vmlinuz-3.10.0-1062.18.1.el7.x86_64 root=UUID=f41e390f-835
b-4223-a9bb-9b45984ddf8d ro console=tty0 console=ttyS0,115200n8 crashkernel=auto
 console=ttyS0,115200 LANG=en_US.UTF-8
        initrd16 /boot/initramfs-3.10.0-1062.18.1.el7.x86_64.img
}

menuentry 'CentOS Linux (0-rescue-05cb8c7b39fe0f70e3ce97e5beab809d) 7 (Core)' --class centos --class gnu-l
inux --class gnu --class os --unrestricted $menuentry_id_option 'gnulinux-0-rescue-05cb8c7b39fe0f70e3ce97e
5beab809d-advanced-f41e390f-835b-4223-a9bb-9b45984ddf8d' {
        load_video
        insmod gzio
        insmod part_msdos
        insmod xfs
        set root='hd0,msdos1'
        if [ x$feature_platform_search_hint = xy ]; then
          search --no-floppy --fs-uuid --set=root --hint='hd0,msdos1'  f41e390f-835b-4223-a9bb-9b45984ddf8
d
        else
          search --no-floppy --fs-uuid --set=root f41e390f-835b-4223-a9bb-9b45984ddf8d
        fi
        linux16 /boot/vmlinuz-0-rescue-05cb8c7b39fe0f70e3ce97e5beab809d root=UUID=f41e390f-835b-4223-a9bb-
9b45984ddf8d ro console=tty0 console=ttyS0,115200n8 crashkernel=auto console=ttyS0,115200
        initrd16 /boot/initramfs-0-rescue-05cb8c7b39fe0f70e3ce97e5beab809d.img
}

### END /etc/grub.d/10_linux ###

### BEGIN /etc/grub.d/20_linux_xen ###
### END /etc/grub.d/20_linux_xen ###

### BEGIN /etc/grub.d/20_ppc_terminfo ###
### END /etc/grub.d/20_ppc_terminfo ###

### BEGIN /etc/grub.d/30_os-prober ###
### END /etc/grub.d/30_os-prober ###

### BEGIN /etc/grub.d/40_custom ###
# This file provides an easy way to add custom menu entries.  Simply type the
# menu entries you want to add after this comment.  Be careful not to change
# the 'exec tail' line above.
### END /etc/grub.d/40_custom ###

### BEGIN /etc/grub.d/41_custom ###
if [ -f  ${config_directory}/custom.cfg ]; then
  source ${config_directory}/custom.cfg
elif [ -z "${config_directory}" -a -f  $prefix/custom.cfg ]; then
  source $prefix/custom.cfg;
fi
### END /etc/grub.d/41_custom ###

 

由menuentry开头的这段代码 按照系统启动时，加载的内核。

menuentry 'CentOS Linux (3.10.0-957.1.3.el7.x86_64) 7 (Core)' --class centos --c
lass gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnuli
nux-3.10.0-957.1.3.el7.x86_64-advanced-f41e390f-835b-4223-a9bb-9b45984ddf8d' {
        load_video
        set gfxpayload=keep
        insmod gzio
        insmod part_msdos
        insmod xfs
        set root='hd0,msdos1'
        if [ x$feature_platform_search_hint = xy ]; then
          search --no-floppy --fs-uuid --set=root --hint='hd0,msdos1'  f41e390f-
835b-4223-a9bb-9b45984ddf8d
        else
          search --no-floppy --fs-uuid --set=root f41e390f-835b-4223-a9bb-9b45984ddf8d
        fi
        linux16 /boot/vmlinuz-3.10.0-957.1.3.el7.x86_64 root=UUID=f41e390f-835b-4223-a9bb-9b45984ddf8d ro
console=tty0 console=ttyS0,115200n8 crashkernel=auto console=ttyS0,115200 LANG=en_US.UTF-8
        initrd16 /boot/initramfs-3.10.0-957.1.3.el7.x86_64.img
}

因为系统是升级内核导致的无法启动，那么需要回退旧版本。

 

在这里只保留最原始的版本即可，删除其余内核代码文件只保留这一段内核代码加首尾内容

menuentry 'CentOS Linux (3.10.0-1062.18.1.el7.x86_64) 7 (Core)' --class centos -
-class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnu
linux-3.10.0-957.1.3.el7.x86_64-advanced-f41e390f-835b-4223-a9bb-9b45984ddf8d' {
        load_video
        set gfxpayload=keep
        insmod gzio
        insmod part_msdos
        insmod xfs
        set root='hd0,msdos1'
        if [ x$feature_platform_search_hint = xy ]; then
          search --no-floppy --fs-uuid --set=root --hint='hd0,msdos1'  f41e390f-
835b-4223-a9bb-9b45984ddf8d
        else
          search --no-floppy --fs-uuid --set=root f41e390f-835b-4223-a9bb-9b4598
4ddf8d
        fi
        linux16 /boot/vmlinuz-3.10.0-1062.18.1.el7.x86_64 root=UUID=f41e390f-835
b-4223-a9bb-9b45984ddf8d ro console=tty0 console=ttyS0,115200n8 crashkernel=auto
 console=ttyS0,115200 LANG=en_US.UTF-8
        initrd16 /boot/initramfs-3.10.0-1062.18.1.el7.x86_64.img
}