Kernel panic流程及处理

一、kernel panic代码流程

kernel panic主要包括die和panic两部分流程。

1.1 die流程

die流程中主要做以下事情：

1）对所有注册die chain的模块进行callback回调

2）PC、LR、SP 等寄存器的信息，同时打印调用堆栈信息

3）打印PC指针信息

// arch/arm64/mm/fault.c
 
static void die_kernel_fault(const char *msg, unsigned long addr,
                             unsigned int esr, struct pt_regs *regs)
{
        bust_spinlocks(1);
 
        pr_alert("Unable to handle kernel %s at virtual address %016lx\n", msg,
                 addr);
 
        mem_abort_decode(esr);
 
        show_pte(addr);
        die("Oops", regs, esr);
        bust_spinlocks(0);
        do_exit(SIGKILL);
}

die() 会进行oops 异常处理：

// arch/arm64/kernel/traps.c
 
static DEFINE_RAW_SPINLOCK(die_lock);
 
/*
 * This function is protected against re-entrancy.
 */
void die(const char *str, struct pt_regs *regs, int err)
{
        enum bug_trap_type bug_type = BUG_TRAP_TYPE_NONE;
        unsigned long flags = oops_begin();
        int sig = SIGSEGV;

        if (!user_mode(regs))
                bug_type = report_bug(regs->ARM_pc, regs);
        if (bug_type != BUG_TRAP_TYPE_NONE)
                str = "Oops - BUG";

        if (__die(str, err, regs))
                sig = 0;

        oops_end(flags, regs, sig);
}
 
static int __die(const char *str, int err, struct pt_regs *regs)
{
        static int die_counter;
        int ret;
 
        pr_emerg("Internal error: %s: %x [#%d]" S_PREEMPT S_SMP "\n",
                 str, err, ++die_counter);
 
        /* trap and error numbers are mostly meaningless on ARM */
        // 通知所有对 Oops 感兴趣的模块并进行callback
        ret = notify_die(DIE_OOPS, str, regs, err, 0, SIGSEGV);
        if (ret == NOTIFY_STOP)
                return ret;
 
        print_modules();
        // PC、LR、SP 等寄存器的信息，同时打印调用堆栈信息
        show_regs(regs);
         // 打印PC指针信息
        dump_kernel_instr(KERN_EMERG, regs);
 
        return ret;
}

模块会通过函数 register_die_notifier() 将callback 注册到全局结构体变量 die_chain 中，然后在通过 notify_die() 函数去解析这个 die_chain，并分别调用callback：

// kernel/notifier.c
 
static ATOMIC_NOTIFIER_HEAD(die_chain);
 
int notrace notify_die(enum die_val val, const char *str,
               struct pt_regs *regs, long err, int trap, int sig)
{
        struct die_args args = {
                .regs        = regs,
                .str        = str,
                .err        = err,
                .trapnr        = trap,
                .signr        = sig,
 
        };
        RCU_LOCKDEP_WARN(!rcu_is_watching(),
                           "notify_die called but RCU thinks we're quiescent");
        return atomic_notifier_call_chain(&die_chain, val, &args);
}
NOKPROBE_SYMBOL(notify_die);
 
int register_die_notifier(struct notifier_block *nb)
{
        vmalloc_sync_mappings();
        return atomic_notifier_chain_register(&die_chain, nb);
}

mtk的aee异常引擎在kernel初始化的时候会去注册到die和panic通知链:

int __init aee_ipanic_init(void)
{
     spin_lock_init(&ipanic_lock);
     mrdump_init();
     atomic_notifier_chain_register(&panic_notifier_list, &panic_blk);
     register_die_notifier(&die_blk);
     register_ipanic_ops(&ipanic_oops_ops);
     ipanic_log_temp_init();
     ipanic_msdc_init();
     LOGI("ipanic: startup, partition assgined %s\n", AEE_IPANIC_PLABEL);
     return 0;
}

1.2 panic流程

panic流程中，主要做的事情：

1）关闭中断和抢占

2）打印输出panic log

3）停止其它cpu，只保留当前的cpu干活

4）回调注册panic chain模块的calbback

5）dump kmsg

6）抓取kdump

7）系统重启

// kernel/panic.c
 
/**
 *        panic - halt the system
 *        @fmt: The text string to print
 *
 *        Display a message, then perform cleanups.
 *
 *        This function never returns.
 */
void panic(const char *fmt, ...)
{
        static char buf[1024];
        va_list args;
        long i, i_next = 0, len;
        int state = 0;
        int old_cpu, this_cpu;
        bool _crash_kexec_post_notifiers = crash_kexec_post_notifiers;
 
 
    //禁止本地中断，避免出现死锁，因为无法防止中断处理程序(在获得panic锁后运行)再次被调用panic
        local_irq_disable();
    //禁止任务抢占
        preempt_disable_notrace();
 
    //通过this_cpu确认是否调用panic() 的cpu是否为panic_cpu；
        //即，只允许一个CPU执行该代码，通过 panic_smp_self_stop() 保证当一个CPU执行panic时，
    //其他CPU处于停止或等待状态；
        this_cpu = raw_smp_processor_id();
        old_cpu  = atomic_cmpxchg(&panic_cpu, PANIC_CPU_INVALID, this_cpu);
 
        if (old_cpu != PANIC_CPU_INVALID && old_cpu != this_cpu)
                panic_smp_self_stop();
 
    //把console的打印级别放开
        console_verbose();
        bust_spinlocks(1);
        va_start(args, fmt);
        len = vscnprintf(buf, sizeof(buf), fmt, args);
        va_end(args);
 
        if (len && buf[len - 1] == '\n')
                buf[len - 1] = '\0';
 
    //解析panic所携带的message，前缀为Kernel panic - not syncing
        pr_emerg("Kernel panic - not syncing: %s\n", buf);
#ifdef CONFIG_DEBUG_BUGVERBOSE
        /*
         * Avoid nested stack-dumping if a panic occurs during oops processing
         */
        if (!test_taint(TAINT_DIE) && oops_in_progress <= 1)
                dump_stack();
#endif
 
    //如果kgdb使能，即CONFIG_KGDB为y，在停掉所有其他CPU之前，跳转kgdb断点运行
        kgdb_panic(buf);
 
        if (!_crash_kexec_post_notifiers) {
                printk_safe_flush_on_panic();
        //会根据当前是否设置了转储内核(使能CONFIG_KEXEC_CORE)确定是否实际执行转储操作；
        //如果执行转储则会通过 kexec 将系统切换到新的kdump 内核，并且不会再返回；
        //如果不执行转储，则继续后面流程；
                __crash_kexec(NULL);
 
                //停掉其他CPU，只留下当前CPU干活
                smp_send_stop();
        } else {
                /*
                 * If we want to do crash dump after notifier calls and
                 * kmsg_dump, we will need architecture dependent extra
                 * works in addition to stopping other CPUs.
                 */
                crash_smp_send_stop();
        }
 
    //通知所有对panic感兴趣的模块进行回调，添加一些kmsg信息到输出
        atomic_notifier_call_chain(&panic_notifier_list, 0, buf);
 
        /* Call flush even twice. It tries harder with a single online CPU */
        printk_safe_flush_on_panic();
 
    //dump 内核log buffer中的log信息
        kmsg_dump(KMSG_DUMP_PANIC);
 
        /*
         * If you doubt kdump always works fine in any situation,
         * "crash_kexec_post_notifiers" offers you a chance to run
         * panic_notifiers and dumping kmsg before kdump.
         * Note: since some panic_notifiers can make crashed kernel
         * more unstable, it can increase risks of the kdump failure too.
         *
         * Bypass the panic_cpu check and call __crash_kexec directly.
         */
        if (_crash_kexec_post_notifiers)
                __crash_kexec(NULL);
 
#ifdef CONFIG_VT
        unblank_screen();
#endif
        console_unblank();
 
    //关掉所有debug锁
        debug_locks_off();
        console_flush_on_panic(CONSOLE_FLUSH_PENDING);
 
        panic_print_sys_info();
 
        if (!panic_blink)
                panic_blink = no_blink;
 
    //如果sysctl配置了panic_timeout > 0则在panic_timeout后重启系统
    //首先，这里会每隔100ms重启 NMI watchdog
        if (panic_timeout > 0) {
                /*
                 * Delay timeout seconds before rebooting the machine.
                 * We can't use the "normal" timers since we just panicked.
                 */
                pr_emerg("Rebooting in %d seconds..\n", panic_timeout);
 
                for (i = 0; i < panic_timeout * 1000; i += PANIC_TIMER_STEP) {
                        touch_nmi_watchdog();
                        if (i >= i_next) {
                                i += panic_blink(state ^= 1);
                                i_next = i + 3600 / PANIC_BLINK_SPD;
                        }
                        mdelay(PANIC_TIMER_STEP);
                }
        }
    //其次，这里确定reboot_mode，并重启系统
        if (panic_timeout != 0) {
                /*
                 * This will not be a clean reboot, with everything
                 * shutting down.  But if there is a chance of
                 * rebooting the system it will be rebooted.
                 */
                if (panic_reboot_mode != REBOOT_UNDEFINED)
                        reboot_mode = panic_reboot_mode;
                emergency_restart();
        }
#ifdef __sparc__
        {
                extern int stop_a_enabled;
                /* Make sure the user can actually press Stop-A (L1-A) */
                stop_a_enabled = 1;
                pr_emerg("Press Stop-A (L1-A) from sun keyboard or send break\n"
                         "twice on console to return to the boot prom\n");
        }
#endif
#if defined(CONFIG_S390)
        disabled_wait();
#endif
        pr_emerg("---[ end Kernel panic - not syncing: %s ]---\n", buf);
 
        /* Do not scroll important messages printed above */
        suppress_printk = 1;
        local_irq_enable();
        for (i = 0; ; i += PANIC_TIMER_STEP) {
                touch_softlockup_watchdog();
                if (i >= i_next) {
                        i += panic_blink(state ^= 1);
                        i_next = i + 3600 / PANIC_BLINK_SPD;
                }
                mdelay(PANIC_TIMER_STEP);
        }
}
 
EXPORT_SYMBOL(panic);

二、kernel panic问题分析

根据kernel log可以发现，当发生kernel panic时，PC指针停留在regulator_is_enabled函数ffffff8008474b88的地方，，通过addr2line工具结合内核符号映射表 vmlinux 就可以定位出具体代码所在文件行号：

arm-linux-androideabi-addr2line -e out/target/product/$project/obj/KERNEL_OBJ/vmlinux -f -C ffffff8008474b88

部分kernel异常log，如下：

[ 328.574562] -(6)[585:camerahalserver]Internal error: Oops: 96000005 [#1] PREEMPT SMP

[ 329.574612] -(6)[585:camerahalserver]CPU: 6 PID: 585 Comm: camerahalserver Tainted: P S W O 4.4.146+ #1

[ 329.574634] -(6)[585:camerahalserver]PC is at regulator_is_enabled+0x10/0x84

[ 329.574644] -(6)[585:camerahalserver]pc : [<ffffff8008474b88>] lr : [<ffffff800876c738>] pstate: 80400145