Linux soft lockup

最新推荐文章于 2024-05-31 10:58:25 发布

枫之所向

最新推荐文章于 2024-05-31 10:58:25 发布

阅读量1.1k

点赞数

分类专栏： linux 文章标签： linux

本文链接：https://blog.csdn.net/qq_42733114/article/details/123714853

版权

linux 专栏收录该内容

5 篇文章 0 订阅

订阅专栏

Linux soft lockup分析

关键词：watchdog、soft lockup、percpu thread、lockdep等。

近日遇到一个soft lockup问题，打印类似“[ 80.002856] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [poweroff:965]“。

这是lockup检测机制在起作用，lockup检测机制包括soft lockup detector和hard lockup detector。

借机分析下soft lockup机制以及什么情况下导致soft watchdog异常、对watchdog的配置、如何定位异常点。

这里跳过hard lockup detector的分析。

本文用到__this_cpu_write和__this_cpu_read,可以保重读取的变量或者更新的变量是原子操作

1. soft lockup机制分析

lockup_detector_init函数首先获取sample_period以及watchdog_cpumask，然后根据情况创建线程，启动喂狗程序；创建hrtimer启动看门狗。

然后有两个重点一个是创建内核线程的API以及struct smp_hotplug_thread结构体。

void __init lockup_detector_init(void)
{
	/* 获取变量sample_period，为watchdog_thresh*2/5，这是后面设置的4s的hrtime中断。 */
    set_sample_period();
...
    cpumask_copy(&watchdog_cpumask, cpu_possible_mask);

    if (watchdog_enabled)
        watchdog_enable_all_cpus();
}

static int watchdog_enable_all_cpus(void)
{
    int err = 0;

    if (!watchdog_running) {
    /* 如果当前watchdog_running没有再运行，那么为每个CPU创建一个watchdog/x线程，
     * 这些线程每隔sample_period时间喂一次狗。watchdog_threads时watchdog/x线程的主要输入参数，
     * watchdog_cpumask规定了为哪些CPU创建线程。
     */
     
        err = smpboot_register_percpu_thread_cpumask(&watchdog_threads,
                                 &watchdog_cpumask);
        if (err)
            pr_err("Failed to create watchdog threads, disabled\n");
        else
            watchdog_running = 1;
    } else {
        err = update_watchdog_all_cpus();

        if (err) {
            watchdog_disable_all_cpus();
            pr_err("Failed to update lockup detectors, disabled\n");
        }
    }

    if (err)
        watchdog_enabled = 0;

    return err;
}

static int update_watchdog_all_cpus(void)
{
    int ret;

    ret = watchdog_park_threads();
    if (ret)
        return ret;

    watchdog_unpark_threads();

    return 0;
}

static int watchdog_park_threads(void)
{
    int cpu, ret = 0;

    atomic_set(&watchdog_park_in_progress, 1);

    for_each_watchdog_cpu(cpu) {
    	/* 设置struct kthread->flags的KTHREAD_SHOULD_PARK位
    	 * 在watchdog/x线程中会调用unpark成员函数进行处理
    	 */
        ret = kthread_park(per_cpu(softlockup_watchdog, cpu));
        if (ret)
            break;
    }

    atomic_set(&watchdog_park_in_progress, 0);

    return ret;
}

1.1 watchdog_threads结构体介绍

在介绍如何创建watchdog/x线程之前，有必要先介绍一些struct smp_hotplug_thread线程。

struct smp_hotplug_thread {
    struct task_struct __percpu    **store; /*存放percpu strcut task_strcut指针的指针*/
    struct list_head        list;
    int                (*thread_should_run)(unsigned int cpu);  /* 检查是否应该运行watchdog/x线程 */
    void                (*thread_fn)(unsigned int cpu);			/* watchdog/x线程的主函数 */
    void                (*create)(unsigned int cpu);
    void                (*setup)(unsigned int cpu);				/* 运行watchdog/x线程之前的准备工作 */
    void                (*cleanup)(unsigned int cpu, bool online); /* 在退出watchdog/x线程之后的清楚工作 */
    void                (*park)(unsigned int cpu);				/*当CPU offline时，需要临时停止*/
    void                (*unpark)(unsigned int cpu);			/*当CPU变成online时，进行准备工作*/
    cpumask_var_t            cpumask;							/* 允许哪些CPU online */
    bool                selfparking;
    const char            *thread_comm;							/* watchdog/x线程名称 */
};

watchdog_threads是soft lockup监控线程的实体，基于此创建 watchdog/x线程。

static struct smp_hotplug_thread watchdog_threads = {
    .store            = &softlockup_watchdog,
    .thread_should_run    = watchdog_should_run,
    .thread_fn        = watchdog,
    .thread_comm        = "watchdog/%u",
    .setup            = watchdog_enable,
    .cleanup        = watchdog_cleanup,
    .park            = watchdog_disable,
    .unpark            = watchdog_enable,
};

static void watchdog_enable(unsigned int cpu)
{
    struct hrtimer *hrtimer = raw_cpu_ptr(&watchdog_hrtimer);

    /* kick off the timer for the hardlockup detector */
    hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
    /* 创建一个hrtimer，超时函数为watchdog_timer_fn，
     * 这里面会检查watchdog_touch_ts变量是否超过20秒没有被更新。
     * 如果是，则有soft lockup
     */
    hrtimer->function = watchdog_timer_fn;

    /* Enable the perf event */
    watchdog_nmi_enable(cpu);

    /* done here because hrtimer_start can only pin to smp_processor_id() */
    /* 启动一个超时为sample_period(4秒)的hrtimer，HRTIMER_MODE_REL_PINNED表示此hrtimer和当前CPU绑定 */
    hrtimer_start(hrtimer, ns_to_ktime(sample_period),
              HRTIMER_MODE_REL_PINNED);

    /* initialize timestamp */
    /* 设置当前线程为实时FIFO，并且优先级为实时99.这个优先级表示高于所有的非实时线程，但是实时优先级最低的*/
    watchdog_set_prio(SCHED_FIFO, MAX_RT_PRIO - 1);
    /* 更新watchdog_touch_ts变量，相当于喂狗操作。 */
    __touch_watchdog();
}

static void watchdog_set_prio(unsigned int policy, unsigned int prio)
{
    struct sched_param param = { .sched_priority = prio };

    sched_setscheduler(current, policy, &param);
}

/* Commands for resetting the watchdog */
static void __touch_watchdog(void)
{
	/* 喂狗的操作就是更新watchdog_touch_ts变量，就是当前时间戳更新到watchdog_touch_ts中*/
    __this_cpu_write(watchdog_touch_ts, get_timestamp());
}


static int watchdog_should_run(unsigned int cpu)
{
	/* hrtimer_interrupts记录了产生hrtimer的次数；
	 * 在watchdog()中，将hrtimer_interrupts赋给soft_lockup_hrtimer_cnt。两者相等表示没有hrtimer产生，
	 * 不需要运行watchdog/x线程；相反不等，则需要watchdog/x线程运行。
	 */
    return __this_cpu_read(hrtimer_interrupts) !=
        __this_cpu_read(soft_lockup_hrtimer_cnt);
}
static void watchdog(unsigned int cpu)
{
	/* 更新soft_lockup_hrtimer_cnt，在watch_should_run()中就返回false，
	 * 表示线程不需要运行，即不需要喂狗。
	 */
    __this_cpu_write(soft_lockup_hrtimer_cnt,
             __this_cpu_read(hrtimer_interrupts));
    /*虽然就是一句话，但是却很重要的喂狗操作。*/
    __touch_watchdog();

    if (!(watchdog_enabled & NMI_WATCHDOG_ENABLED))
        watchdog_nmi_disable(cpu);
}

1.2 创建喂狗线程watchdog/x

在分析了[watchdog_threads]之后，再来看看如何创建watchdog/x线程。

int smpboot_register_percpu_thread_cpumask(struct smp_hotplug_thread *plug_thread,
                       const struct cpumask *cpumask)
{
    unsigned int cpu;
    int ret = 0;

    if (!alloc_cpumask_var(&plug_thread->cpumask, GFP_KERNEL))
        return -ENOMEM;
    cpumask_copy(plug_thread->cpumask, cpumask);

    get_online_cpus();
    mutex_lock(&smpboot_threads_lock);
    for_each_online_cpu(cpu) {------------------------------------------------遍历所有online CPU，为每个CPU创建一个percpu的watchdog/x线程。
        ret = __smpboot_create_thread(plug_thread, cpu);
        if (ret) {
            smpboot_destroy_threads(plug_thread);-----------------------------创建失败则释放相关资源。
            free_cpumask_var(plug_thread->cpumask);
            goto out;
        }
        if (cpumask_test_cpu(cpu, cpumask))
            smpboot_unpark_thread(plug_thread, cpu);--------------------------如果当前CPU不在cpumask中，则清空KTHREAD_SHOULD_PARK，进而调用watchdog_therads的umpark成员函数。
    }
    list_add(&plug_thread->list, &hotplug_threads);
out:
    mutex_unlock(&smpboot_threads_lock);
    put_online_cpus();
    return ret;
}

static int
__smpboot_create_thread(struct smp_hotplug_thread *ht, unsigned int cpu)
{
    struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu);
    struct smpboot_thread_data *td;

    if (tsk)
        return 0;

    td = kzalloc_node(sizeof(*td), GFP_KERNEL, cpu_to_node(cpu));
    if (!td)
        return -ENOMEM;
    td->cpu = cpu;
    td->ht = ht;

    tsk =kthread_create_on_cpu(smpboot_thread_fn, td, cpu,
                    ht->thread_comm);-----------------------------------------在指定CPU上创建watchdog/x线程，处理函数为smpboot_thread_fn()。
    if (IS_ERR(tsk)) {
        kfree(td);
        return PTR_ERR(tsk);
    }
    /*
     * Park the thread so that it could start right on the CPU
     * when it is available.
     */
    kthread_park(tsk);--------------------------------------------------------在CPU上立即启动watchdog/x线程。
    get_task_struct(tsk);-----------------------------------------------------增加对线程的引用计数。
    *per_cpu_ptr(ht->store, cpu) = tsk;---------------------------------------store存放线程结构体指针的指针。
    if (ht->create) {
        if (!wait_task_inactive(tsk, TASK_PARKED))
            WARN_ON(1);
        else
            ht->create(cpu);
    }
    return 0;
}

static int smpboot_thread_fn(void *data)
{
    struct smpboot_thread_data *td = data;
    struct smp_hotplug_thread *ht = td->ht;

    while (1) {
        set_current_state(TASK_INTERRUPTIBLE);
        preempt_disable();
        if (kthread_should_stop()) {----------------------------------------如果可以终止线程，调用cleanup，退出线程。
            __set_current_state(TASK_RUNNING);
            preempt_enable();
            /* cleanup must mirror setup */
            if (ht->cleanup && td->status != HP_THREAD_NONE)
                ht->cleanup(td->cpu, cpu_online(td->cpu));
            kfree(td);
            return 0;
        }

        if (kthread_should_park()) {----------------------------------------如果KTHREAD_SHOULD_PARK置位，调用park()暂停进程执行。
            __set_current_state(TASK_RUNNING);
            preempt_enable();
            if (ht->park && td->status == HP_THREAD_ACTIVE) {
                BUG_ON(td->cpu != smp_processor_id());
                ht->park(td->cpu);
                td->status = HP_THREAD_PARKED;
            }
            kthread_parkme();
            /* We might have been woken for stop */
            continue;
        }

        BUG_ON(td->cpu != smp_processor_id());

        /* Check for state change setup */
        switch (td->status) {
        case HP_THREAD_NONE:-----------------------------------------------相当于第一次运行，调用setup()进行初始化操作。
            __set_current_state(TASK_RUNNING);
            preempt_enable();
            if (ht->setup)
                ht->setup(td->cpu);
            td->status = HP_THREAD_ACTIVE;
            continue;

        case HP_THREAD_PARKED:---------------------------------------------从parked状态恢复。
            __set_current_state(TASK_RUNNING);
            preempt_enable();
            if (ht->unpark)
                ht->unpark(td->cpu);
            td->status = HP_THREAD_ACTIVE;
            continue;
        }

        if (!ht->thread_should_run(td->cpu)) {-----------------------------如果不需要进程运行，schedule()主动放弃CPU给其他线程使用。
            preempt_enable_no_resched();
            schedule();
        } else {
            __set_current_state(TASK_RUNNING);
            preempt_enable();
            ht->thread_fn(td->cpu);----------------------------------------调用struct smpboot_thread_fn->thread_fn及watchdog()，进行喂狗操作。
        }
    }
}

void smpboot_unregister_percpu_thread(struct smp_hotplug_thread *plug_thread)----将创建的内核线程移除操作。
{
    get_online_cpus();
    mutex_lock(&smpboot_threads_lock);
    list_del(&plug_thread->list);
    smpboot_destroy_threads(plug_thread);
    mutex_unlock(&smpboot_threads_lock);
    put_online_cpus();
    free_cpumask_var(plug_thread->cpumask);
}

static void smpboot_destroy_threads(struct smp_hotplug_thread *ht)
{
    unsigned int cpu;

    /* We need to destroy also the parked threads of offline cpus */
    for_each_possible_cpu(cpu) {
        struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu);

        if (tsk) {
            kthread_stop(tsk);
            put_task_struct(tsk);
            *per_cpu_ptr(ht->store, cpu) = NULL;
        }
    }
}

1.3 hrtimer看门狗

在分析了喂狗线程watchdog/x之后，再来分析看门狗是如何实现的？

看门狗是通过启动一个周期为4秒的hrtimer来实现的，这个hrtimer和CPU绑定，使用的变量都是percpu的。确保每个CPU之间不相互干扰。

每次hrtimer超时，都会唤醒watchdog/x线程，并进行一次喂狗操作。

因为hrtimer超时函数在软中断中调用，在中断产生后会比线程优先得到执行。

所以在watchdog/x线程没有得到执行的情况下，通过is_softlockup来判断看门狗是否超过20秒没有得到喂狗。

static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
{
    unsigned long touch_ts = __this_cpu_read(watchdog_touch_ts);
    struct pt_regs *regs = get_irq_regs();
    int duration;
    int softlockup_all_cpu_backtrace = sysctl_softlockup_all_cpu_backtrace;

    if (atomic_read(&watchdog_park_in_progress) != 0)
        return HRTIMER_NORESTART;

    /* kick the hardlockup detector */
   	/*每产生一次中断，hrtimer_interrupts计数加1.hrtimer_interrupts记录了产生hrtimer的次数。*/
    watchdog_interrupt_count();

    /* kick the softlockup detector */
    /*唤醒watchdog/x线程，进行喂狗操作,如果此时的调度被关闭，被喂狗线程得不到执行，可手动preempt_disbale()关闭调度测试softlock*/
    wake_up_process(__this_cpu_read(softlockup_watchdog));

    /* .. and repeat 新设置超时点，形成周期性时钟。*/
    hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period));
...
	/* 返回非0表示，看门狗超时。 */
    duration = is_softlockup(touch_ts);
    if (unlikely(duration)) {
        if (kvm_check_and_clear_guest_paused())
            return HRTIMER_RESTART;

        /* only warn once */
        if (__this_cpu_read(soft_watchdog_warn) == true) {
            if (__this_cpu_read(softlockup_task_ptr_saved) !=
                current) {
                __this_cpu_write(soft_watchdog_warn, false);
                __touch_watchdog();
            }
            return HRTIMER_RESTART;
        }

        if (softlockup_all_cpu_backtrace) {
            if (test_and_set_bit(0, &soft_lockup_nmi_warn)) {
                /* Someone else will report us. Let's give up */
                __this_cpu_write(soft_watchdog_warn, true);
                return HRTIMER_RESTART;
            }
        }

		/* 打印哪个CPU被卡死duration秒，以及死在哪个进程。*/
        pr_emerg("BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n",
            smp_processor_id(), duration,
            current->comm, task_pid_nr(current));
        __this_cpu_write(softlockup_task_ptr_saved, current);
        print_modules();
        /* 显示开关中断、软中断信息，禁止中断和软中断也是造成soft lockup的一个原因。 */
        print_irqtrace_events(current);
        
        /* 有寄存器显示寄存器信息，同时显示栈信息。 */
        if (regs)
            show_regs(regs);
        else
            dump_stack();

        if (softlockup_all_cpu_backtrace) {
            trigger_allbutself_cpu_backtrace();

            clear_bit(0, &soft_lockup_nmi_warn);
            /* Barrier to sync with other cpus */
            smp_mb__after_atomic();
        }

        add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
        /* 如果定义softlockup_panic则进入panic()。 */
        if (softlockup_panic)
            panic("softlockup: hung tasks");
        __this_cpu_write(soft_watchdog_warn, true);
    } else
        __this_cpu_write(soft_watchdog_warn, false);

    return HRTIMER_RESTART;
}

static void watchdog_interrupt_count(void)
{
__this_cpu_inc(hrtimer_interrupts);
}

static int is_softlockup(unsigned long touch_ts)
{
    unsigned long now = get_timestamp();

    if ((watchdog_enabled & SOFT_WATCHDOG_ENABLED) && watchdog_thresh){
        /* Warn about unreasonable delays. */
        if (time_after(now, touch_ts + get_softlockup_thresh()))
            return now - touch_ts;
    }
    return 0;
}

2. 对watchdog的设置

对watchdog行为的设置有两个途径：通过命令行传入参数和通过proc设置。

2.1 通过命令行设置

static int __init softlockup_panic_setup(char *str)
{
    softlockup_panic = simple_strtoul(str, NULL, 0);

    return 1;
}
__setup("softlockup_panic=", softlockup_panic_setup);

static int __init nowatchdog_setup(char *str)
{
    watchdog_enabled = 0;
    return 1;
}
__setup("nowatchdog", nowatchdog_setup);

static int __init nosoftlockup_setup(char *str)
{
    watchdog_enabled &= ~SOFT_WATCHDOG_ENABLED;
    return 1;
}
__setup("nosoftlockup", nosoftlockup_setup);

#ifdef CONFIG_SMP
static int __init softlockup_all_cpu_backtrace_setup(char *str)
{
    sysctl_softlockup_all_cpu_backtrace =
        !!simple_strtol(str, NULL, 0);
    return 1;
}
__setup("softlockup_all_cpu_backtrace=", softlockup_all_cpu_backtrace_setup);
static int __init hardlockup_all_cpu_backtrace_setup(char *str)
{
    sysctl_hardlockup_all_cpu_backtrace =
        !!simple_strtol(str, NULL, 0);
    return 1;
}
__setup("hardlockup_all_cpu_backtrace=", hardlockup_all_cpu_backtrace_setup);
#endif

2.2 通过sysfs节点调节watchdog

watchdog相关的配置还可以通过proc文件系统进行配置。

/proc/sys/kernel/nmi_watchdog-------------------------hard lockup开关，proc_nmi_watchdog()。
/proc/sys/kernel/soft_watchdog------------------------soft lockup开关，proc_soft_watchdog()。
/proc/sys/kernel/watchdog-----------------------------watchdog总开关，proc_watchdog()。
/proc/sys/kernel/watchdog_cpumask---------------------watchdog cpumaks，proc_watchdog_cpumask()。
/proc/sys/kernel/watchdog_thresh----------------------watchdog超时阈值设置，proc_watchdog_thresh()。

3. 定位soft lockup异常

引起soft lockup的原因一般是死循环或者死锁，死循环可以通过栈回溯找到问题点；死锁问题需要打开内核的lockdep功能。

打开内核的lockdep功能可以参考《Linux死锁检测-Lockdep》。

下面看一个关闭调度并while(1)引起的soft lockup异常分析：

[  664.049439] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [init:980]
[  664.057346] Modules linked in: [last unloaded: videobuf2_dma_contig]
[  664.064495] CPU: 0 PID: 980 Comm: init Not tainted 4.9.118 #118
[  664.071135] Hardware name: sun8iw19
[  664.075050] task: ddc27a40 task.stack: ddc6e000
[  664.080143] PC is at axp2101_shutdown+0x24/0x30	/* 这里PC指针直接打印出了函数名 */
[  664.085229] LR is at axp2101_shutdown+0x24/0x30
[  664.090315] pc : [<c0324c24>]    lr : [<c0324c24>]    psr: 60000013
[  664.090315] sp : ddc6fe58  ip : c0793238  fp : 00000000
[  664.103200] r10: 00000000  r9 : fee1dead  r8 : df23d844
[  664.109063] r7 : ddc6e000  r6 : df23d020  r5 : df23d810  r4 : df23d81c
[  664.116388] r3 : 00000000  r2 : 00000001  r1 : 00000002  r0 : 00000023
[  664.123715] Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
[  664.131723] Control: 10c5387d  Table: 5dc50059  DAC: 00000051
[  664.138168] 
[  664.138168] PC: 0xc0324ba4:
[  664.142959] 4ba4  ebfd2738 e3a00000 e8bd8010 c0686844 e92d4010 e5904064 e594327c e3530000
[  664.152195] 4bc4  1a000002 e594102c e1a00004 ebfffd50 e2840f8e ebf441d4 e1a00004 e3a01000
[  664.161430] 4be4  ebffff5d e2842fa1 e3a01081 e5940224 ebfd229e e3a00000 e8bd8010 e92d4010
[  664.170664] 4c04  e59f001c ebf54f6c e59f0018 ebf54f6a e3a00001 ebf462ce e59f0004 ebf54f66
[  664.179899] 4c24  eafffffe c05a3aaf c05e013c e92d47f0 e24dd028 e1a04000 e28d2010 e3a010a2
[  664.189133] 4c44  e5100014 e3a03000 e1cd30be ebfd2287 e59d3010 e3130010 0a000019 e28d2020
[  664.198366] 4c64  e3a010b0 e5140014 ebfd2280 e59d3020 e353002e 1a000012 e28d2010 e3a010b2
[  664.207601] 4c84  e5140014 ebfd2279 e3a010b3 e5140014 e28d2014 ebfd2275 e59d2010 e59d3014
[  664.216837] 
[  664.216837] LR: 0xc0324ba4:
[  664.221627] 4ba4  ebfd2738 e3a00000 e8bd8010 c0686844 e92d4010 e5904064 e594327c e3530000
[  664.230861] 4bc4  1a000002 e594102c e1a00004 ebfffd50 e2840f8e ebf441d4 e1a00004 e3a01000
[  664.240095] 4be4  ebffff5d e2842fa1 e3a01081 e5940224 ebfd229e e3a00000 e8bd8010 e92d4010
[  664.249329] 4c04  e59f001c ebf54f6c e59f0018 ebf54f6a e3a00001 ebf462ce e59f0004 ebf54f66
[  664.258563] 4c24  eafffffe c05a3aaf c05e013c e92d47f0 e24dd028 e1a04000 e28d2010 e3a010a2
[  664.267789] 4c44  e5100014 e3a03000 e1cd30be ebfd2287 e59d3010 e3130010 0a000019 e28d2020
[  664.277023] 4c64  e3a010b0 e5140014 ebfd2280 e59d3020 e353002e 1a000012 e28d2010 e3a010b2
[  664.286257] 4c84  e5140014 ebfd2279 e3a010b3 e5140014 e28d2014 ebfd2275 e59d2010 e59d3014
[  664.295492] 
[  664.295492] SP: 0xddc6fdd8:
[  664.300282] fdd8  e0803000 ddc6fe08 df23d844 ddc6e000 00000000 c0009494 c0324c24 60000013
[  664.309516] fdf8  ffffffff ddc6fe3c df23d844 c001348c 00000023 00000002 00000001 00000000
[  664.318749] fe18  df23d81c df23d810 df23d020 ddc6e000 df23d844 fee1dead 00000000 00000000
[  664.327982] fe38  c0793238 ddc6fe58 c0324c24 c0324c24 60000013 ffffffff 00000051 bf000000
[  664.337207] fe58  df23d81c c0257ecc 4321fedc 4321fedc c068680c c068d010 00000000 c003ba24
[  664.346441] fe78  4321fedc c003bc08 ddc6fe8c c003eb48 ffffe000 c0021d94 ddc6fe90 de608870
[  664.355675] fe98  bea61000 00000055 ddc52fa8 de683184 de262038 00000000 ddc6ffb0 ddc6ffb0
[  664.364908] feb8  00000055 bea61c4c ddc27a40 0000080f de262000 de262038 00000800 c00198fc
[  664.374143] 
[  664.374143] IP: 0xc07931b8:
[  664.378933] 31b8  00000001 00000000 00000000 00000000 00000000 c07931cc c07931cc 00000000
[  664.388166] 31d8  00000000 c06938f0 00000000 00000001 00000001 00000000 c051e1d8 c07931f4
[  664.397399] 31f8  c07931f4 00400000 00000001 00000000 00000000 00000000 00000000 00000000
[  664.406631] 3218  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[  664.415863] 3238  00000001 00000002 00000000 00000000 00000000 00000000 00000000 df2fff00
[  664.425094] 3258  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[  664.434326] 3278  00000001 00000004 00000000 00000000 00000000 00000000 00000000 df2d0380
[  664.443558] 3298  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[  664.452791] 
[  664.452791] R4: 0xdf23d79c:
[  664.457581] d79c  6f74616c 00000072 00000000 00000000 00000000 00000000 00000000 00000000
[  664.466814] d7bc  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[  664.476045] d7dc  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[  664.485277] d7fc  00000000 df23d990 00000000 00000000 00000000 df23d020 df20ef00 df22fa80
[  664.494509] d81c  df23d81c df23d81c df23d028 df02e140 c0755c04 df224550 00000008 00000007
[  664.503742] d83c  00000000 c0756bdc 00000000 df23d848 df23d848 ddc27a40 df23d844 c0755ed0
[  664.512976] d85c  c076c09c 00000000 df3fe010 00000000 00000080 df23da70 df2730b0 7fffffff
[  664.522209] d87c  df23d87c df23d87c 00000000 00000000 00000000 00000000 00000000 c0262db4
[  664.531443] 
[  664.531443] R5: 0xdf23d790:
[  664.536233] d790  32707861 2d313031 75676572 6f74616c 00000072 00000000 00000000 00000000
[  664.545467] d7b0  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[  664.554697] d7d0  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[  664.563928] d7f0  00000000 00000000 00000000 00000000 df23d990 00000000 00000000 00000000
[  664.573159] d810  df23d020 df20ef00 df22fa80 df23d81c df23d81c df23d028 df02e140 c0755c04
[  664.582393] d830  df224550 00000008 00000007 00000000 c0756bdc 00000000 df23d848 df23d848
[  664.591626] d850  ddc27a40 df23d844 c0755ed0 c076c09c 00000000 df3fe010 00000000 00000080
[  664.600860] d870  df23da70 df2730b0 7fffffff df23d87c df23d87c 00000000 00000000 00000000
[  664.610094] 
[  664.610094] R6: 0xdf23cfa0:
[  664.614885] cfa0  00000001 00000000 df23cf50 c05a129a df23e061 df23e100 df23e0b0 00000000
[  664.624117] cfc0  4e121040 00000000 c04e23d8 00000000 00001000 00000000 00000000 00000000
[  664.633349] cfe0  c068fcd8 81240052 00002178 00000000 00000000 00000000 00000000 00000000
[  664.642581] d000  00340080 32707861 00313031 00000000 00000000 00000000 df215010 00000000
[  664.651813] d020  df215048 df255c00 df22b140 df23d41c df215054 df215050 df02e140 c0755c04
[  664.661047] d040  df23c910 00000037 00000007 00000000 c076abd0 00000000 df23d058 df23d058
[  664.670280] d060  ddc27a40 df23d054 c076ac00 c07587a4 00000000 df22be90 00000000 00000081
[  664.679512] d080  df23d470 df2150a8 7fffffff df23d08c df23d08c df255e00 00000000 00000000
[  664.688738] 
[  664.688738] R7: 0xddc6df80:
[  664.693529] df80  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[  664.702761] dfa0  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[  664.711992] dfc0  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[  664.721223] dfe0  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[  664.730454] e000  00000002 00010001 00000000 ddc27a40 00000000 00000000 ddc27a40 c068d360
[  664.739686] e020  c0688c78 de262000 00000000 fee1dead 00000000 ddc6fda4 ddc6fd80 c04d63ec
[  664.748920] e040  00000000 00000000 00000000 00000000 00000000 01010000 00000000 b6f8f540
[  664.758151] e060  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[  664.767383] 
[  664.767383] R8: 0xdf23d7c4:
[  664.772173] d7c4  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[  664.781404] d7e4  00000000 00000000 00000000 00000000 00000000 00000000 00000000 df23d990
[  664.790635] d804  00000000 00000000 00000000 df23d020 df20ef00 df22fa80 df23d81c df23d81c
[  664.799867] d824  df23d028 df02e140 c0755c04 df224550 00000008 00000007 00000000 c0756bdc
[  664.809092] d844  00000000 df23d848 df23d848 ddc27a40 df23d844 c0755ed0 c076c09c 00000000
[  664.818325] d864  df3fe010 00000000 00000080 df23da70 df2730b0 7fffffff df23d87c df23d87c
[  664.827557] d884  00000000 00000000 00000000 00000000 00000000 c0262db4 df23d810 00000000
[  664.836789] d8a4  00000000 ffffffe0 df23d8ac df23d8ac c0262e4c df23d8b8 df23d8b8 00000000
[  664.846015] 
[  664.846015] R9: 0xfee1de2d:
[  664.850805] de2c  ******** ******** ******** ******** ******** ******** ******** ********
[  664.860046] de4c  ******** ******** ******** ******** ******** ******** ******** ********
[  664.869279] de6c  ******** ******** ******** ******** ******** ******** ******** ********
[  664.878511] de8c  ******** ******** ******** ******** ******** ******** ******** ********
[  664.887743] deac  ******** ******** ******** ******** ******** ******** ******** ********
[  664.896976] decc  ******** ******** ******** ******** ******** ******** ******** ********
[  664.906208] deec  ******** ******** ******** ******** ******** ******** ******** ********
[  664.915441] df0c  ******** ******** ******** ******** ******** ******** ******** ********
[  664.924673] df2c  ******** ******** ******** ******** ******** ******** ******** ********
[  664.933913] CPU: 0 PID: 980 Comm: init Not tainted 4.9.118 #118
[  664.940553] Hardware name: sun8iw19
[  664.944494] [<c00158d8>] (unwind_backtrace) from [<c0012ba4>] (show_stack+0x10/0x14)
[  664.953203] [<c0012ba4>] (show_stack) from [<c0075484>] (watchdog_timer_fn+0x1ec/0x27c)
[  664.962203] [<c0075484>] (watchdog_timer_fn) from [<c005dd74>] (__hrtimer_run_queues.constprop.4+0xdc/0x184)
[  664.973247] [<c005dd74>] (__hrtimer_run_queues.constprop.4) from [<c005dfdc>] (hrtimer_interrupt+0xa8/0x230)
[  664.984290] [<c005dfdc>] (hrtimer_interrupt) from [<c035708c>] (arch_timer_handler_phys+0x28/0x30)
[  664.994360] [<c035708c>] (arch_timer_handler_phys) from [<c00520b0>] (handle_percpu_devid_irq+0x70/0x118)
[  665.005113] [<c00520b0>] (handle_percpu_devid_irq) from [<c004de0c>] (generic_handle_irq+0x18/0x28)
[  665.015279] [<c004de0c>] (generic_handle_irq) from [<c004e394>] (__handle_domain_irq+0xb0/0xcc)
[  665.025053] [<c004e394>] (__handle_domain_irq) from [<c0009494>] (gic_handle_irq+0x44/0x64)
[  665.034433] [<c0009494>] (gic_handle_irq) from [<c001348c>] (__irq_svc+0x6c/0xa8)
[  665.042830] Exception stack(0xddc6fe08 to 0xddc6fe50)
[  665.048504] fe00:                   00000023 00000002 00000001 00000000 df23d81c df23d810
[  665.057689] fe20: df23d020 ddc6e000 df23d844 fee1dead 00000000 00000000 c0793238 ddc6fe58
[  665.066868] fe40: c0324c24 c0324c24 60000013 ffffffff
[  665.072547] [<c001348c>] (__irq_svc) from [<c0324c24>] (axp2101_shutdown+0x24/0x30)
[  665.081154] [<c0324c24>] (axp2101_shutdown) from [<c0257ecc>] (device_shutdown+0x1b0/0x200)
[  665.090542] [<c0257ecc>] (device_shutdown) from [<c003ba24>] (kernel_power_off+0x2c/0x70)
[  665.099732] [<c003ba24>] (kernel_power_off) from [<c003bc08>] (SyS_reboot+0x158/0x1d4)
[  665.108628] [<c003bc08>] (SyS_reboot) from [<c000ed80>] (ret_fast_syscall+0x0/0x48)
[  665.117217] Kernel panic - not syncing: softlockup: hung tasks
[  665.123766] CPU: 0 PID: 980 Comm: init Tainted: G             L  4.9.118 #118
[  665.131770] Hardware name: sun8iw19
[  665.135696] [<c00158d8>] (unwind_backtrace) from [<c0012ba4>] (show_stack+0x10/0x14)
[  665.144397] [<c0012ba4>] (show_stack) from [<c0078798>] (panic+0xac/0x278)
[  665.152123] [<c0078798>] (panic) from [<c00754a4>] (watchdog_timer_fn+0x20c/0x27c)
[  665.160631] [<c00754a4>] (watchdog_timer_fn) from [<c005dd74>] (__hrtimer_run_queues.constprop.4+0xdc/0x184)
[  665.171671] [<c005dd74>] (__hrtimer_run_queues.constprop.4) from [<c005dfdc>] (hrtimer_interrupt+0xa8/0x230)
[  665.182713] [<c005dfdc>] (hrtimer_interrupt) from [<c035708c>] (arch_timer_handler_phys+0x28/0x30)
[  665.192778] [<c035708c>] (arch_timer_handler_phys) from [<c00520b0>] (handle_percpu_devid_irq+0x70/0x118)
[  665.203527] [<c00520b0>] (handle_percpu_devid_irq) from [<c004de0c>] (generic_handle_irq+0x18/0x28)
[  665.213691] [<c004de0c>] (generic_handle_irq) from [<c004e394>] (__handle_domain_irq+0xb0/0xcc)
[  665.223463] [<c004e394>] (__handle_domain_irq) from [<c0009494>] (gic_handle_irq+0x44/0x64)
[  665.232842] [<c0009494>] (gic_handle_irq) from [<c001348c>] (__irq_svc+0x6c/0xa8)
[  665.241239] Exception stack(0xddc6fe08 to 0xddc6fe50)
[  665.246911] fe00:                   00000023 00000002 00000001 00000000 df23d81c df23d810
[  665.256094] fe20: df23d020 ddc6e000 df23d844 fee1dead 00000000 00000000 c0793238 ddc6fe58
[  665.265274] fe40: c0324c24 c0324c24 60000013 ffffffff
[  665.270952] [<c001348c>] (__irq_svc) from [<c0324c24>] (axp2101_shutdown+0x24/0x30)
[  665.279555] [<c0324c24>] (axp2101_shutdown) from [<c0257ecc>] (device_shutdown+0x1b0/0x200)
[  665.288939] [<c0257ecc>] (device_shutdown) from [<c003ba24>] (kernel_power_off+0x2c/0x70)
[  665.298128] [<c003ba24>] (kernel_power_off) from [<c003bc08>] (SyS_reboot+0x158/0x1d4)
[  665.307024] [<c003bc08>] (SyS_reboot) from [<c000ed80>] (ret_fast_syscall+0x0/0x48)
[  665.315636] ---[ end Kernel panic - not syncing: softlockup: hung tasks

枫之所向

关注

0
点赞
踩
2

收藏

觉得还不错? 一键收藏
0
评论
Linux soft lockup

Linux soft lockup分析关键词：watchdog、soft lockup、percpu thread、lockdep等。近日遇到一个soft lockup问题，打印类似“[ 80.002856] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [poweroff:965]“。这是lockup检测机制在起作用，lockup检测机制包括soft lockup detector和hard lockup detector。借机分析
复制链接

扫一扫

专栏目录