关于单核/多核死机问题

这个我好像学过

已于 2023-10-05 23:27:01 修改

阅读量987

点赞数 1

文章标签：单片机嵌入式硬件

于 2023-08-08 22:38:34 首次发布

本文链接：https://blog.csdn.net/qq_42693685/article/details/132177585

版权

以下信息来自https://www.cnblogs.com/jerry116/p/8799355.html。另外还未看代码进行验证

https://www.cnblogs.com/sky-heaven/p/16616530.html

linux-3.16

1、对于非抢占内核。如果代码中出现死锁（未屏蔽软中断、本cpu中断）或者死循环。那么出现死锁的这个cpu将一直卡住，无法进行任务调度。

对于这种情况需要一种检测机制去发现这种问题。具体就是每个核有一个喂狗线程（优先级最高？）和一个喂狗软中断（定时器？？）。喂狗线程负责不断更新时间戳。喂狗软中断负责检测时间戳是否更新。

即使出现了cpu无法调度的情况（不考虑中断被屏蔽）。由于软中断能够打断线程或者进程的运行。当喂狗软中断触发时，先去检查上一次喂狗线程的时间。如果当前时间距离上一次喂狗的时间超过了一定的阈值。则认为未进行任务调度，打印调用栈信息（由于任务切换不了，那么软中断打断的一定是死锁的地方）。如果时间戳未超，说明喂狗线程还能被调度，cpu还能进行任务切换

编译内核的时候需要加上这些配置

static struct smp_hotplug_thread watchdog_threads = {
	.store			= &softlockup_watchdog,
	.thread_should_run	= watchdog_should_run,
	.thread_fn		= watchdog,
	.thread_comm		= "watchdog/%u",
	.setup			= watchdog_enable,
	.cleanup		= watchdog_cleanup,
	.park			= watchdog_disable,
	.unpark			= watchdog_enable,
};

喂狗线程

lockup_detector_init会为每个cpu创建一个watchdog内核线程

void __init lockup_detector_init(void)
{
	set_sample_period();

	if (watchdog_user_enabled)
		watchdog_enable_all_cpus(false);
}
static int watchdog_enable_all_cpus(bool sample_period_changed)
{
.................
		err = smpboot_register_percpu_thread(&watchdog_threads);
........................
	return err;
}

__smpboot_create_thread->smpboot_thread_fn。

static int smpboot_thread_fn(void *data)
{
.......................................
			set_current_state(TASK_RUNNING);
			preempt_enable();
			ht->thread_fn(td->cpu);
...............................
}

其中ht->thread_fn就对应了回调函数watchdog。可以看到喂狗线程的回调函数更新了一下时间戳（__touch_watchdog）。

那喂狗线程何时被调度呢？下面进行解析

static void watchdog(unsigned int cpu)
{
	__this_cpu_write(soft_lockup_hrtimer_cnt,
			 __this_cpu_read(hrtimer_interrupts));
	__touch_watchdog();
}
static void __touch_watchdog(void)
{
	__this_cpu_write(watchdog_touch_ts, get_timestamp());
}

喂狗软中断（其实就是高精度定时器），检测cpu是否无法调度

linux3.16 watchdog_enable是在这个地方进行注册的

watchdog_enable在smpboot_thread_fn第一次执行时得到执行，随后其状态设为HP_THREAD_ACTIVE。

static void watchdog_enable(unsigned int cpu)
{
	struct hrtimer *hrtimer = &__raw_get_cpu_var(watchdog_hrtimer);

	/* kick off the timer for the hardlockup detector */
	hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	hrtimer->function = watchdog_timer_fn;//高精度定时器回调函数

	/* Enable the perf event */
	watchdog_nmi_enable(cpu);
	/* 启动一个定时器 */
	/* done here because hrtimer_start can only pin to smp_processor_id() */
	hrtimer_start(hrtimer, ns_to_ktime(sample_period),
		      HRTIMER_MODE_REL_PINNED);

	/* initialize timestamp */
	watchdog_set_prio(SCHED_FIFO, MAX_RT_PRIO - 1);
	__touch_watchdog();
}

高精度定时器，看网上说都是在中断上下文中执行的（中断上半部以及软中断中）

linux 内核高精度定时器（hrtimer）实现机制_hrtimer_start_老王不让用的博客-CSDN博客
HRTIMER_MODE_SOFT：表示该定时器是否是“软”的，也就是定时器到期回调函数是在软中断（HRTIMER_SOFTIRQ，高精度定时器）下被执行的。
HRTIMER_MODE_HARD：表示该定时器是否是“硬”的，也就是定时器到期回调函数是
在中断处理程序中被执行的。看打印好像确实是这样的看打印好像确实是这样的
static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
{
.........................
		printk(KERN_EMERG "\r\n softlockup simulate, in_interrupt %u in_softirq %u, cpu id %d\n", in_interrupt(), in_softirq(), smp_processor_id());
........................

	return HRTIMER_RESTART;
}

从函数watchdog_enable里面可以看到高精度定时器的回调函数是watchdog_timer_fn

1、可以看到喂狗线程就是在高精度定时器里面被唤醒的。

wake_up_process(__this_cpu_read(softlockup_watchdog));

2、定时器的下一次触发时间也是在这里更新

hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period));

3、检测是否发生soft lockup也是在这个里面is_softlockup

/* 高精度定时器的回调是在中断上下文执行的 */
/* watchdog kicker functions */
static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
{
	unsigned long touch_ts = __this_cpu_read(watchdog_touch_ts);
	struct pt_regs *regs = get_irq_regs();
	int duration;
	int softlockup_all_cpu_backtrace = sysctl_softlockup_all_cpu_backtrace;

	/* kick the hardlockup detector */
	watchdog_interrupt_count();//增加计数,这个应该是hard lockup使用
	/* 唤醒喂狗线程 */
	/* kick the softlockup detector */
	wake_up_process(__this_cpu_read(softlockup_watchdog));

	/* 增加高精度定时器的超时时间,并重启定时器 */
	/* .. and repeat */
	hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period));

	if (touch_ts == 0) {//为什么时间戳会是0呢？难道是第一次执行？？
		if (unlikely(__this_cpu_read(softlockup_touch_sync))) {
			/*
			 * If the time stamp was touched atomically
			 * make sure the scheduler tick is up to date.
			 */
			__this_cpu_write(softlockup_touch_sync, false);
			sched_clock_tick();
		}

		/* Clear the guest paused flag on watchdog reset */
		kvm_check_and_clear_guest_paused();
		__touch_watchdog();
		return HRTIMER_RESTART;
	}
	 /* 现在时间到上次时间戳的位置已经超了 */
	duration = is_softlockup(touch_ts);
	if (unlikely(duration)) {//非0说明已经触发检测机制,cpu在规定时间内未进行调度
		if (kvm_check_and_clear_guest_paused())
			return HRTIMER_RESTART;

		/* only warn once */
		if (__this_cpu_read(soft_watchdog_warn) == true)
			return HRTIMER_RESTART;

		if (softlockup_all_cpu_backtrace) {
			if (test_and_set_bit(0, &soft_lockup_nmi_warn)) {
				/* Someone else will report us. Let's give up */
				__this_cpu_write(soft_watchdog_warn, true);
				return HRTIMER_RESTART;
			}
		}

		printk(KERN_EMERG "BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n",
			smp_processor_id(), duration,
			current->comm, task_pid_nr(current));
		print_modules();
		print_irqtrace_events(current);
		if (regs)
			show_regs(regs);
		else
			dump_stack();

		if (softlockup_all_cpu_backtrace) {
			trigger_allbutself_cpu_backtrace();

			clear_bit(0, &soft_lockup_nmi_warn);
			/* Barrier to sync with other cpus */
			smp_mb__after_atomic();
		}

		if (softlockup_panic)
			panic("softlockup: hung tasks");
		__this_cpu_write(soft_watchdog_warn, true);
	} else
		__this_cpu_write(soft_watchdog_warn, false);

	return HRTIMER_RESTART;
}

static int is_softlockup(unsigned long touch_ts)
{
	unsigned long now = get_timestamp();
	/* 当前时间已经超过上一次的时间戳 +  检测阈值   */
	/* Warn about unreasonable delays: */
	if (time_after(now, touch_ts + get_softlockup_thresh()))
		return now - touch_ts;

	return 0;
}

可以看到基于软件实现的喂狗原理大致如下：

首先存在一个喂狗线程，和一个喂狗软中断（其实就是高精度定时器）。其中喂狗线程被调度的时候就更新时间戳watchdog_touch_ts（per cpu变量，每个cpu都有一个）。喂狗定时器里面负责唤醒喂狗线程去更新时间戳，并且检查当前时间和时间戳的差值。如果差值超过一定的阈值，则认为触发soft lockup.

代码示例：非抢占内核

从调用栈也能看到是test_thread里面出了问题

创建了一个内核线程进行死循环

static struct task_struct *test_task;
int test_thread(void* a)
{
    unsigned long flags;
    printk(KERN_EMERG "\r\n softlockup simulate, in_interrupt %u in_softirq %u, cpu id %d\n", in_interrupt(), in_softirq(), smp_processor_id());
    while (1){}
    return 0;
}

 softlockup simulate, in_interrupt 0 in_softirq 0, cpu id 3
BUG: soft lockup - CPU#3 stuck for 23s! [test_task:586]
Modules linked in:

CPU: 3 PID: 586 Comm: test_task Not tainted 3.16.0 #41
task: ee01c800 ti: ee206000 task.ti: ee206000
PC is at test_thread+0x30/0x38
LR is at test_thread+0x30/0x38
pc : [<c0305d58>]    lr : [<c0305d58>]    psr: 60000013
sp : ee207f60  ip : 00000001  fp : 00000000
r10: 00000000  r9 : 00000000  r8 : 00000000
r7 : c0305d28  r6 : 00000000  r5 : 00000000  r4 : ee1fd400
r3 : 000004f0  r2 : c089f494  r1 : 20000093  r0 : 0000003d
Flags: nZCv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment kernel
Control: 10c53c7d  Table: 8e2f806a  DAC: 00000015
CPU: 3 PID: 586 Comm: test_task Not tainted 3.16.0 #41
[<c00142d0>] (unwind_backtrace) from [<c0010f64>] (show_stack+0x10/0x14)
[<c0010f64>] (show_stack) from [<c045cb60>] (dump_stack+0x74/0x90)
[<c045cb60>] (dump_stack) from [<c0087e08>] (watchdog_timer_fn+0x158/0x1bc)
[<c0087e08>] (watchdog_timer_fn) from [<c003f1d0>] (hrtimer_run_queues+0xcc/0x23c)
[<c003f1d0>] (hrtimer_run_queues) from [<c002c244>] (run_local_timers+0x8/0x14)
[<c002c244>] (run_local_timers) from [<c002c27c>] (update_process_times+0x2c/0x58)
[<c002c27c>] (update_process_times) from [<c006a8e8>] (tick_periodic+0x34/0xbc)
[<c006a8e8>] (tick_periodic) from [<c006a9e4>] (tick_handle_periodic+0x2c/0x94)
[<c006a9e4>] (tick_handle_periodic) from [<c00138d8>] (twd_handler+0x2c/0x40)
[<c00138d8>] (twd_handler) from [<c005cd88>] (handle_percpu_devid_irq+0x68/0x84)
[<c005cd88>] (handle_percpu_devid_irq) from [<c00594e0>] (generic_handle_irq+0x20/0x30)
[<c00594e0>] (generic_handle_irq) from [<c000ecf4>] (handle_IRQ+0x38/0x94)
[<c000ecf4>] (handle_IRQ) from [<c00085b0>] (gic_handle_irq+0x28/0x5c)
[<c00085b0>] (gic_handle_irq) from [<c0011a40>] (__irq_svc+0x40/0x50)
Exception stack(0xee207f18 to 0xee207f60)
7f00:                                                       0000003d 20000093
7f20: c089f494 000004f0 ee1fd400 00000000 00000000 c0305d28 00000000 00000000
7f40: 00000000 00000000 00000001 ee207f60 c0305d58 c0305d58 60000013 ffffffff
[<c0011a40>] (__irq_svc) from [<c0305d58>] (test_thread+0x30/0x38)
[<c0305d58>] (test_thread) from [<c003c49c>] (kthread+0xcc/0xe8)
[<c003c49c>] (kthread) from [<c000e4b8>] (ret_from_fork+0x14/0x3c)
Kernel panic - not syncing: softlockup: hung tasks
CPU: 3 PID: 586 Comm: test_task Not tainted 3.16.0 #41
[<c00142d0>] (unwind_backtrace) from [<c0010f64>] (show_stack+0x10/0x14)
[<c0010f64>] (show_stack) from [<c045cb60>] (dump_stack+0x74/0x90)
[<c045cb60>] (dump_stack) from [<c0459fb8>] (panic+0x90/0x204)
[<c0459fb8>] (panic) from [<c0087e50>] (watchdog_timer_fn+0x1a0/0x1bc)
[<c0087e50>] (watchdog_timer_fn) from [<c003f1d0>] (hrtimer_run_queues+0xcc/0x23c)
[<c003f1d0>] (hrtimer_run_queues) from [<c002c244>] (run_local_timers+0x8/0x14)
[<c002c244>] (run_local_timers) from [<c002c27c>] (update_process_times+0x2c/0x58)
[<c002c27c>] (update_process_times) from [<c006a8e8>] (tick_periodic+0x34/0xbc)
[<c006a8e8>] (tick_periodic) from [<c006a9e4>] (tick_handle_periodic+0x2c/0x94)
[<c006a9e4>] (tick_handle_periodic) from [<c00138d8>] (twd_handler+0x2c/0x40)
[<c00138d8>] (twd_handler) from [<c005cd88>] (handle_percpu_devid_irq+0x68/0x84)
[<c005cd88>] (handle_percpu_devid_irq) from [<c00594e0>] (generic_handle_irq+0x20/0x30)
[<c00594e0>] (generic_handle_irq) from [<c000ecf4>] (handle_IRQ+0x38/0x94)
[<c000ecf4>] (handle_IRQ) from [<c00085b0>] (gic_handle_irq+0x28/0x5c)
[<c00085b0>] (gic_handle_irq) from [<c0011a40>] (__irq_svc+0x40/0x50)
Exception stack(0xee207f18 to 0xee207f60)
7f00:                                                       0000003d 20000093
7f20: c089f494 000004f0 ee1fd400 00000000 00000000 c0305d28 00000000 00000000
7f40: 00000000 00000000 00000001 ee207f60 c0305d58 c0305d58 60000013 ffffffff
[<c0011a40>] (__irq_svc) from [<c0305d58>] (test_thread+0x30/0x38)
[<c0305d58>] (test_thread) from [<c003c49c>] (kthread+0xcc/0xe8)
[<c003c49c>] (kthread) from [<c000e4b8>] (ret_from_fork+0x14/0x3c)
CPU0: stopping
CPU: 0 PID: 0 Comm: swapper/0 Not tainted 3.16.0 #41
[<c00142d0>] (unwind_backtrace) from [<c0010f64>] (show_stack+0x10/0x14)
[<c0010f64>] (show_stack) from [<c045cb60>] (dump_stack+0x74/0x90)
[<c045cb60>] (dump_stack) from [<c0012e9c>] (handle_IPI+0x134/0x170)
[<c0012e9c>] (handle_IPI) from [<c00085dc>] (gic_handle_irq+0x54/0x5c)
[<c00085dc>] (gic_handle_irq) from [<c0011a40>] (__irq_svc+0x40/0x50)
Exception stack(0xc088ff60 to 0xc088ffa8)
ff60: ffffffed 00000000 ffffffed 00000000 c088e000 00000000 00000000 c0896464
ff80: c0463dc4 00000000 c088cb30 0000004c 00000000 c088ffa8 c000efd4 c000efd8
ffa0: 60000013 ffffffff
[<c0011a40>] (__irq_svc) from [<c000efd8>] (arch_cpu_idle+0x28/0x30)
[<c000efd8>] (arch_cpu_idle) from [<c0052150>] (cpu_startup_entry+0x1ac/0x1f0)
[<c0052150>] (cpu_startup_entry) from [<c05d1b3c>] (start_kernel+0x328/0x38c)
CPU1: stopping
CPU: 1 PID: 0 Comm: swapper/1 Not tainted 3.16.0 #41
[<c00142d0>] (unwind_backtrace) from [<c0010f64>] (show_stack+0x10/0x14)
[<c0010f64>] (show_stack) from [<c045cb60>] (dump_stack+0x74/0x90)
[<c045cb60>] (dump_stack) from [<c0012e9c>] (handle_IPI+0x134/0x170)
[<c0012e9c>] (handle_IPI) from [<c00085dc>] (gic_handle_irq+0x54/0x5c)
[<c00085dc>] (gic_handle_irq) from [<c0011a40>] (__irq_svc+0x40/0x50)
Exception stack(0xee891f90 to 0xee891fd8)
1f80:                                     ffffffed 00000000 ffffffed 00000000
1fa0: ee890000 00000000 00000000 c0896464 c0463dc4 00000000 c088cb30 0000004c
1fc0: 00000000 ee891fd8 c000efd4 c000efd8 60000013 ffffffff
[<c0011a40>] (__irq_svc) from [<c000efd8>] (arch_cpu_idle+0x28/0x30)
[<c000efd8>] (arch_cpu_idle) from [<c0052150>] (cpu_startup_entry+0x1ac/0x1f0)
[<c0052150>] (cpu_startup_entry) from [<60008684>] (0x60008684)
CPU2: stopping
CPU: 2 PID: 0 Comm: swapper/2 Not tainted 3.16.0 #41
[<c00142d0>] (unwind_backtrace) from [<c0010f64>] (show_stack+0x10/0x14)
[<c0010f64>] (show_stack) from [<c045cb60>] (dump_stack+0x74/0x90)
[<c045cb60>] (dump_stack) from [<c0012e9c>] (handle_IPI+0x134/0x170)
[<c0012e9c>] (handle_IPI) from [<c00085dc>] (gic_handle_irq+0x54/0x5c)
[<c00085dc>] (gic_handle_irq) from [<c0011a40>] (__irq_svc+0x40/0x50)
Exception stack(0xee893f90 to 0xee893fd8)
3f80:                                     ffffffed 00000000 ffffffed 00000000
3fa0: ee892000 00000000 00000000 c0896464 c0463dc4 00000000 c088cb30 0000004c
3fc0: 00000000 ee893fd8 c000efd4 c000efd8 60000013 ffffffff
[<c0011a40>] (__irq_svc) from [<c000efd8>] (arch_cpu_idle+0x28/0x30)
[<c000efd8>] (arch_cpu_idle) from [<c0052150>] (cpu_startup_entry+0x1ac/0x1f0)
[<c0052150>] (cpu_startup_entry) from [<60008684>] (0x60008684)
PANIC: softlockup: hung tasks

Entering kdb (current=0xee01c800, pid 586) on processor 3 due to Keyboard Entry
[3]kdb>

2、单核挂死并屏蔽了本cpu中断的情况。eg local_irq_save, spin_lock_irqsave。对于这种情况由于CPU无法接受到中断信息了。显然中断都无法接收了，上面第一种方法就失效了。但是某些芯片有不可屏蔽中断NMI。

在多核系统里面，每个核都可以去检测其他核的中断接受情况，如果检测到某个核未接受中断了，就可以给该核发生一个不可屏蔽中断的消息，同样在这个中断处理函数里面把调用栈打出来。

该方法有部分也是和方法1公用的

watchdog_enable->watchdog_nmi_enable

static void watchdog_enable(unsigned int cpu)
{
	struct hrtimer *hrtimer = &__raw_get_cpu_var(watchdog_hrtimer);

	....................
	/* Enable the perf event */
	watchdog_nmi_enable(cpu);
..............................
}

watchdog_nmi_enable里面注册了一个hard lockup检测事件。其回调函数是watchdog_overflow_callback

我感觉可能arm也是这个样子把。看x86就是每个一段时间发出一个NMI中断。然后在回调函数里面检查中断触发的次数是否增加吧

这个硬件在x86里叫performance monitoring，这个硬件有一个功能就是在cpu clock经过了多少个周期后发出一个NMI中断出来。

static int watchdog_nmi_enable(unsigned int cpu)
{
	......................
	wd_attr->sample_period = hw_nmi_get_sample_period(watchdog_thresh);

	/* Try to register using hardware perf events */
	event = perf_event_create_kernel_counter(wd_attr, cpu, NULL, watchdog_overflow_callback, NULL);

	............................
}

hard lockup判定：watchdog_overflow_callback->is_hardlockup

is_hardlockup：可以看到它先去读取中断被触发的次数。然后再去比较上一次NMI中断触发时保存的中断次数（hrtimer_interrupts_saved）。如果相等说明出现了hard lockup.

另外hrtimer_interrupts这个变量在方法1的喂狗软中断里面就会更新。

static int is_hardlockup(void)
{
	unsigned long hrint = __this_cpu_read(hrtimer_interrupts);

	if (__this_cpu_read(hrtimer_interrupts_saved) == hrint)
		return 1;

	__this_cpu_write(hrtimer_interrupts_saved, hrint);
	return 0;
}

下面这种hard lockup怎么构造不出来呢？反而一直报rcu相关的问题

int test_thread(void* a)
{
	unsigned long flags;
	printk(KERN_EMERG "\r\n softlockup simulate, in_interrupt %u in_softirq %u, cpu id %d\n", in_interrupt(), in_softirq(), smp_processor_id());
	
	local_irq_disable();
	while (1){}
	
	return 0;
}

[root@arm_test ]# INFO: rcu_sched detected stalls on CPUs/tasks: { 1} (detected by 0, t=8407 jiffies, g=-62, c=-63, q=308)
Task dump for CPU 1:
test_task R running 0 653 2 0x00000002
[<c045de78>] (__schedule) from [<c003c49c>] (kthread+0xcc/0xe8)
[<c003c49c>] (kthread) from [<c000e4b8>] (ret_from_fork+0x14/0x3c)
INFO: rcu_sched detected stalls on CPUs/tasks: { 1} (detected by 0, t=14712 jiffies, g=-62, c=-63, q=308)
Task dump for CPU 1:
test_task R running 0 653 2 0x00000002
[<c045de78>] (__schedule) from [<c003c49c>] (kthread+0xcc/0xe8)
[<c003c49c>] (kthread) from [<c000e4b8>] (ret_from_fork+0x14/0x3c)

通过top和interrupt能看到是哪个进程出问题了。

softlockup simulate, in_interrupt 0 in_softirq 0, cpu id 1//内核线程在1核上

[root@arm_test ]# cat /proc/interrupts
CPU0 CPU1 CPU2 CPU3
29: 9131 1444 9128 9112 GIC 29 twd

root@arm_test ]# cat /proc/interrupts
CPU0 CPU1 CPU2 CPU3
29: 9357 1444 9350 9332 GIC 29 twd
34: 6 0 0 0 GIC 34 timer
可以看到cpu1的twd中断不增加

top可以看到test_task一直占用cpu1

Mem: 19528K used, 1013232K free, 0K shrd, 252K buff, 8944K cached
CPU: 0.3% usr 1.5% sys 0.0% nic 96.4% idle 0.0% io 0.0% irq 1.5% sirq
Load average: 1.65 0.55 0.19 2/54 787
PID PPID USER STAT VSZ %VSZ CPU %CPU COMMAND
653 2 0 RW 0 0.0 1 3.2 [test_task]
787 781 0 R 2432 0.2 0 0.2 top
7 2 0 SW 0 0.0 2 0.0 [rcu_sched]

后面知道为啥构造不出来了

hard lockup：需要CPU支持NMI（不可屏蔽中断，通常是通过CPU里的PMU单元实现的），如果PMU发现长时间（这个cycle是借助NMI来计算的，因为定时器可能不工作了）一个中断都不来，就知道发生了hard lockup，这时（触发NMI中断，中断处理函数中）分析栈就知道在哪里锁住中断的。需要把CONFIG_HARDLOCKUP_DETECTOR打开。

由于ARM里面没有NMI，因此内核不支持ARM的hard lockup detector。但有一些内核patch可以用，比如用FIQ模拟MNI（如果FIQ用于其他地方了，这里就用不了了），或者用CPU1去检测CPU0是否被hard lockup(但CPU1没办法获得线程的栈，只能知道lockup了)，但这两个patch都没在主线上。FIQ在Linux中基本不用的（一般只做特殊的debugger，常规代码不用）。
————————————————
版权声明：本文为CSDN博主「落尘纷扰」的原创文章，遵循CC 4.0 BY-SA版权协议，转载请附上原文出处链接及本声明。
原文链接：https://blog.csdn.net/jasonchen_gbd/article/details/79465405

如果没有这种不可屏蔽中断的芯片（例如arm），这种怎么处理呢？？要不明天试试自己构造一个检测的

还是利用is_hardlockup里面的两个每cpu变量。起一个高精度定时器。每隔一段时间检查一下这两个值是不是相等。如果是相等的说明该cpu关中断了。唯一需要注意的是：假设cpu0关中断了，它自己是不能发现的，只有其他cpu帮忙检查才行。

主要代码如下watch_dog.c：

主要使用者三个变量

/*****************************************/
static DEFINE_PER_CPU(struct hrtimer, hardlock_check_hrtimer);
static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts);
static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts_saved);
/*****************************************/

watchdog_enable里面再起一个定时器，检查是否有hardlockup

static void watchdog_enable(unsigned int cpu)
{
.............................................................

	/* 添加hardlock定时器,用于检查hardlock的情况 */
	hrtimer = &__raw_get_cpu_var(hardlock_check_hrtimer);
	/* kick off the timer for the hardlockup detector */
	
	hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	hrtimer->function = hardlock_check_callback;

	__this_cpu_write(hrtimer_interrupts_saved, 0);
	__this_cpu_write(hrtimer_interrupts, 0);
	/* done here because hrtimer_start can only pin to smp_processor_id() */
	hrtimer_start(hrtimer, ns_to_ktime(sample_period * 2),
		      HRTIMER_MODE_REL_PINNED);
}

定时器回调函数

static void hardlock_check_callback(struct hrtimer *hrtimer)
{

	/* check for a hardlockup
	 * This is done by making sure our timer interrupt
	 * is incrementing.  The timer interrupt should have
	 * fired multiple times before we overflow'd.  If it hasn't
	 * then this is a good indication the cpu is stuck
	 */
	if (is_hardlockup()) {
		int this_cpu = smp_processor_id();
			WARN(1, "CPU %d happen hard LOCKUP detect by cpu %d",  (this_cpu + 1) % 4, this_cpu);

		//return;
	}
	/* 增加高精度定时器的超时时间,并重启定时器 */
	/* .. and repeat */
	hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period * 2));
	return;
}

watchdog_timer_fn->watchdog_interrupt_count:这个里面会更新这个值的，所以我们不用管自己用即可hrtimer_interrupts

cpu0->cpu1->cpu2->cpu3。cpu3反过来监控cpu0的计数

static int is_hardlockup(void)
{
    /* 每次进来先更新当前cpu的中断计数 */
	unsigned long cur_hrint = __this_cpu_read(hrtimer_interrupts);
	__this_cpu_write(hrtimer_interrupts_saved, cur_hrint);

    /* 读取需要监控的cpu的计数，判断是否出现hardlock */
	int cpu = smp_processor_id();
    int check_cpu = (cpu + 1) % 4;
	unsigned long hrint = per_cpu(hrtimer_interrupts, check_cpu);
	unsigned long saved_hrint = per_cpu(hrtimer_interrupts_saved, check_cpu);

	if (saved_hrint == hrint)
	{
		return 1;
	}
	/*
    更新监控的cpu计数
    当我监控完之后，该cpu出现了一直关中断的情况，如果不更新，那么interrupts和saved是一直不相等的	 
	*/
	per_cpu(hrtimer_interrupts_saved, check_cpu) = hrint;
	return 0;
}

测试代码

int test_thread(void* a)
{
	unsigned long flags;
	printk(KERN_EMERG "\r\n softlockup simulate, in_interrupt %u in_softirq %u, cpu id %d\n", in_interrupt(), in_softirq(), smp_processor_id());
	
	local_irq_disable();
	while (1){}
	//f1(10, 20);
	return 0;
}

运行在cpu1上，cpu0负责监控cpu1（实测有误报的情况）