softlockup(watchdog)用于检测系统调度是否正常,即软锁的情况,当发生softlockup时,内核不能调度,但还能响应中断,对用户的表现可能为:能ping通,但无法登陆系统,无法进行正常操作。
其基本原理为:
为每个CPU启动一个内核线程(watchdog/x)
,
此线程为优先级最高的实时线程,在该线程得到调度时,会更新相应的计数(时间戳),同时会启动定时器,当定时器到期时检查相应的时间戳,如果超过指定时间,都没有更新,则说明这段时间内都没有发生调度(因为此线程优先级最高)
,则打印相应告警或根据配置可以进入panic流程。
基本代码分析(2.6.32)
rest_init->kernel_init->lockup_detector_init->cpu_callback->watchdog_prepare_cpu
(初始化watchdog定时器):
- static int watchdog_prepare_cpu(int cpu)
- {
- struct hrtimer *hrtimer = &per_cpu(watchdog_hrtimer, cpu);
-
- WARN_ON(per_cpu(softlockup_watchdog, cpu));
- hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);//初始化高精度定时器
- hrtimer->function = watchdog_timer_fn; //设置定时器处理函数
-
- return 0;
- }
看门狗定时器处理函数:
/* watchdog kicker functions */
.hrtimer定时器调用watchdog_timer_fn进行清狗的时间检查,而线程则每次重置清狗时间,如果watchdog_timer_fn发现狗的重置时间已经和当前时间差出危险值,则根据开关进行panic处理。
- static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
- {
- //获取计数watchdog_touch_ts,该计数在watchdog内核线程被调度时更新
- unsigned longtouch_ts = __get_cpu_var(watchdog_touch_ts);
- struct pt_regs *regs = get_irq_regs();
- int duration;
-
- /* kick the hardlockup detector */
- //增加中断计数,证明没有发生硬锁(关中断死锁)
- watchdog_interrupt_count();
-
- /* kick the softlockup detector */
- //唤醒wathdog内核线程
- wake_up_process(__get_cpu_var(softlockup_watchdog));//就是watchdog线程
-
- /* .. and repeat */
- //重启定时器
- hrtimer_forward_now(hrtimer, ns_to_ktime(get_sample_period()));
- if (touch_ts == 0) {
- if (unlikely(__get_cpu_var(softlockup_touch_sync))) {
- /*
- * If the time stamp was touched atomically
- * make sure the scheduler tick is up to date.
- */
- __get_cpu_var(softlockup_touch_sync) = false;
- sched_clock_tick();
- }
- __touch_watchdog();
- return HRTIMER_RESTART;
- }
-
- /* check for a softlockup
- * This is done by making sure a high priority task is
- * being scheduled. The task touches the watchdog to
- * indicate it is getting cpu time. If it hasn't then
- * this is a good indication some task is hogging the cpu
- */
- //判断是否发生了软锁,原理是判断touch_ts(时间戳)是否超过一定时间没有更新
- duration = is_softlockup(touch_ts);
- if (unlikely(duration)) {
- /* only warn once */
- /*soft_watchdog_warn标识会在已经出现了一次看门狗超时的情况下置位,
- 此处的用意是对于同一个死锁进程,内核只做一次报警动作,如果死锁的进程发生了改变,
- 那该标识会重新设置为false,将可以重新触发报警。
- */
- if (__get_cpu_var(soft_watchdog_warn) == true)
- return HRTIMER_RESTART;
-
- //发生了软锁后,进行一些列的信息记录和告警。
- printk(KERN_EMERG "BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n",
- smp_processor_id(), duration,current->comm, task_pid_nr(current));
-
- print_modules();
- print_irqtrace_events(current);
- if (regs)
- show_regs(regs);
- else
- dump_stack();
- //如果配置了softlockup_panic(proc中配置),则panic
- if (softlockup_panic)
- panic("softlockup: hung tasks");
-
- __get_cpu_var(soft_watchdog_warn) = true; //出现了一次超时,softlock。
- } else
- __get_cpu_var(soft_watchdog_warn) = false;
-
- return HRTIMER_RESTART;
- }
启动看门狗,即创建watchdog内核线程。
- static int watchdog_enable(int cpu)
- {
- struct task_struct *p = per_cpu(softlockup_watchdog, cpu);
- int err = 0;
-
- /* enable the perf event */
- err = watchdog_nmi_enable(cpu);
-
- /* Regardless of err above, fall through and start softlockup */
- /* create the watchdog thread */
- if (!p) {
- //创建watchdog内核线程
- p = kthread_create(watchdog, (void *)(unsigned long)cpu, "watchdog/%d", cpu);
- if (IS_ERR(p)) {
- printk(KERN_ERR "softlockup watchdog for %i failed\n", cpu);
- if (!err)
- /* if hardlockup hasn't already set this */
- err = PTR_ERR(p);
- goto out;
- }
- kthread_bind(p, cpu);
- per_cpu(watchdog_touch_ts, cpu) = 0;
- per_cpu(softlockup_watchdog, cpu) = p;
- wake_up_process(p);
- }
-
- out:
- return err;
- }
watchdog内核线程执行主函数,主要是要更新计数(时间戳)
- static int watchdog(void *unused)
- {
- //设置为最高优先级
- struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
- struct hrtimer *hrtimer = &__raw_get_cpu_var(watchdog_hrtimer);
- //设置为实时线程
- sched_setscheduler(current, SCHED_FIFO, ¶m);
-
- /* initialize timestamp */
- //初始化计数(时间戳)
- __touch_watchdog();
-
- /* kick off the timer for the hardlockup detector */
- /* done here because hrtimer_start can only pin to smp_processor_id() */
- //启动定时器,用于检测是否发生软锁
- hrtimer_start(hrtimer, ns_to_ktime(get_sample_period()),HRTIMER_MODE_REL_PINNED);
-
- //睡眠
- set_current_state(TASK_INTERRUPTIBLE);
-
- /*
- * Run briefly once per second to reset the softlockup timestamp.
- * If this gets delayed for more than 60 seconds then the
- * debug-printout triggers in watchdog_timer_fn().
- */
- while (!kthread_should_stop())
- {
- //更新计数watchdog_touch_ts
- __touch_watchdog();
-
- schedule();
-
- if (kthread_should_stop())
- break;
-
- set_current_state(TASK_INTERRUPTIBLE);
- }
- __set_current_state(TASK_RUNNING);
-
- return 0;
- }
/* Commands for resetting the watchdog */
static void __touch_watchdog(void)
{
__this_cpu_write(watchdog_touch_ts, get_timestamp());
}
判断是否发生软锁:
is_softlockup
点击(
此处
)折叠或打开
- static int is_softlockup(unsigned long touch_ts)
- {
- unsigned long now = get_timestamp();
-
- /* Warn about unreasonable delays: */
- /*检测多久没有更新相关计数了,默认阈值为10s,touch_us为上一次更新的时间*/
- if (time_after(now, touch_ts + get_softlockup_thresh()))
- return now - touch_ts;
-
- return 0;
- }
==============================================================================
root@lrc-0:~# ps -ewLo user,pid,ppid,tid,flags,
pri,rtprio,cls,psr,pmem,time,args | grep watchdog ?? why is TS
root63 2 63 5
19 - TS 0 0.0 00:00:00 [watchdog/0]
root64 2 64 5 19 - TS 1 0.0 00:00:00 [watchdog/1]
root71 2 71 5 19 - TS 2 0.0 00:00:00 [watchdog/2]
root78 2 78 5 19 - TS 3 0.0 00:00:00 [watchdog/3]
root85 2 85 5 19 - TS 4 0.0 00:00:00 [watchdog/4]
root92 2 92 5 19 - TS 5 0.0 00:00:00 [watchdog/5]
root99 2 99 5 19 - TS 6 0.0 00:00:00 [watchdog/6]
root106 2 106 5 19 - TS 7 0.0 00:00:00 [watchdog/7]
root113 2 113 5 19 - TS 8 0.0 00:00:00 [watchdog/8]
root120 2 120 5 19 - TS 9 0.0 00:00:00 [watchdog/9]
root127 2 127 5 19 - TS 10 0.0 00:00:00 [watchdog/10]
root134 2 134 5 19 - TS 11 0.0 00:00:00 [watchdog/11]
root141 2 141 5 19 - TS 12 0.0 00:00:00 [watchdog/12]
root148 2 148 5 19 - TS 13 0.0 00:00:00 [watchdog/13]
root155 2 155 5 19 - TS 14 0.0 00:00:00 [watchdog/14]
root162 2 162 5 19 - TS 15 0.0 00:00:00 [watchdog/15]
root10670 1189 10670 0