smpboot_create_threads 会在每一个cpu上建立一个thread。当cpu down的时候对应的thread 会被kill掉.具体分析如下.
566 void __init smp_init(void)
567 {
568 unsigned int cpu;
569
570 idle_threads_init();
571 cpuhp_threads_init();
572
573 /* FIXME: This should be done in userspace --RR */
574 for_each_present_cpu(cpu) {
575 if (num_online_cpus() >= setup_max_cpus)
576 break;
577 if (!cpu_online(cpu))
578 cpu_up(cpu);
579 }
580
581 /* Any cleanup work */
582 smp_announce();
583 smp_cpus_done(setup_max_cpus);
584 }
这个函数中会调用cpu_up 来boot cpu,需要注意的是cpu是从0开始的,即使现在cpu 0已经正常工作了
1011 int cpu_up(unsigned int cpu)
1012 {
1013 _cpu_up(cpu);
1014 }
继续看_cpu_up
453 static int _cpu_up(unsigned int cpu, int tasks_frozen)
454 {
472
473 ret = smpboot_create_threads(cpu);
504 }
这个函数中的会调用 smpboot_create_threads
203 int smpboot_create_threads(unsigned int cpu)
204 {
205 struct smp_hotplug_thread *cur;
206 int ret = 0;
207
208 mutex_lock(&smpboot_threads_lock);
209 list_for_each_entry(cur, &hotplug_threads, list) {
210 ret = __smpboot_create_thread(cur, cpu);
211 if (ret)
212 break;
213 }
214 mutex_unlock(&smpboot_threads_lock);
215 return ret;
216 }
这个函数会从hotplug_threads list中去下每一个smp_hotplug_thread ,那又是在哪里往这个list上添加smp_hotplug_thread呢?
277 int smpboot_register_percpu_thread(struct smp_hotplug_thread *plug_thread)
278 {
279 unsigned int cpu;
280 int ret = 0;
281
282 mutex_lock(&smpboot_threads_lock);
283 for_each_online_cpu(cpu) {
284 ret = __smpboot_create_thread(plug_thread, cpu);
285 if (ret) {
286 smpboot_destroy_threads(plug_thread);
287 goto out;
288 }
289 smpboot_unpark_thread(plug_thread, cpu);
290 }
291 list_add(&plug_thread->list, &hotplug_threads);
292 out:
293 mutex_unlock(&smpboot_threads_lock);
294 return ret;
295 }
原来调用smpboot_register_percpu_thread 函数就可以往hotplug_threads 中添加smp_hotplug_thread
那谁会调用smpboot_register_percpu_thread 呢?
http://lxr.free-electrons.com/source/kernel/stop_machine.c?v=3.18#L542
519 static struct smp_hotplug_thread cpu_stop_threads = {
520 .store = &cpu_stopper_task,
521 .thread_should_run = cpu_stop_should_run,
522 .thread_fn = cpu_stopper_thread,
523 .thread_comm = "migration/%u",
524 .create = cpu_stop_create,
525 .setup = cpu_stop_unpark,
526 .park = cpu_stop_park,
527 .pre_unpark = cpu_stop_unpark,
528 .selfparking = true,
529 };
530
531 static int __init cpu_stop_init(void)
532 {
533 unsigned int cpu;
534
535 for_each_possible_cpu(cpu) {
536 struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
537
538 spin_lock_init(&stopper->lock);
539 INIT_LIST_HEAD(&stopper->works);
540 }
541
542 BUG_ON(smpboot_register_percpu_thread(&cpu_stop_threads));
543 stop_machine_initialized = true;
544 return 0;
545 }
546 early_initcall(cpu_stop_init);
可以看到cpu_stop_init 有调用smpboot_register_percpu_thread在cpuhotplug的时候在每一个cpu上创建一个thread
从523行我们可以看到thread的那么为migration/0,1,2,3等,即每个cpu上都与一个.
我们在看看smpboot_create_threads中__smpboot_create_thread的实现
166 __smpboot_create_thread(struct smp_hotplug_thread *ht, unsigned int cpu)
167 {
168 struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu);
169 struct smpboot_thread_data *td;
170
171 if (tsk)
172 return 0;
173
174 td = kzalloc_node(sizeof(*td), GFP_KERNEL, cpu_to_node(cpu));
175 if (!td)
176 return -ENOMEM;
177 td->cpu = cpu;
178 td->ht = ht;
179
180 tsk = kthread_create_on_cpu(smpboot_thread_fn, td, cpu,
181 ht->thread_comm);
182 if (IS_ERR(tsk)) {
183 kfree(td);
184 return PTR_ERR(tsk);
185 }
186 get_task_struct(tsk);
187 *per_cpu_ptr(ht->store, cpu) = tsk;
188 if (ht->create) {
189 /*
190 * Make sure that the task has actually scheduled out
191 * into park position, before calling the create
192 * callback. At least the migration thread callback
193 * requires that the task is off the runqueue.
194 */
195 if (!wait_task_inactive(tsk, TASK_PARKED))
196 WARN_ON(1);
197 else
198 ht->create(cpu);
199 }
200 return 0;
201 }
调用kthread_create_on_cpu 在当前的cpu上创建一个thread,回调函数是smpboot_thread_fn,ht->thread_comm是thread的名字,就是例子中的migration/0,1,2,3
104 static int smpboot_thread_fn(void *data)
105 {
106 struct smpboot_thread_data *td = data;
107 struct smp_hotplug_thread *ht = td->ht;
108
109 while (1) {
110 set_current_state(TASK_INTERRUPTIBLE);
111 preempt_disable();
112 if (kthread_should_stop()) {
113 set_current_state(TASK_RUNNING);
114 preempt_enable();
115 if (ht->cleanup)
116 ht->cleanup(td->cpu, cpu_online(td->cpu));
117 kfree(td);
118 return 0;
119 }
120
121 if (kthread_should_park()) {
122 __set_current_state(TASK_RUNNING);
123 preempt_enable();
124 if (ht->park && td->status == HP_THREAD_ACTIVE) {
125 BUG_ON(td->cpu != smp_processor_id());
126 ht->park(td->cpu);
127 td->status = HP_THREAD_PARKED;
128 }
129 kthread_parkme();
130 /* We might have been woken for stop */
131 continue;
132 }
133
134 BUG_ON(td->cpu != smp_processor_id());
135
136 /* Check for state change setup */
137 switch (td->status) {
138 case HP_THREAD_NONE:
139 preempt_enable();
140 if (ht->setup)
141 ht->setup(td->cpu);
142 td->status = HP_THREAD_ACTIVE;
143 preempt_disable();
144 break;
145 case HP_THREAD_PARKED:
146 preempt_enable();
147 if (ht->unpark)
148 ht->unpark(td->cpu);
149 td->status = HP_THREAD_ACTIVE;
150 preempt_disable();
151 break;
152 }
153
154 if (!ht->thread_should_run(td->cpu)) {
155 preempt_enable();
156 schedule();
157 } else {
158 set_current_state(TASK_RUNNING);
159 preempt_enable();
160 ht->thread_fn(td->cpu);
161 }
162 }
163 }
从137~152航根据thread的状况来调用不同的函数,如HP_THREAD_NONE,就调用ht->setup(td->cpu);也就是例子中的cpu_stop_unpark。
如果当前thread不是停止状态的话,就跑160行的ht->thread_fn(td->cpu);即例子中的cpu_stopper_thread
566 void __init smp_init(void)
567 {
568 unsigned int cpu;
569
570 idle_threads_init();
571 cpuhp_threads_init();
572
573 /* FIXME: This should be done in userspace --RR */
574 for_each_present_cpu(cpu) {
575 if (num_online_cpus() >= setup_max_cpus)
576 break;
577 if (!cpu_online(cpu))
578 cpu_up(cpu);
579 }
580
581 /* Any cleanup work */
582 smp_announce();
583 smp_cpus_done(setup_max_cpus);
584 }
这个函数中会调用cpu_up 来boot cpu,需要注意的是cpu是从0开始的,即使现在cpu 0已经正常工作了
1011 int cpu_up(unsigned int cpu)
1012 {
1013 _cpu_up(cpu);
1014 }
继续看_cpu_up
453 static int _cpu_up(unsigned int cpu, int tasks_frozen)
454 {
472
473 ret = smpboot_create_threads(cpu);
504 }
这个函数中的会调用 smpboot_create_threads
203 int smpboot_create_threads(unsigned int cpu)
204 {
205 struct smp_hotplug_thread *cur;
206 int ret = 0;
207
208 mutex_lock(&smpboot_threads_lock);
209 list_for_each_entry(cur, &hotplug_threads, list) {
210 ret = __smpboot_create_thread(cur, cpu);
211 if (ret)
212 break;
213 }
214 mutex_unlock(&smpboot_threads_lock);
215 return ret;
216 }
这个函数会从hotplug_threads list中去下每一个smp_hotplug_thread ,那又是在哪里往这个list上添加smp_hotplug_thread呢?
277 int smpboot_register_percpu_thread(struct smp_hotplug_thread *plug_thread)
278 {
279 unsigned int cpu;
280 int ret = 0;
281
282 mutex_lock(&smpboot_threads_lock);
283 for_each_online_cpu(cpu) {
284 ret = __smpboot_create_thread(plug_thread, cpu);
285 if (ret) {
286 smpboot_destroy_threads(plug_thread);
287 goto out;
288 }
289 smpboot_unpark_thread(plug_thread, cpu);
290 }
291 list_add(&plug_thread->list, &hotplug_threads);
292 out:
293 mutex_unlock(&smpboot_threads_lock);
294 return ret;
295 }
原来调用smpboot_register_percpu_thread 函数就可以往hotplug_threads 中添加smp_hotplug_thread
那谁会调用smpboot_register_percpu_thread 呢?
http://lxr.free-electrons.com/source/kernel/stop_machine.c?v=3.18#L542
519 static struct smp_hotplug_thread cpu_stop_threads = {
520 .store = &cpu_stopper_task,
521 .thread_should_run = cpu_stop_should_run,
522 .thread_fn = cpu_stopper_thread,
523 .thread_comm = "migration/%u",
524 .create = cpu_stop_create,
525 .setup = cpu_stop_unpark,
526 .park = cpu_stop_park,
527 .pre_unpark = cpu_stop_unpark,
528 .selfparking = true,
529 };
530
531 static int __init cpu_stop_init(void)
532 {
533 unsigned int cpu;
534
535 for_each_possible_cpu(cpu) {
536 struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
537
538 spin_lock_init(&stopper->lock);
539 INIT_LIST_HEAD(&stopper->works);
540 }
541
542 BUG_ON(smpboot_register_percpu_thread(&cpu_stop_threads));
543 stop_machine_initialized = true;
544 return 0;
545 }
546 early_initcall(cpu_stop_init);
可以看到cpu_stop_init 有调用smpboot_register_percpu_thread在cpuhotplug的时候在每一个cpu上创建一个thread
从523行我们可以看到thread的那么为migration/0,1,2,3等,即每个cpu上都与一个.
我们在看看smpboot_create_threads中__smpboot_create_thread的实现
166 __smpboot_create_thread(struct smp_hotplug_thread *ht, unsigned int cpu)
167 {
168 struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu);
169 struct smpboot_thread_data *td;
170
171 if (tsk)
172 return 0;
173
174 td = kzalloc_node(sizeof(*td), GFP_KERNEL, cpu_to_node(cpu));
175 if (!td)
176 return -ENOMEM;
177 td->cpu = cpu;
178 td->ht = ht;
179
180 tsk = kthread_create_on_cpu(smpboot_thread_fn, td, cpu,
181 ht->thread_comm);
182 if (IS_ERR(tsk)) {
183 kfree(td);
184 return PTR_ERR(tsk);
185 }
186 get_task_struct(tsk);
187 *per_cpu_ptr(ht->store, cpu) = tsk;
188 if (ht->create) {
189 /*
190 * Make sure that the task has actually scheduled out
191 * into park position, before calling the create
192 * callback. At least the migration thread callback
193 * requires that the task is off the runqueue.
194 */
195 if (!wait_task_inactive(tsk, TASK_PARKED))
196 WARN_ON(1);
197 else
198 ht->create(cpu);
199 }
200 return 0;
201 }
调用kthread_create_on_cpu 在当前的cpu上创建一个thread,回调函数是smpboot_thread_fn,ht->thread_comm是thread的名字,就是例子中的migration/0,1,2,3
104 static int smpboot_thread_fn(void *data)
105 {
106 struct smpboot_thread_data *td = data;
107 struct smp_hotplug_thread *ht = td->ht;
108
109 while (1) {
110 set_current_state(TASK_INTERRUPTIBLE);
111 preempt_disable();
112 if (kthread_should_stop()) {
113 set_current_state(TASK_RUNNING);
114 preempt_enable();
115 if (ht->cleanup)
116 ht->cleanup(td->cpu, cpu_online(td->cpu));
117 kfree(td);
118 return 0;
119 }
120
121 if (kthread_should_park()) {
122 __set_current_state(TASK_RUNNING);
123 preempt_enable();
124 if (ht->park && td->status == HP_THREAD_ACTIVE) {
125 BUG_ON(td->cpu != smp_processor_id());
126 ht->park(td->cpu);
127 td->status = HP_THREAD_PARKED;
128 }
129 kthread_parkme();
130 /* We might have been woken for stop */
131 continue;
132 }
133
134 BUG_ON(td->cpu != smp_processor_id());
135
136 /* Check for state change setup */
137 switch (td->status) {
138 case HP_THREAD_NONE:
139 preempt_enable();
140 if (ht->setup)
141 ht->setup(td->cpu);
142 td->status = HP_THREAD_ACTIVE;
143 preempt_disable();
144 break;
145 case HP_THREAD_PARKED:
146 preempt_enable();
147 if (ht->unpark)
148 ht->unpark(td->cpu);
149 td->status = HP_THREAD_ACTIVE;
150 preempt_disable();
151 break;
152 }
153
154 if (!ht->thread_should_run(td->cpu)) {
155 preempt_enable();
156 schedule();
157 } else {
158 set_current_state(TASK_RUNNING);
159 preempt_enable();
160 ht->thread_fn(td->cpu);
161 }
162 }
163 }
从137~152航根据thread的状况来调用不同的函数,如HP_THREAD_NONE,就调用ht->setup(td->cpu);也就是例子中的cpu_stop_unpark。
如果当前thread不是停止状态的话,就跑160行的ht->thread_fn(td->cpu);即例子中的cpu_stopper_thread