1 #include <linux/stop_machine.h>
2 #include <linux/kthread.h>
3 #include <linux/sched.h>
6 #include <linux/syscalls.h>
7 #include <linux/kthread.h>
8 #include <asm/atomic.h>
9 #include <asm/semaphore.h>
10 #include <asm/uaccess.h>
12 /* Since we effect priority and affinity (both of which are visible
13 * to, and settable by outside processes) we do indirection via a
16 /* Thread to stop each CPU in user context. */
17 enum stopmachine_state
{
20 STOPMACHINE_DISABLE_IRQ
,
24 static enum stopmachine_state stopmachine_state
;
25 static unsigned int stopmachine_num_threads
;
26 static atomic_t stopmachine_thread_ack
;
27 static DECLARE_MUTEX(stopmachine_mutex
);
29 static int stopmachine(void *unused
)
31 int irqs_disabled
= 0;
34 /* Ack: we are alive */
35 smp_mb(); /* Theoretically the ack = 0 might not be on this CPU yet. */
36 atomic_inc(&stopmachine_thread_ack
);
38 /* Simple state machine */
39 while (stopmachine_state
!= STOPMACHINE_EXIT
) {
40 if (stopmachine_state
== STOPMACHINE_DISABLE_IRQ
44 /* Ack: irqs disabled. */
45 smp_mb(); /* Must read state first. */
46 atomic_inc(&stopmachine_thread_ack
);
47 } else if (stopmachine_state
== STOPMACHINE_PREPARE
49 /* Everyone is in place, hold CPU. */
52 smp_mb(); /* Must read state first. */
53 atomic_inc(&stopmachine_thread_ack
);
55 /* Yield in first stage: migration threads need to
56 * help our sisters onto their CPUs. */
57 if (!prepared
&& !irqs_disabled
)
63 /* Ack: we are exiting. */
64 smp_mb(); /* Must read state first. */
65 atomic_inc(&stopmachine_thread_ack
);
75 /* Change the thread state */
76 static void stopmachine_set_state(enum stopmachine_state state
)
78 atomic_set(&stopmachine_thread_ack
, 0);
80 stopmachine_state
= state
;
81 while (atomic_read(&stopmachine_thread_ack
) != stopmachine_num_threads
)
85 static int stop_machine(void)
89 struct sched_param param
= { .sched_priority
= MAX_RT_PRIO
-1 };
91 /* One high-prio thread per cpu. We'll do this one. */
92 sched_setscheduler(current
, SCHED_FIFO
, ¶m
);
94 atomic_set(&stopmachine_thread_ack
, 0);
95 stopmachine_num_threads
= 0;
96 stopmachine_state
= STOPMACHINE_WAIT
;
98 for_each_online_cpu(i
) {
99 struct task_struct
*tsk
;
100 if (i
== raw_smp_processor_id())
102 tsk
= kthread_create(stopmachine
, NULL
, "stopmachine");
107 kthread_bind(tsk
, i
);
108 wake_up_process(tsk
);
109 stopmachine_num_threads
++;
112 /* Wait for them all to come to life. */
113 while (atomic_read(&stopmachine_thread_ack
) != stopmachine_num_threads
)
116 /* If some failed, kill them all. */
118 stopmachine_set_state(STOPMACHINE_EXIT
);
119 up(&stopmachine_mutex
);
123 /* Now they are all started, make them hold the CPUs, ready. */
125 stopmachine_set_state(STOPMACHINE_PREPARE
);
127 /* Make them disable irqs. */
129 stopmachine_set_state(STOPMACHINE_DISABLE_IRQ
);
134 static void restart_machine(void)
136 stopmachine_set_state(STOPMACHINE_EXIT
);
138 preempt_enable_no_resched();
141 struct stop_machine_data
145 struct completion done
;
148 static int do_stop(void *_smdata
)
150 struct stop_machine_data
*smdata
= _smdata
;
153 ret
= stop_machine();
155 ret
= smdata
->fn(smdata
->data
);
159 /* We're done: you can kthread_stop us now */
160 complete(&smdata
->done
);
162 /* Wait for kthread_stop */
163 set_current_state(TASK_INTERRUPTIBLE
);
164 while (!kthread_should_stop()) {
166 set_current_state(TASK_INTERRUPTIBLE
);
168 __set_current_state(TASK_RUNNING
);
172 struct task_struct
*__stop_machine_run(int (*fn
)(void *), void *data
,
175 struct stop_machine_data smdata
;
176 struct task_struct
*p
;
180 init_completion(&smdata
.done
);
182 down(&stopmachine_mutex
);
184 /* If they don't care which CPU fn runs on, bind to any online one. */
186 cpu
= raw_smp_processor_id();
188 p
= kthread_create(do_stop
, &smdata
, "kstopmachine");
190 kthread_bind(p
, cpu
);
192 wait_for_completion(&smdata
.done
);
194 up(&stopmachine_mutex
);
198 int stop_machine_run(int (*fn
)(void *), void *data
, unsigned int cpu
)
200 struct task_struct
*p
;
203 /* No CPUs can come up or down during this. */
205 p
= __stop_machine_run(fn
, data
, cpu
);
207 ret
= kthread_stop(p
);
210 unlock_cpu_hotplug();