kernel/stop_machine.c

   1 /* Copyright 2008, 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation.
   2  * GPL v2 and any later version.
   3  */
   4 #include <linux/cpu.h>
   5 #include <linux/err.h>
   6 #include <linux/kthread.h>
   7 #include <linux/module.h>
   8 #include <linux/sched.h>
   9 #include <linux/stop_machine.h>
  10 #include <linux/syscalls.h>
  11 #include <linux/interrupt.h>
  12
  13 #include <asm/atomic.h>
  14 #include <asm/uaccess.h>
  15
  16 /* This controls the threads on each CPU. */
  17 enum stopmachine_state {
  18         /* Dummy starting state for thread. */
  19         STOPMACHINE_NONE,
  20         /* Awaiting everyone to be scheduled. */
  21         STOPMACHINE_PREPARE,
  22         /* Disable interrupts. */
  23         STOPMACHINE_DISABLE_IRQ,
  24         /* Run the function */
  25         STOPMACHINE_RUN,
  26         /* Exit */
  27         STOPMACHINE_EXIT,
  28 };
  29 static enum stopmachine_state state;
  30
  31 struct stop_machine_data {
  32         int (*fn)(void *);
  33         void *data;
  34         int fnret;
  35 };
  36
  37 /* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
  38 static unsigned int num_threads;
  39 static atomic_t thread_ack;
  40 static DEFINE_MUTEX(lock);
  41 /* setup_lock protects refcount, stop_machine_wq and stop_machine_work. */
  42 static DEFINE_MUTEX(setup_lock);
  43 /* do not start up until all worklets have been placed: */
  44 static DEFINE_MUTEX(startup_lock);
  45 /* Users of stop_machine. */
  46 static int refcount;
  47 static struct workqueue_struct *stop_machine_wq;
  48 static struct stop_machine_data active, idle;
  49 static const cpumask_t *active_cpus;
  50 static void *stop_machine_work;
  51
  52 static void set_state(enum stopmachine_state newstate)
  53 {
  54         /* Reset ack counter. */
  55         atomic_set(&thread_ack, num_threads);
  56         smp_wmb();
  57         state = newstate;
  58 }
  59
  60 /* Last one to ack a state moves to the next state. */
  61 static void ack_state(void)
  62 {
  63         if (atomic_dec_and_test(&thread_ack))
  64                 set_state(state + 1);
  65 }
  66
  67 /* This is the actual function which stops the CPU. It runs
  68  * in the context of a dedicated stopmachine workqueue. */
  69 static void stop_cpu(struct work_struct *unused)
  70 {
  71         enum stopmachine_state curstate = STOPMACHINE_NONE;
  72         struct stop_machine_data *smdata = &idle;
  73         int cpu = smp_processor_id();
  74         int err;
  75
  76         /*
  77          * Wait for the startup loop to finish:
  78          */
  79         mutex_lock(&startup_lock);
  80         /*
  81          * Let other threads continue too:
  82          */
  83         mutex_unlock(&startup_lock);
  84
  85         if (!active_cpus) {
  86                 if (cpu == cpumask_first(cpu_online_mask))
  87                         smdata = &active;
  88         } else {
  89                 if (cpumask_test_cpu(cpu, active_cpus))
  90                         smdata = &active;
  91         }
  92         /* Simple state machine */
  93         do {
  94                 /* Chill out and ensure we re-read stopmachine_state. */
  95                 cpu_relax();
  96                 if (state != curstate) {
  97                         curstate = state;
  98                         switch (curstate) {
  99                         case STOPMACHINE_DISABLE_IRQ:
 100                                 local_irq_disable();
 101                                 hard_irq_disable();
 102                                 break;
 103                         case STOPMACHINE_RUN:
 104                                 /* On multiple CPUs only a single error code
 105                                  * is needed to tell that something failed. */
 106                                 err = smdata->fn(smdata->data);
 107                                 if (err)
 108                                         smdata->fnret = err;
 109                                 break;
 110                         default:
 111                                 break;
 112                         }
 113                         ack_state();
 114                 }
 115         } while (curstate != STOPMACHINE_EXIT);
 116
 117         local_irq_enable();
 118 }
 119
 120 /* Callback for CPUs which aren't supposed to do anything. */
 121 static int chill(void *unused)
 122 {
 123         return 0;
 124 }
 125
 126 int stop_machine_create(void)
 127 {
 128         mutex_lock(&setup_lock);
 129         if (refcount)
 130                 goto done;
 131         stop_machine_wq = create_rt_workqueue("kstop");
 132         if (!stop_machine_wq)
 133                 goto err_out;
 134         stop_machine_work = alloc_percpu(struct work_struct);
 135         if (!stop_machine_work)
 136                 goto err_out;
 137 done:
 138         refcount++;
 139         mutex_unlock(&setup_lock);
 140         return 0;
 141
 142 err_out:
 143         if (stop_machine_wq)
 144                 destroy_workqueue(stop_machine_wq);
 145         mutex_unlock(&setup_lock);
 146         return -ENOMEM;
 147 }
 148 EXPORT_SYMBOL_GPL(stop_machine_create);
 149
 150 void stop_machine_destroy(void)
 151 {
 152         mutex_lock(&setup_lock);
 153         refcount--;
 154         if (refcount)
 155                 goto done;
 156         destroy_workqueue(stop_machine_wq);
 157         free_percpu(stop_machine_work);
 158 done:
 159         mutex_unlock(&setup_lock);
 160 }
 161 EXPORT_SYMBOL_GPL(stop_machine_destroy);
 162
 163 int __stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
 164 {
 165         struct work_struct *sm_work;
 166         int i, ret;
 167
 168         /* Set up initial state. */
 169         mutex_lock(&lock);
 170         num_threads = num_online_cpus();
 171         active_cpus = cpus;
 172         active.fn = fn;
 173         active.data = data;
 174         active.fnret = 0;
 175         idle.fn = chill;
 176         idle.data = NULL;
 177
 178         set_state(STOPMACHINE_PREPARE);
 179
 180         /*
 181          * Schedule the stop_cpu work on all cpus before allowing any
 182          * of the CPUs to execute it:
 183          */
 184         mutex_lock(&startup_lock);
 185
 186         for_each_online_cpu(i) {
 187                 sm_work = per_cpu_ptr(stop_machine_work, i);
 188                 INIT_WORK(sm_work, stop_cpu);
 189                 queue_work_on(i, stop_machine_wq, sm_work);
 190         }
 191
 192         /* This will release the thread on all CPUs: */
 193         mutex_unlock(&startup_lock);
 194
 195         flush_workqueue(stop_machine_wq);
 196         ret = active.fnret;
 197         mutex_unlock(&lock);
 198         return ret;
 199 }
 200
 201 int stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
 202 {
 203         int ret;
 204
 205         ret = stop_machine_create();
 206         if (ret)
 207                 return ret;
 208         /* No CPUs can come up or down during this. */
 209         get_online_cpus();
 210         ret = __stop_machine(fn, data, cpus);
 211         put_online_cpus();
 212         stop_machine_destroy();
 213         return ret;
 214 }
 215 EXPORT_SYMBOL_GPL(stop_machine);