drivers/powercap/idle_inject.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright 2018 Linaro Limited
   4  *
   5  * Author: Daniel Lezcano <daniel.lezcano@linaro.org>
   6  *
   7  * The idle injection framework provides a way to force CPUs to enter idle
   8  * states for a specified fraction of time over a specified period.
   9  *
  10  * It relies on the smpboot kthreads feature providing common code for CPU
  11  * hotplug and thread [un]parking.
  12  *
  13  * All of the kthreads used for idle injection are created at init time.
  14  *
  15  * Next, the users of the idle injection framework provide a cpumask via
  16  * its register function. The kthreads will be synchronized with respect to
  17  * this cpumask.
  18  *
  19  * The idle + run duration is specified via separate helpers and that allows
  20  * idle injection to be started.
  21  *
  22  * The idle injection kthreads will call play_idle_precise() with the idle
  23  * duration and max allowed latency specified as per the above.
  24  *
  25  * After all of them have been woken up, a timer is set to start the next idle
  26  * injection cycle.
  27  *
  28  * The timer interrupt handler will wake up the idle injection kthreads for
  29  * all of the CPUs in the cpumask provided by the user.
  30  *
  31  * Idle injection is stopped synchronously and no leftover idle injection
  32  * kthread activity after its completion is guaranteed.
  33  *
  34  * It is up to the user of this framework to provide a lock for higher-level
  35  * synchronization to prevent race conditions like starting idle injection
  36  * while unregistering from the framework.
  37  */
  38 #define pr_fmt(fmt) "ii_dev: " fmt
  39
  40 #include <linux/cpu.h>
  41 #include <linux/hrtimer.h>
  42 #include <linux/kthread.h>
  43 #include <linux/sched.h>
  44 #include <linux/slab.h>
  45 #include <linux/smpboot.h>
  46 #include <linux/idle_inject.h>
  47
  48 #include <uapi/linux/sched/types.h>
  49
  50 /**
  51  * struct idle_inject_thread - task on/off switch structure
  52  * @tsk: task injecting the idle cycles
  53  * @should_run: whether or not to run the task (for the smpboot kthread API)
  54  */
  55 struct idle_inject_thread {
  56         struct task_struct *tsk;
  57         int should_run;
  58 };
  59
  60 /**
  61  * struct idle_inject_device - idle injection data
  62  * @timer: idle injection period timer
  63  * @idle_duration_us: duration of CPU idle time to inject
  64  * @run_duration_us: duration of CPU run time to allow
  65  * @latency_us: max allowed latency
  66  * @update: Optional callback deciding whether or not to skip idle
  67  *              injection in the given cycle.
  68  * @cpumask: mask of CPUs affected by idle injection
  69  *
  70  * This structure is used to define per instance idle inject device data. Each
  71  * instance has an idle duration, a run duration and mask of CPUs to inject
  72  * idle.
  73  *
  74  * Actual CPU idle time is injected by calling kernel scheduler interface
  75  * play_idle_precise(). There is one optional callback that can be registered
  76  * by calling idle_inject_register_full():
  77  *
  78  * update() - This callback is invoked just before waking up CPUs to inject
  79  * idle. If it returns false, CPUs are not woken up to inject idle in the given
  80  * cycle. It also allows the caller to readjust the idle and run duration by
  81  * calling idle_inject_set_duration() for the next cycle.
  82  */
  83 struct idle_inject_device {
  84         struct hrtimer timer;
  85         unsigned int idle_duration_us;
  86         unsigned int run_duration_us;
  87         unsigned int latency_us;
  88         bool (*update)(void);
  89         unsigned long cpumask[];
  90 };
  91
  92 static DEFINE_PER_CPU(struct idle_inject_thread, idle_inject_thread);
  93 static DEFINE_PER_CPU(struct idle_inject_device *, idle_inject_device);
  94
  95 /**
  96  * idle_inject_wakeup - Wake up idle injection threads
  97  * @ii_dev: target idle injection device
  98  *
  99  * Every idle injection task associated with the given idle injection device
 100  * and running on an online CPU will be woken up.
 101  */
 102 static void idle_inject_wakeup(struct idle_inject_device *ii_dev)
 103 {
 104         struct idle_inject_thread *iit;
 105         unsigned int cpu;
 106
 107         for_each_cpu_and(cpu, to_cpumask(ii_dev->cpumask), cpu_online_mask) {
 108                 iit = per_cpu_ptr(&idle_inject_thread, cpu);
 109                 iit->should_run = 1;
 110                 wake_up_process(iit->tsk);
 111         }
 112 }
 113
 114 /**
 115  * idle_inject_timer_fn - idle injection timer function
 116  * @timer: idle injection hrtimer
 117  *
 118  * This function is called when the idle injection timer expires.  It wakes up
 119  * idle injection tasks associated with the timer and they, in turn, invoke
 120  * play_idle_precise() to inject a specified amount of CPU idle time.
 121  *
 122  * Return: HRTIMER_RESTART.
 123  */
 124 static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer)
 125 {
 126         unsigned int duration_us;
 127         struct idle_inject_device *ii_dev =
 128                 container_of(timer, struct idle_inject_device, timer);
 129
 130         if (!ii_dev->update || ii_dev->update())
 131                 idle_inject_wakeup(ii_dev);
 132
 133         duration_us = READ_ONCE(ii_dev->run_duration_us);
 134         duration_us += READ_ONCE(ii_dev->idle_duration_us);
 135
 136         hrtimer_forward_now(timer, ns_to_ktime(duration_us * NSEC_PER_USEC));
 137
 138         return HRTIMER_RESTART;
 139 }
 140
 141 /**
 142  * idle_inject_fn - idle injection work function
 143  * @cpu: the CPU owning the task
 144  *
 145  * This function calls play_idle_precise() to inject a specified amount of CPU
 146  * idle time.
 147  */
 148 static void idle_inject_fn(unsigned int cpu)
 149 {
 150         struct idle_inject_device *ii_dev;
 151         struct idle_inject_thread *iit;
 152
 153         ii_dev = per_cpu(idle_inject_device, cpu);
 154         iit = per_cpu_ptr(&idle_inject_thread, cpu);
 155
 156         /*
 157          * Let the smpboot main loop know that the task should not run again.
 158          */
 159         iit->should_run = 0;
 160
 161         play_idle_precise(READ_ONCE(ii_dev->idle_duration_us) * NSEC_PER_USEC,
 162                           READ_ONCE(ii_dev->latency_us) * NSEC_PER_USEC);
 163 }
 164
 165 /**
 166  * idle_inject_set_duration - idle and run duration update helper
 167  * @ii_dev: idle injection control device structure
 168  * @run_duration_us: CPU run time to allow in microseconds
 169  * @idle_duration_us: CPU idle time to inject in microseconds
 170  */
 171 void idle_inject_set_duration(struct idle_inject_device *ii_dev,
 172                               unsigned int run_duration_us,
 173                               unsigned int idle_duration_us)
 174 {
 175         if (run_duration_us + idle_duration_us) {
 176                 WRITE_ONCE(ii_dev->run_duration_us, run_duration_us);
 177                 WRITE_ONCE(ii_dev->idle_duration_us, idle_duration_us);
 178         }
 179         if (!run_duration_us)
 180                 pr_debug("CPU is forced to 100 percent idle\n");
 181 }
 182 EXPORT_SYMBOL_NS_GPL(idle_inject_set_duration, "IDLE_INJECT");
 183
 184 /**
 185  * idle_inject_get_duration - idle and run duration retrieval helper
 186  * @ii_dev: idle injection control device structure
 187  * @run_duration_us: memory location to store the current CPU run time
 188  * @idle_duration_us: memory location to store the current CPU idle time
 189  */
 190 void idle_inject_get_duration(struct idle_inject_device *ii_dev,
 191                               unsigned int *run_duration_us,
 192                               unsigned int *idle_duration_us)
 193 {
 194         *run_duration_us = READ_ONCE(ii_dev->run_duration_us);
 195         *idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
 196 }
 197 EXPORT_SYMBOL_NS_GPL(idle_inject_get_duration, "IDLE_INJECT");
 198
 199 /**
 200  * idle_inject_set_latency - set the maximum latency allowed
 201  * @ii_dev: idle injection control device structure
 202  * @latency_us: set the latency requirement for the idle state
 203  */
 204 void idle_inject_set_latency(struct idle_inject_device *ii_dev,
 205                              unsigned int latency_us)
 206 {
 207         WRITE_ONCE(ii_dev->latency_us, latency_us);
 208 }
 209 EXPORT_SYMBOL_NS_GPL(idle_inject_set_latency, "IDLE_INJECT");
 210
 211 /**
 212  * idle_inject_start - start idle injections
 213  * @ii_dev: idle injection control device structure
 214  *
 215  * The function starts idle injection by first waking up all of the idle
 216  * injection kthreads associated with @ii_dev to let them inject CPU idle time
 217  * sets up a timer to start the next idle injection period.
 218  *
 219  * Return: -EINVAL if the CPU idle or CPU run time is not set or 0 on success.
 220  */
 221 int idle_inject_start(struct idle_inject_device *ii_dev)
 222 {
 223         unsigned int idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
 224         unsigned int run_duration_us = READ_ONCE(ii_dev->run_duration_us);
 225
 226         if (!(idle_duration_us + run_duration_us))
 227                 return -EINVAL;
 228
 229         pr_debug("Starting injecting idle cycles on CPUs '%*pbl'\n",
 230                  cpumask_pr_args(to_cpumask(ii_dev->cpumask)));
 231
 232         idle_inject_wakeup(ii_dev);
 233
 234         hrtimer_start(&ii_dev->timer,
 235                       ns_to_ktime((idle_duration_us + run_duration_us) *
 236                                   NSEC_PER_USEC),
 237                       HRTIMER_MODE_REL);
 238
 239         return 0;
 240 }
 241 EXPORT_SYMBOL_NS_GPL(idle_inject_start, "IDLE_INJECT");
 242
 243 /**
 244  * idle_inject_stop - stops idle injections
 245  * @ii_dev: idle injection control device structure
 246  *
 247  * The function stops idle injection and waits for the threads to finish work.
 248  * If CPU idle time is being injected when this function runs, then it will
 249  * wait until the end of the cycle.
 250  *
 251  * When it returns, there is no more idle injection kthread activity.  The
 252  * kthreads are scheduled out and the periodic timer is off.
 253  */
 254 void idle_inject_stop(struct idle_inject_device *ii_dev)
 255 {
 256         struct idle_inject_thread *iit;
 257         unsigned int cpu;
 258
 259         pr_debug("Stopping idle injection on CPUs '%*pbl'\n",
 260                  cpumask_pr_args(to_cpumask(ii_dev->cpumask)));
 261
 262         hrtimer_cancel(&ii_dev->timer);
 263
 264         /*
 265          * Stopping idle injection requires all of the idle injection kthreads
 266          * associated with the given cpumask to be parked and stay that way, so
 267          * prevent CPUs from going online at this point.  Any CPUs going online
 268          * after the loop below will be covered by clearing the should_run flag
 269          * that will cause the smpboot main loop to schedule them out.
 270          */
 271         cpu_hotplug_disable();
 272
 273         /*
 274          * Iterate over all (online + offline) CPUs here in case one of them
 275          * goes offline with the should_run flag set so as to prevent its idle
 276          * injection kthread from running when the CPU goes online again after
 277          * the ii_dev has been freed.
 278          */
 279         for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {
 280                 iit = per_cpu_ptr(&idle_inject_thread, cpu);
 281                 iit->should_run = 0;
 282
 283                 wait_task_inactive(iit->tsk, TASK_ANY);
 284         }
 285
 286         cpu_hotplug_enable();
 287 }
 288 EXPORT_SYMBOL_NS_GPL(idle_inject_stop, "IDLE_INJECT");
 289
 290 /**
 291  * idle_inject_setup - prepare the current task for idle injection
 292  * @cpu: not used
 293  *
 294  * Called once, this function is in charge of setting the current task's
 295  * scheduler parameters to make it an RT task.
 296  */
 297 static void idle_inject_setup(unsigned int cpu)
 298 {
 299         sched_set_fifo(current);
 300 }
 301
 302 /**
 303  * idle_inject_should_run - function helper for the smpboot API
 304  * @cpu: CPU the kthread is running on
 305  *
 306  * Return: whether or not the thread can run.
 307  */
 308 static int idle_inject_should_run(unsigned int cpu)
 309 {
 310         struct idle_inject_thread *iit =
 311                 per_cpu_ptr(&idle_inject_thread, cpu);
 312
 313         return iit->should_run;
 314 }
 315
 316 /**
 317  * idle_inject_register_full - initialize idle injection on a set of CPUs
 318  * @cpumask: CPUs to be affected by idle injection
 319  * @update: This callback is called just before waking up CPUs to inject
 320  * idle
 321  *
 322  * This function creates an idle injection control device structure for the
 323  * given set of CPUs and initializes the timer associated with it. This
 324  * function also allows to register update()callback.
 325  * It does not start any injection cycles.
 326  *
 327  * Return: NULL if memory allocation fails, idle injection control device
 328  * pointer on success.
 329  */
 330
 331 struct idle_inject_device *idle_inject_register_full(struct cpumask *cpumask,
 332                                                      bool (*update)(void))
 333 {
 334         struct idle_inject_device *ii_dev;
 335         int cpu, cpu_rb;
 336
 337         ii_dev = kzalloc(sizeof(*ii_dev) + cpumask_size(), GFP_KERNEL);
 338         if (!ii_dev)
 339                 return NULL;
 340
 341         cpumask_copy(to_cpumask(ii_dev->cpumask), cpumask);
 342         hrtimer_init(&ii_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 343         ii_dev->timer.function = idle_inject_timer_fn;
 344         ii_dev->latency_us = UINT_MAX;
 345         ii_dev->update = update;
 346
 347         for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {
 348
 349                 if (per_cpu(idle_inject_device, cpu)) {
 350                         pr_err("cpu%d is already registered\n", cpu);
 351                         goto out_rollback;
 352                 }
 353
 354                 per_cpu(idle_inject_device, cpu) = ii_dev;
 355         }
 356
 357         return ii_dev;
 358
 359 out_rollback:
 360         for_each_cpu(cpu_rb, to_cpumask(ii_dev->cpumask)) {
 361                 if (cpu == cpu_rb)
 362                         break;
 363                 per_cpu(idle_inject_device, cpu_rb) = NULL;
 364         }
 365
 366         kfree(ii_dev);
 367
 368         return NULL;
 369 }
 370 EXPORT_SYMBOL_NS_GPL(idle_inject_register_full, "IDLE_INJECT");
 371
 372 /**
 373  * idle_inject_register - initialize idle injection on a set of CPUs
 374  * @cpumask: CPUs to be affected by idle injection
 375  *
 376  * This function creates an idle injection control device structure for the
 377  * given set of CPUs and initializes the timer associated with it.  It does not
 378  * start any injection cycles.
 379  *
 380  * Return: NULL if memory allocation fails, idle injection control device
 381  * pointer on success.
 382  */
 383 struct idle_inject_device *idle_inject_register(struct cpumask *cpumask)
 384 {
 385         return idle_inject_register_full(cpumask, NULL);
 386 }
 387 EXPORT_SYMBOL_NS_GPL(idle_inject_register, "IDLE_INJECT");
 388
 389 /**
 390  * idle_inject_unregister - unregister idle injection control device
 391  * @ii_dev: idle injection control device to unregister
 392  *
 393  * The function stops idle injection for the given control device,
 394  * unregisters its kthreads and frees memory allocated when that device was
 395  * created.
 396  */
 397 void idle_inject_unregister(struct idle_inject_device *ii_dev)
 398 {
 399         unsigned int cpu;
 400
 401         idle_inject_stop(ii_dev);
 402
 403         for_each_cpu(cpu, to_cpumask(ii_dev->cpumask))
 404                 per_cpu(idle_inject_device, cpu) = NULL;
 405
 406         kfree(ii_dev);
 407 }
 408 EXPORT_SYMBOL_NS_GPL(idle_inject_unregister, "IDLE_INJECT");
 409
 410 static struct smp_hotplug_thread idle_inject_threads = {
 411         .store = &idle_inject_thread.tsk,
 412         .setup = idle_inject_setup,
 413         .thread_fn = idle_inject_fn,
 414         .thread_comm = "idle_inject/%u",
 415         .thread_should_run = idle_inject_should_run,
 416 };
 417
 418 static int __init idle_inject_init(void)
 419 {
 420         return smpboot_register_percpu_thread(&idle_inject_threads);
 421 }
 422 early_initcall(idle_inject_init);