1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright 2018 Linaro Limited
5 * Author: Daniel Lezcano <daniel.lezcano@linaro.org>
7 * The idle injection framework provides a way to force CPUs to enter idle
8 * states for a specified fraction of time over a specified period.
10 * It relies on the smpboot kthreads feature providing common code for CPU
11 * hotplug and thread [un]parking.
13 * All of the kthreads used for idle injection are created at init time.
15 * Next, the users of the the idle injection framework provide a cpumask via
16 * its register function. The kthreads will be synchronized with respect to
19 * The idle + run duration is specified via separate helpers and that allows
20 * idle injection to be started.
22 * The idle injection kthreads will call play_idle() with the idle duration
23 * specified as per the above.
25 * After all of them have been woken up, a timer is set to start the next idle
28 * The timer interrupt handler will wake up the idle injection kthreads for
29 * all of the CPUs in the cpumask provided by the user.
31 * Idle injection is stopped synchronously and no leftover idle injection
32 * kthread activity after its completion is guaranteed.
34 * It is up to the user of this framework to provide a lock for higher-level
35 * synchronization to prevent race conditions like starting idle injection
36 * while unregistering from the framework.
38 #define pr_fmt(fmt) "ii_dev: " fmt
40 #include <linux/cpu.h>
41 #include <linux/hrtimer.h>
42 #include <linux/kthread.h>
43 #include <linux/sched.h>
44 #include <linux/slab.h>
45 #include <linux/smpboot.h>
47 #include <uapi/linux/sched/types.h>
50 * struct idle_inject_thread - task on/off switch structure
51 * @tsk: task injecting the idle cycles
52 * @should_run: whether or not to run the task (for the smpboot kthread API)
54 struct idle_inject_thread
{
55 struct task_struct
*tsk
;
60 * struct idle_inject_device - idle injection data
61 * @timer: idle injection period timer
62 * @idle_duration_us: duration of CPU idle time to inject
63 * @run_duration_us: duration of CPU run time to allow
64 * @cpumask: mask of CPUs affected by idle injection
66 struct idle_inject_device
{
68 unsigned int idle_duration_us
;
69 unsigned int run_duration_us
;
70 unsigned long int cpumask
[0];
73 static DEFINE_PER_CPU(struct idle_inject_thread
, idle_inject_thread
);
74 static DEFINE_PER_CPU(struct idle_inject_device
*, idle_inject_device
);
77 * idle_inject_wakeup - Wake up idle injection threads
78 * @ii_dev: target idle injection device
80 * Every idle injection task associated with the given idle injection device
81 * and running on an online CPU will be woken up.
83 static void idle_inject_wakeup(struct idle_inject_device
*ii_dev
)
85 struct idle_inject_thread
*iit
;
88 for_each_cpu_and(cpu
, to_cpumask(ii_dev
->cpumask
), cpu_online_mask
) {
89 iit
= per_cpu_ptr(&idle_inject_thread
, cpu
);
91 wake_up_process(iit
->tsk
);
96 * idle_inject_timer_fn - idle injection timer function
97 * @timer: idle injection hrtimer
99 * This function is called when the idle injection timer expires. It wakes up
100 * idle injection tasks associated with the timer and they, in turn, invoke
101 * play_idle() to inject a specified amount of CPU idle time.
103 * Return: HRTIMER_RESTART.
105 static enum hrtimer_restart
idle_inject_timer_fn(struct hrtimer
*timer
)
107 unsigned int duration_us
;
108 struct idle_inject_device
*ii_dev
=
109 container_of(timer
, struct idle_inject_device
, timer
);
111 duration_us
= READ_ONCE(ii_dev
->run_duration_us
);
112 duration_us
+= READ_ONCE(ii_dev
->idle_duration_us
);
114 idle_inject_wakeup(ii_dev
);
116 hrtimer_forward_now(timer
, ns_to_ktime(duration_us
* NSEC_PER_USEC
));
118 return HRTIMER_RESTART
;
122 * idle_inject_fn - idle injection work function
123 * @cpu: the CPU owning the task
125 * This function calls play_idle() to inject a specified amount of CPU idle
128 static void idle_inject_fn(unsigned int cpu
)
130 struct idle_inject_device
*ii_dev
;
131 struct idle_inject_thread
*iit
;
133 ii_dev
= per_cpu(idle_inject_device
, cpu
);
134 iit
= per_cpu_ptr(&idle_inject_thread
, cpu
);
137 * Let the smpboot main loop know that the task should not run again.
141 play_idle(READ_ONCE(ii_dev
->idle_duration_us
));
145 * idle_inject_set_duration - idle and run duration update helper
146 * @run_duration_us: CPU run time to allow in microseconds
147 * @idle_duration_us: CPU idle time to inject in microseconds
149 void idle_inject_set_duration(struct idle_inject_device
*ii_dev
,
150 unsigned int run_duration_us
,
151 unsigned int idle_duration_us
)
153 if (run_duration_us
&& idle_duration_us
) {
154 WRITE_ONCE(ii_dev
->run_duration_us
, run_duration_us
);
155 WRITE_ONCE(ii_dev
->idle_duration_us
, idle_duration_us
);
160 * idle_inject_get_duration - idle and run duration retrieval helper
161 * @run_duration_us: memory location to store the current CPU run time
162 * @idle_duration_us: memory location to store the current CPU idle time
164 void idle_inject_get_duration(struct idle_inject_device
*ii_dev
,
165 unsigned int *run_duration_us
,
166 unsigned int *idle_duration_us
)
168 *run_duration_us
= READ_ONCE(ii_dev
->run_duration_us
);
169 *idle_duration_us
= READ_ONCE(ii_dev
->idle_duration_us
);
173 * idle_inject_start - start idle injections
174 * @ii_dev: idle injection control device structure
176 * The function starts idle injection by first waking up all of the idle
177 * injection kthreads associated with @ii_dev to let them inject CPU idle time
178 * sets up a timer to start the next idle injection period.
180 * Return: -EINVAL if the CPU idle or CPU run time is not set or 0 on success.
182 int idle_inject_start(struct idle_inject_device
*ii_dev
)
184 unsigned int idle_duration_us
= READ_ONCE(ii_dev
->idle_duration_us
);
185 unsigned int run_duration_us
= READ_ONCE(ii_dev
->run_duration_us
);
187 if (!idle_duration_us
|| !run_duration_us
)
190 pr_debug("Starting injecting idle cycles on CPUs '%*pbl'\n",
191 cpumask_pr_args(to_cpumask(ii_dev
->cpumask
)));
193 idle_inject_wakeup(ii_dev
);
195 hrtimer_start(&ii_dev
->timer
,
196 ns_to_ktime((idle_duration_us
+ run_duration_us
) *
204 * idle_inject_stop - stops idle injections
205 * @ii_dev: idle injection control device structure
207 * The function stops idle injection and waits for the threads to finish work.
208 * If CPU idle time is being injected when this function runs, then it will
209 * wait until the end of the cycle.
211 * When it returns, there is no more idle injection kthread activity. The
212 * kthreads are scheduled out and the periodic timer is off.
214 void idle_inject_stop(struct idle_inject_device
*ii_dev
)
216 struct idle_inject_thread
*iit
;
219 pr_debug("Stopping idle injection on CPUs '%*pbl'\n",
220 cpumask_pr_args(to_cpumask(ii_dev
->cpumask
)));
222 hrtimer_cancel(&ii_dev
->timer
);
225 * Stopping idle injection requires all of the idle injection kthreads
226 * associated with the given cpumask to be parked and stay that way, so
227 * prevent CPUs from going online at this point. Any CPUs going online
228 * after the loop below will be covered by clearing the should_run flag
229 * that will cause the smpboot main loop to schedule them out.
231 cpu_hotplug_disable();
234 * Iterate over all (online + offline) CPUs here in case one of them
235 * goes offline with the should_run flag set so as to prevent its idle
236 * injection kthread from running when the CPU goes online again after
237 * the ii_dev has been freed.
239 for_each_cpu(cpu
, to_cpumask(ii_dev
->cpumask
)) {
240 iit
= per_cpu_ptr(&idle_inject_thread
, cpu
);
243 wait_task_inactive(iit
->tsk
, 0);
246 cpu_hotplug_enable();
250 * idle_inject_setup - prepare the current task for idle injection
253 * Called once, this function is in charge of setting the current task's
254 * scheduler parameters to make it an RT task.
256 static void idle_inject_setup(unsigned int cpu
)
258 struct sched_param param
= { .sched_priority
= MAX_USER_RT_PRIO
/ 2 };
260 sched_setscheduler(current
, SCHED_FIFO
, ¶m
);
264 * idle_inject_should_run - function helper for the smpboot API
265 * @cpu: CPU the kthread is running on
267 * Return: whether or not the thread can run.
269 static int idle_inject_should_run(unsigned int cpu
)
271 struct idle_inject_thread
*iit
=
272 per_cpu_ptr(&idle_inject_thread
, cpu
);
274 return iit
->should_run
;
278 * idle_inject_register - initialize idle injection on a set of CPUs
279 * @cpumask: CPUs to be affected by idle injection
281 * This function creates an idle injection control device structure for the
282 * given set of CPUs and initializes the timer associated with it. It does not
283 * start any injection cycles.
285 * Return: NULL if memory allocation fails, idle injection control device
286 * pointer on success.
288 struct idle_inject_device
*idle_inject_register(struct cpumask
*cpumask
)
290 struct idle_inject_device
*ii_dev
;
293 ii_dev
= kzalloc(sizeof(*ii_dev
) + cpumask_size(), GFP_KERNEL
);
297 cpumask_copy(to_cpumask(ii_dev
->cpumask
), cpumask
);
298 hrtimer_init(&ii_dev
->timer
, CLOCK_MONOTONIC
, HRTIMER_MODE_REL
);
299 ii_dev
->timer
.function
= idle_inject_timer_fn
;
301 for_each_cpu(cpu
, to_cpumask(ii_dev
->cpumask
)) {
303 if (per_cpu(idle_inject_device
, cpu
)) {
304 pr_err("cpu%d is already registered\n", cpu
);
308 per_cpu(idle_inject_device
, cpu
) = ii_dev
;
314 for_each_cpu(cpu_rb
, to_cpumask(ii_dev
->cpumask
)) {
317 per_cpu(idle_inject_device
, cpu_rb
) = NULL
;
326 * idle_inject_unregister - unregister idle injection control device
327 * @ii_dev: idle injection control device to unregister
329 * The function stops idle injection for the given control device,
330 * unregisters its kthreads and frees memory allocated when that device was
333 void idle_inject_unregister(struct idle_inject_device
*ii_dev
)
337 idle_inject_stop(ii_dev
);
339 for_each_cpu(cpu
, to_cpumask(ii_dev
->cpumask
))
340 per_cpu(idle_inject_device
, cpu
) = NULL
;
345 static struct smp_hotplug_thread idle_inject_threads
= {
346 .store
= &idle_inject_thread
.tsk
,
347 .setup
= idle_inject_setup
,
348 .thread_fn
= idle_inject_fn
,
349 .thread_comm
= "idle_inject/%u",
350 .thread_should_run
= idle_inject_should_run
,
353 static int __init
idle_inject_init(void)
355 return smpboot_register_percpu_thread(&idle_inject_threads
);
357 early_initcall(idle_inject_init
);