kernel/sched/idle.c

   1 /*
   2  * Generic entry point for the idle threads
   3  */
   4 #include <linux/sched.h>
   5 #include <linux/sched/idle.h>
   6 #include <linux/cpu.h>
   7 #include <linux/cpuidle.h>
   8 #include <linux/cpuhotplug.h>
   9 #include <linux/tick.h>
  10 #include <linux/mm.h>
  11 #include <linux/stackprotector.h>
  12 #include <linux/suspend.h>
  13
  14 #include <asm/tlb.h>
  15
  16 #include <trace/events/power.h>
  17
  18 #include "sched.h"
  19
  20 /* Linker adds these: start and end of __cpuidle functions */
  21 extern char __cpuidle_text_start[], __cpuidle_text_end[];
  22
  23 /**
  24  * sched_idle_set_state - Record idle state for the current CPU.
  25  * @idle_state: State to record.
  26  */
  27 void sched_idle_set_state(struct cpuidle_state *idle_state)
  28 {
  29         idle_set_state(this_rq(), idle_state);
  30 }
  31
  32 static int __read_mostly cpu_idle_force_poll;
  33
  34 void cpu_idle_poll_ctrl(bool enable)
  35 {
  36         if (enable) {
  37                 cpu_idle_force_poll++;
  38         } else {
  39                 cpu_idle_force_poll--;
  40                 WARN_ON_ONCE(cpu_idle_force_poll < 0);
  41         }
  42 }
  43
  44 #ifdef CONFIG_GENERIC_IDLE_POLL_SETUP
  45 static int __init cpu_idle_poll_setup(char *__unused)
  46 {
  47         cpu_idle_force_poll = 1;
  48         return 1;
  49 }
  50 __setup("nohlt", cpu_idle_poll_setup);
  51
  52 static int __init cpu_idle_nopoll_setup(char *__unused)
  53 {
  54         cpu_idle_force_poll = 0;
  55         return 1;
  56 }
  57 __setup("hlt", cpu_idle_nopoll_setup);
  58 #endif
  59
  60 static noinline int __cpuidle cpu_idle_poll(void)
  61 {
  62         rcu_idle_enter();
  63         trace_cpu_idle_rcuidle(0, smp_processor_id());
  64         local_irq_enable();
  65         stop_critical_timings();
  66         while (!tif_need_resched() &&
  67                 (cpu_idle_force_poll || tick_check_broadcast_expired()))
  68                 cpu_relax();
  69         start_critical_timings();
  70         trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
  71         rcu_idle_exit();
  72         return 1;
  73 }
  74
  75 /* Weak implementations for optional arch specific functions */
  76 void __weak arch_cpu_idle_prepare(void) { }
  77 void __weak arch_cpu_idle_enter(void) { }
  78 void __weak arch_cpu_idle_exit(void) { }
  79 void __weak arch_cpu_idle_dead(void) { }
  80 void __weak arch_cpu_idle(void)
  81 {
  82         cpu_idle_force_poll = 1;
  83         local_irq_enable();
  84 }
  85
  86 /**
  87  * default_idle_call - Default CPU idle routine.
  88  *
  89  * To use when the cpuidle framework cannot be used.
  90  */
  91 void __cpuidle default_idle_call(void)
  92 {
  93         if (current_clr_polling_and_test()) {
  94                 local_irq_enable();
  95         } else {
  96                 stop_critical_timings();
  97                 arch_cpu_idle();
  98                 start_critical_timings();
  99         }
 100 }
 101
 102 static int call_cpuidle(struct cpuidle_driver *drv, struct cpuidle_device *dev,
 103                       int next_state)
 104 {
 105         /*
 106          * The idle task must be scheduled, it is pointless to go to idle, just
 107          * update no idle residency and return.
 108          */
 109         if (current_clr_polling_and_test()) {
 110                 dev->last_residency = 0;
 111                 local_irq_enable();
 112                 return -EBUSY;
 113         }
 114
 115         /*
 116          * Enter the idle state previously returned by the governor decision.
 117          * This function will block until an interrupt occurs and will take
 118          * care of re-enabling the local interrupts
 119          */
 120         return cpuidle_enter(drv, dev, next_state);
 121 }
 122
 123 /**
 124  * cpuidle_idle_call - the main idle function
 125  *
 126  * NOTE: no locks or semaphores should be used here
 127  *
 128  * On archs that support TIF_POLLING_NRFLAG, is called with polling
 129  * set, and it returns with polling set.  If it ever stops polling, it
 130  * must clear the polling bit.
 131  */
 132 static void cpuidle_idle_call(void)
 133 {
 134         struct cpuidle_device *dev = cpuidle_get_device();
 135         struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
 136         int next_state, entered_state;
 137
 138         /*
 139          * Check if the idle task must be rescheduled. If it is the
 140          * case, exit the function after re-enabling the local irq.
 141          */
 142         if (need_resched()) {
 143                 local_irq_enable();
 144                 return;
 145         }
 146
 147         /*
 148          * Tell the RCU framework we are entering an idle section,
 149          * so no more rcu read side critical sections and one more
 150          * step to the grace period
 151          */
 152         rcu_idle_enter();
 153
 154         if (cpuidle_not_available(drv, dev)) {
 155                 default_idle_call();
 156                 goto exit_idle;
 157         }
 158
 159         /*
 160          * Suspend-to-idle ("freeze") is a system state in which all user space
 161          * has been frozen, all I/O devices have been suspended and the only
 162          * activity happens here and in iterrupts (if any).  In that case bypass
 163          * the cpuidle governor and go stratight for the deepest idle state
 164          * available.  Possibly also suspend the local tick and the entire
 165          * timekeeping to prevent timer interrupts from kicking us out of idle
 166          * until a proper wakeup interrupt happens.
 167          */
 168
 169         if (idle_should_freeze() || dev->use_deepest_state) {
 170                 if (idle_should_freeze()) {
 171                         entered_state = cpuidle_enter_freeze(drv, dev);
 172                         if (entered_state > 0) {
 173                                 local_irq_enable();
 174                                 goto exit_idle;
 175                         }
 176                 }
 177
 178                 next_state = cpuidle_find_deepest_state(drv, dev);
 179                 call_cpuidle(drv, dev, next_state);
 180         } else {
 181                 /*
 182                  * Ask the cpuidle framework to choose a convenient idle state.
 183                  */
 184                 next_state = cpuidle_select(drv, dev);
 185                 entered_state = call_cpuidle(drv, dev, next_state);
 186                 /*
 187                  * Give the governor an opportunity to reflect on the outcome
 188                  */
 189                 cpuidle_reflect(dev, entered_state);
 190         }
 191
 192 exit_idle:
 193         __current_set_polling();
 194
 195         /*
 196          * It is up to the idle functions to reenable local interrupts
 197          */
 198         if (WARN_ON_ONCE(irqs_disabled()))
 199                 local_irq_enable();
 200
 201         rcu_idle_exit();
 202 }
 203
 204 /*
 205  * Generic idle loop implementation
 206  *
 207  * Called with polling cleared.
 208  */
 209 static void do_idle(void)
 210 {
 211         /*
 212          * If the arch has a polling bit, we maintain an invariant:
 213          *
 214          * Our polling bit is clear if we're not scheduled (i.e. if rq->curr !=
 215          * rq->idle). This means that, if rq->idle has the polling bit set,
 216          * then setting need_resched is guaranteed to cause the CPU to
 217          * reschedule.
 218          */
 219
 220         __current_set_polling();
 221         tick_nohz_idle_enter();
 222
 223         while (!need_resched()) {
 224                 check_pgt_cache();
 225                 rmb();
 226
 227                 if (cpu_is_offline(smp_processor_id())) {
 228                         cpuhp_report_idle_dead();
 229                         arch_cpu_idle_dead();
 230                 }
 231
 232                 local_irq_disable();
 233                 arch_cpu_idle_enter();
 234
 235                 /*
 236                  * In poll mode we reenable interrupts and spin. Also if we
 237                  * detected in the wakeup from idle path that the tick
 238                  * broadcast device expired for us, we don't want to go deep
 239                  * idle as we know that the IPI is going to arrive right away.
 240                  */
 241                 if (cpu_idle_force_poll || tick_check_broadcast_expired())
 242                         cpu_idle_poll();
 243                 else
 244                         cpuidle_idle_call();
 245                 arch_cpu_idle_exit();
 246         }
 247
 248         /*
 249          * Since we fell out of the loop above, we know TIF_NEED_RESCHED must
 250          * be set, propagate it into PREEMPT_NEED_RESCHED.
 251          *
 252          * This is required because for polling idle loops we will not have had
 253          * an IPI to fold the state for us.
 254          */
 255         preempt_set_need_resched();
 256         tick_nohz_idle_exit();
 257         __current_clr_polling();
 258
 259         /*
 260          * We promise to call sched_ttwu_pending() and reschedule if
 261          * need_resched() is set while polling is set. That means that clearing
 262          * polling needs to be visible before doing these things.
 263          */
 264         smp_mb__after_atomic();
 265
 266         sched_ttwu_pending();
 267         schedule_preempt_disabled();
 268 }
 269
 270 bool cpu_in_idle(unsigned long pc)
 271 {
 272         return pc >= (unsigned long)__cpuidle_text_start &&
 273                 pc < (unsigned long)__cpuidle_text_end;
 274 }
 275
 276 struct idle_timer {
 277         struct hrtimer timer;
 278         int done;
 279 };
 280
 281 static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer)
 282 {
 283         struct idle_timer *it = container_of(timer, struct idle_timer, timer);
 284
 285         WRITE_ONCE(it->done, 1);
 286         set_tsk_need_resched(current);
 287
 288         return HRTIMER_NORESTART;
 289 }
 290
 291 void play_idle(unsigned long duration_ms)
 292 {
 293         struct idle_timer it;
 294
 295         /*
 296          * Only FIFO tasks can disable the tick since they don't need the forced
 297          * preemption.
 298          */
 299         WARN_ON_ONCE(current->policy != SCHED_FIFO);
 300         WARN_ON_ONCE(current->nr_cpus_allowed != 1);
 301         WARN_ON_ONCE(!(current->flags & PF_KTHREAD));
 302         WARN_ON_ONCE(!(current->flags & PF_NO_SETAFFINITY));
 303         WARN_ON_ONCE(!duration_ms);
 304
 305         rcu_sleep_check();
 306         preempt_disable();
 307         current->flags |= PF_IDLE;
 308         cpuidle_use_deepest_state(true);
 309
 310         it.done = 0;
 311         hrtimer_init_on_stack(&it.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 312         it.timer.function = idle_inject_timer_fn;
 313         hrtimer_start(&it.timer, ms_to_ktime(duration_ms), HRTIMER_MODE_REL_PINNED);
 314
 315         while (!READ_ONCE(it.done))
 316                 do_idle();
 317
 318         cpuidle_use_deepest_state(false);
 319         current->flags &= ~PF_IDLE;
 320
 321         preempt_fold_need_resched();
 322         preempt_enable();
 323 }
 324 EXPORT_SYMBOL_GPL(play_idle);
 325
 326 void cpu_startup_entry(enum cpuhp_state state)
 327 {
 328         /*
 329          * This #ifdef needs to die, but it's too late in the cycle to
 330          * make this generic (arm and sh have never invoked the canary
 331          * init for the non boot cpus!). Will be fixed in 3.11
 332          */
 333 #ifdef CONFIG_X86
 334         /*
 335          * If we're the non-boot CPU, nothing set the stack canary up
 336          * for us. The boot CPU already has it initialized but no harm
 337          * in doing it again. This is a good place for updating it, as
 338          * we wont ever return from this function (so the invalid
 339          * canaries already on the stack wont ever trigger).
 340          */
 341         boot_init_stack_canary();
 342 #endif
 343         arch_cpu_idle_prepare();
 344         cpuhp_online_idle(state);
 345         while (1)
 346                 do_idle();
 347 }