2 * intel_idle.c - native hardware idle loop for modern Intel processors
4 * Copyright (c) 2013, Intel Corporation.
5 * Len Brown <len.brown@intel.com>
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2, as published by the Free Software Foundation.
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * You should have received a copy of the GNU General Public License along with
17 * this program; if not, write to the Free Software Foundation, Inc.,
18 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
22 * intel_idle is a cpuidle driver that loads on specific Intel processors
23 * in lieu of the legacy ACPI processor_idle driver. The intent is to
24 * make Linux more efficient on these processors, as intel_idle knows
25 * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs.
31 * All CPUs have same idle states as boot CPU
33 * Chipset BM_STS (bus master status) bit is a NOP
34 * for preventing entry into deep C-stats
40 * The driver currently initializes for_each_online_cpu() upon modprobe.
41 * It it unaware of subsequent processors hot-added to the system.
42 * This means that if you boot with maxcpus=n and later online
43 * processors above n, those processors will use C1 only.
45 * ACPI has a .suspend hack to turn off deep c-statees during suspend
46 * to avoid complications with the lapic timer workaround.
47 * Have not seen issues with suspend, but may need same workaround here.
49 * There is currently no kernel-based automatic probing/loading mechanism
50 * if the driver is built as a module.
53 /* un-comment DEBUG to enable pr_debug() statements */
56 #include <linux/kernel.h>
57 #include <linux/cpuidle.h>
58 #include <linux/tick.h>
59 #include <trace/events/power.h>
60 #include <linux/sched.h>
61 #include <linux/notifier.h>
62 #include <linux/cpu.h>
63 #include <linux/module.h>
64 #include <asm/cpu_device_id.h>
65 #include <asm/mwait.h>
68 #define INTEL_IDLE_VERSION "0.4.1"
69 #define PREFIX "intel_idle: "
71 static struct cpuidle_driver intel_idle_driver
= {
75 /* intel_idle.max_cstate=0 disables driver */
76 static int max_cstate
= CPUIDLE_STATE_MAX
- 1;
78 static unsigned int mwait_substates
;
80 #define LAPIC_TIMER_ALWAYS_RELIABLE 0xFFFFFFFF
81 /* Reliable LAPIC Timer States, bit 1 for C1 etc. */
82 static unsigned int lapic_timer_reliable_states
= (1 << 1); /* Default to only C1 */
85 struct cpuidle_state
*state_table
;
88 * Hardware C-state auto-demotion may not always be optimal.
89 * Indicate which enable bits to clear here.
91 unsigned long auto_demotion_disable_flags
;
92 bool byt_auto_demotion_disable_flag
;
93 bool disable_promotion_to_c1e
;
96 static const struct idle_cpu
*icpu
;
97 static struct cpuidle_device __percpu
*intel_idle_cpuidle_devices
;
98 static int intel_idle(struct cpuidle_device
*dev
,
99 struct cpuidle_driver
*drv
, int index
);
100 static void intel_idle_freeze(struct cpuidle_device
*dev
,
101 struct cpuidle_driver
*drv
, int index
);
102 static int intel_idle_cpu_init(int cpu
);
104 static struct cpuidle_state
*cpuidle_state_table
;
107 * Set this flag for states where the HW flushes the TLB for us
108 * and so we don't need cross-calls to keep it consistent.
109 * If this flag is set, SW flushes the TLB, so even if the
110 * HW doesn't do the flushing, this flag is safe to use.
112 #define CPUIDLE_FLAG_TLB_FLUSHED 0x10000
115 * MWAIT takes an 8-bit "hint" in EAX "suggesting"
116 * the C-state (top nibble) and sub-state (bottom nibble)
117 * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc.
119 * We store the hint at the top of our "flags" for each state.
121 #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF)
122 #define MWAIT2flg(eax) ((eax & 0xFF) << 24)
125 * States are indexed by the cstate number,
126 * which is also the index into the MWAIT hint array.
127 * Thus C0 is a dummy.
129 static struct cpuidle_state nehalem_cstates
[] = {
132 .desc
= "MWAIT 0x00",
133 .flags
= MWAIT2flg(0x00),
135 .target_residency
= 6,
136 .enter
= &intel_idle
,
137 .enter_freeze
= intel_idle_freeze
, },
140 .desc
= "MWAIT 0x01",
141 .flags
= MWAIT2flg(0x01),
143 .target_residency
= 20,
144 .enter
= &intel_idle
,
145 .enter_freeze
= intel_idle_freeze
, },
148 .desc
= "MWAIT 0x10",
149 .flags
= MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED
,
151 .target_residency
= 80,
152 .enter
= &intel_idle
,
153 .enter_freeze
= intel_idle_freeze
, },
156 .desc
= "MWAIT 0x20",
157 .flags
= MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED
,
159 .target_residency
= 800,
160 .enter
= &intel_idle
,
161 .enter_freeze
= intel_idle_freeze
, },
166 static struct cpuidle_state snb_cstates
[] = {
169 .desc
= "MWAIT 0x00",
170 .flags
= MWAIT2flg(0x00),
172 .target_residency
= 2,
173 .enter
= &intel_idle
,
174 .enter_freeze
= intel_idle_freeze
, },
177 .desc
= "MWAIT 0x01",
178 .flags
= MWAIT2flg(0x01),
180 .target_residency
= 20,
181 .enter
= &intel_idle
,
182 .enter_freeze
= intel_idle_freeze
, },
185 .desc
= "MWAIT 0x10",
186 .flags
= MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED
,
188 .target_residency
= 211,
189 .enter
= &intel_idle
,
190 .enter_freeze
= intel_idle_freeze
, },
193 .desc
= "MWAIT 0x20",
194 .flags
= MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED
,
196 .target_residency
= 345,
197 .enter
= &intel_idle
,
198 .enter_freeze
= intel_idle_freeze
, },
201 .desc
= "MWAIT 0x30",
202 .flags
= MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED
,
204 .target_residency
= 345,
205 .enter
= &intel_idle
,
206 .enter_freeze
= intel_idle_freeze
, },
211 static struct cpuidle_state byt_cstates
[] = {
214 .desc
= "MWAIT 0x00",
215 .flags
= MWAIT2flg(0x00),
217 .target_residency
= 1,
218 .enter
= &intel_idle
,
219 .enter_freeze
= intel_idle_freeze
, },
222 .desc
= "MWAIT 0x58",
223 .flags
= MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED
,
225 .target_residency
= 275,
226 .enter
= &intel_idle
,
227 .enter_freeze
= intel_idle_freeze
, },
230 .desc
= "MWAIT 0x52",
231 .flags
= MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED
,
233 .target_residency
= 560,
234 .enter
= &intel_idle
,
235 .enter_freeze
= intel_idle_freeze
, },
238 .desc
= "MWAIT 0x60",
239 .flags
= MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED
,
240 .exit_latency
= 1200,
241 .target_residency
= 4000,
242 .enter
= &intel_idle
,
243 .enter_freeze
= intel_idle_freeze
, },
246 .desc
= "MWAIT 0x64",
247 .flags
= MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED
,
248 .exit_latency
= 10000,
249 .target_residency
= 20000,
250 .enter
= &intel_idle
,
251 .enter_freeze
= intel_idle_freeze
, },
256 static struct cpuidle_state cht_cstates
[] = {
259 .desc
= "MWAIT 0x00",
260 .flags
= MWAIT2flg(0x00),
262 .target_residency
= 1,
263 .enter
= &intel_idle
,
264 .enter_freeze
= intel_idle_freeze
, },
267 .desc
= "MWAIT 0x58",
268 .flags
= MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED
,
270 .target_residency
= 275,
271 .enter
= &intel_idle
,
272 .enter_freeze
= intel_idle_freeze
, },
275 .desc
= "MWAIT 0x52",
276 .flags
= MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED
,
278 .target_residency
= 560,
279 .enter
= &intel_idle
,
280 .enter_freeze
= intel_idle_freeze
, },
283 .desc
= "MWAIT 0x60",
284 .flags
= MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED
,
285 .exit_latency
= 1200,
286 .target_residency
= 4000,
287 .enter
= &intel_idle
,
288 .enter_freeze
= intel_idle_freeze
, },
291 .desc
= "MWAIT 0x64",
292 .flags
= MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED
,
293 .exit_latency
= 10000,
294 .target_residency
= 20000,
295 .enter
= &intel_idle
,
296 .enter_freeze
= intel_idle_freeze
, },
301 static struct cpuidle_state ivb_cstates
[] = {
304 .desc
= "MWAIT 0x00",
305 .flags
= MWAIT2flg(0x00),
307 .target_residency
= 1,
308 .enter
= &intel_idle
,
309 .enter_freeze
= intel_idle_freeze
, },
312 .desc
= "MWAIT 0x01",
313 .flags
= MWAIT2flg(0x01),
315 .target_residency
= 20,
316 .enter
= &intel_idle
,
317 .enter_freeze
= intel_idle_freeze
, },
320 .desc
= "MWAIT 0x10",
321 .flags
= MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED
,
323 .target_residency
= 156,
324 .enter
= &intel_idle
,
325 .enter_freeze
= intel_idle_freeze
, },
328 .desc
= "MWAIT 0x20",
329 .flags
= MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED
,
331 .target_residency
= 300,
332 .enter
= &intel_idle
,
333 .enter_freeze
= intel_idle_freeze
, },
336 .desc
= "MWAIT 0x30",
337 .flags
= MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED
,
339 .target_residency
= 300,
340 .enter
= &intel_idle
,
341 .enter_freeze
= intel_idle_freeze
, },
346 static struct cpuidle_state ivt_cstates
[] = {
349 .desc
= "MWAIT 0x00",
350 .flags
= MWAIT2flg(0x00),
352 .target_residency
= 1,
353 .enter
= &intel_idle
,
354 .enter_freeze
= intel_idle_freeze
, },
357 .desc
= "MWAIT 0x01",
358 .flags
= MWAIT2flg(0x01),
360 .target_residency
= 80,
361 .enter
= &intel_idle
,
362 .enter_freeze
= intel_idle_freeze
, },
365 .desc
= "MWAIT 0x10",
366 .flags
= MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED
,
368 .target_residency
= 156,
369 .enter
= &intel_idle
,
370 .enter_freeze
= intel_idle_freeze
, },
373 .desc
= "MWAIT 0x20",
374 .flags
= MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED
,
376 .target_residency
= 300,
377 .enter
= &intel_idle
,
378 .enter_freeze
= intel_idle_freeze
, },
383 static struct cpuidle_state ivt_cstates_4s
[] = {
386 .desc
= "MWAIT 0x00",
387 .flags
= MWAIT2flg(0x00),
389 .target_residency
= 1,
390 .enter
= &intel_idle
,
391 .enter_freeze
= intel_idle_freeze
, },
393 .name
= "C1E-IVT-4S",
394 .desc
= "MWAIT 0x01",
395 .flags
= MWAIT2flg(0x01),
397 .target_residency
= 250,
398 .enter
= &intel_idle
,
399 .enter_freeze
= intel_idle_freeze
, },
402 .desc
= "MWAIT 0x10",
403 .flags
= MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED
,
405 .target_residency
= 300,
406 .enter
= &intel_idle
,
407 .enter_freeze
= intel_idle_freeze
, },
410 .desc
= "MWAIT 0x20",
411 .flags
= MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED
,
413 .target_residency
= 400,
414 .enter
= &intel_idle
,
415 .enter_freeze
= intel_idle_freeze
, },
420 static struct cpuidle_state ivt_cstates_8s
[] = {
423 .desc
= "MWAIT 0x00",
424 .flags
= MWAIT2flg(0x00),
426 .target_residency
= 1,
427 .enter
= &intel_idle
,
428 .enter_freeze
= intel_idle_freeze
, },
430 .name
= "C1E-IVT-8S",
431 .desc
= "MWAIT 0x01",
432 .flags
= MWAIT2flg(0x01),
434 .target_residency
= 500,
435 .enter
= &intel_idle
,
436 .enter_freeze
= intel_idle_freeze
, },
439 .desc
= "MWAIT 0x10",
440 .flags
= MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED
,
442 .target_residency
= 600,
443 .enter
= &intel_idle
,
444 .enter_freeze
= intel_idle_freeze
, },
447 .desc
= "MWAIT 0x20",
448 .flags
= MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED
,
450 .target_residency
= 700,
451 .enter
= &intel_idle
,
452 .enter_freeze
= intel_idle_freeze
, },
457 static struct cpuidle_state hsw_cstates
[] = {
460 .desc
= "MWAIT 0x00",
461 .flags
= MWAIT2flg(0x00),
463 .target_residency
= 2,
464 .enter
= &intel_idle
,
465 .enter_freeze
= intel_idle_freeze
, },
468 .desc
= "MWAIT 0x01",
469 .flags
= MWAIT2flg(0x01),
471 .target_residency
= 20,
472 .enter
= &intel_idle
,
473 .enter_freeze
= intel_idle_freeze
, },
476 .desc
= "MWAIT 0x10",
477 .flags
= MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED
,
479 .target_residency
= 100,
480 .enter
= &intel_idle
,
481 .enter_freeze
= intel_idle_freeze
, },
484 .desc
= "MWAIT 0x20",
485 .flags
= MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED
,
487 .target_residency
= 400,
488 .enter
= &intel_idle
,
489 .enter_freeze
= intel_idle_freeze
, },
492 .desc
= "MWAIT 0x32",
493 .flags
= MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED
,
495 .target_residency
= 500,
496 .enter
= &intel_idle
,
497 .enter_freeze
= intel_idle_freeze
, },
500 .desc
= "MWAIT 0x40",
501 .flags
= MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED
,
503 .target_residency
= 900,
504 .enter
= &intel_idle
,
505 .enter_freeze
= intel_idle_freeze
, },
508 .desc
= "MWAIT 0x50",
509 .flags
= MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED
,
511 .target_residency
= 1800,
512 .enter
= &intel_idle
,
513 .enter_freeze
= intel_idle_freeze
, },
516 .desc
= "MWAIT 0x60",
517 .flags
= MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED
,
518 .exit_latency
= 2600,
519 .target_residency
= 7700,
520 .enter
= &intel_idle
,
521 .enter_freeze
= intel_idle_freeze
, },
525 static struct cpuidle_state bdw_cstates
[] = {
528 .desc
= "MWAIT 0x00",
529 .flags
= MWAIT2flg(0x00),
531 .target_residency
= 2,
532 .enter
= &intel_idle
,
533 .enter_freeze
= intel_idle_freeze
, },
536 .desc
= "MWAIT 0x01",
537 .flags
= MWAIT2flg(0x01),
539 .target_residency
= 20,
540 .enter
= &intel_idle
,
541 .enter_freeze
= intel_idle_freeze
, },
544 .desc
= "MWAIT 0x10",
545 .flags
= MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED
,
547 .target_residency
= 100,
548 .enter
= &intel_idle
,
549 .enter_freeze
= intel_idle_freeze
, },
552 .desc
= "MWAIT 0x20",
553 .flags
= MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED
,
555 .target_residency
= 400,
556 .enter
= &intel_idle
,
557 .enter_freeze
= intel_idle_freeze
, },
560 .desc
= "MWAIT 0x32",
561 .flags
= MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED
,
563 .target_residency
= 500,
564 .enter
= &intel_idle
,
565 .enter_freeze
= intel_idle_freeze
, },
568 .desc
= "MWAIT 0x40",
569 .flags
= MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED
,
571 .target_residency
= 900,
572 .enter
= &intel_idle
,
573 .enter_freeze
= intel_idle_freeze
, },
576 .desc
= "MWAIT 0x50",
577 .flags
= MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED
,
579 .target_residency
= 1800,
580 .enter
= &intel_idle
,
581 .enter_freeze
= intel_idle_freeze
, },
584 .desc
= "MWAIT 0x60",
585 .flags
= MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED
,
586 .exit_latency
= 2600,
587 .target_residency
= 7700,
588 .enter
= &intel_idle
,
589 .enter_freeze
= intel_idle_freeze
, },
594 static struct cpuidle_state skl_cstates
[] = {
597 .desc
= "MWAIT 0x00",
598 .flags
= MWAIT2flg(0x00),
600 .target_residency
= 2,
601 .enter
= &intel_idle
,
602 .enter_freeze
= intel_idle_freeze
, },
605 .desc
= "MWAIT 0x01",
606 .flags
= MWAIT2flg(0x01),
608 .target_residency
= 20,
609 .enter
= &intel_idle
,
610 .enter_freeze
= intel_idle_freeze
, },
613 .desc
= "MWAIT 0x10",
614 .flags
= MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED
,
616 .target_residency
= 100,
617 .enter
= &intel_idle
,
618 .enter_freeze
= intel_idle_freeze
, },
621 .desc
= "MWAIT 0x20",
622 .flags
= MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED
,
624 .target_residency
= 200,
625 .enter
= &intel_idle
,
626 .enter_freeze
= intel_idle_freeze
, },
629 .desc
= "MWAIT 0x33",
630 .flags
= MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED
,
632 .target_residency
= 800,
633 .enter
= &intel_idle
,
634 .enter_freeze
= intel_idle_freeze
, },
637 .desc
= "MWAIT 0x40",
638 .flags
= MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED
,
640 .target_residency
= 800,
641 .enter
= &intel_idle
,
642 .enter_freeze
= intel_idle_freeze
, },
645 .desc
= "MWAIT 0x50",
646 .flags
= MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED
,
648 .target_residency
= 5000,
649 .enter
= &intel_idle
,
650 .enter_freeze
= intel_idle_freeze
, },
653 .desc
= "MWAIT 0x60",
654 .flags
= MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED
,
656 .target_residency
= 5000,
657 .enter
= &intel_idle
,
658 .enter_freeze
= intel_idle_freeze
, },
663 static struct cpuidle_state skx_cstates
[] = {
666 .desc
= "MWAIT 0x00",
667 .flags
= MWAIT2flg(0x00),
669 .target_residency
= 2,
670 .enter
= &intel_idle
,
671 .enter_freeze
= intel_idle_freeze
, },
674 .desc
= "MWAIT 0x01",
675 .flags
= MWAIT2flg(0x01),
677 .target_residency
= 20,
678 .enter
= &intel_idle
,
679 .enter_freeze
= intel_idle_freeze
, },
682 .desc
= "MWAIT 0x20",
683 .flags
= MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED
,
685 .target_residency
= 600,
686 .enter
= &intel_idle
,
687 .enter_freeze
= intel_idle_freeze
, },
692 static struct cpuidle_state atom_cstates
[] = {
695 .desc
= "MWAIT 0x00",
696 .flags
= MWAIT2flg(0x00),
698 .target_residency
= 20,
699 .enter
= &intel_idle
,
700 .enter_freeze
= intel_idle_freeze
, },
703 .desc
= "MWAIT 0x10",
704 .flags
= MWAIT2flg(0x10),
706 .target_residency
= 80,
707 .enter
= &intel_idle
,
708 .enter_freeze
= intel_idle_freeze
, },
711 .desc
= "MWAIT 0x30",
712 .flags
= MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED
,
714 .target_residency
= 400,
715 .enter
= &intel_idle
,
716 .enter_freeze
= intel_idle_freeze
, },
719 .desc
= "MWAIT 0x52",
720 .flags
= MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED
,
722 .target_residency
= 560,
723 .enter
= &intel_idle
,
724 .enter_freeze
= intel_idle_freeze
, },
728 static struct cpuidle_state avn_cstates
[] = {
731 .desc
= "MWAIT 0x00",
732 .flags
= MWAIT2flg(0x00),
734 .target_residency
= 2,
735 .enter
= &intel_idle
,
736 .enter_freeze
= intel_idle_freeze
, },
739 .desc
= "MWAIT 0x51",
740 .flags
= MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED
,
742 .target_residency
= 45,
743 .enter
= &intel_idle
,
744 .enter_freeze
= intel_idle_freeze
, },
748 static struct cpuidle_state knl_cstates
[] = {
751 .desc
= "MWAIT 0x00",
752 .flags
= MWAIT2flg(0x00),
754 .target_residency
= 2,
755 .enter
= &intel_idle
,
756 .enter_freeze
= intel_idle_freeze
},
759 .desc
= "MWAIT 0x10",
760 .flags
= MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED
,
762 .target_residency
= 500,
763 .enter
= &intel_idle
,
764 .enter_freeze
= intel_idle_freeze
},
771 * @dev: cpuidle_device
772 * @drv: cpuidle driver
773 * @index: index of cpuidle state
775 * Must be called under local_irq_disable().
777 static int intel_idle(struct cpuidle_device
*dev
,
778 struct cpuidle_driver
*drv
, int index
)
780 unsigned long ecx
= 1; /* break on interrupt flag */
781 struct cpuidle_state
*state
= &drv
->states
[index
];
782 unsigned long eax
= flg2MWAIT(state
->flags
);
784 int cpu
= smp_processor_id();
786 cstate
= (((eax
) >> MWAIT_SUBSTATE_SIZE
) & MWAIT_CSTATE_MASK
) + 1;
789 * leave_mm() to avoid costly and often unnecessary wakeups
790 * for flushing the user TLB's associated with the active mm.
792 if (state
->flags
& CPUIDLE_FLAG_TLB_FLUSHED
)
795 if (!(lapic_timer_reliable_states
& (1 << (cstate
))))
796 tick_broadcast_enter();
798 mwait_idle_with_hints(eax
, ecx
);
800 if (!(lapic_timer_reliable_states
& (1 << (cstate
))))
801 tick_broadcast_exit();
807 * intel_idle_freeze - simplified "enter" callback routine for suspend-to-idle
808 * @dev: cpuidle_device
809 * @drv: cpuidle driver
810 * @index: state index
812 static void intel_idle_freeze(struct cpuidle_device
*dev
,
813 struct cpuidle_driver
*drv
, int index
)
815 unsigned long ecx
= 1; /* break on interrupt flag */
816 unsigned long eax
= flg2MWAIT(drv
->states
[index
].flags
);
818 mwait_idle_with_hints(eax
, ecx
);
821 static void __setup_broadcast_timer(void *arg
)
823 unsigned long on
= (unsigned long)arg
;
826 tick_broadcast_enable();
828 tick_broadcast_disable();
831 static int cpu_hotplug_notify(struct notifier_block
*n
,
832 unsigned long action
, void *hcpu
)
834 int hotcpu
= (unsigned long)hcpu
;
835 struct cpuidle_device
*dev
;
837 switch (action
& ~CPU_TASKS_FROZEN
) {
840 if (lapic_timer_reliable_states
!= LAPIC_TIMER_ALWAYS_RELIABLE
)
841 smp_call_function_single(hotcpu
, __setup_broadcast_timer
,
845 * Some systems can hotplug a cpu at runtime after
846 * the kernel has booted, we have to initialize the
847 * driver in this case
849 dev
= per_cpu_ptr(intel_idle_cpuidle_devices
, hotcpu
);
853 if (intel_idle_cpu_init(hotcpu
))
861 static struct notifier_block cpu_hotplug_notifier
= {
862 .notifier_call
= cpu_hotplug_notify
,
865 static void auto_demotion_disable(void *dummy
)
867 unsigned long long msr_bits
;
869 rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL
, msr_bits
);
870 msr_bits
&= ~(icpu
->auto_demotion_disable_flags
);
871 wrmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL
, msr_bits
);
873 static void c1e_promotion_disable(void *dummy
)
875 unsigned long long msr_bits
;
877 rdmsrl(MSR_IA32_POWER_CTL
, msr_bits
);
879 wrmsrl(MSR_IA32_POWER_CTL
, msr_bits
);
882 static const struct idle_cpu idle_cpu_nehalem
= {
883 .state_table
= nehalem_cstates
,
884 .auto_demotion_disable_flags
= NHM_C1_AUTO_DEMOTE
| NHM_C3_AUTO_DEMOTE
,
885 .disable_promotion_to_c1e
= true,
888 static const struct idle_cpu idle_cpu_atom
= {
889 .state_table
= atom_cstates
,
892 static const struct idle_cpu idle_cpu_lincroft
= {
893 .state_table
= atom_cstates
,
894 .auto_demotion_disable_flags
= ATM_LNC_C6_AUTO_DEMOTE
,
897 static const struct idle_cpu idle_cpu_snb
= {
898 .state_table
= snb_cstates
,
899 .disable_promotion_to_c1e
= true,
902 static const struct idle_cpu idle_cpu_byt
= {
903 .state_table
= byt_cstates
,
904 .disable_promotion_to_c1e
= true,
905 .byt_auto_demotion_disable_flag
= true,
908 static const struct idle_cpu idle_cpu_cht
= {
909 .state_table
= cht_cstates
,
910 .disable_promotion_to_c1e
= true,
911 .byt_auto_demotion_disable_flag
= true,
914 static const struct idle_cpu idle_cpu_ivb
= {
915 .state_table
= ivb_cstates
,
916 .disable_promotion_to_c1e
= true,
919 static const struct idle_cpu idle_cpu_ivt
= {
920 .state_table
= ivt_cstates
,
921 .disable_promotion_to_c1e
= true,
924 static const struct idle_cpu idle_cpu_hsw
= {
925 .state_table
= hsw_cstates
,
926 .disable_promotion_to_c1e
= true,
929 static const struct idle_cpu idle_cpu_bdw
= {
930 .state_table
= bdw_cstates
,
931 .disable_promotion_to_c1e
= true,
934 static const struct idle_cpu idle_cpu_skl
= {
935 .state_table
= skl_cstates
,
936 .disable_promotion_to_c1e
= true,
939 static const struct idle_cpu idle_cpu_skx
= {
940 .state_table
= skx_cstates
,
941 .disable_promotion_to_c1e
= true,
944 static const struct idle_cpu idle_cpu_avn
= {
945 .state_table
= avn_cstates
,
946 .disable_promotion_to_c1e
= true,
949 static const struct idle_cpu idle_cpu_knl
= {
950 .state_table
= knl_cstates
,
953 #define ICPU(model, cpu) \
954 { X86_VENDOR_INTEL, 6, model, X86_FEATURE_MWAIT, (unsigned long)&cpu }
956 static const struct x86_cpu_id intel_idle_ids
[] __initconst
= {
957 ICPU(0x1a, idle_cpu_nehalem
),
958 ICPU(0x1e, idle_cpu_nehalem
),
959 ICPU(0x1f, idle_cpu_nehalem
),
960 ICPU(0x25, idle_cpu_nehalem
),
961 ICPU(0x2c, idle_cpu_nehalem
),
962 ICPU(0x2e, idle_cpu_nehalem
),
963 ICPU(0x1c, idle_cpu_atom
),
964 ICPU(0x26, idle_cpu_lincroft
),
965 ICPU(0x2f, idle_cpu_nehalem
),
966 ICPU(0x2a, idle_cpu_snb
),
967 ICPU(0x2d, idle_cpu_snb
),
968 ICPU(0x36, idle_cpu_atom
),
969 ICPU(0x37, idle_cpu_byt
),
970 ICPU(0x4c, idle_cpu_cht
),
971 ICPU(0x3a, idle_cpu_ivb
),
972 ICPU(0x3e, idle_cpu_ivt
),
973 ICPU(0x3c, idle_cpu_hsw
),
974 ICPU(0x3f, idle_cpu_hsw
),
975 ICPU(0x45, idle_cpu_hsw
),
976 ICPU(0x46, idle_cpu_hsw
),
977 ICPU(0x4d, idle_cpu_avn
),
978 ICPU(0x3d, idle_cpu_bdw
),
979 ICPU(0x47, idle_cpu_bdw
),
980 ICPU(0x4f, idle_cpu_bdw
),
981 ICPU(0x56, idle_cpu_bdw
),
982 ICPU(0x4e, idle_cpu_skl
),
983 ICPU(0x5e, idle_cpu_skl
),
984 ICPU(0x8e, idle_cpu_skl
),
985 ICPU(0x9e, idle_cpu_skl
),
986 ICPU(0x55, idle_cpu_skx
),
987 ICPU(0x57, idle_cpu_knl
),
990 MODULE_DEVICE_TABLE(x86cpu
, intel_idle_ids
);
995 static int __init
intel_idle_probe(void)
997 unsigned int eax
, ebx
, ecx
;
998 const struct x86_cpu_id
*id
;
1000 if (max_cstate
== 0) {
1001 pr_debug(PREFIX
"disabled\n");
1005 id
= x86_match_cpu(intel_idle_ids
);
1007 if (boot_cpu_data
.x86_vendor
== X86_VENDOR_INTEL
&&
1008 boot_cpu_data
.x86
== 6)
1009 pr_debug(PREFIX
"does not run on family %d model %d\n",
1010 boot_cpu_data
.x86
, boot_cpu_data
.x86_model
);
1014 if (boot_cpu_data
.cpuid_level
< CPUID_MWAIT_LEAF
)
1017 cpuid(CPUID_MWAIT_LEAF
, &eax
, &ebx
, &ecx
, &mwait_substates
);
1019 if (!(ecx
& CPUID5_ECX_EXTENSIONS_SUPPORTED
) ||
1020 !(ecx
& CPUID5_ECX_INTERRUPT_BREAK
) ||
1024 pr_debug(PREFIX
"MWAIT substates: 0x%x\n", mwait_substates
);
1026 icpu
= (const struct idle_cpu
*)id
->driver_data
;
1027 cpuidle_state_table
= icpu
->state_table
;
1029 pr_debug(PREFIX
"v" INTEL_IDLE_VERSION
1030 " model 0x%X\n", boot_cpu_data
.x86_model
);
1036 * intel_idle_cpuidle_devices_uninit()
1037 * Unregisters the cpuidle devices.
1039 static void intel_idle_cpuidle_devices_uninit(void)
1042 struct cpuidle_device
*dev
;
1044 for_each_online_cpu(i
) {
1045 dev
= per_cpu_ptr(intel_idle_cpuidle_devices
, i
);
1046 cpuidle_unregister_device(dev
);
1051 * ivt_idle_state_table_update(void)
1053 * Tune IVT multi-socket targets
1054 * Assumption: num_sockets == (max_package_num + 1)
1056 static void ivt_idle_state_table_update(void)
1058 /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */
1059 int cpu
, package_num
, num_sockets
= 1;
1061 for_each_online_cpu(cpu
) {
1062 package_num
= topology_physical_package_id(cpu
);
1063 if (package_num
+ 1 > num_sockets
) {
1064 num_sockets
= package_num
+ 1;
1066 if (num_sockets
> 4) {
1067 cpuidle_state_table
= ivt_cstates_8s
;
1073 if (num_sockets
> 2)
1074 cpuidle_state_table
= ivt_cstates_4s
;
1076 /* else, 1 and 2 socket systems use default ivt_cstates */
1079 * sklh_idle_state_table_update(void)
1081 * On SKL-H (model 0x5e) disable C8 and C9 if:
1082 * C10 is enabled and SGX disabled
1084 static void sklh_idle_state_table_update(void)
1086 unsigned long long msr
;
1087 unsigned int eax
, ebx
, ecx
, edx
;
1090 /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */
1091 if (max_cstate
<= 7)
1094 /* if PC10 not present in CPUID.MWAIT.EDX */
1095 if ((mwait_substates
& (0xF << 28)) == 0)
1098 rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL
, msr
);
1100 /* PC10 is not enabled in PKG C-state limit */
1101 if ((msr
& 0xF) != 8)
1105 cpuid(7, &eax
, &ebx
, &ecx
, &edx
);
1107 /* if SGX is present */
1108 if (ebx
& (1 << 2)) {
1110 rdmsrl(MSR_IA32_FEATURE_CONTROL
, msr
);
1112 /* if SGX is enabled */
1113 if (msr
& (1 << 18))
1117 skl_cstates
[5].disabled
= 1; /* C8-SKL */
1118 skl_cstates
[6].disabled
= 1; /* C9-SKL */
1121 * intel_idle_state_table_update()
1123 * Update the default state_table for this CPU-id
1126 static void intel_idle_state_table_update(void)
1128 switch (boot_cpu_data
.x86_model
) {
1130 case 0x3e: /* IVT */
1131 ivt_idle_state_table_update();
1133 case 0x5e: /* SKL-H */
1134 sklh_idle_state_table_update();
1140 * intel_idle_cpuidle_driver_init()
1141 * allocate, initialize cpuidle_states
1143 static void __init
intel_idle_cpuidle_driver_init(void)
1146 struct cpuidle_driver
*drv
= &intel_idle_driver
;
1148 intel_idle_state_table_update();
1150 drv
->state_count
= 1;
1152 for (cstate
= 0; cstate
< CPUIDLE_STATE_MAX
; ++cstate
) {
1153 int num_substates
, mwait_hint
, mwait_cstate
;
1155 if ((cpuidle_state_table
[cstate
].enter
== NULL
) &&
1156 (cpuidle_state_table
[cstate
].enter_freeze
== NULL
))
1159 if (cstate
+ 1 > max_cstate
) {
1160 printk(PREFIX
"max_cstate %d reached\n",
1165 mwait_hint
= flg2MWAIT(cpuidle_state_table
[cstate
].flags
);
1166 mwait_cstate
= MWAIT_HINT2CSTATE(mwait_hint
);
1168 /* number of sub-states for this state in CPUID.MWAIT */
1169 num_substates
= (mwait_substates
>> ((mwait_cstate
+ 1) * 4))
1170 & MWAIT_SUBSTATE_MASK
;
1172 /* if NO sub-states for this state in CPUID, skip it */
1173 if (num_substates
== 0)
1176 /* if state marked as disabled, skip it */
1177 if (cpuidle_state_table
[cstate
].disabled
!= 0) {
1178 pr_debug(PREFIX
"state %s is disabled",
1179 cpuidle_state_table
[cstate
].name
);
1184 if (((mwait_cstate
+ 1) > 2) &&
1185 !boot_cpu_has(X86_FEATURE_NONSTOP_TSC
))
1186 mark_tsc_unstable("TSC halts in idle"
1187 " states deeper than C2");
1189 drv
->states
[drv
->state_count
] = /* structure copy */
1190 cpuidle_state_table
[cstate
];
1192 drv
->state_count
+= 1;
1195 if (icpu
->byt_auto_demotion_disable_flag
) {
1196 wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG
, 0);
1197 wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG
, 0);
1203 * intel_idle_cpu_init()
1204 * allocate, initialize, register cpuidle_devices
1205 * @cpu: cpu/core to initialize
1207 static int intel_idle_cpu_init(int cpu
)
1209 struct cpuidle_device
*dev
;
1211 dev
= per_cpu_ptr(intel_idle_cpuidle_devices
, cpu
);
1215 if (cpuidle_register_device(dev
)) {
1216 pr_debug(PREFIX
"cpuidle_register_device %d failed!\n", cpu
);
1220 if (icpu
->auto_demotion_disable_flags
)
1221 smp_call_function_single(cpu
, auto_demotion_disable
, NULL
, 1);
1223 if (icpu
->disable_promotion_to_c1e
)
1224 smp_call_function_single(cpu
, c1e_promotion_disable
, NULL
, 1);
1229 static int __init
intel_idle_init(void)
1233 /* Do not load intel_idle at all for now if idle= is passed */
1234 if (boot_option_idle_override
!= IDLE_NO_OVERRIDE
)
1237 retval
= intel_idle_probe();
1241 intel_idle_cpuidle_devices
= alloc_percpu(struct cpuidle_device
);
1242 if (intel_idle_cpuidle_devices
== NULL
)
1245 intel_idle_cpuidle_driver_init();
1246 retval
= cpuidle_register_driver(&intel_idle_driver
);
1248 struct cpuidle_driver
*drv
= cpuidle_get_driver();
1249 printk(KERN_DEBUG PREFIX
"intel_idle yielding to %s",
1250 drv
? drv
->name
: "none");
1251 free_percpu(intel_idle_cpuidle_devices
);
1255 cpu_notifier_register_begin();
1257 for_each_online_cpu(i
) {
1258 retval
= intel_idle_cpu_init(i
);
1260 intel_idle_cpuidle_devices_uninit();
1261 cpu_notifier_register_done();
1262 cpuidle_unregister_driver(&intel_idle_driver
);
1263 free_percpu(intel_idle_cpuidle_devices
);
1267 __register_cpu_notifier(&cpu_hotplug_notifier
);
1269 if (boot_cpu_has(X86_FEATURE_ARAT
)) /* Always Reliable APIC Timer */
1270 lapic_timer_reliable_states
= LAPIC_TIMER_ALWAYS_RELIABLE
;
1272 on_each_cpu(__setup_broadcast_timer
, (void *)true, 1);
1274 cpu_notifier_register_done();
1276 pr_debug(PREFIX
"lapic_timer_reliable_states 0x%x\n",
1277 lapic_timer_reliable_states
);
1282 static void __exit
intel_idle_exit(void)
1284 struct cpuidle_device
*dev
;
1287 cpu_notifier_register_begin();
1289 if (lapic_timer_reliable_states
!= LAPIC_TIMER_ALWAYS_RELIABLE
)
1290 on_each_cpu(__setup_broadcast_timer
, (void *)false, 1);
1291 __unregister_cpu_notifier(&cpu_hotplug_notifier
);
1293 for_each_possible_cpu(i
) {
1294 dev
= per_cpu_ptr(intel_idle_cpuidle_devices
, i
);
1295 cpuidle_unregister_device(dev
);
1298 cpu_notifier_register_done();
1300 cpuidle_unregister_driver(&intel_idle_driver
);
1301 free_percpu(intel_idle_cpuidle_devices
);
1304 module_init(intel_idle_init
);
1305 module_exit(intel_idle_exit
);
1307 module_param(max_cstate
, int, 0444);
1309 MODULE_AUTHOR("Len Brown <len.brown@intel.com>");
1310 MODULE_DESCRIPTION("Cpuidle driver for Intel Hardware v" INTEL_IDLE_VERSION
);
1311 MODULE_LICENSE("GPL");