?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
treewide: remove redundant IS_ERR() before error code check
[linux/fpc-iii.git] / drivers / idle / intel_idle.c
blob7833e650789f815fca66db5ab786a89ceb6f4eed
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * intel_idle.c - native hardware idle loop for modern Intel processors
4 *
5 * Copyright (c) 2013, Intel Corporation.
6 * Len Brown <len.brown@intel.com>
7 */
8
9 /*
10 * intel_idle is a cpuidle driver that loads on specific Intel processors
11 * in lieu of the legacy ACPI processor_idle driver. The intent is to
12 * make Linux more efficient on these processors, as intel_idle knows
13 * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs.
14 */
15
16 /*
17 * Design Assumptions
18 *
19 * All CPUs have same idle states as boot CPU
20 *
21 * Chipset BM_STS (bus master status) bit is a NOP
22 * for preventing entry into deep C-stats
23 */
24
25 /*
26 * Known limitations
27 *
28 * The driver currently initializes for_each_online_cpu() upon modprobe.
29 * It it unaware of subsequent processors hot-added to the system.
30 * This means that if you boot with maxcpus=n and later online
31 * processors above n, those processors will use C1 only.
32 *
33 * ACPI has a .suspend hack to turn off deep c-statees during suspend
34 * to avoid complications with the lapic timer workaround.
35 * Have not seen issues with suspend, but may need same workaround here.
36 *
37 */
38
39 /* un-comment DEBUG to enable pr_debug() statements */
40 #define DEBUG
41
42 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
43
44 #include <linux/acpi.h>
45 #include <linux/kernel.h>
46 #include <linux/cpuidle.h>
47 #include <linux/tick.h>
48 #include <trace/events/power.h>
49 #include <linux/sched.h>
50 #include <linux/notifier.h>
51 #include <linux/cpu.h>
52 #include <linux/moduleparam.h>
53 #include <asm/cpu_device_id.h>
54 #include <asm/intel-family.h>
55 #include <asm/mwait.h>
56 #include <asm/msr.h>
57
58 #define INTEL_IDLE_VERSION "0.4.1"
59
60 static struct cpuidle_driver intel_idle_driver = {
61 .name = "intel_idle",
62 .owner = THIS_MODULE,
63 };
64 /* intel_idle.max_cstate=0 disables driver */
65 static int max_cstate = CPUIDLE_STATE_MAX - 1;
66
67 static unsigned int mwait_substates;
68
69 #define LAPIC_TIMER_ALWAYS_RELIABLE 0xFFFFFFFF
70 /* Reliable LAPIC Timer States, bit 1 for C1 etc. */
71 static unsigned int lapic_timer_reliable_states = (1 << 1); /* Default to only C1 */
72
73 struct idle_cpu {
74 struct cpuidle_state *state_table;
75
76 /*
77 * Hardware C-state auto-demotion may not always be optimal.
78 * Indicate which enable bits to clear here.
79 */
80 unsigned long auto_demotion_disable_flags;
81 bool byt_auto_demotion_disable_flag;
82 bool disable_promotion_to_c1e;
83 bool use_acpi;
84 };
85
86 static const struct idle_cpu *icpu;
87 static struct cpuidle_device __percpu *intel_idle_cpuidle_devices;
88 static int intel_idle(struct cpuidle_device *dev,
89 struct cpuidle_driver *drv, int index);
90 static void intel_idle_s2idle(struct cpuidle_device *dev,
91 struct cpuidle_driver *drv, int index);
92 static struct cpuidle_state *cpuidle_state_table;
93
94 /*
95 * Enable this state by default even if the ACPI _CST does not list it.
96 */
97 #define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15)
98
99 /*
100 * Set this flag for states where the HW flushes the TLB for us
101 * and so we don't need cross-calls to keep it consistent.
102 * If this flag is set, SW flushes the TLB, so even if the
103 * HW doesn't do the flushing, this flag is safe to use.
104 */
105 #define CPUIDLE_FLAG_TLB_FLUSHED 0x10000
106
107 /*
108 * MWAIT takes an 8-bit "hint" in EAX "suggesting"
109 * the C-state (top nibble) and sub-state (bottom nibble)
110 * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc.
111 *
112 * We store the hint at the top of our "flags" for each state.
113 */
114 #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF)
115 #define MWAIT2flg(eax) ((eax & 0xFF) << 24)
116
117 /*
118 * States are indexed by the cstate number,
119 * which is also the index into the MWAIT hint array.
120 * Thus C0 is a dummy.
121 */
122 static struct cpuidle_state nehalem_cstates[] = {
123 {
124 .name = "C1",
125 .desc = "MWAIT 0x00",
126 .flags = MWAIT2flg(0x00),
127 .exit_latency = 3,
128 .target_residency = 6,
129 .enter = &intel_idle,
130 .enter_s2idle = intel_idle_s2idle, },
131 {
132 .name = "C1E",
133 .desc = "MWAIT 0x01",
134 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
135 .exit_latency = 10,
136 .target_residency = 20,
137 .enter = &intel_idle,
138 .enter_s2idle = intel_idle_s2idle, },
139 {
140 .name = "C3",
141 .desc = "MWAIT 0x10",
142 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
143 .exit_latency = 20,
144 .target_residency = 80,
145 .enter = &intel_idle,
146 .enter_s2idle = intel_idle_s2idle, },
147 {
148 .name = "C6",
149 .desc = "MWAIT 0x20",
150 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
151 .exit_latency = 200,
152 .target_residency = 800,
153 .enter = &intel_idle,
154 .enter_s2idle = intel_idle_s2idle, },
155 {
156 .enter = NULL }
157 };
158
159 static struct cpuidle_state snb_cstates[] = {
160 {
161 .name = "C1",
162 .desc = "MWAIT 0x00",
163 .flags = MWAIT2flg(0x00),
164 .exit_latency = 2,
165 .target_residency = 2,
166 .enter = &intel_idle,
167 .enter_s2idle = intel_idle_s2idle, },
168 {
169 .name = "C1E",
170 .desc = "MWAIT 0x01",
171 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
172 .exit_latency = 10,
173 .target_residency = 20,
174 .enter = &intel_idle,
175 .enter_s2idle = intel_idle_s2idle, },
176 {
177 .name = "C3",
178 .desc = "MWAIT 0x10",
179 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
180 .exit_latency = 80,
181 .target_residency = 211,
182 .enter = &intel_idle,
183 .enter_s2idle = intel_idle_s2idle, },
184 {
185 .name = "C6",
186 .desc = "MWAIT 0x20",
187 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
188 .exit_latency = 104,
189 .target_residency = 345,
190 .enter = &intel_idle,
191 .enter_s2idle = intel_idle_s2idle, },
192 {
193 .name = "C7",
194 .desc = "MWAIT 0x30",
195 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
196 .exit_latency = 109,
197 .target_residency = 345,
198 .enter = &intel_idle,
199 .enter_s2idle = intel_idle_s2idle, },
200 {
201 .enter = NULL }
202 };
203
204 static struct cpuidle_state byt_cstates[] = {
205 {
206 .name = "C1",
207 .desc = "MWAIT 0x00",
208 .flags = MWAIT2flg(0x00),
209 .exit_latency = 1,
210 .target_residency = 1,
211 .enter = &intel_idle,
212 .enter_s2idle = intel_idle_s2idle, },
213 {
214 .name = "C6N",
215 .desc = "MWAIT 0x58",
216 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
217 .exit_latency = 300,
218 .target_residency = 275,
219 .enter = &intel_idle,
220 .enter_s2idle = intel_idle_s2idle, },
221 {
222 .name = "C6S",
223 .desc = "MWAIT 0x52",
224 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
225 .exit_latency = 500,
226 .target_residency = 560,
227 .enter = &intel_idle,
228 .enter_s2idle = intel_idle_s2idle, },
229 {
230 .name = "C7",
231 .desc = "MWAIT 0x60",
232 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
233 .exit_latency = 1200,
234 .target_residency = 4000,
235 .enter = &intel_idle,
236 .enter_s2idle = intel_idle_s2idle, },
237 {
238 .name = "C7S",
239 .desc = "MWAIT 0x64",
240 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
241 .exit_latency = 10000,
242 .target_residency = 20000,
243 .enter = &intel_idle,
244 .enter_s2idle = intel_idle_s2idle, },
245 {
246 .enter = NULL }
247 };
248
249 static struct cpuidle_state cht_cstates[] = {
250 {
251 .name = "C1",
252 .desc = "MWAIT 0x00",
253 .flags = MWAIT2flg(0x00),
254 .exit_latency = 1,
255 .target_residency = 1,
256 .enter = &intel_idle,
257 .enter_s2idle = intel_idle_s2idle, },
258 {
259 .name = "C6N",
260 .desc = "MWAIT 0x58",
261 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
262 .exit_latency = 80,
263 .target_residency = 275,
264 .enter = &intel_idle,
265 .enter_s2idle = intel_idle_s2idle, },
266 {
267 .name = "C6S",
268 .desc = "MWAIT 0x52",
269 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
270 .exit_latency = 200,
271 .target_residency = 560,
272 .enter = &intel_idle,
273 .enter_s2idle = intel_idle_s2idle, },
274 {
275 .name = "C7",
276 .desc = "MWAIT 0x60",
277 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
278 .exit_latency = 1200,
279 .target_residency = 4000,
280 .enter = &intel_idle,
281 .enter_s2idle = intel_idle_s2idle, },
282 {
283 .name = "C7S",
284 .desc = "MWAIT 0x64",
285 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
286 .exit_latency = 10000,
287 .target_residency = 20000,
288 .enter = &intel_idle,
289 .enter_s2idle = intel_idle_s2idle, },
290 {
291 .enter = NULL }
292 };
293
294 static struct cpuidle_state ivb_cstates[] = {
295 {
296 .name = "C1",
297 .desc = "MWAIT 0x00",
298 .flags = MWAIT2flg(0x00),
299 .exit_latency = 1,
300 .target_residency = 1,
301 .enter = &intel_idle,
302 .enter_s2idle = intel_idle_s2idle, },
303 {
304 .name = "C1E",
305 .desc = "MWAIT 0x01",
306 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
307 .exit_latency = 10,
308 .target_residency = 20,
309 .enter = &intel_idle,
310 .enter_s2idle = intel_idle_s2idle, },
311 {
312 .name = "C3",
313 .desc = "MWAIT 0x10",
314 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
315 .exit_latency = 59,
316 .target_residency = 156,
317 .enter = &intel_idle,
318 .enter_s2idle = intel_idle_s2idle, },
319 {
320 .name = "C6",
321 .desc = "MWAIT 0x20",
322 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
323 .exit_latency = 80,
324 .target_residency = 300,
325 .enter = &intel_idle,
326 .enter_s2idle = intel_idle_s2idle, },
327 {
328 .name = "C7",
329 .desc = "MWAIT 0x30",
330 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
331 .exit_latency = 87,
332 .target_residency = 300,
333 .enter = &intel_idle,
334 .enter_s2idle = intel_idle_s2idle, },
335 {
336 .enter = NULL }
337 };
338
339 static struct cpuidle_state ivt_cstates[] = {
340 {
341 .name = "C1",
342 .desc = "MWAIT 0x00",
343 .flags = MWAIT2flg(0x00),
344 .exit_latency = 1,
345 .target_residency = 1,
346 .enter = &intel_idle,
347 .enter_s2idle = intel_idle_s2idle, },
348 {
349 .name = "C1E",
350 .desc = "MWAIT 0x01",
351 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
352 .exit_latency = 10,
353 .target_residency = 80,
354 .enter = &intel_idle,
355 .enter_s2idle = intel_idle_s2idle, },
356 {
357 .name = "C3",
358 .desc = "MWAIT 0x10",
359 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
360 .exit_latency = 59,
361 .target_residency = 156,
362 .enter = &intel_idle,
363 .enter_s2idle = intel_idle_s2idle, },
364 {
365 .name = "C6",
366 .desc = "MWAIT 0x20",
367 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
368 .exit_latency = 82,
369 .target_residency = 300,
370 .enter = &intel_idle,
371 .enter_s2idle = intel_idle_s2idle, },
372 {
373 .enter = NULL }
374 };
375
376 static struct cpuidle_state ivt_cstates_4s[] = {
377 {
378 .name = "C1",
379 .desc = "MWAIT 0x00",
380 .flags = MWAIT2flg(0x00),
381 .exit_latency = 1,
382 .target_residency = 1,
383 .enter = &intel_idle,
384 .enter_s2idle = intel_idle_s2idle, },
385 {
386 .name = "C1E",
387 .desc = "MWAIT 0x01",
388 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
389 .exit_latency = 10,
390 .target_residency = 250,
391 .enter = &intel_idle,
392 .enter_s2idle = intel_idle_s2idle, },
393 {
394 .name = "C3",
395 .desc = "MWAIT 0x10",
396 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
397 .exit_latency = 59,
398 .target_residency = 300,
399 .enter = &intel_idle,
400 .enter_s2idle = intel_idle_s2idle, },
401 {
402 .name = "C6",
403 .desc = "MWAIT 0x20",
404 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
405 .exit_latency = 84,
406 .target_residency = 400,
407 .enter = &intel_idle,
408 .enter_s2idle = intel_idle_s2idle, },
409 {
410 .enter = NULL }
411 };
412
413 static struct cpuidle_state ivt_cstates_8s[] = {
414 {
415 .name = "C1",
416 .desc = "MWAIT 0x00",
417 .flags = MWAIT2flg(0x00),
418 .exit_latency = 1,
419 .target_residency = 1,
420 .enter = &intel_idle,
421 .enter_s2idle = intel_idle_s2idle, },
422 {
423 .name = "C1E",
424 .desc = "MWAIT 0x01",
425 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
426 .exit_latency = 10,
427 .target_residency = 500,
428 .enter = &intel_idle,
429 .enter_s2idle = intel_idle_s2idle, },
430 {
431 .name = "C3",
432 .desc = "MWAIT 0x10",
433 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
434 .exit_latency = 59,
435 .target_residency = 600,
436 .enter = &intel_idle,
437 .enter_s2idle = intel_idle_s2idle, },
438 {
439 .name = "C6",
440 .desc = "MWAIT 0x20",
441 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
442 .exit_latency = 88,
443 .target_residency = 700,
444 .enter = &intel_idle,
445 .enter_s2idle = intel_idle_s2idle, },
446 {
447 .enter = NULL }
448 };
449
450 static struct cpuidle_state hsw_cstates[] = {
451 {
452 .name = "C1",
453 .desc = "MWAIT 0x00",
454 .flags = MWAIT2flg(0x00),
455 .exit_latency = 2,
456 .target_residency = 2,
457 .enter = &intel_idle,
458 .enter_s2idle = intel_idle_s2idle, },
459 {
460 .name = "C1E",
461 .desc = "MWAIT 0x01",
462 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
463 .exit_latency = 10,
464 .target_residency = 20,
465 .enter = &intel_idle,
466 .enter_s2idle = intel_idle_s2idle, },
467 {
468 .name = "C3",
469 .desc = "MWAIT 0x10",
470 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
471 .exit_latency = 33,
472 .target_residency = 100,
473 .enter = &intel_idle,
474 .enter_s2idle = intel_idle_s2idle, },
475 {
476 .name = "C6",
477 .desc = "MWAIT 0x20",
478 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
479 .exit_latency = 133,
480 .target_residency = 400,
481 .enter = &intel_idle,
482 .enter_s2idle = intel_idle_s2idle, },
483 {
484 .name = "C7s",
485 .desc = "MWAIT 0x32",
486 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
487 .exit_latency = 166,
488 .target_residency = 500,
489 .enter = &intel_idle,
490 .enter_s2idle = intel_idle_s2idle, },
491 {
492 .name = "C8",
493 .desc = "MWAIT 0x40",
494 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
495 .exit_latency = 300,
496 .target_residency = 900,
497 .enter = &intel_idle,
498 .enter_s2idle = intel_idle_s2idle, },
499 {
500 .name = "C9",
501 .desc = "MWAIT 0x50",
502 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
503 .exit_latency = 600,
504 .target_residency = 1800,
505 .enter = &intel_idle,
506 .enter_s2idle = intel_idle_s2idle, },
507 {
508 .name = "C10",
509 .desc = "MWAIT 0x60",
510 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
511 .exit_latency = 2600,
512 .target_residency = 7700,
513 .enter = &intel_idle,
514 .enter_s2idle = intel_idle_s2idle, },
515 {
516 .enter = NULL }
517 };
518 static struct cpuidle_state bdw_cstates[] = {
519 {
520 .name = "C1",
521 .desc = "MWAIT 0x00",
522 .flags = MWAIT2flg(0x00),
523 .exit_latency = 2,
524 .target_residency = 2,
525 .enter = &intel_idle,
526 .enter_s2idle = intel_idle_s2idle, },
527 {
528 .name = "C1E",
529 .desc = "MWAIT 0x01",
530 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
531 .exit_latency = 10,
532 .target_residency = 20,
533 .enter = &intel_idle,
534 .enter_s2idle = intel_idle_s2idle, },
535 {
536 .name = "C3",
537 .desc = "MWAIT 0x10",
538 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
539 .exit_latency = 40,
540 .target_residency = 100,
541 .enter = &intel_idle,
542 .enter_s2idle = intel_idle_s2idle, },
543 {
544 .name = "C6",
545 .desc = "MWAIT 0x20",
546 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
547 .exit_latency = 133,
548 .target_residency = 400,
549 .enter = &intel_idle,
550 .enter_s2idle = intel_idle_s2idle, },
551 {
552 .name = "C7s",
553 .desc = "MWAIT 0x32",
554 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
555 .exit_latency = 166,
556 .target_residency = 500,
557 .enter = &intel_idle,
558 .enter_s2idle = intel_idle_s2idle, },
559 {
560 .name = "C8",
561 .desc = "MWAIT 0x40",
562 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
563 .exit_latency = 300,
564 .target_residency = 900,
565 .enter = &intel_idle,
566 .enter_s2idle = intel_idle_s2idle, },
567 {
568 .name = "C9",
569 .desc = "MWAIT 0x50",
570 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
571 .exit_latency = 600,
572 .target_residency = 1800,
573 .enter = &intel_idle,
574 .enter_s2idle = intel_idle_s2idle, },
575 {
576 .name = "C10",
577 .desc = "MWAIT 0x60",
578 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
579 .exit_latency = 2600,
580 .target_residency = 7700,
581 .enter = &intel_idle,
582 .enter_s2idle = intel_idle_s2idle, },
583 {
584 .enter = NULL }
585 };
586
587 static struct cpuidle_state skl_cstates[] = {
588 {
589 .name = "C1",
590 .desc = "MWAIT 0x00",
591 .flags = MWAIT2flg(0x00),
592 .exit_latency = 2,
593 .target_residency = 2,
594 .enter = &intel_idle,
595 .enter_s2idle = intel_idle_s2idle, },
596 {
597 .name = "C1E",
598 .desc = "MWAIT 0x01",
599 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
600 .exit_latency = 10,
601 .target_residency = 20,
602 .enter = &intel_idle,
603 .enter_s2idle = intel_idle_s2idle, },
604 {
605 .name = "C3",
606 .desc = "MWAIT 0x10",
607 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
608 .exit_latency = 70,
609 .target_residency = 100,
610 .enter = &intel_idle,
611 .enter_s2idle = intel_idle_s2idle, },
612 {
613 .name = "C6",
614 .desc = "MWAIT 0x20",
615 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
616 .exit_latency = 85,
617 .target_residency = 200,
618 .enter = &intel_idle,
619 .enter_s2idle = intel_idle_s2idle, },
620 {
621 .name = "C7s",
622 .desc = "MWAIT 0x33",
623 .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED,
624 .exit_latency = 124,
625 .target_residency = 800,
626 .enter = &intel_idle,
627 .enter_s2idle = intel_idle_s2idle, },
628 {
629 .name = "C8",
630 .desc = "MWAIT 0x40",
631 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
632 .exit_latency = 200,
633 .target_residency = 800,
634 .enter = &intel_idle,
635 .enter_s2idle = intel_idle_s2idle, },
636 {
637 .name = "C9",
638 .desc = "MWAIT 0x50",
639 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
640 .exit_latency = 480,
641 .target_residency = 5000,
642 .enter = &intel_idle,
643 .enter_s2idle = intel_idle_s2idle, },
644 {
645 .name = "C10",
646 .desc = "MWAIT 0x60",
647 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
648 .exit_latency = 890,
649 .target_residency = 5000,
650 .enter = &intel_idle,
651 .enter_s2idle = intel_idle_s2idle, },
652 {
653 .enter = NULL }
654 };
655
656 static struct cpuidle_state skx_cstates[] = {
657 {
658 .name = "C1",
659 .desc = "MWAIT 0x00",
660 .flags = MWAIT2flg(0x00),
661 .exit_latency = 2,
662 .target_residency = 2,
663 .enter = &intel_idle,
664 .enter_s2idle = intel_idle_s2idle, },
665 {
666 .name = "C1E",
667 .desc = "MWAIT 0x01",
668 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
669 .exit_latency = 10,
670 .target_residency = 20,
671 .enter = &intel_idle,
672 .enter_s2idle = intel_idle_s2idle, },
673 {
674 .name = "C6",
675 .desc = "MWAIT 0x20",
676 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
677 .exit_latency = 133,
678 .target_residency = 600,
679 .enter = &intel_idle,
680 .enter_s2idle = intel_idle_s2idle, },
681 {
682 .enter = NULL }
683 };
684
685 static struct cpuidle_state atom_cstates[] = {
686 {
687 .name = "C1E",
688 .desc = "MWAIT 0x00",
689 .flags = MWAIT2flg(0x00),
690 .exit_latency = 10,
691 .target_residency = 20,
692 .enter = &intel_idle,
693 .enter_s2idle = intel_idle_s2idle, },
694 {
695 .name = "C2",
696 .desc = "MWAIT 0x10",
697 .flags = MWAIT2flg(0x10),
698 .exit_latency = 20,
699 .target_residency = 80,
700 .enter = &intel_idle,
701 .enter_s2idle = intel_idle_s2idle, },
702 {
703 .name = "C4",
704 .desc = "MWAIT 0x30",
705 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
706 .exit_latency = 100,
707 .target_residency = 400,
708 .enter = &intel_idle,
709 .enter_s2idle = intel_idle_s2idle, },
710 {
711 .name = "C6",
712 .desc = "MWAIT 0x52",
713 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
714 .exit_latency = 140,
715 .target_residency = 560,
716 .enter = &intel_idle,
717 .enter_s2idle = intel_idle_s2idle, },
718 {
719 .enter = NULL }
720 };
721 static struct cpuidle_state tangier_cstates[] = {
722 {
723 .name = "C1",
724 .desc = "MWAIT 0x00",
725 .flags = MWAIT2flg(0x00),
726 .exit_latency = 1,
727 .target_residency = 4,
728 .enter = &intel_idle,
729 .enter_s2idle = intel_idle_s2idle, },
730 {
731 .name = "C4",
732 .desc = "MWAIT 0x30",
733 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
734 .exit_latency = 100,
735 .target_residency = 400,
736 .enter = &intel_idle,
737 .enter_s2idle = intel_idle_s2idle, },
738 {
739 .name = "C6",
740 .desc = "MWAIT 0x52",
741 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
742 .exit_latency = 140,
743 .target_residency = 560,
744 .enter = &intel_idle,
745 .enter_s2idle = intel_idle_s2idle, },
746 {
747 .name = "C7",
748 .desc = "MWAIT 0x60",
749 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
750 .exit_latency = 1200,
751 .target_residency = 4000,
752 .enter = &intel_idle,
753 .enter_s2idle = intel_idle_s2idle, },
754 {
755 .name = "C9",
756 .desc = "MWAIT 0x64",
757 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
758 .exit_latency = 10000,
759 .target_residency = 20000,
760 .enter = &intel_idle,
761 .enter_s2idle = intel_idle_s2idle, },
762 {
763 .enter = NULL }
764 };
765 static struct cpuidle_state avn_cstates[] = {
766 {
767 .name = "C1",
768 .desc = "MWAIT 0x00",
769 .flags = MWAIT2flg(0x00),
770 .exit_latency = 2,
771 .target_residency = 2,
772 .enter = &intel_idle,
773 .enter_s2idle = intel_idle_s2idle, },
774 {
775 .name = "C6",
776 .desc = "MWAIT 0x51",
777 .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED,
778 .exit_latency = 15,
779 .target_residency = 45,
780 .enter = &intel_idle,
781 .enter_s2idle = intel_idle_s2idle, },
782 {
783 .enter = NULL }
784 };
785 static struct cpuidle_state knl_cstates[] = {
786 {
787 .name = "C1",
788 .desc = "MWAIT 0x00",
789 .flags = MWAIT2flg(0x00),
790 .exit_latency = 1,
791 .target_residency = 2,
792 .enter = &intel_idle,
793 .enter_s2idle = intel_idle_s2idle },
794 {
795 .name = "C6",
796 .desc = "MWAIT 0x10",
797 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
798 .exit_latency = 120,
799 .target_residency = 500,
800 .enter = &intel_idle,
801 .enter_s2idle = intel_idle_s2idle },
802 {
803 .enter = NULL }
804 };
805
806 static struct cpuidle_state bxt_cstates[] = {
807 {
808 .name = "C1",
809 .desc = "MWAIT 0x00",
810 .flags = MWAIT2flg(0x00),
811 .exit_latency = 2,
812 .target_residency = 2,
813 .enter = &intel_idle,
814 .enter_s2idle = intel_idle_s2idle, },
815 {
816 .name = "C1E",
817 .desc = "MWAIT 0x01",
818 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
819 .exit_latency = 10,
820 .target_residency = 20,
821 .enter = &intel_idle,
822 .enter_s2idle = intel_idle_s2idle, },
823 {
824 .name = "C6",
825 .desc = "MWAIT 0x20",
826 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
827 .exit_latency = 133,
828 .target_residency = 133,
829 .enter = &intel_idle,
830 .enter_s2idle = intel_idle_s2idle, },
831 {
832 .name = "C7s",
833 .desc = "MWAIT 0x31",
834 .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED,
835 .exit_latency = 155,
836 .target_residency = 155,
837 .enter = &intel_idle,
838 .enter_s2idle = intel_idle_s2idle, },
839 {
840 .name = "C8",
841 .desc = "MWAIT 0x40",
842 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
843 .exit_latency = 1000,
844 .target_residency = 1000,
845 .enter = &intel_idle,
846 .enter_s2idle = intel_idle_s2idle, },
847 {
848 .name = "C9",
849 .desc = "MWAIT 0x50",
850 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
851 .exit_latency = 2000,
852 .target_residency = 2000,
853 .enter = &intel_idle,
854 .enter_s2idle = intel_idle_s2idle, },
855 {
856 .name = "C10",
857 .desc = "MWAIT 0x60",
858 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
859 .exit_latency = 10000,
860 .target_residency = 10000,
861 .enter = &intel_idle,
862 .enter_s2idle = intel_idle_s2idle, },
863 {
864 .enter = NULL }
865 };
866
867 static struct cpuidle_state dnv_cstates[] = {
868 {
869 .name = "C1",
870 .desc = "MWAIT 0x00",
871 .flags = MWAIT2flg(0x00),
872 .exit_latency = 2,
873 .target_residency = 2,
874 .enter = &intel_idle,
875 .enter_s2idle = intel_idle_s2idle, },
876 {
877 .name = "C1E",
878 .desc = "MWAIT 0x01",
879 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
880 .exit_latency = 10,
881 .target_residency = 20,
882 .enter = &intel_idle,
883 .enter_s2idle = intel_idle_s2idle, },
884 {
885 .name = "C6",
886 .desc = "MWAIT 0x20",
887 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
888 .exit_latency = 50,
889 .target_residency = 500,
890 .enter = &intel_idle,
891 .enter_s2idle = intel_idle_s2idle, },
892 {
893 .enter = NULL }
894 };
895
896 /**
897 * intel_idle
898 * @dev: cpuidle_device
899 * @drv: cpuidle driver
900 * @index: index of cpuidle state
901 *
902 * Must be called under local_irq_disable().
903 */
904 static __cpuidle int intel_idle(struct cpuidle_device *dev,
905 struct cpuidle_driver *drv, int index)
906 {
907 unsigned long ecx = 1; /* break on interrupt flag */
908 struct cpuidle_state *state = &drv->states[index];
909 unsigned long eax = flg2MWAIT(state->flags);
910 unsigned int cstate;
911 bool uninitialized_var(tick);
912 int cpu = smp_processor_id();
913
914 /*
915 * leave_mm() to avoid costly and often unnecessary wakeups
916 * for flushing the user TLB's associated with the active mm.
917 */
918 if (state->flags & CPUIDLE_FLAG_TLB_FLUSHED)
919 leave_mm(cpu);
920
921 if (!static_cpu_has(X86_FEATURE_ARAT)) {
922 cstate = (((eax) >> MWAIT_SUBSTATE_SIZE) &
923 MWAIT_CSTATE_MASK) + 1;
924 tick = false;
925 if (!(lapic_timer_reliable_states & (1 << (cstate)))) {
926 tick = true;
927 tick_broadcast_enter();
928 }
929 }
930
931 mwait_idle_with_hints(eax, ecx);
932
933 if (!static_cpu_has(X86_FEATURE_ARAT) && tick)
934 tick_broadcast_exit();
935
936 return index;
937 }
938
939 /**
940 * intel_idle_s2idle - simplified "enter" callback routine for suspend-to-idle
941 * @dev: cpuidle_device
942 * @drv: cpuidle driver
943 * @index: state index
944 */
945 static void intel_idle_s2idle(struct cpuidle_device *dev,
946 struct cpuidle_driver *drv, int index)
947 {
948 unsigned long ecx = 1; /* break on interrupt flag */
949 unsigned long eax = flg2MWAIT(drv->states[index].flags);
950
951 mwait_idle_with_hints(eax, ecx);
952 }
953
954 static const struct idle_cpu idle_cpu_nehalem = {
955 .state_table = nehalem_cstates,
956 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
957 .disable_promotion_to_c1e = true,
958 };
959
960 static const struct idle_cpu idle_cpu_nhx = {
961 .state_table = nehalem_cstates,
962 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
963 .disable_promotion_to_c1e = true,
964 .use_acpi = true,
965 };
966
967 static const struct idle_cpu idle_cpu_atom = {
968 .state_table = atom_cstates,
969 };
970
971 static const struct idle_cpu idle_cpu_tangier = {
972 .state_table = tangier_cstates,
973 };
974
975 static const struct idle_cpu idle_cpu_lincroft = {
976 .state_table = atom_cstates,
977 .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE,
978 };
979
980 static const struct idle_cpu idle_cpu_snb = {
981 .state_table = snb_cstates,
982 .disable_promotion_to_c1e = true,
983 };
984
985 static const struct idle_cpu idle_cpu_snx = {
986 .state_table = snb_cstates,
987 .disable_promotion_to_c1e = true,
988 .use_acpi = true,
989 };
990
991 static const struct idle_cpu idle_cpu_byt = {
992 .state_table = byt_cstates,
993 .disable_promotion_to_c1e = true,
994 .byt_auto_demotion_disable_flag = true,
995 };
996
997 static const struct idle_cpu idle_cpu_cht = {
998 .state_table = cht_cstates,
999 .disable_promotion_to_c1e = true,
1000 .byt_auto_demotion_disable_flag = true,
1001 };
1002
1003 static const struct idle_cpu idle_cpu_ivb = {
1004 .state_table = ivb_cstates,
1005 .disable_promotion_to_c1e = true,
1006 };
1007
1008 static const struct idle_cpu idle_cpu_ivt = {
1009 .state_table = ivt_cstates,
1010 .disable_promotion_to_c1e = true,
1011 .use_acpi = true,
1012 };
1013
1014 static const struct idle_cpu idle_cpu_hsw = {
1015 .state_table = hsw_cstates,
1016 .disable_promotion_to_c1e = true,
1017 };
1018
1019 static const struct idle_cpu idle_cpu_hsx = {
1020 .state_table = hsw_cstates,
1021 .disable_promotion_to_c1e = true,
1022 .use_acpi = true,
1023 };
1024
1025 static const struct idle_cpu idle_cpu_bdw = {
1026 .state_table = bdw_cstates,
1027 .disable_promotion_to_c1e = true,
1028 };
1029
1030 static const struct idle_cpu idle_cpu_bdx = {
1031 .state_table = bdw_cstates,
1032 .disable_promotion_to_c1e = true,
1033 .use_acpi = true,
1034 };
1035
1036 static const struct idle_cpu idle_cpu_skl = {
1037 .state_table = skl_cstates,
1038 .disable_promotion_to_c1e = true,
1039 };
1040
1041 static const struct idle_cpu idle_cpu_skx = {
1042 .state_table = skx_cstates,
1043 .disable_promotion_to_c1e = true,
1044 .use_acpi = true,
1045 };
1046
1047 static const struct idle_cpu idle_cpu_avn = {
1048 .state_table = avn_cstates,
1049 .disable_promotion_to_c1e = true,
1050 .use_acpi = true,
1051 };
1052
1053 static const struct idle_cpu idle_cpu_knl = {
1054 .state_table = knl_cstates,
1055 .use_acpi = true,
1056 };
1057
1058 static const struct idle_cpu idle_cpu_bxt = {
1059 .state_table = bxt_cstates,
1060 .disable_promotion_to_c1e = true,
1061 };
1062
1063 static const struct idle_cpu idle_cpu_dnv = {
1064 .state_table = dnv_cstates,
1065 .disable_promotion_to_c1e = true,
1066 .use_acpi = true,
1067 };
1068
1069 static const struct x86_cpu_id intel_idle_ids[] __initconst = {
1070 INTEL_CPU_FAM6(NEHALEM_EP, idle_cpu_nhx),
1071 INTEL_CPU_FAM6(NEHALEM, idle_cpu_nehalem),
1072 INTEL_CPU_FAM6(NEHALEM_G, idle_cpu_nehalem),
1073 INTEL_CPU_FAM6(WESTMERE, idle_cpu_nehalem),
1074 INTEL_CPU_FAM6(WESTMERE_EP, idle_cpu_nhx),
1075 INTEL_CPU_FAM6(NEHALEM_EX, idle_cpu_nhx),
1076 INTEL_CPU_FAM6(ATOM_BONNELL, idle_cpu_atom),
1077 INTEL_CPU_FAM6(ATOM_BONNELL_MID, idle_cpu_lincroft),
1078 INTEL_CPU_FAM6(WESTMERE_EX, idle_cpu_nhx),
1079 INTEL_CPU_FAM6(SANDYBRIDGE, idle_cpu_snb),
1080 INTEL_CPU_FAM6(SANDYBRIDGE_X, idle_cpu_snx),
1081 INTEL_CPU_FAM6(ATOM_SALTWELL, idle_cpu_atom),
1082 INTEL_CPU_FAM6(ATOM_SILVERMONT, idle_cpu_byt),
1083 INTEL_CPU_FAM6(ATOM_SILVERMONT_MID, idle_cpu_tangier),
1084 INTEL_CPU_FAM6(ATOM_AIRMONT, idle_cpu_cht),
1085 INTEL_CPU_FAM6(IVYBRIDGE, idle_cpu_ivb),
1086 INTEL_CPU_FAM6(IVYBRIDGE_X, idle_cpu_ivt),
1087 INTEL_CPU_FAM6(HASWELL, idle_cpu_hsw),
1088 INTEL_CPU_FAM6(HASWELL_X, idle_cpu_hsx),
1089 INTEL_CPU_FAM6(HASWELL_L, idle_cpu_hsw),
1090 INTEL_CPU_FAM6(HASWELL_G, idle_cpu_hsw),
1091 INTEL_CPU_FAM6(ATOM_SILVERMONT_D, idle_cpu_avn),
1092 INTEL_CPU_FAM6(BROADWELL, idle_cpu_bdw),
1093 INTEL_CPU_FAM6(BROADWELL_G, idle_cpu_bdw),
1094 INTEL_CPU_FAM6(BROADWELL_X, idle_cpu_bdx),
1095 INTEL_CPU_FAM6(BROADWELL_D, idle_cpu_bdx),
1096 INTEL_CPU_FAM6(SKYLAKE_L, idle_cpu_skl),
1097 INTEL_CPU_FAM6(SKYLAKE, idle_cpu_skl),
1098 INTEL_CPU_FAM6(KABYLAKE_L, idle_cpu_skl),
1099 INTEL_CPU_FAM6(KABYLAKE, idle_cpu_skl),
1100 INTEL_CPU_FAM6(SKYLAKE_X, idle_cpu_skx),
1101 INTEL_CPU_FAM6(XEON_PHI_KNL, idle_cpu_knl),
1102 INTEL_CPU_FAM6(XEON_PHI_KNM, idle_cpu_knl),
1103 INTEL_CPU_FAM6(ATOM_GOLDMONT, idle_cpu_bxt),
1104 INTEL_CPU_FAM6(ATOM_GOLDMONT_PLUS, idle_cpu_bxt),
1105 INTEL_CPU_FAM6(ATOM_GOLDMONT_D, idle_cpu_dnv),
1106 INTEL_CPU_FAM6(ATOM_TREMONT_D, idle_cpu_dnv),
1107 {}
1108 };
1109
1110 #define INTEL_CPU_FAM6_MWAIT \
1111 { X86_VENDOR_INTEL, 6, X86_MODEL_ANY, X86_FEATURE_MWAIT, 0 }
1112
1113 static const struct x86_cpu_id intel_mwait_ids[] __initconst = {
1114 INTEL_CPU_FAM6_MWAIT,
1115 {}
1116 };
1117
1118 static bool __init intel_idle_max_cstate_reached(int cstate)
1119 {
1120 if (cstate + 1 > max_cstate) {
1121 pr_info("max_cstate %d reached\n", max_cstate);
1122 return true;
1123 }
1124 return false;
1125 }
1126
1127 #ifdef CONFIG_ACPI_PROCESSOR_CSTATE
1128 #include <acpi/processor.h>
1129
1130 static bool no_acpi __read_mostly;
1131 module_param(no_acpi, bool, 0444);
1132 MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list");
1133
1134 static struct acpi_processor_power acpi_state_table __initdata;
1135
1136 /**
1137 * intel_idle_cst_usable - Check if the _CST information can be used.
1138 *
1139 * Check if all of the C-states listed by _CST in the max_cstate range are
1140 * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT.
1141 */
1142 static bool __init intel_idle_cst_usable(void)
1143 {
1144 int cstate, limit;
1145
1146 limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1),
1147 acpi_state_table.count);
1148
1149 for (cstate = 1; cstate < limit; cstate++) {
1150 struct acpi_processor_cx *cx = &acpi_state_table.states[cstate];
1151
1152 if (cx->entry_method != ACPI_CSTATE_FFH)
1153 return false;
1154 }
1155
1156 return true;
1157 }
1158
1159 static bool __init intel_idle_acpi_cst_extract(void)
1160 {
1161 unsigned int cpu;
1162
1163 if (no_acpi) {
1164 pr_debug("Not allowed to use ACPI _CST\n");
1165 return false;
1166 }
1167
1168 for_each_possible_cpu(cpu) {
1169 struct acpi_processor *pr = per_cpu(processors, cpu);
1170
1171 if (!pr)
1172 continue;
1173
1174 if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table))
1175 continue;
1176
1177 acpi_state_table.count++;
1178
1179 if (!intel_idle_cst_usable())
1180 continue;
1181
1182 if (!acpi_processor_claim_cst_control()) {
1183 acpi_state_table.count = 0;
1184 return false;
1185 }
1186
1187 return true;
1188 }
1189
1190 pr_debug("ACPI _CST not found or not usable\n");
1191 return false;
1192 }
1193
1194 static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv)
1195 {
1196 int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
1197
1198 /*
1199 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
1200 * the interesting states are ACPI_CSTATE_FFH.
1201 */
1202 for (cstate = 1; cstate < limit; cstate++) {
1203 struct acpi_processor_cx *cx;
1204 struct cpuidle_state *state;
1205
1206 if (intel_idle_max_cstate_reached(cstate))
1207 break;
1208
1209 cx = &acpi_state_table.states[cstate];
1210
1211 state = &drv->states[drv->state_count++];
1212
1213 snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate);
1214 strlcpy(state->desc, cx->desc, CPUIDLE_DESC_LEN);
1215 state->exit_latency = cx->latency;
1216 /*
1217 * For C1-type C-states use the same number for both the exit
1218 * latency and target residency, because that is the case for
1219 * C1 in the majority of the static C-states tables above.
1220 * For the other types of C-states, however, set the target
1221 * residency to 3 times the exit latency which should lead to
1222 * a reasonable balance between energy-efficiency and
1223 * performance in the majority of interesting cases.
1224 */
1225 state->target_residency = cx->latency;
1226 if (cx->type > ACPI_STATE_C1)
1227 state->target_residency *= 3;
1228
1229 state->flags = MWAIT2flg(cx->address);
1230 if (cx->type > ACPI_STATE_C2)
1231 state->flags |= CPUIDLE_FLAG_TLB_FLUSHED;
1232
1233 state->enter = intel_idle;
1234 state->enter_s2idle = intel_idle_s2idle;
1235 }
1236 }
1237
1238 static bool __init intel_idle_off_by_default(u32 mwait_hint)
1239 {
1240 int cstate, limit;
1241
1242 /*
1243 * If there are no _CST C-states, do not disable any C-states by
1244 * default.
1245 */
1246 if (!acpi_state_table.count)
1247 return false;
1248
1249 limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
1250 /*
1251 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
1252 * the interesting states are ACPI_CSTATE_FFH.
1253 */
1254 for (cstate = 1; cstate < limit; cstate++) {
1255 if (acpi_state_table.states[cstate].address == mwait_hint)
1256 return false;
1257 }
1258 return true;
1259 }
1260 #else /* !CONFIG_ACPI_PROCESSOR_CSTATE */
1261 static inline bool intel_idle_acpi_cst_extract(void) { return false; }
1262 static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { }
1263 static inline bool intel_idle_off_by_default(u32 mwait_hint) { return false; }
1264 #endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */
1265
1266 /*
1267 * ivt_idle_state_table_update(void)
1268 *
1269 * Tune IVT multi-socket targets
1270 * Assumption: num_sockets == (max_package_num + 1)
1271 */
1272 static void __init ivt_idle_state_table_update(void)
1273 {
1274 /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */
1275 int cpu, package_num, num_sockets = 1;
1276
1277 for_each_online_cpu(cpu) {
1278 package_num = topology_physical_package_id(cpu);
1279 if (package_num + 1 > num_sockets) {
1280 num_sockets = package_num + 1;
1281
1282 if (num_sockets > 4) {
1283 cpuidle_state_table = ivt_cstates_8s;
1284 return;
1285 }
1286 }
1287 }
1288
1289 if (num_sockets > 2)
1290 cpuidle_state_table = ivt_cstates_4s;
1291
1292 /* else, 1 and 2 socket systems use default ivt_cstates */
1293 }
1294
1295 /**
1296 * irtl_2_usec - IRTL to microseconds conversion.
1297 * @irtl: IRTL MSR value.
1298 *
1299 * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds.
1300 */
1301 static unsigned long long __init irtl_2_usec(unsigned long long irtl)
1302 {
1303 static const unsigned int irtl_ns_units[] __initconst = {
1304 1, 32, 1024, 32768, 1048576, 33554432, 0, 0
1305 };
1306 unsigned long long ns;
1307
1308 if (!irtl)
1309 return 0;
1310
1311 ns = irtl_ns_units[(irtl >> 10) & 0x7];
1312
1313 return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC);
1314 }
1315
1316 /*
1317 * bxt_idle_state_table_update(void)
1318 *
1319 * On BXT, we trust the IRTL to show the definitive maximum latency
1320 * We use the same value for target_residency.
1321 */
1322 static void __init bxt_idle_state_table_update(void)
1323 {
1324 unsigned long long msr;
1325 unsigned int usec;
1326
1327 rdmsrl(MSR_PKGC6_IRTL, msr);
1328 usec = irtl_2_usec(msr);
1329 if (usec) {
1330 bxt_cstates[2].exit_latency = usec;
1331 bxt_cstates[2].target_residency = usec;
1332 }
1333
1334 rdmsrl(MSR_PKGC7_IRTL, msr);
1335 usec = irtl_2_usec(msr);
1336 if (usec) {
1337 bxt_cstates[3].exit_latency = usec;
1338 bxt_cstates[3].target_residency = usec;
1339 }
1340
1341 rdmsrl(MSR_PKGC8_IRTL, msr);
1342 usec = irtl_2_usec(msr);
1343 if (usec) {
1344 bxt_cstates[4].exit_latency = usec;
1345 bxt_cstates[4].target_residency = usec;
1346 }
1347
1348 rdmsrl(MSR_PKGC9_IRTL, msr);
1349 usec = irtl_2_usec(msr);
1350 if (usec) {
1351 bxt_cstates[5].exit_latency = usec;
1352 bxt_cstates[5].target_residency = usec;
1353 }
1354
1355 rdmsrl(MSR_PKGC10_IRTL, msr);
1356 usec = irtl_2_usec(msr);
1357 if (usec) {
1358 bxt_cstates[6].exit_latency = usec;
1359 bxt_cstates[6].target_residency = usec;
1360 }
1361
1362 }
1363 /*
1364 * sklh_idle_state_table_update(void)
1365 *
1366 * On SKL-H (model 0x5e) disable C8 and C9 if:
1367 * C10 is enabled and SGX disabled
1368 */
1369 static void __init sklh_idle_state_table_update(void)
1370 {
1371 unsigned long long msr;
1372 unsigned int eax, ebx, ecx, edx;
1373
1374
1375 /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */
1376 if (max_cstate <= 7)
1377 return;
1378
1379 /* if PC10 not present in CPUID.MWAIT.EDX */
1380 if ((mwait_substates & (0xF << 28)) == 0)
1381 return;
1382
1383 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
1384
1385 /* PC10 is not enabled in PKG C-state limit */
1386 if ((msr & 0xF) != 8)
1387 return;
1388
1389 ecx = 0;
1390 cpuid(7, &eax, &ebx, &ecx, &edx);
1391
1392 /* if SGX is present */
1393 if (ebx & (1 << 2)) {
1394
1395 rdmsrl(MSR_IA32_FEAT_CTL, msr);
1396
1397 /* if SGX is enabled */
1398 if (msr & (1 << 18))
1399 return;
1400 }
1401
1402 skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */
1403 skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */
1404 }
1405
1406 static bool __init intel_idle_verify_cstate(unsigned int mwait_hint)
1407 {
1408 unsigned int mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint) + 1;
1409 unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) &
1410 MWAIT_SUBSTATE_MASK;
1411
1412 /* Ignore the C-state if there are NO sub-states in CPUID for it. */
1413 if (num_substates == 0)
1414 return false;
1415
1416 if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
1417 mark_tsc_unstable("TSC halts in idle states deeper than C2");
1418
1419 return true;
1420 }
1421
1422 static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv)
1423 {
1424 int cstate;
1425
1426 switch (boot_cpu_data.x86_model) {
1427 case INTEL_FAM6_IVYBRIDGE_X:
1428 ivt_idle_state_table_update();
1429 break;
1430 case INTEL_FAM6_ATOM_GOLDMONT:
1431 case INTEL_FAM6_ATOM_GOLDMONT_PLUS:
1432 bxt_idle_state_table_update();
1433 break;
1434 case INTEL_FAM6_SKYLAKE:
1435 sklh_idle_state_table_update();
1436 break;
1437 }
1438
1439 for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) {
1440 unsigned int mwait_hint;
1441
1442 if (intel_idle_max_cstate_reached(cstate))
1443 break;
1444
1445 if (!cpuidle_state_table[cstate].enter &&
1446 !cpuidle_state_table[cstate].enter_s2idle)
1447 break;
1448
1449 /* If marked as unusable, skip this state. */
1450 if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) {
1451 pr_debug("state %s is disabled\n",
1452 cpuidle_state_table[cstate].name);
1453 continue;
1454 }
1455
1456 mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags);
1457 if (!intel_idle_verify_cstate(mwait_hint))
1458 continue;
1459
1460 /* Structure copy. */
1461 drv->states[drv->state_count] = cpuidle_state_table[cstate];
1462
1463 if (icpu->use_acpi && intel_idle_off_by_default(mwait_hint) &&
1464 !(cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_ALWAYS_ENABLE))
1465 drv->states[drv->state_count].flags |= CPUIDLE_FLAG_OFF;
1466
1467 drv->state_count++;
1468 }
1469
1470 if (icpu->byt_auto_demotion_disable_flag) {
1471 wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0);
1472 wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0);
1473 }
1474 }
1475
1476 /*
1477 * intel_idle_cpuidle_driver_init()
1478 * allocate, initialize cpuidle_states
1479 */
1480 static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv)
1481 {
1482 cpuidle_poll_state_init(drv);
1483 drv->state_count = 1;
1484
1485 if (icpu)
1486 intel_idle_init_cstates_icpu(drv);
1487 else
1488 intel_idle_init_cstates_acpi(drv);
1489 }
1490
1491 static void auto_demotion_disable(void)
1492 {
1493 unsigned long long msr_bits;
1494
1495 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
1496 msr_bits &= ~(icpu->auto_demotion_disable_flags);
1497 wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
1498 }
1499
1500 static void c1e_promotion_disable(void)
1501 {
1502 unsigned long long msr_bits;
1503
1504 rdmsrl(MSR_IA32_POWER_CTL, msr_bits);
1505 msr_bits &= ~0x2;
1506 wrmsrl(MSR_IA32_POWER_CTL, msr_bits);
1507 }
1508
1509 /*
1510 * intel_idle_cpu_init()
1511 * allocate, initialize, register cpuidle_devices
1512 * @cpu: cpu/core to initialize
1513 */
1514 static int intel_idle_cpu_init(unsigned int cpu)
1515 {
1516 struct cpuidle_device *dev;
1517
1518 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
1519 dev->cpu = cpu;
1520
1521 if (cpuidle_register_device(dev)) {
1522 pr_debug("cpuidle_register_device %d failed!\n", cpu);
1523 return -EIO;
1524 }
1525
1526 if (!icpu)
1527 return 0;
1528
1529 if (icpu->auto_demotion_disable_flags)
1530 auto_demotion_disable();
1531
1532 if (icpu->disable_promotion_to_c1e)
1533 c1e_promotion_disable();
1534
1535 return 0;
1536 }
1537
1538 static int intel_idle_cpu_online(unsigned int cpu)
1539 {
1540 struct cpuidle_device *dev;
1541
1542 if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE)
1543 tick_broadcast_enable();
1544
1545 /*
1546 * Some systems can hotplug a cpu at runtime after
1547 * the kernel has booted, we have to initialize the
1548 * driver in this case
1549 */
1550 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
1551 if (!dev->registered)
1552 return intel_idle_cpu_init(cpu);
1553
1554 return 0;
1555 }
1556
1557 /**
1558 * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices.
1559 */
1560 static void __init intel_idle_cpuidle_devices_uninit(void)
1561 {
1562 int i;
1563
1564 for_each_online_cpu(i)
1565 cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i));
1566 }
1567
1568 static int __init intel_idle_init(void)
1569 {
1570 const struct x86_cpu_id *id;
1571 unsigned int eax, ebx, ecx;
1572 int retval;
1573
1574 /* Do not load intel_idle at all for now if idle= is passed */
1575 if (boot_option_idle_override != IDLE_NO_OVERRIDE)
1576 return -ENODEV;
1577
1578 if (max_cstate == 0) {
1579 pr_debug("disabled\n");
1580 return -EPERM;
1581 }
1582
1583 id = x86_match_cpu(intel_idle_ids);
1584 if (id) {
1585 if (!boot_cpu_has(X86_FEATURE_MWAIT)) {
1586 pr_debug("Please enable MWAIT in BIOS SETUP\n");
1587 return -ENODEV;
1588 }
1589 } else {
1590 id = x86_match_cpu(intel_mwait_ids);
1591 if (!id)
1592 return -ENODEV;
1593 }
1594
1595 if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
1596 return -ENODEV;
1597
1598 cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates);
1599
1600 if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
1601 !(ecx & CPUID5_ECX_INTERRUPT_BREAK) ||
1602 !mwait_substates)
1603 return -ENODEV;
1604
1605 pr_debug("MWAIT substates: 0x%x\n", mwait_substates);
1606
1607 icpu = (const struct idle_cpu *)id->driver_data;
1608 if (icpu) {
1609 cpuidle_state_table = icpu->state_table;
1610 if (icpu->use_acpi)
1611 intel_idle_acpi_cst_extract();
1612 } else if (!intel_idle_acpi_cst_extract()) {
1613 return -ENODEV;
1614 }
1615
1616 pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n",
1617 boot_cpu_data.x86_model);
1618
1619 intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device);
1620 if (!intel_idle_cpuidle_devices)
1621 return -ENOMEM;
1622
1623 intel_idle_cpuidle_driver_init(&intel_idle_driver);
1624
1625 retval = cpuidle_register_driver(&intel_idle_driver);
1626 if (retval) {
1627 struct cpuidle_driver *drv = cpuidle_get_driver();
1628 printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"),
1629 drv ? drv->name : "none");
1630 goto init_driver_fail;
1631 }
1632
1633 if (boot_cpu_has(X86_FEATURE_ARAT)) /* Always Reliable APIC Timer */
1634 lapic_timer_reliable_states = LAPIC_TIMER_ALWAYS_RELIABLE;
1635
1636 retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online",
1637 intel_idle_cpu_online, NULL);
1638 if (retval < 0)
1639 goto hp_setup_fail;
1640
1641 pr_debug("lapic_timer_reliable_states 0x%x\n",
1642 lapic_timer_reliable_states);
1643
1644 return 0;
1645
1646 hp_setup_fail:
1647 intel_idle_cpuidle_devices_uninit();
1648 cpuidle_unregister_driver(&intel_idle_driver);
1649 init_driver_fail:
1650 free_percpu(intel_idle_cpuidle_devices);
1651 return retval;
1652
1653 }
1654 device_initcall(intel_idle_init);
1655
1656 /*
1657 * We are not really modular, but we used to support that. Meaning we also
1658 * support "intel_idle.max_cstate=..." at boot and also a read-only export of
1659 * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param
1660 * is the easiest way (currently) to continue doing that.
1661 */
1662 module_param(max_cstate, int, 0444);
Linux with added FPC-III support