?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
target: Fix kref->refcount underflow in transport_cmd_finish_abort
[linux/fpc-iii.git] / drivers / idle / intel_idle.c
blob5ded9b22b01523580ccf88e9cf6498103609fc2e
1 /*
2 * intel_idle.c - native hardware idle loop for modern Intel processors
3 *
4 * Copyright (c) 2013, Intel Corporation.
5 * Len Brown <len.brown@intel.com>
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2, as published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * more details.
15 *
16 * You should have received a copy of the GNU General Public License along with
17 * this program; if not, write to the Free Software Foundation, Inc.,
18 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
19 */
20
21 /*
22 * intel_idle is a cpuidle driver that loads on specific Intel processors
23 * in lieu of the legacy ACPI processor_idle driver. The intent is to
24 * make Linux more efficient on these processors, as intel_idle knows
25 * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs.
26 */
27
28 /*
29 * Design Assumptions
30 *
31 * All CPUs have same idle states as boot CPU
32 *
33 * Chipset BM_STS (bus master status) bit is a NOP
34 * for preventing entry into deep C-stats
35 */
36
37 /*
38 * Known limitations
39 *
40 * The driver currently initializes for_each_online_cpu() upon modprobe.
41 * It it unaware of subsequent processors hot-added to the system.
42 * This means that if you boot with maxcpus=n and later online
43 * processors above n, those processors will use C1 only.
44 *
45 * ACPI has a .suspend hack to turn off deep c-statees during suspend
46 * to avoid complications with the lapic timer workaround.
47 * Have not seen issues with suspend, but may need same workaround here.
48 *
49 */
50
51 /* un-comment DEBUG to enable pr_debug() statements */
52 #define DEBUG
53
54 #include <linux/kernel.h>
55 #include <linux/cpuidle.h>
56 #include <linux/tick.h>
57 #include <trace/events/power.h>
58 #include <linux/sched.h>
59 #include <linux/notifier.h>
60 #include <linux/cpu.h>
61 #include <linux/moduleparam.h>
62 #include <asm/cpu_device_id.h>
63 #include <asm/intel-family.h>
64 #include <asm/mwait.h>
65 #include <asm/msr.h>
66
67 #define INTEL_IDLE_VERSION "0.4.1"
68 #define PREFIX "intel_idle: "
69
70 static struct cpuidle_driver intel_idle_driver = {
71 .name = "intel_idle",
72 .owner = THIS_MODULE,
73 };
74 /* intel_idle.max_cstate=0 disables driver */
75 static int max_cstate = CPUIDLE_STATE_MAX - 1;
76
77 static unsigned int mwait_substates;
78
79 #define LAPIC_TIMER_ALWAYS_RELIABLE 0xFFFFFFFF
80 /* Reliable LAPIC Timer States, bit 1 for C1 etc. */
81 static unsigned int lapic_timer_reliable_states = (1 << 1); /* Default to only C1 */
82
83 struct idle_cpu {
84 struct cpuidle_state *state_table;
85
86 /*
87 * Hardware C-state auto-demotion may not always be optimal.
88 * Indicate which enable bits to clear here.
89 */
90 unsigned long auto_demotion_disable_flags;
91 bool byt_auto_demotion_disable_flag;
92 bool disable_promotion_to_c1e;
93 };
94
95 static const struct idle_cpu *icpu;
96 static struct cpuidle_device __percpu *intel_idle_cpuidle_devices;
97 static int intel_idle(struct cpuidle_device *dev,
98 struct cpuidle_driver *drv, int index);
99 static void intel_idle_freeze(struct cpuidle_device *dev,
100 struct cpuidle_driver *drv, int index);
101 static int intel_idle_cpu_init(int cpu);
102
103 static struct cpuidle_state *cpuidle_state_table;
104
105 /*
106 * Set this flag for states where the HW flushes the TLB for us
107 * and so we don't need cross-calls to keep it consistent.
108 * If this flag is set, SW flushes the TLB, so even if the
109 * HW doesn't do the flushing, this flag is safe to use.
110 */
111 #define CPUIDLE_FLAG_TLB_FLUSHED 0x10000
112
113 /*
114 * MWAIT takes an 8-bit "hint" in EAX "suggesting"
115 * the C-state (top nibble) and sub-state (bottom nibble)
116 * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc.
117 *
118 * We store the hint at the top of our "flags" for each state.
119 */
120 #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF)
121 #define MWAIT2flg(eax) ((eax & 0xFF) << 24)
122
123 /*
124 * States are indexed by the cstate number,
125 * which is also the index into the MWAIT hint array.
126 * Thus C0 is a dummy.
127 */
128 static struct cpuidle_state nehalem_cstates[] = {
129 {
130 .name = "C1-NHM",
131 .desc = "MWAIT 0x00",
132 .flags = MWAIT2flg(0x00),
133 .exit_latency = 3,
134 .target_residency = 6,
135 .enter = &intel_idle,
136 .enter_freeze = intel_idle_freeze, },
137 {
138 .name = "C1E-NHM",
139 .desc = "MWAIT 0x01",
140 .flags = MWAIT2flg(0x01),
141 .exit_latency = 10,
142 .target_residency = 20,
143 .enter = &intel_idle,
144 .enter_freeze = intel_idle_freeze, },
145 {
146 .name = "C3-NHM",
147 .desc = "MWAIT 0x10",
148 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
149 .exit_latency = 20,
150 .target_residency = 80,
151 .enter = &intel_idle,
152 .enter_freeze = intel_idle_freeze, },
153 {
154 .name = "C6-NHM",
155 .desc = "MWAIT 0x20",
156 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
157 .exit_latency = 200,
158 .target_residency = 800,
159 .enter = &intel_idle,
160 .enter_freeze = intel_idle_freeze, },
161 {
162 .enter = NULL }
163 };
164
165 static struct cpuidle_state snb_cstates[] = {
166 {
167 .name = "C1-SNB",
168 .desc = "MWAIT 0x00",
169 .flags = MWAIT2flg(0x00),
170 .exit_latency = 2,
171 .target_residency = 2,
172 .enter = &intel_idle,
173 .enter_freeze = intel_idle_freeze, },
174 {
175 .name = "C1E-SNB",
176 .desc = "MWAIT 0x01",
177 .flags = MWAIT2flg(0x01),
178 .exit_latency = 10,
179 .target_residency = 20,
180 .enter = &intel_idle,
181 .enter_freeze = intel_idle_freeze, },
182 {
183 .name = "C3-SNB",
184 .desc = "MWAIT 0x10",
185 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
186 .exit_latency = 80,
187 .target_residency = 211,
188 .enter = &intel_idle,
189 .enter_freeze = intel_idle_freeze, },
190 {
191 .name = "C6-SNB",
192 .desc = "MWAIT 0x20",
193 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
194 .exit_latency = 104,
195 .target_residency = 345,
196 .enter = &intel_idle,
197 .enter_freeze = intel_idle_freeze, },
198 {
199 .name = "C7-SNB",
200 .desc = "MWAIT 0x30",
201 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
202 .exit_latency = 109,
203 .target_residency = 345,
204 .enter = &intel_idle,
205 .enter_freeze = intel_idle_freeze, },
206 {
207 .enter = NULL }
208 };
209
210 static struct cpuidle_state byt_cstates[] = {
211 {
212 .name = "C1-BYT",
213 .desc = "MWAIT 0x00",
214 .flags = MWAIT2flg(0x00),
215 .exit_latency = 1,
216 .target_residency = 1,
217 .enter = &intel_idle,
218 .enter_freeze = intel_idle_freeze, },
219 {
220 .name = "C6N-BYT",
221 .desc = "MWAIT 0x58",
222 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
223 .exit_latency = 300,
224 .target_residency = 275,
225 .enter = &intel_idle,
226 .enter_freeze = intel_idle_freeze, },
227 {
228 .name = "C6S-BYT",
229 .desc = "MWAIT 0x52",
230 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
231 .exit_latency = 500,
232 .target_residency = 560,
233 .enter = &intel_idle,
234 .enter_freeze = intel_idle_freeze, },
235 {
236 .name = "C7-BYT",
237 .desc = "MWAIT 0x60",
238 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
239 .exit_latency = 1200,
240 .target_residency = 4000,
241 .enter = &intel_idle,
242 .enter_freeze = intel_idle_freeze, },
243 {
244 .name = "C7S-BYT",
245 .desc = "MWAIT 0x64",
246 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
247 .exit_latency = 10000,
248 .target_residency = 20000,
249 .enter = &intel_idle,
250 .enter_freeze = intel_idle_freeze, },
251 {
252 .enter = NULL }
253 };
254
255 static struct cpuidle_state cht_cstates[] = {
256 {
257 .name = "C1-CHT",
258 .desc = "MWAIT 0x00",
259 .flags = MWAIT2flg(0x00),
260 .exit_latency = 1,
261 .target_residency = 1,
262 .enter = &intel_idle,
263 .enter_freeze = intel_idle_freeze, },
264 {
265 .name = "C6N-CHT",
266 .desc = "MWAIT 0x58",
267 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
268 .exit_latency = 80,
269 .target_residency = 275,
270 .enter = &intel_idle,
271 .enter_freeze = intel_idle_freeze, },
272 {
273 .name = "C6S-CHT",
274 .desc = "MWAIT 0x52",
275 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
276 .exit_latency = 200,
277 .target_residency = 560,
278 .enter = &intel_idle,
279 .enter_freeze = intel_idle_freeze, },
280 {
281 .name = "C7-CHT",
282 .desc = "MWAIT 0x60",
283 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
284 .exit_latency = 1200,
285 .target_residency = 4000,
286 .enter = &intel_idle,
287 .enter_freeze = intel_idle_freeze, },
288 {
289 .name = "C7S-CHT",
290 .desc = "MWAIT 0x64",
291 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
292 .exit_latency = 10000,
293 .target_residency = 20000,
294 .enter = &intel_idle,
295 .enter_freeze = intel_idle_freeze, },
296 {
297 .enter = NULL }
298 };
299
300 static struct cpuidle_state ivb_cstates[] = {
301 {
302 .name = "C1-IVB",
303 .desc = "MWAIT 0x00",
304 .flags = MWAIT2flg(0x00),
305 .exit_latency = 1,
306 .target_residency = 1,
307 .enter = &intel_idle,
308 .enter_freeze = intel_idle_freeze, },
309 {
310 .name = "C1E-IVB",
311 .desc = "MWAIT 0x01",
312 .flags = MWAIT2flg(0x01),
313 .exit_latency = 10,
314 .target_residency = 20,
315 .enter = &intel_idle,
316 .enter_freeze = intel_idle_freeze, },
317 {
318 .name = "C3-IVB",
319 .desc = "MWAIT 0x10",
320 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
321 .exit_latency = 59,
322 .target_residency = 156,
323 .enter = &intel_idle,
324 .enter_freeze = intel_idle_freeze, },
325 {
326 .name = "C6-IVB",
327 .desc = "MWAIT 0x20",
328 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
329 .exit_latency = 80,
330 .target_residency = 300,
331 .enter = &intel_idle,
332 .enter_freeze = intel_idle_freeze, },
333 {
334 .name = "C7-IVB",
335 .desc = "MWAIT 0x30",
336 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
337 .exit_latency = 87,
338 .target_residency = 300,
339 .enter = &intel_idle,
340 .enter_freeze = intel_idle_freeze, },
341 {
342 .enter = NULL }
343 };
344
345 static struct cpuidle_state ivt_cstates[] = {
346 {
347 .name = "C1-IVT",
348 .desc = "MWAIT 0x00",
349 .flags = MWAIT2flg(0x00),
350 .exit_latency = 1,
351 .target_residency = 1,
352 .enter = &intel_idle,
353 .enter_freeze = intel_idle_freeze, },
354 {
355 .name = "C1E-IVT",
356 .desc = "MWAIT 0x01",
357 .flags = MWAIT2flg(0x01),
358 .exit_latency = 10,
359 .target_residency = 80,
360 .enter = &intel_idle,
361 .enter_freeze = intel_idle_freeze, },
362 {
363 .name = "C3-IVT",
364 .desc = "MWAIT 0x10",
365 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
366 .exit_latency = 59,
367 .target_residency = 156,
368 .enter = &intel_idle,
369 .enter_freeze = intel_idle_freeze, },
370 {
371 .name = "C6-IVT",
372 .desc = "MWAIT 0x20",
373 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
374 .exit_latency = 82,
375 .target_residency = 300,
376 .enter = &intel_idle,
377 .enter_freeze = intel_idle_freeze, },
378 {
379 .enter = NULL }
380 };
381
382 static struct cpuidle_state ivt_cstates_4s[] = {
383 {
384 .name = "C1-IVT-4S",
385 .desc = "MWAIT 0x00",
386 .flags = MWAIT2flg(0x00),
387 .exit_latency = 1,
388 .target_residency = 1,
389 .enter = &intel_idle,
390 .enter_freeze = intel_idle_freeze, },
391 {
392 .name = "C1E-IVT-4S",
393 .desc = "MWAIT 0x01",
394 .flags = MWAIT2flg(0x01),
395 .exit_latency = 10,
396 .target_residency = 250,
397 .enter = &intel_idle,
398 .enter_freeze = intel_idle_freeze, },
399 {
400 .name = "C3-IVT-4S",
401 .desc = "MWAIT 0x10",
402 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
403 .exit_latency = 59,
404 .target_residency = 300,
405 .enter = &intel_idle,
406 .enter_freeze = intel_idle_freeze, },
407 {
408 .name = "C6-IVT-4S",
409 .desc = "MWAIT 0x20",
410 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
411 .exit_latency = 84,
412 .target_residency = 400,
413 .enter = &intel_idle,
414 .enter_freeze = intel_idle_freeze, },
415 {
416 .enter = NULL }
417 };
418
419 static struct cpuidle_state ivt_cstates_8s[] = {
420 {
421 .name = "C1-IVT-8S",
422 .desc = "MWAIT 0x00",
423 .flags = MWAIT2flg(0x00),
424 .exit_latency = 1,
425 .target_residency = 1,
426 .enter = &intel_idle,
427 .enter_freeze = intel_idle_freeze, },
428 {
429 .name = "C1E-IVT-8S",
430 .desc = "MWAIT 0x01",
431 .flags = MWAIT2flg(0x01),
432 .exit_latency = 10,
433 .target_residency = 500,
434 .enter = &intel_idle,
435 .enter_freeze = intel_idle_freeze, },
436 {
437 .name = "C3-IVT-8S",
438 .desc = "MWAIT 0x10",
439 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
440 .exit_latency = 59,
441 .target_residency = 600,
442 .enter = &intel_idle,
443 .enter_freeze = intel_idle_freeze, },
444 {
445 .name = "C6-IVT-8S",
446 .desc = "MWAIT 0x20",
447 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
448 .exit_latency = 88,
449 .target_residency = 700,
450 .enter = &intel_idle,
451 .enter_freeze = intel_idle_freeze, },
452 {
453 .enter = NULL }
454 };
455
456 static struct cpuidle_state hsw_cstates[] = {
457 {
458 .name = "C1-HSW",
459 .desc = "MWAIT 0x00",
460 .flags = MWAIT2flg(0x00),
461 .exit_latency = 2,
462 .target_residency = 2,
463 .enter = &intel_idle,
464 .enter_freeze = intel_idle_freeze, },
465 {
466 .name = "C1E-HSW",
467 .desc = "MWAIT 0x01",
468 .flags = MWAIT2flg(0x01),
469 .exit_latency = 10,
470 .target_residency = 20,
471 .enter = &intel_idle,
472 .enter_freeze = intel_idle_freeze, },
473 {
474 .name = "C3-HSW",
475 .desc = "MWAIT 0x10",
476 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
477 .exit_latency = 33,
478 .target_residency = 100,
479 .enter = &intel_idle,
480 .enter_freeze = intel_idle_freeze, },
481 {
482 .name = "C6-HSW",
483 .desc = "MWAIT 0x20",
484 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
485 .exit_latency = 133,
486 .target_residency = 400,
487 .enter = &intel_idle,
488 .enter_freeze = intel_idle_freeze, },
489 {
490 .name = "C7s-HSW",
491 .desc = "MWAIT 0x32",
492 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
493 .exit_latency = 166,
494 .target_residency = 500,
495 .enter = &intel_idle,
496 .enter_freeze = intel_idle_freeze, },
497 {
498 .name = "C8-HSW",
499 .desc = "MWAIT 0x40",
500 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
501 .exit_latency = 300,
502 .target_residency = 900,
503 .enter = &intel_idle,
504 .enter_freeze = intel_idle_freeze, },
505 {
506 .name = "C9-HSW",
507 .desc = "MWAIT 0x50",
508 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
509 .exit_latency = 600,
510 .target_residency = 1800,
511 .enter = &intel_idle,
512 .enter_freeze = intel_idle_freeze, },
513 {
514 .name = "C10-HSW",
515 .desc = "MWAIT 0x60",
516 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
517 .exit_latency = 2600,
518 .target_residency = 7700,
519 .enter = &intel_idle,
520 .enter_freeze = intel_idle_freeze, },
521 {
522 .enter = NULL }
523 };
524 static struct cpuidle_state bdw_cstates[] = {
525 {
526 .name = "C1-BDW",
527 .desc = "MWAIT 0x00",
528 .flags = MWAIT2flg(0x00),
529 .exit_latency = 2,
530 .target_residency = 2,
531 .enter = &intel_idle,
532 .enter_freeze = intel_idle_freeze, },
533 {
534 .name = "C1E-BDW",
535 .desc = "MWAIT 0x01",
536 .flags = MWAIT2flg(0x01),
537 .exit_latency = 10,
538 .target_residency = 20,
539 .enter = &intel_idle,
540 .enter_freeze = intel_idle_freeze, },
541 {
542 .name = "C3-BDW",
543 .desc = "MWAIT 0x10",
544 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
545 .exit_latency = 40,
546 .target_residency = 100,
547 .enter = &intel_idle,
548 .enter_freeze = intel_idle_freeze, },
549 {
550 .name = "C6-BDW",
551 .desc = "MWAIT 0x20",
552 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
553 .exit_latency = 133,
554 .target_residency = 400,
555 .enter = &intel_idle,
556 .enter_freeze = intel_idle_freeze, },
557 {
558 .name = "C7s-BDW",
559 .desc = "MWAIT 0x32",
560 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
561 .exit_latency = 166,
562 .target_residency = 500,
563 .enter = &intel_idle,
564 .enter_freeze = intel_idle_freeze, },
565 {
566 .name = "C8-BDW",
567 .desc = "MWAIT 0x40",
568 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
569 .exit_latency = 300,
570 .target_residency = 900,
571 .enter = &intel_idle,
572 .enter_freeze = intel_idle_freeze, },
573 {
574 .name = "C9-BDW",
575 .desc = "MWAIT 0x50",
576 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
577 .exit_latency = 600,
578 .target_residency = 1800,
579 .enter = &intel_idle,
580 .enter_freeze = intel_idle_freeze, },
581 {
582 .name = "C10-BDW",
583 .desc = "MWAIT 0x60",
584 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
585 .exit_latency = 2600,
586 .target_residency = 7700,
587 .enter = &intel_idle,
588 .enter_freeze = intel_idle_freeze, },
589 {
590 .enter = NULL }
591 };
592
593 static struct cpuidle_state skl_cstates[] = {
594 {
595 .name = "C1-SKL",
596 .desc = "MWAIT 0x00",
597 .flags = MWAIT2flg(0x00),
598 .exit_latency = 2,
599 .target_residency = 2,
600 .enter = &intel_idle,
601 .enter_freeze = intel_idle_freeze, },
602 {
603 .name = "C1E-SKL",
604 .desc = "MWAIT 0x01",
605 .flags = MWAIT2flg(0x01),
606 .exit_latency = 10,
607 .target_residency = 20,
608 .enter = &intel_idle,
609 .enter_freeze = intel_idle_freeze, },
610 {
611 .name = "C3-SKL",
612 .desc = "MWAIT 0x10",
613 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
614 .exit_latency = 70,
615 .target_residency = 100,
616 .enter = &intel_idle,
617 .enter_freeze = intel_idle_freeze, },
618 {
619 .name = "C6-SKL",
620 .desc = "MWAIT 0x20",
621 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
622 .exit_latency = 85,
623 .target_residency = 200,
624 .enter = &intel_idle,
625 .enter_freeze = intel_idle_freeze, },
626 {
627 .name = "C7s-SKL",
628 .desc = "MWAIT 0x33",
629 .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED,
630 .exit_latency = 124,
631 .target_residency = 800,
632 .enter = &intel_idle,
633 .enter_freeze = intel_idle_freeze, },
634 {
635 .name = "C8-SKL",
636 .desc = "MWAIT 0x40",
637 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
638 .exit_latency = 200,
639 .target_residency = 800,
640 .enter = &intel_idle,
641 .enter_freeze = intel_idle_freeze, },
642 {
643 .name = "C9-SKL",
644 .desc = "MWAIT 0x50",
645 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
646 .exit_latency = 480,
647 .target_residency = 5000,
648 .enter = &intel_idle,
649 .enter_freeze = intel_idle_freeze, },
650 {
651 .name = "C10-SKL",
652 .desc = "MWAIT 0x60",
653 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
654 .exit_latency = 890,
655 .target_residency = 5000,
656 .enter = &intel_idle,
657 .enter_freeze = intel_idle_freeze, },
658 {
659 .enter = NULL }
660 };
661
662 static struct cpuidle_state skx_cstates[] = {
663 {
664 .name = "C1-SKX",
665 .desc = "MWAIT 0x00",
666 .flags = MWAIT2flg(0x00),
667 .exit_latency = 2,
668 .target_residency = 2,
669 .enter = &intel_idle,
670 .enter_freeze = intel_idle_freeze, },
671 {
672 .name = "C1E-SKX",
673 .desc = "MWAIT 0x01",
674 .flags = MWAIT2flg(0x01),
675 .exit_latency = 10,
676 .target_residency = 20,
677 .enter = &intel_idle,
678 .enter_freeze = intel_idle_freeze, },
679 {
680 .name = "C6-SKX",
681 .desc = "MWAIT 0x20",
682 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
683 .exit_latency = 133,
684 .target_residency = 600,
685 .enter = &intel_idle,
686 .enter_freeze = intel_idle_freeze, },
687 {
688 .enter = NULL }
689 };
690
691 static struct cpuidle_state atom_cstates[] = {
692 {
693 .name = "C1E-ATM",
694 .desc = "MWAIT 0x00",
695 .flags = MWAIT2flg(0x00),
696 .exit_latency = 10,
697 .target_residency = 20,
698 .enter = &intel_idle,
699 .enter_freeze = intel_idle_freeze, },
700 {
701 .name = "C2-ATM",
702 .desc = "MWAIT 0x10",
703 .flags = MWAIT2flg(0x10),
704 .exit_latency = 20,
705 .target_residency = 80,
706 .enter = &intel_idle,
707 .enter_freeze = intel_idle_freeze, },
708 {
709 .name = "C4-ATM",
710 .desc = "MWAIT 0x30",
711 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
712 .exit_latency = 100,
713 .target_residency = 400,
714 .enter = &intel_idle,
715 .enter_freeze = intel_idle_freeze, },
716 {
717 .name = "C6-ATM",
718 .desc = "MWAIT 0x52",
719 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
720 .exit_latency = 140,
721 .target_residency = 560,
722 .enter = &intel_idle,
723 .enter_freeze = intel_idle_freeze, },
724 {
725 .enter = NULL }
726 };
727 static struct cpuidle_state tangier_cstates[] = {
728 {
729 .name = "C1-TNG",
730 .desc = "MWAIT 0x00",
731 .flags = MWAIT2flg(0x00),
732 .exit_latency = 1,
733 .target_residency = 4,
734 .enter = &intel_idle,
735 .enter_freeze = intel_idle_freeze, },
736 {
737 .name = "C4-TNG",
738 .desc = "MWAIT 0x30",
739 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
740 .exit_latency = 100,
741 .target_residency = 400,
742 .enter = &intel_idle,
743 .enter_freeze = intel_idle_freeze, },
744 {
745 .name = "C6-TNG",
746 .desc = "MWAIT 0x52",
747 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
748 .exit_latency = 140,
749 .target_residency = 560,
750 .enter = &intel_idle,
751 .enter_freeze = intel_idle_freeze, },
752 {
753 .name = "C7-TNG",
754 .desc = "MWAIT 0x60",
755 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
756 .exit_latency = 1200,
757 .target_residency = 4000,
758 .enter = &intel_idle,
759 .enter_freeze = intel_idle_freeze, },
760 {
761 .name = "C9-TNG",
762 .desc = "MWAIT 0x64",
763 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
764 .exit_latency = 10000,
765 .target_residency = 20000,
766 .enter = &intel_idle,
767 .enter_freeze = intel_idle_freeze, },
768 {
769 .enter = NULL }
770 };
771 static struct cpuidle_state avn_cstates[] = {
772 {
773 .name = "C1-AVN",
774 .desc = "MWAIT 0x00",
775 .flags = MWAIT2flg(0x00),
776 .exit_latency = 2,
777 .target_residency = 2,
778 .enter = &intel_idle,
779 .enter_freeze = intel_idle_freeze, },
780 {
781 .name = "C6-AVN",
782 .desc = "MWAIT 0x51",
783 .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED,
784 .exit_latency = 15,
785 .target_residency = 45,
786 .enter = &intel_idle,
787 .enter_freeze = intel_idle_freeze, },
788 {
789 .enter = NULL }
790 };
791 static struct cpuidle_state knl_cstates[] = {
792 {
793 .name = "C1-KNL",
794 .desc = "MWAIT 0x00",
795 .flags = MWAIT2flg(0x00),
796 .exit_latency = 1,
797 .target_residency = 2,
798 .enter = &intel_idle,
799 .enter_freeze = intel_idle_freeze },
800 {
801 .name = "C6-KNL",
802 .desc = "MWAIT 0x10",
803 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
804 .exit_latency = 120,
805 .target_residency = 500,
806 .enter = &intel_idle,
807 .enter_freeze = intel_idle_freeze },
808 {
809 .enter = NULL }
810 };
811
812 static struct cpuidle_state bxt_cstates[] = {
813 {
814 .name = "C1-BXT",
815 .desc = "MWAIT 0x00",
816 .flags = MWAIT2flg(0x00),
817 .exit_latency = 2,
818 .target_residency = 2,
819 .enter = &intel_idle,
820 .enter_freeze = intel_idle_freeze, },
821 {
822 .name = "C1E-BXT",
823 .desc = "MWAIT 0x01",
824 .flags = MWAIT2flg(0x01),
825 .exit_latency = 10,
826 .target_residency = 20,
827 .enter = &intel_idle,
828 .enter_freeze = intel_idle_freeze, },
829 {
830 .name = "C6-BXT",
831 .desc = "MWAIT 0x20",
832 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
833 .exit_latency = 133,
834 .target_residency = 133,
835 .enter = &intel_idle,
836 .enter_freeze = intel_idle_freeze, },
837 {
838 .name = "C7s-BXT",
839 .desc = "MWAIT 0x31",
840 .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED,
841 .exit_latency = 155,
842 .target_residency = 155,
843 .enter = &intel_idle,
844 .enter_freeze = intel_idle_freeze, },
845 {
846 .name = "C8-BXT",
847 .desc = "MWAIT 0x40",
848 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
849 .exit_latency = 1000,
850 .target_residency = 1000,
851 .enter = &intel_idle,
852 .enter_freeze = intel_idle_freeze, },
853 {
854 .name = "C9-BXT",
855 .desc = "MWAIT 0x50",
856 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
857 .exit_latency = 2000,
858 .target_residency = 2000,
859 .enter = &intel_idle,
860 .enter_freeze = intel_idle_freeze, },
861 {
862 .name = "C10-BXT",
863 .desc = "MWAIT 0x60",
864 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
865 .exit_latency = 10000,
866 .target_residency = 10000,
867 .enter = &intel_idle,
868 .enter_freeze = intel_idle_freeze, },
869 {
870 .enter = NULL }
871 };
872
873 static struct cpuidle_state dnv_cstates[] = {
874 {
875 .name = "C1-DNV",
876 .desc = "MWAIT 0x00",
877 .flags = MWAIT2flg(0x00),
878 .exit_latency = 2,
879 .target_residency = 2,
880 .enter = &intel_idle,
881 .enter_freeze = intel_idle_freeze, },
882 {
883 .name = "C1E-DNV",
884 .desc = "MWAIT 0x01",
885 .flags = MWAIT2flg(0x01),
886 .exit_latency = 10,
887 .target_residency = 20,
888 .enter = &intel_idle,
889 .enter_freeze = intel_idle_freeze, },
890 {
891 .name = "C6-DNV",
892 .desc = "MWAIT 0x20",
893 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
894 .exit_latency = 50,
895 .target_residency = 500,
896 .enter = &intel_idle,
897 .enter_freeze = intel_idle_freeze, },
898 {
899 .enter = NULL }
900 };
901
902 /**
903 * intel_idle
904 * @dev: cpuidle_device
905 * @drv: cpuidle driver
906 * @index: index of cpuidle state
907 *
908 * Must be called under local_irq_disable().
909 */
910 static __cpuidle int intel_idle(struct cpuidle_device *dev,
911 struct cpuidle_driver *drv, int index)
912 {
913 unsigned long ecx = 1; /* break on interrupt flag */
914 struct cpuidle_state *state = &drv->states[index];
915 unsigned long eax = flg2MWAIT(state->flags);
916 unsigned int cstate;
917 int cpu = smp_processor_id();
918
919 cstate = (((eax) >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK) + 1;
920
921 /*
922 * leave_mm() to avoid costly and often unnecessary wakeups
923 * for flushing the user TLB's associated with the active mm.
924 */
925 if (state->flags & CPUIDLE_FLAG_TLB_FLUSHED)
926 leave_mm(cpu);
927
928 if (!(lapic_timer_reliable_states & (1 << (cstate))))
929 tick_broadcast_enter();
930
931 mwait_idle_with_hints(eax, ecx);
932
933 if (!(lapic_timer_reliable_states & (1 << (cstate))))
934 tick_broadcast_exit();
935
936 return index;
937 }
938
939 /**
940 * intel_idle_freeze - simplified "enter" callback routine for suspend-to-idle
941 * @dev: cpuidle_device
942 * @drv: cpuidle driver
943 * @index: state index
944 */
945 static void intel_idle_freeze(struct cpuidle_device *dev,
946 struct cpuidle_driver *drv, int index)
947 {
948 unsigned long ecx = 1; /* break on interrupt flag */
949 unsigned long eax = flg2MWAIT(drv->states[index].flags);
950
951 mwait_idle_with_hints(eax, ecx);
952 }
953
954 static void __setup_broadcast_timer(void *arg)
955 {
956 unsigned long on = (unsigned long)arg;
957
958 if (on)
959 tick_broadcast_enable();
960 else
961 tick_broadcast_disable();
962 }
963
964 static int cpu_hotplug_notify(struct notifier_block *n,
965 unsigned long action, void *hcpu)
966 {
967 int hotcpu = (unsigned long)hcpu;
968 struct cpuidle_device *dev;
969
970 switch (action & ~CPU_TASKS_FROZEN) {
971 case CPU_ONLINE:
972
973 if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE)
974 smp_call_function_single(hotcpu, __setup_broadcast_timer,
975 (void *)true, 1);
976
977 /*
978 * Some systems can hotplug a cpu at runtime after
979 * the kernel has booted, we have to initialize the
980 * driver in this case
981 */
982 dev = per_cpu_ptr(intel_idle_cpuidle_devices, hotcpu);
983 if (dev->registered)
984 break;
985
986 if (intel_idle_cpu_init(hotcpu))
987 return NOTIFY_BAD;
988
989 break;
990 }
991 return NOTIFY_OK;
992 }
993
994 static struct notifier_block cpu_hotplug_notifier = {
995 .notifier_call = cpu_hotplug_notify,
996 };
997
998 static void auto_demotion_disable(void *dummy)
999 {
1000 unsigned long long msr_bits;
1001
1002 rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits);
1003 msr_bits &= ~(icpu->auto_demotion_disable_flags);
1004 wrmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits);
1005 }
1006 static void c1e_promotion_disable(void *dummy)
1007 {
1008 unsigned long long msr_bits;
1009
1010 rdmsrl(MSR_IA32_POWER_CTL, msr_bits);
1011 msr_bits &= ~0x2;
1012 wrmsrl(MSR_IA32_POWER_CTL, msr_bits);
1013 }
1014
1015 static const struct idle_cpu idle_cpu_nehalem = {
1016 .state_table = nehalem_cstates,
1017 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
1018 .disable_promotion_to_c1e = true,
1019 };
1020
1021 static const struct idle_cpu idle_cpu_atom = {
1022 .state_table = atom_cstates,
1023 };
1024
1025 static const struct idle_cpu idle_cpu_tangier = {
1026 .state_table = tangier_cstates,
1027 };
1028
1029 static const struct idle_cpu idle_cpu_lincroft = {
1030 .state_table = atom_cstates,
1031 .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE,
1032 };
1033
1034 static const struct idle_cpu idle_cpu_snb = {
1035 .state_table = snb_cstates,
1036 .disable_promotion_to_c1e = true,
1037 };
1038
1039 static const struct idle_cpu idle_cpu_byt = {
1040 .state_table = byt_cstates,
1041 .disable_promotion_to_c1e = true,
1042 .byt_auto_demotion_disable_flag = true,
1043 };
1044
1045 static const struct idle_cpu idle_cpu_cht = {
1046 .state_table = cht_cstates,
1047 .disable_promotion_to_c1e = true,
1048 .byt_auto_demotion_disable_flag = true,
1049 };
1050
1051 static const struct idle_cpu idle_cpu_ivb = {
1052 .state_table = ivb_cstates,
1053 .disable_promotion_to_c1e = true,
1054 };
1055
1056 static const struct idle_cpu idle_cpu_ivt = {
1057 .state_table = ivt_cstates,
1058 .disable_promotion_to_c1e = true,
1059 };
1060
1061 static const struct idle_cpu idle_cpu_hsw = {
1062 .state_table = hsw_cstates,
1063 .disable_promotion_to_c1e = true,
1064 };
1065
1066 static const struct idle_cpu idle_cpu_bdw = {
1067 .state_table = bdw_cstates,
1068 .disable_promotion_to_c1e = true,
1069 };
1070
1071 static const struct idle_cpu idle_cpu_skl = {
1072 .state_table = skl_cstates,
1073 .disable_promotion_to_c1e = true,
1074 };
1075
1076 static const struct idle_cpu idle_cpu_skx = {
1077 .state_table = skx_cstates,
1078 .disable_promotion_to_c1e = true,
1079 };
1080
1081 static const struct idle_cpu idle_cpu_avn = {
1082 .state_table = avn_cstates,
1083 .disable_promotion_to_c1e = true,
1084 };
1085
1086 static const struct idle_cpu idle_cpu_knl = {
1087 .state_table = knl_cstates,
1088 };
1089
1090 static const struct idle_cpu idle_cpu_bxt = {
1091 .state_table = bxt_cstates,
1092 .disable_promotion_to_c1e = true,
1093 };
1094
1095 static const struct idle_cpu idle_cpu_dnv = {
1096 .state_table = dnv_cstates,
1097 .disable_promotion_to_c1e = true,
1098 };
1099
1100 #define ICPU(model, cpu) \
1101 { X86_VENDOR_INTEL, 6, model, X86_FEATURE_MWAIT, (unsigned long)&cpu }
1102
1103 static const struct x86_cpu_id intel_idle_ids[] __initconst = {
1104 ICPU(INTEL_FAM6_NEHALEM_EP, idle_cpu_nehalem),
1105 ICPU(INTEL_FAM6_NEHALEM, idle_cpu_nehalem),
1106 ICPU(INTEL_FAM6_NEHALEM_G, idle_cpu_nehalem),
1107 ICPU(INTEL_FAM6_WESTMERE, idle_cpu_nehalem),
1108 ICPU(INTEL_FAM6_WESTMERE_EP, idle_cpu_nehalem),
1109 ICPU(INTEL_FAM6_NEHALEM_EX, idle_cpu_nehalem),
1110 ICPU(INTEL_FAM6_ATOM_PINEVIEW, idle_cpu_atom),
1111 ICPU(INTEL_FAM6_ATOM_LINCROFT, idle_cpu_lincroft),
1112 ICPU(INTEL_FAM6_WESTMERE_EX, idle_cpu_nehalem),
1113 ICPU(INTEL_FAM6_SANDYBRIDGE, idle_cpu_snb),
1114 ICPU(INTEL_FAM6_SANDYBRIDGE_X, idle_cpu_snb),
1115 ICPU(INTEL_FAM6_ATOM_CEDARVIEW, idle_cpu_atom),
1116 ICPU(INTEL_FAM6_ATOM_SILVERMONT1, idle_cpu_byt),
1117 ICPU(INTEL_FAM6_ATOM_MERRIFIELD, idle_cpu_tangier),
1118 ICPU(INTEL_FAM6_ATOM_AIRMONT, idle_cpu_cht),
1119 ICPU(INTEL_FAM6_IVYBRIDGE, idle_cpu_ivb),
1120 ICPU(INTEL_FAM6_IVYBRIDGE_X, idle_cpu_ivt),
1121 ICPU(INTEL_FAM6_HASWELL_CORE, idle_cpu_hsw),
1122 ICPU(INTEL_FAM6_HASWELL_X, idle_cpu_hsw),
1123 ICPU(INTEL_FAM6_HASWELL_ULT, idle_cpu_hsw),
1124 ICPU(INTEL_FAM6_HASWELL_GT3E, idle_cpu_hsw),
1125 ICPU(INTEL_FAM6_ATOM_SILVERMONT2, idle_cpu_avn),
1126 ICPU(INTEL_FAM6_BROADWELL_CORE, idle_cpu_bdw),
1127 ICPU(INTEL_FAM6_BROADWELL_GT3E, idle_cpu_bdw),
1128 ICPU(INTEL_FAM6_BROADWELL_X, idle_cpu_bdw),
1129 ICPU(INTEL_FAM6_BROADWELL_XEON_D, idle_cpu_bdw),
1130 ICPU(INTEL_FAM6_SKYLAKE_MOBILE, idle_cpu_skl),
1131 ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, idle_cpu_skl),
1132 ICPU(INTEL_FAM6_KABYLAKE_MOBILE, idle_cpu_skl),
1133 ICPU(INTEL_FAM6_KABYLAKE_DESKTOP, idle_cpu_skl),
1134 ICPU(INTEL_FAM6_SKYLAKE_X, idle_cpu_skx),
1135 ICPU(INTEL_FAM6_XEON_PHI_KNL, idle_cpu_knl),
1136 ICPU(INTEL_FAM6_ATOM_GOLDMONT, idle_cpu_bxt),
1137 ICPU(INTEL_FAM6_ATOM_DENVERTON, idle_cpu_dnv),
1138 {}
1139 };
1140
1141 /*
1142 * intel_idle_probe()
1143 */
1144 static int __init intel_idle_probe(void)
1145 {
1146 unsigned int eax, ebx, ecx;
1147 const struct x86_cpu_id *id;
1148
1149 if (max_cstate == 0) {
1150 pr_debug(PREFIX "disabled\n");
1151 return -EPERM;
1152 }
1153
1154 id = x86_match_cpu(intel_idle_ids);
1155 if (!id) {
1156 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
1157 boot_cpu_data.x86 == 6)
1158 pr_debug(PREFIX "does not run on family %d model %d\n",
1159 boot_cpu_data.x86, boot_cpu_data.x86_model);
1160 return -ENODEV;
1161 }
1162
1163 if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
1164 return -ENODEV;
1165
1166 cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates);
1167
1168 if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
1169 !(ecx & CPUID5_ECX_INTERRUPT_BREAK) ||
1170 !mwait_substates)
1171 return -ENODEV;
1172
1173 pr_debug(PREFIX "MWAIT substates: 0x%x\n", mwait_substates);
1174
1175 icpu = (const struct idle_cpu *)id->driver_data;
1176 cpuidle_state_table = icpu->state_table;
1177
1178 pr_debug(PREFIX "v" INTEL_IDLE_VERSION
1179 " model 0x%X\n", boot_cpu_data.x86_model);
1180
1181 return 0;
1182 }
1183
1184 /*
1185 * intel_idle_cpuidle_devices_uninit()
1186 * Unregisters the cpuidle devices.
1187 */
1188 static void intel_idle_cpuidle_devices_uninit(void)
1189 {
1190 int i;
1191 struct cpuidle_device *dev;
1192
1193 for_each_online_cpu(i) {
1194 dev = per_cpu_ptr(intel_idle_cpuidle_devices, i);
1195 cpuidle_unregister_device(dev);
1196 }
1197 }
1198
1199 /*
1200 * ivt_idle_state_table_update(void)
1201 *
1202 * Tune IVT multi-socket targets
1203 * Assumption: num_sockets == (max_package_num + 1)
1204 */
1205 static void ivt_idle_state_table_update(void)
1206 {
1207 /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */
1208 int cpu, package_num, num_sockets = 1;
1209
1210 for_each_online_cpu(cpu) {
1211 package_num = topology_physical_package_id(cpu);
1212 if (package_num + 1 > num_sockets) {
1213 num_sockets = package_num + 1;
1214
1215 if (num_sockets > 4) {
1216 cpuidle_state_table = ivt_cstates_8s;
1217 return;
1218 }
1219 }
1220 }
1221
1222 if (num_sockets > 2)
1223 cpuidle_state_table = ivt_cstates_4s;
1224
1225 /* else, 1 and 2 socket systems use default ivt_cstates */
1226 }
1227
1228 /*
1229 * Translate IRTL (Interrupt Response Time Limit) MSR to usec
1230 */
1231
1232 static unsigned int irtl_ns_units[] = {
1233 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 };
1234
1235 static unsigned long long irtl_2_usec(unsigned long long irtl)
1236 {
1237 unsigned long long ns;
1238
1239 if (!irtl)
1240 return 0;
1241
1242 ns = irtl_ns_units[(irtl >> 10) & 0x7];
1243
1244 return div64_u64((irtl & 0x3FF) * ns, 1000);
1245 }
1246 /*
1247 * bxt_idle_state_table_update(void)
1248 *
1249 * On BXT, we trust the IRTL to show the definitive maximum latency
1250 * We use the same value for target_residency.
1251 */
1252 static void bxt_idle_state_table_update(void)
1253 {
1254 unsigned long long msr;
1255 unsigned int usec;
1256
1257 rdmsrl(MSR_PKGC6_IRTL, msr);
1258 usec = irtl_2_usec(msr);
1259 if (usec) {
1260 bxt_cstates[2].exit_latency = usec;
1261 bxt_cstates[2].target_residency = usec;
1262 }
1263
1264 rdmsrl(MSR_PKGC7_IRTL, msr);
1265 usec = irtl_2_usec(msr);
1266 if (usec) {
1267 bxt_cstates[3].exit_latency = usec;
1268 bxt_cstates[3].target_residency = usec;
1269 }
1270
1271 rdmsrl(MSR_PKGC8_IRTL, msr);
1272 usec = irtl_2_usec(msr);
1273 if (usec) {
1274 bxt_cstates[4].exit_latency = usec;
1275 bxt_cstates[4].target_residency = usec;
1276 }
1277
1278 rdmsrl(MSR_PKGC9_IRTL, msr);
1279 usec = irtl_2_usec(msr);
1280 if (usec) {
1281 bxt_cstates[5].exit_latency = usec;
1282 bxt_cstates[5].target_residency = usec;
1283 }
1284
1285 rdmsrl(MSR_PKGC10_IRTL, msr);
1286 usec = irtl_2_usec(msr);
1287 if (usec) {
1288 bxt_cstates[6].exit_latency = usec;
1289 bxt_cstates[6].target_residency = usec;
1290 }
1291
1292 }
1293 /*
1294 * sklh_idle_state_table_update(void)
1295 *
1296 * On SKL-H (model 0x5e) disable C8 and C9 if:
1297 * C10 is enabled and SGX disabled
1298 */
1299 static void sklh_idle_state_table_update(void)
1300 {
1301 unsigned long long msr;
1302 unsigned int eax, ebx, ecx, edx;
1303
1304
1305 /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */
1306 if (max_cstate <= 7)
1307 return;
1308
1309 /* if PC10 not present in CPUID.MWAIT.EDX */
1310 if ((mwait_substates & (0xF << 28)) == 0)
1311 return;
1312
1313 rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr);
1314
1315 /* PC10 is not enabled in PKG C-state limit */
1316 if ((msr & 0xF) != 8)
1317 return;
1318
1319 ecx = 0;
1320 cpuid(7, &eax, &ebx, &ecx, &edx);
1321
1322 /* if SGX is present */
1323 if (ebx & (1 << 2)) {
1324
1325 rdmsrl(MSR_IA32_FEATURE_CONTROL, msr);
1326
1327 /* if SGX is enabled */
1328 if (msr & (1 << 18))
1329 return;
1330 }
1331
1332 skl_cstates[5].disabled = 1; /* C8-SKL */
1333 skl_cstates[6].disabled = 1; /* C9-SKL */
1334 }
1335 /*
1336 * intel_idle_state_table_update()
1337 *
1338 * Update the default state_table for this CPU-id
1339 */
1340
1341 static void intel_idle_state_table_update(void)
1342 {
1343 switch (boot_cpu_data.x86_model) {
1344
1345 case INTEL_FAM6_IVYBRIDGE_X:
1346 ivt_idle_state_table_update();
1347 break;
1348 case INTEL_FAM6_ATOM_GOLDMONT:
1349 bxt_idle_state_table_update();
1350 break;
1351 case INTEL_FAM6_SKYLAKE_DESKTOP:
1352 sklh_idle_state_table_update();
1353 break;
1354 }
1355 }
1356
1357 /*
1358 * intel_idle_cpuidle_driver_init()
1359 * allocate, initialize cpuidle_states
1360 */
1361 static void __init intel_idle_cpuidle_driver_init(void)
1362 {
1363 int cstate;
1364 struct cpuidle_driver *drv = &intel_idle_driver;
1365
1366 intel_idle_state_table_update();
1367
1368 drv->state_count = 1;
1369
1370 for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) {
1371 int num_substates, mwait_hint, mwait_cstate;
1372
1373 if ((cpuidle_state_table[cstate].enter == NULL) &&
1374 (cpuidle_state_table[cstate].enter_freeze == NULL))
1375 break;
1376
1377 if (cstate + 1 > max_cstate) {
1378 printk(PREFIX "max_cstate %d reached\n",
1379 max_cstate);
1380 break;
1381 }
1382
1383 mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags);
1384 mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint);
1385
1386 /* number of sub-states for this state in CPUID.MWAIT */
1387 num_substates = (mwait_substates >> ((mwait_cstate + 1) * 4))
1388 & MWAIT_SUBSTATE_MASK;
1389
1390 /* if NO sub-states for this state in CPUID, skip it */
1391 if (num_substates == 0)
1392 continue;
1393
1394 /* if state marked as disabled, skip it */
1395 if (cpuidle_state_table[cstate].disabled != 0) {
1396 pr_debug(PREFIX "state %s is disabled",
1397 cpuidle_state_table[cstate].name);
1398 continue;
1399 }
1400
1401
1402 if (((mwait_cstate + 1) > 2) &&
1403 !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
1404 mark_tsc_unstable("TSC halts in idle"
1405 " states deeper than C2");
1406
1407 drv->states[drv->state_count] = /* structure copy */
1408 cpuidle_state_table[cstate];
1409
1410 drv->state_count += 1;
1411 }
1412
1413 if (icpu->byt_auto_demotion_disable_flag) {
1414 wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0);
1415 wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0);
1416 }
1417 }
1418
1419
1420 /*
1421 * intel_idle_cpu_init()
1422 * allocate, initialize, register cpuidle_devices
1423 * @cpu: cpu/core to initialize
1424 */
1425 static int intel_idle_cpu_init(int cpu)
1426 {
1427 struct cpuidle_device *dev;
1428
1429 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
1430
1431 dev->cpu = cpu;
1432
1433 if (cpuidle_register_device(dev)) {
1434 pr_debug(PREFIX "cpuidle_register_device %d failed!\n", cpu);
1435 return -EIO;
1436 }
1437
1438 if (icpu->auto_demotion_disable_flags)
1439 smp_call_function_single(cpu, auto_demotion_disable, NULL, 1);
1440
1441 if (icpu->disable_promotion_to_c1e)
1442 smp_call_function_single(cpu, c1e_promotion_disable, NULL, 1);
1443
1444 return 0;
1445 }
1446
1447 static int __init intel_idle_init(void)
1448 {
1449 int retval, i;
1450
1451 /* Do not load intel_idle at all for now if idle= is passed */
1452 if (boot_option_idle_override != IDLE_NO_OVERRIDE)
1453 return -ENODEV;
1454
1455 retval = intel_idle_probe();
1456 if (retval)
1457 return retval;
1458
1459 intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device);
1460 if (intel_idle_cpuidle_devices == NULL)
1461 return -ENOMEM;
1462
1463 intel_idle_cpuidle_driver_init();
1464 retval = cpuidle_register_driver(&intel_idle_driver);
1465 if (retval) {
1466 struct cpuidle_driver *drv = cpuidle_get_driver();
1467 printk(KERN_DEBUG PREFIX "intel_idle yielding to %s",
1468 drv ? drv->name : "none");
1469 free_percpu(intel_idle_cpuidle_devices);
1470 return retval;
1471 }
1472
1473 cpu_notifier_register_begin();
1474
1475 for_each_online_cpu(i) {
1476 retval = intel_idle_cpu_init(i);
1477 if (retval) {
1478 intel_idle_cpuidle_devices_uninit();
1479 cpu_notifier_register_done();
1480 cpuidle_unregister_driver(&intel_idle_driver);
1481 free_percpu(intel_idle_cpuidle_devices);
1482 return retval;
1483 }
1484 }
1485 __register_cpu_notifier(&cpu_hotplug_notifier);
1486
1487 if (boot_cpu_has(X86_FEATURE_ARAT)) /* Always Reliable APIC Timer */
1488 lapic_timer_reliable_states = LAPIC_TIMER_ALWAYS_RELIABLE;
1489 else
1490 on_each_cpu(__setup_broadcast_timer, (void *)true, 1);
1491
1492 cpu_notifier_register_done();
1493
1494 pr_debug(PREFIX "lapic_timer_reliable_states 0x%x\n",
1495 lapic_timer_reliable_states);
1496
1497 return 0;
1498 }
1499 device_initcall(intel_idle_init);
1500
1501 /*
1502 * We are not really modular, but we used to support that. Meaning we also
1503 * support "intel_idle.max_cstate=..." at boot and also a read-only export of
1504 * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param
1505 * is the easiest way (currently) to continue doing that.
1506 */
1507 module_param(max_cstate, int, 0444);
Linux with added FPC-III support