1 // SPDX-License-Identifier: GPL-2.0-only
3 * intel_pstate.c: Native P state management for Intel processors
5 * (C) Copyright 2012 Intel Corporation
6 * Author: Dirk Brandewie <dirk.j.brandewie@intel.com>
9 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
11 #include <linux/kernel.h>
12 #include <linux/kernel_stat.h>
13 #include <linux/module.h>
14 #include <linux/ktime.h>
15 #include <linux/hrtimer.h>
16 #include <linux/tick.h>
17 #include <linux/slab.h>
18 #include <linux/sched/cpufreq.h>
19 #include <linux/sched/smt.h>
20 #include <linux/list.h>
21 #include <linux/cpu.h>
22 #include <linux/cpufreq.h>
23 #include <linux/sysfs.h>
24 #include <linux/types.h>
26 #include <linux/acpi.h>
27 #include <linux/vmalloc.h>
28 #include <linux/pm_qos.h>
29 #include <linux/bitfield.h>
30 #include <trace/events/power.h>
33 #include <asm/div64.h>
35 #include <asm/cpu_device_id.h>
36 #include <asm/cpufeature.h>
37 #include <asm/intel-family.h>
38 #include "../drivers/thermal/intel/thermal_interrupt.h"
40 #define INTEL_PSTATE_SAMPLING_INTERVAL (10 * NSEC_PER_MSEC)
42 #define INTEL_CPUFREQ_TRANSITION_LATENCY 20000
43 #define INTEL_CPUFREQ_TRANSITION_DELAY_HWP 5000
44 #define INTEL_CPUFREQ_TRANSITION_DELAY 500
47 #include <acpi/processor.h>
48 #include <acpi/cppc_acpi.h>
52 #define int_tofp(X) ((int64_t)(X) << FRAC_BITS)
53 #define fp_toint(X) ((X) >> FRAC_BITS)
55 #define ONE_EIGHTH_FP ((int64_t)1 << (FRAC_BITS - 3))
58 #define EXT_FRAC_BITS (EXT_BITS + FRAC_BITS)
59 #define fp_ext_toint(X) ((X) >> EXT_FRAC_BITS)
60 #define int_ext_tofp(X) ((int64_t)(X) << EXT_FRAC_BITS)
62 static inline int32_t mul_fp(int32_t x
, int32_t y
)
64 return ((int64_t)x
* (int64_t)y
) >> FRAC_BITS
;
67 static inline int32_t div_fp(s64 x
, s64 y
)
69 return div64_s64((int64_t)x
<< FRAC_BITS
, y
);
72 static inline int ceiling_fp(int32_t x
)
77 mask
= (1 << FRAC_BITS
) - 1;
83 static inline u64
mul_ext_fp(u64 x
, u64 y
)
85 return (x
* y
) >> EXT_FRAC_BITS
;
88 static inline u64
div_ext_fp(u64 x
, u64 y
)
90 return div64_u64(x
<< EXT_FRAC_BITS
, y
);
94 * struct sample - Store performance sample
95 * @core_avg_perf: Ratio of APERF/MPERF which is the actual average
96 * performance during last sample period
97 * @busy_scaled: Scaled busy value which is used to calculate next
98 * P state. This can be different than core_avg_perf
99 * to account for cpu idle period
100 * @aperf: Difference of actual performance frequency clock count
101 * read from APERF MSR between last and current sample
102 * @mperf: Difference of maximum performance frequency clock count
103 * read from MPERF MSR between last and current sample
104 * @tsc: Difference of time stamp counter between last and
106 * @time: Current time from scheduler
108 * This structure is used in the cpudata structure to store performance sample
109 * data for choosing next P State.
112 int32_t core_avg_perf
;
121 * struct pstate_data - Store P state data
122 * @current_pstate: Current requested P state
123 * @min_pstate: Min P state possible for this platform
124 * @max_pstate: Max P state possible for this platform
125 * @max_pstate_physical:This is physical Max P state for a processor
126 * This can be higher than the max_pstate which can
127 * be limited by platform thermal design power limits
128 * @perf_ctl_scaling: PERF_CTL P-state to frequency scaling factor
129 * @scaling: Scaling factor between performance and frequency
130 * @turbo_pstate: Max Turbo P state possible for this platform
131 * @min_freq: @min_pstate frequency in cpufreq units
132 * @max_freq: @max_pstate frequency in cpufreq units
133 * @turbo_freq: @turbo_pstate frequency in cpufreq units
135 * Stores the per cpu model P state limits and current P state.
141 int max_pstate_physical
;
142 int perf_ctl_scaling
;
145 unsigned int min_freq
;
146 unsigned int max_freq
;
147 unsigned int turbo_freq
;
151 * struct vid_data - Stores voltage information data
152 * @min: VID data for this platform corresponding to
154 * @max: VID data corresponding to the highest P State.
155 * @turbo: VID data for turbo P state
156 * @ratio: Ratio of (vid max - vid min) /
157 * (max P state - Min P State)
159 * Stores the voltage data for DVFS (Dynamic Voltage and Frequency Scaling)
160 * This data is used in Atom platforms, where in addition to target P state,
161 * the voltage data needs to be specified to select next P State.
171 * struct global_params - Global parameters, mostly tunable via sysfs.
172 * @no_turbo: Whether or not to use turbo P-states.
173 * @turbo_disabled: Whether or not turbo P-states are available at all,
174 * based on the MSR_IA32_MISC_ENABLE value and whether or
175 * not the maximum reported turbo P-state is different from
176 * the maximum reported non-turbo one.
177 * @min_perf_pct: Minimum capacity limit in percent of the maximum turbo
179 * @max_perf_pct: Maximum capacity limit in percent of the maximum turbo
182 struct global_params
{
190 * struct cpudata - Per CPU instance data storage
191 * @cpu: CPU number for this instance data
192 * @policy: CPUFreq policy value
193 * @update_util: CPUFreq utility callback information
194 * @update_util_set: CPUFreq utility callback is set
195 * @iowait_boost: iowait-related boost fraction
196 * @last_update: Time of the last update.
197 * @pstate: Stores P state limits for this CPU
198 * @vid: Stores VID limits for this CPU
199 * @last_sample_time: Last Sample time
200 * @aperf_mperf_shift: APERF vs MPERF counting frequency difference
201 * @prev_aperf: Last APERF value read from APERF MSR
202 * @prev_mperf: Last MPERF value read from MPERF MSR
203 * @prev_tsc: Last timestamp counter (TSC) value
204 * @sample: Storage for storing last Sample data
205 * @min_perf_ratio: Minimum capacity in terms of PERF or HWP ratios
206 * @max_perf_ratio: Maximum capacity in terms of PERF or HWP ratios
207 * @acpi_perf_data: Stores ACPI perf information read from _PSS
208 * @valid_pss_table: Set to true for valid ACPI _PSS entries found
209 * @epp_powersave: Last saved HWP energy performance preference
210 * (EPP) or energy performance bias (EPB),
211 * when policy switched to performance
212 * @epp_policy: Last saved policy used to set EPP/EPB
213 * @epp_default: Power on default HWP energy performance
215 * @epp_cached: Cached HWP energy-performance preference value
216 * @hwp_req_cached: Cached value of the last HWP Request MSR
217 * @hwp_cap_cached: Cached value of the last HWP Capabilities MSR
218 * @last_io_update: Last time when IO wake flag was set
219 * @capacity_perf: Highest perf used for scale invariance
220 * @sched_flags: Store scheduler flags for possible cross CPU update
221 * @hwp_boost_min: Last HWP boosted min performance
222 * @suspended: Whether or not the driver has been suspended.
223 * @hwp_notify_work: workqueue for HWP notifications.
225 * This structure stores per CPU instance data for all CPUs.
231 struct update_util_data update_util
;
232 bool update_util_set
;
234 struct pstate_data pstate
;
238 u64 last_sample_time
;
239 u64 aperf_mperf_shift
;
243 struct sample sample
;
244 int32_t min_perf_ratio
;
245 int32_t max_perf_ratio
;
247 struct acpi_processor_performance acpi_perf_data
;
248 bool valid_pss_table
;
250 unsigned int iowait_boost
;
258 unsigned int capacity_perf
;
259 unsigned int sched_flags
;
262 struct delayed_work hwp_notify_work
;
265 static struct cpudata
**all_cpu_data
;
268 * struct pstate_funcs - Per CPU model specific callbacks
269 * @get_max: Callback to get maximum non turbo effective P state
270 * @get_max_physical: Callback to get maximum non turbo physical P state
271 * @get_min: Callback to get minimum P state
272 * @get_turbo: Callback to get turbo P state
273 * @get_scaling: Callback to get frequency scaling factor
274 * @get_cpu_scaling: Get frequency scaling factor for a given cpu
275 * @get_aperf_mperf_shift: Callback to get the APERF vs MPERF frequency difference
276 * @get_val: Callback to convert P state to actual MSR write value
277 * @get_vid: Callback to get VID data for Atom platforms
279 * Core and Atom CPU models have different way to get P State limits. This
280 * structure is used to store those callbacks.
282 struct pstate_funcs
{
283 int (*get_max
)(int cpu
);
284 int (*get_max_physical
)(int cpu
);
285 int (*get_min
)(int cpu
);
286 int (*get_turbo
)(int cpu
);
287 int (*get_scaling
)(void);
288 int (*get_cpu_scaling
)(int cpu
);
289 int (*get_aperf_mperf_shift
)(void);
290 u64 (*get_val
)(struct cpudata
*, int pstate
);
291 void (*get_vid
)(struct cpudata
*);
294 static struct pstate_funcs pstate_funcs __read_mostly
;
296 static bool hwp_active __ro_after_init
;
297 static int hwp_mode_bdw __ro_after_init
;
298 static bool per_cpu_limits __ro_after_init
;
299 static bool hwp_forced __ro_after_init
;
300 static bool hwp_boost __read_mostly
;
301 static bool hwp_is_hybrid
;
303 static struct cpufreq_driver
*intel_pstate_driver __read_mostly
;
305 #define HYBRID_SCALING_FACTOR 78741
306 #define HYBRID_SCALING_FACTOR_MTL 80000
307 #define HYBRID_SCALING_FACTOR_LNL 86957
309 static int hybrid_scaling_factor
= HYBRID_SCALING_FACTOR
;
311 static inline int core_get_scaling(void)
317 static bool acpi_ppc
;
320 static struct global_params global
;
322 static DEFINE_MUTEX(intel_pstate_driver_lock
);
323 static DEFINE_MUTEX(intel_pstate_limits_lock
);
327 static bool intel_pstate_acpi_pm_profile_server(void)
329 if (acpi_gbl_FADT
.preferred_profile
== PM_ENTERPRISE_SERVER
||
330 acpi_gbl_FADT
.preferred_profile
== PM_PERFORMANCE_SERVER
)
336 static bool intel_pstate_get_ppc_enable_status(void)
338 if (intel_pstate_acpi_pm_profile_server())
344 #ifdef CONFIG_ACPI_CPPC_LIB
346 /* The work item is needed to avoid CPU hotplug locking issues */
347 static void intel_pstste_sched_itmt_work_fn(struct work_struct
*work
)
349 sched_set_itmt_support();
352 static DECLARE_WORK(sched_itmt_work
, intel_pstste_sched_itmt_work_fn
);
354 #define CPPC_MAX_PERF U8_MAX
356 static void intel_pstate_set_itmt_prio(int cpu
)
358 struct cppc_perf_caps cppc_perf
;
359 static u32 max_highest_perf
= 0, min_highest_perf
= U32_MAX
;
362 ret
= cppc_get_perf_caps(cpu
, &cppc_perf
);
364 * If CPPC is not available, fall back to MSR_HWP_CAPABILITIES bits [8:0].
366 * Also, on some systems with overclocking enabled, CPPC.highest_perf is
367 * hardcoded to 0xff, so CPPC.highest_perf cannot be used to enable ITMT.
368 * Fall back to MSR_HWP_CAPABILITIES then too.
370 if (ret
|| cppc_perf
.highest_perf
== CPPC_MAX_PERF
)
371 cppc_perf
.highest_perf
= HWP_HIGHEST_PERF(READ_ONCE(all_cpu_data
[cpu
]->hwp_cap_cached
));
374 * The priorities can be set regardless of whether or not
375 * sched_set_itmt_support(true) has been called and it is valid to
376 * update them at any time after it has been called.
378 sched_set_itmt_core_prio(cppc_perf
.highest_perf
, cpu
);
380 if (max_highest_perf
<= min_highest_perf
) {
381 if (cppc_perf
.highest_perf
> max_highest_perf
)
382 max_highest_perf
= cppc_perf
.highest_perf
;
384 if (cppc_perf
.highest_perf
< min_highest_perf
)
385 min_highest_perf
= cppc_perf
.highest_perf
;
387 if (max_highest_perf
> min_highest_perf
) {
389 * This code can be run during CPU online under the
390 * CPU hotplug locks, so sched_set_itmt_support()
391 * cannot be called from here. Queue up a work item
394 schedule_work(&sched_itmt_work
);
399 static int intel_pstate_get_cppc_guaranteed(int cpu
)
401 struct cppc_perf_caps cppc_perf
;
404 ret
= cppc_get_perf_caps(cpu
, &cppc_perf
);
408 if (cppc_perf
.guaranteed_perf
)
409 return cppc_perf
.guaranteed_perf
;
411 return cppc_perf
.nominal_perf
;
414 static int intel_pstate_cppc_get_scaling(int cpu
)
416 struct cppc_perf_caps cppc_perf
;
419 ret
= cppc_get_perf_caps(cpu
, &cppc_perf
);
422 * If the nominal frequency and the nominal performance are not
423 * zero and the ratio between them is not 100, return the hybrid
426 if (!ret
&& cppc_perf
.nominal_perf
&& cppc_perf
.nominal_freq
&&
427 cppc_perf
.nominal_perf
* 100 != cppc_perf
.nominal_freq
)
428 return hybrid_scaling_factor
;
430 return core_get_scaling();
433 #else /* CONFIG_ACPI_CPPC_LIB */
434 static inline void intel_pstate_set_itmt_prio(int cpu
)
437 #endif /* CONFIG_ACPI_CPPC_LIB */
439 static void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy
*policy
)
446 intel_pstate_set_itmt_prio(policy
->cpu
);
450 if (!intel_pstate_get_ppc_enable_status())
453 cpu
= all_cpu_data
[policy
->cpu
];
455 ret
= acpi_processor_register_performance(&cpu
->acpi_perf_data
,
461 * Check if the control value in _PSS is for PERF_CTL MSR, which should
462 * guarantee that the states returned by it map to the states in our
465 if (cpu
->acpi_perf_data
.control_register
.space_id
!=
466 ACPI_ADR_SPACE_FIXED_HARDWARE
)
470 * If there is only one entry _PSS, simply ignore _PSS and continue as
471 * usual without taking _PSS into account
473 if (cpu
->acpi_perf_data
.state_count
< 2)
476 pr_debug("CPU%u - ACPI _PSS perf data\n", policy
->cpu
);
477 for (i
= 0; i
< cpu
->acpi_perf_data
.state_count
; i
++) {
478 pr_debug(" %cP%d: %u MHz, %u mW, 0x%x\n",
479 (i
== cpu
->acpi_perf_data
.state
? '*' : ' '), i
,
480 (u32
) cpu
->acpi_perf_data
.states
[i
].core_frequency
,
481 (u32
) cpu
->acpi_perf_data
.states
[i
].power
,
482 (u32
) cpu
->acpi_perf_data
.states
[i
].control
);
485 cpu
->valid_pss_table
= true;
486 pr_debug("_PPC limits will be enforced\n");
491 cpu
->valid_pss_table
= false;
492 acpi_processor_unregister_performance(policy
->cpu
);
495 static void intel_pstate_exit_perf_limits(struct cpufreq_policy
*policy
)
499 cpu
= all_cpu_data
[policy
->cpu
];
500 if (!cpu
->valid_pss_table
)
503 acpi_processor_unregister_performance(policy
->cpu
);
505 #else /* CONFIG_ACPI */
506 static inline void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy
*policy
)
510 static inline void intel_pstate_exit_perf_limits(struct cpufreq_policy
*policy
)
514 static inline bool intel_pstate_acpi_pm_profile_server(void)
518 #endif /* CONFIG_ACPI */
520 #ifndef CONFIG_ACPI_CPPC_LIB
521 static inline int intel_pstate_get_cppc_guaranteed(int cpu
)
526 static int intel_pstate_cppc_get_scaling(int cpu
)
528 return core_get_scaling();
530 #endif /* CONFIG_ACPI_CPPC_LIB */
532 static int intel_pstate_freq_to_hwp_rel(struct cpudata
*cpu
, int freq
,
533 unsigned int relation
)
535 if (freq
== cpu
->pstate
.turbo_freq
)
536 return cpu
->pstate
.turbo_pstate
;
538 if (freq
== cpu
->pstate
.max_freq
)
539 return cpu
->pstate
.max_pstate
;
542 case CPUFREQ_RELATION_H
:
543 return freq
/ cpu
->pstate
.scaling
;
544 case CPUFREQ_RELATION_C
:
545 return DIV_ROUND_CLOSEST(freq
, cpu
->pstate
.scaling
);
548 return DIV_ROUND_UP(freq
, cpu
->pstate
.scaling
);
551 static int intel_pstate_freq_to_hwp(struct cpudata
*cpu
, int freq
)
553 return intel_pstate_freq_to_hwp_rel(cpu
, freq
, CPUFREQ_RELATION_L
);
557 * intel_pstate_hybrid_hwp_adjust - Calibrate HWP performance levels.
560 * On hybrid processors, HWP may expose more performance levels than there are
561 * P-states accessible through the PERF_CTL interface. If that happens, the
562 * scaling factor between HWP performance levels and CPU frequency will be less
563 * than the scaling factor between P-state values and CPU frequency.
565 * In that case, adjust the CPU parameters used in computations accordingly.
567 static void intel_pstate_hybrid_hwp_adjust(struct cpudata
*cpu
)
569 int perf_ctl_max_phys
= cpu
->pstate
.max_pstate_physical
;
570 int perf_ctl_scaling
= cpu
->pstate
.perf_ctl_scaling
;
571 int perf_ctl_turbo
= pstate_funcs
.get_turbo(cpu
->cpu
);
572 int scaling
= cpu
->pstate
.scaling
;
575 pr_debug("CPU%d: perf_ctl_max_phys = %d\n", cpu
->cpu
, perf_ctl_max_phys
);
576 pr_debug("CPU%d: perf_ctl_turbo = %d\n", cpu
->cpu
, perf_ctl_turbo
);
577 pr_debug("CPU%d: perf_ctl_scaling = %d\n", cpu
->cpu
, perf_ctl_scaling
);
578 pr_debug("CPU%d: HWP_CAP guaranteed = %d\n", cpu
->cpu
, cpu
->pstate
.max_pstate
);
579 pr_debug("CPU%d: HWP_CAP highest = %d\n", cpu
->cpu
, cpu
->pstate
.turbo_pstate
);
580 pr_debug("CPU%d: HWP-to-frequency scaling factor: %d\n", cpu
->cpu
, scaling
);
582 cpu
->pstate
.turbo_freq
= rounddown(cpu
->pstate
.turbo_pstate
* scaling
,
584 cpu
->pstate
.max_freq
= rounddown(cpu
->pstate
.max_pstate
* scaling
,
587 freq
= perf_ctl_max_phys
* perf_ctl_scaling
;
588 cpu
->pstate
.max_pstate_physical
= intel_pstate_freq_to_hwp(cpu
, freq
);
590 freq
= cpu
->pstate
.min_pstate
* perf_ctl_scaling
;
591 cpu
->pstate
.min_freq
= freq
;
593 * Cast the min P-state value retrieved via pstate_funcs.get_min() to
594 * the effective range of HWP performance levels.
596 cpu
->pstate
.min_pstate
= intel_pstate_freq_to_hwp(cpu
, freq
);
599 static bool turbo_is_disabled(void)
603 rdmsrl(MSR_IA32_MISC_ENABLE
, misc_en
);
605 return !!(misc_en
& MSR_IA32_MISC_ENABLE_TURBO_DISABLE
);
608 static int min_perf_pct_min(void)
610 struct cpudata
*cpu
= all_cpu_data
[0];
611 int turbo_pstate
= cpu
->pstate
.turbo_pstate
;
613 return turbo_pstate
?
614 (cpu
->pstate
.min_pstate
* 100 / turbo_pstate
) : 0;
617 static s16
intel_pstate_get_epb(struct cpudata
*cpu_data
)
622 if (!boot_cpu_has(X86_FEATURE_EPB
))
625 ret
= rdmsrl_on_cpu(cpu_data
->cpu
, MSR_IA32_ENERGY_PERF_BIAS
, &epb
);
629 return (s16
)(epb
& 0x0f);
632 static s16
intel_pstate_get_epp(struct cpudata
*cpu_data
, u64 hwp_req_data
)
636 if (boot_cpu_has(X86_FEATURE_HWP_EPP
)) {
638 * When hwp_req_data is 0, means that caller didn't read
639 * MSR_HWP_REQUEST, so need to read and get EPP.
642 epp
= rdmsrl_on_cpu(cpu_data
->cpu
, MSR_HWP_REQUEST
,
647 epp
= (hwp_req_data
>> 24) & 0xff;
649 /* When there is no EPP present, HWP uses EPB settings */
650 epp
= intel_pstate_get_epb(cpu_data
);
656 static int intel_pstate_set_epb(int cpu
, s16 pref
)
661 if (!boot_cpu_has(X86_FEATURE_EPB
))
664 ret
= rdmsrl_on_cpu(cpu
, MSR_IA32_ENERGY_PERF_BIAS
, &epb
);
668 epb
= (epb
& ~0x0f) | pref
;
669 wrmsrl_on_cpu(cpu
, MSR_IA32_ENERGY_PERF_BIAS
, epb
);
675 * EPP/EPB display strings corresponding to EPP index in the
676 * energy_perf_strings[]
678 *-------------------------------------
681 * 2 balance_performance
686 enum energy_perf_value_index
{
687 EPP_INDEX_DEFAULT
= 0,
688 EPP_INDEX_PERFORMANCE
,
689 EPP_INDEX_BALANCE_PERFORMANCE
,
690 EPP_INDEX_BALANCE_POWERSAVE
,
694 static const char * const energy_perf_strings
[] = {
695 [EPP_INDEX_DEFAULT
] = "default",
696 [EPP_INDEX_PERFORMANCE
] = "performance",
697 [EPP_INDEX_BALANCE_PERFORMANCE
] = "balance_performance",
698 [EPP_INDEX_BALANCE_POWERSAVE
] = "balance_power",
699 [EPP_INDEX_POWERSAVE
] = "power",
702 static unsigned int epp_values
[] = {
703 [EPP_INDEX_DEFAULT
] = 0, /* Unused index */
704 [EPP_INDEX_PERFORMANCE
] = HWP_EPP_PERFORMANCE
,
705 [EPP_INDEX_BALANCE_PERFORMANCE
] = HWP_EPP_BALANCE_PERFORMANCE
,
706 [EPP_INDEX_BALANCE_POWERSAVE
] = HWP_EPP_BALANCE_POWERSAVE
,
707 [EPP_INDEX_POWERSAVE
] = HWP_EPP_POWERSAVE
,
710 static int intel_pstate_get_energy_pref_index(struct cpudata
*cpu_data
, int *raw_epp
)
716 epp
= intel_pstate_get_epp(cpu_data
, 0);
720 if (boot_cpu_has(X86_FEATURE_HWP_EPP
)) {
721 if (epp
== epp_values
[EPP_INDEX_PERFORMANCE
])
722 return EPP_INDEX_PERFORMANCE
;
723 if (epp
== epp_values
[EPP_INDEX_BALANCE_PERFORMANCE
])
724 return EPP_INDEX_BALANCE_PERFORMANCE
;
725 if (epp
== epp_values
[EPP_INDEX_BALANCE_POWERSAVE
])
726 return EPP_INDEX_BALANCE_POWERSAVE
;
727 if (epp
== epp_values
[EPP_INDEX_POWERSAVE
])
728 return EPP_INDEX_POWERSAVE
;
731 } else if (boot_cpu_has(X86_FEATURE_EPB
)) {
734 * 0x00-0x03 : Performance
735 * 0x04-0x07 : Balance performance
736 * 0x08-0x0B : Balance power
738 * The EPB is a 4 bit value, but our ranges restrict the
739 * value which can be set. Here only using top two bits
742 index
= (epp
>> 2) + 1;
748 static int intel_pstate_set_epp(struct cpudata
*cpu
, u32 epp
)
753 * Use the cached HWP Request MSR value, because in the active mode the
754 * register itself may be updated by intel_pstate_hwp_boost_up() or
755 * intel_pstate_hwp_boost_down() at any time.
757 u64 value
= READ_ONCE(cpu
->hwp_req_cached
);
759 value
&= ~GENMASK_ULL(31, 24);
760 value
|= (u64
)epp
<< 24;
762 * The only other updater of hwp_req_cached in the active mode,
763 * intel_pstate_hwp_set(), is called under the same lock as this
764 * function, so it cannot run in parallel with the update below.
766 WRITE_ONCE(cpu
->hwp_req_cached
, value
);
767 ret
= wrmsrl_on_cpu(cpu
->cpu
, MSR_HWP_REQUEST
, value
);
769 cpu
->epp_cached
= epp
;
774 static int intel_pstate_set_energy_pref_index(struct cpudata
*cpu_data
,
775 int pref_index
, bool use_raw
,
782 epp
= cpu_data
->epp_default
;
784 if (boot_cpu_has(X86_FEATURE_HWP_EPP
)) {
787 else if (epp
== -EINVAL
)
788 epp
= epp_values
[pref_index
];
791 * To avoid confusion, refuse to set EPP to any values different
792 * from 0 (performance) if the current policy is "performance",
793 * because those values would be overridden.
795 if (epp
> 0 && cpu_data
->policy
== CPUFREQ_POLICY_PERFORMANCE
)
798 ret
= intel_pstate_set_epp(cpu_data
, epp
);
801 epp
= (pref_index
- 1) << 2;
802 ret
= intel_pstate_set_epb(cpu_data
->cpu
, epp
);
808 static ssize_t
show_energy_performance_available_preferences(
809 struct cpufreq_policy
*policy
, char *buf
)
814 while (energy_perf_strings
[i
] != NULL
)
815 ret
+= sprintf(&buf
[ret
], "%s ", energy_perf_strings
[i
++]);
817 ret
+= sprintf(&buf
[ret
], "\n");
822 cpufreq_freq_attr_ro(energy_performance_available_preferences
);
824 static struct cpufreq_driver intel_pstate
;
826 static ssize_t
store_energy_performance_preference(
827 struct cpufreq_policy
*policy
, const char *buf
, size_t count
)
829 struct cpudata
*cpu
= all_cpu_data
[policy
->cpu
];
830 char str_preference
[21];
835 ret
= sscanf(buf
, "%20s", str_preference
);
839 ret
= match_string(energy_perf_strings
, -1, str_preference
);
841 if (!boot_cpu_has(X86_FEATURE_HWP_EPP
))
844 ret
= kstrtouint(buf
, 10, &epp
);
855 * This function runs with the policy R/W semaphore held, which
856 * guarantees that the driver pointer will not change while it is
859 if (!intel_pstate_driver
)
862 mutex_lock(&intel_pstate_limits_lock
);
864 if (intel_pstate_driver
== &intel_pstate
) {
865 ret
= intel_pstate_set_energy_pref_index(cpu
, ret
, raw
, epp
);
868 * In the passive mode the governor needs to be stopped on the
869 * target CPU before the EPP update and restarted after it,
870 * which is super-heavy-weight, so make sure it is worth doing
874 epp
= ret
? epp_values
[ret
] : cpu
->epp_default
;
876 if (cpu
->epp_cached
!= epp
) {
879 cpufreq_stop_governor(policy
);
880 ret
= intel_pstate_set_epp(cpu
, epp
);
881 err
= cpufreq_start_governor(policy
);
889 mutex_unlock(&intel_pstate_limits_lock
);
894 static ssize_t
show_energy_performance_preference(
895 struct cpufreq_policy
*policy
, char *buf
)
897 struct cpudata
*cpu_data
= all_cpu_data
[policy
->cpu
];
898 int preference
, raw_epp
;
900 preference
= intel_pstate_get_energy_pref_index(cpu_data
, &raw_epp
);
905 return sprintf(buf
, "%d\n", raw_epp
);
907 return sprintf(buf
, "%s\n", energy_perf_strings
[preference
]);
910 cpufreq_freq_attr_rw(energy_performance_preference
);
912 static ssize_t
show_base_frequency(struct cpufreq_policy
*policy
, char *buf
)
914 struct cpudata
*cpu
= all_cpu_data
[policy
->cpu
];
917 ratio
= intel_pstate_get_cppc_guaranteed(policy
->cpu
);
921 rdmsrl_on_cpu(policy
->cpu
, MSR_HWP_CAPABILITIES
, &cap
);
922 ratio
= HWP_GUARANTEED_PERF(cap
);
925 freq
= ratio
* cpu
->pstate
.scaling
;
926 if (cpu
->pstate
.scaling
!= cpu
->pstate
.perf_ctl_scaling
)
927 freq
= rounddown(freq
, cpu
->pstate
.perf_ctl_scaling
);
929 return sprintf(buf
, "%d\n", freq
);
932 cpufreq_freq_attr_ro(base_frequency
);
934 static struct freq_attr
*hwp_cpufreq_attrs
[] = {
935 &energy_performance_preference
,
936 &energy_performance_available_preferences
,
941 static struct cpudata
*hybrid_max_perf_cpu __read_mostly
;
943 * Protects hybrid_max_perf_cpu, the capacity_perf fields in struct cpudata,
944 * and the x86 arch scale-invariance information from concurrent updates.
946 static DEFINE_MUTEX(hybrid_capacity_lock
);
948 static void hybrid_set_cpu_capacity(struct cpudata
*cpu
)
950 arch_set_cpu_capacity(cpu
->cpu
, cpu
->capacity_perf
,
951 hybrid_max_perf_cpu
->capacity_perf
,
953 cpu
->pstate
.max_pstate_physical
);
955 pr_debug("CPU%d: perf = %u, max. perf = %u, base perf = %d\n", cpu
->cpu
,
956 cpu
->capacity_perf
, hybrid_max_perf_cpu
->capacity_perf
,
957 cpu
->pstate
.max_pstate_physical
);
960 static void hybrid_clear_cpu_capacity(unsigned int cpunum
)
962 arch_set_cpu_capacity(cpunum
, 1, 1, 1, 1);
965 static void hybrid_get_capacity_perf(struct cpudata
*cpu
)
967 if (READ_ONCE(global
.no_turbo
)) {
968 cpu
->capacity_perf
= cpu
->pstate
.max_pstate_physical
;
972 cpu
->capacity_perf
= HWP_HIGHEST_PERF(READ_ONCE(cpu
->hwp_cap_cached
));
975 static void hybrid_set_capacity_of_cpus(void)
979 for_each_online_cpu(cpunum
) {
980 struct cpudata
*cpu
= all_cpu_data
[cpunum
];
983 hybrid_set_cpu_capacity(cpu
);
987 static void hybrid_update_cpu_capacity_scaling(void)
989 struct cpudata
*max_perf_cpu
= NULL
;
990 unsigned int max_cap_perf
= 0;
993 for_each_online_cpu(cpunum
) {
994 struct cpudata
*cpu
= all_cpu_data
[cpunum
];
1000 * During initialization, CPU performance at full capacity needs
1003 if (!hybrid_max_perf_cpu
)
1004 hybrid_get_capacity_perf(cpu
);
1007 * If hybrid_max_perf_cpu is not NULL at this point, it is
1008 * being replaced, so don't take it into account when looking
1011 if (cpu
== hybrid_max_perf_cpu
)
1014 if (cpu
->capacity_perf
> max_cap_perf
) {
1015 max_cap_perf
= cpu
->capacity_perf
;
1021 hybrid_max_perf_cpu
= max_perf_cpu
;
1022 hybrid_set_capacity_of_cpus();
1024 pr_info("Found no CPUs with nonzero maximum performance\n");
1025 /* Revert to the flat CPU capacity structure. */
1026 for_each_online_cpu(cpunum
)
1027 hybrid_clear_cpu_capacity(cpunum
);
1031 static void __hybrid_refresh_cpu_capacity_scaling(void)
1033 hybrid_max_perf_cpu
= NULL
;
1034 hybrid_update_cpu_capacity_scaling();
1037 static void hybrid_refresh_cpu_capacity_scaling(void)
1039 guard(mutex
)(&hybrid_capacity_lock
);
1041 __hybrid_refresh_cpu_capacity_scaling();
1044 static void hybrid_init_cpu_capacity_scaling(bool refresh
)
1047 * If hybrid_max_perf_cpu is set at this point, the hybrid CPU capacity
1048 * scaling has been enabled already and the driver is just changing the
1052 hybrid_refresh_cpu_capacity_scaling();
1057 * On hybrid systems, use asym capacity instead of ITMT, but because
1058 * the capacity of SMT threads is not deterministic even approximately,
1059 * do not do that when SMT is in use.
1061 if (hwp_is_hybrid
&& !sched_smt_active() && arch_enable_hybrid_capacity_scale()) {
1062 hybrid_refresh_cpu_capacity_scaling();
1064 * Disabling ITMT causes sched domains to be rebuilt to disable asym
1065 * packing and enable asym capacity.
1067 sched_clear_itmt_support();
1071 static bool hybrid_clear_max_perf_cpu(void)
1075 guard(mutex
)(&hybrid_capacity_lock
);
1077 ret
= !!hybrid_max_perf_cpu
;
1078 hybrid_max_perf_cpu
= NULL
;
1083 static void __intel_pstate_get_hwp_cap(struct cpudata
*cpu
)
1087 rdmsrl_on_cpu(cpu
->cpu
, MSR_HWP_CAPABILITIES
, &cap
);
1088 WRITE_ONCE(cpu
->hwp_cap_cached
, cap
);
1089 cpu
->pstate
.max_pstate
= HWP_GUARANTEED_PERF(cap
);
1090 cpu
->pstate
.turbo_pstate
= HWP_HIGHEST_PERF(cap
);
1093 static void intel_pstate_get_hwp_cap(struct cpudata
*cpu
)
1095 int scaling
= cpu
->pstate
.scaling
;
1097 __intel_pstate_get_hwp_cap(cpu
);
1099 cpu
->pstate
.max_freq
= cpu
->pstate
.max_pstate
* scaling
;
1100 cpu
->pstate
.turbo_freq
= cpu
->pstate
.turbo_pstate
* scaling
;
1101 if (scaling
!= cpu
->pstate
.perf_ctl_scaling
) {
1102 int perf_ctl_scaling
= cpu
->pstate
.perf_ctl_scaling
;
1104 cpu
->pstate
.max_freq
= rounddown(cpu
->pstate
.max_freq
,
1106 cpu
->pstate
.turbo_freq
= rounddown(cpu
->pstate
.turbo_freq
,
1111 static void hybrid_update_capacity(struct cpudata
*cpu
)
1113 unsigned int max_cap_perf
;
1115 mutex_lock(&hybrid_capacity_lock
);
1117 if (!hybrid_max_perf_cpu
)
1121 * The maximum performance of the CPU may have changed, but assume
1122 * that the performance of the other CPUs has not changed.
1124 max_cap_perf
= hybrid_max_perf_cpu
->capacity_perf
;
1126 intel_pstate_get_hwp_cap(cpu
);
1128 hybrid_get_capacity_perf(cpu
);
1129 /* Should hybrid_max_perf_cpu be replaced by this CPU? */
1130 if (cpu
->capacity_perf
> max_cap_perf
) {
1131 hybrid_max_perf_cpu
= cpu
;
1132 hybrid_set_capacity_of_cpus();
1136 /* If this CPU is hybrid_max_perf_cpu, should it be replaced? */
1137 if (cpu
== hybrid_max_perf_cpu
&& cpu
->capacity_perf
< max_cap_perf
) {
1138 hybrid_update_cpu_capacity_scaling();
1142 hybrid_set_cpu_capacity(cpu
);
1145 mutex_unlock(&hybrid_capacity_lock
);
1148 static void intel_pstate_hwp_set(unsigned int cpu
)
1150 struct cpudata
*cpu_data
= all_cpu_data
[cpu
];
1155 max
= cpu_data
->max_perf_ratio
;
1156 min
= cpu_data
->min_perf_ratio
;
1158 if (cpu_data
->policy
== CPUFREQ_POLICY_PERFORMANCE
)
1161 rdmsrl_on_cpu(cpu
, MSR_HWP_REQUEST
, &value
);
1163 value
&= ~HWP_MIN_PERF(~0L);
1164 value
|= HWP_MIN_PERF(min
);
1166 value
&= ~HWP_MAX_PERF(~0L);
1167 value
|= HWP_MAX_PERF(max
);
1169 if (cpu_data
->epp_policy
== cpu_data
->policy
)
1172 cpu_data
->epp_policy
= cpu_data
->policy
;
1174 if (cpu_data
->policy
== CPUFREQ_POLICY_PERFORMANCE
) {
1175 epp
= intel_pstate_get_epp(cpu_data
, value
);
1176 cpu_data
->epp_powersave
= epp
;
1177 /* If EPP read was failed, then don't try to write */
1183 /* skip setting EPP, when saved value is invalid */
1184 if (cpu_data
->epp_powersave
< 0)
1188 * No need to restore EPP when it is not zero. This
1190 * - Policy is not changed
1191 * - user has manually changed
1192 * - Error reading EPB
1194 epp
= intel_pstate_get_epp(cpu_data
, value
);
1198 epp
= cpu_data
->epp_powersave
;
1200 if (boot_cpu_has(X86_FEATURE_HWP_EPP
)) {
1201 value
&= ~GENMASK_ULL(31, 24);
1202 value
|= (u64
)epp
<< 24;
1204 intel_pstate_set_epb(cpu
, epp
);
1207 WRITE_ONCE(cpu_data
->hwp_req_cached
, value
);
1208 wrmsrl_on_cpu(cpu
, MSR_HWP_REQUEST
, value
);
1211 static void intel_pstate_disable_hwp_interrupt(struct cpudata
*cpudata
);
1213 static void intel_pstate_hwp_offline(struct cpudata
*cpu
)
1215 u64 value
= READ_ONCE(cpu
->hwp_req_cached
);
1218 intel_pstate_disable_hwp_interrupt(cpu
);
1220 if (boot_cpu_has(X86_FEATURE_HWP_EPP
)) {
1222 * In case the EPP has been set to "performance" by the
1223 * active mode "performance" scaling algorithm, replace that
1224 * temporary value with the cached EPP one.
1226 value
&= ~GENMASK_ULL(31, 24);
1227 value
|= HWP_ENERGY_PERF_PREFERENCE(cpu
->epp_cached
);
1229 * However, make sure that EPP will be set to "performance" when
1230 * the CPU is brought back online again and the "performance"
1231 * scaling algorithm is still in effect.
1233 cpu
->epp_policy
= CPUFREQ_POLICY_UNKNOWN
;
1237 * Clear the desired perf field in the cached HWP request value to
1238 * prevent nonzero desired values from being leaked into the active
1241 value
&= ~HWP_DESIRED_PERF(~0L);
1242 WRITE_ONCE(cpu
->hwp_req_cached
, value
);
1244 value
&= ~GENMASK_ULL(31, 0);
1245 min_perf
= HWP_LOWEST_PERF(READ_ONCE(cpu
->hwp_cap_cached
));
1247 /* Set hwp_max = hwp_min */
1248 value
|= HWP_MAX_PERF(min_perf
);
1249 value
|= HWP_MIN_PERF(min_perf
);
1251 /* Set EPP to min */
1252 if (boot_cpu_has(X86_FEATURE_HWP_EPP
))
1253 value
|= HWP_ENERGY_PERF_PREFERENCE(HWP_EPP_POWERSAVE
);
1255 wrmsrl_on_cpu(cpu
->cpu
, MSR_HWP_REQUEST
, value
);
1257 mutex_lock(&hybrid_capacity_lock
);
1259 if (!hybrid_max_perf_cpu
) {
1260 mutex_unlock(&hybrid_capacity_lock
);
1265 if (hybrid_max_perf_cpu
== cpu
)
1266 hybrid_update_cpu_capacity_scaling();
1268 mutex_unlock(&hybrid_capacity_lock
);
1270 /* Reset the capacity of the CPU going offline to the initial value. */
1271 hybrid_clear_cpu_capacity(cpu
->cpu
);
1274 #define POWER_CTL_EE_ENABLE 1
1275 #define POWER_CTL_EE_DISABLE 2
1277 static int power_ctl_ee_state
;
1279 static void set_power_ctl_ee_state(bool input
)
1283 mutex_lock(&intel_pstate_driver_lock
);
1284 rdmsrl(MSR_IA32_POWER_CTL
, power_ctl
);
1286 power_ctl
&= ~BIT(MSR_IA32_POWER_CTL_BIT_EE
);
1287 power_ctl_ee_state
= POWER_CTL_EE_ENABLE
;
1289 power_ctl
|= BIT(MSR_IA32_POWER_CTL_BIT_EE
);
1290 power_ctl_ee_state
= POWER_CTL_EE_DISABLE
;
1292 wrmsrl(MSR_IA32_POWER_CTL
, power_ctl
);
1293 mutex_unlock(&intel_pstate_driver_lock
);
1296 static void intel_pstate_hwp_enable(struct cpudata
*cpudata
);
1298 static void intel_pstate_hwp_reenable(struct cpudata
*cpu
)
1300 intel_pstate_hwp_enable(cpu
);
1301 wrmsrl_on_cpu(cpu
->cpu
, MSR_HWP_REQUEST
, READ_ONCE(cpu
->hwp_req_cached
));
1304 static int intel_pstate_suspend(struct cpufreq_policy
*policy
)
1306 struct cpudata
*cpu
= all_cpu_data
[policy
->cpu
];
1308 pr_debug("CPU %d suspending\n", cpu
->cpu
);
1310 cpu
->suspended
= true;
1312 /* disable HWP interrupt and cancel any pending work */
1313 intel_pstate_disable_hwp_interrupt(cpu
);
1318 static int intel_pstate_resume(struct cpufreq_policy
*policy
)
1320 struct cpudata
*cpu
= all_cpu_data
[policy
->cpu
];
1322 pr_debug("CPU %d resuming\n", cpu
->cpu
);
1324 /* Only restore if the system default is changed */
1325 if (power_ctl_ee_state
== POWER_CTL_EE_ENABLE
)
1326 set_power_ctl_ee_state(true);
1327 else if (power_ctl_ee_state
== POWER_CTL_EE_DISABLE
)
1328 set_power_ctl_ee_state(false);
1330 if (cpu
->suspended
&& hwp_active
) {
1331 mutex_lock(&intel_pstate_limits_lock
);
1333 /* Re-enable HWP, because "online" has not done that. */
1334 intel_pstate_hwp_reenable(cpu
);
1336 mutex_unlock(&intel_pstate_limits_lock
);
1339 cpu
->suspended
= false;
1344 static void intel_pstate_update_policies(void)
1348 for_each_possible_cpu(cpu
)
1349 cpufreq_update_policy(cpu
);
1352 static void __intel_pstate_update_max_freq(struct cpudata
*cpudata
,
1353 struct cpufreq_policy
*policy
)
1356 intel_pstate_get_hwp_cap(cpudata
);
1358 policy
->cpuinfo
.max_freq
= READ_ONCE(global
.no_turbo
) ?
1359 cpudata
->pstate
.max_freq
: cpudata
->pstate
.turbo_freq
;
1361 refresh_frequency_limits(policy
);
1364 static void intel_pstate_update_limits(unsigned int cpu
)
1366 struct cpufreq_policy
*policy
= cpufreq_cpu_acquire(cpu
);
1367 struct cpudata
*cpudata
;
1372 cpudata
= all_cpu_data
[cpu
];
1374 __intel_pstate_update_max_freq(cpudata
, policy
);
1376 /* Prevent the driver from being unregistered now. */
1377 mutex_lock(&intel_pstate_driver_lock
);
1379 cpufreq_cpu_release(policy
);
1381 hybrid_update_capacity(cpudata
);
1383 mutex_unlock(&intel_pstate_driver_lock
);
1386 static void intel_pstate_update_limits_for_all(void)
1390 for_each_possible_cpu(cpu
) {
1391 struct cpufreq_policy
*policy
= cpufreq_cpu_acquire(cpu
);
1396 __intel_pstate_update_max_freq(all_cpu_data
[cpu
], policy
);
1398 cpufreq_cpu_release(policy
);
1401 mutex_lock(&hybrid_capacity_lock
);
1403 if (hybrid_max_perf_cpu
)
1404 __hybrid_refresh_cpu_capacity_scaling();
1406 mutex_unlock(&hybrid_capacity_lock
);
1409 /************************** sysfs begin ************************/
1410 #define show_one(file_name, object) \
1411 static ssize_t show_##file_name \
1412 (struct kobject *kobj, struct kobj_attribute *attr, char *buf) \
1414 return sprintf(buf, "%u\n", global.object); \
1417 static ssize_t
intel_pstate_show_status(char *buf
);
1418 static int intel_pstate_update_status(const char *buf
, size_t size
);
1420 static ssize_t
show_status(struct kobject
*kobj
,
1421 struct kobj_attribute
*attr
, char *buf
)
1425 mutex_lock(&intel_pstate_driver_lock
);
1426 ret
= intel_pstate_show_status(buf
);
1427 mutex_unlock(&intel_pstate_driver_lock
);
1432 static ssize_t
store_status(struct kobject
*a
, struct kobj_attribute
*b
,
1433 const char *buf
, size_t count
)
1435 char *p
= memchr(buf
, '\n', count
);
1438 mutex_lock(&intel_pstate_driver_lock
);
1439 ret
= intel_pstate_update_status(buf
, p
? p
- buf
: count
);
1440 mutex_unlock(&intel_pstate_driver_lock
);
1442 return ret
< 0 ? ret
: count
;
1445 static ssize_t
show_turbo_pct(struct kobject
*kobj
,
1446 struct kobj_attribute
*attr
, char *buf
)
1448 struct cpudata
*cpu
;
1449 int total
, no_turbo
, turbo_pct
;
1452 mutex_lock(&intel_pstate_driver_lock
);
1454 if (!intel_pstate_driver
) {
1455 mutex_unlock(&intel_pstate_driver_lock
);
1459 cpu
= all_cpu_data
[0];
1461 total
= cpu
->pstate
.turbo_pstate
- cpu
->pstate
.min_pstate
+ 1;
1462 no_turbo
= cpu
->pstate
.max_pstate
- cpu
->pstate
.min_pstate
+ 1;
1463 turbo_fp
= div_fp(no_turbo
, total
);
1464 turbo_pct
= 100 - fp_toint(mul_fp(turbo_fp
, int_tofp(100)));
1466 mutex_unlock(&intel_pstate_driver_lock
);
1468 return sprintf(buf
, "%u\n", turbo_pct
);
1471 static ssize_t
show_num_pstates(struct kobject
*kobj
,
1472 struct kobj_attribute
*attr
, char *buf
)
1474 struct cpudata
*cpu
;
1477 mutex_lock(&intel_pstate_driver_lock
);
1479 if (!intel_pstate_driver
) {
1480 mutex_unlock(&intel_pstate_driver_lock
);
1484 cpu
= all_cpu_data
[0];
1485 total
= cpu
->pstate
.turbo_pstate
- cpu
->pstate
.min_pstate
+ 1;
1487 mutex_unlock(&intel_pstate_driver_lock
);
1489 return sprintf(buf
, "%u\n", total
);
1492 static ssize_t
show_no_turbo(struct kobject
*kobj
,
1493 struct kobj_attribute
*attr
, char *buf
)
1497 mutex_lock(&intel_pstate_driver_lock
);
1499 if (!intel_pstate_driver
) {
1500 mutex_unlock(&intel_pstate_driver_lock
);
1504 ret
= sprintf(buf
, "%u\n", global
.no_turbo
);
1506 mutex_unlock(&intel_pstate_driver_lock
);
1511 static ssize_t
store_no_turbo(struct kobject
*a
, struct kobj_attribute
*b
,
1512 const char *buf
, size_t count
)
1517 if (sscanf(buf
, "%u", &input
) != 1)
1520 mutex_lock(&intel_pstate_driver_lock
);
1522 if (!intel_pstate_driver
) {
1527 no_turbo
= !!clamp_t(int, input
, 0, 1);
1529 WRITE_ONCE(global
.turbo_disabled
, turbo_is_disabled());
1530 if (global
.turbo_disabled
&& !no_turbo
) {
1531 pr_notice("Turbo disabled by BIOS or unavailable on processor\n");
1533 if (global
.no_turbo
)
1539 if (no_turbo
== global
.no_turbo
) {
1543 WRITE_ONCE(global
.no_turbo
, no_turbo
);
1545 mutex_lock(&intel_pstate_limits_lock
);
1548 struct cpudata
*cpu
= all_cpu_data
[0];
1549 int pct
= cpu
->pstate
.max_pstate
* 100 / cpu
->pstate
.turbo_pstate
;
1551 /* Squash the global minimum into the permitted range. */
1552 if (global
.min_perf_pct
> pct
)
1553 global
.min_perf_pct
= pct
;
1556 mutex_unlock(&intel_pstate_limits_lock
);
1558 intel_pstate_update_limits_for_all();
1559 arch_set_max_freq_ratio(no_turbo
);
1562 mutex_unlock(&intel_pstate_driver_lock
);
1567 static void update_qos_request(enum freq_qos_req_type type
)
1569 struct freq_qos_request
*req
;
1570 struct cpufreq_policy
*policy
;
1573 for_each_possible_cpu(i
) {
1574 struct cpudata
*cpu
= all_cpu_data
[i
];
1575 unsigned int freq
, perf_pct
;
1577 policy
= cpufreq_cpu_get(i
);
1581 req
= policy
->driver_data
;
1582 cpufreq_cpu_put(policy
);
1588 intel_pstate_get_hwp_cap(cpu
);
1590 if (type
== FREQ_QOS_MIN
) {
1591 perf_pct
= global
.min_perf_pct
;
1594 perf_pct
= global
.max_perf_pct
;
1597 freq
= DIV_ROUND_UP(cpu
->pstate
.turbo_freq
* perf_pct
, 100);
1599 if (freq_qos_update_request(req
, freq
) < 0)
1600 pr_warn("Failed to update freq constraint: CPU%d\n", i
);
1604 static ssize_t
store_max_perf_pct(struct kobject
*a
, struct kobj_attribute
*b
,
1605 const char *buf
, size_t count
)
1610 ret
= sscanf(buf
, "%u", &input
);
1614 mutex_lock(&intel_pstate_driver_lock
);
1616 if (!intel_pstate_driver
) {
1617 mutex_unlock(&intel_pstate_driver_lock
);
1621 mutex_lock(&intel_pstate_limits_lock
);
1623 global
.max_perf_pct
= clamp_t(int, input
, global
.min_perf_pct
, 100);
1625 mutex_unlock(&intel_pstate_limits_lock
);
1627 if (intel_pstate_driver
== &intel_pstate
)
1628 intel_pstate_update_policies();
1630 update_qos_request(FREQ_QOS_MAX
);
1632 mutex_unlock(&intel_pstate_driver_lock
);
1637 static ssize_t
store_min_perf_pct(struct kobject
*a
, struct kobj_attribute
*b
,
1638 const char *buf
, size_t count
)
1643 ret
= sscanf(buf
, "%u", &input
);
1647 mutex_lock(&intel_pstate_driver_lock
);
1649 if (!intel_pstate_driver
) {
1650 mutex_unlock(&intel_pstate_driver_lock
);
1654 mutex_lock(&intel_pstate_limits_lock
);
1656 global
.min_perf_pct
= clamp_t(int, input
,
1657 min_perf_pct_min(), global
.max_perf_pct
);
1659 mutex_unlock(&intel_pstate_limits_lock
);
1661 if (intel_pstate_driver
== &intel_pstate
)
1662 intel_pstate_update_policies();
1664 update_qos_request(FREQ_QOS_MIN
);
1666 mutex_unlock(&intel_pstate_driver_lock
);
1671 static ssize_t
show_hwp_dynamic_boost(struct kobject
*kobj
,
1672 struct kobj_attribute
*attr
, char *buf
)
1674 return sprintf(buf
, "%u\n", hwp_boost
);
1677 static ssize_t
store_hwp_dynamic_boost(struct kobject
*a
,
1678 struct kobj_attribute
*b
,
1679 const char *buf
, size_t count
)
1684 ret
= kstrtouint(buf
, 10, &input
);
1688 mutex_lock(&intel_pstate_driver_lock
);
1689 hwp_boost
= !!input
;
1690 intel_pstate_update_policies();
1691 mutex_unlock(&intel_pstate_driver_lock
);
1696 static ssize_t
show_energy_efficiency(struct kobject
*kobj
, struct kobj_attribute
*attr
,
1702 rdmsrl(MSR_IA32_POWER_CTL
, power_ctl
);
1703 enable
= !!(power_ctl
& BIT(MSR_IA32_POWER_CTL_BIT_EE
));
1704 return sprintf(buf
, "%d\n", !enable
);
1707 static ssize_t
store_energy_efficiency(struct kobject
*a
, struct kobj_attribute
*b
,
1708 const char *buf
, size_t count
)
1713 ret
= kstrtobool(buf
, &input
);
1717 set_power_ctl_ee_state(input
);
1722 show_one(max_perf_pct
, max_perf_pct
);
1723 show_one(min_perf_pct
, min_perf_pct
);
1725 define_one_global_rw(status
);
1726 define_one_global_rw(no_turbo
);
1727 define_one_global_rw(max_perf_pct
);
1728 define_one_global_rw(min_perf_pct
);
1729 define_one_global_ro(turbo_pct
);
1730 define_one_global_ro(num_pstates
);
1731 define_one_global_rw(hwp_dynamic_boost
);
1732 define_one_global_rw(energy_efficiency
);
1734 static struct attribute
*intel_pstate_attributes
[] = {
1740 static const struct attribute_group intel_pstate_attr_group
= {
1741 .attrs
= intel_pstate_attributes
,
1744 static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids
[];
1746 static struct kobject
*intel_pstate_kobject
;
1748 static void __init
intel_pstate_sysfs_expose_params(void)
1750 struct device
*dev_root
= bus_get_dev_root(&cpu_subsys
);
1754 intel_pstate_kobject
= kobject_create_and_add("intel_pstate", &dev_root
->kobj
);
1755 put_device(dev_root
);
1757 if (WARN_ON(!intel_pstate_kobject
))
1760 rc
= sysfs_create_group(intel_pstate_kobject
, &intel_pstate_attr_group
);
1764 if (!boot_cpu_has(X86_FEATURE_HYBRID_CPU
)) {
1765 rc
= sysfs_create_file(intel_pstate_kobject
, &turbo_pct
.attr
);
1768 rc
= sysfs_create_file(intel_pstate_kobject
, &num_pstates
.attr
);
1773 * If per cpu limits are enforced there are no global limits, so
1774 * return without creating max/min_perf_pct attributes
1779 rc
= sysfs_create_file(intel_pstate_kobject
, &max_perf_pct
.attr
);
1782 rc
= sysfs_create_file(intel_pstate_kobject
, &min_perf_pct
.attr
);
1785 if (x86_match_cpu(intel_pstate_cpu_ee_disable_ids
)) {
1786 rc
= sysfs_create_file(intel_pstate_kobject
, &energy_efficiency
.attr
);
1791 static void __init
intel_pstate_sysfs_remove(void)
1793 if (!intel_pstate_kobject
)
1796 sysfs_remove_group(intel_pstate_kobject
, &intel_pstate_attr_group
);
1798 if (!boot_cpu_has(X86_FEATURE_HYBRID_CPU
)) {
1799 sysfs_remove_file(intel_pstate_kobject
, &num_pstates
.attr
);
1800 sysfs_remove_file(intel_pstate_kobject
, &turbo_pct
.attr
);
1803 if (!per_cpu_limits
) {
1804 sysfs_remove_file(intel_pstate_kobject
, &max_perf_pct
.attr
);
1805 sysfs_remove_file(intel_pstate_kobject
, &min_perf_pct
.attr
);
1807 if (x86_match_cpu(intel_pstate_cpu_ee_disable_ids
))
1808 sysfs_remove_file(intel_pstate_kobject
, &energy_efficiency
.attr
);
1811 kobject_put(intel_pstate_kobject
);
1814 static void intel_pstate_sysfs_expose_hwp_dynamic_boost(void)
1821 rc
= sysfs_create_file(intel_pstate_kobject
, &hwp_dynamic_boost
.attr
);
1825 static void intel_pstate_sysfs_hide_hwp_dynamic_boost(void)
1830 sysfs_remove_file(intel_pstate_kobject
, &hwp_dynamic_boost
.attr
);
1833 /************************** sysfs end ************************/
1835 static void intel_pstate_notify_work(struct work_struct
*work
)
1837 struct cpudata
*cpudata
=
1838 container_of(to_delayed_work(work
), struct cpudata
, hwp_notify_work
);
1839 struct cpufreq_policy
*policy
= cpufreq_cpu_acquire(cpudata
->cpu
);
1842 __intel_pstate_update_max_freq(cpudata
, policy
);
1844 cpufreq_cpu_release(policy
);
1847 * The driver will not be unregistered while this function is
1848 * running, so update the capacity without acquiring the driver
1851 hybrid_update_capacity(cpudata
);
1854 wrmsrl_on_cpu(cpudata
->cpu
, MSR_HWP_STATUS
, 0);
1857 static DEFINE_RAW_SPINLOCK(hwp_notify_lock
);
1858 static cpumask_t hwp_intr_enable_mask
;
1860 #define HWP_GUARANTEED_PERF_CHANGE_STATUS BIT(0)
1861 #define HWP_HIGHEST_PERF_CHANGE_STATUS BIT(3)
1863 void notify_hwp_interrupt(void)
1865 unsigned int this_cpu
= smp_processor_id();
1866 u64 value
, status_mask
;
1867 unsigned long flags
;
1869 if (!hwp_active
|| !cpu_feature_enabled(X86_FEATURE_HWP_NOTIFY
))
1872 status_mask
= HWP_GUARANTEED_PERF_CHANGE_STATUS
;
1873 if (cpu_feature_enabled(X86_FEATURE_HWP_HIGHEST_PERF_CHANGE
))
1874 status_mask
|= HWP_HIGHEST_PERF_CHANGE_STATUS
;
1876 rdmsrl_safe(MSR_HWP_STATUS
, &value
);
1877 if (!(value
& status_mask
))
1880 raw_spin_lock_irqsave(&hwp_notify_lock
, flags
);
1882 if (!cpumask_test_cpu(this_cpu
, &hwp_intr_enable_mask
))
1885 schedule_delayed_work(&all_cpu_data
[this_cpu
]->hwp_notify_work
,
1886 msecs_to_jiffies(10));
1888 raw_spin_unlock_irqrestore(&hwp_notify_lock
, flags
);
1893 wrmsrl_safe(MSR_HWP_STATUS
, 0);
1894 raw_spin_unlock_irqrestore(&hwp_notify_lock
, flags
);
1897 static void intel_pstate_disable_hwp_interrupt(struct cpudata
*cpudata
)
1901 if (!cpu_feature_enabled(X86_FEATURE_HWP_NOTIFY
))
1904 /* wrmsrl_on_cpu has to be outside spinlock as this can result in IPC */
1905 wrmsrl_on_cpu(cpudata
->cpu
, MSR_HWP_INTERRUPT
, 0x00);
1907 raw_spin_lock_irq(&hwp_notify_lock
);
1908 cancel_work
= cpumask_test_and_clear_cpu(cpudata
->cpu
, &hwp_intr_enable_mask
);
1909 raw_spin_unlock_irq(&hwp_notify_lock
);
1912 cancel_delayed_work_sync(&cpudata
->hwp_notify_work
);
1915 #define HWP_GUARANTEED_PERF_CHANGE_REQ BIT(0)
1916 #define HWP_HIGHEST_PERF_CHANGE_REQ BIT(2)
1918 static void intel_pstate_enable_hwp_interrupt(struct cpudata
*cpudata
)
1920 /* Enable HWP notification interrupt for performance change */
1921 if (boot_cpu_has(X86_FEATURE_HWP_NOTIFY
)) {
1922 u64 interrupt_mask
= HWP_GUARANTEED_PERF_CHANGE_REQ
;
1924 raw_spin_lock_irq(&hwp_notify_lock
);
1925 INIT_DELAYED_WORK(&cpudata
->hwp_notify_work
, intel_pstate_notify_work
);
1926 cpumask_set_cpu(cpudata
->cpu
, &hwp_intr_enable_mask
);
1927 raw_spin_unlock_irq(&hwp_notify_lock
);
1929 if (cpu_feature_enabled(X86_FEATURE_HWP_HIGHEST_PERF_CHANGE
))
1930 interrupt_mask
|= HWP_HIGHEST_PERF_CHANGE_REQ
;
1932 /* wrmsrl_on_cpu has to be outside spinlock as this can result in IPC */
1933 wrmsrl_on_cpu(cpudata
->cpu
, MSR_HWP_INTERRUPT
, interrupt_mask
);
1934 wrmsrl_on_cpu(cpudata
->cpu
, MSR_HWP_STATUS
, 0);
1938 static void intel_pstate_update_epp_defaults(struct cpudata
*cpudata
)
1940 cpudata
->epp_default
= intel_pstate_get_epp(cpudata
, 0);
1943 * If the EPP is set by firmware, which means that firmware enabled HWP
1944 * - Is equal or less than 0x80 (default balance_perf EPP)
1945 * - But less performance oriented than performance EPP
1946 * then use this as new balance_perf EPP.
1948 if (hwp_forced
&& cpudata
->epp_default
<= HWP_EPP_BALANCE_PERFORMANCE
&&
1949 cpudata
->epp_default
> HWP_EPP_PERFORMANCE
) {
1950 epp_values
[EPP_INDEX_BALANCE_PERFORMANCE
] = cpudata
->epp_default
;
1955 * If this CPU gen doesn't call for change in balance_perf
1958 if (epp_values
[EPP_INDEX_BALANCE_PERFORMANCE
] == HWP_EPP_BALANCE_PERFORMANCE
)
1962 * Use hard coded value per gen to update the balance_perf
1965 cpudata
->epp_default
= epp_values
[EPP_INDEX_BALANCE_PERFORMANCE
];
1966 intel_pstate_set_epp(cpudata
, cpudata
->epp_default
);
1969 static void intel_pstate_hwp_enable(struct cpudata
*cpudata
)
1971 /* First disable HWP notification interrupt till we activate again */
1972 if (boot_cpu_has(X86_FEATURE_HWP_NOTIFY
))
1973 wrmsrl_on_cpu(cpudata
->cpu
, MSR_HWP_INTERRUPT
, 0x00);
1975 wrmsrl_on_cpu(cpudata
->cpu
, MSR_PM_ENABLE
, 0x1);
1977 intel_pstate_enable_hwp_interrupt(cpudata
);
1979 if (cpudata
->epp_default
>= 0)
1982 intel_pstate_update_epp_defaults(cpudata
);
1985 static int atom_get_min_pstate(int not_used
)
1989 rdmsrl(MSR_ATOM_CORE_RATIOS
, value
);
1990 return (value
>> 8) & 0x7F;
1993 static int atom_get_max_pstate(int not_used
)
1997 rdmsrl(MSR_ATOM_CORE_RATIOS
, value
);
1998 return (value
>> 16) & 0x7F;
2001 static int atom_get_turbo_pstate(int not_used
)
2005 rdmsrl(MSR_ATOM_CORE_TURBO_RATIOS
, value
);
2006 return value
& 0x7F;
2009 static u64
atom_get_val(struct cpudata
*cpudata
, int pstate
)
2015 val
= (u64
)pstate
<< 8;
2016 if (READ_ONCE(global
.no_turbo
) && !READ_ONCE(global
.turbo_disabled
))
2017 val
|= (u64
)1 << 32;
2019 vid_fp
= cpudata
->vid
.min
+ mul_fp(
2020 int_tofp(pstate
- cpudata
->pstate
.min_pstate
),
2021 cpudata
->vid
.ratio
);
2023 vid_fp
= clamp_t(int32_t, vid_fp
, cpudata
->vid
.min
, cpudata
->vid
.max
);
2024 vid
= ceiling_fp(vid_fp
);
2026 if (pstate
> cpudata
->pstate
.max_pstate
)
2027 vid
= cpudata
->vid
.turbo
;
2032 static int silvermont_get_scaling(void)
2036 /* Defined in Table 35-6 from SDM (Sept 2015) */
2037 static int silvermont_freq_table
[] = {
2038 83300, 100000, 133300, 116700, 80000};
2040 rdmsrl(MSR_FSB_FREQ
, value
);
2044 return silvermont_freq_table
[i
];
2047 static int airmont_get_scaling(void)
2051 /* Defined in Table 35-10 from SDM (Sept 2015) */
2052 static int airmont_freq_table
[] = {
2053 83300, 100000, 133300, 116700, 80000,
2054 93300, 90000, 88900, 87500};
2056 rdmsrl(MSR_FSB_FREQ
, value
);
2060 return airmont_freq_table
[i
];
2063 static void atom_get_vid(struct cpudata
*cpudata
)
2067 rdmsrl(MSR_ATOM_CORE_VIDS
, value
);
2068 cpudata
->vid
.min
= int_tofp((value
>> 8) & 0x7f);
2069 cpudata
->vid
.max
= int_tofp((value
>> 16) & 0x7f);
2070 cpudata
->vid
.ratio
= div_fp(
2071 cpudata
->vid
.max
- cpudata
->vid
.min
,
2072 int_tofp(cpudata
->pstate
.max_pstate
-
2073 cpudata
->pstate
.min_pstate
));
2075 rdmsrl(MSR_ATOM_CORE_TURBO_VIDS
, value
);
2076 cpudata
->vid
.turbo
= value
& 0x7f;
2079 static int core_get_min_pstate(int cpu
)
2083 rdmsrl_on_cpu(cpu
, MSR_PLATFORM_INFO
, &value
);
2084 return (value
>> 40) & 0xFF;
2087 static int core_get_max_pstate_physical(int cpu
)
2091 rdmsrl_on_cpu(cpu
, MSR_PLATFORM_INFO
, &value
);
2092 return (value
>> 8) & 0xFF;
2095 static int core_get_tdp_ratio(int cpu
, u64 plat_info
)
2097 /* Check how many TDP levels present */
2098 if (plat_info
& 0x600000000) {
2104 /* Get the TDP level (0, 1, 2) to get ratios */
2105 err
= rdmsrl_safe_on_cpu(cpu
, MSR_CONFIG_TDP_CONTROL
, &tdp_ctrl
);
2109 /* TDP MSR are continuous starting at 0x648 */
2110 tdp_msr
= MSR_CONFIG_TDP_NOMINAL
+ (tdp_ctrl
& 0x03);
2111 err
= rdmsrl_safe_on_cpu(cpu
, tdp_msr
, &tdp_ratio
);
2115 /* For level 1 and 2, bits[23:16] contain the ratio */
2116 if (tdp_ctrl
& 0x03)
2119 tdp_ratio
&= 0xff; /* ratios are only 8 bits long */
2120 pr_debug("tdp_ratio %x\n", (int)tdp_ratio
);
2122 return (int)tdp_ratio
;
2128 static int core_get_max_pstate(int cpu
)
2136 rdmsrl_on_cpu(cpu
, MSR_PLATFORM_INFO
, &plat_info
);
2137 max_pstate
= (plat_info
>> 8) & 0xFF;
2139 tdp_ratio
= core_get_tdp_ratio(cpu
, plat_info
);
2144 /* Turbo activation ratio is not used on HWP platforms */
2148 err
= rdmsrl_safe_on_cpu(cpu
, MSR_TURBO_ACTIVATION_RATIO
, &tar
);
2152 /* Do some sanity checking for safety */
2153 tar_levels
= tar
& 0xff;
2154 if (tdp_ratio
- 1 == tar_levels
) {
2155 max_pstate
= tar_levels
;
2156 pr_debug("max_pstate=TAC %x\n", max_pstate
);
2163 static int core_get_turbo_pstate(int cpu
)
2168 rdmsrl_on_cpu(cpu
, MSR_TURBO_RATIO_LIMIT
, &value
);
2169 nont
= core_get_max_pstate(cpu
);
2170 ret
= (value
) & 255;
2176 static u64
core_get_val(struct cpudata
*cpudata
, int pstate
)
2180 val
= (u64
)pstate
<< 8;
2181 if (READ_ONCE(global
.no_turbo
) && !READ_ONCE(global
.turbo_disabled
))
2182 val
|= (u64
)1 << 32;
2187 static int knl_get_aperf_mperf_shift(void)
2192 static int knl_get_turbo_pstate(int cpu
)
2197 rdmsrl_on_cpu(cpu
, MSR_TURBO_RATIO_LIMIT
, &value
);
2198 nont
= core_get_max_pstate(cpu
);
2199 ret
= (((value
) >> 8) & 0xFF);
2205 static void hybrid_get_type(void *data
)
2207 u8
*cpu_type
= data
;
2209 *cpu_type
= get_this_hybrid_cpu_type();
2212 static int hwp_get_cpu_scaling(int cpu
)
2216 smp_call_function_single(cpu
, hybrid_get_type
, &cpu_type
, 1);
2217 /* P-cores have a smaller perf level-to-freqency scaling factor. */
2218 if (cpu_type
== 0x40)
2219 return hybrid_scaling_factor
;
2221 /* Use default core scaling for E-cores */
2222 if (cpu_type
== 0x20)
2223 return core_get_scaling();
2226 * If reached here, this system is either non-hybrid (like Tiger
2227 * Lake) or hybrid-capable (like Alder Lake or Raptor Lake) with
2228 * no E cores (in which case CPUID for hybrid support is 0).
2230 * The CPPC nominal_frequency field is 0 for non-hybrid systems,
2231 * so the default core scaling will be used for them.
2233 return intel_pstate_cppc_get_scaling(cpu
);
2236 static void intel_pstate_set_pstate(struct cpudata
*cpu
, int pstate
)
2238 trace_cpu_frequency(pstate
* cpu
->pstate
.scaling
, cpu
->cpu
);
2239 cpu
->pstate
.current_pstate
= pstate
;
2241 * Generally, there is no guarantee that this code will always run on
2242 * the CPU being updated, so force the register update to run on the
2245 wrmsrl_on_cpu(cpu
->cpu
, MSR_IA32_PERF_CTL
,
2246 pstate_funcs
.get_val(cpu
, pstate
));
2249 static void intel_pstate_set_min_pstate(struct cpudata
*cpu
)
2251 intel_pstate_set_pstate(cpu
, cpu
->pstate
.min_pstate
);
2254 static void intel_pstate_get_cpu_pstates(struct cpudata
*cpu
)
2256 int perf_ctl_max_phys
= pstate_funcs
.get_max_physical(cpu
->cpu
);
2257 int perf_ctl_scaling
= pstate_funcs
.get_scaling();
2259 cpu
->pstate
.min_pstate
= pstate_funcs
.get_min(cpu
->cpu
);
2260 cpu
->pstate
.max_pstate_physical
= perf_ctl_max_phys
;
2261 cpu
->pstate
.perf_ctl_scaling
= perf_ctl_scaling
;
2263 if (hwp_active
&& !hwp_mode_bdw
) {
2264 __intel_pstate_get_hwp_cap(cpu
);
2266 if (pstate_funcs
.get_cpu_scaling
) {
2267 cpu
->pstate
.scaling
= pstate_funcs
.get_cpu_scaling(cpu
->cpu
);
2268 if (cpu
->pstate
.scaling
!= perf_ctl_scaling
) {
2269 intel_pstate_hybrid_hwp_adjust(cpu
);
2270 hwp_is_hybrid
= true;
2273 cpu
->pstate
.scaling
= perf_ctl_scaling
;
2276 * If the CPU is going online for the first time and it was
2277 * offline initially, asym capacity scaling needs to be updated.
2279 hybrid_update_capacity(cpu
);
2281 cpu
->pstate
.scaling
= perf_ctl_scaling
;
2282 cpu
->pstate
.max_pstate
= pstate_funcs
.get_max(cpu
->cpu
);
2283 cpu
->pstate
.turbo_pstate
= pstate_funcs
.get_turbo(cpu
->cpu
);
2286 if (cpu
->pstate
.scaling
== perf_ctl_scaling
) {
2287 cpu
->pstate
.min_freq
= cpu
->pstate
.min_pstate
* perf_ctl_scaling
;
2288 cpu
->pstate
.max_freq
= cpu
->pstate
.max_pstate
* perf_ctl_scaling
;
2289 cpu
->pstate
.turbo_freq
= cpu
->pstate
.turbo_pstate
* perf_ctl_scaling
;
2292 if (pstate_funcs
.get_aperf_mperf_shift
)
2293 cpu
->aperf_mperf_shift
= pstate_funcs
.get_aperf_mperf_shift();
2295 if (pstate_funcs
.get_vid
)
2296 pstate_funcs
.get_vid(cpu
);
2298 intel_pstate_set_min_pstate(cpu
);
2302 * Long hold time will keep high perf limits for long time,
2303 * which negatively impacts perf/watt for some workloads,
2304 * like specpower. 3ms is based on experiements on some
2307 static int hwp_boost_hold_time_ns
= 3 * NSEC_PER_MSEC
;
2309 static inline void intel_pstate_hwp_boost_up(struct cpudata
*cpu
)
2311 u64 hwp_req
= READ_ONCE(cpu
->hwp_req_cached
);
2312 u64 hwp_cap
= READ_ONCE(cpu
->hwp_cap_cached
);
2313 u32 max_limit
= (hwp_req
& 0xff00) >> 8;
2314 u32 min_limit
= (hwp_req
& 0xff);
2318 * Cases to consider (User changes via sysfs or boot time):
2319 * If, P0 (Turbo max) = P1 (Guaranteed max) = min:
2321 * If, P0 (Turbo max) > P1 (Guaranteed max) = min:
2322 * Should result in one level boost only for P0.
2323 * If, P0 (Turbo max) = P1 (Guaranteed max) > min:
2324 * Should result in two level boost:
2325 * (min + p1)/2 and P1.
2326 * If, P0 (Turbo max) > P1 (Guaranteed max) > min:
2327 * Should result in three level boost:
2328 * (min + p1)/2, P1 and P0.
2331 /* If max and min are equal or already at max, nothing to boost */
2332 if (max_limit
== min_limit
|| cpu
->hwp_boost_min
>= max_limit
)
2335 if (!cpu
->hwp_boost_min
)
2336 cpu
->hwp_boost_min
= min_limit
;
2338 /* level at half way mark between min and guranteed */
2339 boost_level1
= (HWP_GUARANTEED_PERF(hwp_cap
) + min_limit
) >> 1;
2341 if (cpu
->hwp_boost_min
< boost_level1
)
2342 cpu
->hwp_boost_min
= boost_level1
;
2343 else if (cpu
->hwp_boost_min
< HWP_GUARANTEED_PERF(hwp_cap
))
2344 cpu
->hwp_boost_min
= HWP_GUARANTEED_PERF(hwp_cap
);
2345 else if (cpu
->hwp_boost_min
== HWP_GUARANTEED_PERF(hwp_cap
) &&
2346 max_limit
!= HWP_GUARANTEED_PERF(hwp_cap
))
2347 cpu
->hwp_boost_min
= max_limit
;
2351 hwp_req
= (hwp_req
& ~GENMASK_ULL(7, 0)) | cpu
->hwp_boost_min
;
2352 wrmsrl(MSR_HWP_REQUEST
, hwp_req
);
2353 cpu
->last_update
= cpu
->sample
.time
;
2356 static inline void intel_pstate_hwp_boost_down(struct cpudata
*cpu
)
2358 if (cpu
->hwp_boost_min
) {
2361 /* Check if we are idle for hold time to boost down */
2362 expired
= time_after64(cpu
->sample
.time
, cpu
->last_update
+
2363 hwp_boost_hold_time_ns
);
2365 wrmsrl(MSR_HWP_REQUEST
, cpu
->hwp_req_cached
);
2366 cpu
->hwp_boost_min
= 0;
2369 cpu
->last_update
= cpu
->sample
.time
;
2372 static inline void intel_pstate_update_util_hwp_local(struct cpudata
*cpu
,
2375 cpu
->sample
.time
= time
;
2377 if (cpu
->sched_flags
& SCHED_CPUFREQ_IOWAIT
) {
2380 cpu
->sched_flags
= 0;
2382 * Set iowait_boost flag and update time. Since IO WAIT flag
2383 * is set all the time, we can't just conclude that there is
2384 * some IO bound activity is scheduled on this CPU with just
2385 * one occurrence. If we receive at least two in two
2386 * consecutive ticks, then we treat as boost candidate.
2388 if (time_before64(time
, cpu
->last_io_update
+ 2 * TICK_NSEC
))
2391 cpu
->last_io_update
= time
;
2394 intel_pstate_hwp_boost_up(cpu
);
2397 intel_pstate_hwp_boost_down(cpu
);
2401 static inline void intel_pstate_update_util_hwp(struct update_util_data
*data
,
2402 u64 time
, unsigned int flags
)
2404 struct cpudata
*cpu
= container_of(data
, struct cpudata
, update_util
);
2406 cpu
->sched_flags
|= flags
;
2408 if (smp_processor_id() == cpu
->cpu
)
2409 intel_pstate_update_util_hwp_local(cpu
, time
);
2412 static inline void intel_pstate_calc_avg_perf(struct cpudata
*cpu
)
2414 struct sample
*sample
= &cpu
->sample
;
2416 sample
->core_avg_perf
= div_ext_fp(sample
->aperf
, sample
->mperf
);
2419 static inline bool intel_pstate_sample(struct cpudata
*cpu
, u64 time
)
2422 unsigned long flags
;
2425 local_irq_save(flags
);
2426 rdmsrl(MSR_IA32_APERF
, aperf
);
2427 rdmsrl(MSR_IA32_MPERF
, mperf
);
2429 if (cpu
->prev_mperf
== mperf
|| cpu
->prev_tsc
== tsc
) {
2430 local_irq_restore(flags
);
2433 local_irq_restore(flags
);
2435 cpu
->last_sample_time
= cpu
->sample
.time
;
2436 cpu
->sample
.time
= time
;
2437 cpu
->sample
.aperf
= aperf
;
2438 cpu
->sample
.mperf
= mperf
;
2439 cpu
->sample
.tsc
= tsc
;
2440 cpu
->sample
.aperf
-= cpu
->prev_aperf
;
2441 cpu
->sample
.mperf
-= cpu
->prev_mperf
;
2442 cpu
->sample
.tsc
-= cpu
->prev_tsc
;
2444 cpu
->prev_aperf
= aperf
;
2445 cpu
->prev_mperf
= mperf
;
2446 cpu
->prev_tsc
= tsc
;
2448 * First time this function is invoked in a given cycle, all of the
2449 * previous sample data fields are equal to zero or stale and they must
2450 * be populated with meaningful numbers for things to work, so assume
2451 * that sample.time will always be reset before setting the utilization
2452 * update hook and make the caller skip the sample then.
2454 if (cpu
->last_sample_time
) {
2455 intel_pstate_calc_avg_perf(cpu
);
2461 static inline int32_t get_avg_frequency(struct cpudata
*cpu
)
2463 return mul_ext_fp(cpu
->sample
.core_avg_perf
, cpu_khz
);
2466 static inline int32_t get_avg_pstate(struct cpudata
*cpu
)
2468 return mul_ext_fp(cpu
->pstate
.max_pstate_physical
,
2469 cpu
->sample
.core_avg_perf
);
2472 static inline int32_t get_target_pstate(struct cpudata
*cpu
)
2474 struct sample
*sample
= &cpu
->sample
;
2476 int target
, avg_pstate
;
2478 busy_frac
= div_fp(sample
->mperf
<< cpu
->aperf_mperf_shift
,
2481 if (busy_frac
< cpu
->iowait_boost
)
2482 busy_frac
= cpu
->iowait_boost
;
2484 sample
->busy_scaled
= busy_frac
* 100;
2486 target
= READ_ONCE(global
.no_turbo
) ?
2487 cpu
->pstate
.max_pstate
: cpu
->pstate
.turbo_pstate
;
2488 target
+= target
>> 2;
2489 target
= mul_fp(target
, busy_frac
);
2490 if (target
< cpu
->pstate
.min_pstate
)
2491 target
= cpu
->pstate
.min_pstate
;
2494 * If the average P-state during the previous cycle was higher than the
2495 * current target, add 50% of the difference to the target to reduce
2496 * possible performance oscillations and offset possible performance
2497 * loss related to moving the workload from one CPU to another within
2500 avg_pstate
= get_avg_pstate(cpu
);
2501 if (avg_pstate
> target
)
2502 target
+= (avg_pstate
- target
) >> 1;
2507 static int intel_pstate_prepare_request(struct cpudata
*cpu
, int pstate
)
2509 int min_pstate
= max(cpu
->pstate
.min_pstate
, cpu
->min_perf_ratio
);
2510 int max_pstate
= max(min_pstate
, cpu
->max_perf_ratio
);
2512 return clamp_t(int, pstate
, min_pstate
, max_pstate
);
2515 static void intel_pstate_update_pstate(struct cpudata
*cpu
, int pstate
)
2517 if (pstate
== cpu
->pstate
.current_pstate
)
2520 cpu
->pstate
.current_pstate
= pstate
;
2521 wrmsrl(MSR_IA32_PERF_CTL
, pstate_funcs
.get_val(cpu
, pstate
));
2524 static void intel_pstate_adjust_pstate(struct cpudata
*cpu
)
2526 int from
= cpu
->pstate
.current_pstate
;
2527 struct sample
*sample
;
2530 target_pstate
= get_target_pstate(cpu
);
2531 target_pstate
= intel_pstate_prepare_request(cpu
, target_pstate
);
2532 trace_cpu_frequency(target_pstate
* cpu
->pstate
.scaling
, cpu
->cpu
);
2533 intel_pstate_update_pstate(cpu
, target_pstate
);
2535 sample
= &cpu
->sample
;
2536 trace_pstate_sample(mul_ext_fp(100, sample
->core_avg_perf
),
2537 fp_toint(sample
->busy_scaled
),
2539 cpu
->pstate
.current_pstate
,
2543 get_avg_frequency(cpu
),
2544 fp_toint(cpu
->iowait_boost
* 100));
2547 static void intel_pstate_update_util(struct update_util_data
*data
, u64 time
,
2550 struct cpudata
*cpu
= container_of(data
, struct cpudata
, update_util
);
2553 /* Don't allow remote callbacks */
2554 if (smp_processor_id() != cpu
->cpu
)
2557 delta_ns
= time
- cpu
->last_update
;
2558 if (flags
& SCHED_CPUFREQ_IOWAIT
) {
2559 /* Start over if the CPU may have been idle. */
2560 if (delta_ns
> TICK_NSEC
) {
2561 cpu
->iowait_boost
= ONE_EIGHTH_FP
;
2562 } else if (cpu
->iowait_boost
>= ONE_EIGHTH_FP
) {
2563 cpu
->iowait_boost
<<= 1;
2564 if (cpu
->iowait_boost
> int_tofp(1))
2565 cpu
->iowait_boost
= int_tofp(1);
2567 cpu
->iowait_boost
= ONE_EIGHTH_FP
;
2569 } else if (cpu
->iowait_boost
) {
2570 /* Clear iowait_boost if the CPU may have been idle. */
2571 if (delta_ns
> TICK_NSEC
)
2572 cpu
->iowait_boost
= 0;
2574 cpu
->iowait_boost
>>= 1;
2576 cpu
->last_update
= time
;
2577 delta_ns
= time
- cpu
->sample
.time
;
2578 if ((s64
)delta_ns
< INTEL_PSTATE_SAMPLING_INTERVAL
)
2581 if (intel_pstate_sample(cpu
, time
))
2582 intel_pstate_adjust_pstate(cpu
);
2585 static struct pstate_funcs core_funcs
= {
2586 .get_max
= core_get_max_pstate
,
2587 .get_max_physical
= core_get_max_pstate_physical
,
2588 .get_min
= core_get_min_pstate
,
2589 .get_turbo
= core_get_turbo_pstate
,
2590 .get_scaling
= core_get_scaling
,
2591 .get_val
= core_get_val
,
2594 static const struct pstate_funcs silvermont_funcs
= {
2595 .get_max
= atom_get_max_pstate
,
2596 .get_max_physical
= atom_get_max_pstate
,
2597 .get_min
= atom_get_min_pstate
,
2598 .get_turbo
= atom_get_turbo_pstate
,
2599 .get_val
= atom_get_val
,
2600 .get_scaling
= silvermont_get_scaling
,
2601 .get_vid
= atom_get_vid
,
2604 static const struct pstate_funcs airmont_funcs
= {
2605 .get_max
= atom_get_max_pstate
,
2606 .get_max_physical
= atom_get_max_pstate
,
2607 .get_min
= atom_get_min_pstate
,
2608 .get_turbo
= atom_get_turbo_pstate
,
2609 .get_val
= atom_get_val
,
2610 .get_scaling
= airmont_get_scaling
,
2611 .get_vid
= atom_get_vid
,
2614 static const struct pstate_funcs knl_funcs
= {
2615 .get_max
= core_get_max_pstate
,
2616 .get_max_physical
= core_get_max_pstate_physical
,
2617 .get_min
= core_get_min_pstate
,
2618 .get_turbo
= knl_get_turbo_pstate
,
2619 .get_aperf_mperf_shift
= knl_get_aperf_mperf_shift
,
2620 .get_scaling
= core_get_scaling
,
2621 .get_val
= core_get_val
,
2624 #define X86_MATCH(vfm, policy) \
2625 X86_MATCH_VFM_FEATURE(vfm, X86_FEATURE_APERFMPERF, &policy)
2627 static const struct x86_cpu_id intel_pstate_cpu_ids
[] = {
2628 X86_MATCH(INTEL_SANDYBRIDGE
, core_funcs
),
2629 X86_MATCH(INTEL_SANDYBRIDGE_X
, core_funcs
),
2630 X86_MATCH(INTEL_ATOM_SILVERMONT
, silvermont_funcs
),
2631 X86_MATCH(INTEL_IVYBRIDGE
, core_funcs
),
2632 X86_MATCH(INTEL_HASWELL
, core_funcs
),
2633 X86_MATCH(INTEL_BROADWELL
, core_funcs
),
2634 X86_MATCH(INTEL_IVYBRIDGE_X
, core_funcs
),
2635 X86_MATCH(INTEL_HASWELL_X
, core_funcs
),
2636 X86_MATCH(INTEL_HASWELL_L
, core_funcs
),
2637 X86_MATCH(INTEL_HASWELL_G
, core_funcs
),
2638 X86_MATCH(INTEL_BROADWELL_G
, core_funcs
),
2639 X86_MATCH(INTEL_ATOM_AIRMONT
, airmont_funcs
),
2640 X86_MATCH(INTEL_SKYLAKE_L
, core_funcs
),
2641 X86_MATCH(INTEL_BROADWELL_X
, core_funcs
),
2642 X86_MATCH(INTEL_SKYLAKE
, core_funcs
),
2643 X86_MATCH(INTEL_BROADWELL_D
, core_funcs
),
2644 X86_MATCH(INTEL_XEON_PHI_KNL
, knl_funcs
),
2645 X86_MATCH(INTEL_XEON_PHI_KNM
, knl_funcs
),
2646 X86_MATCH(INTEL_ATOM_GOLDMONT
, core_funcs
),
2647 X86_MATCH(INTEL_ATOM_GOLDMONT_PLUS
, core_funcs
),
2648 X86_MATCH(INTEL_SKYLAKE_X
, core_funcs
),
2649 X86_MATCH(INTEL_COMETLAKE
, core_funcs
),
2650 X86_MATCH(INTEL_ICELAKE_X
, core_funcs
),
2651 X86_MATCH(INTEL_TIGERLAKE
, core_funcs
),
2652 X86_MATCH(INTEL_SAPPHIRERAPIDS_X
, core_funcs
),
2653 X86_MATCH(INTEL_EMERALDRAPIDS_X
, core_funcs
),
2656 MODULE_DEVICE_TABLE(x86cpu
, intel_pstate_cpu_ids
);
2659 static const struct x86_cpu_id intel_pstate_cpu_oob_ids
[] __initconst
= {
2660 X86_MATCH(INTEL_BROADWELL_D
, core_funcs
),
2661 X86_MATCH(INTEL_BROADWELL_X
, core_funcs
),
2662 X86_MATCH(INTEL_SKYLAKE_X
, core_funcs
),
2663 X86_MATCH(INTEL_ICELAKE_X
, core_funcs
),
2664 X86_MATCH(INTEL_SAPPHIRERAPIDS_X
, core_funcs
),
2665 X86_MATCH(INTEL_EMERALDRAPIDS_X
, core_funcs
),
2666 X86_MATCH(INTEL_GRANITERAPIDS_D
, core_funcs
),
2667 X86_MATCH(INTEL_GRANITERAPIDS_X
, core_funcs
),
2668 X86_MATCH(INTEL_ATOM_CRESTMONT
, core_funcs
),
2669 X86_MATCH(INTEL_ATOM_CRESTMONT_X
, core_funcs
),
2674 static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids
[] = {
2675 X86_MATCH(INTEL_KABYLAKE
, core_funcs
),
2679 static int intel_pstate_init_cpu(unsigned int cpunum
)
2681 struct cpudata
*cpu
;
2683 cpu
= all_cpu_data
[cpunum
];
2686 cpu
= kzalloc(sizeof(*cpu
), GFP_KERNEL
);
2690 WRITE_ONCE(all_cpu_data
[cpunum
], cpu
);
2694 cpu
->epp_default
= -EINVAL
;
2697 intel_pstate_hwp_enable(cpu
);
2699 if (intel_pstate_acpi_pm_profile_server())
2702 } else if (hwp_active
) {
2704 * Re-enable HWP in case this happens after a resume from ACPI
2705 * S3 if the CPU was offline during the whole system/resume
2708 intel_pstate_hwp_reenable(cpu
);
2711 cpu
->epp_powersave
= -EINVAL
;
2712 cpu
->epp_policy
= 0;
2714 intel_pstate_get_cpu_pstates(cpu
);
2716 pr_debug("controlling: cpu %d\n", cpunum
);
2721 static void intel_pstate_set_update_util_hook(unsigned int cpu_num
)
2723 struct cpudata
*cpu
= all_cpu_data
[cpu_num
];
2725 if (hwp_active
&& !hwp_boost
)
2728 if (cpu
->update_util_set
)
2731 /* Prevent intel_pstate_update_util() from using stale data. */
2732 cpu
->sample
.time
= 0;
2733 cpufreq_add_update_util_hook(cpu_num
, &cpu
->update_util
,
2735 intel_pstate_update_util_hwp
:
2736 intel_pstate_update_util
));
2737 cpu
->update_util_set
= true;
2740 static void intel_pstate_clear_update_util_hook(unsigned int cpu
)
2742 struct cpudata
*cpu_data
= all_cpu_data
[cpu
];
2744 if (!cpu_data
->update_util_set
)
2747 cpufreq_remove_update_util_hook(cpu
);
2748 cpu_data
->update_util_set
= false;
2752 static int intel_pstate_get_max_freq(struct cpudata
*cpu
)
2754 return READ_ONCE(global
.no_turbo
) ?
2755 cpu
->pstate
.max_freq
: cpu
->pstate
.turbo_freq
;
2758 static void intel_pstate_update_perf_limits(struct cpudata
*cpu
,
2759 unsigned int policy_min
,
2760 unsigned int policy_max
)
2762 int perf_ctl_scaling
= cpu
->pstate
.perf_ctl_scaling
;
2763 int32_t max_policy_perf
, min_policy_perf
;
2765 max_policy_perf
= policy_max
/ perf_ctl_scaling
;
2766 if (policy_max
== policy_min
) {
2767 min_policy_perf
= max_policy_perf
;
2769 min_policy_perf
= policy_min
/ perf_ctl_scaling
;
2770 min_policy_perf
= clamp_t(int32_t, min_policy_perf
,
2771 0, max_policy_perf
);
2775 * HWP needs some special consideration, because HWP_REQUEST uses
2776 * abstract values to represent performance rather than pure ratios.
2778 if (hwp_active
&& cpu
->pstate
.scaling
!= perf_ctl_scaling
) {
2781 freq
= max_policy_perf
* perf_ctl_scaling
;
2782 max_policy_perf
= intel_pstate_freq_to_hwp(cpu
, freq
);
2783 freq
= min_policy_perf
* perf_ctl_scaling
;
2784 min_policy_perf
= intel_pstate_freq_to_hwp(cpu
, freq
);
2787 pr_debug("cpu:%d min_policy_perf:%d max_policy_perf:%d\n",
2788 cpu
->cpu
, min_policy_perf
, max_policy_perf
);
2790 /* Normalize user input to [min_perf, max_perf] */
2791 if (per_cpu_limits
) {
2792 cpu
->min_perf_ratio
= min_policy_perf
;
2793 cpu
->max_perf_ratio
= max_policy_perf
;
2795 int turbo_max
= cpu
->pstate
.turbo_pstate
;
2796 int32_t global_min
, global_max
;
2798 /* Global limits are in percent of the maximum turbo P-state. */
2799 global_max
= DIV_ROUND_UP(turbo_max
* global
.max_perf_pct
, 100);
2800 global_min
= DIV_ROUND_UP(turbo_max
* global
.min_perf_pct
, 100);
2801 global_min
= clamp_t(int32_t, global_min
, 0, global_max
);
2803 pr_debug("cpu:%d global_min:%d global_max:%d\n", cpu
->cpu
,
2804 global_min
, global_max
);
2806 cpu
->min_perf_ratio
= max(min_policy_perf
, global_min
);
2807 cpu
->min_perf_ratio
= min(cpu
->min_perf_ratio
, max_policy_perf
);
2808 cpu
->max_perf_ratio
= min(max_policy_perf
, global_max
);
2809 cpu
->max_perf_ratio
= max(min_policy_perf
, cpu
->max_perf_ratio
);
2811 /* Make sure min_perf <= max_perf */
2812 cpu
->min_perf_ratio
= min(cpu
->min_perf_ratio
,
2813 cpu
->max_perf_ratio
);
2816 pr_debug("cpu:%d max_perf_ratio:%d min_perf_ratio:%d\n", cpu
->cpu
,
2817 cpu
->max_perf_ratio
,
2818 cpu
->min_perf_ratio
);
2821 static int intel_pstate_set_policy(struct cpufreq_policy
*policy
)
2823 struct cpudata
*cpu
;
2825 if (!policy
->cpuinfo
.max_freq
)
2828 pr_debug("set_policy cpuinfo.max %u policy->max %u\n",
2829 policy
->cpuinfo
.max_freq
, policy
->max
);
2831 cpu
= all_cpu_data
[policy
->cpu
];
2832 cpu
->policy
= policy
->policy
;
2834 mutex_lock(&intel_pstate_limits_lock
);
2836 intel_pstate_update_perf_limits(cpu
, policy
->min
, policy
->max
);
2838 if (cpu
->policy
== CPUFREQ_POLICY_PERFORMANCE
) {
2839 int pstate
= max(cpu
->pstate
.min_pstate
, cpu
->max_perf_ratio
);
2842 * NOHZ_FULL CPUs need this as the governor callback may not
2843 * be invoked on them.
2845 intel_pstate_clear_update_util_hook(policy
->cpu
);
2846 intel_pstate_set_pstate(cpu
, pstate
);
2848 intel_pstate_set_update_util_hook(policy
->cpu
);
2853 * When hwp_boost was active before and dynamically it
2854 * was turned off, in that case we need to clear the
2858 intel_pstate_clear_update_util_hook(policy
->cpu
);
2859 intel_pstate_hwp_set(policy
->cpu
);
2862 * policy->cur is never updated with the intel_pstate driver, but it
2863 * is used as a stale frequency value. So, keep it within limits.
2865 policy
->cur
= policy
->min
;
2867 mutex_unlock(&intel_pstate_limits_lock
);
2872 static void intel_pstate_adjust_policy_max(struct cpudata
*cpu
,
2873 struct cpufreq_policy_data
*policy
)
2876 cpu
->pstate
.max_pstate_physical
> cpu
->pstate
.max_pstate
&&
2877 policy
->max
< policy
->cpuinfo
.max_freq
&&
2878 policy
->max
> cpu
->pstate
.max_freq
) {
2879 pr_debug("policy->max > max non turbo frequency\n");
2880 policy
->max
= policy
->cpuinfo
.max_freq
;
2884 static void intel_pstate_verify_cpu_policy(struct cpudata
*cpu
,
2885 struct cpufreq_policy_data
*policy
)
2890 intel_pstate_get_hwp_cap(cpu
);
2891 max_freq
= READ_ONCE(global
.no_turbo
) ?
2892 cpu
->pstate
.max_freq
: cpu
->pstate
.turbo_freq
;
2894 max_freq
= intel_pstate_get_max_freq(cpu
);
2896 cpufreq_verify_within_limits(policy
, policy
->cpuinfo
.min_freq
, max_freq
);
2898 intel_pstate_adjust_policy_max(cpu
, policy
);
2901 static int intel_pstate_verify_policy(struct cpufreq_policy_data
*policy
)
2903 intel_pstate_verify_cpu_policy(all_cpu_data
[policy
->cpu
], policy
);
2908 static int intel_cpufreq_cpu_offline(struct cpufreq_policy
*policy
)
2910 struct cpudata
*cpu
= all_cpu_data
[policy
->cpu
];
2912 pr_debug("CPU %d going offline\n", cpu
->cpu
);
2918 * If the CPU is an SMT thread and it goes offline with the performance
2919 * settings different from the minimum, it will prevent its sibling
2920 * from getting to lower performance levels, so force the minimum
2921 * performance on CPU offline to prevent that from happening.
2924 intel_pstate_hwp_offline(cpu
);
2926 intel_pstate_set_min_pstate(cpu
);
2928 intel_pstate_exit_perf_limits(policy
);
2933 static int intel_pstate_cpu_online(struct cpufreq_policy
*policy
)
2935 struct cpudata
*cpu
= all_cpu_data
[policy
->cpu
];
2937 pr_debug("CPU %d going online\n", cpu
->cpu
);
2939 intel_pstate_init_acpi_perf_limits(policy
);
2943 * Re-enable HWP and clear the "suspended" flag to let "resume"
2944 * know that it need not do that.
2946 intel_pstate_hwp_reenable(cpu
);
2947 cpu
->suspended
= false;
2949 hybrid_update_capacity(cpu
);
2955 static int intel_pstate_cpu_offline(struct cpufreq_policy
*policy
)
2957 intel_pstate_clear_update_util_hook(policy
->cpu
);
2959 return intel_cpufreq_cpu_offline(policy
);
2962 static void intel_pstate_cpu_exit(struct cpufreq_policy
*policy
)
2964 pr_debug("CPU %d exiting\n", policy
->cpu
);
2966 policy
->fast_switch_possible
= false;
2969 static int __intel_pstate_cpu_init(struct cpufreq_policy
*policy
)
2971 struct cpudata
*cpu
;
2974 rc
= intel_pstate_init_cpu(policy
->cpu
);
2978 cpu
= all_cpu_data
[policy
->cpu
];
2980 cpu
->max_perf_ratio
= 0xFF;
2981 cpu
->min_perf_ratio
= 0;
2983 /* cpuinfo and default policy values */
2984 policy
->cpuinfo
.min_freq
= cpu
->pstate
.min_freq
;
2985 policy
->cpuinfo
.max_freq
= READ_ONCE(global
.no_turbo
) ?
2986 cpu
->pstate
.max_freq
: cpu
->pstate
.turbo_freq
;
2988 policy
->min
= policy
->cpuinfo
.min_freq
;
2989 policy
->max
= policy
->cpuinfo
.max_freq
;
2991 intel_pstate_init_acpi_perf_limits(policy
);
2993 policy
->fast_switch_possible
= true;
2998 static int intel_pstate_cpu_init(struct cpufreq_policy
*policy
)
3000 int ret
= __intel_pstate_cpu_init(policy
);
3006 * Set the policy to powersave to provide a valid fallback value in case
3007 * the default cpufreq governor is neither powersave nor performance.
3009 policy
->policy
= CPUFREQ_POLICY_POWERSAVE
;
3012 struct cpudata
*cpu
= all_cpu_data
[policy
->cpu
];
3014 cpu
->epp_cached
= intel_pstate_get_epp(cpu
, 0);
3020 static struct cpufreq_driver intel_pstate
= {
3021 .flags
= CPUFREQ_CONST_LOOPS
,
3022 .verify
= intel_pstate_verify_policy
,
3023 .setpolicy
= intel_pstate_set_policy
,
3024 .suspend
= intel_pstate_suspend
,
3025 .resume
= intel_pstate_resume
,
3026 .init
= intel_pstate_cpu_init
,
3027 .exit
= intel_pstate_cpu_exit
,
3028 .offline
= intel_pstate_cpu_offline
,
3029 .online
= intel_pstate_cpu_online
,
3030 .update_limits
= intel_pstate_update_limits
,
3031 .name
= "intel_pstate",
3034 static int intel_cpufreq_verify_policy(struct cpufreq_policy_data
*policy
)
3036 struct cpudata
*cpu
= all_cpu_data
[policy
->cpu
];
3038 intel_pstate_verify_cpu_policy(cpu
, policy
);
3039 intel_pstate_update_perf_limits(cpu
, policy
->min
, policy
->max
);
3044 /* Use of trace in passive mode:
3046 * In passive mode the trace core_busy field (also known as the
3047 * performance field, and lablelled as such on the graphs; also known as
3048 * core_avg_perf) is not needed and so is re-assigned to indicate if the
3049 * driver call was via the normal or fast switch path. Various graphs
3050 * output from the intel_pstate_tracer.py utility that include core_busy
3051 * (or performance or core_avg_perf) have a fixed y-axis from 0 to 100%,
3052 * so we use 10 to indicate the normal path through the driver, and
3053 * 90 to indicate the fast switch path through the driver.
3054 * The scaled_busy field is not used, and is set to 0.
3057 #define INTEL_PSTATE_TRACE_TARGET 10
3058 #define INTEL_PSTATE_TRACE_FAST_SWITCH 90
3060 static void intel_cpufreq_trace(struct cpudata
*cpu
, unsigned int trace_type
, int old_pstate
)
3062 struct sample
*sample
;
3064 if (!trace_pstate_sample_enabled())
3067 if (!intel_pstate_sample(cpu
, ktime_get()))
3070 sample
= &cpu
->sample
;
3071 trace_pstate_sample(trace_type
,
3074 cpu
->pstate
.current_pstate
,
3078 get_avg_frequency(cpu
),
3079 fp_toint(cpu
->iowait_boost
* 100));
3082 static void intel_cpufreq_hwp_update(struct cpudata
*cpu
, u32 min
, u32 max
,
3083 u32 desired
, bool fast_switch
)
3085 u64 prev
= READ_ONCE(cpu
->hwp_req_cached
), value
= prev
;
3087 value
&= ~HWP_MIN_PERF(~0L);
3088 value
|= HWP_MIN_PERF(min
);
3090 value
&= ~HWP_MAX_PERF(~0L);
3091 value
|= HWP_MAX_PERF(max
);
3093 value
&= ~HWP_DESIRED_PERF(~0L);
3094 value
|= HWP_DESIRED_PERF(desired
);
3099 WRITE_ONCE(cpu
->hwp_req_cached
, value
);
3101 wrmsrl(MSR_HWP_REQUEST
, value
);
3103 wrmsrl_on_cpu(cpu
->cpu
, MSR_HWP_REQUEST
, value
);
3106 static void intel_cpufreq_perf_ctl_update(struct cpudata
*cpu
,
3107 u32 target_pstate
, bool fast_switch
)
3110 wrmsrl(MSR_IA32_PERF_CTL
,
3111 pstate_funcs
.get_val(cpu
, target_pstate
));
3113 wrmsrl_on_cpu(cpu
->cpu
, MSR_IA32_PERF_CTL
,
3114 pstate_funcs
.get_val(cpu
, target_pstate
));
3117 static int intel_cpufreq_update_pstate(struct cpufreq_policy
*policy
,
3118 int target_pstate
, bool fast_switch
)
3120 struct cpudata
*cpu
= all_cpu_data
[policy
->cpu
];
3121 int old_pstate
= cpu
->pstate
.current_pstate
;
3123 target_pstate
= intel_pstate_prepare_request(cpu
, target_pstate
);
3125 int max_pstate
= policy
->strict_target
?
3126 target_pstate
: cpu
->max_perf_ratio
;
3128 intel_cpufreq_hwp_update(cpu
, target_pstate
, max_pstate
, 0,
3130 } else if (target_pstate
!= old_pstate
) {
3131 intel_cpufreq_perf_ctl_update(cpu
, target_pstate
, fast_switch
);
3134 cpu
->pstate
.current_pstate
= target_pstate
;
3136 intel_cpufreq_trace(cpu
, fast_switch
? INTEL_PSTATE_TRACE_FAST_SWITCH
:
3137 INTEL_PSTATE_TRACE_TARGET
, old_pstate
);
3139 return target_pstate
;
3142 static int intel_cpufreq_target(struct cpufreq_policy
*policy
,
3143 unsigned int target_freq
,
3144 unsigned int relation
)
3146 struct cpudata
*cpu
= all_cpu_data
[policy
->cpu
];
3147 struct cpufreq_freqs freqs
;
3150 freqs
.old
= policy
->cur
;
3151 freqs
.new = target_freq
;
3153 cpufreq_freq_transition_begin(policy
, &freqs
);
3155 target_pstate
= intel_pstate_freq_to_hwp_rel(cpu
, freqs
.new, relation
);
3156 target_pstate
= intel_cpufreq_update_pstate(policy
, target_pstate
, false);
3158 freqs
.new = target_pstate
* cpu
->pstate
.scaling
;
3160 cpufreq_freq_transition_end(policy
, &freqs
, false);
3165 static unsigned int intel_cpufreq_fast_switch(struct cpufreq_policy
*policy
,
3166 unsigned int target_freq
)
3168 struct cpudata
*cpu
= all_cpu_data
[policy
->cpu
];
3171 target_pstate
= intel_pstate_freq_to_hwp(cpu
, target_freq
);
3173 target_pstate
= intel_cpufreq_update_pstate(policy
, target_pstate
, true);
3175 return target_pstate
* cpu
->pstate
.scaling
;
3178 static void intel_cpufreq_adjust_perf(unsigned int cpunum
,
3179 unsigned long min_perf
,
3180 unsigned long target_perf
,
3181 unsigned long capacity
)
3183 struct cpudata
*cpu
= all_cpu_data
[cpunum
];
3184 u64 hwp_cap
= READ_ONCE(cpu
->hwp_cap_cached
);
3185 int old_pstate
= cpu
->pstate
.current_pstate
;
3186 int cap_pstate
, min_pstate
, max_pstate
, target_pstate
;
3188 cap_pstate
= READ_ONCE(global
.no_turbo
) ?
3189 HWP_GUARANTEED_PERF(hwp_cap
) :
3190 HWP_HIGHEST_PERF(hwp_cap
);
3192 /* Optimization: Avoid unnecessary divisions. */
3194 target_pstate
= cap_pstate
;
3195 if (target_perf
< capacity
)
3196 target_pstate
= DIV_ROUND_UP(cap_pstate
* target_perf
, capacity
);
3198 min_pstate
= cap_pstate
;
3199 if (min_perf
< capacity
)
3200 min_pstate
= DIV_ROUND_UP(cap_pstate
* min_perf
, capacity
);
3202 if (min_pstate
< cpu
->pstate
.min_pstate
)
3203 min_pstate
= cpu
->pstate
.min_pstate
;
3205 if (min_pstate
< cpu
->min_perf_ratio
)
3206 min_pstate
= cpu
->min_perf_ratio
;
3208 if (min_pstate
> cpu
->max_perf_ratio
)
3209 min_pstate
= cpu
->max_perf_ratio
;
3211 max_pstate
= min(cap_pstate
, cpu
->max_perf_ratio
);
3212 if (max_pstate
< min_pstate
)
3213 max_pstate
= min_pstate
;
3215 target_pstate
= clamp_t(int, target_pstate
, min_pstate
, max_pstate
);
3217 intel_cpufreq_hwp_update(cpu
, min_pstate
, max_pstate
, target_pstate
, true);
3219 cpu
->pstate
.current_pstate
= target_pstate
;
3220 intel_cpufreq_trace(cpu
, INTEL_PSTATE_TRACE_FAST_SWITCH
, old_pstate
);
3223 static int intel_cpufreq_cpu_init(struct cpufreq_policy
*policy
)
3225 struct freq_qos_request
*req
;
3226 struct cpudata
*cpu
;
3230 dev
= get_cpu_device(policy
->cpu
);
3234 ret
= __intel_pstate_cpu_init(policy
);
3238 policy
->cpuinfo
.transition_latency
= INTEL_CPUFREQ_TRANSITION_LATENCY
;
3239 /* This reflects the intel_pstate_get_cpu_pstates() setting. */
3240 policy
->cur
= policy
->cpuinfo
.min_freq
;
3242 req
= kcalloc(2, sizeof(*req
), GFP_KERNEL
);
3248 cpu
= all_cpu_data
[policy
->cpu
];
3253 policy
->transition_delay_us
= INTEL_CPUFREQ_TRANSITION_DELAY_HWP
;
3255 intel_pstate_get_hwp_cap(cpu
);
3257 rdmsrl_on_cpu(cpu
->cpu
, MSR_HWP_REQUEST
, &value
);
3258 WRITE_ONCE(cpu
->hwp_req_cached
, value
);
3260 cpu
->epp_cached
= intel_pstate_get_epp(cpu
, value
);
3262 policy
->transition_delay_us
= INTEL_CPUFREQ_TRANSITION_DELAY
;
3265 freq
= DIV_ROUND_UP(cpu
->pstate
.turbo_freq
* global
.min_perf_pct
, 100);
3267 ret
= freq_qos_add_request(&policy
->constraints
, req
, FREQ_QOS_MIN
,
3270 dev_err(dev
, "Failed to add min-freq constraint (%d)\n", ret
);
3274 freq
= DIV_ROUND_UP(cpu
->pstate
.turbo_freq
* global
.max_perf_pct
, 100);
3276 ret
= freq_qos_add_request(&policy
->constraints
, req
+ 1, FREQ_QOS_MAX
,
3279 dev_err(dev
, "Failed to add max-freq constraint (%d)\n", ret
);
3280 goto remove_min_req
;
3283 policy
->driver_data
= req
;
3288 freq_qos_remove_request(req
);
3292 intel_pstate_exit_perf_limits(policy
);
3297 static void intel_cpufreq_cpu_exit(struct cpufreq_policy
*policy
)
3299 struct freq_qos_request
*req
;
3301 req
= policy
->driver_data
;
3303 freq_qos_remove_request(req
+ 1);
3304 freq_qos_remove_request(req
);
3307 intel_pstate_cpu_exit(policy
);
3310 static int intel_cpufreq_suspend(struct cpufreq_policy
*policy
)
3312 intel_pstate_suspend(policy
);
3315 struct cpudata
*cpu
= all_cpu_data
[policy
->cpu
];
3316 u64 value
= READ_ONCE(cpu
->hwp_req_cached
);
3319 * Clear the desired perf field in MSR_HWP_REQUEST in case
3320 * intel_cpufreq_adjust_perf() is in use and the last value
3321 * written by it may not be suitable.
3323 value
&= ~HWP_DESIRED_PERF(~0L);
3324 wrmsrl_on_cpu(cpu
->cpu
, MSR_HWP_REQUEST
, value
);
3325 WRITE_ONCE(cpu
->hwp_req_cached
, value
);
3331 static struct cpufreq_driver intel_cpufreq
= {
3332 .flags
= CPUFREQ_CONST_LOOPS
,
3333 .verify
= intel_cpufreq_verify_policy
,
3334 .target
= intel_cpufreq_target
,
3335 .fast_switch
= intel_cpufreq_fast_switch
,
3336 .init
= intel_cpufreq_cpu_init
,
3337 .exit
= intel_cpufreq_cpu_exit
,
3338 .offline
= intel_cpufreq_cpu_offline
,
3339 .online
= intel_pstate_cpu_online
,
3340 .suspend
= intel_cpufreq_suspend
,
3341 .resume
= intel_pstate_resume
,
3342 .update_limits
= intel_pstate_update_limits
,
3343 .name
= "intel_cpufreq",
3346 static struct cpufreq_driver
*default_driver
;
3348 static void intel_pstate_driver_cleanup(void)
3353 for_each_online_cpu(cpu
) {
3354 if (all_cpu_data
[cpu
]) {
3355 if (intel_pstate_driver
== &intel_pstate
)
3356 intel_pstate_clear_update_util_hook(cpu
);
3358 kfree(all_cpu_data
[cpu
]);
3359 WRITE_ONCE(all_cpu_data
[cpu
], NULL
);
3364 intel_pstate_driver
= NULL
;
3367 static int intel_pstate_register_driver(struct cpufreq_driver
*driver
)
3369 bool refresh_cpu_cap_scaling
;
3372 if (driver
== &intel_pstate
)
3373 intel_pstate_sysfs_expose_hwp_dynamic_boost();
3375 memset(&global
, 0, sizeof(global
));
3376 global
.max_perf_pct
= 100;
3377 global
.turbo_disabled
= turbo_is_disabled();
3378 global
.no_turbo
= global
.turbo_disabled
;
3380 arch_set_max_freq_ratio(global
.turbo_disabled
);
3382 refresh_cpu_cap_scaling
= hybrid_clear_max_perf_cpu();
3384 intel_pstate_driver
= driver
;
3385 ret
= cpufreq_register_driver(intel_pstate_driver
);
3387 intel_pstate_driver_cleanup();
3391 global
.min_perf_pct
= min_perf_pct_min();
3393 hybrid_init_cpu_capacity_scaling(refresh_cpu_cap_scaling
);
3398 static ssize_t
intel_pstate_show_status(char *buf
)
3400 if (!intel_pstate_driver
)
3401 return sprintf(buf
, "off\n");
3403 return sprintf(buf
, "%s\n", intel_pstate_driver
== &intel_pstate
?
3404 "active" : "passive");
3407 static int intel_pstate_update_status(const char *buf
, size_t size
)
3409 if (size
== 3 && !strncmp(buf
, "off", size
)) {
3410 if (!intel_pstate_driver
)
3416 cpufreq_unregister_driver(intel_pstate_driver
);
3417 intel_pstate_driver_cleanup();
3421 if (size
== 6 && !strncmp(buf
, "active", size
)) {
3422 if (intel_pstate_driver
) {
3423 if (intel_pstate_driver
== &intel_pstate
)
3426 cpufreq_unregister_driver(intel_pstate_driver
);
3429 return intel_pstate_register_driver(&intel_pstate
);
3432 if (size
== 7 && !strncmp(buf
, "passive", size
)) {
3433 if (intel_pstate_driver
) {
3434 if (intel_pstate_driver
== &intel_cpufreq
)
3437 cpufreq_unregister_driver(intel_pstate_driver
);
3438 intel_pstate_sysfs_hide_hwp_dynamic_boost();
3441 return intel_pstate_register_driver(&intel_cpufreq
);
3447 static int no_load __initdata
;
3448 static int no_hwp __initdata
;
3449 static int hwp_only __initdata
;
3450 static unsigned int force_load __initdata
;
3452 static int __init
intel_pstate_msrs_not_valid(void)
3454 if (!pstate_funcs
.get_max(0) ||
3455 !pstate_funcs
.get_min(0) ||
3456 !pstate_funcs
.get_turbo(0))
3462 static void __init
copy_cpu_funcs(struct pstate_funcs
*funcs
)
3464 pstate_funcs
.get_max
= funcs
->get_max
;
3465 pstate_funcs
.get_max_physical
= funcs
->get_max_physical
;
3466 pstate_funcs
.get_min
= funcs
->get_min
;
3467 pstate_funcs
.get_turbo
= funcs
->get_turbo
;
3468 pstate_funcs
.get_scaling
= funcs
->get_scaling
;
3469 pstate_funcs
.get_val
= funcs
->get_val
;
3470 pstate_funcs
.get_vid
= funcs
->get_vid
;
3471 pstate_funcs
.get_aperf_mperf_shift
= funcs
->get_aperf_mperf_shift
;
3476 static bool __init
intel_pstate_no_acpi_pss(void)
3480 for_each_possible_cpu(i
) {
3482 union acpi_object
*pss
;
3483 struct acpi_buffer buffer
= { ACPI_ALLOCATE_BUFFER
, NULL
};
3484 struct acpi_processor
*pr
= per_cpu(processors
, i
);
3489 status
= acpi_evaluate_object(pr
->handle
, "_PSS", NULL
, &buffer
);
3490 if (ACPI_FAILURE(status
))
3493 pss
= buffer
.pointer
;
3494 if (pss
&& pss
->type
== ACPI_TYPE_PACKAGE
) {
3502 pr_debug("ACPI _PSS not found\n");
3506 static bool __init
intel_pstate_no_acpi_pcch(void)
3511 status
= acpi_get_handle(NULL
, "\\_SB", &handle
);
3512 if (ACPI_FAILURE(status
))
3515 if (acpi_has_method(handle
, "PCCH"))
3519 pr_debug("ACPI PCCH not found\n");
3523 static bool __init
intel_pstate_has_acpi_ppc(void)
3527 for_each_possible_cpu(i
) {
3528 struct acpi_processor
*pr
= per_cpu(processors
, i
);
3532 if (acpi_has_method(pr
->handle
, "_PPC"))
3535 pr_debug("ACPI _PPC not found\n");
3544 /* Hardware vendor-specific info that has its own power management modes */
3545 static struct acpi_platform_list plat_info
[] __initdata
= {
3546 {"HP ", "ProLiant", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PSS
},
3547 {"ORACLE", "X4-2 ", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
3548 {"ORACLE", "X4-2L ", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
3549 {"ORACLE", "X4-2B ", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
3550 {"ORACLE", "X3-2 ", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
3551 {"ORACLE", "X3-2L ", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
3552 {"ORACLE", "X3-2B ", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
3553 {"ORACLE", "X4470M2 ", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
3554 {"ORACLE", "X4270M3 ", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
3555 {"ORACLE", "X4270M2 ", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
3556 {"ORACLE", "X4170M2 ", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
3557 {"ORACLE", "X4170 M3", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
3558 {"ORACLE", "X4275 M3", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
3559 {"ORACLE", "X6-2 ", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
3560 {"ORACLE", "Sudbury ", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
3564 #define BITMASK_OOB (BIT(8) | BIT(18))
3566 static bool __init
intel_pstate_platform_pwr_mgmt_exists(void)
3568 const struct x86_cpu_id
*id
;
3572 id
= x86_match_cpu(intel_pstate_cpu_oob_ids
);
3574 rdmsrl(MSR_MISC_PWR_MGMT
, misc_pwr
);
3575 if (misc_pwr
& BITMASK_OOB
) {
3576 pr_debug("Bit 8 or 18 in the MISC_PWR_MGMT MSR set\n");
3577 pr_debug("P states are controlled in Out of Band mode by the firmware/hardware\n");
3582 idx
= acpi_match_platform_list(plat_info
);
3586 switch (plat_info
[idx
].data
) {
3588 if (!intel_pstate_no_acpi_pss())
3591 return intel_pstate_no_acpi_pcch();
3593 return intel_pstate_has_acpi_ppc() && !force_load
;
3599 static void intel_pstate_request_control_from_smm(void)
3602 * It may be unsafe to request P-states control from SMM if _PPC support
3603 * has not been enabled.
3606 acpi_processor_pstate_control();
3608 #else /* CONFIG_ACPI not enabled */
3609 static inline bool intel_pstate_platform_pwr_mgmt_exists(void) { return false; }
3610 static inline bool intel_pstate_has_acpi_ppc(void) { return false; }
3611 static inline void intel_pstate_request_control_from_smm(void) {}
3612 #endif /* CONFIG_ACPI */
3614 #define INTEL_PSTATE_HWP_BROADWELL 0x01
3616 #define X86_MATCH_HWP(vfm, hwp_mode) \
3617 X86_MATCH_VFM_FEATURE(vfm, X86_FEATURE_HWP, hwp_mode)
3619 static const struct x86_cpu_id hwp_support_ids
[] __initconst
= {
3620 X86_MATCH_HWP(INTEL_BROADWELL_X
, INTEL_PSTATE_HWP_BROADWELL
),
3621 X86_MATCH_HWP(INTEL_BROADWELL_D
, INTEL_PSTATE_HWP_BROADWELL
),
3622 X86_MATCH_HWP(INTEL_ANY
, 0),
3626 static bool intel_pstate_hwp_is_enabled(void)
3630 rdmsrl(MSR_PM_ENABLE
, value
);
3631 return !!(value
& 0x1);
3634 #define POWERSAVE_MASK GENMASK(7, 0)
3635 #define BALANCE_POWER_MASK GENMASK(15, 8)
3636 #define BALANCE_PERFORMANCE_MASK GENMASK(23, 16)
3637 #define PERFORMANCE_MASK GENMASK(31, 24)
3639 #define HWP_SET_EPP_VALUES(powersave, balance_power, balance_perf, performance) \
3640 (FIELD_PREP_CONST(POWERSAVE_MASK, powersave) |\
3641 FIELD_PREP_CONST(BALANCE_POWER_MASK, balance_power) |\
3642 FIELD_PREP_CONST(BALANCE_PERFORMANCE_MASK, balance_perf) |\
3643 FIELD_PREP_CONST(PERFORMANCE_MASK, performance))
3645 #define HWP_SET_DEF_BALANCE_PERF_EPP(balance_perf) \
3646 (HWP_SET_EPP_VALUES(HWP_EPP_POWERSAVE, HWP_EPP_BALANCE_POWERSAVE,\
3647 balance_perf, HWP_EPP_PERFORMANCE))
3649 static const struct x86_cpu_id intel_epp_default
[] = {
3651 * Set EPP value as 102, this is the max suggested EPP
3652 * which can result in one core turbo frequency for
3653 * AlderLake Mobile CPUs.
3655 X86_MATCH_VFM(INTEL_ALDERLAKE_L
, HWP_SET_DEF_BALANCE_PERF_EPP(102)),
3656 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X
, HWP_SET_DEF_BALANCE_PERF_EPP(32)),
3657 X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X
, HWP_SET_DEF_BALANCE_PERF_EPP(32)),
3658 X86_MATCH_VFM(INTEL_GRANITERAPIDS_X
, HWP_SET_DEF_BALANCE_PERF_EPP(32)),
3659 X86_MATCH_VFM(INTEL_GRANITERAPIDS_D
, HWP_SET_DEF_BALANCE_PERF_EPP(32)),
3660 X86_MATCH_VFM(INTEL_METEORLAKE_L
, HWP_SET_EPP_VALUES(HWP_EPP_POWERSAVE
,
3662 X86_MATCH_VFM(INTEL_ARROWLAKE
, HWP_SET_EPP_VALUES(HWP_EPP_POWERSAVE
,
3667 static const struct x86_cpu_id intel_hybrid_scaling_factor
[] = {
3668 X86_MATCH_VFM(INTEL_METEORLAKE_L
, HYBRID_SCALING_FACTOR_MTL
),
3669 X86_MATCH_VFM(INTEL_ARROWLAKE
, HYBRID_SCALING_FACTOR_MTL
),
3670 X86_MATCH_VFM(INTEL_LUNARLAKE_M
, HYBRID_SCALING_FACTOR_LNL
),
3674 static int __init
intel_pstate_init(void)
3676 static struct cpudata
**_all_cpu_data
;
3677 const struct x86_cpu_id
*id
;
3680 if (boot_cpu_data
.x86_vendor
!= X86_VENDOR_INTEL
)
3683 id
= x86_match_cpu(hwp_support_ids
);
3685 hwp_forced
= intel_pstate_hwp_is_enabled();
3688 pr_info("HWP enabled by BIOS\n");
3692 copy_cpu_funcs(&core_funcs
);
3694 * Avoid enabling HWP for processors without EPP support,
3695 * because that means incomplete HWP implementation which is a
3696 * corner case and supporting it is generally problematic.
3698 * If HWP is enabled already, though, there is no choice but to
3701 if ((!no_hwp
&& boot_cpu_has(X86_FEATURE_HWP_EPP
)) || hwp_forced
) {
3703 hwp_mode_bdw
= id
->driver_data
;
3704 intel_pstate
.attr
= hwp_cpufreq_attrs
;
3705 intel_cpufreq
.attr
= hwp_cpufreq_attrs
;
3706 intel_cpufreq
.flags
|= CPUFREQ_NEED_UPDATE_LIMITS
;
3707 intel_cpufreq
.adjust_perf
= intel_cpufreq_adjust_perf
;
3708 if (!default_driver
)
3709 default_driver
= &intel_pstate
;
3711 pstate_funcs
.get_cpu_scaling
= hwp_get_cpu_scaling
;
3713 goto hwp_cpu_matched
;
3715 pr_info("HWP not enabled\n");
3720 id
= x86_match_cpu(intel_pstate_cpu_ids
);
3722 pr_info("CPU model not supported\n");
3726 copy_cpu_funcs((struct pstate_funcs
*)id
->driver_data
);
3729 if (intel_pstate_msrs_not_valid()) {
3730 pr_info("Invalid MSRs\n");
3733 /* Without HWP start in the passive mode. */
3734 if (!default_driver
)
3735 default_driver
= &intel_cpufreq
;
3739 * The Intel pstate driver will be ignored if the platform
3740 * firmware has its own power management modes.
3742 if (intel_pstate_platform_pwr_mgmt_exists()) {
3743 pr_info("P-states controlled by the platform\n");
3747 if (!hwp_active
&& hwp_only
)
3750 pr_info("Intel P-state driver initializing\n");
3752 _all_cpu_data
= vzalloc(array_size(sizeof(void *), num_possible_cpus()));
3756 WRITE_ONCE(all_cpu_data
, _all_cpu_data
);
3758 intel_pstate_request_control_from_smm();
3760 intel_pstate_sysfs_expose_params();
3763 const struct x86_cpu_id
*id
= x86_match_cpu(intel_epp_default
);
3764 const struct x86_cpu_id
*hybrid_id
= x86_match_cpu(intel_hybrid_scaling_factor
);
3767 epp_values
[EPP_INDEX_POWERSAVE
] =
3768 FIELD_GET(POWERSAVE_MASK
, id
->driver_data
);
3769 epp_values
[EPP_INDEX_BALANCE_POWERSAVE
] =
3770 FIELD_GET(BALANCE_POWER_MASK
, id
->driver_data
);
3771 epp_values
[EPP_INDEX_BALANCE_PERFORMANCE
] =
3772 FIELD_GET(BALANCE_PERFORMANCE_MASK
, id
->driver_data
);
3773 epp_values
[EPP_INDEX_PERFORMANCE
] =
3774 FIELD_GET(PERFORMANCE_MASK
, id
->driver_data
);
3775 pr_debug("Updated EPPs powersave:%x balanced power:%x balanced perf:%x performance:%x\n",
3776 epp_values
[EPP_INDEX_POWERSAVE
],
3777 epp_values
[EPP_INDEX_BALANCE_POWERSAVE
],
3778 epp_values
[EPP_INDEX_BALANCE_PERFORMANCE
],
3779 epp_values
[EPP_INDEX_PERFORMANCE
]);
3783 hybrid_scaling_factor
= hybrid_id
->driver_data
;
3784 pr_debug("hybrid scaling factor: %d\n", hybrid_scaling_factor
);
3789 mutex_lock(&intel_pstate_driver_lock
);
3790 rc
= intel_pstate_register_driver(default_driver
);
3791 mutex_unlock(&intel_pstate_driver_lock
);
3793 intel_pstate_sysfs_remove();
3798 const struct x86_cpu_id
*id
;
3800 id
= x86_match_cpu(intel_pstate_cpu_ee_disable_ids
);
3802 set_power_ctl_ee_state(false);
3803 pr_info("Disabling energy efficiency optimization\n");
3806 pr_info("HWP enabled\n");
3807 } else if (boot_cpu_has(X86_FEATURE_HYBRID_CPU
)) {
3808 pr_warn("Problematic setup: Hybrid processor with disabled HWP\n");
3813 device_initcall(intel_pstate_init
);
3815 static int __init
intel_pstate_setup(char *str
)
3820 if (!strcmp(str
, "disable"))
3822 else if (!strcmp(str
, "active"))
3823 default_driver
= &intel_pstate
;
3824 else if (!strcmp(str
, "passive"))
3825 default_driver
= &intel_cpufreq
;
3827 if (!strcmp(str
, "no_hwp"))
3830 if (!strcmp(str
, "force"))
3832 if (!strcmp(str
, "hwp_only"))
3834 if (!strcmp(str
, "per_cpu_perf_limits"))
3835 per_cpu_limits
= true;
3838 if (!strcmp(str
, "support_acpi_ppc"))
3844 early_param("intel_pstate", intel_pstate_setup
);
3846 MODULE_AUTHOR("Dirk Brandewie <dirk.j.brandewie@intel.com>");
3847 MODULE_DESCRIPTION("'intel_pstate' - P state driver Intel Core processors");