2 * intel_pstate.c: Native P state management for Intel processors
4 * (C) Copyright 2012 Intel Corporation
5 * Author: Dirk Brandewie <dirk.j.brandewie@intel.com>
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; version 2
13 #include <linux/kernel.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/module.h>
16 #include <linux/ktime.h>
17 #include <linux/hrtimer.h>
18 #include <linux/tick.h>
19 #include <linux/slab.h>
20 #include <linux/sched.h>
21 #include <linux/list.h>
22 #include <linux/cpu.h>
23 #include <linux/cpufreq.h>
24 #include <linux/sysfs.h>
25 #include <linux/types.h>
27 #include <linux/debugfs.h>
28 #include <linux/acpi.h>
29 #include <linux/vmalloc.h>
30 #include <trace/events/power.h>
32 #include <asm/div64.h>
34 #include <asm/cpu_device_id.h>
35 #include <asm/cpufeature.h>
37 #define ATOM_RATIOS 0x66a
38 #define ATOM_VIDS 0x66b
39 #define ATOM_TURBO_RATIOS 0x66c
40 #define ATOM_TURBO_VIDS 0x66d
43 #define int_tofp(X) ((int64_t)(X) << FRAC_BITS)
44 #define fp_toint(X) ((X) >> FRAC_BITS)
46 static inline int32_t mul_fp(int32_t x
, int32_t y
)
48 return ((int64_t)x
* (int64_t)y
) >> FRAC_BITS
;
51 static inline int32_t div_fp(s64 x
, s64 y
)
53 return div64_s64((int64_t)x
<< FRAC_BITS
, y
);
56 static inline int ceiling_fp(int32_t x
)
61 mask
= (1 << FRAC_BITS
) - 1;
68 int32_t core_pct_busy
;
80 int max_pstate_physical
;
105 struct timer_list timer
;
107 struct pstate_data pstate
;
111 ktime_t last_sample_time
;
115 struct sample sample
;
118 static struct cpudata
**all_cpu_data
;
119 struct pstate_adjust_policy
{
128 struct pstate_funcs
{
129 int (*get_max
)(void);
130 int (*get_max_physical
)(void);
131 int (*get_min
)(void);
132 int (*get_turbo
)(void);
133 int (*get_scaling
)(void);
134 void (*set
)(struct cpudata
*, int pstate
);
135 void (*get_vid
)(struct cpudata
*);
138 struct cpu_defaults
{
139 struct pstate_adjust_policy pid_policy
;
140 struct pstate_funcs funcs
;
143 static struct pstate_adjust_policy pid_params
;
144 static struct pstate_funcs pstate_funcs
;
145 static int hwp_active
;
160 static struct perf_limits performance_limits
= {
164 .max_perf
= int_tofp(1),
166 .min_perf
= int_tofp(1),
167 .max_policy_pct
= 100,
168 .max_sysfs_pct
= 100,
173 static struct perf_limits powersave_limits
= {
177 .max_perf
= int_tofp(1),
180 .max_policy_pct
= 100,
181 .max_sysfs_pct
= 100,
186 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE
187 static struct perf_limits
*limits
= &performance_limits
;
189 static struct perf_limits
*limits
= &powersave_limits
;
192 static inline void pid_reset(struct _pid
*pid
, int setpoint
, int busy
,
193 int deadband
, int integral
) {
194 pid
->setpoint
= setpoint
;
195 pid
->deadband
= deadband
;
196 pid
->integral
= int_tofp(integral
);
197 pid
->last_err
= int_tofp(setpoint
) - int_tofp(busy
);
200 static inline void pid_p_gain_set(struct _pid
*pid
, int percent
)
202 pid
->p_gain
= div_fp(int_tofp(percent
), int_tofp(100));
205 static inline void pid_i_gain_set(struct _pid
*pid
, int percent
)
207 pid
->i_gain
= div_fp(int_tofp(percent
), int_tofp(100));
210 static inline void pid_d_gain_set(struct _pid
*pid
, int percent
)
212 pid
->d_gain
= div_fp(int_tofp(percent
), int_tofp(100));
215 static signed int pid_calc(struct _pid
*pid
, int32_t busy
)
218 int32_t pterm
, dterm
, fp_error
;
219 int32_t integral_limit
;
221 fp_error
= int_tofp(pid
->setpoint
) - busy
;
223 if (abs(fp_error
) <= int_tofp(pid
->deadband
))
226 pterm
= mul_fp(pid
->p_gain
, fp_error
);
228 pid
->integral
+= fp_error
;
231 * We limit the integral here so that it will never
232 * get higher than 30. This prevents it from becoming
233 * too large an input over long periods of time and allows
234 * it to get factored out sooner.
236 * The value of 30 was chosen through experimentation.
238 integral_limit
= int_tofp(30);
239 if (pid
->integral
> integral_limit
)
240 pid
->integral
= integral_limit
;
241 if (pid
->integral
< -integral_limit
)
242 pid
->integral
= -integral_limit
;
244 dterm
= mul_fp(pid
->d_gain
, fp_error
- pid
->last_err
);
245 pid
->last_err
= fp_error
;
247 result
= pterm
+ mul_fp(pid
->integral
, pid
->i_gain
) + dterm
;
248 result
= result
+ (1 << (FRAC_BITS
-1));
249 return (signed int)fp_toint(result
);
252 static inline void intel_pstate_busy_pid_reset(struct cpudata
*cpu
)
254 pid_p_gain_set(&cpu
->pid
, pid_params
.p_gain_pct
);
255 pid_d_gain_set(&cpu
->pid
, pid_params
.d_gain_pct
);
256 pid_i_gain_set(&cpu
->pid
, pid_params
.i_gain_pct
);
258 pid_reset(&cpu
->pid
, pid_params
.setpoint
, 100, pid_params
.deadband
, 0);
261 static inline void intel_pstate_reset_all_pid(void)
265 for_each_online_cpu(cpu
) {
266 if (all_cpu_data
[cpu
])
267 intel_pstate_busy_pid_reset(all_cpu_data
[cpu
]);
271 static inline void update_turbo_state(void)
276 cpu
= all_cpu_data
[0];
277 rdmsrl(MSR_IA32_MISC_ENABLE
, misc_en
);
278 limits
->turbo_disabled
=
279 (misc_en
& MSR_IA32_MISC_ENABLE_TURBO_DISABLE
||
280 cpu
->pstate
.max_pstate
== cpu
->pstate
.turbo_pstate
);
283 static void intel_pstate_hwp_set(void)
285 int min
, hw_min
, max
, hw_max
, cpu
, range
, adj_range
;
290 for_each_online_cpu(cpu
) {
291 rdmsrl_on_cpu(cpu
, MSR_HWP_CAPABILITIES
, &cap
);
292 hw_min
= HWP_LOWEST_PERF(cap
);
293 hw_max
= HWP_HIGHEST_PERF(cap
);
294 range
= hw_max
- hw_min
;
296 rdmsrl_on_cpu(cpu
, MSR_HWP_REQUEST
, &value
);
297 adj_range
= limits
->min_perf_pct
* range
/ 100;
298 min
= hw_min
+ adj_range
;
299 value
&= ~HWP_MIN_PERF(~0L);
300 value
|= HWP_MIN_PERF(min
);
302 adj_range
= limits
->max_perf_pct
* range
/ 100;
303 max
= hw_min
+ adj_range
;
304 if (limits
->no_turbo
) {
305 hw_max
= HWP_GUARANTEED_PERF(cap
);
310 value
&= ~HWP_MAX_PERF(~0L);
311 value
|= HWP_MAX_PERF(max
);
312 wrmsrl_on_cpu(cpu
, MSR_HWP_REQUEST
, value
);
318 /************************** debugfs begin ************************/
319 static int pid_param_set(void *data
, u64 val
)
322 intel_pstate_reset_all_pid();
326 static int pid_param_get(void *data
, u64
*val
)
331 DEFINE_SIMPLE_ATTRIBUTE(fops_pid_param
, pid_param_get
, pid_param_set
, "%llu\n");
338 static struct pid_param pid_files
[] = {
339 {"sample_rate_ms", &pid_params
.sample_rate_ms
},
340 {"d_gain_pct", &pid_params
.d_gain_pct
},
341 {"i_gain_pct", &pid_params
.i_gain_pct
},
342 {"deadband", &pid_params
.deadband
},
343 {"setpoint", &pid_params
.setpoint
},
344 {"p_gain_pct", &pid_params
.p_gain_pct
},
348 static void __init
intel_pstate_debug_expose_params(void)
350 struct dentry
*debugfs_parent
;
355 debugfs_parent
= debugfs_create_dir("pstate_snb", NULL
);
356 if (IS_ERR_OR_NULL(debugfs_parent
))
358 while (pid_files
[i
].name
) {
359 debugfs_create_file(pid_files
[i
].name
, 0660,
360 debugfs_parent
, pid_files
[i
].value
,
366 /************************** debugfs end ************************/
368 /************************** sysfs begin ************************/
369 #define show_one(file_name, object) \
370 static ssize_t show_##file_name \
371 (struct kobject *kobj, struct attribute *attr, char *buf) \
373 return sprintf(buf, "%u\n", limits->object); \
376 static ssize_t
show_turbo_pct(struct kobject
*kobj
,
377 struct attribute
*attr
, char *buf
)
380 int total
, no_turbo
, turbo_pct
;
383 cpu
= all_cpu_data
[0];
385 total
= cpu
->pstate
.turbo_pstate
- cpu
->pstate
.min_pstate
+ 1;
386 no_turbo
= cpu
->pstate
.max_pstate
- cpu
->pstate
.min_pstate
+ 1;
387 turbo_fp
= div_fp(int_tofp(no_turbo
), int_tofp(total
));
388 turbo_pct
= 100 - fp_toint(mul_fp(turbo_fp
, int_tofp(100)));
389 return sprintf(buf
, "%u\n", turbo_pct
);
392 static ssize_t
show_num_pstates(struct kobject
*kobj
,
393 struct attribute
*attr
, char *buf
)
398 cpu
= all_cpu_data
[0];
399 total
= cpu
->pstate
.turbo_pstate
- cpu
->pstate
.min_pstate
+ 1;
400 return sprintf(buf
, "%u\n", total
);
403 static ssize_t
show_no_turbo(struct kobject
*kobj
,
404 struct attribute
*attr
, char *buf
)
408 update_turbo_state();
409 if (limits
->turbo_disabled
)
410 ret
= sprintf(buf
, "%u\n", limits
->turbo_disabled
);
412 ret
= sprintf(buf
, "%u\n", limits
->no_turbo
);
417 static ssize_t
store_no_turbo(struct kobject
*a
, struct attribute
*b
,
418 const char *buf
, size_t count
)
423 ret
= sscanf(buf
, "%u", &input
);
427 update_turbo_state();
428 if (limits
->turbo_disabled
) {
429 pr_warn("intel_pstate: Turbo disabled by BIOS or unavailable on processor\n");
433 limits
->no_turbo
= clamp_t(int, input
, 0, 1);
436 intel_pstate_hwp_set();
441 static ssize_t
store_max_perf_pct(struct kobject
*a
, struct attribute
*b
,
442 const char *buf
, size_t count
)
447 ret
= sscanf(buf
, "%u", &input
);
451 limits
->max_sysfs_pct
= clamp_t(int, input
, 0 , 100);
452 limits
->max_perf_pct
= min(limits
->max_policy_pct
,
453 limits
->max_sysfs_pct
);
454 limits
->max_perf_pct
= max(limits
->min_policy_pct
,
455 limits
->max_perf_pct
);
456 limits
->max_perf_pct
= max(limits
->min_perf_pct
,
457 limits
->max_perf_pct
);
458 limits
->max_perf
= div_fp(int_tofp(limits
->max_perf_pct
),
462 intel_pstate_hwp_set();
466 static ssize_t
store_min_perf_pct(struct kobject
*a
, struct attribute
*b
,
467 const char *buf
, size_t count
)
472 ret
= sscanf(buf
, "%u", &input
);
476 limits
->min_sysfs_pct
= clamp_t(int, input
, 0 , 100);
477 limits
->min_perf_pct
= max(limits
->min_policy_pct
,
478 limits
->min_sysfs_pct
);
479 limits
->min_perf_pct
= min(limits
->max_policy_pct
,
480 limits
->min_perf_pct
);
481 limits
->min_perf_pct
= min(limits
->max_perf_pct
,
482 limits
->min_perf_pct
);
483 limits
->min_perf
= div_fp(int_tofp(limits
->min_perf_pct
),
487 intel_pstate_hwp_set();
491 show_one(max_perf_pct
, max_perf_pct
);
492 show_one(min_perf_pct
, min_perf_pct
);
494 define_one_global_rw(no_turbo
);
495 define_one_global_rw(max_perf_pct
);
496 define_one_global_rw(min_perf_pct
);
497 define_one_global_ro(turbo_pct
);
498 define_one_global_ro(num_pstates
);
500 static struct attribute
*intel_pstate_attributes
[] = {
509 static struct attribute_group intel_pstate_attr_group
= {
510 .attrs
= intel_pstate_attributes
,
513 static void __init
intel_pstate_sysfs_expose_params(void)
515 struct kobject
*intel_pstate_kobject
;
518 intel_pstate_kobject
= kobject_create_and_add("intel_pstate",
519 &cpu_subsys
.dev_root
->kobj
);
520 BUG_ON(!intel_pstate_kobject
);
521 rc
= sysfs_create_group(intel_pstate_kobject
, &intel_pstate_attr_group
);
524 /************************** sysfs end ************************/
526 static void intel_pstate_hwp_enable(struct cpudata
*cpudata
)
528 wrmsrl_on_cpu(cpudata
->cpu
, MSR_PM_ENABLE
, 0x1);
531 static int atom_get_min_pstate(void)
535 rdmsrl(ATOM_RATIOS
, value
);
536 return (value
>> 8) & 0x7F;
539 static int atom_get_max_pstate(void)
543 rdmsrl(ATOM_RATIOS
, value
);
544 return (value
>> 16) & 0x7F;
547 static int atom_get_turbo_pstate(void)
551 rdmsrl(ATOM_TURBO_RATIOS
, value
);
555 static void atom_set_pstate(struct cpudata
*cpudata
, int pstate
)
561 val
= (u64
)pstate
<< 8;
562 if (limits
->no_turbo
&& !limits
->turbo_disabled
)
565 vid_fp
= cpudata
->vid
.min
+ mul_fp(
566 int_tofp(pstate
- cpudata
->pstate
.min_pstate
),
569 vid_fp
= clamp_t(int32_t, vid_fp
, cpudata
->vid
.min
, cpudata
->vid
.max
);
570 vid
= ceiling_fp(vid_fp
);
572 if (pstate
> cpudata
->pstate
.max_pstate
)
573 vid
= cpudata
->vid
.turbo
;
577 wrmsrl_on_cpu(cpudata
->cpu
, MSR_IA32_PERF_CTL
, val
);
580 static int silvermont_get_scaling(void)
584 /* Defined in Table 35-6 from SDM (Sept 2015) */
585 static int silvermont_freq_table
[] = {
586 83300, 100000, 133300, 116700, 80000};
588 rdmsrl(MSR_FSB_FREQ
, value
);
592 return silvermont_freq_table
[i
];
595 static int airmont_get_scaling(void)
599 /* Defined in Table 35-10 from SDM (Sept 2015) */
600 static int airmont_freq_table
[] = {
601 83300, 100000, 133300, 116700, 80000,
602 93300, 90000, 88900, 87500};
604 rdmsrl(MSR_FSB_FREQ
, value
);
608 return airmont_freq_table
[i
];
611 static void atom_get_vid(struct cpudata
*cpudata
)
615 rdmsrl(ATOM_VIDS
, value
);
616 cpudata
->vid
.min
= int_tofp((value
>> 8) & 0x7f);
617 cpudata
->vid
.max
= int_tofp((value
>> 16) & 0x7f);
618 cpudata
->vid
.ratio
= div_fp(
619 cpudata
->vid
.max
- cpudata
->vid
.min
,
620 int_tofp(cpudata
->pstate
.max_pstate
-
621 cpudata
->pstate
.min_pstate
));
623 rdmsrl(ATOM_TURBO_VIDS
, value
);
624 cpudata
->vid
.turbo
= value
& 0x7f;
627 static int core_get_min_pstate(void)
631 rdmsrl(MSR_PLATFORM_INFO
, value
);
632 return (value
>> 40) & 0xFF;
635 static int core_get_max_pstate_physical(void)
639 rdmsrl(MSR_PLATFORM_INFO
, value
);
640 return (value
>> 8) & 0xFF;
643 static int core_get_max_pstate(void)
650 rdmsrl(MSR_PLATFORM_INFO
, plat_info
);
651 max_pstate
= (plat_info
>> 8) & 0xFF;
653 err
= rdmsrl_safe(MSR_TURBO_ACTIVATION_RATIO
, &tar
);
655 /* Do some sanity checking for safety */
656 if (plat_info
& 0x600000000) {
661 err
= rdmsrl_safe(MSR_CONFIG_TDP_CONTROL
, &tdp_ctrl
);
665 tdp_msr
= MSR_CONFIG_TDP_NOMINAL
+ (tdp_ctrl
& 0x3);
666 err
= rdmsrl_safe(tdp_msr
, &tdp_ratio
);
670 /* For level 1 and 2, bits[23:16] contain the ratio */
674 tdp_ratio
&= 0xff; /* ratios are only 8 bits long */
675 if (tdp_ratio
- 1 == tar
) {
677 pr_debug("max_pstate=TAC %x\n", max_pstate
);
688 static int core_get_turbo_pstate(void)
693 rdmsrl(MSR_NHM_TURBO_RATIO_LIMIT
, value
);
694 nont
= core_get_max_pstate();
701 static inline int core_get_scaling(void)
706 static void core_set_pstate(struct cpudata
*cpudata
, int pstate
)
710 val
= (u64
)pstate
<< 8;
711 if (limits
->no_turbo
&& !limits
->turbo_disabled
)
714 wrmsrl_on_cpu(cpudata
->cpu
, MSR_IA32_PERF_CTL
, val
);
717 static int knl_get_turbo_pstate(void)
722 rdmsrl(MSR_NHM_TURBO_RATIO_LIMIT
, value
);
723 nont
= core_get_max_pstate();
724 ret
= (((value
) >> 8) & 0xFF);
730 static struct cpu_defaults core_params
= {
732 .sample_rate_ms
= 10,
740 .get_max
= core_get_max_pstate
,
741 .get_max_physical
= core_get_max_pstate_physical
,
742 .get_min
= core_get_min_pstate
,
743 .get_turbo
= core_get_turbo_pstate
,
744 .get_scaling
= core_get_scaling
,
745 .set
= core_set_pstate
,
749 static struct cpu_defaults silvermont_params
= {
751 .sample_rate_ms
= 10,
759 .get_max
= atom_get_max_pstate
,
760 .get_max_physical
= atom_get_max_pstate
,
761 .get_min
= atom_get_min_pstate
,
762 .get_turbo
= atom_get_turbo_pstate
,
763 .set
= atom_set_pstate
,
764 .get_scaling
= silvermont_get_scaling
,
765 .get_vid
= atom_get_vid
,
769 static struct cpu_defaults airmont_params
= {
771 .sample_rate_ms
= 10,
779 .get_max
= atom_get_max_pstate
,
780 .get_max_physical
= atom_get_max_pstate
,
781 .get_min
= atom_get_min_pstate
,
782 .get_turbo
= atom_get_turbo_pstate
,
783 .set
= atom_set_pstate
,
784 .get_scaling
= airmont_get_scaling
,
785 .get_vid
= atom_get_vid
,
789 static struct cpu_defaults knl_params
= {
791 .sample_rate_ms
= 10,
799 .get_max
= core_get_max_pstate
,
800 .get_max_physical
= core_get_max_pstate_physical
,
801 .get_min
= core_get_min_pstate
,
802 .get_turbo
= knl_get_turbo_pstate
,
803 .get_scaling
= core_get_scaling
,
804 .set
= core_set_pstate
,
808 static void intel_pstate_get_min_max(struct cpudata
*cpu
, int *min
, int *max
)
810 int max_perf
= cpu
->pstate
.turbo_pstate
;
814 if (limits
->no_turbo
|| limits
->turbo_disabled
)
815 max_perf
= cpu
->pstate
.max_pstate
;
818 * performance can be limited by user through sysfs, by cpufreq
819 * policy, or by cpu specific default values determined through
822 max_perf_adj
= fp_toint(mul_fp(int_tofp(max_perf
), limits
->max_perf
));
823 *max
= clamp_t(int, max_perf_adj
,
824 cpu
->pstate
.min_pstate
, cpu
->pstate
.turbo_pstate
);
826 min_perf
= fp_toint(mul_fp(int_tofp(max_perf
), limits
->min_perf
));
827 *min
= clamp_t(int, min_perf
, cpu
->pstate
.min_pstate
, max_perf
);
830 static void intel_pstate_set_pstate(struct cpudata
*cpu
, int pstate
, bool force
)
832 int max_perf
, min_perf
;
835 update_turbo_state();
837 intel_pstate_get_min_max(cpu
, &min_perf
, &max_perf
);
839 pstate
= clamp_t(int, pstate
, min_perf
, max_perf
);
841 if (pstate
== cpu
->pstate
.current_pstate
)
844 trace_cpu_frequency(pstate
* cpu
->pstate
.scaling
, cpu
->cpu
);
846 cpu
->pstate
.current_pstate
= pstate
;
848 pstate_funcs
.set(cpu
, pstate
);
851 static void intel_pstate_get_cpu_pstates(struct cpudata
*cpu
)
853 cpu
->pstate
.min_pstate
= pstate_funcs
.get_min();
854 cpu
->pstate
.max_pstate
= pstate_funcs
.get_max();
855 cpu
->pstate
.max_pstate_physical
= pstate_funcs
.get_max_physical();
856 cpu
->pstate
.turbo_pstate
= pstate_funcs
.get_turbo();
857 cpu
->pstate
.scaling
= pstate_funcs
.get_scaling();
859 if (pstate_funcs
.get_vid
)
860 pstate_funcs
.get_vid(cpu
);
861 intel_pstate_set_pstate(cpu
, cpu
->pstate
.min_pstate
, false);
864 static inline void intel_pstate_calc_busy(struct cpudata
*cpu
)
866 struct sample
*sample
= &cpu
->sample
;
869 core_pct
= int_tofp(sample
->aperf
) * int_tofp(100);
870 core_pct
= div64_u64(core_pct
, int_tofp(sample
->mperf
));
872 sample
->freq
= fp_toint(
874 cpu
->pstate
.max_pstate_physical
*
875 cpu
->pstate
.scaling
/ 100),
878 sample
->core_pct_busy
= (int32_t)core_pct
;
881 static inline void intel_pstate_sample(struct cpudata
*cpu
)
887 local_irq_save(flags
);
888 rdmsrl(MSR_IA32_APERF
, aperf
);
889 rdmsrl(MSR_IA32_MPERF
, mperf
);
890 if (cpu
->prev_mperf
== mperf
) {
891 local_irq_restore(flags
);
896 local_irq_restore(flags
);
898 cpu
->last_sample_time
= cpu
->sample
.time
;
899 cpu
->sample
.time
= ktime_get();
900 cpu
->sample
.aperf
= aperf
;
901 cpu
->sample
.mperf
= mperf
;
902 cpu
->sample
.tsc
= tsc
;
903 cpu
->sample
.aperf
-= cpu
->prev_aperf
;
904 cpu
->sample
.mperf
-= cpu
->prev_mperf
;
905 cpu
->sample
.tsc
-= cpu
->prev_tsc
;
907 intel_pstate_calc_busy(cpu
);
909 cpu
->prev_aperf
= aperf
;
910 cpu
->prev_mperf
= mperf
;
914 static inline void intel_hwp_set_sample_time(struct cpudata
*cpu
)
918 delay
= msecs_to_jiffies(50);
919 mod_timer_pinned(&cpu
->timer
, jiffies
+ delay
);
922 static inline void intel_pstate_set_sample_time(struct cpudata
*cpu
)
926 delay
= msecs_to_jiffies(pid_params
.sample_rate_ms
);
927 mod_timer_pinned(&cpu
->timer
, jiffies
+ delay
);
930 static inline int32_t intel_pstate_get_scaled_busy(struct cpudata
*cpu
)
932 int32_t core_busy
, max_pstate
, current_pstate
, sample_ratio
;
937 * core_busy is the ratio of actual performance to max
938 * max_pstate is the max non turbo pstate available
939 * current_pstate was the pstate that was requested during
940 * the last sample period.
942 * We normalize core_busy, which was our actual percent
943 * performance to what we requested during the last sample
944 * period. The result will be a percentage of busy at a
947 core_busy
= cpu
->sample
.core_pct_busy
;
948 max_pstate
= int_tofp(cpu
->pstate
.max_pstate_physical
);
949 current_pstate
= int_tofp(cpu
->pstate
.current_pstate
);
950 core_busy
= mul_fp(core_busy
, div_fp(max_pstate
, current_pstate
));
953 * Since we have a deferred timer, it will not fire unless
954 * we are in C0. So, determine if the actual elapsed time
955 * is significantly greater (3x) than our sample interval. If it
956 * is, then we were idle for a long enough period of time
957 * to adjust our busyness.
959 sample_time
= pid_params
.sample_rate_ms
* USEC_PER_MSEC
;
960 duration_us
= ktime_us_delta(cpu
->sample
.time
,
961 cpu
->last_sample_time
);
962 if (duration_us
> sample_time
* 3) {
963 sample_ratio
= div_fp(int_tofp(sample_time
),
964 int_tofp(duration_us
));
965 core_busy
= mul_fp(core_busy
, sample_ratio
);
971 static inline void intel_pstate_adjust_busy_pstate(struct cpudata
*cpu
)
977 struct sample
*sample
;
979 from
= cpu
->pstate
.current_pstate
;
982 busy_scaled
= intel_pstate_get_scaled_busy(cpu
);
984 ctl
= pid_calc(pid
, busy_scaled
);
986 /* Negative values of ctl increase the pstate and vice versa */
987 intel_pstate_set_pstate(cpu
, cpu
->pstate
.current_pstate
- ctl
, true);
989 sample
= &cpu
->sample
;
990 trace_pstate_sample(fp_toint(sample
->core_pct_busy
),
991 fp_toint(busy_scaled
),
993 cpu
->pstate
.current_pstate
,
1000 static void intel_hwp_timer_func(unsigned long __data
)
1002 struct cpudata
*cpu
= (struct cpudata
*) __data
;
1004 intel_pstate_sample(cpu
);
1005 intel_hwp_set_sample_time(cpu
);
1008 static void intel_pstate_timer_func(unsigned long __data
)
1010 struct cpudata
*cpu
= (struct cpudata
*) __data
;
1012 intel_pstate_sample(cpu
);
1014 intel_pstate_adjust_busy_pstate(cpu
);
1016 intel_pstate_set_sample_time(cpu
);
1019 #define ICPU(model, policy) \
1020 { X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\
1021 (unsigned long)&policy }
1023 static const struct x86_cpu_id intel_pstate_cpu_ids
[] = {
1024 ICPU(0x2a, core_params
),
1025 ICPU(0x2d, core_params
),
1026 ICPU(0x37, silvermont_params
),
1027 ICPU(0x3a, core_params
),
1028 ICPU(0x3c, core_params
),
1029 ICPU(0x3d, core_params
),
1030 ICPU(0x3e, core_params
),
1031 ICPU(0x3f, core_params
),
1032 ICPU(0x45, core_params
),
1033 ICPU(0x46, core_params
),
1034 ICPU(0x47, core_params
),
1035 ICPU(0x4c, airmont_params
),
1036 ICPU(0x4e, core_params
),
1037 ICPU(0x4f, core_params
),
1038 ICPU(0x5e, core_params
),
1039 ICPU(0x56, core_params
),
1040 ICPU(0x57, knl_params
),
1043 MODULE_DEVICE_TABLE(x86cpu
, intel_pstate_cpu_ids
);
1045 static const struct x86_cpu_id intel_pstate_cpu_oob_ids
[] = {
1046 ICPU(0x56, core_params
),
1050 static int intel_pstate_init_cpu(unsigned int cpunum
)
1052 struct cpudata
*cpu
;
1054 if (!all_cpu_data
[cpunum
])
1055 all_cpu_data
[cpunum
] = kzalloc(sizeof(struct cpudata
),
1057 if (!all_cpu_data
[cpunum
])
1060 cpu
= all_cpu_data
[cpunum
];
1065 intel_pstate_hwp_enable(cpu
);
1067 intel_pstate_get_cpu_pstates(cpu
);
1069 init_timer_deferrable(&cpu
->timer
);
1070 cpu
->timer
.data
= (unsigned long)cpu
;
1071 cpu
->timer
.expires
= jiffies
+ HZ
/100;
1074 cpu
->timer
.function
= intel_pstate_timer_func
;
1076 cpu
->timer
.function
= intel_hwp_timer_func
;
1078 intel_pstate_busy_pid_reset(cpu
);
1079 intel_pstate_sample(cpu
);
1081 add_timer_on(&cpu
->timer
, cpunum
);
1083 pr_debug("intel_pstate: controlling: cpu %d\n", cpunum
);
1088 static unsigned int intel_pstate_get(unsigned int cpu_num
)
1090 struct sample
*sample
;
1091 struct cpudata
*cpu
;
1093 cpu
= all_cpu_data
[cpu_num
];
1096 sample
= &cpu
->sample
;
1097 return sample
->freq
;
1100 static int intel_pstate_set_policy(struct cpufreq_policy
*policy
)
1102 if (!policy
->cpuinfo
.max_freq
)
1105 if (policy
->policy
== CPUFREQ_POLICY_PERFORMANCE
&&
1106 policy
->max
>= policy
->cpuinfo
.max_freq
) {
1107 pr_debug("intel_pstate: set performance\n");
1108 limits
= &performance_limits
;
1110 intel_pstate_hwp_set();
1114 pr_debug("intel_pstate: set powersave\n");
1115 limits
= &powersave_limits
;
1116 limits
->min_policy_pct
= (policy
->min
* 100) / policy
->cpuinfo
.max_freq
;
1117 limits
->min_policy_pct
= clamp_t(int, limits
->min_policy_pct
, 0 , 100);
1118 limits
->max_policy_pct
= DIV_ROUND_UP(policy
->max
* 100,
1119 policy
->cpuinfo
.max_freq
);
1120 limits
->max_policy_pct
= clamp_t(int, limits
->max_policy_pct
, 0 , 100);
1122 /* Normalize user input to [min_policy_pct, max_policy_pct] */
1123 limits
->min_perf_pct
= max(limits
->min_policy_pct
,
1124 limits
->min_sysfs_pct
);
1125 limits
->min_perf_pct
= min(limits
->max_policy_pct
,
1126 limits
->min_perf_pct
);
1127 limits
->max_perf_pct
= min(limits
->max_policy_pct
,
1128 limits
->max_sysfs_pct
);
1129 limits
->max_perf_pct
= max(limits
->min_policy_pct
,
1130 limits
->max_perf_pct
);
1131 limits
->max_perf
= round_up(limits
->max_perf
, FRAC_BITS
);
1133 /* Make sure min_perf_pct <= max_perf_pct */
1134 limits
->min_perf_pct
= min(limits
->max_perf_pct
, limits
->min_perf_pct
);
1136 limits
->min_perf
= div_fp(int_tofp(limits
->min_perf_pct
),
1138 limits
->max_perf
= div_fp(int_tofp(limits
->max_perf_pct
),
1142 intel_pstate_hwp_set();
1147 static int intel_pstate_verify_policy(struct cpufreq_policy
*policy
)
1149 cpufreq_verify_within_cpu_limits(policy
);
1151 if (policy
->policy
!= CPUFREQ_POLICY_POWERSAVE
&&
1152 policy
->policy
!= CPUFREQ_POLICY_PERFORMANCE
)
1158 static void intel_pstate_stop_cpu(struct cpufreq_policy
*policy
)
1160 int cpu_num
= policy
->cpu
;
1161 struct cpudata
*cpu
= all_cpu_data
[cpu_num
];
1163 pr_debug("intel_pstate: CPU %d exiting\n", cpu_num
);
1165 del_timer_sync(&all_cpu_data
[cpu_num
]->timer
);
1169 intel_pstate_set_pstate(cpu
, cpu
->pstate
.min_pstate
, false);
1172 static int intel_pstate_cpu_init(struct cpufreq_policy
*policy
)
1174 struct cpudata
*cpu
;
1177 rc
= intel_pstate_init_cpu(policy
->cpu
);
1181 cpu
= all_cpu_data
[policy
->cpu
];
1183 if (limits
->min_perf_pct
== 100 && limits
->max_perf_pct
== 100)
1184 policy
->policy
= CPUFREQ_POLICY_PERFORMANCE
;
1186 policy
->policy
= CPUFREQ_POLICY_POWERSAVE
;
1188 policy
->min
= cpu
->pstate
.min_pstate
* cpu
->pstate
.scaling
;
1189 policy
->max
= cpu
->pstate
.turbo_pstate
* cpu
->pstate
.scaling
;
1191 /* cpuinfo and default policy values */
1192 policy
->cpuinfo
.min_freq
= cpu
->pstate
.min_pstate
* cpu
->pstate
.scaling
;
1193 policy
->cpuinfo
.max_freq
=
1194 cpu
->pstate
.turbo_pstate
* cpu
->pstate
.scaling
;
1195 policy
->cpuinfo
.transition_latency
= CPUFREQ_ETERNAL
;
1196 cpumask_set_cpu(policy
->cpu
, policy
->cpus
);
1201 static struct cpufreq_driver intel_pstate_driver
= {
1202 .flags
= CPUFREQ_CONST_LOOPS
,
1203 .verify
= intel_pstate_verify_policy
,
1204 .setpolicy
= intel_pstate_set_policy
,
1205 .get
= intel_pstate_get
,
1206 .init
= intel_pstate_cpu_init
,
1207 .stop_cpu
= intel_pstate_stop_cpu
,
1208 .name
= "intel_pstate",
1211 static int __initdata no_load
;
1212 static int __initdata no_hwp
;
1213 static int __initdata hwp_only
;
1214 static unsigned int force_load
;
1216 static int intel_pstate_msrs_not_valid(void)
1218 if (!pstate_funcs
.get_max() ||
1219 !pstate_funcs
.get_min() ||
1220 !pstate_funcs
.get_turbo())
1226 static void copy_pid_params(struct pstate_adjust_policy
*policy
)
1228 pid_params
.sample_rate_ms
= policy
->sample_rate_ms
;
1229 pid_params
.p_gain_pct
= policy
->p_gain_pct
;
1230 pid_params
.i_gain_pct
= policy
->i_gain_pct
;
1231 pid_params
.d_gain_pct
= policy
->d_gain_pct
;
1232 pid_params
.deadband
= policy
->deadband
;
1233 pid_params
.setpoint
= policy
->setpoint
;
1236 static void copy_cpu_funcs(struct pstate_funcs
*funcs
)
1238 pstate_funcs
.get_max
= funcs
->get_max
;
1239 pstate_funcs
.get_max_physical
= funcs
->get_max_physical
;
1240 pstate_funcs
.get_min
= funcs
->get_min
;
1241 pstate_funcs
.get_turbo
= funcs
->get_turbo
;
1242 pstate_funcs
.get_scaling
= funcs
->get_scaling
;
1243 pstate_funcs
.set
= funcs
->set
;
1244 pstate_funcs
.get_vid
= funcs
->get_vid
;
1247 #if IS_ENABLED(CONFIG_ACPI)
1248 #include <acpi/processor.h>
1250 static bool intel_pstate_no_acpi_pss(void)
1254 for_each_possible_cpu(i
) {
1256 union acpi_object
*pss
;
1257 struct acpi_buffer buffer
= { ACPI_ALLOCATE_BUFFER
, NULL
};
1258 struct acpi_processor
*pr
= per_cpu(processors
, i
);
1263 status
= acpi_evaluate_object(pr
->handle
, "_PSS", NULL
, &buffer
);
1264 if (ACPI_FAILURE(status
))
1267 pss
= buffer
.pointer
;
1268 if (pss
&& pss
->type
== ACPI_TYPE_PACKAGE
) {
1279 static bool intel_pstate_has_acpi_ppc(void)
1283 for_each_possible_cpu(i
) {
1284 struct acpi_processor
*pr
= per_cpu(processors
, i
);
1288 if (acpi_has_method(pr
->handle
, "_PPC"))
1299 struct hw_vendor_info
{
1301 char oem_id
[ACPI_OEM_ID_SIZE
];
1302 char oem_table_id
[ACPI_OEM_TABLE_ID_SIZE
];
1306 /* Hardware vendor-specific info that has its own power management modes */
1307 static struct hw_vendor_info vendor_info
[] = {
1308 {1, "HP ", "ProLiant", PSS
},
1309 {1, "ORACLE", "X4-2 ", PPC
},
1310 {1, "ORACLE", "X4-2L ", PPC
},
1311 {1, "ORACLE", "X4-2B ", PPC
},
1312 {1, "ORACLE", "X3-2 ", PPC
},
1313 {1, "ORACLE", "X3-2L ", PPC
},
1314 {1, "ORACLE", "X3-2B ", PPC
},
1315 {1, "ORACLE", "X4470M2 ", PPC
},
1316 {1, "ORACLE", "X4270M3 ", PPC
},
1317 {1, "ORACLE", "X4270M2 ", PPC
},
1318 {1, "ORACLE", "X4170M2 ", PPC
},
1319 {1, "ORACLE", "X4170 M3", PPC
},
1320 {1, "ORACLE", "X4275 M3", PPC
},
1321 {1, "ORACLE", "X6-2 ", PPC
},
1322 {1, "ORACLE", "Sudbury ", PPC
},
1326 static bool intel_pstate_platform_pwr_mgmt_exists(void)
1328 struct acpi_table_header hdr
;
1329 struct hw_vendor_info
*v_info
;
1330 const struct x86_cpu_id
*id
;
1333 id
= x86_match_cpu(intel_pstate_cpu_oob_ids
);
1335 rdmsrl(MSR_MISC_PWR_MGMT
, misc_pwr
);
1336 if ( misc_pwr
& (1 << 8))
1340 if (acpi_disabled
||
1341 ACPI_FAILURE(acpi_get_table_header(ACPI_SIG_FADT
, 0, &hdr
)))
1344 for (v_info
= vendor_info
; v_info
->valid
; v_info
++) {
1345 if (!strncmp(hdr
.oem_id
, v_info
->oem_id
, ACPI_OEM_ID_SIZE
) &&
1346 !strncmp(hdr
.oem_table_id
, v_info
->oem_table_id
,
1347 ACPI_OEM_TABLE_ID_SIZE
))
1348 switch (v_info
->oem_pwr_table
) {
1350 return intel_pstate_no_acpi_pss();
1352 return intel_pstate_has_acpi_ppc() &&
1359 #else /* CONFIG_ACPI not enabled */
1360 static inline bool intel_pstate_platform_pwr_mgmt_exists(void) { return false; }
1361 static inline bool intel_pstate_has_acpi_ppc(void) { return false; }
1362 #endif /* CONFIG_ACPI */
1364 static const struct x86_cpu_id hwp_support_ids
[] __initconst
= {
1365 { X86_VENDOR_INTEL
, 6, X86_MODEL_ANY
, X86_FEATURE_HWP
},
1369 static int __init
intel_pstate_init(void)
1372 const struct x86_cpu_id
*id
;
1373 struct cpu_defaults
*cpu_def
;
1378 if (x86_match_cpu(hwp_support_ids
) && !no_hwp
) {
1379 copy_cpu_funcs(&core_params
.funcs
);
1381 goto hwp_cpu_matched
;
1384 id
= x86_match_cpu(intel_pstate_cpu_ids
);
1388 cpu_def
= (struct cpu_defaults
*)id
->driver_data
;
1390 copy_pid_params(&cpu_def
->pid_policy
);
1391 copy_cpu_funcs(&cpu_def
->funcs
);
1393 if (intel_pstate_msrs_not_valid())
1398 * The Intel pstate driver will be ignored if the platform
1399 * firmware has its own power management modes.
1401 if (intel_pstate_platform_pwr_mgmt_exists())
1404 pr_info("Intel P-state driver initializing.\n");
1406 all_cpu_data
= vzalloc(sizeof(void *) * num_possible_cpus());
1410 if (!hwp_active
&& hwp_only
)
1413 rc
= cpufreq_register_driver(&intel_pstate_driver
);
1417 intel_pstate_debug_expose_params();
1418 intel_pstate_sysfs_expose_params();
1421 pr_info("intel_pstate: HWP enabled\n");
1426 for_each_online_cpu(cpu
) {
1427 if (all_cpu_data
[cpu
]) {
1428 del_timer_sync(&all_cpu_data
[cpu
]->timer
);
1429 kfree(all_cpu_data
[cpu
]);
1434 vfree(all_cpu_data
);
1437 device_initcall(intel_pstate_init
);
1439 static int __init
intel_pstate_setup(char *str
)
1444 if (!strcmp(str
, "disable"))
1446 if (!strcmp(str
, "no_hwp")) {
1447 pr_info("intel_pstate: HWP disabled\n");
1450 if (!strcmp(str
, "force"))
1452 if (!strcmp(str
, "hwp_only"))
1456 early_param("intel_pstate", intel_pstate_setup
);
1458 MODULE_AUTHOR("Dirk Brandewie <dirk.j.brandewie@intel.com>");
1459 MODULE_DESCRIPTION("'intel_pstate' - P state driver Intel Core processors");
1460 MODULE_LICENSE("GPL");