2 * arch/arm/kernel/topology.c
4 * Copyright (C) 2011 Linaro Limited.
5 * Written by: Vincent Guittot
7 * based on arch/sh/kernel/topology.c
9 * This file is subject to the terms and conditions of the GNU General Public
10 * License. See the file "COPYING" in the main directory of this archive
14 #include <linux/cpu.h>
15 #include <linux/cpumask.h>
16 #include <linux/export.h>
17 #include <linux/init.h>
18 #include <linux/percpu.h>
19 #include <linux/node.h>
20 #include <linux/nodemask.h>
22 #include <linux/sched.h>
23 #include <linux/slab.h>
25 #include <asm/cputype.h>
26 #include <asm/topology.h>
29 * cpu capacity scale management
34 * This per cpu data structure describes the relative capacity of each core.
35 * On a heteregenous system, cores don't have the same computation capacity
36 * and we reflect that difference in the cpu_capacity field so the scheduler
37 * can take this difference into account during load balance. A per cpu
38 * structure is preferred because each CPU updates its own cpu_capacity field
39 * during the load balance except for idle cores. One idle core is selected
40 * to run the rebalance_domains for all idle cores and the cpu_capacity can be
41 * updated during this sequence.
43 static DEFINE_PER_CPU(unsigned long, cpu_scale
);
45 unsigned long arch_scale_freq_capacity(struct sched_domain
*sd
, int cpu
)
47 return per_cpu(cpu_scale
, cpu
);
50 static void set_capacity_scale(unsigned int cpu
, unsigned long capacity
)
52 per_cpu(cpu_scale
, cpu
) = capacity
;
56 struct cpu_efficiency
{
57 const char *compatible
;
58 unsigned long efficiency
;
62 * Table of relative efficiency of each processors
63 * The efficiency value must fit in 20bit and the final
64 * cpu_scale value must be in the range
65 * 0 < cpu_scale < 3*SCHED_CAPACITY_SCALE/2
66 * in order to return at most 1 when DIV_ROUND_CLOSEST
67 * is used to compute the capacity of a CPU.
68 * Processors that are not defined in the table,
69 * use the default SCHED_CAPACITY_SCALE value for cpu_scale.
71 static const struct cpu_efficiency table_efficiency
[] = {
72 {"arm,cortex-a15", 3891},
73 {"arm,cortex-a7", 2048},
77 static unsigned long *__cpu_capacity
;
78 #define cpu_capacity(cpu) __cpu_capacity[cpu]
80 static unsigned long middle_capacity
= 1;
83 * Iterate all CPUs' descriptor in DT and compute the efficiency
84 * (as per table_efficiency). Also calculate a middle efficiency
85 * as close as possible to (max{eff_i} - min{eff_i}) / 2
86 * This is later used to scale the cpu_capacity field such that an
87 * 'average' CPU is of middle capacity. Also see the comments near
88 * table_efficiency[] and update_cpu_capacity().
90 static void __init
parse_dt_topology(void)
92 const struct cpu_efficiency
*cpu_eff
;
93 struct device_node
*cn
= NULL
;
94 unsigned long min_capacity
= ULONG_MAX
;
95 unsigned long max_capacity
= 0;
96 unsigned long capacity
= 0;
99 __cpu_capacity
= kcalloc(nr_cpu_ids
, sizeof(*__cpu_capacity
),
102 for_each_possible_cpu(cpu
) {
106 /* too early to use cpu->of_node */
107 cn
= of_get_cpu_node(cpu
, NULL
);
109 pr_err("missing device node for CPU %d\n", cpu
);
113 for (cpu_eff
= table_efficiency
; cpu_eff
->compatible
; cpu_eff
++)
114 if (of_device_is_compatible(cn
, cpu_eff
->compatible
))
117 if (cpu_eff
->compatible
== NULL
)
120 rate
= of_get_property(cn
, "clock-frequency", &len
);
121 if (!rate
|| len
!= 4) {
122 pr_err("%s missing clock-frequency property\n",
127 capacity
= ((be32_to_cpup(rate
)) >> 20) * cpu_eff
->efficiency
;
129 /* Save min capacity of the system */
130 if (capacity
< min_capacity
)
131 min_capacity
= capacity
;
133 /* Save max capacity of the system */
134 if (capacity
> max_capacity
)
135 max_capacity
= capacity
;
137 cpu_capacity(cpu
) = capacity
;
140 /* If min and max capacities are equals, we bypass the update of the
141 * cpu_scale because all CPUs have the same capacity. Otherwise, we
142 * compute a middle_capacity factor that will ensure that the capacity
143 * of an 'average' CPU of the system will be as close as possible to
144 * SCHED_CAPACITY_SCALE, which is the default value, but with the
145 * constraint explained near table_efficiency[].
147 if (4*max_capacity
< (3*(max_capacity
+ min_capacity
)))
148 middle_capacity
= (min_capacity
+ max_capacity
)
149 >> (SCHED_CAPACITY_SHIFT
+1);
151 middle_capacity
= ((max_capacity
/ 3)
152 >> (SCHED_CAPACITY_SHIFT
-1)) + 1;
157 * Look for a customed capacity of a CPU in the cpu_capacity table during the
158 * boot. The update of all CPUs is in O(n^2) for heteregeneous system but the
159 * function returns directly for SMP system.
161 static void update_cpu_capacity(unsigned int cpu
)
163 if (!cpu_capacity(cpu
))
166 set_capacity_scale(cpu
, cpu_capacity(cpu
) / middle_capacity
);
168 printk(KERN_INFO
"CPU%u: update cpu_capacity %lu\n",
169 cpu
, arch_scale_freq_capacity(NULL
, cpu
));
173 static inline void parse_dt_topology(void) {}
174 static inline void update_cpu_capacity(unsigned int cpuid
) {}
180 struct cputopo_arm cpu_topology
[NR_CPUS
];
181 EXPORT_SYMBOL_GPL(cpu_topology
);
183 const struct cpumask
*cpu_coregroup_mask(int cpu
)
185 return &cpu_topology
[cpu
].core_sibling
;
189 * The current assumption is that we can power gate each core independently.
190 * This will be superseded by DT binding once available.
192 const struct cpumask
*cpu_corepower_mask(int cpu
)
194 return &cpu_topology
[cpu
].thread_sibling
;
197 static void update_siblings_masks(unsigned int cpuid
)
199 struct cputopo_arm
*cpu_topo
, *cpuid_topo
= &cpu_topology
[cpuid
];
202 /* update core and thread sibling masks */
203 for_each_possible_cpu(cpu
) {
204 cpu_topo
= &cpu_topology
[cpu
];
206 if (cpuid_topo
->socket_id
!= cpu_topo
->socket_id
)
209 cpumask_set_cpu(cpuid
, &cpu_topo
->core_sibling
);
211 cpumask_set_cpu(cpu
, &cpuid_topo
->core_sibling
);
213 if (cpuid_topo
->core_id
!= cpu_topo
->core_id
)
216 cpumask_set_cpu(cpuid
, &cpu_topo
->thread_sibling
);
218 cpumask_set_cpu(cpu
, &cpuid_topo
->thread_sibling
);
224 * store_cpu_topology is called at boot when only one cpu is running
225 * and with the mutex cpu_hotplug.lock locked, when several cpus have booted,
226 * which prevents simultaneous write access to cpu_topology array
228 void store_cpu_topology(unsigned int cpuid
)
230 struct cputopo_arm
*cpuid_topo
= &cpu_topology
[cpuid
];
233 /* If the cpu topology has been already set, just return */
234 if (cpuid_topo
->core_id
!= -1)
237 mpidr
= read_cpuid_mpidr();
239 /* create cpu topology mapping */
240 if ((mpidr
& MPIDR_SMP_BITMASK
) == MPIDR_SMP_VALUE
) {
242 * This is a multiprocessor system
243 * multiprocessor format & multiprocessor mode field are set
246 if (mpidr
& MPIDR_MT_BITMASK
) {
247 /* core performance interdependency */
248 cpuid_topo
->thread_id
= MPIDR_AFFINITY_LEVEL(mpidr
, 0);
249 cpuid_topo
->core_id
= MPIDR_AFFINITY_LEVEL(mpidr
, 1);
250 cpuid_topo
->socket_id
= MPIDR_AFFINITY_LEVEL(mpidr
, 2);
252 /* largely independent cores */
253 cpuid_topo
->thread_id
= -1;
254 cpuid_topo
->core_id
= MPIDR_AFFINITY_LEVEL(mpidr
, 0);
255 cpuid_topo
->socket_id
= MPIDR_AFFINITY_LEVEL(mpidr
, 1);
259 * This is an uniprocessor system
260 * we are in multiprocessor format but uniprocessor system
261 * or in the old uniprocessor format
263 cpuid_topo
->thread_id
= -1;
264 cpuid_topo
->core_id
= 0;
265 cpuid_topo
->socket_id
= -1;
268 update_siblings_masks(cpuid
);
270 update_cpu_capacity(cpuid
);
272 printk(KERN_INFO
"CPU%u: thread %d, cpu %d, socket %d, mpidr %x\n",
273 cpuid
, cpu_topology
[cpuid
].thread_id
,
274 cpu_topology
[cpuid
].core_id
,
275 cpu_topology
[cpuid
].socket_id
, mpidr
);
278 static inline const int cpu_corepower_flags(void)
280 return SD_SHARE_PKG_RESOURCES
| SD_SHARE_POWERDOMAIN
;
283 static struct sched_domain_topology_level arm_topology
[] = {
284 #ifdef CONFIG_SCHED_MC
285 { cpu_corepower_mask
, cpu_corepower_flags
, SD_INIT_NAME(GMC
) },
286 { cpu_coregroup_mask
, cpu_core_flags
, SD_INIT_NAME(MC
) },
288 { cpu_cpu_mask
, SD_INIT_NAME(DIE
) },
293 * init_cpu_topology is called at boot when only one cpu is running
294 * which prevent simultaneous write access to cpu_topology array
296 void __init
init_cpu_topology(void)
300 /* init core mask and capacity */
301 for_each_possible_cpu(cpu
) {
302 struct cputopo_arm
*cpu_topo
= &(cpu_topology
[cpu
]);
304 cpu_topo
->thread_id
= -1;
305 cpu_topo
->core_id
= -1;
306 cpu_topo
->socket_id
= -1;
307 cpumask_clear(&cpu_topo
->core_sibling
);
308 cpumask_clear(&cpu_topo
->thread_sibling
);
310 set_capacity_scale(cpu
, SCHED_CAPACITY_SCALE
);
316 /* Set scheduler topology descriptor */
317 set_sched_topology(arm_topology
);