2 * CCI cache coherent interconnect driver
4 * Copyright (C) 2013 ARM Ltd.
5 * Author: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
11 * This program is distributed "as is" WITHOUT ANY WARRANTY of any
12 * kind, whether express or implied; without even the implied warranty
13 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
17 #include <linux/arm-cci.h>
19 #include <linux/module.h>
20 #include <linux/of_address.h>
21 #include <linux/slab.h>
23 #include <asm/cacheflush.h>
24 #include <asm/smp_plat.h>
26 #define CCI_PORT_CTRL 0x0
27 #define CCI_CTRL_STATUS 0xc
29 #define CCI_ENABLE_SNOOP_REQ 0x1
30 #define CCI_ENABLE_DVM_REQ 0x2
31 #define CCI_ENABLE_REQ (CCI_ENABLE_SNOOP_REQ | CCI_ENABLE_DVM_REQ)
35 unsigned int nb_ace_lite
;
38 enum cci_ace_port_type
{
39 ACE_INVALID_PORT
= 0x0,
47 enum cci_ace_port_type type
;
48 struct device_node
*dn
;
51 static struct cci_ace_port
*ports
;
52 static unsigned int nb_cci_ports
;
54 static void __iomem
*cci_ctrl_base
;
55 static unsigned long cci_ctrl_phys
;
63 * Use the port MSB as valid flag, shift can be made dynamic
64 * by computing number of bits required for port indexes.
65 * Code disabling CCI cpu ports runs with D-cache invalidated
66 * and SCTLR bit clear so data accesses must be kept to a minimum
67 * to improve performance; for now shift is left static to
68 * avoid one more data access while disabling the CCI port.
70 #define PORT_VALID_SHIFT 31
71 #define PORT_VALID (0x1 << PORT_VALID_SHIFT)
73 static inline void init_cpu_port(struct cpu_port
*port
, u32 index
, u64 mpidr
)
75 port
->port
= PORT_VALID
| index
;
79 static inline bool cpu_port_is_valid(struct cpu_port
*port
)
81 return !!(port
->port
& PORT_VALID
);
84 static inline bool cpu_port_match(struct cpu_port
*port
, u64 mpidr
)
86 return port
->mpidr
== (mpidr
& MPIDR_HWID_BITMASK
);
89 static struct cpu_port cpu_port
[NR_CPUS
];
92 * __cci_ace_get_port - Function to retrieve the port index connected to
95 * @dn: device node of the device to look-up
99 * - CCI port index if success
100 * - -ENODEV if failure
102 static int __cci_ace_get_port(struct device_node
*dn
, int type
)
106 struct device_node
*cci_portn
;
108 cci_portn
= of_parse_phandle(dn
, "cci-control-port", 0);
109 for (i
= 0; i
< nb_cci_ports
; i
++) {
110 ace_match
= ports
[i
].type
== type
;
111 if (ace_match
&& cci_portn
== ports
[i
].dn
)
117 int cci_ace_get_port(struct device_node
*dn
)
119 return __cci_ace_get_port(dn
, ACE_LITE_PORT
);
121 EXPORT_SYMBOL_GPL(cci_ace_get_port
);
123 static void __init
cci_ace_init_ports(void)
126 struct device_node
*cpun
;
129 * Port index look-up speeds up the function disabling ports by CPU,
130 * since the logical to port index mapping is done once and does
131 * not change after system boot.
132 * The stashed index array is initialized for all possible CPUs
135 for_each_possible_cpu(cpu
) {
136 /* too early to use cpu->of_node */
137 cpun
= of_get_cpu_node(cpu
, NULL
);
139 if (WARN(!cpun
, "Missing cpu device node\n"))
142 port
= __cci_ace_get_port(cpun
, ACE_PORT
);
146 init_cpu_port(&cpu_port
[cpu
], port
, cpu_logical_map(cpu
));
149 for_each_possible_cpu(cpu
) {
150 WARN(!cpu_port_is_valid(&cpu_port
[cpu
]),
151 "CPU %u does not have an associated CCI port\n",
156 * Functions to enable/disable a CCI interconnect slave port
158 * They are called by low-level power management code to disable slave
159 * interfaces snoops and DVM broadcast.
160 * Since they may execute with cache data allocation disabled and
161 * after the caches have been cleaned and invalidated the functions provide
162 * no explicit locking since they may run with D-cache disabled, so normal
163 * cacheable kernel locks based on ldrex/strex may not work.
164 * Locking has to be provided by BSP implementations to ensure proper
169 * cci_port_control() - function to control a CCI port
171 * @port: index of the port to setup
172 * @enable: if true enables the port, if false disables it
174 static void notrace
cci_port_control(unsigned int port
, bool enable
)
176 void __iomem
*base
= ports
[port
].base
;
178 writel_relaxed(enable
? CCI_ENABLE_REQ
: 0, base
+ CCI_PORT_CTRL
);
180 * This function is called from power down procedures
181 * and must not execute any instruction that might
182 * cause the processor to be put in a quiescent state
183 * (eg wfi). Hence, cpu_relax() can not be added to this
184 * read loop to optimize power, since it might hide possibly
185 * disruptive operations.
187 while (readl_relaxed(cci_ctrl_base
+ CCI_CTRL_STATUS
) & 0x1)
192 * cci_disable_port_by_cpu() - function to disable a CCI port by CPU
195 * @mpidr: mpidr of the CPU whose CCI port should be disabled
197 * Disabling a CCI port for a CPU implies disabling the CCI port
198 * controlling that CPU cluster. Code disabling CPU CCI ports
199 * must make sure that the CPU running the code is the last active CPU
200 * in the cluster ie all other CPUs are quiescent in a low power state.
204 * -ENODEV on port look-up failure
206 int notrace
cci_disable_port_by_cpu(u64 mpidr
)
210 for (cpu
= 0; cpu
< nr_cpu_ids
; cpu
++) {
211 is_valid
= cpu_port_is_valid(&cpu_port
[cpu
]);
212 if (is_valid
&& cpu_port_match(&cpu_port
[cpu
], mpidr
)) {
213 cci_port_control(cpu_port
[cpu
].port
, false);
219 EXPORT_SYMBOL_GPL(cci_disable_port_by_cpu
);
222 * cci_enable_port_for_self() - enable a CCI port for calling CPU
224 * Enabling a CCI port for the calling CPU implies enabling the CCI
225 * port controlling that CPU's cluster. Caller must make sure that the
226 * CPU running the code is the first active CPU in the cluster and all
227 * other CPUs are quiescent in a low power state or waiting for this CPU
228 * to complete the CCI initialization.
230 * Because this is called when the MMU is still off and with no stack,
231 * the code must be position independent and ideally rely on callee
232 * clobbered registers only. To achieve this we must code this function
233 * entirely in assembler.
235 * On success this returns with the proper CCI port enabled. In case of
236 * any failure this never returns as the inability to enable the CCI is
237 * fatal and there is no possible recovery at this stage.
239 asmlinkage
void __naked
cci_enable_port_for_self(void)
243 " mrc p15, 0, r0, c0, c0, 5 @ get MPIDR value \n"
244 " and r0, r0, #"__stringify(MPIDR_HWID_BITMASK
)" \n"
247 " add r1, r1, r2 @ &cpu_port \n"
248 " add ip, r1, %[sizeof_cpu_port] \n"
250 /* Loop over the cpu_port array looking for a matching MPIDR */
251 "1: ldr r2, [r1, %[offsetof_cpu_port_mpidr_lsb]] \n"
252 " cmp r2, r0 @ compare MPIDR \n"
255 /* Found a match, now test port validity */
256 " ldr r3, [r1, %[offsetof_cpu_port_port]] \n"
257 " tst r3, #"__stringify(PORT_VALID
)" \n"
260 /* no match, loop with the next cpu_port entry */
261 "2: add r1, r1, %[sizeof_struct_cpu_port] \n"
262 " cmp r1, ip @ done? \n"
265 /* CCI port not found -- cheaply try to stall this CPU */
266 "cci_port_not_found: \n"
269 " b cci_port_not_found \n"
271 /* Use matched port index to look up the corresponding ports entry */
272 "3: bic r3, r3, #"__stringify(PORT_VALID
)" \n"
274 " ldmia r0, {r1, r2} \n"
275 " sub r1, r1, r0 @ virt - phys \n"
276 " ldr r0, [r0, r2] @ *(&ports) \n"
277 " mov r2, %[sizeof_struct_ace_port] \n"
278 " mla r0, r2, r3, r0 @ &ports[index] \n"
279 " sub r0, r0, r1 @ virt_to_phys() \n"
281 /* Enable the CCI port */
282 " ldr r0, [r0, %[offsetof_port_phys]] \n"
283 " mov r3, #"__stringify(CCI_ENABLE_REQ
)" \n"
284 " str r3, [r0, #"__stringify(CCI_PORT_CTRL
)"] \n"
286 /* poll the status reg for completion */
289 " ldr r0, [r0, r1] @ cci_ctrl_base \n"
290 "4: ldr r1, [r0, #"__stringify(CCI_CTRL_STATUS
)"] \n"
298 "5: .word cpu_port - . \n"
300 " .word ports - 6b \n"
301 "7: .word cci_ctrl_phys - . \n"
303 [sizeof_cpu_port
] "i" (sizeof(cpu_port
)),
305 [offsetof_cpu_port_mpidr_lsb
] "i" (offsetof(struct cpu_port
, mpidr
)),
307 [offsetof_cpu_port_mpidr_lsb
] "i" (offsetof(struct cpu_port
, mpidr
)+4),
309 [offsetof_cpu_port_port
] "i" (offsetof(struct cpu_port
, port
)),
310 [sizeof_struct_cpu_port
] "i" (sizeof(struct cpu_port
)),
311 [sizeof_struct_ace_port
] "i" (sizeof(struct cci_ace_port
)),
312 [offsetof_port_phys
] "i" (offsetof(struct cci_ace_port
, phys
)) );
318 * __cci_control_port_by_device() - function to control a CCI port by device
321 * @dn: device node pointer of the device whose CCI port should be
323 * @enable: if true enables the port, if false disables it
327 * -ENODEV on port look-up failure
329 int notrace
__cci_control_port_by_device(struct device_node
*dn
, bool enable
)
336 port
= __cci_ace_get_port(dn
, ACE_LITE_PORT
);
337 if (WARN_ONCE(port
< 0, "node %s ACE lite port look-up failure\n",
340 cci_port_control(port
, enable
);
343 EXPORT_SYMBOL_GPL(__cci_control_port_by_device
);
346 * __cci_control_port_by_index() - function to control a CCI port by port index
348 * @port: port index previously retrieved with cci_ace_get_port()
349 * @enable: if true enables the port, if false disables it
353 * -ENODEV on port index out of range
354 * -EPERM if operation carried out on an ACE PORT
356 int notrace
__cci_control_port_by_index(u32 port
, bool enable
)
358 if (port
>= nb_cci_ports
|| ports
[port
].type
== ACE_INVALID_PORT
)
361 * CCI control for ports connected to CPUS is extremely fragile
362 * and must be made to go through a specific and controlled
363 * interface (ie cci_disable_port_by_cpu(); control by general purpose
364 * indexing is therefore disabled for ACE ports.
366 if (ports
[port
].type
== ACE_PORT
)
369 cci_port_control(port
, enable
);
372 EXPORT_SYMBOL_GPL(__cci_control_port_by_index
);
374 static const struct cci_nb_ports cci400_ports
= {
379 static const struct of_device_id arm_cci_matches
[] = {
380 {.compatible
= "arm,cci-400", .data
= &cci400_ports
},
384 static const struct of_device_id arm_cci_ctrl_if_matches
[] = {
385 {.compatible
= "arm,cci-400-ctrl-if", },
389 static int __init
cci_probe(void)
391 struct cci_nb_ports
const *cci_config
;
392 int ret
, i
, nb_ace
= 0, nb_ace_lite
= 0;
393 struct device_node
*np
, *cp
;
395 const char *match_str
;
398 np
= of_find_matching_node(NULL
, arm_cci_matches
);
402 cci_config
= of_match_node(arm_cci_matches
, np
)->data
;
406 nb_cci_ports
= cci_config
->nb_ace
+ cci_config
->nb_ace_lite
;
408 ports
= kcalloc(sizeof(*ports
), nb_cci_ports
, GFP_KERNEL
);
412 ret
= of_address_to_resource(np
, 0, &res
);
414 cci_ctrl_base
= ioremap(res
.start
, resource_size(&res
));
415 cci_ctrl_phys
= res
.start
;
417 if (ret
|| !cci_ctrl_base
) {
418 WARN(1, "unable to ioremap CCI ctrl\n");
423 for_each_child_of_node(np
, cp
) {
424 if (!of_match_node(arm_cci_ctrl_if_matches
, cp
))
427 i
= nb_ace
+ nb_ace_lite
;
429 if (i
>= nb_cci_ports
)
432 if (of_property_read_string(cp
, "interface-type",
434 WARN(1, "node %s missing interface-type property\n",
438 is_ace
= strcmp(match_str
, "ace") == 0;
439 if (!is_ace
&& strcmp(match_str
, "ace-lite")) {
440 WARN(1, "node %s containing invalid interface-type property, skipping it\n",
445 ret
= of_address_to_resource(cp
, 0, &res
);
447 ports
[i
].base
= ioremap(res
.start
, resource_size(&res
));
448 ports
[i
].phys
= res
.start
;
450 if (ret
|| !ports
[i
].base
) {
451 WARN(1, "unable to ioremap CCI port %d\n", i
);
456 if (WARN_ON(nb_ace
>= cci_config
->nb_ace
))
458 ports
[i
].type
= ACE_PORT
;
461 if (WARN_ON(nb_ace_lite
>= cci_config
->nb_ace_lite
))
463 ports
[i
].type
= ACE_LITE_PORT
;
469 /* initialize a stashed array of ACE ports to speed-up look-up */
470 cci_ace_init_ports();
473 * Multi-cluster systems may need this data when non-coherent, during
474 * cluster power-up/power-down. Make sure it reaches main memory.
476 sync_cache_w(&cci_ctrl_base
);
477 sync_cache_w(&cci_ctrl_phys
);
478 sync_cache_w(&ports
);
479 sync_cache_w(&cpu_port
);
480 __sync_cache_range_w(ports
, sizeof(*ports
) * nb_cci_ports
);
481 pr_info("ARM CCI driver probed\n");
490 static int cci_init_status
= -EAGAIN
;
491 static DEFINE_MUTEX(cci_probing
);
493 static int __init
cci_init(void)
495 if (cci_init_status
!= -EAGAIN
)
496 return cci_init_status
;
498 mutex_lock(&cci_probing
);
499 if (cci_init_status
== -EAGAIN
)
500 cci_init_status
= cci_probe();
501 mutex_unlock(&cci_probing
);
502 return cci_init_status
;
506 * To sort out early init calls ordering a helper function is provided to
507 * check if the CCI driver has beed initialized. Function check if the driver
508 * has been initialized, if not it calls the init function that probes
509 * the driver and updates the return value.
511 bool __init
cci_probed(void)
513 return cci_init() == 0;
515 EXPORT_SYMBOL_GPL(cci_probed
);
517 early_initcall(cci_init
);
518 MODULE_LICENSE("GPL");
519 MODULE_DESCRIPTION("ARM CCI support");