1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (C) 2012-2019 ARM Limited or its affiliates. */
4 #include <linux/kernel.h>
5 #include <linux/module.h>
7 #include <linux/crypto.h>
8 #include <linux/moduleparam.h>
9 #include <linux/types.h>
10 #include <linux/interrupt.h>
11 #include <linux/platform_device.h>
12 #include <linux/slab.h>
13 #include <linux/spinlock.h>
15 #include <linux/clk.h>
16 #include <linux/of_address.h>
17 #include <linux/of_device.h>
18 #include <linux/pm_runtime.h>
20 #include "cc_driver.h"
21 #include "cc_request_mgr.h"
22 #include "cc_buffer_mgr.h"
23 #include "cc_debugfs.h"
24 #include "cc_cipher.h"
27 #include "cc_sram_mgr.h"
32 module_param_named(dump_desc
, cc_dump_desc
, bool, 0600);
33 MODULE_PARM_DESC(cc_dump_desc
, "Dump descriptors to kernel log as debugging aid");
35 module_param_named(dump_bytes
, cc_dump_bytes
, bool, 0600);
36 MODULE_PARM_DESC(cc_dump_bytes
, "Dump buffers to kernel log as debugging aid");
38 static bool cc_sec_disable
;
39 module_param_named(sec_disable
, cc_sec_disable
, bool, 0600);
40 MODULE_PARM_DESC(cc_sec_disable
, "Disable security functions");
52 #define CC_HW_RESET_LOOP_COUNT 10
54 /* Note: PIDR3 holds CMOD/Rev so ignored for HW identification purposes */
55 static const u32 pidr_0124_offsets
[CC_NUM_IDRS
] = {
56 CC_REG(PERIPHERAL_ID_0
), CC_REG(PERIPHERAL_ID_1
),
57 CC_REG(PERIPHERAL_ID_2
), CC_REG(PERIPHERAL_ID_4
)
60 static const u32 cidr_0123_offsets
[CC_NUM_IDRS
] = {
61 CC_REG(COMPONENT_ID_0
), CC_REG(COMPONENT_ID_1
),
62 CC_REG(COMPONENT_ID_2
), CC_REG(COMPONENT_ID_3
)
65 /* Hardware revisions defs. */
67 /* The 703 is a OSCCA only variant of the 713 */
68 static const struct cc_hw_data cc703_hw
= {
69 .name
= "703", .rev
= CC_HW_REV_713
, .cidr_0123
= 0xB105F00DU
,
70 .pidr_0124
= 0x040BB0D0U
, .std_bodies
= CC_STD_OSCCA
73 static const struct cc_hw_data cc713_hw
= {
74 .name
= "713", .rev
= CC_HW_REV_713
, .cidr_0123
= 0xB105F00DU
,
75 .pidr_0124
= 0x040BB0D0U
, .std_bodies
= CC_STD_ALL
78 static const struct cc_hw_data cc712_hw
= {
79 .name
= "712", .rev
= CC_HW_REV_712
, .sig
= 0xDCC71200U
,
80 .std_bodies
= CC_STD_ALL
83 static const struct cc_hw_data cc710_hw
= {
84 .name
= "710", .rev
= CC_HW_REV_710
, .sig
= 0xDCC63200U
,
85 .std_bodies
= CC_STD_ALL
88 static const struct cc_hw_data cc630p_hw
= {
89 .name
= "630P", .rev
= CC_HW_REV_630
, .sig
= 0xDCC63000U
,
90 .std_bodies
= CC_STD_ALL
93 static const struct of_device_id arm_ccree_dev_of_match
[] = {
94 { .compatible
= "arm,cryptocell-703-ree", .data
= &cc703_hw
},
95 { .compatible
= "arm,cryptocell-713-ree", .data
= &cc713_hw
},
96 { .compatible
= "arm,cryptocell-712-ree", .data
= &cc712_hw
},
97 { .compatible
= "arm,cryptocell-710-ree", .data
= &cc710_hw
},
98 { .compatible
= "arm,cryptocell-630p-ree", .data
= &cc630p_hw
},
101 MODULE_DEVICE_TABLE(of
, arm_ccree_dev_of_match
);
103 static u32
cc_read_idr(struct cc_drvdata
*drvdata
, const u32
*idr_offsets
)
107 u8 regs
[CC_NUM_IDRS
];
111 for (i
= 0; i
< CC_NUM_IDRS
; ++i
)
112 idr
.regs
[i
] = cc_ioread(drvdata
, idr_offsets
[i
]);
114 return le32_to_cpu(idr
.val
);
117 void __dump_byte_array(const char *name
, const u8
*buf
, size_t len
)
124 snprintf(prefix
, sizeof(prefix
), "%s[%zu]: ", name
, len
);
126 print_hex_dump(KERN_DEBUG
, prefix
, DUMP_PREFIX_ADDRESS
, 16, 1, buf
,
130 static irqreturn_t
cc_isr(int irq
, void *dev_id
)
132 struct cc_drvdata
*drvdata
= (struct cc_drvdata
*)dev_id
;
133 struct device
*dev
= drvdata_to_dev(drvdata
);
137 /* STAT_OP_TYPE_GENERIC STAT_PHASE_0: Interrupt */
138 /* if driver suspended return, probably shared interrupt */
139 if (pm_runtime_suspended(dev
))
142 /* read the interrupt status */
143 irr
= cc_ioread(drvdata
, CC_REG(HOST_IRR
));
144 dev_dbg(dev
, "Got IRR=0x%08X\n", irr
);
146 if (irr
== 0) /* Probably shared interrupt line */
149 imr
= cc_ioread(drvdata
, CC_REG(HOST_IMR
));
151 /* clear interrupt - must be before processing events */
152 cc_iowrite(drvdata
, CC_REG(HOST_ICR
), irr
);
155 /* Completion interrupt - most probable */
156 if (irr
& drvdata
->comp_mask
) {
157 /* Mask all completion interrupts - will be unmasked in
158 * deferred service handler
160 cc_iowrite(drvdata
, CC_REG(HOST_IMR
), imr
| drvdata
->comp_mask
);
161 irr
&= ~drvdata
->comp_mask
;
162 complete_request(drvdata
);
164 #ifdef CONFIG_CRYPTO_FIPS
165 /* TEE FIPS interrupt */
166 if (irr
& CC_GPR0_IRQ_MASK
) {
167 /* Mask interrupt - will be unmasked in Deferred service
170 cc_iowrite(drvdata
, CC_REG(HOST_IMR
), imr
| CC_GPR0_IRQ_MASK
);
171 irr
&= ~CC_GPR0_IRQ_MASK
;
172 fips_handler(drvdata
);
175 /* AXI error interrupt */
176 if (irr
& CC_AXI_ERR_IRQ_MASK
) {
179 /* Read the AXI error ID */
180 axi_err
= cc_ioread(drvdata
, CC_REG(AXIM_MON_ERR
));
181 dev_dbg(dev
, "AXI completion error: axim_mon_err=0x%08X\n",
184 irr
&= ~CC_AXI_ERR_IRQ_MASK
;
188 dev_dbg_ratelimited(dev
, "IRR includes unknown cause bits (0x%08X)\n",
196 bool cc_wait_for_reset_completion(struct cc_drvdata
*drvdata
)
201 /* 712/710/63 has no reset completion indication, always return true */
202 if (drvdata
->hw_rev
<= CC_HW_REV_712
)
205 for (i
= 0; i
< CC_HW_RESET_LOOP_COUNT
; i
++) {
206 /* in cc7x3 NVM_IS_IDLE indicates that CC reset is
207 * completed and device is fully functional
209 val
= cc_ioread(drvdata
, CC_REG(NVM_IS_IDLE
));
210 if (val
& CC_NVM_IS_IDLE_MASK
) {
211 /* hw indicate reset completed */
214 /* allow scheduling other process on the processor */
217 /* reset not completed */
221 int init_cc_regs(struct cc_drvdata
*drvdata
, bool is_probe
)
223 unsigned int val
, cache_params
;
224 struct device
*dev
= drvdata_to_dev(drvdata
);
226 /* Unmask all AXI interrupt sources AXI_CFG1 register */
227 /* AXI interrupt config are obsoleted startign at cc7x3 */
228 if (drvdata
->hw_rev
<= CC_HW_REV_712
) {
229 val
= cc_ioread(drvdata
, CC_REG(AXIM_CFG
));
230 cc_iowrite(drvdata
, CC_REG(AXIM_CFG
), val
& ~CC_AXI_IRQ_MASK
);
231 dev_dbg(dev
, "AXIM_CFG=0x%08X\n",
232 cc_ioread(drvdata
, CC_REG(AXIM_CFG
)));
235 /* Clear all pending interrupts */
236 val
= cc_ioread(drvdata
, CC_REG(HOST_IRR
));
237 dev_dbg(dev
, "IRR=0x%08X\n", val
);
238 cc_iowrite(drvdata
, CC_REG(HOST_ICR
), val
);
240 /* Unmask relevant interrupt cause */
241 val
= drvdata
->comp_mask
| CC_AXI_ERR_IRQ_MASK
;
243 if (drvdata
->hw_rev
>= CC_HW_REV_712
)
244 val
|= CC_GPR0_IRQ_MASK
;
246 cc_iowrite(drvdata
, CC_REG(HOST_IMR
), ~val
);
248 cache_params
= (drvdata
->coherent
? CC_COHERENT_CACHE_PARAMS
: 0x0);
250 val
= cc_ioread(drvdata
, CC_REG(AXIM_CACHE_PARAMS
));
253 dev_dbg(dev
, "Cache params previous: 0x%08X\n", val
);
255 cc_iowrite(drvdata
, CC_REG(AXIM_CACHE_PARAMS
), cache_params
);
256 val
= cc_ioread(drvdata
, CC_REG(AXIM_CACHE_PARAMS
));
259 dev_dbg(dev
, "Cache params current: 0x%08X (expect: 0x%08X)\n",
265 static int init_cc_resources(struct platform_device
*plat_dev
)
267 struct resource
*req_mem_cc_regs
= NULL
;
268 struct cc_drvdata
*new_drvdata
;
269 struct device
*dev
= &plat_dev
->dev
;
270 struct device_node
*np
= dev
->of_node
;
271 u32 val
, hw_rev_pidr
, sig_cidr
;
273 const struct cc_hw_data
*hw_rev
;
278 new_drvdata
= devm_kzalloc(dev
, sizeof(*new_drvdata
), GFP_KERNEL
);
282 hw_rev
= of_device_get_match_data(dev
);
283 new_drvdata
->hw_rev_name
= hw_rev
->name
;
284 new_drvdata
->hw_rev
= hw_rev
->rev
;
285 new_drvdata
->std_bodies
= hw_rev
->std_bodies
;
287 if (hw_rev
->rev
>= CC_HW_REV_712
) {
288 new_drvdata
->axim_mon_offset
= CC_REG(AXIM_MON_COMP
);
289 new_drvdata
->sig_offset
= CC_REG(HOST_SIGNATURE_712
);
290 new_drvdata
->ver_offset
= CC_REG(HOST_VERSION_712
);
292 new_drvdata
->axim_mon_offset
= CC_REG(AXIM_MON_COMP8
);
293 new_drvdata
->sig_offset
= CC_REG(HOST_SIGNATURE_630
);
294 new_drvdata
->ver_offset
= CC_REG(HOST_VERSION_630
);
297 new_drvdata
->comp_mask
= CC_COMP_IRQ_MASK
;
299 platform_set_drvdata(plat_dev
, new_drvdata
);
300 new_drvdata
->plat_dev
= plat_dev
;
302 clk
= devm_clk_get_optional(dev
, NULL
);
304 if (PTR_ERR(clk
) != -EPROBE_DEFER
)
305 dev_err(dev
, "Error getting clock: %pe\n", clk
);
308 new_drvdata
->clk
= clk
;
310 new_drvdata
->coherent
= of_dma_is_coherent(np
);
312 /* Get device resources */
313 /* First CC registers space */
314 req_mem_cc_regs
= platform_get_resource(plat_dev
, IORESOURCE_MEM
, 0);
315 /* Map registers space */
316 new_drvdata
->cc_base
= devm_ioremap_resource(dev
, req_mem_cc_regs
);
317 if (IS_ERR(new_drvdata
->cc_base
)) {
318 dev_err(dev
, "Failed to ioremap registers");
319 return PTR_ERR(new_drvdata
->cc_base
);
322 dev_dbg(dev
, "Got MEM resource (%s): %pR\n", req_mem_cc_regs
->name
,
324 dev_dbg(dev
, "CC registers mapped from %pa to 0x%p\n",
325 &req_mem_cc_regs
->start
, new_drvdata
->cc_base
);
328 irq
= platform_get_irq(plat_dev
, 0);
332 init_completion(&new_drvdata
->hw_queue_avail
);
335 dev
->dma_mask
= &dev
->coherent_dma_mask
;
337 dma_mask
= DMA_BIT_MASK(DMA_BIT_MASK_LEN
);
338 while (dma_mask
> 0x7fffffffUL
) {
339 if (dma_supported(dev
, dma_mask
)) {
340 rc
= dma_set_coherent_mask(dev
, dma_mask
);
348 dev_err(dev
, "Failed in dma_set_mask, mask=%llx\n", dma_mask
);
352 rc
= clk_prepare_enable(new_drvdata
->clk
);
354 dev_err(dev
, "Failed to enable clock");
358 new_drvdata
->sec_disabled
= cc_sec_disable
;
360 pm_runtime_set_autosuspend_delay(dev
, CC_SUSPEND_TIMEOUT
);
361 pm_runtime_use_autosuspend(dev
);
362 pm_runtime_set_active(dev
);
363 pm_runtime_enable(dev
);
364 rc
= pm_runtime_get_sync(dev
);
366 dev_err(dev
, "pm_runtime_get_sync() failed: %d\n", rc
);
370 /* Wait for Cryptocell reset completion */
371 if (!cc_wait_for_reset_completion(new_drvdata
)) {
372 dev_err(dev
, "Cryptocell reset not completed");
375 if (hw_rev
->rev
<= CC_HW_REV_712
) {
376 /* Verify correct mapping */
377 val
= cc_ioread(new_drvdata
, new_drvdata
->sig_offset
);
378 if (val
!= hw_rev
->sig
) {
379 dev_err(dev
, "Invalid CC signature: SIGNATURE=0x%08X != expected=0x%08X\n",
385 hw_rev_pidr
= cc_ioread(new_drvdata
, new_drvdata
->ver_offset
);
387 /* Verify correct mapping */
388 val
= cc_read_idr(new_drvdata
, pidr_0124_offsets
);
389 if (val
!= hw_rev
->pidr_0124
) {
390 dev_err(dev
, "Invalid CC PIDR: PIDR0124=0x%08X != expected=0x%08X\n",
391 val
, hw_rev
->pidr_0124
);
397 val
= cc_read_idr(new_drvdata
, cidr_0123_offsets
);
398 if (val
!= hw_rev
->cidr_0123
) {
399 dev_err(dev
, "Invalid CC CIDR: CIDR0123=0x%08X != expected=0x%08X\n",
400 val
, hw_rev
->cidr_0123
);
406 /* Check HW engine configuration */
407 val
= cc_ioread(new_drvdata
, CC_REG(HOST_REMOVE_INPUT_PINS
));
413 if (new_drvdata
->std_bodies
& CC_STD_NIST
) {
414 dev_warn(dev
, "703 mode forced due to HW configuration.\n");
415 new_drvdata
->std_bodies
= CC_STD_OSCCA
;
419 dev_err(dev
, "Unsupported engines configuration.\n");
424 /* Check security disable state */
425 val
= cc_ioread(new_drvdata
, CC_REG(SECURITY_DISABLED
));
426 val
&= CC_SECURITY_DISABLED_MASK
;
427 new_drvdata
->sec_disabled
|= !!val
;
429 if (!new_drvdata
->sec_disabled
) {
430 new_drvdata
->comp_mask
|= CC_CPP_SM4_ABORT_MASK
;
431 if (new_drvdata
->std_bodies
& CC_STD_NIST
)
432 new_drvdata
->comp_mask
|= CC_CPP_AES_ABORT_MASK
;
436 if (new_drvdata
->sec_disabled
)
437 dev_info(dev
, "Security Disabled mode is in effect. Security functions disabled.\n");
439 /* Display HW versions */
440 dev_info(dev
, "ARM CryptoCell %s Driver: HW version 0x%08X/0x%8X, Driver version %s\n",
441 hw_rev
->name
, hw_rev_pidr
, sig_cidr
, DRV_MODULE_VERSION
);
442 /* register the driver isr function */
443 rc
= devm_request_irq(dev
, irq
, cc_isr
, IRQF_SHARED
, "ccree",
446 dev_err(dev
, "Could not register to interrupt %d\n", irq
);
449 dev_dbg(dev
, "Registered to IRQ: %d\n", irq
);
451 rc
= init_cc_regs(new_drvdata
, true);
453 dev_err(dev
, "init_cc_regs failed\n");
457 rc
= cc_debugfs_init(new_drvdata
);
459 dev_err(dev
, "Failed registering debugfs interface\n");
463 rc
= cc_fips_init(new_drvdata
);
465 dev_err(dev
, "cc_fips_init failed 0x%x\n", rc
);
466 goto post_debugfs_err
;
468 rc
= cc_sram_mgr_init(new_drvdata
);
470 dev_err(dev
, "cc_sram_mgr_init failed\n");
471 goto post_fips_init_err
;
474 new_drvdata
->mlli_sram_addr
=
475 cc_sram_alloc(new_drvdata
, MAX_MLLI_BUFF_SIZE
);
476 if (new_drvdata
->mlli_sram_addr
== NULL_SRAM_ADDR
) {
478 goto post_fips_init_err
;
481 rc
= cc_req_mgr_init(new_drvdata
);
483 dev_err(dev
, "cc_req_mgr_init failed\n");
484 goto post_fips_init_err
;
487 rc
= cc_buffer_mgr_init(new_drvdata
);
489 dev_err(dev
, "cc_buffer_mgr_init failed\n");
490 goto post_req_mgr_err
;
493 /* Allocate crypto algs */
494 rc
= cc_cipher_alloc(new_drvdata
);
496 dev_err(dev
, "cc_cipher_alloc failed\n");
497 goto post_buf_mgr_err
;
500 /* hash must be allocated before aead since hash exports APIs */
501 rc
= cc_hash_alloc(new_drvdata
);
503 dev_err(dev
, "cc_hash_alloc failed\n");
504 goto post_cipher_err
;
507 rc
= cc_aead_alloc(new_drvdata
);
509 dev_err(dev
, "cc_aead_alloc failed\n");
513 /* If we got here and FIPS mode is enabled
514 * it means all FIPS test passed, so let TEE
517 cc_set_ree_fips_status(new_drvdata
, true);
523 cc_hash_free(new_drvdata
);
525 cc_cipher_free(new_drvdata
);
527 cc_buffer_mgr_fini(new_drvdata
);
529 cc_req_mgr_fini(new_drvdata
);
531 cc_fips_fini(new_drvdata
);
533 cc_debugfs_fini(new_drvdata
);
535 fini_cc_regs(new_drvdata
);
537 pm_runtime_put_noidle(dev
);
538 pm_runtime_disable(dev
);
539 pm_runtime_set_suspended(dev
);
540 clk_disable_unprepare(new_drvdata
->clk
);
544 void fini_cc_regs(struct cc_drvdata
*drvdata
)
546 /* Mask all interrupts */
547 cc_iowrite(drvdata
, CC_REG(HOST_IMR
), 0xFFFFFFFF);
550 static void cleanup_cc_resources(struct platform_device
*plat_dev
)
552 struct device
*dev
= &plat_dev
->dev
;
553 struct cc_drvdata
*drvdata
=
554 (struct cc_drvdata
*)platform_get_drvdata(plat_dev
);
556 cc_aead_free(drvdata
);
557 cc_hash_free(drvdata
);
558 cc_cipher_free(drvdata
);
559 cc_buffer_mgr_fini(drvdata
);
560 cc_req_mgr_fini(drvdata
);
561 cc_fips_fini(drvdata
);
562 cc_debugfs_fini(drvdata
);
563 fini_cc_regs(drvdata
);
564 pm_runtime_put_noidle(dev
);
565 pm_runtime_disable(dev
);
566 pm_runtime_set_suspended(dev
);
567 clk_disable_unprepare(drvdata
->clk
);
570 unsigned int cc_get_default_hash_len(struct cc_drvdata
*drvdata
)
572 if (drvdata
->hw_rev
>= CC_HW_REV_712
)
573 return HASH_LEN_SIZE_712
;
575 return HASH_LEN_SIZE_630
;
578 static int ccree_probe(struct platform_device
*plat_dev
)
581 struct device
*dev
= &plat_dev
->dev
;
583 /* Map registers space */
584 rc
= init_cc_resources(plat_dev
);
588 dev_info(dev
, "ARM ccree device initialized\n");
593 static int ccree_remove(struct platform_device
*plat_dev
)
595 struct device
*dev
= &plat_dev
->dev
;
597 dev_dbg(dev
, "Releasing ccree resources...\n");
599 cleanup_cc_resources(plat_dev
);
601 dev_info(dev
, "ARM ccree device terminated\n");
606 static struct platform_driver ccree_driver
= {
609 .of_match_table
= arm_ccree_dev_of_match
,
614 .probe
= ccree_probe
,
615 .remove
= ccree_remove
,
618 static int __init
ccree_init(void)
620 cc_debugfs_global_init();
622 return platform_driver_register(&ccree_driver
);
624 module_init(ccree_init
);
626 static void __exit
ccree_exit(void)
628 platform_driver_unregister(&ccree_driver
);
629 cc_debugfs_global_fini();
631 module_exit(ccree_exit
);
633 /* Module description */
634 MODULE_DESCRIPTION("ARM TrustZone CryptoCell REE Driver");
635 MODULE_VERSION(DRV_MODULE_VERSION
);
636 MODULE_AUTHOR("ARM");
637 MODULE_LICENSE("GPL v2");