1 // SPDX-License-Identifier: GPL-2.0-only
3 * AMD Cryptographic Coprocessor (CCP) driver
5 * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
7 * Author: Tom Lendacky <thomas.lendacky@amd.com>
8 * Author: Gary R Hook <gary.hook@amd.com>
11 #include <linux/module.h>
12 #include <linux/kernel.h>
13 #include <linux/kthread.h>
14 #include <linux/interrupt.h>
15 #include <linux/ccp.h>
19 static u32
ccp_alloc_ksb(struct ccp_cmd_queue
*cmd_q
, unsigned int count
)
22 struct ccp_device
*ccp
= cmd_q
->ccp
;
25 mutex_lock(&ccp
->sb_mutex
);
27 start
= (u32
)bitmap_find_next_zero_area(ccp
->sb
,
31 if (start
<= ccp
->sb_count
) {
32 bitmap_set(ccp
->sb
, start
, count
);
34 mutex_unlock(&ccp
->sb_mutex
);
40 mutex_unlock(&ccp
->sb_mutex
);
42 /* Wait for KSB entries to become available */
43 if (wait_event_interruptible(ccp
->sb_queue
, ccp
->sb_avail
))
47 return KSB_START
+ start
;
50 static void ccp_free_ksb(struct ccp_cmd_queue
*cmd_q
, unsigned int start
,
53 struct ccp_device
*ccp
= cmd_q
->ccp
;
58 mutex_lock(&ccp
->sb_mutex
);
60 bitmap_clear(ccp
->sb
, start
- KSB_START
, count
);
64 mutex_unlock(&ccp
->sb_mutex
);
66 wake_up_interruptible_all(&ccp
->sb_queue
);
69 static unsigned int ccp_get_free_slots(struct ccp_cmd_queue
*cmd_q
)
71 return CMD_Q_DEPTH(ioread32(cmd_q
->reg_status
));
74 static int ccp_do_cmd(struct ccp_op
*op
, u32
*cr
, unsigned int cr_count
)
76 struct ccp_cmd_queue
*cmd_q
= op
->cmd_q
;
77 struct ccp_device
*ccp
= cmd_q
->ccp
;
78 void __iomem
*cr_addr
;
83 /* We could read a status register to see how many free slots
84 * are actually available, but reading that register resets it
85 * and you could lose some error information.
89 cr0
= (cmd_q
->id
<< REQ0_CMD_Q_SHIFT
)
90 | (op
->jobid
<< REQ0_JOBID_SHIFT
)
91 | REQ0_WAIT_FOR_WRITE
;
94 cr0
|= REQ0_STOP_ON_COMPLETE
95 | REQ0_INT_ON_COMPLETE
;
97 if (op
->ioc
|| !cmd_q
->free_slots
)
98 cr0
|= REQ0_INT_ON_COMPLETE
;
100 /* Start at CMD_REQ1 */
101 cr_addr
= ccp
->io_regs
+ CMD_REQ0
+ CMD_REQ_INCR
;
103 mutex_lock(&ccp
->req_mutex
);
105 /* Write CMD_REQ1 through CMD_REQx first */
106 for (i
= 0; i
< cr_count
; i
++, cr_addr
+= CMD_REQ_INCR
)
107 iowrite32(*(cr
+ i
), cr_addr
);
109 /* Tell the CCP to start */
111 iowrite32(cr0
, ccp
->io_regs
+ CMD_REQ0
);
113 mutex_unlock(&ccp
->req_mutex
);
115 if (cr0
& REQ0_INT_ON_COMPLETE
) {
116 /* Wait for the job to complete */
117 ret
= wait_event_interruptible(cmd_q
->int_queue
,
119 if (ret
|| cmd_q
->cmd_error
) {
120 /* On error delete all related jobs from the queue */
121 cmd
= (cmd_q
->id
<< DEL_Q_ID_SHIFT
)
123 if (cmd_q
->cmd_error
)
124 ccp_log_error(cmd_q
->ccp
,
127 iowrite32(cmd
, ccp
->io_regs
+ DEL_CMD_Q_JOB
);
131 } else if (op
->soc
) {
132 /* Delete just head job from the queue on SoC */
134 | (cmd_q
->id
<< DEL_Q_ID_SHIFT
)
137 iowrite32(cmd
, ccp
->io_regs
+ DEL_CMD_Q_JOB
);
140 cmd_q
->free_slots
= CMD_Q_DEPTH(cmd_q
->q_status
);
148 static int ccp_perform_aes(struct ccp_op
*op
)
152 /* Fill out the register contents for REQ1 through REQ6 */
153 cr
[0] = (CCP_ENGINE_AES
<< REQ1_ENGINE_SHIFT
)
154 | (op
->u
.aes
.type
<< REQ1_AES_TYPE_SHIFT
)
155 | (op
->u
.aes
.mode
<< REQ1_AES_MODE_SHIFT
)
156 | (op
->u
.aes
.action
<< REQ1_AES_ACTION_SHIFT
)
157 | (op
->sb_key
<< REQ1_KEY_KSB_SHIFT
);
158 cr
[1] = op
->src
.u
.dma
.length
- 1;
159 cr
[2] = ccp_addr_lo(&op
->src
.u
.dma
);
160 cr
[3] = (op
->sb_ctx
<< REQ4_KSB_SHIFT
)
161 | (CCP_MEMTYPE_SYSTEM
<< REQ4_MEMTYPE_SHIFT
)
162 | ccp_addr_hi(&op
->src
.u
.dma
);
163 cr
[4] = ccp_addr_lo(&op
->dst
.u
.dma
);
164 cr
[5] = (CCP_MEMTYPE_SYSTEM
<< REQ6_MEMTYPE_SHIFT
)
165 | ccp_addr_hi(&op
->dst
.u
.dma
);
167 if (op
->u
.aes
.mode
== CCP_AES_MODE_CFB
)
168 cr
[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT
);
176 return ccp_do_cmd(op
, cr
, ARRAY_SIZE(cr
));
179 static int ccp_perform_xts_aes(struct ccp_op
*op
)
183 /* Fill out the register contents for REQ1 through REQ6 */
184 cr
[0] = (CCP_ENGINE_XTS_AES_128
<< REQ1_ENGINE_SHIFT
)
185 | (op
->u
.xts
.action
<< REQ1_AES_ACTION_SHIFT
)
186 | (op
->u
.xts
.unit_size
<< REQ1_XTS_AES_SIZE_SHIFT
)
187 | (op
->sb_key
<< REQ1_KEY_KSB_SHIFT
);
188 cr
[1] = op
->src
.u
.dma
.length
- 1;
189 cr
[2] = ccp_addr_lo(&op
->src
.u
.dma
);
190 cr
[3] = (op
->sb_ctx
<< REQ4_KSB_SHIFT
)
191 | (CCP_MEMTYPE_SYSTEM
<< REQ4_MEMTYPE_SHIFT
)
192 | ccp_addr_hi(&op
->src
.u
.dma
);
193 cr
[4] = ccp_addr_lo(&op
->dst
.u
.dma
);
194 cr
[5] = (CCP_MEMTYPE_SYSTEM
<< REQ6_MEMTYPE_SHIFT
)
195 | ccp_addr_hi(&op
->dst
.u
.dma
);
203 return ccp_do_cmd(op
, cr
, ARRAY_SIZE(cr
));
206 static int ccp_perform_sha(struct ccp_op
*op
)
210 /* Fill out the register contents for REQ1 through REQ6 */
211 cr
[0] = (CCP_ENGINE_SHA
<< REQ1_ENGINE_SHIFT
)
212 | (op
->u
.sha
.type
<< REQ1_SHA_TYPE_SHIFT
)
214 cr
[1] = op
->src
.u
.dma
.length
- 1;
215 cr
[2] = ccp_addr_lo(&op
->src
.u
.dma
);
216 cr
[3] = (op
->sb_ctx
<< REQ4_KSB_SHIFT
)
217 | (CCP_MEMTYPE_SYSTEM
<< REQ4_MEMTYPE_SHIFT
)
218 | ccp_addr_hi(&op
->src
.u
.dma
);
222 cr
[4] = lower_32_bits(op
->u
.sha
.msg_bits
);
223 cr
[5] = upper_32_bits(op
->u
.sha
.msg_bits
);
229 return ccp_do_cmd(op
, cr
, ARRAY_SIZE(cr
));
232 static int ccp_perform_rsa(struct ccp_op
*op
)
236 /* Fill out the register contents for REQ1 through REQ6 */
237 cr
[0] = (CCP_ENGINE_RSA
<< REQ1_ENGINE_SHIFT
)
238 | (op
->u
.rsa
.mod_size
<< REQ1_RSA_MOD_SIZE_SHIFT
)
239 | (op
->sb_key
<< REQ1_KEY_KSB_SHIFT
)
241 cr
[1] = op
->u
.rsa
.input_len
- 1;
242 cr
[2] = ccp_addr_lo(&op
->src
.u
.dma
);
243 cr
[3] = (op
->sb_ctx
<< REQ4_KSB_SHIFT
)
244 | (CCP_MEMTYPE_SYSTEM
<< REQ4_MEMTYPE_SHIFT
)
245 | ccp_addr_hi(&op
->src
.u
.dma
);
246 cr
[4] = ccp_addr_lo(&op
->dst
.u
.dma
);
247 cr
[5] = (CCP_MEMTYPE_SYSTEM
<< REQ6_MEMTYPE_SHIFT
)
248 | ccp_addr_hi(&op
->dst
.u
.dma
);
250 return ccp_do_cmd(op
, cr
, ARRAY_SIZE(cr
));
253 static int ccp_perform_passthru(struct ccp_op
*op
)
257 /* Fill out the register contents for REQ1 through REQ6 */
258 cr
[0] = (CCP_ENGINE_PASSTHRU
<< REQ1_ENGINE_SHIFT
)
259 | (op
->u
.passthru
.bit_mod
<< REQ1_PT_BW_SHIFT
)
260 | (op
->u
.passthru
.byte_swap
<< REQ1_PT_BS_SHIFT
);
262 if (op
->src
.type
== CCP_MEMTYPE_SYSTEM
)
263 cr
[1] = op
->src
.u
.dma
.length
- 1;
265 cr
[1] = op
->dst
.u
.dma
.length
- 1;
267 if (op
->src
.type
== CCP_MEMTYPE_SYSTEM
) {
268 cr
[2] = ccp_addr_lo(&op
->src
.u
.dma
);
269 cr
[3] = (CCP_MEMTYPE_SYSTEM
<< REQ4_MEMTYPE_SHIFT
)
270 | ccp_addr_hi(&op
->src
.u
.dma
);
272 if (op
->u
.passthru
.bit_mod
!= CCP_PASSTHRU_BITWISE_NOOP
)
273 cr
[3] |= (op
->sb_key
<< REQ4_KSB_SHIFT
);
275 cr
[2] = op
->src
.u
.sb
* CCP_SB_BYTES
;
276 cr
[3] = (CCP_MEMTYPE_SB
<< REQ4_MEMTYPE_SHIFT
);
279 if (op
->dst
.type
== CCP_MEMTYPE_SYSTEM
) {
280 cr
[4] = ccp_addr_lo(&op
->dst
.u
.dma
);
281 cr
[5] = (CCP_MEMTYPE_SYSTEM
<< REQ6_MEMTYPE_SHIFT
)
282 | ccp_addr_hi(&op
->dst
.u
.dma
);
284 cr
[4] = op
->dst
.u
.sb
* CCP_SB_BYTES
;
285 cr
[5] = (CCP_MEMTYPE_SB
<< REQ6_MEMTYPE_SHIFT
);
291 return ccp_do_cmd(op
, cr
, ARRAY_SIZE(cr
));
294 static int ccp_perform_ecc(struct ccp_op
*op
)
298 /* Fill out the register contents for REQ1 through REQ6 */
299 cr
[0] = REQ1_ECC_AFFINE_CONVERT
300 | (CCP_ENGINE_ECC
<< REQ1_ENGINE_SHIFT
)
301 | (op
->u
.ecc
.function
<< REQ1_ECC_FUNCTION_SHIFT
)
303 cr
[1] = op
->src
.u
.dma
.length
- 1;
304 cr
[2] = ccp_addr_lo(&op
->src
.u
.dma
);
305 cr
[3] = (CCP_MEMTYPE_SYSTEM
<< REQ4_MEMTYPE_SHIFT
)
306 | ccp_addr_hi(&op
->src
.u
.dma
);
307 cr
[4] = ccp_addr_lo(&op
->dst
.u
.dma
);
308 cr
[5] = (CCP_MEMTYPE_SYSTEM
<< REQ6_MEMTYPE_SHIFT
)
309 | ccp_addr_hi(&op
->dst
.u
.dma
);
311 return ccp_do_cmd(op
, cr
, ARRAY_SIZE(cr
));
314 static void ccp_disable_queue_interrupts(struct ccp_device
*ccp
)
316 iowrite32(0x00, ccp
->io_regs
+ IRQ_MASK_REG
);
319 static void ccp_enable_queue_interrupts(struct ccp_device
*ccp
)
321 iowrite32(ccp
->qim
, ccp
->io_regs
+ IRQ_MASK_REG
);
324 static void ccp_irq_bh(unsigned long data
)
326 struct ccp_device
*ccp
= (struct ccp_device
*)data
;
327 struct ccp_cmd_queue
*cmd_q
;
331 status
= ioread32(ccp
->io_regs
+ IRQ_STATUS_REG
);
333 for (i
= 0; i
< ccp
->cmd_q_count
; i
++) {
334 cmd_q
= &ccp
->cmd_q
[i
];
336 q_int
= status
& (cmd_q
->int_ok
| cmd_q
->int_err
);
338 cmd_q
->int_status
= status
;
339 cmd_q
->q_status
= ioread32(cmd_q
->reg_status
);
340 cmd_q
->q_int_status
= ioread32(cmd_q
->reg_int_status
);
342 /* On error, only save the first error value */
343 if ((q_int
& cmd_q
->int_err
) && !cmd_q
->cmd_error
)
344 cmd_q
->cmd_error
= CMD_Q_ERROR(cmd_q
->q_status
);
348 /* Acknowledge the interrupt and wake the kthread */
349 iowrite32(q_int
, ccp
->io_regs
+ IRQ_STATUS_REG
);
350 wake_up_interruptible(&cmd_q
->int_queue
);
353 ccp_enable_queue_interrupts(ccp
);
356 static irqreturn_t
ccp_irq_handler(int irq
, void *data
)
358 struct ccp_device
*ccp
= (struct ccp_device
*)data
;
360 ccp_disable_queue_interrupts(ccp
);
361 if (ccp
->use_tasklet
)
362 tasklet_schedule(&ccp
->irq_tasklet
);
364 ccp_irq_bh((unsigned long)ccp
);
369 static int ccp_init(struct ccp_device
*ccp
)
371 struct device
*dev
= ccp
->dev
;
372 struct ccp_cmd_queue
*cmd_q
;
373 struct dma_pool
*dma_pool
;
374 char dma_pool_name
[MAX_DMAPOOL_NAME_LEN
];
378 /* Find available queues */
380 qmr
= ioread32(ccp
->io_regs
+ Q_MASK_REG
);
381 for (i
= 0; (i
< MAX_HW_QUEUES
) && (ccp
->cmd_q_count
< ccp
->max_q_count
); i
++) {
382 if (!(qmr
& (1 << i
)))
385 /* Allocate a dma pool for this queue */
386 snprintf(dma_pool_name
, sizeof(dma_pool_name
), "%s_q%d",
388 dma_pool
= dma_pool_create(dma_pool_name
, dev
,
389 CCP_DMAPOOL_MAX_SIZE
,
390 CCP_DMAPOOL_ALIGN
, 0);
392 dev_err(dev
, "unable to allocate dma pool\n");
397 cmd_q
= &ccp
->cmd_q
[ccp
->cmd_q_count
];
402 cmd_q
->dma_pool
= dma_pool
;
404 /* Reserve 2 KSB regions for the queue */
405 cmd_q
->sb_key
= KSB_START
+ ccp
->sb_start
++;
406 cmd_q
->sb_ctx
= KSB_START
+ ccp
->sb_start
++;
409 /* Preset some register values and masks that are queue
412 cmd_q
->reg_status
= ccp
->io_regs
+ CMD_Q_STATUS_BASE
+
413 (CMD_Q_STATUS_INCR
* i
);
414 cmd_q
->reg_int_status
= ccp
->io_regs
+ CMD_Q_INT_STATUS_BASE
+
415 (CMD_Q_STATUS_INCR
* i
);
416 cmd_q
->int_ok
= 1 << (i
* 2);
417 cmd_q
->int_err
= 1 << ((i
* 2) + 1);
419 cmd_q
->free_slots
= ccp_get_free_slots(cmd_q
);
421 init_waitqueue_head(&cmd_q
->int_queue
);
423 /* Build queue interrupt mask (two interrupts per queue) */
424 ccp
->qim
|= cmd_q
->int_ok
| cmd_q
->int_err
;
427 /* For arm64 set the recommended queue cache settings */
428 iowrite32(ccp
->axcache
, ccp
->io_regs
+ CMD_Q_CACHE_BASE
+
429 (CMD_Q_CACHE_INC
* i
));
432 dev_dbg(dev
, "queue #%u available\n", i
);
434 if (ccp
->cmd_q_count
== 0) {
435 dev_notice(dev
, "no command queues available\n");
439 dev_notice(dev
, "%u command queues available\n", ccp
->cmd_q_count
);
441 /* Disable and clear interrupts until ready */
442 ccp_disable_queue_interrupts(ccp
);
443 for (i
= 0; i
< ccp
->cmd_q_count
; i
++) {
444 cmd_q
= &ccp
->cmd_q
[i
];
446 ioread32(cmd_q
->reg_int_status
);
447 ioread32(cmd_q
->reg_status
);
449 iowrite32(ccp
->qim
, ccp
->io_regs
+ IRQ_STATUS_REG
);
452 ret
= sp_request_ccp_irq(ccp
->sp
, ccp_irq_handler
, ccp
->name
, ccp
);
454 dev_err(dev
, "unable to allocate an IRQ\n");
458 /* Initialize the ISR tasklet? */
459 if (ccp
->use_tasklet
)
460 tasklet_init(&ccp
->irq_tasklet
, ccp_irq_bh
,
463 dev_dbg(dev
, "Starting threads...\n");
464 /* Create a kthread for each queue */
465 for (i
= 0; i
< ccp
->cmd_q_count
; i
++) {
466 struct task_struct
*kthread
;
468 cmd_q
= &ccp
->cmd_q
[i
];
470 kthread
= kthread_run(ccp_cmd_queue_thread
, cmd_q
,
471 "%s-q%u", ccp
->name
, cmd_q
->id
);
472 if (IS_ERR(kthread
)) {
473 dev_err(dev
, "error creating queue thread (%ld)\n",
475 ret
= PTR_ERR(kthread
);
479 cmd_q
->kthread
= kthread
;
482 dev_dbg(dev
, "Enabling interrupts...\n");
483 /* Enable interrupts */
484 ccp_enable_queue_interrupts(ccp
);
486 dev_dbg(dev
, "Registering device...\n");
489 ret
= ccp_register_rng(ccp
);
493 /* Register the DMA engine support */
494 ret
= ccp_dmaengine_register(ccp
);
501 ccp_unregister_rng(ccp
);
504 for (i
= 0; i
< ccp
->cmd_q_count
; i
++)
505 if (ccp
->cmd_q
[i
].kthread
)
506 kthread_stop(ccp
->cmd_q
[i
].kthread
);
508 sp_free_ccp_irq(ccp
->sp
, ccp
);
511 for (i
= 0; i
< ccp
->cmd_q_count
; i
++)
512 dma_pool_destroy(ccp
->cmd_q
[i
].dma_pool
);
517 static void ccp_destroy(struct ccp_device
*ccp
)
519 struct ccp_cmd_queue
*cmd_q
;
523 /* Unregister the DMA engine */
524 ccp_dmaengine_unregister(ccp
);
526 /* Unregister the RNG */
527 ccp_unregister_rng(ccp
);
529 /* Remove this device from the list of available units */
532 /* Disable and clear interrupts */
533 ccp_disable_queue_interrupts(ccp
);
534 for (i
= 0; i
< ccp
->cmd_q_count
; i
++) {
535 cmd_q
= &ccp
->cmd_q
[i
];
537 ioread32(cmd_q
->reg_int_status
);
538 ioread32(cmd_q
->reg_status
);
540 iowrite32(ccp
->qim
, ccp
->io_regs
+ IRQ_STATUS_REG
);
542 /* Stop the queue kthreads */
543 for (i
= 0; i
< ccp
->cmd_q_count
; i
++)
544 if (ccp
->cmd_q
[i
].kthread
)
545 kthread_stop(ccp
->cmd_q
[i
].kthread
);
547 sp_free_ccp_irq(ccp
->sp
, ccp
);
549 for (i
= 0; i
< ccp
->cmd_q_count
; i
++)
550 dma_pool_destroy(ccp
->cmd_q
[i
].dma_pool
);
552 /* Flush the cmd and backlog queue */
553 while (!list_empty(&ccp
->cmd
)) {
554 /* Invoke the callback directly with an error code */
555 cmd
= list_first_entry(&ccp
->cmd
, struct ccp_cmd
, entry
);
556 list_del(&cmd
->entry
);
557 cmd
->callback(cmd
->data
, -ENODEV
);
559 while (!list_empty(&ccp
->backlog
)) {
560 /* Invoke the callback directly with an error code */
561 cmd
= list_first_entry(&ccp
->backlog
, struct ccp_cmd
, entry
);
562 list_del(&cmd
->entry
);
563 cmd
->callback(cmd
->data
, -ENODEV
);
567 static const struct ccp_actions ccp3_actions
= {
568 .aes
= ccp_perform_aes
,
569 .xts_aes
= ccp_perform_xts_aes
,
571 .sha
= ccp_perform_sha
,
572 .rsa
= ccp_perform_rsa
,
573 .passthru
= ccp_perform_passthru
,
574 .ecc
= ccp_perform_ecc
,
575 .sballoc
= ccp_alloc_ksb
,
576 .sbfree
= ccp_free_ksb
,
578 .destroy
= ccp_destroy
,
579 .get_free_slots
= ccp_get_free_slots
,
580 .irqhandler
= ccp_irq_handler
,
583 const struct ccp_vdata ccpv3_platform
= {
584 .version
= CCP_VERSION(3, 0),
586 .perform
= &ccp3_actions
,
588 .rsamax
= CCP_RSA_MAX_WIDTH
,
591 const struct ccp_vdata ccpv3
= {
592 .version
= CCP_VERSION(3, 0),
594 .perform
= &ccp3_actions
,
596 .rsamax
= CCP_RSA_MAX_WIDTH
,