1 // SPDX-License-Identifier: GPL-2.0-only
3 * AMD Cryptographic Coprocessor (CCP) driver
5 * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
7 * Author: Tom Lendacky <thomas.lendacky@amd.com>
8 * Author: Gary R Hook <gary.hook@amd.com>
11 #include <linux/module.h>
12 #include <linux/kernel.h>
13 #include <linux/pci.h>
14 #include <linux/kthread.h>
15 #include <linux/interrupt.h>
16 #include <linux/ccp.h>
20 static u32
ccp_alloc_ksb(struct ccp_cmd_queue
*cmd_q
, unsigned int count
)
23 struct ccp_device
*ccp
= cmd_q
->ccp
;
26 mutex_lock(&ccp
->sb_mutex
);
28 start
= (u32
)bitmap_find_next_zero_area(ccp
->sb
,
32 if (start
<= ccp
->sb_count
) {
33 bitmap_set(ccp
->sb
, start
, count
);
35 mutex_unlock(&ccp
->sb_mutex
);
41 mutex_unlock(&ccp
->sb_mutex
);
43 /* Wait for KSB entries to become available */
44 if (wait_event_interruptible(ccp
->sb_queue
, ccp
->sb_avail
))
48 return KSB_START
+ start
;
51 static void ccp_free_ksb(struct ccp_cmd_queue
*cmd_q
, unsigned int start
,
54 struct ccp_device
*ccp
= cmd_q
->ccp
;
59 mutex_lock(&ccp
->sb_mutex
);
61 bitmap_clear(ccp
->sb
, start
- KSB_START
, count
);
65 mutex_unlock(&ccp
->sb_mutex
);
67 wake_up_interruptible_all(&ccp
->sb_queue
);
70 static unsigned int ccp_get_free_slots(struct ccp_cmd_queue
*cmd_q
)
72 return CMD_Q_DEPTH(ioread32(cmd_q
->reg_status
));
75 static int ccp_do_cmd(struct ccp_op
*op
, u32
*cr
, unsigned int cr_count
)
77 struct ccp_cmd_queue
*cmd_q
= op
->cmd_q
;
78 struct ccp_device
*ccp
= cmd_q
->ccp
;
79 void __iomem
*cr_addr
;
84 /* We could read a status register to see how many free slots
85 * are actually available, but reading that register resets it
86 * and you could lose some error information.
90 cr0
= (cmd_q
->id
<< REQ0_CMD_Q_SHIFT
)
91 | (op
->jobid
<< REQ0_JOBID_SHIFT
)
92 | REQ0_WAIT_FOR_WRITE
;
95 cr0
|= REQ0_STOP_ON_COMPLETE
96 | REQ0_INT_ON_COMPLETE
;
98 if (op
->ioc
|| !cmd_q
->free_slots
)
99 cr0
|= REQ0_INT_ON_COMPLETE
;
101 /* Start at CMD_REQ1 */
102 cr_addr
= ccp
->io_regs
+ CMD_REQ0
+ CMD_REQ_INCR
;
104 mutex_lock(&ccp
->req_mutex
);
106 /* Write CMD_REQ1 through CMD_REQx first */
107 for (i
= 0; i
< cr_count
; i
++, cr_addr
+= CMD_REQ_INCR
)
108 iowrite32(*(cr
+ i
), cr_addr
);
110 /* Tell the CCP to start */
112 iowrite32(cr0
, ccp
->io_regs
+ CMD_REQ0
);
114 mutex_unlock(&ccp
->req_mutex
);
116 if (cr0
& REQ0_INT_ON_COMPLETE
) {
117 /* Wait for the job to complete */
118 ret
= wait_event_interruptible(cmd_q
->int_queue
,
120 if (ret
|| cmd_q
->cmd_error
) {
121 /* On error delete all related jobs from the queue */
122 cmd
= (cmd_q
->id
<< DEL_Q_ID_SHIFT
)
124 if (cmd_q
->cmd_error
)
125 ccp_log_error(cmd_q
->ccp
,
128 iowrite32(cmd
, ccp
->io_regs
+ DEL_CMD_Q_JOB
);
132 } else if (op
->soc
) {
133 /* Delete just head job from the queue on SoC */
135 | (cmd_q
->id
<< DEL_Q_ID_SHIFT
)
138 iowrite32(cmd
, ccp
->io_regs
+ DEL_CMD_Q_JOB
);
141 cmd_q
->free_slots
= CMD_Q_DEPTH(cmd_q
->q_status
);
149 static int ccp_perform_aes(struct ccp_op
*op
)
153 /* Fill out the register contents for REQ1 through REQ6 */
154 cr
[0] = (CCP_ENGINE_AES
<< REQ1_ENGINE_SHIFT
)
155 | (op
->u
.aes
.type
<< REQ1_AES_TYPE_SHIFT
)
156 | (op
->u
.aes
.mode
<< REQ1_AES_MODE_SHIFT
)
157 | (op
->u
.aes
.action
<< REQ1_AES_ACTION_SHIFT
)
158 | (op
->sb_key
<< REQ1_KEY_KSB_SHIFT
);
159 cr
[1] = op
->src
.u
.dma
.length
- 1;
160 cr
[2] = ccp_addr_lo(&op
->src
.u
.dma
);
161 cr
[3] = (op
->sb_ctx
<< REQ4_KSB_SHIFT
)
162 | (CCP_MEMTYPE_SYSTEM
<< REQ4_MEMTYPE_SHIFT
)
163 | ccp_addr_hi(&op
->src
.u
.dma
);
164 cr
[4] = ccp_addr_lo(&op
->dst
.u
.dma
);
165 cr
[5] = (CCP_MEMTYPE_SYSTEM
<< REQ6_MEMTYPE_SHIFT
)
166 | ccp_addr_hi(&op
->dst
.u
.dma
);
168 if (op
->u
.aes
.mode
== CCP_AES_MODE_CFB
)
169 cr
[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT
);
177 return ccp_do_cmd(op
, cr
, ARRAY_SIZE(cr
));
180 static int ccp_perform_xts_aes(struct ccp_op
*op
)
184 /* Fill out the register contents for REQ1 through REQ6 */
185 cr
[0] = (CCP_ENGINE_XTS_AES_128
<< REQ1_ENGINE_SHIFT
)
186 | (op
->u
.xts
.action
<< REQ1_AES_ACTION_SHIFT
)
187 | (op
->u
.xts
.unit_size
<< REQ1_XTS_AES_SIZE_SHIFT
)
188 | (op
->sb_key
<< REQ1_KEY_KSB_SHIFT
);
189 cr
[1] = op
->src
.u
.dma
.length
- 1;
190 cr
[2] = ccp_addr_lo(&op
->src
.u
.dma
);
191 cr
[3] = (op
->sb_ctx
<< REQ4_KSB_SHIFT
)
192 | (CCP_MEMTYPE_SYSTEM
<< REQ4_MEMTYPE_SHIFT
)
193 | ccp_addr_hi(&op
->src
.u
.dma
);
194 cr
[4] = ccp_addr_lo(&op
->dst
.u
.dma
);
195 cr
[5] = (CCP_MEMTYPE_SYSTEM
<< REQ6_MEMTYPE_SHIFT
)
196 | ccp_addr_hi(&op
->dst
.u
.dma
);
204 return ccp_do_cmd(op
, cr
, ARRAY_SIZE(cr
));
207 static int ccp_perform_sha(struct ccp_op
*op
)
211 /* Fill out the register contents for REQ1 through REQ6 */
212 cr
[0] = (CCP_ENGINE_SHA
<< REQ1_ENGINE_SHIFT
)
213 | (op
->u
.sha
.type
<< REQ1_SHA_TYPE_SHIFT
)
215 cr
[1] = op
->src
.u
.dma
.length
- 1;
216 cr
[2] = ccp_addr_lo(&op
->src
.u
.dma
);
217 cr
[3] = (op
->sb_ctx
<< REQ4_KSB_SHIFT
)
218 | (CCP_MEMTYPE_SYSTEM
<< REQ4_MEMTYPE_SHIFT
)
219 | ccp_addr_hi(&op
->src
.u
.dma
);
223 cr
[4] = lower_32_bits(op
->u
.sha
.msg_bits
);
224 cr
[5] = upper_32_bits(op
->u
.sha
.msg_bits
);
230 return ccp_do_cmd(op
, cr
, ARRAY_SIZE(cr
));
233 static int ccp_perform_rsa(struct ccp_op
*op
)
237 /* Fill out the register contents for REQ1 through REQ6 */
238 cr
[0] = (CCP_ENGINE_RSA
<< REQ1_ENGINE_SHIFT
)
239 | (op
->u
.rsa
.mod_size
<< REQ1_RSA_MOD_SIZE_SHIFT
)
240 | (op
->sb_key
<< REQ1_KEY_KSB_SHIFT
)
242 cr
[1] = op
->u
.rsa
.input_len
- 1;
243 cr
[2] = ccp_addr_lo(&op
->src
.u
.dma
);
244 cr
[3] = (op
->sb_ctx
<< REQ4_KSB_SHIFT
)
245 | (CCP_MEMTYPE_SYSTEM
<< REQ4_MEMTYPE_SHIFT
)
246 | ccp_addr_hi(&op
->src
.u
.dma
);
247 cr
[4] = ccp_addr_lo(&op
->dst
.u
.dma
);
248 cr
[5] = (CCP_MEMTYPE_SYSTEM
<< REQ6_MEMTYPE_SHIFT
)
249 | ccp_addr_hi(&op
->dst
.u
.dma
);
251 return ccp_do_cmd(op
, cr
, ARRAY_SIZE(cr
));
254 static int ccp_perform_passthru(struct ccp_op
*op
)
258 /* Fill out the register contents for REQ1 through REQ6 */
259 cr
[0] = (CCP_ENGINE_PASSTHRU
<< REQ1_ENGINE_SHIFT
)
260 | (op
->u
.passthru
.bit_mod
<< REQ1_PT_BW_SHIFT
)
261 | (op
->u
.passthru
.byte_swap
<< REQ1_PT_BS_SHIFT
);
263 if (op
->src
.type
== CCP_MEMTYPE_SYSTEM
)
264 cr
[1] = op
->src
.u
.dma
.length
- 1;
266 cr
[1] = op
->dst
.u
.dma
.length
- 1;
268 if (op
->src
.type
== CCP_MEMTYPE_SYSTEM
) {
269 cr
[2] = ccp_addr_lo(&op
->src
.u
.dma
);
270 cr
[3] = (CCP_MEMTYPE_SYSTEM
<< REQ4_MEMTYPE_SHIFT
)
271 | ccp_addr_hi(&op
->src
.u
.dma
);
273 if (op
->u
.passthru
.bit_mod
!= CCP_PASSTHRU_BITWISE_NOOP
)
274 cr
[3] |= (op
->sb_key
<< REQ4_KSB_SHIFT
);
276 cr
[2] = op
->src
.u
.sb
* CCP_SB_BYTES
;
277 cr
[3] = (CCP_MEMTYPE_SB
<< REQ4_MEMTYPE_SHIFT
);
280 if (op
->dst
.type
== CCP_MEMTYPE_SYSTEM
) {
281 cr
[4] = ccp_addr_lo(&op
->dst
.u
.dma
);
282 cr
[5] = (CCP_MEMTYPE_SYSTEM
<< REQ6_MEMTYPE_SHIFT
)
283 | ccp_addr_hi(&op
->dst
.u
.dma
);
285 cr
[4] = op
->dst
.u
.sb
* CCP_SB_BYTES
;
286 cr
[5] = (CCP_MEMTYPE_SB
<< REQ6_MEMTYPE_SHIFT
);
292 return ccp_do_cmd(op
, cr
, ARRAY_SIZE(cr
));
295 static int ccp_perform_ecc(struct ccp_op
*op
)
299 /* Fill out the register contents for REQ1 through REQ6 */
300 cr
[0] = REQ1_ECC_AFFINE_CONVERT
301 | (CCP_ENGINE_ECC
<< REQ1_ENGINE_SHIFT
)
302 | (op
->u
.ecc
.function
<< REQ1_ECC_FUNCTION_SHIFT
)
304 cr
[1] = op
->src
.u
.dma
.length
- 1;
305 cr
[2] = ccp_addr_lo(&op
->src
.u
.dma
);
306 cr
[3] = (CCP_MEMTYPE_SYSTEM
<< REQ4_MEMTYPE_SHIFT
)
307 | ccp_addr_hi(&op
->src
.u
.dma
);
308 cr
[4] = ccp_addr_lo(&op
->dst
.u
.dma
);
309 cr
[5] = (CCP_MEMTYPE_SYSTEM
<< REQ6_MEMTYPE_SHIFT
)
310 | ccp_addr_hi(&op
->dst
.u
.dma
);
312 return ccp_do_cmd(op
, cr
, ARRAY_SIZE(cr
));
315 static void ccp_disable_queue_interrupts(struct ccp_device
*ccp
)
317 iowrite32(0x00, ccp
->io_regs
+ IRQ_MASK_REG
);
320 static void ccp_enable_queue_interrupts(struct ccp_device
*ccp
)
322 iowrite32(ccp
->qim
, ccp
->io_regs
+ IRQ_MASK_REG
);
325 static void ccp_irq_bh(unsigned long data
)
327 struct ccp_device
*ccp
= (struct ccp_device
*)data
;
328 struct ccp_cmd_queue
*cmd_q
;
332 status
= ioread32(ccp
->io_regs
+ IRQ_STATUS_REG
);
334 for (i
= 0; i
< ccp
->cmd_q_count
; i
++) {
335 cmd_q
= &ccp
->cmd_q
[i
];
337 q_int
= status
& (cmd_q
->int_ok
| cmd_q
->int_err
);
339 cmd_q
->int_status
= status
;
340 cmd_q
->q_status
= ioread32(cmd_q
->reg_status
);
341 cmd_q
->q_int_status
= ioread32(cmd_q
->reg_int_status
);
343 /* On error, only save the first error value */
344 if ((q_int
& cmd_q
->int_err
) && !cmd_q
->cmd_error
)
345 cmd_q
->cmd_error
= CMD_Q_ERROR(cmd_q
->q_status
);
349 /* Acknowledge the interrupt and wake the kthread */
350 iowrite32(q_int
, ccp
->io_regs
+ IRQ_STATUS_REG
);
351 wake_up_interruptible(&cmd_q
->int_queue
);
354 ccp_enable_queue_interrupts(ccp
);
357 static irqreturn_t
ccp_irq_handler(int irq
, void *data
)
359 struct ccp_device
*ccp
= (struct ccp_device
*)data
;
361 ccp_disable_queue_interrupts(ccp
);
362 if (ccp
->use_tasklet
)
363 tasklet_schedule(&ccp
->irq_tasklet
);
365 ccp_irq_bh((unsigned long)ccp
);
370 static int ccp_init(struct ccp_device
*ccp
)
372 struct device
*dev
= ccp
->dev
;
373 struct ccp_cmd_queue
*cmd_q
;
374 struct dma_pool
*dma_pool
;
375 char dma_pool_name
[MAX_DMAPOOL_NAME_LEN
];
379 /* Find available queues */
381 qmr
= ioread32(ccp
->io_regs
+ Q_MASK_REG
);
382 for (i
= 0; i
< MAX_HW_QUEUES
; i
++) {
383 if (!(qmr
& (1 << i
)))
386 /* Allocate a dma pool for this queue */
387 snprintf(dma_pool_name
, sizeof(dma_pool_name
), "%s_q%d",
389 dma_pool
= dma_pool_create(dma_pool_name
, dev
,
390 CCP_DMAPOOL_MAX_SIZE
,
391 CCP_DMAPOOL_ALIGN
, 0);
393 dev_err(dev
, "unable to allocate dma pool\n");
398 cmd_q
= &ccp
->cmd_q
[ccp
->cmd_q_count
];
403 cmd_q
->dma_pool
= dma_pool
;
405 /* Reserve 2 KSB regions for the queue */
406 cmd_q
->sb_key
= KSB_START
+ ccp
->sb_start
++;
407 cmd_q
->sb_ctx
= KSB_START
+ ccp
->sb_start
++;
410 /* Preset some register values and masks that are queue
413 cmd_q
->reg_status
= ccp
->io_regs
+ CMD_Q_STATUS_BASE
+
414 (CMD_Q_STATUS_INCR
* i
);
415 cmd_q
->reg_int_status
= ccp
->io_regs
+ CMD_Q_INT_STATUS_BASE
+
416 (CMD_Q_STATUS_INCR
* i
);
417 cmd_q
->int_ok
= 1 << (i
* 2);
418 cmd_q
->int_err
= 1 << ((i
* 2) + 1);
420 cmd_q
->free_slots
= ccp_get_free_slots(cmd_q
);
422 init_waitqueue_head(&cmd_q
->int_queue
);
424 /* Build queue interrupt mask (two interrupts per queue) */
425 ccp
->qim
|= cmd_q
->int_ok
| cmd_q
->int_err
;
428 /* For arm64 set the recommended queue cache settings */
429 iowrite32(ccp
->axcache
, ccp
->io_regs
+ CMD_Q_CACHE_BASE
+
430 (CMD_Q_CACHE_INC
* i
));
433 dev_dbg(dev
, "queue #%u available\n", i
);
435 if (ccp
->cmd_q_count
== 0) {
436 dev_notice(dev
, "no command queues available\n");
440 dev_notice(dev
, "%u command queues available\n", ccp
->cmd_q_count
);
442 /* Disable and clear interrupts until ready */
443 ccp_disable_queue_interrupts(ccp
);
444 for (i
= 0; i
< ccp
->cmd_q_count
; i
++) {
445 cmd_q
= &ccp
->cmd_q
[i
];
447 ioread32(cmd_q
->reg_int_status
);
448 ioread32(cmd_q
->reg_status
);
450 iowrite32(ccp
->qim
, ccp
->io_regs
+ IRQ_STATUS_REG
);
453 ret
= sp_request_ccp_irq(ccp
->sp
, ccp_irq_handler
, ccp
->name
, ccp
);
455 dev_err(dev
, "unable to allocate an IRQ\n");
459 /* Initialize the ISR tasklet? */
460 if (ccp
->use_tasklet
)
461 tasklet_init(&ccp
->irq_tasklet
, ccp_irq_bh
,
464 dev_dbg(dev
, "Starting threads...\n");
465 /* Create a kthread for each queue */
466 for (i
= 0; i
< ccp
->cmd_q_count
; i
++) {
467 struct task_struct
*kthread
;
469 cmd_q
= &ccp
->cmd_q
[i
];
471 kthread
= kthread_create(ccp_cmd_queue_thread
, cmd_q
,
472 "%s-q%u", ccp
->name
, cmd_q
->id
);
473 if (IS_ERR(kthread
)) {
474 dev_err(dev
, "error creating queue thread (%ld)\n",
476 ret
= PTR_ERR(kthread
);
480 cmd_q
->kthread
= kthread
;
481 wake_up_process(kthread
);
484 dev_dbg(dev
, "Enabling interrupts...\n");
485 /* Enable interrupts */
486 ccp_enable_queue_interrupts(ccp
);
488 dev_dbg(dev
, "Registering device...\n");
491 ret
= ccp_register_rng(ccp
);
495 /* Register the DMA engine support */
496 ret
= ccp_dmaengine_register(ccp
);
503 ccp_unregister_rng(ccp
);
506 for (i
= 0; i
< ccp
->cmd_q_count
; i
++)
507 if (ccp
->cmd_q
[i
].kthread
)
508 kthread_stop(ccp
->cmd_q
[i
].kthread
);
510 sp_free_ccp_irq(ccp
->sp
, ccp
);
513 for (i
= 0; i
< ccp
->cmd_q_count
; i
++)
514 dma_pool_destroy(ccp
->cmd_q
[i
].dma_pool
);
519 static void ccp_destroy(struct ccp_device
*ccp
)
521 struct ccp_cmd_queue
*cmd_q
;
525 /* Unregister the DMA engine */
526 ccp_dmaengine_unregister(ccp
);
528 /* Unregister the RNG */
529 ccp_unregister_rng(ccp
);
531 /* Remove this device from the list of available units */
534 /* Disable and clear interrupts */
535 ccp_disable_queue_interrupts(ccp
);
536 for (i
= 0; i
< ccp
->cmd_q_count
; i
++) {
537 cmd_q
= &ccp
->cmd_q
[i
];
539 ioread32(cmd_q
->reg_int_status
);
540 ioread32(cmd_q
->reg_status
);
542 iowrite32(ccp
->qim
, ccp
->io_regs
+ IRQ_STATUS_REG
);
544 /* Stop the queue kthreads */
545 for (i
= 0; i
< ccp
->cmd_q_count
; i
++)
546 if (ccp
->cmd_q
[i
].kthread
)
547 kthread_stop(ccp
->cmd_q
[i
].kthread
);
549 sp_free_ccp_irq(ccp
->sp
, ccp
);
551 for (i
= 0; i
< ccp
->cmd_q_count
; i
++)
552 dma_pool_destroy(ccp
->cmd_q
[i
].dma_pool
);
554 /* Flush the cmd and backlog queue */
555 while (!list_empty(&ccp
->cmd
)) {
556 /* Invoke the callback directly with an error code */
557 cmd
= list_first_entry(&ccp
->cmd
, struct ccp_cmd
, entry
);
558 list_del(&cmd
->entry
);
559 cmd
->callback(cmd
->data
, -ENODEV
);
561 while (!list_empty(&ccp
->backlog
)) {
562 /* Invoke the callback directly with an error code */
563 cmd
= list_first_entry(&ccp
->backlog
, struct ccp_cmd
, entry
);
564 list_del(&cmd
->entry
);
565 cmd
->callback(cmd
->data
, -ENODEV
);
569 static const struct ccp_actions ccp3_actions
= {
570 .aes
= ccp_perform_aes
,
571 .xts_aes
= ccp_perform_xts_aes
,
573 .sha
= ccp_perform_sha
,
574 .rsa
= ccp_perform_rsa
,
575 .passthru
= ccp_perform_passthru
,
576 .ecc
= ccp_perform_ecc
,
577 .sballoc
= ccp_alloc_ksb
,
578 .sbfree
= ccp_free_ksb
,
580 .destroy
= ccp_destroy
,
581 .get_free_slots
= ccp_get_free_slots
,
582 .irqhandler
= ccp_irq_handler
,
585 const struct ccp_vdata ccpv3_platform
= {
586 .version
= CCP_VERSION(3, 0),
588 .perform
= &ccp3_actions
,
592 const struct ccp_vdata ccpv3
= {
593 .version
= CCP_VERSION(3, 0),
595 .perform
= &ccp3_actions
,
597 .rsamax
= CCP_RSA_MAX_WIDTH
,