staging: rtl8192u: remove redundant assignment to pointer crypt
[linux/fpc-iii.git] / drivers / crypto / ccp / ccp-dev-v3.c
blob2b7d47ed5c74638900e2b39115734f978036bd17
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * AMD Cryptographic Coprocessor (CCP) driver
5 * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
7 * Author: Tom Lendacky <thomas.lendacky@amd.com>
8 * Author: Gary R Hook <gary.hook@amd.com>
9 */
11 #include <linux/module.h>
12 #include <linux/kernel.h>
13 #include <linux/pci.h>
14 #include <linux/kthread.h>
15 #include <linux/interrupt.h>
16 #include <linux/ccp.h>
18 #include "ccp-dev.h"
20 static u32 ccp_alloc_ksb(struct ccp_cmd_queue *cmd_q, unsigned int count)
22 int start;
23 struct ccp_device *ccp = cmd_q->ccp;
25 for (;;) {
26 mutex_lock(&ccp->sb_mutex);
28 start = (u32)bitmap_find_next_zero_area(ccp->sb,
29 ccp->sb_count,
30 ccp->sb_start,
31 count, 0);
32 if (start <= ccp->sb_count) {
33 bitmap_set(ccp->sb, start, count);
35 mutex_unlock(&ccp->sb_mutex);
36 break;
39 ccp->sb_avail = 0;
41 mutex_unlock(&ccp->sb_mutex);
43 /* Wait for KSB entries to become available */
44 if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail))
45 return 0;
48 return KSB_START + start;
51 static void ccp_free_ksb(struct ccp_cmd_queue *cmd_q, unsigned int start,
52 unsigned int count)
54 struct ccp_device *ccp = cmd_q->ccp;
56 if (!start)
57 return;
59 mutex_lock(&ccp->sb_mutex);
61 bitmap_clear(ccp->sb, start - KSB_START, count);
63 ccp->sb_avail = 1;
65 mutex_unlock(&ccp->sb_mutex);
67 wake_up_interruptible_all(&ccp->sb_queue);
70 static unsigned int ccp_get_free_slots(struct ccp_cmd_queue *cmd_q)
72 return CMD_Q_DEPTH(ioread32(cmd_q->reg_status));
75 static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count)
77 struct ccp_cmd_queue *cmd_q = op->cmd_q;
78 struct ccp_device *ccp = cmd_q->ccp;
79 void __iomem *cr_addr;
80 u32 cr0, cmd;
81 unsigned int i;
82 int ret = 0;
84 /* We could read a status register to see how many free slots
85 * are actually available, but reading that register resets it
86 * and you could lose some error information.
88 cmd_q->free_slots--;
90 cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT)
91 | (op->jobid << REQ0_JOBID_SHIFT)
92 | REQ0_WAIT_FOR_WRITE;
94 if (op->soc)
95 cr0 |= REQ0_STOP_ON_COMPLETE
96 | REQ0_INT_ON_COMPLETE;
98 if (op->ioc || !cmd_q->free_slots)
99 cr0 |= REQ0_INT_ON_COMPLETE;
101 /* Start at CMD_REQ1 */
102 cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR;
104 mutex_lock(&ccp->req_mutex);
106 /* Write CMD_REQ1 through CMD_REQx first */
107 for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR)
108 iowrite32(*(cr + i), cr_addr);
110 /* Tell the CCP to start */
111 wmb();
112 iowrite32(cr0, ccp->io_regs + CMD_REQ0);
114 mutex_unlock(&ccp->req_mutex);
116 if (cr0 & REQ0_INT_ON_COMPLETE) {
117 /* Wait for the job to complete */
118 ret = wait_event_interruptible(cmd_q->int_queue,
119 cmd_q->int_rcvd);
120 if (ret || cmd_q->cmd_error) {
121 /* On error delete all related jobs from the queue */
122 cmd = (cmd_q->id << DEL_Q_ID_SHIFT)
123 | op->jobid;
124 if (cmd_q->cmd_error)
125 ccp_log_error(cmd_q->ccp,
126 cmd_q->cmd_error);
128 iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
130 if (!ret)
131 ret = -EIO;
132 } else if (op->soc) {
133 /* Delete just head job from the queue on SoC */
134 cmd = DEL_Q_ACTIVE
135 | (cmd_q->id << DEL_Q_ID_SHIFT)
136 | op->jobid;
138 iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
141 cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status);
143 cmd_q->int_rcvd = 0;
146 return ret;
149 static int ccp_perform_aes(struct ccp_op *op)
151 u32 cr[6];
153 /* Fill out the register contents for REQ1 through REQ6 */
154 cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT)
155 | (op->u.aes.type << REQ1_AES_TYPE_SHIFT)
156 | (op->u.aes.mode << REQ1_AES_MODE_SHIFT)
157 | (op->u.aes.action << REQ1_AES_ACTION_SHIFT)
158 | (op->sb_key << REQ1_KEY_KSB_SHIFT);
159 cr[1] = op->src.u.dma.length - 1;
160 cr[2] = ccp_addr_lo(&op->src.u.dma);
161 cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)
162 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
163 | ccp_addr_hi(&op->src.u.dma);
164 cr[4] = ccp_addr_lo(&op->dst.u.dma);
165 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
166 | ccp_addr_hi(&op->dst.u.dma);
168 if (op->u.aes.mode == CCP_AES_MODE_CFB)
169 cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT);
171 if (op->eom)
172 cr[0] |= REQ1_EOM;
174 if (op->init)
175 cr[0] |= REQ1_INIT;
177 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
180 static int ccp_perform_xts_aes(struct ccp_op *op)
182 u32 cr[6];
184 /* Fill out the register contents for REQ1 through REQ6 */
185 cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT)
186 | (op->u.xts.action << REQ1_AES_ACTION_SHIFT)
187 | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT)
188 | (op->sb_key << REQ1_KEY_KSB_SHIFT);
189 cr[1] = op->src.u.dma.length - 1;
190 cr[2] = ccp_addr_lo(&op->src.u.dma);
191 cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)
192 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
193 | ccp_addr_hi(&op->src.u.dma);
194 cr[4] = ccp_addr_lo(&op->dst.u.dma);
195 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
196 | ccp_addr_hi(&op->dst.u.dma);
198 if (op->eom)
199 cr[0] |= REQ1_EOM;
201 if (op->init)
202 cr[0] |= REQ1_INIT;
204 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
207 static int ccp_perform_sha(struct ccp_op *op)
209 u32 cr[6];
211 /* Fill out the register contents for REQ1 through REQ6 */
212 cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT)
213 | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT)
214 | REQ1_INIT;
215 cr[1] = op->src.u.dma.length - 1;
216 cr[2] = ccp_addr_lo(&op->src.u.dma);
217 cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)
218 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
219 | ccp_addr_hi(&op->src.u.dma);
221 if (op->eom) {
222 cr[0] |= REQ1_EOM;
223 cr[4] = lower_32_bits(op->u.sha.msg_bits);
224 cr[5] = upper_32_bits(op->u.sha.msg_bits);
225 } else {
226 cr[4] = 0;
227 cr[5] = 0;
230 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
233 static int ccp_perform_rsa(struct ccp_op *op)
235 u32 cr[6];
237 /* Fill out the register contents for REQ1 through REQ6 */
238 cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT)
239 | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT)
240 | (op->sb_key << REQ1_KEY_KSB_SHIFT)
241 | REQ1_EOM;
242 cr[1] = op->u.rsa.input_len - 1;
243 cr[2] = ccp_addr_lo(&op->src.u.dma);
244 cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)
245 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
246 | ccp_addr_hi(&op->src.u.dma);
247 cr[4] = ccp_addr_lo(&op->dst.u.dma);
248 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
249 | ccp_addr_hi(&op->dst.u.dma);
251 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
254 static int ccp_perform_passthru(struct ccp_op *op)
256 u32 cr[6];
258 /* Fill out the register contents for REQ1 through REQ6 */
259 cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT)
260 | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT)
261 | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT);
263 if (op->src.type == CCP_MEMTYPE_SYSTEM)
264 cr[1] = op->src.u.dma.length - 1;
265 else
266 cr[1] = op->dst.u.dma.length - 1;
268 if (op->src.type == CCP_MEMTYPE_SYSTEM) {
269 cr[2] = ccp_addr_lo(&op->src.u.dma);
270 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
271 | ccp_addr_hi(&op->src.u.dma);
273 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
274 cr[3] |= (op->sb_key << REQ4_KSB_SHIFT);
275 } else {
276 cr[2] = op->src.u.sb * CCP_SB_BYTES;
277 cr[3] = (CCP_MEMTYPE_SB << REQ4_MEMTYPE_SHIFT);
280 if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
281 cr[4] = ccp_addr_lo(&op->dst.u.dma);
282 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
283 | ccp_addr_hi(&op->dst.u.dma);
284 } else {
285 cr[4] = op->dst.u.sb * CCP_SB_BYTES;
286 cr[5] = (CCP_MEMTYPE_SB << REQ6_MEMTYPE_SHIFT);
289 if (op->eom)
290 cr[0] |= REQ1_EOM;
292 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
295 static int ccp_perform_ecc(struct ccp_op *op)
297 u32 cr[6];
299 /* Fill out the register contents for REQ1 through REQ6 */
300 cr[0] = REQ1_ECC_AFFINE_CONVERT
301 | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT)
302 | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT)
303 | REQ1_EOM;
304 cr[1] = op->src.u.dma.length - 1;
305 cr[2] = ccp_addr_lo(&op->src.u.dma);
306 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
307 | ccp_addr_hi(&op->src.u.dma);
308 cr[4] = ccp_addr_lo(&op->dst.u.dma);
309 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
310 | ccp_addr_hi(&op->dst.u.dma);
312 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
315 static void ccp_disable_queue_interrupts(struct ccp_device *ccp)
317 iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG);
320 static void ccp_enable_queue_interrupts(struct ccp_device *ccp)
322 iowrite32(ccp->qim, ccp->io_regs + IRQ_MASK_REG);
325 static void ccp_irq_bh(unsigned long data)
327 struct ccp_device *ccp = (struct ccp_device *)data;
328 struct ccp_cmd_queue *cmd_q;
329 u32 q_int, status;
330 unsigned int i;
332 status = ioread32(ccp->io_regs + IRQ_STATUS_REG);
334 for (i = 0; i < ccp->cmd_q_count; i++) {
335 cmd_q = &ccp->cmd_q[i];
337 q_int = status & (cmd_q->int_ok | cmd_q->int_err);
338 if (q_int) {
339 cmd_q->int_status = status;
340 cmd_q->q_status = ioread32(cmd_q->reg_status);
341 cmd_q->q_int_status = ioread32(cmd_q->reg_int_status);
343 /* On error, only save the first error value */
344 if ((q_int & cmd_q->int_err) && !cmd_q->cmd_error)
345 cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
347 cmd_q->int_rcvd = 1;
349 /* Acknowledge the interrupt and wake the kthread */
350 iowrite32(q_int, ccp->io_regs + IRQ_STATUS_REG);
351 wake_up_interruptible(&cmd_q->int_queue);
354 ccp_enable_queue_interrupts(ccp);
357 static irqreturn_t ccp_irq_handler(int irq, void *data)
359 struct ccp_device *ccp = (struct ccp_device *)data;
361 ccp_disable_queue_interrupts(ccp);
362 if (ccp->use_tasklet)
363 tasklet_schedule(&ccp->irq_tasklet);
364 else
365 ccp_irq_bh((unsigned long)ccp);
367 return IRQ_HANDLED;
370 static int ccp_init(struct ccp_device *ccp)
372 struct device *dev = ccp->dev;
373 struct ccp_cmd_queue *cmd_q;
374 struct dma_pool *dma_pool;
375 char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
376 unsigned int qmr, i;
377 int ret;
379 /* Find available queues */
380 ccp->qim = 0;
381 qmr = ioread32(ccp->io_regs + Q_MASK_REG);
382 for (i = 0; i < MAX_HW_QUEUES; i++) {
383 if (!(qmr & (1 << i)))
384 continue;
386 /* Allocate a dma pool for this queue */
387 snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d",
388 ccp->name, i);
389 dma_pool = dma_pool_create(dma_pool_name, dev,
390 CCP_DMAPOOL_MAX_SIZE,
391 CCP_DMAPOOL_ALIGN, 0);
392 if (!dma_pool) {
393 dev_err(dev, "unable to allocate dma pool\n");
394 ret = -ENOMEM;
395 goto e_pool;
398 cmd_q = &ccp->cmd_q[ccp->cmd_q_count];
399 ccp->cmd_q_count++;
401 cmd_q->ccp = ccp;
402 cmd_q->id = i;
403 cmd_q->dma_pool = dma_pool;
405 /* Reserve 2 KSB regions for the queue */
406 cmd_q->sb_key = KSB_START + ccp->sb_start++;
407 cmd_q->sb_ctx = KSB_START + ccp->sb_start++;
408 ccp->sb_count -= 2;
410 /* Preset some register values and masks that are queue
411 * number dependent
413 cmd_q->reg_status = ccp->io_regs + CMD_Q_STATUS_BASE +
414 (CMD_Q_STATUS_INCR * i);
415 cmd_q->reg_int_status = ccp->io_regs + CMD_Q_INT_STATUS_BASE +
416 (CMD_Q_STATUS_INCR * i);
417 cmd_q->int_ok = 1 << (i * 2);
418 cmd_q->int_err = 1 << ((i * 2) + 1);
420 cmd_q->free_slots = ccp_get_free_slots(cmd_q);
422 init_waitqueue_head(&cmd_q->int_queue);
424 /* Build queue interrupt mask (two interrupts per queue) */
425 ccp->qim |= cmd_q->int_ok | cmd_q->int_err;
427 #ifdef CONFIG_ARM64
428 /* For arm64 set the recommended queue cache settings */
429 iowrite32(ccp->axcache, ccp->io_regs + CMD_Q_CACHE_BASE +
430 (CMD_Q_CACHE_INC * i));
431 #endif
433 dev_dbg(dev, "queue #%u available\n", i);
435 if (ccp->cmd_q_count == 0) {
436 dev_notice(dev, "no command queues available\n");
437 ret = -EIO;
438 goto e_pool;
440 dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count);
442 /* Disable and clear interrupts until ready */
443 ccp_disable_queue_interrupts(ccp);
444 for (i = 0; i < ccp->cmd_q_count; i++) {
445 cmd_q = &ccp->cmd_q[i];
447 ioread32(cmd_q->reg_int_status);
448 ioread32(cmd_q->reg_status);
450 iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG);
452 /* Request an irq */
453 ret = sp_request_ccp_irq(ccp->sp, ccp_irq_handler, ccp->name, ccp);
454 if (ret) {
455 dev_err(dev, "unable to allocate an IRQ\n");
456 goto e_pool;
459 /* Initialize the ISR tasklet? */
460 if (ccp->use_tasklet)
461 tasklet_init(&ccp->irq_tasklet, ccp_irq_bh,
462 (unsigned long)ccp);
464 dev_dbg(dev, "Starting threads...\n");
465 /* Create a kthread for each queue */
466 for (i = 0; i < ccp->cmd_q_count; i++) {
467 struct task_struct *kthread;
469 cmd_q = &ccp->cmd_q[i];
471 kthread = kthread_create(ccp_cmd_queue_thread, cmd_q,
472 "%s-q%u", ccp->name, cmd_q->id);
473 if (IS_ERR(kthread)) {
474 dev_err(dev, "error creating queue thread (%ld)\n",
475 PTR_ERR(kthread));
476 ret = PTR_ERR(kthread);
477 goto e_kthread;
480 cmd_q->kthread = kthread;
481 wake_up_process(kthread);
484 dev_dbg(dev, "Enabling interrupts...\n");
485 /* Enable interrupts */
486 ccp_enable_queue_interrupts(ccp);
488 dev_dbg(dev, "Registering device...\n");
489 ccp_add_device(ccp);
491 ret = ccp_register_rng(ccp);
492 if (ret)
493 goto e_kthread;
495 /* Register the DMA engine support */
496 ret = ccp_dmaengine_register(ccp);
497 if (ret)
498 goto e_hwrng;
500 return 0;
502 e_hwrng:
503 ccp_unregister_rng(ccp);
505 e_kthread:
506 for (i = 0; i < ccp->cmd_q_count; i++)
507 if (ccp->cmd_q[i].kthread)
508 kthread_stop(ccp->cmd_q[i].kthread);
510 sp_free_ccp_irq(ccp->sp, ccp);
512 e_pool:
513 for (i = 0; i < ccp->cmd_q_count; i++)
514 dma_pool_destroy(ccp->cmd_q[i].dma_pool);
516 return ret;
519 static void ccp_destroy(struct ccp_device *ccp)
521 struct ccp_cmd_queue *cmd_q;
522 struct ccp_cmd *cmd;
523 unsigned int i;
525 /* Unregister the DMA engine */
526 ccp_dmaengine_unregister(ccp);
528 /* Unregister the RNG */
529 ccp_unregister_rng(ccp);
531 /* Remove this device from the list of available units */
532 ccp_del_device(ccp);
534 /* Disable and clear interrupts */
535 ccp_disable_queue_interrupts(ccp);
536 for (i = 0; i < ccp->cmd_q_count; i++) {
537 cmd_q = &ccp->cmd_q[i];
539 ioread32(cmd_q->reg_int_status);
540 ioread32(cmd_q->reg_status);
542 iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG);
544 /* Stop the queue kthreads */
545 for (i = 0; i < ccp->cmd_q_count; i++)
546 if (ccp->cmd_q[i].kthread)
547 kthread_stop(ccp->cmd_q[i].kthread);
549 sp_free_ccp_irq(ccp->sp, ccp);
551 for (i = 0; i < ccp->cmd_q_count; i++)
552 dma_pool_destroy(ccp->cmd_q[i].dma_pool);
554 /* Flush the cmd and backlog queue */
555 while (!list_empty(&ccp->cmd)) {
556 /* Invoke the callback directly with an error code */
557 cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);
558 list_del(&cmd->entry);
559 cmd->callback(cmd->data, -ENODEV);
561 while (!list_empty(&ccp->backlog)) {
562 /* Invoke the callback directly with an error code */
563 cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry);
564 list_del(&cmd->entry);
565 cmd->callback(cmd->data, -ENODEV);
569 static const struct ccp_actions ccp3_actions = {
570 .aes = ccp_perform_aes,
571 .xts_aes = ccp_perform_xts_aes,
572 .des3 = NULL,
573 .sha = ccp_perform_sha,
574 .rsa = ccp_perform_rsa,
575 .passthru = ccp_perform_passthru,
576 .ecc = ccp_perform_ecc,
577 .sballoc = ccp_alloc_ksb,
578 .sbfree = ccp_free_ksb,
579 .init = ccp_init,
580 .destroy = ccp_destroy,
581 .get_free_slots = ccp_get_free_slots,
582 .irqhandler = ccp_irq_handler,
585 const struct ccp_vdata ccpv3_platform = {
586 .version = CCP_VERSION(3, 0),
587 .setup = NULL,
588 .perform = &ccp3_actions,
589 .offset = 0,
592 const struct ccp_vdata ccpv3 = {
593 .version = CCP_VERSION(3, 0),
594 .setup = NULL,
595 .perform = &ccp3_actions,
596 .offset = 0x20000,
597 .rsamax = CCP_RSA_MAX_WIDTH,