?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Input: xpad - add support for Xbox1 PDP Camo series gamepad
[linux/fpc-iii.git] / drivers / crypto / ccp / ccp-dev-v5.c
blob71980c41283bf985b1a2d4c4556aed449dd5e522
1 /*
2 * AMD Cryptographic Coprocessor (CCP) driver
3 *
4 * Copyright (C) 2016 Advanced Micro Devices, Inc.
5 *
6 * Author: Gary R Hook <gary.hook@amd.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13 #include <linux/module.h>
14 #include <linux/kernel.h>
15 #include <linux/pci.h>
16 #include <linux/kthread.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/interrupt.h>
19 #include <linux/compiler.h>
20 #include <linux/ccp.h>
21
22 #include "ccp-dev.h"
23
24 static u32 ccp_lsb_alloc(struct ccp_cmd_queue *cmd_q, unsigned int count)
25 {
26 struct ccp_device *ccp;
27 int start;
28
29 /* First look at the map for the queue */
30 if (cmd_q->lsb >= 0) {
31 start = (u32)bitmap_find_next_zero_area(cmd_q->lsbmap,
32 LSB_SIZE,
33 0, count, 0);
34 if (start < LSB_SIZE) {
35 bitmap_set(cmd_q->lsbmap, start, count);
36 return start + cmd_q->lsb * LSB_SIZE;
37 }
38 }
39
40 /* No joy; try to get an entry from the shared blocks */
41 ccp = cmd_q->ccp;
42 for (;;) {
43 mutex_lock(&ccp->sb_mutex);
44
45 start = (u32)bitmap_find_next_zero_area(ccp->lsbmap,
46 MAX_LSB_CNT * LSB_SIZE,
47 0,
48 count, 0);
49 if (start <= MAX_LSB_CNT * LSB_SIZE) {
50 bitmap_set(ccp->lsbmap, start, count);
51
52 mutex_unlock(&ccp->sb_mutex);
53 return start * LSB_ITEM_SIZE;
54 }
55
56 ccp->sb_avail = 0;
57
58 mutex_unlock(&ccp->sb_mutex);
59
60 /* Wait for KSB entries to become available */
61 if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail))
62 return 0;
63 }
64 }
65
66 static void ccp_lsb_free(struct ccp_cmd_queue *cmd_q, unsigned int start,
67 unsigned int count)
68 {
69 int lsbno = start / LSB_SIZE;
70
71 if (!start)
72 return;
73
74 if (cmd_q->lsb == lsbno) {
75 /* An entry from the private LSB */
76 bitmap_clear(cmd_q->lsbmap, start % LSB_SIZE, count);
77 } else {
78 /* From the shared LSBs */
79 struct ccp_device *ccp = cmd_q->ccp;
80
81 mutex_lock(&ccp->sb_mutex);
82 bitmap_clear(ccp->lsbmap, start, count);
83 ccp->sb_avail = 1;
84 mutex_unlock(&ccp->sb_mutex);
85 wake_up_interruptible_all(&ccp->sb_queue);
86 }
87 }
88
89 /* CCP version 5: Union to define the function field (cmd_reg1/dword0) */
90 union ccp_function {
91 struct {
92 u16 size:7;
93 u16 encrypt:1;
94 u16 mode:5;
95 u16 type:2;
96 } aes;
97 struct {
98 u16 size:7;
99 u16 encrypt:1;
100 u16 rsvd:5;
101 u16 type:2;
102 } aes_xts;
103 struct {
104 u16 rsvd1:10;
105 u16 type:4;
106 u16 rsvd2:1;
107 } sha;
108 struct {
109 u16 mode:3;
110 u16 size:12;
111 } rsa;
112 struct {
113 u16 byteswap:2;
114 u16 bitwise:3;
115 u16 reflect:2;
116 u16 rsvd:8;
117 } pt;
118 struct {
119 u16 rsvd:13;
120 } zlib;
121 struct {
122 u16 size:10;
123 u16 type:2;
124 u16 mode:3;
125 } ecc;
126 u16 raw;
127 };
128
129 #define CCP_AES_SIZE(p) ((p)->aes.size)
130 #define CCP_AES_ENCRYPT(p) ((p)->aes.encrypt)
131 #define CCP_AES_MODE(p) ((p)->aes.mode)
132 #define CCP_AES_TYPE(p) ((p)->aes.type)
133 #define CCP_XTS_SIZE(p) ((p)->aes_xts.size)
134 #define CCP_XTS_TYPE(p) ((p)->aes_xts.type)
135 #define CCP_XTS_ENCRYPT(p) ((p)->aes_xts.encrypt)
136 #define CCP_SHA_TYPE(p) ((p)->sha.type)
137 #define CCP_RSA_SIZE(p) ((p)->rsa.size)
138 #define CCP_PT_BYTESWAP(p) ((p)->pt.byteswap)
139 #define CCP_PT_BITWISE(p) ((p)->pt.bitwise)
140 #define CCP_ECC_MODE(p) ((p)->ecc.mode)
141 #define CCP_ECC_AFFINE(p) ((p)->ecc.one)
142
143 /* Word 0 */
144 #define CCP5_CMD_DW0(p) ((p)->dw0)
145 #define CCP5_CMD_SOC(p) (CCP5_CMD_DW0(p).soc)
146 #define CCP5_CMD_IOC(p) (CCP5_CMD_DW0(p).ioc)
147 #define CCP5_CMD_INIT(p) (CCP5_CMD_DW0(p).init)
148 #define CCP5_CMD_EOM(p) (CCP5_CMD_DW0(p).eom)
149 #define CCP5_CMD_FUNCTION(p) (CCP5_CMD_DW0(p).function)
150 #define CCP5_CMD_ENGINE(p) (CCP5_CMD_DW0(p).engine)
151 #define CCP5_CMD_PROT(p) (CCP5_CMD_DW0(p).prot)
152
153 /* Word 1 */
154 #define CCP5_CMD_DW1(p) ((p)->length)
155 #define CCP5_CMD_LEN(p) (CCP5_CMD_DW1(p))
156
157 /* Word 2 */
158 #define CCP5_CMD_DW2(p) ((p)->src_lo)
159 #define CCP5_CMD_SRC_LO(p) (CCP5_CMD_DW2(p))
160
161 /* Word 3 */
162 #define CCP5_CMD_DW3(p) ((p)->dw3)
163 #define CCP5_CMD_SRC_MEM(p) ((p)->dw3.src_mem)
164 #define CCP5_CMD_SRC_HI(p) ((p)->dw3.src_hi)
165 #define CCP5_CMD_LSB_ID(p) ((p)->dw3.lsb_cxt_id)
166 #define CCP5_CMD_FIX_SRC(p) ((p)->dw3.fixed)
167
168 /* Words 4/5 */
169 #define CCP5_CMD_DW4(p) ((p)->dw4)
170 #define CCP5_CMD_DST_LO(p) (CCP5_CMD_DW4(p).dst_lo)
171 #define CCP5_CMD_DW5(p) ((p)->dw5.fields.dst_hi)
172 #define CCP5_CMD_DST_HI(p) (CCP5_CMD_DW5(p))
173 #define CCP5_CMD_DST_MEM(p) ((p)->dw5.fields.dst_mem)
174 #define CCP5_CMD_FIX_DST(p) ((p)->dw5.fields.fixed)
175 #define CCP5_CMD_SHA_LO(p) ((p)->dw4.sha_len_lo)
176 #define CCP5_CMD_SHA_HI(p) ((p)->dw5.sha_len_hi)
177
178 /* Word 6/7 */
179 #define CCP5_CMD_DW6(p) ((p)->key_lo)
180 #define CCP5_CMD_KEY_LO(p) (CCP5_CMD_DW6(p))
181 #define CCP5_CMD_DW7(p) ((p)->dw7)
182 #define CCP5_CMD_KEY_HI(p) ((p)->dw7.key_hi)
183 #define CCP5_CMD_KEY_MEM(p) ((p)->dw7.key_mem)
184
185 static inline u32 low_address(unsigned long addr)
186 {
187 return (u64)addr & 0x0ffffffff;
188 }
189
190 static inline u32 high_address(unsigned long addr)
191 {
192 return ((u64)addr >> 32) & 0x00000ffff;
193 }
194
195 static unsigned int ccp5_get_free_slots(struct ccp_cmd_queue *cmd_q)
196 {
197 unsigned int head_idx, n;
198 u32 head_lo, queue_start;
199
200 queue_start = low_address(cmd_q->qdma_tail);
201 head_lo = ioread32(cmd_q->reg_head_lo);
202 head_idx = (head_lo - queue_start) / sizeof(struct ccp5_desc);
203
204 n = head_idx + COMMANDS_PER_QUEUE - cmd_q->qidx - 1;
205
206 return n % COMMANDS_PER_QUEUE; /* Always one unused spot */
207 }
208
209 static int ccp5_do_cmd(struct ccp5_desc *desc,
210 struct ccp_cmd_queue *cmd_q)
211 {
212 u32 *mP;
213 __le32 *dP;
214 u32 tail;
215 int i;
216 int ret = 0;
217
218 if (CCP5_CMD_SOC(desc)) {
219 CCP5_CMD_IOC(desc) = 1;
220 CCP5_CMD_SOC(desc) = 0;
221 }
222 mutex_lock(&cmd_q->q_mutex);
223
224 mP = (u32 *) &cmd_q->qbase[cmd_q->qidx];
225 dP = (__le32 *) desc;
226 for (i = 0; i < 8; i++)
227 mP[i] = cpu_to_le32(dP[i]); /* handle endianness */
228
229 cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
230
231 /* The data used by this command must be flushed to memory */
232 wmb();
233
234 /* Write the new tail address back to the queue register */
235 tail = low_address(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);
236 iowrite32(tail, cmd_q->reg_tail_lo);
237
238 /* Turn the queue back on using our cached control register */
239 iowrite32(cmd_q->qcontrol | CMD5_Q_RUN, cmd_q->reg_control);
240 mutex_unlock(&cmd_q->q_mutex);
241
242 if (CCP5_CMD_IOC(desc)) {
243 /* Wait for the job to complete */
244 ret = wait_event_interruptible(cmd_q->int_queue,
245 cmd_q->int_rcvd);
246 if (ret || cmd_q->cmd_error) {
247 if (cmd_q->cmd_error)
248 ccp_log_error(cmd_q->ccp,
249 cmd_q->cmd_error);
250 /* A version 5 device doesn't use Job IDs... */
251 if (!ret)
252 ret = -EIO;
253 }
254 cmd_q->int_rcvd = 0;
255 }
256
257 return 0;
258 }
259
260 static int ccp5_perform_aes(struct ccp_op *op)
261 {
262 struct ccp5_desc desc;
263 union ccp_function function;
264 u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
265
266 /* Zero out all the fields of the command desc */
267 memset(&desc, 0, Q_DESC_SIZE);
268
269 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_AES;
270
271 CCP5_CMD_SOC(&desc) = op->soc;
272 CCP5_CMD_IOC(&desc) = 1;
273 CCP5_CMD_INIT(&desc) = op->init;
274 CCP5_CMD_EOM(&desc) = op->eom;
275 CCP5_CMD_PROT(&desc) = 0;
276
277 function.raw = 0;
278 CCP_AES_ENCRYPT(&function) = op->u.aes.action;
279 CCP_AES_MODE(&function) = op->u.aes.mode;
280 CCP_AES_TYPE(&function) = op->u.aes.type;
281 CCP_AES_SIZE(&function) = op->u.aes.size;
282
283 CCP5_CMD_FUNCTION(&desc) = function.raw;
284
285 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
286
287 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
288 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
289 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
290
291 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
292 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
293 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
294
295 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
296 CCP5_CMD_KEY_HI(&desc) = 0;
297 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
298 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
299
300 return ccp5_do_cmd(&desc, op->cmd_q);
301 }
302
303 static int ccp5_perform_xts_aes(struct ccp_op *op)
304 {
305 struct ccp5_desc desc;
306 union ccp_function function;
307 u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
308
309 /* Zero out all the fields of the command desc */
310 memset(&desc, 0, Q_DESC_SIZE);
311
312 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_XTS_AES_128;
313
314 CCP5_CMD_SOC(&desc) = op->soc;
315 CCP5_CMD_IOC(&desc) = 1;
316 CCP5_CMD_INIT(&desc) = op->init;
317 CCP5_CMD_EOM(&desc) = op->eom;
318 CCP5_CMD_PROT(&desc) = 0;
319
320 function.raw = 0;
321 CCP_XTS_TYPE(&function) = op->u.xts.type;
322 CCP_XTS_ENCRYPT(&function) = op->u.xts.action;
323 CCP_XTS_SIZE(&function) = op->u.xts.unit_size;
324 CCP5_CMD_FUNCTION(&desc) = function.raw;
325
326 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
327
328 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
329 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
330 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
331
332 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
333 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
334 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
335
336 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
337 CCP5_CMD_KEY_HI(&desc) = 0;
338 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
339 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
340
341 return ccp5_do_cmd(&desc, op->cmd_q);
342 }
343
344 static int ccp5_perform_sha(struct ccp_op *op)
345 {
346 struct ccp5_desc desc;
347 union ccp_function function;
348
349 /* Zero out all the fields of the command desc */
350 memset(&desc, 0, Q_DESC_SIZE);
351
352 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_SHA;
353
354 CCP5_CMD_SOC(&desc) = op->soc;
355 CCP5_CMD_IOC(&desc) = 1;
356 CCP5_CMD_INIT(&desc) = 1;
357 CCP5_CMD_EOM(&desc) = op->eom;
358 CCP5_CMD_PROT(&desc) = 0;
359
360 function.raw = 0;
361 CCP_SHA_TYPE(&function) = op->u.sha.type;
362 CCP5_CMD_FUNCTION(&desc) = function.raw;
363
364 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
365
366 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
367 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
368 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
369
370 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
371
372 if (op->eom) {
373 CCP5_CMD_SHA_LO(&desc) = lower_32_bits(op->u.sha.msg_bits);
374 CCP5_CMD_SHA_HI(&desc) = upper_32_bits(op->u.sha.msg_bits);
375 } else {
376 CCP5_CMD_SHA_LO(&desc) = 0;
377 CCP5_CMD_SHA_HI(&desc) = 0;
378 }
379
380 return ccp5_do_cmd(&desc, op->cmd_q);
381 }
382
383 static int ccp5_perform_rsa(struct ccp_op *op)
384 {
385 struct ccp5_desc desc;
386 union ccp_function function;
387
388 /* Zero out all the fields of the command desc */
389 memset(&desc, 0, Q_DESC_SIZE);
390
391 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_RSA;
392
393 CCP5_CMD_SOC(&desc) = op->soc;
394 CCP5_CMD_IOC(&desc) = 1;
395 CCP5_CMD_INIT(&desc) = 0;
396 CCP5_CMD_EOM(&desc) = 1;
397 CCP5_CMD_PROT(&desc) = 0;
398
399 function.raw = 0;
400 CCP_RSA_SIZE(&function) = op->u.rsa.mod_size;
401 CCP5_CMD_FUNCTION(&desc) = function.raw;
402
403 CCP5_CMD_LEN(&desc) = op->u.rsa.input_len;
404
405 /* Source is from external memory */
406 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
407 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
408 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
409
410 /* Destination is in external memory */
411 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
412 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
413 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
414
415 /* Key (Exponent) is in external memory */
416 CCP5_CMD_KEY_LO(&desc) = ccp_addr_lo(&op->exp.u.dma);
417 CCP5_CMD_KEY_HI(&desc) = ccp_addr_hi(&op->exp.u.dma);
418 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
419
420 return ccp5_do_cmd(&desc, op->cmd_q);
421 }
422
423 static int ccp5_perform_passthru(struct ccp_op *op)
424 {
425 struct ccp5_desc desc;
426 union ccp_function function;
427 struct ccp_dma_info *saddr = &op->src.u.dma;
428 struct ccp_dma_info *daddr = &op->dst.u.dma;
429
430 memset(&desc, 0, Q_DESC_SIZE);
431
432 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_PASSTHRU;
433
434 CCP5_CMD_SOC(&desc) = 0;
435 CCP5_CMD_IOC(&desc) = 1;
436 CCP5_CMD_INIT(&desc) = 0;
437 CCP5_CMD_EOM(&desc) = op->eom;
438 CCP5_CMD_PROT(&desc) = 0;
439
440 function.raw = 0;
441 CCP_PT_BYTESWAP(&function) = op->u.passthru.byte_swap;
442 CCP_PT_BITWISE(&function) = op->u.passthru.bit_mod;
443 CCP5_CMD_FUNCTION(&desc) = function.raw;
444
445 /* Length of source data is always 256 bytes */
446 if (op->src.type == CCP_MEMTYPE_SYSTEM)
447 CCP5_CMD_LEN(&desc) = saddr->length;
448 else
449 CCP5_CMD_LEN(&desc) = daddr->length;
450
451 if (op->src.type == CCP_MEMTYPE_SYSTEM) {
452 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
453 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
454 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
455
456 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
457 CCP5_CMD_LSB_ID(&desc) = op->sb_key;
458 } else {
459 u32 key_addr = op->src.u.sb * CCP_SB_BYTES;
460
461 CCP5_CMD_SRC_LO(&desc) = lower_32_bits(key_addr);
462 CCP5_CMD_SRC_HI(&desc) = 0;
463 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SB;
464 }
465
466 if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
467 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
468 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
469 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
470 } else {
471 u32 key_addr = op->dst.u.sb * CCP_SB_BYTES;
472
473 CCP5_CMD_DST_LO(&desc) = lower_32_bits(key_addr);
474 CCP5_CMD_DST_HI(&desc) = 0;
475 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SB;
476 }
477
478 return ccp5_do_cmd(&desc, op->cmd_q);
479 }
480
481 static int ccp5_perform_ecc(struct ccp_op *op)
482 {
483 struct ccp5_desc desc;
484 union ccp_function function;
485
486 /* Zero out all the fields of the command desc */
487 memset(&desc, 0, Q_DESC_SIZE);
488
489 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_ECC;
490
491 CCP5_CMD_SOC(&desc) = 0;
492 CCP5_CMD_IOC(&desc) = 1;
493 CCP5_CMD_INIT(&desc) = 0;
494 CCP5_CMD_EOM(&desc) = 1;
495 CCP5_CMD_PROT(&desc) = 0;
496
497 function.raw = 0;
498 function.ecc.mode = op->u.ecc.function;
499 CCP5_CMD_FUNCTION(&desc) = function.raw;
500
501 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
502
503 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
504 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
505 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
506
507 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
508 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
509 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
510
511 return ccp5_do_cmd(&desc, op->cmd_q);
512 }
513
514 static int ccp_find_lsb_regions(struct ccp_cmd_queue *cmd_q, u64 status)
515 {
516 int q_mask = 1 << cmd_q->id;
517 int queues = 0;
518 int j;
519
520 /* Build a bit mask to know which LSBs this queue has access to.
521 * Don't bother with segment 0 as it has special privileges.
522 */
523 for (j = 1; j < MAX_LSB_CNT; j++) {
524 if (status & q_mask)
525 bitmap_set(cmd_q->lsbmask, j, 1);
526 status >>= LSB_REGION_WIDTH;
527 }
528 queues = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);
529 dev_info(cmd_q->ccp->dev, "Queue %d can access %d LSB regions\n",
530 cmd_q->id, queues);
531
532 return queues ? 0 : -EINVAL;
533 }
534
535
536 static int ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp,
537 int lsb_cnt, int n_lsbs,
538 unsigned long *lsb_pub)
539 {
540 DECLARE_BITMAP(qlsb, MAX_LSB_CNT);
541 int bitno;
542 int qlsb_wgt;
543 int i;
544
545 /* For each queue:
546 * If the count of potential LSBs available to a queue matches the
547 * ordinal given to us in lsb_cnt:
548 * Copy the mask of possible LSBs for this queue into "qlsb";
549 * For each bit in qlsb, see if the corresponding bit in the
550 * aggregation mask is set; if so, we have a match.
551 * If we have a match, clear the bit in the aggregation to
552 * mark it as no longer available.
553 * If there is no match, clear the bit in qlsb and keep looking.
554 */
555 for (i = 0; i < ccp->cmd_q_count; i++) {
556 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];
557
558 qlsb_wgt = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);
559
560 if (qlsb_wgt == lsb_cnt) {
561 bitmap_copy(qlsb, cmd_q->lsbmask, MAX_LSB_CNT);
562
563 bitno = find_first_bit(qlsb, MAX_LSB_CNT);
564 while (bitno < MAX_LSB_CNT) {
565 if (test_bit(bitno, lsb_pub)) {
566 /* We found an available LSB
567 * that this queue can access
568 */
569 cmd_q->lsb = bitno;
570 bitmap_clear(lsb_pub, bitno, 1);
571 dev_info(ccp->dev,
572 "Queue %d gets LSB %d\n",
573 i, bitno);
574 break;
575 }
576 bitmap_clear(qlsb, bitno, 1);
577 bitno = find_first_bit(qlsb, MAX_LSB_CNT);
578 }
579 if (bitno >= MAX_LSB_CNT)
580 return -EINVAL;
581 n_lsbs--;
582 }
583 }
584 return n_lsbs;
585 }
586
587 /* For each queue, from the most- to least-constrained:
588 * find an LSB that can be assigned to the queue. If there are N queues that
589 * can only use M LSBs, where N > M, fail; otherwise, every queue will get a
590 * dedicated LSB. Remaining LSB regions become a shared resource.
591 * If we have fewer LSBs than queues, all LSB regions become shared resources.
592 */
593 static int ccp_assign_lsbs(struct ccp_device *ccp)
594 {
595 DECLARE_BITMAP(lsb_pub, MAX_LSB_CNT);
596 DECLARE_BITMAP(qlsb, MAX_LSB_CNT);
597 int n_lsbs = 0;
598 int bitno;
599 int i, lsb_cnt;
600 int rc = 0;
601
602 bitmap_zero(lsb_pub, MAX_LSB_CNT);
603
604 /* Create an aggregate bitmap to get a total count of available LSBs */
605 for (i = 0; i < ccp->cmd_q_count; i++)
606 bitmap_or(lsb_pub,
607 lsb_pub, ccp->cmd_q[i].lsbmask,
608 MAX_LSB_CNT);
609
610 n_lsbs = bitmap_weight(lsb_pub, MAX_LSB_CNT);
611
612 if (n_lsbs >= ccp->cmd_q_count) {
613 /* We have enough LSBS to give every queue a private LSB.
614 * Brute force search to start with the queues that are more
615 * constrained in LSB choice. When an LSB is privately
616 * assigned, it is removed from the public mask.
617 * This is an ugly N squared algorithm with some optimization.
618 */
619 for (lsb_cnt = 1;
620 n_lsbs && (lsb_cnt <= MAX_LSB_CNT);
621 lsb_cnt++) {
622 rc = ccp_find_and_assign_lsb_to_q(ccp, lsb_cnt, n_lsbs,
623 lsb_pub);
624 if (rc < 0)
625 return -EINVAL;
626 n_lsbs = rc;
627 }
628 }
629
630 rc = 0;
631 /* What's left of the LSBs, according to the public mask, now become
632 * shared. Any zero bits in the lsb_pub mask represent an LSB region
633 * that can't be used as a shared resource, so mark the LSB slots for
634 * them as "in use".
635 */
636 bitmap_copy(qlsb, lsb_pub, MAX_LSB_CNT);
637
638 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);
639 while (bitno < MAX_LSB_CNT) {
640 bitmap_set(ccp->lsbmap, bitno * LSB_SIZE, LSB_SIZE);
641 bitmap_set(qlsb, bitno, 1);
642 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);
643 }
644
645 return rc;
646 }
647
648 static void ccp5_disable_queue_interrupts(struct ccp_device *ccp)
649 {
650 unsigned int i;
651
652 for (i = 0; i < ccp->cmd_q_count; i++)
653 iowrite32(0x0, ccp->cmd_q[i].reg_int_enable);
654 }
655
656 static void ccp5_enable_queue_interrupts(struct ccp_device *ccp)
657 {
658 unsigned int i;
659
660 for (i = 0; i < ccp->cmd_q_count; i++)
661 iowrite32(SUPPORTED_INTERRUPTS, ccp->cmd_q[i].reg_int_enable);
662 }
663
664 static void ccp5_irq_bh(unsigned long data)
665 {
666 struct ccp_device *ccp = (struct ccp_device *)data;
667 u32 status;
668 unsigned int i;
669
670 for (i = 0; i < ccp->cmd_q_count; i++) {
671 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];
672
673 status = ioread32(cmd_q->reg_interrupt_status);
674
675 if (status) {
676 cmd_q->int_status = status;
677 cmd_q->q_status = ioread32(cmd_q->reg_status);
678 cmd_q->q_int_status = ioread32(cmd_q->reg_int_status);
679
680 /* On error, only save the first error value */
681 if ((status & INT_ERROR) && !cmd_q->cmd_error)
682 cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
683
684 cmd_q->int_rcvd = 1;
685
686 /* Acknowledge the interrupt and wake the kthread */
687 iowrite32(status, cmd_q->reg_interrupt_status);
688 wake_up_interruptible(&cmd_q->int_queue);
689 }
690 }
691 ccp5_enable_queue_interrupts(ccp);
692 }
693
694 static irqreturn_t ccp5_irq_handler(int irq, void *data)
695 {
696 struct device *dev = data;
697 struct ccp_device *ccp = dev_get_drvdata(dev);
698
699 ccp5_disable_queue_interrupts(ccp);
700 if (ccp->use_tasklet)
701 tasklet_schedule(&ccp->irq_tasklet);
702 else
703 ccp5_irq_bh((unsigned long)ccp);
704 return IRQ_HANDLED;
705 }
706
707 static int ccp5_init(struct ccp_device *ccp)
708 {
709 struct device *dev = ccp->dev;
710 struct ccp_cmd_queue *cmd_q;
711 struct dma_pool *dma_pool;
712 char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
713 unsigned int qmr, qim, i;
714 u64 status;
715 u32 status_lo, status_hi;
716 int ret;
717
718 /* Find available queues */
719 qim = 0;
720 qmr = ioread32(ccp->io_regs + Q_MASK_REG);
721 for (i = 0; i < MAX_HW_QUEUES; i++) {
722
723 if (!(qmr & (1 << i)))
724 continue;
725
726 /* Allocate a dma pool for this queue */
727 snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d",
728 ccp->name, i);
729 dma_pool = dma_pool_create(dma_pool_name, dev,
730 CCP_DMAPOOL_MAX_SIZE,
731 CCP_DMAPOOL_ALIGN, 0);
732 if (!dma_pool) {
733 dev_err(dev, "unable to allocate dma pool\n");
734 ret = -ENOMEM;
735 }
736
737 cmd_q = &ccp->cmd_q[ccp->cmd_q_count];
738 ccp->cmd_q_count++;
739
740 cmd_q->ccp = ccp;
741 cmd_q->id = i;
742 cmd_q->dma_pool = dma_pool;
743 mutex_init(&cmd_q->q_mutex);
744
745 /* Page alignment satisfies our needs for N <= 128 */
746 BUILD_BUG_ON(COMMANDS_PER_QUEUE > 128);
747 cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);
748 cmd_q->qbase = dma_zalloc_coherent(dev, cmd_q->qsize,
749 &cmd_q->qbase_dma,
750 GFP_KERNEL);
751 if (!cmd_q->qbase) {
752 dev_err(dev, "unable to allocate command queue\n");
753 ret = -ENOMEM;
754 goto e_pool;
755 }
756
757 cmd_q->qidx = 0;
758 /* Preset some register values and masks that are queue
759 * number dependent
760 */
761 cmd_q->reg_control = ccp->io_regs +
762 CMD5_Q_STATUS_INCR * (i + 1);
763 cmd_q->reg_tail_lo = cmd_q->reg_control + CMD5_Q_TAIL_LO_BASE;
764 cmd_q->reg_head_lo = cmd_q->reg_control + CMD5_Q_HEAD_LO_BASE;
765 cmd_q->reg_int_enable = cmd_q->reg_control +
766 CMD5_Q_INT_ENABLE_BASE;
767 cmd_q->reg_interrupt_status = cmd_q->reg_control +
768 CMD5_Q_INTERRUPT_STATUS_BASE;
769 cmd_q->reg_status = cmd_q->reg_control + CMD5_Q_STATUS_BASE;
770 cmd_q->reg_int_status = cmd_q->reg_control +
771 CMD5_Q_INT_STATUS_BASE;
772 cmd_q->reg_dma_status = cmd_q->reg_control +
773 CMD5_Q_DMA_STATUS_BASE;
774 cmd_q->reg_dma_read_status = cmd_q->reg_control +
775 CMD5_Q_DMA_READ_STATUS_BASE;
776 cmd_q->reg_dma_write_status = cmd_q->reg_control +
777 CMD5_Q_DMA_WRITE_STATUS_BASE;
778
779 init_waitqueue_head(&cmd_q->int_queue);
780
781 dev_dbg(dev, "queue #%u available\n", i);
782 }
783 if (ccp->cmd_q_count == 0) {
784 dev_notice(dev, "no command queues available\n");
785 ret = -EIO;
786 goto e_pool;
787 }
788 dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count);
789
790 /* Turn off the queues and disable interrupts until ready */
791 ccp5_disable_queue_interrupts(ccp);
792 for (i = 0; i < ccp->cmd_q_count; i++) {
793 cmd_q = &ccp->cmd_q[i];
794
795 cmd_q->qcontrol = 0; /* Start with nothing */
796 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
797
798 ioread32(cmd_q->reg_int_status);
799 ioread32(cmd_q->reg_status);
800
801 /* Clear the interrupt status */
802 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status);
803 }
804
805 dev_dbg(dev, "Requesting an IRQ...\n");
806 /* Request an irq */
807 ret = ccp->get_irq(ccp);
808 if (ret) {
809 dev_err(dev, "unable to allocate an IRQ\n");
810 goto e_pool;
811 }
812 /* Initialize the ISR tasklet */
813 if (ccp->use_tasklet)
814 tasklet_init(&ccp->irq_tasklet, ccp5_irq_bh,
815 (unsigned long)ccp);
816
817 /* Initialize the queue used to suspend */
818 init_waitqueue_head(&ccp->suspend_queue);
819
820 dev_dbg(dev, "Loading LSB map...\n");
821 /* Copy the private LSB mask to the public registers */
822 status_lo = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);
823 status_hi = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);
824 iowrite32(status_lo, ccp->io_regs + LSB_PUBLIC_MASK_LO_OFFSET);
825 iowrite32(status_hi, ccp->io_regs + LSB_PUBLIC_MASK_HI_OFFSET);
826 status = ((u64)status_hi<<30) | (u64)status_lo;
827
828 dev_dbg(dev, "Configuring virtual queues...\n");
829 /* Configure size of each virtual queue accessible to host */
830 for (i = 0; i < ccp->cmd_q_count; i++) {
831 u32 dma_addr_lo;
832 u32 dma_addr_hi;
833
834 cmd_q = &ccp->cmd_q[i];
835
836 cmd_q->qcontrol &= ~(CMD5_Q_SIZE << CMD5_Q_SHIFT);
837 cmd_q->qcontrol |= QUEUE_SIZE_VAL << CMD5_Q_SHIFT;
838
839 cmd_q->qdma_tail = cmd_q->qbase_dma;
840 dma_addr_lo = low_address(cmd_q->qdma_tail);
841 iowrite32((u32)dma_addr_lo, cmd_q->reg_tail_lo);
842 iowrite32((u32)dma_addr_lo, cmd_q->reg_head_lo);
843
844 dma_addr_hi = high_address(cmd_q->qdma_tail);
845 cmd_q->qcontrol |= (dma_addr_hi << 16);
846 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
847
848 /* Find the LSB regions accessible to the queue */
849 ccp_find_lsb_regions(cmd_q, status);
850 cmd_q->lsb = -1; /* Unassigned value */
851 }
852
853 dev_dbg(dev, "Assigning LSBs...\n");
854 ret = ccp_assign_lsbs(ccp);
855 if (ret) {
856 dev_err(dev, "Unable to assign LSBs (%d)\n", ret);
857 goto e_irq;
858 }
859
860 /* Optimization: pre-allocate LSB slots for each queue */
861 for (i = 0; i < ccp->cmd_q_count; i++) {
862 ccp->cmd_q[i].sb_key = ccp_lsb_alloc(&ccp->cmd_q[i], 2);
863 ccp->cmd_q[i].sb_ctx = ccp_lsb_alloc(&ccp->cmd_q[i], 2);
864 }
865
866 dev_dbg(dev, "Starting threads...\n");
867 /* Create a kthread for each queue */
868 for (i = 0; i < ccp->cmd_q_count; i++) {
869 struct task_struct *kthread;
870
871 cmd_q = &ccp->cmd_q[i];
872
873 kthread = kthread_create(ccp_cmd_queue_thread, cmd_q,
874 "%s-q%u", ccp->name, cmd_q->id);
875 if (IS_ERR(kthread)) {
876 dev_err(dev, "error creating queue thread (%ld)\n",
877 PTR_ERR(kthread));
878 ret = PTR_ERR(kthread);
879 goto e_kthread;
880 }
881
882 cmd_q->kthread = kthread;
883 wake_up_process(kthread);
884 }
885
886 dev_dbg(dev, "Enabling interrupts...\n");
887 ccp5_enable_queue_interrupts(ccp);
888
889 dev_dbg(dev, "Registering device...\n");
890 /* Put this on the unit list to make it available */
891 ccp_add_device(ccp);
892
893 ret = ccp_register_rng(ccp);
894 if (ret)
895 goto e_kthread;
896
897 /* Register the DMA engine support */
898 ret = ccp_dmaengine_register(ccp);
899 if (ret)
900 goto e_hwrng;
901
902 return 0;
903
904 e_hwrng:
905 ccp_unregister_rng(ccp);
906
907 e_kthread:
908 for (i = 0; i < ccp->cmd_q_count; i++)
909 if (ccp->cmd_q[i].kthread)
910 kthread_stop(ccp->cmd_q[i].kthread);
911
912 e_irq:
913 ccp->free_irq(ccp);
914
915 e_pool:
916 for (i = 0; i < ccp->cmd_q_count; i++)
917 dma_pool_destroy(ccp->cmd_q[i].dma_pool);
918
919 return ret;
920 }
921
922 static void ccp5_destroy(struct ccp_device *ccp)
923 {
924 struct device *dev = ccp->dev;
925 struct ccp_cmd_queue *cmd_q;
926 struct ccp_cmd *cmd;
927 unsigned int i;
928
929 /* Unregister the DMA engine */
930 ccp_dmaengine_unregister(ccp);
931
932 /* Unregister the RNG */
933 ccp_unregister_rng(ccp);
934
935 /* Remove this device from the list of available units first */
936 ccp_del_device(ccp);
937
938 /* Disable and clear interrupts */
939 ccp5_disable_queue_interrupts(ccp);
940 for (i = 0; i < ccp->cmd_q_count; i++) {
941 cmd_q = &ccp->cmd_q[i];
942
943 /* Turn off the run bit */
944 iowrite32(cmd_q->qcontrol & ~CMD5_Q_RUN, cmd_q->reg_control);
945
946 /* Clear the interrupt status */
947 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status);
948 ioread32(cmd_q->reg_int_status);
949 ioread32(cmd_q->reg_status);
950 }
951
952 /* Stop the queue kthreads */
953 for (i = 0; i < ccp->cmd_q_count; i++)
954 if (ccp->cmd_q[i].kthread)
955 kthread_stop(ccp->cmd_q[i].kthread);
956
957 ccp->free_irq(ccp);
958
959 for (i = 0; i < ccp->cmd_q_count; i++) {
960 cmd_q = &ccp->cmd_q[i];
961 dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase,
962 cmd_q->qbase_dma);
963 }
964
965 /* Flush the cmd and backlog queue */
966 while (!list_empty(&ccp->cmd)) {
967 /* Invoke the callback directly with an error code */
968 cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);
969 list_del(&cmd->entry);
970 cmd->callback(cmd->data, -ENODEV);
971 }
972 while (!list_empty(&ccp->backlog)) {
973 /* Invoke the callback directly with an error code */
974 cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry);
975 list_del(&cmd->entry);
976 cmd->callback(cmd->data, -ENODEV);
977 }
978 }
979
980 static void ccp5_config(struct ccp_device *ccp)
981 {
982 /* Public side */
983 iowrite32(0x0, ccp->io_regs + CMD5_REQID_CONFIG_OFFSET);
984 }
985
986 static void ccp5other_config(struct ccp_device *ccp)
987 {
988 int i;
989 u32 rnd;
990
991 /* We own all of the queues on the NTB CCP */
992
993 iowrite32(0x00012D57, ccp->io_regs + CMD5_TRNG_CTL_OFFSET);
994 iowrite32(0x00000003, ccp->io_regs + CMD5_CONFIG_0_OFFSET);
995 for (i = 0; i < 12; i++) {
996 rnd = ioread32(ccp->io_regs + TRNG_OUT_REG);
997 iowrite32(rnd, ccp->io_regs + CMD5_AES_MASK_OFFSET);
998 }
999
1000 iowrite32(0x0000001F, ccp->io_regs + CMD5_QUEUE_MASK_OFFSET);
1001 iowrite32(0x00005B6D, ccp->io_regs + CMD5_QUEUE_PRIO_OFFSET);
1002 iowrite32(0x00000000, ccp->io_regs + CMD5_CMD_TIMEOUT_OFFSET);
1003
1004 iowrite32(0x3FFFFFFF, ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);
1005 iowrite32(0x000003FF, ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);
1006
1007 iowrite32(0x00108823, ccp->io_regs + CMD5_CLK_GATE_CTL_OFFSET);
1008
1009 ccp5_config(ccp);
1010 }
1011
1012 /* Version 5 adds some function, but is essentially the same as v5 */
1013 static const struct ccp_actions ccp5_actions = {
1014 .aes = ccp5_perform_aes,
1015 .xts_aes = ccp5_perform_xts_aes,
1016 .sha = ccp5_perform_sha,
1017 .rsa = ccp5_perform_rsa,
1018 .passthru = ccp5_perform_passthru,
1019 .ecc = ccp5_perform_ecc,
1020 .sballoc = ccp_lsb_alloc,
1021 .sbfree = ccp_lsb_free,
1022 .init = ccp5_init,
1023 .destroy = ccp5_destroy,
1024 .get_free_slots = ccp5_get_free_slots,
1025 .irqhandler = ccp5_irq_handler,
1026 };
1027
1028 const struct ccp_vdata ccpv5a = {
1029 .version = CCP_VERSION(5, 0),
1030 .setup = ccp5_config,
1031 .perform = &ccp5_actions,
1032 .bar = 2,
1033 .offset = 0x0,
1034 };
1035
1036 const struct ccp_vdata ccpv5b = {
1037 .version = CCP_VERSION(5, 0),
1038 .setup = ccp5other_config,
1039 .perform = &ccp5_actions,
1040 .bar = 2,
1041 .offset = 0x0,
1042 };
Linux with added FPC-III support