?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge branch 'akpm' (patches from Andrew)
[linux/fpc-iii.git] / drivers / crypto / picoxcell_crypto.c
blob7384e91c8b32b58280504d9db825df05657555c9
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Copyright (c) 2010-2011 Picochip Ltd., Jamie Iles
4 */
5 #include <crypto/internal/aead.h>
6 #include <crypto/aes.h>
7 #include <crypto/algapi.h>
8 #include <crypto/authenc.h>
9 #include <crypto/internal/des.h>
10 #include <crypto/md5.h>
11 #include <crypto/sha.h>
12 #include <crypto/internal/skcipher.h>
13 #include <linux/clk.h>
14 #include <linux/crypto.h>
15 #include <linux/delay.h>
16 #include <linux/dma-mapping.h>
17 #include <linux/dmapool.h>
18 #include <linux/err.h>
19 #include <linux/init.h>
20 #include <linux/interrupt.h>
21 #include <linux/io.h>
22 #include <linux/list.h>
23 #include <linux/module.h>
24 #include <linux/of.h>
25 #include <linux/platform_device.h>
26 #include <linux/pm.h>
27 #include <linux/rtnetlink.h>
28 #include <linux/scatterlist.h>
29 #include <linux/sched.h>
30 #include <linux/sizes.h>
31 #include <linux/slab.h>
32 #include <linux/timer.h>
33
34 #include "picoxcell_crypto_regs.h"
35
36 /*
37 * The threshold for the number of entries in the CMD FIFO available before
38 * the CMD0_CNT interrupt is raised. Increasing this value will reduce the
39 * number of interrupts raised to the CPU.
40 */
41 #define CMD0_IRQ_THRESHOLD 1
42
43 /*
44 * The timeout period (in jiffies) for a PDU. When the the number of PDUs in
45 * flight is greater than the STAT_IRQ_THRESHOLD or 0 the timer is disabled.
46 * When there are packets in flight but lower than the threshold, we enable
47 * the timer and at expiry, attempt to remove any processed packets from the
48 * queue and if there are still packets left, schedule the timer again.
49 */
50 #define PACKET_TIMEOUT 1
51
52 /* The priority to register each algorithm with. */
53 #define SPACC_CRYPTO_ALG_PRIORITY 10000
54
55 #define SPACC_CRYPTO_KASUMI_F8_KEY_LEN 16
56 #define SPACC_CRYPTO_IPSEC_CIPHER_PG_SZ 64
57 #define SPACC_CRYPTO_IPSEC_HASH_PG_SZ 64
58 #define SPACC_CRYPTO_IPSEC_MAX_CTXS 32
59 #define SPACC_CRYPTO_IPSEC_FIFO_SZ 32
60 #define SPACC_CRYPTO_L2_CIPHER_PG_SZ 64
61 #define SPACC_CRYPTO_L2_HASH_PG_SZ 64
62 #define SPACC_CRYPTO_L2_MAX_CTXS 128
63 #define SPACC_CRYPTO_L2_FIFO_SZ 128
64
65 #define MAX_DDT_LEN 16
66
67 /* DDT format. This must match the hardware DDT format exactly. */
68 struct spacc_ddt {
69 dma_addr_t p;
70 u32 len;
71 };
72
73 /*
74 * Asynchronous crypto request structure.
75 *
76 * This structure defines a request that is either queued for processing or
77 * being processed.
78 */
79 struct spacc_req {
80 struct list_head list;
81 struct spacc_engine *engine;
82 struct crypto_async_request *req;
83 int result;
84 bool is_encrypt;
85 unsigned ctx_id;
86 dma_addr_t src_addr, dst_addr;
87 struct spacc_ddt *src_ddt, *dst_ddt;
88 void (*complete)(struct spacc_req *req);
89 };
90
91 struct spacc_aead {
92 unsigned long ctrl_default;
93 unsigned long type;
94 struct aead_alg alg;
95 struct spacc_engine *engine;
96 struct list_head entry;
97 int key_offs;
98 int iv_offs;
99 };
100
101 struct spacc_engine {
102 void __iomem *regs;
103 struct list_head pending;
104 int next_ctx;
105 spinlock_t hw_lock;
106 int in_flight;
107 struct list_head completed;
108 struct list_head in_progress;
109 struct tasklet_struct complete;
110 unsigned long fifo_sz;
111 void __iomem *cipher_ctx_base;
112 void __iomem *hash_key_base;
113 struct spacc_alg *algs;
114 unsigned num_algs;
115 struct list_head registered_algs;
116 struct spacc_aead *aeads;
117 unsigned num_aeads;
118 struct list_head registered_aeads;
119 size_t cipher_pg_sz;
120 size_t hash_pg_sz;
121 const char *name;
122 struct clk *clk;
123 struct device *dev;
124 unsigned max_ctxs;
125 struct timer_list packet_timeout;
126 unsigned stat_irq_thresh;
127 struct dma_pool *req_pool;
128 };
129
130 /* Algorithm type mask. */
131 #define SPACC_CRYPTO_ALG_MASK 0x7
132
133 /* SPACC definition of a crypto algorithm. */
134 struct spacc_alg {
135 unsigned long ctrl_default;
136 unsigned long type;
137 struct skcipher_alg alg;
138 struct spacc_engine *engine;
139 struct list_head entry;
140 int key_offs;
141 int iv_offs;
142 };
143
144 /* Generic context structure for any algorithm type. */
145 struct spacc_generic_ctx {
146 struct spacc_engine *engine;
147 int flags;
148 int key_offs;
149 int iv_offs;
150 };
151
152 /* Block cipher context. */
153 struct spacc_ablk_ctx {
154 struct spacc_generic_ctx generic;
155 u8 key[AES_MAX_KEY_SIZE];
156 u8 key_len;
157 /*
158 * The fallback cipher. If the operation can't be done in hardware,
159 * fallback to a software version.
160 */
161 struct crypto_sync_skcipher *sw_cipher;
162 };
163
164 /* AEAD cipher context. */
165 struct spacc_aead_ctx {
166 struct spacc_generic_ctx generic;
167 u8 cipher_key[AES_MAX_KEY_SIZE];
168 u8 hash_ctx[SPACC_CRYPTO_IPSEC_HASH_PG_SZ];
169 u8 cipher_key_len;
170 u8 hash_key_len;
171 struct crypto_aead *sw_cipher;
172 };
173
174 static int spacc_ablk_submit(struct spacc_req *req);
175
176 static inline struct spacc_alg *to_spacc_skcipher(struct skcipher_alg *alg)
177 {
178 return alg ? container_of(alg, struct spacc_alg, alg) : NULL;
179 }
180
181 static inline struct spacc_aead *to_spacc_aead(struct aead_alg *alg)
182 {
183 return container_of(alg, struct spacc_aead, alg);
184 }
185
186 static inline int spacc_fifo_cmd_full(struct spacc_engine *engine)
187 {
188 u32 fifo_stat = readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET);
189
190 return fifo_stat & SPA_FIFO_CMD_FULL;
191 }
192
193 /*
194 * Given a cipher context, and a context number, get the base address of the
195 * context page.
196 *
197 * Returns the address of the context page where the key/context may
198 * be written.
199 */
200 static inline void __iomem *spacc_ctx_page_addr(struct spacc_generic_ctx *ctx,
201 unsigned indx,
202 bool is_cipher_ctx)
203 {
204 return is_cipher_ctx ? ctx->engine->cipher_ctx_base +
205 (indx * ctx->engine->cipher_pg_sz) :
206 ctx->engine->hash_key_base + (indx * ctx->engine->hash_pg_sz);
207 }
208
209 /* The context pages can only be written with 32-bit accesses. */
210 static inline void memcpy_toio32(u32 __iomem *dst, const void *src,
211 unsigned count)
212 {
213 const u32 *src32 = (const u32 *) src;
214
215 while (count--)
216 writel(*src32++, dst++);
217 }
218
219 static void spacc_cipher_write_ctx(struct spacc_generic_ctx *ctx,
220 void __iomem *page_addr, const u8 *key,
221 size_t key_len, const u8 *iv, size_t iv_len)
222 {
223 void __iomem *key_ptr = page_addr + ctx->key_offs;
224 void __iomem *iv_ptr = page_addr + ctx->iv_offs;
225
226 memcpy_toio32(key_ptr, key, key_len / 4);
227 memcpy_toio32(iv_ptr, iv, iv_len / 4);
228 }
229
230 /*
231 * Load a context into the engines context memory.
232 *
233 * Returns the index of the context page where the context was loaded.
234 */
235 static unsigned spacc_load_ctx(struct spacc_generic_ctx *ctx,
236 const u8 *ciph_key, size_t ciph_len,
237 const u8 *iv, size_t ivlen, const u8 *hash_key,
238 size_t hash_len)
239 {
240 unsigned indx = ctx->engine->next_ctx++;
241 void __iomem *ciph_page_addr, *hash_page_addr;
242
243 ciph_page_addr = spacc_ctx_page_addr(ctx, indx, 1);
244 hash_page_addr = spacc_ctx_page_addr(ctx, indx, 0);
245
246 ctx->engine->next_ctx &= ctx->engine->fifo_sz - 1;
247 spacc_cipher_write_ctx(ctx, ciph_page_addr, ciph_key, ciph_len, iv,
248 ivlen);
249 writel(ciph_len | (indx << SPA_KEY_SZ_CTX_INDEX_OFFSET) |
250 (1 << SPA_KEY_SZ_CIPHER_OFFSET),
251 ctx->engine->regs + SPA_KEY_SZ_REG_OFFSET);
252
253 if (hash_key) {
254 memcpy_toio32(hash_page_addr, hash_key, hash_len / 4);
255 writel(hash_len | (indx << SPA_KEY_SZ_CTX_INDEX_OFFSET),
256 ctx->engine->regs + SPA_KEY_SZ_REG_OFFSET);
257 }
258
259 return indx;
260 }
261
262 static inline void ddt_set(struct spacc_ddt *ddt, dma_addr_t phys, size_t len)
263 {
264 ddt->p = phys;
265 ddt->len = len;
266 }
267
268 /*
269 * Take a crypto request and scatterlists for the data and turn them into DDTs
270 * for passing to the crypto engines. This also DMA maps the data so that the
271 * crypto engines can DMA to/from them.
272 */
273 static struct spacc_ddt *spacc_sg_to_ddt(struct spacc_engine *engine,
274 struct scatterlist *payload,
275 unsigned nbytes,
276 enum dma_data_direction dir,
277 dma_addr_t *ddt_phys)
278 {
279 unsigned mapped_ents;
280 struct scatterlist *cur;
281 struct spacc_ddt *ddt;
282 int i;
283 int nents;
284
285 nents = sg_nents_for_len(payload, nbytes);
286 if (nents < 0) {
287 dev_err(engine->dev, "Invalid numbers of SG.\n");
288 return NULL;
289 }
290 mapped_ents = dma_map_sg(engine->dev, payload, nents, dir);
291
292 if (mapped_ents + 1 > MAX_DDT_LEN)
293 goto out;
294
295 ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, ddt_phys);
296 if (!ddt)
297 goto out;
298
299 for_each_sg(payload, cur, mapped_ents, i)
300 ddt_set(&ddt[i], sg_dma_address(cur), sg_dma_len(cur));
301 ddt_set(&ddt[mapped_ents], 0, 0);
302
303 return ddt;
304
305 out:
306 dma_unmap_sg(engine->dev, payload, nents, dir);
307 return NULL;
308 }
309
310 static int spacc_aead_make_ddts(struct aead_request *areq)
311 {
312 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
313 struct spacc_req *req = aead_request_ctx(areq);
314 struct spacc_engine *engine = req->engine;
315 struct spacc_ddt *src_ddt, *dst_ddt;
316 unsigned total;
317 int src_nents, dst_nents;
318 struct scatterlist *cur;
319 int i, dst_ents, src_ents;
320
321 total = areq->assoclen + areq->cryptlen;
322 if (req->is_encrypt)
323 total += crypto_aead_authsize(aead);
324
325 src_nents = sg_nents_for_len(areq->src, total);
326 if (src_nents < 0) {
327 dev_err(engine->dev, "Invalid numbers of src SG.\n");
328 return src_nents;
329 }
330 if (src_nents + 1 > MAX_DDT_LEN)
331 return -E2BIG;
332
333 dst_nents = 0;
334 if (areq->src != areq->dst) {
335 dst_nents = sg_nents_for_len(areq->dst, total);
336 if (dst_nents < 0) {
337 dev_err(engine->dev, "Invalid numbers of dst SG.\n");
338 return dst_nents;
339 }
340 if (src_nents + 1 > MAX_DDT_LEN)
341 return -E2BIG;
342 }
343
344 src_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->src_addr);
345 if (!src_ddt)
346 goto err;
347
348 dst_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->dst_addr);
349 if (!dst_ddt)
350 goto err_free_src;
351
352 req->src_ddt = src_ddt;
353 req->dst_ddt = dst_ddt;
354
355 if (dst_nents) {
356 src_ents = dma_map_sg(engine->dev, areq->src, src_nents,
357 DMA_TO_DEVICE);
358 if (!src_ents)
359 goto err_free_dst;
360
361 dst_ents = dma_map_sg(engine->dev, areq->dst, dst_nents,
362 DMA_FROM_DEVICE);
363
364 if (!dst_ents) {
365 dma_unmap_sg(engine->dev, areq->src, src_nents,
366 DMA_TO_DEVICE);
367 goto err_free_dst;
368 }
369 } else {
370 src_ents = dma_map_sg(engine->dev, areq->src, src_nents,
371 DMA_BIDIRECTIONAL);
372 if (!src_ents)
373 goto err_free_dst;
374 dst_ents = src_ents;
375 }
376
377 /*
378 * Now map in the payload for the source and destination and terminate
379 * with the NULL pointers.
380 */
381 for_each_sg(areq->src, cur, src_ents, i)
382 ddt_set(src_ddt++, sg_dma_address(cur), sg_dma_len(cur));
383
384 /* For decryption we need to skip the associated data. */
385 total = req->is_encrypt ? 0 : areq->assoclen;
386 for_each_sg(areq->dst, cur, dst_ents, i) {
387 unsigned len = sg_dma_len(cur);
388
389 if (len <= total) {
390 total -= len;
391 continue;
392 }
393
394 ddt_set(dst_ddt++, sg_dma_address(cur) + total, len - total);
395 }
396
397 ddt_set(src_ddt, 0, 0);
398 ddt_set(dst_ddt, 0, 0);
399
400 return 0;
401
402 err_free_dst:
403 dma_pool_free(engine->req_pool, dst_ddt, req->dst_addr);
404 err_free_src:
405 dma_pool_free(engine->req_pool, src_ddt, req->src_addr);
406 err:
407 return -ENOMEM;
408 }
409
410 static void spacc_aead_free_ddts(struct spacc_req *req)
411 {
412 struct aead_request *areq = container_of(req->req, struct aead_request,
413 base);
414 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
415 unsigned total = areq->assoclen + areq->cryptlen +
416 (req->is_encrypt ? crypto_aead_authsize(aead) : 0);
417 struct spacc_aead_ctx *aead_ctx = crypto_aead_ctx(aead);
418 struct spacc_engine *engine = aead_ctx->generic.engine;
419 int nents = sg_nents_for_len(areq->src, total);
420
421 /* sg_nents_for_len should not fail since it works when mapping sg */
422 if (unlikely(nents < 0)) {
423 dev_err(engine->dev, "Invalid numbers of src SG.\n");
424 return;
425 }
426
427 if (areq->src != areq->dst) {
428 dma_unmap_sg(engine->dev, areq->src, nents, DMA_TO_DEVICE);
429 nents = sg_nents_for_len(areq->dst, total);
430 if (unlikely(nents < 0)) {
431 dev_err(engine->dev, "Invalid numbers of dst SG.\n");
432 return;
433 }
434 dma_unmap_sg(engine->dev, areq->dst, nents, DMA_FROM_DEVICE);
435 } else
436 dma_unmap_sg(engine->dev, areq->src, nents, DMA_BIDIRECTIONAL);
437
438 dma_pool_free(engine->req_pool, req->src_ddt, req->src_addr);
439 dma_pool_free(engine->req_pool, req->dst_ddt, req->dst_addr);
440 }
441
442 static void spacc_free_ddt(struct spacc_req *req, struct spacc_ddt *ddt,
443 dma_addr_t ddt_addr, struct scatterlist *payload,
444 unsigned nbytes, enum dma_data_direction dir)
445 {
446 int nents = sg_nents_for_len(payload, nbytes);
447
448 if (nents < 0) {
449 dev_err(req->engine->dev, "Invalid numbers of SG.\n");
450 return;
451 }
452
453 dma_unmap_sg(req->engine->dev, payload, nents, dir);
454 dma_pool_free(req->engine->req_pool, ddt, ddt_addr);
455 }
456
457 static int spacc_aead_setkey(struct crypto_aead *tfm, const u8 *key,
458 unsigned int keylen)
459 {
460 struct spacc_aead_ctx *ctx = crypto_aead_ctx(tfm);
461 struct crypto_authenc_keys keys;
462 int err;
463
464 crypto_aead_clear_flags(ctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
465 crypto_aead_set_flags(ctx->sw_cipher, crypto_aead_get_flags(tfm) &
466 CRYPTO_TFM_REQ_MASK);
467 err = crypto_aead_setkey(ctx->sw_cipher, key, keylen);
468 if (err)
469 return err;
470
471 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
472 goto badkey;
473
474 if (keys.enckeylen > AES_MAX_KEY_SIZE)
475 goto badkey;
476
477 if (keys.authkeylen > sizeof(ctx->hash_ctx))
478 goto badkey;
479
480 memcpy(ctx->cipher_key, keys.enckey, keys.enckeylen);
481 ctx->cipher_key_len = keys.enckeylen;
482
483 memcpy(ctx->hash_ctx, keys.authkey, keys.authkeylen);
484 ctx->hash_key_len = keys.authkeylen;
485
486 memzero_explicit(&keys, sizeof(keys));
487 return 0;
488
489 badkey:
490 memzero_explicit(&keys, sizeof(keys));
491 return -EINVAL;
492 }
493
494 static int spacc_aead_setauthsize(struct crypto_aead *tfm,
495 unsigned int authsize)
496 {
497 struct spacc_aead_ctx *ctx = crypto_tfm_ctx(crypto_aead_tfm(tfm));
498
499 return crypto_aead_setauthsize(ctx->sw_cipher, authsize);
500 }
501
502 /*
503 * Check if an AEAD request requires a fallback operation. Some requests can't
504 * be completed in hardware because the hardware may not support certain key
505 * sizes. In these cases we need to complete the request in software.
506 */
507 static int spacc_aead_need_fallback(struct aead_request *aead_req)
508 {
509 struct crypto_aead *aead = crypto_aead_reqtfm(aead_req);
510 struct aead_alg *alg = crypto_aead_alg(aead);
511 struct spacc_aead *spacc_alg = to_spacc_aead(alg);
512 struct spacc_aead_ctx *ctx = crypto_aead_ctx(aead);
513
514 /*
515 * If we have a non-supported key-length, then we need to do a
516 * software fallback.
517 */
518 if ((spacc_alg->ctrl_default & SPACC_CRYPTO_ALG_MASK) ==
519 SPA_CTRL_CIPH_ALG_AES &&
520 ctx->cipher_key_len != AES_KEYSIZE_128 &&
521 ctx->cipher_key_len != AES_KEYSIZE_256)
522 return 1;
523
524 return 0;
525 }
526
527 static int spacc_aead_do_fallback(struct aead_request *req, unsigned alg_type,
528 bool is_encrypt)
529 {
530 struct crypto_tfm *old_tfm = crypto_aead_tfm(crypto_aead_reqtfm(req));
531 struct spacc_aead_ctx *ctx = crypto_tfm_ctx(old_tfm);
532 struct aead_request *subreq = aead_request_ctx(req);
533
534 aead_request_set_tfm(subreq, ctx->sw_cipher);
535 aead_request_set_callback(subreq, req->base.flags,
536 req->base.complete, req->base.data);
537 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
538 req->iv);
539 aead_request_set_ad(subreq, req->assoclen);
540
541 return is_encrypt ? crypto_aead_encrypt(subreq) :
542 crypto_aead_decrypt(subreq);
543 }
544
545 static void spacc_aead_complete(struct spacc_req *req)
546 {
547 spacc_aead_free_ddts(req);
548 req->req->complete(req->req, req->result);
549 }
550
551 static int spacc_aead_submit(struct spacc_req *req)
552 {
553 struct aead_request *aead_req =
554 container_of(req->req, struct aead_request, base);
555 struct crypto_aead *aead = crypto_aead_reqtfm(aead_req);
556 unsigned int authsize = crypto_aead_authsize(aead);
557 struct spacc_aead_ctx *ctx = crypto_aead_ctx(aead);
558 struct aead_alg *alg = crypto_aead_alg(aead);
559 struct spacc_aead *spacc_alg = to_spacc_aead(alg);
560 struct spacc_engine *engine = ctx->generic.engine;
561 u32 ctrl, proc_len, assoc_len;
562
563 req->result = -EINPROGRESS;
564 req->ctx_id = spacc_load_ctx(&ctx->generic, ctx->cipher_key,
565 ctx->cipher_key_len, aead_req->iv, crypto_aead_ivsize(aead),
566 ctx->hash_ctx, ctx->hash_key_len);
567
568 /* Set the source and destination DDT pointers. */
569 writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET);
570 writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET);
571 writel(0, engine->regs + SPA_OFFSET_REG_OFFSET);
572
573 assoc_len = aead_req->assoclen;
574 proc_len = aead_req->cryptlen + assoc_len;
575
576 /*
577 * If we are decrypting, we need to take the length of the ICV out of
578 * the processing length.
579 */
580 if (!req->is_encrypt)
581 proc_len -= authsize;
582
583 writel(proc_len, engine->regs + SPA_PROC_LEN_REG_OFFSET);
584 writel(assoc_len, engine->regs + SPA_AAD_LEN_REG_OFFSET);
585 writel(authsize, engine->regs + SPA_ICV_LEN_REG_OFFSET);
586 writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET);
587 writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET);
588
589 ctrl = spacc_alg->ctrl_default | (req->ctx_id << SPA_CTRL_CTX_IDX) |
590 (1 << SPA_CTRL_ICV_APPEND);
591 if (req->is_encrypt)
592 ctrl |= (1 << SPA_CTRL_ENCRYPT_IDX) | (1 << SPA_CTRL_AAD_COPY);
593 else
594 ctrl |= (1 << SPA_CTRL_KEY_EXP);
595
596 mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT);
597
598 writel(ctrl, engine->regs + SPA_CTRL_REG_OFFSET);
599
600 return -EINPROGRESS;
601 }
602
603 static int spacc_req_submit(struct spacc_req *req);
604
605 static void spacc_push(struct spacc_engine *engine)
606 {
607 struct spacc_req *req;
608
609 while (!list_empty(&engine->pending) &&
610 engine->in_flight + 1 <= engine->fifo_sz) {
611
612 ++engine->in_flight;
613 req = list_first_entry(&engine->pending, struct spacc_req,
614 list);
615 list_move_tail(&req->list, &engine->in_progress);
616
617 req->result = spacc_req_submit(req);
618 }
619 }
620
621 /*
622 * Setup an AEAD request for processing. This will configure the engine, load
623 * the context and then start the packet processing.
624 */
625 static int spacc_aead_setup(struct aead_request *req,
626 unsigned alg_type, bool is_encrypt)
627 {
628 struct crypto_aead *aead = crypto_aead_reqtfm(req);
629 struct aead_alg *alg = crypto_aead_alg(aead);
630 struct spacc_engine *engine = to_spacc_aead(alg)->engine;
631 struct spacc_req *dev_req = aead_request_ctx(req);
632 int err;
633 unsigned long flags;
634
635 dev_req->req = &req->base;
636 dev_req->is_encrypt = is_encrypt;
637 dev_req->result = -EBUSY;
638 dev_req->engine = engine;
639 dev_req->complete = spacc_aead_complete;
640
641 if (unlikely(spacc_aead_need_fallback(req) ||
642 ((err = spacc_aead_make_ddts(req)) == -E2BIG)))
643 return spacc_aead_do_fallback(req, alg_type, is_encrypt);
644
645 if (err)
646 goto out;
647
648 err = -EINPROGRESS;
649 spin_lock_irqsave(&engine->hw_lock, flags);
650 if (unlikely(spacc_fifo_cmd_full(engine)) ||
651 engine->in_flight + 1 > engine->fifo_sz) {
652 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
653 err = -EBUSY;
654 spin_unlock_irqrestore(&engine->hw_lock, flags);
655 goto out_free_ddts;
656 }
657 list_add_tail(&dev_req->list, &engine->pending);
658 } else {
659 list_add_tail(&dev_req->list, &engine->pending);
660 spacc_push(engine);
661 }
662 spin_unlock_irqrestore(&engine->hw_lock, flags);
663
664 goto out;
665
666 out_free_ddts:
667 spacc_aead_free_ddts(dev_req);
668 out:
669 return err;
670 }
671
672 static int spacc_aead_encrypt(struct aead_request *req)
673 {
674 struct crypto_aead *aead = crypto_aead_reqtfm(req);
675 struct spacc_aead *alg = to_spacc_aead(crypto_aead_alg(aead));
676
677 return spacc_aead_setup(req, alg->type, 1);
678 }
679
680 static int spacc_aead_decrypt(struct aead_request *req)
681 {
682 struct crypto_aead *aead = crypto_aead_reqtfm(req);
683 struct spacc_aead *alg = to_spacc_aead(crypto_aead_alg(aead));
684
685 return spacc_aead_setup(req, alg->type, 0);
686 }
687
688 /*
689 * Initialise a new AEAD context. This is responsible for allocating the
690 * fallback cipher and initialising the context.
691 */
692 static int spacc_aead_cra_init(struct crypto_aead *tfm)
693 {
694 struct spacc_aead_ctx *ctx = crypto_aead_ctx(tfm);
695 struct aead_alg *alg = crypto_aead_alg(tfm);
696 struct spacc_aead *spacc_alg = to_spacc_aead(alg);
697 struct spacc_engine *engine = spacc_alg->engine;
698
699 ctx->generic.flags = spacc_alg->type;
700 ctx->generic.engine = engine;
701 ctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0,
702 CRYPTO_ALG_NEED_FALLBACK);
703 if (IS_ERR(ctx->sw_cipher))
704 return PTR_ERR(ctx->sw_cipher);
705 ctx->generic.key_offs = spacc_alg->key_offs;
706 ctx->generic.iv_offs = spacc_alg->iv_offs;
707
708 crypto_aead_set_reqsize(
709 tfm,
710 max(sizeof(struct spacc_req),
711 sizeof(struct aead_request) +
712 crypto_aead_reqsize(ctx->sw_cipher)));
713
714 return 0;
715 }
716
717 /*
718 * Destructor for an AEAD context. This is called when the transform is freed
719 * and must free the fallback cipher.
720 */
721 static void spacc_aead_cra_exit(struct crypto_aead *tfm)
722 {
723 struct spacc_aead_ctx *ctx = crypto_aead_ctx(tfm);
724
725 crypto_free_aead(ctx->sw_cipher);
726 }
727
728 /*
729 * Set the DES key for a block cipher transform. This also performs weak key
730 * checking if the transform has requested it.
731 */
732 static int spacc_des_setkey(struct crypto_skcipher *cipher, const u8 *key,
733 unsigned int len)
734 {
735 struct spacc_ablk_ctx *ctx = crypto_skcipher_ctx(cipher);
736 int err;
737
738 err = verify_skcipher_des_key(cipher, key);
739 if (err)
740 return err;
741
742 memcpy(ctx->key, key, len);
743 ctx->key_len = len;
744
745 return 0;
746 }
747
748 /*
749 * Set the 3DES key for a block cipher transform. This also performs weak key
750 * checking if the transform has requested it.
751 */
752 static int spacc_des3_setkey(struct crypto_skcipher *cipher, const u8 *key,
753 unsigned int len)
754 {
755 struct spacc_ablk_ctx *ctx = crypto_skcipher_ctx(cipher);
756 int err;
757
758 err = verify_skcipher_des3_key(cipher, key);
759 if (err)
760 return err;
761
762 memcpy(ctx->key, key, len);
763 ctx->key_len = len;
764
765 return 0;
766 }
767
768 /*
769 * Set the key for an AES block cipher. Some key lengths are not supported in
770 * hardware so this must also check whether a fallback is needed.
771 */
772 static int spacc_aes_setkey(struct crypto_skcipher *cipher, const u8 *key,
773 unsigned int len)
774 {
775 struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher);
776 struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm);
777 int err = 0;
778
779 if (len > AES_MAX_KEY_SIZE)
780 return -EINVAL;
781
782 /*
783 * IPSec engine only supports 128 and 256 bit AES keys. If we get a
784 * request for any other size (192 bits) then we need to do a software
785 * fallback.
786 */
787 if (len != AES_KEYSIZE_128 && len != AES_KEYSIZE_256) {
788 if (!ctx->sw_cipher)
789 return -EINVAL;
790
791 /*
792 * Set the fallback transform to use the same request flags as
793 * the hardware transform.
794 */
795 crypto_sync_skcipher_clear_flags(ctx->sw_cipher,
796 CRYPTO_TFM_REQ_MASK);
797 crypto_sync_skcipher_set_flags(ctx->sw_cipher,
798 cipher->base.crt_flags &
799 CRYPTO_TFM_REQ_MASK);
800
801 err = crypto_sync_skcipher_setkey(ctx->sw_cipher, key, len);
802 if (err)
803 goto sw_setkey_failed;
804 }
805
806 memcpy(ctx->key, key, len);
807 ctx->key_len = len;
808
809 sw_setkey_failed:
810 return err;
811 }
812
813 static int spacc_kasumi_f8_setkey(struct crypto_skcipher *cipher,
814 const u8 *key, unsigned int len)
815 {
816 struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher);
817 struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm);
818 int err = 0;
819
820 if (len > AES_MAX_KEY_SIZE) {
821 err = -EINVAL;
822 goto out;
823 }
824
825 memcpy(ctx->key, key, len);
826 ctx->key_len = len;
827
828 out:
829 return err;
830 }
831
832 static int spacc_ablk_need_fallback(struct spacc_req *req)
833 {
834 struct skcipher_request *ablk_req = skcipher_request_cast(req->req);
835 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(ablk_req);
836 struct spacc_alg *spacc_alg = to_spacc_skcipher(crypto_skcipher_alg(tfm));
837 struct spacc_ablk_ctx *ctx;
838
839 ctx = crypto_skcipher_ctx(tfm);
840
841 return (spacc_alg->ctrl_default & SPACC_CRYPTO_ALG_MASK) ==
842 SPA_CTRL_CIPH_ALG_AES &&
843 ctx->key_len != AES_KEYSIZE_128 &&
844 ctx->key_len != AES_KEYSIZE_256;
845 }
846
847 static void spacc_ablk_complete(struct spacc_req *req)
848 {
849 struct skcipher_request *ablk_req = skcipher_request_cast(req->req);
850
851 if (ablk_req->src != ablk_req->dst) {
852 spacc_free_ddt(req, req->src_ddt, req->src_addr, ablk_req->src,
853 ablk_req->cryptlen, DMA_TO_DEVICE);
854 spacc_free_ddt(req, req->dst_ddt, req->dst_addr, ablk_req->dst,
855 ablk_req->cryptlen, DMA_FROM_DEVICE);
856 } else
857 spacc_free_ddt(req, req->dst_ddt, req->dst_addr, ablk_req->dst,
858 ablk_req->cryptlen, DMA_BIDIRECTIONAL);
859
860 req->req->complete(req->req, req->result);
861 }
862
863 static int spacc_ablk_submit(struct spacc_req *req)
864 {
865 struct skcipher_request *ablk_req = skcipher_request_cast(req->req);
866 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(ablk_req);
867 struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
868 struct spacc_alg *spacc_alg = to_spacc_skcipher(alg);
869 struct spacc_ablk_ctx *ctx = crypto_skcipher_ctx(tfm);
870 struct spacc_engine *engine = ctx->generic.engine;
871 u32 ctrl;
872
873 req->ctx_id = spacc_load_ctx(&ctx->generic, ctx->key,
874 ctx->key_len, ablk_req->iv, alg->ivsize,
875 NULL, 0);
876
877 writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET);
878 writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET);
879 writel(0, engine->regs + SPA_OFFSET_REG_OFFSET);
880
881 writel(ablk_req->cryptlen, engine->regs + SPA_PROC_LEN_REG_OFFSET);
882 writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET);
883 writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET);
884 writel(0, engine->regs + SPA_AAD_LEN_REG_OFFSET);
885
886 ctrl = spacc_alg->ctrl_default | (req->ctx_id << SPA_CTRL_CTX_IDX) |
887 (req->is_encrypt ? (1 << SPA_CTRL_ENCRYPT_IDX) :
888 (1 << SPA_CTRL_KEY_EXP));
889
890 mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT);
891
892 writel(ctrl, engine->regs + SPA_CTRL_REG_OFFSET);
893
894 return -EINPROGRESS;
895 }
896
897 static int spacc_ablk_do_fallback(struct skcipher_request *req,
898 unsigned alg_type, bool is_encrypt)
899 {
900 struct crypto_tfm *old_tfm =
901 crypto_skcipher_tfm(crypto_skcipher_reqtfm(req));
902 struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(old_tfm);
903 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->sw_cipher);
904 int err;
905
906 /*
907 * Change the request to use the software fallback transform, and once
908 * the ciphering has completed, put the old transform back into the
909 * request.
910 */
911 skcipher_request_set_sync_tfm(subreq, ctx->sw_cipher);
912 skcipher_request_set_callback(subreq, req->base.flags, NULL, NULL);
913 skcipher_request_set_crypt(subreq, req->src, req->dst,
914 req->cryptlen, req->iv);
915 err = is_encrypt ? crypto_skcipher_encrypt(subreq) :
916 crypto_skcipher_decrypt(subreq);
917 skcipher_request_zero(subreq);
918
919 return err;
920 }
921
922 static int spacc_ablk_setup(struct skcipher_request *req, unsigned alg_type,
923 bool is_encrypt)
924 {
925 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
926 struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
927 struct spacc_engine *engine = to_spacc_skcipher(alg)->engine;
928 struct spacc_req *dev_req = skcipher_request_ctx(req);
929 unsigned long flags;
930 int err = -ENOMEM;
931
932 dev_req->req = &req->base;
933 dev_req->is_encrypt = is_encrypt;
934 dev_req->engine = engine;
935 dev_req->complete = spacc_ablk_complete;
936 dev_req->result = -EINPROGRESS;
937
938 if (unlikely(spacc_ablk_need_fallback(dev_req)))
939 return spacc_ablk_do_fallback(req, alg_type, is_encrypt);
940
941 /*
942 * Create the DDT's for the engine. If we share the same source and
943 * destination then we can optimize by reusing the DDT's.
944 */
945 if (req->src != req->dst) {
946 dev_req->src_ddt = spacc_sg_to_ddt(engine, req->src,
947 req->cryptlen, DMA_TO_DEVICE, &dev_req->src_addr);
948 if (!dev_req->src_ddt)
949 goto out;
950
951 dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst,
952 req->cryptlen, DMA_FROM_DEVICE, &dev_req->dst_addr);
953 if (!dev_req->dst_ddt)
954 goto out_free_src;
955 } else {
956 dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst,
957 req->cryptlen, DMA_BIDIRECTIONAL, &dev_req->dst_addr);
958 if (!dev_req->dst_ddt)
959 goto out;
960
961 dev_req->src_ddt = NULL;
962 dev_req->src_addr = dev_req->dst_addr;
963 }
964
965 err = -EINPROGRESS;
966 spin_lock_irqsave(&engine->hw_lock, flags);
967 /*
968 * Check if the engine will accept the operation now. If it won't then
969 * we either stick it on the end of a pending list if we can backlog,
970 * or bailout with an error if not.
971 */
972 if (unlikely(spacc_fifo_cmd_full(engine)) ||
973 engine->in_flight + 1 > engine->fifo_sz) {
974 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
975 err = -EBUSY;
976 spin_unlock_irqrestore(&engine->hw_lock, flags);
977 goto out_free_ddts;
978 }
979 list_add_tail(&dev_req->list, &engine->pending);
980 } else {
981 list_add_tail(&dev_req->list, &engine->pending);
982 spacc_push(engine);
983 }
984 spin_unlock_irqrestore(&engine->hw_lock, flags);
985
986 goto out;
987
988 out_free_ddts:
989 spacc_free_ddt(dev_req, dev_req->dst_ddt, dev_req->dst_addr, req->dst,
990 req->cryptlen, req->src == req->dst ?
991 DMA_BIDIRECTIONAL : DMA_FROM_DEVICE);
992 out_free_src:
993 if (req->src != req->dst)
994 spacc_free_ddt(dev_req, dev_req->src_ddt, dev_req->src_addr,
995 req->src, req->cryptlen, DMA_TO_DEVICE);
996 out:
997 return err;
998 }
999
1000 static int spacc_ablk_init_tfm(struct crypto_skcipher *tfm)
1001 {
1002 struct spacc_ablk_ctx *ctx = crypto_skcipher_ctx(tfm);
1003 struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
1004 struct spacc_alg *spacc_alg = to_spacc_skcipher(alg);
1005 struct spacc_engine *engine = spacc_alg->engine;
1006
1007 ctx->generic.flags = spacc_alg->type;
1008 ctx->generic.engine = engine;
1009 if (alg->base.cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
1010 ctx->sw_cipher = crypto_alloc_sync_skcipher(
1011 alg->base.cra_name, 0, CRYPTO_ALG_NEED_FALLBACK);
1012 if (IS_ERR(ctx->sw_cipher)) {
1013 dev_warn(engine->dev, "failed to allocate fallback for %s\n",
1014 alg->base.cra_name);
1015 return PTR_ERR(ctx->sw_cipher);
1016 }
1017 }
1018 ctx->generic.key_offs = spacc_alg->key_offs;
1019 ctx->generic.iv_offs = spacc_alg->iv_offs;
1020
1021 crypto_skcipher_set_reqsize(tfm, sizeof(struct spacc_req));
1022
1023 return 0;
1024 }
1025
1026 static void spacc_ablk_exit_tfm(struct crypto_skcipher *tfm)
1027 {
1028 struct spacc_ablk_ctx *ctx = crypto_skcipher_ctx(tfm);
1029
1030 crypto_free_sync_skcipher(ctx->sw_cipher);
1031 }
1032
1033 static int spacc_ablk_encrypt(struct skcipher_request *req)
1034 {
1035 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
1036 struct skcipher_alg *alg = crypto_skcipher_alg(cipher);
1037 struct spacc_alg *spacc_alg = to_spacc_skcipher(alg);
1038
1039 return spacc_ablk_setup(req, spacc_alg->type, 1);
1040 }
1041
1042 static int spacc_ablk_decrypt(struct skcipher_request *req)
1043 {
1044 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
1045 struct skcipher_alg *alg = crypto_skcipher_alg(cipher);
1046 struct spacc_alg *spacc_alg = to_spacc_skcipher(alg);
1047
1048 return spacc_ablk_setup(req, spacc_alg->type, 0);
1049 }
1050
1051 static inline int spacc_fifo_stat_empty(struct spacc_engine *engine)
1052 {
1053 return readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET) &
1054 SPA_FIFO_STAT_EMPTY;
1055 }
1056
1057 static void spacc_process_done(struct spacc_engine *engine)
1058 {
1059 struct spacc_req *req;
1060 unsigned long flags;
1061
1062 spin_lock_irqsave(&engine->hw_lock, flags);
1063
1064 while (!spacc_fifo_stat_empty(engine)) {
1065 req = list_first_entry(&engine->in_progress, struct spacc_req,
1066 list);
1067 list_move_tail(&req->list, &engine->completed);
1068 --engine->in_flight;
1069
1070 /* POP the status register. */
1071 writel(~0, engine->regs + SPA_STAT_POP_REG_OFFSET);
1072 req->result = (readl(engine->regs + SPA_STATUS_REG_OFFSET) &
1073 SPA_STATUS_RES_CODE_MASK) >> SPA_STATUS_RES_CODE_OFFSET;
1074
1075 /*
1076 * Convert the SPAcc error status into the standard POSIX error
1077 * codes.
1078 */
1079 if (unlikely(req->result)) {
1080 switch (req->result) {
1081 case SPA_STATUS_ICV_FAIL:
1082 req->result = -EBADMSG;
1083 break;
1084
1085 case SPA_STATUS_MEMORY_ERROR:
1086 dev_warn(engine->dev,
1087 "memory error triggered\n");
1088 req->result = -EFAULT;
1089 break;
1090
1091 case SPA_STATUS_BLOCK_ERROR:
1092 dev_warn(engine->dev,
1093 "block error triggered\n");
1094 req->result = -EIO;
1095 break;
1096 }
1097 }
1098 }
1099
1100 tasklet_schedule(&engine->complete);
1101
1102 spin_unlock_irqrestore(&engine->hw_lock, flags);
1103 }
1104
1105 static irqreturn_t spacc_spacc_irq(int irq, void *dev)
1106 {
1107 struct spacc_engine *engine = (struct spacc_engine *)dev;
1108 u32 spacc_irq_stat = readl(engine->regs + SPA_IRQ_STAT_REG_OFFSET);
1109
1110 writel(spacc_irq_stat, engine->regs + SPA_IRQ_STAT_REG_OFFSET);
1111 spacc_process_done(engine);
1112
1113 return IRQ_HANDLED;
1114 }
1115
1116 static void spacc_packet_timeout(struct timer_list *t)
1117 {
1118 struct spacc_engine *engine = from_timer(engine, t, packet_timeout);
1119
1120 spacc_process_done(engine);
1121 }
1122
1123 static int spacc_req_submit(struct spacc_req *req)
1124 {
1125 struct crypto_alg *alg = req->req->tfm->__crt_alg;
1126
1127 if (CRYPTO_ALG_TYPE_AEAD == (CRYPTO_ALG_TYPE_MASK & alg->cra_flags))
1128 return spacc_aead_submit(req);
1129 else
1130 return spacc_ablk_submit(req);
1131 }
1132
1133 static void spacc_spacc_complete(unsigned long data)
1134 {
1135 struct spacc_engine *engine = (struct spacc_engine *)data;
1136 struct spacc_req *req, *tmp;
1137 unsigned long flags;
1138 LIST_HEAD(completed);
1139
1140 spin_lock_irqsave(&engine->hw_lock, flags);
1141
1142 list_splice_init(&engine->completed, &completed);
1143 spacc_push(engine);
1144 if (engine->in_flight)
1145 mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT);
1146
1147 spin_unlock_irqrestore(&engine->hw_lock, flags);
1148
1149 list_for_each_entry_safe(req, tmp, &completed, list) {
1150 list_del(&req->list);
1151 req->complete(req);
1152 }
1153 }
1154
1155 #ifdef CONFIG_PM
1156 static int spacc_suspend(struct device *dev)
1157 {
1158 struct spacc_engine *engine = dev_get_drvdata(dev);
1159
1160 /*
1161 * We only support standby mode. All we have to do is gate the clock to
1162 * the spacc. The hardware will preserve state until we turn it back
1163 * on again.
1164 */
1165 clk_disable(engine->clk);
1166
1167 return 0;
1168 }
1169
1170 static int spacc_resume(struct device *dev)
1171 {
1172 struct spacc_engine *engine = dev_get_drvdata(dev);
1173
1174 return clk_enable(engine->clk);
1175 }
1176
1177 static const struct dev_pm_ops spacc_pm_ops = {
1178 .suspend = spacc_suspend,
1179 .resume = spacc_resume,
1180 };
1181 #endif /* CONFIG_PM */
1182
1183 static inline struct spacc_engine *spacc_dev_to_engine(struct device *dev)
1184 {
1185 return dev ? dev_get_drvdata(dev) : NULL;
1186 }
1187
1188 static ssize_t spacc_stat_irq_thresh_show(struct device *dev,
1189 struct device_attribute *attr,
1190 char *buf)
1191 {
1192 struct spacc_engine *engine = spacc_dev_to_engine(dev);
1193
1194 return snprintf(buf, PAGE_SIZE, "%u\n", engine->stat_irq_thresh);
1195 }
1196
1197 static ssize_t spacc_stat_irq_thresh_store(struct device *dev,
1198 struct device_attribute *attr,
1199 const char *buf, size_t len)
1200 {
1201 struct spacc_engine *engine = spacc_dev_to_engine(dev);
1202 unsigned long thresh;
1203
1204 if (kstrtoul(buf, 0, &thresh))
1205 return -EINVAL;
1206
1207 thresh = clamp(thresh, 1UL, engine->fifo_sz - 1);
1208
1209 engine->stat_irq_thresh = thresh;
1210 writel(engine->stat_irq_thresh << SPA_IRQ_CTRL_STAT_CNT_OFFSET,
1211 engine->regs + SPA_IRQ_CTRL_REG_OFFSET);
1212
1213 return len;
1214 }
1215 static DEVICE_ATTR(stat_irq_thresh, 0644, spacc_stat_irq_thresh_show,
1216 spacc_stat_irq_thresh_store);
1217
1218 static struct spacc_alg ipsec_engine_algs[] = {
1219 {
1220 .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_CBC,
1221 .key_offs = 0,
1222 .iv_offs = AES_MAX_KEY_SIZE,
1223 .alg = {
1224 .base.cra_name = "cbc(aes)",
1225 .base.cra_driver_name = "cbc-aes-picoxcell",
1226 .base.cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1227 .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
1228 CRYPTO_ALG_ASYNC |
1229 CRYPTO_ALG_NEED_FALLBACK,
1230 .base.cra_blocksize = AES_BLOCK_SIZE,
1231 .base.cra_ctxsize = sizeof(struct spacc_ablk_ctx),
1232 .base.cra_module = THIS_MODULE,
1233
1234 .setkey = spacc_aes_setkey,
1235 .encrypt = spacc_ablk_encrypt,
1236 .decrypt = spacc_ablk_decrypt,
1237 .min_keysize = AES_MIN_KEY_SIZE,
1238 .max_keysize = AES_MAX_KEY_SIZE,
1239 .ivsize = AES_BLOCK_SIZE,
1240 .init = spacc_ablk_init_tfm,
1241 .exit = spacc_ablk_exit_tfm,
1242 },
1243 },
1244 {
1245 .key_offs = 0,
1246 .iv_offs = AES_MAX_KEY_SIZE,
1247 .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_ECB,
1248 .alg = {
1249 .base.cra_name = "ecb(aes)",
1250 .base.cra_driver_name = "ecb-aes-picoxcell",
1251 .base.cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1252 .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
1253 CRYPTO_ALG_ASYNC |
1254 CRYPTO_ALG_NEED_FALLBACK,
1255 .base.cra_blocksize = AES_BLOCK_SIZE,
1256 .base.cra_ctxsize = sizeof(struct spacc_ablk_ctx),
1257 .base.cra_module = THIS_MODULE,
1258
1259 .setkey = spacc_aes_setkey,
1260 .encrypt = spacc_ablk_encrypt,
1261 .decrypt = spacc_ablk_decrypt,
1262 .min_keysize = AES_MIN_KEY_SIZE,
1263 .max_keysize = AES_MAX_KEY_SIZE,
1264 .init = spacc_ablk_init_tfm,
1265 .exit = spacc_ablk_exit_tfm,
1266 },
1267 },
1268 {
1269 .key_offs = DES_BLOCK_SIZE,
1270 .iv_offs = 0,
1271 .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_CBC,
1272 .alg = {
1273 .base.cra_name = "cbc(des)",
1274 .base.cra_driver_name = "cbc-des-picoxcell",
1275 .base.cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1276 .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
1277 CRYPTO_ALG_ASYNC,
1278 .base.cra_blocksize = DES_BLOCK_SIZE,
1279 .base.cra_ctxsize = sizeof(struct spacc_ablk_ctx),
1280 .base.cra_module = THIS_MODULE,
1281
1282 .setkey = spacc_des_setkey,
1283 .encrypt = spacc_ablk_encrypt,
1284 .decrypt = spacc_ablk_decrypt,
1285 .min_keysize = DES_KEY_SIZE,
1286 .max_keysize = DES_KEY_SIZE,
1287 .ivsize = DES_BLOCK_SIZE,
1288 .init = spacc_ablk_init_tfm,
1289 .exit = spacc_ablk_exit_tfm,
1290 },
1291 },
1292 {
1293 .key_offs = DES_BLOCK_SIZE,
1294 .iv_offs = 0,
1295 .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_ECB,
1296 .alg = {
1297 .base.cra_name = "ecb(des)",
1298 .base.cra_driver_name = "ecb-des-picoxcell",
1299 .base.cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1300 .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
1301 CRYPTO_ALG_ASYNC,
1302 .base.cra_blocksize = DES_BLOCK_SIZE,
1303 .base.cra_ctxsize = sizeof(struct spacc_ablk_ctx),
1304 .base.cra_module = THIS_MODULE,
1305
1306 .setkey = spacc_des_setkey,
1307 .encrypt = spacc_ablk_encrypt,
1308 .decrypt = spacc_ablk_decrypt,
1309 .min_keysize = DES_KEY_SIZE,
1310 .max_keysize = DES_KEY_SIZE,
1311 .init = spacc_ablk_init_tfm,
1312 .exit = spacc_ablk_exit_tfm,
1313 },
1314 },
1315 {
1316 .key_offs = DES_BLOCK_SIZE,
1317 .iv_offs = 0,
1318 .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_CBC,
1319 .alg = {
1320 .base.cra_name = "cbc(des3_ede)",
1321 .base.cra_driver_name = "cbc-des3-ede-picoxcell",
1322 .base.cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1323 .base.cra_flags = CRYPTO_ALG_ASYNC |
1324 CRYPTO_ALG_KERN_DRIVER_ONLY,
1325 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1326 .base.cra_ctxsize = sizeof(struct spacc_ablk_ctx),
1327 .base.cra_module = THIS_MODULE,
1328
1329 .setkey = spacc_des3_setkey,
1330 .encrypt = spacc_ablk_encrypt,
1331 .decrypt = spacc_ablk_decrypt,
1332 .min_keysize = DES3_EDE_KEY_SIZE,
1333 .max_keysize = DES3_EDE_KEY_SIZE,
1334 .ivsize = DES3_EDE_BLOCK_SIZE,
1335 .init = spacc_ablk_init_tfm,
1336 .exit = spacc_ablk_exit_tfm,
1337 },
1338 },
1339 {
1340 .key_offs = DES_BLOCK_SIZE,
1341 .iv_offs = 0,
1342 .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_ECB,
1343 .alg = {
1344 .base.cra_name = "ecb(des3_ede)",
1345 .base.cra_driver_name = "ecb-des3-ede-picoxcell",
1346 .base.cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1347 .base.cra_flags = CRYPTO_ALG_ASYNC |
1348 CRYPTO_ALG_KERN_DRIVER_ONLY,
1349 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1350 .base.cra_ctxsize = sizeof(struct spacc_ablk_ctx),
1351 .base.cra_module = THIS_MODULE,
1352
1353 .setkey = spacc_des3_setkey,
1354 .encrypt = spacc_ablk_encrypt,
1355 .decrypt = spacc_ablk_decrypt,
1356 .min_keysize = DES3_EDE_KEY_SIZE,
1357 .max_keysize = DES3_EDE_KEY_SIZE,
1358 .init = spacc_ablk_init_tfm,
1359 .exit = spacc_ablk_exit_tfm,
1360 },
1361 },
1362 };
1363
1364 static struct spacc_aead ipsec_engine_aeads[] = {
1365 {
1366 .ctrl_default = SPA_CTRL_CIPH_ALG_AES |
1367 SPA_CTRL_CIPH_MODE_CBC |
1368 SPA_CTRL_HASH_ALG_SHA |
1369 SPA_CTRL_HASH_MODE_HMAC,
1370 .key_offs = 0,
1371 .iv_offs = AES_MAX_KEY_SIZE,
1372 .alg = {
1373 .base = {
1374 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1375 .cra_driver_name = "authenc-hmac-sha1-"
1376 "cbc-aes-picoxcell",
1377 .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1378 .cra_flags = CRYPTO_ALG_ASYNC |
1379 CRYPTO_ALG_NEED_FALLBACK |
1380 CRYPTO_ALG_KERN_DRIVER_ONLY,
1381 .cra_blocksize = AES_BLOCK_SIZE,
1382 .cra_ctxsize = sizeof(struct spacc_aead_ctx),
1383 .cra_module = THIS_MODULE,
1384 },
1385 .setkey = spacc_aead_setkey,
1386 .setauthsize = spacc_aead_setauthsize,
1387 .encrypt = spacc_aead_encrypt,
1388 .decrypt = spacc_aead_decrypt,
1389 .ivsize = AES_BLOCK_SIZE,
1390 .maxauthsize = SHA1_DIGEST_SIZE,
1391 .init = spacc_aead_cra_init,
1392 .exit = spacc_aead_cra_exit,
1393 },
1394 },
1395 {
1396 .ctrl_default = SPA_CTRL_CIPH_ALG_AES |
1397 SPA_CTRL_CIPH_MODE_CBC |
1398 SPA_CTRL_HASH_ALG_SHA256 |
1399 SPA_CTRL_HASH_MODE_HMAC,
1400 .key_offs = 0,
1401 .iv_offs = AES_MAX_KEY_SIZE,
1402 .alg = {
1403 .base = {
1404 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1405 .cra_driver_name = "authenc-hmac-sha256-"
1406 "cbc-aes-picoxcell",
1407 .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1408 .cra_flags = CRYPTO_ALG_ASYNC |
1409 CRYPTO_ALG_NEED_FALLBACK |
1410 CRYPTO_ALG_KERN_DRIVER_ONLY,
1411 .cra_blocksize = AES_BLOCK_SIZE,
1412 .cra_ctxsize = sizeof(struct spacc_aead_ctx),
1413 .cra_module = THIS_MODULE,
1414 },
1415 .setkey = spacc_aead_setkey,
1416 .setauthsize = spacc_aead_setauthsize,
1417 .encrypt = spacc_aead_encrypt,
1418 .decrypt = spacc_aead_decrypt,
1419 .ivsize = AES_BLOCK_SIZE,
1420 .maxauthsize = SHA256_DIGEST_SIZE,
1421 .init = spacc_aead_cra_init,
1422 .exit = spacc_aead_cra_exit,
1423 },
1424 },
1425 {
1426 .key_offs = 0,
1427 .iv_offs = AES_MAX_KEY_SIZE,
1428 .ctrl_default = SPA_CTRL_CIPH_ALG_AES |
1429 SPA_CTRL_CIPH_MODE_CBC |
1430 SPA_CTRL_HASH_ALG_MD5 |
1431 SPA_CTRL_HASH_MODE_HMAC,
1432 .alg = {
1433 .base = {
1434 .cra_name = "authenc(hmac(md5),cbc(aes))",
1435 .cra_driver_name = "authenc-hmac-md5-"
1436 "cbc-aes-picoxcell",
1437 .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1438 .cra_flags = CRYPTO_ALG_ASYNC |
1439 CRYPTO_ALG_NEED_FALLBACK |
1440 CRYPTO_ALG_KERN_DRIVER_ONLY,
1441 .cra_blocksize = AES_BLOCK_SIZE,
1442 .cra_ctxsize = sizeof(struct spacc_aead_ctx),
1443 .cra_module = THIS_MODULE,
1444 },
1445 .setkey = spacc_aead_setkey,
1446 .setauthsize = spacc_aead_setauthsize,
1447 .encrypt = spacc_aead_encrypt,
1448 .decrypt = spacc_aead_decrypt,
1449 .ivsize = AES_BLOCK_SIZE,
1450 .maxauthsize = MD5_DIGEST_SIZE,
1451 .init = spacc_aead_cra_init,
1452 .exit = spacc_aead_cra_exit,
1453 },
1454 },
1455 {
1456 .key_offs = DES_BLOCK_SIZE,
1457 .iv_offs = 0,
1458 .ctrl_default = SPA_CTRL_CIPH_ALG_DES |
1459 SPA_CTRL_CIPH_MODE_CBC |
1460 SPA_CTRL_HASH_ALG_SHA |
1461 SPA_CTRL_HASH_MODE_HMAC,
1462 .alg = {
1463 .base = {
1464 .cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
1465 .cra_driver_name = "authenc-hmac-sha1-"
1466 "cbc-3des-picoxcell",
1467 .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1468 .cra_flags = CRYPTO_ALG_ASYNC |
1469 CRYPTO_ALG_NEED_FALLBACK |
1470 CRYPTO_ALG_KERN_DRIVER_ONLY,
1471 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1472 .cra_ctxsize = sizeof(struct spacc_aead_ctx),
1473 .cra_module = THIS_MODULE,
1474 },
1475 .setkey = spacc_aead_setkey,
1476 .setauthsize = spacc_aead_setauthsize,
1477 .encrypt = spacc_aead_encrypt,
1478 .decrypt = spacc_aead_decrypt,
1479 .ivsize = DES3_EDE_BLOCK_SIZE,
1480 .maxauthsize = SHA1_DIGEST_SIZE,
1481 .init = spacc_aead_cra_init,
1482 .exit = spacc_aead_cra_exit,
1483 },
1484 },
1485 {
1486 .key_offs = DES_BLOCK_SIZE,
1487 .iv_offs = 0,
1488 .ctrl_default = SPA_CTRL_CIPH_ALG_AES |
1489 SPA_CTRL_CIPH_MODE_CBC |
1490 SPA_CTRL_HASH_ALG_SHA256 |
1491 SPA_CTRL_HASH_MODE_HMAC,
1492 .alg = {
1493 .base = {
1494 .cra_name = "authenc(hmac(sha256),"
1495 "cbc(des3_ede))",
1496 .cra_driver_name = "authenc-hmac-sha256-"
1497 "cbc-3des-picoxcell",
1498 .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1499 .cra_flags = CRYPTO_ALG_ASYNC |
1500 CRYPTO_ALG_NEED_FALLBACK |
1501 CRYPTO_ALG_KERN_DRIVER_ONLY,
1502 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1503 .cra_ctxsize = sizeof(struct spacc_aead_ctx),
1504 .cra_module = THIS_MODULE,
1505 },
1506 .setkey = spacc_aead_setkey,
1507 .setauthsize = spacc_aead_setauthsize,
1508 .encrypt = spacc_aead_encrypt,
1509 .decrypt = spacc_aead_decrypt,
1510 .ivsize = DES3_EDE_BLOCK_SIZE,
1511 .maxauthsize = SHA256_DIGEST_SIZE,
1512 .init = spacc_aead_cra_init,
1513 .exit = spacc_aead_cra_exit,
1514 },
1515 },
1516 {
1517 .key_offs = DES_BLOCK_SIZE,
1518 .iv_offs = 0,
1519 .ctrl_default = SPA_CTRL_CIPH_ALG_DES |
1520 SPA_CTRL_CIPH_MODE_CBC |
1521 SPA_CTRL_HASH_ALG_MD5 |
1522 SPA_CTRL_HASH_MODE_HMAC,
1523 .alg = {
1524 .base = {
1525 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
1526 .cra_driver_name = "authenc-hmac-md5-"
1527 "cbc-3des-picoxcell",
1528 .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1529 .cra_flags = CRYPTO_ALG_ASYNC |
1530 CRYPTO_ALG_NEED_FALLBACK |
1531 CRYPTO_ALG_KERN_DRIVER_ONLY,
1532 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1533 .cra_ctxsize = sizeof(struct spacc_aead_ctx),
1534 .cra_module = THIS_MODULE,
1535 },
1536 .setkey = spacc_aead_setkey,
1537 .setauthsize = spacc_aead_setauthsize,
1538 .encrypt = spacc_aead_encrypt,
1539 .decrypt = spacc_aead_decrypt,
1540 .ivsize = DES3_EDE_BLOCK_SIZE,
1541 .maxauthsize = MD5_DIGEST_SIZE,
1542 .init = spacc_aead_cra_init,
1543 .exit = spacc_aead_cra_exit,
1544 },
1545 },
1546 };
1547
1548 static struct spacc_alg l2_engine_algs[] = {
1549 {
1550 .key_offs = 0,
1551 .iv_offs = SPACC_CRYPTO_KASUMI_F8_KEY_LEN,
1552 .ctrl_default = SPA_CTRL_CIPH_ALG_KASUMI |
1553 SPA_CTRL_CIPH_MODE_F8,
1554 .alg = {
1555 .base.cra_name = "f8(kasumi)",
1556 .base.cra_driver_name = "f8-kasumi-picoxcell",
1557 .base.cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
1558 .base.cra_flags = CRYPTO_ALG_ASYNC |
1559 CRYPTO_ALG_KERN_DRIVER_ONLY,
1560 .base.cra_blocksize = 8,
1561 .base.cra_ctxsize = sizeof(struct spacc_ablk_ctx),
1562 .base.cra_module = THIS_MODULE,
1563
1564 .setkey = spacc_kasumi_f8_setkey,
1565 .encrypt = spacc_ablk_encrypt,
1566 .decrypt = spacc_ablk_decrypt,
1567 .min_keysize = 16,
1568 .max_keysize = 16,
1569 .ivsize = 8,
1570 .init = spacc_ablk_init_tfm,
1571 .exit = spacc_ablk_exit_tfm,
1572 },
1573 },
1574 };
1575
1576 #ifdef CONFIG_OF
1577 static const struct of_device_id spacc_of_id_table[] = {
1578 { .compatible = "picochip,spacc-ipsec" },
1579 { .compatible = "picochip,spacc-l2" },
1580 {}
1581 };
1582 MODULE_DEVICE_TABLE(of, spacc_of_id_table);
1583 #endif /* CONFIG_OF */
1584
1585 static void spacc_tasklet_kill(void *data)
1586 {
1587 tasklet_kill(data);
1588 }
1589
1590 static int spacc_probe(struct platform_device *pdev)
1591 {
1592 int i, err, ret;
1593 struct resource *irq;
1594 struct device_node *np = pdev->dev.of_node;
1595 struct spacc_engine *engine = devm_kzalloc(&pdev->dev, sizeof(*engine),
1596 GFP_KERNEL);
1597 if (!engine)
1598 return -ENOMEM;
1599
1600 if (of_device_is_compatible(np, "picochip,spacc-ipsec")) {
1601 engine->max_ctxs = SPACC_CRYPTO_IPSEC_MAX_CTXS;
1602 engine->cipher_pg_sz = SPACC_CRYPTO_IPSEC_CIPHER_PG_SZ;
1603 engine->hash_pg_sz = SPACC_CRYPTO_IPSEC_HASH_PG_SZ;
1604 engine->fifo_sz = SPACC_CRYPTO_IPSEC_FIFO_SZ;
1605 engine->algs = ipsec_engine_algs;
1606 engine->num_algs = ARRAY_SIZE(ipsec_engine_algs);
1607 engine->aeads = ipsec_engine_aeads;
1608 engine->num_aeads = ARRAY_SIZE(ipsec_engine_aeads);
1609 } else if (of_device_is_compatible(np, "picochip,spacc-l2")) {
1610 engine->max_ctxs = SPACC_CRYPTO_L2_MAX_CTXS;
1611 engine->cipher_pg_sz = SPACC_CRYPTO_L2_CIPHER_PG_SZ;
1612 engine->hash_pg_sz = SPACC_CRYPTO_L2_HASH_PG_SZ;
1613 engine->fifo_sz = SPACC_CRYPTO_L2_FIFO_SZ;
1614 engine->algs = l2_engine_algs;
1615 engine->num_algs = ARRAY_SIZE(l2_engine_algs);
1616 } else {
1617 return -EINVAL;
1618 }
1619
1620 engine->name = dev_name(&pdev->dev);
1621
1622 engine->regs = devm_platform_ioremap_resource(pdev, 0);
1623 if (IS_ERR(engine->regs))
1624 return PTR_ERR(engine->regs);
1625
1626 irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
1627 if (!irq) {
1628 dev_err(&pdev->dev, "no memory/irq resource for engine\n");
1629 return -ENXIO;
1630 }
1631
1632 tasklet_init(&engine->complete, spacc_spacc_complete,
1633 (unsigned long)engine);
1634
1635 ret = devm_add_action(&pdev->dev, spacc_tasklet_kill,
1636 &engine->complete);
1637 if (ret)
1638 return ret;
1639
1640 if (devm_request_irq(&pdev->dev, irq->start, spacc_spacc_irq, 0,
1641 engine->name, engine)) {
1642 dev_err(engine->dev, "failed to request IRQ\n");
1643 return -EBUSY;
1644 }
1645
1646 engine->dev = &pdev->dev;
1647 engine->cipher_ctx_base = engine->regs + SPA_CIPH_KEY_BASE_REG_OFFSET;
1648 engine->hash_key_base = engine->regs + SPA_HASH_KEY_BASE_REG_OFFSET;
1649
1650 engine->req_pool = dmam_pool_create(engine->name, engine->dev,
1651 MAX_DDT_LEN * sizeof(struct spacc_ddt), 8, SZ_64K);
1652 if (!engine->req_pool)
1653 return -ENOMEM;
1654
1655 spin_lock_init(&engine->hw_lock);
1656
1657 engine->clk = clk_get(&pdev->dev, "ref");
1658 if (IS_ERR(engine->clk)) {
1659 dev_info(&pdev->dev, "clk unavailable\n");
1660 return PTR_ERR(engine->clk);
1661 }
1662
1663 if (clk_prepare_enable(engine->clk)) {
1664 dev_info(&pdev->dev, "unable to prepare/enable clk\n");
1665 ret = -EIO;
1666 goto err_clk_put;
1667 }
1668
1669 ret = device_create_file(&pdev->dev, &dev_attr_stat_irq_thresh);
1670 if (ret)
1671 goto err_clk_disable;
1672
1673
1674 /*
1675 * Use an IRQ threshold of 50% as a default. This seems to be a
1676 * reasonable trade off of latency against throughput but can be
1677 * changed at runtime.
1678 */
1679 engine->stat_irq_thresh = (engine->fifo_sz / 2);
1680
1681 /*
1682 * Configure the interrupts. We only use the STAT_CNT interrupt as we
1683 * only submit a new packet for processing when we complete another in
1684 * the queue. This minimizes time spent in the interrupt handler.
1685 */
1686 writel(engine->stat_irq_thresh << SPA_IRQ_CTRL_STAT_CNT_OFFSET,
1687 engine->regs + SPA_IRQ_CTRL_REG_OFFSET);
1688 writel(SPA_IRQ_EN_STAT_EN | SPA_IRQ_EN_GLBL_EN,
1689 engine->regs + SPA_IRQ_EN_REG_OFFSET);
1690
1691 timer_setup(&engine->packet_timeout, spacc_packet_timeout, 0);
1692
1693 INIT_LIST_HEAD(&engine->pending);
1694 INIT_LIST_HEAD(&engine->completed);
1695 INIT_LIST_HEAD(&engine->in_progress);
1696 engine->in_flight = 0;
1697
1698 platform_set_drvdata(pdev, engine);
1699
1700 ret = -EINVAL;
1701 INIT_LIST_HEAD(&engine->registered_algs);
1702 for (i = 0; i < engine->num_algs; ++i) {
1703 engine->algs[i].engine = engine;
1704 err = crypto_register_skcipher(&engine->algs[i].alg);
1705 if (!err) {
1706 list_add_tail(&engine->algs[i].entry,
1707 &engine->registered_algs);
1708 ret = 0;
1709 }
1710 if (err)
1711 dev_err(engine->dev, "failed to register alg \"%s\"\n",
1712 engine->algs[i].alg.base.cra_name);
1713 else
1714 dev_dbg(engine->dev, "registered alg \"%s\"\n",
1715 engine->algs[i].alg.base.cra_name);
1716 }
1717
1718 INIT_LIST_HEAD(&engine->registered_aeads);
1719 for (i = 0; i < engine->num_aeads; ++i) {
1720 engine->aeads[i].engine = engine;
1721 err = crypto_register_aead(&engine->aeads[i].alg);
1722 if (!err) {
1723 list_add_tail(&engine->aeads[i].entry,
1724 &engine->registered_aeads);
1725 ret = 0;
1726 }
1727 if (err)
1728 dev_err(engine->dev, "failed to register alg \"%s\"\n",
1729 engine->aeads[i].alg.base.cra_name);
1730 else
1731 dev_dbg(engine->dev, "registered alg \"%s\"\n",
1732 engine->aeads[i].alg.base.cra_name);
1733 }
1734
1735 if (!ret)
1736 return 0;
1737
1738 del_timer_sync(&engine->packet_timeout);
1739 device_remove_file(&pdev->dev, &dev_attr_stat_irq_thresh);
1740 err_clk_disable:
1741 clk_disable_unprepare(engine->clk);
1742 err_clk_put:
1743 clk_put(engine->clk);
1744
1745 return ret;
1746 }
1747
1748 static int spacc_remove(struct platform_device *pdev)
1749 {
1750 struct spacc_aead *aead, *an;
1751 struct spacc_alg *alg, *next;
1752 struct spacc_engine *engine = platform_get_drvdata(pdev);
1753
1754 del_timer_sync(&engine->packet_timeout);
1755 device_remove_file(&pdev->dev, &dev_attr_stat_irq_thresh);
1756
1757 list_for_each_entry_safe(aead, an, &engine->registered_aeads, entry) {
1758 list_del(&aead->entry);
1759 crypto_unregister_aead(&aead->alg);
1760 }
1761
1762 list_for_each_entry_safe(alg, next, &engine->registered_algs, entry) {
1763 list_del(&alg->entry);
1764 crypto_unregister_skcipher(&alg->alg);
1765 }
1766
1767 clk_disable_unprepare(engine->clk);
1768 clk_put(engine->clk);
1769
1770 return 0;
1771 }
1772
1773 static struct platform_driver spacc_driver = {
1774 .probe = spacc_probe,
1775 .remove = spacc_remove,
1776 .driver = {
1777 .name = "picochip,spacc",
1778 #ifdef CONFIG_PM
1779 .pm = &spacc_pm_ops,
1780 #endif /* CONFIG_PM */
1781 .of_match_table = of_match_ptr(spacc_of_id_table),
1782 },
1783 };
1784
1785 module_platform_driver(spacc_driver);
1786
1787 MODULE_LICENSE("GPL");
1788 MODULE_AUTHOR("Jamie Iles");
Linux with added FPC-III support