?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
of/platform: Initialise default DMA masks
[linux/fpc-iii.git] / drivers / crypto / inside-secure / safexcel.c
blob4e86f864a9529c44e555f08645148c0e833836f0
1 /*
2 * Copyright (C) 2017 Marvell
3 *
4 * Antoine Tenart <antoine.tenart@free-electrons.com>
5 *
6 * This file is licensed under the terms of the GNU General Public
7 * License version 2. This program is licensed "as is" without any
8 * warranty of any kind, whether express or implied.
9 */
10
11 #include <linux/clk.h>
12 #include <linux/device.h>
13 #include <linux/dma-mapping.h>
14 #include <linux/dmapool.h>
15 #include <linux/firmware.h>
16 #include <linux/interrupt.h>
17 #include <linux/module.h>
18 #include <linux/of_platform.h>
19 #include <linux/of_irq.h>
20 #include <linux/platform_device.h>
21 #include <linux/workqueue.h>
22
23 #include <crypto/internal/aead.h>
24 #include <crypto/internal/hash.h>
25 #include <crypto/internal/skcipher.h>
26
27 #include "safexcel.h"
28
29 static u32 max_rings = EIP197_MAX_RINGS;
30 module_param(max_rings, uint, 0644);
31 MODULE_PARM_DESC(max_rings, "Maximum number of rings to use.");
32
33 static void eip197_trc_cache_init(struct safexcel_crypto_priv *priv)
34 {
35 u32 val, htable_offset;
36 int i;
37
38 /* Enable the record cache memory access */
39 val = readl(priv->base + EIP197_CS_RAM_CTRL);
40 val &= ~EIP197_TRC_ENABLE_MASK;
41 val |= EIP197_TRC_ENABLE_0;
42 writel(val, priv->base + EIP197_CS_RAM_CTRL);
43
44 /* Clear all ECC errors */
45 writel(0, priv->base + EIP197_TRC_ECCCTRL);
46
47 /*
48 * Make sure the cache memory is accessible by taking record cache into
49 * reset.
50 */
51 val = readl(priv->base + EIP197_TRC_PARAMS);
52 val |= EIP197_TRC_PARAMS_SW_RESET;
53 val &= ~EIP197_TRC_PARAMS_DATA_ACCESS;
54 writel(val, priv->base + EIP197_TRC_PARAMS);
55
56 /* Clear all records */
57 for (i = 0; i < EIP197_CS_RC_MAX; i++) {
58 u32 val, offset = EIP197_CLASSIFICATION_RAMS + i * EIP197_CS_RC_SIZE;
59
60 writel(EIP197_CS_RC_NEXT(EIP197_RC_NULL) |
61 EIP197_CS_RC_PREV(EIP197_RC_NULL),
62 priv->base + offset);
63
64 val = EIP197_CS_RC_NEXT(i+1) | EIP197_CS_RC_PREV(i-1);
65 if (i == 0)
66 val |= EIP197_CS_RC_PREV(EIP197_RC_NULL);
67 else if (i == EIP197_CS_RC_MAX - 1)
68 val |= EIP197_CS_RC_NEXT(EIP197_RC_NULL);
69 writel(val, priv->base + offset + sizeof(u32));
70 }
71
72 /* Clear the hash table entries */
73 htable_offset = EIP197_CS_RC_MAX * EIP197_CS_RC_SIZE;
74 for (i = 0; i < 64; i++)
75 writel(GENMASK(29, 0),
76 priv->base + EIP197_CLASSIFICATION_RAMS + htable_offset + i * sizeof(u32));
77
78 /* Disable the record cache memory access */
79 val = readl(priv->base + EIP197_CS_RAM_CTRL);
80 val &= ~EIP197_TRC_ENABLE_MASK;
81 writel(val, priv->base + EIP197_CS_RAM_CTRL);
82
83 /* Write head and tail pointers of the record free chain */
84 val = EIP197_TRC_FREECHAIN_HEAD_PTR(0) |
85 EIP197_TRC_FREECHAIN_TAIL_PTR(EIP197_CS_RC_MAX - 1);
86 writel(val, priv->base + EIP197_TRC_FREECHAIN);
87
88 /* Configure the record cache #1 */
89 val = EIP197_TRC_PARAMS2_RC_SZ_SMALL(EIP197_CS_TRC_REC_WC) |
90 EIP197_TRC_PARAMS2_HTABLE_PTR(EIP197_CS_RC_MAX);
91 writel(val, priv->base + EIP197_TRC_PARAMS2);
92
93 /* Configure the record cache #2 */
94 val = EIP197_TRC_PARAMS_RC_SZ_LARGE(EIP197_CS_TRC_LG_REC_WC) |
95 EIP197_TRC_PARAMS_BLK_TIMER_SPEED(1) |
96 EIP197_TRC_PARAMS_HTABLE_SZ(2);
97 writel(val, priv->base + EIP197_TRC_PARAMS);
98 }
99
100 static void eip197_write_firmware(struct safexcel_crypto_priv *priv,
101 const struct firmware *fw, u32 ctrl,
102 u32 prog_en)
103 {
104 const u32 *data = (const u32 *)fw->data;
105 u32 val;
106 int i;
107
108 /* Reset the engine to make its program memory accessible */
109 writel(EIP197_PE_ICE_x_CTRL_SW_RESET |
110 EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR |
111 EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR,
112 EIP197_PE(priv) + ctrl);
113
114 /* Enable access to the program memory */
115 writel(prog_en, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL);
116
117 /* Write the firmware */
118 for (i = 0; i < fw->size / sizeof(u32); i++)
119 writel(be32_to_cpu(data[i]),
120 priv->base + EIP197_CLASSIFICATION_RAMS + i * sizeof(u32));
121
122 /* Disable access to the program memory */
123 writel(0, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL);
124
125 /* Release engine from reset */
126 val = readl(EIP197_PE(priv) + ctrl);
127 val &= ~EIP197_PE_ICE_x_CTRL_SW_RESET;
128 writel(val, EIP197_PE(priv) + ctrl);
129 }
130
131 static int eip197_load_firmwares(struct safexcel_crypto_priv *priv)
132 {
133 const char *fw_name[] = {"ifpp.bin", "ipue.bin"};
134 const struct firmware *fw[FW_NB];
135 int i, j, ret = 0;
136 u32 val;
137
138 for (i = 0; i < FW_NB; i++) {
139 ret = request_firmware(&fw[i], fw_name[i], priv->dev);
140 if (ret) {
141 dev_err(priv->dev,
142 "Failed to request firmware %s (%d)\n",
143 fw_name[i], ret);
144 goto release_fw;
145 }
146 }
147
148 /* Clear the scratchpad memory */
149 val = readl(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL);
150 val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER |
151 EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN |
152 EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS |
153 EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS;
154 writel(val, EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL);
155
156 memset_io(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_RAM, 0,
157 EIP197_NUM_OF_SCRATCH_BLOCKS * sizeof(u32));
158
159 eip197_write_firmware(priv, fw[FW_IFPP], EIP197_PE_ICE_FPP_CTRL,
160 EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN);
161
162 eip197_write_firmware(priv, fw[FW_IPUE], EIP197_PE_ICE_PUE_CTRL,
163 EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN);
164
165 release_fw:
166 for (j = 0; j < i; j++)
167 release_firmware(fw[j]);
168
169 return ret;
170 }
171
172 static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv)
173 {
174 u32 hdw, cd_size_rnd, val;
175 int i;
176
177 hdw = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS);
178 hdw &= GENMASK(27, 25);
179 hdw >>= 25;
180
181 cd_size_rnd = (priv->config.cd_size + (BIT(hdw) - 1)) >> hdw;
182
183 for (i = 0; i < priv->config.rings; i++) {
184 /* ring base address */
185 writel(lower_32_bits(priv->ring[i].cdr.base_dma),
186 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
187 writel(upper_32_bits(priv->ring[i].cdr.base_dma),
188 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
189
190 writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.cd_offset << 16) |
191 priv->config.cd_size,
192 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE);
193 writel(((EIP197_FETCH_COUNT * (cd_size_rnd << hdw)) << 16) |
194 (EIP197_FETCH_COUNT * priv->config.cd_offset),
195 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG);
196
197 /* Configure DMA tx control */
198 val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
199 val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
200 writel(val, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DMA_CFG);
201
202 /* clear any pending interrupt */
203 writel(GENMASK(5, 0),
204 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT);
205 }
206
207 return 0;
208 }
209
210 static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv)
211 {
212 u32 hdw, rd_size_rnd, val;
213 int i;
214
215 hdw = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS);
216 hdw &= GENMASK(27, 25);
217 hdw >>= 25;
218
219 rd_size_rnd = (priv->config.rd_size + (BIT(hdw) - 1)) >> hdw;
220
221 for (i = 0; i < priv->config.rings; i++) {
222 /* ring base address */
223 writel(lower_32_bits(priv->ring[i].rdr.base_dma),
224 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
225 writel(upper_32_bits(priv->ring[i].rdr.base_dma),
226 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
227
228 writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.rd_offset << 16) |
229 priv->config.rd_size,
230 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE);
231
232 writel(((EIP197_FETCH_COUNT * (rd_size_rnd << hdw)) << 16) |
233 (EIP197_FETCH_COUNT * priv->config.rd_offset),
234 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG);
235
236 /* Configure DMA tx control */
237 val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
238 val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
239 val |= EIP197_HIA_xDR_WR_RES_BUF | EIP197_HIA_xDR_WR_CTRL_BUF;
240 writel(val,
241 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DMA_CFG);
242
243 /* clear any pending interrupt */
244 writel(GENMASK(7, 0),
245 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT);
246
247 /* enable ring interrupt */
248 val = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
249 val |= EIP197_RDR_IRQ(i);
250 writel(val, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
251 }
252
253 return 0;
254 }
255
256 static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
257 {
258 u32 version, val;
259 int i, ret;
260
261 /* Determine endianess and configure byte swap */
262 version = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_VERSION);
263 val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
264
265 if ((version & 0xffff) == EIP197_HIA_VERSION_BE)
266 val |= EIP197_MST_CTRL_BYTE_SWAP;
267 else if (((version >> 16) & 0xffff) == EIP197_HIA_VERSION_LE)
268 val |= (EIP197_MST_CTRL_NO_BYTE_SWAP >> 24);
269
270 writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
271
272 /* Configure wr/rd cache values */
273 writel(EIP197_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) |
274 EIP197_MST_CTRL_WD_CACHE(WR_CACHE_4BITS),
275 EIP197_HIA_GEN_CFG(priv) + EIP197_MST_CTRL);
276
277 /* Interrupts reset */
278
279 /* Disable all global interrupts */
280 writel(0, EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ENABLE_CTRL);
281
282 /* Clear any pending interrupt */
283 writel(GENMASK(31, 0), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK);
284
285 /* Data Fetch Engine configuration */
286
287 /* Reset all DFE threads */
288 writel(EIP197_DxE_THR_CTRL_RESET_PE,
289 EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL);
290
291 if (priv->version == EIP197) {
292 /* Reset HIA input interface arbiter */
293 writel(EIP197_HIA_RA_PE_CTRL_RESET,
294 EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL);
295 }
296
297 /* DMA transfer size to use */
298 val = EIP197_HIA_DFE_CFG_DIS_DEBUG;
299 val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(5) | EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(9);
300 val |= EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(5) | EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(7);
301 val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS);
302 val |= EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS);
303 writel(val, EIP197_HIA_DFE(priv) + EIP197_HIA_DFE_CFG);
304
305 /* Leave the DFE threads reset state */
306 writel(0, EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL);
307
308 /* Configure the procesing engine thresholds */
309 writel(EIP197_PE_IN_xBUF_THRES_MIN(5) | EIP197_PE_IN_xBUF_THRES_MAX(9),
310 EIP197_PE(priv) + EIP197_PE_IN_DBUF_THRES);
311 writel(EIP197_PE_IN_xBUF_THRES_MIN(5) | EIP197_PE_IN_xBUF_THRES_MAX(7),
312 EIP197_PE(priv) + EIP197_PE_IN_TBUF_THRES);
313
314 if (priv->version == EIP197) {
315 /* enable HIA input interface arbiter and rings */
316 writel(EIP197_HIA_RA_PE_CTRL_EN |
317 GENMASK(priv->config.rings - 1, 0),
318 EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL);
319 }
320
321 /* Data Store Engine configuration */
322
323 /* Reset all DSE threads */
324 writel(EIP197_DxE_THR_CTRL_RESET_PE,
325 EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL);
326
327 /* Wait for all DSE threads to complete */
328 while ((readl(EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_STAT) &
329 GENMASK(15, 12)) != GENMASK(15, 12))
330 ;
331
332 /* DMA transfer size to use */
333 val = EIP197_HIA_DSE_CFG_DIS_DEBUG;
334 val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(7) | EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(8);
335 val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS);
336 val |= EIP197_HIA_DSE_CFG_ALWAYS_BUFFERABLE;
337 /* FIXME: instability issues can occur for EIP97 but disabling it impact
338 * performances.
339 */
340 if (priv->version == EIP197)
341 val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR;
342 writel(val, EIP197_HIA_DSE(priv) + EIP197_HIA_DSE_CFG);
343
344 /* Leave the DSE threads reset state */
345 writel(0, EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL);
346
347 /* Configure the procesing engine thresholds */
348 writel(EIP197_PE_OUT_DBUF_THRES_MIN(7) | EIP197_PE_OUT_DBUF_THRES_MAX(8),
349 EIP197_PE(priv) + EIP197_PE_OUT_DBUF_THRES);
350
351 /* Processing Engine configuration */
352
353 /* H/W capabilities selection */
354 val = EIP197_FUNCTION_RSVD;
355 val |= EIP197_PROTOCOL_ENCRYPT_ONLY | EIP197_PROTOCOL_HASH_ONLY;
356 val |= EIP197_PROTOCOL_ENCRYPT_HASH | EIP197_PROTOCOL_HASH_DECRYPT;
357 val |= EIP197_ALG_AES_ECB | EIP197_ALG_AES_CBC;
358 val |= EIP197_ALG_SHA1 | EIP197_ALG_HMAC_SHA1;
359 val |= EIP197_ALG_SHA2 | EIP197_ALG_HMAC_SHA2;
360 writel(val, EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION_EN);
361
362 /* Command Descriptor Rings prepare */
363 for (i = 0; i < priv->config.rings; i++) {
364 /* Clear interrupts for this ring */
365 writel(GENMASK(31, 0),
366 EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CLR(i));
367
368 /* Disable external triggering */
369 writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG);
370
371 /* Clear the pending prepared counter */
372 writel(EIP197_xDR_PREP_CLR_COUNT,
373 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT);
374
375 /* Clear the pending processed counter */
376 writel(EIP197_xDR_PROC_CLR_COUNT,
377 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT);
378
379 writel(0,
380 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR);
381 writel(0,
382 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR);
383
384 writel((EIP197_DEFAULT_RING_SIZE * priv->config.cd_offset) << 2,
385 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_SIZE);
386 }
387
388 /* Result Descriptor Ring prepare */
389 for (i = 0; i < priv->config.rings; i++) {
390 /* Disable external triggering*/
391 writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG);
392
393 /* Clear the pending prepared counter */
394 writel(EIP197_xDR_PREP_CLR_COUNT,
395 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT);
396
397 /* Clear the pending processed counter */
398 writel(EIP197_xDR_PROC_CLR_COUNT,
399 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT);
400
401 writel(0,
402 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR);
403 writel(0,
404 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR);
405
406 /* Ring size */
407 writel((EIP197_DEFAULT_RING_SIZE * priv->config.rd_offset) << 2,
408 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_SIZE);
409 }
410
411 /* Enable command descriptor rings */
412 writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
413 EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL);
414
415 /* Enable result descriptor rings */
416 writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
417 EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL);
418
419 /* Clear any HIA interrupt */
420 writel(GENMASK(30, 20), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK);
421
422 if (priv->version == EIP197) {
423 eip197_trc_cache_init(priv);
424
425 ret = eip197_load_firmwares(priv);
426 if (ret)
427 return ret;
428 }
429
430 safexcel_hw_setup_cdesc_rings(priv);
431 safexcel_hw_setup_rdesc_rings(priv);
432
433 return 0;
434 }
435
436 /* Called with ring's lock taken */
437 static void safexcel_try_push_requests(struct safexcel_crypto_priv *priv,
438 int ring)
439 {
440 int coal = min_t(int, priv->ring[ring].requests, EIP197_MAX_BATCH_SZ);
441
442 if (!coal)
443 return;
444
445 /* Configure when we want an interrupt */
446 writel(EIP197_HIA_RDR_THRESH_PKT_MODE |
447 EIP197_HIA_RDR_THRESH_PROC_PKT(coal),
448 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_THRESH);
449 }
450
451 void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring)
452 {
453 struct crypto_async_request *req, *backlog;
454 struct safexcel_context *ctx;
455 struct safexcel_request *request;
456 int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results;
457
458 /* If a request wasn't properly dequeued because of a lack of resources,
459 * proceeded it first,
460 */
461 req = priv->ring[ring].req;
462 backlog = priv->ring[ring].backlog;
463 if (req)
464 goto handle_req;
465
466 while (true) {
467 spin_lock_bh(&priv->ring[ring].queue_lock);
468 backlog = crypto_get_backlog(&priv->ring[ring].queue);
469 req = crypto_dequeue_request(&priv->ring[ring].queue);
470 spin_unlock_bh(&priv->ring[ring].queue_lock);
471
472 if (!req) {
473 priv->ring[ring].req = NULL;
474 priv->ring[ring].backlog = NULL;
475 goto finalize;
476 }
477
478 handle_req:
479 request = kzalloc(sizeof(*request), EIP197_GFP_FLAGS(*req));
480 if (!request)
481 goto request_failed;
482
483 ctx = crypto_tfm_ctx(req->tfm);
484 ret = ctx->send(req, ring, request, &commands, &results);
485 if (ret) {
486 kfree(request);
487 goto request_failed;
488 }
489
490 if (backlog)
491 backlog->complete(backlog, -EINPROGRESS);
492
493 /* In case the send() helper did not issue any command to push
494 * to the engine because the input data was cached, continue to
495 * dequeue other requests as this is valid and not an error.
496 */
497 if (!commands && !results) {
498 kfree(request);
499 continue;
500 }
501
502 spin_lock_bh(&priv->ring[ring].egress_lock);
503 list_add_tail(&request->list, &priv->ring[ring].list);
504 spin_unlock_bh(&priv->ring[ring].egress_lock);
505
506 cdesc += commands;
507 rdesc += results;
508 nreq++;
509 }
510
511 request_failed:
512 /* Not enough resources to handle all the requests. Bail out and save
513 * the request and the backlog for the next dequeue call (per-ring).
514 */
515 priv->ring[ring].req = req;
516 priv->ring[ring].backlog = backlog;
517
518 finalize:
519 if (!nreq)
520 return;
521
522 spin_lock_bh(&priv->ring[ring].egress_lock);
523
524 priv->ring[ring].requests += nreq;
525
526 if (!priv->ring[ring].busy) {
527 safexcel_try_push_requests(priv, ring);
528 priv->ring[ring].busy = true;
529 }
530
531 spin_unlock_bh(&priv->ring[ring].egress_lock);
532
533 /* let the RDR know we have pending descriptors */
534 writel((rdesc * priv->config.rd_offset) << 2,
535 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT);
536
537 /* let the CDR know we have pending descriptors */
538 writel((cdesc * priv->config.cd_offset) << 2,
539 EIP197_HIA_CDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT);
540 }
541
542 inline int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv,
543 struct safexcel_result_desc *rdesc)
544 {
545 if (likely(!rdesc->result_data.error_code))
546 return 0;
547
548 if (rdesc->result_data.error_code & 0x407f) {
549 /* Fatal error (bits 0-7, 14) */
550 dev_err(priv->dev,
551 "cipher: result: result descriptor error (%d)\n",
552 rdesc->result_data.error_code);
553 return -EIO;
554 } else if (rdesc->result_data.error_code == BIT(9)) {
555 /* Authentication failed */
556 return -EBADMSG;
557 }
558
559 /* All other non-fatal errors */
560 return -EINVAL;
561 }
562
563 void safexcel_complete(struct safexcel_crypto_priv *priv, int ring)
564 {
565 struct safexcel_command_desc *cdesc;
566
567 /* Acknowledge the command descriptors */
568 do {
569 cdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].cdr);
570 if (IS_ERR(cdesc)) {
571 dev_err(priv->dev,
572 "Could not retrieve the command descriptor\n");
573 return;
574 }
575 } while (!cdesc->last_seg);
576 }
577
578 void safexcel_inv_complete(struct crypto_async_request *req, int error)
579 {
580 struct safexcel_inv_result *result = req->data;
581
582 if (error == -EINPROGRESS)
583 return;
584
585 result->error = error;
586 complete(&result->completion);
587 }
588
589 int safexcel_invalidate_cache(struct crypto_async_request *async,
590 struct safexcel_crypto_priv *priv,
591 dma_addr_t ctxr_dma, int ring,
592 struct safexcel_request *request)
593 {
594 struct safexcel_command_desc *cdesc;
595 struct safexcel_result_desc *rdesc;
596 int ret = 0;
597
598 spin_lock_bh(&priv->ring[ring].egress_lock);
599
600 /* Prepare command descriptor */
601 cdesc = safexcel_add_cdesc(priv, ring, true, true, 0, 0, 0, ctxr_dma);
602 if (IS_ERR(cdesc)) {
603 ret = PTR_ERR(cdesc);
604 goto unlock;
605 }
606
607 cdesc->control_data.type = EIP197_TYPE_EXTENDED;
608 cdesc->control_data.options = 0;
609 cdesc->control_data.refresh = 0;
610 cdesc->control_data.control0 = CONTEXT_CONTROL_INV_TR;
611
612 /* Prepare result descriptor */
613 rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0);
614
615 if (IS_ERR(rdesc)) {
616 ret = PTR_ERR(rdesc);
617 goto cdesc_rollback;
618 }
619
620 request->req = async;
621 goto unlock;
622
623 cdesc_rollback:
624 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
625
626 unlock:
627 spin_unlock_bh(&priv->ring[ring].egress_lock);
628 return ret;
629 }
630
631 static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv *priv,
632 int ring)
633 {
634 struct safexcel_request *sreq;
635 struct safexcel_context *ctx;
636 int ret, i, nreq, ndesc, tot_descs, handled = 0;
637 bool should_complete;
638
639 handle_results:
640 tot_descs = 0;
641
642 nreq = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT);
643 nreq >>= EIP197_xDR_PROC_xD_PKT_OFFSET;
644 nreq &= EIP197_xDR_PROC_xD_PKT_MASK;
645 if (!nreq)
646 goto requests_left;
647
648 for (i = 0; i < nreq; i++) {
649 spin_lock_bh(&priv->ring[ring].egress_lock);
650 sreq = list_first_entry(&priv->ring[ring].list,
651 struct safexcel_request, list);
652 list_del(&sreq->list);
653 spin_unlock_bh(&priv->ring[ring].egress_lock);
654
655 ctx = crypto_tfm_ctx(sreq->req->tfm);
656 ndesc = ctx->handle_result(priv, ring, sreq->req,
657 &should_complete, &ret);
658 if (ndesc < 0) {
659 kfree(sreq);
660 dev_err(priv->dev, "failed to handle result (%d)", ndesc);
661 goto acknowledge;
662 }
663
664 if (should_complete) {
665 local_bh_disable();
666 sreq->req->complete(sreq->req, ret);
667 local_bh_enable();
668 }
669
670 kfree(sreq);
671 tot_descs += ndesc;
672 handled++;
673 }
674
675 acknowledge:
676 if (i) {
677 writel(EIP197_xDR_PROC_xD_PKT(i) |
678 EIP197_xDR_PROC_xD_COUNT(tot_descs * priv->config.rd_offset),
679 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT);
680 }
681
682 /* If the number of requests overflowed the counter, try to proceed more
683 * requests.
684 */
685 if (nreq == EIP197_xDR_PROC_xD_PKT_MASK)
686 goto handle_results;
687
688 requests_left:
689 spin_lock_bh(&priv->ring[ring].egress_lock);
690
691 priv->ring[ring].requests -= handled;
692 safexcel_try_push_requests(priv, ring);
693
694 if (!priv->ring[ring].requests)
695 priv->ring[ring].busy = false;
696
697 spin_unlock_bh(&priv->ring[ring].egress_lock);
698 }
699
700 static void safexcel_dequeue_work(struct work_struct *work)
701 {
702 struct safexcel_work_data *data =
703 container_of(work, struct safexcel_work_data, work);
704
705 safexcel_dequeue(data->priv, data->ring);
706 }
707
708 struct safexcel_ring_irq_data {
709 struct safexcel_crypto_priv *priv;
710 int ring;
711 };
712
713 static irqreturn_t safexcel_irq_ring(int irq, void *data)
714 {
715 struct safexcel_ring_irq_data *irq_data = data;
716 struct safexcel_crypto_priv *priv = irq_data->priv;
717 int ring = irq_data->ring, rc = IRQ_NONE;
718 u32 status, stat;
719
720 status = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLED_STAT(ring));
721 if (!status)
722 return rc;
723
724 /* RDR interrupts */
725 if (status & EIP197_RDR_IRQ(ring)) {
726 stat = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT);
727
728 if (unlikely(stat & EIP197_xDR_ERR)) {
729 /*
730 * Fatal error, the RDR is unusable and must be
731 * reinitialized. This should not happen under
732 * normal circumstances.
733 */
734 dev_err(priv->dev, "RDR: fatal error.");
735 } else if (likely(stat & EIP197_xDR_THRESH)) {
736 rc = IRQ_WAKE_THREAD;
737 }
738
739 /* ACK the interrupts */
740 writel(stat & 0xff,
741 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT);
742 }
743
744 /* ACK the interrupts */
745 writel(status, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ACK(ring));
746
747 return rc;
748 }
749
750 static irqreturn_t safexcel_irq_ring_thread(int irq, void *data)
751 {
752 struct safexcel_ring_irq_data *irq_data = data;
753 struct safexcel_crypto_priv *priv = irq_data->priv;
754 int ring = irq_data->ring;
755
756 safexcel_handle_result_descriptor(priv, ring);
757
758 queue_work(priv->ring[ring].workqueue,
759 &priv->ring[ring].work_data.work);
760
761 return IRQ_HANDLED;
762 }
763
764 static int safexcel_request_ring_irq(struct platform_device *pdev, const char *name,
765 irq_handler_t handler,
766 irq_handler_t threaded_handler,
767 struct safexcel_ring_irq_data *ring_irq_priv)
768 {
769 int ret, irq = platform_get_irq_byname(pdev, name);
770
771 if (irq < 0) {
772 dev_err(&pdev->dev, "unable to get IRQ '%s'\n", name);
773 return irq;
774 }
775
776 ret = devm_request_threaded_irq(&pdev->dev, irq, handler,
777 threaded_handler, IRQF_ONESHOT,
778 dev_name(&pdev->dev), ring_irq_priv);
779 if (ret) {
780 dev_err(&pdev->dev, "unable to request IRQ %d\n", irq);
781 return ret;
782 }
783
784 return irq;
785 }
786
787 static struct safexcel_alg_template *safexcel_algs[] = {
788 &safexcel_alg_ecb_aes,
789 &safexcel_alg_cbc_aes,
790 &safexcel_alg_sha1,
791 &safexcel_alg_sha224,
792 &safexcel_alg_sha256,
793 &safexcel_alg_hmac_sha1,
794 &safexcel_alg_hmac_sha224,
795 &safexcel_alg_hmac_sha256,
796 &safexcel_alg_authenc_hmac_sha1_cbc_aes,
797 &safexcel_alg_authenc_hmac_sha224_cbc_aes,
798 &safexcel_alg_authenc_hmac_sha256_cbc_aes,
799 };
800
801 static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv)
802 {
803 int i, j, ret = 0;
804
805 for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
806 safexcel_algs[i]->priv = priv;
807
808 if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
809 ret = crypto_register_skcipher(&safexcel_algs[i]->alg.skcipher);
810 else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD)
811 ret = crypto_register_aead(&safexcel_algs[i]->alg.aead);
812 else
813 ret = crypto_register_ahash(&safexcel_algs[i]->alg.ahash);
814
815 if (ret)
816 goto fail;
817 }
818
819 return 0;
820
821 fail:
822 for (j = 0; j < i; j++) {
823 if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
824 crypto_unregister_skcipher(&safexcel_algs[j]->alg.skcipher);
825 else if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_AEAD)
826 crypto_unregister_aead(&safexcel_algs[j]->alg.aead);
827 else
828 crypto_unregister_ahash(&safexcel_algs[j]->alg.ahash);
829 }
830
831 return ret;
832 }
833
834 static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv)
835 {
836 int i;
837
838 for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
839 if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
840 crypto_unregister_skcipher(&safexcel_algs[i]->alg.skcipher);
841 else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD)
842 crypto_unregister_aead(&safexcel_algs[i]->alg.aead);
843 else
844 crypto_unregister_ahash(&safexcel_algs[i]->alg.ahash);
845 }
846 }
847
848 static void safexcel_configure(struct safexcel_crypto_priv *priv)
849 {
850 u32 val, mask;
851
852 val = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS);
853 val = (val & GENMASK(27, 25)) >> 25;
854 mask = BIT(val) - 1;
855
856 val = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS);
857 priv->config.rings = min_t(u32, val & GENMASK(3, 0), max_rings);
858
859 priv->config.cd_size = (sizeof(struct safexcel_command_desc) / sizeof(u32));
860 priv->config.cd_offset = (priv->config.cd_size + mask) & ~mask;
861
862 priv->config.rd_size = (sizeof(struct safexcel_result_desc) / sizeof(u32));
863 priv->config.rd_offset = (priv->config.rd_size + mask) & ~mask;
864 }
865
866 static void safexcel_init_register_offsets(struct safexcel_crypto_priv *priv)
867 {
868 struct safexcel_register_offsets *offsets = &priv->offsets;
869
870 if (priv->version == EIP197) {
871 offsets->hia_aic = EIP197_HIA_AIC_BASE;
872 offsets->hia_aic_g = EIP197_HIA_AIC_G_BASE;
873 offsets->hia_aic_r = EIP197_HIA_AIC_R_BASE;
874 offsets->hia_aic_xdr = EIP197_HIA_AIC_xDR_BASE;
875 offsets->hia_dfe = EIP197_HIA_DFE_BASE;
876 offsets->hia_dfe_thr = EIP197_HIA_DFE_THR_BASE;
877 offsets->hia_dse = EIP197_HIA_DSE_BASE;
878 offsets->hia_dse_thr = EIP197_HIA_DSE_THR_BASE;
879 offsets->hia_gen_cfg = EIP197_HIA_GEN_CFG_BASE;
880 offsets->pe = EIP197_PE_BASE;
881 } else {
882 offsets->hia_aic = EIP97_HIA_AIC_BASE;
883 offsets->hia_aic_g = EIP97_HIA_AIC_G_BASE;
884 offsets->hia_aic_r = EIP97_HIA_AIC_R_BASE;
885 offsets->hia_aic_xdr = EIP97_HIA_AIC_xDR_BASE;
886 offsets->hia_dfe = EIP97_HIA_DFE_BASE;
887 offsets->hia_dfe_thr = EIP97_HIA_DFE_THR_BASE;
888 offsets->hia_dse = EIP97_HIA_DSE_BASE;
889 offsets->hia_dse_thr = EIP97_HIA_DSE_THR_BASE;
890 offsets->hia_gen_cfg = EIP97_HIA_GEN_CFG_BASE;
891 offsets->pe = EIP97_PE_BASE;
892 }
893 }
894
895 static int safexcel_probe(struct platform_device *pdev)
896 {
897 struct device *dev = &pdev->dev;
898 struct resource *res;
899 struct safexcel_crypto_priv *priv;
900 int i, ret;
901
902 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
903 if (!priv)
904 return -ENOMEM;
905
906 priv->dev = dev;
907 priv->version = (enum safexcel_eip_version)of_device_get_match_data(dev);
908
909 safexcel_init_register_offsets(priv);
910
911 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
912 priv->base = devm_ioremap_resource(dev, res);
913 if (IS_ERR(priv->base)) {
914 dev_err(dev, "failed to get resource\n");
915 return PTR_ERR(priv->base);
916 }
917
918 priv->clk = devm_clk_get(&pdev->dev, NULL);
919 ret = PTR_ERR_OR_ZERO(priv->clk);
920 /* The clock isn't mandatory */
921 if (ret != -ENOENT) {
922 if (ret)
923 return ret;
924
925 ret = clk_prepare_enable(priv->clk);
926 if (ret) {
927 dev_err(dev, "unable to enable clk (%d)\n", ret);
928 return ret;
929 }
930 }
931
932 priv->reg_clk = devm_clk_get(&pdev->dev, "reg");
933 ret = PTR_ERR_OR_ZERO(priv->reg_clk);
934 /* The clock isn't mandatory */
935 if (ret != -ENOENT) {
936 if (ret)
937 goto err_core_clk;
938
939 ret = clk_prepare_enable(priv->reg_clk);
940 if (ret) {
941 dev_err(dev, "unable to enable reg clk (%d)\n", ret);
942 goto err_core_clk;
943 }
944 }
945
946 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
947 if (ret)
948 goto err_reg_clk;
949
950 priv->context_pool = dmam_pool_create("safexcel-context", dev,
951 sizeof(struct safexcel_context_record),
952 1, 0);
953 if (!priv->context_pool) {
954 ret = -ENOMEM;
955 goto err_reg_clk;
956 }
957
958 safexcel_configure(priv);
959
960 for (i = 0; i < priv->config.rings; i++) {
961 char irq_name[6] = {0}; /* "ringX\0" */
962 char wq_name[9] = {0}; /* "wq_ringX\0" */
963 int irq;
964 struct safexcel_ring_irq_data *ring_irq;
965
966 ret = safexcel_init_ring_descriptors(priv,
967 &priv->ring[i].cdr,
968 &priv->ring[i].rdr);
969 if (ret)
970 goto err_reg_clk;
971
972 ring_irq = devm_kzalloc(dev, sizeof(*ring_irq), GFP_KERNEL);
973 if (!ring_irq) {
974 ret = -ENOMEM;
975 goto err_reg_clk;
976 }
977
978 ring_irq->priv = priv;
979 ring_irq->ring = i;
980
981 snprintf(irq_name, 6, "ring%d", i);
982 irq = safexcel_request_ring_irq(pdev, irq_name, safexcel_irq_ring,
983 safexcel_irq_ring_thread,
984 ring_irq);
985 if (irq < 0) {
986 ret = irq;
987 goto err_reg_clk;
988 }
989
990 priv->ring[i].work_data.priv = priv;
991 priv->ring[i].work_data.ring = i;
992 INIT_WORK(&priv->ring[i].work_data.work, safexcel_dequeue_work);
993
994 snprintf(wq_name, 9, "wq_ring%d", i);
995 priv->ring[i].workqueue = create_singlethread_workqueue(wq_name);
996 if (!priv->ring[i].workqueue) {
997 ret = -ENOMEM;
998 goto err_reg_clk;
999 }
1000
1001 priv->ring[i].requests = 0;
1002 priv->ring[i].busy = false;
1003
1004 crypto_init_queue(&priv->ring[i].queue,
1005 EIP197_DEFAULT_RING_SIZE);
1006
1007 INIT_LIST_HEAD(&priv->ring[i].list);
1008 spin_lock_init(&priv->ring[i].lock);
1009 spin_lock_init(&priv->ring[i].egress_lock);
1010 spin_lock_init(&priv->ring[i].queue_lock);
1011 }
1012
1013 platform_set_drvdata(pdev, priv);
1014 atomic_set(&priv->ring_used, 0);
1015
1016 ret = safexcel_hw_init(priv);
1017 if (ret) {
1018 dev_err(dev, "EIP h/w init failed (%d)\n", ret);
1019 goto err_reg_clk;
1020 }
1021
1022 ret = safexcel_register_algorithms(priv);
1023 if (ret) {
1024 dev_err(dev, "Failed to register algorithms (%d)\n", ret);
1025 goto err_reg_clk;
1026 }
1027
1028 return 0;
1029
1030 err_reg_clk:
1031 clk_disable_unprepare(priv->reg_clk);
1032 err_core_clk:
1033 clk_disable_unprepare(priv->clk);
1034 return ret;
1035 }
1036
1037
1038 static int safexcel_remove(struct platform_device *pdev)
1039 {
1040 struct safexcel_crypto_priv *priv = platform_get_drvdata(pdev);
1041 int i;
1042
1043 safexcel_unregister_algorithms(priv);
1044 clk_disable_unprepare(priv->clk);
1045
1046 for (i = 0; i < priv->config.rings; i++)
1047 destroy_workqueue(priv->ring[i].workqueue);
1048
1049 return 0;
1050 }
1051
1052 static const struct of_device_id safexcel_of_match_table[] = {
1053 {
1054 .compatible = "inside-secure,safexcel-eip97",
1055 .data = (void *)EIP97,
1056 },
1057 {
1058 .compatible = "inside-secure,safexcel-eip197",
1059 .data = (void *)EIP197,
1060 },
1061 {},
1062 };
1063
1064
1065 static struct platform_driver crypto_safexcel = {
1066 .probe = safexcel_probe,
1067 .remove = safexcel_remove,
1068 .driver = {
1069 .name = "crypto-safexcel",
1070 .of_match_table = safexcel_of_match_table,
1071 },
1072 };
1073 module_platform_driver(crypto_safexcel);
1074
1075 MODULE_AUTHOR("Antoine Tenart <antoine.tenart@free-electrons.com>");
1076 MODULE_AUTHOR("Ofer Heifetz <oferh@marvell.com>");
1077 MODULE_AUTHOR("Igal Liberman <igall@marvell.com>");
1078 MODULE_DESCRIPTION("Support for SafeXcel cryptographic engine EIP197");
1079 MODULE_LICENSE("GPL v2");
Linux with added FPC-III support