1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Handle async block request by crypto hardware engine.
5 * Copyright (C) 2016 Linaro, Inc.
7 * Author: Baolin Wang <baolin.wang@linaro.org>
10 #include <linux/err.h>
11 #include <linux/delay.h>
12 #include <linux/device.h>
13 #include <crypto/engine.h>
14 #include <uapi/linux/sched/types.h>
17 #define CRYPTO_ENGINE_MAX_QLEN 10
20 * crypto_finalize_request - finalize one request if the request is done
21 * @engine: the hardware engine
22 * @req: the request need to be finalized
25 static void crypto_finalize_request(struct crypto_engine
*engine
,
26 struct crypto_async_request
*req
, int err
)
29 bool finalize_req
= false;
31 struct crypto_engine_ctx
*enginectx
;
34 * If hardware cannot enqueue more requests
35 * and retry mechanism is not supported
36 * make sure we are completing the current request
38 if (!engine
->retry_support
) {
39 spin_lock_irqsave(&engine
->queue_lock
, flags
);
40 if (engine
->cur_req
== req
) {
42 engine
->cur_req
= NULL
;
44 spin_unlock_irqrestore(&engine
->queue_lock
, flags
);
47 if (finalize_req
|| engine
->retry_support
) {
48 enginectx
= crypto_tfm_ctx(req
->tfm
);
49 if (enginectx
->op
.prepare_request
&&
50 enginectx
->op
.unprepare_request
) {
51 ret
= enginectx
->op
.unprepare_request(engine
, req
);
53 dev_err(engine
->dev
, "failed to unprepare request\n");
56 req
->complete(req
, err
);
58 kthread_queue_work(engine
->kworker
, &engine
->pump_requests
);
62 * crypto_pump_requests - dequeue one request from engine queue to process
63 * @engine: the hardware engine
64 * @in_kthread: true if we are in the context of the request pump thread
66 * This function checks if there is any request in the engine queue that
67 * needs processing and if so call out to the driver to initialize hardware
68 * and handle each request.
70 static void crypto_pump_requests(struct crypto_engine
*engine
,
73 struct crypto_async_request
*async_req
, *backlog
;
75 bool was_busy
= false;
77 struct crypto_engine_ctx
*enginectx
;
79 spin_lock_irqsave(&engine
->queue_lock
, flags
);
81 /* Make sure we are not already running a request */
82 if (!engine
->retry_support
&& engine
->cur_req
)
85 /* If another context is idling then defer */
87 kthread_queue_work(engine
->kworker
, &engine
->pump_requests
);
91 /* Check if the engine queue is idle */
92 if (!crypto_queue_len(&engine
->queue
) || !engine
->running
) {
96 /* Only do teardown in the thread */
98 kthread_queue_work(engine
->kworker
,
99 &engine
->pump_requests
);
103 engine
->busy
= false;
104 engine
->idling
= true;
105 spin_unlock_irqrestore(&engine
->queue_lock
, flags
);
107 if (engine
->unprepare_crypt_hardware
&&
108 engine
->unprepare_crypt_hardware(engine
))
109 dev_err(engine
->dev
, "failed to unprepare crypt hardware\n");
111 spin_lock_irqsave(&engine
->queue_lock
, flags
);
112 engine
->idling
= false;
117 /* Get the fist request from the engine queue to handle */
118 backlog
= crypto_get_backlog(&engine
->queue
);
119 async_req
= crypto_dequeue_request(&engine
->queue
);
124 * If hardware doesn't support the retry mechanism,
125 * keep track of the request we are processing now.
126 * We'll need it on completion (crypto_finalize_request).
128 if (!engine
->retry_support
)
129 engine
->cur_req
= async_req
;
132 backlog
->complete(backlog
, -EINPROGRESS
);
139 spin_unlock_irqrestore(&engine
->queue_lock
, flags
);
141 /* Until here we get the request need to be encrypted successfully */
142 if (!was_busy
&& engine
->prepare_crypt_hardware
) {
143 ret
= engine
->prepare_crypt_hardware(engine
);
145 dev_err(engine
->dev
, "failed to prepare crypt hardware\n");
150 enginectx
= crypto_tfm_ctx(async_req
->tfm
);
152 if (enginectx
->op
.prepare_request
) {
153 ret
= enginectx
->op
.prepare_request(engine
, async_req
);
155 dev_err(engine
->dev
, "failed to prepare request: %d\n",
160 if (!enginectx
->op
.do_one_request
) {
161 dev_err(engine
->dev
, "failed to do request\n");
166 ret
= enginectx
->op
.do_one_request(engine
, async_req
);
168 /* Request unsuccessfully executed by hardware */
171 * If hardware queue is full (-ENOSPC), requeue request
172 * regardless of backlog flag.
173 * Otherwise, unprepare and complete the request.
175 if (!engine
->retry_support
||
178 "Failed to do one request from queue: %d\n",
183 * If retry mechanism is supported,
184 * unprepare current request and
185 * enqueue it back into crypto-engine queue.
187 if (enginectx
->op
.unprepare_request
) {
188 ret
= enginectx
->op
.unprepare_request(engine
,
192 "failed to unprepare request\n");
194 spin_lock_irqsave(&engine
->queue_lock
, flags
);
196 * If hardware was unable to execute request, enqueue it
197 * back in front of crypto-engine queue, to keep the order
200 crypto_enqueue_request_head(&engine
->queue
, async_req
);
202 kthread_queue_work(engine
->kworker
, &engine
->pump_requests
);
209 if (enginectx
->op
.unprepare_request
) {
210 ret
= enginectx
->op
.unprepare_request(engine
, async_req
);
212 dev_err(engine
->dev
, "failed to unprepare request\n");
216 async_req
->complete(async_req
, ret
);
219 /* If retry mechanism is supported, send new requests to engine */
220 if (engine
->retry_support
) {
221 spin_lock_irqsave(&engine
->queue_lock
, flags
);
227 spin_unlock_irqrestore(&engine
->queue_lock
, flags
);
230 * Batch requests is possible only if
231 * hardware can enqueue multiple requests
233 if (engine
->do_batch_requests
) {
234 ret
= engine
->do_batch_requests(engine
);
236 dev_err(engine
->dev
, "failed to do batch requests: %d\n",
243 static void crypto_pump_work(struct kthread_work
*work
)
245 struct crypto_engine
*engine
=
246 container_of(work
, struct crypto_engine
, pump_requests
);
248 crypto_pump_requests(engine
, true);
252 * crypto_transfer_request - transfer the new request into the engine queue
253 * @engine: the hardware engine
254 * @req: the request need to be listed into the engine queue
256 static int crypto_transfer_request(struct crypto_engine
*engine
,
257 struct crypto_async_request
*req
,
263 spin_lock_irqsave(&engine
->queue_lock
, flags
);
265 if (!engine
->running
) {
266 spin_unlock_irqrestore(&engine
->queue_lock
, flags
);
270 ret
= crypto_enqueue_request(&engine
->queue
, req
);
272 if (!engine
->busy
&& need_pump
)
273 kthread_queue_work(engine
->kworker
, &engine
->pump_requests
);
275 spin_unlock_irqrestore(&engine
->queue_lock
, flags
);
280 * crypto_transfer_request_to_engine - transfer one request to list
281 * into the engine queue
282 * @engine: the hardware engine
283 * @req: the request need to be listed into the engine queue
285 static int crypto_transfer_request_to_engine(struct crypto_engine
*engine
,
286 struct crypto_async_request
*req
)
288 return crypto_transfer_request(engine
, req
, true);
292 * crypto_transfer_aead_request_to_engine - transfer one aead_request
293 * to list into the engine queue
294 * @engine: the hardware engine
295 * @req: the request need to be listed into the engine queue
297 int crypto_transfer_aead_request_to_engine(struct crypto_engine
*engine
,
298 struct aead_request
*req
)
300 return crypto_transfer_request_to_engine(engine
, &req
->base
);
302 EXPORT_SYMBOL_GPL(crypto_transfer_aead_request_to_engine
);
305 * crypto_transfer_akcipher_request_to_engine - transfer one akcipher_request
306 * to list into the engine queue
307 * @engine: the hardware engine
308 * @req: the request need to be listed into the engine queue
310 int crypto_transfer_akcipher_request_to_engine(struct crypto_engine
*engine
,
311 struct akcipher_request
*req
)
313 return crypto_transfer_request_to_engine(engine
, &req
->base
);
315 EXPORT_SYMBOL_GPL(crypto_transfer_akcipher_request_to_engine
);
318 * crypto_transfer_hash_request_to_engine - transfer one ahash_request
319 * to list into the engine queue
320 * @engine: the hardware engine
321 * @req: the request need to be listed into the engine queue
323 int crypto_transfer_hash_request_to_engine(struct crypto_engine
*engine
,
324 struct ahash_request
*req
)
326 return crypto_transfer_request_to_engine(engine
, &req
->base
);
328 EXPORT_SYMBOL_GPL(crypto_transfer_hash_request_to_engine
);
331 * crypto_transfer_skcipher_request_to_engine - transfer one skcipher_request
332 * to list into the engine queue
333 * @engine: the hardware engine
334 * @req: the request need to be listed into the engine queue
336 int crypto_transfer_skcipher_request_to_engine(struct crypto_engine
*engine
,
337 struct skcipher_request
*req
)
339 return crypto_transfer_request_to_engine(engine
, &req
->base
);
341 EXPORT_SYMBOL_GPL(crypto_transfer_skcipher_request_to_engine
);
344 * crypto_finalize_aead_request - finalize one aead_request if
345 * the request is done
346 * @engine: the hardware engine
347 * @req: the request need to be finalized
350 void crypto_finalize_aead_request(struct crypto_engine
*engine
,
351 struct aead_request
*req
, int err
)
353 return crypto_finalize_request(engine
, &req
->base
, err
);
355 EXPORT_SYMBOL_GPL(crypto_finalize_aead_request
);
358 * crypto_finalize_akcipher_request - finalize one akcipher_request if
359 * the request is done
360 * @engine: the hardware engine
361 * @req: the request need to be finalized
364 void crypto_finalize_akcipher_request(struct crypto_engine
*engine
,
365 struct akcipher_request
*req
, int err
)
367 return crypto_finalize_request(engine
, &req
->base
, err
);
369 EXPORT_SYMBOL_GPL(crypto_finalize_akcipher_request
);
372 * crypto_finalize_hash_request - finalize one ahash_request if
373 * the request is done
374 * @engine: the hardware engine
375 * @req: the request need to be finalized
378 void crypto_finalize_hash_request(struct crypto_engine
*engine
,
379 struct ahash_request
*req
, int err
)
381 return crypto_finalize_request(engine
, &req
->base
, err
);
383 EXPORT_SYMBOL_GPL(crypto_finalize_hash_request
);
386 * crypto_finalize_skcipher_request - finalize one skcipher_request if
387 * the request is done
388 * @engine: the hardware engine
389 * @req: the request need to be finalized
392 void crypto_finalize_skcipher_request(struct crypto_engine
*engine
,
393 struct skcipher_request
*req
, int err
)
395 return crypto_finalize_request(engine
, &req
->base
, err
);
397 EXPORT_SYMBOL_GPL(crypto_finalize_skcipher_request
);
400 * crypto_engine_start - start the hardware engine
401 * @engine: the hardware engine need to be started
403 * Return 0 on success, else on fail.
405 int crypto_engine_start(struct crypto_engine
*engine
)
409 spin_lock_irqsave(&engine
->queue_lock
, flags
);
411 if (engine
->running
|| engine
->busy
) {
412 spin_unlock_irqrestore(&engine
->queue_lock
, flags
);
416 engine
->running
= true;
417 spin_unlock_irqrestore(&engine
->queue_lock
, flags
);
419 kthread_queue_work(engine
->kworker
, &engine
->pump_requests
);
423 EXPORT_SYMBOL_GPL(crypto_engine_start
);
426 * crypto_engine_stop - stop the hardware engine
427 * @engine: the hardware engine need to be stopped
429 * Return 0 on success, else on fail.
431 int crypto_engine_stop(struct crypto_engine
*engine
)
434 unsigned int limit
= 500;
437 spin_lock_irqsave(&engine
->queue_lock
, flags
);
440 * If the engine queue is not empty or the engine is on busy state,
441 * we need to wait for a while to pump the requests of engine queue.
443 while ((crypto_queue_len(&engine
->queue
) || engine
->busy
) && limit
--) {
444 spin_unlock_irqrestore(&engine
->queue_lock
, flags
);
446 spin_lock_irqsave(&engine
->queue_lock
, flags
);
449 if (crypto_queue_len(&engine
->queue
) || engine
->busy
)
452 engine
->running
= false;
454 spin_unlock_irqrestore(&engine
->queue_lock
, flags
);
457 dev_warn(engine
->dev
, "could not stop engine\n");
461 EXPORT_SYMBOL_GPL(crypto_engine_stop
);
464 * crypto_engine_alloc_init_and_set - allocate crypto hardware engine structure
465 * and initialize it by setting the maximum number of entries in the software
466 * crypto-engine queue.
467 * @dev: the device attached with one hardware engine
468 * @retry_support: whether hardware has support for retry mechanism
469 * @cbk_do_batch: pointer to a callback function to be invoked when executing
470 * a batch of requests.
472 * callback(struct crypto_engine *engine)
474 * @engine: the crypto engine structure.
475 * @rt: whether this queue is set to run as a realtime task
476 * @qlen: maximum size of the crypto-engine queue
478 * This must be called from context that can sleep.
479 * Return: the crypto engine structure on success, else NULL.
481 struct crypto_engine
*crypto_engine_alloc_init_and_set(struct device
*dev
,
483 int (*cbk_do_batch
)(struct crypto_engine
*engine
),
486 struct crypto_engine
*engine
;
491 engine
= devm_kzalloc(dev
, sizeof(*engine
), GFP_KERNEL
);
497 engine
->running
= false;
498 engine
->busy
= false;
499 engine
->idling
= false;
500 engine
->retry_support
= retry_support
;
501 engine
->priv_data
= dev
;
503 * Batch requests is possible only if
504 * hardware has support for retry mechanism.
506 engine
->do_batch_requests
= retry_support
? cbk_do_batch
: NULL
;
508 snprintf(engine
->name
, sizeof(engine
->name
),
509 "%s-engine", dev_name(dev
));
511 crypto_init_queue(&engine
->queue
, qlen
);
512 spin_lock_init(&engine
->queue_lock
);
514 engine
->kworker
= kthread_create_worker(0, "%s", engine
->name
);
515 if (IS_ERR(engine
->kworker
)) {
516 dev_err(dev
, "failed to create crypto request pump task\n");
519 kthread_init_work(&engine
->pump_requests
, crypto_pump_work
);
522 dev_info(dev
, "will run requests pump with realtime priority\n");
523 sched_set_fifo(engine
->kworker
->task
);
528 EXPORT_SYMBOL_GPL(crypto_engine_alloc_init_and_set
);
531 * crypto_engine_alloc_init - allocate crypto hardware engine structure and
533 * @dev: the device attached with one hardware engine
534 * @rt: whether this queue is set to run as a realtime task
536 * This must be called from context that can sleep.
537 * Return: the crypto engine structure on success, else NULL.
539 struct crypto_engine
*crypto_engine_alloc_init(struct device
*dev
, bool rt
)
541 return crypto_engine_alloc_init_and_set(dev
, false, NULL
, rt
,
542 CRYPTO_ENGINE_MAX_QLEN
);
544 EXPORT_SYMBOL_GPL(crypto_engine_alloc_init
);
547 * crypto_engine_exit - free the resources of hardware engine when exit
548 * @engine: the hardware engine need to be freed
550 * Return 0 for success.
552 int crypto_engine_exit(struct crypto_engine
*engine
)
556 ret
= crypto_engine_stop(engine
);
560 kthread_destroy_worker(engine
->kworker
);
564 EXPORT_SYMBOL_GPL(crypto_engine_exit
);
566 MODULE_LICENSE("GPL");
567 MODULE_DESCRIPTION("Crypto hardware engine framework");