crypto/crypto_engine.c

   1 /*
   2  * Handle async block request by crypto hardware engine.
   3  *
   4  * Copyright (C) 2016 Linaro, Inc.
   5  *
   6  * Author: Baolin Wang <baolin.wang@linaro.org>
   7  *
   8  * This program is free software; you can redistribute it and/or modify it
   9  * under the terms of the GNU General Public License as published by the Free
  10  * Software Foundation; either version 2 of the License, or (at your option)
  11  * any later version.
  12  *
  13  */
  14
  15 #include <linux/err.h>
  16 #include <linux/delay.h>
  17 #include "internal.h"
  18
  19 #define CRYPTO_ENGINE_MAX_QLEN 10
  20
  21 void crypto_finalize_request(struct crypto_engine *engine,
  22                              struct ablkcipher_request *req, int err);
  23
  24 /**
  25  * crypto_pump_requests - dequeue one request from engine queue to process
  26  * @engine: the hardware engine
  27  * @in_kthread: true if we are in the context of the request pump thread
  28  *
  29  * This function checks if there is any request in the engine queue that
  30  * needs processing and if so call out to the driver to initialize hardware
  31  * and handle each request.
  32  */
  33 static void crypto_pump_requests(struct crypto_engine *engine,
  34                                  bool in_kthread)
  35 {
  36         struct crypto_async_request *async_req, *backlog;
  37         struct ablkcipher_request *req;
  38         unsigned long flags;
  39         bool was_busy = false;
  40         int ret;
  41
  42         spin_lock_irqsave(&engine->queue_lock, flags);
  43
  44         /* Make sure we are not already running a request */
  45         if (engine->cur_req)
  46                 goto out;
  47
  48         /* If another context is idling then defer */
  49         if (engine->idling) {
  50                 queue_kthread_work(&engine->kworker, &engine->pump_requests);
  51                 goto out;
  52         }
  53
  54         /* Check if the engine queue is idle */
  55         if (!crypto_queue_len(&engine->queue) || !engine->running) {
  56                 if (!engine->busy)
  57                         goto out;
  58
  59                 /* Only do teardown in the thread */
  60                 if (!in_kthread) {
  61                         queue_kthread_work(&engine->kworker,
  62                                            &engine->pump_requests);
  63                         goto out;
  64                 }
  65
  66                 engine->busy = false;
  67                 engine->idling = true;
  68                 spin_unlock_irqrestore(&engine->queue_lock, flags);
  69
  70                 if (engine->unprepare_crypt_hardware &&
  71                     engine->unprepare_crypt_hardware(engine))
  72                         pr_err("failed to unprepare crypt hardware\n");
  73
  74                 spin_lock_irqsave(&engine->queue_lock, flags);
  75                 engine->idling = false;
  76                 goto out;
  77         }
  78
  79         /* Get the fist request from the engine queue to handle */
  80         backlog = crypto_get_backlog(&engine->queue);
  81         async_req = crypto_dequeue_request(&engine->queue);
  82         if (!async_req)
  83                 goto out;
  84
  85         req = ablkcipher_request_cast(async_req);
  86
  87         engine->cur_req = req;
  88         if (backlog)
  89                 backlog->complete(backlog, -EINPROGRESS);
  90
  91         if (engine->busy)
  92                 was_busy = true;
  93         else
  94                 engine->busy = true;
  95
  96         spin_unlock_irqrestore(&engine->queue_lock, flags);
  97
  98         /* Until here we get the request need to be encrypted successfully */
  99         if (!was_busy && engine->prepare_crypt_hardware) {
 100                 ret = engine->prepare_crypt_hardware(engine);
 101                 if (ret) {
 102                         pr_err("failed to prepare crypt hardware\n");
 103                         goto req_err;
 104                 }
 105         }
 106
 107         if (engine->prepare_request) {
 108                 ret = engine->prepare_request(engine, engine->cur_req);
 109                 if (ret) {
 110                         pr_err("failed to prepare request: %d\n", ret);
 111                         goto req_err;
 112                 }
 113                 engine->cur_req_prepared = true;
 114         }
 115
 116         ret = engine->crypt_one_request(engine, engine->cur_req);
 117         if (ret) {
 118                 pr_err("failed to crypt one request from queue\n");
 119                 goto req_err;
 120         }
 121         return;
 122
 123 req_err:
 124         crypto_finalize_request(engine, engine->cur_req, ret);
 125         return;
 126
 127 out:
 128         spin_unlock_irqrestore(&engine->queue_lock, flags);
 129 }
 130
 131 static void crypto_pump_work(struct kthread_work *work)
 132 {
 133         struct crypto_engine *engine =
 134                 container_of(work, struct crypto_engine, pump_requests);
 135
 136         crypto_pump_requests(engine, true);
 137 }
 138
 139 /**
 140  * crypto_transfer_request - transfer the new request into the engine queue
 141  * @engine: the hardware engine
 142  * @req: the request need to be listed into the engine queue
 143  */
 144 int crypto_transfer_request(struct crypto_engine *engine,
 145                             struct ablkcipher_request *req, bool need_pump)
 146 {
 147         unsigned long flags;
 148         int ret;
 149
 150         spin_lock_irqsave(&engine->queue_lock, flags);
 151
 152         if (!engine->running) {
 153                 spin_unlock_irqrestore(&engine->queue_lock, flags);
 154                 return -ESHUTDOWN;
 155         }
 156
 157         ret = ablkcipher_enqueue_request(&engine->queue, req);
 158
 159         if (!engine->busy && need_pump)
 160                 queue_kthread_work(&engine->kworker, &engine->pump_requests);
 161
 162         spin_unlock_irqrestore(&engine->queue_lock, flags);
 163         return ret;
 164 }
 165 EXPORT_SYMBOL_GPL(crypto_transfer_request);
 166
 167 /**
 168  * crypto_transfer_request_to_engine - transfer one request to list into the
 169  * engine queue
 170  * @engine: the hardware engine
 171  * @req: the request need to be listed into the engine queue
 172  */
 173 int crypto_transfer_request_to_engine(struct crypto_engine *engine,
 174                                       struct ablkcipher_request *req)
 175 {
 176         return crypto_transfer_request(engine, req, true);
 177 }
 178 EXPORT_SYMBOL_GPL(crypto_transfer_request_to_engine);
 179
 180 /**
 181  * crypto_finalize_request - finalize one request if the request is done
 182  * @engine: the hardware engine
 183  * @req: the request need to be finalized
 184  * @err: error number
 185  */
 186 void crypto_finalize_request(struct crypto_engine *engine,
 187                              struct ablkcipher_request *req, int err)
 188 {
 189         unsigned long flags;
 190         bool finalize_cur_req = false;
 191         int ret;
 192
 193         spin_lock_irqsave(&engine->queue_lock, flags);
 194         if (engine->cur_req == req)
 195                 finalize_cur_req = true;
 196         spin_unlock_irqrestore(&engine->queue_lock, flags);
 197
 198         if (finalize_cur_req) {
 199                 if (engine->cur_req_prepared && engine->unprepare_request) {
 200                         ret = engine->unprepare_request(engine, req);
 201                         if (ret)
 202                                 pr_err("failed to unprepare request\n");
 203                 }
 204
 205                 spin_lock_irqsave(&engine->queue_lock, flags);
 206                 engine->cur_req = NULL;
 207                 engine->cur_req_prepared = false;
 208                 spin_unlock_irqrestore(&engine->queue_lock, flags);
 209         }
 210
 211         req->base.complete(&req->base, err);
 212
 213         queue_kthread_work(&engine->kworker, &engine->pump_requests);
 214 }
 215 EXPORT_SYMBOL_GPL(crypto_finalize_request);
 216
 217 /**
 218  * crypto_engine_start - start the hardware engine
 219  * @engine: the hardware engine need to be started
 220  *
 221  * Return 0 on success, else on fail.
 222  */
 223 int crypto_engine_start(struct crypto_engine *engine)
 224 {
 225         unsigned long flags;
 226
 227         spin_lock_irqsave(&engine->queue_lock, flags);
 228
 229         if (engine->running || engine->busy) {
 230                 spin_unlock_irqrestore(&engine->queue_lock, flags);
 231                 return -EBUSY;
 232         }
 233
 234         engine->running = true;
 235         spin_unlock_irqrestore(&engine->queue_lock, flags);
 236
 237         queue_kthread_work(&engine->kworker, &engine->pump_requests);
 238
 239         return 0;
 240 }
 241 EXPORT_SYMBOL_GPL(crypto_engine_start);
 242
 243 /**
 244  * crypto_engine_stop - stop the hardware engine
 245  * @engine: the hardware engine need to be stopped
 246  *
 247  * Return 0 on success, else on fail.
 248  */
 249 int crypto_engine_stop(struct crypto_engine *engine)
 250 {
 251         unsigned long flags;
 252         unsigned limit = 500;
 253         int ret = 0;
 254
 255         spin_lock_irqsave(&engine->queue_lock, flags);
 256
 257         /*
 258          * If the engine queue is not empty or the engine is on busy state,
 259          * we need to wait for a while to pump the requests of engine queue.
 260          */
 261         while ((crypto_queue_len(&engine->queue) || engine->busy) && limit--) {
 262                 spin_unlock_irqrestore(&engine->queue_lock, flags);
 263                 msleep(20);
 264                 spin_lock_irqsave(&engine->queue_lock, flags);
 265         }
 266
 267         if (crypto_queue_len(&engine->queue) || engine->busy)
 268                 ret = -EBUSY;
 269         else
 270                 engine->running = false;
 271
 272         spin_unlock_irqrestore(&engine->queue_lock, flags);
 273
 274         if (ret)
 275                 pr_warn("could not stop engine\n");
 276
 277         return ret;
 278 }
 279 EXPORT_SYMBOL_GPL(crypto_engine_stop);
 280
 281 /**
 282  * crypto_engine_alloc_init - allocate crypto hardware engine structure and
 283  * initialize it.
 284  * @dev: the device attached with one hardware engine
 285  * @rt: whether this queue is set to run as a realtime task
 286  *
 287  * This must be called from context that can sleep.
 288  * Return: the crypto engine structure on success, else NULL.
 289  */
 290 struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt)
 291 {
 292         struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
 293         struct crypto_engine *engine;
 294
 295         if (!dev)
 296                 return NULL;
 297
 298         engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL);
 299         if (!engine)
 300                 return NULL;
 301
 302         engine->rt = rt;
 303         engine->running = false;
 304         engine->busy = false;
 305         engine->idling = false;
 306         engine->cur_req_prepared = false;
 307         engine->priv_data = dev;
 308         snprintf(engine->name, sizeof(engine->name),
 309                  "%s-engine", dev_name(dev));
 310
 311         crypto_init_queue(&engine->queue, CRYPTO_ENGINE_MAX_QLEN);
 312         spin_lock_init(&engine->queue_lock);
 313
 314         init_kthread_worker(&engine->kworker);
 315         engine->kworker_task = kthread_run(kthread_worker_fn,
 316                                            &engine->kworker, "%s",
 317                                            engine->name);
 318         if (IS_ERR(engine->kworker_task)) {
 319                 dev_err(dev, "failed to create crypto request pump task\n");
 320                 return NULL;
 321         }
 322         init_kthread_work(&engine->pump_requests, crypto_pump_work);
 323
 324         if (engine->rt) {
 325                 dev_info(dev, "will run requests pump with realtime priority\n");
 326                 sched_setscheduler(engine->kworker_task, SCHED_FIFO, &param);
 327         }
 328
 329         return engine;
 330 }
 331 EXPORT_SYMBOL_GPL(crypto_engine_alloc_init);
 332
 333 /**
 334  * crypto_engine_exit - free the resources of hardware engine when exit
 335  * @engine: the hardware engine need to be freed
 336  *
 337  * Return 0 for success.
 338  */
 339 int crypto_engine_exit(struct crypto_engine *engine)
 340 {
 341         int ret;
 342
 343         ret = crypto_engine_stop(engine);
 344         if (ret)
 345                 return ret;
 346
 347         flush_kthread_worker(&engine->kworker);
 348         kthread_stop(engine->kworker_task);
 349
 350         return 0;
 351 }
 352 EXPORT_SYMBOL_GPL(crypto_engine_exit);
 353
 354 MODULE_LICENSE("GPL");
 355 MODULE_DESCRIPTION("Crypto hardware engine framework");