drivers/crypto/cavium/nitrox/nitrox_reqmgr.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/gfp.h>
   3 #include <linux/workqueue.h>
   4 #include <crypto/internal/skcipher.h>
   5
   6 #include "nitrox_dev.h"
   7 #include "nitrox_req.h"
   8 #include "nitrox_csr.h"
   9
  10 /* SLC_STORE_INFO */
  11 #define MIN_UDD_LEN 16
  12 /* PKT_IN_HDR + SLC_STORE_INFO */
  13 #define FDATA_SIZE 32
  14 /* Base destination port for the solicited requests */
  15 #define SOLICIT_BASE_DPORT 256
  16
  17 #define REQ_NOT_POSTED 1
  18 #define REQ_BACKLOG    2
  19 #define REQ_POSTED     3
  20
  21 /**
  22  * Response codes from SE microcode
  23  * 0x00 - Success
  24  *   Completion with no error
  25  * 0x43 - ERR_GC_DATA_LEN_INVALID
  26  *   Invalid Data length if Encryption Data length is
  27  *   less than 16 bytes for AES-XTS and AES-CTS.
  28  * 0x45 - ERR_GC_CTX_LEN_INVALID
  29  *   Invalid context length: CTXL != 23 words.
  30  * 0x4F - ERR_GC_DOCSIS_CIPHER_INVALID
  31  *   DOCSIS support is enabled with other than
  32  *   AES/DES-CBC mode encryption.
  33  * 0x50 - ERR_GC_DOCSIS_OFFSET_INVALID
  34  *   Authentication offset is other than 0 with
  35  *   Encryption IV source = 0.
  36  *   Authentication offset is other than 8 (DES)/16 (AES)
  37  *   with Encryption IV source = 1
  38  * 0x51 - ERR_GC_CRC32_INVALID_SELECTION
  39  *   CRC32 is enabled for other than DOCSIS encryption.
  40  * 0x52 - ERR_GC_AES_CCM_FLAG_INVALID
  41  *   Invalid flag options in AES-CCM IV.
  42  */
  43
  44 static inline int incr_index(int index, int count, int max)
  45 {
  46         if ((index + count) >= max)
  47                 index = index + count - max;
  48         else
  49                 index += count;
  50
  51         return index;
  52 }
  53
  54 static void softreq_unmap_sgbufs(struct nitrox_softreq *sr)
  55 {
  56         struct nitrox_device *ndev = sr->ndev;
  57         struct device *dev = DEV(ndev);
  58
  59
  60         dma_unmap_sg(dev, sr->in.sg, sr->in.sgmap_cnt, DMA_BIDIRECTIONAL);
  61         dma_unmap_single(dev, sr->in.sgcomp_dma, sr->in.sgcomp_len,
  62                          DMA_TO_DEVICE);
  63         kfree(sr->in.sgcomp);
  64         sr->in.sg = NULL;
  65         sr->in.sgmap_cnt = 0;
  66
  67         dma_unmap_sg(dev, sr->out.sg, sr->out.sgmap_cnt,
  68                      DMA_BIDIRECTIONAL);
  69         dma_unmap_single(dev, sr->out.sgcomp_dma, sr->out.sgcomp_len,
  70                          DMA_TO_DEVICE);
  71         kfree(sr->out.sgcomp);
  72         sr->out.sg = NULL;
  73         sr->out.sgmap_cnt = 0;
  74 }
  75
  76 static void softreq_destroy(struct nitrox_softreq *sr)
  77 {
  78         softreq_unmap_sgbufs(sr);
  79         kfree(sr);
  80 }
  81
  82 /**
  83  * create_sg_component - create SG componets for N5 device.
  84  * @sr: Request structure
  85  * @sgtbl: SG table
  86  * @map_nents: number of dma mapped entries
  87  *
  88  * Component structure
  89  *
  90  *   63     48 47     32 31    16 15      0
  91  *   --------------------------------------
  92  *   |   LEN0  |  LEN1  |  LEN2  |  LEN3  |
  93  *   |-------------------------------------
  94  *   |               PTR0                 |
  95  *   --------------------------------------
  96  *   |               PTR1                 |
  97  *   --------------------------------------
  98  *   |               PTR2                 |
  99  *   --------------------------------------
 100  *   |               PTR3                 |
 101  *   --------------------------------------
 102  *
 103  *   Returns 0 if success or a negative errno code on error.
 104  */
 105 static int create_sg_component(struct nitrox_softreq *sr,
 106                                struct nitrox_sgtable *sgtbl, int map_nents)
 107 {
 108         struct nitrox_device *ndev = sr->ndev;
 109         struct nitrox_sgcomp *sgcomp;
 110         struct scatterlist *sg;
 111         dma_addr_t dma;
 112         size_t sz_comp;
 113         int i, j, nr_sgcomp;
 114
 115         nr_sgcomp = roundup(map_nents, 4) / 4;
 116
 117         /* each component holds 4 dma pointers */
 118         sz_comp = nr_sgcomp * sizeof(*sgcomp);
 119         sgcomp = kzalloc(sz_comp, sr->gfp);
 120         if (!sgcomp)
 121                 return -ENOMEM;
 122
 123         sgtbl->sgcomp = sgcomp;
 124
 125         sg = sgtbl->sg;
 126         /* populate device sg component */
 127         for (i = 0; i < nr_sgcomp; i++) {
 128                 for (j = 0; j < 4 && sg; j++) {
 129                         sgcomp[i].len[j] = cpu_to_be16(sg_dma_len(sg));
 130                         sgcomp[i].dma[j] = cpu_to_be64(sg_dma_address(sg));
 131                         sg = sg_next(sg);
 132                 }
 133         }
 134         /* map the device sg component */
 135         dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE);
 136         if (dma_mapping_error(DEV(ndev), dma)) {
 137                 kfree(sgtbl->sgcomp);
 138                 sgtbl->sgcomp = NULL;
 139                 return -ENOMEM;
 140         }
 141
 142         sgtbl->sgcomp_dma = dma;
 143         sgtbl->sgcomp_len = sz_comp;
 144
 145         return 0;
 146 }
 147
 148 /**
 149  * dma_map_inbufs - DMA map input sglist and creates sglist component
 150  *                  for N5 device.
 151  * @sr: Request structure
 152  * @req: Crypto request structre
 153  *
 154  * Returns 0 if successful or a negative errno code on error.
 155  */
 156 static int dma_map_inbufs(struct nitrox_softreq *sr,
 157                           struct se_crypto_request *req)
 158 {
 159         struct device *dev = DEV(sr->ndev);
 160         struct scatterlist *sg = req->src;
 161         int i, nents, ret = 0;
 162
 163         nents = dma_map_sg(dev, req->src, sg_nents(req->src),
 164                            DMA_BIDIRECTIONAL);
 165         if (!nents)
 166                 return -EINVAL;
 167
 168         for_each_sg(req->src, sg, nents, i)
 169                 sr->in.total_bytes += sg_dma_len(sg);
 170
 171         sr->in.sg = req->src;
 172         sr->in.sgmap_cnt = nents;
 173         ret = create_sg_component(sr, &sr->in, sr->in.sgmap_cnt);
 174         if (ret)
 175                 goto incomp_err;
 176
 177         return 0;
 178
 179 incomp_err:
 180         dma_unmap_sg(dev, req->src, nents, DMA_BIDIRECTIONAL);
 181         sr->in.sgmap_cnt = 0;
 182         return ret;
 183 }
 184
 185 static int dma_map_outbufs(struct nitrox_softreq *sr,
 186                            struct se_crypto_request *req)
 187 {
 188         struct device *dev = DEV(sr->ndev);
 189         int nents, ret = 0;
 190
 191         nents = dma_map_sg(dev, req->dst, sg_nents(req->dst),
 192                            DMA_BIDIRECTIONAL);
 193         if (!nents)
 194                 return -EINVAL;
 195
 196         sr->out.sg = req->dst;
 197         sr->out.sgmap_cnt = nents;
 198         ret = create_sg_component(sr, &sr->out, sr->out.sgmap_cnt);
 199         if (ret)
 200                 goto outcomp_map_err;
 201
 202         return 0;
 203
 204 outcomp_map_err:
 205         dma_unmap_sg(dev, req->dst, nents, DMA_BIDIRECTIONAL);
 206         sr->out.sgmap_cnt = 0;
 207         sr->out.sg = NULL;
 208         return ret;
 209 }
 210
 211 static inline int softreq_map_iobuf(struct nitrox_softreq *sr,
 212                                     struct se_crypto_request *creq)
 213 {
 214         int ret;
 215
 216         ret = dma_map_inbufs(sr, creq);
 217         if (ret)
 218                 return ret;
 219
 220         ret = dma_map_outbufs(sr, creq);
 221         if (ret)
 222                 softreq_unmap_sgbufs(sr);
 223
 224         return ret;
 225 }
 226
 227 static inline void backlog_list_add(struct nitrox_softreq *sr,
 228                                     struct nitrox_cmdq *cmdq)
 229 {
 230         INIT_LIST_HEAD(&sr->backlog);
 231
 232         spin_lock_bh(&cmdq->backlog_qlock);
 233         list_add_tail(&sr->backlog, &cmdq->backlog_head);
 234         atomic_inc(&cmdq->backlog_count);
 235         atomic_set(&sr->status, REQ_BACKLOG);
 236         spin_unlock_bh(&cmdq->backlog_qlock);
 237 }
 238
 239 static inline void response_list_add(struct nitrox_softreq *sr,
 240                                      struct nitrox_cmdq *cmdq)
 241 {
 242         INIT_LIST_HEAD(&sr->response);
 243
 244         spin_lock_bh(&cmdq->resp_qlock);
 245         list_add_tail(&sr->response, &cmdq->response_head);
 246         spin_unlock_bh(&cmdq->resp_qlock);
 247 }
 248
 249 static inline void response_list_del(struct nitrox_softreq *sr,
 250                                      struct nitrox_cmdq *cmdq)
 251 {
 252         spin_lock_bh(&cmdq->resp_qlock);
 253         list_del(&sr->response);
 254         spin_unlock_bh(&cmdq->resp_qlock);
 255 }
 256
 257 static struct nitrox_softreq *
 258 get_first_response_entry(struct nitrox_cmdq *cmdq)
 259 {
 260         return list_first_entry_or_null(&cmdq->response_head,
 261                                         struct nitrox_softreq, response);
 262 }
 263
 264 static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen)
 265 {
 266         if (atomic_inc_return(&cmdq->pending_count) > qlen) {
 267                 atomic_dec(&cmdq->pending_count);
 268                 /* sync with other cpus */
 269                 smp_mb__after_atomic();
 270                 return true;
 271         }
 272         /* sync with other cpus */
 273         smp_mb__after_atomic();
 274         return false;
 275 }
 276
 277 /**
 278  * post_se_instr - Post SE instruction to Packet Input ring
 279  * @sr: Request structure
 280  *
 281  * Returns 0 if successful or a negative error code,
 282  * if no space in ring.
 283  */
 284 static void post_se_instr(struct nitrox_softreq *sr,
 285                           struct nitrox_cmdq *cmdq)
 286 {
 287         struct nitrox_device *ndev = sr->ndev;
 288         int idx;
 289         u8 *ent;
 290
 291         spin_lock_bh(&cmdq->cmd_qlock);
 292
 293         idx = cmdq->write_idx;
 294         /* copy the instruction */
 295         ent = cmdq->base + (idx * cmdq->instr_size);
 296         memcpy(ent, &sr->instr, cmdq->instr_size);
 297
 298         atomic_set(&sr->status, REQ_POSTED);
 299         response_list_add(sr, cmdq);
 300         sr->tstamp = jiffies;
 301         /* flush the command queue updates */
 302         dma_wmb();
 303
 304         /* Ring doorbell with count 1 */
 305         writeq(1, cmdq->dbell_csr_addr);
 306
 307         cmdq->write_idx = incr_index(idx, 1, ndev->qlen);
 308
 309         spin_unlock_bh(&cmdq->cmd_qlock);
 310
 311         /* increment the posted command count */
 312         atomic64_inc(&ndev->stats.posted);
 313 }
 314
 315 static int post_backlog_cmds(struct nitrox_cmdq *cmdq)
 316 {
 317         struct nitrox_device *ndev = cmdq->ndev;
 318         struct nitrox_softreq *sr, *tmp;
 319         int ret = 0;
 320
 321         if (!atomic_read(&cmdq->backlog_count))
 322                 return 0;
 323
 324         spin_lock_bh(&cmdq->backlog_qlock);
 325
 326         list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) {
 327                 /* submit until space available */
 328                 if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
 329                         ret = -ENOSPC;
 330                         break;
 331                 }
 332                 /* delete from backlog list */
 333                 list_del(&sr->backlog);
 334                 atomic_dec(&cmdq->backlog_count);
 335                 /* sync with other cpus */
 336                 smp_mb__after_atomic();
 337
 338                 /* post the command */
 339                 post_se_instr(sr, cmdq);
 340         }
 341         spin_unlock_bh(&cmdq->backlog_qlock);
 342
 343         return ret;
 344 }
 345
 346 static int nitrox_enqueue_request(struct nitrox_softreq *sr)
 347 {
 348         struct nitrox_cmdq *cmdq = sr->cmdq;
 349         struct nitrox_device *ndev = sr->ndev;
 350
 351         /* try to post backlog requests */
 352         post_backlog_cmds(cmdq);
 353
 354         if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
 355                 if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
 356                         /* increment drop count */
 357                         atomic64_inc(&ndev->stats.dropped);
 358                         return -ENOSPC;
 359                 }
 360                 /* add to backlog list */
 361                 backlog_list_add(sr, cmdq);
 362                 return -EINPROGRESS;
 363         }
 364         post_se_instr(sr, cmdq);
 365
 366         return -EINPROGRESS;
 367 }
 368
 369 /**
 370  * nitrox_se_request - Send request to SE core
 371  * @ndev: NITROX device
 372  * @req: Crypto request
 373  *
 374  * Returns 0 on success, or a negative error code.
 375  */
 376 int nitrox_process_se_request(struct nitrox_device *ndev,
 377                               struct se_crypto_request *req,
 378                               completion_t callback,
 379                               void *cb_arg)
 380 {
 381         struct nitrox_softreq *sr;
 382         dma_addr_t ctx_handle = 0;
 383         int qno, ret = 0;
 384
 385         if (!nitrox_ready(ndev))
 386                 return -ENODEV;
 387
 388         sr = kzalloc(sizeof(*sr), req->gfp);
 389         if (!sr)
 390                 return -ENOMEM;
 391
 392         sr->ndev = ndev;
 393         sr->flags = req->flags;
 394         sr->gfp = req->gfp;
 395         sr->callback = callback;
 396         sr->cb_arg = cb_arg;
 397
 398         atomic_set(&sr->status, REQ_NOT_POSTED);
 399
 400         sr->resp.orh = req->orh;
 401         sr->resp.completion = req->comp;
 402
 403         ret = softreq_map_iobuf(sr, req);
 404         if (ret) {
 405                 kfree(sr);
 406                 return ret;
 407         }
 408
 409         /* get the context handle */
 410         if (req->ctx_handle) {
 411                 struct ctx_hdr *hdr;
 412                 u8 *ctx_ptr;
 413
 414                 ctx_ptr = (u8 *)(uintptr_t)req->ctx_handle;
 415                 hdr = (struct ctx_hdr *)(ctx_ptr - sizeof(struct ctx_hdr));
 416                 ctx_handle = hdr->ctx_dma;
 417         }
 418
 419         /* select the queue */
 420         qno = smp_processor_id() % ndev->nr_queues;
 421
 422         sr->cmdq = &ndev->pkt_inq[qno];
 423
 424         /*
 425          * 64-Byte Instruction Format
 426          *
 427          *  ----------------------
 428          *  |      DPTR0         | 8 bytes
 429          *  ----------------------
 430          *  |  PKT_IN_INSTR_HDR  | 8 bytes
 431          *  ----------------------
 432          *  |    PKT_IN_HDR      | 16 bytes
 433          *  ----------------------
 434          *  |    SLC_INFO        | 16 bytes
 435          *  ----------------------
 436          *  |   Front data       | 16 bytes
 437          *  ----------------------
 438          */
 439
 440         /* fill the packet instruction */
 441         /* word 0 */
 442         sr->instr.dptr0 = cpu_to_be64(sr->in.sgcomp_dma);
 443
 444         /* word 1 */
 445         sr->instr.ih.value = 0;
 446         sr->instr.ih.s.g = 1;
 447         sr->instr.ih.s.gsz = sr->in.sgmap_cnt;
 448         sr->instr.ih.s.ssz = sr->out.sgmap_cnt;
 449         sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr);
 450         sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes;
 451         sr->instr.ih.value = cpu_to_be64(sr->instr.ih.value);
 452
 453         /* word 2 */
 454         sr->instr.irh.value[0] = 0;
 455         sr->instr.irh.s.uddl = MIN_UDD_LEN;
 456         /* context length in 64-bit words */
 457         sr->instr.irh.s.ctxl = (req->ctrl.s.ctxl / 8);
 458         /* offset from solicit base port 256 */
 459         sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno;
 460         sr->instr.irh.s.ctxc = req->ctrl.s.ctxc;
 461         sr->instr.irh.s.arg = req->ctrl.s.arg;
 462         sr->instr.irh.s.opcode = req->opcode;
 463         sr->instr.irh.value[0] = cpu_to_be64(sr->instr.irh.value[0]);
 464
 465         /* word 3 */
 466         sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle);
 467
 468         /* word 4 */
 469         sr->instr.slc.value[0] = 0;
 470         sr->instr.slc.s.ssz = sr->out.sgmap_cnt;
 471         sr->instr.slc.value[0] = cpu_to_be64(sr->instr.slc.value[0]);
 472
 473         /* word 5 */
 474         sr->instr.slc.s.rptr = cpu_to_be64(sr->out.sgcomp_dma);
 475
 476         /*
 477          * No conversion for front data,
 478          * It goes into payload
 479          * put GP Header in front data
 480          */
 481         sr->instr.fdata[0] = *((u64 *)&req->gph);
 482         sr->instr.fdata[1] = 0;
 483
 484         ret = nitrox_enqueue_request(sr);
 485         if (ret == -ENOSPC)
 486                 goto send_fail;
 487
 488         return ret;
 489
 490 send_fail:
 491         softreq_destroy(sr);
 492         return ret;
 493 }
 494
 495 static inline int cmd_timeout(unsigned long tstamp, unsigned long timeout)
 496 {
 497         return time_after_eq(jiffies, (tstamp + timeout));
 498 }
 499
 500 void backlog_qflush_work(struct work_struct *work)
 501 {
 502         struct nitrox_cmdq *cmdq;
 503
 504         cmdq = container_of(work, struct nitrox_cmdq, backlog_qflush);
 505         post_backlog_cmds(cmdq);
 506 }
 507
 508 static bool sr_completed(struct nitrox_softreq *sr)
 509 {
 510         u64 orh = READ_ONCE(*sr->resp.orh);
 511         unsigned long timeout = jiffies + msecs_to_jiffies(1);
 512
 513         if ((orh != PENDING_SIG) && (orh & 0xff))
 514                 return true;
 515
 516         while (READ_ONCE(*sr->resp.completion) == PENDING_SIG) {
 517                 if (time_after(jiffies, timeout)) {
 518                         pr_err("comp not done\n");
 519                         return false;
 520                 }
 521         }
 522
 523         return true;
 524 }
 525
 526 /**
 527  * process_request_list - process completed requests
 528  * @ndev: N5 device
 529  * @qno: queue to operate
 530  *
 531  * Returns the number of responses processed.
 532  */
 533 static void process_response_list(struct nitrox_cmdq *cmdq)
 534 {
 535         struct nitrox_device *ndev = cmdq->ndev;
 536         struct nitrox_softreq *sr;
 537         int req_completed = 0, err = 0, budget;
 538         completion_t callback;
 539         void *cb_arg;
 540
 541         /* check all pending requests */
 542         budget = atomic_read(&cmdq->pending_count);
 543
 544         while (req_completed < budget) {
 545                 sr = get_first_response_entry(cmdq);
 546                 if (!sr)
 547                         break;
 548
 549                 if (atomic_read(&sr->status) != REQ_POSTED)
 550                         break;
 551
 552                 /* check orh and completion bytes updates */
 553                 if (!sr_completed(sr)) {
 554                         /* request not completed, check for timeout */
 555                         if (!cmd_timeout(sr->tstamp, ndev->timeout))
 556                                 break;
 557                         dev_err_ratelimited(DEV(ndev),
 558                                             "Request timeout, orh 0x%016llx\n",
 559                                             READ_ONCE(*sr->resp.orh));
 560                 }
 561                 atomic_dec(&cmdq->pending_count);
 562                 atomic64_inc(&ndev->stats.completed);
 563                 /* sync with other cpus */
 564                 smp_mb__after_atomic();
 565                 /* remove from response list */
 566                 response_list_del(sr, cmdq);
 567                 /* ORH error code */
 568                 err = READ_ONCE(*sr->resp.orh) & 0xff;
 569                 callback = sr->callback;
 570                 cb_arg = sr->cb_arg;
 571                 softreq_destroy(sr);
 572                 if (callback)
 573                         callback(cb_arg, err);
 574
 575                 req_completed++;
 576         }
 577 }
 578
 579 /**
 580  * pkt_slc_resp_tasklet - post processing of SE responses
 581  */
 582 void pkt_slc_resp_tasklet(unsigned long data)
 583 {
 584         struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data);
 585         struct nitrox_cmdq *cmdq = qvec->cmdq;
 586         union nps_pkt_slc_cnts slc_cnts;
 587
 588         /* read completion count */
 589         slc_cnts.value = readq(cmdq->compl_cnt_csr_addr);
 590         /* resend the interrupt if more work to do */
 591         slc_cnts.s.resend = 1;
 592
 593         process_response_list(cmdq);
 594
 595         /*
 596          * clear the interrupt with resend bit enabled,
 597          * MSI-X interrupt generates if Completion count > Threshold
 598          */
 599         writeq(slc_cnts.value, cmdq->compl_cnt_csr_addr);
 600
 601         if (atomic_read(&cmdq->backlog_count))
 602                 schedule_work(&cmdq->backlog_qflush);
 603 }