drivers/crypto/caam/qi.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * CAAM/SEC 4.x QI transport/backend driver
   4  * Queue Interface backend functionality
   5  *
   6  * Copyright 2013-2016 Freescale Semiconductor, Inc.
   7  * Copyright 2016-2017 NXP
   8  */
   9
  10 #include <linux/cpumask.h>
  11 #include <linux/kthread.h>
  12 #include <soc/fsl/qman.h>
  13
  14 #include "regs.h"
  15 #include "qi.h"
  16 #include "desc.h"
  17 #include "intern.h"
  18 #include "desc_constr.h"
  19
  20 #define PREHDR_RSLS_SHIFT       31
  21
  22 /*
  23  * Use a reasonable backlog of frames (per CPU) as congestion threshold,
  24  * so that resources used by the in-flight buffers do not become a memory hog.
  25  */
  26 #define MAX_RSP_FQ_BACKLOG_PER_CPU      256
  27
  28 #define CAAM_QI_ENQUEUE_RETRIES 10000
  29
  30 #define CAAM_NAPI_WEIGHT        63
  31
  32 /*
  33  * caam_napi - struct holding CAAM NAPI-related params
  34  * @irqtask: IRQ task for QI backend
  35  * @p: QMan portal
  36  */
  37 struct caam_napi {
  38         struct napi_struct irqtask;
  39         struct qman_portal *p;
  40 };
  41
  42 /*
  43  * caam_qi_pcpu_priv - percpu private data structure to main list of pending
  44  *                     responses expected on each cpu.
  45  * @caam_napi: CAAM NAPI params
  46  * @net_dev: netdev used by NAPI
  47  * @rsp_fq: response FQ from CAAM
  48  */
  49 struct caam_qi_pcpu_priv {
  50         struct caam_napi caam_napi;
  51         struct net_device net_dev;
  52         struct qman_fq *rsp_fq;
  53 } ____cacheline_aligned;
  54
  55 static DEFINE_PER_CPU(struct caam_qi_pcpu_priv, pcpu_qipriv);
  56 static DEFINE_PER_CPU(int, last_cpu);
  57
  58 /*
  59  * caam_qi_priv - CAAM QI backend private params
  60  * @cgr: QMan congestion group
  61  * @qi_pdev: platform device for QI backend
  62  */
  63 struct caam_qi_priv {
  64         struct qman_cgr cgr;
  65         struct platform_device *qi_pdev;
  66 };
  67
  68 static struct caam_qi_priv qipriv ____cacheline_aligned;
  69
  70 /*
  71  * This is written by only one core - the one that initialized the CGR - and
  72  * read by multiple cores (all the others).
  73  */
  74 bool caam_congested __read_mostly;
  75 EXPORT_SYMBOL(caam_congested);
  76
  77 #ifdef CONFIG_DEBUG_FS
  78 /*
  79  * This is a counter for the number of times the congestion group (where all
  80  * the request and response queueus are) reached congestion. Incremented
  81  * each time the congestion callback is called with congested == true.
  82  */
  83 static u64 times_congested;
  84 #endif
  85
  86 /*
  87  * CPU from where the module initialised. This is required because QMan driver
  88  * requires CGRs to be removed from same CPU from where they were originally
  89  * allocated.
  90  */
  91 static int mod_init_cpu;
  92
  93 /*
  94  * This is a a cache of buffers, from which the users of CAAM QI driver
  95  * can allocate short (CAAM_QI_MEMCACHE_SIZE) buffers. It's faster than
  96  * doing malloc on the hotpath.
  97  * NOTE: A more elegant solution would be to have some headroom in the frames
  98  *       being processed. This could be added by the dpaa-ethernet driver.
  99  *       This would pose a problem for userspace application processing which
 100  *       cannot know of this limitation. So for now, this will work.
 101  * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here
 102  */
 103 static struct kmem_cache *qi_cache;
 104
 105 int caam_qi_enqueue(struct device *qidev, struct caam_drv_req *req)
 106 {
 107         struct qm_fd fd;
 108         dma_addr_t addr;
 109         int ret;
 110         int num_retries = 0;
 111
 112         qm_fd_clear_fd(&fd);
 113         qm_fd_set_compound(&fd, qm_sg_entry_get_len(&req->fd_sgt[1]));
 114
 115         addr = dma_map_single(qidev, req->fd_sgt, sizeof(req->fd_sgt),
 116                               DMA_BIDIRECTIONAL);
 117         if (dma_mapping_error(qidev, addr)) {
 118                 dev_err(qidev, "DMA mapping error for QI enqueue request\n");
 119                 return -EIO;
 120         }
 121         qm_fd_addr_set64(&fd, addr);
 122
 123         do {
 124                 ret = qman_enqueue(req->drv_ctx->req_fq, &fd);
 125                 if (likely(!ret))
 126                         return 0;
 127
 128                 if (ret != -EBUSY)
 129                         break;
 130                 num_retries++;
 131         } while (num_retries < CAAM_QI_ENQUEUE_RETRIES);
 132
 133         dev_err(qidev, "qman_enqueue failed: %d\n", ret);
 134
 135         return ret;
 136 }
 137 EXPORT_SYMBOL(caam_qi_enqueue);
 138
 139 static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq,
 140                            const union qm_mr_entry *msg)
 141 {
 142         const struct qm_fd *fd;
 143         struct caam_drv_req *drv_req;
 144         struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev);
 145
 146         fd = &msg->ern.fd;
 147
 148         if (qm_fd_get_format(fd) != qm_fd_compound) {
 149                 dev_err(qidev, "Non-compound FD from CAAM\n");
 150                 return;
 151         }
 152
 153         drv_req = (struct caam_drv_req *)phys_to_virt(qm_fd_addr_get64(fd));
 154         if (!drv_req) {
 155                 dev_err(qidev,
 156                         "Can't find original request for CAAM response\n");
 157                 return;
 158         }
 159
 160         dma_unmap_single(drv_req->drv_ctx->qidev, qm_fd_addr(fd),
 161                          sizeof(drv_req->fd_sgt), DMA_BIDIRECTIONAL);
 162
 163         drv_req->cbk(drv_req, -EIO);
 164 }
 165
 166 static struct qman_fq *create_caam_req_fq(struct device *qidev,
 167                                           struct qman_fq *rsp_fq,
 168                                           dma_addr_t hwdesc,
 169                                           int fq_sched_flag)
 170 {
 171         int ret;
 172         struct qman_fq *req_fq;
 173         struct qm_mcc_initfq opts;
 174
 175         req_fq = kzalloc(sizeof(*req_fq), GFP_ATOMIC);
 176         if (!req_fq)
 177                 return ERR_PTR(-ENOMEM);
 178
 179         req_fq->cb.ern = caam_fq_ern_cb;
 180         req_fq->cb.fqs = NULL;
 181
 182         ret = qman_create_fq(0, QMAN_FQ_FLAG_DYNAMIC_FQID |
 183                                 QMAN_FQ_FLAG_TO_DCPORTAL, req_fq);
 184         if (ret) {
 185                 dev_err(qidev, "Failed to create session req FQ\n");
 186                 goto create_req_fq_fail;
 187         }
 188
 189         memset(&opts, 0, sizeof(opts));
 190         opts.we_mask = cpu_to_be16(QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_DESTWQ |
 191                                    QM_INITFQ_WE_CONTEXTB |
 192                                    QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CGID);
 193         opts.fqd.fq_ctrl = cpu_to_be16(QM_FQCTRL_CPCSTASH | QM_FQCTRL_CGE);
 194         qm_fqd_set_destwq(&opts.fqd, qm_channel_caam, 2);
 195         opts.fqd.context_b = cpu_to_be32(qman_fq_fqid(rsp_fq));
 196         qm_fqd_context_a_set64(&opts.fqd, hwdesc);
 197         opts.fqd.cgid = qipriv.cgr.cgrid;
 198
 199         ret = qman_init_fq(req_fq, fq_sched_flag, &opts);
 200         if (ret) {
 201                 dev_err(qidev, "Failed to init session req FQ\n");
 202                 goto init_req_fq_fail;
 203         }
 204
 205         dev_dbg(qidev, "Allocated request FQ %u for CPU %u\n", req_fq->fqid,
 206                 smp_processor_id());
 207         return req_fq;
 208
 209 init_req_fq_fail:
 210         qman_destroy_fq(req_fq);
 211 create_req_fq_fail:
 212         kfree(req_fq);
 213         return ERR_PTR(ret);
 214 }
 215
 216 static int empty_retired_fq(struct device *qidev, struct qman_fq *fq)
 217 {
 218         int ret;
 219
 220         ret = qman_volatile_dequeue(fq, QMAN_VOLATILE_FLAG_WAIT_INT |
 221                                     QMAN_VOLATILE_FLAG_FINISH,
 222                                     QM_VDQCR_PRECEDENCE_VDQCR |
 223                                     QM_VDQCR_NUMFRAMES_TILLEMPTY);
 224         if (ret) {
 225                 dev_err(qidev, "Volatile dequeue fail for FQ: %u\n", fq->fqid);
 226                 return ret;
 227         }
 228
 229         do {
 230                 struct qman_portal *p;
 231
 232                 p = qman_get_affine_portal(smp_processor_id());
 233                 qman_p_poll_dqrr(p, 16);
 234         } while (fq->flags & QMAN_FQ_STATE_NE);
 235
 236         return 0;
 237 }
 238
 239 static int kill_fq(struct device *qidev, struct qman_fq *fq)
 240 {
 241         u32 flags;
 242         int ret;
 243
 244         ret = qman_retire_fq(fq, &flags);
 245         if (ret < 0) {
 246                 dev_err(qidev, "qman_retire_fq failed: %d\n", ret);
 247                 return ret;
 248         }
 249
 250         if (!ret)
 251                 goto empty_fq;
 252
 253         /* Async FQ retirement condition */
 254         if (ret == 1) {
 255                 /* Retry till FQ gets in retired state */
 256                 do {
 257                         msleep(20);
 258                 } while (fq->state != qman_fq_state_retired);
 259
 260                 WARN_ON(fq->flags & QMAN_FQ_STATE_BLOCKOOS);
 261                 WARN_ON(fq->flags & QMAN_FQ_STATE_ORL);
 262         }
 263
 264 empty_fq:
 265         if (fq->flags & QMAN_FQ_STATE_NE) {
 266                 ret = empty_retired_fq(qidev, fq);
 267                 if (ret) {
 268                         dev_err(qidev, "empty_retired_fq fail for FQ: %u\n",
 269                                 fq->fqid);
 270                         return ret;
 271                 }
 272         }
 273
 274         ret = qman_oos_fq(fq);
 275         if (ret)
 276                 dev_err(qidev, "OOS of FQID: %u failed\n", fq->fqid);
 277
 278         qman_destroy_fq(fq);
 279         kfree(fq);
 280
 281         return ret;
 282 }
 283
 284 static int empty_caam_fq(struct qman_fq *fq)
 285 {
 286         int ret;
 287         struct qm_mcr_queryfq_np np;
 288
 289         /* Wait till the older CAAM FQ get empty */
 290         do {
 291                 ret = qman_query_fq_np(fq, &np);
 292                 if (ret)
 293                         return ret;
 294
 295                 if (!qm_mcr_np_get(&np, frm_cnt))
 296                         break;
 297
 298                 msleep(20);
 299         } while (1);
 300
 301         /*
 302          * Give extra time for pending jobs from this FQ in holding tanks
 303          * to get processed
 304          */
 305         msleep(20);
 306         return 0;
 307 }
 308
 309 int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc)
 310 {
 311         int ret;
 312         u32 num_words;
 313         struct qman_fq *new_fq, *old_fq;
 314         struct device *qidev = drv_ctx->qidev;
 315
 316         num_words = desc_len(sh_desc);
 317         if (num_words > MAX_SDLEN) {
 318                 dev_err(qidev, "Invalid descriptor len: %d words\n", num_words);
 319                 return -EINVAL;
 320         }
 321
 322         /* Note down older req FQ */
 323         old_fq = drv_ctx->req_fq;
 324
 325         /* Create a new req FQ in parked state */
 326         new_fq = create_caam_req_fq(drv_ctx->qidev, drv_ctx->rsp_fq,
 327                                     drv_ctx->context_a, 0);
 328         if (unlikely(IS_ERR_OR_NULL(new_fq))) {
 329                 dev_err(qidev, "FQ allocation for shdesc update failed\n");
 330                 return PTR_ERR(new_fq);
 331         }
 332
 333         /* Hook up new FQ to context so that new requests keep queuing */
 334         drv_ctx->req_fq = new_fq;
 335
 336         /* Empty and remove the older FQ */
 337         ret = empty_caam_fq(old_fq);
 338         if (ret) {
 339                 dev_err(qidev, "Old CAAM FQ empty failed: %d\n", ret);
 340
 341                 /* We can revert to older FQ */
 342                 drv_ctx->req_fq = old_fq;
 343
 344                 if (kill_fq(qidev, new_fq))
 345                         dev_warn(qidev, "New CAAM FQ kill failed\n");
 346
 347                 return ret;
 348         }
 349
 350         /*
 351          * Re-initialise pre-header. Set RSLS and SDLEN.
 352          * Update the shared descriptor for driver context.
 353          */
 354         drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) |
 355                                            num_words);
 356         memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc));
 357         dma_sync_single_for_device(qidev, drv_ctx->context_a,
 358                                    sizeof(drv_ctx->sh_desc) +
 359                                    sizeof(drv_ctx->prehdr),
 360                                    DMA_BIDIRECTIONAL);
 361
 362         /* Put the new FQ in scheduled state */
 363         ret = qman_schedule_fq(new_fq);
 364         if (ret) {
 365                 dev_err(qidev, "Fail to sched new CAAM FQ, ecode = %d\n", ret);
 366
 367                 /*
 368                  * We can kill new FQ and revert to old FQ.
 369                  * Since the desc is already modified, it is success case
 370                  */
 371
 372                 drv_ctx->req_fq = old_fq;
 373
 374                 if (kill_fq(qidev, new_fq))
 375                         dev_warn(qidev, "New CAAM FQ kill failed\n");
 376         } else if (kill_fq(qidev, old_fq)) {
 377                 dev_warn(qidev, "Old CAAM FQ kill failed\n");
 378         }
 379
 380         return 0;
 381 }
 382 EXPORT_SYMBOL(caam_drv_ctx_update);
 383
 384 struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev,
 385                                        int *cpu,
 386                                        u32 *sh_desc)
 387 {
 388         size_t size;
 389         u32 num_words;
 390         dma_addr_t hwdesc;
 391         struct caam_drv_ctx *drv_ctx;
 392         const cpumask_t *cpus = qman_affine_cpus();
 393
 394         num_words = desc_len(sh_desc);
 395         if (num_words > MAX_SDLEN) {
 396                 dev_err(qidev, "Invalid descriptor len: %d words\n",
 397                         num_words);
 398                 return ERR_PTR(-EINVAL);
 399         }
 400
 401         drv_ctx = kzalloc(sizeof(*drv_ctx), GFP_ATOMIC);
 402         if (!drv_ctx)
 403                 return ERR_PTR(-ENOMEM);
 404
 405         /*
 406          * Initialise pre-header - set RSLS and SDLEN - and shared descriptor
 407          * and dma-map them.
 408          */
 409         drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) |
 410                                            num_words);
 411         memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc));
 412         size = sizeof(drv_ctx->prehdr) + sizeof(drv_ctx->sh_desc);
 413         hwdesc = dma_map_single(qidev, drv_ctx->prehdr, size,
 414                                 DMA_BIDIRECTIONAL);
 415         if (dma_mapping_error(qidev, hwdesc)) {
 416                 dev_err(qidev, "DMA map error for preheader + shdesc\n");
 417                 kfree(drv_ctx);
 418                 return ERR_PTR(-ENOMEM);
 419         }
 420         drv_ctx->context_a = hwdesc;
 421
 422         /* If given CPU does not own the portal, choose another one that does */
 423         if (!cpumask_test_cpu(*cpu, cpus)) {
 424                 int *pcpu = &get_cpu_var(last_cpu);
 425
 426                 *pcpu = cpumask_next(*pcpu, cpus);
 427                 if (*pcpu >= nr_cpu_ids)
 428                         *pcpu = cpumask_first(cpus);
 429                 *cpu = *pcpu;
 430
 431                 put_cpu_var(last_cpu);
 432         }
 433         drv_ctx->cpu = *cpu;
 434
 435         /* Find response FQ hooked with this CPU */
 436         drv_ctx->rsp_fq = per_cpu(pcpu_qipriv.rsp_fq, drv_ctx->cpu);
 437
 438         /* Attach request FQ */
 439         drv_ctx->req_fq = create_caam_req_fq(qidev, drv_ctx->rsp_fq, hwdesc,
 440                                              QMAN_INITFQ_FLAG_SCHED);
 441         if (unlikely(IS_ERR_OR_NULL(drv_ctx->req_fq))) {
 442                 dev_err(qidev, "create_caam_req_fq failed\n");
 443                 dma_unmap_single(qidev, hwdesc, size, DMA_BIDIRECTIONAL);
 444                 kfree(drv_ctx);
 445                 return ERR_PTR(-ENOMEM);
 446         }
 447
 448         drv_ctx->qidev = qidev;
 449         return drv_ctx;
 450 }
 451 EXPORT_SYMBOL(caam_drv_ctx_init);
 452
 453 void *qi_cache_alloc(gfp_t flags)
 454 {
 455         return kmem_cache_alloc(qi_cache, flags);
 456 }
 457 EXPORT_SYMBOL(qi_cache_alloc);
 458
 459 void qi_cache_free(void *obj)
 460 {
 461         kmem_cache_free(qi_cache, obj);
 462 }
 463 EXPORT_SYMBOL(qi_cache_free);
 464
 465 static int caam_qi_poll(struct napi_struct *napi, int budget)
 466 {
 467         struct caam_napi *np = container_of(napi, struct caam_napi, irqtask);
 468
 469         int cleaned = qman_p_poll_dqrr(np->p, budget);
 470
 471         if (cleaned < budget) {
 472                 napi_complete(napi);
 473                 qman_p_irqsource_add(np->p, QM_PIRQ_DQRI);
 474         }
 475
 476         return cleaned;
 477 }
 478
 479 void caam_drv_ctx_rel(struct caam_drv_ctx *drv_ctx)
 480 {
 481         if (IS_ERR_OR_NULL(drv_ctx))
 482                 return;
 483
 484         /* Remove request FQ */
 485         if (kill_fq(drv_ctx->qidev, drv_ctx->req_fq))
 486                 dev_err(drv_ctx->qidev, "Crypto session req FQ kill failed\n");
 487
 488         dma_unmap_single(drv_ctx->qidev, drv_ctx->context_a,
 489                          sizeof(drv_ctx->sh_desc) + sizeof(drv_ctx->prehdr),
 490                          DMA_BIDIRECTIONAL);
 491         kfree(drv_ctx);
 492 }
 493 EXPORT_SYMBOL(caam_drv_ctx_rel);
 494
 495 int caam_qi_shutdown(struct device *qidev)
 496 {
 497         int i, ret;
 498         struct caam_qi_priv *priv = dev_get_drvdata(qidev);
 499         const cpumask_t *cpus = qman_affine_cpus();
 500         struct cpumask old_cpumask = current->cpus_allowed;
 501
 502         for_each_cpu(i, cpus) {
 503                 struct napi_struct *irqtask;
 504
 505                 irqtask = &per_cpu_ptr(&pcpu_qipriv.caam_napi, i)->irqtask;
 506                 napi_disable(irqtask);
 507                 netif_napi_del(irqtask);
 508
 509                 if (kill_fq(qidev, per_cpu(pcpu_qipriv.rsp_fq, i)))
 510                         dev_err(qidev, "Rsp FQ kill failed, cpu: %d\n", i);
 511         }
 512
 513         /*
 514          * QMan driver requires CGRs to be deleted from same CPU from where they
 515          * were instantiated. Hence we get the module removal execute from the
 516          * same CPU from where it was originally inserted.
 517          */
 518         set_cpus_allowed_ptr(current, get_cpu_mask(mod_init_cpu));
 519
 520         ret = qman_delete_cgr(&priv->cgr);
 521         if (ret)
 522                 dev_err(qidev, "Deletion of CGR failed: %d\n", ret);
 523         else
 524                 qman_release_cgrid(priv->cgr.cgrid);
 525
 526         kmem_cache_destroy(qi_cache);
 527
 528         /* Now that we're done with the CGRs, restore the cpus allowed mask */
 529         set_cpus_allowed_ptr(current, &old_cpumask);
 530
 531         platform_device_unregister(priv->qi_pdev);
 532         return ret;
 533 }
 534
 535 static void cgr_cb(struct qman_portal *qm, struct qman_cgr *cgr, int congested)
 536 {
 537         caam_congested = congested;
 538
 539         if (congested) {
 540 #ifdef CONFIG_DEBUG_FS
 541                 times_congested++;
 542 #endif
 543                 pr_debug_ratelimited("CAAM entered congestion\n");
 544
 545         } else {
 546                 pr_debug_ratelimited("CAAM exited congestion\n");
 547         }
 548 }
 549
 550 static int caam_qi_napi_schedule(struct qman_portal *p, struct caam_napi *np)
 551 {
 552         /*
 553          * In case of threaded ISR, for RT kernels in_irq() does not return
 554          * appropriate value, so use in_serving_softirq to distinguish between
 555          * softirq and irq contexts.
 556          */
 557         if (unlikely(in_irq() || !in_serving_softirq())) {
 558                 /* Disable QMan IRQ source and invoke NAPI */
 559                 qman_p_irqsource_remove(p, QM_PIRQ_DQRI);
 560                 np->p = p;
 561                 napi_schedule(&np->irqtask);
 562                 return 1;
 563         }
 564         return 0;
 565 }
 566
 567 static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p,
 568                                                     struct qman_fq *rsp_fq,
 569                                                     const struct qm_dqrr_entry *dqrr)
 570 {
 571         struct caam_napi *caam_napi = raw_cpu_ptr(&pcpu_qipriv.caam_napi);
 572         struct caam_drv_req *drv_req;
 573         const struct qm_fd *fd;
 574         struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev);
 575         u32 status;
 576
 577         if (caam_qi_napi_schedule(p, caam_napi))
 578                 return qman_cb_dqrr_stop;
 579
 580         fd = &dqrr->fd;
 581         status = be32_to_cpu(fd->status);
 582         if (unlikely(status))
 583                 dev_err(qidev, "Error: %#x in CAAM response FD\n", status);
 584
 585         if (unlikely(qm_fd_get_format(fd) != qm_fd_compound)) {
 586                 dev_err(qidev, "Non-compound FD from CAAM\n");
 587                 return qman_cb_dqrr_consume;
 588         }
 589
 590         drv_req = (struct caam_drv_req *)phys_to_virt(qm_fd_addr_get64(fd));
 591         if (unlikely(!drv_req)) {
 592                 dev_err(qidev,
 593                         "Can't find original request for caam response\n");
 594                 return qman_cb_dqrr_consume;
 595         }
 596
 597         dma_unmap_single(drv_req->drv_ctx->qidev, qm_fd_addr(fd),
 598                          sizeof(drv_req->fd_sgt), DMA_BIDIRECTIONAL);
 599
 600         drv_req->cbk(drv_req, status);
 601         return qman_cb_dqrr_consume;
 602 }
 603
 604 static int alloc_rsp_fq_cpu(struct device *qidev, unsigned int cpu)
 605 {
 606         struct qm_mcc_initfq opts;
 607         struct qman_fq *fq;
 608         int ret;
 609
 610         fq = kzalloc(sizeof(*fq), GFP_KERNEL | GFP_DMA);
 611         if (!fq)
 612                 return -ENOMEM;
 613
 614         fq->cb.dqrr = caam_rsp_fq_dqrr_cb;
 615
 616         ret = qman_create_fq(0, QMAN_FQ_FLAG_NO_ENQUEUE |
 617                              QMAN_FQ_FLAG_DYNAMIC_FQID, fq);
 618         if (ret) {
 619                 dev_err(qidev, "Rsp FQ create failed\n");
 620                 kfree(fq);
 621                 return -ENODEV;
 622         }
 623
 624         memset(&opts, 0, sizeof(opts));
 625         opts.we_mask = cpu_to_be16(QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_DESTWQ |
 626                                    QM_INITFQ_WE_CONTEXTB |
 627                                    QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CGID);
 628         opts.fqd.fq_ctrl = cpu_to_be16(QM_FQCTRL_CTXASTASHING |
 629                                        QM_FQCTRL_CPCSTASH | QM_FQCTRL_CGE);
 630         qm_fqd_set_destwq(&opts.fqd, qman_affine_channel(cpu), 3);
 631         opts.fqd.cgid = qipriv.cgr.cgrid;
 632         opts.fqd.context_a.stashing.exclusive = QM_STASHING_EXCL_CTX |
 633                                                 QM_STASHING_EXCL_DATA;
 634         qm_fqd_set_stashing(&opts.fqd, 0, 1, 1);
 635
 636         ret = qman_init_fq(fq, QMAN_INITFQ_FLAG_SCHED, &opts);
 637         if (ret) {
 638                 dev_err(qidev, "Rsp FQ init failed\n");
 639                 kfree(fq);
 640                 return -ENODEV;
 641         }
 642
 643         per_cpu(pcpu_qipriv.rsp_fq, cpu) = fq;
 644
 645         dev_dbg(qidev, "Allocated response FQ %u for CPU %u", fq->fqid, cpu);
 646         return 0;
 647 }
 648
 649 static int init_cgr(struct device *qidev)
 650 {
 651         int ret;
 652         struct qm_mcc_initcgr opts;
 653         const u64 cpus = *(u64 *)qman_affine_cpus();
 654         const int num_cpus = hweight64(cpus);
 655         const u64 val = num_cpus * MAX_RSP_FQ_BACKLOG_PER_CPU;
 656
 657         ret = qman_alloc_cgrid(&qipriv.cgr.cgrid);
 658         if (ret) {
 659                 dev_err(qidev, "CGR alloc failed for rsp FQs: %d\n", ret);
 660                 return ret;
 661         }
 662
 663         qipriv.cgr.cb = cgr_cb;
 664         memset(&opts, 0, sizeof(opts));
 665         opts.we_mask = cpu_to_be16(QM_CGR_WE_CSCN_EN | QM_CGR_WE_CS_THRES |
 666                                    QM_CGR_WE_MODE);
 667         opts.cgr.cscn_en = QM_CGR_EN;
 668         opts.cgr.mode = QMAN_CGR_MODE_FRAME;
 669         qm_cgr_cs_thres_set64(&opts.cgr.cs_thres, val, 1);
 670
 671         ret = qman_create_cgr(&qipriv.cgr, QMAN_CGR_FLAG_USE_INIT, &opts);
 672         if (ret) {
 673                 dev_err(qidev, "Error %d creating CAAM CGRID: %u\n", ret,
 674                         qipriv.cgr.cgrid);
 675                 return ret;
 676         }
 677
 678         dev_dbg(qidev, "Congestion threshold set to %llu\n", val);
 679         return 0;
 680 }
 681
 682 static int alloc_rsp_fqs(struct device *qidev)
 683 {
 684         int ret, i;
 685         const cpumask_t *cpus = qman_affine_cpus();
 686
 687         /*Now create response FQs*/
 688         for_each_cpu(i, cpus) {
 689                 ret = alloc_rsp_fq_cpu(qidev, i);
 690                 if (ret) {
 691                         dev_err(qidev, "CAAM rsp FQ alloc failed, cpu: %u", i);
 692                         return ret;
 693                 }
 694         }
 695
 696         return 0;
 697 }
 698
 699 static void free_rsp_fqs(void)
 700 {
 701         int i;
 702         const cpumask_t *cpus = qman_affine_cpus();
 703
 704         for_each_cpu(i, cpus)
 705                 kfree(per_cpu(pcpu_qipriv.rsp_fq, i));
 706 }
 707
 708 int caam_qi_init(struct platform_device *caam_pdev)
 709 {
 710         int err, i;
 711         struct platform_device *qi_pdev;
 712         struct device *ctrldev = &caam_pdev->dev, *qidev;
 713         struct caam_drv_private *ctrlpriv;
 714         const cpumask_t *cpus = qman_affine_cpus();
 715         struct cpumask old_cpumask = current->cpus_allowed;
 716         static struct platform_device_info qi_pdev_info = {
 717                 .name = "caam_qi",
 718                 .id = PLATFORM_DEVID_NONE
 719         };
 720
 721         /*
 722          * QMAN requires CGRs to be removed from same CPU+portal from where it
 723          * was originally allocated. Hence we need to note down the
 724          * initialisation CPU and use the same CPU for module exit.
 725          * We select the first CPU to from the list of portal owning CPUs.
 726          * Then we pin module init to this CPU.
 727          */
 728         mod_init_cpu = cpumask_first(cpus);
 729         set_cpus_allowed_ptr(current, get_cpu_mask(mod_init_cpu));
 730
 731         qi_pdev_info.parent = ctrldev;
 732         qi_pdev_info.dma_mask = dma_get_mask(ctrldev);
 733         qi_pdev = platform_device_register_full(&qi_pdev_info);
 734         if (IS_ERR(qi_pdev))
 735                 return PTR_ERR(qi_pdev);
 736         set_dma_ops(&qi_pdev->dev, get_dma_ops(ctrldev));
 737
 738         ctrlpriv = dev_get_drvdata(ctrldev);
 739         qidev = &qi_pdev->dev;
 740
 741         qipriv.qi_pdev = qi_pdev;
 742         dev_set_drvdata(qidev, &qipriv);
 743
 744         /* Initialize the congestion detection */
 745         err = init_cgr(qidev);
 746         if (err) {
 747                 dev_err(qidev, "CGR initialization failed: %d\n", err);
 748                 platform_device_unregister(qi_pdev);
 749                 return err;
 750         }
 751
 752         /* Initialise response FQs */
 753         err = alloc_rsp_fqs(qidev);
 754         if (err) {
 755                 dev_err(qidev, "Can't allocate CAAM response FQs: %d\n", err);
 756                 free_rsp_fqs();
 757                 platform_device_unregister(qi_pdev);
 758                 return err;
 759         }
 760
 761         /*
 762          * Enable the NAPI contexts on each of the core which has an affine
 763          * portal.
 764          */
 765         for_each_cpu(i, cpus) {
 766                 struct caam_qi_pcpu_priv *priv = per_cpu_ptr(&pcpu_qipriv, i);
 767                 struct caam_napi *caam_napi = &priv->caam_napi;
 768                 struct napi_struct *irqtask = &caam_napi->irqtask;
 769                 struct net_device *net_dev = &priv->net_dev;
 770
 771                 net_dev->dev = *qidev;
 772                 INIT_LIST_HEAD(&net_dev->napi_list);
 773
 774                 netif_napi_add(net_dev, irqtask, caam_qi_poll,
 775                                CAAM_NAPI_WEIGHT);
 776
 777                 napi_enable(irqtask);
 778         }
 779
 780         /* Hook up QI device to parent controlling caam device */
 781         ctrlpriv->qidev = qidev;
 782
 783         qi_cache = kmem_cache_create("caamqicache", CAAM_QI_MEMCACHE_SIZE, 0,
 784                                      SLAB_CACHE_DMA, NULL);
 785         if (!qi_cache) {
 786                 dev_err(qidev, "Can't allocate CAAM cache\n");
 787                 free_rsp_fqs();
 788                 platform_device_unregister(qi_pdev);
 789                 return -ENOMEM;
 790         }
 791
 792         /* Done with the CGRs; restore the cpus allowed mask */
 793         set_cpus_allowed_ptr(current, &old_cpumask);
 794 #ifdef CONFIG_DEBUG_FS
 795         debugfs_create_file("qi_congested", 0444, ctrlpriv->ctl,
 796                             &times_congested, &caam_fops_u64_ro);
 797 #endif
 798         dev_info(qidev, "Linux CAAM Queue I/F driver initialised\n");
 799         return 0;
 800 }