drivers/crypto/marvell/octeontx/otx_cptvf_main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Marvell OcteonTX CPT driver
   3  *
   4  * Copyright (C) 2019 Marvell International Ltd.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10
  11 #include <linux/interrupt.h>
  12 #include <linux/module.h>
  13 #include "otx_cptvf.h"
  14 #include "otx_cptvf_algs.h"
  15 #include "otx_cptvf_reqmgr.h"
  16
  17 #define DRV_NAME        "octeontx-cptvf"
  18 #define DRV_VERSION     "1.0"
  19
  20 static void vq_work_handler(unsigned long data)
  21 {
  22         struct otx_cptvf_wqe_info *cwqe_info =
  23                                         (struct otx_cptvf_wqe_info *) data;
  24
  25         otx_cpt_post_process(&cwqe_info->vq_wqe[0]);
  26 }
  27
  28 static int init_worker_threads(struct otx_cptvf *cptvf)
  29 {
  30         struct pci_dev *pdev = cptvf->pdev;
  31         struct otx_cptvf_wqe_info *cwqe_info;
  32         int i;
  33
  34         cwqe_info = kzalloc(sizeof(*cwqe_info), GFP_KERNEL);
  35         if (!cwqe_info)
  36                 return -ENOMEM;
  37
  38         if (cptvf->num_queues) {
  39                 dev_dbg(&pdev->dev, "Creating VQ worker threads (%d)\n",
  40                         cptvf->num_queues);
  41         }
  42
  43         for (i = 0; i < cptvf->num_queues; i++) {
  44                 tasklet_init(&cwqe_info->vq_wqe[i].twork, vq_work_handler,
  45                              (u64)cwqe_info);
  46                 cwqe_info->vq_wqe[i].cptvf = cptvf;
  47         }
  48         cptvf->wqe_info = cwqe_info;
  49
  50         return 0;
  51 }
  52
  53 static void cleanup_worker_threads(struct otx_cptvf *cptvf)
  54 {
  55         struct pci_dev *pdev = cptvf->pdev;
  56         struct otx_cptvf_wqe_info *cwqe_info;
  57         int i;
  58
  59         cwqe_info = (struct otx_cptvf_wqe_info *)cptvf->wqe_info;
  60         if (!cwqe_info)
  61                 return;
  62
  63         if (cptvf->num_queues) {
  64                 dev_dbg(&pdev->dev, "Cleaning VQ worker threads (%u)\n",
  65                         cptvf->num_queues);
  66         }
  67
  68         for (i = 0; i < cptvf->num_queues; i++)
  69                 tasklet_kill(&cwqe_info->vq_wqe[i].twork);
  70
  71         kfree_sensitive(cwqe_info);
  72         cptvf->wqe_info = NULL;
  73 }
  74
  75 static void free_pending_queues(struct otx_cpt_pending_qinfo *pqinfo)
  76 {
  77         struct otx_cpt_pending_queue *queue;
  78         int i;
  79
  80         for_each_pending_queue(pqinfo, queue, i) {
  81                 if (!queue->head)
  82                         continue;
  83
  84                 /* free single queue */
  85                 kfree_sensitive((queue->head));
  86                 queue->front = 0;
  87                 queue->rear = 0;
  88                 queue->qlen = 0;
  89         }
  90         pqinfo->num_queues = 0;
  91 }
  92
  93 static int alloc_pending_queues(struct otx_cpt_pending_qinfo *pqinfo, u32 qlen,
  94                                 u32 num_queues)
  95 {
  96         struct otx_cpt_pending_queue *queue = NULL;
  97         int ret;
  98         u32 i;
  99
 100         pqinfo->num_queues = num_queues;
 101
 102         for_each_pending_queue(pqinfo, queue, i) {
 103                 queue->head = kcalloc(qlen, sizeof(*queue->head), GFP_KERNEL);
 104                 if (!queue->head) {
 105                         ret = -ENOMEM;
 106                         goto pending_qfail;
 107                 }
 108
 109                 queue->pending_count = 0;
 110                 queue->front = 0;
 111                 queue->rear = 0;
 112                 queue->qlen = qlen;
 113
 114                 /* init queue spin lock */
 115                 spin_lock_init(&queue->lock);
 116         }
 117         return 0;
 118
 119 pending_qfail:
 120         free_pending_queues(pqinfo);
 121
 122         return ret;
 123 }
 124
 125 static int init_pending_queues(struct otx_cptvf *cptvf, u32 qlen,
 126                                u32 num_queues)
 127 {
 128         struct pci_dev *pdev = cptvf->pdev;
 129         int ret;
 130
 131         if (!num_queues)
 132                 return 0;
 133
 134         ret = alloc_pending_queues(&cptvf->pqinfo, qlen, num_queues);
 135         if (ret) {
 136                 dev_err(&pdev->dev, "Failed to setup pending queues (%u)\n",
 137                         num_queues);
 138                 return ret;
 139         }
 140         return 0;
 141 }
 142
 143 static void cleanup_pending_queues(struct otx_cptvf *cptvf)
 144 {
 145         struct pci_dev *pdev = cptvf->pdev;
 146
 147         if (!cptvf->num_queues)
 148                 return;
 149
 150         dev_dbg(&pdev->dev, "Cleaning VQ pending queue (%u)\n",
 151                 cptvf->num_queues);
 152         free_pending_queues(&cptvf->pqinfo);
 153 }
 154
 155 static void free_command_queues(struct otx_cptvf *cptvf,
 156                                 struct otx_cpt_cmd_qinfo *cqinfo)
 157 {
 158         struct otx_cpt_cmd_queue *queue = NULL;
 159         struct otx_cpt_cmd_chunk *chunk = NULL;
 160         struct pci_dev *pdev = cptvf->pdev;
 161         int i;
 162
 163         /* clean up for each queue */
 164         for (i = 0; i < cptvf->num_queues; i++) {
 165                 queue = &cqinfo->queue[i];
 166
 167                 while (!list_empty(&cqinfo->queue[i].chead)) {
 168                         chunk = list_first_entry(&cqinfo->queue[i].chead,
 169                                         struct otx_cpt_cmd_chunk, nextchunk);
 170
 171                         dma_free_coherent(&pdev->dev, chunk->size,
 172                                           chunk->head,
 173                                           chunk->dma_addr);
 174                         chunk->head = NULL;
 175                         chunk->dma_addr = 0;
 176                         list_del(&chunk->nextchunk);
 177                         kfree_sensitive(chunk);
 178                 }
 179                 queue->num_chunks = 0;
 180                 queue->idx = 0;
 181
 182         }
 183 }
 184
 185 static int alloc_command_queues(struct otx_cptvf *cptvf,
 186                                 struct otx_cpt_cmd_qinfo *cqinfo,
 187                                 u32 qlen)
 188 {
 189         struct otx_cpt_cmd_chunk *curr, *first, *last;
 190         struct otx_cpt_cmd_queue *queue = NULL;
 191         struct pci_dev *pdev = cptvf->pdev;
 192         size_t q_size, c_size, rem_q_size;
 193         u32 qcsize_bytes;
 194         int i;
 195
 196
 197         /* Qsize in dwords, needed for SADDR config, 1-next chunk pointer */
 198         cptvf->qsize = min(qlen, cqinfo->qchunksize) *
 199                        OTX_CPT_NEXT_CHUNK_PTR_SIZE + 1;
 200         /* Qsize in bytes to create space for alignment */
 201         q_size = qlen * OTX_CPT_INST_SIZE;
 202
 203         qcsize_bytes = cqinfo->qchunksize * OTX_CPT_INST_SIZE;
 204
 205         /* per queue initialization */
 206         for (i = 0; i < cptvf->num_queues; i++) {
 207                 rem_q_size = q_size;
 208                 first = NULL;
 209                 last = NULL;
 210
 211                 queue = &cqinfo->queue[i];
 212                 INIT_LIST_HEAD(&queue->chead);
 213                 do {
 214                         curr = kzalloc(sizeof(*curr), GFP_KERNEL);
 215                         if (!curr)
 216                                 goto cmd_qfail;
 217
 218                         c_size = (rem_q_size > qcsize_bytes) ? qcsize_bytes :
 219                                         rem_q_size;
 220                         curr->head = dma_alloc_coherent(&pdev->dev,
 221                                            c_size + OTX_CPT_NEXT_CHUNK_PTR_SIZE,
 222                                            &curr->dma_addr, GFP_KERNEL);
 223                         if (!curr->head) {
 224                                 dev_err(&pdev->dev,
 225                                 "Command Q (%d) chunk (%d) allocation failed\n",
 226                                         i, queue->num_chunks);
 227                                 goto free_curr;
 228                         }
 229                         curr->size = c_size;
 230
 231                         if (queue->num_chunks == 0) {
 232                                 first = curr;
 233                                 queue->base  = first;
 234                         }
 235                         list_add_tail(&curr->nextchunk,
 236                                       &cqinfo->queue[i].chead);
 237
 238                         queue->num_chunks++;
 239                         rem_q_size -= c_size;
 240                         if (last)
 241                                 *((u64 *)(&last->head[last->size])) =
 242                                         (u64)curr->dma_addr;
 243
 244                         last = curr;
 245                 } while (rem_q_size);
 246
 247                 /*
 248                  * Make the queue circular, tie back last chunk entry to head
 249                  */
 250                 curr = first;
 251                 *((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr;
 252                 queue->qhead = curr;
 253         }
 254         return 0;
 255 free_curr:
 256         kfree(curr);
 257 cmd_qfail:
 258         free_command_queues(cptvf, cqinfo);
 259         return -ENOMEM;
 260 }
 261
 262 static int init_command_queues(struct otx_cptvf *cptvf, u32 qlen)
 263 {
 264         struct pci_dev *pdev = cptvf->pdev;
 265         int ret;
 266
 267         /* setup command queues */
 268         ret = alloc_command_queues(cptvf, &cptvf->cqinfo, qlen);
 269         if (ret) {
 270                 dev_err(&pdev->dev, "Failed to allocate command queues (%u)\n",
 271                         cptvf->num_queues);
 272                 return ret;
 273         }
 274         return ret;
 275 }
 276
 277 static void cleanup_command_queues(struct otx_cptvf *cptvf)
 278 {
 279         struct pci_dev *pdev = cptvf->pdev;
 280
 281         if (!cptvf->num_queues)
 282                 return;
 283
 284         dev_dbg(&pdev->dev, "Cleaning VQ command queue (%u)\n",
 285                 cptvf->num_queues);
 286         free_command_queues(cptvf, &cptvf->cqinfo);
 287 }
 288
 289 static void cptvf_sw_cleanup(struct otx_cptvf *cptvf)
 290 {
 291         cleanup_worker_threads(cptvf);
 292         cleanup_pending_queues(cptvf);
 293         cleanup_command_queues(cptvf);
 294 }
 295
 296 static int cptvf_sw_init(struct otx_cptvf *cptvf, u32 qlen, u32 num_queues)
 297 {
 298         struct pci_dev *pdev = cptvf->pdev;
 299         u32 max_dev_queues = 0;
 300         int ret;
 301
 302         max_dev_queues = OTX_CPT_NUM_QS_PER_VF;
 303         /* possible cpus */
 304         num_queues = min_t(u32, num_queues, max_dev_queues);
 305         cptvf->num_queues = num_queues;
 306
 307         ret = init_command_queues(cptvf, qlen);
 308         if (ret) {
 309                 dev_err(&pdev->dev, "Failed to setup command queues (%u)\n",
 310                         num_queues);
 311                 return ret;
 312         }
 313
 314         ret = init_pending_queues(cptvf, qlen, num_queues);
 315         if (ret) {
 316                 dev_err(&pdev->dev, "Failed to setup pending queues (%u)\n",
 317                         num_queues);
 318                 goto setup_pqfail;
 319         }
 320
 321         /* Create worker threads for BH processing */
 322         ret = init_worker_threads(cptvf);
 323         if (ret) {
 324                 dev_err(&pdev->dev, "Failed to setup worker threads\n");
 325                 goto init_work_fail;
 326         }
 327         return 0;
 328
 329 init_work_fail:
 330         cleanup_worker_threads(cptvf);
 331         cleanup_pending_queues(cptvf);
 332
 333 setup_pqfail:
 334         cleanup_command_queues(cptvf);
 335
 336         return ret;
 337 }
 338
 339 static void cptvf_free_irq_affinity(struct otx_cptvf *cptvf, int vec)
 340 {
 341         irq_set_affinity_hint(pci_irq_vector(cptvf->pdev, vec), NULL);
 342         free_cpumask_var(cptvf->affinity_mask[vec]);
 343 }
 344
 345 static void cptvf_write_vq_ctl(struct otx_cptvf *cptvf, bool val)
 346 {
 347         union otx_cptx_vqx_ctl vqx_ctl;
 348
 349         vqx_ctl.u = readq(cptvf->reg_base + OTX_CPT_VQX_CTL(0));
 350         vqx_ctl.s.ena = val;
 351         writeq(vqx_ctl.u, cptvf->reg_base + OTX_CPT_VQX_CTL(0));
 352 }
 353
 354 void otx_cptvf_write_vq_doorbell(struct otx_cptvf *cptvf, u32 val)
 355 {
 356         union otx_cptx_vqx_doorbell vqx_dbell;
 357
 358         vqx_dbell.u = readq(cptvf->reg_base + OTX_CPT_VQX_DOORBELL(0));
 359         vqx_dbell.s.dbell_cnt = val * 8; /* Num of Instructions * 8 words */
 360         writeq(vqx_dbell.u, cptvf->reg_base + OTX_CPT_VQX_DOORBELL(0));
 361 }
 362
 363 static void cptvf_write_vq_inprog(struct otx_cptvf *cptvf, u8 val)
 364 {
 365         union otx_cptx_vqx_inprog vqx_inprg;
 366
 367         vqx_inprg.u = readq(cptvf->reg_base + OTX_CPT_VQX_INPROG(0));
 368         vqx_inprg.s.inflight = val;
 369         writeq(vqx_inprg.u, cptvf->reg_base + OTX_CPT_VQX_INPROG(0));
 370 }
 371
 372 static void cptvf_write_vq_done_numwait(struct otx_cptvf *cptvf, u32 val)
 373 {
 374         union otx_cptx_vqx_done_wait vqx_dwait;
 375
 376         vqx_dwait.u = readq(cptvf->reg_base + OTX_CPT_VQX_DONE_WAIT(0));
 377         vqx_dwait.s.num_wait = val;
 378         writeq(vqx_dwait.u, cptvf->reg_base + OTX_CPT_VQX_DONE_WAIT(0));
 379 }
 380
 381 static u32 cptvf_read_vq_done_numwait(struct otx_cptvf *cptvf)
 382 {
 383         union otx_cptx_vqx_done_wait vqx_dwait;
 384
 385         vqx_dwait.u = readq(cptvf->reg_base + OTX_CPT_VQX_DONE_WAIT(0));
 386         return vqx_dwait.s.num_wait;
 387 }
 388
 389 static void cptvf_write_vq_done_timewait(struct otx_cptvf *cptvf, u16 time)
 390 {
 391         union otx_cptx_vqx_done_wait vqx_dwait;
 392
 393         vqx_dwait.u = readq(cptvf->reg_base + OTX_CPT_VQX_DONE_WAIT(0));
 394         vqx_dwait.s.time_wait = time;
 395         writeq(vqx_dwait.u, cptvf->reg_base + OTX_CPT_VQX_DONE_WAIT(0));
 396 }
 397
 398
 399 static u16 cptvf_read_vq_done_timewait(struct otx_cptvf *cptvf)
 400 {
 401         union otx_cptx_vqx_done_wait vqx_dwait;
 402
 403         vqx_dwait.u = readq(cptvf->reg_base + OTX_CPT_VQX_DONE_WAIT(0));
 404         return vqx_dwait.s.time_wait;
 405 }
 406
 407 static void cptvf_enable_swerr_interrupts(struct otx_cptvf *cptvf)
 408 {
 409         union otx_cptx_vqx_misc_ena_w1s vqx_misc_ena;
 410
 411         vqx_misc_ena.u = readq(cptvf->reg_base + OTX_CPT_VQX_MISC_ENA_W1S(0));
 412         /* Enable SWERR interrupts for the requested VF */
 413         vqx_misc_ena.s.swerr = 1;
 414         writeq(vqx_misc_ena.u, cptvf->reg_base + OTX_CPT_VQX_MISC_ENA_W1S(0));
 415 }
 416
 417 static void cptvf_enable_mbox_interrupts(struct otx_cptvf *cptvf)
 418 {
 419         union otx_cptx_vqx_misc_ena_w1s vqx_misc_ena;
 420
 421         vqx_misc_ena.u = readq(cptvf->reg_base + OTX_CPT_VQX_MISC_ENA_W1S(0));
 422         /* Enable MBOX interrupt for the requested VF */
 423         vqx_misc_ena.s.mbox = 1;
 424         writeq(vqx_misc_ena.u, cptvf->reg_base + OTX_CPT_VQX_MISC_ENA_W1S(0));
 425 }
 426
 427 static void cptvf_enable_done_interrupts(struct otx_cptvf *cptvf)
 428 {
 429         union otx_cptx_vqx_done_ena_w1s vqx_done_ena;
 430
 431         vqx_done_ena.u = readq(cptvf->reg_base + OTX_CPT_VQX_DONE_ENA_W1S(0));
 432         /* Enable DONE interrupt for the requested VF */
 433         vqx_done_ena.s.done = 1;
 434         writeq(vqx_done_ena.u, cptvf->reg_base + OTX_CPT_VQX_DONE_ENA_W1S(0));
 435 }
 436
 437 static void cptvf_clear_dovf_intr(struct otx_cptvf *cptvf)
 438 {
 439         union otx_cptx_vqx_misc_int vqx_misc_int;
 440
 441         vqx_misc_int.u = readq(cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0));
 442         /* W1C for the VF */
 443         vqx_misc_int.s.dovf = 1;
 444         writeq(vqx_misc_int.u, cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0));
 445 }
 446
 447 static void cptvf_clear_irde_intr(struct otx_cptvf *cptvf)
 448 {
 449         union otx_cptx_vqx_misc_int vqx_misc_int;
 450
 451         vqx_misc_int.u = readq(cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0));
 452         /* W1C for the VF */
 453         vqx_misc_int.s.irde = 1;
 454         writeq(vqx_misc_int.u, cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0));
 455 }
 456
 457 static void cptvf_clear_nwrp_intr(struct otx_cptvf *cptvf)
 458 {
 459         union otx_cptx_vqx_misc_int vqx_misc_int;
 460
 461         vqx_misc_int.u = readq(cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0));
 462         /* W1C for the VF */
 463         vqx_misc_int.s.nwrp = 1;
 464         writeq(vqx_misc_int.u, cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0));
 465 }
 466
 467 static void cptvf_clear_mbox_intr(struct otx_cptvf *cptvf)
 468 {
 469         union otx_cptx_vqx_misc_int vqx_misc_int;
 470
 471         vqx_misc_int.u = readq(cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0));
 472         /* W1C for the VF */
 473         vqx_misc_int.s.mbox = 1;
 474         writeq(vqx_misc_int.u, cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0));
 475 }
 476
 477 static void cptvf_clear_swerr_intr(struct otx_cptvf *cptvf)
 478 {
 479         union otx_cptx_vqx_misc_int vqx_misc_int;
 480
 481         vqx_misc_int.u = readq(cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0));
 482         /* W1C for the VF */
 483         vqx_misc_int.s.swerr = 1;
 484         writeq(vqx_misc_int.u, cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0));
 485 }
 486
 487 static u64 cptvf_read_vf_misc_intr_status(struct otx_cptvf *cptvf)
 488 {
 489         return readq(cptvf->reg_base + OTX_CPT_VQX_MISC_INT(0));
 490 }
 491
 492 static irqreturn_t cptvf_misc_intr_handler(int __always_unused irq,
 493                                            void *arg)
 494 {
 495         struct otx_cptvf *cptvf = arg;
 496         struct pci_dev *pdev = cptvf->pdev;
 497         u64 intr;
 498
 499         intr = cptvf_read_vf_misc_intr_status(cptvf);
 500         /* Check for MISC interrupt types */
 501         if (likely(intr & OTX_CPT_VF_INTR_MBOX_MASK)) {
 502                 dev_dbg(&pdev->dev, "Mailbox interrupt 0x%llx on CPT VF %d\n",
 503                         intr, cptvf->vfid);
 504                 otx_cptvf_handle_mbox_intr(cptvf);
 505                 cptvf_clear_mbox_intr(cptvf);
 506         } else if (unlikely(intr & OTX_CPT_VF_INTR_DOVF_MASK)) {
 507                 cptvf_clear_dovf_intr(cptvf);
 508                 /* Clear doorbell count */
 509                 otx_cptvf_write_vq_doorbell(cptvf, 0);
 510                 dev_err(&pdev->dev,
 511                 "Doorbell overflow error interrupt 0x%llx on CPT VF %d\n",
 512                         intr, cptvf->vfid);
 513         } else if (unlikely(intr & OTX_CPT_VF_INTR_IRDE_MASK)) {
 514                 cptvf_clear_irde_intr(cptvf);
 515                 dev_err(&pdev->dev,
 516                 "Instruction NCB read error interrupt 0x%llx on CPT VF %d\n",
 517                         intr, cptvf->vfid);
 518         } else if (unlikely(intr & OTX_CPT_VF_INTR_NWRP_MASK)) {
 519                 cptvf_clear_nwrp_intr(cptvf);
 520                 dev_err(&pdev->dev,
 521                 "NCB response write error interrupt 0x%llx on CPT VF %d\n",
 522                         intr, cptvf->vfid);
 523         } else if (unlikely(intr & OTX_CPT_VF_INTR_SERR_MASK)) {
 524                 cptvf_clear_swerr_intr(cptvf);
 525                 dev_err(&pdev->dev,
 526                         "Software error interrupt 0x%llx on CPT VF %d\n",
 527                         intr, cptvf->vfid);
 528         } else {
 529                 dev_err(&pdev->dev, "Unhandled interrupt in OTX_CPT VF %d\n",
 530                         cptvf->vfid);
 531         }
 532
 533         return IRQ_HANDLED;
 534 }
 535
 536 static inline struct otx_cptvf_wqe *get_cptvf_vq_wqe(struct otx_cptvf *cptvf,
 537                                                      int qno)
 538 {
 539         struct otx_cptvf_wqe_info *nwqe_info;
 540
 541         if (unlikely(qno >= cptvf->num_queues))
 542                 return NULL;
 543         nwqe_info = (struct otx_cptvf_wqe_info *)cptvf->wqe_info;
 544
 545         return &nwqe_info->vq_wqe[qno];
 546 }
 547
 548 static inline u32 cptvf_read_vq_done_count(struct otx_cptvf *cptvf)
 549 {
 550         union otx_cptx_vqx_done vqx_done;
 551
 552         vqx_done.u = readq(cptvf->reg_base + OTX_CPT_VQX_DONE(0));
 553         return vqx_done.s.done;
 554 }
 555
 556 static inline void cptvf_write_vq_done_ack(struct otx_cptvf *cptvf,
 557                                            u32 ackcnt)
 558 {
 559         union otx_cptx_vqx_done_ack vqx_dack_cnt;
 560
 561         vqx_dack_cnt.u = readq(cptvf->reg_base + OTX_CPT_VQX_DONE_ACK(0));
 562         vqx_dack_cnt.s.done_ack = ackcnt;
 563         writeq(vqx_dack_cnt.u, cptvf->reg_base + OTX_CPT_VQX_DONE_ACK(0));
 564 }
 565
 566 static irqreturn_t cptvf_done_intr_handler(int __always_unused irq,
 567                                            void *cptvf_dev)
 568 {
 569         struct otx_cptvf *cptvf = (struct otx_cptvf *)cptvf_dev;
 570         struct pci_dev *pdev = cptvf->pdev;
 571         /* Read the number of completions */
 572         u32 intr = cptvf_read_vq_done_count(cptvf);
 573
 574         if (intr) {
 575                 struct otx_cptvf_wqe *wqe;
 576
 577                 /*
 578                  * Acknowledge the number of scheduled completions for
 579                  * processing
 580                  */
 581                 cptvf_write_vq_done_ack(cptvf, intr);
 582                 wqe = get_cptvf_vq_wqe(cptvf, 0);
 583                 if (unlikely(!wqe)) {
 584                         dev_err(&pdev->dev, "No work to schedule for VF (%d)\n",
 585                                 cptvf->vfid);
 586                         return IRQ_NONE;
 587                 }
 588                 tasklet_hi_schedule(&wqe->twork);
 589         }
 590
 591         return IRQ_HANDLED;
 592 }
 593
 594 static void cptvf_set_irq_affinity(struct otx_cptvf *cptvf, int vec)
 595 {
 596         struct pci_dev *pdev = cptvf->pdev;
 597         int cpu;
 598
 599         if (!zalloc_cpumask_var(&cptvf->affinity_mask[vec],
 600                                 GFP_KERNEL)) {
 601                 dev_err(&pdev->dev,
 602                         "Allocation failed for affinity_mask for VF %d\n",
 603                         cptvf->vfid);
 604                 return;
 605         }
 606
 607         cpu = cptvf->vfid % num_online_cpus();
 608         cpumask_set_cpu(cpumask_local_spread(cpu, cptvf->node),
 609                         cptvf->affinity_mask[vec]);
 610         irq_set_affinity_hint(pci_irq_vector(pdev, vec),
 611                               cptvf->affinity_mask[vec]);
 612 }
 613
 614 static void cptvf_write_vq_saddr(struct otx_cptvf *cptvf, u64 val)
 615 {
 616         union otx_cptx_vqx_saddr vqx_saddr;
 617
 618         vqx_saddr.u = val;
 619         writeq(vqx_saddr.u, cptvf->reg_base + OTX_CPT_VQX_SADDR(0));
 620 }
 621
 622 static void cptvf_device_init(struct otx_cptvf *cptvf)
 623 {
 624         u64 base_addr = 0;
 625
 626         /* Disable the VQ */
 627         cptvf_write_vq_ctl(cptvf, 0);
 628         /* Reset the doorbell */
 629         otx_cptvf_write_vq_doorbell(cptvf, 0);
 630         /* Clear inflight */
 631         cptvf_write_vq_inprog(cptvf, 0);
 632         /* Write VQ SADDR */
 633         base_addr = (u64)(cptvf->cqinfo.queue[0].qhead->dma_addr);
 634         cptvf_write_vq_saddr(cptvf, base_addr);
 635         /* Configure timerhold / coalescence */
 636         cptvf_write_vq_done_timewait(cptvf, OTX_CPT_TIMER_HOLD);
 637         cptvf_write_vq_done_numwait(cptvf, OTX_CPT_COUNT_HOLD);
 638         /* Enable the VQ */
 639         cptvf_write_vq_ctl(cptvf, 1);
 640         /* Flag the VF ready */
 641         cptvf->flags |= OTX_CPT_FLAG_DEVICE_READY;
 642 }
 643
 644 static ssize_t vf_type_show(struct device *dev,
 645                             struct device_attribute *attr,
 646                             char *buf)
 647 {
 648         struct otx_cptvf *cptvf = dev_get_drvdata(dev);
 649         char *msg;
 650
 651         switch (cptvf->vftype) {
 652         case OTX_CPT_AE_TYPES:
 653                 msg = "AE";
 654                 break;
 655
 656         case OTX_CPT_SE_TYPES:
 657                 msg = "SE";
 658                 break;
 659
 660         default:
 661                 msg = "Invalid";
 662         }
 663
 664         return sysfs_emit(buf, "%s\n", msg);
 665 }
 666
 667 static ssize_t vf_engine_group_show(struct device *dev,
 668                                     struct device_attribute *attr,
 669                                     char *buf)
 670 {
 671         struct otx_cptvf *cptvf = dev_get_drvdata(dev);
 672
 673         return sysfs_emit(buf, "%d\n", cptvf->vfgrp);
 674 }
 675
 676 static ssize_t vf_engine_group_store(struct device *dev,
 677                                      struct device_attribute *attr,
 678                                      const char *buf, size_t count)
 679 {
 680         struct otx_cptvf *cptvf = dev_get_drvdata(dev);
 681         int val, ret;
 682
 683         ret = kstrtoint(buf, 10, &val);
 684         if (ret)
 685                 return ret;
 686
 687         if (val < 0)
 688                 return -EINVAL;
 689
 690         if (val >= OTX_CPT_MAX_ENGINE_GROUPS) {
 691                 dev_err(dev, "Engine group >= than max available groups %d\n",
 692                         OTX_CPT_MAX_ENGINE_GROUPS);
 693                 return -EINVAL;
 694         }
 695
 696         ret = otx_cptvf_send_vf_to_grp_msg(cptvf, val);
 697         if (ret)
 698                 return ret;
 699
 700         return count;
 701 }
 702
 703 static ssize_t vf_coalesc_time_wait_show(struct device *dev,
 704                                          struct device_attribute *attr,
 705                                          char *buf)
 706 {
 707         struct otx_cptvf *cptvf = dev_get_drvdata(dev);
 708
 709         return sysfs_emit(buf, "%d\n",
 710                          cptvf_read_vq_done_timewait(cptvf));
 711 }
 712
 713 static ssize_t vf_coalesc_num_wait_show(struct device *dev,
 714                                         struct device_attribute *attr,
 715                                         char *buf)
 716 {
 717         struct otx_cptvf *cptvf = dev_get_drvdata(dev);
 718
 719         return sysfs_emit(buf, "%d\n",
 720                          cptvf_read_vq_done_numwait(cptvf));
 721 }
 722
 723 static ssize_t vf_coalesc_time_wait_store(struct device *dev,
 724                                           struct device_attribute *attr,
 725                                           const char *buf, size_t count)
 726 {
 727         struct otx_cptvf *cptvf = dev_get_drvdata(dev);
 728         long val;
 729         int ret;
 730
 731         ret = kstrtol(buf, 10, &val);
 732         if (ret != 0)
 733                 return ret;
 734
 735         if (val < OTX_CPT_COALESC_MIN_TIME_WAIT ||
 736             val > OTX_CPT_COALESC_MAX_TIME_WAIT)
 737                 return -EINVAL;
 738
 739         cptvf_write_vq_done_timewait(cptvf, val);
 740         return count;
 741 }
 742
 743 static ssize_t vf_coalesc_num_wait_store(struct device *dev,
 744                                          struct device_attribute *attr,
 745                                          const char *buf, size_t count)
 746 {
 747         struct otx_cptvf *cptvf = dev_get_drvdata(dev);
 748         long val;
 749         int ret;
 750
 751         ret = kstrtol(buf, 10, &val);
 752         if (ret != 0)
 753                 return ret;
 754
 755         if (val < OTX_CPT_COALESC_MIN_NUM_WAIT ||
 756             val > OTX_CPT_COALESC_MAX_NUM_WAIT)
 757                 return -EINVAL;
 758
 759         cptvf_write_vq_done_numwait(cptvf, val);
 760         return count;
 761 }
 762
 763 static DEVICE_ATTR_RO(vf_type);
 764 static DEVICE_ATTR_RW(vf_engine_group);
 765 static DEVICE_ATTR_RW(vf_coalesc_time_wait);
 766 static DEVICE_ATTR_RW(vf_coalesc_num_wait);
 767
 768 static struct attribute *otx_cptvf_attrs[] = {
 769         &dev_attr_vf_type.attr,
 770         &dev_attr_vf_engine_group.attr,
 771         &dev_attr_vf_coalesc_time_wait.attr,
 772         &dev_attr_vf_coalesc_num_wait.attr,
 773         NULL
 774 };
 775
 776 static const struct attribute_group otx_cptvf_sysfs_group = {
 777         .attrs = otx_cptvf_attrs,
 778 };
 779
 780 static int otx_cptvf_probe(struct pci_dev *pdev,
 781                            const struct pci_device_id *ent)
 782 {
 783         struct device *dev = &pdev->dev;
 784         struct otx_cptvf *cptvf;
 785         int err;
 786
 787         cptvf = devm_kzalloc(dev, sizeof(*cptvf), GFP_KERNEL);
 788         if (!cptvf)
 789                 return -ENOMEM;
 790
 791         pci_set_drvdata(pdev, cptvf);
 792         cptvf->pdev = pdev;
 793
 794         err = pci_enable_device(pdev);
 795         if (err) {
 796                 dev_err(dev, "Failed to enable PCI device\n");
 797                 goto clear_drvdata;
 798         }
 799         err = pci_request_regions(pdev, DRV_NAME);
 800         if (err) {
 801                 dev_err(dev, "PCI request regions failed 0x%x\n", err);
 802                 goto disable_device;
 803         }
 804         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48));
 805         if (err) {
 806                 dev_err(dev, "Unable to get usable 48-bit DMA configuration\n");
 807                 goto release_regions;
 808         }
 809
 810         /* MAP PF's configuration registers */
 811         cptvf->reg_base = pci_iomap(pdev, OTX_CPT_VF_PCI_CFG_BAR, 0);
 812         if (!cptvf->reg_base) {
 813                 dev_err(dev, "Cannot map config register space, aborting\n");
 814                 err = -ENOMEM;
 815                 goto release_regions;
 816         }
 817
 818         cptvf->node = dev_to_node(&pdev->dev);
 819         err = pci_alloc_irq_vectors(pdev, OTX_CPT_VF_MSIX_VECTORS,
 820                                     OTX_CPT_VF_MSIX_VECTORS, PCI_IRQ_MSIX);
 821         if (err < 0) {
 822                 dev_err(dev, "Request for #%d msix vectors failed\n",
 823                         OTX_CPT_VF_MSIX_VECTORS);
 824                 goto unmap_region;
 825         }
 826
 827         err = request_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC),
 828                           cptvf_misc_intr_handler, 0, "CPT VF misc intr",
 829                           cptvf);
 830         if (err) {
 831                 dev_err(dev, "Failed to request misc irq\n");
 832                 goto free_vectors;
 833         }
 834
 835         /* Enable mailbox interrupt */
 836         cptvf_enable_mbox_interrupts(cptvf);
 837         cptvf_enable_swerr_interrupts(cptvf);
 838
 839         /* Check cpt pf status, gets chip ID / device Id from PF if ready */
 840         err = otx_cptvf_check_pf_ready(cptvf);
 841         if (err)
 842                 goto free_misc_irq;
 843
 844         /* CPT VF software resources initialization */
 845         cptvf->cqinfo.qchunksize = OTX_CPT_CMD_QCHUNK_SIZE;
 846         err = cptvf_sw_init(cptvf, OTX_CPT_CMD_QLEN, OTX_CPT_NUM_QS_PER_VF);
 847         if (err) {
 848                 dev_err(dev, "cptvf_sw_init() failed\n");
 849                 goto free_misc_irq;
 850         }
 851         /* Convey VQ LEN to PF */
 852         err = otx_cptvf_send_vq_size_msg(cptvf);
 853         if (err)
 854                 goto sw_cleanup;
 855
 856         /* CPT VF device initialization */
 857         cptvf_device_init(cptvf);
 858         /* Send msg to PF to assign currnet Q to required group */
 859         err = otx_cptvf_send_vf_to_grp_msg(cptvf, cptvf->vfgrp);
 860         if (err)
 861                 goto sw_cleanup;
 862
 863         cptvf->priority = 1;
 864         err = otx_cptvf_send_vf_priority_msg(cptvf);
 865         if (err)
 866                 goto sw_cleanup;
 867
 868         err = request_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE),
 869                           cptvf_done_intr_handler, 0, "CPT VF done intr",
 870                           cptvf);
 871         if (err) {
 872                 dev_err(dev, "Failed to request done irq\n");
 873                 goto free_done_irq;
 874         }
 875
 876         /* Enable done interrupt */
 877         cptvf_enable_done_interrupts(cptvf);
 878
 879         /* Set irq affinity masks */
 880         cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC);
 881         cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE);
 882
 883         err = otx_cptvf_send_vf_up(cptvf);
 884         if (err)
 885                 goto free_irq_affinity;
 886
 887         /* Initialize algorithms and set ops */
 888         err = otx_cpt_crypto_init(pdev, THIS_MODULE,
 889                     cptvf->vftype == OTX_CPT_SE_TYPES ? OTX_CPT_SE : OTX_CPT_AE,
 890                     cptvf->vftype, 1, cptvf->num_vfs);
 891         if (err) {
 892                 dev_err(dev, "Failed to register crypto algs\n");
 893                 goto free_irq_affinity;
 894         }
 895
 896         err = sysfs_create_group(&dev->kobj, &otx_cptvf_sysfs_group);
 897         if (err) {
 898                 dev_err(dev, "Creating sysfs entries failed\n");
 899                 goto crypto_exit;
 900         }
 901
 902         return 0;
 903
 904 crypto_exit:
 905         otx_cpt_crypto_exit(pdev, THIS_MODULE, cptvf->vftype);
 906 free_irq_affinity:
 907         cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE);
 908         cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC);
 909 free_done_irq:
 910         free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE), cptvf);
 911 sw_cleanup:
 912         cptvf_sw_cleanup(cptvf);
 913 free_misc_irq:
 914         free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf);
 915 free_vectors:
 916         pci_free_irq_vectors(cptvf->pdev);
 917 unmap_region:
 918         pci_iounmap(pdev, cptvf->reg_base);
 919 release_regions:
 920         pci_release_regions(pdev);
 921 disable_device:
 922         pci_disable_device(pdev);
 923 clear_drvdata:
 924         pci_set_drvdata(pdev, NULL);
 925
 926         return err;
 927 }
 928
 929 static void otx_cptvf_remove(struct pci_dev *pdev)
 930 {
 931         struct otx_cptvf *cptvf = pci_get_drvdata(pdev);
 932
 933         if (!cptvf) {
 934                 dev_err(&pdev->dev, "Invalid CPT-VF device\n");
 935                 return;
 936         }
 937
 938         /* Convey DOWN to PF */
 939         if (otx_cptvf_send_vf_down(cptvf)) {
 940                 dev_err(&pdev->dev, "PF not responding to DOWN msg\n");
 941         } else {
 942                 sysfs_remove_group(&pdev->dev.kobj, &otx_cptvf_sysfs_group);
 943                 otx_cpt_crypto_exit(pdev, THIS_MODULE, cptvf->vftype);
 944                 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE);
 945                 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC);
 946                 free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE), cptvf);
 947                 free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf);
 948                 cptvf_sw_cleanup(cptvf);
 949                 pci_free_irq_vectors(cptvf->pdev);
 950                 pci_iounmap(pdev, cptvf->reg_base);
 951                 pci_release_regions(pdev);
 952                 pci_disable_device(pdev);
 953                 pci_set_drvdata(pdev, NULL);
 954         }
 955 }
 956
 957 /* Supported devices */
 958 static const struct pci_device_id otx_cptvf_id_table[] = {
 959         {PCI_VDEVICE(CAVIUM, OTX_CPT_PCI_VF_DEVICE_ID), 0},
 960         { 0, }  /* end of table */
 961 };
 962
 963 static struct pci_driver otx_cptvf_pci_driver = {
 964         .name = DRV_NAME,
 965         .id_table = otx_cptvf_id_table,
 966         .probe = otx_cptvf_probe,
 967         .remove = otx_cptvf_remove,
 968 };
 969
 970 module_pci_driver(otx_cptvf_pci_driver);
 971
 972 MODULE_AUTHOR("Marvell International Ltd.");
 973 MODULE_DESCRIPTION("Marvell OcteonTX CPT Virtual Function Driver");
 974 MODULE_LICENSE("GPL v2");
 975 MODULE_VERSION(DRV_VERSION);
 976 MODULE_DEVICE_TABLE(pci, otx_cptvf_id_table);