drivers/crypto/cavium/nitrox/nitrox_main.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 #include <linux/aer.h>
   3 #include <linux/delay.h>
   4 #include <linux/firmware.h>
   5 #include <linux/list.h>
   6 #include <linux/module.h>
   7 #include <linux/mutex.h>
   8 #include <linux/pci.h>
   9 #include <linux/pci_ids.h>
  10
  11 #include "nitrox_dev.h"
  12 #include "nitrox_common.h"
  13 #include "nitrox_csr.h"
  14 #include "nitrox_hal.h"
  15 #include "nitrox_isr.h"
  16 #include "nitrox_debugfs.h"
  17
  18 #define CNN55XX_DEV_ID  0x12
  19 #define UCODE_HLEN 48
  20 #define DEFAULT_SE_GROUP 0
  21 #define DEFAULT_AE_GROUP 0
  22
  23 #define DRIVER_VERSION "1.2"
  24 #define CNN55XX_UCD_BLOCK_SIZE 32768
  25 #define CNN55XX_MAX_UCODE_SIZE (CNN55XX_UCD_BLOCK_SIZE * 2)
  26 #define FW_DIR "cavium/"
  27 /* SE microcode */
  28 #define SE_FW   FW_DIR "cnn55xx_se.fw"
  29 /* AE microcode */
  30 #define AE_FW   FW_DIR "cnn55xx_ae.fw"
  31
  32 static const char nitrox_driver_name[] = "CNN55XX";
  33
  34 static LIST_HEAD(ndevlist);
  35 static DEFINE_MUTEX(devlist_lock);
  36 static unsigned int num_devices;
  37
  38 /**
  39  * nitrox_pci_tbl - PCI Device ID Table
  40  */
  41 static const struct pci_device_id nitrox_pci_tbl[] = {
  42         {PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
  43         /* required last entry */
  44         {0, }
  45 };
  46 MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);
  47
  48 static unsigned int qlen = DEFAULT_CMD_QLEN;
  49 module_param(qlen, uint, 0644);
  50 MODULE_PARM_DESC(qlen, "Command queue length - default 2048");
  51
  52 #ifdef CONFIG_PCI_IOV
  53 int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs);
  54 #else
  55 int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs)
  56 {
  57         return 0;
  58 }
  59 #endif
  60
  61 /**
  62  * struct ucode - Firmware Header
  63  * @id: microcode ID
  64  * @version: firmware version
  65  * @code_size: code section size
  66  * @raz: alignment
  67  * @code: code section
  68  */
  69 struct ucode {
  70         u8 id;
  71         char version[VERSION_LEN - 1];
  72         __be32 code_size;
  73         u8 raz[12];
  74         u64 code[0];
  75 };
  76
  77 /**
  78  * write_to_ucd_unit - Write Firmware to NITROX UCD unit
  79  */
  80 static void write_to_ucd_unit(struct nitrox_device *ndev, u32 ucode_size,
  81                               u64 *ucode_data, int block_num)
  82 {
  83         u32 code_size;
  84         u64 offset, data;
  85         int i = 0;
  86
  87         /*
  88          * UCD structure
  89          *
  90          *  -------------
  91          *  |    BLK 7  |
  92          *  -------------
  93          *  |    BLK 6  |
  94          *  -------------
  95          *  |    ...    |
  96          *  -------------
  97          *  |    BLK 0  |
  98          *  -------------
  99          *  Total of 8 blocks, each size 32KB
 100          */
 101
 102         /* set the block number */
 103         offset = UCD_UCODE_LOAD_BLOCK_NUM;
 104         nitrox_write_csr(ndev, offset, block_num);
 105
 106         code_size = ucode_size;
 107         code_size = roundup(code_size, 8);
 108         while (code_size) {
 109                 data = ucode_data[i];
 110                 /* write 8 bytes at a time */
 111                 offset = UCD_UCODE_LOAD_IDX_DATAX(i);
 112                 nitrox_write_csr(ndev, offset, data);
 113                 code_size -= 8;
 114                 i++;
 115         }
 116
 117         usleep_range(300, 400);
 118 }
 119
 120 static int nitrox_load_fw(struct nitrox_device *ndev)
 121 {
 122         const struct firmware *fw;
 123         const char *fw_name;
 124         struct ucode *ucode;
 125         u64 *ucode_data;
 126         u64 offset;
 127         union ucd_core_eid_ucode_block_num core_2_eid_val;
 128         union aqm_grp_execmsk_lo aqm_grp_execmask_lo;
 129         union aqm_grp_execmsk_hi aqm_grp_execmask_hi;
 130         u32 ucode_size;
 131         int ret, i = 0;
 132
 133         fw_name = SE_FW;
 134         dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
 135
 136         ret = request_firmware(&fw, fw_name, DEV(ndev));
 137         if (ret < 0) {
 138                 dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
 139                 return ret;
 140         }
 141
 142         ucode = (struct ucode *)fw->data;
 143
 144         ucode_size = be32_to_cpu(ucode->code_size) * 2;
 145         if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) {
 146                 dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n",
 147                         ucode_size, fw_name);
 148                 release_firmware(fw);
 149                 return -EINVAL;
 150         }
 151         ucode_data = ucode->code;
 152
 153         /* copy the firmware version */
 154         memcpy(&ndev->hw.fw_name[0][0], ucode->version, (VERSION_LEN - 2));
 155         ndev->hw.fw_name[0][VERSION_LEN - 1] = '\0';
 156
 157         /* Load SE Firmware on UCD Block 0 */
 158         write_to_ucd_unit(ndev, ucode_size, ucode_data, 0);
 159
 160         release_firmware(fw);
 161
 162         /* put all SE cores in DEFAULT_SE_GROUP */
 163         offset = POM_GRP_EXECMASKX(DEFAULT_SE_GROUP);
 164         nitrox_write_csr(ndev, offset, (~0ULL));
 165
 166         /* write block number and firmware length
 167          * bit:<2:0> block number
 168          * bit:3 is set SE uses 32KB microcode
 169          * bit:3 is clear SE uses 64KB microcode
 170          */
 171         core_2_eid_val.value = 0ULL;
 172         core_2_eid_val.ucode_blk = 0;
 173         if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE)
 174                 core_2_eid_val.ucode_len = 1;
 175         else
 176                 core_2_eid_val.ucode_len = 0;
 177
 178         for (i = 0; i < ndev->hw.se_cores; i++) {
 179                 offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
 180                 nitrox_write_csr(ndev, offset, core_2_eid_val.value);
 181         }
 182
 183
 184         fw_name = AE_FW;
 185         dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
 186
 187         ret = request_firmware(&fw, fw_name, DEV(ndev));
 188         if (ret < 0) {
 189                 dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
 190                 return ret;
 191         }
 192
 193         ucode = (struct ucode *)fw->data;
 194
 195         ucode_size = be32_to_cpu(ucode->code_size) * 2;
 196         if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) {
 197                 dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n",
 198                         ucode_size, fw_name);
 199                 release_firmware(fw);
 200                 return -EINVAL;
 201         }
 202         ucode_data = ucode->code;
 203
 204         /* copy the firmware version */
 205         memcpy(&ndev->hw.fw_name[1][0], ucode->version, (VERSION_LEN - 2));
 206         ndev->hw.fw_name[1][VERSION_LEN - 1] = '\0';
 207
 208         /* Load AE Firmware on UCD Block 2 */
 209         write_to_ucd_unit(ndev, ucode_size, ucode_data, 2);
 210
 211         release_firmware(fw);
 212
 213         /* put all AE cores in DEFAULT_AE_GROUP */
 214         offset = AQM_GRP_EXECMSK_LOX(DEFAULT_AE_GROUP);
 215         aqm_grp_execmask_lo.exec_0_to_39 = 0xFFFFFFFFFFULL;
 216         nitrox_write_csr(ndev, offset, aqm_grp_execmask_lo.value);
 217         offset = AQM_GRP_EXECMSK_HIX(DEFAULT_AE_GROUP);
 218         aqm_grp_execmask_hi.exec_40_to_79 = 0xFFFFFFFFFFULL;
 219         nitrox_write_csr(ndev, offset, aqm_grp_execmask_hi.value);
 220
 221         /* write block number and firmware length
 222          * bit:<2:0> block number
 223          * bit:3 is set SE uses 32KB microcode
 224          * bit:3 is clear SE uses 64KB microcode
 225          */
 226         core_2_eid_val.value = 0ULL;
 227         core_2_eid_val.ucode_blk = 0;
 228         if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE)
 229                 core_2_eid_val.ucode_len = 1;
 230         else
 231                 core_2_eid_val.ucode_len = 0;
 232
 233         for (i = 0; i < ndev->hw.ae_cores; i++) {
 234                 offset = UCD_AE_EID_UCODE_BLOCK_NUMX(i);
 235                 nitrox_write_csr(ndev, offset, core_2_eid_val.value);
 236         }
 237
 238         return 0;
 239 }
 240
 241 /**
 242  * nitrox_add_to_devlist - add NITROX device to global device list
 243  * @ndev: NITROX device
 244  */
 245 static int nitrox_add_to_devlist(struct nitrox_device *ndev)
 246 {
 247         struct nitrox_device *dev;
 248         int ret = 0;
 249
 250         INIT_LIST_HEAD(&ndev->list);
 251         refcount_set(&ndev->refcnt, 1);
 252
 253         mutex_lock(&devlist_lock);
 254         list_for_each_entry(dev, &ndevlist, list) {
 255                 if (dev == ndev) {
 256                         ret = -EEXIST;
 257                         goto unlock;
 258                 }
 259         }
 260         ndev->idx = num_devices++;
 261         list_add_tail(&ndev->list, &ndevlist);
 262 unlock:
 263         mutex_unlock(&devlist_lock);
 264         return ret;
 265 }
 266
 267 /**
 268  * nitrox_remove_from_devlist - remove NITROX device from
 269  *   global device list
 270  * @ndev: NITROX device
 271  */
 272 static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
 273 {
 274         mutex_lock(&devlist_lock);
 275         list_del(&ndev->list);
 276         num_devices--;
 277         mutex_unlock(&devlist_lock);
 278 }
 279
 280 struct nitrox_device *nitrox_get_first_device(void)
 281 {
 282         struct nitrox_device *ndev = NULL;
 283
 284         mutex_lock(&devlist_lock);
 285         list_for_each_entry(ndev, &ndevlist, list) {
 286                 if (nitrox_ready(ndev))
 287                         break;
 288         }
 289         mutex_unlock(&devlist_lock);
 290         if (!ndev)
 291                 return NULL;
 292
 293         refcount_inc(&ndev->refcnt);
 294         /* barrier to sync with other cpus */
 295         smp_mb__after_atomic();
 296         return ndev;
 297 }
 298
 299 void nitrox_put_device(struct nitrox_device *ndev)
 300 {
 301         if (!ndev)
 302                 return;
 303
 304         refcount_dec(&ndev->refcnt);
 305         /* barrier to sync with other cpus */
 306         smp_mb__after_atomic();
 307 }
 308
 309 static int nitrox_device_flr(struct pci_dev *pdev)
 310 {
 311         int pos = 0;
 312
 313         pos = pci_save_state(pdev);
 314         if (pos) {
 315                 dev_err(&pdev->dev, "Failed to save pci state\n");
 316                 return -ENOMEM;
 317         }
 318
 319         /* check flr support */
 320         if (pcie_has_flr(pdev))
 321                 pcie_flr(pdev);
 322
 323         pci_restore_state(pdev);
 324
 325         return 0;
 326 }
 327
 328 static int nitrox_pf_sw_init(struct nitrox_device *ndev)
 329 {
 330         int err;
 331
 332         err = nitrox_common_sw_init(ndev);
 333         if (err)
 334                 return err;
 335
 336         err = nitrox_register_interrupts(ndev);
 337         if (err)
 338                 nitrox_common_sw_cleanup(ndev);
 339
 340         return err;
 341 }
 342
 343 static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
 344 {
 345         nitrox_unregister_interrupts(ndev);
 346         nitrox_common_sw_cleanup(ndev);
 347 }
 348
 349 /**
 350  * nitrox_bist_check - Check NITORX BIST registers status
 351  * @ndev: NITROX device
 352  */
 353 static int nitrox_bist_check(struct nitrox_device *ndev)
 354 {
 355         u64 value = 0;
 356         int i;
 357
 358         for (i = 0; i < NR_CLUSTERS; i++) {
 359                 value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
 360                 value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
 361         }
 362         value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
 363         value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
 364         value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
 365         value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
 366         value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
 367         value += nitrox_read_csr(ndev, POM_BIST_REG);
 368         value += nitrox_read_csr(ndev, BMI_BIST_REG);
 369         value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
 370         value += nitrox_read_csr(ndev, BMO_BIST_REG);
 371         value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
 372         value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
 373         if (value)
 374                 return -EIO;
 375         return 0;
 376 }
 377
 378 static int nitrox_pf_hw_init(struct nitrox_device *ndev)
 379 {
 380         int err;
 381
 382         err = nitrox_bist_check(ndev);
 383         if (err) {
 384                 dev_err(&ndev->pdev->dev, "BIST check failed\n");
 385                 return err;
 386         }
 387         /* get cores information */
 388         nitrox_get_hwinfo(ndev);
 389
 390         nitrox_config_nps_core_unit(ndev);
 391         nitrox_config_aqm_unit(ndev);
 392         nitrox_config_nps_pkt_unit(ndev);
 393         nitrox_config_pom_unit(ndev);
 394         nitrox_config_efl_unit(ndev);
 395         /* configure IO units */
 396         nitrox_config_bmi_unit(ndev);
 397         nitrox_config_bmo_unit(ndev);
 398         /* configure Local Buffer Cache */
 399         nitrox_config_lbc_unit(ndev);
 400         nitrox_config_rand_unit(ndev);
 401
 402         /* load firmware on cores */
 403         err = nitrox_load_fw(ndev);
 404         if (err)
 405                 return err;
 406
 407         nitrox_config_emu_unit(ndev);
 408
 409         return 0;
 410 }
 411
 412 /**
 413  * nitrox_probe - NITROX Initialization function.
 414  * @pdev: PCI device information struct
 415  * @id: entry in nitrox_pci_tbl
 416  *
 417  * Return: 0, if the driver is bound to the device, or
 418  *         a negative error if there is failure.
 419  */
 420 static int nitrox_probe(struct pci_dev *pdev,
 421                         const struct pci_device_id *id)
 422 {
 423         struct nitrox_device *ndev;
 424         int err;
 425
 426         dev_info_once(&pdev->dev, "%s driver version %s\n",
 427                       nitrox_driver_name, DRIVER_VERSION);
 428
 429         err = pci_enable_device_mem(pdev);
 430         if (err)
 431                 return err;
 432
 433         /* do FLR */
 434         err = nitrox_device_flr(pdev);
 435         if (err) {
 436                 dev_err(&pdev->dev, "FLR failed\n");
 437                 pci_disable_device(pdev);
 438                 return err;
 439         }
 440
 441         if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
 442                 dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
 443         } else {
 444                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
 445                 if (err) {
 446                         dev_err(&pdev->dev, "DMA configuration failed\n");
 447                         pci_disable_device(pdev);
 448                         return err;
 449                 }
 450         }
 451
 452         err = pci_request_mem_regions(pdev, nitrox_driver_name);
 453         if (err) {
 454                 pci_disable_device(pdev);
 455                 return err;
 456         }
 457         pci_set_master(pdev);
 458
 459         ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
 460         if (!ndev) {
 461                 err = -ENOMEM;
 462                 goto ndev_fail;
 463         }
 464
 465         pci_set_drvdata(pdev, ndev);
 466         ndev->pdev = pdev;
 467
 468         /* add to device list */
 469         nitrox_add_to_devlist(ndev);
 470
 471         ndev->hw.vendor_id = pdev->vendor;
 472         ndev->hw.device_id = pdev->device;
 473         ndev->hw.revision_id = pdev->revision;
 474         /* command timeout in jiffies */
 475         ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
 476         ndev->node = dev_to_node(&pdev->dev);
 477         if (ndev->node == NUMA_NO_NODE)
 478                 ndev->node = 0;
 479
 480         ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
 481                                  pci_resource_len(pdev, 0));
 482         if (!ndev->bar_addr) {
 483                 err = -EIO;
 484                 goto ioremap_err;
 485         }
 486         /* allocate command queus based on cpus, max queues are 64 */
 487         ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
 488         ndev->qlen = qlen;
 489
 490         err = nitrox_pf_sw_init(ndev);
 491         if (err)
 492                 goto ioremap_err;
 493
 494         err = nitrox_pf_hw_init(ndev);
 495         if (err)
 496                 goto pf_hw_fail;
 497
 498         nitrox_debugfs_init(ndev);
 499
 500         /* clear the statistics */
 501         atomic64_set(&ndev->stats.posted, 0);
 502         atomic64_set(&ndev->stats.completed, 0);
 503         atomic64_set(&ndev->stats.dropped, 0);
 504
 505         atomic_set(&ndev->state, __NDEV_READY);
 506         /* barrier to sync with other cpus */
 507         smp_mb__after_atomic();
 508
 509         err = nitrox_crypto_register();
 510         if (err)
 511                 goto crypto_fail;
 512
 513         return 0;
 514
 515 crypto_fail:
 516         nitrox_debugfs_exit(ndev);
 517         atomic_set(&ndev->state, __NDEV_NOT_READY);
 518         /* barrier to sync with other cpus */
 519         smp_mb__after_atomic();
 520 pf_hw_fail:
 521         nitrox_pf_sw_cleanup(ndev);
 522 ioremap_err:
 523         nitrox_remove_from_devlist(ndev);
 524         kfree(ndev);
 525         pci_set_drvdata(pdev, NULL);
 526 ndev_fail:
 527         pci_release_mem_regions(pdev);
 528         pci_disable_device(pdev);
 529         return err;
 530 }
 531
 532 /**
 533  * nitrox_remove - Unbind the driver from the device.
 534  * @pdev: PCI device information struct
 535  */
 536 static void nitrox_remove(struct pci_dev *pdev)
 537 {
 538         struct nitrox_device *ndev = pci_get_drvdata(pdev);
 539
 540         if (!ndev)
 541                 return;
 542
 543         if (!refcount_dec_and_test(&ndev->refcnt)) {
 544                 dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
 545                         refcount_read(&ndev->refcnt));
 546                 return;
 547         }
 548
 549         dev_info(DEV(ndev), "Removing Device %x:%x\n",
 550                  ndev->hw.vendor_id, ndev->hw.device_id);
 551
 552         atomic_set(&ndev->state, __NDEV_NOT_READY);
 553         /* barrier to sync with other cpus */
 554         smp_mb__after_atomic();
 555
 556         nitrox_remove_from_devlist(ndev);
 557
 558 #ifdef CONFIG_PCI_IOV
 559         /* disable SR-IOV */
 560         nitrox_sriov_configure(pdev, 0);
 561 #endif
 562         nitrox_crypto_unregister();
 563         nitrox_debugfs_exit(ndev);
 564         nitrox_pf_sw_cleanup(ndev);
 565
 566         iounmap(ndev->bar_addr);
 567         kfree(ndev);
 568
 569         pci_set_drvdata(pdev, NULL);
 570         pci_release_mem_regions(pdev);
 571         pci_disable_device(pdev);
 572 }
 573
 574 static void nitrox_shutdown(struct pci_dev *pdev)
 575 {
 576         pci_set_drvdata(pdev, NULL);
 577         pci_release_mem_regions(pdev);
 578         pci_disable_device(pdev);
 579 }
 580
 581 static struct pci_driver nitrox_driver = {
 582         .name = nitrox_driver_name,
 583         .id_table = nitrox_pci_tbl,
 584         .probe = nitrox_probe,
 585         .remove = nitrox_remove,
 586         .shutdown = nitrox_shutdown,
 587 #ifdef CONFIG_PCI_IOV
 588         .sriov_configure = nitrox_sriov_configure,
 589 #endif
 590 };
 591
 592 module_pci_driver(nitrox_driver);
 593
 594 MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
 595 MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
 596 MODULE_LICENSE("GPL");
 597 MODULE_VERSION(DRIVER_VERSION);
 598 MODULE_FIRMWARE(SE_FW);