drivers/crypto/cavium/nitrox/nitrox_main.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 #include <linux/aer.h>
   3 #include <linux/delay.h>
   4 #include <linux/firmware.h>
   5 #include <linux/list.h>
   6 #include <linux/module.h>
   7 #include <linux/mutex.h>
   8 #include <linux/pci.h>
   9 #include <linux/pci_ids.h>
  10
  11 #include "nitrox_dev.h"
  12 #include "nitrox_common.h"
  13 #include "nitrox_csr.h"
  14 #include "nitrox_hal.h"
  15 #include "nitrox_isr.h"
  16 #include "nitrox_debugfs.h"
  17
  18 #define CNN55XX_DEV_ID  0x12
  19 #define UCODE_HLEN 48
  20 #define DEFAULT_SE_GROUP 0
  21 #define DEFAULT_AE_GROUP 0
  22
  23 #define DRIVER_VERSION "1.2"
  24 #define CNN55XX_UCD_BLOCK_SIZE 32768
  25 #define CNN55XX_MAX_UCODE_SIZE (CNN55XX_UCD_BLOCK_SIZE * 2)
  26 #define FW_DIR "cavium/"
  27 /* SE microcode */
  28 #define SE_FW   FW_DIR "cnn55xx_se.fw"
  29 /* AE microcode */
  30 #define AE_FW   FW_DIR "cnn55xx_ae.fw"
  31
  32 static const char nitrox_driver_name[] = "CNN55XX";
  33
  34 static LIST_HEAD(ndevlist);
  35 static DEFINE_MUTEX(devlist_lock);
  36 static unsigned int num_devices;
  37
  38 /**
  39  * nitrox_pci_tbl - PCI Device ID Table
  40  */
  41 static const struct pci_device_id nitrox_pci_tbl[] = {
  42         {PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
  43         /* required last entry */
  44         {0, }
  45 };
  46 MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);
  47
  48 static unsigned int qlen = DEFAULT_CMD_QLEN;
  49 module_param(qlen, uint, 0644);
  50 MODULE_PARM_DESC(qlen, "Command queue length - default 2048");
  51
  52 #ifdef CONFIG_PCI_IOV
  53 int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs);
  54 #else
  55 int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs)
  56 {
  57         return 0;
  58 }
  59 #endif
  60
  61 /**
  62  * struct ucode - Firmware Header
  63  * @id: microcode ID
  64  * @version: firmware version
  65  * @code_size: code section size
  66  * @raz: alignment
  67  * @code: code section
  68  */
  69 struct ucode {
  70         u8 id;
  71         char version[VERSION_LEN - 1];
  72         __be32 code_size;
  73         u8 raz[12];
  74         u64 code[0];
  75 };
  76
  77 /**
  78  * write_to_ucd_unit - Write Firmware to NITROX UCD unit
  79  */
  80 static void write_to_ucd_unit(struct nitrox_device *ndev, u32 ucode_size,
  81                               u64 *ucode_data, int block_num)
  82 {
  83         u32 code_size;
  84         u64 offset, data;
  85         int i = 0;
  86
  87         /*
  88          * UCD structure
  89          *
  90          *  -------------
  91          *  |    BLK 7  |
  92          *  -------------
  93          *  |    BLK 6  |
  94          *  -------------
  95          *  |    ...    |
  96          *  -------------
  97          *  |    BLK 0  |
  98          *  -------------
  99          *  Total of 8 blocks, each size 32KB
 100          */
 101
 102         /* set the block number */
 103         offset = UCD_UCODE_LOAD_BLOCK_NUM;
 104         nitrox_write_csr(ndev, offset, block_num);
 105
 106         code_size = roundup(ucode_size, 16);
 107         while (code_size) {
 108                 data = ucode_data[i];
 109                 /* write 8 bytes at a time */
 110                 offset = UCD_UCODE_LOAD_IDX_DATAX(i);
 111                 nitrox_write_csr(ndev, offset, data);
 112                 code_size -= 8;
 113                 i++;
 114         }
 115
 116         usleep_range(300, 400);
 117 }
 118
 119 static int nitrox_load_fw(struct nitrox_device *ndev)
 120 {
 121         const struct firmware *fw;
 122         const char *fw_name;
 123         struct ucode *ucode;
 124         u64 *ucode_data;
 125         u64 offset;
 126         union ucd_core_eid_ucode_block_num core_2_eid_val;
 127         union aqm_grp_execmsk_lo aqm_grp_execmask_lo;
 128         union aqm_grp_execmsk_hi aqm_grp_execmask_hi;
 129         u32 ucode_size;
 130         int ret, i = 0;
 131
 132         fw_name = SE_FW;
 133         dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
 134
 135         ret = request_firmware(&fw, fw_name, DEV(ndev));
 136         if (ret < 0) {
 137                 dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
 138                 return ret;
 139         }
 140
 141         ucode = (struct ucode *)fw->data;
 142
 143         ucode_size = be32_to_cpu(ucode->code_size) * 2;
 144         if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) {
 145                 dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n",
 146                         ucode_size, fw_name);
 147                 release_firmware(fw);
 148                 return -EINVAL;
 149         }
 150         ucode_data = ucode->code;
 151
 152         /* copy the firmware version */
 153         memcpy(&ndev->hw.fw_name[0][0], ucode->version, (VERSION_LEN - 2));
 154         ndev->hw.fw_name[0][VERSION_LEN - 1] = '\0';
 155
 156         /* Load SE Firmware on UCD Block 0 */
 157         write_to_ucd_unit(ndev, ucode_size, ucode_data, 0);
 158
 159         release_firmware(fw);
 160
 161         /* put all SE cores in DEFAULT_SE_GROUP */
 162         offset = POM_GRP_EXECMASKX(DEFAULT_SE_GROUP);
 163         nitrox_write_csr(ndev, offset, (~0ULL));
 164
 165         /* write block number and firmware length
 166          * bit:<2:0> block number
 167          * bit:3 is set SE uses 32KB microcode
 168          * bit:3 is clear SE uses 64KB microcode
 169          */
 170         core_2_eid_val.value = 0ULL;
 171         core_2_eid_val.ucode_blk = 0;
 172         if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE)
 173                 core_2_eid_val.ucode_len = 1;
 174         else
 175                 core_2_eid_val.ucode_len = 0;
 176
 177         for (i = 0; i < ndev->hw.se_cores; i++) {
 178                 offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
 179                 nitrox_write_csr(ndev, offset, core_2_eid_val.value);
 180         }
 181
 182
 183         fw_name = AE_FW;
 184         dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
 185
 186         ret = request_firmware(&fw, fw_name, DEV(ndev));
 187         if (ret < 0) {
 188                 dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
 189                 return ret;
 190         }
 191
 192         ucode = (struct ucode *)fw->data;
 193
 194         ucode_size = be32_to_cpu(ucode->code_size) * 2;
 195         if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) {
 196                 dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n",
 197                         ucode_size, fw_name);
 198                 release_firmware(fw);
 199                 return -EINVAL;
 200         }
 201         ucode_data = ucode->code;
 202
 203         /* copy the firmware version */
 204         memcpy(&ndev->hw.fw_name[1][0], ucode->version, (VERSION_LEN - 2));
 205         ndev->hw.fw_name[1][VERSION_LEN - 1] = '\0';
 206
 207         /* Load AE Firmware on UCD Block 2 */
 208         write_to_ucd_unit(ndev, ucode_size, ucode_data, 2);
 209
 210         release_firmware(fw);
 211
 212         /* put all AE cores in DEFAULT_AE_GROUP */
 213         offset = AQM_GRP_EXECMSK_LOX(DEFAULT_AE_GROUP);
 214         aqm_grp_execmask_lo.exec_0_to_39 = 0xFFFFFFFFFFULL;
 215         nitrox_write_csr(ndev, offset, aqm_grp_execmask_lo.value);
 216         offset = AQM_GRP_EXECMSK_HIX(DEFAULT_AE_GROUP);
 217         aqm_grp_execmask_hi.exec_40_to_79 = 0xFFFFFFFFFFULL;
 218         nitrox_write_csr(ndev, offset, aqm_grp_execmask_hi.value);
 219
 220         /* write block number and firmware length
 221          * bit:<2:0> block number
 222          * bit:3 is set AE uses 32KB microcode
 223          * bit:3 is clear AE uses 64KB microcode
 224          */
 225         core_2_eid_val.value = 0ULL;
 226         core_2_eid_val.ucode_blk = 2;
 227         if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE)
 228                 core_2_eid_val.ucode_len = 1;
 229         else
 230                 core_2_eid_val.ucode_len = 0;
 231
 232         for (i = 0; i < ndev->hw.ae_cores; i++) {
 233                 offset = UCD_AE_EID_UCODE_BLOCK_NUMX(i);
 234                 nitrox_write_csr(ndev, offset, core_2_eid_val.value);
 235         }
 236
 237         return 0;
 238 }
 239
 240 /**
 241  * nitrox_add_to_devlist - add NITROX device to global device list
 242  * @ndev: NITROX device
 243  */
 244 static int nitrox_add_to_devlist(struct nitrox_device *ndev)
 245 {
 246         struct nitrox_device *dev;
 247         int ret = 0;
 248
 249         INIT_LIST_HEAD(&ndev->list);
 250         refcount_set(&ndev->refcnt, 1);
 251
 252         mutex_lock(&devlist_lock);
 253         list_for_each_entry(dev, &ndevlist, list) {
 254                 if (dev == ndev) {
 255                         ret = -EEXIST;
 256                         goto unlock;
 257                 }
 258         }
 259         ndev->idx = num_devices++;
 260         list_add_tail(&ndev->list, &ndevlist);
 261 unlock:
 262         mutex_unlock(&devlist_lock);
 263         return ret;
 264 }
 265
 266 /**
 267  * nitrox_remove_from_devlist - remove NITROX device from
 268  *   global device list
 269  * @ndev: NITROX device
 270  */
 271 static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
 272 {
 273         mutex_lock(&devlist_lock);
 274         list_del(&ndev->list);
 275         num_devices--;
 276         mutex_unlock(&devlist_lock);
 277 }
 278
 279 struct nitrox_device *nitrox_get_first_device(void)
 280 {
 281         struct nitrox_device *ndev = NULL;
 282
 283         mutex_lock(&devlist_lock);
 284         list_for_each_entry(ndev, &ndevlist, list) {
 285                 if (nitrox_ready(ndev))
 286                         break;
 287         }
 288         mutex_unlock(&devlist_lock);
 289         if (!ndev)
 290                 return NULL;
 291
 292         refcount_inc(&ndev->refcnt);
 293         /* barrier to sync with other cpus */
 294         smp_mb__after_atomic();
 295         return ndev;
 296 }
 297
 298 void nitrox_put_device(struct nitrox_device *ndev)
 299 {
 300         if (!ndev)
 301                 return;
 302
 303         refcount_dec(&ndev->refcnt);
 304         /* barrier to sync with other cpus */
 305         smp_mb__after_atomic();
 306 }
 307
 308 static int nitrox_device_flr(struct pci_dev *pdev)
 309 {
 310         int pos = 0;
 311
 312         pos = pci_save_state(pdev);
 313         if (pos) {
 314                 dev_err(&pdev->dev, "Failed to save pci state\n");
 315                 return -ENOMEM;
 316         }
 317
 318         /* check flr support */
 319         if (pcie_has_flr(pdev))
 320                 pcie_flr(pdev);
 321
 322         pci_restore_state(pdev);
 323
 324         return 0;
 325 }
 326
 327 static int nitrox_pf_sw_init(struct nitrox_device *ndev)
 328 {
 329         int err;
 330
 331         err = nitrox_common_sw_init(ndev);
 332         if (err)
 333                 return err;
 334
 335         err = nitrox_register_interrupts(ndev);
 336         if (err)
 337                 nitrox_common_sw_cleanup(ndev);
 338
 339         return err;
 340 }
 341
 342 static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
 343 {
 344         nitrox_unregister_interrupts(ndev);
 345         nitrox_common_sw_cleanup(ndev);
 346 }
 347
 348 /**
 349  * nitrox_bist_check - Check NITORX BIST registers status
 350  * @ndev: NITROX device
 351  */
 352 static int nitrox_bist_check(struct nitrox_device *ndev)
 353 {
 354         u64 value = 0;
 355         int i;
 356
 357         for (i = 0; i < NR_CLUSTERS; i++) {
 358                 value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
 359                 value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
 360         }
 361         value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
 362         value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
 363         value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
 364         value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
 365         value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
 366         value += nitrox_read_csr(ndev, POM_BIST_REG);
 367         value += nitrox_read_csr(ndev, BMI_BIST_REG);
 368         value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
 369         value += nitrox_read_csr(ndev, BMO_BIST_REG);
 370         value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
 371         value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
 372         if (value)
 373                 return -EIO;
 374         return 0;
 375 }
 376
 377 static int nitrox_pf_hw_init(struct nitrox_device *ndev)
 378 {
 379         int err;
 380
 381         err = nitrox_bist_check(ndev);
 382         if (err) {
 383                 dev_err(&ndev->pdev->dev, "BIST check failed\n");
 384                 return err;
 385         }
 386         /* get cores information */
 387         nitrox_get_hwinfo(ndev);
 388
 389         nitrox_config_nps_core_unit(ndev);
 390         nitrox_config_aqm_unit(ndev);
 391         nitrox_config_nps_pkt_unit(ndev);
 392         nitrox_config_pom_unit(ndev);
 393         nitrox_config_efl_unit(ndev);
 394         /* configure IO units */
 395         nitrox_config_bmi_unit(ndev);
 396         nitrox_config_bmo_unit(ndev);
 397         /* configure Local Buffer Cache */
 398         nitrox_config_lbc_unit(ndev);
 399         nitrox_config_rand_unit(ndev);
 400
 401         /* load firmware on cores */
 402         err = nitrox_load_fw(ndev);
 403         if (err)
 404                 return err;
 405
 406         nitrox_config_emu_unit(ndev);
 407
 408         return 0;
 409 }
 410
 411 /**
 412  * nitrox_probe - NITROX Initialization function.
 413  * @pdev: PCI device information struct
 414  * @id: entry in nitrox_pci_tbl
 415  *
 416  * Return: 0, if the driver is bound to the device, or
 417  *         a negative error if there is failure.
 418  */
 419 static int nitrox_probe(struct pci_dev *pdev,
 420                         const struct pci_device_id *id)
 421 {
 422         struct nitrox_device *ndev;
 423         int err;
 424
 425         dev_info_once(&pdev->dev, "%s driver version %s\n",
 426                       nitrox_driver_name, DRIVER_VERSION);
 427
 428         err = pci_enable_device_mem(pdev);
 429         if (err)
 430                 return err;
 431
 432         /* do FLR */
 433         err = nitrox_device_flr(pdev);
 434         if (err) {
 435                 dev_err(&pdev->dev, "FLR failed\n");
 436                 pci_disable_device(pdev);
 437                 return err;
 438         }
 439
 440         if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
 441                 dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
 442         } else {
 443                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
 444                 if (err) {
 445                         dev_err(&pdev->dev, "DMA configuration failed\n");
 446                         pci_disable_device(pdev);
 447                         return err;
 448                 }
 449         }
 450
 451         err = pci_request_mem_regions(pdev, nitrox_driver_name);
 452         if (err) {
 453                 pci_disable_device(pdev);
 454                 return err;
 455         }
 456         pci_set_master(pdev);
 457
 458         ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
 459         if (!ndev) {
 460                 err = -ENOMEM;
 461                 goto ndev_fail;
 462         }
 463
 464         pci_set_drvdata(pdev, ndev);
 465         ndev->pdev = pdev;
 466
 467         /* add to device list */
 468         nitrox_add_to_devlist(ndev);
 469
 470         ndev->hw.vendor_id = pdev->vendor;
 471         ndev->hw.device_id = pdev->device;
 472         ndev->hw.revision_id = pdev->revision;
 473         /* command timeout in jiffies */
 474         ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
 475         ndev->node = dev_to_node(&pdev->dev);
 476         if (ndev->node == NUMA_NO_NODE)
 477                 ndev->node = 0;
 478
 479         ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
 480                                  pci_resource_len(pdev, 0));
 481         if (!ndev->bar_addr) {
 482                 err = -EIO;
 483                 goto ioremap_err;
 484         }
 485         /* allocate command queus based on cpus, max queues are 64 */
 486         ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
 487         ndev->qlen = qlen;
 488
 489         err = nitrox_pf_sw_init(ndev);
 490         if (err)
 491                 goto ioremap_err;
 492
 493         err = nitrox_pf_hw_init(ndev);
 494         if (err)
 495                 goto pf_hw_fail;
 496
 497         nitrox_debugfs_init(ndev);
 498
 499         /* clear the statistics */
 500         atomic64_set(&ndev->stats.posted, 0);
 501         atomic64_set(&ndev->stats.completed, 0);
 502         atomic64_set(&ndev->stats.dropped, 0);
 503
 504         atomic_set(&ndev->state, __NDEV_READY);
 505         /* barrier to sync with other cpus */
 506         smp_mb__after_atomic();
 507
 508         err = nitrox_crypto_register();
 509         if (err)
 510                 goto crypto_fail;
 511
 512         return 0;
 513
 514 crypto_fail:
 515         nitrox_debugfs_exit(ndev);
 516         atomic_set(&ndev->state, __NDEV_NOT_READY);
 517         /* barrier to sync with other cpus */
 518         smp_mb__after_atomic();
 519 pf_hw_fail:
 520         nitrox_pf_sw_cleanup(ndev);
 521 ioremap_err:
 522         nitrox_remove_from_devlist(ndev);
 523         kfree(ndev);
 524         pci_set_drvdata(pdev, NULL);
 525 ndev_fail:
 526         pci_release_mem_regions(pdev);
 527         pci_disable_device(pdev);
 528         return err;
 529 }
 530
 531 /**
 532  * nitrox_remove - Unbind the driver from the device.
 533  * @pdev: PCI device information struct
 534  */
 535 static void nitrox_remove(struct pci_dev *pdev)
 536 {
 537         struct nitrox_device *ndev = pci_get_drvdata(pdev);
 538
 539         if (!ndev)
 540                 return;
 541
 542         if (!refcount_dec_and_test(&ndev->refcnt)) {
 543                 dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
 544                         refcount_read(&ndev->refcnt));
 545                 return;
 546         }
 547
 548         dev_info(DEV(ndev), "Removing Device %x:%x\n",
 549                  ndev->hw.vendor_id, ndev->hw.device_id);
 550
 551         atomic_set(&ndev->state, __NDEV_NOT_READY);
 552         /* barrier to sync with other cpus */
 553         smp_mb__after_atomic();
 554
 555         nitrox_remove_from_devlist(ndev);
 556
 557 #ifdef CONFIG_PCI_IOV
 558         /* disable SR-IOV */
 559         nitrox_sriov_configure(pdev, 0);
 560 #endif
 561         nitrox_crypto_unregister();
 562         nitrox_debugfs_exit(ndev);
 563         nitrox_pf_sw_cleanup(ndev);
 564
 565         iounmap(ndev->bar_addr);
 566         kfree(ndev);
 567
 568         pci_set_drvdata(pdev, NULL);
 569         pci_release_mem_regions(pdev);
 570         pci_disable_device(pdev);
 571 }
 572
 573 static void nitrox_shutdown(struct pci_dev *pdev)
 574 {
 575         pci_set_drvdata(pdev, NULL);
 576         pci_release_mem_regions(pdev);
 577         pci_disable_device(pdev);
 578 }
 579
 580 static struct pci_driver nitrox_driver = {
 581         .name = nitrox_driver_name,
 582         .id_table = nitrox_pci_tbl,
 583         .probe = nitrox_probe,
 584         .remove = nitrox_remove,
 585         .shutdown = nitrox_shutdown,
 586 #ifdef CONFIG_PCI_IOV
 587         .sriov_configure = nitrox_sriov_configure,
 588 #endif
 589 };
 590
 591 module_pci_driver(nitrox_driver);
 592
 593 MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
 594 MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
 595 MODULE_LICENSE("GPL");
 596 MODULE_VERSION(DRIVER_VERSION);
 597 MODULE_FIRMWARE(SE_FW);