drivers/crypto/cavium/nitrox/nitrox_main.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 #include <linux/delay.h>
   3 #include <linux/firmware.h>
   4 #include <linux/list.h>
   5 #include <linux/module.h>
   6 #include <linux/mutex.h>
   7 #include <linux/pci.h>
   8 #include <linux/pci_ids.h>
   9
  10 #include "nitrox_dev.h"
  11 #include "nitrox_common.h"
  12 #include "nitrox_csr.h"
  13 #include "nitrox_hal.h"
  14 #include "nitrox_isr.h"
  15 #include "nitrox_debugfs.h"
  16
  17 #define CNN55XX_DEV_ID  0x12
  18 #define UCODE_HLEN 48
  19 #define DEFAULT_SE_GROUP 0
  20 #define DEFAULT_AE_GROUP 0
  21
  22 #define DRIVER_VERSION "1.2"
  23 #define CNN55XX_UCD_BLOCK_SIZE 32768
  24 #define CNN55XX_MAX_UCODE_SIZE (CNN55XX_UCD_BLOCK_SIZE * 2)
  25 #define FW_DIR "cavium/"
  26 /* SE microcode */
  27 #define SE_FW   FW_DIR "cnn55xx_se.fw"
  28 /* AE microcode */
  29 #define AE_FW   FW_DIR "cnn55xx_ae.fw"
  30
  31 static const char nitrox_driver_name[] = "CNN55XX";
  32
  33 static LIST_HEAD(ndevlist);
  34 static DEFINE_MUTEX(devlist_lock);
  35 static unsigned int num_devices;
  36
  37 /*
  38  * nitrox_pci_tbl - PCI Device ID Table
  39  */
  40 static const struct pci_device_id nitrox_pci_tbl[] = {
  41         {PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
  42         /* required last entry */
  43         {0, }
  44 };
  45 MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);
  46
  47 static unsigned int qlen = DEFAULT_CMD_QLEN;
  48 module_param(qlen, uint, 0644);
  49 MODULE_PARM_DESC(qlen, "Command queue length - default 2048");
  50
  51 /**
  52  * struct ucode - Firmware Header
  53  * @id: microcode ID
  54  * @version: firmware version
  55  * @code_size: code section size
  56  * @raz: alignment
  57  * @code: code section
  58  */
  59 struct ucode {
  60         u8 id;
  61         char version[VERSION_LEN - 1];
  62         __be32 code_size;
  63         u8 raz[12];
  64         u64 code[];
  65 };
  66
  67 /*
  68  * write_to_ucd_unit - Write Firmware to NITROX UCD unit
  69  */
  70 static void write_to_ucd_unit(struct nitrox_device *ndev, u32 ucode_size,
  71                               u64 *ucode_data, int block_num)
  72 {
  73         u32 code_size;
  74         u64 offset, data;
  75         int i = 0;
  76
  77         /*
  78          * UCD structure
  79          *
  80          *  -------------
  81          *  |    BLK 7  |
  82          *  -------------
  83          *  |    BLK 6  |
  84          *  -------------
  85          *  |    ...    |
  86          *  -------------
  87          *  |    BLK 0  |
  88          *  -------------
  89          *  Total of 8 blocks, each size 32KB
  90          */
  91
  92         /* set the block number */
  93         offset = UCD_UCODE_LOAD_BLOCK_NUM;
  94         nitrox_write_csr(ndev, offset, block_num);
  95
  96         code_size = roundup(ucode_size, 16);
  97         while (code_size) {
  98                 data = ucode_data[i];
  99                 /* write 8 bytes at a time */
 100                 offset = UCD_UCODE_LOAD_IDX_DATAX(i);
 101                 nitrox_write_csr(ndev, offset, data);
 102                 code_size -= 8;
 103                 i++;
 104         }
 105
 106         usleep_range(300, 400);
 107 }
 108
 109 static int nitrox_load_fw(struct nitrox_device *ndev)
 110 {
 111         const struct firmware *fw;
 112         const char *fw_name;
 113         struct ucode *ucode;
 114         u64 *ucode_data;
 115         u64 offset;
 116         union ucd_core_eid_ucode_block_num core_2_eid_val;
 117         union aqm_grp_execmsk_lo aqm_grp_execmask_lo;
 118         union aqm_grp_execmsk_hi aqm_grp_execmask_hi;
 119         u32 ucode_size;
 120         int ret, i = 0;
 121
 122         fw_name = SE_FW;
 123         dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
 124
 125         ret = request_firmware(&fw, fw_name, DEV(ndev));
 126         if (ret < 0) {
 127                 dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
 128                 return ret;
 129         }
 130
 131         ucode = (struct ucode *)fw->data;
 132
 133         ucode_size = be32_to_cpu(ucode->code_size) * 2;
 134         if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) {
 135                 dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n",
 136                         ucode_size, fw_name);
 137                 release_firmware(fw);
 138                 return -EINVAL;
 139         }
 140         ucode_data = ucode->code;
 141
 142         /* copy the firmware version */
 143         memcpy(&ndev->hw.fw_name[0][0], ucode->version, (VERSION_LEN - 2));
 144         ndev->hw.fw_name[0][VERSION_LEN - 1] = '\0';
 145
 146         /* Load SE Firmware on UCD Block 0 */
 147         write_to_ucd_unit(ndev, ucode_size, ucode_data, 0);
 148
 149         release_firmware(fw);
 150
 151         /* put all SE cores in DEFAULT_SE_GROUP */
 152         offset = POM_GRP_EXECMASKX(DEFAULT_SE_GROUP);
 153         nitrox_write_csr(ndev, offset, (~0ULL));
 154
 155         /* write block number and firmware length
 156          * bit:<2:0> block number
 157          * bit:3 is set SE uses 32KB microcode
 158          * bit:3 is clear SE uses 64KB microcode
 159          */
 160         core_2_eid_val.value = 0ULL;
 161         core_2_eid_val.ucode_blk = 0;
 162         if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE)
 163                 core_2_eid_val.ucode_len = 1;
 164         else
 165                 core_2_eid_val.ucode_len = 0;
 166
 167         for (i = 0; i < ndev->hw.se_cores; i++) {
 168                 offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
 169                 nitrox_write_csr(ndev, offset, core_2_eid_val.value);
 170         }
 171
 172
 173         fw_name = AE_FW;
 174         dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
 175
 176         ret = request_firmware(&fw, fw_name, DEV(ndev));
 177         if (ret < 0) {
 178                 dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
 179                 return ret;
 180         }
 181
 182         ucode = (struct ucode *)fw->data;
 183
 184         ucode_size = be32_to_cpu(ucode->code_size) * 2;
 185         if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) {
 186                 dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n",
 187                         ucode_size, fw_name);
 188                 release_firmware(fw);
 189                 return -EINVAL;
 190         }
 191         ucode_data = ucode->code;
 192
 193         /* copy the firmware version */
 194         memcpy(&ndev->hw.fw_name[1][0], ucode->version, (VERSION_LEN - 2));
 195         ndev->hw.fw_name[1][VERSION_LEN - 1] = '\0';
 196
 197         /* Load AE Firmware on UCD Block 2 */
 198         write_to_ucd_unit(ndev, ucode_size, ucode_data, 2);
 199
 200         release_firmware(fw);
 201
 202         /* put all AE cores in DEFAULT_AE_GROUP */
 203         offset = AQM_GRP_EXECMSK_LOX(DEFAULT_AE_GROUP);
 204         aqm_grp_execmask_lo.exec_0_to_39 = 0xFFFFFFFFFFULL;
 205         nitrox_write_csr(ndev, offset, aqm_grp_execmask_lo.value);
 206         offset = AQM_GRP_EXECMSK_HIX(DEFAULT_AE_GROUP);
 207         aqm_grp_execmask_hi.exec_40_to_79 = 0xFFFFFFFFFFULL;
 208         nitrox_write_csr(ndev, offset, aqm_grp_execmask_hi.value);
 209
 210         /* write block number and firmware length
 211          * bit:<2:0> block number
 212          * bit:3 is set AE uses 32KB microcode
 213          * bit:3 is clear AE uses 64KB microcode
 214          */
 215         core_2_eid_val.value = 0ULL;
 216         core_2_eid_val.ucode_blk = 2;
 217         if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE)
 218                 core_2_eid_val.ucode_len = 1;
 219         else
 220                 core_2_eid_val.ucode_len = 0;
 221
 222         for (i = 0; i < ndev->hw.ae_cores; i++) {
 223                 offset = UCD_AE_EID_UCODE_BLOCK_NUMX(i);
 224                 nitrox_write_csr(ndev, offset, core_2_eid_val.value);
 225         }
 226
 227         return 0;
 228 }
 229
 230 /**
 231  * nitrox_add_to_devlist - add NITROX device to global device list
 232  * @ndev: NITROX device
 233  */
 234 static int nitrox_add_to_devlist(struct nitrox_device *ndev)
 235 {
 236         struct nitrox_device *dev;
 237         int ret = 0;
 238
 239         INIT_LIST_HEAD(&ndev->list);
 240         refcount_set(&ndev->refcnt, 1);
 241
 242         mutex_lock(&devlist_lock);
 243         list_for_each_entry(dev, &ndevlist, list) {
 244                 if (dev == ndev) {
 245                         ret = -EEXIST;
 246                         goto unlock;
 247                 }
 248         }
 249         ndev->idx = num_devices++;
 250         list_add_tail(&ndev->list, &ndevlist);
 251 unlock:
 252         mutex_unlock(&devlist_lock);
 253         return ret;
 254 }
 255
 256 /**
 257  * nitrox_remove_from_devlist - remove NITROX device from
 258  *   global device list
 259  * @ndev: NITROX device
 260  */
 261 static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
 262 {
 263         mutex_lock(&devlist_lock);
 264         list_del(&ndev->list);
 265         num_devices--;
 266         mutex_unlock(&devlist_lock);
 267 }
 268
 269 struct nitrox_device *nitrox_get_first_device(void)
 270 {
 271         struct nitrox_device *ndev = NULL, *iter;
 272
 273         mutex_lock(&devlist_lock);
 274         list_for_each_entry(iter, &ndevlist, list) {
 275                 if (nitrox_ready(iter)) {
 276                         ndev = iter;
 277                         break;
 278                 }
 279         }
 280         mutex_unlock(&devlist_lock);
 281         if (!ndev)
 282                 return NULL;
 283
 284         refcount_inc(&ndev->refcnt);
 285         /* barrier to sync with other cpus */
 286         smp_mb__after_atomic();
 287         return ndev;
 288 }
 289
 290 void nitrox_put_device(struct nitrox_device *ndev)
 291 {
 292         if (!ndev)
 293                 return;
 294
 295         refcount_dec(&ndev->refcnt);
 296         /* barrier to sync with other cpus */
 297         smp_mb__after_atomic();
 298 }
 299
 300 static int nitrox_device_flr(struct pci_dev *pdev)
 301 {
 302         int pos = 0;
 303
 304         pos = pci_save_state(pdev);
 305         if (pos) {
 306                 dev_err(&pdev->dev, "Failed to save pci state\n");
 307                 return -ENOMEM;
 308         }
 309
 310         pcie_reset_flr(pdev, PCI_RESET_DO_RESET);
 311
 312         pci_restore_state(pdev);
 313
 314         return 0;
 315 }
 316
 317 static int nitrox_pf_sw_init(struct nitrox_device *ndev)
 318 {
 319         int err;
 320
 321         err = nitrox_common_sw_init(ndev);
 322         if (err)
 323                 return err;
 324
 325         err = nitrox_register_interrupts(ndev);
 326         if (err)
 327                 nitrox_common_sw_cleanup(ndev);
 328
 329         return err;
 330 }
 331
 332 static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
 333 {
 334         nitrox_unregister_interrupts(ndev);
 335         nitrox_common_sw_cleanup(ndev);
 336 }
 337
 338 /**
 339  * nitrox_bist_check - Check NITROX BIST registers status
 340  * @ndev: NITROX device
 341  */
 342 static int nitrox_bist_check(struct nitrox_device *ndev)
 343 {
 344         u64 value = 0;
 345         int i;
 346
 347         for (i = 0; i < NR_CLUSTERS; i++) {
 348                 value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
 349                 value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
 350         }
 351         value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
 352         value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
 353         value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
 354         value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
 355         value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
 356         value += nitrox_read_csr(ndev, POM_BIST_REG);
 357         value += nitrox_read_csr(ndev, BMI_BIST_REG);
 358         value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
 359         value += nitrox_read_csr(ndev, BMO_BIST_REG);
 360         value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
 361         value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
 362         if (value)
 363                 return -EIO;
 364         return 0;
 365 }
 366
 367 static int nitrox_pf_hw_init(struct nitrox_device *ndev)
 368 {
 369         int err;
 370
 371         err = nitrox_bist_check(ndev);
 372         if (err) {
 373                 dev_err(&ndev->pdev->dev, "BIST check failed\n");
 374                 return err;
 375         }
 376         /* get cores information */
 377         nitrox_get_hwinfo(ndev);
 378
 379         nitrox_config_nps_core_unit(ndev);
 380         nitrox_config_aqm_unit(ndev);
 381         nitrox_config_nps_pkt_unit(ndev);
 382         nitrox_config_pom_unit(ndev);
 383         nitrox_config_efl_unit(ndev);
 384         /* configure IO units */
 385         nitrox_config_bmi_unit(ndev);
 386         nitrox_config_bmo_unit(ndev);
 387         /* configure Local Buffer Cache */
 388         nitrox_config_lbc_unit(ndev);
 389         nitrox_config_rand_unit(ndev);
 390
 391         /* load firmware on cores */
 392         err = nitrox_load_fw(ndev);
 393         if (err)
 394                 return err;
 395
 396         nitrox_config_emu_unit(ndev);
 397
 398         return 0;
 399 }
 400
 401 /**
 402  * nitrox_probe - NITROX Initialization function.
 403  * @pdev: PCI device information struct
 404  * @id: entry in nitrox_pci_tbl
 405  *
 406  * Return: 0, if the driver is bound to the device, or
 407  *         a negative error if there is failure.
 408  */
 409 static int nitrox_probe(struct pci_dev *pdev,
 410                         const struct pci_device_id *id)
 411 {
 412         struct nitrox_device *ndev;
 413         int err;
 414
 415         dev_info_once(&pdev->dev, "%s driver version %s\n",
 416                       nitrox_driver_name, DRIVER_VERSION);
 417
 418         err = pci_enable_device_mem(pdev);
 419         if (err)
 420                 return err;
 421
 422         /* do FLR */
 423         err = nitrox_device_flr(pdev);
 424         if (err) {
 425                 dev_err(&pdev->dev, "FLR failed\n");
 426                 goto flr_fail;
 427         }
 428
 429         if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
 430                 dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
 431         } else {
 432                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
 433                 if (err) {
 434                         dev_err(&pdev->dev, "DMA configuration failed\n");
 435                         goto flr_fail;
 436                 }
 437         }
 438
 439         err = pci_request_mem_regions(pdev, nitrox_driver_name);
 440         if (err)
 441                 goto flr_fail;
 442         pci_set_master(pdev);
 443
 444         ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
 445         if (!ndev) {
 446                 err = -ENOMEM;
 447                 goto ndev_fail;
 448         }
 449
 450         pci_set_drvdata(pdev, ndev);
 451         ndev->pdev = pdev;
 452
 453         /* add to device list */
 454         nitrox_add_to_devlist(ndev);
 455
 456         ndev->hw.vendor_id = pdev->vendor;
 457         ndev->hw.device_id = pdev->device;
 458         ndev->hw.revision_id = pdev->revision;
 459         /* command timeout in jiffies */
 460         ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
 461         ndev->node = dev_to_node(&pdev->dev);
 462         if (ndev->node == NUMA_NO_NODE)
 463                 ndev->node = 0;
 464
 465         ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
 466                                  pci_resource_len(pdev, 0));
 467         if (!ndev->bar_addr) {
 468                 err = -EIO;
 469                 goto ioremap_err;
 470         }
 471         /* allocate command queus based on cpus, max queues are 64 */
 472         ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
 473         ndev->qlen = qlen;
 474
 475         err = nitrox_pf_sw_init(ndev);
 476         if (err)
 477                 goto pf_sw_fail;
 478
 479         err = nitrox_pf_hw_init(ndev);
 480         if (err)
 481                 goto pf_hw_fail;
 482
 483         nitrox_debugfs_init(ndev);
 484
 485         /* clear the statistics */
 486         atomic64_set(&ndev->stats.posted, 0);
 487         atomic64_set(&ndev->stats.completed, 0);
 488         atomic64_set(&ndev->stats.dropped, 0);
 489
 490         atomic_set(&ndev->state, __NDEV_READY);
 491         /* barrier to sync with other cpus */
 492         smp_mb__after_atomic();
 493
 494         err = nitrox_crypto_register();
 495         if (err)
 496                 goto crypto_fail;
 497
 498         return 0;
 499
 500 crypto_fail:
 501         nitrox_debugfs_exit(ndev);
 502         atomic_set(&ndev->state, __NDEV_NOT_READY);
 503         /* barrier to sync with other cpus */
 504         smp_mb__after_atomic();
 505 pf_hw_fail:
 506         nitrox_pf_sw_cleanup(ndev);
 507 pf_sw_fail:
 508         iounmap(ndev->bar_addr);
 509 ioremap_err:
 510         nitrox_remove_from_devlist(ndev);
 511         kfree(ndev);
 512         pci_set_drvdata(pdev, NULL);
 513 ndev_fail:
 514         pci_release_mem_regions(pdev);
 515 flr_fail:
 516         pci_disable_device(pdev);
 517         return err;
 518 }
 519
 520 /**
 521  * nitrox_remove - Unbind the driver from the device.
 522  * @pdev: PCI device information struct
 523  */
 524 static void nitrox_remove(struct pci_dev *pdev)
 525 {
 526         struct nitrox_device *ndev = pci_get_drvdata(pdev);
 527
 528         if (!ndev)
 529                 return;
 530
 531         if (!refcount_dec_and_test(&ndev->refcnt)) {
 532                 dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
 533                         refcount_read(&ndev->refcnt));
 534                 return;
 535         }
 536
 537         dev_info(DEV(ndev), "Removing Device %x:%x\n",
 538                  ndev->hw.vendor_id, ndev->hw.device_id);
 539
 540         atomic_set(&ndev->state, __NDEV_NOT_READY);
 541         /* barrier to sync with other cpus */
 542         smp_mb__after_atomic();
 543
 544         nitrox_remove_from_devlist(ndev);
 545
 546         /* disable SR-IOV */
 547         nitrox_sriov_configure(pdev, 0);
 548         nitrox_crypto_unregister();
 549         nitrox_debugfs_exit(ndev);
 550         nitrox_pf_sw_cleanup(ndev);
 551
 552         iounmap(ndev->bar_addr);
 553         kfree(ndev);
 554
 555         pci_set_drvdata(pdev, NULL);
 556         pci_release_mem_regions(pdev);
 557         pci_disable_device(pdev);
 558 }
 559
 560 static void nitrox_shutdown(struct pci_dev *pdev)
 561 {
 562         pci_set_drvdata(pdev, NULL);
 563         pci_release_mem_regions(pdev);
 564         pci_disable_device(pdev);
 565 }
 566
 567 static struct pci_driver nitrox_driver = {
 568         .name = nitrox_driver_name,
 569         .id_table = nitrox_pci_tbl,
 570         .probe = nitrox_probe,
 571         .remove = nitrox_remove,
 572         .shutdown = nitrox_shutdown,
 573         .sriov_configure = nitrox_sriov_configure,
 574 };
 575
 576 module_pci_driver(nitrox_driver);
 577
 578 MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
 579 MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
 580 MODULE_LICENSE("GPL");
 581 MODULE_VERSION(DRIVER_VERSION);
 582 MODULE_FIRMWARE(SE_FW);