drivers/crypto/qat/qat_dh895xcc/adf_drv.c

   1 /*
   2   This file is provided under a dual BSD/GPLv2 license.  When using or
   3   redistributing this file, you may do so under either license.
   4
   5   GPL LICENSE SUMMARY
   6   Copyright(c) 2014 Intel Corporation.
   7   This program is free software; you can redistribute it and/or modify
   8   it under the terms of version 2 of the GNU General Public License as
   9   published by the Free Software Foundation.
  10
  11   This program is distributed in the hope that it will be useful, but
  12   WITHOUT ANY WARRANTY; without even the implied warranty of
  13   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14   General Public License for more details.
  15
  16   Contact Information:
  17   qat-linux@intel.com
  18
  19   BSD LICENSE
  20   Copyright(c) 2014 Intel Corporation.
  21   Redistribution and use in source and binary forms, with or without
  22   modification, are permitted provided that the following conditions
  23   are met:
  24
  25     * Redistributions of source code must retain the above copyright
  26       notice, this list of conditions and the following disclaimer.
  27     * Redistributions in binary form must reproduce the above copyright
  28       notice, this list of conditions and the following disclaimer in
  29       the documentation and/or other materials provided with the
  30       distribution.
  31     * Neither the name of Intel Corporation nor the names of its
  32       contributors may be used to endorse or promote products derived
  33       from this software without specific prior written permission.
  34
  35   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  36   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  37   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  38   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  39   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  40   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  41   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  42   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  43   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  44   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  45   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  46 */
  47 #include <linux/kernel.h>
  48 #include <linux/module.h>
  49 #include <linux/pci.h>
  50 #include <linux/init.h>
  51 #include <linux/types.h>
  52 #include <linux/fs.h>
  53 #include <linux/slab.h>
  54 #include <linux/errno.h>
  55 #include <linux/device.h>
  56 #include <linux/dma-mapping.h>
  57 #include <linux/platform_device.h>
  58 #include <linux/workqueue.h>
  59 #include <linux/io.h>
  60 #include <adf_accel_devices.h>
  61 #include <adf_common_drv.h>
  62 #include <adf_cfg.h>
  63 #include <adf_transport_access_macros.h>
  64 #include "adf_dh895xcc_hw_data.h"
  65 #include "adf_drv.h"
  66
  67 static const char adf_driver_name[] = ADF_DH895XCC_DEVICE_NAME;
  68
  69 #define ADF_SYSTEM_DEVICE(device_id) \
  70         {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
  71
  72 static const struct pci_device_id adf_pci_tbl[] = {
  73         ADF_SYSTEM_DEVICE(ADF_DH895XCC_PCI_DEVICE_ID),
  74         {0,}
  75 };
  76 MODULE_DEVICE_TABLE(pci, adf_pci_tbl);
  77
  78 static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent);
  79 static void adf_remove(struct pci_dev *dev);
  80
  81 static struct pci_driver adf_driver = {
  82         .id_table = adf_pci_tbl,
  83         .name = adf_driver_name,
  84         .probe = adf_probe,
  85         .remove = adf_remove,
  86         .sriov_configure = adf_sriov_configure,
  87 };
  88
  89 static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev)
  90 {
  91         pci_release_regions(accel_dev->accel_pci_dev.pci_dev);
  92         pci_disable_device(accel_dev->accel_pci_dev.pci_dev);
  93 }
  94
  95 static void adf_cleanup_accel(struct adf_accel_dev *accel_dev)
  96 {
  97         struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev;
  98         int i;
  99
 100         for (i = 0; i < ADF_PCI_MAX_BARS; i++) {
 101                 struct adf_bar *bar = &accel_pci_dev->pci_bars[i];
 102
 103                 if (bar->virt_addr)
 104                         pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr);
 105         }
 106
 107         if (accel_dev->hw_device) {
 108                 switch (accel_pci_dev->pci_dev->device) {
 109                 case ADF_DH895XCC_PCI_DEVICE_ID:
 110                         adf_clean_hw_data_dh895xcc(accel_dev->hw_device);
 111                         break;
 112                 default:
 113                         break;
 114                 }
 115                 kfree(accel_dev->hw_device);
 116                 accel_dev->hw_device = NULL;
 117         }
 118         adf_cfg_dev_remove(accel_dev);
 119         debugfs_remove(accel_dev->debugfs_dir);
 120         adf_devmgr_rm_dev(accel_dev, NULL);
 121 }
 122
 123 static int adf_dev_configure(struct adf_accel_dev *accel_dev)
 124 {
 125         int cpus = num_online_cpus();
 126         int banks = GET_MAX_BANKS(accel_dev);
 127         int instances = min(cpus, banks);
 128         char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
 129         int i;
 130         unsigned long val;
 131
 132         if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC))
 133                 goto err;
 134         if (adf_cfg_section_add(accel_dev, "Accelerator0"))
 135                 goto err;
 136         for (i = 0; i < instances; i++) {
 137                 val = i;
 138                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i);
 139                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 140                                                 key, (void *)&val, ADF_DEC))
 141                         goto err;
 142
 143                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY,
 144                          i);
 145                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 146                                                 key, (void *)&val, ADF_DEC))
 147                         goto err;
 148
 149                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i);
 150                 val = 128;
 151                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 152                                                 key, (void *)&val, ADF_DEC))
 153                         goto err;
 154
 155                 val = 512;
 156                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i);
 157                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 158                                                 key, (void *)&val, ADF_DEC))
 159                         goto err;
 160
 161                 val = 0;
 162                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i);
 163                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 164                                                 key, (void *)&val, ADF_DEC))
 165                         goto err;
 166
 167                 val = 2;
 168                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i);
 169                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 170                                                 key, (void *)&val, ADF_DEC))
 171                         goto err;
 172
 173                 val = 8;
 174                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i);
 175                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 176                                                 key, (void *)&val, ADF_DEC))
 177                         goto err;
 178
 179                 val = 10;
 180                 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i);
 181                 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 182                                                 key, (void *)&val, ADF_DEC))
 183                         goto err;
 184
 185                 val = ADF_COALESCING_DEF_TIME;
 186                 snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i);
 187                 if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0",
 188                                                 key, (void *)&val, ADF_DEC))
 189                         goto err;
 190         }
 191
 192         val = i;
 193         if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
 194                                         ADF_NUM_CY, (void *)&val, ADF_DEC))
 195                 goto err;
 196
 197         set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
 198         return 0;
 199 err:
 200         dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n");
 201         return -EINVAL;
 202 }
 203
 204 static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 205 {
 206         struct adf_accel_dev *accel_dev;
 207         struct adf_accel_pci *accel_pci_dev;
 208         struct adf_hw_device_data *hw_data;
 209         char name[ADF_DEVICE_NAME_LENGTH];
 210         unsigned int i, bar_nr;
 211         int ret, bar_mask;
 212
 213         switch (ent->device) {
 214         case ADF_DH895XCC_PCI_DEVICE_ID:
 215                 break;
 216         default:
 217                 dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device);
 218                 return -ENODEV;
 219         }
 220
 221         if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) {
 222                 /* If the accelerator is connected to a node with no memory
 223                  * there is no point in using the accelerator since the remote
 224                  * memory transaction will be very slow. */
 225                 dev_err(&pdev->dev, "Invalid NUMA configuration.\n");
 226                 return -EINVAL;
 227         }
 228
 229         accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL,
 230                                  dev_to_node(&pdev->dev));
 231         if (!accel_dev)
 232                 return -ENOMEM;
 233
 234         INIT_LIST_HEAD(&accel_dev->crypto_list);
 235         accel_pci_dev = &accel_dev->accel_pci_dev;
 236         accel_pci_dev->pci_dev = pdev;
 237
 238         /* Add accel device to accel table.
 239          * This should be called before adf_cleanup_accel is called */
 240         if (adf_devmgr_add_dev(accel_dev, NULL)) {
 241                 dev_err(&pdev->dev, "Failed to add new accelerator device.\n");
 242                 kfree(accel_dev);
 243                 return -EFAULT;
 244         }
 245
 246         accel_dev->owner = THIS_MODULE;
 247         /* Allocate and configure device configuration structure */
 248         hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL,
 249                                dev_to_node(&pdev->dev));
 250         if (!hw_data) {
 251                 ret = -ENOMEM;
 252                 goto out_err;
 253         }
 254
 255         accel_dev->hw_device = hw_data;
 256         switch (ent->device) {
 257         case ADF_DH895XCC_PCI_DEVICE_ID:
 258                 adf_init_hw_data_dh895xcc(accel_dev->hw_device);
 259                 break;
 260         default:
 261                 return -ENODEV;
 262         }
 263         pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid);
 264         pci_read_config_dword(pdev, ADF_DH895XCC_FUSECTL_OFFSET,
 265                               &hw_data->fuses);
 266
 267         /* Get Accelerators and Accelerators Engines masks */
 268         hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses);
 269         hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses);
 270         accel_pci_dev->sku = hw_data->get_sku(hw_data);
 271         /* If the device has no acceleration engines then ignore it. */
 272         if (!hw_data->accel_mask || !hw_data->ae_mask ||
 273             ((~hw_data->ae_mask) & 0x01)) {
 274                 dev_err(&pdev->dev, "No acceleration units found");
 275                 ret = -EFAULT;
 276                 goto out_err;
 277         }
 278
 279         /* Create dev top level debugfs entry */
 280         snprintf(name, sizeof(name), "%s%s_%02x:%02d.%02d",
 281                  ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name,
 282                  pdev->bus->number, PCI_SLOT(pdev->devfn),
 283                  PCI_FUNC(pdev->devfn));
 284
 285         accel_dev->debugfs_dir = debugfs_create_dir(name, NULL);
 286         if (!accel_dev->debugfs_dir) {
 287                 dev_err(&pdev->dev, "Could not create debugfs dir %s\n", name);
 288                 ret = -EINVAL;
 289                 goto out_err;
 290         }
 291
 292         /* Create device configuration table */
 293         ret = adf_cfg_dev_add(accel_dev);
 294         if (ret)
 295                 goto out_err;
 296
 297         pcie_set_readrq(pdev, 1024);
 298
 299         /* enable PCI device */
 300         if (pci_enable_device(pdev)) {
 301                 ret = -EFAULT;
 302                 goto out_err;
 303         }
 304
 305         /* set dma identifier */
 306         if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
 307                 if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) {
 308                         dev_err(&pdev->dev, "No usable DMA configuration\n");
 309                         ret = -EFAULT;
 310                         goto out_err_disable;
 311                 } else {
 312                         pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
 313                 }
 314
 315         } else {
 316                 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
 317         }
 318
 319         if (pci_request_regions(pdev, adf_driver_name)) {
 320                 ret = -EFAULT;
 321                 goto out_err_disable;
 322         }
 323
 324         /* Read accelerator capabilities mask */
 325         pci_read_config_dword(pdev, ADF_DH895XCC_LEGFUSE_OFFSET,
 326                               &hw_data->accel_capabilities_mask);
 327
 328         /* Find and map all the device's BARS */
 329         i = 0;
 330         bar_mask = pci_select_bars(pdev, IORESOURCE_MEM);
 331         for_each_set_bit(bar_nr, (const unsigned long *)&bar_mask,
 332                          ADF_PCI_MAX_BARS * 2) {
 333                 struct adf_bar *bar = &accel_pci_dev->pci_bars[i++];
 334
 335                 bar->base_addr = pci_resource_start(pdev, bar_nr);
 336                 if (!bar->base_addr)
 337                         break;
 338                 bar->size = pci_resource_len(pdev, bar_nr);
 339                 bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0);
 340                 if (!bar->virt_addr) {
 341                         dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr);
 342                         ret = -EFAULT;
 343                         goto out_err_free_reg;
 344                 }
 345         }
 346         pci_set_master(pdev);
 347
 348         if (adf_enable_aer(accel_dev, &adf_driver)) {
 349                 dev_err(&pdev->dev, "Failed to enable aer\n");
 350                 ret = -EFAULT;
 351                 goto out_err_free_reg;
 352         }
 353
 354         if (pci_save_state(pdev)) {
 355                 dev_err(&pdev->dev, "Failed to save pci state\n");
 356                 ret = -ENOMEM;
 357                 goto out_err_free_reg;
 358         }
 359
 360         ret = adf_dev_configure(accel_dev);
 361         if (ret)
 362                 goto out_err_free_reg;
 363
 364         ret = adf_dev_init(accel_dev);
 365         if (ret)
 366                 goto out_err_dev_shutdown;
 367
 368         ret = adf_dev_start(accel_dev);
 369         if (ret)
 370                 goto out_err_dev_stop;
 371
 372         return ret;
 373
 374 out_err_dev_stop:
 375         adf_dev_stop(accel_dev);
 376 out_err_dev_shutdown:
 377         adf_dev_shutdown(accel_dev);
 378 out_err_free_reg:
 379         pci_release_regions(accel_pci_dev->pci_dev);
 380 out_err_disable:
 381         pci_disable_device(accel_pci_dev->pci_dev);
 382 out_err:
 383         adf_cleanup_accel(accel_dev);
 384         kfree(accel_dev);
 385         return ret;
 386 }
 387
 388 static void adf_remove(struct pci_dev *pdev)
 389 {
 390         struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev);
 391
 392         if (!accel_dev) {
 393                 pr_err("QAT: Driver removal failed\n");
 394                 return;
 395         }
 396         if (adf_dev_stop(accel_dev))
 397                 dev_err(&GET_DEV(accel_dev), "Failed to stop QAT accel dev\n");
 398
 399         adf_dev_shutdown(accel_dev);
 400         adf_disable_aer(accel_dev);
 401         adf_cleanup_accel(accel_dev);
 402         adf_cleanup_pci_dev(accel_dev);
 403         kfree(accel_dev);
 404 }
 405
 406 static int __init adfdrv_init(void)
 407 {
 408         request_module("intel_qat");
 409
 410         if (pci_register_driver(&adf_driver)) {
 411                 pr_err("QAT: Driver initialization failed\n");
 412                 return -EFAULT;
 413         }
 414         return 0;
 415 }
 416
 417 static void __exit adfdrv_release(void)
 418 {
 419         pci_unregister_driver(&adf_driver);
 420 }
 421
 422 module_init(adfdrv_init);
 423 module_exit(adfdrv_release);
 424
 425 MODULE_LICENSE("Dual BSD/GPL");
 426 MODULE_AUTHOR("Intel");
 427 MODULE_FIRMWARE(ADF_DH895XCC_FW);
 428 MODULE_DESCRIPTION("Intel(R) QuickAssist Technology");
 429 MODULE_VERSION(ADF_DRV_VERSION);