drivers/crypto/ccree/cc_driver.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (C) 2012-2018 ARM Limited or its affiliates. */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/module.h>
   6
   7 #include <linux/crypto.h>
   8 #include <linux/moduleparam.h>
   9 #include <linux/types.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/platform_device.h>
  12 #include <linux/slab.h>
  13 #include <linux/spinlock.h>
  14 #include <linux/of.h>
  15 #include <linux/clk.h>
  16 #include <linux/of_address.h>
  17
  18 #include "cc_driver.h"
  19 #include "cc_request_mgr.h"
  20 #include "cc_buffer_mgr.h"
  21 #include "cc_debugfs.h"
  22 #include "cc_cipher.h"
  23 #include "cc_aead.h"
  24 #include "cc_hash.h"
  25 #include "cc_ivgen.h"
  26 #include "cc_sram_mgr.h"
  27 #include "cc_pm.h"
  28 #include "cc_fips.h"
  29
  30 bool cc_dump_desc;
  31 module_param_named(dump_desc, cc_dump_desc, bool, 0600);
  32 MODULE_PARM_DESC(cc_dump_desc, "Dump descriptors to kernel log as debugging aid");
  33
  34 bool cc_dump_bytes;
  35 module_param_named(dump_bytes, cc_dump_bytes, bool, 0600);
  36 MODULE_PARM_DESC(cc_dump_bytes, "Dump buffers to kernel log as debugging aid");
  37
  38 struct cc_hw_data {
  39         char *name;
  40         enum cc_hw_rev rev;
  41         u32 sig;
  42 };
  43
  44 /* Hardware revisions defs. */
  45
  46 static const struct cc_hw_data cc712_hw = {
  47         .name = "712", .rev = CC_HW_REV_712, .sig =  0xDCC71200U
  48 };
  49
  50 static const struct cc_hw_data cc710_hw = {
  51         .name = "710", .rev = CC_HW_REV_710, .sig =  0xDCC63200U
  52 };
  53
  54 static const struct cc_hw_data cc630p_hw = {
  55         .name = "630P", .rev = CC_HW_REV_630, .sig = 0xDCC63000U
  56 };
  57
  58 static const struct of_device_id arm_ccree_dev_of_match[] = {
  59         { .compatible = "arm,cryptocell-712-ree", .data = &cc712_hw },
  60         { .compatible = "arm,cryptocell-710-ree", .data = &cc710_hw },
  61         { .compatible = "arm,cryptocell-630p-ree", .data = &cc630p_hw },
  62         {}
  63 };
  64 MODULE_DEVICE_TABLE(of, arm_ccree_dev_of_match);
  65
  66 void __dump_byte_array(const char *name, const u8 *buf, size_t len)
  67 {
  68         char prefix[64];
  69
  70         if (!buf)
  71                 return;
  72
  73         snprintf(prefix, sizeof(prefix), "%s[%zu]: ", name, len);
  74
  75         print_hex_dump(KERN_DEBUG, prefix, DUMP_PREFIX_ADDRESS, 16, 1, buf,
  76                        len, false);
  77 }
  78
  79 static irqreturn_t cc_isr(int irq, void *dev_id)
  80 {
  81         struct cc_drvdata *drvdata = (struct cc_drvdata *)dev_id;
  82         struct device *dev = drvdata_to_dev(drvdata);
  83         u32 irr;
  84         u32 imr;
  85
  86         /* STAT_OP_TYPE_GENERIC STAT_PHASE_0: Interrupt */
  87
  88         /* read the interrupt status */
  89         irr = cc_ioread(drvdata, CC_REG(HOST_IRR));
  90         dev_dbg(dev, "Got IRR=0x%08X\n", irr);
  91         if (irr == 0) { /* Probably shared interrupt line */
  92                 dev_err(dev, "Got interrupt with empty IRR\n");
  93                 return IRQ_NONE;
  94         }
  95         imr = cc_ioread(drvdata, CC_REG(HOST_IMR));
  96
  97         /* clear interrupt - must be before processing events */
  98         cc_iowrite(drvdata, CC_REG(HOST_ICR), irr);
  99
 100         drvdata->irq = irr;
 101         /* Completion interrupt - most probable */
 102         if (irr & CC_COMP_IRQ_MASK) {
 103                 /* Mask AXI completion interrupt - will be unmasked in
 104                  * Deferred service handler
 105                  */
 106                 cc_iowrite(drvdata, CC_REG(HOST_IMR), imr | CC_COMP_IRQ_MASK);
 107                 irr &= ~CC_COMP_IRQ_MASK;
 108                 complete_request(drvdata);
 109         }
 110 #ifdef CONFIG_CRYPTO_FIPS
 111         /* TEE FIPS interrupt */
 112         if (irr & CC_GPR0_IRQ_MASK) {
 113                 /* Mask interrupt - will be unmasked in Deferred service
 114                  * handler
 115                  */
 116                 cc_iowrite(drvdata, CC_REG(HOST_IMR), imr | CC_GPR0_IRQ_MASK);
 117                 irr &= ~CC_GPR0_IRQ_MASK;
 118                 fips_handler(drvdata);
 119         }
 120 #endif
 121         /* AXI error interrupt */
 122         if (irr & CC_AXI_ERR_IRQ_MASK) {
 123                 u32 axi_err;
 124
 125                 /* Read the AXI error ID */
 126                 axi_err = cc_ioread(drvdata, CC_REG(AXIM_MON_ERR));
 127                 dev_dbg(dev, "AXI completion error: axim_mon_err=0x%08X\n",
 128                         axi_err);
 129
 130                 irr &= ~CC_AXI_ERR_IRQ_MASK;
 131         }
 132
 133         if (irr) {
 134                 dev_dbg(dev, "IRR includes unknown cause bits (0x%08X)\n",
 135                         irr);
 136                 /* Just warning */
 137         }
 138
 139         return IRQ_HANDLED;
 140 }
 141
 142 int init_cc_regs(struct cc_drvdata *drvdata, bool is_probe)
 143 {
 144         unsigned int val, cache_params;
 145         struct device *dev = drvdata_to_dev(drvdata);
 146
 147         /* Unmask all AXI interrupt sources AXI_CFG1 register */
 148         val = cc_ioread(drvdata, CC_REG(AXIM_CFG));
 149         cc_iowrite(drvdata, CC_REG(AXIM_CFG), val & ~CC_AXI_IRQ_MASK);
 150         dev_dbg(dev, "AXIM_CFG=0x%08X\n",
 151                 cc_ioread(drvdata, CC_REG(AXIM_CFG)));
 152
 153         /* Clear all pending interrupts */
 154         val = cc_ioread(drvdata, CC_REG(HOST_IRR));
 155         dev_dbg(dev, "IRR=0x%08X\n", val);
 156         cc_iowrite(drvdata, CC_REG(HOST_ICR), val);
 157
 158         /* Unmask relevant interrupt cause */
 159         val = CC_COMP_IRQ_MASK | CC_AXI_ERR_IRQ_MASK;
 160
 161         if (drvdata->hw_rev >= CC_HW_REV_712)
 162                 val |= CC_GPR0_IRQ_MASK;
 163
 164         cc_iowrite(drvdata, CC_REG(HOST_IMR), ~val);
 165
 166         cache_params = (drvdata->coherent ? CC_COHERENT_CACHE_PARAMS : 0x0);
 167
 168         val = cc_ioread(drvdata, CC_REG(AXIM_CACHE_PARAMS));
 169
 170         if (is_probe)
 171                 dev_info(dev, "Cache params previous: 0x%08X\n", val);
 172
 173         cc_iowrite(drvdata, CC_REG(AXIM_CACHE_PARAMS), cache_params);
 174         val = cc_ioread(drvdata, CC_REG(AXIM_CACHE_PARAMS));
 175
 176         if (is_probe)
 177                 dev_info(dev, "Cache params current: 0x%08X (expect: 0x%08X)\n",
 178                          val, cache_params);
 179
 180         return 0;
 181 }
 182
 183 static int init_cc_resources(struct platform_device *plat_dev)
 184 {
 185         struct resource *req_mem_cc_regs = NULL;
 186         struct cc_drvdata *new_drvdata;
 187         struct device *dev = &plat_dev->dev;
 188         struct device_node *np = dev->of_node;
 189         u32 signature_val;
 190         u64 dma_mask;
 191         const struct cc_hw_data *hw_rev;
 192         const struct of_device_id *dev_id;
 193         int rc = 0;
 194
 195         new_drvdata = devm_kzalloc(dev, sizeof(*new_drvdata), GFP_KERNEL);
 196         if (!new_drvdata)
 197                 return -ENOMEM;
 198
 199         dev_id = of_match_node(arm_ccree_dev_of_match, np);
 200         if (!dev_id)
 201                 return -ENODEV;
 202
 203         hw_rev = (struct cc_hw_data *)dev_id->data;
 204         new_drvdata->hw_rev_name = hw_rev->name;
 205         new_drvdata->hw_rev = hw_rev->rev;
 206
 207         if (hw_rev->rev >= CC_HW_REV_712) {
 208                 new_drvdata->hash_len_sz = HASH_LEN_SIZE_712;
 209                 new_drvdata->axim_mon_offset = CC_REG(AXIM_MON_COMP);
 210         } else {
 211                 new_drvdata->hash_len_sz = HASH_LEN_SIZE_630;
 212                 new_drvdata->axim_mon_offset = CC_REG(AXIM_MON_COMP8);
 213         }
 214
 215         platform_set_drvdata(plat_dev, new_drvdata);
 216         new_drvdata->plat_dev = plat_dev;
 217
 218         new_drvdata->clk = of_clk_get(np, 0);
 219         new_drvdata->coherent = of_dma_is_coherent(np);
 220
 221         /* Get device resources */
 222         /* First CC registers space */
 223         req_mem_cc_regs = platform_get_resource(plat_dev, IORESOURCE_MEM, 0);
 224         /* Map registers space */
 225         new_drvdata->cc_base = devm_ioremap_resource(dev, req_mem_cc_regs);
 226         if (IS_ERR(new_drvdata->cc_base)) {
 227                 dev_err(dev, "Failed to ioremap registers");
 228                 return PTR_ERR(new_drvdata->cc_base);
 229         }
 230
 231         dev_dbg(dev, "Got MEM resource (%s): %pR\n", req_mem_cc_regs->name,
 232                 req_mem_cc_regs);
 233         dev_dbg(dev, "CC registers mapped from %pa to 0x%p\n",
 234                 &req_mem_cc_regs->start, new_drvdata->cc_base);
 235
 236         /* Then IRQ */
 237         new_drvdata->irq = platform_get_irq(plat_dev, 0);
 238         if (new_drvdata->irq < 0) {
 239                 dev_err(dev, "Failed getting IRQ resource\n");
 240                 return new_drvdata->irq;
 241         }
 242
 243         rc = devm_request_irq(dev, new_drvdata->irq, cc_isr,
 244                               IRQF_SHARED, "ccree", new_drvdata);
 245         if (rc) {
 246                 dev_err(dev, "Could not register to interrupt %d\n",
 247                         new_drvdata->irq);
 248                 return rc;
 249         }
 250         dev_dbg(dev, "Registered to IRQ: %d\n", new_drvdata->irq);
 251
 252         init_completion(&new_drvdata->hw_queue_avail);
 253
 254         if (!plat_dev->dev.dma_mask)
 255                 plat_dev->dev.dma_mask = &plat_dev->dev.coherent_dma_mask;
 256
 257         dma_mask = DMA_BIT_MASK(DMA_BIT_MASK_LEN);
 258         while (dma_mask > 0x7fffffffUL) {
 259                 if (dma_supported(&plat_dev->dev, dma_mask)) {
 260                         rc = dma_set_coherent_mask(&plat_dev->dev, dma_mask);
 261                         if (!rc)
 262                                 break;
 263                 }
 264                 dma_mask >>= 1;
 265         }
 266
 267         if (rc) {
 268                 dev_err(dev, "Failed in dma_set_mask, mask=%pad\n", &dma_mask);
 269                 return rc;
 270         }
 271
 272         rc = cc_clk_on(new_drvdata);
 273         if (rc) {
 274                 dev_err(dev, "Failed to enable clock");
 275                 return rc;
 276         }
 277
 278         /* Verify correct mapping */
 279         signature_val = cc_ioread(new_drvdata, CC_REG(HOST_SIGNATURE));
 280         if (signature_val != hw_rev->sig) {
 281                 dev_err(dev, "Invalid CC signature: SIGNATURE=0x%08X != expected=0x%08X\n",
 282                         signature_val, hw_rev->sig);
 283                 rc = -EINVAL;
 284                 goto post_clk_err;
 285         }
 286         dev_dbg(dev, "CC SIGNATURE=0x%08X\n", signature_val);
 287
 288         /* Display HW versions */
 289         dev_info(dev, "ARM CryptoCell %s Driver: HW version 0x%08X, Driver version %s\n",
 290                  hw_rev->name, cc_ioread(new_drvdata, CC_REG(HOST_VERSION)),
 291                  DRV_MODULE_VERSION);
 292
 293         rc = init_cc_regs(new_drvdata, true);
 294         if (rc) {
 295                 dev_err(dev, "init_cc_regs failed\n");
 296                 goto post_clk_err;
 297         }
 298
 299         rc = cc_debugfs_init(new_drvdata);
 300         if (rc) {
 301                 dev_err(dev, "Failed registering debugfs interface\n");
 302                 goto post_regs_err;
 303         }
 304
 305         rc = cc_fips_init(new_drvdata);
 306         if (rc) {
 307                 dev_err(dev, "CC_FIPS_INIT failed 0x%x\n", rc);
 308                 goto post_debugfs_err;
 309         }
 310         rc = cc_sram_mgr_init(new_drvdata);
 311         if (rc) {
 312                 dev_err(dev, "cc_sram_mgr_init failed\n");
 313                 goto post_fips_init_err;
 314         }
 315
 316         new_drvdata->mlli_sram_addr =
 317                 cc_sram_alloc(new_drvdata, MAX_MLLI_BUFF_SIZE);
 318         if (new_drvdata->mlli_sram_addr == NULL_SRAM_ADDR) {
 319                 dev_err(dev, "Failed to alloc MLLI Sram buffer\n");
 320                 rc = -ENOMEM;
 321                 goto post_sram_mgr_err;
 322         }
 323
 324         rc = cc_req_mgr_init(new_drvdata);
 325         if (rc) {
 326                 dev_err(dev, "cc_req_mgr_init failed\n");
 327                 goto post_sram_mgr_err;
 328         }
 329
 330         rc = cc_buffer_mgr_init(new_drvdata);
 331         if (rc) {
 332                 dev_err(dev, "buffer_mgr_init failed\n");
 333                 goto post_req_mgr_err;
 334         }
 335
 336         rc = cc_pm_init(new_drvdata);
 337         if (rc) {
 338                 dev_err(dev, "ssi_power_mgr_init failed\n");
 339                 goto post_buf_mgr_err;
 340         }
 341
 342         rc = cc_ivgen_init(new_drvdata);
 343         if (rc) {
 344                 dev_err(dev, "cc_ivgen_init failed\n");
 345                 goto post_power_mgr_err;
 346         }
 347
 348         /* Allocate crypto algs */
 349         rc = cc_cipher_alloc(new_drvdata);
 350         if (rc) {
 351                 dev_err(dev, "cc_cipher_alloc failed\n");
 352                 goto post_ivgen_err;
 353         }
 354
 355         /* hash must be allocated before aead since hash exports APIs */
 356         rc = cc_hash_alloc(new_drvdata);
 357         if (rc) {
 358                 dev_err(dev, "cc_hash_alloc failed\n");
 359                 goto post_cipher_err;
 360         }
 361
 362         rc = cc_aead_alloc(new_drvdata);
 363         if (rc) {
 364                 dev_err(dev, "cc_aead_alloc failed\n");
 365                 goto post_hash_err;
 366         }
 367
 368         /* If we got here and FIPS mode is enabled
 369          * it means all FIPS test passed, so let TEE
 370          * know we're good.
 371          */
 372         cc_set_ree_fips_status(new_drvdata, true);
 373
 374         return 0;
 375
 376 post_hash_err:
 377         cc_hash_free(new_drvdata);
 378 post_cipher_err:
 379         cc_cipher_free(new_drvdata);
 380 post_ivgen_err:
 381         cc_ivgen_fini(new_drvdata);
 382 post_power_mgr_err:
 383         cc_pm_fini(new_drvdata);
 384 post_buf_mgr_err:
 385          cc_buffer_mgr_fini(new_drvdata);
 386 post_req_mgr_err:
 387         cc_req_mgr_fini(new_drvdata);
 388 post_sram_mgr_err:
 389         cc_sram_mgr_fini(new_drvdata);
 390 post_fips_init_err:
 391         cc_fips_fini(new_drvdata);
 392 post_debugfs_err:
 393         cc_debugfs_fini(new_drvdata);
 394 post_regs_err:
 395         fini_cc_regs(new_drvdata);
 396 post_clk_err:
 397         cc_clk_off(new_drvdata);
 398         return rc;
 399 }
 400
 401 void fini_cc_regs(struct cc_drvdata *drvdata)
 402 {
 403         /* Mask all interrupts */
 404         cc_iowrite(drvdata, CC_REG(HOST_IMR), 0xFFFFFFFF);
 405 }
 406
 407 static void cleanup_cc_resources(struct platform_device *plat_dev)
 408 {
 409         struct cc_drvdata *drvdata =
 410                 (struct cc_drvdata *)platform_get_drvdata(plat_dev);
 411
 412         cc_aead_free(drvdata);
 413         cc_hash_free(drvdata);
 414         cc_cipher_free(drvdata);
 415         cc_ivgen_fini(drvdata);
 416         cc_pm_fini(drvdata);
 417         cc_buffer_mgr_fini(drvdata);
 418         cc_req_mgr_fini(drvdata);
 419         cc_sram_mgr_fini(drvdata);
 420         cc_fips_fini(drvdata);
 421         cc_debugfs_fini(drvdata);
 422         fini_cc_regs(drvdata);
 423         cc_clk_off(drvdata);
 424 }
 425
 426 int cc_clk_on(struct cc_drvdata *drvdata)
 427 {
 428         struct clk *clk = drvdata->clk;
 429         int rc;
 430
 431         if (IS_ERR(clk))
 432                 /* Not all devices have a clock associated with CCREE  */
 433                 return 0;
 434
 435         rc = clk_prepare_enable(clk);
 436         if (rc)
 437                 return rc;
 438
 439         return 0;
 440 }
 441
 442 void cc_clk_off(struct cc_drvdata *drvdata)
 443 {
 444         struct clk *clk = drvdata->clk;
 445
 446         if (IS_ERR(clk))
 447                 /* Not all devices have a clock associated with CCREE */
 448                 return;
 449
 450         clk_disable_unprepare(clk);
 451 }
 452
 453 static int ccree_probe(struct platform_device *plat_dev)
 454 {
 455         int rc;
 456         struct device *dev = &plat_dev->dev;
 457
 458         /* Map registers space */
 459         rc = init_cc_resources(plat_dev);
 460         if (rc)
 461                 return rc;
 462
 463         dev_info(dev, "ARM ccree device initialized\n");
 464
 465         return 0;
 466 }
 467
 468 static int ccree_remove(struct platform_device *plat_dev)
 469 {
 470         struct device *dev = &plat_dev->dev;
 471
 472         dev_dbg(dev, "Releasing ccree resources...\n");
 473
 474         cleanup_cc_resources(plat_dev);
 475
 476         dev_info(dev, "ARM ccree device terminated\n");
 477
 478         return 0;
 479 }
 480
 481 static struct platform_driver ccree_driver = {
 482         .driver = {
 483                    .name = "ccree",
 484                    .of_match_table = arm_ccree_dev_of_match,
 485 #ifdef CONFIG_PM
 486                    .pm = &ccree_pm,
 487 #endif
 488         },
 489         .probe = ccree_probe,
 490         .remove = ccree_remove,
 491 };
 492
 493 static int __init ccree_init(void)
 494 {
 495         int ret;
 496
 497         cc_hash_global_init();
 498
 499         ret = cc_debugfs_global_init();
 500         if (ret)
 501                 return ret;
 502
 503         return platform_driver_register(&ccree_driver);
 504 }
 505 module_init(ccree_init);
 506
 507 static void __exit ccree_exit(void)
 508 {
 509         platform_driver_unregister(&ccree_driver);
 510         cc_debugfs_global_fini();
 511 }
 512 module_exit(ccree_exit);
 513
 514 /* Module description */
 515 MODULE_DESCRIPTION("ARM TrustZone CryptoCell REE Driver");
 516 MODULE_VERSION(DRV_MODULE_VERSION);
 517 MODULE_AUTHOR("ARM");
 518 MODULE_LICENSE("GPL v2");