drivers/memory/tegra/mc.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2014 NVIDIA CORPORATION.  All rights reserved.
   4  */
   5
   6 #include <linux/clk.h>
   7 #include <linux/delay.h>
   8 #include <linux/dma-mapping.h>
   9 #include <linux/interrupt.h>
  10 #include <linux/kernel.h>
  11 #include <linux/module.h>
  12 #include <linux/of.h>
  13 #include <linux/of_device.h>
  14 #include <linux/platform_device.h>
  15 #include <linux/slab.h>
  16 #include <linux/sort.h>
  17
  18 #include <soc/tegra/fuse.h>
  19
  20 #include "mc.h"
  21
  22 static const struct of_device_id tegra_mc_of_match[] = {
  23 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
  24         { .compatible = "nvidia,tegra20-mc-gart", .data = &tegra20_mc_soc },
  25 #endif
  26 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
  27         { .compatible = "nvidia,tegra30-mc", .data = &tegra30_mc_soc },
  28 #endif
  29 #ifdef CONFIG_ARCH_TEGRA_114_SOC
  30         { .compatible = "nvidia,tegra114-mc", .data = &tegra114_mc_soc },
  31 #endif
  32 #ifdef CONFIG_ARCH_TEGRA_124_SOC
  33         { .compatible = "nvidia,tegra124-mc", .data = &tegra124_mc_soc },
  34 #endif
  35 #ifdef CONFIG_ARCH_TEGRA_132_SOC
  36         { .compatible = "nvidia,tegra132-mc", .data = &tegra132_mc_soc },
  37 #endif
  38 #ifdef CONFIG_ARCH_TEGRA_210_SOC
  39         { .compatible = "nvidia,tegra210-mc", .data = &tegra210_mc_soc },
  40 #endif
  41         { }
  42 };
  43 MODULE_DEVICE_TABLE(of, tegra_mc_of_match);
  44
  45 static int tegra_mc_block_dma_common(struct tegra_mc *mc,
  46                                      const struct tegra_mc_reset *rst)
  47 {
  48         unsigned long flags;
  49         u32 value;
  50
  51         spin_lock_irqsave(&mc->lock, flags);
  52
  53         value = mc_readl(mc, rst->control) | BIT(rst->bit);
  54         mc_writel(mc, value, rst->control);
  55
  56         spin_unlock_irqrestore(&mc->lock, flags);
  57
  58         return 0;
  59 }
  60
  61 static bool tegra_mc_dma_idling_common(struct tegra_mc *mc,
  62                                        const struct tegra_mc_reset *rst)
  63 {
  64         return (mc_readl(mc, rst->status) & BIT(rst->bit)) != 0;
  65 }
  66
  67 static int tegra_mc_unblock_dma_common(struct tegra_mc *mc,
  68                                        const struct tegra_mc_reset *rst)
  69 {
  70         unsigned long flags;
  71         u32 value;
  72
  73         spin_lock_irqsave(&mc->lock, flags);
  74
  75         value = mc_readl(mc, rst->control) & ~BIT(rst->bit);
  76         mc_writel(mc, value, rst->control);
  77
  78         spin_unlock_irqrestore(&mc->lock, flags);
  79
  80         return 0;
  81 }
  82
  83 static int tegra_mc_reset_status_common(struct tegra_mc *mc,
  84                                         const struct tegra_mc_reset *rst)
  85 {
  86         return (mc_readl(mc, rst->control) & BIT(rst->bit)) != 0;
  87 }
  88
  89 const struct tegra_mc_reset_ops tegra_mc_reset_ops_common = {
  90         .block_dma = tegra_mc_block_dma_common,
  91         .dma_idling = tegra_mc_dma_idling_common,
  92         .unblock_dma = tegra_mc_unblock_dma_common,
  93         .reset_status = tegra_mc_reset_status_common,
  94 };
  95
  96 static inline struct tegra_mc *reset_to_mc(struct reset_controller_dev *rcdev)
  97 {
  98         return container_of(rcdev, struct tegra_mc, reset);
  99 }
 100
 101 static const struct tegra_mc_reset *tegra_mc_reset_find(struct tegra_mc *mc,
 102                                                         unsigned long id)
 103 {
 104         unsigned int i;
 105
 106         for (i = 0; i < mc->soc->num_resets; i++)
 107                 if (mc->soc->resets[i].id == id)
 108                         return &mc->soc->resets[i];
 109
 110         return NULL;
 111 }
 112
 113 static int tegra_mc_hotreset_assert(struct reset_controller_dev *rcdev,
 114                                     unsigned long id)
 115 {
 116         struct tegra_mc *mc = reset_to_mc(rcdev);
 117         const struct tegra_mc_reset_ops *rst_ops;
 118         const struct tegra_mc_reset *rst;
 119         int retries = 500;
 120         int err;
 121
 122         rst = tegra_mc_reset_find(mc, id);
 123         if (!rst)
 124                 return -ENODEV;
 125
 126         rst_ops = mc->soc->reset_ops;
 127         if (!rst_ops)
 128                 return -ENODEV;
 129
 130         if (rst_ops->block_dma) {
 131                 /* block clients DMA requests */
 132                 err = rst_ops->block_dma(mc, rst);
 133                 if (err) {
 134                         dev_err(mc->dev, "failed to block %s DMA: %d\n",
 135                                 rst->name, err);
 136                         return err;
 137                 }
 138         }
 139
 140         if (rst_ops->dma_idling) {
 141                 /* wait for completion of the outstanding DMA requests */
 142                 while (!rst_ops->dma_idling(mc, rst)) {
 143                         if (!retries--) {
 144                                 dev_err(mc->dev, "failed to flush %s DMA\n",
 145                                         rst->name);
 146                                 return -EBUSY;
 147                         }
 148
 149                         usleep_range(10, 100);
 150                 }
 151         }
 152
 153         if (rst_ops->hotreset_assert) {
 154                 /* clear clients DMA requests sitting before arbitration */
 155                 err = rst_ops->hotreset_assert(mc, rst);
 156                 if (err) {
 157                         dev_err(mc->dev, "failed to hot reset %s: %d\n",
 158                                 rst->name, err);
 159                         return err;
 160                 }
 161         }
 162
 163         return 0;
 164 }
 165
 166 static int tegra_mc_hotreset_deassert(struct reset_controller_dev *rcdev,
 167                                       unsigned long id)
 168 {
 169         struct tegra_mc *mc = reset_to_mc(rcdev);
 170         const struct tegra_mc_reset_ops *rst_ops;
 171         const struct tegra_mc_reset *rst;
 172         int err;
 173
 174         rst = tegra_mc_reset_find(mc, id);
 175         if (!rst)
 176                 return -ENODEV;
 177
 178         rst_ops = mc->soc->reset_ops;
 179         if (!rst_ops)
 180                 return -ENODEV;
 181
 182         if (rst_ops->hotreset_deassert) {
 183                 /* take out client from hot reset */
 184                 err = rst_ops->hotreset_deassert(mc, rst);
 185                 if (err) {
 186                         dev_err(mc->dev, "failed to deassert hot reset %s: %d\n",
 187                                 rst->name, err);
 188                         return err;
 189                 }
 190         }
 191
 192         if (rst_ops->unblock_dma) {
 193                 /* allow new DMA requests to proceed to arbitration */
 194                 err = rst_ops->unblock_dma(mc, rst);
 195                 if (err) {
 196                         dev_err(mc->dev, "failed to unblock %s DMA : %d\n",
 197                                 rst->name, err);
 198                         return err;
 199                 }
 200         }
 201
 202         return 0;
 203 }
 204
 205 static int tegra_mc_hotreset_status(struct reset_controller_dev *rcdev,
 206                                     unsigned long id)
 207 {
 208         struct tegra_mc *mc = reset_to_mc(rcdev);
 209         const struct tegra_mc_reset_ops *rst_ops;
 210         const struct tegra_mc_reset *rst;
 211
 212         rst = tegra_mc_reset_find(mc, id);
 213         if (!rst)
 214                 return -ENODEV;
 215
 216         rst_ops = mc->soc->reset_ops;
 217         if (!rst_ops)
 218                 return -ENODEV;
 219
 220         return rst_ops->reset_status(mc, rst);
 221 }
 222
 223 static const struct reset_control_ops tegra_mc_reset_ops = {
 224         .assert = tegra_mc_hotreset_assert,
 225         .deassert = tegra_mc_hotreset_deassert,
 226         .status = tegra_mc_hotreset_status,
 227 };
 228
 229 static int tegra_mc_reset_setup(struct tegra_mc *mc)
 230 {
 231         int err;
 232
 233         mc->reset.ops = &tegra_mc_reset_ops;
 234         mc->reset.owner = THIS_MODULE;
 235         mc->reset.of_node = mc->dev->of_node;
 236         mc->reset.of_reset_n_cells = 1;
 237         mc->reset.nr_resets = mc->soc->num_resets;
 238
 239         err = reset_controller_register(&mc->reset);
 240         if (err < 0)
 241                 return err;
 242
 243         return 0;
 244 }
 245
 246 static int tegra_mc_setup_latency_allowance(struct tegra_mc *mc)
 247 {
 248         unsigned long long tick;
 249         unsigned int i;
 250         u32 value;
 251
 252         /* compute the number of MC clock cycles per tick */
 253         tick = (unsigned long long)mc->tick * clk_get_rate(mc->clk);
 254         do_div(tick, NSEC_PER_SEC);
 255
 256         value = mc_readl(mc, MC_EMEM_ARB_CFG);
 257         value &= ~MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK;
 258         value |= MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(tick);
 259         mc_writel(mc, value, MC_EMEM_ARB_CFG);
 260
 261         /* write latency allowance defaults */
 262         for (i = 0; i < mc->soc->num_clients; i++) {
 263                 const struct tegra_mc_la *la = &mc->soc->clients[i].la;
 264                 u32 value;
 265
 266                 value = mc_readl(mc, la->reg);
 267                 value &= ~(la->mask << la->shift);
 268                 value |= (la->def & la->mask) << la->shift;
 269                 mc_writel(mc, value, la->reg);
 270         }
 271
 272         /* latch new values */
 273         mc_writel(mc, MC_TIMING_UPDATE, MC_TIMING_CONTROL);
 274
 275         return 0;
 276 }
 277
 278 int tegra_mc_write_emem_configuration(struct tegra_mc *mc, unsigned long rate)
 279 {
 280         unsigned int i;
 281         struct tegra_mc_timing *timing = NULL;
 282
 283         for (i = 0; i < mc->num_timings; i++) {
 284                 if (mc->timings[i].rate == rate) {
 285                         timing = &mc->timings[i];
 286                         break;
 287                 }
 288         }
 289
 290         if (!timing) {
 291                 dev_err(mc->dev, "no memory timing registered for rate %lu\n",
 292                         rate);
 293                 return -EINVAL;
 294         }
 295
 296         for (i = 0; i < mc->soc->num_emem_regs; ++i)
 297                 mc_writel(mc, timing->emem_data[i], mc->soc->emem_regs[i]);
 298
 299         return 0;
 300 }
 301
 302 unsigned int tegra_mc_get_emem_device_count(struct tegra_mc *mc)
 303 {
 304         u8 dram_count;
 305
 306         dram_count = mc_readl(mc, MC_EMEM_ADR_CFG);
 307         dram_count &= MC_EMEM_ADR_CFG_EMEM_NUMDEV;
 308         dram_count++;
 309
 310         return dram_count;
 311 }
 312
 313 static int load_one_timing(struct tegra_mc *mc,
 314                            struct tegra_mc_timing *timing,
 315                            struct device_node *node)
 316 {
 317         int err;
 318         u32 tmp;
 319
 320         err = of_property_read_u32(node, "clock-frequency", &tmp);
 321         if (err) {
 322                 dev_err(mc->dev,
 323                         "timing %pOFn: failed to read rate\n", node);
 324                 return err;
 325         }
 326
 327         timing->rate = tmp;
 328         timing->emem_data = devm_kcalloc(mc->dev, mc->soc->num_emem_regs,
 329                                          sizeof(u32), GFP_KERNEL);
 330         if (!timing->emem_data)
 331                 return -ENOMEM;
 332
 333         err = of_property_read_u32_array(node, "nvidia,emem-configuration",
 334                                          timing->emem_data,
 335                                          mc->soc->num_emem_regs);
 336         if (err) {
 337                 dev_err(mc->dev,
 338                         "timing %pOFn: failed to read EMEM configuration\n",
 339                         node);
 340                 return err;
 341         }
 342
 343         return 0;
 344 }
 345
 346 static int load_timings(struct tegra_mc *mc, struct device_node *node)
 347 {
 348         struct device_node *child;
 349         struct tegra_mc_timing *timing;
 350         int child_count = of_get_child_count(node);
 351         int i = 0, err;
 352
 353         mc->timings = devm_kcalloc(mc->dev, child_count, sizeof(*timing),
 354                                    GFP_KERNEL);
 355         if (!mc->timings)
 356                 return -ENOMEM;
 357
 358         mc->num_timings = child_count;
 359
 360         for_each_child_of_node(node, child) {
 361                 timing = &mc->timings[i++];
 362
 363                 err = load_one_timing(mc, timing, child);
 364                 if (err) {
 365                         of_node_put(child);
 366                         return err;
 367                 }
 368         }
 369
 370         return 0;
 371 }
 372
 373 static int tegra_mc_setup_timings(struct tegra_mc *mc)
 374 {
 375         struct device_node *node;
 376         u32 ram_code, node_ram_code;
 377         int err;
 378
 379         ram_code = tegra_read_ram_code();
 380
 381         mc->num_timings = 0;
 382
 383         for_each_child_of_node(mc->dev->of_node, node) {
 384                 err = of_property_read_u32(node, "nvidia,ram-code",
 385                                            &node_ram_code);
 386                 if (err || (node_ram_code != ram_code))
 387                         continue;
 388
 389                 err = load_timings(mc, node);
 390                 of_node_put(node);
 391                 if (err)
 392                         return err;
 393                 break;
 394         }
 395
 396         if (mc->num_timings == 0)
 397                 dev_warn(mc->dev,
 398                          "no memory timings for RAM code %u registered\n",
 399                          ram_code);
 400
 401         return 0;
 402 }
 403
 404 static const char *const status_names[32] = {
 405         [ 1] = "External interrupt",
 406         [ 6] = "EMEM address decode error",
 407         [ 7] = "GART page fault",
 408         [ 8] = "Security violation",
 409         [ 9] = "EMEM arbitration error",
 410         [10] = "Page fault",
 411         [11] = "Invalid APB ASID update",
 412         [12] = "VPR violation",
 413         [13] = "Secure carveout violation",
 414         [16] = "MTS carveout violation",
 415 };
 416
 417 static const char *const error_names[8] = {
 418         [2] = "EMEM decode error",
 419         [3] = "TrustZone violation",
 420         [4] = "Carveout violation",
 421         [6] = "SMMU translation error",
 422 };
 423
 424 static irqreturn_t tegra_mc_irq(int irq, void *data)
 425 {
 426         struct tegra_mc *mc = data;
 427         unsigned long status;
 428         unsigned int bit;
 429
 430         /* mask all interrupts to avoid flooding */
 431         status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
 432         if (!status)
 433                 return IRQ_NONE;
 434
 435         for_each_set_bit(bit, &status, 32) {
 436                 const char *error = status_names[bit] ?: "unknown";
 437                 const char *client = "unknown", *desc;
 438                 const char *direction, *secure;
 439                 phys_addr_t addr = 0;
 440                 unsigned int i;
 441                 char perm[7];
 442                 u8 id, type;
 443                 u32 value;
 444
 445                 value = mc_readl(mc, MC_ERR_STATUS);
 446
 447 #ifdef CONFIG_PHYS_ADDR_T_64BIT
 448                 if (mc->soc->num_address_bits > 32) {
 449                         addr = ((value >> MC_ERR_STATUS_ADR_HI_SHIFT) &
 450                                 MC_ERR_STATUS_ADR_HI_MASK);
 451                         addr <<= 32;
 452                 }
 453 #endif
 454
 455                 if (value & MC_ERR_STATUS_RW)
 456                         direction = "write";
 457                 else
 458                         direction = "read";
 459
 460                 if (value & MC_ERR_STATUS_SECURITY)
 461                         secure = "secure ";
 462                 else
 463                         secure = "";
 464
 465                 id = value & mc->soc->client_id_mask;
 466
 467                 for (i = 0; i < mc->soc->num_clients; i++) {
 468                         if (mc->soc->clients[i].id == id) {
 469                                 client = mc->soc->clients[i].name;
 470                                 break;
 471                         }
 472                 }
 473
 474                 type = (value & MC_ERR_STATUS_TYPE_MASK) >>
 475                        MC_ERR_STATUS_TYPE_SHIFT;
 476                 desc = error_names[type];
 477
 478                 switch (value & MC_ERR_STATUS_TYPE_MASK) {
 479                 case MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE:
 480                         perm[0] = ' ';
 481                         perm[1] = '[';
 482
 483                         if (value & MC_ERR_STATUS_READABLE)
 484                                 perm[2] = 'R';
 485                         else
 486                                 perm[2] = '-';
 487
 488                         if (value & MC_ERR_STATUS_WRITABLE)
 489                                 perm[3] = 'W';
 490                         else
 491                                 perm[3] = '-';
 492
 493                         if (value & MC_ERR_STATUS_NONSECURE)
 494                                 perm[4] = '-';
 495                         else
 496                                 perm[4] = 'S';
 497
 498                         perm[5] = ']';
 499                         perm[6] = '\0';
 500                         break;
 501
 502                 default:
 503                         perm[0] = '\0';
 504                         break;
 505                 }
 506
 507                 value = mc_readl(mc, MC_ERR_ADR);
 508                 addr |= value;
 509
 510                 dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s%s)\n",
 511                                     client, secure, direction, &addr, error,
 512                                     desc, perm);
 513         }
 514
 515         /* clear interrupts */
 516         mc_writel(mc, status, MC_INTSTATUS);
 517
 518         return IRQ_HANDLED;
 519 }
 520
 521 static __maybe_unused irqreturn_t tegra20_mc_irq(int irq, void *data)
 522 {
 523         struct tegra_mc *mc = data;
 524         unsigned long status;
 525         unsigned int bit;
 526
 527         /* mask all interrupts to avoid flooding */
 528         status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
 529         if (!status)
 530                 return IRQ_NONE;
 531
 532         for_each_set_bit(bit, &status, 32) {
 533                 const char *direction = "read", *secure = "";
 534                 const char *error = status_names[bit];
 535                 const char *client, *desc;
 536                 phys_addr_t addr;
 537                 u32 value, reg;
 538                 u8 id, type;
 539
 540                 switch (BIT(bit)) {
 541                 case MC_INT_DECERR_EMEM:
 542                         reg = MC_DECERR_EMEM_OTHERS_STATUS;
 543                         value = mc_readl(mc, reg);
 544
 545                         id = value & mc->soc->client_id_mask;
 546                         desc = error_names[2];
 547
 548                         if (value & BIT(31))
 549                                 direction = "write";
 550                         break;
 551
 552                 case MC_INT_INVALID_GART_PAGE:
 553                         reg = MC_GART_ERROR_REQ;
 554                         value = mc_readl(mc, reg);
 555
 556                         id = (value >> 1) & mc->soc->client_id_mask;
 557                         desc = error_names[2];
 558
 559                         if (value & BIT(0))
 560                                 direction = "write";
 561                         break;
 562
 563                 case MC_INT_SECURITY_VIOLATION:
 564                         reg = MC_SECURITY_VIOLATION_STATUS;
 565                         value = mc_readl(mc, reg);
 566
 567                         id = value & mc->soc->client_id_mask;
 568                         type = (value & BIT(30)) ? 4 : 3;
 569                         desc = error_names[type];
 570                         secure = "secure ";
 571
 572                         if (value & BIT(31))
 573                                 direction = "write";
 574                         break;
 575
 576                 default:
 577                         continue;
 578                 }
 579
 580                 client = mc->soc->clients[id].name;
 581                 addr = mc_readl(mc, reg + sizeof(u32));
 582
 583                 dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s)\n",
 584                                     client, secure, direction, &addr, error,
 585                                     desc);
 586         }
 587
 588         /* clear interrupts */
 589         mc_writel(mc, status, MC_INTSTATUS);
 590
 591         return IRQ_HANDLED;
 592 }
 593
 594 static int tegra_mc_probe(struct platform_device *pdev)
 595 {
 596         struct resource *res;
 597         struct tegra_mc *mc;
 598         void *isr;
 599         u64 mask;
 600         int err;
 601
 602         mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL);
 603         if (!mc)
 604                 return -ENOMEM;
 605
 606         platform_set_drvdata(pdev, mc);
 607         spin_lock_init(&mc->lock);
 608         mc->soc = of_device_get_match_data(&pdev->dev);
 609         mc->dev = &pdev->dev;
 610
 611         mask = DMA_BIT_MASK(mc->soc->num_address_bits);
 612
 613         err = dma_coerce_mask_and_coherent(&pdev->dev, mask);
 614         if (err < 0) {
 615                 dev_err(&pdev->dev, "failed to set DMA mask: %d\n", err);
 616                 return err;
 617         }
 618
 619         /* length of MC tick in nanoseconds */
 620         mc->tick = 30;
 621
 622         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 623         mc->regs = devm_ioremap_resource(&pdev->dev, res);
 624         if (IS_ERR(mc->regs))
 625                 return PTR_ERR(mc->regs);
 626
 627         mc->clk = devm_clk_get(&pdev->dev, "mc");
 628         if (IS_ERR(mc->clk)) {
 629                 dev_err(&pdev->dev, "failed to get MC clock: %ld\n",
 630                         PTR_ERR(mc->clk));
 631                 return PTR_ERR(mc->clk);
 632         }
 633
 634 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
 635         if (mc->soc == &tegra20_mc_soc) {
 636                 isr = tegra20_mc_irq;
 637         } else
 638 #endif
 639         {
 640                 /* ensure that debug features are disabled */
 641                 mc_writel(mc, 0x00000000, MC_TIMING_CONTROL_DBG);
 642
 643                 err = tegra_mc_setup_latency_allowance(mc);
 644                 if (err < 0) {
 645                         dev_err(&pdev->dev,
 646                                 "failed to setup latency allowance: %d\n",
 647                                 err);
 648                         return err;
 649                 }
 650
 651                 isr = tegra_mc_irq;
 652
 653                 err = tegra_mc_setup_timings(mc);
 654                 if (err < 0) {
 655                         dev_err(&pdev->dev, "failed to setup timings: %d\n",
 656                                 err);
 657                         return err;
 658                 }
 659         }
 660
 661         mc->irq = platform_get_irq(pdev, 0);
 662         if (mc->irq < 0) {
 663                 dev_err(&pdev->dev, "interrupt not specified\n");
 664                 return mc->irq;
 665         }
 666
 667         WARN(!mc->soc->client_id_mask, "missing client ID mask for this SoC\n");
 668
 669         mc_writel(mc, mc->soc->intmask, MC_INTMASK);
 670
 671         err = devm_request_irq(&pdev->dev, mc->irq, isr, 0,
 672                                dev_name(&pdev->dev), mc);
 673         if (err < 0) {
 674                 dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", mc->irq,
 675                         err);
 676                 return err;
 677         }
 678
 679         err = tegra_mc_reset_setup(mc);
 680         if (err < 0)
 681                 dev_err(&pdev->dev, "failed to register reset controller: %d\n",
 682                         err);
 683
 684         if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU) && mc->soc->smmu) {
 685                 mc->smmu = tegra_smmu_probe(&pdev->dev, mc->soc->smmu, mc);
 686                 if (IS_ERR(mc->smmu)) {
 687                         dev_err(&pdev->dev, "failed to probe SMMU: %ld\n",
 688                                 PTR_ERR(mc->smmu));
 689                         mc->smmu = NULL;
 690                 }
 691         }
 692
 693         if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && !mc->soc->smmu) {
 694                 mc->gart = tegra_gart_probe(&pdev->dev, mc);
 695                 if (IS_ERR(mc->gart)) {
 696                         dev_err(&pdev->dev, "failed to probe GART: %ld\n",
 697                                 PTR_ERR(mc->gart));
 698                         mc->gart = NULL;
 699                 }
 700         }
 701
 702         return 0;
 703 }
 704
 705 static int tegra_mc_suspend(struct device *dev)
 706 {
 707         struct tegra_mc *mc = dev_get_drvdata(dev);
 708         int err;
 709
 710         if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && mc->gart) {
 711                 err = tegra_gart_suspend(mc->gart);
 712                 if (err)
 713                         return err;
 714         }
 715
 716         return 0;
 717 }
 718
 719 static int tegra_mc_resume(struct device *dev)
 720 {
 721         struct tegra_mc *mc = dev_get_drvdata(dev);
 722         int err;
 723
 724         if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && mc->gart) {
 725                 err = tegra_gart_resume(mc->gart);
 726                 if (err)
 727                         return err;
 728         }
 729
 730         return 0;
 731 }
 732
 733 static const struct dev_pm_ops tegra_mc_pm_ops = {
 734         .suspend = tegra_mc_suspend,
 735         .resume = tegra_mc_resume,
 736 };
 737
 738 static struct platform_driver tegra_mc_driver = {
 739         .driver = {
 740                 .name = "tegra-mc",
 741                 .of_match_table = tegra_mc_of_match,
 742                 .pm = &tegra_mc_pm_ops,
 743                 .suppress_bind_attrs = true,
 744         },
 745         .prevent_deferred_probe = true,
 746         .probe = tegra_mc_probe,
 747 };
 748
 749 static int tegra_mc_init(void)
 750 {
 751         return platform_driver_register(&tegra_mc_driver);
 752 }
 753 arch_initcall(tegra_mc_init);
 754
 755 MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
 756 MODULE_DESCRIPTION("NVIDIA Tegra Memory Controller driver");
 757 MODULE_LICENSE("GPL v2");