drivers/memory/tegra/mc.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2014 NVIDIA CORPORATION.  All rights reserved.
   4  */
   5
   6 #include <linux/clk.h>
   7 #include <linux/delay.h>
   8 #include <linux/dma-mapping.h>
   9 #include <linux/export.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/kernel.h>
  12 #include <linux/module.h>
  13 #include <linux/of.h>
  14 #include <linux/of_device.h>
  15 #include <linux/platform_device.h>
  16 #include <linux/slab.h>
  17 #include <linux/sort.h>
  18
  19 #include <soc/tegra/fuse.h>
  20
  21 #include "mc.h"
  22
  23 static const struct of_device_id tegra_mc_of_match[] = {
  24 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
  25         { .compatible = "nvidia,tegra20-mc-gart", .data = &tegra20_mc_soc },
  26 #endif
  27 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
  28         { .compatible = "nvidia,tegra30-mc", .data = &tegra30_mc_soc },
  29 #endif
  30 #ifdef CONFIG_ARCH_TEGRA_114_SOC
  31         { .compatible = "nvidia,tegra114-mc", .data = &tegra114_mc_soc },
  32 #endif
  33 #ifdef CONFIG_ARCH_TEGRA_124_SOC
  34         { .compatible = "nvidia,tegra124-mc", .data = &tegra124_mc_soc },
  35 #endif
  36 #ifdef CONFIG_ARCH_TEGRA_132_SOC
  37         { .compatible = "nvidia,tegra132-mc", .data = &tegra132_mc_soc },
  38 #endif
  39 #ifdef CONFIG_ARCH_TEGRA_210_SOC
  40         { .compatible = "nvidia,tegra210-mc", .data = &tegra210_mc_soc },
  41 #endif
  42         { }
  43 };
  44 MODULE_DEVICE_TABLE(of, tegra_mc_of_match);
  45
  46 static void tegra_mc_devm_action_put_device(void *data)
  47 {
  48         struct tegra_mc *mc = data;
  49
  50         put_device(mc->dev);
  51 }
  52
  53 /**
  54  * devm_tegra_memory_controller_get() - get Tegra Memory Controller handle
  55  * @dev: device pointer for the consumer device
  56  *
  57  * This function will search for the Memory Controller node in a device-tree
  58  * and retrieve the Memory Controller handle.
  59  *
  60  * Return: ERR_PTR() on error or a valid pointer to a struct tegra_mc.
  61  */
  62 struct tegra_mc *devm_tegra_memory_controller_get(struct device *dev)
  63 {
  64         struct platform_device *pdev;
  65         struct device_node *np;
  66         struct tegra_mc *mc;
  67         int err;
  68
  69         np = of_parse_phandle(dev->of_node, "nvidia,memory-controller", 0);
  70         if (!np)
  71                 return ERR_PTR(-ENOENT);
  72
  73         pdev = of_find_device_by_node(np);
  74         of_node_put(np);
  75         if (!pdev)
  76                 return ERR_PTR(-ENODEV);
  77
  78         mc = platform_get_drvdata(pdev);
  79         if (!mc) {
  80                 put_device(&pdev->dev);
  81                 return ERR_PTR(-EPROBE_DEFER);
  82         }
  83
  84         err = devm_add_action(dev, tegra_mc_devm_action_put_device, mc);
  85         if (err) {
  86                 put_device(mc->dev);
  87                 return ERR_PTR(err);
  88         }
  89
  90         return mc;
  91 }
  92 EXPORT_SYMBOL_GPL(devm_tegra_memory_controller_get);
  93
  94 static int tegra_mc_block_dma_common(struct tegra_mc *mc,
  95                                      const struct tegra_mc_reset *rst)
  96 {
  97         unsigned long flags;
  98         u32 value;
  99
 100         spin_lock_irqsave(&mc->lock, flags);
 101
 102         value = mc_readl(mc, rst->control) | BIT(rst->bit);
 103         mc_writel(mc, value, rst->control);
 104
 105         spin_unlock_irqrestore(&mc->lock, flags);
 106
 107         return 0;
 108 }
 109
 110 static bool tegra_mc_dma_idling_common(struct tegra_mc *mc,
 111                                        const struct tegra_mc_reset *rst)
 112 {
 113         return (mc_readl(mc, rst->status) & BIT(rst->bit)) != 0;
 114 }
 115
 116 static int tegra_mc_unblock_dma_common(struct tegra_mc *mc,
 117                                        const struct tegra_mc_reset *rst)
 118 {
 119         unsigned long flags;
 120         u32 value;
 121
 122         spin_lock_irqsave(&mc->lock, flags);
 123
 124         value = mc_readl(mc, rst->control) & ~BIT(rst->bit);
 125         mc_writel(mc, value, rst->control);
 126
 127         spin_unlock_irqrestore(&mc->lock, flags);
 128
 129         return 0;
 130 }
 131
 132 static int tegra_mc_reset_status_common(struct tegra_mc *mc,
 133                                         const struct tegra_mc_reset *rst)
 134 {
 135         return (mc_readl(mc, rst->control) & BIT(rst->bit)) != 0;
 136 }
 137
 138 const struct tegra_mc_reset_ops tegra_mc_reset_ops_common = {
 139         .block_dma = tegra_mc_block_dma_common,
 140         .dma_idling = tegra_mc_dma_idling_common,
 141         .unblock_dma = tegra_mc_unblock_dma_common,
 142         .reset_status = tegra_mc_reset_status_common,
 143 };
 144
 145 static inline struct tegra_mc *reset_to_mc(struct reset_controller_dev *rcdev)
 146 {
 147         return container_of(rcdev, struct tegra_mc, reset);
 148 }
 149
 150 static const struct tegra_mc_reset *tegra_mc_reset_find(struct tegra_mc *mc,
 151                                                         unsigned long id)
 152 {
 153         unsigned int i;
 154
 155         for (i = 0; i < mc->soc->num_resets; i++)
 156                 if (mc->soc->resets[i].id == id)
 157                         return &mc->soc->resets[i];
 158
 159         return NULL;
 160 }
 161
 162 static int tegra_mc_hotreset_assert(struct reset_controller_dev *rcdev,
 163                                     unsigned long id)
 164 {
 165         struct tegra_mc *mc = reset_to_mc(rcdev);
 166         const struct tegra_mc_reset_ops *rst_ops;
 167         const struct tegra_mc_reset *rst;
 168         int retries = 500;
 169         int err;
 170
 171         rst = tegra_mc_reset_find(mc, id);
 172         if (!rst)
 173                 return -ENODEV;
 174
 175         rst_ops = mc->soc->reset_ops;
 176         if (!rst_ops)
 177                 return -ENODEV;
 178
 179         if (rst_ops->block_dma) {
 180                 /* block clients DMA requests */
 181                 err = rst_ops->block_dma(mc, rst);
 182                 if (err) {
 183                         dev_err(mc->dev, "failed to block %s DMA: %d\n",
 184                                 rst->name, err);
 185                         return err;
 186                 }
 187         }
 188
 189         if (rst_ops->dma_idling) {
 190                 /* wait for completion of the outstanding DMA requests */
 191                 while (!rst_ops->dma_idling(mc, rst)) {
 192                         if (!retries--) {
 193                                 dev_err(mc->dev, "failed to flush %s DMA\n",
 194                                         rst->name);
 195                                 return -EBUSY;
 196                         }
 197
 198                         usleep_range(10, 100);
 199                 }
 200         }
 201
 202         if (rst_ops->hotreset_assert) {
 203                 /* clear clients DMA requests sitting before arbitration */
 204                 err = rst_ops->hotreset_assert(mc, rst);
 205                 if (err) {
 206                         dev_err(mc->dev, "failed to hot reset %s: %d\n",
 207                                 rst->name, err);
 208                         return err;
 209                 }
 210         }
 211
 212         return 0;
 213 }
 214
 215 static int tegra_mc_hotreset_deassert(struct reset_controller_dev *rcdev,
 216                                       unsigned long id)
 217 {
 218         struct tegra_mc *mc = reset_to_mc(rcdev);
 219         const struct tegra_mc_reset_ops *rst_ops;
 220         const struct tegra_mc_reset *rst;
 221         int err;
 222
 223         rst = tegra_mc_reset_find(mc, id);
 224         if (!rst)
 225                 return -ENODEV;
 226
 227         rst_ops = mc->soc->reset_ops;
 228         if (!rst_ops)
 229                 return -ENODEV;
 230
 231         if (rst_ops->hotreset_deassert) {
 232                 /* take out client from hot reset */
 233                 err = rst_ops->hotreset_deassert(mc, rst);
 234                 if (err) {
 235                         dev_err(mc->dev, "failed to deassert hot reset %s: %d\n",
 236                                 rst->name, err);
 237                         return err;
 238                 }
 239         }
 240
 241         if (rst_ops->unblock_dma) {
 242                 /* allow new DMA requests to proceed to arbitration */
 243                 err = rst_ops->unblock_dma(mc, rst);
 244                 if (err) {
 245                         dev_err(mc->dev, "failed to unblock %s DMA : %d\n",
 246                                 rst->name, err);
 247                         return err;
 248                 }
 249         }
 250
 251         return 0;
 252 }
 253
 254 static int tegra_mc_hotreset_status(struct reset_controller_dev *rcdev,
 255                                     unsigned long id)
 256 {
 257         struct tegra_mc *mc = reset_to_mc(rcdev);
 258         const struct tegra_mc_reset_ops *rst_ops;
 259         const struct tegra_mc_reset *rst;
 260
 261         rst = tegra_mc_reset_find(mc, id);
 262         if (!rst)
 263                 return -ENODEV;
 264
 265         rst_ops = mc->soc->reset_ops;
 266         if (!rst_ops)
 267                 return -ENODEV;
 268
 269         return rst_ops->reset_status(mc, rst);
 270 }
 271
 272 static const struct reset_control_ops tegra_mc_reset_ops = {
 273         .assert = tegra_mc_hotreset_assert,
 274         .deassert = tegra_mc_hotreset_deassert,
 275         .status = tegra_mc_hotreset_status,
 276 };
 277
 278 static int tegra_mc_reset_setup(struct tegra_mc *mc)
 279 {
 280         int err;
 281
 282         mc->reset.ops = &tegra_mc_reset_ops;
 283         mc->reset.owner = THIS_MODULE;
 284         mc->reset.of_node = mc->dev->of_node;
 285         mc->reset.of_reset_n_cells = 1;
 286         mc->reset.nr_resets = mc->soc->num_resets;
 287
 288         err = reset_controller_register(&mc->reset);
 289         if (err < 0)
 290                 return err;
 291
 292         return 0;
 293 }
 294
 295 static int tegra_mc_setup_latency_allowance(struct tegra_mc *mc)
 296 {
 297         unsigned long long tick;
 298         unsigned int i;
 299         u32 value;
 300
 301         /* compute the number of MC clock cycles per tick */
 302         tick = (unsigned long long)mc->tick * clk_get_rate(mc->clk);
 303         do_div(tick, NSEC_PER_SEC);
 304
 305         value = mc_readl(mc, MC_EMEM_ARB_CFG);
 306         value &= ~MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK;
 307         value |= MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(tick);
 308         mc_writel(mc, value, MC_EMEM_ARB_CFG);
 309
 310         /* write latency allowance defaults */
 311         for (i = 0; i < mc->soc->num_clients; i++) {
 312                 const struct tegra_mc_la *la = &mc->soc->clients[i].la;
 313                 u32 value;
 314
 315                 value = mc_readl(mc, la->reg);
 316                 value &= ~(la->mask << la->shift);
 317                 value |= (la->def & la->mask) << la->shift;
 318                 mc_writel(mc, value, la->reg);
 319         }
 320
 321         /* latch new values */
 322         mc_writel(mc, MC_TIMING_UPDATE, MC_TIMING_CONTROL);
 323
 324         return 0;
 325 }
 326
 327 int tegra_mc_write_emem_configuration(struct tegra_mc *mc, unsigned long rate)
 328 {
 329         unsigned int i;
 330         struct tegra_mc_timing *timing = NULL;
 331
 332         for (i = 0; i < mc->num_timings; i++) {
 333                 if (mc->timings[i].rate == rate) {
 334                         timing = &mc->timings[i];
 335                         break;
 336                 }
 337         }
 338
 339         if (!timing) {
 340                 dev_err(mc->dev, "no memory timing registered for rate %lu\n",
 341                         rate);
 342                 return -EINVAL;
 343         }
 344
 345         for (i = 0; i < mc->soc->num_emem_regs; ++i)
 346                 mc_writel(mc, timing->emem_data[i], mc->soc->emem_regs[i]);
 347
 348         return 0;
 349 }
 350 EXPORT_SYMBOL_GPL(tegra_mc_write_emem_configuration);
 351
 352 unsigned int tegra_mc_get_emem_device_count(struct tegra_mc *mc)
 353 {
 354         u8 dram_count;
 355
 356         dram_count = mc_readl(mc, MC_EMEM_ADR_CFG);
 357         dram_count &= MC_EMEM_ADR_CFG_EMEM_NUMDEV;
 358         dram_count++;
 359
 360         return dram_count;
 361 }
 362 EXPORT_SYMBOL_GPL(tegra_mc_get_emem_device_count);
 363
 364 static int load_one_timing(struct tegra_mc *mc,
 365                            struct tegra_mc_timing *timing,
 366                            struct device_node *node)
 367 {
 368         int err;
 369         u32 tmp;
 370
 371         err = of_property_read_u32(node, "clock-frequency", &tmp);
 372         if (err) {
 373                 dev_err(mc->dev,
 374                         "timing %pOFn: failed to read rate\n", node);
 375                 return err;
 376         }
 377
 378         timing->rate = tmp;
 379         timing->emem_data = devm_kcalloc(mc->dev, mc->soc->num_emem_regs,
 380                                          sizeof(u32), GFP_KERNEL);
 381         if (!timing->emem_data)
 382                 return -ENOMEM;
 383
 384         err = of_property_read_u32_array(node, "nvidia,emem-configuration",
 385                                          timing->emem_data,
 386                                          mc->soc->num_emem_regs);
 387         if (err) {
 388                 dev_err(mc->dev,
 389                         "timing %pOFn: failed to read EMEM configuration\n",
 390                         node);
 391                 return err;
 392         }
 393
 394         return 0;
 395 }
 396
 397 static int load_timings(struct tegra_mc *mc, struct device_node *node)
 398 {
 399         struct device_node *child;
 400         struct tegra_mc_timing *timing;
 401         int child_count = of_get_child_count(node);
 402         int i = 0, err;
 403
 404         mc->timings = devm_kcalloc(mc->dev, child_count, sizeof(*timing),
 405                                    GFP_KERNEL);
 406         if (!mc->timings)
 407                 return -ENOMEM;
 408
 409         mc->num_timings = child_count;
 410
 411         for_each_child_of_node(node, child) {
 412                 timing = &mc->timings[i++];
 413
 414                 err = load_one_timing(mc, timing, child);
 415                 if (err) {
 416                         of_node_put(child);
 417                         return err;
 418                 }
 419         }
 420
 421         return 0;
 422 }
 423
 424 static int tegra_mc_setup_timings(struct tegra_mc *mc)
 425 {
 426         struct device_node *node;
 427         u32 ram_code, node_ram_code;
 428         int err;
 429
 430         ram_code = tegra_read_ram_code();
 431
 432         mc->num_timings = 0;
 433
 434         for_each_child_of_node(mc->dev->of_node, node) {
 435                 err = of_property_read_u32(node, "nvidia,ram-code",
 436                                            &node_ram_code);
 437                 if (err || (node_ram_code != ram_code))
 438                         continue;
 439
 440                 err = load_timings(mc, node);
 441                 of_node_put(node);
 442                 if (err)
 443                         return err;
 444                 break;
 445         }
 446
 447         if (mc->num_timings == 0)
 448                 dev_warn(mc->dev,
 449                          "no memory timings for RAM code %u registered\n",
 450                          ram_code);
 451
 452         return 0;
 453 }
 454
 455 static const char *const status_names[32] = {
 456         [ 1] = "External interrupt",
 457         [ 6] = "EMEM address decode error",
 458         [ 7] = "GART page fault",
 459         [ 8] = "Security violation",
 460         [ 9] = "EMEM arbitration error",
 461         [10] = "Page fault",
 462         [11] = "Invalid APB ASID update",
 463         [12] = "VPR violation",
 464         [13] = "Secure carveout violation",
 465         [16] = "MTS carveout violation",
 466 };
 467
 468 static const char *const error_names[8] = {
 469         [2] = "EMEM decode error",
 470         [3] = "TrustZone violation",
 471         [4] = "Carveout violation",
 472         [6] = "SMMU translation error",
 473 };
 474
 475 static irqreturn_t tegra_mc_irq(int irq, void *data)
 476 {
 477         struct tegra_mc *mc = data;
 478         unsigned long status;
 479         unsigned int bit;
 480
 481         /* mask all interrupts to avoid flooding */
 482         status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
 483         if (!status)
 484                 return IRQ_NONE;
 485
 486         for_each_set_bit(bit, &status, 32) {
 487                 const char *error = status_names[bit] ?: "unknown";
 488                 const char *client = "unknown", *desc;
 489                 const char *direction, *secure;
 490                 phys_addr_t addr = 0;
 491                 unsigned int i;
 492                 char perm[7];
 493                 u8 id, type;
 494                 u32 value;
 495
 496                 value = mc_readl(mc, MC_ERR_STATUS);
 497
 498 #ifdef CONFIG_PHYS_ADDR_T_64BIT
 499                 if (mc->soc->num_address_bits > 32) {
 500                         addr = ((value >> MC_ERR_STATUS_ADR_HI_SHIFT) &
 501                                 MC_ERR_STATUS_ADR_HI_MASK);
 502                         addr <<= 32;
 503                 }
 504 #endif
 505
 506                 if (value & MC_ERR_STATUS_RW)
 507                         direction = "write";
 508                 else
 509                         direction = "read";
 510
 511                 if (value & MC_ERR_STATUS_SECURITY)
 512                         secure = "secure ";
 513                 else
 514                         secure = "";
 515
 516                 id = value & mc->soc->client_id_mask;
 517
 518                 for (i = 0; i < mc->soc->num_clients; i++) {
 519                         if (mc->soc->clients[i].id == id) {
 520                                 client = mc->soc->clients[i].name;
 521                                 break;
 522                         }
 523                 }
 524
 525                 type = (value & MC_ERR_STATUS_TYPE_MASK) >>
 526                        MC_ERR_STATUS_TYPE_SHIFT;
 527                 desc = error_names[type];
 528
 529                 switch (value & MC_ERR_STATUS_TYPE_MASK) {
 530                 case MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE:
 531                         perm[0] = ' ';
 532                         perm[1] = '[';
 533
 534                         if (value & MC_ERR_STATUS_READABLE)
 535                                 perm[2] = 'R';
 536                         else
 537                                 perm[2] = '-';
 538
 539                         if (value & MC_ERR_STATUS_WRITABLE)
 540                                 perm[3] = 'W';
 541                         else
 542                                 perm[3] = '-';
 543
 544                         if (value & MC_ERR_STATUS_NONSECURE)
 545                                 perm[4] = '-';
 546                         else
 547                                 perm[4] = 'S';
 548
 549                         perm[5] = ']';
 550                         perm[6] = '\0';
 551                         break;
 552
 553                 default:
 554                         perm[0] = '\0';
 555                         break;
 556                 }
 557
 558                 value = mc_readl(mc, MC_ERR_ADR);
 559                 addr |= value;
 560
 561                 dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s%s)\n",
 562                                     client, secure, direction, &addr, error,
 563                                     desc, perm);
 564         }
 565
 566         /* clear interrupts */
 567         mc_writel(mc, status, MC_INTSTATUS);
 568
 569         return IRQ_HANDLED;
 570 }
 571
 572 static __maybe_unused irqreturn_t tegra20_mc_irq(int irq, void *data)
 573 {
 574         struct tegra_mc *mc = data;
 575         unsigned long status;
 576         unsigned int bit;
 577
 578         /* mask all interrupts to avoid flooding */
 579         status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
 580         if (!status)
 581                 return IRQ_NONE;
 582
 583         for_each_set_bit(bit, &status, 32) {
 584                 const char *direction = "read", *secure = "";
 585                 const char *error = status_names[bit];
 586                 const char *client, *desc;
 587                 phys_addr_t addr;
 588                 u32 value, reg;
 589                 u8 id, type;
 590
 591                 switch (BIT(bit)) {
 592                 case MC_INT_DECERR_EMEM:
 593                         reg = MC_DECERR_EMEM_OTHERS_STATUS;
 594                         value = mc_readl(mc, reg);
 595
 596                         id = value & mc->soc->client_id_mask;
 597                         desc = error_names[2];
 598
 599                         if (value & BIT(31))
 600                                 direction = "write";
 601                         break;
 602
 603                 case MC_INT_INVALID_GART_PAGE:
 604                         reg = MC_GART_ERROR_REQ;
 605                         value = mc_readl(mc, reg);
 606
 607                         id = (value >> 1) & mc->soc->client_id_mask;
 608                         desc = error_names[2];
 609
 610                         if (value & BIT(0))
 611                                 direction = "write";
 612                         break;
 613
 614                 case MC_INT_SECURITY_VIOLATION:
 615                         reg = MC_SECURITY_VIOLATION_STATUS;
 616                         value = mc_readl(mc, reg);
 617
 618                         id = value & mc->soc->client_id_mask;
 619                         type = (value & BIT(30)) ? 4 : 3;
 620                         desc = error_names[type];
 621                         secure = "secure ";
 622
 623                         if (value & BIT(31))
 624                                 direction = "write";
 625                         break;
 626
 627                 default:
 628                         continue;
 629                 }
 630
 631                 client = mc->soc->clients[id].name;
 632                 addr = mc_readl(mc, reg + sizeof(u32));
 633
 634                 dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s)\n",
 635                                     client, secure, direction, &addr, error,
 636                                     desc);
 637         }
 638
 639         /* clear interrupts */
 640         mc_writel(mc, status, MC_INTSTATUS);
 641
 642         return IRQ_HANDLED;
 643 }
 644
 645 /*
 646  * Memory Controller (MC) has few Memory Clients that are issuing memory
 647  * bandwidth allocation requests to the MC interconnect provider. The MC
 648  * provider aggregates the requests and then sends the aggregated request
 649  * up to the External Memory Controller (EMC) interconnect provider which
 650  * re-configures hardware interface to External Memory (EMEM) in accordance
 651  * to the required bandwidth. Each MC interconnect node represents an
 652  * individual Memory Client.
 653  *
 654  * Memory interconnect topology:
 655  *
 656  *               +----+
 657  * +--------+    |    |
 658  * | TEXSRD +--->+    |
 659  * +--------+    |    |
 660  *               |    |    +-----+    +------+
 661  *    ...        | MC +--->+ EMC +--->+ EMEM |
 662  *               |    |    +-----+    +------+
 663  * +--------+    |    |
 664  * | DISP.. +--->+    |
 665  * +--------+    |    |
 666  *               +----+
 667  */
 668 static int tegra_mc_interconnect_setup(struct tegra_mc *mc)
 669 {
 670         struct icc_node *node;
 671         unsigned int i;
 672         int err;
 673
 674         /* older device-trees don't have interconnect properties */
 675         if (!device_property_present(mc->dev, "#interconnect-cells") ||
 676             !mc->soc->icc_ops)
 677                 return 0;
 678
 679         mc->provider.dev = mc->dev;
 680         mc->provider.data = &mc->provider;
 681         mc->provider.set = mc->soc->icc_ops->set;
 682         mc->provider.aggregate = mc->soc->icc_ops->aggregate;
 683         mc->provider.xlate_extended = mc->soc->icc_ops->xlate_extended;
 684
 685         err = icc_provider_add(&mc->provider);
 686         if (err)
 687                 return err;
 688
 689         /* create Memory Controller node */
 690         node = icc_node_create(TEGRA_ICC_MC);
 691         if (IS_ERR(node)) {
 692                 err = PTR_ERR(node);
 693                 goto del_provider;
 694         }
 695
 696         node->name = "Memory Controller";
 697         icc_node_add(node, &mc->provider);
 698
 699         /* link Memory Controller to External Memory Controller */
 700         err = icc_link_create(node, TEGRA_ICC_EMC);
 701         if (err)
 702                 goto remove_nodes;
 703
 704         for (i = 0; i < mc->soc->num_clients; i++) {
 705                 /* create MC client node */
 706                 node = icc_node_create(mc->soc->clients[i].id);
 707                 if (IS_ERR(node)) {
 708                         err = PTR_ERR(node);
 709                         goto remove_nodes;
 710                 }
 711
 712                 node->name = mc->soc->clients[i].name;
 713                 icc_node_add(node, &mc->provider);
 714
 715                 /* link Memory Client to Memory Controller */
 716                 err = icc_link_create(node, TEGRA_ICC_MC);
 717                 if (err)
 718                         goto remove_nodes;
 719         }
 720
 721         /*
 722          * MC driver is registered too early, so early that generic driver
 723          * syncing doesn't work for the MC. But it doesn't really matter
 724          * since syncing works for the EMC drivers, hence we can sync the
 725          * MC driver by ourselves and then EMC will complete syncing of
 726          * the whole ICC state.
 727          */
 728         icc_sync_state(mc->dev);
 729
 730         return 0;
 731
 732 remove_nodes:
 733         icc_nodes_remove(&mc->provider);
 734 del_provider:
 735         icc_provider_del(&mc->provider);
 736
 737         return err;
 738 }
 739
 740 static int tegra_mc_probe(struct platform_device *pdev)
 741 {
 742         struct resource *res;
 743         struct tegra_mc *mc;
 744         void *isr;
 745         u64 mask;
 746         int err;
 747
 748         mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL);
 749         if (!mc)
 750                 return -ENOMEM;
 751
 752         platform_set_drvdata(pdev, mc);
 753         spin_lock_init(&mc->lock);
 754         mc->soc = of_device_get_match_data(&pdev->dev);
 755         mc->dev = &pdev->dev;
 756
 757         mask = DMA_BIT_MASK(mc->soc->num_address_bits);
 758
 759         err = dma_coerce_mask_and_coherent(&pdev->dev, mask);
 760         if (err < 0) {
 761                 dev_err(&pdev->dev, "failed to set DMA mask: %d\n", err);
 762                 return err;
 763         }
 764
 765         /* length of MC tick in nanoseconds */
 766         mc->tick = 30;
 767
 768         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 769         mc->regs = devm_ioremap_resource(&pdev->dev, res);
 770         if (IS_ERR(mc->regs))
 771                 return PTR_ERR(mc->regs);
 772
 773         mc->clk = devm_clk_get(&pdev->dev, "mc");
 774         if (IS_ERR(mc->clk)) {
 775                 dev_err(&pdev->dev, "failed to get MC clock: %ld\n",
 776                         PTR_ERR(mc->clk));
 777                 return PTR_ERR(mc->clk);
 778         }
 779
 780 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
 781         if (mc->soc == &tegra20_mc_soc) {
 782                 isr = tegra20_mc_irq;
 783         } else
 784 #endif
 785         {
 786                 /* ensure that debug features are disabled */
 787                 mc_writel(mc, 0x00000000, MC_TIMING_CONTROL_DBG);
 788
 789                 err = tegra_mc_setup_latency_allowance(mc);
 790                 if (err < 0) {
 791                         dev_err(&pdev->dev,
 792                                 "failed to setup latency allowance: %d\n",
 793                                 err);
 794                         return err;
 795                 }
 796
 797                 isr = tegra_mc_irq;
 798
 799                 err = tegra_mc_setup_timings(mc);
 800                 if (err < 0) {
 801                         dev_err(&pdev->dev, "failed to setup timings: %d\n",
 802                                 err);
 803                         return err;
 804                 }
 805         }
 806
 807         mc->irq = platform_get_irq(pdev, 0);
 808         if (mc->irq < 0)
 809                 return mc->irq;
 810
 811         WARN(!mc->soc->client_id_mask, "missing client ID mask for this SoC\n");
 812
 813         mc_writel(mc, mc->soc->intmask, MC_INTMASK);
 814
 815         err = devm_request_irq(&pdev->dev, mc->irq, isr, 0,
 816                                dev_name(&pdev->dev), mc);
 817         if (err < 0) {
 818                 dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", mc->irq,
 819                         err);
 820                 return err;
 821         }
 822
 823         err = tegra_mc_reset_setup(mc);
 824         if (err < 0)
 825                 dev_err(&pdev->dev, "failed to register reset controller: %d\n",
 826                         err);
 827
 828         err = tegra_mc_interconnect_setup(mc);
 829         if (err < 0)
 830                 dev_err(&pdev->dev, "failed to initialize interconnect: %d\n",
 831                         err);
 832
 833         if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU) && mc->soc->smmu) {
 834                 mc->smmu = tegra_smmu_probe(&pdev->dev, mc->soc->smmu, mc);
 835                 if (IS_ERR(mc->smmu)) {
 836                         dev_err(&pdev->dev, "failed to probe SMMU: %ld\n",
 837                                 PTR_ERR(mc->smmu));
 838                         mc->smmu = NULL;
 839                 }
 840         }
 841
 842         if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && !mc->soc->smmu) {
 843                 mc->gart = tegra_gart_probe(&pdev->dev, mc);
 844                 if (IS_ERR(mc->gart)) {
 845                         dev_err(&pdev->dev, "failed to probe GART: %ld\n",
 846                                 PTR_ERR(mc->gart));
 847                         mc->gart = NULL;
 848                 }
 849         }
 850
 851         return 0;
 852 }
 853
 854 static int tegra_mc_suspend(struct device *dev)
 855 {
 856         struct tegra_mc *mc = dev_get_drvdata(dev);
 857         int err;
 858
 859         if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && mc->gart) {
 860                 err = tegra_gart_suspend(mc->gart);
 861                 if (err)
 862                         return err;
 863         }
 864
 865         return 0;
 866 }
 867
 868 static int tegra_mc_resume(struct device *dev)
 869 {
 870         struct tegra_mc *mc = dev_get_drvdata(dev);
 871         int err;
 872
 873         if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && mc->gart) {
 874                 err = tegra_gart_resume(mc->gart);
 875                 if (err)
 876                         return err;
 877         }
 878
 879         return 0;
 880 }
 881
 882 static const struct dev_pm_ops tegra_mc_pm_ops = {
 883         .suspend = tegra_mc_suspend,
 884         .resume = tegra_mc_resume,
 885 };
 886
 887 static struct platform_driver tegra_mc_driver = {
 888         .driver = {
 889                 .name = "tegra-mc",
 890                 .of_match_table = tegra_mc_of_match,
 891                 .pm = &tegra_mc_pm_ops,
 892                 .suppress_bind_attrs = true,
 893         },
 894         .prevent_deferred_probe = true,
 895         .probe = tegra_mc_probe,
 896 };
 897
 898 static int tegra_mc_init(void)
 899 {
 900         return platform_driver_register(&tegra_mc_driver);
 901 }
 902 arch_initcall(tegra_mc_init);
 903
 904 MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
 905 MODULE_DESCRIPTION("NVIDIA Tegra Memory Controller driver");
 906 MODULE_LICENSE("GPL v2");