?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'block-5.11-2021-01-10' of git://git.kernel.dk/linux-block
[linux/fpc-iii.git] / drivers / iommu / arm / arm-smmu / arm-smmu.c
blobd8c6bfde6a61587864092c3df89789c0c7671f34
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * IOMMU API for ARM architected SMMU implementations.
4 *
5 * Copyright (C) 2013 ARM Limited
6 *
7 * Author: Will Deacon <will.deacon@arm.com>
8 *
9 * This driver currently supports:
10 * - SMMUv1 and v2 implementations
11 * - Stream-matching and stream-indexing
12 * - v7/v8 long-descriptor format
13 * - Non-secure access to the SMMU
14 * - Context fault reporting
15 * - Extended Stream ID (16 bit)
16 */
17
18 #define pr_fmt(fmt) "arm-smmu: " fmt
19
20 #include <linux/acpi.h>
21 #include <linux/acpi_iort.h>
22 #include <linux/bitfield.h>
23 #include <linux/delay.h>
24 #include <linux/dma-iommu.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/err.h>
27 #include <linux/interrupt.h>
28 #include <linux/io.h>
29 #include <linux/iopoll.h>
30 #include <linux/module.h>
31 #include <linux/of.h>
32 #include <linux/of_address.h>
33 #include <linux/of_device.h>
34 #include <linux/of_iommu.h>
35 #include <linux/pci.h>
36 #include <linux/platform_device.h>
37 #include <linux/pm_runtime.h>
38 #include <linux/ratelimit.h>
39 #include <linux/slab.h>
40
41 #include <linux/amba/bus.h>
42 #include <linux/fsl/mc.h>
43
44 #include "arm-smmu.h"
45
46 /*
47 * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
48 * global register space are still, in fact, using a hypervisor to mediate it
49 * by trapping and emulating register accesses. Sadly, some deployed versions
50 * of said trapping code have bugs wherein they go horribly wrong for stores
51 * using r31 (i.e. XZR/WZR) as the source register.
52 */
53 #define QCOM_DUMMY_VAL -1
54
55 #define MSI_IOVA_BASE 0x8000000
56 #define MSI_IOVA_LENGTH 0x100000
57
58 static int force_stage;
59 module_param(force_stage, int, S_IRUGO);
60 MODULE_PARM_DESC(force_stage,
61 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
62 static bool disable_bypass =
63 IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
64 module_param(disable_bypass, bool, S_IRUGO);
65 MODULE_PARM_DESC(disable_bypass,
66 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
67
68 #define s2cr_init_val (struct arm_smmu_s2cr){ \
69 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
70 }
71
72 static bool using_legacy_binding, using_generic_binding;
73
74 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
75 {
76 if (pm_runtime_enabled(smmu->dev))
77 return pm_runtime_get_sync(smmu->dev);
78
79 return 0;
80 }
81
82 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
83 {
84 if (pm_runtime_enabled(smmu->dev))
85 pm_runtime_put_autosuspend(smmu->dev);
86 }
87
88 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
89 {
90 return container_of(dom, struct arm_smmu_domain, domain);
91 }
92
93 static struct platform_driver arm_smmu_driver;
94 static struct iommu_ops arm_smmu_ops;
95
96 #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
97 static int arm_smmu_bus_init(struct iommu_ops *ops);
98
99 static struct device_node *dev_get_dev_node(struct device *dev)
100 {
101 if (dev_is_pci(dev)) {
102 struct pci_bus *bus = to_pci_dev(dev)->bus;
103
104 while (!pci_is_root_bus(bus))
105 bus = bus->parent;
106 return of_node_get(bus->bridge->parent->of_node);
107 }
108
109 return of_node_get(dev->of_node);
110 }
111
112 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
113 {
114 *((__be32 *)data) = cpu_to_be32(alias);
115 return 0; /* Continue walking */
116 }
117
118 static int __find_legacy_master_phandle(struct device *dev, void *data)
119 {
120 struct of_phandle_iterator *it = *(void **)data;
121 struct device_node *np = it->node;
122 int err;
123
124 of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
125 "#stream-id-cells", -1)
126 if (it->node == np) {
127 *(void **)data = dev;
128 return 1;
129 }
130 it->node = np;
131 return err == -ENOENT ? 0 : err;
132 }
133
134 static int arm_smmu_register_legacy_master(struct device *dev,
135 struct arm_smmu_device **smmu)
136 {
137 struct device *smmu_dev;
138 struct device_node *np;
139 struct of_phandle_iterator it;
140 void *data = &it;
141 u32 *sids;
142 __be32 pci_sid;
143 int err;
144
145 np = dev_get_dev_node(dev);
146 if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
147 of_node_put(np);
148 return -ENODEV;
149 }
150
151 it.node = np;
152 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
153 __find_legacy_master_phandle);
154 smmu_dev = data;
155 of_node_put(np);
156 if (err == 0)
157 return -ENODEV;
158 if (err < 0)
159 return err;
160
161 if (dev_is_pci(dev)) {
162 /* "mmu-masters" assumes Stream ID == Requester ID */
163 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
164 &pci_sid);
165 it.cur = &pci_sid;
166 it.cur_count = 1;
167 }
168
169 err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
170 &arm_smmu_ops);
171 if (err)
172 return err;
173
174 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
175 if (!sids)
176 return -ENOMEM;
177
178 *smmu = dev_get_drvdata(smmu_dev);
179 of_phandle_iterator_args(&it, sids, it.cur_count);
180 err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
181 kfree(sids);
182 return err;
183 }
184
185 /*
186 * With the legacy DT binding in play, we have no guarantees about
187 * probe order, but then we're also not doing default domains, so we can
188 * delay setting bus ops until we're sure every possible SMMU is ready,
189 * and that way ensure that no probe_device() calls get missed.
190 */
191 static int arm_smmu_legacy_bus_init(void)
192 {
193 if (using_legacy_binding)
194 return arm_smmu_bus_init(&arm_smmu_ops);
195 return 0;
196 }
197 device_initcall_sync(arm_smmu_legacy_bus_init);
198 #else
199 static int arm_smmu_register_legacy_master(struct device *dev,
200 struct arm_smmu_device **smmu)
201 {
202 return -ENODEV;
203 }
204 #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
205
206 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
207 {
208 clear_bit(idx, map);
209 }
210
211 /* Wait for any pending TLB invalidations to complete */
212 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
213 int sync, int status)
214 {
215 unsigned int spin_cnt, delay;
216 u32 reg;
217
218 if (smmu->impl && unlikely(smmu->impl->tlb_sync))
219 return smmu->impl->tlb_sync(smmu, page, sync, status);
220
221 arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
222 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
223 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
224 reg = arm_smmu_readl(smmu, page, status);
225 if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
226 return;
227 cpu_relax();
228 }
229 udelay(delay);
230 }
231 dev_err_ratelimited(smmu->dev,
232 "TLB sync timed out -- SMMU may be deadlocked\n");
233 }
234
235 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
236 {
237 unsigned long flags;
238
239 spin_lock_irqsave(&smmu->global_sync_lock, flags);
240 __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
241 ARM_SMMU_GR0_sTLBGSTATUS);
242 spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
243 }
244
245 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain)
246 {
247 struct arm_smmu_device *smmu = smmu_domain->smmu;
248 unsigned long flags;
249
250 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
251 __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
252 ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
253 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
254 }
255
256 static void arm_smmu_tlb_inv_context_s1(void *cookie)
257 {
258 struct arm_smmu_domain *smmu_domain = cookie;
259 /*
260 * The TLBI write may be relaxed, so ensure that PTEs cleared by the
261 * current CPU are visible beforehand.
262 */
263 wmb();
264 arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
265 ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
266 arm_smmu_tlb_sync_context(smmu_domain);
267 }
268
269 static void arm_smmu_tlb_inv_context_s2(void *cookie)
270 {
271 struct arm_smmu_domain *smmu_domain = cookie;
272 struct arm_smmu_device *smmu = smmu_domain->smmu;
273
274 /* See above */
275 wmb();
276 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
277 arm_smmu_tlb_sync_global(smmu);
278 }
279
280 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
281 size_t granule, void *cookie, int reg)
282 {
283 struct arm_smmu_domain *smmu_domain = cookie;
284 struct arm_smmu_device *smmu = smmu_domain->smmu;
285 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
286 int idx = cfg->cbndx;
287
288 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
289 wmb();
290
291 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
292 iova = (iova >> 12) << 12;
293 iova |= cfg->asid;
294 do {
295 arm_smmu_cb_write(smmu, idx, reg, iova);
296 iova += granule;
297 } while (size -= granule);
298 } else {
299 iova >>= 12;
300 iova |= (u64)cfg->asid << 48;
301 do {
302 arm_smmu_cb_writeq(smmu, idx, reg, iova);
303 iova += granule >> 12;
304 } while (size -= granule);
305 }
306 }
307
308 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
309 size_t granule, void *cookie, int reg)
310 {
311 struct arm_smmu_domain *smmu_domain = cookie;
312 struct arm_smmu_device *smmu = smmu_domain->smmu;
313 int idx = smmu_domain->cfg.cbndx;
314
315 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
316 wmb();
317
318 iova >>= 12;
319 do {
320 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
321 arm_smmu_cb_writeq(smmu, idx, reg, iova);
322 else
323 arm_smmu_cb_write(smmu, idx, reg, iova);
324 iova += granule >> 12;
325 } while (size -= granule);
326 }
327
328 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size,
329 size_t granule, void *cookie)
330 {
331 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
332 ARM_SMMU_CB_S1_TLBIVA);
333 arm_smmu_tlb_sync_context(cookie);
334 }
335
336 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather,
337 unsigned long iova, size_t granule,
338 void *cookie)
339 {
340 arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie,
341 ARM_SMMU_CB_S1_TLBIVAL);
342 }
343
344 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size,
345 size_t granule, void *cookie)
346 {
347 arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
348 ARM_SMMU_CB_S2_TLBIIPAS2);
349 arm_smmu_tlb_sync_context(cookie);
350 }
351
352 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather,
353 unsigned long iova, size_t granule,
354 void *cookie)
355 {
356 arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie,
357 ARM_SMMU_CB_S2_TLBIIPAS2L);
358 }
359
360 static void arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova, size_t size,
361 size_t granule, void *cookie)
362 {
363 arm_smmu_tlb_inv_context_s2(cookie);
364 }
365 /*
366 * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
367 * almost negligible, but the benefit of getting the first one in as far ahead
368 * of the sync as possible is significant, hence we don't just make this a
369 * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might
370 * think.
371 */
372 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather,
373 unsigned long iova, size_t granule,
374 void *cookie)
375 {
376 struct arm_smmu_domain *smmu_domain = cookie;
377 struct arm_smmu_device *smmu = smmu_domain->smmu;
378
379 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
380 wmb();
381
382 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
383 }
384
385 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = {
386 .tlb_flush_all = arm_smmu_tlb_inv_context_s1,
387 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1,
388 .tlb_add_page = arm_smmu_tlb_add_page_s1,
389 };
390
391 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
392 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
393 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2,
394 .tlb_add_page = arm_smmu_tlb_add_page_s2,
395 };
396
397 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
398 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
399 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2_v1,
400 .tlb_add_page = arm_smmu_tlb_add_page_s2_v1,
401 };
402
403 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
404 {
405 u32 fsr, fsynr, cbfrsynra;
406 unsigned long iova;
407 struct iommu_domain *domain = dev;
408 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
409 struct arm_smmu_device *smmu = smmu_domain->smmu;
410 int idx = smmu_domain->cfg.cbndx;
411
412 fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
413 if (!(fsr & ARM_SMMU_FSR_FAULT))
414 return IRQ_NONE;
415
416 fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
417 iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
418 cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
419
420 dev_err_ratelimited(smmu->dev,
421 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
422 fsr, iova, fsynr, cbfrsynra, idx);
423
424 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);
425 return IRQ_HANDLED;
426 }
427
428 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
429 {
430 u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
431 struct arm_smmu_device *smmu = dev;
432 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
433 DEFAULT_RATELIMIT_BURST);
434
435 gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
436 gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
437 gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
438 gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
439
440 if (!gfsr)
441 return IRQ_NONE;
442
443 if (__ratelimit(&rs)) {
444 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
445 (gfsr & ARM_SMMU_sGFSR_USF))
446 dev_err(smmu->dev,
447 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
448 (u16)gfsynr1);
449 else
450 dev_err(smmu->dev,
451 "Unexpected global fault, this could be serious\n");
452 dev_err(smmu->dev,
453 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
454 gfsr, gfsynr0, gfsynr1, gfsynr2);
455 }
456
457 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
458 return IRQ_HANDLED;
459 }
460
461 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
462 struct io_pgtable_cfg *pgtbl_cfg)
463 {
464 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
465 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
466 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
467
468 cb->cfg = cfg;
469
470 /* TCR */
471 if (stage1) {
472 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
473 cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
474 } else {
475 cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg);
476 cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg);
477 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
478 cb->tcr[1] |= ARM_SMMU_TCR2_AS;
479 else
480 cb->tcr[0] |= ARM_SMMU_TCR_EAE;
481 }
482 } else {
483 cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg);
484 }
485
486 /* TTBRs */
487 if (stage1) {
488 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
489 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
490 cb->ttbr[1] = 0;
491 } else {
492 cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
493 cfg->asid);
494 cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
495 cfg->asid);
496
497 if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
498 cb->ttbr[1] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
499 else
500 cb->ttbr[0] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
501 }
502 } else {
503 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
504 }
505
506 /* MAIRs (stage-1 only) */
507 if (stage1) {
508 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
509 cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
510 cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
511 } else {
512 cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair;
513 cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32;
514 }
515 }
516 }
517
518 void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
519 {
520 u32 reg;
521 bool stage1;
522 struct arm_smmu_cb *cb = &smmu->cbs[idx];
523 struct arm_smmu_cfg *cfg = cb->cfg;
524
525 /* Unassigned context banks only need disabling */
526 if (!cfg) {
527 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
528 return;
529 }
530
531 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
532
533 /* CBA2R */
534 if (smmu->version > ARM_SMMU_V1) {
535 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
536 reg = ARM_SMMU_CBA2R_VA64;
537 else
538 reg = 0;
539 /* 16-bit VMIDs live in CBA2R */
540 if (smmu->features & ARM_SMMU_FEAT_VMID16)
541 reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid);
542
543 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
544 }
545
546 /* CBAR */
547 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar);
548 if (smmu->version < ARM_SMMU_V2)
549 reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx);
550
551 /*
552 * Use the weakest shareability/memory types, so they are
553 * overridden by the ttbcr/pte.
554 */
555 if (stage1) {
556 reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG,
557 ARM_SMMU_CBAR_S1_BPSHCFG_NSH) |
558 FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR,
559 ARM_SMMU_CBAR_S1_MEMATTR_WB);
560 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
561 /* 8-bit VMIDs live in CBAR */
562 reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid);
563 }
564 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
565
566 /*
567 * TCR
568 * We must write this before the TTBRs, since it determines the
569 * access behaviour of some fields (in particular, ASID[15:8]).
570 */
571 if (stage1 && smmu->version > ARM_SMMU_V1)
572 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
573 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
574
575 /* TTBRs */
576 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
577 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
578 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
579 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
580 } else {
581 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
582 if (stage1)
583 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
584 cb->ttbr[1]);
585 }
586
587 /* MAIRs (stage-1 only) */
588 if (stage1) {
589 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
590 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
591 }
592
593 /* SCTLR */
594 reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE |
595 ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M;
596 if (stage1)
597 reg |= ARM_SMMU_SCTLR_S1_ASIDPNE;
598 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
599 reg |= ARM_SMMU_SCTLR_E;
600
601 if (smmu->impl && smmu->impl->write_sctlr)
602 smmu->impl->write_sctlr(smmu, idx, reg);
603 else
604 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
605 }
606
607 static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
608 struct arm_smmu_device *smmu,
609 struct device *dev, unsigned int start)
610 {
611 if (smmu->impl && smmu->impl->alloc_context_bank)
612 return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start);
613
614 return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks);
615 }
616
617 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
618 struct arm_smmu_device *smmu,
619 struct device *dev)
620 {
621 int irq, start, ret = 0;
622 unsigned long ias, oas;
623 struct io_pgtable_ops *pgtbl_ops;
624 struct io_pgtable_cfg pgtbl_cfg;
625 enum io_pgtable_fmt fmt;
626 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
627 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
628 irqreturn_t (*context_fault)(int irq, void *dev);
629
630 mutex_lock(&smmu_domain->init_mutex);
631 if (smmu_domain->smmu)
632 goto out_unlock;
633
634 if (domain->type == IOMMU_DOMAIN_IDENTITY) {
635 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
636 smmu_domain->smmu = smmu;
637 goto out_unlock;
638 }
639
640 /*
641 * Mapping the requested stage onto what we support is surprisingly
642 * complicated, mainly because the spec allows S1+S2 SMMUs without
643 * support for nested translation. That means we end up with the
644 * following table:
645 *
646 * Requested Supported Actual
647 * S1 N S1
648 * S1 S1+S2 S1
649 * S1 S2 S2
650 * S1 S1 S1
651 * N N N
652 * N S1+S2 S2
653 * N S2 S2
654 * N S1 S1
655 *
656 * Note that you can't actually request stage-2 mappings.
657 */
658 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
659 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
660 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
661 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
662
663 /*
664 * Choosing a suitable context format is even more fiddly. Until we
665 * grow some way for the caller to express a preference, and/or move
666 * the decision into the io-pgtable code where it arguably belongs,
667 * just aim for the closest thing to the rest of the system, and hope
668 * that the hardware isn't esoteric enough that we can't assume AArch64
669 * support to be a superset of AArch32 support...
670 */
671 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
672 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
673 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
674 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
675 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
676 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
677 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
678 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
679 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
680 ARM_SMMU_FEAT_FMT_AARCH64_16K |
681 ARM_SMMU_FEAT_FMT_AARCH64_4K)))
682 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
683
684 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
685 ret = -EINVAL;
686 goto out_unlock;
687 }
688
689 switch (smmu_domain->stage) {
690 case ARM_SMMU_DOMAIN_S1:
691 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
692 start = smmu->num_s2_context_banks;
693 ias = smmu->va_size;
694 oas = smmu->ipa_size;
695 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
696 fmt = ARM_64_LPAE_S1;
697 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
698 fmt = ARM_32_LPAE_S1;
699 ias = min(ias, 32UL);
700 oas = min(oas, 40UL);
701 } else {
702 fmt = ARM_V7S;
703 ias = min(ias, 32UL);
704 oas = min(oas, 32UL);
705 }
706 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
707 break;
708 case ARM_SMMU_DOMAIN_NESTED:
709 /*
710 * We will likely want to change this if/when KVM gets
711 * involved.
712 */
713 case ARM_SMMU_DOMAIN_S2:
714 cfg->cbar = CBAR_TYPE_S2_TRANS;
715 start = 0;
716 ias = smmu->ipa_size;
717 oas = smmu->pa_size;
718 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
719 fmt = ARM_64_LPAE_S2;
720 } else {
721 fmt = ARM_32_LPAE_S2;
722 ias = min(ias, 40UL);
723 oas = min(oas, 40UL);
724 }
725 if (smmu->version == ARM_SMMU_V2)
726 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
727 else
728 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
729 break;
730 default:
731 ret = -EINVAL;
732 goto out_unlock;
733 }
734
735 ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start);
736 if (ret < 0) {
737 goto out_unlock;
738 }
739
740 smmu_domain->smmu = smmu;
741
742 cfg->cbndx = ret;
743 if (smmu->version < ARM_SMMU_V2) {
744 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
745 cfg->irptndx %= smmu->num_context_irqs;
746 } else {
747 cfg->irptndx = cfg->cbndx;
748 }
749
750 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
751 cfg->vmid = cfg->cbndx + 1;
752 else
753 cfg->asid = cfg->cbndx;
754
755 pgtbl_cfg = (struct io_pgtable_cfg) {
756 .pgsize_bitmap = smmu->pgsize_bitmap,
757 .ias = ias,
758 .oas = oas,
759 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
760 .tlb = smmu_domain->flush_ops,
761 .iommu_dev = smmu->dev,
762 };
763
764 if (smmu->impl && smmu->impl->init_context) {
765 ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev);
766 if (ret)
767 goto out_clear_smmu;
768 }
769
770 if (smmu_domain->pgtbl_cfg.quirks)
771 pgtbl_cfg.quirks |= smmu_domain->pgtbl_cfg.quirks;
772
773 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
774 if (!pgtbl_ops) {
775 ret = -ENOMEM;
776 goto out_clear_smmu;
777 }
778
779 /* Update the domain's page sizes to reflect the page table format */
780 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
781
782 if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
783 domain->geometry.aperture_start = ~0UL << ias;
784 domain->geometry.aperture_end = ~0UL;
785 } else {
786 domain->geometry.aperture_end = (1UL << ias) - 1;
787 }
788
789 domain->geometry.force_aperture = true;
790
791 /* Initialise the context bank with our page table cfg */
792 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
793 arm_smmu_write_context_bank(smmu, cfg->cbndx);
794
795 /*
796 * Request context fault interrupt. Do this last to avoid the
797 * handler seeing a half-initialised domain state.
798 */
799 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
800
801 if (smmu->impl && smmu->impl->context_fault)
802 context_fault = smmu->impl->context_fault;
803 else
804 context_fault = arm_smmu_context_fault;
805
806 ret = devm_request_irq(smmu->dev, irq, context_fault,
807 IRQF_SHARED, "arm-smmu-context-fault", domain);
808 if (ret < 0) {
809 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
810 cfg->irptndx, irq);
811 cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
812 }
813
814 mutex_unlock(&smmu_domain->init_mutex);
815
816 /* Publish page table ops for map/unmap */
817 smmu_domain->pgtbl_ops = pgtbl_ops;
818 return 0;
819
820 out_clear_smmu:
821 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
822 smmu_domain->smmu = NULL;
823 out_unlock:
824 mutex_unlock(&smmu_domain->init_mutex);
825 return ret;
826 }
827
828 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
829 {
830 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
831 struct arm_smmu_device *smmu = smmu_domain->smmu;
832 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
833 int ret, irq;
834
835 if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
836 return;
837
838 ret = arm_smmu_rpm_get(smmu);
839 if (ret < 0)
840 return;
841
842 /*
843 * Disable the context bank and free the page tables before freeing
844 * it.
845 */
846 smmu->cbs[cfg->cbndx].cfg = NULL;
847 arm_smmu_write_context_bank(smmu, cfg->cbndx);
848
849 if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
850 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
851 devm_free_irq(smmu->dev, irq, domain);
852 }
853
854 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
855 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
856
857 arm_smmu_rpm_put(smmu);
858 }
859
860 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
861 {
862 struct arm_smmu_domain *smmu_domain;
863
864 if (type != IOMMU_DOMAIN_UNMANAGED &&
865 type != IOMMU_DOMAIN_DMA &&
866 type != IOMMU_DOMAIN_IDENTITY)
867 return NULL;
868 /*
869 * Allocate the domain and initialise some of its data structures.
870 * We can't really do anything meaningful until we've added a
871 * master.
872 */
873 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
874 if (!smmu_domain)
875 return NULL;
876
877 if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
878 iommu_get_dma_cookie(&smmu_domain->domain))) {
879 kfree(smmu_domain);
880 return NULL;
881 }
882
883 mutex_init(&smmu_domain->init_mutex);
884 spin_lock_init(&smmu_domain->cb_lock);
885
886 return &smmu_domain->domain;
887 }
888
889 static void arm_smmu_domain_free(struct iommu_domain *domain)
890 {
891 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
892
893 /*
894 * Free the domain resources. We assume that all devices have
895 * already been detached.
896 */
897 iommu_put_dma_cookie(domain);
898 arm_smmu_destroy_domain_context(domain);
899 kfree(smmu_domain);
900 }
901
902 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
903 {
904 struct arm_smmu_smr *smr = smmu->smrs + idx;
905 u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) |
906 FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask);
907
908 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
909 reg |= ARM_SMMU_SMR_VALID;
910 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
911 }
912
913 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
914 {
915 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
916 u32 reg;
917
918 if (smmu->impl && smmu->impl->write_s2cr) {
919 smmu->impl->write_s2cr(smmu, idx);
920 return;
921 }
922
923 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) |
924 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) |
925 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
926
927 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
928 smmu->smrs[idx].valid)
929 reg |= ARM_SMMU_S2CR_EXIDVALID;
930 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
931 }
932
933 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
934 {
935 arm_smmu_write_s2cr(smmu, idx);
936 if (smmu->smrs)
937 arm_smmu_write_smr(smmu, idx);
938 }
939
940 /*
941 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
942 * should be called after sCR0 is written.
943 */
944 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
945 {
946 u32 smr;
947 int i;
948
949 if (!smmu->smrs)
950 return;
951 /*
952 * If we've had to accommodate firmware memory regions, we may
953 * have live SMRs by now; tread carefully...
954 *
955 * Somewhat perversely, not having a free SMR for this test implies we
956 * can get away without it anyway, as we'll only be able to 'allocate'
957 * these SMRs for the ID/mask values we're already trusting to be OK.
958 */
959 for (i = 0; i < smmu->num_mapping_groups; i++)
960 if (!smmu->smrs[i].valid)
961 goto smr_ok;
962 return;
963 smr_ok:
964 /*
965 * SMR.ID bits may not be preserved if the corresponding MASK
966 * bits are set, so check each one separately. We can reject
967 * masters later if they try to claim IDs outside these masks.
968 */
969 smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);
970 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
971 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
972 smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);
973
974 smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);
975 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
976 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
977 smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
978 }
979
980 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
981 {
982 struct arm_smmu_smr *smrs = smmu->smrs;
983 int i, free_idx = -ENOSPC;
984
985 /* Stream indexing is blissfully easy */
986 if (!smrs)
987 return id;
988
989 /* Validating SMRs is... less so */
990 for (i = 0; i < smmu->num_mapping_groups; ++i) {
991 if (!smrs[i].valid) {
992 /*
993 * Note the first free entry we come across, which
994 * we'll claim in the end if nothing else matches.
995 */
996 if (free_idx < 0)
997 free_idx = i;
998 continue;
999 }
1000 /*
1001 * If the new entry is _entirely_ matched by an existing entry,
1002 * then reuse that, with the guarantee that there also cannot
1003 * be any subsequent conflicting entries. In normal use we'd
1004 * expect simply identical entries for this case, but there's
1005 * no harm in accommodating the generalisation.
1006 */
1007 if ((mask & smrs[i].mask) == mask &&
1008 !((id ^ smrs[i].id) & ~smrs[i].mask))
1009 return i;
1010 /*
1011 * If the new entry has any other overlap with an existing one,
1012 * though, then there always exists at least one stream ID
1013 * which would cause a conflict, and we can't allow that risk.
1014 */
1015 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1016 return -EINVAL;
1017 }
1018
1019 return free_idx;
1020 }
1021
1022 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1023 {
1024 if (--smmu->s2crs[idx].count)
1025 return false;
1026
1027 smmu->s2crs[idx] = s2cr_init_val;
1028 if (smmu->smrs)
1029 smmu->smrs[idx].valid = false;
1030
1031 return true;
1032 }
1033
1034 static int arm_smmu_master_alloc_smes(struct device *dev)
1035 {
1036 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1037 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1038 struct arm_smmu_device *smmu = cfg->smmu;
1039 struct arm_smmu_smr *smrs = smmu->smrs;
1040 int i, idx, ret;
1041
1042 mutex_lock(&smmu->stream_map_mutex);
1043 /* Figure out a viable stream map entry allocation */
1044 for_each_cfg_sme(cfg, fwspec, i, idx) {
1045 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1046 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1047
1048 if (idx != INVALID_SMENDX) {
1049 ret = -EEXIST;
1050 goto out_err;
1051 }
1052
1053 ret = arm_smmu_find_sme(smmu, sid, mask);
1054 if (ret < 0)
1055 goto out_err;
1056
1057 idx = ret;
1058 if (smrs && smmu->s2crs[idx].count == 0) {
1059 smrs[idx].id = sid;
1060 smrs[idx].mask = mask;
1061 smrs[idx].valid = true;
1062 }
1063 smmu->s2crs[idx].count++;
1064 cfg->smendx[i] = (s16)idx;
1065 }
1066
1067 /* It worked! Now, poke the actual hardware */
1068 for_each_cfg_sme(cfg, fwspec, i, idx)
1069 arm_smmu_write_sme(smmu, idx);
1070
1071 mutex_unlock(&smmu->stream_map_mutex);
1072 return 0;
1073
1074 out_err:
1075 while (i--) {
1076 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1077 cfg->smendx[i] = INVALID_SMENDX;
1078 }
1079 mutex_unlock(&smmu->stream_map_mutex);
1080 return ret;
1081 }
1082
1083 static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg,
1084 struct iommu_fwspec *fwspec)
1085 {
1086 struct arm_smmu_device *smmu = cfg->smmu;
1087 int i, idx;
1088
1089 mutex_lock(&smmu->stream_map_mutex);
1090 for_each_cfg_sme(cfg, fwspec, i, idx) {
1091 if (arm_smmu_free_sme(smmu, idx))
1092 arm_smmu_write_sme(smmu, idx);
1093 cfg->smendx[i] = INVALID_SMENDX;
1094 }
1095 mutex_unlock(&smmu->stream_map_mutex);
1096 }
1097
1098 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1099 struct arm_smmu_master_cfg *cfg,
1100 struct iommu_fwspec *fwspec)
1101 {
1102 struct arm_smmu_device *smmu = smmu_domain->smmu;
1103 struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1104 u8 cbndx = smmu_domain->cfg.cbndx;
1105 enum arm_smmu_s2cr_type type;
1106 int i, idx;
1107
1108 if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
1109 type = S2CR_TYPE_BYPASS;
1110 else
1111 type = S2CR_TYPE_TRANS;
1112
1113 for_each_cfg_sme(cfg, fwspec, i, idx) {
1114 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1115 continue;
1116
1117 s2cr[idx].type = type;
1118 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1119 s2cr[idx].cbndx = cbndx;
1120 arm_smmu_write_s2cr(smmu, idx);
1121 }
1122 return 0;
1123 }
1124
1125 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1126 {
1127 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1128 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1129 struct arm_smmu_master_cfg *cfg;
1130 struct arm_smmu_device *smmu;
1131 int ret;
1132
1133 if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1134 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1135 return -ENXIO;
1136 }
1137
1138 /*
1139 * FIXME: The arch/arm DMA API code tries to attach devices to its own
1140 * domains between of_xlate() and probe_device() - we have no way to cope
1141 * with that, so until ARM gets converted to rely on groups and default
1142 * domains, just say no (but more politely than by dereferencing NULL).
1143 * This should be at least a WARN_ON once that's sorted.
1144 */
1145 cfg = dev_iommu_priv_get(dev);
1146 if (!cfg)
1147 return -ENODEV;
1148
1149 smmu = cfg->smmu;
1150
1151 ret = arm_smmu_rpm_get(smmu);
1152 if (ret < 0)
1153 return ret;
1154
1155 /* Ensure that the domain is finalised */
1156 ret = arm_smmu_init_domain_context(domain, smmu, dev);
1157 if (ret < 0)
1158 goto rpm_put;
1159
1160 /*
1161 * Sanity check the domain. We don't support domains across
1162 * different SMMUs.
1163 */
1164 if (smmu_domain->smmu != smmu) {
1165 dev_err(dev,
1166 "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1167 dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1168 ret = -EINVAL;
1169 goto rpm_put;
1170 }
1171
1172 /* Looks ok, so add the device to the domain */
1173 ret = arm_smmu_domain_add_master(smmu_domain, cfg, fwspec);
1174
1175 /*
1176 * Setup an autosuspend delay to avoid bouncing runpm state.
1177 * Otherwise, if a driver for a suspended consumer device
1178 * unmaps buffers, it will runpm resume/suspend for each one.
1179 *
1180 * For example, when used by a GPU device, when an application
1181 * or game exits, it can trigger unmapping 100s or 1000s of
1182 * buffers. With a runpm cycle for each buffer, that adds up
1183 * to 5-10sec worth of reprogramming the context bank, while
1184 * the system appears to be locked up to the user.
1185 */
1186 pm_runtime_set_autosuspend_delay(smmu->dev, 20);
1187 pm_runtime_use_autosuspend(smmu->dev);
1188
1189 rpm_put:
1190 arm_smmu_rpm_put(smmu);
1191 return ret;
1192 }
1193
1194 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1195 phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
1196 {
1197 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1198 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1199 int ret;
1200
1201 if (!ops)
1202 return -ENODEV;
1203
1204 arm_smmu_rpm_get(smmu);
1205 ret = ops->map(ops, iova, paddr, size, prot, gfp);
1206 arm_smmu_rpm_put(smmu);
1207
1208 return ret;
1209 }
1210
1211 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1212 size_t size, struct iommu_iotlb_gather *gather)
1213 {
1214 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1215 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1216 size_t ret;
1217
1218 if (!ops)
1219 return 0;
1220
1221 arm_smmu_rpm_get(smmu);
1222 ret = ops->unmap(ops, iova, size, gather);
1223 arm_smmu_rpm_put(smmu);
1224
1225 return ret;
1226 }
1227
1228 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1229 {
1230 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1231 struct arm_smmu_device *smmu = smmu_domain->smmu;
1232
1233 if (smmu_domain->flush_ops) {
1234 arm_smmu_rpm_get(smmu);
1235 smmu_domain->flush_ops->tlb_flush_all(smmu_domain);
1236 arm_smmu_rpm_put(smmu);
1237 }
1238 }
1239
1240 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1241 struct iommu_iotlb_gather *gather)
1242 {
1243 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1244 struct arm_smmu_device *smmu = smmu_domain->smmu;
1245
1246 if (!smmu)
1247 return;
1248
1249 arm_smmu_rpm_get(smmu);
1250 if (smmu->version == ARM_SMMU_V2 ||
1251 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1252 arm_smmu_tlb_sync_context(smmu_domain);
1253 else
1254 arm_smmu_tlb_sync_global(smmu);
1255 arm_smmu_rpm_put(smmu);
1256 }
1257
1258 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1259 dma_addr_t iova)
1260 {
1261 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1262 struct arm_smmu_device *smmu = smmu_domain->smmu;
1263 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1264 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1265 struct device *dev = smmu->dev;
1266 void __iomem *reg;
1267 u32 tmp;
1268 u64 phys;
1269 unsigned long va, flags;
1270 int ret, idx = cfg->cbndx;
1271
1272 ret = arm_smmu_rpm_get(smmu);
1273 if (ret < 0)
1274 return 0;
1275
1276 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1277 va = iova & ~0xfffUL;
1278 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1279 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1280 else
1281 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1282
1283 reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1284 if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_ATSR_ACTIVE),
1285 5, 50)) {
1286 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1287 dev_err(dev,
1288 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1289 &iova);
1290 return ops->iova_to_phys(ops, iova);
1291 }
1292
1293 phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1294 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1295 if (phys & ARM_SMMU_CB_PAR_F) {
1296 dev_err(dev, "translation fault!\n");
1297 dev_err(dev, "PAR = 0x%llx\n", phys);
1298 return 0;
1299 }
1300
1301 arm_smmu_rpm_put(smmu);
1302
1303 return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1304 }
1305
1306 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1307 dma_addr_t iova)
1308 {
1309 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1310 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1311
1312 if (domain->type == IOMMU_DOMAIN_IDENTITY)
1313 return iova;
1314
1315 if (!ops)
1316 return 0;
1317
1318 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1319 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1320 return arm_smmu_iova_to_phys_hard(domain, iova);
1321
1322 return ops->iova_to_phys(ops, iova);
1323 }
1324
1325 static bool arm_smmu_capable(enum iommu_cap cap)
1326 {
1327 switch (cap) {
1328 case IOMMU_CAP_CACHE_COHERENCY:
1329 /*
1330 * Return true here as the SMMU can always send out coherent
1331 * requests.
1332 */
1333 return true;
1334 case IOMMU_CAP_NOEXEC:
1335 return true;
1336 default:
1337 return false;
1338 }
1339 }
1340
1341 static
1342 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1343 {
1344 struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
1345 fwnode);
1346 put_device(dev);
1347 return dev ? dev_get_drvdata(dev) : NULL;
1348 }
1349
1350 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
1351 {
1352 struct arm_smmu_device *smmu = NULL;
1353 struct arm_smmu_master_cfg *cfg;
1354 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1355 int i, ret;
1356
1357 if (using_legacy_binding) {
1358 ret = arm_smmu_register_legacy_master(dev, &smmu);
1359
1360 /*
1361 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1362 * will allocate/initialise a new one. Thus we need to update fwspec for
1363 * later use.
1364 */
1365 fwspec = dev_iommu_fwspec_get(dev);
1366 if (ret)
1367 goto out_free;
1368 } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1369 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1370 } else {
1371 return ERR_PTR(-ENODEV);
1372 }
1373
1374 ret = -EINVAL;
1375 for (i = 0; i < fwspec->num_ids; i++) {
1376 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1377 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1378
1379 if (sid & ~smmu->streamid_mask) {
1380 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1381 sid, smmu->streamid_mask);
1382 goto out_free;
1383 }
1384 if (mask & ~smmu->smr_mask_mask) {
1385 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1386 mask, smmu->smr_mask_mask);
1387 goto out_free;
1388 }
1389 }
1390
1391 ret = -ENOMEM;
1392 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1393 GFP_KERNEL);
1394 if (!cfg)
1395 goto out_free;
1396
1397 cfg->smmu = smmu;
1398 dev_iommu_priv_set(dev, cfg);
1399 while (i--)
1400 cfg->smendx[i] = INVALID_SMENDX;
1401
1402 ret = arm_smmu_rpm_get(smmu);
1403 if (ret < 0)
1404 goto out_cfg_free;
1405
1406 ret = arm_smmu_master_alloc_smes(dev);
1407 arm_smmu_rpm_put(smmu);
1408
1409 if (ret)
1410 goto out_cfg_free;
1411
1412 device_link_add(dev, smmu->dev,
1413 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1414
1415 return &smmu->iommu;
1416
1417 out_cfg_free:
1418 kfree(cfg);
1419 out_free:
1420 iommu_fwspec_free(dev);
1421 return ERR_PTR(ret);
1422 }
1423
1424 static void arm_smmu_release_device(struct device *dev)
1425 {
1426 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1427 struct arm_smmu_master_cfg *cfg;
1428 struct arm_smmu_device *smmu;
1429 int ret;
1430
1431 if (!fwspec || fwspec->ops != &arm_smmu_ops)
1432 return;
1433
1434 cfg = dev_iommu_priv_get(dev);
1435 smmu = cfg->smmu;
1436
1437 ret = arm_smmu_rpm_get(smmu);
1438 if (ret < 0)
1439 return;
1440
1441 arm_smmu_master_free_smes(cfg, fwspec);
1442
1443 arm_smmu_rpm_put(smmu);
1444
1445 dev_iommu_priv_set(dev, NULL);
1446 kfree(cfg);
1447 iommu_fwspec_free(dev);
1448 }
1449
1450 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1451 {
1452 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1453 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1454 struct arm_smmu_device *smmu = cfg->smmu;
1455 struct iommu_group *group = NULL;
1456 int i, idx;
1457
1458 for_each_cfg_sme(cfg, fwspec, i, idx) {
1459 if (group && smmu->s2crs[idx].group &&
1460 group != smmu->s2crs[idx].group)
1461 return ERR_PTR(-EINVAL);
1462
1463 group = smmu->s2crs[idx].group;
1464 }
1465
1466 if (group)
1467 return iommu_group_ref_get(group);
1468
1469 if (dev_is_pci(dev))
1470 group = pci_device_group(dev);
1471 else if (dev_is_fsl_mc(dev))
1472 group = fsl_mc_device_group(dev);
1473 else
1474 group = generic_device_group(dev);
1475
1476 /* Remember group for faster lookups */
1477 if (!IS_ERR(group))
1478 for_each_cfg_sme(cfg, fwspec, i, idx)
1479 smmu->s2crs[idx].group = group;
1480
1481 return group;
1482 }
1483
1484 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1485 enum iommu_attr attr, void *data)
1486 {
1487 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1488
1489 switch(domain->type) {
1490 case IOMMU_DOMAIN_UNMANAGED:
1491 switch (attr) {
1492 case DOMAIN_ATTR_NESTING:
1493 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1494 return 0;
1495 case DOMAIN_ATTR_IO_PGTABLE_CFG: {
1496 struct io_pgtable_domain_attr *pgtbl_cfg = data;
1497 *pgtbl_cfg = smmu_domain->pgtbl_cfg;
1498
1499 return 0;
1500 }
1501 default:
1502 return -ENODEV;
1503 }
1504 break;
1505 case IOMMU_DOMAIN_DMA:
1506 switch (attr) {
1507 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE: {
1508 bool non_strict = smmu_domain->pgtbl_cfg.quirks &
1509 IO_PGTABLE_QUIRK_NON_STRICT;
1510 *(int *)data = non_strict;
1511 return 0;
1512 }
1513 default:
1514 return -ENODEV;
1515 }
1516 break;
1517 default:
1518 return -EINVAL;
1519 }
1520 }
1521
1522 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1523 enum iommu_attr attr, void *data)
1524 {
1525 int ret = 0;
1526 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1527
1528 mutex_lock(&smmu_domain->init_mutex);
1529
1530 switch(domain->type) {
1531 case IOMMU_DOMAIN_UNMANAGED:
1532 switch (attr) {
1533 case DOMAIN_ATTR_NESTING:
1534 if (smmu_domain->smmu) {
1535 ret = -EPERM;
1536 goto out_unlock;
1537 }
1538
1539 if (*(int *)data)
1540 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1541 else
1542 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1543 break;
1544 case DOMAIN_ATTR_IO_PGTABLE_CFG: {
1545 struct io_pgtable_domain_attr *pgtbl_cfg = data;
1546
1547 if (smmu_domain->smmu) {
1548 ret = -EPERM;
1549 goto out_unlock;
1550 }
1551
1552 smmu_domain->pgtbl_cfg = *pgtbl_cfg;
1553 break;
1554 }
1555 default:
1556 ret = -ENODEV;
1557 }
1558 break;
1559 case IOMMU_DOMAIN_DMA:
1560 switch (attr) {
1561 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1562 if (*(int *)data)
1563 smmu_domain->pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
1564 else
1565 smmu_domain->pgtbl_cfg.quirks &= ~IO_PGTABLE_QUIRK_NON_STRICT;
1566 break;
1567 default:
1568 ret = -ENODEV;
1569 }
1570 break;
1571 default:
1572 ret = -EINVAL;
1573 }
1574 out_unlock:
1575 mutex_unlock(&smmu_domain->init_mutex);
1576 return ret;
1577 }
1578
1579 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1580 {
1581 u32 mask, fwid = 0;
1582
1583 if (args->args_count > 0)
1584 fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]);
1585
1586 if (args->args_count > 1)
1587 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]);
1588 else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1589 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask);
1590
1591 return iommu_fwspec_add_ids(dev, &fwid, 1);
1592 }
1593
1594 static void arm_smmu_get_resv_regions(struct device *dev,
1595 struct list_head *head)
1596 {
1597 struct iommu_resv_region *region;
1598 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1599
1600 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1601 prot, IOMMU_RESV_SW_MSI);
1602 if (!region)
1603 return;
1604
1605 list_add_tail(&region->list, head);
1606
1607 iommu_dma_get_resv_regions(dev, head);
1608 }
1609
1610 static int arm_smmu_def_domain_type(struct device *dev)
1611 {
1612 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1613 const struct arm_smmu_impl *impl = cfg->smmu->impl;
1614
1615 if (impl && impl->def_domain_type)
1616 return impl->def_domain_type(dev);
1617
1618 return 0;
1619 }
1620
1621 static struct iommu_ops arm_smmu_ops = {
1622 .capable = arm_smmu_capable,
1623 .domain_alloc = arm_smmu_domain_alloc,
1624 .domain_free = arm_smmu_domain_free,
1625 .attach_dev = arm_smmu_attach_dev,
1626 .map = arm_smmu_map,
1627 .unmap = arm_smmu_unmap,
1628 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
1629 .iotlb_sync = arm_smmu_iotlb_sync,
1630 .iova_to_phys = arm_smmu_iova_to_phys,
1631 .probe_device = arm_smmu_probe_device,
1632 .release_device = arm_smmu_release_device,
1633 .device_group = arm_smmu_device_group,
1634 .domain_get_attr = arm_smmu_domain_get_attr,
1635 .domain_set_attr = arm_smmu_domain_set_attr,
1636 .of_xlate = arm_smmu_of_xlate,
1637 .get_resv_regions = arm_smmu_get_resv_regions,
1638 .put_resv_regions = generic_iommu_put_resv_regions,
1639 .def_domain_type = arm_smmu_def_domain_type,
1640 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1641 };
1642
1643 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1644 {
1645 int i;
1646 u32 reg;
1647
1648 /* clear global FSR */
1649 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1650 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1651
1652 /*
1653 * Reset stream mapping groups: Initial values mark all SMRn as
1654 * invalid and all S2CRn as bypass unless overridden.
1655 */
1656 for (i = 0; i < smmu->num_mapping_groups; ++i)
1657 arm_smmu_write_sme(smmu, i);
1658
1659 /* Make sure all context banks are disabled and clear CB_FSR */
1660 for (i = 0; i < smmu->num_context_banks; ++i) {
1661 arm_smmu_write_context_bank(smmu, i);
1662 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_FSR_FAULT);
1663 }
1664
1665 /* Invalidate the TLB, just in case */
1666 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1667 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1668
1669 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1670
1671 /* Enable fault reporting */
1672 reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE |
1673 ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE);
1674
1675 /* Disable TLB broadcasting. */
1676 reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM);
1677
1678 /* Enable client access, handling unmatched streams as appropriate */
1679 reg &= ~ARM_SMMU_sCR0_CLIENTPD;
1680 if (disable_bypass)
1681 reg |= ARM_SMMU_sCR0_USFCFG;
1682 else
1683 reg &= ~ARM_SMMU_sCR0_USFCFG;
1684
1685 /* Disable forced broadcasting */
1686 reg &= ~ARM_SMMU_sCR0_FB;
1687
1688 /* Don't upgrade barriers */
1689 reg &= ~(ARM_SMMU_sCR0_BSU);
1690
1691 if (smmu->features & ARM_SMMU_FEAT_VMID16)
1692 reg |= ARM_SMMU_sCR0_VMID16EN;
1693
1694 if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1695 reg |= ARM_SMMU_sCR0_EXIDENABLE;
1696
1697 if (smmu->impl && smmu->impl->reset)
1698 smmu->impl->reset(smmu);
1699
1700 /* Push the button */
1701 arm_smmu_tlb_sync_global(smmu);
1702 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1703 }
1704
1705 static int arm_smmu_id_size_to_bits(int size)
1706 {
1707 switch (size) {
1708 case 0:
1709 return 32;
1710 case 1:
1711 return 36;
1712 case 2:
1713 return 40;
1714 case 3:
1715 return 42;
1716 case 4:
1717 return 44;
1718 case 5:
1719 default:
1720 return 48;
1721 }
1722 }
1723
1724 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1725 {
1726 unsigned int size;
1727 u32 id;
1728 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1729 int i, ret;
1730
1731 dev_notice(smmu->dev, "probing hardware configuration...\n");
1732 dev_notice(smmu->dev, "SMMUv%d with:\n",
1733 smmu->version == ARM_SMMU_V2 ? 2 : 1);
1734
1735 /* ID0 */
1736 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1737
1738 /* Restrict available stages based on module parameter */
1739 if (force_stage == 1)
1740 id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS);
1741 else if (force_stage == 2)
1742 id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS);
1743
1744 if (id & ARM_SMMU_ID0_S1TS) {
1745 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1746 dev_notice(smmu->dev, "\tstage 1 translation\n");
1747 }
1748
1749 if (id & ARM_SMMU_ID0_S2TS) {
1750 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1751 dev_notice(smmu->dev, "\tstage 2 translation\n");
1752 }
1753
1754 if (id & ARM_SMMU_ID0_NTS) {
1755 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1756 dev_notice(smmu->dev, "\tnested translation\n");
1757 }
1758
1759 if (!(smmu->features &
1760 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1761 dev_err(smmu->dev, "\tno translation support!\n");
1762 return -ENODEV;
1763 }
1764
1765 if ((id & ARM_SMMU_ID0_S1TS) &&
1766 ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) {
1767 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1768 dev_notice(smmu->dev, "\taddress translation ops\n");
1769 }
1770
1771 /*
1772 * In order for DMA API calls to work properly, we must defer to what
1773 * the FW says about coherency, regardless of what the hardware claims.
1774 * Fortunately, this also opens up a workaround for systems where the
1775 * ID register value has ended up configured incorrectly.
1776 */
1777 cttw_reg = !!(id & ARM_SMMU_ID0_CTTW);
1778 if (cttw_fw || cttw_reg)
1779 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1780 cttw_fw ? "" : "non-");
1781 if (cttw_fw != cttw_reg)
1782 dev_notice(smmu->dev,
1783 "\t(IDR0.CTTW overridden by FW configuration)\n");
1784
1785 /* Max. number of entries we have for stream matching/indexing */
1786 if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {
1787 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1788 size = 1 << 16;
1789 } else {
1790 size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id);
1791 }
1792 smmu->streamid_mask = size - 1;
1793 if (id & ARM_SMMU_ID0_SMS) {
1794 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1795 size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id);
1796 if (size == 0) {
1797 dev_err(smmu->dev,
1798 "stream-matching supported, but no SMRs present!\n");
1799 return -ENODEV;
1800 }
1801
1802 /* Zero-initialised to mark as invalid */
1803 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1804 GFP_KERNEL);
1805 if (!smmu->smrs)
1806 return -ENOMEM;
1807
1808 dev_notice(smmu->dev,
1809 "\tstream matching with %u register groups", size);
1810 }
1811 /* s2cr->type == 0 means translation, so initialise explicitly */
1812 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1813 GFP_KERNEL);
1814 if (!smmu->s2crs)
1815 return -ENOMEM;
1816 for (i = 0; i < size; i++)
1817 smmu->s2crs[i] = s2cr_init_val;
1818
1819 smmu->num_mapping_groups = size;
1820 mutex_init(&smmu->stream_map_mutex);
1821 spin_lock_init(&smmu->global_sync_lock);
1822
1823 if (smmu->version < ARM_SMMU_V2 ||
1824 !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) {
1825 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1826 if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S))
1827 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1828 }
1829
1830 /* ID1 */
1831 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1832 smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;
1833
1834 /* Check for size mismatch of SMMU address space from mapped region */
1835 size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1);
1836 if (smmu->numpage != 2 * size << smmu->pgshift)
1837 dev_warn(smmu->dev,
1838 "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1839 2 * size << smmu->pgshift, smmu->numpage);
1840 /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1841 smmu->numpage = size;
1842
1843 smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);
1844 smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);
1845 if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1846 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1847 return -ENODEV;
1848 }
1849 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1850 smmu->num_context_banks, smmu->num_s2_context_banks);
1851 smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1852 sizeof(*smmu->cbs), GFP_KERNEL);
1853 if (!smmu->cbs)
1854 return -ENOMEM;
1855
1856 /* ID2 */
1857 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1858 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id));
1859 smmu->ipa_size = size;
1860
1861 /* The output mask is also applied for bypass */
1862 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id));
1863 smmu->pa_size = size;
1864
1865 if (id & ARM_SMMU_ID2_VMID16)
1866 smmu->features |= ARM_SMMU_FEAT_VMID16;
1867
1868 /*
1869 * What the page table walker can address actually depends on which
1870 * descriptor format is in use, but since a) we don't know that yet,
1871 * and b) it can vary per context bank, this will have to do...
1872 */
1873 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1874 dev_warn(smmu->dev,
1875 "failed to set DMA mask for table walker\n");
1876
1877 if (smmu->version < ARM_SMMU_V2) {
1878 smmu->va_size = smmu->ipa_size;
1879 if (smmu->version == ARM_SMMU_V1_64K)
1880 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1881 } else {
1882 size = FIELD_GET(ARM_SMMU_ID2_UBS, id);
1883 smmu->va_size = arm_smmu_id_size_to_bits(size);
1884 if (id & ARM_SMMU_ID2_PTFS_4K)
1885 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1886 if (id & ARM_SMMU_ID2_PTFS_16K)
1887 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1888 if (id & ARM_SMMU_ID2_PTFS_64K)
1889 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1890 }
1891
1892 if (smmu->impl && smmu->impl->cfg_probe) {
1893 ret = smmu->impl->cfg_probe(smmu);
1894 if (ret)
1895 return ret;
1896 }
1897
1898 /* Now we've corralled the various formats, what'll it do? */
1899 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1900 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1901 if (smmu->features &
1902 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1903 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1904 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1905 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1906 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1907 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1908
1909 if (arm_smmu_ops.pgsize_bitmap == -1UL)
1910 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1911 else
1912 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1913 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1914 smmu->pgsize_bitmap);
1915
1916
1917 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1918 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1919 smmu->va_size, smmu->ipa_size);
1920
1921 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1922 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1923 smmu->ipa_size, smmu->pa_size);
1924
1925 return 0;
1926 }
1927
1928 struct arm_smmu_match_data {
1929 enum arm_smmu_arch_version version;
1930 enum arm_smmu_implementation model;
1931 };
1932
1933 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \
1934 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1935
1936 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1937 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1938 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1939 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1940 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1941 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1942
1943 static const struct of_device_id arm_smmu_of_match[] = {
1944 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1945 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1946 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1947 { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1948 { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1949 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1950 { .compatible = "nvidia,smmu-500", .data = &arm_mmu500 },
1951 { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1952 { },
1953 };
1954 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1955
1956 #ifdef CONFIG_ACPI
1957 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1958 {
1959 int ret = 0;
1960
1961 switch (model) {
1962 case ACPI_IORT_SMMU_V1:
1963 case ACPI_IORT_SMMU_CORELINK_MMU400:
1964 smmu->version = ARM_SMMU_V1;
1965 smmu->model = GENERIC_SMMU;
1966 break;
1967 case ACPI_IORT_SMMU_CORELINK_MMU401:
1968 smmu->version = ARM_SMMU_V1_64K;
1969 smmu->model = GENERIC_SMMU;
1970 break;
1971 case ACPI_IORT_SMMU_V2:
1972 smmu->version = ARM_SMMU_V2;
1973 smmu->model = GENERIC_SMMU;
1974 break;
1975 case ACPI_IORT_SMMU_CORELINK_MMU500:
1976 smmu->version = ARM_SMMU_V2;
1977 smmu->model = ARM_MMU500;
1978 break;
1979 case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1980 smmu->version = ARM_SMMU_V2;
1981 smmu->model = CAVIUM_SMMUV2;
1982 break;
1983 default:
1984 ret = -ENODEV;
1985 }
1986
1987 return ret;
1988 }
1989
1990 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
1991 struct arm_smmu_device *smmu)
1992 {
1993 struct device *dev = smmu->dev;
1994 struct acpi_iort_node *node =
1995 *(struct acpi_iort_node **)dev_get_platdata(dev);
1996 struct acpi_iort_smmu *iort_smmu;
1997 int ret;
1998
1999 /* Retrieve SMMU1/2 specific data */
2000 iort_smmu = (struct acpi_iort_smmu *)node->node_data;
2001
2002 ret = acpi_smmu_get_data(iort_smmu->model, smmu);
2003 if (ret < 0)
2004 return ret;
2005
2006 /* Ignore the configuration access interrupt */
2007 smmu->num_global_irqs = 1;
2008
2009 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
2010 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2011
2012 return 0;
2013 }
2014 #else
2015 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2016 struct arm_smmu_device *smmu)
2017 {
2018 return -ENODEV;
2019 }
2020 #endif
2021
2022 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
2023 struct arm_smmu_device *smmu)
2024 {
2025 const struct arm_smmu_match_data *data;
2026 struct device *dev = &pdev->dev;
2027 bool legacy_binding;
2028
2029 if (of_property_read_u32(dev->of_node, "#global-interrupts",
2030 &smmu->num_global_irqs)) {
2031 dev_err(dev, "missing #global-interrupts property\n");
2032 return -ENODEV;
2033 }
2034
2035 data = of_device_get_match_data(dev);
2036 smmu->version = data->version;
2037 smmu->model = data->model;
2038
2039 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2040 if (legacy_binding && !using_generic_binding) {
2041 if (!using_legacy_binding) {
2042 pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
2043 IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
2044 }
2045 using_legacy_binding = true;
2046 } else if (!legacy_binding && !using_legacy_binding) {
2047 using_generic_binding = true;
2048 } else {
2049 dev_err(dev, "not probing due to mismatched DT properties\n");
2050 return -ENODEV;
2051 }
2052
2053 if (of_dma_is_coherent(dev->of_node))
2054 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2055
2056 return 0;
2057 }
2058
2059 static int arm_smmu_bus_init(struct iommu_ops *ops)
2060 {
2061 int err;
2062
2063 /* Oh, for a proper bus abstraction */
2064 if (!iommu_present(&platform_bus_type)) {
2065 err = bus_set_iommu(&platform_bus_type, ops);
2066 if (err)
2067 return err;
2068 }
2069 #ifdef CONFIG_ARM_AMBA
2070 if (!iommu_present(&amba_bustype)) {
2071 err = bus_set_iommu(&amba_bustype, ops);
2072 if (err)
2073 goto err_reset_platform_ops;
2074 }
2075 #endif
2076 #ifdef CONFIG_PCI
2077 if (!iommu_present(&pci_bus_type)) {
2078 err = bus_set_iommu(&pci_bus_type, ops);
2079 if (err)
2080 goto err_reset_amba_ops;
2081 }
2082 #endif
2083 #ifdef CONFIG_FSL_MC_BUS
2084 if (!iommu_present(&fsl_mc_bus_type)) {
2085 err = bus_set_iommu(&fsl_mc_bus_type, ops);
2086 if (err)
2087 goto err_reset_pci_ops;
2088 }
2089 #endif
2090 return 0;
2091
2092 err_reset_pci_ops: __maybe_unused;
2093 #ifdef CONFIG_PCI
2094 bus_set_iommu(&pci_bus_type, NULL);
2095 #endif
2096 err_reset_amba_ops: __maybe_unused;
2097 #ifdef CONFIG_ARM_AMBA
2098 bus_set_iommu(&amba_bustype, NULL);
2099 #endif
2100 err_reset_platform_ops: __maybe_unused;
2101 bus_set_iommu(&platform_bus_type, NULL);
2102 return err;
2103 }
2104
2105 static int arm_smmu_device_probe(struct platform_device *pdev)
2106 {
2107 struct resource *res;
2108 resource_size_t ioaddr;
2109 struct arm_smmu_device *smmu;
2110 struct device *dev = &pdev->dev;
2111 int num_irqs, i, err;
2112 irqreturn_t (*global_fault)(int irq, void *dev);
2113
2114 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2115 if (!smmu) {
2116 dev_err(dev, "failed to allocate arm_smmu_device\n");
2117 return -ENOMEM;
2118 }
2119 smmu->dev = dev;
2120
2121 if (dev->of_node)
2122 err = arm_smmu_device_dt_probe(pdev, smmu);
2123 else
2124 err = arm_smmu_device_acpi_probe(pdev, smmu);
2125
2126 if (err)
2127 return err;
2128
2129 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2130 ioaddr = res->start;
2131 smmu->base = devm_ioremap_resource(dev, res);
2132 if (IS_ERR(smmu->base))
2133 return PTR_ERR(smmu->base);
2134 /*
2135 * The resource size should effectively match the value of SMMU_TOP;
2136 * stash that temporarily until we know PAGESIZE to validate it with.
2137 */
2138 smmu->numpage = resource_size(res);
2139
2140 smmu = arm_smmu_impl_init(smmu);
2141 if (IS_ERR(smmu))
2142 return PTR_ERR(smmu);
2143
2144 num_irqs = 0;
2145 while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2146 num_irqs++;
2147 if (num_irqs > smmu->num_global_irqs)
2148 smmu->num_context_irqs++;
2149 }
2150
2151 if (!smmu->num_context_irqs) {
2152 dev_err(dev, "found %d interrupts but expected at least %d\n",
2153 num_irqs, smmu->num_global_irqs + 1);
2154 return -ENODEV;
2155 }
2156
2157 smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs),
2158 GFP_KERNEL);
2159 if (!smmu->irqs) {
2160 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2161 return -ENOMEM;
2162 }
2163
2164 for (i = 0; i < num_irqs; ++i) {
2165 int irq = platform_get_irq(pdev, i);
2166
2167 if (irq < 0)
2168 return -ENODEV;
2169 smmu->irqs[i] = irq;
2170 }
2171
2172 err = devm_clk_bulk_get_all(dev, &smmu->clks);
2173 if (err < 0) {
2174 dev_err(dev, "failed to get clocks %d\n", err);
2175 return err;
2176 }
2177 smmu->num_clks = err;
2178
2179 err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2180 if (err)
2181 return err;
2182
2183 err = arm_smmu_device_cfg_probe(smmu);
2184 if (err)
2185 return err;
2186
2187 if (smmu->version == ARM_SMMU_V2) {
2188 if (smmu->num_context_banks > smmu->num_context_irqs) {
2189 dev_err(dev,
2190 "found only %d context irq(s) but %d required\n",
2191 smmu->num_context_irqs, smmu->num_context_banks);
2192 return -ENODEV;
2193 }
2194
2195 /* Ignore superfluous interrupts */
2196 smmu->num_context_irqs = smmu->num_context_banks;
2197 }
2198
2199 if (smmu->impl && smmu->impl->global_fault)
2200 global_fault = smmu->impl->global_fault;
2201 else
2202 global_fault = arm_smmu_global_fault;
2203
2204 for (i = 0; i < smmu->num_global_irqs; ++i) {
2205 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2206 global_fault,
2207 IRQF_SHARED,
2208 "arm-smmu global fault",
2209 smmu);
2210 if (err) {
2211 dev_err(dev, "failed to request global IRQ %d (%u)\n",
2212 i, smmu->irqs[i]);
2213 return err;
2214 }
2215 }
2216
2217 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2218 "smmu.%pa", &ioaddr);
2219 if (err) {
2220 dev_err(dev, "Failed to register iommu in sysfs\n");
2221 return err;
2222 }
2223
2224 iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
2225 iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
2226
2227 err = iommu_device_register(&smmu->iommu);
2228 if (err) {
2229 dev_err(dev, "Failed to register iommu\n");
2230 return err;
2231 }
2232
2233 platform_set_drvdata(pdev, smmu);
2234 arm_smmu_device_reset(smmu);
2235 arm_smmu_test_smr_masks(smmu);
2236
2237 /*
2238 * We want to avoid touching dev->power.lock in fastpaths unless
2239 * it's really going to do something useful - pm_runtime_enabled()
2240 * can serve as an ideal proxy for that decision. So, conditionally
2241 * enable pm_runtime.
2242 */
2243 if (dev->pm_domain) {
2244 pm_runtime_set_active(dev);
2245 pm_runtime_enable(dev);
2246 }
2247
2248 /*
2249 * For ACPI and generic DT bindings, an SMMU will be probed before
2250 * any device which might need it, so we want the bus ops in place
2251 * ready to handle default domain setup as soon as any SMMU exists.
2252 */
2253 if (!using_legacy_binding)
2254 return arm_smmu_bus_init(&arm_smmu_ops);
2255
2256 return 0;
2257 }
2258
2259 static int arm_smmu_device_remove(struct platform_device *pdev)
2260 {
2261 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2262
2263 if (!smmu)
2264 return -ENODEV;
2265
2266 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2267 dev_notice(&pdev->dev, "disabling translation\n");
2268
2269 arm_smmu_bus_init(NULL);
2270 iommu_device_unregister(&smmu->iommu);
2271 iommu_device_sysfs_remove(&smmu->iommu);
2272
2273 arm_smmu_rpm_get(smmu);
2274 /* Turn the thing off */
2275 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);
2276 arm_smmu_rpm_put(smmu);
2277
2278 if (pm_runtime_enabled(smmu->dev))
2279 pm_runtime_force_suspend(smmu->dev);
2280 else
2281 clk_bulk_disable(smmu->num_clks, smmu->clks);
2282
2283 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2284 return 0;
2285 }
2286
2287 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2288 {
2289 arm_smmu_device_remove(pdev);
2290 }
2291
2292 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2293 {
2294 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2295 int ret;
2296
2297 ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2298 if (ret)
2299 return ret;
2300
2301 arm_smmu_device_reset(smmu);
2302
2303 return 0;
2304 }
2305
2306 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2307 {
2308 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2309
2310 clk_bulk_disable(smmu->num_clks, smmu->clks);
2311
2312 return 0;
2313 }
2314
2315 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2316 {
2317 if (pm_runtime_suspended(dev))
2318 return 0;
2319
2320 return arm_smmu_runtime_resume(dev);
2321 }
2322
2323 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2324 {
2325 if (pm_runtime_suspended(dev))
2326 return 0;
2327
2328 return arm_smmu_runtime_suspend(dev);
2329 }
2330
2331 static const struct dev_pm_ops arm_smmu_pm_ops = {
2332 SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2333 SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2334 arm_smmu_runtime_resume, NULL)
2335 };
2336
2337 static struct platform_driver arm_smmu_driver = {
2338 .driver = {
2339 .name = "arm-smmu",
2340 .of_match_table = arm_smmu_of_match,
2341 .pm = &arm_smmu_pm_ops,
2342 .suppress_bind_attrs = true,
2343 },
2344 .probe = arm_smmu_device_probe,
2345 .remove = arm_smmu_device_remove,
2346 .shutdown = arm_smmu_device_shutdown,
2347 };
2348 module_platform_driver(arm_smmu_driver);
2349
2350 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2351 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
2352 MODULE_ALIAS("platform:arm-smmu");
2353 MODULE_LICENSE("GPL v2");
Linux with added FPC-III support