?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
ocfs2: avoid inode removal while nfsd is accessing it
[linux/fpc-iii.git] / drivers / iommu / arm-smmu.c
blob243bc4cb2705b331d71329c61d753f303499c998
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * IOMMU API for ARM architected SMMU implementations.
4 *
5 * Copyright (C) 2013 ARM Limited
6 *
7 * Author: Will Deacon <will.deacon@arm.com>
8 *
9 * This driver currently supports:
10 * - SMMUv1 and v2 implementations
11 * - Stream-matching and stream-indexing
12 * - v7/v8 long-descriptor format
13 * - Non-secure access to the SMMU
14 * - Context fault reporting
15 * - Extended Stream ID (16 bit)
16 */
17
18 #define pr_fmt(fmt) "arm-smmu: " fmt
19
20 #include <linux/acpi.h>
21 #include <linux/acpi_iort.h>
22 #include <linux/bitfield.h>
23 #include <linux/delay.h>
24 #include <linux/dma-iommu.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/err.h>
27 #include <linux/interrupt.h>
28 #include <linux/io.h>
29 #include <linux/iopoll.h>
30 #include <linux/module.h>
31 #include <linux/of.h>
32 #include <linux/of_address.h>
33 #include <linux/of_device.h>
34 #include <linux/of_iommu.h>
35 #include <linux/pci.h>
36 #include <linux/platform_device.h>
37 #include <linux/pm_runtime.h>
38 #include <linux/ratelimit.h>
39 #include <linux/slab.h>
40
41 #include <linux/amba/bus.h>
42 #include <linux/fsl/mc.h>
43
44 #include "arm-smmu.h"
45
46 /*
47 * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
48 * global register space are still, in fact, using a hypervisor to mediate it
49 * by trapping and emulating register accesses. Sadly, some deployed versions
50 * of said trapping code have bugs wherein they go horribly wrong for stores
51 * using r31 (i.e. XZR/WZR) as the source register.
52 */
53 #define QCOM_DUMMY_VAL -1
54
55 #define TLB_LOOP_TIMEOUT 1000000 /* 1s! */
56 #define TLB_SPIN_COUNT 10
57
58 #define MSI_IOVA_BASE 0x8000000
59 #define MSI_IOVA_LENGTH 0x100000
60
61 static int force_stage;
62 module_param(force_stage, int, S_IRUGO);
63 MODULE_PARM_DESC(force_stage,
64 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
65 static bool disable_bypass =
66 IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
67 module_param(disable_bypass, bool, S_IRUGO);
68 MODULE_PARM_DESC(disable_bypass,
69 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
70
71 struct arm_smmu_s2cr {
72 struct iommu_group *group;
73 int count;
74 enum arm_smmu_s2cr_type type;
75 enum arm_smmu_s2cr_privcfg privcfg;
76 u8 cbndx;
77 };
78
79 #define s2cr_init_val (struct arm_smmu_s2cr){ \
80 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
81 }
82
83 struct arm_smmu_smr {
84 u16 mask;
85 u16 id;
86 bool valid;
87 };
88
89 struct arm_smmu_cb {
90 u64 ttbr[2];
91 u32 tcr[2];
92 u32 mair[2];
93 struct arm_smmu_cfg *cfg;
94 };
95
96 struct arm_smmu_master_cfg {
97 struct arm_smmu_device *smmu;
98 s16 smendx[];
99 };
100 #define INVALID_SMENDX -1
101 #define cfg_smendx(cfg, fw, i) \
102 (i >= fw->num_ids ? INVALID_SMENDX : cfg->smendx[i])
103 #define for_each_cfg_sme(cfg, fw, i, idx) \
104 for (i = 0; idx = cfg_smendx(cfg, fw, i), i < fw->num_ids; ++i)
105
106 static bool using_legacy_binding, using_generic_binding;
107
108 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
109 {
110 if (pm_runtime_enabled(smmu->dev))
111 return pm_runtime_get_sync(smmu->dev);
112
113 return 0;
114 }
115
116 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
117 {
118 if (pm_runtime_enabled(smmu->dev))
119 pm_runtime_put_autosuspend(smmu->dev);
120 }
121
122 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
123 {
124 return container_of(dom, struct arm_smmu_domain, domain);
125 }
126
127 static struct platform_driver arm_smmu_driver;
128 static struct iommu_ops arm_smmu_ops;
129
130 #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
131 static int arm_smmu_bus_init(struct iommu_ops *ops);
132
133 static struct device_node *dev_get_dev_node(struct device *dev)
134 {
135 if (dev_is_pci(dev)) {
136 struct pci_bus *bus = to_pci_dev(dev)->bus;
137
138 while (!pci_is_root_bus(bus))
139 bus = bus->parent;
140 return of_node_get(bus->bridge->parent->of_node);
141 }
142
143 return of_node_get(dev->of_node);
144 }
145
146 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
147 {
148 *((__be32 *)data) = cpu_to_be32(alias);
149 return 0; /* Continue walking */
150 }
151
152 static int __find_legacy_master_phandle(struct device *dev, void *data)
153 {
154 struct of_phandle_iterator *it = *(void **)data;
155 struct device_node *np = it->node;
156 int err;
157
158 of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
159 "#stream-id-cells", -1)
160 if (it->node == np) {
161 *(void **)data = dev;
162 return 1;
163 }
164 it->node = np;
165 return err == -ENOENT ? 0 : err;
166 }
167
168 static int arm_smmu_register_legacy_master(struct device *dev,
169 struct arm_smmu_device **smmu)
170 {
171 struct device *smmu_dev;
172 struct device_node *np;
173 struct of_phandle_iterator it;
174 void *data = &it;
175 u32 *sids;
176 __be32 pci_sid;
177 int err;
178
179 np = dev_get_dev_node(dev);
180 if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
181 of_node_put(np);
182 return -ENODEV;
183 }
184
185 it.node = np;
186 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
187 __find_legacy_master_phandle);
188 smmu_dev = data;
189 of_node_put(np);
190 if (err == 0)
191 return -ENODEV;
192 if (err < 0)
193 return err;
194
195 if (dev_is_pci(dev)) {
196 /* "mmu-masters" assumes Stream ID == Requester ID */
197 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
198 &pci_sid);
199 it.cur = &pci_sid;
200 it.cur_count = 1;
201 }
202
203 err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
204 &arm_smmu_ops);
205 if (err)
206 return err;
207
208 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
209 if (!sids)
210 return -ENOMEM;
211
212 *smmu = dev_get_drvdata(smmu_dev);
213 of_phandle_iterator_args(&it, sids, it.cur_count);
214 err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
215 kfree(sids);
216 return err;
217 }
218
219 /*
220 * With the legacy DT binding in play, we have no guarantees about
221 * probe order, but then we're also not doing default domains, so we can
222 * delay setting bus ops until we're sure every possible SMMU is ready,
223 * and that way ensure that no probe_device() calls get missed.
224 */
225 static int arm_smmu_legacy_bus_init(void)
226 {
227 if (using_legacy_binding)
228 return arm_smmu_bus_init(&arm_smmu_ops);
229 return 0;
230 }
231 device_initcall_sync(arm_smmu_legacy_bus_init);
232 #else
233 static int arm_smmu_register_legacy_master(struct device *dev,
234 struct arm_smmu_device **smmu)
235 {
236 return -ENODEV;
237 }
238 #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
239
240 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
241 {
242 int idx;
243
244 do {
245 idx = find_next_zero_bit(map, end, start);
246 if (idx == end)
247 return -ENOSPC;
248 } while (test_and_set_bit(idx, map));
249
250 return idx;
251 }
252
253 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
254 {
255 clear_bit(idx, map);
256 }
257
258 /* Wait for any pending TLB invalidations to complete */
259 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
260 int sync, int status)
261 {
262 unsigned int spin_cnt, delay;
263 u32 reg;
264
265 if (smmu->impl && unlikely(smmu->impl->tlb_sync))
266 return smmu->impl->tlb_sync(smmu, page, sync, status);
267
268 arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
269 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
270 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
271 reg = arm_smmu_readl(smmu, page, status);
272 if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
273 return;
274 cpu_relax();
275 }
276 udelay(delay);
277 }
278 dev_err_ratelimited(smmu->dev,
279 "TLB sync timed out -- SMMU may be deadlocked\n");
280 }
281
282 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
283 {
284 unsigned long flags;
285
286 spin_lock_irqsave(&smmu->global_sync_lock, flags);
287 __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
288 ARM_SMMU_GR0_sTLBGSTATUS);
289 spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
290 }
291
292 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain)
293 {
294 struct arm_smmu_device *smmu = smmu_domain->smmu;
295 unsigned long flags;
296
297 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
298 __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
299 ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
300 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
301 }
302
303 static void arm_smmu_tlb_inv_context_s1(void *cookie)
304 {
305 struct arm_smmu_domain *smmu_domain = cookie;
306 /*
307 * The TLBI write may be relaxed, so ensure that PTEs cleared by the
308 * current CPU are visible beforehand.
309 */
310 wmb();
311 arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
312 ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
313 arm_smmu_tlb_sync_context(smmu_domain);
314 }
315
316 static void arm_smmu_tlb_inv_context_s2(void *cookie)
317 {
318 struct arm_smmu_domain *smmu_domain = cookie;
319 struct arm_smmu_device *smmu = smmu_domain->smmu;
320
321 /* See above */
322 wmb();
323 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
324 arm_smmu_tlb_sync_global(smmu);
325 }
326
327 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
328 size_t granule, void *cookie, int reg)
329 {
330 struct arm_smmu_domain *smmu_domain = cookie;
331 struct arm_smmu_device *smmu = smmu_domain->smmu;
332 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
333 int idx = cfg->cbndx;
334
335 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
336 wmb();
337
338 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
339 iova = (iova >> 12) << 12;
340 iova |= cfg->asid;
341 do {
342 arm_smmu_cb_write(smmu, idx, reg, iova);
343 iova += granule;
344 } while (size -= granule);
345 } else {
346 iova >>= 12;
347 iova |= (u64)cfg->asid << 48;
348 do {
349 arm_smmu_cb_writeq(smmu, idx, reg, iova);
350 iova += granule >> 12;
351 } while (size -= granule);
352 }
353 }
354
355 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
356 size_t granule, void *cookie, int reg)
357 {
358 struct arm_smmu_domain *smmu_domain = cookie;
359 struct arm_smmu_device *smmu = smmu_domain->smmu;
360 int idx = smmu_domain->cfg.cbndx;
361
362 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
363 wmb();
364
365 iova >>= 12;
366 do {
367 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
368 arm_smmu_cb_writeq(smmu, idx, reg, iova);
369 else
370 arm_smmu_cb_write(smmu, idx, reg, iova);
371 iova += granule >> 12;
372 } while (size -= granule);
373 }
374
375 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size,
376 size_t granule, void *cookie)
377 {
378 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
379 ARM_SMMU_CB_S1_TLBIVA);
380 arm_smmu_tlb_sync_context(cookie);
381 }
382
383 static void arm_smmu_tlb_inv_leaf_s1(unsigned long iova, size_t size,
384 size_t granule, void *cookie)
385 {
386 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
387 ARM_SMMU_CB_S1_TLBIVAL);
388 arm_smmu_tlb_sync_context(cookie);
389 }
390
391 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather,
392 unsigned long iova, size_t granule,
393 void *cookie)
394 {
395 arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie,
396 ARM_SMMU_CB_S1_TLBIVAL);
397 }
398
399 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size,
400 size_t granule, void *cookie)
401 {
402 arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
403 ARM_SMMU_CB_S2_TLBIIPAS2);
404 arm_smmu_tlb_sync_context(cookie);
405 }
406
407 static void arm_smmu_tlb_inv_leaf_s2(unsigned long iova, size_t size,
408 size_t granule, void *cookie)
409 {
410 arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
411 ARM_SMMU_CB_S2_TLBIIPAS2L);
412 arm_smmu_tlb_sync_context(cookie);
413 }
414
415 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather,
416 unsigned long iova, size_t granule,
417 void *cookie)
418 {
419 arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie,
420 ARM_SMMU_CB_S2_TLBIIPAS2L);
421 }
422
423 static void arm_smmu_tlb_inv_any_s2_v1(unsigned long iova, size_t size,
424 size_t granule, void *cookie)
425 {
426 arm_smmu_tlb_inv_context_s2(cookie);
427 }
428 /*
429 * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
430 * almost negligible, but the benefit of getting the first one in as far ahead
431 * of the sync as possible is significant, hence we don't just make this a
432 * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might
433 * think.
434 */
435 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather,
436 unsigned long iova, size_t granule,
437 void *cookie)
438 {
439 struct arm_smmu_domain *smmu_domain = cookie;
440 struct arm_smmu_device *smmu = smmu_domain->smmu;
441
442 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
443 wmb();
444
445 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
446 }
447
448 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = {
449 .tlb_flush_all = arm_smmu_tlb_inv_context_s1,
450 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1,
451 .tlb_flush_leaf = arm_smmu_tlb_inv_leaf_s1,
452 .tlb_add_page = arm_smmu_tlb_add_page_s1,
453 };
454
455 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
456 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
457 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2,
458 .tlb_flush_leaf = arm_smmu_tlb_inv_leaf_s2,
459 .tlb_add_page = arm_smmu_tlb_add_page_s2,
460 };
461
462 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
463 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
464 .tlb_flush_walk = arm_smmu_tlb_inv_any_s2_v1,
465 .tlb_flush_leaf = arm_smmu_tlb_inv_any_s2_v1,
466 .tlb_add_page = arm_smmu_tlb_add_page_s2_v1,
467 };
468
469 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
470 {
471 u32 fsr, fsynr, cbfrsynra;
472 unsigned long iova;
473 struct iommu_domain *domain = dev;
474 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
475 struct arm_smmu_device *smmu = smmu_domain->smmu;
476 int idx = smmu_domain->cfg.cbndx;
477
478 fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
479 if (!(fsr & ARM_SMMU_FSR_FAULT))
480 return IRQ_NONE;
481
482 fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
483 iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
484 cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
485
486 dev_err_ratelimited(smmu->dev,
487 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
488 fsr, iova, fsynr, cbfrsynra, idx);
489
490 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);
491 return IRQ_HANDLED;
492 }
493
494 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
495 {
496 u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
497 struct arm_smmu_device *smmu = dev;
498 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
499 DEFAULT_RATELIMIT_BURST);
500
501 gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
502 gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
503 gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
504 gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
505
506 if (!gfsr)
507 return IRQ_NONE;
508
509 if (__ratelimit(&rs)) {
510 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
511 (gfsr & ARM_SMMU_sGFSR_USF))
512 dev_err(smmu->dev,
513 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
514 (u16)gfsynr1);
515 else
516 dev_err(smmu->dev,
517 "Unexpected global fault, this could be serious\n");
518 dev_err(smmu->dev,
519 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
520 gfsr, gfsynr0, gfsynr1, gfsynr2);
521 }
522
523 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
524 return IRQ_HANDLED;
525 }
526
527 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
528 struct io_pgtable_cfg *pgtbl_cfg)
529 {
530 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
531 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
532 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
533
534 cb->cfg = cfg;
535
536 /* TCR */
537 if (stage1) {
538 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
539 cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
540 } else {
541 cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg);
542 cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg);
543 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
544 cb->tcr[1] |= ARM_SMMU_TCR2_AS;
545 else
546 cb->tcr[0] |= ARM_SMMU_TCR_EAE;
547 }
548 } else {
549 cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg);
550 }
551
552 /* TTBRs */
553 if (stage1) {
554 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
555 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
556 cb->ttbr[1] = 0;
557 } else {
558 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
559 cb->ttbr[0] |= FIELD_PREP(ARM_SMMU_TTBRn_ASID,
560 cfg->asid);
561 cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
562 cfg->asid);
563 }
564 } else {
565 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
566 }
567
568 /* MAIRs (stage-1 only) */
569 if (stage1) {
570 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
571 cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
572 cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
573 } else {
574 cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair;
575 cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32;
576 }
577 }
578 }
579
580 static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
581 {
582 u32 reg;
583 bool stage1;
584 struct arm_smmu_cb *cb = &smmu->cbs[idx];
585 struct arm_smmu_cfg *cfg = cb->cfg;
586
587 /* Unassigned context banks only need disabling */
588 if (!cfg) {
589 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
590 return;
591 }
592
593 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
594
595 /* CBA2R */
596 if (smmu->version > ARM_SMMU_V1) {
597 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
598 reg = ARM_SMMU_CBA2R_VA64;
599 else
600 reg = 0;
601 /* 16-bit VMIDs live in CBA2R */
602 if (smmu->features & ARM_SMMU_FEAT_VMID16)
603 reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid);
604
605 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
606 }
607
608 /* CBAR */
609 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar);
610 if (smmu->version < ARM_SMMU_V2)
611 reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx);
612
613 /*
614 * Use the weakest shareability/memory types, so they are
615 * overridden by the ttbcr/pte.
616 */
617 if (stage1) {
618 reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG,
619 ARM_SMMU_CBAR_S1_BPSHCFG_NSH) |
620 FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR,
621 ARM_SMMU_CBAR_S1_MEMATTR_WB);
622 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
623 /* 8-bit VMIDs live in CBAR */
624 reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid);
625 }
626 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
627
628 /*
629 * TCR
630 * We must write this before the TTBRs, since it determines the
631 * access behaviour of some fields (in particular, ASID[15:8]).
632 */
633 if (stage1 && smmu->version > ARM_SMMU_V1)
634 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
635 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
636
637 /* TTBRs */
638 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
639 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
640 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
641 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
642 } else {
643 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
644 if (stage1)
645 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
646 cb->ttbr[1]);
647 }
648
649 /* MAIRs (stage-1 only) */
650 if (stage1) {
651 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
652 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
653 }
654
655 /* SCTLR */
656 reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE |
657 ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M;
658 if (stage1)
659 reg |= ARM_SMMU_SCTLR_S1_ASIDPNE;
660 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
661 reg |= ARM_SMMU_SCTLR_E;
662
663 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
664 }
665
666 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
667 struct arm_smmu_device *smmu)
668 {
669 int irq, start, ret = 0;
670 unsigned long ias, oas;
671 struct io_pgtable_ops *pgtbl_ops;
672 struct io_pgtable_cfg pgtbl_cfg;
673 enum io_pgtable_fmt fmt;
674 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
675 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
676
677 mutex_lock(&smmu_domain->init_mutex);
678 if (smmu_domain->smmu)
679 goto out_unlock;
680
681 if (domain->type == IOMMU_DOMAIN_IDENTITY) {
682 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
683 smmu_domain->smmu = smmu;
684 goto out_unlock;
685 }
686
687 /*
688 * Mapping the requested stage onto what we support is surprisingly
689 * complicated, mainly because the spec allows S1+S2 SMMUs without
690 * support for nested translation. That means we end up with the
691 * following table:
692 *
693 * Requested Supported Actual
694 * S1 N S1
695 * S1 S1+S2 S1
696 * S1 S2 S2
697 * S1 S1 S1
698 * N N N
699 * N S1+S2 S2
700 * N S2 S2
701 * N S1 S1
702 *
703 * Note that you can't actually request stage-2 mappings.
704 */
705 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
706 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
707 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
708 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
709
710 /*
711 * Choosing a suitable context format is even more fiddly. Until we
712 * grow some way for the caller to express a preference, and/or move
713 * the decision into the io-pgtable code where it arguably belongs,
714 * just aim for the closest thing to the rest of the system, and hope
715 * that the hardware isn't esoteric enough that we can't assume AArch64
716 * support to be a superset of AArch32 support...
717 */
718 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
719 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
720 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
721 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
722 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
723 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
724 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
725 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
726 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
727 ARM_SMMU_FEAT_FMT_AARCH64_16K |
728 ARM_SMMU_FEAT_FMT_AARCH64_4K)))
729 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
730
731 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
732 ret = -EINVAL;
733 goto out_unlock;
734 }
735
736 switch (smmu_domain->stage) {
737 case ARM_SMMU_DOMAIN_S1:
738 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
739 start = smmu->num_s2_context_banks;
740 ias = smmu->va_size;
741 oas = smmu->ipa_size;
742 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
743 fmt = ARM_64_LPAE_S1;
744 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
745 fmt = ARM_32_LPAE_S1;
746 ias = min(ias, 32UL);
747 oas = min(oas, 40UL);
748 } else {
749 fmt = ARM_V7S;
750 ias = min(ias, 32UL);
751 oas = min(oas, 32UL);
752 }
753 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
754 break;
755 case ARM_SMMU_DOMAIN_NESTED:
756 /*
757 * We will likely want to change this if/when KVM gets
758 * involved.
759 */
760 case ARM_SMMU_DOMAIN_S2:
761 cfg->cbar = CBAR_TYPE_S2_TRANS;
762 start = 0;
763 ias = smmu->ipa_size;
764 oas = smmu->pa_size;
765 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
766 fmt = ARM_64_LPAE_S2;
767 } else {
768 fmt = ARM_32_LPAE_S2;
769 ias = min(ias, 40UL);
770 oas = min(oas, 40UL);
771 }
772 if (smmu->version == ARM_SMMU_V2)
773 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
774 else
775 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
776 break;
777 default:
778 ret = -EINVAL;
779 goto out_unlock;
780 }
781 ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
782 smmu->num_context_banks);
783 if (ret < 0)
784 goto out_unlock;
785
786 cfg->cbndx = ret;
787 if (smmu->version < ARM_SMMU_V2) {
788 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
789 cfg->irptndx %= smmu->num_context_irqs;
790 } else {
791 cfg->irptndx = cfg->cbndx;
792 }
793
794 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
795 cfg->vmid = cfg->cbndx + 1;
796 else
797 cfg->asid = cfg->cbndx;
798
799 smmu_domain->smmu = smmu;
800 if (smmu->impl && smmu->impl->init_context) {
801 ret = smmu->impl->init_context(smmu_domain);
802 if (ret)
803 goto out_unlock;
804 }
805
806 pgtbl_cfg = (struct io_pgtable_cfg) {
807 .pgsize_bitmap = smmu->pgsize_bitmap,
808 .ias = ias,
809 .oas = oas,
810 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
811 .tlb = smmu_domain->flush_ops,
812 .iommu_dev = smmu->dev,
813 };
814
815 if (smmu_domain->non_strict)
816 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
817
818 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
819 if (!pgtbl_ops) {
820 ret = -ENOMEM;
821 goto out_clear_smmu;
822 }
823
824 /* Update the domain's page sizes to reflect the page table format */
825 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
826 domain->geometry.aperture_end = (1UL << ias) - 1;
827 domain->geometry.force_aperture = true;
828
829 /* Initialise the context bank with our page table cfg */
830 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
831 arm_smmu_write_context_bank(smmu, cfg->cbndx);
832
833 /*
834 * Request context fault interrupt. Do this last to avoid the
835 * handler seeing a half-initialised domain state.
836 */
837 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
838 ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
839 IRQF_SHARED, "arm-smmu-context-fault", domain);
840 if (ret < 0) {
841 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
842 cfg->irptndx, irq);
843 cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
844 }
845
846 mutex_unlock(&smmu_domain->init_mutex);
847
848 /* Publish page table ops for map/unmap */
849 smmu_domain->pgtbl_ops = pgtbl_ops;
850 return 0;
851
852 out_clear_smmu:
853 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
854 smmu_domain->smmu = NULL;
855 out_unlock:
856 mutex_unlock(&smmu_domain->init_mutex);
857 return ret;
858 }
859
860 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
861 {
862 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
863 struct arm_smmu_device *smmu = smmu_domain->smmu;
864 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
865 int ret, irq;
866
867 if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
868 return;
869
870 ret = arm_smmu_rpm_get(smmu);
871 if (ret < 0)
872 return;
873
874 /*
875 * Disable the context bank and free the page tables before freeing
876 * it.
877 */
878 smmu->cbs[cfg->cbndx].cfg = NULL;
879 arm_smmu_write_context_bank(smmu, cfg->cbndx);
880
881 if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
882 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
883 devm_free_irq(smmu->dev, irq, domain);
884 }
885
886 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
887 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
888
889 arm_smmu_rpm_put(smmu);
890 }
891
892 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
893 {
894 struct arm_smmu_domain *smmu_domain;
895
896 if (type != IOMMU_DOMAIN_UNMANAGED &&
897 type != IOMMU_DOMAIN_DMA &&
898 type != IOMMU_DOMAIN_IDENTITY)
899 return NULL;
900 /*
901 * Allocate the domain and initialise some of its data structures.
902 * We can't really do anything meaningful until we've added a
903 * master.
904 */
905 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
906 if (!smmu_domain)
907 return NULL;
908
909 if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
910 iommu_get_dma_cookie(&smmu_domain->domain))) {
911 kfree(smmu_domain);
912 return NULL;
913 }
914
915 mutex_init(&smmu_domain->init_mutex);
916 spin_lock_init(&smmu_domain->cb_lock);
917
918 return &smmu_domain->domain;
919 }
920
921 static void arm_smmu_domain_free(struct iommu_domain *domain)
922 {
923 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
924
925 /*
926 * Free the domain resources. We assume that all devices have
927 * already been detached.
928 */
929 iommu_put_dma_cookie(domain);
930 arm_smmu_destroy_domain_context(domain);
931 kfree(smmu_domain);
932 }
933
934 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
935 {
936 struct arm_smmu_smr *smr = smmu->smrs + idx;
937 u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) |
938 FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask);
939
940 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
941 reg |= ARM_SMMU_SMR_VALID;
942 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
943 }
944
945 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
946 {
947 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
948 u32 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) |
949 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) |
950 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
951
952 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
953 smmu->smrs[idx].valid)
954 reg |= ARM_SMMU_S2CR_EXIDVALID;
955 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
956 }
957
958 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
959 {
960 arm_smmu_write_s2cr(smmu, idx);
961 if (smmu->smrs)
962 arm_smmu_write_smr(smmu, idx);
963 }
964
965 /*
966 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
967 * should be called after sCR0 is written.
968 */
969 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
970 {
971 u32 smr;
972 int i;
973
974 if (!smmu->smrs)
975 return;
976 /*
977 * If we've had to accommodate firmware memory regions, we may
978 * have live SMRs by now; tread carefully...
979 *
980 * Somewhat perversely, not having a free SMR for this test implies we
981 * can get away without it anyway, as we'll only be able to 'allocate'
982 * these SMRs for the ID/mask values we're already trusting to be OK.
983 */
984 for (i = 0; i < smmu->num_mapping_groups; i++)
985 if (!smmu->smrs[i].valid)
986 goto smr_ok;
987 return;
988 smr_ok:
989 /*
990 * SMR.ID bits may not be preserved if the corresponding MASK
991 * bits are set, so check each one separately. We can reject
992 * masters later if they try to claim IDs outside these masks.
993 */
994 smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);
995 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
996 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
997 smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);
998
999 smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);
1000 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
1001 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
1002 smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
1003 }
1004
1005 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1006 {
1007 struct arm_smmu_smr *smrs = smmu->smrs;
1008 int i, free_idx = -ENOSPC;
1009
1010 /* Stream indexing is blissfully easy */
1011 if (!smrs)
1012 return id;
1013
1014 /* Validating SMRs is... less so */
1015 for (i = 0; i < smmu->num_mapping_groups; ++i) {
1016 if (!smrs[i].valid) {
1017 /*
1018 * Note the first free entry we come across, which
1019 * we'll claim in the end if nothing else matches.
1020 */
1021 if (free_idx < 0)
1022 free_idx = i;
1023 continue;
1024 }
1025 /*
1026 * If the new entry is _entirely_ matched by an existing entry,
1027 * then reuse that, with the guarantee that there also cannot
1028 * be any subsequent conflicting entries. In normal use we'd
1029 * expect simply identical entries for this case, but there's
1030 * no harm in accommodating the generalisation.
1031 */
1032 if ((mask & smrs[i].mask) == mask &&
1033 !((id ^ smrs[i].id) & ~smrs[i].mask))
1034 return i;
1035 /*
1036 * If the new entry has any other overlap with an existing one,
1037 * though, then there always exists at least one stream ID
1038 * which would cause a conflict, and we can't allow that risk.
1039 */
1040 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1041 return -EINVAL;
1042 }
1043
1044 return free_idx;
1045 }
1046
1047 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1048 {
1049 if (--smmu->s2crs[idx].count)
1050 return false;
1051
1052 smmu->s2crs[idx] = s2cr_init_val;
1053 if (smmu->smrs)
1054 smmu->smrs[idx].valid = false;
1055
1056 return true;
1057 }
1058
1059 static int arm_smmu_master_alloc_smes(struct device *dev)
1060 {
1061 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1062 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1063 struct arm_smmu_device *smmu = cfg->smmu;
1064 struct arm_smmu_smr *smrs = smmu->smrs;
1065 int i, idx, ret;
1066
1067 mutex_lock(&smmu->stream_map_mutex);
1068 /* Figure out a viable stream map entry allocation */
1069 for_each_cfg_sme(cfg, fwspec, i, idx) {
1070 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1071 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1072
1073 if (idx != INVALID_SMENDX) {
1074 ret = -EEXIST;
1075 goto out_err;
1076 }
1077
1078 ret = arm_smmu_find_sme(smmu, sid, mask);
1079 if (ret < 0)
1080 goto out_err;
1081
1082 idx = ret;
1083 if (smrs && smmu->s2crs[idx].count == 0) {
1084 smrs[idx].id = sid;
1085 smrs[idx].mask = mask;
1086 smrs[idx].valid = true;
1087 }
1088 smmu->s2crs[idx].count++;
1089 cfg->smendx[i] = (s16)idx;
1090 }
1091
1092 /* It worked! Now, poke the actual hardware */
1093 for_each_cfg_sme(cfg, fwspec, i, idx)
1094 arm_smmu_write_sme(smmu, idx);
1095
1096 mutex_unlock(&smmu->stream_map_mutex);
1097 return 0;
1098
1099 out_err:
1100 while (i--) {
1101 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1102 cfg->smendx[i] = INVALID_SMENDX;
1103 }
1104 mutex_unlock(&smmu->stream_map_mutex);
1105 return ret;
1106 }
1107
1108 static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg,
1109 struct iommu_fwspec *fwspec)
1110 {
1111 struct arm_smmu_device *smmu = cfg->smmu;
1112 int i, idx;
1113
1114 mutex_lock(&smmu->stream_map_mutex);
1115 for_each_cfg_sme(cfg, fwspec, i, idx) {
1116 if (arm_smmu_free_sme(smmu, idx))
1117 arm_smmu_write_sme(smmu, idx);
1118 cfg->smendx[i] = INVALID_SMENDX;
1119 }
1120 mutex_unlock(&smmu->stream_map_mutex);
1121 }
1122
1123 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1124 struct arm_smmu_master_cfg *cfg,
1125 struct iommu_fwspec *fwspec)
1126 {
1127 struct arm_smmu_device *smmu = smmu_domain->smmu;
1128 struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1129 u8 cbndx = smmu_domain->cfg.cbndx;
1130 enum arm_smmu_s2cr_type type;
1131 int i, idx;
1132
1133 if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
1134 type = S2CR_TYPE_BYPASS;
1135 else
1136 type = S2CR_TYPE_TRANS;
1137
1138 for_each_cfg_sme(cfg, fwspec, i, idx) {
1139 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1140 continue;
1141
1142 s2cr[idx].type = type;
1143 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1144 s2cr[idx].cbndx = cbndx;
1145 arm_smmu_write_s2cr(smmu, idx);
1146 }
1147 return 0;
1148 }
1149
1150 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1151 {
1152 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1153 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1154 struct arm_smmu_master_cfg *cfg;
1155 struct arm_smmu_device *smmu;
1156 int ret;
1157
1158 if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1159 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1160 return -ENXIO;
1161 }
1162
1163 /*
1164 * FIXME: The arch/arm DMA API code tries to attach devices to its own
1165 * domains between of_xlate() and probe_device() - we have no way to cope
1166 * with that, so until ARM gets converted to rely on groups and default
1167 * domains, just say no (but more politely than by dereferencing NULL).
1168 * This should be at least a WARN_ON once that's sorted.
1169 */
1170 cfg = dev_iommu_priv_get(dev);
1171 if (!cfg)
1172 return -ENODEV;
1173
1174 smmu = cfg->smmu;
1175
1176 ret = arm_smmu_rpm_get(smmu);
1177 if (ret < 0)
1178 return ret;
1179
1180 /* Ensure that the domain is finalised */
1181 ret = arm_smmu_init_domain_context(domain, smmu);
1182 if (ret < 0)
1183 goto rpm_put;
1184
1185 /*
1186 * Sanity check the domain. We don't support domains across
1187 * different SMMUs.
1188 */
1189 if (smmu_domain->smmu != smmu) {
1190 dev_err(dev,
1191 "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1192 dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1193 ret = -EINVAL;
1194 goto rpm_put;
1195 }
1196
1197 /* Looks ok, so add the device to the domain */
1198 ret = arm_smmu_domain_add_master(smmu_domain, cfg, fwspec);
1199
1200 /*
1201 * Setup an autosuspend delay to avoid bouncing runpm state.
1202 * Otherwise, if a driver for a suspended consumer device
1203 * unmaps buffers, it will runpm resume/suspend for each one.
1204 *
1205 * For example, when used by a GPU device, when an application
1206 * or game exits, it can trigger unmapping 100s or 1000s of
1207 * buffers. With a runpm cycle for each buffer, that adds up
1208 * to 5-10sec worth of reprogramming the context bank, while
1209 * the system appears to be locked up to the user.
1210 */
1211 pm_runtime_set_autosuspend_delay(smmu->dev, 20);
1212 pm_runtime_use_autosuspend(smmu->dev);
1213
1214 rpm_put:
1215 arm_smmu_rpm_put(smmu);
1216 return ret;
1217 }
1218
1219 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1220 phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
1221 {
1222 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1223 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1224 int ret;
1225
1226 if (!ops)
1227 return -ENODEV;
1228
1229 arm_smmu_rpm_get(smmu);
1230 ret = ops->map(ops, iova, paddr, size, prot);
1231 arm_smmu_rpm_put(smmu);
1232
1233 return ret;
1234 }
1235
1236 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1237 size_t size, struct iommu_iotlb_gather *gather)
1238 {
1239 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1240 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1241 size_t ret;
1242
1243 if (!ops)
1244 return 0;
1245
1246 arm_smmu_rpm_get(smmu);
1247 ret = ops->unmap(ops, iova, size, gather);
1248 arm_smmu_rpm_put(smmu);
1249
1250 return ret;
1251 }
1252
1253 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1254 {
1255 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1256 struct arm_smmu_device *smmu = smmu_domain->smmu;
1257
1258 if (smmu_domain->flush_ops) {
1259 arm_smmu_rpm_get(smmu);
1260 smmu_domain->flush_ops->tlb_flush_all(smmu_domain);
1261 arm_smmu_rpm_put(smmu);
1262 }
1263 }
1264
1265 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1266 struct iommu_iotlb_gather *gather)
1267 {
1268 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1269 struct arm_smmu_device *smmu = smmu_domain->smmu;
1270
1271 if (!smmu)
1272 return;
1273
1274 arm_smmu_rpm_get(smmu);
1275 if (smmu->version == ARM_SMMU_V2 ||
1276 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1277 arm_smmu_tlb_sync_context(smmu_domain);
1278 else
1279 arm_smmu_tlb_sync_global(smmu);
1280 arm_smmu_rpm_put(smmu);
1281 }
1282
1283 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1284 dma_addr_t iova)
1285 {
1286 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1287 struct arm_smmu_device *smmu = smmu_domain->smmu;
1288 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1289 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1290 struct device *dev = smmu->dev;
1291 void __iomem *reg;
1292 u32 tmp;
1293 u64 phys;
1294 unsigned long va, flags;
1295 int ret, idx = cfg->cbndx;
1296
1297 ret = arm_smmu_rpm_get(smmu);
1298 if (ret < 0)
1299 return 0;
1300
1301 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1302 va = iova & ~0xfffUL;
1303 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1304 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1305 else
1306 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1307
1308 reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1309 if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_ATSR_ACTIVE),
1310 5, 50)) {
1311 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1312 dev_err(dev,
1313 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1314 &iova);
1315 return ops->iova_to_phys(ops, iova);
1316 }
1317
1318 phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1319 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1320 if (phys & ARM_SMMU_CB_PAR_F) {
1321 dev_err(dev, "translation fault!\n");
1322 dev_err(dev, "PAR = 0x%llx\n", phys);
1323 return 0;
1324 }
1325
1326 arm_smmu_rpm_put(smmu);
1327
1328 return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1329 }
1330
1331 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1332 dma_addr_t iova)
1333 {
1334 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1335 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1336
1337 if (domain->type == IOMMU_DOMAIN_IDENTITY)
1338 return iova;
1339
1340 if (!ops)
1341 return 0;
1342
1343 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1344 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1345 return arm_smmu_iova_to_phys_hard(domain, iova);
1346
1347 return ops->iova_to_phys(ops, iova);
1348 }
1349
1350 static bool arm_smmu_capable(enum iommu_cap cap)
1351 {
1352 switch (cap) {
1353 case IOMMU_CAP_CACHE_COHERENCY:
1354 /*
1355 * Return true here as the SMMU can always send out coherent
1356 * requests.
1357 */
1358 return true;
1359 case IOMMU_CAP_NOEXEC:
1360 return true;
1361 default:
1362 return false;
1363 }
1364 }
1365
1366 static
1367 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1368 {
1369 struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
1370 fwnode);
1371 put_device(dev);
1372 return dev ? dev_get_drvdata(dev) : NULL;
1373 }
1374
1375 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
1376 {
1377 struct arm_smmu_device *smmu = NULL;
1378 struct arm_smmu_master_cfg *cfg;
1379 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1380 int i, ret;
1381
1382 if (using_legacy_binding) {
1383 ret = arm_smmu_register_legacy_master(dev, &smmu);
1384
1385 /*
1386 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1387 * will allocate/initialise a new one. Thus we need to update fwspec for
1388 * later use.
1389 */
1390 fwspec = dev_iommu_fwspec_get(dev);
1391 if (ret)
1392 goto out_free;
1393 } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1394 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1395 } else {
1396 return ERR_PTR(-ENODEV);
1397 }
1398
1399 ret = -EINVAL;
1400 for (i = 0; i < fwspec->num_ids; i++) {
1401 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1402 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1403
1404 if (sid & ~smmu->streamid_mask) {
1405 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1406 sid, smmu->streamid_mask);
1407 goto out_free;
1408 }
1409 if (mask & ~smmu->smr_mask_mask) {
1410 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1411 mask, smmu->smr_mask_mask);
1412 goto out_free;
1413 }
1414 }
1415
1416 ret = -ENOMEM;
1417 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1418 GFP_KERNEL);
1419 if (!cfg)
1420 goto out_free;
1421
1422 cfg->smmu = smmu;
1423 dev_iommu_priv_set(dev, cfg);
1424 while (i--)
1425 cfg->smendx[i] = INVALID_SMENDX;
1426
1427 ret = arm_smmu_rpm_get(smmu);
1428 if (ret < 0)
1429 goto out_cfg_free;
1430
1431 ret = arm_smmu_master_alloc_smes(dev);
1432 arm_smmu_rpm_put(smmu);
1433
1434 if (ret)
1435 goto out_cfg_free;
1436
1437 device_link_add(dev, smmu->dev,
1438 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1439
1440 return &smmu->iommu;
1441
1442 out_cfg_free:
1443 kfree(cfg);
1444 out_free:
1445 iommu_fwspec_free(dev);
1446 return ERR_PTR(ret);
1447 }
1448
1449 static void arm_smmu_release_device(struct device *dev)
1450 {
1451 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1452 struct arm_smmu_master_cfg *cfg;
1453 struct arm_smmu_device *smmu;
1454 int ret;
1455
1456 if (!fwspec || fwspec->ops != &arm_smmu_ops)
1457 return;
1458
1459 cfg = dev_iommu_priv_get(dev);
1460 smmu = cfg->smmu;
1461
1462 ret = arm_smmu_rpm_get(smmu);
1463 if (ret < 0)
1464 return;
1465
1466 arm_smmu_master_free_smes(cfg, fwspec);
1467
1468 arm_smmu_rpm_put(smmu);
1469
1470 dev_iommu_priv_set(dev, NULL);
1471 kfree(cfg);
1472 iommu_fwspec_free(dev);
1473 }
1474
1475 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1476 {
1477 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1478 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1479 struct arm_smmu_device *smmu = cfg->smmu;
1480 struct iommu_group *group = NULL;
1481 int i, idx;
1482
1483 for_each_cfg_sme(cfg, fwspec, i, idx) {
1484 if (group && smmu->s2crs[idx].group &&
1485 group != smmu->s2crs[idx].group)
1486 return ERR_PTR(-EINVAL);
1487
1488 group = smmu->s2crs[idx].group;
1489 }
1490
1491 if (group)
1492 return iommu_group_ref_get(group);
1493
1494 if (dev_is_pci(dev))
1495 group = pci_device_group(dev);
1496 else if (dev_is_fsl_mc(dev))
1497 group = fsl_mc_device_group(dev);
1498 else
1499 group = generic_device_group(dev);
1500
1501 /* Remember group for faster lookups */
1502 if (!IS_ERR(group))
1503 for_each_cfg_sme(cfg, fwspec, i, idx)
1504 smmu->s2crs[idx].group = group;
1505
1506 return group;
1507 }
1508
1509 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1510 enum iommu_attr attr, void *data)
1511 {
1512 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1513
1514 switch(domain->type) {
1515 case IOMMU_DOMAIN_UNMANAGED:
1516 switch (attr) {
1517 case DOMAIN_ATTR_NESTING:
1518 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1519 return 0;
1520 default:
1521 return -ENODEV;
1522 }
1523 break;
1524 case IOMMU_DOMAIN_DMA:
1525 switch (attr) {
1526 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1527 *(int *)data = smmu_domain->non_strict;
1528 return 0;
1529 default:
1530 return -ENODEV;
1531 }
1532 break;
1533 default:
1534 return -EINVAL;
1535 }
1536 }
1537
1538 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1539 enum iommu_attr attr, void *data)
1540 {
1541 int ret = 0;
1542 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1543
1544 mutex_lock(&smmu_domain->init_mutex);
1545
1546 switch(domain->type) {
1547 case IOMMU_DOMAIN_UNMANAGED:
1548 switch (attr) {
1549 case DOMAIN_ATTR_NESTING:
1550 if (smmu_domain->smmu) {
1551 ret = -EPERM;
1552 goto out_unlock;
1553 }
1554
1555 if (*(int *)data)
1556 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1557 else
1558 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1559 break;
1560 default:
1561 ret = -ENODEV;
1562 }
1563 break;
1564 case IOMMU_DOMAIN_DMA:
1565 switch (attr) {
1566 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1567 smmu_domain->non_strict = *(int *)data;
1568 break;
1569 default:
1570 ret = -ENODEV;
1571 }
1572 break;
1573 default:
1574 ret = -EINVAL;
1575 }
1576 out_unlock:
1577 mutex_unlock(&smmu_domain->init_mutex);
1578 return ret;
1579 }
1580
1581 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1582 {
1583 u32 mask, fwid = 0;
1584
1585 if (args->args_count > 0)
1586 fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]);
1587
1588 if (args->args_count > 1)
1589 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]);
1590 else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1591 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask);
1592
1593 return iommu_fwspec_add_ids(dev, &fwid, 1);
1594 }
1595
1596 static void arm_smmu_get_resv_regions(struct device *dev,
1597 struct list_head *head)
1598 {
1599 struct iommu_resv_region *region;
1600 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1601
1602 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1603 prot, IOMMU_RESV_SW_MSI);
1604 if (!region)
1605 return;
1606
1607 list_add_tail(&region->list, head);
1608
1609 iommu_dma_get_resv_regions(dev, head);
1610 }
1611
1612 static int arm_smmu_def_domain_type(struct device *dev)
1613 {
1614 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1615 const struct arm_smmu_impl *impl = cfg->smmu->impl;
1616
1617 if (impl && impl->def_domain_type)
1618 return impl->def_domain_type(dev);
1619
1620 return 0;
1621 }
1622
1623 static struct iommu_ops arm_smmu_ops = {
1624 .capable = arm_smmu_capable,
1625 .domain_alloc = arm_smmu_domain_alloc,
1626 .domain_free = arm_smmu_domain_free,
1627 .attach_dev = arm_smmu_attach_dev,
1628 .map = arm_smmu_map,
1629 .unmap = arm_smmu_unmap,
1630 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
1631 .iotlb_sync = arm_smmu_iotlb_sync,
1632 .iova_to_phys = arm_smmu_iova_to_phys,
1633 .probe_device = arm_smmu_probe_device,
1634 .release_device = arm_smmu_release_device,
1635 .device_group = arm_smmu_device_group,
1636 .domain_get_attr = arm_smmu_domain_get_attr,
1637 .domain_set_attr = arm_smmu_domain_set_attr,
1638 .of_xlate = arm_smmu_of_xlate,
1639 .get_resv_regions = arm_smmu_get_resv_regions,
1640 .put_resv_regions = generic_iommu_put_resv_regions,
1641 .def_domain_type = arm_smmu_def_domain_type,
1642 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1643 };
1644
1645 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1646 {
1647 int i;
1648 u32 reg;
1649
1650 /* clear global FSR */
1651 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1652 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1653
1654 /*
1655 * Reset stream mapping groups: Initial values mark all SMRn as
1656 * invalid and all S2CRn as bypass unless overridden.
1657 */
1658 for (i = 0; i < smmu->num_mapping_groups; ++i)
1659 arm_smmu_write_sme(smmu, i);
1660
1661 /* Make sure all context banks are disabled and clear CB_FSR */
1662 for (i = 0; i < smmu->num_context_banks; ++i) {
1663 arm_smmu_write_context_bank(smmu, i);
1664 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_FSR_FAULT);
1665 }
1666
1667 /* Invalidate the TLB, just in case */
1668 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1669 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1670
1671 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1672
1673 /* Enable fault reporting */
1674 reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE |
1675 ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE);
1676
1677 /* Disable TLB broadcasting. */
1678 reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM);
1679
1680 /* Enable client access, handling unmatched streams as appropriate */
1681 reg &= ~ARM_SMMU_sCR0_CLIENTPD;
1682 if (disable_bypass)
1683 reg |= ARM_SMMU_sCR0_USFCFG;
1684 else
1685 reg &= ~ARM_SMMU_sCR0_USFCFG;
1686
1687 /* Disable forced broadcasting */
1688 reg &= ~ARM_SMMU_sCR0_FB;
1689
1690 /* Don't upgrade barriers */
1691 reg &= ~(ARM_SMMU_sCR0_BSU);
1692
1693 if (smmu->features & ARM_SMMU_FEAT_VMID16)
1694 reg |= ARM_SMMU_sCR0_VMID16EN;
1695
1696 if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1697 reg |= ARM_SMMU_sCR0_EXIDENABLE;
1698
1699 if (smmu->impl && smmu->impl->reset)
1700 smmu->impl->reset(smmu);
1701
1702 /* Push the button */
1703 arm_smmu_tlb_sync_global(smmu);
1704 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1705 }
1706
1707 static int arm_smmu_id_size_to_bits(int size)
1708 {
1709 switch (size) {
1710 case 0:
1711 return 32;
1712 case 1:
1713 return 36;
1714 case 2:
1715 return 40;
1716 case 3:
1717 return 42;
1718 case 4:
1719 return 44;
1720 case 5:
1721 default:
1722 return 48;
1723 }
1724 }
1725
1726 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1727 {
1728 unsigned int size;
1729 u32 id;
1730 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1731 int i;
1732
1733 dev_notice(smmu->dev, "probing hardware configuration...\n");
1734 dev_notice(smmu->dev, "SMMUv%d with:\n",
1735 smmu->version == ARM_SMMU_V2 ? 2 : 1);
1736
1737 /* ID0 */
1738 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1739
1740 /* Restrict available stages based on module parameter */
1741 if (force_stage == 1)
1742 id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS);
1743 else if (force_stage == 2)
1744 id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS);
1745
1746 if (id & ARM_SMMU_ID0_S1TS) {
1747 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1748 dev_notice(smmu->dev, "\tstage 1 translation\n");
1749 }
1750
1751 if (id & ARM_SMMU_ID0_S2TS) {
1752 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1753 dev_notice(smmu->dev, "\tstage 2 translation\n");
1754 }
1755
1756 if (id & ARM_SMMU_ID0_NTS) {
1757 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1758 dev_notice(smmu->dev, "\tnested translation\n");
1759 }
1760
1761 if (!(smmu->features &
1762 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1763 dev_err(smmu->dev, "\tno translation support!\n");
1764 return -ENODEV;
1765 }
1766
1767 if ((id & ARM_SMMU_ID0_S1TS) &&
1768 ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) {
1769 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1770 dev_notice(smmu->dev, "\taddress translation ops\n");
1771 }
1772
1773 /*
1774 * In order for DMA API calls to work properly, we must defer to what
1775 * the FW says about coherency, regardless of what the hardware claims.
1776 * Fortunately, this also opens up a workaround for systems where the
1777 * ID register value has ended up configured incorrectly.
1778 */
1779 cttw_reg = !!(id & ARM_SMMU_ID0_CTTW);
1780 if (cttw_fw || cttw_reg)
1781 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1782 cttw_fw ? "" : "non-");
1783 if (cttw_fw != cttw_reg)
1784 dev_notice(smmu->dev,
1785 "\t(IDR0.CTTW overridden by FW configuration)\n");
1786
1787 /* Max. number of entries we have for stream matching/indexing */
1788 if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {
1789 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1790 size = 1 << 16;
1791 } else {
1792 size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id);
1793 }
1794 smmu->streamid_mask = size - 1;
1795 if (id & ARM_SMMU_ID0_SMS) {
1796 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1797 size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id);
1798 if (size == 0) {
1799 dev_err(smmu->dev,
1800 "stream-matching supported, but no SMRs present!\n");
1801 return -ENODEV;
1802 }
1803
1804 /* Zero-initialised to mark as invalid */
1805 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1806 GFP_KERNEL);
1807 if (!smmu->smrs)
1808 return -ENOMEM;
1809
1810 dev_notice(smmu->dev,
1811 "\tstream matching with %u register groups", size);
1812 }
1813 /* s2cr->type == 0 means translation, so initialise explicitly */
1814 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1815 GFP_KERNEL);
1816 if (!smmu->s2crs)
1817 return -ENOMEM;
1818 for (i = 0; i < size; i++)
1819 smmu->s2crs[i] = s2cr_init_val;
1820
1821 smmu->num_mapping_groups = size;
1822 mutex_init(&smmu->stream_map_mutex);
1823 spin_lock_init(&smmu->global_sync_lock);
1824
1825 if (smmu->version < ARM_SMMU_V2 ||
1826 !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) {
1827 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1828 if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S))
1829 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1830 }
1831
1832 /* ID1 */
1833 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1834 smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;
1835
1836 /* Check for size mismatch of SMMU address space from mapped region */
1837 size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1);
1838 if (smmu->numpage != 2 * size << smmu->pgshift)
1839 dev_warn(smmu->dev,
1840 "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1841 2 * size << smmu->pgshift, smmu->numpage);
1842 /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1843 smmu->numpage = size;
1844
1845 smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);
1846 smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);
1847 if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1848 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1849 return -ENODEV;
1850 }
1851 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1852 smmu->num_context_banks, smmu->num_s2_context_banks);
1853 smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1854 sizeof(*smmu->cbs), GFP_KERNEL);
1855 if (!smmu->cbs)
1856 return -ENOMEM;
1857
1858 /* ID2 */
1859 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1860 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id));
1861 smmu->ipa_size = size;
1862
1863 /* The output mask is also applied for bypass */
1864 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id));
1865 smmu->pa_size = size;
1866
1867 if (id & ARM_SMMU_ID2_VMID16)
1868 smmu->features |= ARM_SMMU_FEAT_VMID16;
1869
1870 /*
1871 * What the page table walker can address actually depends on which
1872 * descriptor format is in use, but since a) we don't know that yet,
1873 * and b) it can vary per context bank, this will have to do...
1874 */
1875 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1876 dev_warn(smmu->dev,
1877 "failed to set DMA mask for table walker\n");
1878
1879 if (smmu->version < ARM_SMMU_V2) {
1880 smmu->va_size = smmu->ipa_size;
1881 if (smmu->version == ARM_SMMU_V1_64K)
1882 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1883 } else {
1884 size = FIELD_GET(ARM_SMMU_ID2_UBS, id);
1885 smmu->va_size = arm_smmu_id_size_to_bits(size);
1886 if (id & ARM_SMMU_ID2_PTFS_4K)
1887 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1888 if (id & ARM_SMMU_ID2_PTFS_16K)
1889 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1890 if (id & ARM_SMMU_ID2_PTFS_64K)
1891 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1892 }
1893
1894 /* Now we've corralled the various formats, what'll it do? */
1895 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1896 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1897 if (smmu->features &
1898 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1899 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1900 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1901 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1902 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1903 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1904
1905 if (arm_smmu_ops.pgsize_bitmap == -1UL)
1906 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1907 else
1908 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1909 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1910 smmu->pgsize_bitmap);
1911
1912
1913 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1914 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1915 smmu->va_size, smmu->ipa_size);
1916
1917 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1918 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1919 smmu->ipa_size, smmu->pa_size);
1920
1921 if (smmu->impl && smmu->impl->cfg_probe)
1922 return smmu->impl->cfg_probe(smmu);
1923
1924 return 0;
1925 }
1926
1927 struct arm_smmu_match_data {
1928 enum arm_smmu_arch_version version;
1929 enum arm_smmu_implementation model;
1930 };
1931
1932 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \
1933 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1934
1935 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1936 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1937 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1938 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1939 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1940 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1941
1942 static const struct of_device_id arm_smmu_of_match[] = {
1943 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1944 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1945 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1946 { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1947 { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1948 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1949 { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1950 { },
1951 };
1952 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1953
1954 #ifdef CONFIG_ACPI
1955 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1956 {
1957 int ret = 0;
1958
1959 switch (model) {
1960 case ACPI_IORT_SMMU_V1:
1961 case ACPI_IORT_SMMU_CORELINK_MMU400:
1962 smmu->version = ARM_SMMU_V1;
1963 smmu->model = GENERIC_SMMU;
1964 break;
1965 case ACPI_IORT_SMMU_CORELINK_MMU401:
1966 smmu->version = ARM_SMMU_V1_64K;
1967 smmu->model = GENERIC_SMMU;
1968 break;
1969 case ACPI_IORT_SMMU_V2:
1970 smmu->version = ARM_SMMU_V2;
1971 smmu->model = GENERIC_SMMU;
1972 break;
1973 case ACPI_IORT_SMMU_CORELINK_MMU500:
1974 smmu->version = ARM_SMMU_V2;
1975 smmu->model = ARM_MMU500;
1976 break;
1977 case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1978 smmu->version = ARM_SMMU_V2;
1979 smmu->model = CAVIUM_SMMUV2;
1980 break;
1981 default:
1982 ret = -ENODEV;
1983 }
1984
1985 return ret;
1986 }
1987
1988 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
1989 struct arm_smmu_device *smmu)
1990 {
1991 struct device *dev = smmu->dev;
1992 struct acpi_iort_node *node =
1993 *(struct acpi_iort_node **)dev_get_platdata(dev);
1994 struct acpi_iort_smmu *iort_smmu;
1995 int ret;
1996
1997 /* Retrieve SMMU1/2 specific data */
1998 iort_smmu = (struct acpi_iort_smmu *)node->node_data;
1999
2000 ret = acpi_smmu_get_data(iort_smmu->model, smmu);
2001 if (ret < 0)
2002 return ret;
2003
2004 /* Ignore the configuration access interrupt */
2005 smmu->num_global_irqs = 1;
2006
2007 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
2008 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2009
2010 return 0;
2011 }
2012 #else
2013 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2014 struct arm_smmu_device *smmu)
2015 {
2016 return -ENODEV;
2017 }
2018 #endif
2019
2020 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
2021 struct arm_smmu_device *smmu)
2022 {
2023 const struct arm_smmu_match_data *data;
2024 struct device *dev = &pdev->dev;
2025 bool legacy_binding;
2026
2027 if (of_property_read_u32(dev->of_node, "#global-interrupts",
2028 &smmu->num_global_irqs)) {
2029 dev_err(dev, "missing #global-interrupts property\n");
2030 return -ENODEV;
2031 }
2032
2033 data = of_device_get_match_data(dev);
2034 smmu->version = data->version;
2035 smmu->model = data->model;
2036
2037 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2038 if (legacy_binding && !using_generic_binding) {
2039 if (!using_legacy_binding) {
2040 pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
2041 IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
2042 }
2043 using_legacy_binding = true;
2044 } else if (!legacy_binding && !using_legacy_binding) {
2045 using_generic_binding = true;
2046 } else {
2047 dev_err(dev, "not probing due to mismatched DT properties\n");
2048 return -ENODEV;
2049 }
2050
2051 if (of_dma_is_coherent(dev->of_node))
2052 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2053
2054 return 0;
2055 }
2056
2057 static int arm_smmu_bus_init(struct iommu_ops *ops)
2058 {
2059 int err;
2060
2061 /* Oh, for a proper bus abstraction */
2062 if (!iommu_present(&platform_bus_type)) {
2063 err = bus_set_iommu(&platform_bus_type, ops);
2064 if (err)
2065 return err;
2066 }
2067 #ifdef CONFIG_ARM_AMBA
2068 if (!iommu_present(&amba_bustype)) {
2069 err = bus_set_iommu(&amba_bustype, ops);
2070 if (err)
2071 goto err_reset_platform_ops;
2072 }
2073 #endif
2074 #ifdef CONFIG_PCI
2075 if (!iommu_present(&pci_bus_type)) {
2076 err = bus_set_iommu(&pci_bus_type, ops);
2077 if (err)
2078 goto err_reset_amba_ops;
2079 }
2080 #endif
2081 #ifdef CONFIG_FSL_MC_BUS
2082 if (!iommu_present(&fsl_mc_bus_type)) {
2083 err = bus_set_iommu(&fsl_mc_bus_type, ops);
2084 if (err)
2085 goto err_reset_pci_ops;
2086 }
2087 #endif
2088 return 0;
2089
2090 err_reset_pci_ops: __maybe_unused;
2091 #ifdef CONFIG_PCI
2092 bus_set_iommu(&pci_bus_type, NULL);
2093 #endif
2094 err_reset_amba_ops: __maybe_unused;
2095 #ifdef CONFIG_ARM_AMBA
2096 bus_set_iommu(&amba_bustype, NULL);
2097 #endif
2098 err_reset_platform_ops: __maybe_unused;
2099 bus_set_iommu(&platform_bus_type, NULL);
2100 return err;
2101 }
2102
2103 static int arm_smmu_device_probe(struct platform_device *pdev)
2104 {
2105 struct resource *res;
2106 resource_size_t ioaddr;
2107 struct arm_smmu_device *smmu;
2108 struct device *dev = &pdev->dev;
2109 int num_irqs, i, err;
2110
2111 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2112 if (!smmu) {
2113 dev_err(dev, "failed to allocate arm_smmu_device\n");
2114 return -ENOMEM;
2115 }
2116 smmu->dev = dev;
2117
2118 if (dev->of_node)
2119 err = arm_smmu_device_dt_probe(pdev, smmu);
2120 else
2121 err = arm_smmu_device_acpi_probe(pdev, smmu);
2122
2123 if (err)
2124 return err;
2125
2126 smmu = arm_smmu_impl_init(smmu);
2127 if (IS_ERR(smmu))
2128 return PTR_ERR(smmu);
2129
2130 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2131 ioaddr = res->start;
2132 smmu->base = devm_ioremap_resource(dev, res);
2133 if (IS_ERR(smmu->base))
2134 return PTR_ERR(smmu->base);
2135 /*
2136 * The resource size should effectively match the value of SMMU_TOP;
2137 * stash that temporarily until we know PAGESIZE to validate it with.
2138 */
2139 smmu->numpage = resource_size(res);
2140
2141 num_irqs = 0;
2142 while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2143 num_irqs++;
2144 if (num_irqs > smmu->num_global_irqs)
2145 smmu->num_context_irqs++;
2146 }
2147
2148 if (!smmu->num_context_irqs) {
2149 dev_err(dev, "found %d interrupts but expected at least %d\n",
2150 num_irqs, smmu->num_global_irqs + 1);
2151 return -ENODEV;
2152 }
2153
2154 smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs),
2155 GFP_KERNEL);
2156 if (!smmu->irqs) {
2157 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2158 return -ENOMEM;
2159 }
2160
2161 for (i = 0; i < num_irqs; ++i) {
2162 int irq = platform_get_irq(pdev, i);
2163
2164 if (irq < 0)
2165 return -ENODEV;
2166 smmu->irqs[i] = irq;
2167 }
2168
2169 err = devm_clk_bulk_get_all(dev, &smmu->clks);
2170 if (err < 0) {
2171 dev_err(dev, "failed to get clocks %d\n", err);
2172 return err;
2173 }
2174 smmu->num_clks = err;
2175
2176 err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2177 if (err)
2178 return err;
2179
2180 err = arm_smmu_device_cfg_probe(smmu);
2181 if (err)
2182 return err;
2183
2184 if (smmu->version == ARM_SMMU_V2) {
2185 if (smmu->num_context_banks > smmu->num_context_irqs) {
2186 dev_err(dev,
2187 "found only %d context irq(s) but %d required\n",
2188 smmu->num_context_irqs, smmu->num_context_banks);
2189 return -ENODEV;
2190 }
2191
2192 /* Ignore superfluous interrupts */
2193 smmu->num_context_irqs = smmu->num_context_banks;
2194 }
2195
2196 for (i = 0; i < smmu->num_global_irqs; ++i) {
2197 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2198 arm_smmu_global_fault,
2199 IRQF_SHARED,
2200 "arm-smmu global fault",
2201 smmu);
2202 if (err) {
2203 dev_err(dev, "failed to request global IRQ %d (%u)\n",
2204 i, smmu->irqs[i]);
2205 return err;
2206 }
2207 }
2208
2209 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2210 "smmu.%pa", &ioaddr);
2211 if (err) {
2212 dev_err(dev, "Failed to register iommu in sysfs\n");
2213 return err;
2214 }
2215
2216 iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
2217 iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
2218
2219 err = iommu_device_register(&smmu->iommu);
2220 if (err) {
2221 dev_err(dev, "Failed to register iommu\n");
2222 return err;
2223 }
2224
2225 platform_set_drvdata(pdev, smmu);
2226 arm_smmu_device_reset(smmu);
2227 arm_smmu_test_smr_masks(smmu);
2228
2229 /*
2230 * We want to avoid touching dev->power.lock in fastpaths unless
2231 * it's really going to do something useful - pm_runtime_enabled()
2232 * can serve as an ideal proxy for that decision. So, conditionally
2233 * enable pm_runtime.
2234 */
2235 if (dev->pm_domain) {
2236 pm_runtime_set_active(dev);
2237 pm_runtime_enable(dev);
2238 }
2239
2240 /*
2241 * For ACPI and generic DT bindings, an SMMU will be probed before
2242 * any device which might need it, so we want the bus ops in place
2243 * ready to handle default domain setup as soon as any SMMU exists.
2244 */
2245 if (!using_legacy_binding)
2246 return arm_smmu_bus_init(&arm_smmu_ops);
2247
2248 return 0;
2249 }
2250
2251 static int arm_smmu_device_remove(struct platform_device *pdev)
2252 {
2253 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2254
2255 if (!smmu)
2256 return -ENODEV;
2257
2258 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2259 dev_notice(&pdev->dev, "disabling translation\n");
2260
2261 arm_smmu_bus_init(NULL);
2262 iommu_device_unregister(&smmu->iommu);
2263 iommu_device_sysfs_remove(&smmu->iommu);
2264
2265 arm_smmu_rpm_get(smmu);
2266 /* Turn the thing off */
2267 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);
2268 arm_smmu_rpm_put(smmu);
2269
2270 if (pm_runtime_enabled(smmu->dev))
2271 pm_runtime_force_suspend(smmu->dev);
2272 else
2273 clk_bulk_disable(smmu->num_clks, smmu->clks);
2274
2275 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2276 return 0;
2277 }
2278
2279 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2280 {
2281 arm_smmu_device_remove(pdev);
2282 }
2283
2284 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2285 {
2286 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2287 int ret;
2288
2289 ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2290 if (ret)
2291 return ret;
2292
2293 arm_smmu_device_reset(smmu);
2294
2295 return 0;
2296 }
2297
2298 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2299 {
2300 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2301
2302 clk_bulk_disable(smmu->num_clks, smmu->clks);
2303
2304 return 0;
2305 }
2306
2307 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2308 {
2309 if (pm_runtime_suspended(dev))
2310 return 0;
2311
2312 return arm_smmu_runtime_resume(dev);
2313 }
2314
2315 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2316 {
2317 if (pm_runtime_suspended(dev))
2318 return 0;
2319
2320 return arm_smmu_runtime_suspend(dev);
2321 }
2322
2323 static const struct dev_pm_ops arm_smmu_pm_ops = {
2324 SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2325 SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2326 arm_smmu_runtime_resume, NULL)
2327 };
2328
2329 static struct platform_driver arm_smmu_driver = {
2330 .driver = {
2331 .name = "arm-smmu",
2332 .of_match_table = arm_smmu_of_match,
2333 .pm = &arm_smmu_pm_ops,
2334 .suppress_bind_attrs = true,
2335 },
2336 .probe = arm_smmu_device_probe,
2337 .remove = arm_smmu_device_remove,
2338 .shutdown = arm_smmu_device_shutdown,
2339 };
2340 module_platform_driver(arm_smmu_driver);
2341
2342 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2343 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
2344 MODULE_ALIAS("platform:arm-smmu");
2345 MODULE_LICENSE("GPL v2");
Linux with added FPC-III support