?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Revert "tty: hvc: Fix data abort due to race in hvc_open"
[linux/fpc-iii.git] / drivers / iommu / arm-smmu.c
bloba6a5796e9c41f2ca371c9b2a2b38093234fbe59e
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * IOMMU API for ARM architected SMMU implementations.
4 *
5 * Copyright (C) 2013 ARM Limited
6 *
7 * Author: Will Deacon <will.deacon@arm.com>
8 *
9 * This driver currently supports:
10 * - SMMUv1 and v2 implementations
11 * - Stream-matching and stream-indexing
12 * - v7/v8 long-descriptor format
13 * - Non-secure access to the SMMU
14 * - Context fault reporting
15 * - Extended Stream ID (16 bit)
16 */
17
18 #define pr_fmt(fmt) "arm-smmu: " fmt
19
20 #include <linux/acpi.h>
21 #include <linux/acpi_iort.h>
22 #include <linux/bitfield.h>
23 #include <linux/delay.h>
24 #include <linux/dma-iommu.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/err.h>
27 #include <linux/interrupt.h>
28 #include <linux/io.h>
29 #include <linux/iopoll.h>
30 #include <linux/module.h>
31 #include <linux/of.h>
32 #include <linux/of_address.h>
33 #include <linux/of_device.h>
34 #include <linux/of_iommu.h>
35 #include <linux/pci.h>
36 #include <linux/platform_device.h>
37 #include <linux/pm_runtime.h>
38 #include <linux/ratelimit.h>
39 #include <linux/slab.h>
40
41 #include <linux/amba/bus.h>
42 #include <linux/fsl/mc.h>
43
44 #include "arm-smmu.h"
45
46 /*
47 * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
48 * global register space are still, in fact, using a hypervisor to mediate it
49 * by trapping and emulating register accesses. Sadly, some deployed versions
50 * of said trapping code have bugs wherein they go horribly wrong for stores
51 * using r31 (i.e. XZR/WZR) as the source register.
52 */
53 #define QCOM_DUMMY_VAL -1
54
55 #define TLB_LOOP_TIMEOUT 1000000 /* 1s! */
56 #define TLB_SPIN_COUNT 10
57
58 #define MSI_IOVA_BASE 0x8000000
59 #define MSI_IOVA_LENGTH 0x100000
60
61 static int force_stage;
62 module_param(force_stage, int, S_IRUGO);
63 MODULE_PARM_DESC(force_stage,
64 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
65 static bool disable_bypass =
66 IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
67 module_param(disable_bypass, bool, S_IRUGO);
68 MODULE_PARM_DESC(disable_bypass,
69 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
70
71 struct arm_smmu_s2cr {
72 struct iommu_group *group;
73 int count;
74 enum arm_smmu_s2cr_type type;
75 enum arm_smmu_s2cr_privcfg privcfg;
76 u8 cbndx;
77 };
78
79 #define s2cr_init_val (struct arm_smmu_s2cr){ \
80 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
81 }
82
83 struct arm_smmu_smr {
84 u16 mask;
85 u16 id;
86 bool valid;
87 };
88
89 struct arm_smmu_cb {
90 u64 ttbr[2];
91 u32 tcr[2];
92 u32 mair[2];
93 struct arm_smmu_cfg *cfg;
94 };
95
96 struct arm_smmu_master_cfg {
97 struct arm_smmu_device *smmu;
98 s16 smendx[];
99 };
100 #define INVALID_SMENDX -1
101 #define cfg_smendx(cfg, fw, i) \
102 (i >= fw->num_ids ? INVALID_SMENDX : cfg->smendx[i])
103 #define for_each_cfg_sme(cfg, fw, i, idx) \
104 for (i = 0; idx = cfg_smendx(cfg, fw, i), i < fw->num_ids; ++i)
105
106 static bool using_legacy_binding, using_generic_binding;
107
108 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
109 {
110 if (pm_runtime_enabled(smmu->dev))
111 return pm_runtime_get_sync(smmu->dev);
112
113 return 0;
114 }
115
116 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
117 {
118 if (pm_runtime_enabled(smmu->dev))
119 pm_runtime_put_autosuspend(smmu->dev);
120 }
121
122 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
123 {
124 return container_of(dom, struct arm_smmu_domain, domain);
125 }
126
127 static struct platform_driver arm_smmu_driver;
128 static struct iommu_ops arm_smmu_ops;
129
130 #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
131 static int arm_smmu_bus_init(struct iommu_ops *ops);
132
133 static struct device_node *dev_get_dev_node(struct device *dev)
134 {
135 if (dev_is_pci(dev)) {
136 struct pci_bus *bus = to_pci_dev(dev)->bus;
137
138 while (!pci_is_root_bus(bus))
139 bus = bus->parent;
140 return of_node_get(bus->bridge->parent->of_node);
141 }
142
143 return of_node_get(dev->of_node);
144 }
145
146 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
147 {
148 *((__be32 *)data) = cpu_to_be32(alias);
149 return 0; /* Continue walking */
150 }
151
152 static int __find_legacy_master_phandle(struct device *dev, void *data)
153 {
154 struct of_phandle_iterator *it = *(void **)data;
155 struct device_node *np = it->node;
156 int err;
157
158 of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
159 "#stream-id-cells", -1)
160 if (it->node == np) {
161 *(void **)data = dev;
162 return 1;
163 }
164 it->node = np;
165 return err == -ENOENT ? 0 : err;
166 }
167
168 static int arm_smmu_register_legacy_master(struct device *dev,
169 struct arm_smmu_device **smmu)
170 {
171 struct device *smmu_dev;
172 struct device_node *np;
173 struct of_phandle_iterator it;
174 void *data = &it;
175 u32 *sids;
176 __be32 pci_sid;
177 int err;
178
179 np = dev_get_dev_node(dev);
180 if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
181 of_node_put(np);
182 return -ENODEV;
183 }
184
185 it.node = np;
186 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
187 __find_legacy_master_phandle);
188 smmu_dev = data;
189 of_node_put(np);
190 if (err == 0)
191 return -ENODEV;
192 if (err < 0)
193 return err;
194
195 if (dev_is_pci(dev)) {
196 /* "mmu-masters" assumes Stream ID == Requester ID */
197 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
198 &pci_sid);
199 it.cur = &pci_sid;
200 it.cur_count = 1;
201 }
202
203 err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
204 &arm_smmu_ops);
205 if (err)
206 return err;
207
208 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
209 if (!sids)
210 return -ENOMEM;
211
212 *smmu = dev_get_drvdata(smmu_dev);
213 of_phandle_iterator_args(&it, sids, it.cur_count);
214 err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
215 kfree(sids);
216 return err;
217 }
218
219 /*
220 * With the legacy DT binding in play, we have no guarantees about
221 * probe order, but then we're also not doing default domains, so we can
222 * delay setting bus ops until we're sure every possible SMMU is ready,
223 * and that way ensure that no add_device() calls get missed.
224 */
225 static int arm_smmu_legacy_bus_init(void)
226 {
227 if (using_legacy_binding)
228 return arm_smmu_bus_init(&arm_smmu_ops);
229 return 0;
230 }
231 device_initcall_sync(arm_smmu_legacy_bus_init);
232 #else
233 static int arm_smmu_register_legacy_master(struct device *dev,
234 struct arm_smmu_device **smmu)
235 {
236 return -ENODEV;
237 }
238 #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
239
240 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
241 {
242 int idx;
243
244 do {
245 idx = find_next_zero_bit(map, end, start);
246 if (idx == end)
247 return -ENOSPC;
248 } while (test_and_set_bit(idx, map));
249
250 return idx;
251 }
252
253 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
254 {
255 clear_bit(idx, map);
256 }
257
258 /* Wait for any pending TLB invalidations to complete */
259 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
260 int sync, int status)
261 {
262 unsigned int spin_cnt, delay;
263 u32 reg;
264
265 if (smmu->impl && unlikely(smmu->impl->tlb_sync))
266 return smmu->impl->tlb_sync(smmu, page, sync, status);
267
268 arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
269 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
270 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
271 reg = arm_smmu_readl(smmu, page, status);
272 if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
273 return;
274 cpu_relax();
275 }
276 udelay(delay);
277 }
278 dev_err_ratelimited(smmu->dev,
279 "TLB sync timed out -- SMMU may be deadlocked\n");
280 }
281
282 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
283 {
284 unsigned long flags;
285
286 spin_lock_irqsave(&smmu->global_sync_lock, flags);
287 __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
288 ARM_SMMU_GR0_sTLBGSTATUS);
289 spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
290 }
291
292 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain)
293 {
294 struct arm_smmu_device *smmu = smmu_domain->smmu;
295 unsigned long flags;
296
297 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
298 __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
299 ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
300 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
301 }
302
303 static void arm_smmu_tlb_inv_context_s1(void *cookie)
304 {
305 struct arm_smmu_domain *smmu_domain = cookie;
306 /*
307 * The TLBI write may be relaxed, so ensure that PTEs cleared by the
308 * current CPU are visible beforehand.
309 */
310 wmb();
311 arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
312 ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
313 arm_smmu_tlb_sync_context(smmu_domain);
314 }
315
316 static void arm_smmu_tlb_inv_context_s2(void *cookie)
317 {
318 struct arm_smmu_domain *smmu_domain = cookie;
319 struct arm_smmu_device *smmu = smmu_domain->smmu;
320
321 /* See above */
322 wmb();
323 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
324 arm_smmu_tlb_sync_global(smmu);
325 }
326
327 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
328 size_t granule, void *cookie, int reg)
329 {
330 struct arm_smmu_domain *smmu_domain = cookie;
331 struct arm_smmu_device *smmu = smmu_domain->smmu;
332 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
333 int idx = cfg->cbndx;
334
335 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
336 wmb();
337
338 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
339 iova = (iova >> 12) << 12;
340 iova |= cfg->asid;
341 do {
342 arm_smmu_cb_write(smmu, idx, reg, iova);
343 iova += granule;
344 } while (size -= granule);
345 } else {
346 iova >>= 12;
347 iova |= (u64)cfg->asid << 48;
348 do {
349 arm_smmu_cb_writeq(smmu, idx, reg, iova);
350 iova += granule >> 12;
351 } while (size -= granule);
352 }
353 }
354
355 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
356 size_t granule, void *cookie, int reg)
357 {
358 struct arm_smmu_domain *smmu_domain = cookie;
359 struct arm_smmu_device *smmu = smmu_domain->smmu;
360 int idx = smmu_domain->cfg.cbndx;
361
362 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
363 wmb();
364
365 iova >>= 12;
366 do {
367 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
368 arm_smmu_cb_writeq(smmu, idx, reg, iova);
369 else
370 arm_smmu_cb_write(smmu, idx, reg, iova);
371 iova += granule >> 12;
372 } while (size -= granule);
373 }
374
375 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size,
376 size_t granule, void *cookie)
377 {
378 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
379 ARM_SMMU_CB_S1_TLBIVA);
380 arm_smmu_tlb_sync_context(cookie);
381 }
382
383 static void arm_smmu_tlb_inv_leaf_s1(unsigned long iova, size_t size,
384 size_t granule, void *cookie)
385 {
386 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
387 ARM_SMMU_CB_S1_TLBIVAL);
388 arm_smmu_tlb_sync_context(cookie);
389 }
390
391 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather,
392 unsigned long iova, size_t granule,
393 void *cookie)
394 {
395 arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie,
396 ARM_SMMU_CB_S1_TLBIVAL);
397 }
398
399 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size,
400 size_t granule, void *cookie)
401 {
402 arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
403 ARM_SMMU_CB_S2_TLBIIPAS2);
404 arm_smmu_tlb_sync_context(cookie);
405 }
406
407 static void arm_smmu_tlb_inv_leaf_s2(unsigned long iova, size_t size,
408 size_t granule, void *cookie)
409 {
410 arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
411 ARM_SMMU_CB_S2_TLBIIPAS2L);
412 arm_smmu_tlb_sync_context(cookie);
413 }
414
415 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather,
416 unsigned long iova, size_t granule,
417 void *cookie)
418 {
419 arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie,
420 ARM_SMMU_CB_S2_TLBIIPAS2L);
421 }
422
423 static void arm_smmu_tlb_inv_any_s2_v1(unsigned long iova, size_t size,
424 size_t granule, void *cookie)
425 {
426 arm_smmu_tlb_inv_context_s2(cookie);
427 }
428 /*
429 * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
430 * almost negligible, but the benefit of getting the first one in as far ahead
431 * of the sync as possible is significant, hence we don't just make this a
432 * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might
433 * think.
434 */
435 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather,
436 unsigned long iova, size_t granule,
437 void *cookie)
438 {
439 struct arm_smmu_domain *smmu_domain = cookie;
440 struct arm_smmu_device *smmu = smmu_domain->smmu;
441
442 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
443 wmb();
444
445 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
446 }
447
448 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = {
449 .tlb_flush_all = arm_smmu_tlb_inv_context_s1,
450 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1,
451 .tlb_flush_leaf = arm_smmu_tlb_inv_leaf_s1,
452 .tlb_add_page = arm_smmu_tlb_add_page_s1,
453 };
454
455 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
456 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
457 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2,
458 .tlb_flush_leaf = arm_smmu_tlb_inv_leaf_s2,
459 .tlb_add_page = arm_smmu_tlb_add_page_s2,
460 };
461
462 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
463 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
464 .tlb_flush_walk = arm_smmu_tlb_inv_any_s2_v1,
465 .tlb_flush_leaf = arm_smmu_tlb_inv_any_s2_v1,
466 .tlb_add_page = arm_smmu_tlb_add_page_s2_v1,
467 };
468
469 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
470 {
471 u32 fsr, fsynr, cbfrsynra;
472 unsigned long iova;
473 struct iommu_domain *domain = dev;
474 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
475 struct arm_smmu_device *smmu = smmu_domain->smmu;
476 int idx = smmu_domain->cfg.cbndx;
477
478 fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
479 if (!(fsr & ARM_SMMU_FSR_FAULT))
480 return IRQ_NONE;
481
482 fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
483 iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
484 cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
485
486 dev_err_ratelimited(smmu->dev,
487 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
488 fsr, iova, fsynr, cbfrsynra, idx);
489
490 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);
491 return IRQ_HANDLED;
492 }
493
494 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
495 {
496 u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
497 struct arm_smmu_device *smmu = dev;
498 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
499 DEFAULT_RATELIMIT_BURST);
500
501 gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
502 gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
503 gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
504 gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
505
506 if (!gfsr)
507 return IRQ_NONE;
508
509 if (__ratelimit(&rs)) {
510 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
511 (gfsr & ARM_SMMU_sGFSR_USF))
512 dev_err(smmu->dev,
513 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
514 (u16)gfsynr1);
515 else
516 dev_err(smmu->dev,
517 "Unexpected global fault, this could be serious\n");
518 dev_err(smmu->dev,
519 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
520 gfsr, gfsynr0, gfsynr1, gfsynr2);
521 }
522
523 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
524 return IRQ_HANDLED;
525 }
526
527 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
528 struct io_pgtable_cfg *pgtbl_cfg)
529 {
530 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
531 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
532 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
533
534 cb->cfg = cfg;
535
536 /* TCR */
537 if (stage1) {
538 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
539 cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
540 } else {
541 cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg);
542 cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg);
543 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
544 cb->tcr[1] |= ARM_SMMU_TCR2_AS;
545 else
546 cb->tcr[0] |= ARM_SMMU_TCR_EAE;
547 }
548 } else {
549 cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg);
550 }
551
552 /* TTBRs */
553 if (stage1) {
554 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
555 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
556 cb->ttbr[1] = 0;
557 } else {
558 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
559 cb->ttbr[0] |= FIELD_PREP(ARM_SMMU_TTBRn_ASID,
560 cfg->asid);
561 cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
562 cfg->asid);
563 }
564 } else {
565 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
566 }
567
568 /* MAIRs (stage-1 only) */
569 if (stage1) {
570 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
571 cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
572 cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
573 } else {
574 cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair;
575 cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32;
576 }
577 }
578 }
579
580 static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
581 {
582 u32 reg;
583 bool stage1;
584 struct arm_smmu_cb *cb = &smmu->cbs[idx];
585 struct arm_smmu_cfg *cfg = cb->cfg;
586
587 /* Unassigned context banks only need disabling */
588 if (!cfg) {
589 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
590 return;
591 }
592
593 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
594
595 /* CBA2R */
596 if (smmu->version > ARM_SMMU_V1) {
597 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
598 reg = ARM_SMMU_CBA2R_VA64;
599 else
600 reg = 0;
601 /* 16-bit VMIDs live in CBA2R */
602 if (smmu->features & ARM_SMMU_FEAT_VMID16)
603 reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid);
604
605 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
606 }
607
608 /* CBAR */
609 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar);
610 if (smmu->version < ARM_SMMU_V2)
611 reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx);
612
613 /*
614 * Use the weakest shareability/memory types, so they are
615 * overridden by the ttbcr/pte.
616 */
617 if (stage1) {
618 reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG,
619 ARM_SMMU_CBAR_S1_BPSHCFG_NSH) |
620 FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR,
621 ARM_SMMU_CBAR_S1_MEMATTR_WB);
622 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
623 /* 8-bit VMIDs live in CBAR */
624 reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid);
625 }
626 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
627
628 /*
629 * TCR
630 * We must write this before the TTBRs, since it determines the
631 * access behaviour of some fields (in particular, ASID[15:8]).
632 */
633 if (stage1 && smmu->version > ARM_SMMU_V1)
634 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
635 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
636
637 /* TTBRs */
638 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
639 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
640 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
641 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
642 } else {
643 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
644 if (stage1)
645 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
646 cb->ttbr[1]);
647 }
648
649 /* MAIRs (stage-1 only) */
650 if (stage1) {
651 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
652 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
653 }
654
655 /* SCTLR */
656 reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE |
657 ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M;
658 if (stage1)
659 reg |= ARM_SMMU_SCTLR_S1_ASIDPNE;
660 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
661 reg |= ARM_SMMU_SCTLR_E;
662
663 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
664 }
665
666 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
667 struct arm_smmu_device *smmu)
668 {
669 int irq, start, ret = 0;
670 unsigned long ias, oas;
671 struct io_pgtable_ops *pgtbl_ops;
672 struct io_pgtable_cfg pgtbl_cfg;
673 enum io_pgtable_fmt fmt;
674 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
675 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
676
677 mutex_lock(&smmu_domain->init_mutex);
678 if (smmu_domain->smmu)
679 goto out_unlock;
680
681 if (domain->type == IOMMU_DOMAIN_IDENTITY) {
682 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
683 smmu_domain->smmu = smmu;
684 goto out_unlock;
685 }
686
687 /*
688 * Mapping the requested stage onto what we support is surprisingly
689 * complicated, mainly because the spec allows S1+S2 SMMUs without
690 * support for nested translation. That means we end up with the
691 * following table:
692 *
693 * Requested Supported Actual
694 * S1 N S1
695 * S1 S1+S2 S1
696 * S1 S2 S2
697 * S1 S1 S1
698 * N N N
699 * N S1+S2 S2
700 * N S2 S2
701 * N S1 S1
702 *
703 * Note that you can't actually request stage-2 mappings.
704 */
705 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
706 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
707 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
708 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
709
710 /*
711 * Choosing a suitable context format is even more fiddly. Until we
712 * grow some way for the caller to express a preference, and/or move
713 * the decision into the io-pgtable code where it arguably belongs,
714 * just aim for the closest thing to the rest of the system, and hope
715 * that the hardware isn't esoteric enough that we can't assume AArch64
716 * support to be a superset of AArch32 support...
717 */
718 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
719 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
720 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
721 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
722 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
723 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
724 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
725 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
726 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
727 ARM_SMMU_FEAT_FMT_AARCH64_16K |
728 ARM_SMMU_FEAT_FMT_AARCH64_4K)))
729 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
730
731 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
732 ret = -EINVAL;
733 goto out_unlock;
734 }
735
736 switch (smmu_domain->stage) {
737 case ARM_SMMU_DOMAIN_S1:
738 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
739 start = smmu->num_s2_context_banks;
740 ias = smmu->va_size;
741 oas = smmu->ipa_size;
742 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
743 fmt = ARM_64_LPAE_S1;
744 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
745 fmt = ARM_32_LPAE_S1;
746 ias = min(ias, 32UL);
747 oas = min(oas, 40UL);
748 } else {
749 fmt = ARM_V7S;
750 ias = min(ias, 32UL);
751 oas = min(oas, 32UL);
752 }
753 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
754 break;
755 case ARM_SMMU_DOMAIN_NESTED:
756 /*
757 * We will likely want to change this if/when KVM gets
758 * involved.
759 */
760 case ARM_SMMU_DOMAIN_S2:
761 cfg->cbar = CBAR_TYPE_S2_TRANS;
762 start = 0;
763 ias = smmu->ipa_size;
764 oas = smmu->pa_size;
765 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
766 fmt = ARM_64_LPAE_S2;
767 } else {
768 fmt = ARM_32_LPAE_S2;
769 ias = min(ias, 40UL);
770 oas = min(oas, 40UL);
771 }
772 if (smmu->version == ARM_SMMU_V2)
773 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
774 else
775 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
776 break;
777 default:
778 ret = -EINVAL;
779 goto out_unlock;
780 }
781 ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
782 smmu->num_context_banks);
783 if (ret < 0)
784 goto out_unlock;
785
786 cfg->cbndx = ret;
787 if (smmu->version < ARM_SMMU_V2) {
788 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
789 cfg->irptndx %= smmu->num_context_irqs;
790 } else {
791 cfg->irptndx = cfg->cbndx;
792 }
793
794 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
795 cfg->vmid = cfg->cbndx + 1;
796 else
797 cfg->asid = cfg->cbndx;
798
799 smmu_domain->smmu = smmu;
800 if (smmu->impl && smmu->impl->init_context) {
801 ret = smmu->impl->init_context(smmu_domain);
802 if (ret)
803 goto out_unlock;
804 }
805
806 pgtbl_cfg = (struct io_pgtable_cfg) {
807 .pgsize_bitmap = smmu->pgsize_bitmap,
808 .ias = ias,
809 .oas = oas,
810 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
811 .tlb = smmu_domain->flush_ops,
812 .iommu_dev = smmu->dev,
813 };
814
815 if (smmu_domain->non_strict)
816 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
817
818 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
819 if (!pgtbl_ops) {
820 ret = -ENOMEM;
821 goto out_clear_smmu;
822 }
823
824 /* Update the domain's page sizes to reflect the page table format */
825 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
826 domain->geometry.aperture_end = (1UL << ias) - 1;
827 domain->geometry.force_aperture = true;
828
829 /* Initialise the context bank with our page table cfg */
830 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
831 arm_smmu_write_context_bank(smmu, cfg->cbndx);
832
833 /*
834 * Request context fault interrupt. Do this last to avoid the
835 * handler seeing a half-initialised domain state.
836 */
837 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
838 ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
839 IRQF_SHARED, "arm-smmu-context-fault", domain);
840 if (ret < 0) {
841 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
842 cfg->irptndx, irq);
843 cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
844 }
845
846 mutex_unlock(&smmu_domain->init_mutex);
847
848 /* Publish page table ops for map/unmap */
849 smmu_domain->pgtbl_ops = pgtbl_ops;
850 return 0;
851
852 out_clear_smmu:
853 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
854 smmu_domain->smmu = NULL;
855 out_unlock:
856 mutex_unlock(&smmu_domain->init_mutex);
857 return ret;
858 }
859
860 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
861 {
862 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
863 struct arm_smmu_device *smmu = smmu_domain->smmu;
864 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
865 int ret, irq;
866
867 if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
868 return;
869
870 ret = arm_smmu_rpm_get(smmu);
871 if (ret < 0)
872 return;
873
874 /*
875 * Disable the context bank and free the page tables before freeing
876 * it.
877 */
878 smmu->cbs[cfg->cbndx].cfg = NULL;
879 arm_smmu_write_context_bank(smmu, cfg->cbndx);
880
881 if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
882 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
883 devm_free_irq(smmu->dev, irq, domain);
884 }
885
886 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
887 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
888
889 arm_smmu_rpm_put(smmu);
890 }
891
892 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
893 {
894 struct arm_smmu_domain *smmu_domain;
895
896 if (type != IOMMU_DOMAIN_UNMANAGED &&
897 type != IOMMU_DOMAIN_DMA &&
898 type != IOMMU_DOMAIN_IDENTITY)
899 return NULL;
900 /*
901 * Allocate the domain and initialise some of its data structures.
902 * We can't really do anything meaningful until we've added a
903 * master.
904 */
905 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
906 if (!smmu_domain)
907 return NULL;
908
909 if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
910 iommu_get_dma_cookie(&smmu_domain->domain))) {
911 kfree(smmu_domain);
912 return NULL;
913 }
914
915 mutex_init(&smmu_domain->init_mutex);
916 spin_lock_init(&smmu_domain->cb_lock);
917
918 return &smmu_domain->domain;
919 }
920
921 static void arm_smmu_domain_free(struct iommu_domain *domain)
922 {
923 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
924
925 /*
926 * Free the domain resources. We assume that all devices have
927 * already been detached.
928 */
929 iommu_put_dma_cookie(domain);
930 arm_smmu_destroy_domain_context(domain);
931 kfree(smmu_domain);
932 }
933
934 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
935 {
936 struct arm_smmu_smr *smr = smmu->smrs + idx;
937 u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) |
938 FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask);
939
940 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
941 reg |= ARM_SMMU_SMR_VALID;
942 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
943 }
944
945 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
946 {
947 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
948 u32 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) |
949 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) |
950 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
951
952 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
953 smmu->smrs[idx].valid)
954 reg |= ARM_SMMU_S2CR_EXIDVALID;
955 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
956 }
957
958 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
959 {
960 arm_smmu_write_s2cr(smmu, idx);
961 if (smmu->smrs)
962 arm_smmu_write_smr(smmu, idx);
963 }
964
965 /*
966 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
967 * should be called after sCR0 is written.
968 */
969 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
970 {
971 u32 smr;
972 int i;
973
974 if (!smmu->smrs)
975 return;
976 /*
977 * If we've had to accommodate firmware memory regions, we may
978 * have live SMRs by now; tread carefully...
979 *
980 * Somewhat perversely, not having a free SMR for this test implies we
981 * can get away without it anyway, as we'll only be able to 'allocate'
982 * these SMRs for the ID/mask values we're already trusting to be OK.
983 */
984 for (i = 0; i < smmu->num_mapping_groups; i++)
985 if (!smmu->smrs[i].valid)
986 goto smr_ok;
987 return;
988 smr_ok:
989 /*
990 * SMR.ID bits may not be preserved if the corresponding MASK
991 * bits are set, so check each one separately. We can reject
992 * masters later if they try to claim IDs outside these masks.
993 */
994 smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);
995 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
996 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
997 smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);
998
999 smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);
1000 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
1001 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
1002 smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
1003 }
1004
1005 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1006 {
1007 struct arm_smmu_smr *smrs = smmu->smrs;
1008 int i, free_idx = -ENOSPC;
1009
1010 /* Stream indexing is blissfully easy */
1011 if (!smrs)
1012 return id;
1013
1014 /* Validating SMRs is... less so */
1015 for (i = 0; i < smmu->num_mapping_groups; ++i) {
1016 if (!smrs[i].valid) {
1017 /*
1018 * Note the first free entry we come across, which
1019 * we'll claim in the end if nothing else matches.
1020 */
1021 if (free_idx < 0)
1022 free_idx = i;
1023 continue;
1024 }
1025 /*
1026 * If the new entry is _entirely_ matched by an existing entry,
1027 * then reuse that, with the guarantee that there also cannot
1028 * be any subsequent conflicting entries. In normal use we'd
1029 * expect simply identical entries for this case, but there's
1030 * no harm in accommodating the generalisation.
1031 */
1032 if ((mask & smrs[i].mask) == mask &&
1033 !((id ^ smrs[i].id) & ~smrs[i].mask))
1034 return i;
1035 /*
1036 * If the new entry has any other overlap with an existing one,
1037 * though, then there always exists at least one stream ID
1038 * which would cause a conflict, and we can't allow that risk.
1039 */
1040 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1041 return -EINVAL;
1042 }
1043
1044 return free_idx;
1045 }
1046
1047 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1048 {
1049 if (--smmu->s2crs[idx].count)
1050 return false;
1051
1052 smmu->s2crs[idx] = s2cr_init_val;
1053 if (smmu->smrs)
1054 smmu->smrs[idx].valid = false;
1055
1056 return true;
1057 }
1058
1059 static int arm_smmu_master_alloc_smes(struct device *dev)
1060 {
1061 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1062 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1063 struct arm_smmu_device *smmu = cfg->smmu;
1064 struct arm_smmu_smr *smrs = smmu->smrs;
1065 struct iommu_group *group;
1066 int i, idx, ret;
1067
1068 mutex_lock(&smmu->stream_map_mutex);
1069 /* Figure out a viable stream map entry allocation */
1070 for_each_cfg_sme(cfg, fwspec, i, idx) {
1071 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1072 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1073
1074 if (idx != INVALID_SMENDX) {
1075 ret = -EEXIST;
1076 goto out_err;
1077 }
1078
1079 ret = arm_smmu_find_sme(smmu, sid, mask);
1080 if (ret < 0)
1081 goto out_err;
1082
1083 idx = ret;
1084 if (smrs && smmu->s2crs[idx].count == 0) {
1085 smrs[idx].id = sid;
1086 smrs[idx].mask = mask;
1087 smrs[idx].valid = true;
1088 }
1089 smmu->s2crs[idx].count++;
1090 cfg->smendx[i] = (s16)idx;
1091 }
1092
1093 group = iommu_group_get_for_dev(dev);
1094 if (IS_ERR(group)) {
1095 ret = PTR_ERR(group);
1096 goto out_err;
1097 }
1098 iommu_group_put(group);
1099
1100 /* It worked! Now, poke the actual hardware */
1101 for_each_cfg_sme(cfg, fwspec, i, idx) {
1102 arm_smmu_write_sme(smmu, idx);
1103 smmu->s2crs[idx].group = group;
1104 }
1105
1106 mutex_unlock(&smmu->stream_map_mutex);
1107 return 0;
1108
1109 out_err:
1110 while (i--) {
1111 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1112 cfg->smendx[i] = INVALID_SMENDX;
1113 }
1114 mutex_unlock(&smmu->stream_map_mutex);
1115 return ret;
1116 }
1117
1118 static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg,
1119 struct iommu_fwspec *fwspec)
1120 {
1121 struct arm_smmu_device *smmu = cfg->smmu;
1122 int i, idx;
1123
1124 mutex_lock(&smmu->stream_map_mutex);
1125 for_each_cfg_sme(cfg, fwspec, i, idx) {
1126 if (arm_smmu_free_sme(smmu, idx))
1127 arm_smmu_write_sme(smmu, idx);
1128 cfg->smendx[i] = INVALID_SMENDX;
1129 }
1130 mutex_unlock(&smmu->stream_map_mutex);
1131 }
1132
1133 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1134 struct arm_smmu_master_cfg *cfg,
1135 struct iommu_fwspec *fwspec)
1136 {
1137 struct arm_smmu_device *smmu = smmu_domain->smmu;
1138 struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1139 u8 cbndx = smmu_domain->cfg.cbndx;
1140 enum arm_smmu_s2cr_type type;
1141 int i, idx;
1142
1143 if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
1144 type = S2CR_TYPE_BYPASS;
1145 else
1146 type = S2CR_TYPE_TRANS;
1147
1148 for_each_cfg_sme(cfg, fwspec, i, idx) {
1149 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1150 continue;
1151
1152 s2cr[idx].type = type;
1153 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1154 s2cr[idx].cbndx = cbndx;
1155 arm_smmu_write_s2cr(smmu, idx);
1156 }
1157 return 0;
1158 }
1159
1160 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1161 {
1162 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1163 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1164 struct arm_smmu_master_cfg *cfg;
1165 struct arm_smmu_device *smmu;
1166 int ret;
1167
1168 if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1169 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1170 return -ENXIO;
1171 }
1172
1173 /*
1174 * FIXME: The arch/arm DMA API code tries to attach devices to its own
1175 * domains between of_xlate() and add_device() - we have no way to cope
1176 * with that, so until ARM gets converted to rely on groups and default
1177 * domains, just say no (but more politely than by dereferencing NULL).
1178 * This should be at least a WARN_ON once that's sorted.
1179 */
1180 cfg = dev_iommu_priv_get(dev);
1181 if (!cfg)
1182 return -ENODEV;
1183
1184 smmu = cfg->smmu;
1185
1186 ret = arm_smmu_rpm_get(smmu);
1187 if (ret < 0)
1188 return ret;
1189
1190 /* Ensure that the domain is finalised */
1191 ret = arm_smmu_init_domain_context(domain, smmu);
1192 if (ret < 0)
1193 goto rpm_put;
1194
1195 /*
1196 * Sanity check the domain. We don't support domains across
1197 * different SMMUs.
1198 */
1199 if (smmu_domain->smmu != smmu) {
1200 dev_err(dev,
1201 "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1202 dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1203 ret = -EINVAL;
1204 goto rpm_put;
1205 }
1206
1207 /* Looks ok, so add the device to the domain */
1208 ret = arm_smmu_domain_add_master(smmu_domain, cfg, fwspec);
1209
1210 /*
1211 * Setup an autosuspend delay to avoid bouncing runpm state.
1212 * Otherwise, if a driver for a suspended consumer device
1213 * unmaps buffers, it will runpm resume/suspend for each one.
1214 *
1215 * For example, when used by a GPU device, when an application
1216 * or game exits, it can trigger unmapping 100s or 1000s of
1217 * buffers. With a runpm cycle for each buffer, that adds up
1218 * to 5-10sec worth of reprogramming the context bank, while
1219 * the system appears to be locked up to the user.
1220 */
1221 pm_runtime_set_autosuspend_delay(smmu->dev, 20);
1222 pm_runtime_use_autosuspend(smmu->dev);
1223
1224 rpm_put:
1225 arm_smmu_rpm_put(smmu);
1226 return ret;
1227 }
1228
1229 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1230 phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
1231 {
1232 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1233 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1234 int ret;
1235
1236 if (!ops)
1237 return -ENODEV;
1238
1239 arm_smmu_rpm_get(smmu);
1240 ret = ops->map(ops, iova, paddr, size, prot);
1241 arm_smmu_rpm_put(smmu);
1242
1243 return ret;
1244 }
1245
1246 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1247 size_t size, struct iommu_iotlb_gather *gather)
1248 {
1249 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1250 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1251 size_t ret;
1252
1253 if (!ops)
1254 return 0;
1255
1256 arm_smmu_rpm_get(smmu);
1257 ret = ops->unmap(ops, iova, size, gather);
1258 arm_smmu_rpm_put(smmu);
1259
1260 return ret;
1261 }
1262
1263 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1264 {
1265 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1266 struct arm_smmu_device *smmu = smmu_domain->smmu;
1267
1268 if (smmu_domain->flush_ops) {
1269 arm_smmu_rpm_get(smmu);
1270 smmu_domain->flush_ops->tlb_flush_all(smmu_domain);
1271 arm_smmu_rpm_put(smmu);
1272 }
1273 }
1274
1275 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1276 struct iommu_iotlb_gather *gather)
1277 {
1278 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1279 struct arm_smmu_device *smmu = smmu_domain->smmu;
1280
1281 if (!smmu)
1282 return;
1283
1284 arm_smmu_rpm_get(smmu);
1285 if (smmu->version == ARM_SMMU_V2 ||
1286 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1287 arm_smmu_tlb_sync_context(smmu_domain);
1288 else
1289 arm_smmu_tlb_sync_global(smmu);
1290 arm_smmu_rpm_put(smmu);
1291 }
1292
1293 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1294 dma_addr_t iova)
1295 {
1296 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1297 struct arm_smmu_device *smmu = smmu_domain->smmu;
1298 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1299 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1300 struct device *dev = smmu->dev;
1301 void __iomem *reg;
1302 u32 tmp;
1303 u64 phys;
1304 unsigned long va, flags;
1305 int ret, idx = cfg->cbndx;
1306
1307 ret = arm_smmu_rpm_get(smmu);
1308 if (ret < 0)
1309 return 0;
1310
1311 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1312 va = iova & ~0xfffUL;
1313 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1314 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1315 else
1316 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1317
1318 reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1319 if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_ATSR_ACTIVE),
1320 5, 50)) {
1321 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1322 dev_err(dev,
1323 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1324 &iova);
1325 return ops->iova_to_phys(ops, iova);
1326 }
1327
1328 phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1329 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1330 if (phys & ARM_SMMU_CB_PAR_F) {
1331 dev_err(dev, "translation fault!\n");
1332 dev_err(dev, "PAR = 0x%llx\n", phys);
1333 return 0;
1334 }
1335
1336 arm_smmu_rpm_put(smmu);
1337
1338 return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1339 }
1340
1341 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1342 dma_addr_t iova)
1343 {
1344 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1345 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1346
1347 if (domain->type == IOMMU_DOMAIN_IDENTITY)
1348 return iova;
1349
1350 if (!ops)
1351 return 0;
1352
1353 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1354 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1355 return arm_smmu_iova_to_phys_hard(domain, iova);
1356
1357 return ops->iova_to_phys(ops, iova);
1358 }
1359
1360 static bool arm_smmu_capable(enum iommu_cap cap)
1361 {
1362 switch (cap) {
1363 case IOMMU_CAP_CACHE_COHERENCY:
1364 /*
1365 * Return true here as the SMMU can always send out coherent
1366 * requests.
1367 */
1368 return true;
1369 case IOMMU_CAP_NOEXEC:
1370 return true;
1371 default:
1372 return false;
1373 }
1374 }
1375
1376 static
1377 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1378 {
1379 struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
1380 fwnode);
1381 put_device(dev);
1382 return dev ? dev_get_drvdata(dev) : NULL;
1383 }
1384
1385 static int arm_smmu_add_device(struct device *dev)
1386 {
1387 struct arm_smmu_device *smmu = NULL;
1388 struct arm_smmu_master_cfg *cfg;
1389 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1390 int i, ret;
1391
1392 if (using_legacy_binding) {
1393 ret = arm_smmu_register_legacy_master(dev, &smmu);
1394
1395 /*
1396 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1397 * will allocate/initialise a new one. Thus we need to update fwspec for
1398 * later use.
1399 */
1400 fwspec = dev_iommu_fwspec_get(dev);
1401 if (ret)
1402 goto out_free;
1403 } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1404 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1405 } else {
1406 return -ENODEV;
1407 }
1408
1409 ret = -EINVAL;
1410 for (i = 0; i < fwspec->num_ids; i++) {
1411 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1412 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1413
1414 if (sid & ~smmu->streamid_mask) {
1415 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1416 sid, smmu->streamid_mask);
1417 goto out_free;
1418 }
1419 if (mask & ~smmu->smr_mask_mask) {
1420 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1421 mask, smmu->smr_mask_mask);
1422 goto out_free;
1423 }
1424 }
1425
1426 ret = -ENOMEM;
1427 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1428 GFP_KERNEL);
1429 if (!cfg)
1430 goto out_free;
1431
1432 cfg->smmu = smmu;
1433 dev_iommu_priv_set(dev, cfg);
1434 while (i--)
1435 cfg->smendx[i] = INVALID_SMENDX;
1436
1437 ret = arm_smmu_rpm_get(smmu);
1438 if (ret < 0)
1439 goto out_cfg_free;
1440
1441 ret = arm_smmu_master_alloc_smes(dev);
1442 arm_smmu_rpm_put(smmu);
1443
1444 if (ret)
1445 goto out_cfg_free;
1446
1447 iommu_device_link(&smmu->iommu, dev);
1448
1449 device_link_add(dev, smmu->dev,
1450 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1451
1452 return 0;
1453
1454 out_cfg_free:
1455 kfree(cfg);
1456 out_free:
1457 iommu_fwspec_free(dev);
1458 return ret;
1459 }
1460
1461 static void arm_smmu_remove_device(struct device *dev)
1462 {
1463 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1464 struct arm_smmu_master_cfg *cfg;
1465 struct arm_smmu_device *smmu;
1466 int ret;
1467
1468 if (!fwspec || fwspec->ops != &arm_smmu_ops)
1469 return;
1470
1471 cfg = dev_iommu_priv_get(dev);
1472 smmu = cfg->smmu;
1473
1474 ret = arm_smmu_rpm_get(smmu);
1475 if (ret < 0)
1476 return;
1477
1478 iommu_device_unlink(&smmu->iommu, dev);
1479 arm_smmu_master_free_smes(cfg, fwspec);
1480
1481 arm_smmu_rpm_put(smmu);
1482
1483 dev_iommu_priv_set(dev, NULL);
1484 iommu_group_remove_device(dev);
1485 kfree(cfg);
1486 iommu_fwspec_free(dev);
1487 }
1488
1489 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1490 {
1491 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1492 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1493 struct arm_smmu_device *smmu = cfg->smmu;
1494 struct iommu_group *group = NULL;
1495 int i, idx;
1496
1497 for_each_cfg_sme(cfg, fwspec, i, idx) {
1498 if (group && smmu->s2crs[idx].group &&
1499 group != smmu->s2crs[idx].group)
1500 return ERR_PTR(-EINVAL);
1501
1502 group = smmu->s2crs[idx].group;
1503 }
1504
1505 if (group)
1506 return iommu_group_ref_get(group);
1507
1508 if (dev_is_pci(dev))
1509 group = pci_device_group(dev);
1510 else if (dev_is_fsl_mc(dev))
1511 group = fsl_mc_device_group(dev);
1512 else
1513 group = generic_device_group(dev);
1514
1515 return group;
1516 }
1517
1518 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1519 enum iommu_attr attr, void *data)
1520 {
1521 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1522
1523 switch(domain->type) {
1524 case IOMMU_DOMAIN_UNMANAGED:
1525 switch (attr) {
1526 case DOMAIN_ATTR_NESTING:
1527 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1528 return 0;
1529 default:
1530 return -ENODEV;
1531 }
1532 break;
1533 case IOMMU_DOMAIN_DMA:
1534 switch (attr) {
1535 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1536 *(int *)data = smmu_domain->non_strict;
1537 return 0;
1538 default:
1539 return -ENODEV;
1540 }
1541 break;
1542 default:
1543 return -EINVAL;
1544 }
1545 }
1546
1547 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1548 enum iommu_attr attr, void *data)
1549 {
1550 int ret = 0;
1551 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1552
1553 mutex_lock(&smmu_domain->init_mutex);
1554
1555 switch(domain->type) {
1556 case IOMMU_DOMAIN_UNMANAGED:
1557 switch (attr) {
1558 case DOMAIN_ATTR_NESTING:
1559 if (smmu_domain->smmu) {
1560 ret = -EPERM;
1561 goto out_unlock;
1562 }
1563
1564 if (*(int *)data)
1565 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1566 else
1567 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1568 break;
1569 default:
1570 ret = -ENODEV;
1571 }
1572 break;
1573 case IOMMU_DOMAIN_DMA:
1574 switch (attr) {
1575 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1576 smmu_domain->non_strict = *(int *)data;
1577 break;
1578 default:
1579 ret = -ENODEV;
1580 }
1581 break;
1582 default:
1583 ret = -EINVAL;
1584 }
1585 out_unlock:
1586 mutex_unlock(&smmu_domain->init_mutex);
1587 return ret;
1588 }
1589
1590 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1591 {
1592 u32 mask, fwid = 0;
1593
1594 if (args->args_count > 0)
1595 fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]);
1596
1597 if (args->args_count > 1)
1598 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]);
1599 else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1600 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask);
1601
1602 return iommu_fwspec_add_ids(dev, &fwid, 1);
1603 }
1604
1605 static void arm_smmu_get_resv_regions(struct device *dev,
1606 struct list_head *head)
1607 {
1608 struct iommu_resv_region *region;
1609 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1610
1611 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1612 prot, IOMMU_RESV_SW_MSI);
1613 if (!region)
1614 return;
1615
1616 list_add_tail(&region->list, head);
1617
1618 iommu_dma_get_resv_regions(dev, head);
1619 }
1620
1621 static struct iommu_ops arm_smmu_ops = {
1622 .capable = arm_smmu_capable,
1623 .domain_alloc = arm_smmu_domain_alloc,
1624 .domain_free = arm_smmu_domain_free,
1625 .attach_dev = arm_smmu_attach_dev,
1626 .map = arm_smmu_map,
1627 .unmap = arm_smmu_unmap,
1628 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
1629 .iotlb_sync = arm_smmu_iotlb_sync,
1630 .iova_to_phys = arm_smmu_iova_to_phys,
1631 .add_device = arm_smmu_add_device,
1632 .remove_device = arm_smmu_remove_device,
1633 .device_group = arm_smmu_device_group,
1634 .domain_get_attr = arm_smmu_domain_get_attr,
1635 .domain_set_attr = arm_smmu_domain_set_attr,
1636 .of_xlate = arm_smmu_of_xlate,
1637 .get_resv_regions = arm_smmu_get_resv_regions,
1638 .put_resv_regions = generic_iommu_put_resv_regions,
1639 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1640 };
1641
1642 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1643 {
1644 int i;
1645 u32 reg;
1646
1647 /* clear global FSR */
1648 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1649 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1650
1651 /*
1652 * Reset stream mapping groups: Initial values mark all SMRn as
1653 * invalid and all S2CRn as bypass unless overridden.
1654 */
1655 for (i = 0; i < smmu->num_mapping_groups; ++i)
1656 arm_smmu_write_sme(smmu, i);
1657
1658 /* Make sure all context banks are disabled and clear CB_FSR */
1659 for (i = 0; i < smmu->num_context_banks; ++i) {
1660 arm_smmu_write_context_bank(smmu, i);
1661 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_FSR_FAULT);
1662 }
1663
1664 /* Invalidate the TLB, just in case */
1665 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1666 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1667
1668 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1669
1670 /* Enable fault reporting */
1671 reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE |
1672 ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE);
1673
1674 /* Disable TLB broadcasting. */
1675 reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM);
1676
1677 /* Enable client access, handling unmatched streams as appropriate */
1678 reg &= ~ARM_SMMU_sCR0_CLIENTPD;
1679 if (disable_bypass)
1680 reg |= ARM_SMMU_sCR0_USFCFG;
1681 else
1682 reg &= ~ARM_SMMU_sCR0_USFCFG;
1683
1684 /* Disable forced broadcasting */
1685 reg &= ~ARM_SMMU_sCR0_FB;
1686
1687 /* Don't upgrade barriers */
1688 reg &= ~(ARM_SMMU_sCR0_BSU);
1689
1690 if (smmu->features & ARM_SMMU_FEAT_VMID16)
1691 reg |= ARM_SMMU_sCR0_VMID16EN;
1692
1693 if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1694 reg |= ARM_SMMU_sCR0_EXIDENABLE;
1695
1696 if (smmu->impl && smmu->impl->reset)
1697 smmu->impl->reset(smmu);
1698
1699 /* Push the button */
1700 arm_smmu_tlb_sync_global(smmu);
1701 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1702 }
1703
1704 static int arm_smmu_id_size_to_bits(int size)
1705 {
1706 switch (size) {
1707 case 0:
1708 return 32;
1709 case 1:
1710 return 36;
1711 case 2:
1712 return 40;
1713 case 3:
1714 return 42;
1715 case 4:
1716 return 44;
1717 case 5:
1718 default:
1719 return 48;
1720 }
1721 }
1722
1723 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1724 {
1725 unsigned int size;
1726 u32 id;
1727 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1728 int i;
1729
1730 dev_notice(smmu->dev, "probing hardware configuration...\n");
1731 dev_notice(smmu->dev, "SMMUv%d with:\n",
1732 smmu->version == ARM_SMMU_V2 ? 2 : 1);
1733
1734 /* ID0 */
1735 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1736
1737 /* Restrict available stages based on module parameter */
1738 if (force_stage == 1)
1739 id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS);
1740 else if (force_stage == 2)
1741 id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS);
1742
1743 if (id & ARM_SMMU_ID0_S1TS) {
1744 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1745 dev_notice(smmu->dev, "\tstage 1 translation\n");
1746 }
1747
1748 if (id & ARM_SMMU_ID0_S2TS) {
1749 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1750 dev_notice(smmu->dev, "\tstage 2 translation\n");
1751 }
1752
1753 if (id & ARM_SMMU_ID0_NTS) {
1754 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1755 dev_notice(smmu->dev, "\tnested translation\n");
1756 }
1757
1758 if (!(smmu->features &
1759 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1760 dev_err(smmu->dev, "\tno translation support!\n");
1761 return -ENODEV;
1762 }
1763
1764 if ((id & ARM_SMMU_ID0_S1TS) &&
1765 ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) {
1766 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1767 dev_notice(smmu->dev, "\taddress translation ops\n");
1768 }
1769
1770 /*
1771 * In order for DMA API calls to work properly, we must defer to what
1772 * the FW says about coherency, regardless of what the hardware claims.
1773 * Fortunately, this also opens up a workaround for systems where the
1774 * ID register value has ended up configured incorrectly.
1775 */
1776 cttw_reg = !!(id & ARM_SMMU_ID0_CTTW);
1777 if (cttw_fw || cttw_reg)
1778 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1779 cttw_fw ? "" : "non-");
1780 if (cttw_fw != cttw_reg)
1781 dev_notice(smmu->dev,
1782 "\t(IDR0.CTTW overridden by FW configuration)\n");
1783
1784 /* Max. number of entries we have for stream matching/indexing */
1785 if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {
1786 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1787 size = 1 << 16;
1788 } else {
1789 size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id);
1790 }
1791 smmu->streamid_mask = size - 1;
1792 if (id & ARM_SMMU_ID0_SMS) {
1793 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1794 size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id);
1795 if (size == 0) {
1796 dev_err(smmu->dev,
1797 "stream-matching supported, but no SMRs present!\n");
1798 return -ENODEV;
1799 }
1800
1801 /* Zero-initialised to mark as invalid */
1802 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1803 GFP_KERNEL);
1804 if (!smmu->smrs)
1805 return -ENOMEM;
1806
1807 dev_notice(smmu->dev,
1808 "\tstream matching with %u register groups", size);
1809 }
1810 /* s2cr->type == 0 means translation, so initialise explicitly */
1811 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1812 GFP_KERNEL);
1813 if (!smmu->s2crs)
1814 return -ENOMEM;
1815 for (i = 0; i < size; i++)
1816 smmu->s2crs[i] = s2cr_init_val;
1817
1818 smmu->num_mapping_groups = size;
1819 mutex_init(&smmu->stream_map_mutex);
1820 spin_lock_init(&smmu->global_sync_lock);
1821
1822 if (smmu->version < ARM_SMMU_V2 ||
1823 !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) {
1824 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1825 if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S))
1826 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1827 }
1828
1829 /* ID1 */
1830 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1831 smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;
1832
1833 /* Check for size mismatch of SMMU address space from mapped region */
1834 size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1);
1835 if (smmu->numpage != 2 * size << smmu->pgshift)
1836 dev_warn(smmu->dev,
1837 "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1838 2 * size << smmu->pgshift, smmu->numpage);
1839 /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1840 smmu->numpage = size;
1841
1842 smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);
1843 smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);
1844 if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1845 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1846 return -ENODEV;
1847 }
1848 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1849 smmu->num_context_banks, smmu->num_s2_context_banks);
1850 smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1851 sizeof(*smmu->cbs), GFP_KERNEL);
1852 if (!smmu->cbs)
1853 return -ENOMEM;
1854
1855 /* ID2 */
1856 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1857 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id));
1858 smmu->ipa_size = size;
1859
1860 /* The output mask is also applied for bypass */
1861 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id));
1862 smmu->pa_size = size;
1863
1864 if (id & ARM_SMMU_ID2_VMID16)
1865 smmu->features |= ARM_SMMU_FEAT_VMID16;
1866
1867 /*
1868 * What the page table walker can address actually depends on which
1869 * descriptor format is in use, but since a) we don't know that yet,
1870 * and b) it can vary per context bank, this will have to do...
1871 */
1872 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1873 dev_warn(smmu->dev,
1874 "failed to set DMA mask for table walker\n");
1875
1876 if (smmu->version < ARM_SMMU_V2) {
1877 smmu->va_size = smmu->ipa_size;
1878 if (smmu->version == ARM_SMMU_V1_64K)
1879 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1880 } else {
1881 size = FIELD_GET(ARM_SMMU_ID2_UBS, id);
1882 smmu->va_size = arm_smmu_id_size_to_bits(size);
1883 if (id & ARM_SMMU_ID2_PTFS_4K)
1884 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1885 if (id & ARM_SMMU_ID2_PTFS_16K)
1886 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1887 if (id & ARM_SMMU_ID2_PTFS_64K)
1888 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1889 }
1890
1891 /* Now we've corralled the various formats, what'll it do? */
1892 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1893 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1894 if (smmu->features &
1895 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1896 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1897 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1898 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1899 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1900 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1901
1902 if (arm_smmu_ops.pgsize_bitmap == -1UL)
1903 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1904 else
1905 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1906 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1907 smmu->pgsize_bitmap);
1908
1909
1910 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1911 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1912 smmu->va_size, smmu->ipa_size);
1913
1914 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1915 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1916 smmu->ipa_size, smmu->pa_size);
1917
1918 if (smmu->impl && smmu->impl->cfg_probe)
1919 return smmu->impl->cfg_probe(smmu);
1920
1921 return 0;
1922 }
1923
1924 struct arm_smmu_match_data {
1925 enum arm_smmu_arch_version version;
1926 enum arm_smmu_implementation model;
1927 };
1928
1929 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \
1930 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1931
1932 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1933 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1934 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1935 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1936 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1937 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1938
1939 static const struct of_device_id arm_smmu_of_match[] = {
1940 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1941 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1942 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1943 { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1944 { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1945 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1946 { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1947 { },
1948 };
1949 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1950
1951 #ifdef CONFIG_ACPI
1952 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1953 {
1954 int ret = 0;
1955
1956 switch (model) {
1957 case ACPI_IORT_SMMU_V1:
1958 case ACPI_IORT_SMMU_CORELINK_MMU400:
1959 smmu->version = ARM_SMMU_V1;
1960 smmu->model = GENERIC_SMMU;
1961 break;
1962 case ACPI_IORT_SMMU_CORELINK_MMU401:
1963 smmu->version = ARM_SMMU_V1_64K;
1964 smmu->model = GENERIC_SMMU;
1965 break;
1966 case ACPI_IORT_SMMU_V2:
1967 smmu->version = ARM_SMMU_V2;
1968 smmu->model = GENERIC_SMMU;
1969 break;
1970 case ACPI_IORT_SMMU_CORELINK_MMU500:
1971 smmu->version = ARM_SMMU_V2;
1972 smmu->model = ARM_MMU500;
1973 break;
1974 case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1975 smmu->version = ARM_SMMU_V2;
1976 smmu->model = CAVIUM_SMMUV2;
1977 break;
1978 default:
1979 ret = -ENODEV;
1980 }
1981
1982 return ret;
1983 }
1984
1985 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
1986 struct arm_smmu_device *smmu)
1987 {
1988 struct device *dev = smmu->dev;
1989 struct acpi_iort_node *node =
1990 *(struct acpi_iort_node **)dev_get_platdata(dev);
1991 struct acpi_iort_smmu *iort_smmu;
1992 int ret;
1993
1994 /* Retrieve SMMU1/2 specific data */
1995 iort_smmu = (struct acpi_iort_smmu *)node->node_data;
1996
1997 ret = acpi_smmu_get_data(iort_smmu->model, smmu);
1998 if (ret < 0)
1999 return ret;
2000
2001 /* Ignore the configuration access interrupt */
2002 smmu->num_global_irqs = 1;
2003
2004 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
2005 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2006
2007 return 0;
2008 }
2009 #else
2010 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2011 struct arm_smmu_device *smmu)
2012 {
2013 return -ENODEV;
2014 }
2015 #endif
2016
2017 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
2018 struct arm_smmu_device *smmu)
2019 {
2020 const struct arm_smmu_match_data *data;
2021 struct device *dev = &pdev->dev;
2022 bool legacy_binding;
2023
2024 if (of_property_read_u32(dev->of_node, "#global-interrupts",
2025 &smmu->num_global_irqs)) {
2026 dev_err(dev, "missing #global-interrupts property\n");
2027 return -ENODEV;
2028 }
2029
2030 data = of_device_get_match_data(dev);
2031 smmu->version = data->version;
2032 smmu->model = data->model;
2033
2034 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2035 if (legacy_binding && !using_generic_binding) {
2036 if (!using_legacy_binding) {
2037 pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
2038 IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
2039 }
2040 using_legacy_binding = true;
2041 } else if (!legacy_binding && !using_legacy_binding) {
2042 using_generic_binding = true;
2043 } else {
2044 dev_err(dev, "not probing due to mismatched DT properties\n");
2045 return -ENODEV;
2046 }
2047
2048 if (of_dma_is_coherent(dev->of_node))
2049 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2050
2051 return 0;
2052 }
2053
2054 static int arm_smmu_bus_init(struct iommu_ops *ops)
2055 {
2056 int err;
2057
2058 /* Oh, for a proper bus abstraction */
2059 if (!iommu_present(&platform_bus_type)) {
2060 err = bus_set_iommu(&platform_bus_type, ops);
2061 if (err)
2062 return err;
2063 }
2064 #ifdef CONFIG_ARM_AMBA
2065 if (!iommu_present(&amba_bustype)) {
2066 err = bus_set_iommu(&amba_bustype, ops);
2067 if (err)
2068 goto err_reset_platform_ops;
2069 }
2070 #endif
2071 #ifdef CONFIG_PCI
2072 if (!iommu_present(&pci_bus_type)) {
2073 err = bus_set_iommu(&pci_bus_type, ops);
2074 if (err)
2075 goto err_reset_amba_ops;
2076 }
2077 #endif
2078 #ifdef CONFIG_FSL_MC_BUS
2079 if (!iommu_present(&fsl_mc_bus_type)) {
2080 err = bus_set_iommu(&fsl_mc_bus_type, ops);
2081 if (err)
2082 goto err_reset_pci_ops;
2083 }
2084 #endif
2085 return 0;
2086
2087 err_reset_pci_ops: __maybe_unused;
2088 #ifdef CONFIG_PCI
2089 bus_set_iommu(&pci_bus_type, NULL);
2090 #endif
2091 err_reset_amba_ops: __maybe_unused;
2092 #ifdef CONFIG_ARM_AMBA
2093 bus_set_iommu(&amba_bustype, NULL);
2094 #endif
2095 err_reset_platform_ops: __maybe_unused;
2096 bus_set_iommu(&platform_bus_type, NULL);
2097 return err;
2098 }
2099
2100 static int arm_smmu_device_probe(struct platform_device *pdev)
2101 {
2102 struct resource *res;
2103 resource_size_t ioaddr;
2104 struct arm_smmu_device *smmu;
2105 struct device *dev = &pdev->dev;
2106 int num_irqs, i, err;
2107
2108 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2109 if (!smmu) {
2110 dev_err(dev, "failed to allocate arm_smmu_device\n");
2111 return -ENOMEM;
2112 }
2113 smmu->dev = dev;
2114
2115 if (dev->of_node)
2116 err = arm_smmu_device_dt_probe(pdev, smmu);
2117 else
2118 err = arm_smmu_device_acpi_probe(pdev, smmu);
2119
2120 if (err)
2121 return err;
2122
2123 smmu = arm_smmu_impl_init(smmu);
2124 if (IS_ERR(smmu))
2125 return PTR_ERR(smmu);
2126
2127 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2128 ioaddr = res->start;
2129 smmu->base = devm_ioremap_resource(dev, res);
2130 if (IS_ERR(smmu->base))
2131 return PTR_ERR(smmu->base);
2132 /*
2133 * The resource size should effectively match the value of SMMU_TOP;
2134 * stash that temporarily until we know PAGESIZE to validate it with.
2135 */
2136 smmu->numpage = resource_size(res);
2137
2138 num_irqs = 0;
2139 while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2140 num_irqs++;
2141 if (num_irqs > smmu->num_global_irqs)
2142 smmu->num_context_irqs++;
2143 }
2144
2145 if (!smmu->num_context_irqs) {
2146 dev_err(dev, "found %d interrupts but expected at least %d\n",
2147 num_irqs, smmu->num_global_irqs + 1);
2148 return -ENODEV;
2149 }
2150
2151 smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs),
2152 GFP_KERNEL);
2153 if (!smmu->irqs) {
2154 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2155 return -ENOMEM;
2156 }
2157
2158 for (i = 0; i < num_irqs; ++i) {
2159 int irq = platform_get_irq(pdev, i);
2160
2161 if (irq < 0)
2162 return -ENODEV;
2163 smmu->irqs[i] = irq;
2164 }
2165
2166 err = devm_clk_bulk_get_all(dev, &smmu->clks);
2167 if (err < 0) {
2168 dev_err(dev, "failed to get clocks %d\n", err);
2169 return err;
2170 }
2171 smmu->num_clks = err;
2172
2173 err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2174 if (err)
2175 return err;
2176
2177 err = arm_smmu_device_cfg_probe(smmu);
2178 if (err)
2179 return err;
2180
2181 if (smmu->version == ARM_SMMU_V2) {
2182 if (smmu->num_context_banks > smmu->num_context_irqs) {
2183 dev_err(dev,
2184 "found only %d context irq(s) but %d required\n",
2185 smmu->num_context_irqs, smmu->num_context_banks);
2186 return -ENODEV;
2187 }
2188
2189 /* Ignore superfluous interrupts */
2190 smmu->num_context_irqs = smmu->num_context_banks;
2191 }
2192
2193 for (i = 0; i < smmu->num_global_irqs; ++i) {
2194 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2195 arm_smmu_global_fault,
2196 IRQF_SHARED,
2197 "arm-smmu global fault",
2198 smmu);
2199 if (err) {
2200 dev_err(dev, "failed to request global IRQ %d (%u)\n",
2201 i, smmu->irqs[i]);
2202 return err;
2203 }
2204 }
2205
2206 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2207 "smmu.%pa", &ioaddr);
2208 if (err) {
2209 dev_err(dev, "Failed to register iommu in sysfs\n");
2210 return err;
2211 }
2212
2213 iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
2214 iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
2215
2216 err = iommu_device_register(&smmu->iommu);
2217 if (err) {
2218 dev_err(dev, "Failed to register iommu\n");
2219 return err;
2220 }
2221
2222 platform_set_drvdata(pdev, smmu);
2223 arm_smmu_device_reset(smmu);
2224 arm_smmu_test_smr_masks(smmu);
2225
2226 /*
2227 * We want to avoid touching dev->power.lock in fastpaths unless
2228 * it's really going to do something useful - pm_runtime_enabled()
2229 * can serve as an ideal proxy for that decision. So, conditionally
2230 * enable pm_runtime.
2231 */
2232 if (dev->pm_domain) {
2233 pm_runtime_set_active(dev);
2234 pm_runtime_enable(dev);
2235 }
2236
2237 /*
2238 * For ACPI and generic DT bindings, an SMMU will be probed before
2239 * any device which might need it, so we want the bus ops in place
2240 * ready to handle default domain setup as soon as any SMMU exists.
2241 */
2242 if (!using_legacy_binding)
2243 return arm_smmu_bus_init(&arm_smmu_ops);
2244
2245 return 0;
2246 }
2247
2248 static int arm_smmu_device_remove(struct platform_device *pdev)
2249 {
2250 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2251
2252 if (!smmu)
2253 return -ENODEV;
2254
2255 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2256 dev_err(&pdev->dev, "removing device with active domains!\n");
2257
2258 arm_smmu_bus_init(NULL);
2259 iommu_device_unregister(&smmu->iommu);
2260 iommu_device_sysfs_remove(&smmu->iommu);
2261
2262 arm_smmu_rpm_get(smmu);
2263 /* Turn the thing off */
2264 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);
2265 arm_smmu_rpm_put(smmu);
2266
2267 if (pm_runtime_enabled(smmu->dev))
2268 pm_runtime_force_suspend(smmu->dev);
2269 else
2270 clk_bulk_disable(smmu->num_clks, smmu->clks);
2271
2272 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2273 return 0;
2274 }
2275
2276 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2277 {
2278 arm_smmu_device_remove(pdev);
2279 }
2280
2281 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2282 {
2283 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2284 int ret;
2285
2286 ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2287 if (ret)
2288 return ret;
2289
2290 arm_smmu_device_reset(smmu);
2291
2292 return 0;
2293 }
2294
2295 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2296 {
2297 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2298
2299 clk_bulk_disable(smmu->num_clks, smmu->clks);
2300
2301 return 0;
2302 }
2303
2304 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2305 {
2306 if (pm_runtime_suspended(dev))
2307 return 0;
2308
2309 return arm_smmu_runtime_resume(dev);
2310 }
2311
2312 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2313 {
2314 if (pm_runtime_suspended(dev))
2315 return 0;
2316
2317 return arm_smmu_runtime_suspend(dev);
2318 }
2319
2320 static const struct dev_pm_ops arm_smmu_pm_ops = {
2321 SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2322 SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2323 arm_smmu_runtime_resume, NULL)
2324 };
2325
2326 static struct platform_driver arm_smmu_driver = {
2327 .driver = {
2328 .name = "arm-smmu",
2329 .of_match_table = arm_smmu_of_match,
2330 .pm = &arm_smmu_pm_ops,
2331 .suppress_bind_attrs = true,
2332 },
2333 .probe = arm_smmu_device_probe,
2334 .remove = arm_smmu_device_remove,
2335 .shutdown = arm_smmu_device_shutdown,
2336 };
2337 module_platform_driver(arm_smmu_driver);
2338
2339 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2340 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
2341 MODULE_ALIAS("platform:arm-smmu");
2342 MODULE_LICENSE("GPL v2");
Linux with added FPC-III support