?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 5.1.15
[linux/fpc-iii.git] / drivers / iommu / arm-smmu.c
blob7e63a15fa724e67701c667f6b60b55f722f1bee4
1 /*
2 * IOMMU API for ARM architected SMMU implementations.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
16 *
17 * Copyright (C) 2013 ARM Limited
18 *
19 * Author: Will Deacon <will.deacon@arm.com>
20 *
21 * This driver currently supports:
22 * - SMMUv1 and v2 implementations
23 * - Stream-matching and stream-indexing
24 * - v7/v8 long-descriptor format
25 * - Non-secure access to the SMMU
26 * - Context fault reporting
27 * - Extended Stream ID (16 bit)
28 */
29
30 #define pr_fmt(fmt) "arm-smmu: " fmt
31
32 #include <linux/acpi.h>
33 #include <linux/acpi_iort.h>
34 #include <linux/atomic.h>
35 #include <linux/delay.h>
36 #include <linux/dma-iommu.h>
37 #include <linux/dma-mapping.h>
38 #include <linux/err.h>
39 #include <linux/interrupt.h>
40 #include <linux/io.h>
41 #include <linux/io-64-nonatomic-hi-lo.h>
42 #include <linux/io-pgtable.h>
43 #include <linux/iommu.h>
44 #include <linux/iopoll.h>
45 #include <linux/init.h>
46 #include <linux/moduleparam.h>
47 #include <linux/of.h>
48 #include <linux/of_address.h>
49 #include <linux/of_device.h>
50 #include <linux/of_iommu.h>
51 #include <linux/pci.h>
52 #include <linux/platform_device.h>
53 #include <linux/pm_runtime.h>
54 #include <linux/slab.h>
55 #include <linux/spinlock.h>
56
57 #include <linux/amba/bus.h>
58 #include <linux/fsl/mc.h>
59
60 #include "arm-smmu-regs.h"
61
62 /*
63 * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
64 * global register space are still, in fact, using a hypervisor to mediate it
65 * by trapping and emulating register accesses. Sadly, some deployed versions
66 * of said trapping code have bugs wherein they go horribly wrong for stores
67 * using r31 (i.e. XZR/WZR) as the source register.
68 */
69 #define QCOM_DUMMY_VAL -1
70
71 #define ARM_MMU500_ACTLR_CPRE (1 << 1)
72
73 #define ARM_MMU500_ACR_CACHE_LOCK (1 << 26)
74 #define ARM_MMU500_ACR_S2CRB_TLBEN (1 << 10)
75 #define ARM_MMU500_ACR_SMTNMB_TLBEN (1 << 8)
76
77 #define TLB_LOOP_TIMEOUT 1000000 /* 1s! */
78 #define TLB_SPIN_COUNT 10
79
80 /* Maximum number of context banks per SMMU */
81 #define ARM_SMMU_MAX_CBS 128
82
83 /* SMMU global address space */
84 #define ARM_SMMU_GR0(smmu) ((smmu)->base)
85 #define ARM_SMMU_GR1(smmu) ((smmu)->base + (1 << (smmu)->pgshift))
86
87 /*
88 * SMMU global address space with conditional offset to access secure
89 * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
90 * nsGFSYNR0: 0x450)
91 */
92 #define ARM_SMMU_GR0_NS(smmu) \
93 ((smmu)->base + \
94 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) \
95 ? 0x400 : 0))
96
97 /*
98 * Some 64-bit registers only make sense to write atomically, but in such
99 * cases all the data relevant to AArch32 formats lies within the lower word,
100 * therefore this actually makes more sense than it might first appear.
101 */
102 #ifdef CONFIG_64BIT
103 #define smmu_write_atomic_lq writeq_relaxed
104 #else
105 #define smmu_write_atomic_lq writel_relaxed
106 #endif
107
108 /* Translation context bank */
109 #define ARM_SMMU_CB(smmu, n) ((smmu)->cb_base + ((n) << (smmu)->pgshift))
110
111 #define MSI_IOVA_BASE 0x8000000
112 #define MSI_IOVA_LENGTH 0x100000
113
114 static int force_stage;
115 /*
116 * not really modular, but the easiest way to keep compat with existing
117 * bootargs behaviour is to continue using module_param() here.
118 */
119 module_param(force_stage, int, S_IRUGO);
120 MODULE_PARM_DESC(force_stage,
121 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
122 static bool disable_bypass;
123 module_param(disable_bypass, bool, S_IRUGO);
124 MODULE_PARM_DESC(disable_bypass,
125 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
126
127 enum arm_smmu_arch_version {
128 ARM_SMMU_V1,
129 ARM_SMMU_V1_64K,
130 ARM_SMMU_V2,
131 };
132
133 enum arm_smmu_implementation {
134 GENERIC_SMMU,
135 ARM_MMU500,
136 CAVIUM_SMMUV2,
137 QCOM_SMMUV2,
138 };
139
140 struct arm_smmu_s2cr {
141 struct iommu_group *group;
142 int count;
143 enum arm_smmu_s2cr_type type;
144 enum arm_smmu_s2cr_privcfg privcfg;
145 u8 cbndx;
146 };
147
148 #define s2cr_init_val (struct arm_smmu_s2cr){ \
149 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
150 }
151
152 struct arm_smmu_smr {
153 u16 mask;
154 u16 id;
155 bool valid;
156 };
157
158 struct arm_smmu_cb {
159 u64 ttbr[2];
160 u32 tcr[2];
161 u32 mair[2];
162 struct arm_smmu_cfg *cfg;
163 };
164
165 struct arm_smmu_master_cfg {
166 struct arm_smmu_device *smmu;
167 s16 smendx[];
168 };
169 #define INVALID_SMENDX -1
170 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
171 #define fwspec_smmu(fw) (__fwspec_cfg(fw)->smmu)
172 #define fwspec_smendx(fw, i) \
173 (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i])
174 #define for_each_cfg_sme(fw, i, idx) \
175 for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)
176
177 struct arm_smmu_device {
178 struct device *dev;
179
180 void __iomem *base;
181 void __iomem *cb_base;
182 unsigned long pgshift;
183
184 #define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0)
185 #define ARM_SMMU_FEAT_STREAM_MATCH (1 << 1)
186 #define ARM_SMMU_FEAT_TRANS_S1 (1 << 2)
187 #define ARM_SMMU_FEAT_TRANS_S2 (1 << 3)
188 #define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4)
189 #define ARM_SMMU_FEAT_TRANS_OPS (1 << 5)
190 #define ARM_SMMU_FEAT_VMID16 (1 << 6)
191 #define ARM_SMMU_FEAT_FMT_AARCH64_4K (1 << 7)
192 #define ARM_SMMU_FEAT_FMT_AARCH64_16K (1 << 8)
193 #define ARM_SMMU_FEAT_FMT_AARCH64_64K (1 << 9)
194 #define ARM_SMMU_FEAT_FMT_AARCH32_L (1 << 10)
195 #define ARM_SMMU_FEAT_FMT_AARCH32_S (1 << 11)
196 #define ARM_SMMU_FEAT_EXIDS (1 << 12)
197 u32 features;
198
199 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
200 u32 options;
201 enum arm_smmu_arch_version version;
202 enum arm_smmu_implementation model;
203
204 u32 num_context_banks;
205 u32 num_s2_context_banks;
206 DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
207 struct arm_smmu_cb *cbs;
208 atomic_t irptndx;
209
210 u32 num_mapping_groups;
211 u16 streamid_mask;
212 u16 smr_mask_mask;
213 struct arm_smmu_smr *smrs;
214 struct arm_smmu_s2cr *s2crs;
215 struct mutex stream_map_mutex;
216
217 unsigned long va_size;
218 unsigned long ipa_size;
219 unsigned long pa_size;
220 unsigned long pgsize_bitmap;
221
222 u32 num_global_irqs;
223 u32 num_context_irqs;
224 unsigned int *irqs;
225 struct clk_bulk_data *clks;
226 int num_clks;
227
228 u32 cavium_id_base; /* Specific to Cavium */
229
230 spinlock_t global_sync_lock;
231
232 /* IOMMU core code handle */
233 struct iommu_device iommu;
234 };
235
236 enum arm_smmu_context_fmt {
237 ARM_SMMU_CTX_FMT_NONE,
238 ARM_SMMU_CTX_FMT_AARCH64,
239 ARM_SMMU_CTX_FMT_AARCH32_L,
240 ARM_SMMU_CTX_FMT_AARCH32_S,
241 };
242
243 struct arm_smmu_cfg {
244 u8 cbndx;
245 u8 irptndx;
246 union {
247 u16 asid;
248 u16 vmid;
249 };
250 u32 cbar;
251 enum arm_smmu_context_fmt fmt;
252 };
253 #define INVALID_IRPTNDX 0xff
254
255 enum arm_smmu_domain_stage {
256 ARM_SMMU_DOMAIN_S1 = 0,
257 ARM_SMMU_DOMAIN_S2,
258 ARM_SMMU_DOMAIN_NESTED,
259 ARM_SMMU_DOMAIN_BYPASS,
260 };
261
262 struct arm_smmu_domain {
263 struct arm_smmu_device *smmu;
264 struct io_pgtable_ops *pgtbl_ops;
265 const struct iommu_gather_ops *tlb_ops;
266 struct arm_smmu_cfg cfg;
267 enum arm_smmu_domain_stage stage;
268 bool non_strict;
269 struct mutex init_mutex; /* Protects smmu pointer */
270 spinlock_t cb_lock; /* Serialises ATS1* ops and TLB syncs */
271 struct iommu_domain domain;
272 };
273
274 struct arm_smmu_option_prop {
275 u32 opt;
276 const char *prop;
277 };
278
279 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
280
281 static bool using_legacy_binding, using_generic_binding;
282
283 static struct arm_smmu_option_prop arm_smmu_options[] = {
284 { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
285 { 0, NULL},
286 };
287
288 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
289 {
290 if (pm_runtime_enabled(smmu->dev))
291 return pm_runtime_get_sync(smmu->dev);
292
293 return 0;
294 }
295
296 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
297 {
298 if (pm_runtime_enabled(smmu->dev))
299 pm_runtime_put(smmu->dev);
300 }
301
302 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
303 {
304 return container_of(dom, struct arm_smmu_domain, domain);
305 }
306
307 static void parse_driver_options(struct arm_smmu_device *smmu)
308 {
309 int i = 0;
310
311 do {
312 if (of_property_read_bool(smmu->dev->of_node,
313 arm_smmu_options[i].prop)) {
314 smmu->options |= arm_smmu_options[i].opt;
315 dev_notice(smmu->dev, "option %s\n",
316 arm_smmu_options[i].prop);
317 }
318 } while (arm_smmu_options[++i].opt);
319 }
320
321 static struct device_node *dev_get_dev_node(struct device *dev)
322 {
323 if (dev_is_pci(dev)) {
324 struct pci_bus *bus = to_pci_dev(dev)->bus;
325
326 while (!pci_is_root_bus(bus))
327 bus = bus->parent;
328 return of_node_get(bus->bridge->parent->of_node);
329 }
330
331 return of_node_get(dev->of_node);
332 }
333
334 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
335 {
336 *((__be32 *)data) = cpu_to_be32(alias);
337 return 0; /* Continue walking */
338 }
339
340 static int __find_legacy_master_phandle(struct device *dev, void *data)
341 {
342 struct of_phandle_iterator *it = *(void **)data;
343 struct device_node *np = it->node;
344 int err;
345
346 of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
347 "#stream-id-cells", 0)
348 if (it->node == np) {
349 *(void **)data = dev;
350 return 1;
351 }
352 it->node = np;
353 return err == -ENOENT ? 0 : err;
354 }
355
356 static struct platform_driver arm_smmu_driver;
357 static struct iommu_ops arm_smmu_ops;
358
359 static int arm_smmu_register_legacy_master(struct device *dev,
360 struct arm_smmu_device **smmu)
361 {
362 struct device *smmu_dev;
363 struct device_node *np;
364 struct of_phandle_iterator it;
365 void *data = &it;
366 u32 *sids;
367 __be32 pci_sid;
368 int err;
369
370 np = dev_get_dev_node(dev);
371 if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
372 of_node_put(np);
373 return -ENODEV;
374 }
375
376 it.node = np;
377 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
378 __find_legacy_master_phandle);
379 smmu_dev = data;
380 of_node_put(np);
381 if (err == 0)
382 return -ENODEV;
383 if (err < 0)
384 return err;
385
386 if (dev_is_pci(dev)) {
387 /* "mmu-masters" assumes Stream ID == Requester ID */
388 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
389 &pci_sid);
390 it.cur = &pci_sid;
391 it.cur_count = 1;
392 }
393
394 err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
395 &arm_smmu_ops);
396 if (err)
397 return err;
398
399 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
400 if (!sids)
401 return -ENOMEM;
402
403 *smmu = dev_get_drvdata(smmu_dev);
404 of_phandle_iterator_args(&it, sids, it.cur_count);
405 err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
406 kfree(sids);
407 return err;
408 }
409
410 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
411 {
412 int idx;
413
414 do {
415 idx = find_next_zero_bit(map, end, start);
416 if (idx == end)
417 return -ENOSPC;
418 } while (test_and_set_bit(idx, map));
419
420 return idx;
421 }
422
423 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
424 {
425 clear_bit(idx, map);
426 }
427
428 /* Wait for any pending TLB invalidations to complete */
429 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu,
430 void __iomem *sync, void __iomem *status)
431 {
432 unsigned int spin_cnt, delay;
433
434 writel_relaxed(QCOM_DUMMY_VAL, sync);
435 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
436 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
437 if (!(readl_relaxed(status) & sTLBGSTATUS_GSACTIVE))
438 return;
439 cpu_relax();
440 }
441 udelay(delay);
442 }
443 dev_err_ratelimited(smmu->dev,
444 "TLB sync timed out -- SMMU may be deadlocked\n");
445 }
446
447 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
448 {
449 void __iomem *base = ARM_SMMU_GR0(smmu);
450 unsigned long flags;
451
452 spin_lock_irqsave(&smmu->global_sync_lock, flags);
453 __arm_smmu_tlb_sync(smmu, base + ARM_SMMU_GR0_sTLBGSYNC,
454 base + ARM_SMMU_GR0_sTLBGSTATUS);
455 spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
456 }
457
458 static void arm_smmu_tlb_sync_context(void *cookie)
459 {
460 struct arm_smmu_domain *smmu_domain = cookie;
461 struct arm_smmu_device *smmu = smmu_domain->smmu;
462 void __iomem *base = ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx);
463 unsigned long flags;
464
465 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
466 __arm_smmu_tlb_sync(smmu, base + ARM_SMMU_CB_TLBSYNC,
467 base + ARM_SMMU_CB_TLBSTATUS);
468 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
469 }
470
471 static void arm_smmu_tlb_sync_vmid(void *cookie)
472 {
473 struct arm_smmu_domain *smmu_domain = cookie;
474
475 arm_smmu_tlb_sync_global(smmu_domain->smmu);
476 }
477
478 static void arm_smmu_tlb_inv_context_s1(void *cookie)
479 {
480 struct arm_smmu_domain *smmu_domain = cookie;
481 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
482 void __iomem *base = ARM_SMMU_CB(smmu_domain->smmu, cfg->cbndx);
483
484 /*
485 * NOTE: this is not a relaxed write; it needs to guarantee that PTEs
486 * cleared by the current CPU are visible to the SMMU before the TLBI.
487 */
488 writel(cfg->asid, base + ARM_SMMU_CB_S1_TLBIASID);
489 arm_smmu_tlb_sync_context(cookie);
490 }
491
492 static void arm_smmu_tlb_inv_context_s2(void *cookie)
493 {
494 struct arm_smmu_domain *smmu_domain = cookie;
495 struct arm_smmu_device *smmu = smmu_domain->smmu;
496 void __iomem *base = ARM_SMMU_GR0(smmu);
497
498 /* NOTE: see above */
499 writel(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID);
500 arm_smmu_tlb_sync_global(smmu);
501 }
502
503 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
504 size_t granule, bool leaf, void *cookie)
505 {
506 struct arm_smmu_domain *smmu_domain = cookie;
507 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
508 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
509 void __iomem *reg = ARM_SMMU_CB(smmu_domain->smmu, cfg->cbndx);
510
511 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
512 wmb();
513
514 if (stage1) {
515 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
516
517 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
518 iova &= ~12UL;
519 iova |= cfg->asid;
520 do {
521 writel_relaxed(iova, reg);
522 iova += granule;
523 } while (size -= granule);
524 } else {
525 iova >>= 12;
526 iova |= (u64)cfg->asid << 48;
527 do {
528 writeq_relaxed(iova, reg);
529 iova += granule >> 12;
530 } while (size -= granule);
531 }
532 } else {
533 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
534 ARM_SMMU_CB_S2_TLBIIPAS2;
535 iova >>= 12;
536 do {
537 smmu_write_atomic_lq(iova, reg);
538 iova += granule >> 12;
539 } while (size -= granule);
540 }
541 }
542
543 /*
544 * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
545 * almost negligible, but the benefit of getting the first one in as far ahead
546 * of the sync as possible is significant, hence we don't just make this a
547 * no-op and set .tlb_sync to arm_smmu_inv_context_s2() as you might think.
548 */
549 static void arm_smmu_tlb_inv_vmid_nosync(unsigned long iova, size_t size,
550 size_t granule, bool leaf, void *cookie)
551 {
552 struct arm_smmu_domain *smmu_domain = cookie;
553 void __iomem *base = ARM_SMMU_GR0(smmu_domain->smmu);
554
555 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
556 wmb();
557
558 writel_relaxed(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID);
559 }
560
561 static const struct iommu_gather_ops arm_smmu_s1_tlb_ops = {
562 .tlb_flush_all = arm_smmu_tlb_inv_context_s1,
563 .tlb_add_flush = arm_smmu_tlb_inv_range_nosync,
564 .tlb_sync = arm_smmu_tlb_sync_context,
565 };
566
567 static const struct iommu_gather_ops arm_smmu_s2_tlb_ops_v2 = {
568 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
569 .tlb_add_flush = arm_smmu_tlb_inv_range_nosync,
570 .tlb_sync = arm_smmu_tlb_sync_context,
571 };
572
573 static const struct iommu_gather_ops arm_smmu_s2_tlb_ops_v1 = {
574 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
575 .tlb_add_flush = arm_smmu_tlb_inv_vmid_nosync,
576 .tlb_sync = arm_smmu_tlb_sync_vmid,
577 };
578
579 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
580 {
581 u32 fsr, fsynr;
582 unsigned long iova;
583 struct iommu_domain *domain = dev;
584 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
585 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
586 struct arm_smmu_device *smmu = smmu_domain->smmu;
587 void __iomem *cb_base;
588
589 cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
590 fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
591
592 if (!(fsr & FSR_FAULT))
593 return IRQ_NONE;
594
595 fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
596 iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
597
598 dev_err_ratelimited(smmu->dev,
599 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d\n",
600 fsr, iova, fsynr, cfg->cbndx);
601
602 writel(fsr, cb_base + ARM_SMMU_CB_FSR);
603 return IRQ_HANDLED;
604 }
605
606 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
607 {
608 u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
609 struct arm_smmu_device *smmu = dev;
610 void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
611
612 gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
613 gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
614 gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
615 gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
616
617 if (!gfsr)
618 return IRQ_NONE;
619
620 dev_err_ratelimited(smmu->dev,
621 "Unexpected global fault, this could be serious\n");
622 dev_err_ratelimited(smmu->dev,
623 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
624 gfsr, gfsynr0, gfsynr1, gfsynr2);
625
626 writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
627 return IRQ_HANDLED;
628 }
629
630 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
631 struct io_pgtable_cfg *pgtbl_cfg)
632 {
633 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
634 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
635 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
636
637 cb->cfg = cfg;
638
639 /* TTBCR */
640 if (stage1) {
641 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
642 cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
643 } else {
644 cb->tcr[0] = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
645 cb->tcr[1] = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
646 cb->tcr[1] |= TTBCR2_SEP_UPSTREAM;
647 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
648 cb->tcr[1] |= TTBCR2_AS;
649 }
650 } else {
651 cb->tcr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
652 }
653
654 /* TTBRs */
655 if (stage1) {
656 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
657 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
658 cb->ttbr[1] = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
659 } else {
660 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
661 cb->ttbr[0] |= (u64)cfg->asid << TTBRn_ASID_SHIFT;
662 cb->ttbr[1] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
663 cb->ttbr[1] |= (u64)cfg->asid << TTBRn_ASID_SHIFT;
664 }
665 } else {
666 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
667 }
668
669 /* MAIRs (stage-1 only) */
670 if (stage1) {
671 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
672 cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
673 cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
674 } else {
675 cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
676 cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
677 }
678 }
679 }
680
681 static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
682 {
683 u32 reg;
684 bool stage1;
685 struct arm_smmu_cb *cb = &smmu->cbs[idx];
686 struct arm_smmu_cfg *cfg = cb->cfg;
687 void __iomem *cb_base, *gr1_base;
688
689 cb_base = ARM_SMMU_CB(smmu, idx);
690
691 /* Unassigned context banks only need disabling */
692 if (!cfg) {
693 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
694 return;
695 }
696
697 gr1_base = ARM_SMMU_GR1(smmu);
698 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
699
700 /* CBA2R */
701 if (smmu->version > ARM_SMMU_V1) {
702 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
703 reg = CBA2R_RW64_64BIT;
704 else
705 reg = CBA2R_RW64_32BIT;
706 /* 16-bit VMIDs live in CBA2R */
707 if (smmu->features & ARM_SMMU_FEAT_VMID16)
708 reg |= cfg->vmid << CBA2R_VMID_SHIFT;
709
710 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(idx));
711 }
712
713 /* CBAR */
714 reg = cfg->cbar;
715 if (smmu->version < ARM_SMMU_V2)
716 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
717
718 /*
719 * Use the weakest shareability/memory types, so they are
720 * overridden by the ttbcr/pte.
721 */
722 if (stage1) {
723 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
724 (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
725 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
726 /* 8-bit VMIDs live in CBAR */
727 reg |= cfg->vmid << CBAR_VMID_SHIFT;
728 }
729 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(idx));
730
731 /*
732 * TTBCR
733 * We must write this before the TTBRs, since it determines the
734 * access behaviour of some fields (in particular, ASID[15:8]).
735 */
736 if (stage1 && smmu->version > ARM_SMMU_V1)
737 writel_relaxed(cb->tcr[1], cb_base + ARM_SMMU_CB_TTBCR2);
738 writel_relaxed(cb->tcr[0], cb_base + ARM_SMMU_CB_TTBCR);
739
740 /* TTBRs */
741 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
742 writel_relaxed(cfg->asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
743 writel_relaxed(cb->ttbr[0], cb_base + ARM_SMMU_CB_TTBR0);
744 writel_relaxed(cb->ttbr[1], cb_base + ARM_SMMU_CB_TTBR1);
745 } else {
746 writeq_relaxed(cb->ttbr[0], cb_base + ARM_SMMU_CB_TTBR0);
747 if (stage1)
748 writeq_relaxed(cb->ttbr[1], cb_base + ARM_SMMU_CB_TTBR1);
749 }
750
751 /* MAIRs (stage-1 only) */
752 if (stage1) {
753 writel_relaxed(cb->mair[0], cb_base + ARM_SMMU_CB_S1_MAIR0);
754 writel_relaxed(cb->mair[1], cb_base + ARM_SMMU_CB_S1_MAIR1);
755 }
756
757 /* SCTLR */
758 reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
759 if (stage1)
760 reg |= SCTLR_S1_ASIDPNE;
761 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
762 reg |= SCTLR_E;
763
764 writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
765 }
766
767 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
768 struct arm_smmu_device *smmu)
769 {
770 int irq, start, ret = 0;
771 unsigned long ias, oas;
772 struct io_pgtable_ops *pgtbl_ops;
773 struct io_pgtable_cfg pgtbl_cfg;
774 enum io_pgtable_fmt fmt;
775 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
776 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
777
778 mutex_lock(&smmu_domain->init_mutex);
779 if (smmu_domain->smmu)
780 goto out_unlock;
781
782 if (domain->type == IOMMU_DOMAIN_IDENTITY) {
783 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
784 smmu_domain->smmu = smmu;
785 goto out_unlock;
786 }
787
788 /*
789 * Mapping the requested stage onto what we support is surprisingly
790 * complicated, mainly because the spec allows S1+S2 SMMUs without
791 * support for nested translation. That means we end up with the
792 * following table:
793 *
794 * Requested Supported Actual
795 * S1 N S1
796 * S1 S1+S2 S1
797 * S1 S2 S2
798 * S1 S1 S1
799 * N N N
800 * N S1+S2 S2
801 * N S2 S2
802 * N S1 S1
803 *
804 * Note that you can't actually request stage-2 mappings.
805 */
806 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
807 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
808 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
809 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
810
811 /*
812 * Choosing a suitable context format is even more fiddly. Until we
813 * grow some way for the caller to express a preference, and/or move
814 * the decision into the io-pgtable code where it arguably belongs,
815 * just aim for the closest thing to the rest of the system, and hope
816 * that the hardware isn't esoteric enough that we can't assume AArch64
817 * support to be a superset of AArch32 support...
818 */
819 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
820 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
821 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
822 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
823 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
824 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
825 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
826 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
827 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
828 ARM_SMMU_FEAT_FMT_AARCH64_16K |
829 ARM_SMMU_FEAT_FMT_AARCH64_4K)))
830 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
831
832 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
833 ret = -EINVAL;
834 goto out_unlock;
835 }
836
837 switch (smmu_domain->stage) {
838 case ARM_SMMU_DOMAIN_S1:
839 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
840 start = smmu->num_s2_context_banks;
841 ias = smmu->va_size;
842 oas = smmu->ipa_size;
843 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
844 fmt = ARM_64_LPAE_S1;
845 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
846 fmt = ARM_32_LPAE_S1;
847 ias = min(ias, 32UL);
848 oas = min(oas, 40UL);
849 } else {
850 fmt = ARM_V7S;
851 ias = min(ias, 32UL);
852 oas = min(oas, 32UL);
853 }
854 smmu_domain->tlb_ops = &arm_smmu_s1_tlb_ops;
855 break;
856 case ARM_SMMU_DOMAIN_NESTED:
857 /*
858 * We will likely want to change this if/when KVM gets
859 * involved.
860 */
861 case ARM_SMMU_DOMAIN_S2:
862 cfg->cbar = CBAR_TYPE_S2_TRANS;
863 start = 0;
864 ias = smmu->ipa_size;
865 oas = smmu->pa_size;
866 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
867 fmt = ARM_64_LPAE_S2;
868 } else {
869 fmt = ARM_32_LPAE_S2;
870 ias = min(ias, 40UL);
871 oas = min(oas, 40UL);
872 }
873 if (smmu->version == ARM_SMMU_V2)
874 smmu_domain->tlb_ops = &arm_smmu_s2_tlb_ops_v2;
875 else
876 smmu_domain->tlb_ops = &arm_smmu_s2_tlb_ops_v1;
877 break;
878 default:
879 ret = -EINVAL;
880 goto out_unlock;
881 }
882 ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
883 smmu->num_context_banks);
884 if (ret < 0)
885 goto out_unlock;
886
887 cfg->cbndx = ret;
888 if (smmu->version < ARM_SMMU_V2) {
889 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
890 cfg->irptndx %= smmu->num_context_irqs;
891 } else {
892 cfg->irptndx = cfg->cbndx;
893 }
894
895 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
896 cfg->vmid = cfg->cbndx + 1 + smmu->cavium_id_base;
897 else
898 cfg->asid = cfg->cbndx + smmu->cavium_id_base;
899
900 pgtbl_cfg = (struct io_pgtable_cfg) {
901 .pgsize_bitmap = smmu->pgsize_bitmap,
902 .ias = ias,
903 .oas = oas,
904 .tlb = smmu_domain->tlb_ops,
905 .iommu_dev = smmu->dev,
906 };
907
908 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
909 pgtbl_cfg.quirks = IO_PGTABLE_QUIRK_NO_DMA;
910
911 if (smmu_domain->non_strict)
912 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
913
914 smmu_domain->smmu = smmu;
915 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
916 if (!pgtbl_ops) {
917 ret = -ENOMEM;
918 goto out_clear_smmu;
919 }
920
921 /* Update the domain's page sizes to reflect the page table format */
922 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
923 domain->geometry.aperture_end = (1UL << ias) - 1;
924 domain->geometry.force_aperture = true;
925
926 /* Initialise the context bank with our page table cfg */
927 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
928 arm_smmu_write_context_bank(smmu, cfg->cbndx);
929
930 /*
931 * Request context fault interrupt. Do this last to avoid the
932 * handler seeing a half-initialised domain state.
933 */
934 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
935 ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
936 IRQF_SHARED, "arm-smmu-context-fault", domain);
937 if (ret < 0) {
938 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
939 cfg->irptndx, irq);
940 cfg->irptndx = INVALID_IRPTNDX;
941 }
942
943 mutex_unlock(&smmu_domain->init_mutex);
944
945 /* Publish page table ops for map/unmap */
946 smmu_domain->pgtbl_ops = pgtbl_ops;
947 return 0;
948
949 out_clear_smmu:
950 smmu_domain->smmu = NULL;
951 out_unlock:
952 mutex_unlock(&smmu_domain->init_mutex);
953 return ret;
954 }
955
956 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
957 {
958 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
959 struct arm_smmu_device *smmu = smmu_domain->smmu;
960 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
961 int ret, irq;
962
963 if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
964 return;
965
966 ret = arm_smmu_rpm_get(smmu);
967 if (ret < 0)
968 return;
969
970 /*
971 * Disable the context bank and free the page tables before freeing
972 * it.
973 */
974 smmu->cbs[cfg->cbndx].cfg = NULL;
975 arm_smmu_write_context_bank(smmu, cfg->cbndx);
976
977 if (cfg->irptndx != INVALID_IRPTNDX) {
978 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
979 devm_free_irq(smmu->dev, irq, domain);
980 }
981
982 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
983 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
984
985 arm_smmu_rpm_put(smmu);
986 }
987
988 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
989 {
990 struct arm_smmu_domain *smmu_domain;
991
992 if (type != IOMMU_DOMAIN_UNMANAGED &&
993 type != IOMMU_DOMAIN_DMA &&
994 type != IOMMU_DOMAIN_IDENTITY)
995 return NULL;
996 /*
997 * Allocate the domain and initialise some of its data structures.
998 * We can't really do anything meaningful until we've added a
999 * master.
1000 */
1001 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1002 if (!smmu_domain)
1003 return NULL;
1004
1005 if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
1006 iommu_get_dma_cookie(&smmu_domain->domain))) {
1007 kfree(smmu_domain);
1008 return NULL;
1009 }
1010
1011 mutex_init(&smmu_domain->init_mutex);
1012 spin_lock_init(&smmu_domain->cb_lock);
1013
1014 return &smmu_domain->domain;
1015 }
1016
1017 static void arm_smmu_domain_free(struct iommu_domain *domain)
1018 {
1019 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1020
1021 /*
1022 * Free the domain resources. We assume that all devices have
1023 * already been detached.
1024 */
1025 iommu_put_dma_cookie(domain);
1026 arm_smmu_destroy_domain_context(domain);
1027 kfree(smmu_domain);
1028 }
1029
1030 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
1031 {
1032 struct arm_smmu_smr *smr = smmu->smrs + idx;
1033 u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT;
1034
1035 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
1036 reg |= SMR_VALID;
1037 writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx));
1038 }
1039
1040 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
1041 {
1042 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
1043 u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT |
1044 (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT |
1045 (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT;
1046
1047 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
1048 smmu->smrs[idx].valid)
1049 reg |= S2CR_EXIDVALID;
1050 writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx));
1051 }
1052
1053 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
1054 {
1055 arm_smmu_write_s2cr(smmu, idx);
1056 if (smmu->smrs)
1057 arm_smmu_write_smr(smmu, idx);
1058 }
1059
1060 /*
1061 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
1062 * should be called after sCR0 is written.
1063 */
1064 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
1065 {
1066 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1067 u32 smr;
1068
1069 if (!smmu->smrs)
1070 return;
1071
1072 /*
1073 * SMR.ID bits may not be preserved if the corresponding MASK
1074 * bits are set, so check each one separately. We can reject
1075 * masters later if they try to claim IDs outside these masks.
1076 */
1077 smr = smmu->streamid_mask << SMR_ID_SHIFT;
1078 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1079 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1080 smmu->streamid_mask = smr >> SMR_ID_SHIFT;
1081
1082 smr = smmu->streamid_mask << SMR_MASK_SHIFT;
1083 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1084 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1085 smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT;
1086 }
1087
1088 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1089 {
1090 struct arm_smmu_smr *smrs = smmu->smrs;
1091 int i, free_idx = -ENOSPC;
1092
1093 /* Stream indexing is blissfully easy */
1094 if (!smrs)
1095 return id;
1096
1097 /* Validating SMRs is... less so */
1098 for (i = 0; i < smmu->num_mapping_groups; ++i) {
1099 if (!smrs[i].valid) {
1100 /*
1101 * Note the first free entry we come across, which
1102 * we'll claim in the end if nothing else matches.
1103 */
1104 if (free_idx < 0)
1105 free_idx = i;
1106 continue;
1107 }
1108 /*
1109 * If the new entry is _entirely_ matched by an existing entry,
1110 * then reuse that, with the guarantee that there also cannot
1111 * be any subsequent conflicting entries. In normal use we'd
1112 * expect simply identical entries for this case, but there's
1113 * no harm in accommodating the generalisation.
1114 */
1115 if ((mask & smrs[i].mask) == mask &&
1116 !((id ^ smrs[i].id) & ~smrs[i].mask))
1117 return i;
1118 /*
1119 * If the new entry has any other overlap with an existing one,
1120 * though, then there always exists at least one stream ID
1121 * which would cause a conflict, and we can't allow that risk.
1122 */
1123 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1124 return -EINVAL;
1125 }
1126
1127 return free_idx;
1128 }
1129
1130 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1131 {
1132 if (--smmu->s2crs[idx].count)
1133 return false;
1134
1135 smmu->s2crs[idx] = s2cr_init_val;
1136 if (smmu->smrs)
1137 smmu->smrs[idx].valid = false;
1138
1139 return true;
1140 }
1141
1142 static int arm_smmu_master_alloc_smes(struct device *dev)
1143 {
1144 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1145 struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1146 struct arm_smmu_device *smmu = cfg->smmu;
1147 struct arm_smmu_smr *smrs = smmu->smrs;
1148 struct iommu_group *group;
1149 int i, idx, ret;
1150
1151 mutex_lock(&smmu->stream_map_mutex);
1152 /* Figure out a viable stream map entry allocation */
1153 for_each_cfg_sme(fwspec, i, idx) {
1154 u16 sid = fwspec->ids[i];
1155 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1156
1157 if (idx != INVALID_SMENDX) {
1158 ret = -EEXIST;
1159 goto out_err;
1160 }
1161
1162 ret = arm_smmu_find_sme(smmu, sid, mask);
1163 if (ret < 0)
1164 goto out_err;
1165
1166 idx = ret;
1167 if (smrs && smmu->s2crs[idx].count == 0) {
1168 smrs[idx].id = sid;
1169 smrs[idx].mask = mask;
1170 smrs[idx].valid = true;
1171 }
1172 smmu->s2crs[idx].count++;
1173 cfg->smendx[i] = (s16)idx;
1174 }
1175
1176 group = iommu_group_get_for_dev(dev);
1177 if (!group)
1178 group = ERR_PTR(-ENOMEM);
1179 if (IS_ERR(group)) {
1180 ret = PTR_ERR(group);
1181 goto out_err;
1182 }
1183 iommu_group_put(group);
1184
1185 /* It worked! Now, poke the actual hardware */
1186 for_each_cfg_sme(fwspec, i, idx) {
1187 arm_smmu_write_sme(smmu, idx);
1188 smmu->s2crs[idx].group = group;
1189 }
1190
1191 mutex_unlock(&smmu->stream_map_mutex);
1192 return 0;
1193
1194 out_err:
1195 while (i--) {
1196 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1197 cfg->smendx[i] = INVALID_SMENDX;
1198 }
1199 mutex_unlock(&smmu->stream_map_mutex);
1200 return ret;
1201 }
1202
1203 static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
1204 {
1205 struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1206 struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1207 int i, idx;
1208
1209 mutex_lock(&smmu->stream_map_mutex);
1210 for_each_cfg_sme(fwspec, i, idx) {
1211 if (arm_smmu_free_sme(smmu, idx))
1212 arm_smmu_write_sme(smmu, idx);
1213 cfg->smendx[i] = INVALID_SMENDX;
1214 }
1215 mutex_unlock(&smmu->stream_map_mutex);
1216 }
1217
1218 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1219 struct iommu_fwspec *fwspec)
1220 {
1221 struct arm_smmu_device *smmu = smmu_domain->smmu;
1222 struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1223 u8 cbndx = smmu_domain->cfg.cbndx;
1224 enum arm_smmu_s2cr_type type;
1225 int i, idx;
1226
1227 if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
1228 type = S2CR_TYPE_BYPASS;
1229 else
1230 type = S2CR_TYPE_TRANS;
1231
1232 for_each_cfg_sme(fwspec, i, idx) {
1233 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1234 continue;
1235
1236 s2cr[idx].type = type;
1237 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1238 s2cr[idx].cbndx = cbndx;
1239 arm_smmu_write_s2cr(smmu, idx);
1240 }
1241 return 0;
1242 }
1243
1244 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1245 {
1246 int ret;
1247 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1248 struct arm_smmu_device *smmu;
1249 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1250
1251 if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1252 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1253 return -ENXIO;
1254 }
1255
1256 /*
1257 * FIXME: The arch/arm DMA API code tries to attach devices to its own
1258 * domains between of_xlate() and add_device() - we have no way to cope
1259 * with that, so until ARM gets converted to rely on groups and default
1260 * domains, just say no (but more politely than by dereferencing NULL).
1261 * This should be at least a WARN_ON once that's sorted.
1262 */
1263 if (!fwspec->iommu_priv)
1264 return -ENODEV;
1265
1266 smmu = fwspec_smmu(fwspec);
1267
1268 ret = arm_smmu_rpm_get(smmu);
1269 if (ret < 0)
1270 return ret;
1271
1272 /* Ensure that the domain is finalised */
1273 ret = arm_smmu_init_domain_context(domain, smmu);
1274 if (ret < 0)
1275 goto rpm_put;
1276
1277 /*
1278 * Sanity check the domain. We don't support domains across
1279 * different SMMUs.
1280 */
1281 if (smmu_domain->smmu != smmu) {
1282 dev_err(dev,
1283 "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1284 dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1285 ret = -EINVAL;
1286 goto rpm_put;
1287 }
1288
1289 /* Looks ok, so add the device to the domain */
1290 ret = arm_smmu_domain_add_master(smmu_domain, fwspec);
1291
1292 rpm_put:
1293 arm_smmu_rpm_put(smmu);
1294 return ret;
1295 }
1296
1297 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1298 phys_addr_t paddr, size_t size, int prot)
1299 {
1300 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1301 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1302 int ret;
1303
1304 if (!ops)
1305 return -ENODEV;
1306
1307 arm_smmu_rpm_get(smmu);
1308 ret = ops->map(ops, iova, paddr, size, prot);
1309 arm_smmu_rpm_put(smmu);
1310
1311 return ret;
1312 }
1313
1314 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1315 size_t size)
1316 {
1317 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1318 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1319 size_t ret;
1320
1321 if (!ops)
1322 return 0;
1323
1324 arm_smmu_rpm_get(smmu);
1325 ret = ops->unmap(ops, iova, size);
1326 arm_smmu_rpm_put(smmu);
1327
1328 return ret;
1329 }
1330
1331 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1332 {
1333 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1334 struct arm_smmu_device *smmu = smmu_domain->smmu;
1335
1336 if (smmu_domain->tlb_ops) {
1337 arm_smmu_rpm_get(smmu);
1338 smmu_domain->tlb_ops->tlb_flush_all(smmu_domain);
1339 arm_smmu_rpm_put(smmu);
1340 }
1341 }
1342
1343 static void arm_smmu_iotlb_sync(struct iommu_domain *domain)
1344 {
1345 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1346 struct arm_smmu_device *smmu = smmu_domain->smmu;
1347
1348 if (smmu_domain->tlb_ops) {
1349 arm_smmu_rpm_get(smmu);
1350 smmu_domain->tlb_ops->tlb_sync(smmu_domain);
1351 arm_smmu_rpm_put(smmu);
1352 }
1353 }
1354
1355 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1356 dma_addr_t iova)
1357 {
1358 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1359 struct arm_smmu_device *smmu = smmu_domain->smmu;
1360 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1361 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1362 struct device *dev = smmu->dev;
1363 void __iomem *cb_base;
1364 u32 tmp;
1365 u64 phys;
1366 unsigned long va, flags;
1367 int ret;
1368
1369 ret = arm_smmu_rpm_get(smmu);
1370 if (ret < 0)
1371 return 0;
1372
1373 cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
1374
1375 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1376 /* ATS1 registers can only be written atomically */
1377 va = iova & ~0xfffUL;
1378 if (smmu->version == ARM_SMMU_V2)
1379 smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1380 else /* Register is only 32-bit in v1 */
1381 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1382
1383 if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1384 !(tmp & ATSR_ACTIVE), 5, 50)) {
1385 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1386 dev_err(dev,
1387 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1388 &iova);
1389 return ops->iova_to_phys(ops, iova);
1390 }
1391
1392 phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
1393 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1394 if (phys & CB_PAR_F) {
1395 dev_err(dev, "translation fault!\n");
1396 dev_err(dev, "PAR = 0x%llx\n", phys);
1397 return 0;
1398 }
1399
1400 arm_smmu_rpm_put(smmu);
1401
1402 return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1403 }
1404
1405 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1406 dma_addr_t iova)
1407 {
1408 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1409 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1410
1411 if (domain->type == IOMMU_DOMAIN_IDENTITY)
1412 return iova;
1413
1414 if (!ops)
1415 return 0;
1416
1417 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1418 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1419 return arm_smmu_iova_to_phys_hard(domain, iova);
1420
1421 return ops->iova_to_phys(ops, iova);
1422 }
1423
1424 static bool arm_smmu_capable(enum iommu_cap cap)
1425 {
1426 switch (cap) {
1427 case IOMMU_CAP_CACHE_COHERENCY:
1428 /*
1429 * Return true here as the SMMU can always send out coherent
1430 * requests.
1431 */
1432 return true;
1433 case IOMMU_CAP_NOEXEC:
1434 return true;
1435 default:
1436 return false;
1437 }
1438 }
1439
1440 static int arm_smmu_match_node(struct device *dev, void *data)
1441 {
1442 return dev->fwnode == data;
1443 }
1444
1445 static
1446 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1447 {
1448 struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
1449 fwnode, arm_smmu_match_node);
1450 put_device(dev);
1451 return dev ? dev_get_drvdata(dev) : NULL;
1452 }
1453
1454 static int arm_smmu_add_device(struct device *dev)
1455 {
1456 struct arm_smmu_device *smmu;
1457 struct arm_smmu_master_cfg *cfg;
1458 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1459 int i, ret;
1460
1461 if (using_legacy_binding) {
1462 ret = arm_smmu_register_legacy_master(dev, &smmu);
1463
1464 /*
1465 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1466 * will allocate/initialise a new one. Thus we need to update fwspec for
1467 * later use.
1468 */
1469 fwspec = dev_iommu_fwspec_get(dev);
1470 if (ret)
1471 goto out_free;
1472 } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1473 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1474 } else {
1475 return -ENODEV;
1476 }
1477
1478 ret = -EINVAL;
1479 for (i = 0; i < fwspec->num_ids; i++) {
1480 u16 sid = fwspec->ids[i];
1481 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1482
1483 if (sid & ~smmu->streamid_mask) {
1484 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1485 sid, smmu->streamid_mask);
1486 goto out_free;
1487 }
1488 if (mask & ~smmu->smr_mask_mask) {
1489 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1490 mask, smmu->smr_mask_mask);
1491 goto out_free;
1492 }
1493 }
1494
1495 ret = -ENOMEM;
1496 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1497 GFP_KERNEL);
1498 if (!cfg)
1499 goto out_free;
1500
1501 cfg->smmu = smmu;
1502 fwspec->iommu_priv = cfg;
1503 while (i--)
1504 cfg->smendx[i] = INVALID_SMENDX;
1505
1506 ret = arm_smmu_rpm_get(smmu);
1507 if (ret < 0)
1508 goto out_cfg_free;
1509
1510 ret = arm_smmu_master_alloc_smes(dev);
1511 arm_smmu_rpm_put(smmu);
1512
1513 if (ret)
1514 goto out_cfg_free;
1515
1516 iommu_device_link(&smmu->iommu, dev);
1517
1518 device_link_add(dev, smmu->dev,
1519 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1520
1521 return 0;
1522
1523 out_cfg_free:
1524 kfree(cfg);
1525 out_free:
1526 iommu_fwspec_free(dev);
1527 return ret;
1528 }
1529
1530 static void arm_smmu_remove_device(struct device *dev)
1531 {
1532 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1533 struct arm_smmu_master_cfg *cfg;
1534 struct arm_smmu_device *smmu;
1535 int ret;
1536
1537 if (!fwspec || fwspec->ops != &arm_smmu_ops)
1538 return;
1539
1540 cfg = fwspec->iommu_priv;
1541 smmu = cfg->smmu;
1542
1543 ret = arm_smmu_rpm_get(smmu);
1544 if (ret < 0)
1545 return;
1546
1547 iommu_device_unlink(&smmu->iommu, dev);
1548 arm_smmu_master_free_smes(fwspec);
1549
1550 arm_smmu_rpm_put(smmu);
1551
1552 iommu_group_remove_device(dev);
1553 kfree(fwspec->iommu_priv);
1554 iommu_fwspec_free(dev);
1555 }
1556
1557 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1558 {
1559 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1560 struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1561 struct iommu_group *group = NULL;
1562 int i, idx;
1563
1564 for_each_cfg_sme(fwspec, i, idx) {
1565 if (group && smmu->s2crs[idx].group &&
1566 group != smmu->s2crs[idx].group)
1567 return ERR_PTR(-EINVAL);
1568
1569 group = smmu->s2crs[idx].group;
1570 }
1571
1572 if (group)
1573 return iommu_group_ref_get(group);
1574
1575 if (dev_is_pci(dev))
1576 group = pci_device_group(dev);
1577 else if (dev_is_fsl_mc(dev))
1578 group = fsl_mc_device_group(dev);
1579 else
1580 group = generic_device_group(dev);
1581
1582 return group;
1583 }
1584
1585 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1586 enum iommu_attr attr, void *data)
1587 {
1588 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1589
1590 switch(domain->type) {
1591 case IOMMU_DOMAIN_UNMANAGED:
1592 switch (attr) {
1593 case DOMAIN_ATTR_NESTING:
1594 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1595 return 0;
1596 default:
1597 return -ENODEV;
1598 }
1599 break;
1600 case IOMMU_DOMAIN_DMA:
1601 switch (attr) {
1602 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1603 *(int *)data = smmu_domain->non_strict;
1604 return 0;
1605 default:
1606 return -ENODEV;
1607 }
1608 break;
1609 default:
1610 return -EINVAL;
1611 }
1612 }
1613
1614 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1615 enum iommu_attr attr, void *data)
1616 {
1617 int ret = 0;
1618 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1619
1620 mutex_lock(&smmu_domain->init_mutex);
1621
1622 switch(domain->type) {
1623 case IOMMU_DOMAIN_UNMANAGED:
1624 switch (attr) {
1625 case DOMAIN_ATTR_NESTING:
1626 if (smmu_domain->smmu) {
1627 ret = -EPERM;
1628 goto out_unlock;
1629 }
1630
1631 if (*(int *)data)
1632 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1633 else
1634 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1635 break;
1636 default:
1637 ret = -ENODEV;
1638 }
1639 break;
1640 case IOMMU_DOMAIN_DMA:
1641 switch (attr) {
1642 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1643 smmu_domain->non_strict = *(int *)data;
1644 break;
1645 default:
1646 ret = -ENODEV;
1647 }
1648 break;
1649 default:
1650 ret = -EINVAL;
1651 }
1652 out_unlock:
1653 mutex_unlock(&smmu_domain->init_mutex);
1654 return ret;
1655 }
1656
1657 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1658 {
1659 u32 mask, fwid = 0;
1660
1661 if (args->args_count > 0)
1662 fwid |= (u16)args->args[0];
1663
1664 if (args->args_count > 1)
1665 fwid |= (u16)args->args[1] << SMR_MASK_SHIFT;
1666 else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1667 fwid |= (u16)mask << SMR_MASK_SHIFT;
1668
1669 return iommu_fwspec_add_ids(dev, &fwid, 1);
1670 }
1671
1672 static void arm_smmu_get_resv_regions(struct device *dev,
1673 struct list_head *head)
1674 {
1675 struct iommu_resv_region *region;
1676 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1677
1678 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1679 prot, IOMMU_RESV_SW_MSI);
1680 if (!region)
1681 return;
1682
1683 list_add_tail(&region->list, head);
1684
1685 iommu_dma_get_resv_regions(dev, head);
1686 }
1687
1688 static void arm_smmu_put_resv_regions(struct device *dev,
1689 struct list_head *head)
1690 {
1691 struct iommu_resv_region *entry, *next;
1692
1693 list_for_each_entry_safe(entry, next, head, list)
1694 kfree(entry);
1695 }
1696
1697 static struct iommu_ops arm_smmu_ops = {
1698 .capable = arm_smmu_capable,
1699 .domain_alloc = arm_smmu_domain_alloc,
1700 .domain_free = arm_smmu_domain_free,
1701 .attach_dev = arm_smmu_attach_dev,
1702 .map = arm_smmu_map,
1703 .unmap = arm_smmu_unmap,
1704 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
1705 .iotlb_sync = arm_smmu_iotlb_sync,
1706 .iova_to_phys = arm_smmu_iova_to_phys,
1707 .add_device = arm_smmu_add_device,
1708 .remove_device = arm_smmu_remove_device,
1709 .device_group = arm_smmu_device_group,
1710 .domain_get_attr = arm_smmu_domain_get_attr,
1711 .domain_set_attr = arm_smmu_domain_set_attr,
1712 .of_xlate = arm_smmu_of_xlate,
1713 .get_resv_regions = arm_smmu_get_resv_regions,
1714 .put_resv_regions = arm_smmu_put_resv_regions,
1715 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1716 };
1717
1718 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1719 {
1720 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1721 int i;
1722 u32 reg, major;
1723
1724 /* clear global FSR */
1725 reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1726 writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1727
1728 /*
1729 * Reset stream mapping groups: Initial values mark all SMRn as
1730 * invalid and all S2CRn as bypass unless overridden.
1731 */
1732 for (i = 0; i < smmu->num_mapping_groups; ++i)
1733 arm_smmu_write_sme(smmu, i);
1734
1735 if (smmu->model == ARM_MMU500) {
1736 /*
1737 * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1738 * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1739 * bit is only present in MMU-500r2 onwards.
1740 */
1741 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
1742 major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
1743 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
1744 if (major >= 2)
1745 reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1746 /*
1747 * Allow unmatched Stream IDs to allocate bypass
1748 * TLB entries for reduced latency.
1749 */
1750 reg |= ARM_MMU500_ACR_SMTNMB_TLBEN | ARM_MMU500_ACR_S2CRB_TLBEN;
1751 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
1752 }
1753
1754 /* Make sure all context banks are disabled and clear CB_FSR */
1755 for (i = 0; i < smmu->num_context_banks; ++i) {
1756 void __iomem *cb_base = ARM_SMMU_CB(smmu, i);
1757
1758 arm_smmu_write_context_bank(smmu, i);
1759 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1760 /*
1761 * Disable MMU-500's not-particularly-beneficial next-page
1762 * prefetcher for the sake of errata #841119 and #826419.
1763 */
1764 if (smmu->model == ARM_MMU500) {
1765 reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1766 reg &= ~ARM_MMU500_ACTLR_CPRE;
1767 writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1768 }
1769 }
1770
1771 /* Invalidate the TLB, just in case */
1772 writel_relaxed(QCOM_DUMMY_VAL, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1773 writel_relaxed(QCOM_DUMMY_VAL, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1774
1775 reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1776
1777 /* Enable fault reporting */
1778 reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1779
1780 /* Disable TLB broadcasting. */
1781 reg |= (sCR0_VMIDPNE | sCR0_PTM);
1782
1783 /* Enable client access, handling unmatched streams as appropriate */
1784 reg &= ~sCR0_CLIENTPD;
1785 if (disable_bypass)
1786 reg |= sCR0_USFCFG;
1787 else
1788 reg &= ~sCR0_USFCFG;
1789
1790 /* Disable forced broadcasting */
1791 reg &= ~sCR0_FB;
1792
1793 /* Don't upgrade barriers */
1794 reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
1795
1796 if (smmu->features & ARM_SMMU_FEAT_VMID16)
1797 reg |= sCR0_VMID16EN;
1798
1799 if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1800 reg |= sCR0_EXIDENABLE;
1801
1802 /* Push the button */
1803 arm_smmu_tlb_sync_global(smmu);
1804 writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1805 }
1806
1807 static int arm_smmu_id_size_to_bits(int size)
1808 {
1809 switch (size) {
1810 case 0:
1811 return 32;
1812 case 1:
1813 return 36;
1814 case 2:
1815 return 40;
1816 case 3:
1817 return 42;
1818 case 4:
1819 return 44;
1820 case 5:
1821 default:
1822 return 48;
1823 }
1824 }
1825
1826 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1827 {
1828 unsigned long size;
1829 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1830 u32 id;
1831 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1832 int i;
1833
1834 dev_notice(smmu->dev, "probing hardware configuration...\n");
1835 dev_notice(smmu->dev, "SMMUv%d with:\n",
1836 smmu->version == ARM_SMMU_V2 ? 2 : 1);
1837
1838 /* ID0 */
1839 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1840
1841 /* Restrict available stages based on module parameter */
1842 if (force_stage == 1)
1843 id &= ~(ID0_S2TS | ID0_NTS);
1844 else if (force_stage == 2)
1845 id &= ~(ID0_S1TS | ID0_NTS);
1846
1847 if (id & ID0_S1TS) {
1848 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1849 dev_notice(smmu->dev, "\tstage 1 translation\n");
1850 }
1851
1852 if (id & ID0_S2TS) {
1853 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1854 dev_notice(smmu->dev, "\tstage 2 translation\n");
1855 }
1856
1857 if (id & ID0_NTS) {
1858 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1859 dev_notice(smmu->dev, "\tnested translation\n");
1860 }
1861
1862 if (!(smmu->features &
1863 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1864 dev_err(smmu->dev, "\tno translation support!\n");
1865 return -ENODEV;
1866 }
1867
1868 if ((id & ID0_S1TS) &&
1869 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1870 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1871 dev_notice(smmu->dev, "\taddress translation ops\n");
1872 }
1873
1874 /*
1875 * In order for DMA API calls to work properly, we must defer to what
1876 * the FW says about coherency, regardless of what the hardware claims.
1877 * Fortunately, this also opens up a workaround for systems where the
1878 * ID register value has ended up configured incorrectly.
1879 */
1880 cttw_reg = !!(id & ID0_CTTW);
1881 if (cttw_fw || cttw_reg)
1882 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1883 cttw_fw ? "" : "non-");
1884 if (cttw_fw != cttw_reg)
1885 dev_notice(smmu->dev,
1886 "\t(IDR0.CTTW overridden by FW configuration)\n");
1887
1888 /* Max. number of entries we have for stream matching/indexing */
1889 if (smmu->version == ARM_SMMU_V2 && id & ID0_EXIDS) {
1890 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1891 size = 1 << 16;
1892 } else {
1893 size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK);
1894 }
1895 smmu->streamid_mask = size - 1;
1896 if (id & ID0_SMS) {
1897 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1898 size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK;
1899 if (size == 0) {
1900 dev_err(smmu->dev,
1901 "stream-matching supported, but no SMRs present!\n");
1902 return -ENODEV;
1903 }
1904
1905 /* Zero-initialised to mark as invalid */
1906 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1907 GFP_KERNEL);
1908 if (!smmu->smrs)
1909 return -ENOMEM;
1910
1911 dev_notice(smmu->dev,
1912 "\tstream matching with %lu register groups", size);
1913 }
1914 /* s2cr->type == 0 means translation, so initialise explicitly */
1915 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1916 GFP_KERNEL);
1917 if (!smmu->s2crs)
1918 return -ENOMEM;
1919 for (i = 0; i < size; i++)
1920 smmu->s2crs[i] = s2cr_init_val;
1921
1922 smmu->num_mapping_groups = size;
1923 mutex_init(&smmu->stream_map_mutex);
1924 spin_lock_init(&smmu->global_sync_lock);
1925
1926 if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1927 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1928 if (!(id & ID0_PTFS_NO_AARCH32S))
1929 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1930 }
1931
1932 /* ID1 */
1933 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1934 smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1935
1936 /* Check for size mismatch of SMMU address space from mapped region */
1937 size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1938 size <<= smmu->pgshift;
1939 if (smmu->cb_base != gr0_base + size)
1940 dev_warn(smmu->dev,
1941 "SMMU address space size (0x%lx) differs from mapped region size (0x%tx)!\n",
1942 size * 2, (smmu->cb_base - gr0_base) * 2);
1943
1944 smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
1945 smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1946 if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1947 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1948 return -ENODEV;
1949 }
1950 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1951 smmu->num_context_banks, smmu->num_s2_context_banks);
1952 /*
1953 * Cavium CN88xx erratum #27704.
1954 * Ensure ASID and VMID allocation is unique across all SMMUs in
1955 * the system.
1956 */
1957 if (smmu->model == CAVIUM_SMMUV2) {
1958 smmu->cavium_id_base =
1959 atomic_add_return(smmu->num_context_banks,
1960 &cavium_smmu_context_count);
1961 smmu->cavium_id_base -= smmu->num_context_banks;
1962 dev_notice(smmu->dev, "\tenabling workaround for Cavium erratum 27704\n");
1963 }
1964 smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1965 sizeof(*smmu->cbs), GFP_KERNEL);
1966 if (!smmu->cbs)
1967 return -ENOMEM;
1968
1969 /* ID2 */
1970 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1971 size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1972 smmu->ipa_size = size;
1973
1974 /* The output mask is also applied for bypass */
1975 size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1976 smmu->pa_size = size;
1977
1978 if (id & ID2_VMID16)
1979 smmu->features |= ARM_SMMU_FEAT_VMID16;
1980
1981 /*
1982 * What the page table walker can address actually depends on which
1983 * descriptor format is in use, but since a) we don't know that yet,
1984 * and b) it can vary per context bank, this will have to do...
1985 */
1986 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1987 dev_warn(smmu->dev,
1988 "failed to set DMA mask for table walker\n");
1989
1990 if (smmu->version < ARM_SMMU_V2) {
1991 smmu->va_size = smmu->ipa_size;
1992 if (smmu->version == ARM_SMMU_V1_64K)
1993 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1994 } else {
1995 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
1996 smmu->va_size = arm_smmu_id_size_to_bits(size);
1997 if (id & ID2_PTFS_4K)
1998 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1999 if (id & ID2_PTFS_16K)
2000 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
2001 if (id & ID2_PTFS_64K)
2002 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
2003 }
2004
2005 /* Now we've corralled the various formats, what'll it do? */
2006 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
2007 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
2008 if (smmu->features &
2009 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
2010 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
2011 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
2012 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
2013 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
2014 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
2015
2016 if (arm_smmu_ops.pgsize_bitmap == -1UL)
2017 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
2018 else
2019 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
2020 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
2021 smmu->pgsize_bitmap);
2022
2023
2024 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
2025 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
2026 smmu->va_size, smmu->ipa_size);
2027
2028 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
2029 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
2030 smmu->ipa_size, smmu->pa_size);
2031
2032 return 0;
2033 }
2034
2035 struct arm_smmu_match_data {
2036 enum arm_smmu_arch_version version;
2037 enum arm_smmu_implementation model;
2038 };
2039
2040 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \
2041 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
2042
2043 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
2044 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
2045 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
2046 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
2047 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
2048 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
2049
2050 static const struct of_device_id arm_smmu_of_match[] = {
2051 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
2052 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
2053 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
2054 { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
2055 { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
2056 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
2057 { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
2058 { },
2059 };
2060
2061 #ifdef CONFIG_ACPI
2062 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
2063 {
2064 int ret = 0;
2065
2066 switch (model) {
2067 case ACPI_IORT_SMMU_V1:
2068 case ACPI_IORT_SMMU_CORELINK_MMU400:
2069 smmu->version = ARM_SMMU_V1;
2070 smmu->model = GENERIC_SMMU;
2071 break;
2072 case ACPI_IORT_SMMU_CORELINK_MMU401:
2073 smmu->version = ARM_SMMU_V1_64K;
2074 smmu->model = GENERIC_SMMU;
2075 break;
2076 case ACPI_IORT_SMMU_V2:
2077 smmu->version = ARM_SMMU_V2;
2078 smmu->model = GENERIC_SMMU;
2079 break;
2080 case ACPI_IORT_SMMU_CORELINK_MMU500:
2081 smmu->version = ARM_SMMU_V2;
2082 smmu->model = ARM_MMU500;
2083 break;
2084 case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
2085 smmu->version = ARM_SMMU_V2;
2086 smmu->model = CAVIUM_SMMUV2;
2087 break;
2088 default:
2089 ret = -ENODEV;
2090 }
2091
2092 return ret;
2093 }
2094
2095 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2096 struct arm_smmu_device *smmu)
2097 {
2098 struct device *dev = smmu->dev;
2099 struct acpi_iort_node *node =
2100 *(struct acpi_iort_node **)dev_get_platdata(dev);
2101 struct acpi_iort_smmu *iort_smmu;
2102 int ret;
2103
2104 /* Retrieve SMMU1/2 specific data */
2105 iort_smmu = (struct acpi_iort_smmu *)node->node_data;
2106
2107 ret = acpi_smmu_get_data(iort_smmu->model, smmu);
2108 if (ret < 0)
2109 return ret;
2110
2111 /* Ignore the configuration access interrupt */
2112 smmu->num_global_irqs = 1;
2113
2114 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
2115 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2116
2117 return 0;
2118 }
2119 #else
2120 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2121 struct arm_smmu_device *smmu)
2122 {
2123 return -ENODEV;
2124 }
2125 #endif
2126
2127 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
2128 struct arm_smmu_device *smmu)
2129 {
2130 const struct arm_smmu_match_data *data;
2131 struct device *dev = &pdev->dev;
2132 bool legacy_binding;
2133
2134 if (of_property_read_u32(dev->of_node, "#global-interrupts",
2135 &smmu->num_global_irqs)) {
2136 dev_err(dev, "missing #global-interrupts property\n");
2137 return -ENODEV;
2138 }
2139
2140 data = of_device_get_match_data(dev);
2141 smmu->version = data->version;
2142 smmu->model = data->model;
2143
2144 parse_driver_options(smmu);
2145
2146 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2147 if (legacy_binding && !using_generic_binding) {
2148 if (!using_legacy_binding)
2149 pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
2150 using_legacy_binding = true;
2151 } else if (!legacy_binding && !using_legacy_binding) {
2152 using_generic_binding = true;
2153 } else {
2154 dev_err(dev, "not probing due to mismatched DT properties\n");
2155 return -ENODEV;
2156 }
2157
2158 if (of_dma_is_coherent(dev->of_node))
2159 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2160
2161 return 0;
2162 }
2163
2164 static void arm_smmu_bus_init(void)
2165 {
2166 /* Oh, for a proper bus abstraction */
2167 if (!iommu_present(&platform_bus_type))
2168 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2169 #ifdef CONFIG_ARM_AMBA
2170 if (!iommu_present(&amba_bustype))
2171 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2172 #endif
2173 #ifdef CONFIG_PCI
2174 if (!iommu_present(&pci_bus_type)) {
2175 pci_request_acs();
2176 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2177 }
2178 #endif
2179 #ifdef CONFIG_FSL_MC_BUS
2180 if (!iommu_present(&fsl_mc_bus_type))
2181 bus_set_iommu(&fsl_mc_bus_type, &arm_smmu_ops);
2182 #endif
2183 }
2184
2185 static int arm_smmu_device_probe(struct platform_device *pdev)
2186 {
2187 struct resource *res;
2188 resource_size_t ioaddr;
2189 struct arm_smmu_device *smmu;
2190 struct device *dev = &pdev->dev;
2191 int num_irqs, i, err;
2192
2193 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2194 if (!smmu) {
2195 dev_err(dev, "failed to allocate arm_smmu_device\n");
2196 return -ENOMEM;
2197 }
2198 smmu->dev = dev;
2199
2200 if (dev->of_node)
2201 err = arm_smmu_device_dt_probe(pdev, smmu);
2202 else
2203 err = arm_smmu_device_acpi_probe(pdev, smmu);
2204
2205 if (err)
2206 return err;
2207
2208 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2209 ioaddr = res->start;
2210 smmu->base = devm_ioremap_resource(dev, res);
2211 if (IS_ERR(smmu->base))
2212 return PTR_ERR(smmu->base);
2213 smmu->cb_base = smmu->base + resource_size(res) / 2;
2214
2215 num_irqs = 0;
2216 while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2217 num_irqs++;
2218 if (num_irqs > smmu->num_global_irqs)
2219 smmu->num_context_irqs++;
2220 }
2221
2222 if (!smmu->num_context_irqs) {
2223 dev_err(dev, "found %d interrupts but expected at least %d\n",
2224 num_irqs, smmu->num_global_irqs + 1);
2225 return -ENODEV;
2226 }
2227
2228 smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs),
2229 GFP_KERNEL);
2230 if (!smmu->irqs) {
2231 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2232 return -ENOMEM;
2233 }
2234
2235 for (i = 0; i < num_irqs; ++i) {
2236 int irq = platform_get_irq(pdev, i);
2237
2238 if (irq < 0) {
2239 dev_err(dev, "failed to get irq index %d\n", i);
2240 return -ENODEV;
2241 }
2242 smmu->irqs[i] = irq;
2243 }
2244
2245 err = devm_clk_bulk_get_all(dev, &smmu->clks);
2246 if (err < 0) {
2247 dev_err(dev, "failed to get clocks %d\n", err);
2248 return err;
2249 }
2250 smmu->num_clks = err;
2251
2252 err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2253 if (err)
2254 return err;
2255
2256 err = arm_smmu_device_cfg_probe(smmu);
2257 if (err)
2258 return err;
2259
2260 if (smmu->version == ARM_SMMU_V2) {
2261 if (smmu->num_context_banks > smmu->num_context_irqs) {
2262 dev_err(dev,
2263 "found only %d context irq(s) but %d required\n",
2264 smmu->num_context_irqs, smmu->num_context_banks);
2265 return -ENODEV;
2266 }
2267
2268 /* Ignore superfluous interrupts */
2269 smmu->num_context_irqs = smmu->num_context_banks;
2270 }
2271
2272 for (i = 0; i < smmu->num_global_irqs; ++i) {
2273 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2274 arm_smmu_global_fault,
2275 IRQF_SHARED,
2276 "arm-smmu global fault",
2277 smmu);
2278 if (err) {
2279 dev_err(dev, "failed to request global IRQ %d (%u)\n",
2280 i, smmu->irqs[i]);
2281 return err;
2282 }
2283 }
2284
2285 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2286 "smmu.%pa", &ioaddr);
2287 if (err) {
2288 dev_err(dev, "Failed to register iommu in sysfs\n");
2289 return err;
2290 }
2291
2292 iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
2293 iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
2294
2295 err = iommu_device_register(&smmu->iommu);
2296 if (err) {
2297 dev_err(dev, "Failed to register iommu\n");
2298 return err;
2299 }
2300
2301 platform_set_drvdata(pdev, smmu);
2302 arm_smmu_device_reset(smmu);
2303 arm_smmu_test_smr_masks(smmu);
2304
2305 /*
2306 * We want to avoid touching dev->power.lock in fastpaths unless
2307 * it's really going to do something useful - pm_runtime_enabled()
2308 * can serve as an ideal proxy for that decision. So, conditionally
2309 * enable pm_runtime.
2310 */
2311 if (dev->pm_domain) {
2312 pm_runtime_set_active(dev);
2313 pm_runtime_enable(dev);
2314 }
2315
2316 /*
2317 * For ACPI and generic DT bindings, an SMMU will be probed before
2318 * any device which might need it, so we want the bus ops in place
2319 * ready to handle default domain setup as soon as any SMMU exists.
2320 */
2321 if (!using_legacy_binding)
2322 arm_smmu_bus_init();
2323
2324 return 0;
2325 }
2326
2327 /*
2328 * With the legacy DT binding in play, though, we have no guarantees about
2329 * probe order, but then we're also not doing default domains, so we can
2330 * delay setting bus ops until we're sure every possible SMMU is ready,
2331 * and that way ensure that no add_device() calls get missed.
2332 */
2333 static int arm_smmu_legacy_bus_init(void)
2334 {
2335 if (using_legacy_binding)
2336 arm_smmu_bus_init();
2337 return 0;
2338 }
2339 device_initcall_sync(arm_smmu_legacy_bus_init);
2340
2341 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2342 {
2343 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2344
2345 if (!smmu)
2346 return;
2347
2348 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2349 dev_err(&pdev->dev, "removing device with active domains!\n");
2350
2351 arm_smmu_rpm_get(smmu);
2352 /* Turn the thing off */
2353 writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
2354 arm_smmu_rpm_put(smmu);
2355
2356 if (pm_runtime_enabled(smmu->dev))
2357 pm_runtime_force_suspend(smmu->dev);
2358 else
2359 clk_bulk_disable(smmu->num_clks, smmu->clks);
2360
2361 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2362 }
2363
2364 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2365 {
2366 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2367 int ret;
2368
2369 ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2370 if (ret)
2371 return ret;
2372
2373 arm_smmu_device_reset(smmu);
2374
2375 return 0;
2376 }
2377
2378 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2379 {
2380 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2381
2382 clk_bulk_disable(smmu->num_clks, smmu->clks);
2383
2384 return 0;
2385 }
2386
2387 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2388 {
2389 if (pm_runtime_suspended(dev))
2390 return 0;
2391
2392 return arm_smmu_runtime_resume(dev);
2393 }
2394
2395 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2396 {
2397 if (pm_runtime_suspended(dev))
2398 return 0;
2399
2400 return arm_smmu_runtime_suspend(dev);
2401 }
2402
2403 static const struct dev_pm_ops arm_smmu_pm_ops = {
2404 SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2405 SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2406 arm_smmu_runtime_resume, NULL)
2407 };
2408
2409 static struct platform_driver arm_smmu_driver = {
2410 .driver = {
2411 .name = "arm-smmu",
2412 .of_match_table = of_match_ptr(arm_smmu_of_match),
2413 .pm = &arm_smmu_pm_ops,
2414 .suppress_bind_attrs = true,
2415 },
2416 .probe = arm_smmu_device_probe,
2417 .shutdown = arm_smmu_device_shutdown,
2418 };
2419 builtin_platform_driver(arm_smmu_driver);
Linux with added FPC-III support