?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge branch 'perf-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...
[cris-mirror.git] / drivers / iommu / arm-smmu.c
blob2409e3bd3df21e973db1231360b7a50fba66efa1
1 /*
2 * IOMMU API for ARM architected SMMU implementations.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
16 *
17 * Copyright (C) 2013 ARM Limited
18 *
19 * Author: Will Deacon <will.deacon@arm.com>
20 *
21 * This driver currently supports:
22 * - SMMUv1 and v2 implementations
23 * - Stream-matching and stream-indexing
24 * - v7/v8 long-descriptor format
25 * - Non-secure access to the SMMU
26 * - Context fault reporting
27 */
28
29 #define pr_fmt(fmt) "arm-smmu: " fmt
30
31 #include <linux/delay.h>
32 #include <linux/dma-iommu.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/err.h>
35 #include <linux/interrupt.h>
36 #include <linux/io.h>
37 #include <linux/iommu.h>
38 #include <linux/iopoll.h>
39 #include <linux/module.h>
40 #include <linux/of.h>
41 #include <linux/of_address.h>
42 #include <linux/pci.h>
43 #include <linux/platform_device.h>
44 #include <linux/slab.h>
45 #include <linux/spinlock.h>
46
47 #include <linux/amba/bus.h>
48
49 #include "io-pgtable.h"
50
51 /* Maximum number of stream IDs assigned to a single device */
52 #define MAX_MASTER_STREAMIDS MAX_PHANDLE_ARGS
53
54 /* Maximum number of context banks per SMMU */
55 #define ARM_SMMU_MAX_CBS 128
56
57 /* Maximum number of mapping groups per SMMU */
58 #define ARM_SMMU_MAX_SMRS 128
59
60 /* SMMU global address space */
61 #define ARM_SMMU_GR0(smmu) ((smmu)->base)
62 #define ARM_SMMU_GR1(smmu) ((smmu)->base + (1 << (smmu)->pgshift))
63
64 /*
65 * SMMU global address space with conditional offset to access secure
66 * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
67 * nsGFSYNR0: 0x450)
68 */
69 #define ARM_SMMU_GR0_NS(smmu) \
70 ((smmu)->base + \
71 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) \
72 ? 0x400 : 0))
73
74 #ifdef CONFIG_64BIT
75 #define smmu_writeq writeq_relaxed
76 #else
77 #define smmu_writeq(reg64, addr) \
78 do { \
79 u64 __val = (reg64); \
80 void __iomem *__addr = (addr); \
81 writel_relaxed(__val >> 32, __addr + 4); \
82 writel_relaxed(__val, __addr); \
83 } while (0)
84 #endif
85
86 /* Configuration registers */
87 #define ARM_SMMU_GR0_sCR0 0x0
88 #define sCR0_CLIENTPD (1 << 0)
89 #define sCR0_GFRE (1 << 1)
90 #define sCR0_GFIE (1 << 2)
91 #define sCR0_GCFGFRE (1 << 4)
92 #define sCR0_GCFGFIE (1 << 5)
93 #define sCR0_USFCFG (1 << 10)
94 #define sCR0_VMIDPNE (1 << 11)
95 #define sCR0_PTM (1 << 12)
96 #define sCR0_FB (1 << 13)
97 #define sCR0_BSU_SHIFT 14
98 #define sCR0_BSU_MASK 0x3
99
100 /* Identification registers */
101 #define ARM_SMMU_GR0_ID0 0x20
102 #define ARM_SMMU_GR0_ID1 0x24
103 #define ARM_SMMU_GR0_ID2 0x28
104 #define ARM_SMMU_GR0_ID3 0x2c
105 #define ARM_SMMU_GR0_ID4 0x30
106 #define ARM_SMMU_GR0_ID5 0x34
107 #define ARM_SMMU_GR0_ID6 0x38
108 #define ARM_SMMU_GR0_ID7 0x3c
109 #define ARM_SMMU_GR0_sGFSR 0x48
110 #define ARM_SMMU_GR0_sGFSYNR0 0x50
111 #define ARM_SMMU_GR0_sGFSYNR1 0x54
112 #define ARM_SMMU_GR0_sGFSYNR2 0x58
113
114 #define ID0_S1TS (1 << 30)
115 #define ID0_S2TS (1 << 29)
116 #define ID0_NTS (1 << 28)
117 #define ID0_SMS (1 << 27)
118 #define ID0_ATOSNS (1 << 26)
119 #define ID0_CTTW (1 << 14)
120 #define ID0_NUMIRPT_SHIFT 16
121 #define ID0_NUMIRPT_MASK 0xff
122 #define ID0_NUMSIDB_SHIFT 9
123 #define ID0_NUMSIDB_MASK 0xf
124 #define ID0_NUMSMRG_SHIFT 0
125 #define ID0_NUMSMRG_MASK 0xff
126
127 #define ID1_PAGESIZE (1 << 31)
128 #define ID1_NUMPAGENDXB_SHIFT 28
129 #define ID1_NUMPAGENDXB_MASK 7
130 #define ID1_NUMS2CB_SHIFT 16
131 #define ID1_NUMS2CB_MASK 0xff
132 #define ID1_NUMCB_SHIFT 0
133 #define ID1_NUMCB_MASK 0xff
134
135 #define ID2_OAS_SHIFT 4
136 #define ID2_OAS_MASK 0xf
137 #define ID2_IAS_SHIFT 0
138 #define ID2_IAS_MASK 0xf
139 #define ID2_UBS_SHIFT 8
140 #define ID2_UBS_MASK 0xf
141 #define ID2_PTFS_4K (1 << 12)
142 #define ID2_PTFS_16K (1 << 13)
143 #define ID2_PTFS_64K (1 << 14)
144
145 /* Global TLB invalidation */
146 #define ARM_SMMU_GR0_TLBIVMID 0x64
147 #define ARM_SMMU_GR0_TLBIALLNSNH 0x68
148 #define ARM_SMMU_GR0_TLBIALLH 0x6c
149 #define ARM_SMMU_GR0_sTLBGSYNC 0x70
150 #define ARM_SMMU_GR0_sTLBGSTATUS 0x74
151 #define sTLBGSTATUS_GSACTIVE (1 << 0)
152 #define TLB_LOOP_TIMEOUT 1000000 /* 1s! */
153
154 /* Stream mapping registers */
155 #define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2))
156 #define SMR_VALID (1 << 31)
157 #define SMR_MASK_SHIFT 16
158 #define SMR_MASK_MASK 0x7fff
159 #define SMR_ID_SHIFT 0
160 #define SMR_ID_MASK 0x7fff
161
162 #define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2))
163 #define S2CR_CBNDX_SHIFT 0
164 #define S2CR_CBNDX_MASK 0xff
165 #define S2CR_TYPE_SHIFT 16
166 #define S2CR_TYPE_MASK 0x3
167 #define S2CR_TYPE_TRANS (0 << S2CR_TYPE_SHIFT)
168 #define S2CR_TYPE_BYPASS (1 << S2CR_TYPE_SHIFT)
169 #define S2CR_TYPE_FAULT (2 << S2CR_TYPE_SHIFT)
170
171 #define S2CR_PRIVCFG_SHIFT 24
172 #define S2CR_PRIVCFG_UNPRIV (2 << S2CR_PRIVCFG_SHIFT)
173
174 /* Context bank attribute registers */
175 #define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2))
176 #define CBAR_VMID_SHIFT 0
177 #define CBAR_VMID_MASK 0xff
178 #define CBAR_S1_BPSHCFG_SHIFT 8
179 #define CBAR_S1_BPSHCFG_MASK 3
180 #define CBAR_S1_BPSHCFG_NSH 3
181 #define CBAR_S1_MEMATTR_SHIFT 12
182 #define CBAR_S1_MEMATTR_MASK 0xf
183 #define CBAR_S1_MEMATTR_WB 0xf
184 #define CBAR_TYPE_SHIFT 16
185 #define CBAR_TYPE_MASK 0x3
186 #define CBAR_TYPE_S2_TRANS (0 << CBAR_TYPE_SHIFT)
187 #define CBAR_TYPE_S1_TRANS_S2_BYPASS (1 << CBAR_TYPE_SHIFT)
188 #define CBAR_TYPE_S1_TRANS_S2_FAULT (2 << CBAR_TYPE_SHIFT)
189 #define CBAR_TYPE_S1_TRANS_S2_TRANS (3 << CBAR_TYPE_SHIFT)
190 #define CBAR_IRPTNDX_SHIFT 24
191 #define CBAR_IRPTNDX_MASK 0xff
192
193 #define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2))
194 #define CBA2R_RW64_32BIT (0 << 0)
195 #define CBA2R_RW64_64BIT (1 << 0)
196
197 /* Translation context bank */
198 #define ARM_SMMU_CB_BASE(smmu) ((smmu)->base + ((smmu)->size >> 1))
199 #define ARM_SMMU_CB(smmu, n) ((n) * (1 << (smmu)->pgshift))
200
201 #define ARM_SMMU_CB_SCTLR 0x0
202 #define ARM_SMMU_CB_RESUME 0x8
203 #define ARM_SMMU_CB_TTBCR2 0x10
204 #define ARM_SMMU_CB_TTBR0 0x20
205 #define ARM_SMMU_CB_TTBR1 0x28
206 #define ARM_SMMU_CB_TTBCR 0x30
207 #define ARM_SMMU_CB_S1_MAIR0 0x38
208 #define ARM_SMMU_CB_S1_MAIR1 0x3c
209 #define ARM_SMMU_CB_PAR_LO 0x50
210 #define ARM_SMMU_CB_PAR_HI 0x54
211 #define ARM_SMMU_CB_FSR 0x58
212 #define ARM_SMMU_CB_FAR_LO 0x60
213 #define ARM_SMMU_CB_FAR_HI 0x64
214 #define ARM_SMMU_CB_FSYNR0 0x68
215 #define ARM_SMMU_CB_S1_TLBIVA 0x600
216 #define ARM_SMMU_CB_S1_TLBIASID 0x610
217 #define ARM_SMMU_CB_S1_TLBIVAL 0x620
218 #define ARM_SMMU_CB_S2_TLBIIPAS2 0x630
219 #define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638
220 #define ARM_SMMU_CB_ATS1PR 0x800
221 #define ARM_SMMU_CB_ATSR 0x8f0
222
223 #define SCTLR_S1_ASIDPNE (1 << 12)
224 #define SCTLR_CFCFG (1 << 7)
225 #define SCTLR_CFIE (1 << 6)
226 #define SCTLR_CFRE (1 << 5)
227 #define SCTLR_E (1 << 4)
228 #define SCTLR_AFE (1 << 2)
229 #define SCTLR_TRE (1 << 1)
230 #define SCTLR_M (1 << 0)
231 #define SCTLR_EAE_SBOP (SCTLR_AFE | SCTLR_TRE)
232
233 #define CB_PAR_F (1 << 0)
234
235 #define ATSR_ACTIVE (1 << 0)
236
237 #define RESUME_RETRY (0 << 0)
238 #define RESUME_TERMINATE (1 << 0)
239
240 #define TTBCR2_SEP_SHIFT 15
241 #define TTBCR2_SEP_UPSTREAM (0x7 << TTBCR2_SEP_SHIFT)
242
243 #define TTBRn_ASID_SHIFT 48
244
245 #define FSR_MULTI (1 << 31)
246 #define FSR_SS (1 << 30)
247 #define FSR_UUT (1 << 8)
248 #define FSR_ASF (1 << 7)
249 #define FSR_TLBLKF (1 << 6)
250 #define FSR_TLBMCF (1 << 5)
251 #define FSR_EF (1 << 4)
252 #define FSR_PF (1 << 3)
253 #define FSR_AFF (1 << 2)
254 #define FSR_TF (1 << 1)
255
256 #define FSR_IGN (FSR_AFF | FSR_ASF | \
257 FSR_TLBMCF | FSR_TLBLKF)
258 #define FSR_FAULT (FSR_MULTI | FSR_SS | FSR_UUT | \
259 FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
260
261 #define FSYNR0_WNR (1 << 4)
262
263 static int force_stage;
264 module_param(force_stage, int, S_IRUGO);
265 MODULE_PARM_DESC(force_stage,
266 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
267 static bool disable_bypass;
268 module_param(disable_bypass, bool, S_IRUGO);
269 MODULE_PARM_DESC(disable_bypass,
270 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
271
272 enum arm_smmu_arch_version {
273 ARM_SMMU_V1 = 1,
274 ARM_SMMU_V2,
275 };
276
277 struct arm_smmu_smr {
278 u8 idx;
279 u16 mask;
280 u16 id;
281 };
282
283 struct arm_smmu_master_cfg {
284 int num_streamids;
285 u16 streamids[MAX_MASTER_STREAMIDS];
286 struct arm_smmu_smr *smrs;
287 };
288
289 struct arm_smmu_master {
290 struct device_node *of_node;
291 struct rb_node node;
292 struct arm_smmu_master_cfg cfg;
293 };
294
295 struct arm_smmu_device {
296 struct device *dev;
297
298 void __iomem *base;
299 unsigned long size;
300 unsigned long pgshift;
301
302 #define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0)
303 #define ARM_SMMU_FEAT_STREAM_MATCH (1 << 1)
304 #define ARM_SMMU_FEAT_TRANS_S1 (1 << 2)
305 #define ARM_SMMU_FEAT_TRANS_S2 (1 << 3)
306 #define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4)
307 #define ARM_SMMU_FEAT_TRANS_OPS (1 << 5)
308 u32 features;
309
310 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
311 u32 options;
312 enum arm_smmu_arch_version version;
313
314 u32 num_context_banks;
315 u32 num_s2_context_banks;
316 DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
317 atomic_t irptndx;
318
319 u32 num_mapping_groups;
320 DECLARE_BITMAP(smr_map, ARM_SMMU_MAX_SMRS);
321
322 unsigned long va_size;
323 unsigned long ipa_size;
324 unsigned long pa_size;
325
326 u32 num_global_irqs;
327 u32 num_context_irqs;
328 unsigned int *irqs;
329
330 struct list_head list;
331 struct rb_root masters;
332 };
333
334 struct arm_smmu_cfg {
335 u8 cbndx;
336 u8 irptndx;
337 u32 cbar;
338 };
339 #define INVALID_IRPTNDX 0xff
340
341 #define ARM_SMMU_CB_ASID(cfg) ((cfg)->cbndx)
342 #define ARM_SMMU_CB_VMID(cfg) ((cfg)->cbndx + 1)
343
344 enum arm_smmu_domain_stage {
345 ARM_SMMU_DOMAIN_S1 = 0,
346 ARM_SMMU_DOMAIN_S2,
347 ARM_SMMU_DOMAIN_NESTED,
348 };
349
350 struct arm_smmu_domain {
351 struct arm_smmu_device *smmu;
352 struct io_pgtable_ops *pgtbl_ops;
353 spinlock_t pgtbl_lock;
354 struct arm_smmu_cfg cfg;
355 enum arm_smmu_domain_stage stage;
356 struct mutex init_mutex; /* Protects smmu pointer */
357 struct iommu_domain domain;
358 };
359
360 static struct iommu_ops arm_smmu_ops;
361
362 static DEFINE_SPINLOCK(arm_smmu_devices_lock);
363 static LIST_HEAD(arm_smmu_devices);
364
365 struct arm_smmu_option_prop {
366 u32 opt;
367 const char *prop;
368 };
369
370 static struct arm_smmu_option_prop arm_smmu_options[] = {
371 { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
372 { 0, NULL},
373 };
374
375 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
376 {
377 return container_of(dom, struct arm_smmu_domain, domain);
378 }
379
380 static void parse_driver_options(struct arm_smmu_device *smmu)
381 {
382 int i = 0;
383
384 do {
385 if (of_property_read_bool(smmu->dev->of_node,
386 arm_smmu_options[i].prop)) {
387 smmu->options |= arm_smmu_options[i].opt;
388 dev_notice(smmu->dev, "option %s\n",
389 arm_smmu_options[i].prop);
390 }
391 } while (arm_smmu_options[++i].opt);
392 }
393
394 static struct device_node *dev_get_dev_node(struct device *dev)
395 {
396 if (dev_is_pci(dev)) {
397 struct pci_bus *bus = to_pci_dev(dev)->bus;
398
399 while (!pci_is_root_bus(bus))
400 bus = bus->parent;
401 return bus->bridge->parent->of_node;
402 }
403
404 return dev->of_node;
405 }
406
407 static struct arm_smmu_master *find_smmu_master(struct arm_smmu_device *smmu,
408 struct device_node *dev_node)
409 {
410 struct rb_node *node = smmu->masters.rb_node;
411
412 while (node) {
413 struct arm_smmu_master *master;
414
415 master = container_of(node, struct arm_smmu_master, node);
416
417 if (dev_node < master->of_node)
418 node = node->rb_left;
419 else if (dev_node > master->of_node)
420 node = node->rb_right;
421 else
422 return master;
423 }
424
425 return NULL;
426 }
427
428 static struct arm_smmu_master_cfg *
429 find_smmu_master_cfg(struct device *dev)
430 {
431 struct arm_smmu_master_cfg *cfg = NULL;
432 struct iommu_group *group = iommu_group_get(dev);
433
434 if (group) {
435 cfg = iommu_group_get_iommudata(group);
436 iommu_group_put(group);
437 }
438
439 return cfg;
440 }
441
442 static int insert_smmu_master(struct arm_smmu_device *smmu,
443 struct arm_smmu_master *master)
444 {
445 struct rb_node **new, *parent;
446
447 new = &smmu->masters.rb_node;
448 parent = NULL;
449 while (*new) {
450 struct arm_smmu_master *this
451 = container_of(*new, struct arm_smmu_master, node);
452
453 parent = *new;
454 if (master->of_node < this->of_node)
455 new = &((*new)->rb_left);
456 else if (master->of_node > this->of_node)
457 new = &((*new)->rb_right);
458 else
459 return -EEXIST;
460 }
461
462 rb_link_node(&master->node, parent, new);
463 rb_insert_color(&master->node, &smmu->masters);
464 return 0;
465 }
466
467 static int register_smmu_master(struct arm_smmu_device *smmu,
468 struct device *dev,
469 struct of_phandle_args *masterspec)
470 {
471 int i;
472 struct arm_smmu_master *master;
473
474 master = find_smmu_master(smmu, masterspec->np);
475 if (master) {
476 dev_err(dev,
477 "rejecting multiple registrations for master device %s\n",
478 masterspec->np->name);
479 return -EBUSY;
480 }
481
482 if (masterspec->args_count > MAX_MASTER_STREAMIDS) {
483 dev_err(dev,
484 "reached maximum number (%d) of stream IDs for master device %s\n",
485 MAX_MASTER_STREAMIDS, masterspec->np->name);
486 return -ENOSPC;
487 }
488
489 master = devm_kzalloc(dev, sizeof(*master), GFP_KERNEL);
490 if (!master)
491 return -ENOMEM;
492
493 master->of_node = masterspec->np;
494 master->cfg.num_streamids = masterspec->args_count;
495
496 for (i = 0; i < master->cfg.num_streamids; ++i) {
497 u16 streamid = masterspec->args[i];
498
499 if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) &&
500 (streamid >= smmu->num_mapping_groups)) {
501 dev_err(dev,
502 "stream ID for master device %s greater than maximum allowed (%d)\n",
503 masterspec->np->name, smmu->num_mapping_groups);
504 return -ERANGE;
505 }
506 master->cfg.streamids[i] = streamid;
507 }
508 return insert_smmu_master(smmu, master);
509 }
510
511 static struct arm_smmu_device *find_smmu_for_device(struct device *dev)
512 {
513 struct arm_smmu_device *smmu;
514 struct arm_smmu_master *master = NULL;
515 struct device_node *dev_node = dev_get_dev_node(dev);
516
517 spin_lock(&arm_smmu_devices_lock);
518 list_for_each_entry(smmu, &arm_smmu_devices, list) {
519 master = find_smmu_master(smmu, dev_node);
520 if (master)
521 break;
522 }
523 spin_unlock(&arm_smmu_devices_lock);
524
525 return master ? smmu : NULL;
526 }
527
528 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
529 {
530 int idx;
531
532 do {
533 idx = find_next_zero_bit(map, end, start);
534 if (idx == end)
535 return -ENOSPC;
536 } while (test_and_set_bit(idx, map));
537
538 return idx;
539 }
540
541 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
542 {
543 clear_bit(idx, map);
544 }
545
546 /* Wait for any pending TLB invalidations to complete */
547 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
548 {
549 int count = 0;
550 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
551
552 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_sTLBGSYNC);
553 while (readl_relaxed(gr0_base + ARM_SMMU_GR0_sTLBGSTATUS)
554 & sTLBGSTATUS_GSACTIVE) {
555 cpu_relax();
556 if (++count == TLB_LOOP_TIMEOUT) {
557 dev_err_ratelimited(smmu->dev,
558 "TLB sync timed out -- SMMU may be deadlocked\n");
559 return;
560 }
561 udelay(1);
562 }
563 }
564
565 static void arm_smmu_tlb_sync(void *cookie)
566 {
567 struct arm_smmu_domain *smmu_domain = cookie;
568 __arm_smmu_tlb_sync(smmu_domain->smmu);
569 }
570
571 static void arm_smmu_tlb_inv_context(void *cookie)
572 {
573 struct arm_smmu_domain *smmu_domain = cookie;
574 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
575 struct arm_smmu_device *smmu = smmu_domain->smmu;
576 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
577 void __iomem *base;
578
579 if (stage1) {
580 base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
581 writel_relaxed(ARM_SMMU_CB_ASID(cfg),
582 base + ARM_SMMU_CB_S1_TLBIASID);
583 } else {
584 base = ARM_SMMU_GR0(smmu);
585 writel_relaxed(ARM_SMMU_CB_VMID(cfg),
586 base + ARM_SMMU_GR0_TLBIVMID);
587 }
588
589 __arm_smmu_tlb_sync(smmu);
590 }
591
592 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
593 size_t granule, bool leaf, void *cookie)
594 {
595 struct arm_smmu_domain *smmu_domain = cookie;
596 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
597 struct arm_smmu_device *smmu = smmu_domain->smmu;
598 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
599 void __iomem *reg;
600
601 if (stage1) {
602 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
603 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
604
605 if (!IS_ENABLED(CONFIG_64BIT) || smmu->version == ARM_SMMU_V1) {
606 iova &= ~12UL;
607 iova |= ARM_SMMU_CB_ASID(cfg);
608 do {
609 writel_relaxed(iova, reg);
610 iova += granule;
611 } while (size -= granule);
612 #ifdef CONFIG_64BIT
613 } else {
614 iova >>= 12;
615 iova |= (u64)ARM_SMMU_CB_ASID(cfg) << 48;
616 do {
617 writeq_relaxed(iova, reg);
618 iova += granule >> 12;
619 } while (size -= granule);
620 #endif
621 }
622 #ifdef CONFIG_64BIT
623 } else if (smmu->version == ARM_SMMU_V2) {
624 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
625 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
626 ARM_SMMU_CB_S2_TLBIIPAS2;
627 iova >>= 12;
628 do {
629 writeq_relaxed(iova, reg);
630 iova += granule >> 12;
631 } while (size -= granule);
632 #endif
633 } else {
634 reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;
635 writel_relaxed(ARM_SMMU_CB_VMID(cfg), reg);
636 }
637 }
638
639 static struct iommu_gather_ops arm_smmu_gather_ops = {
640 .tlb_flush_all = arm_smmu_tlb_inv_context,
641 .tlb_add_flush = arm_smmu_tlb_inv_range_nosync,
642 .tlb_sync = arm_smmu_tlb_sync,
643 };
644
645 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
646 {
647 int flags, ret;
648 u32 fsr, far, fsynr, resume;
649 unsigned long iova;
650 struct iommu_domain *domain = dev;
651 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
652 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
653 struct arm_smmu_device *smmu = smmu_domain->smmu;
654 void __iomem *cb_base;
655
656 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
657 fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
658
659 if (!(fsr & FSR_FAULT))
660 return IRQ_NONE;
661
662 if (fsr & FSR_IGN)
663 dev_err_ratelimited(smmu->dev,
664 "Unexpected context fault (fsr 0x%x)\n",
665 fsr);
666
667 fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
668 flags = fsynr & FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ;
669
670 far = readl_relaxed(cb_base + ARM_SMMU_CB_FAR_LO);
671 iova = far;
672 #ifdef CONFIG_64BIT
673 far = readl_relaxed(cb_base + ARM_SMMU_CB_FAR_HI);
674 iova |= ((unsigned long)far << 32);
675 #endif
676
677 if (!report_iommu_fault(domain, smmu->dev, iova, flags)) {
678 ret = IRQ_HANDLED;
679 resume = RESUME_RETRY;
680 } else {
681 dev_err_ratelimited(smmu->dev,
682 "Unhandled context fault: iova=0x%08lx, fsynr=0x%x, cb=%d\n",
683 iova, fsynr, cfg->cbndx);
684 ret = IRQ_NONE;
685 resume = RESUME_TERMINATE;
686 }
687
688 /* Clear the faulting FSR */
689 writel(fsr, cb_base + ARM_SMMU_CB_FSR);
690
691 /* Retry or terminate any stalled transactions */
692 if (fsr & FSR_SS)
693 writel_relaxed(resume, cb_base + ARM_SMMU_CB_RESUME);
694
695 return ret;
696 }
697
698 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
699 {
700 u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
701 struct arm_smmu_device *smmu = dev;
702 void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
703
704 gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
705 gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
706 gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
707 gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
708
709 if (!gfsr)
710 return IRQ_NONE;
711
712 dev_err_ratelimited(smmu->dev,
713 "Unexpected global fault, this could be serious\n");
714 dev_err_ratelimited(smmu->dev,
715 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
716 gfsr, gfsynr0, gfsynr1, gfsynr2);
717
718 writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
719 return IRQ_HANDLED;
720 }
721
722 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
723 struct io_pgtable_cfg *pgtbl_cfg)
724 {
725 u32 reg;
726 u64 reg64;
727 bool stage1;
728 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
729 struct arm_smmu_device *smmu = smmu_domain->smmu;
730 void __iomem *cb_base, *gr1_base;
731
732 gr1_base = ARM_SMMU_GR1(smmu);
733 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
734 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
735
736 if (smmu->version > ARM_SMMU_V1) {
737 /*
738 * CBA2R.
739 * *Must* be initialised before CBAR thanks to VMID16
740 * architectural oversight affected some implementations.
741 */
742 #ifdef CONFIG_64BIT
743 reg = CBA2R_RW64_64BIT;
744 #else
745 reg = CBA2R_RW64_32BIT;
746 #endif
747 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
748 }
749
750 /* CBAR */
751 reg = cfg->cbar;
752 if (smmu->version == ARM_SMMU_V1)
753 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
754
755 /*
756 * Use the weakest shareability/memory types, so they are
757 * overridden by the ttbcr/pte.
758 */
759 if (stage1) {
760 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
761 (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
762 } else {
763 reg |= ARM_SMMU_CB_VMID(cfg) << CBAR_VMID_SHIFT;
764 }
765 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
766
767 /* TTBRs */
768 if (stage1) {
769 reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
770
771 reg64 |= ((u64)ARM_SMMU_CB_ASID(cfg)) << TTBRn_ASID_SHIFT;
772 smmu_writeq(reg64, cb_base + ARM_SMMU_CB_TTBR0);
773
774 reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
775 reg64 |= ((u64)ARM_SMMU_CB_ASID(cfg)) << TTBRn_ASID_SHIFT;
776 smmu_writeq(reg64, cb_base + ARM_SMMU_CB_TTBR1);
777 } else {
778 reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
779 smmu_writeq(reg64, cb_base + ARM_SMMU_CB_TTBR0);
780 }
781
782 /* TTBCR */
783 if (stage1) {
784 reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
785 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
786 if (smmu->version > ARM_SMMU_V1) {
787 reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
788 reg |= TTBCR2_SEP_UPSTREAM;
789 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR2);
790 }
791 } else {
792 reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
793 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
794 }
795
796 /* MAIRs (stage-1 only) */
797 if (stage1) {
798 reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
799 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
800 reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
801 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR1);
802 }
803
804 /* SCTLR */
805 reg = SCTLR_CFCFG | SCTLR_CFIE | SCTLR_CFRE | SCTLR_M | SCTLR_EAE_SBOP;
806 if (stage1)
807 reg |= SCTLR_S1_ASIDPNE;
808 #ifdef __BIG_ENDIAN
809 reg |= SCTLR_E;
810 #endif
811 writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
812 }
813
814 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
815 struct arm_smmu_device *smmu)
816 {
817 int irq, start, ret = 0;
818 unsigned long ias, oas;
819 struct io_pgtable_ops *pgtbl_ops;
820 struct io_pgtable_cfg pgtbl_cfg;
821 enum io_pgtable_fmt fmt;
822 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
823 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
824
825 mutex_lock(&smmu_domain->init_mutex);
826 if (smmu_domain->smmu)
827 goto out_unlock;
828
829 /*
830 * Mapping the requested stage onto what we support is surprisingly
831 * complicated, mainly because the spec allows S1+S2 SMMUs without
832 * support for nested translation. That means we end up with the
833 * following table:
834 *
835 * Requested Supported Actual
836 * S1 N S1
837 * S1 S1+S2 S1
838 * S1 S2 S2
839 * S1 S1 S1
840 * N N N
841 * N S1+S2 S2
842 * N S2 S2
843 * N S1 S1
844 *
845 * Note that you can't actually request stage-2 mappings.
846 */
847 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
848 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
849 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
850 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
851
852 switch (smmu_domain->stage) {
853 case ARM_SMMU_DOMAIN_S1:
854 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
855 start = smmu->num_s2_context_banks;
856 ias = smmu->va_size;
857 oas = smmu->ipa_size;
858 if (IS_ENABLED(CONFIG_64BIT))
859 fmt = ARM_64_LPAE_S1;
860 else
861 fmt = ARM_32_LPAE_S1;
862 break;
863 case ARM_SMMU_DOMAIN_NESTED:
864 /*
865 * We will likely want to change this if/when KVM gets
866 * involved.
867 */
868 case ARM_SMMU_DOMAIN_S2:
869 cfg->cbar = CBAR_TYPE_S2_TRANS;
870 start = 0;
871 ias = smmu->ipa_size;
872 oas = smmu->pa_size;
873 if (IS_ENABLED(CONFIG_64BIT))
874 fmt = ARM_64_LPAE_S2;
875 else
876 fmt = ARM_32_LPAE_S2;
877 break;
878 default:
879 ret = -EINVAL;
880 goto out_unlock;
881 }
882
883 ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
884 smmu->num_context_banks);
885 if (IS_ERR_VALUE(ret))
886 goto out_unlock;
887
888 cfg->cbndx = ret;
889 if (smmu->version == ARM_SMMU_V1) {
890 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
891 cfg->irptndx %= smmu->num_context_irqs;
892 } else {
893 cfg->irptndx = cfg->cbndx;
894 }
895
896 pgtbl_cfg = (struct io_pgtable_cfg) {
897 .pgsize_bitmap = arm_smmu_ops.pgsize_bitmap,
898 .ias = ias,
899 .oas = oas,
900 .tlb = &arm_smmu_gather_ops,
901 .iommu_dev = smmu->dev,
902 };
903
904 smmu_domain->smmu = smmu;
905 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
906 if (!pgtbl_ops) {
907 ret = -ENOMEM;
908 goto out_clear_smmu;
909 }
910
911 /* Update our support page sizes to reflect the page table format */
912 arm_smmu_ops.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
913
914 /* Initialise the context bank with our page table cfg */
915 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
916
917 /*
918 * Request context fault interrupt. Do this last to avoid the
919 * handler seeing a half-initialised domain state.
920 */
921 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
922 ret = request_irq(irq, arm_smmu_context_fault, IRQF_SHARED,
923 "arm-smmu-context-fault", domain);
924 if (IS_ERR_VALUE(ret)) {
925 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
926 cfg->irptndx, irq);
927 cfg->irptndx = INVALID_IRPTNDX;
928 }
929
930 mutex_unlock(&smmu_domain->init_mutex);
931
932 /* Publish page table ops for map/unmap */
933 smmu_domain->pgtbl_ops = pgtbl_ops;
934 return 0;
935
936 out_clear_smmu:
937 smmu_domain->smmu = NULL;
938 out_unlock:
939 mutex_unlock(&smmu_domain->init_mutex);
940 return ret;
941 }
942
943 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
944 {
945 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
946 struct arm_smmu_device *smmu = smmu_domain->smmu;
947 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
948 void __iomem *cb_base;
949 int irq;
950
951 if (!smmu)
952 return;
953
954 /*
955 * Disable the context bank and free the page tables before freeing
956 * it.
957 */
958 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
959 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
960
961 if (cfg->irptndx != INVALID_IRPTNDX) {
962 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
963 free_irq(irq, domain);
964 }
965
966 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
967 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
968 }
969
970 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
971 {
972 struct arm_smmu_domain *smmu_domain;
973
974 if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
975 return NULL;
976 /*
977 * Allocate the domain and initialise some of its data structures.
978 * We can't really do anything meaningful until we've added a
979 * master.
980 */
981 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
982 if (!smmu_domain)
983 return NULL;
984
985 if (type == IOMMU_DOMAIN_DMA &&
986 iommu_get_dma_cookie(&smmu_domain->domain)) {
987 kfree(smmu_domain);
988 return NULL;
989 }
990
991 mutex_init(&smmu_domain->init_mutex);
992 spin_lock_init(&smmu_domain->pgtbl_lock);
993
994 return &smmu_domain->domain;
995 }
996
997 static void arm_smmu_domain_free(struct iommu_domain *domain)
998 {
999 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1000
1001 /*
1002 * Free the domain resources. We assume that all devices have
1003 * already been detached.
1004 */
1005 iommu_put_dma_cookie(domain);
1006 arm_smmu_destroy_domain_context(domain);
1007 kfree(smmu_domain);
1008 }
1009
1010 static int arm_smmu_master_configure_smrs(struct arm_smmu_device *smmu,
1011 struct arm_smmu_master_cfg *cfg)
1012 {
1013 int i;
1014 struct arm_smmu_smr *smrs;
1015 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1016
1017 if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH))
1018 return 0;
1019
1020 if (cfg->smrs)
1021 return -EEXIST;
1022
1023 smrs = kmalloc_array(cfg->num_streamids, sizeof(*smrs), GFP_KERNEL);
1024 if (!smrs) {
1025 dev_err(smmu->dev, "failed to allocate %d SMRs\n",
1026 cfg->num_streamids);
1027 return -ENOMEM;
1028 }
1029
1030 /* Allocate the SMRs on the SMMU */
1031 for (i = 0; i < cfg->num_streamids; ++i) {
1032 int idx = __arm_smmu_alloc_bitmap(smmu->smr_map, 0,
1033 smmu->num_mapping_groups);
1034 if (IS_ERR_VALUE(idx)) {
1035 dev_err(smmu->dev, "failed to allocate free SMR\n");
1036 goto err_free_smrs;
1037 }
1038
1039 smrs[i] = (struct arm_smmu_smr) {
1040 .idx = idx,
1041 .mask = 0, /* We don't currently share SMRs */
1042 .id = cfg->streamids[i],
1043 };
1044 }
1045
1046 /* It worked! Now, poke the actual hardware */
1047 for (i = 0; i < cfg->num_streamids; ++i) {
1048 u32 reg = SMR_VALID | smrs[i].id << SMR_ID_SHIFT |
1049 smrs[i].mask << SMR_MASK_SHIFT;
1050 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_SMR(smrs[i].idx));
1051 }
1052
1053 cfg->smrs = smrs;
1054 return 0;
1055
1056 err_free_smrs:
1057 while (--i >= 0)
1058 __arm_smmu_free_bitmap(smmu->smr_map, smrs[i].idx);
1059 kfree(smrs);
1060 return -ENOSPC;
1061 }
1062
1063 static void arm_smmu_master_free_smrs(struct arm_smmu_device *smmu,
1064 struct arm_smmu_master_cfg *cfg)
1065 {
1066 int i;
1067 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1068 struct arm_smmu_smr *smrs = cfg->smrs;
1069
1070 if (!smrs)
1071 return;
1072
1073 /* Invalidate the SMRs before freeing back to the allocator */
1074 for (i = 0; i < cfg->num_streamids; ++i) {
1075 u8 idx = smrs[i].idx;
1076
1077 writel_relaxed(~SMR_VALID, gr0_base + ARM_SMMU_GR0_SMR(idx));
1078 __arm_smmu_free_bitmap(smmu->smr_map, idx);
1079 }
1080
1081 cfg->smrs = NULL;
1082 kfree(smrs);
1083 }
1084
1085 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1086 struct arm_smmu_master_cfg *cfg)
1087 {
1088 int i, ret;
1089 struct arm_smmu_device *smmu = smmu_domain->smmu;
1090 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1091
1092 /* Devices in an IOMMU group may already be configured */
1093 ret = arm_smmu_master_configure_smrs(smmu, cfg);
1094 if (ret)
1095 return ret == -EEXIST ? 0 : ret;
1096
1097 /*
1098 * FIXME: This won't be needed once we have IOMMU-backed DMA ops
1099 * for all devices behind the SMMU.
1100 */
1101 if (smmu_domain->domain.type == IOMMU_DOMAIN_DMA)
1102 return 0;
1103
1104 for (i = 0; i < cfg->num_streamids; ++i) {
1105 u32 idx, s2cr;
1106
1107 idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i];
1108 s2cr = S2CR_TYPE_TRANS | S2CR_PRIVCFG_UNPRIV |
1109 (smmu_domain->cfg.cbndx << S2CR_CBNDX_SHIFT);
1110 writel_relaxed(s2cr, gr0_base + ARM_SMMU_GR0_S2CR(idx));
1111 }
1112
1113 return 0;
1114 }
1115
1116 static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain,
1117 struct arm_smmu_master_cfg *cfg)
1118 {
1119 int i;
1120 struct arm_smmu_device *smmu = smmu_domain->smmu;
1121 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1122
1123 /* An IOMMU group is torn down by the first device to be removed */
1124 if ((smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) && !cfg->smrs)
1125 return;
1126
1127 /*
1128 * We *must* clear the S2CR first, because freeing the SMR means
1129 * that it can be re-allocated immediately.
1130 */
1131 for (i = 0; i < cfg->num_streamids; ++i) {
1132 u32 idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i];
1133 u32 reg = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS;
1134
1135 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_S2CR(idx));
1136 }
1137
1138 arm_smmu_master_free_smrs(smmu, cfg);
1139 }
1140
1141 static void arm_smmu_detach_dev(struct device *dev,
1142 struct arm_smmu_master_cfg *cfg)
1143 {
1144 struct iommu_domain *domain = dev->archdata.iommu;
1145 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1146
1147 dev->archdata.iommu = NULL;
1148 arm_smmu_domain_remove_master(smmu_domain, cfg);
1149 }
1150
1151 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1152 {
1153 int ret;
1154 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1155 struct arm_smmu_device *smmu;
1156 struct arm_smmu_master_cfg *cfg;
1157
1158 smmu = find_smmu_for_device(dev);
1159 if (!smmu) {
1160 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1161 return -ENXIO;
1162 }
1163
1164 /* Ensure that the domain is finalised */
1165 ret = arm_smmu_init_domain_context(domain, smmu);
1166 if (IS_ERR_VALUE(ret))
1167 return ret;
1168
1169 /*
1170 * Sanity check the domain. We don't support domains across
1171 * different SMMUs.
1172 */
1173 if (smmu_domain->smmu != smmu) {
1174 dev_err(dev,
1175 "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1176 dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1177 return -EINVAL;
1178 }
1179
1180 /* Looks ok, so add the device to the domain */
1181 cfg = find_smmu_master_cfg(dev);
1182 if (!cfg)
1183 return -ENODEV;
1184
1185 /* Detach the dev from its current domain */
1186 if (dev->archdata.iommu)
1187 arm_smmu_detach_dev(dev, cfg);
1188
1189 ret = arm_smmu_domain_add_master(smmu_domain, cfg);
1190 if (!ret)
1191 dev->archdata.iommu = domain;
1192 return ret;
1193 }
1194
1195 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1196 phys_addr_t paddr, size_t size, int prot)
1197 {
1198 int ret;
1199 unsigned long flags;
1200 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1201 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1202
1203 if (!ops)
1204 return -ENODEV;
1205
1206 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1207 ret = ops->map(ops, iova, paddr, size, prot);
1208 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1209 return ret;
1210 }
1211
1212 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1213 size_t size)
1214 {
1215 size_t ret;
1216 unsigned long flags;
1217 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1218 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1219
1220 if (!ops)
1221 return 0;
1222
1223 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1224 ret = ops->unmap(ops, iova, size);
1225 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1226 return ret;
1227 }
1228
1229 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1230 dma_addr_t iova)
1231 {
1232 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1233 struct arm_smmu_device *smmu = smmu_domain->smmu;
1234 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1235 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1236 struct device *dev = smmu->dev;
1237 void __iomem *cb_base;
1238 u32 tmp;
1239 u64 phys;
1240 unsigned long va;
1241
1242 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1243
1244 /* ATS1 registers can only be written atomically */
1245 va = iova & ~0xfffUL;
1246 if (smmu->version == ARM_SMMU_V2)
1247 smmu_writeq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1248 else
1249 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1250
1251 if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1252 !(tmp & ATSR_ACTIVE), 5, 50)) {
1253 dev_err(dev,
1254 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1255 &iova);
1256 return ops->iova_to_phys(ops, iova);
1257 }
1258
1259 phys = readl_relaxed(cb_base + ARM_SMMU_CB_PAR_LO);
1260 phys |= ((u64)readl_relaxed(cb_base + ARM_SMMU_CB_PAR_HI)) << 32;
1261
1262 if (phys & CB_PAR_F) {
1263 dev_err(dev, "translation fault!\n");
1264 dev_err(dev, "PAR = 0x%llx\n", phys);
1265 return 0;
1266 }
1267
1268 return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1269 }
1270
1271 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1272 dma_addr_t iova)
1273 {
1274 phys_addr_t ret;
1275 unsigned long flags;
1276 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1277 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1278
1279 if (!ops)
1280 return 0;
1281
1282 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1283 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1284 smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1285 ret = arm_smmu_iova_to_phys_hard(domain, iova);
1286 } else {
1287 ret = ops->iova_to_phys(ops, iova);
1288 }
1289
1290 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1291
1292 return ret;
1293 }
1294
1295 static bool arm_smmu_capable(enum iommu_cap cap)
1296 {
1297 switch (cap) {
1298 case IOMMU_CAP_CACHE_COHERENCY:
1299 /*
1300 * Return true here as the SMMU can always send out coherent
1301 * requests.
1302 */
1303 return true;
1304 case IOMMU_CAP_INTR_REMAP:
1305 return true; /* MSIs are just memory writes */
1306 case IOMMU_CAP_NOEXEC:
1307 return true;
1308 default:
1309 return false;
1310 }
1311 }
1312
1313 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
1314 {
1315 *((u16 *)data) = alias;
1316 return 0; /* Continue walking */
1317 }
1318
1319 static void __arm_smmu_release_pci_iommudata(void *data)
1320 {
1321 kfree(data);
1322 }
1323
1324 static int arm_smmu_init_pci_device(struct pci_dev *pdev,
1325 struct iommu_group *group)
1326 {
1327 struct arm_smmu_master_cfg *cfg;
1328 u16 sid;
1329 int i;
1330
1331 cfg = iommu_group_get_iommudata(group);
1332 if (!cfg) {
1333 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
1334 if (!cfg)
1335 return -ENOMEM;
1336
1337 iommu_group_set_iommudata(group, cfg,
1338 __arm_smmu_release_pci_iommudata);
1339 }
1340
1341 if (cfg->num_streamids >= MAX_MASTER_STREAMIDS)
1342 return -ENOSPC;
1343
1344 /*
1345 * Assume Stream ID == Requester ID for now.
1346 * We need a way to describe the ID mappings in FDT.
1347 */
1348 pci_for_each_dma_alias(pdev, __arm_smmu_get_pci_sid, &sid);
1349 for (i = 0; i < cfg->num_streamids; ++i)
1350 if (cfg->streamids[i] == sid)
1351 break;
1352
1353 /* Avoid duplicate SIDs, as this can lead to SMR conflicts */
1354 if (i == cfg->num_streamids)
1355 cfg->streamids[cfg->num_streamids++] = sid;
1356
1357 return 0;
1358 }
1359
1360 static int arm_smmu_init_platform_device(struct device *dev,
1361 struct iommu_group *group)
1362 {
1363 struct arm_smmu_device *smmu = find_smmu_for_device(dev);
1364 struct arm_smmu_master *master;
1365
1366 if (!smmu)
1367 return -ENODEV;
1368
1369 master = find_smmu_master(smmu, dev->of_node);
1370 if (!master)
1371 return -ENODEV;
1372
1373 iommu_group_set_iommudata(group, &master->cfg, NULL);
1374
1375 return 0;
1376 }
1377
1378 static int arm_smmu_add_device(struct device *dev)
1379 {
1380 struct iommu_group *group;
1381
1382 group = iommu_group_get_for_dev(dev);
1383 if (IS_ERR(group))
1384 return PTR_ERR(group);
1385
1386 iommu_group_put(group);
1387 return 0;
1388 }
1389
1390 static void arm_smmu_remove_device(struct device *dev)
1391 {
1392 iommu_group_remove_device(dev);
1393 }
1394
1395 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1396 {
1397 struct iommu_group *group;
1398 int ret;
1399
1400 if (dev_is_pci(dev))
1401 group = pci_device_group(dev);
1402 else
1403 group = generic_device_group(dev);
1404
1405 if (IS_ERR(group))
1406 return group;
1407
1408 if (dev_is_pci(dev))
1409 ret = arm_smmu_init_pci_device(to_pci_dev(dev), group);
1410 else
1411 ret = arm_smmu_init_platform_device(dev, group);
1412
1413 if (ret) {
1414 iommu_group_put(group);
1415 group = ERR_PTR(ret);
1416 }
1417
1418 return group;
1419 }
1420
1421 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1422 enum iommu_attr attr, void *data)
1423 {
1424 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1425
1426 switch (attr) {
1427 case DOMAIN_ATTR_NESTING:
1428 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1429 return 0;
1430 default:
1431 return -ENODEV;
1432 }
1433 }
1434
1435 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1436 enum iommu_attr attr, void *data)
1437 {
1438 int ret = 0;
1439 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1440
1441 mutex_lock(&smmu_domain->init_mutex);
1442
1443 switch (attr) {
1444 case DOMAIN_ATTR_NESTING:
1445 if (smmu_domain->smmu) {
1446 ret = -EPERM;
1447 goto out_unlock;
1448 }
1449
1450 if (*(int *)data)
1451 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1452 else
1453 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1454
1455 break;
1456 default:
1457 ret = -ENODEV;
1458 }
1459
1460 out_unlock:
1461 mutex_unlock(&smmu_domain->init_mutex);
1462 return ret;
1463 }
1464
1465 static struct iommu_ops arm_smmu_ops = {
1466 .capable = arm_smmu_capable,
1467 .domain_alloc = arm_smmu_domain_alloc,
1468 .domain_free = arm_smmu_domain_free,
1469 .attach_dev = arm_smmu_attach_dev,
1470 .map = arm_smmu_map,
1471 .unmap = arm_smmu_unmap,
1472 .map_sg = default_iommu_map_sg,
1473 .iova_to_phys = arm_smmu_iova_to_phys,
1474 .add_device = arm_smmu_add_device,
1475 .remove_device = arm_smmu_remove_device,
1476 .device_group = arm_smmu_device_group,
1477 .domain_get_attr = arm_smmu_domain_get_attr,
1478 .domain_set_attr = arm_smmu_domain_set_attr,
1479 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1480 };
1481
1482 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1483 {
1484 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1485 void __iomem *cb_base;
1486 int i = 0;
1487 u32 reg;
1488
1489 /* clear global FSR */
1490 reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1491 writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1492
1493 /* Mark all SMRn as invalid and all S2CRn as bypass unless overridden */
1494 reg = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS;
1495 for (i = 0; i < smmu->num_mapping_groups; ++i) {
1496 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_SMR(i));
1497 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_S2CR(i));
1498 }
1499
1500 /* Make sure all context banks are disabled and clear CB_FSR */
1501 for (i = 0; i < smmu->num_context_banks; ++i) {
1502 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);
1503 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1504 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1505 }
1506
1507 /* Invalidate the TLB, just in case */
1508 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1509 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1510
1511 reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1512
1513 /* Enable fault reporting */
1514 reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1515
1516 /* Disable TLB broadcasting. */
1517 reg |= (sCR0_VMIDPNE | sCR0_PTM);
1518
1519 /* Enable client access, handling unmatched streams as appropriate */
1520 reg &= ~sCR0_CLIENTPD;
1521 if (disable_bypass)
1522 reg |= sCR0_USFCFG;
1523 else
1524 reg &= ~sCR0_USFCFG;
1525
1526 /* Disable forced broadcasting */
1527 reg &= ~sCR0_FB;
1528
1529 /* Don't upgrade barriers */
1530 reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
1531
1532 /* Push the button */
1533 __arm_smmu_tlb_sync(smmu);
1534 writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1535 }
1536
1537 static int arm_smmu_id_size_to_bits(int size)
1538 {
1539 switch (size) {
1540 case 0:
1541 return 32;
1542 case 1:
1543 return 36;
1544 case 2:
1545 return 40;
1546 case 3:
1547 return 42;
1548 case 4:
1549 return 44;
1550 case 5:
1551 default:
1552 return 48;
1553 }
1554 }
1555
1556 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1557 {
1558 unsigned long size;
1559 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1560 u32 id;
1561 bool cttw_dt, cttw_reg;
1562
1563 dev_notice(smmu->dev, "probing hardware configuration...\n");
1564 dev_notice(smmu->dev, "SMMUv%d with:\n", smmu->version);
1565
1566 /* ID0 */
1567 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1568
1569 /* Restrict available stages based on module parameter */
1570 if (force_stage == 1)
1571 id &= ~(ID0_S2TS | ID0_NTS);
1572 else if (force_stage == 2)
1573 id &= ~(ID0_S1TS | ID0_NTS);
1574
1575 if (id & ID0_S1TS) {
1576 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1577 dev_notice(smmu->dev, "\tstage 1 translation\n");
1578 }
1579
1580 if (id & ID0_S2TS) {
1581 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1582 dev_notice(smmu->dev, "\tstage 2 translation\n");
1583 }
1584
1585 if (id & ID0_NTS) {
1586 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1587 dev_notice(smmu->dev, "\tnested translation\n");
1588 }
1589
1590 if (!(smmu->features &
1591 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1592 dev_err(smmu->dev, "\tno translation support!\n");
1593 return -ENODEV;
1594 }
1595
1596 if ((id & ID0_S1TS) && ((smmu->version == 1) || !(id & ID0_ATOSNS))) {
1597 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1598 dev_notice(smmu->dev, "\taddress translation ops\n");
1599 }
1600
1601 /*
1602 * In order for DMA API calls to work properly, we must defer to what
1603 * the DT says about coherency, regardless of what the hardware claims.
1604 * Fortunately, this also opens up a workaround for systems where the
1605 * ID register value has ended up configured incorrectly.
1606 */
1607 cttw_dt = of_dma_is_coherent(smmu->dev->of_node);
1608 cttw_reg = !!(id & ID0_CTTW);
1609 if (cttw_dt)
1610 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1611 if (cttw_dt || cttw_reg)
1612 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1613 cttw_dt ? "" : "non-");
1614 if (cttw_dt != cttw_reg)
1615 dev_notice(smmu->dev,
1616 "\t(IDR0.CTTW overridden by dma-coherent property)\n");
1617
1618 if (id & ID0_SMS) {
1619 u32 smr, sid, mask;
1620
1621 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1622 smmu->num_mapping_groups = (id >> ID0_NUMSMRG_SHIFT) &
1623 ID0_NUMSMRG_MASK;
1624 if (smmu->num_mapping_groups == 0) {
1625 dev_err(smmu->dev,
1626 "stream-matching supported, but no SMRs present!\n");
1627 return -ENODEV;
1628 }
1629
1630 smr = SMR_MASK_MASK << SMR_MASK_SHIFT;
1631 smr |= (SMR_ID_MASK << SMR_ID_SHIFT);
1632 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1633 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1634
1635 mask = (smr >> SMR_MASK_SHIFT) & SMR_MASK_MASK;
1636 sid = (smr >> SMR_ID_SHIFT) & SMR_ID_MASK;
1637 if ((mask & sid) != sid) {
1638 dev_err(smmu->dev,
1639 "SMR mask bits (0x%x) insufficient for ID field (0x%x)\n",
1640 mask, sid);
1641 return -ENODEV;
1642 }
1643
1644 dev_notice(smmu->dev,
1645 "\tstream matching with %u register groups, mask 0x%x",
1646 smmu->num_mapping_groups, mask);
1647 } else {
1648 smmu->num_mapping_groups = (id >> ID0_NUMSIDB_SHIFT) &
1649 ID0_NUMSIDB_MASK;
1650 }
1651
1652 /* ID1 */
1653 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1654 smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1655
1656 /* Check for size mismatch of SMMU address space from mapped region */
1657 size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1658 size *= 2 << smmu->pgshift;
1659 if (smmu->size != size)
1660 dev_warn(smmu->dev,
1661 "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
1662 size, smmu->size);
1663
1664 smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
1665 smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1666 if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1667 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1668 return -ENODEV;
1669 }
1670 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1671 smmu->num_context_banks, smmu->num_s2_context_banks);
1672
1673 /* ID2 */
1674 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1675 size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1676 smmu->ipa_size = size;
1677
1678 /* The output mask is also applied for bypass */
1679 size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1680 smmu->pa_size = size;
1681
1682 /*
1683 * What the page table walker can address actually depends on which
1684 * descriptor format is in use, but since a) we don't know that yet,
1685 * and b) it can vary per context bank, this will have to do...
1686 */
1687 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1688 dev_warn(smmu->dev,
1689 "failed to set DMA mask for table walker\n");
1690
1691 if (smmu->version == ARM_SMMU_V1) {
1692 smmu->va_size = smmu->ipa_size;
1693 size = SZ_4K | SZ_2M | SZ_1G;
1694 } else {
1695 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
1696 smmu->va_size = arm_smmu_id_size_to_bits(size);
1697 #ifndef CONFIG_64BIT
1698 smmu->va_size = min(32UL, smmu->va_size);
1699 #endif
1700 size = 0;
1701 if (id & ID2_PTFS_4K)
1702 size |= SZ_4K | SZ_2M | SZ_1G;
1703 if (id & ID2_PTFS_16K)
1704 size |= SZ_16K | SZ_32M;
1705 if (id & ID2_PTFS_64K)
1706 size |= SZ_64K | SZ_512M;
1707 }
1708
1709 arm_smmu_ops.pgsize_bitmap &= size;
1710 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n", size);
1711
1712 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1713 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1714 smmu->va_size, smmu->ipa_size);
1715
1716 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1717 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1718 smmu->ipa_size, smmu->pa_size);
1719
1720 return 0;
1721 }
1722
1723 static const struct of_device_id arm_smmu_of_match[] = {
1724 { .compatible = "arm,smmu-v1", .data = (void *)ARM_SMMU_V1 },
1725 { .compatible = "arm,smmu-v2", .data = (void *)ARM_SMMU_V2 },
1726 { .compatible = "arm,mmu-400", .data = (void *)ARM_SMMU_V1 },
1727 { .compatible = "arm,mmu-401", .data = (void *)ARM_SMMU_V1 },
1728 { .compatible = "arm,mmu-500", .data = (void *)ARM_SMMU_V2 },
1729 { },
1730 };
1731 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1732
1733 static int arm_smmu_device_dt_probe(struct platform_device *pdev)
1734 {
1735 const struct of_device_id *of_id;
1736 struct resource *res;
1737 struct arm_smmu_device *smmu;
1738 struct device *dev = &pdev->dev;
1739 struct rb_node *node;
1740 struct of_phandle_args masterspec;
1741 int num_irqs, i, err;
1742
1743 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
1744 if (!smmu) {
1745 dev_err(dev, "failed to allocate arm_smmu_device\n");
1746 return -ENOMEM;
1747 }
1748 smmu->dev = dev;
1749
1750 of_id = of_match_node(arm_smmu_of_match, dev->of_node);
1751 smmu->version = (enum arm_smmu_arch_version)of_id->data;
1752
1753 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1754 smmu->base = devm_ioremap_resource(dev, res);
1755 if (IS_ERR(smmu->base))
1756 return PTR_ERR(smmu->base);
1757 smmu->size = resource_size(res);
1758
1759 if (of_property_read_u32(dev->of_node, "#global-interrupts",
1760 &smmu->num_global_irqs)) {
1761 dev_err(dev, "missing #global-interrupts property\n");
1762 return -ENODEV;
1763 }
1764
1765 num_irqs = 0;
1766 while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
1767 num_irqs++;
1768 if (num_irqs > smmu->num_global_irqs)
1769 smmu->num_context_irqs++;
1770 }
1771
1772 if (!smmu->num_context_irqs) {
1773 dev_err(dev, "found %d interrupts but expected at least %d\n",
1774 num_irqs, smmu->num_global_irqs + 1);
1775 return -ENODEV;
1776 }
1777
1778 smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,
1779 GFP_KERNEL);
1780 if (!smmu->irqs) {
1781 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
1782 return -ENOMEM;
1783 }
1784
1785 for (i = 0; i < num_irqs; ++i) {
1786 int irq = platform_get_irq(pdev, i);
1787
1788 if (irq < 0) {
1789 dev_err(dev, "failed to get irq index %d\n", i);
1790 return -ENODEV;
1791 }
1792 smmu->irqs[i] = irq;
1793 }
1794
1795 err = arm_smmu_device_cfg_probe(smmu);
1796 if (err)
1797 return err;
1798
1799 i = 0;
1800 smmu->masters = RB_ROOT;
1801 while (!of_parse_phandle_with_args(dev->of_node, "mmu-masters",
1802 "#stream-id-cells", i,
1803 &masterspec)) {
1804 err = register_smmu_master(smmu, dev, &masterspec);
1805 if (err) {
1806 dev_err(dev, "failed to add master %s\n",
1807 masterspec.np->name);
1808 goto out_put_masters;
1809 }
1810
1811 i++;
1812 }
1813 dev_notice(dev, "registered %d master devices\n", i);
1814
1815 parse_driver_options(smmu);
1816
1817 if (smmu->version > ARM_SMMU_V1 &&
1818 smmu->num_context_banks != smmu->num_context_irqs) {
1819 dev_err(dev,
1820 "found only %d context interrupt(s) but %d required\n",
1821 smmu->num_context_irqs, smmu->num_context_banks);
1822 err = -ENODEV;
1823 goto out_put_masters;
1824 }
1825
1826 for (i = 0; i < smmu->num_global_irqs; ++i) {
1827 err = request_irq(smmu->irqs[i],
1828 arm_smmu_global_fault,
1829 IRQF_SHARED,
1830 "arm-smmu global fault",
1831 smmu);
1832 if (err) {
1833 dev_err(dev, "failed to request global IRQ %d (%u)\n",
1834 i, smmu->irqs[i]);
1835 goto out_free_irqs;
1836 }
1837 }
1838
1839 INIT_LIST_HEAD(&smmu->list);
1840 spin_lock(&arm_smmu_devices_lock);
1841 list_add(&smmu->list, &arm_smmu_devices);
1842 spin_unlock(&arm_smmu_devices_lock);
1843
1844 arm_smmu_device_reset(smmu);
1845 return 0;
1846
1847 out_free_irqs:
1848 while (i--)
1849 free_irq(smmu->irqs[i], smmu);
1850
1851 out_put_masters:
1852 for (node = rb_first(&smmu->masters); node; node = rb_next(node)) {
1853 struct arm_smmu_master *master
1854 = container_of(node, struct arm_smmu_master, node);
1855 of_node_put(master->of_node);
1856 }
1857
1858 return err;
1859 }
1860
1861 static int arm_smmu_device_remove(struct platform_device *pdev)
1862 {
1863 int i;
1864 struct device *dev = &pdev->dev;
1865 struct arm_smmu_device *curr, *smmu = NULL;
1866 struct rb_node *node;
1867
1868 spin_lock(&arm_smmu_devices_lock);
1869 list_for_each_entry(curr, &arm_smmu_devices, list) {
1870 if (curr->dev == dev) {
1871 smmu = curr;
1872 list_del(&smmu->list);
1873 break;
1874 }
1875 }
1876 spin_unlock(&arm_smmu_devices_lock);
1877
1878 if (!smmu)
1879 return -ENODEV;
1880
1881 for (node = rb_first(&smmu->masters); node; node = rb_next(node)) {
1882 struct arm_smmu_master *master
1883 = container_of(node, struct arm_smmu_master, node);
1884 of_node_put(master->of_node);
1885 }
1886
1887 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
1888 dev_err(dev, "removing device with active domains!\n");
1889
1890 for (i = 0; i < smmu->num_global_irqs; ++i)
1891 free_irq(smmu->irqs[i], smmu);
1892
1893 /* Turn the thing off */
1894 writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1895 return 0;
1896 }
1897
1898 static struct platform_driver arm_smmu_driver = {
1899 .driver = {
1900 .name = "arm-smmu",
1901 .of_match_table = of_match_ptr(arm_smmu_of_match),
1902 },
1903 .probe = arm_smmu_device_dt_probe,
1904 .remove = arm_smmu_device_remove,
1905 };
1906
1907 static int __init arm_smmu_init(void)
1908 {
1909 struct device_node *np;
1910 int ret;
1911
1912 /*
1913 * Play nice with systems that don't have an ARM SMMU by checking that
1914 * an ARM SMMU exists in the system before proceeding with the driver
1915 * and IOMMU bus operation registration.
1916 */
1917 np = of_find_matching_node(NULL, arm_smmu_of_match);
1918 if (!np)
1919 return 0;
1920
1921 of_node_put(np);
1922
1923 ret = platform_driver_register(&arm_smmu_driver);
1924 if (ret)
1925 return ret;
1926
1927 /* Oh, for a proper bus abstraction */
1928 if (!iommu_present(&platform_bus_type))
1929 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
1930
1931 #ifdef CONFIG_ARM_AMBA
1932 if (!iommu_present(&amba_bustype))
1933 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
1934 #endif
1935
1936 #ifdef CONFIG_PCI
1937 if (!iommu_present(&pci_bus_type))
1938 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
1939 #endif
1940
1941 return 0;
1942 }
1943
1944 static void __exit arm_smmu_exit(void)
1945 {
1946 return platform_driver_unregister(&arm_smmu_driver);
1947 }
1948
1949 subsys_initcall(arm_smmu_init);
1950 module_exit(arm_smmu_exit);
1951
1952 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
1953 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
1954 MODULE_LICENSE("GPL v2");
CRIS linux kernel tree