?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 5.1.15
[linux/fpc-iii.git] / drivers / iommu / tegra-smmu.c
blob8d30653cd13a77056c05337a1e47dc9634e6b779
1 /*
2 * Copyright (C) 2011-2014 NVIDIA CORPORATION. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 */
8
9 #include <linux/bitops.h>
10 #include <linux/debugfs.h>
11 #include <linux/err.h>
12 #include <linux/iommu.h>
13 #include <linux/kernel.h>
14 #include <linux/of.h>
15 #include <linux/of_device.h>
16 #include <linux/platform_device.h>
17 #include <linux/slab.h>
18 #include <linux/dma-mapping.h>
19
20 #include <soc/tegra/ahb.h>
21 #include <soc/tegra/mc.h>
22
23 struct tegra_smmu_group {
24 struct list_head list;
25 const struct tegra_smmu_group_soc *soc;
26 struct iommu_group *group;
27 };
28
29 struct tegra_smmu {
30 void __iomem *regs;
31 struct device *dev;
32
33 struct tegra_mc *mc;
34 const struct tegra_smmu_soc *soc;
35
36 struct list_head groups;
37
38 unsigned long pfn_mask;
39 unsigned long tlb_mask;
40
41 unsigned long *asids;
42 struct mutex lock;
43
44 struct list_head list;
45
46 struct dentry *debugfs;
47
48 struct iommu_device iommu; /* IOMMU Core code handle */
49 };
50
51 struct tegra_smmu_as {
52 struct iommu_domain domain;
53 struct tegra_smmu *smmu;
54 unsigned int use_count;
55 u32 *count;
56 struct page **pts;
57 struct page *pd;
58 dma_addr_t pd_dma;
59 unsigned id;
60 u32 attr;
61 };
62
63 static struct tegra_smmu_as *to_smmu_as(struct iommu_domain *dom)
64 {
65 return container_of(dom, struct tegra_smmu_as, domain);
66 }
67
68 static inline void smmu_writel(struct tegra_smmu *smmu, u32 value,
69 unsigned long offset)
70 {
71 writel(value, smmu->regs + offset);
72 }
73
74 static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset)
75 {
76 return readl(smmu->regs + offset);
77 }
78
79 #define SMMU_CONFIG 0x010
80 #define SMMU_CONFIG_ENABLE (1 << 0)
81
82 #define SMMU_TLB_CONFIG 0x14
83 #define SMMU_TLB_CONFIG_HIT_UNDER_MISS (1 << 29)
84 #define SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION (1 << 28)
85 #define SMMU_TLB_CONFIG_ACTIVE_LINES(smmu) \
86 ((smmu)->soc->num_tlb_lines & (smmu)->tlb_mask)
87
88 #define SMMU_PTC_CONFIG 0x18
89 #define SMMU_PTC_CONFIG_ENABLE (1 << 29)
90 #define SMMU_PTC_CONFIG_REQ_LIMIT(x) (((x) & 0x0f) << 24)
91 #define SMMU_PTC_CONFIG_INDEX_MAP(x) ((x) & 0x3f)
92
93 #define SMMU_PTB_ASID 0x01c
94 #define SMMU_PTB_ASID_VALUE(x) ((x) & 0x7f)
95
96 #define SMMU_PTB_DATA 0x020
97 #define SMMU_PTB_DATA_VALUE(dma, attr) ((dma) >> 12 | (attr))
98
99 #define SMMU_MK_PDE(dma, attr) ((dma) >> SMMU_PTE_SHIFT | (attr))
100
101 #define SMMU_TLB_FLUSH 0x030
102 #define SMMU_TLB_FLUSH_VA_MATCH_ALL (0 << 0)
103 #define SMMU_TLB_FLUSH_VA_MATCH_SECTION (2 << 0)
104 #define SMMU_TLB_FLUSH_VA_MATCH_GROUP (3 << 0)
105 #define SMMU_TLB_FLUSH_VA_SECTION(addr) ((((addr) & 0xffc00000) >> 12) | \
106 SMMU_TLB_FLUSH_VA_MATCH_SECTION)
107 #define SMMU_TLB_FLUSH_VA_GROUP(addr) ((((addr) & 0xffffc000) >> 12) | \
108 SMMU_TLB_FLUSH_VA_MATCH_GROUP)
109 #define SMMU_TLB_FLUSH_ASID_MATCH (1 << 31)
110
111 #define SMMU_PTC_FLUSH 0x034
112 #define SMMU_PTC_FLUSH_TYPE_ALL (0 << 0)
113 #define SMMU_PTC_FLUSH_TYPE_ADR (1 << 0)
114
115 #define SMMU_PTC_FLUSH_HI 0x9b8
116 #define SMMU_PTC_FLUSH_HI_MASK 0x3
117
118 /* per-SWGROUP SMMU_*_ASID register */
119 #define SMMU_ASID_ENABLE (1 << 31)
120 #define SMMU_ASID_MASK 0x7f
121 #define SMMU_ASID_VALUE(x) ((x) & SMMU_ASID_MASK)
122
123 /* page table definitions */
124 #define SMMU_NUM_PDE 1024
125 #define SMMU_NUM_PTE 1024
126
127 #define SMMU_SIZE_PD (SMMU_NUM_PDE * 4)
128 #define SMMU_SIZE_PT (SMMU_NUM_PTE * 4)
129
130 #define SMMU_PDE_SHIFT 22
131 #define SMMU_PTE_SHIFT 12
132
133 #define SMMU_PD_READABLE (1 << 31)
134 #define SMMU_PD_WRITABLE (1 << 30)
135 #define SMMU_PD_NONSECURE (1 << 29)
136
137 #define SMMU_PDE_READABLE (1 << 31)
138 #define SMMU_PDE_WRITABLE (1 << 30)
139 #define SMMU_PDE_NONSECURE (1 << 29)
140 #define SMMU_PDE_NEXT (1 << 28)
141
142 #define SMMU_PTE_READABLE (1 << 31)
143 #define SMMU_PTE_WRITABLE (1 << 30)
144 #define SMMU_PTE_NONSECURE (1 << 29)
145
146 #define SMMU_PDE_ATTR (SMMU_PDE_READABLE | SMMU_PDE_WRITABLE | \
147 SMMU_PDE_NONSECURE)
148 #define SMMU_PTE_ATTR (SMMU_PTE_READABLE | SMMU_PTE_WRITABLE | \
149 SMMU_PTE_NONSECURE)
150
151 static unsigned int iova_pd_index(unsigned long iova)
152 {
153 return (iova >> SMMU_PDE_SHIFT) & (SMMU_NUM_PDE - 1);
154 }
155
156 static unsigned int iova_pt_index(unsigned long iova)
157 {
158 return (iova >> SMMU_PTE_SHIFT) & (SMMU_NUM_PTE - 1);
159 }
160
161 static bool smmu_dma_addr_valid(struct tegra_smmu *smmu, dma_addr_t addr)
162 {
163 addr >>= 12;
164 return (addr & smmu->pfn_mask) == addr;
165 }
166
167 static dma_addr_t smmu_pde_to_dma(u32 pde)
168 {
169 return pde << 12;
170 }
171
172 static void smmu_flush_ptc_all(struct tegra_smmu *smmu)
173 {
174 smmu_writel(smmu, SMMU_PTC_FLUSH_TYPE_ALL, SMMU_PTC_FLUSH);
175 }
176
177 static inline void smmu_flush_ptc(struct tegra_smmu *smmu, dma_addr_t dma,
178 unsigned long offset)
179 {
180 u32 value;
181
182 offset &= ~(smmu->mc->soc->atom_size - 1);
183
184 if (smmu->mc->soc->num_address_bits > 32) {
185 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
186 value = (dma >> 32) & SMMU_PTC_FLUSH_HI_MASK;
187 #else
188 value = 0;
189 #endif
190 smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI);
191 }
192
193 value = (dma + offset) | SMMU_PTC_FLUSH_TYPE_ADR;
194 smmu_writel(smmu, value, SMMU_PTC_FLUSH);
195 }
196
197 static inline void smmu_flush_tlb(struct tegra_smmu *smmu)
198 {
199 smmu_writel(smmu, SMMU_TLB_FLUSH_VA_MATCH_ALL, SMMU_TLB_FLUSH);
200 }
201
202 static inline void smmu_flush_tlb_asid(struct tegra_smmu *smmu,
203 unsigned long asid)
204 {
205 u32 value;
206
207 if (smmu->soc->num_asids == 4)
208 value = (asid & 0x3) << 29;
209 else
210 value = (asid & 0x7f) << 24;
211
212 value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_MATCH_ALL;
213 smmu_writel(smmu, value, SMMU_TLB_FLUSH);
214 }
215
216 static inline void smmu_flush_tlb_section(struct tegra_smmu *smmu,
217 unsigned long asid,
218 unsigned long iova)
219 {
220 u32 value;
221
222 if (smmu->soc->num_asids == 4)
223 value = (asid & 0x3) << 29;
224 else
225 value = (asid & 0x7f) << 24;
226
227 value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_SECTION(iova);
228 smmu_writel(smmu, value, SMMU_TLB_FLUSH);
229 }
230
231 static inline void smmu_flush_tlb_group(struct tegra_smmu *smmu,
232 unsigned long asid,
233 unsigned long iova)
234 {
235 u32 value;
236
237 if (smmu->soc->num_asids == 4)
238 value = (asid & 0x3) << 29;
239 else
240 value = (asid & 0x7f) << 24;
241
242 value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_GROUP(iova);
243 smmu_writel(smmu, value, SMMU_TLB_FLUSH);
244 }
245
246 static inline void smmu_flush(struct tegra_smmu *smmu)
247 {
248 smmu_readl(smmu, SMMU_CONFIG);
249 }
250
251 static int tegra_smmu_alloc_asid(struct tegra_smmu *smmu, unsigned int *idp)
252 {
253 unsigned long id;
254
255 mutex_lock(&smmu->lock);
256
257 id = find_first_zero_bit(smmu->asids, smmu->soc->num_asids);
258 if (id >= smmu->soc->num_asids) {
259 mutex_unlock(&smmu->lock);
260 return -ENOSPC;
261 }
262
263 set_bit(id, smmu->asids);
264 *idp = id;
265
266 mutex_unlock(&smmu->lock);
267 return 0;
268 }
269
270 static void tegra_smmu_free_asid(struct tegra_smmu *smmu, unsigned int id)
271 {
272 mutex_lock(&smmu->lock);
273 clear_bit(id, smmu->asids);
274 mutex_unlock(&smmu->lock);
275 }
276
277 static bool tegra_smmu_capable(enum iommu_cap cap)
278 {
279 return false;
280 }
281
282 static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type)
283 {
284 struct tegra_smmu_as *as;
285
286 if (type != IOMMU_DOMAIN_UNMANAGED)
287 return NULL;
288
289 as = kzalloc(sizeof(*as), GFP_KERNEL);
290 if (!as)
291 return NULL;
292
293 as->attr = SMMU_PD_READABLE | SMMU_PD_WRITABLE | SMMU_PD_NONSECURE;
294
295 as->pd = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO);
296 if (!as->pd) {
297 kfree(as);
298 return NULL;
299 }
300
301 as->count = kcalloc(SMMU_NUM_PDE, sizeof(u32), GFP_KERNEL);
302 if (!as->count) {
303 __free_page(as->pd);
304 kfree(as);
305 return NULL;
306 }
307
308 as->pts = kcalloc(SMMU_NUM_PDE, sizeof(*as->pts), GFP_KERNEL);
309 if (!as->pts) {
310 kfree(as->count);
311 __free_page(as->pd);
312 kfree(as);
313 return NULL;
314 }
315
316 /* setup aperture */
317 as->domain.geometry.aperture_start = 0;
318 as->domain.geometry.aperture_end = 0xffffffff;
319 as->domain.geometry.force_aperture = true;
320
321 return &as->domain;
322 }
323
324 static void tegra_smmu_domain_free(struct iommu_domain *domain)
325 {
326 struct tegra_smmu_as *as = to_smmu_as(domain);
327
328 /* TODO: free page directory and page tables */
329
330 kfree(as);
331 }
332
333 static const struct tegra_smmu_swgroup *
334 tegra_smmu_find_swgroup(struct tegra_smmu *smmu, unsigned int swgroup)
335 {
336 const struct tegra_smmu_swgroup *group = NULL;
337 unsigned int i;
338
339 for (i = 0; i < smmu->soc->num_swgroups; i++) {
340 if (smmu->soc->swgroups[i].swgroup == swgroup) {
341 group = &smmu->soc->swgroups[i];
342 break;
343 }
344 }
345
346 return group;
347 }
348
349 static void tegra_smmu_enable(struct tegra_smmu *smmu, unsigned int swgroup,
350 unsigned int asid)
351 {
352 const struct tegra_smmu_swgroup *group;
353 unsigned int i;
354 u32 value;
355
356 for (i = 0; i < smmu->soc->num_clients; i++) {
357 const struct tegra_mc_client *client = &smmu->soc->clients[i];
358
359 if (client->swgroup != swgroup)
360 continue;
361
362 value = smmu_readl(smmu, client->smmu.reg);
363 value |= BIT(client->smmu.bit);
364 smmu_writel(smmu, value, client->smmu.reg);
365 }
366
367 group = tegra_smmu_find_swgroup(smmu, swgroup);
368 if (group) {
369 value = smmu_readl(smmu, group->reg);
370 value &= ~SMMU_ASID_MASK;
371 value |= SMMU_ASID_VALUE(asid);
372 value |= SMMU_ASID_ENABLE;
373 smmu_writel(smmu, value, group->reg);
374 }
375 }
376
377 static void tegra_smmu_disable(struct tegra_smmu *smmu, unsigned int swgroup,
378 unsigned int asid)
379 {
380 const struct tegra_smmu_swgroup *group;
381 unsigned int i;
382 u32 value;
383
384 group = tegra_smmu_find_swgroup(smmu, swgroup);
385 if (group) {
386 value = smmu_readl(smmu, group->reg);
387 value &= ~SMMU_ASID_MASK;
388 value |= SMMU_ASID_VALUE(asid);
389 value &= ~SMMU_ASID_ENABLE;
390 smmu_writel(smmu, value, group->reg);
391 }
392
393 for (i = 0; i < smmu->soc->num_clients; i++) {
394 const struct tegra_mc_client *client = &smmu->soc->clients[i];
395
396 if (client->swgroup != swgroup)
397 continue;
398
399 value = smmu_readl(smmu, client->smmu.reg);
400 value &= ~BIT(client->smmu.bit);
401 smmu_writel(smmu, value, client->smmu.reg);
402 }
403 }
404
405 static int tegra_smmu_as_prepare(struct tegra_smmu *smmu,
406 struct tegra_smmu_as *as)
407 {
408 u32 value;
409 int err;
410
411 if (as->use_count > 0) {
412 as->use_count++;
413 return 0;
414 }
415
416 as->pd_dma = dma_map_page(smmu->dev, as->pd, 0, SMMU_SIZE_PD,
417 DMA_TO_DEVICE);
418 if (dma_mapping_error(smmu->dev, as->pd_dma))
419 return -ENOMEM;
420
421 /* We can't handle 64-bit DMA addresses */
422 if (!smmu_dma_addr_valid(smmu, as->pd_dma)) {
423 err = -ENOMEM;
424 goto err_unmap;
425 }
426
427 err = tegra_smmu_alloc_asid(smmu, &as->id);
428 if (err < 0)
429 goto err_unmap;
430
431 smmu_flush_ptc(smmu, as->pd_dma, 0);
432 smmu_flush_tlb_asid(smmu, as->id);
433
434 smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID);
435 value = SMMU_PTB_DATA_VALUE(as->pd_dma, as->attr);
436 smmu_writel(smmu, value, SMMU_PTB_DATA);
437 smmu_flush(smmu);
438
439 as->smmu = smmu;
440 as->use_count++;
441
442 return 0;
443
444 err_unmap:
445 dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);
446 return err;
447 }
448
449 static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu,
450 struct tegra_smmu_as *as)
451 {
452 if (--as->use_count > 0)
453 return;
454
455 tegra_smmu_free_asid(smmu, as->id);
456
457 dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);
458
459 as->smmu = NULL;
460 }
461
462 static int tegra_smmu_attach_dev(struct iommu_domain *domain,
463 struct device *dev)
464 {
465 struct tegra_smmu *smmu = dev->archdata.iommu;
466 struct tegra_smmu_as *as = to_smmu_as(domain);
467 struct device_node *np = dev->of_node;
468 struct of_phandle_args args;
469 unsigned int index = 0;
470 int err = 0;
471
472 while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index,
473 &args)) {
474 unsigned int swgroup = args.args[0];
475
476 if (args.np != smmu->dev->of_node) {
477 of_node_put(args.np);
478 continue;
479 }
480
481 of_node_put(args.np);
482
483 err = tegra_smmu_as_prepare(smmu, as);
484 if (err < 0)
485 return err;
486
487 tegra_smmu_enable(smmu, swgroup, as->id);
488 index++;
489 }
490
491 if (index == 0)
492 return -ENODEV;
493
494 return 0;
495 }
496
497 static void tegra_smmu_detach_dev(struct iommu_domain *domain, struct device *dev)
498 {
499 struct tegra_smmu_as *as = to_smmu_as(domain);
500 struct device_node *np = dev->of_node;
501 struct tegra_smmu *smmu = as->smmu;
502 struct of_phandle_args args;
503 unsigned int index = 0;
504
505 while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index,
506 &args)) {
507 unsigned int swgroup = args.args[0];
508
509 if (args.np != smmu->dev->of_node) {
510 of_node_put(args.np);
511 continue;
512 }
513
514 of_node_put(args.np);
515
516 tegra_smmu_disable(smmu, swgroup, as->id);
517 tegra_smmu_as_unprepare(smmu, as);
518 index++;
519 }
520 }
521
522 static void tegra_smmu_set_pde(struct tegra_smmu_as *as, unsigned long iova,
523 u32 value)
524 {
525 unsigned int pd_index = iova_pd_index(iova);
526 struct tegra_smmu *smmu = as->smmu;
527 u32 *pd = page_address(as->pd);
528 unsigned long offset = pd_index * sizeof(*pd);
529
530 /* Set the page directory entry first */
531 pd[pd_index] = value;
532
533 /* The flush the page directory entry from caches */
534 dma_sync_single_range_for_device(smmu->dev, as->pd_dma, offset,
535 sizeof(*pd), DMA_TO_DEVICE);
536
537 /* And flush the iommu */
538 smmu_flush_ptc(smmu, as->pd_dma, offset);
539 smmu_flush_tlb_section(smmu, as->id, iova);
540 smmu_flush(smmu);
541 }
542
543 static u32 *tegra_smmu_pte_offset(struct page *pt_page, unsigned long iova)
544 {
545 u32 *pt = page_address(pt_page);
546
547 return pt + iova_pt_index(iova);
548 }
549
550 static u32 *tegra_smmu_pte_lookup(struct tegra_smmu_as *as, unsigned long iova,
551 dma_addr_t *dmap)
552 {
553 unsigned int pd_index = iova_pd_index(iova);
554 struct page *pt_page;
555 u32 *pd;
556
557 pt_page = as->pts[pd_index];
558 if (!pt_page)
559 return NULL;
560
561 pd = page_address(as->pd);
562 *dmap = smmu_pde_to_dma(pd[pd_index]);
563
564 return tegra_smmu_pte_offset(pt_page, iova);
565 }
566
567 static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova,
568 dma_addr_t *dmap)
569 {
570 unsigned int pde = iova_pd_index(iova);
571 struct tegra_smmu *smmu = as->smmu;
572
573 if (!as->pts[pde]) {
574 struct page *page;
575 dma_addr_t dma;
576
577 page = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO);
578 if (!page)
579 return NULL;
580
581 dma = dma_map_page(smmu->dev, page, 0, SMMU_SIZE_PT,
582 DMA_TO_DEVICE);
583 if (dma_mapping_error(smmu->dev, dma)) {
584 __free_page(page);
585 return NULL;
586 }
587
588 if (!smmu_dma_addr_valid(smmu, dma)) {
589 dma_unmap_page(smmu->dev, dma, SMMU_SIZE_PT,
590 DMA_TO_DEVICE);
591 __free_page(page);
592 return NULL;
593 }
594
595 as->pts[pde] = page;
596
597 tegra_smmu_set_pde(as, iova, SMMU_MK_PDE(dma, SMMU_PDE_ATTR |
598 SMMU_PDE_NEXT));
599
600 *dmap = dma;
601 } else {
602 u32 *pd = page_address(as->pd);
603
604 *dmap = smmu_pde_to_dma(pd[pde]);
605 }
606
607 return tegra_smmu_pte_offset(as->pts[pde], iova);
608 }
609
610 static void tegra_smmu_pte_get_use(struct tegra_smmu_as *as, unsigned long iova)
611 {
612 unsigned int pd_index = iova_pd_index(iova);
613
614 as->count[pd_index]++;
615 }
616
617 static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova)
618 {
619 unsigned int pde = iova_pd_index(iova);
620 struct page *page = as->pts[pde];
621
622 /*
623 * When no entries in this page table are used anymore, return the
624 * memory page to the system.
625 */
626 if (--as->count[pde] == 0) {
627 struct tegra_smmu *smmu = as->smmu;
628 u32 *pd = page_address(as->pd);
629 dma_addr_t pte_dma = smmu_pde_to_dma(pd[pde]);
630
631 tegra_smmu_set_pde(as, iova, 0);
632
633 dma_unmap_page(smmu->dev, pte_dma, SMMU_SIZE_PT, DMA_TO_DEVICE);
634 __free_page(page);
635 as->pts[pde] = NULL;
636 }
637 }
638
639 static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova,
640 u32 *pte, dma_addr_t pte_dma, u32 val)
641 {
642 struct tegra_smmu *smmu = as->smmu;
643 unsigned long offset = offset_in_page(pte);
644
645 *pte = val;
646
647 dma_sync_single_range_for_device(smmu->dev, pte_dma, offset,
648 4, DMA_TO_DEVICE);
649 smmu_flush_ptc(smmu, pte_dma, offset);
650 smmu_flush_tlb_group(smmu, as->id, iova);
651 smmu_flush(smmu);
652 }
653
654 static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,
655 phys_addr_t paddr, size_t size, int prot)
656 {
657 struct tegra_smmu_as *as = to_smmu_as(domain);
658 dma_addr_t pte_dma;
659 u32 *pte;
660
661 pte = as_get_pte(as, iova, &pte_dma);
662 if (!pte)
663 return -ENOMEM;
664
665 /* If we aren't overwriting a pre-existing entry, increment use */
666 if (*pte == 0)
667 tegra_smmu_pte_get_use(as, iova);
668
669 tegra_smmu_set_pte(as, iova, pte, pte_dma,
670 __phys_to_pfn(paddr) | SMMU_PTE_ATTR);
671
672 return 0;
673 }
674
675 static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
676 size_t size)
677 {
678 struct tegra_smmu_as *as = to_smmu_as(domain);
679 dma_addr_t pte_dma;
680 u32 *pte;
681
682 pte = tegra_smmu_pte_lookup(as, iova, &pte_dma);
683 if (!pte || !*pte)
684 return 0;
685
686 tegra_smmu_set_pte(as, iova, pte, pte_dma, 0);
687 tegra_smmu_pte_put_use(as, iova);
688
689 return size;
690 }
691
692 static phys_addr_t tegra_smmu_iova_to_phys(struct iommu_domain *domain,
693 dma_addr_t iova)
694 {
695 struct tegra_smmu_as *as = to_smmu_as(domain);
696 unsigned long pfn;
697 dma_addr_t pte_dma;
698 u32 *pte;
699
700 pte = tegra_smmu_pte_lookup(as, iova, &pte_dma);
701 if (!pte || !*pte)
702 return 0;
703
704 pfn = *pte & as->smmu->pfn_mask;
705
706 return PFN_PHYS(pfn);
707 }
708
709 static struct tegra_smmu *tegra_smmu_find(struct device_node *np)
710 {
711 struct platform_device *pdev;
712 struct tegra_mc *mc;
713
714 pdev = of_find_device_by_node(np);
715 if (!pdev)
716 return NULL;
717
718 mc = platform_get_drvdata(pdev);
719 if (!mc)
720 return NULL;
721
722 return mc->smmu;
723 }
724
725 static int tegra_smmu_configure(struct tegra_smmu *smmu, struct device *dev,
726 struct of_phandle_args *args)
727 {
728 const struct iommu_ops *ops = smmu->iommu.ops;
729 int err;
730
731 err = iommu_fwspec_init(dev, &dev->of_node->fwnode, ops);
732 if (err < 0) {
733 dev_err(dev, "failed to initialize fwspec: %d\n", err);
734 return err;
735 }
736
737 err = ops->of_xlate(dev, args);
738 if (err < 0) {
739 dev_err(dev, "failed to parse SW group ID: %d\n", err);
740 iommu_fwspec_free(dev);
741 return err;
742 }
743
744 return 0;
745 }
746
747 static int tegra_smmu_add_device(struct device *dev)
748 {
749 struct device_node *np = dev->of_node;
750 struct tegra_smmu *smmu = NULL;
751 struct iommu_group *group;
752 struct of_phandle_args args;
753 unsigned int index = 0;
754 int err;
755
756 while (of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index,
757 &args) == 0) {
758 smmu = tegra_smmu_find(args.np);
759 if (smmu) {
760 err = tegra_smmu_configure(smmu, dev, &args);
761 of_node_put(args.np);
762
763 if (err < 0)
764 return err;
765
766 /*
767 * Only a single IOMMU master interface is currently
768 * supported by the Linux kernel, so abort after the
769 * first match.
770 */
771 dev->archdata.iommu = smmu;
772
773 iommu_device_link(&smmu->iommu, dev);
774
775 break;
776 }
777
778 of_node_put(args.np);
779 index++;
780 }
781
782 if (!smmu)
783 return -ENODEV;
784
785 group = iommu_group_get_for_dev(dev);
786 if (IS_ERR(group))
787 return PTR_ERR(group);
788
789 iommu_group_put(group);
790
791 return 0;
792 }
793
794 static void tegra_smmu_remove_device(struct device *dev)
795 {
796 struct tegra_smmu *smmu = dev->archdata.iommu;
797
798 if (smmu)
799 iommu_device_unlink(&smmu->iommu, dev);
800
801 dev->archdata.iommu = NULL;
802 iommu_group_remove_device(dev);
803 }
804
805 static const struct tegra_smmu_group_soc *
806 tegra_smmu_find_group(struct tegra_smmu *smmu, unsigned int swgroup)
807 {
808 unsigned int i, j;
809
810 for (i = 0; i < smmu->soc->num_groups; i++)
811 for (j = 0; j < smmu->soc->groups[i].num_swgroups; j++)
812 if (smmu->soc->groups[i].swgroups[j] == swgroup)
813 return &smmu->soc->groups[i];
814
815 return NULL;
816 }
817
818 static struct iommu_group *tegra_smmu_group_get(struct tegra_smmu *smmu,
819 unsigned int swgroup)
820 {
821 const struct tegra_smmu_group_soc *soc;
822 struct tegra_smmu_group *group;
823
824 soc = tegra_smmu_find_group(smmu, swgroup);
825 if (!soc)
826 return NULL;
827
828 mutex_lock(&smmu->lock);
829
830 list_for_each_entry(group, &smmu->groups, list)
831 if (group->soc == soc) {
832 mutex_unlock(&smmu->lock);
833 return group->group;
834 }
835
836 group = devm_kzalloc(smmu->dev, sizeof(*group), GFP_KERNEL);
837 if (!group) {
838 mutex_unlock(&smmu->lock);
839 return NULL;
840 }
841
842 INIT_LIST_HEAD(&group->list);
843 group->soc = soc;
844
845 group->group = iommu_group_alloc();
846 if (IS_ERR(group->group)) {
847 devm_kfree(smmu->dev, group);
848 mutex_unlock(&smmu->lock);
849 return NULL;
850 }
851
852 list_add_tail(&group->list, &smmu->groups);
853 mutex_unlock(&smmu->lock);
854
855 return group->group;
856 }
857
858 static struct iommu_group *tegra_smmu_device_group(struct device *dev)
859 {
860 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
861 struct tegra_smmu *smmu = dev->archdata.iommu;
862 struct iommu_group *group;
863
864 group = tegra_smmu_group_get(smmu, fwspec->ids[0]);
865 if (!group)
866 group = generic_device_group(dev);
867
868 return group;
869 }
870
871 static int tegra_smmu_of_xlate(struct device *dev,
872 struct of_phandle_args *args)
873 {
874 u32 id = args->args[0];
875
876 return iommu_fwspec_add_ids(dev, &id, 1);
877 }
878
879 static const struct iommu_ops tegra_smmu_ops = {
880 .capable = tegra_smmu_capable,
881 .domain_alloc = tegra_smmu_domain_alloc,
882 .domain_free = tegra_smmu_domain_free,
883 .attach_dev = tegra_smmu_attach_dev,
884 .detach_dev = tegra_smmu_detach_dev,
885 .add_device = tegra_smmu_add_device,
886 .remove_device = tegra_smmu_remove_device,
887 .device_group = tegra_smmu_device_group,
888 .map = tegra_smmu_map,
889 .unmap = tegra_smmu_unmap,
890 .iova_to_phys = tegra_smmu_iova_to_phys,
891 .of_xlate = tegra_smmu_of_xlate,
892 .pgsize_bitmap = SZ_4K,
893 };
894
895 static void tegra_smmu_ahb_enable(void)
896 {
897 static const struct of_device_id ahb_match[] = {
898 { .compatible = "nvidia,tegra30-ahb", },
899 { }
900 };
901 struct device_node *ahb;
902
903 ahb = of_find_matching_node(NULL, ahb_match);
904 if (ahb) {
905 tegra_ahb_enable_smmu(ahb);
906 of_node_put(ahb);
907 }
908 }
909
910 static int tegra_smmu_swgroups_show(struct seq_file *s, void *data)
911 {
912 struct tegra_smmu *smmu = s->private;
913 unsigned int i;
914 u32 value;
915
916 seq_printf(s, "swgroup enabled ASID\n");
917 seq_printf(s, "------------------------\n");
918
919 for (i = 0; i < smmu->soc->num_swgroups; i++) {
920 const struct tegra_smmu_swgroup *group = &smmu->soc->swgroups[i];
921 const char *status;
922 unsigned int asid;
923
924 value = smmu_readl(smmu, group->reg);
925
926 if (value & SMMU_ASID_ENABLE)
927 status = "yes";
928 else
929 status = "no";
930
931 asid = value & SMMU_ASID_MASK;
932
933 seq_printf(s, "%-9s %-7s %#04x\n", group->name, status,
934 asid);
935 }
936
937 return 0;
938 }
939
940 DEFINE_SHOW_ATTRIBUTE(tegra_smmu_swgroups);
941
942 static int tegra_smmu_clients_show(struct seq_file *s, void *data)
943 {
944 struct tegra_smmu *smmu = s->private;
945 unsigned int i;
946 u32 value;
947
948 seq_printf(s, "client enabled\n");
949 seq_printf(s, "--------------------\n");
950
951 for (i = 0; i < smmu->soc->num_clients; i++) {
952 const struct tegra_mc_client *client = &smmu->soc->clients[i];
953 const char *status;
954
955 value = smmu_readl(smmu, client->smmu.reg);
956
957 if (value & BIT(client->smmu.bit))
958 status = "yes";
959 else
960 status = "no";
961
962 seq_printf(s, "%-12s %s\n", client->name, status);
963 }
964
965 return 0;
966 }
967
968 DEFINE_SHOW_ATTRIBUTE(tegra_smmu_clients);
969
970 static void tegra_smmu_debugfs_init(struct tegra_smmu *smmu)
971 {
972 smmu->debugfs = debugfs_create_dir("smmu", NULL);
973 if (!smmu->debugfs)
974 return;
975
976 debugfs_create_file("swgroups", S_IRUGO, smmu->debugfs, smmu,
977 &tegra_smmu_swgroups_fops);
978 debugfs_create_file("clients", S_IRUGO, smmu->debugfs, smmu,
979 &tegra_smmu_clients_fops);
980 }
981
982 static void tegra_smmu_debugfs_exit(struct tegra_smmu *smmu)
983 {
984 debugfs_remove_recursive(smmu->debugfs);
985 }
986
987 struct tegra_smmu *tegra_smmu_probe(struct device *dev,
988 const struct tegra_smmu_soc *soc,
989 struct tegra_mc *mc)
990 {
991 struct tegra_smmu *smmu;
992 size_t size;
993 u32 value;
994 int err;
995
996 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
997 if (!smmu)
998 return ERR_PTR(-ENOMEM);
999
1000 /*
1001 * This is a bit of a hack. Ideally we'd want to simply return this
1002 * value. However the IOMMU registration process will attempt to add
1003 * all devices to the IOMMU when bus_set_iommu() is called. In order
1004 * not to rely on global variables to track the IOMMU instance, we
1005 * set it here so that it can be looked up from the .add_device()
1006 * callback via the IOMMU device's .drvdata field.
1007 */
1008 mc->smmu = smmu;
1009
1010 size = BITS_TO_LONGS(soc->num_asids) * sizeof(long);
1011
1012 smmu->asids = devm_kzalloc(dev, size, GFP_KERNEL);
1013 if (!smmu->asids)
1014 return ERR_PTR(-ENOMEM);
1015
1016 INIT_LIST_HEAD(&smmu->groups);
1017 mutex_init(&smmu->lock);
1018
1019 smmu->regs = mc->regs;
1020 smmu->soc = soc;
1021 smmu->dev = dev;
1022 smmu->mc = mc;
1023
1024 smmu->pfn_mask = BIT_MASK(mc->soc->num_address_bits - PAGE_SHIFT) - 1;
1025 dev_dbg(dev, "address bits: %u, PFN mask: %#lx\n",
1026 mc->soc->num_address_bits, smmu->pfn_mask);
1027 smmu->tlb_mask = (smmu->soc->num_tlb_lines << 1) - 1;
1028 dev_dbg(dev, "TLB lines: %u, mask: %#lx\n", smmu->soc->num_tlb_lines,
1029 smmu->tlb_mask);
1030
1031 value = SMMU_PTC_CONFIG_ENABLE | SMMU_PTC_CONFIG_INDEX_MAP(0x3f);
1032
1033 if (soc->supports_request_limit)
1034 value |= SMMU_PTC_CONFIG_REQ_LIMIT(8);
1035
1036 smmu_writel(smmu, value, SMMU_PTC_CONFIG);
1037
1038 value = SMMU_TLB_CONFIG_HIT_UNDER_MISS |
1039 SMMU_TLB_CONFIG_ACTIVE_LINES(smmu);
1040
1041 if (soc->supports_round_robin_arbitration)
1042 value |= SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION;
1043
1044 smmu_writel(smmu, value, SMMU_TLB_CONFIG);
1045
1046 smmu_flush_ptc_all(smmu);
1047 smmu_flush_tlb(smmu);
1048 smmu_writel(smmu, SMMU_CONFIG_ENABLE, SMMU_CONFIG);
1049 smmu_flush(smmu);
1050
1051 tegra_smmu_ahb_enable();
1052
1053 err = iommu_device_sysfs_add(&smmu->iommu, dev, NULL, dev_name(dev));
1054 if (err)
1055 return ERR_PTR(err);
1056
1057 iommu_device_set_ops(&smmu->iommu, &tegra_smmu_ops);
1058 iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
1059
1060 err = iommu_device_register(&smmu->iommu);
1061 if (err) {
1062 iommu_device_sysfs_remove(&smmu->iommu);
1063 return ERR_PTR(err);
1064 }
1065
1066 err = bus_set_iommu(&platform_bus_type, &tegra_smmu_ops);
1067 if (err < 0) {
1068 iommu_device_unregister(&smmu->iommu);
1069 iommu_device_sysfs_remove(&smmu->iommu);
1070 return ERR_PTR(err);
1071 }
1072
1073 if (IS_ENABLED(CONFIG_DEBUG_FS))
1074 tegra_smmu_debugfs_init(smmu);
1075
1076 return smmu;
1077 }
1078
1079 void tegra_smmu_remove(struct tegra_smmu *smmu)
1080 {
1081 iommu_device_unregister(&smmu->iommu);
1082 iommu_device_sysfs_remove(&smmu->iommu);
1083
1084 if (IS_ENABLED(CONFIG_DEBUG_FS))
1085 tegra_smmu_debugfs_exit(smmu);
1086 }
Linux with added FPC-III support