1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2011-2014 NVIDIA CORPORATION. All rights reserved.
6 #include <linux/bitops.h>
7 #include <linux/debugfs.h>
9 #include <linux/iommu.h>
10 #include <linux/kernel.h>
12 #include <linux/of_device.h>
13 #include <linux/platform_device.h>
14 #include <linux/slab.h>
15 #include <linux/dma-mapping.h>
17 #include <soc/tegra/ahb.h>
18 #include <soc/tegra/mc.h>
20 struct tegra_smmu_group
{
21 struct list_head list
;
22 const struct tegra_smmu_group_soc
*soc
;
23 struct iommu_group
*group
;
31 const struct tegra_smmu_soc
*soc
;
33 struct list_head groups
;
35 unsigned long pfn_mask
;
36 unsigned long tlb_mask
;
41 struct list_head list
;
43 struct dentry
*debugfs
;
45 struct iommu_device iommu
; /* IOMMU Core code handle */
48 struct tegra_smmu_as
{
49 struct iommu_domain domain
;
50 struct tegra_smmu
*smmu
;
51 unsigned int use_count
;
60 static struct tegra_smmu_as
*to_smmu_as(struct iommu_domain
*dom
)
62 return container_of(dom
, struct tegra_smmu_as
, domain
);
65 static inline void smmu_writel(struct tegra_smmu
*smmu
, u32 value
,
68 writel(value
, smmu
->regs
+ offset
);
71 static inline u32
smmu_readl(struct tegra_smmu
*smmu
, unsigned long offset
)
73 return readl(smmu
->regs
+ offset
);
76 #define SMMU_CONFIG 0x010
77 #define SMMU_CONFIG_ENABLE (1 << 0)
79 #define SMMU_TLB_CONFIG 0x14
80 #define SMMU_TLB_CONFIG_HIT_UNDER_MISS (1 << 29)
81 #define SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION (1 << 28)
82 #define SMMU_TLB_CONFIG_ACTIVE_LINES(smmu) \
83 ((smmu)->soc->num_tlb_lines & (smmu)->tlb_mask)
85 #define SMMU_PTC_CONFIG 0x18
86 #define SMMU_PTC_CONFIG_ENABLE (1 << 29)
87 #define SMMU_PTC_CONFIG_REQ_LIMIT(x) (((x) & 0x0f) << 24)
88 #define SMMU_PTC_CONFIG_INDEX_MAP(x) ((x) & 0x3f)
90 #define SMMU_PTB_ASID 0x01c
91 #define SMMU_PTB_ASID_VALUE(x) ((x) & 0x7f)
93 #define SMMU_PTB_DATA 0x020
94 #define SMMU_PTB_DATA_VALUE(dma, attr) ((dma) >> 12 | (attr))
96 #define SMMU_MK_PDE(dma, attr) ((dma) >> SMMU_PTE_SHIFT | (attr))
98 #define SMMU_TLB_FLUSH 0x030
99 #define SMMU_TLB_FLUSH_VA_MATCH_ALL (0 << 0)
100 #define SMMU_TLB_FLUSH_VA_MATCH_SECTION (2 << 0)
101 #define SMMU_TLB_FLUSH_VA_MATCH_GROUP (3 << 0)
102 #define SMMU_TLB_FLUSH_VA_SECTION(addr) ((((addr) & 0xffc00000) >> 12) | \
103 SMMU_TLB_FLUSH_VA_MATCH_SECTION)
104 #define SMMU_TLB_FLUSH_VA_GROUP(addr) ((((addr) & 0xffffc000) >> 12) | \
105 SMMU_TLB_FLUSH_VA_MATCH_GROUP)
106 #define SMMU_TLB_FLUSH_ASID_MATCH (1 << 31)
108 #define SMMU_PTC_FLUSH 0x034
109 #define SMMU_PTC_FLUSH_TYPE_ALL (0 << 0)
110 #define SMMU_PTC_FLUSH_TYPE_ADR (1 << 0)
112 #define SMMU_PTC_FLUSH_HI 0x9b8
113 #define SMMU_PTC_FLUSH_HI_MASK 0x3
115 /* per-SWGROUP SMMU_*_ASID register */
116 #define SMMU_ASID_ENABLE (1 << 31)
117 #define SMMU_ASID_MASK 0x7f
118 #define SMMU_ASID_VALUE(x) ((x) & SMMU_ASID_MASK)
120 /* page table definitions */
121 #define SMMU_NUM_PDE 1024
122 #define SMMU_NUM_PTE 1024
124 #define SMMU_SIZE_PD (SMMU_NUM_PDE * 4)
125 #define SMMU_SIZE_PT (SMMU_NUM_PTE * 4)
127 #define SMMU_PDE_SHIFT 22
128 #define SMMU_PTE_SHIFT 12
130 #define SMMU_PD_READABLE (1 << 31)
131 #define SMMU_PD_WRITABLE (1 << 30)
132 #define SMMU_PD_NONSECURE (1 << 29)
134 #define SMMU_PDE_READABLE (1 << 31)
135 #define SMMU_PDE_WRITABLE (1 << 30)
136 #define SMMU_PDE_NONSECURE (1 << 29)
137 #define SMMU_PDE_NEXT (1 << 28)
139 #define SMMU_PTE_READABLE (1 << 31)
140 #define SMMU_PTE_WRITABLE (1 << 30)
141 #define SMMU_PTE_NONSECURE (1 << 29)
143 #define SMMU_PDE_ATTR (SMMU_PDE_READABLE | SMMU_PDE_WRITABLE | \
146 static unsigned int iova_pd_index(unsigned long iova
)
148 return (iova
>> SMMU_PDE_SHIFT
) & (SMMU_NUM_PDE
- 1);
151 static unsigned int iova_pt_index(unsigned long iova
)
153 return (iova
>> SMMU_PTE_SHIFT
) & (SMMU_NUM_PTE
- 1);
156 static bool smmu_dma_addr_valid(struct tegra_smmu
*smmu
, dma_addr_t addr
)
159 return (addr
& smmu
->pfn_mask
) == addr
;
162 static dma_addr_t
smmu_pde_to_dma(struct tegra_smmu
*smmu
, u32 pde
)
164 return (dma_addr_t
)(pde
& smmu
->pfn_mask
) << 12;
167 static void smmu_flush_ptc_all(struct tegra_smmu
*smmu
)
169 smmu_writel(smmu
, SMMU_PTC_FLUSH_TYPE_ALL
, SMMU_PTC_FLUSH
);
172 static inline void smmu_flush_ptc(struct tegra_smmu
*smmu
, dma_addr_t dma
,
173 unsigned long offset
)
177 offset
&= ~(smmu
->mc
->soc
->atom_size
- 1);
179 if (smmu
->mc
->soc
->num_address_bits
> 32) {
180 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
181 value
= (dma
>> 32) & SMMU_PTC_FLUSH_HI_MASK
;
185 smmu_writel(smmu
, value
, SMMU_PTC_FLUSH_HI
);
188 value
= (dma
+ offset
) | SMMU_PTC_FLUSH_TYPE_ADR
;
189 smmu_writel(smmu
, value
, SMMU_PTC_FLUSH
);
192 static inline void smmu_flush_tlb(struct tegra_smmu
*smmu
)
194 smmu_writel(smmu
, SMMU_TLB_FLUSH_VA_MATCH_ALL
, SMMU_TLB_FLUSH
);
197 static inline void smmu_flush_tlb_asid(struct tegra_smmu
*smmu
,
202 if (smmu
->soc
->num_asids
== 4)
203 value
= (asid
& 0x3) << 29;
205 value
= (asid
& 0x7f) << 24;
207 value
|= SMMU_TLB_FLUSH_ASID_MATCH
| SMMU_TLB_FLUSH_VA_MATCH_ALL
;
208 smmu_writel(smmu
, value
, SMMU_TLB_FLUSH
);
211 static inline void smmu_flush_tlb_section(struct tegra_smmu
*smmu
,
217 if (smmu
->soc
->num_asids
== 4)
218 value
= (asid
& 0x3) << 29;
220 value
= (asid
& 0x7f) << 24;
222 value
|= SMMU_TLB_FLUSH_ASID_MATCH
| SMMU_TLB_FLUSH_VA_SECTION(iova
);
223 smmu_writel(smmu
, value
, SMMU_TLB_FLUSH
);
226 static inline void smmu_flush_tlb_group(struct tegra_smmu
*smmu
,
232 if (smmu
->soc
->num_asids
== 4)
233 value
= (asid
& 0x3) << 29;
235 value
= (asid
& 0x7f) << 24;
237 value
|= SMMU_TLB_FLUSH_ASID_MATCH
| SMMU_TLB_FLUSH_VA_GROUP(iova
);
238 smmu_writel(smmu
, value
, SMMU_TLB_FLUSH
);
241 static inline void smmu_flush(struct tegra_smmu
*smmu
)
243 smmu_readl(smmu
, SMMU_CONFIG
);
246 static int tegra_smmu_alloc_asid(struct tegra_smmu
*smmu
, unsigned int *idp
)
250 mutex_lock(&smmu
->lock
);
252 id
= find_first_zero_bit(smmu
->asids
, smmu
->soc
->num_asids
);
253 if (id
>= smmu
->soc
->num_asids
) {
254 mutex_unlock(&smmu
->lock
);
258 set_bit(id
, smmu
->asids
);
261 mutex_unlock(&smmu
->lock
);
265 static void tegra_smmu_free_asid(struct tegra_smmu
*smmu
, unsigned int id
)
267 mutex_lock(&smmu
->lock
);
268 clear_bit(id
, smmu
->asids
);
269 mutex_unlock(&smmu
->lock
);
272 static bool tegra_smmu_capable(enum iommu_cap cap
)
277 static struct iommu_domain
*tegra_smmu_domain_alloc(unsigned type
)
279 struct tegra_smmu_as
*as
;
281 if (type
!= IOMMU_DOMAIN_UNMANAGED
)
284 as
= kzalloc(sizeof(*as
), GFP_KERNEL
);
288 as
->attr
= SMMU_PD_READABLE
| SMMU_PD_WRITABLE
| SMMU_PD_NONSECURE
;
290 as
->pd
= alloc_page(GFP_KERNEL
| __GFP_DMA
| __GFP_ZERO
);
296 as
->count
= kcalloc(SMMU_NUM_PDE
, sizeof(u32
), GFP_KERNEL
);
303 as
->pts
= kcalloc(SMMU_NUM_PDE
, sizeof(*as
->pts
), GFP_KERNEL
);
312 as
->domain
.geometry
.aperture_start
= 0;
313 as
->domain
.geometry
.aperture_end
= 0xffffffff;
314 as
->domain
.geometry
.force_aperture
= true;
319 static void tegra_smmu_domain_free(struct iommu_domain
*domain
)
321 struct tegra_smmu_as
*as
= to_smmu_as(domain
);
323 /* TODO: free page directory and page tables */
325 WARN_ON_ONCE(as
->use_count
);
331 static const struct tegra_smmu_swgroup
*
332 tegra_smmu_find_swgroup(struct tegra_smmu
*smmu
, unsigned int swgroup
)
334 const struct tegra_smmu_swgroup
*group
= NULL
;
337 for (i
= 0; i
< smmu
->soc
->num_swgroups
; i
++) {
338 if (smmu
->soc
->swgroups
[i
].swgroup
== swgroup
) {
339 group
= &smmu
->soc
->swgroups
[i
];
347 static void tegra_smmu_enable(struct tegra_smmu
*smmu
, unsigned int swgroup
,
350 const struct tegra_smmu_swgroup
*group
;
354 for (i
= 0; i
< smmu
->soc
->num_clients
; i
++) {
355 const struct tegra_mc_client
*client
= &smmu
->soc
->clients
[i
];
357 if (client
->swgroup
!= swgroup
)
360 value
= smmu_readl(smmu
, client
->smmu
.reg
);
361 value
|= BIT(client
->smmu
.bit
);
362 smmu_writel(smmu
, value
, client
->smmu
.reg
);
365 group
= tegra_smmu_find_swgroup(smmu
, swgroup
);
367 value
= smmu_readl(smmu
, group
->reg
);
368 value
&= ~SMMU_ASID_MASK
;
369 value
|= SMMU_ASID_VALUE(asid
);
370 value
|= SMMU_ASID_ENABLE
;
371 smmu_writel(smmu
, value
, group
->reg
);
375 static void tegra_smmu_disable(struct tegra_smmu
*smmu
, unsigned int swgroup
,
378 const struct tegra_smmu_swgroup
*group
;
382 group
= tegra_smmu_find_swgroup(smmu
, swgroup
);
384 value
= smmu_readl(smmu
, group
->reg
);
385 value
&= ~SMMU_ASID_MASK
;
386 value
|= SMMU_ASID_VALUE(asid
);
387 value
&= ~SMMU_ASID_ENABLE
;
388 smmu_writel(smmu
, value
, group
->reg
);
391 for (i
= 0; i
< smmu
->soc
->num_clients
; i
++) {
392 const struct tegra_mc_client
*client
= &smmu
->soc
->clients
[i
];
394 if (client
->swgroup
!= swgroup
)
397 value
= smmu_readl(smmu
, client
->smmu
.reg
);
398 value
&= ~BIT(client
->smmu
.bit
);
399 smmu_writel(smmu
, value
, client
->smmu
.reg
);
403 static int tegra_smmu_as_prepare(struct tegra_smmu
*smmu
,
404 struct tegra_smmu_as
*as
)
409 if (as
->use_count
> 0) {
414 as
->pd_dma
= dma_map_page(smmu
->dev
, as
->pd
, 0, SMMU_SIZE_PD
,
416 if (dma_mapping_error(smmu
->dev
, as
->pd_dma
))
419 /* We can't handle 64-bit DMA addresses */
420 if (!smmu_dma_addr_valid(smmu
, as
->pd_dma
)) {
425 err
= tegra_smmu_alloc_asid(smmu
, &as
->id
);
429 smmu_flush_ptc(smmu
, as
->pd_dma
, 0);
430 smmu_flush_tlb_asid(smmu
, as
->id
);
432 smmu_writel(smmu
, as
->id
& 0x7f, SMMU_PTB_ASID
);
433 value
= SMMU_PTB_DATA_VALUE(as
->pd_dma
, as
->attr
);
434 smmu_writel(smmu
, value
, SMMU_PTB_DATA
);
443 dma_unmap_page(smmu
->dev
, as
->pd_dma
, SMMU_SIZE_PD
, DMA_TO_DEVICE
);
447 static void tegra_smmu_as_unprepare(struct tegra_smmu
*smmu
,
448 struct tegra_smmu_as
*as
)
450 if (--as
->use_count
> 0)
453 tegra_smmu_free_asid(smmu
, as
->id
);
455 dma_unmap_page(smmu
->dev
, as
->pd_dma
, SMMU_SIZE_PD
, DMA_TO_DEVICE
);
460 static int tegra_smmu_attach_dev(struct iommu_domain
*domain
,
463 struct tegra_smmu
*smmu
= dev
->archdata
.iommu
;
464 struct tegra_smmu_as
*as
= to_smmu_as(domain
);
465 struct device_node
*np
= dev
->of_node
;
466 struct of_phandle_args args
;
467 unsigned int index
= 0;
470 while (!of_parse_phandle_with_args(np
, "iommus", "#iommu-cells", index
,
472 unsigned int swgroup
= args
.args
[0];
474 if (args
.np
!= smmu
->dev
->of_node
) {
475 of_node_put(args
.np
);
479 of_node_put(args
.np
);
481 err
= tegra_smmu_as_prepare(smmu
, as
);
485 tegra_smmu_enable(smmu
, swgroup
, as
->id
);
495 static void tegra_smmu_detach_dev(struct iommu_domain
*domain
, struct device
*dev
)
497 struct tegra_smmu_as
*as
= to_smmu_as(domain
);
498 struct device_node
*np
= dev
->of_node
;
499 struct tegra_smmu
*smmu
= as
->smmu
;
500 struct of_phandle_args args
;
501 unsigned int index
= 0;
503 while (!of_parse_phandle_with_args(np
, "iommus", "#iommu-cells", index
,
505 unsigned int swgroup
= args
.args
[0];
507 if (args
.np
!= smmu
->dev
->of_node
) {
508 of_node_put(args
.np
);
512 of_node_put(args
.np
);
514 tegra_smmu_disable(smmu
, swgroup
, as
->id
);
515 tegra_smmu_as_unprepare(smmu
, as
);
520 static void tegra_smmu_set_pde(struct tegra_smmu_as
*as
, unsigned long iova
,
523 unsigned int pd_index
= iova_pd_index(iova
);
524 struct tegra_smmu
*smmu
= as
->smmu
;
525 u32
*pd
= page_address(as
->pd
);
526 unsigned long offset
= pd_index
* sizeof(*pd
);
528 /* Set the page directory entry first */
529 pd
[pd_index
] = value
;
531 /* The flush the page directory entry from caches */
532 dma_sync_single_range_for_device(smmu
->dev
, as
->pd_dma
, offset
,
533 sizeof(*pd
), DMA_TO_DEVICE
);
535 /* And flush the iommu */
536 smmu_flush_ptc(smmu
, as
->pd_dma
, offset
);
537 smmu_flush_tlb_section(smmu
, as
->id
, iova
);
541 static u32
*tegra_smmu_pte_offset(struct page
*pt_page
, unsigned long iova
)
543 u32
*pt
= page_address(pt_page
);
545 return pt
+ iova_pt_index(iova
);
548 static u32
*tegra_smmu_pte_lookup(struct tegra_smmu_as
*as
, unsigned long iova
,
551 unsigned int pd_index
= iova_pd_index(iova
);
552 struct tegra_smmu
*smmu
= as
->smmu
;
553 struct page
*pt_page
;
556 pt_page
= as
->pts
[pd_index
];
560 pd
= page_address(as
->pd
);
561 *dmap
= smmu_pde_to_dma(smmu
, pd
[pd_index
]);
563 return tegra_smmu_pte_offset(pt_page
, iova
);
566 static u32
*as_get_pte(struct tegra_smmu_as
*as
, dma_addr_t iova
,
569 unsigned int pde
= iova_pd_index(iova
);
570 struct tegra_smmu
*smmu
= as
->smmu
;
576 page
= alloc_page(GFP_KERNEL
| __GFP_DMA
| __GFP_ZERO
);
580 dma
= dma_map_page(smmu
->dev
, page
, 0, SMMU_SIZE_PT
,
582 if (dma_mapping_error(smmu
->dev
, dma
)) {
587 if (!smmu_dma_addr_valid(smmu
, dma
)) {
588 dma_unmap_page(smmu
->dev
, dma
, SMMU_SIZE_PT
,
596 tegra_smmu_set_pde(as
, iova
, SMMU_MK_PDE(dma
, SMMU_PDE_ATTR
|
601 u32
*pd
= page_address(as
->pd
);
603 *dmap
= smmu_pde_to_dma(smmu
, pd
[pde
]);
606 return tegra_smmu_pte_offset(as
->pts
[pde
], iova
);
609 static void tegra_smmu_pte_get_use(struct tegra_smmu_as
*as
, unsigned long iova
)
611 unsigned int pd_index
= iova_pd_index(iova
);
613 as
->count
[pd_index
]++;
616 static void tegra_smmu_pte_put_use(struct tegra_smmu_as
*as
, unsigned long iova
)
618 unsigned int pde
= iova_pd_index(iova
);
619 struct page
*page
= as
->pts
[pde
];
622 * When no entries in this page table are used anymore, return the
623 * memory page to the system.
625 if (--as
->count
[pde
] == 0) {
626 struct tegra_smmu
*smmu
= as
->smmu
;
627 u32
*pd
= page_address(as
->pd
);
628 dma_addr_t pte_dma
= smmu_pde_to_dma(smmu
, pd
[pde
]);
630 tegra_smmu_set_pde(as
, iova
, 0);
632 dma_unmap_page(smmu
->dev
, pte_dma
, SMMU_SIZE_PT
, DMA_TO_DEVICE
);
638 static void tegra_smmu_set_pte(struct tegra_smmu_as
*as
, unsigned long iova
,
639 u32
*pte
, dma_addr_t pte_dma
, u32 val
)
641 struct tegra_smmu
*smmu
= as
->smmu
;
642 unsigned long offset
= offset_in_page(pte
);
646 dma_sync_single_range_for_device(smmu
->dev
, pte_dma
, offset
,
648 smmu_flush_ptc(smmu
, pte_dma
, offset
);
649 smmu_flush_tlb_group(smmu
, as
->id
, iova
);
653 static int tegra_smmu_map(struct iommu_domain
*domain
, unsigned long iova
,
654 phys_addr_t paddr
, size_t size
, int prot
)
656 struct tegra_smmu_as
*as
= to_smmu_as(domain
);
661 pte
= as_get_pte(as
, iova
, &pte_dma
);
665 /* If we aren't overwriting a pre-existing entry, increment use */
667 tegra_smmu_pte_get_use(as
, iova
);
669 pte_attrs
= SMMU_PTE_NONSECURE
;
671 if (prot
& IOMMU_READ
)
672 pte_attrs
|= SMMU_PTE_READABLE
;
674 if (prot
& IOMMU_WRITE
)
675 pte_attrs
|= SMMU_PTE_WRITABLE
;
677 tegra_smmu_set_pte(as
, iova
, pte
, pte_dma
,
678 __phys_to_pfn(paddr
) | pte_attrs
);
683 static size_t tegra_smmu_unmap(struct iommu_domain
*domain
, unsigned long iova
,
684 size_t size
, struct iommu_iotlb_gather
*gather
)
686 struct tegra_smmu_as
*as
= to_smmu_as(domain
);
690 pte
= tegra_smmu_pte_lookup(as
, iova
, &pte_dma
);
694 tegra_smmu_set_pte(as
, iova
, pte
, pte_dma
, 0);
695 tegra_smmu_pte_put_use(as
, iova
);
700 static phys_addr_t
tegra_smmu_iova_to_phys(struct iommu_domain
*domain
,
703 struct tegra_smmu_as
*as
= to_smmu_as(domain
);
708 pte
= tegra_smmu_pte_lookup(as
, iova
, &pte_dma
);
712 pfn
= *pte
& as
->smmu
->pfn_mask
;
714 return PFN_PHYS(pfn
);
717 static struct tegra_smmu
*tegra_smmu_find(struct device_node
*np
)
719 struct platform_device
*pdev
;
722 pdev
= of_find_device_by_node(np
);
726 mc
= platform_get_drvdata(pdev
);
733 static int tegra_smmu_configure(struct tegra_smmu
*smmu
, struct device
*dev
,
734 struct of_phandle_args
*args
)
736 const struct iommu_ops
*ops
= smmu
->iommu
.ops
;
739 err
= iommu_fwspec_init(dev
, &dev
->of_node
->fwnode
, ops
);
741 dev_err(dev
, "failed to initialize fwspec: %d\n", err
);
745 err
= ops
->of_xlate(dev
, args
);
747 dev_err(dev
, "failed to parse SW group ID: %d\n", err
);
748 iommu_fwspec_free(dev
);
755 static int tegra_smmu_add_device(struct device
*dev
)
757 struct device_node
*np
= dev
->of_node
;
758 struct tegra_smmu
*smmu
= NULL
;
759 struct iommu_group
*group
;
760 struct of_phandle_args args
;
761 unsigned int index
= 0;
764 while (of_parse_phandle_with_args(np
, "iommus", "#iommu-cells", index
,
766 smmu
= tegra_smmu_find(args
.np
);
768 err
= tegra_smmu_configure(smmu
, dev
, &args
);
769 of_node_put(args
.np
);
775 * Only a single IOMMU master interface is currently
776 * supported by the Linux kernel, so abort after the
779 dev
->archdata
.iommu
= smmu
;
781 iommu_device_link(&smmu
->iommu
, dev
);
786 of_node_put(args
.np
);
793 group
= iommu_group_get_for_dev(dev
);
795 return PTR_ERR(group
);
797 iommu_group_put(group
);
802 static void tegra_smmu_remove_device(struct device
*dev
)
804 struct tegra_smmu
*smmu
= dev
->archdata
.iommu
;
807 iommu_device_unlink(&smmu
->iommu
, dev
);
809 dev
->archdata
.iommu
= NULL
;
810 iommu_group_remove_device(dev
);
813 static const struct tegra_smmu_group_soc
*
814 tegra_smmu_find_group(struct tegra_smmu
*smmu
, unsigned int swgroup
)
818 for (i
= 0; i
< smmu
->soc
->num_groups
; i
++)
819 for (j
= 0; j
< smmu
->soc
->groups
[i
].num_swgroups
; j
++)
820 if (smmu
->soc
->groups
[i
].swgroups
[j
] == swgroup
)
821 return &smmu
->soc
->groups
[i
];
826 static struct iommu_group
*tegra_smmu_group_get(struct tegra_smmu
*smmu
,
827 unsigned int swgroup
)
829 const struct tegra_smmu_group_soc
*soc
;
830 struct tegra_smmu_group
*group
;
832 soc
= tegra_smmu_find_group(smmu
, swgroup
);
836 mutex_lock(&smmu
->lock
);
838 list_for_each_entry(group
, &smmu
->groups
, list
)
839 if (group
->soc
== soc
) {
840 mutex_unlock(&smmu
->lock
);
844 group
= devm_kzalloc(smmu
->dev
, sizeof(*group
), GFP_KERNEL
);
846 mutex_unlock(&smmu
->lock
);
850 INIT_LIST_HEAD(&group
->list
);
853 group
->group
= iommu_group_alloc();
854 if (IS_ERR(group
->group
)) {
855 devm_kfree(smmu
->dev
, group
);
856 mutex_unlock(&smmu
->lock
);
860 list_add_tail(&group
->list
, &smmu
->groups
);
861 mutex_unlock(&smmu
->lock
);
866 static struct iommu_group
*tegra_smmu_device_group(struct device
*dev
)
868 struct iommu_fwspec
*fwspec
= dev_iommu_fwspec_get(dev
);
869 struct tegra_smmu
*smmu
= dev
->archdata
.iommu
;
870 struct iommu_group
*group
;
872 group
= tegra_smmu_group_get(smmu
, fwspec
->ids
[0]);
874 group
= generic_device_group(dev
);
879 static int tegra_smmu_of_xlate(struct device
*dev
,
880 struct of_phandle_args
*args
)
882 u32 id
= args
->args
[0];
884 return iommu_fwspec_add_ids(dev
, &id
, 1);
887 static const struct iommu_ops tegra_smmu_ops
= {
888 .capable
= tegra_smmu_capable
,
889 .domain_alloc
= tegra_smmu_domain_alloc
,
890 .domain_free
= tegra_smmu_domain_free
,
891 .attach_dev
= tegra_smmu_attach_dev
,
892 .detach_dev
= tegra_smmu_detach_dev
,
893 .add_device
= tegra_smmu_add_device
,
894 .remove_device
= tegra_smmu_remove_device
,
895 .device_group
= tegra_smmu_device_group
,
896 .map
= tegra_smmu_map
,
897 .unmap
= tegra_smmu_unmap
,
898 .iova_to_phys
= tegra_smmu_iova_to_phys
,
899 .of_xlate
= tegra_smmu_of_xlate
,
900 .pgsize_bitmap
= SZ_4K
,
903 static void tegra_smmu_ahb_enable(void)
905 static const struct of_device_id ahb_match
[] = {
906 { .compatible
= "nvidia,tegra30-ahb", },
909 struct device_node
*ahb
;
911 ahb
= of_find_matching_node(NULL
, ahb_match
);
913 tegra_ahb_enable_smmu(ahb
);
918 static int tegra_smmu_swgroups_show(struct seq_file
*s
, void *data
)
920 struct tegra_smmu
*smmu
= s
->private;
924 seq_printf(s
, "swgroup enabled ASID\n");
925 seq_printf(s
, "------------------------\n");
927 for (i
= 0; i
< smmu
->soc
->num_swgroups
; i
++) {
928 const struct tegra_smmu_swgroup
*group
= &smmu
->soc
->swgroups
[i
];
932 value
= smmu_readl(smmu
, group
->reg
);
934 if (value
& SMMU_ASID_ENABLE
)
939 asid
= value
& SMMU_ASID_MASK
;
941 seq_printf(s
, "%-9s %-7s %#04x\n", group
->name
, status
,
948 DEFINE_SHOW_ATTRIBUTE(tegra_smmu_swgroups
);
950 static int tegra_smmu_clients_show(struct seq_file
*s
, void *data
)
952 struct tegra_smmu
*smmu
= s
->private;
956 seq_printf(s
, "client enabled\n");
957 seq_printf(s
, "--------------------\n");
959 for (i
= 0; i
< smmu
->soc
->num_clients
; i
++) {
960 const struct tegra_mc_client
*client
= &smmu
->soc
->clients
[i
];
963 value
= smmu_readl(smmu
, client
->smmu
.reg
);
965 if (value
& BIT(client
->smmu
.bit
))
970 seq_printf(s
, "%-12s %s\n", client
->name
, status
);
976 DEFINE_SHOW_ATTRIBUTE(tegra_smmu_clients
);
978 static void tegra_smmu_debugfs_init(struct tegra_smmu
*smmu
)
980 smmu
->debugfs
= debugfs_create_dir("smmu", NULL
);
984 debugfs_create_file("swgroups", S_IRUGO
, smmu
->debugfs
, smmu
,
985 &tegra_smmu_swgroups_fops
);
986 debugfs_create_file("clients", S_IRUGO
, smmu
->debugfs
, smmu
,
987 &tegra_smmu_clients_fops
);
990 static void tegra_smmu_debugfs_exit(struct tegra_smmu
*smmu
)
992 debugfs_remove_recursive(smmu
->debugfs
);
995 struct tegra_smmu
*tegra_smmu_probe(struct device
*dev
,
996 const struct tegra_smmu_soc
*soc
,
999 struct tegra_smmu
*smmu
;
1004 smmu
= devm_kzalloc(dev
, sizeof(*smmu
), GFP_KERNEL
);
1006 return ERR_PTR(-ENOMEM
);
1009 * This is a bit of a hack. Ideally we'd want to simply return this
1010 * value. However the IOMMU registration process will attempt to add
1011 * all devices to the IOMMU when bus_set_iommu() is called. In order
1012 * not to rely on global variables to track the IOMMU instance, we
1013 * set it here so that it can be looked up from the .add_device()
1014 * callback via the IOMMU device's .drvdata field.
1018 size
= BITS_TO_LONGS(soc
->num_asids
) * sizeof(long);
1020 smmu
->asids
= devm_kzalloc(dev
, size
, GFP_KERNEL
);
1022 return ERR_PTR(-ENOMEM
);
1024 INIT_LIST_HEAD(&smmu
->groups
);
1025 mutex_init(&smmu
->lock
);
1027 smmu
->regs
= mc
->regs
;
1032 smmu
->pfn_mask
= BIT_MASK(mc
->soc
->num_address_bits
- PAGE_SHIFT
) - 1;
1033 dev_dbg(dev
, "address bits: %u, PFN mask: %#lx\n",
1034 mc
->soc
->num_address_bits
, smmu
->pfn_mask
);
1035 smmu
->tlb_mask
= (smmu
->soc
->num_tlb_lines
<< 1) - 1;
1036 dev_dbg(dev
, "TLB lines: %u, mask: %#lx\n", smmu
->soc
->num_tlb_lines
,
1039 value
= SMMU_PTC_CONFIG_ENABLE
| SMMU_PTC_CONFIG_INDEX_MAP(0x3f);
1041 if (soc
->supports_request_limit
)
1042 value
|= SMMU_PTC_CONFIG_REQ_LIMIT(8);
1044 smmu_writel(smmu
, value
, SMMU_PTC_CONFIG
);
1046 value
= SMMU_TLB_CONFIG_HIT_UNDER_MISS
|
1047 SMMU_TLB_CONFIG_ACTIVE_LINES(smmu
);
1049 if (soc
->supports_round_robin_arbitration
)
1050 value
|= SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION
;
1052 smmu_writel(smmu
, value
, SMMU_TLB_CONFIG
);
1054 smmu_flush_ptc_all(smmu
);
1055 smmu_flush_tlb(smmu
);
1056 smmu_writel(smmu
, SMMU_CONFIG_ENABLE
, SMMU_CONFIG
);
1059 tegra_smmu_ahb_enable();
1061 err
= iommu_device_sysfs_add(&smmu
->iommu
, dev
, NULL
, dev_name(dev
));
1063 return ERR_PTR(err
);
1065 iommu_device_set_ops(&smmu
->iommu
, &tegra_smmu_ops
);
1066 iommu_device_set_fwnode(&smmu
->iommu
, dev
->fwnode
);
1068 err
= iommu_device_register(&smmu
->iommu
);
1070 iommu_device_sysfs_remove(&smmu
->iommu
);
1071 return ERR_PTR(err
);
1074 err
= bus_set_iommu(&platform_bus_type
, &tegra_smmu_ops
);
1076 iommu_device_unregister(&smmu
->iommu
);
1077 iommu_device_sysfs_remove(&smmu
->iommu
);
1078 return ERR_PTR(err
);
1081 if (IS_ENABLED(CONFIG_DEBUG_FS
))
1082 tegra_smmu_debugfs_init(smmu
);
1087 void tegra_smmu_remove(struct tegra_smmu
*smmu
)
1089 iommu_device_unregister(&smmu
->iommu
);
1090 iommu_device_sysfs_remove(&smmu
->iommu
);
1092 if (IS_ENABLED(CONFIG_DEBUG_FS
))
1093 tegra_smmu_debugfs_exit(smmu
);