1 // SPDX-License-Identifier: GPL-2.0-only
4 * Copyright (C) 2013 Freescale Semiconductor, Inc.
5 * Author: Varun Sethi <varun.sethi@freescale.com>
8 #define pr_fmt(fmt) "fsl-pamu-domain: %s: " fmt, __func__
10 #include "fsl_pamu_domain.h"
12 #include <sysdev/fsl_pci.h>
15 * Global spinlock that needs to be held while
18 static DEFINE_SPINLOCK(iommu_lock
);
20 static struct kmem_cache
*fsl_pamu_domain_cache
;
21 static struct kmem_cache
*iommu_devinfo_cache
;
22 static DEFINE_SPINLOCK(device_domain_lock
);
24 struct iommu_device pamu_iommu
; /* IOMMU core code handle */
26 static struct fsl_dma_domain
*to_fsl_dma_domain(struct iommu_domain
*dom
)
28 return container_of(dom
, struct fsl_dma_domain
, iommu_domain
);
31 static int __init
iommu_init_mempool(void)
33 fsl_pamu_domain_cache
= kmem_cache_create("fsl_pamu_domain",
34 sizeof(struct fsl_dma_domain
),
38 if (!fsl_pamu_domain_cache
) {
39 pr_debug("Couldn't create fsl iommu_domain cache\n");
43 iommu_devinfo_cache
= kmem_cache_create("iommu_devinfo",
44 sizeof(struct device_domain_info
),
48 if (!iommu_devinfo_cache
) {
49 pr_debug("Couldn't create devinfo cache\n");
50 kmem_cache_destroy(fsl_pamu_domain_cache
);
57 static phys_addr_t
get_phys_addr(struct fsl_dma_domain
*dma_domain
, dma_addr_t iova
)
59 u32 win_cnt
= dma_domain
->win_cnt
;
60 struct dma_window
*win_ptr
= &dma_domain
->win_arr
[0];
61 struct iommu_domain_geometry
*geom
;
63 geom
= &dma_domain
->iommu_domain
.geometry
;
65 if (!win_cnt
|| !dma_domain
->geom_size
) {
66 pr_debug("Number of windows/geometry not configured for the domain\n");
72 dma_addr_t subwin_iova
;
75 subwin_size
= dma_domain
->geom_size
>> ilog2(win_cnt
);
76 subwin_iova
= iova
& ~(subwin_size
- 1);
77 wnd
= (subwin_iova
- geom
->aperture_start
) >> ilog2(subwin_size
);
78 win_ptr
= &dma_domain
->win_arr
[wnd
];
82 return win_ptr
->paddr
+ (iova
& (win_ptr
->size
- 1));
87 static int map_subwins(int liodn
, struct fsl_dma_domain
*dma_domain
)
89 struct dma_window
*sub_win_ptr
= &dma_domain
->win_arr
[0];
91 unsigned long rpn
, flags
;
93 for (i
= 0; i
< dma_domain
->win_cnt
; i
++) {
94 if (sub_win_ptr
[i
].valid
) {
95 rpn
= sub_win_ptr
[i
].paddr
>> PAMU_PAGE_SHIFT
;
96 spin_lock_irqsave(&iommu_lock
, flags
);
97 ret
= pamu_config_spaace(liodn
, dma_domain
->win_cnt
, i
,
101 dma_domain
->snoop_id
,
102 dma_domain
->stash_id
,
104 sub_win_ptr
[i
].prot
);
105 spin_unlock_irqrestore(&iommu_lock
, flags
);
107 pr_debug("SPAACE configuration failed for liodn %d\n",
117 static int map_win(int liodn
, struct fsl_dma_domain
*dma_domain
)
120 struct dma_window
*wnd
= &dma_domain
->win_arr
[0];
121 phys_addr_t wnd_addr
= dma_domain
->iommu_domain
.geometry
.aperture_start
;
124 spin_lock_irqsave(&iommu_lock
, flags
);
125 ret
= pamu_config_ppaace(liodn
, wnd_addr
,
128 wnd
->paddr
>> PAMU_PAGE_SHIFT
,
129 dma_domain
->snoop_id
, dma_domain
->stash_id
,
131 spin_unlock_irqrestore(&iommu_lock
, flags
);
133 pr_debug("PAACE configuration failed for liodn %d\n", liodn
);
138 /* Map the DMA window corresponding to the LIODN */
139 static int map_liodn(int liodn
, struct fsl_dma_domain
*dma_domain
)
141 if (dma_domain
->win_cnt
> 1)
142 return map_subwins(liodn
, dma_domain
);
144 return map_win(liodn
, dma_domain
);
147 /* Update window/subwindow mapping for the LIODN */
148 static int update_liodn(int liodn
, struct fsl_dma_domain
*dma_domain
, u32 wnd_nr
)
151 struct dma_window
*wnd
= &dma_domain
->win_arr
[wnd_nr
];
154 spin_lock_irqsave(&iommu_lock
, flags
);
155 if (dma_domain
->win_cnt
> 1) {
156 ret
= pamu_config_spaace(liodn
, dma_domain
->win_cnt
, wnd_nr
,
159 wnd
->paddr
>> PAMU_PAGE_SHIFT
,
160 dma_domain
->snoop_id
,
161 dma_domain
->stash_id
,
162 (wnd_nr
> 0) ? 1 : 0,
165 pr_debug("Subwindow reconfiguration failed for liodn %d\n",
168 phys_addr_t wnd_addr
;
170 wnd_addr
= dma_domain
->iommu_domain
.geometry
.aperture_start
;
172 ret
= pamu_config_ppaace(liodn
, wnd_addr
,
175 wnd
->paddr
>> PAMU_PAGE_SHIFT
,
176 dma_domain
->snoop_id
, dma_domain
->stash_id
,
179 pr_debug("Window reconfiguration failed for liodn %d\n",
183 spin_unlock_irqrestore(&iommu_lock
, flags
);
188 static int update_liodn_stash(int liodn
, struct fsl_dma_domain
*dma_domain
,
194 spin_lock_irqsave(&iommu_lock
, flags
);
195 if (!dma_domain
->win_arr
) {
196 pr_debug("Windows not configured, stash destination update failed for liodn %d\n",
198 spin_unlock_irqrestore(&iommu_lock
, flags
);
202 for (i
= 0; i
< dma_domain
->win_cnt
; i
++) {
203 ret
= pamu_update_paace_stash(liodn
, i
, val
);
205 pr_debug("Failed to update SPAACE %d field for liodn %d\n ",
207 spin_unlock_irqrestore(&iommu_lock
, flags
);
212 spin_unlock_irqrestore(&iommu_lock
, flags
);
217 /* Set the geometry parameters for a LIODN */
218 static int pamu_set_liodn(int liodn
, struct device
*dev
,
219 struct fsl_dma_domain
*dma_domain
,
220 struct iommu_domain_geometry
*geom_attr
,
223 phys_addr_t window_addr
, window_size
;
224 phys_addr_t subwin_size
;
226 u32 omi_index
= ~(u32
)0;
230 * Configure the omi_index at the geometry setup time.
231 * This is a static value which depends on the type of
232 * device and would not change thereafter.
234 get_ome_index(&omi_index
, dev
);
236 window_addr
= geom_attr
->aperture_start
;
237 window_size
= dma_domain
->geom_size
;
239 spin_lock_irqsave(&iommu_lock
, flags
);
240 ret
= pamu_disable_liodn(liodn
);
242 ret
= pamu_config_ppaace(liodn
, window_addr
, window_size
, omi_index
,
243 0, dma_domain
->snoop_id
,
244 dma_domain
->stash_id
, win_cnt
, 0);
245 spin_unlock_irqrestore(&iommu_lock
, flags
);
247 pr_debug("PAACE configuration failed for liodn %d, win_cnt =%d\n",
253 subwin_size
= window_size
>> ilog2(win_cnt
);
254 for (i
= 0; i
< win_cnt
; i
++) {
255 spin_lock_irqsave(&iommu_lock
, flags
);
256 ret
= pamu_disable_spaace(liodn
, i
);
258 ret
= pamu_config_spaace(liodn
, win_cnt
, i
,
259 subwin_size
, omi_index
,
260 0, dma_domain
->snoop_id
,
261 dma_domain
->stash_id
,
263 spin_unlock_irqrestore(&iommu_lock
, flags
);
265 pr_debug("SPAACE configuration failed for liodn %d\n",
275 static int check_size(u64 size
, dma_addr_t iova
)
278 * Size must be a power of two and at least be equal
281 if ((size
& (size
- 1)) || size
< PAMU_PAGE_SIZE
) {
282 pr_debug("Size too small or not a power of two\n");
286 /* iova must be page size aligned */
287 if (iova
& (size
- 1)) {
288 pr_debug("Address is not aligned with window size\n");
295 static struct fsl_dma_domain
*iommu_alloc_dma_domain(void)
297 struct fsl_dma_domain
*domain
;
299 domain
= kmem_cache_zalloc(fsl_pamu_domain_cache
, GFP_KERNEL
);
303 domain
->stash_id
= ~(u32
)0;
304 domain
->snoop_id
= ~(u32
)0;
305 domain
->win_cnt
= pamu_get_max_subwin_cnt();
306 domain
->geom_size
= 0;
308 INIT_LIST_HEAD(&domain
->devices
);
310 spin_lock_init(&domain
->domain_lock
);
315 static void remove_device_ref(struct device_domain_info
*info
, u32 win_cnt
)
319 list_del(&info
->link
);
320 spin_lock_irqsave(&iommu_lock
, flags
);
322 pamu_free_subwins(info
->liodn
);
323 pamu_disable_liodn(info
->liodn
);
324 spin_unlock_irqrestore(&iommu_lock
, flags
);
325 spin_lock_irqsave(&device_domain_lock
, flags
);
326 dev_iommu_priv_set(info
->dev
, NULL
);
327 kmem_cache_free(iommu_devinfo_cache
, info
);
328 spin_unlock_irqrestore(&device_domain_lock
, flags
);
331 static void detach_device(struct device
*dev
, struct fsl_dma_domain
*dma_domain
)
333 struct device_domain_info
*info
, *tmp
;
336 spin_lock_irqsave(&dma_domain
->domain_lock
, flags
);
337 /* Remove the device from the domain device list */
338 list_for_each_entry_safe(info
, tmp
, &dma_domain
->devices
, link
) {
339 if (!dev
|| (info
->dev
== dev
))
340 remove_device_ref(info
, dma_domain
->win_cnt
);
342 spin_unlock_irqrestore(&dma_domain
->domain_lock
, flags
);
345 static void attach_device(struct fsl_dma_domain
*dma_domain
, int liodn
, struct device
*dev
)
347 struct device_domain_info
*info
, *old_domain_info
;
350 spin_lock_irqsave(&device_domain_lock
, flags
);
352 * Check here if the device is already attached to domain or not.
353 * If the device is already attached to a domain detach it.
355 old_domain_info
= dev_iommu_priv_get(dev
);
356 if (old_domain_info
&& old_domain_info
->domain
!= dma_domain
) {
357 spin_unlock_irqrestore(&device_domain_lock
, flags
);
358 detach_device(dev
, old_domain_info
->domain
);
359 spin_lock_irqsave(&device_domain_lock
, flags
);
362 info
= kmem_cache_zalloc(iommu_devinfo_cache
, GFP_ATOMIC
);
366 info
->domain
= dma_domain
;
368 list_add(&info
->link
, &dma_domain
->devices
);
370 * In case of devices with multiple LIODNs just store
371 * the info for the first LIODN as all
372 * LIODNs share the same domain
374 if (!dev_iommu_priv_get(dev
))
375 dev_iommu_priv_set(dev
, info
);
376 spin_unlock_irqrestore(&device_domain_lock
, flags
);
379 static phys_addr_t
fsl_pamu_iova_to_phys(struct iommu_domain
*domain
,
382 struct fsl_dma_domain
*dma_domain
= to_fsl_dma_domain(domain
);
384 if (iova
< domain
->geometry
.aperture_start
||
385 iova
> domain
->geometry
.aperture_end
)
388 return get_phys_addr(dma_domain
, iova
);
391 static bool fsl_pamu_capable(enum iommu_cap cap
)
393 return cap
== IOMMU_CAP_CACHE_COHERENCY
;
396 static void fsl_pamu_domain_free(struct iommu_domain
*domain
)
398 struct fsl_dma_domain
*dma_domain
= to_fsl_dma_domain(domain
);
400 /* remove all the devices from the device list */
401 detach_device(NULL
, dma_domain
);
403 dma_domain
->enabled
= 0;
404 dma_domain
->mapped
= 0;
406 kmem_cache_free(fsl_pamu_domain_cache
, dma_domain
);
409 static struct iommu_domain
*fsl_pamu_domain_alloc(unsigned type
)
411 struct fsl_dma_domain
*dma_domain
;
413 if (type
!= IOMMU_DOMAIN_UNMANAGED
)
416 dma_domain
= iommu_alloc_dma_domain();
418 pr_debug("dma_domain allocation failed\n");
421 /* defaul geometry 64 GB i.e. maximum system address */
422 dma_domain
->iommu_domain
. geometry
.aperture_start
= 0;
423 dma_domain
->iommu_domain
.geometry
.aperture_end
= (1ULL << 36) - 1;
424 dma_domain
->iommu_domain
.geometry
.force_aperture
= true;
426 return &dma_domain
->iommu_domain
;
429 /* Configure geometry settings for all LIODNs associated with domain */
430 static int pamu_set_domain_geometry(struct fsl_dma_domain
*dma_domain
,
431 struct iommu_domain_geometry
*geom_attr
,
434 struct device_domain_info
*info
;
437 list_for_each_entry(info
, &dma_domain
->devices
, link
) {
438 ret
= pamu_set_liodn(info
->liodn
, info
->dev
, dma_domain
,
447 /* Update stash destination for all LIODNs associated with the domain */
448 static int update_domain_stash(struct fsl_dma_domain
*dma_domain
, u32 val
)
450 struct device_domain_info
*info
;
453 list_for_each_entry(info
, &dma_domain
->devices
, link
) {
454 ret
= update_liodn_stash(info
->liodn
, dma_domain
, val
);
462 /* Update domain mappings for all LIODNs associated with the domain */
463 static int update_domain_mapping(struct fsl_dma_domain
*dma_domain
, u32 wnd_nr
)
465 struct device_domain_info
*info
;
468 list_for_each_entry(info
, &dma_domain
->devices
, link
) {
469 ret
= update_liodn(info
->liodn
, dma_domain
, wnd_nr
);
476 static int disable_domain_win(struct fsl_dma_domain
*dma_domain
, u32 wnd_nr
)
478 struct device_domain_info
*info
;
481 list_for_each_entry(info
, &dma_domain
->devices
, link
) {
482 if (dma_domain
->win_cnt
== 1 && dma_domain
->enabled
) {
483 ret
= pamu_disable_liodn(info
->liodn
);
485 dma_domain
->enabled
= 0;
487 ret
= pamu_disable_spaace(info
->liodn
, wnd_nr
);
494 static void fsl_pamu_window_disable(struct iommu_domain
*domain
, u32 wnd_nr
)
496 struct fsl_dma_domain
*dma_domain
= to_fsl_dma_domain(domain
);
500 spin_lock_irqsave(&dma_domain
->domain_lock
, flags
);
501 if (!dma_domain
->win_arr
) {
502 pr_debug("Number of windows not configured\n");
503 spin_unlock_irqrestore(&dma_domain
->domain_lock
, flags
);
507 if (wnd_nr
>= dma_domain
->win_cnt
) {
508 pr_debug("Invalid window index\n");
509 spin_unlock_irqrestore(&dma_domain
->domain_lock
, flags
);
513 if (dma_domain
->win_arr
[wnd_nr
].valid
) {
514 ret
= disable_domain_win(dma_domain
, wnd_nr
);
516 dma_domain
->win_arr
[wnd_nr
].valid
= 0;
517 dma_domain
->mapped
--;
521 spin_unlock_irqrestore(&dma_domain
->domain_lock
, flags
);
524 static int fsl_pamu_window_enable(struct iommu_domain
*domain
, u32 wnd_nr
,
525 phys_addr_t paddr
, u64 size
, int prot
)
527 struct fsl_dma_domain
*dma_domain
= to_fsl_dma_domain(domain
);
528 struct dma_window
*wnd
;
534 if (prot
& IOMMU_READ
)
535 pamu_prot
|= PAACE_AP_PERMS_QUERY
;
536 if (prot
& IOMMU_WRITE
)
537 pamu_prot
|= PAACE_AP_PERMS_UPDATE
;
539 spin_lock_irqsave(&dma_domain
->domain_lock
, flags
);
540 if (!dma_domain
->win_arr
) {
541 pr_debug("Number of windows not configured\n");
542 spin_unlock_irqrestore(&dma_domain
->domain_lock
, flags
);
546 if (wnd_nr
>= dma_domain
->win_cnt
) {
547 pr_debug("Invalid window index\n");
548 spin_unlock_irqrestore(&dma_domain
->domain_lock
, flags
);
552 win_size
= dma_domain
->geom_size
>> ilog2(dma_domain
->win_cnt
);
553 if (size
> win_size
) {
554 pr_debug("Invalid window size\n");
555 spin_unlock_irqrestore(&dma_domain
->domain_lock
, flags
);
559 if (dma_domain
->win_cnt
== 1) {
560 if (dma_domain
->enabled
) {
561 pr_debug("Disable the window before updating the mapping\n");
562 spin_unlock_irqrestore(&dma_domain
->domain_lock
, flags
);
566 ret
= check_size(size
, domain
->geometry
.aperture_start
);
568 pr_debug("Aperture start not aligned to the size\n");
569 spin_unlock_irqrestore(&dma_domain
->domain_lock
, flags
);
574 wnd
= &dma_domain
->win_arr
[wnd_nr
];
578 wnd
->prot
= pamu_prot
;
580 ret
= update_domain_mapping(dma_domain
, wnd_nr
);
583 dma_domain
->mapped
++;
586 pr_debug("Disable the window before updating the mapping\n");
590 spin_unlock_irqrestore(&dma_domain
->domain_lock
, flags
);
596 * Attach the LIODN to the DMA domain and configure the geometry
597 * and window mappings.
599 static int handle_attach_device(struct fsl_dma_domain
*dma_domain
,
600 struct device
*dev
, const u32
*liodn
,
604 struct iommu_domain
*domain
= &dma_domain
->iommu_domain
;
608 spin_lock_irqsave(&dma_domain
->domain_lock
, flags
);
609 for (i
= 0; i
< num
; i
++) {
610 /* Ensure that LIODN value is valid */
611 if (liodn
[i
] >= PAACE_NUMBER_ENTRIES
) {
612 pr_debug("Invalid liodn %d, attach device failed for %pOF\n",
613 liodn
[i
], dev
->of_node
);
618 attach_device(dma_domain
, liodn
[i
], dev
);
620 * Check if geometry has already been configured
621 * for the domain. If yes, set the geometry for
624 if (dma_domain
->win_arr
) {
625 u32 win_cnt
= dma_domain
->win_cnt
> 1 ? dma_domain
->win_cnt
: 0;
627 ret
= pamu_set_liodn(liodn
[i
], dev
, dma_domain
,
628 &domain
->geometry
, win_cnt
);
631 if (dma_domain
->mapped
) {
633 * Create window/subwindow mapping for
636 ret
= map_liodn(liodn
[i
], dma_domain
);
642 spin_unlock_irqrestore(&dma_domain
->domain_lock
, flags
);
647 static int fsl_pamu_attach_device(struct iommu_domain
*domain
,
650 struct fsl_dma_domain
*dma_domain
= to_fsl_dma_domain(domain
);
654 struct pci_dev
*pdev
= NULL
;
655 struct pci_controller
*pci_ctl
;
658 * Use LIODN of the PCI controller while attaching a
661 if (dev_is_pci(dev
)) {
662 pdev
= to_pci_dev(dev
);
663 pci_ctl
= pci_bus_to_host(pdev
->bus
);
665 * make dev point to pci controller device
666 * so we can get the LIODN programmed by
669 dev
= pci_ctl
->parent
;
672 liodn
= of_get_property(dev
->of_node
, "fsl,liodn", &len
);
674 liodn_cnt
= len
/ sizeof(u32
);
675 ret
= handle_attach_device(dma_domain
, dev
, liodn
, liodn_cnt
);
677 pr_debug("missing fsl,liodn property at %pOF\n", dev
->of_node
);
684 static void fsl_pamu_detach_device(struct iommu_domain
*domain
,
687 struct fsl_dma_domain
*dma_domain
= to_fsl_dma_domain(domain
);
690 struct pci_dev
*pdev
= NULL
;
691 struct pci_controller
*pci_ctl
;
694 * Use LIODN of the PCI controller while detaching a
697 if (dev_is_pci(dev
)) {
698 pdev
= to_pci_dev(dev
);
699 pci_ctl
= pci_bus_to_host(pdev
->bus
);
701 * make dev point to pci controller device
702 * so we can get the LIODN programmed by
705 dev
= pci_ctl
->parent
;
708 prop
= of_get_property(dev
->of_node
, "fsl,liodn", &len
);
710 detach_device(dev
, dma_domain
);
712 pr_debug("missing fsl,liodn property at %pOF\n", dev
->of_node
);
715 static int configure_domain_geometry(struct iommu_domain
*domain
, void *data
)
717 struct iommu_domain_geometry
*geom_attr
= data
;
718 struct fsl_dma_domain
*dma_domain
= to_fsl_dma_domain(domain
);
719 dma_addr_t geom_size
;
722 geom_size
= geom_attr
->aperture_end
- geom_attr
->aperture_start
+ 1;
724 * Sanity check the geometry size. Also, we do not support
725 * DMA outside of the geometry.
727 if (check_size(geom_size
, geom_attr
->aperture_start
) ||
728 !geom_attr
->force_aperture
) {
729 pr_debug("Invalid PAMU geometry attributes\n");
733 spin_lock_irqsave(&dma_domain
->domain_lock
, flags
);
734 if (dma_domain
->enabled
) {
735 pr_debug("Can't set geometry attributes as domain is active\n");
736 spin_unlock_irqrestore(&dma_domain
->domain_lock
, flags
);
740 /* Copy the domain geometry information */
741 memcpy(&domain
->geometry
, geom_attr
,
742 sizeof(struct iommu_domain_geometry
));
743 dma_domain
->geom_size
= geom_size
;
745 spin_unlock_irqrestore(&dma_domain
->domain_lock
, flags
);
750 /* Set the domain stash attribute */
751 static int configure_domain_stash(struct fsl_dma_domain
*dma_domain
, void *data
)
753 struct pamu_stash_attribute
*stash_attr
= data
;
757 spin_lock_irqsave(&dma_domain
->domain_lock
, flags
);
759 memcpy(&dma_domain
->dma_stash
, stash_attr
,
760 sizeof(struct pamu_stash_attribute
));
762 dma_domain
->stash_id
= get_stash_id(stash_attr
->cache
,
764 if (dma_domain
->stash_id
== ~(u32
)0) {
765 pr_debug("Invalid stash attributes\n");
766 spin_unlock_irqrestore(&dma_domain
->domain_lock
, flags
);
770 ret
= update_domain_stash(dma_domain
, dma_domain
->stash_id
);
772 spin_unlock_irqrestore(&dma_domain
->domain_lock
, flags
);
777 /* Configure domain dma state i.e. enable/disable DMA */
778 static int configure_domain_dma_state(struct fsl_dma_domain
*dma_domain
, bool enable
)
780 struct device_domain_info
*info
;
784 spin_lock_irqsave(&dma_domain
->domain_lock
, flags
);
786 if (enable
&& !dma_domain
->mapped
) {
787 pr_debug("Can't enable DMA domain without valid mapping\n");
788 spin_unlock_irqrestore(&dma_domain
->domain_lock
, flags
);
792 dma_domain
->enabled
= enable
;
793 list_for_each_entry(info
, &dma_domain
->devices
, link
) {
794 ret
= (enable
) ? pamu_enable_liodn(info
->liodn
) :
795 pamu_disable_liodn(info
->liodn
);
797 pr_debug("Unable to set dma state for liodn %d",
800 spin_unlock_irqrestore(&dma_domain
->domain_lock
, flags
);
805 static int fsl_pamu_set_windows(struct iommu_domain
*domain
, u32 w_count
)
807 struct fsl_dma_domain
*dma_domain
= to_fsl_dma_domain(domain
);
811 spin_lock_irqsave(&dma_domain
->domain_lock
, flags
);
812 /* Ensure domain is inactive i.e. DMA should be disabled for the domain */
813 if (dma_domain
->enabled
) {
814 pr_debug("Can't set geometry attributes as domain is active\n");
815 spin_unlock_irqrestore(&dma_domain
->domain_lock
, flags
);
819 /* Ensure that the geometry has been set for the domain */
820 if (!dma_domain
->geom_size
) {
821 pr_debug("Please configure geometry before setting the number of windows\n");
822 spin_unlock_irqrestore(&dma_domain
->domain_lock
, flags
);
827 * Ensure we have valid window count i.e. it should be less than
828 * maximum permissible limit and should be a power of two.
830 if (w_count
> pamu_get_max_subwin_cnt() || !is_power_of_2(w_count
)) {
831 pr_debug("Invalid window count\n");
832 spin_unlock_irqrestore(&dma_domain
->domain_lock
, flags
);
836 ret
= pamu_set_domain_geometry(dma_domain
, &domain
->geometry
,
837 w_count
> 1 ? w_count
: 0);
839 kfree(dma_domain
->win_arr
);
840 dma_domain
->win_arr
= kcalloc(w_count
,
841 sizeof(*dma_domain
->win_arr
),
843 if (!dma_domain
->win_arr
) {
844 spin_unlock_irqrestore(&dma_domain
->domain_lock
, flags
);
847 dma_domain
->win_cnt
= w_count
;
849 spin_unlock_irqrestore(&dma_domain
->domain_lock
, flags
);
854 static int fsl_pamu_set_domain_attr(struct iommu_domain
*domain
,
855 enum iommu_attr attr_type
, void *data
)
857 struct fsl_dma_domain
*dma_domain
= to_fsl_dma_domain(domain
);
861 case DOMAIN_ATTR_GEOMETRY
:
862 ret
= configure_domain_geometry(domain
, data
);
864 case DOMAIN_ATTR_FSL_PAMU_STASH
:
865 ret
= configure_domain_stash(dma_domain
, data
);
867 case DOMAIN_ATTR_FSL_PAMU_ENABLE
:
868 ret
= configure_domain_dma_state(dma_domain
, *(int *)data
);
870 case DOMAIN_ATTR_WINDOWS
:
871 ret
= fsl_pamu_set_windows(domain
, *(u32
*)data
);
874 pr_debug("Unsupported attribute type\n");
882 static int fsl_pamu_get_domain_attr(struct iommu_domain
*domain
,
883 enum iommu_attr attr_type
, void *data
)
885 struct fsl_dma_domain
*dma_domain
= to_fsl_dma_domain(domain
);
889 case DOMAIN_ATTR_FSL_PAMU_STASH
:
890 memcpy(data
, &dma_domain
->dma_stash
,
891 sizeof(struct pamu_stash_attribute
));
893 case DOMAIN_ATTR_FSL_PAMU_ENABLE
:
894 *(int *)data
= dma_domain
->enabled
;
896 case DOMAIN_ATTR_FSL_PAMUV1
:
897 *(int *)data
= DOMAIN_ATTR_FSL_PAMUV1
;
899 case DOMAIN_ATTR_WINDOWS
:
900 *(u32
*)data
= dma_domain
->win_cnt
;
903 pr_debug("Unsupported attribute type\n");
911 static struct iommu_group
*get_device_iommu_group(struct device
*dev
)
913 struct iommu_group
*group
;
915 group
= iommu_group_get(dev
);
917 group
= iommu_group_alloc();
922 static bool check_pci_ctl_endpt_part(struct pci_controller
*pci_ctl
)
926 /* Check the PCI controller version number by readding BRR1 register */
927 version
= in_be32(pci_ctl
->cfg_addr
+ (PCI_FSL_BRR1
>> 2));
928 version
&= PCI_FSL_BRR1_VER
;
929 /* If PCI controller version is >= 0x204 we can partition endpoints */
930 return version
>= 0x204;
933 /* Get iommu group information from peer devices or devices on the parent bus */
934 static struct iommu_group
*get_shared_pci_device_group(struct pci_dev
*pdev
)
937 struct iommu_group
*group
;
938 struct pci_bus
*bus
= pdev
->bus
;
941 * Traverese the pci bus device list to get
942 * the shared iommu group.
945 list_for_each_entry(tmp
, &bus
->devices
, bus_list
) {
948 group
= iommu_group_get(&tmp
->dev
);
959 static struct iommu_group
*get_pci_device_group(struct pci_dev
*pdev
)
961 struct pci_controller
*pci_ctl
;
962 bool pci_endpt_partitioning
;
963 struct iommu_group
*group
= NULL
;
965 pci_ctl
= pci_bus_to_host(pdev
->bus
);
966 pci_endpt_partitioning
= check_pci_ctl_endpt_part(pci_ctl
);
967 /* We can partition PCIe devices so assign device group to the device */
968 if (pci_endpt_partitioning
) {
969 group
= pci_device_group(&pdev
->dev
);
972 * PCIe controller is not a paritionable entity
973 * free the controller device iommu_group.
975 if (pci_ctl
->parent
->iommu_group
)
976 iommu_group_remove_device(pci_ctl
->parent
);
979 * All devices connected to the controller will share the
980 * PCI controllers device group. If this is the first
981 * device to be probed for the pci controller, copy the
982 * device group information from the PCI controller device
983 * node and remove the PCI controller iommu group.
984 * For subsequent devices, the iommu group information can
985 * be obtained from sibling devices (i.e. from the bus_devices
988 if (pci_ctl
->parent
->iommu_group
) {
989 group
= get_device_iommu_group(pci_ctl
->parent
);
990 iommu_group_remove_device(pci_ctl
->parent
);
992 group
= get_shared_pci_device_group(pdev
);
997 group
= ERR_PTR(-ENODEV
);
1002 static struct iommu_group
*fsl_pamu_device_group(struct device
*dev
)
1004 struct iommu_group
*group
= ERR_PTR(-ENODEV
);
1008 * For platform devices we allocate a separate group for
1009 * each of the devices.
1011 if (dev_is_pci(dev
))
1012 group
= get_pci_device_group(to_pci_dev(dev
));
1013 else if (of_get_property(dev
->of_node
, "fsl,liodn", &len
))
1014 group
= get_device_iommu_group(dev
);
1019 static struct iommu_device
*fsl_pamu_probe_device(struct device
*dev
)
1024 static void fsl_pamu_release_device(struct device
*dev
)
1028 static const struct iommu_ops fsl_pamu_ops
= {
1029 .capable
= fsl_pamu_capable
,
1030 .domain_alloc
= fsl_pamu_domain_alloc
,
1031 .domain_free
= fsl_pamu_domain_free
,
1032 .attach_dev
= fsl_pamu_attach_device
,
1033 .detach_dev
= fsl_pamu_detach_device
,
1034 .domain_window_enable
= fsl_pamu_window_enable
,
1035 .domain_window_disable
= fsl_pamu_window_disable
,
1036 .iova_to_phys
= fsl_pamu_iova_to_phys
,
1037 .domain_set_attr
= fsl_pamu_set_domain_attr
,
1038 .domain_get_attr
= fsl_pamu_get_domain_attr
,
1039 .probe_device
= fsl_pamu_probe_device
,
1040 .release_device
= fsl_pamu_release_device
,
1041 .device_group
= fsl_pamu_device_group
,
1044 int __init
pamu_domain_init(void)
1048 ret
= iommu_init_mempool();
1052 ret
= iommu_device_sysfs_add(&pamu_iommu
, NULL
, NULL
, "iommu0");
1056 iommu_device_set_ops(&pamu_iommu
, &fsl_pamu_ops
);
1058 ret
= iommu_device_register(&pamu_iommu
);
1060 iommu_device_sysfs_remove(&pamu_iommu
);
1061 pr_err("Can't register iommu device\n");
1065 bus_set_iommu(&platform_bus_type
, &fsl_pamu_ops
);
1066 bus_set_iommu(&pci_bus_type
, &fsl_pamu_ops
);