1 // SPDX-License-Identifier: GPL-2.0-only
3 #include <linux/dma-mapping.h>
5 #include <linux/highmem.h>
6 #include <linux/export.h>
7 #include <linux/memblock.h>
8 #include <linux/of_address.h>
9 #include <linux/slab.h>
10 #include <linux/types.h>
11 #include <linux/vmalloc.h>
12 #include <linux/swiotlb.h>
15 #include <xen/interface/grant_table.h>
16 #include <xen/interface/memory.h>
18 #include <xen/swiotlb-xen.h>
20 #include <asm/cacheflush.h>
21 #include <asm/xen/hypercall.h>
22 #include <asm/xen/interface.h>
24 unsigned long xen_get_swiotlb_free_pages(unsigned int order
)
26 struct memblock_region
*reg
;
27 gfp_t flags
= __GFP_NOWARN
|__GFP_KSWAPD_RECLAIM
;
29 for_each_memblock(memory
, reg
) {
30 if (reg
->base
< (phys_addr_t
)0xffffffff) {
35 return __get_free_pages(flags
, order
);
42 static bool hypercall_cflush
= false;
44 /* functions called by SWIOTLB */
46 static void dma_cache_maint(dma_addr_t handle
, unsigned long offset
,
47 size_t size
, enum dma_data_direction dir
, enum dma_cache_op op
)
49 struct gnttab_cache_flush cflush
;
50 unsigned long xen_pfn
;
53 xen_pfn
= (handle
>> XEN_PAGE_SHIFT
) + offset
/ XEN_PAGE_SIZE
;
54 offset
%= XEN_PAGE_SIZE
;
59 /* buffers in highmem or foreign pages cannot cross page
61 if (len
+ offset
> XEN_PAGE_SIZE
)
62 len
= XEN_PAGE_SIZE
- offset
;
65 cflush
.a
.dev_bus_addr
= xen_pfn
<< XEN_PAGE_SHIFT
;
66 cflush
.offset
= offset
;
69 if (op
== DMA_UNMAP
&& dir
!= DMA_TO_DEVICE
)
70 cflush
.op
= GNTTAB_CACHE_INVAL
;
72 if (dir
== DMA_FROM_DEVICE
)
73 cflush
.op
= GNTTAB_CACHE_INVAL
;
75 cflush
.op
= GNTTAB_CACHE_CLEAN
;
78 HYPERVISOR_grant_table_op(GNTTABOP_cache_flush
, &cflush
, 1);
86 static void __xen_dma_page_dev_to_cpu(struct device
*hwdev
, dma_addr_t handle
,
87 size_t size
, enum dma_data_direction dir
)
89 dma_cache_maint(handle
& PAGE_MASK
, handle
& ~PAGE_MASK
, size
, dir
, DMA_UNMAP
);
92 static void __xen_dma_page_cpu_to_dev(struct device
*hwdev
, dma_addr_t handle
,
93 size_t size
, enum dma_data_direction dir
)
95 dma_cache_maint(handle
& PAGE_MASK
, handle
& ~PAGE_MASK
, size
, dir
, DMA_MAP
);
98 void __xen_dma_map_page(struct device
*hwdev
, struct page
*page
,
99 dma_addr_t dev_addr
, unsigned long offset
, size_t size
,
100 enum dma_data_direction dir
, unsigned long attrs
)
102 if (is_device_dma_coherent(hwdev
))
104 if (attrs
& DMA_ATTR_SKIP_CPU_SYNC
)
107 __xen_dma_page_cpu_to_dev(hwdev
, dev_addr
, size
, dir
);
110 void __xen_dma_unmap_page(struct device
*hwdev
, dma_addr_t handle
,
111 size_t size
, enum dma_data_direction dir
,
115 if (is_device_dma_coherent(hwdev
))
117 if (attrs
& DMA_ATTR_SKIP_CPU_SYNC
)
120 __xen_dma_page_dev_to_cpu(hwdev
, handle
, size
, dir
);
123 void __xen_dma_sync_single_for_cpu(struct device
*hwdev
,
124 dma_addr_t handle
, size_t size
, enum dma_data_direction dir
)
126 if (is_device_dma_coherent(hwdev
))
128 __xen_dma_page_dev_to_cpu(hwdev
, handle
, size
, dir
);
131 void __xen_dma_sync_single_for_device(struct device
*hwdev
,
132 dma_addr_t handle
, size_t size
, enum dma_data_direction dir
)
134 if (is_device_dma_coherent(hwdev
))
136 __xen_dma_page_cpu_to_dev(hwdev
, handle
, size
, dir
);
139 bool xen_arch_need_swiotlb(struct device
*dev
,
143 unsigned int xen_pfn
= XEN_PFN_DOWN(phys
);
144 unsigned int bfn
= XEN_PFN_DOWN(dev_addr
);
147 * The swiotlb buffer should be used if
148 * - Xen doesn't have the cache flush hypercall
149 * - The Linux page refers to foreign memory
150 * - The device doesn't support coherent DMA request
152 * The Linux page may be spanned acrros multiple Xen page, although
153 * it's not possible to have a mix of local and foreign Xen page.
154 * Furthermore, range_straddles_page_boundary is already checking
155 * if buffer is physically contiguous in the host RAM.
157 * Therefore we only need to check the first Xen page to know if we
158 * require a bounce buffer because the device doesn't support coherent
159 * memory and we are not able to flush the cache.
161 return (!hypercall_cflush
&& (xen_pfn
!= bfn
) &&
162 !is_device_dma_coherent(dev
));
165 int xen_create_contiguous_region(phys_addr_t pstart
, unsigned int order
,
166 unsigned int address_bits
,
167 dma_addr_t
*dma_handle
)
169 if (!xen_initial_domain())
172 /* we assume that dom0 is mapped 1:1 for now */
173 *dma_handle
= pstart
;
176 EXPORT_SYMBOL_GPL(xen_create_contiguous_region
);
178 void xen_destroy_contiguous_region(phys_addr_t pstart
, unsigned int order
)
182 EXPORT_SYMBOL_GPL(xen_destroy_contiguous_region
);
184 const struct dma_map_ops
*xen_dma_ops
;
185 EXPORT_SYMBOL(xen_dma_ops
);
187 int __init
xen_mm_init(void)
189 struct gnttab_cache_flush cflush
;
190 if (!xen_initial_domain())
192 xen_swiotlb_init(1, false);
193 xen_dma_ops
= &xen_swiotlb_dma_ops
;
196 cflush
.a
.dev_bus_addr
= 0;
199 if (HYPERVISOR_grant_table_op(GNTTABOP_cache_flush
, &cflush
, 1) != -ENOSYS
)
200 hypercall_cflush
= true;
203 arch_initcall(xen_mm_init
);