2 * SWIOTLB-based DMA API implementation
4 * Copyright (C) 2012 ARM Ltd.
5 * Author: Catalin Marinas <catalin.marinas@arm.com>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
20 #include <linux/gfp.h>
21 #include <linux/export.h>
22 #include <linux/slab.h>
23 #include <linux/dma-mapping.h>
24 #include <linux/dma-contiguous.h>
25 #include <linux/vmalloc.h>
26 #include <linux/swiotlb.h>
28 #include <asm/cacheflush.h>
30 struct dma_map_ops
*dma_ops
;
31 EXPORT_SYMBOL(dma_ops
);
33 static pgprot_t
__get_dma_pgprot(struct dma_attrs
*attrs
, pgprot_t prot
,
36 if (!coherent
|| dma_get_attr(DMA_ATTR_WRITE_COMBINE
, attrs
))
37 return pgprot_writecombine(prot
);
41 static void *__dma_alloc_coherent(struct device
*dev
, size_t size
,
42 dma_addr_t
*dma_handle
, gfp_t flags
,
43 struct dma_attrs
*attrs
)
46 WARN_ONCE(1, "Use an actual device structure for DMA allocation\n");
50 if (IS_ENABLED(CONFIG_ZONE_DMA
) &&
51 dev
->coherent_dma_mask
<= DMA_BIT_MASK(32))
53 if (IS_ENABLED(CONFIG_DMA_CMA
)) {
56 size
= PAGE_ALIGN(size
);
57 page
= dma_alloc_from_contiguous(dev
, size
>> PAGE_SHIFT
,
62 *dma_handle
= phys_to_dma(dev
, page_to_phys(page
));
63 return page_address(page
);
65 return swiotlb_alloc_coherent(dev
, size
, dma_handle
, flags
);
69 static void __dma_free_coherent(struct device
*dev
, size_t size
,
70 void *vaddr
, dma_addr_t dma_handle
,
71 struct dma_attrs
*attrs
)
74 WARN_ONCE(1, "Use an actual device structure for DMA allocation\n");
78 if (IS_ENABLED(CONFIG_DMA_CMA
)) {
79 phys_addr_t paddr
= dma_to_phys(dev
, dma_handle
);
81 dma_release_from_contiguous(dev
,
85 swiotlb_free_coherent(dev
, size
, vaddr
, dma_handle
);
89 static void *__dma_alloc_noncoherent(struct device
*dev
, size_t size
,
90 dma_addr_t
*dma_handle
, gfp_t flags
,
91 struct dma_attrs
*attrs
)
93 struct page
*page
, **map
;
94 void *ptr
, *coherent_ptr
;
97 size
= PAGE_ALIGN(size
);
98 order
= get_order(size
);
100 ptr
= __dma_alloc_coherent(dev
, size
, dma_handle
, flags
, attrs
);
103 map
= kmalloc(sizeof(struct page
*) << order
, flags
& ~GFP_DMA
);
107 /* remove any dirty cache lines on the kernel alias */
108 __dma_flush_range(ptr
, ptr
+ size
);
110 /* create a coherent mapping */
111 page
= virt_to_page(ptr
);
112 for (i
= 0; i
< (size
>> PAGE_SHIFT
); i
++)
114 coherent_ptr
= vmap(map
, size
>> PAGE_SHIFT
, VM_MAP
,
115 __get_dma_pgprot(attrs
, pgprot_default
, false));
123 __dma_free_coherent(dev
, size
, ptr
, *dma_handle
, attrs
);
129 static void __dma_free_noncoherent(struct device
*dev
, size_t size
,
130 void *vaddr
, dma_addr_t dma_handle
,
131 struct dma_attrs
*attrs
)
133 void *swiotlb_addr
= phys_to_virt(dma_to_phys(dev
, dma_handle
));
136 __dma_free_coherent(dev
, size
, swiotlb_addr
, dma_handle
, attrs
);
139 static dma_addr_t
__swiotlb_map_page(struct device
*dev
, struct page
*page
,
140 unsigned long offset
, size_t size
,
141 enum dma_data_direction dir
,
142 struct dma_attrs
*attrs
)
146 dev_addr
= swiotlb_map_page(dev
, page
, offset
, size
, dir
, attrs
);
147 __dma_map_area(phys_to_virt(dma_to_phys(dev
, dev_addr
)), size
, dir
);
153 static void __swiotlb_unmap_page(struct device
*dev
, dma_addr_t dev_addr
,
154 size_t size
, enum dma_data_direction dir
,
155 struct dma_attrs
*attrs
)
157 __dma_unmap_area(phys_to_virt(dma_to_phys(dev
, dev_addr
)), size
, dir
);
158 swiotlb_unmap_page(dev
, dev_addr
, size
, dir
, attrs
);
161 static int __swiotlb_map_sg_attrs(struct device
*dev
, struct scatterlist
*sgl
,
162 int nelems
, enum dma_data_direction dir
,
163 struct dma_attrs
*attrs
)
165 struct scatterlist
*sg
;
168 ret
= swiotlb_map_sg_attrs(dev
, sgl
, nelems
, dir
, attrs
);
169 for_each_sg(sgl
, sg
, ret
, i
)
170 __dma_map_area(phys_to_virt(dma_to_phys(dev
, sg
->dma_address
)),
176 static void __swiotlb_unmap_sg_attrs(struct device
*dev
,
177 struct scatterlist
*sgl
, int nelems
,
178 enum dma_data_direction dir
,
179 struct dma_attrs
*attrs
)
181 struct scatterlist
*sg
;
184 for_each_sg(sgl
, sg
, nelems
, i
)
185 __dma_unmap_area(phys_to_virt(dma_to_phys(dev
, sg
->dma_address
)),
187 swiotlb_unmap_sg_attrs(dev
, sgl
, nelems
, dir
, attrs
);
190 static void __swiotlb_sync_single_for_cpu(struct device
*dev
,
191 dma_addr_t dev_addr
, size_t size
,
192 enum dma_data_direction dir
)
194 __dma_unmap_area(phys_to_virt(dma_to_phys(dev
, dev_addr
)), size
, dir
);
195 swiotlb_sync_single_for_cpu(dev
, dev_addr
, size
, dir
);
198 static void __swiotlb_sync_single_for_device(struct device
*dev
,
199 dma_addr_t dev_addr
, size_t size
,
200 enum dma_data_direction dir
)
202 swiotlb_sync_single_for_device(dev
, dev_addr
, size
, dir
);
203 __dma_map_area(phys_to_virt(dma_to_phys(dev
, dev_addr
)), size
, dir
);
206 static void __swiotlb_sync_sg_for_cpu(struct device
*dev
,
207 struct scatterlist
*sgl
, int nelems
,
208 enum dma_data_direction dir
)
210 struct scatterlist
*sg
;
213 for_each_sg(sgl
, sg
, nelems
, i
)
214 __dma_unmap_area(phys_to_virt(dma_to_phys(dev
, sg
->dma_address
)),
216 swiotlb_sync_sg_for_cpu(dev
, sgl
, nelems
, dir
);
219 static void __swiotlb_sync_sg_for_device(struct device
*dev
,
220 struct scatterlist
*sgl
, int nelems
,
221 enum dma_data_direction dir
)
223 struct scatterlist
*sg
;
226 swiotlb_sync_sg_for_device(dev
, sgl
, nelems
, dir
);
227 for_each_sg(sgl
, sg
, nelems
, i
)
228 __dma_map_area(phys_to_virt(dma_to_phys(dev
, sg
->dma_address
)),
232 /* vma->vm_page_prot must be set appropriately before calling this function */
233 static int __dma_common_mmap(struct device
*dev
, struct vm_area_struct
*vma
,
234 void *cpu_addr
, dma_addr_t dma_addr
, size_t size
)
237 unsigned long nr_vma_pages
= (vma
->vm_end
- vma
->vm_start
) >>
239 unsigned long nr_pages
= PAGE_ALIGN(size
) >> PAGE_SHIFT
;
240 unsigned long pfn
= dma_to_phys(dev
, dma_addr
) >> PAGE_SHIFT
;
241 unsigned long off
= vma
->vm_pgoff
;
243 if (dma_mmap_from_coherent(dev
, vma
, cpu_addr
, size
, &ret
))
246 if (off
< nr_pages
&& nr_vma_pages
<= (nr_pages
- off
)) {
247 ret
= remap_pfn_range(vma
, vma
->vm_start
,
249 vma
->vm_end
- vma
->vm_start
,
256 static int __swiotlb_mmap_noncoherent(struct device
*dev
,
257 struct vm_area_struct
*vma
,
258 void *cpu_addr
, dma_addr_t dma_addr
, size_t size
,
259 struct dma_attrs
*attrs
)
261 vma
->vm_page_prot
= __get_dma_pgprot(attrs
, vma
->vm_page_prot
, false);
262 return __dma_common_mmap(dev
, vma
, cpu_addr
, dma_addr
, size
);
265 static int __swiotlb_mmap_coherent(struct device
*dev
,
266 struct vm_area_struct
*vma
,
267 void *cpu_addr
, dma_addr_t dma_addr
, size_t size
,
268 struct dma_attrs
*attrs
)
270 /* Just use whatever page_prot attributes were specified */
271 return __dma_common_mmap(dev
, vma
, cpu_addr
, dma_addr
, size
);
274 struct dma_map_ops noncoherent_swiotlb_dma_ops
= {
275 .alloc
= __dma_alloc_noncoherent
,
276 .free
= __dma_free_noncoherent
,
277 .mmap
= __swiotlb_mmap_noncoherent
,
278 .map_page
= __swiotlb_map_page
,
279 .unmap_page
= __swiotlb_unmap_page
,
280 .map_sg
= __swiotlb_map_sg_attrs
,
281 .unmap_sg
= __swiotlb_unmap_sg_attrs
,
282 .sync_single_for_cpu
= __swiotlb_sync_single_for_cpu
,
283 .sync_single_for_device
= __swiotlb_sync_single_for_device
,
284 .sync_sg_for_cpu
= __swiotlb_sync_sg_for_cpu
,
285 .sync_sg_for_device
= __swiotlb_sync_sg_for_device
,
286 .dma_supported
= swiotlb_dma_supported
,
287 .mapping_error
= swiotlb_dma_mapping_error
,
289 EXPORT_SYMBOL(noncoherent_swiotlb_dma_ops
);
291 struct dma_map_ops coherent_swiotlb_dma_ops
= {
292 .alloc
= __dma_alloc_coherent
,
293 .free
= __dma_free_coherent
,
294 .mmap
= __swiotlb_mmap_coherent
,
295 .map_page
= swiotlb_map_page
,
296 .unmap_page
= swiotlb_unmap_page
,
297 .map_sg
= swiotlb_map_sg_attrs
,
298 .unmap_sg
= swiotlb_unmap_sg_attrs
,
299 .sync_single_for_cpu
= swiotlb_sync_single_for_cpu
,
300 .sync_single_for_device
= swiotlb_sync_single_for_device
,
301 .sync_sg_for_cpu
= swiotlb_sync_sg_for_cpu
,
302 .sync_sg_for_device
= swiotlb_sync_sg_for_device
,
303 .dma_supported
= swiotlb_dma_supported
,
304 .mapping_error
= swiotlb_dma_mapping_error
,
306 EXPORT_SYMBOL(coherent_swiotlb_dma_ops
);
308 extern int swiotlb_late_init_with_default_size(size_t default_size
);
310 static int __init
swiotlb_late_init(void)
312 size_t swiotlb_size
= min(SZ_64M
, MAX_ORDER_NR_PAGES
<< PAGE_SHIFT
);
314 dma_ops
= &coherent_swiotlb_dma_ops
;
316 return swiotlb_late_init_with_default_size(swiotlb_size
);
318 subsys_initcall(swiotlb_late_init
);
320 #define PREALLOC_DMA_DEBUG_ENTRIES 4096
322 static int __init
dma_debug_do_init(void)
324 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES
);
327 fs_initcall(dma_debug_do_init
);