2 * DMA coherent memory allocation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the
6 * Free Software Foundation; either version 2 of the License, or (at your
7 * option) any later version.
9 * Copyright (C) 2002 - 2005 Tensilica Inc.
10 * Copyright (C) 2015 Cadence Design Systems Inc.
12 * Based on version for i386.
14 * Chris Zankel <chris@zankel.net>
15 * Joe Taylor <joe@tensilica.com, joetylr@yahoo.com>
18 #include <linux/dma-contiguous.h>
19 #include <linux/gfp.h>
20 #include <linux/highmem.h>
22 #include <linux/module.h>
23 #include <linux/pci.h>
24 #include <linux/string.h>
25 #include <linux/types.h>
26 #include <asm/cacheflush.h>
29 void dma_cache_sync(struct device
*dev
, void *vaddr
, size_t size
,
30 enum dma_data_direction dir
)
33 case DMA_BIDIRECTIONAL
:
34 __flush_invalidate_dcache_range((unsigned long)vaddr
, size
);
38 __invalidate_dcache_range((unsigned long)vaddr
, size
);
42 __flush_dcache_range((unsigned long)vaddr
, size
);
50 EXPORT_SYMBOL(dma_cache_sync
);
52 static void do_cache_op(dma_addr_t dma_handle
, size_t size
,
53 void (*fn
)(unsigned long, unsigned long))
55 unsigned long off
= dma_handle
& (PAGE_SIZE
- 1);
56 unsigned long pfn
= PFN_DOWN(dma_handle
);
57 struct page
*page
= pfn_to_page(pfn
);
59 if (!PageHighMem(page
))
60 fn((unsigned long)bus_to_virt(dma_handle
), size
);
63 size_t sz
= min_t(size_t, size
, PAGE_SIZE
- off
);
64 void *vaddr
= kmap_atomic(page
);
66 fn((unsigned long)vaddr
+ off
, sz
);
74 static void xtensa_sync_single_for_cpu(struct device
*dev
,
75 dma_addr_t dma_handle
, size_t size
,
76 enum dma_data_direction dir
)
79 case DMA_BIDIRECTIONAL
:
81 do_cache_op(dma_handle
, size
, __invalidate_dcache_range
);
93 static void xtensa_sync_single_for_device(struct device
*dev
,
94 dma_addr_t dma_handle
, size_t size
,
95 enum dma_data_direction dir
)
98 case DMA_BIDIRECTIONAL
:
100 if (XCHAL_DCACHE_IS_WRITEBACK
)
101 do_cache_op(dma_handle
, size
, __flush_dcache_range
);
113 static void xtensa_sync_sg_for_cpu(struct device
*dev
,
114 struct scatterlist
*sg
, int nents
,
115 enum dma_data_direction dir
)
117 struct scatterlist
*s
;
120 for_each_sg(sg
, s
, nents
, i
) {
121 xtensa_sync_single_for_cpu(dev
, sg_dma_address(s
),
126 static void xtensa_sync_sg_for_device(struct device
*dev
,
127 struct scatterlist
*sg
, int nents
,
128 enum dma_data_direction dir
)
130 struct scatterlist
*s
;
133 for_each_sg(sg
, s
, nents
, i
) {
134 xtensa_sync_single_for_device(dev
, sg_dma_address(s
),
140 * Note: We assume that the full memory space is always mapped to 'kseg'
141 * Otherwise we have to use page attributes (not implemented).
144 static void *xtensa_dma_alloc(struct device
*dev
, size_t size
,
145 dma_addr_t
*handle
, gfp_t flag
,
149 unsigned long uncached
= 0;
150 unsigned long count
= PAGE_ALIGN(size
) >> PAGE_SHIFT
;
151 struct page
*page
= NULL
;
153 /* ignore region speicifiers */
155 flag
&= ~(__GFP_DMA
| __GFP_HIGHMEM
);
157 if (dev
== NULL
|| (dev
->coherent_dma_mask
< 0xffffffff))
160 if (gfpflags_allow_blocking(flag
))
161 page
= dma_alloc_from_contiguous(dev
, count
, get_order(size
),
165 page
= alloc_pages(flag
, get_order(size
));
170 ret
= (unsigned long)page_address(page
);
172 /* We currently don't support coherent memory outside KSEG */
174 BUG_ON(ret
< XCHAL_KSEG_CACHED_VADDR
||
175 ret
> XCHAL_KSEG_CACHED_VADDR
+ XCHAL_KSEG_SIZE
- 1);
177 uncached
= ret
+ XCHAL_KSEG_BYPASS_VADDR
- XCHAL_KSEG_CACHED_VADDR
;
178 *handle
= virt_to_bus((void *)ret
);
179 __invalidate_dcache_range(ret
, size
);
181 return (void *)uncached
;
184 static void xtensa_dma_free(struct device
*dev
, size_t size
, void *vaddr
,
185 dma_addr_t dma_handle
, unsigned long attrs
)
187 unsigned long addr
= (unsigned long)vaddr
+
188 XCHAL_KSEG_CACHED_VADDR
- XCHAL_KSEG_BYPASS_VADDR
;
189 struct page
*page
= virt_to_page(addr
);
190 unsigned long count
= PAGE_ALIGN(size
) >> PAGE_SHIFT
;
192 BUG_ON(addr
< XCHAL_KSEG_CACHED_VADDR
||
193 addr
> XCHAL_KSEG_CACHED_VADDR
+ XCHAL_KSEG_SIZE
- 1);
195 if (!dma_release_from_contiguous(dev
, page
, count
))
196 __free_pages(page
, get_order(size
));
199 static dma_addr_t
xtensa_map_page(struct device
*dev
, struct page
*page
,
200 unsigned long offset
, size_t size
,
201 enum dma_data_direction dir
,
204 dma_addr_t dma_handle
= page_to_phys(page
) + offset
;
206 if (!(attrs
& DMA_ATTR_SKIP_CPU_SYNC
))
207 xtensa_sync_single_for_device(dev
, dma_handle
, size
, dir
);
212 static void xtensa_unmap_page(struct device
*dev
, dma_addr_t dma_handle
,
213 size_t size
, enum dma_data_direction dir
,
216 if (!(attrs
& DMA_ATTR_SKIP_CPU_SYNC
))
217 xtensa_sync_single_for_cpu(dev
, dma_handle
, size
, dir
);
220 static int xtensa_map_sg(struct device
*dev
, struct scatterlist
*sg
,
221 int nents
, enum dma_data_direction dir
,
224 struct scatterlist
*s
;
227 for_each_sg(sg
, s
, nents
, i
) {
228 s
->dma_address
= xtensa_map_page(dev
, sg_page(s
), s
->offset
,
229 s
->length
, dir
, attrs
);
234 static void xtensa_unmap_sg(struct device
*dev
,
235 struct scatterlist
*sg
, int nents
,
236 enum dma_data_direction dir
,
239 struct scatterlist
*s
;
242 for_each_sg(sg
, s
, nents
, i
) {
243 xtensa_unmap_page(dev
, sg_dma_address(s
),
244 sg_dma_len(s
), dir
, attrs
);
248 int xtensa_dma_mapping_error(struct device
*dev
, dma_addr_t dma_addr
)
253 const struct dma_map_ops xtensa_dma_map_ops
= {
254 .alloc
= xtensa_dma_alloc
,
255 .free
= xtensa_dma_free
,
256 .map_page
= xtensa_map_page
,
257 .unmap_page
= xtensa_unmap_page
,
258 .map_sg
= xtensa_map_sg
,
259 .unmap_sg
= xtensa_unmap_sg
,
260 .sync_single_for_cpu
= xtensa_sync_single_for_cpu
,
261 .sync_single_for_device
= xtensa_sync_single_for_device
,
262 .sync_sg_for_cpu
= xtensa_sync_sg_for_cpu
,
263 .sync_sg_for_device
= xtensa_sync_sg_for_device
,
264 .mapping_error
= xtensa_dma_mapping_error
,
266 EXPORT_SYMBOL(xtensa_dma_map_ops
);
268 #define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
270 static int __init
xtensa_dma_init(void)
272 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES
);
275 fs_initcall(xtensa_dma_init
);