2 * linux/arch/arm/mm/dma-mapping.c
4 * Copyright (C) 2000-2004 Russell King
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 * DMA uncached mapping support.
12 #include <linux/module.h>
14 #include <linux/slab.h>
15 #include <linux/errno.h>
16 #include <linux/list.h>
17 #include <linux/init.h>
18 #include <linux/device.h>
19 #include <linux/dma-mapping.h>
21 #include <asm/memory.h>
22 #include <asm/highmem.h>
23 #include <asm/cacheflush.h>
24 #include <asm/tlbflush.h>
25 #include <asm/sizes.h>
27 /* Sanity check size */
28 #if (CONSISTENT_DMA_SIZE % SZ_2M)
29 #error "CONSISTENT_DMA_SIZE must be multiple of 2MiB"
32 #define CONSISTENT_END (0xffe00000)
33 #define CONSISTENT_BASE (CONSISTENT_END - CONSISTENT_DMA_SIZE)
35 #define CONSISTENT_OFFSET(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PAGE_SHIFT)
36 #define CONSISTENT_PTE_INDEX(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PGDIR_SHIFT)
37 #define NUM_CONSISTENT_PTES (CONSISTENT_DMA_SIZE >> PGDIR_SHIFT)
39 static u64
get_coherent_dma_mask(struct device
*dev
)
41 u64 mask
= ISA_DMA_THRESHOLD
;
44 mask
= dev
->coherent_dma_mask
;
47 * Sanity check the DMA mask - it must be non-zero, and
48 * must be able to be satisfied by a DMA allocation.
51 dev_warn(dev
, "coherent DMA mask is unset\n");
55 if ((~mask
) & ISA_DMA_THRESHOLD
) {
56 dev_warn(dev
, "coherent DMA mask %#llx is smaller "
57 "than system GFP_DMA mask %#llx\n",
58 mask
, (unsigned long long)ISA_DMA_THRESHOLD
);
68 * These are the page tables (2MB each) covering uncached, DMA consistent allocations
70 static pte_t
*consistent_pte
[NUM_CONSISTENT_PTES
];
71 static DEFINE_SPINLOCK(consistent_lock
);
74 * VM region handling support.
76 * This should become something generic, handling VM region allocations for
77 * vmalloc and similar (ioremap, module space, etc).
79 * I envisage vmalloc()'s supporting vm_struct becoming:
82 * struct vm_region region;
83 * unsigned long flags;
84 * struct page **pages;
85 * unsigned int nr_pages;
86 * unsigned long phys_addr;
89 * get_vm_area() would then call vm_region_alloc with an appropriate
90 * struct vm_region head (eg):
92 * struct vm_region vmalloc_head = {
93 * .vm_list = LIST_HEAD_INIT(vmalloc_head.vm_list),
94 * .vm_start = VMALLOC_START,
95 * .vm_end = VMALLOC_END,
98 * However, vmalloc_head.vm_start is variable (typically, it is dependent on
99 * the amount of RAM found at boot time.) I would imagine that get_vm_area()
100 * would have to initialise this each time prior to calling vm_region_alloc().
102 struct arm_vm_region
{
103 struct list_head vm_list
;
104 unsigned long vm_start
;
105 unsigned long vm_end
;
106 struct page
*vm_pages
;
110 static struct arm_vm_region consistent_head
= {
111 .vm_list
= LIST_HEAD_INIT(consistent_head
.vm_list
),
112 .vm_start
= CONSISTENT_BASE
,
113 .vm_end
= CONSISTENT_END
,
116 static struct arm_vm_region
*
117 arm_vm_region_alloc(struct arm_vm_region
*head
, size_t size
, gfp_t gfp
)
119 unsigned long addr
= head
->vm_start
, end
= head
->vm_end
- size
;
121 struct arm_vm_region
*c
, *new;
123 new = kmalloc(sizeof(struct arm_vm_region
), gfp
);
127 spin_lock_irqsave(&consistent_lock
, flags
);
129 list_for_each_entry(c
, &head
->vm_list
, vm_list
) {
130 if ((addr
+ size
) < addr
)
132 if ((addr
+ size
) <= c
->vm_start
)
141 * Insert this entry _before_ the one we found.
143 list_add_tail(&new->vm_list
, &c
->vm_list
);
144 new->vm_start
= addr
;
145 new->vm_end
= addr
+ size
;
148 spin_unlock_irqrestore(&consistent_lock
, flags
);
152 spin_unlock_irqrestore(&consistent_lock
, flags
);
158 static struct arm_vm_region
*arm_vm_region_find(struct arm_vm_region
*head
, unsigned long addr
)
160 struct arm_vm_region
*c
;
162 list_for_each_entry(c
, &head
->vm_list
, vm_list
) {
163 if (c
->vm_active
&& c
->vm_start
== addr
)
171 #ifdef CONFIG_HUGETLB_PAGE
172 #error ARM Coherent DMA allocator does not (yet) support huge TLB
176 __dma_alloc(struct device
*dev
, size_t size
, dma_addr_t
*handle
, gfp_t gfp
,
180 struct arm_vm_region
*c
;
182 u64 mask
= get_coherent_dma_mask(dev
);
185 if (!consistent_pte
[0]) {
186 printk(KERN_ERR
"%s: not initialised\n", __func__
);
195 * Sanity check the allocation size.
197 size
= PAGE_ALIGN(size
);
198 limit
= (mask
+ 1) & ~mask
;
199 if ((limit
&& size
>= limit
) ||
200 size
>= (CONSISTENT_END
- CONSISTENT_BASE
)) {
201 printk(KERN_WARNING
"coherent allocation too big "
202 "(requested %#x mask %#llx)\n", size
, mask
);
206 order
= get_order(size
);
208 if (mask
!= 0xffffffff)
211 page
= alloc_pages(gfp
, order
);
216 * Invalidate any data that might be lurking in the
217 * kernel direct-mapped region for device DMA.
220 void *ptr
= page_address(page
);
221 memset(ptr
, 0, size
);
222 dmac_flush_range(ptr
, ptr
+ size
);
223 outer_flush_range(__pa(ptr
), __pa(ptr
) + size
);
227 * Allocate a virtual address in the consistent mapping region.
229 c
= arm_vm_region_alloc(&consistent_head
, size
,
230 gfp
& ~(__GFP_DMA
| __GFP_HIGHMEM
));
233 struct page
*end
= page
+ (1 << order
);
234 int idx
= CONSISTENT_PTE_INDEX(c
->vm_start
);
235 u32 off
= CONSISTENT_OFFSET(c
->vm_start
) & (PTRS_PER_PTE
-1);
237 pte
= consistent_pte
[idx
] + off
;
240 split_page(page
, order
);
243 * Set the "dma handle"
245 *handle
= page_to_dma(dev
, page
);
248 BUG_ON(!pte_none(*pte
));
251 * x86 does not mark the pages reserved...
253 SetPageReserved(page
);
254 set_pte_ext(pte
, mk_pte(page
, prot
), 0);
258 if (off
>= PTRS_PER_PTE
) {
260 pte
= consistent_pte
[++idx
];
262 } while (size
-= PAGE_SIZE
);
265 * Free the otherwise unused pages.
272 return (void *)c
->vm_start
;
276 __free_pages(page
, order
);
281 #else /* !CONFIG_MMU */
283 __dma_alloc(struct device
*dev
, size_t size
, dma_addr_t
*handle
, gfp_t gfp
,
287 u64 mask
= get_coherent_dma_mask(dev
);
292 if (mask
!= 0xffffffff)
294 virt
= kmalloc(size
, gfp
);
298 *handle
= virt_to_dma(dev
, virt
);
305 #endif /* CONFIG_MMU */
308 * Allocate DMA-coherent memory space and return both the kernel remapped
309 * virtual and bus address for that space.
312 dma_alloc_coherent(struct device
*dev
, size_t size
, dma_addr_t
*handle
, gfp_t gfp
)
316 if (dma_alloc_from_coherent(dev
, size
, handle
, &memory
))
319 if (arch_is_coherent()) {
322 virt
= kmalloc(size
, gfp
);
325 *handle
= virt_to_dma(dev
, virt
);
330 return __dma_alloc(dev
, size
, handle
, gfp
,
331 pgprot_noncached(pgprot_kernel
));
333 EXPORT_SYMBOL(dma_alloc_coherent
);
336 * Allocate a writecombining region, in much the same way as
337 * dma_alloc_coherent above.
340 dma_alloc_writecombine(struct device
*dev
, size_t size
, dma_addr_t
*handle
, gfp_t gfp
)
342 return __dma_alloc(dev
, size
, handle
, gfp
,
343 pgprot_writecombine(pgprot_kernel
));
345 EXPORT_SYMBOL(dma_alloc_writecombine
);
347 static int dma_mmap(struct device
*dev
, struct vm_area_struct
*vma
,
348 void *cpu_addr
, dma_addr_t dma_addr
, size_t size
)
352 unsigned long flags
, user_size
, kern_size
;
353 struct arm_vm_region
*c
;
355 user_size
= (vma
->vm_end
- vma
->vm_start
) >> PAGE_SHIFT
;
357 spin_lock_irqsave(&consistent_lock
, flags
);
358 c
= arm_vm_region_find(&consistent_head
, (unsigned long)cpu_addr
);
359 spin_unlock_irqrestore(&consistent_lock
, flags
);
362 unsigned long off
= vma
->vm_pgoff
;
364 kern_size
= (c
->vm_end
- c
->vm_start
) >> PAGE_SHIFT
;
366 if (off
< kern_size
&&
367 user_size
<= (kern_size
- off
)) {
368 ret
= remap_pfn_range(vma
, vma
->vm_start
,
369 page_to_pfn(c
->vm_pages
) + off
,
370 user_size
<< PAGE_SHIFT
,
374 #endif /* CONFIG_MMU */
379 int dma_mmap_coherent(struct device
*dev
, struct vm_area_struct
*vma
,
380 void *cpu_addr
, dma_addr_t dma_addr
, size_t size
)
382 vma
->vm_page_prot
= pgprot_noncached(vma
->vm_page_prot
);
383 return dma_mmap(dev
, vma
, cpu_addr
, dma_addr
, size
);
385 EXPORT_SYMBOL(dma_mmap_coherent
);
387 int dma_mmap_writecombine(struct device
*dev
, struct vm_area_struct
*vma
,
388 void *cpu_addr
, dma_addr_t dma_addr
, size_t size
)
390 vma
->vm_page_prot
= pgprot_writecombine(vma
->vm_page_prot
);
391 return dma_mmap(dev
, vma
, cpu_addr
, dma_addr
, size
);
393 EXPORT_SYMBOL(dma_mmap_writecombine
);
396 * free a page as defined by the above mapping.
397 * Must not be called with IRQs disabled.
400 void dma_free_coherent(struct device
*dev
, size_t size
, void *cpu_addr
, dma_addr_t handle
)
402 struct arm_vm_region
*c
;
403 unsigned long flags
, addr
;
408 WARN_ON(irqs_disabled());
410 if (dma_release_from_coherent(dev
, get_order(size
), cpu_addr
))
413 if (arch_is_coherent()) {
418 size
= PAGE_ALIGN(size
);
420 spin_lock_irqsave(&consistent_lock
, flags
);
421 c
= arm_vm_region_find(&consistent_head
, (unsigned long)cpu_addr
);
426 spin_unlock_irqrestore(&consistent_lock
, flags
);
428 if ((c
->vm_end
- c
->vm_start
) != size
) {
429 printk(KERN_ERR
"%s: freeing wrong coherent size (%ld != %d)\n",
430 __func__
, c
->vm_end
- c
->vm_start
, size
);
432 size
= c
->vm_end
- c
->vm_start
;
435 idx
= CONSISTENT_PTE_INDEX(c
->vm_start
);
436 off
= CONSISTENT_OFFSET(c
->vm_start
) & (PTRS_PER_PTE
-1);
437 ptep
= consistent_pte
[idx
] + off
;
440 pte_t pte
= ptep_get_and_clear(&init_mm
, addr
, ptep
);
446 if (off
>= PTRS_PER_PTE
) {
448 ptep
= consistent_pte
[++idx
];
451 if (!pte_none(pte
) && pte_present(pte
)) {
454 if (pfn_valid(pfn
)) {
455 struct page
*page
= pfn_to_page(pfn
);
458 * x86 does not mark the pages reserved...
460 ClearPageReserved(page
);
467 printk(KERN_CRIT
"%s: bad page in kernel page table\n",
469 } while (size
-= PAGE_SIZE
);
471 flush_tlb_kernel_range(c
->vm_start
, c
->vm_end
);
473 spin_lock_irqsave(&consistent_lock
, flags
);
474 list_del(&c
->vm_list
);
475 spin_unlock_irqrestore(&consistent_lock
, flags
);
481 spin_unlock_irqrestore(&consistent_lock
, flags
);
482 printk(KERN_ERR
"%s: trying to free invalid coherent area: %p\n",
486 #else /* !CONFIG_MMU */
487 void dma_free_coherent(struct device
*dev
, size_t size
, void *cpu_addr
, dma_addr_t handle
)
489 if (dma_release_from_coherent(dev
, get_order(size
), cpu_addr
))
493 #endif /* CONFIG_MMU */
494 EXPORT_SYMBOL(dma_free_coherent
);
497 * Initialise the consistent memory allocation.
499 static int __init
consistent_init(void)
507 u32 base
= CONSISTENT_BASE
;
510 pgd
= pgd_offset(&init_mm
, base
);
511 pmd
= pmd_alloc(&init_mm
, pgd
, base
);
513 printk(KERN_ERR
"%s: no pmd tables\n", __func__
);
517 WARN_ON(!pmd_none(*pmd
));
519 pte
= pte_alloc_kernel(pmd
, base
);
521 printk(KERN_ERR
"%s: no pte tables\n", __func__
);
526 consistent_pte
[i
++] = pte
;
527 base
+= (1 << PGDIR_SHIFT
);
528 } while (base
< CONSISTENT_END
);
529 #endif /* !CONFIG_MMU */
534 core_initcall(consistent_init
);
537 * Make an area consistent for devices.
538 * Note: Drivers should NOT use this function directly, as it will break
539 * platforms with CONFIG_DMABOUNCE.
540 * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
542 void dma_cache_maint(const void *start
, size_t size
, int direction
)
544 void (*inner_op
)(const void *, const void *);
545 void (*outer_op
)(unsigned long, unsigned long);
547 BUG_ON(!virt_addr_valid(start
) || !virt_addr_valid(start
+ size
- 1));
550 case DMA_FROM_DEVICE
: /* invalidate only */
551 inner_op
= dmac_inv_range
;
552 outer_op
= outer_inv_range
;
554 case DMA_TO_DEVICE
: /* writeback only */
555 inner_op
= dmac_clean_range
;
556 outer_op
= outer_clean_range
;
558 case DMA_BIDIRECTIONAL
: /* writeback and invalidate */
559 inner_op
= dmac_flush_range
;
560 outer_op
= outer_flush_range
;
566 inner_op(start
, start
+ size
);
567 outer_op(__pa(start
), __pa(start
) + size
);
569 EXPORT_SYMBOL(dma_cache_maint
);
571 static void dma_cache_maint_contiguous(struct page
*page
, unsigned long offset
,
572 size_t size
, int direction
)
576 void (*inner_op
)(const void *, const void *);
577 void (*outer_op
)(unsigned long, unsigned long);
580 case DMA_FROM_DEVICE
: /* invalidate only */
581 inner_op
= dmac_inv_range
;
582 outer_op
= outer_inv_range
;
584 case DMA_TO_DEVICE
: /* writeback only */
585 inner_op
= dmac_clean_range
;
586 outer_op
= outer_clean_range
;
588 case DMA_BIDIRECTIONAL
: /* writeback and invalidate */
589 inner_op
= dmac_flush_range
;
590 outer_op
= outer_flush_range
;
596 if (!PageHighMem(page
)) {
597 vaddr
= page_address(page
) + offset
;
598 inner_op(vaddr
, vaddr
+ size
);
600 vaddr
= kmap_high_get(page
);
603 inner_op(vaddr
, vaddr
+ size
);
608 paddr
= page_to_phys(page
) + offset
;
609 outer_op(paddr
, paddr
+ size
);
612 void dma_cache_maint_page(struct page
*page
, unsigned long offset
,
613 size_t size
, int dir
)
616 * A single sg entry may refer to multiple physically contiguous
617 * pages. But we still need to process highmem pages individually.
618 * If highmem is not configured then the bulk of this loop gets
624 if (PageHighMem(page
) && len
+ offset
> PAGE_SIZE
) {
625 if (offset
>= PAGE_SIZE
) {
626 page
+= offset
/ PAGE_SIZE
;
629 len
= PAGE_SIZE
- offset
;
631 dma_cache_maint_contiguous(page
, offset
, len
, dir
);
637 EXPORT_SYMBOL(dma_cache_maint_page
);
640 * dma_map_sg - map a set of SG buffers for streaming mode DMA
641 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
642 * @sg: list of buffers
643 * @nents: number of buffers to map
644 * @dir: DMA transfer direction
646 * Map a set of buffers described by scatterlist in streaming mode for DMA.
647 * This is the scatter-gather version of the dma_map_single interface.
648 * Here the scatter gather list elements are each tagged with the
649 * appropriate dma address and length. They are obtained via
650 * sg_dma_{address,length}.
652 * Device ownership issues as mentioned for dma_map_single are the same
655 int dma_map_sg(struct device
*dev
, struct scatterlist
*sg
, int nents
,
656 enum dma_data_direction dir
)
658 struct scatterlist
*s
;
661 for_each_sg(sg
, s
, nents
, i
) {
662 s
->dma_address
= dma_map_page(dev
, sg_page(s
), s
->offset
,
664 if (dma_mapping_error(dev
, s
->dma_address
))
670 for_each_sg(sg
, s
, i
, j
)
671 dma_unmap_page(dev
, sg_dma_address(s
), sg_dma_len(s
), dir
);
674 EXPORT_SYMBOL(dma_map_sg
);
677 * dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
678 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
679 * @sg: list of buffers
680 * @nents: number of buffers to unmap (returned from dma_map_sg)
681 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
683 * Unmap a set of streaming mode DMA translations. Again, CPU access
684 * rules concerning calls here are the same as for dma_unmap_single().
686 void dma_unmap_sg(struct device
*dev
, struct scatterlist
*sg
, int nents
,
687 enum dma_data_direction dir
)
689 struct scatterlist
*s
;
692 for_each_sg(sg
, s
, nents
, i
)
693 dma_unmap_page(dev
, sg_dma_address(s
), sg_dma_len(s
), dir
);
695 EXPORT_SYMBOL(dma_unmap_sg
);
698 * dma_sync_sg_for_cpu
699 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
700 * @sg: list of buffers
701 * @nents: number of buffers to map (returned from dma_map_sg)
702 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
704 void dma_sync_sg_for_cpu(struct device
*dev
, struct scatterlist
*sg
,
705 int nents
, enum dma_data_direction dir
)
707 struct scatterlist
*s
;
710 for_each_sg(sg
, s
, nents
, i
) {
711 dmabounce_sync_for_cpu(dev
, sg_dma_address(s
), 0,
715 EXPORT_SYMBOL(dma_sync_sg_for_cpu
);
718 * dma_sync_sg_for_device
719 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
720 * @sg: list of buffers
721 * @nents: number of buffers to map (returned from dma_map_sg)
722 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
724 void dma_sync_sg_for_device(struct device
*dev
, struct scatterlist
*sg
,
725 int nents
, enum dma_data_direction dir
)
727 struct scatterlist
*s
;
730 for_each_sg(sg
, s
, nents
, i
) {
731 if (!dmabounce_sync_for_device(dev
, sg_dma_address(s
), 0,
735 if (!arch_is_coherent())
736 dma_cache_maint_page(sg_page(s
), s
->offset
,
740 EXPORT_SYMBOL(dma_sync_sg_for_device
);