2 * omap iommu: simple virtual address space management
4 * Copyright (C) 2008-2009 Nokia Corporation
6 * Written by Hiroshi DOYU <Hiroshi.DOYU@nokia.com>
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
13 #include <linux/err.h>
14 #include <linux/vmalloc.h>
15 #include <linux/device.h>
16 #include <linux/scatterlist.h>
18 #include <asm/cacheflush.h>
19 #include <asm/mach/map.h>
21 #include <mach/iommu.h>
22 #include <mach/iovmm.h>
24 #include "iopgtable.h"
27 * A device driver needs to create address mappings between:
29 * - iommu/device address
31 * - mpu virtual address
33 * There are 4 possible patterns for them:
35 * |iova/ mapping iommu_ page
36 * | da pa va (d)-(p)-(v) function type
37 * ---------------------------------------------------------------------------
38 * 1 | c c c 1 - 1 - 1 _kmap() / _kunmap() s
39 * 2 | c c,a c 1 - 1 - 1 _kmalloc()/ _kfree() s
40 * 3 | c d c 1 - n - 1 _vmap() / _vunmap() s
41 * 4 | c d,a c 1 - n - 1 _vmalloc()/ _vfree() n*
44 * 'iova': device iommu virtual address
45 * 'da': alias of 'iova'
46 * 'pa': physical address
47 * 'va': mpu virtual address
49 * 'c': contiguous memory area
50 * 'd': dicontiguous memory area
51 * 'a': anonymous memory allocation
52 * '()': optional feature
54 * 'n': a normal page(4KB) size is used.
55 * 's': multiple iommu superpage(16MB, 1MB, 64KB, 4KB) size is used.
57 * '*': not yet, but feasible.
60 static struct kmem_cache
*iovm_area_cachep
;
62 /* return total bytes of sg buffers */
63 static size_t sgtable_len(const struct sg_table
*sgt
)
65 unsigned int i
, total
= 0;
66 struct scatterlist
*sg
;
71 for_each_sg(sgt
->sgl
, sg
, sgt
->nents
, i
) {
74 bytes
= sg_dma_len(sg
);
76 if (!iopgsz_ok(bytes
)) {
77 pr_err("%s: sg[%d] not iommu pagesize(%x)\n",
87 #define sgtable_ok(x) (!!sgtable_len(x))
90 * calculate the optimal number sg elements from total bytes based on
93 static unsigned int sgtable_nents(size_t bytes
)
96 unsigned int nr_entries
;
97 const unsigned long pagesize
[] = { SZ_16M
, SZ_1M
, SZ_64K
, SZ_4K
, };
99 if (!IS_ALIGNED(bytes
, PAGE_SIZE
)) {
100 pr_err("%s: wrong size %08x\n", __func__
, bytes
);
105 for (i
= 0; i
< ARRAY_SIZE(pagesize
); i
++) {
106 if (bytes
>= pagesize
[i
]) {
107 nr_entries
+= (bytes
/ pagesize
[i
]);
108 bytes
%= pagesize
[i
];
116 /* allocate and initialize sg_table header(a kind of 'superblock') */
117 static struct sg_table
*sgtable_alloc(const size_t bytes
, u32 flags
)
119 unsigned int nr_entries
;
121 struct sg_table
*sgt
;
124 return ERR_PTR(-EINVAL
);
126 if (!IS_ALIGNED(bytes
, PAGE_SIZE
))
127 return ERR_PTR(-EINVAL
);
129 /* FIXME: IOVMF_DA_FIXED should support 'superpages' */
130 if ((flags
& IOVMF_LINEAR
) && (flags
& IOVMF_DA_ANON
)) {
131 nr_entries
= sgtable_nents(bytes
);
133 return ERR_PTR(-EINVAL
);
135 nr_entries
= bytes
/ PAGE_SIZE
;
137 sgt
= kzalloc(sizeof(*sgt
), GFP_KERNEL
);
139 return ERR_PTR(-ENOMEM
);
141 err
= sg_alloc_table(sgt
, nr_entries
, GFP_KERNEL
);
145 pr_debug("%s: sgt:%p(%d entries)\n", __func__
, sgt
, nr_entries
);
150 /* free sg_table header(a kind of superblock) */
151 static void sgtable_free(struct sg_table
*sgt
)
159 pr_debug("%s: sgt:%p\n", __func__
, sgt
);
162 /* map 'sglist' to a contiguous mpu virtual area and return 'va' */
163 static void *vmap_sg(const struct sg_table
*sgt
)
168 struct scatterlist
*sg
;
169 struct vm_struct
*new;
170 const struct mem_type
*mtype
;
172 mtype
= get_mem_type(MT_DEVICE
);
174 return ERR_PTR(-EINVAL
);
176 total
= sgtable_len(sgt
);
178 return ERR_PTR(-EINVAL
);
180 new = __get_vm_area(total
, VM_IOREMAP
, VMALLOC_START
, VMALLOC_END
);
182 return ERR_PTR(-ENOMEM
);
185 for_each_sg(sgt
->sgl
, sg
, sgt
->nents
, i
) {
191 bytes
= sg_dma_len(sg
);
193 BUG_ON(bytes
!= PAGE_SIZE
);
195 err
= ioremap_page(va
, pa
, mtype
);
202 flush_cache_vmap(new->addr
, total
);
206 WARN_ON(1); /* FIXME: cleanup some mpu mappings */
208 return ERR_PTR(-EAGAIN
);
211 static inline void vunmap_sg(const void *va
)
216 static struct iovm_struct
*__find_iovm_area(struct iommu
*obj
, const u32 da
)
218 struct iovm_struct
*tmp
;
220 list_for_each_entry(tmp
, &obj
->mmap
, list
) {
221 if ((da
>= tmp
->da_start
) && (da
< tmp
->da_end
)) {
224 len
= tmp
->da_end
- tmp
->da_start
;
226 dev_dbg(obj
->dev
, "%s: %08x-%08x-%08x(%x) %08x\n",
227 __func__
, tmp
->da_start
, da
, tmp
->da_end
, len
,
238 * find_iovm_area - find iovma which includes @da
239 * @da: iommu device virtual address
241 * Find the existing iovma starting at @da
243 struct iovm_struct
*find_iovm_area(struct iommu
*obj
, u32 da
)
245 struct iovm_struct
*area
;
247 mutex_lock(&obj
->mmap_lock
);
248 area
= __find_iovm_area(obj
, da
);
249 mutex_unlock(&obj
->mmap_lock
);
253 EXPORT_SYMBOL_GPL(find_iovm_area
);
256 * This finds the hole(area) which fits the requested address and len
257 * in iovmas mmap, and returns the new allocated iovma.
259 static struct iovm_struct
*alloc_iovm_area(struct iommu
*obj
, u32 da
,
260 size_t bytes
, u32 flags
)
262 struct iovm_struct
*new, *tmp
;
263 u32 start
, prev_end
, alignement
;
266 return ERR_PTR(-EINVAL
);
269 alignement
= PAGE_SIZE
;
271 if (flags
& IOVMF_DA_ANON
) {
273 * Reserve the first page for NULL
276 if (flags
& IOVMF_LINEAR
)
277 alignement
= iopgsz_max(bytes
);
278 start
= roundup(start
, alignement
);
282 if (list_empty(&obj
->mmap
))
286 list_for_each_entry(tmp
, &obj
->mmap
, list
) {
288 if ((prev_end
<= start
) && (start
+ bytes
< tmp
->da_start
))
291 if (flags
& IOVMF_DA_ANON
)
292 start
= roundup(tmp
->da_end
, alignement
);
294 prev_end
= tmp
->da_end
;
297 if ((start
>= prev_end
) && (ULONG_MAX
- start
>= bytes
))
300 dev_dbg(obj
->dev
, "%s: no space to fit %08x(%x) flags: %08x\n",
301 __func__
, da
, bytes
, flags
);
303 return ERR_PTR(-EINVAL
);
306 new = kmem_cache_zalloc(iovm_area_cachep
, GFP_KERNEL
);
308 return ERR_PTR(-ENOMEM
);
311 new->da_start
= start
;
312 new->da_end
= start
+ bytes
;
316 * keep ascending order of iovmas
319 list_add_tail(&new->list
, &tmp
->list
);
321 list_add(&new->list
, &obj
->mmap
);
323 dev_dbg(obj
->dev
, "%s: found %08x-%08x-%08x(%x) %08x\n",
324 __func__
, new->da_start
, start
, new->da_end
, bytes
, flags
);
329 static void free_iovm_area(struct iommu
*obj
, struct iovm_struct
*area
)
333 BUG_ON(!obj
|| !area
);
335 bytes
= area
->da_end
- area
->da_start
;
337 dev_dbg(obj
->dev
, "%s: %08x-%08x(%x) %08x\n",
338 __func__
, area
->da_start
, area
->da_end
, bytes
, area
->flags
);
340 list_del(&area
->list
);
341 kmem_cache_free(iovm_area_cachep
, area
);
345 * da_to_va - convert (d) to (v)
346 * @obj: objective iommu
347 * @da: iommu device virtual address
348 * @va: mpu virtual address
350 * Returns mpu virtual addr which corresponds to a given device virtual addr
352 void *da_to_va(struct iommu
*obj
, u32 da
)
355 struct iovm_struct
*area
;
357 mutex_lock(&obj
->mmap_lock
);
359 area
= __find_iovm_area(obj
, da
);
361 dev_dbg(obj
->dev
, "%s: no da area(%08x)\n", __func__
, da
);
365 mutex_unlock(&obj
->mmap_lock
);
369 EXPORT_SYMBOL_GPL(da_to_va
);
371 static void sgtable_fill_vmalloc(struct sg_table
*sgt
, void *_va
)
374 struct scatterlist
*sg
;
378 for_each_sg(sgt
->sgl
, sg
, sgt
->nents
, i
) {
380 const size_t bytes
= PAGE_SIZE
;
383 * iommu 'superpage' isn't supported with 'iommu_vmalloc()'
385 pg
= vmalloc_to_page(va
);
387 sg_set_page(sg
, pg
, bytes
, 0);
392 va_end
= _va
+ PAGE_SIZE
* i
;
393 flush_cache_vmap(_va
, va_end
);
396 static inline void sgtable_drain_vmalloc(struct sg_table
*sgt
)
399 * Actually this is not necessary at all, just exists for
400 * consistency of the code readibility.
405 static void sgtable_fill_kmalloc(struct sg_table
*sgt
, u32 pa
, size_t len
)
408 struct scatterlist
*sg
;
411 va
= phys_to_virt(pa
);
413 for_each_sg(sgt
->sgl
, sg
, sgt
->nents
, i
) {
416 bytes
= iopgsz_max(len
);
418 BUG_ON(!iopgsz_ok(bytes
));
420 sg_set_buf(sg
, phys_to_virt(pa
), bytes
);
422 * 'pa' is cotinuous(linear).
429 clean_dcache_area(va
, len
);
432 static inline void sgtable_drain_kmalloc(struct sg_table
*sgt
)
435 * Actually this is not necessary at all, just exists for
436 * consistency of the code readibility
441 /* create 'da' <-> 'pa' mapping from 'sgt' */
442 static int map_iovm_area(struct iommu
*obj
, struct iovm_struct
*new,
443 const struct sg_table
*sgt
, u32 flags
)
447 struct scatterlist
*sg
;
448 u32 da
= new->da_start
;
450 if (!obj
|| !new || !sgt
)
453 BUG_ON(!sgtable_ok(sgt
));
455 for_each_sg(sgt
->sgl
, sg
, sgt
->nents
, i
) {
459 struct iotlb_entry e
;
462 bytes
= sg_dma_len(sg
);
464 flags
&= ~IOVMF_PGSZ_MASK
;
465 pgsz
= bytes_to_iopgsz(bytes
);
470 pr_debug("%s: [%d] %08x %08x(%x)\n", __func__
,
473 iotlb_init_entry(&e
, da
, pa
, flags
);
474 err
= iopgtable_store_entry(obj
, &e
);
485 for_each_sg(sgt
->sgl
, sg
, i
, j
) {
488 bytes
= iopgtable_clear_entry(obj
, da
);
490 BUG_ON(!iopgsz_ok(bytes
));
497 /* release 'da' <-> 'pa' mapping */
498 static void unmap_iovm_area(struct iommu
*obj
, struct iovm_struct
*area
)
501 size_t total
= area
->da_end
- area
->da_start
;
503 BUG_ON((!total
) || !IS_ALIGNED(total
, PAGE_SIZE
));
505 start
= area
->da_start
;
509 bytes
= iopgtable_clear_entry(obj
, start
);
513 dev_dbg(obj
->dev
, "%s: unmap %08x(%x) %08x\n",
514 __func__
, start
, bytes
, area
->flags
);
516 BUG_ON(!IS_ALIGNED(bytes
, PAGE_SIZE
));
524 /* template function for all unmapping */
525 static struct sg_table
*unmap_vm_area(struct iommu
*obj
, const u32 da
,
526 void (*fn
)(const void *), u32 flags
)
528 struct sg_table
*sgt
= NULL
;
529 struct iovm_struct
*area
;
531 if (!IS_ALIGNED(da
, PAGE_SIZE
)) {
532 dev_err(obj
->dev
, "%s: alignment err(%08x)\n", __func__
, da
);
536 mutex_lock(&obj
->mmap_lock
);
538 area
= __find_iovm_area(obj
, da
);
540 dev_dbg(obj
->dev
, "%s: no da area(%08x)\n", __func__
, da
);
544 if ((area
->flags
& flags
) != flags
) {
545 dev_err(obj
->dev
, "%s: wrong flags(%08x)\n", __func__
,
549 sgt
= (struct sg_table
*)area
->sgt
;
551 unmap_iovm_area(obj
, area
);
555 dev_dbg(obj
->dev
, "%s: %08x-%08x-%08x(%x) %08x\n", __func__
,
556 area
->da_start
, da
, area
->da_end
,
557 area
->da_end
- area
->da_start
, area
->flags
);
559 free_iovm_area(obj
, area
);
561 mutex_unlock(&obj
->mmap_lock
);
566 static u32
map_iommu_region(struct iommu
*obj
, u32 da
,
567 const struct sg_table
*sgt
, void *va
, size_t bytes
, u32 flags
)
570 struct iovm_struct
*new;
572 mutex_lock(&obj
->mmap_lock
);
574 new = alloc_iovm_area(obj
, da
, bytes
, flags
);
577 goto err_alloc_iovma
;
582 if (map_iovm_area(obj
, new, sgt
, new->flags
))
585 mutex_unlock(&obj
->mmap_lock
);
587 dev_dbg(obj
->dev
, "%s: da:%08x(%x) flags:%08x va:%p\n",
588 __func__
, new->da_start
, bytes
, new->flags
, va
);
590 return new->da_start
;
593 free_iovm_area(obj
, new);
595 mutex_unlock(&obj
->mmap_lock
);
599 static inline u32
__iommu_vmap(struct iommu
*obj
, u32 da
,
600 const struct sg_table
*sgt
, void *va
, size_t bytes
, u32 flags
)
602 return map_iommu_region(obj
, da
, sgt
, va
, bytes
, flags
);
606 * iommu_vmap - (d)-(p)-(v) address mapper
607 * @obj: objective iommu
608 * @sgt: address of scatter gather table
609 * @flags: iovma and page property
611 * Creates 1-n-1 mapping with given @sgt and returns @da.
612 * All @sgt element must be io page size aligned.
614 u32
iommu_vmap(struct iommu
*obj
, u32 da
, const struct sg_table
*sgt
,
620 if (!obj
|| !obj
->dev
|| !sgt
)
623 bytes
= sgtable_len(sgt
);
626 bytes
= PAGE_ALIGN(bytes
);
632 flags
&= IOVMF_HW_MASK
;
633 flags
|= IOVMF_DISCONT
;
635 flags
|= (da
? IOVMF_DA_FIXED
: IOVMF_DA_ANON
);
637 da
= __iommu_vmap(obj
, da
, sgt
, va
, bytes
, flags
);
638 if (IS_ERR_VALUE(da
))
643 EXPORT_SYMBOL_GPL(iommu_vmap
);
646 * iommu_vunmap - release virtual mapping obtained by 'iommu_vmap()'
647 * @obj: objective iommu
648 * @da: iommu device virtual address
650 * Free the iommu virtually contiguous memory area starting at
651 * @da, which was returned by 'iommu_vmap()'.
653 struct sg_table
*iommu_vunmap(struct iommu
*obj
, u32 da
)
655 struct sg_table
*sgt
;
657 * 'sgt' is allocated before 'iommu_vmalloc()' is called.
658 * Just returns 'sgt' to the caller to free
660 sgt
= unmap_vm_area(obj
, da
, vunmap_sg
, IOVMF_DISCONT
| IOVMF_MMIO
);
662 dev_dbg(obj
->dev
, "%s: No sgt\n", __func__
);
665 EXPORT_SYMBOL_GPL(iommu_vunmap
);
668 * iommu_vmalloc - (d)-(p)-(v) address allocator and mapper
669 * @obj: objective iommu
670 * @da: contiguous iommu virtual memory
671 * @bytes: allocation size
672 * @flags: iovma and page property
674 * Allocate @bytes linearly and creates 1-n-1 mapping and returns
675 * @da again, which might be adjusted if 'IOVMF_DA_ANON' is set.
677 u32
iommu_vmalloc(struct iommu
*obj
, u32 da
, size_t bytes
, u32 flags
)
680 struct sg_table
*sgt
;
682 if (!obj
|| !obj
->dev
|| !bytes
)
685 bytes
= PAGE_ALIGN(bytes
);
691 sgt
= sgtable_alloc(bytes
, flags
);
696 sgtable_fill_vmalloc(sgt
, va
);
698 flags
&= IOVMF_HW_MASK
;
699 flags
|= IOVMF_DISCONT
;
700 flags
|= IOVMF_ALLOC
;
701 flags
|= (da
? IOVMF_DA_FIXED
: IOVMF_DA_ANON
);
703 da
= __iommu_vmap(obj
, da
, sgt
, va
, bytes
, flags
);
704 if (IS_ERR_VALUE(da
))
710 sgtable_drain_vmalloc(sgt
);
716 EXPORT_SYMBOL_GPL(iommu_vmalloc
);
719 * iommu_vfree - release memory allocated by 'iommu_vmalloc()'
720 * @obj: objective iommu
721 * @da: iommu device virtual address
723 * Frees the iommu virtually continuous memory area starting at
724 * @da, as obtained from 'iommu_vmalloc()'.
726 void iommu_vfree(struct iommu
*obj
, const u32 da
)
728 struct sg_table
*sgt
;
730 sgt
= unmap_vm_area(obj
, da
, vfree
, IOVMF_DISCONT
| IOVMF_ALLOC
);
732 dev_dbg(obj
->dev
, "%s: No sgt\n", __func__
);
735 EXPORT_SYMBOL_GPL(iommu_vfree
);
737 static u32
__iommu_kmap(struct iommu
*obj
, u32 da
, u32 pa
, void *va
,
738 size_t bytes
, u32 flags
)
740 struct sg_table
*sgt
;
742 sgt
= sgtable_alloc(bytes
, flags
);
746 sgtable_fill_kmalloc(sgt
, pa
, bytes
);
748 da
= map_iommu_region(obj
, da
, sgt
, va
, bytes
, flags
);
749 if (IS_ERR_VALUE(da
)) {
750 sgtable_drain_kmalloc(sgt
);
758 * iommu_kmap - (d)-(p)-(v) address mapper
759 * @obj: objective iommu
760 * @da: contiguous iommu virtual memory
761 * @pa: contiguous physical memory
762 * @flags: iovma and page property
764 * Creates 1-1-1 mapping and returns @da again, which can be
765 * adjusted if 'IOVMF_DA_ANON' is set.
767 u32
iommu_kmap(struct iommu
*obj
, u32 da
, u32 pa
, size_t bytes
,
772 if (!obj
|| !obj
->dev
|| !bytes
)
775 bytes
= PAGE_ALIGN(bytes
);
777 va
= ioremap(pa
, bytes
);
781 flags
&= IOVMF_HW_MASK
;
782 flags
|= IOVMF_LINEAR
;
784 flags
|= (da
? IOVMF_DA_FIXED
: IOVMF_DA_ANON
);
786 da
= __iommu_kmap(obj
, da
, pa
, va
, bytes
, flags
);
787 if (IS_ERR_VALUE(da
))
792 EXPORT_SYMBOL_GPL(iommu_kmap
);
795 * iommu_kunmap - release virtual mapping obtained by 'iommu_kmap()'
796 * @obj: objective iommu
797 * @da: iommu device virtual address
799 * Frees the iommu virtually contiguous memory area starting at
800 * @da, which was passed to and was returned by'iommu_kmap()'.
802 void iommu_kunmap(struct iommu
*obj
, u32 da
)
804 struct sg_table
*sgt
;
805 typedef void (*func_t
)(const void *);
807 sgt
= unmap_vm_area(obj
, da
, (func_t
)__iounmap
,
808 IOVMF_LINEAR
| IOVMF_MMIO
);
810 dev_dbg(obj
->dev
, "%s: No sgt\n", __func__
);
813 EXPORT_SYMBOL_GPL(iommu_kunmap
);
816 * iommu_kmalloc - (d)-(p)-(v) address allocator and mapper
817 * @obj: objective iommu
818 * @da: contiguous iommu virtual memory
819 * @bytes: bytes for allocation
820 * @flags: iovma and page property
822 * Allocate @bytes linearly and creates 1-1-1 mapping and returns
823 * @da again, which might be adjusted if 'IOVMF_DA_ANON' is set.
825 u32
iommu_kmalloc(struct iommu
*obj
, u32 da
, size_t bytes
, u32 flags
)
830 if (!obj
|| !obj
->dev
|| !bytes
)
833 bytes
= PAGE_ALIGN(bytes
);
835 va
= kmalloc(bytes
, GFP_KERNEL
| GFP_DMA
);
838 pa
= virt_to_phys(va
);
840 flags
&= IOVMF_HW_MASK
;
841 flags
|= IOVMF_LINEAR
;
842 flags
|= IOVMF_ALLOC
;
843 flags
|= (da
? IOVMF_DA_FIXED
: IOVMF_DA_ANON
);
845 da
= __iommu_kmap(obj
, da
, pa
, va
, bytes
, flags
);
846 if (IS_ERR_VALUE(da
))
851 EXPORT_SYMBOL_GPL(iommu_kmalloc
);
854 * iommu_kfree - release virtual mapping obtained by 'iommu_kmalloc()'
855 * @obj: objective iommu
856 * @da: iommu device virtual address
858 * Frees the iommu virtually contiguous memory area starting at
859 * @da, which was passed to and was returned by'iommu_kmalloc()'.
861 void iommu_kfree(struct iommu
*obj
, u32 da
)
863 struct sg_table
*sgt
;
865 sgt
= unmap_vm_area(obj
, da
, kfree
, IOVMF_LINEAR
| IOVMF_ALLOC
);
867 dev_dbg(obj
->dev
, "%s: No sgt\n", __func__
);
870 EXPORT_SYMBOL_GPL(iommu_kfree
);
873 static int __init
iovmm_init(void)
875 const unsigned long flags
= SLAB_HWCACHE_ALIGN
;
876 struct kmem_cache
*p
;
878 p
= kmem_cache_create("iovm_area_cache", sizeof(struct iovm_struct
), 0,
882 iovm_area_cachep
= p
;
886 module_init(iovmm_init
);
888 static void __exit
iovmm_exit(void)
890 kmem_cache_destroy(iovm_area_cachep
);
892 module_exit(iovmm_exit
);
894 MODULE_DESCRIPTION("omap iommu: simple virtual address space management");
895 MODULE_AUTHOR("Hiroshi DOYU <Hiroshi.DOYU@nokia.com>");
896 MODULE_LICENSE("GPL v2");