2 * Copyright IBM Corp. 2012
5 * Jan Glauber <jang@linux.vnet.ibm.com>
8 #include <linux/kernel.h>
9 #include <linux/slab.h>
10 #include <linux/export.h>
11 #include <linux/iommu-helper.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/vmalloc.h>
14 #include <linux/pci.h>
15 #include <asm/pci_dma.h>
17 static struct kmem_cache
*dma_region_table_cache
;
18 static struct kmem_cache
*dma_page_table_cache
;
19 static int s390_iommu_strict
;
21 static int zpci_refresh_global(struct zpci_dev
*zdev
)
23 return zpci_refresh_trans((u64
) zdev
->fh
<< 32, zdev
->start_dma
,
24 zdev
->iommu_pages
* PAGE_SIZE
);
27 static unsigned long *dma_alloc_cpu_table(void)
29 unsigned long *table
, *entry
;
31 table
= kmem_cache_alloc(dma_region_table_cache
, GFP_ATOMIC
);
35 for (entry
= table
; entry
< table
+ ZPCI_TABLE_ENTRIES
; entry
++)
36 *entry
= ZPCI_TABLE_INVALID
| ZPCI_TABLE_PROTECTED
;
40 static void dma_free_cpu_table(void *table
)
42 kmem_cache_free(dma_region_table_cache
, table
);
45 static unsigned long *dma_alloc_page_table(void)
47 unsigned long *table
, *entry
;
49 table
= kmem_cache_alloc(dma_page_table_cache
, GFP_ATOMIC
);
53 for (entry
= table
; entry
< table
+ ZPCI_PT_ENTRIES
; entry
++)
54 *entry
= ZPCI_PTE_INVALID
| ZPCI_TABLE_PROTECTED
;
58 static void dma_free_page_table(void *table
)
60 kmem_cache_free(dma_page_table_cache
, table
);
63 static unsigned long *dma_get_seg_table_origin(unsigned long *entry
)
67 if (reg_entry_isvalid(*entry
))
68 sto
= get_rt_sto(*entry
);
70 sto
= dma_alloc_cpu_table();
74 set_rt_sto(entry
, sto
);
75 validate_rt_entry(entry
);
76 entry_clr_protected(entry
);
81 static unsigned long *dma_get_page_table_origin(unsigned long *entry
)
85 if (reg_entry_isvalid(*entry
))
86 pto
= get_st_pto(*entry
);
88 pto
= dma_alloc_page_table();
91 set_st_pto(entry
, pto
);
92 validate_st_entry(entry
);
93 entry_clr_protected(entry
);
98 static unsigned long *dma_walk_cpu_trans(unsigned long *rto
, dma_addr_t dma_addr
)
100 unsigned long *sto
, *pto
;
101 unsigned int rtx
, sx
, px
;
103 rtx
= calc_rtx(dma_addr
);
104 sto
= dma_get_seg_table_origin(&rto
[rtx
]);
108 sx
= calc_sx(dma_addr
);
109 pto
= dma_get_page_table_origin(&sto
[sx
]);
113 px
= calc_px(dma_addr
);
117 static void dma_update_cpu_trans(struct zpci_dev
*zdev
, void *page_addr
,
118 dma_addr_t dma_addr
, int flags
)
120 unsigned long *entry
;
122 entry
= dma_walk_cpu_trans(zdev
->dma_table
, dma_addr
);
128 if (flags
& ZPCI_PTE_INVALID
) {
129 invalidate_pt_entry(entry
);
132 set_pt_pfaa(entry
, page_addr
);
133 validate_pt_entry(entry
);
136 if (flags
& ZPCI_TABLE_PROTECTED
)
137 entry_set_protected(entry
);
139 entry_clr_protected(entry
);
142 static int dma_update_trans(struct zpci_dev
*zdev
, unsigned long pa
,
143 dma_addr_t dma_addr
, size_t size
, int flags
)
145 unsigned int nr_pages
= PAGE_ALIGN(size
) >> PAGE_SHIFT
;
146 u8
*page_addr
= (u8
*) (pa
& PAGE_MASK
);
147 dma_addr_t start_dma_addr
= dma_addr
;
148 unsigned long irq_flags
;
154 spin_lock_irqsave(&zdev
->dma_table_lock
, irq_flags
);
155 if (!zdev
->dma_table
)
158 for (i
= 0; i
< nr_pages
; i
++) {
159 dma_update_cpu_trans(zdev
, page_addr
, dma_addr
, flags
);
160 page_addr
+= PAGE_SIZE
;
161 dma_addr
+= PAGE_SIZE
;
165 * With zdev->tlb_refresh == 0, rpcit is not required to establish new
166 * translations when previously invalid translation-table entries are
167 * validated. With lazy unmap, it also is skipped for previously valid
168 * entries, but a global rpcit is then required before any address can
169 * be re-used, i.e. after each iommu bitmap wrap-around.
171 if (!zdev
->tlb_refresh
&&
172 (!s390_iommu_strict
||
173 ((flags
& ZPCI_PTE_VALID_MASK
) == ZPCI_PTE_VALID
)))
176 rc
= zpci_refresh_trans((u64
) zdev
->fh
<< 32, start_dma_addr
,
177 nr_pages
* PAGE_SIZE
);
180 spin_unlock_irqrestore(&zdev
->dma_table_lock
, irq_flags
);
184 static void dma_free_seg_table(unsigned long entry
)
186 unsigned long *sto
= get_rt_sto(entry
);
189 for (sx
= 0; sx
< ZPCI_TABLE_ENTRIES
; sx
++)
190 if (reg_entry_isvalid(sto
[sx
]))
191 dma_free_page_table(get_st_pto(sto
[sx
]));
193 dma_free_cpu_table(sto
);
196 static void dma_cleanup_tables(struct zpci_dev
*zdev
)
198 unsigned long *table
;
201 if (!zdev
|| !zdev
->dma_table
)
204 table
= zdev
->dma_table
;
205 for (rtx
= 0; rtx
< ZPCI_TABLE_ENTRIES
; rtx
++)
206 if (reg_entry_isvalid(table
[rtx
]))
207 dma_free_seg_table(table
[rtx
]);
209 dma_free_cpu_table(table
);
210 zdev
->dma_table
= NULL
;
213 static unsigned long __dma_alloc_iommu(struct zpci_dev
*zdev
,
214 unsigned long start
, int size
)
216 unsigned long boundary_size
;
218 boundary_size
= ALIGN(dma_get_seg_boundary(&zdev
->pdev
->dev
) + 1,
219 PAGE_SIZE
) >> PAGE_SHIFT
;
220 return iommu_area_alloc(zdev
->iommu_bitmap
, zdev
->iommu_pages
,
221 start
, size
, 0, boundary_size
, 0);
224 static unsigned long dma_alloc_iommu(struct zpci_dev
*zdev
, int size
)
226 unsigned long offset
, flags
;
229 spin_lock_irqsave(&zdev
->iommu_bitmap_lock
, flags
);
230 offset
= __dma_alloc_iommu(zdev
, zdev
->next_bit
, size
);
233 offset
= __dma_alloc_iommu(zdev
, 0, size
);
238 zdev
->next_bit
= offset
+ size
;
239 if (!zdev
->tlb_refresh
&& !s390_iommu_strict
&& wrap
)
240 /* global flush after wrap-around with lazy unmap */
241 zpci_refresh_global(zdev
);
243 spin_unlock_irqrestore(&zdev
->iommu_bitmap_lock
, flags
);
247 static void dma_free_iommu(struct zpci_dev
*zdev
, unsigned long offset
, int size
)
251 spin_lock_irqsave(&zdev
->iommu_bitmap_lock
, flags
);
252 if (!zdev
->iommu_bitmap
)
254 bitmap_clear(zdev
->iommu_bitmap
, offset
, size
);
256 * Lazy flush for unmap: need to move next_bit to avoid address re-use
259 if (!s390_iommu_strict
&& offset
>= zdev
->next_bit
)
260 zdev
->next_bit
= offset
+ size
;
262 spin_unlock_irqrestore(&zdev
->iommu_bitmap_lock
, flags
);
265 int dma_set_mask(struct device
*dev
, u64 mask
)
267 if (!dev
->dma_mask
|| !dma_supported(dev
, mask
))
270 *dev
->dma_mask
= mask
;
273 EXPORT_SYMBOL_GPL(dma_set_mask
);
275 static dma_addr_t
s390_dma_map_pages(struct device
*dev
, struct page
*page
,
276 unsigned long offset
, size_t size
,
277 enum dma_data_direction direction
,
278 struct dma_attrs
*attrs
)
280 struct zpci_dev
*zdev
= get_zdev(to_pci_dev(dev
));
281 unsigned long nr_pages
, iommu_page_index
;
282 unsigned long pa
= page_to_phys(page
) + offset
;
283 int flags
= ZPCI_PTE_VALID
;
286 /* This rounds up number of pages based on size and offset */
287 nr_pages
= iommu_num_pages(pa
, size
, PAGE_SIZE
);
288 iommu_page_index
= dma_alloc_iommu(zdev
, nr_pages
);
289 if (iommu_page_index
== -1)
292 /* Use rounded up size */
293 size
= nr_pages
* PAGE_SIZE
;
295 dma_addr
= zdev
->start_dma
+ iommu_page_index
* PAGE_SIZE
;
296 if (dma_addr
+ size
> zdev
->end_dma
)
299 if (direction
== DMA_NONE
|| direction
== DMA_TO_DEVICE
)
300 flags
|= ZPCI_TABLE_PROTECTED
;
302 if (!dma_update_trans(zdev
, pa
, dma_addr
, size
, flags
)) {
303 atomic64_add(nr_pages
, &zdev
->fmb
->mapped_pages
);
304 return dma_addr
+ (offset
& ~PAGE_MASK
);
308 dma_free_iommu(zdev
, iommu_page_index
, nr_pages
);
310 zpci_err("map error:\n");
311 zpci_err_hex(&pa
, sizeof(pa
));
312 return DMA_ERROR_CODE
;
315 static void s390_dma_unmap_pages(struct device
*dev
, dma_addr_t dma_addr
,
316 size_t size
, enum dma_data_direction direction
,
317 struct dma_attrs
*attrs
)
319 struct zpci_dev
*zdev
= get_zdev(to_pci_dev(dev
));
320 unsigned long iommu_page_index
;
323 npages
= iommu_num_pages(dma_addr
, size
, PAGE_SIZE
);
324 dma_addr
= dma_addr
& PAGE_MASK
;
325 if (dma_update_trans(zdev
, 0, dma_addr
, npages
* PAGE_SIZE
,
326 ZPCI_TABLE_PROTECTED
| ZPCI_PTE_INVALID
)) {
327 zpci_err("unmap error:\n");
328 zpci_err_hex(&dma_addr
, sizeof(dma_addr
));
331 atomic64_add(npages
, &zdev
->fmb
->unmapped_pages
);
332 iommu_page_index
= (dma_addr
- zdev
->start_dma
) >> PAGE_SHIFT
;
333 dma_free_iommu(zdev
, iommu_page_index
, npages
);
336 static void *s390_dma_alloc(struct device
*dev
, size_t size
,
337 dma_addr_t
*dma_handle
, gfp_t flag
,
338 struct dma_attrs
*attrs
)
340 struct zpci_dev
*zdev
= get_zdev(to_pci_dev(dev
));
345 size
= PAGE_ALIGN(size
);
346 page
= alloc_pages(flag
, get_order(size
));
350 pa
= page_to_phys(page
);
351 memset((void *) pa
, 0, size
);
353 map
= s390_dma_map_pages(dev
, page
, pa
% PAGE_SIZE
,
354 size
, DMA_BIDIRECTIONAL
, NULL
);
355 if (dma_mapping_error(dev
, map
)) {
356 free_pages(pa
, get_order(size
));
360 atomic64_add(size
/ PAGE_SIZE
, &zdev
->fmb
->allocated_pages
);
366 static void s390_dma_free(struct device
*dev
, size_t size
,
367 void *pa
, dma_addr_t dma_handle
,
368 struct dma_attrs
*attrs
)
370 struct zpci_dev
*zdev
= get_zdev(to_pci_dev(dev
));
372 size
= PAGE_ALIGN(size
);
373 atomic64_sub(size
/ PAGE_SIZE
, &zdev
->fmb
->allocated_pages
);
374 s390_dma_unmap_pages(dev
, dma_handle
, size
, DMA_BIDIRECTIONAL
, NULL
);
375 free_pages((unsigned long) pa
, get_order(size
));
378 static int s390_dma_map_sg(struct device
*dev
, struct scatterlist
*sg
,
379 int nr_elements
, enum dma_data_direction dir
,
380 struct dma_attrs
*attrs
)
382 int mapped_elements
= 0;
383 struct scatterlist
*s
;
386 for_each_sg(sg
, s
, nr_elements
, i
) {
387 struct page
*page
= sg_page(s
);
388 s
->dma_address
= s390_dma_map_pages(dev
, page
, s
->offset
,
389 s
->length
, dir
, NULL
);
390 if (!dma_mapping_error(dev
, s
->dma_address
)) {
391 s
->dma_length
= s
->length
;
397 return mapped_elements
;
400 for_each_sg(sg
, s
, mapped_elements
, i
) {
402 s390_dma_unmap_pages(dev
, s
->dma_address
, s
->dma_length
,
411 static void s390_dma_unmap_sg(struct device
*dev
, struct scatterlist
*sg
,
412 int nr_elements
, enum dma_data_direction dir
,
413 struct dma_attrs
*attrs
)
415 struct scatterlist
*s
;
418 for_each_sg(sg
, s
, nr_elements
, i
) {
419 s390_dma_unmap_pages(dev
, s
->dma_address
, s
->dma_length
, dir
, NULL
);
425 int zpci_dma_init_device(struct zpci_dev
*zdev
)
429 spin_lock_init(&zdev
->iommu_bitmap_lock
);
430 spin_lock_init(&zdev
->dma_table_lock
);
432 zdev
->dma_table
= dma_alloc_cpu_table();
433 if (!zdev
->dma_table
) {
438 zdev
->iommu_size
= (unsigned long) high_memory
- PAGE_OFFSET
;
439 zdev
->iommu_pages
= zdev
->iommu_size
>> PAGE_SHIFT
;
440 zdev
->iommu_bitmap
= vzalloc(zdev
->iommu_pages
/ 8);
441 if (!zdev
->iommu_bitmap
) {
446 rc
= zpci_register_ioat(zdev
,
448 zdev
->start_dma
+ PAGE_OFFSET
,
449 zdev
->start_dma
+ zdev
->iommu_size
- 1,
450 (u64
) zdev
->dma_table
);
456 dma_free_cpu_table(zdev
->dma_table
);
461 void zpci_dma_exit_device(struct zpci_dev
*zdev
)
463 zpci_unregister_ioat(zdev
, 0);
464 dma_cleanup_tables(zdev
);
465 vfree(zdev
->iommu_bitmap
);
466 zdev
->iommu_bitmap
= NULL
;
470 static int __init
dma_alloc_cpu_table_caches(void)
472 dma_region_table_cache
= kmem_cache_create("PCI_DMA_region_tables",
473 ZPCI_TABLE_SIZE
, ZPCI_TABLE_ALIGN
,
475 if (!dma_region_table_cache
)
478 dma_page_table_cache
= kmem_cache_create("PCI_DMA_page_tables",
479 ZPCI_PT_SIZE
, ZPCI_PT_ALIGN
,
481 if (!dma_page_table_cache
) {
482 kmem_cache_destroy(dma_region_table_cache
);
488 int __init
zpci_dma_init(void)
490 return dma_alloc_cpu_table_caches();
493 void zpci_dma_exit(void)
495 kmem_cache_destroy(dma_page_table_cache
);
496 kmem_cache_destroy(dma_region_table_cache
);
499 #define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
501 static int __init
dma_debug_do_init(void)
503 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES
);
506 fs_initcall(dma_debug_do_init
);
508 struct dma_map_ops s390_dma_ops
= {
509 .alloc
= s390_dma_alloc
,
510 .free
= s390_dma_free
,
511 .map_sg
= s390_dma_map_sg
,
512 .unmap_sg
= s390_dma_unmap_sg
,
513 .map_page
= s390_dma_map_pages
,
514 .unmap_page
= s390_dma_unmap_pages
,
515 /* if we support direct DMA this must be conditional */
517 /* dma_supported is unconditionally true without a callback */
519 EXPORT_SYMBOL_GPL(s390_dma_ops
);
521 static int __init
s390_iommu_setup(char *str
)
523 if (!strncmp(str
, "strict", 6))
524 s390_iommu_strict
= 1;
528 __setup("s390_iommu=", s390_iommu_setup
);