2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
6 * Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com>
7 * Copyright (C) 2000, 2001 Ralf Baechle <ralf@gnu.org>
8 * Copyright (C) 2005 Ilya A. Volynets-Evenbakh <ilya@total-knowledge.com>
9 * swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
10 * IP32 changes by Ilya.
11 * Copyright (C) 2010 Cavium Networks, Inc.
13 #include <linux/dma-mapping.h>
14 #include <linux/scatterlist.h>
15 #include <linux/bootmem.h>
16 #include <linux/export.h>
17 #include <linux/swiotlb.h>
18 #include <linux/types.h>
19 #include <linux/init.h>
22 #include <asm/bootinfo.h>
24 #include <asm/octeon/octeon.h>
27 #include <asm/octeon/pci-octeon.h>
28 #include <asm/octeon/cvmx-npi-defs.h>
29 #include <asm/octeon/cvmx-pci-defs.h>
31 static dma_addr_t
octeon_hole_phys_to_dma(phys_addr_t paddr
)
33 if (paddr
>= CVMX_PCIE_BAR1_PHYS_BASE
&& paddr
< (CVMX_PCIE_BAR1_PHYS_BASE
+ CVMX_PCIE_BAR1_PHYS_SIZE
))
34 return paddr
- CVMX_PCIE_BAR1_PHYS_BASE
+ CVMX_PCIE_BAR1_RC_BASE
;
39 static phys_addr_t
octeon_hole_dma_to_phys(dma_addr_t daddr
)
41 if (daddr
>= CVMX_PCIE_BAR1_RC_BASE
)
42 return daddr
+ CVMX_PCIE_BAR1_PHYS_BASE
- CVMX_PCIE_BAR1_RC_BASE
;
47 static dma_addr_t
octeon_gen1_phys_to_dma(struct device
*dev
, phys_addr_t paddr
)
49 if (paddr
>= 0x410000000ull
&& paddr
< 0x420000000ull
)
50 paddr
-= 0x400000000ull
;
51 return octeon_hole_phys_to_dma(paddr
);
54 static phys_addr_t
octeon_gen1_dma_to_phys(struct device
*dev
, dma_addr_t daddr
)
56 daddr
= octeon_hole_dma_to_phys(daddr
);
58 if (daddr
>= 0x10000000ull
&& daddr
< 0x20000000ull
)
59 daddr
+= 0x400000000ull
;
64 static dma_addr_t
octeon_gen2_phys_to_dma(struct device
*dev
, phys_addr_t paddr
)
66 return octeon_hole_phys_to_dma(paddr
);
69 static phys_addr_t
octeon_gen2_dma_to_phys(struct device
*dev
, dma_addr_t daddr
)
71 return octeon_hole_dma_to_phys(daddr
);
74 static dma_addr_t
octeon_big_phys_to_dma(struct device
*dev
, phys_addr_t paddr
)
76 if (paddr
>= 0x410000000ull
&& paddr
< 0x420000000ull
)
77 paddr
-= 0x400000000ull
;
79 /* Anything in the BAR1 hole or above goes via BAR2 */
80 if (paddr
>= 0xf0000000ull
)
81 paddr
= OCTEON_BAR2_PCI_ADDRESS
+ paddr
;
86 static phys_addr_t
octeon_big_dma_to_phys(struct device
*dev
, dma_addr_t daddr
)
88 if (daddr
>= OCTEON_BAR2_PCI_ADDRESS
)
89 daddr
-= OCTEON_BAR2_PCI_ADDRESS
;
91 if (daddr
>= 0x10000000ull
&& daddr
< 0x20000000ull
)
92 daddr
+= 0x400000000ull
;
96 static dma_addr_t
octeon_small_phys_to_dma(struct device
*dev
,
99 if (paddr
>= 0x410000000ull
&& paddr
< 0x420000000ull
)
100 paddr
-= 0x400000000ull
;
102 /* Anything not in the BAR1 range goes via BAR2 */
103 if (paddr
>= octeon_bar1_pci_phys
&& paddr
< octeon_bar1_pci_phys
+ 0x8000000ull
)
104 paddr
= paddr
- octeon_bar1_pci_phys
;
106 paddr
= OCTEON_BAR2_PCI_ADDRESS
+ paddr
;
111 static phys_addr_t
octeon_small_dma_to_phys(struct device
*dev
,
114 if (daddr
>= OCTEON_BAR2_PCI_ADDRESS
)
115 daddr
-= OCTEON_BAR2_PCI_ADDRESS
;
117 daddr
+= octeon_bar1_pci_phys
;
119 if (daddr
>= 0x10000000ull
&& daddr
< 0x20000000ull
)
120 daddr
+= 0x400000000ull
;
124 #endif /* CONFIG_PCI */
126 static dma_addr_t
octeon_dma_map_page(struct device
*dev
, struct page
*page
,
127 unsigned long offset
, size_t size
, enum dma_data_direction direction
,
128 struct dma_attrs
*attrs
)
130 dma_addr_t daddr
= swiotlb_map_page(dev
, page
, offset
, size
,
137 static int octeon_dma_map_sg(struct device
*dev
, struct scatterlist
*sg
,
138 int nents
, enum dma_data_direction direction
, struct dma_attrs
*attrs
)
140 int r
= swiotlb_map_sg_attrs(dev
, sg
, nents
, direction
, attrs
);
145 static void octeon_dma_sync_single_for_device(struct device
*dev
,
146 dma_addr_t dma_handle
, size_t size
, enum dma_data_direction direction
)
148 swiotlb_sync_single_for_device(dev
, dma_handle
, size
, direction
);
152 static void octeon_dma_sync_sg_for_device(struct device
*dev
,
153 struct scatterlist
*sg
, int nelems
, enum dma_data_direction direction
)
155 swiotlb_sync_sg_for_device(dev
, sg
, nelems
, direction
);
159 static void *octeon_dma_alloc_coherent(struct device
*dev
, size_t size
,
160 dma_addr_t
*dma_handle
, gfp_t gfp
, struct dma_attrs
*attrs
)
164 if (dma_alloc_from_coherent(dev
, size
, dma_handle
, &ret
))
167 /* ignore region specifiers */
168 gfp
&= ~(__GFP_DMA
| __GFP_DMA32
| __GFP_HIGHMEM
);
170 #ifdef CONFIG_ZONE_DMA
173 else if (dev
->coherent_dma_mask
<= DMA_BIT_MASK(24))
177 #ifdef CONFIG_ZONE_DMA32
178 if (dev
->coherent_dma_mask
<= DMA_BIT_MASK(32))
184 /* Don't invoke OOM killer */
185 gfp
|= __GFP_NORETRY
;
187 ret
= swiotlb_alloc_coherent(dev
, size
, dma_handle
, gfp
);
194 static void octeon_dma_free_coherent(struct device
*dev
, size_t size
,
195 void *vaddr
, dma_addr_t dma_handle
, struct dma_attrs
*attrs
)
197 int order
= get_order(size
);
199 if (dma_release_from_coherent(dev
, order
, vaddr
))
202 swiotlb_free_coherent(dev
, size
, vaddr
, dma_handle
);
205 static dma_addr_t
octeon_unity_phys_to_dma(struct device
*dev
, phys_addr_t paddr
)
210 static phys_addr_t
octeon_unity_dma_to_phys(struct device
*dev
, dma_addr_t daddr
)
215 struct octeon_dma_map_ops
{
216 struct dma_map_ops dma_map_ops
;
217 dma_addr_t (*phys_to_dma
)(struct device
*dev
, phys_addr_t paddr
);
218 phys_addr_t (*dma_to_phys
)(struct device
*dev
, dma_addr_t daddr
);
221 dma_addr_t
phys_to_dma(struct device
*dev
, phys_addr_t paddr
)
223 struct octeon_dma_map_ops
*ops
= container_of(get_dma_ops(dev
),
224 struct octeon_dma_map_ops
,
227 return ops
->phys_to_dma(dev
, paddr
);
229 EXPORT_SYMBOL(phys_to_dma
);
231 phys_addr_t
dma_to_phys(struct device
*dev
, dma_addr_t daddr
)
233 struct octeon_dma_map_ops
*ops
= container_of(get_dma_ops(dev
),
234 struct octeon_dma_map_ops
,
237 return ops
->dma_to_phys(dev
, daddr
);
239 EXPORT_SYMBOL(dma_to_phys
);
241 static struct octeon_dma_map_ops octeon_linear_dma_map_ops
= {
243 .alloc
= octeon_dma_alloc_coherent
,
244 .free
= octeon_dma_free_coherent
,
245 .map_page
= octeon_dma_map_page
,
246 .unmap_page
= swiotlb_unmap_page
,
247 .map_sg
= octeon_dma_map_sg
,
248 .unmap_sg
= swiotlb_unmap_sg_attrs
,
249 .sync_single_for_cpu
= swiotlb_sync_single_for_cpu
,
250 .sync_single_for_device
= octeon_dma_sync_single_for_device
,
251 .sync_sg_for_cpu
= swiotlb_sync_sg_for_cpu
,
252 .sync_sg_for_device
= octeon_dma_sync_sg_for_device
,
253 .mapping_error
= swiotlb_dma_mapping_error
,
254 .dma_supported
= swiotlb_dma_supported
256 .phys_to_dma
= octeon_unity_phys_to_dma
,
257 .dma_to_phys
= octeon_unity_dma_to_phys
260 char *octeon_swiotlb
;
262 void __init
plat_swiotlb_setup(void)
265 phys_addr_t max_addr
;
266 phys_addr_t addr_size
;
268 unsigned long swiotlb_nslabs
;
273 for (i
= 0 ; i
< boot_mem_map
.nr_map
; i
++) {
274 struct boot_mem_map_entry
*e
= &boot_mem_map
.map
[i
];
275 if (e
->type
!= BOOT_MEM_RAM
&& e
->type
!= BOOT_MEM_INIT_RAM
)
278 /* These addresses map low for PCI. */
279 if (e
->addr
> 0x410000000ull
&& !OCTEON_IS_OCTEON2())
282 addr_size
+= e
->size
;
284 if (max_addr
< e
->addr
+ e
->size
)
285 max_addr
= e
->addr
+ e
->size
;
289 swiotlbsize
= PAGE_SIZE
;
293 * For OCTEON_DMA_BAR_TYPE_SMALL, size the iotlb at 1/4 memory
294 * size to a maximum of 64MB
296 if (OCTEON_IS_MODEL(OCTEON_CN31XX
)
297 || OCTEON_IS_MODEL(OCTEON_CN38XX_PASS2
)) {
298 swiotlbsize
= addr_size
/ 4;
299 if (swiotlbsize
> 64 * (1<<20))
300 swiotlbsize
= 64 * (1<<20);
301 } else if (max_addr
> 0xf0000000ul
) {
303 * Otherwise only allocate a big iotlb if there is
304 * memory past the BAR1 hole.
306 swiotlbsize
= 64 * (1<<20);
309 #ifdef CONFIG_USB_OHCI_HCD_PLATFORM
310 /* OCTEON II ohci is only 32-bit. */
311 if (OCTEON_IS_OCTEON2() && max_addr
>= 0x100000000ul
)
312 swiotlbsize
= 64 * (1<<20);
314 swiotlb_nslabs
= swiotlbsize
>> IO_TLB_SHIFT
;
315 swiotlb_nslabs
= ALIGN(swiotlb_nslabs
, IO_TLB_SEGSIZE
);
316 swiotlbsize
= swiotlb_nslabs
<< IO_TLB_SHIFT
;
318 octeon_swiotlb
= alloc_bootmem_low_pages(swiotlbsize
);
320 if (swiotlb_init_with_tbl(octeon_swiotlb
, swiotlb_nslabs
, 1) == -ENOMEM
)
321 panic("Cannot allocate SWIOTLB buffer");
323 mips_dma_map_ops
= &octeon_linear_dma_map_ops
.dma_map_ops
;
327 static struct octeon_dma_map_ops _octeon_pci_dma_map_ops
= {
329 .alloc
= octeon_dma_alloc_coherent
,
330 .free
= octeon_dma_free_coherent
,
331 .map_page
= octeon_dma_map_page
,
332 .unmap_page
= swiotlb_unmap_page
,
333 .map_sg
= octeon_dma_map_sg
,
334 .unmap_sg
= swiotlb_unmap_sg_attrs
,
335 .sync_single_for_cpu
= swiotlb_sync_single_for_cpu
,
336 .sync_single_for_device
= octeon_dma_sync_single_for_device
,
337 .sync_sg_for_cpu
= swiotlb_sync_sg_for_cpu
,
338 .sync_sg_for_device
= octeon_dma_sync_sg_for_device
,
339 .mapping_error
= swiotlb_dma_mapping_error
,
340 .dma_supported
= swiotlb_dma_supported
344 struct dma_map_ops
*octeon_pci_dma_map_ops
;
346 void __init
octeon_pci_dma_init(void)
348 switch (octeon_dma_bar_type
) {
349 case OCTEON_DMA_BAR_TYPE_PCIE2
:
350 _octeon_pci_dma_map_ops
.phys_to_dma
= octeon_gen2_phys_to_dma
;
351 _octeon_pci_dma_map_ops
.dma_to_phys
= octeon_gen2_dma_to_phys
;
353 case OCTEON_DMA_BAR_TYPE_PCIE
:
354 _octeon_pci_dma_map_ops
.phys_to_dma
= octeon_gen1_phys_to_dma
;
355 _octeon_pci_dma_map_ops
.dma_to_phys
= octeon_gen1_dma_to_phys
;
357 case OCTEON_DMA_BAR_TYPE_BIG
:
358 _octeon_pci_dma_map_ops
.phys_to_dma
= octeon_big_phys_to_dma
;
359 _octeon_pci_dma_map_ops
.dma_to_phys
= octeon_big_dma_to_phys
;
361 case OCTEON_DMA_BAR_TYPE_SMALL
:
362 _octeon_pci_dma_map_ops
.phys_to_dma
= octeon_small_phys_to_dma
;
363 _octeon_pci_dma_map_ops
.dma_to_phys
= octeon_small_dma_to_phys
;
368 octeon_pci_dma_map_ops
= &_octeon_pci_dma_map_ops
.dma_map_ops
;
370 #endif /* CONFIG_PCI */