1 // SPDX-License-Identifier: GPL-2.0-only
3 * linux/arch/arm/mm/ioremap.c
5 * Re-map IO memory to kernel address space so that we can access it.
7 * (C) Copyright 1995 1996 Linus Torvalds
9 * Hacked for ARM by Phil Blundell <philb@gnu.org>
10 * Hacked to allow all architectures to build, and various cleanups
13 * This allows a driver to remap an arbitrary region of bus memory into
14 * virtual space. One should *only* use readl, writel, memcpy_toio and
15 * so on with such remapped areas.
17 * Because the ARM only has a 32-bit address space we can't address the
18 * whole of the (physical) PCI space at once. PCI huge-mode addressing
19 * allows us to circumvent this restriction by splitting PCI space into
20 * two 2GB chunks and mapping only one at a time into processor memory.
21 * We use MMU protection domains to trap any attempt to access the bank
22 * that is not currently mapped. (This isn't fully implemented yet.)
24 #include <linux/module.h>
25 #include <linux/errno.h>
27 #include <linux/vmalloc.h>
29 #include <linux/sizes.h>
32 #include <asm/cputype.h>
33 #include <asm/cacheflush.h>
34 #include <asm/early_ioremap.h>
35 #include <asm/mmu_context.h>
36 #include <asm/pgalloc.h>
37 #include <asm/tlbflush.h>
38 #include <asm/system_info.h>
40 #include <asm/mach/map.h>
41 #include <asm/mach/pci.h>
45 LIST_HEAD(static_vmlist
);
47 static struct static_vm
*find_static_vm_paddr(phys_addr_t paddr
,
48 size_t size
, unsigned int mtype
)
50 struct static_vm
*svm
;
53 list_for_each_entry(svm
, &static_vmlist
, list
) {
55 if (!(vm
->flags
& VM_ARM_STATIC_MAPPING
))
57 if ((vm
->flags
& VM_ARM_MTYPE_MASK
) != VM_ARM_MTYPE(mtype
))
60 if (vm
->phys_addr
> paddr
||
61 paddr
+ size
- 1 > vm
->phys_addr
+ vm
->size
- 1)
70 struct static_vm
*find_static_vm_vaddr(void *vaddr
)
72 struct static_vm
*svm
;
75 list_for_each_entry(svm
, &static_vmlist
, list
) {
78 /* static_vmlist is ascending order */
82 if (vm
->addr
<= vaddr
&& vm
->addr
+ vm
->size
> vaddr
)
89 void __init
add_static_vm_early(struct static_vm
*svm
)
91 struct static_vm
*curr_svm
;
96 vm_area_add_early(vm
);
99 list_for_each_entry(curr_svm
, &static_vmlist
, list
) {
102 if (vm
->addr
> vaddr
)
105 list_add_tail(&svm
->list
, &curr_svm
->list
);
108 int ioremap_page(unsigned long virt
, unsigned long phys
,
109 const struct mem_type
*mtype
)
111 return ioremap_page_range(virt
, virt
+ PAGE_SIZE
, phys
,
112 __pgprot(mtype
->prot_pte
));
114 EXPORT_SYMBOL(ioremap_page
);
116 void __check_vmalloc_seq(struct mm_struct
*mm
)
121 seq
= init_mm
.context
.vmalloc_seq
;
122 memcpy(pgd_offset(mm
, VMALLOC_START
),
123 pgd_offset_k(VMALLOC_START
),
124 sizeof(pgd_t
) * (pgd_index(VMALLOC_END
) -
125 pgd_index(VMALLOC_START
)));
126 mm
->context
.vmalloc_seq
= seq
;
127 } while (seq
!= init_mm
.context
.vmalloc_seq
);
130 #if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE)
132 * Section support is unsafe on SMP - If you iounmap and ioremap a region,
133 * the other CPUs will not see this change until their next context switch.
134 * Meanwhile, (eg) if an interrupt comes in on one of those other CPUs
135 * which requires the new ioremap'd region to be referenced, the CPU will
136 * reference the _old_ region.
138 * Note that get_vm_area_caller() allocates a guard 4K page, so we need to
139 * mask the size back to 1MB aligned or we will overflow in the loop below.
141 static void unmap_area_sections(unsigned long virt
, unsigned long size
)
143 unsigned long addr
= virt
, end
= virt
+ (size
& ~(SZ_1M
- 1));
148 flush_cache_vunmap(addr
, end
);
149 pgd
= pgd_offset_k(addr
);
150 pud
= pud_offset(pgd
, addr
);
151 pmdp
= pmd_offset(pud
, addr
);
155 if (!pmd_none(pmd
)) {
157 * Clear the PMD from the page table, and
158 * increment the vmalloc sequence so others
159 * notice this change.
161 * Note: this is still racy on SMP machines.
164 init_mm
.context
.vmalloc_seq
++;
167 * Free the page table, if there was one.
169 if ((pmd_val(pmd
) & PMD_TYPE_MASK
) == PMD_TYPE_TABLE
)
170 pte_free_kernel(&init_mm
, pmd_page_vaddr(pmd
));
175 } while (addr
< end
);
178 * Ensure that the active_mm is up to date - we want to
179 * catch any use-after-iounmap cases.
181 if (current
->active_mm
->context
.vmalloc_seq
!= init_mm
.context
.vmalloc_seq
)
182 __check_vmalloc_seq(current
->active_mm
);
184 flush_tlb_kernel_range(virt
, end
);
188 remap_area_sections(unsigned long virt
, unsigned long pfn
,
189 size_t size
, const struct mem_type
*type
)
191 unsigned long addr
= virt
, end
= virt
+ size
;
197 * Remove and free any PTE-based mapping, and
198 * sync the current kernel mapping.
200 unmap_area_sections(virt
, size
);
202 pgd
= pgd_offset_k(addr
);
203 pud
= pud_offset(pgd
, addr
);
204 pmd
= pmd_offset(pud
, addr
);
206 pmd
[0] = __pmd(__pfn_to_phys(pfn
) | type
->prot_sect
);
207 pfn
+= SZ_1M
>> PAGE_SHIFT
;
208 pmd
[1] = __pmd(__pfn_to_phys(pfn
) | type
->prot_sect
);
209 pfn
+= SZ_1M
>> PAGE_SHIFT
;
210 flush_pmd_entry(pmd
);
214 } while (addr
< end
);
220 remap_area_supersections(unsigned long virt
, unsigned long pfn
,
221 size_t size
, const struct mem_type
*type
)
223 unsigned long addr
= virt
, end
= virt
+ size
;
229 * Remove and free any PTE-based mapping, and
230 * sync the current kernel mapping.
232 unmap_area_sections(virt
, size
);
234 pgd
= pgd_offset_k(virt
);
235 pud
= pud_offset(pgd
, addr
);
236 pmd
= pmd_offset(pud
, addr
);
238 unsigned long super_pmd_val
, i
;
240 super_pmd_val
= __pfn_to_phys(pfn
) | type
->prot_sect
|
242 super_pmd_val
|= ((pfn
>> (32 - PAGE_SHIFT
)) & 0xf) << 20;
244 for (i
= 0; i
< 8; i
++) {
245 pmd
[0] = __pmd(super_pmd_val
);
246 pmd
[1] = __pmd(super_pmd_val
);
247 flush_pmd_entry(pmd
);
253 pfn
+= SUPERSECTION_SIZE
>> PAGE_SHIFT
;
254 } while (addr
< end
);
260 static void __iomem
* __arm_ioremap_pfn_caller(unsigned long pfn
,
261 unsigned long offset
, size_t size
, unsigned int mtype
, void *caller
)
263 const struct mem_type
*type
;
266 struct vm_struct
*area
;
267 phys_addr_t paddr
= __pfn_to_phys(pfn
);
269 #ifndef CONFIG_ARM_LPAE
271 * High mappings must be supersection aligned
273 if (pfn
>= 0x100000 && (paddr
& ~SUPERSECTION_MASK
))
277 type
= get_mem_type(mtype
);
282 * Page align the mapping size, taking account of any offset.
284 size
= PAGE_ALIGN(offset
+ size
);
287 * Try to reuse one of the static mapping whenever possible.
289 if (size
&& !(sizeof(phys_addr_t
) == 4 && pfn
>= 0x100000)) {
290 struct static_vm
*svm
;
292 svm
= find_static_vm_paddr(paddr
, size
, mtype
);
294 addr
= (unsigned long)svm
->vm
.addr
;
295 addr
+= paddr
- svm
->vm
.phys_addr
;
296 return (void __iomem
*) (offset
+ addr
);
301 * Don't allow RAM to be mapped with mismatched attributes - this
302 * causes problems with ARMv6+
304 if (WARN_ON(pfn_valid(pfn
) && mtype
!= MT_MEMORY_RW
))
307 area
= get_vm_area_caller(size
, VM_IOREMAP
, caller
);
310 addr
= (unsigned long)area
->addr
;
311 area
->phys_addr
= paddr
;
313 #if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE)
314 if (DOMAIN_IO
== 0 &&
315 (((cpu_architecture() >= CPU_ARCH_ARMv6
) && (get_cr() & CR_XP
)) ||
316 cpu_is_xsc3()) && pfn
>= 0x100000 &&
317 !((paddr
| size
| addr
) & ~SUPERSECTION_MASK
)) {
318 area
->flags
|= VM_ARM_SECTION_MAPPING
;
319 err
= remap_area_supersections(addr
, pfn
, size
, type
);
320 } else if (!((paddr
| size
| addr
) & ~PMD_MASK
)) {
321 area
->flags
|= VM_ARM_SECTION_MAPPING
;
322 err
= remap_area_sections(addr
, pfn
, size
, type
);
325 err
= ioremap_page_range(addr
, addr
+ size
, paddr
,
326 __pgprot(type
->prot_pte
));
329 vunmap((void *)addr
);
333 flush_cache_vmap(addr
, addr
+ size
);
334 return (void __iomem
*) (offset
+ addr
);
337 void __iomem
*__arm_ioremap_caller(phys_addr_t phys_addr
, size_t size
,
338 unsigned int mtype
, void *caller
)
340 phys_addr_t last_addr
;
341 unsigned long offset
= phys_addr
& ~PAGE_MASK
;
342 unsigned long pfn
= __phys_to_pfn(phys_addr
);
345 * Don't allow wraparound or zero size
347 last_addr
= phys_addr
+ size
- 1;
348 if (!size
|| last_addr
< phys_addr
)
351 return __arm_ioremap_pfn_caller(pfn
, offset
, size
, mtype
,
356 * Remap an arbitrary physical address space into the kernel virtual
357 * address space. Needed when the kernel wants to access high addresses
360 * NOTE! We need to allow non-page-aligned mappings too: we will obviously
361 * have to convert them into an offset in a page-aligned mapping, but the
362 * caller shouldn't need to know that small detail.
365 __arm_ioremap_pfn(unsigned long pfn
, unsigned long offset
, size_t size
,
368 return __arm_ioremap_pfn_caller(pfn
, offset
, size
, mtype
,
369 __builtin_return_address(0));
371 EXPORT_SYMBOL(__arm_ioremap_pfn
);
373 void __iomem
* (*arch_ioremap_caller
)(phys_addr_t
, size_t,
374 unsigned int, void *) =
375 __arm_ioremap_caller
;
377 void __iomem
*ioremap(resource_size_t res_cookie
, size_t size
)
379 return arch_ioremap_caller(res_cookie
, size
, MT_DEVICE
,
380 __builtin_return_address(0));
382 EXPORT_SYMBOL(ioremap
);
384 void __iomem
*ioremap_cache(resource_size_t res_cookie
, size_t size
)
386 return arch_ioremap_caller(res_cookie
, size
, MT_DEVICE_CACHED
,
387 __builtin_return_address(0));
389 EXPORT_SYMBOL(ioremap_cache
);
391 void __iomem
*ioremap_wc(resource_size_t res_cookie
, size_t size
)
393 return arch_ioremap_caller(res_cookie
, size
, MT_DEVICE_WC
,
394 __builtin_return_address(0));
396 EXPORT_SYMBOL(ioremap_wc
);
399 * Remap an arbitrary physical address space into the kernel virtual
400 * address space as memory. Needed when the kernel wants to execute
401 * code in external memory. This is needed for reprogramming source
402 * clocks that would affect normal memory for example. Please see
403 * CONFIG_GENERIC_ALLOCATOR for allocating external memory.
406 __arm_ioremap_exec(phys_addr_t phys_addr
, size_t size
, bool cached
)
411 mtype
= MT_MEMORY_RWX
;
413 mtype
= MT_MEMORY_RWX_NONCACHED
;
415 return __arm_ioremap_caller(phys_addr
, size
, mtype
,
416 __builtin_return_address(0));
419 void *arch_memremap_wb(phys_addr_t phys_addr
, size_t size
)
421 return (__force
void *)arch_ioremap_caller(phys_addr
, size
,
423 __builtin_return_address(0));
426 void __iounmap(volatile void __iomem
*io_addr
)
428 void *addr
= (void *)(PAGE_MASK
& (unsigned long)io_addr
);
429 struct static_vm
*svm
;
431 /* If this is a static mapping, we must leave it alone */
432 svm
= find_static_vm_vaddr(addr
);
436 #if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE)
438 struct vm_struct
*vm
;
440 vm
= find_vm_area(addr
);
443 * If this is a section based mapping we need to handle it
444 * specially as the VM subsystem does not know how to handle
447 if (vm
&& (vm
->flags
& VM_ARM_SECTION_MAPPING
))
448 unmap_area_sections((unsigned long)vm
->addr
, vm
->size
);
455 void (*arch_iounmap
)(volatile void __iomem
*) = __iounmap
;
457 void iounmap(volatile void __iomem
*cookie
)
459 arch_iounmap(cookie
);
461 EXPORT_SYMBOL(iounmap
);
464 static int pci_ioremap_mem_type
= MT_DEVICE
;
466 void pci_ioremap_set_mem_type(int mem_type
)
468 pci_ioremap_mem_type
= mem_type
;
471 int pci_ioremap_io(unsigned int offset
, phys_addr_t phys_addr
)
473 BUG_ON(offset
+ SZ_64K
- 1 > IO_SPACE_LIMIT
);
475 return ioremap_page_range(PCI_IO_VIRT_BASE
+ offset
,
476 PCI_IO_VIRT_BASE
+ offset
+ SZ_64K
,
478 __pgprot(get_mem_type(pci_ioremap_mem_type
)->prot_pte
));
480 EXPORT_SYMBOL_GPL(pci_ioremap_io
);
482 void __iomem
*pci_remap_cfgspace(resource_size_t res_cookie
, size_t size
)
484 return arch_ioremap_caller(res_cookie
, size
, MT_UNCACHED
,
485 __builtin_return_address(0));
487 EXPORT_SYMBOL_GPL(pci_remap_cfgspace
);
491 * Must be called after early_fixmap_init
493 void __init
early_ioremap_init(void)
495 early_ioremap_setup();