2 * This file contains ioremap and related functions for 64-bit machines.
4 * Derived from arch/ppc64/mm/init.c
5 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
7 * Modifications by Paul Mackerras (PowerMac) (paulus@samba.org)
8 * and Cort Dougan (PReP) (cort@cs.nmt.edu)
9 * Copyright (C) 1996 Paul Mackerras
11 * Derived from "arch/i386/mm/init.c"
12 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
14 * Dave Engebretsen <engebret@us.ibm.com>
15 * Rework for PPC64 port.
17 * This program is free software; you can redistribute it and/or
18 * modify it under the terms of the GNU General Public License
19 * as published by the Free Software Foundation; either version
20 * 2 of the License, or (at your option) any later version.
24 #include <linux/signal.h>
25 #include <linux/sched.h>
26 #include <linux/kernel.h>
27 #include <linux/errno.h>
28 #include <linux/string.h>
29 #include <linux/types.h>
30 #include <linux/mman.h>
32 #include <linux/swap.h>
33 #include <linux/stddef.h>
34 #include <linux/vmalloc.h>
35 #include <linux/init.h>
36 #include <linux/bootmem.h>
37 #include <linux/memblock.h>
38 #include <linux/slab.h>
40 #include <asm/pgalloc.h>
44 #include <asm/mmu_context.h>
45 #include <asm/pgtable.h>
48 #include <asm/machdep.h>
50 #include <asm/processor.h>
51 #include <asm/cputable.h>
52 #include <asm/sections.h>
53 #include <asm/system.h>
54 #include <asm/abs_addr.h>
55 #include <asm/firmware.h>
59 unsigned long ioremap_bot
= IOREMAP_BASE
;
62 #ifdef CONFIG_PPC_MMU_NOHASH
63 static void *early_alloc_pgtable(unsigned long size
)
67 if (init_bootmem_done
)
68 pt
= __alloc_bootmem(size
, size
, __pa(MAX_DMA_ADDRESS
));
70 pt
= __va(memblock_alloc_base(size
, size
,
71 __pa(MAX_DMA_ADDRESS
)));
76 #endif /* CONFIG_PPC_MMU_NOHASH */
79 * map_kernel_page currently only called by __ioremap
80 * map_kernel_page adds an entry to the ioremap page table
81 * and adds an entry to the HPT, possibly bolting it
83 int map_kernel_page(unsigned long ea
, unsigned long pa
, int flags
)
90 if (slab_is_available()) {
91 pgdp
= pgd_offset_k(ea
);
92 pudp
= pud_alloc(&init_mm
, pgdp
, ea
);
95 pmdp
= pmd_alloc(&init_mm
, pudp
, ea
);
98 ptep
= pte_alloc_kernel(pmdp
, ea
);
101 set_pte_at(&init_mm
, ea
, ptep
, pfn_pte(pa
>> PAGE_SHIFT
,
104 #ifdef CONFIG_PPC_MMU_NOHASH
105 /* Warning ! This will blow up if bootmem is not initialized
106 * which our ppc64 code is keen to do that, we'll need to
107 * fix it and/or be more careful
109 pgdp
= pgd_offset_k(ea
);
110 #ifdef PUD_TABLE_SIZE
111 if (pgd_none(*pgdp
)) {
112 pudp
= early_alloc_pgtable(PUD_TABLE_SIZE
);
113 BUG_ON(pudp
== NULL
);
114 pgd_populate(&init_mm
, pgdp
, pudp
);
116 #endif /* PUD_TABLE_SIZE */
117 pudp
= pud_offset(pgdp
, ea
);
118 if (pud_none(*pudp
)) {
119 pmdp
= early_alloc_pgtable(PMD_TABLE_SIZE
);
120 BUG_ON(pmdp
== NULL
);
121 pud_populate(&init_mm
, pudp
, pmdp
);
123 pmdp
= pmd_offset(pudp
, ea
);
124 if (!pmd_present(*pmdp
)) {
125 ptep
= early_alloc_pgtable(PAGE_SIZE
);
126 BUG_ON(ptep
== NULL
);
127 pmd_populate_kernel(&init_mm
, pmdp
, ptep
);
129 ptep
= pte_offset_kernel(pmdp
, ea
);
130 set_pte_at(&init_mm
, ea
, ptep
, pfn_pte(pa
>> PAGE_SHIFT
,
132 #else /* CONFIG_PPC_MMU_NOHASH */
134 * If the mm subsystem is not fully up, we cannot create a
135 * linux page table entry for this mapping. Simply bolt an
136 * entry in the hardware page table.
139 if (htab_bolt_mapping(ea
, ea
+ PAGE_SIZE
, pa
, flags
,
140 mmu_io_psize
, mmu_kernel_ssize
)) {
141 printk(KERN_ERR
"Failed to do bolted mapping IO "
142 "memory at %016lx !\n", pa
);
145 #endif /* !CONFIG_PPC_MMU_NOHASH */
152 * __ioremap_at - Low level function to establish the page tables
155 void __iomem
* __ioremap_at(phys_addr_t pa
, void *ea
, unsigned long size
,
160 /* Make sure we have the base flags */
161 if ((flags
& _PAGE_PRESENT
) == 0)
162 flags
|= pgprot_val(PAGE_KERNEL
);
164 /* Non-cacheable page cannot be coherent */
165 if (flags
& _PAGE_NO_CACHE
)
166 flags
&= ~_PAGE_COHERENT
;
168 /* We don't support the 4K PFN hack with ioremap */
169 if (flags
& _PAGE_4K_PFN
)
172 WARN_ON(pa
& ~PAGE_MASK
);
173 WARN_ON(((unsigned long)ea
) & ~PAGE_MASK
);
174 WARN_ON(size
& ~PAGE_MASK
);
176 for (i
= 0; i
< size
; i
+= PAGE_SIZE
)
177 if (map_kernel_page((unsigned long)ea
+i
, pa
+i
, flags
))
180 return (void __iomem
*)ea
;
184 * __iounmap_from - Low level function to tear down the page tables
185 * for an IO mapping. This is used for mappings that
186 * are manipulated manually, like partial unmapping of
187 * PCI IOs or ISA space.
189 void __iounmap_at(void *ea
, unsigned long size
)
191 WARN_ON(((unsigned long)ea
) & ~PAGE_MASK
);
192 WARN_ON(size
& ~PAGE_MASK
);
194 unmap_kernel_range((unsigned long)ea
, size
);
197 void __iomem
* __ioremap_caller(phys_addr_t addr
, unsigned long size
,
198 unsigned long flags
, void *caller
)
200 phys_addr_t paligned
;
204 * Choose an address to map it to.
205 * Once the imalloc system is running, we use it.
206 * Before that, we map using addresses going
207 * up from ioremap_bot. imalloc will use
208 * the addresses from ioremap_bot through
212 paligned
= addr
& PAGE_MASK
;
213 size
= PAGE_ALIGN(addr
+ size
) - paligned
;
215 if ((size
== 0) || (paligned
== 0))
219 struct vm_struct
*area
;
221 area
= __get_vm_area_caller(size
, VM_IOREMAP
,
222 ioremap_bot
, IOREMAP_END
,
227 area
->phys_addr
= paligned
;
228 ret
= __ioremap_at(paligned
, area
->addr
, size
, flags
);
232 ret
= __ioremap_at(paligned
, (void *)ioremap_bot
, size
, flags
);
238 ret
+= addr
& ~PAGE_MASK
;
242 void __iomem
* __ioremap(phys_addr_t addr
, unsigned long size
,
245 return __ioremap_caller(addr
, size
, flags
, __builtin_return_address(0));
248 void __iomem
* ioremap(phys_addr_t addr
, unsigned long size
)
250 unsigned long flags
= _PAGE_NO_CACHE
| _PAGE_GUARDED
;
251 void *caller
= __builtin_return_address(0);
254 return ppc_md
.ioremap(addr
, size
, flags
, caller
);
255 return __ioremap_caller(addr
, size
, flags
, caller
);
258 void __iomem
* ioremap_wc(phys_addr_t addr
, unsigned long size
)
260 unsigned long flags
= _PAGE_NO_CACHE
;
261 void *caller
= __builtin_return_address(0);
264 return ppc_md
.ioremap(addr
, size
, flags
, caller
);
265 return __ioremap_caller(addr
, size
, flags
, caller
);
268 void __iomem
* ioremap_prot(phys_addr_t addr
, unsigned long size
,
271 void *caller
= __builtin_return_address(0);
273 /* writeable implies dirty for kernel addresses */
274 if (flags
& _PAGE_RW
)
275 flags
|= _PAGE_DIRTY
;
277 /* we don't want to let _PAGE_USER and _PAGE_EXEC leak out */
278 flags
&= ~(_PAGE_USER
| _PAGE_EXEC
);
281 /* _PAGE_USER contains _PAGE_BAP_SR on BookE using the new PTE format
282 * which means that we just cleared supervisor access... oops ;-) This
285 flags
|= _PAGE_BAP_SR
;
289 return ppc_md
.ioremap(addr
, size
, flags
, caller
);
290 return __ioremap_caller(addr
, size
, flags
, caller
);
295 * Unmap an IO region and remove it from imalloc'd list.
296 * Access to IO memory should be serialized by driver.
298 void __iounmap(volatile void __iomem
*token
)
305 addr
= (void *) ((unsigned long __force
)
306 PCI_FIX_ADDR(token
) & PAGE_MASK
);
307 if ((unsigned long)addr
< ioremap_bot
) {
308 printk(KERN_WARNING
"Attempt to iounmap early bolted mapping"
315 void iounmap(volatile void __iomem
*token
)
318 ppc_md
.iounmap(token
);
323 EXPORT_SYMBOL(ioremap
);
324 EXPORT_SYMBOL(ioremap_wc
);
325 EXPORT_SYMBOL(ioremap_prot
);
326 EXPORT_SYMBOL(__ioremap
);
327 EXPORT_SYMBOL(__ioremap_at
);
328 EXPORT_SYMBOL(iounmap
);
329 EXPORT_SYMBOL(__iounmap
);
330 EXPORT_SYMBOL(__iounmap_at
);