| blob | 7090c126797cb27e3db102e830d5de72c14c3310 |
1 /*
2 * Low-level CPU initialisation
3 * Based on arch/arm/kernel/head.S
4 *
5 * Copyright (C) 1994-2002 Russell King
6 * Copyright (C) 2003-2012 ARM Ltd.
7 * Authors: Catalin Marinas <catalin.marinas@arm.com>
8 * Will Deacon <will.deacon@arm.com>
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License version 2 as
12 * published by the Free Software Foundation.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this program. If not, see <http://www.gnu.org/licenses/>.
21 */
23 #include <linux/linkage.h>
24 #include <linux/init.h>
26 #include <asm/assembler.h>
27 #include <asm/ptrace.h>
28 #include <asm/asm-offsets.h>
29 #include <asm/cputype.h>
30 #include <asm/memory.h>
31 #include <asm/thread_info.h>
32 #include <asm/pgtable-hwdef.h>
33 #include <asm/pgtable.h>
34 #include <asm/page.h>
35 #include <asm/virt.h>
37 /*
38 * swapper_pg_dir is the virtual address of the initial page table. We place
39 * the page tables 3 * PAGE_SIZE below KERNEL_RAM_VADDR. The idmap_pg_dir has
40 * 2 pages and is placed below swapper_pg_dir.
41 */
42 #define KERNEL_RAM_VADDR (PAGE_OFFSET + TEXT_OFFSET)
44 #if (KERNEL_RAM_VADDR & 0xfffff) != 0x80000
45 #error KERNEL_RAM_VADDR must start at 0xXXX80000
46 #endif
48 #define SWAPPER_DIR_SIZE (3 * PAGE_SIZE)
49 #define IDMAP_DIR_SIZE (2 * PAGE_SIZE)
51 .globl swapper_pg_dir
52 .equ swapper_pg_dir, KERNEL_RAM_VADDR - SWAPPER_DIR_SIZE
54 .globl idmap_pg_dir
55 .equ idmap_pg_dir, swapper_pg_dir - IDMAP_DIR_SIZE
57 .macro pgtbl, ttb0, ttb1, phys
58 add \ttb1, \phys, #TEXT_OFFSET - SWAPPER_DIR_SIZE
59 sub \ttb0, \ttb1, #IDMAP_DIR_SIZE
60 .endm
62 #ifdef CONFIG_ARM64_64K_PAGES
63 #define BLOCK_SHIFT PAGE_SHIFT
64 #define BLOCK_SIZE PAGE_SIZE
65 #else
66 #define BLOCK_SHIFT SECTION_SHIFT
67 #define BLOCK_SIZE SECTION_SIZE
68 #endif
70 #define KERNEL_START KERNEL_RAM_VADDR
71 #define KERNEL_END _end
73 /*
74 * Initial memory map attributes.
75 */
76 #ifndef CONFIG_SMP
77 #define PTE_FLAGS PTE_TYPE_PAGE | PTE_AF
78 #define PMD_FLAGS PMD_TYPE_SECT | PMD_SECT_AF
79 #else
80 #define PTE_FLAGS PTE_TYPE_PAGE | PTE_AF | PTE_SHARED
81 #define PMD_FLAGS PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S
82 #endif
84 #ifdef CONFIG_ARM64_64K_PAGES
85 #define MM_MMUFLAGS PTE_ATTRINDX(MT_NORMAL) | PTE_FLAGS
86 #else
87 #define MM_MMUFLAGS PMD_ATTRINDX(MT_NORMAL) | PMD_FLAGS
88 #endif
90 /*
91 * Kernel startup entry point.
92 * ---------------------------
93 *
94 * The requirements are:
95 * MMU = off, D-cache = off, I-cache = on or off,
96 * x0 = physical address to the FDT blob.
97 *
98 * This code is mostly position independent so you call this at
99 * __pa(PAGE_OFFSET + TEXT_OFFSET).
100 *
101 * Note that the callee-saved registers are used for storing variables
102 * that are useful before the MMU is enabled. The allocations are described
103 * in the entry routines.
104 */
105 __HEAD
107 /*
108 * DO NOT MODIFY. Image header expected by Linux boot-loaders.
109 */
110 b stext // branch to kernel start, magic
111 .long 0 // reserved
112 .quad TEXT_OFFSET // Image load offset from start of RAM
113 .quad 0 // reserved
114 .quad 0 // reserved
115 .quad 0 // reserved
116 .quad 0 // reserved
117 .quad 0 // reserved
118 .byte 0x41 // Magic number, "ARM\x64"
119 .byte 0x52
120 .byte 0x4d
121 .byte 0x64
122 .word 0 // reserved
124 ENTRY(stext)
125 mov x21, x0 // x21=FDT
126 bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET
127 bl el2_setup // Drop to EL1
128 mrs x22, midr_el1 // x22=cpuid
129 mov x0, x22
130 bl lookup_processor_type
131 mov x23, x0 // x23=current cpu_table
132 cbz x23, __error_p // invalid processor (x23=0)?
133 bl __vet_fdt
134 bl __create_page_tables // x25=TTBR0, x26=TTBR1
135 /*
136 * The following calls CPU specific code in a position independent
137 * manner. See arch/arm64/mm/proc.S for details. x23 = base of
138 * cpu_info structure selected by lookup_processor_type above.
139 * On return, the CPU will be ready for the MMU to be turned on and
140 * the TCR will have been set.
141 */
142 ldr x27, __switch_data // address to jump to after
143 // MMU has been enabled
144 adr lr, __enable_mmu // return (PIC) address
145 ldr x12, [x23, #CPU_INFO_SETUP]
146 add x12, x12, x28 // __virt_to_phys
147 br x12 // initialise processor
148 ENDPROC(stext)
150 /*
151 * If we're fortunate enough to boot at EL2, ensure that the world is
152 * sane before dropping to EL1.
153 */
154 ENTRY(el2_setup)
155 mrs x0, CurrentEL
156 cmp x0, #PSR_MODE_EL2t
157 ccmp x0, #PSR_MODE_EL2h, #0x4, ne
158 ldr x0, =__boot_cpu_mode // Compute __boot_cpu_mode
159 add x0, x0, x28
160 b.eq 1f
161 str wzr, [x0] // Remember we don't have EL2...
162 ret
164 /* Hyp configuration. */
165 1: ldr w1, =BOOT_CPU_MODE_EL2
166 str w1, [x0, #4] // This CPU has EL2
167 mov x0, #(1 << 31) // 64-bit EL1
168 msr hcr_el2, x0
170 /* Generic timers. */
171 mrs x0, cnthctl_el2
172 orr x0, x0, #3 // Enable EL1 physical timers
173 msr cnthctl_el2, x0
174 msr cntvoff_el2, xzr // Clear virtual offset
176 /* Populate ID registers. */
177 mrs x0, midr_el1
178 mrs x1, mpidr_el1
179 msr vpidr_el2, x0
180 msr vmpidr_el2, x1
182 /* sctlr_el1 */
183 mov x0, #0x0800 // Set/clear RES{1,0} bits
184 movk x0, #0x30d0, lsl #16
185 msr sctlr_el1, x0
187 /* Coprocessor traps. */
188 mov x0, #0x33ff
189 msr cptr_el2, x0 // Disable copro. traps to EL2
191 #ifdef CONFIG_COMPAT
192 msr hstr_el2, xzr // Disable CP15 traps to EL2
193 #endif
195 /* Stage-2 translation */
196 msr vttbr_el2, xzr
198 /* Hypervisor stub */
199 adr x0, __hyp_stub_vectors
200 msr vbar_el2, x0
202 /* spsr */
203 mov x0, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\
204 PSR_MODE_EL1h)
205 msr spsr_el2, x0
206 msr elr_el2, lr
207 eret
208 ENDPROC(el2_setup)
210 /*
211 * We need to find out the CPU boot mode long after boot, so we need to
212 * store it in a writable variable.
213 *
214 * This is not in .bss, because we set it sufficiently early that the boot-time
215 * zeroing of .bss would clobber it.
216 */
217 .pushsection .data
218 ENTRY(__boot_cpu_mode)
219 .long BOOT_CPU_MODE_EL2
220 .long 0
221 .popsection
223 .align 3
224 2: .quad .
225 .quad PAGE_OFFSET
227 #ifdef CONFIG_SMP
228 .pushsection .smp.pen.text, "ax"
229 .align 3
230 1: .quad .
231 .quad secondary_holding_pen_release
233 /*
234 * This provides a "holding pen" for platforms to hold all secondary
235 * cores are held until we're ready for them to initialise.
236 */
237 ENTRY(secondary_holding_pen)
238 bl __calc_phys_offset // x24=phys offset
239 bl el2_setup // Drop to EL1
240 mrs x0, mpidr_el1
241 ldr x1, =MPIDR_HWID_BITMASK
242 and x0, x0, x1
243 adr x1, 1b
244 ldp x2, x3, [x1]
245 sub x1, x1, x2
246 add x3, x3, x1
247 pen: ldr x4, [x3]
248 cmp x4, x0
249 b.eq secondary_startup
250 wfe
251 b pen
252 ENDPROC(secondary_holding_pen)
253 .popsection
255 ENTRY(secondary_startup)
256 /*
257 * Common entry point for secondary CPUs.
258 */
259 mrs x22, midr_el1 // x22=cpuid
260 mov x0, x22
261 bl lookup_processor_type
262 mov x23, x0 // x23=current cpu_table
263 cbz x23, __error_p // invalid processor (x23=0)?
265 pgtbl x25, x26, x24 // x25=TTBR0, x26=TTBR1
266 ldr x12, [x23, #CPU_INFO_SETUP]
267 add x12, x12, x28 // __virt_to_phys
268 blr x12 // initialise processor
270 ldr x21, =secondary_data
271 ldr x27, =__secondary_switched // address to jump to after enabling the MMU
272 b __enable_mmu
273 ENDPROC(secondary_startup)
275 ENTRY(__secondary_switched)
276 ldr x0, [x21] // get secondary_data.stack
277 mov sp, x0
278 mov x29, #0
279 b secondary_start_kernel
280 ENDPROC(__secondary_switched)
281 #endif /* CONFIG_SMP */
283 /*
284 * Setup common bits before finally enabling the MMU. Essentially this is just
285 * loading the page table pointer and vector base registers.
286 *
287 * On entry to this code, x0 must contain the SCTLR_EL1 value for turning on
288 * the MMU.
289 */
290 __enable_mmu:
291 ldr x5, =vectors
292 msr vbar_el1, x5
293 msr ttbr0_el1, x25 // load TTBR0
294 msr ttbr1_el1, x26 // load TTBR1
295 isb
296 b __turn_mmu_on
297 ENDPROC(__enable_mmu)
299 /*
300 * Enable the MMU. This completely changes the structure of the visible memory
301 * space. You will not be able to trace execution through this.
302 *
303 * x0 = system control register
304 * x27 = *virtual* address to jump to upon completion
305 *
306 * other registers depend on the function called upon completion
307 */
308 .align 6
309 __turn_mmu_on:
310 msr sctlr_el1, x0
311 isb
312 br x27
313 ENDPROC(__turn_mmu_on)
315 /*
316 * Calculate the start of physical memory.
317 */
318 __calc_phys_offset:
319 adr x0, 1f
320 ldp x1, x2, [x0]
321 sub x28, x0, x1 // x28 = PHYS_OFFSET - PAGE_OFFSET
322 add x24, x2, x28 // x24 = PHYS_OFFSET
323 ret
324 ENDPROC(__calc_phys_offset)
326 .align 3
327 1: .quad .
328 .quad PAGE_OFFSET
330 /*
331 * Macro to populate the PGD for the corresponding block entry in the next
332 * level (tbl) for the given virtual address.
333 *
334 * Preserves: pgd, tbl, virt
335 * Corrupts: tmp1, tmp2
336 */
337 .macro create_pgd_entry, pgd, tbl, virt, tmp1, tmp2
338 lsr \tmp1, \virt, #PGDIR_SHIFT
339 and \tmp1, \tmp1, #PTRS_PER_PGD - 1 // PGD index
340 orr \tmp2, \tbl, #3 // PGD entry table type
341 str \tmp2, [\pgd, \tmp1, lsl #3]
342 .endm
344 /*
345 * Macro to populate block entries in the page table for the start..end
346 * virtual range (inclusive).
347 *
348 * Preserves: tbl, flags
349 * Corrupts: phys, start, end, pstate
350 */
351 .macro create_block_map, tbl, flags, phys, start, end, idmap=0
352 lsr \phys, \phys, #BLOCK_SHIFT
353 .if \idmap
354 and \start, \phys, #PTRS_PER_PTE - 1 // table index
355 .else
356 lsr \start, \start, #BLOCK_SHIFT
357 and \start, \start, #PTRS_PER_PTE - 1 // table index
358 .endif
359 orr \phys, \flags, \phys, lsl #BLOCK_SHIFT // table entry
360 .ifnc \start,\end
361 lsr \end, \end, #BLOCK_SHIFT
362 and \end, \end, #PTRS_PER_PTE - 1 // table end index
363 .endif
364 9999: str \phys, [\tbl, \start, lsl #3] // store the entry
365 .ifnc \start,\end
366 add \start, \start, #1 // next entry
367 add \phys, \phys, #BLOCK_SIZE // next block
368 cmp \start, \end
369 b.ls 9999b
370 .endif
371 .endm
373 /*
374 * Setup the initial page tables. We only setup the barest amount which is
375 * required to get the kernel running. The following sections are required:
376 * - identity mapping to enable the MMU (low address, TTBR0)
377 * - first few MB of the kernel linear mapping to jump to once the MMU has
378 * been enabled, including the FDT blob (TTBR1)
379 * - UART mapping if CONFIG_EARLY_PRINTK is enabled (TTBR1)
380 */
381 __create_page_tables:
382 pgtbl x25, x26, x24 // idmap_pg_dir and swapper_pg_dir addresses
384 /*
385 * Clear the idmap and swapper page tables.
386 */
387 mov x0, x25
388 add x6, x26, #SWAPPER_DIR_SIZE
389 1: stp xzr, xzr, [x0], #16
390 stp xzr, xzr, [x0], #16
391 stp xzr, xzr, [x0], #16
392 stp xzr, xzr, [x0], #16
393 cmp x0, x6
394 b.lo 1b
396 ldr x7, =MM_MMUFLAGS
398 /*
399 * Create the identity mapping.
400 */
401 add x0, x25, #PAGE_SIZE // section table address
402 adr x3, __turn_mmu_on // virtual/physical address
403 create_pgd_entry x25, x0, x3, x5, x6
404 create_block_map x0, x7, x3, x5, x5, idmap=1
406 /*
407 * Map the kernel image (starting with PHYS_OFFSET).
408 */
409 add x0, x26, #PAGE_SIZE // section table address
410 mov x5, #PAGE_OFFSET
411 create_pgd_entry x26, x0, x5, x3, x6
412 ldr x6, =KERNEL_END - 1
413 mov x3, x24 // phys offset
414 create_block_map x0, x7, x3, x5, x6
416 /*
417 * Map the FDT blob (maximum 2MB; must be within 512MB of
418 * PHYS_OFFSET).
419 */
420 mov x3, x21 // FDT phys address
421 and x3, x3, #~((1 << 21) - 1) // 2MB aligned
422 mov x6, #PAGE_OFFSET
423 sub x5, x3, x24 // subtract PHYS_OFFSET
424 tst x5, #~((1 << 29) - 1) // within 512MB?
425 csel x21, xzr, x21, ne // zero the FDT pointer
426 b.ne 1f
427 add x5, x5, x6 // __va(FDT blob)
428 add x6, x5, #1 << 21 // 2MB for the FDT blob
429 sub x6, x6, #1 // inclusive range
430 create_block_map x0, x7, x3, x5, x6
431 1:
432 #ifdef CONFIG_EARLY_PRINTK
433 /*
434 * Create the pgd entry for the UART mapping. The full mapping is done
435 * later based earlyprintk kernel parameter.
436 */
437 ldr x5, =EARLYCON_IOBASE // UART virtual address
438 add x0, x26, #2 * PAGE_SIZE // section table address
439 create_pgd_entry x26, x0, x5, x6, x7
440 #endif
441 ret
442 ENDPROC(__create_page_tables)
443 .ltorg
445 .align 3
446 .type __switch_data, %object
447 __switch_data:
448 .quad __mmap_switched
449 .quad __data_loc // x4
450 .quad _data // x5
451 .quad __bss_start // x6
452 .quad _end // x7
453 .quad processor_id // x4
454 .quad __fdt_pointer // x5
455 .quad memstart_addr // x6
456 .quad init_thread_union + THREAD_START_SP // sp
458 /*
459 * The following fragment of code is executed with the MMU on in MMU mode, and
460 * uses absolute addresses; this is not position independent.
461 */
462 __mmap_switched:
463 adr x3, __switch_data + 8
465 ldp x4, x5, [x3], #16
466 ldp x6, x7, [x3], #16
467 cmp x4, x5 // Copy data segment if needed
468 1: ccmp x5, x6, #4, ne
469 b.eq 2f
470 ldr x16, [x4], #8
471 str x16, [x5], #8
472 b 1b
473 2:
474 1: cmp x6, x7
475 b.hs 2f
476 str xzr, [x6], #8 // Clear BSS
477 b 1b
478 2:
479 ldp x4, x5, [x3], #16
480 ldr x6, [x3], #8
481 ldr x16, [x3]
482 mov sp, x16
483 str x22, [x4] // Save processor ID
484 str x21, [x5] // Save FDT pointer
485 str x24, [x6] // Save PHYS_OFFSET
486 mov x29, #0
487 b start_kernel
488 ENDPROC(__mmap_switched)
490 /*
491 * Exception handling. Something went wrong and we can't proceed. We ought to
492 * tell the user, but since we don't have any guarantee that we're even
493 * running on the right architecture, we do virtually nothing.
494 */
495 __error_p:
496 ENDPROC(__error_p)
498 __error:
499 1: nop
500 b 1b
501 ENDPROC(__error)
503 /*
504 * This function gets the processor ID in w0 and searches the cpu_table[] for
505 * a match. It returns a pointer to the struct cpu_info it found. The
506 * cpu_table[] must end with an empty (all zeros) structure.
507 *
508 * This routine can be called via C code and it needs to work with the MMU
509 * both disabled and enabled (the offset is calculated automatically).
510 */
511 ENTRY(lookup_processor_type)
512 adr x1, __lookup_processor_type_data
513 ldp x2, x3, [x1]
514 sub x1, x1, x2 // get offset between VA and PA
515 add x3, x3, x1 // convert VA to PA
516 1:
517 ldp w5, w6, [x3] // load cpu_id_val and cpu_id_mask
518 cbz w5, 2f // end of list?
519 and w6, w6, w0
520 cmp w5, w6
521 b.eq 3f
522 add x3, x3, #CPU_INFO_SZ
523 b 1b
524 2:
525 mov x3, #0 // unknown processor
526 3:
527 mov x0, x3
528 ret
529 ENDPROC(lookup_processor_type)
531 .align 3
532 .type __lookup_processor_type_data, %object
533 __lookup_processor_type_data:
534 .quad .
535 .quad cpu_table
536 .size __lookup_processor_type_data, . - __lookup_processor_type_data
538 /*
539 * Determine validity of the x21 FDT pointer.
540 * The dtb must be 8-byte aligned and live in the first 512M of memory.
541 */
542 __vet_fdt:
543 tst x21, #0x7
544 b.ne 1f
545 cmp x21, x24
546 b.lt 1f
547 mov x0, #(1 << 29)
548 add x0, x0, x24
549 cmp x21, x0
550 b.ge 1f
551 ret
552 1:
553 mov x21, #0
554 ret
555 ENDPROC(__vet_fdt)