2 * linux/arch/arm/mm/fault.c
4 * Copyright (C) 1995 Linus Torvalds
5 * Modifications for ARM processor (c) 1995-2004 Russell King
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
11 #include <linux/module.h>
12 #include <linux/signal.h>
14 #include <linux/hardirq.h>
15 #include <linux/init.h>
16 #include <linux/kprobes.h>
17 #include <linux/uaccess.h>
18 #include <linux/page-flags.h>
19 #include <linux/sched.h>
20 #include <linux/highmem.h>
21 #include <linux/perf_event.h>
23 #include <asm/exception.h>
24 #include <asm/pgtable.h>
25 #include <asm/system_misc.h>
26 #include <asm/system_info.h>
27 #include <asm/tlbflush.h>
34 static inline int notify_page_fault(struct pt_regs
*regs
, unsigned int fsr
)
38 if (!user_mode(regs
)) {
39 /* kprobe_running() needs smp_processor_id() */
41 if (kprobe_running() && kprobe_fault_handler(regs
, fsr
))
49 static inline int notify_page_fault(struct pt_regs
*regs
, unsigned int fsr
)
56 * This is useful to dump out the page tables associated with
59 void show_pte(struct mm_struct
*mm
, unsigned long addr
)
66 pr_alert("pgd = %p\n", mm
->pgd
);
67 pgd
= pgd_offset(mm
, addr
);
68 pr_alert("[%08lx] *pgd=%08llx",
69 addr
, (long long)pgd_val(*pgd
));
84 pud
= pud_offset(pgd
, addr
);
85 if (PTRS_PER_PUD
!= 1)
86 pr_cont(", *pud=%08llx", (long long)pud_val(*pud
));
96 pmd
= pmd_offset(pud
, addr
);
97 if (PTRS_PER_PMD
!= 1)
98 pr_cont(", *pmd=%08llx", (long long)pmd_val(*pmd
));
108 /* We must not map this if we have highmem enabled */
109 if (PageHighMem(pfn_to_page(pmd_val(*pmd
) >> PAGE_SHIFT
)))
112 pte
= pte_offset_map(pmd
, addr
);
113 pr_cont(", *pte=%08llx", (long long)pte_val(*pte
));
114 #ifndef CONFIG_ARM_LPAE
115 pr_cont(", *ppte=%08llx",
116 (long long)pte_val(pte
[PTE_HWTABLE_PTRS
]));
123 #else /* CONFIG_MMU */
124 void show_pte(struct mm_struct
*mm
, unsigned long addr
)
126 #endif /* CONFIG_MMU */
129 * Oops. The kernel tried to access some page that wasn't present.
132 __do_kernel_fault(struct mm_struct
*mm
, unsigned long addr
, unsigned int fsr
,
133 struct pt_regs
*regs
)
136 * Are we prepared to handle this kernel fault?
138 if (fixup_exception(regs
))
142 * No handler, we'll have to terminate things with extreme prejudice.
145 pr_alert("Unable to handle kernel %s at virtual address %08lx\n",
146 (addr
< PAGE_SIZE
) ? "NULL pointer dereference" :
147 "paging request", addr
);
150 die("Oops", regs
, fsr
);
156 * Something tried to access memory that isn't in our memory map..
157 * User mode accesses just cause a SIGSEGV
160 __do_user_fault(struct task_struct
*tsk
, unsigned long addr
,
161 unsigned int fsr
, unsigned int sig
, int code
,
162 struct pt_regs
*regs
)
166 #ifdef CONFIG_DEBUG_USER
167 if (((user_debug
& UDBG_SEGV
) && (sig
== SIGSEGV
)) ||
168 ((user_debug
& UDBG_BUS
) && (sig
== SIGBUS
))) {
169 printk(KERN_DEBUG
"%s: unhandled page fault (%d) at 0x%08lx, code 0x%03x\n",
170 tsk
->comm
, sig
, addr
, fsr
);
171 show_pte(tsk
->mm
, addr
);
176 tsk
->thread
.address
= addr
;
177 tsk
->thread
.error_code
= fsr
;
178 tsk
->thread
.trap_no
= 14;
182 si
.si_addr
= (void __user
*)addr
;
183 force_sig_info(sig
, &si
, tsk
);
186 void do_bad_area(unsigned long addr
, unsigned int fsr
, struct pt_regs
*regs
)
188 struct task_struct
*tsk
= current
;
189 struct mm_struct
*mm
= tsk
->active_mm
;
192 * If we are in kernel mode at this point, we
193 * have no context to handle this fault with.
196 __do_user_fault(tsk
, addr
, fsr
, SIGSEGV
, SEGV_MAPERR
, regs
);
198 __do_kernel_fault(mm
, addr
, fsr
, regs
);
202 #define VM_FAULT_BADMAP 0x010000
203 #define VM_FAULT_BADACCESS 0x020000
206 * Check that the permissions on the VMA allow for the fault which occurred.
207 * If we encountered a write fault, we must have write permission, otherwise
208 * we allow any permission.
210 static inline bool access_error(unsigned int fsr
, struct vm_area_struct
*vma
)
212 unsigned int mask
= VM_READ
| VM_WRITE
| VM_EXEC
;
216 if (fsr
& FSR_LNX_PF
)
219 return vma
->vm_flags
& mask
? false : true;
223 __do_page_fault(struct mm_struct
*mm
, unsigned long addr
, unsigned int fsr
,
224 unsigned int flags
, struct task_struct
*tsk
)
226 struct vm_area_struct
*vma
;
229 vma
= find_vma(mm
, addr
);
230 fault
= VM_FAULT_BADMAP
;
233 if (unlikely(vma
->vm_start
> addr
))
237 * Ok, we have a good vm_area for this
238 * memory access, so we can handle it.
241 if (access_error(fsr
, vma
)) {
242 fault
= VM_FAULT_BADACCESS
;
246 return handle_mm_fault(mm
, vma
, addr
& PAGE_MASK
, flags
);
249 /* Don't allow expansion below FIRST_USER_ADDRESS */
250 if (vma
->vm_flags
& VM_GROWSDOWN
&&
251 addr
>= FIRST_USER_ADDRESS
&& !expand_stack(vma
, addr
))
258 do_page_fault(unsigned long addr
, unsigned int fsr
, struct pt_regs
*regs
)
260 struct task_struct
*tsk
;
261 struct mm_struct
*mm
;
262 int fault
, sig
, code
;
263 unsigned int flags
= FAULT_FLAG_ALLOW_RETRY
| FAULT_FLAG_KILLABLE
;
265 if (notify_page_fault(regs
, fsr
))
271 /* Enable interrupts if they were enabled in the parent context. */
272 if (interrupts_enabled(regs
))
276 * If we're in an interrupt or have no user
277 * context, we must not take the fault..
279 if (faulthandler_disabled() || !mm
)
283 flags
|= FAULT_FLAG_USER
;
285 flags
|= FAULT_FLAG_WRITE
;
288 * As per x86, we may deadlock here. However, since the kernel only
289 * validly references user space from well defined areas of the code,
290 * we can bug out early if this is from code which shouldn't.
292 if (!down_read_trylock(&mm
->mmap_sem
)) {
293 if (!user_mode(regs
) && !search_exception_tables(regs
->ARM_pc
))
296 down_read(&mm
->mmap_sem
);
299 * The above down_read_trylock() might have succeeded in
300 * which case, we'll have missed the might_sleep() from
304 #ifdef CONFIG_DEBUG_VM
305 if (!user_mode(regs
) &&
306 !search_exception_tables(regs
->ARM_pc
))
311 fault
= __do_page_fault(mm
, addr
, fsr
, flags
, tsk
);
313 /* If we need to retry but a fatal signal is pending, handle the
314 * signal first. We do not need to release the mmap_sem because
315 * it would already be released in __lock_page_or_retry in
317 if ((fault
& VM_FAULT_RETRY
) && fatal_signal_pending(current
)) {
318 if (!user_mode(regs
))
324 * Major/minor page fault accounting is only done on the
325 * initial attempt. If we go through a retry, it is extremely
326 * likely that the page will be found in page cache at that point.
329 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS
, 1, regs
, addr
);
330 if (!(fault
& VM_FAULT_ERROR
) && flags
& FAULT_FLAG_ALLOW_RETRY
) {
331 if (fault
& VM_FAULT_MAJOR
) {
333 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ
, 1,
337 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN
, 1,
340 if (fault
& VM_FAULT_RETRY
) {
341 /* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk
343 flags
&= ~FAULT_FLAG_ALLOW_RETRY
;
344 flags
|= FAULT_FLAG_TRIED
;
349 up_read(&mm
->mmap_sem
);
352 * Handle the "normal" case first - VM_FAULT_MAJOR / VM_FAULT_MINOR
354 if (likely(!(fault
& (VM_FAULT_ERROR
| VM_FAULT_BADMAP
| VM_FAULT_BADACCESS
))))
358 * If we are in kernel mode at this point, we
359 * have no context to handle this fault with.
361 if (!user_mode(regs
))
364 if (fault
& VM_FAULT_OOM
) {
366 * We ran out of memory, call the OOM killer, and return to
367 * userspace (which will retry the fault, or kill us if we
370 pagefault_out_of_memory();
374 if (fault
& VM_FAULT_SIGBUS
) {
376 * We had some memory, but were unable to
377 * successfully fix up this page fault.
383 * Something tried to access memory that
384 * isn't in our memory map..
387 code
= fault
== VM_FAULT_BADACCESS
?
388 SEGV_ACCERR
: SEGV_MAPERR
;
391 __do_user_fault(tsk
, addr
, fsr
, sig
, code
, regs
);
395 __do_kernel_fault(mm
, addr
, fsr
, regs
);
398 #else /* CONFIG_MMU */
400 do_page_fault(unsigned long addr
, unsigned int fsr
, struct pt_regs
*regs
)
404 #endif /* CONFIG_MMU */
407 * First Level Translation Fault Handler
409 * We enter here because the first level page table doesn't contain
410 * a valid entry for the address.
412 * If the address is in kernel space (>= TASK_SIZE), then we are
413 * probably faulting in the vmalloc() area.
415 * If the init_task's first level page tables contains the relevant
416 * entry, we copy the it to this task. If not, we send the process
417 * a signal, fixup the exception, or oops the kernel.
419 * NOTE! We MUST NOT take any locks for this case. We may be in an
420 * interrupt or a critical region, and should only copy the information
421 * from the master page table, nothing more.
425 do_translation_fault(unsigned long addr
, unsigned int fsr
,
426 struct pt_regs
*regs
)
433 if (addr
< TASK_SIZE
)
434 return do_page_fault(addr
, fsr
, regs
);
439 index
= pgd_index(addr
);
441 pgd
= cpu_get_pgd() + index
;
442 pgd_k
= init_mm
.pgd
+ index
;
444 if (pgd_none(*pgd_k
))
446 if (!pgd_present(*pgd
))
447 set_pgd(pgd
, *pgd_k
);
449 pud
= pud_offset(pgd
, addr
);
450 pud_k
= pud_offset(pgd_k
, addr
);
452 if (pud_none(*pud_k
))
454 if (!pud_present(*pud
))
455 set_pud(pud
, *pud_k
);
457 pmd
= pmd_offset(pud
, addr
);
458 pmd_k
= pmd_offset(pud_k
, addr
);
460 #ifdef CONFIG_ARM_LPAE
462 * Only one hardware entry per PMD with LPAE.
467 * On ARM one Linux PGD entry contains two hardware entries (see page
468 * tables layout in pgtable.h). We normally guarantee that we always
469 * fill both L1 entries. But create_mapping() doesn't follow the rule.
470 * It can create inidividual L1 entries, so here we have to call
471 * pmd_none() check for the entry really corresponded to address, not
472 * for the first of pair.
474 index
= (addr
>> SECTION_SHIFT
) & 1;
476 if (pmd_none(pmd_k
[index
]))
479 copy_pmd(pmd
, pmd_k
);
483 do_bad_area(addr
, fsr
, regs
);
486 #else /* CONFIG_MMU */
488 do_translation_fault(unsigned long addr
, unsigned int fsr
,
489 struct pt_regs
*regs
)
493 #endif /* CONFIG_MMU */
496 * Some section permission faults need to be handled gracefully.
497 * They can happen due to a __{get,put}_user during an oops.
499 #ifndef CONFIG_ARM_LPAE
501 do_sect_fault(unsigned long addr
, unsigned int fsr
, struct pt_regs
*regs
)
503 do_bad_area(addr
, fsr
, regs
);
506 #endif /* CONFIG_ARM_LPAE */
509 * This abort handler always returns "fault".
512 do_bad(unsigned long addr
, unsigned int fsr
, struct pt_regs
*regs
)
518 int (*fn
)(unsigned long addr
, unsigned int fsr
, struct pt_regs
*regs
);
525 #ifdef CONFIG_ARM_LPAE
526 #include "fsr-3level.c"
528 #include "fsr-2level.c"
532 hook_fault_code(int nr
, int (*fn
)(unsigned long, unsigned int, struct pt_regs
*),
533 int sig
, int code
, const char *name
)
535 if (nr
< 0 || nr
>= ARRAY_SIZE(fsr_info
))
538 fsr_info
[nr
].fn
= fn
;
539 fsr_info
[nr
].sig
= sig
;
540 fsr_info
[nr
].code
= code
;
541 fsr_info
[nr
].name
= name
;
545 * Dispatch a data abort to the relevant handler.
547 asmlinkage
void __exception
548 do_DataAbort(unsigned long addr
, unsigned int fsr
, struct pt_regs
*regs
)
550 const struct fsr_info
*inf
= fsr_info
+ fsr_fs(fsr
);
553 if (!inf
->fn(addr
, fsr
& ~FSR_LNX_PF
, regs
))
556 pr_alert("Unhandled fault: %s (0x%03x) at 0x%08lx\n",
557 inf
->name
, fsr
, addr
);
558 show_pte(current
->mm
, addr
);
560 info
.si_signo
= inf
->sig
;
562 info
.si_code
= inf
->code
;
563 info
.si_addr
= (void __user
*)addr
;
564 arm_notify_die("", regs
, &info
, fsr
, 0);
568 hook_ifault_code(int nr
, int (*fn
)(unsigned long, unsigned int, struct pt_regs
*),
569 int sig
, int code
, const char *name
)
571 if (nr
< 0 || nr
>= ARRAY_SIZE(ifsr_info
))
574 ifsr_info
[nr
].fn
= fn
;
575 ifsr_info
[nr
].sig
= sig
;
576 ifsr_info
[nr
].code
= code
;
577 ifsr_info
[nr
].name
= name
;
580 asmlinkage
void __exception
581 do_PrefetchAbort(unsigned long addr
, unsigned int ifsr
, struct pt_regs
*regs
)
583 const struct fsr_info
*inf
= ifsr_info
+ fsr_fs(ifsr
);
586 if (!inf
->fn(addr
, ifsr
| FSR_LNX_PF
, regs
))
589 pr_alert("Unhandled prefetch abort: %s (0x%03x) at 0x%08lx\n",
590 inf
->name
, ifsr
, addr
);
592 info
.si_signo
= inf
->sig
;
594 info
.si_code
= inf
->code
;
595 info
.si_addr
= (void __user
*)addr
;
596 arm_notify_die("", regs
, &info
, ifsr
, 0);
600 * Abort handler to be used only during first unmasking of asynchronous aborts
601 * on the boot CPU. This makes sure that the machine will not die if the
602 * firmware/bootloader left an imprecise abort pending for us to trip over.
604 static int __init
early_abort_handler(unsigned long addr
, unsigned int fsr
,
605 struct pt_regs
*regs
)
607 pr_warn("Hit pending asynchronous external abort (FSR=0x%08x) during "
608 "first unmask, this is most likely caused by a "
609 "firmware/bootloader bug.\n", fsr
);
614 void __init
early_abt_enable(void)
616 fsr_info
[FSR_FS_AEA
].fn
= early_abort_handler
;
618 fsr_info
[FSR_FS_AEA
].fn
= do_bad
;
621 #ifndef CONFIG_ARM_LPAE
622 static int __init
exceptions_init(void)
624 if (cpu_architecture() >= CPU_ARCH_ARMv6
) {
625 hook_fault_code(4, do_translation_fault
, SIGSEGV
, SEGV_MAPERR
,
626 "I-cache maintenance fault");
629 if (cpu_architecture() >= CPU_ARCH_ARMv7
) {
631 * TODO: Access flag faults introduced in ARMv6K.
632 * Runtime check for 'K' extension is needed
634 hook_fault_code(3, do_bad
, SIGSEGV
, SEGV_MAPERR
,
635 "section access flag fault");
636 hook_fault_code(6, do_bad
, SIGSEGV
, SEGV_MAPERR
,
637 "section access flag fault");
643 arch_initcall(exceptions_init
);