2 * linux/arch/arm/mm/fault.c
4 * Copyright (C) 1995 Linus Torvalds
5 * Modifications for ARM processor (c) 1995-2004 Russell King
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
11 #include <linux/module.h>
12 #include <linux/signal.h>
14 #include <linux/hardirq.h>
15 #include <linux/init.h>
16 #include <linux/kprobes.h>
17 #include <linux/uaccess.h>
18 #include <linux/page-flags.h>
19 #include <linux/sched.h>
20 #include <linux/highmem.h>
21 #include <linux/perf_event.h>
23 #include <asm/system.h>
24 #include <asm/pgtable.h>
25 #include <asm/tlbflush.h>
30 * Fault status register encodings. We steal bit 31 for our own purposes.
32 #define FSR_LNX_PF (1 << 31)
33 #define FSR_WRITE (1 << 11)
34 #define FSR_FS4 (1 << 10)
35 #define FSR_FS3_0 (15)
37 static inline int fsr_fs(unsigned int fsr
)
39 return (fsr
& FSR_FS3_0
) | (fsr
& FSR_FS4
) >> 6;
45 static inline int notify_page_fault(struct pt_regs
*regs
, unsigned int fsr
)
49 if (!user_mode(regs
)) {
50 /* kprobe_running() needs smp_processor_id() */
52 if (kprobe_running() && kprobe_fault_handler(regs
, fsr
))
60 static inline int notify_page_fault(struct pt_regs
*regs
, unsigned int fsr
)
67 * This is useful to dump out the page tables associated with
70 void show_pte(struct mm_struct
*mm
, unsigned long addr
)
77 printk(KERN_ALERT
"pgd = %p\n", mm
->pgd
);
78 pgd
= pgd_offset(mm
, addr
);
79 printk(KERN_ALERT
"[%08lx] *pgd=%08llx",
80 addr
, (long long)pgd_val(*pgd
));
95 pud
= pud_offset(pgd
, addr
);
96 if (PTRS_PER_PUD
!= 1)
97 printk(", *pud=%08lx", pud_val(*pud
));
107 pmd
= pmd_offset(pud
, addr
);
108 if (PTRS_PER_PMD
!= 1)
109 printk(", *pmd=%08llx", (long long)pmd_val(*pmd
));
119 /* We must not map this if we have highmem enabled */
120 if (PageHighMem(pfn_to_page(pmd_val(*pmd
) >> PAGE_SHIFT
)))
123 pte
= pte_offset_map(pmd
, addr
);
124 printk(", *pte=%08llx", (long long)pte_val(*pte
));
125 printk(", *ppte=%08llx",
126 (long long)pte_val(pte
[PTE_HWTABLE_PTRS
]));
132 #else /* CONFIG_MMU */
133 void show_pte(struct mm_struct
*mm
, unsigned long addr
)
135 #endif /* CONFIG_MMU */
138 * Oops. The kernel tried to access some page that wasn't present.
141 __do_kernel_fault(struct mm_struct
*mm
, unsigned long addr
, unsigned int fsr
,
142 struct pt_regs
*regs
)
145 * Are we prepared to handle this kernel fault?
147 if (fixup_exception(regs
))
151 * No handler, we'll have to terminate things with extreme prejudice.
155 "Unable to handle kernel %s at virtual address %08lx\n",
156 (addr
< PAGE_SIZE
) ? "NULL pointer dereference" :
157 "paging request", addr
);
160 die("Oops", regs
, fsr
);
166 * Something tried to access memory that isn't in our memory map..
167 * User mode accesses just cause a SIGSEGV
170 __do_user_fault(struct task_struct
*tsk
, unsigned long addr
,
171 unsigned int fsr
, unsigned int sig
, int code
,
172 struct pt_regs
*regs
)
176 #ifdef CONFIG_DEBUG_USER
177 if (user_debug
& UDBG_SEGV
) {
178 printk(KERN_DEBUG
"%s: unhandled page fault (%d) at 0x%08lx, code 0x%03x\n",
179 tsk
->comm
, sig
, addr
, fsr
);
180 show_pte(tsk
->mm
, addr
);
185 tsk
->thread
.address
= addr
;
186 tsk
->thread
.error_code
= fsr
;
187 tsk
->thread
.trap_no
= 14;
191 si
.si_addr
= (void __user
*)addr
;
192 force_sig_info(sig
, &si
, tsk
);
195 void do_bad_area(unsigned long addr
, unsigned int fsr
, struct pt_regs
*regs
)
197 struct task_struct
*tsk
= current
;
198 struct mm_struct
*mm
= tsk
->active_mm
;
201 * If we are in kernel mode at this point, we
202 * have no context to handle this fault with.
205 __do_user_fault(tsk
, addr
, fsr
, SIGSEGV
, SEGV_MAPERR
, regs
);
207 __do_kernel_fault(mm
, addr
, fsr
, regs
);
211 #define VM_FAULT_BADMAP 0x010000
212 #define VM_FAULT_BADACCESS 0x020000
215 * Check that the permissions on the VMA allow for the fault which occurred.
216 * If we encountered a write fault, we must have write permission, otherwise
217 * we allow any permission.
219 static inline bool access_error(unsigned int fsr
, struct vm_area_struct
*vma
)
221 unsigned int mask
= VM_READ
| VM_WRITE
| VM_EXEC
;
225 if (fsr
& FSR_LNX_PF
)
228 return vma
->vm_flags
& mask
? false : true;
232 __do_page_fault(struct mm_struct
*mm
, unsigned long addr
, unsigned int fsr
,
233 struct task_struct
*tsk
)
235 struct vm_area_struct
*vma
;
238 vma
= find_vma(mm
, addr
);
239 fault
= VM_FAULT_BADMAP
;
242 if (unlikely(vma
->vm_start
> addr
))
246 * Ok, we have a good vm_area for this
247 * memory access, so we can handle it.
250 if (access_error(fsr
, vma
)) {
251 fault
= VM_FAULT_BADACCESS
;
256 * If for any reason at all we couldn't handle the fault, make
257 * sure we exit gracefully rather than endlessly redo the fault.
259 fault
= handle_mm_fault(mm
, vma
, addr
& PAGE_MASK
, (fsr
& FSR_WRITE
) ? FAULT_FLAG_WRITE
: 0);
260 if (unlikely(fault
& VM_FAULT_ERROR
))
262 if (fault
& VM_FAULT_MAJOR
)
269 if (vma
->vm_flags
& VM_GROWSDOWN
&& !expand_stack(vma
, addr
))
276 do_page_fault(unsigned long addr
, unsigned int fsr
, struct pt_regs
*regs
)
278 struct task_struct
*tsk
;
279 struct mm_struct
*mm
;
280 int fault
, sig
, code
;
282 if (notify_page_fault(regs
, fsr
))
289 * If we're in an interrupt or have no user
290 * context, we must not take the fault..
292 if (in_atomic() || !mm
)
296 * As per x86, we may deadlock here. However, since the kernel only
297 * validly references user space from well defined areas of the code,
298 * we can bug out early if this is from code which shouldn't.
300 if (!down_read_trylock(&mm
->mmap_sem
)) {
301 if (!user_mode(regs
) && !search_exception_tables(regs
->ARM_pc
))
303 down_read(&mm
->mmap_sem
);
306 * The above down_read_trylock() might have succeeded in
307 * which case, we'll have missed the might_sleep() from
311 #ifdef CONFIG_DEBUG_VM
312 if (!user_mode(regs
) &&
313 !search_exception_tables(regs
->ARM_pc
))
318 fault
= __do_page_fault(mm
, addr
, fsr
, tsk
);
319 up_read(&mm
->mmap_sem
);
321 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS
, 1, 0, regs
, addr
);
322 if (fault
& VM_FAULT_MAJOR
)
323 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ
, 1, 0, regs
, addr
);
324 else if (fault
& VM_FAULT_MINOR
)
325 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN
, 1, 0, regs
, addr
);
328 * Handle the "normal" case first - VM_FAULT_MAJOR / VM_FAULT_MINOR
330 if (likely(!(fault
& (VM_FAULT_ERROR
| VM_FAULT_BADMAP
| VM_FAULT_BADACCESS
))))
333 if (fault
& VM_FAULT_OOM
) {
335 * We ran out of memory, call the OOM killer, and return to
336 * userspace (which will retry the fault, or kill us if we
339 pagefault_out_of_memory();
344 * If we are in kernel mode at this point, we
345 * have no context to handle this fault with.
347 if (!user_mode(regs
))
350 if (fault
& VM_FAULT_SIGBUS
) {
352 * We had some memory, but were unable to
353 * successfully fix up this page fault.
359 * Something tried to access memory that
360 * isn't in our memory map..
363 code
= fault
== VM_FAULT_BADACCESS
?
364 SEGV_ACCERR
: SEGV_MAPERR
;
367 __do_user_fault(tsk
, addr
, fsr
, sig
, code
, regs
);
371 __do_kernel_fault(mm
, addr
, fsr
, regs
);
374 #else /* CONFIG_MMU */
376 do_page_fault(unsigned long addr
, unsigned int fsr
, struct pt_regs
*regs
)
380 #endif /* CONFIG_MMU */
383 * First Level Translation Fault Handler
385 * We enter here because the first level page table doesn't contain
386 * a valid entry for the address.
388 * If the address is in kernel space (>= TASK_SIZE), then we are
389 * probably faulting in the vmalloc() area.
391 * If the init_task's first level page tables contains the relevant
392 * entry, we copy the it to this task. If not, we send the process
393 * a signal, fixup the exception, or oops the kernel.
395 * NOTE! We MUST NOT take any locks for this case. We may be in an
396 * interrupt or a critical region, and should only copy the information
397 * from the master page table, nothing more.
401 do_translation_fault(unsigned long addr
, unsigned int fsr
,
402 struct pt_regs
*regs
)
409 if (addr
< TASK_SIZE
)
410 return do_page_fault(addr
, fsr
, regs
);
415 index
= pgd_index(addr
);
418 * FIXME: CP15 C1 is write only on ARMv3 architectures.
420 pgd
= cpu_get_pgd() + index
;
421 pgd_k
= init_mm
.pgd
+ index
;
423 if (pgd_none(*pgd_k
))
425 if (!pgd_present(*pgd
))
426 set_pgd(pgd
, *pgd_k
);
428 pud
= pud_offset(pgd
, addr
);
429 pud_k
= pud_offset(pgd_k
, addr
);
431 if (pud_none(*pud_k
))
433 if (!pud_present(*pud
))
434 set_pud(pud
, *pud_k
);
436 pmd
= pmd_offset(pud
, addr
);
437 pmd_k
= pmd_offset(pud_k
, addr
);
440 * On ARM one Linux PGD entry contains two hardware entries (see page
441 * tables layout in pgtable.h). We normally guarantee that we always
442 * fill both L1 entries. But create_mapping() doesn't follow the rule.
443 * It can create inidividual L1 entries, so here we have to call
444 * pmd_none() check for the entry really corresponded to address, not
445 * for the first of pair.
447 index
= (addr
>> SECTION_SHIFT
) & 1;
448 if (pmd_none(pmd_k
[index
]))
451 copy_pmd(pmd
, pmd_k
);
455 do_bad_area(addr
, fsr
, regs
);
458 #else /* CONFIG_MMU */
460 do_translation_fault(unsigned long addr
, unsigned int fsr
,
461 struct pt_regs
*regs
)
465 #endif /* CONFIG_MMU */
468 * Some section permission faults need to be handled gracefully.
469 * They can happen due to a __{get,put}_user during an oops.
472 do_sect_fault(unsigned long addr
, unsigned int fsr
, struct pt_regs
*regs
)
474 do_bad_area(addr
, fsr
, regs
);
479 * This abort handler always returns "fault".
482 do_bad(unsigned long addr
, unsigned int fsr
, struct pt_regs
*regs
)
487 static struct fsr_info
{
488 int (*fn
)(unsigned long addr
, unsigned int fsr
, struct pt_regs
*regs
);
494 * The following are the standard ARMv3 and ARMv4 aborts. ARMv5
495 * defines these to be "precise" aborts.
497 { do_bad
, SIGSEGV
, 0, "vector exception" },
498 { do_bad
, SIGBUS
, BUS_ADRALN
, "alignment exception" },
499 { do_bad
, SIGKILL
, 0, "terminal exception" },
500 { do_bad
, SIGBUS
, BUS_ADRALN
, "alignment exception" },
501 { do_bad
, SIGBUS
, 0, "external abort on linefetch" },
502 { do_translation_fault
, SIGSEGV
, SEGV_MAPERR
, "section translation fault" },
503 { do_bad
, SIGBUS
, 0, "external abort on linefetch" },
504 { do_page_fault
, SIGSEGV
, SEGV_MAPERR
, "page translation fault" },
505 { do_bad
, SIGBUS
, 0, "external abort on non-linefetch" },
506 { do_bad
, SIGSEGV
, SEGV_ACCERR
, "section domain fault" },
507 { do_bad
, SIGBUS
, 0, "external abort on non-linefetch" },
508 { do_bad
, SIGSEGV
, SEGV_ACCERR
, "page domain fault" },
509 { do_bad
, SIGBUS
, 0, "external abort on translation" },
510 { do_sect_fault
, SIGSEGV
, SEGV_ACCERR
, "section permission fault" },
511 { do_bad
, SIGBUS
, 0, "external abort on translation" },
512 { do_page_fault
, SIGSEGV
, SEGV_ACCERR
, "page permission fault" },
514 * The following are "imprecise" aborts, which are signalled by bit
515 * 10 of the FSR, and may not be recoverable. These are only
516 * supported if the CPU abort handler supports bit 10.
518 { do_bad
, SIGBUS
, 0, "unknown 16" },
519 { do_bad
, SIGBUS
, 0, "unknown 17" },
520 { do_bad
, SIGBUS
, 0, "unknown 18" },
521 { do_bad
, SIGBUS
, 0, "unknown 19" },
522 { do_bad
, SIGBUS
, 0, "lock abort" }, /* xscale */
523 { do_bad
, SIGBUS
, 0, "unknown 21" },
524 { do_bad
, SIGBUS
, BUS_OBJERR
, "imprecise external abort" }, /* xscale */
525 { do_bad
, SIGBUS
, 0, "unknown 23" },
526 { do_bad
, SIGBUS
, 0, "dcache parity error" }, /* xscale */
527 { do_bad
, SIGBUS
, 0, "unknown 25" },
528 { do_bad
, SIGBUS
, 0, "unknown 26" },
529 { do_bad
, SIGBUS
, 0, "unknown 27" },
530 { do_bad
, SIGBUS
, 0, "unknown 28" },
531 { do_bad
, SIGBUS
, 0, "unknown 29" },
532 { do_bad
, SIGBUS
, 0, "unknown 30" },
533 { do_bad
, SIGBUS
, 0, "unknown 31" }
537 hook_fault_code(int nr
, int (*fn
)(unsigned long, unsigned int, struct pt_regs
*),
538 int sig
, int code
, const char *name
)
540 if (nr
< 0 || nr
>= ARRAY_SIZE(fsr_info
))
543 fsr_info
[nr
].fn
= fn
;
544 fsr_info
[nr
].sig
= sig
;
545 fsr_info
[nr
].code
= code
;
546 fsr_info
[nr
].name
= name
;
550 * Dispatch a data abort to the relevant handler.
552 asmlinkage
void __exception
553 do_DataAbort(unsigned long addr
, unsigned int fsr
, struct pt_regs
*regs
)
555 const struct fsr_info
*inf
= fsr_info
+ fsr_fs(fsr
);
558 if (!inf
->fn(addr
, fsr
& ~FSR_LNX_PF
, regs
))
561 printk(KERN_ALERT
"Unhandled fault: %s (0x%03x) at 0x%08lx\n",
562 inf
->name
, fsr
, addr
);
564 info
.si_signo
= inf
->sig
;
566 info
.si_code
= inf
->code
;
567 info
.si_addr
= (void __user
*)addr
;
568 arm_notify_die("", regs
, &info
, fsr
, 0);
572 static struct fsr_info ifsr_info
[] = {
573 { do_bad
, SIGBUS
, 0, "unknown 0" },
574 { do_bad
, SIGBUS
, 0, "unknown 1" },
575 { do_bad
, SIGBUS
, 0, "debug event" },
576 { do_bad
, SIGSEGV
, SEGV_ACCERR
, "section access flag fault" },
577 { do_bad
, SIGBUS
, 0, "unknown 4" },
578 { do_translation_fault
, SIGSEGV
, SEGV_MAPERR
, "section translation fault" },
579 { do_bad
, SIGSEGV
, SEGV_ACCERR
, "page access flag fault" },
580 { do_page_fault
, SIGSEGV
, SEGV_MAPERR
, "page translation fault" },
581 { do_bad
, SIGBUS
, 0, "external abort on non-linefetch" },
582 { do_bad
, SIGSEGV
, SEGV_ACCERR
, "section domain fault" },
583 { do_bad
, SIGBUS
, 0, "unknown 10" },
584 { do_bad
, SIGSEGV
, SEGV_ACCERR
, "page domain fault" },
585 { do_bad
, SIGBUS
, 0, "external abort on translation" },
586 { do_sect_fault
, SIGSEGV
, SEGV_ACCERR
, "section permission fault" },
587 { do_bad
, SIGBUS
, 0, "external abort on translation" },
588 { do_page_fault
, SIGSEGV
, SEGV_ACCERR
, "page permission fault" },
589 { do_bad
, SIGBUS
, 0, "unknown 16" },
590 { do_bad
, SIGBUS
, 0, "unknown 17" },
591 { do_bad
, SIGBUS
, 0, "unknown 18" },
592 { do_bad
, SIGBUS
, 0, "unknown 19" },
593 { do_bad
, SIGBUS
, 0, "unknown 20" },
594 { do_bad
, SIGBUS
, 0, "unknown 21" },
595 { do_bad
, SIGBUS
, 0, "unknown 22" },
596 { do_bad
, SIGBUS
, 0, "unknown 23" },
597 { do_bad
, SIGBUS
, 0, "unknown 24" },
598 { do_bad
, SIGBUS
, 0, "unknown 25" },
599 { do_bad
, SIGBUS
, 0, "unknown 26" },
600 { do_bad
, SIGBUS
, 0, "unknown 27" },
601 { do_bad
, SIGBUS
, 0, "unknown 28" },
602 { do_bad
, SIGBUS
, 0, "unknown 29" },
603 { do_bad
, SIGBUS
, 0, "unknown 30" },
604 { do_bad
, SIGBUS
, 0, "unknown 31" },
608 hook_ifault_code(int nr
, int (*fn
)(unsigned long, unsigned int, struct pt_regs
*),
609 int sig
, int code
, const char *name
)
611 if (nr
< 0 || nr
>= ARRAY_SIZE(ifsr_info
))
614 ifsr_info
[nr
].fn
= fn
;
615 ifsr_info
[nr
].sig
= sig
;
616 ifsr_info
[nr
].code
= code
;
617 ifsr_info
[nr
].name
= name
;
620 asmlinkage
void __exception
621 do_PrefetchAbort(unsigned long addr
, unsigned int ifsr
, struct pt_regs
*regs
)
623 const struct fsr_info
*inf
= ifsr_info
+ fsr_fs(ifsr
);
626 if (!inf
->fn(addr
, ifsr
| FSR_LNX_PF
, regs
))
629 printk(KERN_ALERT
"Unhandled prefetch abort: %s (0x%03x) at 0x%08lx\n",
630 inf
->name
, ifsr
, addr
);
632 info
.si_signo
= inf
->sig
;
634 info
.si_code
= inf
->code
;
635 info
.si_addr
= (void __user
*)addr
;
636 arm_notify_die("", regs
, &info
, ifsr
, 0);
639 static int __init
exceptions_init(void)
641 if (cpu_architecture() >= CPU_ARCH_ARMv6
) {
642 hook_fault_code(4, do_translation_fault
, SIGSEGV
, SEGV_MAPERR
,
643 "I-cache maintenance fault");
646 if (cpu_architecture() >= CPU_ARCH_ARMv7
) {
648 * TODO: Access flag faults introduced in ARMv6K.
649 * Runtime check for 'K' extension is needed
651 hook_fault_code(3, do_bad
, SIGSEGV
, SEGV_MAPERR
,
652 "section access flag fault");
653 hook_fault_code(6, do_bad
, SIGSEGV
, SEGV_MAPERR
,
654 "section access flag fault");
660 arch_initcall(exceptions_init
);