[PATCH] ppc64: Remove unused code
[linux-2.6/verdex.git] / arch / x86_64 / mm / fault.c
blob816732d8858c3195c7b6d78419b2b0b537de3f10
1 /*
2 * linux/arch/x86-64/mm/fault.c
4 * Copyright (C) 1995 Linus Torvalds
5 * Copyright (C) 2001,2002 Andi Kleen, SuSE Labs.
6 */
8 #include <linux/config.h>
9 #include <linux/signal.h>
10 #include <linux/sched.h>
11 #include <linux/kernel.h>
12 #include <linux/errno.h>
13 #include <linux/string.h>
14 #include <linux/types.h>
15 #include <linux/ptrace.h>
16 #include <linux/mman.h>
17 #include <linux/mm.h>
18 #include <linux/smp.h>
19 #include <linux/smp_lock.h>
20 #include <linux/interrupt.h>
21 #include <linux/init.h>
22 #include <linux/tty.h>
23 #include <linux/vt_kern.h> /* For unblank_screen() */
24 #include <linux/compiler.h>
25 #include <linux/module.h>
26 #include <linux/kprobes.h>
28 #include <asm/system.h>
29 #include <asm/uaccess.h>
30 #include <asm/pgalloc.h>
31 #include <asm/smp.h>
32 #include <asm/tlbflush.h>
33 #include <asm/proto.h>
34 #include <asm/kdebug.h>
35 #include <asm-generic/sections.h>
36 #include <asm/kdebug.h>
38 void bust_spinlocks(int yes)
40 int loglevel_save = console_loglevel;
41 if (yes) {
42 oops_in_progress = 1;
43 } else {
44 #ifdef CONFIG_VT
45 unblank_screen();
46 #endif
47 oops_in_progress = 0;
49 * OK, the message is on the console. Now we call printk()
50 * without oops_in_progress set so that printk will give klogd
51 * a poke. Hold onto your hats...
53 console_loglevel = 15; /* NMI oopser may have shut the console up */
54 printk(" ");
55 console_loglevel = loglevel_save;
59 /* Sometimes the CPU reports invalid exceptions on prefetch.
60 Check that here and ignore.
61 Opcode checker based on code by Richard Brunner */
62 static noinline int is_prefetch(struct pt_regs *regs, unsigned long addr,
63 unsigned long error_code)
65 unsigned char *instr;
66 int scan_more = 1;
67 int prefetch = 0;
68 unsigned char *max_instr;
70 /* If it was a exec fault ignore */
71 if (error_code & (1<<4))
72 return 0;
74 instr = (unsigned char *)convert_rip_to_linear(current, regs);
75 max_instr = instr + 15;
77 if (user_mode(regs) && instr >= (unsigned char *)TASK_SIZE)
78 return 0;
80 while (scan_more && instr < max_instr) {
81 unsigned char opcode;
82 unsigned char instr_hi;
83 unsigned char instr_lo;
85 if (__get_user(opcode, instr))
86 break;
88 instr_hi = opcode & 0xf0;
89 instr_lo = opcode & 0x0f;
90 instr++;
92 switch (instr_hi) {
93 case 0x20:
94 case 0x30:
95 /* Values 0x26,0x2E,0x36,0x3E are valid x86
96 prefixes. In long mode, the CPU will signal
97 invalid opcode if some of these prefixes are
98 present so we will never get here anyway */
99 scan_more = ((instr_lo & 7) == 0x6);
100 break;
102 case 0x40:
103 /* In AMD64 long mode, 0x40 to 0x4F are valid REX prefixes
104 Need to figure out under what instruction mode the
105 instruction was issued ... */
106 /* Could check the LDT for lm, but for now it's good
107 enough to assume that long mode only uses well known
108 segments or kernel. */
109 scan_more = (!user_mode(regs)) || (regs->cs == __USER_CS);
110 break;
112 case 0x60:
113 /* 0x64 thru 0x67 are valid prefixes in all modes. */
114 scan_more = (instr_lo & 0xC) == 0x4;
115 break;
116 case 0xF0:
117 /* 0xF0, 0xF2, and 0xF3 are valid prefixes in all modes. */
118 scan_more = !instr_lo || (instr_lo>>1) == 1;
119 break;
120 case 0x00:
121 /* Prefetch instruction is 0x0F0D or 0x0F18 */
122 scan_more = 0;
123 if (__get_user(opcode, instr))
124 break;
125 prefetch = (instr_lo == 0xF) &&
126 (opcode == 0x0D || opcode == 0x18);
127 break;
128 default:
129 scan_more = 0;
130 break;
133 return prefetch;
136 static int bad_address(void *p)
138 unsigned long dummy;
139 return __get_user(dummy, (unsigned long *)p);
142 void dump_pagetable(unsigned long address)
144 pgd_t *pgd;
145 pud_t *pud;
146 pmd_t *pmd;
147 pte_t *pte;
149 asm("movq %%cr3,%0" : "=r" (pgd));
151 pgd = __va((unsigned long)pgd & PHYSICAL_PAGE_MASK);
152 pgd += pgd_index(address);
153 printk("PGD %lx ", pgd_val(*pgd));
154 if (bad_address(pgd)) goto bad;
155 if (!pgd_present(*pgd)) goto ret;
157 pud = __pud_offset_k((pud_t *)pgd_page(*pgd), address);
158 if (bad_address(pud)) goto bad;
159 printk("PUD %lx ", pud_val(*pud));
160 if (!pud_present(*pud)) goto ret;
162 pmd = pmd_offset(pud, address);
163 if (bad_address(pmd)) goto bad;
164 printk("PMD %lx ", pmd_val(*pmd));
165 if (!pmd_present(*pmd)) goto ret;
167 pte = pte_offset_kernel(pmd, address);
168 if (bad_address(pte)) goto bad;
169 printk("PTE %lx", pte_val(*pte));
170 ret:
171 printk("\n");
172 return;
173 bad:
174 printk("BAD\n");
177 static const char errata93_warning[] =
178 KERN_ERR "******* Your BIOS seems to not contain a fix for K8 errata #93\n"
179 KERN_ERR "******* Working around it, but it may cause SEGVs or burn power.\n"
180 KERN_ERR "******* Please consider a BIOS update.\n"
181 KERN_ERR "******* Disabling USB legacy in the BIOS may also help.\n";
183 /* Workaround for K8 erratum #93 & buggy BIOS.
184 BIOS SMM functions are required to use a specific workaround
185 to avoid corruption of the 64bit RIP register on C stepping K8.
186 A lot of BIOS that didn't get tested properly miss this.
187 The OS sees this as a page fault with the upper 32bits of RIP cleared.
188 Try to work around it here.
189 Note we only handle faults in kernel here. */
191 static int is_errata93(struct pt_regs *regs, unsigned long address)
193 static int warned;
194 if (address != regs->rip)
195 return 0;
196 if ((address >> 32) != 0)
197 return 0;
198 address |= 0xffffffffUL << 32;
199 if ((address >= (u64)_stext && address <= (u64)_etext) ||
200 (address >= MODULES_VADDR && address <= MODULES_END)) {
201 if (!warned) {
202 printk(errata93_warning);
203 warned = 1;
205 regs->rip = address;
206 return 1;
208 return 0;
211 int unhandled_signal(struct task_struct *tsk, int sig)
213 if (tsk->pid == 1)
214 return 1;
215 if (tsk->ptrace & PT_PTRACED)
216 return 0;
217 return (tsk->sighand->action[sig-1].sa.sa_handler == SIG_IGN) ||
218 (tsk->sighand->action[sig-1].sa.sa_handler == SIG_DFL);
221 static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs,
222 unsigned long error_code)
224 oops_begin();
225 printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
226 current->comm, address);
227 dump_pagetable(address);
228 __die("Bad pagetable", regs, error_code);
229 oops_end();
230 do_exit(SIGKILL);
234 * Handle a fault on the vmalloc or module mapping area
236 * This assumes no large pages in there.
238 static int vmalloc_fault(unsigned long address)
240 pgd_t *pgd, *pgd_ref;
241 pud_t *pud, *pud_ref;
242 pmd_t *pmd, *pmd_ref;
243 pte_t *pte, *pte_ref;
245 /* Copy kernel mappings over when needed. This can also
246 happen within a race in page table update. In the later
247 case just flush. */
249 pgd = pgd_offset(current->mm ?: &init_mm, address);
250 pgd_ref = pgd_offset_k(address);
251 if (pgd_none(*pgd_ref))
252 return -1;
253 if (pgd_none(*pgd))
254 set_pgd(pgd, *pgd_ref);
256 /* Below here mismatches are bugs because these lower tables
257 are shared */
259 pud = pud_offset(pgd, address);
260 pud_ref = pud_offset(pgd_ref, address);
261 if (pud_none(*pud_ref))
262 return -1;
263 if (pud_none(*pud) || pud_page(*pud) != pud_page(*pud_ref))
264 BUG();
265 pmd = pmd_offset(pud, address);
266 pmd_ref = pmd_offset(pud_ref, address);
267 if (pmd_none(*pmd_ref))
268 return -1;
269 if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref))
270 BUG();
271 pte_ref = pte_offset_kernel(pmd_ref, address);
272 if (!pte_present(*pte_ref))
273 return -1;
274 pte = pte_offset_kernel(pmd, address);
275 /* Don't use pte_page here, because the mappings can point
276 outside mem_map, and the NUMA hash lookup cannot handle
277 that. */
278 if (!pte_present(*pte) || pte_pfn(*pte) != pte_pfn(*pte_ref))
279 BUG();
280 __flush_tlb_all();
281 return 0;
284 int page_fault_trace = 0;
285 int exception_trace = 1;
288 * This routine handles page faults. It determines the address,
289 * and the problem, and then passes it off to one of the appropriate
290 * routines.
292 * error_code:
293 * bit 0 == 0 means no page found, 1 means protection fault
294 * bit 1 == 0 means read, 1 means write
295 * bit 2 == 0 means kernel, 1 means user-mode
296 * bit 3 == 1 means fault was an instruction fetch
298 asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
299 unsigned long error_code)
301 struct task_struct *tsk;
302 struct mm_struct *mm;
303 struct vm_area_struct * vma;
304 unsigned long address;
305 const struct exception_table_entry *fixup;
306 int write;
307 siginfo_t info;
309 #ifdef CONFIG_CHECKING
311 unsigned long gs;
312 struct x8664_pda *pda = cpu_pda + stack_smp_processor_id();
313 rdmsrl(MSR_GS_BASE, gs);
314 if (gs != (unsigned long)pda) {
315 wrmsrl(MSR_GS_BASE, pda);
316 printk("page_fault: wrong gs %lx expected %p\n", gs, pda);
319 #endif
321 /* get the address */
322 __asm__("movq %%cr2,%0":"=r" (address));
323 if (notify_die(DIE_PAGE_FAULT, "page fault", regs, error_code, 14,
324 SIGSEGV) == NOTIFY_STOP)
325 return;
327 if (likely(regs->eflags & X86_EFLAGS_IF))
328 local_irq_enable();
330 if (unlikely(page_fault_trace))
331 printk("pagefault rip:%lx rsp:%lx cs:%lu ss:%lu address %lx error %lx\n",
332 regs->rip,regs->rsp,regs->cs,regs->ss,address,error_code);
334 tsk = current;
335 mm = tsk->mm;
336 info.si_code = SEGV_MAPERR;
340 * We fault-in kernel-space virtual memory on-demand. The
341 * 'reference' page table is init_mm.pgd.
343 * NOTE! We MUST NOT take any locks for this case. We may
344 * be in an interrupt or a critical region, and should
345 * only copy the information from the master page table,
346 * nothing more.
348 * This verifies that the fault happens in kernel space
349 * (error_code & 4) == 0, and that the fault was not a
350 * protection error (error_code & 1) == 0.
352 if (unlikely(address >= TASK_SIZE64)) {
353 if (!(error_code & 5) &&
354 ((address >= VMALLOC_START && address < VMALLOC_END) ||
355 (address >= MODULES_VADDR && address < MODULES_END))) {
356 if (vmalloc_fault(address) < 0)
357 goto bad_area_nosemaphore;
358 return;
361 * Don't take the mm semaphore here. If we fixup a prefetch
362 * fault we could otherwise deadlock.
364 goto bad_area_nosemaphore;
367 if (unlikely(error_code & (1 << 3)))
368 pgtable_bad(address, regs, error_code);
371 * If we're in an interrupt or have no user
372 * context, we must not take the fault..
374 if (unlikely(in_atomic() || !mm))
375 goto bad_area_nosemaphore;
377 again:
378 /* When running in the kernel we expect faults to occur only to
379 * addresses in user space. All other faults represent errors in the
380 * kernel and should generate an OOPS. Unfortunatly, in the case of an
381 * erroneous fault occuring in a code path which already holds mmap_sem
382 * we will deadlock attempting to validate the fault against the
383 * address space. Luckily the kernel only validly references user
384 * space from well defined areas of code, which are listed in the
385 * exceptions table.
387 * As the vast majority of faults will be valid we will only perform
388 * the source reference check when there is a possibilty of a deadlock.
389 * Attempt to lock the address space, if we cannot we then validate the
390 * source. If this is invalid we can skip the address space check,
391 * thus avoiding the deadlock.
393 if (!down_read_trylock(&mm->mmap_sem)) {
394 if ((error_code & 4) == 0 &&
395 !search_exception_tables(regs->rip))
396 goto bad_area_nosemaphore;
397 down_read(&mm->mmap_sem);
400 vma = find_vma(mm, address);
401 if (!vma)
402 goto bad_area;
403 if (likely(vma->vm_start <= address))
404 goto good_area;
405 if (!(vma->vm_flags & VM_GROWSDOWN))
406 goto bad_area;
407 if (error_code & 4) {
408 // XXX: align red zone size with ABI
409 if (address + 128 < regs->rsp)
410 goto bad_area;
412 if (expand_stack(vma, address))
413 goto bad_area;
415 * Ok, we have a good vm_area for this memory access, so
416 * we can handle it..
418 good_area:
419 info.si_code = SEGV_ACCERR;
420 write = 0;
421 switch (error_code & 3) {
422 default: /* 3: write, present */
423 /* fall through */
424 case 2: /* write, not present */
425 if (!(vma->vm_flags & VM_WRITE))
426 goto bad_area;
427 write++;
428 break;
429 case 1: /* read, present */
430 goto bad_area;
431 case 0: /* read, not present */
432 if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
433 goto bad_area;
437 * If for any reason at all we couldn't handle the fault,
438 * make sure we exit gracefully rather than endlessly redo
439 * the fault.
441 switch (handle_mm_fault(mm, vma, address, write)) {
442 case VM_FAULT_MINOR:
443 tsk->min_flt++;
444 break;
445 case VM_FAULT_MAJOR:
446 tsk->maj_flt++;
447 break;
448 case VM_FAULT_SIGBUS:
449 goto do_sigbus;
450 default:
451 goto out_of_memory;
454 up_read(&mm->mmap_sem);
455 return;
458 * Something tried to access memory that isn't in our memory map..
459 * Fix it, but check if it's kernel or user first..
461 bad_area:
462 up_read(&mm->mmap_sem);
464 bad_area_nosemaphore:
465 /* User mode accesses just cause a SIGSEGV */
466 if (error_code & 4) {
467 if (is_prefetch(regs, address, error_code))
468 return;
470 /* Work around K8 erratum #100 K8 in compat mode
471 occasionally jumps to illegal addresses >4GB. We
472 catch this here in the page fault handler because
473 these addresses are not reachable. Just detect this
474 case and return. Any code segment in LDT is
475 compatibility mode. */
476 if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) &&
477 (address >> 32))
478 return;
480 if (exception_trace && unhandled_signal(tsk, SIGSEGV)) {
481 printk(
482 "%s%s[%d]: segfault at %016lx rip %016lx rsp %016lx error %lx\n",
483 tsk->pid > 1 ? KERN_INFO : KERN_EMERG,
484 tsk->comm, tsk->pid, address, regs->rip,
485 regs->rsp, error_code);
488 tsk->thread.cr2 = address;
489 /* Kernel addresses are always protection faults */
490 tsk->thread.error_code = error_code | (address >= TASK_SIZE);
491 tsk->thread.trap_no = 14;
492 info.si_signo = SIGSEGV;
493 info.si_errno = 0;
494 /* info.si_code has been set above */
495 info.si_addr = (void __user *)address;
496 force_sig_info(SIGSEGV, &info, tsk);
497 return;
500 no_context:
502 /* Are we prepared to handle this kernel fault? */
503 fixup = search_exception_tables(regs->rip);
504 if (fixup) {
505 regs->rip = fixup->fixup;
506 return;
510 * Hall of shame of CPU/BIOS bugs.
513 if (is_prefetch(regs, address, error_code))
514 return;
516 if (is_errata93(regs, address))
517 return;
520 * Oops. The kernel tried to access some bad page. We'll have to
521 * terminate things with extreme prejudice.
524 oops_begin();
526 if (address < PAGE_SIZE)
527 printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
528 else
529 printk(KERN_ALERT "Unable to handle kernel paging request");
530 printk(" at %016lx RIP: \n" KERN_ALERT,address);
531 printk_address(regs->rip);
532 printk("\n");
533 dump_pagetable(address);
534 __die("Oops", regs, error_code);
535 /* Executive summary in case the body of the oops scrolled away */
536 printk(KERN_EMERG "CR2: %016lx\n", address);
537 oops_end();
538 do_exit(SIGKILL);
541 * We ran out of memory, or some other thing happened to us that made
542 * us unable to handle the page fault gracefully.
544 out_of_memory:
545 up_read(&mm->mmap_sem);
546 if (current->pid == 1) {
547 yield();
548 goto again;
550 printk("VM: killing process %s\n", tsk->comm);
551 if (error_code & 4)
552 do_exit(SIGKILL);
553 goto no_context;
555 do_sigbus:
556 up_read(&mm->mmap_sem);
558 /* Kernel mode? Handle exceptions or die */
559 if (!(error_code & 4))
560 goto no_context;
562 tsk->thread.cr2 = address;
563 tsk->thread.error_code = error_code;
564 tsk->thread.trap_no = 14;
565 info.si_signo = SIGBUS;
566 info.si_errno = 0;
567 info.si_code = BUS_ADRERR;
568 info.si_addr = (void __user *)address;
569 force_sig_info(SIGBUS, &info, tsk);
570 return;