arch/sh/mm/fault.c

   1 /*
   2  * Page fault handler for SH with an MMU.
   3  *
   4  *  Copyright (C) 1999  Niibe Yutaka
   5  *  Copyright (C) 2003 - 2012  Paul Mundt
   6  *
   7  *  Based on linux/arch/i386/mm/fault.c:
   8  *   Copyright (C) 1995  Linus Torvalds
   9  *
  10  * This file is subject to the terms and conditions of the GNU General Public
  11  * License.  See the file "COPYING" in the main directory of this archive
  12  * for more details.
  13  */
  14 #include <linux/kernel.h>
  15 #include <linux/mm.h>
  16 #include <linux/sched/signal.h>
  17 #include <linux/hardirq.h>
  18 #include <linux/kprobes.h>
  19 #include <linux/perf_event.h>
  20 #include <linux/kdebug.h>
  21 #include <linux/uaccess.h>
  22 #include <asm/io_trapped.h>
  23 #include <asm/mmu_context.h>
  24 #include <asm/tlbflush.h>
  25 #include <asm/traps.h>
  26
  27 static void
  28 force_sig_info_fault(int si_signo, int si_code, unsigned long address)
  29 {
  30         force_sig_fault(si_signo, si_code, (void __user *)address);
  31 }
  32
  33 /*
  34  * This is useful to dump out the page tables associated with
  35  * 'addr' in mm 'mm'.
  36  */
  37 static void show_pte(struct mm_struct *mm, unsigned long addr)
  38 {
  39         pgd_t *pgd;
  40
  41         if (mm) {
  42                 pgd = mm->pgd;
  43         } else {
  44                 pgd = get_TTB();
  45
  46                 if (unlikely(!pgd))
  47                         pgd = swapper_pg_dir;
  48         }
  49
  50         printk(KERN_ALERT "pgd = %p\n", pgd);
  51         pgd += pgd_index(addr);
  52         printk(KERN_ALERT "[%08lx] *pgd=%0*Lx", addr,
  53                (u32)(sizeof(*pgd) * 2), (u64)pgd_val(*pgd));
  54
  55         do {
  56                 pud_t *pud;
  57                 pmd_t *pmd;
  58                 pte_t *pte;
  59
  60                 if (pgd_none(*pgd))
  61                         break;
  62
  63                 if (pgd_bad(*pgd)) {
  64                         printk("(bad)");
  65                         break;
  66                 }
  67
  68                 pud = pud_offset(pgd, addr);
  69                 if (PTRS_PER_PUD != 1)
  70                         printk(", *pud=%0*Lx", (u32)(sizeof(*pud) * 2),
  71                                (u64)pud_val(*pud));
  72
  73                 if (pud_none(*pud))
  74                         break;
  75
  76                 if (pud_bad(*pud)) {
  77                         printk("(bad)");
  78                         break;
  79                 }
  80
  81                 pmd = pmd_offset(pud, addr);
  82                 if (PTRS_PER_PMD != 1)
  83                         printk(", *pmd=%0*Lx", (u32)(sizeof(*pmd) * 2),
  84                                (u64)pmd_val(*pmd));
  85
  86                 if (pmd_none(*pmd))
  87                         break;
  88
  89                 if (pmd_bad(*pmd)) {
  90                         printk("(bad)");
  91                         break;
  92                 }
  93
  94                 /* We must not map this if we have highmem enabled */
  95                 if (PageHighMem(pfn_to_page(pmd_val(*pmd) >> PAGE_SHIFT)))
  96                         break;
  97
  98                 pte = pte_offset_kernel(pmd, addr);
  99                 printk(", *pte=%0*Lx", (u32)(sizeof(*pte) * 2),
 100                        (u64)pte_val(*pte));
 101         } while (0);
 102
 103         printk("\n");
 104 }
 105
 106 static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
 107 {
 108         unsigned index = pgd_index(address);
 109         pgd_t *pgd_k;
 110         pud_t *pud, *pud_k;
 111         pmd_t *pmd, *pmd_k;
 112
 113         pgd += index;
 114         pgd_k = init_mm.pgd + index;
 115
 116         if (!pgd_present(*pgd_k))
 117                 return NULL;
 118
 119         pud = pud_offset(pgd, address);
 120         pud_k = pud_offset(pgd_k, address);
 121         if (!pud_present(*pud_k))
 122                 return NULL;
 123
 124         if (!pud_present(*pud))
 125             set_pud(pud, *pud_k);
 126
 127         pmd = pmd_offset(pud, address);
 128         pmd_k = pmd_offset(pud_k, address);
 129         if (!pmd_present(*pmd_k))
 130                 return NULL;
 131
 132         if (!pmd_present(*pmd))
 133                 set_pmd(pmd, *pmd_k);
 134         else {
 135                 /*
 136                  * The page tables are fully synchronised so there must
 137                  * be another reason for the fault. Return NULL here to
 138                  * signal that we have not taken care of the fault.
 139                  */
 140                 BUG_ON(pmd_page(*pmd) != pmd_page(*pmd_k));
 141                 return NULL;
 142         }
 143
 144         return pmd_k;
 145 }
 146
 147 #ifdef CONFIG_SH_STORE_QUEUES
 148 #define __FAULT_ADDR_LIMIT      P3_ADDR_MAX
 149 #else
 150 #define __FAULT_ADDR_LIMIT      VMALLOC_END
 151 #endif
 152
 153 /*
 154  * Handle a fault on the vmalloc or module mapping area
 155  */
 156 static noinline int vmalloc_fault(unsigned long address)
 157 {
 158         pgd_t *pgd_k;
 159         pmd_t *pmd_k;
 160         pte_t *pte_k;
 161
 162         /* Make sure we are in vmalloc/module/P3 area: */
 163         if (!(address >= VMALLOC_START && address < __FAULT_ADDR_LIMIT))
 164                 return -1;
 165
 166         /*
 167          * Synchronize this task's top level page-table
 168          * with the 'reference' page table.
 169          *
 170          * Do _not_ use "current" here. We might be inside
 171          * an interrupt in the middle of a task switch..
 172          */
 173         pgd_k = get_TTB();
 174         pmd_k = vmalloc_sync_one(pgd_k, address);
 175         if (!pmd_k)
 176                 return -1;
 177
 178         pte_k = pte_offset_kernel(pmd_k, address);
 179         if (!pte_present(*pte_k))
 180                 return -1;
 181
 182         return 0;
 183 }
 184
 185 static void
 186 show_fault_oops(struct pt_regs *regs, unsigned long address)
 187 {
 188         if (!oops_may_print())
 189                 return;
 190
 191         printk(KERN_ALERT "BUG: unable to handle kernel ");
 192         if (address < PAGE_SIZE)
 193                 printk(KERN_CONT "NULL pointer dereference");
 194         else
 195                 printk(KERN_CONT "paging request");
 196
 197         printk(KERN_CONT " at %08lx\n", address);
 198         printk(KERN_ALERT "PC:");
 199         printk_address(regs->pc, 1);
 200
 201         show_pte(NULL, address);
 202 }
 203
 204 static noinline void
 205 no_context(struct pt_regs *regs, unsigned long error_code,
 206            unsigned long address)
 207 {
 208         /* Are we prepared to handle this kernel fault?  */
 209         if (fixup_exception(regs))
 210                 return;
 211
 212         if (handle_trapped_io(regs, address))
 213                 return;
 214
 215         /*
 216          * Oops. The kernel tried to access some bad page. We'll have to
 217          * terminate things with extreme prejudice.
 218          */
 219         bust_spinlocks(1);
 220
 221         show_fault_oops(regs, address);
 222
 223         die("Oops", regs, error_code);
 224         bust_spinlocks(0);
 225         do_exit(SIGKILL);
 226 }
 227
 228 static void
 229 __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
 230                        unsigned long address, int si_code)
 231 {
 232         /* User mode accesses just cause a SIGSEGV */
 233         if (user_mode(regs)) {
 234                 /*
 235                  * It's possible to have interrupts off here:
 236                  */
 237                 local_irq_enable();
 238
 239                 force_sig_info_fault(SIGSEGV, si_code, address);
 240
 241                 return;
 242         }
 243
 244         no_context(regs, error_code, address);
 245 }
 246
 247 static noinline void
 248 bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
 249                      unsigned long address)
 250 {
 251         __bad_area_nosemaphore(regs, error_code, address, SEGV_MAPERR);
 252 }
 253
 254 static void
 255 __bad_area(struct pt_regs *regs, unsigned long error_code,
 256            unsigned long address, int si_code)
 257 {
 258         struct mm_struct *mm = current->mm;
 259
 260         /*
 261          * Something tried to access memory that isn't in our memory map..
 262          * Fix it, but check if it's kernel or user first..
 263          */
 264         up_read(&mm->mmap_sem);
 265
 266         __bad_area_nosemaphore(regs, error_code, address, si_code);
 267 }
 268
 269 static noinline void
 270 bad_area(struct pt_regs *regs, unsigned long error_code, unsigned long address)
 271 {
 272         __bad_area(regs, error_code, address, SEGV_MAPERR);
 273 }
 274
 275 static noinline void
 276 bad_area_access_error(struct pt_regs *regs, unsigned long error_code,
 277                       unsigned long address)
 278 {
 279         __bad_area(regs, error_code, address, SEGV_ACCERR);
 280 }
 281
 282 static void
 283 do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address)
 284 {
 285         struct task_struct *tsk = current;
 286         struct mm_struct *mm = tsk->mm;
 287
 288         up_read(&mm->mmap_sem);
 289
 290         /* Kernel mode? Handle exceptions or die: */
 291         if (!user_mode(regs))
 292                 no_context(regs, error_code, address);
 293
 294         force_sig_info_fault(SIGBUS, BUS_ADRERR, address);
 295 }
 296
 297 static noinline int
 298 mm_fault_error(struct pt_regs *regs, unsigned long error_code,
 299                unsigned long address, vm_fault_t fault)
 300 {
 301         /*
 302          * Pagefault was interrupted by SIGKILL. We have no reason to
 303          * continue pagefault.
 304          */
 305         if (fatal_signal_pending(current)) {
 306                 if (!(fault & VM_FAULT_RETRY))
 307                         up_read(&current->mm->mmap_sem);
 308                 if (!user_mode(regs))
 309                         no_context(regs, error_code, address);
 310                 return 1;
 311         }
 312
 313         if (!(fault & VM_FAULT_ERROR))
 314                 return 0;
 315
 316         if (fault & VM_FAULT_OOM) {
 317                 /* Kernel mode? Handle exceptions or die: */
 318                 if (!user_mode(regs)) {
 319                         up_read(&current->mm->mmap_sem);
 320                         no_context(regs, error_code, address);
 321                         return 1;
 322                 }
 323                 up_read(&current->mm->mmap_sem);
 324
 325                 /*
 326                  * We ran out of memory, call the OOM killer, and return the
 327                  * userspace (which will retry the fault, or kill us if we got
 328                  * oom-killed):
 329                  */
 330                 pagefault_out_of_memory();
 331         } else {
 332                 if (fault & VM_FAULT_SIGBUS)
 333                         do_sigbus(regs, error_code, address);
 334                 else if (fault & VM_FAULT_SIGSEGV)
 335                         bad_area(regs, error_code, address);
 336                 else
 337                         BUG();
 338         }
 339
 340         return 1;
 341 }
 342
 343 static inline int access_error(int error_code, struct vm_area_struct *vma)
 344 {
 345         if (error_code & FAULT_CODE_WRITE) {
 346                 /* write, present and write, not present: */
 347                 if (unlikely(!(vma->vm_flags & VM_WRITE)))
 348                         return 1;
 349                 return 0;
 350         }
 351
 352         /* ITLB miss on NX page */
 353         if (unlikely((error_code & FAULT_CODE_ITLB) &&
 354                      !(vma->vm_flags & VM_EXEC)))
 355                 return 1;
 356
 357         /* read, not present: */
 358         if (unlikely(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))))
 359                 return 1;
 360
 361         return 0;
 362 }
 363
 364 static int fault_in_kernel_space(unsigned long address)
 365 {
 366         return address >= TASK_SIZE;
 367 }
 368
 369 /*
 370  * This routine handles page faults.  It determines the address,
 371  * and the problem, and then passes it off to one of the appropriate
 372  * routines.
 373  */
 374 asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
 375                                         unsigned long error_code,
 376                                         unsigned long address)
 377 {
 378         unsigned long vec;
 379         struct task_struct *tsk;
 380         struct mm_struct *mm;
 381         struct vm_area_struct * vma;
 382         vm_fault_t fault;
 383         unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
 384
 385         tsk = current;
 386         mm = tsk->mm;
 387         vec = lookup_exception_vector();
 388
 389         /*
 390          * We fault-in kernel-space virtual memory on-demand. The
 391          * 'reference' page table is init_mm.pgd.
 392          *
 393          * NOTE! We MUST NOT take any locks for this case. We may
 394          * be in an interrupt or a critical region, and should
 395          * only copy the information from the master page table,
 396          * nothing more.
 397          */
 398         if (unlikely(fault_in_kernel_space(address))) {
 399                 if (vmalloc_fault(address) >= 0)
 400                         return;
 401                 if (kprobe_page_fault(regs, vec))
 402                         return;
 403
 404                 bad_area_nosemaphore(regs, error_code, address);
 405                 return;
 406         }
 407
 408         if (unlikely(kprobe_page_fault(regs, vec)))
 409                 return;
 410
 411         /* Only enable interrupts if they were on before the fault */
 412         if ((regs->sr & SR_IMASK) != SR_IMASK)
 413                 local_irq_enable();
 414
 415         perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
 416
 417         /*
 418          * If we're in an interrupt, have no user context or are running
 419          * with pagefaults disabled then we must not take the fault:
 420          */
 421         if (unlikely(faulthandler_disabled() || !mm)) {
 422                 bad_area_nosemaphore(regs, error_code, address);
 423                 return;
 424         }
 425
 426 retry:
 427         down_read(&mm->mmap_sem);
 428
 429         vma = find_vma(mm, address);
 430         if (unlikely(!vma)) {
 431                 bad_area(regs, error_code, address);
 432                 return;
 433         }
 434         if (likely(vma->vm_start <= address))
 435                 goto good_area;
 436         if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) {
 437                 bad_area(regs, error_code, address);
 438                 return;
 439         }
 440         if (unlikely(expand_stack(vma, address))) {
 441                 bad_area(regs, error_code, address);
 442                 return;
 443         }
 444
 445         /*
 446          * Ok, we have a good vm_area for this memory access, so
 447          * we can handle it..
 448          */
 449 good_area:
 450         if (unlikely(access_error(error_code, vma))) {
 451                 bad_area_access_error(regs, error_code, address);
 452                 return;
 453         }
 454
 455         set_thread_fault_code(error_code);
 456
 457         if (user_mode(regs))
 458                 flags |= FAULT_FLAG_USER;
 459         if (error_code & FAULT_CODE_WRITE)
 460                 flags |= FAULT_FLAG_WRITE;
 461
 462         /*
 463          * If for any reason at all we couldn't handle the fault,
 464          * make sure we exit gracefully rather than endlessly redo
 465          * the fault.
 466          */
 467         fault = handle_mm_fault(vma, address, flags);
 468
 469         if (unlikely(fault & (VM_FAULT_RETRY | VM_FAULT_ERROR)))
 470                 if (mm_fault_error(regs, error_code, address, fault))
 471                         return;
 472
 473         if (flags & FAULT_FLAG_ALLOW_RETRY) {
 474                 if (fault & VM_FAULT_MAJOR) {
 475                         tsk->maj_flt++;
 476                         perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1,
 477                                       regs, address);
 478                 } else {
 479                         tsk->min_flt++;
 480                         perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1,
 481                                       regs, address);
 482                 }
 483                 if (fault & VM_FAULT_RETRY) {
 484                         flags &= ~FAULT_FLAG_ALLOW_RETRY;
 485                         flags |= FAULT_FLAG_TRIED;
 486
 487                         /*
 488                          * No need to up_read(&mm->mmap_sem) as we would
 489                          * have already released it in __lock_page_or_retry
 490                          * in mm/filemap.c.
 491                          */
 492                         goto retry;
 493                 }
 494         }
 495
 496         up_read(&mm->mmap_sem);
 497 }