arch/sh/mm/tlbflush_64.c

   1 /*
   2  * arch/sh/mm/tlb-flush_64.c
   3  *
   4  * Copyright (C) 2000, 2001  Paolo Alberelli
   5  * Copyright (C) 2003  Richard Curnow (/proc/tlb, bug fixes)
   6  * Copyright (C) 2003  Paul Mundt
   7  *
   8  * This file is subject to the terms and conditions of the GNU General Public
   9  * License.  See the file "COPYING" in the main directory of this archive
  10  * for more details.
  11  */
  12 #include <linux/signal.h>
  13 #include <linux/rwsem.h>
  14 #include <linux/sched.h>
  15 #include <linux/kernel.h>
  16 #include <linux/errno.h>
  17 #include <linux/string.h>
  18 #include <linux/types.h>
  19 #include <linux/ptrace.h>
  20 #include <linux/mman.h>
  21 #include <linux/mm.h>
  22 #include <linux/smp.h>
  23 #include <linux/interrupt.h>
  24 #include <asm/system.h>
  25 #include <asm/io.h>
  26 #include <asm/tlb.h>
  27 #include <asm/uaccess.h>
  28 #include <asm/pgalloc.h>
  29 #include <asm/mmu_context.h>
  30
  31 extern void die(const char *,struct pt_regs *,long);
  32
  33 #define PFLAG(val,flag)   (( (val) & (flag) ) ? #flag : "" )
  34 #define PPROT(flag) PFLAG(pgprot_val(prot),flag)
  35
  36 static inline void print_prots(pgprot_t prot)
  37 {
  38         printk("prot is 0x%08lx\n",pgprot_val(prot));
  39
  40         printk("%s %s %s %s %s\n",PPROT(_PAGE_SHARED),PPROT(_PAGE_READ),
  41                PPROT(_PAGE_EXECUTE),PPROT(_PAGE_WRITE),PPROT(_PAGE_USER));
  42 }
  43
  44 static inline void print_vma(struct vm_area_struct *vma)
  45 {
  46         printk("vma start 0x%08lx\n", vma->vm_start);
  47         printk("vma end   0x%08lx\n", vma->vm_end);
  48
  49         print_prots(vma->vm_page_prot);
  50         printk("vm_flags 0x%08lx\n", vma->vm_flags);
  51 }
  52
  53 static inline void print_task(struct task_struct *tsk)
  54 {
  55         printk("Task pid %d\n", task_pid_nr(tsk));
  56 }
  57
  58 static pte_t *lookup_pte(struct mm_struct *mm, unsigned long address)
  59 {
  60         pgd_t *dir;
  61         pud_t *pud;
  62         pmd_t *pmd;
  63         pte_t *pte;
  64         pte_t entry;
  65
  66         dir = pgd_offset(mm, address);
  67         if (pgd_none(*dir))
  68                 return NULL;
  69
  70         pud = pud_offset(dir, address);
  71         if (pud_none(*pud))
  72                 return NULL;
  73
  74         pmd = pmd_offset(pud, address);
  75         if (pmd_none(*pmd))
  76                 return NULL;
  77
  78         pte = pte_offset_kernel(pmd, address);
  79         entry = *pte;
  80         if (pte_none(entry) || !pte_present(entry))
  81                 return NULL;
  82
  83         return pte;
  84 }
  85
  86 /*
  87  * This routine handles page faults.  It determines the address,
  88  * and the problem, and then passes it off to one of the appropriate
  89  * routines.
  90  */
  91 asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long writeaccess,
  92                               unsigned long textaccess, unsigned long address)
  93 {
  94         struct task_struct *tsk;
  95         struct mm_struct *mm;
  96         struct vm_area_struct * vma;
  97         const struct exception_table_entry *fixup;
  98         pte_t *pte;
  99         int fault;
 100
 101         /* SIM
 102          * Note this is now called with interrupts still disabled
 103          * This is to cope with being called for a missing IO port
 104          * address with interrupts disabled. This should be fixed as
 105          * soon as we have a better 'fast path' miss handler.
 106          *
 107          * Plus take care how you try and debug this stuff.
 108          * For example, writing debug data to a port which you
 109          * have just faulted on is not going to work.
 110          */
 111
 112         tsk = current;
 113         mm = tsk->mm;
 114
 115         /* Not an IO address, so reenable interrupts */
 116         local_irq_enable();
 117
 118         /*
 119          * If we're in an interrupt or have no user
 120          * context, we must not take the fault..
 121          */
 122         if (in_atomic() || !mm)
 123                 goto no_context;
 124
 125         /* TLB misses upon some cache flushes get done under cli() */
 126         down_read(&mm->mmap_sem);
 127
 128         vma = find_vma(mm, address);
 129
 130         if (!vma) {
 131 #ifdef DEBUG_FAULT
 132                 print_task(tsk);
 133                 printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
 134 <<<<<<< HEAD:arch/sh/mm/tlbflush_64.c
 135                        __FUNCTION__,__LINE__,
 136 =======
 137                        __func__, __LINE__,
 138 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:arch/sh/mm/tlbflush_64.c
 139                        address,regs->pc,textaccess,writeaccess);
 140                 show_regs(regs);
 141 #endif
 142                 goto bad_area;
 143         }
 144         if (vma->vm_start <= address) {
 145                 goto good_area;
 146         }
 147
 148         if (!(vma->vm_flags & VM_GROWSDOWN)) {
 149 #ifdef DEBUG_FAULT
 150                 print_task(tsk);
 151                 printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
 152 <<<<<<< HEAD:arch/sh/mm/tlbflush_64.c
 153                        __FUNCTION__,__LINE__,
 154 =======
 155                        __func__, __LINE__,
 156 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:arch/sh/mm/tlbflush_64.c
 157                        address,regs->pc,textaccess,writeaccess);
 158                 show_regs(regs);
 159
 160                 print_vma(vma);
 161 #endif
 162                 goto bad_area;
 163         }
 164         if (expand_stack(vma, address)) {
 165 #ifdef DEBUG_FAULT
 166                 print_task(tsk);
 167                 printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
 168 <<<<<<< HEAD:arch/sh/mm/tlbflush_64.c
 169                        __FUNCTION__,__LINE__,
 170 =======
 171                        __func__, __LINE__,
 172 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:arch/sh/mm/tlbflush_64.c
 173                        address,regs->pc,textaccess,writeaccess);
 174                 show_regs(regs);
 175 #endif
 176                 goto bad_area;
 177         }
 178 /*
 179  * Ok, we have a good vm_area for this memory access, so
 180  * we can handle it..
 181  */
 182 good_area:
 183         if (textaccess) {
 184                 if (!(vma->vm_flags & VM_EXEC))
 185                         goto bad_area;
 186         } else {
 187                 if (writeaccess) {
 188                         if (!(vma->vm_flags & VM_WRITE))
 189                                 goto bad_area;
 190                 } else {
 191                         if (!(vma->vm_flags & VM_READ))
 192                                 goto bad_area;
 193                 }
 194         }
 195
 196         /*
 197          * If for any reason at all we couldn't handle the fault,
 198          * make sure we exit gracefully rather than endlessly redo
 199          * the fault.
 200          */
 201 survive:
 202         fault = handle_mm_fault(mm, vma, address, writeaccess);
 203         if (unlikely(fault & VM_FAULT_ERROR)) {
 204                 if (fault & VM_FAULT_OOM)
 205                         goto out_of_memory;
 206                 else if (fault & VM_FAULT_SIGBUS)
 207                         goto do_sigbus;
 208                 BUG();
 209         }
 210         if (fault & VM_FAULT_MAJOR)
 211                 tsk->maj_flt++;
 212         else
 213                 tsk->min_flt++;
 214
 215         /* If we get here, the page fault has been handled.  Do the TLB refill
 216            now from the newly-setup PTE, to avoid having to fault again right
 217            away on the same instruction. */
 218         pte = lookup_pte (mm, address);
 219         if (!pte) {
 220                 /* From empirical evidence, we can get here, due to
 221                    !pte_present(pte).  (e.g. if a swap-in occurs, and the page
 222                    is swapped back out again before the process that wanted it
 223                    gets rescheduled?) */
 224                 goto no_pte;
 225         }
 226
 227         __do_tlb_refill(address, textaccess, pte);
 228
 229 no_pte:
 230
 231         up_read(&mm->mmap_sem);
 232         return;
 233
 234 /*
 235  * Something tried to access memory that isn't in our memory map..
 236  * Fix it, but check if it's kernel or user first..
 237  */
 238 bad_area:
 239 #ifdef DEBUG_FAULT
 240         printk("fault:bad area\n");
 241 #endif
 242         up_read(&mm->mmap_sem);
 243
 244         if (user_mode(regs)) {
 245                 static int count=0;
 246                 siginfo_t info;
 247                 if (count < 4) {
 248                         /* This is really to help debug faults when starting
 249                          * usermode, so only need a few */
 250                         count++;
 251                         printk("user mode bad_area address=%08lx pid=%d (%s) pc=%08lx\n",
 252                                 address, task_pid_nr(current), current->comm,
 253                                 (unsigned long) regs->pc);
 254 #if 0
 255                         show_regs(regs);
 256 #endif
 257                 }
 258                 if (is_global_init(tsk)) {
 259                         panic("INIT had user mode bad_area\n");
 260                 }
 261                 tsk->thread.address = address;
 262                 tsk->thread.error_code = writeaccess;
 263                 info.si_signo = SIGSEGV;
 264                 info.si_errno = 0;
 265                 info.si_addr = (void *) address;
 266                 force_sig_info(SIGSEGV, &info, tsk);
 267                 return;
 268         }
 269
 270 no_context:
 271 #ifdef DEBUG_FAULT
 272         printk("fault:No context\n");
 273 #endif
 274         /* Are we prepared to handle this kernel fault?  */
 275         fixup = search_exception_tables(regs->pc);
 276         if (fixup) {
 277                 regs->pc = fixup->fixup;
 278                 return;
 279         }
 280
 281 /*
 282  * Oops. The kernel tried to access some bad page. We'll have to
 283  * terminate things with extreme prejudice.
 284  *
 285  */
 286         if (address < PAGE_SIZE)
 287                 printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
 288         else
 289                 printk(KERN_ALERT "Unable to handle kernel paging request");
 290         printk(" at virtual address %08lx\n", address);
 291         printk(KERN_ALERT "pc = %08Lx%08Lx\n", regs->pc >> 32, regs->pc & 0xffffffff);
 292         die("Oops", regs, writeaccess);
 293         do_exit(SIGKILL);
 294
 295 /*
 296  * We ran out of memory, or some other thing happened to us that made
 297  * us unable to handle the page fault gracefully.
 298  */
 299 out_of_memory:
 300         if (is_global_init(current)) {
 301                 panic("INIT out of memory\n");
 302                 yield();
 303                 goto survive;
 304         }
 305         printk("fault:Out of memory\n");
 306         up_read(&mm->mmap_sem);
 307         if (is_global_init(current)) {
 308                 yield();
 309                 down_read(&mm->mmap_sem);
 310                 goto survive;
 311         }
 312         printk("VM: killing process %s\n", tsk->comm);
 313         if (user_mode(regs))
 314                 do_group_exit(SIGKILL);
 315         goto no_context;
 316
 317 do_sigbus:
 318         printk("fault:Do sigbus\n");
 319         up_read(&mm->mmap_sem);
 320
 321         /*
 322          * Send a sigbus, regardless of whether we were in kernel
 323          * or user mode.
 324          */
 325         tsk->thread.address = address;
 326         tsk->thread.error_code = writeaccess;
 327         tsk->thread.trap_no = 14;
 328         force_sig(SIGBUS, tsk);
 329
 330         /* Kernel mode? Handle exceptions or die */
 331         if (!user_mode(regs))
 332                 goto no_context;
 333 }
 334
 335 void update_mmu_cache(struct vm_area_struct * vma,
 336                         unsigned long address, pte_t pte)
 337 {
 338         /*
 339          * This appears to get called once for every pte entry that gets
 340          * established => I don't think it's efficient to try refilling the
 341          * TLBs with the pages - some may not get accessed even.  Also, for
 342          * executable pages, it is impossible to determine reliably here which
 343          * TLB they should be mapped into (or both even).
 344          *
 345          * So, just do nothing here and handle faults on demand.  In the
 346          * TLBMISS handling case, the refill is now done anyway after the pte
 347          * has been fixed up, so that deals with most useful cases.
 348          */
 349 }
 350
 351 void local_flush_tlb_one(unsigned long asid, unsigned long page)
 352 {
 353         unsigned long long match, pteh=0, lpage;
 354         unsigned long tlb;
 355
 356         /*
 357          * Sign-extend based on neff.
 358          */
 359         lpage = (page & NEFF_SIGN) ? (page | NEFF_MASK) : page;
 360         match = (asid << PTEH_ASID_SHIFT) | PTEH_VALID;
 361         match |= lpage;
 362
 363         for_each_itlb_entry(tlb) {
 364                 asm volatile ("getcfg   %1, 0, %0"
 365                               : "=r" (pteh)
 366                               : "r" (tlb) );
 367
 368                 if (pteh == match) {
 369                         __flush_tlb_slot(tlb);
 370                         break;
 371                 }
 372         }
 373
 374         for_each_dtlb_entry(tlb) {
 375                 asm volatile ("getcfg   %1, 0, %0"
 376                               : "=r" (pteh)
 377                               : "r" (tlb) );
 378
 379                 if (pteh == match) {
 380                         __flush_tlb_slot(tlb);
 381                         break;
 382                 }
 383
 384         }
 385 }
 386
 387 void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
 388 {
 389         unsigned long flags;
 390
 391         if (vma->vm_mm) {
 392                 page &= PAGE_MASK;
 393                 local_irq_save(flags);
 394                 local_flush_tlb_one(get_asid(), page);
 395                 local_irq_restore(flags);
 396         }
 397 }
 398
 399 void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 400                            unsigned long end)
 401 {
 402         unsigned long flags;
 403         unsigned long long match, pteh=0, pteh_epn, pteh_low;
 404         unsigned long tlb;
 405         unsigned int cpu = smp_processor_id();
 406         struct mm_struct *mm;
 407
 408         mm = vma->vm_mm;
 409         if (cpu_context(cpu, mm) == NO_CONTEXT)
 410                 return;
 411
 412         local_irq_save(flags);
 413
 414         start &= PAGE_MASK;
 415         end &= PAGE_MASK;
 416
 417         match = (cpu_asid(cpu, mm) << PTEH_ASID_SHIFT) | PTEH_VALID;
 418
 419         /* Flush ITLB */
 420         for_each_itlb_entry(tlb) {
 421                 asm volatile ("getcfg   %1, 0, %0"
 422                               : "=r" (pteh)
 423                               : "r" (tlb) );
 424
 425                 pteh_epn = pteh & PAGE_MASK;
 426                 pteh_low = pteh & ~PAGE_MASK;
 427
 428                 if (pteh_low == match && pteh_epn >= start && pteh_epn <= end)
 429                         __flush_tlb_slot(tlb);
 430         }
 431
 432         /* Flush DTLB */
 433         for_each_dtlb_entry(tlb) {
 434                 asm volatile ("getcfg   %1, 0, %0"
 435                               : "=r" (pteh)
 436                               : "r" (tlb) );
 437
 438                 pteh_epn = pteh & PAGE_MASK;
 439                 pteh_low = pteh & ~PAGE_MASK;
 440
 441                 if (pteh_low == match && pteh_epn >= start && pteh_epn <= end)
 442                         __flush_tlb_slot(tlb);
 443         }
 444
 445         local_irq_restore(flags);
 446 }
 447
 448 void local_flush_tlb_mm(struct mm_struct *mm)
 449 {
 450         unsigned long flags;
 451         unsigned int cpu = smp_processor_id();
 452
 453         if (cpu_context(cpu, mm) == NO_CONTEXT)
 454                 return;
 455
 456         local_irq_save(flags);
 457
 458         cpu_context(cpu, mm) = NO_CONTEXT;
 459         if (mm == current->mm)
 460                 activate_context(mm, cpu);
 461
 462         local_irq_restore(flags);
 463 }
 464
 465 void local_flush_tlb_all(void)
 466 {
 467         /* Invalidate all, including shared pages, excluding fixed TLBs */
 468         unsigned long flags, tlb;
 469
 470         local_irq_save(flags);
 471
 472         /* Flush each ITLB entry */
 473         for_each_itlb_entry(tlb)
 474                 __flush_tlb_slot(tlb);
 475
 476         /* Flush each DTLB entry */
 477         for_each_dtlb_entry(tlb)
 478                 __flush_tlb_slot(tlb);
 479
 480         local_irq_restore(flags);
 481 }
 482
 483 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
 484 {
 485         /* FIXME: Optimize this later.. */
 486         flush_tlb_all();
 487 }