arch/mips/mm/tlb-r4k.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
   7  * Copyright (C) 1997, 1998, 1999, 2000 Ralf Baechle ralf@gnu.org
   8  * Carsten Langgaard, carstenl@mips.com
   9  * Copyright (C) 2002 MIPS Technologies, Inc.  All rights reserved.
  10  */
  11 #include <linux/init.h>
  12 #include <linux/sched.h>
  13 #include <linux/mm.h>
  14
  15 #include <asm/cpu.h>
  16 #include <asm/bootinfo.h>
  17 #include <asm/mmu_context.h>
  18 #include <asm/pgtable.h>
  19 #include <asm/system.h>
  20
  21 extern void build_tlb_refill_handler(void);
  22
  23 /*
  24  * Make sure all entries differ.  If they're not different
  25  * MIPS32 will take revenge ...
  26  */
  27 #define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1)))
  28
  29 /* CP0 hazard avoidance. */
  30 #define BARRIER __asm__ __volatile__(".set noreorder\n\t" \
  31                                      "nop; nop; nop; nop; nop; nop;\n\t" \
  32                                      ".set reorder\n\t")
  33
  34 /* Atomicity and interruptability */
  35 #ifdef CONFIG_MIPS_MT_SMTC
  36
  37 #include <asm/smtc.h>
  38 #include <asm/mipsmtregs.h>
  39
  40 #define ENTER_CRITICAL(flags) \
  41         { \
  42         unsigned int mvpflags; \
  43         local_irq_save(flags);\
  44         mvpflags = dvpe()
  45 #define EXIT_CRITICAL(flags) \
  46         evpe(mvpflags); \
  47         local_irq_restore(flags); \
  48         }
  49 #else
  50
  51 #define ENTER_CRITICAL(flags) local_irq_save(flags)
  52 #define EXIT_CRITICAL(flags) local_irq_restore(flags)
  53
  54 #endif /* CONFIG_MIPS_MT_SMTC */
  55
  56 void local_flush_tlb_all(void)
  57 {
  58         unsigned long flags;
  59         unsigned long old_ctx;
  60         int entry;
  61
  62         ENTER_CRITICAL(flags);
  63         /* Save old context and create impossible VPN2 value */
  64         old_ctx = read_c0_entryhi();
  65         write_c0_entrylo0(0);
  66         write_c0_entrylo1(0);
  67
  68         entry = read_c0_wired();
  69
  70         /* Blast 'em all away. */
  71         while (entry < current_cpu_data.tlbsize) {
  72                 /* Make sure all entries differ. */
  73                 write_c0_entryhi(UNIQUE_ENTRYHI(entry));
  74                 write_c0_index(entry);
  75                 mtc0_tlbw_hazard();
  76                 tlb_write_indexed();
  77                 entry++;
  78         }
  79         tlbw_use_hazard();
  80         write_c0_entryhi(old_ctx);
  81         EXIT_CRITICAL(flags);
  82 }
  83
  84 /* All entries common to a mm share an asid.  To effectively flush
  85    these entries, we just bump the asid. */
  86 void local_flush_tlb_mm(struct mm_struct *mm)
  87 {
  88         int cpu;
  89
  90         preempt_disable();
  91
  92         cpu = smp_processor_id();
  93
  94         if (cpu_context(cpu, mm) != 0) {
  95                 drop_mmu_context(mm, cpu);
  96         }
  97
  98         preempt_enable();
  99 }
 100
 101 void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 102         unsigned long end)
 103 {
 104         struct mm_struct *mm = vma->vm_mm;
 105         int cpu = smp_processor_id();
 106
 107         if (cpu_context(cpu, mm) != 0) {
 108                 unsigned long flags;
 109                 int size;
 110
 111                 ENTER_CRITICAL(flags);
 112                 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
 113                 size = (size + 1) >> 1;
 114                 local_irq_save(flags);
 115                 if (size <= current_cpu_data.tlbsize/2) {
 116                         int oldpid = read_c0_entryhi();
 117                         int newpid = cpu_asid(cpu, mm);
 118
 119                         start &= (PAGE_MASK << 1);
 120                         end += ((PAGE_SIZE << 1) - 1);
 121                         end &= (PAGE_MASK << 1);
 122                         while (start < end) {
 123                                 int idx;
 124
 125                                 write_c0_entryhi(start | newpid);
 126                                 start += (PAGE_SIZE << 1);
 127                                 mtc0_tlbw_hazard();
 128                                 tlb_probe();
 129                                 BARRIER;
 130                                 idx = read_c0_index();
 131                                 write_c0_entrylo0(0);
 132                                 write_c0_entrylo1(0);
 133                                 if (idx < 0)
 134                                         continue;
 135                                 /* Make sure all entries differ. */
 136                                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 137                                 mtc0_tlbw_hazard();
 138                                 tlb_write_indexed();
 139                         }
 140                         tlbw_use_hazard();
 141                         write_c0_entryhi(oldpid);
 142                 } else {
 143                         drop_mmu_context(mm, cpu);
 144                 }
 145                 EXIT_CRITICAL(flags);
 146         }
 147 }
 148
 149 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
 150 {
 151         unsigned long flags;
 152         int size;
 153
 154         ENTER_CRITICAL(flags);
 155         size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
 156         size = (size + 1) >> 1;
 157         if (size <= current_cpu_data.tlbsize / 2) {
 158                 int pid = read_c0_entryhi();
 159
 160                 start &= (PAGE_MASK << 1);
 161                 end += ((PAGE_SIZE << 1) - 1);
 162                 end &= (PAGE_MASK << 1);
 163
 164                 while (start < end) {
 165                         int idx;
 166
 167                         write_c0_entryhi(start);
 168                         start += (PAGE_SIZE << 1);
 169                         mtc0_tlbw_hazard();
 170                         tlb_probe();
 171                         BARRIER;
 172                         idx = read_c0_index();
 173                         write_c0_entrylo0(0);
 174                         write_c0_entrylo1(0);
 175                         if (idx < 0)
 176                                 continue;
 177                         /* Make sure all entries differ. */
 178                         write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 179                         mtc0_tlbw_hazard();
 180                         tlb_write_indexed();
 181                 }
 182                 tlbw_use_hazard();
 183                 write_c0_entryhi(pid);
 184         } else {
 185                 local_flush_tlb_all();
 186         }
 187         EXIT_CRITICAL(flags);
 188 }
 189
 190 void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
 191 {
 192         int cpu = smp_processor_id();
 193
 194         if (cpu_context(cpu, vma->vm_mm) != 0) {
 195                 unsigned long flags;
 196                 int oldpid, newpid, idx;
 197
 198                 newpid = cpu_asid(cpu, vma->vm_mm);
 199                 page &= (PAGE_MASK << 1);
 200                 ENTER_CRITICAL(flags);
 201                 oldpid = read_c0_entryhi();
 202                 write_c0_entryhi(page | newpid);
 203                 mtc0_tlbw_hazard();
 204                 tlb_probe();
 205                 BARRIER;
 206                 idx = read_c0_index();
 207                 write_c0_entrylo0(0);
 208                 write_c0_entrylo1(0);
 209                 if (idx < 0)
 210                         goto finish;
 211                 /* Make sure all entries differ. */
 212                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 213                 mtc0_tlbw_hazard();
 214                 tlb_write_indexed();
 215                 tlbw_use_hazard();
 216
 217         finish:
 218                 write_c0_entryhi(oldpid);
 219                 EXIT_CRITICAL(flags);
 220         }
 221 }
 222
 223 /*
 224  * This one is only used for pages with the global bit set so we don't care
 225  * much about the ASID.
 226  */
 227 void local_flush_tlb_one(unsigned long page)
 228 {
 229         unsigned long flags;
 230         int oldpid, idx;
 231
 232         ENTER_CRITICAL(flags);
 233         oldpid = read_c0_entryhi();
 234         page &= (PAGE_MASK << 1);
 235         write_c0_entryhi(page);
 236         mtc0_tlbw_hazard();
 237         tlb_probe();
 238         BARRIER;
 239         idx = read_c0_index();
 240         write_c0_entrylo0(0);
 241         write_c0_entrylo1(0);
 242         if (idx >= 0) {
 243                 /* Make sure all entries differ. */
 244                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 245                 mtc0_tlbw_hazard();
 246                 tlb_write_indexed();
 247                 tlbw_use_hazard();
 248         }
 249         write_c0_entryhi(oldpid);
 250
 251         EXIT_CRITICAL(flags);
 252 }
 253
 254 /*
 255  * We will need multiple versions of update_mmu_cache(), one that just
 256  * updates the TLB with the new pte(s), and another which also checks
 257  * for the R4k "end of page" hardware bug and does the needy.
 258  */
 259 void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
 260 {
 261         unsigned long flags;
 262         pgd_t *pgdp;
 263         pud_t *pudp;
 264         pmd_t *pmdp;
 265         pte_t *ptep;
 266         int idx, pid;
 267
 268         /*
 269          * Handle debugger faulting in for debugee.
 270          */
 271         if (current->active_mm != vma->vm_mm)
 272                 return;
 273
 274         ENTER_CRITICAL(flags);
 275
 276         pid = read_c0_entryhi() & ASID_MASK;
 277         address &= (PAGE_MASK << 1);
 278         write_c0_entryhi(address | pid);
 279         pgdp = pgd_offset(vma->vm_mm, address);
 280         mtc0_tlbw_hazard();
 281         tlb_probe();
 282         BARRIER;
 283         pudp = pud_offset(pgdp, address);
 284         pmdp = pmd_offset(pudp, address);
 285         idx = read_c0_index();
 286         ptep = pte_offset_map(pmdp, address);
 287
 288 #if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32_R1)
 289         write_c0_entrylo0(ptep->pte_high);
 290         ptep++;
 291         write_c0_entrylo1(ptep->pte_high);
 292 #else
 293         write_c0_entrylo0(pte_val(*ptep++) >> 6);
 294         write_c0_entrylo1(pte_val(*ptep) >> 6);
 295 #endif
 296         mtc0_tlbw_hazard();
 297         if (idx < 0)
 298                 tlb_write_random();
 299         else
 300                 tlb_write_indexed();
 301         tlbw_use_hazard();
 302         EXIT_CRITICAL(flags);
 303 }
 304
 305 #if 0
 306 static void r4k_update_mmu_cache_hwbug(struct vm_area_struct * vma,
 307                                        unsigned long address, pte_t pte)
 308 {
 309         unsigned long flags;
 310         unsigned int asid;
 311         pgd_t *pgdp;
 312         pmd_t *pmdp;
 313         pte_t *ptep;
 314         int idx;
 315
 316         ENTER_CRITICAL(flags);
 317         address &= (PAGE_MASK << 1);
 318         asid = read_c0_entryhi() & ASID_MASK;
 319         write_c0_entryhi(address | asid);
 320         pgdp = pgd_offset(vma->vm_mm, address);
 321         mtc0_tlbw_hazard();
 322         tlb_probe();
 323         BARRIER;
 324         pmdp = pmd_offset(pgdp, address);
 325         idx = read_c0_index();
 326         ptep = pte_offset_map(pmdp, address);
 327         write_c0_entrylo0(pte_val(*ptep++) >> 6);
 328         write_c0_entrylo1(pte_val(*ptep) >> 6);
 329         mtc0_tlbw_hazard();
 330         if (idx < 0)
 331                 tlb_write_random();
 332         else
 333                 tlb_write_indexed();
 334         tlbw_use_hazard();
 335         EXIT_CRITICAL(flags);
 336 }
 337 #endif
 338
 339 void __init add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
 340         unsigned long entryhi, unsigned long pagemask)
 341 {
 342         unsigned long flags;
 343         unsigned long wired;
 344         unsigned long old_pagemask;
 345         unsigned long old_ctx;
 346
 347         ENTER_CRITICAL(flags);
 348         /* Save old context and create impossible VPN2 value */
 349         old_ctx = read_c0_entryhi();
 350         old_pagemask = read_c0_pagemask();
 351         wired = read_c0_wired();
 352         write_c0_wired(wired + 1);
 353         write_c0_index(wired);
 354         BARRIER;
 355         write_c0_pagemask(pagemask);
 356         write_c0_entryhi(entryhi);
 357         write_c0_entrylo0(entrylo0);
 358         write_c0_entrylo1(entrylo1);
 359         mtc0_tlbw_hazard();
 360         tlb_write_indexed();
 361         tlbw_use_hazard();
 362
 363         write_c0_entryhi(old_ctx);
 364         BARRIER;
 365         write_c0_pagemask(old_pagemask);
 366         local_flush_tlb_all();
 367         EXIT_CRITICAL(flags);
 368 }
 369
 370 /*
 371  * Used for loading TLB entries before trap_init() has started, when we
 372  * don't actually want to add a wired entry which remains throughout the
 373  * lifetime of the system
 374  */
 375
 376 static int temp_tlb_entry __initdata;
 377
 378 __init int add_temporary_entry(unsigned long entrylo0, unsigned long entrylo1,
 379                                unsigned long entryhi, unsigned long pagemask)
 380 {
 381         int ret = 0;
 382         unsigned long flags;
 383         unsigned long wired;
 384         unsigned long old_pagemask;
 385         unsigned long old_ctx;
 386
 387         ENTER_CRITICAL(flags);
 388         /* Save old context and create impossible VPN2 value */
 389         old_ctx = read_c0_entryhi();
 390         old_pagemask = read_c0_pagemask();
 391         wired = read_c0_wired();
 392         if (--temp_tlb_entry < wired) {
 393                 printk(KERN_WARNING
 394                        "No TLB space left for add_temporary_entry\n");
 395                 ret = -ENOSPC;
 396                 goto out;
 397         }
 398
 399         write_c0_index(temp_tlb_entry);
 400         write_c0_pagemask(pagemask);
 401         write_c0_entryhi(entryhi);
 402         write_c0_entrylo0(entrylo0);
 403         write_c0_entrylo1(entrylo1);
 404         mtc0_tlbw_hazard();
 405         tlb_write_indexed();
 406         tlbw_use_hazard();
 407
 408         write_c0_entryhi(old_ctx);
 409         write_c0_pagemask(old_pagemask);
 410 out:
 411         EXIT_CRITICAL(flags);
 412         return ret;
 413 }
 414
 415 static void __init probe_tlb(unsigned long config)
 416 {
 417         struct cpuinfo_mips *c = &current_cpu_data;
 418         unsigned int reg;
 419
 420         /*
 421          * If this isn't a MIPS32 / MIPS64 compliant CPU.  Config 1 register
 422          * is not supported, we assume R4k style.  Cpu probing already figured
 423          * out the number of tlb entries.
 424          */
 425         if ((c->processor_id & 0xff0000) == PRID_COMP_LEGACY)
 426                 return;
 427 #ifdef CONFIG_MIPS_MT_SMTC
 428         /*
 429          * If TLB is shared in SMTC system, total size already
 430          * has been calculated and written into cpu_data tlbsize
 431          */
 432         if((smtc_status & SMTC_TLB_SHARED) == SMTC_TLB_SHARED)
 433                 return;
 434 #endif /* CONFIG_MIPS_MT_SMTC */
 435
 436         reg = read_c0_config1();
 437         if (!((config >> 7) & 3))
 438                 panic("No TLB present");
 439
 440         c->tlbsize = ((reg >> 25) & 0x3f) + 1;
 441 }
 442
 443 static int __initdata ntlb = 0;
 444 static int __init set_ntlb(char *str)
 445 {
 446         get_option(&str, &ntlb);
 447         return 1;
 448 }
 449
 450 __setup("ntlb=", set_ntlb);
 451
 452 void __init tlb_init(void)
 453 {
 454         unsigned int config = read_c0_config();
 455
 456         /*
 457          * You should never change this register:
 458          *   - On R4600 1.7 the tlbp never hits for pages smaller than
 459          *     the value in the c0_pagemask register.
 460          *   - The entire mm handling assumes the c0_pagemask register to
 461          *     be set for 4kb pages.
 462          */
 463         probe_tlb(config);
 464         write_c0_pagemask(PM_DEFAULT_MASK);
 465         write_c0_wired(0);
 466         write_c0_framemask(0);
 467         temp_tlb_entry = current_cpu_data.tlbsize - 1;
 468
 469         /* From this point on the ARC firmware is dead.  */
 470         local_flush_tlb_all();
 471
 472         /* Did I tell you that ARC SUCKS?  */
 473
 474         if (ntlb) {
 475                 if (ntlb > 1 && ntlb <= current_cpu_data.tlbsize) {
 476                         int wired = current_cpu_data.tlbsize - ntlb;
 477                         write_c0_wired(wired);
 478                         write_c0_index(wired-1);
 479                         printk ("Restricting TLB to %d entries\n", ntlb);
 480                 } else
 481                         printk("Ignoring invalid argument ntlb=%d\n", ntlb);
 482         }
 483
 484         build_tlb_refill_handler();
 485 }