arch/powerpc/mm/book3s64/radix_pgtable.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Page table handling routines for radix page table.
   4  *
   5  * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation.
   6  */
   7
   8 #define pr_fmt(fmt) "radix-mmu: " fmt
   9
  10 #include <linux/io.h>
  11 #include <linux/kernel.h>
  12 #include <linux/sched/mm.h>
  13 #include <linux/memblock.h>
  14 #include <linux/of_fdt.h>
  15 #include <linux/mm.h>
  16 #include <linux/hugetlb.h>
  17 #include <linux/string_helpers.h>
  18 #include <linux/memory.h>
  19
  20 #include <asm/pgalloc.h>
  21 #include <asm/mmu_context.h>
  22 #include <asm/dma.h>
  23 #include <asm/machdep.h>
  24 #include <asm/mmu.h>
  25 #include <asm/firmware.h>
  26 #include <asm/powernv.h>
  27 #include <asm/sections.h>
  28 #include <asm/smp.h>
  29 #include <asm/trace.h>
  30 #include <asm/uaccess.h>
  31 #include <asm/ultravisor.h>
  32
  33 #include <trace/events/thp.h>
  34
  35 unsigned int mmu_pid_bits;
  36 unsigned int mmu_base_pid;
  37 unsigned long radix_mem_block_size __ro_after_init;
  38
  39 static __ref void *early_alloc_pgtable(unsigned long size, int nid,
  40                         unsigned long region_start, unsigned long region_end)
  41 {
  42         phys_addr_t min_addr = MEMBLOCK_LOW_LIMIT;
  43         phys_addr_t max_addr = MEMBLOCK_ALLOC_ANYWHERE;
  44         void *ptr;
  45
  46         if (region_start)
  47                 min_addr = region_start;
  48         if (region_end)
  49                 max_addr = region_end;
  50
  51         ptr = memblock_alloc_try_nid(size, size, min_addr, max_addr, nid);
  52
  53         if (!ptr)
  54                 panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa max_addr=%pa\n",
  55                       __func__, size, size, nid, &min_addr, &max_addr);
  56
  57         return ptr;
  58 }
  59
  60 /*
  61  * When allocating pud or pmd pointers, we allocate a complete page
  62  * of PAGE_SIZE rather than PUD_TABLE_SIZE or PMD_TABLE_SIZE. This
  63  * is to ensure that the page obtained from the memblock allocator
  64  * can be completely used as page table page and can be freed
  65  * correctly when the page table entries are removed.
  66  */
  67 static int early_map_kernel_page(unsigned long ea, unsigned long pa,
  68                           pgprot_t flags,
  69                           unsigned int map_page_size,
  70                           int nid,
  71                           unsigned long region_start, unsigned long region_end)
  72 {
  73         unsigned long pfn = pa >> PAGE_SHIFT;
  74         pgd_t *pgdp;
  75         p4d_t *p4dp;
  76         pud_t *pudp;
  77         pmd_t *pmdp;
  78         pte_t *ptep;
  79
  80         pgdp = pgd_offset_k(ea);
  81         p4dp = p4d_offset(pgdp, ea);
  82         if (p4d_none(*p4dp)) {
  83                 pudp = early_alloc_pgtable(PAGE_SIZE, nid,
  84                                            region_start, region_end);
  85                 p4d_populate(&init_mm, p4dp, pudp);
  86         }
  87         pudp = pud_offset(p4dp, ea);
  88         if (map_page_size == PUD_SIZE) {
  89                 ptep = (pte_t *)pudp;
  90                 goto set_the_pte;
  91         }
  92         if (pud_none(*pudp)) {
  93                 pmdp = early_alloc_pgtable(PAGE_SIZE, nid, region_start,
  94                                            region_end);
  95                 pud_populate(&init_mm, pudp, pmdp);
  96         }
  97         pmdp = pmd_offset(pudp, ea);
  98         if (map_page_size == PMD_SIZE) {
  99                 ptep = pmdp_ptep(pmdp);
 100                 goto set_the_pte;
 101         }
 102         if (!pmd_present(*pmdp)) {
 103                 ptep = early_alloc_pgtable(PAGE_SIZE, nid,
 104                                                 region_start, region_end);
 105                 pmd_populate_kernel(&init_mm, pmdp, ptep);
 106         }
 107         ptep = pte_offset_kernel(pmdp, ea);
 108
 109 set_the_pte:
 110         set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
 111         smp_wmb();
 112         return 0;
 113 }
 114
 115 /*
 116  * nid, region_start, and region_end are hints to try to place the page
 117  * table memory in the same node or region.
 118  */
 119 static int __map_kernel_page(unsigned long ea, unsigned long pa,
 120                           pgprot_t flags,
 121                           unsigned int map_page_size,
 122                           int nid,
 123                           unsigned long region_start, unsigned long region_end)
 124 {
 125         unsigned long pfn = pa >> PAGE_SHIFT;
 126         pgd_t *pgdp;
 127         p4d_t *p4dp;
 128         pud_t *pudp;
 129         pmd_t *pmdp;
 130         pte_t *ptep;
 131         /*
 132          * Make sure task size is correct as per the max adddr
 133          */
 134         BUILD_BUG_ON(TASK_SIZE_USER64 > RADIX_PGTABLE_RANGE);
 135
 136 #ifdef CONFIG_PPC_64K_PAGES
 137         BUILD_BUG_ON(RADIX_KERN_MAP_SIZE != (1UL << MAX_EA_BITS_PER_CONTEXT));
 138 #endif
 139
 140         if (unlikely(!slab_is_available()))
 141                 return early_map_kernel_page(ea, pa, flags, map_page_size,
 142                                                 nid, region_start, region_end);
 143
 144         /*
 145          * Should make page table allocation functions be able to take a
 146          * node, so we can place kernel page tables on the right nodes after
 147          * boot.
 148          */
 149         pgdp = pgd_offset_k(ea);
 150         p4dp = p4d_offset(pgdp, ea);
 151         pudp = pud_alloc(&init_mm, p4dp, ea);
 152         if (!pudp)
 153                 return -ENOMEM;
 154         if (map_page_size == PUD_SIZE) {
 155                 ptep = (pte_t *)pudp;
 156                 goto set_the_pte;
 157         }
 158         pmdp = pmd_alloc(&init_mm, pudp, ea);
 159         if (!pmdp)
 160                 return -ENOMEM;
 161         if (map_page_size == PMD_SIZE) {
 162                 ptep = pmdp_ptep(pmdp);
 163                 goto set_the_pte;
 164         }
 165         ptep = pte_alloc_kernel(pmdp, ea);
 166         if (!ptep)
 167                 return -ENOMEM;
 168
 169 set_the_pte:
 170         set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
 171         smp_wmb();
 172         return 0;
 173 }
 174
 175 int radix__map_kernel_page(unsigned long ea, unsigned long pa,
 176                           pgprot_t flags,
 177                           unsigned int map_page_size)
 178 {
 179         return __map_kernel_page(ea, pa, flags, map_page_size, -1, 0, 0);
 180 }
 181
 182 #ifdef CONFIG_STRICT_KERNEL_RWX
 183 void radix__change_memory_range(unsigned long start, unsigned long end,
 184                                 unsigned long clear)
 185 {
 186         unsigned long idx;
 187         pgd_t *pgdp;
 188         p4d_t *p4dp;
 189         pud_t *pudp;
 190         pmd_t *pmdp;
 191         pte_t *ptep;
 192
 193         start = ALIGN_DOWN(start, PAGE_SIZE);
 194         end = PAGE_ALIGN(end); // aligns up
 195
 196         pr_debug("Changing flags on range %lx-%lx removing 0x%lx\n",
 197                  start, end, clear);
 198
 199         for (idx = start; idx < end; idx += PAGE_SIZE) {
 200                 pgdp = pgd_offset_k(idx);
 201                 p4dp = p4d_offset(pgdp, idx);
 202                 pudp = pud_alloc(&init_mm, p4dp, idx);
 203                 if (!pudp)
 204                         continue;
 205                 if (pud_is_leaf(*pudp)) {
 206                         ptep = (pte_t *)pudp;
 207                         goto update_the_pte;
 208                 }
 209                 pmdp = pmd_alloc(&init_mm, pudp, idx);
 210                 if (!pmdp)
 211                         continue;
 212                 if (pmd_is_leaf(*pmdp)) {
 213                         ptep = pmdp_ptep(pmdp);
 214                         goto update_the_pte;
 215                 }
 216                 ptep = pte_alloc_kernel(pmdp, idx);
 217                 if (!ptep)
 218                         continue;
 219 update_the_pte:
 220                 radix__pte_update(&init_mm, idx, ptep, clear, 0, 0);
 221         }
 222
 223         radix__flush_tlb_kernel_range(start, end);
 224 }
 225
 226 void radix__mark_rodata_ro(void)
 227 {
 228         unsigned long start, end;
 229
 230         start = (unsigned long)_stext;
 231         end = (unsigned long)__init_begin;
 232
 233         radix__change_memory_range(start, end, _PAGE_WRITE);
 234 }
 235
 236 void radix__mark_initmem_nx(void)
 237 {
 238         unsigned long start = (unsigned long)__init_begin;
 239         unsigned long end = (unsigned long)__init_end;
 240
 241         radix__change_memory_range(start, end, _PAGE_EXEC);
 242 }
 243 #endif /* CONFIG_STRICT_KERNEL_RWX */
 244
 245 static inline void __meminit
 246 print_mapping(unsigned long start, unsigned long end, unsigned long size, bool exec)
 247 {
 248         char buf[10];
 249
 250         if (end <= start)
 251                 return;
 252
 253         string_get_size(size, 1, STRING_UNITS_2, buf, sizeof(buf));
 254
 255         pr_info("Mapped 0x%016lx-0x%016lx with %s pages%s\n", start, end, buf,
 256                 exec ? " (exec)" : "");
 257 }
 258
 259 static unsigned long next_boundary(unsigned long addr, unsigned long end)
 260 {
 261 #ifdef CONFIG_STRICT_KERNEL_RWX
 262         if (addr < __pa_symbol(__init_begin))
 263                 return __pa_symbol(__init_begin);
 264 #endif
 265         return end;
 266 }
 267
 268 static int __meminit create_physical_mapping(unsigned long start,
 269                                              unsigned long end,
 270                                              unsigned long max_mapping_size,
 271                                              int nid, pgprot_t _prot)
 272 {
 273         unsigned long vaddr, addr, mapping_size = 0;
 274         bool prev_exec, exec = false;
 275         pgprot_t prot;
 276         int psize;
 277
 278         start = ALIGN(start, PAGE_SIZE);
 279         end   = ALIGN_DOWN(end, PAGE_SIZE);
 280         for (addr = start; addr < end; addr += mapping_size) {
 281                 unsigned long gap, previous_size;
 282                 int rc;
 283
 284                 gap = next_boundary(addr, end) - addr;
 285                 if (gap > max_mapping_size)
 286                         gap = max_mapping_size;
 287                 previous_size = mapping_size;
 288                 prev_exec = exec;
 289
 290                 if (IS_ALIGNED(addr, PUD_SIZE) && gap >= PUD_SIZE &&
 291                     mmu_psize_defs[MMU_PAGE_1G].shift) {
 292                         mapping_size = PUD_SIZE;
 293                         psize = MMU_PAGE_1G;
 294                 } else if (IS_ALIGNED(addr, PMD_SIZE) && gap >= PMD_SIZE &&
 295                            mmu_psize_defs[MMU_PAGE_2M].shift) {
 296                         mapping_size = PMD_SIZE;
 297                         psize = MMU_PAGE_2M;
 298                 } else {
 299                         mapping_size = PAGE_SIZE;
 300                         psize = mmu_virtual_psize;
 301                 }
 302
 303                 vaddr = (unsigned long)__va(addr);
 304
 305                 if (overlaps_kernel_text(vaddr, vaddr + mapping_size) ||
 306                     overlaps_interrupt_vector_text(vaddr, vaddr + mapping_size)) {
 307                         prot = PAGE_KERNEL_X;
 308                         exec = true;
 309                 } else {
 310                         prot = _prot;
 311                         exec = false;
 312                 }
 313
 314                 if (mapping_size != previous_size || exec != prev_exec) {
 315                         print_mapping(start, addr, previous_size, prev_exec);
 316                         start = addr;
 317                 }
 318
 319                 rc = __map_kernel_page(vaddr, addr, prot, mapping_size, nid, start, end);
 320                 if (rc)
 321                         return rc;
 322
 323                 update_page_count(psize, 1);
 324         }
 325
 326         print_mapping(start, addr, mapping_size, exec);
 327         return 0;
 328 }
 329
 330 static void __init radix_init_pgtable(void)
 331 {
 332         unsigned long rts_field;
 333         phys_addr_t start, end;
 334         u64 i;
 335
 336         /* We don't support slb for radix */
 337         mmu_slb_size = 0;
 338
 339         /*
 340          * Create the linear mapping
 341          */
 342         for_each_mem_range(i, &start, &end) {
 343                 /*
 344                  * The memblock allocator  is up at this point, so the
 345                  * page tables will be allocated within the range. No
 346                  * need or a node (which we don't have yet).
 347                  */
 348
 349                 if (end >= RADIX_VMALLOC_START) {
 350                         pr_warn("Outside the supported range\n");
 351                         continue;
 352                 }
 353
 354                 WARN_ON(create_physical_mapping(start, end,
 355                                                 radix_mem_block_size,
 356                                                 -1, PAGE_KERNEL));
 357         }
 358
 359         /* Find out how many PID bits are supported */
 360         if (!cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) {
 361                 if (!mmu_pid_bits)
 362                         mmu_pid_bits = 20;
 363                 mmu_base_pid = 1;
 364         } else if (cpu_has_feature(CPU_FTR_HVMODE)) {
 365                 if (!mmu_pid_bits)
 366                         mmu_pid_bits = 20;
 367 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
 368                 /*
 369                  * When KVM is possible, we only use the top half of the
 370                  * PID space to avoid collisions between host and guest PIDs
 371                  * which can cause problems due to prefetch when exiting the
 372                  * guest with AIL=3
 373                  */
 374                 mmu_base_pid = 1 << (mmu_pid_bits - 1);
 375 #else
 376                 mmu_base_pid = 1;
 377 #endif
 378         } else {
 379                 /* The guest uses the bottom half of the PID space */
 380                 if (!mmu_pid_bits)
 381                         mmu_pid_bits = 19;
 382                 mmu_base_pid = 1;
 383         }
 384
 385         /*
 386          * Allocate Partition table and process table for the
 387          * host.
 388          */
 389         BUG_ON(PRTB_SIZE_SHIFT > 36);
 390         process_tb = early_alloc_pgtable(1UL << PRTB_SIZE_SHIFT, -1, 0, 0);
 391         /*
 392          * Fill in the process table.
 393          */
 394         rts_field = radix__get_tree_size();
 395         process_tb->prtb0 = cpu_to_be64(rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE);
 396
 397         /*
 398          * The init_mm context is given the first available (non-zero) PID,
 399          * which is the "guard PID" and contains no page table. PIDR should
 400          * never be set to zero because that duplicates the kernel address
 401          * space at the 0x0... offset (quadrant 0)!
 402          *
 403          * An arbitrary PID that may later be allocated by the PID allocator
 404          * for userspace processes must not be used either, because that
 405          * would cause stale user mappings for that PID on CPUs outside of
 406          * the TLB invalidation scheme (because it won't be in mm_cpumask).
 407          *
 408          * So permanently carve out one PID for the purpose of a guard PID.
 409          */
 410         init_mm.context.id = mmu_base_pid;
 411         mmu_base_pid++;
 412 }
 413
 414 static void __init radix_init_partition_table(void)
 415 {
 416         unsigned long rts_field, dw0, dw1;
 417
 418         mmu_partition_table_init();
 419         rts_field = radix__get_tree_size();
 420         dw0 = rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE | PATB_HR;
 421         dw1 = __pa(process_tb) | (PRTB_SIZE_SHIFT - 12) | PATB_GR;
 422         mmu_partition_table_set_entry(0, dw0, dw1, false);
 423
 424         pr_info("Initializing Radix MMU\n");
 425 }
 426
 427 static int __init get_idx_from_shift(unsigned int shift)
 428 {
 429         int idx = -1;
 430
 431         switch (shift) {
 432         case 0xc:
 433                 idx = MMU_PAGE_4K;
 434                 break;
 435         case 0x10:
 436                 idx = MMU_PAGE_64K;
 437                 break;
 438         case 0x15:
 439                 idx = MMU_PAGE_2M;
 440                 break;
 441         case 0x1e:
 442                 idx = MMU_PAGE_1G;
 443                 break;
 444         }
 445         return idx;
 446 }
 447
 448 static int __init radix_dt_scan_page_sizes(unsigned long node,
 449                                            const char *uname, int depth,
 450                                            void *data)
 451 {
 452         int size = 0;
 453         int shift, idx;
 454         unsigned int ap;
 455         const __be32 *prop;
 456         const char *type = of_get_flat_dt_prop(node, "device_type", NULL);
 457
 458         /* We are scanning "cpu" nodes only */
 459         if (type == NULL || strcmp(type, "cpu") != 0)
 460                 return 0;
 461
 462         /* Find MMU PID size */
 463         prop = of_get_flat_dt_prop(node, "ibm,mmu-pid-bits", &size);
 464         if (prop && size == 4)
 465                 mmu_pid_bits = be32_to_cpup(prop);
 466
 467         /* Grab page size encodings */
 468         prop = of_get_flat_dt_prop(node, "ibm,processor-radix-AP-encodings", &size);
 469         if (!prop)
 470                 return 0;
 471
 472         pr_info("Page sizes from device-tree:\n");
 473         for (; size >= 4; size -= 4, ++prop) {
 474
 475                 struct mmu_psize_def *def;
 476
 477                 /* top 3 bit is AP encoding */
 478                 shift = be32_to_cpu(prop[0]) & ~(0xe << 28);
 479                 ap = be32_to_cpu(prop[0]) >> 29;
 480                 pr_info("Page size shift = %d AP=0x%x\n", shift, ap);
 481
 482                 idx = get_idx_from_shift(shift);
 483                 if (idx < 0)
 484                         continue;
 485
 486                 def = &mmu_psize_defs[idx];
 487                 def->shift = shift;
 488                 def->ap  = ap;
 489         }
 490
 491         /* needed ? */
 492         cur_cpu_spec->mmu_features &= ~MMU_FTR_NO_SLBIE_B;
 493         return 1;
 494 }
 495
 496 #ifdef CONFIG_MEMORY_HOTPLUG
 497 static int __init probe_memory_block_size(unsigned long node, const char *uname, int
 498                                           depth, void *data)
 499 {
 500         unsigned long *mem_block_size = (unsigned long *)data;
 501         const __be32 *prop;
 502         int len;
 503
 504         if (depth != 1)
 505                 return 0;
 506
 507         if (strcmp(uname, "ibm,dynamic-reconfiguration-memory"))
 508                 return 0;
 509
 510         prop = of_get_flat_dt_prop(node, "ibm,lmb-size", &len);
 511
 512         if (!prop || len < dt_root_size_cells * sizeof(__be32))
 513                 /*
 514                  * Nothing in the device tree
 515                  */
 516                 *mem_block_size = MIN_MEMORY_BLOCK_SIZE;
 517         else
 518                 *mem_block_size = of_read_number(prop, dt_root_size_cells);
 519         return 1;
 520 }
 521
 522 static unsigned long radix_memory_block_size(void)
 523 {
 524         unsigned long mem_block_size = MIN_MEMORY_BLOCK_SIZE;
 525
 526         /*
 527          * OPAL firmware feature is set by now. Hence we are ok
 528          * to test OPAL feature.
 529          */
 530         if (firmware_has_feature(FW_FEATURE_OPAL))
 531                 mem_block_size = 1UL * 1024 * 1024 * 1024;
 532         else
 533                 of_scan_flat_dt(probe_memory_block_size, &mem_block_size);
 534
 535         return mem_block_size;
 536 }
 537
 538 #else   /* CONFIG_MEMORY_HOTPLUG */
 539
 540 static unsigned long radix_memory_block_size(void)
 541 {
 542         return 1UL * 1024 * 1024 * 1024;
 543 }
 544
 545 #endif /* CONFIG_MEMORY_HOTPLUG */
 546
 547
 548 void __init radix__early_init_devtree(void)
 549 {
 550         int rc;
 551
 552         /*
 553          * Try to find the available page sizes in the device-tree
 554          */
 555         rc = of_scan_flat_dt(radix_dt_scan_page_sizes, NULL);
 556         if (!rc) {
 557                 /*
 558                  * No page size details found in device tree.
 559                  * Let's assume we have page 4k and 64k support
 560                  */
 561                 mmu_psize_defs[MMU_PAGE_4K].shift = 12;
 562                 mmu_psize_defs[MMU_PAGE_4K].ap = 0x0;
 563
 564                 mmu_psize_defs[MMU_PAGE_64K].shift = 16;
 565                 mmu_psize_defs[MMU_PAGE_64K].ap = 0x5;
 566         }
 567
 568         /*
 569          * Max mapping size used when mapping pages. We don't use
 570          * ppc_md.memory_block_size() here because this get called
 571          * early and we don't have machine probe called yet. Also
 572          * the pseries implementation only check for ibm,lmb-size.
 573          * All hypervisor supporting radix do expose that device
 574          * tree node.
 575          */
 576         radix_mem_block_size = radix_memory_block_size();
 577         return;
 578 }
 579
 580 static void radix_init_amor(void)
 581 {
 582         /*
 583         * In HV mode, we init AMOR (Authority Mask Override Register) so that
 584         * the hypervisor and guest can setup IAMR (Instruction Authority Mask
 585         * Register), enable key 0 and set it to 1.
 586         *
 587         * AMOR = 0b1100 .... 0000 (Mask for key 0 is 11)
 588         */
 589         mtspr(SPRN_AMOR, (3ul << 62));
 590 }
 591
 592 void __init radix__early_init_mmu(void)
 593 {
 594         unsigned long lpcr;
 595
 596 #ifdef CONFIG_PPC_64K_PAGES
 597         /* PAGE_SIZE mappings */
 598         mmu_virtual_psize = MMU_PAGE_64K;
 599 #else
 600         mmu_virtual_psize = MMU_PAGE_4K;
 601 #endif
 602
 603 #ifdef CONFIG_SPARSEMEM_VMEMMAP
 604         /* vmemmap mapping */
 605         if (mmu_psize_defs[MMU_PAGE_2M].shift) {
 606                 /*
 607                  * map vmemmap using 2M if available
 608                  */
 609                 mmu_vmemmap_psize = MMU_PAGE_2M;
 610         } else
 611                 mmu_vmemmap_psize = mmu_virtual_psize;
 612 #endif
 613         /*
 614          * initialize page table size
 615          */
 616         __pte_index_size = RADIX_PTE_INDEX_SIZE;
 617         __pmd_index_size = RADIX_PMD_INDEX_SIZE;
 618         __pud_index_size = RADIX_PUD_INDEX_SIZE;
 619         __pgd_index_size = RADIX_PGD_INDEX_SIZE;
 620         __pud_cache_index = RADIX_PUD_INDEX_SIZE;
 621         __pte_table_size = RADIX_PTE_TABLE_SIZE;
 622         __pmd_table_size = RADIX_PMD_TABLE_SIZE;
 623         __pud_table_size = RADIX_PUD_TABLE_SIZE;
 624         __pgd_table_size = RADIX_PGD_TABLE_SIZE;
 625
 626         __pmd_val_bits = RADIX_PMD_VAL_BITS;
 627         __pud_val_bits = RADIX_PUD_VAL_BITS;
 628         __pgd_val_bits = RADIX_PGD_VAL_BITS;
 629
 630         __kernel_virt_start = RADIX_KERN_VIRT_START;
 631         __vmalloc_start = RADIX_VMALLOC_START;
 632         __vmalloc_end = RADIX_VMALLOC_END;
 633         __kernel_io_start = RADIX_KERN_IO_START;
 634         __kernel_io_end = RADIX_KERN_IO_END;
 635         vmemmap = (struct page *)RADIX_VMEMMAP_START;
 636         ioremap_bot = IOREMAP_BASE;
 637
 638 #ifdef CONFIG_PCI
 639         pci_io_base = ISA_IO_BASE;
 640 #endif
 641         __pte_frag_nr = RADIX_PTE_FRAG_NR;
 642         __pte_frag_size_shift = RADIX_PTE_FRAG_SIZE_SHIFT;
 643         __pmd_frag_nr = RADIX_PMD_FRAG_NR;
 644         __pmd_frag_size_shift = RADIX_PMD_FRAG_SIZE_SHIFT;
 645
 646         radix_init_pgtable();
 647
 648         if (!firmware_has_feature(FW_FEATURE_LPAR)) {
 649                 lpcr = mfspr(SPRN_LPCR);
 650                 mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
 651                 radix_init_partition_table();
 652                 radix_init_amor();
 653         } else {
 654                 radix_init_pseries();
 655         }
 656
 657         memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
 658
 659         /* Switch to the guard PID before turning on MMU */
 660         radix__switch_mmu_context(NULL, &init_mm);
 661         tlbiel_all();
 662 }
 663
 664 void radix__early_init_mmu_secondary(void)
 665 {
 666         unsigned long lpcr;
 667         /*
 668          * update partition table control register and UPRT
 669          */
 670         if (!firmware_has_feature(FW_FEATURE_LPAR)) {
 671                 lpcr = mfspr(SPRN_LPCR);
 672                 mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
 673
 674                 set_ptcr_when_no_uv(__pa(partition_tb) |
 675                                     (PATB_SIZE_SHIFT - 12));
 676
 677                 radix_init_amor();
 678         }
 679
 680         radix__switch_mmu_context(NULL, &init_mm);
 681         tlbiel_all();
 682
 683         /* Make sure userspace can't change the AMR */
 684         mtspr(SPRN_UAMOR, 0);
 685 }
 686
 687 void radix__mmu_cleanup_all(void)
 688 {
 689         unsigned long lpcr;
 690
 691         if (!firmware_has_feature(FW_FEATURE_LPAR)) {
 692                 lpcr = mfspr(SPRN_LPCR);
 693                 mtspr(SPRN_LPCR, lpcr & ~LPCR_UPRT);
 694                 set_ptcr_when_no_uv(0);
 695                 powernv_set_nmmu_ptcr(0);
 696                 radix__flush_tlb_all();
 697         }
 698 }
 699
 700 #ifdef CONFIG_MEMORY_HOTPLUG
 701 static void free_pte_table(pte_t *pte_start, pmd_t *pmd)
 702 {
 703         pte_t *pte;
 704         int i;
 705
 706         for (i = 0; i < PTRS_PER_PTE; i++) {
 707                 pte = pte_start + i;
 708                 if (!pte_none(*pte))
 709                         return;
 710         }
 711
 712         pte_free_kernel(&init_mm, pte_start);
 713         pmd_clear(pmd);
 714 }
 715
 716 static void free_pmd_table(pmd_t *pmd_start, pud_t *pud)
 717 {
 718         pmd_t *pmd;
 719         int i;
 720
 721         for (i = 0; i < PTRS_PER_PMD; i++) {
 722                 pmd = pmd_start + i;
 723                 if (!pmd_none(*pmd))
 724                         return;
 725         }
 726
 727         pmd_free(&init_mm, pmd_start);
 728         pud_clear(pud);
 729 }
 730
 731 static void free_pud_table(pud_t *pud_start, p4d_t *p4d)
 732 {
 733         pud_t *pud;
 734         int i;
 735
 736         for (i = 0; i < PTRS_PER_PUD; i++) {
 737                 pud = pud_start + i;
 738                 if (!pud_none(*pud))
 739                         return;
 740         }
 741
 742         pud_free(&init_mm, pud_start);
 743         p4d_clear(p4d);
 744 }
 745
 746 static void remove_pte_table(pte_t *pte_start, unsigned long addr,
 747                              unsigned long end)
 748 {
 749         unsigned long next;
 750         pte_t *pte;
 751
 752         pte = pte_start + pte_index(addr);
 753         for (; addr < end; addr = next, pte++) {
 754                 next = (addr + PAGE_SIZE) & PAGE_MASK;
 755                 if (next > end)
 756                         next = end;
 757
 758                 if (!pte_present(*pte))
 759                         continue;
 760
 761                 if (!PAGE_ALIGNED(addr) || !PAGE_ALIGNED(next)) {
 762                         /*
 763                          * The vmemmap_free() and remove_section_mapping()
 764                          * codepaths call us with aligned addresses.
 765                          */
 766                         WARN_ONCE(1, "%s: unaligned range\n", __func__);
 767                         continue;
 768                 }
 769
 770                 pte_clear(&init_mm, addr, pte);
 771         }
 772 }
 773
 774 static void __meminit remove_pmd_table(pmd_t *pmd_start, unsigned long addr,
 775                              unsigned long end)
 776 {
 777         unsigned long next;
 778         pte_t *pte_base;
 779         pmd_t *pmd;
 780
 781         pmd = pmd_start + pmd_index(addr);
 782         for (; addr < end; addr = next, pmd++) {
 783                 next = pmd_addr_end(addr, end);
 784
 785                 if (!pmd_present(*pmd))
 786                         continue;
 787
 788                 if (pmd_is_leaf(*pmd)) {
 789                         if (!IS_ALIGNED(addr, PMD_SIZE) ||
 790                             !IS_ALIGNED(next, PMD_SIZE)) {
 791                                 WARN_ONCE(1, "%s: unaligned range\n", __func__);
 792                                 continue;
 793                         }
 794                         pte_clear(&init_mm, addr, (pte_t *)pmd);
 795                         continue;
 796                 }
 797
 798                 pte_base = (pte_t *)pmd_page_vaddr(*pmd);
 799                 remove_pte_table(pte_base, addr, next);
 800                 free_pte_table(pte_base, pmd);
 801         }
 802 }
 803
 804 static void __meminit remove_pud_table(pud_t *pud_start, unsigned long addr,
 805                              unsigned long end)
 806 {
 807         unsigned long next;
 808         pmd_t *pmd_base;
 809         pud_t *pud;
 810
 811         pud = pud_start + pud_index(addr);
 812         for (; addr < end; addr = next, pud++) {
 813                 next = pud_addr_end(addr, end);
 814
 815                 if (!pud_present(*pud))
 816                         continue;
 817
 818                 if (pud_is_leaf(*pud)) {
 819                         if (!IS_ALIGNED(addr, PUD_SIZE) ||
 820                             !IS_ALIGNED(next, PUD_SIZE)) {
 821                                 WARN_ONCE(1, "%s: unaligned range\n", __func__);
 822                                 continue;
 823                         }
 824                         pte_clear(&init_mm, addr, (pte_t *)pud);
 825                         continue;
 826                 }
 827
 828                 pmd_base = (pmd_t *)pud_page_vaddr(*pud);
 829                 remove_pmd_table(pmd_base, addr, next);
 830                 free_pmd_table(pmd_base, pud);
 831         }
 832 }
 833
 834 static void __meminit remove_pagetable(unsigned long start, unsigned long end)
 835 {
 836         unsigned long addr, next;
 837         pud_t *pud_base;
 838         pgd_t *pgd;
 839         p4d_t *p4d;
 840
 841         spin_lock(&init_mm.page_table_lock);
 842
 843         for (addr = start; addr < end; addr = next) {
 844                 next = pgd_addr_end(addr, end);
 845
 846                 pgd = pgd_offset_k(addr);
 847                 p4d = p4d_offset(pgd, addr);
 848                 if (!p4d_present(*p4d))
 849                         continue;
 850
 851                 if (p4d_is_leaf(*p4d)) {
 852                         if (!IS_ALIGNED(addr, P4D_SIZE) ||
 853                             !IS_ALIGNED(next, P4D_SIZE)) {
 854                                 WARN_ONCE(1, "%s: unaligned range\n", __func__);
 855                                 continue;
 856                         }
 857
 858                         pte_clear(&init_mm, addr, (pte_t *)pgd);
 859                         continue;
 860                 }
 861
 862                 pud_base = (pud_t *)p4d_page_vaddr(*p4d);
 863                 remove_pud_table(pud_base, addr, next);
 864                 free_pud_table(pud_base, p4d);
 865         }
 866
 867         spin_unlock(&init_mm.page_table_lock);
 868         radix__flush_tlb_kernel_range(start, end);
 869 }
 870
 871 int __meminit radix__create_section_mapping(unsigned long start,
 872                                             unsigned long end, int nid,
 873                                             pgprot_t prot)
 874 {
 875         if (end >= RADIX_VMALLOC_START) {
 876                 pr_warn("Outside the supported range\n");
 877                 return -1;
 878         }
 879
 880         return create_physical_mapping(__pa(start), __pa(end),
 881                                        radix_mem_block_size, nid, prot);
 882 }
 883
 884 int __meminit radix__remove_section_mapping(unsigned long start, unsigned long end)
 885 {
 886         remove_pagetable(start, end);
 887         return 0;
 888 }
 889 #endif /* CONFIG_MEMORY_HOTPLUG */
 890
 891 #ifdef CONFIG_SPARSEMEM_VMEMMAP
 892 static int __map_kernel_page_nid(unsigned long ea, unsigned long pa,
 893                                  pgprot_t flags, unsigned int map_page_size,
 894                                  int nid)
 895 {
 896         return __map_kernel_page(ea, pa, flags, map_page_size, nid, 0, 0);
 897 }
 898
 899 int __meminit radix__vmemmap_create_mapping(unsigned long start,
 900                                       unsigned long page_size,
 901                                       unsigned long phys)
 902 {
 903         /* Create a PTE encoding */
 904         unsigned long flags = _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_KERNEL_RW;
 905         int nid = early_pfn_to_nid(phys >> PAGE_SHIFT);
 906         int ret;
 907
 908         if ((start + page_size) >= RADIX_VMEMMAP_END) {
 909                 pr_warn("Outside the supported range\n");
 910                 return -1;
 911         }
 912
 913         ret = __map_kernel_page_nid(start, phys, __pgprot(flags), page_size, nid);
 914         BUG_ON(ret);
 915
 916         return 0;
 917 }
 918
 919 #ifdef CONFIG_MEMORY_HOTPLUG
 920 void __meminit radix__vmemmap_remove_mapping(unsigned long start, unsigned long page_size)
 921 {
 922         remove_pagetable(start, start + page_size);
 923 }
 924 #endif
 925 #endif
 926
 927 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 928
 929 unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
 930                                   pmd_t *pmdp, unsigned long clr,
 931                                   unsigned long set)
 932 {
 933         unsigned long old;
 934
 935 #ifdef CONFIG_DEBUG_VM
 936         WARN_ON(!radix__pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp));
 937         assert_spin_locked(pmd_lockptr(mm, pmdp));
 938 #endif
 939
 940         old = radix__pte_update(mm, addr, (pte_t *)pmdp, clr, set, 1);
 941         trace_hugepage_update(addr, old, clr, set);
 942
 943         return old;
 944 }
 945
 946 pmd_t radix__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
 947                         pmd_t *pmdp)
 948
 949 {
 950         pmd_t pmd;
 951
 952         VM_BUG_ON(address & ~HPAGE_PMD_MASK);
 953         VM_BUG_ON(radix__pmd_trans_huge(*pmdp));
 954         VM_BUG_ON(pmd_devmap(*pmdp));
 955         /*
 956          * khugepaged calls this for normal pmd
 957          */
 958         pmd = *pmdp;
 959         pmd_clear(pmdp);
 960
 961         /*
 962          * pmdp collapse_flush need to ensure that there are no parallel gup
 963          * walk after this call. This is needed so that we can have stable
 964          * page ref count when collapsing a page. We don't allow a collapse page
 965          * if we have gup taken on the page. We can ensure that by sending IPI
 966          * because gup walk happens with IRQ disabled.
 967          */
 968         serialize_against_pte_lookup(vma->vm_mm);
 969
 970         radix__flush_tlb_collapsed_pmd(vma->vm_mm, address);
 971
 972         return pmd;
 973 }
 974
 975 /*
 976  * For us pgtable_t is pte_t *. Inorder to save the deposisted
 977  * page table, we consider the allocated page table as a list
 978  * head. On withdraw we need to make sure we zero out the used
 979  * list_head memory area.
 980  */
 981 void radix__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
 982                                  pgtable_t pgtable)
 983 {
 984         struct list_head *lh = (struct list_head *) pgtable;
 985
 986         assert_spin_locked(pmd_lockptr(mm, pmdp));
 987
 988         /* FIFO */
 989         if (!pmd_huge_pte(mm, pmdp))
 990                 INIT_LIST_HEAD(lh);
 991         else
 992                 list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
 993         pmd_huge_pte(mm, pmdp) = pgtable;
 994 }
 995
 996 pgtable_t radix__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
 997 {
 998         pte_t *ptep;
 999         pgtable_t pgtable;
1000         struct list_head *lh;
1001
1002         assert_spin_locked(pmd_lockptr(mm, pmdp));
1003
1004         /* FIFO */
1005         pgtable = pmd_huge_pte(mm, pmdp);
1006         lh = (struct list_head *) pgtable;
1007         if (list_empty(lh))
1008                 pmd_huge_pte(mm, pmdp) = NULL;
1009         else {
1010                 pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
1011                 list_del(lh);
1012         }
1013         ptep = (pte_t *) pgtable;
1014         *ptep = __pte(0);
1015         ptep++;
1016         *ptep = __pte(0);
1017         return pgtable;
1018 }
1019
1020 pmd_t radix__pmdp_huge_get_and_clear(struct mm_struct *mm,
1021                                      unsigned long addr, pmd_t *pmdp)
1022 {
1023         pmd_t old_pmd;
1024         unsigned long old;
1025
1026         old = radix__pmd_hugepage_update(mm, addr, pmdp, ~0UL, 0);
1027         old_pmd = __pmd(old);
1028         return old_pmd;
1029 }
1030
1031 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1032
1033 void radix__ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep,
1034                                   pte_t entry, unsigned long address, int psize)
1035 {
1036         struct mm_struct *mm = vma->vm_mm;
1037         unsigned long set = pte_val(entry) & (_PAGE_DIRTY | _PAGE_ACCESSED |
1038                                               _PAGE_RW | _PAGE_EXEC);
1039
1040         unsigned long change = pte_val(entry) ^ pte_val(*ptep);
1041         /*
1042          * To avoid NMMU hang while relaxing access, we need mark
1043          * the pte invalid in between.
1044          */
1045         if ((change & _PAGE_RW) && atomic_read(&mm->context.copros) > 0) {
1046                 unsigned long old_pte, new_pte;
1047
1048                 old_pte = __radix_pte_update(ptep, _PAGE_PRESENT, _PAGE_INVALID);
1049                 /*
1050                  * new value of pte
1051                  */
1052                 new_pte = old_pte | set;
1053                 radix__flush_tlb_page_psize(mm, address, psize);
1054                 __radix_pte_update(ptep, _PAGE_INVALID, new_pte);
1055         } else {
1056                 __radix_pte_update(ptep, 0, set);
1057                 /*
1058                  * Book3S does not require a TLB flush when relaxing access
1059                  * restrictions when the address space is not attached to a
1060                  * NMMU, because the core MMU will reload the pte after taking
1061                  * an access fault, which is defined by the architectue.
1062                  */
1063         }
1064         /* See ptesync comment in radix__set_pte_at */
1065 }
1066
1067 void radix__ptep_modify_prot_commit(struct vm_area_struct *vma,
1068                                     unsigned long addr, pte_t *ptep,
1069                                     pte_t old_pte, pte_t pte)
1070 {
1071         struct mm_struct *mm = vma->vm_mm;
1072
1073         /*
1074          * To avoid NMMU hang while relaxing access we need to flush the tlb before
1075          * we set the new value. We need to do this only for radix, because hash
1076          * translation does flush when updating the linux pte.
1077          */
1078         if (is_pte_rw_upgrade(pte_val(old_pte), pte_val(pte)) &&
1079             (atomic_read(&mm->context.copros) > 0))
1080                 radix__flush_tlb_page(vma, addr);
1081
1082         set_pte_at(mm, addr, ptep, pte);
1083 }
1084
1085 int __init arch_ioremap_pud_supported(void)
1086 {
1087         /* HPT does not cope with large pages in the vmalloc area */
1088         return radix_enabled();
1089 }
1090
1091 int __init arch_ioremap_pmd_supported(void)
1092 {
1093         return radix_enabled();
1094 }
1095
1096 int p4d_free_pud_page(p4d_t *p4d, unsigned long addr)
1097 {
1098         return 0;
1099 }
1100
1101 int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)
1102 {
1103         pte_t *ptep = (pte_t *)pud;
1104         pte_t new_pud = pfn_pte(__phys_to_pfn(addr), prot);
1105
1106         if (!radix_enabled())
1107                 return 0;
1108
1109         set_pte_at(&init_mm, 0 /* radix unused */, ptep, new_pud);
1110
1111         return 1;
1112 }
1113
1114 int pud_clear_huge(pud_t *pud)
1115 {
1116         if (pud_huge(*pud)) {
1117                 pud_clear(pud);
1118                 return 1;
1119         }
1120
1121         return 0;
1122 }
1123
1124 int pud_free_pmd_page(pud_t *pud, unsigned long addr)
1125 {
1126         pmd_t *pmd;
1127         int i;
1128
1129         pmd = (pmd_t *)pud_page_vaddr(*pud);
1130         pud_clear(pud);
1131
1132         flush_tlb_kernel_range(addr, addr + PUD_SIZE);
1133
1134         for (i = 0; i < PTRS_PER_PMD; i++) {
1135                 if (!pmd_none(pmd[i])) {
1136                         pte_t *pte;
1137                         pte = (pte_t *)pmd_page_vaddr(pmd[i]);
1138
1139                         pte_free_kernel(&init_mm, pte);
1140                 }
1141         }
1142
1143         pmd_free(&init_mm, pmd);
1144
1145         return 1;
1146 }
1147
1148 int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot)
1149 {
1150         pte_t *ptep = (pte_t *)pmd;
1151         pte_t new_pmd = pfn_pte(__phys_to_pfn(addr), prot);
1152
1153         if (!radix_enabled())
1154                 return 0;
1155
1156         set_pte_at(&init_mm, 0 /* radix unused */, ptep, new_pmd);
1157
1158         return 1;
1159 }
1160
1161 int pmd_clear_huge(pmd_t *pmd)
1162 {
1163         if (pmd_huge(*pmd)) {
1164                 pmd_clear(pmd);
1165                 return 1;
1166         }
1167
1168         return 0;
1169 }
1170
1171 int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
1172 {
1173         pte_t *pte;
1174
1175         pte = (pte_t *)pmd_page_vaddr(*pmd);
1176         pmd_clear(pmd);
1177
1178         flush_tlb_kernel_range(addr, addr + PMD_SIZE);
1179
1180         pte_free_kernel(&init_mm, pte);
1181
1182         return 1;
1183 }
1184
1185 int __init arch_ioremap_p4d_supported(void)
1186 {
1187         return 0;
1188 }