arch/powerpc/mm/nohash/tlb_64e.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Copyright 2008,2009 Ben Herrenschmidt <benh@kernel.crashing.org>
   4  *                     IBM Corp.
   5  *
   6  *  Derived from arch/ppc/mm/init.c:
   7  *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
   8  *
   9  *  Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
  10  *  and Cort Dougan (PReP) (cort@cs.nmt.edu)
  11  *    Copyright (C) 1996 Paul Mackerras
  12  *
  13  *  Derived from "arch/i386/mm/init.c"
  14  *    Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
  15  */
  16
  17 #include <linux/kernel.h>
  18 #include <linux/export.h>
  19 #include <linux/mm.h>
  20 #include <linux/init.h>
  21 #include <linux/pagemap.h>
  22 #include <linux/memblock.h>
  23
  24 #include <asm/pgalloc.h>
  25 #include <asm/tlbflush.h>
  26 #include <asm/tlb.h>
  27 #include <asm/text-patching.h>
  28 #include <asm/cputhreads.h>
  29
  30 #include <mm/mmu_decl.h>
  31
  32 /* The variables below are currently only used on 64-bit Book3E
  33  * though this will probably be made common with other nohash
  34  * implementations at some point
  35  */
  36 static int mmu_pte_psize;       /* Page size used for PTE pages */
  37 int mmu_vmemmap_psize;          /* Page size used for the virtual mem map */
  38 int book3e_htw_mode;            /* HW tablewalk?  Value is PPC_HTW_* */
  39 unsigned long linear_map_top;   /* Top of linear mapping */
  40
  41
  42 /*
  43  * Number of bytes to add to SPRN_SPRG_TLB_EXFRAME on crit/mcheck/debug
  44  * exceptions.  This is used for bolted and e6500 TLB miss handlers which
  45  * do not modify this SPRG in the TLB miss code; for other TLB miss handlers,
  46  * this is set to zero.
  47  */
  48 int extlb_level_exc;
  49
  50 /*
  51  * Handling of virtual linear page tables or indirect TLB entries
  52  * flushing when PTE pages are freed
  53  */
  54 void tlb_flush_pgtable(struct mmu_gather *tlb, unsigned long address)
  55 {
  56         int tsize = mmu_psize_defs[mmu_pte_psize].shift - 10;
  57
  58         if (book3e_htw_mode != PPC_HTW_NONE) {
  59                 unsigned long start = address & PMD_MASK;
  60                 unsigned long end = address + PMD_SIZE;
  61                 unsigned long size = 1UL << mmu_psize_defs[mmu_pte_psize].shift;
  62
  63                 /* This isn't the most optimal, ideally we would factor out the
  64                  * while preempt & CPU mask mucking around, or even the IPI but
  65                  * it will do for now
  66                  */
  67                 while (start < end) {
  68                         __flush_tlb_page(tlb->mm, start, tsize, 1);
  69                         start += size;
  70                 }
  71         } else {
  72                 unsigned long rmask = 0xf000000000000000ul;
  73                 unsigned long rid = (address & rmask) | 0x1000000000000000ul;
  74                 unsigned long vpte = address & ~rmask;
  75
  76                 vpte = (vpte >> (PAGE_SHIFT - 3)) & ~0xffful;
  77                 vpte |= rid;
  78                 __flush_tlb_page(tlb->mm, vpte, tsize, 0);
  79         }
  80 }
  81
  82 static void __init setup_page_sizes(void)
  83 {
  84         unsigned int tlb0cfg;
  85         unsigned int eptcfg;
  86         int psize;
  87
  88         unsigned int mmucfg = mfspr(SPRN_MMUCFG);
  89
  90         if ((mmucfg & MMUCFG_MAVN) == MMUCFG_MAVN_V1) {
  91                 unsigned int tlb1cfg = mfspr(SPRN_TLB1CFG);
  92                 unsigned int min_pg, max_pg;
  93
  94                 min_pg = (tlb1cfg & TLBnCFG_MINSIZE) >> TLBnCFG_MINSIZE_SHIFT;
  95                 max_pg = (tlb1cfg & TLBnCFG_MAXSIZE) >> TLBnCFG_MAXSIZE_SHIFT;
  96
  97                 for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
  98                         struct mmu_psize_def *def;
  99                         unsigned int shift;
 100
 101                         def = &mmu_psize_defs[psize];
 102                         shift = def->shift;
 103
 104                         if (shift == 0 || shift & 1)
 105                                 continue;
 106
 107                         /* adjust to be in terms of 4^shift Kb */
 108                         shift = (shift - 10) >> 1;
 109
 110                         if ((shift >= min_pg) && (shift <= max_pg))
 111                                 def->flags |= MMU_PAGE_SIZE_DIRECT;
 112                 }
 113
 114                 goto out;
 115         }
 116
 117         if ((mmucfg & MMUCFG_MAVN) == MMUCFG_MAVN_V2) {
 118                 u32 tlb1cfg, tlb1ps;
 119
 120                 tlb0cfg = mfspr(SPRN_TLB0CFG);
 121                 tlb1cfg = mfspr(SPRN_TLB1CFG);
 122                 tlb1ps = mfspr(SPRN_TLB1PS);
 123                 eptcfg = mfspr(SPRN_EPTCFG);
 124
 125                 if ((tlb1cfg & TLBnCFG_IND) && (tlb0cfg & TLBnCFG_PT))
 126                         book3e_htw_mode = PPC_HTW_E6500;
 127
 128                 /*
 129                  * We expect 4K subpage size and unrestricted indirect size.
 130                  * The lack of a restriction on indirect size is a Freescale
 131                  * extension, indicated by PSn = 0 but SPSn != 0.
 132                  */
 133                 if (eptcfg != 2)
 134                         book3e_htw_mode = PPC_HTW_NONE;
 135
 136                 for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
 137                         struct mmu_psize_def *def = &mmu_psize_defs[psize];
 138
 139                         if (!def->shift)
 140                                 continue;
 141
 142                         if (tlb1ps & (1U << (def->shift - 10))) {
 143                                 def->flags |= MMU_PAGE_SIZE_DIRECT;
 144
 145                                 if (book3e_htw_mode && psize == MMU_PAGE_2M)
 146                                         def->flags |= MMU_PAGE_SIZE_INDIRECT;
 147                         }
 148                 }
 149
 150                 goto out;
 151         }
 152 out:
 153         /* Cleanup array and print summary */
 154         pr_info("MMU: Supported page sizes\n");
 155         for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
 156                 struct mmu_psize_def *def = &mmu_psize_defs[psize];
 157                 const char *__page_type_names[] = {
 158                         "unsupported",
 159                         "direct",
 160                         "indirect",
 161                         "direct & indirect"
 162                 };
 163                 if (def->flags == 0) {
 164                         def->shift = 0;
 165                         continue;
 166                 }
 167                 pr_info("  %8ld KB as %s\n", 1ul << (def->shift - 10),
 168                         __page_type_names[def->flags & 0x3]);
 169         }
 170 }
 171
 172 /*
 173  * Early initialization of the MMU TLB code
 174  */
 175 static void early_init_this_mmu(void)
 176 {
 177         unsigned int mas4;
 178
 179         /* Set MAS4 based on page table setting */
 180
 181         mas4 = 0x4 << MAS4_WIMGED_SHIFT;
 182         switch (book3e_htw_mode) {
 183         case PPC_HTW_E6500:
 184                 mas4 |= MAS4_INDD;
 185                 mas4 |= BOOK3E_PAGESZ_2M << MAS4_TSIZED_SHIFT;
 186                 mas4 |= MAS4_TLBSELD(1);
 187                 mmu_pte_psize = MMU_PAGE_2M;
 188                 break;
 189
 190         case PPC_HTW_NONE:
 191                 mas4 |= BOOK3E_PAGESZ_4K << MAS4_TSIZED_SHIFT;
 192                 mmu_pte_psize = mmu_virtual_psize;
 193                 break;
 194         }
 195         mtspr(SPRN_MAS4, mas4);
 196
 197         unsigned int num_cams;
 198         bool map = true;
 199
 200         /* use a quarter of the TLBCAM for bolted linear map */
 201         num_cams = (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) / 4;
 202
 203         /*
 204          * Only do the mapping once per core, or else the
 205          * transient mapping would cause problems.
 206          */
 207 #ifdef CONFIG_SMP
 208         if (hweight32(get_tensr()) > 1)
 209                 map = false;
 210 #endif
 211
 212         if (map)
 213                 linear_map_top = map_mem_in_cams(linear_map_top,
 214                                                  num_cams, false, true);
 215
 216         /* A sync won't hurt us after mucking around with
 217          * the MMU configuration
 218          */
 219         mb();
 220 }
 221
 222 static void __init early_init_mmu_global(void)
 223 {
 224         /*
 225          * Freescale booke only supports 4K pages in TLB0, so use that.
 226          */
 227         mmu_vmemmap_psize = MMU_PAGE_4K;
 228
 229         /* XXX This code only checks for TLB 0 capabilities and doesn't
 230          *     check what page size combos are supported by the HW. It
 231          *     also doesn't handle the case where a separate array holds
 232          *     the IND entries from the array loaded by the PT.
 233          */
 234         /* Look for supported page sizes */
 235         setup_page_sizes();
 236
 237         /*
 238          * If we want to use HW tablewalk, enable it by patching the TLB miss
 239          * handlers to branch to the one dedicated to it.
 240          */
 241         extlb_level_exc = EX_TLB_SIZE;
 242         switch (book3e_htw_mode) {
 243         case PPC_HTW_E6500:
 244                 patch_exception(0x1c0, exc_data_tlb_miss_e6500_book3e);
 245                 patch_exception(0x1e0, exc_instruction_tlb_miss_e6500_book3e);
 246                 break;
 247         }
 248
 249         pr_info("MMU: Book3E HW tablewalk %s\n",
 250                 book3e_htw_mode != PPC_HTW_NONE ? "enabled" : "not supported");
 251
 252         /* Set the global containing the top of the linear mapping
 253          * for use by the TLB miss code
 254          */
 255         linear_map_top = memblock_end_of_DRAM();
 256
 257         ioremap_bot = IOREMAP_BASE;
 258 }
 259
 260 static void __init early_mmu_set_memory_limit(void)
 261 {
 262         /*
 263          * Limit memory so we dont have linear faults.
 264          * Unlike memblock_set_current_limit, which limits
 265          * memory available during early boot, this permanently
 266          * reduces the memory available to Linux.  We need to
 267          * do this because highmem is not supported on 64-bit.
 268          */
 269         memblock_enforce_memory_limit(linear_map_top);
 270
 271         memblock_set_current_limit(linear_map_top);
 272 }
 273
 274 /* boot cpu only */
 275 void __init early_init_mmu(void)
 276 {
 277         early_init_mmu_global();
 278         early_init_this_mmu();
 279         early_mmu_set_memory_limit();
 280 }
 281
 282 void early_init_mmu_secondary(void)
 283 {
 284         early_init_this_mmu();
 285 }
 286
 287 void setup_initial_memory_limit(phys_addr_t first_memblock_base,
 288                                 phys_addr_t first_memblock_size)
 289 {
 290         /*
 291          * On FSL Embedded 64-bit, usually all RAM is bolted, but with
 292          * unusual memory sizes it's possible for some RAM to not be mapped
 293          * (such RAM is not used at all by Linux, since we don't support
 294          * highmem on 64-bit).  We limit ppc64_rma_size to what would be
 295          * mappable if this memblock is the only one.  Additional memblocks
 296          * can only increase, not decrease, the amount that ends up getting
 297          * mapped.  We still limit max to 1G even if we'll eventually map
 298          * more.  This is due to what the early init code is set up to do.
 299          *
 300          * We crop it to the size of the first MEMBLOCK to
 301          * avoid going over total available memory just in case...
 302          */
 303         unsigned long linear_sz;
 304         unsigned int num_cams;
 305
 306         /* use a quarter of the TLBCAM for bolted linear map */
 307         num_cams = (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) / 4;
 308
 309         linear_sz = map_mem_in_cams(first_memblock_size, num_cams, true, true);
 310         ppc64_rma_size = min_t(u64, linear_sz, 0x40000000);
 311
 312         /* Finally limit subsequent allocations */
 313         memblock_set_current_limit(first_memblock_base + ppc64_rma_size);
 314 }