arch/x86/kernel/espfix_64.c

   1 /* ----------------------------------------------------------------------- *
   2  *
   3  *   Copyright 2014 Intel Corporation; author: H. Peter Anvin
   4  *
   5  *   This program is free software; you can redistribute it and/or modify it
   6  *   under the terms and conditions of the GNU General Public License,
   7  *   version 2, as published by the Free Software Foundation.
   8  *
   9  *   This program is distributed in the hope it will be useful, but WITHOUT
  10  *   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  *   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  *   more details.
  13  *
  14  * ----------------------------------------------------------------------- */
  15
  16 /*
  17  * The IRET instruction, when returning to a 16-bit segment, only
  18  * restores the bottom 16 bits of the user space stack pointer.  This
  19  * causes some 16-bit software to break, but it also leaks kernel state
  20  * to user space.
  21  *
  22  * This works around this by creating percpu "ministacks", each of which
  23  * is mapped 2^16 times 64K apart.  When we detect that the return SS is
  24  * on the LDT, we copy the IRET frame to the ministack and use the
  25  * relevant alias to return to userspace.  The ministacks are mapped
  26  * readonly, so if the IRET fault we promote #GP to #DF which is an IST
  27  * vector and thus has its own stack; we then do the fixup in the #DF
  28  * handler.
  29  *
  30  * This file sets up the ministacks and the related page tables.  The
  31  * actual ministack invocation is in entry_64.S.
  32  */
  33
  34 #include <linux/init.h>
  35 #include <linux/init_task.h>
  36 #include <linux/kernel.h>
  37 #include <linux/percpu.h>
  38 #include <linux/gfp.h>
  39 #include <linux/random.h>
  40 #include <asm/pgtable.h>
  41 #include <asm/pgalloc.h>
  42 #include <asm/setup.h>
  43 #include <asm/espfix.h>
  44 #include <asm/kaiser.h>
  45
  46 /*
  47  * Note: we only need 6*8 = 48 bytes for the espfix stack, but round
  48  * it up to a cache line to avoid unnecessary sharing.
  49  */
  50 #define ESPFIX_STACK_SIZE       (8*8UL)
  51 #define ESPFIX_STACKS_PER_PAGE  (PAGE_SIZE/ESPFIX_STACK_SIZE)
  52
  53 /* There is address space for how many espfix pages? */
  54 #define ESPFIX_PAGE_SPACE       (1UL << (PGDIR_SHIFT-PAGE_SHIFT-16))
  55
  56 #define ESPFIX_MAX_CPUS         (ESPFIX_STACKS_PER_PAGE * ESPFIX_PAGE_SPACE)
  57 #if CONFIG_NR_CPUS > ESPFIX_MAX_CPUS
  58 # error "Need more than one PGD for the ESPFIX hack"
  59 #endif
  60
  61 #define PGALLOC_GFP (GFP_KERNEL | __GFP_NOTRACK | __GFP_ZERO)
  62
  63 /* This contains the *bottom* address of the espfix stack */
  64 DEFINE_PER_CPU_READ_MOSTLY(unsigned long, espfix_stack);
  65 DEFINE_PER_CPU_READ_MOSTLY(unsigned long, espfix_waddr);
  66
  67 /* Initialization mutex - should this be a spinlock? */
  68 static DEFINE_MUTEX(espfix_init_mutex);
  69
  70 /* Page allocation bitmap - each page serves ESPFIX_STACKS_PER_PAGE CPUs */
  71 #define ESPFIX_MAX_PAGES  DIV_ROUND_UP(CONFIG_NR_CPUS, ESPFIX_STACKS_PER_PAGE)
  72 static void *espfix_pages[ESPFIX_MAX_PAGES];
  73
  74 static __page_aligned_bss pud_t espfix_pud_page[PTRS_PER_PUD]
  75         __aligned(PAGE_SIZE);
  76
  77 static unsigned int page_random, slot_random;
  78
  79 /*
  80  * This returns the bottom address of the espfix stack for a specific CPU.
  81  * The math allows for a non-power-of-two ESPFIX_STACK_SIZE, in which case
  82  * we have to account for some amount of padding at the end of each page.
  83  */
  84 static inline unsigned long espfix_base_addr(unsigned int cpu)
  85 {
  86         unsigned long page, slot;
  87         unsigned long addr;
  88
  89         page = (cpu / ESPFIX_STACKS_PER_PAGE) ^ page_random;
  90         slot = (cpu + slot_random) % ESPFIX_STACKS_PER_PAGE;
  91         addr = (page << PAGE_SHIFT) + (slot * ESPFIX_STACK_SIZE);
  92         addr = (addr & 0xffffUL) | ((addr & ~0xffffUL) << 16);
  93         addr += ESPFIX_BASE_ADDR;
  94         return addr;
  95 }
  96
  97 #define PTE_STRIDE        (65536/PAGE_SIZE)
  98 #define ESPFIX_PTE_CLONES (PTRS_PER_PTE/PTE_STRIDE)
  99 #define ESPFIX_PMD_CLONES PTRS_PER_PMD
 100 #define ESPFIX_PUD_CLONES (65536/(ESPFIX_PTE_CLONES*ESPFIX_PMD_CLONES))
 101
 102 #define PGTABLE_PROT      ((_KERNPG_TABLE & ~_PAGE_RW) | _PAGE_NX)
 103
 104 static void init_espfix_random(void)
 105 {
 106         unsigned long rand;
 107
 108         /*
 109          * This is run before the entropy pools are initialized,
 110          * but this is hopefully better than nothing.
 111          */
 112         if (!arch_get_random_long(&rand)) {
 113                 /* The constant is an arbitrary large prime */
 114                 rand = rdtsc();
 115                 rand *= 0xc345c6b72fd16123UL;
 116         }
 117
 118         slot_random = rand % ESPFIX_STACKS_PER_PAGE;
 119         page_random = (rand / ESPFIX_STACKS_PER_PAGE)
 120                 & (ESPFIX_PAGE_SPACE - 1);
 121 }
 122
 123 void __init init_espfix_bsp(void)
 124 {
 125         pgd_t *pgd_p;
 126
 127         /* Install the espfix pud into the kernel page directory */
 128         pgd_p = &init_level4_pgt[pgd_index(ESPFIX_BASE_ADDR)];
 129         pgd_populate(&init_mm, pgd_p, (pud_t *)espfix_pud_page);
 130         /*
 131          * Just copy the top-level PGD that is mapping the espfix
 132          * area to ensure it is mapped into the shadow user page
 133          * tables.
 134          */
 135         if (kaiser_enabled) {
 136                 set_pgd(native_get_shadow_pgd(pgd_p),
 137                         __pgd(_KERNPG_TABLE | __pa((pud_t *)espfix_pud_page)));
 138         }
 139
 140         /* Randomize the locations */
 141         init_espfix_random();
 142
 143         /* The rest is the same as for any other processor */
 144         init_espfix_ap(0);
 145 }
 146
 147 void init_espfix_ap(int cpu)
 148 {
 149         unsigned int page;
 150         unsigned long addr;
 151         pud_t pud, *pud_p;
 152         pmd_t pmd, *pmd_p;
 153         pte_t pte, *pte_p;
 154         int n, node;
 155         void *stack_page;
 156         pteval_t ptemask;
 157
 158         /* We only have to do this once... */
 159         if (likely(per_cpu(espfix_stack, cpu)))
 160                 return;         /* Already initialized */
 161
 162         addr = espfix_base_addr(cpu);
 163         page = cpu/ESPFIX_STACKS_PER_PAGE;
 164
 165         /* Did another CPU already set this up? */
 166         stack_page = ACCESS_ONCE(espfix_pages[page]);
 167         if (likely(stack_page))
 168                 goto done;
 169
 170         mutex_lock(&espfix_init_mutex);
 171
 172         /* Did we race on the lock? */
 173         stack_page = ACCESS_ONCE(espfix_pages[page]);
 174         if (stack_page)
 175                 goto unlock_done;
 176
 177         node = cpu_to_node(cpu);
 178         ptemask = __supported_pte_mask;
 179
 180         pud_p = &espfix_pud_page[pud_index(addr)];
 181         pud = *pud_p;
 182         if (!pud_present(pud)) {
 183                 struct page *page = alloc_pages_node(node, PGALLOC_GFP, 0);
 184
 185                 pmd_p = (pmd_t *)page_address(page);
 186                 pud = __pud(__pa(pmd_p) | (PGTABLE_PROT & ptemask));
 187                 paravirt_alloc_pmd(&init_mm, __pa(pmd_p) >> PAGE_SHIFT);
 188                 for (n = 0; n < ESPFIX_PUD_CLONES; n++)
 189                         set_pud(&pud_p[n], pud);
 190         }
 191
 192         pmd_p = pmd_offset(&pud, addr);
 193         pmd = *pmd_p;
 194         if (!pmd_present(pmd)) {
 195                 struct page *page = alloc_pages_node(node, PGALLOC_GFP, 0);
 196
 197                 pte_p = (pte_t *)page_address(page);
 198                 pmd = __pmd(__pa(pte_p) | (PGTABLE_PROT & ptemask));
 199                 paravirt_alloc_pte(&init_mm, __pa(pte_p) >> PAGE_SHIFT);
 200                 for (n = 0; n < ESPFIX_PMD_CLONES; n++)
 201                         set_pmd(&pmd_p[n], pmd);
 202         }
 203
 204         pte_p = pte_offset_kernel(&pmd, addr);
 205         stack_page = page_address(alloc_pages_node(node, GFP_KERNEL, 0));
 206         pte = __pte(__pa(stack_page) | (__PAGE_KERNEL_RO & ptemask));
 207         for (n = 0; n < ESPFIX_PTE_CLONES; n++)
 208                 set_pte(&pte_p[n*PTE_STRIDE], pte);
 209
 210         /* Job is done for this CPU and any CPU which shares this page */
 211         ACCESS_ONCE(espfix_pages[page]) = stack_page;
 212
 213 unlock_done:
 214         mutex_unlock(&espfix_init_mutex);
 215 done:
 216         per_cpu(espfix_stack, cpu) = addr;
 217         per_cpu(espfix_waddr, cpu) = (unsigned long)stack_page
 218                                       + (addr & ~PAGE_MASK);
 219 }