arch/powerpc/kernel/vdso.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2
   3 /*
   4  *    Copyright (C) 2004 Benjamin Herrenschmidt, IBM Corp.
   5  *                       <benh@kernel.crashing.org>
   6  */
   7
   8 #include <linux/errno.h>
   9 #include <linux/sched.h>
  10 #include <linux/kernel.h>
  11 #include <linux/mm.h>
  12 #include <linux/smp.h>
  13 #include <linux/stddef.h>
  14 #include <linux/unistd.h>
  15 #include <linux/slab.h>
  16 #include <linux/user.h>
  17 #include <linux/elf.h>
  18 #include <linux/security.h>
  19 #include <linux/memblock.h>
  20 #include <linux/syscalls.h>
  21 #include <vdso/datapage.h>
  22
  23 #include <asm/syscall.h>
  24 #include <asm/processor.h>
  25 #include <asm/mmu.h>
  26 #include <asm/mmu_context.h>
  27 #include <asm/prom.h>
  28 #include <asm/machdep.h>
  29 #include <asm/cputable.h>
  30 #include <asm/sections.h>
  31 #include <asm/firmware.h>
  32 #include <asm/vdso.h>
  33 #include <asm/vdso_datapage.h>
  34 #include <asm/setup.h>
  35
  36 /* The alignment of the vDSO */
  37 #define VDSO_ALIGNMENT  (1 << 16)
  38
  39 extern char vdso32_start, vdso32_end;
  40 extern char vdso64_start, vdso64_end;
  41
  42 /*
  43  * The vdso data page (aka. systemcfg for old ppc64 fans) is here.
  44  * Once the early boot kernel code no longer needs to muck around
  45  * with it, it will become dynamically allocated
  46  */
  47 static union {
  48         struct vdso_arch_data   data;
  49         u8                      page[PAGE_SIZE];
  50 } vdso_data_store __page_aligned_data;
  51 struct vdso_arch_data *vdso_data = &vdso_data_store.data;
  52
  53 static int vdso_mremap(const struct vm_special_mapping *sm, struct vm_area_struct *new_vma,
  54                        unsigned long text_size)
  55 {
  56         unsigned long new_size = new_vma->vm_end - new_vma->vm_start;
  57
  58         if (new_size != text_size + PAGE_SIZE)
  59                 return -EINVAL;
  60
  61         current->mm->context.vdso = (void __user *)new_vma->vm_start + PAGE_SIZE;
  62
  63         return 0;
  64 }
  65
  66 static int vdso32_mremap(const struct vm_special_mapping *sm, struct vm_area_struct *new_vma)
  67 {
  68         return vdso_mremap(sm, new_vma, &vdso32_end - &vdso32_start);
  69 }
  70
  71 static int vdso64_mremap(const struct vm_special_mapping *sm, struct vm_area_struct *new_vma)
  72 {
  73         return vdso_mremap(sm, new_vma, &vdso64_end - &vdso64_start);
  74 }
  75
  76 static struct vm_special_mapping vdso32_spec __ro_after_init = {
  77         .name = "[vdso]",
  78         .mremap = vdso32_mremap,
  79 };
  80
  81 static struct vm_special_mapping vdso64_spec __ro_after_init = {
  82         .name = "[vdso]",
  83         .mremap = vdso64_mremap,
  84 };
  85
  86 /*
  87  * This is called from binfmt_elf, we create the special vma for the
  88  * vDSO and insert it into the mm struct tree
  89  */
  90 static int __arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
  91 {
  92         struct mm_struct *mm = current->mm;
  93         struct vm_special_mapping *vdso_spec;
  94         struct vm_area_struct *vma;
  95         unsigned long vdso_size;
  96         unsigned long vdso_base;
  97
  98         if (is_32bit_task()) {
  99                 vdso_spec = &vdso32_spec;
 100                 vdso_size = &vdso32_end - &vdso32_start;
 101                 vdso_base = VDSO32_MBASE;
 102         } else {
 103                 vdso_spec = &vdso64_spec;
 104                 vdso_size = &vdso64_end - &vdso64_start;
 105                 /*
 106                  * On 64bit we don't have a preferred map address. This
 107                  * allows get_unmapped_area to find an area near other mmaps
 108                  * and most likely share a SLB entry.
 109                  */
 110                 vdso_base = 0;
 111         }
 112
 113         /* Add a page to the vdso size for the data page */
 114         vdso_size += PAGE_SIZE;
 115
 116         /*
 117          * pick a base address for the vDSO in process space. We try to put it
 118          * at vdso_base which is the "natural" base for it, but we might fail
 119          * and end up putting it elsewhere.
 120          * Add enough to the size so that the result can be aligned.
 121          */
 122         vdso_base = get_unmapped_area(NULL, vdso_base,
 123                                       vdso_size + ((VDSO_ALIGNMENT - 1) & PAGE_MASK),
 124                                       0, 0);
 125         if (IS_ERR_VALUE(vdso_base))
 126                 return vdso_base;
 127
 128         /* Add required alignment. */
 129         vdso_base = ALIGN(vdso_base, VDSO_ALIGNMENT);
 130
 131         /*
 132          * Put vDSO base into mm struct. We need to do this before calling
 133          * install_special_mapping or the perf counter mmap tracking code
 134          * will fail to recognise it as a vDSO.
 135          */
 136         mm->context.vdso = (void __user *)vdso_base + PAGE_SIZE;
 137
 138         /*
 139          * our vma flags don't have VM_WRITE so by default, the process isn't
 140          * allowed to write those pages.
 141          * gdb can break that with ptrace interface, and thus trigger COW on
 142          * those pages but it's then your responsibility to never do that on
 143          * the "data" page of the vDSO or you'll stop getting kernel updates
 144          * and your nice userland gettimeofday will be totally dead.
 145          * It's fine to use that for setting breakpoints in the vDSO code
 146          * pages though.
 147          */
 148         vma = _install_special_mapping(mm, vdso_base, vdso_size,
 149                                        VM_READ | VM_EXEC | VM_MAYREAD |
 150                                        VM_MAYWRITE | VM_MAYEXEC, vdso_spec);
 151         return PTR_ERR_OR_ZERO(vma);
 152 }
 153
 154 int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
 155 {
 156         struct mm_struct *mm = current->mm;
 157         int rc;
 158
 159         mm->context.vdso = NULL;
 160
 161         if (mmap_write_lock_killable(mm))
 162                 return -EINTR;
 163
 164         rc = __arch_setup_additional_pages(bprm, uses_interp);
 165         if (rc)
 166                 mm->context.vdso = NULL;
 167
 168         mmap_write_unlock(mm);
 169         return rc;
 170 }
 171
 172 #define VDSO_DO_FIXUPS(type, value, bits, sec) do {                                     \
 173         void *__start = (void *)VDSO##bits##_SYMBOL(&vdso##bits##_start, sec##_start);  \
 174         void *__end = (void *)VDSO##bits##_SYMBOL(&vdso##bits##_start, sec##_end);      \
 175                                                                                         \
 176         do_##type##_fixups((value), __start, __end);                                    \
 177 } while (0)
 178
 179 static void __init vdso_fixup_features(void)
 180 {
 181 #ifdef CONFIG_PPC64
 182         VDSO_DO_FIXUPS(feature, cur_cpu_spec->cpu_features, 64, ftr_fixup);
 183         VDSO_DO_FIXUPS(feature, cur_cpu_spec->mmu_features, 64, mmu_ftr_fixup);
 184         VDSO_DO_FIXUPS(feature, powerpc_firmware_features, 64, fw_ftr_fixup);
 185         VDSO_DO_FIXUPS(lwsync, cur_cpu_spec->cpu_features, 64, lwsync_fixup);
 186 #endif /* CONFIG_PPC64 */
 187
 188 #ifdef CONFIG_VDSO32
 189         VDSO_DO_FIXUPS(feature, cur_cpu_spec->cpu_features, 32, ftr_fixup);
 190         VDSO_DO_FIXUPS(feature, cur_cpu_spec->mmu_features, 32, mmu_ftr_fixup);
 191 #ifdef CONFIG_PPC64
 192         VDSO_DO_FIXUPS(feature, powerpc_firmware_features, 32, fw_ftr_fixup);
 193 #endif /* CONFIG_PPC64 */
 194         VDSO_DO_FIXUPS(lwsync, cur_cpu_spec->cpu_features, 32, lwsync_fixup);
 195 #endif
 196 }
 197
 198 /*
 199  * Called from setup_arch to initialize the bitmap of available
 200  * syscalls in the systemcfg page
 201  */
 202 static void __init vdso_setup_syscall_map(void)
 203 {
 204         unsigned int i;
 205
 206         for (i = 0; i < NR_syscalls; i++) {
 207                 if (sys_call_table[i] != (unsigned long)&sys_ni_syscall)
 208                         vdso_data->syscall_map[i >> 5] |= 0x80000000UL >> (i & 0x1f);
 209                 if (IS_ENABLED(CONFIG_COMPAT) &&
 210                     compat_sys_call_table[i] != (unsigned long)&sys_ni_syscall)
 211                         vdso_data->compat_syscall_map[i >> 5] |= 0x80000000UL >> (i & 0x1f);
 212         }
 213 }
 214
 215 #ifdef CONFIG_PPC64
 216 int vdso_getcpu_init(void)
 217 {
 218         unsigned long cpu, node, val;
 219
 220         /*
 221          * SPRG_VDSO contains the CPU in the bottom 16 bits and the NUMA node
 222          * in the next 16 bits.  The VDSO uses this to implement getcpu().
 223          */
 224         cpu = get_cpu();
 225         WARN_ON_ONCE(cpu > 0xffff);
 226
 227         node = cpu_to_node(cpu);
 228         WARN_ON_ONCE(node > 0xffff);
 229
 230         val = (cpu & 0xffff) | ((node & 0xffff) << 16);
 231         mtspr(SPRN_SPRG_VDSO_WRITE, val);
 232         get_paca()->sprg_vdso = val;
 233
 234         put_cpu();
 235
 236         return 0;
 237 }
 238 /* We need to call this before SMP init */
 239 early_initcall(vdso_getcpu_init);
 240 #endif
 241
 242 static struct page ** __init vdso_setup_pages(void *start, void *end)
 243 {
 244         int i;
 245         struct page **pagelist;
 246         int pages = (end - start) >> PAGE_SHIFT;
 247
 248         pagelist = kcalloc(pages + 1, sizeof(struct page *), GFP_KERNEL);
 249         if (!pagelist)
 250                 panic("%s: Cannot allocate page list for VDSO", __func__);
 251
 252         pagelist[0] = virt_to_page(vdso_data);
 253
 254         for (i = 0; i < pages; i++)
 255                 pagelist[i + 1] = virt_to_page(start + i * PAGE_SIZE);
 256
 257         return pagelist;
 258 }
 259
 260 static int __init vdso_init(void)
 261 {
 262 #ifdef CONFIG_PPC64
 263         /*
 264          * Fill up the "systemcfg" stuff for backward compatibility
 265          */
 266         strcpy((char *)vdso_data->eye_catcher, "SYSTEMCFG:PPC64");
 267         vdso_data->version.major = SYSTEMCFG_MAJOR;
 268         vdso_data->version.minor = SYSTEMCFG_MINOR;
 269         vdso_data->processor = mfspr(SPRN_PVR);
 270         /*
 271          * Fake the old platform number for pSeries and add
 272          * in LPAR bit if necessary
 273          */
 274         vdso_data->platform = 0x100;
 275         if (firmware_has_feature(FW_FEATURE_LPAR))
 276                 vdso_data->platform |= 1;
 277         vdso_data->physicalMemorySize = memblock_phys_mem_size();
 278         vdso_data->dcache_size = ppc64_caches.l1d.size;
 279         vdso_data->dcache_line_size = ppc64_caches.l1d.line_size;
 280         vdso_data->icache_size = ppc64_caches.l1i.size;
 281         vdso_data->icache_line_size = ppc64_caches.l1i.line_size;
 282         vdso_data->dcache_block_size = ppc64_caches.l1d.block_size;
 283         vdso_data->icache_block_size = ppc64_caches.l1i.block_size;
 284         vdso_data->dcache_log_block_size = ppc64_caches.l1d.log_block_size;
 285         vdso_data->icache_log_block_size = ppc64_caches.l1i.log_block_size;
 286 #endif /* CONFIG_PPC64 */
 287
 288         vdso_setup_syscall_map();
 289
 290         vdso_fixup_features();
 291
 292         if (IS_ENABLED(CONFIG_VDSO32))
 293                 vdso32_spec.pages = vdso_setup_pages(&vdso32_start, &vdso32_end);
 294
 295         if (IS_ENABLED(CONFIG_PPC64))
 296                 vdso64_spec.pages = vdso_setup_pages(&vdso64_start, &vdso64_end);
 297
 298         smp_wmb();
 299
 300         return 0;
 301 }
 302 arch_initcall(vdso_init);