arch/x86_64/kernel/vsyscall.c

   1 /*
   2  *  linux/arch/x86_64/kernel/vsyscall.c
   3  *
   4  *  Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
   5  *  Copyright 2003 Andi Kleen, SuSE Labs.
   6  *
   7  *  Thanks to hpa@transmeta.com for some useful hint.
   8  *  Special thanks to Ingo Molnar for his early experience with
   9  *  a different vsyscall implementation for Linux/IA32 and for the name.
  10  *
  11  *  vsyscall 1 is located at -10Mbyte, vsyscall 2 is located
  12  *  at virtual address -10Mbyte+1024bytes etc... There are at max 4
  13  *  vsyscalls. One vsyscall can reserve more than 1 slot to avoid
  14  *  jumping out of line if necessary. We cannot add more with this
  15  *  mechanism because older kernels won't return -ENOSYS.
  16  *  If we want more than four we need a vDSO.
  17  *
  18  *  Note: the concept clashes with user mode linux. If you use UML and
  19  *  want per guest time just set the kernel.vsyscall64 sysctl to 0.
  20  */
  21
  22 #include <linux/time.h>
  23 #include <linux/init.h>
  24 #include <linux/kernel.h>
  25 #include <linux/timer.h>
  26 #include <linux/seqlock.h>
  27 #include <linux/jiffies.h>
  28 #include <linux/sysctl.h>
  29 #include <linux/getcpu.h>
  30 #include <linux/cpu.h>
  31 #include <linux/smp.h>
  32 #include <linux/notifier.h>
  33
  34 #include <asm/vsyscall.h>
  35 #include <asm/pgtable.h>
  36 #include <asm/page.h>
  37 #include <asm/fixmap.h>
  38 #include <asm/errno.h>
  39 #include <asm/io.h>
  40 #include <asm/segment.h>
  41 #include <asm/desc.h>
  42 #include <asm/topology.h>
  43
  44 #define __vsyscall(nr) __attribute__ ((unused,__section__(".vsyscall_" #nr)))
  45
  46 int __sysctl_vsyscall __section_sysctl_vsyscall = 1;
  47 seqlock_t __xtime_lock __section_xtime_lock = SEQLOCK_UNLOCKED;
  48 int __vgetcpu_mode __section_vgetcpu_mode;
  49
  50 #include <asm/unistd.h>
  51
  52 static __always_inline void timeval_normalize(struct timeval * tv)
  53 {
  54         time_t __sec;
  55
  56         __sec = tv->tv_usec / 1000000;
  57         if (__sec) {
  58                 tv->tv_usec %= 1000000;
  59                 tv->tv_sec += __sec;
  60         }
  61 }
  62
  63 static __always_inline void do_vgettimeofday(struct timeval * tv)
  64 {
  65         long sequence, t;
  66         unsigned long sec, usec;
  67
  68         do {
  69                 sequence = read_seqbegin(&__xtime_lock);
  70
  71                 sec = __xtime.tv_sec;
  72                 usec = __xtime.tv_nsec / 1000;
  73
  74                 if (__vxtime.mode != VXTIME_HPET) {
  75                         t = get_cycles_sync();
  76                         if (t < __vxtime.last_tsc)
  77                                 t = __vxtime.last_tsc;
  78                         usec += ((t - __vxtime.last_tsc) *
  79                                  __vxtime.tsc_quot) >> 32;
  80                         /* See comment in x86_64 do_gettimeofday. */
  81                 } else {
  82                         usec += ((readl((void __iomem *)
  83                                    fix_to_virt(VSYSCALL_HPET) + 0xf0) -
  84                                   __vxtime.last) * __vxtime.quot) >> 32;
  85                 }
  86         } while (read_seqretry(&__xtime_lock, sequence));
  87
  88         tv->tv_sec = sec + usec / 1000000;
  89         tv->tv_usec = usec % 1000000;
  90 }
  91
  92 /* RED-PEN may want to readd seq locking, but then the variable should be write-once. */
  93 static __always_inline void do_get_tz(struct timezone * tz)
  94 {
  95         *tz = __sys_tz;
  96 }
  97
  98 static __always_inline int gettimeofday(struct timeval *tv, struct timezone *tz)
  99 {
 100         int ret;
 101         asm volatile("vsysc2: syscall"
 102                 : "=a" (ret)
 103                 : "0" (__NR_gettimeofday),"D" (tv),"S" (tz) : __syscall_clobber );
 104         return ret;
 105 }
 106
 107 static __always_inline long time_syscall(long *t)
 108 {
 109         long secs;
 110         asm volatile("vsysc1: syscall"
 111                 : "=a" (secs)
 112                 : "0" (__NR_time),"D" (t) : __syscall_clobber);
 113         return secs;
 114 }
 115
 116 int __vsyscall(0) vgettimeofday(struct timeval * tv, struct timezone * tz)
 117 {
 118         if (!__sysctl_vsyscall)
 119                 return gettimeofday(tv,tz);
 120         if (tv)
 121                 do_vgettimeofday(tv);
 122         if (tz)
 123                 do_get_tz(tz);
 124         return 0;
 125 }
 126
 127 /* This will break when the xtime seconds get inaccurate, but that is
 128  * unlikely */
 129 time_t __vsyscall(1) vtime(time_t *t)
 130 {
 131         if (!__sysctl_vsyscall)
 132                 return time_syscall(t);
 133         else if (t)
 134                 *t = __xtime.tv_sec;
 135         return __xtime.tv_sec;
 136 }
 137
 138 /* Fast way to get current CPU and node.
 139    This helps to do per node and per CPU caches in user space.
 140    The result is not guaranteed without CPU affinity, but usually
 141    works out because the scheduler tries to keep a thread on the same
 142    CPU.
 143
 144    tcache must point to a two element sized long array.
 145    All arguments can be NULL. */
 146 long __vsyscall(2)
 147 vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache)
 148 {
 149         unsigned int dummy, p;
 150         unsigned long j = 0;
 151
 152         /* Fast cache - only recompute value once per jiffies and avoid
 153            relatively costly rdtscp/cpuid otherwise.
 154            This works because the scheduler usually keeps the process
 155            on the same CPU and this syscall doesn't guarantee its
 156            results anyways.
 157            We do this here because otherwise user space would do it on
 158            its own in a likely inferior way (no access to jiffies).
 159            If you don't like it pass NULL. */
 160         if (tcache && tcache->blob[0] == (j = __jiffies)) {
 161                 p = tcache->blob[1];
 162         } else if (__vgetcpu_mode == VGETCPU_RDTSCP) {
 163                 /* Load per CPU data from RDTSCP */
 164                 rdtscp(dummy, dummy, p);
 165         } else {
 166                 /* Load per CPU data from GDT */
 167                 asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG));
 168         }
 169         if (tcache) {
 170                 tcache->blob[0] = j;
 171                 tcache->blob[1] = p;
 172         }
 173         if (cpu)
 174                 *cpu = p & 0xfff;
 175         if (node)
 176                 *node = p >> 12;
 177         return 0;
 178 }
 179
 180 long __vsyscall(3) venosys_1(void)
 181 {
 182         return -ENOSYS;
 183 }
 184
 185 #ifdef CONFIG_SYSCTL
 186
 187 #define SYSCALL 0x050f
 188 #define NOP2    0x9090
 189
 190 /*
 191  * NOP out syscall in vsyscall page when not needed.
 192  */
 193 static int vsyscall_sysctl_change(ctl_table *ctl, int write, struct file * filp,
 194                         void __user *buffer, size_t *lenp, loff_t *ppos)
 195 {
 196         extern u16 vsysc1, vsysc2;
 197         u16 __iomem *map1;
 198         u16 __iomem *map2;
 199         int ret = proc_dointvec(ctl, write, filp, buffer, lenp, ppos);
 200         if (!write)
 201                 return ret;
 202         /* gcc has some trouble with __va(__pa()), so just do it this
 203            way. */
 204         map1 = ioremap(__pa_symbol(&vsysc1), 2);
 205         if (!map1)
 206                 return -ENOMEM;
 207         map2 = ioremap(__pa_symbol(&vsysc2), 2);
 208         if (!map2) {
 209                 ret = -ENOMEM;
 210                 goto out;
 211         }
 212         if (!sysctl_vsyscall) {
 213                 writew(SYSCALL, map1);
 214                 writew(SYSCALL, map2);
 215         } else {
 216                 writew(NOP2, map1);
 217                 writew(NOP2, map2);
 218         }
 219         iounmap(map2);
 220 out:
 221         iounmap(map1);
 222         return ret;
 223 }
 224
 225 static int vsyscall_sysctl_nostrat(ctl_table *t, int __user *name, int nlen,
 226                                 void __user *oldval, size_t __user *oldlenp,
 227                                 void __user *newval, size_t newlen,
 228                                 void **context)
 229 {
 230         return -ENOSYS;
 231 }
 232
 233 static ctl_table kernel_table2[] = {
 234         { .ctl_name = 99, .procname = "vsyscall64",
 235           .data = &sysctl_vsyscall, .maxlen = sizeof(int), .mode = 0644,
 236           .strategy = vsyscall_sysctl_nostrat,
 237           .proc_handler = vsyscall_sysctl_change },
 238         { 0, }
 239 };
 240
 241 static ctl_table kernel_root_table2[] = {
 242         { .ctl_name = CTL_KERN, .procname = "kernel", .mode = 0555,
 243           .child = kernel_table2 },
 244         { 0 },
 245 };
 246
 247 #endif
 248
 249 /* Assume __initcall executes before all user space. Hopefully kmod
 250    doesn't violate that. We'll find out if it does. */
 251 static void __cpuinit vsyscall_set_cpu(int cpu)
 252 {
 253         unsigned long *d;
 254         unsigned long node = 0;
 255 #ifdef CONFIG_NUMA
 256         node = cpu_to_node[cpu];
 257 #endif
 258         if (cpu_has(&cpu_data[cpu], X86_FEATURE_RDTSCP))
 259                 write_rdtscp_aux((node << 12) | cpu);
 260
 261         /* Store cpu number in limit so that it can be loaded quickly
 262            in user space in vgetcpu.
 263            12 bits for the CPU and 8 bits for the node. */
 264         d = (unsigned long *)(cpu_gdt(cpu) + GDT_ENTRY_PER_CPU);
 265         *d = 0x0f40000000000ULL;
 266         *d |= cpu;
 267         *d |= (node & 0xf) << 12;
 268         *d |= (node >> 4) << 48;
 269 }
 270
 271 static void __cpuinit cpu_vsyscall_init(void *arg)
 272 {
 273         /* preemption should be already off */
 274         vsyscall_set_cpu(raw_smp_processor_id());
 275 }
 276
 277 #ifdef CONFIG_HOTPLUG_CPU
 278 static int __cpuinit
 279 cpu_vsyscall_notifier(struct notifier_block *n, unsigned long action, void *arg)
 280 {
 281         long cpu = (long)arg;
 282         if (action == CPU_ONLINE)
 283                 smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 0, 1);
 284         return NOTIFY_DONE;
 285 }
 286 #endif
 287
 288 static void __init map_vsyscall(void)
 289 {
 290         extern char __vsyscall_0;
 291         unsigned long physaddr_page0 = __pa_symbol(&__vsyscall_0);
 292
 293         __set_fixmap(VSYSCALL_FIRST_PAGE, physaddr_page0, PAGE_KERNEL_VSYSCALL);
 294 }
 295
 296 static int __init vsyscall_init(void)
 297 {
 298         BUG_ON(((unsigned long) &vgettimeofday !=
 299                         VSYSCALL_ADDR(__NR_vgettimeofday)));
 300         BUG_ON((unsigned long) &vtime != VSYSCALL_ADDR(__NR_vtime));
 301         BUG_ON((VSYSCALL_ADDR(0) != __fix_to_virt(VSYSCALL_FIRST_PAGE)));
 302         BUG_ON((unsigned long) &vgetcpu != VSYSCALL_ADDR(__NR_vgetcpu));
 303         map_vsyscall();
 304 #ifdef CONFIG_SYSCTL
 305         register_sysctl_table(kernel_root_table2, 0);
 306 #endif
 307         on_each_cpu(cpu_vsyscall_init, NULL, 0, 1);
 308         hotcpu_notifier(cpu_vsyscall_notifier, 0);
 309         return 0;
 310 }
 311
 312 __initcall(vsyscall_init);