arch/x86/kernel/vsyscall_64.c

   1 /*
   2  *  Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
   3  *  Copyright 2003 Andi Kleen, SuSE Labs.
   4  *
   5  *  Thanks to hpa@transmeta.com for some useful hint.
   6  *  Special thanks to Ingo Molnar for his early experience with
   7  *  a different vsyscall implementation for Linux/IA32 and for the name.
   8  *
   9  *  vsyscall 1 is located at -10Mbyte, vsyscall 2 is located
  10  *  at virtual address -10Mbyte+1024bytes etc... There are at max 4
  11  *  vsyscalls. One vsyscall can reserve more than 1 slot to avoid
  12  *  jumping out of line if necessary. We cannot add more with this
  13  *  mechanism because older kernels won't return -ENOSYS.
  14  *  If we want more than four we need a vDSO.
  15  *
  16  *  Note: the concept clashes with user mode linux. If you use UML and
  17  *  want per guest time just set the kernel.vsyscall64 sysctl to 0.
  18  */
  19
  20 /* Disable profiling for userspace code: */
  21 #define DISABLE_BRANCH_PROFILING
  22
  23 #include <linux/time.h>
  24 #include <linux/init.h>
  25 #include <linux/kernel.h>
  26 #include <linux/timer.h>
  27 #include <linux/seqlock.h>
  28 #include <linux/jiffies.h>
  29 #include <linux/sysctl.h>
  30 #include <linux/clocksource.h>
  31 #include <linux/getcpu.h>
  32 #include <linux/cpu.h>
  33 #include <linux/smp.h>
  34 #include <linux/notifier.h>
  35
  36 #include <asm/vsyscall.h>
  37 #include <asm/pgtable.h>
  38 #include <asm/page.h>
  39 #include <asm/unistd.h>
  40 #include <asm/fixmap.h>
  41 #include <asm/errno.h>
  42 #include <asm/io.h>
  43 #include <asm/segment.h>
  44 #include <asm/desc.h>
  45 #include <asm/topology.h>
  46 #include <asm/vgtod.h>
  47
  48 #define __vsyscall(nr) \
  49                 __attribute__ ((unused, __section__(".vsyscall_" #nr))) notrace
  50 #define __syscall_clobber "r11","cx","memory"
  51
  52 /*
  53  * vsyscall_gtod_data contains data that is :
  54  * - readonly from vsyscalls
  55  * - written by timer interrupt or systcl (/proc/sys/kernel/vsyscall64)
  56  * Try to keep this structure as small as possible to avoid cache line ping pongs
  57  */
  58 int __vgetcpu_mode __section_vgetcpu_mode;
  59
  60 struct vsyscall_gtod_data __vsyscall_gtod_data __section_vsyscall_gtod_data =
  61 {
  62         .lock = SEQLOCK_UNLOCKED,
  63         .sysctl_enabled = 1,
  64 };
  65
  66 void update_vsyscall_tz(void)
  67 {
  68         unsigned long flags;
  69
  70         write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
  71         /* sys_tz has changed */
  72         vsyscall_gtod_data.sys_tz = sys_tz;
  73         write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
  74 }
  75
  76 void update_vsyscall(struct timespec *wall_time, struct clocksource *clock,
  77                      u32 mult)
  78 {
  79         unsigned long flags;
  80
  81         write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
  82         /* copy vsyscall data */
  83         vsyscall_gtod_data.clock.vread = clock->vread;
  84         vsyscall_gtod_data.clock.cycle_last = clock->cycle_last;
  85         vsyscall_gtod_data.clock.mask = clock->mask;
  86         vsyscall_gtod_data.clock.mult = mult;
  87         vsyscall_gtod_data.clock.shift = clock->shift;
  88         vsyscall_gtod_data.wall_time_sec = wall_time->tv_sec;
  89         vsyscall_gtod_data.wall_time_nsec = wall_time->tv_nsec;
  90         vsyscall_gtod_data.wall_to_monotonic = wall_to_monotonic;
  91         vsyscall_gtod_data.wall_time_coarse = __current_kernel_time();
  92         write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
  93 }
  94
  95 /* RED-PEN may want to readd seq locking, but then the variable should be
  96  * write-once.
  97  */
  98 static __always_inline void do_get_tz(struct timezone * tz)
  99 {
 100         *tz = __vsyscall_gtod_data.sys_tz;
 101 }
 102
 103 static __always_inline int gettimeofday(struct timeval *tv, struct timezone *tz)
 104 {
 105         int ret;
 106         asm volatile("syscall"
 107                 : "=a" (ret)
 108                 : "0" (__NR_gettimeofday),"D" (tv),"S" (tz)
 109                 : __syscall_clobber );
 110         return ret;
 111 }
 112
 113 static __always_inline long time_syscall(long *t)
 114 {
 115         long secs;
 116         asm volatile("syscall"
 117                 : "=a" (secs)
 118                 : "0" (__NR_time),"D" (t) : __syscall_clobber);
 119         return secs;
 120 }
 121
 122 static __always_inline void do_vgettimeofday(struct timeval * tv)
 123 {
 124         cycle_t now, base, mask, cycle_delta;
 125         unsigned seq;
 126         unsigned long mult, shift, nsec;
 127         cycle_t (*vread)(void);
 128         do {
 129                 seq = read_seqbegin(&__vsyscall_gtod_data.lock);
 130
 131                 vread = __vsyscall_gtod_data.clock.vread;
 132                 if (unlikely(!__vsyscall_gtod_data.sysctl_enabled || !vread)) {
 133                         gettimeofday(tv,NULL);
 134                         return;
 135                 }
 136
 137                 now = vread();
 138                 base = __vsyscall_gtod_data.clock.cycle_last;
 139                 mask = __vsyscall_gtod_data.clock.mask;
 140                 mult = __vsyscall_gtod_data.clock.mult;
 141                 shift = __vsyscall_gtod_data.clock.shift;
 142
 143                 tv->tv_sec = __vsyscall_gtod_data.wall_time_sec;
 144                 nsec = __vsyscall_gtod_data.wall_time_nsec;
 145         } while (read_seqretry(&__vsyscall_gtod_data.lock, seq));
 146
 147         /* calculate interval: */
 148         cycle_delta = (now - base) & mask;
 149         /* convert to nsecs: */
 150         nsec += (cycle_delta * mult) >> shift;
 151
 152         while (nsec >= NSEC_PER_SEC) {
 153                 tv->tv_sec += 1;
 154                 nsec -= NSEC_PER_SEC;
 155         }
 156         tv->tv_usec = nsec / NSEC_PER_USEC;
 157 }
 158
 159 int __vsyscall(0) vgettimeofday(struct timeval * tv, struct timezone * tz)
 160 {
 161         if (tv)
 162                 do_vgettimeofday(tv);
 163         if (tz)
 164                 do_get_tz(tz);
 165         return 0;
 166 }
 167
 168 /* This will break when the xtime seconds get inaccurate, but that is
 169  * unlikely */
 170 time_t __vsyscall(1) vtime(time_t *t)
 171 {
 172         struct timeval tv;
 173         time_t result;
 174         if (unlikely(!__vsyscall_gtod_data.sysctl_enabled))
 175                 return time_syscall(t);
 176
 177         vgettimeofday(&tv, NULL);
 178         result = tv.tv_sec;
 179         if (t)
 180                 *t = result;
 181         return result;
 182 }
 183
 184 /* Fast way to get current CPU and node.
 185    This helps to do per node and per CPU caches in user space.
 186    The result is not guaranteed without CPU affinity, but usually
 187    works out because the scheduler tries to keep a thread on the same
 188    CPU.
 189
 190    tcache must point to a two element sized long array.
 191    All arguments can be NULL. */
 192 long __vsyscall(2)
 193 vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache)
 194 {
 195         unsigned int p;
 196         unsigned long j = 0;
 197
 198         /* Fast cache - only recompute value once per jiffies and avoid
 199            relatively costly rdtscp/cpuid otherwise.
 200            This works because the scheduler usually keeps the process
 201            on the same CPU and this syscall doesn't guarantee its
 202            results anyways.
 203            We do this here because otherwise user space would do it on
 204            its own in a likely inferior way (no access to jiffies).
 205            If you don't like it pass NULL. */
 206         if (tcache && tcache->blob[0] == (j = __jiffies)) {
 207                 p = tcache->blob[1];
 208         } else if (__vgetcpu_mode == VGETCPU_RDTSCP) {
 209                 /* Load per CPU data from RDTSCP */
 210                 native_read_tscp(&p);
 211         } else {
 212                 /* Load per CPU data from GDT */
 213                 asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG));
 214         }
 215         if (tcache) {
 216                 tcache->blob[0] = j;
 217                 tcache->blob[1] = p;
 218         }
 219         if (cpu)
 220                 *cpu = p & 0xfff;
 221         if (node)
 222                 *node = p >> 12;
 223         return 0;
 224 }
 225
 226 static long __vsyscall(3) venosys_1(void)
 227 {
 228         return -ENOSYS;
 229 }
 230
 231 #ifdef CONFIG_SYSCTL
 232 static ctl_table kernel_table2[] = {
 233         { .procname = "vsyscall64",
 234           .data = &vsyscall_gtod_data.sysctl_enabled, .maxlen = sizeof(int),
 235           .mode = 0644,
 236           .proc_handler = proc_dointvec },
 237         {}
 238 };
 239
 240 static ctl_table kernel_root_table2[] = {
 241         { .procname = "kernel", .mode = 0555,
 242           .child = kernel_table2 },
 243         {}
 244 };
 245 #endif
 246
 247 /* Assume __initcall executes before all user space. Hopefully kmod
 248    doesn't violate that. We'll find out if it does. */
 249 static void __cpuinit vsyscall_set_cpu(int cpu)
 250 {
 251         unsigned long d;
 252         unsigned long node = 0;
 253 #ifdef CONFIG_NUMA
 254         node = cpu_to_node(cpu);
 255 #endif
 256         if (cpu_has(&cpu_data(cpu), X86_FEATURE_RDTSCP))
 257                 write_rdtscp_aux((node << 12) | cpu);
 258
 259         /* Store cpu number in limit so that it can be loaded quickly
 260            in user space in vgetcpu.
 261            12 bits for the CPU and 8 bits for the node. */
 262         d = 0x0f40000000000ULL;
 263         d |= cpu;
 264         d |= (node & 0xf) << 12;
 265         d |= (node >> 4) << 48;
 266         write_gdt_entry(get_cpu_gdt_table(cpu), GDT_ENTRY_PER_CPU, &d, DESCTYPE_S);
 267 }
 268
 269 static void __cpuinit cpu_vsyscall_init(void *arg)
 270 {
 271         /* preemption should be already off */
 272         vsyscall_set_cpu(raw_smp_processor_id());
 273 }
 274
 275 static int __cpuinit
 276 cpu_vsyscall_notifier(struct notifier_block *n, unsigned long action, void *arg)
 277 {
 278         long cpu = (long)arg;
 279         if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN)
 280                 smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 1);
 281         return NOTIFY_DONE;
 282 }
 283
 284 void __init map_vsyscall(void)
 285 {
 286         extern char __vsyscall_0;
 287         unsigned long physaddr_page0 = __pa_symbol(&__vsyscall_0);
 288
 289         /* Note that VSYSCALL_MAPPED_PAGES must agree with the code below. */
 290         __set_fixmap(VSYSCALL_FIRST_PAGE, physaddr_page0, PAGE_KERNEL_VSYSCALL);
 291 }
 292
 293 static int __init vsyscall_init(void)
 294 {
 295         BUG_ON(((unsigned long) &vgettimeofday !=
 296                         VSYSCALL_ADDR(__NR_vgettimeofday)));
 297         BUG_ON((unsigned long) &vtime != VSYSCALL_ADDR(__NR_vtime));
 298         BUG_ON((VSYSCALL_ADDR(0) != __fix_to_virt(VSYSCALL_FIRST_PAGE)));
 299         BUG_ON((unsigned long) &vgetcpu != VSYSCALL_ADDR(__NR_vgetcpu));
 300 #ifdef CONFIG_SYSCTL
 301         register_sysctl_table(kernel_root_table2);
 302 #endif
 303         on_each_cpu(cpu_vsyscall_init, NULL, 1);
 304         /* notifier priority > KVM */
 305         hotcpu_notifier(cpu_vsyscall_notifier, 30);
 306         return 0;
 307 }
 308
 309 __initcall(vsyscall_init);