arch/x86/entry/vdso/vclock_gettime.c

   1 /*
   2  * Copyright 2006 Andi Kleen, SUSE Labs.
   3  * Subject to the GNU Public License, v.2
   4  *
   5  * Fast user context implementation of clock_gettime, gettimeofday, and time.
   6  *
   7  * 32 Bit compat layer by Stefani Seibold <stefani@seibold.net>
   8  *  sponsored by Rohde & Schwarz GmbH & Co. KG Munich/Germany
   9  *
  10  * The code should have no internal unresolved relocations.
  11  * Check with readelf after changing.
  12  */
  13
  14 #include <uapi/linux/time.h>
  15 #include <asm/vgtod.h>
  16 #include <asm/hpet.h>
  17 #include <asm/vvar.h>
  18 #include <asm/unistd.h>
  19 #include <asm/msr.h>
  20 #include <asm/pvclock.h>
  21 #include <linux/math64.h>
  22 #include <linux/time.h>
  23 #include <linux/kernel.h>
  24
  25 #define gtod (&VVAR(vsyscall_gtod_data))
  26
  27 extern int __vdso_clock_gettime(clockid_t clock, struct timespec *ts);
  28 extern int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz);
  29 extern time_t __vdso_time(time_t *t);
  30
  31 #ifdef CONFIG_HPET_TIMER
  32 extern u8 hpet_page
  33         __attribute__((visibility("hidden")));
  34
  35 static notrace cycle_t vread_hpet(void)
  36 {
  37         return *(const volatile u32 *)(&hpet_page + HPET_COUNTER);
  38 }
  39 #endif
  40
  41 #ifdef CONFIG_PARAVIRT_CLOCK
  42 extern u8 pvclock_page
  43         __attribute__((visibility("hidden")));
  44 #endif
  45
  46 #ifndef BUILD_VDSO32
  47
  48 notrace static long vdso_fallback_gettime(long clock, struct timespec *ts)
  49 {
  50         long ret;
  51         asm("syscall" : "=a" (ret) :
  52             "0" (__NR_clock_gettime), "D" (clock), "S" (ts) : "memory");
  53         return ret;
  54 }
  55
  56 notrace static long vdso_fallback_gtod(struct timeval *tv, struct timezone *tz)
  57 {
  58         long ret;
  59
  60         asm("syscall" : "=a" (ret) :
  61             "0" (__NR_gettimeofday), "D" (tv), "S" (tz) : "memory");
  62         return ret;
  63 }
  64
  65
  66 #else
  67
  68 notrace static long vdso_fallback_gettime(long clock, struct timespec *ts)
  69 {
  70         long ret;
  71
  72         asm(
  73                 "mov %%ebx, %%edx \n"
  74                 "mov %2, %%ebx \n"
  75                 "call __kernel_vsyscall \n"
  76                 "mov %%edx, %%ebx \n"
  77                 : "=a" (ret)
  78                 : "0" (__NR_clock_gettime), "g" (clock), "c" (ts)
  79                 : "memory", "edx");
  80         return ret;
  81 }
  82
  83 notrace static long vdso_fallback_gtod(struct timeval *tv, struct timezone *tz)
  84 {
  85         long ret;
  86
  87         asm(
  88                 "mov %%ebx, %%edx \n"
  89                 "mov %2, %%ebx \n"
  90                 "call __kernel_vsyscall \n"
  91                 "mov %%edx, %%ebx \n"
  92                 : "=a" (ret)
  93                 : "0" (__NR_gettimeofday), "g" (tv), "c" (tz)
  94                 : "memory", "edx");
  95         return ret;
  96 }
  97
  98 #endif
  99
 100 #ifdef CONFIG_PARAVIRT_CLOCK
 101 static notrace const struct pvclock_vsyscall_time_info *get_pvti0(void)
 102 {
 103         return (const struct pvclock_vsyscall_time_info *)&pvclock_page;
 104 }
 105
 106 static notrace cycle_t vread_pvclock(int *mode)
 107 {
 108         const struct pvclock_vcpu_time_info *pvti = &get_pvti0()->pvti;
 109         cycle_t ret;
 110         u64 tsc, pvti_tsc;
 111         u64 last, delta, pvti_system_time;
 112         u32 version, pvti_tsc_to_system_mul, pvti_tsc_shift;
 113
 114         /*
 115          * Note: The kernel and hypervisor must guarantee that cpu ID
 116          * number maps 1:1 to per-CPU pvclock time info.
 117          *
 118          * Because the hypervisor is entirely unaware of guest userspace
 119          * preemption, it cannot guarantee that per-CPU pvclock time
 120          * info is updated if the underlying CPU changes or that that
 121          * version is increased whenever underlying CPU changes.
 122          *
 123          * On KVM, we are guaranteed that pvti updates for any vCPU are
 124          * atomic as seen by *all* vCPUs.  This is an even stronger
 125          * guarantee than we get with a normal seqlock.
 126          *
 127          * On Xen, we don't appear to have that guarantee, but Xen still
 128          * supplies a valid seqlock using the version field.
 129          *
 130          * We only do pvclock vdso timing at all if
 131          * PVCLOCK_TSC_STABLE_BIT is set, and we interpret that bit to
 132          * mean that all vCPUs have matching pvti and that the TSC is
 133          * synced, so we can just look at vCPU 0's pvti.
 134          */
 135
 136         do {
 137                 version = pvti->version;
 138
 139                 smp_rmb();
 140
 141                 if (unlikely(!(pvti->flags & PVCLOCK_TSC_STABLE_BIT))) {
 142                         *mode = VCLOCK_NONE;
 143                         return 0;
 144                 }
 145
 146                 tsc = rdtsc_ordered();
 147                 pvti_tsc_to_system_mul = pvti->tsc_to_system_mul;
 148                 pvti_tsc_shift = pvti->tsc_shift;
 149                 pvti_system_time = pvti->system_time;
 150                 pvti_tsc = pvti->tsc_timestamp;
 151
 152                 /* Make sure that the version double-check is last. */
 153                 smp_rmb();
 154         } while (unlikely((version & 1) || version != pvti->version));
 155
 156         delta = tsc - pvti_tsc;
 157         ret = pvti_system_time +
 158                 pvclock_scale_delta(delta, pvti_tsc_to_system_mul,
 159                                     pvti_tsc_shift);
 160
 161         /* refer to vread_tsc() comment for rationale */
 162         last = gtod->cycle_last;
 163
 164         if (likely(ret >= last))
 165                 return ret;
 166
 167         return last;
 168 }
 169 #endif
 170
 171 notrace static cycle_t vread_tsc(void)
 172 {
 173         cycle_t ret = (cycle_t)rdtsc_ordered();
 174         u64 last = gtod->cycle_last;
 175
 176         if (likely(ret >= last))
 177                 return ret;
 178
 179         /*
 180          * GCC likes to generate cmov here, but this branch is extremely
 181          * predictable (it's just a function of time and the likely is
 182          * very likely) and there's a data dependence, so force GCC
 183          * to generate a branch instead.  I don't barrier() because
 184          * we don't actually need a barrier, and if this function
 185          * ever gets inlined it will generate worse code.
 186          */
 187         asm volatile ("");
 188         return last;
 189 }
 190
 191 notrace static inline u64 vgetsns(int *mode)
 192 {
 193         u64 v;
 194         cycles_t cycles;
 195
 196         if (gtod->vclock_mode == VCLOCK_TSC)
 197                 cycles = vread_tsc();
 198 #ifdef CONFIG_HPET_TIMER
 199         else if (gtod->vclock_mode == VCLOCK_HPET)
 200                 cycles = vread_hpet();
 201 #endif
 202 #ifdef CONFIG_PARAVIRT_CLOCK
 203         else if (gtod->vclock_mode == VCLOCK_PVCLOCK)
 204                 cycles = vread_pvclock(mode);
 205 #endif
 206         else
 207                 return 0;
 208         v = (cycles - gtod->cycle_last) & gtod->mask;
 209         return v * gtod->mult;
 210 }
 211
 212 /* Code size doesn't matter (vdso is 4k anyway) and this is faster. */
 213 notrace static int __always_inline do_realtime(struct timespec *ts)
 214 {
 215         unsigned long seq;
 216         u64 ns;
 217         int mode;
 218
 219         do {
 220                 seq = gtod_read_begin(gtod);
 221                 mode = gtod->vclock_mode;
 222                 ts->tv_sec = gtod->wall_time_sec;
 223                 ns = gtod->wall_time_snsec;
 224                 ns += vgetsns(&mode);
 225                 ns >>= gtod->shift;
 226         } while (unlikely(gtod_read_retry(gtod, seq)));
 227
 228         ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
 229         ts->tv_nsec = ns;
 230
 231         return mode;
 232 }
 233
 234 notrace static int __always_inline do_monotonic(struct timespec *ts)
 235 {
 236         unsigned long seq;
 237         u64 ns;
 238         int mode;
 239
 240         do {
 241                 seq = gtod_read_begin(gtod);
 242                 mode = gtod->vclock_mode;
 243                 ts->tv_sec = gtod->monotonic_time_sec;
 244                 ns = gtod->monotonic_time_snsec;
 245                 ns += vgetsns(&mode);
 246                 ns >>= gtod->shift;
 247         } while (unlikely(gtod_read_retry(gtod, seq)));
 248
 249         ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
 250         ts->tv_nsec = ns;
 251
 252         return mode;
 253 }
 254
 255 notrace static void do_realtime_coarse(struct timespec *ts)
 256 {
 257         unsigned long seq;
 258         do {
 259                 seq = gtod_read_begin(gtod);
 260                 ts->tv_sec = gtod->wall_time_coarse_sec;
 261                 ts->tv_nsec = gtod->wall_time_coarse_nsec;
 262         } while (unlikely(gtod_read_retry(gtod, seq)));
 263 }
 264
 265 notrace static void do_monotonic_coarse(struct timespec *ts)
 266 {
 267         unsigned long seq;
 268         do {
 269                 seq = gtod_read_begin(gtod);
 270                 ts->tv_sec = gtod->monotonic_time_coarse_sec;
 271                 ts->tv_nsec = gtod->monotonic_time_coarse_nsec;
 272         } while (unlikely(gtod_read_retry(gtod, seq)));
 273 }
 274
 275 notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
 276 {
 277         switch (clock) {
 278         case CLOCK_REALTIME:
 279                 if (do_realtime(ts) == VCLOCK_NONE)
 280                         goto fallback;
 281                 break;
 282         case CLOCK_MONOTONIC:
 283                 if (do_monotonic(ts) == VCLOCK_NONE)
 284                         goto fallback;
 285                 break;
 286         case CLOCK_REALTIME_COARSE:
 287                 do_realtime_coarse(ts);
 288                 break;
 289         case CLOCK_MONOTONIC_COARSE:
 290                 do_monotonic_coarse(ts);
 291                 break;
 292         default:
 293                 goto fallback;
 294         }
 295
 296         return 0;
 297 fallback:
 298         return vdso_fallback_gettime(clock, ts);
 299 }
 300 int clock_gettime(clockid_t, struct timespec *)
 301         __attribute__((weak, alias("__vdso_clock_gettime")));
 302
 303 notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
 304 {
 305         if (likely(tv != NULL)) {
 306                 if (unlikely(do_realtime((struct timespec *)tv) == VCLOCK_NONE))
 307                         return vdso_fallback_gtod(tv, tz);
 308                 tv->tv_usec /= 1000;
 309         }
 310         if (unlikely(tz != NULL)) {
 311                 tz->tz_minuteswest = gtod->tz_minuteswest;
 312                 tz->tz_dsttime = gtod->tz_dsttime;
 313         }
 314
 315         return 0;
 316 }
 317 int gettimeofday(struct timeval *, struct timezone *)
 318         __attribute__((weak, alias("__vdso_gettimeofday")));
 319
 320 /*
 321  * This will break when the xtime seconds get inaccurate, but that is
 322  * unlikely
 323  */
 324 notrace time_t __vdso_time(time_t *t)
 325 {
 326         /* This is atomic on x86 so we don't need any locks. */
 327         time_t result = ACCESS_ONCE(gtod->wall_time_sec);
 328
 329         if (t)
 330                 *t = result;
 331         return result;
 332 }
 333 int time(time_t *t)
 334         __attribute__((weak, alias("__vdso_time")));