arch/sparc/vdso/vclock_gettime.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright 2006 Andi Kleen, SUSE Labs.
   4  *
   5  * Fast user context implementation of clock_gettime, gettimeofday, and time.
   6  *
   7  * The code should have no internal unresolved relocations.
   8  * Check with readelf after changing.
   9  * Also alternative() doesn't work.
  10  */
  11 /*
  12  * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved.
  13  */
  14
  15 #include <linux/kernel.h>
  16 #include <linux/time.h>
  17 #include <linux/string.h>
  18 #include <asm/io.h>
  19 #include <asm/unistd.h>
  20 #include <asm/timex.h>
  21 #include <asm/clocksource.h>
  22 #include <asm/vvar.h>
  23
  24 #ifdef  CONFIG_SPARC64
  25 #define SYSCALL_STRING                                                  \
  26         "ta     0x6d;"                                                  \
  27         "bcs,a  1f;"                                                    \
  28         " sub   %%g0, %%o0, %%o0;"                                      \
  29         "1:"
  30 #else
  31 #define SYSCALL_STRING                                                  \
  32         "ta     0x10;"                                                  \
  33         "bcs,a  1f;"                                                    \
  34         " sub   %%g0, %%o0, %%o0;"                                      \
  35         "1:"
  36 #endif
  37
  38 #define SYSCALL_CLOBBERS                                                \
  39         "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7",                 \
  40         "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15",           \
  41         "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",         \
  42         "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31",         \
  43         "f32", "f34", "f36", "f38", "f40", "f42", "f44", "f46",         \
  44         "f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62",         \
  45         "cc", "memory"
  46
  47 /*
  48  * Compute the vvar page's address in the process address space, and return it
  49  * as a pointer to the vvar_data.
  50  */
  51 notrace static __always_inline struct vvar_data *get_vvar_data(void)
  52 {
  53         unsigned long ret;
  54
  55         /*
  56          * vdso data page is the first vDSO page so grab the PC
  57          * and move up a page to get to the data page.
  58          */
  59         __asm__("rd %%pc, %0" : "=r" (ret));
  60         ret &= ~(8192 - 1);
  61         ret -= 8192;
  62
  63         return (struct vvar_data *) ret;
  64 }
  65
  66 notrace static long vdso_fallback_gettime(long clock, struct __kernel_old_timespec *ts)
  67 {
  68         register long num __asm__("g1") = __NR_clock_gettime;
  69         register long o0 __asm__("o0") = clock;
  70         register long o1 __asm__("o1") = (long) ts;
  71
  72         __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
  73                              "0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
  74         return o0;
  75 }
  76
  77 notrace static long vdso_fallback_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
  78 {
  79         register long num __asm__("g1") = __NR_gettimeofday;
  80         register long o0 __asm__("o0") = (long) tv;
  81         register long o1 __asm__("o1") = (long) tz;
  82
  83         __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
  84                              "0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
  85         return o0;
  86 }
  87
  88 #ifdef  CONFIG_SPARC64
  89 notrace static __always_inline u64 __shr64(u64 val, int amt)
  90 {
  91         return val >> amt;
  92 }
  93
  94 notrace static __always_inline u64 vread_tick(void)
  95 {
  96         u64     ret;
  97
  98         __asm__ __volatile__("rd %%tick, %0" : "=r" (ret));
  99         return ret;
 100 }
 101
 102 notrace static __always_inline u64 vread_tick_stick(void)
 103 {
 104         u64     ret;
 105
 106         __asm__ __volatile__("rd %%asr24, %0" : "=r" (ret));
 107         return ret;
 108 }
 109 #else
 110 notrace static __always_inline u64 __shr64(u64 val, int amt)
 111 {
 112         u64 ret;
 113
 114         __asm__ __volatile__("sllx %H1, 32, %%g1\n\t"
 115                              "srl %L1, 0, %L1\n\t"
 116                              "or %%g1, %L1, %%g1\n\t"
 117                              "srlx %%g1, %2, %L0\n\t"
 118                              "srlx %L0, 32, %H0"
 119                              : "=r" (ret)
 120                              : "r" (val), "r" (amt)
 121                              : "g1");
 122         return ret;
 123 }
 124
 125 notrace static __always_inline u64 vread_tick(void)
 126 {
 127         register unsigned long long ret asm("o4");
 128
 129         __asm__ __volatile__("rd %%tick, %L0\n\t"
 130                              "srlx %L0, 32, %H0"
 131                              : "=r" (ret));
 132         return ret;
 133 }
 134
 135 notrace static __always_inline u64 vread_tick_stick(void)
 136 {
 137         register unsigned long long ret asm("o4");
 138
 139         __asm__ __volatile__("rd %%asr24, %L0\n\t"
 140                              "srlx %L0, 32, %H0"
 141                              : "=r" (ret));
 142         return ret;
 143 }
 144 #endif
 145
 146 notrace static __always_inline u64 vgetsns(struct vvar_data *vvar)
 147 {
 148         u64 v;
 149         u64 cycles;
 150
 151         cycles = vread_tick();
 152         v = (cycles - vvar->clock.cycle_last) & vvar->clock.mask;
 153         return v * vvar->clock.mult;
 154 }
 155
 156 notrace static __always_inline u64 vgetsns_stick(struct vvar_data *vvar)
 157 {
 158         u64 v;
 159         u64 cycles;
 160
 161         cycles = vread_tick_stick();
 162         v = (cycles - vvar->clock.cycle_last) & vvar->clock.mask;
 163         return v * vvar->clock.mult;
 164 }
 165
 166 notrace static __always_inline int do_realtime(struct vvar_data *vvar,
 167                                                struct __kernel_old_timespec *ts)
 168 {
 169         unsigned long seq;
 170         u64 ns;
 171
 172         do {
 173                 seq = vvar_read_begin(vvar);
 174                 ts->tv_sec = vvar->wall_time_sec;
 175                 ns = vvar->wall_time_snsec;
 176                 ns += vgetsns(vvar);
 177                 ns = __shr64(ns, vvar->clock.shift);
 178         } while (unlikely(vvar_read_retry(vvar, seq)));
 179
 180         ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
 181         ts->tv_nsec = ns;
 182
 183         return 0;
 184 }
 185
 186 notrace static __always_inline int do_realtime_stick(struct vvar_data *vvar,
 187                                                      struct __kernel_old_timespec *ts)
 188 {
 189         unsigned long seq;
 190         u64 ns;
 191
 192         do {
 193                 seq = vvar_read_begin(vvar);
 194                 ts->tv_sec = vvar->wall_time_sec;
 195                 ns = vvar->wall_time_snsec;
 196                 ns += vgetsns_stick(vvar);
 197                 ns = __shr64(ns, vvar->clock.shift);
 198         } while (unlikely(vvar_read_retry(vvar, seq)));
 199
 200         ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
 201         ts->tv_nsec = ns;
 202
 203         return 0;
 204 }
 205
 206 notrace static __always_inline int do_monotonic(struct vvar_data *vvar,
 207                                                 struct __kernel_old_timespec *ts)
 208 {
 209         unsigned long seq;
 210         u64 ns;
 211
 212         do {
 213                 seq = vvar_read_begin(vvar);
 214                 ts->tv_sec = vvar->monotonic_time_sec;
 215                 ns = vvar->monotonic_time_snsec;
 216                 ns += vgetsns(vvar);
 217                 ns = __shr64(ns, vvar->clock.shift);
 218         } while (unlikely(vvar_read_retry(vvar, seq)));
 219
 220         ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
 221         ts->tv_nsec = ns;
 222
 223         return 0;
 224 }
 225
 226 notrace static __always_inline int do_monotonic_stick(struct vvar_data *vvar,
 227                                                       struct __kernel_old_timespec *ts)
 228 {
 229         unsigned long seq;
 230         u64 ns;
 231
 232         do {
 233                 seq = vvar_read_begin(vvar);
 234                 ts->tv_sec = vvar->monotonic_time_sec;
 235                 ns = vvar->monotonic_time_snsec;
 236                 ns += vgetsns_stick(vvar);
 237                 ns = __shr64(ns, vvar->clock.shift);
 238         } while (unlikely(vvar_read_retry(vvar, seq)));
 239
 240         ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
 241         ts->tv_nsec = ns;
 242
 243         return 0;
 244 }
 245
 246 notrace static int do_realtime_coarse(struct vvar_data *vvar,
 247                                       struct __kernel_old_timespec *ts)
 248 {
 249         unsigned long seq;
 250
 251         do {
 252                 seq = vvar_read_begin(vvar);
 253                 ts->tv_sec = vvar->wall_time_coarse_sec;
 254                 ts->tv_nsec = vvar->wall_time_coarse_nsec;
 255         } while (unlikely(vvar_read_retry(vvar, seq)));
 256         return 0;
 257 }
 258
 259 notrace static int do_monotonic_coarse(struct vvar_data *vvar,
 260                                        struct __kernel_old_timespec *ts)
 261 {
 262         unsigned long seq;
 263
 264         do {
 265                 seq = vvar_read_begin(vvar);
 266                 ts->tv_sec = vvar->monotonic_time_coarse_sec;
 267                 ts->tv_nsec = vvar->monotonic_time_coarse_nsec;
 268         } while (unlikely(vvar_read_retry(vvar, seq)));
 269
 270         return 0;
 271 }
 272
 273 notrace int
 274 __vdso_clock_gettime(clockid_t clock, struct __kernel_old_timespec *ts)
 275 {
 276         struct vvar_data *vvd = get_vvar_data();
 277
 278         switch (clock) {
 279         case CLOCK_REALTIME:
 280                 if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
 281                         break;
 282                 return do_realtime(vvd, ts);
 283         case CLOCK_MONOTONIC:
 284                 if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
 285                         break;
 286                 return do_monotonic(vvd, ts);
 287         case CLOCK_REALTIME_COARSE:
 288                 return do_realtime_coarse(vvd, ts);
 289         case CLOCK_MONOTONIC_COARSE:
 290                 return do_monotonic_coarse(vvd, ts);
 291         }
 292         /*
 293          * Unknown clock ID ? Fall back to the syscall.
 294          */
 295         return vdso_fallback_gettime(clock, ts);
 296 }
 297 int
 298 clock_gettime(clockid_t, struct __kernel_old_timespec *)
 299         __attribute__((weak, alias("__vdso_clock_gettime")));
 300
 301 notrace int
 302 __vdso_clock_gettime_stick(clockid_t clock, struct __kernel_old_timespec *ts)
 303 {
 304         struct vvar_data *vvd = get_vvar_data();
 305
 306         switch (clock) {
 307         case CLOCK_REALTIME:
 308                 if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
 309                         break;
 310                 return do_realtime_stick(vvd, ts);
 311         case CLOCK_MONOTONIC:
 312                 if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
 313                         break;
 314                 return do_monotonic_stick(vvd, ts);
 315         case CLOCK_REALTIME_COARSE:
 316                 return do_realtime_coarse(vvd, ts);
 317         case CLOCK_MONOTONIC_COARSE:
 318                 return do_monotonic_coarse(vvd, ts);
 319         }
 320         /*
 321          * Unknown clock ID ? Fall back to the syscall.
 322          */
 323         return vdso_fallback_gettime(clock, ts);
 324 }
 325
 326 notrace int
 327 __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
 328 {
 329         struct vvar_data *vvd = get_vvar_data();
 330
 331         if (likely(vvd->vclock_mode != VCLOCK_NONE)) {
 332                 if (likely(tv != NULL)) {
 333                         union tstv_t {
 334                                 struct __kernel_old_timespec ts;
 335                                 struct __kernel_old_timeval tv;
 336                         } *tstv = (union tstv_t *) tv;
 337                         do_realtime(vvd, &tstv->ts);
 338                         /*
 339                          * Assign before dividing to ensure that the division is
 340                          * done in the type of tv_usec, not tv_nsec.
 341                          *
 342                          * There cannot be > 1 billion usec in a second:
 343                          * do_realtime() has already distributed such overflow
 344                          * into tv_sec.  So we can assign it to an int safely.
 345                          */
 346                         tstv->tv.tv_usec = tstv->ts.tv_nsec;
 347                         tstv->tv.tv_usec /= 1000;
 348                 }
 349                 if (unlikely(tz != NULL)) {
 350                         /* Avoid memcpy. Some old compilers fail to inline it */
 351                         tz->tz_minuteswest = vvd->tz_minuteswest;
 352                         tz->tz_dsttime = vvd->tz_dsttime;
 353                 }
 354                 return 0;
 355         }
 356         return vdso_fallback_gettimeofday(tv, tz);
 357 }
 358 int
 359 gettimeofday(struct __kernel_old_timeval *, struct timezone *)
 360         __attribute__((weak, alias("__vdso_gettimeofday")));
 361
 362 notrace int
 363 __vdso_gettimeofday_stick(struct __kernel_old_timeval *tv, struct timezone *tz)
 364 {
 365         struct vvar_data *vvd = get_vvar_data();
 366
 367         if (likely(vvd->vclock_mode != VCLOCK_NONE)) {
 368                 if (likely(tv != NULL)) {
 369                         union tstv_t {
 370                                 struct __kernel_old_timespec ts;
 371                                 struct __kernel_old_timeval tv;
 372                         } *tstv = (union tstv_t *) tv;
 373                         do_realtime_stick(vvd, &tstv->ts);
 374                         /*
 375                          * Assign before dividing to ensure that the division is
 376                          * done in the type of tv_usec, not tv_nsec.
 377                          *
 378                          * There cannot be > 1 billion usec in a second:
 379                          * do_realtime() has already distributed such overflow
 380                          * into tv_sec.  So we can assign it to an int safely.
 381                          */
 382                         tstv->tv.tv_usec = tstv->ts.tv_nsec;
 383                         tstv->tv.tv_usec /= 1000;
 384                 }
 385                 if (unlikely(tz != NULL)) {
 386                         /* Avoid memcpy. Some old compilers fail to inline it */
 387                         tz->tz_minuteswest = vvd->tz_minuteswest;
 388                         tz->tz_dsttime = vvd->tz_dsttime;
 389                 }
 390                 return 0;
 391         }
 392         return vdso_fallback_gettimeofday(tv, tz);
 393 }