usr/src/lib/commpage/amd64/cp_subr.s

   1 /*
   2  * This file and its contents are supplied under the terms of the
   3  * Common Development and Distribution License ("CDDL"), version 1.0.
   4  * You may only use this file in accordance with the terms of version
   5  * 1.0 of the CDDL.
   6  *
   7  * A full copy of the text of the CDDL should have accompanied this
   8  * source.  A copy of the CDDL is also available via the Internet at
   9  * http://www.illumos.org/license/CDDL.
  10  */
  11
  12 /*
  13  * Copyright 2016 Joyent, Inc.
  14  */
  15
  16 #include <sys/asm_linkage.h>
  17 #include <sys/segments.h>
  18 #include <sys/time_impl.h>
  19 #include <sys/tsc.h>
  20 #include <cp_offsets.h>
  21
  22 #define GETCPU_GDT_OFFSET       SEL_GDT(GDT_CPUID, SEL_UPL)
  23
  24         .file   "cp_subr.s"
  25
  26 /*
  27  * These are cloned from TSC and time related code in the kernel.  They should
  28  * be kept in sync in the case that the source values are changed.
  29  * See: uts/i86pc/os/timestamp.c
  30  */
  31 #define NSEC_SHIFT      5
  32 #define ADJ_SHIFT       4
  33 #define NANOSEC         0x3b9aca00
  34
  35 /*
  36  * hrtime_t
  37  * __cp_tsc_read(comm_page_t *cp)
  38  *
  39  * Stack usage: 0 bytes
  40  */
  41         ENTRY_NP(__cp_tsc_read)
  42         movl    CP_TSC_TYPE(%rdi), %esi
  43         movl    CP_TSC_NCPU(%rdi), %r8d
  44         leaq    CP_TSC_SYNC_TICK_DELTA(%rdi), %r9
  45
  46         cmpl    $TSC_TSCP, %esi
  47         jne     2f
  48         rdtscp
  49         /*
  50          * When the TSC is read, the low 32 bits are placed in %eax while the
  51          * high 32 bits are placed in %edx.  They are shifted and ORed together
  52          * to obtain the full 64-bit value.
  53          */
  54         shlq    $0x20, %rdx
  55         orq     %rdx, %rax
  56         cmpl    $0, %esi
  57         jne     1f
  58         ret
  59 1:
  60         /*
  61          * When cp_tsc_ncpu is non-zero, it indicates the length of the
  62          * cp_tsc_sync_tick_delta array, which contains per-CPU offsets for the
  63          * TSC.  The CPU ID furnished by the IA32_TSC_AUX register via rdtscp
  64          * is used to look up an offset value in that array and apply it to the
  65          * TSC reading.
  66          */
  67         movq    (%r9, %rcx, 8), %rdx
  68         addq    %rdx, %rax
  69         ret
  70
  71 2:
  72         /*
  73          * Without rdtscp, there is no way to perform a TSC reading and
  74          * simultaneously query the current CPU.  If tsc_ncpu indicates that
  75          * per-CPU TSC offsets are present, the ID of the current CPU is
  76          * queried before performing a TSC reading.  It will be later compared
  77          * to a second CPU ID lookup to catch CPU migrations.
  78          *
  79          * This method will catch all but the most pathological scheduling.
  80          */
  81         cmpl    $0, %r8d
  82         je      3f
  83         movl    $GETCPU_GDT_OFFSET, %edx
  84         lsl     %dx, %edx
  85
  86 3:
  87         /* Save the most recently queried CPU ID for later comparison. */
  88         movl    %edx, %r10d
  89
  90         cmpl    $TSC_RDTSC_MFENCE, %esi
  91         jne     4f
  92         mfence
  93         rdtsc
  94         jmp     7f
  95
  96 4:
  97         cmpl    $TSC_RDTSC_LFENCE, %esi
  98         jne     5f
  99         lfence
 100         rdtsc
 101         jmp     7f
 102
 103 5:
 104         cmpl    $TSC_RDTSC_CPUID, %esi
 105         jne     6f
 106         /*
 107          * Since the amd64 ABI dictates that %rbx is callee-saved, it must be
 108          * preserved here.  Its contents will be overwritten when cpuid is used
 109          * as a serializing instruction.
 110          */
 111         movq    %rbx, %r11
 112         xorl    %eax, %eax
 113         cpuid
 114         rdtsc
 115         movq    %r11, %rbx
 116         jmp     7f
 117
 118 6:
 119         /*
 120          * Other protections should have prevented this function from being
 121          * called in the first place.  The only sane action is to abort.
 122          * The easiest means in this context is via SIGILL.
 123          */
 124         ud2a
 125
 126 7:
 127         shlq    $0x20, %rdx
 128         orq     %rdx, %rax
 129
 130         /*
 131          * Query the current CPU again if a per-CPU offset is being applied to
 132          * the TSC reading.  If the result differs from the earlier reading,
 133          * then a migration has occured and the TSC must be read again.
 134          */
 135         cmpl    $0, %r8d
 136         je      8f
 137         movl    $GETCPU_GDT_OFFSET, %edx
 138         lsl     %dx, %edx
 139         cmpl    %edx, %r10d
 140         jne     3b
 141         movq    (%r9, %rdx, 8), %rdx
 142         addq    %rdx, %rax
 143 8:
 144         ret
 145         SET_SIZE(__cp_tsc_read)
 146
 147
 148 /*
 149  * uint_t
 150  * __cp_getcpu(comm_page_t *)
 151  *
 152  * Stack usage: 0 bytes
 153  */
 154         ENTRY_NP(__cp_getcpu)
 155         movl    CP_TSC_TYPE(%rdi), %edi
 156         /*
 157          * If RDTSCP is available, it is a quick way to grab the cpu_id which
 158          * is stored in the TSC_AUX MSR by the kernel.
 159          */
 160         cmpl    $TSC_TSCP, %edi
 161         jne     1f
 162         rdtscp
 163         movl    %ecx, %eax
 164         ret
 165 1:
 166         mov     $GETCPU_GDT_OFFSET, %eax
 167         lsl     %ax, %eax
 168         ret
 169         SET_SIZE(__cp_getcpu)
 170
 171 /*
 172  * hrtime_t
 173  * __cp_gethrtime(comm_page_t *cp)
 174  *
 175  * Stack usage: 0x20 local + 0x8 call = 0x28 bytes
 176  *
 177  * %rsp+0x00 - hrtime_t tsc_last
 178  * %rsp+0x08 - hrtime_t hrtime_base
 179  * %rsp+0x10 - commpage_t *cp
 180  * %rsp+0x18 - int hres_lock
 181  */
 182         ENTRY_NP(__cp_gethrtime)
 183         subq    $0x20, %rsp
 184         movq    %rdi, 0x10(%rsp)
 185 1:
 186         movl    CP_HRES_LOCK(%rdi), %r9d
 187         movl    %r9d, 0x18(%rsp)
 188
 189         movq    CP_TSC_LAST(%rdi), %rax
 190         movq    CP_TSC_HRTIME_BASE(%rdi), %rdx
 191         movq    %rax, (%rsp)
 192         movq    %rdx, 0x8(%rsp)
 193
 194         call    __cp_tsc_read
 195         movq    0x10(%rsp), %rdi
 196
 197         movl    0x18(%rsp), %r9d
 198         movl    CP_HRES_LOCK(%rdi), %edx
 199         andl    $0xfffffffe, %r9d
 200         cmpl    %r9d, %edx
 201         jne     1b
 202
 203         /*
 204          * The in-kernel logic for calculating hrtime performs several checks
 205          * to protect against edge cases.  That logic is summarized as:
 206          * if (tsc >= tsc_last) {
 207          *         delta -= tsc_last;
 208          * } else if (tsc >= tsc_last - 2*tsc_max_delta) {
 209          *         delta = 0;
 210          * } else {
 211          *         delta = MIN(tsc, tsc_resume_cap);
 212          * }
 213          *
 214          * The below implementation achieves the same result, although it is
 215          * structured for speed and optimized for the fast path:
 216          *
 217          * delta = tsc - tsc_last;
 218          * if (delta < 0) {
 219          *         delta += (tsc_max_delta << 1);
 220          *         if (delta >= 0) {
 221          *                 delta = 0;
 222          *         } else {
 223          *                 delta = MIN(tsc, tsc_resume_cap);
 224          *         }
 225          * }
 226          */
 227         movq    (%rsp), %rdx
 228         subq    %rdx, %rax              /* delta = tsc - tsc_last */
 229         jbe     3f                      /* if (delta < 0) */
 230
 231 2:
 232         /*
 233          * Optimized TSC_CONVERT_AND_ADD:
 234          * hrtime_base += (tsc_delta * nsec_scale) >> (32 - NSEC_SHIFT)
 235          *
 236          * Since the multiply and shift are done in 128-bit, there is no need
 237          * to worry about overflow.
 238          */
 239         movl    CP_NSEC_SCALE(%rdi), %ecx
 240         mulq    %rcx
 241         shrdq   $_CONST(32 - NSEC_SHIFT), %rdx, %rax
 242         movq    0x8(%rsp), %r8
 243         addq    %r8, %rax
 244
 245         addq    $0x20, %rsp
 246         ret
 247
 248 3:
 249         movq    %rax, %r9               /* save (tsc - tsc_last) in r9 */
 250         movl    CP_TSC_MAX_DELTA(%rdi), %ecx
 251         sall    $1, %ecx
 252         addq    %rcx, %rax              /* delta += (tsc_max_delta << 1) */
 253         jae     4f                      /* delta < 0 */
 254         xorq    %rax, %rax
 255         jmp     2b
 256
 257 4:
 258         /*
 259          * Repopulate %rax with the TSC reading by adding tsc_last to %r9
 260          * (which holds tsc - tsc_last)
 261          */
 262         movq    (%rsp), %rax
 263         addq    %r9, %rax
 264
 265         /* delta = MIN(tsc, resume_cap) */
 266         movq    CP_TSC_RESUME_CAP(%rdi), %rcx
 267         cmpq    %rcx, %rax
 268         jbe     5f
 269         movq    %rcx, %rax
 270 5:
 271         jmp     2b
 272
 273         SET_SIZE(__cp_gethrtime)
 274
 275 /*
 276  * int
 277  * __cp_clock_gettime_monotonic(comm_page_t *cp, timespec_t *tsp)
 278  *
 279  * Stack usage: 0x8 local + 0x8 call + 0x28 called func. = 0x38 bytes
 280  *
 281  * %rsp+0x00 - timespec_t *tsp
 282  */
 283         ENTRY_NP(__cp_clock_gettime_monotonic)
 284         subq    $0x8, %rsp
 285         movq    %rsi, (%rsp)
 286
 287         call    __cp_gethrtime
 288
 289         /*
 290          * Convert from hrtime_t (int64_t in nanoseconds) to timespec_t.
 291          * This uses the same approach as hrt2ts, although it has been updated
 292          * to utilize 64-bit math.
 293          * 1 / 1,000,000,000 =
 294          * 1000100101110000010111110100000100110110101101001010110110011B-26
 295          * = 0x112e0be826d694b3 * 2^-26
 296          *
 297          * secs = (nsecs * 0x112e0be826d694b3) >> 26
 298          *
 299          * In order to account for the 2s-compliment of negative inputs, a
 300          * final operation completes the process:
 301          *
 302          * secs -= (nsecs >> 63)
 303          */
 304         movq    %rax, %r11
 305         movq    $0x112e0be826d694b3, %rdx
 306         imulq   %rdx
 307         sarq    $0x1a, %rdx
 308         movq    %r11, %rax
 309         sarq    $0x3f, %rax
 310         subq    %rax, %rdx
 311         movq    (%rsp), %rsi
 312         movq    %rdx, (%rsi)
 313         /*
 314          * Populating tv_nsec is easier:
 315          * tv_nsec = nsecs - (secs * NANOSEC)
 316          */
 317         imulq   $NANOSEC, %rdx, %rdx
 318         subq    %rdx, %r11
 319         movq    %r11, 0x8(%rsi)
 320
 321         xorl    %eax, %eax
 322         addq    $0x8, %rsp
 323         ret
 324         SET_SIZE(__cp_clock_gettime_monotonic)
 325
 326 /*
 327  * int
 328  * __cp_clock_gettime_realtime(comm_page_t *cp, timespec_t *tsp)
 329  *
 330  * Stack usage: 0x18 local + 0x8 call + 0x28 called func. = 0x48 bytes
 331  *
 332  * %rsp+0x00 - commpage_t *cp
 333  * %rsp+0x08 - timespec_t *tsp
 334  * %rsp+0x10 - int hres_lock
 335  */
 336         ENTRY_NP(__cp_clock_gettime_realtime)
 337         subq    $0x18, %rsp
 338         movq    %rdi, (%rsp)
 339         movq    %rsi, 0x8(%rsp)
 340
 341 1:
 342         movl    CP_HRES_LOCK(%rdi), %eax
 343         movl    %eax, 0x10(%rsp)
 344
 345         call    __cp_gethrtime
 346         movq    (%rsp), %rdi
 347         movq    CP_HRES_LAST_TICK(%rdi), %rdx
 348         subq    %rdx, %rax                      /* nslt = hrtime - last_tick */
 349         jb      1b
 350         movq    CP_HRESTIME(%rdi), %r9
 351         movq    _CONST(CP_HRESTIME + CP_HRESTIME_INCR)(%rdi), %r10
 352         movl    CP_HRESTIME_ADJ(%rdi), %r11d
 353
 354         addq    %rax, %r10                      /* now.tv_nsec += nslt */
 355
 356         cmpl    $0, %r11d
 357         jb      4f                              /* hres_adj > 0 */
 358         ja      6f                              /* hres_adj < 0 */
 359
 360 2:
 361         cmpq    $NANOSEC, %r10
 362         jae     8f                              /* tv_nsec >= NANOSEC */
 363
 364 3:
 365         movl    0x10(%rsp), %eax
 366         movl    CP_HRES_LOCK(%rdi), %edx
 367         andl    $0xfffffffe, %edx
 368         cmpl    %eax, %edx
 369         jne     1b
 370
 371         movq    0x8(%rsp), %rsi
 372         movq    %r9, (%rsi)
 373         movq    %r10, 0x8(%rsi)
 374
 375         xorl    %eax, %eax
 376         addq    $0x18, %rsp
 377         ret
 378
 379
 380 4:                                              /* hres_adj > 0 */
 381         sarq    $ADJ_SHIFT, %rax
 382         cmpl    %r11d, %eax
 383         jbe     5f
 384         movl    %r11d, %eax
 385 5:
 386         addq    %rax, %r10
 387         jmp     2b
 388
 389 6:                                              /* hres_adj < 0 */
 390         sarq    $ADJ_SHIFT, %rax
 391         negl    %r11d
 392         cmpl    %r11d, %eax
 393         jbe     7f
 394         movl    %r11d, %eax
 395 7:
 396         subq    %rax, %r10
 397         jmp     2b
 398
 399 8:                                              /* tv_nsec >= NANOSEC */
 400         subq    $NANOSEC, %r10
 401         incq    %r9
 402         cmpq    $NANOSEC, %r10
 403         jae     8b
 404         jmp     3b
 405
 406         SET_SIZE(__cp_clock_gettime_realtime)