usr/src/lib/commpage/common/cp_main.c

   1 /*
   2  * This file and its contents are supplied under the terms of the
   3  * Common Development and Distribution License ("CDDL"), version 1.0.
   4  * You may only use this file in accordance with the terms of version
   5  * 1.0 of the CDDL.
   6  *
   7  * A full copy of the text of the CDDL should have accompanied this
   8  * source.  A copy of the CDDL is also available via the Internet at
   9  * http://www.illumos.org/license/CDDL.
  10  */
  11
  12 /*
  13  * Copyright 2016 Joyent, Inc.
  14  */
  15
  16 #include <sys/comm_page.h>
  17 #include <sys/tsc.h>
  18
  19
  20 /*
  21  * Interrogate if querying the clock via the comm page is possible.
  22  */
  23 int
  24 __cp_can_gettime(comm_page_t *cp)
  25 {
  26         switch (cp->cp_tsc_type) {
  27         case TSC_TSCP:
  28         case TSC_RDTSC_MFENCE:
  29         case TSC_RDTSC_LFENCE:
  30         case TSC_RDTSC_CPUID:
  31                 return (1);
  32         default:
  33                 break;
  34         }
  35         return (0);
  36 }
  37
  38 #ifdef __amd64
  39
  40 /*
  41  * The functions used for calculating time (both monotonic and wall-clock) are
  42  * implemented in assembly on amd64.  This is primarily for stack conservation.
  43  */
  44
  45 #else /* i386 below */
  46
  47 /*
  48  * ASM-defined functions.
  49  */
  50 extern hrtime_t __cp_tsc_read(comm_page_t *);
  51
  52 /*
  53  * These are cloned from TSC and time related code in the kernel.  The should
  54  * be kept in sync in the case that the source values are changed.
  55  */
  56 #define NSEC_SHIFT      5
  57 #define ADJ_SHIFT       4
  58 #define NANOSEC         1000000000LL
  59
  60 #define TSC_CONVERT_AND_ADD(tsc, hrt, scale) do {               \
  61         uint32_t *_l = (uint32_t *)&(tsc);                      \
  62         uint64_t sc = (uint32_t)(scale);                        \
  63         (hrt) += (uint64_t)(_l[1] * sc) << NSEC_SHIFT;          \
  64         (hrt) += (uint64_t)(_l[0] * sc) >> (32 - NSEC_SHIFT);   \
  65 } while (0)
  66
  67 /*
  68  * Userspace version of tsc_gethrtime.
  69  * See: uts/i86pc/os/timestamp.c
  70  */
  71 hrtime_t
  72 __cp_gethrtime(comm_page_t *cp)
  73 {
  74         uint32_t old_hres_lock;
  75         hrtime_t tsc, hrt, tsc_last;
  76
  77         /*
  78          * Several precautions must be taken when collecting the data necessary
  79          * to perform an accurate gethrtime calculation.
  80          *
  81          * While much of the TSC state stored in the comm page is unchanging
  82          * after boot, portions of it are periodically updated during OS ticks.
  83          * Changes to hres_lock during the course of the copy indicates a
  84          * potentially inconsistent snapshot, necessitating a loop.
  85          *
  86          * Even more complicated is the handling for TSCs which require sync
  87          * offsets between different CPUs.  Since userspace lacks the luxury of
  88          * disabling interrupts, a validation loop checking for CPU migrations
  89          * is used.  Pathological scheduling could, in theory, "outwit"
  90          * this check.  Such a possibility is considered an acceptable risk.
  91          *
  92          */
  93         do {
  94                 old_hres_lock = cp->cp_hres_lock;
  95                 tsc_last = cp->cp_tsc_last;
  96                 hrt = cp->cp_tsc_hrtime_base;
  97                 tsc = __cp_tsc_read(cp);
  98         } while ((old_hres_lock & ~1) != cp->cp_hres_lock);
  99
 100         if (tsc >= tsc_last) {
 101                 tsc -= tsc_last;
 102         } else if (tsc >= tsc_last - (2 * cp->cp_tsc_max_delta)) {
 103                 tsc = 0;
 104         } else if (tsc > cp->cp_tsc_resume_cap) {
 105                 tsc = cp->cp_tsc_resume_cap;
 106         }
 107         TSC_CONVERT_AND_ADD(tsc, hrt, cp->cp_nsec_scale);
 108
 109         return (hrt);
 110 }
 111
 112 /*
 113  * Userspace version of pc_gethrestime.
 114  * See: uts/i86pc/os/machdep.c
 115  */
 116 int
 117 __cp_clock_gettime_realtime(comm_page_t *cp, timespec_t *tsp)
 118 {
 119         int lock_prev, nslt;
 120         timespec_t now;
 121         int64_t hres_adj;
 122
 123 loop:
 124         lock_prev = cp->cp_hres_lock;
 125         now.tv_sec = cp->cp_hrestime[0];
 126         now.tv_nsec = cp->cp_hrestime[1];
 127         nslt = (int)(__cp_gethrtime(cp) - cp->cp_hres_last_tick);
 128         hres_adj = cp->cp_hrestime_adj;
 129         if (nslt < 0) {
 130                 /*
 131                  * Tick came between sampling hrtime and hres_last_tick;
 132                  */
 133                 goto loop;
 134         }
 135         now.tv_nsec += nslt;
 136
 137         /*
 138          * Apply hres_adj skew, if needed.
 139          */
 140         if (hres_adj > 0) {
 141                 nslt = (nslt >> ADJ_SHIFT);
 142                 if (nslt > hres_adj)
 143                         nslt = (int)hres_adj;
 144                 now.tv_nsec += nslt;
 145         } else if (hres_adj < 0) {
 146                 nslt = -(nslt >> ADJ_SHIFT);
 147                 if (nslt < hres_adj)
 148                         nslt = (int)hres_adj;
 149                 now.tv_nsec += nslt;
 150         }
 151
 152         /*
 153          * Rope in tv_nsec from any excessive adjustments.
 154          */
 155         while ((unsigned long)now.tv_nsec >= NANOSEC) {
 156                 now.tv_nsec -= NANOSEC;
 157                 now.tv_sec++;
 158         }
 159
 160         if ((cp->cp_hres_lock & ~1) != lock_prev)
 161                 goto loop;
 162
 163         *tsp = now;
 164         return (0);
 165 }
 166
 167 /*
 168  * The __cp_clock_gettime_monotonic function expects that hrt2ts be present
 169  * when the code is finally linked.
 170  * (The amd64 version has no such requirement.)
 171  */
 172 extern void hrt2ts(hrtime_t, timespec_t *);
 173
 174 int
 175 __cp_clock_gettime_monotonic(comm_page_t *cp, timespec_t *tsp)
 176 {
 177         hrtime_t hrt;
 178
 179         hrt = __cp_gethrtime(cp);
 180         hrt2ts(hrt, tsp);
 181         return (0);
 182 }
 183
 184 #endif /* __amd64 */