third-party/benchmark/src/cycleclock.h

   1 // ----------------------------------------------------------------------
   2 // CycleClock
   3 //    A CycleClock tells you the current time in Cycles.  The "time"
   4 //    is actually time since power-on.  This is like time() but doesn't
   5 //    involve a system call and is much more precise.
   6 //
   7 // NOTE: Not all cpu/platform/kernel combinations guarantee that this
   8 // clock increments at a constant rate or is synchronized across all logical
   9 // cpus in a system.
  10 //
  11 // If you need the above guarantees, please consider using a different
  12 // API. There are efforts to provide an interface which provides a millisecond
  13 // granularity and implemented as a memory read. A memory read is generally
  14 // cheaper than the CycleClock for many architectures.
  15 //
  16 // Also, in some out of order CPU implementations, the CycleClock is not
  17 // serializing. So if you're trying to count at cycles granularity, your
  18 // data might be inaccurate due to out of order instruction execution.
  19 // ----------------------------------------------------------------------
  20
  21 #ifndef BENCHMARK_CYCLECLOCK_H_
  22 #define BENCHMARK_CYCLECLOCK_H_
  23
  24 #include <cstdint>
  25
  26 #include "benchmark/benchmark.h"
  27 #include "internal_macros.h"
  28
  29 #if defined(BENCHMARK_OS_MACOSX)
  30 #include <mach/mach_time.h>
  31 #endif
  32 // For MSVC, we want to use '_asm rdtsc' when possible (since it works
  33 // with even ancient MSVC compilers), and when not possible the
  34 // __rdtsc intrinsic, declared in <intrin.h>.  Unfortunately, in some
  35 // environments, <windows.h> and <intrin.h> have conflicting
  36 // declarations of some other intrinsics, breaking compilation.
  37 // Therefore, we simply declare __rdtsc ourselves. See also
  38 // http://connect.microsoft.com/VisualStudio/feedback/details/262047
  39 #if defined(COMPILER_MSVC) && !defined(_M_IX86) && !defined(_M_ARM64)
  40 extern "C" uint64_t __rdtsc();
  41 #pragma intrinsic(__rdtsc)
  42 #endif
  43
  44 #if !defined(BENCHMARK_OS_WINDOWS) || defined(BENCHMARK_OS_MINGW)
  45 #include <sys/time.h>
  46 #include <time.h>
  47 #endif
  48
  49 #ifdef BENCHMARK_OS_EMSCRIPTEN
  50 #include <emscripten.h>
  51 #endif
  52
  53 namespace benchmark {
  54 // NOTE: only i386 and x86_64 have been well tested.
  55 // PPC, sparc, alpha, and ia64 are based on
  56 //    http://peter.kuscsik.com/wordpress/?p=14
  57 // with modifications by m3b.  See also
  58 //    https://setisvn.ssl.berkeley.edu/svn/lib/fftw-3.0.1/kernel/cycle.h
  59 namespace cycleclock {
  60 // This should return the number of cycles since power-on.  Thread-safe.
  61 inline BENCHMARK_ALWAYS_INLINE int64_t Now() {
  62 #if defined(BENCHMARK_OS_MACOSX)
  63   // this goes at the top because we need ALL Macs, regardless of
  64   // architecture, to return the number of "mach time units" that
  65   // have passed since startup.  See sysinfo.cc where
  66   // InitializeSystemInfo() sets the supposed cpu clock frequency of
  67   // macs to the number of mach time units per second, not actual
  68   // CPU clock frequency (which can change in the face of CPU
  69   // frequency scaling).  Also note that when the Mac sleeps, this
  70   // counter pauses; it does not continue counting, nor does it
  71   // reset to zero.
  72   return mach_absolute_time();
  73 #elif defined(BENCHMARK_OS_EMSCRIPTEN)
  74   // this goes above x86-specific code because old versions of Emscripten
  75   // define __x86_64__, although they have nothing to do with it.
  76   return static_cast<int64_t>(emscripten_get_now() * 1e+6);
  77 #elif defined(__i386__)
  78   int64_t ret;
  79   __asm__ volatile("rdtsc" : "=A"(ret));
  80   return ret;
  81 #elif defined(__x86_64__) || defined(__amd64__)
  82   uint64_t low, high;
  83   __asm__ volatile("rdtsc" : "=a"(low), "=d"(high));
  84   return (high << 32) | low;
  85 #elif defined(__powerpc__) || defined(__ppc__)
  86   // This returns a time-base, which is not always precisely a cycle-count.
  87 #if defined(__powerpc64__) || defined(__ppc64__)
  88   int64_t tb;
  89   asm volatile("mfspr %0, 268" : "=r"(tb));
  90   return tb;
  91 #else
  92   uint32_t tbl, tbu0, tbu1;
  93   asm volatile(
  94       "mftbu %0\n"
  95       "mftb %1\n"
  96       "mftbu %2"
  97       : "=r"(tbu0), "=r"(tbl), "=r"(tbu1));
  98   tbl &= -static_cast<int32_t>(tbu0 == tbu1);
  99   // high 32 bits in tbu1; low 32 bits in tbl  (tbu0 is no longer needed)
 100   return (static_cast<uint64_t>(tbu1) << 32) | tbl;
 101 #endif
 102 #elif defined(__sparc__)
 103   int64_t tick;
 104   asm(".byte 0x83, 0x41, 0x00, 0x00");
 105   asm("mov   %%g1, %0" : "=r"(tick));
 106   return tick;
 107 #elif defined(__ia64__)
 108   int64_t itc;
 109   asm("mov %0 = ar.itc" : "=r"(itc));
 110   return itc;
 111 #elif defined(COMPILER_MSVC) && defined(_M_IX86)
 112   // Older MSVC compilers (like 7.x) don't seem to support the
 113   // __rdtsc intrinsic properly, so I prefer to use _asm instead
 114   // when I know it will work.  Otherwise, I'll use __rdtsc and hope
 115   // the code is being compiled with a non-ancient compiler.
 116   _asm rdtsc
 117 #elif defined(COMPILER_MSVC) && defined(_M_ARM64)
 118   // See // https://docs.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics
 119   // and https://reviews.llvm.org/D53115
 120   int64_t virtual_timer_value;
 121   virtual_timer_value = _ReadStatusReg(ARM64_CNTVCT);
 122   return virtual_timer_value;
 123 #elif defined(COMPILER_MSVC)
 124   return __rdtsc();
 125 #elif defined(BENCHMARK_OS_NACL)
 126   // Native Client validator on x86/x86-64 allows RDTSC instructions,
 127   // and this case is handled above. Native Client validator on ARM
 128   // rejects MRC instructions (used in the ARM-specific sequence below),
 129   // so we handle it here. Portable Native Client compiles to
 130   // architecture-agnostic bytecode, which doesn't provide any
 131   // cycle counter access mnemonics.
 132
 133   // Native Client does not provide any API to access cycle counter.
 134   // Use clock_gettime(CLOCK_MONOTONIC, ...) instead of gettimeofday
 135   // because is provides nanosecond resolution (which is noticable at
 136   // least for PNaCl modules running on x86 Mac & Linux).
 137   // Initialize to always return 0 if clock_gettime fails.
 138   struct timespec ts = {0, 0};
 139   clock_gettime(CLOCK_MONOTONIC, &ts);
 140   return static_cast<int64_t>(ts.tv_sec) * 1000000000 + ts.tv_nsec;
 141 #elif defined(__aarch64__)
 142   // System timer of ARMv8 runs at a different frequency than the CPU's.
 143   // The frequency is fixed, typically in the range 1-50MHz.  It can be
 144   // read at CNTFRQ special register.  We assume the OS has set up
 145   // the virtual timer properly.
 146   int64_t virtual_timer_value;
 147   asm volatile("mrs %0, cntvct_el0" : "=r"(virtual_timer_value));
 148   return virtual_timer_value;
 149 #elif defined(__ARM_ARCH)
 150   // V6 is the earliest arch that has a standard cyclecount
 151   // Native Client validator doesn't allow MRC instructions.
 152 #if (__ARM_ARCH >= 6)
 153   uint32_t pmccntr;
 154   uint32_t pmuseren;
 155   uint32_t pmcntenset;
 156   // Read the user mode perf monitor counter access permissions.
 157   asm volatile("mrc p15, 0, %0, c9, c14, 0" : "=r"(pmuseren));
 158   if (pmuseren & 1) {  // Allows reading perfmon counters for user mode code.
 159     asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r"(pmcntenset));
 160     if (pmcntenset & 0x80000000ul) {  // Is it counting?
 161       asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r"(pmccntr));
 162       // The counter is set up to count every 64th cycle
 163       return static_cast<int64_t>(pmccntr) * 64;  // Should optimize to << 6
 164     }
 165   }
 166 #endif
 167   struct timeval tv;
 168   gettimeofday(&tv, nullptr);
 169   return static_cast<int64_t>(tv.tv_sec) * 1000000 + tv.tv_usec;
 170 #elif defined(__mips__) || defined(__m68k__)
 171   // mips apparently only allows rdtsc for superusers, so we fall
 172   // back to gettimeofday.  It's possible clock_gettime would be better.
 173   struct timeval tv;
 174   gettimeofday(&tv, nullptr);
 175   return static_cast<int64_t>(tv.tv_sec) * 1000000 + tv.tv_usec;
 176 #elif defined(__loongarch__)
 177   struct timeval tv;
 178   gettimeofday(&tv, nullptr);
 179   return static_cast<int64_t>(tv.tv_sec) * 1000000 + tv.tv_usec;
 180 #elif defined(__s390__)  // Covers both s390 and s390x.
 181   // Return the CPU clock.
 182   uint64_t tsc;
 183 #if defined(BENCHMARK_OS_ZOS) && defined(COMPILER_IBMXL)
 184   // z/OS XL compiler HLASM syntax.
 185   asm(" stck %0" : "=m"(tsc) : : "cc");
 186 #else
 187   asm("stck %0" : "=Q"(tsc) : : "cc");
 188 #endif
 189   return tsc;
 190 #elif defined(__riscv)  // RISC-V
 191   // Use RDCYCLE (and RDCYCLEH on riscv32)
 192 #if __riscv_xlen == 32
 193   uint32_t cycles_lo, cycles_hi0, cycles_hi1;
 194   // This asm also includes the PowerPC overflow handling strategy, as above.
 195   // Implemented in assembly because Clang insisted on branching.
 196   asm volatile(
 197       "rdcycleh %0\n"
 198       "rdcycle %1\n"
 199       "rdcycleh %2\n"
 200       "sub %0, %0, %2\n"
 201       "seqz %0, %0\n"
 202       "sub %0, zero, %0\n"
 203       "and %1, %1, %0\n"
 204       : "=r"(cycles_hi0), "=r"(cycles_lo), "=r"(cycles_hi1));
 205   return (static_cast<uint64_t>(cycles_hi1) << 32) | cycles_lo;
 206 #else
 207   uint64_t cycles;
 208   asm volatile("rdcycle %0" : "=r"(cycles));
 209   return cycles;
 210 #endif
 211 #elif defined(__e2k__) || defined(__elbrus__)
 212   struct timeval tv;
 213   gettimeofday(&tv, nullptr);
 214   return static_cast<int64_t>(tv.tv_sec) * 1000000 + tv.tv_usec;
 215 #else
 216 // The soft failover to a generic implementation is automatic only for ARM.
 217 // For other platforms the developer is expected to make an attempt to create
 218 // a fast implementation and use generic version if nothing better is available.
 219 #error You need to define CycleTimer for your OS and CPU
 220 #endif
 221 }
 222 }  // end namespace cycleclock
 223 }  // end namespace benchmark
 224
 225 #endif  // BENCHMARK_CYCLECLOCK_H_