compiler-rt/lib/nsan/nsan.cpp

   1 //===-- nsan.cc -----------------------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // NumericalStabilitySanitizer runtime.
  10 //
  11 // This implements:
  12 //  - The public nsan interface (include/sanitizer/nsan_interface.h).
  13 //  - The private nsan interface (./nsan.h).
  14 //  - The internal instrumentation interface. These are function emitted by the
  15 //    instrumentation pass:
  16 //        * __nsan_get_shadow_ptr_for_{float,double,longdouble}_load
  17 //          These return the shadow memory pointer for loading the shadow value,
  18 //          after checking that the types are consistent. If the types are not
  19 //          consistent, returns nullptr.
  20 //        * __nsan_get_shadow_ptr_for_{float,double,longdouble}_store
  21 //          Sets the shadow types appropriately and returns the shadow memory
  22 //          pointer for storing the shadow value.
  23 //        * __nsan_internal_check_{float,double,long double}_{f,d,l} checks the
  24 //          accuracy of a value against its shadow and emits a warning depending
  25 //          on the runtime configuration. The middle part indicates the type of
  26 //          the application value, the suffix (f,d,l) indicates the type of the
  27 //          shadow, and depends on the instrumentation configuration.
  28 //        * __nsan_fcmp_fail_* emits a warning for a fcmp instruction whose
  29 //          corresponding shadow fcmp result differs.
  30 //
  31 //===----------------------------------------------------------------------===//
  32
  33 #include "nsan.h"
  34 #include "nsan_flags.h"
  35 #include "nsan_stats.h"
  36 #include "nsan_suppressions.h"
  37 #include "nsan_thread.h"
  38
  39 #include <assert.h>
  40 #include <math.h>
  41 #include <stdint.h>
  42 #include <stdio.h>
  43 #include <stdlib.h>
  44
  45 #include "sanitizer_common/sanitizer_atomic.h"
  46 #include "sanitizer_common/sanitizer_common.h"
  47 #include "sanitizer_common/sanitizer_libc.h"
  48 #include "sanitizer_common/sanitizer_report_decorator.h"
  49 #include "sanitizer_common/sanitizer_stacktrace.h"
  50 #include "sanitizer_common/sanitizer_symbolizer.h"
  51
  52 using namespace __sanitizer;
  53 using namespace __nsan;
  54
  55 constexpr int kMaxVectorWidth = 8;
  56
  57 // When copying application memory, we also copy its shadow and shadow type.
  58 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
  59 __nsan_copy_values(const void *daddr, const void *saddr, uptr size) {
  60   internal_memmove(GetShadowTypeAddrFor(daddr), GetShadowTypeAddrFor(saddr),
  61                    size);
  62   internal_memmove(GetShadowAddrFor(daddr), GetShadowAddrFor(saddr),
  63                    size * kShadowScale);
  64 }
  65
  66 #define NSAN_COPY_VALUES_N(N)                                                  \
  67   extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_copy_##N(               \
  68       const u8 *daddr, const u8 *saddr) {                                      \
  69     __builtin_memmove(GetShadowTypeAddrFor(daddr),                             \
  70                       GetShadowTypeAddrFor(saddr), N);                         \
  71     __builtin_memmove(GetShadowAddrFor(daddr), GetShadowAddrFor(saddr),        \
  72                       N *kShadowScale);                                        \
  73   }
  74
  75 NSAN_COPY_VALUES_N(4)
  76 NSAN_COPY_VALUES_N(8)
  77 NSAN_COPY_VALUES_N(16)
  78
  79 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
  80 __nsan_set_value_unknown(const void *addr, uptr size) {
  81   internal_memset(GetShadowTypeAddrFor(addr), 0, size);
  82 }
  83
  84 #define NSAN_SET_VALUE_UNKNOWN_N(N)                                            \
  85   extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_set_value_unknown_##N(  \
  86       const u8 *daddr) {                                                       \
  87     __builtin_memset(GetShadowTypeAddrFor(daddr), 0, N);                       \
  88   }
  89
  90 NSAN_SET_VALUE_UNKNOWN_N(4)
  91 NSAN_SET_VALUE_UNKNOWN_N(8)
  92 NSAN_SET_VALUE_UNKNOWN_N(16)
  93
  94 const char *FTInfo<float>::kCppTypeName = "float";
  95 const char *FTInfo<double>::kCppTypeName = "double";
  96 const char *FTInfo<long double>::kCppTypeName = "long double";
  97 const char *FTInfo<__float128>::kCppTypeName = "__float128";
  98
  99 const char FTInfo<float>::kTypePattern[sizeof(float)];
 100 const char FTInfo<double>::kTypePattern[sizeof(double)];
 101 const char FTInfo<long double>::kTypePattern[sizeof(long double)];
 102
 103 // Helper for __nsan_dump_shadow_mem: Reads the value at address `ptr`,
 104 // identified by its type id.
 105 template <typename ShadowFT>
 106 static __float128 ReadShadowInternal(const u8 *ptr) {
 107   ShadowFT Shadow;
 108   __builtin_memcpy(&Shadow, ptr, sizeof(Shadow));
 109   return Shadow;
 110 }
 111
 112 static __float128 ReadShadow(const u8 *ptr, const char ShadowTypeId) {
 113   switch (ShadowTypeId) {
 114   case 'd':
 115     return ReadShadowInternal<double>(ptr);
 116   case 'l':
 117     return ReadShadowInternal<long double>(ptr);
 118   case 'q':
 119     return ReadShadowInternal<__float128>(ptr);
 120   default:
 121     return 0.0;
 122   }
 123 }
 124
 125 namespace {
 126 class Decorator : public __sanitizer::SanitizerCommonDecorator {
 127 public:
 128   Decorator() : SanitizerCommonDecorator() {}
 129   const char *Warning() { return Red(); }
 130   const char *Name() { return Green(); }
 131   const char *End() { return Default(); }
 132 };
 133
 134 // Workaround for the fact that Printf() does not support floats.
 135 struct PrintBuffer {
 136   char Buffer[64];
 137 };
 138 template <typename FT> struct FTPrinter {};
 139
 140 template <> struct FTPrinter<double> {
 141   static PrintBuffer dec(double value) {
 142     PrintBuffer result;
 143     snprintf(result.Buffer, sizeof(result.Buffer) - 1, "%.20f", value);
 144     return result;
 145   }
 146   static PrintBuffer hex(double value) {
 147     PrintBuffer result;
 148     snprintf(result.Buffer, sizeof(result.Buffer) - 1, "%.20a", value);
 149     return result;
 150   }
 151 };
 152
 153 template <> struct FTPrinter<float> : FTPrinter<double> {};
 154
 155 template <> struct FTPrinter<long double> {
 156   static PrintBuffer dec(long double value) {
 157     PrintBuffer result;
 158     snprintf(result.Buffer, sizeof(result.Buffer) - 1, "%.20Lf", value);
 159     return result;
 160   }
 161   static PrintBuffer hex(long double value) {
 162     PrintBuffer result;
 163     snprintf(result.Buffer, sizeof(result.Buffer) - 1, "%.20La", value);
 164     return result;
 165   }
 166 };
 167
 168 // FIXME: print with full precision.
 169 template <> struct FTPrinter<__float128> : FTPrinter<long double> {};
 170
 171 // This is a template so that there are no implicit conversions.
 172 template <typename FT> inline FT ftAbs(FT v);
 173
 174 template <> inline long double ftAbs(long double v) { return fabsl(v); }
 175 template <> inline double ftAbs(double v) { return fabs(v); }
 176
 177 // We don't care about nans.
 178 // std::abs(__float128) code is suboptimal and generates a function call to
 179 // __getf2().
 180 template <typename FT> inline FT ftAbs(FT v) { return v >= FT{0} ? v : -v; }
 181
 182 template <typename FT1, typename FT2, bool Enable> struct LargestFTImpl {
 183   using type = FT2;
 184 };
 185
 186 template <typename FT1, typename FT2> struct LargestFTImpl<FT1, FT2, true> {
 187   using type = FT1;
 188 };
 189
 190 template <typename FT1, typename FT2>
 191 using LargestFT =
 192     typename LargestFTImpl<FT1, FT2, (sizeof(FT1) > sizeof(FT2))>::type;
 193
 194 template <typename T> T max(T a, T b) { return a < b ? b : a; }
 195
 196 } // end anonymous namespace
 197
 198 void __sanitizer::BufferedStackTrace::UnwindImpl(uptr pc, uptr bp,
 199                                                  void *context,
 200                                                  bool request_fast,
 201                                                  u32 max_depth) {
 202   using namespace __nsan;
 203   NsanThread *t = GetCurrentThread();
 204   if (!t || !StackTrace::WillUseFastUnwind(request_fast))
 205     return Unwind(max_depth, pc, bp, context, t ? t->stack_top() : 0,
 206                   t ? t->stack_bottom() : 0, false);
 207   if (StackTrace::WillUseFastUnwind(request_fast))
 208     Unwind(max_depth, pc, bp, nullptr, t->stack_top(), t->stack_bottom(), true);
 209   else
 210     Unwind(max_depth, pc, 0, context, 0, 0, false);
 211 }
 212
 213 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_print_accumulated_stats() {
 214   if (nsan_stats)
 215     nsan_stats->Print();
 216 }
 217
 218 static void NsanAtexit() {
 219   Printf("Numerical Sanitizer exit stats:\n");
 220   __nsan_print_accumulated_stats();
 221   nsan_stats = nullptr;
 222 }
 223
 224 // The next three functions return a pointer for storing a shadow value for `n`
 225 // values, after setting the shadow types. We return the pointer instead of
 226 // storing ourselves because it avoids having to rely on the calling convention
 227 // around long double being the same for nsan and the target application.
 228 // We have to have 3 versions because we need to know which type we are storing
 229 // since we are setting the type shadow memory.
 230 template <typename FT> static u8 *getShadowPtrForStore(u8 *store_addr, uptr n) {
 231   unsigned char *shadow_type = GetShadowTypeAddrFor(store_addr);
 232   for (uptr i = 0; i < n; ++i) {
 233     __builtin_memcpy(shadow_type + i * sizeof(FT), FTInfo<FT>::kTypePattern,
 234                      sizeof(FTInfo<FT>::kTypePattern));
 235   }
 236   return GetShadowAddrFor(store_addr);
 237 }
 238
 239 extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *
 240 __nsan_get_shadow_ptr_for_float_store(u8 *store_addr, uptr n) {
 241   return getShadowPtrForStore<float>(store_addr, n);
 242 }
 243
 244 extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *
 245 __nsan_get_shadow_ptr_for_double_store(u8 *store_addr, uptr n) {
 246   return getShadowPtrForStore<double>(store_addr, n);
 247 }
 248
 249 extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *
 250 __nsan_get_shadow_ptr_for_longdouble_store(u8 *store_addr, uptr n) {
 251   return getShadowPtrForStore<long double>(store_addr, n);
 252 }
 253
 254 template <typename FT> static bool IsValidShadowType(const u8 *shadow_type) {
 255   return __builtin_memcmp(shadow_type, FTInfo<FT>::kTypePattern, sizeof(FT)) ==
 256          0;
 257 }
 258
 259 template <int kSize, typename T> static bool IsZero(const T *ptr) {
 260   constexpr const char kZeros[kSize] = {}; // Zero initialized.
 261   return __builtin_memcmp(ptr, kZeros, kSize) == 0;
 262 }
 263
 264 template <typename FT> static bool IsUnknownShadowType(const u8 *shadow_type) {
 265   return IsZero<sizeof(FTInfo<FT>::kTypePattern)>(shadow_type);
 266 }
 267
 268 // The three folowing functions check that the address stores a complete
 269 // shadow value of the given type and return a pointer for loading.
 270 // They return nullptr if the type of the value is unknown or incomplete.
 271 template <typename FT>
 272 static const u8 *getShadowPtrForLoad(const u8 *load_addr, uptr n) {
 273   const u8 *const shadow_type = GetShadowTypeAddrFor(load_addr);
 274   for (uptr i = 0; i < n; ++i) {
 275     if (!IsValidShadowType<FT>(shadow_type + i * sizeof(FT))) {
 276       // If loadtracking stats are enabled, log loads with invalid types
 277       // (tampered with through type punning).
 278       if (flags().enable_loadtracking_stats) {
 279         if (IsUnknownShadowType<FT>(shadow_type + i * sizeof(FT))) {
 280           // Warn only if the value is non-zero. Zero is special because
 281           // applications typically initialize large buffers to zero in an
 282           // untyped way.
 283           if (!IsZero<sizeof(FT)>(load_addr)) {
 284             GET_CALLER_PC_BP;
 285             nsan_stats->AddUnknownLoadTrackingEvent(pc, bp);
 286           }
 287         } else {
 288           GET_CALLER_PC_BP;
 289           nsan_stats->AddInvalidLoadTrackingEvent(pc, bp);
 290         }
 291       }
 292       return nullptr;
 293     }
 294   }
 295   return GetShadowAddrFor(load_addr);
 296 }
 297
 298 extern "C" SANITIZER_INTERFACE_ATTRIBUTE const u8 *
 299 __nsan_get_shadow_ptr_for_float_load(const u8 *load_addr, uptr n) {
 300   return getShadowPtrForLoad<float>(load_addr, n);
 301 }
 302
 303 extern "C" SANITIZER_INTERFACE_ATTRIBUTE const u8 *
 304 __nsan_get_shadow_ptr_for_double_load(const u8 *load_addr, uptr n) {
 305   return getShadowPtrForLoad<double>(load_addr, n);
 306 }
 307
 308 extern "C" SANITIZER_INTERFACE_ATTRIBUTE const u8 *
 309 __nsan_get_shadow_ptr_for_longdouble_load(const u8 *load_addr, uptr n) {
 310   return getShadowPtrForLoad<long double>(load_addr, n);
 311 }
 312
 313 // Returns the raw shadow pointer. The returned pointer should be considered
 314 // opaque.
 315 extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *
 316 __nsan_internal_get_raw_shadow_ptr(const u8 *addr) {
 317   return GetShadowAddrFor(addr);
 318 }
 319
 320 // Returns the raw shadow type pointer. The returned pointer should be
 321 // considered opaque.
 322 extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *
 323 __nsan_internal_get_raw_shadow_type_ptr(const u8 *addr) {
 324   return reinterpret_cast<u8 *>(GetShadowTypeAddrFor(addr));
 325 }
 326
 327 static ValueType getValueType(u8 c) { return static_cast<ValueType>(c & 0x3); }
 328
 329 static int getValuePos(u8 c) { return c >> kValueSizeSizeBits; }
 330
 331 // Checks the consistency of the value types at the given type pointer.
 332 // If the value is inconsistent, returns ValueType::kUnknown. Else, return the
 333 // consistent type.
 334 template <typename FT>
 335 static bool checkValueConsistency(const u8 *shadow_type) {
 336   const int pos = getValuePos(*shadow_type);
 337   // Check that all bytes from the start of the value are ordered.
 338   for (uptr i = 0; i < sizeof(FT); ++i) {
 339     const u8 T = *(shadow_type - pos + i);
 340     if (!(getValueType(T) == FTInfo<FT>::kValueType && getValuePos(T) == i))
 341       return false;
 342   }
 343   return true;
 344 }
 345
 346 // The instrumentation automatically appends `shadow_value_type_ids`, see
 347 // maybeAddSuffixForNsanInterface.
 348 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
 349 __nsan_dump_shadow_mem(const u8 *addr, size_t size_bytes, size_t bytes_per_line,
 350                        size_t shadow_value_type_ids) {
 351   const u8 *const shadow_type = GetShadowTypeAddrFor(addr);
 352   const u8 *const shadow = GetShadowAddrFor(addr);
 353
 354   constexpr int kMaxNumDecodedValues = 16;
 355   __float128 decoded_values[kMaxNumDecodedValues];
 356   int num_decoded_values = 0;
 357   if (bytes_per_line > 4 * kMaxNumDecodedValues)
 358     bytes_per_line = 4 * kMaxNumDecodedValues;
 359
 360   // We keep track of the current type and position as we go.
 361   ValueType LastValueTy = kUnknownValueType;
 362   int LastPos = -1;
 363   size_t Offset = 0;
 364   for (size_t R = 0; R < (size_bytes + bytes_per_line - 1) / bytes_per_line;
 365        ++R) {
 366     printf("%p:    ", (void *)(addr + R * bytes_per_line));
 367     for (size_t C = 0; C < bytes_per_line && Offset < size_bytes; ++C) {
 368       const ValueType ValueTy = getValueType(shadow_type[Offset]);
 369       const int pos = getValuePos(shadow_type[Offset]);
 370       if (ValueTy == LastValueTy && pos == LastPos + 1) {
 371         ++LastPos;
 372       } else {
 373         LastValueTy = ValueTy;
 374         LastPos = pos == 0 ? 0 : -1;
 375       }
 376
 377       switch (ValueTy) {
 378       case kUnknownValueType:
 379         printf("__ ");
 380         break;
 381       case kFloatValueType:
 382         printf("f%x ", pos);
 383         if (LastPos == sizeof(float) - 1) {
 384           decoded_values[num_decoded_values] =
 385               ReadShadow(shadow + kShadowScale * (Offset + 1 - sizeof(float)),
 386                          static_cast<char>(shadow_value_type_ids & 0xff));
 387           ++num_decoded_values;
 388         }
 389         break;
 390       case kDoubleValueType:
 391         printf("d%x ", pos);
 392         if (LastPos == sizeof(double) - 1) {
 393           decoded_values[num_decoded_values] = ReadShadow(
 394               shadow + kShadowScale * (Offset + 1 - sizeof(double)),
 395               static_cast<char>((shadow_value_type_ids >> 8) & 0xff));
 396           ++num_decoded_values;
 397         }
 398         break;
 399       case kFp80ValueType:
 400         printf("l%x ", pos);
 401         if (LastPos == sizeof(long double) - 1) {
 402           decoded_values[num_decoded_values] = ReadShadow(
 403               shadow + kShadowScale * (Offset + 1 - sizeof(long double)),
 404               static_cast<char>((shadow_value_type_ids >> 16) & 0xff));
 405           ++num_decoded_values;
 406         }
 407         break;
 408       }
 409       ++Offset;
 410     }
 411     for (int i = 0; i < num_decoded_values; ++i) {
 412       printf("  (%s)", FTPrinter<__float128>::dec(decoded_values[i]).Buffer);
 413     }
 414     num_decoded_values = 0;
 415     printf("\n");
 416   }
 417 }
 418
 419 alignas(64) SANITIZER_INTERFACE_ATTRIBUTE
 420     thread_local uptr __nsan_shadow_ret_tag = 0;
 421
 422 alignas(64) SANITIZER_INTERFACE_ATTRIBUTE
 423     thread_local char __nsan_shadow_ret_ptr[kMaxVectorWidth *
 424                                             sizeof(__float128)];
 425
 426 alignas(64) SANITIZER_INTERFACE_ATTRIBUTE
 427     thread_local uptr __nsan_shadow_args_tag = 0;
 428
 429 // Maximum number of args. This should be enough for anyone (tm). An alternate
 430 // scheme is to have the generated code create an alloca and make
 431 // __nsan_shadow_args_ptr point ot the alloca.
 432 constexpr const int kMaxNumArgs = 128;
 433 alignas(64) SANITIZER_INTERFACE_ATTRIBUTE
 434     thread_local char __nsan_shadow_args_ptr[kMaxVectorWidth * kMaxNumArgs *
 435                                              sizeof(__float128)];
 436
 437 enum ContinuationType { // Keep in sync with instrumentation pass.
 438   kContinueWithShadow = 0,
 439   kResumeFromValue = 1,
 440 };
 441
 442 // Checks the consistency between application and shadow value. Returns true
 443 // when the instrumented code should resume computations from the original value
 444 // rather than the shadow value. This prevents one error to propagate to all
 445 // subsequent operations. This behaviour is tunable with flags.
 446 template <typename FT, typename ShadowFT>
 447 int32_t checkFT(const FT value, ShadowFT Shadow, CheckTypeT CheckType,
 448                 uptr CheckArg) {
 449   // We do all comparisons in the InternalFT domain, which is the largest FT
 450   // type.
 451   using InternalFT = LargestFT<FT, ShadowFT>;
 452   const InternalFT check_value = value;
 453   const InternalFT check_shadow = Shadow;
 454
 455   // We only check for NaNs in the value, not the shadow.
 456   if (flags().check_nan && isnan(value)) {
 457     GET_CALLER_PC_BP;
 458     BufferedStackTrace stack;
 459     stack.Unwind(pc, bp, nullptr, false);
 460     if (GetSuppressionForStack(&stack, CheckKind::Consistency)) {
 461       // FIXME: optionally print.
 462       return flags().resume_after_suppression ? kResumeFromValue
 463                                               : kContinueWithShadow;
 464     }
 465     Decorator D;
 466     Printf("%s", D.Warning());
 467     Printf("WARNING: NumericalStabilitySanitizer: NaN detected\n");
 468     Printf("%s", D.Default());
 469     stack.Print();
 470     if (flags().halt_on_error) {
 471       if (common_flags()->abort_on_error)
 472         Printf("ABORTING\n");
 473       else
 474         Printf("Exiting\n");
 475       Die();
 476     }
 477     // Performing other tests for NaN values is meaningless when dealing with numbers.
 478     return kResumeFromValue;
 479   }
 480
 481   // See this article for an interesting discussion of how to compare floats:
 482   // https://randomascii.wordpress.com/2012/02/25/comparing-floating-point-numbers-2012-edition/
 483   static constexpr const FT Eps = FTInfo<FT>::kEpsilon;
 484
 485   const InternalFT abs_err = ftAbs(check_value - check_shadow);
 486
 487   if (flags().enable_check_stats) {
 488     GET_CALLER_PC_BP;
 489     // We are re-computing `largest` here because this is a cold branch, and we
 490     // want to avoid having to move the computation of `largest` before the
 491     // absolute value check when this branch is not taken.
 492     const InternalFT largest = max(ftAbs(check_value), ftAbs(check_shadow));
 493     nsan_stats->AddCheck(CheckType, pc, bp, abs_err / largest);
 494   }
 495
 496   // Note: writing the comparison that way ensures that when `abs_err` is Nan
 497   // (value and shadow are inf or -inf), we pass the test.
 498   if (!(abs_err >= flags().cached_absolute_error_threshold))
 499     return kContinueWithShadow;
 500
 501   const InternalFT largest = max(ftAbs(check_value), ftAbs(check_shadow));
 502   if (abs_err * (1ull << flags().log2_max_relative_error) <= largest)
 503     return kContinueWithShadow; // No problem here.
 504
 505   if (!flags().disable_warnings) {
 506     GET_CALLER_PC_BP;
 507     UNINITIALIZED BufferedStackTrace stack;
 508     stack.Unwind(pc, bp, nullptr, false);
 509     if (GetSuppressionForStack(&stack, CheckKind::Consistency)) {
 510       // FIXME: optionally print.
 511       return flags().resume_after_suppression ? kResumeFromValue
 512                                               : kContinueWithShadow;
 513     }
 514
 515     Decorator D;
 516     Printf("%s", D.Warning());
 517     // Printf does not support float formatting.
 518     char RelErrBuf[64] = "inf";
 519     if (largest > Eps) {
 520       snprintf(RelErrBuf, sizeof(RelErrBuf) - 1, "%.20Lf%% (2^%.0Lf epsilons)",
 521                static_cast<long double>(100.0 * abs_err / largest),
 522                log2l(static_cast<long double>(abs_err / largest / Eps)));
 523     }
 524     char ulp_err_buf[128] = "";
 525     const double shadow_ulp_diff = GetULPDiff(check_value, check_shadow);
 526     if (shadow_ulp_diff != kMaxULPDiff) {
 527       // This is the ULP diff in the internal domain. The user actually cares
 528       // about that in the original domain.
 529       const double ulp_diff =
 530           shadow_ulp_diff / (u64{1} << (FTInfo<InternalFT>::kMantissaBits -
 531                                         FTInfo<FT>::kMantissaBits));
 532       snprintf(ulp_err_buf, sizeof(ulp_err_buf) - 1,
 533                "(%.0f ULPs == %.1f digits == %.1f bits)", ulp_diff,
 534                log10(ulp_diff), log2(ulp_diff));
 535     }
 536     Printf("WARNING: NumericalStabilitySanitizer: inconsistent shadow results");
 537     switch (CheckType) {
 538     case CheckTypeT::kUnknown:
 539     case CheckTypeT::kFcmp:
 540     case CheckTypeT::kMaxCheckType:
 541       break;
 542     case CheckTypeT::kRet:
 543       Printf(" while checking return value");
 544       break;
 545     case CheckTypeT::kArg:
 546       Printf(" while checking call argument #%d", static_cast<int>(CheckArg));
 547       break;
 548     case CheckTypeT::kLoad:
 549       Printf(
 550           " while checking load from address 0x%lx. This is due to incorrect "
 551           "shadow memory tracking, typically due to uninstrumented code "
 552           "writing to memory.",
 553           CheckArg);
 554       break;
 555     case CheckTypeT::kStore:
 556       Printf(" while checking store to address 0x%lx", CheckArg);
 557       break;
 558     case CheckTypeT::kInsert:
 559       Printf(" while checking vector insert");
 560       break;
 561     case CheckTypeT::kUser:
 562       Printf(" in user-initiated check");
 563       break;
 564     }
 565     using ValuePrinter = FTPrinter<FT>;
 566     using ShadowPrinter = FTPrinter<ShadowFT>;
 567     Printf("%s", D.Default());
 568
 569     Printf("\n"
 570            "%-12s precision  (native): dec: %s  hex: %s\n"
 571            "%-12s precision  (shadow): dec: %s  hex: %s\n"
 572            "shadow truncated to %-12s: dec: %s  hex: %s\n"
 573            "Relative error: %s\n"
 574            "Absolute error: %s\n"
 575            "%s\n",
 576            FTInfo<FT>::kCppTypeName, ValuePrinter::dec(value).Buffer,
 577            ValuePrinter::hex(value).Buffer, FTInfo<ShadowFT>::kCppTypeName,
 578            ShadowPrinter::dec(Shadow).Buffer, ShadowPrinter::hex(Shadow).Buffer,
 579            FTInfo<FT>::kCppTypeName, ValuePrinter::dec(Shadow).Buffer,
 580            ValuePrinter::hex(Shadow).Buffer, RelErrBuf,
 581            ValuePrinter::hex(abs_err).Buffer, ulp_err_buf);
 582     stack.Print();
 583   }
 584
 585   if (flags().enable_warning_stats) {
 586     GET_CALLER_PC_BP;
 587     nsan_stats->AddWarning(CheckType, pc, bp, abs_err / largest);
 588   }
 589
 590   if (flags().halt_on_error) {
 591     if (common_flags()->abort_on_error)
 592       Printf("ABORTING\n");
 593     else
 594       Printf("Exiting\n");
 595     Die();
 596   }
 597   return flags().resume_after_warning ? kResumeFromValue : kContinueWithShadow;
 598 }
 599
 600 extern "C" SANITIZER_INTERFACE_ATTRIBUTE int32_t __nsan_internal_check_float_d(
 601     float value, double shadow, int32_t check_type, uptr check_arg) {
 602   return checkFT(value, shadow, static_cast<CheckTypeT>(check_type), check_arg);
 603 }
 604
 605 extern "C" SANITIZER_INTERFACE_ATTRIBUTE int32_t __nsan_internal_check_double_l(
 606     double value, long double shadow, int32_t check_type, uptr check_arg) {
 607   return checkFT(value, shadow, static_cast<CheckTypeT>(check_type), check_arg);
 608 }
 609
 610 extern "C" SANITIZER_INTERFACE_ATTRIBUTE int32_t __nsan_internal_check_double_q(
 611     double value, __float128 shadow, int32_t check_type, uptr check_arg) {
 612   return checkFT(value, shadow, static_cast<CheckTypeT>(check_type), check_arg);
 613 }
 614
 615 extern "C" SANITIZER_INTERFACE_ATTRIBUTE int32_t
 616 __nsan_internal_check_longdouble_q(long double value, __float128 shadow,
 617                                    int32_t check_type, uptr check_arg) {
 618   return checkFT(value, shadow, static_cast<CheckTypeT>(check_type), check_arg);
 619 }
 620
 621 static const char *GetTruthValueName(bool v) { return v ? "true" : "false"; }
 622
 623 // This uses the same values as CmpInst::Predicate.
 624 static const char *GetPredicateName(int v) {
 625   switch (v) {
 626   case 0:
 627     return "(false)";
 628   case 1:
 629     return "==";
 630   case 2:
 631     return ">";
 632   case 3:
 633     return ">=";
 634   case 4:
 635     return "<";
 636   case 5:
 637     return "<=";
 638   case 6:
 639     return "!=";
 640   case 7:
 641     return "(ordered)";
 642   case 8:
 643     return "(unordered)";
 644   case 9:
 645     return "==";
 646   case 10:
 647     return ">";
 648   case 11:
 649     return ">=";
 650   case 12:
 651     return "<";
 652   case 13:
 653     return "<=";
 654   case 14:
 655     return "!=";
 656   case 15:
 657     return "(true)";
 658   }
 659   return "??";
 660 }
 661
 662 template <typename FT, typename ShadowFT>
 663 void fCmpFailFT(const FT Lhs, const FT Rhs, ShadowFT LhsShadow,
 664                 ShadowFT RhsShadow, int Predicate, bool result,
 665                 bool ShadowResult) {
 666   if (result == ShadowResult) {
 667     // When a vector comparison fails, we fail each element of the comparison
 668     // to simplify instrumented code. Skip elements where the shadow comparison
 669     // gave the same result as the original one.
 670     return;
 671   }
 672
 673   GET_CALLER_PC_BP;
 674   UNINITIALIZED BufferedStackTrace stack;
 675   stack.Unwind(pc, bp, nullptr, false);
 676
 677   if (GetSuppressionForStack(&stack, CheckKind::Fcmp)) {
 678     // FIXME: optionally print.
 679     return;
 680   }
 681
 682   if (flags().enable_warning_stats)
 683     nsan_stats->AddWarning(CheckTypeT::kFcmp, pc, bp, 0.0);
 684
 685   if (flags().disable_warnings || !flags().check_cmp)
 686     return;
 687
 688   // FIXME: ideally we would print the shadow value as FP128. Right now because
 689   // we truncate to long double we can sometimes see stuff like:
 690   // shadow <value> == <value> (false)
 691   using ValuePrinter = FTPrinter<FT>;
 692   using ShadowPrinter = FTPrinter<ShadowFT>;
 693   Decorator D;
 694   const char *const PredicateName = GetPredicateName(Predicate);
 695   Printf("%s", D.Warning());
 696   Printf("WARNING: NumericalStabilitySanitizer: floating-point comparison "
 697          "results depend on precision\n");
 698   Printf("%s", D.Default());
 699   Printf("%-12s precision dec (native): %s %s %s (%s)\n"
 700          "%-12s precision dec (shadow): %s %s %s (%s)\n"
 701          "%-12s precision hex (native): %s %s %s (%s)\n"
 702          "%-12s precision hex (shadow): %s %s %s (%s)\n"
 703          "%s",
 704          // Native, decimal.
 705          FTInfo<FT>::kCppTypeName, ValuePrinter::dec(Lhs).Buffer, PredicateName,
 706          ValuePrinter::dec(Rhs).Buffer, GetTruthValueName(result),
 707          // Shadow, decimal
 708          FTInfo<ShadowFT>::kCppTypeName, ShadowPrinter::dec(LhsShadow).Buffer,
 709          PredicateName, ShadowPrinter::dec(RhsShadow).Buffer,
 710          GetTruthValueName(ShadowResult),
 711          // Native, hex.
 712          FTInfo<FT>::kCppTypeName, ValuePrinter::hex(Lhs).Buffer, PredicateName,
 713          ValuePrinter::hex(Rhs).Buffer, GetTruthValueName(result),
 714          // Shadow, hex
 715          FTInfo<ShadowFT>::kCppTypeName, ShadowPrinter::hex(LhsShadow).Buffer,
 716          PredicateName, ShadowPrinter::hex(RhsShadow).Buffer,
 717          GetTruthValueName(ShadowResult), D.End());
 718   stack.Print();
 719   if (flags().halt_on_error) {
 720     Printf("Exiting\n");
 721     Die();
 722   }
 723 }
 724
 725 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
 726 __nsan_fcmp_fail_float_d(float lhs, float rhs, double lhs_shadow,
 727                          double rhs_shadow, int predicate, bool result,
 728                          bool shadow_result) {
 729   fCmpFailFT(lhs, rhs, lhs_shadow, rhs_shadow, predicate, result,
 730              shadow_result);
 731 }
 732
 733 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
 734 __nsan_fcmp_fail_double_q(double lhs, double rhs, __float128 lhs_shadow,
 735                           __float128 rhs_shadow, int predicate, bool result,
 736                           bool shadow_result) {
 737   fCmpFailFT(lhs, rhs, lhs_shadow, rhs_shadow, predicate, result,
 738              shadow_result);
 739 }
 740
 741 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
 742 __nsan_fcmp_fail_double_l(double lhs, double rhs, long double lhs_shadow,
 743                           long double rhs_shadow, int predicate, bool result,
 744                           bool shadow_result) {
 745   fCmpFailFT(lhs, rhs, lhs_shadow, rhs_shadow, predicate, result,
 746              shadow_result);
 747 }
 748
 749 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
 750 __nsan_fcmp_fail_longdouble_q(long double lhs, long double rhs,
 751                               __float128 lhs_shadow, __float128 rhs_shadow,
 752                               int predicate, bool result, bool shadow_result) {
 753   fCmpFailFT(lhs, rhs, lhs_shadow, rhs_shadow, predicate, result,
 754              shadow_result);
 755 }
 756
 757 template <typename FT> void checkFTFromShadowStack(const FT value) {
 758   // Get the shadow 2FT value from the shadow stack. Note that
 759   // __nsan_check_{float,double,long double} is a function like any other, so
 760   // the instrumentation will have placed the shadow value on the shadow stack.
 761   using ShadowFT = typename FTInfo<FT>::shadow_type;
 762   ShadowFT Shadow;
 763   __builtin_memcpy(&Shadow, __nsan_shadow_args_ptr, sizeof(ShadowFT));
 764   checkFT(value, Shadow, CheckTypeT::kUser, 0);
 765 }
 766
 767 // FIXME: Add suffixes and let the instrumentation pass automatically add
 768 // suffixes.
 769 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_check_float(float value) {
 770   assert(__nsan_shadow_args_tag == (uptr)&__nsan_check_float &&
 771          "__nsan_check_float called from non-instrumented function");
 772   checkFTFromShadowStack(value);
 773 }
 774
 775 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
 776 __nsan_check_double(double value) {
 777   assert(__nsan_shadow_args_tag == (uptr)&__nsan_check_double &&
 778          "__nsan_check_double called from non-instrumented function");
 779   checkFTFromShadowStack(value);
 780 }
 781
 782 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
 783 __nsan_check_longdouble(long double value) {
 784   assert(__nsan_shadow_args_tag == (uptr)&__nsan_check_longdouble &&
 785          "__nsan_check_longdouble called from non-instrumented function");
 786   checkFTFromShadowStack(value);
 787 }
 788
 789 template <typename FT> static void dumpFTFromShadowStack(const FT value) {
 790   // Get the shadow 2FT value from the shadow stack. Note that
 791   // __nsan_dump_{float,double,long double} is a function like any other, so
 792   // the instrumentation will have placed the shadow value on the shadow stack.
 793   using ShadowFT = typename FTInfo<FT>::shadow_type;
 794   ShadowFT shadow;
 795   __builtin_memcpy(&shadow, __nsan_shadow_args_ptr, sizeof(ShadowFT));
 796   using ValuePrinter = FTPrinter<FT>;
 797   using ShadowPrinter = FTPrinter<typename FTInfo<FT>::shadow_type>;
 798   printf("value  dec:%s hex:%s\n"
 799          "shadow dec:%s hex:%s\n",
 800          ValuePrinter::dec(value).Buffer, ValuePrinter::hex(value).Buffer,
 801          ShadowPrinter::dec(shadow).Buffer, ShadowPrinter::hex(shadow).Buffer);
 802 }
 803
 804 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_float(float value) {
 805   assert(__nsan_shadow_args_tag == (uptr)&__nsan_dump_float &&
 806          "__nsan_dump_float called from non-instrumented function");
 807   dumpFTFromShadowStack(value);
 808 }
 809
 810 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_double(double value) {
 811   assert(__nsan_shadow_args_tag == (uptr)&__nsan_dump_double &&
 812          "__nsan_dump_double called from non-instrumented function");
 813   dumpFTFromShadowStack(value);
 814 }
 815
 816 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
 817 __nsan_dump_longdouble(long double value) {
 818   assert(__nsan_shadow_args_tag == (uptr)&__nsan_dump_longdouble &&
 819          "__nsan_dump_longdouble called from non-instrumented function");
 820   dumpFTFromShadowStack(value);
 821 }
 822
 823 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_shadow_ret() {
 824   printf("ret tag: %lx\n", __nsan_shadow_ret_tag);
 825   double v;
 826   __builtin_memcpy(&v, __nsan_shadow_ret_ptr, sizeof(double));
 827   printf("double value: %f\n", v);
 828   // FIXME: float128 value.
 829 }
 830
 831 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_shadow_args() {
 832   printf("args tag: %lx\n", __nsan_shadow_args_tag);
 833 }
 834
 835 bool __nsan::nsan_initialized;
 836 bool __nsan::nsan_init_is_running;
 837
 838 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_init() {
 839   CHECK(!nsan_init_is_running);
 840   if (nsan_initialized)
 841     return;
 842   nsan_init_is_running = true;
 843   SanitizerToolName = "NumericalStabilitySanitizer";
 844
 845   InitializeFlags();
 846   InitializeSuppressions();
 847   InitializePlatformEarly();
 848
 849   DisableCoreDumperIfNecessary();
 850
 851   if (!MmapFixedNoReserve(TypesAddr(), AllocatorAddr() - TypesAddr()))
 852     Die();
 853
 854   InitializeInterceptors();
 855   NsanTSDInit(NsanTSDDtor);
 856   NsanAllocatorInit();
 857
 858   NsanThread *main_thread = NsanThread::Create(nullptr, nullptr);
 859   SetCurrentThread(main_thread);
 860   main_thread->Init();
 861
 862   InitializeStats();
 863   if (flags().print_stats_on_exit)
 864     Atexit(NsanAtexit);
 865
 866   nsan_init_is_running = false;
 867   nsan_initialized = true;
 868 }