compiler-rt/lib/hwasan/hwasan_linux.cpp

   1 //===-- hwasan_linux.cpp ----------------------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 ///
   9 /// \file
  10 /// This file is a part of HWAddressSanitizer and contains Linux-, NetBSD- and
  11 /// FreeBSD-specific code.
  12 ///
  13 //===----------------------------------------------------------------------===//
  14
  15 #include "sanitizer_common/sanitizer_platform.h"
  16 #if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
  17
  18 #  include <dlfcn.h>
  19 #  include <elf.h>
  20 #  include <errno.h>
  21 #  include <link.h>
  22 #  include <pthread.h>
  23 #  include <signal.h>
  24 #  include <stdio.h>
  25 #  include <stdlib.h>
  26 #  include <sys/prctl.h>
  27 #  include <sys/resource.h>
  28 #  include <sys/time.h>
  29 #  include <unistd.h>
  30 #  include <unwind.h>
  31
  32 #  include "hwasan.h"
  33 #  include "hwasan_dynamic_shadow.h"
  34 #  include "hwasan_interface_internal.h"
  35 #  include "hwasan_mapping.h"
  36 #  include "hwasan_report.h"
  37 #  include "hwasan_thread.h"
  38 #  include "hwasan_thread_list.h"
  39 #  include "sanitizer_common/sanitizer_common.h"
  40 #  include "sanitizer_common/sanitizer_procmaps.h"
  41 #  include "sanitizer_common/sanitizer_stackdepot.h"
  42
  43 // Configurations of HWASAN_WITH_INTERCEPTORS and SANITIZER_ANDROID.
  44 //
  45 // HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=OFF
  46 //   Not currently tested.
  47 // HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=ON
  48 //   Integration tests downstream exist.
  49 // HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=OFF
  50 //    Tested with check-hwasan on x86_64-linux.
  51 // HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=ON
  52 //    Tested with check-hwasan on aarch64-linux-android.
  53 #  if !SANITIZER_ANDROID
  54 SANITIZER_INTERFACE_ATTRIBUTE
  55 THREADLOCAL uptr __hwasan_tls;
  56 #  endif
  57
  58 namespace __hwasan {
  59
  60 // With the zero shadow base we can not actually map pages starting from 0.
  61 // This constant is somewhat arbitrary.
  62 constexpr uptr kZeroBaseShadowStart = 0;
  63 constexpr uptr kZeroBaseMaxShadowStart = 1 << 18;
  64
  65 static void ProtectGap(uptr addr, uptr size) {
  66   __sanitizer::ProtectGap(addr, size, kZeroBaseShadowStart,
  67                           kZeroBaseMaxShadowStart);
  68 }
  69
  70 uptr kLowMemStart;
  71 uptr kLowMemEnd;
  72 uptr kHighMemStart;
  73 uptr kHighMemEnd;
  74
  75 static void PrintRange(uptr start, uptr end, const char *name) {
  76   Printf("|| [%p, %p] || %.*s ||\n", (void *)start, (void *)end, 10, name);
  77 }
  78
  79 static void PrintAddressSpaceLayout() {
  80   PrintRange(kHighMemStart, kHighMemEnd, "HighMem");
  81   if (kHighShadowEnd + 1 < kHighMemStart)
  82     PrintRange(kHighShadowEnd + 1, kHighMemStart - 1, "ShadowGap");
  83   else
  84     CHECK_EQ(kHighShadowEnd + 1, kHighMemStart);
  85   PrintRange(kHighShadowStart, kHighShadowEnd, "HighShadow");
  86   if (kLowShadowEnd + 1 < kHighShadowStart)
  87     PrintRange(kLowShadowEnd + 1, kHighShadowStart - 1, "ShadowGap");
  88   else
  89     CHECK_EQ(kLowMemEnd + 1, kHighShadowStart);
  90   PrintRange(kLowShadowStart, kLowShadowEnd, "LowShadow");
  91   if (kLowMemEnd + 1 < kLowShadowStart)
  92     PrintRange(kLowMemEnd + 1, kLowShadowStart - 1, "ShadowGap");
  93   else
  94     CHECK_EQ(kLowMemEnd + 1, kLowShadowStart);
  95   PrintRange(kLowMemStart, kLowMemEnd, "LowMem");
  96   CHECK_EQ(0, kLowMemStart);
  97 }
  98
  99 static uptr GetHighMemEnd() {
 100   // HighMem covers the upper part of the address space.
 101   uptr max_address = GetMaxUserVirtualAddress();
 102   // Adjust max address to make sure that kHighMemEnd and kHighMemStart are
 103   // properly aligned:
 104   max_address |= (GetMmapGranularity() << kShadowScale) - 1;
 105   return max_address;
 106 }
 107
 108 static void InitializeShadowBaseAddress(uptr shadow_size_bytes) {
 109   __hwasan_shadow_memory_dynamic_address =
 110       FindDynamicShadowStart(shadow_size_bytes);
 111 }
 112
 113 static void MaybeDieIfNoTaggingAbi(const char *message) {
 114   if (!flags()->fail_without_syscall_abi)
 115     return;
 116   Printf("FATAL: %s\n", message);
 117   Die();
 118 }
 119
 120 #  define PR_SET_TAGGED_ADDR_CTRL 55
 121 #  define PR_GET_TAGGED_ADDR_CTRL 56
 122 #  define PR_TAGGED_ADDR_ENABLE (1UL << 0)
 123 #  define ARCH_GET_UNTAG_MASK 0x4001
 124 #  define ARCH_ENABLE_TAGGED_ADDR 0x4002
 125 #  define ARCH_GET_MAX_TAG_BITS 0x4003
 126
 127 static bool CanUseTaggingAbi() {
 128 #  if defined(__x86_64__)
 129   unsigned long num_bits = 0;
 130   // Check for x86 LAM support. This API is based on a currently unsubmitted
 131   // patch to the Linux kernel (as of August 2022) and is thus subject to
 132   // change. The patch is here:
 133   // https://lore.kernel.org/all/20220815041803.17954-1-kirill.shutemov@linux.intel.com/
 134   //
 135   // arch_prctl(ARCH_GET_MAX_TAG_BITS, &bits) returns the maximum number of tag
 136   // bits the user can request, or zero if LAM is not supported by the hardware.
 137   if (internal_iserror(internal_arch_prctl(ARCH_GET_MAX_TAG_BITS,
 138                                            reinterpret_cast<uptr>(&num_bits))))
 139     return false;
 140   // The platform must provide enough bits for HWASan tags.
 141   if (num_bits < kTagBits)
 142     return false;
 143   return true;
 144 #  else
 145   // Check for ARM TBI support.
 146   return !internal_iserror(internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0));
 147 #  endif // __x86_64__
 148 }
 149
 150 static bool EnableTaggingAbi() {
 151 #  if defined(__x86_64__)
 152   // Enable x86 LAM tagging for the process.
 153   //
 154   // arch_prctl(ARCH_ENABLE_TAGGED_ADDR, bits) enables tagging if the number of
 155   // tag bits requested by the user does not exceed that provided by the system.
 156   // arch_prctl(ARCH_GET_UNTAG_MASK, &mask) returns the mask of significant
 157   // address bits. It is ~0ULL if either LAM is disabled for the process or LAM
 158   // is not supported by the hardware.
 159   if (internal_iserror(internal_arch_prctl(ARCH_ENABLE_TAGGED_ADDR, kTagBits)))
 160     return false;
 161   unsigned long mask = 0;
 162   // Make sure the tag bits are where we expect them to be.
 163   if (internal_iserror(internal_arch_prctl(ARCH_GET_UNTAG_MASK,
 164                                            reinterpret_cast<uptr>(&mask))))
 165     return false;
 166   // @mask has ones for non-tag bits, whereas @kAddressTagMask has ones for tag
 167   // bits. Therefore these masks must not overlap.
 168   if (mask & kAddressTagMask)
 169     return false;
 170   return true;
 171 #  else
 172   // Enable ARM TBI tagging for the process. If for some reason tagging is not
 173   // supported, prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE) returns
 174   // -EINVAL.
 175   if (internal_iserror(internal_prctl(PR_SET_TAGGED_ADDR_CTRL,
 176                                       PR_TAGGED_ADDR_ENABLE, 0, 0, 0)))
 177     return false;
 178   // Ensure that TBI is enabled.
 179   if (internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0) !=
 180       PR_TAGGED_ADDR_ENABLE)
 181     return false;
 182   return true;
 183 #  endif // __x86_64__
 184 }
 185
 186 void InitializeOsSupport() {
 187   // Check we're running on a kernel that can use the tagged address ABI.
 188   bool has_abi = CanUseTaggingAbi();
 189
 190   if (!has_abi) {
 191 #  if SANITIZER_ANDROID || defined(HWASAN_ALIASING_MODE)
 192     // Some older Android kernels have the tagged pointer ABI on
 193     // unconditionally, and hence don't have the tagged-addr prctl while still
 194     // allow the ABI.
 195     // If targeting Android and the prctl is not around we assume this is the
 196     // case.
 197     return;
 198 #  else
 199     MaybeDieIfNoTaggingAbi(
 200         "HWAddressSanitizer requires a kernel with tagged address ABI.");
 201 #  endif
 202   }
 203
 204   if (EnableTaggingAbi())
 205     return;
 206
 207 #  if SANITIZER_ANDROID
 208   MaybeDieIfNoTaggingAbi(
 209       "HWAddressSanitizer failed to enable tagged address syscall ABI.\n"
 210       "Check the `sysctl abi.tagged_addr_disabled` configuration.");
 211 #  else
 212   MaybeDieIfNoTaggingAbi(
 213       "HWAddressSanitizer failed to enable tagged address syscall ABI.\n");
 214 #  endif
 215 }
 216
 217 bool InitShadow() {
 218   // Define the entire memory range.
 219   kHighMemEnd = GetHighMemEnd();
 220
 221   // Determine shadow memory base offset.
 222   InitializeShadowBaseAddress(MemToShadowSize(kHighMemEnd));
 223
 224   // Place the low memory first.
 225   kLowMemEnd = __hwasan_shadow_memory_dynamic_address - 1;
 226   kLowMemStart = 0;
 227
 228   // Define the low shadow based on the already placed low memory.
 229   kLowShadowEnd = MemToShadow(kLowMemEnd);
 230   kLowShadowStart = __hwasan_shadow_memory_dynamic_address;
 231
 232   // High shadow takes whatever memory is left up there (making sure it is not
 233   // interfering with low memory in the fixed case).
 234   kHighShadowEnd = MemToShadow(kHighMemEnd);
 235   kHighShadowStart = Max(kLowMemEnd, MemToShadow(kHighShadowEnd)) + 1;
 236
 237   // High memory starts where allocated shadow allows.
 238   kHighMemStart = ShadowToMem(kHighShadowStart);
 239
 240   // Check the sanity of the defined memory ranges (there might be gaps).
 241   CHECK_EQ(kHighMemStart % GetMmapGranularity(), 0);
 242   CHECK_GT(kHighMemStart, kHighShadowEnd);
 243   CHECK_GT(kHighShadowEnd, kHighShadowStart);
 244   CHECK_GT(kHighShadowStart, kLowMemEnd);
 245   CHECK_GT(kLowMemEnd, kLowMemStart);
 246   CHECK_GT(kLowShadowEnd, kLowShadowStart);
 247   CHECK_GT(kLowShadowStart, kLowMemEnd);
 248
 249   if (Verbosity())
 250     PrintAddressSpaceLayout();
 251
 252   // Reserve shadow memory.
 253   ReserveShadowMemoryRange(kLowShadowStart, kLowShadowEnd, "low shadow");
 254   ReserveShadowMemoryRange(kHighShadowStart, kHighShadowEnd, "high shadow");
 255
 256   // Protect all the gaps.
 257   ProtectGap(0, Min(kLowMemStart, kLowShadowStart));
 258   if (kLowMemEnd + 1 < kLowShadowStart)
 259     ProtectGap(kLowMemEnd + 1, kLowShadowStart - kLowMemEnd - 1);
 260   if (kLowShadowEnd + 1 < kHighShadowStart)
 261     ProtectGap(kLowShadowEnd + 1, kHighShadowStart - kLowShadowEnd - 1);
 262   if (kHighShadowEnd + 1 < kHighMemStart)
 263     ProtectGap(kHighShadowEnd + 1, kHighMemStart - kHighShadowEnd - 1);
 264
 265   return true;
 266 }
 267
 268 void InitThreads() {
 269   CHECK(__hwasan_shadow_memory_dynamic_address);
 270   uptr guard_page_size = GetMmapGranularity();
 271   uptr thread_space_start =
 272       __hwasan_shadow_memory_dynamic_address - (1ULL << kShadowBaseAlignment);
 273   uptr thread_space_end =
 274       __hwasan_shadow_memory_dynamic_address - guard_page_size;
 275   ReserveShadowMemoryRange(thread_space_start, thread_space_end - 1,
 276                            "hwasan threads", /*madvise_shadow*/ false);
 277   ProtectGap(thread_space_end,
 278              __hwasan_shadow_memory_dynamic_address - thread_space_end);
 279   InitThreadList(thread_space_start, thread_space_end - thread_space_start);
 280   hwasanThreadList().CreateCurrentThread();
 281 }
 282
 283 bool MemIsApp(uptr p) {
 284 // Memory outside the alias range has non-zero tags.
 285 #  if !defined(HWASAN_ALIASING_MODE)
 286   CHECK(GetTagFromPointer(p) == 0);
 287 #  endif
 288
 289   return (p >= kHighMemStart && p <= kHighMemEnd) ||
 290          (p >= kLowMemStart && p <= kLowMemEnd);
 291 }
 292
 293 void InstallAtExitHandler() { atexit(HwasanAtExit); }
 294
 295 // ---------------------- TSD ---------------- {{{1
 296
 297 extern "C" void __hwasan_thread_enter() {
 298   hwasanThreadList().CreateCurrentThread()->EnsureRandomStateInited();
 299 }
 300
 301 extern "C" void __hwasan_thread_exit() {
 302   Thread *t = GetCurrentThread();
 303   // Make sure that signal handler can not see a stale current thread pointer.
 304   atomic_signal_fence(memory_order_seq_cst);
 305   if (t)
 306     hwasanThreadList().ReleaseThread(t);
 307 }
 308
 309 #  if HWASAN_WITH_INTERCEPTORS
 310 static pthread_key_t tsd_key;
 311 static bool tsd_key_inited = false;
 312
 313 void HwasanTSDThreadInit() {
 314   if (tsd_key_inited)
 315     CHECK_EQ(0, pthread_setspecific(tsd_key,
 316                                     (void *)GetPthreadDestructorIterations()));
 317 }
 318
 319 void HwasanTSDDtor(void *tsd) {
 320   uptr iterations = (uptr)tsd;
 321   if (iterations > 1) {
 322     CHECK_EQ(0, pthread_setspecific(tsd_key, (void *)(iterations - 1)));
 323     return;
 324   }
 325   __hwasan_thread_exit();
 326 }
 327
 328 void HwasanTSDInit() {
 329   CHECK(!tsd_key_inited);
 330   tsd_key_inited = true;
 331   CHECK_EQ(0, pthread_key_create(&tsd_key, HwasanTSDDtor));
 332 }
 333 #  else
 334 void HwasanTSDInit() {}
 335 void HwasanTSDThreadInit() {}
 336 #  endif
 337
 338 #  if SANITIZER_ANDROID
 339 uptr *GetCurrentThreadLongPtr() { return (uptr *)get_android_tls_ptr(); }
 340 #  else
 341 uptr *GetCurrentThreadLongPtr() { return &__hwasan_tls; }
 342 #  endif
 343
 344 #  if SANITIZER_ANDROID
 345 void AndroidTestTlsSlot() {
 346   uptr kMagicValue = 0x010203040A0B0C0D;
 347   uptr *tls_ptr = GetCurrentThreadLongPtr();
 348   uptr old_value = *tls_ptr;
 349   *tls_ptr = kMagicValue;
 350   dlerror();
 351   if (*(uptr *)get_android_tls_ptr() != kMagicValue) {
 352     Printf(
 353         "ERROR: Incompatible version of Android: TLS_SLOT_SANITIZER(6) is used "
 354         "for dlerror().\n");
 355     Die();
 356   }
 357   *tls_ptr = old_value;
 358 }
 359 #  else
 360 void AndroidTestTlsSlot() {}
 361 #  endif
 362
 363 static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
 364   // Access type is passed in a platform dependent way (see below) and encoded
 365   // as 0xXY, where X&1 is 1 for store, 0 for load, and X&2 is 1 if the error is
 366   // recoverable. Valid values of Y are 0 to 4, which are interpreted as
 367   // log2(access_size), and 0xF, which means that access size is passed via
 368   // platform dependent register (see below).
 369 #  if defined(__aarch64__)
 370   // Access type is encoded in BRK immediate as 0x900 + 0xXY. For Y == 0xF,
 371   // access size is stored in X1 register. Access address is always in X0
 372   // register.
 373   uptr pc = (uptr)info->si_addr;
 374   const unsigned code = ((*(u32 *)pc) >> 5) & 0xffff;
 375   if ((code & 0xff00) != 0x900)
 376     return AccessInfo{};  // Not ours.
 377
 378   const bool is_store = code & 0x10;
 379   const bool recover = code & 0x20;
 380   const uptr addr = uc->uc_mcontext.regs[0];
 381   const unsigned size_log = code & 0xf;
 382   if (size_log > 4 && size_log != 0xf)
 383     return AccessInfo{};  // Not ours.
 384   const uptr size = size_log == 0xf ? uc->uc_mcontext.regs[1] : 1U << size_log;
 385
 386 #  elif defined(__x86_64__)
 387   // Access type is encoded in the instruction following INT3 as
 388   // NOP DWORD ptr [EAX + 0x40 + 0xXY]. For Y == 0xF, access size is stored in
 389   // RSI register. Access address is always in RDI register.
 390   uptr pc = (uptr)uc->uc_mcontext.gregs[REG_RIP];
 391   uint8_t *nop = (uint8_t *)pc;
 392   if (*nop != 0x0f || *(nop + 1) != 0x1f || *(nop + 2) != 0x40 ||
 393       *(nop + 3) < 0x40)
 394     return AccessInfo{};  // Not ours.
 395   const unsigned code = *(nop + 3);
 396
 397   const bool is_store = code & 0x10;
 398   const bool recover = code & 0x20;
 399   const uptr addr = uc->uc_mcontext.gregs[REG_RDI];
 400   const unsigned size_log = code & 0xf;
 401   if (size_log > 4 && size_log != 0xf)
 402     return AccessInfo{};  // Not ours.
 403   const uptr size =
 404       size_log == 0xf ? uc->uc_mcontext.gregs[REG_RSI] : 1U << size_log;
 405
 406 #  elif SANITIZER_RISCV64
 407   // Access type is encoded in the instruction following EBREAK as
 408   // ADDI x0, x0, [0x40 + 0xXY]. For Y == 0xF, access size is stored in
 409   // X11 register. Access address is always in X10 register.
 410   uptr pc = (uptr)uc->uc_mcontext.__gregs[REG_PC];
 411   uint8_t byte1 = *((u8 *)(pc + 0));
 412   uint8_t byte2 = *((u8 *)(pc + 1));
 413   uint8_t byte3 = *((u8 *)(pc + 2));
 414   uint8_t byte4 = *((u8 *)(pc + 3));
 415   uint32_t ebreak = (byte1 | (byte2 << 8) | (byte3 << 16) | (byte4 << 24));
 416   bool isFaultShort = false;
 417   bool isEbreak = (ebreak == 0x100073);
 418   bool isShortEbreak = false;
 419 #    if defined(__riscv_compressed)
 420   isFaultShort = ((ebreak & 0x3) != 0x3);
 421   isShortEbreak = ((ebreak & 0xffff) == 0x9002);
 422 #    endif
 423   // faulted insn is not ebreak, not our case
 424   if (!(isEbreak || isShortEbreak))
 425     return AccessInfo{};
 426   // advance pc to point after ebreak and reconstruct addi instruction
 427   pc += isFaultShort ? 2 : 4;
 428   byte1 = *((u8 *)(pc + 0));
 429   byte2 = *((u8 *)(pc + 1));
 430   byte3 = *((u8 *)(pc + 2));
 431   byte4 = *((u8 *)(pc + 3));
 432   // reconstruct instruction
 433   uint32_t instr = (byte1 | (byte2 << 8) | (byte3 << 16) | (byte4 << 24));
 434   // check if this is really 32 bit instruction
 435   // code is encoded in top 12 bits, since instruction is supposed to be with
 436   // imm
 437   const unsigned code = (instr >> 20) & 0xffff;
 438   const uptr addr = uc->uc_mcontext.__gregs[10];
 439   const bool is_store = code & 0x10;
 440   const bool recover = code & 0x20;
 441   const unsigned size_log = code & 0xf;
 442   if (size_log > 4 && size_log != 0xf)
 443     return AccessInfo{};  // Not our case
 444   const uptr size =
 445       size_log == 0xf ? uc->uc_mcontext.__gregs[11] : 1U << size_log;
 446
 447 #  else
 448 #    error Unsupported architecture
 449 #  endif
 450
 451   return AccessInfo{addr, size, is_store, !is_store, recover};
 452 }
 453
 454 static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) {
 455   AccessInfo ai = GetAccessInfo(info, uc);
 456   if (!ai.is_store && !ai.is_load)
 457     return false;
 458
 459   SignalContext sig{info, uc};
 460   HandleTagMismatch(ai, StackTrace::GetNextInstructionPc(sig.pc), sig.bp, uc);
 461
 462 #  if defined(__aarch64__)
 463   uc->uc_mcontext.pc += 4;
 464 #  elif defined(__x86_64__)
 465 #  elif SANITIZER_RISCV64
 466   // pc points to EBREAK which is 2 bytes long
 467   uint8_t *exception_source = (uint8_t *)(uc->uc_mcontext.__gregs[REG_PC]);
 468   uint8_t byte1 = (uint8_t)(*(exception_source + 0));
 469   uint8_t byte2 = (uint8_t)(*(exception_source + 1));
 470   uint8_t byte3 = (uint8_t)(*(exception_source + 2));
 471   uint8_t byte4 = (uint8_t)(*(exception_source + 3));
 472   uint32_t faulted = (byte1 | (byte2 << 8) | (byte3 << 16) | (byte4 << 24));
 473   bool isFaultShort = false;
 474 #    if defined(__riscv_compressed)
 475   isFaultShort = ((faulted & 0x3) != 0x3);
 476 #    endif
 477   uc->uc_mcontext.__gregs[REG_PC] += isFaultShort ? 2 : 4;
 478 #  else
 479 #    error Unsupported architecture
 480 #  endif
 481   return true;
 482 }
 483
 484 static void OnStackUnwind(const SignalContext &sig, const void *,
 485                           BufferedStackTrace *stack) {
 486   stack->Unwind(StackTrace::GetNextInstructionPc(sig.pc), sig.bp, sig.context,
 487                 common_flags()->fast_unwind_on_fatal);
 488 }
 489
 490 void HwasanOnDeadlySignal(int signo, void *info, void *context) {
 491   // Probably a tag mismatch.
 492   if (signo == SIGTRAP)
 493     if (HwasanOnSIGTRAP(signo, (siginfo_t *)info, (ucontext_t *)context))
 494       return;
 495
 496   HandleDeadlySignal(info, context, GetTid(), &OnStackUnwind, nullptr);
 497 }
 498
 499 void Thread::InitStackAndTls(const InitState *) {
 500   uptr tls_size;
 501   uptr stack_size;
 502   GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size, &tls_begin_,
 503                        &tls_size);
 504   stack_top_ = stack_bottom_ + stack_size;
 505   tls_end_ = tls_begin_ + tls_size;
 506 }
 507
 508 uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) {
 509   CHECK(IsAligned(p, kShadowAlignment));
 510   CHECK(IsAligned(size, kShadowAlignment));
 511   uptr shadow_start = MemToShadow(p);
 512   uptr shadow_size = MemToShadowSize(size);
 513
 514   uptr page_size = GetPageSizeCached();
 515   uptr page_start = RoundUpTo(shadow_start, page_size);
 516   uptr page_end = RoundDownTo(shadow_start + shadow_size, page_size);
 517   uptr threshold = common_flags()->clear_shadow_mmap_threshold;
 518   if (SANITIZER_LINUX &&
 519       UNLIKELY(page_end >= page_start + threshold && tag == 0)) {
 520     internal_memset((void *)shadow_start, tag, page_start - shadow_start);
 521     internal_memset((void *)page_end, tag,
 522                     shadow_start + shadow_size - page_end);
 523     // For an anonymous private mapping MADV_DONTNEED will return a zero page on
 524     // Linux.
 525     ReleaseMemoryPagesToOSAndZeroFill(page_start, page_end);
 526   } else {
 527     internal_memset((void *)shadow_start, tag, shadow_size);
 528   }
 529   return AddTagToPointer(p, tag);
 530 }
 531
 532 void HwasanInstallAtForkHandler() {
 533   auto before = []() {
 534     HwasanAllocatorLock();
 535     StackDepotLockAll();
 536   };
 537   auto after = []() {
 538     StackDepotUnlockAll();
 539     HwasanAllocatorUnlock();
 540   };
 541   pthread_atfork(before, after, after);
 542 }
 543
 544 void InstallAtExitCheckLeaks() {
 545   if (CAN_SANITIZE_LEAKS) {
 546     if (common_flags()->detect_leaks && common_flags()->leak_check_at_exit) {
 547       if (flags()->halt_on_error)
 548         Atexit(__lsan::DoLeakCheck);
 549       else
 550         Atexit(__lsan::DoRecoverableLeakCheckVoid);
 551     }
 552   }
 553 }
 554
 555 }  // namespace __hwasan
 556
 557 #endif  // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD