compiler-rt/lib/scudo/standalone/memtag.h

   1 //===-- memtag.h ------------------------------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #ifndef SCUDO_MEMTAG_H_
  10 #define SCUDO_MEMTAG_H_
  11
  12 #include "internal_defs.h"
  13
  14 #if SCUDO_CAN_USE_MTE
  15 #include <sys/auxv.h>
  16 #include <sys/prctl.h>
  17 #endif
  18
  19 namespace scudo {
  20
  21 #if (__clang_major__ >= 12 && defined(__aarch64__) && !defined(__ILP32__)) ||  \
  22     defined(SCUDO_FUZZ)
  23
  24 // We assume that Top-Byte Ignore is enabled if the architecture supports memory
  25 // tagging. Not all operating systems enable TBI, so we only claim architectural
  26 // support for memory tagging if the operating system enables TBI.
  27 // HWASan uses the top byte for its own purpose and Scudo should not touch it.
  28 #if SCUDO_CAN_USE_MTE && !defined(SCUDO_DISABLE_TBI) &&                        \
  29     !__has_feature(hwaddress_sanitizer)
  30 inline constexpr bool archSupportsMemoryTagging() { return true; }
  31 #else
  32 inline constexpr bool archSupportsMemoryTagging() { return false; }
  33 #endif
  34
  35 inline constexpr uptr archMemoryTagGranuleSize() { return 16; }
  36
  37 inline uptr untagPointer(uptr Ptr) { return Ptr & ((1ULL << 56) - 1); }
  38
  39 inline uint8_t extractTag(uptr Ptr) { return (Ptr >> 56) & 0xf; }
  40
  41 #else
  42
  43 inline constexpr bool archSupportsMemoryTagging() { return false; }
  44
  45 inline NORETURN uptr archMemoryTagGranuleSize() {
  46   UNREACHABLE("memory tagging not supported");
  47 }
  48
  49 inline NORETURN uptr untagPointer(uptr Ptr) {
  50   (void)Ptr;
  51   UNREACHABLE("memory tagging not supported");
  52 }
  53
  54 inline NORETURN uint8_t extractTag(uptr Ptr) {
  55   (void)Ptr;
  56   UNREACHABLE("memory tagging not supported");
  57 }
  58
  59 #endif
  60
  61 #if __clang_major__ >= 12 && defined(__aarch64__) && !defined(__ILP32__)
  62
  63 #if SCUDO_CAN_USE_MTE
  64
  65 inline bool systemSupportsMemoryTagging() {
  66 #ifndef HWCAP2_MTE
  67 #define HWCAP2_MTE (1 << 18)
  68 #endif
  69   return getauxval(AT_HWCAP2) & HWCAP2_MTE;
  70 }
  71
  72 inline bool systemDetectsMemoryTagFaultsTestOnly() {
  73 #ifndef PR_SET_TAGGED_ADDR_CTRL
  74 #define PR_SET_TAGGED_ADDR_CTRL 54
  75 #endif
  76 #ifndef PR_GET_TAGGED_ADDR_CTRL
  77 #define PR_GET_TAGGED_ADDR_CTRL 56
  78 #endif
  79 #ifndef PR_TAGGED_ADDR_ENABLE
  80 #define PR_TAGGED_ADDR_ENABLE (1UL << 0)
  81 #endif
  82 #ifndef PR_MTE_TCF_SHIFT
  83 #define PR_MTE_TCF_SHIFT 1
  84 #endif
  85 #ifndef PR_MTE_TAG_SHIFT
  86 #define PR_MTE_TAG_SHIFT 3
  87 #endif
  88 #ifndef PR_MTE_TCF_NONE
  89 #define PR_MTE_TCF_NONE (0UL << PR_MTE_TCF_SHIFT)
  90 #endif
  91 #ifndef PR_MTE_TCF_SYNC
  92 #define PR_MTE_TCF_SYNC (1UL << PR_MTE_TCF_SHIFT)
  93 #endif
  94 #ifndef PR_MTE_TCF_MASK
  95 #define PR_MTE_TCF_MASK (3UL << PR_MTE_TCF_SHIFT)
  96 #endif
  97   int res = prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0);
  98   if (res == -1)
  99     return false;
 100   return (static_cast<unsigned long>(res) & PR_MTE_TCF_MASK) != PR_MTE_TCF_NONE;
 101 }
 102
 103 inline void enableSystemMemoryTaggingTestOnly() {
 104   prctl(PR_SET_TAGGED_ADDR_CTRL,
 105         PR_TAGGED_ADDR_ENABLE | PR_MTE_TCF_SYNC | (0xfffe << PR_MTE_TAG_SHIFT),
 106         0, 0, 0);
 107 }
 108
 109 #else // !SCUDO_CAN_USE_MTE
 110
 111 inline bool systemSupportsMemoryTagging() { return false; }
 112
 113 inline NORETURN bool systemDetectsMemoryTagFaultsTestOnly() {
 114   UNREACHABLE("memory tagging not supported");
 115 }
 116
 117 inline NORETURN void enableSystemMemoryTaggingTestOnly() {
 118   UNREACHABLE("memory tagging not supported");
 119 }
 120
 121 #endif // SCUDO_CAN_USE_MTE
 122
 123 class ScopedDisableMemoryTagChecks {
 124   uptr PrevTCO;
 125   bool active;
 126
 127 public:
 128   ScopedDisableMemoryTagChecks(bool cond = true) : active(cond) {
 129     if (!active)
 130       return;
 131     __asm__ __volatile__(
 132         R"(
 133         .arch_extension memtag
 134         mrs %0, tco
 135         msr tco, #1
 136         )"
 137         : "=r"(PrevTCO));
 138   }
 139
 140   ~ScopedDisableMemoryTagChecks() {
 141     if (!active)
 142       return;
 143     __asm__ __volatile__(
 144         R"(
 145         .arch_extension memtag
 146         msr tco, %0
 147         )"
 148         :
 149         : "r"(PrevTCO));
 150   }
 151 };
 152
 153 inline uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) {
 154   ExcludeMask |= 1; // Always exclude Tag 0.
 155   uptr TaggedPtr;
 156   __asm__ __volatile__(
 157       R"(
 158       .arch_extension memtag
 159       irg %[TaggedPtr], %[Ptr], %[ExcludeMask]
 160       )"
 161       : [TaggedPtr] "=r"(TaggedPtr)
 162       : [Ptr] "r"(Ptr), [ExcludeMask] "r"(ExcludeMask));
 163   return TaggedPtr;
 164 }
 165
 166 inline uptr addFixedTag(uptr Ptr, uptr Tag) {
 167   DCHECK_LT(Tag, 16);
 168   DCHECK_EQ(untagPointer(Ptr), Ptr);
 169   return Ptr | (Tag << 56);
 170 }
 171
 172 inline uptr storeTags(uptr Begin, uptr End) {
 173   DCHECK_EQ(0, Begin % 16);
 174   uptr LineSize, Next, Tmp;
 175   __asm__ __volatile__(
 176       R"(
 177     .arch_extension memtag
 178
 179     // Compute the cache line size in bytes (DCZID_EL0 stores it as the log2
 180     // of the number of 4-byte words) and bail out to the slow path if DCZID_EL0
 181     // indicates that the DC instructions are unavailable.
 182     DCZID .req %[Tmp]
 183     mrs DCZID, dczid_el0
 184     tbnz DCZID, #4, 3f
 185     and DCZID, DCZID, #15
 186     mov %[LineSize], #4
 187     lsl %[LineSize], %[LineSize], DCZID
 188     .unreq DCZID
 189
 190     // Our main loop doesn't handle the case where we don't need to perform any
 191     // DC GZVA operations. If the size of our tagged region is less than
 192     // twice the cache line size, bail out to the slow path since it's not
 193     // guaranteed that we'll be able to do a DC GZVA.
 194     Size .req %[Tmp]
 195     sub Size, %[End], %[Cur]
 196     cmp Size, %[LineSize], lsl #1
 197     b.lt 3f
 198     .unreq Size
 199
 200     LineMask .req %[Tmp]
 201     sub LineMask, %[LineSize], #1
 202
 203     // STZG until the start of the next cache line.
 204     orr %[Next], %[Cur], LineMask
 205   1:
 206     stzg %[Cur], [%[Cur]], #16
 207     cmp %[Cur], %[Next]
 208     b.lt 1b
 209
 210     // DC GZVA cache lines until we have no more full cache lines.
 211     bic %[Next], %[End], LineMask
 212     .unreq LineMask
 213   2:
 214     dc gzva, %[Cur]
 215     add %[Cur], %[Cur], %[LineSize]
 216     cmp %[Cur], %[Next]
 217     b.lt 2b
 218
 219     // STZG until the end of the tagged region. This loop is also used to handle
 220     // slow path cases.
 221   3:
 222     cmp %[Cur], %[End]
 223     b.ge 4f
 224     stzg %[Cur], [%[Cur]], #16
 225     b 3b
 226
 227   4:
 228   )"
 229       : [Cur] "+&r"(Begin), [LineSize] "=&r"(LineSize), [Next] "=&r"(Next),
 230         [Tmp] "=&r"(Tmp)
 231       : [End] "r"(End)
 232       : "memory");
 233   DCHECK_EQ(0, Begin % 16);
 234   return Begin;
 235 }
 236
 237 inline void storeTag(uptr Ptr) {
 238   DCHECK_EQ(0, Ptr % 16);
 239   __asm__ __volatile__(R"(
 240     .arch_extension memtag
 241     stg %0, [%0]
 242   )"
 243                        :
 244                        : "r"(Ptr)
 245                        : "memory");
 246 }
 247
 248 inline uptr loadTag(uptr Ptr) {
 249   DCHECK_EQ(0, Ptr % 16);
 250   uptr TaggedPtr = Ptr;
 251   __asm__ __volatile__(
 252       R"(
 253       .arch_extension memtag
 254       ldg %0, [%0]
 255       )"
 256       : "+r"(TaggedPtr)
 257       :
 258       : "memory");
 259   return TaggedPtr;
 260 }
 261
 262 #else
 263
 264 inline NORETURN bool systemSupportsMemoryTagging() {
 265   UNREACHABLE("memory tagging not supported");
 266 }
 267
 268 inline NORETURN bool systemDetectsMemoryTagFaultsTestOnly() {
 269   UNREACHABLE("memory tagging not supported");
 270 }
 271
 272 inline NORETURN void enableSystemMemoryTaggingTestOnly() {
 273   UNREACHABLE("memory tagging not supported");
 274 }
 275
 276 struct ScopedDisableMemoryTagChecks {
 277   ScopedDisableMemoryTagChecks(UNUSED bool cond = true) {}
 278 };
 279
 280 inline NORETURN uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) {
 281   (void)Ptr;
 282   (void)ExcludeMask;
 283   UNREACHABLE("memory tagging not supported");
 284 }
 285
 286 inline NORETURN uptr addFixedTag(uptr Ptr, uptr Tag) {
 287   (void)Ptr;
 288   (void)Tag;
 289   UNREACHABLE("memory tagging not supported");
 290 }
 291
 292 inline NORETURN uptr storeTags(uptr Begin, uptr End) {
 293   (void)Begin;
 294   (void)End;
 295   UNREACHABLE("memory tagging not supported");
 296 }
 297
 298 inline NORETURN void storeTag(uptr Ptr) {
 299   (void)Ptr;
 300   UNREACHABLE("memory tagging not supported");
 301 }
 302
 303 inline NORETURN uptr loadTag(uptr Ptr) {
 304   (void)Ptr;
 305   UNREACHABLE("memory tagging not supported");
 306 }
 307
 308 #endif
 309
 310 #pragma GCC diagnostic push
 311 #pragma GCC diagnostic ignored "-Wmissing-noreturn"
 312 inline void setRandomTag(void *Ptr, uptr Size, uptr ExcludeMask,
 313                          uptr *TaggedBegin, uptr *TaggedEnd) {
 314   *TaggedBegin = selectRandomTag(reinterpret_cast<uptr>(Ptr), ExcludeMask);
 315   *TaggedEnd = storeTags(*TaggedBegin, *TaggedBegin + Size);
 316 }
 317 #pragma GCC diagnostic pop
 318
 319 inline void *untagPointer(void *Ptr) {
 320   return reinterpret_cast<void *>(untagPointer(reinterpret_cast<uptr>(Ptr)));
 321 }
 322
 323 inline void *loadTag(void *Ptr) {
 324   return reinterpret_cast<void *>(loadTag(reinterpret_cast<uptr>(Ptr)));
 325 }
 326
 327 inline void *addFixedTag(void *Ptr, uptr Tag) {
 328   return reinterpret_cast<void *>(
 329       addFixedTag(reinterpret_cast<uptr>(Ptr), Tag));
 330 }
 331
 332 template <typename Config>
 333 inline constexpr bool allocatorSupportsMemoryTagging() {
 334   return archSupportsMemoryTagging() && Config::getMaySupportMemoryTagging() &&
 335          (1 << SCUDO_MIN_ALIGNMENT_LOG) >= archMemoryTagGranuleSize();
 336 }
 337
 338 } // namespace scudo
 339
 340 #endif