compiler-rt/lib/scudo/standalone/memtag.h

   1 //===-- memtag.h ------------------------------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #ifndef SCUDO_MEMTAG_H_
  10 #define SCUDO_MEMTAG_H_
  11
  12 #include "internal_defs.h"
  13
  14 #if SCUDO_CAN_USE_MTE
  15 #include <sys/auxv.h>
  16 #include <sys/prctl.h>
  17 #endif
  18
  19 namespace scudo {
  20
  21 #if (__clang_major__ >= 12 && defined(__aarch64__) && !defined(__ILP32__)) ||  \
  22     defined(SCUDO_FUZZ)
  23
  24 // We assume that Top-Byte Ignore is enabled if the architecture supports memory
  25 // tagging. Not all operating systems enable TBI, so we only claim architectural
  26 // support for memory tagging if the operating system enables TBI.
  27 // HWASan uses the top byte for its own purpose and Scudo should not touch it.
  28 #if SCUDO_CAN_USE_MTE && !defined(SCUDO_DISABLE_TBI) &&                        \
  29     !__has_feature(hwaddress_sanitizer)
  30 inline constexpr bool archSupportsMemoryTagging() { return true; }
  31 #else
  32 inline constexpr bool archSupportsMemoryTagging() { return false; }
  33 #endif
  34
  35 inline constexpr uptr archMemoryTagGranuleSize() { return 16; }
  36
  37 inline uptr untagPointer(uptr Ptr) { return Ptr & ((1ULL << 56) - 1); }
  38
  39 inline uint8_t extractTag(uptr Ptr) { return (Ptr >> 56) & 0xf; }
  40
  41 #else
  42
  43 inline constexpr bool archSupportsMemoryTagging() { return false; }
  44
  45 inline NORETURN uptr archMemoryTagGranuleSize() {
  46   UNREACHABLE("memory tagging not supported");
  47 }
  48
  49 inline NORETURN uptr untagPointer(uptr Ptr) {
  50   (void)Ptr;
  51   UNREACHABLE("memory tagging not supported");
  52 }
  53
  54 inline NORETURN uint8_t extractTag(uptr Ptr) {
  55   (void)Ptr;
  56   UNREACHABLE("memory tagging not supported");
  57 }
  58
  59 #endif
  60
  61 #if __clang_major__ >= 12 && defined(__aarch64__) && !defined(__ILP32__)
  62
  63 #if SCUDO_CAN_USE_MTE
  64
  65 inline bool systemSupportsMemoryTagging() {
  66 #ifndef HWCAP2_MTE
  67 #define HWCAP2_MTE (1 << 18)
  68 #endif
  69   return getauxval(AT_HWCAP2) & HWCAP2_MTE;
  70 }
  71
  72 inline bool systemDetectsMemoryTagFaultsTestOnly() {
  73 #ifndef PR_SET_TAGGED_ADDR_CTRL
  74 #define PR_SET_TAGGED_ADDR_CTRL 54
  75 #endif
  76 #ifndef PR_GET_TAGGED_ADDR_CTRL
  77 #define PR_GET_TAGGED_ADDR_CTRL 56
  78 #endif
  79 #ifndef PR_TAGGED_ADDR_ENABLE
  80 #define PR_TAGGED_ADDR_ENABLE (1UL << 0)
  81 #endif
  82 #ifndef PR_MTE_TCF_SHIFT
  83 #define PR_MTE_TCF_SHIFT 1
  84 #endif
  85 #ifndef PR_MTE_TAG_SHIFT
  86 #define PR_MTE_TAG_SHIFT 3
  87 #endif
  88 #ifndef PR_MTE_TCF_NONE
  89 #define PR_MTE_TCF_NONE (0UL << PR_MTE_TCF_SHIFT)
  90 #endif
  91 #ifndef PR_MTE_TCF_SYNC
  92 #define PR_MTE_TCF_SYNC (1UL << PR_MTE_TCF_SHIFT)
  93 #endif
  94 #ifndef PR_MTE_TCF_MASK
  95 #define PR_MTE_TCF_MASK (3UL << PR_MTE_TCF_SHIFT)
  96 #endif
  97   int res = prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0);
  98   if (res == -1)
  99     return false;
 100   return (static_cast<unsigned long>(res) & PR_MTE_TCF_MASK) != PR_MTE_TCF_NONE;
 101 }
 102
 103 inline void enableSystemMemoryTaggingTestOnly() {
 104   prctl(PR_SET_TAGGED_ADDR_CTRL,
 105         PR_TAGGED_ADDR_ENABLE | PR_MTE_TCF_SYNC | (0xfffe << PR_MTE_TAG_SHIFT),
 106         0, 0, 0);
 107 }
 108
 109 #else // !SCUDO_CAN_USE_MTE
 110
 111 inline bool systemSupportsMemoryTagging() { return false; }
 112
 113 inline NORETURN bool systemDetectsMemoryTagFaultsTestOnly() {
 114   UNREACHABLE("memory tagging not supported");
 115 }
 116
 117 inline NORETURN void enableSystemMemoryTaggingTestOnly() {
 118   UNREACHABLE("memory tagging not supported");
 119 }
 120
 121 #endif // SCUDO_CAN_USE_MTE
 122
 123 class ScopedDisableMemoryTagChecks {
 124   uptr PrevTCO;
 125
 126 public:
 127   ScopedDisableMemoryTagChecks() {
 128     __asm__ __volatile__(
 129         R"(
 130         .arch_extension memtag
 131         mrs %0, tco
 132         msr tco, #1
 133         )"
 134         : "=r"(PrevTCO));
 135   }
 136
 137   ~ScopedDisableMemoryTagChecks() {
 138     __asm__ __volatile__(
 139         R"(
 140         .arch_extension memtag
 141         msr tco, %0
 142         )"
 143         :
 144         : "r"(PrevTCO));
 145   }
 146 };
 147
 148 inline uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) {
 149   ExcludeMask |= 1; // Always exclude Tag 0.
 150   uptr TaggedPtr;
 151   __asm__ __volatile__(
 152       R"(
 153       .arch_extension memtag
 154       irg %[TaggedPtr], %[Ptr], %[ExcludeMask]
 155       )"
 156       : [TaggedPtr] "=r"(TaggedPtr)
 157       : [Ptr] "r"(Ptr), [ExcludeMask] "r"(ExcludeMask));
 158   return TaggedPtr;
 159 }
 160
 161 inline uptr addFixedTag(uptr Ptr, uptr Tag) {
 162   DCHECK_LT(Tag, 16);
 163   DCHECK_EQ(untagPointer(Ptr), Ptr);
 164   return Ptr | (Tag << 56);
 165 }
 166
 167 inline uptr storeTags(uptr Begin, uptr End) {
 168   DCHECK_EQ(0, Begin % 16);
 169   uptr LineSize, Next, Tmp;
 170   __asm__ __volatile__(
 171       R"(
 172     .arch_extension memtag
 173
 174     // Compute the cache line size in bytes (DCZID_EL0 stores it as the log2
 175     // of the number of 4-byte words) and bail out to the slow path if DCZID_EL0
 176     // indicates that the DC instructions are unavailable.
 177     DCZID .req %[Tmp]
 178     mrs DCZID, dczid_el0
 179     tbnz DCZID, #4, 3f
 180     and DCZID, DCZID, #15
 181     mov %[LineSize], #4
 182     lsl %[LineSize], %[LineSize], DCZID
 183     .unreq DCZID
 184
 185     // Our main loop doesn't handle the case where we don't need to perform any
 186     // DC GZVA operations. If the size of our tagged region is less than
 187     // twice the cache line size, bail out to the slow path since it's not
 188     // guaranteed that we'll be able to do a DC GZVA.
 189     Size .req %[Tmp]
 190     sub Size, %[End], %[Cur]
 191     cmp Size, %[LineSize], lsl #1
 192     b.lt 3f
 193     .unreq Size
 194
 195     LineMask .req %[Tmp]
 196     sub LineMask, %[LineSize], #1
 197
 198     // STZG until the start of the next cache line.
 199     orr %[Next], %[Cur], LineMask
 200   1:
 201     stzg %[Cur], [%[Cur]], #16
 202     cmp %[Cur], %[Next]
 203     b.lt 1b
 204
 205     // DC GZVA cache lines until we have no more full cache lines.
 206     bic %[Next], %[End], LineMask
 207     .unreq LineMask
 208   2:
 209     dc gzva, %[Cur]
 210     add %[Cur], %[Cur], %[LineSize]
 211     cmp %[Cur], %[Next]
 212     b.lt 2b
 213
 214     // STZG until the end of the tagged region. This loop is also used to handle
 215     // slow path cases.
 216   3:
 217     cmp %[Cur], %[End]
 218     b.ge 4f
 219     stzg %[Cur], [%[Cur]], #16
 220     b 3b
 221
 222   4:
 223   )"
 224       : [Cur] "+&r"(Begin), [LineSize] "=&r"(LineSize), [Next] "=&r"(Next),
 225         [Tmp] "=&r"(Tmp)
 226       : [End] "r"(End)
 227       : "memory");
 228   DCHECK_EQ(0, Begin % 16);
 229   return Begin;
 230 }
 231
 232 inline void storeTag(uptr Ptr) {
 233   DCHECK_EQ(0, Ptr % 16);
 234   __asm__ __volatile__(R"(
 235     .arch_extension memtag
 236     stg %0, [%0]
 237   )"
 238                        :
 239                        : "r"(Ptr)
 240                        : "memory");
 241 }
 242
 243 inline uptr loadTag(uptr Ptr) {
 244   DCHECK_EQ(0, Ptr % 16);
 245   uptr TaggedPtr = Ptr;
 246   __asm__ __volatile__(
 247       R"(
 248       .arch_extension memtag
 249       ldg %0, [%0]
 250       )"
 251       : "+r"(TaggedPtr)
 252       :
 253       : "memory");
 254   return TaggedPtr;
 255 }
 256
 257 #else
 258
 259 inline NORETURN bool systemSupportsMemoryTagging() {
 260   UNREACHABLE("memory tagging not supported");
 261 }
 262
 263 inline NORETURN bool systemDetectsMemoryTagFaultsTestOnly() {
 264   UNREACHABLE("memory tagging not supported");
 265 }
 266
 267 inline NORETURN void enableSystemMemoryTaggingTestOnly() {
 268   UNREACHABLE("memory tagging not supported");
 269 }
 270
 271 struct ScopedDisableMemoryTagChecks {
 272   ScopedDisableMemoryTagChecks() {}
 273 };
 274
 275 inline NORETURN uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) {
 276   (void)Ptr;
 277   (void)ExcludeMask;
 278   UNREACHABLE("memory tagging not supported");
 279 }
 280
 281 inline NORETURN uptr addFixedTag(uptr Ptr, uptr Tag) {
 282   (void)Ptr;
 283   (void)Tag;
 284   UNREACHABLE("memory tagging not supported");
 285 }
 286
 287 inline NORETURN uptr storeTags(uptr Begin, uptr End) {
 288   (void)Begin;
 289   (void)End;
 290   UNREACHABLE("memory tagging not supported");
 291 }
 292
 293 inline NORETURN void storeTag(uptr Ptr) {
 294   (void)Ptr;
 295   UNREACHABLE("memory tagging not supported");
 296 }
 297
 298 inline NORETURN uptr loadTag(uptr Ptr) {
 299   (void)Ptr;
 300   UNREACHABLE("memory tagging not supported");
 301 }
 302
 303 #endif
 304
 305 #pragma GCC diagnostic push
 306 #pragma GCC diagnostic ignored "-Wmissing-noreturn"
 307 inline void setRandomTag(void *Ptr, uptr Size, uptr ExcludeMask,
 308                          uptr *TaggedBegin, uptr *TaggedEnd) {
 309   *TaggedBegin = selectRandomTag(reinterpret_cast<uptr>(Ptr), ExcludeMask);
 310   *TaggedEnd = storeTags(*TaggedBegin, *TaggedBegin + Size);
 311 }
 312 #pragma GCC diagnostic pop
 313
 314 inline void *untagPointer(void *Ptr) {
 315   return reinterpret_cast<void *>(untagPointer(reinterpret_cast<uptr>(Ptr)));
 316 }
 317
 318 inline void *loadTag(void *Ptr) {
 319   return reinterpret_cast<void *>(loadTag(reinterpret_cast<uptr>(Ptr)));
 320 }
 321
 322 inline void *addFixedTag(void *Ptr, uptr Tag) {
 323   return reinterpret_cast<void *>(
 324       addFixedTag(reinterpret_cast<uptr>(Ptr), Tag));
 325 }
 326
 327 template <typename Config>
 328 inline constexpr bool allocatorSupportsMemoryTagging() {
 329   return archSupportsMemoryTagging() && Config::MaySupportMemoryTagging &&
 330          (1 << SCUDO_MIN_ALIGNMENT_LOG) >= archMemoryTagGranuleSize();
 331 }
 332
 333 } // namespace scudo
 334
 335 #endif