llvm/lib/ExecutionEngine/JITLink/aarch32.cpp

   1 //===--------- aarch32.cpp - Generic JITLink arm/thumb utilities ----------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // Generic utilities for graphs representing arm/thumb objects.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "llvm/ExecutionEngine/JITLink/aarch32.h"
  14
  15 #include "llvm/ADT/StringExtras.h"
  16 #include "llvm/BinaryFormat/ELF.h"
  17 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
  18 #include "llvm/ExecutionEngine/Orc/Shared/MemoryFlags.h"
  19 #include "llvm/Object/ELFObjectFile.h"
  20 #include "llvm/Support/Endian.h"
  21 #include "llvm/Support/ManagedStatic.h"
  22 #include "llvm/Support/MathExtras.h"
  23
  24 #define DEBUG_TYPE "jitlink"
  25
  26 namespace llvm {
  27 namespace jitlink {
  28 namespace aarch32 {
  29
  30 /// Check whether the given target flags are set for this Symbol.
  31 bool hasTargetFlags(Symbol &Sym, TargetFlagsType Flags) {
  32   return static_cast<TargetFlagsType>(Sym.getTargetFlags()) & Flags;
  33 }
  34
  35 /// Encode 22-bit immediate value for branch instructions without J1J2 range
  36 /// extension (formats B T4, BL T1 and BLX T2).
  37 ///
  38 ///   00000:Imm11H:Imm11L:0 -> [ 00000:Imm11H, 00000:Imm11L ]
  39 ///                                            J1^ ^J2 will always be 1
  40 ///
  41 HalfWords encodeImmBT4BlT1BlxT2(int64_t Value) {
  42   constexpr uint32_t J1J2 = 0x2800;
  43   uint32_t Imm11H = (Value >> 12) & 0x07ff;
  44   uint32_t Imm11L = (Value >> 1) & 0x07ff;
  45   return HalfWords{Imm11H, Imm11L | J1J2};
  46 }
  47
  48 /// Decode 22-bit immediate value for branch instructions without J1J2 range
  49 /// extension (formats B T4, BL T1 and BLX T2).
  50 ///
  51 ///   [ 00000:Imm11H, 00000:Imm11L ] -> 00000:Imm11H:Imm11L:0
  52 ///                   J1^ ^J2 will always be 1
  53 ///
  54 int64_t decodeImmBT4BlT1BlxT2(uint32_t Hi, uint32_t Lo) {
  55   uint32_t Imm11H = Hi & 0x07ff;
  56   uint32_t Imm11L = Lo & 0x07ff;
  57   return SignExtend64<22>(Imm11H << 12 | Imm11L << 1);
  58 }
  59
  60 /// Encode 25-bit immediate value for branch instructions with J1J2 range
  61 /// extension (formats B T4, BL T1 and BLX T2).
  62 ///
  63 ///   S:I1:I2:Imm10:Imm11:0 -> [ 00000:S:Imm10, 00:J1:0:J2:Imm11 ]
  64 ///
  65 HalfWords encodeImmBT4BlT1BlxT2_J1J2(int64_t Value) {
  66   uint32_t S = (Value >> 14) & 0x0400;
  67   uint32_t J1 = (((~(Value >> 10)) ^ (Value >> 11)) & 0x2000);
  68   uint32_t J2 = (((~(Value >> 11)) ^ (Value >> 13)) & 0x0800);
  69   uint32_t Imm10 = (Value >> 12) & 0x03ff;
  70   uint32_t Imm11 = (Value >> 1) & 0x07ff;
  71   return HalfWords{S | Imm10, J1 | J2 | Imm11};
  72 }
  73
  74 /// Decode 25-bit immediate value for branch instructions with J1J2 range
  75 /// extension (formats B T4, BL T1 and BLX T2).
  76 ///
  77 ///   [ 00000:S:Imm10, 00:J1:0:J2:Imm11] -> S:I1:I2:Imm10:Imm11:0
  78 ///
  79 int64_t decodeImmBT4BlT1BlxT2_J1J2(uint32_t Hi, uint32_t Lo) {
  80   uint32_t S = Hi & 0x0400;
  81   uint32_t I1 = ~((Lo ^ (Hi << 3)) << 10) & 0x00800000;
  82   uint32_t I2 = ~((Lo ^ (Hi << 1)) << 11) & 0x00400000;
  83   uint32_t Imm10 = Hi & 0x03ff;
  84   uint32_t Imm11 = Lo & 0x07ff;
  85   return SignExtend64<25>(S << 14 | I1 | I2 | Imm10 << 12 | Imm11 << 1);
  86 }
  87
  88 /// Encode 26-bit immediate value for branch instructions
  89 /// (formats B A1, BL A1 and BLX A2).
  90 ///
  91 ///   Imm24:00 ->  00000000:Imm24
  92 ///
  93 uint32_t encodeImmBA1BlA1BlxA2(int64_t Value) {
  94   return (Value >> 2) & 0x00ffffff;
  95 }
  96
  97 /// Decode 26-bit immediate value for branch instructions
  98 /// (formats B A1, BL A1 and BLX A2).
  99 ///
 100 ///   00000000:Imm24 ->  Imm24:00
 101 ///
 102 int64_t decodeImmBA1BlA1BlxA2(int64_t Value) {
 103   return SignExtend64<26>((Value & 0x00ffffff) << 2);
 104 }
 105
 106 /// Encode 16-bit immediate value for move instruction formats MOVT T1 and
 107 /// MOVW T3.
 108 ///
 109 ///   Imm4:Imm1:Imm3:Imm8 -> [ 00000:i:000000:Imm4, 0:Imm3:0000:Imm8 ]
 110 ///
 111 HalfWords encodeImmMovtT1MovwT3(uint16_t Value) {
 112   uint32_t Imm4 = (Value >> 12) & 0x0f;
 113   uint32_t Imm1 = (Value >> 11) & 0x01;
 114   uint32_t Imm3 = (Value >> 8) & 0x07;
 115   uint32_t Imm8 = Value & 0xff;
 116   return HalfWords{Imm1 << 10 | Imm4, Imm3 << 12 | Imm8};
 117 }
 118
 119 /// Decode 16-bit immediate value from move instruction formats MOVT T1 and
 120 /// MOVW T3.
 121 ///
 122 ///   [ 00000:i:000000:Imm4, 0:Imm3:0000:Imm8 ] -> Imm4:Imm1:Imm3:Imm8
 123 ///
 124 uint16_t decodeImmMovtT1MovwT3(uint32_t Hi, uint32_t Lo) {
 125   uint32_t Imm4 = Hi & 0x0f;
 126   uint32_t Imm1 = (Hi >> 10) & 0x01;
 127   uint32_t Imm3 = (Lo >> 12) & 0x07;
 128   uint32_t Imm8 = Lo & 0xff;
 129   uint32_t Imm16 = Imm4 << 12 | Imm1 << 11 | Imm3 << 8 | Imm8;
 130   assert(Imm16 <= 0xffff && "Decoded value out-of-range");
 131   return Imm16;
 132 }
 133
 134 /// Encode register ID for instruction formats MOVT T1 and MOVW T3.
 135 ///
 136 ///   Rd4 -> [0000000000000000, 0000:Rd4:00000000]
 137 ///
 138 HalfWords encodeRegMovtT1MovwT3(int64_t Value) {
 139   uint32_t Rd4 = (Value & 0x0f) << 8;
 140   return HalfWords{0, Rd4};
 141 }
 142
 143 /// Decode register ID from instruction formats MOVT T1 and MOVW T3.
 144 ///
 145 ///   [0000000000000000, 0000:Rd4:00000000] -> Rd4
 146 ///
 147 int64_t decodeRegMovtT1MovwT3(uint32_t Hi, uint32_t Lo) {
 148   uint32_t Rd4 = (Lo >> 8) & 0x0f;
 149   return Rd4;
 150 }
 151
 152 /// Encode 16-bit immediate value for move instruction formats MOVT A1 and
 153 /// MOVW A2.
 154 ///
 155 ///   Imm4:Imm12 -> 000000000000:Imm4:0000:Imm12
 156 ///
 157 uint32_t encodeImmMovtA1MovwA2(uint16_t Value) {
 158   uint32_t Imm4 = (Value >> 12) & 0x0f;
 159   uint32_t Imm12 = Value & 0x0fff;
 160   return (Imm4 << 16) | Imm12;
 161 }
 162
 163 /// Decode 16-bit immediate value for move instruction formats MOVT A1 and
 164 /// MOVW A2.
 165 ///
 166 ///   000000000000:Imm4:0000:Imm12 -> Imm4:Imm12
 167 ///
 168 uint16_t decodeImmMovtA1MovwA2(uint64_t Value) {
 169   uint32_t Imm4 = (Value >> 16) & 0x0f;
 170   uint32_t Imm12 = Value & 0x0fff;
 171   return (Imm4 << 12) | Imm12;
 172 }
 173
 174 /// Encode register ID for instruction formats MOVT A1 and
 175 /// MOVW A2.
 176 ///
 177 ///   Rd4 -> 0000000000000000:Rd4:000000000000
 178 ///
 179 uint32_t encodeRegMovtA1MovwA2(int64_t Value) {
 180   uint32_t Rd4 = (Value & 0x00000f) << 12;
 181   return Rd4;
 182 }
 183
 184 /// Decode register ID for instruction formats MOVT A1 and
 185 /// MOVW A2.
 186 ///
 187 ///   0000000000000000:Rd4:000000000000 -> Rd4
 188 ///
 189 int64_t decodeRegMovtA1MovwA2(uint64_t Value) {
 190   uint32_t Rd4 = (Value >> 12) & 0x00000f;
 191   return Rd4;
 192 }
 193
 194 namespace {
 195
 196 /// 32-bit Thumb instructions are stored as two little-endian halfwords.
 197 /// An instruction at address A encodes bytes A+1, A in the first halfword (Hi),
 198 /// followed by bytes A+3, A+2 in the second halfword (Lo).
 199 struct WritableThumbRelocation {
 200   /// Create a writable reference to a Thumb32 fixup.
 201   WritableThumbRelocation(char *FixupPtr)
 202       : Hi{*reinterpret_cast<support::ulittle16_t *>(FixupPtr)},
 203         Lo{*reinterpret_cast<support::ulittle16_t *>(FixupPtr + 2)} {}
 204
 205   support::ulittle16_t &Hi; // First halfword
 206   support::ulittle16_t &Lo; // Second halfword
 207 };
 208
 209 struct ThumbRelocation {
 210   /// Create a read-only reference to a Thumb32 fixup.
 211   ThumbRelocation(const char *FixupPtr)
 212       : Hi{*reinterpret_cast<const support::ulittle16_t *>(FixupPtr)},
 213         Lo{*reinterpret_cast<const support::ulittle16_t *>(FixupPtr + 2)} {}
 214
 215   /// Create a read-only Thumb32 fixup from a writeable one.
 216   ThumbRelocation(WritableThumbRelocation &Writable)
 217       : Hi{Writable.Hi}, Lo(Writable.Lo) {}
 218
 219   const support::ulittle16_t &Hi; // First halfword
 220   const support::ulittle16_t &Lo; // Second halfword
 221 };
 222
 223 struct WritableArmRelocation {
 224   WritableArmRelocation(char *FixupPtr)
 225       : Wd{*reinterpret_cast<support::ulittle32_t *>(FixupPtr)} {}
 226
 227   support::ulittle32_t &Wd;
 228 };
 229
 230 struct ArmRelocation {
 231   ArmRelocation(const char *FixupPtr)
 232       : Wd{*reinterpret_cast<const support::ulittle32_t *>(FixupPtr)} {}
 233
 234   ArmRelocation(WritableArmRelocation &Writable) : Wd{Writable.Wd} {}
 235
 236   const support::ulittle32_t &Wd;
 237 };
 238
 239 Error makeUnexpectedOpcodeError(const LinkGraph &G, const ThumbRelocation &R,
 240                                 Edge::Kind Kind) {
 241   return make_error<JITLinkError>(
 242       formatv("Invalid opcode [ {0:x4}, {1:x4} ] for relocation: {2}",
 243               static_cast<uint16_t>(R.Hi), static_cast<uint16_t>(R.Lo),
 244               G.getEdgeKindName(Kind)));
 245 }
 246
 247 Error makeUnexpectedOpcodeError(const LinkGraph &G, const ArmRelocation &R,
 248                                 Edge::Kind Kind) {
 249   return make_error<JITLinkError>(
 250       formatv("Invalid opcode {0:x8} for relocation: {1}",
 251               static_cast<uint32_t>(R.Wd), G.getEdgeKindName(Kind)));
 252 }
 253
 254 template <EdgeKind_aarch32 K> constexpr bool isArm() {
 255   return FirstArmRelocation <= K && K <= LastArmRelocation;
 256 }
 257 template <EdgeKind_aarch32 K> constexpr bool isThumb() {
 258   return FirstThumbRelocation <= K && K <= LastThumbRelocation;
 259 }
 260
 261 template <EdgeKind_aarch32 K> static bool checkOpcodeArm(uint32_t Wd) {
 262   return (Wd & FixupInfo<K>::OpcodeMask) == FixupInfo<K>::Opcode;
 263 }
 264
 265 template <EdgeKind_aarch32 K>
 266 static bool checkOpcodeThumb(uint16_t Hi, uint16_t Lo) {
 267   return (Hi & FixupInfo<K>::OpcodeMask.Hi) == FixupInfo<K>::Opcode.Hi &&
 268          (Lo & FixupInfo<K>::OpcodeMask.Lo) == FixupInfo<K>::Opcode.Lo;
 269 }
 270
 271 class FixupInfoTable {
 272   static constexpr size_t Items = LastRelocation + 1;
 273
 274 public:
 275   FixupInfoTable() {
 276     populateEntries<FirstArmRelocation, LastArmRelocation>();
 277     populateEntries<FirstThumbRelocation, LastThumbRelocation>();
 278   }
 279
 280   const FixupInfoBase *getEntry(Edge::Kind K) {
 281     assert(K < Data.size() && "Index out of bounds");
 282     return Data.at(K).get();
 283   }
 284
 285 private:
 286   template <EdgeKind_aarch32 K, EdgeKind_aarch32 LastK> void populateEntries() {
 287     assert(K < Data.size() && "Index out of range");
 288     assert(Data.at(K) == nullptr && "Initialized entries are immutable");
 289     Data[K] = initEntry<K>();
 290     if constexpr (K < LastK) {
 291       constexpr auto Next = static_cast<EdgeKind_aarch32>(K + 1);
 292       populateEntries<Next, LastK>();
 293     }
 294   }
 295
 296   template <EdgeKind_aarch32 K>
 297   static std::unique_ptr<FixupInfoBase> initEntry() {
 298     auto Entry = std::make_unique<FixupInfo<K>>();
 299     static_assert(isArm<K>() != isThumb<K>(), "Classes are mutually exclusive");
 300     if constexpr (isArm<K>())
 301       Entry->checkOpcode = checkOpcodeArm<K>;
 302     if constexpr (isThumb<K>())
 303       Entry->checkOpcode = checkOpcodeThumb<K>;
 304     return Entry;
 305   }
 306
 307 private:
 308   std::array<std::unique_ptr<FixupInfoBase>, Items> Data;
 309 };
 310
 311 ManagedStatic<FixupInfoTable> DynFixupInfos;
 312
 313 } // namespace
 314
 315 static Error checkOpcode(LinkGraph &G, const ArmRelocation &R,
 316                          Edge::Kind Kind) {
 317   assert(Kind >= FirstArmRelocation && Kind <= LastArmRelocation &&
 318          "Edge kind must be Arm relocation");
 319   const FixupInfoBase *Entry = DynFixupInfos->getEntry(Kind);
 320   const FixupInfoArm &Info = *static_cast<const FixupInfoArm *>(Entry);
 321   assert(Info.checkOpcode && "Opcode check is mandatory for Arm edges");
 322   if (!Info.checkOpcode(R.Wd))
 323     return makeUnexpectedOpcodeError(G, R, Kind);
 324
 325   return Error::success();
 326 }
 327
 328 static Error checkOpcode(LinkGraph &G, const ThumbRelocation &R,
 329                          Edge::Kind Kind) {
 330   assert(Kind >= FirstThumbRelocation && Kind <= LastThumbRelocation &&
 331          "Edge kind must be Thumb relocation");
 332   const FixupInfoBase *Entry = DynFixupInfos->getEntry(Kind);
 333   const FixupInfoThumb &Info = *static_cast<const FixupInfoThumb *>(Entry);
 334   assert(Info.checkOpcode && "Opcode check is mandatory for Thumb edges");
 335   if (!Info.checkOpcode(R.Hi, R.Lo))
 336     return makeUnexpectedOpcodeError(G, R, Kind);
 337
 338   return Error::success();
 339 }
 340
 341 const FixupInfoBase *FixupInfoBase::getDynFixupInfo(Edge::Kind K) {
 342   return DynFixupInfos->getEntry(K);
 343 }
 344
 345 template <EdgeKind_aarch32 Kind>
 346 bool checkRegister(const ThumbRelocation &R, HalfWords Reg) {
 347   uint16_t Hi = R.Hi & FixupInfo<Kind>::RegMask.Hi;
 348   uint16_t Lo = R.Lo & FixupInfo<Kind>::RegMask.Lo;
 349   return Hi == Reg.Hi && Lo == Reg.Lo;
 350 }
 351
 352 template <EdgeKind_aarch32 Kind>
 353 bool checkRegister(const ArmRelocation &R, uint32_t Reg) {
 354   uint32_t Wd = R.Wd & FixupInfo<Kind>::RegMask;
 355   return Wd == Reg;
 356 }
 357
 358 template <EdgeKind_aarch32 Kind>
 359 void writeRegister(WritableThumbRelocation &R, HalfWords Reg) {
 360   static constexpr HalfWords Mask = FixupInfo<Kind>::RegMask;
 361   assert((Mask.Hi & Reg.Hi) == Reg.Hi && (Mask.Lo & Reg.Lo) == Reg.Lo &&
 362          "Value bits exceed bit range of given mask");
 363   R.Hi = (R.Hi & ~Mask.Hi) | Reg.Hi;
 364   R.Lo = (R.Lo & ~Mask.Lo) | Reg.Lo;
 365 }
 366
 367 template <EdgeKind_aarch32 Kind>
 368 void writeRegister(WritableArmRelocation &R, uint32_t Reg) {
 369   static constexpr uint32_t Mask = FixupInfo<Kind>::RegMask;
 370   assert((Mask & Reg) == Reg && "Value bits exceed bit range of given mask");
 371   R.Wd = (R.Wd & ~Mask) | Reg;
 372 }
 373
 374 template <EdgeKind_aarch32 Kind>
 375 void writeImmediate(WritableThumbRelocation &R, HalfWords Imm) {
 376   static constexpr HalfWords Mask = FixupInfo<Kind>::ImmMask;
 377   assert((Mask.Hi & Imm.Hi) == Imm.Hi && (Mask.Lo & Imm.Lo) == Imm.Lo &&
 378          "Value bits exceed bit range of given mask");
 379   R.Hi = (R.Hi & ~Mask.Hi) | Imm.Hi;
 380   R.Lo = (R.Lo & ~Mask.Lo) | Imm.Lo;
 381 }
 382
 383 template <EdgeKind_aarch32 Kind>
 384 void writeImmediate(WritableArmRelocation &R, uint32_t Imm) {
 385   static constexpr uint32_t Mask = FixupInfo<Kind>::ImmMask;
 386   assert((Mask & Imm) == Imm && "Value bits exceed bit range of given mask");
 387   R.Wd = (R.Wd & ~Mask) | Imm;
 388 }
 389
 390 Expected<int64_t> readAddendData(LinkGraph &G, Block &B, Edge::OffsetT Offset,
 391                                  Edge::Kind Kind) {
 392   endianness Endian = G.getEndianness();
 393   const char *BlockWorkingMem = B.getContent().data();
 394   const char *FixupPtr = BlockWorkingMem + Offset;
 395
 396   switch (Kind) {
 397   case Data_Delta32:
 398   case Data_Pointer32:
 399   case Data_RequestGOTAndTransformToDelta32:
 400     return SignExtend64<32>(support::endian::read32(FixupPtr, Endian));
 401   case Data_PRel31:
 402     return SignExtend64<31>(support::endian::read32(FixupPtr, Endian));
 403   default:
 404     return make_error<JITLinkError>(
 405         "In graph " + G.getName() + ", section " + B.getSection().getName() +
 406         " can not read implicit addend for aarch32 edge kind " +
 407         G.getEdgeKindName(Kind));
 408   }
 409 }
 410
 411 Expected<int64_t> readAddendArm(LinkGraph &G, Block &B, Edge::OffsetT Offset,
 412                                 Edge::Kind Kind) {
 413   ArmRelocation R(B.getContent().data() + Offset);
 414   if (Error Err = checkOpcode(G, R, Kind))
 415     return std::move(Err);
 416
 417   switch (Kind) {
 418   case Arm_Call:
 419   case Arm_Jump24:
 420     return decodeImmBA1BlA1BlxA2(R.Wd);
 421
 422   case Arm_MovtAbs:
 423   case Arm_MovwAbsNC:
 424     return decodeImmMovtA1MovwA2(R.Wd);
 425
 426   default:
 427     return make_error<JITLinkError>(
 428         "In graph " + G.getName() + ", section " + B.getSection().getName() +
 429         " can not read implicit addend for aarch32 edge kind " +
 430         G.getEdgeKindName(Kind));
 431   }
 432 }
 433
 434 Expected<int64_t> readAddendThumb(LinkGraph &G, Block &B, Edge::OffsetT Offset,
 435                                   Edge::Kind Kind, const ArmConfig &ArmCfg) {
 436   ThumbRelocation R(B.getContent().data() + Offset);
 437   if (Error Err = checkOpcode(G, R, Kind))
 438     return std::move(Err);
 439
 440   switch (Kind) {
 441   case Thumb_Call:
 442   case Thumb_Jump24:
 443     return LLVM_LIKELY(ArmCfg.J1J2BranchEncoding)
 444                ? decodeImmBT4BlT1BlxT2_J1J2(R.Hi, R.Lo)
 445                : decodeImmBT4BlT1BlxT2(R.Hi, R.Lo);
 446
 447   case Thumb_MovwAbsNC:
 448   case Thumb_MovwPrelNC:
 449     // Initial addend is interpreted as a signed value
 450     return SignExtend64<16>(decodeImmMovtT1MovwT3(R.Hi, R.Lo));
 451
 452   case Thumb_MovtAbs:
 453   case Thumb_MovtPrel:
 454     // Initial addend is interpreted as a signed value
 455     return SignExtend64<16>(decodeImmMovtT1MovwT3(R.Hi, R.Lo));
 456
 457   default:
 458     return make_error<JITLinkError>(
 459         "In graph " + G.getName() + ", section " + B.getSection().getName() +
 460         " can not read implicit addend for aarch32 edge kind " +
 461         G.getEdgeKindName(Kind));
 462   }
 463 }
 464
 465 Error applyFixupData(LinkGraph &G, Block &B, const Edge &E) {
 466   using namespace support;
 467
 468   char *BlockWorkingMem = B.getAlreadyMutableContent().data();
 469   char *FixupPtr = BlockWorkingMem + E.getOffset();
 470
 471   Edge::Kind Kind = E.getKind();
 472   uint64_t FixupAddress = (B.getAddress() + E.getOffset()).getValue();
 473   int64_t Addend = E.getAddend();
 474   Symbol &TargetSymbol = E.getTarget();
 475   uint64_t TargetAddress = TargetSymbol.getAddress().getValue();
 476
 477   // Data relocations have alignment 1, size 4 (except R_ARM_ABS8 and
 478   // R_ARM_ABS16) and write the full 32-bit result (except R_ARM_PREL31).
 479   switch (Kind) {
 480   case Data_Delta32: {
 481     int64_t Value = TargetAddress - FixupAddress + Addend;
 482     if (!isInt<32>(Value))
 483       return makeTargetOutOfRangeError(G, B, E);
 484     if (LLVM_LIKELY(G.getEndianness() == endianness::little))
 485       endian::write32le(FixupPtr, Value);
 486     else
 487       endian::write32be(FixupPtr, Value);
 488     return Error::success();
 489   }
 490   case Data_Pointer32: {
 491     int64_t Value = TargetAddress + Addend;
 492     if (!isUInt<32>(Value))
 493       return makeTargetOutOfRangeError(G, B, E);
 494     if (LLVM_LIKELY(G.getEndianness() == endianness::little))
 495       endian::write32le(FixupPtr, Value);
 496     else
 497       endian::write32be(FixupPtr, Value);
 498     return Error::success();
 499   }
 500   case Data_PRel31: {
 501     int64_t Value = TargetAddress - FixupAddress + Addend;
 502     if (!isInt<31>(Value))
 503       return makeTargetOutOfRangeError(G, B, E);
 504     if (LLVM_LIKELY(G.getEndianness() == endianness::little)) {
 505       uint32_t MSB = endian::read32le(FixupPtr) & 0x80000000;
 506       endian::write32le(FixupPtr, MSB | (Value & ~0x80000000));
 507     } else {
 508       uint32_t MSB = endian::read32be(FixupPtr) & 0x80000000;
 509       endian::write32be(FixupPtr, MSB | (Value & ~0x80000000));
 510     }
 511     return Error::success();
 512   }
 513   case Data_RequestGOTAndTransformToDelta32:
 514     llvm_unreachable("Should be transformed");
 515   default:
 516     return make_error<JITLinkError>(
 517         "In graph " + G.getName() + ", section " + B.getSection().getName() +
 518         " encountered unfixable aarch32 edge kind " +
 519         G.getEdgeKindName(E.getKind()));
 520   }
 521 }
 522
 523 Error applyFixupArm(LinkGraph &G, Block &B, const Edge &E) {
 524   WritableArmRelocation R(B.getAlreadyMutableContent().data() + E.getOffset());
 525   Edge::Kind Kind = E.getKind();
 526   if (Error Err = checkOpcode(G, R, Kind))
 527     return Err;
 528
 529   uint64_t FixupAddress = (B.getAddress() + E.getOffset()).getValue();
 530   int64_t Addend = E.getAddend();
 531   Symbol &TargetSymbol = E.getTarget();
 532   uint64_t TargetAddress = TargetSymbol.getAddress().getValue();
 533
 534   switch (Kind) {
 535   case Arm_Jump24: {
 536     if (hasTargetFlags(TargetSymbol, ThumbSymbol))
 537       return make_error<JITLinkError>("Branch relocation needs interworking "
 538                                       "stub when bridging to Thumb: " +
 539                                       StringRef(G.getEdgeKindName(Kind)));
 540
 541     int64_t Value = TargetAddress - FixupAddress + Addend;
 542
 543     if (!isInt<26>(Value))
 544       return makeTargetOutOfRangeError(G, B, E);
 545     writeImmediate<Arm_Jump24>(R, encodeImmBA1BlA1BlxA2(Value));
 546
 547     return Error::success();
 548   }
 549   case Arm_Call: {
 550     if ((R.Wd & FixupInfo<Arm_Call>::CondMask) !=
 551         FixupInfo<Arm_Call>::Unconditional)
 552       return make_error<JITLinkError>("Relocation expects an unconditional "
 553                                       "BL/BLX branch instruction: " +
 554                                       StringRef(G.getEdgeKindName(Kind)));
 555
 556     int64_t Value = TargetAddress - FixupAddress + Addend;
 557
 558     // The call instruction itself is Arm. The call destination can either be
 559     // Thumb or Arm. We use BL to stay in Arm and BLX to change to Thumb.
 560     bool TargetIsThumb = hasTargetFlags(TargetSymbol, ThumbSymbol);
 561     bool InstrIsBlx = (~R.Wd & FixupInfo<Arm_Call>::BitBlx) == 0;
 562     if (TargetIsThumb != InstrIsBlx) {
 563       if (LLVM_LIKELY(TargetIsThumb)) {
 564         // Change opcode BL -> BLX
 565         R.Wd = R.Wd | FixupInfo<Arm_Call>::BitBlx;
 566         R.Wd = R.Wd & ~FixupInfo<Arm_Call>::BitH;
 567       } else {
 568         // Change opcode BLX -> BL
 569         R.Wd = R.Wd & ~FixupInfo<Arm_Call>::BitBlx;
 570       }
 571     }
 572
 573     if (!isInt<26>(Value))
 574       return makeTargetOutOfRangeError(G, B, E);
 575     writeImmediate<Arm_Call>(R, encodeImmBA1BlA1BlxA2(Value));
 576
 577     return Error::success();
 578   }
 579   case Arm_MovwAbsNC: {
 580     uint16_t Value = (TargetAddress + Addend) & 0xffff;
 581     writeImmediate<Arm_MovwAbsNC>(R, encodeImmMovtA1MovwA2(Value));
 582     return Error::success();
 583   }
 584   case Arm_MovtAbs: {
 585     uint16_t Value = ((TargetAddress + Addend) >> 16) & 0xffff;
 586     writeImmediate<Arm_MovtAbs>(R, encodeImmMovtA1MovwA2(Value));
 587     return Error::success();
 588   }
 589   default:
 590     return make_error<JITLinkError>(
 591         "In graph " + G.getName() + ", section " + B.getSection().getName() +
 592         " encountered unfixable aarch32 edge kind " +
 593         G.getEdgeKindName(E.getKind()));
 594   }
 595 }
 596
 597 Error applyFixupThumb(LinkGraph &G, Block &B, const Edge &E,
 598                       const ArmConfig &ArmCfg) {
 599   WritableThumbRelocation R(B.getAlreadyMutableContent().data() +
 600                             E.getOffset());
 601   Edge::Kind Kind = E.getKind();
 602   if (Error Err = checkOpcode(G, R, Kind))
 603     return Err;
 604
 605   uint64_t FixupAddress = (B.getAddress() + E.getOffset()).getValue();
 606   int64_t Addend = E.getAddend();
 607   Symbol &TargetSymbol = E.getTarget();
 608   uint64_t TargetAddress = TargetSymbol.getAddress().getValue();
 609
 610   switch (Kind) {
 611   case Thumb_Jump24: {
 612     if (!hasTargetFlags(TargetSymbol, ThumbSymbol))
 613       return make_error<JITLinkError>("Branch relocation needs interworking "
 614                                       "stub when bridging to ARM: " +
 615                                       StringRef(G.getEdgeKindName(Kind)));
 616
 617     int64_t Value = TargetAddress - FixupAddress + Addend;
 618     if (LLVM_LIKELY(ArmCfg.J1J2BranchEncoding)) {
 619       if (!isInt<25>(Value))
 620         return makeTargetOutOfRangeError(G, B, E);
 621       writeImmediate<Thumb_Jump24>(R, encodeImmBT4BlT1BlxT2_J1J2(Value));
 622     } else {
 623       if (!isInt<22>(Value))
 624         return makeTargetOutOfRangeError(G, B, E);
 625       writeImmediate<Thumb_Jump24>(R, encodeImmBT4BlT1BlxT2(Value));
 626     }
 627
 628     return Error::success();
 629   }
 630
 631   case Thumb_Call: {
 632     int64_t Value = TargetAddress - FixupAddress + Addend;
 633
 634     // The call instruction itself is Thumb. The call destination can either be
 635     // Thumb or Arm. We use BL to stay in Thumb and BLX to change to Arm.
 636     bool TargetIsArm = !hasTargetFlags(TargetSymbol, ThumbSymbol);
 637     bool InstrIsBlx = (R.Lo & FixupInfo<Thumb_Call>::LoBitNoBlx) == 0;
 638     if (TargetIsArm != InstrIsBlx) {
 639       if (LLVM_LIKELY(TargetIsArm)) {
 640         // Change opcode BL -> BLX and fix range value: account for 4-byte
 641         // aligned destination while instruction may only be 2-byte aligned
 642         R.Lo = R.Lo & ~FixupInfo<Thumb_Call>::LoBitNoBlx;
 643         R.Lo = R.Lo & ~FixupInfo<Thumb_Call>::LoBitH;
 644         Value = alignTo(Value, 4);
 645       } else {
 646         // Change opcode BLX -> BL
 647         R.Lo = R.Lo & ~FixupInfo<Thumb_Call>::LoBitNoBlx;
 648       }
 649     }
 650
 651     if (LLVM_LIKELY(ArmCfg.J1J2BranchEncoding)) {
 652       if (!isInt<25>(Value))
 653         return makeTargetOutOfRangeError(G, B, E);
 654       writeImmediate<Thumb_Call>(R, encodeImmBT4BlT1BlxT2_J1J2(Value));
 655     } else {
 656       if (!isInt<22>(Value))
 657         return makeTargetOutOfRangeError(G, B, E);
 658       writeImmediate<Thumb_Call>(R, encodeImmBT4BlT1BlxT2(Value));
 659     }
 660
 661     assert(((R.Lo & FixupInfo<Thumb_Call>::LoBitNoBlx) ||
 662             (R.Lo & FixupInfo<Thumb_Call>::LoBitH) == 0) &&
 663            "Opcode BLX implies H bit is clear (avoid UB in BLX T2)");
 664     return Error::success();
 665   }
 666
 667   case Thumb_MovwAbsNC: {
 668     uint16_t Value = (TargetAddress + Addend) & 0xffff;
 669     writeImmediate<Thumb_MovwAbsNC>(R, encodeImmMovtT1MovwT3(Value));
 670     return Error::success();
 671   }
 672   case Thumb_MovtAbs: {
 673     uint16_t Value = ((TargetAddress + Addend) >> 16) & 0xffff;
 674     writeImmediate<Thumb_MovtAbs>(R, encodeImmMovtT1MovwT3(Value));
 675     return Error::success();
 676   }
 677   case Thumb_MovwPrelNC: {
 678     uint16_t Value = ((TargetAddress + Addend - FixupAddress) & 0xffff);
 679     writeImmediate<Thumb_MovwPrelNC>(R, encodeImmMovtT1MovwT3(Value));
 680     return Error::success();
 681   }
 682   case Thumb_MovtPrel: {
 683     uint16_t Value = (((TargetAddress + Addend - FixupAddress) >> 16) & 0xffff);
 684     writeImmediate<Thumb_MovtPrel>(R, encodeImmMovtT1MovwT3(Value));
 685     return Error::success();
 686   }
 687
 688   default:
 689     return make_error<JITLinkError>(
 690         "In graph " + G.getName() + ", section " + B.getSection().getName() +
 691         " encountered unfixable aarch32 edge kind " +
 692         G.getEdgeKindName(E.getKind()));
 693   }
 694 }
 695
 696 const uint8_t GOTEntryInit[] = {
 697     0x00,
 698     0x00,
 699     0x00,
 700     0x00,
 701 };
 702
 703 /// Create a new node in the link-graph for the given pointer value.
 704 template <size_t Size>
 705 static Block &allocPointer(LinkGraph &G, Section &S,
 706                            const uint8_t (&Content)[Size]) {
 707   static_assert(Size == 4, "Pointers are 32-bit");
 708   constexpr uint64_t Alignment = 4;
 709   ArrayRef<char> Init(reinterpret_cast<const char *>(Content), Size);
 710   return G.createContentBlock(S, Init, orc::ExecutorAddr(), Alignment, 0);
 711 }
 712
 713 Symbol &GOTBuilder::createEntry(LinkGraph &G, Symbol &Target) {
 714   if (!GOTSection)
 715     GOTSection = &G.createSection(getSectionName(), orc::MemProt::Read);
 716   Block &B = allocPointer(G, *GOTSection, GOTEntryInit);
 717   constexpr int64_t GOTEntryAddend = 0;
 718   B.addEdge(Data_Pointer32, 0, Target, GOTEntryAddend);
 719   return G.addAnonymousSymbol(B, 0, B.getSize(), false, false);
 720 }
 721
 722 bool GOTBuilder::visitEdge(LinkGraph &G, Block *B, Edge &E) {
 723   Edge::Kind KindToSet = Edge::Invalid;
 724   switch (E.getKind()) {
 725   case aarch32::Data_RequestGOTAndTransformToDelta32: {
 726     KindToSet = aarch32::Data_Delta32;
 727     break;
 728   }
 729   default:
 730     return false;
 731   }
 732   LLVM_DEBUG(dbgs() << "  Transforming " << G.getEdgeKindName(E.getKind())
 733                     << " edge at " << B->getFixupAddress(E) << " ("
 734                     << B->getAddress() << " + "
 735                     << formatv("{0:x}", E.getOffset()) << ") into "
 736                     << G.getEdgeKindName(KindToSet) << "\n");
 737   E.setKind(KindToSet);
 738   E.setTarget(getEntryForTarget(G, E.getTarget()));
 739   return true;
 740 }
 741
 742 const uint8_t ArmThumbv5LdrPc[] = {
 743     0x78, 0x47,             // bx pc
 744     0xfd, 0xe7,             // b #-6 ; Arm recommended sequence to follow bx pc
 745     0x04, 0xf0, 0x1f, 0xe5, // ldr pc, [pc,#-4] ; L1
 746     0x00, 0x00, 0x00, 0x00, // L1: .word S
 747 };
 748
 749 const uint8_t Armv7ABS[] = {
 750     0x00, 0xc0, 0x00, 0xe3, // movw r12, #0x0000     ; lower 16-bit
 751     0x00, 0xc0, 0x40, 0xe3, // movt r12, #0x0000     ; upper 16-bit
 752     0x1c, 0xff, 0x2f, 0xe1  // bx   r12
 753 };
 754
 755 const uint8_t Thumbv7ABS[] = {
 756     0x40, 0xf2, 0x00, 0x0c, // movw r12, #0x0000    ; lower 16-bit
 757     0xc0, 0xf2, 0x00, 0x0c, // movt r12, #0x0000    ; upper 16-bit
 758     0x60, 0x47              // bx   r12
 759 };
 760
 761 /// Create a new node in the link-graph for the given stub template.
 762 template <size_t Size>
 763 static Block &allocStub(LinkGraph &G, Section &S, const uint8_t (&Code)[Size]) {
 764   constexpr uint64_t Alignment = 4;
 765   ArrayRef<char> Template(reinterpret_cast<const char *>(Code), Size);
 766   return G.createContentBlock(S, Template, orc::ExecutorAddr(), Alignment, 0);
 767 }
 768
 769 static Block &createStubPrev7(LinkGraph &G, Section &S, Symbol &Target) {
 770   Block &B = allocStub(G, S, ArmThumbv5LdrPc);
 771   B.addEdge(Data_Pointer32, 8, Target, 0);
 772   return B;
 773 }
 774
 775 static Block &createStubThumbv7(LinkGraph &G, Section &S, Symbol &Target) {
 776   Block &B = allocStub(G, S, Thumbv7ABS);
 777   B.addEdge(Thumb_MovwAbsNC, 0, Target, 0);
 778   B.addEdge(Thumb_MovtAbs, 4, Target, 0);
 779
 780   [[maybe_unused]] const char *StubPtr = B.getContent().data();
 781   [[maybe_unused]] HalfWords Reg12 = encodeRegMovtT1MovwT3(12);
 782   assert(checkRegister<Thumb_MovwAbsNC>(StubPtr, Reg12) &&
 783          checkRegister<Thumb_MovtAbs>(StubPtr + 4, Reg12) &&
 784          "Linker generated stubs may only corrupt register r12 (IP)");
 785   return B;
 786 }
 787
 788 static Block &createStubArmv7(LinkGraph &G, Section &S, Symbol &Target) {
 789   Block &B = allocStub(G, S, Armv7ABS);
 790   B.addEdge(Arm_MovwAbsNC, 0, Target, 0);
 791   B.addEdge(Arm_MovtAbs, 4, Target, 0);
 792
 793   [[maybe_unused]] const char *StubPtr = B.getContent().data();
 794   [[maybe_unused]] uint32_t Reg12 = encodeRegMovtA1MovwA2(12);
 795   assert(checkRegister<Arm_MovwAbsNC>(StubPtr, Reg12) &&
 796          checkRegister<Arm_MovtAbs>(StubPtr + 4, Reg12) &&
 797          "Linker generated stubs may only corrupt register r12 (IP)");
 798   return B;
 799 }
 800
 801 static bool needsStub(const Edge &E) {
 802   Symbol &Target = E.getTarget();
 803
 804   // Create stubs for external branch targets.
 805   if (!Target.isDefined()) {
 806     switch (E.getKind()) {
 807     case Arm_Call:
 808     case Arm_Jump24:
 809     case Thumb_Call:
 810     case Thumb_Jump24:
 811       return true;
 812     default:
 813       return false;
 814     }
 815   }
 816
 817   // For local targets, create interworking stubs if we switch Arm/Thumb with an
 818   // instruction that cannot switch the instruction set state natively.
 819   bool TargetIsThumb = Target.getTargetFlags() & ThumbSymbol;
 820   switch (E.getKind()) {
 821   case Arm_Jump24:
 822     return TargetIsThumb; // Branch to Thumb needs interworking stub
 823   case Thumb_Jump24:
 824     return !TargetIsThumb; // Branch to Arm needs interworking stub
 825   default:
 826     break;
 827   }
 828
 829   return false;
 830 }
 831
 832 // The ArmThumbv5LdrPc stub has 2 entrypoints: Thumb at offset 0 is taken only
 833 // for Thumb B instructions. Thumb BL is rewritten to BLX and takes the Arm
 834 // entrypoint at offset 4. Arm branches always use that one.
 835 Symbol *StubsManager_prev7::getOrCreateSlotEntrypoint(LinkGraph &G,
 836                                                       StubMapEntry &Slot,
 837                                                       bool Thumb) {
 838   constexpr orc::ExecutorAddrDiff ThumbEntrypointOffset = 0;
 839   constexpr orc::ExecutorAddrDiff ArmEntrypointOffset = 4;
 840   if (Thumb && !Slot.ThumbEntry) {
 841     Slot.ThumbEntry =
 842         &G.addAnonymousSymbol(*Slot.B, ThumbEntrypointOffset, 4, true, false);
 843     Slot.ThumbEntry->setTargetFlags(ThumbSymbol);
 844   }
 845   if (!Thumb && !Slot.ArmEntry)
 846     Slot.ArmEntry =
 847         &G.addAnonymousSymbol(*Slot.B, ArmEntrypointOffset, 8, true, false);
 848   return Thumb ? Slot.ThumbEntry : Slot.ArmEntry;
 849 }
 850
 851 bool StubsManager_prev7::visitEdge(LinkGraph &G, Block *B, Edge &E) {
 852   if (!needsStub(E))
 853     return false;
 854
 855   Symbol &Target = E.getTarget();
 856   assert(Target.hasName() && "Edge cannot point to anonymous target");
 857   auto [Slot, NewStub] = getStubMapSlot(Target.getName());
 858
 859   if (NewStub) {
 860     if (!StubsSection)
 861       StubsSection = &G.createSection(getSectionName(),
 862                                       orc::MemProt::Read | orc::MemProt::Exec);
 863     LLVM_DEBUG({
 864       dbgs() << "    Created stub entry for " << Target.getName() << " in "
 865              << StubsSection->getName() << "\n";
 866     });
 867     Slot->B = &createStubPrev7(G, *StubsSection, Target);
 868   }
 869
 870   // The ArmThumbv5LdrPc stub has 2 entrypoints: Thumb at offset 0 is taken only
 871   // for Thumb B instructions. Thumb BL is rewritten to BLX and takes the Arm
 872   // entrypoint at offset 4. Arm branches always use that one.
 873   bool UseThumb = E.getKind() == Thumb_Jump24;
 874   Symbol *StubEntrypoint = getOrCreateSlotEntrypoint(G, *Slot, UseThumb);
 875
 876   LLVM_DEBUG({
 877     dbgs() << "    Using " << (UseThumb ? "Thumb" : "Arm") << " entrypoint "
 878            << *StubEntrypoint << " in "
 879            << StubEntrypoint->getBlock().getSection().getName() << "\n";
 880   });
 881
 882   E.setTarget(*StubEntrypoint);
 883   return true;
 884 }
 885
 886 bool StubsManager_v7::visitEdge(LinkGraph &G, Block *B, Edge &E) {
 887   if (!needsStub(E))
 888     return false;
 889
 890   // Stub Arm/Thumb follows instruction set state at relocation site.
 891   // TODO: We may reduce them at relaxation time and reuse freed slots.
 892   bool MakeThumb = (E.getKind() > LastArmRelocation);
 893   LLVM_DEBUG(dbgs() << "  Preparing " << (MakeThumb ? "Thumb" : "Arm")
 894                     << " stub for " << G.getEdgeKindName(E.getKind())
 895                     << " edge at " << B->getFixupAddress(E) << " ("
 896                     << B->getAddress() << " + "
 897                     << formatv("{0:x}", E.getOffset()) << ")\n");
 898
 899   Symbol &Target = E.getTarget();
 900   assert(Target.hasName() && "Edge cannot point to anonymous target");
 901   Symbol *&StubSymbol = getStubSymbolSlot(Target.getName(), MakeThumb);
 902
 903   if (!StubSymbol) {
 904     if (!StubsSection)
 905       StubsSection = &G.createSection(getSectionName(),
 906                                       orc::MemProt::Read | orc::MemProt::Exec);
 907     Block &B = MakeThumb ? createStubThumbv7(G, *StubsSection, Target)
 908                          : createStubArmv7(G, *StubsSection, Target);
 909     StubSymbol = &G.addAnonymousSymbol(B, 0, B.getSize(), true, false);
 910     if (MakeThumb)
 911       StubSymbol->setTargetFlags(ThumbSymbol);
 912
 913     LLVM_DEBUG({
 914       dbgs() << "    Created " << (MakeThumb ? "Thumb" : "Arm") << " entry for "
 915              << Target.getName() << " in " << StubsSection->getName() << ": "
 916              << *StubSymbol << "\n";
 917     });
 918   }
 919
 920   assert(MakeThumb == (StubSymbol->getTargetFlags() & ThumbSymbol) &&
 921          "Instruction set states of stub and relocation site should be equal");
 922   LLVM_DEBUG({
 923     dbgs() << "    Using " << (MakeThumb ? "Thumb" : "Arm") << " entry "
 924            << *StubSymbol << " in "
 925            << StubSymbol->getBlock().getSection().getName() << "\n";
 926   });
 927
 928   E.setTarget(*StubSymbol);
 929   return true;
 930 }
 931
 932 const char *getEdgeKindName(Edge::Kind K) {
 933 #define KIND_NAME_CASE(K)                                                      \
 934   case K:                                                                      \
 935     return #K;
 936
 937   switch (K) {
 938     KIND_NAME_CASE(Data_Delta32)
 939     KIND_NAME_CASE(Data_Pointer32)
 940     KIND_NAME_CASE(Data_PRel31)
 941     KIND_NAME_CASE(Data_RequestGOTAndTransformToDelta32)
 942     KIND_NAME_CASE(Arm_Call)
 943     KIND_NAME_CASE(Arm_Jump24)
 944     KIND_NAME_CASE(Arm_MovwAbsNC)
 945     KIND_NAME_CASE(Arm_MovtAbs)
 946     KIND_NAME_CASE(Thumb_Call)
 947     KIND_NAME_CASE(Thumb_Jump24)
 948     KIND_NAME_CASE(Thumb_MovwAbsNC)
 949     KIND_NAME_CASE(Thumb_MovtAbs)
 950     KIND_NAME_CASE(Thumb_MovwPrelNC)
 951     KIND_NAME_CASE(Thumb_MovtPrel)
 952     KIND_NAME_CASE(None)
 953   default:
 954     return getGenericEdgeKindName(K);
 955   }
 956 #undef KIND_NAME_CASE
 957 }
 958
 959 const char *getCPUArchName(ARMBuildAttrs::CPUArch K) {
 960 #define CPUARCH_NAME_CASE(K)                                                   \
 961   case K:                                                                      \
 962     return #K;
 963
 964   using namespace ARMBuildAttrs;
 965   switch (K) {
 966     CPUARCH_NAME_CASE(Pre_v4)
 967     CPUARCH_NAME_CASE(v4)
 968     CPUARCH_NAME_CASE(v4T)
 969     CPUARCH_NAME_CASE(v5T)
 970     CPUARCH_NAME_CASE(v5TE)
 971     CPUARCH_NAME_CASE(v5TEJ)
 972     CPUARCH_NAME_CASE(v6)
 973     CPUARCH_NAME_CASE(v6KZ)
 974     CPUARCH_NAME_CASE(v6T2)
 975     CPUARCH_NAME_CASE(v6K)
 976     CPUARCH_NAME_CASE(v7)
 977     CPUARCH_NAME_CASE(v6_M)
 978     CPUARCH_NAME_CASE(v6S_M)
 979     CPUARCH_NAME_CASE(v7E_M)
 980     CPUARCH_NAME_CASE(v8_A)
 981     CPUARCH_NAME_CASE(v8_R)
 982     CPUARCH_NAME_CASE(v8_M_Base)
 983     CPUARCH_NAME_CASE(v8_M_Main)
 984     CPUARCH_NAME_CASE(v8_1_M_Main)
 985     CPUARCH_NAME_CASE(v9_A)
 986   }
 987   llvm_unreachable("Missing CPUArch in switch?");
 988 #undef CPUARCH_NAME_CASE
 989 }
 990
 991 } // namespace aarch32
 992 } // namespace jitlink
 993 } // namespace llvm