llvm/lib/ExecutionEngine/JITLink/aarch64.cpp

   1 //===---- aarch64.cpp - Generic JITLink aarch64 edge kinds, utilities -----===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // Generic utilities for graphs representing aarch64 objects.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
  14
  15 #include "llvm/Support/BinaryStreamWriter.h"
  16
  17 #define DEBUG_TYPE "jitlink"
  18
  19 namespace llvm {
  20 namespace jitlink {
  21 namespace aarch64 {
  22
  23 const char NullPointerContent[8] = {0x00, 0x00, 0x00, 0x00,
  24                                     0x00, 0x00, 0x00, 0x00};
  25
  26 const char PointerJumpStubContent[12] = {
  27     0x10, 0x00, 0x00, (char)0x90u, // ADRP x16, <imm>@page21
  28     0x10, 0x02, 0x40, (char)0xf9u, // LDR x16, [x16, <imm>@pageoff12]
  29     0x00, 0x02, 0x1f, (char)0xd6u  // BR  x16
  30 };
  31
  32 const char ReentryTrampolineContent[8] = {
  33     (char)0xfd, 0x7b, (char)0xbf, (char)0xa9, // STP x30, [sp, #-8]
  34     0x00,       0x00, 0x00,       (char)0x94  // BL
  35 };
  36
  37 const char *getEdgeKindName(Edge::Kind R) {
  38   switch (R) {
  39   case Pointer64:
  40     return "Pointer64";
  41   case Pointer32:
  42     return "Pointer32";
  43   case Delta64:
  44     return "Delta64";
  45   case Delta32:
  46     return "Delta32";
  47   case NegDelta64:
  48     return "NegDelta64";
  49   case NegDelta32:
  50     return "NegDelta32";
  51   case Branch26PCRel:
  52     return "Branch26PCRel";
  53   case MoveWide16:
  54     return "MoveWide16";
  55   case LDRLiteral19:
  56     return "LDRLiteral19";
  57   case TestAndBranch14PCRel:
  58     return "TestAndBranch14PCRel";
  59   case CondBranch19PCRel:
  60     return "CondBranch19PCRel";
  61   case ADRLiteral21:
  62     return "ADRLiteral21";
  63   case Page21:
  64     return "Page21";
  65   case PageOffset12:
  66     return "PageOffset12";
  67   case GotPageOffset15:
  68     return "GotPageOffset15";
  69   case RequestGOTAndTransformToPage21:
  70     return "RequestGOTAndTransformToPage21";
  71   case RequestGOTAndTransformToPageOffset12:
  72     return "RequestGOTAndTransformToPageOffset12";
  73   case RequestGOTAndTransformToPageOffset15:
  74     return "RequestGOTAndTransformToPageOffset15";
  75   case RequestGOTAndTransformToDelta32:
  76     return "RequestGOTAndTransformToDelta32";
  77   case RequestTLVPAndTransformToPage21:
  78     return "RequestTLVPAndTransformToPage21";
  79   case RequestTLVPAndTransformToPageOffset12:
  80     return "RequestTLVPAndTransformToPageOffset12";
  81   case RequestTLSDescEntryAndTransformToPage21:
  82     return "RequestTLSDescEntryAndTransformToPage21";
  83   case RequestTLSDescEntryAndTransformToPageOffset12:
  84     return "RequestTLSDescEntryAndTransformToPageOffset12";
  85   default:
  86     return getGenericEdgeKindName(static_cast<Edge::Kind>(R));
  87   }
  88 }
  89
  90 // Write a 64-bit GPR -> GPR move.
  91 template <typename AppendFtor>
  92 static Error writeMovRegRegSeq(AppendFtor &Append, uint64_t DstReg,
  93                                uint64_t SrcReg) {
  94   assert(DstReg < 32 && "Dst reg out of range");
  95   assert(SrcReg < 32 && "Src reg out of range");
  96
  97   if (DstReg == SrcReg)
  98     return Error::success();
  99
 100   constexpr uint32_t MOVGPR64Template = 0xaa0003e0;
 101   constexpr uint32_t DstRegIndex = 0;
 102   constexpr uint32_t SrcRegIndex = 16;
 103   uint32_t Instr = MOVGPR64Template;
 104   Instr |= DstReg << DstRegIndex;
 105   Instr |= SrcReg << SrcRegIndex;
 106   return Append(Instr);
 107 }
 108
 109 // Generate a sequence of imm writes to assign the given value.
 110 template <typename AppendFtor>
 111 static Error writeMovRegImm64Seq(AppendFtor &Append, uint64_t Reg,
 112                                  uint64_t Imm) {
 113   assert(Reg < 32 && "Invalid register number");
 114
 115   constexpr uint32_t MovRegImm64Template = 0xd2800000;
 116   constexpr unsigned PreserveBitIndex = 29;
 117   constexpr unsigned ShiftBitsIndex = 21;
 118   constexpr unsigned ImmBitsIndex = 5;
 119
 120   bool PreserveRegValue = false;
 121   for (unsigned I = 0; I != 4; ++I) {
 122     uint32_t ImmBits = Imm & 0xffff;
 123     Imm >>= 16;
 124
 125     // Skip any all-zero immediates after the first one.
 126     if (PreserveRegValue && !ImmBits)
 127       continue;
 128
 129     uint32_t Instr = MovRegImm64Template;
 130     Instr |= PreserveRegValue << PreserveBitIndex;
 131     Instr |= (I << ShiftBitsIndex);
 132     Instr |= ImmBits << ImmBitsIndex;
 133     Instr |= Reg;
 134     if (auto Err = Append(Instr))
 135       return Err;
 136     PreserveRegValue = true;
 137   }
 138
 139   return Error::success();
 140 }
 141
 142 template <typename AppendFtor>
 143 static Error
 144 writePACSignSeq(AppendFtor &Append, unsigned DstReg, orc::ExecutorAddr RawAddr,
 145                 unsigned RawAddrReg, unsigned DiscriminatorReg, unsigned Key,
 146                 uint64_t EncodedDiscriminator, bool AddressDiversify) {
 147   assert(DstReg < 32 && "DstReg out of range");
 148   assert(RawAddrReg < 32 && "AddrReg out of range");
 149   assert(DiscriminatorReg < 32 && "DiscriminatorReg out of range");
 150   assert(EncodedDiscriminator < 0x10000 && "EncodedDiscriminator out of range");
 151
 152   if (AddressDiversify) {
 153     // Move the address into the discriminator register.
 154     if (auto Err = writeMovRegRegSeq(Append, DiscriminatorReg, RawAddrReg))
 155       return Err;
 156     // Blend encoded discriminator if there is one.
 157     if (EncodedDiscriminator) {
 158       constexpr uint32_t MOVKTemplate = 0xf2e00000;
 159       constexpr unsigned ImmIndex = 5;
 160       uint32_t BlendInstr = MOVKTemplate;
 161       BlendInstr |= EncodedDiscriminator << ImmIndex;
 162       BlendInstr |= DiscriminatorReg;
 163       if (auto Err = Append(BlendInstr))
 164         return Err;
 165     }
 166   } else if (EncodedDiscriminator) {
 167     // Move the encoded discriminator into the discriminator register.
 168     if (auto Err =
 169             writeMovRegImm64Seq(Append, DiscriminatorReg, EncodedDiscriminator))
 170       return Err;
 171   } else
 172     DiscriminatorReg = 31; // WZR
 173
 174   constexpr uint32_t PACTemplate = 0xdac10000;
 175   constexpr unsigned ZBitIndex = 13;
 176   constexpr unsigned KeyIndex = 10;
 177   constexpr unsigned DiscriminatorRegIndex = 5;
 178
 179   uint32_t Instr = PACTemplate;
 180   Instr |= (DiscriminatorReg == 31) << ZBitIndex;
 181   Instr |= Key << KeyIndex;
 182   Instr |= DiscriminatorReg << DiscriminatorRegIndex;
 183   Instr |= DstReg;
 184
 185   return Append(Instr);
 186 }
 187
 188 template <typename AppendFtor>
 189 static Error writeStoreRegSeq(AppendFtor &Append, unsigned DstLocReg,
 190                               unsigned SrcReg) {
 191   assert(DstLocReg < 32 && "DstLocReg out of range");
 192   assert(SrcReg < 32 && "SrcReg out of range");
 193
 194   constexpr uint32_t STRTemplate = 0xf9000000;
 195   constexpr unsigned DstLocRegIndex = 5;
 196   constexpr unsigned SrcRegIndex = 0;
 197
 198   uint32_t Instr = STRTemplate;
 199   Instr |= DstLocReg << DstLocRegIndex;
 200   Instr |= SrcReg << SrcRegIndex;
 201
 202   return Append(Instr);
 203 }
 204
 205 const char *getPointerSigningFunctionSectionName() { return "$__ptrauth_sign"; }
 206
 207 /// Creates a pointer signing function section, block, and symbol to reserve
 208 /// space for a signing function for this LinkGraph. Clients should insert this
 209 /// pass in the post-prune phase, and add the paired
 210 /// lowerPointer64AuthEdgesToSigningFunction pass to the pre-fixup phase.
 211 Error createEmptyPointerSigningFunction(LinkGraph &G) {
 212   LLVM_DEBUG({
 213     dbgs() << "Creating empty pointer signing function for " << G.getName()
 214            << "\n";
 215   });
 216
 217   // FIXME: We could put a tighter bound on this if we inspected the ptrauth
 218   // info encoded in the addend -- the only actually unknown quantity is the
 219   // fixup location, and we can probably put constraints even on that.
 220   size_t NumPtrAuthFixupLocations = 0;
 221   for (auto *B : G.blocks())
 222     for (auto &E : B->edges())
 223       NumPtrAuthFixupLocations +=
 224           E.getKind() == aarch64::Pointer64Authenticated;
 225
 226   constexpr size_t MaxPtrSignSeqLength =
 227       4 + // To materialize the value to sign.
 228       4 + // To materialize the fixup location.
 229       3 + // To copy, blend discriminator, and sign
 230       1;  // To store the result.
 231
 232   // The maximum number of signing instructions required is the maximum per
 233   // location, times the number of locations, plus three instructions to
 234   // materialize the return value and return.
 235   size_t NumSigningInstrs = NumPtrAuthFixupLocations * MaxPtrSignSeqLength + 3;
 236
 237   // Create signing function section.
 238   auto &SigningSection =
 239       G.createSection(getPointerSigningFunctionSectionName(),
 240                       orc::MemProt::Read | orc::MemProt::Exec);
 241   SigningSection.setMemLifetime(orc::MemLifetime::Finalize);
 242
 243   size_t SigningFunctionSize = NumSigningInstrs * 4;
 244   auto &SigningFunctionBlock = G.createMutableContentBlock(
 245       SigningSection, G.allocateBuffer(SigningFunctionSize),
 246       orc::ExecutorAddr(), 4, 0);
 247   G.addAnonymousSymbol(SigningFunctionBlock, 0, SigningFunctionBlock.getSize(),
 248                        true, true);
 249
 250   LLVM_DEBUG({
 251     dbgs() << "  " << NumPtrAuthFixupLocations << " location(s) to sign, up to "
 252            << NumSigningInstrs << " instructions required ("
 253            << formatv("{0:x}", SigningFunctionBlock.getSize()) << " bytes)\n";
 254   });
 255
 256   return Error::success();
 257 }
 258
 259 /// Given a LinkGraph containing Pointer64Auth edges, transform those edges to
 260 /// Pointer64 and add code to sign the pointers in the executor.
 261 ///
 262 /// This function will add a $__ptrauth_sign section with finalization-lifetime
 263 /// containing an anonymous function that will sign all pointers in the graph.
 264 /// An allocation action will be added to run this function during finalization.
 265 Error lowerPointer64AuthEdgesToSigningFunction(LinkGraph &G) {
 266   LLVM_DEBUG({
 267     dbgs() << "Writing pointer signing function for " << G.getName() << "\n";
 268   });
 269
 270   constexpr unsigned Reg1 = 8;  // Holds pointer value to sign.
 271   constexpr unsigned Reg2 = 9;  // Holds fixup address.
 272   constexpr unsigned Reg3 = 10; // Temporary for discriminator value if needed.
 273
 274   // Find the signing function.
 275   auto *SigningSection =
 276       G.findSectionByName(getPointerSigningFunctionSectionName());
 277   assert(SigningSection && "Siging section missing");
 278   assert(SigningSection->blocks_size() == 1 &&
 279          "Unexpected number of blocks in signing section");
 280   assert(SigningSection->symbols_size() == 1 &&
 281          "Unexpected number of symbols in signing section");
 282
 283   auto &SigningFunctionSym = **SigningSection->symbols().begin();
 284   auto &SigningFunctionBlock = SigningFunctionSym.getBlock();
 285   auto SigningFunctionBuf = SigningFunctionBlock.getAlreadyMutableContent();
 286
 287   // Write the instructions to the block content.
 288   BinaryStreamWriter InstrWriter(
 289       {reinterpret_cast<uint8_t *>(SigningFunctionBuf.data()),
 290        SigningFunctionBuf.size()},
 291       G.getEndianness());
 292
 293   auto AppendInstr = [&](uint32_t Instr) {
 294     return InstrWriter.writeInteger(Instr);
 295   };
 296
 297   for (auto *B : G.blocks()) {
 298     for (auto EI = B->edges().begin(); EI != B->edges().end();) {
 299       auto &E = *EI;
 300       if (E.getKind() == aarch64::Pointer64Authenticated) {
 301         uint64_t EncodedInfo = E.getAddend();
 302         int32_t RealAddend = (uint32_t)(EncodedInfo & 0xffffffff);
 303         uint32_t InitialDiscriminator = (EncodedInfo >> 32) & 0xffff;
 304         bool AddressDiversify = (EncodedInfo >> 48) & 0x1;
 305         uint32_t Key = (EncodedInfo >> 49) & 0x3;
 306         uint32_t HighBits = EncodedInfo >> 51;
 307         auto ValueToSign = E.getTarget().getAddress() + RealAddend;
 308
 309         if (HighBits != 0x1000)
 310           return make_error<JITLinkError>(
 311               "Pointer64Auth edge at " +
 312               formatv("{0:x}", B->getFixupAddress(E).getValue()) +
 313               " has invalid encoded addend  " + formatv("{0:x}", EncodedInfo));
 314
 315 #ifndef NDEBUG
 316         const char *const KeyNames[] = {"IA", "IB", "DA", "DB"};
 317 #endif // NDEBUG
 318         LLVM_DEBUG({
 319           dbgs() << "  " << B->getFixupAddress(E) << " <- " << ValueToSign
 320                  << " : key = " << KeyNames[Key] << ", discriminator = "
 321                  << formatv("{0:x4}", InitialDiscriminator)
 322                  << ", address diversified = "
 323                  << (AddressDiversify ? "yes" : "no") << "\n";
 324         });
 325
 326         // Materialize pointer value.
 327         cantFail(
 328             writeMovRegImm64Seq(AppendInstr, Reg1, ValueToSign.getValue()));
 329
 330         // Materialize fixup pointer.
 331         cantFail(writeMovRegImm64Seq(AppendInstr, Reg2,
 332                                      B->getFixupAddress(E).getValue()));
 333
 334         // Write signing instruction(s).
 335         cantFail(writePACSignSeq(AppendInstr, Reg1, ValueToSign, Reg2, Reg3,
 336                                  Key, InitialDiscriminator, AddressDiversify));
 337
 338         // Store signed pointer.
 339         cantFail(writeStoreRegSeq(AppendInstr, Reg2, Reg1));
 340
 341         // Remove this edge.
 342         EI = B->removeEdge(EI);
 343       } else
 344         ++EI;
 345     }
 346   }
 347
 348   // Write epilogue. x0 = 0, x1 = 1 is an SPS serialized Error::success value.
 349   constexpr uint32_t RETInstr = 0xd65f03c0;
 350   cantFail(writeMovRegImm64Seq(AppendInstr, 0, 0)); // mov x0, #0
 351   cantFail(writeMovRegImm64Seq(AppendInstr, 1, 1)); // mov x1, #1
 352   cantFail(AppendInstr(RETInstr));                  // ret
 353
 354   // Add an allocation action to call the signing function.
 355   using namespace orc::shared;
 356   G.allocActions().push_back(
 357       {cantFail(WrapperFunctionCall::Create<SPSArgList<>>(
 358            SigningFunctionSym.getAddress())),
 359        {}});
 360
 361   return Error::success();
 362 }
 363
 364 } // namespace aarch64
 365 } // namespace jitlink
 366 } // namespace llvm