llvm/lib/Target/AArch64/AArch64A53Fix835769.cpp

   1 //===-- AArch64A53Fix835769.cpp -------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 // This pass changes code to work around Cortex-A53 erratum 835769.
   9 // It works around it by inserting a nop instruction in code sequences that
  10 // in some circumstances may trigger the erratum.
  11 // It inserts a nop instruction between a sequence of the following 2 classes
  12 // of instructions:
  13 // instr 1: mem-instr (including loads, stores and prefetches).
  14 // instr 2: non-SIMD integer multiply-accumulate writing 64-bit X registers.
  15 //===----------------------------------------------------------------------===//
  16
  17 #include "AArch64.h"
  18 #include "AArch64Subtarget.h"
  19 #include "llvm/ADT/Statistic.h"
  20 #include "llvm/CodeGen/MachineFunction.h"
  21 #include "llvm/CodeGen/MachineFunctionPass.h"
  22 #include "llvm/CodeGen/MachineInstr.h"
  23 #include "llvm/CodeGen/MachineInstrBuilder.h"
  24 #include "llvm/CodeGen/MachineRegisterInfo.h"
  25 #include "llvm/CodeGen/TargetInstrInfo.h"
  26 #include "llvm/Support/Debug.h"
  27 #include "llvm/Support/raw_ostream.h"
  28
  29 using namespace llvm;
  30
  31 #define DEBUG_TYPE "aarch64-fix-cortex-a53-835769"
  32
  33 STATISTIC(NumNopsAdded, "Number of Nops added to work around erratum 835769");
  34
  35 //===----------------------------------------------------------------------===//
  36 // Helper functions
  37
  38 // Is the instruction a match for the instruction that comes first in the
  39 // sequence of instructions that can trigger the erratum?
  40 static bool isFirstInstructionInSequence(MachineInstr *MI) {
  41   // Must return true if this instruction is a load, a store or a prefetch.
  42   switch (MI->getOpcode()) {
  43   case AArch64::PRFMl:
  44   case AArch64::PRFMroW:
  45   case AArch64::PRFMroX:
  46   case AArch64::PRFMui:
  47   case AArch64::PRFUMi:
  48     return true;
  49   default:
  50     return MI->mayLoadOrStore();
  51   }
  52 }
  53
  54 // Is the instruction a match for the instruction that comes second in the
  55 // sequence that can trigger the erratum?
  56 static bool isSecondInstructionInSequence(MachineInstr *MI) {
  57   // Must return true for non-SIMD integer multiply-accumulates, writing
  58   // to a 64-bit register.
  59   switch (MI->getOpcode()) {
  60   // Erratum cannot be triggered when the destination register is 32 bits,
  61   // therefore only include the following.
  62   case AArch64::MSUBXrrr:
  63   case AArch64::MADDXrrr:
  64   case AArch64::SMADDLrrr:
  65   case AArch64::SMSUBLrrr:
  66   case AArch64::UMADDLrrr:
  67   case AArch64::UMSUBLrrr:
  68     // Erratum can only be triggered by multiply-adds, not by regular
  69     // non-accumulating multiplies, i.e. when Ra=XZR='11111'
  70     return MI->getOperand(3).getReg() != AArch64::XZR;
  71   default:
  72     return false;
  73   }
  74 }
  75
  76
  77 //===----------------------------------------------------------------------===//
  78
  79 namespace {
  80 class AArch64A53Fix835769 : public MachineFunctionPass {
  81   const TargetInstrInfo *TII;
  82
  83 public:
  84   static char ID;
  85   explicit AArch64A53Fix835769() : MachineFunctionPass(ID) {
  86     initializeAArch64A53Fix835769Pass(*PassRegistry::getPassRegistry());
  87   }
  88
  89   bool runOnMachineFunction(MachineFunction &F) override;
  90
  91   MachineFunctionProperties getRequiredProperties() const override {
  92     return MachineFunctionProperties().set(
  93         MachineFunctionProperties::Property::NoVRegs);
  94   }
  95
  96   StringRef getPassName() const override {
  97     return "Workaround A53 erratum 835769 pass";
  98   }
  99
 100   void getAnalysisUsage(AnalysisUsage &AU) const override {
 101     AU.setPreservesCFG();
 102     MachineFunctionPass::getAnalysisUsage(AU);
 103   }
 104
 105 private:
 106   bool runOnBasicBlock(MachineBasicBlock &MBB);
 107 };
 108 char AArch64A53Fix835769::ID = 0;
 109
 110 } // end anonymous namespace
 111
 112 INITIALIZE_PASS(AArch64A53Fix835769, "aarch64-fix-cortex-a53-835769-pass",
 113                 "AArch64 fix for A53 erratum 835769", false, false)
 114
 115 //===----------------------------------------------------------------------===//
 116
 117 bool
 118 AArch64A53Fix835769::runOnMachineFunction(MachineFunction &F) {
 119   LLVM_DEBUG(dbgs() << "***** AArch64A53Fix835769 *****\n");
 120   auto &STI = F.getSubtarget<AArch64Subtarget>();
 121   // Fix not requested, skip pass.
 122   if (!STI.fixCortexA53_835769())
 123     return false;
 124
 125   bool Changed = false;
 126   TII = STI.getInstrInfo();
 127
 128   for (auto &MBB : F) {
 129     Changed |= runOnBasicBlock(MBB);
 130   }
 131   return Changed;
 132 }
 133
 134 // Return the block that was fallen through to get to MBB, if any,
 135 // otherwise nullptr.
 136 static MachineBasicBlock *getBBFallenThrough(MachineBasicBlock *MBB,
 137                                              const TargetInstrInfo *TII) {
 138   // Get the previous machine basic block in the function.
 139   MachineFunction::iterator MBBI(MBB);
 140
 141   // Can't go off top of function.
 142   if (MBBI == MBB->getParent()->begin())
 143     return nullptr;
 144
 145   MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
 146   SmallVector<MachineOperand, 2> Cond;
 147
 148   MachineBasicBlock *PrevBB = &*std::prev(MBBI);
 149   for (MachineBasicBlock *S : MBB->predecessors())
 150     if (S == PrevBB && !TII->analyzeBranch(*PrevBB, TBB, FBB, Cond) && !TBB &&
 151         !FBB)
 152       return S;
 153
 154   return nullptr;
 155 }
 156
 157 // Iterate through fallen through blocks trying to find a previous non-pseudo if
 158 // there is one, otherwise return nullptr. Only look for instructions in
 159 // previous blocks, not the current block, since we only use this to look at
 160 // previous blocks.
 161 static MachineInstr *getLastNonPseudo(MachineBasicBlock &MBB,
 162                                       const TargetInstrInfo *TII) {
 163   MachineBasicBlock *FMBB = &MBB;
 164
 165   // If there is no non-pseudo in the current block, loop back around and try
 166   // the previous block (if there is one).
 167   while ((FMBB = getBBFallenThrough(FMBB, TII))) {
 168     for (MachineInstr &I : llvm::reverse(*FMBB))
 169       if (!I.isPseudo())
 170         return &I;
 171   }
 172
 173   // There was no previous non-pseudo in the fallen through blocks
 174   return nullptr;
 175 }
 176
 177 static void insertNopBeforeInstruction(MachineBasicBlock &MBB, MachineInstr* MI,
 178                                        const TargetInstrInfo *TII) {
 179   // If we are the first instruction of the block, put the NOP at the end of
 180   // the previous fallthrough block
 181   if (MI == &MBB.front()) {
 182     MachineInstr *I = getLastNonPseudo(MBB, TII);
 183     assert(I && "Expected instruction");
 184     DebugLoc DL = I->getDebugLoc();
 185     BuildMI(I->getParent(), DL, TII->get(AArch64::HINT)).addImm(0);
 186   }
 187   else {
 188     DebugLoc DL = MI->getDebugLoc();
 189     BuildMI(MBB, MI, DL, TII->get(AArch64::HINT)).addImm(0);
 190   }
 191
 192   ++NumNopsAdded;
 193 }
 194
 195 bool
 196 AArch64A53Fix835769::runOnBasicBlock(MachineBasicBlock &MBB) {
 197   bool Changed = false;
 198   LLVM_DEBUG(dbgs() << "Running on MBB: " << MBB
 199                     << " - scanning instructions...\n");
 200
 201   // First, scan the basic block, looking for a sequence of 2 instructions
 202   // that match the conditions under which the erratum may trigger.
 203
 204   // List of terminating instructions in matching sequences
 205   std::vector<MachineInstr*> Sequences;
 206   unsigned Idx = 0;
 207   MachineInstr *PrevInstr = nullptr;
 208
 209   // Try and find the last non-pseudo instruction in any fallen through blocks,
 210   // if there isn't one, then we use nullptr to represent that.
 211   PrevInstr = getLastNonPseudo(MBB, TII);
 212
 213   for (auto &MI : MBB) {
 214     MachineInstr *CurrInstr = &MI;
 215     LLVM_DEBUG(dbgs() << "  Examining: " << MI);
 216     if (PrevInstr) {
 217       LLVM_DEBUG(dbgs() << "    PrevInstr: " << *PrevInstr
 218                         << "    CurrInstr: " << *CurrInstr
 219                         << "    isFirstInstructionInSequence(PrevInstr): "
 220                         << isFirstInstructionInSequence(PrevInstr) << "\n"
 221                         << "    isSecondInstructionInSequence(CurrInstr): "
 222                         << isSecondInstructionInSequence(CurrInstr) << "\n");
 223       if (isFirstInstructionInSequence(PrevInstr) &&
 224           isSecondInstructionInSequence(CurrInstr)) {
 225         LLVM_DEBUG(dbgs() << "   ** pattern found at Idx " << Idx << "!\n");
 226         (void) Idx;
 227         Sequences.push_back(CurrInstr);
 228       }
 229     }
 230     if (!CurrInstr->isPseudo())
 231       PrevInstr = CurrInstr;
 232     ++Idx;
 233   }
 234
 235   LLVM_DEBUG(dbgs() << "Scan complete, " << Sequences.size()
 236                     << " occurrences of pattern found.\n");
 237
 238   // Then update the basic block, inserting nops between the detected sequences.
 239   for (auto &MI : Sequences) {
 240     Changed = true;
 241     insertNopBeforeInstruction(MBB, MI, TII);
 242   }
 243
 244   return Changed;
 245 }
 246
 247 // Factory function used by AArch64TargetMachine to add the pass to
 248 // the passmanager.
 249 FunctionPass *llvm::createAArch64A53Fix835769() {
 250   return new AArch64A53Fix835769();
 251 }