lib/Target/AArch64/AArch64A53Fix835769.cpp

   1 //===-- AArch64A53Fix835769.cpp -------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 // This pass changes code to work around Cortex-A53 erratum 835769.
   9 // It works around it by inserting a nop instruction in code sequences that
  10 // in some circumstances may trigger the erratum.
  11 // It inserts a nop instruction between a sequence of the following 2 classes
  12 // of instructions:
  13 // instr 1: mem-instr (including loads, stores and prefetches).
  14 // instr 2: non-SIMD integer multiply-accumulate writing 64-bit X registers.
  15 //===----------------------------------------------------------------------===//
  16
  17 #include "AArch64.h"
  18 #include "llvm/ADT/Statistic.h"
  19 #include "llvm/CodeGen/MachineFunction.h"
  20 #include "llvm/CodeGen/MachineFunctionPass.h"
  21 #include "llvm/CodeGen/MachineInstr.h"
  22 #include "llvm/CodeGen/MachineInstrBuilder.h"
  23 #include "llvm/CodeGen/MachineRegisterInfo.h"
  24 #include "llvm/CodeGen/TargetInstrInfo.h"
  25 #include "llvm/Support/Debug.h"
  26 #include "llvm/Support/raw_ostream.h"
  27
  28 using namespace llvm;
  29
  30 #define DEBUG_TYPE "aarch64-fix-cortex-a53-835769"
  31
  32 STATISTIC(NumNopsAdded, "Number of Nops added to work around erratum 835769");
  33
  34 //===----------------------------------------------------------------------===//
  35 // Helper functions
  36
  37 // Is the instruction a match for the instruction that comes first in the
  38 // sequence of instructions that can trigger the erratum?
  39 static bool isFirstInstructionInSequence(MachineInstr *MI) {
  40   // Must return true if this instruction is a load, a store or a prefetch.
  41   switch (MI->getOpcode()) {
  42   case AArch64::PRFMl:
  43   case AArch64::PRFMroW:
  44   case AArch64::PRFMroX:
  45   case AArch64::PRFMui:
  46   case AArch64::PRFUMi:
  47     return true;
  48   default:
  49     return MI->mayLoadOrStore();
  50   }
  51 }
  52
  53 // Is the instruction a match for the instruction that comes second in the
  54 // sequence that can trigger the erratum?
  55 static bool isSecondInstructionInSequence(MachineInstr *MI) {
  56   // Must return true for non-SIMD integer multiply-accumulates, writing
  57   // to a 64-bit register.
  58   switch (MI->getOpcode()) {
  59   // Erratum cannot be triggered when the destination register is 32 bits,
  60   // therefore only include the following.
  61   case AArch64::MSUBXrrr:
  62   case AArch64::MADDXrrr:
  63   case AArch64::SMADDLrrr:
  64   case AArch64::SMSUBLrrr:
  65   case AArch64::UMADDLrrr:
  66   case AArch64::UMSUBLrrr:
  67     // Erratum can only be triggered by multiply-adds, not by regular
  68     // non-accumulating multiplies, i.e. when Ra=XZR='11111'
  69     return MI->getOperand(3).getReg() != AArch64::XZR;
  70   default:
  71     return false;
  72   }
  73 }
  74
  75
  76 //===----------------------------------------------------------------------===//
  77
  78 namespace {
  79 class AArch64A53Fix835769 : public MachineFunctionPass {
  80   const TargetInstrInfo *TII;
  81
  82 public:
  83   static char ID;
  84   explicit AArch64A53Fix835769() : MachineFunctionPass(ID) {
  85     initializeAArch64A53Fix835769Pass(*PassRegistry::getPassRegistry());
  86   }
  87
  88   bool runOnMachineFunction(MachineFunction &F) override;
  89
  90   MachineFunctionProperties getRequiredProperties() const override {
  91     return MachineFunctionProperties().set(
  92         MachineFunctionProperties::Property::NoVRegs);
  93   }
  94
  95   StringRef getPassName() const override {
  96     return "Workaround A53 erratum 835769 pass";
  97   }
  98
  99   void getAnalysisUsage(AnalysisUsage &AU) const override {
 100     AU.setPreservesCFG();
 101     MachineFunctionPass::getAnalysisUsage(AU);
 102   }
 103
 104 private:
 105   bool runOnBasicBlock(MachineBasicBlock &MBB);
 106 };
 107 char AArch64A53Fix835769::ID = 0;
 108
 109 } // end anonymous namespace
 110
 111 INITIALIZE_PASS(AArch64A53Fix835769, "aarch64-fix-cortex-a53-835769-pass",
 112                 "AArch64 fix for A53 erratum 835769", false, false)
 113
 114 //===----------------------------------------------------------------------===//
 115
 116 bool
 117 AArch64A53Fix835769::runOnMachineFunction(MachineFunction &F) {
 118   LLVM_DEBUG(dbgs() << "***** AArch64A53Fix835769 *****\n");
 119   bool Changed = false;
 120   TII = F.getSubtarget().getInstrInfo();
 121
 122   for (auto &MBB : F) {
 123     Changed |= runOnBasicBlock(MBB);
 124   }
 125   return Changed;
 126 }
 127
 128 // Return the block that was fallen through to get to MBB, if any,
 129 // otherwise nullptr.
 130 static MachineBasicBlock *getBBFallenThrough(MachineBasicBlock *MBB,
 131                                              const TargetInstrInfo *TII) {
 132   // Get the previous machine basic block in the function.
 133   MachineFunction::iterator MBBI(MBB);
 134
 135   // Can't go off top of function.
 136   if (MBBI == MBB->getParent()->begin())
 137     return nullptr;
 138
 139   MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
 140   SmallVector<MachineOperand, 2> Cond;
 141
 142   MachineBasicBlock *PrevBB = &*std::prev(MBBI);
 143   for (MachineBasicBlock *S : MBB->predecessors())
 144     if (S == PrevBB && !TII->analyzeBranch(*PrevBB, TBB, FBB, Cond) && !TBB &&
 145         !FBB)
 146       return S;
 147
 148   return nullptr;
 149 }
 150
 151 // Iterate through fallen through blocks trying to find a previous non-pseudo if
 152 // there is one, otherwise return nullptr. Only look for instructions in
 153 // previous blocks, not the current block, since we only use this to look at
 154 // previous blocks.
 155 static MachineInstr *getLastNonPseudo(MachineBasicBlock &MBB,
 156                                       const TargetInstrInfo *TII) {
 157   MachineBasicBlock *FMBB = &MBB;
 158
 159   // If there is no non-pseudo in the current block, loop back around and try
 160   // the previous block (if there is one).
 161   while ((FMBB = getBBFallenThrough(FMBB, TII))) {
 162     for (MachineInstr &I : make_range(FMBB->rbegin(), FMBB->rend()))
 163       if (!I.isPseudo())
 164         return &I;
 165   }
 166
 167   // There was no previous non-pseudo in the fallen through blocks
 168   return nullptr;
 169 }
 170
 171 static void insertNopBeforeInstruction(MachineBasicBlock &MBB, MachineInstr* MI,
 172                                        const TargetInstrInfo *TII) {
 173   // If we are the first instruction of the block, put the NOP at the end of
 174   // the previous fallthrough block
 175   if (MI == &MBB.front()) {
 176     MachineInstr *I = getLastNonPseudo(MBB, TII);
 177     assert(I && "Expected instruction");
 178     DebugLoc DL = I->getDebugLoc();
 179     BuildMI(I->getParent(), DL, TII->get(AArch64::HINT)).addImm(0);
 180   }
 181   else {
 182     DebugLoc DL = MI->getDebugLoc();
 183     BuildMI(MBB, MI, DL, TII->get(AArch64::HINT)).addImm(0);
 184   }
 185
 186   ++NumNopsAdded;
 187 }
 188
 189 bool
 190 AArch64A53Fix835769::runOnBasicBlock(MachineBasicBlock &MBB) {
 191   bool Changed = false;
 192   LLVM_DEBUG(dbgs() << "Running on MBB: " << MBB
 193                     << " - scanning instructions...\n");
 194
 195   // First, scan the basic block, looking for a sequence of 2 instructions
 196   // that match the conditions under which the erratum may trigger.
 197
 198   // List of terminating instructions in matching sequences
 199   std::vector<MachineInstr*> Sequences;
 200   unsigned Idx = 0;
 201   MachineInstr *PrevInstr = nullptr;
 202
 203   // Try and find the last non-pseudo instruction in any fallen through blocks,
 204   // if there isn't one, then we use nullptr to represent that.
 205   PrevInstr = getLastNonPseudo(MBB, TII);
 206
 207   for (auto &MI : MBB) {
 208     MachineInstr *CurrInstr = &MI;
 209     LLVM_DEBUG(dbgs() << "  Examining: " << MI);
 210     if (PrevInstr) {
 211       LLVM_DEBUG(dbgs() << "    PrevInstr: " << *PrevInstr
 212                         << "    CurrInstr: " << *CurrInstr
 213                         << "    isFirstInstructionInSequence(PrevInstr): "
 214                         << isFirstInstructionInSequence(PrevInstr) << "\n"
 215                         << "    isSecondInstructionInSequence(CurrInstr): "
 216                         << isSecondInstructionInSequence(CurrInstr) << "\n");
 217       if (isFirstInstructionInSequence(PrevInstr) &&
 218           isSecondInstructionInSequence(CurrInstr)) {
 219         LLVM_DEBUG(dbgs() << "   ** pattern found at Idx " << Idx << "!\n");
 220         Sequences.push_back(CurrInstr);
 221       }
 222     }
 223     if (!CurrInstr->isPseudo())
 224       PrevInstr = CurrInstr;
 225     ++Idx;
 226   }
 227
 228   LLVM_DEBUG(dbgs() << "Scan complete, " << Sequences.size()
 229                     << " occurrences of pattern found.\n");
 230
 231   // Then update the basic block, inserting nops between the detected sequences.
 232   for (auto &MI : Sequences) {
 233     Changed = true;
 234     insertNopBeforeInstruction(MBB, MI, TII);
 235   }
 236
 237   return Changed;
 238 }
 239
 240 // Factory function used by AArch64TargetMachine to add the pass to
 241 // the passmanager.
 242 FunctionPass *llvm::createAArch64A53Fix835769() {
 243   return new AArch64A53Fix835769();
 244 }