llvm/lib/Target/Mips/MipsExpandPseudo.cpp

   1 //===-- MipsExpandPseudoInsts.cpp - Expand pseudo instructions ------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file contains a pass that expands pseudo instructions into target
  10 // instructions to allow proper scheduling, if-conversion, and other late
  11 // optimizations. This pass should be run after register allocation but before
  12 // the post-regalloc scheduling pass.
  13 //
  14 // This is currently only used for expanding atomic pseudos after register
  15 // allocation. We do this to avoid the fast register allocator introducing
  16 // spills between ll and sc. These stores cause some MIPS implementations to
  17 // abort the atomic RMW sequence.
  18 //
  19 //===----------------------------------------------------------------------===//
  20
  21 #include "Mips.h"
  22 #include "MipsInstrInfo.h"
  23 #include "MipsSubtarget.h"
  24 #include "llvm/CodeGen/LivePhysRegs.h"
  25 #include "llvm/CodeGen/MachineFunctionPass.h"
  26 #include "llvm/CodeGen/MachineInstrBuilder.h"
  27
  28 using namespace llvm;
  29
  30 #define DEBUG_TYPE "mips-pseudo"
  31
  32 namespace {
  33   class MipsExpandPseudo : public MachineFunctionPass {
  34   public:
  35     static char ID;
  36     MipsExpandPseudo() : MachineFunctionPass(ID) {}
  37
  38     const MipsInstrInfo *TII;
  39     const MipsSubtarget *STI;
  40
  41     bool runOnMachineFunction(MachineFunction &Fn) override;
  42
  43     MachineFunctionProperties getRequiredProperties() const override {
  44       return MachineFunctionProperties().set(
  45           MachineFunctionProperties::Property::NoVRegs);
  46     }
  47
  48     StringRef getPassName() const override {
  49       return "Mips pseudo instruction expansion pass";
  50     }
  51
  52   private:
  53     bool expandAtomicCmpSwap(MachineBasicBlock &MBB,
  54                              MachineBasicBlock::iterator MBBI,
  55                              MachineBasicBlock::iterator &NextMBBI);
  56     bool expandAtomicCmpSwapSubword(MachineBasicBlock &MBB,
  57                                     MachineBasicBlock::iterator MBBI,
  58                                     MachineBasicBlock::iterator &NextMBBI);
  59
  60     bool expandAtomicBinOp(MachineBasicBlock &BB,
  61                            MachineBasicBlock::iterator I,
  62                            MachineBasicBlock::iterator &NMBBI, unsigned Size);
  63     bool expandAtomicBinOpSubword(MachineBasicBlock &BB,
  64                                   MachineBasicBlock::iterator I,
  65                                   MachineBasicBlock::iterator &NMBBI);
  66
  67     bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
  68                   MachineBasicBlock::iterator &NMBB);
  69     bool expandMBB(MachineBasicBlock &MBB);
  70    };
  71   char MipsExpandPseudo::ID = 0;
  72 }
  73
  74 bool MipsExpandPseudo::expandAtomicCmpSwapSubword(
  75     MachineBasicBlock &BB, MachineBasicBlock::iterator I,
  76     MachineBasicBlock::iterator &NMBBI) {
  77
  78   MachineFunction *MF = BB.getParent();
  79
  80   const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
  81   DebugLoc DL = I->getDebugLoc();
  82   unsigned LL, SC;
  83
  84   unsigned ZERO = Mips::ZERO;
  85   unsigned BNE = Mips::BNE;
  86   unsigned BEQ = Mips::BEQ;
  87   unsigned SEOp =
  88       I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I8_POSTRA ? Mips::SEB : Mips::SEH;
  89
  90   if (STI->inMicroMipsMode()) {
  91       LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
  92       SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
  93       BNE = STI->hasMips32r6() ? Mips::BNEC_MMR6 : Mips::BNE_MM;
  94       BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
  95   } else {
  96     LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
  97                             : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
  98     SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
  99                             : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
 100   }
 101
 102   Register Dest = I->getOperand(0).getReg();
 103   Register Ptr = I->getOperand(1).getReg();
 104   Register Mask = I->getOperand(2).getReg();
 105   Register ShiftCmpVal = I->getOperand(3).getReg();
 106   Register Mask2 = I->getOperand(4).getReg();
 107   Register ShiftNewVal = I->getOperand(5).getReg();
 108   Register ShiftAmnt = I->getOperand(6).getReg();
 109   Register Scratch = I->getOperand(7).getReg();
 110   Register Scratch2 = I->getOperand(8).getReg();
 111
 112   // insert new blocks after the current block
 113   const BasicBlock *LLVM_BB = BB.getBasicBlock();
 114   MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB);
 115   MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB);
 116   MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB);
 117   MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
 118   MachineFunction::iterator It = ++BB.getIterator();
 119   MF->insert(It, loop1MBB);
 120   MF->insert(It, loop2MBB);
 121   MF->insert(It, sinkMBB);
 122   MF->insert(It, exitMBB);
 123
 124   // Transfer the remainder of BB and its successor edges to exitMBB.
 125   exitMBB->splice(exitMBB->begin(), &BB,
 126                   std::next(MachineBasicBlock::iterator(I)), BB.end());
 127   exitMBB->transferSuccessorsAndUpdatePHIs(&BB);
 128
 129   //  thisMBB:
 130   //    ...
 131   //    fallthrough --> loop1MBB
 132   BB.addSuccessor(loop1MBB, BranchProbability::getOne());
 133   loop1MBB->addSuccessor(sinkMBB);
 134   loop1MBB->addSuccessor(loop2MBB);
 135   loop1MBB->normalizeSuccProbs();
 136   loop2MBB->addSuccessor(loop1MBB);
 137   loop2MBB->addSuccessor(sinkMBB);
 138   loop2MBB->normalizeSuccProbs();
 139   sinkMBB->addSuccessor(exitMBB, BranchProbability::getOne());
 140
 141   // loop1MBB:
 142   //   ll dest, 0(ptr)
 143   //   and Mask', dest, Mask
 144   //   bne Mask', ShiftCmpVal, exitMBB
 145   BuildMI(loop1MBB, DL, TII->get(LL), Scratch).addReg(Ptr).addImm(0);
 146   BuildMI(loop1MBB, DL, TII->get(Mips::AND), Scratch2)
 147       .addReg(Scratch)
 148       .addReg(Mask);
 149   BuildMI(loop1MBB, DL, TII->get(BNE))
 150     .addReg(Scratch2).addReg(ShiftCmpVal).addMBB(sinkMBB);
 151
 152   // loop2MBB:
 153   //   and dest, dest, mask2
 154   //   or dest, dest, ShiftNewVal
 155   //   sc dest, dest, 0(ptr)
 156   //   beq dest, $0, loop1MBB
 157   BuildMI(loop2MBB, DL, TII->get(Mips::AND), Scratch)
 158       .addReg(Scratch, RegState::Kill)
 159       .addReg(Mask2);
 160   BuildMI(loop2MBB, DL, TII->get(Mips::OR), Scratch)
 161       .addReg(Scratch, RegState::Kill)
 162       .addReg(ShiftNewVal);
 163   BuildMI(loop2MBB, DL, TII->get(SC), Scratch)
 164       .addReg(Scratch, RegState::Kill)
 165       .addReg(Ptr)
 166       .addImm(0);
 167   BuildMI(loop2MBB, DL, TII->get(BEQ))
 168       .addReg(Scratch, RegState::Kill)
 169       .addReg(ZERO)
 170       .addMBB(loop1MBB);
 171
 172   //  sinkMBB:
 173   //    srl     srlres, Mask', shiftamt
 174   //    sign_extend dest,srlres
 175   BuildMI(sinkMBB, DL, TII->get(Mips::SRLV), Dest)
 176       .addReg(Scratch2)
 177       .addReg(ShiftAmnt);
 178   if (STI->hasMips32r2()) {
 179     BuildMI(sinkMBB, DL, TII->get(SEOp), Dest).addReg(Dest);
 180   } else {
 181     const unsigned ShiftImm =
 182         I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I16_POSTRA ? 16 : 24;
 183     BuildMI(sinkMBB, DL, TII->get(Mips::SLL), Dest)
 184         .addReg(Dest, RegState::Kill)
 185         .addImm(ShiftImm);
 186     BuildMI(sinkMBB, DL, TII->get(Mips::SRA), Dest)
 187         .addReg(Dest, RegState::Kill)
 188         .addImm(ShiftImm);
 189   }
 190
 191   LivePhysRegs LiveRegs;
 192   computeAndAddLiveIns(LiveRegs, *loop1MBB);
 193   computeAndAddLiveIns(LiveRegs, *loop2MBB);
 194   computeAndAddLiveIns(LiveRegs, *sinkMBB);
 195   computeAndAddLiveIns(LiveRegs, *exitMBB);
 196
 197   NMBBI = BB.end();
 198   I->eraseFromParent();
 199   return true;
 200 }
 201
 202 bool MipsExpandPseudo::expandAtomicCmpSwap(MachineBasicBlock &BB,
 203                                            MachineBasicBlock::iterator I,
 204                                            MachineBasicBlock::iterator &NMBBI) {
 205
 206   const unsigned Size =
 207       I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I32_POSTRA ? 4 : 8;
 208   MachineFunction *MF = BB.getParent();
 209
 210   const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
 211   DebugLoc DL = I->getDebugLoc();
 212
 213   unsigned LL, SC, ZERO, BNE, BEQ, MOVE;
 214
 215   if (Size == 4) {
 216     if (STI->inMicroMipsMode()) {
 217       LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
 218       SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
 219       BNE = STI->hasMips32r6() ? Mips::BNEC_MMR6 : Mips::BNE_MM;
 220       BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
 221     } else {
 222       LL = STI->hasMips32r6()
 223                ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
 224                : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
 225       SC = STI->hasMips32r6()
 226                ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
 227                : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
 228       BNE = Mips::BNE;
 229       BEQ = Mips::BEQ;
 230     }
 231
 232     ZERO = Mips::ZERO;
 233     MOVE = Mips::OR;
 234   } else {
 235     LL = STI->hasMips64r6() ? Mips::LLD_R6 : Mips::LLD;
 236     SC = STI->hasMips64r6() ? Mips::SCD_R6 : Mips::SCD;
 237     ZERO = Mips::ZERO_64;
 238     BNE = Mips::BNE64;
 239     BEQ = Mips::BEQ64;
 240     MOVE = Mips::OR64;
 241   }
 242
 243   Register Dest = I->getOperand(0).getReg();
 244   Register Ptr = I->getOperand(1).getReg();
 245   Register OldVal = I->getOperand(2).getReg();
 246   Register NewVal = I->getOperand(3).getReg();
 247   Register Scratch = I->getOperand(4).getReg();
 248
 249   // insert new blocks after the current block
 250   const BasicBlock *LLVM_BB = BB.getBasicBlock();
 251   MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB);
 252   MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB);
 253   MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
 254   MachineFunction::iterator It = ++BB.getIterator();
 255   MF->insert(It, loop1MBB);
 256   MF->insert(It, loop2MBB);
 257   MF->insert(It, exitMBB);
 258
 259   // Transfer the remainder of BB and its successor edges to exitMBB.
 260   exitMBB->splice(exitMBB->begin(), &BB,
 261                   std::next(MachineBasicBlock::iterator(I)), BB.end());
 262   exitMBB->transferSuccessorsAndUpdatePHIs(&BB);
 263
 264   //  thisMBB:
 265   //    ...
 266   //    fallthrough --> loop1MBB
 267   BB.addSuccessor(loop1MBB, BranchProbability::getOne());
 268   loop1MBB->addSuccessor(exitMBB);
 269   loop1MBB->addSuccessor(loop2MBB);
 270   loop1MBB->normalizeSuccProbs();
 271   loop2MBB->addSuccessor(loop1MBB);
 272   loop2MBB->addSuccessor(exitMBB);
 273   loop2MBB->normalizeSuccProbs();
 274
 275   // loop1MBB:
 276   //   ll dest, 0(ptr)
 277   //   bne dest, oldval, exitMBB
 278   BuildMI(loop1MBB, DL, TII->get(LL), Dest).addReg(Ptr).addImm(0);
 279   BuildMI(loop1MBB, DL, TII->get(BNE))
 280     .addReg(Dest, RegState::Kill).addReg(OldVal).addMBB(exitMBB);
 281
 282   // loop2MBB:
 283   //   move scratch, NewVal
 284   //   sc Scratch, Scratch, 0(ptr)
 285   //   beq Scratch, $0, loop1MBB
 286   BuildMI(loop2MBB, DL, TII->get(MOVE), Scratch).addReg(NewVal).addReg(ZERO);
 287   BuildMI(loop2MBB, DL, TII->get(SC), Scratch)
 288     .addReg(Scratch).addReg(Ptr).addImm(0);
 289   BuildMI(loop2MBB, DL, TII->get(BEQ))
 290     .addReg(Scratch, RegState::Kill).addReg(ZERO).addMBB(loop1MBB);
 291
 292   LivePhysRegs LiveRegs;
 293   computeAndAddLiveIns(LiveRegs, *loop1MBB);
 294   computeAndAddLiveIns(LiveRegs, *loop2MBB);
 295   computeAndAddLiveIns(LiveRegs, *exitMBB);
 296
 297   NMBBI = BB.end();
 298   I->eraseFromParent();
 299   return true;
 300 }
 301
 302 bool MipsExpandPseudo::expandAtomicBinOpSubword(
 303     MachineBasicBlock &BB, MachineBasicBlock::iterator I,
 304     MachineBasicBlock::iterator &NMBBI) {
 305
 306   MachineFunction *MF = BB.getParent();
 307
 308   const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
 309   DebugLoc DL = I->getDebugLoc();
 310
 311   unsigned LL, SC, SLT, SLTu, OR, MOVN, MOVZ, SELNEZ, SELEQZ;
 312   unsigned BEQ = Mips::BEQ;
 313   unsigned SEOp = Mips::SEH;
 314
 315   if (STI->inMicroMipsMode()) {
 316       LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
 317       SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
 318       BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
 319       SLT = Mips::SLT_MM;
 320       SLTu = Mips::SLTu_MM;
 321       OR = STI->hasMips32r6() ? Mips::OR_MMR6 : Mips::OR_MM;
 322       MOVN = Mips::MOVN_I_MM;
 323       MOVZ = Mips::MOVZ_I_MM;
 324       SELNEZ = STI->hasMips32r6() ? Mips::SELNEZ_MMR6 : Mips::SELNEZ;
 325       SELEQZ = STI->hasMips32r6() ? Mips::SELEQZ_MMR6 : Mips::SELEQZ;
 326   } else {
 327     LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
 328                             : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
 329     SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
 330                             : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
 331     SLT = Mips::SLT;
 332     SLTu = Mips::SLTu;
 333     OR = Mips::OR;
 334     MOVN = Mips::MOVN_I_I;
 335     MOVZ = Mips::MOVZ_I_I;
 336     SELNEZ = Mips::SELNEZ;
 337     SELEQZ = Mips::SELEQZ;
 338   }
 339
 340   bool IsSwap = false;
 341   bool IsNand = false;
 342   bool IsMin = false;
 343   bool IsMax = false;
 344   bool IsUnsigned = false;
 345
 346   unsigned Opcode = 0;
 347   switch (I->getOpcode()) {
 348   case Mips::ATOMIC_LOAD_NAND_I8_POSTRA:
 349     SEOp = Mips::SEB;
 350     [[fallthrough]];
 351   case Mips::ATOMIC_LOAD_NAND_I16_POSTRA:
 352     IsNand = true;
 353     break;
 354   case Mips::ATOMIC_SWAP_I8_POSTRA:
 355     SEOp = Mips::SEB;
 356     [[fallthrough]];
 357   case Mips::ATOMIC_SWAP_I16_POSTRA:
 358     IsSwap = true;
 359     break;
 360   case Mips::ATOMIC_LOAD_ADD_I8_POSTRA:
 361     SEOp = Mips::SEB;
 362     [[fallthrough]];
 363   case Mips::ATOMIC_LOAD_ADD_I16_POSTRA:
 364     Opcode = Mips::ADDu;
 365     break;
 366   case Mips::ATOMIC_LOAD_SUB_I8_POSTRA:
 367     SEOp = Mips::SEB;
 368     [[fallthrough]];
 369   case Mips::ATOMIC_LOAD_SUB_I16_POSTRA:
 370     Opcode = Mips::SUBu;
 371     break;
 372   case Mips::ATOMIC_LOAD_AND_I8_POSTRA:
 373     SEOp = Mips::SEB;
 374     [[fallthrough]];
 375   case Mips::ATOMIC_LOAD_AND_I16_POSTRA:
 376     Opcode = Mips::AND;
 377     break;
 378   case Mips::ATOMIC_LOAD_OR_I8_POSTRA:
 379     SEOp = Mips::SEB;
 380     [[fallthrough]];
 381   case Mips::ATOMIC_LOAD_OR_I16_POSTRA:
 382     Opcode = Mips::OR;
 383     break;
 384   case Mips::ATOMIC_LOAD_XOR_I8_POSTRA:
 385     SEOp = Mips::SEB;
 386     [[fallthrough]];
 387   case Mips::ATOMIC_LOAD_XOR_I16_POSTRA:
 388     Opcode = Mips::XOR;
 389     break;
 390   case Mips::ATOMIC_LOAD_UMIN_I8_POSTRA:
 391   case Mips::ATOMIC_LOAD_UMIN_I16_POSTRA:
 392     IsUnsigned = true;
 393     [[fallthrough]];
 394   case Mips::ATOMIC_LOAD_MIN_I8_POSTRA:
 395   case Mips::ATOMIC_LOAD_MIN_I16_POSTRA:
 396     IsMin = true;
 397     break;
 398   case Mips::ATOMIC_LOAD_UMAX_I8_POSTRA:
 399   case Mips::ATOMIC_LOAD_UMAX_I16_POSTRA:
 400     IsUnsigned = true;
 401     [[fallthrough]];
 402   case Mips::ATOMIC_LOAD_MAX_I8_POSTRA:
 403   case Mips::ATOMIC_LOAD_MAX_I16_POSTRA:
 404     IsMax = true;
 405     break;
 406   default:
 407     llvm_unreachable("Unknown subword atomic pseudo for expansion!");
 408   }
 409
 410   Register Dest = I->getOperand(0).getReg();
 411   Register Ptr = I->getOperand(1).getReg();
 412   Register Incr = I->getOperand(2).getReg();
 413   Register Mask = I->getOperand(3).getReg();
 414   Register Mask2 = I->getOperand(4).getReg();
 415   Register ShiftAmnt = I->getOperand(5).getReg();
 416   Register OldVal = I->getOperand(6).getReg();
 417   Register BinOpRes = I->getOperand(7).getReg();
 418   Register StoreVal = I->getOperand(8).getReg();
 419
 420   const BasicBlock *LLVM_BB = BB.getBasicBlock();
 421   MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
 422   MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB);
 423   MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
 424   MachineFunction::iterator It = ++BB.getIterator();
 425   MF->insert(It, loopMBB);
 426   MF->insert(It, sinkMBB);
 427   MF->insert(It, exitMBB);
 428
 429   exitMBB->splice(exitMBB->begin(), &BB, std::next(I), BB.end());
 430   exitMBB->transferSuccessorsAndUpdatePHIs(&BB);
 431
 432   BB.addSuccessor(loopMBB, BranchProbability::getOne());
 433   loopMBB->addSuccessor(sinkMBB);
 434   loopMBB->addSuccessor(loopMBB);
 435   loopMBB->normalizeSuccProbs();
 436
 437   BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0);
 438   if (IsNand) {
 439     //  and andres, oldval, incr2
 440     //  nor binopres, $0, andres
 441     //  and newval, binopres, mask
 442     BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
 443         .addReg(OldVal)
 444         .addReg(Incr);
 445     BuildMI(loopMBB, DL, TII->get(Mips::NOR), BinOpRes)
 446         .addReg(Mips::ZERO)
 447         .addReg(BinOpRes);
 448     BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
 449         .addReg(BinOpRes)
 450         .addReg(Mask);
 451   } else if (IsMin || IsMax) {
 452
 453     assert(I->getNumOperands() == 10 &&
 454            "Atomics min|max|umin|umax use an additional register");
 455     Register Scratch4 = I->getOperand(9).getReg();
 456
 457     unsigned SLTScratch4 = IsUnsigned ? SLTu : SLT;
 458     unsigned SELIncr = IsMax ? SELNEZ : SELEQZ;
 459     unsigned SELOldVal = IsMax ? SELEQZ : SELNEZ;
 460     unsigned MOVIncr = IsMax ? MOVN : MOVZ;
 461
 462     // For little endian we need to clear uninterested bits.
 463     if (STI->isLittle()) {
 464       // and OldVal, OldVal, Mask
 465       // and Incr, Incr, Mask
 466       BuildMI(loopMBB, DL, TII->get(Mips::AND), OldVal)
 467           .addReg(OldVal)
 468           .addReg(Mask);
 469       BuildMI(loopMBB, DL, TII->get(Mips::AND), Incr).addReg(Incr).addReg(Mask);
 470     }
 471
 472     // unsigned: sltu Scratch4, oldVal, Incr
 473     // signed:   slt Scratch4, oldVal, Incr
 474     BuildMI(loopMBB, DL, TII->get(SLTScratch4), Scratch4)
 475         .addReg(OldVal)
 476         .addReg(Incr);
 477
 478     if (STI->hasMips64r6() || STI->hasMips32r6()) {
 479       // max: seleqz BinOpRes, OldVal, Scratch4
 480       //      selnez Scratch4, Incr, Scratch4
 481       //      or BinOpRes, BinOpRes, Scratch4
 482       // min: selnqz BinOpRes, OldVal, Scratch4
 483       //      seleqz Scratch4, Incr, Scratch4
 484       //      or BinOpRes, BinOpRes, Scratch4
 485       BuildMI(loopMBB, DL, TII->get(SELOldVal), BinOpRes)
 486           .addReg(OldVal)
 487           .addReg(Scratch4);
 488       BuildMI(loopMBB, DL, TII->get(SELIncr), Scratch4)
 489           .addReg(Incr)
 490           .addReg(Scratch4);
 491       BuildMI(loopMBB, DL, TII->get(OR), BinOpRes)
 492           .addReg(BinOpRes)
 493           .addReg(Scratch4);
 494     } else {
 495       // max: move BinOpRes, OldVal
 496       //      movn BinOpRes, Incr, Scratch4, BinOpRes
 497       // min: move BinOpRes, OldVal
 498       //      movz BinOpRes, Incr, Scratch4, BinOpRes
 499       BuildMI(loopMBB, DL, TII->get(OR), BinOpRes)
 500           .addReg(OldVal)
 501           .addReg(Mips::ZERO);
 502       BuildMI(loopMBB, DL, TII->get(MOVIncr), BinOpRes)
 503           .addReg(Incr)
 504           .addReg(Scratch4)
 505           .addReg(BinOpRes);
 506     }
 507
 508     //  and BinOpRes, BinOpRes, Mask
 509     BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
 510         .addReg(BinOpRes)
 511         .addReg(Mask);
 512
 513   } else if (!IsSwap) {
 514     //  <binop> binopres, oldval, incr2
 515     //  and newval, binopres, mask
 516     BuildMI(loopMBB, DL, TII->get(Opcode), BinOpRes)
 517         .addReg(OldVal)
 518         .addReg(Incr);
 519     BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
 520         .addReg(BinOpRes)
 521         .addReg(Mask);
 522   } else { // atomic.swap
 523     //  and newval, incr2, mask
 524     BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
 525         .addReg(Incr)
 526         .addReg(Mask);
 527   }
 528
 529   // and StoreVal, OlddVal, Mask2
 530   // or StoreVal, StoreVal, BinOpRes
 531   // StoreVal<tied1> = sc StoreVal, 0(Ptr)
 532   // beq StoreVal, zero, loopMBB
 533   BuildMI(loopMBB, DL, TII->get(Mips::AND), StoreVal)
 534     .addReg(OldVal).addReg(Mask2);
 535   BuildMI(loopMBB, DL, TII->get(Mips::OR), StoreVal)
 536     .addReg(StoreVal).addReg(BinOpRes);
 537   BuildMI(loopMBB, DL, TII->get(SC), StoreVal)
 538     .addReg(StoreVal).addReg(Ptr).addImm(0);
 539   BuildMI(loopMBB, DL, TII->get(BEQ))
 540     .addReg(StoreVal).addReg(Mips::ZERO).addMBB(loopMBB);
 541
 542   //  sinkMBB:
 543   //    and     maskedoldval1,oldval,mask
 544   //    srl     srlres,maskedoldval1,shiftamt
 545   //    sign_extend dest,srlres
 546
 547   sinkMBB->addSuccessor(exitMBB, BranchProbability::getOne());
 548
 549   BuildMI(sinkMBB, DL, TII->get(Mips::AND), Dest)
 550     .addReg(OldVal).addReg(Mask);
 551   BuildMI(sinkMBB, DL, TII->get(Mips::SRLV), Dest)
 552       .addReg(Dest).addReg(ShiftAmnt);
 553
 554   if (STI->hasMips32r2()) {
 555     BuildMI(sinkMBB, DL, TII->get(SEOp), Dest).addReg(Dest);
 556   } else {
 557     const unsigned ShiftImm = SEOp == Mips::SEH ? 16 : 24;
 558     BuildMI(sinkMBB, DL, TII->get(Mips::SLL), Dest)
 559         .addReg(Dest, RegState::Kill)
 560         .addImm(ShiftImm);
 561     BuildMI(sinkMBB, DL, TII->get(Mips::SRA), Dest)
 562         .addReg(Dest, RegState::Kill)
 563         .addImm(ShiftImm);
 564   }
 565
 566   LivePhysRegs LiveRegs;
 567   computeAndAddLiveIns(LiveRegs, *loopMBB);
 568   computeAndAddLiveIns(LiveRegs, *sinkMBB);
 569   computeAndAddLiveIns(LiveRegs, *exitMBB);
 570
 571   NMBBI = BB.end();
 572   I->eraseFromParent();
 573
 574   return true;
 575 }
 576
 577 bool MipsExpandPseudo::expandAtomicBinOp(MachineBasicBlock &BB,
 578                                          MachineBasicBlock::iterator I,
 579                                          MachineBasicBlock::iterator &NMBBI,
 580                                          unsigned Size) {
 581   MachineFunction *MF = BB.getParent();
 582
 583   const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
 584   DebugLoc DL = I->getDebugLoc();
 585
 586   unsigned LL, SC, ZERO, BEQ, SLT, SLTu, OR, MOVN, MOVZ, SELNEZ, SELEQZ;
 587
 588   if (Size == 4) {
 589     if (STI->inMicroMipsMode()) {
 590       LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
 591       SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
 592       BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
 593       SLT = Mips::SLT_MM;
 594       SLTu = Mips::SLTu_MM;
 595       OR = STI->hasMips32r6() ? Mips::OR_MMR6 : Mips::OR_MM;
 596       MOVN = Mips::MOVN_I_MM;
 597       MOVZ = Mips::MOVZ_I_MM;
 598       SELNEZ = STI->hasMips32r6() ? Mips::SELNEZ_MMR6 : Mips::SELNEZ;
 599       SELEQZ = STI->hasMips32r6() ? Mips::SELEQZ_MMR6 : Mips::SELEQZ;
 600     } else {
 601       LL = STI->hasMips32r6()
 602                ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
 603                : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
 604       SC = STI->hasMips32r6()
 605                ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
 606                : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
 607       BEQ = Mips::BEQ;
 608       SLT = Mips::SLT;
 609       SLTu = Mips::SLTu;
 610       OR = Mips::OR;
 611       MOVN = Mips::MOVN_I_I;
 612       MOVZ = Mips::MOVZ_I_I;
 613       SELNEZ = Mips::SELNEZ;
 614       SELEQZ = Mips::SELEQZ;
 615     }
 616
 617     ZERO = Mips::ZERO;
 618   } else {
 619     LL = STI->hasMips64r6() ? Mips::LLD_R6 : Mips::LLD;
 620     SC = STI->hasMips64r6() ? Mips::SCD_R6 : Mips::SCD;
 621     ZERO = Mips::ZERO_64;
 622     BEQ = Mips::BEQ64;
 623     SLT = Mips::SLT64;
 624     SLTu = Mips::SLTu64;
 625     OR = Mips::OR64;
 626     MOVN = Mips::MOVN_I64_I64;
 627     MOVZ = Mips::MOVZ_I64_I64;
 628     SELNEZ = Mips::SELNEZ64;
 629     SELEQZ = Mips::SELEQZ64;
 630   }
 631
 632   Register OldVal = I->getOperand(0).getReg();
 633   Register Ptr = I->getOperand(1).getReg();
 634   Register Incr = I->getOperand(2).getReg();
 635   Register Scratch = I->getOperand(3).getReg();
 636
 637   unsigned Opcode = 0;
 638   unsigned AND = 0;
 639   unsigned NOR = 0;
 640
 641   bool IsOr = false;
 642   bool IsNand = false;
 643   bool IsMin = false;
 644   bool IsMax = false;
 645   bool IsUnsigned = false;
 646
 647   switch (I->getOpcode()) {
 648   case Mips::ATOMIC_LOAD_ADD_I32_POSTRA:
 649     Opcode = Mips::ADDu;
 650     break;
 651   case Mips::ATOMIC_LOAD_SUB_I32_POSTRA:
 652     Opcode = Mips::SUBu;
 653     break;
 654   case Mips::ATOMIC_LOAD_AND_I32_POSTRA:
 655     Opcode = Mips::AND;
 656     break;
 657   case Mips::ATOMIC_LOAD_OR_I32_POSTRA:
 658     Opcode = Mips::OR;
 659     break;
 660   case Mips::ATOMIC_LOAD_XOR_I32_POSTRA:
 661     Opcode = Mips::XOR;
 662     break;
 663   case Mips::ATOMIC_LOAD_NAND_I32_POSTRA:
 664     IsNand = true;
 665     AND = Mips::AND;
 666     NOR = Mips::NOR;
 667     break;
 668   case Mips::ATOMIC_SWAP_I32_POSTRA:
 669     IsOr = true;
 670     break;
 671   case Mips::ATOMIC_LOAD_ADD_I64_POSTRA:
 672     Opcode = Mips::DADDu;
 673     break;
 674   case Mips::ATOMIC_LOAD_SUB_I64_POSTRA:
 675     Opcode = Mips::DSUBu;
 676     break;
 677   case Mips::ATOMIC_LOAD_AND_I64_POSTRA:
 678     Opcode = Mips::AND64;
 679     break;
 680   case Mips::ATOMIC_LOAD_OR_I64_POSTRA:
 681     Opcode = Mips::OR64;
 682     break;
 683   case Mips::ATOMIC_LOAD_XOR_I64_POSTRA:
 684     Opcode = Mips::XOR64;
 685     break;
 686   case Mips::ATOMIC_LOAD_NAND_I64_POSTRA:
 687     IsNand = true;
 688     AND = Mips::AND64;
 689     NOR = Mips::NOR64;
 690     break;
 691   case Mips::ATOMIC_SWAP_I64_POSTRA:
 692     IsOr = true;
 693     break;
 694   case Mips::ATOMIC_LOAD_UMIN_I32_POSTRA:
 695   case Mips::ATOMIC_LOAD_UMIN_I64_POSTRA:
 696     IsUnsigned = true;
 697     [[fallthrough]];
 698   case Mips::ATOMIC_LOAD_MIN_I32_POSTRA:
 699   case Mips::ATOMIC_LOAD_MIN_I64_POSTRA:
 700     IsMin = true;
 701     break;
 702   case Mips::ATOMIC_LOAD_UMAX_I32_POSTRA:
 703   case Mips::ATOMIC_LOAD_UMAX_I64_POSTRA:
 704     IsUnsigned = true;
 705     [[fallthrough]];
 706   case Mips::ATOMIC_LOAD_MAX_I32_POSTRA:
 707   case Mips::ATOMIC_LOAD_MAX_I64_POSTRA:
 708     IsMax = true;
 709     break;
 710   default:
 711     llvm_unreachable("Unknown pseudo atomic!");
 712   }
 713
 714   const BasicBlock *LLVM_BB = BB.getBasicBlock();
 715   MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
 716   MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
 717   MachineFunction::iterator It = ++BB.getIterator();
 718   MF->insert(It, loopMBB);
 719   MF->insert(It, exitMBB);
 720
 721   exitMBB->splice(exitMBB->begin(), &BB, std::next(I), BB.end());
 722   exitMBB->transferSuccessorsAndUpdatePHIs(&BB);
 723
 724   BB.addSuccessor(loopMBB, BranchProbability::getOne());
 725   loopMBB->addSuccessor(exitMBB);
 726   loopMBB->addSuccessor(loopMBB);
 727   loopMBB->normalizeSuccProbs();
 728
 729   BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0);
 730   assert((OldVal != Ptr) && "Clobbered the wrong ptr reg!");
 731   assert((OldVal != Incr) && "Clobbered the wrong reg!");
 732   if (IsMin || IsMax) {
 733
 734     assert(I->getNumOperands() == 5 &&
 735            "Atomics min|max|umin|umax use an additional register");
 736     MCRegister Scratch2 = I->getOperand(4).getReg().asMCReg();
 737
 738     // On Mips64 result of slt is GPR32.
 739     MCRegister Scratch2_32 =
 740         (Size == 8) ? STI->getRegisterInfo()->getSubReg(Scratch2, Mips::sub_32)
 741                     : Scratch2;
 742
 743     unsigned SLTScratch2 = IsUnsigned ? SLTu : SLT;
 744     unsigned SELIncr = IsMax ? SELNEZ : SELEQZ;
 745     unsigned SELOldVal = IsMax ? SELEQZ : SELNEZ;
 746     unsigned MOVIncr = IsMax ? MOVN : MOVZ;
 747
 748     // unsigned: sltu Scratch2, oldVal, Incr
 749     // signed:   slt Scratch2, oldVal, Incr
 750     BuildMI(loopMBB, DL, TII->get(SLTScratch2), Scratch2_32)
 751         .addReg(OldVal)
 752         .addReg(Incr);
 753
 754     if (STI->hasMips64r6() || STI->hasMips32r6()) {
 755       // max: seleqz Scratch, OldVal, Scratch2
 756       //      selnez Scratch2, Incr, Scratch2
 757       //      or Scratch, Scratch, Scratch2
 758       // min: selnez Scratch, OldVal, Scratch2
 759       //      seleqz Scratch2, Incr, Scratch2
 760       //      or Scratch, Scratch, Scratch2
 761       BuildMI(loopMBB, DL, TII->get(SELOldVal), Scratch)
 762           .addReg(OldVal)
 763           .addReg(Scratch2);
 764       BuildMI(loopMBB, DL, TII->get(SELIncr), Scratch2)
 765           .addReg(Incr)
 766           .addReg(Scratch2);
 767       BuildMI(loopMBB, DL, TII->get(OR), Scratch)
 768           .addReg(Scratch)
 769           .addReg(Scratch2);
 770     } else {
 771       // max: move Scratch, OldVal
 772       //      movn Scratch, Incr, Scratch2, Scratch
 773       // min: move Scratch, OldVal
 774       //      movz Scratch, Incr, Scratch2, Scratch
 775       BuildMI(loopMBB, DL, TII->get(OR), Scratch)
 776           .addReg(OldVal)
 777           .addReg(ZERO);
 778       BuildMI(loopMBB, DL, TII->get(MOVIncr), Scratch)
 779           .addReg(Incr)
 780           .addReg(Scratch2)
 781           .addReg(Scratch);
 782     }
 783
 784   } else if (Opcode) {
 785     BuildMI(loopMBB, DL, TII->get(Opcode), Scratch).addReg(OldVal).addReg(Incr);
 786   } else if (IsNand) {
 787     assert(AND && NOR &&
 788            "Unknown nand instruction for atomic pseudo expansion");
 789     BuildMI(loopMBB, DL, TII->get(AND), Scratch).addReg(OldVal).addReg(Incr);
 790     BuildMI(loopMBB, DL, TII->get(NOR), Scratch).addReg(ZERO).addReg(Scratch);
 791   } else {
 792     assert(IsOr && OR && "Unknown instruction for atomic pseudo expansion!");
 793     (void)IsOr;
 794     BuildMI(loopMBB, DL, TII->get(OR), Scratch).addReg(Incr).addReg(ZERO);
 795   }
 796
 797   BuildMI(loopMBB, DL, TII->get(SC), Scratch)
 798       .addReg(Scratch)
 799       .addReg(Ptr)
 800       .addImm(0);
 801   BuildMI(loopMBB, DL, TII->get(BEQ))
 802       .addReg(Scratch)
 803       .addReg(ZERO)
 804       .addMBB(loopMBB);
 805
 806   NMBBI = BB.end();
 807   I->eraseFromParent();
 808
 809   LivePhysRegs LiveRegs;
 810   computeAndAddLiveIns(LiveRegs, *loopMBB);
 811   computeAndAddLiveIns(LiveRegs, *exitMBB);
 812
 813   return true;
 814 }
 815
 816 bool MipsExpandPseudo::expandMI(MachineBasicBlock &MBB,
 817                                 MachineBasicBlock::iterator MBBI,
 818                                 MachineBasicBlock::iterator &NMBB) {
 819
 820   bool Modified = false;
 821
 822   switch (MBBI->getOpcode()) {
 823   case Mips::ATOMIC_CMP_SWAP_I32_POSTRA:
 824   case Mips::ATOMIC_CMP_SWAP_I64_POSTRA:
 825     return expandAtomicCmpSwap(MBB, MBBI, NMBB);
 826   case Mips::ATOMIC_CMP_SWAP_I8_POSTRA:
 827   case Mips::ATOMIC_CMP_SWAP_I16_POSTRA:
 828     return expandAtomicCmpSwapSubword(MBB, MBBI, NMBB);
 829   case Mips::ATOMIC_SWAP_I8_POSTRA:
 830   case Mips::ATOMIC_SWAP_I16_POSTRA:
 831   case Mips::ATOMIC_LOAD_NAND_I8_POSTRA:
 832   case Mips::ATOMIC_LOAD_NAND_I16_POSTRA:
 833   case Mips::ATOMIC_LOAD_ADD_I8_POSTRA:
 834   case Mips::ATOMIC_LOAD_ADD_I16_POSTRA:
 835   case Mips::ATOMIC_LOAD_SUB_I8_POSTRA:
 836   case Mips::ATOMIC_LOAD_SUB_I16_POSTRA:
 837   case Mips::ATOMIC_LOAD_AND_I8_POSTRA:
 838   case Mips::ATOMIC_LOAD_AND_I16_POSTRA:
 839   case Mips::ATOMIC_LOAD_OR_I8_POSTRA:
 840   case Mips::ATOMIC_LOAD_OR_I16_POSTRA:
 841   case Mips::ATOMIC_LOAD_XOR_I8_POSTRA:
 842   case Mips::ATOMIC_LOAD_XOR_I16_POSTRA:
 843   case Mips::ATOMIC_LOAD_MIN_I8_POSTRA:
 844   case Mips::ATOMIC_LOAD_MIN_I16_POSTRA:
 845   case Mips::ATOMIC_LOAD_MAX_I8_POSTRA:
 846   case Mips::ATOMIC_LOAD_MAX_I16_POSTRA:
 847   case Mips::ATOMIC_LOAD_UMIN_I8_POSTRA:
 848   case Mips::ATOMIC_LOAD_UMIN_I16_POSTRA:
 849   case Mips::ATOMIC_LOAD_UMAX_I8_POSTRA:
 850   case Mips::ATOMIC_LOAD_UMAX_I16_POSTRA:
 851     return expandAtomicBinOpSubword(MBB, MBBI, NMBB);
 852   case Mips::ATOMIC_LOAD_ADD_I32_POSTRA:
 853   case Mips::ATOMIC_LOAD_SUB_I32_POSTRA:
 854   case Mips::ATOMIC_LOAD_AND_I32_POSTRA:
 855   case Mips::ATOMIC_LOAD_OR_I32_POSTRA:
 856   case Mips::ATOMIC_LOAD_XOR_I32_POSTRA:
 857   case Mips::ATOMIC_LOAD_NAND_I32_POSTRA:
 858   case Mips::ATOMIC_SWAP_I32_POSTRA:
 859   case Mips::ATOMIC_LOAD_MIN_I32_POSTRA:
 860   case Mips::ATOMIC_LOAD_MAX_I32_POSTRA:
 861   case Mips::ATOMIC_LOAD_UMIN_I32_POSTRA:
 862   case Mips::ATOMIC_LOAD_UMAX_I32_POSTRA:
 863     return expandAtomicBinOp(MBB, MBBI, NMBB, 4);
 864   case Mips::ATOMIC_LOAD_ADD_I64_POSTRA:
 865   case Mips::ATOMIC_LOAD_SUB_I64_POSTRA:
 866   case Mips::ATOMIC_LOAD_AND_I64_POSTRA:
 867   case Mips::ATOMIC_LOAD_OR_I64_POSTRA:
 868   case Mips::ATOMIC_LOAD_XOR_I64_POSTRA:
 869   case Mips::ATOMIC_LOAD_NAND_I64_POSTRA:
 870   case Mips::ATOMIC_SWAP_I64_POSTRA:
 871   case Mips::ATOMIC_LOAD_MIN_I64_POSTRA:
 872   case Mips::ATOMIC_LOAD_MAX_I64_POSTRA:
 873   case Mips::ATOMIC_LOAD_UMIN_I64_POSTRA:
 874   case Mips::ATOMIC_LOAD_UMAX_I64_POSTRA:
 875     return expandAtomicBinOp(MBB, MBBI, NMBB, 8);
 876   default:
 877     return Modified;
 878   }
 879 }
 880
 881 bool MipsExpandPseudo::expandMBB(MachineBasicBlock &MBB) {
 882   bool Modified = false;
 883
 884   MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
 885   while (MBBI != E) {
 886     MachineBasicBlock::iterator NMBBI = std::next(MBBI);
 887     Modified |= expandMI(MBB, MBBI, NMBBI);
 888     MBBI = NMBBI;
 889   }
 890
 891   return Modified;
 892 }
 893
 894 bool MipsExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
 895   STI = &MF.getSubtarget<MipsSubtarget>();
 896   TII = STI->getInstrInfo();
 897
 898   bool Modified = false;
 899   for (MachineBasicBlock &MBB : MF)
 900     Modified |= expandMBB(MBB);
 901
 902   if (Modified)
 903     MF.RenumberBlocks();
 904
 905   return Modified;
 906 }
 907
 908 /// createMipsExpandPseudoPass - returns an instance of the pseudo instruction
 909 /// expansion pass.
 910 FunctionPass *llvm::createMipsExpandPseudoPass() {
 911   return new MipsExpandPseudo();
 912 }