llvm/lib/Target/Mips/MipsExpandPseudo.cpp

   1 //===-- MipsExpandPseudoInsts.cpp - Expand pseudo instructions ------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file contains a pass that expands pseudo instructions into target
  10 // instructions to allow proper scheduling, if-conversion, and other late
  11 // optimizations. This pass should be run after register allocation but before
  12 // the post-regalloc scheduling pass.
  13 //
  14 // This is currently only used for expanding atomic pseudos after register
  15 // allocation. We do this to avoid the fast register allocator introducing
  16 // spills between ll and sc. These stores cause some MIPS implementations to
  17 // abort the atomic RMW sequence.
  18 //
  19 //===----------------------------------------------------------------------===//
  20
  21 #include "Mips.h"
  22 #include "MipsInstrInfo.h"
  23 #include "MipsSubtarget.h"
  24 #include "llvm/CodeGen/LivePhysRegs.h"
  25 #include "llvm/CodeGen/MachineFunctionPass.h"
  26 #include "llvm/CodeGen/MachineInstrBuilder.h"
  27
  28 using namespace llvm;
  29
  30 #define DEBUG_TYPE "mips-pseudo"
  31
  32 namespace {
  33   class MipsExpandPseudo : public MachineFunctionPass {
  34   public:
  35     static char ID;
  36     MipsExpandPseudo() : MachineFunctionPass(ID) {}
  37
  38     const MipsInstrInfo *TII;
  39     const MipsSubtarget *STI;
  40
  41     bool runOnMachineFunction(MachineFunction &Fn) override;
  42
  43     MachineFunctionProperties getRequiredProperties() const override {
  44       return MachineFunctionProperties().set(
  45           MachineFunctionProperties::Property::NoVRegs);
  46     }
  47
  48     StringRef getPassName() const override {
  49       return "Mips pseudo instruction expansion pass";
  50     }
  51
  52   private:
  53     bool expandAtomicCmpSwap(MachineBasicBlock &MBB,
  54                              MachineBasicBlock::iterator MBBI,
  55                              MachineBasicBlock::iterator &NextMBBI);
  56     bool expandAtomicCmpSwapSubword(MachineBasicBlock &MBB,
  57                                     MachineBasicBlock::iterator MBBI,
  58                                     MachineBasicBlock::iterator &NextMBBI);
  59
  60     bool expandAtomicBinOp(MachineBasicBlock &BB,
  61                            MachineBasicBlock::iterator I,
  62                            MachineBasicBlock::iterator &NMBBI, unsigned Size);
  63     bool expandAtomicBinOpSubword(MachineBasicBlock &BB,
  64                                   MachineBasicBlock::iterator I,
  65                                   MachineBasicBlock::iterator &NMBBI);
  66
  67     bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
  68                   MachineBasicBlock::iterator &NMBB);
  69     bool expandMBB(MachineBasicBlock &MBB);
  70    };
  71   char MipsExpandPseudo::ID = 0;
  72 }
  73
  74 bool MipsExpandPseudo::expandAtomicCmpSwapSubword(
  75     MachineBasicBlock &BB, MachineBasicBlock::iterator I,
  76     MachineBasicBlock::iterator &NMBBI) {
  77
  78   MachineFunction *MF = BB.getParent();
  79
  80   const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
  81   DebugLoc DL = I->getDebugLoc();
  82   unsigned LL, SC;
  83
  84   unsigned ZERO = Mips::ZERO;
  85   unsigned BNE = Mips::BNE;
  86   unsigned BEQ = Mips::BEQ;
  87   unsigned SEOp =
  88       I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I8_POSTRA ? Mips::SEB : Mips::SEH;
  89
  90   if (STI->inMicroMipsMode()) {
  91       LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
  92       SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
  93       BNE = STI->hasMips32r6() ? Mips::BNEC_MMR6 : Mips::BNE_MM;
  94       BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
  95   } else {
  96     LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
  97                             : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
  98     SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
  99                             : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
 100   }
 101
 102   Register Dest = I->getOperand(0).getReg();
 103   Register Ptr = I->getOperand(1).getReg();
 104   Register Mask = I->getOperand(2).getReg();
 105   Register ShiftCmpVal = I->getOperand(3).getReg();
 106   Register Mask2 = I->getOperand(4).getReg();
 107   Register ShiftNewVal = I->getOperand(5).getReg();
 108   Register ShiftAmnt = I->getOperand(6).getReg();
 109   Register Scratch = I->getOperand(7).getReg();
 110   Register Scratch2 = I->getOperand(8).getReg();
 111
 112   // insert new blocks after the current block
 113   const BasicBlock *LLVM_BB = BB.getBasicBlock();
 114   MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB);
 115   MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB);
 116   MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB);
 117   MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
 118   MachineFunction::iterator It = ++BB.getIterator();
 119   MF->insert(It, loop1MBB);
 120   MF->insert(It, loop2MBB);
 121   MF->insert(It, sinkMBB);
 122   MF->insert(It, exitMBB);
 123
 124   // Transfer the remainder of BB and its successor edges to exitMBB.
 125   exitMBB->splice(exitMBB->begin(), &BB,
 126                   std::next(MachineBasicBlock::iterator(I)), BB.end());
 127   exitMBB->transferSuccessorsAndUpdatePHIs(&BB);
 128
 129   //  thisMBB:
 130   //    ...
 131   //    fallthrough --> loop1MBB
 132   BB.addSuccessor(loop1MBB, BranchProbability::getOne());
 133   loop1MBB->addSuccessor(sinkMBB);
 134   loop1MBB->addSuccessor(loop2MBB);
 135   loop1MBB->normalizeSuccProbs();
 136   loop2MBB->addSuccessor(loop1MBB);
 137   loop2MBB->addSuccessor(sinkMBB);
 138   loop2MBB->normalizeSuccProbs();
 139   sinkMBB->addSuccessor(exitMBB, BranchProbability::getOne());
 140
 141   // loop1MBB:
 142   //   ll dest, 0(ptr)
 143   //   and Mask', dest, Mask
 144   //   bne Mask', ShiftCmpVal, exitMBB
 145   BuildMI(loop1MBB, DL, TII->get(LL), Scratch).addReg(Ptr).addImm(0);
 146   BuildMI(loop1MBB, DL, TII->get(Mips::AND), Scratch2)
 147       .addReg(Scratch)
 148       .addReg(Mask);
 149   BuildMI(loop1MBB, DL, TII->get(BNE))
 150     .addReg(Scratch2).addReg(ShiftCmpVal).addMBB(sinkMBB);
 151
 152   // loop2MBB:
 153   //   and dest, dest, mask2
 154   //   or dest, dest, ShiftNewVal
 155   //   sc dest, dest, 0(ptr)
 156   //   beq dest, $0, loop1MBB
 157   BuildMI(loop2MBB, DL, TII->get(Mips::AND), Scratch)
 158       .addReg(Scratch, RegState::Kill)
 159       .addReg(Mask2);
 160   BuildMI(loop2MBB, DL, TII->get(Mips::OR), Scratch)
 161       .addReg(Scratch, RegState::Kill)
 162       .addReg(ShiftNewVal);
 163   BuildMI(loop2MBB, DL, TII->get(SC), Scratch)
 164       .addReg(Scratch, RegState::Kill)
 165       .addReg(Ptr)
 166       .addImm(0);
 167   BuildMI(loop2MBB, DL, TII->get(BEQ))
 168       .addReg(Scratch, RegState::Kill)
 169       .addReg(ZERO)
 170       .addMBB(loop1MBB);
 171
 172   //  sinkMBB:
 173   //    srl     srlres, Mask', shiftamt
 174   //    sign_extend dest,srlres
 175   BuildMI(sinkMBB, DL, TII->get(Mips::SRLV), Dest)
 176       .addReg(Scratch2)
 177       .addReg(ShiftAmnt);
 178   if (STI->hasMips32r2()) {
 179     BuildMI(sinkMBB, DL, TII->get(SEOp), Dest).addReg(Dest);
 180   } else {
 181     const unsigned ShiftImm =
 182         I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I16_POSTRA ? 16 : 24;
 183     BuildMI(sinkMBB, DL, TII->get(Mips::SLL), Dest)
 184         .addReg(Dest, RegState::Kill)
 185         .addImm(ShiftImm);
 186     BuildMI(sinkMBB, DL, TII->get(Mips::SRA), Dest)
 187         .addReg(Dest, RegState::Kill)
 188         .addImm(ShiftImm);
 189   }
 190
 191   LivePhysRegs LiveRegs;
 192   computeAndAddLiveIns(LiveRegs, *loop1MBB);
 193   computeAndAddLiveIns(LiveRegs, *loop2MBB);
 194   computeAndAddLiveIns(LiveRegs, *sinkMBB);
 195   computeAndAddLiveIns(LiveRegs, *exitMBB);
 196
 197   NMBBI = BB.end();
 198   I->eraseFromParent();
 199   return true;
 200 }
 201
 202 bool MipsExpandPseudo::expandAtomicCmpSwap(MachineBasicBlock &BB,
 203                                            MachineBasicBlock::iterator I,
 204                                            MachineBasicBlock::iterator &NMBBI) {
 205
 206   const unsigned Size =
 207       I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I32_POSTRA ? 4 : 8;
 208   MachineFunction *MF = BB.getParent();
 209
 210   const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
 211   DebugLoc DL = I->getDebugLoc();
 212
 213   unsigned LL, SC, ZERO, BNE, BEQ, MOVE;
 214
 215   if (Size == 4) {
 216     if (STI->inMicroMipsMode()) {
 217       LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
 218       SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
 219       BNE = STI->hasMips32r6() ? Mips::BNEC_MMR6 : Mips::BNE_MM;
 220       BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
 221     } else {
 222       LL = STI->hasMips32r6()
 223                ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
 224                : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
 225       SC = STI->hasMips32r6()
 226                ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
 227                : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
 228       BNE = Mips::BNE;
 229       BEQ = Mips::BEQ;
 230     }
 231
 232     ZERO = Mips::ZERO;
 233     MOVE = Mips::OR;
 234   } else {
 235     LL = STI->hasMips64r6() ? Mips::LLD_R6 : Mips::LLD;
 236     SC = STI->hasMips64r6() ? Mips::SCD_R6 : Mips::SCD;
 237     ZERO = Mips::ZERO_64;
 238     BNE = Mips::BNE64;
 239     BEQ = Mips::BEQ64;
 240     MOVE = Mips::OR64;
 241   }
 242
 243   Register Dest = I->getOperand(0).getReg();
 244   Register Ptr = I->getOperand(1).getReg();
 245   Register OldVal = I->getOperand(2).getReg();
 246   Register NewVal = I->getOperand(3).getReg();
 247   Register Scratch = I->getOperand(4).getReg();
 248
 249   // insert new blocks after the current block
 250   const BasicBlock *LLVM_BB = BB.getBasicBlock();
 251   MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB);
 252   MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB);
 253   MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
 254   MachineFunction::iterator It = ++BB.getIterator();
 255   MF->insert(It, loop1MBB);
 256   MF->insert(It, loop2MBB);
 257   MF->insert(It, exitMBB);
 258
 259   // Transfer the remainder of BB and its successor edges to exitMBB.
 260   exitMBB->splice(exitMBB->begin(), &BB,
 261                   std::next(MachineBasicBlock::iterator(I)), BB.end());
 262   exitMBB->transferSuccessorsAndUpdatePHIs(&BB);
 263
 264   //  thisMBB:
 265   //    ...
 266   //    fallthrough --> loop1MBB
 267   BB.addSuccessor(loop1MBB, BranchProbability::getOne());
 268   loop1MBB->addSuccessor(exitMBB);
 269   loop1MBB->addSuccessor(loop2MBB);
 270   loop1MBB->normalizeSuccProbs();
 271   loop2MBB->addSuccessor(loop1MBB);
 272   loop2MBB->addSuccessor(exitMBB);
 273   loop2MBB->normalizeSuccProbs();
 274
 275   // loop1MBB:
 276   //   ll dest, 0(ptr)
 277   //   bne dest, oldval, exitMBB
 278   BuildMI(loop1MBB, DL, TII->get(LL), Dest).addReg(Ptr).addImm(0);
 279   BuildMI(loop1MBB, DL, TII->get(BNE))
 280     .addReg(Dest, RegState::Kill).addReg(OldVal).addMBB(exitMBB);
 281
 282   // loop2MBB:
 283   //   move scratch, NewVal
 284   //   sc Scratch, Scratch, 0(ptr)
 285   //   beq Scratch, $0, loop1MBB
 286   BuildMI(loop2MBB, DL, TII->get(MOVE), Scratch).addReg(NewVal).addReg(ZERO);
 287   BuildMI(loop2MBB, DL, TII->get(SC), Scratch)
 288     .addReg(Scratch).addReg(Ptr).addImm(0);
 289   BuildMI(loop2MBB, DL, TII->get(BEQ))
 290     .addReg(Scratch, RegState::Kill).addReg(ZERO).addMBB(loop1MBB);
 291
 292   LivePhysRegs LiveRegs;
 293   computeAndAddLiveIns(LiveRegs, *loop1MBB);
 294   computeAndAddLiveIns(LiveRegs, *loop2MBB);
 295   computeAndAddLiveIns(LiveRegs, *exitMBB);
 296
 297   NMBBI = BB.end();
 298   I->eraseFromParent();
 299   return true;
 300 }
 301
 302 bool MipsExpandPseudo::expandAtomicBinOpSubword(
 303     MachineBasicBlock &BB, MachineBasicBlock::iterator I,
 304     MachineBasicBlock::iterator &NMBBI) {
 305
 306   MachineFunction *MF = BB.getParent();
 307
 308   const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
 309   DebugLoc DL = I->getDebugLoc();
 310
 311   unsigned LL, SC, SLT, SLTu, OR, MOVN, MOVZ, SELNEZ, SELEQZ;
 312   unsigned BEQ = Mips::BEQ;
 313   unsigned SEOp = Mips::SEH;
 314
 315   if (STI->inMicroMipsMode()) {
 316       LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
 317       SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
 318       BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
 319       SLT = Mips::SLT_MM;
 320       SLTu = Mips::SLTu_MM;
 321       OR = STI->hasMips32r6() ? Mips::OR_MMR6 : Mips::OR_MM;
 322       MOVN = Mips::MOVN_I_MM;
 323       MOVZ = Mips::MOVZ_I_MM;
 324       SELNEZ = STI->hasMips32r6() ? Mips::SELNEZ_MMR6 : Mips::SELNEZ;
 325       SELEQZ = STI->hasMips32r6() ? Mips::SELEQZ_MMR6 : Mips::SELEQZ;
 326   } else {
 327     LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
 328                             : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
 329     SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
 330                             : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
 331     SLT = Mips::SLT;
 332     SLTu = Mips::SLTu;
 333     OR = Mips::OR;
 334     MOVN = Mips::MOVN_I_I;
 335     MOVZ = Mips::MOVZ_I_I;
 336     SELNEZ = Mips::SELNEZ;
 337     SELEQZ = Mips::SELEQZ;
 338   }
 339
 340   bool IsSwap = false;
 341   bool IsNand = false;
 342   bool IsMin = false;
 343   bool IsMax = false;
 344   bool IsUnsigned = false;
 345   bool DestOK = false;
 346
 347   unsigned Opcode = 0;
 348   switch (I->getOpcode()) {
 349   case Mips::ATOMIC_LOAD_NAND_I8_POSTRA:
 350     SEOp = Mips::SEB;
 351     [[fallthrough]];
 352   case Mips::ATOMIC_LOAD_NAND_I16_POSTRA:
 353     IsNand = true;
 354     break;
 355   case Mips::ATOMIC_SWAP_I8_POSTRA:
 356     SEOp = Mips::SEB;
 357     [[fallthrough]];
 358   case Mips::ATOMIC_SWAP_I16_POSTRA:
 359     IsSwap = true;
 360     break;
 361   case Mips::ATOMIC_LOAD_ADD_I8_POSTRA:
 362     SEOp = Mips::SEB;
 363     [[fallthrough]];
 364   case Mips::ATOMIC_LOAD_ADD_I16_POSTRA:
 365     Opcode = Mips::ADDu;
 366     break;
 367   case Mips::ATOMIC_LOAD_SUB_I8_POSTRA:
 368     SEOp = Mips::SEB;
 369     [[fallthrough]];
 370   case Mips::ATOMIC_LOAD_SUB_I16_POSTRA:
 371     Opcode = Mips::SUBu;
 372     break;
 373   case Mips::ATOMIC_LOAD_AND_I8_POSTRA:
 374     SEOp = Mips::SEB;
 375     [[fallthrough]];
 376   case Mips::ATOMIC_LOAD_AND_I16_POSTRA:
 377     Opcode = Mips::AND;
 378     break;
 379   case Mips::ATOMIC_LOAD_OR_I8_POSTRA:
 380     SEOp = Mips::SEB;
 381     [[fallthrough]];
 382   case Mips::ATOMIC_LOAD_OR_I16_POSTRA:
 383     Opcode = Mips::OR;
 384     break;
 385   case Mips::ATOMIC_LOAD_XOR_I8_POSTRA:
 386     SEOp = Mips::SEB;
 387     [[fallthrough]];
 388   case Mips::ATOMIC_LOAD_XOR_I16_POSTRA:
 389     Opcode = Mips::XOR;
 390     break;
 391   case Mips::ATOMIC_LOAD_UMIN_I8_POSTRA:
 392     SEOp = Mips::SEB;
 393     IsUnsigned = true;
 394     IsMin = true;
 395     break;
 396   case Mips::ATOMIC_LOAD_UMIN_I16_POSTRA:
 397     IsUnsigned = true;
 398     IsMin = true;
 399     break;
 400   case Mips::ATOMIC_LOAD_MIN_I8_POSTRA:
 401     SEOp = Mips::SEB;
 402     IsMin = true;
 403     break;
 404   case Mips::ATOMIC_LOAD_MIN_I16_POSTRA:
 405     IsMin = true;
 406     break;
 407   case Mips::ATOMIC_LOAD_UMAX_I8_POSTRA:
 408     SEOp = Mips::SEB;
 409     IsUnsigned = true;
 410     IsMax = true;
 411     break;
 412   case Mips::ATOMIC_LOAD_UMAX_I16_POSTRA:
 413     IsUnsigned = true;
 414     IsMax = true;
 415     break;
 416   case Mips::ATOMIC_LOAD_MAX_I8_POSTRA:
 417     SEOp = Mips::SEB;
 418     IsMax = true;
 419     break;
 420   case Mips::ATOMIC_LOAD_MAX_I16_POSTRA:
 421     IsMax = true;
 422     break;
 423   default:
 424     llvm_unreachable("Unknown subword atomic pseudo for expansion!");
 425   }
 426
 427   Register Dest = I->getOperand(0).getReg();
 428   Register Ptr = I->getOperand(1).getReg();
 429   Register Incr = I->getOperand(2).getReg();
 430   Register Mask = I->getOperand(3).getReg();
 431   Register Mask2 = I->getOperand(4).getReg();
 432   Register ShiftAmnt = I->getOperand(5).getReg();
 433   Register OldVal = I->getOperand(6).getReg();
 434   Register BinOpRes = I->getOperand(7).getReg();
 435   Register StoreVal = I->getOperand(8).getReg();
 436
 437   const BasicBlock *LLVM_BB = BB.getBasicBlock();
 438   MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
 439   MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB);
 440   MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
 441   MachineFunction::iterator It = ++BB.getIterator();
 442   MF->insert(It, loopMBB);
 443   MF->insert(It, sinkMBB);
 444   MF->insert(It, exitMBB);
 445
 446   exitMBB->splice(exitMBB->begin(), &BB, std::next(I), BB.end());
 447   exitMBB->transferSuccessorsAndUpdatePHIs(&BB);
 448
 449   BB.addSuccessor(loopMBB, BranchProbability::getOne());
 450   loopMBB->addSuccessor(sinkMBB);
 451   loopMBB->addSuccessor(loopMBB);
 452   loopMBB->normalizeSuccProbs();
 453
 454   BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0);
 455   if (IsNand) {
 456     //  and andres, oldval, incr2
 457     //  nor binopres, $0, andres
 458     //  and newval, binopres, mask
 459     BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
 460         .addReg(OldVal)
 461         .addReg(Incr);
 462     BuildMI(loopMBB, DL, TII->get(Mips::NOR), BinOpRes)
 463         .addReg(Mips::ZERO)
 464         .addReg(BinOpRes);
 465     BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
 466         .addReg(BinOpRes)
 467         .addReg(Mask);
 468   } else if (IsMin || IsMax) {
 469
 470     assert(I->getNumOperands() == 10 &&
 471            "Atomics min|max|umin|umax use an additional register");
 472     Register Scratch4 = I->getOperand(9).getReg();
 473
 474     unsigned SLTScratch4 = IsUnsigned ? SLTu : SLT;
 475     unsigned SELIncr = IsMax ? SELNEZ : SELEQZ;
 476     unsigned SELOldVal = IsMax ? SELEQZ : SELNEZ;
 477     unsigned MOVIncr = IsMax ? MOVN : MOVZ;
 478
 479     BuildMI(loopMBB, DL, TII->get(Mips::SRAV), StoreVal)
 480         .addReg(OldVal)
 481         .addReg(ShiftAmnt);
 482     if (IsUnsigned) {
 483       const unsigned OpMask = SEOp == Mips::SEH ? 0xffff : 0xff;
 484       BuildMI(loopMBB, DL, TII->get(Mips::ANDi), StoreVal)
 485           .addReg(StoreVal)
 486           .addImm(OpMask);
 487     } else if (STI->hasMips32r2()) {
 488       BuildMI(loopMBB, DL, TII->get(SEOp), StoreVal).addReg(StoreVal);
 489     } else {
 490       const unsigned ShiftImm = SEOp == Mips::SEH ? 16 : 24;
 491       const unsigned SROp = IsUnsigned ? Mips::SRL : Mips::SRA;
 492       BuildMI(loopMBB, DL, TII->get(Mips::SLL), StoreVal)
 493           .addReg(StoreVal, RegState::Kill)
 494           .addImm(ShiftImm);
 495       BuildMI(loopMBB, DL, TII->get(SROp), StoreVal)
 496           .addReg(StoreVal, RegState::Kill)
 497           .addImm(ShiftImm);
 498     }
 499     BuildMI(loopMBB, DL, TII->get(Mips::OR), Dest)
 500         .addReg(Mips::ZERO)
 501         .addReg(StoreVal);
 502     DestOK = true;
 503     BuildMI(loopMBB, DL, TII->get(Mips::SLLV), StoreVal)
 504         .addReg(StoreVal)
 505         .addReg(ShiftAmnt);
 506
 507     // unsigned: sltu Scratch4, StoreVal, Incr
 508     // signed:   slt Scratch4, StoreVal, Incr
 509     BuildMI(loopMBB, DL, TII->get(SLTScratch4), Scratch4)
 510         .addReg(StoreVal)
 511         .addReg(Incr);
 512
 513     if (STI->hasMips64r6() || STI->hasMips32r6()) {
 514       // max: seleqz BinOpRes, OldVal, Scratch4
 515       //      selnez Scratch4, Incr, Scratch4
 516       //      or BinOpRes, BinOpRes, Scratch4
 517       // min: selnqz BinOpRes, OldVal, Scratch4
 518       //      seleqz Scratch4, Incr, Scratch4
 519       //      or BinOpRes, BinOpRes, Scratch4
 520       BuildMI(loopMBB, DL, TII->get(SELOldVal), BinOpRes)
 521           .addReg(StoreVal)
 522           .addReg(Scratch4);
 523       BuildMI(loopMBB, DL, TII->get(SELIncr), Scratch4)
 524           .addReg(Incr)
 525           .addReg(Scratch4);
 526       BuildMI(loopMBB, DL, TII->get(OR), BinOpRes)
 527           .addReg(BinOpRes)
 528           .addReg(Scratch4);
 529     } else {
 530       // max: move BinOpRes, StoreVal
 531       //      movn BinOpRes, Incr, Scratch4, BinOpRes
 532       // min: move BinOpRes, StoreVal
 533       //      movz BinOpRes, Incr, Scratch4, BinOpRes
 534       BuildMI(loopMBB, DL, TII->get(OR), BinOpRes)
 535           .addReg(StoreVal)
 536           .addReg(Mips::ZERO);
 537       BuildMI(loopMBB, DL, TII->get(MOVIncr), BinOpRes)
 538           .addReg(Incr)
 539           .addReg(Scratch4)
 540           .addReg(BinOpRes);
 541     }
 542
 543     //  and BinOpRes, BinOpRes, Mask
 544     BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
 545         .addReg(BinOpRes)
 546         .addReg(Mask);
 547
 548   } else if (!IsSwap) {
 549     //  <binop> binopres, oldval, incr2
 550     //  and newval, binopres, mask
 551     BuildMI(loopMBB, DL, TII->get(Opcode), BinOpRes)
 552         .addReg(OldVal)
 553         .addReg(Incr);
 554     BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
 555         .addReg(BinOpRes)
 556         .addReg(Mask);
 557   } else { // atomic.swap
 558     //  and newval, incr2, mask
 559     BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
 560         .addReg(Incr)
 561         .addReg(Mask);
 562   }
 563
 564   // and StoreVal, OlddVal, Mask2
 565   // or StoreVal, StoreVal, BinOpRes
 566   // StoreVal<tied1> = sc StoreVal, 0(Ptr)
 567   // beq StoreVal, zero, loopMBB
 568   BuildMI(loopMBB, DL, TII->get(Mips::AND), StoreVal)
 569     .addReg(OldVal).addReg(Mask2);
 570   BuildMI(loopMBB, DL, TII->get(Mips::OR), StoreVal)
 571     .addReg(StoreVal).addReg(BinOpRes);
 572   BuildMI(loopMBB, DL, TII->get(SC), StoreVal)
 573     .addReg(StoreVal).addReg(Ptr).addImm(0);
 574   BuildMI(loopMBB, DL, TII->get(BEQ))
 575     .addReg(StoreVal).addReg(Mips::ZERO).addMBB(loopMBB);
 576
 577   //  sinkMBB:
 578   //    and     maskedoldval1,oldval,mask
 579   //    srl     srlres,maskedoldval1,shiftamt
 580   //    sign_extend dest,srlres
 581
 582   if (!DestOK) {
 583     sinkMBB->addSuccessor(exitMBB, BranchProbability::getOne());
 584     BuildMI(sinkMBB, DL, TII->get(Mips::AND), Dest).addReg(OldVal).addReg(Mask);
 585     BuildMI(sinkMBB, DL, TII->get(Mips::SRLV), Dest)
 586         .addReg(Dest)
 587         .addReg(ShiftAmnt);
 588
 589     if (STI->hasMips32r2()) {
 590       BuildMI(sinkMBB, DL, TII->get(SEOp), Dest).addReg(Dest);
 591     } else {
 592       const unsigned ShiftImm = SEOp == Mips::SEH ? 16 : 24;
 593       BuildMI(sinkMBB, DL, TII->get(Mips::SLL), Dest)
 594           .addReg(Dest, RegState::Kill)
 595           .addImm(ShiftImm);
 596       BuildMI(sinkMBB, DL, TII->get(Mips::SRA), Dest)
 597           .addReg(Dest, RegState::Kill)
 598           .addImm(ShiftImm);
 599     }
 600   }
 601
 602   LivePhysRegs LiveRegs;
 603   computeAndAddLiveIns(LiveRegs, *loopMBB);
 604   computeAndAddLiveIns(LiveRegs, *sinkMBB);
 605   computeAndAddLiveIns(LiveRegs, *exitMBB);
 606
 607   NMBBI = BB.end();
 608   I->eraseFromParent();
 609
 610   return true;
 611 }
 612
 613 bool MipsExpandPseudo::expandAtomicBinOp(MachineBasicBlock &BB,
 614                                          MachineBasicBlock::iterator I,
 615                                          MachineBasicBlock::iterator &NMBBI,
 616                                          unsigned Size) {
 617   MachineFunction *MF = BB.getParent();
 618
 619   const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
 620   DebugLoc DL = I->getDebugLoc();
 621
 622   unsigned LL, SC, ZERO, BEQ, SLT, SLTu, OR, MOVN, MOVZ, SELNEZ, SELEQZ;
 623
 624   if (Size == 4) {
 625     if (STI->inMicroMipsMode()) {
 626       LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
 627       SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
 628       BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
 629       SLT = Mips::SLT_MM;
 630       SLTu = Mips::SLTu_MM;
 631       OR = STI->hasMips32r6() ? Mips::OR_MMR6 : Mips::OR_MM;
 632       MOVN = Mips::MOVN_I_MM;
 633       MOVZ = Mips::MOVZ_I_MM;
 634       SELNEZ = STI->hasMips32r6() ? Mips::SELNEZ_MMR6 : Mips::SELNEZ;
 635       SELEQZ = STI->hasMips32r6() ? Mips::SELEQZ_MMR6 : Mips::SELEQZ;
 636     } else {
 637       LL = STI->hasMips32r6()
 638                ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
 639                : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
 640       SC = STI->hasMips32r6()
 641                ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
 642                : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
 643       BEQ = Mips::BEQ;
 644       SLT = Mips::SLT;
 645       SLTu = Mips::SLTu;
 646       OR = Mips::OR;
 647       MOVN = Mips::MOVN_I_I;
 648       MOVZ = Mips::MOVZ_I_I;
 649       SELNEZ = Mips::SELNEZ;
 650       SELEQZ = Mips::SELEQZ;
 651     }
 652
 653     ZERO = Mips::ZERO;
 654   } else {
 655     LL = STI->hasMips64r6() ? Mips::LLD_R6 : Mips::LLD;
 656     SC = STI->hasMips64r6() ? Mips::SCD_R6 : Mips::SCD;
 657     ZERO = Mips::ZERO_64;
 658     BEQ = Mips::BEQ64;
 659     SLT = Mips::SLT64;
 660     SLTu = Mips::SLTu64;
 661     OR = Mips::OR64;
 662     MOVN = Mips::MOVN_I64_I64;
 663     MOVZ = Mips::MOVZ_I64_I64;
 664     SELNEZ = Mips::SELNEZ64;
 665     SELEQZ = Mips::SELEQZ64;
 666   }
 667
 668   Register OldVal = I->getOperand(0).getReg();
 669   Register Ptr = I->getOperand(1).getReg();
 670   Register Incr = I->getOperand(2).getReg();
 671   Register Scratch = I->getOperand(3).getReg();
 672
 673   unsigned Opcode = 0;
 674   unsigned AND = 0;
 675   unsigned NOR = 0;
 676
 677   bool IsOr = false;
 678   bool IsNand = false;
 679   bool IsMin = false;
 680   bool IsMax = false;
 681   bool IsUnsigned = false;
 682
 683   switch (I->getOpcode()) {
 684   case Mips::ATOMIC_LOAD_ADD_I32_POSTRA:
 685     Opcode = Mips::ADDu;
 686     break;
 687   case Mips::ATOMIC_LOAD_SUB_I32_POSTRA:
 688     Opcode = Mips::SUBu;
 689     break;
 690   case Mips::ATOMIC_LOAD_AND_I32_POSTRA:
 691     Opcode = Mips::AND;
 692     break;
 693   case Mips::ATOMIC_LOAD_OR_I32_POSTRA:
 694     Opcode = Mips::OR;
 695     break;
 696   case Mips::ATOMIC_LOAD_XOR_I32_POSTRA:
 697     Opcode = Mips::XOR;
 698     break;
 699   case Mips::ATOMIC_LOAD_NAND_I32_POSTRA:
 700     IsNand = true;
 701     AND = Mips::AND;
 702     NOR = Mips::NOR;
 703     break;
 704   case Mips::ATOMIC_SWAP_I32_POSTRA:
 705     IsOr = true;
 706     break;
 707   case Mips::ATOMIC_LOAD_ADD_I64_POSTRA:
 708     Opcode = Mips::DADDu;
 709     break;
 710   case Mips::ATOMIC_LOAD_SUB_I64_POSTRA:
 711     Opcode = Mips::DSUBu;
 712     break;
 713   case Mips::ATOMIC_LOAD_AND_I64_POSTRA:
 714     Opcode = Mips::AND64;
 715     break;
 716   case Mips::ATOMIC_LOAD_OR_I64_POSTRA:
 717     Opcode = Mips::OR64;
 718     break;
 719   case Mips::ATOMIC_LOAD_XOR_I64_POSTRA:
 720     Opcode = Mips::XOR64;
 721     break;
 722   case Mips::ATOMIC_LOAD_NAND_I64_POSTRA:
 723     IsNand = true;
 724     AND = Mips::AND64;
 725     NOR = Mips::NOR64;
 726     break;
 727   case Mips::ATOMIC_SWAP_I64_POSTRA:
 728     IsOr = true;
 729     break;
 730   case Mips::ATOMIC_LOAD_UMIN_I32_POSTRA:
 731   case Mips::ATOMIC_LOAD_UMIN_I64_POSTRA:
 732     IsUnsigned = true;
 733     [[fallthrough]];
 734   case Mips::ATOMIC_LOAD_MIN_I32_POSTRA:
 735   case Mips::ATOMIC_LOAD_MIN_I64_POSTRA:
 736     IsMin = true;
 737     break;
 738   case Mips::ATOMIC_LOAD_UMAX_I32_POSTRA:
 739   case Mips::ATOMIC_LOAD_UMAX_I64_POSTRA:
 740     IsUnsigned = true;
 741     [[fallthrough]];
 742   case Mips::ATOMIC_LOAD_MAX_I32_POSTRA:
 743   case Mips::ATOMIC_LOAD_MAX_I64_POSTRA:
 744     IsMax = true;
 745     break;
 746   default:
 747     llvm_unreachable("Unknown pseudo atomic!");
 748   }
 749
 750   const BasicBlock *LLVM_BB = BB.getBasicBlock();
 751   MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
 752   MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
 753   MachineFunction::iterator It = ++BB.getIterator();
 754   MF->insert(It, loopMBB);
 755   MF->insert(It, exitMBB);
 756
 757   exitMBB->splice(exitMBB->begin(), &BB, std::next(I), BB.end());
 758   exitMBB->transferSuccessorsAndUpdatePHIs(&BB);
 759
 760   BB.addSuccessor(loopMBB, BranchProbability::getOne());
 761   loopMBB->addSuccessor(exitMBB);
 762   loopMBB->addSuccessor(loopMBB);
 763   loopMBB->normalizeSuccProbs();
 764
 765   BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0);
 766   assert((OldVal != Ptr) && "Clobbered the wrong ptr reg!");
 767   assert((OldVal != Incr) && "Clobbered the wrong reg!");
 768   if (IsMin || IsMax) {
 769
 770     assert(I->getNumOperands() == 5 &&
 771            "Atomics min|max|umin|umax use an additional register");
 772     MCRegister Scratch2 = I->getOperand(4).getReg().asMCReg();
 773
 774     // On Mips64 result of slt is GPR32.
 775     MCRegister Scratch2_32 =
 776         (Size == 8) ? STI->getRegisterInfo()->getSubReg(Scratch2, Mips::sub_32)
 777                     : Scratch2;
 778
 779     unsigned SLTScratch2 = IsUnsigned ? SLTu : SLT;
 780     unsigned SELIncr = IsMax ? SELNEZ : SELEQZ;
 781     unsigned SELOldVal = IsMax ? SELEQZ : SELNEZ;
 782     unsigned MOVIncr = IsMax ? MOVN : MOVZ;
 783
 784     // unsigned: sltu Scratch2, oldVal, Incr
 785     // signed:   slt Scratch2, oldVal, Incr
 786     BuildMI(loopMBB, DL, TII->get(SLTScratch2), Scratch2_32)
 787         .addReg(OldVal)
 788         .addReg(Incr);
 789
 790     if (STI->hasMips64r6() || STI->hasMips32r6()) {
 791       // max: seleqz Scratch, OldVal, Scratch2
 792       //      selnez Scratch2, Incr, Scratch2
 793       //      or Scratch, Scratch, Scratch2
 794       // min: selnez Scratch, OldVal, Scratch2
 795       //      seleqz Scratch2, Incr, Scratch2
 796       //      or Scratch, Scratch, Scratch2
 797       BuildMI(loopMBB, DL, TII->get(SELOldVal), Scratch)
 798           .addReg(OldVal)
 799           .addReg(Scratch2);
 800       BuildMI(loopMBB, DL, TII->get(SELIncr), Scratch2)
 801           .addReg(Incr)
 802           .addReg(Scratch2);
 803       BuildMI(loopMBB, DL, TII->get(OR), Scratch)
 804           .addReg(Scratch)
 805           .addReg(Scratch2);
 806     } else {
 807       // max: move Scratch, OldVal
 808       //      movn Scratch, Incr, Scratch2, Scratch
 809       // min: move Scratch, OldVal
 810       //      movz Scratch, Incr, Scratch2, Scratch
 811       BuildMI(loopMBB, DL, TII->get(OR), Scratch)
 812           .addReg(OldVal)
 813           .addReg(ZERO);
 814       BuildMI(loopMBB, DL, TII->get(MOVIncr), Scratch)
 815           .addReg(Incr)
 816           .addReg(Scratch2)
 817           .addReg(Scratch);
 818     }
 819
 820   } else if (Opcode) {
 821     BuildMI(loopMBB, DL, TII->get(Opcode), Scratch).addReg(OldVal).addReg(Incr);
 822   } else if (IsNand) {
 823     assert(AND && NOR &&
 824            "Unknown nand instruction for atomic pseudo expansion");
 825     BuildMI(loopMBB, DL, TII->get(AND), Scratch).addReg(OldVal).addReg(Incr);
 826     BuildMI(loopMBB, DL, TII->get(NOR), Scratch).addReg(ZERO).addReg(Scratch);
 827   } else {
 828     assert(IsOr && OR && "Unknown instruction for atomic pseudo expansion!");
 829     (void)IsOr;
 830     BuildMI(loopMBB, DL, TII->get(OR), Scratch).addReg(Incr).addReg(ZERO);
 831   }
 832
 833   BuildMI(loopMBB, DL, TII->get(SC), Scratch)
 834       .addReg(Scratch)
 835       .addReg(Ptr)
 836       .addImm(0);
 837   BuildMI(loopMBB, DL, TII->get(BEQ))
 838       .addReg(Scratch)
 839       .addReg(ZERO)
 840       .addMBB(loopMBB);
 841
 842   NMBBI = BB.end();
 843   I->eraseFromParent();
 844
 845   LivePhysRegs LiveRegs;
 846   computeAndAddLiveIns(LiveRegs, *loopMBB);
 847   computeAndAddLiveIns(LiveRegs, *exitMBB);
 848
 849   return true;
 850 }
 851
 852 bool MipsExpandPseudo::expandMI(MachineBasicBlock &MBB,
 853                                 MachineBasicBlock::iterator MBBI,
 854                                 MachineBasicBlock::iterator &NMBB) {
 855
 856   bool Modified = false;
 857
 858   switch (MBBI->getOpcode()) {
 859   case Mips::ATOMIC_CMP_SWAP_I32_POSTRA:
 860   case Mips::ATOMIC_CMP_SWAP_I64_POSTRA:
 861     return expandAtomicCmpSwap(MBB, MBBI, NMBB);
 862   case Mips::ATOMIC_CMP_SWAP_I8_POSTRA:
 863   case Mips::ATOMIC_CMP_SWAP_I16_POSTRA:
 864     return expandAtomicCmpSwapSubword(MBB, MBBI, NMBB);
 865   case Mips::ATOMIC_SWAP_I8_POSTRA:
 866   case Mips::ATOMIC_SWAP_I16_POSTRA:
 867   case Mips::ATOMIC_LOAD_NAND_I8_POSTRA:
 868   case Mips::ATOMIC_LOAD_NAND_I16_POSTRA:
 869   case Mips::ATOMIC_LOAD_ADD_I8_POSTRA:
 870   case Mips::ATOMIC_LOAD_ADD_I16_POSTRA:
 871   case Mips::ATOMIC_LOAD_SUB_I8_POSTRA:
 872   case Mips::ATOMIC_LOAD_SUB_I16_POSTRA:
 873   case Mips::ATOMIC_LOAD_AND_I8_POSTRA:
 874   case Mips::ATOMIC_LOAD_AND_I16_POSTRA:
 875   case Mips::ATOMIC_LOAD_OR_I8_POSTRA:
 876   case Mips::ATOMIC_LOAD_OR_I16_POSTRA:
 877   case Mips::ATOMIC_LOAD_XOR_I8_POSTRA:
 878   case Mips::ATOMIC_LOAD_XOR_I16_POSTRA:
 879   case Mips::ATOMIC_LOAD_MIN_I8_POSTRA:
 880   case Mips::ATOMIC_LOAD_MIN_I16_POSTRA:
 881   case Mips::ATOMIC_LOAD_MAX_I8_POSTRA:
 882   case Mips::ATOMIC_LOAD_MAX_I16_POSTRA:
 883   case Mips::ATOMIC_LOAD_UMIN_I8_POSTRA:
 884   case Mips::ATOMIC_LOAD_UMIN_I16_POSTRA:
 885   case Mips::ATOMIC_LOAD_UMAX_I8_POSTRA:
 886   case Mips::ATOMIC_LOAD_UMAX_I16_POSTRA:
 887     return expandAtomicBinOpSubword(MBB, MBBI, NMBB);
 888   case Mips::ATOMIC_LOAD_ADD_I32_POSTRA:
 889   case Mips::ATOMIC_LOAD_SUB_I32_POSTRA:
 890   case Mips::ATOMIC_LOAD_AND_I32_POSTRA:
 891   case Mips::ATOMIC_LOAD_OR_I32_POSTRA:
 892   case Mips::ATOMIC_LOAD_XOR_I32_POSTRA:
 893   case Mips::ATOMIC_LOAD_NAND_I32_POSTRA:
 894   case Mips::ATOMIC_SWAP_I32_POSTRA:
 895   case Mips::ATOMIC_LOAD_MIN_I32_POSTRA:
 896   case Mips::ATOMIC_LOAD_MAX_I32_POSTRA:
 897   case Mips::ATOMIC_LOAD_UMIN_I32_POSTRA:
 898   case Mips::ATOMIC_LOAD_UMAX_I32_POSTRA:
 899     return expandAtomicBinOp(MBB, MBBI, NMBB, 4);
 900   case Mips::ATOMIC_LOAD_ADD_I64_POSTRA:
 901   case Mips::ATOMIC_LOAD_SUB_I64_POSTRA:
 902   case Mips::ATOMIC_LOAD_AND_I64_POSTRA:
 903   case Mips::ATOMIC_LOAD_OR_I64_POSTRA:
 904   case Mips::ATOMIC_LOAD_XOR_I64_POSTRA:
 905   case Mips::ATOMIC_LOAD_NAND_I64_POSTRA:
 906   case Mips::ATOMIC_SWAP_I64_POSTRA:
 907   case Mips::ATOMIC_LOAD_MIN_I64_POSTRA:
 908   case Mips::ATOMIC_LOAD_MAX_I64_POSTRA:
 909   case Mips::ATOMIC_LOAD_UMIN_I64_POSTRA:
 910   case Mips::ATOMIC_LOAD_UMAX_I64_POSTRA:
 911     return expandAtomicBinOp(MBB, MBBI, NMBB, 8);
 912   default:
 913     return Modified;
 914   }
 915 }
 916
 917 bool MipsExpandPseudo::expandMBB(MachineBasicBlock &MBB) {
 918   bool Modified = false;
 919
 920   MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
 921   while (MBBI != E) {
 922     MachineBasicBlock::iterator NMBBI = std::next(MBBI);
 923     Modified |= expandMI(MBB, MBBI, NMBBI);
 924     MBBI = NMBBI;
 925   }
 926
 927   return Modified;
 928 }
 929
 930 bool MipsExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
 931   STI = &MF.getSubtarget<MipsSubtarget>();
 932   TII = STI->getInstrInfo();
 933
 934   bool Modified = false;
 935   for (MachineBasicBlock &MBB : MF)
 936     Modified |= expandMBB(MBB);
 937
 938   if (Modified)
 939     MF.RenumberBlocks();
 940
 941   return Modified;
 942 }
 943
 944 /// createMipsExpandPseudoPass - returns an instance of the pseudo instruction
 945 /// expansion pass.
 946 FunctionPass *llvm::createMipsExpandPseudoPass() {
 947   return new MipsExpandPseudo();
 948 }