llvm/lib/Transforms/Scalar/LowerAtomic.cpp

   1 //===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This pass lowers atomic intrinsics to non-atomic form for use in a known
  10 // non-preemptible environment.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/Transforms/Scalar/LowerAtomic.h"
  15 #include "llvm/IR/Function.h"
  16 #include "llvm/IR/IRBuilder.h"
  17 #include "llvm/InitializePasses.h"
  18 #include "llvm/Pass.h"
  19 #include "llvm/Transforms/Scalar.h"
  20 using namespace llvm;
  21
  22 #define DEBUG_TYPE "loweratomic"
  23
  24 static bool LowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) {
  25   IRBuilder<> Builder(CXI);
  26   Value *Ptr = CXI->getPointerOperand();
  27   Value *Cmp = CXI->getCompareOperand();
  28   Value *Val = CXI->getNewValOperand();
  29
  30   LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr);
  31   Value *Equal = Builder.CreateICmpEQ(Orig, Cmp);
  32   Value *Res = Builder.CreateSelect(Equal, Val, Orig);
  33   Builder.CreateStore(Res, Ptr);
  34
  35   Res = Builder.CreateInsertValue(UndefValue::get(CXI->getType()), Orig, 0);
  36   Res = Builder.CreateInsertValue(Res, Equal, 1);
  37
  38   CXI->replaceAllUsesWith(Res);
  39   CXI->eraseFromParent();
  40   return true;
  41 }
  42
  43 bool llvm::lowerAtomicRMWInst(AtomicRMWInst *RMWI) {
  44   IRBuilder<> Builder(RMWI);
  45   Value *Ptr = RMWI->getPointerOperand();
  46   Value *Val = RMWI->getValOperand();
  47
  48   LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr);
  49   Value *Res = nullptr;
  50
  51   switch (RMWI->getOperation()) {
  52   default: llvm_unreachable("Unexpected RMW operation");
  53   case AtomicRMWInst::Xchg:
  54     Res = Val;
  55     break;
  56   case AtomicRMWInst::Add:
  57     Res = Builder.CreateAdd(Orig, Val);
  58     break;
  59   case AtomicRMWInst::Sub:
  60     Res = Builder.CreateSub(Orig, Val);
  61     break;
  62   case AtomicRMWInst::And:
  63     Res = Builder.CreateAnd(Orig, Val);
  64     break;
  65   case AtomicRMWInst::Nand:
  66     Res = Builder.CreateNot(Builder.CreateAnd(Orig, Val));
  67     break;
  68   case AtomicRMWInst::Or:
  69     Res = Builder.CreateOr(Orig, Val);
  70     break;
  71   case AtomicRMWInst::Xor:
  72     Res = Builder.CreateXor(Orig, Val);
  73     break;
  74   case AtomicRMWInst::Max:
  75     Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val),
  76                                Val, Orig);
  77     break;
  78   case AtomicRMWInst::Min:
  79     Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val),
  80                                Orig, Val);
  81     break;
  82   case AtomicRMWInst::UMax:
  83     Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val),
  84                                Val, Orig);
  85     break;
  86   case AtomicRMWInst::UMin:
  87     Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val),
  88                                Orig, Val);
  89     break;
  90   case AtomicRMWInst::FAdd:
  91     Res = Builder.CreateFAdd(Orig, Val);
  92     break;
  93   case AtomicRMWInst::FSub:
  94     Res = Builder.CreateFSub(Orig, Val);
  95     break;
  96   }
  97   Builder.CreateStore(Res, Ptr);
  98   RMWI->replaceAllUsesWith(Orig);
  99   RMWI->eraseFromParent();
 100   return true;
 101 }
 102
 103 static bool LowerFenceInst(FenceInst *FI) {
 104   FI->eraseFromParent();
 105   return true;
 106 }
 107
 108 static bool LowerLoadInst(LoadInst *LI) {
 109   LI->setAtomic(AtomicOrdering::NotAtomic);
 110   return true;
 111 }
 112
 113 static bool LowerStoreInst(StoreInst *SI) {
 114   SI->setAtomic(AtomicOrdering::NotAtomic);
 115   return true;
 116 }
 117
 118 static bool runOnBasicBlock(BasicBlock &BB) {
 119   bool Changed = false;
 120   for (Instruction &Inst : make_early_inc_range(BB)) {
 121     if (FenceInst *FI = dyn_cast<FenceInst>(&Inst))
 122       Changed |= LowerFenceInst(FI);
 123     else if (AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(&Inst))
 124       Changed |= LowerAtomicCmpXchgInst(CXI);
 125     else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(&Inst))
 126       Changed |= lowerAtomicRMWInst(RMWI);
 127     else if (LoadInst *LI = dyn_cast<LoadInst>(&Inst)) {
 128       if (LI->isAtomic())
 129         LowerLoadInst(LI);
 130     } else if (StoreInst *SI = dyn_cast<StoreInst>(&Inst)) {
 131       if (SI->isAtomic())
 132         LowerStoreInst(SI);
 133     }
 134   }
 135   return Changed;
 136 }
 137
 138 static bool lowerAtomics(Function &F) {
 139   bool Changed = false;
 140   for (BasicBlock &BB : F) {
 141     Changed |= runOnBasicBlock(BB);
 142   }
 143   return Changed;
 144 }
 145
 146 PreservedAnalyses LowerAtomicPass::run(Function &F, FunctionAnalysisManager &) {
 147   if (lowerAtomics(F))
 148     return PreservedAnalyses::none();
 149   return PreservedAnalyses::all();
 150 }
 151
 152 namespace {
 153 class LowerAtomicLegacyPass : public FunctionPass {
 154 public:
 155   static char ID;
 156
 157   LowerAtomicLegacyPass() : FunctionPass(ID) {
 158     initializeLowerAtomicLegacyPassPass(*PassRegistry::getPassRegistry());
 159   }
 160
 161   bool runOnFunction(Function &F) override {
 162     // Don't skip optnone functions; atomics still need to be lowered.
 163     FunctionAnalysisManager DummyFAM;
 164     auto PA = Impl.run(F, DummyFAM);
 165     return !PA.areAllPreserved();
 166   }
 167
 168 private:
 169   LowerAtomicPass Impl;
 170   };
 171 }
 172
 173 char LowerAtomicLegacyPass::ID = 0;
 174 INITIALIZE_PASS(LowerAtomicLegacyPass, "loweratomic",
 175                 "Lower atomic intrinsics to non-atomic form", false, false)
 176
 177 Pass *llvm::createLowerAtomicPass() { return new LowerAtomicLegacyPass(); }