lib/Transforms/Scalar/LowerAtomic.cpp

   1 //===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This pass lowers atomic intrinsics to non-atomic form for use in a known
  10 // non-preemptible environment.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/Transforms/Scalar/LowerAtomic.h"
  15 #include "llvm/IR/Function.h"
  16 #include "llvm/IR/IRBuilder.h"
  17 #include "llvm/Pass.h"
  18 #include "llvm/Transforms/Scalar.h"
  19 using namespace llvm;
  20
  21 #define DEBUG_TYPE "loweratomic"
  22
  23 static bool LowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) {
  24   IRBuilder<> Builder(CXI);
  25   Value *Ptr = CXI->getPointerOperand();
  26   Value *Cmp = CXI->getCompareOperand();
  27   Value *Val = CXI->getNewValOperand();
  28
  29   LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr);
  30   Value *Equal = Builder.CreateICmpEQ(Orig, Cmp);
  31   Value *Res = Builder.CreateSelect(Equal, Val, Orig);
  32   Builder.CreateStore(Res, Ptr);
  33
  34   Res = Builder.CreateInsertValue(UndefValue::get(CXI->getType()), Orig, 0);
  35   Res = Builder.CreateInsertValue(Res, Equal, 1);
  36
  37   CXI->replaceAllUsesWith(Res);
  38   CXI->eraseFromParent();
  39   return true;
  40 }
  41
  42 static bool LowerAtomicRMWInst(AtomicRMWInst *RMWI) {
  43   IRBuilder<> Builder(RMWI);
  44   Value *Ptr = RMWI->getPointerOperand();
  45   Value *Val = RMWI->getValOperand();
  46
  47   LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr);
  48   Value *Res = nullptr;
  49
  50   switch (RMWI->getOperation()) {
  51   default: llvm_unreachable("Unexpected RMW operation");
  52   case AtomicRMWInst::Xchg:
  53     Res = Val;
  54     break;
  55   case AtomicRMWInst::Add:
  56     Res = Builder.CreateAdd(Orig, Val);
  57     break;
  58   case AtomicRMWInst::Sub:
  59     Res = Builder.CreateSub(Orig, Val);
  60     break;
  61   case AtomicRMWInst::And:
  62     Res = Builder.CreateAnd(Orig, Val);
  63     break;
  64   case AtomicRMWInst::Nand:
  65     Res = Builder.CreateNot(Builder.CreateAnd(Orig, Val));
  66     break;
  67   case AtomicRMWInst::Or:
  68     Res = Builder.CreateOr(Orig, Val);
  69     break;
  70   case AtomicRMWInst::Xor:
  71     Res = Builder.CreateXor(Orig, Val);
  72     break;
  73   case AtomicRMWInst::Max:
  74     Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val),
  75                                Val, Orig);
  76     break;
  77   case AtomicRMWInst::Min:
  78     Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val),
  79                                Orig, Val);
  80     break;
  81   case AtomicRMWInst::UMax:
  82     Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val),
  83                                Val, Orig);
  84     break;
  85   case AtomicRMWInst::UMin:
  86     Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val),
  87                                Orig, Val);
  88     break;
  89   }
  90   Builder.CreateStore(Res, Ptr);
  91   RMWI->replaceAllUsesWith(Orig);
  92   RMWI->eraseFromParent();
  93   return true;
  94 }
  95
  96 static bool LowerFenceInst(FenceInst *FI) {
  97   FI->eraseFromParent();
  98   return true;
  99 }
 100
 101 static bool LowerLoadInst(LoadInst *LI) {
 102   LI->setAtomic(AtomicOrdering::NotAtomic);
 103   return true;
 104 }
 105
 106 static bool LowerStoreInst(StoreInst *SI) {
 107   SI->setAtomic(AtomicOrdering::NotAtomic);
 108   return true;
 109 }
 110
 111 static bool runOnBasicBlock(BasicBlock &BB) {
 112   bool Changed = false;
 113   for (BasicBlock::iterator DI = BB.begin(), DE = BB.end(); DI != DE;) {
 114     Instruction *Inst = &*DI++;
 115     if (FenceInst *FI = dyn_cast<FenceInst>(Inst))
 116       Changed |= LowerFenceInst(FI);
 117     else if (AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(Inst))
 118       Changed |= LowerAtomicCmpXchgInst(CXI);
 119     else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(Inst))
 120       Changed |= LowerAtomicRMWInst(RMWI);
 121     else if (LoadInst *LI = dyn_cast<LoadInst>(Inst)) {
 122       if (LI->isAtomic())
 123         LowerLoadInst(LI);
 124     } else if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
 125       if (SI->isAtomic())
 126         LowerStoreInst(SI);
 127     }
 128   }
 129   return Changed;
 130 }
 131
 132 static bool lowerAtomics(Function &F) {
 133   bool Changed = false;
 134   for (BasicBlock &BB : F) {
 135     Changed |= runOnBasicBlock(BB);
 136   }
 137   return Changed;
 138 }
 139
 140 PreservedAnalyses LowerAtomicPass::run(Function &F, FunctionAnalysisManager &) {
 141   if (lowerAtomics(F))
 142     return PreservedAnalyses::none();
 143   return PreservedAnalyses::all();
 144 }
 145
 146 namespace {
 147 class LowerAtomicLegacyPass : public FunctionPass {
 148 public:
 149   static char ID;
 150
 151   LowerAtomicLegacyPass() : FunctionPass(ID) {
 152     initializeLowerAtomicLegacyPassPass(*PassRegistry::getPassRegistry());
 153   }
 154
 155   bool runOnFunction(Function &F) override {
 156     // Don't skip optnone functions; atomics still need to be lowered.
 157     FunctionAnalysisManager DummyFAM;
 158     auto PA = Impl.run(F, DummyFAM);
 159     return !PA.areAllPreserved();
 160   }
 161
 162 private:
 163   LowerAtomicPass Impl;
 164   };
 165 }
 166
 167 char LowerAtomicLegacyPass::ID = 0;
 168 INITIALIZE_PASS(LowerAtomicLegacyPass, "loweratomic",
 169                 "Lower atomic intrinsics to non-atomic form", false, false)
 170
 171 Pass *llvm::createLowerAtomicPass() { return new LowerAtomicLegacyPass(); }