llvm/lib/Transforms/Utils/LowerAtomic.cpp

   1 //===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This pass lowers atomic intrinsics to non-atomic form for use in a known
  10 // non-preemptible environment.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/Transforms/Utils/LowerAtomic.h"
  15 #include "llvm/IR/Function.h"
  16 #include "llvm/IR/IRBuilder.h"
  17
  18 using namespace llvm;
  19
  20 #define DEBUG_TYPE "loweratomic"
  21
  22 bool llvm::lowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) {
  23   IRBuilder<> Builder(CXI);
  24   Value *Ptr = CXI->getPointerOperand();
  25   Value *Cmp = CXI->getCompareOperand();
  26   Value *Val = CXI->getNewValOperand();
  27
  28   auto [Orig, Equal] =
  29       buildCmpXchgValue(Builder, Ptr, Cmp, Val, CXI->getAlign());
  30
  31   Value *Res =
  32       Builder.CreateInsertValue(PoisonValue::get(CXI->getType()), Orig, 0);
  33   Res = Builder.CreateInsertValue(Res, Equal, 1);
  34
  35   CXI->replaceAllUsesWith(Res);
  36   CXI->eraseFromParent();
  37   return true;
  38 }
  39
  40 std::pair<Value *, Value *> llvm::buildCmpXchgValue(IRBuilderBase &Builder,
  41                                                     Value *Ptr, Value *Cmp,
  42                                                     Value *Val,
  43                                                     Align Alignment) {
  44   LoadInst *Orig = Builder.CreateAlignedLoad(Val->getType(), Ptr, Alignment);
  45   Value *Equal = Builder.CreateICmpEQ(Orig, Cmp);
  46   Value *Res = Builder.CreateSelect(Equal, Val, Orig);
  47   Builder.CreateAlignedStore(Res, Ptr, Alignment);
  48
  49   return {Orig, Equal};
  50 }
  51
  52 Value *llvm::buildAtomicRMWValue(AtomicRMWInst::BinOp Op,
  53                                  IRBuilderBase &Builder, Value *Loaded,
  54                                  Value *Val) {
  55   Value *NewVal;
  56   switch (Op) {
  57   case AtomicRMWInst::Xchg:
  58     return Val;
  59   case AtomicRMWInst::Add:
  60     return Builder.CreateAdd(Loaded, Val, "new");
  61   case AtomicRMWInst::Sub:
  62     return Builder.CreateSub(Loaded, Val, "new");
  63   case AtomicRMWInst::And:
  64     return Builder.CreateAnd(Loaded, Val, "new");
  65   case AtomicRMWInst::Nand:
  66     return Builder.CreateNot(Builder.CreateAnd(Loaded, Val), "new");
  67   case AtomicRMWInst::Or:
  68     return Builder.CreateOr(Loaded, Val, "new");
  69   case AtomicRMWInst::Xor:
  70     return Builder.CreateXor(Loaded, Val, "new");
  71   case AtomicRMWInst::Max:
  72     NewVal = Builder.CreateICmpSGT(Loaded, Val);
  73     return Builder.CreateSelect(NewVal, Loaded, Val, "new");
  74   case AtomicRMWInst::Min:
  75     NewVal = Builder.CreateICmpSLE(Loaded, Val);
  76     return Builder.CreateSelect(NewVal, Loaded, Val, "new");
  77   case AtomicRMWInst::UMax:
  78     NewVal = Builder.CreateICmpUGT(Loaded, Val);
  79     return Builder.CreateSelect(NewVal, Loaded, Val, "new");
  80   case AtomicRMWInst::UMin:
  81     NewVal = Builder.CreateICmpULE(Loaded, Val);
  82     return Builder.CreateSelect(NewVal, Loaded, Val, "new");
  83   case AtomicRMWInst::FAdd:
  84     return Builder.CreateFAdd(Loaded, Val, "new");
  85   case AtomicRMWInst::FSub:
  86     return Builder.CreateFSub(Loaded, Val, "new");
  87   case AtomicRMWInst::FMax:
  88     return Builder.CreateMaxNum(Loaded, Val);
  89   case AtomicRMWInst::FMin:
  90     return Builder.CreateMinNum(Loaded, Val);
  91   case AtomicRMWInst::UIncWrap: {
  92     Constant *One = ConstantInt::get(Loaded->getType(), 1);
  93     Value *Inc = Builder.CreateAdd(Loaded, One);
  94     Value *Cmp = Builder.CreateICmpUGE(Loaded, Val);
  95     Constant *Zero = ConstantInt::get(Loaded->getType(), 0);
  96     return Builder.CreateSelect(Cmp, Zero, Inc, "new");
  97   }
  98   case AtomicRMWInst::UDecWrap: {
  99     Constant *Zero = ConstantInt::get(Loaded->getType(), 0);
 100     Constant *One = ConstantInt::get(Loaded->getType(), 1);
 101
 102     Value *Dec = Builder.CreateSub(Loaded, One);
 103     Value *CmpEq0 = Builder.CreateICmpEQ(Loaded, Zero);
 104     Value *CmpOldGtVal = Builder.CreateICmpUGT(Loaded, Val);
 105     Value *Or = Builder.CreateOr(CmpEq0, CmpOldGtVal);
 106     return Builder.CreateSelect(Or, Val, Dec, "new");
 107   }
 108   case AtomicRMWInst::USubCond: {
 109     Value *Cmp = Builder.CreateICmpUGE(Loaded, Val);
 110     Value *Sub = Builder.CreateSub(Loaded, Val);
 111     return Builder.CreateSelect(Cmp, Sub, Loaded, "new");
 112   }
 113   case AtomicRMWInst::USubSat:
 114     return Builder.CreateIntrinsic(Intrinsic::usub_sat, Loaded->getType(),
 115                                    {Loaded, Val}, nullptr, "new");
 116   default:
 117     llvm_unreachable("Unknown atomic op");
 118   }
 119 }
 120
 121 bool llvm::lowerAtomicRMWInst(AtomicRMWInst *RMWI) {
 122   IRBuilder<> Builder(RMWI);
 123   Builder.setIsFPConstrained(
 124       RMWI->getFunction()->hasFnAttribute(Attribute::StrictFP));
 125
 126   Value *Ptr = RMWI->getPointerOperand();
 127   Value *Val = RMWI->getValOperand();
 128
 129   LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr);
 130   Value *Res = buildAtomicRMWValue(RMWI->getOperation(), Builder, Orig, Val);
 131   Builder.CreateStore(Res, Ptr);
 132   RMWI->replaceAllUsesWith(Orig);
 133   RMWI->eraseFromParent();
 134   return true;
 135 }