llvm/lib/Transforms/Utils/LowerAtomic.cpp

   1 //===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This pass lowers atomic intrinsics to non-atomic form for use in a known
  10 // non-preemptible environment.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/Transforms/Utils/LowerAtomic.h"
  15 #include "llvm/IR/Function.h"
  16 #include "llvm/IR/IRBuilder.h"
  17
  18 using namespace llvm;
  19
  20 #define DEBUG_TYPE "loweratomic"
  21
  22 bool llvm::lowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) {
  23   IRBuilder<> Builder(CXI);
  24   Value *Ptr = CXI->getPointerOperand();
  25   Value *Cmp = CXI->getCompareOperand();
  26   Value *Val = CXI->getNewValOperand();
  27
  28   LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr);
  29   Value *Equal = Builder.CreateICmpEQ(Orig, Cmp);
  30   Value *Res = Builder.CreateSelect(Equal, Val, Orig);
  31   Builder.CreateStore(Res, Ptr);
  32
  33   Res = Builder.CreateInsertValue(PoisonValue::get(CXI->getType()), Orig, 0);
  34   Res = Builder.CreateInsertValue(Res, Equal, 1);
  35
  36   CXI->replaceAllUsesWith(Res);
  37   CXI->eraseFromParent();
  38   return true;
  39 }
  40
  41 Value *llvm::buildAtomicRMWValue(AtomicRMWInst::BinOp Op,
  42                                  IRBuilderBase &Builder, Value *Loaded,
  43                                  Value *Val) {
  44   Value *NewVal;
  45   switch (Op) {
  46   case AtomicRMWInst::Xchg:
  47     return Val;
  48   case AtomicRMWInst::Add:
  49     return Builder.CreateAdd(Loaded, Val, "new");
  50   case AtomicRMWInst::Sub:
  51     return Builder.CreateSub(Loaded, Val, "new");
  52   case AtomicRMWInst::And:
  53     return Builder.CreateAnd(Loaded, Val, "new");
  54   case AtomicRMWInst::Nand:
  55     return Builder.CreateNot(Builder.CreateAnd(Loaded, Val), "new");
  56   case AtomicRMWInst::Or:
  57     return Builder.CreateOr(Loaded, Val, "new");
  58   case AtomicRMWInst::Xor:
  59     return Builder.CreateXor(Loaded, Val, "new");
  60   case AtomicRMWInst::Max:
  61     NewVal = Builder.CreateICmpSGT(Loaded, Val);
  62     return Builder.CreateSelect(NewVal, Loaded, Val, "new");
  63   case AtomicRMWInst::Min:
  64     NewVal = Builder.CreateICmpSLE(Loaded, Val);
  65     return Builder.CreateSelect(NewVal, Loaded, Val, "new");
  66   case AtomicRMWInst::UMax:
  67     NewVal = Builder.CreateICmpUGT(Loaded, Val);
  68     return Builder.CreateSelect(NewVal, Loaded, Val, "new");
  69   case AtomicRMWInst::UMin:
  70     NewVal = Builder.CreateICmpULE(Loaded, Val);
  71     return Builder.CreateSelect(NewVal, Loaded, Val, "new");
  72   case AtomicRMWInst::FAdd:
  73     return Builder.CreateFAdd(Loaded, Val, "new");
  74   case AtomicRMWInst::FSub:
  75     return Builder.CreateFSub(Loaded, Val, "new");
  76   case AtomicRMWInst::FMax:
  77     return Builder.CreateMaxNum(Loaded, Val);
  78   case AtomicRMWInst::FMin:
  79     return Builder.CreateMinNum(Loaded, Val);
  80   case AtomicRMWInst::UIncWrap: {
  81     Constant *One = ConstantInt::get(Loaded->getType(), 1);
  82     Value *Inc = Builder.CreateAdd(Loaded, One);
  83     Value *Cmp = Builder.CreateICmpUGE(Loaded, Val);
  84     Constant *Zero = ConstantInt::get(Loaded->getType(), 0);
  85     return Builder.CreateSelect(Cmp, Zero, Inc, "new");
  86   }
  87   case AtomicRMWInst::UDecWrap: {
  88     Constant *Zero = ConstantInt::get(Loaded->getType(), 0);
  89     Constant *One = ConstantInt::get(Loaded->getType(), 1);
  90
  91     Value *Dec = Builder.CreateSub(Loaded, One);
  92     Value *CmpEq0 = Builder.CreateICmpEQ(Loaded, Zero);
  93     Value *CmpOldGtVal = Builder.CreateICmpUGT(Loaded, Val);
  94     Value *Or = Builder.CreateOr(CmpEq0, CmpOldGtVal);
  95     return Builder.CreateSelect(Or, Val, Dec, "new");
  96   }
  97   default:
  98     llvm_unreachable("Unknown atomic op");
  99   }
 100 }
 101
 102 bool llvm::lowerAtomicRMWInst(AtomicRMWInst *RMWI) {
 103   IRBuilder<> Builder(RMWI);
 104   Builder.setIsFPConstrained(
 105       RMWI->getFunction()->hasFnAttribute(Attribute::StrictFP));
 106
 107   Value *Ptr = RMWI->getPointerOperand();
 108   Value *Val = RMWI->getValOperand();
 109
 110   LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr);
 111   Value *Res = buildAtomicRMWValue(RMWI->getOperation(), Builder, Orig, Val);
 112   Builder.CreateStore(Res, Ptr);
 113   RMWI->replaceAllUsesWith(Orig);
 114   RMWI->eraseFromParent();
 115   return true;
 116 }