lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp

   1 //===- InstCombineAtomicRMW.cpp -------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the visit functions for atomic rmw instructions.
  10 //
  11 //===----------------------------------------------------------------------===//
  12 #include "InstCombineInternal.h"
  13 #include "llvm/IR/Instructions.h"
  14
  15 using namespace llvm;
  16
  17 namespace {
  18 /// Return true if and only if the given instruction does not modify the memory
  19 /// location referenced.  Note that an idemptent atomicrmw may still have
  20 /// ordering effects on nearby instructions, or be volatile.
  21 /// TODO: Common w/ the version in AtomicExpandPass, and change the term used.
  22 /// Idemptotent is confusing in this context.
  23 bool isIdempotentRMW(AtomicRMWInst& RMWI) {
  24   if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand()))
  25     switch(RMWI.getOperation()) {
  26     case AtomicRMWInst::FAdd: // -0.0
  27       return CF->isZero() && CF->isNegative();
  28     case AtomicRMWInst::FSub: // +0.0
  29       return CF->isZero() && !CF->isNegative();
  30     default:
  31       return false;
  32     };
  33
  34   auto C = dyn_cast<ConstantInt>(RMWI.getValOperand());
  35   if(!C)
  36     return false;
  37
  38   switch(RMWI.getOperation()) {
  39     case AtomicRMWInst::Add:
  40     case AtomicRMWInst::Sub:
  41     case AtomicRMWInst::Or:
  42     case AtomicRMWInst::Xor:
  43       return C->isZero();
  44     case AtomicRMWInst::And:
  45       return C->isMinusOne();
  46     case AtomicRMWInst::Min:
  47       return C->isMaxValue(true);
  48     case AtomicRMWInst::Max:
  49       return C->isMinValue(true);
  50     case AtomicRMWInst::UMin:
  51       return C->isMaxValue(false);
  52     case AtomicRMWInst::UMax:
  53       return C->isMinValue(false);
  54     default:
  55       return false;
  56   }
  57 }
  58
  59 /// Return true if the given instruction always produces a value in memory
  60 /// equivalent to its value operand.
  61 bool isSaturating(AtomicRMWInst& RMWI) {
  62   if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand()))
  63     switch(RMWI.getOperation()) {
  64     case AtomicRMWInst::FAdd:
  65     case AtomicRMWInst::FSub:
  66       return CF->isNaN();
  67     default:
  68       return false;
  69     };
  70
  71   auto C = dyn_cast<ConstantInt>(RMWI.getValOperand());
  72   if(!C)
  73     return false;
  74
  75   switch(RMWI.getOperation()) {
  76   default:
  77     return false;
  78   case AtomicRMWInst::Xchg:
  79     return true;
  80   case AtomicRMWInst::Or:
  81     return C->isAllOnesValue();
  82   case AtomicRMWInst::And:
  83     return C->isZero();
  84   case AtomicRMWInst::Min:
  85     return C->isMinValue(true);
  86   case AtomicRMWInst::Max:
  87     return C->isMaxValue(true);
  88   case AtomicRMWInst::UMin:
  89     return C->isMinValue(false);
  90   case AtomicRMWInst::UMax:
  91     return C->isMaxValue(false);
  92   };
  93 }
  94 }
  95
  96 Instruction *InstCombiner::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
  97
  98   // Volatile RMWs perform a load and a store, we cannot replace this by just a
  99   // load or just a store. We chose not to canonicalize out of general paranoia
 100   // about user expectations around volatile.
 101   if (RMWI.isVolatile())
 102     return nullptr;
 103
 104   // Any atomicrmw op which produces a known result in memory can be
 105   // replaced w/an atomicrmw xchg.
 106   if (isSaturating(RMWI) &&
 107       RMWI.getOperation() != AtomicRMWInst::Xchg) {
 108     RMWI.setOperation(AtomicRMWInst::Xchg);
 109     return &RMWI;
 110   }
 111
 112   AtomicOrdering Ordering = RMWI.getOrdering();
 113   assert(Ordering != AtomicOrdering::NotAtomic &&
 114          Ordering != AtomicOrdering::Unordered &&
 115          "AtomicRMWs don't make sense with Unordered or NotAtomic");
 116
 117   // Any atomicrmw xchg with no uses can be converted to a atomic store if the
 118   // ordering is compatible.
 119   if (RMWI.getOperation() == AtomicRMWInst::Xchg &&
 120       RMWI.use_empty()) {
 121     if (Ordering != AtomicOrdering::Release &&
 122         Ordering != AtomicOrdering::Monotonic)
 123       return nullptr;
 124     auto *SI = new StoreInst(RMWI.getValOperand(),
 125                              RMWI.getPointerOperand(), &RMWI);
 126     SI->setAtomic(Ordering, RMWI.getSyncScopeID());
 127     SI->setAlignment(MaybeAlign(DL.getABITypeAlignment(RMWI.getType())));
 128     return eraseInstFromFunction(RMWI);
 129   }
 130
 131   if (!isIdempotentRMW(RMWI))
 132     return nullptr;
 133
 134   // We chose to canonicalize all idempotent operations to an single
 135   // operation code and constant.  This makes it easier for the rest of the
 136   // optimizer to match easily.  The choices of or w/0 and fadd w/-0.0 are
 137   // arbitrary.
 138   if (RMWI.getType()->isIntegerTy() &&
 139       RMWI.getOperation() != AtomicRMWInst::Or) {
 140     RMWI.setOperation(AtomicRMWInst::Or);
 141     RMWI.setOperand(1, ConstantInt::get(RMWI.getType(), 0));
 142     return &RMWI;
 143   } else if (RMWI.getType()->isFloatingPointTy() &&
 144              RMWI.getOperation() != AtomicRMWInst::FAdd) {
 145     RMWI.setOperation(AtomicRMWInst::FAdd);
 146     RMWI.setOperand(1, ConstantFP::getNegativeZero(RMWI.getType()));
 147     return &RMWI;
 148   }
 149
 150   // Check if the required ordering is compatible with an atomic load.
 151   if (Ordering != AtomicOrdering::Acquire &&
 152       Ordering != AtomicOrdering::Monotonic)
 153     return nullptr;
 154
 155   LoadInst *Load = new LoadInst(RMWI.getType(), RMWI.getPointerOperand());
 156   Load->setAtomic(Ordering, RMWI.getSyncScopeID());
 157   Load->setAlignment(MaybeAlign(DL.getABITypeAlignment(RMWI.getType())));
 158   return Load;
 159 }