llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp

   1 //===- InstCombineAtomicRMW.cpp -------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the visit functions for atomic rmw instructions.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "InstCombineInternal.h"
  14 #include "llvm/IR/Instructions.h"
  15
  16 using namespace llvm;
  17
  18 namespace {
  19 /// Return true if and only if the given instruction does not modify the memory
  20 /// location referenced.  Note that an idemptent atomicrmw may still have
  21 /// ordering effects on nearby instructions, or be volatile.
  22 /// TODO: Common w/ the version in AtomicExpandPass, and change the term used.
  23 /// Idemptotent is confusing in this context.
  24 bool isIdempotentRMW(AtomicRMWInst& RMWI) {
  25   if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand()))
  26     switch(RMWI.getOperation()) {
  27     case AtomicRMWInst::FAdd: // -0.0
  28       return CF->isZero() && CF->isNegative();
  29     case AtomicRMWInst::FSub: // +0.0
  30       return CF->isZero() && !CF->isNegative();
  31     default:
  32       return false;
  33     };
  34
  35   auto C = dyn_cast<ConstantInt>(RMWI.getValOperand());
  36   if(!C)
  37     return false;
  38
  39   switch(RMWI.getOperation()) {
  40     case AtomicRMWInst::Add:
  41     case AtomicRMWInst::Sub:
  42     case AtomicRMWInst::Or:
  43     case AtomicRMWInst::Xor:
  44       return C->isZero();
  45     case AtomicRMWInst::And:
  46       return C->isMinusOne();
  47     case AtomicRMWInst::Min:
  48       return C->isMaxValue(true);
  49     case AtomicRMWInst::Max:
  50       return C->isMinValue(true);
  51     case AtomicRMWInst::UMin:
  52       return C->isMaxValue(false);
  53     case AtomicRMWInst::UMax:
  54       return C->isMinValue(false);
  55     default:
  56       return false;
  57   }
  58 }
  59
  60 /// Return true if the given instruction always produces a value in memory
  61 /// equivalent to its value operand.
  62 bool isSaturating(AtomicRMWInst& RMWI) {
  63   if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand()))
  64     switch (RMWI.getOperation()) {
  65     case AtomicRMWInst::FMax:
  66       // maxnum(x, +inf) -> +inf
  67       return !CF->isNegative() && CF->isInfinity();
  68     case AtomicRMWInst::FMin:
  69       // minnum(x, -inf) -> +inf
  70       return CF->isNegative() && CF->isInfinity();
  71     case AtomicRMWInst::FAdd:
  72     case AtomicRMWInst::FSub:
  73       return CF->isNaN();
  74     default:
  75       return false;
  76     };
  77
  78   auto C = dyn_cast<ConstantInt>(RMWI.getValOperand());
  79   if(!C)
  80     return false;
  81
  82   switch(RMWI.getOperation()) {
  83   default:
  84     return false;
  85   case AtomicRMWInst::Xchg:
  86     return true;
  87   case AtomicRMWInst::Or:
  88     return C->isAllOnesValue();
  89   case AtomicRMWInst::And:
  90     return C->isZero();
  91   case AtomicRMWInst::Min:
  92     return C->isMinValue(true);
  93   case AtomicRMWInst::Max:
  94     return C->isMaxValue(true);
  95   case AtomicRMWInst::UMin:
  96     return C->isMinValue(false);
  97   case AtomicRMWInst::UMax:
  98     return C->isMaxValue(false);
  99   };
 100 }
 101 } // namespace
 102
 103 Instruction *InstCombinerImpl::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
 104
 105   // Volatile RMWs perform a load and a store, we cannot replace this by just a
 106   // load or just a store. We chose not to canonicalize out of general paranoia
 107   // about user expectations around volatile.
 108   if (RMWI.isVolatile())
 109     return nullptr;
 110
 111   // Any atomicrmw op which produces a known result in memory can be
 112   // replaced w/an atomicrmw xchg.
 113   if (isSaturating(RMWI) &&
 114       RMWI.getOperation() != AtomicRMWInst::Xchg) {
 115     RMWI.setOperation(AtomicRMWInst::Xchg);
 116     return &RMWI;
 117   }
 118
 119   assert(RMWI.getOrdering() != AtomicOrdering::NotAtomic &&
 120          RMWI.getOrdering() != AtomicOrdering::Unordered &&
 121          "AtomicRMWs don't make sense with Unordered or NotAtomic");
 122
 123   if (!isIdempotentRMW(RMWI))
 124     return nullptr;
 125
 126   // We chose to canonicalize all idempotent operations to an single
 127   // operation code and constant.  This makes it easier for the rest of the
 128   // optimizer to match easily.  The choices of or w/0 and fadd w/-0.0 are
 129   // arbitrary.
 130   if (RMWI.getType()->isIntegerTy() &&
 131       RMWI.getOperation() != AtomicRMWInst::Or) {
 132     RMWI.setOperation(AtomicRMWInst::Or);
 133     return replaceOperand(RMWI, 1, ConstantInt::get(RMWI.getType(), 0));
 134   } else if (RMWI.getType()->isFloatingPointTy() &&
 135              RMWI.getOperation() != AtomicRMWInst::FAdd) {
 136     RMWI.setOperation(AtomicRMWInst::FAdd);
 137     return replaceOperand(RMWI, 1, ConstantFP::getNegativeZero(RMWI.getType()));
 138   }
 139
 140   return nullptr;
 141 }