clang/lib/CodeGen/CGAtomic.cpp

   1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file contains the code for emitting atomic operations.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "CGCall.h"
  14 #include "CGRecordLayout.h"
  15 #include "CodeGenFunction.h"
  16 #include "CodeGenModule.h"
  17 #include "TargetInfo.h"
  18 #include "clang/AST/ASTContext.h"
  19 #include "clang/CodeGen/CGFunctionInfo.h"
  20 #include "clang/Frontend/FrontendDiagnostic.h"
  21 #include "llvm/ADT/DenseMap.h"
  22 #include "llvm/IR/DataLayout.h"
  23 #include "llvm/IR/Intrinsics.h"
  24 #include "llvm/IR/Operator.h"
  25
  26 using namespace clang;
  27 using namespace CodeGen;
  28
  29 namespace {
  30   class AtomicInfo {
  31     CodeGenFunction &CGF;
  32     QualType AtomicTy;
  33     QualType ValueTy;
  34     uint64_t AtomicSizeInBits;
  35     uint64_t ValueSizeInBits;
  36     CharUnits AtomicAlign;
  37     CharUnits ValueAlign;
  38     TypeEvaluationKind EvaluationKind;
  39     bool UseLibcall;
  40     LValue LVal;
  41     CGBitFieldInfo BFI;
  42   public:
  43     AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
  44         : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
  45           EvaluationKind(TEK_Scalar), UseLibcall(true) {
  46       assert(!lvalue.isGlobalReg());
  47       ASTContext &C = CGF.getContext();
  48       if (lvalue.isSimple()) {
  49         AtomicTy = lvalue.getType();
  50         if (auto *ATy = AtomicTy->getAs<AtomicType>())
  51           ValueTy = ATy->getValueType();
  52         else
  53           ValueTy = AtomicTy;
  54         EvaluationKind = CGF.getEvaluationKind(ValueTy);
  55
  56         uint64_t ValueAlignInBits;
  57         uint64_t AtomicAlignInBits;
  58         TypeInfo ValueTI = C.getTypeInfo(ValueTy);
  59         ValueSizeInBits = ValueTI.Width;
  60         ValueAlignInBits = ValueTI.Align;
  61
  62         TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
  63         AtomicSizeInBits = AtomicTI.Width;
  64         AtomicAlignInBits = AtomicTI.Align;
  65
  66         assert(ValueSizeInBits <= AtomicSizeInBits);
  67         assert(ValueAlignInBits <= AtomicAlignInBits);
  68
  69         AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
  70         ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
  71         if (lvalue.getAlignment().isZero())
  72           lvalue.setAlignment(AtomicAlign);
  73
  74         LVal = lvalue;
  75       } else if (lvalue.isBitField()) {
  76         ValueTy = lvalue.getType();
  77         ValueSizeInBits = C.getTypeSize(ValueTy);
  78         auto &OrigBFI = lvalue.getBitFieldInfo();
  79         auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
  80         AtomicSizeInBits = C.toBits(
  81             C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
  82                 .alignTo(lvalue.getAlignment()));
  83         llvm::Value *BitFieldPtr = lvalue.getBitFieldPointer();
  84         auto OffsetInChars =
  85             (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
  86             lvalue.getAlignment();
  87         llvm::Value *StoragePtr = CGF.Builder.CreateConstGEP1_64(
  88             CGF.Int8Ty, BitFieldPtr, OffsetInChars.getQuantity());
  89         StoragePtr = CGF.Builder.CreateAddrSpaceCast(
  90             StoragePtr, llvm::PointerType::getUnqual(CGF.getLLVMContext()),
  91             "atomic_bitfield_base");
  92         BFI = OrigBFI;
  93         BFI.Offset = Offset;
  94         BFI.StorageSize = AtomicSizeInBits;
  95         BFI.StorageOffset += OffsetInChars;
  96         llvm::Type *StorageTy = CGF.Builder.getIntNTy(AtomicSizeInBits);
  97         LVal = LValue::MakeBitfield(
  98             Address(StoragePtr, StorageTy, lvalue.getAlignment()), BFI,
  99             lvalue.getType(), lvalue.getBaseInfo(), lvalue.getTBAAInfo());
 100         AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
 101         if (AtomicTy.isNull()) {
 102           llvm::APInt Size(
 103               /*numBits=*/32,
 104               C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
 105           AtomicTy =
 106               C.getConstantArrayType(C.CharTy, Size, nullptr, ArrayType::Normal,
 107                                      /*IndexTypeQuals=*/0);
 108         }
 109         AtomicAlign = ValueAlign = lvalue.getAlignment();
 110       } else if (lvalue.isVectorElt()) {
 111         ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType();
 112         ValueSizeInBits = C.getTypeSize(ValueTy);
 113         AtomicTy = lvalue.getType();
 114         AtomicSizeInBits = C.getTypeSize(AtomicTy);
 115         AtomicAlign = ValueAlign = lvalue.getAlignment();
 116         LVal = lvalue;
 117       } else {
 118         assert(lvalue.isExtVectorElt());
 119         ValueTy = lvalue.getType();
 120         ValueSizeInBits = C.getTypeSize(ValueTy);
 121         AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
 122             lvalue.getType(), cast<llvm::FixedVectorType>(
 123                                   lvalue.getExtVectorAddress().getElementType())
 124                                   ->getNumElements());
 125         AtomicSizeInBits = C.getTypeSize(AtomicTy);
 126         AtomicAlign = ValueAlign = lvalue.getAlignment();
 127         LVal = lvalue;
 128       }
 129       UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
 130           AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
 131     }
 132
 133     QualType getAtomicType() const { return AtomicTy; }
 134     QualType getValueType() const { return ValueTy; }
 135     CharUnits getAtomicAlignment() const { return AtomicAlign; }
 136     uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
 137     uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
 138     TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
 139     bool shouldUseLibcall() const { return UseLibcall; }
 140     const LValue &getAtomicLValue() const { return LVal; }
 141     llvm::Value *getAtomicPointer() const {
 142       if (LVal.isSimple())
 143         return LVal.getPointer(CGF);
 144       else if (LVal.isBitField())
 145         return LVal.getBitFieldPointer();
 146       else if (LVal.isVectorElt())
 147         return LVal.getVectorPointer();
 148       assert(LVal.isExtVectorElt());
 149       return LVal.getExtVectorPointer();
 150     }
 151     Address getAtomicAddress() const {
 152       llvm::Type *ElTy;
 153       if (LVal.isSimple())
 154         ElTy = LVal.getAddress(CGF).getElementType();
 155       else if (LVal.isBitField())
 156         ElTy = LVal.getBitFieldAddress().getElementType();
 157       else if (LVal.isVectorElt())
 158         ElTy = LVal.getVectorAddress().getElementType();
 159       else
 160         ElTy = LVal.getExtVectorAddress().getElementType();
 161       return Address(getAtomicPointer(), ElTy, getAtomicAlignment());
 162     }
 163
 164     Address getAtomicAddressAsAtomicIntPointer() const {
 165       return castToAtomicIntPointer(getAtomicAddress());
 166     }
 167
 168     /// Is the atomic size larger than the underlying value type?
 169     ///
 170     /// Note that the absence of padding does not mean that atomic
 171     /// objects are completely interchangeable with non-atomic
 172     /// objects: we might have promoted the alignment of a type
 173     /// without making it bigger.
 174     bool hasPadding() const {
 175       return (ValueSizeInBits != AtomicSizeInBits);
 176     }
 177
 178     bool emitMemSetZeroIfNecessary() const;
 179
 180     llvm::Value *getAtomicSizeValue() const {
 181       CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
 182       return CGF.CGM.getSize(size);
 183     }
 184
 185     /// Cast the given pointer to an integer pointer suitable for atomic
 186     /// operations if the source.
 187     Address castToAtomicIntPointer(Address Addr) const;
 188
 189     /// If Addr is compatible with the iN that will be used for an atomic
 190     /// operation, bitcast it. Otherwise, create a temporary that is suitable
 191     /// and copy the value across.
 192     Address convertToAtomicIntPointer(Address Addr) const;
 193
 194     /// Turn an atomic-layout object into an r-value.
 195     RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
 196                                      SourceLocation loc, bool AsValue) const;
 197
 198     /// Converts a rvalue to integer value.
 199     llvm::Value *convertRValueToInt(RValue RVal) const;
 200
 201     RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
 202                                      AggValueSlot ResultSlot,
 203                                      SourceLocation Loc, bool AsValue) const;
 204
 205     /// Copy an atomic r-value into atomic-layout memory.
 206     void emitCopyIntoMemory(RValue rvalue) const;
 207
 208     /// Project an l-value down to the value field.
 209     LValue projectValue() const {
 210       assert(LVal.isSimple());
 211       Address addr = getAtomicAddress();
 212       if (hasPadding())
 213         addr = CGF.Builder.CreateStructGEP(addr, 0);
 214
 215       return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
 216                               LVal.getBaseInfo(), LVal.getTBAAInfo());
 217     }
 218
 219     /// Emits atomic load.
 220     /// \returns Loaded value.
 221     RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
 222                           bool AsValue, llvm::AtomicOrdering AO,
 223                           bool IsVolatile);
 224
 225     /// Emits atomic compare-and-exchange sequence.
 226     /// \param Expected Expected value.
 227     /// \param Desired Desired value.
 228     /// \param Success Atomic ordering for success operation.
 229     /// \param Failure Atomic ordering for failed operation.
 230     /// \param IsWeak true if atomic operation is weak, false otherwise.
 231     /// \returns Pair of values: previous value from storage (value type) and
 232     /// boolean flag (i1 type) with true if success and false otherwise.
 233     std::pair<RValue, llvm::Value *>
 234     EmitAtomicCompareExchange(RValue Expected, RValue Desired,
 235                               llvm::AtomicOrdering Success =
 236                                   llvm::AtomicOrdering::SequentiallyConsistent,
 237                               llvm::AtomicOrdering Failure =
 238                                   llvm::AtomicOrdering::SequentiallyConsistent,
 239                               bool IsWeak = false);
 240
 241     /// Emits atomic update.
 242     /// \param AO Atomic ordering.
 243     /// \param UpdateOp Update operation for the current lvalue.
 244     void EmitAtomicUpdate(llvm::AtomicOrdering AO,
 245                           const llvm::function_ref<RValue(RValue)> &UpdateOp,
 246                           bool IsVolatile);
 247     /// Emits atomic update.
 248     /// \param AO Atomic ordering.
 249     void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
 250                           bool IsVolatile);
 251
 252     /// Materialize an atomic r-value in atomic-layout memory.
 253     Address materializeRValue(RValue rvalue) const;
 254
 255     /// Creates temp alloca for intermediate operations on atomic value.
 256     Address CreateTempAlloca() const;
 257   private:
 258     bool requiresMemSetZero(llvm::Type *type) const;
 259
 260
 261     /// Emits atomic load as a libcall.
 262     void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
 263                                llvm::AtomicOrdering AO, bool IsVolatile);
 264     /// Emits atomic load as LLVM instruction.
 265     llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
 266     /// Emits atomic compare-and-exchange op as a libcall.
 267     llvm::Value *EmitAtomicCompareExchangeLibcall(
 268         llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
 269         llvm::AtomicOrdering Success =
 270             llvm::AtomicOrdering::SequentiallyConsistent,
 271         llvm::AtomicOrdering Failure =
 272             llvm::AtomicOrdering::SequentiallyConsistent);
 273     /// Emits atomic compare-and-exchange op as LLVM instruction.
 274     std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
 275         llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
 276         llvm::AtomicOrdering Success =
 277             llvm::AtomicOrdering::SequentiallyConsistent,
 278         llvm::AtomicOrdering Failure =
 279             llvm::AtomicOrdering::SequentiallyConsistent,
 280         bool IsWeak = false);
 281     /// Emit atomic update as libcalls.
 282     void
 283     EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
 284                             const llvm::function_ref<RValue(RValue)> &UpdateOp,
 285                             bool IsVolatile);
 286     /// Emit atomic update as LLVM instructions.
 287     void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
 288                             const llvm::function_ref<RValue(RValue)> &UpdateOp,
 289                             bool IsVolatile);
 290     /// Emit atomic update as libcalls.
 291     void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
 292                                  bool IsVolatile);
 293     /// Emit atomic update as LLVM instructions.
 294     void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
 295                             bool IsVolatile);
 296   };
 297 }
 298
 299 Address AtomicInfo::CreateTempAlloca() const {
 300   Address TempAlloca = CGF.CreateMemTemp(
 301       (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
 302                                                                 : AtomicTy,
 303       getAtomicAlignment(),
 304       "atomic-temp");
 305   // Cast to pointer to value type for bitfields.
 306   if (LVal.isBitField())
 307     return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
 308         TempAlloca, getAtomicAddress().getType(),
 309         getAtomicAddress().getElementType());
 310   return TempAlloca;
 311 }
 312
 313 static RValue emitAtomicLibcall(CodeGenFunction &CGF,
 314                                 StringRef fnName,
 315                                 QualType resultType,
 316                                 CallArgList &args) {
 317   const CGFunctionInfo &fnInfo =
 318     CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
 319   llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
 320   llvm::AttrBuilder fnAttrB(CGF.getLLVMContext());
 321   fnAttrB.addAttribute(llvm::Attribute::NoUnwind);
 322   fnAttrB.addAttribute(llvm::Attribute::WillReturn);
 323   llvm::AttributeList fnAttrs = llvm::AttributeList::get(
 324       CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB);
 325
 326   llvm::FunctionCallee fn =
 327       CGF.CGM.CreateRuntimeFunction(fnTy, fnName, fnAttrs);
 328   auto callee = CGCallee::forDirect(fn);
 329   return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);
 330 }
 331
 332 /// Does a store of the given IR type modify the full expected width?
 333 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
 334                            uint64_t expectedSize) {
 335   return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
 336 }
 337
 338 /// Does the atomic type require memsetting to zero before initialization?
 339 ///
 340 /// The IR type is provided as a way of making certain queries faster.
 341 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
 342   // If the atomic type has size padding, we definitely need a memset.
 343   if (hasPadding()) return true;
 344
 345   // Otherwise, do some simple heuristics to try to avoid it:
 346   switch (getEvaluationKind()) {
 347   // For scalars and complexes, check whether the store size of the
 348   // type uses the full size.
 349   case TEK_Scalar:
 350     return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
 351   case TEK_Complex:
 352     return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
 353                            AtomicSizeInBits / 2);
 354
 355   // Padding in structs has an undefined bit pattern.  User beware.
 356   case TEK_Aggregate:
 357     return false;
 358   }
 359   llvm_unreachable("bad evaluation kind");
 360 }
 361
 362 bool AtomicInfo::emitMemSetZeroIfNecessary() const {
 363   assert(LVal.isSimple());
 364   Address addr = LVal.getAddress(CGF);
 365   if (!requiresMemSetZero(addr.getElementType()))
 366     return false;
 367
 368   CGF.Builder.CreateMemSet(
 369       addr.getPointer(), llvm::ConstantInt::get(CGF.Int8Ty, 0),
 370       CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
 371       LVal.getAlignment().getAsAlign());
 372   return true;
 373 }
 374
 375 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
 376                               Address Dest, Address Ptr,
 377                               Address Val1, Address Val2,
 378                               uint64_t Size,
 379                               llvm::AtomicOrdering SuccessOrder,
 380                               llvm::AtomicOrdering FailureOrder,
 381                               llvm::SyncScope::ID Scope) {
 382   // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
 383   llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
 384   llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
 385
 386   llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
 387       Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder,
 388       Scope);
 389   Pair->setVolatile(E->isVolatile());
 390   Pair->setWeak(IsWeak);
 391
 392   // Cmp holds the result of the compare-exchange operation: true on success,
 393   // false on failure.
 394   llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
 395   llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
 396
 397   // This basic block is used to hold the store instruction if the operation
 398   // failed.
 399   llvm::BasicBlock *StoreExpectedBB =
 400       CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
 401
 402   // This basic block is the exit point of the operation, we should end up
 403   // here regardless of whether or not the operation succeeded.
 404   llvm::BasicBlock *ContinueBB =
 405       CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
 406
 407   // Update Expected if Expected isn't equal to Old, otherwise branch to the
 408   // exit point.
 409   CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
 410
 411   CGF.Builder.SetInsertPoint(StoreExpectedBB);
 412   // Update the memory at Expected with Old's value.
 413   CGF.Builder.CreateStore(Old, Val1);
 414   // Finally, branch to the exit point.
 415   CGF.Builder.CreateBr(ContinueBB);
 416
 417   CGF.Builder.SetInsertPoint(ContinueBB);
 418   // Update the memory at Dest with Cmp's value.
 419   CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
 420 }
 421
 422 /// Given an ordering required on success, emit all possible cmpxchg
 423 /// instructions to cope with the provided (but possibly only dynamically known)
 424 /// FailureOrder.
 425 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
 426                                         bool IsWeak, Address Dest, Address Ptr,
 427                                         Address Val1, Address Val2,
 428                                         llvm::Value *FailureOrderVal,
 429                                         uint64_t Size,
 430                                         llvm::AtomicOrdering SuccessOrder,
 431                                         llvm::SyncScope::ID Scope) {
 432   llvm::AtomicOrdering FailureOrder;
 433   if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
 434     auto FOS = FO->getSExtValue();
 435     if (!llvm::isValidAtomicOrderingCABI(FOS))
 436       FailureOrder = llvm::AtomicOrdering::Monotonic;
 437     else
 438       switch ((llvm::AtomicOrderingCABI)FOS) {
 439       case llvm::AtomicOrderingCABI::relaxed:
 440       // 31.7.2.18: "The failure argument shall not be memory_order_release
 441       // nor memory_order_acq_rel". Fallback to monotonic.
 442       case llvm::AtomicOrderingCABI::release:
 443       case llvm::AtomicOrderingCABI::acq_rel:
 444         FailureOrder = llvm::AtomicOrdering::Monotonic;
 445         break;
 446       case llvm::AtomicOrderingCABI::consume:
 447       case llvm::AtomicOrderingCABI::acquire:
 448         FailureOrder = llvm::AtomicOrdering::Acquire;
 449         break;
 450       case llvm::AtomicOrderingCABI::seq_cst:
 451         FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
 452         break;
 453       }
 454     // Prior to c++17, "the failure argument shall be no stronger than the
 455     // success argument". This condition has been lifted and the only
 456     // precondition is 31.7.2.18. Effectively treat this as a DR and skip
 457     // language version checks.
 458     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
 459                       FailureOrder, Scope);
 460     return;
 461   }
 462
 463   // Create all the relevant BB's
 464   auto *MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
 465   auto *AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
 466   auto *SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
 467   auto *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
 468
 469   // MonotonicBB is arbitrarily chosen as the default case; in practice, this
 470   // doesn't matter unless someone is crazy enough to use something that
 471   // doesn't fold to a constant for the ordering.
 472   llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
 473   // Implemented as acquire, since it's the closest in LLVM.
 474   SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
 475               AcquireBB);
 476   SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
 477               AcquireBB);
 478   SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
 479               SeqCstBB);
 480
 481   // Emit all the different atomics
 482   CGF.Builder.SetInsertPoint(MonotonicBB);
 483   emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
 484                     Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);
 485   CGF.Builder.CreateBr(ContBB);
 486
 487   CGF.Builder.SetInsertPoint(AcquireBB);
 488   emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
 489                     llvm::AtomicOrdering::Acquire, Scope);
 490   CGF.Builder.CreateBr(ContBB);
 491
 492   CGF.Builder.SetInsertPoint(SeqCstBB);
 493   emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
 494                     llvm::AtomicOrdering::SequentiallyConsistent, Scope);
 495   CGF.Builder.CreateBr(ContBB);
 496
 497   CGF.Builder.SetInsertPoint(ContBB);
 498 }
 499
 500 /// Duplicate the atomic min/max operation in conventional IR for the builtin
 501 /// variants that return the new rather than the original value.
 502 static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder,
 503                                          AtomicExpr::AtomicOp Op,
 504                                          bool IsSigned,
 505                                          llvm::Value *OldVal,
 506                                          llvm::Value *RHS) {
 507   llvm::CmpInst::Predicate Pred;
 508   switch (Op) {
 509   default:
 510     llvm_unreachable("Unexpected min/max operation");
 511   case AtomicExpr::AO__atomic_max_fetch:
 512     Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT;
 513     break;
 514   case AtomicExpr::AO__atomic_min_fetch:
 515     Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT;
 516     break;
 517   }
 518   llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst");
 519   return Builder.CreateSelect(Cmp, OldVal, RHS, "newval");
 520 }
 521
 522 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
 523                          Address Ptr, Address Val1, Address Val2,
 524                          llvm::Value *IsWeak, llvm::Value *FailureOrder,
 525                          uint64_t Size, llvm::AtomicOrdering Order,
 526                          llvm::SyncScope::ID Scope) {
 527   llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
 528   bool PostOpMinMax = false;
 529   unsigned PostOp = 0;
 530
 531   switch (E->getOp()) {
 532   case AtomicExpr::AO__c11_atomic_init:
 533   case AtomicExpr::AO__opencl_atomic_init:
 534     llvm_unreachable("Already handled!");
 535
 536   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
 537   case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
 538   case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
 539     emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
 540                                 FailureOrder, Size, Order, Scope);
 541     return;
 542   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
 543   case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
 544   case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
 545     emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
 546                                 FailureOrder, Size, Order, Scope);
 547     return;
 548   case AtomicExpr::AO__atomic_compare_exchange:
 549   case AtomicExpr::AO__atomic_compare_exchange_n: {
 550     if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
 551       emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
 552                                   Val1, Val2, FailureOrder, Size, Order, Scope);
 553     } else {
 554       // Create all the relevant BB's
 555       llvm::BasicBlock *StrongBB =
 556           CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
 557       llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
 558       llvm::BasicBlock *ContBB =
 559           CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
 560
 561       llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
 562       SI->addCase(CGF.Builder.getInt1(false), StrongBB);
 563
 564       CGF.Builder.SetInsertPoint(StrongBB);
 565       emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
 566                                   FailureOrder, Size, Order, Scope);
 567       CGF.Builder.CreateBr(ContBB);
 568
 569       CGF.Builder.SetInsertPoint(WeakBB);
 570       emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
 571                                   FailureOrder, Size, Order, Scope);
 572       CGF.Builder.CreateBr(ContBB);
 573
 574       CGF.Builder.SetInsertPoint(ContBB);
 575     }
 576     return;
 577   }
 578   case AtomicExpr::AO__c11_atomic_load:
 579   case AtomicExpr::AO__opencl_atomic_load:
 580   case AtomicExpr::AO__hip_atomic_load:
 581   case AtomicExpr::AO__atomic_load_n:
 582   case AtomicExpr::AO__atomic_load: {
 583     llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
 584     Load->setAtomic(Order, Scope);
 585     Load->setVolatile(E->isVolatile());
 586     CGF.Builder.CreateStore(Load, Dest);
 587     return;
 588   }
 589
 590   case AtomicExpr::AO__c11_atomic_store:
 591   case AtomicExpr::AO__opencl_atomic_store:
 592   case AtomicExpr::AO__hip_atomic_store:
 593   case AtomicExpr::AO__atomic_store:
 594   case AtomicExpr::AO__atomic_store_n: {
 595     llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
 596     llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
 597     Store->setAtomic(Order, Scope);
 598     Store->setVolatile(E->isVolatile());
 599     return;
 600   }
 601
 602   case AtomicExpr::AO__c11_atomic_exchange:
 603   case AtomicExpr::AO__hip_atomic_exchange:
 604   case AtomicExpr::AO__opencl_atomic_exchange:
 605   case AtomicExpr::AO__atomic_exchange_n:
 606   case AtomicExpr::AO__atomic_exchange:
 607     Op = llvm::AtomicRMWInst::Xchg;
 608     break;
 609
 610   case AtomicExpr::AO__atomic_add_fetch:
 611     PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd
 612                                                  : llvm::Instruction::Add;
 613     [[fallthrough]];
 614   case AtomicExpr::AO__c11_atomic_fetch_add:
 615   case AtomicExpr::AO__hip_atomic_fetch_add:
 616   case AtomicExpr::AO__opencl_atomic_fetch_add:
 617   case AtomicExpr::AO__atomic_fetch_add:
 618     Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd
 619                                              : llvm::AtomicRMWInst::Add;
 620     break;
 621
 622   case AtomicExpr::AO__atomic_sub_fetch:
 623     PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub
 624                                                  : llvm::Instruction::Sub;
 625     [[fallthrough]];
 626   case AtomicExpr::AO__c11_atomic_fetch_sub:
 627   case AtomicExpr::AO__hip_atomic_fetch_sub:
 628   case AtomicExpr::AO__opencl_atomic_fetch_sub:
 629   case AtomicExpr::AO__atomic_fetch_sub:
 630     Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub
 631                                              : llvm::AtomicRMWInst::Sub;
 632     break;
 633
 634   case AtomicExpr::AO__atomic_min_fetch:
 635     PostOpMinMax = true;
 636     [[fallthrough]];
 637   case AtomicExpr::AO__c11_atomic_fetch_min:
 638   case AtomicExpr::AO__hip_atomic_fetch_min:
 639   case AtomicExpr::AO__opencl_atomic_fetch_min:
 640   case AtomicExpr::AO__atomic_fetch_min:
 641     Op = E->getValueType()->isFloatingType()
 642              ? llvm::AtomicRMWInst::FMin
 643              : (E->getValueType()->isSignedIntegerType()
 644                     ? llvm::AtomicRMWInst::Min
 645                     : llvm::AtomicRMWInst::UMin);
 646     break;
 647
 648   case AtomicExpr::AO__atomic_max_fetch:
 649     PostOpMinMax = true;
 650     [[fallthrough]];
 651   case AtomicExpr::AO__c11_atomic_fetch_max:
 652   case AtomicExpr::AO__hip_atomic_fetch_max:
 653   case AtomicExpr::AO__opencl_atomic_fetch_max:
 654   case AtomicExpr::AO__atomic_fetch_max:
 655     Op = E->getValueType()->isFloatingType()
 656              ? llvm::AtomicRMWInst::FMax
 657              : (E->getValueType()->isSignedIntegerType()
 658                     ? llvm::AtomicRMWInst::Max
 659                     : llvm::AtomicRMWInst::UMax);
 660     break;
 661
 662   case AtomicExpr::AO__atomic_and_fetch:
 663     PostOp = llvm::Instruction::And;
 664     [[fallthrough]];
 665   case AtomicExpr::AO__c11_atomic_fetch_and:
 666   case AtomicExpr::AO__hip_atomic_fetch_and:
 667   case AtomicExpr::AO__opencl_atomic_fetch_and:
 668   case AtomicExpr::AO__atomic_fetch_and:
 669     Op = llvm::AtomicRMWInst::And;
 670     break;
 671
 672   case AtomicExpr::AO__atomic_or_fetch:
 673     PostOp = llvm::Instruction::Or;
 674     [[fallthrough]];
 675   case AtomicExpr::AO__c11_atomic_fetch_or:
 676   case AtomicExpr::AO__hip_atomic_fetch_or:
 677   case AtomicExpr::AO__opencl_atomic_fetch_or:
 678   case AtomicExpr::AO__atomic_fetch_or:
 679     Op = llvm::AtomicRMWInst::Or;
 680     break;
 681
 682   case AtomicExpr::AO__atomic_xor_fetch:
 683     PostOp = llvm::Instruction::Xor;
 684     [[fallthrough]];
 685   case AtomicExpr::AO__c11_atomic_fetch_xor:
 686   case AtomicExpr::AO__hip_atomic_fetch_xor:
 687   case AtomicExpr::AO__opencl_atomic_fetch_xor:
 688   case AtomicExpr::AO__atomic_fetch_xor:
 689     Op = llvm::AtomicRMWInst::Xor;
 690     break;
 691
 692   case AtomicExpr::AO__atomic_nand_fetch:
 693     PostOp = llvm::Instruction::And; // the NOT is special cased below
 694     [[fallthrough]];
 695   case AtomicExpr::AO__c11_atomic_fetch_nand:
 696   case AtomicExpr::AO__atomic_fetch_nand:
 697     Op = llvm::AtomicRMWInst::Nand;
 698     break;
 699   }
 700
 701   llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
 702   llvm::AtomicRMWInst *RMWI =
 703       CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope);
 704   RMWI->setVolatile(E->isVolatile());
 705
 706   // For __atomic_*_fetch operations, perform the operation again to
 707   // determine the value which was written.
 708   llvm::Value *Result = RMWI;
 709   if (PostOpMinMax)
 710     Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(),
 711                                   E->getValueType()->isSignedIntegerType(),
 712                                   RMWI, LoadVal1);
 713   else if (PostOp)
 714     Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI,
 715                                      LoadVal1);
 716   if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
 717     Result = CGF.Builder.CreateNot(Result);
 718   CGF.Builder.CreateStore(Result, Dest);
 719 }
 720
 721 // This function emits any expression (scalar, complex, or aggregate)
 722 // into a temporary alloca.
 723 static Address
 724 EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
 725   Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
 726   CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
 727                        /*Init*/ true);
 728   return DeclPtr;
 729 }
 730
 731 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
 732                          Address Ptr, Address Val1, Address Val2,
 733                          llvm::Value *IsWeak, llvm::Value *FailureOrder,
 734                          uint64_t Size, llvm::AtomicOrdering Order,
 735                          llvm::Value *Scope) {
 736   auto ScopeModel = Expr->getScopeModel();
 737
 738   // LLVM atomic instructions always have synch scope. If clang atomic
 739   // expression has no scope operand, use default LLVM synch scope.
 740   if (!ScopeModel) {
 741     EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
 742                  Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID(""));
 743     return;
 744   }
 745
 746   // Handle constant scope.
 747   if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) {
 748     auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
 749         CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()),
 750         Order, CGF.CGM.getLLVMContext());
 751     EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
 752                  Order, SCID);
 753     return;
 754   }
 755
 756   // Handle non-constant scope.
 757   auto &Builder = CGF.Builder;
 758   auto Scopes = ScopeModel->getRuntimeValues();
 759   llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
 760   for (auto S : Scopes)
 761     BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);
 762
 763   llvm::BasicBlock *ContBB =
 764       CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn);
 765
 766   auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false);
 767   // If unsupported synch scope is encountered at run time, assume a fallback
 768   // synch scope value.
 769   auto FallBack = ScopeModel->getFallBackValue();
 770   llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);
 771   for (auto S : Scopes) {
 772     auto *B = BB[S];
 773     if (S != FallBack)
 774       SI->addCase(Builder.getInt32(S), B);
 775
 776     Builder.SetInsertPoint(B);
 777     EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
 778                  Order,
 779                  CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(),
 780                                                          ScopeModel->map(S),
 781                                                          Order,
 782                                                          CGF.getLLVMContext()));
 783     Builder.CreateBr(ContBB);
 784   }
 785
 786   Builder.SetInsertPoint(ContBB);
 787 }
 788
 789 static void
 790 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
 791                   bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
 792                   SourceLocation Loc, CharUnits SizeInChars) {
 793   if (UseOptimizedLibcall) {
 794     // Load value and pass it to the function directly.
 795     CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
 796     int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
 797     ValTy =
 798         CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
 799     llvm::Type *ITy = llvm::IntegerType::get(CGF.getLLVMContext(), SizeInBits);
 800     Address Ptr = Address(Val, ITy, Align);
 801     Val = CGF.EmitLoadOfScalar(Ptr, false,
 802                                CGF.getContext().getPointerType(ValTy),
 803                                Loc);
 804     // Coerce the value into an appropriately sized integer type.
 805     Args.add(RValue::get(Val), ValTy);
 806   } else {
 807     // Non-optimized functions always take a reference.
 808     Args.add(RValue::get(Val), CGF.getContext().VoidPtrTy);
 809   }
 810 }
 811
 812 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
 813   QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
 814   QualType MemTy = AtomicTy;
 815   if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
 816     MemTy = AT->getValueType();
 817   llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
 818
 819   Address Val1 = Address::invalid();
 820   Address Val2 = Address::invalid();
 821   Address Dest = Address::invalid();
 822   Address Ptr = EmitPointerWithAlignment(E->getPtr());
 823
 824   if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
 825       E->getOp() == AtomicExpr::AO__opencl_atomic_init) {
 826     LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
 827     EmitAtomicInit(E->getVal1(), lvalue);
 828     return RValue::get(nullptr);
 829   }
 830
 831   auto TInfo = getContext().getTypeInfoInChars(AtomicTy);
 832   uint64_t Size = TInfo.Width.getQuantity();
 833   unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
 834
 835   bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits;
 836   bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0;
 837   bool UseLibcall = Misaligned | Oversized;
 838   bool ShouldCastToIntPtrTy = true;
 839
 840   CharUnits MaxInlineWidth =
 841       getContext().toCharUnitsFromBits(MaxInlineWidthInBits);
 842
 843   DiagnosticsEngine &Diags = CGM.getDiags();
 844
 845   if (Misaligned) {
 846     Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned)
 847         << (int)TInfo.Width.getQuantity()
 848         << (int)Ptr.getAlignment().getQuantity();
 849   }
 850
 851   if (Oversized) {
 852     Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized)
 853         << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity();
 854   }
 855
 856   llvm::Value *Order = EmitScalarExpr(E->getOrder());
 857   llvm::Value *Scope =
 858       E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr;
 859
 860   switch (E->getOp()) {
 861   case AtomicExpr::AO__c11_atomic_init:
 862   case AtomicExpr::AO__opencl_atomic_init:
 863     llvm_unreachable("Already handled above with EmitAtomicInit!");
 864
 865   case AtomicExpr::AO__c11_atomic_load:
 866   case AtomicExpr::AO__opencl_atomic_load:
 867   case AtomicExpr::AO__hip_atomic_load:
 868   case AtomicExpr::AO__atomic_load_n:
 869     break;
 870
 871   case AtomicExpr::AO__atomic_load:
 872     Dest = EmitPointerWithAlignment(E->getVal1());
 873     break;
 874
 875   case AtomicExpr::AO__atomic_store:
 876     Val1 = EmitPointerWithAlignment(E->getVal1());
 877     break;
 878
 879   case AtomicExpr::AO__atomic_exchange:
 880     Val1 = EmitPointerWithAlignment(E->getVal1());
 881     Dest = EmitPointerWithAlignment(E->getVal2());
 882     break;
 883
 884   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
 885   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
 886   case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
 887   case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
 888   case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
 889   case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
 890   case AtomicExpr::AO__atomic_compare_exchange_n:
 891   case AtomicExpr::AO__atomic_compare_exchange:
 892     Val1 = EmitPointerWithAlignment(E->getVal1());
 893     if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
 894       Val2 = EmitPointerWithAlignment(E->getVal2());
 895     else
 896       Val2 = EmitValToTemp(*this, E->getVal2());
 897     OrderFail = EmitScalarExpr(E->getOrderFail());
 898     if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
 899         E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
 900       IsWeak = EmitScalarExpr(E->getWeak());
 901     break;
 902
 903   case AtomicExpr::AO__c11_atomic_fetch_add:
 904   case AtomicExpr::AO__c11_atomic_fetch_sub:
 905   case AtomicExpr::AO__hip_atomic_fetch_add:
 906   case AtomicExpr::AO__hip_atomic_fetch_sub:
 907   case AtomicExpr::AO__opencl_atomic_fetch_add:
 908   case AtomicExpr::AO__opencl_atomic_fetch_sub:
 909     if (MemTy->isPointerType()) {
 910       // For pointer arithmetic, we're required to do a bit of math:
 911       // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
 912       // ... but only for the C11 builtins. The GNU builtins expect the
 913       // user to multiply by sizeof(T).
 914       QualType Val1Ty = E->getVal1()->getType();
 915       llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
 916       CharUnits PointeeIncAmt =
 917           getContext().getTypeSizeInChars(MemTy->getPointeeType());
 918       Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
 919       auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
 920       Val1 = Temp;
 921       EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
 922       break;
 923     }
 924     [[fallthrough]];
 925   case AtomicExpr::AO__atomic_fetch_add:
 926   case AtomicExpr::AO__atomic_fetch_max:
 927   case AtomicExpr::AO__atomic_fetch_min:
 928   case AtomicExpr::AO__atomic_fetch_sub:
 929   case AtomicExpr::AO__atomic_add_fetch:
 930   case AtomicExpr::AO__atomic_max_fetch:
 931   case AtomicExpr::AO__atomic_min_fetch:
 932   case AtomicExpr::AO__atomic_sub_fetch:
 933   case AtomicExpr::AO__c11_atomic_fetch_max:
 934   case AtomicExpr::AO__c11_atomic_fetch_min:
 935   case AtomicExpr::AO__opencl_atomic_fetch_max:
 936   case AtomicExpr::AO__opencl_atomic_fetch_min:
 937   case AtomicExpr::AO__hip_atomic_fetch_max:
 938   case AtomicExpr::AO__hip_atomic_fetch_min:
 939     ShouldCastToIntPtrTy = !MemTy->isFloatingType();
 940     [[fallthrough]];
 941
 942   case AtomicExpr::AO__c11_atomic_store:
 943   case AtomicExpr::AO__c11_atomic_exchange:
 944   case AtomicExpr::AO__opencl_atomic_store:
 945   case AtomicExpr::AO__hip_atomic_store:
 946   case AtomicExpr::AO__opencl_atomic_exchange:
 947   case AtomicExpr::AO__hip_atomic_exchange:
 948   case AtomicExpr::AO__atomic_store_n:
 949   case AtomicExpr::AO__atomic_exchange_n:
 950   case AtomicExpr::AO__c11_atomic_fetch_and:
 951   case AtomicExpr::AO__c11_atomic_fetch_or:
 952   case AtomicExpr::AO__c11_atomic_fetch_xor:
 953   case AtomicExpr::AO__c11_atomic_fetch_nand:
 954   case AtomicExpr::AO__opencl_atomic_fetch_and:
 955   case AtomicExpr::AO__opencl_atomic_fetch_or:
 956   case AtomicExpr::AO__opencl_atomic_fetch_xor:
 957   case AtomicExpr::AO__atomic_fetch_and:
 958   case AtomicExpr::AO__hip_atomic_fetch_and:
 959   case AtomicExpr::AO__atomic_fetch_or:
 960   case AtomicExpr::AO__hip_atomic_fetch_or:
 961   case AtomicExpr::AO__atomic_fetch_xor:
 962   case AtomicExpr::AO__hip_atomic_fetch_xor:
 963   case AtomicExpr::AO__atomic_fetch_nand:
 964   case AtomicExpr::AO__atomic_and_fetch:
 965   case AtomicExpr::AO__atomic_or_fetch:
 966   case AtomicExpr::AO__atomic_xor_fetch:
 967   case AtomicExpr::AO__atomic_nand_fetch:
 968     Val1 = EmitValToTemp(*this, E->getVal1());
 969     break;
 970   }
 971
 972   QualType RValTy = E->getType().getUnqualifiedType();
 973
 974   // The inlined atomics only function on iN types, where N is a power of 2. We
 975   // need to make sure (via temporaries if necessary) that all incoming values
 976   // are compatible.
 977   LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
 978   AtomicInfo Atomics(*this, AtomicVal);
 979
 980   if (ShouldCastToIntPtrTy) {
 981     Ptr = Atomics.castToAtomicIntPointer(Ptr);
 982     if (Val1.isValid())
 983       Val1 = Atomics.convertToAtomicIntPointer(Val1);
 984     if (Val2.isValid())
 985       Val2 = Atomics.convertToAtomicIntPointer(Val2);
 986   }
 987   if (Dest.isValid()) {
 988     if (ShouldCastToIntPtrTy)
 989       Dest = Atomics.castToAtomicIntPointer(Dest);
 990   } else if (E->isCmpXChg())
 991     Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
 992   else if (!RValTy->isVoidType()) {
 993     Dest = Atomics.CreateTempAlloca();
 994     if (ShouldCastToIntPtrTy)
 995       Dest = Atomics.castToAtomicIntPointer(Dest);
 996   }
 997
 998   // Use a library call.  See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
 999   if (UseLibcall) {
1000     bool UseOptimizedLibcall = false;
1001     switch (E->getOp()) {
1002     case AtomicExpr::AO__c11_atomic_init:
1003     case AtomicExpr::AO__opencl_atomic_init:
1004       llvm_unreachable("Already handled above with EmitAtomicInit!");
1005
1006     case AtomicExpr::AO__c11_atomic_fetch_add:
1007     case AtomicExpr::AO__opencl_atomic_fetch_add:
1008     case AtomicExpr::AO__atomic_fetch_add:
1009     case AtomicExpr::AO__hip_atomic_fetch_add:
1010     case AtomicExpr::AO__c11_atomic_fetch_and:
1011     case AtomicExpr::AO__opencl_atomic_fetch_and:
1012     case AtomicExpr::AO__hip_atomic_fetch_and:
1013     case AtomicExpr::AO__atomic_fetch_and:
1014     case AtomicExpr::AO__c11_atomic_fetch_or:
1015     case AtomicExpr::AO__opencl_atomic_fetch_or:
1016     case AtomicExpr::AO__hip_atomic_fetch_or:
1017     case AtomicExpr::AO__atomic_fetch_or:
1018     case AtomicExpr::AO__c11_atomic_fetch_nand:
1019     case AtomicExpr::AO__atomic_fetch_nand:
1020     case AtomicExpr::AO__c11_atomic_fetch_sub:
1021     case AtomicExpr::AO__opencl_atomic_fetch_sub:
1022     case AtomicExpr::AO__atomic_fetch_sub:
1023     case AtomicExpr::AO__hip_atomic_fetch_sub:
1024     case AtomicExpr::AO__c11_atomic_fetch_xor:
1025     case AtomicExpr::AO__opencl_atomic_fetch_xor:
1026     case AtomicExpr::AO__opencl_atomic_fetch_min:
1027     case AtomicExpr::AO__opencl_atomic_fetch_max:
1028     case AtomicExpr::AO__atomic_fetch_xor:
1029     case AtomicExpr::AO__hip_atomic_fetch_xor:
1030     case AtomicExpr::AO__c11_atomic_fetch_max:
1031     case AtomicExpr::AO__c11_atomic_fetch_min:
1032     case AtomicExpr::AO__atomic_add_fetch:
1033     case AtomicExpr::AO__atomic_and_fetch:
1034     case AtomicExpr::AO__atomic_nand_fetch:
1035     case AtomicExpr::AO__atomic_or_fetch:
1036     case AtomicExpr::AO__atomic_sub_fetch:
1037     case AtomicExpr::AO__atomic_xor_fetch:
1038     case AtomicExpr::AO__atomic_fetch_max:
1039     case AtomicExpr::AO__hip_atomic_fetch_max:
1040     case AtomicExpr::AO__atomic_fetch_min:
1041     case AtomicExpr::AO__hip_atomic_fetch_min:
1042     case AtomicExpr::AO__atomic_max_fetch:
1043     case AtomicExpr::AO__atomic_min_fetch:
1044       // For these, only library calls for certain sizes exist.
1045       UseOptimizedLibcall = true;
1046       break;
1047
1048     case AtomicExpr::AO__atomic_load:
1049     case AtomicExpr::AO__atomic_store:
1050     case AtomicExpr::AO__atomic_exchange:
1051     case AtomicExpr::AO__atomic_compare_exchange:
1052       // Use the generic version if we don't know that the operand will be
1053       // suitably aligned for the optimized version.
1054       if (Misaligned)
1055         break;
1056       [[fallthrough]];
1057     case AtomicExpr::AO__c11_atomic_load:
1058     case AtomicExpr::AO__c11_atomic_store:
1059     case AtomicExpr::AO__c11_atomic_exchange:
1060     case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1061     case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1062     case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1063     case AtomicExpr::AO__opencl_atomic_load:
1064     case AtomicExpr::AO__hip_atomic_load:
1065     case AtomicExpr::AO__opencl_atomic_store:
1066     case AtomicExpr::AO__hip_atomic_store:
1067     case AtomicExpr::AO__opencl_atomic_exchange:
1068     case AtomicExpr::AO__hip_atomic_exchange:
1069     case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1070     case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1071     case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1072     case AtomicExpr::AO__atomic_load_n:
1073     case AtomicExpr::AO__atomic_store_n:
1074     case AtomicExpr::AO__atomic_exchange_n:
1075     case AtomicExpr::AO__atomic_compare_exchange_n:
1076       // Only use optimized library calls for sizes for which they exist.
1077       // FIXME: Size == 16 optimized library functions exist too.
1078       if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
1079         UseOptimizedLibcall = true;
1080       break;
1081     }
1082
1083     CallArgList Args;
1084     if (!UseOptimizedLibcall) {
1085       // For non-optimized library calls, the size is the first parameter
1086       Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
1087                getContext().getSizeType());
1088     }
1089     // Atomic address is the first or second parameter
1090     // The OpenCL atomic library functions only accept pointer arguments to
1091     // generic address space.
1092     auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
1093       if (!E->isOpenCL())
1094         return V;
1095       auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace();
1096       if (AS == LangAS::opencl_generic)
1097         return V;
1098       auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic);
1099       auto *DestType = llvm::PointerType::get(getLLVMContext(), DestAS);
1100
1101       return getTargetHooks().performAddrSpaceCast(
1102           *this, V, AS, LangAS::opencl_generic, DestType, false);
1103     };
1104
1105     Args.add(RValue::get(CastToGenericAddrSpace(Ptr.getPointer(),
1106                                                 E->getPtr()->getType())),
1107              getContext().VoidPtrTy);
1108
1109     std::string LibCallName;
1110     QualType LoweredMemTy =
1111       MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
1112     QualType RetTy;
1113     bool HaveRetTy = false;
1114     llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
1115     bool PostOpMinMax = false;
1116     switch (E->getOp()) {
1117     case AtomicExpr::AO__c11_atomic_init:
1118     case AtomicExpr::AO__opencl_atomic_init:
1119       llvm_unreachable("Already handled!");
1120
1121     // There is only one libcall for compare an exchange, because there is no
1122     // optimisation benefit possible from a libcall version of a weak compare
1123     // and exchange.
1124     // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
1125     //                                void *desired, int success, int failure)
1126     // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
1127     //                                  int success, int failure)
1128     case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1129     case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1130     case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1131     case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1132     case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1133     case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1134     case AtomicExpr::AO__atomic_compare_exchange:
1135     case AtomicExpr::AO__atomic_compare_exchange_n:
1136       LibCallName = "__atomic_compare_exchange";
1137       RetTy = getContext().BoolTy;
1138       HaveRetTy = true;
1139       Args.add(RValue::get(CastToGenericAddrSpace(Val1.getPointer(),
1140                                                   E->getVal1()->getType())),
1141                getContext().VoidPtrTy);
1142       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
1143                         MemTy, E->getExprLoc(), TInfo.Width);
1144       Args.add(RValue::get(Order), getContext().IntTy);
1145       Order = OrderFail;
1146       break;
1147     // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1148     //                        int order)
1149     // T __atomic_exchange_N(T *mem, T val, int order)
1150     case AtomicExpr::AO__c11_atomic_exchange:
1151     case AtomicExpr::AO__opencl_atomic_exchange:
1152     case AtomicExpr::AO__atomic_exchange_n:
1153     case AtomicExpr::AO__atomic_exchange:
1154     case AtomicExpr::AO__hip_atomic_exchange:
1155       LibCallName = "__atomic_exchange";
1156       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1157                         MemTy, E->getExprLoc(), TInfo.Width);
1158       break;
1159     // void __atomic_store(size_t size, void *mem, void *val, int order)
1160     // void __atomic_store_N(T *mem, T val, int order)
1161     case AtomicExpr::AO__c11_atomic_store:
1162     case AtomicExpr::AO__opencl_atomic_store:
1163     case AtomicExpr::AO__hip_atomic_store:
1164     case AtomicExpr::AO__atomic_store:
1165     case AtomicExpr::AO__atomic_store_n:
1166       LibCallName = "__atomic_store";
1167       RetTy = getContext().VoidTy;
1168       HaveRetTy = true;
1169       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1170                         MemTy, E->getExprLoc(), TInfo.Width);
1171       break;
1172     // void __atomic_load(size_t size, void *mem, void *return, int order)
1173     // T __atomic_load_N(T *mem, int order)
1174     case AtomicExpr::AO__c11_atomic_load:
1175     case AtomicExpr::AO__opencl_atomic_load:
1176     case AtomicExpr::AO__hip_atomic_load:
1177     case AtomicExpr::AO__atomic_load:
1178     case AtomicExpr::AO__atomic_load_n:
1179       LibCallName = "__atomic_load";
1180       break;
1181     // T __atomic_add_fetch_N(T *mem, T val, int order)
1182     // T __atomic_fetch_add_N(T *mem, T val, int order)
1183     case AtomicExpr::AO__atomic_add_fetch:
1184       PostOp = llvm::Instruction::Add;
1185       [[fallthrough]];
1186     case AtomicExpr::AO__c11_atomic_fetch_add:
1187     case AtomicExpr::AO__opencl_atomic_fetch_add:
1188     case AtomicExpr::AO__atomic_fetch_add:
1189     case AtomicExpr::AO__hip_atomic_fetch_add:
1190       LibCallName = "__atomic_fetch_add";
1191       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1192                         LoweredMemTy, E->getExprLoc(), TInfo.Width);
1193       break;
1194     // T __atomic_and_fetch_N(T *mem, T val, int order)
1195     // T __atomic_fetch_and_N(T *mem, T val, int order)
1196     case AtomicExpr::AO__atomic_and_fetch:
1197       PostOp = llvm::Instruction::And;
1198       [[fallthrough]];
1199     case AtomicExpr::AO__c11_atomic_fetch_and:
1200     case AtomicExpr::AO__opencl_atomic_fetch_and:
1201     case AtomicExpr::AO__hip_atomic_fetch_and:
1202     case AtomicExpr::AO__atomic_fetch_and:
1203       LibCallName = "__atomic_fetch_and";
1204       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1205                         MemTy, E->getExprLoc(), TInfo.Width);
1206       break;
1207     // T __atomic_or_fetch_N(T *mem, T val, int order)
1208     // T __atomic_fetch_or_N(T *mem, T val, int order)
1209     case AtomicExpr::AO__atomic_or_fetch:
1210       PostOp = llvm::Instruction::Or;
1211       [[fallthrough]];
1212     case AtomicExpr::AO__c11_atomic_fetch_or:
1213     case AtomicExpr::AO__opencl_atomic_fetch_or:
1214     case AtomicExpr::AO__hip_atomic_fetch_or:
1215     case AtomicExpr::AO__atomic_fetch_or:
1216       LibCallName = "__atomic_fetch_or";
1217       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1218                         MemTy, E->getExprLoc(), TInfo.Width);
1219       break;
1220     // T __atomic_sub_fetch_N(T *mem, T val, int order)
1221     // T __atomic_fetch_sub_N(T *mem, T val, int order)
1222     case AtomicExpr::AO__atomic_sub_fetch:
1223       PostOp = llvm::Instruction::Sub;
1224       [[fallthrough]];
1225     case AtomicExpr::AO__c11_atomic_fetch_sub:
1226     case AtomicExpr::AO__opencl_atomic_fetch_sub:
1227     case AtomicExpr::AO__hip_atomic_fetch_sub:
1228     case AtomicExpr::AO__atomic_fetch_sub:
1229       LibCallName = "__atomic_fetch_sub";
1230       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1231                         LoweredMemTy, E->getExprLoc(), TInfo.Width);
1232       break;
1233     // T __atomic_xor_fetch_N(T *mem, T val, int order)
1234     // T __atomic_fetch_xor_N(T *mem, T val, int order)
1235     case AtomicExpr::AO__atomic_xor_fetch:
1236       PostOp = llvm::Instruction::Xor;
1237       [[fallthrough]];
1238     case AtomicExpr::AO__c11_atomic_fetch_xor:
1239     case AtomicExpr::AO__opencl_atomic_fetch_xor:
1240     case AtomicExpr::AO__hip_atomic_fetch_xor:
1241     case AtomicExpr::AO__atomic_fetch_xor:
1242       LibCallName = "__atomic_fetch_xor";
1243       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1244                         MemTy, E->getExprLoc(), TInfo.Width);
1245       break;
1246     case AtomicExpr::AO__atomic_min_fetch:
1247       PostOpMinMax = true;
1248       [[fallthrough]];
1249     case AtomicExpr::AO__c11_atomic_fetch_min:
1250     case AtomicExpr::AO__atomic_fetch_min:
1251     case AtomicExpr::AO__hip_atomic_fetch_min:
1252     case AtomicExpr::AO__opencl_atomic_fetch_min:
1253       LibCallName = E->getValueType()->isSignedIntegerType()
1254                         ? "__atomic_fetch_min"
1255                         : "__atomic_fetch_umin";
1256       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1257                         LoweredMemTy, E->getExprLoc(), TInfo.Width);
1258       break;
1259     case AtomicExpr::AO__atomic_max_fetch:
1260       PostOpMinMax = true;
1261       [[fallthrough]];
1262     case AtomicExpr::AO__c11_atomic_fetch_max:
1263     case AtomicExpr::AO__atomic_fetch_max:
1264     case AtomicExpr::AO__hip_atomic_fetch_max:
1265     case AtomicExpr::AO__opencl_atomic_fetch_max:
1266       LibCallName = E->getValueType()->isSignedIntegerType()
1267                         ? "__atomic_fetch_max"
1268                         : "__atomic_fetch_umax";
1269       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1270                         LoweredMemTy, E->getExprLoc(), TInfo.Width);
1271       break;
1272     // T __atomic_nand_fetch_N(T *mem, T val, int order)
1273     // T __atomic_fetch_nand_N(T *mem, T val, int order)
1274     case AtomicExpr::AO__atomic_nand_fetch:
1275       PostOp = llvm::Instruction::And; // the NOT is special cased below
1276       [[fallthrough]];
1277     case AtomicExpr::AO__c11_atomic_fetch_nand:
1278     case AtomicExpr::AO__atomic_fetch_nand:
1279       LibCallName = "__atomic_fetch_nand";
1280       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1281                         MemTy, E->getExprLoc(), TInfo.Width);
1282       break;
1283     }
1284
1285     if (E->isOpenCL()) {
1286       LibCallName = std::string("__opencl") +
1287           StringRef(LibCallName).drop_front(1).str();
1288
1289     }
1290     // Optimized functions have the size in their name.
1291     if (UseOptimizedLibcall)
1292       LibCallName += "_" + llvm::utostr(Size);
1293     // By default, assume we return a value of the atomic type.
1294     if (!HaveRetTy) {
1295       if (UseOptimizedLibcall) {
1296         // Value is returned directly.
1297         // The function returns an appropriately sized integer type.
1298         RetTy = getContext().getIntTypeForBitwidth(
1299             getContext().toBits(TInfo.Width), /*Signed=*/false);
1300       } else {
1301         // Value is returned through parameter before the order.
1302         RetTy = getContext().VoidTy;
1303         Args.add(RValue::get(Dest.getPointer()), getContext().VoidPtrTy);
1304       }
1305     }
1306     // order is always the last parameter
1307     Args.add(RValue::get(Order),
1308              getContext().IntTy);
1309     if (E->isOpenCL())
1310       Args.add(RValue::get(Scope), getContext().IntTy);
1311
1312     // PostOp is only needed for the atomic_*_fetch operations, and
1313     // thus is only needed for and implemented in the
1314     // UseOptimizedLibcall codepath.
1315     assert(UseOptimizedLibcall || (!PostOp && !PostOpMinMax));
1316
1317     RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1318     // The value is returned directly from the libcall.
1319     if (E->isCmpXChg())
1320       return Res;
1321
1322     // The value is returned directly for optimized libcalls but the expr
1323     // provided an out-param.
1324     if (UseOptimizedLibcall && Res.getScalarVal()) {
1325       llvm::Value *ResVal = Res.getScalarVal();
1326       if (PostOpMinMax) {
1327         llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1328         ResVal = EmitPostAtomicMinMax(Builder, E->getOp(),
1329                                       E->getValueType()->isSignedIntegerType(),
1330                                       ResVal, LoadVal1);
1331       } else if (PostOp) {
1332         llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1333         ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1334       }
1335       if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1336         ResVal = Builder.CreateNot(ResVal);
1337
1338       Builder.CreateStore(ResVal, Dest.withElementType(ResVal->getType()));
1339     }
1340
1341     if (RValTy->isVoidType())
1342       return RValue::get(nullptr);
1343
1344     return convertTempToRValue(Dest.withElementType(ConvertTypeForMem(RValTy)),
1345                                RValTy, E->getExprLoc());
1346   }
1347
1348   bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1349                  E->getOp() == AtomicExpr::AO__opencl_atomic_store ||
1350                  E->getOp() == AtomicExpr::AO__hip_atomic_store ||
1351                  E->getOp() == AtomicExpr::AO__atomic_store ||
1352                  E->getOp() == AtomicExpr::AO__atomic_store_n;
1353   bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1354                 E->getOp() == AtomicExpr::AO__opencl_atomic_load ||
1355                 E->getOp() == AtomicExpr::AO__hip_atomic_load ||
1356                 E->getOp() == AtomicExpr::AO__atomic_load ||
1357                 E->getOp() == AtomicExpr::AO__atomic_load_n;
1358
1359   if (isa<llvm::ConstantInt>(Order)) {
1360     auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1361     // We should not ever get to a case where the ordering isn't a valid C ABI
1362     // value, but it's hard to enforce that in general.
1363     if (llvm::isValidAtomicOrderingCABI(ord))
1364       switch ((llvm::AtomicOrderingCABI)ord) {
1365       case llvm::AtomicOrderingCABI::relaxed:
1366         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1367                      llvm::AtomicOrdering::Monotonic, Scope);
1368         break;
1369       case llvm::AtomicOrderingCABI::consume:
1370       case llvm::AtomicOrderingCABI::acquire:
1371         if (IsStore)
1372           break; // Avoid crashing on code with undefined behavior
1373         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1374                      llvm::AtomicOrdering::Acquire, Scope);
1375         break;
1376       case llvm::AtomicOrderingCABI::release:
1377         if (IsLoad)
1378           break; // Avoid crashing on code with undefined behavior
1379         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1380                      llvm::AtomicOrdering::Release, Scope);
1381         break;
1382       case llvm::AtomicOrderingCABI::acq_rel:
1383         if (IsLoad || IsStore)
1384           break; // Avoid crashing on code with undefined behavior
1385         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1386                      llvm::AtomicOrdering::AcquireRelease, Scope);
1387         break;
1388       case llvm::AtomicOrderingCABI::seq_cst:
1389         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1390                      llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1391         break;
1392       }
1393     if (RValTy->isVoidType())
1394       return RValue::get(nullptr);
1395
1396     return convertTempToRValue(Dest.withElementType(ConvertTypeForMem(RValTy)),
1397                                RValTy, E->getExprLoc());
1398   }
1399
1400   // Long case, when Order isn't obviously constant.
1401
1402   // Create all the relevant BB's
1403   llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1404                    *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1405                    *SeqCstBB = nullptr;
1406   MonotonicBB = createBasicBlock("monotonic", CurFn);
1407   if (!IsStore)
1408     AcquireBB = createBasicBlock("acquire", CurFn);
1409   if (!IsLoad)
1410     ReleaseBB = createBasicBlock("release", CurFn);
1411   if (!IsLoad && !IsStore)
1412     AcqRelBB = createBasicBlock("acqrel", CurFn);
1413   SeqCstBB = createBasicBlock("seqcst", CurFn);
1414   llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1415
1416   // Create the switch for the split
1417   // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1418   // doesn't matter unless someone is crazy enough to use something that
1419   // doesn't fold to a constant for the ordering.
1420   Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1421   llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1422
1423   // Emit all the different atomics
1424   Builder.SetInsertPoint(MonotonicBB);
1425   EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1426                llvm::AtomicOrdering::Monotonic, Scope);
1427   Builder.CreateBr(ContBB);
1428   if (!IsStore) {
1429     Builder.SetInsertPoint(AcquireBB);
1430     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1431                  llvm::AtomicOrdering::Acquire, Scope);
1432     Builder.CreateBr(ContBB);
1433     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1434                 AcquireBB);
1435     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1436                 AcquireBB);
1437   }
1438   if (!IsLoad) {
1439     Builder.SetInsertPoint(ReleaseBB);
1440     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1441                  llvm::AtomicOrdering::Release, Scope);
1442     Builder.CreateBr(ContBB);
1443     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1444                 ReleaseBB);
1445   }
1446   if (!IsLoad && !IsStore) {
1447     Builder.SetInsertPoint(AcqRelBB);
1448     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1449                  llvm::AtomicOrdering::AcquireRelease, Scope);
1450     Builder.CreateBr(ContBB);
1451     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1452                 AcqRelBB);
1453   }
1454   Builder.SetInsertPoint(SeqCstBB);
1455   EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1456                llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1457   Builder.CreateBr(ContBB);
1458   SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1459               SeqCstBB);
1460
1461   // Cleanup and return
1462   Builder.SetInsertPoint(ContBB);
1463   if (RValTy->isVoidType())
1464     return RValue::get(nullptr);
1465
1466   assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1467   return convertTempToRValue(Dest.withElementType(ConvertTypeForMem(RValTy)),
1468                              RValTy, E->getExprLoc());
1469 }
1470
1471 Address AtomicInfo::castToAtomicIntPointer(Address addr) const {
1472   llvm::IntegerType *ty =
1473     llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1474   return addr.withElementType(ty);
1475 }
1476
1477 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1478   llvm::Type *Ty = Addr.getElementType();
1479   uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1480   if (SourceSizeInBits != AtomicSizeInBits) {
1481     Address Tmp = CreateTempAlloca();
1482     CGF.Builder.CreateMemCpy(Tmp, Addr,
1483                              std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1484     Addr = Tmp;
1485   }
1486
1487   return castToAtomicIntPointer(Addr);
1488 }
1489
1490 RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1491                                              AggValueSlot resultSlot,
1492                                              SourceLocation loc,
1493                                              bool asValue) const {
1494   if (LVal.isSimple()) {
1495     if (EvaluationKind == TEK_Aggregate)
1496       return resultSlot.asRValue();
1497
1498     // Drill into the padding structure if we have one.
1499     if (hasPadding())
1500       addr = CGF.Builder.CreateStructGEP(addr, 0);
1501
1502     // Otherwise, just convert the temporary to an r-value using the
1503     // normal conversion routine.
1504     return CGF.convertTempToRValue(addr, getValueType(), loc);
1505   }
1506   if (!asValue)
1507     // Get RValue from temp memory as atomic for non-simple lvalues
1508     return RValue::get(CGF.Builder.CreateLoad(addr));
1509   if (LVal.isBitField())
1510     return CGF.EmitLoadOfBitfieldLValue(
1511         LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1512                              LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1513   if (LVal.isVectorElt())
1514     return CGF.EmitLoadOfLValue(
1515         LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1516                               LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1517   assert(LVal.isExtVectorElt());
1518   return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1519       addr, LVal.getExtVectorElts(), LVal.getType(),
1520       LVal.getBaseInfo(), TBAAAccessInfo()));
1521 }
1522
1523 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1524                                              AggValueSlot ResultSlot,
1525                                              SourceLocation Loc,
1526                                              bool AsValue) const {
1527   // Try not to in some easy cases.
1528   assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1529   if (getEvaluationKind() == TEK_Scalar &&
1530       (((!LVal.isBitField() ||
1531          LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1532         !hasPadding()) ||
1533        !AsValue)) {
1534     auto *ValTy = AsValue
1535                       ? CGF.ConvertTypeForMem(ValueTy)
1536                       : getAtomicAddress().getElementType();
1537     if (ValTy->isIntegerTy()) {
1538       assert(IntVal->getType() == ValTy && "Different integer types.");
1539       return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1540     } else if (ValTy->isPointerTy())
1541       return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1542     else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1543       return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1544   }
1545
1546   // Create a temporary.  This needs to be big enough to hold the
1547   // atomic integer.
1548   Address Temp = Address::invalid();
1549   bool TempIsVolatile = false;
1550   if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1551     assert(!ResultSlot.isIgnored());
1552     Temp = ResultSlot.getAddress();
1553     TempIsVolatile = ResultSlot.isVolatile();
1554   } else {
1555     Temp = CreateTempAlloca();
1556   }
1557
1558   // Slam the integer into the temporary.
1559   Address CastTemp = castToAtomicIntPointer(Temp);
1560   CGF.Builder.CreateStore(IntVal, CastTemp)
1561       ->setVolatile(TempIsVolatile);
1562
1563   return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1564 }
1565
1566 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1567                                        llvm::AtomicOrdering AO, bool) {
1568   // void __atomic_load(size_t size, void *mem, void *return, int order);
1569   CallArgList Args;
1570   Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1571   Args.add(RValue::get(getAtomicPointer()), CGF.getContext().VoidPtrTy);
1572   Args.add(RValue::get(AddForLoaded), CGF.getContext().VoidPtrTy);
1573   Args.add(
1574       RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1575       CGF.getContext().IntTy);
1576   emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1577 }
1578
1579 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1580                                           bool IsVolatile) {
1581   // Okay, we're doing this natively.
1582   Address Addr = getAtomicAddressAsAtomicIntPointer();
1583   llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1584   Load->setAtomic(AO);
1585
1586   // Other decoration.
1587   if (IsVolatile)
1588     Load->setVolatile(true);
1589   CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1590   return Load;
1591 }
1592
1593 /// An LValue is a candidate for having its loads and stores be made atomic if
1594 /// we are operating under /volatile:ms *and* the LValue itself is volatile and
1595 /// performing such an operation can be performed without a libcall.
1596 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1597   if (!CGM.getLangOpts().MSVolatile) return false;
1598   AtomicInfo AI(*this, LV);
1599   bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1600   // An atomic is inline if we don't need to use a libcall.
1601   bool AtomicIsInline = !AI.shouldUseLibcall();
1602   // MSVC doesn't seem to do this for types wider than a pointer.
1603   if (getContext().getTypeSize(LV.getType()) >
1604       getContext().getTypeSize(getContext().getIntPtrType()))
1605     return false;
1606   return IsVolatile && AtomicIsInline;
1607 }
1608
1609 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1610                                        AggValueSlot Slot) {
1611   llvm::AtomicOrdering AO;
1612   bool IsVolatile = LV.isVolatileQualified();
1613   if (LV.getType()->isAtomicType()) {
1614     AO = llvm::AtomicOrdering::SequentiallyConsistent;
1615   } else {
1616     AO = llvm::AtomicOrdering::Acquire;
1617     IsVolatile = true;
1618   }
1619   return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1620 }
1621
1622 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1623                                   bool AsValue, llvm::AtomicOrdering AO,
1624                                   bool IsVolatile) {
1625   // Check whether we should use a library call.
1626   if (shouldUseLibcall()) {
1627     Address TempAddr = Address::invalid();
1628     if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1629       assert(getEvaluationKind() == TEK_Aggregate);
1630       TempAddr = ResultSlot.getAddress();
1631     } else
1632       TempAddr = CreateTempAlloca();
1633
1634     EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1635
1636     // Okay, turn that back into the original value or whole atomic (for
1637     // non-simple lvalues) type.
1638     return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1639   }
1640
1641   // Okay, we're doing this natively.
1642   auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1643
1644   // If we're ignoring an aggregate return, don't do anything.
1645   if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1646     return RValue::getAggregate(Address::invalid(), false);
1647
1648   // Okay, turn that back into the original value or atomic (for non-simple
1649   // lvalues) type.
1650   return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1651 }
1652
1653 /// Emit a load from an l-value of atomic type.  Note that the r-value
1654 /// we produce is an r-value of the atomic *value* type.
1655 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1656                                        llvm::AtomicOrdering AO, bool IsVolatile,
1657                                        AggValueSlot resultSlot) {
1658   AtomicInfo Atomics(*this, src);
1659   return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1660                                 IsVolatile);
1661 }
1662
1663 /// Copy an r-value into memory as part of storing to an atomic type.
1664 /// This needs to create a bit-pattern suitable for atomic operations.
1665 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1666   assert(LVal.isSimple());
1667   // If we have an r-value, the rvalue should be of the atomic type,
1668   // which means that the caller is responsible for having zeroed
1669   // any padding.  Just do an aggregate copy of that type.
1670   if (rvalue.isAggregate()) {
1671     LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType());
1672     LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(),
1673                                     getAtomicType());
1674     bool IsVolatile = rvalue.isVolatileQualified() ||
1675                       LVal.isVolatileQualified();
1676     CGF.EmitAggregateCopy(Dest, Src, getAtomicType(),
1677                           AggValueSlot::DoesNotOverlap, IsVolatile);
1678     return;
1679   }
1680
1681   // Okay, otherwise we're copying stuff.
1682
1683   // Zero out the buffer if necessary.
1684   emitMemSetZeroIfNecessary();
1685
1686   // Drill past the padding if present.
1687   LValue TempLVal = projectValue();
1688
1689   // Okay, store the rvalue in.
1690   if (rvalue.isScalar()) {
1691     CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1692   } else {
1693     CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1694   }
1695 }
1696
1697
1698 /// Materialize an r-value into memory for the purposes of storing it
1699 /// to an atomic type.
1700 Address AtomicInfo::materializeRValue(RValue rvalue) const {
1701   // Aggregate r-values are already in memory, and EmitAtomicStore
1702   // requires them to be values of the atomic type.
1703   if (rvalue.isAggregate())
1704     return rvalue.getAggregateAddress();
1705
1706   // Otherwise, make a temporary and materialize into it.
1707   LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1708   AtomicInfo Atomics(CGF, TempLV);
1709   Atomics.emitCopyIntoMemory(rvalue);
1710   return TempLV.getAddress(CGF);
1711 }
1712
1713 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1714   // If we've got a scalar value of the right size, try to avoid going
1715   // through memory.
1716   if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1717     llvm::Value *Value = RVal.getScalarVal();
1718     if (isa<llvm::IntegerType>(Value->getType()))
1719       return CGF.EmitToMemory(Value, ValueTy);
1720     else {
1721       llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1722           CGF.getLLVMContext(),
1723           LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1724       if (isa<llvm::PointerType>(Value->getType()))
1725         return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1726       else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1727         return CGF.Builder.CreateBitCast(Value, InputIntTy);
1728     }
1729   }
1730   // Otherwise, we need to go through memory.
1731   // Put the r-value in memory.
1732   Address Addr = materializeRValue(RVal);
1733
1734   // Cast the temporary to the atomic int type and pull a value out.
1735   Addr = castToAtomicIntPointer(Addr);
1736   return CGF.Builder.CreateLoad(Addr);
1737 }
1738
1739 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1740     llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1741     llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1742   // Do the atomic store.
1743   Address Addr = getAtomicAddressAsAtomicIntPointer();
1744   auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1745                                                ExpectedVal, DesiredVal,
1746                                                Success, Failure);
1747   // Other decoration.
1748   Inst->setVolatile(LVal.isVolatileQualified());
1749   Inst->setWeak(IsWeak);
1750
1751   // Okay, turn that back into the original value type.
1752   auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1753   auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1754   return std::make_pair(PreviousVal, SuccessFailureVal);
1755 }
1756
1757 llvm::Value *
1758 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1759                                              llvm::Value *DesiredAddr,
1760                                              llvm::AtomicOrdering Success,
1761                                              llvm::AtomicOrdering Failure) {
1762   // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1763   // void *desired, int success, int failure);
1764   CallArgList Args;
1765   Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1766   Args.add(RValue::get(getAtomicPointer()), CGF.getContext().VoidPtrTy);
1767   Args.add(RValue::get(ExpectedAddr), CGF.getContext().VoidPtrTy);
1768   Args.add(RValue::get(DesiredAddr), CGF.getContext().VoidPtrTy);
1769   Args.add(RValue::get(
1770                llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1771            CGF.getContext().IntTy);
1772   Args.add(RValue::get(
1773                llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1774            CGF.getContext().IntTy);
1775   auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1776                                               CGF.getContext().BoolTy, Args);
1777
1778   return SuccessFailureRVal.getScalarVal();
1779 }
1780
1781 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1782     RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1783     llvm::AtomicOrdering Failure, bool IsWeak) {
1784   // Check whether we should use a library call.
1785   if (shouldUseLibcall()) {
1786     // Produce a source address.
1787     Address ExpectedAddr = materializeRValue(Expected);
1788     Address DesiredAddr = materializeRValue(Desired);
1789     auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1790                                                  DesiredAddr.getPointer(),
1791                                                  Success, Failure);
1792     return std::make_pair(
1793         convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1794                                   SourceLocation(), /*AsValue=*/false),
1795         Res);
1796   }
1797
1798   // If we've got a scalar value of the right size, try to avoid going
1799   // through memory.
1800   auto *ExpectedVal = convertRValueToInt(Expected);
1801   auto *DesiredVal = convertRValueToInt(Desired);
1802   auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1803                                          Failure, IsWeak);
1804   return std::make_pair(
1805       ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1806                                 SourceLocation(), /*AsValue=*/false),
1807       Res.second);
1808 }
1809
1810 static void
1811 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1812                       const llvm::function_ref<RValue(RValue)> &UpdateOp,
1813                       Address DesiredAddr) {
1814   RValue UpRVal;
1815   LValue AtomicLVal = Atomics.getAtomicLValue();
1816   LValue DesiredLVal;
1817   if (AtomicLVal.isSimple()) {
1818     UpRVal = OldRVal;
1819     DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1820   } else {
1821     // Build new lvalue for temp address.
1822     Address Ptr = Atomics.materializeRValue(OldRVal);
1823     LValue UpdateLVal;
1824     if (AtomicLVal.isBitField()) {
1825       UpdateLVal =
1826           LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1827                                AtomicLVal.getType(),
1828                                AtomicLVal.getBaseInfo(),
1829                                AtomicLVal.getTBAAInfo());
1830       DesiredLVal =
1831           LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1832                                AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1833                                AtomicLVal.getTBAAInfo());
1834     } else if (AtomicLVal.isVectorElt()) {
1835       UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1836                                          AtomicLVal.getType(),
1837                                          AtomicLVal.getBaseInfo(),
1838                                          AtomicLVal.getTBAAInfo());
1839       DesiredLVal = LValue::MakeVectorElt(
1840           DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1841           AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1842     } else {
1843       assert(AtomicLVal.isExtVectorElt());
1844       UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1845                                             AtomicLVal.getType(),
1846                                             AtomicLVal.getBaseInfo(),
1847                                             AtomicLVal.getTBAAInfo());
1848       DesiredLVal = LValue::MakeExtVectorElt(
1849           DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1850           AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1851     }
1852     UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1853   }
1854   // Store new value in the corresponding memory area.
1855   RValue NewRVal = UpdateOp(UpRVal);
1856   if (NewRVal.isScalar()) {
1857     CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1858   } else {
1859     assert(NewRVal.isComplex());
1860     CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1861                            /*isInit=*/false);
1862   }
1863 }
1864
1865 void AtomicInfo::EmitAtomicUpdateLibcall(
1866     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1867     bool IsVolatile) {
1868   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1869
1870   Address ExpectedAddr = CreateTempAlloca();
1871
1872   EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1873   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1874   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1875   CGF.EmitBlock(ContBB);
1876   Address DesiredAddr = CreateTempAlloca();
1877   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1878       requiresMemSetZero(getAtomicAddress().getElementType())) {
1879     auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1880     CGF.Builder.CreateStore(OldVal, DesiredAddr);
1881   }
1882   auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1883                                            AggValueSlot::ignored(),
1884                                            SourceLocation(), /*AsValue=*/false);
1885   EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1886   auto *Res =
1887       EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1888                                        DesiredAddr.getPointer(),
1889                                        AO, Failure);
1890   CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1891   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1892 }
1893
1894 void AtomicInfo::EmitAtomicUpdateOp(
1895     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1896     bool IsVolatile) {
1897   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1898
1899   // Do the atomic load.
1900   auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1901   // For non-simple lvalues perform compare-and-swap procedure.
1902   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1903   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1904   auto *CurBB = CGF.Builder.GetInsertBlock();
1905   CGF.EmitBlock(ContBB);
1906   llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1907                                              /*NumReservedValues=*/2);
1908   PHI->addIncoming(OldVal, CurBB);
1909   Address NewAtomicAddr = CreateTempAlloca();
1910   Address NewAtomicIntAddr = castToAtomicIntPointer(NewAtomicAddr);
1911   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1912       requiresMemSetZero(getAtomicAddress().getElementType())) {
1913     CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1914   }
1915   auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1916                                            SourceLocation(), /*AsValue=*/false);
1917   EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1918   auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1919   // Try to write new value using cmpxchg operation.
1920   auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1921   PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1922   CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1923   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1924 }
1925
1926 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1927                                   RValue UpdateRVal, Address DesiredAddr) {
1928   LValue AtomicLVal = Atomics.getAtomicLValue();
1929   LValue DesiredLVal;
1930   // Build new lvalue for temp address.
1931   if (AtomicLVal.isBitField()) {
1932     DesiredLVal =
1933         LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1934                              AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1935                              AtomicLVal.getTBAAInfo());
1936   } else if (AtomicLVal.isVectorElt()) {
1937     DesiredLVal =
1938         LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1939                               AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1940                               AtomicLVal.getTBAAInfo());
1941   } else {
1942     assert(AtomicLVal.isExtVectorElt());
1943     DesiredLVal = LValue::MakeExtVectorElt(
1944         DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1945         AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1946   }
1947   // Store new value in the corresponding memory area.
1948   assert(UpdateRVal.isScalar());
1949   CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1950 }
1951
1952 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1953                                          RValue UpdateRVal, bool IsVolatile) {
1954   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1955
1956   Address ExpectedAddr = CreateTempAlloca();
1957
1958   EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1959   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1960   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1961   CGF.EmitBlock(ContBB);
1962   Address DesiredAddr = CreateTempAlloca();
1963   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1964       requiresMemSetZero(getAtomicAddress().getElementType())) {
1965     auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1966     CGF.Builder.CreateStore(OldVal, DesiredAddr);
1967   }
1968   EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1969   auto *Res =
1970       EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1971                                        DesiredAddr.getPointer(),
1972                                        AO, Failure);
1973   CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1974   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1975 }
1976
1977 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1978                                     bool IsVolatile) {
1979   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1980
1981   // Do the atomic load.
1982   auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1983   // For non-simple lvalues perform compare-and-swap procedure.
1984   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1985   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1986   auto *CurBB = CGF.Builder.GetInsertBlock();
1987   CGF.EmitBlock(ContBB);
1988   llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1989                                              /*NumReservedValues=*/2);
1990   PHI->addIncoming(OldVal, CurBB);
1991   Address NewAtomicAddr = CreateTempAlloca();
1992   Address NewAtomicIntAddr = castToAtomicIntPointer(NewAtomicAddr);
1993   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1994       requiresMemSetZero(getAtomicAddress().getElementType())) {
1995     CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1996   }
1997   EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1998   auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1999   // Try to write new value using cmpxchg operation.
2000   auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
2001   PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
2002   CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
2003   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2004 }
2005
2006 void AtomicInfo::EmitAtomicUpdate(
2007     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
2008     bool IsVolatile) {
2009   if (shouldUseLibcall()) {
2010     EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
2011   } else {
2012     EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
2013   }
2014 }
2015
2016 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
2017                                   bool IsVolatile) {
2018   if (shouldUseLibcall()) {
2019     EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
2020   } else {
2021     EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
2022   }
2023 }
2024
2025 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
2026                                       bool isInit) {
2027   bool IsVolatile = lvalue.isVolatileQualified();
2028   llvm::AtomicOrdering AO;
2029   if (lvalue.getType()->isAtomicType()) {
2030     AO = llvm::AtomicOrdering::SequentiallyConsistent;
2031   } else {
2032     AO = llvm::AtomicOrdering::Release;
2033     IsVolatile = true;
2034   }
2035   return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
2036 }
2037
2038 /// Emit a store to an l-value of atomic type.
2039 ///
2040 /// Note that the r-value is expected to be an r-value *of the atomic
2041 /// type*; this means that for aggregate r-values, it should include
2042 /// storage for any padding that was necessary.
2043 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
2044                                       llvm::AtomicOrdering AO, bool IsVolatile,
2045                                       bool isInit) {
2046   // If this is an aggregate r-value, it should agree in type except
2047   // maybe for address-space qualification.
2048   assert(!rvalue.isAggregate() ||
2049          rvalue.getAggregateAddress().getElementType() ==
2050              dest.getAddress(*this).getElementType());
2051
2052   AtomicInfo atomics(*this, dest);
2053   LValue LVal = atomics.getAtomicLValue();
2054
2055   // If this is an initialization, just put the value there normally.
2056   if (LVal.isSimple()) {
2057     if (isInit) {
2058       atomics.emitCopyIntoMemory(rvalue);
2059       return;
2060     }
2061
2062     // Check whether we should use a library call.
2063     if (atomics.shouldUseLibcall()) {
2064       // Produce a source address.
2065       Address srcAddr = atomics.materializeRValue(rvalue);
2066
2067       // void __atomic_store(size_t size, void *mem, void *val, int order)
2068       CallArgList args;
2069       args.add(RValue::get(atomics.getAtomicSizeValue()),
2070                getContext().getSizeType());
2071       args.add(RValue::get(atomics.getAtomicPointer()), getContext().VoidPtrTy);
2072       args.add(RValue::get(srcAddr.getPointer()), getContext().VoidPtrTy);
2073       args.add(
2074           RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
2075           getContext().IntTy);
2076       emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
2077       return;
2078     }
2079
2080     // Okay, we're doing this natively.
2081     llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
2082
2083     // Do the atomic store.
2084     Address addr = atomics.castToAtomicIntPointer(atomics.getAtomicAddress());
2085     intValue = Builder.CreateIntCast(
2086         intValue, addr.getElementType(), /*isSigned=*/false);
2087     llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
2088
2089     if (AO == llvm::AtomicOrdering::Acquire)
2090       AO = llvm::AtomicOrdering::Monotonic;
2091     else if (AO == llvm::AtomicOrdering::AcquireRelease)
2092       AO = llvm::AtomicOrdering::Release;
2093     // Initializations don't need to be atomic.
2094     if (!isInit)
2095       store->setAtomic(AO);
2096
2097     // Other decoration.
2098     if (IsVolatile)
2099       store->setVolatile(true);
2100     CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
2101     return;
2102   }
2103
2104   // Emit simple atomic update operation.
2105   atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
2106 }
2107
2108 /// Emit a compare-and-exchange op for atomic type.
2109 ///
2110 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
2111     LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
2112     llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
2113     AggValueSlot Slot) {
2114   // If this is an aggregate r-value, it should agree in type except
2115   // maybe for address-space qualification.
2116   assert(!Expected.isAggregate() ||
2117          Expected.getAggregateAddress().getElementType() ==
2118              Obj.getAddress(*this).getElementType());
2119   assert(!Desired.isAggregate() ||
2120          Desired.getAggregateAddress().getElementType() ==
2121              Obj.getAddress(*this).getElementType());
2122   AtomicInfo Atomics(*this, Obj);
2123
2124   return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
2125                                            IsWeak);
2126 }
2127
2128 void CodeGenFunction::EmitAtomicUpdate(
2129     LValue LVal, llvm::AtomicOrdering AO,
2130     const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
2131   AtomicInfo Atomics(*this, LVal);
2132   Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2133 }
2134
2135 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
2136   AtomicInfo atomics(*this, dest);
2137
2138   switch (atomics.getEvaluationKind()) {
2139   case TEK_Scalar: {
2140     llvm::Value *value = EmitScalarExpr(init);
2141     atomics.emitCopyIntoMemory(RValue::get(value));
2142     return;
2143   }
2144
2145   case TEK_Complex: {
2146     ComplexPairTy value = EmitComplexExpr(init);
2147     atomics.emitCopyIntoMemory(RValue::getComplex(value));
2148     return;
2149   }
2150
2151   case TEK_Aggregate: {
2152     // Fix up the destination if the initializer isn't an expression
2153     // of atomic type.
2154     bool Zeroed = false;
2155     if (!init->getType()->isAtomicType()) {
2156       Zeroed = atomics.emitMemSetZeroIfNecessary();
2157       dest = atomics.projectValue();
2158     }
2159
2160     // Evaluate the expression directly into the destination.
2161     AggValueSlot slot = AggValueSlot::forLValue(
2162         dest, *this, AggValueSlot::IsNotDestructed,
2163         AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
2164         AggValueSlot::DoesNotOverlap,
2165         Zeroed ? AggValueSlot::IsZeroed : AggValueSlot::IsNotZeroed);
2166
2167     EmitAggExpr(init, slot);
2168     return;
2169   }
2170   }
2171   llvm_unreachable("bad evaluation kind");
2172 }