clang/lib/CodeGen/CGExprComplex.cpp

   1 //===--- CGExprComplex.cpp - Emit LLVM Code for Complex Exprs -------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This contains code to emit Expr nodes with complex types as LLVM code.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "CGOpenMPRuntime.h"
  14 #include "CodeGenFunction.h"
  15 #include "CodeGenModule.h"
  16 #include "ConstantEmitter.h"
  17 #include "clang/AST/StmtVisitor.h"
  18 #include "llvm/IR/Constants.h"
  19 #include "llvm/IR/Instructions.h"
  20 #include "llvm/IR/MDBuilder.h"
  21 #include "llvm/IR/Metadata.h"
  22 using namespace clang;
  23 using namespace CodeGen;
  24
  25 //===----------------------------------------------------------------------===//
  26 //                        Complex Expression Emitter
  27 //===----------------------------------------------------------------------===//
  28
  29 namespace llvm {
  30 extern cl::opt<bool> EnableSingleByteCoverage;
  31 } // namespace llvm
  32
  33 typedef CodeGenFunction::ComplexPairTy ComplexPairTy;
  34
  35 /// Return the complex type that we are meant to emit.
  36 static const ComplexType *getComplexType(QualType type) {
  37   type = type.getCanonicalType();
  38   if (const ComplexType *comp = dyn_cast<ComplexType>(type)) {
  39     return comp;
  40   } else {
  41     return cast<ComplexType>(cast<AtomicType>(type)->getValueType());
  42   }
  43 }
  44
  45 namespace  {
  46 class ComplexExprEmitter
  47   : public StmtVisitor<ComplexExprEmitter, ComplexPairTy> {
  48   CodeGenFunction &CGF;
  49   CGBuilderTy &Builder;
  50   bool IgnoreReal;
  51   bool IgnoreImag;
  52   bool FPHasBeenPromoted;
  53
  54 public:
  55   ComplexExprEmitter(CodeGenFunction &cgf, bool ir = false, bool ii = false)
  56       : CGF(cgf), Builder(CGF.Builder), IgnoreReal(ir), IgnoreImag(ii),
  57         FPHasBeenPromoted(false) {}
  58
  59   //===--------------------------------------------------------------------===//
  60   //                               Utilities
  61   //===--------------------------------------------------------------------===//
  62
  63   bool TestAndClearIgnoreReal() {
  64     bool I = IgnoreReal;
  65     IgnoreReal = false;
  66     return I;
  67   }
  68   bool TestAndClearIgnoreImag() {
  69     bool I = IgnoreImag;
  70     IgnoreImag = false;
  71     return I;
  72   }
  73
  74   /// EmitLoadOfLValue - Given an expression with complex type that represents a
  75   /// value l-value, this method emits the address of the l-value, then loads
  76   /// and returns the result.
  77   ComplexPairTy EmitLoadOfLValue(const Expr *E) {
  78     return EmitLoadOfLValue(CGF.EmitLValue(E), E->getExprLoc());
  79   }
  80
  81   ComplexPairTy EmitLoadOfLValue(LValue LV, SourceLocation Loc);
  82
  83   /// EmitStoreOfComplex - Store the specified real/imag parts into the
  84   /// specified value pointer.
  85   void EmitStoreOfComplex(ComplexPairTy Val, LValue LV, bool isInit);
  86
  87   /// Emit a cast from complex value Val to DestType.
  88   ComplexPairTy EmitComplexToComplexCast(ComplexPairTy Val, QualType SrcType,
  89                                          QualType DestType, SourceLocation Loc);
  90   /// Emit a cast from scalar value Val to DestType.
  91   ComplexPairTy EmitScalarToComplexCast(llvm::Value *Val, QualType SrcType,
  92                                         QualType DestType, SourceLocation Loc);
  93
  94   //===--------------------------------------------------------------------===//
  95   //                            Visitor Methods
  96   //===--------------------------------------------------------------------===//
  97
  98   ComplexPairTy Visit(Expr *E) {
  99     ApplyDebugLocation DL(CGF, E);
 100     return StmtVisitor<ComplexExprEmitter, ComplexPairTy>::Visit(E);
 101   }
 102
 103   ComplexPairTy VisitStmt(Stmt *S) {
 104     S->dump(llvm::errs(), CGF.getContext());
 105     llvm_unreachable("Stmt can't have complex result type!");
 106   }
 107   ComplexPairTy VisitExpr(Expr *S);
 108   ComplexPairTy VisitConstantExpr(ConstantExpr *E) {
 109     if (llvm::Constant *Result = ConstantEmitter(CGF).tryEmitConstantExpr(E))
 110       return ComplexPairTy(Result->getAggregateElement(0U),
 111                            Result->getAggregateElement(1U));
 112     return Visit(E->getSubExpr());
 113   }
 114   ComplexPairTy VisitParenExpr(ParenExpr *PE) { return Visit(PE->getSubExpr());}
 115   ComplexPairTy VisitGenericSelectionExpr(GenericSelectionExpr *GE) {
 116     return Visit(GE->getResultExpr());
 117   }
 118   ComplexPairTy VisitImaginaryLiteral(const ImaginaryLiteral *IL);
 119   ComplexPairTy
 120   VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *PE) {
 121     return Visit(PE->getReplacement());
 122   }
 123   ComplexPairTy VisitCoawaitExpr(CoawaitExpr *S) {
 124     return CGF.EmitCoawaitExpr(*S).getComplexVal();
 125   }
 126   ComplexPairTy VisitCoyieldExpr(CoyieldExpr *S) {
 127     return CGF.EmitCoyieldExpr(*S).getComplexVal();
 128   }
 129   ComplexPairTy VisitUnaryCoawait(const UnaryOperator *E) {
 130     return Visit(E->getSubExpr());
 131   }
 132
 133   ComplexPairTy emitConstant(const CodeGenFunction::ConstantEmission &Constant,
 134                              Expr *E) {
 135     assert(Constant && "not a constant");
 136     if (Constant.isReference())
 137       return EmitLoadOfLValue(Constant.getReferenceLValue(CGF, E),
 138                               E->getExprLoc());
 139
 140     llvm::Constant *pair = Constant.getValue();
 141     return ComplexPairTy(pair->getAggregateElement(0U),
 142                          pair->getAggregateElement(1U));
 143   }
 144
 145   // l-values.
 146   ComplexPairTy VisitDeclRefExpr(DeclRefExpr *E) {
 147     if (CodeGenFunction::ConstantEmission Constant = CGF.tryEmitAsConstant(E))
 148       return emitConstant(Constant, E);
 149     return EmitLoadOfLValue(E);
 150   }
 151   ComplexPairTy VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) {
 152     return EmitLoadOfLValue(E);
 153   }
 154   ComplexPairTy VisitObjCMessageExpr(ObjCMessageExpr *E) {
 155     return CGF.EmitObjCMessageExpr(E).getComplexVal();
 156   }
 157   ComplexPairTy VisitArraySubscriptExpr(Expr *E) { return EmitLoadOfLValue(E); }
 158   ComplexPairTy VisitMemberExpr(MemberExpr *ME) {
 159     if (CodeGenFunction::ConstantEmission Constant =
 160             CGF.tryEmitAsConstant(ME)) {
 161       CGF.EmitIgnoredExpr(ME->getBase());
 162       return emitConstant(Constant, ME);
 163     }
 164     return EmitLoadOfLValue(ME);
 165   }
 166   ComplexPairTy VisitOpaqueValueExpr(OpaqueValueExpr *E) {
 167     if (E->isGLValue())
 168       return EmitLoadOfLValue(CGF.getOrCreateOpaqueLValueMapping(E),
 169                               E->getExprLoc());
 170     return CGF.getOrCreateOpaqueRValueMapping(E).getComplexVal();
 171   }
 172
 173   ComplexPairTy VisitPseudoObjectExpr(PseudoObjectExpr *E) {
 174     return CGF.EmitPseudoObjectRValue(E).getComplexVal();
 175   }
 176
 177   // FIXME: CompoundLiteralExpr
 178
 179   ComplexPairTy EmitCast(CastKind CK, Expr *Op, QualType DestTy);
 180   ComplexPairTy VisitImplicitCastExpr(ImplicitCastExpr *E) {
 181     // Unlike for scalars, we don't have to worry about function->ptr demotion
 182     // here.
 183     if (E->changesVolatileQualification())
 184       return EmitLoadOfLValue(E);
 185     return EmitCast(E->getCastKind(), E->getSubExpr(), E->getType());
 186   }
 187   ComplexPairTy VisitCastExpr(CastExpr *E) {
 188     if (const auto *ECE = dyn_cast<ExplicitCastExpr>(E))
 189       CGF.CGM.EmitExplicitCastExprType(ECE, &CGF);
 190     if (E->changesVolatileQualification())
 191        return EmitLoadOfLValue(E);
 192     return EmitCast(E->getCastKind(), E->getSubExpr(), E->getType());
 193   }
 194   ComplexPairTy VisitCallExpr(const CallExpr *E);
 195   ComplexPairTy VisitStmtExpr(const StmtExpr *E);
 196
 197   // Operators.
 198   ComplexPairTy VisitPrePostIncDec(const UnaryOperator *E,
 199                                    bool isInc, bool isPre) {
 200     LValue LV = CGF.EmitLValue(E->getSubExpr());
 201     return CGF.EmitComplexPrePostIncDec(E, LV, isInc, isPre);
 202   }
 203   ComplexPairTy VisitUnaryPostDec(const UnaryOperator *E) {
 204     return VisitPrePostIncDec(E, false, false);
 205   }
 206   ComplexPairTy VisitUnaryPostInc(const UnaryOperator *E) {
 207     return VisitPrePostIncDec(E, true, false);
 208   }
 209   ComplexPairTy VisitUnaryPreDec(const UnaryOperator *E) {
 210     return VisitPrePostIncDec(E, false, true);
 211   }
 212   ComplexPairTy VisitUnaryPreInc(const UnaryOperator *E) {
 213     return VisitPrePostIncDec(E, true, true);
 214   }
 215   ComplexPairTy VisitUnaryDeref(const Expr *E) { return EmitLoadOfLValue(E); }
 216
 217   ComplexPairTy VisitUnaryPlus(const UnaryOperator *E,
 218                                QualType PromotionType = QualType());
 219   ComplexPairTy VisitPlus(const UnaryOperator *E, QualType PromotionType);
 220   ComplexPairTy VisitUnaryMinus(const UnaryOperator *E,
 221                                 QualType PromotionType = QualType());
 222   ComplexPairTy VisitMinus(const UnaryOperator *E, QualType PromotionType);
 223   ComplexPairTy VisitUnaryNot      (const UnaryOperator *E);
 224   // LNot,Real,Imag never return complex.
 225   ComplexPairTy VisitUnaryExtension(const UnaryOperator *E) {
 226     return Visit(E->getSubExpr());
 227   }
 228   ComplexPairTy VisitCXXDefaultArgExpr(CXXDefaultArgExpr *DAE) {
 229     CodeGenFunction::CXXDefaultArgExprScope Scope(CGF, DAE);
 230     return Visit(DAE->getExpr());
 231   }
 232   ComplexPairTy VisitCXXDefaultInitExpr(CXXDefaultInitExpr *DIE) {
 233     CodeGenFunction::CXXDefaultInitExprScope Scope(CGF, DIE);
 234     return Visit(DIE->getExpr());
 235   }
 236   ComplexPairTy VisitExprWithCleanups(ExprWithCleanups *E) {
 237     CodeGenFunction::RunCleanupsScope Scope(CGF);
 238     ComplexPairTy Vals = Visit(E->getSubExpr());
 239     // Defend against dominance problems caused by jumps out of expression
 240     // evaluation through the shared cleanup block.
 241     Scope.ForceCleanup({&Vals.first, &Vals.second});
 242     return Vals;
 243   }
 244   ComplexPairTy VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) {
 245     assert(E->getType()->isAnyComplexType() && "Expected complex type!");
 246     QualType Elem = E->getType()->castAs<ComplexType>()->getElementType();
 247     llvm::Constant *Null = llvm::Constant::getNullValue(CGF.ConvertType(Elem));
 248     return ComplexPairTy(Null, Null);
 249   }
 250   ComplexPairTy VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) {
 251     assert(E->getType()->isAnyComplexType() && "Expected complex type!");
 252     QualType Elem = E->getType()->castAs<ComplexType>()->getElementType();
 253     llvm::Constant *Null =
 254                        llvm::Constant::getNullValue(CGF.ConvertType(Elem));
 255     return ComplexPairTy(Null, Null);
 256   }
 257
 258   struct BinOpInfo {
 259     ComplexPairTy LHS;
 260     ComplexPairTy RHS;
 261     QualType Ty;  // Computation Type.
 262     FPOptions FPFeatures;
 263   };
 264
 265   BinOpInfo EmitBinOps(const BinaryOperator *E,
 266                        QualType PromotionTy = QualType());
 267   ComplexPairTy EmitPromoted(const Expr *E, QualType PromotionTy);
 268   ComplexPairTy EmitPromotedComplexOperand(const Expr *E, QualType PromotionTy);
 269   LValue EmitCompoundAssignLValue(const CompoundAssignOperator *E,
 270                                   ComplexPairTy (ComplexExprEmitter::*Func)
 271                                   (const BinOpInfo &),
 272                                   RValue &Val);
 273   ComplexPairTy EmitCompoundAssign(const CompoundAssignOperator *E,
 274                                    ComplexPairTy (ComplexExprEmitter::*Func)
 275                                    (const BinOpInfo &));
 276
 277   ComplexPairTy EmitBinAdd(const BinOpInfo &Op);
 278   ComplexPairTy EmitBinSub(const BinOpInfo &Op);
 279   ComplexPairTy EmitBinMul(const BinOpInfo &Op);
 280   ComplexPairTy EmitBinDiv(const BinOpInfo &Op);
 281   ComplexPairTy EmitAlgebraicDiv(llvm::Value *A, llvm::Value *B, llvm::Value *C,
 282                                  llvm::Value *D);
 283   ComplexPairTy EmitRangeReductionDiv(llvm::Value *A, llvm::Value *B,
 284                                       llvm::Value *C, llvm::Value *D);
 285
 286   ComplexPairTy EmitComplexBinOpLibCall(StringRef LibCallName,
 287                                         const BinOpInfo &Op);
 288
 289   QualType HigherPrecisionTypeForComplexArithmetic(QualType ElementType,
 290                                                    bool IsDivOpCode) {
 291     ASTContext &Ctx = CGF.getContext();
 292     const QualType HigherElementType =
 293         Ctx.GetHigherPrecisionFPType(ElementType);
 294     const llvm::fltSemantics &ElementTypeSemantics =
 295         Ctx.getFloatTypeSemantics(ElementType);
 296     const llvm::fltSemantics &HigherElementTypeSemantics =
 297         Ctx.getFloatTypeSemantics(HigherElementType);
 298     // Check that the promoted type can handle the intermediate values without
 299     // overflowing. This can be interpreted as:
 300     // (SmallerType.LargestFiniteVal * SmallerType.LargestFiniteVal) * 2 <=
 301     // LargerType.LargestFiniteVal.
 302     // In terms of exponent it gives this formula:
 303     // (SmallerType.LargestFiniteVal * SmallerType.LargestFiniteVal
 304     // doubles the exponent of SmallerType.LargestFiniteVal)
 305     if (llvm::APFloat::semanticsMaxExponent(ElementTypeSemantics) * 2 + 1 <=
 306         llvm::APFloat::semanticsMaxExponent(HigherElementTypeSemantics)) {
 307       FPHasBeenPromoted = true;
 308       return Ctx.getComplexType(HigherElementType);
 309     } else {
 310       // The intermediate values can't be represented in the promoted type
 311       // without overflowing.
 312       return QualType();
 313     }
 314   }
 315
 316   QualType getPromotionType(FPOptionsOverride Features, QualType Ty,
 317                             bool IsDivOpCode = false) {
 318     if (auto *CT = Ty->getAs<ComplexType>()) {
 319       QualType ElementType = CT->getElementType();
 320       bool IsFloatingType = ElementType->isFloatingType();
 321       bool IsComplexRangePromoted = CGF.getLangOpts().getComplexRange() ==
 322                                     LangOptions::ComplexRangeKind::CX_Promoted;
 323       bool HasNoComplexRangeOverride = !Features.hasComplexRangeOverride();
 324       bool HasMatchingComplexRange = Features.hasComplexRangeOverride() &&
 325                                      Features.getComplexRangeOverride() ==
 326                                          CGF.getLangOpts().getComplexRange();
 327
 328       if (IsDivOpCode && IsFloatingType && IsComplexRangePromoted &&
 329           (HasNoComplexRangeOverride || HasMatchingComplexRange))
 330         return HigherPrecisionTypeForComplexArithmetic(ElementType,
 331                                                        IsDivOpCode);
 332       if (ElementType.UseExcessPrecision(CGF.getContext()))
 333         return CGF.getContext().getComplexType(CGF.getContext().FloatTy);
 334     }
 335     if (Ty.UseExcessPrecision(CGF.getContext()))
 336       return CGF.getContext().FloatTy;
 337     return QualType();
 338   }
 339
 340 #define HANDLEBINOP(OP)                                                        \
 341   ComplexPairTy VisitBin##OP(const BinaryOperator *E) {                        \
 342     QualType promotionTy = getPromotionType(                                   \
 343         E->getStoredFPFeaturesOrDefault(), E->getType(),                       \
 344         (E->getOpcode() == BinaryOperatorKind::BO_Div) ? true : false);        \
 345     ComplexPairTy result = EmitBin##OP(EmitBinOps(E, promotionTy));            \
 346     if (!promotionTy.isNull())                                                 \
 347       result = CGF.EmitUnPromotedValue(result, E->getType());                  \
 348     return result;                                                             \
 349   }
 350
 351   HANDLEBINOP(Mul)
 352   HANDLEBINOP(Div)
 353   HANDLEBINOP(Add)
 354   HANDLEBINOP(Sub)
 355 #undef HANDLEBINOP
 356
 357   ComplexPairTy VisitCXXRewrittenBinaryOperator(CXXRewrittenBinaryOperator *E) {
 358     return Visit(E->getSemanticForm());
 359   }
 360
 361   // Compound assignments.
 362   ComplexPairTy VisitBinAddAssign(const CompoundAssignOperator *E) {
 363     return EmitCompoundAssign(E, &ComplexExprEmitter::EmitBinAdd);
 364   }
 365   ComplexPairTy VisitBinSubAssign(const CompoundAssignOperator *E) {
 366     return EmitCompoundAssign(E, &ComplexExprEmitter::EmitBinSub);
 367   }
 368   ComplexPairTy VisitBinMulAssign(const CompoundAssignOperator *E) {
 369     return EmitCompoundAssign(E, &ComplexExprEmitter::EmitBinMul);
 370   }
 371   ComplexPairTy VisitBinDivAssign(const CompoundAssignOperator *E) {
 372     return EmitCompoundAssign(E, &ComplexExprEmitter::EmitBinDiv);
 373   }
 374
 375   // GCC rejects rem/and/or/xor for integer complex.
 376   // Logical and/or always return int, never complex.
 377
 378   // No comparisons produce a complex result.
 379
 380   LValue EmitBinAssignLValue(const BinaryOperator *E,
 381                              ComplexPairTy &Val);
 382   ComplexPairTy VisitBinAssign     (const BinaryOperator *E);
 383   ComplexPairTy VisitBinComma      (const BinaryOperator *E);
 384
 385
 386   ComplexPairTy
 387   VisitAbstractConditionalOperator(const AbstractConditionalOperator *CO);
 388   ComplexPairTy VisitChooseExpr(ChooseExpr *CE);
 389
 390   ComplexPairTy VisitInitListExpr(InitListExpr *E);
 391
 392   ComplexPairTy VisitCompoundLiteralExpr(CompoundLiteralExpr *E) {
 393     return EmitLoadOfLValue(E);
 394   }
 395
 396   ComplexPairTy VisitVAArgExpr(VAArgExpr *E);
 397
 398   ComplexPairTy VisitAtomicExpr(AtomicExpr *E) {
 399     return CGF.EmitAtomicExpr(E).getComplexVal();
 400   }
 401
 402   ComplexPairTy VisitPackIndexingExpr(PackIndexingExpr *E) {
 403     return Visit(E->getSelectedExpr());
 404   }
 405 };
 406 }  // end anonymous namespace.
 407
 408 //===----------------------------------------------------------------------===//
 409 //                                Utilities
 410 //===----------------------------------------------------------------------===//
 411
 412 Address CodeGenFunction::emitAddrOfRealComponent(Address addr,
 413                                                  QualType complexType) {
 414   return Builder.CreateStructGEP(addr, 0, addr.getName() + ".realp");
 415 }
 416
 417 Address CodeGenFunction::emitAddrOfImagComponent(Address addr,
 418                                                  QualType complexType) {
 419   return Builder.CreateStructGEP(addr, 1, addr.getName() + ".imagp");
 420 }
 421
 422 /// EmitLoadOfLValue - Given an RValue reference for a complex, emit code to
 423 /// load the real and imaginary pieces, returning them as Real/Imag.
 424 ComplexPairTy ComplexExprEmitter::EmitLoadOfLValue(LValue lvalue,
 425                                                    SourceLocation loc) {
 426   assert(lvalue.isSimple() && "non-simple complex l-value?");
 427   if (lvalue.getType()->isAtomicType())
 428     return CGF.EmitAtomicLoad(lvalue, loc).getComplexVal();
 429
 430   Address SrcPtr = lvalue.getAddress();
 431   bool isVolatile = lvalue.isVolatileQualified();
 432
 433   llvm::Value *Real = nullptr, *Imag = nullptr;
 434
 435   if (!IgnoreReal || isVolatile) {
 436     Address RealP = CGF.emitAddrOfRealComponent(SrcPtr, lvalue.getType());
 437     Real = Builder.CreateLoad(RealP, isVolatile, SrcPtr.getName() + ".real");
 438   }
 439
 440   if (!IgnoreImag || isVolatile) {
 441     Address ImagP = CGF.emitAddrOfImagComponent(SrcPtr, lvalue.getType());
 442     Imag = Builder.CreateLoad(ImagP, isVolatile, SrcPtr.getName() + ".imag");
 443   }
 444
 445   return ComplexPairTy(Real, Imag);
 446 }
 447
 448 /// EmitStoreOfComplex - Store the specified real/imag parts into the
 449 /// specified value pointer.
 450 void ComplexExprEmitter::EmitStoreOfComplex(ComplexPairTy Val, LValue lvalue,
 451                                             bool isInit) {
 452   if (lvalue.getType()->isAtomicType() ||
 453       (!isInit && CGF.LValueIsSuitableForInlineAtomic(lvalue)))
 454     return CGF.EmitAtomicStore(RValue::getComplex(Val), lvalue, isInit);
 455
 456   Address Ptr = lvalue.getAddress();
 457   Address RealPtr = CGF.emitAddrOfRealComponent(Ptr, lvalue.getType());
 458   Address ImagPtr = CGF.emitAddrOfImagComponent(Ptr, lvalue.getType());
 459
 460   Builder.CreateStore(Val.first, RealPtr, lvalue.isVolatileQualified());
 461   Builder.CreateStore(Val.second, ImagPtr, lvalue.isVolatileQualified());
 462 }
 463
 464
 465
 466 //===----------------------------------------------------------------------===//
 467 //                            Visitor Methods
 468 //===----------------------------------------------------------------------===//
 469
 470 ComplexPairTy ComplexExprEmitter::VisitExpr(Expr *E) {
 471   CGF.ErrorUnsupported(E, "complex expression");
 472   llvm::Type *EltTy =
 473     CGF.ConvertType(getComplexType(E->getType())->getElementType());
 474   llvm::Value *U = llvm::PoisonValue::get(EltTy);
 475   return ComplexPairTy(U, U);
 476 }
 477
 478 ComplexPairTy ComplexExprEmitter::
 479 VisitImaginaryLiteral(const ImaginaryLiteral *IL) {
 480   llvm::Value *Imag = CGF.EmitScalarExpr(IL->getSubExpr());
 481   return ComplexPairTy(llvm::Constant::getNullValue(Imag->getType()), Imag);
 482 }
 483
 484
 485 ComplexPairTy ComplexExprEmitter::VisitCallExpr(const CallExpr *E) {
 486   if (E->getCallReturnType(CGF.getContext())->isReferenceType())
 487     return EmitLoadOfLValue(E);
 488
 489   return CGF.EmitCallExpr(E).getComplexVal();
 490 }
 491
 492 ComplexPairTy ComplexExprEmitter::VisitStmtExpr(const StmtExpr *E) {
 493   CodeGenFunction::StmtExprEvaluation eval(CGF);
 494   Address RetAlloca = CGF.EmitCompoundStmt(*E->getSubStmt(), true);
 495   assert(RetAlloca.isValid() && "Expected complex return value");
 496   return EmitLoadOfLValue(CGF.MakeAddrLValue(RetAlloca, E->getType()),
 497                           E->getExprLoc());
 498 }
 499
 500 /// Emit a cast from complex value Val to DestType.
 501 ComplexPairTy ComplexExprEmitter::EmitComplexToComplexCast(ComplexPairTy Val,
 502                                                            QualType SrcType,
 503                                                            QualType DestType,
 504                                                            SourceLocation Loc) {
 505   // Get the src/dest element type.
 506   SrcType = SrcType->castAs<ComplexType>()->getElementType();
 507   DestType = DestType->castAs<ComplexType>()->getElementType();
 508
 509   // C99 6.3.1.6: When a value of complex type is converted to another
 510   // complex type, both the real and imaginary parts follow the conversion
 511   // rules for the corresponding real types.
 512   if (Val.first)
 513     Val.first = CGF.EmitScalarConversion(Val.first, SrcType, DestType, Loc);
 514   if (Val.second)
 515     Val.second = CGF.EmitScalarConversion(Val.second, SrcType, DestType, Loc);
 516   return Val;
 517 }
 518
 519 ComplexPairTy ComplexExprEmitter::EmitScalarToComplexCast(llvm::Value *Val,
 520                                                           QualType SrcType,
 521                                                           QualType DestType,
 522                                                           SourceLocation Loc) {
 523   // Convert the input element to the element type of the complex.
 524   DestType = DestType->castAs<ComplexType>()->getElementType();
 525   Val = CGF.EmitScalarConversion(Val, SrcType, DestType, Loc);
 526
 527   // Return (realval, 0).
 528   return ComplexPairTy(Val, llvm::Constant::getNullValue(Val->getType()));
 529 }
 530
 531 ComplexPairTy ComplexExprEmitter::EmitCast(CastKind CK, Expr *Op,
 532                                            QualType DestTy) {
 533   switch (CK) {
 534   case CK_Dependent: llvm_unreachable("dependent cast kind in IR gen!");
 535
 536   // Atomic to non-atomic casts may be more than a no-op for some platforms and
 537   // for some types.
 538   case CK_AtomicToNonAtomic:
 539   case CK_NonAtomicToAtomic:
 540   case CK_NoOp:
 541   case CK_LValueToRValue:
 542   case CK_UserDefinedConversion:
 543     return Visit(Op);
 544
 545   case CK_LValueBitCast: {
 546     LValue origLV = CGF.EmitLValue(Op);
 547     Address V = origLV.getAddress().withElementType(CGF.ConvertType(DestTy));
 548     return EmitLoadOfLValue(CGF.MakeAddrLValue(V, DestTy), Op->getExprLoc());
 549   }
 550
 551   case CK_LValueToRValueBitCast: {
 552     LValue SourceLVal = CGF.EmitLValue(Op);
 553     Address Addr =
 554         SourceLVal.getAddress().withElementType(CGF.ConvertTypeForMem(DestTy));
 555     LValue DestLV = CGF.MakeAddrLValue(Addr, DestTy);
 556     DestLV.setTBAAInfo(TBAAAccessInfo::getMayAliasInfo());
 557     return EmitLoadOfLValue(DestLV, Op->getExprLoc());
 558   }
 559
 560   case CK_BitCast:
 561   case CK_BaseToDerived:
 562   case CK_DerivedToBase:
 563   case CK_UncheckedDerivedToBase:
 564   case CK_Dynamic:
 565   case CK_ToUnion:
 566   case CK_ArrayToPointerDecay:
 567   case CK_FunctionToPointerDecay:
 568   case CK_NullToPointer:
 569   case CK_NullToMemberPointer:
 570   case CK_BaseToDerivedMemberPointer:
 571   case CK_DerivedToBaseMemberPointer:
 572   case CK_MemberPointerToBoolean:
 573   case CK_ReinterpretMemberPointer:
 574   case CK_ConstructorConversion:
 575   case CK_IntegralToPointer:
 576   case CK_PointerToIntegral:
 577   case CK_PointerToBoolean:
 578   case CK_ToVoid:
 579   case CK_VectorSplat:
 580   case CK_IntegralCast:
 581   case CK_BooleanToSignedIntegral:
 582   case CK_IntegralToBoolean:
 583   case CK_IntegralToFloating:
 584   case CK_FloatingToIntegral:
 585   case CK_FloatingToBoolean:
 586   case CK_FloatingCast:
 587   case CK_CPointerToObjCPointerCast:
 588   case CK_BlockPointerToObjCPointerCast:
 589   case CK_AnyPointerToBlockPointerCast:
 590   case CK_ObjCObjectLValueCast:
 591   case CK_FloatingComplexToReal:
 592   case CK_FloatingComplexToBoolean:
 593   case CK_IntegralComplexToReal:
 594   case CK_IntegralComplexToBoolean:
 595   case CK_ARCProduceObject:
 596   case CK_ARCConsumeObject:
 597   case CK_ARCReclaimReturnedObject:
 598   case CK_ARCExtendBlockObject:
 599   case CK_CopyAndAutoreleaseBlockObject:
 600   case CK_BuiltinFnToFnPtr:
 601   case CK_ZeroToOCLOpaqueType:
 602   case CK_AddressSpaceConversion:
 603   case CK_IntToOCLSampler:
 604   case CK_FloatingToFixedPoint:
 605   case CK_FixedPointToFloating:
 606   case CK_FixedPointCast:
 607   case CK_FixedPointToBoolean:
 608   case CK_FixedPointToIntegral:
 609   case CK_IntegralToFixedPoint:
 610   case CK_MatrixCast:
 611   case CK_HLSLVectorTruncation:
 612   case CK_HLSLArrayRValue:
 613     llvm_unreachable("invalid cast kind for complex value");
 614
 615   case CK_FloatingRealToComplex:
 616   case CK_IntegralRealToComplex: {
 617     CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, Op);
 618     return EmitScalarToComplexCast(CGF.EmitScalarExpr(Op), Op->getType(),
 619                                    DestTy, Op->getExprLoc());
 620   }
 621
 622   case CK_FloatingComplexCast:
 623   case CK_FloatingComplexToIntegralComplex:
 624   case CK_IntegralComplexCast:
 625   case CK_IntegralComplexToFloatingComplex: {
 626     CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, Op);
 627     return EmitComplexToComplexCast(Visit(Op), Op->getType(), DestTy,
 628                                     Op->getExprLoc());
 629   }
 630   }
 631
 632   llvm_unreachable("unknown cast resulting in complex value");
 633 }
 634
 635 ComplexPairTy ComplexExprEmitter::VisitUnaryPlus(const UnaryOperator *E,
 636                                                  QualType PromotionType) {
 637   QualType promotionTy =
 638       PromotionType.isNull()
 639           ? getPromotionType(E->getStoredFPFeaturesOrDefault(),
 640                              E->getSubExpr()->getType())
 641           : PromotionType;
 642   ComplexPairTy result = VisitPlus(E, promotionTy);
 643   if (!promotionTy.isNull())
 644     return CGF.EmitUnPromotedValue(result, E->getSubExpr()->getType());
 645   return result;
 646 }
 647
 648 ComplexPairTy ComplexExprEmitter::VisitPlus(const UnaryOperator *E,
 649                                             QualType PromotionType) {
 650   TestAndClearIgnoreReal();
 651   TestAndClearIgnoreImag();
 652   if (!PromotionType.isNull())
 653     return CGF.EmitPromotedComplexExpr(E->getSubExpr(), PromotionType);
 654   return Visit(E->getSubExpr());
 655 }
 656
 657 ComplexPairTy ComplexExprEmitter::VisitUnaryMinus(const UnaryOperator *E,
 658                                                   QualType PromotionType) {
 659   QualType promotionTy =
 660       PromotionType.isNull()
 661           ? getPromotionType(E->getStoredFPFeaturesOrDefault(),
 662                              E->getSubExpr()->getType())
 663           : PromotionType;
 664   ComplexPairTy result = VisitMinus(E, promotionTy);
 665   if (!promotionTy.isNull())
 666     return CGF.EmitUnPromotedValue(result, E->getSubExpr()->getType());
 667   return result;
 668 }
 669 ComplexPairTy ComplexExprEmitter::VisitMinus(const UnaryOperator *E,
 670                                              QualType PromotionType) {
 671   TestAndClearIgnoreReal();
 672   TestAndClearIgnoreImag();
 673   ComplexPairTy Op;
 674   if (!PromotionType.isNull())
 675     Op = CGF.EmitPromotedComplexExpr(E->getSubExpr(), PromotionType);
 676   else
 677     Op = Visit(E->getSubExpr());
 678
 679   llvm::Value *ResR, *ResI;
 680   if (Op.first->getType()->isFloatingPointTy()) {
 681     ResR = Builder.CreateFNeg(Op.first,  "neg.r");
 682     ResI = Builder.CreateFNeg(Op.second, "neg.i");
 683   } else {
 684     ResR = Builder.CreateNeg(Op.first,  "neg.r");
 685     ResI = Builder.CreateNeg(Op.second, "neg.i");
 686   }
 687   return ComplexPairTy(ResR, ResI);
 688 }
 689
 690 ComplexPairTy ComplexExprEmitter::VisitUnaryNot(const UnaryOperator *E) {
 691   TestAndClearIgnoreReal();
 692   TestAndClearIgnoreImag();
 693   // ~(a+ib) = a + i*-b
 694   ComplexPairTy Op = Visit(E->getSubExpr());
 695   llvm::Value *ResI;
 696   if (Op.second->getType()->isFloatingPointTy())
 697     ResI = Builder.CreateFNeg(Op.second, "conj.i");
 698   else
 699     ResI = Builder.CreateNeg(Op.second, "conj.i");
 700
 701   return ComplexPairTy(Op.first, ResI);
 702 }
 703
 704 ComplexPairTy ComplexExprEmitter::EmitBinAdd(const BinOpInfo &Op) {
 705   llvm::Value *ResR, *ResI;
 706
 707   if (Op.LHS.first->getType()->isFloatingPointTy()) {
 708     CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, Op.FPFeatures);
 709     ResR = Builder.CreateFAdd(Op.LHS.first,  Op.RHS.first,  "add.r");
 710     if (Op.LHS.second && Op.RHS.second)
 711       ResI = Builder.CreateFAdd(Op.LHS.second, Op.RHS.second, "add.i");
 712     else
 713       ResI = Op.LHS.second ? Op.LHS.second : Op.RHS.second;
 714     assert(ResI && "Only one operand may be real!");
 715   } else {
 716     ResR = Builder.CreateAdd(Op.LHS.first,  Op.RHS.first,  "add.r");
 717     assert(Op.LHS.second && Op.RHS.second &&
 718            "Both operands of integer complex operators must be complex!");
 719     ResI = Builder.CreateAdd(Op.LHS.second, Op.RHS.second, "add.i");
 720   }
 721   return ComplexPairTy(ResR, ResI);
 722 }
 723
 724 ComplexPairTy ComplexExprEmitter::EmitBinSub(const BinOpInfo &Op) {
 725   llvm::Value *ResR, *ResI;
 726   if (Op.LHS.first->getType()->isFloatingPointTy()) {
 727     CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, Op.FPFeatures);
 728     ResR = Builder.CreateFSub(Op.LHS.first, Op.RHS.first, "sub.r");
 729     if (Op.LHS.second && Op.RHS.second)
 730       ResI = Builder.CreateFSub(Op.LHS.second, Op.RHS.second, "sub.i");
 731     else
 732       ResI = Op.LHS.second ? Op.LHS.second
 733                            : Builder.CreateFNeg(Op.RHS.second, "sub.i");
 734     assert(ResI && "Only one operand may be real!");
 735   } else {
 736     ResR = Builder.CreateSub(Op.LHS.first, Op.RHS.first, "sub.r");
 737     assert(Op.LHS.second && Op.RHS.second &&
 738            "Both operands of integer complex operators must be complex!");
 739     ResI = Builder.CreateSub(Op.LHS.second, Op.RHS.second, "sub.i");
 740   }
 741   return ComplexPairTy(ResR, ResI);
 742 }
 743
 744 /// Emit a libcall for a binary operation on complex types.
 745 ComplexPairTy ComplexExprEmitter::EmitComplexBinOpLibCall(StringRef LibCallName,
 746                                                           const BinOpInfo &Op) {
 747   CallArgList Args;
 748   Args.add(RValue::get(Op.LHS.first),
 749            Op.Ty->castAs<ComplexType>()->getElementType());
 750   Args.add(RValue::get(Op.LHS.second),
 751            Op.Ty->castAs<ComplexType>()->getElementType());
 752   Args.add(RValue::get(Op.RHS.first),
 753            Op.Ty->castAs<ComplexType>()->getElementType());
 754   Args.add(RValue::get(Op.RHS.second),
 755            Op.Ty->castAs<ComplexType>()->getElementType());
 756
 757   // We *must* use the full CG function call building logic here because the
 758   // complex type has special ABI handling. We also should not forget about
 759   // special calling convention which may be used for compiler builtins.
 760
 761   // We create a function qualified type to state that this call does not have
 762   // any exceptions.
 763   FunctionProtoType::ExtProtoInfo EPI;
 764   EPI = EPI.withExceptionSpec(
 765       FunctionProtoType::ExceptionSpecInfo(EST_BasicNoexcept));
 766   SmallVector<QualType, 4> ArgsQTys(
 767       4, Op.Ty->castAs<ComplexType>()->getElementType());
 768   QualType FQTy = CGF.getContext().getFunctionType(Op.Ty, ArgsQTys, EPI);
 769   const CGFunctionInfo &FuncInfo = CGF.CGM.getTypes().arrangeFreeFunctionCall(
 770       Args, cast<FunctionType>(FQTy.getTypePtr()), false);
 771
 772   llvm::FunctionType *FTy = CGF.CGM.getTypes().GetFunctionType(FuncInfo);
 773   llvm::FunctionCallee Func = CGF.CGM.CreateRuntimeFunction(
 774       FTy, LibCallName, llvm::AttributeList(), true);
 775   CGCallee Callee = CGCallee::forDirect(Func, FQTy->getAs<FunctionProtoType>());
 776
 777   llvm::CallBase *Call;
 778   RValue Res = CGF.EmitCall(FuncInfo, Callee, ReturnValueSlot(), Args, &Call);
 779   Call->setCallingConv(CGF.CGM.getRuntimeCC());
 780   return Res.getComplexVal();
 781 }
 782
 783 /// Lookup the libcall name for a given floating point type complex
 784 /// multiply.
 785 static StringRef getComplexMultiplyLibCallName(llvm::Type *Ty) {
 786   switch (Ty->getTypeID()) {
 787   default:
 788     llvm_unreachable("Unsupported floating point type!");
 789   case llvm::Type::HalfTyID:
 790     return "__mulhc3";
 791   case llvm::Type::FloatTyID:
 792     return "__mulsc3";
 793   case llvm::Type::DoubleTyID:
 794     return "__muldc3";
 795   case llvm::Type::PPC_FP128TyID:
 796     return "__multc3";
 797   case llvm::Type::X86_FP80TyID:
 798     return "__mulxc3";
 799   case llvm::Type::FP128TyID:
 800     return "__multc3";
 801   }
 802 }
 803
 804 // See C11 Annex G.5.1 for the semantics of multiplicative operators on complex
 805 // typed values.
 806 ComplexPairTy ComplexExprEmitter::EmitBinMul(const BinOpInfo &Op) {
 807   using llvm::Value;
 808   Value *ResR, *ResI;
 809   llvm::MDBuilder MDHelper(CGF.getLLVMContext());
 810
 811   if (Op.LHS.first->getType()->isFloatingPointTy()) {
 812     // The general formulation is:
 813     // (a + ib) * (c + id) = (a * c - b * d) + i(a * d + b * c)
 814     //
 815     // But we can fold away components which would be zero due to a real
 816     // operand according to C11 Annex G.5.1p2.
 817
 818     CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, Op.FPFeatures);
 819     if (Op.LHS.second && Op.RHS.second) {
 820       // If both operands are complex, emit the core math directly, and then
 821       // test for NaNs. If we find NaNs in the result, we delegate to a libcall
 822       // to carefully re-compute the correct infinity representation if
 823       // possible. The expectation is that the presence of NaNs here is
 824       // *extremely* rare, and so the cost of the libcall is almost irrelevant.
 825       // This is good, because the libcall re-computes the core multiplication
 826       // exactly the same as we do here and re-tests for NaNs in order to be
 827       // a generic complex*complex libcall.
 828
 829       // First compute the four products.
 830       Value *AC = Builder.CreateFMul(Op.LHS.first, Op.RHS.first, "mul_ac");
 831       Value *BD = Builder.CreateFMul(Op.LHS.second, Op.RHS.second, "mul_bd");
 832       Value *AD = Builder.CreateFMul(Op.LHS.first, Op.RHS.second, "mul_ad");
 833       Value *BC = Builder.CreateFMul(Op.LHS.second, Op.RHS.first, "mul_bc");
 834
 835       // The real part is the difference of the first two, the imaginary part is
 836       // the sum of the second.
 837       ResR = Builder.CreateFSub(AC, BD, "mul_r");
 838       ResI = Builder.CreateFAdd(AD, BC, "mul_i");
 839
 840       if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Basic ||
 841           Op.FPFeatures.getComplexRange() == LangOptions::CX_Improved ||
 842           Op.FPFeatures.getComplexRange() == LangOptions::CX_Promoted)
 843         return ComplexPairTy(ResR, ResI);
 844
 845       // Emit the test for the real part becoming NaN and create a branch to
 846       // handle it. We test for NaN by comparing the number to itself.
 847       Value *IsRNaN = Builder.CreateFCmpUNO(ResR, ResR, "isnan_cmp");
 848       llvm::BasicBlock *ContBB = CGF.createBasicBlock("complex_mul_cont");
 849       llvm::BasicBlock *INaNBB = CGF.createBasicBlock("complex_mul_imag_nan");
 850       llvm::Instruction *Branch = Builder.CreateCondBr(IsRNaN, INaNBB, ContBB);
 851       llvm::BasicBlock *OrigBB = Branch->getParent();
 852
 853       // Give hint that we very much don't expect to see NaNs.
 854       llvm::MDNode *BrWeight = MDHelper.createUnlikelyBranchWeights();
 855       Branch->setMetadata(llvm::LLVMContext::MD_prof, BrWeight);
 856
 857       // Now test the imaginary part and create its branch.
 858       CGF.EmitBlock(INaNBB);
 859       Value *IsINaN = Builder.CreateFCmpUNO(ResI, ResI, "isnan_cmp");
 860       llvm::BasicBlock *LibCallBB = CGF.createBasicBlock("complex_mul_libcall");
 861       Branch = Builder.CreateCondBr(IsINaN, LibCallBB, ContBB);
 862       Branch->setMetadata(llvm::LLVMContext::MD_prof, BrWeight);
 863
 864       // Now emit the libcall on this slowest of the slow paths.
 865       CGF.EmitBlock(LibCallBB);
 866       Value *LibCallR, *LibCallI;
 867       std::tie(LibCallR, LibCallI) = EmitComplexBinOpLibCall(
 868           getComplexMultiplyLibCallName(Op.LHS.first->getType()), Op);
 869       Builder.CreateBr(ContBB);
 870
 871       // Finally continue execution by phi-ing together the different
 872       // computation paths.
 873       CGF.EmitBlock(ContBB);
 874       llvm::PHINode *RealPHI = Builder.CreatePHI(ResR->getType(), 3, "real_mul_phi");
 875       RealPHI->addIncoming(ResR, OrigBB);
 876       RealPHI->addIncoming(ResR, INaNBB);
 877       RealPHI->addIncoming(LibCallR, LibCallBB);
 878       llvm::PHINode *ImagPHI = Builder.CreatePHI(ResI->getType(), 3, "imag_mul_phi");
 879       ImagPHI->addIncoming(ResI, OrigBB);
 880       ImagPHI->addIncoming(ResI, INaNBB);
 881       ImagPHI->addIncoming(LibCallI, LibCallBB);
 882       return ComplexPairTy(RealPHI, ImagPHI);
 883     }
 884     assert((Op.LHS.second || Op.RHS.second) &&
 885            "At least one operand must be complex!");
 886
 887     // If either of the operands is a real rather than a complex, the
 888     // imaginary component is ignored when computing the real component of the
 889     // result.
 890     ResR = Builder.CreateFMul(Op.LHS.first, Op.RHS.first, "mul.rl");
 891
 892     ResI = Op.LHS.second
 893                ? Builder.CreateFMul(Op.LHS.second, Op.RHS.first, "mul.il")
 894                : Builder.CreateFMul(Op.LHS.first, Op.RHS.second, "mul.ir");
 895   } else {
 896     assert(Op.LHS.second && Op.RHS.second &&
 897            "Both operands of integer complex operators must be complex!");
 898     Value *ResRl = Builder.CreateMul(Op.LHS.first, Op.RHS.first, "mul.rl");
 899     Value *ResRr = Builder.CreateMul(Op.LHS.second, Op.RHS.second, "mul.rr");
 900     ResR = Builder.CreateSub(ResRl, ResRr, "mul.r");
 901
 902     Value *ResIl = Builder.CreateMul(Op.LHS.second, Op.RHS.first, "mul.il");
 903     Value *ResIr = Builder.CreateMul(Op.LHS.first, Op.RHS.second, "mul.ir");
 904     ResI = Builder.CreateAdd(ResIl, ResIr, "mul.i");
 905   }
 906   return ComplexPairTy(ResR, ResI);
 907 }
 908
 909 ComplexPairTy ComplexExprEmitter::EmitAlgebraicDiv(llvm::Value *LHSr,
 910                                                    llvm::Value *LHSi,
 911                                                    llvm::Value *RHSr,
 912                                                    llvm::Value *RHSi) {
 913   // (a+ib) / (c+id) = ((ac+bd)/(cc+dd)) + i((bc-ad)/(cc+dd))
 914   llvm::Value *DSTr, *DSTi;
 915
 916   llvm::Value *AC = Builder.CreateFMul(LHSr, RHSr); // a*c
 917   llvm::Value *BD = Builder.CreateFMul(LHSi, RHSi); // b*d
 918   llvm::Value *ACpBD = Builder.CreateFAdd(AC, BD);  // ac+bd
 919
 920   llvm::Value *CC = Builder.CreateFMul(RHSr, RHSr); // c*c
 921   llvm::Value *DD = Builder.CreateFMul(RHSi, RHSi); // d*d
 922   llvm::Value *CCpDD = Builder.CreateFAdd(CC, DD);  // cc+dd
 923
 924   llvm::Value *BC = Builder.CreateFMul(LHSi, RHSr); // b*c
 925   llvm::Value *AD = Builder.CreateFMul(LHSr, RHSi); // a*d
 926   llvm::Value *BCmAD = Builder.CreateFSub(BC, AD);  // bc-ad
 927
 928   DSTr = Builder.CreateFDiv(ACpBD, CCpDD);
 929   DSTi = Builder.CreateFDiv(BCmAD, CCpDD);
 930   return ComplexPairTy(DSTr, DSTi);
 931 }
 932
 933 // EmitFAbs - Emit a call to @llvm.fabs.
 934 static llvm::Value *EmitllvmFAbs(CodeGenFunction &CGF, llvm::Value *Value) {
 935   llvm::Function *Func =
 936       CGF.CGM.getIntrinsic(llvm::Intrinsic::fabs, Value->getType());
 937   llvm::Value *Call = CGF.Builder.CreateCall(Func, Value);
 938   return Call;
 939 }
 940
 941 // EmitRangeReductionDiv - Implements Smith's algorithm for complex division.
 942 // SMITH, R. L. Algorithm 116: Complex division. Commun. ACM 5, 8 (1962).
 943 ComplexPairTy ComplexExprEmitter::EmitRangeReductionDiv(llvm::Value *LHSr,
 944                                                         llvm::Value *LHSi,
 945                                                         llvm::Value *RHSr,
 946                                                         llvm::Value *RHSi) {
 947   // FIXME: This could eventually be replaced by an LLVM intrinsic to
 948   // avoid this long IR sequence.
 949
 950   // (a + ib) / (c + id) = (e + if)
 951   llvm::Value *FAbsRHSr = EmitllvmFAbs(CGF, RHSr); // |c|
 952   llvm::Value *FAbsRHSi = EmitllvmFAbs(CGF, RHSi); // |d|
 953   // |c| >= |d|
 954   llvm::Value *IsR = Builder.CreateFCmpUGT(FAbsRHSr, FAbsRHSi, "abs_cmp");
 955
 956   llvm::BasicBlock *TrueBB =
 957       CGF.createBasicBlock("abs_rhsr_greater_or_equal_abs_rhsi");
 958   llvm::BasicBlock *FalseBB =
 959       CGF.createBasicBlock("abs_rhsr_less_than_abs_rhsi");
 960   llvm::BasicBlock *ContBB = CGF.createBasicBlock("complex_div");
 961   Builder.CreateCondBr(IsR, TrueBB, FalseBB);
 962
 963   CGF.EmitBlock(TrueBB);
 964   // abs(c) >= abs(d)
 965   // r = d/c
 966   // tmp = c + rd
 967   // e = (a + br)/tmp
 968   // f = (b - ar)/tmp
 969   llvm::Value *DdC = Builder.CreateFDiv(RHSi, RHSr); // r=d/c
 970
 971   llvm::Value *RD = Builder.CreateFMul(DdC, RHSi);  // rd
 972   llvm::Value *CpRD = Builder.CreateFAdd(RHSr, RD); // tmp=c+rd
 973
 974   llvm::Value *T3 = Builder.CreateFMul(LHSi, DdC);   // br
 975   llvm::Value *T4 = Builder.CreateFAdd(LHSr, T3);    // a+br
 976   llvm::Value *DSTTr = Builder.CreateFDiv(T4, CpRD); // (a+br)/tmp
 977
 978   llvm::Value *T5 = Builder.CreateFMul(LHSr, DdC);   // ar
 979   llvm::Value *T6 = Builder.CreateFSub(LHSi, T5);    // b-ar
 980   llvm::Value *DSTTi = Builder.CreateFDiv(T6, CpRD); // (b-ar)/tmp
 981   Builder.CreateBr(ContBB);
 982
 983   CGF.EmitBlock(FalseBB);
 984   // abs(c) < abs(d)
 985   // r = c/d
 986   // tmp = d + rc
 987   // e = (ar + b)/tmp
 988   // f = (br - a)/tmp
 989   llvm::Value *CdD = Builder.CreateFDiv(RHSr, RHSi); // r=c/d
 990
 991   llvm::Value *RC = Builder.CreateFMul(CdD, RHSr);  // rc
 992   llvm::Value *DpRC = Builder.CreateFAdd(RHSi, RC); // tmp=d+rc
 993
 994   llvm::Value *T7 = Builder.CreateFMul(LHSr, CdD);   // ar
 995   llvm::Value *T8 = Builder.CreateFAdd(T7, LHSi);    // ar+b
 996   llvm::Value *DSTFr = Builder.CreateFDiv(T8, DpRC); // (ar+b)/tmp
 997
 998   llvm::Value *T9 = Builder.CreateFMul(LHSi, CdD);    // br
 999   llvm::Value *T10 = Builder.CreateFSub(T9, LHSr);    // br-a
1000   llvm::Value *DSTFi = Builder.CreateFDiv(T10, DpRC); // (br-a)/tmp
1001   Builder.CreateBr(ContBB);
1002
1003   // Phi together the computation paths.
1004   CGF.EmitBlock(ContBB);
1005   llvm::PHINode *VALr = Builder.CreatePHI(DSTTr->getType(), 2);
1006   VALr->addIncoming(DSTTr, TrueBB);
1007   VALr->addIncoming(DSTFr, FalseBB);
1008   llvm::PHINode *VALi = Builder.CreatePHI(DSTTi->getType(), 2);
1009   VALi->addIncoming(DSTTi, TrueBB);
1010   VALi->addIncoming(DSTFi, FalseBB);
1011   return ComplexPairTy(VALr, VALi);
1012 }
1013
1014 // See C11 Annex G.5.1 for the semantics of multiplicative operators on complex
1015 // typed values.
1016 ComplexPairTy ComplexExprEmitter::EmitBinDiv(const BinOpInfo &Op) {
1017   llvm::Value *LHSr = Op.LHS.first, *LHSi = Op.LHS.second;
1018   llvm::Value *RHSr = Op.RHS.first, *RHSi = Op.RHS.second;
1019   llvm::Value *DSTr, *DSTi;
1020   if (LHSr->getType()->isFloatingPointTy()) {
1021     CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, Op.FPFeatures);
1022     if (!RHSi) {
1023       assert(LHSi && "Can have at most one non-complex operand!");
1024
1025       DSTr = Builder.CreateFDiv(LHSr, RHSr);
1026       DSTi = Builder.CreateFDiv(LHSi, RHSr);
1027       return ComplexPairTy(DSTr, DSTi);
1028     }
1029     llvm::Value *OrigLHSi = LHSi;
1030     if (!LHSi)
1031       LHSi = llvm::Constant::getNullValue(RHSi->getType());
1032     if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Improved ||
1033         (Op.FPFeatures.getComplexRange() == LangOptions::CX_Promoted &&
1034          !FPHasBeenPromoted))
1035       return EmitRangeReductionDiv(LHSr, LHSi, RHSr, RHSi);
1036     else if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Basic ||
1037              Op.FPFeatures.getComplexRange() == LangOptions::CX_Promoted)
1038       return EmitAlgebraicDiv(LHSr, LHSi, RHSr, RHSi);
1039     // '-ffast-math' is used in the command line but followed by an
1040     // '-fno-cx-limited-range' or '-fcomplex-arithmetic=full'.
1041     else if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Full) {
1042       LHSi = OrigLHSi;
1043       // If we have a complex operand on the RHS and FastMath is not allowed, we
1044       // delegate to a libcall to handle all of the complexities and minimize
1045       // underflow/overflow cases. When FastMath is allowed we construct the
1046       // divide inline using the same algorithm as for integer operands.
1047       BinOpInfo LibCallOp = Op;
1048       // If LHS was a real, supply a null imaginary part.
1049       if (!LHSi)
1050         LibCallOp.LHS.second = llvm::Constant::getNullValue(LHSr->getType());
1051
1052       switch (LHSr->getType()->getTypeID()) {
1053       default:
1054         llvm_unreachable("Unsupported floating point type!");
1055       case llvm::Type::HalfTyID:
1056         return EmitComplexBinOpLibCall("__divhc3", LibCallOp);
1057       case llvm::Type::FloatTyID:
1058         return EmitComplexBinOpLibCall("__divsc3", LibCallOp);
1059       case llvm::Type::DoubleTyID:
1060         return EmitComplexBinOpLibCall("__divdc3", LibCallOp);
1061       case llvm::Type::PPC_FP128TyID:
1062         return EmitComplexBinOpLibCall("__divtc3", LibCallOp);
1063       case llvm::Type::X86_FP80TyID:
1064         return EmitComplexBinOpLibCall("__divxc3", LibCallOp);
1065       case llvm::Type::FP128TyID:
1066         return EmitComplexBinOpLibCall("__divtc3", LibCallOp);
1067       }
1068     } else {
1069       return EmitAlgebraicDiv(LHSr, LHSi, RHSr, RHSi);
1070     }
1071   } else {
1072     assert(Op.LHS.second && Op.RHS.second &&
1073            "Both operands of integer complex operators must be complex!");
1074     // (a+ib) / (c+id) = ((ac+bd)/(cc+dd)) + i((bc-ad)/(cc+dd))
1075     llvm::Value *Tmp1 = Builder.CreateMul(LHSr, RHSr); // a*c
1076     llvm::Value *Tmp2 = Builder.CreateMul(LHSi, RHSi); // b*d
1077     llvm::Value *Tmp3 = Builder.CreateAdd(Tmp1, Tmp2); // ac+bd
1078
1079     llvm::Value *Tmp4 = Builder.CreateMul(RHSr, RHSr); // c*c
1080     llvm::Value *Tmp5 = Builder.CreateMul(RHSi, RHSi); // d*d
1081     llvm::Value *Tmp6 = Builder.CreateAdd(Tmp4, Tmp5); // cc+dd
1082
1083     llvm::Value *Tmp7 = Builder.CreateMul(LHSi, RHSr); // b*c
1084     llvm::Value *Tmp8 = Builder.CreateMul(LHSr, RHSi); // a*d
1085     llvm::Value *Tmp9 = Builder.CreateSub(Tmp7, Tmp8); // bc-ad
1086
1087     if (Op.Ty->castAs<ComplexType>()->getElementType()->isUnsignedIntegerType()) {
1088       DSTr = Builder.CreateUDiv(Tmp3, Tmp6);
1089       DSTi = Builder.CreateUDiv(Tmp9, Tmp6);
1090     } else {
1091       DSTr = Builder.CreateSDiv(Tmp3, Tmp6);
1092       DSTi = Builder.CreateSDiv(Tmp9, Tmp6);
1093     }
1094   }
1095
1096   return ComplexPairTy(DSTr, DSTi);
1097 }
1098
1099 ComplexPairTy CodeGenFunction::EmitUnPromotedValue(ComplexPairTy result,
1100                                                    QualType UnPromotionType) {
1101   llvm::Type *ComplexElementTy =
1102       ConvertType(UnPromotionType->castAs<ComplexType>()->getElementType());
1103   if (result.first)
1104     result.first =
1105         Builder.CreateFPTrunc(result.first, ComplexElementTy, "unpromotion");
1106   if (result.second)
1107     result.second =
1108         Builder.CreateFPTrunc(result.second, ComplexElementTy, "unpromotion");
1109   return result;
1110 }
1111
1112 ComplexPairTy CodeGenFunction::EmitPromotedValue(ComplexPairTy result,
1113                                                  QualType PromotionType) {
1114   llvm::Type *ComplexElementTy =
1115       ConvertType(PromotionType->castAs<ComplexType>()->getElementType());
1116   if (result.first)
1117     result.first = Builder.CreateFPExt(result.first, ComplexElementTy, "ext");
1118   if (result.second)
1119     result.second = Builder.CreateFPExt(result.second, ComplexElementTy, "ext");
1120
1121   return result;
1122 }
1123
1124 ComplexPairTy ComplexExprEmitter::EmitPromoted(const Expr *E,
1125                                                QualType PromotionType) {
1126   E = E->IgnoreParens();
1127   if (auto BO = dyn_cast<BinaryOperator>(E)) {
1128     switch (BO->getOpcode()) {
1129 #define HANDLE_BINOP(OP)                                                       \
1130   case BO_##OP:                                                                \
1131     return EmitBin##OP(EmitBinOps(BO, PromotionType));
1132       HANDLE_BINOP(Add)
1133       HANDLE_BINOP(Sub)
1134       HANDLE_BINOP(Mul)
1135       HANDLE_BINOP(Div)
1136 #undef HANDLE_BINOP
1137     default:
1138       break;
1139     }
1140   } else if (auto UO = dyn_cast<UnaryOperator>(E)) {
1141     switch (UO->getOpcode()) {
1142     case UO_Minus:
1143       return VisitMinus(UO, PromotionType);
1144     case UO_Plus:
1145       return VisitPlus(UO, PromotionType);
1146     default:
1147       break;
1148     }
1149   }
1150   auto result = Visit(const_cast<Expr *>(E));
1151   if (!PromotionType.isNull())
1152     return CGF.EmitPromotedValue(result, PromotionType);
1153   else
1154     return result;
1155 }
1156
1157 ComplexPairTy CodeGenFunction::EmitPromotedComplexExpr(const Expr *E,
1158                                                        QualType DstTy) {
1159   return ComplexExprEmitter(*this).EmitPromoted(E, DstTy);
1160 }
1161
1162 ComplexPairTy
1163 ComplexExprEmitter::EmitPromotedComplexOperand(const Expr *E,
1164                                                QualType OverallPromotionType) {
1165   if (E->getType()->isAnyComplexType()) {
1166     if (!OverallPromotionType.isNull())
1167       return CGF.EmitPromotedComplexExpr(E, OverallPromotionType);
1168     else
1169       return Visit(const_cast<Expr *>(E));
1170   } else {
1171     if (!OverallPromotionType.isNull()) {
1172       QualType ComplexElementTy =
1173           OverallPromotionType->castAs<ComplexType>()->getElementType();
1174       return ComplexPairTy(CGF.EmitPromotedScalarExpr(E, ComplexElementTy),
1175                            nullptr);
1176     } else {
1177       return ComplexPairTy(CGF.EmitScalarExpr(E), nullptr);
1178     }
1179   }
1180 }
1181
1182 ComplexExprEmitter::BinOpInfo
1183 ComplexExprEmitter::EmitBinOps(const BinaryOperator *E,
1184                                QualType PromotionType) {
1185   TestAndClearIgnoreReal();
1186   TestAndClearIgnoreImag();
1187   BinOpInfo Ops;
1188
1189   Ops.LHS = EmitPromotedComplexOperand(E->getLHS(), PromotionType);
1190   Ops.RHS = EmitPromotedComplexOperand(E->getRHS(), PromotionType);
1191   if (!PromotionType.isNull())
1192     Ops.Ty = PromotionType;
1193   else
1194     Ops.Ty = E->getType();
1195   Ops.FPFeatures = E->getFPFeaturesInEffect(CGF.getLangOpts());
1196   return Ops;
1197 }
1198
1199
1200 LValue ComplexExprEmitter::
1201 EmitCompoundAssignLValue(const CompoundAssignOperator *E,
1202           ComplexPairTy (ComplexExprEmitter::*Func)(const BinOpInfo&),
1203                          RValue &Val) {
1204   TestAndClearIgnoreReal();
1205   TestAndClearIgnoreImag();
1206   QualType LHSTy = E->getLHS()->getType();
1207   if (const AtomicType *AT = LHSTy->getAs<AtomicType>())
1208     LHSTy = AT->getValueType();
1209
1210   BinOpInfo OpInfo;
1211   OpInfo.FPFeatures = E->getFPFeaturesInEffect(CGF.getLangOpts());
1212   CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, OpInfo.FPFeatures);
1213
1214   // Load the RHS and LHS operands.
1215   // __block variables need to have the rhs evaluated first, plus this should
1216   // improve codegen a little.
1217   QualType PromotionTypeCR;
1218   PromotionTypeCR = getPromotionType(E->getStoredFPFeaturesOrDefault(),
1219                                      E->getComputationResultType());
1220   if (PromotionTypeCR.isNull())
1221     PromotionTypeCR = E->getComputationResultType();
1222   OpInfo.Ty = PromotionTypeCR;
1223   QualType ComplexElementTy =
1224       OpInfo.Ty->castAs<ComplexType>()->getElementType();
1225   QualType PromotionTypeRHS = getPromotionType(
1226       E->getStoredFPFeaturesOrDefault(), E->getRHS()->getType());
1227
1228   // The RHS should have been converted to the computation type.
1229   if (E->getRHS()->getType()->isRealFloatingType()) {
1230     if (!PromotionTypeRHS.isNull())
1231       OpInfo.RHS = ComplexPairTy(
1232           CGF.EmitPromotedScalarExpr(E->getRHS(), PromotionTypeRHS), nullptr);
1233     else {
1234       assert(CGF.getContext().hasSameUnqualifiedType(ComplexElementTy,
1235                                                      E->getRHS()->getType()));
1236
1237       OpInfo.RHS = ComplexPairTy(CGF.EmitScalarExpr(E->getRHS()), nullptr);
1238     }
1239   } else {
1240     if (!PromotionTypeRHS.isNull()) {
1241       OpInfo.RHS = ComplexPairTy(
1242           CGF.EmitPromotedComplexExpr(E->getRHS(), PromotionTypeRHS));
1243     } else {
1244       assert(CGF.getContext().hasSameUnqualifiedType(OpInfo.Ty,
1245                                                      E->getRHS()->getType()));
1246       OpInfo.RHS = Visit(E->getRHS());
1247     }
1248   }
1249
1250   LValue LHS = CGF.EmitLValue(E->getLHS());
1251
1252   // Load from the l-value and convert it.
1253   SourceLocation Loc = E->getExprLoc();
1254   QualType PromotionTypeLHS = getPromotionType(
1255       E->getStoredFPFeaturesOrDefault(), E->getComputationLHSType());
1256   if (LHSTy->isAnyComplexType()) {
1257     ComplexPairTy LHSVal = EmitLoadOfLValue(LHS, Loc);
1258     if (!PromotionTypeLHS.isNull())
1259       OpInfo.LHS =
1260           EmitComplexToComplexCast(LHSVal, LHSTy, PromotionTypeLHS, Loc);
1261     else
1262       OpInfo.LHS = EmitComplexToComplexCast(LHSVal, LHSTy, OpInfo.Ty, Loc);
1263   } else {
1264     llvm::Value *LHSVal = CGF.EmitLoadOfScalar(LHS, Loc);
1265     // For floating point real operands we can directly pass the scalar form
1266     // to the binary operator emission and potentially get more efficient code.
1267     if (LHSTy->isRealFloatingType()) {
1268       QualType PromotedComplexElementTy;
1269       if (!PromotionTypeLHS.isNull()) {
1270         PromotedComplexElementTy =
1271             cast<ComplexType>(PromotionTypeLHS)->getElementType();
1272         if (!CGF.getContext().hasSameUnqualifiedType(PromotedComplexElementTy,
1273                                                      PromotionTypeLHS))
1274           LHSVal = CGF.EmitScalarConversion(LHSVal, LHSTy,
1275                                             PromotedComplexElementTy, Loc);
1276       } else {
1277         if (!CGF.getContext().hasSameUnqualifiedType(ComplexElementTy, LHSTy))
1278           LHSVal =
1279               CGF.EmitScalarConversion(LHSVal, LHSTy, ComplexElementTy, Loc);
1280       }
1281       OpInfo.LHS = ComplexPairTy(LHSVal, nullptr);
1282     } else {
1283       OpInfo.LHS = EmitScalarToComplexCast(LHSVal, LHSTy, OpInfo.Ty, Loc);
1284     }
1285   }
1286
1287   // Expand the binary operator.
1288   ComplexPairTy Result = (this->*Func)(OpInfo);
1289
1290   // Truncate the result and store it into the LHS lvalue.
1291   if (LHSTy->isAnyComplexType()) {
1292     ComplexPairTy ResVal =
1293         EmitComplexToComplexCast(Result, OpInfo.Ty, LHSTy, Loc);
1294     EmitStoreOfComplex(ResVal, LHS, /*isInit*/ false);
1295     Val = RValue::getComplex(ResVal);
1296   } else {
1297     llvm::Value *ResVal =
1298         CGF.EmitComplexToScalarConversion(Result, OpInfo.Ty, LHSTy, Loc);
1299     CGF.EmitStoreOfScalar(ResVal, LHS, /*isInit*/ false);
1300     Val = RValue::get(ResVal);
1301   }
1302
1303   return LHS;
1304 }
1305
1306 // Compound assignments.
1307 ComplexPairTy ComplexExprEmitter::
1308 EmitCompoundAssign(const CompoundAssignOperator *E,
1309                    ComplexPairTy (ComplexExprEmitter::*Func)(const BinOpInfo&)){
1310   RValue Val;
1311   LValue LV = EmitCompoundAssignLValue(E, Func, Val);
1312
1313   // The result of an assignment in C is the assigned r-value.
1314   if (!CGF.getLangOpts().CPlusPlus)
1315     return Val.getComplexVal();
1316
1317   // If the lvalue is non-volatile, return the computed value of the assignment.
1318   if (!LV.isVolatileQualified())
1319     return Val.getComplexVal();
1320
1321   return EmitLoadOfLValue(LV, E->getExprLoc());
1322 }
1323
1324 LValue ComplexExprEmitter::EmitBinAssignLValue(const BinaryOperator *E,
1325                                                ComplexPairTy &Val) {
1326   assert(CGF.getContext().hasSameUnqualifiedType(E->getLHS()->getType(),
1327                                                  E->getRHS()->getType()) &&
1328          "Invalid assignment");
1329   TestAndClearIgnoreReal();
1330   TestAndClearIgnoreImag();
1331
1332   // Emit the RHS.  __block variables need the RHS evaluated first.
1333   Val = Visit(E->getRHS());
1334
1335   // Compute the address to store into.
1336   LValue LHS = CGF.EmitLValue(E->getLHS());
1337
1338   // Store the result value into the LHS lvalue.
1339   EmitStoreOfComplex(Val, LHS, /*isInit*/ false);
1340
1341   return LHS;
1342 }
1343
1344 ComplexPairTy ComplexExprEmitter::VisitBinAssign(const BinaryOperator *E) {
1345   ComplexPairTy Val;
1346   LValue LV = EmitBinAssignLValue(E, Val);
1347
1348   // The result of an assignment in C is the assigned r-value.
1349   if (!CGF.getLangOpts().CPlusPlus)
1350     return Val;
1351
1352   // If the lvalue is non-volatile, return the computed value of the assignment.
1353   if (!LV.isVolatileQualified())
1354     return Val;
1355
1356   return EmitLoadOfLValue(LV, E->getExprLoc());
1357 }
1358
1359 ComplexPairTy ComplexExprEmitter::VisitBinComma(const BinaryOperator *E) {
1360   CGF.EmitIgnoredExpr(E->getLHS());
1361   return Visit(E->getRHS());
1362 }
1363
1364 ComplexPairTy ComplexExprEmitter::
1365 VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
1366   TestAndClearIgnoreReal();
1367   TestAndClearIgnoreImag();
1368   llvm::BasicBlock *LHSBlock = CGF.createBasicBlock("cond.true");
1369   llvm::BasicBlock *RHSBlock = CGF.createBasicBlock("cond.false");
1370   llvm::BasicBlock *ContBlock = CGF.createBasicBlock("cond.end");
1371
1372   // Bind the common expression if necessary.
1373   CodeGenFunction::OpaqueValueMapping binding(CGF, E);
1374
1375
1376   CodeGenFunction::ConditionalEvaluation eval(CGF);
1377   CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock,
1378                            CGF.getProfileCount(E));
1379
1380   eval.begin(CGF);
1381   CGF.EmitBlock(LHSBlock);
1382   if (llvm::EnableSingleByteCoverage)
1383     CGF.incrementProfileCounter(E->getTrueExpr());
1384   else
1385     CGF.incrementProfileCounter(E);
1386
1387   ComplexPairTy LHS = Visit(E->getTrueExpr());
1388   LHSBlock = Builder.GetInsertBlock();
1389   CGF.EmitBranch(ContBlock);
1390   eval.end(CGF);
1391
1392   eval.begin(CGF);
1393   CGF.EmitBlock(RHSBlock);
1394   if (llvm::EnableSingleByteCoverage)
1395     CGF.incrementProfileCounter(E->getFalseExpr());
1396   ComplexPairTy RHS = Visit(E->getFalseExpr());
1397   RHSBlock = Builder.GetInsertBlock();
1398   CGF.EmitBlock(ContBlock);
1399   if (llvm::EnableSingleByteCoverage)
1400     CGF.incrementProfileCounter(E);
1401   eval.end(CGF);
1402
1403   // Create a PHI node for the real part.
1404   llvm::PHINode *RealPN = Builder.CreatePHI(LHS.first->getType(), 2, "cond.r");
1405   RealPN->addIncoming(LHS.first, LHSBlock);
1406   RealPN->addIncoming(RHS.first, RHSBlock);
1407
1408   // Create a PHI node for the imaginary part.
1409   llvm::PHINode *ImagPN = Builder.CreatePHI(LHS.first->getType(), 2, "cond.i");
1410   ImagPN->addIncoming(LHS.second, LHSBlock);
1411   ImagPN->addIncoming(RHS.second, RHSBlock);
1412
1413   return ComplexPairTy(RealPN, ImagPN);
1414 }
1415
1416 ComplexPairTy ComplexExprEmitter::VisitChooseExpr(ChooseExpr *E) {
1417   return Visit(E->getChosenSubExpr());
1418 }
1419
1420 ComplexPairTy ComplexExprEmitter::VisitInitListExpr(InitListExpr *E) {
1421     bool Ignore = TestAndClearIgnoreReal();
1422     (void)Ignore;
1423     assert (Ignore == false && "init list ignored");
1424     Ignore = TestAndClearIgnoreImag();
1425     (void)Ignore;
1426     assert (Ignore == false && "init list ignored");
1427
1428   if (E->getNumInits() == 2) {
1429     llvm::Value *Real = CGF.EmitScalarExpr(E->getInit(0));
1430     llvm::Value *Imag = CGF.EmitScalarExpr(E->getInit(1));
1431     return ComplexPairTy(Real, Imag);
1432   } else if (E->getNumInits() == 1) {
1433     return Visit(E->getInit(0));
1434   }
1435
1436   // Empty init list initializes to null
1437   assert(E->getNumInits() == 0 && "Unexpected number of inits");
1438   QualType Ty = E->getType()->castAs<ComplexType>()->getElementType();
1439   llvm::Type* LTy = CGF.ConvertType(Ty);
1440   llvm::Value* zeroConstant = llvm::Constant::getNullValue(LTy);
1441   return ComplexPairTy(zeroConstant, zeroConstant);
1442 }
1443
1444 ComplexPairTy ComplexExprEmitter::VisitVAArgExpr(VAArgExpr *E) {
1445   Address ArgValue = Address::invalid();
1446   RValue RV = CGF.EmitVAArg(E, ArgValue);
1447
1448   if (!ArgValue.isValid()) {
1449     CGF.ErrorUnsupported(E, "complex va_arg expression");
1450     llvm::Type *EltTy =
1451       CGF.ConvertType(E->getType()->castAs<ComplexType>()->getElementType());
1452     llvm::Value *U = llvm::PoisonValue::get(EltTy);
1453     return ComplexPairTy(U, U);
1454   }
1455
1456   return RV.getComplexVal();
1457 }
1458
1459 //===----------------------------------------------------------------------===//
1460 //                         Entry Point into this File
1461 //===----------------------------------------------------------------------===//
1462
1463 /// EmitComplexExpr - Emit the computation of the specified expression of
1464 /// complex type, ignoring the result.
1465 ComplexPairTy CodeGenFunction::EmitComplexExpr(const Expr *E, bool IgnoreReal,
1466                                                bool IgnoreImag) {
1467   assert(E && getComplexType(E->getType()) &&
1468          "Invalid complex expression to emit");
1469
1470   return ComplexExprEmitter(*this, IgnoreReal, IgnoreImag)
1471       .Visit(const_cast<Expr *>(E));
1472 }
1473
1474 void CodeGenFunction::EmitComplexExprIntoLValue(const Expr *E, LValue dest,
1475                                                 bool isInit) {
1476   assert(E && getComplexType(E->getType()) &&
1477          "Invalid complex expression to emit");
1478   ComplexExprEmitter Emitter(*this);
1479   ComplexPairTy Val = Emitter.Visit(const_cast<Expr*>(E));
1480   Emitter.EmitStoreOfComplex(Val, dest, isInit);
1481 }
1482
1483 /// EmitStoreOfComplex - Store a complex number into the specified l-value.
1484 void CodeGenFunction::EmitStoreOfComplex(ComplexPairTy V, LValue dest,
1485                                          bool isInit) {
1486   ComplexExprEmitter(*this).EmitStoreOfComplex(V, dest, isInit);
1487 }
1488
1489 /// EmitLoadOfComplex - Load a complex number from the specified address.
1490 ComplexPairTy CodeGenFunction::EmitLoadOfComplex(LValue src,
1491                                                  SourceLocation loc) {
1492   return ComplexExprEmitter(*this).EmitLoadOfLValue(src, loc);
1493 }
1494
1495 LValue CodeGenFunction::EmitComplexAssignmentLValue(const BinaryOperator *E) {
1496   assert(E->getOpcode() == BO_Assign);
1497   ComplexPairTy Val; // ignored
1498   LValue LVal = ComplexExprEmitter(*this).EmitBinAssignLValue(E, Val);
1499   if (getLangOpts().OpenMP)
1500     CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(*this,
1501                                                               E->getLHS());
1502   return LVal;
1503 }
1504
1505 typedef ComplexPairTy (ComplexExprEmitter::*CompoundFunc)(
1506     const ComplexExprEmitter::BinOpInfo &);
1507
1508 static CompoundFunc getComplexOp(BinaryOperatorKind Op) {
1509   switch (Op) {
1510   case BO_MulAssign: return &ComplexExprEmitter::EmitBinMul;
1511   case BO_DivAssign: return &ComplexExprEmitter::EmitBinDiv;
1512   case BO_SubAssign: return &ComplexExprEmitter::EmitBinSub;
1513   case BO_AddAssign: return &ComplexExprEmitter::EmitBinAdd;
1514   default:
1515     llvm_unreachable("unexpected complex compound assignment");
1516   }
1517 }
1518
1519 LValue CodeGenFunction::
1520 EmitComplexCompoundAssignmentLValue(const CompoundAssignOperator *E) {
1521   CompoundFunc Op = getComplexOp(E->getOpcode());
1522   RValue Val;
1523   return ComplexExprEmitter(*this).EmitCompoundAssignLValue(E, Op, Val);
1524 }
1525
1526 LValue CodeGenFunction::
1527 EmitScalarCompoundAssignWithComplex(const CompoundAssignOperator *E,
1528                                     llvm::Value *&Result) {
1529   CompoundFunc Op = getComplexOp(E->getOpcode());
1530   RValue Val;
1531   LValue Ret = ComplexExprEmitter(*this).EmitCompoundAssignLValue(E, Op, Val);
1532   Result = Val.getScalarVal();
1533   return Ret;
1534 }