clang/lib/AST/Interp/ByteCodeExprGen.cpp

   1 //===--- ByteCodeExprGen.cpp - Code generator for expressions ---*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "ByteCodeExprGen.h"
  10 #include "ByteCodeEmitter.h"
  11 #include "ByteCodeGenError.h"
  12 #include "ByteCodeStmtGen.h"
  13 #include "Context.h"
  14 #include "Floating.h"
  15 #include "Function.h"
  16 #include "PrimType.h"
  17 #include "Program.h"
  18 #include "State.h"
  19
  20 using namespace clang;
  21 using namespace clang::interp;
  22
  23 using APSInt = llvm::APSInt;
  24
  25 namespace clang {
  26 namespace interp {
  27
  28 /// Scope used to handle temporaries in toplevel variable declarations.
  29 template <class Emitter> class DeclScope final : public VariableScope<Emitter> {
  30 public:
  31   DeclScope(ByteCodeExprGen<Emitter> *Ctx, const ValueDecl *VD)
  32       : VariableScope<Emitter>(Ctx), Scope(Ctx->P, VD),
  33         OldGlobalDecl(Ctx->GlobalDecl) {
  34     Ctx->GlobalDecl = Context::shouldBeGloballyIndexed(VD);
  35   }
  36
  37   void addExtended(const Scope::Local &Local) override {
  38     return this->addLocal(Local);
  39   }
  40
  41   ~DeclScope() { this->Ctx->GlobalDecl = OldGlobalDecl; }
  42
  43 private:
  44   Program::DeclScope Scope;
  45   bool OldGlobalDecl;
  46 };
  47
  48 /// Scope used to handle initialization methods.
  49 template <class Emitter> class OptionScope final {
  50 public:
  51   /// Root constructor, compiling or discarding primitives.
  52   OptionScope(ByteCodeExprGen<Emitter> *Ctx, bool NewDiscardResult,
  53               bool NewInitializing)
  54       : Ctx(Ctx), OldDiscardResult(Ctx->DiscardResult),
  55         OldInitializing(Ctx->Initializing) {
  56     Ctx->DiscardResult = NewDiscardResult;
  57     Ctx->Initializing = NewInitializing;
  58   }
  59
  60   ~OptionScope() {
  61     Ctx->DiscardResult = OldDiscardResult;
  62     Ctx->Initializing = OldInitializing;
  63   }
  64
  65 private:
  66   /// Parent context.
  67   ByteCodeExprGen<Emitter> *Ctx;
  68   /// Old discard flag to restore.
  69   bool OldDiscardResult;
  70   bool OldInitializing;
  71 };
  72
  73 } // namespace interp
  74 } // namespace clang
  75
  76 template <class Emitter>
  77 bool ByteCodeExprGen<Emitter>::VisitCastExpr(const CastExpr *CE) {
  78   auto *SubExpr = CE->getSubExpr();
  79   switch (CE->getCastKind()) {
  80
  81   case CK_LValueToRValue: {
  82     return dereference(
  83         SubExpr, DerefKind::Read,
  84         [](PrimType) {
  85           // Value loaded - nothing to do here.
  86           return true;
  87         },
  88         [this, CE](PrimType T) {
  89           // Pointer on stack - dereference it.
  90           if (!this->emitLoadPop(T, CE))
  91             return false;
  92           return DiscardResult ? this->emitPop(T, CE) : true;
  93         });
  94   }
  95
  96   case CK_UncheckedDerivedToBase:
  97   case CK_DerivedToBase: {
  98     if (!this->visit(SubExpr))
  99       return false;
 100
 101     unsigned DerivedOffset = collectBaseOffset(getRecordTy(CE->getType()),
 102                                                getRecordTy(SubExpr->getType()));
 103
 104     return this->emitGetPtrBasePop(DerivedOffset, CE);
 105   }
 106
 107   case CK_BaseToDerived: {
 108     if (!this->visit(SubExpr))
 109       return false;
 110
 111     unsigned DerivedOffset = collectBaseOffset(getRecordTy(SubExpr->getType()),
 112                                                getRecordTy(CE->getType()));
 113
 114     return this->emitGetPtrDerivedPop(DerivedOffset, CE);
 115   }
 116
 117   case CK_FloatingCast: {
 118     if (!this->visit(SubExpr))
 119       return false;
 120     const auto *TargetSemantics = &Ctx.getFloatSemantics(CE->getType());
 121     return this->emitCastFP(TargetSemantics, getRoundingMode(CE), CE);
 122   }
 123
 124   case CK_IntegralToFloating: {
 125     std::optional<PrimType> FromT = classify(SubExpr->getType());
 126     if (!FromT)
 127       return false;
 128
 129     if (!this->visit(SubExpr))
 130       return false;
 131
 132     const auto *TargetSemantics = &Ctx.getFloatSemantics(CE->getType());
 133     llvm::RoundingMode RM = getRoundingMode(CE);
 134     return this->emitCastIntegralFloating(*FromT, TargetSemantics, RM, CE);
 135   }
 136
 137   case CK_FloatingToBoolean:
 138   case CK_FloatingToIntegral: {
 139     std::optional<PrimType> ToT = classify(CE->getType());
 140
 141     if (!ToT)
 142       return false;
 143
 144     if (!this->visit(SubExpr))
 145       return false;
 146
 147     if (ToT == PT_IntAP)
 148       return this->emitCastFloatingIntegralAP(Ctx.getBitWidth(CE->getType()),
 149                                               CE);
 150     if (ToT == PT_IntAPS)
 151       return this->emitCastFloatingIntegralAPS(Ctx.getBitWidth(CE->getType()),
 152                                                CE);
 153
 154     return this->emitCastFloatingIntegral(*ToT, CE);
 155   }
 156
 157   case CK_NullToPointer:
 158     if (DiscardResult)
 159       return true;
 160     return this->emitNull(classifyPrim(CE->getType()), CE);
 161
 162   case CK_PointerToIntegral: {
 163     // TODO: Discard handling.
 164     if (!this->visit(SubExpr))
 165       return false;
 166
 167     PrimType T = classifyPrim(CE->getType());
 168     return this->emitCastPointerIntegral(T, CE);
 169   }
 170
 171   case CK_ArrayToPointerDecay: {
 172     if (!this->visit(SubExpr))
 173       return false;
 174     if (!this->emitArrayDecay(CE))
 175       return false;
 176     if (DiscardResult)
 177       return this->emitPopPtr(CE);
 178     return true;
 179   }
 180
 181   case CK_AtomicToNonAtomic:
 182   case CK_ConstructorConversion:
 183   case CK_FunctionToPointerDecay:
 184   case CK_NonAtomicToAtomic:
 185   case CK_NoOp:
 186   case CK_UserDefinedConversion:
 187   case CK_BitCast:
 188     return this->delegate(SubExpr);
 189
 190   case CK_IntegralToBoolean:
 191   case CK_IntegralCast: {
 192     if (DiscardResult)
 193       return this->discard(SubExpr);
 194     std::optional<PrimType> FromT = classify(SubExpr->getType());
 195     std::optional<PrimType> ToT = classify(CE->getType());
 196
 197     if (!FromT || !ToT)
 198       return false;
 199
 200     if (!this->visit(SubExpr))
 201       return false;
 202
 203     if (FromT == ToT) {
 204       assert(ToT != PT_IntAP && ToT != PT_IntAPS);
 205       return true;
 206     }
 207
 208     if (ToT == PT_IntAP)
 209       return this->emitCastAP(*FromT, Ctx.getBitWidth(CE->getType()), CE);
 210     if (ToT == PT_IntAPS)
 211       return this->emitCastAPS(*FromT, Ctx.getBitWidth(CE->getType()), CE);
 212
 213     return this->emitCast(*FromT, *ToT, CE);
 214   }
 215
 216   case CK_PointerToBoolean: {
 217     PrimType PtrT = classifyPrim(SubExpr->getType());
 218
 219     // Just emit p != nullptr for this.
 220     if (!this->visit(SubExpr))
 221       return false;
 222
 223     if (!this->emitNull(PtrT, CE))
 224       return false;
 225
 226     return this->emitNE(PtrT, CE);
 227   }
 228
 229   case CK_ToVoid:
 230     return discard(SubExpr);
 231
 232   default:
 233     assert(false && "Cast not implemented");
 234   }
 235   llvm_unreachable("Unhandled clang::CastKind enum");
 236 }
 237
 238 template <class Emitter>
 239 bool ByteCodeExprGen<Emitter>::VisitIntegerLiteral(const IntegerLiteral *LE) {
 240   if (DiscardResult)
 241     return true;
 242
 243   return this->emitConst(LE->getValue(), LE);
 244 }
 245
 246 template <class Emitter>
 247 bool ByteCodeExprGen<Emitter>::VisitFloatingLiteral(const FloatingLiteral *E) {
 248   if (DiscardResult)
 249     return true;
 250
 251   return this->emitConstFloat(E->getValue(), E);
 252 }
 253
 254 template <class Emitter>
 255 bool ByteCodeExprGen<Emitter>::VisitParenExpr(const ParenExpr *E) {
 256   return this->delegate(E->getSubExpr());
 257 }
 258
 259 template <class Emitter>
 260 bool ByteCodeExprGen<Emitter>::VisitBinaryOperator(const BinaryOperator *BO) {
 261   // Need short-circuiting for these.
 262   if (BO->isLogicalOp())
 263     return this->VisitLogicalBinOp(BO);
 264
 265   const Expr *LHS = BO->getLHS();
 266   const Expr *RHS = BO->getRHS();
 267
 268   if (BO->isPtrMemOp())
 269     return this->visit(RHS);
 270
 271   // Typecheck the args.
 272   std::optional<PrimType> LT = classify(LHS->getType());
 273   std::optional<PrimType> RT = classify(RHS->getType());
 274   std::optional<PrimType> T = classify(BO->getType());
 275
 276   // Deal with operations which have composite or void types.
 277   if (BO->isCommaOp()) {
 278     if (!this->discard(LHS))
 279       return false;
 280     if (RHS->getType()->isVoidType())
 281       return this->discard(RHS);
 282
 283     return this->delegate(RHS);
 284   }
 285
 286   // Special case for C++'s three-way/spaceship operator <=>, which
 287   // returns a std::{strong,weak,partial}_ordering (which is a class, so doesn't
 288   // have a PrimType).
 289   if (!T) {
 290     if (DiscardResult)
 291       return true;
 292     const ComparisonCategoryInfo *CmpInfo =
 293         Ctx.getASTContext().CompCategories.lookupInfoForType(BO->getType());
 294     assert(CmpInfo);
 295
 296     // We need a temporary variable holding our return value.
 297     if (!Initializing) {
 298       std::optional<unsigned> ResultIndex = this->allocateLocal(BO, false);
 299       if (!this->emitGetPtrLocal(*ResultIndex, BO))
 300         return false;
 301     }
 302
 303     if (!visit(LHS) || !visit(RHS))
 304       return false;
 305
 306     return this->emitCMP3(*LT, CmpInfo, BO);
 307   }
 308
 309   if (!LT || !RT || !T)
 310     return this->bail(BO);
 311
 312   // Pointer arithmetic special case.
 313   if (BO->getOpcode() == BO_Add || BO->getOpcode() == BO_Sub) {
 314     if (T == PT_Ptr || (LT == PT_Ptr && RT == PT_Ptr))
 315       return this->VisitPointerArithBinOp(BO);
 316   }
 317
 318   if (!visit(LHS) || !visit(RHS))
 319     return false;
 320
 321   // For languages such as C, cast the result of one
 322   // of our comparision opcodes to T (which is usually int).
 323   auto MaybeCastToBool = [this, T, BO](bool Result) {
 324     if (!Result)
 325       return false;
 326     if (DiscardResult)
 327       return this->emitPop(*T, BO);
 328     if (T != PT_Bool)
 329       return this->emitCast(PT_Bool, *T, BO);
 330     return true;
 331   };
 332
 333   auto Discard = [this, T, BO](bool Result) {
 334     if (!Result)
 335       return false;
 336     return DiscardResult ? this->emitPop(*T, BO) : true;
 337   };
 338
 339   switch (BO->getOpcode()) {
 340   case BO_EQ:
 341     return MaybeCastToBool(this->emitEQ(*LT, BO));
 342   case BO_NE:
 343     return MaybeCastToBool(this->emitNE(*LT, BO));
 344   case BO_LT:
 345     return MaybeCastToBool(this->emitLT(*LT, BO));
 346   case BO_LE:
 347     return MaybeCastToBool(this->emitLE(*LT, BO));
 348   case BO_GT:
 349     return MaybeCastToBool(this->emitGT(*LT, BO));
 350   case BO_GE:
 351     return MaybeCastToBool(this->emitGE(*LT, BO));
 352   case BO_Sub:
 353     if (BO->getType()->isFloatingType())
 354       return Discard(this->emitSubf(getRoundingMode(BO), BO));
 355     return Discard(this->emitSub(*T, BO));
 356   case BO_Add:
 357     if (BO->getType()->isFloatingType())
 358       return Discard(this->emitAddf(getRoundingMode(BO), BO));
 359     return Discard(this->emitAdd(*T, BO));
 360   case BO_Mul:
 361     if (BO->getType()->isFloatingType())
 362       return Discard(this->emitMulf(getRoundingMode(BO), BO));
 363     return Discard(this->emitMul(*T, BO));
 364   case BO_Rem:
 365     return Discard(this->emitRem(*T, BO));
 366   case BO_Div:
 367     if (BO->getType()->isFloatingType())
 368       return Discard(this->emitDivf(getRoundingMode(BO), BO));
 369     return Discard(this->emitDiv(*T, BO));
 370   case BO_Assign:
 371     if (DiscardResult)
 372       return LHS->refersToBitField() ? this->emitStoreBitFieldPop(*T, BO)
 373                                      : this->emitStorePop(*T, BO);
 374     return LHS->refersToBitField() ? this->emitStoreBitField(*T, BO)
 375                                    : this->emitStore(*T, BO);
 376   case BO_And:
 377     return Discard(this->emitBitAnd(*T, BO));
 378   case BO_Or:
 379     return Discard(this->emitBitOr(*T, BO));
 380   case BO_Shl:
 381     return Discard(this->emitShl(*LT, *RT, BO));
 382   case BO_Shr:
 383     return Discard(this->emitShr(*LT, *RT, BO));
 384   case BO_Xor:
 385     return Discard(this->emitBitXor(*T, BO));
 386   case BO_LOr:
 387   case BO_LAnd:
 388     llvm_unreachable("Already handled earlier");
 389   default:
 390     return this->bail(BO);
 391   }
 392
 393   llvm_unreachable("Unhandled binary op");
 394 }
 395
 396 /// Perform addition/subtraction of a pointer and an integer or
 397 /// subtraction of two pointers.
 398 template <class Emitter>
 399 bool ByteCodeExprGen<Emitter>::VisitPointerArithBinOp(const BinaryOperator *E) {
 400   BinaryOperatorKind Op = E->getOpcode();
 401   const Expr *LHS = E->getLHS();
 402   const Expr *RHS = E->getRHS();
 403
 404   if ((Op != BO_Add && Op != BO_Sub) ||
 405       (!LHS->getType()->isPointerType() && !RHS->getType()->isPointerType()))
 406     return false;
 407
 408   std::optional<PrimType> LT = classify(LHS);
 409   std::optional<PrimType> RT = classify(RHS);
 410
 411   if (!LT || !RT)
 412     return false;
 413
 414   if (LHS->getType()->isPointerType() && RHS->getType()->isPointerType()) {
 415     if (Op != BO_Sub)
 416       return false;
 417
 418     assert(E->getType()->isIntegerType());
 419     if (!visit(RHS) || !visit(LHS))
 420       return false;
 421
 422     return this->emitSubPtr(classifyPrim(E->getType()), E);
 423   }
 424
 425   PrimType OffsetType;
 426   if (LHS->getType()->isIntegerType()) {
 427     if (!visit(RHS) || !visit(LHS))
 428       return false;
 429     OffsetType = *LT;
 430   } else if (RHS->getType()->isIntegerType()) {
 431     if (!visit(LHS) || !visit(RHS))
 432       return false;
 433     OffsetType = *RT;
 434   } else {
 435     return false;
 436   }
 437
 438   if (Op == BO_Add)
 439     return this->emitAddOffset(OffsetType, E);
 440   else if (Op == BO_Sub)
 441     return this->emitSubOffset(OffsetType, E);
 442
 443   return this->bail(E);
 444 }
 445
 446 template <class Emitter>
 447 bool ByteCodeExprGen<Emitter>::VisitLogicalBinOp(const BinaryOperator *E) {
 448   assert(E->isLogicalOp());
 449   BinaryOperatorKind Op = E->getOpcode();
 450   const Expr *LHS = E->getLHS();
 451   const Expr *RHS = E->getRHS();
 452   std::optional<PrimType> T = classify(E->getType());
 453
 454   if (Op == BO_LOr) {
 455     // Logical OR. Visit LHS and only evaluate RHS if LHS was FALSE.
 456     LabelTy LabelTrue = this->getLabel();
 457     LabelTy LabelEnd = this->getLabel();
 458
 459     if (!this->visitBool(LHS))
 460       return false;
 461     if (!this->jumpTrue(LabelTrue))
 462       return false;
 463
 464     if (!this->visitBool(RHS))
 465       return false;
 466     if (!this->jump(LabelEnd))
 467       return false;
 468
 469     this->emitLabel(LabelTrue);
 470     this->emitConstBool(true, E);
 471     this->fallthrough(LabelEnd);
 472     this->emitLabel(LabelEnd);
 473
 474   } else {
 475     assert(Op == BO_LAnd);
 476     // Logical AND.
 477     // Visit LHS. Only visit RHS if LHS was TRUE.
 478     LabelTy LabelFalse = this->getLabel();
 479     LabelTy LabelEnd = this->getLabel();
 480
 481     if (!this->visitBool(LHS))
 482       return false;
 483     if (!this->jumpFalse(LabelFalse))
 484       return false;
 485
 486     if (!this->visitBool(RHS))
 487       return false;
 488     if (!this->jump(LabelEnd))
 489       return false;
 490
 491     this->emitLabel(LabelFalse);
 492     this->emitConstBool(false, E);
 493     this->fallthrough(LabelEnd);
 494     this->emitLabel(LabelEnd);
 495   }
 496
 497   if (DiscardResult)
 498     return this->emitPopBool(E);
 499
 500   // For C, cast back to integer type.
 501   assert(T);
 502   if (T != PT_Bool)
 503     return this->emitCast(PT_Bool, *T, E);
 504   return true;
 505 }
 506
 507 template <class Emitter>
 508 bool ByteCodeExprGen<Emitter>::VisitImplicitValueInitExpr(const ImplicitValueInitExpr *E) {
 509   QualType QT = E->getType();
 510
 511   if (std::optional<PrimType> T = classify(QT))
 512     return this->visitZeroInitializer(*T, QT, E);
 513
 514   if (QT->isRecordType())
 515     return false;
 516
 517   if (QT->isIncompleteArrayType())
 518     return true;
 519
 520   if (QT->isArrayType()) {
 521     const ArrayType *AT = QT->getAsArrayTypeUnsafe();
 522     assert(AT);
 523     const auto *CAT = cast<ConstantArrayType>(AT);
 524     size_t NumElems = CAT->getSize().getZExtValue();
 525     PrimType ElemT = classifyPrim(CAT->getElementType());
 526
 527     for (size_t I = 0; I != NumElems; ++I) {
 528       if (!this->visitZeroInitializer(ElemT, CAT->getElementType(), E))
 529         return false;
 530       if (!this->emitInitElem(ElemT, I, E))
 531         return false;
 532     }
 533
 534     return true;
 535   }
 536
 537   return false;
 538 }
 539
 540 template <class Emitter>
 541 bool ByteCodeExprGen<Emitter>::VisitArraySubscriptExpr(
 542     const ArraySubscriptExpr *E) {
 543   const Expr *Base = E->getBase();
 544   const Expr *Index = E->getIdx();
 545
 546   if (DiscardResult)
 547     return this->discard(Base) && this->discard(Index);
 548
 549   // Take pointer of LHS, add offset from RHS.
 550   // What's left on the stack after this is a pointer.
 551   if (!this->visit(Base))
 552     return false;
 553
 554   if (!this->visit(Index))
 555     return false;
 556
 557   PrimType IndexT = classifyPrim(Index->getType());
 558   return this->emitArrayElemPtrPop(IndexT, E);
 559 }
 560
 561 template <class Emitter>
 562 bool ByteCodeExprGen<Emitter>::visitInitList(ArrayRef<const Expr *> Inits,
 563                                              const Expr *E) {
 564   assert(E->getType()->isRecordType());
 565   const Record *R = getRecord(E->getType());
 566
 567   unsigned InitIndex = 0;
 568   for (const Expr *Init : Inits) {
 569     if (!this->emitDupPtr(E))
 570       return false;
 571
 572     if (std::optional<PrimType> T = classify(Init)) {
 573       const Record::Field *FieldToInit = R->getField(InitIndex);
 574       if (!this->visit(Init))
 575         return false;
 576
 577       if (FieldToInit->isBitField()) {
 578         if (!this->emitInitBitField(*T, FieldToInit, E))
 579           return false;
 580       } else {
 581         if (!this->emitInitField(*T, FieldToInit->Offset, E))
 582           return false;
 583       }
 584
 585       if (!this->emitPopPtr(E))
 586         return false;
 587       ++InitIndex;
 588     } else {
 589       // Initializer for a direct base class.
 590       if (const Record::Base *B = R->getBase(Init->getType())) {
 591         if (!this->emitGetPtrBasePop(B->Offset, Init))
 592           return false;
 593
 594         if (!this->visitInitializer(Init))
 595           return false;
 596
 597         if (!this->emitInitPtrPop(E))
 598           return false;
 599         // Base initializers don't increase InitIndex, since they don't count
 600         // into the Record's fields.
 601       } else {
 602         const Record::Field *FieldToInit = R->getField(InitIndex);
 603         // Non-primitive case. Get a pointer to the field-to-initialize
 604         // on the stack and recurse into visitInitializer().
 605         if (!this->emitGetPtrField(FieldToInit->Offset, Init))
 606           return false;
 607
 608         if (!this->visitInitializer(Init))
 609           return false;
 610
 611         if (!this->emitPopPtr(E))
 612           return false;
 613         ++InitIndex;
 614       }
 615     }
 616   }
 617   return true;
 618 }
 619
 620 template <class Emitter>
 621 bool ByteCodeExprGen<Emitter>::VisitInitListExpr(const InitListExpr *E) {
 622   // Handle discarding first.
 623   if (DiscardResult) {
 624     for (const Expr *Init : E->inits()) {
 625       if (!this->discard(Init))
 626         return false;
 627     }
 628     return true;
 629   }
 630
 631   // Primitive values.
 632   if (std::optional<PrimType> T = classify(E->getType())) {
 633     assert(!DiscardResult);
 634     if (E->getNumInits() == 0)
 635       return this->visitZeroInitializer(*T, E->getType(), E);
 636     assert(E->getNumInits() == 1);
 637     return this->delegate(E->inits()[0]);
 638   }
 639
 640   QualType T = E->getType();
 641   if (T->isRecordType())
 642     return this->visitInitList(E->inits(), E);
 643
 644   if (T->isArrayType()) {
 645     // FIXME: Array fillers.
 646     unsigned ElementIndex = 0;
 647     for (const Expr *Init : E->inits()) {
 648       if (std::optional<PrimType> T = classify(Init->getType())) {
 649         // Visit the primitive element like normal.
 650         if (!this->visit(Init))
 651           return false;
 652         if (!this->emitInitElem(*T, ElementIndex, Init))
 653           return false;
 654       } else {
 655         // Advance the pointer currently on the stack to the given
 656         // dimension.
 657         if (!this->emitConstUint32(ElementIndex, Init))
 658           return false;
 659         if (!this->emitArrayElemPtrUint32(Init))
 660           return false;
 661         if (!this->visitInitializer(Init))
 662           return false;
 663         if (!this->emitPopPtr(Init))
 664           return false;
 665       }
 666
 667       ++ElementIndex;
 668     }
 669     return true;
 670   }
 671
 672   return false;
 673 }
 674
 675 template <class Emitter>
 676 bool ByteCodeExprGen<Emitter>::VisitCXXParenListInitExpr(
 677     const CXXParenListInitExpr *E) {
 678   if (DiscardResult) {
 679     for (const Expr *Init : E->getInitExprs()) {
 680       if (!this->discard(Init))
 681         return false;
 682     }
 683     return true;
 684   }
 685
 686   assert(E->getType()->isRecordType());
 687   return this->visitInitList(E->getInitExprs(), E);
 688 }
 689
 690 template <class Emitter>
 691 bool ByteCodeExprGen<Emitter>::VisitSubstNonTypeTemplateParmExpr(
 692     const SubstNonTypeTemplateParmExpr *E) {
 693   return this->delegate(E->getReplacement());
 694 }
 695
 696 template <class Emitter>
 697 bool ByteCodeExprGen<Emitter>::VisitConstantExpr(const ConstantExpr *E) {
 698   // Try to emit the APValue directly, without visiting the subexpr.
 699   // This will only fail if we can't emit the APValue, so won't emit any
 700   // diagnostics or any double values.
 701   std::optional<PrimType> T = classify(E->getType());
 702   if (T && E->hasAPValueResult() &&
 703       this->visitAPValue(E->getAPValueResult(), *T, E))
 704     return true;
 705
 706   return this->delegate(E->getSubExpr());
 707 }
 708
 709 static CharUnits AlignOfType(QualType T, const ASTContext &ASTCtx,
 710                              UnaryExprOrTypeTrait Kind) {
 711   bool AlignOfReturnsPreferred =
 712       ASTCtx.getLangOpts().getClangABICompat() <= LangOptions::ClangABI::Ver7;
 713
 714   // C++ [expr.alignof]p3:
 715   //     When alignof is applied to a reference type, the result is the
 716   //     alignment of the referenced type.
 717   if (const auto *Ref = T->getAs<ReferenceType>())
 718     T = Ref->getPointeeType();
 719
 720   // __alignof is defined to return the preferred alignment.
 721   // Before 8, clang returned the preferred alignment for alignof and
 722   // _Alignof as well.
 723   if (Kind == UETT_PreferredAlignOf || AlignOfReturnsPreferred)
 724     return ASTCtx.toCharUnitsFromBits(ASTCtx.getPreferredTypeAlign(T));
 725
 726   return ASTCtx.getTypeAlignInChars(T);
 727 }
 728
 729 template <class Emitter>
 730 bool ByteCodeExprGen<Emitter>::VisitUnaryExprOrTypeTraitExpr(
 731     const UnaryExprOrTypeTraitExpr *E) {
 732   UnaryExprOrTypeTrait Kind = E->getKind();
 733   ASTContext &ASTCtx = Ctx.getASTContext();
 734
 735   if (Kind == UETT_SizeOf) {
 736     QualType ArgType = E->getTypeOfArgument();
 737     CharUnits Size;
 738     if (ArgType->isVoidType() || ArgType->isFunctionType())
 739       Size = CharUnits::One();
 740     else {
 741       if (ArgType->isDependentType() || !ArgType->isConstantSizeType())
 742         return false;
 743
 744       Size = ASTCtx.getTypeSizeInChars(ArgType);
 745     }
 746
 747     if (DiscardResult)
 748       return true;
 749
 750     return this->emitConst(Size.getQuantity(), E);
 751   }
 752
 753   if (Kind == UETT_AlignOf || Kind == UETT_PreferredAlignOf) {
 754     CharUnits Size;
 755
 756     if (E->isArgumentType()) {
 757       QualType ArgType = E->getTypeOfArgument();
 758
 759       Size = AlignOfType(ArgType, ASTCtx, Kind);
 760     } else {
 761       // Argument is an expression, not a type.
 762       const Expr *Arg = E->getArgumentExpr()->IgnoreParens();
 763
 764       // The kinds of expressions that we have special-case logic here for
 765       // should be kept up to date with the special checks for those
 766       // expressions in Sema.
 767
 768       // alignof decl is always accepted, even if it doesn't make sense: we
 769       // default to 1 in those cases.
 770       if (const auto *DRE = dyn_cast<DeclRefExpr>(Arg))
 771         Size = ASTCtx.getDeclAlign(DRE->getDecl(),
 772                                    /*RefAsPointee*/ true);
 773       else if (const auto *ME = dyn_cast<MemberExpr>(Arg))
 774         Size = ASTCtx.getDeclAlign(ME->getMemberDecl(),
 775                                    /*RefAsPointee*/ true);
 776       else
 777         Size = AlignOfType(Arg->getType(), ASTCtx, Kind);
 778     }
 779
 780     if (DiscardResult)
 781       return true;
 782
 783     return this->emitConst(Size.getQuantity(), E);
 784   }
 785
 786   return false;
 787 }
 788
 789 template <class Emitter>
 790 bool ByteCodeExprGen<Emitter>::VisitMemberExpr(const MemberExpr *E) {
 791   // 'Base.Member'
 792   const Expr *Base = E->getBase();
 793
 794   if (DiscardResult)
 795     return this->discard(Base);
 796
 797   if (!this->visit(Base))
 798     return false;
 799
 800   // Base above gives us a pointer on the stack.
 801   // TODO: Implement non-FieldDecl members.
 802   const ValueDecl *Member = E->getMemberDecl();
 803   if (const auto *FD = dyn_cast<FieldDecl>(Member)) {
 804     const RecordDecl *RD = FD->getParent();
 805     const Record *R = getRecord(RD);
 806     const Record::Field *F = R->getField(FD);
 807     // Leave a pointer to the field on the stack.
 808     if (F->Decl->getType()->isReferenceType())
 809       return this->emitGetFieldPop(PT_Ptr, F->Offset, E);
 810     return this->emitGetPtrField(F->Offset, E);
 811   }
 812
 813   return false;
 814 }
 815
 816 template <class Emitter>
 817 bool ByteCodeExprGen<Emitter>::VisitArrayInitIndexExpr(
 818     const ArrayInitIndexExpr *E) {
 819   // ArrayIndex might not be set if a ArrayInitIndexExpr is being evaluated
 820   // stand-alone, e.g. via EvaluateAsInt().
 821   if (!ArrayIndex)
 822     return false;
 823   return this->emitConst(*ArrayIndex, E);
 824 }
 825
 826 template <class Emitter>
 827 bool ByteCodeExprGen<Emitter>::VisitArrayInitLoopExpr(
 828     const ArrayInitLoopExpr *E) {
 829   assert(Initializing);
 830   assert(!DiscardResult);
 831   // TODO: This compiles to quite a lot of bytecode if the array is larger.
 832   //   Investigate compiling this to a loop.
 833
 834   const Expr *SubExpr = E->getSubExpr();
 835   const Expr *CommonExpr = E->getCommonExpr();
 836   size_t Size = E->getArraySize().getZExtValue();
 837   std::optional<PrimType> ElemT = classify(SubExpr->getType());
 838
 839   // If the common expression is an opaque expression, we visit it
 840   // here once so we have its value cached.
 841   // FIXME: This might be necessary (or useful) for all expressions.
 842   if (isa<OpaqueValueExpr>(CommonExpr)) {
 843     if (!this->discard(CommonExpr))
 844       return false;
 845   }
 846
 847   // So, every iteration, we execute an assignment here
 848   // where the LHS is on the stack (the target array)
 849   // and the RHS is our SubExpr.
 850   for (size_t I = 0; I != Size; ++I) {
 851     ArrayIndexScope<Emitter> IndexScope(this, I);
 852     BlockScope<Emitter> BS(this);
 853
 854     if (ElemT) {
 855       if (!this->visit(SubExpr))
 856         return false;
 857       if (!this->emitInitElem(*ElemT, I, E))
 858         return false;
 859     } else {
 860       // Get to our array element and recurse into visitInitializer()
 861       if (!this->emitConstUint64(I, SubExpr))
 862         return false;
 863       if (!this->emitArrayElemPtrUint64(SubExpr))
 864         return false;
 865       if (!visitInitializer(SubExpr))
 866         return false;
 867       if (!this->emitPopPtr(E))
 868         return false;
 869     }
 870   }
 871   return true;
 872 }
 873
 874 template <class Emitter>
 875 bool ByteCodeExprGen<Emitter>::VisitOpaqueValueExpr(const OpaqueValueExpr *E) {
 876   if (Initializing)
 877     return this->visitInitializer(E->getSourceExpr());
 878
 879   PrimType SubExprT = classify(E->getSourceExpr()).value_or(PT_Ptr);
 880   if (auto It = OpaqueExprs.find(E); It != OpaqueExprs.end())
 881     return this->emitGetLocal(SubExprT, It->second, E);
 882
 883   if (!this->visit(E->getSourceExpr()))
 884     return false;
 885
 886   // At this point we either have the evaluated source expression or a pointer
 887   // to an object on the stack. We want to create a local variable that stores
 888   // this value.
 889   std::optional<unsigned> LocalIndex =
 890       allocateLocalPrimitive(E, SubExprT, /*IsConst=*/true);
 891   if (!LocalIndex)
 892     return false;
 893   if (!this->emitSetLocal(SubExprT, *LocalIndex, E))
 894     return false;
 895
 896   // Here the local variable is created but the value is removed from the stack,
 897   // so we put it back, because the caller might need it.
 898   if (!DiscardResult) {
 899     if (!this->emitGetLocal(SubExprT, *LocalIndex, E))
 900       return false;
 901   }
 902
 903   // FIXME: Ideally the cached value should be cleaned up later.
 904   OpaqueExprs.insert({E, *LocalIndex});
 905
 906   return true;
 907 }
 908
 909 template <class Emitter>
 910 bool ByteCodeExprGen<Emitter>::VisitAbstractConditionalOperator(
 911     const AbstractConditionalOperator *E) {
 912   const Expr *Condition = E->getCond();
 913   const Expr *TrueExpr = E->getTrueExpr();
 914   const Expr *FalseExpr = E->getFalseExpr();
 915
 916   LabelTy LabelEnd = this->getLabel();   // Label after the operator.
 917   LabelTy LabelFalse = this->getLabel(); // Label for the false expr.
 918
 919   if (!this->visitBool(Condition))
 920     return false;
 921
 922   if (!this->jumpFalse(LabelFalse))
 923     return false;
 924
 925   if (!this->delegate(TrueExpr))
 926     return false;
 927   if (!this->jump(LabelEnd))
 928     return false;
 929
 930   this->emitLabel(LabelFalse);
 931
 932   if (!this->delegate(FalseExpr))
 933     return false;
 934
 935   this->fallthrough(LabelEnd);
 936   this->emitLabel(LabelEnd);
 937
 938   return true;
 939 }
 940
 941 template <class Emitter>
 942 bool ByteCodeExprGen<Emitter>::VisitStringLiteral(const StringLiteral *E) {
 943   if (DiscardResult)
 944     return true;
 945
 946   if (!Initializing) {
 947     unsigned StringIndex = P.createGlobalString(E);
 948     return this->emitGetPtrGlobal(StringIndex, E);
 949   }
 950
 951   // We are initializing an array on the stack.
 952   const ConstantArrayType *CAT =
 953       Ctx.getASTContext().getAsConstantArrayType(E->getType());
 954   assert(CAT && "a string literal that's not a constant array?");
 955
 956   // If the initializer string is too long, a diagnostic has already been
 957   // emitted. Read only the array length from the string literal.
 958   unsigned ArraySize = CAT->getSize().getZExtValue();
 959   unsigned N = std::min(ArraySize, E->getLength());
 960   size_t CharWidth = E->getCharByteWidth();
 961
 962   for (unsigned I = 0; I != N; ++I) {
 963     uint32_t CodeUnit = E->getCodeUnit(I);
 964
 965     if (CharWidth == 1) {
 966       this->emitConstSint8(CodeUnit, E);
 967       this->emitInitElemSint8(I, E);
 968     } else if (CharWidth == 2) {
 969       this->emitConstUint16(CodeUnit, E);
 970       this->emitInitElemUint16(I, E);
 971     } else if (CharWidth == 4) {
 972       this->emitConstUint32(CodeUnit, E);
 973       this->emitInitElemUint32(I, E);
 974     } else {
 975       llvm_unreachable("unsupported character width");
 976     }
 977   }
 978
 979   // Fill up the rest of the char array with NUL bytes.
 980   for (unsigned I = N; I != ArraySize; ++I) {
 981     if (CharWidth == 1) {
 982       this->emitConstSint8(0, E);
 983       this->emitInitElemSint8(I, E);
 984     } else if (CharWidth == 2) {
 985       this->emitConstUint16(0, E);
 986       this->emitInitElemUint16(I, E);
 987     } else if (CharWidth == 4) {
 988       this->emitConstUint32(0, E);
 989       this->emitInitElemUint32(I, E);
 990     } else {
 991       llvm_unreachable("unsupported character width");
 992     }
 993   }
 994
 995   return true;
 996 }
 997
 998 template <class Emitter>
 999 bool ByteCodeExprGen<Emitter>::VisitCharacterLiteral(
1000     const CharacterLiteral *E) {
1001   if (DiscardResult)
1002     return true;
1003   return this->emitConst(E->getValue(), E);
1004 }
1005
1006 template <class Emitter>
1007 bool ByteCodeExprGen<Emitter>::VisitFloatCompoundAssignOperator(
1008     const CompoundAssignOperator *E) {
1009
1010   const Expr *LHS = E->getLHS();
1011   const Expr *RHS = E->getRHS();
1012   QualType LHSType = LHS->getType();
1013   QualType LHSComputationType = E->getComputationLHSType();
1014   QualType ResultType = E->getComputationResultType();
1015   std::optional<PrimType> LT = classify(LHSComputationType);
1016   std::optional<PrimType> RT = classify(ResultType);
1017
1018   assert(ResultType->isFloatingType());
1019
1020   if (!LT || !RT)
1021     return false;
1022
1023   PrimType LHST = classifyPrim(LHSType);
1024
1025   // C++17 onwards require that we evaluate the RHS first.
1026   // Compute RHS and save it in a temporary variable so we can
1027   // load it again later.
1028   if (!visit(RHS))
1029     return false;
1030
1031   unsigned TempOffset = this->allocateLocalPrimitive(E, *RT, /*IsConst=*/true);
1032   if (!this->emitSetLocal(*RT, TempOffset, E))
1033     return false;
1034
1035   // First, visit LHS.
1036   if (!visit(LHS))
1037     return false;
1038   if (!this->emitLoad(LHST, E))
1039     return false;
1040
1041   // If necessary, convert LHS to its computation type.
1042   if (!this->emitPrimCast(LHST, classifyPrim(LHSComputationType),
1043                           LHSComputationType, E))
1044     return false;
1045
1046   // Now load RHS.
1047   if (!this->emitGetLocal(*RT, TempOffset, E))
1048     return false;
1049
1050   llvm::RoundingMode RM = getRoundingMode(E);
1051   switch (E->getOpcode()) {
1052   case BO_AddAssign:
1053     if (!this->emitAddf(RM, E))
1054       return false;
1055     break;
1056   case BO_SubAssign:
1057     if (!this->emitSubf(RM, E))
1058       return false;
1059     break;
1060   case BO_MulAssign:
1061     if (!this->emitMulf(RM, E))
1062       return false;
1063     break;
1064   case BO_DivAssign:
1065     if (!this->emitDivf(RM, E))
1066       return false;
1067     break;
1068   default:
1069     return false;
1070   }
1071
1072   if (!this->emitPrimCast(classifyPrim(ResultType), LHST, LHS->getType(), E))
1073     return false;
1074
1075   if (DiscardResult)
1076     return this->emitStorePop(LHST, E);
1077   return this->emitStore(LHST, E);
1078 }
1079
1080 template <class Emitter>
1081 bool ByteCodeExprGen<Emitter>::VisitPointerCompoundAssignOperator(
1082     const CompoundAssignOperator *E) {
1083   BinaryOperatorKind Op = E->getOpcode();
1084   const Expr *LHS = E->getLHS();
1085   const Expr *RHS = E->getRHS();
1086   std::optional<PrimType> LT = classify(LHS->getType());
1087   std::optional<PrimType> RT = classify(RHS->getType());
1088
1089   if (Op != BO_AddAssign && Op != BO_SubAssign)
1090     return false;
1091
1092   if (!LT || !RT)
1093     return false;
1094   assert(*LT == PT_Ptr);
1095
1096   if (!visit(LHS))
1097     return false;
1098
1099   if (!this->emitLoadPtr(LHS))
1100     return false;
1101
1102   if (!visit(RHS))
1103     return false;
1104
1105   if (Op == BO_AddAssign)
1106     this->emitAddOffset(*RT, E);
1107   else
1108     this->emitSubOffset(*RT, E);
1109
1110   if (DiscardResult)
1111     return this->emitStorePopPtr(E);
1112   return this->emitStorePtr(E);
1113 }
1114
1115 template <class Emitter>
1116 bool ByteCodeExprGen<Emitter>::VisitCompoundAssignOperator(
1117     const CompoundAssignOperator *E) {
1118
1119   const Expr *LHS = E->getLHS();
1120   const Expr *RHS = E->getRHS();
1121   std::optional<PrimType> LHSComputationT =
1122       classify(E->getComputationLHSType());
1123   std::optional<PrimType> LT = classify(LHS->getType());
1124   std::optional<PrimType> RT = classify(E->getComputationResultType());
1125   std::optional<PrimType> ResultT = classify(E->getType());
1126
1127   if (!LT || !RT || !ResultT || !LHSComputationT)
1128     return false;
1129
1130   // Handle floating point operations separately here, since they
1131   // require special care.
1132
1133   if (ResultT == PT_Float || RT == PT_Float)
1134     return VisitFloatCompoundAssignOperator(E);
1135
1136   if (E->getType()->isPointerType())
1137     return VisitPointerCompoundAssignOperator(E);
1138
1139   assert(!E->getType()->isPointerType() && "Handled above");
1140   assert(!E->getType()->isFloatingType() && "Handled above");
1141
1142   // C++17 onwards require that we evaluate the RHS first.
1143   // Compute RHS and save it in a temporary variable so we can
1144   // load it again later.
1145   // FIXME: Compound assignments are unsequenced in C, so we might
1146   //   have to figure out how to reject them.
1147   if (!visit(RHS))
1148     return false;
1149
1150   unsigned TempOffset = this->allocateLocalPrimitive(E, *RT, /*IsConst=*/true);
1151
1152   if (!this->emitSetLocal(*RT, TempOffset, E))
1153     return false;
1154
1155   // Get LHS pointer, load its value and cast it to the
1156   // computation type if necessary.
1157   if (!visit(LHS))
1158     return false;
1159   if (!this->emitLoad(*LT, E))
1160     return false;
1161   if (*LT != *LHSComputationT) {
1162     if (!this->emitCast(*LT, *LHSComputationT, E))
1163       return false;
1164   }
1165
1166   // Get the RHS value on the stack.
1167   if (!this->emitGetLocal(*RT, TempOffset, E))
1168     return false;
1169
1170   // Perform operation.
1171   switch (E->getOpcode()) {
1172   case BO_AddAssign:
1173     if (!this->emitAdd(*LHSComputationT, E))
1174       return false;
1175     break;
1176   case BO_SubAssign:
1177     if (!this->emitSub(*LHSComputationT, E))
1178       return false;
1179     break;
1180   case BO_MulAssign:
1181     if (!this->emitMul(*LHSComputationT, E))
1182       return false;
1183     break;
1184   case BO_DivAssign:
1185     if (!this->emitDiv(*LHSComputationT, E))
1186       return false;
1187     break;
1188   case BO_RemAssign:
1189     if (!this->emitRem(*LHSComputationT, E))
1190       return false;
1191     break;
1192   case BO_ShlAssign:
1193     if (!this->emitShl(*LHSComputationT, *RT, E))
1194       return false;
1195     break;
1196   case BO_ShrAssign:
1197     if (!this->emitShr(*LHSComputationT, *RT, E))
1198       return false;
1199     break;
1200   case BO_AndAssign:
1201     if (!this->emitBitAnd(*LHSComputationT, E))
1202       return false;
1203     break;
1204   case BO_XorAssign:
1205     if (!this->emitBitXor(*LHSComputationT, E))
1206       return false;
1207     break;
1208   case BO_OrAssign:
1209     if (!this->emitBitOr(*LHSComputationT, E))
1210       return false;
1211     break;
1212   default:
1213     llvm_unreachable("Unimplemented compound assign operator");
1214   }
1215
1216   // And now cast from LHSComputationT to ResultT.
1217   if (*ResultT != *LHSComputationT) {
1218     if (!this->emitCast(*LHSComputationT, *ResultT, E))
1219       return false;
1220   }
1221
1222   // And store the result in LHS.
1223   if (DiscardResult) {
1224     if (LHS->refersToBitField())
1225       return this->emitStoreBitFieldPop(*ResultT, E);
1226     return this->emitStorePop(*ResultT, E);
1227   }
1228   if (LHS->refersToBitField())
1229     return this->emitStoreBitField(*ResultT, E);
1230   return this->emitStore(*ResultT, E);
1231 }
1232
1233 template <class Emitter>
1234 bool ByteCodeExprGen<Emitter>::VisitExprWithCleanups(
1235     const ExprWithCleanups *E) {
1236   const Expr *SubExpr = E->getSubExpr();
1237
1238   assert(E->getNumObjects() == 0 && "TODO: Implement cleanups");
1239
1240   return this->delegate(SubExpr);
1241 }
1242
1243 template <class Emitter>
1244 bool ByteCodeExprGen<Emitter>::VisitMaterializeTemporaryExpr(
1245     const MaterializeTemporaryExpr *E) {
1246   const Expr *SubExpr = E->getSubExpr();
1247
1248   if (Initializing) {
1249     // We already have a value, just initialize that.
1250     return this->visitInitializer(SubExpr);
1251   }
1252   // If we don't end up using the materialized temporary anyway, don't
1253   // bother creating it.
1254   if (DiscardResult)
1255     return this->discard(SubExpr);
1256
1257   // When we're initializing a global variable *or* the storage duration of
1258   // the temporary is explicitly static, create a global variable.
1259   std::optional<PrimType> SubExprT = classify(SubExpr);
1260   bool IsStatic = E->getStorageDuration() == SD_Static;
1261   if (GlobalDecl || IsStatic) {
1262     std::optional<unsigned> GlobalIndex = P.createGlobal(E);
1263     if (!GlobalIndex)
1264       return false;
1265
1266     const LifetimeExtendedTemporaryDecl *TempDecl =
1267         E->getLifetimeExtendedTemporaryDecl();
1268     if (IsStatic)
1269       assert(TempDecl);
1270
1271     if (SubExprT) {
1272       if (!this->visit(SubExpr))
1273         return false;
1274       if (IsStatic) {
1275         if (!this->emitInitGlobalTemp(*SubExprT, *GlobalIndex, TempDecl, E))
1276           return false;
1277       } else {
1278         if (!this->emitInitGlobal(*SubExprT, *GlobalIndex, E))
1279           return false;
1280       }
1281       return this->emitGetPtrGlobal(*GlobalIndex, E);
1282     }
1283
1284     // Non-primitive values.
1285     if (!this->emitGetPtrGlobal(*GlobalIndex, E))
1286       return false;
1287     if (!this->visitInitializer(SubExpr))
1288       return false;
1289     if (IsStatic)
1290       return this->emitInitGlobalTempComp(TempDecl, E);
1291     return true;
1292   }
1293
1294   // For everyhing else, use local variables.
1295   if (SubExprT) {
1296     if (std::optional<unsigned> LocalIndex = allocateLocalPrimitive(
1297             SubExpr, *SubExprT, /*IsConst=*/true, /*IsExtended=*/true)) {
1298       if (!this->visit(SubExpr))
1299         return false;
1300       this->emitSetLocal(*SubExprT, *LocalIndex, E);
1301       return this->emitGetPtrLocal(*LocalIndex, E);
1302     }
1303   } else {
1304     if (std::optional<unsigned> LocalIndex =
1305             allocateLocal(SubExpr, /*IsExtended=*/true)) {
1306       if (!this->emitGetPtrLocal(*LocalIndex, E))
1307         return false;
1308       return this->visitInitializer(SubExpr);
1309     }
1310   }
1311   return false;
1312 }
1313
1314 template <class Emitter>
1315 bool ByteCodeExprGen<Emitter>::VisitCXXBindTemporaryExpr(
1316     const CXXBindTemporaryExpr *E) {
1317   return this->delegate(E->getSubExpr());
1318 }
1319
1320 template <class Emitter>
1321 bool ByteCodeExprGen<Emitter>::VisitCompoundLiteralExpr(
1322     const CompoundLiteralExpr *E) {
1323   const Expr *Init = E->getInitializer();
1324   if (Initializing) {
1325     // We already have a value, just initialize that.
1326     return this->visitInitializer(Init);
1327   }
1328
1329   std::optional<PrimType> T = classify(E->getType());
1330   if (E->isFileScope()) {
1331     if (std::optional<unsigned> GlobalIndex = P.createGlobal(E)) {
1332       if (classify(E->getType()))
1333         return this->visit(Init);
1334       if (!this->emitGetPtrGlobal(*GlobalIndex, E))
1335         return false;
1336       return this->visitInitializer(Init);
1337     }
1338   }
1339
1340   // Otherwise, use a local variable.
1341   if (T) {
1342     // For primitive types, we just visit the initializer.
1343     return this->delegate(Init);
1344   } else {
1345     if (std::optional<unsigned> LocalIndex = allocateLocal(Init)) {
1346       if (!this->emitGetPtrLocal(*LocalIndex, E))
1347         return false;
1348       if (!this->visitInitializer(Init))
1349         return false;
1350       if (DiscardResult)
1351         return this->emitPopPtr(E);
1352       return true;
1353     }
1354   }
1355
1356   return false;
1357 }
1358
1359 template <class Emitter>
1360 bool ByteCodeExprGen<Emitter>::VisitTypeTraitExpr(const TypeTraitExpr *E) {
1361   if (DiscardResult)
1362     return true;
1363   return this->emitConstBool(E->getValue(), E);
1364 }
1365
1366 template <class Emitter>
1367 bool ByteCodeExprGen<Emitter>::VisitLambdaExpr(const LambdaExpr *E) {
1368   assert(Initializing);
1369   const Record *R = P.getOrCreateRecord(E->getLambdaClass());
1370
1371   auto *CaptureInitIt = E->capture_init_begin();
1372   // Initialize all fields (which represent lambda captures) of the
1373   // record with their initializers.
1374   for (const Record::Field &F : R->fields()) {
1375     const Expr *Init = *CaptureInitIt;
1376     ++CaptureInitIt;
1377
1378     if (std::optional<PrimType> T = classify(Init)) {
1379       if (!this->visit(Init))
1380         return false;
1381
1382       if (!this->emitSetField(*T, F.Offset, E))
1383         return false;
1384     } else {
1385       if (!this->emitDupPtr(E))
1386         return false;
1387
1388       if (!this->emitGetPtrField(F.Offset, E))
1389         return false;
1390
1391       if (!this->visitInitializer(Init))
1392         return false;
1393
1394       if (!this->emitPopPtr(E))
1395         return false;
1396     }
1397   }
1398
1399   return true;
1400 }
1401
1402 template <class Emitter>
1403 bool ByteCodeExprGen<Emitter>::VisitPredefinedExpr(const PredefinedExpr *E) {
1404   if (DiscardResult)
1405     return true;
1406
1407   assert(!Initializing);
1408   return this->visit(E->getFunctionName());
1409 }
1410
1411 template <class Emitter>
1412 bool ByteCodeExprGen<Emitter>::VisitCXXThrowExpr(const CXXThrowExpr *E) {
1413   if (E->getSubExpr() && !this->discard(E->getSubExpr()))
1414     return false;
1415
1416   return this->emitInvalid(E);
1417 }
1418
1419 template <class Emitter>
1420 bool ByteCodeExprGen<Emitter>::VisitCXXReinterpretCastExpr(
1421     const CXXReinterpretCastExpr *E) {
1422   if (!this->discard(E->getSubExpr()))
1423     return false;
1424
1425   return this->emitInvalidCast(CastKind::Reinterpret, E);
1426 }
1427
1428 template <class Emitter>
1429 bool ByteCodeExprGen<Emitter>::VisitCXXNoexceptExpr(const CXXNoexceptExpr *E) {
1430   assert(E->getType()->isBooleanType());
1431
1432   if (DiscardResult)
1433     return true;
1434   return this->emitConstBool(E->getValue(), E);
1435 }
1436
1437 template <class Emitter>
1438 bool ByteCodeExprGen<Emitter>::VisitCXXConstructExpr(
1439     const CXXConstructExpr *E) {
1440   QualType T = E->getType();
1441   assert(!classify(T));
1442
1443   if (T->isRecordType()) {
1444     const CXXConstructorDecl *Ctor = E->getConstructor();
1445
1446     // Trivial zero initialization.
1447     if (E->requiresZeroInitialization() && Ctor->isTrivial()) {
1448       const Record *R = getRecord(E->getType());
1449       return this->visitZeroRecordInitializer(R, E);
1450     }
1451
1452     const Function *Func = getFunction(Ctor);
1453
1454     if (!Func)
1455       return false;
1456
1457     assert(Func->hasThisPointer());
1458     assert(!Func->hasRVO());
1459
1460     // If we're discarding a construct expression, we still need
1461     // to allocate a variable and call the constructor and destructor.
1462     if (DiscardResult) {
1463       assert(!Initializing);
1464       std::optional<unsigned> LocalIndex =
1465           allocateLocal(E, /*IsExtended=*/true);
1466
1467       if (!LocalIndex)
1468         return false;
1469
1470       if (!this->emitGetPtrLocal(*LocalIndex, E))
1471         return false;
1472     }
1473
1474     //  The This pointer is already on the stack because this is an initializer,
1475     //  but we need to dup() so the call() below has its own copy.
1476     if (!this->emitDupPtr(E))
1477       return false;
1478
1479     // Constructor arguments.
1480     for (const auto *Arg : E->arguments()) {
1481       if (!this->visit(Arg))
1482         return false;
1483     }
1484
1485     if (!this->emitCall(Func, E))
1486       return false;
1487
1488     // Immediately call the destructor if we have to.
1489     if (DiscardResult) {
1490       if (!this->emitPopPtr(E))
1491         return false;
1492     }
1493     return true;
1494   }
1495
1496   if (T->isArrayType()) {
1497     const ConstantArrayType *CAT =
1498         Ctx.getASTContext().getAsConstantArrayType(E->getType());
1499     assert(CAT);
1500     size_t NumElems = CAT->getSize().getZExtValue();
1501     const Function *Func = getFunction(E->getConstructor());
1502     if (!Func || !Func->isConstexpr())
1503       return false;
1504
1505     // FIXME(perf): We're calling the constructor once per array element here,
1506     //   in the old intepreter we had a special-case for trivial constructors.
1507     for (size_t I = 0; I != NumElems; ++I) {
1508       if (!this->emitConstUint64(I, E))
1509         return false;
1510       if (!this->emitArrayElemPtrUint64(E))
1511         return false;
1512
1513       // Constructor arguments.
1514       for (const auto *Arg : E->arguments()) {
1515         if (!this->visit(Arg))
1516           return false;
1517       }
1518
1519       if (!this->emitCall(Func, E))
1520         return false;
1521     }
1522     return true;
1523   }
1524
1525   return false;
1526 }
1527
1528 template <class Emitter>
1529 bool ByteCodeExprGen<Emitter>::VisitSourceLocExpr(const SourceLocExpr *E) {
1530   if (DiscardResult)
1531     return true;
1532
1533   const APValue Val =
1534       E->EvaluateInContext(Ctx.getASTContext(), SourceLocDefaultExpr);
1535
1536   // Things like __builtin_LINE().
1537   if (E->getType()->isIntegerType()) {
1538     assert(Val.isInt());
1539     const APSInt &I = Val.getInt();
1540     return this->emitConst(I, E);
1541   }
1542   // Otherwise, the APValue is an LValue, with only one element.
1543   // Theoretically, we don't need the APValue at all of course.
1544   assert(E->getType()->isPointerType());
1545   assert(Val.isLValue());
1546   const APValue::LValueBase &Base = Val.getLValueBase();
1547   if (const Expr *LValueExpr = Base.dyn_cast<const Expr *>())
1548     return this->visit(LValueExpr);
1549
1550   // Otherwise, we have a decl (which is the case for
1551   // __builtin_source_location).
1552   assert(Base.is<const ValueDecl *>());
1553   assert(Val.getLValuePath().size() == 0);
1554   const auto *BaseDecl = Base.dyn_cast<const ValueDecl *>();
1555   assert(BaseDecl);
1556
1557   auto *UGCD = cast<UnnamedGlobalConstantDecl>(BaseDecl);
1558
1559   std::optional<unsigned> GlobalIndex = P.getOrCreateGlobal(UGCD);
1560   if (!GlobalIndex)
1561     return false;
1562
1563   if (!this->emitGetPtrGlobal(*GlobalIndex, E))
1564     return false;
1565
1566   const Record *R = getRecord(E->getType());
1567   const APValue &V = UGCD->getValue();
1568   for (unsigned I = 0, N = R->getNumFields(); I != N; ++I) {
1569     const Record::Field *F = R->getField(I);
1570     const APValue &FieldValue = V.getStructField(I);
1571
1572     PrimType FieldT = classifyPrim(F->Decl->getType());
1573
1574     if (!this->visitAPValue(FieldValue, FieldT, E))
1575       return false;
1576     if (!this->emitInitField(FieldT, F->Offset, E))
1577       return false;
1578   }
1579
1580   // Leave the pointer to the global on the stack.
1581   return true;
1582 }
1583
1584 template <class Emitter>
1585 bool ByteCodeExprGen<Emitter>::VisitOffsetOfExpr(const OffsetOfExpr *E) {
1586   unsigned N = E->getNumComponents();
1587   if (N == 0)
1588     return false;
1589
1590   for (unsigned I = 0; I != N; ++I) {
1591     const OffsetOfNode &Node = E->getComponent(I);
1592     if (Node.getKind() == OffsetOfNode::Array) {
1593       const Expr *ArrayIndexExpr = E->getIndexExpr(Node.getArrayExprIndex());
1594       PrimType IndexT = classifyPrim(ArrayIndexExpr->getType());
1595
1596       if (DiscardResult) {
1597         if (!this->discard(ArrayIndexExpr))
1598           return false;
1599         continue;
1600       }
1601
1602       if (!this->visit(ArrayIndexExpr))
1603         return false;
1604       // Cast to Sint64.
1605       if (IndexT != PT_Sint64) {
1606         if (!this->emitCast(IndexT, PT_Sint64, E))
1607           return false;
1608       }
1609     }
1610   }
1611
1612   if (DiscardResult)
1613     return true;
1614
1615   PrimType T = classifyPrim(E->getType());
1616   return this->emitOffsetOf(T, E, E);
1617 }
1618
1619 template <class Emitter>
1620 bool ByteCodeExprGen<Emitter>::VisitCXXScalarValueInitExpr(
1621     const CXXScalarValueInitExpr *E) {
1622   QualType Ty = E->getType();
1623
1624   if (Ty->isVoidType())
1625     return true;
1626
1627   return this->visitZeroInitializer(classifyPrim(Ty), Ty, E);
1628 }
1629
1630 template <class Emitter> bool ByteCodeExprGen<Emitter>::discard(const Expr *E) {
1631   if (E->containsErrors())
1632     return false;
1633
1634   OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/true,
1635                              /*NewInitializing=*/false);
1636   return this->Visit(E);
1637 }
1638
1639 template <class Emitter>
1640 bool ByteCodeExprGen<Emitter>::delegate(const Expr *E) {
1641   if (E->containsErrors())
1642     return false;
1643
1644   // We're basically doing:
1645   // OptionScope<Emitter> Scope(this, DicardResult, Initializing);
1646   // but that's unnecessary of course.
1647   return this->Visit(E);
1648 }
1649
1650 template <class Emitter> bool ByteCodeExprGen<Emitter>::visit(const Expr *E) {
1651   if (E->containsErrors())
1652     return false;
1653
1654   if (E->getType()->isVoidType())
1655     return this->discard(E);
1656
1657   // Create local variable to hold the return value.
1658   if (!E->isGLValue() && !classify(E->getType())) {
1659     std::optional<unsigned> LocalIndex = allocateLocal(E, /*IsExtended=*/true);
1660     if (!LocalIndex)
1661       return false;
1662
1663     if (!this->emitGetPtrLocal(*LocalIndex, E))
1664       return false;
1665     return this->visitInitializer(E);
1666   }
1667
1668   //  Otherwise,we have a primitive return value, produce the value directly
1669   //  and push it on the stack.
1670   OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/false,
1671                              /*NewInitializing=*/false);
1672   return this->Visit(E);
1673 }
1674
1675 template <class Emitter>
1676 bool ByteCodeExprGen<Emitter>::visitInitializer(const Expr *E) {
1677   assert(!classify(E->getType()));
1678
1679   if (E->containsErrors())
1680     return false;
1681
1682   OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/false,
1683                              /*NewInitializing=*/true);
1684   return this->Visit(E);
1685 }
1686
1687 template <class Emitter>
1688 bool ByteCodeExprGen<Emitter>::visitBool(const Expr *E) {
1689   std::optional<PrimType> T = classify(E->getType());
1690   if (!T)
1691     return false;
1692
1693   if (!this->visit(E))
1694     return false;
1695
1696   if (T == PT_Bool)
1697     return true;
1698
1699   // Convert pointers to bool.
1700   if (T == PT_Ptr || T == PT_FnPtr) {
1701     if (!this->emitNull(*T, E))
1702       return false;
1703     return this->emitNE(*T, E);
1704   }
1705
1706   // Or Floats.
1707   if (T == PT_Float)
1708     return this->emitCastFloatingIntegralBool(E);
1709
1710   // Or anything else we can.
1711   return this->emitCast(*T, PT_Bool, E);
1712 }
1713
1714 template <class Emitter>
1715 bool ByteCodeExprGen<Emitter>::visitZeroInitializer(PrimType T, QualType QT,
1716                                                     const Expr *E) {
1717   switch (T) {
1718   case PT_Bool:
1719     return this->emitZeroBool(E);
1720   case PT_Sint8:
1721     return this->emitZeroSint8(E);
1722   case PT_Uint8:
1723     return this->emitZeroUint8(E);
1724   case PT_Sint16:
1725     return this->emitZeroSint16(E);
1726   case PT_Uint16:
1727     return this->emitZeroUint16(E);
1728   case PT_Sint32:
1729     return this->emitZeroSint32(E);
1730   case PT_Uint32:
1731     return this->emitZeroUint32(E);
1732   case PT_Sint64:
1733     return this->emitZeroSint64(E);
1734   case PT_Uint64:
1735     return this->emitZeroUint64(E);
1736   case PT_IntAP:
1737     return this->emitZeroIntAP(Ctx.getBitWidth(QT), E);
1738   case PT_IntAPS:
1739     return this->emitZeroIntAPS(Ctx.getBitWidth(QT), E);
1740   case PT_Ptr:
1741     return this->emitNullPtr(E);
1742   case PT_FnPtr:
1743     return this->emitNullFnPtr(E);
1744   case PT_Float: {
1745     return this->emitConstFloat(APFloat::getZero(Ctx.getFloatSemantics(QT)), E);
1746   }
1747   }
1748   llvm_unreachable("unknown primitive type");
1749 }
1750
1751 template <class Emitter>
1752 bool ByteCodeExprGen<Emitter>::visitZeroRecordInitializer(const Record *R,
1753                                                           const Expr *E) {
1754   assert(E);
1755   assert(R);
1756   // Fields
1757   for (const Record::Field &Field : R->fields()) {
1758     const Descriptor *D = Field.Desc;
1759     if (D->isPrimitive()) {
1760       QualType QT = D->getType();
1761       PrimType T = classifyPrim(D->getType());
1762       if (!this->visitZeroInitializer(T, QT, E))
1763         return false;
1764       if (!this->emitInitField(T, Field.Offset, E))
1765         return false;
1766       continue;
1767     }
1768
1769     // TODO: Add GetPtrFieldPop and get rid of this dup.
1770     if (!this->emitDupPtr(E))
1771       return false;
1772     if (!this->emitGetPtrField(Field.Offset, E))
1773       return false;
1774
1775     if (D->isPrimitiveArray()) {
1776       QualType ET = D->getElemQualType();
1777       PrimType T = classifyPrim(ET);
1778       for (uint32_t I = 0, N = D->getNumElems(); I != N; ++I) {
1779         if (!this->visitZeroInitializer(T, ET, E))
1780           return false;
1781         if (!this->emitInitElem(T, I, E))
1782           return false;
1783       }
1784     } else if (D->isCompositeArray()) {
1785       const Record *ElemRecord = D->ElemDesc->ElemRecord;
1786       assert(D->ElemDesc->ElemRecord);
1787       for (uint32_t I = 0, N = D->getNumElems(); I != N; ++I) {
1788         if (!this->emitConstUint32(I, E))
1789           return false;
1790         if (!this->emitArrayElemPtr(PT_Uint32, E))
1791           return false;
1792         if (!this->visitZeroRecordInitializer(ElemRecord, E))
1793           return false;
1794         if (!this->emitPopPtr(E))
1795           return false;
1796       }
1797     } else if (D->isRecord()) {
1798       if (!this->visitZeroRecordInitializer(D->ElemRecord, E))
1799         return false;
1800     } else {
1801       assert(false);
1802     }
1803
1804     if (!this->emitPopPtr(E))
1805       return false;
1806   }
1807
1808   for (const Record::Base &B : R->bases()) {
1809     if (!this->emitGetPtrBase(B.Offset, E))
1810       return false;
1811     if (!this->visitZeroRecordInitializer(B.R, E))
1812       return false;
1813     if (!this->emitInitPtrPop(E))
1814       return false;
1815   }
1816
1817   // FIXME: Virtual bases.
1818
1819   return true;
1820 }
1821
1822 template <class Emitter>
1823 bool ByteCodeExprGen<Emitter>::dereference(
1824     const Expr *LV, DerefKind AK, llvm::function_ref<bool(PrimType)> Direct,
1825     llvm::function_ref<bool(PrimType)> Indirect) {
1826   if (std::optional<PrimType> T = classify(LV->getType())) {
1827     if (!LV->refersToBitField()) {
1828       // Only primitive, non bit-field types can be dereferenced directly.
1829       if (const auto *DE = dyn_cast<DeclRefExpr>(LV)) {
1830         if (!DE->getDecl()->getType()->isReferenceType()) {
1831           if (const auto *PD = dyn_cast<ParmVarDecl>(DE->getDecl()))
1832             return dereferenceParam(LV, *T, PD, AK, Direct, Indirect);
1833           if (const auto *VD = dyn_cast<VarDecl>(DE->getDecl()))
1834             return dereferenceVar(LV, *T, VD, AK, Direct, Indirect);
1835         }
1836       }
1837     }
1838
1839     if (!visit(LV))
1840       return false;
1841     return Indirect(*T);
1842   }
1843
1844   return false;
1845 }
1846
1847 template <class Emitter>
1848 bool ByteCodeExprGen<Emitter>::dereferenceParam(
1849     const Expr *LV, PrimType T, const ParmVarDecl *PD, DerefKind AK,
1850     llvm::function_ref<bool(PrimType)> Direct,
1851     llvm::function_ref<bool(PrimType)> Indirect) {
1852   auto It = this->Params.find(PD);
1853   if (It != this->Params.end()) {
1854     unsigned Idx = It->second.Offset;
1855     switch (AK) {
1856     case DerefKind::Read:
1857       return DiscardResult ? true : this->emitGetParam(T, Idx, LV);
1858
1859     case DerefKind::Write:
1860       if (!Direct(T))
1861         return false;
1862       if (!this->emitSetParam(T, Idx, LV))
1863         return false;
1864       return DiscardResult ? true : this->emitGetPtrParam(Idx, LV);
1865
1866     case DerefKind::ReadWrite:
1867       if (!this->emitGetParam(T, Idx, LV))
1868         return false;
1869       if (!Direct(T))
1870         return false;
1871       if (!this->emitSetParam(T, Idx, LV))
1872         return false;
1873       return DiscardResult ? true : this->emitGetPtrParam(Idx, LV);
1874     }
1875     return true;
1876   }
1877
1878   // If the param is a pointer, we can dereference a dummy value.
1879   if (!DiscardResult && T == PT_Ptr && AK == DerefKind::Read) {
1880     if (auto Idx = P.getOrCreateDummy(PD))
1881       return this->emitGetPtrGlobal(*Idx, PD);
1882     return false;
1883   }
1884
1885   // Value cannot be produced - try to emit pointer and do stuff with it.
1886   return visit(LV) && Indirect(T);
1887 }
1888
1889 template <class Emitter>
1890 bool ByteCodeExprGen<Emitter>::dereferenceVar(
1891     const Expr *LV, PrimType T, const VarDecl *VD, DerefKind AK,
1892     llvm::function_ref<bool(PrimType)> Direct,
1893     llvm::function_ref<bool(PrimType)> Indirect) {
1894   auto It = Locals.find(VD);
1895   if (It != Locals.end()) {
1896     const auto &L = It->second;
1897     switch (AK) {
1898     case DerefKind::Read:
1899       if (!this->emitGetLocal(T, L.Offset, LV))
1900         return false;
1901       return DiscardResult ? this->emitPop(T, LV) : true;
1902
1903     case DerefKind::Write:
1904       if (!Direct(T))
1905         return false;
1906       if (!this->emitSetLocal(T, L.Offset, LV))
1907         return false;
1908       return DiscardResult ? true : this->emitGetPtrLocal(L.Offset, LV);
1909
1910     case DerefKind::ReadWrite:
1911       if (!this->emitGetLocal(T, L.Offset, LV))
1912         return false;
1913       if (!Direct(T))
1914         return false;
1915       if (!this->emitSetLocal(T, L.Offset, LV))
1916         return false;
1917       return DiscardResult ? true : this->emitGetPtrLocal(L.Offset, LV);
1918     }
1919   } else if (auto Idx = P.getGlobal(VD)) {
1920     switch (AK) {
1921     case DerefKind::Read:
1922       if (!this->emitGetGlobal(T, *Idx, LV))
1923         return false;
1924       return DiscardResult ? this->emitPop(T, LV) : true;
1925
1926     case DerefKind::Write:
1927       if (!Direct(T))
1928         return false;
1929       if (!this->emitSetGlobal(T, *Idx, LV))
1930         return false;
1931       return DiscardResult ? true : this->emitGetPtrGlobal(*Idx, LV);
1932
1933     case DerefKind::ReadWrite:
1934       if (!this->emitGetGlobal(T, *Idx, LV))
1935         return false;
1936       if (!Direct(T))
1937         return false;
1938       if (!this->emitSetGlobal(T, *Idx, LV))
1939         return false;
1940       return DiscardResult ? true : this->emitGetPtrGlobal(*Idx, LV);
1941     }
1942   }
1943
1944   // If the declaration is a constant value, emit it here even
1945   // though the declaration was not evaluated in the current scope.
1946   // The access mode can only be read in this case.
1947   if (!DiscardResult && AK == DerefKind::Read) {
1948     if (VD->hasLocalStorage() && VD->hasInit() && !VD->isConstexpr()) {
1949       QualType VT = VD->getType();
1950       if (VT.isConstQualified() && VT->isFundamentalType())
1951         return this->visit(VD->getInit());
1952     }
1953   }
1954
1955   // Value cannot be produced - try to emit pointer.
1956   return visit(LV) && Indirect(T);
1957 }
1958
1959 template <class Emitter>
1960 template <typename T>
1961 bool ByteCodeExprGen<Emitter>::emitConst(T Value, PrimType Ty, const Expr *E) {
1962   switch (Ty) {
1963   case PT_Sint8:
1964     return this->emitConstSint8(Value, E);
1965   case PT_Uint8:
1966     return this->emitConstUint8(Value, E);
1967   case PT_Sint16:
1968     return this->emitConstSint16(Value, E);
1969   case PT_Uint16:
1970     return this->emitConstUint16(Value, E);
1971   case PT_Sint32:
1972     return this->emitConstSint32(Value, E);
1973   case PT_Uint32:
1974     return this->emitConstUint32(Value, E);
1975   case PT_Sint64:
1976     return this->emitConstSint64(Value, E);
1977   case PT_Uint64:
1978     return this->emitConstUint64(Value, E);
1979   case PT_IntAP:
1980   case PT_IntAPS:
1981     assert(false);
1982     return false;
1983   case PT_Bool:
1984     return this->emitConstBool(Value, E);
1985   case PT_Ptr:
1986   case PT_FnPtr:
1987   case PT_Float:
1988     llvm_unreachable("Invalid integral type");
1989     break;
1990   }
1991   llvm_unreachable("unknown primitive type");
1992 }
1993
1994 template <class Emitter>
1995 template <typename T>
1996 bool ByteCodeExprGen<Emitter>::emitConst(T Value, const Expr *E) {
1997   return this->emitConst(Value, classifyPrim(E->getType()), E);
1998 }
1999
2000 template <class Emitter>
2001 bool ByteCodeExprGen<Emitter>::emitConst(const APSInt &Value, PrimType Ty,
2002                                          const Expr *E) {
2003   if (Value.isSigned())
2004     return this->emitConst(Value.getSExtValue(), Ty, E);
2005   return this->emitConst(Value.getZExtValue(), Ty, E);
2006 }
2007
2008 template <class Emitter>
2009 bool ByteCodeExprGen<Emitter>::emitConst(const APSInt &Value, const Expr *E) {
2010   return this->emitConst(Value, classifyPrim(E->getType()), E);
2011 }
2012
2013 template <class Emitter>
2014 unsigned ByteCodeExprGen<Emitter>::allocateLocalPrimitive(DeclTy &&Src,
2015                                                           PrimType Ty,
2016                                                           bool IsConst,
2017                                                           bool IsExtended) {
2018   // Make sure we don't accidentally register the same decl twice.
2019   if (const auto *VD =
2020           dyn_cast_if_present<ValueDecl>(Src.dyn_cast<const Decl *>())) {
2021     assert(!P.getGlobal(VD));
2022     assert(!Locals.contains(VD));
2023   }
2024
2025   // FIXME: There are cases where Src.is<Expr*>() is wrong, e.g.
2026   //   (int){12} in C. Consider using Expr::isTemporaryObject() instead
2027   //   or isa<MaterializeTemporaryExpr>().
2028   Descriptor *D = P.createDescriptor(Src, Ty, Descriptor::InlineDescMD, IsConst,
2029                                      Src.is<const Expr *>());
2030   Scope::Local Local = this->createLocal(D);
2031   if (auto *VD = dyn_cast_if_present<ValueDecl>(Src.dyn_cast<const Decl *>()))
2032     Locals.insert({VD, Local});
2033   VarScope->add(Local, IsExtended);
2034   return Local.Offset;
2035 }
2036
2037 template <class Emitter>
2038 std::optional<unsigned>
2039 ByteCodeExprGen<Emitter>::allocateLocal(DeclTy &&Src, bool IsExtended) {
2040   // Make sure we don't accidentally register the same decl twice.
2041   if ([[maybe_unused]]  const auto *VD =
2042           dyn_cast_if_present<ValueDecl>(Src.dyn_cast<const Decl *>())) {
2043     assert(!P.getGlobal(VD));
2044     assert(!Locals.contains(VD));
2045   }
2046
2047   QualType Ty;
2048   const ValueDecl *Key = nullptr;
2049   const Expr *Init = nullptr;
2050   bool IsTemporary = false;
2051   if (auto *VD = dyn_cast_if_present<ValueDecl>(Src.dyn_cast<const Decl *>())) {
2052     Key = VD;
2053     Ty = VD->getType();
2054
2055     if (const auto *VarD = dyn_cast<VarDecl>(VD))
2056       Init = VarD->getInit();
2057   }
2058   if (auto *E = Src.dyn_cast<const Expr *>()) {
2059     IsTemporary = true;
2060     Ty = E->getType();
2061   }
2062
2063   Descriptor *D = P.createDescriptor(
2064       Src, Ty.getTypePtr(), Descriptor::InlineDescMD, Ty.isConstQualified(),
2065       IsTemporary, /*IsMutable=*/false, Init);
2066   if (!D)
2067     return {};
2068
2069   Scope::Local Local = this->createLocal(D);
2070   if (Key)
2071     Locals.insert({Key, Local});
2072   VarScope->add(Local, IsExtended);
2073   return Local.Offset;
2074 }
2075
2076 template <class Emitter>
2077 const RecordType *ByteCodeExprGen<Emitter>::getRecordTy(QualType Ty) {
2078   if (const PointerType *PT = dyn_cast<PointerType>(Ty))
2079     return PT->getPointeeType()->getAs<RecordType>();
2080   return Ty->getAs<RecordType>();
2081 }
2082
2083 template <class Emitter>
2084 Record *ByteCodeExprGen<Emitter>::getRecord(QualType Ty) {
2085   if (const auto *RecordTy = getRecordTy(Ty))
2086     return getRecord(RecordTy->getDecl());
2087   return nullptr;
2088 }
2089
2090 template <class Emitter>
2091 Record *ByteCodeExprGen<Emitter>::getRecord(const RecordDecl *RD) {
2092   return P.getOrCreateRecord(RD);
2093 }
2094
2095 template <class Emitter>
2096 const Function *ByteCodeExprGen<Emitter>::getFunction(const FunctionDecl *FD) {
2097   return Ctx.getOrCreateFunction(FD);
2098 }
2099
2100 template <class Emitter>
2101 bool ByteCodeExprGen<Emitter>::visitExpr(const Expr *E) {
2102   ExprScope<Emitter> RootScope(this);
2103   if (!visit(E))
2104     return false;
2105
2106   if (E->getType()->isVoidType())
2107     return this->emitRetVoid(E);
2108
2109   if (std::optional<PrimType> T = classify(E))
2110     return this->emitRet(*T, E);
2111   return this->emitRetValue(E);
2112 }
2113
2114 /// Toplevel visitDecl().
2115 /// We get here from evaluateAsInitializer().
2116 /// We need to evaluate the initializer and return its value.
2117 template <class Emitter>
2118 bool ByteCodeExprGen<Emitter>::visitDecl(const VarDecl *VD) {
2119   assert(!VD->isInvalidDecl() && "Trying to constant evaluate an invalid decl");
2120
2121   // Create and initialize the variable.
2122   if (!this->visitVarDecl(VD))
2123     return false;
2124
2125   // Get a pointer to the variable
2126   if (Context::shouldBeGloballyIndexed(VD)) {
2127     auto GlobalIndex = P.getGlobal(VD);
2128     assert(GlobalIndex); // visitVarDecl() didn't return false.
2129     if (!this->emitGetPtrGlobal(*GlobalIndex, VD))
2130       return false;
2131   } else {
2132     auto Local = Locals.find(VD);
2133     assert(Local != Locals.end()); // Same here.
2134     if (!this->emitGetPtrLocal(Local->second.Offset, VD))
2135       return false;
2136   }
2137
2138   // Return the value
2139   if (std::optional<PrimType> VarT = classify(VD->getType())) {
2140     if (!this->emitLoadPop(*VarT, VD))
2141       return false;
2142
2143     return this->emitRet(*VarT, VD);
2144   }
2145
2146   return this->emitRetValue(VD);
2147 }
2148
2149 template <class Emitter>
2150 bool ByteCodeExprGen<Emitter>::visitVarDecl(const VarDecl *VD) {
2151   // We don't know what to do with these, so just return false.
2152   if (VD->getType().isNull())
2153     return false;
2154
2155   const Expr *Init = VD->getInit();
2156   std::optional<PrimType> VarT = classify(VD->getType());
2157
2158   if (Context::shouldBeGloballyIndexed(VD)) {
2159     // We've already seen and initialized this global.
2160     if (P.getGlobal(VD))
2161       return true;
2162
2163     std::optional<unsigned> GlobalIndex = P.createGlobal(VD, Init);
2164
2165     if (!GlobalIndex)
2166       return this->bail(VD);
2167
2168     assert(Init);
2169     {
2170       DeclScope<Emitter> LocalScope(this, VD);
2171
2172       if (VarT) {
2173         if (!this->visit(Init))
2174           return false;
2175         return this->emitInitGlobal(*VarT, *GlobalIndex, VD);
2176       }
2177       return this->visitGlobalInitializer(Init, *GlobalIndex);
2178     }
2179   } else {
2180     VariableScope<Emitter> LocalScope(this);
2181     if (VarT) {
2182       unsigned Offset = this->allocateLocalPrimitive(
2183           VD, *VarT, VD->getType().isConstQualified());
2184       if (Init) {
2185         // Compile the initializer in its own scope.
2186         ExprScope<Emitter> Scope(this);
2187         if (!this->visit(Init))
2188           return false;
2189
2190         return this->emitSetLocal(*VarT, Offset, VD);
2191       }
2192     } else {
2193       if (std::optional<unsigned> Offset = this->allocateLocal(VD)) {
2194         if (Init)
2195           return this->visitLocalInitializer(Init, *Offset);
2196       }
2197     }
2198     return true;
2199   }
2200
2201   return false;
2202 }
2203
2204 template <class Emitter>
2205 bool ByteCodeExprGen<Emitter>::visitAPValue(const APValue &Val,
2206                                             PrimType ValType, const Expr *E) {
2207   assert(!DiscardResult);
2208   if (Val.isInt())
2209     return this->emitConst(Val.getInt(), ValType, E);
2210
2211   if (Val.isLValue()) {
2212     APValue::LValueBase Base = Val.getLValueBase();
2213     if (const Expr *BaseExpr = Base.dyn_cast<const Expr *>())
2214       return this->visit(BaseExpr);
2215   }
2216
2217   return false;
2218 }
2219
2220 template <class Emitter>
2221 bool ByteCodeExprGen<Emitter>::VisitBuiltinCallExpr(const CallExpr *E) {
2222   const Function *Func = getFunction(E->getDirectCallee());
2223   if (!Func)
2224     return false;
2225
2226   // Put arguments on the stack.
2227   for (const auto *Arg : E->arguments()) {
2228     if (!this->visit(Arg))
2229       return false;
2230   }
2231
2232   if (!this->emitCallBI(Func, E, E))
2233     return false;
2234
2235   QualType ReturnType = E->getCallReturnType(Ctx.getASTContext());
2236   if (DiscardResult && !ReturnType->isVoidType()) {
2237     PrimType T = classifyPrim(ReturnType);
2238     return this->emitPop(T, E);
2239   }
2240
2241   return true;
2242 }
2243
2244 template <class Emitter>
2245 bool ByteCodeExprGen<Emitter>::VisitCallExpr(const CallExpr *E) {
2246   if (E->getBuiltinCallee())
2247     return VisitBuiltinCallExpr(E);
2248
2249   QualType ReturnType = E->getCallReturnType(Ctx.getASTContext());
2250   std::optional<PrimType> T = classify(ReturnType);
2251   bool HasRVO = !ReturnType->isVoidType() && !T;
2252
2253   if (HasRVO) {
2254     if (DiscardResult) {
2255       // If we need to discard the return value but the function returns its
2256       // value via an RVO pointer, we need to create one such pointer just
2257       // for this call.
2258       if (std::optional<unsigned> LocalIndex = allocateLocal(E)) {
2259         if (!this->emitGetPtrLocal(*LocalIndex, E))
2260           return false;
2261       }
2262     } else {
2263       assert(Initializing);
2264       if (!isa<CXXMemberCallExpr>(E)) {
2265         if (!this->emitDupPtr(E))
2266           return false;
2267       }
2268     }
2269   }
2270
2271   // Put arguments on the stack.
2272   for (const auto *Arg : E->arguments()) {
2273     if (!this->visit(Arg))
2274       return false;
2275   }
2276
2277   if (const FunctionDecl *FuncDecl = E->getDirectCallee()) {
2278     const Function *Func = getFunction(FuncDecl);
2279     if (!Func)
2280       return false;
2281     // If the function is being compiled right now, this is a recursive call.
2282     // In that case, the function can't be valid yet, even though it will be
2283     // later.
2284     // If the function is already fully compiled but not constexpr, it was
2285     // found to be faulty earlier on, so bail out.
2286     if (Func->isFullyCompiled() && !Func->isConstexpr())
2287       return false;
2288
2289     assert(HasRVO == Func->hasRVO());
2290
2291     bool HasQualifier = false;
2292     if (const auto *ME = dyn_cast<MemberExpr>(E->getCallee()))
2293       HasQualifier = ME->hasQualifier();
2294
2295     bool IsVirtual = false;
2296     if (const auto *MD = dyn_cast<CXXMethodDecl>(FuncDecl))
2297       IsVirtual = MD->isVirtual();
2298
2299     // In any case call the function. The return value will end up on the stack
2300     // and if the function has RVO, we already have the pointer on the stack to
2301     // write the result into.
2302     if (IsVirtual && !HasQualifier) {
2303       if (!this->emitCallVirt(Func, E))
2304         return false;
2305     } else {
2306       if (!this->emitCall(Func, E))
2307         return false;
2308     }
2309   } else {
2310     // Indirect call. Visit the callee, which will leave a FunctionPointer on
2311     // the stack. Cleanup of the returned value if necessary will be done after
2312     // the function call completed.
2313     if (!this->visit(E->getCallee()))
2314       return false;
2315
2316     if (!this->emitCallPtr(E))
2317       return false;
2318   }
2319
2320   // Cleanup for discarded return values.
2321   if (DiscardResult && !ReturnType->isVoidType() && T)
2322     return this->emitPop(*T, E);
2323
2324   return true;
2325 }
2326
2327 template <class Emitter>
2328 bool ByteCodeExprGen<Emitter>::VisitCXXMemberCallExpr(
2329     const CXXMemberCallExpr *E) {
2330   if (Initializing) {
2331     // If we're initializing, the current stack top is the pointer to
2332     // initialize, so dup that so this call has its own version.
2333     if (!this->emitDupPtr(E))
2334       return false;
2335   }
2336
2337   if (!this->visit(E->getImplicitObjectArgument()))
2338     return false;
2339
2340   return VisitCallExpr(E);
2341 }
2342
2343 template <class Emitter>
2344 bool ByteCodeExprGen<Emitter>::VisitCXXDefaultInitExpr(
2345     const CXXDefaultInitExpr *E) {
2346   SourceLocScope<Emitter> SLS(this, E);
2347   if (Initializing)
2348     return this->visitInitializer(E->getExpr());
2349
2350   assert(classify(E->getType()));
2351   return this->visit(E->getExpr());
2352 }
2353
2354 template <class Emitter>
2355 bool ByteCodeExprGen<Emitter>::VisitCXXDefaultArgExpr(
2356     const CXXDefaultArgExpr *E) {
2357   SourceLocScope<Emitter> SLS(this, E);
2358
2359   const Expr *SubExpr = E->getExpr();
2360   if (std::optional<PrimType> T = classify(E->getExpr()))
2361     return this->visit(SubExpr);
2362
2363   assert(Initializing);
2364   return this->visitInitializer(SubExpr);
2365 }
2366
2367 template <class Emitter>
2368 bool ByteCodeExprGen<Emitter>::VisitCXXBoolLiteralExpr(
2369     const CXXBoolLiteralExpr *E) {
2370   if (DiscardResult)
2371     return true;
2372
2373   return this->emitConstBool(E->getValue(), E);
2374 }
2375
2376 template <class Emitter>
2377 bool ByteCodeExprGen<Emitter>::VisitCXXNullPtrLiteralExpr(
2378     const CXXNullPtrLiteralExpr *E) {
2379   if (DiscardResult)
2380     return true;
2381
2382   return this->emitNullPtr(E);
2383 }
2384
2385 template <class Emitter>
2386 bool ByteCodeExprGen<Emitter>::VisitGNUNullExpr(const GNUNullExpr *E) {
2387   if (DiscardResult)
2388     return true;
2389
2390   assert(E->getType()->isIntegerType());
2391
2392   PrimType T = classifyPrim(E->getType());
2393   return this->emitZero(T, E);
2394 }
2395
2396 template <class Emitter>
2397 bool ByteCodeExprGen<Emitter>::VisitCXXThisExpr(const CXXThisExpr *E) {
2398   if (DiscardResult)
2399     return true;
2400
2401   if (this->LambdaThisCapture > 0)
2402     return this->emitGetThisFieldPtr(this->LambdaThisCapture, E);
2403
2404   return this->emitThis(E);
2405 }
2406
2407 template <class Emitter>
2408 bool ByteCodeExprGen<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
2409   const Expr *SubExpr = E->getSubExpr();
2410   std::optional<PrimType> T = classify(SubExpr->getType());
2411
2412   switch (E->getOpcode()) {
2413   case UO_PostInc: { // x++
2414     if (!this->visit(SubExpr))
2415       return false;
2416
2417     if (T == PT_Ptr) {
2418       if (!this->emitIncPtr(E))
2419         return false;
2420
2421       return DiscardResult ? this->emitPopPtr(E) : true;
2422     }
2423
2424     if (T == PT_Float) {
2425       return DiscardResult ? this->emitIncfPop(getRoundingMode(E), E)
2426                            : this->emitIncf(getRoundingMode(E), E);
2427     }
2428
2429     return DiscardResult ? this->emitIncPop(*T, E) : this->emitInc(*T, E);
2430   }
2431   case UO_PostDec: { // x--
2432     if (!this->visit(SubExpr))
2433       return false;
2434
2435     if (T == PT_Ptr) {
2436       if (!this->emitDecPtr(E))
2437         return false;
2438
2439       return DiscardResult ? this->emitPopPtr(E) : true;
2440     }
2441
2442     if (T == PT_Float) {
2443       return DiscardResult ? this->emitDecfPop(getRoundingMode(E), E)
2444                            : this->emitDecf(getRoundingMode(E), E);
2445     }
2446
2447     return DiscardResult ? this->emitDecPop(*T, E) : this->emitDec(*T, E);
2448   }
2449   case UO_PreInc: { // ++x
2450     if (!this->visit(SubExpr))
2451       return false;
2452
2453     if (T == PT_Ptr) {
2454       if (!this->emitLoadPtr(E))
2455         return false;
2456       if (!this->emitConstUint8(1, E))
2457         return false;
2458       if (!this->emitAddOffsetUint8(E))
2459         return false;
2460       return DiscardResult ? this->emitStorePopPtr(E) : this->emitStorePtr(E);
2461     }
2462
2463     // Post-inc and pre-inc are the same if the value is to be discarded.
2464     if (DiscardResult) {
2465       if (T == PT_Float)
2466         return this->emitIncfPop(getRoundingMode(E), E);
2467       return this->emitIncPop(*T, E);
2468     }
2469
2470     if (T == PT_Float) {
2471       const auto &TargetSemantics = Ctx.getFloatSemantics(E->getType());
2472       if (!this->emitLoadFloat(E))
2473         return false;
2474       if (!this->emitConstFloat(llvm::APFloat(TargetSemantics, 1), E))
2475         return false;
2476       if (!this->emitAddf(getRoundingMode(E), E))
2477         return false;
2478       return this->emitStoreFloat(E);
2479     }
2480     if (!this->emitLoad(*T, E))
2481       return false;
2482     if (!this->emitConst(1, E))
2483       return false;
2484     if (!this->emitAdd(*T, E))
2485       return false;
2486     return this->emitStore(*T, E);
2487   }
2488   case UO_PreDec: { // --x
2489     if (!this->visit(SubExpr))
2490       return false;
2491
2492     if (T == PT_Ptr) {
2493       if (!this->emitLoadPtr(E))
2494         return false;
2495       if (!this->emitConstUint8(1, E))
2496         return false;
2497       if (!this->emitSubOffsetUint8(E))
2498         return false;
2499       return DiscardResult ? this->emitStorePopPtr(E) : this->emitStorePtr(E);
2500     }
2501
2502     // Post-dec and pre-dec are the same if the value is to be discarded.
2503     if (DiscardResult) {
2504       if (T == PT_Float)
2505         return this->emitDecfPop(getRoundingMode(E), E);
2506       return this->emitDecPop(*T, E);
2507     }
2508
2509     if (T == PT_Float) {
2510       const auto &TargetSemantics = Ctx.getFloatSemantics(E->getType());
2511       if (!this->emitLoadFloat(E))
2512         return false;
2513       if (!this->emitConstFloat(llvm::APFloat(TargetSemantics, 1), E))
2514         return false;
2515       if (!this->emitSubf(getRoundingMode(E), E))
2516         return false;
2517       return this->emitStoreFloat(E);
2518     }
2519     if (!this->emitLoad(*T, E))
2520       return false;
2521     if (!this->emitConst(1, E))
2522       return false;
2523     if (!this->emitSub(*T, E))
2524       return false;
2525     return this->emitStore(*T, E);
2526   }
2527   case UO_LNot: // !x
2528     if (DiscardResult)
2529       return this->discard(SubExpr);
2530
2531     if (!this->visitBool(SubExpr))
2532       return false;
2533
2534     if (!this->emitInvBool(E))
2535       return false;
2536
2537     if (PrimType ET = classifyPrim(E->getType()); ET != PT_Bool)
2538       return this->emitCast(PT_Bool, ET, E);
2539     return true;
2540   case UO_Minus: // -x
2541     if (!this->visit(SubExpr))
2542       return false;
2543     return DiscardResult ? this->emitPop(*T, E) : this->emitNeg(*T, E);
2544   case UO_Plus:  // +x
2545     if (!this->visit(SubExpr)) // noop
2546       return false;
2547     return DiscardResult ? this->emitPop(*T, E) : true;
2548   case UO_AddrOf: // &x
2549     // We should already have a pointer when we get here.
2550     return this->delegate(SubExpr);
2551   case UO_Deref:  // *x
2552     return dereference(
2553         SubExpr, DerefKind::Read,
2554         [](PrimType) {
2555           llvm_unreachable("Dereferencing requires a pointer");
2556           return false;
2557         },
2558         [this, E](PrimType T) {
2559           return DiscardResult ? this->emitPop(T, E) : true;
2560         });
2561   case UO_Not:    // ~x
2562     if (!this->visit(SubExpr))
2563       return false;
2564     return DiscardResult ? this->emitPop(*T, E) : this->emitComp(*T, E);
2565   case UO_Real:   // __real x
2566   case UO_Imag:   // __imag x
2567   case UO_Extension:
2568     return this->delegate(SubExpr);
2569   case UO_Coawait:
2570     assert(false && "Unhandled opcode");
2571   }
2572
2573   return false;
2574 }
2575
2576 template <class Emitter>
2577 bool ByteCodeExprGen<Emitter>::VisitDeclRefExpr(const DeclRefExpr *E) {
2578   if (DiscardResult)
2579     return true;
2580
2581   const auto *D = E->getDecl();
2582
2583   if (const auto *ECD = dyn_cast<EnumConstantDecl>(D)) {
2584     return this->emitConst(ECD->getInitVal(), E);
2585   } else if (const auto *BD = dyn_cast<BindingDecl>(D)) {
2586     return this->visit(BD->getBinding());
2587   } else if (const auto *FuncDecl = dyn_cast<FunctionDecl>(D)) {
2588     const Function *F = getFunction(FuncDecl);
2589     return F && this->emitGetFnPtr(F, E);
2590   }
2591
2592   // References are implemented via pointers, so when we see a DeclRefExpr
2593   // pointing to a reference, we need to get its value directly (i.e. the
2594   // pointer to the actual value) instead of a pointer to the pointer to the
2595   // value.
2596   bool IsReference = D->getType()->isReferenceType();
2597
2598   // Check for local/global variables and parameters.
2599   if (auto It = Locals.find(D); It != Locals.end()) {
2600     const unsigned Offset = It->second.Offset;
2601
2602     if (IsReference)
2603       return this->emitGetLocal(PT_Ptr, Offset, E);
2604     return this->emitGetPtrLocal(Offset, E);
2605   } else if (auto GlobalIndex = P.getGlobal(D)) {
2606     if (IsReference)
2607       return this->emitGetGlobalPtr(*GlobalIndex, E);
2608
2609     return this->emitGetPtrGlobal(*GlobalIndex, E);
2610   } else if (const auto *PVD = dyn_cast<ParmVarDecl>(D)) {
2611     if (auto It = this->Params.find(PVD); It != this->Params.end()) {
2612       if (IsReference || !It->second.IsPtr)
2613         return this->emitGetParamPtr(It->second.Offset, E);
2614
2615       return this->emitGetPtrParam(It->second.Offset, E);
2616     }
2617   }
2618
2619   // Handle lambda captures.
2620   if (auto It = this->LambdaCaptures.find(D);
2621       It != this->LambdaCaptures.end()) {
2622     auto [Offset, IsPtr] = It->second;
2623
2624     if (IsPtr)
2625       return this->emitGetThisFieldPtr(Offset, E);
2626     return this->emitGetPtrThisField(Offset, E);
2627   }
2628
2629   // Lazily visit global declarations we haven't seen yet.
2630   // This happens in C.
2631   if (!Ctx.getLangOpts().CPlusPlus) {
2632     if (const auto *VD = dyn_cast<VarDecl>(D);
2633         VD && VD->hasGlobalStorage() && VD->getAnyInitializer() &&
2634         VD->getType().isConstQualified()) {
2635       if (!this->visitVarDecl(VD))
2636         return false;
2637       // Retry.
2638       return this->VisitDeclRefExpr(E);
2639     }
2640
2641     if (std::optional<unsigned> I = P.getOrCreateDummy(D))
2642       return this->emitGetPtrGlobal(*I, E);
2643   }
2644
2645   return this->emitInvalidDeclRef(E, E);
2646 }
2647
2648 template <class Emitter>
2649 void ByteCodeExprGen<Emitter>::emitCleanup() {
2650   for (VariableScope<Emitter> *C = VarScope; C; C = C->getParent())
2651     C->emitDestruction();
2652 }
2653
2654 template <class Emitter>
2655 unsigned
2656 ByteCodeExprGen<Emitter>::collectBaseOffset(const RecordType *BaseType,
2657                                             const RecordType *DerivedType) {
2658   const auto *FinalDecl = cast<CXXRecordDecl>(BaseType->getDecl());
2659   const RecordDecl *CurDecl = DerivedType->getDecl();
2660   const Record *CurRecord = getRecord(CurDecl);
2661   assert(CurDecl && FinalDecl);
2662
2663   unsigned OffsetSum = 0;
2664   for (;;) {
2665     assert(CurRecord->getNumBases() > 0);
2666     // One level up
2667     for (const Record::Base &B : CurRecord->bases()) {
2668       const auto *BaseDecl = cast<CXXRecordDecl>(B.Decl);
2669
2670       if (BaseDecl == FinalDecl || BaseDecl->isDerivedFrom(FinalDecl)) {
2671         OffsetSum += B.Offset;
2672         CurRecord = B.R;
2673         CurDecl = BaseDecl;
2674         break;
2675       }
2676     }
2677     if (CurDecl == FinalDecl)
2678       break;
2679   }
2680
2681   assert(OffsetSum > 0);
2682   return OffsetSum;
2683 }
2684
2685 /// Emit casts from a PrimType to another PrimType.
2686 template <class Emitter>
2687 bool ByteCodeExprGen<Emitter>::emitPrimCast(PrimType FromT, PrimType ToT,
2688                                             QualType ToQT, const Expr *E) {
2689
2690   if (FromT == PT_Float) {
2691     // Floating to floating.
2692     if (ToT == PT_Float) {
2693       const llvm::fltSemantics *ToSem = &Ctx.getFloatSemantics(ToQT);
2694       return this->emitCastFP(ToSem, getRoundingMode(E), E);
2695     }
2696
2697     // Float to integral.
2698     if (isIntegralType(ToT) || ToT == PT_Bool)
2699       return this->emitCastFloatingIntegral(ToT, E);
2700   }
2701
2702   if (isIntegralType(FromT) || FromT == PT_Bool) {
2703     // Integral to integral.
2704     if (isIntegralType(ToT) || ToT == PT_Bool)
2705       return FromT != ToT ? this->emitCast(FromT, ToT, E) : true;
2706
2707     if (ToT == PT_Float) {
2708       // Integral to floating.
2709       const llvm::fltSemantics *ToSem = &Ctx.getFloatSemantics(ToQT);
2710       return this->emitCastIntegralFloating(FromT, ToSem, getRoundingMode(E),
2711                                             E);
2712     }
2713   }
2714
2715   return false;
2716 }
2717
2718 /// When calling this, we have a pointer of the local-to-destroy
2719 /// on the stack.
2720 /// Emit destruction of record types (or arrays of record types).
2721 /// FIXME: Handle virtual destructors.
2722 template <class Emitter>
2723 bool ByteCodeExprGen<Emitter>::emitRecordDestruction(const Descriptor *Desc) {
2724   assert(Desc);
2725   assert(!Desc->isPrimitive());
2726   assert(!Desc->isPrimitiveArray());
2727
2728   // Arrays.
2729   if (Desc->isArray()) {
2730     const Descriptor *ElemDesc = Desc->ElemDesc;
2731     assert(ElemDesc);
2732
2733     // Don't need to do anything for these.
2734     if (ElemDesc->isPrimitiveArray())
2735       return this->emitPopPtr(SourceInfo{});
2736
2737     // If this is an array of record types, check if we need
2738     // to call the element destructors at all. If not, try
2739     // to save the work.
2740     if (const Record *ElemRecord = ElemDesc->ElemRecord) {
2741       if (const CXXDestructorDecl *Dtor = ElemRecord->getDestructor();
2742           !Dtor || Dtor->isTrivial())
2743         return this->emitPopPtr(SourceInfo{});
2744     }
2745
2746     for (ssize_t I = Desc->getNumElems() - 1; I >= 0; --I) {
2747       if (!this->emitConstUint64(I, SourceInfo{}))
2748         return false;
2749       if (!this->emitArrayElemPtrUint64(SourceInfo{}))
2750         return false;
2751       if (!this->emitRecordDestruction(ElemDesc))
2752         return false;
2753     }
2754     return this->emitPopPtr(SourceInfo{});
2755   }
2756
2757   const Record *R = Desc->ElemRecord;
2758   assert(R);
2759   // First, destroy all fields.
2760   for (const Record::Field &Field : llvm::reverse(R->fields())) {
2761     const Descriptor *D = Field.Desc;
2762     if (!D->isPrimitive() && !D->isPrimitiveArray()) {
2763       if (!this->emitDupPtr(SourceInfo{}))
2764         return false;
2765       if (!this->emitGetPtrField(Field.Offset, SourceInfo{}))
2766         return false;
2767       if (!this->emitRecordDestruction(D))
2768         return false;
2769     }
2770   }
2771
2772   // FIXME: Unions need to be handled differently here. We don't want to
2773   //   call the destructor of its members.
2774
2775   // Now emit the destructor and recurse into base classes.
2776   if (const CXXDestructorDecl *Dtor = R->getDestructor();
2777       Dtor && !Dtor->isTrivial()) {
2778     if (const Function *DtorFunc = getFunction(Dtor)) {
2779       assert(DtorFunc->hasThisPointer());
2780       assert(DtorFunc->getNumParams() == 1);
2781       if (!this->emitDupPtr(SourceInfo{}))
2782         return false;
2783       if (!this->emitCall(DtorFunc, SourceInfo{}))
2784         return false;
2785     }
2786   }
2787
2788   for (const Record::Base &Base : llvm::reverse(R->bases())) {
2789     if (!this->emitGetPtrBase(Base.Offset, SourceInfo{}))
2790       return false;
2791     if (!this->emitRecordDestruction(Base.Desc))
2792       return false;
2793   }
2794   // FIXME: Virtual bases.
2795
2796   // Remove the instance pointer.
2797   return this->emitPopPtr(SourceInfo{});
2798 }
2799
2800 namespace clang {
2801 namespace interp {
2802
2803 template class ByteCodeExprGen<ByteCodeEmitter>;
2804 template class ByteCodeExprGen<EvalEmitter>;
2805
2806 } // namespace interp
2807 } // namespace clang