clang/lib/AST/Interp/ByteCodeExprGen.cpp

   1 //===--- ByteCodeExprGen.cpp - Code generator for expressions ---*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "ByteCodeExprGen.h"
  10 #include "ByteCodeEmitter.h"
  11 #include "ByteCodeGenError.h"
  12 #include "ByteCodeStmtGen.h"
  13 #include "Context.h"
  14 #include "Floating.h"
  15 #include "Function.h"
  16 #include "PrimType.h"
  17 #include "Program.h"
  18
  19 using namespace clang;
  20 using namespace clang::interp;
  21
  22 using APSInt = llvm::APSInt;
  23
  24 namespace clang {
  25 namespace interp {
  26
  27 /// Scope used to handle temporaries in toplevel variable declarations.
  28 template <class Emitter> class DeclScope final : public VariableScope<Emitter> {
  29 public:
  30   DeclScope(ByteCodeExprGen<Emitter> *Ctx, const ValueDecl *VD)
  31       : VariableScope<Emitter>(Ctx), Scope(Ctx->P, VD),
  32         OldGlobalDecl(Ctx->GlobalDecl) {
  33     Ctx->GlobalDecl = Context::shouldBeGloballyIndexed(VD);
  34   }
  35
  36   void addExtended(const Scope::Local &Local) override {
  37     return this->addLocal(Local);
  38   }
  39
  40   ~DeclScope() { this->Ctx->GlobalDecl = OldGlobalDecl; }
  41
  42 private:
  43   Program::DeclScope Scope;
  44   bool OldGlobalDecl;
  45 };
  46
  47 /// Scope used to handle initialization methods.
  48 template <class Emitter> class OptionScope final {
  49 public:
  50   /// Root constructor, compiling or discarding primitives.
  51   OptionScope(ByteCodeExprGen<Emitter> *Ctx, bool NewDiscardResult,
  52               bool NewInitializing)
  53       : Ctx(Ctx), OldDiscardResult(Ctx->DiscardResult),
  54         OldInitializing(Ctx->Initializing) {
  55     Ctx->DiscardResult = NewDiscardResult;
  56     Ctx->Initializing = NewInitializing;
  57   }
  58
  59   ~OptionScope() {
  60     Ctx->DiscardResult = OldDiscardResult;
  61     Ctx->Initializing = OldInitializing;
  62   }
  63
  64 private:
  65   /// Parent context.
  66   ByteCodeExprGen<Emitter> *Ctx;
  67   /// Old discard flag to restore.
  68   bool OldDiscardResult;
  69   bool OldInitializing;
  70 };
  71
  72 } // namespace interp
  73 } // namespace clang
  74
  75 template <class Emitter>
  76 bool ByteCodeExprGen<Emitter>::VisitCastExpr(const CastExpr *CE) {
  77   const Expr *SubExpr = CE->getSubExpr();
  78   switch (CE->getCastKind()) {
  79
  80   case CK_LValueToRValue: {
  81     return dereference(
  82         SubExpr, DerefKind::Read,
  83         [](PrimType) {
  84           // Value loaded - nothing to do here.
  85           return true;
  86         },
  87         [this, CE](PrimType T) {
  88           // Pointer on stack - dereference it.
  89           if (!this->emitLoadPop(T, CE))
  90             return false;
  91           return DiscardResult ? this->emitPop(T, CE) : true;
  92         });
  93   }
  94
  95   case CK_UncheckedDerivedToBase:
  96   case CK_DerivedToBase: {
  97     if (!this->visit(SubExpr))
  98       return false;
  99
 100     unsigned DerivedOffset = collectBaseOffset(getRecordTy(CE->getType()),
 101                                                getRecordTy(SubExpr->getType()));
 102
 103     return this->emitGetPtrBasePop(DerivedOffset, CE);
 104   }
 105
 106   case CK_BaseToDerived: {
 107     if (!this->visit(SubExpr))
 108       return false;
 109
 110     unsigned DerivedOffset = collectBaseOffset(getRecordTy(SubExpr->getType()),
 111                                                getRecordTy(CE->getType()));
 112
 113     return this->emitGetPtrDerivedPop(DerivedOffset, CE);
 114   }
 115
 116   case CK_FloatingCast: {
 117     if (DiscardResult)
 118       return this->discard(SubExpr);
 119     if (!this->visit(SubExpr))
 120       return false;
 121     const auto *TargetSemantics = &Ctx.getFloatSemantics(CE->getType());
 122     return this->emitCastFP(TargetSemantics, getRoundingMode(CE), CE);
 123   }
 124
 125   case CK_IntegralToFloating: {
 126     if (DiscardResult)
 127       return this->discard(SubExpr);
 128     std::optional<PrimType> FromT = classify(SubExpr->getType());
 129     if (!FromT)
 130       return false;
 131
 132     if (!this->visit(SubExpr))
 133       return false;
 134
 135     const auto *TargetSemantics = &Ctx.getFloatSemantics(CE->getType());
 136     llvm::RoundingMode RM = getRoundingMode(CE);
 137     return this->emitCastIntegralFloating(*FromT, TargetSemantics, RM, CE);
 138   }
 139
 140   case CK_FloatingToBoolean:
 141   case CK_FloatingToIntegral: {
 142     if (DiscardResult)
 143       return this->discard(SubExpr);
 144
 145     std::optional<PrimType> ToT = classify(CE->getType());
 146
 147     if (!ToT)
 148       return false;
 149
 150     if (!this->visit(SubExpr))
 151       return false;
 152
 153     if (ToT == PT_IntAP)
 154       return this->emitCastFloatingIntegralAP(Ctx.getBitWidth(CE->getType()),
 155                                               CE);
 156     if (ToT == PT_IntAPS)
 157       return this->emitCastFloatingIntegralAPS(Ctx.getBitWidth(CE->getType()),
 158                                                CE);
 159
 160     return this->emitCastFloatingIntegral(*ToT, CE);
 161   }
 162
 163   case CK_NullToPointer:
 164     if (DiscardResult)
 165       return true;
 166     return this->emitNull(classifyPrim(CE->getType()), CE);
 167
 168   case CK_PointerToIntegral: {
 169     // TODO: Discard handling.
 170     if (!this->visit(SubExpr))
 171       return false;
 172
 173     PrimType T = classifyPrim(CE->getType());
 174     return this->emitCastPointerIntegral(T, CE);
 175   }
 176
 177   case CK_ArrayToPointerDecay: {
 178     if (!this->visit(SubExpr))
 179       return false;
 180     if (!this->emitArrayDecay(CE))
 181       return false;
 182     if (DiscardResult)
 183       return this->emitPopPtr(CE);
 184     return true;
 185   }
 186
 187   case CK_AtomicToNonAtomic:
 188   case CK_ConstructorConversion:
 189   case CK_FunctionToPointerDecay:
 190   case CK_NonAtomicToAtomic:
 191   case CK_NoOp:
 192   case CK_UserDefinedConversion:
 193   case CK_BitCast:
 194     return this->delegate(SubExpr);
 195
 196   case CK_IntegralToBoolean:
 197   case CK_IntegralCast: {
 198     if (DiscardResult)
 199       return this->discard(SubExpr);
 200     std::optional<PrimType> FromT = classify(SubExpr->getType());
 201     std::optional<PrimType> ToT = classify(CE->getType());
 202
 203     if (!FromT || !ToT)
 204       return false;
 205
 206     if (!this->visit(SubExpr))
 207       return false;
 208
 209     if (ToT == PT_IntAP)
 210       return this->emitCastAP(*FromT, Ctx.getBitWidth(CE->getType()), CE);
 211     if (ToT == PT_IntAPS)
 212       return this->emitCastAPS(*FromT, Ctx.getBitWidth(CE->getType()), CE);
 213
 214     if (FromT == ToT)
 215       return true;
 216     return this->emitCast(*FromT, *ToT, CE);
 217   }
 218
 219   case CK_PointerToBoolean: {
 220     PrimType PtrT = classifyPrim(SubExpr->getType());
 221
 222     // Just emit p != nullptr for this.
 223     if (!this->visit(SubExpr))
 224       return false;
 225
 226     if (!this->emitNull(PtrT, CE))
 227       return false;
 228
 229     return this->emitNE(PtrT, CE);
 230   }
 231
 232   case CK_IntegralComplexToBoolean:
 233   case CK_FloatingComplexToBoolean: {
 234     std::optional<PrimType> ElemT =
 235         classifyComplexElementType(SubExpr->getType());
 236     if (!ElemT)
 237       return false;
 238     // We emit the expression (__real(E) != 0 || __imag(E) != 0)
 239     // for us, that means (bool)E[0] || (bool)E[1]
 240     if (!this->visit(SubExpr))
 241       return false;
 242     if (!this->emitConstUint8(0, CE))
 243       return false;
 244     if (!this->emitArrayElemPtrUint8(CE))
 245       return false;
 246     if (!this->emitLoadPop(*ElemT, CE))
 247       return false;
 248     if (*ElemT == PT_Float) {
 249       if (!this->emitCastFloatingIntegral(PT_Bool, CE))
 250         return false;
 251     } else {
 252       if (!this->emitCast(*ElemT, PT_Bool, CE))
 253         return false;
 254     }
 255
 256     // We now have the bool value of E[0] on the stack.
 257     LabelTy LabelTrue = this->getLabel();
 258     if (!this->jumpTrue(LabelTrue))
 259       return false;
 260
 261     if (!this->emitConstUint8(1, CE))
 262       return false;
 263     if (!this->emitArrayElemPtrPopUint8(CE))
 264       return false;
 265     if (!this->emitLoadPop(*ElemT, CE))
 266       return false;
 267     if (*ElemT == PT_Float) {
 268       if (!this->emitCastFloatingIntegral(PT_Bool, CE))
 269         return false;
 270     } else {
 271       if (!this->emitCast(*ElemT, PT_Bool, CE))
 272         return false;
 273     }
 274     // Leave the boolean value of E[1] on the stack.
 275     LabelTy EndLabel = this->getLabel();
 276     this->jump(EndLabel);
 277
 278     this->emitLabel(LabelTrue);
 279     if (!this->emitPopPtr(CE))
 280       return false;
 281     if (!this->emitConstBool(true, CE))
 282       return false;
 283
 284     this->fallthrough(EndLabel);
 285     this->emitLabel(EndLabel);
 286
 287     return true;
 288   }
 289
 290   case CK_IntegralComplexToReal:
 291   case CK_FloatingComplexToReal:
 292     return this->emitComplexReal(SubExpr);
 293
 294   case CK_ToVoid:
 295     return discard(SubExpr);
 296
 297   default:
 298     assert(false && "Cast not implemented");
 299   }
 300   llvm_unreachable("Unhandled clang::CastKind enum");
 301 }
 302
 303 template <class Emitter>
 304 bool ByteCodeExprGen<Emitter>::VisitIntegerLiteral(const IntegerLiteral *LE) {
 305   if (DiscardResult)
 306     return true;
 307
 308   return this->emitConst(LE->getValue(), LE);
 309 }
 310
 311 template <class Emitter>
 312 bool ByteCodeExprGen<Emitter>::VisitFloatingLiteral(const FloatingLiteral *E) {
 313   if (DiscardResult)
 314     return true;
 315
 316   return this->emitConstFloat(E->getValue(), E);
 317 }
 318
 319 template <class Emitter>
 320 bool ByteCodeExprGen<Emitter>::VisitParenExpr(const ParenExpr *E) {
 321   return this->delegate(E->getSubExpr());
 322 }
 323
 324 template <class Emitter>
 325 bool ByteCodeExprGen<Emitter>::VisitBinaryOperator(const BinaryOperator *BO) {
 326   // Need short-circuiting for these.
 327   if (BO->isLogicalOp())
 328     return this->VisitLogicalBinOp(BO);
 329
 330   if (BO->getType()->isAnyComplexType())
 331     return this->VisitComplexBinOp(BO);
 332
 333   const Expr *LHS = BO->getLHS();
 334   const Expr *RHS = BO->getRHS();
 335
 336   if (BO->isPtrMemOp())
 337     return this->visit(RHS);
 338
 339   // Typecheck the args.
 340   std::optional<PrimType> LT = classify(LHS->getType());
 341   std::optional<PrimType> RT = classify(RHS->getType());
 342   std::optional<PrimType> T = classify(BO->getType());
 343
 344   // Deal with operations which have composite or void types.
 345   if (BO->isCommaOp()) {
 346     if (!this->discard(LHS))
 347       return false;
 348     if (RHS->getType()->isVoidType())
 349       return this->discard(RHS);
 350
 351     return this->delegate(RHS);
 352   }
 353
 354   // Special case for C++'s three-way/spaceship operator <=>, which
 355   // returns a std::{strong,weak,partial}_ordering (which is a class, so doesn't
 356   // have a PrimType).
 357   if (!T) {
 358     if (DiscardResult)
 359       return true;
 360     const ComparisonCategoryInfo *CmpInfo =
 361         Ctx.getASTContext().CompCategories.lookupInfoForType(BO->getType());
 362     assert(CmpInfo);
 363
 364     // We need a temporary variable holding our return value.
 365     if (!Initializing) {
 366       std::optional<unsigned> ResultIndex = this->allocateLocal(BO, false);
 367       if (!this->emitGetPtrLocal(*ResultIndex, BO))
 368         return false;
 369     }
 370
 371     if (!visit(LHS) || !visit(RHS))
 372       return false;
 373
 374     return this->emitCMP3(*LT, CmpInfo, BO);
 375   }
 376
 377   if (!LT || !RT || !T)
 378     return this->bail(BO);
 379
 380   // Pointer arithmetic special case.
 381   if (BO->getOpcode() == BO_Add || BO->getOpcode() == BO_Sub) {
 382     if (T == PT_Ptr || (LT == PT_Ptr && RT == PT_Ptr))
 383       return this->VisitPointerArithBinOp(BO);
 384   }
 385
 386   if (!visit(LHS) || !visit(RHS))
 387     return false;
 388
 389   // For languages such as C, cast the result of one
 390   // of our comparision opcodes to T (which is usually int).
 391   auto MaybeCastToBool = [this, T, BO](bool Result) {
 392     if (!Result)
 393       return false;
 394     if (DiscardResult)
 395       return this->emitPop(*T, BO);
 396     if (T != PT_Bool)
 397       return this->emitCast(PT_Bool, *T, BO);
 398     return true;
 399   };
 400
 401   auto Discard = [this, T, BO](bool Result) {
 402     if (!Result)
 403       return false;
 404     return DiscardResult ? this->emitPop(*T, BO) : true;
 405   };
 406
 407   switch (BO->getOpcode()) {
 408   case BO_EQ:
 409     return MaybeCastToBool(this->emitEQ(*LT, BO));
 410   case BO_NE:
 411     return MaybeCastToBool(this->emitNE(*LT, BO));
 412   case BO_LT:
 413     return MaybeCastToBool(this->emitLT(*LT, BO));
 414   case BO_LE:
 415     return MaybeCastToBool(this->emitLE(*LT, BO));
 416   case BO_GT:
 417     return MaybeCastToBool(this->emitGT(*LT, BO));
 418   case BO_GE:
 419     return MaybeCastToBool(this->emitGE(*LT, BO));
 420   case BO_Sub:
 421     if (BO->getType()->isFloatingType())
 422       return Discard(this->emitSubf(getRoundingMode(BO), BO));
 423     return Discard(this->emitSub(*T, BO));
 424   case BO_Add:
 425     if (BO->getType()->isFloatingType())
 426       return Discard(this->emitAddf(getRoundingMode(BO), BO));
 427     return Discard(this->emitAdd(*T, BO));
 428   case BO_Mul:
 429     if (BO->getType()->isFloatingType())
 430       return Discard(this->emitMulf(getRoundingMode(BO), BO));
 431     return Discard(this->emitMul(*T, BO));
 432   case BO_Rem:
 433     return Discard(this->emitRem(*T, BO));
 434   case BO_Div:
 435     if (BO->getType()->isFloatingType())
 436       return Discard(this->emitDivf(getRoundingMode(BO), BO));
 437     return Discard(this->emitDiv(*T, BO));
 438   case BO_Assign:
 439     if (DiscardResult)
 440       return LHS->refersToBitField() ? this->emitStoreBitFieldPop(*T, BO)
 441                                      : this->emitStorePop(*T, BO);
 442     return LHS->refersToBitField() ? this->emitStoreBitField(*T, BO)
 443                                    : this->emitStore(*T, BO);
 444   case BO_And:
 445     return Discard(this->emitBitAnd(*T, BO));
 446   case BO_Or:
 447     return Discard(this->emitBitOr(*T, BO));
 448   case BO_Shl:
 449     return Discard(this->emitShl(*LT, *RT, BO));
 450   case BO_Shr:
 451     return Discard(this->emitShr(*LT, *RT, BO));
 452   case BO_Xor:
 453     return Discard(this->emitBitXor(*T, BO));
 454   case BO_LOr:
 455   case BO_LAnd:
 456     llvm_unreachable("Already handled earlier");
 457   default:
 458     return this->bail(BO);
 459   }
 460
 461   llvm_unreachable("Unhandled binary op");
 462 }
 463
 464 /// Perform addition/subtraction of a pointer and an integer or
 465 /// subtraction of two pointers.
 466 template <class Emitter>
 467 bool ByteCodeExprGen<Emitter>::VisitPointerArithBinOp(const BinaryOperator *E) {
 468   BinaryOperatorKind Op = E->getOpcode();
 469   const Expr *LHS = E->getLHS();
 470   const Expr *RHS = E->getRHS();
 471
 472   if ((Op != BO_Add && Op != BO_Sub) ||
 473       (!LHS->getType()->isPointerType() && !RHS->getType()->isPointerType()))
 474     return false;
 475
 476   std::optional<PrimType> LT = classify(LHS);
 477   std::optional<PrimType> RT = classify(RHS);
 478
 479   if (!LT || !RT)
 480     return false;
 481
 482   if (LHS->getType()->isPointerType() && RHS->getType()->isPointerType()) {
 483     if (Op != BO_Sub)
 484       return false;
 485
 486     assert(E->getType()->isIntegerType());
 487     if (!visit(RHS) || !visit(LHS))
 488       return false;
 489
 490     return this->emitSubPtr(classifyPrim(E->getType()), E);
 491   }
 492
 493   PrimType OffsetType;
 494   if (LHS->getType()->isIntegerType()) {
 495     if (!visit(RHS) || !visit(LHS))
 496       return false;
 497     OffsetType = *LT;
 498   } else if (RHS->getType()->isIntegerType()) {
 499     if (!visit(LHS) || !visit(RHS))
 500       return false;
 501     OffsetType = *RT;
 502   } else {
 503     return false;
 504   }
 505
 506   if (Op == BO_Add)
 507     return this->emitAddOffset(OffsetType, E);
 508   else if (Op == BO_Sub)
 509     return this->emitSubOffset(OffsetType, E);
 510
 511   return this->bail(E);
 512 }
 513
 514 template <class Emitter>
 515 bool ByteCodeExprGen<Emitter>::VisitLogicalBinOp(const BinaryOperator *E) {
 516   assert(E->isLogicalOp());
 517   BinaryOperatorKind Op = E->getOpcode();
 518   const Expr *LHS = E->getLHS();
 519   const Expr *RHS = E->getRHS();
 520   std::optional<PrimType> T = classify(E->getType());
 521
 522   if (Op == BO_LOr) {
 523     // Logical OR. Visit LHS and only evaluate RHS if LHS was FALSE.
 524     LabelTy LabelTrue = this->getLabel();
 525     LabelTy LabelEnd = this->getLabel();
 526
 527     if (!this->visitBool(LHS))
 528       return false;
 529     if (!this->jumpTrue(LabelTrue))
 530       return false;
 531
 532     if (!this->visitBool(RHS))
 533       return false;
 534     if (!this->jump(LabelEnd))
 535       return false;
 536
 537     this->emitLabel(LabelTrue);
 538     this->emitConstBool(true, E);
 539     this->fallthrough(LabelEnd);
 540     this->emitLabel(LabelEnd);
 541
 542   } else {
 543     assert(Op == BO_LAnd);
 544     // Logical AND.
 545     // Visit LHS. Only visit RHS if LHS was TRUE.
 546     LabelTy LabelFalse = this->getLabel();
 547     LabelTy LabelEnd = this->getLabel();
 548
 549     if (!this->visitBool(LHS))
 550       return false;
 551     if (!this->jumpFalse(LabelFalse))
 552       return false;
 553
 554     if (!this->visitBool(RHS))
 555       return false;
 556     if (!this->jump(LabelEnd))
 557       return false;
 558
 559     this->emitLabel(LabelFalse);
 560     this->emitConstBool(false, E);
 561     this->fallthrough(LabelEnd);
 562     this->emitLabel(LabelEnd);
 563   }
 564
 565   if (DiscardResult)
 566     return this->emitPopBool(E);
 567
 568   // For C, cast back to integer type.
 569   assert(T);
 570   if (T != PT_Bool)
 571     return this->emitCast(PT_Bool, *T, E);
 572   return true;
 573 }
 574
 575 template <class Emitter>
 576 bool ByteCodeExprGen<Emitter>::VisitComplexBinOp(const BinaryOperator *E) {
 577   assert(Initializing);
 578
 579   const Expr *LHS = E->getLHS();
 580   const Expr *RHS = E->getRHS();
 581   PrimType LHSElemT = *this->classifyComplexElementType(LHS->getType());
 582   PrimType RHSElemT = *this->classifyComplexElementType(RHS->getType());
 583
 584   unsigned LHSOffset = this->allocateLocalPrimitive(LHS, PT_Ptr, true, false);
 585   unsigned RHSOffset = this->allocateLocalPrimitive(RHS, PT_Ptr, true, false);
 586   unsigned ResultOffset = ~0u;
 587   if (!this->DiscardResult)
 588     ResultOffset = this->allocateLocalPrimitive(E, PT_Ptr, true, false);
 589
 590   assert(LHSElemT == RHSElemT);
 591
 592   // Save result pointer in ResultOffset
 593   if (!this->DiscardResult) {
 594     if (!this->emitDupPtr(E))
 595       return false;
 596     if (!this->emitSetLocal(PT_Ptr, ResultOffset, E))
 597       return false;
 598   }
 599
 600   // Evaluate LHS and save value to LHSOffset.
 601   if (!this->visit(LHS))
 602     return false;
 603   if (!this->emitSetLocal(PT_Ptr, LHSOffset, E))
 604     return false;
 605
 606   // Same with RHS.
 607   if (!this->visit(RHS))
 608     return false;
 609   if (!this->emitSetLocal(PT_Ptr, RHSOffset, E))
 610     return false;
 611
 612   // Now we can get pointers to the LHS and RHS from the offsets above.
 613   BinaryOperatorKind Op = E->getOpcode();
 614   for (unsigned ElemIndex = 0; ElemIndex != 2; ++ElemIndex) {
 615     // Result pointer for the store later.
 616     if (!this->DiscardResult) {
 617       if (!this->emitGetLocal(PT_Ptr, ResultOffset, E))
 618         return false;
 619     }
 620
 621     if (!this->emitGetLocal(PT_Ptr, LHSOffset, E))
 622       return false;
 623     if (!this->emitConstUint8(ElemIndex, E))
 624       return false;
 625     if (!this->emitArrayElemPtrPopUint8(E))
 626       return false;
 627     if (!this->emitLoadPop(LHSElemT, E))
 628       return false;
 629
 630     if (!this->emitGetLocal(PT_Ptr, RHSOffset, E))
 631       return false;
 632     if (!this->emitConstUint8(ElemIndex, E))
 633       return false;
 634     if (!this->emitArrayElemPtrPopUint8(E))
 635       return false;
 636     if (!this->emitLoadPop(RHSElemT, E))
 637       return false;
 638
 639     // The actual operation.
 640     switch (Op) {
 641     case BO_Add:
 642       if (LHSElemT == PT_Float) {
 643         if (!this->emitAddf(getRoundingMode(E), E))
 644           return false;
 645       } else {
 646         if (!this->emitAdd(LHSElemT, E))
 647           return false;
 648       }
 649       break;
 650     case BO_Sub:
 651       if (LHSElemT == PT_Float) {
 652         if (!this->emitSubf(getRoundingMode(E), E))
 653           return false;
 654       } else {
 655         if (!this->emitSub(LHSElemT, E))
 656           return false;
 657       }
 658       break;
 659
 660     default:
 661       return false;
 662     }
 663
 664     if (!this->DiscardResult) {
 665       // Initialize array element with the value we just computed.
 666       if (!this->emitInitElemPop(LHSElemT, ElemIndex, E))
 667         return false;
 668     } else {
 669       if (!this->emitPop(LHSElemT, E))
 670         return false;
 671     }
 672   }
 673   return true;
 674 }
 675
 676 template <class Emitter>
 677 bool ByteCodeExprGen<Emitter>::VisitImplicitValueInitExpr(const ImplicitValueInitExpr *E) {
 678   QualType QT = E->getType();
 679
 680   if (std::optional<PrimType> T = classify(QT))
 681     return this->visitZeroInitializer(*T, QT, E);
 682
 683   if (QT->isRecordType())
 684     return false;
 685
 686   if (QT->isIncompleteArrayType())
 687     return true;
 688
 689   if (QT->isArrayType()) {
 690     const ArrayType *AT = QT->getAsArrayTypeUnsafe();
 691     assert(AT);
 692     const auto *CAT = cast<ConstantArrayType>(AT);
 693     size_t NumElems = CAT->getSize().getZExtValue();
 694     PrimType ElemT = classifyPrim(CAT->getElementType());
 695
 696     for (size_t I = 0; I != NumElems; ++I) {
 697       if (!this->visitZeroInitializer(ElemT, CAT->getElementType(), E))
 698         return false;
 699       if (!this->emitInitElem(ElemT, I, E))
 700         return false;
 701     }
 702
 703     return true;
 704   }
 705
 706   return false;
 707 }
 708
 709 template <class Emitter>
 710 bool ByteCodeExprGen<Emitter>::VisitArraySubscriptExpr(
 711     const ArraySubscriptExpr *E) {
 712   const Expr *Base = E->getBase();
 713   const Expr *Index = E->getIdx();
 714
 715   if (DiscardResult)
 716     return this->discard(Base) && this->discard(Index);
 717
 718   // Take pointer of LHS, add offset from RHS.
 719   // What's left on the stack after this is a pointer.
 720   if (!this->visit(Base))
 721     return false;
 722
 723   if (!this->visit(Index))
 724     return false;
 725
 726   PrimType IndexT = classifyPrim(Index->getType());
 727   return this->emitArrayElemPtrPop(IndexT, E);
 728 }
 729
 730 template <class Emitter>
 731 bool ByteCodeExprGen<Emitter>::visitInitList(ArrayRef<const Expr *> Inits,
 732                                              const Expr *E) {
 733   assert(E->getType()->isRecordType());
 734   const Record *R = getRecord(E->getType());
 735
 736   unsigned InitIndex = 0;
 737   for (const Expr *Init : Inits) {
 738     if (!this->emitDupPtr(E))
 739       return false;
 740
 741     if (std::optional<PrimType> T = classify(Init)) {
 742       const Record::Field *FieldToInit = R->getField(InitIndex);
 743       if (!this->visit(Init))
 744         return false;
 745
 746       if (FieldToInit->isBitField()) {
 747         if (!this->emitInitBitField(*T, FieldToInit, E))
 748           return false;
 749       } else {
 750         if (!this->emitInitField(*T, FieldToInit->Offset, E))
 751           return false;
 752       }
 753
 754       if (!this->emitPopPtr(E))
 755         return false;
 756       ++InitIndex;
 757     } else {
 758       // Initializer for a direct base class.
 759       if (const Record::Base *B = R->getBase(Init->getType())) {
 760         if (!this->emitGetPtrBasePop(B->Offset, Init))
 761           return false;
 762
 763         if (!this->visitInitializer(Init))
 764           return false;
 765
 766         if (!this->emitInitPtrPop(E))
 767           return false;
 768         // Base initializers don't increase InitIndex, since they don't count
 769         // into the Record's fields.
 770       } else {
 771         const Record::Field *FieldToInit = R->getField(InitIndex);
 772         // Non-primitive case. Get a pointer to the field-to-initialize
 773         // on the stack and recurse into visitInitializer().
 774         if (!this->emitGetPtrField(FieldToInit->Offset, Init))
 775           return false;
 776
 777         if (!this->visitInitializer(Init))
 778           return false;
 779
 780         if (!this->emitPopPtr(E))
 781           return false;
 782         ++InitIndex;
 783       }
 784     }
 785   }
 786   return true;
 787 }
 788
 789 /// Pointer to the array(not the element!) must be on the stack when calling
 790 /// this.
 791 template <class Emitter>
 792 bool ByteCodeExprGen<Emitter>::visitArrayElemInit(unsigned ElemIndex,
 793                                                   const Expr *Init) {
 794   if (std::optional<PrimType> T = classify(Init->getType())) {
 795     // Visit the primitive element like normal.
 796     if (!this->visit(Init))
 797       return false;
 798     return this->emitInitElem(*T, ElemIndex, Init);
 799   }
 800
 801   // Advance the pointer currently on the stack to the given
 802   // dimension.
 803   if (!this->emitConstUint32(ElemIndex, Init))
 804     return false;
 805   if (!this->emitArrayElemPtrUint32(Init))
 806     return false;
 807   if (!this->visitInitializer(Init))
 808     return false;
 809   return this->emitPopPtr(Init);
 810 }
 811
 812 template <class Emitter>
 813 bool ByteCodeExprGen<Emitter>::VisitInitListExpr(const InitListExpr *E) {
 814   // Handle discarding first.
 815   if (DiscardResult) {
 816     for (const Expr *Init : E->inits()) {
 817       if (!this->discard(Init))
 818         return false;
 819     }
 820     return true;
 821   }
 822
 823   // Primitive values.
 824   if (std::optional<PrimType> T = classify(E->getType())) {
 825     assert(!DiscardResult);
 826     if (E->getNumInits() == 0)
 827       return this->visitZeroInitializer(*T, E->getType(), E);
 828     assert(E->getNumInits() == 1);
 829     return this->delegate(E->inits()[0]);
 830   }
 831
 832   QualType T = E->getType();
 833   if (T->isRecordType())
 834     return this->visitInitList(E->inits(), E);
 835
 836   if (T->isArrayType()) {
 837     // FIXME: Array fillers.
 838     unsigned ElementIndex = 0;
 839     for (const Expr *Init : E->inits()) {
 840       if (!this->visitArrayElemInit(ElementIndex, Init))
 841         return false;
 842       ++ElementIndex;
 843     }
 844     return true;
 845   }
 846
 847   if (T->isAnyComplexType()) {
 848     unsigned NumInits = E->getNumInits();
 849     QualType ElemQT = E->getType()->getAs<ComplexType>()->getElementType();
 850     PrimType ElemT = classifyPrim(ElemQT);
 851     if (NumInits == 0) {
 852       // Zero-initialize both elements.
 853       for (unsigned I = 0; I < 2; ++I) {
 854         if (!this->visitZeroInitializer(ElemT, ElemQT, E))
 855           return false;
 856         if (!this->emitInitElem(ElemT, I, E))
 857           return false;
 858       }
 859     } else if (NumInits == 2) {
 860       unsigned InitIndex = 0;
 861       for (const Expr *Init : E->inits()) {
 862         if (!this->visit(Init))
 863           return false;
 864
 865         if (!this->emitInitElem(ElemT, InitIndex, E))
 866           return false;
 867         ++InitIndex;
 868       }
 869     }
 870     return true;
 871   }
 872
 873   return false;
 874 }
 875
 876 template <class Emitter>
 877 bool ByteCodeExprGen<Emitter>::VisitCXXParenListInitExpr(
 878     const CXXParenListInitExpr *E) {
 879   if (DiscardResult) {
 880     for (const Expr *Init : E->getInitExprs()) {
 881       if (!this->discard(Init))
 882         return false;
 883     }
 884     return true;
 885   }
 886
 887   assert(E->getType()->isRecordType());
 888   return this->visitInitList(E->getInitExprs(), E);
 889 }
 890
 891 template <class Emitter>
 892 bool ByteCodeExprGen<Emitter>::VisitSubstNonTypeTemplateParmExpr(
 893     const SubstNonTypeTemplateParmExpr *E) {
 894   return this->delegate(E->getReplacement());
 895 }
 896
 897 template <class Emitter>
 898 bool ByteCodeExprGen<Emitter>::VisitConstantExpr(const ConstantExpr *E) {
 899   // Try to emit the APValue directly, without visiting the subexpr.
 900   // This will only fail if we can't emit the APValue, so won't emit any
 901   // diagnostics or any double values.
 902   std::optional<PrimType> T = classify(E->getType());
 903   if (T && E->hasAPValueResult() &&
 904       this->visitAPValue(E->getAPValueResult(), *T, E))
 905     return true;
 906
 907   return this->delegate(E->getSubExpr());
 908 }
 909
 910 static CharUnits AlignOfType(QualType T, const ASTContext &ASTCtx,
 911                              UnaryExprOrTypeTrait Kind) {
 912   bool AlignOfReturnsPreferred =
 913       ASTCtx.getLangOpts().getClangABICompat() <= LangOptions::ClangABI::Ver7;
 914
 915   // C++ [expr.alignof]p3:
 916   //     When alignof is applied to a reference type, the result is the
 917   //     alignment of the referenced type.
 918   if (const auto *Ref = T->getAs<ReferenceType>())
 919     T = Ref->getPointeeType();
 920
 921   // __alignof is defined to return the preferred alignment.
 922   // Before 8, clang returned the preferred alignment for alignof and
 923   // _Alignof as well.
 924   if (Kind == UETT_PreferredAlignOf || AlignOfReturnsPreferred)
 925     return ASTCtx.toCharUnitsFromBits(ASTCtx.getPreferredTypeAlign(T));
 926
 927   return ASTCtx.getTypeAlignInChars(T);
 928 }
 929
 930 template <class Emitter>
 931 bool ByteCodeExprGen<Emitter>::VisitUnaryExprOrTypeTraitExpr(
 932     const UnaryExprOrTypeTraitExpr *E) {
 933   UnaryExprOrTypeTrait Kind = E->getKind();
 934   ASTContext &ASTCtx = Ctx.getASTContext();
 935
 936   if (Kind == UETT_SizeOf) {
 937     QualType ArgType = E->getTypeOfArgument();
 938     CharUnits Size;
 939     if (ArgType->isVoidType() || ArgType->isFunctionType())
 940       Size = CharUnits::One();
 941     else {
 942       if (ArgType->isDependentType() || !ArgType->isConstantSizeType())
 943         return false;
 944
 945       Size = ASTCtx.getTypeSizeInChars(ArgType);
 946     }
 947
 948     if (DiscardResult)
 949       return true;
 950
 951     return this->emitConst(Size.getQuantity(), E);
 952   }
 953
 954   if (Kind == UETT_AlignOf || Kind == UETT_PreferredAlignOf) {
 955     CharUnits Size;
 956
 957     if (E->isArgumentType()) {
 958       QualType ArgType = E->getTypeOfArgument();
 959
 960       Size = AlignOfType(ArgType, ASTCtx, Kind);
 961     } else {
 962       // Argument is an expression, not a type.
 963       const Expr *Arg = E->getArgumentExpr()->IgnoreParens();
 964
 965       // The kinds of expressions that we have special-case logic here for
 966       // should be kept up to date with the special checks for those
 967       // expressions in Sema.
 968
 969       // alignof decl is always accepted, even if it doesn't make sense: we
 970       // default to 1 in those cases.
 971       if (const auto *DRE = dyn_cast<DeclRefExpr>(Arg))
 972         Size = ASTCtx.getDeclAlign(DRE->getDecl(),
 973                                    /*RefAsPointee*/ true);
 974       else if (const auto *ME = dyn_cast<MemberExpr>(Arg))
 975         Size = ASTCtx.getDeclAlign(ME->getMemberDecl(),
 976                                    /*RefAsPointee*/ true);
 977       else
 978         Size = AlignOfType(Arg->getType(), ASTCtx, Kind);
 979     }
 980
 981     if (DiscardResult)
 982       return true;
 983
 984     return this->emitConst(Size.getQuantity(), E);
 985   }
 986
 987   return false;
 988 }
 989
 990 template <class Emitter>
 991 bool ByteCodeExprGen<Emitter>::VisitMemberExpr(const MemberExpr *E) {
 992   // 'Base.Member'
 993   const Expr *Base = E->getBase();
 994
 995   if (DiscardResult)
 996     return this->discard(Base);
 997
 998   if (!this->visit(Base))
 999     return false;
1000
1001   // Base above gives us a pointer on the stack.
1002   // TODO: Implement non-FieldDecl members.
1003   const ValueDecl *Member = E->getMemberDecl();
1004   if (const auto *FD = dyn_cast<FieldDecl>(Member)) {
1005     const RecordDecl *RD = FD->getParent();
1006     const Record *R = getRecord(RD);
1007     const Record::Field *F = R->getField(FD);
1008     // Leave a pointer to the field on the stack.
1009     if (F->Decl->getType()->isReferenceType())
1010       return this->emitGetFieldPop(PT_Ptr, F->Offset, E);
1011     return this->emitGetPtrField(F->Offset, E);
1012   }
1013
1014   return false;
1015 }
1016
1017 template <class Emitter>
1018 bool ByteCodeExprGen<Emitter>::VisitArrayInitIndexExpr(
1019     const ArrayInitIndexExpr *E) {
1020   // ArrayIndex might not be set if a ArrayInitIndexExpr is being evaluated
1021   // stand-alone, e.g. via EvaluateAsInt().
1022   if (!ArrayIndex)
1023     return false;
1024   return this->emitConst(*ArrayIndex, E);
1025 }
1026
1027 template <class Emitter>
1028 bool ByteCodeExprGen<Emitter>::VisitArrayInitLoopExpr(
1029     const ArrayInitLoopExpr *E) {
1030   assert(Initializing);
1031   assert(!DiscardResult);
1032   // TODO: This compiles to quite a lot of bytecode if the array is larger.
1033   //   Investigate compiling this to a loop.
1034
1035   const Expr *SubExpr = E->getSubExpr();
1036   const Expr *CommonExpr = E->getCommonExpr();
1037   size_t Size = E->getArraySize().getZExtValue();
1038
1039   // If the common expression is an opaque expression, we visit it
1040   // here once so we have its value cached.
1041   // FIXME: This might be necessary (or useful) for all expressions.
1042   if (isa<OpaqueValueExpr>(CommonExpr)) {
1043     if (!this->discard(CommonExpr))
1044       return false;
1045   }
1046
1047   // So, every iteration, we execute an assignment here
1048   // where the LHS is on the stack (the target array)
1049   // and the RHS is our SubExpr.
1050   for (size_t I = 0; I != Size; ++I) {
1051     ArrayIndexScope<Emitter> IndexScope(this, I);
1052     BlockScope<Emitter> BS(this);
1053
1054     if (!this->visitArrayElemInit(I, SubExpr))
1055       return false;
1056   }
1057   return true;
1058 }
1059
1060 template <class Emitter>
1061 bool ByteCodeExprGen<Emitter>::VisitOpaqueValueExpr(const OpaqueValueExpr *E) {
1062   if (Initializing)
1063     return this->visitInitializer(E->getSourceExpr());
1064
1065   PrimType SubExprT = classify(E->getSourceExpr()).value_or(PT_Ptr);
1066   if (auto It = OpaqueExprs.find(E); It != OpaqueExprs.end())
1067     return this->emitGetLocal(SubExprT, It->second, E);
1068
1069   if (!this->visit(E->getSourceExpr()))
1070     return false;
1071
1072   // At this point we either have the evaluated source expression or a pointer
1073   // to an object on the stack. We want to create a local variable that stores
1074   // this value.
1075   std::optional<unsigned> LocalIndex =
1076       allocateLocalPrimitive(E, SubExprT, /*IsConst=*/true);
1077   if (!LocalIndex)
1078     return false;
1079   if (!this->emitSetLocal(SubExprT, *LocalIndex, E))
1080     return false;
1081
1082   // Here the local variable is created but the value is removed from the stack,
1083   // so we put it back, because the caller might need it.
1084   if (!DiscardResult) {
1085     if (!this->emitGetLocal(SubExprT, *LocalIndex, E))
1086       return false;
1087   }
1088
1089   // FIXME: Ideally the cached value should be cleaned up later.
1090   OpaqueExprs.insert({E, *LocalIndex});
1091
1092   return true;
1093 }
1094
1095 template <class Emitter>
1096 bool ByteCodeExprGen<Emitter>::VisitAbstractConditionalOperator(
1097     const AbstractConditionalOperator *E) {
1098   const Expr *Condition = E->getCond();
1099   const Expr *TrueExpr = E->getTrueExpr();
1100   const Expr *FalseExpr = E->getFalseExpr();
1101
1102   LabelTy LabelEnd = this->getLabel();   // Label after the operator.
1103   LabelTy LabelFalse = this->getLabel(); // Label for the false expr.
1104
1105   if (!this->visitBool(Condition))
1106     return false;
1107
1108   if (!this->jumpFalse(LabelFalse))
1109     return false;
1110
1111   if (!this->delegate(TrueExpr))
1112     return false;
1113   if (!this->jump(LabelEnd))
1114     return false;
1115
1116   this->emitLabel(LabelFalse);
1117
1118   if (!this->delegate(FalseExpr))
1119     return false;
1120
1121   this->fallthrough(LabelEnd);
1122   this->emitLabel(LabelEnd);
1123
1124   return true;
1125 }
1126
1127 template <class Emitter>
1128 bool ByteCodeExprGen<Emitter>::VisitStringLiteral(const StringLiteral *E) {
1129   if (DiscardResult)
1130     return true;
1131
1132   if (!Initializing) {
1133     unsigned StringIndex = P.createGlobalString(E);
1134     return this->emitGetPtrGlobal(StringIndex, E);
1135   }
1136
1137   // We are initializing an array on the stack.
1138   const ConstantArrayType *CAT =
1139       Ctx.getASTContext().getAsConstantArrayType(E->getType());
1140   assert(CAT && "a string literal that's not a constant array?");
1141
1142   // If the initializer string is too long, a diagnostic has already been
1143   // emitted. Read only the array length from the string literal.
1144   unsigned ArraySize = CAT->getSize().getZExtValue();
1145   unsigned N = std::min(ArraySize, E->getLength());
1146   size_t CharWidth = E->getCharByteWidth();
1147
1148   for (unsigned I = 0; I != N; ++I) {
1149     uint32_t CodeUnit = E->getCodeUnit(I);
1150
1151     if (CharWidth == 1) {
1152       this->emitConstSint8(CodeUnit, E);
1153       this->emitInitElemSint8(I, E);
1154     } else if (CharWidth == 2) {
1155       this->emitConstUint16(CodeUnit, E);
1156       this->emitInitElemUint16(I, E);
1157     } else if (CharWidth == 4) {
1158       this->emitConstUint32(CodeUnit, E);
1159       this->emitInitElemUint32(I, E);
1160     } else {
1161       llvm_unreachable("unsupported character width");
1162     }
1163   }
1164
1165   // Fill up the rest of the char array with NUL bytes.
1166   for (unsigned I = N; I != ArraySize; ++I) {
1167     if (CharWidth == 1) {
1168       this->emitConstSint8(0, E);
1169       this->emitInitElemSint8(I, E);
1170     } else if (CharWidth == 2) {
1171       this->emitConstUint16(0, E);
1172       this->emitInitElemUint16(I, E);
1173     } else if (CharWidth == 4) {
1174       this->emitConstUint32(0, E);
1175       this->emitInitElemUint32(I, E);
1176     } else {
1177       llvm_unreachable("unsupported character width");
1178     }
1179   }
1180
1181   return true;
1182 }
1183
1184 template <class Emitter>
1185 bool ByteCodeExprGen<Emitter>::VisitCharacterLiteral(
1186     const CharacterLiteral *E) {
1187   if (DiscardResult)
1188     return true;
1189   return this->emitConst(E->getValue(), E);
1190 }
1191
1192 template <class Emitter>
1193 bool ByteCodeExprGen<Emitter>::VisitFloatCompoundAssignOperator(
1194     const CompoundAssignOperator *E) {
1195
1196   const Expr *LHS = E->getLHS();
1197   const Expr *RHS = E->getRHS();
1198   QualType LHSType = LHS->getType();
1199   QualType LHSComputationType = E->getComputationLHSType();
1200   QualType ResultType = E->getComputationResultType();
1201   std::optional<PrimType> LT = classify(LHSComputationType);
1202   std::optional<PrimType> RT = classify(ResultType);
1203
1204   assert(ResultType->isFloatingType());
1205
1206   if (!LT || !RT)
1207     return false;
1208
1209   PrimType LHST = classifyPrim(LHSType);
1210
1211   // C++17 onwards require that we evaluate the RHS first.
1212   // Compute RHS and save it in a temporary variable so we can
1213   // load it again later.
1214   if (!visit(RHS))
1215     return false;
1216
1217   unsigned TempOffset = this->allocateLocalPrimitive(E, *RT, /*IsConst=*/true);
1218   if (!this->emitSetLocal(*RT, TempOffset, E))
1219     return false;
1220
1221   // First, visit LHS.
1222   if (!visit(LHS))
1223     return false;
1224   if (!this->emitLoad(LHST, E))
1225     return false;
1226
1227   // If necessary, convert LHS to its computation type.
1228   if (!this->emitPrimCast(LHST, classifyPrim(LHSComputationType),
1229                           LHSComputationType, E))
1230     return false;
1231
1232   // Now load RHS.
1233   if (!this->emitGetLocal(*RT, TempOffset, E))
1234     return false;
1235
1236   llvm::RoundingMode RM = getRoundingMode(E);
1237   switch (E->getOpcode()) {
1238   case BO_AddAssign:
1239     if (!this->emitAddf(RM, E))
1240       return false;
1241     break;
1242   case BO_SubAssign:
1243     if (!this->emitSubf(RM, E))
1244       return false;
1245     break;
1246   case BO_MulAssign:
1247     if (!this->emitMulf(RM, E))
1248       return false;
1249     break;
1250   case BO_DivAssign:
1251     if (!this->emitDivf(RM, E))
1252       return false;
1253     break;
1254   default:
1255     return false;
1256   }
1257
1258   if (!this->emitPrimCast(classifyPrim(ResultType), LHST, LHS->getType(), E))
1259     return false;
1260
1261   if (DiscardResult)
1262     return this->emitStorePop(LHST, E);
1263   return this->emitStore(LHST, E);
1264 }
1265
1266 template <class Emitter>
1267 bool ByteCodeExprGen<Emitter>::VisitPointerCompoundAssignOperator(
1268     const CompoundAssignOperator *E) {
1269   BinaryOperatorKind Op = E->getOpcode();
1270   const Expr *LHS = E->getLHS();
1271   const Expr *RHS = E->getRHS();
1272   std::optional<PrimType> LT = classify(LHS->getType());
1273   std::optional<PrimType> RT = classify(RHS->getType());
1274
1275   if (Op != BO_AddAssign && Op != BO_SubAssign)
1276     return false;
1277
1278   if (!LT || !RT)
1279     return false;
1280   assert(*LT == PT_Ptr);
1281
1282   if (!visit(LHS))
1283     return false;
1284
1285   if (!this->emitLoadPtr(LHS))
1286     return false;
1287
1288   if (!visit(RHS))
1289     return false;
1290
1291   if (Op == BO_AddAssign)
1292     this->emitAddOffset(*RT, E);
1293   else
1294     this->emitSubOffset(*RT, E);
1295
1296   if (DiscardResult)
1297     return this->emitStorePopPtr(E);
1298   return this->emitStorePtr(E);
1299 }
1300
1301 template <class Emitter>
1302 bool ByteCodeExprGen<Emitter>::VisitCompoundAssignOperator(
1303     const CompoundAssignOperator *E) {
1304
1305   const Expr *LHS = E->getLHS();
1306   const Expr *RHS = E->getRHS();
1307   std::optional<PrimType> LHSComputationT =
1308       classify(E->getComputationLHSType());
1309   std::optional<PrimType> LT = classify(LHS->getType());
1310   std::optional<PrimType> RT = classify(E->getComputationResultType());
1311   std::optional<PrimType> ResultT = classify(E->getType());
1312
1313   if (!LT || !RT || !ResultT || !LHSComputationT)
1314     return false;
1315
1316   // Handle floating point operations separately here, since they
1317   // require special care.
1318
1319   if (ResultT == PT_Float || RT == PT_Float)
1320     return VisitFloatCompoundAssignOperator(E);
1321
1322   if (E->getType()->isPointerType())
1323     return VisitPointerCompoundAssignOperator(E);
1324
1325   assert(!E->getType()->isPointerType() && "Handled above");
1326   assert(!E->getType()->isFloatingType() && "Handled above");
1327
1328   // C++17 onwards require that we evaluate the RHS first.
1329   // Compute RHS and save it in a temporary variable so we can
1330   // load it again later.
1331   // FIXME: Compound assignments are unsequenced in C, so we might
1332   //   have to figure out how to reject them.
1333   if (!visit(RHS))
1334     return false;
1335
1336   unsigned TempOffset = this->allocateLocalPrimitive(E, *RT, /*IsConst=*/true);
1337
1338   if (!this->emitSetLocal(*RT, TempOffset, E))
1339     return false;
1340
1341   // Get LHS pointer, load its value and cast it to the
1342   // computation type if necessary.
1343   if (!visit(LHS))
1344     return false;
1345   if (!this->emitLoad(*LT, E))
1346     return false;
1347   if (*LT != *LHSComputationT) {
1348     if (!this->emitCast(*LT, *LHSComputationT, E))
1349       return false;
1350   }
1351
1352   // Get the RHS value on the stack.
1353   if (!this->emitGetLocal(*RT, TempOffset, E))
1354     return false;
1355
1356   // Perform operation.
1357   switch (E->getOpcode()) {
1358   case BO_AddAssign:
1359     if (!this->emitAdd(*LHSComputationT, E))
1360       return false;
1361     break;
1362   case BO_SubAssign:
1363     if (!this->emitSub(*LHSComputationT, E))
1364       return false;
1365     break;
1366   case BO_MulAssign:
1367     if (!this->emitMul(*LHSComputationT, E))
1368       return false;
1369     break;
1370   case BO_DivAssign:
1371     if (!this->emitDiv(*LHSComputationT, E))
1372       return false;
1373     break;
1374   case BO_RemAssign:
1375     if (!this->emitRem(*LHSComputationT, E))
1376       return false;
1377     break;
1378   case BO_ShlAssign:
1379     if (!this->emitShl(*LHSComputationT, *RT, E))
1380       return false;
1381     break;
1382   case BO_ShrAssign:
1383     if (!this->emitShr(*LHSComputationT, *RT, E))
1384       return false;
1385     break;
1386   case BO_AndAssign:
1387     if (!this->emitBitAnd(*LHSComputationT, E))
1388       return false;
1389     break;
1390   case BO_XorAssign:
1391     if (!this->emitBitXor(*LHSComputationT, E))
1392       return false;
1393     break;
1394   case BO_OrAssign:
1395     if (!this->emitBitOr(*LHSComputationT, E))
1396       return false;
1397     break;
1398   default:
1399     llvm_unreachable("Unimplemented compound assign operator");
1400   }
1401
1402   // And now cast from LHSComputationT to ResultT.
1403   if (*ResultT != *LHSComputationT) {
1404     if (!this->emitCast(*LHSComputationT, *ResultT, E))
1405       return false;
1406   }
1407
1408   // And store the result in LHS.
1409   if (DiscardResult) {
1410     if (LHS->refersToBitField())
1411       return this->emitStoreBitFieldPop(*ResultT, E);
1412     return this->emitStorePop(*ResultT, E);
1413   }
1414   if (LHS->refersToBitField())
1415     return this->emitStoreBitField(*ResultT, E);
1416   return this->emitStore(*ResultT, E);
1417 }
1418
1419 template <class Emitter>
1420 bool ByteCodeExprGen<Emitter>::VisitExprWithCleanups(
1421     const ExprWithCleanups *E) {
1422   const Expr *SubExpr = E->getSubExpr();
1423
1424   assert(E->getNumObjects() == 0 && "TODO: Implement cleanups");
1425
1426   return this->delegate(SubExpr);
1427 }
1428
1429 template <class Emitter>
1430 bool ByteCodeExprGen<Emitter>::VisitMaterializeTemporaryExpr(
1431     const MaterializeTemporaryExpr *E) {
1432   const Expr *SubExpr = E->getSubExpr();
1433
1434   if (Initializing) {
1435     // We already have a value, just initialize that.
1436     return this->visitInitializer(SubExpr);
1437   }
1438   // If we don't end up using the materialized temporary anyway, don't
1439   // bother creating it.
1440   if (DiscardResult)
1441     return this->discard(SubExpr);
1442
1443   // When we're initializing a global variable *or* the storage duration of
1444   // the temporary is explicitly static, create a global variable.
1445   std::optional<PrimType> SubExprT = classify(SubExpr);
1446   bool IsStatic = E->getStorageDuration() == SD_Static;
1447   if (GlobalDecl || IsStatic) {
1448     std::optional<unsigned> GlobalIndex = P.createGlobal(E);
1449     if (!GlobalIndex)
1450       return false;
1451
1452     const LifetimeExtendedTemporaryDecl *TempDecl =
1453         E->getLifetimeExtendedTemporaryDecl();
1454     if (IsStatic)
1455       assert(TempDecl);
1456
1457     if (SubExprT) {
1458       if (!this->visit(SubExpr))
1459         return false;
1460       if (IsStatic) {
1461         if (!this->emitInitGlobalTemp(*SubExprT, *GlobalIndex, TempDecl, E))
1462           return false;
1463       } else {
1464         if (!this->emitInitGlobal(*SubExprT, *GlobalIndex, E))
1465           return false;
1466       }
1467       return this->emitGetPtrGlobal(*GlobalIndex, E);
1468     }
1469
1470     // Non-primitive values.
1471     if (!this->emitGetPtrGlobal(*GlobalIndex, E))
1472       return false;
1473     if (!this->visitInitializer(SubExpr))
1474       return false;
1475     if (IsStatic)
1476       return this->emitInitGlobalTempComp(TempDecl, E);
1477     return true;
1478   }
1479
1480   // For everyhing else, use local variables.
1481   if (SubExprT) {
1482     if (std::optional<unsigned> LocalIndex = allocateLocalPrimitive(
1483             SubExpr, *SubExprT, /*IsConst=*/true, /*IsExtended=*/true)) {
1484       if (!this->visit(SubExpr))
1485         return false;
1486       this->emitSetLocal(*SubExprT, *LocalIndex, E);
1487       return this->emitGetPtrLocal(*LocalIndex, E);
1488     }
1489   } else {
1490     if (std::optional<unsigned> LocalIndex =
1491             allocateLocal(SubExpr, /*IsExtended=*/true)) {
1492       if (!this->emitGetPtrLocal(*LocalIndex, E))
1493         return false;
1494       return this->visitInitializer(SubExpr);
1495     }
1496   }
1497   return false;
1498 }
1499
1500 template <class Emitter>
1501 bool ByteCodeExprGen<Emitter>::VisitCXXBindTemporaryExpr(
1502     const CXXBindTemporaryExpr *E) {
1503   return this->delegate(E->getSubExpr());
1504 }
1505
1506 template <class Emitter>
1507 bool ByteCodeExprGen<Emitter>::VisitCompoundLiteralExpr(
1508     const CompoundLiteralExpr *E) {
1509   const Expr *Init = E->getInitializer();
1510   if (Initializing) {
1511     // We already have a value, just initialize that.
1512     return this->visitInitializer(Init);
1513   }
1514
1515   std::optional<PrimType> T = classify(E->getType());
1516   if (E->isFileScope()) {
1517     if (std::optional<unsigned> GlobalIndex = P.createGlobal(E)) {
1518       if (classify(E->getType()))
1519         return this->visit(Init);
1520       if (!this->emitGetPtrGlobal(*GlobalIndex, E))
1521         return false;
1522       return this->visitInitializer(Init);
1523     }
1524   }
1525
1526   // Otherwise, use a local variable.
1527   if (T) {
1528     // For primitive types, we just visit the initializer.
1529     return this->delegate(Init);
1530   } else {
1531     if (std::optional<unsigned> LocalIndex = allocateLocal(Init)) {
1532       if (!this->emitGetPtrLocal(*LocalIndex, E))
1533         return false;
1534       if (!this->visitInitializer(Init))
1535         return false;
1536       if (DiscardResult)
1537         return this->emitPopPtr(E);
1538       return true;
1539     }
1540   }
1541
1542   return false;
1543 }
1544
1545 template <class Emitter>
1546 bool ByteCodeExprGen<Emitter>::VisitTypeTraitExpr(const TypeTraitExpr *E) {
1547   if (DiscardResult)
1548     return true;
1549   return this->emitConstBool(E->getValue(), E);
1550 }
1551
1552 template <class Emitter>
1553 bool ByteCodeExprGen<Emitter>::VisitLambdaExpr(const LambdaExpr *E) {
1554   assert(Initializing);
1555   const Record *R = P.getOrCreateRecord(E->getLambdaClass());
1556
1557   auto *CaptureInitIt = E->capture_init_begin();
1558   // Initialize all fields (which represent lambda captures) of the
1559   // record with their initializers.
1560   for (const Record::Field &F : R->fields()) {
1561     const Expr *Init = *CaptureInitIt;
1562     ++CaptureInitIt;
1563
1564     if (std::optional<PrimType> T = classify(Init)) {
1565       if (!this->visit(Init))
1566         return false;
1567
1568       if (!this->emitSetField(*T, F.Offset, E))
1569         return false;
1570     } else {
1571       if (!this->emitDupPtr(E))
1572         return false;
1573
1574       if (!this->emitGetPtrField(F.Offset, E))
1575         return false;
1576
1577       if (!this->visitInitializer(Init))
1578         return false;
1579
1580       if (!this->emitPopPtr(E))
1581         return false;
1582     }
1583   }
1584
1585   return true;
1586 }
1587
1588 template <class Emitter>
1589 bool ByteCodeExprGen<Emitter>::VisitPredefinedExpr(const PredefinedExpr *E) {
1590   if (DiscardResult)
1591     return true;
1592
1593   assert(!Initializing);
1594   return this->visit(E->getFunctionName());
1595 }
1596
1597 template <class Emitter>
1598 bool ByteCodeExprGen<Emitter>::VisitCXXThrowExpr(const CXXThrowExpr *E) {
1599   if (E->getSubExpr() && !this->discard(E->getSubExpr()))
1600     return false;
1601
1602   return this->emitInvalid(E);
1603 }
1604
1605 template <class Emitter>
1606 bool ByteCodeExprGen<Emitter>::VisitCXXReinterpretCastExpr(
1607     const CXXReinterpretCastExpr *E) {
1608   if (!this->discard(E->getSubExpr()))
1609     return false;
1610
1611   return this->emitInvalidCast(CastKind::Reinterpret, E);
1612 }
1613
1614 template <class Emitter>
1615 bool ByteCodeExprGen<Emitter>::VisitCXXNoexceptExpr(const CXXNoexceptExpr *E) {
1616   assert(E->getType()->isBooleanType());
1617
1618   if (DiscardResult)
1619     return true;
1620   return this->emitConstBool(E->getValue(), E);
1621 }
1622
1623 template <class Emitter>
1624 bool ByteCodeExprGen<Emitter>::VisitCXXConstructExpr(
1625     const CXXConstructExpr *E) {
1626   QualType T = E->getType();
1627   assert(!classify(T));
1628
1629   if (T->isRecordType()) {
1630     const CXXConstructorDecl *Ctor = E->getConstructor();
1631
1632     // Trivial zero initialization.
1633     if (E->requiresZeroInitialization() && Ctor->isTrivial()) {
1634       const Record *R = getRecord(E->getType());
1635       return this->visitZeroRecordInitializer(R, E);
1636     }
1637
1638     const Function *Func = getFunction(Ctor);
1639
1640     if (!Func)
1641       return false;
1642
1643     assert(Func->hasThisPointer());
1644     assert(!Func->hasRVO());
1645
1646     // If we're discarding a construct expression, we still need
1647     // to allocate a variable and call the constructor and destructor.
1648     if (DiscardResult) {
1649       assert(!Initializing);
1650       std::optional<unsigned> LocalIndex =
1651           allocateLocal(E, /*IsExtended=*/true);
1652
1653       if (!LocalIndex)
1654         return false;
1655
1656       if (!this->emitGetPtrLocal(*LocalIndex, E))
1657         return false;
1658     }
1659
1660     //  The This pointer is already on the stack because this is an initializer,
1661     //  but we need to dup() so the call() below has its own copy.
1662     if (!this->emitDupPtr(E))
1663       return false;
1664
1665     // Constructor arguments.
1666     for (const auto *Arg : E->arguments()) {
1667       if (!this->visit(Arg))
1668         return false;
1669     }
1670
1671     if (!this->emitCall(Func, E))
1672       return false;
1673
1674     // Immediately call the destructor if we have to.
1675     if (DiscardResult) {
1676       if (!this->emitPopPtr(E))
1677         return false;
1678     }
1679     return true;
1680   }
1681
1682   if (T->isArrayType()) {
1683     const ConstantArrayType *CAT =
1684         Ctx.getASTContext().getAsConstantArrayType(E->getType());
1685     assert(CAT);
1686     size_t NumElems = CAT->getSize().getZExtValue();
1687     const Function *Func = getFunction(E->getConstructor());
1688     if (!Func || !Func->isConstexpr())
1689       return false;
1690
1691     // FIXME(perf): We're calling the constructor once per array element here,
1692     //   in the old intepreter we had a special-case for trivial constructors.
1693     for (size_t I = 0; I != NumElems; ++I) {
1694       if (!this->emitConstUint64(I, E))
1695         return false;
1696       if (!this->emitArrayElemPtrUint64(E))
1697         return false;
1698
1699       // Constructor arguments.
1700       for (const auto *Arg : E->arguments()) {
1701         if (!this->visit(Arg))
1702           return false;
1703       }
1704
1705       if (!this->emitCall(Func, E))
1706         return false;
1707     }
1708     return true;
1709   }
1710
1711   return false;
1712 }
1713
1714 template <class Emitter>
1715 bool ByteCodeExprGen<Emitter>::VisitSourceLocExpr(const SourceLocExpr *E) {
1716   if (DiscardResult)
1717     return true;
1718
1719   const APValue Val =
1720       E->EvaluateInContext(Ctx.getASTContext(), SourceLocDefaultExpr);
1721
1722   // Things like __builtin_LINE().
1723   if (E->getType()->isIntegerType()) {
1724     assert(Val.isInt());
1725     const APSInt &I = Val.getInt();
1726     return this->emitConst(I, E);
1727   }
1728   // Otherwise, the APValue is an LValue, with only one element.
1729   // Theoretically, we don't need the APValue at all of course.
1730   assert(E->getType()->isPointerType());
1731   assert(Val.isLValue());
1732   const APValue::LValueBase &Base = Val.getLValueBase();
1733   if (const Expr *LValueExpr = Base.dyn_cast<const Expr *>())
1734     return this->visit(LValueExpr);
1735
1736   // Otherwise, we have a decl (which is the case for
1737   // __builtin_source_location).
1738   assert(Base.is<const ValueDecl *>());
1739   assert(Val.getLValuePath().size() == 0);
1740   const auto *BaseDecl = Base.dyn_cast<const ValueDecl *>();
1741   assert(BaseDecl);
1742
1743   auto *UGCD = cast<UnnamedGlobalConstantDecl>(BaseDecl);
1744
1745   std::optional<unsigned> GlobalIndex = P.getOrCreateGlobal(UGCD);
1746   if (!GlobalIndex)
1747     return false;
1748
1749   if (!this->emitGetPtrGlobal(*GlobalIndex, E))
1750     return false;
1751
1752   const Record *R = getRecord(E->getType());
1753   const APValue &V = UGCD->getValue();
1754   for (unsigned I = 0, N = R->getNumFields(); I != N; ++I) {
1755     const Record::Field *F = R->getField(I);
1756     const APValue &FieldValue = V.getStructField(I);
1757
1758     PrimType FieldT = classifyPrim(F->Decl->getType());
1759
1760     if (!this->visitAPValue(FieldValue, FieldT, E))
1761       return false;
1762     if (!this->emitInitField(FieldT, F->Offset, E))
1763       return false;
1764   }
1765
1766   // Leave the pointer to the global on the stack.
1767   return true;
1768 }
1769
1770 template <class Emitter>
1771 bool ByteCodeExprGen<Emitter>::VisitOffsetOfExpr(const OffsetOfExpr *E) {
1772   unsigned N = E->getNumComponents();
1773   if (N == 0)
1774     return false;
1775
1776   for (unsigned I = 0; I != N; ++I) {
1777     const OffsetOfNode &Node = E->getComponent(I);
1778     if (Node.getKind() == OffsetOfNode::Array) {
1779       const Expr *ArrayIndexExpr = E->getIndexExpr(Node.getArrayExprIndex());
1780       PrimType IndexT = classifyPrim(ArrayIndexExpr->getType());
1781
1782       if (DiscardResult) {
1783         if (!this->discard(ArrayIndexExpr))
1784           return false;
1785         continue;
1786       }
1787
1788       if (!this->visit(ArrayIndexExpr))
1789         return false;
1790       // Cast to Sint64.
1791       if (IndexT != PT_Sint64) {
1792         if (!this->emitCast(IndexT, PT_Sint64, E))
1793           return false;
1794       }
1795     }
1796   }
1797
1798   if (DiscardResult)
1799     return true;
1800
1801   PrimType T = classifyPrim(E->getType());
1802   return this->emitOffsetOf(T, E, E);
1803 }
1804
1805 template <class Emitter>
1806 bool ByteCodeExprGen<Emitter>::VisitCXXScalarValueInitExpr(
1807     const CXXScalarValueInitExpr *E) {
1808   QualType Ty = E->getType();
1809
1810   if (Ty->isVoidType())
1811     return true;
1812
1813   return this->visitZeroInitializer(classifyPrim(Ty), Ty, E);
1814 }
1815
1816 template <class Emitter>
1817 bool ByteCodeExprGen<Emitter>::VisitSizeOfPackExpr(const SizeOfPackExpr *E) {
1818   return this->emitConst(E->getPackLength(), E);
1819 }
1820
1821 template <class Emitter> bool ByteCodeExprGen<Emitter>::discard(const Expr *E) {
1822   if (E->containsErrors())
1823     return false;
1824
1825   OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/true,
1826                              /*NewInitializing=*/false);
1827   return this->Visit(E);
1828 }
1829
1830 template <class Emitter>
1831 bool ByteCodeExprGen<Emitter>::delegate(const Expr *E) {
1832   if (E->containsErrors())
1833     return false;
1834
1835   // We're basically doing:
1836   // OptionScope<Emitter> Scope(this, DicardResult, Initializing);
1837   // but that's unnecessary of course.
1838   return this->Visit(E);
1839 }
1840
1841 template <class Emitter> bool ByteCodeExprGen<Emitter>::visit(const Expr *E) {
1842   if (E->containsErrors())
1843     return false;
1844
1845   if (E->getType()->isVoidType())
1846     return this->discard(E);
1847
1848   // Create local variable to hold the return value.
1849   if (!E->isGLValue() && !E->getType()->isAnyComplexType() &&
1850       !classify(E->getType())) {
1851     std::optional<unsigned> LocalIndex = allocateLocal(E, /*IsExtended=*/true);
1852     if (!LocalIndex)
1853       return false;
1854
1855     if (!this->emitGetPtrLocal(*LocalIndex, E))
1856       return false;
1857     return this->visitInitializer(E);
1858   }
1859
1860   //  Otherwise,we have a primitive return value, produce the value directly
1861   //  and push it on the stack.
1862   OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/false,
1863                              /*NewInitializing=*/false);
1864   return this->Visit(E);
1865 }
1866
1867 template <class Emitter>
1868 bool ByteCodeExprGen<Emitter>::visitInitializer(const Expr *E) {
1869   assert(!classify(E->getType()));
1870
1871   if (E->containsErrors())
1872     return false;
1873
1874   OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/false,
1875                              /*NewInitializing=*/true);
1876   return this->Visit(E);
1877 }
1878
1879 template <class Emitter>
1880 bool ByteCodeExprGen<Emitter>::visitBool(const Expr *E) {
1881   std::optional<PrimType> T = classify(E->getType());
1882   if (!T)
1883     return false;
1884
1885   if (!this->visit(E))
1886     return false;
1887
1888   if (T == PT_Bool)
1889     return true;
1890
1891   // Convert pointers to bool.
1892   if (T == PT_Ptr || T == PT_FnPtr) {
1893     if (!this->emitNull(*T, E))
1894       return false;
1895     return this->emitNE(*T, E);
1896   }
1897
1898   // Or Floats.
1899   if (T == PT_Float)
1900     return this->emitCastFloatingIntegralBool(E);
1901
1902   // Or anything else we can.
1903   return this->emitCast(*T, PT_Bool, E);
1904 }
1905
1906 template <class Emitter>
1907 bool ByteCodeExprGen<Emitter>::visitZeroInitializer(PrimType T, QualType QT,
1908                                                     const Expr *E) {
1909   switch (T) {
1910   case PT_Bool:
1911     return this->emitZeroBool(E);
1912   case PT_Sint8:
1913     return this->emitZeroSint8(E);
1914   case PT_Uint8:
1915     return this->emitZeroUint8(E);
1916   case PT_Sint16:
1917     return this->emitZeroSint16(E);
1918   case PT_Uint16:
1919     return this->emitZeroUint16(E);
1920   case PT_Sint32:
1921     return this->emitZeroSint32(E);
1922   case PT_Uint32:
1923     return this->emitZeroUint32(E);
1924   case PT_Sint64:
1925     return this->emitZeroSint64(E);
1926   case PT_Uint64:
1927     return this->emitZeroUint64(E);
1928   case PT_IntAP:
1929     return this->emitZeroIntAP(Ctx.getBitWidth(QT), E);
1930   case PT_IntAPS:
1931     return this->emitZeroIntAPS(Ctx.getBitWidth(QT), E);
1932   case PT_Ptr:
1933     return this->emitNullPtr(E);
1934   case PT_FnPtr:
1935     return this->emitNullFnPtr(E);
1936   case PT_Float: {
1937     return this->emitConstFloat(APFloat::getZero(Ctx.getFloatSemantics(QT)), E);
1938   }
1939   }
1940   llvm_unreachable("unknown primitive type");
1941 }
1942
1943 template <class Emitter>
1944 bool ByteCodeExprGen<Emitter>::visitZeroRecordInitializer(const Record *R,
1945                                                           const Expr *E) {
1946   assert(E);
1947   assert(R);
1948   // Fields
1949   for (const Record::Field &Field : R->fields()) {
1950     const Descriptor *D = Field.Desc;
1951     if (D->isPrimitive()) {
1952       QualType QT = D->getType();
1953       PrimType T = classifyPrim(D->getType());
1954       if (!this->visitZeroInitializer(T, QT, E))
1955         return false;
1956       if (!this->emitInitField(T, Field.Offset, E))
1957         return false;
1958       continue;
1959     }
1960
1961     // TODO: Add GetPtrFieldPop and get rid of this dup.
1962     if (!this->emitDupPtr(E))
1963       return false;
1964     if (!this->emitGetPtrField(Field.Offset, E))
1965       return false;
1966
1967     if (D->isPrimitiveArray()) {
1968       QualType ET = D->getElemQualType();
1969       PrimType T = classifyPrim(ET);
1970       for (uint32_t I = 0, N = D->getNumElems(); I != N; ++I) {
1971         if (!this->visitZeroInitializer(T, ET, E))
1972           return false;
1973         if (!this->emitInitElem(T, I, E))
1974           return false;
1975       }
1976     } else if (D->isCompositeArray()) {
1977       const Record *ElemRecord = D->ElemDesc->ElemRecord;
1978       assert(D->ElemDesc->ElemRecord);
1979       for (uint32_t I = 0, N = D->getNumElems(); I != N; ++I) {
1980         if (!this->emitConstUint32(I, E))
1981           return false;
1982         if (!this->emitArrayElemPtr(PT_Uint32, E))
1983           return false;
1984         if (!this->visitZeroRecordInitializer(ElemRecord, E))
1985           return false;
1986         if (!this->emitPopPtr(E))
1987           return false;
1988       }
1989     } else if (D->isRecord()) {
1990       if (!this->visitZeroRecordInitializer(D->ElemRecord, E))
1991         return false;
1992     } else {
1993       assert(false);
1994     }
1995
1996     if (!this->emitPopPtr(E))
1997       return false;
1998   }
1999
2000   for (const Record::Base &B : R->bases()) {
2001     if (!this->emitGetPtrBase(B.Offset, E))
2002       return false;
2003     if (!this->visitZeroRecordInitializer(B.R, E))
2004       return false;
2005     if (!this->emitInitPtrPop(E))
2006       return false;
2007   }
2008
2009   // FIXME: Virtual bases.
2010
2011   return true;
2012 }
2013
2014 template <class Emitter>
2015 bool ByteCodeExprGen<Emitter>::dereference(
2016     const Expr *LV, DerefKind AK, llvm::function_ref<bool(PrimType)> Direct,
2017     llvm::function_ref<bool(PrimType)> Indirect) {
2018   if (std::optional<PrimType> T = classify(LV->getType())) {
2019     if (!LV->refersToBitField()) {
2020       // Only primitive, non bit-field types can be dereferenced directly.
2021       if (const auto *DE = dyn_cast<DeclRefExpr>(LV)) {
2022         if (!DE->getDecl()->getType()->isReferenceType()) {
2023           if (const auto *PD = dyn_cast<ParmVarDecl>(DE->getDecl()))
2024             return dereferenceParam(LV, *T, PD, AK, Direct, Indirect);
2025           if (const auto *VD = dyn_cast<VarDecl>(DE->getDecl()))
2026             return dereferenceVar(LV, *T, VD, AK, Direct, Indirect);
2027         }
2028       }
2029     }
2030
2031     if (!visit(LV))
2032       return false;
2033     return Indirect(*T);
2034   }
2035
2036   if (LV->getType()->isAnyComplexType())
2037     return this->delegate(LV);
2038
2039   return false;
2040 }
2041
2042 template <class Emitter>
2043 bool ByteCodeExprGen<Emitter>::dereferenceParam(
2044     const Expr *LV, PrimType T, const ParmVarDecl *PD, DerefKind AK,
2045     llvm::function_ref<bool(PrimType)> Direct,
2046     llvm::function_ref<bool(PrimType)> Indirect) {
2047   auto It = this->Params.find(PD);
2048   if (It != this->Params.end()) {
2049     unsigned Idx = It->second.Offset;
2050     switch (AK) {
2051     case DerefKind::Read:
2052       return DiscardResult ? true : this->emitGetParam(T, Idx, LV);
2053
2054     case DerefKind::Write:
2055       if (!Direct(T))
2056         return false;
2057       if (!this->emitSetParam(T, Idx, LV))
2058         return false;
2059       return DiscardResult ? true : this->emitGetPtrParam(Idx, LV);
2060
2061     case DerefKind::ReadWrite:
2062       if (!this->emitGetParam(T, Idx, LV))
2063         return false;
2064       if (!Direct(T))
2065         return false;
2066       if (!this->emitSetParam(T, Idx, LV))
2067         return false;
2068       return DiscardResult ? true : this->emitGetPtrParam(Idx, LV);
2069     }
2070     return true;
2071   }
2072
2073   // If the param is a pointer, we can dereference a dummy value.
2074   if (!DiscardResult && T == PT_Ptr && AK == DerefKind::Read) {
2075     if (auto Idx = P.getOrCreateDummy(PD))
2076       return this->emitGetPtrGlobal(*Idx, PD);
2077     return false;
2078   }
2079
2080   // Value cannot be produced - try to emit pointer and do stuff with it.
2081   return visit(LV) && Indirect(T);
2082 }
2083
2084 template <class Emitter>
2085 bool ByteCodeExprGen<Emitter>::dereferenceVar(
2086     const Expr *LV, PrimType T, const VarDecl *VD, DerefKind AK,
2087     llvm::function_ref<bool(PrimType)> Direct,
2088     llvm::function_ref<bool(PrimType)> Indirect) {
2089   auto It = Locals.find(VD);
2090   if (It != Locals.end()) {
2091     const auto &L = It->second;
2092     switch (AK) {
2093     case DerefKind::Read:
2094       if (!this->emitGetLocal(T, L.Offset, LV))
2095         return false;
2096       return DiscardResult ? this->emitPop(T, LV) : true;
2097
2098     case DerefKind::Write:
2099       if (!Direct(T))
2100         return false;
2101       if (!this->emitSetLocal(T, L.Offset, LV))
2102         return false;
2103       return DiscardResult ? true : this->emitGetPtrLocal(L.Offset, LV);
2104
2105     case DerefKind::ReadWrite:
2106       if (!this->emitGetLocal(T, L.Offset, LV))
2107         return false;
2108       if (!Direct(T))
2109         return false;
2110       if (!this->emitSetLocal(T, L.Offset, LV))
2111         return false;
2112       return DiscardResult ? true : this->emitGetPtrLocal(L.Offset, LV);
2113     }
2114   } else if (auto Idx = P.getGlobal(VD)) {
2115     switch (AK) {
2116     case DerefKind::Read:
2117       if (!this->emitGetGlobal(T, *Idx, LV))
2118         return false;
2119       return DiscardResult ? this->emitPop(T, LV) : true;
2120
2121     case DerefKind::Write:
2122       if (!Direct(T))
2123         return false;
2124       if (!this->emitSetGlobal(T, *Idx, LV))
2125         return false;
2126       return DiscardResult ? true : this->emitGetPtrGlobal(*Idx, LV);
2127
2128     case DerefKind::ReadWrite:
2129       if (!this->emitGetGlobal(T, *Idx, LV))
2130         return false;
2131       if (!Direct(T))
2132         return false;
2133       if (!this->emitSetGlobal(T, *Idx, LV))
2134         return false;
2135       return DiscardResult ? true : this->emitGetPtrGlobal(*Idx, LV);
2136     }
2137   }
2138
2139   // If the declaration is a constant value, emit it here even
2140   // though the declaration was not evaluated in the current scope.
2141   // The access mode can only be read in this case.
2142   if (!DiscardResult && AK == DerefKind::Read) {
2143     if (VD->hasLocalStorage() && VD->hasInit() && !VD->isConstexpr()) {
2144       QualType VT = VD->getType();
2145       if (VT.isConstQualified() && VT->isFundamentalType())
2146         return this->visit(VD->getInit());
2147     }
2148   }
2149
2150   // Value cannot be produced - try to emit pointer.
2151   return visit(LV) && Indirect(T);
2152 }
2153
2154 template <class Emitter>
2155 template <typename T>
2156 bool ByteCodeExprGen<Emitter>::emitConst(T Value, PrimType Ty, const Expr *E) {
2157   switch (Ty) {
2158   case PT_Sint8:
2159     return this->emitConstSint8(Value, E);
2160   case PT_Uint8:
2161     return this->emitConstUint8(Value, E);
2162   case PT_Sint16:
2163     return this->emitConstSint16(Value, E);
2164   case PT_Uint16:
2165     return this->emitConstUint16(Value, E);
2166   case PT_Sint32:
2167     return this->emitConstSint32(Value, E);
2168   case PT_Uint32:
2169     return this->emitConstUint32(Value, E);
2170   case PT_Sint64:
2171     return this->emitConstSint64(Value, E);
2172   case PT_Uint64:
2173     return this->emitConstUint64(Value, E);
2174   case PT_IntAP:
2175   case PT_IntAPS:
2176     assert(false);
2177     return false;
2178   case PT_Bool:
2179     return this->emitConstBool(Value, E);
2180   case PT_Ptr:
2181   case PT_FnPtr:
2182   case PT_Float:
2183     llvm_unreachable("Invalid integral type");
2184     break;
2185   }
2186   llvm_unreachable("unknown primitive type");
2187 }
2188
2189 template <class Emitter>
2190 template <typename T>
2191 bool ByteCodeExprGen<Emitter>::emitConst(T Value, const Expr *E) {
2192   return this->emitConst(Value, classifyPrim(E->getType()), E);
2193 }
2194
2195 template <class Emitter>
2196 bool ByteCodeExprGen<Emitter>::emitConst(const APSInt &Value, PrimType Ty,
2197                                          const Expr *E) {
2198   if (Value.isSigned())
2199     return this->emitConst(Value.getSExtValue(), Ty, E);
2200   return this->emitConst(Value.getZExtValue(), Ty, E);
2201 }
2202
2203 template <class Emitter>
2204 bool ByteCodeExprGen<Emitter>::emitConst(const APSInt &Value, const Expr *E) {
2205   return this->emitConst(Value, classifyPrim(E->getType()), E);
2206 }
2207
2208 template <class Emitter>
2209 unsigned ByteCodeExprGen<Emitter>::allocateLocalPrimitive(DeclTy &&Src,
2210                                                           PrimType Ty,
2211                                                           bool IsConst,
2212                                                           bool IsExtended) {
2213   // Make sure we don't accidentally register the same decl twice.
2214   if (const auto *VD =
2215           dyn_cast_if_present<ValueDecl>(Src.dyn_cast<const Decl *>())) {
2216     assert(!P.getGlobal(VD));
2217     assert(!Locals.contains(VD));
2218   }
2219
2220   // FIXME: There are cases where Src.is<Expr*>() is wrong, e.g.
2221   //   (int){12} in C. Consider using Expr::isTemporaryObject() instead
2222   //   or isa<MaterializeTemporaryExpr>().
2223   Descriptor *D = P.createDescriptor(Src, Ty, Descriptor::InlineDescMD, IsConst,
2224                                      Src.is<const Expr *>());
2225   Scope::Local Local = this->createLocal(D);
2226   if (auto *VD = dyn_cast_if_present<ValueDecl>(Src.dyn_cast<const Decl *>()))
2227     Locals.insert({VD, Local});
2228   VarScope->add(Local, IsExtended);
2229   return Local.Offset;
2230 }
2231
2232 template <class Emitter>
2233 std::optional<unsigned>
2234 ByteCodeExprGen<Emitter>::allocateLocal(DeclTy &&Src, bool IsExtended) {
2235   // Make sure we don't accidentally register the same decl twice.
2236   if ([[maybe_unused]]  const auto *VD =
2237           dyn_cast_if_present<ValueDecl>(Src.dyn_cast<const Decl *>())) {
2238     assert(!P.getGlobal(VD));
2239     assert(!Locals.contains(VD));
2240   }
2241
2242   QualType Ty;
2243   const ValueDecl *Key = nullptr;
2244   const Expr *Init = nullptr;
2245   bool IsTemporary = false;
2246   if (auto *VD = dyn_cast_if_present<ValueDecl>(Src.dyn_cast<const Decl *>())) {
2247     Key = VD;
2248     Ty = VD->getType();
2249
2250     if (const auto *VarD = dyn_cast<VarDecl>(VD))
2251       Init = VarD->getInit();
2252   }
2253   if (auto *E = Src.dyn_cast<const Expr *>()) {
2254     IsTemporary = true;
2255     Ty = E->getType();
2256   }
2257
2258   Descriptor *D = P.createDescriptor(
2259       Src, Ty.getTypePtr(), Descriptor::InlineDescMD, Ty.isConstQualified(),
2260       IsTemporary, /*IsMutable=*/false, Init);
2261   if (!D)
2262     return {};
2263
2264   Scope::Local Local = this->createLocal(D);
2265   if (Key)
2266     Locals.insert({Key, Local});
2267   VarScope->add(Local, IsExtended);
2268   return Local.Offset;
2269 }
2270
2271 template <class Emitter>
2272 const RecordType *ByteCodeExprGen<Emitter>::getRecordTy(QualType Ty) {
2273   if (const PointerType *PT = dyn_cast<PointerType>(Ty))
2274     return PT->getPointeeType()->getAs<RecordType>();
2275   return Ty->getAs<RecordType>();
2276 }
2277
2278 template <class Emitter>
2279 Record *ByteCodeExprGen<Emitter>::getRecord(QualType Ty) {
2280   if (const auto *RecordTy = getRecordTy(Ty))
2281     return getRecord(RecordTy->getDecl());
2282   return nullptr;
2283 }
2284
2285 template <class Emitter>
2286 Record *ByteCodeExprGen<Emitter>::getRecord(const RecordDecl *RD) {
2287   return P.getOrCreateRecord(RD);
2288 }
2289
2290 template <class Emitter>
2291 const Function *ByteCodeExprGen<Emitter>::getFunction(const FunctionDecl *FD) {
2292   return Ctx.getOrCreateFunction(FD);
2293 }
2294
2295 template <class Emitter>
2296 bool ByteCodeExprGen<Emitter>::visitExpr(const Expr *E) {
2297   ExprScope<Emitter> RootScope(this);
2298   // Void expressions.
2299   if (E->getType()->isVoidType()) {
2300     if (!visit(E))
2301       return false;
2302     return this->emitRetVoid(E);
2303   }
2304
2305   // Expressions with a primitive return type.
2306   if (std::optional<PrimType> T = classify(E)) {
2307     if (!visit(E))
2308       return false;
2309     return this->emitRet(*T, E);
2310   }
2311
2312   // Expressions with a composite return type.
2313   // For us, that means everything we don't
2314   // have a PrimType for.
2315   if (std::optional<unsigned> LocalOffset = this->allocateLocal(E)) {
2316     if (!this->visitLocalInitializer(E, *LocalOffset))
2317       return false;
2318
2319     if (!this->emitGetPtrLocal(*LocalOffset, E))
2320       return false;
2321     return this->emitRetValue(E);
2322   }
2323
2324   return false;
2325 }
2326
2327 /// Toplevel visitDecl().
2328 /// We get here from evaluateAsInitializer().
2329 /// We need to evaluate the initializer and return its value.
2330 template <class Emitter>
2331 bool ByteCodeExprGen<Emitter>::visitDecl(const VarDecl *VD) {
2332   assert(!VD->isInvalidDecl() && "Trying to constant evaluate an invalid decl");
2333
2334   // Create and initialize the variable.
2335   if (!this->visitVarDecl(VD))
2336     return false;
2337
2338   std::optional<PrimType> VarT = classify(VD->getType());
2339   // Get a pointer to the variable
2340   if (Context::shouldBeGloballyIndexed(VD)) {
2341     auto GlobalIndex = P.getGlobal(VD);
2342     assert(GlobalIndex); // visitVarDecl() didn't return false.
2343     if (VarT) {
2344       if (!this->emitGetGlobalUnchecked(*VarT, *GlobalIndex, VD))
2345         return false;
2346     } else {
2347       if (!this->emitGetPtrGlobal(*GlobalIndex, VD))
2348         return false;
2349     }
2350   } else {
2351     auto Local = Locals.find(VD);
2352     assert(Local != Locals.end()); // Same here.
2353     if (VarT) {
2354       if (!this->emitGetLocal(*VarT, Local->second.Offset, VD))
2355         return false;
2356     } else {
2357       if (!this->emitGetPtrLocal(Local->second.Offset, VD))
2358         return false;
2359     }
2360   }
2361
2362   // Return the value
2363   if (VarT)
2364     return this->emitRet(*VarT, VD);
2365   return this->emitRetValue(VD);
2366 }
2367
2368 template <class Emitter>
2369 bool ByteCodeExprGen<Emitter>::visitVarDecl(const VarDecl *VD) {
2370   // We don't know what to do with these, so just return false.
2371   if (VD->getType().isNull())
2372     return false;
2373
2374   const Expr *Init = VD->getInit();
2375   std::optional<PrimType> VarT = classify(VD->getType());
2376
2377   if (Context::shouldBeGloballyIndexed(VD)) {
2378     // We've already seen and initialized this global.
2379     if (P.getGlobal(VD))
2380       return true;
2381
2382     std::optional<unsigned> GlobalIndex = P.createGlobal(VD, Init);
2383
2384     if (!GlobalIndex)
2385       return this->bail(VD);
2386
2387     assert(Init);
2388     {
2389       DeclScope<Emitter> LocalScope(this, VD);
2390
2391       if (VarT) {
2392         if (!this->visit(Init))
2393           return false;
2394         return this->emitInitGlobal(*VarT, *GlobalIndex, VD);
2395       }
2396       return this->visitGlobalInitializer(Init, *GlobalIndex);
2397     }
2398   } else {
2399     VariableScope<Emitter> LocalScope(this);
2400     if (VarT) {
2401       unsigned Offset = this->allocateLocalPrimitive(
2402           VD, *VarT, VD->getType().isConstQualified());
2403       if (Init) {
2404         // Compile the initializer in its own scope.
2405         ExprScope<Emitter> Scope(this);
2406         if (!this->visit(Init))
2407           return false;
2408
2409         return this->emitSetLocal(*VarT, Offset, VD);
2410       }
2411     } else {
2412       if (std::optional<unsigned> Offset = this->allocateLocal(VD)) {
2413         if (Init)
2414           return this->visitLocalInitializer(Init, *Offset);
2415       }
2416     }
2417     return true;
2418   }
2419
2420   return false;
2421 }
2422
2423 template <class Emitter>
2424 bool ByteCodeExprGen<Emitter>::visitAPValue(const APValue &Val,
2425                                             PrimType ValType, const Expr *E) {
2426   assert(!DiscardResult);
2427   if (Val.isInt())
2428     return this->emitConst(Val.getInt(), ValType, E);
2429
2430   if (Val.isLValue()) {
2431     APValue::LValueBase Base = Val.getLValueBase();
2432     if (const Expr *BaseExpr = Base.dyn_cast<const Expr *>())
2433       return this->visit(BaseExpr);
2434   }
2435
2436   return false;
2437 }
2438
2439 template <class Emitter>
2440 bool ByteCodeExprGen<Emitter>::VisitBuiltinCallExpr(const CallExpr *E) {
2441   const Function *Func = getFunction(E->getDirectCallee());
2442   if (!Func)
2443     return false;
2444
2445   if (!Func->isUnevaluatedBuiltin()) {
2446     // Put arguments on the stack.
2447     for (const auto *Arg : E->arguments()) {
2448       if (!this->visit(Arg))
2449         return false;
2450     }
2451   }
2452
2453   if (!this->emitCallBI(Func, E, E))
2454     return false;
2455
2456   QualType ReturnType = E->getCallReturnType(Ctx.getASTContext());
2457   if (DiscardResult && !ReturnType->isVoidType()) {
2458     PrimType T = classifyPrim(ReturnType);
2459     return this->emitPop(T, E);
2460   }
2461
2462   return true;
2463 }
2464
2465 template <class Emitter>
2466 bool ByteCodeExprGen<Emitter>::VisitCallExpr(const CallExpr *E) {
2467   if (E->getBuiltinCallee())
2468     return VisitBuiltinCallExpr(E);
2469
2470   QualType ReturnType = E->getCallReturnType(Ctx.getASTContext());
2471   std::optional<PrimType> T = classify(ReturnType);
2472   bool HasRVO = !ReturnType->isVoidType() && !T;
2473
2474   if (HasRVO) {
2475     if (DiscardResult) {
2476       // If we need to discard the return value but the function returns its
2477       // value via an RVO pointer, we need to create one such pointer just
2478       // for this call.
2479       if (std::optional<unsigned> LocalIndex = allocateLocal(E)) {
2480         if (!this->emitGetPtrLocal(*LocalIndex, E))
2481           return false;
2482       }
2483     } else {
2484       assert(Initializing);
2485       if (!this->emitDupPtr(E))
2486         return false;
2487     }
2488   }
2489
2490   // Add the (optional, implicit) This pointer.
2491   if (const auto *MC = dyn_cast<CXXMemberCallExpr>(E)) {
2492     if (!this->visit(MC->getImplicitObjectArgument()))
2493       return false;
2494   }
2495
2496   // Put arguments on the stack.
2497   for (const auto *Arg : E->arguments()) {
2498     if (!this->visit(Arg))
2499       return false;
2500   }
2501
2502   if (const FunctionDecl *FuncDecl = E->getDirectCallee()) {
2503     const Function *Func = getFunction(FuncDecl);
2504     if (!Func)
2505       return false;
2506     // If the function is being compiled right now, this is a recursive call.
2507     // In that case, the function can't be valid yet, even though it will be
2508     // later.
2509     // If the function is already fully compiled but not constexpr, it was
2510     // found to be faulty earlier on, so bail out.
2511     if (Func->isFullyCompiled() && !Func->isConstexpr())
2512       return false;
2513
2514     assert(HasRVO == Func->hasRVO());
2515
2516     bool HasQualifier = false;
2517     if (const auto *ME = dyn_cast<MemberExpr>(E->getCallee()))
2518       HasQualifier = ME->hasQualifier();
2519
2520     bool IsVirtual = false;
2521     if (const auto *MD = dyn_cast<CXXMethodDecl>(FuncDecl))
2522       IsVirtual = MD->isVirtual();
2523
2524     // In any case call the function. The return value will end up on the stack
2525     // and if the function has RVO, we already have the pointer on the stack to
2526     // write the result into.
2527     if (IsVirtual && !HasQualifier) {
2528       if (!this->emitCallVirt(Func, E))
2529         return false;
2530     } else {
2531       if (!this->emitCall(Func, E))
2532         return false;
2533     }
2534   } else {
2535     // Indirect call. Visit the callee, which will leave a FunctionPointer on
2536     // the stack. Cleanup of the returned value if necessary will be done after
2537     // the function call completed.
2538     if (!this->visit(E->getCallee()))
2539       return false;
2540
2541     if (!this->emitCallPtr(E))
2542       return false;
2543   }
2544
2545   // Cleanup for discarded return values.
2546   if (DiscardResult && !ReturnType->isVoidType() && T)
2547     return this->emitPop(*T, E);
2548
2549   return true;
2550 }
2551
2552 template <class Emitter>
2553 bool ByteCodeExprGen<Emitter>::VisitCXXDefaultInitExpr(
2554     const CXXDefaultInitExpr *E) {
2555   SourceLocScope<Emitter> SLS(this, E);
2556   if (Initializing)
2557     return this->visitInitializer(E->getExpr());
2558
2559   assert(classify(E->getType()));
2560   return this->visit(E->getExpr());
2561 }
2562
2563 template <class Emitter>
2564 bool ByteCodeExprGen<Emitter>::VisitCXXDefaultArgExpr(
2565     const CXXDefaultArgExpr *E) {
2566   SourceLocScope<Emitter> SLS(this, E);
2567
2568   const Expr *SubExpr = E->getExpr();
2569   if (std::optional<PrimType> T = classify(E->getExpr()))
2570     return this->visit(SubExpr);
2571
2572   assert(Initializing);
2573   return this->visitInitializer(SubExpr);
2574 }
2575
2576 template <class Emitter>
2577 bool ByteCodeExprGen<Emitter>::VisitCXXBoolLiteralExpr(
2578     const CXXBoolLiteralExpr *E) {
2579   if (DiscardResult)
2580     return true;
2581
2582   return this->emitConstBool(E->getValue(), E);
2583 }
2584
2585 template <class Emitter>
2586 bool ByteCodeExprGen<Emitter>::VisitCXXNullPtrLiteralExpr(
2587     const CXXNullPtrLiteralExpr *E) {
2588   if (DiscardResult)
2589     return true;
2590
2591   return this->emitNullPtr(E);
2592 }
2593
2594 template <class Emitter>
2595 bool ByteCodeExprGen<Emitter>::VisitGNUNullExpr(const GNUNullExpr *E) {
2596   if (DiscardResult)
2597     return true;
2598
2599   assert(E->getType()->isIntegerType());
2600
2601   PrimType T = classifyPrim(E->getType());
2602   return this->emitZero(T, E);
2603 }
2604
2605 template <class Emitter>
2606 bool ByteCodeExprGen<Emitter>::VisitCXXThisExpr(const CXXThisExpr *E) {
2607   if (DiscardResult)
2608     return true;
2609
2610   if (this->LambdaThisCapture > 0)
2611     return this->emitGetThisFieldPtr(this->LambdaThisCapture, E);
2612
2613   return this->emitThis(E);
2614 }
2615
2616 template <class Emitter>
2617 bool ByteCodeExprGen<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
2618   const Expr *SubExpr = E->getSubExpr();
2619   std::optional<PrimType> T = classify(SubExpr->getType());
2620
2621   switch (E->getOpcode()) {
2622   case UO_PostInc: { // x++
2623     if (!this->visit(SubExpr))
2624       return false;
2625
2626     if (T == PT_Ptr) {
2627       if (!this->emitIncPtr(E))
2628         return false;
2629
2630       return DiscardResult ? this->emitPopPtr(E) : true;
2631     }
2632
2633     if (T == PT_Float) {
2634       return DiscardResult ? this->emitIncfPop(getRoundingMode(E), E)
2635                            : this->emitIncf(getRoundingMode(E), E);
2636     }
2637
2638     return DiscardResult ? this->emitIncPop(*T, E) : this->emitInc(*T, E);
2639   }
2640   case UO_PostDec: { // x--
2641     if (!this->visit(SubExpr))
2642       return false;
2643
2644     if (T == PT_Ptr) {
2645       if (!this->emitDecPtr(E))
2646         return false;
2647
2648       return DiscardResult ? this->emitPopPtr(E) : true;
2649     }
2650
2651     if (T == PT_Float) {
2652       return DiscardResult ? this->emitDecfPop(getRoundingMode(E), E)
2653                            : this->emitDecf(getRoundingMode(E), E);
2654     }
2655
2656     return DiscardResult ? this->emitDecPop(*T, E) : this->emitDec(*T, E);
2657   }
2658   case UO_PreInc: { // ++x
2659     if (!this->visit(SubExpr))
2660       return false;
2661
2662     if (T == PT_Ptr) {
2663       if (!this->emitLoadPtr(E))
2664         return false;
2665       if (!this->emitConstUint8(1, E))
2666         return false;
2667       if (!this->emitAddOffsetUint8(E))
2668         return false;
2669       return DiscardResult ? this->emitStorePopPtr(E) : this->emitStorePtr(E);
2670     }
2671
2672     // Post-inc and pre-inc are the same if the value is to be discarded.
2673     if (DiscardResult) {
2674       if (T == PT_Float)
2675         return this->emitIncfPop(getRoundingMode(E), E);
2676       return this->emitIncPop(*T, E);
2677     }
2678
2679     if (T == PT_Float) {
2680       const auto &TargetSemantics = Ctx.getFloatSemantics(E->getType());
2681       if (!this->emitLoadFloat(E))
2682         return false;
2683       if (!this->emitConstFloat(llvm::APFloat(TargetSemantics, 1), E))
2684         return false;
2685       if (!this->emitAddf(getRoundingMode(E), E))
2686         return false;
2687       return this->emitStoreFloat(E);
2688     }
2689     if (!this->emitLoad(*T, E))
2690       return false;
2691     if (!this->emitConst(1, E))
2692       return false;
2693     if (!this->emitAdd(*T, E))
2694       return false;
2695     return this->emitStore(*T, E);
2696   }
2697   case UO_PreDec: { // --x
2698     if (!this->visit(SubExpr))
2699       return false;
2700
2701     if (T == PT_Ptr) {
2702       if (!this->emitLoadPtr(E))
2703         return false;
2704       if (!this->emitConstUint8(1, E))
2705         return false;
2706       if (!this->emitSubOffsetUint8(E))
2707         return false;
2708       return DiscardResult ? this->emitStorePopPtr(E) : this->emitStorePtr(E);
2709     }
2710
2711     // Post-dec and pre-dec are the same if the value is to be discarded.
2712     if (DiscardResult) {
2713       if (T == PT_Float)
2714         return this->emitDecfPop(getRoundingMode(E), E);
2715       return this->emitDecPop(*T, E);
2716     }
2717
2718     if (T == PT_Float) {
2719       const auto &TargetSemantics = Ctx.getFloatSemantics(E->getType());
2720       if (!this->emitLoadFloat(E))
2721         return false;
2722       if (!this->emitConstFloat(llvm::APFloat(TargetSemantics, 1), E))
2723         return false;
2724       if (!this->emitSubf(getRoundingMode(E), E))
2725         return false;
2726       return this->emitStoreFloat(E);
2727     }
2728     if (!this->emitLoad(*T, E))
2729       return false;
2730     if (!this->emitConst(1, E))
2731       return false;
2732     if (!this->emitSub(*T, E))
2733       return false;
2734     return this->emitStore(*T, E);
2735   }
2736   case UO_LNot: // !x
2737     if (DiscardResult)
2738       return this->discard(SubExpr);
2739
2740     if (!this->visitBool(SubExpr))
2741       return false;
2742
2743     if (!this->emitInvBool(E))
2744       return false;
2745
2746     if (PrimType ET = classifyPrim(E->getType()); ET != PT_Bool)
2747       return this->emitCast(PT_Bool, ET, E);
2748     return true;
2749   case UO_Minus: // -x
2750     if (!this->visit(SubExpr))
2751       return false;
2752     return DiscardResult ? this->emitPop(*T, E) : this->emitNeg(*T, E);
2753   case UO_Plus:  // +x
2754     if (!this->visit(SubExpr)) // noop
2755       return false;
2756     return DiscardResult ? this->emitPop(*T, E) : true;
2757   case UO_AddrOf: // &x
2758     // We should already have a pointer when we get here.
2759     return this->delegate(SubExpr);
2760   case UO_Deref:  // *x
2761     return dereference(
2762         SubExpr, DerefKind::Read,
2763         [](PrimType) {
2764           llvm_unreachable("Dereferencing requires a pointer");
2765           return false;
2766         },
2767         [this, E](PrimType T) {
2768           return DiscardResult ? this->emitPop(T, E) : true;
2769         });
2770   case UO_Not:    // ~x
2771     if (!this->visit(SubExpr))
2772       return false;
2773     return DiscardResult ? this->emitPop(*T, E) : this->emitComp(*T, E);
2774   case UO_Real: // __real x
2775     if (T)
2776       return this->delegate(SubExpr);
2777     return this->emitComplexReal(SubExpr);
2778   case UO_Imag: { // __imag x
2779     if (T) {
2780       if (!this->discard(SubExpr))
2781         return false;
2782       return this->visitZeroInitializer(*T, SubExpr->getType(), SubExpr);
2783     }
2784     if (!this->visit(SubExpr))
2785       return false;
2786     if (!this->emitConstUint8(1, E))
2787       return false;
2788     if (!this->emitArrayElemPtrPopUint8(E))
2789       return false;
2790
2791     // Since our _Complex implementation does not map to a primitive type,
2792     // we sometimes have to do the lvalue-to-rvalue conversion here manually.
2793     if (!SubExpr->isLValue())
2794       return this->emitLoadPop(classifyPrim(E->getType()), E);
2795     return true;
2796   }
2797   case UO_Extension:
2798     return this->delegate(SubExpr);
2799   case UO_Coawait:
2800     assert(false && "Unhandled opcode");
2801   }
2802
2803   return false;
2804 }
2805
2806 template <class Emitter>
2807 bool ByteCodeExprGen<Emitter>::VisitDeclRefExpr(const DeclRefExpr *E) {
2808   if (DiscardResult)
2809     return true;
2810
2811   const auto *D = E->getDecl();
2812
2813   if (const auto *ECD = dyn_cast<EnumConstantDecl>(D)) {
2814     return this->emitConst(ECD->getInitVal(), E);
2815   } else if (const auto *BD = dyn_cast<BindingDecl>(D)) {
2816     return this->visit(BD->getBinding());
2817   } else if (const auto *FuncDecl = dyn_cast<FunctionDecl>(D)) {
2818     const Function *F = getFunction(FuncDecl);
2819     return F && this->emitGetFnPtr(F, E);
2820   }
2821
2822   // References are implemented via pointers, so when we see a DeclRefExpr
2823   // pointing to a reference, we need to get its value directly (i.e. the
2824   // pointer to the actual value) instead of a pointer to the pointer to the
2825   // value.
2826   bool IsReference = D->getType()->isReferenceType();
2827
2828   // Check for local/global variables and parameters.
2829   if (auto It = Locals.find(D); It != Locals.end()) {
2830     const unsigned Offset = It->second.Offset;
2831
2832     if (IsReference)
2833       return this->emitGetLocal(PT_Ptr, Offset, E);
2834     return this->emitGetPtrLocal(Offset, E);
2835   } else if (auto GlobalIndex = P.getGlobal(D)) {
2836     if (IsReference)
2837       return this->emitGetGlobalPtr(*GlobalIndex, E);
2838
2839     return this->emitGetPtrGlobal(*GlobalIndex, E);
2840   } else if (const auto *PVD = dyn_cast<ParmVarDecl>(D)) {
2841     if (auto It = this->Params.find(PVD); It != this->Params.end()) {
2842       if (IsReference || !It->second.IsPtr)
2843         return this->emitGetParamPtr(It->second.Offset, E);
2844
2845       return this->emitGetPtrParam(It->second.Offset, E);
2846     }
2847   }
2848
2849   // Handle lambda captures.
2850   if (auto It = this->LambdaCaptures.find(D);
2851       It != this->LambdaCaptures.end()) {
2852     auto [Offset, IsPtr] = It->second;
2853
2854     if (IsPtr)
2855       return this->emitGetThisFieldPtr(Offset, E);
2856     return this->emitGetPtrThisField(Offset, E);
2857   }
2858
2859   // Lazily visit global declarations we haven't seen yet.
2860   // This happens in C.
2861   if (!Ctx.getLangOpts().CPlusPlus) {
2862     if (const auto *VD = dyn_cast<VarDecl>(D);
2863         VD && VD->hasGlobalStorage() && VD->getAnyInitializer() &&
2864         VD->getType().isConstQualified()) {
2865       if (!this->visitVarDecl(VD))
2866         return false;
2867       // Retry.
2868       return this->VisitDeclRefExpr(E);
2869     }
2870
2871     if (std::optional<unsigned> I = P.getOrCreateDummy(D))
2872       return this->emitGetPtrGlobal(*I, E);
2873   }
2874
2875   return this->emitInvalidDeclRef(E, E);
2876 }
2877
2878 template <class Emitter>
2879 void ByteCodeExprGen<Emitter>::emitCleanup() {
2880   for (VariableScope<Emitter> *C = VarScope; C; C = C->getParent())
2881     C->emitDestruction();
2882 }
2883
2884 template <class Emitter>
2885 unsigned
2886 ByteCodeExprGen<Emitter>::collectBaseOffset(const RecordType *BaseType,
2887                                             const RecordType *DerivedType) {
2888   const auto *FinalDecl = cast<CXXRecordDecl>(BaseType->getDecl());
2889   const RecordDecl *CurDecl = DerivedType->getDecl();
2890   const Record *CurRecord = getRecord(CurDecl);
2891   assert(CurDecl && FinalDecl);
2892
2893   unsigned OffsetSum = 0;
2894   for (;;) {
2895     assert(CurRecord->getNumBases() > 0);
2896     // One level up
2897     for (const Record::Base &B : CurRecord->bases()) {
2898       const auto *BaseDecl = cast<CXXRecordDecl>(B.Decl);
2899
2900       if (BaseDecl == FinalDecl || BaseDecl->isDerivedFrom(FinalDecl)) {
2901         OffsetSum += B.Offset;
2902         CurRecord = B.R;
2903         CurDecl = BaseDecl;
2904         break;
2905       }
2906     }
2907     if (CurDecl == FinalDecl)
2908       break;
2909   }
2910
2911   assert(OffsetSum > 0);
2912   return OffsetSum;
2913 }
2914
2915 /// Emit casts from a PrimType to another PrimType.
2916 template <class Emitter>
2917 bool ByteCodeExprGen<Emitter>::emitPrimCast(PrimType FromT, PrimType ToT,
2918                                             QualType ToQT, const Expr *E) {
2919
2920   if (FromT == PT_Float) {
2921     // Floating to floating.
2922     if (ToT == PT_Float) {
2923       const llvm::fltSemantics *ToSem = &Ctx.getFloatSemantics(ToQT);
2924       return this->emitCastFP(ToSem, getRoundingMode(E), E);
2925     }
2926
2927     // Float to integral.
2928     if (isIntegralType(ToT) || ToT == PT_Bool)
2929       return this->emitCastFloatingIntegral(ToT, E);
2930   }
2931
2932   if (isIntegralType(FromT) || FromT == PT_Bool) {
2933     // Integral to integral.
2934     if (isIntegralType(ToT) || ToT == PT_Bool)
2935       return FromT != ToT ? this->emitCast(FromT, ToT, E) : true;
2936
2937     if (ToT == PT_Float) {
2938       // Integral to floating.
2939       const llvm::fltSemantics *ToSem = &Ctx.getFloatSemantics(ToQT);
2940       return this->emitCastIntegralFloating(FromT, ToSem, getRoundingMode(E),
2941                                             E);
2942     }
2943   }
2944
2945   return false;
2946 }
2947
2948 /// Emits __real(SubExpr)
2949 template <class Emitter>
2950 bool ByteCodeExprGen<Emitter>::emitComplexReal(const Expr *SubExpr) {
2951   assert(SubExpr->getType()->isAnyComplexType());
2952
2953   if (DiscardResult)
2954     return this->discard(SubExpr);
2955
2956   if (!this->visit(SubExpr))
2957     return false;
2958   if (!this->emitConstUint8(0, SubExpr))
2959     return false;
2960   if (!this->emitArrayElemPtrPopUint8(SubExpr))
2961     return false;
2962
2963   // Since our _Complex implementation does not map to a primitive type,
2964   // we sometimes have to do the lvalue-to-rvalue conversion here manually.
2965   if (!SubExpr->isLValue())
2966     return this->emitLoadPop(*classifyComplexElementType(SubExpr->getType()),
2967                              SubExpr);
2968   return true;
2969 }
2970
2971 /// When calling this, we have a pointer of the local-to-destroy
2972 /// on the stack.
2973 /// Emit destruction of record types (or arrays of record types).
2974 template <class Emitter>
2975 bool ByteCodeExprGen<Emitter>::emitRecordDestruction(const Descriptor *Desc) {
2976   assert(Desc);
2977   assert(!Desc->isPrimitive());
2978   assert(!Desc->isPrimitiveArray());
2979
2980   // Arrays.
2981   if (Desc->isArray()) {
2982     const Descriptor *ElemDesc = Desc->ElemDesc;
2983     assert(ElemDesc);
2984
2985     // Don't need to do anything for these.
2986     if (ElemDesc->isPrimitiveArray())
2987       return this->emitPopPtr(SourceInfo{});
2988
2989     // If this is an array of record types, check if we need
2990     // to call the element destructors at all. If not, try
2991     // to save the work.
2992     if (const Record *ElemRecord = ElemDesc->ElemRecord) {
2993       if (const CXXDestructorDecl *Dtor = ElemRecord->getDestructor();
2994           !Dtor || Dtor->isTrivial())
2995         return this->emitPopPtr(SourceInfo{});
2996     }
2997
2998     for (ssize_t I = Desc->getNumElems() - 1; I >= 0; --I) {
2999       if (!this->emitConstUint64(I, SourceInfo{}))
3000         return false;
3001       if (!this->emitArrayElemPtrUint64(SourceInfo{}))
3002         return false;
3003       if (!this->emitRecordDestruction(ElemDesc))
3004         return false;
3005     }
3006     return this->emitPopPtr(SourceInfo{});
3007   }
3008
3009   const Record *R = Desc->ElemRecord;
3010   assert(R);
3011   // First, destroy all fields.
3012   for (const Record::Field &Field : llvm::reverse(R->fields())) {
3013     const Descriptor *D = Field.Desc;
3014     if (!D->isPrimitive() && !D->isPrimitiveArray()) {
3015       if (!this->emitDupPtr(SourceInfo{}))
3016         return false;
3017       if (!this->emitGetPtrField(Field.Offset, SourceInfo{}))
3018         return false;
3019       if (!this->emitRecordDestruction(D))
3020         return false;
3021     }
3022   }
3023
3024   // FIXME: Unions need to be handled differently here. We don't want to
3025   //   call the destructor of its members.
3026
3027   // Now emit the destructor and recurse into base classes.
3028   if (const CXXDestructorDecl *Dtor = R->getDestructor();
3029       Dtor && !Dtor->isTrivial()) {
3030     if (const Function *DtorFunc = getFunction(Dtor)) {
3031       assert(DtorFunc->hasThisPointer());
3032       assert(DtorFunc->getNumParams() == 1);
3033       if (!this->emitDupPtr(SourceInfo{}))
3034         return false;
3035       if (!this->emitCall(DtorFunc, SourceInfo{}))
3036         return false;
3037     }
3038   }
3039
3040   for (const Record::Base &Base : llvm::reverse(R->bases())) {
3041     if (!this->emitGetPtrBase(Base.Offset, SourceInfo{}))
3042       return false;
3043     if (!this->emitRecordDestruction(Base.Desc))
3044       return false;
3045   }
3046   // FIXME: Virtual bases.
3047
3048   // Remove the instance pointer.
3049   return this->emitPopPtr(SourceInfo{});
3050 }
3051
3052 namespace clang {
3053 namespace interp {
3054
3055 template class ByteCodeExprGen<ByteCodeEmitter>;
3056 template class ByteCodeExprGen<EvalEmitter>;
3057
3058 } // namespace interp
3059 } // namespace clang