llvm/lib/Transforms/Coroutines/Coroutines.cpp

   1 //===- Coroutines.cpp -----------------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the common infrastructure for Coroutine Passes.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "CoroInternal.h"
  14 #include "llvm/ADT/SmallVector.h"
  15 #include "llvm/ADT/StringRef.h"
  16 #include "llvm/Analysis/CallGraph.h"
  17 #include "llvm/IR/Attributes.h"
  18 #include "llvm/IR/Constants.h"
  19 #include "llvm/IR/DerivedTypes.h"
  20 #include "llvm/IR/Function.h"
  21 #include "llvm/IR/InstIterator.h"
  22 #include "llvm/IR/Instructions.h"
  23 #include "llvm/IR/IntrinsicInst.h"
  24 #include "llvm/IR/Intrinsics.h"
  25 #include "llvm/IR/Module.h"
  26 #include "llvm/IR/Type.h"
  27 #include "llvm/Support/Casting.h"
  28 #include "llvm/Support/ErrorHandling.h"
  29 #include "llvm/Transforms/Coroutines/ABI.h"
  30 #include "llvm/Transforms/Coroutines/CoroInstr.h"
  31 #include "llvm/Transforms/Coroutines/CoroShape.h"
  32 #include "llvm/Transforms/Utils/Local.h"
  33 #include <cassert>
  34 #include <cstddef>
  35 #include <utility>
  36
  37 using namespace llvm;
  38
  39 // Construct the lowerer base class and initialize its members.
  40 coro::LowererBase::LowererBase(Module &M)
  41     : TheModule(M), Context(M.getContext()),
  42       Int8Ptr(PointerType::get(Context, 0)),
  43       ResumeFnType(FunctionType::get(Type::getVoidTy(Context), Int8Ptr,
  44                                      /*isVarArg=*/false)),
  45       NullPtr(ConstantPointerNull::get(Int8Ptr)) {}
  46
  47 // Creates a call to llvm.coro.subfn.addr to obtain a resume function address.
  48 // It generates the following:
  49 //
  50 //    call ptr @llvm.coro.subfn.addr(ptr %Arg, i8 %index)
  51
  52 CallInst *coro::LowererBase::makeSubFnCall(Value *Arg, int Index,
  53                                            Instruction *InsertPt) {
  54   auto *IndexVal = ConstantInt::get(Type::getInt8Ty(Context), Index);
  55   auto *Fn =
  56       Intrinsic::getOrInsertDeclaration(&TheModule, Intrinsic::coro_subfn_addr);
  57
  58   assert(Index >= CoroSubFnInst::IndexFirst &&
  59          Index < CoroSubFnInst::IndexLast &&
  60          "makeSubFnCall: Index value out of range");
  61   return CallInst::Create(Fn, {Arg, IndexVal}, "", InsertPt->getIterator());
  62 }
  63
  64 // NOTE: Must be sorted!
  65 static const char *const CoroIntrinsics[] = {
  66     "llvm.coro.align",
  67     "llvm.coro.alloc",
  68     "llvm.coro.async.context.alloc",
  69     "llvm.coro.async.context.dealloc",
  70     "llvm.coro.async.resume",
  71     "llvm.coro.async.size.replace",
  72     "llvm.coro.async.store_resume",
  73     "llvm.coro.await.suspend.bool",
  74     "llvm.coro.await.suspend.handle",
  75     "llvm.coro.await.suspend.void",
  76     "llvm.coro.begin",
  77     "llvm.coro.begin.custom.abi",
  78     "llvm.coro.destroy",
  79     "llvm.coro.done",
  80     "llvm.coro.end",
  81     "llvm.coro.end.async",
  82     "llvm.coro.frame",
  83     "llvm.coro.free",
  84     "llvm.coro.id",
  85     "llvm.coro.id.async",
  86     "llvm.coro.id.retcon",
  87     "llvm.coro.id.retcon.once",
  88     "llvm.coro.noop",
  89     "llvm.coro.prepare.async",
  90     "llvm.coro.prepare.retcon",
  91     "llvm.coro.promise",
  92     "llvm.coro.resume",
  93     "llvm.coro.save",
  94     "llvm.coro.size",
  95     "llvm.coro.subfn.addr",
  96     "llvm.coro.suspend",
  97     "llvm.coro.suspend.async",
  98     "llvm.coro.suspend.retcon",
  99 };
 100
 101 #ifndef NDEBUG
 102 static bool isCoroutineIntrinsicName(StringRef Name) {
 103   return llvm::binary_search(CoroIntrinsics, Name);
 104 }
 105 #endif
 106
 107 bool coro::isSuspendBlock(BasicBlock *BB) {
 108   return isa<AnyCoroSuspendInst>(BB->front());
 109 }
 110
 111 bool coro::declaresAnyIntrinsic(const Module &M) {
 112   for (StringRef Name : CoroIntrinsics) {
 113     if (M.getNamedValue(Name))
 114       return true;
 115   }
 116
 117   return false;
 118 }
 119
 120 // Verifies if a module has named values listed. Also, in debug mode verifies
 121 // that names are intrinsic names.
 122 bool coro::declaresIntrinsics(const Module &M,
 123                               const std::initializer_list<StringRef> List) {
 124   for (StringRef Name : List) {
 125     assert(isCoroutineIntrinsicName(Name) && "not a coroutine intrinsic");
 126     if (M.getNamedValue(Name))
 127       return true;
 128   }
 129
 130   return false;
 131 }
 132
 133 // Replace all coro.frees associated with the provided CoroId either with 'null'
 134 // if Elide is true and with its frame parameter otherwise.
 135 void coro::replaceCoroFree(CoroIdInst *CoroId, bool Elide) {
 136   SmallVector<CoroFreeInst *, 4> CoroFrees;
 137   for (User *U : CoroId->users())
 138     if (auto CF = dyn_cast<CoroFreeInst>(U))
 139       CoroFrees.push_back(CF);
 140
 141   if (CoroFrees.empty())
 142     return;
 143
 144   Value *Replacement =
 145       Elide
 146           ? ConstantPointerNull::get(PointerType::get(CoroId->getContext(), 0))
 147           : CoroFrees.front()->getFrame();
 148
 149   for (CoroFreeInst *CF : CoroFrees) {
 150     CF->replaceAllUsesWith(Replacement);
 151     CF->eraseFromParent();
 152   }
 153 }
 154
 155 void coro::suppressCoroAllocs(CoroIdInst *CoroId) {
 156   SmallVector<CoroAllocInst *, 4> CoroAllocs;
 157   for (User *U : CoroId->users())
 158     if (auto *CA = dyn_cast<CoroAllocInst>(U))
 159       CoroAllocs.push_back(CA);
 160
 161   if (CoroAllocs.empty())
 162     return;
 163
 164   coro::suppressCoroAllocs(CoroId->getContext(), CoroAllocs);
 165 }
 166
 167 // Replacing llvm.coro.alloc with false will suppress dynamic
 168 // allocation as it is expected for the frontend to generate the code that
 169 // looks like:
 170 //   id = coro.id(...)
 171 //   mem = coro.alloc(id) ? malloc(coro.size()) : 0;
 172 //   coro.begin(id, mem)
 173 void coro::suppressCoroAllocs(LLVMContext &Context,
 174                               ArrayRef<CoroAllocInst *> CoroAllocs) {
 175   auto *False = ConstantInt::getFalse(Context);
 176   for (auto *CA : CoroAllocs) {
 177     CA->replaceAllUsesWith(False);
 178     CA->eraseFromParent();
 179   }
 180 }
 181
 182 static CoroSaveInst *createCoroSave(CoroBeginInst *CoroBegin,
 183                                     CoroSuspendInst *SuspendInst) {
 184   Module *M = SuspendInst->getModule();
 185   auto *Fn = Intrinsic::getOrInsertDeclaration(M, Intrinsic::coro_save);
 186   auto *SaveInst = cast<CoroSaveInst>(
 187       CallInst::Create(Fn, CoroBegin, "", SuspendInst->getIterator()));
 188   assert(!SuspendInst->getCoroSave());
 189   SuspendInst->setArgOperand(0, SaveInst);
 190   return SaveInst;
 191 }
 192
 193 // Collect "interesting" coroutine intrinsics.
 194 void coro::Shape::analyze(Function &F,
 195                           SmallVectorImpl<CoroFrameInst *> &CoroFrames,
 196                           SmallVectorImpl<CoroSaveInst *> &UnusedCoroSaves) {
 197   clear();
 198
 199   bool HasFinalSuspend = false;
 200   bool HasUnwindCoroEnd = false;
 201   size_t FinalSuspendIndex = 0;
 202
 203   for (Instruction &I : instructions(F)) {
 204     // FIXME: coro_await_suspend_* are not proper `IntrinisicInst`s
 205     // because they might be invoked
 206     if (auto AWS = dyn_cast<CoroAwaitSuspendInst>(&I)) {
 207       CoroAwaitSuspends.push_back(AWS);
 208     } else if (auto II = dyn_cast<IntrinsicInst>(&I)) {
 209       switch (II->getIntrinsicID()) {
 210       default:
 211         continue;
 212       case Intrinsic::coro_size:
 213         CoroSizes.push_back(cast<CoroSizeInst>(II));
 214         break;
 215       case Intrinsic::coro_align:
 216         CoroAligns.push_back(cast<CoroAlignInst>(II));
 217         break;
 218       case Intrinsic::coro_frame:
 219         CoroFrames.push_back(cast<CoroFrameInst>(II));
 220         break;
 221       case Intrinsic::coro_save:
 222         // After optimizations, coro_suspends using this coro_save might have
 223         // been removed, remember orphaned coro_saves to remove them later.
 224         if (II->use_empty())
 225           UnusedCoroSaves.push_back(cast<CoroSaveInst>(II));
 226         break;
 227       case Intrinsic::coro_suspend_async: {
 228         auto *Suspend = cast<CoroSuspendAsyncInst>(II);
 229         Suspend->checkWellFormed();
 230         CoroSuspends.push_back(Suspend);
 231         break;
 232       }
 233       case Intrinsic::coro_suspend_retcon: {
 234         auto Suspend = cast<CoroSuspendRetconInst>(II);
 235         CoroSuspends.push_back(Suspend);
 236         break;
 237       }
 238       case Intrinsic::coro_suspend: {
 239         auto Suspend = cast<CoroSuspendInst>(II);
 240         CoroSuspends.push_back(Suspend);
 241         if (Suspend->isFinal()) {
 242           if (HasFinalSuspend)
 243             report_fatal_error(
 244               "Only one suspend point can be marked as final");
 245           HasFinalSuspend = true;
 246           FinalSuspendIndex = CoroSuspends.size() - 1;
 247         }
 248         break;
 249       }
 250       case Intrinsic::coro_begin:
 251       case Intrinsic::coro_begin_custom_abi: {
 252         auto CB = cast<CoroBeginInst>(II);
 253
 254         // Ignore coro id's that aren't pre-split.
 255         auto Id = dyn_cast<CoroIdInst>(CB->getId());
 256         if (Id && !Id->getInfo().isPreSplit())
 257           break;
 258
 259         if (CoroBegin)
 260           report_fatal_error(
 261                 "coroutine should have exactly one defining @llvm.coro.begin");
 262         CB->addRetAttr(Attribute::NonNull);
 263         CB->addRetAttr(Attribute::NoAlias);
 264         CB->removeFnAttr(Attribute::NoDuplicate);
 265         CoroBegin = CB;
 266         break;
 267       }
 268       case Intrinsic::coro_end_async:
 269       case Intrinsic::coro_end:
 270         CoroEnds.push_back(cast<AnyCoroEndInst>(II));
 271         if (auto *AsyncEnd = dyn_cast<CoroAsyncEndInst>(II)) {
 272           AsyncEnd->checkWellFormed();
 273         }
 274
 275         if (CoroEnds.back()->isUnwind())
 276           HasUnwindCoroEnd = true;
 277
 278         if (CoroEnds.back()->isFallthrough() && isa<CoroEndInst>(II)) {
 279           // Make sure that the fallthrough coro.end is the first element in the
 280           // CoroEnds vector.
 281           // Note: I don't think this is neccessary anymore.
 282           if (CoroEnds.size() > 1) {
 283             if (CoroEnds.front()->isFallthrough())
 284               report_fatal_error(
 285                   "Only one coro.end can be marked as fallthrough");
 286             std::swap(CoroEnds.front(), CoroEnds.back());
 287           }
 288         }
 289         break;
 290       }
 291     }
 292   }
 293
 294   // If there is no CoroBegin then this is not a coroutine.
 295   if (!CoroBegin)
 296     return;
 297
 298   // Determination of ABI and initializing lowering info
 299   auto Id = CoroBegin->getId();
 300   switch (auto IntrID = Id->getIntrinsicID()) {
 301   case Intrinsic::coro_id: {
 302     ABI = coro::ABI::Switch;
 303     SwitchLowering.HasFinalSuspend = HasFinalSuspend;
 304     SwitchLowering.HasUnwindCoroEnd = HasUnwindCoroEnd;
 305
 306     auto SwitchId = getSwitchCoroId();
 307     SwitchLowering.ResumeSwitch = nullptr;
 308     SwitchLowering.PromiseAlloca = SwitchId->getPromise();
 309     SwitchLowering.ResumeEntryBlock = nullptr;
 310
 311     // Move final suspend to the last element in the CoroSuspends vector.
 312     if (SwitchLowering.HasFinalSuspend &&
 313         FinalSuspendIndex != CoroSuspends.size() - 1)
 314       std::swap(CoroSuspends[FinalSuspendIndex], CoroSuspends.back());
 315     break;
 316   }
 317   case Intrinsic::coro_id_async: {
 318     ABI = coro::ABI::Async;
 319     auto *AsyncId = getAsyncCoroId();
 320     AsyncId->checkWellFormed();
 321     AsyncLowering.Context = AsyncId->getStorage();
 322     AsyncLowering.ContextArgNo = AsyncId->getStorageArgumentIndex();
 323     AsyncLowering.ContextHeaderSize = AsyncId->getStorageSize();
 324     AsyncLowering.ContextAlignment = AsyncId->getStorageAlignment().value();
 325     AsyncLowering.AsyncFuncPointer = AsyncId->getAsyncFunctionPointer();
 326     AsyncLowering.AsyncCC = F.getCallingConv();
 327     break;
 328   }
 329   case Intrinsic::coro_id_retcon:
 330   case Intrinsic::coro_id_retcon_once: {
 331     ABI = IntrID == Intrinsic::coro_id_retcon ? coro::ABI::Retcon
 332                                               : coro::ABI::RetconOnce;
 333     auto ContinuationId = getRetconCoroId();
 334     ContinuationId->checkWellFormed();
 335     auto Prototype = ContinuationId->getPrototype();
 336     RetconLowering.ResumePrototype = Prototype;
 337     RetconLowering.Alloc = ContinuationId->getAllocFunction();
 338     RetconLowering.Dealloc = ContinuationId->getDeallocFunction();
 339     RetconLowering.ReturnBlock = nullptr;
 340     RetconLowering.IsFrameInlineInStorage = false;
 341     break;
 342   }
 343   default:
 344     llvm_unreachable("coro.begin is not dependent on a coro.id call");
 345   }
 346 }
 347
 348 // If for some reason, we were not able to find coro.begin, bailout.
 349 void coro::Shape::invalidateCoroutine(
 350     Function &F, SmallVectorImpl<CoroFrameInst *> &CoroFrames) {
 351   assert(!CoroBegin);
 352   {
 353     // Replace coro.frame which are supposed to be lowered to the result of
 354     // coro.begin with poison.
 355     auto *Poison = PoisonValue::get(PointerType::get(F.getContext(), 0));
 356     for (CoroFrameInst *CF : CoroFrames) {
 357       CF->replaceAllUsesWith(Poison);
 358       CF->eraseFromParent();
 359     }
 360     CoroFrames.clear();
 361
 362     // Replace all coro.suspend with poison and remove related coro.saves if
 363     // present.
 364     for (AnyCoroSuspendInst *CS : CoroSuspends) {
 365       CS->replaceAllUsesWith(PoisonValue::get(CS->getType()));
 366       CS->eraseFromParent();
 367       if (auto *CoroSave = CS->getCoroSave())
 368         CoroSave->eraseFromParent();
 369     }
 370     CoroSuspends.clear();
 371
 372     // Replace all coro.ends with unreachable instruction.
 373     for (AnyCoroEndInst *CE : CoroEnds)
 374       changeToUnreachable(CE);
 375   }
 376 }
 377
 378 void coro::SwitchABI::init() {
 379   assert(Shape.ABI == coro::ABI::Switch);
 380   {
 381     for (auto *AnySuspend : Shape.CoroSuspends) {
 382       auto Suspend = dyn_cast<CoroSuspendInst>(AnySuspend);
 383       if (!Suspend) {
 384 #ifndef NDEBUG
 385         AnySuspend->dump();
 386 #endif
 387         report_fatal_error("coro.id must be paired with coro.suspend");
 388       }
 389
 390       if (!Suspend->getCoroSave())
 391         createCoroSave(Shape.CoroBegin, Suspend);
 392     }
 393   }
 394 }
 395
 396 void coro::AsyncABI::init() { assert(Shape.ABI == coro::ABI::Async); }
 397
 398 void coro::AnyRetconABI::init() {
 399   assert(Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce);
 400   {
 401     // Determine the result value types, and make sure they match up with
 402     // the values passed to the suspends.
 403     auto ResultTys = Shape.getRetconResultTypes();
 404     auto ResumeTys = Shape.getRetconResumeTypes();
 405
 406     for (auto *AnySuspend : Shape.CoroSuspends) {
 407       auto Suspend = dyn_cast<CoroSuspendRetconInst>(AnySuspend);
 408       if (!Suspend) {
 409 #ifndef NDEBUG
 410         AnySuspend->dump();
 411 #endif
 412         report_fatal_error("coro.id.retcon.* must be paired with "
 413                            "coro.suspend.retcon");
 414       }
 415
 416       // Check that the argument types of the suspend match the results.
 417       auto SI = Suspend->value_begin(), SE = Suspend->value_end();
 418       auto RI = ResultTys.begin(), RE = ResultTys.end();
 419       for (; SI != SE && RI != RE; ++SI, ++RI) {
 420         auto SrcTy = (*SI)->getType();
 421         if (SrcTy != *RI) {
 422           // The optimizer likes to eliminate bitcasts leading into variadic
 423           // calls, but that messes with our invariants.  Re-insert the
 424           // bitcast and ignore this type mismatch.
 425           if (CastInst::isBitCastable(SrcTy, *RI)) {
 426             auto BCI = new BitCastInst(*SI, *RI, "", Suspend->getIterator());
 427             SI->set(BCI);
 428             continue;
 429           }
 430
 431 #ifndef NDEBUG
 432           Suspend->dump();
 433           Shape.RetconLowering.ResumePrototype->getFunctionType()->dump();
 434 #endif
 435           report_fatal_error("argument to coro.suspend.retcon does not "
 436                              "match corresponding prototype function result");
 437         }
 438       }
 439       if (SI != SE || RI != RE) {
 440 #ifndef NDEBUG
 441         Suspend->dump();
 442         Shape.RetconLowering.ResumePrototype->getFunctionType()->dump();
 443 #endif
 444         report_fatal_error("wrong number of arguments to coro.suspend.retcon");
 445       }
 446
 447       // Check that the result type of the suspend matches the resume types.
 448       Type *SResultTy = Suspend->getType();
 449       ArrayRef<Type *> SuspendResultTys;
 450       if (SResultTy->isVoidTy()) {
 451         // leave as empty array
 452       } else if (auto SResultStructTy = dyn_cast<StructType>(SResultTy)) {
 453         SuspendResultTys = SResultStructTy->elements();
 454       } else {
 455         // forms an ArrayRef using SResultTy, be careful
 456         SuspendResultTys = SResultTy;
 457       }
 458       if (SuspendResultTys.size() != ResumeTys.size()) {
 459 #ifndef NDEBUG
 460         Suspend->dump();
 461         Shape.RetconLowering.ResumePrototype->getFunctionType()->dump();
 462 #endif
 463         report_fatal_error("wrong number of results from coro.suspend.retcon");
 464       }
 465       for (size_t I = 0, E = ResumeTys.size(); I != E; ++I) {
 466         if (SuspendResultTys[I] != ResumeTys[I]) {
 467 #ifndef NDEBUG
 468           Suspend->dump();
 469           Shape.RetconLowering.ResumePrototype->getFunctionType()->dump();
 470 #endif
 471           report_fatal_error("result from coro.suspend.retcon does not "
 472                              "match corresponding prototype function param");
 473         }
 474       }
 475     }
 476   }
 477 }
 478
 479 void coro::Shape::cleanCoroutine(
 480     SmallVectorImpl<CoroFrameInst *> &CoroFrames,
 481     SmallVectorImpl<CoroSaveInst *> &UnusedCoroSaves) {
 482   // The coro.frame intrinsic is always lowered to the result of coro.begin.
 483   for (CoroFrameInst *CF : CoroFrames) {
 484     CF->replaceAllUsesWith(CoroBegin);
 485     CF->eraseFromParent();
 486   }
 487   CoroFrames.clear();
 488
 489   // Remove orphaned coro.saves.
 490   for (CoroSaveInst *CoroSave : UnusedCoroSaves)
 491     CoroSave->eraseFromParent();
 492   UnusedCoroSaves.clear();
 493 }
 494
 495 static void propagateCallAttrsFromCallee(CallInst *Call, Function *Callee) {
 496   Call->setCallingConv(Callee->getCallingConv());
 497   // TODO: attributes?
 498 }
 499
 500 static void addCallToCallGraph(CallGraph *CG, CallInst *Call, Function *Callee){
 501   if (CG)
 502     (*CG)[Call->getFunction()]->addCalledFunction(Call, (*CG)[Callee]);
 503 }
 504
 505 Value *coro::Shape::emitAlloc(IRBuilder<> &Builder, Value *Size,
 506                               CallGraph *CG) const {
 507   switch (ABI) {
 508   case coro::ABI::Switch:
 509     llvm_unreachable("can't allocate memory in coro switch-lowering");
 510
 511   case coro::ABI::Retcon:
 512   case coro::ABI::RetconOnce: {
 513     auto Alloc = RetconLowering.Alloc;
 514     Size = Builder.CreateIntCast(Size,
 515                                  Alloc->getFunctionType()->getParamType(0),
 516                                  /*is signed*/ false);
 517     auto *Call = Builder.CreateCall(Alloc, Size);
 518     propagateCallAttrsFromCallee(Call, Alloc);
 519     addCallToCallGraph(CG, Call, Alloc);
 520     return Call;
 521   }
 522   case coro::ABI::Async:
 523     llvm_unreachable("can't allocate memory in coro async-lowering");
 524   }
 525   llvm_unreachable("Unknown coro::ABI enum");
 526 }
 527
 528 void coro::Shape::emitDealloc(IRBuilder<> &Builder, Value *Ptr,
 529                               CallGraph *CG) const {
 530   switch (ABI) {
 531   case coro::ABI::Switch:
 532     llvm_unreachable("can't allocate memory in coro switch-lowering");
 533
 534   case coro::ABI::Retcon:
 535   case coro::ABI::RetconOnce: {
 536     auto Dealloc = RetconLowering.Dealloc;
 537     Ptr = Builder.CreateBitCast(Ptr,
 538                                 Dealloc->getFunctionType()->getParamType(0));
 539     auto *Call = Builder.CreateCall(Dealloc, Ptr);
 540     propagateCallAttrsFromCallee(Call, Dealloc);
 541     addCallToCallGraph(CG, Call, Dealloc);
 542     return;
 543   }
 544   case coro::ABI::Async:
 545     llvm_unreachable("can't allocate memory in coro async-lowering");
 546   }
 547   llvm_unreachable("Unknown coro::ABI enum");
 548 }
 549
 550 [[noreturn]] static void fail(const Instruction *I, const char *Reason,
 551                               Value *V) {
 552 #ifndef NDEBUG
 553   I->dump();
 554   if (V) {
 555     errs() << "  Value: ";
 556     V->printAsOperand(llvm::errs());
 557     errs() << '\n';
 558   }
 559 #endif
 560   report_fatal_error(Reason);
 561 }
 562
 563 /// Check that the given value is a well-formed prototype for the
 564 /// llvm.coro.id.retcon.* intrinsics.
 565 static void checkWFRetconPrototype(const AnyCoroIdRetconInst *I, Value *V) {
 566   auto F = dyn_cast<Function>(V->stripPointerCasts());
 567   if (!F)
 568     fail(I, "llvm.coro.id.retcon.* prototype not a Function", V);
 569
 570   auto FT = F->getFunctionType();
 571
 572   if (isa<CoroIdRetconInst>(I)) {
 573     bool ResultOkay;
 574     if (FT->getReturnType()->isPointerTy()) {
 575       ResultOkay = true;
 576     } else if (auto SRetTy = dyn_cast<StructType>(FT->getReturnType())) {
 577       ResultOkay = (!SRetTy->isOpaque() &&
 578                     SRetTy->getNumElements() > 0 &&
 579                     SRetTy->getElementType(0)->isPointerTy());
 580     } else {
 581       ResultOkay = false;
 582     }
 583     if (!ResultOkay)
 584       fail(I, "llvm.coro.id.retcon prototype must return pointer as first "
 585               "result", F);
 586
 587     if (FT->getReturnType() !=
 588           I->getFunction()->getFunctionType()->getReturnType())
 589       fail(I, "llvm.coro.id.retcon prototype return type must be same as"
 590               "current function return type", F);
 591   } else {
 592     // No meaningful validation to do here for llvm.coro.id.unique.once.
 593   }
 594
 595   if (FT->getNumParams() == 0 || !FT->getParamType(0)->isPointerTy())
 596     fail(I, "llvm.coro.id.retcon.* prototype must take pointer as "
 597             "its first parameter", F);
 598 }
 599
 600 /// Check that the given value is a well-formed allocator.
 601 static void checkWFAlloc(const Instruction *I, Value *V) {
 602   auto F = dyn_cast<Function>(V->stripPointerCasts());
 603   if (!F)
 604     fail(I, "llvm.coro.* allocator not a Function", V);
 605
 606   auto FT = F->getFunctionType();
 607   if (!FT->getReturnType()->isPointerTy())
 608     fail(I, "llvm.coro.* allocator must return a pointer", F);
 609
 610   if (FT->getNumParams() != 1 ||
 611       !FT->getParamType(0)->isIntegerTy())
 612     fail(I, "llvm.coro.* allocator must take integer as only param", F);
 613 }
 614
 615 /// Check that the given value is a well-formed deallocator.
 616 static void checkWFDealloc(const Instruction *I, Value *V) {
 617   auto F = dyn_cast<Function>(V->stripPointerCasts());
 618   if (!F)
 619     fail(I, "llvm.coro.* deallocator not a Function", V);
 620
 621   auto FT = F->getFunctionType();
 622   if (!FT->getReturnType()->isVoidTy())
 623     fail(I, "llvm.coro.* deallocator must return void", F);
 624
 625   if (FT->getNumParams() != 1 ||
 626       !FT->getParamType(0)->isPointerTy())
 627     fail(I, "llvm.coro.* deallocator must take pointer as only param", F);
 628 }
 629
 630 static void checkConstantInt(const Instruction *I, Value *V,
 631                              const char *Reason) {
 632   if (!isa<ConstantInt>(V)) {
 633     fail(I, Reason, V);
 634   }
 635 }
 636
 637 void AnyCoroIdRetconInst::checkWellFormed() const {
 638   checkConstantInt(this, getArgOperand(SizeArg),
 639                    "size argument to coro.id.retcon.* must be constant");
 640   checkConstantInt(this, getArgOperand(AlignArg),
 641                    "alignment argument to coro.id.retcon.* must be constant");
 642   checkWFRetconPrototype(this, getArgOperand(PrototypeArg));
 643   checkWFAlloc(this, getArgOperand(AllocArg));
 644   checkWFDealloc(this, getArgOperand(DeallocArg));
 645 }
 646
 647 static void checkAsyncFuncPointer(const Instruction *I, Value *V) {
 648   auto *AsyncFuncPtrAddr = dyn_cast<GlobalVariable>(V->stripPointerCasts());
 649   if (!AsyncFuncPtrAddr)
 650     fail(I, "llvm.coro.id.async async function pointer not a global", V);
 651 }
 652
 653 void CoroIdAsyncInst::checkWellFormed() const {
 654   checkConstantInt(this, getArgOperand(SizeArg),
 655                    "size argument to coro.id.async must be constant");
 656   checkConstantInt(this, getArgOperand(AlignArg),
 657                    "alignment argument to coro.id.async must be constant");
 658   checkConstantInt(this, getArgOperand(StorageArg),
 659                    "storage argument offset to coro.id.async must be constant");
 660   checkAsyncFuncPointer(this, getArgOperand(AsyncFuncPtrArg));
 661 }
 662
 663 static void checkAsyncContextProjectFunction(const Instruction *I,
 664                                              Function *F) {
 665   auto *FunTy = cast<FunctionType>(F->getValueType());
 666   if (!FunTy->getReturnType()->isPointerTy())
 667     fail(I,
 668          "llvm.coro.suspend.async resume function projection function must "
 669          "return a ptr type",
 670          F);
 671   if (FunTy->getNumParams() != 1 || !FunTy->getParamType(0)->isPointerTy())
 672     fail(I,
 673          "llvm.coro.suspend.async resume function projection function must "
 674          "take one ptr type as parameter",
 675          F);
 676 }
 677
 678 void CoroSuspendAsyncInst::checkWellFormed() const {
 679   checkAsyncContextProjectFunction(this, getAsyncContextProjectionFunction());
 680 }
 681
 682 void CoroAsyncEndInst::checkWellFormed() const {
 683   auto *MustTailCallFunc = getMustTailCallFunction();
 684   if (!MustTailCallFunc)
 685     return;
 686   auto *FnTy = MustTailCallFunc->getFunctionType();
 687   if (FnTy->getNumParams() != (arg_size() - 3))
 688     fail(this,
 689          "llvm.coro.end.async must tail call function argument type must "
 690          "match the tail arguments",
 691          MustTailCallFunc);
 692 }