llvm/lib/Bitcode/Writer/BitcodeWriter.cpp

   1 //===- Bitcode/Writer/BitcodeWriter.cpp - Bitcode Writer ------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // Bitcode writer implementation.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "llvm/Bitcode/BitcodeWriter.h"
  14 #include "ValueEnumerator.h"
  15 #include "llvm/ADT/APFloat.h"
  16 #include "llvm/ADT/APInt.h"
  17 #include "llvm/ADT/ArrayRef.h"
  18 #include "llvm/ADT/DenseMap.h"
  19 #include "llvm/ADT/STLExtras.h"
  20 #include "llvm/ADT/SetVector.h"
  21 #include "llvm/ADT/SmallPtrSet.h"
  22 #include "llvm/ADT/SmallString.h"
  23 #include "llvm/ADT/SmallVector.h"
  24 #include "llvm/ADT/StringMap.h"
  25 #include "llvm/ADT/StringRef.h"
  26 #include "llvm/Bitcode/BitcodeCommon.h"
  27 #include "llvm/Bitcode/BitcodeReader.h"
  28 #include "llvm/Bitcode/LLVMBitCodes.h"
  29 #include "llvm/Bitstream/BitCodes.h"
  30 #include "llvm/Bitstream/BitstreamWriter.h"
  31 #include "llvm/Config/llvm-config.h"
  32 #include "llvm/IR/Attributes.h"
  33 #include "llvm/IR/BasicBlock.h"
  34 #include "llvm/IR/Comdat.h"
  35 #include "llvm/IR/Constant.h"
  36 #include "llvm/IR/ConstantRangeList.h"
  37 #include "llvm/IR/Constants.h"
  38 #include "llvm/IR/DebugInfoMetadata.h"
  39 #include "llvm/IR/DebugLoc.h"
  40 #include "llvm/IR/DerivedTypes.h"
  41 #include "llvm/IR/Function.h"
  42 #include "llvm/IR/GlobalAlias.h"
  43 #include "llvm/IR/GlobalIFunc.h"
  44 #include "llvm/IR/GlobalObject.h"
  45 #include "llvm/IR/GlobalValue.h"
  46 #include "llvm/IR/GlobalVariable.h"
  47 #include "llvm/IR/InlineAsm.h"
  48 #include "llvm/IR/InstrTypes.h"
  49 #include "llvm/IR/Instruction.h"
  50 #include "llvm/IR/Instructions.h"
  51 #include "llvm/IR/LLVMContext.h"
  52 #include "llvm/IR/Metadata.h"
  53 #include "llvm/IR/Module.h"
  54 #include "llvm/IR/ModuleSummaryIndex.h"
  55 #include "llvm/IR/Operator.h"
  56 #include "llvm/IR/Type.h"
  57 #include "llvm/IR/UseListOrder.h"
  58 #include "llvm/IR/Value.h"
  59 #include "llvm/IR/ValueSymbolTable.h"
  60 #include "llvm/MC/StringTableBuilder.h"
  61 #include "llvm/MC/TargetRegistry.h"
  62 #include "llvm/Object/IRSymtab.h"
  63 #include "llvm/ProfileData/MemProf.h"
  64 #include "llvm/Support/AtomicOrdering.h"
  65 #include "llvm/Support/Casting.h"
  66 #include "llvm/Support/CommandLine.h"
  67 #include "llvm/Support/Endian.h"
  68 #include "llvm/Support/Error.h"
  69 #include "llvm/Support/ErrorHandling.h"
  70 #include "llvm/Support/MathExtras.h"
  71 #include "llvm/Support/SHA1.h"
  72 #include "llvm/Support/raw_ostream.h"
  73 #include "llvm/TargetParser/Triple.h"
  74 #include <algorithm>
  75 #include <cassert>
  76 #include <cstddef>
  77 #include <cstdint>
  78 #include <iterator>
  79 #include <map>
  80 #include <memory>
  81 #include <optional>
  82 #include <string>
  83 #include <utility>
  84 #include <vector>
  85
  86 using namespace llvm;
  87 using namespace llvm::memprof;
  88
  89 static cl::opt<unsigned>
  90     IndexThreshold("bitcode-mdindex-threshold", cl::Hidden, cl::init(25),
  91                    cl::desc("Number of metadatas above which we emit an index "
  92                             "to enable lazy-loading"));
  93 static cl::opt<uint32_t> FlushThreshold(
  94     "bitcode-flush-threshold", cl::Hidden, cl::init(512),
  95     cl::desc("The threshold (unit M) for flushing LLVM bitcode."));
  96
  97 static cl::opt<bool> WriteRelBFToSummary(
  98     "write-relbf-to-summary", cl::Hidden, cl::init(false),
  99     cl::desc("Write relative block frequency to function summary "));
 100
 101 namespace llvm {
 102 extern FunctionSummary::ForceSummaryHotnessType ForceSummaryEdgesCold;
 103 }
 104
 105 extern bool WriteNewDbgInfoFormatToBitcode;
 106 extern llvm::cl::opt<bool> UseNewDbgInfoFormat;
 107
 108 namespace {
 109
 110 /// These are manifest constants used by the bitcode writer. They do not need to
 111 /// be kept in sync with the reader, but need to be consistent within this file.
 112 enum {
 113   // VALUE_SYMTAB_BLOCK abbrev id's.
 114   VST_ENTRY_8_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
 115   VST_ENTRY_7_ABBREV,
 116   VST_ENTRY_6_ABBREV,
 117   VST_BBENTRY_6_ABBREV,
 118
 119   // CONSTANTS_BLOCK abbrev id's.
 120   CONSTANTS_SETTYPE_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
 121   CONSTANTS_INTEGER_ABBREV,
 122   CONSTANTS_CE_CAST_Abbrev,
 123   CONSTANTS_NULL_Abbrev,
 124
 125   // FUNCTION_BLOCK abbrev id's.
 126   FUNCTION_INST_LOAD_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
 127   FUNCTION_INST_UNOP_ABBREV,
 128   FUNCTION_INST_UNOP_FLAGS_ABBREV,
 129   FUNCTION_INST_BINOP_ABBREV,
 130   FUNCTION_INST_BINOP_FLAGS_ABBREV,
 131   FUNCTION_INST_CAST_ABBREV,
 132   FUNCTION_INST_CAST_FLAGS_ABBREV,
 133   FUNCTION_INST_RET_VOID_ABBREV,
 134   FUNCTION_INST_RET_VAL_ABBREV,
 135   FUNCTION_INST_UNREACHABLE_ABBREV,
 136   FUNCTION_INST_GEP_ABBREV,
 137   FUNCTION_DEBUG_RECORD_VALUE_ABBREV,
 138 };
 139
 140 /// Abstract class to manage the bitcode writing, subclassed for each bitcode
 141 /// file type.
 142 class BitcodeWriterBase {
 143 protected:
 144   /// The stream created and owned by the client.
 145   BitstreamWriter &Stream;
 146
 147   StringTableBuilder &StrtabBuilder;
 148
 149 public:
 150   /// Constructs a BitcodeWriterBase object that writes to the provided
 151   /// \p Stream.
 152   BitcodeWriterBase(BitstreamWriter &Stream, StringTableBuilder &StrtabBuilder)
 153       : Stream(Stream), StrtabBuilder(StrtabBuilder) {}
 154
 155 protected:
 156   void writeModuleVersion();
 157 };
 158
 159 void BitcodeWriterBase::writeModuleVersion() {
 160   // VERSION: [version#]
 161   Stream.EmitRecord(bitc::MODULE_CODE_VERSION, ArrayRef<uint64_t>{2});
 162 }
 163
 164 /// Base class to manage the module bitcode writing, currently subclassed for
 165 /// ModuleBitcodeWriter and ThinLinkBitcodeWriter.
 166 class ModuleBitcodeWriterBase : public BitcodeWriterBase {
 167 protected:
 168   /// The Module to write to bitcode.
 169   const Module &M;
 170
 171   /// Enumerates ids for all values in the module.
 172   ValueEnumerator VE;
 173
 174   /// Optional per-module index to write for ThinLTO.
 175   const ModuleSummaryIndex *Index;
 176
 177   /// Map that holds the correspondence between GUIDs in the summary index,
 178   /// that came from indirect call profiles, and a value id generated by this
 179   /// class to use in the VST and summary block records.
 180   std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
 181
 182   /// Tracks the last value id recorded in the GUIDToValueMap.
 183   unsigned GlobalValueId;
 184
 185   /// Saves the offset of the VSTOffset record that must eventually be
 186   /// backpatched with the offset of the actual VST.
 187   uint64_t VSTOffsetPlaceholder = 0;
 188
 189 public:
 190   /// Constructs a ModuleBitcodeWriterBase object for the given Module,
 191   /// writing to the provided \p Buffer.
 192   ModuleBitcodeWriterBase(const Module &M, StringTableBuilder &StrtabBuilder,
 193                           BitstreamWriter &Stream,
 194                           bool ShouldPreserveUseListOrder,
 195                           const ModuleSummaryIndex *Index)
 196       : BitcodeWriterBase(Stream, StrtabBuilder), M(M),
 197         VE(M, ShouldPreserveUseListOrder), Index(Index) {
 198     // Assign ValueIds to any callee values in the index that came from
 199     // indirect call profiles and were recorded as a GUID not a Value*
 200     // (which would have been assigned an ID by the ValueEnumerator).
 201     // The starting ValueId is just after the number of values in the
 202     // ValueEnumerator, so that they can be emitted in the VST.
 203     GlobalValueId = VE.getValues().size();
 204     if (!Index)
 205       return;
 206     for (const auto &GUIDSummaryLists : *Index)
 207       // Examine all summaries for this GUID.
 208       for (auto &Summary : GUIDSummaryLists.second.SummaryList)
 209         if (auto FS = dyn_cast<FunctionSummary>(Summary.get())) {
 210           // For each call in the function summary, see if the call
 211           // is to a GUID (which means it is for an indirect call,
 212           // otherwise we would have a Value for it). If so, synthesize
 213           // a value id.
 214           for (auto &CallEdge : FS->calls())
 215             if (!CallEdge.first.haveGVs() || !CallEdge.first.getValue())
 216               assignValueId(CallEdge.first.getGUID());
 217
 218           // For each referenced variables in the function summary, see if the
 219           // variable is represented by a GUID (as opposed to a symbol to
 220           // declarations or definitions in the module). If so, synthesize a
 221           // value id.
 222           for (auto &RefEdge : FS->refs())
 223             if (!RefEdge.haveGVs() || !RefEdge.getValue())
 224               assignValueId(RefEdge.getGUID());
 225         }
 226   }
 227
 228 protected:
 229   void writePerModuleGlobalValueSummary();
 230
 231 private:
 232   void writePerModuleFunctionSummaryRecord(
 233       SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
 234       unsigned ValueID, unsigned FSCallsAbbrev, unsigned FSCallsProfileAbbrev,
 235       unsigned CallsiteAbbrev, unsigned AllocAbbrev, unsigned ContextIdAbbvId,
 236       const Function &F, DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
 237       CallStackId &CallStackCount);
 238   void writeModuleLevelReferences(const GlobalVariable &V,
 239                                   SmallVector<uint64_t, 64> &NameVals,
 240                                   unsigned FSModRefsAbbrev,
 241                                   unsigned FSModVTableRefsAbbrev);
 242
 243   void assignValueId(GlobalValue::GUID ValGUID) {
 244     GUIDToValueIdMap[ValGUID] = ++GlobalValueId;
 245   }
 246
 247   unsigned getValueId(GlobalValue::GUID ValGUID) {
 248     const auto &VMI = GUIDToValueIdMap.find(ValGUID);
 249     // Expect that any GUID value had a value Id assigned by an
 250     // earlier call to assignValueId.
 251     assert(VMI != GUIDToValueIdMap.end() &&
 252            "GUID does not have assigned value Id");
 253     return VMI->second;
 254   }
 255
 256   // Helper to get the valueId for the type of value recorded in VI.
 257   unsigned getValueId(ValueInfo VI) {
 258     if (!VI.haveGVs() || !VI.getValue())
 259       return getValueId(VI.getGUID());
 260     return VE.getValueID(VI.getValue());
 261   }
 262
 263   std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; }
 264 };
 265
 266 /// Class to manage the bitcode writing for a module.
 267 class ModuleBitcodeWriter : public ModuleBitcodeWriterBase {
 268   /// True if a module hash record should be written.
 269   bool GenerateHash;
 270
 271   /// If non-null, when GenerateHash is true, the resulting hash is written
 272   /// into ModHash.
 273   ModuleHash *ModHash;
 274
 275   SHA1 Hasher;
 276
 277   /// The start bit of the identification block.
 278   uint64_t BitcodeStartBit;
 279
 280 public:
 281   /// Constructs a ModuleBitcodeWriter object for the given Module,
 282   /// writing to the provided \p Buffer.
 283   ModuleBitcodeWriter(const Module &M, StringTableBuilder &StrtabBuilder,
 284                       BitstreamWriter &Stream, bool ShouldPreserveUseListOrder,
 285                       const ModuleSummaryIndex *Index, bool GenerateHash,
 286                       ModuleHash *ModHash = nullptr)
 287       : ModuleBitcodeWriterBase(M, StrtabBuilder, Stream,
 288                                 ShouldPreserveUseListOrder, Index),
 289         GenerateHash(GenerateHash), ModHash(ModHash),
 290         BitcodeStartBit(Stream.GetCurrentBitNo()) {}
 291
 292   /// Emit the current module to the bitstream.
 293   void write();
 294
 295 private:
 296   uint64_t bitcodeStartBit() { return BitcodeStartBit; }
 297
 298   size_t addToStrtab(StringRef Str);
 299
 300   void writeAttributeGroupTable();
 301   void writeAttributeTable();
 302   void writeTypeTable();
 303   void writeComdats();
 304   void writeValueSymbolTableForwardDecl();
 305   void writeModuleInfo();
 306   void writeValueAsMetadata(const ValueAsMetadata *MD,
 307                             SmallVectorImpl<uint64_t> &Record);
 308   void writeMDTuple(const MDTuple *N, SmallVectorImpl<uint64_t> &Record,
 309                     unsigned Abbrev);
 310   unsigned createDILocationAbbrev();
 311   void writeDILocation(const DILocation *N, SmallVectorImpl<uint64_t> &Record,
 312                        unsigned &Abbrev);
 313   unsigned createGenericDINodeAbbrev();
 314   void writeGenericDINode(const GenericDINode *N,
 315                           SmallVectorImpl<uint64_t> &Record, unsigned &Abbrev);
 316   void writeDISubrange(const DISubrange *N, SmallVectorImpl<uint64_t> &Record,
 317                        unsigned Abbrev);
 318   void writeDIGenericSubrange(const DIGenericSubrange *N,
 319                               SmallVectorImpl<uint64_t> &Record,
 320                               unsigned Abbrev);
 321   void writeDIEnumerator(const DIEnumerator *N,
 322                          SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
 323   void writeDIBasicType(const DIBasicType *N, SmallVectorImpl<uint64_t> &Record,
 324                         unsigned Abbrev);
 325   void writeDIStringType(const DIStringType *N,
 326                          SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
 327   void writeDIDerivedType(const DIDerivedType *N,
 328                           SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
 329   void writeDICompositeType(const DICompositeType *N,
 330                             SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
 331   void writeDISubroutineType(const DISubroutineType *N,
 332                              SmallVectorImpl<uint64_t> &Record,
 333                              unsigned Abbrev);
 334   void writeDIFile(const DIFile *N, SmallVectorImpl<uint64_t> &Record,
 335                    unsigned Abbrev);
 336   void writeDICompileUnit(const DICompileUnit *N,
 337                           SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
 338   void writeDISubprogram(const DISubprogram *N,
 339                          SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
 340   void writeDILexicalBlock(const DILexicalBlock *N,
 341                            SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
 342   void writeDILexicalBlockFile(const DILexicalBlockFile *N,
 343                                SmallVectorImpl<uint64_t> &Record,
 344                                unsigned Abbrev);
 345   void writeDICommonBlock(const DICommonBlock *N,
 346                           SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
 347   void writeDINamespace(const DINamespace *N, SmallVectorImpl<uint64_t> &Record,
 348                         unsigned Abbrev);
 349   void writeDIMacro(const DIMacro *N, SmallVectorImpl<uint64_t> &Record,
 350                     unsigned Abbrev);
 351   void writeDIMacroFile(const DIMacroFile *N, SmallVectorImpl<uint64_t> &Record,
 352                         unsigned Abbrev);
 353   void writeDIArgList(const DIArgList *N, SmallVectorImpl<uint64_t> &Record);
 354   void writeDIModule(const DIModule *N, SmallVectorImpl<uint64_t> &Record,
 355                      unsigned Abbrev);
 356   void writeDIAssignID(const DIAssignID *N, SmallVectorImpl<uint64_t> &Record,
 357                        unsigned Abbrev);
 358   void writeDITemplateTypeParameter(const DITemplateTypeParameter *N,
 359                                     SmallVectorImpl<uint64_t> &Record,
 360                                     unsigned Abbrev);
 361   void writeDITemplateValueParameter(const DITemplateValueParameter *N,
 362                                      SmallVectorImpl<uint64_t> &Record,
 363                                      unsigned Abbrev);
 364   void writeDIGlobalVariable(const DIGlobalVariable *N,
 365                              SmallVectorImpl<uint64_t> &Record,
 366                              unsigned Abbrev);
 367   void writeDILocalVariable(const DILocalVariable *N,
 368                             SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
 369   void writeDILabel(const DILabel *N,
 370                     SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
 371   void writeDIExpression(const DIExpression *N,
 372                          SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
 373   void writeDIGlobalVariableExpression(const DIGlobalVariableExpression *N,
 374                                        SmallVectorImpl<uint64_t> &Record,
 375                                        unsigned Abbrev);
 376   void writeDIObjCProperty(const DIObjCProperty *N,
 377                            SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
 378   void writeDIImportedEntity(const DIImportedEntity *N,
 379                              SmallVectorImpl<uint64_t> &Record,
 380                              unsigned Abbrev);
 381   unsigned createNamedMetadataAbbrev();
 382   void writeNamedMetadata(SmallVectorImpl<uint64_t> &Record);
 383   unsigned createMetadataStringsAbbrev();
 384   void writeMetadataStrings(ArrayRef<const Metadata *> Strings,
 385                             SmallVectorImpl<uint64_t> &Record);
 386   void writeMetadataRecords(ArrayRef<const Metadata *> MDs,
 387                             SmallVectorImpl<uint64_t> &Record,
 388                             std::vector<unsigned> *MDAbbrevs = nullptr,
 389                             std::vector<uint64_t> *IndexPos = nullptr);
 390   void writeModuleMetadata();
 391   void writeFunctionMetadata(const Function &F);
 392   void writeFunctionMetadataAttachment(const Function &F);
 393   void pushGlobalMetadataAttachment(SmallVectorImpl<uint64_t> &Record,
 394                                     const GlobalObject &GO);
 395   void writeModuleMetadataKinds();
 396   void writeOperandBundleTags();
 397   void writeSyncScopeNames();
 398   void writeConstants(unsigned FirstVal, unsigned LastVal, bool isGlobal);
 399   void writeModuleConstants();
 400   bool pushValueAndType(const Value *V, unsigned InstID,
 401                         SmallVectorImpl<unsigned> &Vals);
 402   bool pushValueOrMetadata(const Value *V, unsigned InstID,
 403                            SmallVectorImpl<unsigned> &Vals);
 404   void writeOperandBundles(const CallBase &CB, unsigned InstID);
 405   void pushValue(const Value *V, unsigned InstID,
 406                  SmallVectorImpl<unsigned> &Vals);
 407   void pushValueSigned(const Value *V, unsigned InstID,
 408                        SmallVectorImpl<uint64_t> &Vals);
 409   void writeInstruction(const Instruction &I, unsigned InstID,
 410                         SmallVectorImpl<unsigned> &Vals);
 411   void writeFunctionLevelValueSymbolTable(const ValueSymbolTable &VST);
 412   void writeGlobalValueSymbolTable(
 413       DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex);
 414   void writeUseList(UseListOrder &&Order);
 415   void writeUseListBlock(const Function *F);
 416   void
 417   writeFunction(const Function &F,
 418                 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex);
 419   void writeBlockInfo();
 420   void writeModuleHash(StringRef View);
 421
 422   unsigned getEncodedSyncScopeID(SyncScope::ID SSID) {
 423     return unsigned(SSID);
 424   }
 425
 426   unsigned getEncodedAlign(MaybeAlign Alignment) { return encode(Alignment); }
 427 };
 428
 429 /// Class to manage the bitcode writing for a combined index.
 430 class IndexBitcodeWriter : public BitcodeWriterBase {
 431   /// The combined index to write to bitcode.
 432   const ModuleSummaryIndex &Index;
 433
 434   /// When writing combined summaries, provides the set of global value
 435   /// summaries for which the value (function, function alias, etc) should be
 436   /// imported as a declaration.
 437   const GVSummaryPtrSet *DecSummaries = nullptr;
 438
 439   /// When writing a subset of the index for distributed backends, client
 440   /// provides a map of modules to the corresponding GUIDs/summaries to write.
 441   const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex;
 442
 443   /// Map that holds the correspondence between the GUID used in the combined
 444   /// index and a value id generated by this class to use in references.
 445   std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
 446
 447   // The stack ids used by this index, which will be a subset of those in
 448   // the full index in the case of distributed indexes.
 449   std::vector<uint64_t> StackIds;
 450
 451   // Keep a map of the stack id indices used by records being written for this
 452   // index to the index of the corresponding stack id in the above StackIds
 453   // vector. Ensures we write each referenced stack id once.
 454   DenseMap<unsigned, unsigned> StackIdIndicesToIndex;
 455
 456   /// Tracks the last value id recorded in the GUIDToValueMap.
 457   unsigned GlobalValueId = 0;
 458
 459   /// Tracks the assignment of module paths in the module path string table to
 460   /// an id assigned for use in summary references to the module path.
 461   DenseMap<StringRef, uint64_t> ModuleIdMap;
 462
 463 public:
 464   /// Constructs a IndexBitcodeWriter object for the given combined index,
 465   /// writing to the provided \p Buffer. When writing a subset of the index
 466   /// for a distributed backend, provide a \p ModuleToSummariesForIndex map.
 467   /// If provided, \p DecSummaries specifies the set of summaries for which
 468   /// the corresponding functions or aliased functions should be imported as a
 469   /// declaration (but not definition) for each module.
 470   IndexBitcodeWriter(
 471       BitstreamWriter &Stream, StringTableBuilder &StrtabBuilder,
 472       const ModuleSummaryIndex &Index,
 473       const GVSummaryPtrSet *DecSummaries = nullptr,
 474       const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex = nullptr)
 475       : BitcodeWriterBase(Stream, StrtabBuilder), Index(Index),
 476         DecSummaries(DecSummaries),
 477         ModuleToSummariesForIndex(ModuleToSummariesForIndex) {
 478
 479     // See if the StackIdIndex was already added to the StackId map and
 480     // vector. If not, record it.
 481     auto RecordStackIdReference = [&](unsigned StackIdIndex) {
 482       // If the StackIdIndex is not yet in the map, the below insert ensures
 483       // that it will point to the new StackIds vector entry we push to just
 484       // below.
 485       auto Inserted =
 486           StackIdIndicesToIndex.insert({StackIdIndex, StackIds.size()});
 487       if (Inserted.second)
 488         StackIds.push_back(Index.getStackIdAtIndex(StackIdIndex));
 489     };
 490
 491     // Assign unique value ids to all summaries to be written, for use
 492     // in writing out the call graph edges. Save the mapping from GUID
 493     // to the new global value id to use when writing those edges, which
 494     // are currently saved in the index in terms of GUID.
 495     forEachSummary([&](GVInfo I, bool IsAliasee) {
 496       GUIDToValueIdMap[I.first] = ++GlobalValueId;
 497       // If this is invoked for an aliasee, we want to record the above mapping,
 498       // but not the information needed for its summary entry (if the aliasee is
 499       // to be imported, we will invoke this separately with IsAliasee=false).
 500       if (IsAliasee)
 501         return;
 502       auto *FS = dyn_cast<FunctionSummary>(I.second);
 503       if (!FS)
 504         return;
 505       // Record all stack id indices actually used in the summary entries being
 506       // written, so that we can compact them in the case of distributed ThinLTO
 507       // indexes.
 508       for (auto &CI : FS->callsites()) {
 509         // If the stack id list is empty, this callsite info was synthesized for
 510         // a missing tail call frame. Ensure that the callee's GUID gets a value
 511         // id. Normally we only generate these for defined summaries, which in
 512         // the case of distributed ThinLTO is only the functions already defined
 513         // in the module or that we want to import. We don't bother to include
 514         // all the callee symbols as they aren't normally needed in the backend.
 515         // However, for the synthesized callsite infos we do need the callee
 516         // GUID in the backend so that we can correlate the identified callee
 517         // with this callsite info (which for non-tail calls is done by the
 518         // ordering of the callsite infos and verified via stack ids).
 519         if (CI.StackIdIndices.empty()) {
 520           GUIDToValueIdMap[CI.Callee.getGUID()] = ++GlobalValueId;
 521           continue;
 522         }
 523         for (auto Idx : CI.StackIdIndices)
 524           RecordStackIdReference(Idx);
 525       }
 526       for (auto &AI : FS->allocs())
 527         for (auto &MIB : AI.MIBs)
 528           for (auto Idx : MIB.StackIdIndices)
 529             RecordStackIdReference(Idx);
 530     });
 531   }
 532
 533   /// The below iterator returns the GUID and associated summary.
 534   using GVInfo = std::pair<GlobalValue::GUID, GlobalValueSummary *>;
 535
 536   /// Calls the callback for each value GUID and summary to be written to
 537   /// bitcode. This hides the details of whether they are being pulled from the
 538   /// entire index or just those in a provided ModuleToSummariesForIndex map.
 539   template<typename Functor>
 540   void forEachSummary(Functor Callback) {
 541     if (ModuleToSummariesForIndex) {
 542       for (auto &M : *ModuleToSummariesForIndex)
 543         for (auto &Summary : M.second) {
 544           Callback(Summary, false);
 545           // Ensure aliasee is handled, e.g. for assigning a valueId,
 546           // even if we are not importing the aliasee directly (the
 547           // imported alias will contain a copy of aliasee).
 548           if (auto *AS = dyn_cast<AliasSummary>(Summary.getSecond()))
 549             Callback({AS->getAliaseeGUID(), &AS->getAliasee()}, true);
 550         }
 551     } else {
 552       for (auto &Summaries : Index)
 553         for (auto &Summary : Summaries.second.SummaryList)
 554           Callback({Summaries.first, Summary.get()}, false);
 555     }
 556   }
 557
 558   /// Calls the callback for each entry in the modulePaths StringMap that
 559   /// should be written to the module path string table. This hides the details
 560   /// of whether they are being pulled from the entire index or just those in a
 561   /// provided ModuleToSummariesForIndex map.
 562   template <typename Functor> void forEachModule(Functor Callback) {
 563     if (ModuleToSummariesForIndex) {
 564       for (const auto &M : *ModuleToSummariesForIndex) {
 565         const auto &MPI = Index.modulePaths().find(M.first);
 566         if (MPI == Index.modulePaths().end()) {
 567           // This should only happen if the bitcode file was empty, in which
 568           // case we shouldn't be importing (the ModuleToSummariesForIndex
 569           // would only include the module we are writing and index for).
 570           assert(ModuleToSummariesForIndex->size() == 1);
 571           continue;
 572         }
 573         Callback(*MPI);
 574       }
 575     } else {
 576       // Since StringMap iteration order isn't guaranteed, order by path string
 577       // first.
 578       // FIXME: Make this a vector of StringMapEntry instead to avoid the later
 579       // map lookup.
 580       std::vector<StringRef> ModulePaths;
 581       for (auto &[ModPath, _] : Index.modulePaths())
 582         ModulePaths.push_back(ModPath);
 583       llvm::sort(ModulePaths.begin(), ModulePaths.end());
 584       for (auto &ModPath : ModulePaths)
 585         Callback(*Index.modulePaths().find(ModPath));
 586     }
 587   }
 588
 589   /// Main entry point for writing a combined index to bitcode.
 590   void write();
 591
 592 private:
 593   void writeModStrings();
 594   void writeCombinedGlobalValueSummary();
 595
 596   std::optional<unsigned> getValueId(GlobalValue::GUID ValGUID) {
 597     auto VMI = GUIDToValueIdMap.find(ValGUID);
 598     if (VMI == GUIDToValueIdMap.end())
 599       return std::nullopt;
 600     return VMI->second;
 601   }
 602
 603   std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; }
 604 };
 605
 606 } // end anonymous namespace
 607
 608 static unsigned getEncodedCastOpcode(unsigned Opcode) {
 609   switch (Opcode) {
 610   default: llvm_unreachable("Unknown cast instruction!");
 611   case Instruction::Trunc   : return bitc::CAST_TRUNC;
 612   case Instruction::ZExt    : return bitc::CAST_ZEXT;
 613   case Instruction::SExt    : return bitc::CAST_SEXT;
 614   case Instruction::FPToUI  : return bitc::CAST_FPTOUI;
 615   case Instruction::FPToSI  : return bitc::CAST_FPTOSI;
 616   case Instruction::UIToFP  : return bitc::CAST_UITOFP;
 617   case Instruction::SIToFP  : return bitc::CAST_SITOFP;
 618   case Instruction::FPTrunc : return bitc::CAST_FPTRUNC;
 619   case Instruction::FPExt   : return bitc::CAST_FPEXT;
 620   case Instruction::PtrToInt: return bitc::CAST_PTRTOINT;
 621   case Instruction::IntToPtr: return bitc::CAST_INTTOPTR;
 622   case Instruction::BitCast : return bitc::CAST_BITCAST;
 623   case Instruction::AddrSpaceCast: return bitc::CAST_ADDRSPACECAST;
 624   }
 625 }
 626
 627 static unsigned getEncodedUnaryOpcode(unsigned Opcode) {
 628   switch (Opcode) {
 629   default: llvm_unreachable("Unknown binary instruction!");
 630   case Instruction::FNeg: return bitc::UNOP_FNEG;
 631   }
 632 }
 633
 634 static unsigned getEncodedBinaryOpcode(unsigned Opcode) {
 635   switch (Opcode) {
 636   default: llvm_unreachable("Unknown binary instruction!");
 637   case Instruction::Add:
 638   case Instruction::FAdd: return bitc::BINOP_ADD;
 639   case Instruction::Sub:
 640   case Instruction::FSub: return bitc::BINOP_SUB;
 641   case Instruction::Mul:
 642   case Instruction::FMul: return bitc::BINOP_MUL;
 643   case Instruction::UDiv: return bitc::BINOP_UDIV;
 644   case Instruction::FDiv:
 645   case Instruction::SDiv: return bitc::BINOP_SDIV;
 646   case Instruction::URem: return bitc::BINOP_UREM;
 647   case Instruction::FRem:
 648   case Instruction::SRem: return bitc::BINOP_SREM;
 649   case Instruction::Shl:  return bitc::BINOP_SHL;
 650   case Instruction::LShr: return bitc::BINOP_LSHR;
 651   case Instruction::AShr: return bitc::BINOP_ASHR;
 652   case Instruction::And:  return bitc::BINOP_AND;
 653   case Instruction::Or:   return bitc::BINOP_OR;
 654   case Instruction::Xor:  return bitc::BINOP_XOR;
 655   }
 656 }
 657
 658 static unsigned getEncodedRMWOperation(AtomicRMWInst::BinOp Op) {
 659   switch (Op) {
 660   default: llvm_unreachable("Unknown RMW operation!");
 661   case AtomicRMWInst::Xchg: return bitc::RMW_XCHG;
 662   case AtomicRMWInst::Add: return bitc::RMW_ADD;
 663   case AtomicRMWInst::Sub: return bitc::RMW_SUB;
 664   case AtomicRMWInst::And: return bitc::RMW_AND;
 665   case AtomicRMWInst::Nand: return bitc::RMW_NAND;
 666   case AtomicRMWInst::Or: return bitc::RMW_OR;
 667   case AtomicRMWInst::Xor: return bitc::RMW_XOR;
 668   case AtomicRMWInst::Max: return bitc::RMW_MAX;
 669   case AtomicRMWInst::Min: return bitc::RMW_MIN;
 670   case AtomicRMWInst::UMax: return bitc::RMW_UMAX;
 671   case AtomicRMWInst::UMin: return bitc::RMW_UMIN;
 672   case AtomicRMWInst::FAdd: return bitc::RMW_FADD;
 673   case AtomicRMWInst::FSub: return bitc::RMW_FSUB;
 674   case AtomicRMWInst::FMax: return bitc::RMW_FMAX;
 675   case AtomicRMWInst::FMin: return bitc::RMW_FMIN;
 676   case AtomicRMWInst::UIncWrap:
 677     return bitc::RMW_UINC_WRAP;
 678   case AtomicRMWInst::UDecWrap:
 679     return bitc::RMW_UDEC_WRAP;
 680   case AtomicRMWInst::USubCond:
 681     return bitc::RMW_USUB_COND;
 682   case AtomicRMWInst::USubSat:
 683     return bitc::RMW_USUB_SAT;
 684   }
 685 }
 686
 687 static unsigned getEncodedOrdering(AtomicOrdering Ordering) {
 688   switch (Ordering) {
 689   case AtomicOrdering::NotAtomic: return bitc::ORDERING_NOTATOMIC;
 690   case AtomicOrdering::Unordered: return bitc::ORDERING_UNORDERED;
 691   case AtomicOrdering::Monotonic: return bitc::ORDERING_MONOTONIC;
 692   case AtomicOrdering::Acquire: return bitc::ORDERING_ACQUIRE;
 693   case AtomicOrdering::Release: return bitc::ORDERING_RELEASE;
 694   case AtomicOrdering::AcquireRelease: return bitc::ORDERING_ACQREL;
 695   case AtomicOrdering::SequentiallyConsistent: return bitc::ORDERING_SEQCST;
 696   }
 697   llvm_unreachable("Invalid ordering");
 698 }
 699
 700 static void writeStringRecord(BitstreamWriter &Stream, unsigned Code,
 701                               StringRef Str, unsigned AbbrevToUse) {
 702   SmallVector<unsigned, 64> Vals;
 703
 704   // Code: [strchar x N]
 705   for (char C : Str) {
 706     if (AbbrevToUse && !BitCodeAbbrevOp::isChar6(C))
 707       AbbrevToUse = 0;
 708     Vals.push_back(C);
 709   }
 710
 711   // Emit the finished record.
 712   Stream.EmitRecord(Code, Vals, AbbrevToUse);
 713 }
 714
 715 static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
 716   switch (Kind) {
 717   case Attribute::Alignment:
 718     return bitc::ATTR_KIND_ALIGNMENT;
 719   case Attribute::AllocAlign:
 720     return bitc::ATTR_KIND_ALLOC_ALIGN;
 721   case Attribute::AllocSize:
 722     return bitc::ATTR_KIND_ALLOC_SIZE;
 723   case Attribute::AlwaysInline:
 724     return bitc::ATTR_KIND_ALWAYS_INLINE;
 725   case Attribute::Builtin:
 726     return bitc::ATTR_KIND_BUILTIN;
 727   case Attribute::ByVal:
 728     return bitc::ATTR_KIND_BY_VAL;
 729   case Attribute::Convergent:
 730     return bitc::ATTR_KIND_CONVERGENT;
 731   case Attribute::InAlloca:
 732     return bitc::ATTR_KIND_IN_ALLOCA;
 733   case Attribute::Cold:
 734     return bitc::ATTR_KIND_COLD;
 735   case Attribute::DisableSanitizerInstrumentation:
 736     return bitc::ATTR_KIND_DISABLE_SANITIZER_INSTRUMENTATION;
 737   case Attribute::FnRetThunkExtern:
 738     return bitc::ATTR_KIND_FNRETTHUNK_EXTERN;
 739   case Attribute::Hot:
 740     return bitc::ATTR_KIND_HOT;
 741   case Attribute::ElementType:
 742     return bitc::ATTR_KIND_ELEMENTTYPE;
 743   case Attribute::HybridPatchable:
 744     return bitc::ATTR_KIND_HYBRID_PATCHABLE;
 745   case Attribute::InlineHint:
 746     return bitc::ATTR_KIND_INLINE_HINT;
 747   case Attribute::InReg:
 748     return bitc::ATTR_KIND_IN_REG;
 749   case Attribute::JumpTable:
 750     return bitc::ATTR_KIND_JUMP_TABLE;
 751   case Attribute::MinSize:
 752     return bitc::ATTR_KIND_MIN_SIZE;
 753   case Attribute::AllocatedPointer:
 754     return bitc::ATTR_KIND_ALLOCATED_POINTER;
 755   case Attribute::AllocKind:
 756     return bitc::ATTR_KIND_ALLOC_KIND;
 757   case Attribute::Memory:
 758     return bitc::ATTR_KIND_MEMORY;
 759   case Attribute::NoFPClass:
 760     return bitc::ATTR_KIND_NOFPCLASS;
 761   case Attribute::Naked:
 762     return bitc::ATTR_KIND_NAKED;
 763   case Attribute::Nest:
 764     return bitc::ATTR_KIND_NEST;
 765   case Attribute::NoAlias:
 766     return bitc::ATTR_KIND_NO_ALIAS;
 767   case Attribute::NoBuiltin:
 768     return bitc::ATTR_KIND_NO_BUILTIN;
 769   case Attribute::NoCallback:
 770     return bitc::ATTR_KIND_NO_CALLBACK;
 771   case Attribute::NoCapture:
 772     return bitc::ATTR_KIND_NO_CAPTURE;
 773   case Attribute::NoDivergenceSource:
 774     return bitc::ATTR_KIND_NO_DIVERGENCE_SOURCE;
 775   case Attribute::NoDuplicate:
 776     return bitc::ATTR_KIND_NO_DUPLICATE;
 777   case Attribute::NoFree:
 778     return bitc::ATTR_KIND_NOFREE;
 779   case Attribute::NoImplicitFloat:
 780     return bitc::ATTR_KIND_NO_IMPLICIT_FLOAT;
 781   case Attribute::NoInline:
 782     return bitc::ATTR_KIND_NO_INLINE;
 783   case Attribute::NoRecurse:
 784     return bitc::ATTR_KIND_NO_RECURSE;
 785   case Attribute::NoMerge:
 786     return bitc::ATTR_KIND_NO_MERGE;
 787   case Attribute::NonLazyBind:
 788     return bitc::ATTR_KIND_NON_LAZY_BIND;
 789   case Attribute::NonNull:
 790     return bitc::ATTR_KIND_NON_NULL;
 791   case Attribute::Dereferenceable:
 792     return bitc::ATTR_KIND_DEREFERENCEABLE;
 793   case Attribute::DereferenceableOrNull:
 794     return bitc::ATTR_KIND_DEREFERENCEABLE_OR_NULL;
 795   case Attribute::NoRedZone:
 796     return bitc::ATTR_KIND_NO_RED_ZONE;
 797   case Attribute::NoReturn:
 798     return bitc::ATTR_KIND_NO_RETURN;
 799   case Attribute::NoSync:
 800     return bitc::ATTR_KIND_NOSYNC;
 801   case Attribute::NoCfCheck:
 802     return bitc::ATTR_KIND_NOCF_CHECK;
 803   case Attribute::NoProfile:
 804     return bitc::ATTR_KIND_NO_PROFILE;
 805   case Attribute::SkipProfile:
 806     return bitc::ATTR_KIND_SKIP_PROFILE;
 807   case Attribute::NoUnwind:
 808     return bitc::ATTR_KIND_NO_UNWIND;
 809   case Attribute::NoSanitizeBounds:
 810     return bitc::ATTR_KIND_NO_SANITIZE_BOUNDS;
 811   case Attribute::NoSanitizeCoverage:
 812     return bitc::ATTR_KIND_NO_SANITIZE_COVERAGE;
 813   case Attribute::NullPointerIsValid:
 814     return bitc::ATTR_KIND_NULL_POINTER_IS_VALID;
 815   case Attribute::OptimizeForDebugging:
 816     return bitc::ATTR_KIND_OPTIMIZE_FOR_DEBUGGING;
 817   case Attribute::OptForFuzzing:
 818     return bitc::ATTR_KIND_OPT_FOR_FUZZING;
 819   case Attribute::OptimizeForSize:
 820     return bitc::ATTR_KIND_OPTIMIZE_FOR_SIZE;
 821   case Attribute::OptimizeNone:
 822     return bitc::ATTR_KIND_OPTIMIZE_NONE;
 823   case Attribute::ReadNone:
 824     return bitc::ATTR_KIND_READ_NONE;
 825   case Attribute::ReadOnly:
 826     return bitc::ATTR_KIND_READ_ONLY;
 827   case Attribute::Returned:
 828     return bitc::ATTR_KIND_RETURNED;
 829   case Attribute::ReturnsTwice:
 830     return bitc::ATTR_KIND_RETURNS_TWICE;
 831   case Attribute::SExt:
 832     return bitc::ATTR_KIND_S_EXT;
 833   case Attribute::Speculatable:
 834     return bitc::ATTR_KIND_SPECULATABLE;
 835   case Attribute::StackAlignment:
 836     return bitc::ATTR_KIND_STACK_ALIGNMENT;
 837   case Attribute::StackProtect:
 838     return bitc::ATTR_KIND_STACK_PROTECT;
 839   case Attribute::StackProtectReq:
 840     return bitc::ATTR_KIND_STACK_PROTECT_REQ;
 841   case Attribute::StackProtectStrong:
 842     return bitc::ATTR_KIND_STACK_PROTECT_STRONG;
 843   case Attribute::SafeStack:
 844     return bitc::ATTR_KIND_SAFESTACK;
 845   case Attribute::ShadowCallStack:
 846     return bitc::ATTR_KIND_SHADOWCALLSTACK;
 847   case Attribute::StrictFP:
 848     return bitc::ATTR_KIND_STRICT_FP;
 849   case Attribute::StructRet:
 850     return bitc::ATTR_KIND_STRUCT_RET;
 851   case Attribute::SanitizeAddress:
 852     return bitc::ATTR_KIND_SANITIZE_ADDRESS;
 853   case Attribute::SanitizeHWAddress:
 854     return bitc::ATTR_KIND_SANITIZE_HWADDRESS;
 855   case Attribute::SanitizeThread:
 856     return bitc::ATTR_KIND_SANITIZE_THREAD;
 857   case Attribute::SanitizeType:
 858     return bitc::ATTR_KIND_SANITIZE_TYPE;
 859   case Attribute::SanitizeMemory:
 860     return bitc::ATTR_KIND_SANITIZE_MEMORY;
 861   case Attribute::SanitizeNumericalStability:
 862     return bitc::ATTR_KIND_SANITIZE_NUMERICAL_STABILITY;
 863   case Attribute::SanitizeRealtime:
 864     return bitc::ATTR_KIND_SANITIZE_REALTIME;
 865   case Attribute::SanitizeRealtimeBlocking:
 866     return bitc::ATTR_KIND_SANITIZE_REALTIME_BLOCKING;
 867   case Attribute::SpeculativeLoadHardening:
 868     return bitc::ATTR_KIND_SPECULATIVE_LOAD_HARDENING;
 869   case Attribute::SwiftError:
 870     return bitc::ATTR_KIND_SWIFT_ERROR;
 871   case Attribute::SwiftSelf:
 872     return bitc::ATTR_KIND_SWIFT_SELF;
 873   case Attribute::SwiftAsync:
 874     return bitc::ATTR_KIND_SWIFT_ASYNC;
 875   case Attribute::UWTable:
 876     return bitc::ATTR_KIND_UW_TABLE;
 877   case Attribute::VScaleRange:
 878     return bitc::ATTR_KIND_VSCALE_RANGE;
 879   case Attribute::WillReturn:
 880     return bitc::ATTR_KIND_WILLRETURN;
 881   case Attribute::WriteOnly:
 882     return bitc::ATTR_KIND_WRITEONLY;
 883   case Attribute::ZExt:
 884     return bitc::ATTR_KIND_Z_EXT;
 885   case Attribute::ImmArg:
 886     return bitc::ATTR_KIND_IMMARG;
 887   case Attribute::SanitizeMemTag:
 888     return bitc::ATTR_KIND_SANITIZE_MEMTAG;
 889   case Attribute::Preallocated:
 890     return bitc::ATTR_KIND_PREALLOCATED;
 891   case Attribute::NoUndef:
 892     return bitc::ATTR_KIND_NOUNDEF;
 893   case Attribute::ByRef:
 894     return bitc::ATTR_KIND_BYREF;
 895   case Attribute::MustProgress:
 896     return bitc::ATTR_KIND_MUSTPROGRESS;
 897   case Attribute::PresplitCoroutine:
 898     return bitc::ATTR_KIND_PRESPLIT_COROUTINE;
 899   case Attribute::Writable:
 900     return bitc::ATTR_KIND_WRITABLE;
 901   case Attribute::CoroDestroyOnlyWhenComplete:
 902     return bitc::ATTR_KIND_CORO_ONLY_DESTROY_WHEN_COMPLETE;
 903   case Attribute::CoroElideSafe:
 904     return bitc::ATTR_KIND_CORO_ELIDE_SAFE;
 905   case Attribute::DeadOnUnwind:
 906     return bitc::ATTR_KIND_DEAD_ON_UNWIND;
 907   case Attribute::Range:
 908     return bitc::ATTR_KIND_RANGE;
 909   case Attribute::Initializes:
 910     return bitc::ATTR_KIND_INITIALIZES;
 911   case Attribute::NoExt:
 912     return bitc::ATTR_KIND_NO_EXT;
 913   case Attribute::Captures:
 914     return bitc::ATTR_KIND_CAPTURES;
 915   case Attribute::EndAttrKinds:
 916     llvm_unreachable("Can not encode end-attribute kinds marker.");
 917   case Attribute::None:
 918     llvm_unreachable("Can not encode none-attribute.");
 919   case Attribute::EmptyKey:
 920   case Attribute::TombstoneKey:
 921     llvm_unreachable("Trying to encode EmptyKey/TombstoneKey");
 922   }
 923
 924   llvm_unreachable("Trying to encode unknown attribute");
 925 }
 926
 927 static void emitSignedInt64(SmallVectorImpl<uint64_t> &Vals, uint64_t V) {
 928   if ((int64_t)V >= 0)
 929     Vals.push_back(V << 1);
 930   else
 931     Vals.push_back((-V << 1) | 1);
 932 }
 933
 934 static void emitWideAPInt(SmallVectorImpl<uint64_t> &Vals, const APInt &A) {
 935   // We have an arbitrary precision integer value to write whose
 936   // bit width is > 64. However, in canonical unsigned integer
 937   // format it is likely that the high bits are going to be zero.
 938   // So, we only write the number of active words.
 939   unsigned NumWords = A.getActiveWords();
 940   const uint64_t *RawData = A.getRawData();
 941   for (unsigned i = 0; i < NumWords; i++)
 942     emitSignedInt64(Vals, RawData[i]);
 943 }
 944
 945 static void emitConstantRange(SmallVectorImpl<uint64_t> &Record,
 946                               const ConstantRange &CR, bool EmitBitWidth) {
 947   unsigned BitWidth = CR.getBitWidth();
 948   if (EmitBitWidth)
 949     Record.push_back(BitWidth);
 950   if (BitWidth > 64) {
 951     Record.push_back(CR.getLower().getActiveWords() |
 952                      (uint64_t(CR.getUpper().getActiveWords()) << 32));
 953     emitWideAPInt(Record, CR.getLower());
 954     emitWideAPInt(Record, CR.getUpper());
 955   } else {
 956     emitSignedInt64(Record, CR.getLower().getSExtValue());
 957     emitSignedInt64(Record, CR.getUpper().getSExtValue());
 958   }
 959 }
 960
 961 void ModuleBitcodeWriter::writeAttributeGroupTable() {
 962   const std::vector<ValueEnumerator::IndexAndAttrSet> &AttrGrps =
 963       VE.getAttributeGroups();
 964   if (AttrGrps.empty()) return;
 965
 966   Stream.EnterSubblock(bitc::PARAMATTR_GROUP_BLOCK_ID, 3);
 967
 968   SmallVector<uint64_t, 64> Record;
 969   for (ValueEnumerator::IndexAndAttrSet Pair : AttrGrps) {
 970     unsigned AttrListIndex = Pair.first;
 971     AttributeSet AS = Pair.second;
 972     Record.push_back(VE.getAttributeGroupID(Pair));
 973     Record.push_back(AttrListIndex);
 974
 975     for (Attribute Attr : AS) {
 976       if (Attr.isEnumAttribute()) {
 977         Record.push_back(0);
 978         Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum()));
 979       } else if (Attr.isIntAttribute()) {
 980         Record.push_back(1);
 981         Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum()));
 982         Record.push_back(Attr.getValueAsInt());
 983       } else if (Attr.isStringAttribute()) {
 984         StringRef Kind = Attr.getKindAsString();
 985         StringRef Val = Attr.getValueAsString();
 986
 987         Record.push_back(Val.empty() ? 3 : 4);
 988         Record.append(Kind.begin(), Kind.end());
 989         Record.push_back(0);
 990         if (!Val.empty()) {
 991           Record.append(Val.begin(), Val.end());
 992           Record.push_back(0);
 993         }
 994       } else if (Attr.isTypeAttribute()) {
 995         Type *Ty = Attr.getValueAsType();
 996         Record.push_back(Ty ? 6 : 5);
 997         Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum()));
 998         if (Ty)
 999           Record.push_back(VE.getTypeID(Attr.getValueAsType()));
1000       } else if (Attr.isConstantRangeAttribute()) {
1001         Record.push_back(7);
1002         Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum()));
1003         emitConstantRange(Record, Attr.getValueAsConstantRange(),
1004                           /*EmitBitWidth=*/true);
1005       } else {
1006         assert(Attr.isConstantRangeListAttribute());
1007         Record.push_back(8);
1008         Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum()));
1009         ArrayRef<ConstantRange> Val = Attr.getValueAsConstantRangeList();
1010         Record.push_back(Val.size());
1011         Record.push_back(Val[0].getBitWidth());
1012         for (auto &CR : Val)
1013           emitConstantRange(Record, CR, /*EmitBitWidth=*/false);
1014       }
1015     }
1016
1017     Stream.EmitRecord(bitc::PARAMATTR_GRP_CODE_ENTRY, Record);
1018     Record.clear();
1019   }
1020
1021   Stream.ExitBlock();
1022 }
1023
1024 void ModuleBitcodeWriter::writeAttributeTable() {
1025   const std::vector<AttributeList> &Attrs = VE.getAttributeLists();
1026   if (Attrs.empty()) return;
1027
1028   Stream.EnterSubblock(bitc::PARAMATTR_BLOCK_ID, 3);
1029
1030   SmallVector<uint64_t, 64> Record;
1031   for (const AttributeList &AL : Attrs) {
1032     for (unsigned i : AL.indexes()) {
1033       AttributeSet AS = AL.getAttributes(i);
1034       if (AS.hasAttributes())
1035         Record.push_back(VE.getAttributeGroupID({i, AS}));
1036     }
1037
1038     Stream.EmitRecord(bitc::PARAMATTR_CODE_ENTRY, Record);
1039     Record.clear();
1040   }
1041
1042   Stream.ExitBlock();
1043 }
1044
1045 /// WriteTypeTable - Write out the type table for a module.
1046 void ModuleBitcodeWriter::writeTypeTable() {
1047   const ValueEnumerator::TypeList &TypeList = VE.getTypes();
1048
1049   Stream.EnterSubblock(bitc::TYPE_BLOCK_ID_NEW, 4 /*count from # abbrevs */);
1050   SmallVector<uint64_t, 64> TypeVals;
1051
1052   uint64_t NumBits = VE.computeBitsRequiredForTypeIndices();
1053
1054   // Abbrev for TYPE_CODE_OPAQUE_POINTER.
1055   auto Abbv = std::make_shared<BitCodeAbbrev>();
1056   Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_OPAQUE_POINTER));
1057   Abbv->Add(BitCodeAbbrevOp(0)); // Addrspace = 0
1058   unsigned OpaquePtrAbbrev = Stream.EmitAbbrev(std::move(Abbv));
1059
1060   // Abbrev for TYPE_CODE_FUNCTION.
1061   Abbv = std::make_shared<BitCodeAbbrev>();
1062   Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_FUNCTION));
1063   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));  // isvararg
1064   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1065   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1066   unsigned FunctionAbbrev = Stream.EmitAbbrev(std::move(Abbv));
1067
1068   // Abbrev for TYPE_CODE_STRUCT_ANON.
1069   Abbv = std::make_shared<BitCodeAbbrev>();
1070   Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_ANON));
1071   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));  // ispacked
1072   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1073   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1074   unsigned StructAnonAbbrev = Stream.EmitAbbrev(std::move(Abbv));
1075
1076   // Abbrev for TYPE_CODE_STRUCT_NAME.
1077   Abbv = std::make_shared<BitCodeAbbrev>();
1078   Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAME));
1079   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1080   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
1081   unsigned StructNameAbbrev = Stream.EmitAbbrev(std::move(Abbv));
1082
1083   // Abbrev for TYPE_CODE_STRUCT_NAMED.
1084   Abbv = std::make_shared<BitCodeAbbrev>();
1085   Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAMED));
1086   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));  // ispacked
1087   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1088   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1089   unsigned StructNamedAbbrev = Stream.EmitAbbrev(std::move(Abbv));
1090
1091   // Abbrev for TYPE_CODE_ARRAY.
1092   Abbv = std::make_shared<BitCodeAbbrev>();
1093   Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_ARRAY));
1094   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // size
1095   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1096   unsigned ArrayAbbrev = Stream.EmitAbbrev(std::move(Abbv));
1097
1098   // Emit an entry count so the reader can reserve space.
1099   TypeVals.push_back(TypeList.size());
1100   Stream.EmitRecord(bitc::TYPE_CODE_NUMENTRY, TypeVals);
1101   TypeVals.clear();
1102
1103   // Loop over all of the types, emitting each in turn.
1104   for (Type *T : TypeList) {
1105     int AbbrevToUse = 0;
1106     unsigned Code = 0;
1107
1108     switch (T->getTypeID()) {
1109     case Type::VoidTyID:      Code = bitc::TYPE_CODE_VOID;      break;
1110     case Type::HalfTyID:      Code = bitc::TYPE_CODE_HALF;      break;
1111     case Type::BFloatTyID:    Code = bitc::TYPE_CODE_BFLOAT;    break;
1112     case Type::FloatTyID:     Code = bitc::TYPE_CODE_FLOAT;     break;
1113     case Type::DoubleTyID:    Code = bitc::TYPE_CODE_DOUBLE;    break;
1114     case Type::X86_FP80TyID:  Code = bitc::TYPE_CODE_X86_FP80;  break;
1115     case Type::FP128TyID:     Code = bitc::TYPE_CODE_FP128;     break;
1116     case Type::PPC_FP128TyID: Code = bitc::TYPE_CODE_PPC_FP128; break;
1117     case Type::LabelTyID:     Code = bitc::TYPE_CODE_LABEL;     break;
1118     case Type::MetadataTyID:
1119       Code = bitc::TYPE_CODE_METADATA;
1120       break;
1121     case Type::X86_AMXTyID:   Code = bitc::TYPE_CODE_X86_AMX;   break;
1122     case Type::TokenTyID:     Code = bitc::TYPE_CODE_TOKEN;     break;
1123     case Type::IntegerTyID:
1124       // INTEGER: [width]
1125       Code = bitc::TYPE_CODE_INTEGER;
1126       TypeVals.push_back(cast<IntegerType>(T)->getBitWidth());
1127       break;
1128     case Type::PointerTyID: {
1129       PointerType *PTy = cast<PointerType>(T);
1130       unsigned AddressSpace = PTy->getAddressSpace();
1131       // OPAQUE_POINTER: [address space]
1132       Code = bitc::TYPE_CODE_OPAQUE_POINTER;
1133       TypeVals.push_back(AddressSpace);
1134       if (AddressSpace == 0)
1135         AbbrevToUse = OpaquePtrAbbrev;
1136       break;
1137     }
1138     case Type::FunctionTyID: {
1139       FunctionType *FT = cast<FunctionType>(T);
1140       // FUNCTION: [isvararg, retty, paramty x N]
1141       Code = bitc::TYPE_CODE_FUNCTION;
1142       TypeVals.push_back(FT->isVarArg());
1143       TypeVals.push_back(VE.getTypeID(FT->getReturnType()));
1144       for (unsigned i = 0, e = FT->getNumParams(); i != e; ++i)
1145         TypeVals.push_back(VE.getTypeID(FT->getParamType(i)));
1146       AbbrevToUse = FunctionAbbrev;
1147       break;
1148     }
1149     case Type::StructTyID: {
1150       StructType *ST = cast<StructType>(T);
1151       // STRUCT: [ispacked, eltty x N]
1152       TypeVals.push_back(ST->isPacked());
1153       // Output all of the element types.
1154       for (Type *ET : ST->elements())
1155         TypeVals.push_back(VE.getTypeID(ET));
1156
1157       if (ST->isLiteral()) {
1158         Code = bitc::TYPE_CODE_STRUCT_ANON;
1159         AbbrevToUse = StructAnonAbbrev;
1160       } else {
1161         if (ST->isOpaque()) {
1162           Code = bitc::TYPE_CODE_OPAQUE;
1163         } else {
1164           Code = bitc::TYPE_CODE_STRUCT_NAMED;
1165           AbbrevToUse = StructNamedAbbrev;
1166         }
1167
1168         // Emit the name if it is present.
1169         if (!ST->getName().empty())
1170           writeStringRecord(Stream, bitc::TYPE_CODE_STRUCT_NAME, ST->getName(),
1171                             StructNameAbbrev);
1172       }
1173       break;
1174     }
1175     case Type::ArrayTyID: {
1176       ArrayType *AT = cast<ArrayType>(T);
1177       // ARRAY: [numelts, eltty]
1178       Code = bitc::TYPE_CODE_ARRAY;
1179       TypeVals.push_back(AT->getNumElements());
1180       TypeVals.push_back(VE.getTypeID(AT->getElementType()));
1181       AbbrevToUse = ArrayAbbrev;
1182       break;
1183     }
1184     case Type::FixedVectorTyID:
1185     case Type::ScalableVectorTyID: {
1186       VectorType *VT = cast<VectorType>(T);
1187       // VECTOR [numelts, eltty] or
1188       //        [numelts, eltty, scalable]
1189       Code = bitc::TYPE_CODE_VECTOR;
1190       TypeVals.push_back(VT->getElementCount().getKnownMinValue());
1191       TypeVals.push_back(VE.getTypeID(VT->getElementType()));
1192       if (isa<ScalableVectorType>(VT))
1193         TypeVals.push_back(true);
1194       break;
1195     }
1196     case Type::TargetExtTyID: {
1197       TargetExtType *TET = cast<TargetExtType>(T);
1198       Code = bitc::TYPE_CODE_TARGET_TYPE;
1199       writeStringRecord(Stream, bitc::TYPE_CODE_STRUCT_NAME, TET->getName(),
1200                         StructNameAbbrev);
1201       TypeVals.push_back(TET->getNumTypeParameters());
1202       for (Type *InnerTy : TET->type_params())
1203         TypeVals.push_back(VE.getTypeID(InnerTy));
1204       for (unsigned IntParam : TET->int_params())
1205         TypeVals.push_back(IntParam);
1206       break;
1207     }
1208     case Type::TypedPointerTyID:
1209       llvm_unreachable("Typed pointers cannot be added to IR modules");
1210     }
1211
1212     // Emit the finished record.
1213     Stream.EmitRecord(Code, TypeVals, AbbrevToUse);
1214     TypeVals.clear();
1215   }
1216
1217   Stream.ExitBlock();
1218 }
1219
1220 static unsigned getEncodedLinkage(const GlobalValue::LinkageTypes Linkage) {
1221   switch (Linkage) {
1222   case GlobalValue::ExternalLinkage:
1223     return 0;
1224   case GlobalValue::WeakAnyLinkage:
1225     return 16;
1226   case GlobalValue::AppendingLinkage:
1227     return 2;
1228   case GlobalValue::InternalLinkage:
1229     return 3;
1230   case GlobalValue::LinkOnceAnyLinkage:
1231     return 18;
1232   case GlobalValue::ExternalWeakLinkage:
1233     return 7;
1234   case GlobalValue::CommonLinkage:
1235     return 8;
1236   case GlobalValue::PrivateLinkage:
1237     return 9;
1238   case GlobalValue::WeakODRLinkage:
1239     return 17;
1240   case GlobalValue::LinkOnceODRLinkage:
1241     return 19;
1242   case GlobalValue::AvailableExternallyLinkage:
1243     return 12;
1244   }
1245   llvm_unreachable("Invalid linkage");
1246 }
1247
1248 static unsigned getEncodedLinkage(const GlobalValue &GV) {
1249   return getEncodedLinkage(GV.getLinkage());
1250 }
1251
1252 static uint64_t getEncodedFFlags(FunctionSummary::FFlags Flags) {
1253   uint64_t RawFlags = 0;
1254   RawFlags |= Flags.ReadNone;
1255   RawFlags |= (Flags.ReadOnly << 1);
1256   RawFlags |= (Flags.NoRecurse << 2);
1257   RawFlags |= (Flags.ReturnDoesNotAlias << 3);
1258   RawFlags |= (Flags.NoInline << 4);
1259   RawFlags |= (Flags.AlwaysInline << 5);
1260   RawFlags |= (Flags.NoUnwind << 6);
1261   RawFlags |= (Flags.MayThrow << 7);
1262   RawFlags |= (Flags.HasUnknownCall << 8);
1263   RawFlags |= (Flags.MustBeUnreachable << 9);
1264   return RawFlags;
1265 }
1266
1267 // Decode the flags for GlobalValue in the summary. See getDecodedGVSummaryFlags
1268 // in BitcodeReader.cpp.
1269 static uint64_t getEncodedGVSummaryFlags(GlobalValueSummary::GVFlags Flags,
1270                                          bool ImportAsDecl = false) {
1271   uint64_t RawFlags = 0;
1272
1273   RawFlags |= Flags.NotEligibleToImport; // bool
1274   RawFlags |= (Flags.Live << 1);
1275   RawFlags |= (Flags.DSOLocal << 2);
1276   RawFlags |= (Flags.CanAutoHide << 3);
1277
1278   // Linkage don't need to be remapped at that time for the summary. Any future
1279   // change to the getEncodedLinkage() function will need to be taken into
1280   // account here as well.
1281   RawFlags = (RawFlags << 4) | Flags.Linkage; // 4 bits
1282
1283   RawFlags |= (Flags.Visibility << 8); // 2 bits
1284
1285   unsigned ImportType = Flags.ImportType | ImportAsDecl;
1286   RawFlags |= (ImportType << 10); // 1 bit
1287
1288   return RawFlags;
1289 }
1290
1291 static uint64_t getEncodedGVarFlags(GlobalVarSummary::GVarFlags Flags) {
1292   uint64_t RawFlags = Flags.MaybeReadOnly | (Flags.MaybeWriteOnly << 1) |
1293                       (Flags.Constant << 2) | Flags.VCallVisibility << 3;
1294   return RawFlags;
1295 }
1296
1297 static uint64_t getEncodedHotnessCallEdgeInfo(const CalleeInfo &CI) {
1298   uint64_t RawFlags = 0;
1299
1300   RawFlags |= CI.Hotness;            // 3 bits
1301   RawFlags |= (CI.HasTailCall << 3); // 1 bit
1302
1303   return RawFlags;
1304 }
1305
1306 static uint64_t getEncodedRelBFCallEdgeInfo(const CalleeInfo &CI) {
1307   uint64_t RawFlags = 0;
1308
1309   RawFlags |= CI.RelBlockFreq; // CalleeInfo::RelBlockFreqBits bits
1310   RawFlags |= (CI.HasTailCall << CalleeInfo::RelBlockFreqBits); // 1 bit
1311
1312   return RawFlags;
1313 }
1314
1315 static unsigned getEncodedVisibility(const GlobalValue &GV) {
1316   switch (GV.getVisibility()) {
1317   case GlobalValue::DefaultVisibility:   return 0;
1318   case GlobalValue::HiddenVisibility:    return 1;
1319   case GlobalValue::ProtectedVisibility: return 2;
1320   }
1321   llvm_unreachable("Invalid visibility");
1322 }
1323
1324 static unsigned getEncodedDLLStorageClass(const GlobalValue &GV) {
1325   switch (GV.getDLLStorageClass()) {
1326   case GlobalValue::DefaultStorageClass:   return 0;
1327   case GlobalValue::DLLImportStorageClass: return 1;
1328   case GlobalValue::DLLExportStorageClass: return 2;
1329   }
1330   llvm_unreachable("Invalid DLL storage class");
1331 }
1332
1333 static unsigned getEncodedThreadLocalMode(const GlobalValue &GV) {
1334   switch (GV.getThreadLocalMode()) {
1335     case GlobalVariable::NotThreadLocal:         return 0;
1336     case GlobalVariable::GeneralDynamicTLSModel: return 1;
1337     case GlobalVariable::LocalDynamicTLSModel:   return 2;
1338     case GlobalVariable::InitialExecTLSModel:    return 3;
1339     case GlobalVariable::LocalExecTLSModel:      return 4;
1340   }
1341   llvm_unreachable("Invalid TLS model");
1342 }
1343
1344 static unsigned getEncodedComdatSelectionKind(const Comdat &C) {
1345   switch (C.getSelectionKind()) {
1346   case Comdat::Any:
1347     return bitc::COMDAT_SELECTION_KIND_ANY;
1348   case Comdat::ExactMatch:
1349     return bitc::COMDAT_SELECTION_KIND_EXACT_MATCH;
1350   case Comdat::Largest:
1351     return bitc::COMDAT_SELECTION_KIND_LARGEST;
1352   case Comdat::NoDeduplicate:
1353     return bitc::COMDAT_SELECTION_KIND_NO_DUPLICATES;
1354   case Comdat::SameSize:
1355     return bitc::COMDAT_SELECTION_KIND_SAME_SIZE;
1356   }
1357   llvm_unreachable("Invalid selection kind");
1358 }
1359
1360 static unsigned getEncodedUnnamedAddr(const GlobalValue &GV) {
1361   switch (GV.getUnnamedAddr()) {
1362   case GlobalValue::UnnamedAddr::None:   return 0;
1363   case GlobalValue::UnnamedAddr::Local:  return 2;
1364   case GlobalValue::UnnamedAddr::Global: return 1;
1365   }
1366   llvm_unreachable("Invalid unnamed_addr");
1367 }
1368
1369 size_t ModuleBitcodeWriter::addToStrtab(StringRef Str) {
1370   if (GenerateHash)
1371     Hasher.update(Str);
1372   return StrtabBuilder.add(Str);
1373 }
1374
1375 void ModuleBitcodeWriter::writeComdats() {
1376   SmallVector<unsigned, 64> Vals;
1377   for (const Comdat *C : VE.getComdats()) {
1378     // COMDAT: [strtab offset, strtab size, selection_kind]
1379     Vals.push_back(addToStrtab(C->getName()));
1380     Vals.push_back(C->getName().size());
1381     Vals.push_back(getEncodedComdatSelectionKind(*C));
1382     Stream.EmitRecord(bitc::MODULE_CODE_COMDAT, Vals, /*AbbrevToUse=*/0);
1383     Vals.clear();
1384   }
1385 }
1386
1387 /// Write a record that will eventually hold the word offset of the
1388 /// module-level VST. For now the offset is 0, which will be backpatched
1389 /// after the real VST is written. Saves the bit offset to backpatch.
1390 void ModuleBitcodeWriter::writeValueSymbolTableForwardDecl() {
1391   // Write a placeholder value in for the offset of the real VST,
1392   // which is written after the function blocks so that it can include
1393   // the offset of each function. The placeholder offset will be
1394   // updated when the real VST is written.
1395   auto Abbv = std::make_shared<BitCodeAbbrev>();
1396   Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_VSTOFFSET));
1397   // Blocks are 32-bit aligned, so we can use a 32-bit word offset to
1398   // hold the real VST offset. Must use fixed instead of VBR as we don't
1399   // know how many VBR chunks to reserve ahead of time.
1400   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
1401   unsigned VSTOffsetAbbrev = Stream.EmitAbbrev(std::move(Abbv));
1402
1403   // Emit the placeholder
1404   uint64_t Vals[] = {bitc::MODULE_CODE_VSTOFFSET, 0};
1405   Stream.EmitRecordWithAbbrev(VSTOffsetAbbrev, Vals);
1406
1407   // Compute and save the bit offset to the placeholder, which will be
1408   // patched when the real VST is written. We can simply subtract the 32-bit
1409   // fixed size from the current bit number to get the location to backpatch.
1410   VSTOffsetPlaceholder = Stream.GetCurrentBitNo() - 32;
1411 }
1412
1413 enum StringEncoding { SE_Char6, SE_Fixed7, SE_Fixed8 };
1414
1415 /// Determine the encoding to use for the given string name and length.
1416 static StringEncoding getStringEncoding(StringRef Str) {
1417   bool isChar6 = true;
1418   for (char C : Str) {
1419     if (isChar6)
1420       isChar6 = BitCodeAbbrevOp::isChar6(C);
1421     if ((unsigned char)C & 128)
1422       // don't bother scanning the rest.
1423       return SE_Fixed8;
1424   }
1425   if (isChar6)
1426     return SE_Char6;
1427   return SE_Fixed7;
1428 }
1429
1430 static_assert(sizeof(GlobalValue::SanitizerMetadata) <= sizeof(unsigned),
1431               "Sanitizer Metadata is too large for naive serialization.");
1432 static unsigned
1433 serializeSanitizerMetadata(const GlobalValue::SanitizerMetadata &Meta) {
1434   return Meta.NoAddress | (Meta.NoHWAddress << 1) |
1435          (Meta.Memtag << 2) | (Meta.IsDynInit << 3);
1436 }
1437
1438 /// Emit top-level description of module, including target triple, inline asm,
1439 /// descriptors for global variables, and function prototype info.
1440 /// Returns the bit offset to backpatch with the location of the real VST.
1441 void ModuleBitcodeWriter::writeModuleInfo() {
1442   // Emit various pieces of data attached to a module.
1443   if (!M.getTargetTriple().empty())
1444     writeStringRecord(Stream, bitc::MODULE_CODE_TRIPLE, M.getTargetTriple(),
1445                       0 /*TODO*/);
1446   const std::string &DL = M.getDataLayoutStr();
1447   if (!DL.empty())
1448     writeStringRecord(Stream, bitc::MODULE_CODE_DATALAYOUT, DL, 0 /*TODO*/);
1449   if (!M.getModuleInlineAsm().empty())
1450     writeStringRecord(Stream, bitc::MODULE_CODE_ASM, M.getModuleInlineAsm(),
1451                       0 /*TODO*/);
1452
1453   // Emit information about sections and GC, computing how many there are. Also
1454   // compute the maximum alignment value.
1455   std::map<std::string, unsigned> SectionMap;
1456   std::map<std::string, unsigned> GCMap;
1457   MaybeAlign MaxAlignment;
1458   unsigned MaxGlobalType = 0;
1459   const auto UpdateMaxAlignment = [&MaxAlignment](const MaybeAlign A) {
1460     if (A)
1461       MaxAlignment = !MaxAlignment ? *A : std::max(*MaxAlignment, *A);
1462   };
1463   for (const GlobalVariable &GV : M.globals()) {
1464     UpdateMaxAlignment(GV.getAlign());
1465     MaxGlobalType = std::max(MaxGlobalType, VE.getTypeID(GV.getValueType()));
1466     if (GV.hasSection()) {
1467       // Give section names unique ID's.
1468       unsigned &Entry = SectionMap[std::string(GV.getSection())];
1469       if (!Entry) {
1470         writeStringRecord(Stream, bitc::MODULE_CODE_SECTIONNAME, GV.getSection(),
1471                           0 /*TODO*/);
1472         Entry = SectionMap.size();
1473       }
1474     }
1475   }
1476   for (const Function &F : M) {
1477     UpdateMaxAlignment(F.getAlign());
1478     if (F.hasSection()) {
1479       // Give section names unique ID's.
1480       unsigned &Entry = SectionMap[std::string(F.getSection())];
1481       if (!Entry) {
1482         writeStringRecord(Stream, bitc::MODULE_CODE_SECTIONNAME, F.getSection(),
1483                           0 /*TODO*/);
1484         Entry = SectionMap.size();
1485       }
1486     }
1487     if (F.hasGC()) {
1488       // Same for GC names.
1489       unsigned &Entry = GCMap[F.getGC()];
1490       if (!Entry) {
1491         writeStringRecord(Stream, bitc::MODULE_CODE_GCNAME, F.getGC(),
1492                           0 /*TODO*/);
1493         Entry = GCMap.size();
1494       }
1495     }
1496   }
1497
1498   // Emit abbrev for globals, now that we know # sections and max alignment.
1499   unsigned SimpleGVarAbbrev = 0;
1500   if (!M.global_empty()) {
1501     // Add an abbrev for common globals with no visibility or thread localness.
1502     auto Abbv = std::make_shared<BitCodeAbbrev>();
1503     Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_GLOBALVAR));
1504     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1505     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1506     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1507                               Log2_32_Ceil(MaxGlobalType+1)));
1508     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // AddrSpace << 2
1509                                                            //| explicitType << 1
1510                                                            //| constant
1511     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // Initializer.
1512     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // Linkage.
1513     if (!MaxAlignment)                                     // Alignment.
1514       Abbv->Add(BitCodeAbbrevOp(0));
1515     else {
1516       unsigned MaxEncAlignment = getEncodedAlign(MaxAlignment);
1517       Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1518                                Log2_32_Ceil(MaxEncAlignment+1)));
1519     }
1520     if (SectionMap.empty())                                    // Section.
1521       Abbv->Add(BitCodeAbbrevOp(0));
1522     else
1523       Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1524                                Log2_32_Ceil(SectionMap.size()+1)));
1525     // Don't bother emitting vis + thread local.
1526     SimpleGVarAbbrev = Stream.EmitAbbrev(std::move(Abbv));
1527   }
1528
1529   SmallVector<unsigned, 64> Vals;
1530   // Emit the module's source file name.
1531   {
1532     StringEncoding Bits = getStringEncoding(M.getSourceFileName());
1533     BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8);
1534     if (Bits == SE_Char6)
1535       AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6);
1536     else if (Bits == SE_Fixed7)
1537       AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7);
1538
1539     // MODULE_CODE_SOURCE_FILENAME: [namechar x N]
1540     auto Abbv = std::make_shared<BitCodeAbbrev>();
1541     Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME));
1542     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1543     Abbv->Add(AbbrevOpToUse);
1544     unsigned FilenameAbbrev = Stream.EmitAbbrev(std::move(Abbv));
1545
1546     for (const auto P : M.getSourceFileName())
1547       Vals.push_back((unsigned char)P);
1548
1549     // Emit the finished record.
1550     Stream.EmitRecord(bitc::MODULE_CODE_SOURCE_FILENAME, Vals, FilenameAbbrev);
1551     Vals.clear();
1552   }
1553
1554   // Emit the global variable information.
1555   for (const GlobalVariable &GV : M.globals()) {
1556     unsigned AbbrevToUse = 0;
1557
1558     // GLOBALVAR: [strtab offset, strtab size, type, isconst, initid,
1559     //             linkage, alignment, section, visibility, threadlocal,
1560     //             unnamed_addr, externally_initialized, dllstorageclass,
1561     //             comdat, attributes, DSO_Local, GlobalSanitizer, code_model]
1562     Vals.push_back(addToStrtab(GV.getName()));
1563     Vals.push_back(GV.getName().size());
1564     Vals.push_back(VE.getTypeID(GV.getValueType()));
1565     Vals.push_back(GV.getType()->getAddressSpace() << 2 | 2 | GV.isConstant());
1566     Vals.push_back(GV.isDeclaration() ? 0 :
1567                    (VE.getValueID(GV.getInitializer()) + 1));
1568     Vals.push_back(getEncodedLinkage(GV));
1569     Vals.push_back(getEncodedAlign(GV.getAlign()));
1570     Vals.push_back(GV.hasSection() ? SectionMap[std::string(GV.getSection())]
1571                                    : 0);
1572     if (GV.isThreadLocal() ||
1573         GV.getVisibility() != GlobalValue::DefaultVisibility ||
1574         GV.getUnnamedAddr() != GlobalValue::UnnamedAddr::None ||
1575         GV.isExternallyInitialized() ||
1576         GV.getDLLStorageClass() != GlobalValue::DefaultStorageClass ||
1577         GV.hasComdat() || GV.hasAttributes() || GV.isDSOLocal() ||
1578         GV.hasPartition() || GV.hasSanitizerMetadata() || GV.getCodeModel()) {
1579       Vals.push_back(getEncodedVisibility(GV));
1580       Vals.push_back(getEncodedThreadLocalMode(GV));
1581       Vals.push_back(getEncodedUnnamedAddr(GV));
1582       Vals.push_back(GV.isExternallyInitialized());
1583       Vals.push_back(getEncodedDLLStorageClass(GV));
1584       Vals.push_back(GV.hasComdat() ? VE.getComdatID(GV.getComdat()) : 0);
1585
1586       auto AL = GV.getAttributesAsList(AttributeList::FunctionIndex);
1587       Vals.push_back(VE.getAttributeListID(AL));
1588
1589       Vals.push_back(GV.isDSOLocal());
1590       Vals.push_back(addToStrtab(GV.getPartition()));
1591       Vals.push_back(GV.getPartition().size());
1592
1593       Vals.push_back((GV.hasSanitizerMetadata() ? serializeSanitizerMetadata(
1594                                                       GV.getSanitizerMetadata())
1595                                                 : 0));
1596       Vals.push_back(GV.getCodeModelRaw());
1597     } else {
1598       AbbrevToUse = SimpleGVarAbbrev;
1599     }
1600
1601     Stream.EmitRecord(bitc::MODULE_CODE_GLOBALVAR, Vals, AbbrevToUse);
1602     Vals.clear();
1603   }
1604
1605   // Emit the function proto information.
1606   for (const Function &F : M) {
1607     // FUNCTION:  [strtab offset, strtab size, type, callingconv, isproto,
1608     //             linkage, paramattrs, alignment, section, visibility, gc,
1609     //             unnamed_addr, prologuedata, dllstorageclass, comdat,
1610     //             prefixdata, personalityfn, DSO_Local, addrspace]
1611     Vals.push_back(addToStrtab(F.getName()));
1612     Vals.push_back(F.getName().size());
1613     Vals.push_back(VE.getTypeID(F.getFunctionType()));
1614     Vals.push_back(F.getCallingConv());
1615     Vals.push_back(F.isDeclaration());
1616     Vals.push_back(getEncodedLinkage(F));
1617     Vals.push_back(VE.getAttributeListID(F.getAttributes()));
1618     Vals.push_back(getEncodedAlign(F.getAlign()));
1619     Vals.push_back(F.hasSection() ? SectionMap[std::string(F.getSection())]
1620                                   : 0);
1621     Vals.push_back(getEncodedVisibility(F));
1622     Vals.push_back(F.hasGC() ? GCMap[F.getGC()] : 0);
1623     Vals.push_back(getEncodedUnnamedAddr(F));
1624     Vals.push_back(F.hasPrologueData() ? (VE.getValueID(F.getPrologueData()) + 1)
1625                                        : 0);
1626     Vals.push_back(getEncodedDLLStorageClass(F));
1627     Vals.push_back(F.hasComdat() ? VE.getComdatID(F.getComdat()) : 0);
1628     Vals.push_back(F.hasPrefixData() ? (VE.getValueID(F.getPrefixData()) + 1)
1629                                      : 0);
1630     Vals.push_back(
1631         F.hasPersonalityFn() ? (VE.getValueID(F.getPersonalityFn()) + 1) : 0);
1632
1633     Vals.push_back(F.isDSOLocal());
1634     Vals.push_back(F.getAddressSpace());
1635     Vals.push_back(addToStrtab(F.getPartition()));
1636     Vals.push_back(F.getPartition().size());
1637
1638     unsigned AbbrevToUse = 0;
1639     Stream.EmitRecord(bitc::MODULE_CODE_FUNCTION, Vals, AbbrevToUse);
1640     Vals.clear();
1641   }
1642
1643   // Emit the alias information.
1644   for (const GlobalAlias &A : M.aliases()) {
1645     // ALIAS: [strtab offset, strtab size, alias type, aliasee val#, linkage,
1646     //         visibility, dllstorageclass, threadlocal, unnamed_addr,
1647     //         DSO_Local]
1648     Vals.push_back(addToStrtab(A.getName()));
1649     Vals.push_back(A.getName().size());
1650     Vals.push_back(VE.getTypeID(A.getValueType()));
1651     Vals.push_back(A.getType()->getAddressSpace());
1652     Vals.push_back(VE.getValueID(A.getAliasee()));
1653     Vals.push_back(getEncodedLinkage(A));
1654     Vals.push_back(getEncodedVisibility(A));
1655     Vals.push_back(getEncodedDLLStorageClass(A));
1656     Vals.push_back(getEncodedThreadLocalMode(A));
1657     Vals.push_back(getEncodedUnnamedAddr(A));
1658     Vals.push_back(A.isDSOLocal());
1659     Vals.push_back(addToStrtab(A.getPartition()));
1660     Vals.push_back(A.getPartition().size());
1661
1662     unsigned AbbrevToUse = 0;
1663     Stream.EmitRecord(bitc::MODULE_CODE_ALIAS, Vals, AbbrevToUse);
1664     Vals.clear();
1665   }
1666
1667   // Emit the ifunc information.
1668   for (const GlobalIFunc &I : M.ifuncs()) {
1669     // IFUNC: [strtab offset, strtab size, ifunc type, address space, resolver
1670     //         val#, linkage, visibility, DSO_Local]
1671     Vals.push_back(addToStrtab(I.getName()));
1672     Vals.push_back(I.getName().size());
1673     Vals.push_back(VE.getTypeID(I.getValueType()));
1674     Vals.push_back(I.getType()->getAddressSpace());
1675     Vals.push_back(VE.getValueID(I.getResolver()));
1676     Vals.push_back(getEncodedLinkage(I));
1677     Vals.push_back(getEncodedVisibility(I));
1678     Vals.push_back(I.isDSOLocal());
1679     Vals.push_back(addToStrtab(I.getPartition()));
1680     Vals.push_back(I.getPartition().size());
1681     Stream.EmitRecord(bitc::MODULE_CODE_IFUNC, Vals);
1682     Vals.clear();
1683   }
1684
1685   writeValueSymbolTableForwardDecl();
1686 }
1687
1688 static uint64_t getOptimizationFlags(const Value *V) {
1689   uint64_t Flags = 0;
1690
1691   if (const auto *OBO = dyn_cast<OverflowingBinaryOperator>(V)) {
1692     if (OBO->hasNoSignedWrap())
1693       Flags |= 1 << bitc::OBO_NO_SIGNED_WRAP;
1694     if (OBO->hasNoUnsignedWrap())
1695       Flags |= 1 << bitc::OBO_NO_UNSIGNED_WRAP;
1696   } else if (const auto *PEO = dyn_cast<PossiblyExactOperator>(V)) {
1697     if (PEO->isExact())
1698       Flags |= 1 << bitc::PEO_EXACT;
1699   } else if (const auto *PDI = dyn_cast<PossiblyDisjointInst>(V)) {
1700     if (PDI->isDisjoint())
1701       Flags |= 1 << bitc::PDI_DISJOINT;
1702   } else if (const auto *FPMO = dyn_cast<FPMathOperator>(V)) {
1703     if (FPMO->hasAllowReassoc())
1704       Flags |= bitc::AllowReassoc;
1705     if (FPMO->hasNoNaNs())
1706       Flags |= bitc::NoNaNs;
1707     if (FPMO->hasNoInfs())
1708       Flags |= bitc::NoInfs;
1709     if (FPMO->hasNoSignedZeros())
1710       Flags |= bitc::NoSignedZeros;
1711     if (FPMO->hasAllowReciprocal())
1712       Flags |= bitc::AllowReciprocal;
1713     if (FPMO->hasAllowContract())
1714       Flags |= bitc::AllowContract;
1715     if (FPMO->hasApproxFunc())
1716       Flags |= bitc::ApproxFunc;
1717   } else if (const auto *NNI = dyn_cast<PossiblyNonNegInst>(V)) {
1718     if (NNI->hasNonNeg())
1719       Flags |= 1 << bitc::PNNI_NON_NEG;
1720   } else if (const auto *TI = dyn_cast<TruncInst>(V)) {
1721     if (TI->hasNoSignedWrap())
1722       Flags |= 1 << bitc::TIO_NO_SIGNED_WRAP;
1723     if (TI->hasNoUnsignedWrap())
1724       Flags |= 1 << bitc::TIO_NO_UNSIGNED_WRAP;
1725   } else if (const auto *GEP = dyn_cast<GEPOperator>(V)) {
1726     if (GEP->isInBounds())
1727       Flags |= 1 << bitc::GEP_INBOUNDS;
1728     if (GEP->hasNoUnsignedSignedWrap())
1729       Flags |= 1 << bitc::GEP_NUSW;
1730     if (GEP->hasNoUnsignedWrap())
1731       Flags |= 1 << bitc::GEP_NUW;
1732   } else if (const auto *ICmp = dyn_cast<ICmpInst>(V)) {
1733     if (ICmp->hasSameSign())
1734       Flags |= 1 << bitc::ICMP_SAME_SIGN;
1735   }
1736
1737   return Flags;
1738 }
1739
1740 void ModuleBitcodeWriter::writeValueAsMetadata(
1741     const ValueAsMetadata *MD, SmallVectorImpl<uint64_t> &Record) {
1742   // Mimic an MDNode with a value as one operand.
1743   Value *V = MD->getValue();
1744   Record.push_back(VE.getTypeID(V->getType()));
1745   Record.push_back(VE.getValueID(V));
1746   Stream.EmitRecord(bitc::METADATA_VALUE, Record, 0);
1747   Record.clear();
1748 }
1749
1750 void ModuleBitcodeWriter::writeMDTuple(const MDTuple *N,
1751                                        SmallVectorImpl<uint64_t> &Record,
1752                                        unsigned Abbrev) {
1753   for (const MDOperand &MDO : N->operands()) {
1754     Metadata *MD = MDO;
1755     assert(!(MD && isa<LocalAsMetadata>(MD)) &&
1756            "Unexpected function-local metadata");
1757     Record.push_back(VE.getMetadataOrNullID(MD));
1758   }
1759   Stream.EmitRecord(N->isDistinct() ? bitc::METADATA_DISTINCT_NODE
1760                                     : bitc::METADATA_NODE,
1761                     Record, Abbrev);
1762   Record.clear();
1763 }
1764
1765 unsigned ModuleBitcodeWriter::createDILocationAbbrev() {
1766   // Assume the column is usually under 128, and always output the inlined-at
1767   // location (it's never more expensive than building an array size 1).
1768   auto Abbv = std::make_shared<BitCodeAbbrev>();
1769   Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_LOCATION));
1770   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1771   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1772   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1773   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1774   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1775   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1776   return Stream.EmitAbbrev(std::move(Abbv));
1777 }
1778
1779 void ModuleBitcodeWriter::writeDILocation(const DILocation *N,
1780                                           SmallVectorImpl<uint64_t> &Record,
1781                                           unsigned &Abbrev) {
1782   if (!Abbrev)
1783     Abbrev = createDILocationAbbrev();
1784
1785   Record.push_back(N->isDistinct());
1786   Record.push_back(N->getLine());
1787   Record.push_back(N->getColumn());
1788   Record.push_back(VE.getMetadataID(N->getScope()));
1789   Record.push_back(VE.getMetadataOrNullID(N->getInlinedAt()));
1790   Record.push_back(N->isImplicitCode());
1791
1792   Stream.EmitRecord(bitc::METADATA_LOCATION, Record, Abbrev);
1793   Record.clear();
1794 }
1795
1796 unsigned ModuleBitcodeWriter::createGenericDINodeAbbrev() {
1797   // Assume the column is usually under 128, and always output the inlined-at
1798   // location (it's never more expensive than building an array size 1).
1799   auto Abbv = std::make_shared<BitCodeAbbrev>();
1800   Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_GENERIC_DEBUG));
1801   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1802   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1803   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1804   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1805   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1806   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1807   return Stream.EmitAbbrev(std::move(Abbv));
1808 }
1809
1810 void ModuleBitcodeWriter::writeGenericDINode(const GenericDINode *N,
1811                                              SmallVectorImpl<uint64_t> &Record,
1812                                              unsigned &Abbrev) {
1813   if (!Abbrev)
1814     Abbrev = createGenericDINodeAbbrev();
1815
1816   Record.push_back(N->isDistinct());
1817   Record.push_back(N->getTag());
1818   Record.push_back(0); // Per-tag version field; unused for now.
1819
1820   for (auto &I : N->operands())
1821     Record.push_back(VE.getMetadataOrNullID(I));
1822
1823   Stream.EmitRecord(bitc::METADATA_GENERIC_DEBUG, Record, Abbrev);
1824   Record.clear();
1825 }
1826
1827 void ModuleBitcodeWriter::writeDISubrange(const DISubrange *N,
1828                                           SmallVectorImpl<uint64_t> &Record,
1829                                           unsigned Abbrev) {
1830   const uint64_t Version = 2 << 1;
1831   Record.push_back((uint64_t)N->isDistinct() | Version);
1832   Record.push_back(VE.getMetadataOrNullID(N->getRawCountNode()));
1833   Record.push_back(VE.getMetadataOrNullID(N->getRawLowerBound()));
1834   Record.push_back(VE.getMetadataOrNullID(N->getRawUpperBound()));
1835   Record.push_back(VE.getMetadataOrNullID(N->getRawStride()));
1836
1837   Stream.EmitRecord(bitc::METADATA_SUBRANGE, Record, Abbrev);
1838   Record.clear();
1839 }
1840
1841 void ModuleBitcodeWriter::writeDIGenericSubrange(
1842     const DIGenericSubrange *N, SmallVectorImpl<uint64_t> &Record,
1843     unsigned Abbrev) {
1844   Record.push_back((uint64_t)N->isDistinct());
1845   Record.push_back(VE.getMetadataOrNullID(N->getRawCountNode()));
1846   Record.push_back(VE.getMetadataOrNullID(N->getRawLowerBound()));
1847   Record.push_back(VE.getMetadataOrNullID(N->getRawUpperBound()));
1848   Record.push_back(VE.getMetadataOrNullID(N->getRawStride()));
1849
1850   Stream.EmitRecord(bitc::METADATA_GENERIC_SUBRANGE, Record, Abbrev);
1851   Record.clear();
1852 }
1853
1854 void ModuleBitcodeWriter::writeDIEnumerator(const DIEnumerator *N,
1855                                             SmallVectorImpl<uint64_t> &Record,
1856                                             unsigned Abbrev) {
1857   const uint64_t IsBigInt = 1 << 2;
1858   Record.push_back(IsBigInt | (N->isUnsigned() << 1) | N->isDistinct());
1859   Record.push_back(N->getValue().getBitWidth());
1860   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
1861   emitWideAPInt(Record, N->getValue());
1862
1863   Stream.EmitRecord(bitc::METADATA_ENUMERATOR, Record, Abbrev);
1864   Record.clear();
1865 }
1866
1867 void ModuleBitcodeWriter::writeDIBasicType(const DIBasicType *N,
1868                                            SmallVectorImpl<uint64_t> &Record,
1869                                            unsigned Abbrev) {
1870   Record.push_back(N->isDistinct());
1871   Record.push_back(N->getTag());
1872   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
1873   Record.push_back(N->getSizeInBits());
1874   Record.push_back(N->getAlignInBits());
1875   Record.push_back(N->getEncoding());
1876   Record.push_back(N->getFlags());
1877   Record.push_back(N->getNumExtraInhabitants());
1878
1879   Stream.EmitRecord(bitc::METADATA_BASIC_TYPE, Record, Abbrev);
1880   Record.clear();
1881 }
1882
1883 void ModuleBitcodeWriter::writeDIStringType(const DIStringType *N,
1884                                             SmallVectorImpl<uint64_t> &Record,
1885                                             unsigned Abbrev) {
1886   Record.push_back(N->isDistinct());
1887   Record.push_back(N->getTag());
1888   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
1889   Record.push_back(VE.getMetadataOrNullID(N->getStringLength()));
1890   Record.push_back(VE.getMetadataOrNullID(N->getStringLengthExp()));
1891   Record.push_back(VE.getMetadataOrNullID(N->getStringLocationExp()));
1892   Record.push_back(N->getSizeInBits());
1893   Record.push_back(N->getAlignInBits());
1894   Record.push_back(N->getEncoding());
1895
1896   Stream.EmitRecord(bitc::METADATA_STRING_TYPE, Record, Abbrev);
1897   Record.clear();
1898 }
1899
1900 void ModuleBitcodeWriter::writeDIDerivedType(const DIDerivedType *N,
1901                                              SmallVectorImpl<uint64_t> &Record,
1902                                              unsigned Abbrev) {
1903   Record.push_back(N->isDistinct());
1904   Record.push_back(N->getTag());
1905   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
1906   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
1907   Record.push_back(N->getLine());
1908   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
1909   Record.push_back(VE.getMetadataOrNullID(N->getBaseType()));
1910   Record.push_back(N->getSizeInBits());
1911   Record.push_back(N->getAlignInBits());
1912   Record.push_back(N->getOffsetInBits());
1913   Record.push_back(N->getFlags());
1914   Record.push_back(VE.getMetadataOrNullID(N->getExtraData()));
1915
1916   // DWARF address space is encoded as N->getDWARFAddressSpace() + 1. 0 means
1917   // that there is no DWARF address space associated with DIDerivedType.
1918   if (const auto &DWARFAddressSpace = N->getDWARFAddressSpace())
1919     Record.push_back(*DWARFAddressSpace + 1);
1920   else
1921     Record.push_back(0);
1922
1923   Record.push_back(VE.getMetadataOrNullID(N->getAnnotations().get()));
1924
1925   if (auto PtrAuthData = N->getPtrAuthData())
1926     Record.push_back(PtrAuthData->RawData);
1927   else
1928     Record.push_back(0);
1929
1930   Stream.EmitRecord(bitc::METADATA_DERIVED_TYPE, Record, Abbrev);
1931   Record.clear();
1932 }
1933
1934 void ModuleBitcodeWriter::writeDICompositeType(
1935     const DICompositeType *N, SmallVectorImpl<uint64_t> &Record,
1936     unsigned Abbrev) {
1937   const unsigned IsNotUsedInOldTypeRef = 0x2;
1938   Record.push_back(IsNotUsedInOldTypeRef | (unsigned)N->isDistinct());
1939   Record.push_back(N->getTag());
1940   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
1941   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
1942   Record.push_back(N->getLine());
1943   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
1944   Record.push_back(VE.getMetadataOrNullID(N->getBaseType()));
1945   Record.push_back(N->getSizeInBits());
1946   Record.push_back(N->getAlignInBits());
1947   Record.push_back(N->getOffsetInBits());
1948   Record.push_back(N->getFlags());
1949   Record.push_back(VE.getMetadataOrNullID(N->getElements().get()));
1950   Record.push_back(N->getRuntimeLang());
1951   Record.push_back(VE.getMetadataOrNullID(N->getVTableHolder()));
1952   Record.push_back(VE.getMetadataOrNullID(N->getTemplateParams().get()));
1953   Record.push_back(VE.getMetadataOrNullID(N->getRawIdentifier()));
1954   Record.push_back(VE.getMetadataOrNullID(N->getDiscriminator()));
1955   Record.push_back(VE.getMetadataOrNullID(N->getRawDataLocation()));
1956   Record.push_back(VE.getMetadataOrNullID(N->getRawAssociated()));
1957   Record.push_back(VE.getMetadataOrNullID(N->getRawAllocated()));
1958   Record.push_back(VE.getMetadataOrNullID(N->getRawRank()));
1959   Record.push_back(VE.getMetadataOrNullID(N->getAnnotations().get()));
1960   Record.push_back(N->getNumExtraInhabitants());
1961   Record.push_back(VE.getMetadataOrNullID(N->getRawSpecification()));
1962
1963   Stream.EmitRecord(bitc::METADATA_COMPOSITE_TYPE, Record, Abbrev);
1964   Record.clear();
1965 }
1966
1967 void ModuleBitcodeWriter::writeDISubroutineType(
1968     const DISubroutineType *N, SmallVectorImpl<uint64_t> &Record,
1969     unsigned Abbrev) {
1970   const unsigned HasNoOldTypeRefs = 0x2;
1971   Record.push_back(HasNoOldTypeRefs | (unsigned)N->isDistinct());
1972   Record.push_back(N->getFlags());
1973   Record.push_back(VE.getMetadataOrNullID(N->getTypeArray().get()));
1974   Record.push_back(N->getCC());
1975
1976   Stream.EmitRecord(bitc::METADATA_SUBROUTINE_TYPE, Record, Abbrev);
1977   Record.clear();
1978 }
1979
1980 void ModuleBitcodeWriter::writeDIFile(const DIFile *N,
1981                                       SmallVectorImpl<uint64_t> &Record,
1982                                       unsigned Abbrev) {
1983   Record.push_back(N->isDistinct());
1984   Record.push_back(VE.getMetadataOrNullID(N->getRawFilename()));
1985   Record.push_back(VE.getMetadataOrNullID(N->getRawDirectory()));
1986   if (N->getRawChecksum()) {
1987     Record.push_back(N->getRawChecksum()->Kind);
1988     Record.push_back(VE.getMetadataOrNullID(N->getRawChecksum()->Value));
1989   } else {
1990     // Maintain backwards compatibility with the old internal representation of
1991     // CSK_None in ChecksumKind by writing nulls here when Checksum is None.
1992     Record.push_back(0);
1993     Record.push_back(VE.getMetadataOrNullID(nullptr));
1994   }
1995   auto Source = N->getRawSource();
1996   if (Source)
1997     Record.push_back(VE.getMetadataOrNullID(Source));
1998
1999   Stream.EmitRecord(bitc::METADATA_FILE, Record, Abbrev);
2000   Record.clear();
2001 }
2002
2003 void ModuleBitcodeWriter::writeDICompileUnit(const DICompileUnit *N,
2004                                              SmallVectorImpl<uint64_t> &Record,
2005                                              unsigned Abbrev) {
2006   assert(N->isDistinct() && "Expected distinct compile units");
2007   Record.push_back(/* IsDistinct */ true);
2008   Record.push_back(N->getSourceLanguage());
2009   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
2010   Record.push_back(VE.getMetadataOrNullID(N->getRawProducer()));
2011   Record.push_back(N->isOptimized());
2012   Record.push_back(VE.getMetadataOrNullID(N->getRawFlags()));
2013   Record.push_back(N->getRuntimeVersion());
2014   Record.push_back(VE.getMetadataOrNullID(N->getRawSplitDebugFilename()));
2015   Record.push_back(N->getEmissionKind());
2016   Record.push_back(VE.getMetadataOrNullID(N->getEnumTypes().get()));
2017   Record.push_back(VE.getMetadataOrNullID(N->getRetainedTypes().get()));
2018   Record.push_back(/* subprograms */ 0);
2019   Record.push_back(VE.getMetadataOrNullID(N->getGlobalVariables().get()));
2020   Record.push_back(VE.getMetadataOrNullID(N->getImportedEntities().get()));
2021   Record.push_back(N->getDWOId());
2022   Record.push_back(VE.getMetadataOrNullID(N->getMacros().get()));
2023   Record.push_back(N->getSplitDebugInlining());
2024   Record.push_back(N->getDebugInfoForProfiling());
2025   Record.push_back((unsigned)N->getNameTableKind());
2026   Record.push_back(N->getRangesBaseAddress());
2027   Record.push_back(VE.getMetadataOrNullID(N->getRawSysRoot()));
2028   Record.push_back(VE.getMetadataOrNullID(N->getRawSDK()));
2029
2030   Stream.EmitRecord(bitc::METADATA_COMPILE_UNIT, Record, Abbrev);
2031   Record.clear();
2032 }
2033
2034 void ModuleBitcodeWriter::writeDISubprogram(const DISubprogram *N,
2035                                             SmallVectorImpl<uint64_t> &Record,
2036                                             unsigned Abbrev) {
2037   const uint64_t HasUnitFlag = 1 << 1;
2038   const uint64_t HasSPFlagsFlag = 1 << 2;
2039   Record.push_back(uint64_t(N->isDistinct()) | HasUnitFlag | HasSPFlagsFlag);
2040   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
2041   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
2042   Record.push_back(VE.getMetadataOrNullID(N->getRawLinkageName()));
2043   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
2044   Record.push_back(N->getLine());
2045   Record.push_back(VE.getMetadataOrNullID(N->getType()));
2046   Record.push_back(N->getScopeLine());
2047   Record.push_back(VE.getMetadataOrNullID(N->getContainingType()));
2048   Record.push_back(N->getSPFlags());
2049   Record.push_back(N->getVirtualIndex());
2050   Record.push_back(N->getFlags());
2051   Record.push_back(VE.getMetadataOrNullID(N->getRawUnit()));
2052   Record.push_back(VE.getMetadataOrNullID(N->getTemplateParams().get()));
2053   Record.push_back(VE.getMetadataOrNullID(N->getDeclaration()));
2054   Record.push_back(VE.getMetadataOrNullID(N->getRetainedNodes().get()));
2055   Record.push_back(N->getThisAdjustment());
2056   Record.push_back(VE.getMetadataOrNullID(N->getThrownTypes().get()));
2057   Record.push_back(VE.getMetadataOrNullID(N->getAnnotations().get()));
2058   Record.push_back(VE.getMetadataOrNullID(N->getRawTargetFuncName()));
2059
2060   Stream.EmitRecord(bitc::METADATA_SUBPROGRAM, Record, Abbrev);
2061   Record.clear();
2062 }
2063
2064 void ModuleBitcodeWriter::writeDILexicalBlock(const DILexicalBlock *N,
2065                                               SmallVectorImpl<uint64_t> &Record,
2066                                               unsigned Abbrev) {
2067   Record.push_back(N->isDistinct());
2068   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
2069   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
2070   Record.push_back(N->getLine());
2071   Record.push_back(N->getColumn());
2072
2073   Stream.EmitRecord(bitc::METADATA_LEXICAL_BLOCK, Record, Abbrev);
2074   Record.clear();
2075 }
2076
2077 void ModuleBitcodeWriter::writeDILexicalBlockFile(
2078     const DILexicalBlockFile *N, SmallVectorImpl<uint64_t> &Record,
2079     unsigned Abbrev) {
2080   Record.push_back(N->isDistinct());
2081   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
2082   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
2083   Record.push_back(N->getDiscriminator());
2084
2085   Stream.EmitRecord(bitc::METADATA_LEXICAL_BLOCK_FILE, Record, Abbrev);
2086   Record.clear();
2087 }
2088
2089 void ModuleBitcodeWriter::writeDICommonBlock(const DICommonBlock *N,
2090                                              SmallVectorImpl<uint64_t> &Record,
2091                                              unsigned Abbrev) {
2092   Record.push_back(N->isDistinct());
2093   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
2094   Record.push_back(VE.getMetadataOrNullID(N->getDecl()));
2095   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
2096   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
2097   Record.push_back(N->getLineNo());
2098
2099   Stream.EmitRecord(bitc::METADATA_COMMON_BLOCK, Record, Abbrev);
2100   Record.clear();
2101 }
2102
2103 void ModuleBitcodeWriter::writeDINamespace(const DINamespace *N,
2104                                            SmallVectorImpl<uint64_t> &Record,
2105                                            unsigned Abbrev) {
2106   Record.push_back(N->isDistinct() | N->getExportSymbols() << 1);
2107   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
2108   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
2109
2110   Stream.EmitRecord(bitc::METADATA_NAMESPACE, Record, Abbrev);
2111   Record.clear();
2112 }
2113
2114 void ModuleBitcodeWriter::writeDIMacro(const DIMacro *N,
2115                                        SmallVectorImpl<uint64_t> &Record,
2116                                        unsigned Abbrev) {
2117   Record.push_back(N->isDistinct());
2118   Record.push_back(N->getMacinfoType());
2119   Record.push_back(N->getLine());
2120   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
2121   Record.push_back(VE.getMetadataOrNullID(N->getRawValue()));
2122
2123   Stream.EmitRecord(bitc::METADATA_MACRO, Record, Abbrev);
2124   Record.clear();
2125 }
2126
2127 void ModuleBitcodeWriter::writeDIMacroFile(const DIMacroFile *N,
2128                                            SmallVectorImpl<uint64_t> &Record,
2129                                            unsigned Abbrev) {
2130   Record.push_back(N->isDistinct());
2131   Record.push_back(N->getMacinfoType());
2132   Record.push_back(N->getLine());
2133   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
2134   Record.push_back(VE.getMetadataOrNullID(N->getElements().get()));
2135
2136   Stream.EmitRecord(bitc::METADATA_MACRO_FILE, Record, Abbrev);
2137   Record.clear();
2138 }
2139
2140 void ModuleBitcodeWriter::writeDIArgList(const DIArgList *N,
2141                                          SmallVectorImpl<uint64_t> &Record) {
2142   Record.reserve(N->getArgs().size());
2143   for (ValueAsMetadata *MD : N->getArgs())
2144     Record.push_back(VE.getMetadataID(MD));
2145
2146   Stream.EmitRecord(bitc::METADATA_ARG_LIST, Record);
2147   Record.clear();
2148 }
2149
2150 void ModuleBitcodeWriter::writeDIModule(const DIModule *N,
2151                                         SmallVectorImpl<uint64_t> &Record,
2152                                         unsigned Abbrev) {
2153   Record.push_back(N->isDistinct());
2154   for (auto &I : N->operands())
2155     Record.push_back(VE.getMetadataOrNullID(I));
2156   Record.push_back(N->getLineNo());
2157   Record.push_back(N->getIsDecl());
2158
2159   Stream.EmitRecord(bitc::METADATA_MODULE, Record, Abbrev);
2160   Record.clear();
2161 }
2162
2163 void ModuleBitcodeWriter::writeDIAssignID(const DIAssignID *N,
2164                                           SmallVectorImpl<uint64_t> &Record,
2165                                           unsigned Abbrev) {
2166   // There are no arguments for this metadata type.
2167   Record.push_back(N->isDistinct());
2168   Stream.EmitRecord(bitc::METADATA_ASSIGN_ID, Record, Abbrev);
2169   Record.clear();
2170 }
2171
2172 void ModuleBitcodeWriter::writeDITemplateTypeParameter(
2173     const DITemplateTypeParameter *N, SmallVectorImpl<uint64_t> &Record,
2174     unsigned Abbrev) {
2175   Record.push_back(N->isDistinct());
2176   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
2177   Record.push_back(VE.getMetadataOrNullID(N->getType()));
2178   Record.push_back(N->isDefault());
2179
2180   Stream.EmitRecord(bitc::METADATA_TEMPLATE_TYPE, Record, Abbrev);
2181   Record.clear();
2182 }
2183
2184 void ModuleBitcodeWriter::writeDITemplateValueParameter(
2185     const DITemplateValueParameter *N, SmallVectorImpl<uint64_t> &Record,
2186     unsigned Abbrev) {
2187   Record.push_back(N->isDistinct());
2188   Record.push_back(N->getTag());
2189   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
2190   Record.push_back(VE.getMetadataOrNullID(N->getType()));
2191   Record.push_back(N->isDefault());
2192   Record.push_back(VE.getMetadataOrNullID(N->getValue()));
2193
2194   Stream.EmitRecord(bitc::METADATA_TEMPLATE_VALUE, Record, Abbrev);
2195   Record.clear();
2196 }
2197
2198 void ModuleBitcodeWriter::writeDIGlobalVariable(
2199     const DIGlobalVariable *N, SmallVectorImpl<uint64_t> &Record,
2200     unsigned Abbrev) {
2201   const uint64_t Version = 2 << 1;
2202   Record.push_back((uint64_t)N->isDistinct() | Version);
2203   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
2204   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
2205   Record.push_back(VE.getMetadataOrNullID(N->getRawLinkageName()));
2206   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
2207   Record.push_back(N->getLine());
2208   Record.push_back(VE.getMetadataOrNullID(N->getType()));
2209   Record.push_back(N->isLocalToUnit());
2210   Record.push_back(N->isDefinition());
2211   Record.push_back(VE.getMetadataOrNullID(N->getStaticDataMemberDeclaration()));
2212   Record.push_back(VE.getMetadataOrNullID(N->getTemplateParams()));
2213   Record.push_back(N->getAlignInBits());
2214   Record.push_back(VE.getMetadataOrNullID(N->getAnnotations().get()));
2215
2216   Stream.EmitRecord(bitc::METADATA_GLOBAL_VAR, Record, Abbrev);
2217   Record.clear();
2218 }
2219
2220 void ModuleBitcodeWriter::writeDILocalVariable(
2221     const DILocalVariable *N, SmallVectorImpl<uint64_t> &Record,
2222     unsigned Abbrev) {
2223   // In order to support all possible bitcode formats in BitcodeReader we need
2224   // to distinguish the following cases:
2225   // 1) Record has no artificial tag (Record[1]),
2226   //   has no obsolete inlinedAt field (Record[9]).
2227   //   In this case Record size will be 8, HasAlignment flag is false.
2228   // 2) Record has artificial tag (Record[1]),
2229   //   has no obsolete inlignedAt field (Record[9]).
2230   //   In this case Record size will be 9, HasAlignment flag is false.
2231   // 3) Record has both artificial tag (Record[1]) and
2232   //   obsolete inlignedAt field (Record[9]).
2233   //   In this case Record size will be 10, HasAlignment flag is false.
2234   // 4) Record has neither artificial tag, nor inlignedAt field, but
2235   //   HasAlignment flag is true and Record[8] contains alignment value.
2236   const uint64_t HasAlignmentFlag = 1 << 1;
2237   Record.push_back((uint64_t)N->isDistinct() | HasAlignmentFlag);
2238   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
2239   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
2240   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
2241   Record.push_back(N->getLine());
2242   Record.push_back(VE.getMetadataOrNullID(N->getType()));
2243   Record.push_back(N->getArg());
2244   Record.push_back(N->getFlags());
2245   Record.push_back(N->getAlignInBits());
2246   Record.push_back(VE.getMetadataOrNullID(N->getAnnotations().get()));
2247
2248   Stream.EmitRecord(bitc::METADATA_LOCAL_VAR, Record, Abbrev);
2249   Record.clear();
2250 }
2251
2252 void ModuleBitcodeWriter::writeDILabel(
2253     const DILabel *N, SmallVectorImpl<uint64_t> &Record,
2254     unsigned Abbrev) {
2255   Record.push_back((uint64_t)N->isDistinct());
2256   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
2257   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
2258   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
2259   Record.push_back(N->getLine());
2260
2261   Stream.EmitRecord(bitc::METADATA_LABEL, Record, Abbrev);
2262   Record.clear();
2263 }
2264
2265 void ModuleBitcodeWriter::writeDIExpression(const DIExpression *N,
2266                                             SmallVectorImpl<uint64_t> &Record,
2267                                             unsigned Abbrev) {
2268   Record.reserve(N->getElements().size() + 1);
2269   const uint64_t Version = 3 << 1;
2270   Record.push_back((uint64_t)N->isDistinct() | Version);
2271   Record.append(N->elements_begin(), N->elements_end());
2272
2273   Stream.EmitRecord(bitc::METADATA_EXPRESSION, Record, Abbrev);
2274   Record.clear();
2275 }
2276
2277 void ModuleBitcodeWriter::writeDIGlobalVariableExpression(
2278     const DIGlobalVariableExpression *N, SmallVectorImpl<uint64_t> &Record,
2279     unsigned Abbrev) {
2280   Record.push_back(N->isDistinct());
2281   Record.push_back(VE.getMetadataOrNullID(N->getVariable()));
2282   Record.push_back(VE.getMetadataOrNullID(N->getExpression()));
2283
2284   Stream.EmitRecord(bitc::METADATA_GLOBAL_VAR_EXPR, Record, Abbrev);
2285   Record.clear();
2286 }
2287
2288 void ModuleBitcodeWriter::writeDIObjCProperty(const DIObjCProperty *N,
2289                                               SmallVectorImpl<uint64_t> &Record,
2290                                               unsigned Abbrev) {
2291   Record.push_back(N->isDistinct());
2292   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
2293   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
2294   Record.push_back(N->getLine());
2295   Record.push_back(VE.getMetadataOrNullID(N->getRawSetterName()));
2296   Record.push_back(VE.getMetadataOrNullID(N->getRawGetterName()));
2297   Record.push_back(N->getAttributes());
2298   Record.push_back(VE.getMetadataOrNullID(N->getType()));
2299
2300   Stream.EmitRecord(bitc::METADATA_OBJC_PROPERTY, Record, Abbrev);
2301   Record.clear();
2302 }
2303
2304 void ModuleBitcodeWriter::writeDIImportedEntity(
2305     const DIImportedEntity *N, SmallVectorImpl<uint64_t> &Record,
2306     unsigned Abbrev) {
2307   Record.push_back(N->isDistinct());
2308   Record.push_back(N->getTag());
2309   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
2310   Record.push_back(VE.getMetadataOrNullID(N->getEntity()));
2311   Record.push_back(N->getLine());
2312   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
2313   Record.push_back(VE.getMetadataOrNullID(N->getRawFile()));
2314   Record.push_back(VE.getMetadataOrNullID(N->getElements().get()));
2315
2316   Stream.EmitRecord(bitc::METADATA_IMPORTED_ENTITY, Record, Abbrev);
2317   Record.clear();
2318 }
2319
2320 unsigned ModuleBitcodeWriter::createNamedMetadataAbbrev() {
2321   auto Abbv = std::make_shared<BitCodeAbbrev>();
2322   Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_NAME));
2323   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2324   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
2325   return Stream.EmitAbbrev(std::move(Abbv));
2326 }
2327
2328 void ModuleBitcodeWriter::writeNamedMetadata(
2329     SmallVectorImpl<uint64_t> &Record) {
2330   if (M.named_metadata_empty())
2331     return;
2332
2333   unsigned Abbrev = createNamedMetadataAbbrev();
2334   for (const NamedMDNode &NMD : M.named_metadata()) {
2335     // Write name.
2336     StringRef Str = NMD.getName();
2337     Record.append(Str.bytes_begin(), Str.bytes_end());
2338     Stream.EmitRecord(bitc::METADATA_NAME, Record, Abbrev);
2339     Record.clear();
2340
2341     // Write named metadata operands.
2342     for (const MDNode *N : NMD.operands())
2343       Record.push_back(VE.getMetadataID(N));
2344     Stream.EmitRecord(bitc::METADATA_NAMED_NODE, Record, 0);
2345     Record.clear();
2346   }
2347 }
2348
2349 unsigned ModuleBitcodeWriter::createMetadataStringsAbbrev() {
2350   auto Abbv = std::make_shared<BitCodeAbbrev>();
2351   Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_STRINGS));
2352   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // # of strings
2353   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // offset to chars
2354   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
2355   return Stream.EmitAbbrev(std::move(Abbv));
2356 }
2357
2358 /// Write out a record for MDString.
2359 ///
2360 /// All the metadata strings in a metadata block are emitted in a single
2361 /// record.  The sizes and strings themselves are shoved into a blob.
2362 void ModuleBitcodeWriter::writeMetadataStrings(
2363     ArrayRef<const Metadata *> Strings, SmallVectorImpl<uint64_t> &Record) {
2364   if (Strings.empty())
2365     return;
2366
2367   // Start the record with the number of strings.
2368   Record.push_back(bitc::METADATA_STRINGS);
2369   Record.push_back(Strings.size());
2370
2371   // Emit the sizes of the strings in the blob.
2372   SmallString<256> Blob;
2373   {
2374     BitstreamWriter W(Blob);
2375     for (const Metadata *MD : Strings)
2376       W.EmitVBR(cast<MDString>(MD)->getLength(), 6);
2377     W.FlushToWord();
2378   }
2379
2380   // Add the offset to the strings to the record.
2381   Record.push_back(Blob.size());
2382
2383   // Add the strings to the blob.
2384   for (const Metadata *MD : Strings)
2385     Blob.append(cast<MDString>(MD)->getString());
2386
2387   // Emit the final record.
2388   Stream.EmitRecordWithBlob(createMetadataStringsAbbrev(), Record, Blob);
2389   Record.clear();
2390 }
2391
2392 // Generates an enum to use as an index in the Abbrev array of Metadata record.
2393 enum MetadataAbbrev : unsigned {
2394 #define HANDLE_MDNODE_LEAF(CLASS) CLASS##AbbrevID,
2395 #include "llvm/IR/Metadata.def"
2396   LastPlusOne
2397 };
2398
2399 void ModuleBitcodeWriter::writeMetadataRecords(
2400     ArrayRef<const Metadata *> MDs, SmallVectorImpl<uint64_t> &Record,
2401     std::vector<unsigned> *MDAbbrevs, std::vector<uint64_t> *IndexPos) {
2402   if (MDs.empty())
2403     return;
2404
2405   // Initialize MDNode abbreviations.
2406 #define HANDLE_MDNODE_LEAF(CLASS) unsigned CLASS##Abbrev = 0;
2407 #include "llvm/IR/Metadata.def"
2408
2409   for (const Metadata *MD : MDs) {
2410     if (IndexPos)
2411       IndexPos->push_back(Stream.GetCurrentBitNo());
2412     if (const MDNode *N = dyn_cast<MDNode>(MD)) {
2413       assert(N->isResolved() && "Expected forward references to be resolved");
2414
2415       switch (N->getMetadataID()) {
2416       default:
2417         llvm_unreachable("Invalid MDNode subclass");
2418 #define HANDLE_MDNODE_LEAF(CLASS)                                              \
2419   case Metadata::CLASS##Kind:                                                  \
2420     if (MDAbbrevs)                                                             \
2421       write##CLASS(cast<CLASS>(N), Record,                                     \
2422                    (*MDAbbrevs)[MetadataAbbrev::CLASS##AbbrevID]);             \
2423     else                                                                       \
2424       write##CLASS(cast<CLASS>(N), Record, CLASS##Abbrev);                     \
2425     continue;
2426 #include "llvm/IR/Metadata.def"
2427       }
2428     }
2429     if (auto *AL = dyn_cast<DIArgList>(MD)) {
2430       writeDIArgList(AL, Record);
2431       continue;
2432     }
2433     writeValueAsMetadata(cast<ValueAsMetadata>(MD), Record);
2434   }
2435 }
2436
2437 void ModuleBitcodeWriter::writeModuleMetadata() {
2438   if (!VE.hasMDs() && M.named_metadata_empty())
2439     return;
2440
2441   Stream.EnterSubblock(bitc::METADATA_BLOCK_ID, 4);
2442   SmallVector<uint64_t, 64> Record;
2443
2444   // Emit all abbrevs upfront, so that the reader can jump in the middle of the
2445   // block and load any metadata.
2446   std::vector<unsigned> MDAbbrevs;
2447
2448   MDAbbrevs.resize(MetadataAbbrev::LastPlusOne);
2449   MDAbbrevs[MetadataAbbrev::DILocationAbbrevID] = createDILocationAbbrev();
2450   MDAbbrevs[MetadataAbbrev::GenericDINodeAbbrevID] =
2451       createGenericDINodeAbbrev();
2452
2453   auto Abbv = std::make_shared<BitCodeAbbrev>();
2454   Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_INDEX_OFFSET));
2455   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
2456   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
2457   unsigned OffsetAbbrev = Stream.EmitAbbrev(std::move(Abbv));
2458
2459   Abbv = std::make_shared<BitCodeAbbrev>();
2460   Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_INDEX));
2461   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2462   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
2463   unsigned IndexAbbrev = Stream.EmitAbbrev(std::move(Abbv));
2464
2465   // Emit MDStrings together upfront.
2466   writeMetadataStrings(VE.getMDStrings(), Record);
2467
2468   // We only emit an index for the metadata record if we have more than a given
2469   // (naive) threshold of metadatas, otherwise it is not worth it.
2470   if (VE.getNonMDStrings().size() > IndexThreshold) {
2471     // Write a placeholder value in for the offset of the metadata index,
2472     // which is written after the records, so that it can include
2473     // the offset of each entry. The placeholder offset will be
2474     // updated after all records are emitted.
2475     uint64_t Vals[] = {0, 0};
2476     Stream.EmitRecord(bitc::METADATA_INDEX_OFFSET, Vals, OffsetAbbrev);
2477   }
2478
2479   // Compute and save the bit offset to the current position, which will be
2480   // patched when we emit the index later. We can simply subtract the 64-bit
2481   // fixed size from the current bit number to get the location to backpatch.
2482   uint64_t IndexOffsetRecordBitPos = Stream.GetCurrentBitNo();
2483
2484   // This index will contain the bitpos for each individual record.
2485   std::vector<uint64_t> IndexPos;
2486   IndexPos.reserve(VE.getNonMDStrings().size());
2487
2488   // Write all the records
2489   writeMetadataRecords(VE.getNonMDStrings(), Record, &MDAbbrevs, &IndexPos);
2490
2491   if (VE.getNonMDStrings().size() > IndexThreshold) {
2492     // Now that we have emitted all the records we will emit the index. But
2493     // first
2494     // backpatch the forward reference so that the reader can skip the records
2495     // efficiently.
2496     Stream.BackpatchWord64(IndexOffsetRecordBitPos - 64,
2497                            Stream.GetCurrentBitNo() - IndexOffsetRecordBitPos);
2498
2499     // Delta encode the index.
2500     uint64_t PreviousValue = IndexOffsetRecordBitPos;
2501     for (auto &Elt : IndexPos) {
2502       auto EltDelta = Elt - PreviousValue;
2503       PreviousValue = Elt;
2504       Elt = EltDelta;
2505     }
2506     // Emit the index record.
2507     Stream.EmitRecord(bitc::METADATA_INDEX, IndexPos, IndexAbbrev);
2508     IndexPos.clear();
2509   }
2510
2511   // Write the named metadata now.
2512   writeNamedMetadata(Record);
2513
2514   auto AddDeclAttachedMetadata = [&](const GlobalObject &GO) {
2515     SmallVector<uint64_t, 4> Record;
2516     Record.push_back(VE.getValueID(&GO));
2517     pushGlobalMetadataAttachment(Record, GO);
2518     Stream.EmitRecord(bitc::METADATA_GLOBAL_DECL_ATTACHMENT, Record);
2519   };
2520   for (const Function &F : M)
2521     if (F.isDeclaration() && F.hasMetadata())
2522       AddDeclAttachedMetadata(F);
2523   // FIXME: Only store metadata for declarations here, and move data for global
2524   // variable definitions to a separate block (PR28134).
2525   for (const GlobalVariable &GV : M.globals())
2526     if (GV.hasMetadata())
2527       AddDeclAttachedMetadata(GV);
2528
2529   Stream.ExitBlock();
2530 }
2531
2532 void ModuleBitcodeWriter::writeFunctionMetadata(const Function &F) {
2533   if (!VE.hasMDs())
2534     return;
2535
2536   Stream.EnterSubblock(bitc::METADATA_BLOCK_ID, 3);
2537   SmallVector<uint64_t, 64> Record;
2538   writeMetadataStrings(VE.getMDStrings(), Record);
2539   writeMetadataRecords(VE.getNonMDStrings(), Record);
2540   Stream.ExitBlock();
2541 }
2542
2543 void ModuleBitcodeWriter::pushGlobalMetadataAttachment(
2544     SmallVectorImpl<uint64_t> &Record, const GlobalObject &GO) {
2545   // [n x [id, mdnode]]
2546   SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
2547   GO.getAllMetadata(MDs);
2548   for (const auto &I : MDs) {
2549     Record.push_back(I.first);
2550     Record.push_back(VE.getMetadataID(I.second));
2551   }
2552 }
2553
2554 void ModuleBitcodeWriter::writeFunctionMetadataAttachment(const Function &F) {
2555   Stream.EnterSubblock(bitc::METADATA_ATTACHMENT_ID, 3);
2556
2557   SmallVector<uint64_t, 64> Record;
2558
2559   if (F.hasMetadata()) {
2560     pushGlobalMetadataAttachment(Record, F);
2561     Stream.EmitRecord(bitc::METADATA_ATTACHMENT, Record, 0);
2562     Record.clear();
2563   }
2564
2565   // Write metadata attachments
2566   // METADATA_ATTACHMENT - [m x [value, [n x [id, mdnode]]]
2567   SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
2568   for (const BasicBlock &BB : F)
2569     for (const Instruction &I : BB) {
2570       MDs.clear();
2571       I.getAllMetadataOtherThanDebugLoc(MDs);
2572
2573       // If no metadata, ignore instruction.
2574       if (MDs.empty()) continue;
2575
2576       Record.push_back(VE.getInstructionID(&I));
2577
2578       for (unsigned i = 0, e = MDs.size(); i != e; ++i) {
2579         Record.push_back(MDs[i].first);
2580         Record.push_back(VE.getMetadataID(MDs[i].second));
2581       }
2582       Stream.EmitRecord(bitc::METADATA_ATTACHMENT, Record, 0);
2583       Record.clear();
2584     }
2585
2586   Stream.ExitBlock();
2587 }
2588
2589 void ModuleBitcodeWriter::writeModuleMetadataKinds() {
2590   SmallVector<uint64_t, 64> Record;
2591
2592   // Write metadata kinds
2593   // METADATA_KIND - [n x [id, name]]
2594   SmallVector<StringRef, 8> Names;
2595   M.getMDKindNames(Names);
2596
2597   if (Names.empty()) return;
2598
2599   Stream.EnterSubblock(bitc::METADATA_KIND_BLOCK_ID, 3);
2600
2601   for (unsigned MDKindID = 0, e = Names.size(); MDKindID != e; ++MDKindID) {
2602     Record.push_back(MDKindID);
2603     StringRef KName = Names[MDKindID];
2604     Record.append(KName.begin(), KName.end());
2605
2606     Stream.EmitRecord(bitc::METADATA_KIND, Record, 0);
2607     Record.clear();
2608   }
2609
2610   Stream.ExitBlock();
2611 }
2612
2613 void ModuleBitcodeWriter::writeOperandBundleTags() {
2614   // Write metadata kinds
2615   //
2616   // OPERAND_BUNDLE_TAGS_BLOCK_ID : N x OPERAND_BUNDLE_TAG
2617   //
2618   // OPERAND_BUNDLE_TAG - [strchr x N]
2619
2620   SmallVector<StringRef, 8> Tags;
2621   M.getOperandBundleTags(Tags);
2622
2623   if (Tags.empty())
2624     return;
2625
2626   Stream.EnterSubblock(bitc::OPERAND_BUNDLE_TAGS_BLOCK_ID, 3);
2627
2628   SmallVector<uint64_t, 64> Record;
2629
2630   for (auto Tag : Tags) {
2631     Record.append(Tag.begin(), Tag.end());
2632
2633     Stream.EmitRecord(bitc::OPERAND_BUNDLE_TAG, Record, 0);
2634     Record.clear();
2635   }
2636
2637   Stream.ExitBlock();
2638 }
2639
2640 void ModuleBitcodeWriter::writeSyncScopeNames() {
2641   SmallVector<StringRef, 8> SSNs;
2642   M.getContext().getSyncScopeNames(SSNs);
2643   if (SSNs.empty())
2644     return;
2645
2646   Stream.EnterSubblock(bitc::SYNC_SCOPE_NAMES_BLOCK_ID, 2);
2647
2648   SmallVector<uint64_t, 64> Record;
2649   for (auto SSN : SSNs) {
2650     Record.append(SSN.begin(), SSN.end());
2651     Stream.EmitRecord(bitc::SYNC_SCOPE_NAME, Record, 0);
2652     Record.clear();
2653   }
2654
2655   Stream.ExitBlock();
2656 }
2657
2658 void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal,
2659                                          bool isGlobal) {
2660   if (FirstVal == LastVal) return;
2661
2662   Stream.EnterSubblock(bitc::CONSTANTS_BLOCK_ID, 4);
2663
2664   unsigned AggregateAbbrev = 0;
2665   unsigned String8Abbrev = 0;
2666   unsigned CString7Abbrev = 0;
2667   unsigned CString6Abbrev = 0;
2668   // If this is a constant pool for the module, emit module-specific abbrevs.
2669   if (isGlobal) {
2670     // Abbrev for CST_CODE_AGGREGATE.
2671     auto Abbv = std::make_shared<BitCodeAbbrev>();
2672     Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_AGGREGATE));
2673     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2674     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, Log2_32_Ceil(LastVal+1)));
2675     AggregateAbbrev = Stream.EmitAbbrev(std::move(Abbv));
2676
2677     // Abbrev for CST_CODE_STRING.
2678     Abbv = std::make_shared<BitCodeAbbrev>();
2679     Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_STRING));
2680     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2681     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
2682     String8Abbrev = Stream.EmitAbbrev(std::move(Abbv));
2683     // Abbrev for CST_CODE_CSTRING.
2684     Abbv = std::make_shared<BitCodeAbbrev>();
2685     Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_CSTRING));
2686     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2687     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
2688     CString7Abbrev = Stream.EmitAbbrev(std::move(Abbv));
2689     // Abbrev for CST_CODE_CSTRING.
2690     Abbv = std::make_shared<BitCodeAbbrev>();
2691     Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_CSTRING));
2692     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2693     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
2694     CString6Abbrev = Stream.EmitAbbrev(std::move(Abbv));
2695   }
2696
2697   SmallVector<uint64_t, 64> Record;
2698
2699   const ValueEnumerator::ValueList &Vals = VE.getValues();
2700   Type *LastTy = nullptr;
2701   for (unsigned i = FirstVal; i != LastVal; ++i) {
2702     const Value *V = Vals[i].first;
2703     // If we need to switch types, do so now.
2704     if (V->getType() != LastTy) {
2705       LastTy = V->getType();
2706       Record.push_back(VE.getTypeID(LastTy));
2707       Stream.EmitRecord(bitc::CST_CODE_SETTYPE, Record,
2708                         CONSTANTS_SETTYPE_ABBREV);
2709       Record.clear();
2710     }
2711
2712     if (const InlineAsm *IA = dyn_cast<InlineAsm>(V)) {
2713       Record.push_back(VE.getTypeID(IA->getFunctionType()));
2714       Record.push_back(
2715           unsigned(IA->hasSideEffects()) | unsigned(IA->isAlignStack()) << 1 |
2716           unsigned(IA->getDialect() & 1) << 2 | unsigned(IA->canThrow()) << 3);
2717
2718       // Add the asm string.
2719       const std::string &AsmStr = IA->getAsmString();
2720       Record.push_back(AsmStr.size());
2721       Record.append(AsmStr.begin(), AsmStr.end());
2722
2723       // Add the constraint string.
2724       const std::string &ConstraintStr = IA->getConstraintString();
2725       Record.push_back(ConstraintStr.size());
2726       Record.append(ConstraintStr.begin(), ConstraintStr.end());
2727       Stream.EmitRecord(bitc::CST_CODE_INLINEASM, Record);
2728       Record.clear();
2729       continue;
2730     }
2731     const Constant *C = cast<Constant>(V);
2732     unsigned Code = -1U;
2733     unsigned AbbrevToUse = 0;
2734     if (C->isNullValue()) {
2735       Code = bitc::CST_CODE_NULL;
2736     } else if (isa<PoisonValue>(C)) {
2737       Code = bitc::CST_CODE_POISON;
2738     } else if (isa<UndefValue>(C)) {
2739       Code = bitc::CST_CODE_UNDEF;
2740     } else if (const ConstantInt *IV = dyn_cast<ConstantInt>(C)) {
2741       if (IV->getBitWidth() <= 64) {
2742         uint64_t V = IV->getSExtValue();
2743         emitSignedInt64(Record, V);
2744         Code = bitc::CST_CODE_INTEGER;
2745         AbbrevToUse = CONSTANTS_INTEGER_ABBREV;
2746       } else {                             // Wide integers, > 64 bits in size.
2747         emitWideAPInt(Record, IV->getValue());
2748         Code = bitc::CST_CODE_WIDE_INTEGER;
2749       }
2750     } else if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C)) {
2751       Code = bitc::CST_CODE_FLOAT;
2752       Type *Ty = CFP->getType()->getScalarType();
2753       if (Ty->isHalfTy() || Ty->isBFloatTy() || Ty->isFloatTy() ||
2754           Ty->isDoubleTy()) {
2755         Record.push_back(CFP->getValueAPF().bitcastToAPInt().getZExtValue());
2756       } else if (Ty->isX86_FP80Ty()) {
2757         // api needed to prevent premature destruction
2758         // bits are not in the same order as a normal i80 APInt, compensate.
2759         APInt api = CFP->getValueAPF().bitcastToAPInt();
2760         const uint64_t *p = api.getRawData();
2761         Record.push_back((p[1] << 48) | (p[0] >> 16));
2762         Record.push_back(p[0] & 0xffffLL);
2763       } else if (Ty->isFP128Ty() || Ty->isPPC_FP128Ty()) {
2764         APInt api = CFP->getValueAPF().bitcastToAPInt();
2765         const uint64_t *p = api.getRawData();
2766         Record.push_back(p[0]);
2767         Record.push_back(p[1]);
2768       } else {
2769         assert(0 && "Unknown FP type!");
2770       }
2771     } else if (isa<ConstantDataSequential>(C) &&
2772                cast<ConstantDataSequential>(C)->isString()) {
2773       const ConstantDataSequential *Str = cast<ConstantDataSequential>(C);
2774       // Emit constant strings specially.
2775       unsigned NumElts = Str->getNumElements();
2776       // If this is a null-terminated string, use the denser CSTRING encoding.
2777       if (Str->isCString()) {
2778         Code = bitc::CST_CODE_CSTRING;
2779         --NumElts;  // Don't encode the null, which isn't allowed by char6.
2780       } else {
2781         Code = bitc::CST_CODE_STRING;
2782         AbbrevToUse = String8Abbrev;
2783       }
2784       bool isCStr7 = Code == bitc::CST_CODE_CSTRING;
2785       bool isCStrChar6 = Code == bitc::CST_CODE_CSTRING;
2786       for (unsigned i = 0; i != NumElts; ++i) {
2787         unsigned char V = Str->getElementAsInteger(i);
2788         Record.push_back(V);
2789         isCStr7 &= (V & 128) == 0;
2790         if (isCStrChar6)
2791           isCStrChar6 = BitCodeAbbrevOp::isChar6(V);
2792       }
2793
2794       if (isCStrChar6)
2795         AbbrevToUse = CString6Abbrev;
2796       else if (isCStr7)
2797         AbbrevToUse = CString7Abbrev;
2798     } else if (const ConstantDataSequential *CDS =
2799                   dyn_cast<ConstantDataSequential>(C)) {
2800       Code = bitc::CST_CODE_DATA;
2801       Type *EltTy = CDS->getElementType();
2802       if (isa<IntegerType>(EltTy)) {
2803         for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i)
2804           Record.push_back(CDS->getElementAsInteger(i));
2805       } else {
2806         for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i)
2807           Record.push_back(
2808               CDS->getElementAsAPFloat(i).bitcastToAPInt().getLimitedValue());
2809       }
2810     } else if (isa<ConstantAggregate>(C)) {
2811       Code = bitc::CST_CODE_AGGREGATE;
2812       for (const Value *Op : C->operands())
2813         Record.push_back(VE.getValueID(Op));
2814       AbbrevToUse = AggregateAbbrev;
2815     } else if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
2816       switch (CE->getOpcode()) {
2817       default:
2818         if (Instruction::isCast(CE->getOpcode())) {
2819           Code = bitc::CST_CODE_CE_CAST;
2820           Record.push_back(getEncodedCastOpcode(CE->getOpcode()));
2821           Record.push_back(VE.getTypeID(C->getOperand(0)->getType()));
2822           Record.push_back(VE.getValueID(C->getOperand(0)));
2823           AbbrevToUse = CONSTANTS_CE_CAST_Abbrev;
2824         } else {
2825           assert(CE->getNumOperands() == 2 && "Unknown constant expr!");
2826           Code = bitc::CST_CODE_CE_BINOP;
2827           Record.push_back(getEncodedBinaryOpcode(CE->getOpcode()));
2828           Record.push_back(VE.getValueID(C->getOperand(0)));
2829           Record.push_back(VE.getValueID(C->getOperand(1)));
2830           uint64_t Flags = getOptimizationFlags(CE);
2831           if (Flags != 0)
2832             Record.push_back(Flags);
2833         }
2834         break;
2835       case Instruction::FNeg: {
2836         assert(CE->getNumOperands() == 1 && "Unknown constant expr!");
2837         Code = bitc::CST_CODE_CE_UNOP;
2838         Record.push_back(getEncodedUnaryOpcode(CE->getOpcode()));
2839         Record.push_back(VE.getValueID(C->getOperand(0)));
2840         uint64_t Flags = getOptimizationFlags(CE);
2841         if (Flags != 0)
2842           Record.push_back(Flags);
2843         break;
2844       }
2845       case Instruction::GetElementPtr: {
2846         Code = bitc::CST_CODE_CE_GEP;
2847         const auto *GO = cast<GEPOperator>(C);
2848         Record.push_back(VE.getTypeID(GO->getSourceElementType()));
2849         Record.push_back(getOptimizationFlags(GO));
2850         if (std::optional<ConstantRange> Range = GO->getInRange()) {
2851           Code = bitc::CST_CODE_CE_GEP_WITH_INRANGE;
2852           emitConstantRange(Record, *Range, /*EmitBitWidth=*/true);
2853         }
2854         for (const Value *Op : CE->operands()) {
2855           Record.push_back(VE.getTypeID(Op->getType()));
2856           Record.push_back(VE.getValueID(Op));
2857         }
2858         break;
2859       }
2860       case Instruction::ExtractElement:
2861         Code = bitc::CST_CODE_CE_EXTRACTELT;
2862         Record.push_back(VE.getTypeID(C->getOperand(0)->getType()));
2863         Record.push_back(VE.getValueID(C->getOperand(0)));
2864         Record.push_back(VE.getTypeID(C->getOperand(1)->getType()));
2865         Record.push_back(VE.getValueID(C->getOperand(1)));
2866         break;
2867       case Instruction::InsertElement:
2868         Code = bitc::CST_CODE_CE_INSERTELT;
2869         Record.push_back(VE.getValueID(C->getOperand(0)));
2870         Record.push_back(VE.getValueID(C->getOperand(1)));
2871         Record.push_back(VE.getTypeID(C->getOperand(2)->getType()));
2872         Record.push_back(VE.getValueID(C->getOperand(2)));
2873         break;
2874       case Instruction::ShuffleVector:
2875         // If the return type and argument types are the same, this is a
2876         // standard shufflevector instruction.  If the types are different,
2877         // then the shuffle is widening or truncating the input vectors, and
2878         // the argument type must also be encoded.
2879         if (C->getType() == C->getOperand(0)->getType()) {
2880           Code = bitc::CST_CODE_CE_SHUFFLEVEC;
2881         } else {
2882           Code = bitc::CST_CODE_CE_SHUFVEC_EX;
2883           Record.push_back(VE.getTypeID(C->getOperand(0)->getType()));
2884         }
2885         Record.push_back(VE.getValueID(C->getOperand(0)));
2886         Record.push_back(VE.getValueID(C->getOperand(1)));
2887         Record.push_back(VE.getValueID(CE->getShuffleMaskForBitcode()));
2888         break;
2889       }
2890     } else if (const BlockAddress *BA = dyn_cast<BlockAddress>(C)) {
2891       Code = bitc::CST_CODE_BLOCKADDRESS;
2892       Record.push_back(VE.getTypeID(BA->getFunction()->getType()));
2893       Record.push_back(VE.getValueID(BA->getFunction()));
2894       Record.push_back(VE.getGlobalBasicBlockID(BA->getBasicBlock()));
2895     } else if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C)) {
2896       Code = bitc::CST_CODE_DSO_LOCAL_EQUIVALENT;
2897       Record.push_back(VE.getTypeID(Equiv->getGlobalValue()->getType()));
2898       Record.push_back(VE.getValueID(Equiv->getGlobalValue()));
2899     } else if (const auto *NC = dyn_cast<NoCFIValue>(C)) {
2900       Code = bitc::CST_CODE_NO_CFI_VALUE;
2901       Record.push_back(VE.getTypeID(NC->getGlobalValue()->getType()));
2902       Record.push_back(VE.getValueID(NC->getGlobalValue()));
2903     } else if (const auto *CPA = dyn_cast<ConstantPtrAuth>(C)) {
2904       Code = bitc::CST_CODE_PTRAUTH;
2905       Record.push_back(VE.getValueID(CPA->getPointer()));
2906       Record.push_back(VE.getValueID(CPA->getKey()));
2907       Record.push_back(VE.getValueID(CPA->getDiscriminator()));
2908       Record.push_back(VE.getValueID(CPA->getAddrDiscriminator()));
2909     } else {
2910 #ifndef NDEBUG
2911       C->dump();
2912 #endif
2913       llvm_unreachable("Unknown constant!");
2914     }
2915     Stream.EmitRecord(Code, Record, AbbrevToUse);
2916     Record.clear();
2917   }
2918
2919   Stream.ExitBlock();
2920 }
2921
2922 void ModuleBitcodeWriter::writeModuleConstants() {
2923   const ValueEnumerator::ValueList &Vals = VE.getValues();
2924
2925   // Find the first constant to emit, which is the first non-globalvalue value.
2926   // We know globalvalues have been emitted by WriteModuleInfo.
2927   for (unsigned i = 0, e = Vals.size(); i != e; ++i) {
2928     if (!isa<GlobalValue>(Vals[i].first)) {
2929       writeConstants(i, Vals.size(), true);
2930       return;
2931     }
2932   }
2933 }
2934
2935 /// pushValueAndType - The file has to encode both the value and type id for
2936 /// many values, because we need to know what type to create for forward
2937 /// references.  However, most operands are not forward references, so this type
2938 /// field is not needed.
2939 ///
2940 /// This function adds V's value ID to Vals.  If the value ID is higher than the
2941 /// instruction ID, then it is a forward reference, and it also includes the
2942 /// type ID.  The value ID that is written is encoded relative to the InstID.
2943 bool ModuleBitcodeWriter::pushValueAndType(const Value *V, unsigned InstID,
2944                                            SmallVectorImpl<unsigned> &Vals) {
2945   unsigned ValID = VE.getValueID(V);
2946   // Make encoding relative to the InstID.
2947   Vals.push_back(InstID - ValID);
2948   if (ValID >= InstID) {
2949     Vals.push_back(VE.getTypeID(V->getType()));
2950     return true;
2951   }
2952   return false;
2953 }
2954
2955 bool ModuleBitcodeWriter::pushValueOrMetadata(const Value *V, unsigned InstID,
2956                                               SmallVectorImpl<unsigned> &Vals) {
2957   bool IsMetadata = V->getType()->isMetadataTy();
2958   if (IsMetadata) {
2959     Vals.push_back(bitc::OB_METADATA);
2960     Metadata *MD = cast<MetadataAsValue>(V)->getMetadata();
2961     unsigned ValID = VE.getMetadataID(MD);
2962     Vals.push_back(InstID - ValID);
2963     return false;
2964   }
2965   return pushValueAndType(V, InstID, Vals);
2966 }
2967
2968 void ModuleBitcodeWriter::writeOperandBundles(const CallBase &CS,
2969                                               unsigned InstID) {
2970   SmallVector<unsigned, 64> Record;
2971   LLVMContext &C = CS.getContext();
2972
2973   for (unsigned i = 0, e = CS.getNumOperandBundles(); i != e; ++i) {
2974     const auto &Bundle = CS.getOperandBundleAt(i);
2975     Record.push_back(C.getOperandBundleTagID(Bundle.getTagName()));
2976
2977     for (auto &Input : Bundle.Inputs)
2978       pushValueOrMetadata(Input, InstID, Record);
2979
2980     Stream.EmitRecord(bitc::FUNC_CODE_OPERAND_BUNDLE, Record);
2981     Record.clear();
2982   }
2983 }
2984
2985 /// pushValue - Like pushValueAndType, but where the type of the value is
2986 /// omitted (perhaps it was already encoded in an earlier operand).
2987 void ModuleBitcodeWriter::pushValue(const Value *V, unsigned InstID,
2988                                     SmallVectorImpl<unsigned> &Vals) {
2989   unsigned ValID = VE.getValueID(V);
2990   Vals.push_back(InstID - ValID);
2991 }
2992
2993 void ModuleBitcodeWriter::pushValueSigned(const Value *V, unsigned InstID,
2994                                           SmallVectorImpl<uint64_t> &Vals) {
2995   unsigned ValID = VE.getValueID(V);
2996   int64_t diff = ((int32_t)InstID - (int32_t)ValID);
2997   emitSignedInt64(Vals, diff);
2998 }
2999
3000 /// WriteInstruction - Emit an instruction to the specified stream.
3001 void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
3002                                            unsigned InstID,
3003                                            SmallVectorImpl<unsigned> &Vals) {
3004   unsigned Code = 0;
3005   unsigned AbbrevToUse = 0;
3006   VE.setInstructionID(&I);
3007   switch (I.getOpcode()) {
3008   default:
3009     if (Instruction::isCast(I.getOpcode())) {
3010       Code = bitc::FUNC_CODE_INST_CAST;
3011       if (!pushValueAndType(I.getOperand(0), InstID, Vals))
3012         AbbrevToUse = FUNCTION_INST_CAST_ABBREV;
3013       Vals.push_back(VE.getTypeID(I.getType()));
3014       Vals.push_back(getEncodedCastOpcode(I.getOpcode()));
3015       uint64_t Flags = getOptimizationFlags(&I);
3016       if (Flags != 0) {
3017         if (AbbrevToUse == FUNCTION_INST_CAST_ABBREV)
3018           AbbrevToUse = FUNCTION_INST_CAST_FLAGS_ABBREV;
3019         Vals.push_back(Flags);
3020       }
3021     } else {
3022       assert(isa<BinaryOperator>(I) && "Unknown instruction!");
3023       Code = bitc::FUNC_CODE_INST_BINOP;
3024       if (!pushValueAndType(I.getOperand(0), InstID, Vals))
3025         AbbrevToUse = FUNCTION_INST_BINOP_ABBREV;
3026       pushValue(I.getOperand(1), InstID, Vals);
3027       Vals.push_back(getEncodedBinaryOpcode(I.getOpcode()));
3028       uint64_t Flags = getOptimizationFlags(&I);
3029       if (Flags != 0) {
3030         if (AbbrevToUse == FUNCTION_INST_BINOP_ABBREV)
3031           AbbrevToUse = FUNCTION_INST_BINOP_FLAGS_ABBREV;
3032         Vals.push_back(Flags);
3033       }
3034     }
3035     break;
3036   case Instruction::FNeg: {
3037     Code = bitc::FUNC_CODE_INST_UNOP;
3038     if (!pushValueAndType(I.getOperand(0), InstID, Vals))
3039       AbbrevToUse = FUNCTION_INST_UNOP_ABBREV;
3040     Vals.push_back(getEncodedUnaryOpcode(I.getOpcode()));
3041     uint64_t Flags = getOptimizationFlags(&I);
3042     if (Flags != 0) {
3043       if (AbbrevToUse == FUNCTION_INST_UNOP_ABBREV)
3044         AbbrevToUse = FUNCTION_INST_UNOP_FLAGS_ABBREV;
3045       Vals.push_back(Flags);
3046     }
3047     break;
3048   }
3049   case Instruction::GetElementPtr: {
3050     Code = bitc::FUNC_CODE_INST_GEP;
3051     AbbrevToUse = FUNCTION_INST_GEP_ABBREV;
3052     auto &GEPInst = cast<GetElementPtrInst>(I);
3053     Vals.push_back(getOptimizationFlags(&I));
3054     Vals.push_back(VE.getTypeID(GEPInst.getSourceElementType()));
3055     for (const Value *Op : I.operands())
3056       pushValueAndType(Op, InstID, Vals);
3057     break;
3058   }
3059   case Instruction::ExtractValue: {
3060     Code = bitc::FUNC_CODE_INST_EXTRACTVAL;
3061     pushValueAndType(I.getOperand(0), InstID, Vals);
3062     const ExtractValueInst *EVI = cast<ExtractValueInst>(&I);
3063     Vals.append(EVI->idx_begin(), EVI->idx_end());
3064     break;
3065   }
3066   case Instruction::InsertValue: {
3067     Code = bitc::FUNC_CODE_INST_INSERTVAL;
3068     pushValueAndType(I.getOperand(0), InstID, Vals);
3069     pushValueAndType(I.getOperand(1), InstID, Vals);
3070     const InsertValueInst *IVI = cast<InsertValueInst>(&I);
3071     Vals.append(IVI->idx_begin(), IVI->idx_end());
3072     break;
3073   }
3074   case Instruction::Select: {
3075     Code = bitc::FUNC_CODE_INST_VSELECT;
3076     pushValueAndType(I.getOperand(1), InstID, Vals);
3077     pushValue(I.getOperand(2), InstID, Vals);
3078     pushValueAndType(I.getOperand(0), InstID, Vals);
3079     uint64_t Flags = getOptimizationFlags(&I);
3080     if (Flags != 0)
3081       Vals.push_back(Flags);
3082     break;
3083   }
3084   case Instruction::ExtractElement:
3085     Code = bitc::FUNC_CODE_INST_EXTRACTELT;
3086     pushValueAndType(I.getOperand(0), InstID, Vals);
3087     pushValueAndType(I.getOperand(1), InstID, Vals);
3088     break;
3089   case Instruction::InsertElement:
3090     Code = bitc::FUNC_CODE_INST_INSERTELT;
3091     pushValueAndType(I.getOperand(0), InstID, Vals);
3092     pushValue(I.getOperand(1), InstID, Vals);
3093     pushValueAndType(I.getOperand(2), InstID, Vals);
3094     break;
3095   case Instruction::ShuffleVector:
3096     Code = bitc::FUNC_CODE_INST_SHUFFLEVEC;
3097     pushValueAndType(I.getOperand(0), InstID, Vals);
3098     pushValue(I.getOperand(1), InstID, Vals);
3099     pushValue(cast<ShuffleVectorInst>(I).getShuffleMaskForBitcode(), InstID,
3100               Vals);
3101     break;
3102   case Instruction::ICmp:
3103   case Instruction::FCmp: {
3104     // compare returning Int1Ty or vector of Int1Ty
3105     Code = bitc::FUNC_CODE_INST_CMP2;
3106     pushValueAndType(I.getOperand(0), InstID, Vals);
3107     pushValue(I.getOperand(1), InstID, Vals);
3108     Vals.push_back(cast<CmpInst>(I).getPredicate());
3109     uint64_t Flags = getOptimizationFlags(&I);
3110     if (Flags != 0)
3111       Vals.push_back(Flags);
3112     break;
3113   }
3114
3115   case Instruction::Ret:
3116     {
3117       Code = bitc::FUNC_CODE_INST_RET;
3118       unsigned NumOperands = I.getNumOperands();
3119       if (NumOperands == 0)
3120         AbbrevToUse = FUNCTION_INST_RET_VOID_ABBREV;
3121       else if (NumOperands == 1) {
3122         if (!pushValueAndType(I.getOperand(0), InstID, Vals))
3123           AbbrevToUse = FUNCTION_INST_RET_VAL_ABBREV;
3124       } else {
3125         for (const Value *Op : I.operands())
3126           pushValueAndType(Op, InstID, Vals);
3127       }
3128     }
3129     break;
3130   case Instruction::Br:
3131     {
3132       Code = bitc::FUNC_CODE_INST_BR;
3133       const BranchInst &II = cast<BranchInst>(I);
3134       Vals.push_back(VE.getValueID(II.getSuccessor(0)));
3135       if (II.isConditional()) {
3136         Vals.push_back(VE.getValueID(II.getSuccessor(1)));
3137         pushValue(II.getCondition(), InstID, Vals);
3138       }
3139     }
3140     break;
3141   case Instruction::Switch:
3142     {
3143       Code = bitc::FUNC_CODE_INST_SWITCH;
3144       const SwitchInst &SI = cast<SwitchInst>(I);
3145       Vals.push_back(VE.getTypeID(SI.getCondition()->getType()));
3146       pushValue(SI.getCondition(), InstID, Vals);
3147       Vals.push_back(VE.getValueID(SI.getDefaultDest()));
3148       for (auto Case : SI.cases()) {
3149         Vals.push_back(VE.getValueID(Case.getCaseValue()));
3150         Vals.push_back(VE.getValueID(Case.getCaseSuccessor()));
3151       }
3152     }
3153     break;
3154   case Instruction::IndirectBr:
3155     Code = bitc::FUNC_CODE_INST_INDIRECTBR;
3156     Vals.push_back(VE.getTypeID(I.getOperand(0)->getType()));
3157     // Encode the address operand as relative, but not the basic blocks.
3158     pushValue(I.getOperand(0), InstID, Vals);
3159     for (const Value *Op : drop_begin(I.operands()))
3160       Vals.push_back(VE.getValueID(Op));
3161     break;
3162
3163   case Instruction::Invoke: {
3164     const InvokeInst *II = cast<InvokeInst>(&I);
3165     const Value *Callee = II->getCalledOperand();
3166     FunctionType *FTy = II->getFunctionType();
3167
3168     if (II->hasOperandBundles())
3169       writeOperandBundles(*II, InstID);
3170
3171     Code = bitc::FUNC_CODE_INST_INVOKE;
3172
3173     Vals.push_back(VE.getAttributeListID(II->getAttributes()));
3174     Vals.push_back(II->getCallingConv() | 1 << 13);
3175     Vals.push_back(VE.getValueID(II->getNormalDest()));
3176     Vals.push_back(VE.getValueID(II->getUnwindDest()));
3177     Vals.push_back(VE.getTypeID(FTy));
3178     pushValueAndType(Callee, InstID, Vals);
3179
3180     // Emit value #'s for the fixed parameters.
3181     for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3182       pushValue(I.getOperand(i), InstID, Vals); // fixed param.
3183
3184     // Emit type/value pairs for varargs params.
3185     if (FTy->isVarArg()) {
3186       for (unsigned i = FTy->getNumParams(), e = II->arg_size(); i != e; ++i)
3187         pushValueAndType(I.getOperand(i), InstID, Vals); // vararg
3188     }
3189     break;
3190   }
3191   case Instruction::Resume:
3192     Code = bitc::FUNC_CODE_INST_RESUME;
3193     pushValueAndType(I.getOperand(0), InstID, Vals);
3194     break;
3195   case Instruction::CleanupRet: {
3196     Code = bitc::FUNC_CODE_INST_CLEANUPRET;
3197     const auto &CRI = cast<CleanupReturnInst>(I);
3198     pushValue(CRI.getCleanupPad(), InstID, Vals);
3199     if (CRI.hasUnwindDest())
3200       Vals.push_back(VE.getValueID(CRI.getUnwindDest()));
3201     break;
3202   }
3203   case Instruction::CatchRet: {
3204     Code = bitc::FUNC_CODE_INST_CATCHRET;
3205     const auto &CRI = cast<CatchReturnInst>(I);
3206     pushValue(CRI.getCatchPad(), InstID, Vals);
3207     Vals.push_back(VE.getValueID(CRI.getSuccessor()));
3208     break;
3209   }
3210   case Instruction::CleanupPad:
3211   case Instruction::CatchPad: {
3212     const auto &FuncletPad = cast<FuncletPadInst>(I);
3213     Code = isa<CatchPadInst>(FuncletPad) ? bitc::FUNC_CODE_INST_CATCHPAD
3214                                          : bitc::FUNC_CODE_INST_CLEANUPPAD;
3215     pushValue(FuncletPad.getParentPad(), InstID, Vals);
3216
3217     unsigned NumArgOperands = FuncletPad.arg_size();
3218     Vals.push_back(NumArgOperands);
3219     for (unsigned Op = 0; Op != NumArgOperands; ++Op)
3220       pushValueAndType(FuncletPad.getArgOperand(Op), InstID, Vals);
3221     break;
3222   }
3223   case Instruction::CatchSwitch: {
3224     Code = bitc::FUNC_CODE_INST_CATCHSWITCH;
3225     const auto &CatchSwitch = cast<CatchSwitchInst>(I);
3226
3227     pushValue(CatchSwitch.getParentPad(), InstID, Vals);
3228
3229     unsigned NumHandlers = CatchSwitch.getNumHandlers();
3230     Vals.push_back(NumHandlers);
3231     for (const BasicBlock *CatchPadBB : CatchSwitch.handlers())
3232       Vals.push_back(VE.getValueID(CatchPadBB));
3233
3234     if (CatchSwitch.hasUnwindDest())
3235       Vals.push_back(VE.getValueID(CatchSwitch.getUnwindDest()));
3236     break;
3237   }
3238   case Instruction::CallBr: {
3239     const CallBrInst *CBI = cast<CallBrInst>(&I);
3240     const Value *Callee = CBI->getCalledOperand();
3241     FunctionType *FTy = CBI->getFunctionType();
3242
3243     if (CBI->hasOperandBundles())
3244       writeOperandBundles(*CBI, InstID);
3245
3246     Code = bitc::FUNC_CODE_INST_CALLBR;
3247
3248     Vals.push_back(VE.getAttributeListID(CBI->getAttributes()));
3249
3250     Vals.push_back(CBI->getCallingConv() << bitc::CALL_CCONV |
3251                    1 << bitc::CALL_EXPLICIT_TYPE);
3252
3253     Vals.push_back(VE.getValueID(CBI->getDefaultDest()));
3254     Vals.push_back(CBI->getNumIndirectDests());
3255     for (unsigned i = 0, e = CBI->getNumIndirectDests(); i != e; ++i)
3256       Vals.push_back(VE.getValueID(CBI->getIndirectDest(i)));
3257
3258     Vals.push_back(VE.getTypeID(FTy));
3259     pushValueAndType(Callee, InstID, Vals);
3260
3261     // Emit value #'s for the fixed parameters.
3262     for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3263       pushValue(I.getOperand(i), InstID, Vals); // fixed param.
3264
3265     // Emit type/value pairs for varargs params.
3266     if (FTy->isVarArg()) {
3267       for (unsigned i = FTy->getNumParams(), e = CBI->arg_size(); i != e; ++i)
3268         pushValueAndType(I.getOperand(i), InstID, Vals); // vararg
3269     }
3270     break;
3271   }
3272   case Instruction::Unreachable:
3273     Code = bitc::FUNC_CODE_INST_UNREACHABLE;
3274     AbbrevToUse = FUNCTION_INST_UNREACHABLE_ABBREV;
3275     break;
3276
3277   case Instruction::PHI: {
3278     const PHINode &PN = cast<PHINode>(I);
3279     Code = bitc::FUNC_CODE_INST_PHI;
3280     // With the newer instruction encoding, forward references could give
3281     // negative valued IDs.  This is most common for PHIs, so we use
3282     // signed VBRs.
3283     SmallVector<uint64_t, 128> Vals64;
3284     Vals64.push_back(VE.getTypeID(PN.getType()));
3285     for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) {
3286       pushValueSigned(PN.getIncomingValue(i), InstID, Vals64);
3287       Vals64.push_back(VE.getValueID(PN.getIncomingBlock(i)));
3288     }
3289
3290     uint64_t Flags = getOptimizationFlags(&I);
3291     if (Flags != 0)
3292       Vals64.push_back(Flags);
3293
3294     // Emit a Vals64 vector and exit.
3295     Stream.EmitRecord(Code, Vals64, AbbrevToUse);
3296     Vals64.clear();
3297     return;
3298   }
3299
3300   case Instruction::LandingPad: {
3301     const LandingPadInst &LP = cast<LandingPadInst>(I);
3302     Code = bitc::FUNC_CODE_INST_LANDINGPAD;
3303     Vals.push_back(VE.getTypeID(LP.getType()));
3304     Vals.push_back(LP.isCleanup());
3305     Vals.push_back(LP.getNumClauses());
3306     for (unsigned I = 0, E = LP.getNumClauses(); I != E; ++I) {
3307       if (LP.isCatch(I))
3308         Vals.push_back(LandingPadInst::Catch);
3309       else
3310         Vals.push_back(LandingPadInst::Filter);
3311       pushValueAndType(LP.getClause(I), InstID, Vals);
3312     }
3313     break;
3314   }
3315
3316   case Instruction::Alloca: {
3317     Code = bitc::FUNC_CODE_INST_ALLOCA;
3318     const AllocaInst &AI = cast<AllocaInst>(I);
3319     Vals.push_back(VE.getTypeID(AI.getAllocatedType()));
3320     Vals.push_back(VE.getTypeID(I.getOperand(0)->getType()));
3321     Vals.push_back(VE.getValueID(I.getOperand(0))); // size.
3322     using APV = AllocaPackedValues;
3323     unsigned Record = 0;
3324     unsigned EncodedAlign = getEncodedAlign(AI.getAlign());
3325     Bitfield::set<APV::AlignLower>(
3326         Record, EncodedAlign & ((1 << APV::AlignLower::Bits) - 1));
3327     Bitfield::set<APV::AlignUpper>(Record,
3328                                    EncodedAlign >> APV::AlignLower::Bits);
3329     Bitfield::set<APV::UsedWithInAlloca>(Record, AI.isUsedWithInAlloca());
3330     Bitfield::set<APV::ExplicitType>(Record, true);
3331     Bitfield::set<APV::SwiftError>(Record, AI.isSwiftError());
3332     Vals.push_back(Record);
3333
3334     unsigned AS = AI.getAddressSpace();
3335     if (AS != M.getDataLayout().getAllocaAddrSpace())
3336       Vals.push_back(AS);
3337     break;
3338   }
3339
3340   case Instruction::Load:
3341     if (cast<LoadInst>(I).isAtomic()) {
3342       Code = bitc::FUNC_CODE_INST_LOADATOMIC;
3343       pushValueAndType(I.getOperand(0), InstID, Vals);
3344     } else {
3345       Code = bitc::FUNC_CODE_INST_LOAD;
3346       if (!pushValueAndType(I.getOperand(0), InstID, Vals)) // ptr
3347         AbbrevToUse = FUNCTION_INST_LOAD_ABBREV;
3348     }
3349     Vals.push_back(VE.getTypeID(I.getType()));
3350     Vals.push_back(getEncodedAlign(cast<LoadInst>(I).getAlign()));
3351     Vals.push_back(cast<LoadInst>(I).isVolatile());
3352     if (cast<LoadInst>(I).isAtomic()) {
3353       Vals.push_back(getEncodedOrdering(cast<LoadInst>(I).getOrdering()));
3354       Vals.push_back(getEncodedSyncScopeID(cast<LoadInst>(I).getSyncScopeID()));
3355     }
3356     break;
3357   case Instruction::Store:
3358     if (cast<StoreInst>(I).isAtomic())
3359       Code = bitc::FUNC_CODE_INST_STOREATOMIC;
3360     else
3361       Code = bitc::FUNC_CODE_INST_STORE;
3362     pushValueAndType(I.getOperand(1), InstID, Vals); // ptrty + ptr
3363     pushValueAndType(I.getOperand(0), InstID, Vals); // valty + val
3364     Vals.push_back(getEncodedAlign(cast<StoreInst>(I).getAlign()));
3365     Vals.push_back(cast<StoreInst>(I).isVolatile());
3366     if (cast<StoreInst>(I).isAtomic()) {
3367       Vals.push_back(getEncodedOrdering(cast<StoreInst>(I).getOrdering()));
3368       Vals.push_back(
3369           getEncodedSyncScopeID(cast<StoreInst>(I).getSyncScopeID()));
3370     }
3371     break;
3372   case Instruction::AtomicCmpXchg:
3373     Code = bitc::FUNC_CODE_INST_CMPXCHG;
3374     pushValueAndType(I.getOperand(0), InstID, Vals); // ptrty + ptr
3375     pushValueAndType(I.getOperand(1), InstID, Vals); // cmp.
3376     pushValue(I.getOperand(2), InstID, Vals);        // newval.
3377     Vals.push_back(cast<AtomicCmpXchgInst>(I).isVolatile());
3378     Vals.push_back(
3379         getEncodedOrdering(cast<AtomicCmpXchgInst>(I).getSuccessOrdering()));
3380     Vals.push_back(
3381         getEncodedSyncScopeID(cast<AtomicCmpXchgInst>(I).getSyncScopeID()));
3382     Vals.push_back(
3383         getEncodedOrdering(cast<AtomicCmpXchgInst>(I).getFailureOrdering()));
3384     Vals.push_back(cast<AtomicCmpXchgInst>(I).isWeak());
3385     Vals.push_back(getEncodedAlign(cast<AtomicCmpXchgInst>(I).getAlign()));
3386     break;
3387   case Instruction::AtomicRMW:
3388     Code = bitc::FUNC_CODE_INST_ATOMICRMW;
3389     pushValueAndType(I.getOperand(0), InstID, Vals); // ptrty + ptr
3390     pushValueAndType(I.getOperand(1), InstID, Vals); // valty + val
3391     Vals.push_back(
3392         getEncodedRMWOperation(cast<AtomicRMWInst>(I).getOperation()));
3393     Vals.push_back(cast<AtomicRMWInst>(I).isVolatile());
3394     Vals.push_back(getEncodedOrdering(cast<AtomicRMWInst>(I).getOrdering()));
3395     Vals.push_back(
3396         getEncodedSyncScopeID(cast<AtomicRMWInst>(I).getSyncScopeID()));
3397     Vals.push_back(getEncodedAlign(cast<AtomicRMWInst>(I).getAlign()));
3398     break;
3399   case Instruction::Fence:
3400     Code = bitc::FUNC_CODE_INST_FENCE;
3401     Vals.push_back(getEncodedOrdering(cast<FenceInst>(I).getOrdering()));
3402     Vals.push_back(getEncodedSyncScopeID(cast<FenceInst>(I).getSyncScopeID()));
3403     break;
3404   case Instruction::Call: {
3405     const CallInst &CI = cast<CallInst>(I);
3406     FunctionType *FTy = CI.getFunctionType();
3407
3408     if (CI.hasOperandBundles())
3409       writeOperandBundles(CI, InstID);
3410
3411     Code = bitc::FUNC_CODE_INST_CALL;
3412
3413     Vals.push_back(VE.getAttributeListID(CI.getAttributes()));
3414
3415     unsigned Flags = getOptimizationFlags(&I);
3416     Vals.push_back(CI.getCallingConv() << bitc::CALL_CCONV |
3417                    unsigned(CI.isTailCall()) << bitc::CALL_TAIL |
3418                    unsigned(CI.isMustTailCall()) << bitc::CALL_MUSTTAIL |
3419                    1 << bitc::CALL_EXPLICIT_TYPE |
3420                    unsigned(CI.isNoTailCall()) << bitc::CALL_NOTAIL |
3421                    unsigned(Flags != 0) << bitc::CALL_FMF);
3422     if (Flags != 0)
3423       Vals.push_back(Flags);
3424
3425     Vals.push_back(VE.getTypeID(FTy));
3426     pushValueAndType(CI.getCalledOperand(), InstID, Vals); // Callee
3427
3428     // Emit value #'s for the fixed parameters.
3429     for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i) {
3430       // Check for labels (can happen with asm labels).
3431       if (FTy->getParamType(i)->isLabelTy())
3432         Vals.push_back(VE.getValueID(CI.getArgOperand(i)));
3433       else
3434         pushValue(CI.getArgOperand(i), InstID, Vals); // fixed param.
3435     }
3436
3437     // Emit type/value pairs for varargs params.
3438     if (FTy->isVarArg()) {
3439       for (unsigned i = FTy->getNumParams(), e = CI.arg_size(); i != e; ++i)
3440         pushValueAndType(CI.getArgOperand(i), InstID, Vals); // varargs
3441     }
3442     break;
3443   }
3444   case Instruction::VAArg:
3445     Code = bitc::FUNC_CODE_INST_VAARG;
3446     Vals.push_back(VE.getTypeID(I.getOperand(0)->getType()));   // valistty
3447     pushValue(I.getOperand(0), InstID, Vals);                   // valist.
3448     Vals.push_back(VE.getTypeID(I.getType())); // restype.
3449     break;
3450   case Instruction::Freeze:
3451     Code = bitc::FUNC_CODE_INST_FREEZE;
3452     pushValueAndType(I.getOperand(0), InstID, Vals);
3453     break;
3454   }
3455
3456   Stream.EmitRecord(Code, Vals, AbbrevToUse);
3457   Vals.clear();
3458 }
3459
3460 /// Write a GlobalValue VST to the module. The purpose of this data structure is
3461 /// to allow clients to efficiently find the function body.
3462 void ModuleBitcodeWriter::writeGlobalValueSymbolTable(
3463   DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) {
3464   // Get the offset of the VST we are writing, and backpatch it into
3465   // the VST forward declaration record.
3466   uint64_t VSTOffset = Stream.GetCurrentBitNo();
3467   // The BitcodeStartBit was the stream offset of the identification block.
3468   VSTOffset -= bitcodeStartBit();
3469   assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned");
3470   // Note that we add 1 here because the offset is relative to one word
3471   // before the start of the identification block, which was historically
3472   // always the start of the regular bitcode header.
3473   Stream.BackpatchWord(VSTOffsetPlaceholder, VSTOffset / 32 + 1);
3474
3475   Stream.EnterSubblock(bitc::VALUE_SYMTAB_BLOCK_ID, 4);
3476
3477   auto Abbv = std::make_shared<BitCodeAbbrev>();
3478   Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY));
3479   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id
3480   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset
3481   unsigned FnEntryAbbrev = Stream.EmitAbbrev(std::move(Abbv));
3482
3483   for (const Function &F : M) {
3484     uint64_t Record[2];
3485
3486     if (F.isDeclaration())
3487       continue;
3488
3489     Record[0] = VE.getValueID(&F);
3490
3491     // Save the word offset of the function (from the start of the
3492     // actual bitcode written to the stream).
3493     uint64_t BitcodeIndex = FunctionToBitcodeIndex[&F] - bitcodeStartBit();
3494     assert((BitcodeIndex & 31) == 0 && "function block not 32-bit aligned");
3495     // Note that we add 1 here because the offset is relative to one word
3496     // before the start of the identification block, which was historically
3497     // always the start of the regular bitcode header.
3498     Record[1] = BitcodeIndex / 32 + 1;
3499
3500     Stream.EmitRecord(bitc::VST_CODE_FNENTRY, Record, FnEntryAbbrev);
3501   }
3502
3503   Stream.ExitBlock();
3504 }
3505
3506 /// Emit names for arguments, instructions and basic blocks in a function.
3507 void ModuleBitcodeWriter::writeFunctionLevelValueSymbolTable(
3508     const ValueSymbolTable &VST) {
3509   if (VST.empty())
3510     return;
3511
3512   Stream.EnterSubblock(bitc::VALUE_SYMTAB_BLOCK_ID, 4);
3513
3514   // FIXME: Set up the abbrev, we know how many values there are!
3515   // FIXME: We know if the type names can use 7-bit ascii.
3516   SmallVector<uint64_t, 64> NameVals;
3517
3518   for (const ValueName &Name : VST) {
3519     // Figure out the encoding to use for the name.
3520     StringEncoding Bits = getStringEncoding(Name.getKey());
3521
3522     unsigned AbbrevToUse = VST_ENTRY_8_ABBREV;
3523     NameVals.push_back(VE.getValueID(Name.getValue()));
3524
3525     // VST_CODE_ENTRY:   [valueid, namechar x N]
3526     // VST_CODE_BBENTRY: [bbid, namechar x N]
3527     unsigned Code;
3528     if (isa<BasicBlock>(Name.getValue())) {
3529       Code = bitc::VST_CODE_BBENTRY;
3530       if (Bits == SE_Char6)
3531         AbbrevToUse = VST_BBENTRY_6_ABBREV;
3532     } else {
3533       Code = bitc::VST_CODE_ENTRY;
3534       if (Bits == SE_Char6)
3535         AbbrevToUse = VST_ENTRY_6_ABBREV;
3536       else if (Bits == SE_Fixed7)
3537         AbbrevToUse = VST_ENTRY_7_ABBREV;
3538     }
3539
3540     for (const auto P : Name.getKey())
3541       NameVals.push_back((unsigned char)P);
3542
3543     // Emit the finished record.
3544     Stream.EmitRecord(Code, NameVals, AbbrevToUse);
3545     NameVals.clear();
3546   }
3547
3548   Stream.ExitBlock();
3549 }
3550
3551 void ModuleBitcodeWriter::writeUseList(UseListOrder &&Order) {
3552   assert(Order.Shuffle.size() >= 2 && "Shuffle too small");
3553   unsigned Code;
3554   if (isa<BasicBlock>(Order.V))
3555     Code = bitc::USELIST_CODE_BB;
3556   else
3557     Code = bitc::USELIST_CODE_DEFAULT;
3558
3559   SmallVector<uint64_t, 64> Record(Order.Shuffle.begin(), Order.Shuffle.end());
3560   Record.push_back(VE.getValueID(Order.V));
3561   Stream.EmitRecord(Code, Record);
3562 }
3563
3564 void ModuleBitcodeWriter::writeUseListBlock(const Function *F) {
3565   assert(VE.shouldPreserveUseListOrder() &&
3566          "Expected to be preserving use-list order");
3567
3568   auto hasMore = [&]() {
3569     return !VE.UseListOrders.empty() && VE.UseListOrders.back().F == F;
3570   };
3571   if (!hasMore())
3572     // Nothing to do.
3573     return;
3574
3575   Stream.EnterSubblock(bitc::USELIST_BLOCK_ID, 3);
3576   while (hasMore()) {
3577     writeUseList(std::move(VE.UseListOrders.back()));
3578     VE.UseListOrders.pop_back();
3579   }
3580   Stream.ExitBlock();
3581 }
3582
3583 /// Emit a function body to the module stream.
3584 void ModuleBitcodeWriter::writeFunction(
3585     const Function &F,
3586     DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) {
3587   // Save the bitcode index of the start of this function block for recording
3588   // in the VST.
3589   FunctionToBitcodeIndex[&F] = Stream.GetCurrentBitNo();
3590
3591   Stream.EnterSubblock(bitc::FUNCTION_BLOCK_ID, 4);
3592   VE.incorporateFunction(F);
3593
3594   SmallVector<unsigned, 64> Vals;
3595
3596   // Emit the number of basic blocks, so the reader can create them ahead of
3597   // time.
3598   Vals.push_back(VE.getBasicBlocks().size());
3599   Stream.EmitRecord(bitc::FUNC_CODE_DECLAREBLOCKS, Vals);
3600   Vals.clear();
3601
3602   // If there are function-local constants, emit them now.
3603   unsigned CstStart, CstEnd;
3604   VE.getFunctionConstantRange(CstStart, CstEnd);
3605   writeConstants(CstStart, CstEnd, false);
3606
3607   // If there is function-local metadata, emit it now.
3608   writeFunctionMetadata(F);
3609
3610   // Keep a running idea of what the instruction ID is.
3611   unsigned InstID = CstEnd;
3612
3613   bool NeedsMetadataAttachment = F.hasMetadata();
3614
3615   DILocation *LastDL = nullptr;
3616   SmallSetVector<Function *, 4> BlockAddressUsers;
3617
3618   // Finally, emit all the instructions, in order.
3619   for (const BasicBlock &BB : F) {
3620     for (const Instruction &I : BB) {
3621       writeInstruction(I, InstID, Vals);
3622
3623       if (!I.getType()->isVoidTy())
3624         ++InstID;
3625
3626       // If the instruction has metadata, write a metadata attachment later.
3627       NeedsMetadataAttachment |= I.hasMetadataOtherThanDebugLoc();
3628
3629       // If the instruction has a debug location, emit it.
3630       if (DILocation *DL = I.getDebugLoc()) {
3631         if (DL == LastDL) {
3632           // Just repeat the same debug loc as last time.
3633           Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_LOC_AGAIN, Vals);
3634         } else {
3635           Vals.push_back(DL->getLine());
3636           Vals.push_back(DL->getColumn());
3637           Vals.push_back(VE.getMetadataOrNullID(DL->getScope()));
3638           Vals.push_back(VE.getMetadataOrNullID(DL->getInlinedAt()));
3639           Vals.push_back(DL->isImplicitCode());
3640           Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_LOC, Vals);
3641           Vals.clear();
3642           LastDL = DL;
3643         }
3644       }
3645
3646       // If the instruction has DbgRecords attached to it, emit them. Note that
3647       // they come after the instruction so that it's easy to attach them again
3648       // when reading the bitcode, even though conceptually the debug locations
3649       // start "before" the instruction.
3650       if (I.hasDbgRecords() && WriteNewDbgInfoFormatToBitcode) {
3651         /// Try to push the value only (unwrapped), otherwise push the
3652         /// metadata wrapped value. Returns true if the value was pushed
3653         /// without the ValueAsMetadata wrapper.
3654         auto PushValueOrMetadata = [&Vals, InstID,
3655                                     this](Metadata *RawLocation) {
3656           assert(RawLocation &&
3657                  "RawLocation unexpectedly null in DbgVariableRecord");
3658           if (ValueAsMetadata *VAM = dyn_cast<ValueAsMetadata>(RawLocation)) {
3659             SmallVector<unsigned, 2> ValAndType;
3660             // If the value is a fwd-ref the type is also pushed. We don't
3661             // want the type, so fwd-refs are kept wrapped (pushValueAndType
3662             // returns false if the value is pushed without type).
3663             if (!pushValueAndType(VAM->getValue(), InstID, ValAndType)) {
3664               Vals.push_back(ValAndType[0]);
3665               return true;
3666             }
3667           }
3668           // The metadata is a DIArgList, or ValueAsMetadata wrapping a
3669           // fwd-ref. Push the metadata ID.
3670           Vals.push_back(VE.getMetadataID(RawLocation));
3671           return false;
3672         };
3673
3674         // Write out non-instruction debug information attached to this
3675         // instruction. Write it after the instruction so that it's easy to
3676         // re-attach to the instruction reading the records in.
3677         for (DbgRecord &DR : I.DebugMarker->getDbgRecordRange()) {
3678           if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(&DR)) {
3679             Vals.push_back(VE.getMetadataID(&*DLR->getDebugLoc()));
3680             Vals.push_back(VE.getMetadataID(DLR->getLabel()));
3681             Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_RECORD_LABEL, Vals);
3682             Vals.clear();
3683             continue;
3684           }
3685
3686           // First 3 fields are common to all kinds:
3687           //   DILocation, DILocalVariable, DIExpression
3688           // dbg_value (FUNC_CODE_DEBUG_RECORD_VALUE)
3689           //   ..., LocationMetadata
3690           // dbg_value (FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE - abbrev'd)
3691           //   ..., Value
3692           // dbg_declare (FUNC_CODE_DEBUG_RECORD_DECLARE)
3693           //   ..., LocationMetadata
3694           // dbg_assign (FUNC_CODE_DEBUG_RECORD_ASSIGN)
3695           //   ..., LocationMetadata, DIAssignID, DIExpression, LocationMetadata
3696           DbgVariableRecord &DVR = cast<DbgVariableRecord>(DR);
3697           Vals.push_back(VE.getMetadataID(&*DVR.getDebugLoc()));
3698           Vals.push_back(VE.getMetadataID(DVR.getVariable()));
3699           Vals.push_back(VE.getMetadataID(DVR.getExpression()));
3700           if (DVR.isDbgValue()) {
3701             if (PushValueOrMetadata(DVR.getRawLocation()))
3702               Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE, Vals,
3703                                 FUNCTION_DEBUG_RECORD_VALUE_ABBREV);
3704             else
3705               Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_RECORD_VALUE, Vals);
3706           } else if (DVR.isDbgDeclare()) {
3707             Vals.push_back(VE.getMetadataID(DVR.getRawLocation()));
3708             Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_RECORD_DECLARE, Vals);
3709           } else {
3710             assert(DVR.isDbgAssign() && "Unexpected DbgRecord kind");
3711             Vals.push_back(VE.getMetadataID(DVR.getRawLocation()));
3712             Vals.push_back(VE.getMetadataID(DVR.getAssignID()));
3713             Vals.push_back(VE.getMetadataID(DVR.getAddressExpression()));
3714             Vals.push_back(VE.getMetadataID(DVR.getRawAddress()));
3715             Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_RECORD_ASSIGN, Vals);
3716           }
3717           Vals.clear();
3718         }
3719       }
3720     }
3721
3722     if (BlockAddress *BA = BlockAddress::lookup(&BB)) {
3723       SmallVector<Value *> Worklist{BA};
3724       SmallPtrSet<Value *, 8> Visited{BA};
3725       while (!Worklist.empty()) {
3726         Value *V = Worklist.pop_back_val();
3727         for (User *U : V->users()) {
3728           if (auto *I = dyn_cast<Instruction>(U)) {
3729             Function *P = I->getFunction();
3730             if (P != &F)
3731               BlockAddressUsers.insert(P);
3732           } else if (isa<Constant>(U) && !isa<GlobalValue>(U) &&
3733                      Visited.insert(U).second)
3734             Worklist.push_back(U);
3735         }
3736       }
3737     }
3738   }
3739
3740   if (!BlockAddressUsers.empty()) {
3741     Vals.resize(BlockAddressUsers.size());
3742     for (auto I : llvm::enumerate(BlockAddressUsers))
3743       Vals[I.index()] = VE.getValueID(I.value());
3744     Stream.EmitRecord(bitc::FUNC_CODE_BLOCKADDR_USERS, Vals);
3745     Vals.clear();
3746   }
3747
3748   // Emit names for all the instructions etc.
3749   if (auto *Symtab = F.getValueSymbolTable())
3750     writeFunctionLevelValueSymbolTable(*Symtab);
3751
3752   if (NeedsMetadataAttachment)
3753     writeFunctionMetadataAttachment(F);
3754   if (VE.shouldPreserveUseListOrder())
3755     writeUseListBlock(&F);
3756   VE.purgeFunction();
3757   Stream.ExitBlock();
3758 }
3759
3760 // Emit blockinfo, which defines the standard abbreviations etc.
3761 void ModuleBitcodeWriter::writeBlockInfo() {
3762   // We only want to emit block info records for blocks that have multiple
3763   // instances: CONSTANTS_BLOCK, FUNCTION_BLOCK and VALUE_SYMTAB_BLOCK.
3764   // Other blocks can define their abbrevs inline.
3765   Stream.EnterBlockInfoBlock();
3766
3767   { // 8-bit fixed-width VST_CODE_ENTRY/VST_CODE_BBENTRY strings.
3768     auto Abbv = std::make_shared<BitCodeAbbrev>();
3769     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3));
3770     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3771     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3772     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
3773     if (Stream.EmitBlockInfoAbbrev(bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3774         VST_ENTRY_8_ABBREV)
3775       llvm_unreachable("Unexpected abbrev ordering!");
3776   }
3777
3778   { // 7-bit fixed width VST_CODE_ENTRY strings.
3779     auto Abbv = std::make_shared<BitCodeAbbrev>();
3780     Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_ENTRY));
3781     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3782     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3783     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
3784     if (Stream.EmitBlockInfoAbbrev(bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3785         VST_ENTRY_7_ABBREV)
3786       llvm_unreachable("Unexpected abbrev ordering!");
3787   }
3788   { // 6-bit char6 VST_CODE_ENTRY strings.
3789     auto Abbv = std::make_shared<BitCodeAbbrev>();
3790     Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_ENTRY));
3791     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3792     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3793     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
3794     if (Stream.EmitBlockInfoAbbrev(bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3795         VST_ENTRY_6_ABBREV)
3796       llvm_unreachable("Unexpected abbrev ordering!");
3797   }
3798   { // 6-bit char6 VST_CODE_BBENTRY strings.
3799     auto Abbv = std::make_shared<BitCodeAbbrev>();
3800     Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_BBENTRY));
3801     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3802     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3803     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
3804     if (Stream.EmitBlockInfoAbbrev(bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3805         VST_BBENTRY_6_ABBREV)
3806       llvm_unreachable("Unexpected abbrev ordering!");
3807   }
3808
3809   { // SETTYPE abbrev for CONSTANTS_BLOCK.
3810     auto Abbv = std::make_shared<BitCodeAbbrev>();
3811     Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_SETTYPE));
3812     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
3813                               VE.computeBitsRequiredForTypeIndices()));
3814     if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3815         CONSTANTS_SETTYPE_ABBREV)
3816       llvm_unreachable("Unexpected abbrev ordering!");
3817   }
3818
3819   { // INTEGER abbrev for CONSTANTS_BLOCK.
3820     auto Abbv = std::make_shared<BitCodeAbbrev>();
3821     Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_INTEGER));
3822     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3823     if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3824         CONSTANTS_INTEGER_ABBREV)
3825       llvm_unreachable("Unexpected abbrev ordering!");
3826   }
3827
3828   { // CE_CAST abbrev for CONSTANTS_BLOCK.
3829     auto Abbv = std::make_shared<BitCodeAbbrev>();
3830     Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_CE_CAST));
3831     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4));  // cast opc
3832     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,       // typeid
3833                               VE.computeBitsRequiredForTypeIndices()));
3834     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));    // value id
3835
3836     if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3837         CONSTANTS_CE_CAST_Abbrev)
3838       llvm_unreachable("Unexpected abbrev ordering!");
3839   }
3840   { // NULL abbrev for CONSTANTS_BLOCK.
3841     auto Abbv = std::make_shared<BitCodeAbbrev>();
3842     Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_NULL));
3843     if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3844         CONSTANTS_NULL_Abbrev)
3845       llvm_unreachable("Unexpected abbrev ordering!");
3846   }
3847
3848   // FIXME: This should only use space for first class types!
3849
3850   { // INST_LOAD abbrev for FUNCTION_BLOCK.
3851     auto Abbv = std::make_shared<BitCodeAbbrev>();
3852     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_LOAD));
3853     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Ptr
3854     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,    // dest ty
3855                               VE.computeBitsRequiredForTypeIndices()));
3856     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // Align
3857     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile
3858     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
3859         FUNCTION_INST_LOAD_ABBREV)
3860       llvm_unreachable("Unexpected abbrev ordering!");
3861   }
3862   { // INST_UNOP abbrev for FUNCTION_BLOCK.
3863     auto Abbv = std::make_shared<BitCodeAbbrev>();
3864     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
3865     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS
3866     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
3867     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
3868         FUNCTION_INST_UNOP_ABBREV)
3869       llvm_unreachable("Unexpected abbrev ordering!");
3870   }
3871   { // INST_UNOP_FLAGS abbrev for FUNCTION_BLOCK.
3872     auto Abbv = std::make_shared<BitCodeAbbrev>();
3873     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
3874     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS
3875     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
3876     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
3877     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
3878         FUNCTION_INST_UNOP_FLAGS_ABBREV)
3879       llvm_unreachable("Unexpected abbrev ordering!");
3880   }
3881   { // INST_BINOP abbrev for FUNCTION_BLOCK.
3882     auto Abbv = std::make_shared<BitCodeAbbrev>();
3883     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
3884     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS
3885     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // RHS
3886     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
3887     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
3888         FUNCTION_INST_BINOP_ABBREV)
3889       llvm_unreachable("Unexpected abbrev ordering!");
3890   }
3891   { // INST_BINOP_FLAGS abbrev for FUNCTION_BLOCK.
3892     auto Abbv = std::make_shared<BitCodeAbbrev>();
3893     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
3894     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS
3895     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // RHS
3896     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
3897     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
3898     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
3899         FUNCTION_INST_BINOP_FLAGS_ABBREV)
3900       llvm_unreachable("Unexpected abbrev ordering!");
3901   }
3902   { // INST_CAST abbrev for FUNCTION_BLOCK.
3903     auto Abbv = std::make_shared<BitCodeAbbrev>();
3904     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
3905     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));    // OpVal
3906     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,       // dest ty
3907                               VE.computeBitsRequiredForTypeIndices()));
3908     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4));  // opc
3909     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
3910         FUNCTION_INST_CAST_ABBREV)
3911       llvm_unreachable("Unexpected abbrev ordering!");
3912   }
3913   { // INST_CAST_FLAGS abbrev for FUNCTION_BLOCK.
3914     auto Abbv = std::make_shared<BitCodeAbbrev>();
3915     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
3916     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // OpVal
3917     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,    // dest ty
3918                               VE.computeBitsRequiredForTypeIndices()));
3919     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
3920     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
3921     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
3922         FUNCTION_INST_CAST_FLAGS_ABBREV)
3923       llvm_unreachable("Unexpected abbrev ordering!");
3924   }
3925
3926   { // INST_RET abbrev for FUNCTION_BLOCK.
3927     auto Abbv = std::make_shared<BitCodeAbbrev>();
3928     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET));
3929     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
3930         FUNCTION_INST_RET_VOID_ABBREV)
3931       llvm_unreachable("Unexpected abbrev ordering!");
3932   }
3933   { // INST_RET abbrev for FUNCTION_BLOCK.
3934     auto Abbv = std::make_shared<BitCodeAbbrev>();
3935     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET));
3936     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // ValID
3937     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
3938         FUNCTION_INST_RET_VAL_ABBREV)
3939       llvm_unreachable("Unexpected abbrev ordering!");
3940   }
3941   { // INST_UNREACHABLE abbrev for FUNCTION_BLOCK.
3942     auto Abbv = std::make_shared<BitCodeAbbrev>();
3943     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNREACHABLE));
3944     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
3945         FUNCTION_INST_UNREACHABLE_ABBREV)
3946       llvm_unreachable("Unexpected abbrev ordering!");
3947   }
3948   {
3949     auto Abbv = std::make_shared<BitCodeAbbrev>();
3950     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_GEP));
3951     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3));
3952     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // dest ty
3953                               Log2_32_Ceil(VE.getTypes().size() + 1)));
3954     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3955     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
3956     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
3957         FUNCTION_INST_GEP_ABBREV)
3958       llvm_unreachable("Unexpected abbrev ordering!");
3959   }
3960   {
3961     auto Abbv = std::make_shared<BitCodeAbbrev>();
3962     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE));
3963     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // dbgloc
3964     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // var
3965     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // expr
3966     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // val
3967     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
3968         FUNCTION_DEBUG_RECORD_VALUE_ABBREV)
3969       llvm_unreachable("Unexpected abbrev ordering! 1");
3970   }
3971   Stream.ExitBlock();
3972 }
3973
3974 /// Write the module path strings, currently only used when generating
3975 /// a combined index file.
3976 void IndexBitcodeWriter::writeModStrings() {
3977   Stream.EnterSubblock(bitc::MODULE_STRTAB_BLOCK_ID, 3);
3978
3979   // TODO: See which abbrev sizes we actually need to emit
3980
3981   // 8-bit fixed-width MST_ENTRY strings.
3982   auto Abbv = std::make_shared<BitCodeAbbrev>();
3983   Abbv->Add(BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
3984   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3985   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3986   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
3987   unsigned Abbrev8Bit = Stream.EmitAbbrev(std::move(Abbv));
3988
3989   // 7-bit fixed width MST_ENTRY strings.
3990   Abbv = std::make_shared<BitCodeAbbrev>();
3991   Abbv->Add(BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
3992   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3993   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3994   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
3995   unsigned Abbrev7Bit = Stream.EmitAbbrev(std::move(Abbv));
3996
3997   // 6-bit char6 MST_ENTRY strings.
3998   Abbv = std::make_shared<BitCodeAbbrev>();
3999   Abbv->Add(BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
4000   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4001   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4002   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
4003   unsigned Abbrev6Bit = Stream.EmitAbbrev(std::move(Abbv));
4004
4005   // Module Hash, 160 bits SHA1. Optionally, emitted after each MST_CODE_ENTRY.
4006   Abbv = std::make_shared<BitCodeAbbrev>();
4007   Abbv->Add(BitCodeAbbrevOp(bitc::MST_CODE_HASH));
4008   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4009   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4010   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4011   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4012   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4013   unsigned AbbrevHash = Stream.EmitAbbrev(std::move(Abbv));
4014
4015   SmallVector<unsigned, 64> Vals;
4016   forEachModule([&](const StringMapEntry<ModuleHash> &MPSE) {
4017     StringRef Key = MPSE.getKey();
4018     const auto &Hash = MPSE.getValue();
4019     StringEncoding Bits = getStringEncoding(Key);
4020     unsigned AbbrevToUse = Abbrev8Bit;
4021     if (Bits == SE_Char6)
4022       AbbrevToUse = Abbrev6Bit;
4023     else if (Bits == SE_Fixed7)
4024       AbbrevToUse = Abbrev7Bit;
4025
4026     auto ModuleId = ModuleIdMap.size();
4027     ModuleIdMap[Key] = ModuleId;
4028     Vals.push_back(ModuleId);
4029     Vals.append(Key.begin(), Key.end());
4030
4031     // Emit the finished record.
4032     Stream.EmitRecord(bitc::MST_CODE_ENTRY, Vals, AbbrevToUse);
4033
4034     // Emit an optional hash for the module now
4035     if (llvm::any_of(Hash, [](uint32_t H) { return H; })) {
4036       Vals.assign(Hash.begin(), Hash.end());
4037       // Emit the hash record.
4038       Stream.EmitRecord(bitc::MST_CODE_HASH, Vals, AbbrevHash);
4039     }
4040
4041     Vals.clear();
4042   });
4043   Stream.ExitBlock();
4044 }
4045
4046 /// Write the function type metadata related records that need to appear before
4047 /// a function summary entry (whether per-module or combined).
4048 template <typename Fn>
4049 static void writeFunctionTypeMetadataRecords(BitstreamWriter &Stream,
4050                                              FunctionSummary *FS,
4051                                              Fn GetValueID) {
4052   if (!FS->type_tests().empty())
4053     Stream.EmitRecord(bitc::FS_TYPE_TESTS, FS->type_tests());
4054
4055   SmallVector<uint64_t, 64> Record;
4056
4057   auto WriteVFuncIdVec = [&](uint64_t Ty,
4058                              ArrayRef<FunctionSummary::VFuncId> VFs) {
4059     if (VFs.empty())
4060       return;
4061     Record.clear();
4062     for (auto &VF : VFs) {
4063       Record.push_back(VF.GUID);
4064       Record.push_back(VF.Offset);
4065     }
4066     Stream.EmitRecord(Ty, Record);
4067   };
4068
4069   WriteVFuncIdVec(bitc::FS_TYPE_TEST_ASSUME_VCALLS,
4070                   FS->type_test_assume_vcalls());
4071   WriteVFuncIdVec(bitc::FS_TYPE_CHECKED_LOAD_VCALLS,
4072                   FS->type_checked_load_vcalls());
4073
4074   auto WriteConstVCallVec = [&](uint64_t Ty,
4075                                 ArrayRef<FunctionSummary::ConstVCall> VCs) {
4076     for (auto &VC : VCs) {
4077       Record.clear();
4078       Record.push_back(VC.VFunc.GUID);
4079       Record.push_back(VC.VFunc.Offset);
4080       llvm::append_range(Record, VC.Args);
4081       Stream.EmitRecord(Ty, Record);
4082     }
4083   };
4084
4085   WriteConstVCallVec(bitc::FS_TYPE_TEST_ASSUME_CONST_VCALL,
4086                      FS->type_test_assume_const_vcalls());
4087   WriteConstVCallVec(bitc::FS_TYPE_CHECKED_LOAD_CONST_VCALL,
4088                      FS->type_checked_load_const_vcalls());
4089
4090   auto WriteRange = [&](ConstantRange Range) {
4091     Range = Range.sextOrTrunc(FunctionSummary::ParamAccess::RangeWidth);
4092     assert(Range.getLower().getNumWords() == 1);
4093     assert(Range.getUpper().getNumWords() == 1);
4094     emitSignedInt64(Record, *Range.getLower().getRawData());
4095     emitSignedInt64(Record, *Range.getUpper().getRawData());
4096   };
4097
4098   if (!FS->paramAccesses().empty()) {
4099     Record.clear();
4100     for (auto &Arg : FS->paramAccesses()) {
4101       size_t UndoSize = Record.size();
4102       Record.push_back(Arg.ParamNo);
4103       WriteRange(Arg.Use);
4104       Record.push_back(Arg.Calls.size());
4105       for (auto &Call : Arg.Calls) {
4106         Record.push_back(Call.ParamNo);
4107         std::optional<unsigned> ValueID = GetValueID(Call.Callee);
4108         if (!ValueID) {
4109           // If ValueID is unknown we can't drop just this call, we must drop
4110           // entire parameter.
4111           Record.resize(UndoSize);
4112           break;
4113         }
4114         Record.push_back(*ValueID);
4115         WriteRange(Call.Offsets);
4116       }
4117     }
4118     if (!Record.empty())
4119       Stream.EmitRecord(bitc::FS_PARAM_ACCESS, Record);
4120   }
4121 }
4122
4123 /// Collect type IDs from type tests used by function.
4124 static void
4125 getReferencedTypeIds(FunctionSummary *FS,
4126                      std::set<GlobalValue::GUID> &ReferencedTypeIds) {
4127   if (!FS->type_tests().empty())
4128     for (auto &TT : FS->type_tests())
4129       ReferencedTypeIds.insert(TT);
4130
4131   auto GetReferencedTypesFromVFuncIdVec =
4132       [&](ArrayRef<FunctionSummary::VFuncId> VFs) {
4133         for (auto &VF : VFs)
4134           ReferencedTypeIds.insert(VF.GUID);
4135       };
4136
4137   GetReferencedTypesFromVFuncIdVec(FS->type_test_assume_vcalls());
4138   GetReferencedTypesFromVFuncIdVec(FS->type_checked_load_vcalls());
4139
4140   auto GetReferencedTypesFromConstVCallVec =
4141       [&](ArrayRef<FunctionSummary::ConstVCall> VCs) {
4142         for (auto &VC : VCs)
4143           ReferencedTypeIds.insert(VC.VFunc.GUID);
4144       };
4145
4146   GetReferencedTypesFromConstVCallVec(FS->type_test_assume_const_vcalls());
4147   GetReferencedTypesFromConstVCallVec(FS->type_checked_load_const_vcalls());
4148 }
4149
4150 static void writeWholeProgramDevirtResolutionByArg(
4151     SmallVector<uint64_t, 64> &NameVals, const std::vector<uint64_t> &args,
4152     const WholeProgramDevirtResolution::ByArg &ByArg) {
4153   NameVals.push_back(args.size());
4154   llvm::append_range(NameVals, args);
4155
4156   NameVals.push_back(ByArg.TheKind);
4157   NameVals.push_back(ByArg.Info);
4158   NameVals.push_back(ByArg.Byte);
4159   NameVals.push_back(ByArg.Bit);
4160 }
4161
4162 static void writeWholeProgramDevirtResolution(
4163     SmallVector<uint64_t, 64> &NameVals, StringTableBuilder &StrtabBuilder,
4164     uint64_t Id, const WholeProgramDevirtResolution &Wpd) {
4165   NameVals.push_back(Id);
4166
4167   NameVals.push_back(Wpd.TheKind);
4168   NameVals.push_back(StrtabBuilder.add(Wpd.SingleImplName));
4169   NameVals.push_back(Wpd.SingleImplName.size());
4170
4171   NameVals.push_back(Wpd.ResByArg.size());
4172   for (auto &A : Wpd.ResByArg)
4173     writeWholeProgramDevirtResolutionByArg(NameVals, A.first, A.second);
4174 }
4175
4176 static void writeTypeIdSummaryRecord(SmallVector<uint64_t, 64> &NameVals,
4177                                      StringTableBuilder &StrtabBuilder,
4178                                      StringRef Id,
4179                                      const TypeIdSummary &Summary) {
4180   NameVals.push_back(StrtabBuilder.add(Id));
4181   NameVals.push_back(Id.size());
4182
4183   NameVals.push_back(Summary.TTRes.TheKind);
4184   NameVals.push_back(Summary.TTRes.SizeM1BitWidth);
4185   NameVals.push_back(Summary.TTRes.AlignLog2);
4186   NameVals.push_back(Summary.TTRes.SizeM1);
4187   NameVals.push_back(Summary.TTRes.BitMask);
4188   NameVals.push_back(Summary.TTRes.InlineBits);
4189
4190   for (auto &W : Summary.WPDRes)
4191     writeWholeProgramDevirtResolution(NameVals, StrtabBuilder, W.first,
4192                                       W.second);
4193 }
4194
4195 static void writeTypeIdCompatibleVtableSummaryRecord(
4196     SmallVector<uint64_t, 64> &NameVals, StringTableBuilder &StrtabBuilder,
4197     StringRef Id, const TypeIdCompatibleVtableInfo &Summary,
4198     ValueEnumerator &VE) {
4199   NameVals.push_back(StrtabBuilder.add(Id));
4200   NameVals.push_back(Id.size());
4201
4202   for (auto &P : Summary) {
4203     NameVals.push_back(P.AddressPointOffset);
4204     NameVals.push_back(VE.getValueID(P.VTableVI.getValue()));
4205   }
4206 }
4207
4208 // Adds the allocation contexts to the CallStacks map. We simply use the
4209 // size at the time the context was added as the CallStackId. This works because
4210 // when we look up the call stacks later on we process the function summaries
4211 // and their allocation records in the same exact order.
4212 static void collectMemProfCallStacks(
4213     FunctionSummary *FS, std::function<LinearFrameId(unsigned)> GetStackIndex,
4214     MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> &CallStacks) {
4215   // The interfaces in ProfileData/MemProf.h use a type alias for a stack frame
4216   // id offset into the index of the full stack frames. The ModuleSummaryIndex
4217   // currently uses unsigned. Make sure these stay in sync.
4218   static_assert(std::is_same_v<LinearFrameId, unsigned>);
4219   for (auto &AI : FS->allocs()) {
4220     for (auto &MIB : AI.MIBs) {
4221       SmallVector<unsigned> StackIdIndices;
4222       StackIdIndices.reserve(MIB.StackIdIndices.size());
4223       for (auto Id : MIB.StackIdIndices)
4224         StackIdIndices.push_back(GetStackIndex(Id));
4225       // The CallStackId is the size at the time this context was inserted.
4226       CallStacks.insert({CallStacks.size(), StackIdIndices});
4227     }
4228   }
4229 }
4230
4231 // Build the radix tree from the accumulated CallStacks, write out the resulting
4232 // linearized radix tree array, and return the map of call stack positions into
4233 // this array for use when writing the allocation records. The returned map is
4234 // indexed by a CallStackId which in this case is implicitly determined by the
4235 // order of function summaries and their allocation infos being written.
4236 static DenseMap<CallStackId, LinearCallStackId> writeMemoryProfileRadixTree(
4237     MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> &&CallStacks,
4238     BitstreamWriter &Stream, unsigned RadixAbbrev) {
4239   assert(!CallStacks.empty());
4240   DenseMap<unsigned, FrameStat> FrameHistogram =
4241       computeFrameHistogram<LinearFrameId>(CallStacks);
4242   CallStackRadixTreeBuilder<LinearFrameId> Builder;
4243   // We don't need a MemProfFrameIndexes map as we have already converted the
4244   // full stack id hash to a linear offset into the StackIds array.
4245   Builder.build(std::move(CallStacks), /*MemProfFrameIndexes=*/nullptr,
4246                 FrameHistogram);
4247   Stream.EmitRecord(bitc::FS_CONTEXT_RADIX_TREE_ARRAY, Builder.getRadixArray(),
4248                     RadixAbbrev);
4249   return Builder.takeCallStackPos();
4250 }
4251
4252 static void writeFunctionHeapProfileRecords(
4253     BitstreamWriter &Stream, FunctionSummary *FS, unsigned CallsiteAbbrev,
4254     unsigned AllocAbbrev, unsigned ContextIdAbbvId, bool PerModule,
4255     std::function<unsigned(const ValueInfo &VI)> GetValueID,
4256     std::function<unsigned(unsigned)> GetStackIndex,
4257     bool WriteContextSizeInfoIndex,
4258     DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
4259     CallStackId &CallStackCount) {
4260   SmallVector<uint64_t> Record;
4261
4262   for (auto &CI : FS->callsites()) {
4263     Record.clear();
4264     // Per module callsite clones should always have a single entry of
4265     // value 0.
4266     assert(!PerModule || (CI.Clones.size() == 1 && CI.Clones[0] == 0));
4267     Record.push_back(GetValueID(CI.Callee));
4268     if (!PerModule) {
4269       Record.push_back(CI.StackIdIndices.size());
4270       Record.push_back(CI.Clones.size());
4271     }
4272     for (auto Id : CI.StackIdIndices)
4273       Record.push_back(GetStackIndex(Id));
4274     if (!PerModule) {
4275       for (auto V : CI.Clones)
4276         Record.push_back(V);
4277     }
4278     Stream.EmitRecord(PerModule ? bitc::FS_PERMODULE_CALLSITE_INFO
4279                                 : bitc::FS_COMBINED_CALLSITE_INFO,
4280                       Record, CallsiteAbbrev);
4281   }
4282
4283   for (auto &AI : FS->allocs()) {
4284     Record.clear();
4285     // Per module alloc versions should always have a single entry of
4286     // value 0.
4287     assert(!PerModule || (AI.Versions.size() == 1 && AI.Versions[0] == 0));
4288     Record.push_back(AI.MIBs.size());
4289     if (!PerModule)
4290       Record.push_back(AI.Versions.size());
4291     for (auto &MIB : AI.MIBs) {
4292       Record.push_back((uint8_t)MIB.AllocType);
4293       // Record the index into the radix tree array for this context.
4294       assert(CallStackCount <= CallStackPos.size());
4295       Record.push_back(CallStackPos[CallStackCount++]);
4296     }
4297     if (!PerModule) {
4298       for (auto V : AI.Versions)
4299         Record.push_back(V);
4300     }
4301     assert(AI.ContextSizeInfos.empty() ||
4302            AI.ContextSizeInfos.size() == AI.MIBs.size());
4303     // Optionally emit the context size information if it exists.
4304     if (WriteContextSizeInfoIndex && !AI.ContextSizeInfos.empty()) {
4305       // The abbreviation id for the context ids record should have been created
4306       // if we are emitting the per-module index, which is where we write this
4307       // info.
4308       assert(ContextIdAbbvId);
4309       SmallVector<uint32_t> ContextIds;
4310       // At least one context id per ContextSizeInfos entry (MIB), broken into 2
4311       // halves.
4312       ContextIds.reserve(AI.ContextSizeInfos.size() * 2);
4313       for (auto &Infos : AI.ContextSizeInfos) {
4314         Record.push_back(Infos.size());
4315         for (auto [FullStackId, TotalSize] : Infos) {
4316           // The context ids are emitted separately as a fixed width array,
4317           // which is more efficient than a VBR given that these hashes are
4318           // typically close to 64-bits. The max fixed width entry is 32 bits so
4319           // it is split into 2.
4320           ContextIds.push_back(static_cast<uint32_t>(FullStackId >> 32));
4321           ContextIds.push_back(static_cast<uint32_t>(FullStackId));
4322           Record.push_back(TotalSize);
4323         }
4324       }
4325       // The context ids are expected by the reader to immediately precede the
4326       // associated alloc info record.
4327       Stream.EmitRecord(bitc::FS_ALLOC_CONTEXT_IDS, ContextIds,
4328                         ContextIdAbbvId);
4329     }
4330     Stream.EmitRecord(PerModule ? bitc::FS_PERMODULE_ALLOC_INFO
4331                                 : bitc::FS_COMBINED_ALLOC_INFO,
4332                       Record, AllocAbbrev);
4333   }
4334 }
4335
4336 // Helper to emit a single function summary record.
4337 void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord(
4338     SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
4339     unsigned ValueID, unsigned FSCallsRelBFAbbrev,
4340     unsigned FSCallsProfileAbbrev, unsigned CallsiteAbbrev,
4341     unsigned AllocAbbrev, unsigned ContextIdAbbvId, const Function &F,
4342     DenseMap<CallStackId, LinearCallStackId> &CallStackPos,
4343     CallStackId &CallStackCount) {
4344   NameVals.push_back(ValueID);
4345
4346   FunctionSummary *FS = cast<FunctionSummary>(Summary);
4347
4348   writeFunctionTypeMetadataRecords(
4349       Stream, FS, [&](const ValueInfo &VI) -> std::optional<unsigned> {
4350         return {VE.getValueID(VI.getValue())};
4351       });
4352
4353   writeFunctionHeapProfileRecords(
4354       Stream, FS, CallsiteAbbrev, AllocAbbrev, ContextIdAbbvId,
4355       /*PerModule*/ true,
4356       /*GetValueId*/ [&](const ValueInfo &VI) { return getValueId(VI); },
4357       /*GetStackIndex*/ [&](unsigned I) { return I; },
4358       /*WriteContextSizeInfoIndex*/ true, CallStackPos, CallStackCount);
4359
4360   auto SpecialRefCnts = FS->specialRefCounts();
4361   NameVals.push_back(getEncodedGVSummaryFlags(FS->flags()));
4362   NameVals.push_back(FS->instCount());
4363   NameVals.push_back(getEncodedFFlags(FS->fflags()));
4364   NameVals.push_back(FS->refs().size());
4365   NameVals.push_back(SpecialRefCnts.first);  // rorefcnt
4366   NameVals.push_back(SpecialRefCnts.second); // worefcnt
4367
4368   for (auto &RI : FS->refs())
4369     NameVals.push_back(getValueId(RI));
4370
4371   const bool UseRelBFRecord =
4372       WriteRelBFToSummary && !F.hasProfileData() &&
4373       ForceSummaryEdgesCold == FunctionSummary::FSHT_None;
4374   for (auto &ECI : FS->calls()) {
4375     NameVals.push_back(getValueId(ECI.first));
4376     if (UseRelBFRecord)
4377       NameVals.push_back(getEncodedRelBFCallEdgeInfo(ECI.second));
4378     else
4379       NameVals.push_back(getEncodedHotnessCallEdgeInfo(ECI.second));
4380   }
4381
4382   unsigned FSAbbrev =
4383       (UseRelBFRecord ? FSCallsRelBFAbbrev : FSCallsProfileAbbrev);
4384   unsigned Code =
4385       (UseRelBFRecord ? bitc::FS_PERMODULE_RELBF : bitc::FS_PERMODULE_PROFILE);
4386
4387   // Emit the finished record.
4388   Stream.EmitRecord(Code, NameVals, FSAbbrev);
4389   NameVals.clear();
4390 }
4391
4392 // Collect the global value references in the given variable's initializer,
4393 // and emit them in a summary record.
4394 void ModuleBitcodeWriterBase::writeModuleLevelReferences(
4395     const GlobalVariable &V, SmallVector<uint64_t, 64> &NameVals,
4396     unsigned FSModRefsAbbrev, unsigned FSModVTableRefsAbbrev) {
4397   auto VI = Index->getValueInfo(V.getGUID());
4398   if (!VI || VI.getSummaryList().empty()) {
4399     // Only declarations should not have a summary (a declaration might however
4400     // have a summary if the def was in module level asm).
4401     assert(V.isDeclaration());
4402     return;
4403   }
4404   auto *Summary = VI.getSummaryList()[0].get();
4405   NameVals.push_back(VE.getValueID(&V));
4406   GlobalVarSummary *VS = cast<GlobalVarSummary>(Summary);
4407   NameVals.push_back(getEncodedGVSummaryFlags(VS->flags()));
4408   NameVals.push_back(getEncodedGVarFlags(VS->varflags()));
4409
4410   auto VTableFuncs = VS->vTableFuncs();
4411   if (!VTableFuncs.empty())
4412     NameVals.push_back(VS->refs().size());
4413
4414   unsigned SizeBeforeRefs = NameVals.size();
4415   for (auto &RI : VS->refs())
4416     NameVals.push_back(VE.getValueID(RI.getValue()));
4417   // Sort the refs for determinism output, the vector returned by FS->refs() has
4418   // been initialized from a DenseSet.
4419   llvm::sort(drop_begin(NameVals, SizeBeforeRefs));
4420
4421   if (VTableFuncs.empty())
4422     Stream.EmitRecord(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS, NameVals,
4423                       FSModRefsAbbrev);
4424   else {
4425     // VTableFuncs pairs should already be sorted by offset.
4426     for (auto &P : VTableFuncs) {
4427       NameVals.push_back(VE.getValueID(P.FuncVI.getValue()));
4428       NameVals.push_back(P.VTableOffset);
4429     }
4430
4431     Stream.EmitRecord(bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS, NameVals,
4432                       FSModVTableRefsAbbrev);
4433   }
4434   NameVals.clear();
4435 }
4436
4437 /// Emit the per-module summary section alongside the rest of
4438 /// the module's bitcode.
4439 void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() {
4440   // By default we compile with ThinLTO if the module has a summary, but the
4441   // client can request full LTO with a module flag.
4442   bool IsThinLTO = true;
4443   if (auto *MD =
4444           mdconst::extract_or_null<ConstantInt>(M.getModuleFlag("ThinLTO")))
4445     IsThinLTO = MD->getZExtValue();
4446   Stream.EnterSubblock(IsThinLTO ? bitc::GLOBALVAL_SUMMARY_BLOCK_ID
4447                                  : bitc::FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID,
4448                        4);
4449
4450   Stream.EmitRecord(
4451       bitc::FS_VERSION,
4452       ArrayRef<uint64_t>{ModuleSummaryIndex::BitcodeSummaryVersion});
4453
4454   // Write the index flags.
4455   uint64_t Flags = 0;
4456   // Bits 1-3 are set only in the combined index, skip them.
4457   if (Index->enableSplitLTOUnit())
4458     Flags |= 0x8;
4459   if (Index->hasUnifiedLTO())
4460     Flags |= 0x200;
4461
4462   Stream.EmitRecord(bitc::FS_FLAGS, ArrayRef<uint64_t>{Flags});
4463
4464   if (Index->begin() == Index->end()) {
4465     Stream.ExitBlock();
4466     return;
4467   }
4468
4469   auto Abbv = std::make_shared<BitCodeAbbrev>();
4470   Abbv->Add(BitCodeAbbrevOp(bitc::FS_VALUE_GUID));
4471   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4472   // GUIDS often use up most of 64-bits, so encode as two Fixed 32.
4473   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4474   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4475   unsigned ValueGuidAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4476
4477   for (const auto &GVI : valueIds()) {
4478     Stream.EmitRecord(bitc::FS_VALUE_GUID,
4479                       ArrayRef<uint32_t>{GVI.second,
4480                                          static_cast<uint32_t>(GVI.first >> 32),
4481                                          static_cast<uint32_t>(GVI.first)},
4482                       ValueGuidAbbrev);
4483   }
4484
4485   if (!Index->stackIds().empty()) {
4486     auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
4487     StackIdAbbv->Add(BitCodeAbbrevOp(bitc::FS_STACK_IDS));
4488     // numids x stackid
4489     StackIdAbbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4490     // The stack ids are hashes that are close to 64 bits in size, so emitting
4491     // as a pair of 32-bit fixed-width values is more efficient than a VBR.
4492     StackIdAbbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4493     unsigned StackIdAbbvId = Stream.EmitAbbrev(std::move(StackIdAbbv));
4494     SmallVector<uint32_t> Vals;
4495     Vals.reserve(Index->stackIds().size() * 2);
4496     for (auto Id : Index->stackIds()) {
4497       Vals.push_back(static_cast<uint32_t>(Id >> 32));
4498       Vals.push_back(static_cast<uint32_t>(Id));
4499     }
4500     Stream.EmitRecord(bitc::FS_STACK_IDS, Vals, StackIdAbbvId);
4501   }
4502
4503   // n x context id
4504   auto ContextIdAbbv = std::make_shared<BitCodeAbbrev>();
4505   ContextIdAbbv->Add(BitCodeAbbrevOp(bitc::FS_ALLOC_CONTEXT_IDS));
4506   ContextIdAbbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4507   // The context ids are hashes that are close to 64 bits in size, so emitting
4508   // as a pair of 32-bit fixed-width values is more efficient than a VBR.
4509   ContextIdAbbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4510   unsigned ContextIdAbbvId = Stream.EmitAbbrev(std::move(ContextIdAbbv));
4511
4512   // Abbrev for FS_PERMODULE_PROFILE.
4513   Abbv = std::make_shared<BitCodeAbbrev>();
4514   Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE));
4515   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // valueid
4516   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // flags
4517   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // instcount
4518   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // fflags
4519   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // numrefs
4520   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // rorefcnt
4521   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // worefcnt
4522   // numrefs x valueid, n x (valueid, hotness+tailcall flags)
4523   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4524   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4525   unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4526
4527   // Abbrev for FS_PERMODULE_RELBF.
4528   Abbv = std::make_shared<BitCodeAbbrev>();
4529   Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_RELBF));
4530   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // valueid
4531   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // flags
4532   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // instcount
4533   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // fflags
4534   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // numrefs
4535   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // rorefcnt
4536   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // worefcnt
4537   // numrefs x valueid, n x (valueid, rel_block_freq+tailcall])
4538   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4539   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4540   unsigned FSCallsRelBFAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4541
4542   // Abbrev for FS_PERMODULE_GLOBALVAR_INIT_REFS.
4543   Abbv = std::make_shared<BitCodeAbbrev>();
4544   Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS));
4545   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4546   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4547   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));  // valueids
4548   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4549   unsigned FSModRefsAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4550
4551   // Abbrev for FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS.
4552   Abbv = std::make_shared<BitCodeAbbrev>();
4553   Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS));
4554   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4555   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4556   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4557   // numrefs x valueid, n x (valueid , offset)
4558   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4559   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4560   unsigned FSModVTableRefsAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4561
4562   // Abbrev for FS_ALIAS.
4563   Abbv = std::make_shared<BitCodeAbbrev>();
4564   Abbv->Add(BitCodeAbbrevOp(bitc::FS_ALIAS));
4565   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // valueid
4566   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // flags
4567   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // valueid
4568   unsigned FSAliasAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4569
4570   // Abbrev for FS_TYPE_ID_METADATA
4571   Abbv = std::make_shared<BitCodeAbbrev>();
4572   Abbv->Add(BitCodeAbbrevOp(bitc::FS_TYPE_ID_METADATA));
4573   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // typeid strtab index
4574   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // typeid length
4575   // n x (valueid , offset)
4576   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4577   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4578   unsigned TypeIdCompatibleVtableAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4579
4580   Abbv = std::make_shared<BitCodeAbbrev>();
4581   Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_CALLSITE_INFO));
4582   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4583   // n x stackidindex
4584   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4585   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4586   unsigned CallsiteAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4587
4588   Abbv = std::make_shared<BitCodeAbbrev>();
4589   Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_ALLOC_INFO));
4590   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // nummib
4591   // n x (alloc type, context radix tree index)
4592   // optional: nummib x (numcontext x total size)
4593   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4594   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4595   unsigned AllocAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4596
4597   Abbv = std::make_shared<BitCodeAbbrev>();
4598   Abbv->Add(BitCodeAbbrevOp(bitc::FS_CONTEXT_RADIX_TREE_ARRAY));
4599   // n x entry
4600   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4601   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4602   unsigned RadixAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4603
4604   // First walk through all the functions and collect the allocation contexts in
4605   // their associated summaries, for use in constructing a radix tree of
4606   // contexts. Note that we need to do this in the same order as the functions
4607   // are processed further below since the call stack positions in the resulting
4608   // radix tree array are identified based on this order.
4609   MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> CallStacks;
4610   for (const Function &F : M) {
4611     // Summary emission does not support anonymous functions, they have to be
4612     // renamed using the anonymous function renaming pass.
4613     if (!F.hasName())
4614       report_fatal_error("Unexpected anonymous function when writing summary");
4615
4616     ValueInfo VI = Index->getValueInfo(F.getGUID());
4617     if (!VI || VI.getSummaryList().empty()) {
4618       // Only declarations should not have a summary (a declaration might
4619       // however have a summary if the def was in module level asm).
4620       assert(F.isDeclaration());
4621       continue;
4622     }
4623     auto *Summary = VI.getSummaryList()[0].get();
4624     FunctionSummary *FS = cast<FunctionSummary>(Summary);
4625     collectMemProfCallStacks(
4626         FS, /*GetStackIndex*/ [](unsigned I) { return I; }, CallStacks);
4627   }
4628   // Finalize the radix tree, write it out, and get the map of positions in the
4629   // linearized tree array.
4630   DenseMap<CallStackId, LinearCallStackId> CallStackPos;
4631   if (!CallStacks.empty()) {
4632     CallStackPos =
4633         writeMemoryProfileRadixTree(std::move(CallStacks), Stream, RadixAbbrev);
4634   }
4635
4636   // Keep track of the current index into the CallStackPos map.
4637   CallStackId CallStackCount = 0;
4638
4639   SmallVector<uint64_t, 64> NameVals;
4640   // Iterate over the list of functions instead of the Index to
4641   // ensure the ordering is stable.
4642   for (const Function &F : M) {
4643     // Summary emission does not support anonymous functions, they have to
4644     // renamed using the anonymous function renaming pass.
4645     if (!F.hasName())
4646       report_fatal_error("Unexpected anonymous function when writing summary");
4647
4648     ValueInfo VI = Index->getValueInfo(F.getGUID());
4649     if (!VI || VI.getSummaryList().empty()) {
4650       // Only declarations should not have a summary (a declaration might
4651       // however have a summary if the def was in module level asm).
4652       assert(F.isDeclaration());
4653       continue;
4654     }
4655     auto *Summary = VI.getSummaryList()[0].get();
4656     writePerModuleFunctionSummaryRecord(
4657         NameVals, Summary, VE.getValueID(&F), FSCallsRelBFAbbrev,
4658         FSCallsProfileAbbrev, CallsiteAbbrev, AllocAbbrev, ContextIdAbbvId, F,
4659         CallStackPos, CallStackCount);
4660   }
4661
4662   // Capture references from GlobalVariable initializers, which are outside
4663   // of a function scope.
4664   for (const GlobalVariable &G : M.globals())
4665     writeModuleLevelReferences(G, NameVals, FSModRefsAbbrev,
4666                                FSModVTableRefsAbbrev);
4667
4668   for (const GlobalAlias &A : M.aliases()) {
4669     auto *Aliasee = A.getAliaseeObject();
4670     // Skip ifunc and nameless functions which don't have an entry in the
4671     // summary.
4672     if (!Aliasee->hasName() || isa<GlobalIFunc>(Aliasee))
4673       continue;
4674     auto AliasId = VE.getValueID(&A);
4675     auto AliaseeId = VE.getValueID(Aliasee);
4676     NameVals.push_back(AliasId);
4677     auto *Summary = Index->getGlobalValueSummary(A);
4678     AliasSummary *AS = cast<AliasSummary>(Summary);
4679     NameVals.push_back(getEncodedGVSummaryFlags(AS->flags()));
4680     NameVals.push_back(AliaseeId);
4681     Stream.EmitRecord(bitc::FS_ALIAS, NameVals, FSAliasAbbrev);
4682     NameVals.clear();
4683   }
4684
4685   for (auto &S : Index->typeIdCompatibleVtableMap()) {
4686     writeTypeIdCompatibleVtableSummaryRecord(NameVals, StrtabBuilder, S.first,
4687                                              S.second, VE);
4688     Stream.EmitRecord(bitc::FS_TYPE_ID_METADATA, NameVals,
4689                       TypeIdCompatibleVtableAbbrev);
4690     NameVals.clear();
4691   }
4692
4693   if (Index->getBlockCount())
4694     Stream.EmitRecord(bitc::FS_BLOCK_COUNT,
4695                       ArrayRef<uint64_t>{Index->getBlockCount()});
4696
4697   Stream.ExitBlock();
4698 }
4699
4700 /// Emit the combined summary section into the combined index file.
4701 void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
4702   Stream.EnterSubblock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID, 4);
4703   Stream.EmitRecord(
4704       bitc::FS_VERSION,
4705       ArrayRef<uint64_t>{ModuleSummaryIndex::BitcodeSummaryVersion});
4706
4707   // Write the index flags.
4708   Stream.EmitRecord(bitc::FS_FLAGS, ArrayRef<uint64_t>{Index.getFlags()});
4709
4710   auto Abbv = std::make_shared<BitCodeAbbrev>();
4711   Abbv->Add(BitCodeAbbrevOp(bitc::FS_VALUE_GUID));
4712   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4713   // GUIDS often use up most of 64-bits, so encode as two Fixed 32.
4714   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4715   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4716   unsigned ValueGuidAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4717
4718   for (const auto &GVI : valueIds()) {
4719     Stream.EmitRecord(bitc::FS_VALUE_GUID,
4720                       ArrayRef<uint32_t>{GVI.second,
4721                                          static_cast<uint32_t>(GVI.first >> 32),
4722                                          static_cast<uint32_t>(GVI.first)},
4723                       ValueGuidAbbrev);
4724   }
4725
4726   // Write the stack ids used by this index, which will be a subset of those in
4727   // the full index in the case of distributed indexes.
4728   if (!StackIds.empty()) {
4729     auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
4730     StackIdAbbv->Add(BitCodeAbbrevOp(bitc::FS_STACK_IDS));
4731     // numids x stackid
4732     StackIdAbbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4733     // The stack ids are hashes that are close to 64 bits in size, so emitting
4734     // as a pair of 32-bit fixed-width values is more efficient than a VBR.
4735     StackIdAbbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
4736     unsigned StackIdAbbvId = Stream.EmitAbbrev(std::move(StackIdAbbv));
4737     SmallVector<uint32_t> Vals;
4738     Vals.reserve(StackIds.size() * 2);
4739     for (auto Id : StackIds) {
4740       Vals.push_back(static_cast<uint32_t>(Id >> 32));
4741       Vals.push_back(static_cast<uint32_t>(Id));
4742     }
4743     Stream.EmitRecord(bitc::FS_STACK_IDS, Vals, StackIdAbbvId);
4744   }
4745
4746   // Abbrev for FS_COMBINED_PROFILE.
4747   Abbv = std::make_shared<BitCodeAbbrev>();
4748   Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_PROFILE));
4749   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // valueid
4750   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // modid
4751   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // flags
4752   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // instcount
4753   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // fflags
4754   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // entrycount
4755   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // numrefs
4756   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // rorefcnt
4757   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // worefcnt
4758   // numrefs x valueid, n x (valueid, hotness+tailcall flags)
4759   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4760   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4761   unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4762
4763   // Abbrev for FS_COMBINED_GLOBALVAR_INIT_REFS.
4764   Abbv = std::make_shared<BitCodeAbbrev>();
4765   Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS));
4766   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // valueid
4767   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // modid
4768   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // flags
4769   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));    // valueids
4770   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4771   unsigned FSModRefsAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4772
4773   // Abbrev for FS_COMBINED_ALIAS.
4774   Abbv = std::make_shared<BitCodeAbbrev>();
4775   Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_ALIAS));
4776   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // valueid
4777   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // modid
4778   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // flags
4779   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // valueid
4780   unsigned FSAliasAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4781
4782   Abbv = std::make_shared<BitCodeAbbrev>();
4783   Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_CALLSITE_INFO));
4784   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4785   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numstackindices
4786   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver
4787   // numstackindices x stackidindex, numver x version
4788   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4789   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4790   unsigned CallsiteAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4791
4792   Abbv = std::make_shared<BitCodeAbbrev>();
4793   Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_ALLOC_INFO));
4794   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // nummib
4795   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver
4796   // nummib x (alloc type, context radix tree index),
4797   // numver x version
4798   // optional: nummib x total size
4799   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4800   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4801   unsigned AllocAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4802
4803   Abbv = std::make_shared<BitCodeAbbrev>();
4804   Abbv->Add(BitCodeAbbrevOp(bitc::FS_CONTEXT_RADIX_TREE_ARRAY));
4805   // n x entry
4806   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4807   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4808   unsigned RadixAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4809
4810   auto shouldImportValueAsDecl = [&](GlobalValueSummary *GVS) -> bool {
4811     if (DecSummaries == nullptr)
4812       return false;
4813     return DecSummaries->count(GVS);
4814   };
4815
4816   // The aliases are emitted as a post-pass, and will point to the value
4817   // id of the aliasee. Save them in a vector for post-processing.
4818   SmallVector<AliasSummary *, 64> Aliases;
4819
4820   // Save the value id for each summary for alias emission.
4821   DenseMap<const GlobalValueSummary *, unsigned> SummaryToValueIdMap;
4822
4823   SmallVector<uint64_t, 64> NameVals;
4824
4825   // Set that will be populated during call to writeFunctionTypeMetadataRecords
4826   // with the type ids referenced by this index file.
4827   std::set<GlobalValue::GUID> ReferencedTypeIds;
4828
4829   // For local linkage, we also emit the original name separately
4830   // immediately after the record.
4831   auto MaybeEmitOriginalName = [&](GlobalValueSummary &S) {
4832     // We don't need to emit the original name if we are writing the index for
4833     // distributed backends (in which case ModuleToSummariesForIndex is
4834     // non-null). The original name is only needed during the thin link, since
4835     // for SamplePGO the indirect call targets for local functions have
4836     // have the original name annotated in profile.
4837     // Continue to emit it when writing out the entire combined index, which is
4838     // used in testing the thin link via llvm-lto.
4839     if (ModuleToSummariesForIndex || !GlobalValue::isLocalLinkage(S.linkage()))
4840       return;
4841     NameVals.push_back(S.getOriginalName());
4842     Stream.EmitRecord(bitc::FS_COMBINED_ORIGINAL_NAME, NameVals);
4843     NameVals.clear();
4844   };
4845
4846   // First walk through all the functions and collect the allocation contexts in
4847   // their associated summaries, for use in constructing a radix tree of
4848   // contexts. Note that we need to do this in the same order as the functions
4849   // are processed further below since the call stack positions in the resulting
4850   // radix tree array are identified based on this order.
4851   MapVector<CallStackId, llvm::SmallVector<LinearFrameId>> CallStacks;
4852   forEachSummary([&](GVInfo I, bool IsAliasee) {
4853     // Don't collect this when invoked for an aliasee, as it is not needed for
4854     // the alias summary. If the aliasee is to be imported, we will invoke this
4855     // separately with IsAliasee=false.
4856     if (IsAliasee)
4857       return;
4858     GlobalValueSummary *S = I.second;
4859     assert(S);
4860     auto *FS = dyn_cast<FunctionSummary>(S);
4861     if (!FS)
4862       return;
4863     collectMemProfCallStacks(
4864         FS,
4865         /*GetStackIndex*/
4866         [&](unsigned I) {
4867           // Get the corresponding index into the list of StackIds actually
4868           // being written for this combined index (which may be a subset in
4869           // the case of distributed indexes).
4870           assert(StackIdIndicesToIndex.contains(I));
4871           return StackIdIndicesToIndex[I];
4872         },
4873         CallStacks);
4874   });
4875   // Finalize the radix tree, write it out, and get the map of positions in the
4876   // linearized tree array.
4877   DenseMap<CallStackId, LinearCallStackId> CallStackPos;
4878   if (!CallStacks.empty()) {
4879     CallStackPos =
4880         writeMemoryProfileRadixTree(std::move(CallStacks), Stream, RadixAbbrev);
4881   }
4882
4883   // Keep track of the current index into the CallStackPos map.
4884   CallStackId CallStackCount = 0;
4885
4886   DenseSet<GlobalValue::GUID> DefOrUseGUIDs;
4887   forEachSummary([&](GVInfo I, bool IsAliasee) {
4888     GlobalValueSummary *S = I.second;
4889     assert(S);
4890     DefOrUseGUIDs.insert(I.first);
4891     for (const ValueInfo &VI : S->refs())
4892       DefOrUseGUIDs.insert(VI.getGUID());
4893
4894     auto ValueId = getValueId(I.first);
4895     assert(ValueId);
4896     SummaryToValueIdMap[S] = *ValueId;
4897
4898     // If this is invoked for an aliasee, we want to record the above
4899     // mapping, but then not emit a summary entry (if the aliasee is
4900     // to be imported, we will invoke this separately with IsAliasee=false).
4901     if (IsAliasee)
4902       return;
4903
4904     if (auto *AS = dyn_cast<AliasSummary>(S)) {
4905       // Will process aliases as a post-pass because the reader wants all
4906       // global to be loaded first.
4907       Aliases.push_back(AS);
4908       return;
4909     }
4910
4911     if (auto *VS = dyn_cast<GlobalVarSummary>(S)) {
4912       NameVals.push_back(*ValueId);
4913       assert(ModuleIdMap.count(VS->modulePath()));
4914       NameVals.push_back(ModuleIdMap[VS->modulePath()]);
4915       NameVals.push_back(
4916           getEncodedGVSummaryFlags(VS->flags(), shouldImportValueAsDecl(VS)));
4917       NameVals.push_back(getEncodedGVarFlags(VS->varflags()));
4918       for (auto &RI : VS->refs()) {
4919         auto RefValueId = getValueId(RI.getGUID());
4920         if (!RefValueId)
4921           continue;
4922         NameVals.push_back(*RefValueId);
4923       }
4924
4925       // Emit the finished record.
4926       Stream.EmitRecord(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS, NameVals,
4927                         FSModRefsAbbrev);
4928       NameVals.clear();
4929       MaybeEmitOriginalName(*S);
4930       return;
4931     }
4932
4933     auto GetValueId = [&](const ValueInfo &VI) -> std::optional<unsigned> {
4934       if (!VI)
4935         return std::nullopt;
4936       return getValueId(VI.getGUID());
4937     };
4938
4939     auto *FS = cast<FunctionSummary>(S);
4940     writeFunctionTypeMetadataRecords(Stream, FS, GetValueId);
4941     getReferencedTypeIds(FS, ReferencedTypeIds);
4942
4943     writeFunctionHeapProfileRecords(
4944         Stream, FS, CallsiteAbbrev, AllocAbbrev, /*ContextIdAbbvId*/ 0,
4945         /*PerModule*/ false,
4946         /*GetValueId*/
4947         [&](const ValueInfo &VI) -> unsigned {
4948           std::optional<unsigned> ValueID = GetValueId(VI);
4949           // This can happen in shared index files for distributed ThinLTO if
4950           // the callee function summary is not included. Record 0 which we
4951           // will have to deal with conservatively when doing any kind of
4952           // validation in the ThinLTO backends.
4953           if (!ValueID)
4954             return 0;
4955           return *ValueID;
4956         },
4957         /*GetStackIndex*/
4958         [&](unsigned I) {
4959           // Get the corresponding index into the list of StackIds actually
4960           // being written for this combined index (which may be a subset in
4961           // the case of distributed indexes).
4962           assert(StackIdIndicesToIndex.contains(I));
4963           return StackIdIndicesToIndex[I];
4964         },
4965         /*WriteContextSizeInfoIndex*/ false, CallStackPos, CallStackCount);
4966
4967     NameVals.push_back(*ValueId);
4968     assert(ModuleIdMap.count(FS->modulePath()));
4969     NameVals.push_back(ModuleIdMap[FS->modulePath()]);
4970     NameVals.push_back(
4971         getEncodedGVSummaryFlags(FS->flags(), shouldImportValueAsDecl(FS)));
4972     NameVals.push_back(FS->instCount());
4973     NameVals.push_back(getEncodedFFlags(FS->fflags()));
4974     // TODO: Stop writing entry count and bump bitcode version.
4975     NameVals.push_back(0 /* EntryCount */);
4976
4977     // Fill in below
4978     NameVals.push_back(0); // numrefs
4979     NameVals.push_back(0); // rorefcnt
4980     NameVals.push_back(0); // worefcnt
4981
4982     unsigned Count = 0, RORefCnt = 0, WORefCnt = 0;
4983     for (auto &RI : FS->refs()) {
4984       auto RefValueId = getValueId(RI.getGUID());
4985       if (!RefValueId)
4986         continue;
4987       NameVals.push_back(*RefValueId);
4988       if (RI.isReadOnly())
4989         RORefCnt++;
4990       else if (RI.isWriteOnly())
4991         WORefCnt++;
4992       Count++;
4993     }
4994     NameVals[6] = Count;
4995     NameVals[7] = RORefCnt;
4996     NameVals[8] = WORefCnt;
4997
4998     for (auto &EI : FS->calls()) {
4999       // If this GUID doesn't have a value id, it doesn't have a function
5000       // summary and we don't need to record any calls to it.
5001       std::optional<unsigned> CallValueId = GetValueId(EI.first);
5002       if (!CallValueId)
5003         continue;
5004       NameVals.push_back(*CallValueId);
5005       NameVals.push_back(getEncodedHotnessCallEdgeInfo(EI.second));
5006     }
5007
5008     // Emit the finished record.
5009     Stream.EmitRecord(bitc::FS_COMBINED_PROFILE, NameVals,
5010                       FSCallsProfileAbbrev);
5011     NameVals.clear();
5012     MaybeEmitOriginalName(*S);
5013   });
5014
5015   for (auto *AS : Aliases) {
5016     auto AliasValueId = SummaryToValueIdMap[AS];
5017     assert(AliasValueId);
5018     NameVals.push_back(AliasValueId);
5019     assert(ModuleIdMap.count(AS->modulePath()));
5020     NameVals.push_back(ModuleIdMap[AS->modulePath()]);
5021     NameVals.push_back(
5022         getEncodedGVSummaryFlags(AS->flags(), shouldImportValueAsDecl(AS)));
5023     auto AliaseeValueId = SummaryToValueIdMap[&AS->getAliasee()];
5024     assert(AliaseeValueId);
5025     NameVals.push_back(AliaseeValueId);
5026
5027     // Emit the finished record.
5028     Stream.EmitRecord(bitc::FS_COMBINED_ALIAS, NameVals, FSAliasAbbrev);
5029     NameVals.clear();
5030     MaybeEmitOriginalName(*AS);
5031
5032     if (auto *FS = dyn_cast<FunctionSummary>(&AS->getAliasee()))
5033       getReferencedTypeIds(FS, ReferencedTypeIds);
5034   }
5035
5036   if (!Index.cfiFunctionDefs().empty()) {
5037     for (auto &S : Index.cfiFunctionDefs()) {
5038       if (DefOrUseGUIDs.contains(
5039               GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(S)))) {
5040         NameVals.push_back(StrtabBuilder.add(S));
5041         NameVals.push_back(S.size());
5042       }
5043     }
5044     if (!NameVals.empty()) {
5045       Stream.EmitRecord(bitc::FS_CFI_FUNCTION_DEFS, NameVals);
5046       NameVals.clear();
5047     }
5048   }
5049
5050   if (!Index.cfiFunctionDecls().empty()) {
5051     for (auto &S : Index.cfiFunctionDecls()) {
5052       if (DefOrUseGUIDs.contains(
5053               GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(S)))) {
5054         NameVals.push_back(StrtabBuilder.add(S));
5055         NameVals.push_back(S.size());
5056       }
5057     }
5058     if (!NameVals.empty()) {
5059       Stream.EmitRecord(bitc::FS_CFI_FUNCTION_DECLS, NameVals);
5060       NameVals.clear();
5061     }
5062   }
5063
5064   // Walk the GUIDs that were referenced, and write the
5065   // corresponding type id records.
5066   for (auto &T : ReferencedTypeIds) {
5067     auto TidIter = Index.typeIds().equal_range(T);
5068     for (const auto &[GUID, TypeIdPair] : make_range(TidIter)) {
5069       writeTypeIdSummaryRecord(NameVals, StrtabBuilder, TypeIdPair.first,
5070                                TypeIdPair.second);
5071       Stream.EmitRecord(bitc::FS_TYPE_ID, NameVals);
5072       NameVals.clear();
5073     }
5074   }
5075
5076   if (Index.getBlockCount())
5077     Stream.EmitRecord(bitc::FS_BLOCK_COUNT,
5078                       ArrayRef<uint64_t>{Index.getBlockCount()});
5079
5080   Stream.ExitBlock();
5081 }
5082
5083 /// Create the "IDENTIFICATION_BLOCK_ID" containing a single string with the
5084 /// current llvm version, and a record for the epoch number.
5085 static void writeIdentificationBlock(BitstreamWriter &Stream) {
5086   Stream.EnterSubblock(bitc::IDENTIFICATION_BLOCK_ID, 5);
5087
5088   // Write the "user readable" string identifying the bitcode producer
5089   auto Abbv = std::make_shared<BitCodeAbbrev>();
5090   Abbv->Add(BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_STRING));
5091   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5092   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
5093   auto StringAbbrev = Stream.EmitAbbrev(std::move(Abbv));
5094   writeStringRecord(Stream, bitc::IDENTIFICATION_CODE_STRING,
5095                     "LLVM" LLVM_VERSION_STRING, StringAbbrev);
5096
5097   // Write the epoch version
5098   Abbv = std::make_shared<BitCodeAbbrev>();
5099   Abbv->Add(BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_EPOCH));
5100   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
5101   auto EpochAbbrev = Stream.EmitAbbrev(std::move(Abbv));
5102   constexpr std::array<unsigned, 1> Vals = {{bitc::BITCODE_CURRENT_EPOCH}};
5103   Stream.EmitRecord(bitc::IDENTIFICATION_CODE_EPOCH, Vals, EpochAbbrev);
5104   Stream.ExitBlock();
5105 }
5106
5107 void ModuleBitcodeWriter::writeModuleHash(StringRef View) {
5108   // Emit the module's hash.
5109   // MODULE_CODE_HASH: [5*i32]
5110   if (GenerateHash) {
5111     uint32_t Vals[5];
5112     Hasher.update(ArrayRef<uint8_t>(
5113         reinterpret_cast<const uint8_t *>(View.data()), View.size()));
5114     std::array<uint8_t, 20> Hash = Hasher.result();
5115     for (int Pos = 0; Pos < 20; Pos += 4) {
5116       Vals[Pos / 4] = support::endian::read32be(Hash.data() + Pos);
5117     }
5118
5119     // Emit the finished record.
5120     Stream.EmitRecord(bitc::MODULE_CODE_HASH, Vals);
5121
5122     if (ModHash)
5123       // Save the written hash value.
5124       llvm::copy(Vals, std::begin(*ModHash));
5125   }
5126 }
5127
5128 void ModuleBitcodeWriter::write() {
5129   writeIdentificationBlock(Stream);
5130
5131   Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3);
5132   // We will want to write the module hash at this point. Block any flushing so
5133   // we can have access to the whole underlying data later.
5134   Stream.markAndBlockFlushing();
5135
5136   writeModuleVersion();
5137
5138   // Emit blockinfo, which defines the standard abbreviations etc.
5139   writeBlockInfo();
5140
5141   // Emit information describing all of the types in the module.
5142   writeTypeTable();
5143
5144   // Emit information about attribute groups.
5145   writeAttributeGroupTable();
5146
5147   // Emit information about parameter attributes.
5148   writeAttributeTable();
5149
5150   writeComdats();
5151
5152   // Emit top-level description of module, including target triple, inline asm,
5153   // descriptors for global variables, and function prototype info.
5154   writeModuleInfo();
5155
5156   // Emit constants.
5157   writeModuleConstants();
5158
5159   // Emit metadata kind names.
5160   writeModuleMetadataKinds();
5161
5162   // Emit metadata.
5163   writeModuleMetadata();
5164
5165   // Emit module-level use-lists.
5166   if (VE.shouldPreserveUseListOrder())
5167     writeUseListBlock(nullptr);
5168
5169   writeOperandBundleTags();
5170   writeSyncScopeNames();
5171
5172   // Emit function bodies.
5173   DenseMap<const Function *, uint64_t> FunctionToBitcodeIndex;
5174   for (const Function &F : M)
5175     if (!F.isDeclaration())
5176       writeFunction(F, FunctionToBitcodeIndex);
5177
5178   // Need to write after the above call to WriteFunction which populates
5179   // the summary information in the index.
5180   if (Index)
5181     writePerModuleGlobalValueSummary();
5182
5183   writeGlobalValueSymbolTable(FunctionToBitcodeIndex);
5184
5185   writeModuleHash(Stream.getMarkedBufferAndResumeFlushing());
5186
5187   Stream.ExitBlock();
5188 }
5189
5190 static void writeInt32ToBuffer(uint32_t Value, SmallVectorImpl<char> &Buffer,
5191                                uint32_t &Position) {
5192   support::endian::write32le(&Buffer[Position], Value);
5193   Position += 4;
5194 }
5195
5196 /// If generating a bc file on darwin, we have to emit a
5197 /// header and trailer to make it compatible with the system archiver.  To do
5198 /// this we emit the following header, and then emit a trailer that pads the
5199 /// file out to be a multiple of 16 bytes.
5200 ///
5201 /// struct bc_header {
5202 ///   uint32_t Magic;         // 0x0B17C0DE
5203 ///   uint32_t Version;       // Version, currently always 0.
5204 ///   uint32_t BitcodeOffset; // Offset to traditional bitcode file.
5205 ///   uint32_t BitcodeSize;   // Size of traditional bitcode file.
5206 ///   uint32_t CPUType;       // CPU specifier.
5207 ///   ... potentially more later ...
5208 /// };
5209 static void emitDarwinBCHeaderAndTrailer(SmallVectorImpl<char> &Buffer,
5210                                          const Triple &TT) {
5211   unsigned CPUType = ~0U;
5212
5213   // Match x86_64-*, i[3-9]86-*, powerpc-*, powerpc64-*, arm-*, thumb-*,
5214   // armv[0-9]-*, thumbv[0-9]-*, armv5te-*, or armv6t2-*. The CPUType is a magic
5215   // number from /usr/include/mach/machine.h.  It is ok to reproduce the
5216   // specific constants here because they are implicitly part of the Darwin ABI.
5217   enum {
5218     DARWIN_CPU_ARCH_ABI64      = 0x01000000,
5219     DARWIN_CPU_TYPE_X86        = 7,
5220     DARWIN_CPU_TYPE_ARM        = 12,
5221     DARWIN_CPU_TYPE_POWERPC    = 18
5222   };
5223
5224   Triple::ArchType Arch = TT.getArch();
5225   if (Arch == Triple::x86_64)
5226     CPUType = DARWIN_CPU_TYPE_X86 | DARWIN_CPU_ARCH_ABI64;
5227   else if (Arch == Triple::x86)
5228     CPUType = DARWIN_CPU_TYPE_X86;
5229   else if (Arch == Triple::ppc)
5230     CPUType = DARWIN_CPU_TYPE_POWERPC;
5231   else if (Arch == Triple::ppc64)
5232     CPUType = DARWIN_CPU_TYPE_POWERPC | DARWIN_CPU_ARCH_ABI64;
5233   else if (Arch == Triple::arm || Arch == Triple::thumb)
5234     CPUType = DARWIN_CPU_TYPE_ARM;
5235
5236   // Traditional Bitcode starts after header.
5237   assert(Buffer.size() >= BWH_HeaderSize &&
5238          "Expected header size to be reserved");
5239   unsigned BCOffset = BWH_HeaderSize;
5240   unsigned BCSize = Buffer.size() - BWH_HeaderSize;
5241
5242   // Write the magic and version.
5243   unsigned Position = 0;
5244   writeInt32ToBuffer(0x0B17C0DE, Buffer, Position);
5245   writeInt32ToBuffer(0, Buffer, Position); // Version.
5246   writeInt32ToBuffer(BCOffset, Buffer, Position);
5247   writeInt32ToBuffer(BCSize, Buffer, Position);
5248   writeInt32ToBuffer(CPUType, Buffer, Position);
5249
5250   // If the file is not a multiple of 16 bytes, insert dummy padding.
5251   while (Buffer.size() & 15)
5252     Buffer.push_back(0);
5253 }
5254
5255 /// Helper to write the header common to all bitcode files.
5256 static void writeBitcodeHeader(BitstreamWriter &Stream) {
5257   // Emit the file header.
5258   Stream.Emit((unsigned)'B', 8);
5259   Stream.Emit((unsigned)'C', 8);
5260   Stream.Emit(0x0, 4);
5261   Stream.Emit(0xC, 4);
5262   Stream.Emit(0xE, 4);
5263   Stream.Emit(0xD, 4);
5264 }
5265
5266 BitcodeWriter::BitcodeWriter(SmallVectorImpl<char> &Buffer)
5267     : Stream(new BitstreamWriter(Buffer)) {
5268   writeBitcodeHeader(*Stream);
5269 }
5270
5271 BitcodeWriter::BitcodeWriter(raw_ostream &FS)
5272     : Stream(new BitstreamWriter(FS, FlushThreshold)) {
5273   writeBitcodeHeader(*Stream);
5274 }
5275
5276 BitcodeWriter::~BitcodeWriter() { assert(WroteStrtab); }
5277
5278 void BitcodeWriter::writeBlob(unsigned Block, unsigned Record, StringRef Blob) {
5279   Stream->EnterSubblock(Block, 3);
5280
5281   auto Abbv = std::make_shared<BitCodeAbbrev>();
5282   Abbv->Add(BitCodeAbbrevOp(Record));
5283   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
5284   auto AbbrevNo = Stream->EmitAbbrev(std::move(Abbv));
5285
5286   Stream->EmitRecordWithBlob(AbbrevNo, ArrayRef<uint64_t>{Record}, Blob);
5287
5288   Stream->ExitBlock();
5289 }
5290
5291 void BitcodeWriter::writeSymtab() {
5292   assert(!WroteStrtab && !WroteSymtab);
5293
5294   // If any module has module-level inline asm, we will require a registered asm
5295   // parser for the target so that we can create an accurate symbol table for
5296   // the module.
5297   for (Module *M : Mods) {
5298     if (M->getModuleInlineAsm().empty())
5299       continue;
5300
5301     std::string Err;
5302     const Triple TT(M->getTargetTriple());
5303     const Target *T = TargetRegistry::lookupTarget(TT.str(), Err);
5304     if (!T || !T->hasMCAsmParser())
5305       return;
5306   }
5307
5308   WroteSymtab = true;
5309   SmallVector<char, 0> Symtab;
5310   // The irsymtab::build function may be unable to create a symbol table if the
5311   // module is malformed (e.g. it contains an invalid alias). Writing a symbol
5312   // table is not required for correctness, but we still want to be able to
5313   // write malformed modules to bitcode files, so swallow the error.
5314   if (Error E = irsymtab::build(Mods, Symtab, StrtabBuilder, Alloc)) {
5315     consumeError(std::move(E));
5316     return;
5317   }
5318
5319   writeBlob(bitc::SYMTAB_BLOCK_ID, bitc::SYMTAB_BLOB,
5320             {Symtab.data(), Symtab.size()});
5321 }
5322
5323 void BitcodeWriter::writeStrtab() {
5324   assert(!WroteStrtab);
5325
5326   std::vector<char> Strtab;
5327   StrtabBuilder.finalizeInOrder();
5328   Strtab.resize(StrtabBuilder.getSize());
5329   StrtabBuilder.write((uint8_t *)Strtab.data());
5330
5331   writeBlob(bitc::STRTAB_BLOCK_ID, bitc::STRTAB_BLOB,
5332             {Strtab.data(), Strtab.size()});
5333
5334   WroteStrtab = true;
5335 }
5336
5337 void BitcodeWriter::copyStrtab(StringRef Strtab) {
5338   writeBlob(bitc::STRTAB_BLOCK_ID, bitc::STRTAB_BLOB, Strtab);
5339   WroteStrtab = true;
5340 }
5341
5342 void BitcodeWriter::writeModule(const Module &M,
5343                                 bool ShouldPreserveUseListOrder,
5344                                 const ModuleSummaryIndex *Index,
5345                                 bool GenerateHash, ModuleHash *ModHash) {
5346   assert(!WroteStrtab);
5347
5348   // The Mods vector is used by irsymtab::build, which requires non-const
5349   // Modules in case it needs to materialize metadata. But the bitcode writer
5350   // requires that the module is materialized, so we can cast to non-const here,
5351   // after checking that it is in fact materialized.
5352   assert(M.isMaterialized());
5353   Mods.push_back(const_cast<Module *>(&M));
5354
5355   ModuleBitcodeWriter ModuleWriter(M, StrtabBuilder, *Stream,
5356                                    ShouldPreserveUseListOrder, Index,
5357                                    GenerateHash, ModHash);
5358   ModuleWriter.write();
5359 }
5360
5361 void BitcodeWriter::writeIndex(
5362     const ModuleSummaryIndex *Index,
5363     const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex,
5364     const GVSummaryPtrSet *DecSummaries) {
5365   IndexBitcodeWriter IndexWriter(*Stream, StrtabBuilder, *Index, DecSummaries,
5366                                  ModuleToSummariesForIndex);
5367   IndexWriter.write();
5368 }
5369
5370 /// Write the specified module to the specified output stream.
5371 void llvm::WriteBitcodeToFile(const Module &M, raw_ostream &Out,
5372                               bool ShouldPreserveUseListOrder,
5373                               const ModuleSummaryIndex *Index,
5374                               bool GenerateHash, ModuleHash *ModHash) {
5375   auto Write = [&](BitcodeWriter &Writer) {
5376     Writer.writeModule(M, ShouldPreserveUseListOrder, Index, GenerateHash,
5377                        ModHash);
5378     Writer.writeSymtab();
5379     Writer.writeStrtab();
5380   };
5381   Triple TT(M.getTargetTriple());
5382   if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) {
5383     // If this is darwin or another generic macho target, reserve space for the
5384     // header. Note that the header is computed *after* the output is known, so
5385     // we currently explicitly use a buffer, write to it, and then subsequently
5386     // flush to Out.
5387     SmallVector<char, 0> Buffer;
5388     Buffer.reserve(256 * 1024);
5389     Buffer.insert(Buffer.begin(), BWH_HeaderSize, 0);
5390     BitcodeWriter Writer(Buffer);
5391     Write(Writer);
5392     emitDarwinBCHeaderAndTrailer(Buffer, TT);
5393     Out.write(Buffer.data(), Buffer.size());
5394   } else {
5395     BitcodeWriter Writer(Out);
5396     Write(Writer);
5397   }
5398 }
5399
5400 void IndexBitcodeWriter::write() {
5401   Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3);
5402
5403   writeModuleVersion();
5404
5405   // Write the module paths in the combined index.
5406   writeModStrings();
5407
5408   // Write the summary combined index records.
5409   writeCombinedGlobalValueSummary();
5410
5411   Stream.ExitBlock();
5412 }
5413
5414 // Write the specified module summary index to the given raw output stream,
5415 // where it will be written in a new bitcode block. This is used when
5416 // writing the combined index file for ThinLTO. When writing a subset of the
5417 // index for a distributed backend, provide a \p ModuleToSummariesForIndex map.
5418 void llvm::writeIndexToFile(
5419     const ModuleSummaryIndex &Index, raw_ostream &Out,
5420     const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex,
5421     const GVSummaryPtrSet *DecSummaries) {
5422   SmallVector<char, 0> Buffer;
5423   Buffer.reserve(256 * 1024);
5424
5425   BitcodeWriter Writer(Buffer);
5426   Writer.writeIndex(&Index, ModuleToSummariesForIndex, DecSummaries);
5427   Writer.writeStrtab();
5428
5429   Out.write((char *)&Buffer.front(), Buffer.size());
5430 }
5431
5432 namespace {
5433
5434 /// Class to manage the bitcode writing for a thin link bitcode file.
5435 class ThinLinkBitcodeWriter : public ModuleBitcodeWriterBase {
5436   /// ModHash is for use in ThinLTO incremental build, generated while writing
5437   /// the module bitcode file.
5438   const ModuleHash *ModHash;
5439
5440 public:
5441   ThinLinkBitcodeWriter(const Module &M, StringTableBuilder &StrtabBuilder,
5442                         BitstreamWriter &Stream,
5443                         const ModuleSummaryIndex &Index,
5444                         const ModuleHash &ModHash)
5445       : ModuleBitcodeWriterBase(M, StrtabBuilder, Stream,
5446                                 /*ShouldPreserveUseListOrder=*/false, &Index),
5447         ModHash(&ModHash) {}
5448
5449   void write();
5450
5451 private:
5452   void writeSimplifiedModuleInfo();
5453 };
5454
5455 } // end anonymous namespace
5456
5457 // This function writes a simpilified module info for thin link bitcode file.
5458 // It only contains the source file name along with the name(the offset and
5459 // size in strtab) and linkage for global values. For the global value info
5460 // entry, in order to keep linkage at offset 5, there are three zeros used
5461 // as padding.
5462 void ThinLinkBitcodeWriter::writeSimplifiedModuleInfo() {
5463   SmallVector<unsigned, 64> Vals;
5464   // Emit the module's source file name.
5465   {
5466     StringEncoding Bits = getStringEncoding(M.getSourceFileName());
5467     BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8);
5468     if (Bits == SE_Char6)
5469       AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6);
5470     else if (Bits == SE_Fixed7)
5471       AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7);
5472
5473     // MODULE_CODE_SOURCE_FILENAME: [namechar x N]
5474     auto Abbv = std::make_shared<BitCodeAbbrev>();
5475     Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME));
5476     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5477     Abbv->Add(AbbrevOpToUse);
5478     unsigned FilenameAbbrev = Stream.EmitAbbrev(std::move(Abbv));
5479
5480     for (const auto P : M.getSourceFileName())
5481       Vals.push_back((unsigned char)P);
5482
5483     Stream.EmitRecord(bitc::MODULE_CODE_SOURCE_FILENAME, Vals, FilenameAbbrev);
5484     Vals.clear();
5485   }
5486
5487   // Emit the global variable information.
5488   for (const GlobalVariable &GV : M.globals()) {
5489     // GLOBALVAR: [strtab offset, strtab size, 0, 0, 0, linkage]
5490     Vals.push_back(StrtabBuilder.add(GV.getName()));
5491     Vals.push_back(GV.getName().size());
5492     Vals.push_back(0);
5493     Vals.push_back(0);
5494     Vals.push_back(0);
5495     Vals.push_back(getEncodedLinkage(GV));
5496
5497     Stream.EmitRecord(bitc::MODULE_CODE_GLOBALVAR, Vals);
5498     Vals.clear();
5499   }
5500
5501   // Emit the function proto information.
5502   for (const Function &F : M) {
5503     // FUNCTION:  [strtab offset, strtab size, 0, 0, 0, linkage]
5504     Vals.push_back(StrtabBuilder.add(F.getName()));
5505     Vals.push_back(F.getName().size());
5506     Vals.push_back(0);
5507     Vals.push_back(0);
5508     Vals.push_back(0);
5509     Vals.push_back(getEncodedLinkage(F));
5510
5511     Stream.EmitRecord(bitc::MODULE_CODE_FUNCTION, Vals);
5512     Vals.clear();
5513   }
5514
5515   // Emit the alias information.
5516   for (const GlobalAlias &A : M.aliases()) {
5517     // ALIAS: [strtab offset, strtab size, 0, 0, 0, linkage]
5518     Vals.push_back(StrtabBuilder.add(A.getName()));
5519     Vals.push_back(A.getName().size());
5520     Vals.push_back(0);
5521     Vals.push_back(0);
5522     Vals.push_back(0);
5523     Vals.push_back(getEncodedLinkage(A));
5524
5525     Stream.EmitRecord(bitc::MODULE_CODE_ALIAS, Vals);
5526     Vals.clear();
5527   }
5528
5529   // Emit the ifunc information.
5530   for (const GlobalIFunc &I : M.ifuncs()) {
5531     // IFUNC: [strtab offset, strtab size, 0, 0, 0, linkage]
5532     Vals.push_back(StrtabBuilder.add(I.getName()));
5533     Vals.push_back(I.getName().size());
5534     Vals.push_back(0);
5535     Vals.push_back(0);
5536     Vals.push_back(0);
5537     Vals.push_back(getEncodedLinkage(I));
5538
5539     Stream.EmitRecord(bitc::MODULE_CODE_IFUNC, Vals);
5540     Vals.clear();
5541   }
5542 }
5543
5544 void ThinLinkBitcodeWriter::write() {
5545   Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3);
5546
5547   writeModuleVersion();
5548
5549   writeSimplifiedModuleInfo();
5550
5551   writePerModuleGlobalValueSummary();
5552
5553   // Write module hash.
5554   Stream.EmitRecord(bitc::MODULE_CODE_HASH, ArrayRef<uint32_t>(*ModHash));
5555
5556   Stream.ExitBlock();
5557 }
5558
5559 void BitcodeWriter::writeThinLinkBitcode(const Module &M,
5560                                          const ModuleSummaryIndex &Index,
5561                                          const ModuleHash &ModHash) {
5562   assert(!WroteStrtab);
5563
5564   // The Mods vector is used by irsymtab::build, which requires non-const
5565   // Modules in case it needs to materialize metadata. But the bitcode writer
5566   // requires that the module is materialized, so we can cast to non-const here,
5567   // after checking that it is in fact materialized.
5568   assert(M.isMaterialized());
5569   Mods.push_back(const_cast<Module *>(&M));
5570
5571   ThinLinkBitcodeWriter ThinLinkWriter(M, StrtabBuilder, *Stream, Index,
5572                                        ModHash);
5573   ThinLinkWriter.write();
5574 }
5575
5576 // Write the specified thin link bitcode file to the given raw output stream,
5577 // where it will be written in a new bitcode block. This is used when
5578 // writing the per-module index file for ThinLTO.
5579 void llvm::writeThinLinkBitcodeToFile(const Module &M, raw_ostream &Out,
5580                                       const ModuleSummaryIndex &Index,
5581                                       const ModuleHash &ModHash) {
5582   SmallVector<char, 0> Buffer;
5583   Buffer.reserve(256 * 1024);
5584
5585   BitcodeWriter Writer(Buffer);
5586   Writer.writeThinLinkBitcode(M, Index, ModHash);
5587   Writer.writeSymtab();
5588   Writer.writeStrtab();
5589
5590   Out.write((char *)&Buffer.front(), Buffer.size());
5591 }
5592
5593 static const char *getSectionNameForBitcode(const Triple &T) {
5594   switch (T.getObjectFormat()) {
5595   case Triple::MachO:
5596     return "__LLVM,__bitcode";
5597   case Triple::COFF:
5598   case Triple::ELF:
5599   case Triple::Wasm:
5600   case Triple::UnknownObjectFormat:
5601     return ".llvmbc";
5602   case Triple::GOFF:
5603     llvm_unreachable("GOFF is not yet implemented");
5604     break;
5605   case Triple::SPIRV:
5606     if (T.getVendor() == Triple::AMD)
5607       return ".llvmbc";
5608     llvm_unreachable("SPIRV is not yet implemented");
5609     break;
5610   case Triple::XCOFF:
5611     llvm_unreachable("XCOFF is not yet implemented");
5612     break;
5613   case Triple::DXContainer:
5614     llvm_unreachable("DXContainer is not yet implemented");
5615     break;
5616   }
5617   llvm_unreachable("Unimplemented ObjectFormatType");
5618 }
5619
5620 static const char *getSectionNameForCommandline(const Triple &T) {
5621   switch (T.getObjectFormat()) {
5622   case Triple::MachO:
5623     return "__LLVM,__cmdline";
5624   case Triple::COFF:
5625   case Triple::ELF:
5626   case Triple::Wasm:
5627   case Triple::UnknownObjectFormat:
5628     return ".llvmcmd";
5629   case Triple::GOFF:
5630     llvm_unreachable("GOFF is not yet implemented");
5631     break;
5632   case Triple::SPIRV:
5633     if (T.getVendor() == Triple::AMD)
5634       return ".llvmcmd";
5635     llvm_unreachable("SPIRV is not yet implemented");
5636     break;
5637   case Triple::XCOFF:
5638     llvm_unreachable("XCOFF is not yet implemented");
5639     break;
5640   case Triple::DXContainer:
5641     llvm_unreachable("DXC is not yet implemented");
5642     break;
5643   }
5644   llvm_unreachable("Unimplemented ObjectFormatType");
5645 }
5646
5647 void llvm::embedBitcodeInModule(llvm::Module &M, llvm::MemoryBufferRef Buf,
5648                                 bool EmbedBitcode, bool EmbedCmdline,
5649                                 const std::vector<uint8_t> &CmdArgs) {
5650   // Save llvm.compiler.used and remove it.
5651   SmallVector<Constant *, 2> UsedArray;
5652   SmallVector<GlobalValue *, 4> UsedGlobals;
5653   GlobalVariable *Used = collectUsedGlobalVariables(M, UsedGlobals, true);
5654   Type *UsedElementType = Used ? Used->getValueType()->getArrayElementType()
5655                                : PointerType::getUnqual(M.getContext());
5656   for (auto *GV : UsedGlobals) {
5657     if (GV->getName() != "llvm.embedded.module" &&
5658         GV->getName() != "llvm.cmdline")
5659       UsedArray.push_back(
5660           ConstantExpr::getPointerBitCastOrAddrSpaceCast(GV, UsedElementType));
5661   }
5662   if (Used)
5663     Used->eraseFromParent();
5664
5665   // Embed the bitcode for the llvm module.
5666   std::string Data;
5667   ArrayRef<uint8_t> ModuleData;
5668   Triple T(M.getTargetTriple());
5669
5670   if (EmbedBitcode) {
5671     if (Buf.getBufferSize() == 0 ||
5672         !isBitcode((const unsigned char *)Buf.getBufferStart(),
5673                    (const unsigned char *)Buf.getBufferEnd())) {
5674       // If the input is LLVM Assembly, bitcode is produced by serializing
5675       // the module. Use-lists order need to be preserved in this case.
5676       llvm::raw_string_ostream OS(Data);
5677       llvm::WriteBitcodeToFile(M, OS, /* ShouldPreserveUseListOrder */ true);
5678       ModuleData =
5679           ArrayRef<uint8_t>((const uint8_t *)OS.str().data(), OS.str().size());
5680     } else
5681       // If the input is LLVM bitcode, write the input byte stream directly.
5682       ModuleData = ArrayRef<uint8_t>((const uint8_t *)Buf.getBufferStart(),
5683                                      Buf.getBufferSize());
5684   }
5685   llvm::Constant *ModuleConstant =
5686       llvm::ConstantDataArray::get(M.getContext(), ModuleData);
5687   llvm::GlobalVariable *GV = new llvm::GlobalVariable(
5688       M, ModuleConstant->getType(), true, llvm::GlobalValue::PrivateLinkage,
5689       ModuleConstant);
5690   GV->setSection(getSectionNameForBitcode(T));
5691   // Set alignment to 1 to prevent padding between two contributions from input
5692   // sections after linking.
5693   GV->setAlignment(Align(1));
5694   UsedArray.push_back(
5695       ConstantExpr::getPointerBitCastOrAddrSpaceCast(GV, UsedElementType));
5696   if (llvm::GlobalVariable *Old =
5697           M.getGlobalVariable("llvm.embedded.module", true)) {
5698     assert(Old->hasZeroLiveUses() &&
5699            "llvm.embedded.module can only be used once in llvm.compiler.used");
5700     GV->takeName(Old);
5701     Old->eraseFromParent();
5702   } else {
5703     GV->setName("llvm.embedded.module");
5704   }
5705
5706   // Skip if only bitcode needs to be embedded.
5707   if (EmbedCmdline) {
5708     // Embed command-line options.
5709     ArrayRef<uint8_t> CmdData(const_cast<uint8_t *>(CmdArgs.data()),
5710                               CmdArgs.size());
5711     llvm::Constant *CmdConstant =
5712         llvm::ConstantDataArray::get(M.getContext(), CmdData);
5713     GV = new llvm::GlobalVariable(M, CmdConstant->getType(), true,
5714                                   llvm::GlobalValue::PrivateLinkage,
5715                                   CmdConstant);
5716     GV->setSection(getSectionNameForCommandline(T));
5717     GV->setAlignment(Align(1));
5718     UsedArray.push_back(
5719         ConstantExpr::getPointerBitCastOrAddrSpaceCast(GV, UsedElementType));
5720     if (llvm::GlobalVariable *Old = M.getGlobalVariable("llvm.cmdline", true)) {
5721       assert(Old->hasZeroLiveUses() &&
5722              "llvm.cmdline can only be used once in llvm.compiler.used");
5723       GV->takeName(Old);
5724       Old->eraseFromParent();
5725     } else {
5726       GV->setName("llvm.cmdline");
5727     }
5728   }
5729
5730   if (UsedArray.empty())
5731     return;
5732
5733   // Recreate llvm.compiler.used.
5734   ArrayType *ATy = ArrayType::get(UsedElementType, UsedArray.size());
5735   auto *NewUsed = new GlobalVariable(
5736       M, ATy, false, llvm::GlobalValue::AppendingLinkage,
5737       llvm::ConstantArray::get(ATy, UsedArray), "llvm.compiler.used");
5738   NewUsed->setSection("llvm.metadata");
5739 }