llvm/lib/Bitcode/Writer/BitcodeWriter.cpp

   1 //===- Bitcode/Writer/BitcodeWriter.cpp - Bitcode Writer ------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // Bitcode writer implementation.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "llvm/Bitcode/BitcodeWriter.h"
  14 #include "ValueEnumerator.h"
  15 #include "llvm/ADT/APFloat.h"
  16 #include "llvm/ADT/APInt.h"
  17 #include "llvm/ADT/ArrayRef.h"
  18 #include "llvm/ADT/DenseMap.h"
  19 #include "llvm/ADT/STLExtras.h"
  20 #include "llvm/ADT/SetVector.h"
  21 #include "llvm/ADT/SmallPtrSet.h"
  22 #include "llvm/ADT/SmallString.h"
  23 #include "llvm/ADT/SmallVector.h"
  24 #include "llvm/ADT/StringMap.h"
  25 #include "llvm/ADT/StringRef.h"
  26 #include "llvm/Bitcode/BitcodeCommon.h"
  27 #include "llvm/Bitcode/BitcodeReader.h"
  28 #include "llvm/Bitcode/LLVMBitCodes.h"
  29 #include "llvm/Bitstream/BitCodes.h"
  30 #include "llvm/Bitstream/BitstreamWriter.h"
  31 #include "llvm/Config/llvm-config.h"
  32 #include "llvm/IR/Attributes.h"
  33 #include "llvm/IR/BasicBlock.h"
  34 #include "llvm/IR/Comdat.h"
  35 #include "llvm/IR/Constant.h"
  36 #include "llvm/IR/Constants.h"
  37 #include "llvm/IR/DebugInfoMetadata.h"
  38 #include "llvm/IR/DebugLoc.h"
  39 #include "llvm/IR/DerivedTypes.h"
  40 #include "llvm/IR/Function.h"
  41 #include "llvm/IR/GlobalAlias.h"
  42 #include "llvm/IR/GlobalIFunc.h"
  43 #include "llvm/IR/GlobalObject.h"
  44 #include "llvm/IR/GlobalValue.h"
  45 #include "llvm/IR/GlobalVariable.h"
  46 #include "llvm/IR/InlineAsm.h"
  47 #include "llvm/IR/InstrTypes.h"
  48 #include "llvm/IR/Instruction.h"
  49 #include "llvm/IR/Instructions.h"
  50 #include "llvm/IR/LLVMContext.h"
  51 #include "llvm/IR/Metadata.h"
  52 #include "llvm/IR/Module.h"
  53 #include "llvm/IR/ModuleSummaryIndex.h"
  54 #include "llvm/IR/Operator.h"
  55 #include "llvm/IR/Type.h"
  56 #include "llvm/IR/UseListOrder.h"
  57 #include "llvm/IR/Value.h"
  58 #include "llvm/IR/ValueSymbolTable.h"
  59 #include "llvm/MC/StringTableBuilder.h"
  60 #include "llvm/MC/TargetRegistry.h"
  61 #include "llvm/Object/IRSymtab.h"
  62 #include "llvm/Support/AtomicOrdering.h"
  63 #include "llvm/Support/Casting.h"
  64 #include "llvm/Support/CommandLine.h"
  65 #include "llvm/Support/Endian.h"
  66 #include "llvm/Support/Error.h"
  67 #include "llvm/Support/ErrorHandling.h"
  68 #include "llvm/Support/MathExtras.h"
  69 #include "llvm/Support/SHA1.h"
  70 #include "llvm/Support/raw_ostream.h"
  71 #include "llvm/TargetParser/Triple.h"
  72 #include <algorithm>
  73 #include <cassert>
  74 #include <cstddef>
  75 #include <cstdint>
  76 #include <iterator>
  77 #include <map>
  78 #include <memory>
  79 #include <optional>
  80 #include <string>
  81 #include <utility>
  82 #include <vector>
  83
  84 using namespace llvm;
  85
  86 static cl::opt<unsigned>
  87     IndexThreshold("bitcode-mdindex-threshold", cl::Hidden, cl::init(25),
  88                    cl::desc("Number of metadatas above which we emit an index "
  89                             "to enable lazy-loading"));
  90 static cl::opt<uint32_t> FlushThreshold(
  91     "bitcode-flush-threshold", cl::Hidden, cl::init(512),
  92     cl::desc("The threshold (unit M) for flushing LLVM bitcode."));
  93
  94 static cl::opt<bool> WriteRelBFToSummary(
  95     "write-relbf-to-summary", cl::Hidden, cl::init(false),
  96     cl::desc("Write relative block frequency to function summary "));
  97
  98 namespace llvm {
  99 extern FunctionSummary::ForceSummaryHotnessType ForceSummaryEdgesCold;
 100 }
 101
 102 namespace {
 103
 104 /// These are manifest constants used by the bitcode writer. They do not need to
 105 /// be kept in sync with the reader, but need to be consistent within this file.
 106 enum {
 107   // VALUE_SYMTAB_BLOCK abbrev id's.
 108   VST_ENTRY_8_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
 109   VST_ENTRY_7_ABBREV,
 110   VST_ENTRY_6_ABBREV,
 111   VST_BBENTRY_6_ABBREV,
 112
 113   // CONSTANTS_BLOCK abbrev id's.
 114   CONSTANTS_SETTYPE_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
 115   CONSTANTS_INTEGER_ABBREV,
 116   CONSTANTS_CE_CAST_Abbrev,
 117   CONSTANTS_NULL_Abbrev,
 118
 119   // FUNCTION_BLOCK abbrev id's.
 120   FUNCTION_INST_LOAD_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
 121   FUNCTION_INST_UNOP_ABBREV,
 122   FUNCTION_INST_UNOP_FLAGS_ABBREV,
 123   FUNCTION_INST_BINOP_ABBREV,
 124   FUNCTION_INST_BINOP_FLAGS_ABBREV,
 125   FUNCTION_INST_CAST_ABBREV,
 126   FUNCTION_INST_RET_VOID_ABBREV,
 127   FUNCTION_INST_RET_VAL_ABBREV,
 128   FUNCTION_INST_UNREACHABLE_ABBREV,
 129   FUNCTION_INST_GEP_ABBREV,
 130 };
 131
 132 /// Abstract class to manage the bitcode writing, subclassed for each bitcode
 133 /// file type.
 134 class BitcodeWriterBase {
 135 protected:
 136   /// The stream created and owned by the client.
 137   BitstreamWriter &Stream;
 138
 139   StringTableBuilder &StrtabBuilder;
 140
 141 public:
 142   /// Constructs a BitcodeWriterBase object that writes to the provided
 143   /// \p Stream.
 144   BitcodeWriterBase(BitstreamWriter &Stream, StringTableBuilder &StrtabBuilder)
 145       : Stream(Stream), StrtabBuilder(StrtabBuilder) {}
 146
 147 protected:
 148   void writeModuleVersion();
 149 };
 150
 151 void BitcodeWriterBase::writeModuleVersion() {
 152   // VERSION: [version#]
 153   Stream.EmitRecord(bitc::MODULE_CODE_VERSION, ArrayRef<uint64_t>{2});
 154 }
 155
 156 /// Base class to manage the module bitcode writing, currently subclassed for
 157 /// ModuleBitcodeWriter and ThinLinkBitcodeWriter.
 158 class ModuleBitcodeWriterBase : public BitcodeWriterBase {
 159 protected:
 160   /// The Module to write to bitcode.
 161   const Module &M;
 162
 163   /// Enumerates ids for all values in the module.
 164   ValueEnumerator VE;
 165
 166   /// Optional per-module index to write for ThinLTO.
 167   const ModuleSummaryIndex *Index;
 168
 169   /// Map that holds the correspondence between GUIDs in the summary index,
 170   /// that came from indirect call profiles, and a value id generated by this
 171   /// class to use in the VST and summary block records.
 172   std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
 173
 174   /// Tracks the last value id recorded in the GUIDToValueMap.
 175   unsigned GlobalValueId;
 176
 177   /// Saves the offset of the VSTOffset record that must eventually be
 178   /// backpatched with the offset of the actual VST.
 179   uint64_t VSTOffsetPlaceholder = 0;
 180
 181 public:
 182   /// Constructs a ModuleBitcodeWriterBase object for the given Module,
 183   /// writing to the provided \p Buffer.
 184   ModuleBitcodeWriterBase(const Module &M, StringTableBuilder &StrtabBuilder,
 185                           BitstreamWriter &Stream,
 186                           bool ShouldPreserveUseListOrder,
 187                           const ModuleSummaryIndex *Index)
 188       : BitcodeWriterBase(Stream, StrtabBuilder), M(M),
 189         VE(M, ShouldPreserveUseListOrder), Index(Index) {
 190     // Assign ValueIds to any callee values in the index that came from
 191     // indirect call profiles and were recorded as a GUID not a Value*
 192     // (which would have been assigned an ID by the ValueEnumerator).
 193     // The starting ValueId is just after the number of values in the
 194     // ValueEnumerator, so that they can be emitted in the VST.
 195     GlobalValueId = VE.getValues().size();
 196     if (!Index)
 197       return;
 198     for (const auto &GUIDSummaryLists : *Index)
 199       // Examine all summaries for this GUID.
 200       for (auto &Summary : GUIDSummaryLists.second.SummaryList)
 201         if (auto FS = dyn_cast<FunctionSummary>(Summary.get()))
 202           // For each call in the function summary, see if the call
 203           // is to a GUID (which means it is for an indirect call,
 204           // otherwise we would have a Value for it). If so, synthesize
 205           // a value id.
 206           for (auto &CallEdge : FS->calls())
 207             if (!CallEdge.first.haveGVs() || !CallEdge.first.getValue())
 208               assignValueId(CallEdge.first.getGUID());
 209   }
 210
 211 protected:
 212   void writePerModuleGlobalValueSummary();
 213
 214 private:
 215   void writePerModuleFunctionSummaryRecord(
 216       SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
 217       unsigned ValueID, unsigned FSCallsAbbrev, unsigned FSCallsProfileAbbrev,
 218       unsigned CallsiteAbbrev, unsigned AllocAbbrev, const Function &F);
 219   void writeModuleLevelReferences(const GlobalVariable &V,
 220                                   SmallVector<uint64_t, 64> &NameVals,
 221                                   unsigned FSModRefsAbbrev,
 222                                   unsigned FSModVTableRefsAbbrev);
 223
 224   void assignValueId(GlobalValue::GUID ValGUID) {
 225     GUIDToValueIdMap[ValGUID] = ++GlobalValueId;
 226   }
 227
 228   unsigned getValueId(GlobalValue::GUID ValGUID) {
 229     const auto &VMI = GUIDToValueIdMap.find(ValGUID);
 230     // Expect that any GUID value had a value Id assigned by an
 231     // earlier call to assignValueId.
 232     assert(VMI != GUIDToValueIdMap.end() &&
 233            "GUID does not have assigned value Id");
 234     return VMI->second;
 235   }
 236
 237   // Helper to get the valueId for the type of value recorded in VI.
 238   unsigned getValueId(ValueInfo VI) {
 239     if (!VI.haveGVs() || !VI.getValue())
 240       return getValueId(VI.getGUID());
 241     return VE.getValueID(VI.getValue());
 242   }
 243
 244   std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; }
 245 };
 246
 247 /// Class to manage the bitcode writing for a module.
 248 class ModuleBitcodeWriter : public ModuleBitcodeWriterBase {
 249   /// Pointer to the buffer allocated by caller for bitcode writing.
 250   const SmallVectorImpl<char> &Buffer;
 251
 252   /// True if a module hash record should be written.
 253   bool GenerateHash;
 254
 255   /// If non-null, when GenerateHash is true, the resulting hash is written
 256   /// into ModHash.
 257   ModuleHash *ModHash;
 258
 259   SHA1 Hasher;
 260
 261   /// The start bit of the identification block.
 262   uint64_t BitcodeStartBit;
 263
 264 public:
 265   /// Constructs a ModuleBitcodeWriter object for the given Module,
 266   /// writing to the provided \p Buffer.
 267   ModuleBitcodeWriter(const Module &M, SmallVectorImpl<char> &Buffer,
 268                       StringTableBuilder &StrtabBuilder,
 269                       BitstreamWriter &Stream, bool ShouldPreserveUseListOrder,
 270                       const ModuleSummaryIndex *Index, bool GenerateHash,
 271                       ModuleHash *ModHash = nullptr)
 272       : ModuleBitcodeWriterBase(M, StrtabBuilder, Stream,
 273                                 ShouldPreserveUseListOrder, Index),
 274         Buffer(Buffer), GenerateHash(GenerateHash), ModHash(ModHash),
 275         BitcodeStartBit(Stream.GetCurrentBitNo()) {}
 276
 277   /// Emit the current module to the bitstream.
 278   void write();
 279
 280 private:
 281   uint64_t bitcodeStartBit() { return BitcodeStartBit; }
 282
 283   size_t addToStrtab(StringRef Str);
 284
 285   void writeAttributeGroupTable();
 286   void writeAttributeTable();
 287   void writeTypeTable();
 288   void writeComdats();
 289   void writeValueSymbolTableForwardDecl();
 290   void writeModuleInfo();
 291   void writeValueAsMetadata(const ValueAsMetadata *MD,
 292                             SmallVectorImpl<uint64_t> &Record);
 293   void writeMDTuple(const MDTuple *N, SmallVectorImpl<uint64_t> &Record,
 294                     unsigned Abbrev);
 295   unsigned createDILocationAbbrev();
 296   void writeDILocation(const DILocation *N, SmallVectorImpl<uint64_t> &Record,
 297                        unsigned &Abbrev);
 298   unsigned createGenericDINodeAbbrev();
 299   void writeGenericDINode(const GenericDINode *N,
 300                           SmallVectorImpl<uint64_t> &Record, unsigned &Abbrev);
 301   void writeDISubrange(const DISubrange *N, SmallVectorImpl<uint64_t> &Record,
 302                        unsigned Abbrev);
 303   void writeDIGenericSubrange(const DIGenericSubrange *N,
 304                               SmallVectorImpl<uint64_t> &Record,
 305                               unsigned Abbrev);
 306   void writeDIEnumerator(const DIEnumerator *N,
 307                          SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
 308   void writeDIBasicType(const DIBasicType *N, SmallVectorImpl<uint64_t> &Record,
 309                         unsigned Abbrev);
 310   void writeDIStringType(const DIStringType *N,
 311                          SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
 312   void writeDIDerivedType(const DIDerivedType *N,
 313                           SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
 314   void writeDICompositeType(const DICompositeType *N,
 315                             SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
 316   void writeDISubroutineType(const DISubroutineType *N,
 317                              SmallVectorImpl<uint64_t> &Record,
 318                              unsigned Abbrev);
 319   void writeDIFile(const DIFile *N, SmallVectorImpl<uint64_t> &Record,
 320                    unsigned Abbrev);
 321   void writeDICompileUnit(const DICompileUnit *N,
 322                           SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
 323   void writeDISubprogram(const DISubprogram *N,
 324                          SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
 325   void writeDILexicalBlock(const DILexicalBlock *N,
 326                            SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
 327   void writeDILexicalBlockFile(const DILexicalBlockFile *N,
 328                                SmallVectorImpl<uint64_t> &Record,
 329                                unsigned Abbrev);
 330   void writeDICommonBlock(const DICommonBlock *N,
 331                           SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
 332   void writeDINamespace(const DINamespace *N, SmallVectorImpl<uint64_t> &Record,
 333                         unsigned Abbrev);
 334   void writeDIMacro(const DIMacro *N, SmallVectorImpl<uint64_t> &Record,
 335                     unsigned Abbrev);
 336   void writeDIMacroFile(const DIMacroFile *N, SmallVectorImpl<uint64_t> &Record,
 337                         unsigned Abbrev);
 338   void writeDIArgList(const DIArgList *N, SmallVectorImpl<uint64_t> &Record,
 339                       unsigned Abbrev);
 340   void writeDIModule(const DIModule *N, SmallVectorImpl<uint64_t> &Record,
 341                      unsigned Abbrev);
 342   void writeDIAssignID(const DIAssignID *N, SmallVectorImpl<uint64_t> &Record,
 343                        unsigned Abbrev);
 344   void writeDITemplateTypeParameter(const DITemplateTypeParameter *N,
 345                                     SmallVectorImpl<uint64_t> &Record,
 346                                     unsigned Abbrev);
 347   void writeDITemplateValueParameter(const DITemplateValueParameter *N,
 348                                      SmallVectorImpl<uint64_t> &Record,
 349                                      unsigned Abbrev);
 350   void writeDIGlobalVariable(const DIGlobalVariable *N,
 351                              SmallVectorImpl<uint64_t> &Record,
 352                              unsigned Abbrev);
 353   void writeDILocalVariable(const DILocalVariable *N,
 354                             SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
 355   void writeDILabel(const DILabel *N,
 356                     SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
 357   void writeDIExpression(const DIExpression *N,
 358                          SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
 359   void writeDIGlobalVariableExpression(const DIGlobalVariableExpression *N,
 360                                        SmallVectorImpl<uint64_t> &Record,
 361                                        unsigned Abbrev);
 362   void writeDIObjCProperty(const DIObjCProperty *N,
 363                            SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
 364   void writeDIImportedEntity(const DIImportedEntity *N,
 365                              SmallVectorImpl<uint64_t> &Record,
 366                              unsigned Abbrev);
 367   unsigned createNamedMetadataAbbrev();
 368   void writeNamedMetadata(SmallVectorImpl<uint64_t> &Record);
 369   unsigned createMetadataStringsAbbrev();
 370   void writeMetadataStrings(ArrayRef<const Metadata *> Strings,
 371                             SmallVectorImpl<uint64_t> &Record);
 372   void writeMetadataRecords(ArrayRef<const Metadata *> MDs,
 373                             SmallVectorImpl<uint64_t> &Record,
 374                             std::vector<unsigned> *MDAbbrevs = nullptr,
 375                             std::vector<uint64_t> *IndexPos = nullptr);
 376   void writeModuleMetadata();
 377   void writeFunctionMetadata(const Function &F);
 378   void writeFunctionMetadataAttachment(const Function &F);
 379   void pushGlobalMetadataAttachment(SmallVectorImpl<uint64_t> &Record,
 380                                     const GlobalObject &GO);
 381   void writeModuleMetadataKinds();
 382   void writeOperandBundleTags();
 383   void writeSyncScopeNames();
 384   void writeConstants(unsigned FirstVal, unsigned LastVal, bool isGlobal);
 385   void writeModuleConstants();
 386   bool pushValueAndType(const Value *V, unsigned InstID,
 387                         SmallVectorImpl<unsigned> &Vals);
 388   void writeOperandBundles(const CallBase &CB, unsigned InstID);
 389   void pushValue(const Value *V, unsigned InstID,
 390                  SmallVectorImpl<unsigned> &Vals);
 391   void pushValueSigned(const Value *V, unsigned InstID,
 392                        SmallVectorImpl<uint64_t> &Vals);
 393   void writeInstruction(const Instruction &I, unsigned InstID,
 394                         SmallVectorImpl<unsigned> &Vals);
 395   void writeFunctionLevelValueSymbolTable(const ValueSymbolTable &VST);
 396   void writeGlobalValueSymbolTable(
 397       DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex);
 398   void writeUseList(UseListOrder &&Order);
 399   void writeUseListBlock(const Function *F);
 400   void
 401   writeFunction(const Function &F,
 402                 DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex);
 403   void writeBlockInfo();
 404   void writeModuleHash(size_t BlockStartPos);
 405
 406   unsigned getEncodedSyncScopeID(SyncScope::ID SSID) {
 407     return unsigned(SSID);
 408   }
 409
 410   unsigned getEncodedAlign(MaybeAlign Alignment) { return encode(Alignment); }
 411 };
 412
 413 /// Class to manage the bitcode writing for a combined index.
 414 class IndexBitcodeWriter : public BitcodeWriterBase {
 415   /// The combined index to write to bitcode.
 416   const ModuleSummaryIndex &Index;
 417
 418   /// When writing a subset of the index for distributed backends, client
 419   /// provides a map of modules to the corresponding GUIDs/summaries to write.
 420   const std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex;
 421
 422   /// Map that holds the correspondence between the GUID used in the combined
 423   /// index and a value id generated by this class to use in references.
 424   std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
 425
 426   // The sorted stack id indices actually used in the summary entries being
 427   // written, which will be a subset of those in the full index in the case of
 428   // distributed indexes.
 429   std::vector<unsigned> StackIdIndices;
 430
 431   /// Tracks the last value id recorded in the GUIDToValueMap.
 432   unsigned GlobalValueId = 0;
 433
 434 public:
 435   /// Constructs a IndexBitcodeWriter object for the given combined index,
 436   /// writing to the provided \p Buffer. When writing a subset of the index
 437   /// for a distributed backend, provide a \p ModuleToSummariesForIndex map.
 438   IndexBitcodeWriter(BitstreamWriter &Stream, StringTableBuilder &StrtabBuilder,
 439                      const ModuleSummaryIndex &Index,
 440                      const std::map<std::string, GVSummaryMapTy>
 441                          *ModuleToSummariesForIndex = nullptr)
 442       : BitcodeWriterBase(Stream, StrtabBuilder), Index(Index),
 443         ModuleToSummariesForIndex(ModuleToSummariesForIndex) {
 444     // Assign unique value ids to all summaries to be written, for use
 445     // in writing out the call graph edges. Save the mapping from GUID
 446     // to the new global value id to use when writing those edges, which
 447     // are currently saved in the index in terms of GUID.
 448     forEachSummary([&](GVInfo I, bool IsAliasee) {
 449       GUIDToValueIdMap[I.first] = ++GlobalValueId;
 450       if (IsAliasee)
 451         return;
 452       auto *FS = dyn_cast<FunctionSummary>(I.second);
 453       if (!FS)
 454         return;
 455       // Record all stack id indices actually used in the summary entries being
 456       // written, so that we can compact them in the case of distributed ThinLTO
 457       // indexes.
 458       for (auto &CI : FS->callsites())
 459         for (auto Idx : CI.StackIdIndices)
 460           StackIdIndices.push_back(Idx);
 461       for (auto &AI : FS->allocs())
 462         for (auto &MIB : AI.MIBs)
 463           for (auto Idx : MIB.StackIdIndices)
 464             StackIdIndices.push_back(Idx);
 465     });
 466     llvm::sort(StackIdIndices);
 467     StackIdIndices.erase(
 468         std::unique(StackIdIndices.begin(), StackIdIndices.end()),
 469         StackIdIndices.end());
 470   }
 471
 472   /// The below iterator returns the GUID and associated summary.
 473   using GVInfo = std::pair<GlobalValue::GUID, GlobalValueSummary *>;
 474
 475   /// Calls the callback for each value GUID and summary to be written to
 476   /// bitcode. This hides the details of whether they are being pulled from the
 477   /// entire index or just those in a provided ModuleToSummariesForIndex map.
 478   template<typename Functor>
 479   void forEachSummary(Functor Callback) {
 480     if (ModuleToSummariesForIndex) {
 481       for (auto &M : *ModuleToSummariesForIndex)
 482         for (auto &Summary : M.second) {
 483           Callback(Summary, false);
 484           // Ensure aliasee is handled, e.g. for assigning a valueId,
 485           // even if we are not importing the aliasee directly (the
 486           // imported alias will contain a copy of aliasee).
 487           if (auto *AS = dyn_cast<AliasSummary>(Summary.getSecond()))
 488             Callback({AS->getAliaseeGUID(), &AS->getAliasee()}, true);
 489         }
 490     } else {
 491       for (auto &Summaries : Index)
 492         for (auto &Summary : Summaries.second.SummaryList)
 493           Callback({Summaries.first, Summary.get()}, false);
 494     }
 495   }
 496
 497   /// Calls the callback for each entry in the modulePaths StringMap that
 498   /// should be written to the module path string table. This hides the details
 499   /// of whether they are being pulled from the entire index or just those in a
 500   /// provided ModuleToSummariesForIndex map.
 501   template <typename Functor> void forEachModule(Functor Callback) {
 502     if (ModuleToSummariesForIndex) {
 503       for (const auto &M : *ModuleToSummariesForIndex) {
 504         const auto &MPI = Index.modulePaths().find(M.first);
 505         if (MPI == Index.modulePaths().end()) {
 506           // This should only happen if the bitcode file was empty, in which
 507           // case we shouldn't be importing (the ModuleToSummariesForIndex
 508           // would only include the module we are writing and index for).
 509           assert(ModuleToSummariesForIndex->size() == 1);
 510           continue;
 511         }
 512         Callback(*MPI);
 513       }
 514     } else {
 515       for (const auto &MPSE : Index.modulePaths())
 516         Callback(MPSE);
 517     }
 518   }
 519
 520   /// Main entry point for writing a combined index to bitcode.
 521   void write();
 522
 523 private:
 524   void writeModStrings();
 525   void writeCombinedGlobalValueSummary();
 526
 527   std::optional<unsigned> getValueId(GlobalValue::GUID ValGUID) {
 528     auto VMI = GUIDToValueIdMap.find(ValGUID);
 529     if (VMI == GUIDToValueIdMap.end())
 530       return std::nullopt;
 531     return VMI->second;
 532   }
 533
 534   std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; }
 535 };
 536
 537 } // end anonymous namespace
 538
 539 static unsigned getEncodedCastOpcode(unsigned Opcode) {
 540   switch (Opcode) {
 541   default: llvm_unreachable("Unknown cast instruction!");
 542   case Instruction::Trunc   : return bitc::CAST_TRUNC;
 543   case Instruction::ZExt    : return bitc::CAST_ZEXT;
 544   case Instruction::SExt    : return bitc::CAST_SEXT;
 545   case Instruction::FPToUI  : return bitc::CAST_FPTOUI;
 546   case Instruction::FPToSI  : return bitc::CAST_FPTOSI;
 547   case Instruction::UIToFP  : return bitc::CAST_UITOFP;
 548   case Instruction::SIToFP  : return bitc::CAST_SITOFP;
 549   case Instruction::FPTrunc : return bitc::CAST_FPTRUNC;
 550   case Instruction::FPExt   : return bitc::CAST_FPEXT;
 551   case Instruction::PtrToInt: return bitc::CAST_PTRTOINT;
 552   case Instruction::IntToPtr: return bitc::CAST_INTTOPTR;
 553   case Instruction::BitCast : return bitc::CAST_BITCAST;
 554   case Instruction::AddrSpaceCast: return bitc::CAST_ADDRSPACECAST;
 555   }
 556 }
 557
 558 static unsigned getEncodedUnaryOpcode(unsigned Opcode) {
 559   switch (Opcode) {
 560   default: llvm_unreachable("Unknown binary instruction!");
 561   case Instruction::FNeg: return bitc::UNOP_FNEG;
 562   }
 563 }
 564
 565 static unsigned getEncodedBinaryOpcode(unsigned Opcode) {
 566   switch (Opcode) {
 567   default: llvm_unreachable("Unknown binary instruction!");
 568   case Instruction::Add:
 569   case Instruction::FAdd: return bitc::BINOP_ADD;
 570   case Instruction::Sub:
 571   case Instruction::FSub: return bitc::BINOP_SUB;
 572   case Instruction::Mul:
 573   case Instruction::FMul: return bitc::BINOP_MUL;
 574   case Instruction::UDiv: return bitc::BINOP_UDIV;
 575   case Instruction::FDiv:
 576   case Instruction::SDiv: return bitc::BINOP_SDIV;
 577   case Instruction::URem: return bitc::BINOP_UREM;
 578   case Instruction::FRem:
 579   case Instruction::SRem: return bitc::BINOP_SREM;
 580   case Instruction::Shl:  return bitc::BINOP_SHL;
 581   case Instruction::LShr: return bitc::BINOP_LSHR;
 582   case Instruction::AShr: return bitc::BINOP_ASHR;
 583   case Instruction::And:  return bitc::BINOP_AND;
 584   case Instruction::Or:   return bitc::BINOP_OR;
 585   case Instruction::Xor:  return bitc::BINOP_XOR;
 586   }
 587 }
 588
 589 static unsigned getEncodedRMWOperation(AtomicRMWInst::BinOp Op) {
 590   switch (Op) {
 591   default: llvm_unreachable("Unknown RMW operation!");
 592   case AtomicRMWInst::Xchg: return bitc::RMW_XCHG;
 593   case AtomicRMWInst::Add: return bitc::RMW_ADD;
 594   case AtomicRMWInst::Sub: return bitc::RMW_SUB;
 595   case AtomicRMWInst::And: return bitc::RMW_AND;
 596   case AtomicRMWInst::Nand: return bitc::RMW_NAND;
 597   case AtomicRMWInst::Or: return bitc::RMW_OR;
 598   case AtomicRMWInst::Xor: return bitc::RMW_XOR;
 599   case AtomicRMWInst::Max: return bitc::RMW_MAX;
 600   case AtomicRMWInst::Min: return bitc::RMW_MIN;
 601   case AtomicRMWInst::UMax: return bitc::RMW_UMAX;
 602   case AtomicRMWInst::UMin: return bitc::RMW_UMIN;
 603   case AtomicRMWInst::FAdd: return bitc::RMW_FADD;
 604   case AtomicRMWInst::FSub: return bitc::RMW_FSUB;
 605   case AtomicRMWInst::FMax: return bitc::RMW_FMAX;
 606   case AtomicRMWInst::FMin: return bitc::RMW_FMIN;
 607   case AtomicRMWInst::UIncWrap:
 608     return bitc::RMW_UINC_WRAP;
 609   case AtomicRMWInst::UDecWrap:
 610     return bitc::RMW_UDEC_WRAP;
 611   }
 612 }
 613
 614 static unsigned getEncodedOrdering(AtomicOrdering Ordering) {
 615   switch (Ordering) {
 616   case AtomicOrdering::NotAtomic: return bitc::ORDERING_NOTATOMIC;
 617   case AtomicOrdering::Unordered: return bitc::ORDERING_UNORDERED;
 618   case AtomicOrdering::Monotonic: return bitc::ORDERING_MONOTONIC;
 619   case AtomicOrdering::Acquire: return bitc::ORDERING_ACQUIRE;
 620   case AtomicOrdering::Release: return bitc::ORDERING_RELEASE;
 621   case AtomicOrdering::AcquireRelease: return bitc::ORDERING_ACQREL;
 622   case AtomicOrdering::SequentiallyConsistent: return bitc::ORDERING_SEQCST;
 623   }
 624   llvm_unreachable("Invalid ordering");
 625 }
 626
 627 static void writeStringRecord(BitstreamWriter &Stream, unsigned Code,
 628                               StringRef Str, unsigned AbbrevToUse) {
 629   SmallVector<unsigned, 64> Vals;
 630
 631   // Code: [strchar x N]
 632   for (char C : Str) {
 633     if (AbbrevToUse && !BitCodeAbbrevOp::isChar6(C))
 634       AbbrevToUse = 0;
 635     Vals.push_back(C);
 636   }
 637
 638   // Emit the finished record.
 639   Stream.EmitRecord(Code, Vals, AbbrevToUse);
 640 }
 641
 642 static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
 643   switch (Kind) {
 644   case Attribute::Alignment:
 645     return bitc::ATTR_KIND_ALIGNMENT;
 646   case Attribute::AllocAlign:
 647     return bitc::ATTR_KIND_ALLOC_ALIGN;
 648   case Attribute::AllocSize:
 649     return bitc::ATTR_KIND_ALLOC_SIZE;
 650   case Attribute::AlwaysInline:
 651     return bitc::ATTR_KIND_ALWAYS_INLINE;
 652   case Attribute::Builtin:
 653     return bitc::ATTR_KIND_BUILTIN;
 654   case Attribute::ByVal:
 655     return bitc::ATTR_KIND_BY_VAL;
 656   case Attribute::Convergent:
 657     return bitc::ATTR_KIND_CONVERGENT;
 658   case Attribute::InAlloca:
 659     return bitc::ATTR_KIND_IN_ALLOCA;
 660   case Attribute::Cold:
 661     return bitc::ATTR_KIND_COLD;
 662   case Attribute::DisableSanitizerInstrumentation:
 663     return bitc::ATTR_KIND_DISABLE_SANITIZER_INSTRUMENTATION;
 664   case Attribute::FnRetThunkExtern:
 665     return bitc::ATTR_KIND_FNRETTHUNK_EXTERN;
 666   case Attribute::Hot:
 667     return bitc::ATTR_KIND_HOT;
 668   case Attribute::ElementType:
 669     return bitc::ATTR_KIND_ELEMENTTYPE;
 670   case Attribute::InlineHint:
 671     return bitc::ATTR_KIND_INLINE_HINT;
 672   case Attribute::InReg:
 673     return bitc::ATTR_KIND_IN_REG;
 674   case Attribute::JumpTable:
 675     return bitc::ATTR_KIND_JUMP_TABLE;
 676   case Attribute::MinSize:
 677     return bitc::ATTR_KIND_MIN_SIZE;
 678   case Attribute::AllocatedPointer:
 679     return bitc::ATTR_KIND_ALLOCATED_POINTER;
 680   case Attribute::AllocKind:
 681     return bitc::ATTR_KIND_ALLOC_KIND;
 682   case Attribute::Memory:
 683     return bitc::ATTR_KIND_MEMORY;
 684   case Attribute::NoFPClass:
 685     return bitc::ATTR_KIND_NOFPCLASS;
 686   case Attribute::Naked:
 687     return bitc::ATTR_KIND_NAKED;
 688   case Attribute::Nest:
 689     return bitc::ATTR_KIND_NEST;
 690   case Attribute::NoAlias:
 691     return bitc::ATTR_KIND_NO_ALIAS;
 692   case Attribute::NoBuiltin:
 693     return bitc::ATTR_KIND_NO_BUILTIN;
 694   case Attribute::NoCallback:
 695     return bitc::ATTR_KIND_NO_CALLBACK;
 696   case Attribute::NoCapture:
 697     return bitc::ATTR_KIND_NO_CAPTURE;
 698   case Attribute::NoDuplicate:
 699     return bitc::ATTR_KIND_NO_DUPLICATE;
 700   case Attribute::NoFree:
 701     return bitc::ATTR_KIND_NOFREE;
 702   case Attribute::NoImplicitFloat:
 703     return bitc::ATTR_KIND_NO_IMPLICIT_FLOAT;
 704   case Attribute::NoInline:
 705     return bitc::ATTR_KIND_NO_INLINE;
 706   case Attribute::NoRecurse:
 707     return bitc::ATTR_KIND_NO_RECURSE;
 708   case Attribute::NoMerge:
 709     return bitc::ATTR_KIND_NO_MERGE;
 710   case Attribute::NonLazyBind:
 711     return bitc::ATTR_KIND_NON_LAZY_BIND;
 712   case Attribute::NonNull:
 713     return bitc::ATTR_KIND_NON_NULL;
 714   case Attribute::Dereferenceable:
 715     return bitc::ATTR_KIND_DEREFERENCEABLE;
 716   case Attribute::DereferenceableOrNull:
 717     return bitc::ATTR_KIND_DEREFERENCEABLE_OR_NULL;
 718   case Attribute::NoRedZone:
 719     return bitc::ATTR_KIND_NO_RED_ZONE;
 720   case Attribute::NoReturn:
 721     return bitc::ATTR_KIND_NO_RETURN;
 722   case Attribute::NoSync:
 723     return bitc::ATTR_KIND_NOSYNC;
 724   case Attribute::NoCfCheck:
 725     return bitc::ATTR_KIND_NOCF_CHECK;
 726   case Attribute::NoProfile:
 727     return bitc::ATTR_KIND_NO_PROFILE;
 728   case Attribute::SkipProfile:
 729     return bitc::ATTR_KIND_SKIP_PROFILE;
 730   case Attribute::NoUnwind:
 731     return bitc::ATTR_KIND_NO_UNWIND;
 732   case Attribute::NoSanitizeBounds:
 733     return bitc::ATTR_KIND_NO_SANITIZE_BOUNDS;
 734   case Attribute::NoSanitizeCoverage:
 735     return bitc::ATTR_KIND_NO_SANITIZE_COVERAGE;
 736   case Attribute::NullPointerIsValid:
 737     return bitc::ATTR_KIND_NULL_POINTER_IS_VALID;
 738   case Attribute::OptForFuzzing:
 739     return bitc::ATTR_KIND_OPT_FOR_FUZZING;
 740   case Attribute::OptimizeForSize:
 741     return bitc::ATTR_KIND_OPTIMIZE_FOR_SIZE;
 742   case Attribute::OptimizeNone:
 743     return bitc::ATTR_KIND_OPTIMIZE_NONE;
 744   case Attribute::ReadNone:
 745     return bitc::ATTR_KIND_READ_NONE;
 746   case Attribute::ReadOnly:
 747     return bitc::ATTR_KIND_READ_ONLY;
 748   case Attribute::Returned:
 749     return bitc::ATTR_KIND_RETURNED;
 750   case Attribute::ReturnsTwice:
 751     return bitc::ATTR_KIND_RETURNS_TWICE;
 752   case Attribute::SExt:
 753     return bitc::ATTR_KIND_S_EXT;
 754   case Attribute::Speculatable:
 755     return bitc::ATTR_KIND_SPECULATABLE;
 756   case Attribute::StackAlignment:
 757     return bitc::ATTR_KIND_STACK_ALIGNMENT;
 758   case Attribute::StackProtect:
 759     return bitc::ATTR_KIND_STACK_PROTECT;
 760   case Attribute::StackProtectReq:
 761     return bitc::ATTR_KIND_STACK_PROTECT_REQ;
 762   case Attribute::StackProtectStrong:
 763     return bitc::ATTR_KIND_STACK_PROTECT_STRONG;
 764   case Attribute::SafeStack:
 765     return bitc::ATTR_KIND_SAFESTACK;
 766   case Attribute::ShadowCallStack:
 767     return bitc::ATTR_KIND_SHADOWCALLSTACK;
 768   case Attribute::StrictFP:
 769     return bitc::ATTR_KIND_STRICT_FP;
 770   case Attribute::StructRet:
 771     return bitc::ATTR_KIND_STRUCT_RET;
 772   case Attribute::SanitizeAddress:
 773     return bitc::ATTR_KIND_SANITIZE_ADDRESS;
 774   case Attribute::SanitizeHWAddress:
 775     return bitc::ATTR_KIND_SANITIZE_HWADDRESS;
 776   case Attribute::SanitizeThread:
 777     return bitc::ATTR_KIND_SANITIZE_THREAD;
 778   case Attribute::SanitizeMemory:
 779     return bitc::ATTR_KIND_SANITIZE_MEMORY;
 780   case Attribute::SpeculativeLoadHardening:
 781     return bitc::ATTR_KIND_SPECULATIVE_LOAD_HARDENING;
 782   case Attribute::SwiftError:
 783     return bitc::ATTR_KIND_SWIFT_ERROR;
 784   case Attribute::SwiftSelf:
 785     return bitc::ATTR_KIND_SWIFT_SELF;
 786   case Attribute::SwiftAsync:
 787     return bitc::ATTR_KIND_SWIFT_ASYNC;
 788   case Attribute::UWTable:
 789     return bitc::ATTR_KIND_UW_TABLE;
 790   case Attribute::VScaleRange:
 791     return bitc::ATTR_KIND_VSCALE_RANGE;
 792   case Attribute::WillReturn:
 793     return bitc::ATTR_KIND_WILLRETURN;
 794   case Attribute::WriteOnly:
 795     return bitc::ATTR_KIND_WRITEONLY;
 796   case Attribute::ZExt:
 797     return bitc::ATTR_KIND_Z_EXT;
 798   case Attribute::ImmArg:
 799     return bitc::ATTR_KIND_IMMARG;
 800   case Attribute::SanitizeMemTag:
 801     return bitc::ATTR_KIND_SANITIZE_MEMTAG;
 802   case Attribute::Preallocated:
 803     return bitc::ATTR_KIND_PREALLOCATED;
 804   case Attribute::NoUndef:
 805     return bitc::ATTR_KIND_NOUNDEF;
 806   case Attribute::ByRef:
 807     return bitc::ATTR_KIND_BYREF;
 808   case Attribute::MustProgress:
 809     return bitc::ATTR_KIND_MUSTPROGRESS;
 810   case Attribute::PresplitCoroutine:
 811     return bitc::ATTR_KIND_PRESPLIT_COROUTINE;
 812   case Attribute::EndAttrKinds:
 813     llvm_unreachable("Can not encode end-attribute kinds marker.");
 814   case Attribute::None:
 815     llvm_unreachable("Can not encode none-attribute.");
 816   case Attribute::EmptyKey:
 817   case Attribute::TombstoneKey:
 818     llvm_unreachable("Trying to encode EmptyKey/TombstoneKey");
 819   }
 820
 821   llvm_unreachable("Trying to encode unknown attribute");
 822 }
 823
 824 void ModuleBitcodeWriter::writeAttributeGroupTable() {
 825   const std::vector<ValueEnumerator::IndexAndAttrSet> &AttrGrps =
 826       VE.getAttributeGroups();
 827   if (AttrGrps.empty()) return;
 828
 829   Stream.EnterSubblock(bitc::PARAMATTR_GROUP_BLOCK_ID, 3);
 830
 831   SmallVector<uint64_t, 64> Record;
 832   for (ValueEnumerator::IndexAndAttrSet Pair : AttrGrps) {
 833     unsigned AttrListIndex = Pair.first;
 834     AttributeSet AS = Pair.second;
 835     Record.push_back(VE.getAttributeGroupID(Pair));
 836     Record.push_back(AttrListIndex);
 837
 838     for (Attribute Attr : AS) {
 839       if (Attr.isEnumAttribute()) {
 840         Record.push_back(0);
 841         Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum()));
 842       } else if (Attr.isIntAttribute()) {
 843         Record.push_back(1);
 844         Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum()));
 845         Record.push_back(Attr.getValueAsInt());
 846       } else if (Attr.isStringAttribute()) {
 847         StringRef Kind = Attr.getKindAsString();
 848         StringRef Val = Attr.getValueAsString();
 849
 850         Record.push_back(Val.empty() ? 3 : 4);
 851         Record.append(Kind.begin(), Kind.end());
 852         Record.push_back(0);
 853         if (!Val.empty()) {
 854           Record.append(Val.begin(), Val.end());
 855           Record.push_back(0);
 856         }
 857       } else {
 858         assert(Attr.isTypeAttribute());
 859         Type *Ty = Attr.getValueAsType();
 860         Record.push_back(Ty ? 6 : 5);
 861         Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum()));
 862         if (Ty)
 863           Record.push_back(VE.getTypeID(Attr.getValueAsType()));
 864       }
 865     }
 866
 867     Stream.EmitRecord(bitc::PARAMATTR_GRP_CODE_ENTRY, Record);
 868     Record.clear();
 869   }
 870
 871   Stream.ExitBlock();
 872 }
 873
 874 void ModuleBitcodeWriter::writeAttributeTable() {
 875   const std::vector<AttributeList> &Attrs = VE.getAttributeLists();
 876   if (Attrs.empty()) return;
 877
 878   Stream.EnterSubblock(bitc::PARAMATTR_BLOCK_ID, 3);
 879
 880   SmallVector<uint64_t, 64> Record;
 881   for (const AttributeList &AL : Attrs) {
 882     for (unsigned i : AL.indexes()) {
 883       AttributeSet AS = AL.getAttributes(i);
 884       if (AS.hasAttributes())
 885         Record.push_back(VE.getAttributeGroupID({i, AS}));
 886     }
 887
 888     Stream.EmitRecord(bitc::PARAMATTR_CODE_ENTRY, Record);
 889     Record.clear();
 890   }
 891
 892   Stream.ExitBlock();
 893 }
 894
 895 /// WriteTypeTable - Write out the type table for a module.
 896 void ModuleBitcodeWriter::writeTypeTable() {
 897   const ValueEnumerator::TypeList &TypeList = VE.getTypes();
 898
 899   Stream.EnterSubblock(bitc::TYPE_BLOCK_ID_NEW, 4 /*count from # abbrevs */);
 900   SmallVector<uint64_t, 64> TypeVals;
 901
 902   uint64_t NumBits = VE.computeBitsRequiredForTypeIndicies();
 903
 904   // Abbrev for TYPE_CODE_OPAQUE_POINTER.
 905   auto Abbv = std::make_shared<BitCodeAbbrev>();
 906   Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_OPAQUE_POINTER));
 907   Abbv->Add(BitCodeAbbrevOp(0)); // Addrspace = 0
 908   unsigned OpaquePtrAbbrev = Stream.EmitAbbrev(std::move(Abbv));
 909
 910   // Abbrev for TYPE_CODE_FUNCTION.
 911   Abbv = std::make_shared<BitCodeAbbrev>();
 912   Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_FUNCTION));
 913   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));  // isvararg
 914   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
 915   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
 916   unsigned FunctionAbbrev = Stream.EmitAbbrev(std::move(Abbv));
 917
 918   // Abbrev for TYPE_CODE_STRUCT_ANON.
 919   Abbv = std::make_shared<BitCodeAbbrev>();
 920   Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_ANON));
 921   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));  // ispacked
 922   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
 923   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
 924   unsigned StructAnonAbbrev = Stream.EmitAbbrev(std::move(Abbv));
 925
 926   // Abbrev for TYPE_CODE_STRUCT_NAME.
 927   Abbv = std::make_shared<BitCodeAbbrev>();
 928   Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAME));
 929   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
 930   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
 931   unsigned StructNameAbbrev = Stream.EmitAbbrev(std::move(Abbv));
 932
 933   // Abbrev for TYPE_CODE_STRUCT_NAMED.
 934   Abbv = std::make_shared<BitCodeAbbrev>();
 935   Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAMED));
 936   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));  // ispacked
 937   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
 938   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
 939   unsigned StructNamedAbbrev = Stream.EmitAbbrev(std::move(Abbv));
 940
 941   // Abbrev for TYPE_CODE_ARRAY.
 942   Abbv = std::make_shared<BitCodeAbbrev>();
 943   Abbv->Add(BitCodeAbbrevOp(bitc::TYPE_CODE_ARRAY));
 944   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // size
 945   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
 946   unsigned ArrayAbbrev = Stream.EmitAbbrev(std::move(Abbv));
 947
 948   // Emit an entry count so the reader can reserve space.
 949   TypeVals.push_back(TypeList.size());
 950   Stream.EmitRecord(bitc::TYPE_CODE_NUMENTRY, TypeVals);
 951   TypeVals.clear();
 952
 953   // Loop over all of the types, emitting each in turn.
 954   for (Type *T : TypeList) {
 955     int AbbrevToUse = 0;
 956     unsigned Code = 0;
 957
 958     switch (T->getTypeID()) {
 959     case Type::VoidTyID:      Code = bitc::TYPE_CODE_VOID;      break;
 960     case Type::HalfTyID:      Code = bitc::TYPE_CODE_HALF;      break;
 961     case Type::BFloatTyID:    Code = bitc::TYPE_CODE_BFLOAT;    break;
 962     case Type::FloatTyID:     Code = bitc::TYPE_CODE_FLOAT;     break;
 963     case Type::DoubleTyID:    Code = bitc::TYPE_CODE_DOUBLE;    break;
 964     case Type::X86_FP80TyID:  Code = bitc::TYPE_CODE_X86_FP80;  break;
 965     case Type::FP128TyID:     Code = bitc::TYPE_CODE_FP128;     break;
 966     case Type::PPC_FP128TyID: Code = bitc::TYPE_CODE_PPC_FP128; break;
 967     case Type::LabelTyID:     Code = bitc::TYPE_CODE_LABEL;     break;
 968     case Type::MetadataTyID:  Code = bitc::TYPE_CODE_METADATA;  break;
 969     case Type::X86_MMXTyID:   Code = bitc::TYPE_CODE_X86_MMX;   break;
 970     case Type::X86_AMXTyID:   Code = bitc::TYPE_CODE_X86_AMX;   break;
 971     case Type::TokenTyID:     Code = bitc::TYPE_CODE_TOKEN;     break;
 972     case Type::IntegerTyID:
 973       // INTEGER: [width]
 974       Code = bitc::TYPE_CODE_INTEGER;
 975       TypeVals.push_back(cast<IntegerType>(T)->getBitWidth());
 976       break;
 977     case Type::PointerTyID: {
 978       PointerType *PTy = cast<PointerType>(T);
 979       unsigned AddressSpace = PTy->getAddressSpace();
 980       if (PTy->isOpaque()) {
 981         // OPAQUE_POINTER: [address space]
 982         Code = bitc::TYPE_CODE_OPAQUE_POINTER;
 983         TypeVals.push_back(AddressSpace);
 984         if (AddressSpace == 0)
 985           AbbrevToUse = OpaquePtrAbbrev;
 986       } else {
 987         // POINTER: [pointee type, address space]
 988         Code = bitc::TYPE_CODE_POINTER;
 989         TypeVals.push_back(VE.getTypeID(PTy->getNonOpaquePointerElementType()));
 990         TypeVals.push_back(AddressSpace);
 991       }
 992       break;
 993     }
 994     case Type::FunctionTyID: {
 995       FunctionType *FT = cast<FunctionType>(T);
 996       // FUNCTION: [isvararg, retty, paramty x N]
 997       Code = bitc::TYPE_CODE_FUNCTION;
 998       TypeVals.push_back(FT->isVarArg());
 999       TypeVals.push_back(VE.getTypeID(FT->getReturnType()));
1000       for (unsigned i = 0, e = FT->getNumParams(); i != e; ++i)
1001         TypeVals.push_back(VE.getTypeID(FT->getParamType(i)));
1002       AbbrevToUse = FunctionAbbrev;
1003       break;
1004     }
1005     case Type::StructTyID: {
1006       StructType *ST = cast<StructType>(T);
1007       // STRUCT: [ispacked, eltty x N]
1008       TypeVals.push_back(ST->isPacked());
1009       // Output all of the element types.
1010       for (Type *ET : ST->elements())
1011         TypeVals.push_back(VE.getTypeID(ET));
1012
1013       if (ST->isLiteral()) {
1014         Code = bitc::TYPE_CODE_STRUCT_ANON;
1015         AbbrevToUse = StructAnonAbbrev;
1016       } else {
1017         if (ST->isOpaque()) {
1018           Code = bitc::TYPE_CODE_OPAQUE;
1019         } else {
1020           Code = bitc::TYPE_CODE_STRUCT_NAMED;
1021           AbbrevToUse = StructNamedAbbrev;
1022         }
1023
1024         // Emit the name if it is present.
1025         if (!ST->getName().empty())
1026           writeStringRecord(Stream, bitc::TYPE_CODE_STRUCT_NAME, ST->getName(),
1027                             StructNameAbbrev);
1028       }
1029       break;
1030     }
1031     case Type::ArrayTyID: {
1032       ArrayType *AT = cast<ArrayType>(T);
1033       // ARRAY: [numelts, eltty]
1034       Code = bitc::TYPE_CODE_ARRAY;
1035       TypeVals.push_back(AT->getNumElements());
1036       TypeVals.push_back(VE.getTypeID(AT->getElementType()));
1037       AbbrevToUse = ArrayAbbrev;
1038       break;
1039     }
1040     case Type::FixedVectorTyID:
1041     case Type::ScalableVectorTyID: {
1042       VectorType *VT = cast<VectorType>(T);
1043       // VECTOR [numelts, eltty] or
1044       //        [numelts, eltty, scalable]
1045       Code = bitc::TYPE_CODE_VECTOR;
1046       TypeVals.push_back(VT->getElementCount().getKnownMinValue());
1047       TypeVals.push_back(VE.getTypeID(VT->getElementType()));
1048       if (isa<ScalableVectorType>(VT))
1049         TypeVals.push_back(true);
1050       break;
1051     }
1052     case Type::TargetExtTyID: {
1053       TargetExtType *TET = cast<TargetExtType>(T);
1054       Code = bitc::TYPE_CODE_TARGET_TYPE;
1055       writeStringRecord(Stream, bitc::TYPE_CODE_STRUCT_NAME, TET->getName(),
1056                         StructNameAbbrev);
1057       TypeVals.push_back(TET->getNumTypeParameters());
1058       for (Type *InnerTy : TET->type_params())
1059         TypeVals.push_back(VE.getTypeID(InnerTy));
1060       for (unsigned IntParam : TET->int_params())
1061         TypeVals.push_back(IntParam);
1062       break;
1063     }
1064     case Type::TypedPointerTyID:
1065       llvm_unreachable("Typed pointers cannot be added to IR modules");
1066     }
1067
1068     // Emit the finished record.
1069     Stream.EmitRecord(Code, TypeVals, AbbrevToUse);
1070     TypeVals.clear();
1071   }
1072
1073   Stream.ExitBlock();
1074 }
1075
1076 static unsigned getEncodedLinkage(const GlobalValue::LinkageTypes Linkage) {
1077   switch (Linkage) {
1078   case GlobalValue::ExternalLinkage:
1079     return 0;
1080   case GlobalValue::WeakAnyLinkage:
1081     return 16;
1082   case GlobalValue::AppendingLinkage:
1083     return 2;
1084   case GlobalValue::InternalLinkage:
1085     return 3;
1086   case GlobalValue::LinkOnceAnyLinkage:
1087     return 18;
1088   case GlobalValue::ExternalWeakLinkage:
1089     return 7;
1090   case GlobalValue::CommonLinkage:
1091     return 8;
1092   case GlobalValue::PrivateLinkage:
1093     return 9;
1094   case GlobalValue::WeakODRLinkage:
1095     return 17;
1096   case GlobalValue::LinkOnceODRLinkage:
1097     return 19;
1098   case GlobalValue::AvailableExternallyLinkage:
1099     return 12;
1100   }
1101   llvm_unreachable("Invalid linkage");
1102 }
1103
1104 static unsigned getEncodedLinkage(const GlobalValue &GV) {
1105   return getEncodedLinkage(GV.getLinkage());
1106 }
1107
1108 static uint64_t getEncodedFFlags(FunctionSummary::FFlags Flags) {
1109   uint64_t RawFlags = 0;
1110   RawFlags |= Flags.ReadNone;
1111   RawFlags |= (Flags.ReadOnly << 1);
1112   RawFlags |= (Flags.NoRecurse << 2);
1113   RawFlags |= (Flags.ReturnDoesNotAlias << 3);
1114   RawFlags |= (Flags.NoInline << 4);
1115   RawFlags |= (Flags.AlwaysInline << 5);
1116   RawFlags |= (Flags.NoUnwind << 6);
1117   RawFlags |= (Flags.MayThrow << 7);
1118   RawFlags |= (Flags.HasUnknownCall << 8);
1119   RawFlags |= (Flags.MustBeUnreachable << 9);
1120   return RawFlags;
1121 }
1122
1123 // Decode the flags for GlobalValue in the summary. See getDecodedGVSummaryFlags
1124 // in BitcodeReader.cpp.
1125 static uint64_t getEncodedGVSummaryFlags(GlobalValueSummary::GVFlags Flags) {
1126   uint64_t RawFlags = 0;
1127
1128   RawFlags |= Flags.NotEligibleToImport; // bool
1129   RawFlags |= (Flags.Live << 1);
1130   RawFlags |= (Flags.DSOLocal << 2);
1131   RawFlags |= (Flags.CanAutoHide << 3);
1132
1133   // Linkage don't need to be remapped at that time for the summary. Any future
1134   // change to the getEncodedLinkage() function will need to be taken into
1135   // account here as well.
1136   RawFlags = (RawFlags << 4) | Flags.Linkage; // 4 bits
1137
1138   RawFlags |= (Flags.Visibility << 8); // 2 bits
1139
1140   return RawFlags;
1141 }
1142
1143 static uint64_t getEncodedGVarFlags(GlobalVarSummary::GVarFlags Flags) {
1144   uint64_t RawFlags = Flags.MaybeReadOnly | (Flags.MaybeWriteOnly << 1) |
1145                       (Flags.Constant << 2) | Flags.VCallVisibility << 3;
1146   return RawFlags;
1147 }
1148
1149 static unsigned getEncodedVisibility(const GlobalValue &GV) {
1150   switch (GV.getVisibility()) {
1151   case GlobalValue::DefaultVisibility:   return 0;
1152   case GlobalValue::HiddenVisibility:    return 1;
1153   case GlobalValue::ProtectedVisibility: return 2;
1154   }
1155   llvm_unreachable("Invalid visibility");
1156 }
1157
1158 static unsigned getEncodedDLLStorageClass(const GlobalValue &GV) {
1159   switch (GV.getDLLStorageClass()) {
1160   case GlobalValue::DefaultStorageClass:   return 0;
1161   case GlobalValue::DLLImportStorageClass: return 1;
1162   case GlobalValue::DLLExportStorageClass: return 2;
1163   }
1164   llvm_unreachable("Invalid DLL storage class");
1165 }
1166
1167 static unsigned getEncodedThreadLocalMode(const GlobalValue &GV) {
1168   switch (GV.getThreadLocalMode()) {
1169     case GlobalVariable::NotThreadLocal:         return 0;
1170     case GlobalVariable::GeneralDynamicTLSModel: return 1;
1171     case GlobalVariable::LocalDynamicTLSModel:   return 2;
1172     case GlobalVariable::InitialExecTLSModel:    return 3;
1173     case GlobalVariable::LocalExecTLSModel:      return 4;
1174   }
1175   llvm_unreachable("Invalid TLS model");
1176 }
1177
1178 static unsigned getEncodedComdatSelectionKind(const Comdat &C) {
1179   switch (C.getSelectionKind()) {
1180   case Comdat::Any:
1181     return bitc::COMDAT_SELECTION_KIND_ANY;
1182   case Comdat::ExactMatch:
1183     return bitc::COMDAT_SELECTION_KIND_EXACT_MATCH;
1184   case Comdat::Largest:
1185     return bitc::COMDAT_SELECTION_KIND_LARGEST;
1186   case Comdat::NoDeduplicate:
1187     return bitc::COMDAT_SELECTION_KIND_NO_DUPLICATES;
1188   case Comdat::SameSize:
1189     return bitc::COMDAT_SELECTION_KIND_SAME_SIZE;
1190   }
1191   llvm_unreachable("Invalid selection kind");
1192 }
1193
1194 static unsigned getEncodedUnnamedAddr(const GlobalValue &GV) {
1195   switch (GV.getUnnamedAddr()) {
1196   case GlobalValue::UnnamedAddr::None:   return 0;
1197   case GlobalValue::UnnamedAddr::Local:  return 2;
1198   case GlobalValue::UnnamedAddr::Global: return 1;
1199   }
1200   llvm_unreachable("Invalid unnamed_addr");
1201 }
1202
1203 size_t ModuleBitcodeWriter::addToStrtab(StringRef Str) {
1204   if (GenerateHash)
1205     Hasher.update(Str);
1206   return StrtabBuilder.add(Str);
1207 }
1208
1209 void ModuleBitcodeWriter::writeComdats() {
1210   SmallVector<unsigned, 64> Vals;
1211   for (const Comdat *C : VE.getComdats()) {
1212     // COMDAT: [strtab offset, strtab size, selection_kind]
1213     Vals.push_back(addToStrtab(C->getName()));
1214     Vals.push_back(C->getName().size());
1215     Vals.push_back(getEncodedComdatSelectionKind(*C));
1216     Stream.EmitRecord(bitc::MODULE_CODE_COMDAT, Vals, /*AbbrevToUse=*/0);
1217     Vals.clear();
1218   }
1219 }
1220
1221 /// Write a record that will eventually hold the word offset of the
1222 /// module-level VST. For now the offset is 0, which will be backpatched
1223 /// after the real VST is written. Saves the bit offset to backpatch.
1224 void ModuleBitcodeWriter::writeValueSymbolTableForwardDecl() {
1225   // Write a placeholder value in for the offset of the real VST,
1226   // which is written after the function blocks so that it can include
1227   // the offset of each function. The placeholder offset will be
1228   // updated when the real VST is written.
1229   auto Abbv = std::make_shared<BitCodeAbbrev>();
1230   Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_VSTOFFSET));
1231   // Blocks are 32-bit aligned, so we can use a 32-bit word offset to
1232   // hold the real VST offset. Must use fixed instead of VBR as we don't
1233   // know how many VBR chunks to reserve ahead of time.
1234   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
1235   unsigned VSTOffsetAbbrev = Stream.EmitAbbrev(std::move(Abbv));
1236
1237   // Emit the placeholder
1238   uint64_t Vals[] = {bitc::MODULE_CODE_VSTOFFSET, 0};
1239   Stream.EmitRecordWithAbbrev(VSTOffsetAbbrev, Vals);
1240
1241   // Compute and save the bit offset to the placeholder, which will be
1242   // patched when the real VST is written. We can simply subtract the 32-bit
1243   // fixed size from the current bit number to get the location to backpatch.
1244   VSTOffsetPlaceholder = Stream.GetCurrentBitNo() - 32;
1245 }
1246
1247 enum StringEncoding { SE_Char6, SE_Fixed7, SE_Fixed8 };
1248
1249 /// Determine the encoding to use for the given string name and length.
1250 static StringEncoding getStringEncoding(StringRef Str) {
1251   bool isChar6 = true;
1252   for (char C : Str) {
1253     if (isChar6)
1254       isChar6 = BitCodeAbbrevOp::isChar6(C);
1255     if ((unsigned char)C & 128)
1256       // don't bother scanning the rest.
1257       return SE_Fixed8;
1258   }
1259   if (isChar6)
1260     return SE_Char6;
1261   return SE_Fixed7;
1262 }
1263
1264 static_assert(sizeof(GlobalValue::SanitizerMetadata) <= sizeof(unsigned),
1265               "Sanitizer Metadata is too large for naive serialization.");
1266 static unsigned
1267 serializeSanitizerMetadata(const GlobalValue::SanitizerMetadata &Meta) {
1268   return Meta.NoAddress | (Meta.NoHWAddress << 1) |
1269          (Meta.Memtag << 2) | (Meta.IsDynInit << 3);
1270 }
1271
1272 /// Emit top-level description of module, including target triple, inline asm,
1273 /// descriptors for global variables, and function prototype info.
1274 /// Returns the bit offset to backpatch with the location of the real VST.
1275 void ModuleBitcodeWriter::writeModuleInfo() {
1276   // Emit various pieces of data attached to a module.
1277   if (!M.getTargetTriple().empty())
1278     writeStringRecord(Stream, bitc::MODULE_CODE_TRIPLE, M.getTargetTriple(),
1279                       0 /*TODO*/);
1280   const std::string &DL = M.getDataLayoutStr();
1281   if (!DL.empty())
1282     writeStringRecord(Stream, bitc::MODULE_CODE_DATALAYOUT, DL, 0 /*TODO*/);
1283   if (!M.getModuleInlineAsm().empty())
1284     writeStringRecord(Stream, bitc::MODULE_CODE_ASM, M.getModuleInlineAsm(),
1285                       0 /*TODO*/);
1286
1287   // Emit information about sections and GC, computing how many there are. Also
1288   // compute the maximum alignment value.
1289   std::map<std::string, unsigned> SectionMap;
1290   std::map<std::string, unsigned> GCMap;
1291   MaybeAlign MaxAlignment;
1292   unsigned MaxGlobalType = 0;
1293   const auto UpdateMaxAlignment = [&MaxAlignment](const MaybeAlign A) {
1294     if (A)
1295       MaxAlignment = !MaxAlignment ? *A : std::max(*MaxAlignment, *A);
1296   };
1297   for (const GlobalVariable &GV : M.globals()) {
1298     UpdateMaxAlignment(GV.getAlign());
1299     MaxGlobalType = std::max(MaxGlobalType, VE.getTypeID(GV.getValueType()));
1300     if (GV.hasSection()) {
1301       // Give section names unique ID's.
1302       unsigned &Entry = SectionMap[std::string(GV.getSection())];
1303       if (!Entry) {
1304         writeStringRecord(Stream, bitc::MODULE_CODE_SECTIONNAME, GV.getSection(),
1305                           0 /*TODO*/);
1306         Entry = SectionMap.size();
1307       }
1308     }
1309   }
1310   for (const Function &F : M) {
1311     UpdateMaxAlignment(F.getAlign());
1312     if (F.hasSection()) {
1313       // Give section names unique ID's.
1314       unsigned &Entry = SectionMap[std::string(F.getSection())];
1315       if (!Entry) {
1316         writeStringRecord(Stream, bitc::MODULE_CODE_SECTIONNAME, F.getSection(),
1317                           0 /*TODO*/);
1318         Entry = SectionMap.size();
1319       }
1320     }
1321     if (F.hasGC()) {
1322       // Same for GC names.
1323       unsigned &Entry = GCMap[F.getGC()];
1324       if (!Entry) {
1325         writeStringRecord(Stream, bitc::MODULE_CODE_GCNAME, F.getGC(),
1326                           0 /*TODO*/);
1327         Entry = GCMap.size();
1328       }
1329     }
1330   }
1331
1332   // Emit abbrev for globals, now that we know # sections and max alignment.
1333   unsigned SimpleGVarAbbrev = 0;
1334   if (!M.global_empty()) {
1335     // Add an abbrev for common globals with no visibility or thread localness.
1336     auto Abbv = std::make_shared<BitCodeAbbrev>();
1337     Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_GLOBALVAR));
1338     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1339     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1340     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1341                               Log2_32_Ceil(MaxGlobalType+1)));
1342     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // AddrSpace << 2
1343                                                            //| explicitType << 1
1344                                                            //| constant
1345     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // Initializer.
1346     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // Linkage.
1347     if (!MaxAlignment)                                     // Alignment.
1348       Abbv->Add(BitCodeAbbrevOp(0));
1349     else {
1350       unsigned MaxEncAlignment = getEncodedAlign(MaxAlignment);
1351       Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1352                                Log2_32_Ceil(MaxEncAlignment+1)));
1353     }
1354     if (SectionMap.empty())                                    // Section.
1355       Abbv->Add(BitCodeAbbrevOp(0));
1356     else
1357       Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1358                                Log2_32_Ceil(SectionMap.size()+1)));
1359     // Don't bother emitting vis + thread local.
1360     SimpleGVarAbbrev = Stream.EmitAbbrev(std::move(Abbv));
1361   }
1362
1363   SmallVector<unsigned, 64> Vals;
1364   // Emit the module's source file name.
1365   {
1366     StringEncoding Bits = getStringEncoding(M.getSourceFileName());
1367     BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8);
1368     if (Bits == SE_Char6)
1369       AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6);
1370     else if (Bits == SE_Fixed7)
1371       AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7);
1372
1373     // MODULE_CODE_SOURCE_FILENAME: [namechar x N]
1374     auto Abbv = std::make_shared<BitCodeAbbrev>();
1375     Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME));
1376     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1377     Abbv->Add(AbbrevOpToUse);
1378     unsigned FilenameAbbrev = Stream.EmitAbbrev(std::move(Abbv));
1379
1380     for (const auto P : M.getSourceFileName())
1381       Vals.push_back((unsigned char)P);
1382
1383     // Emit the finished record.
1384     Stream.EmitRecord(bitc::MODULE_CODE_SOURCE_FILENAME, Vals, FilenameAbbrev);
1385     Vals.clear();
1386   }
1387
1388   // Emit the global variable information.
1389   for (const GlobalVariable &GV : M.globals()) {
1390     unsigned AbbrevToUse = 0;
1391
1392     // GLOBALVAR: [strtab offset, strtab size, type, isconst, initid,
1393     //             linkage, alignment, section, visibility, threadlocal,
1394     //             unnamed_addr, externally_initialized, dllstorageclass,
1395     //             comdat, attributes, DSO_Local, GlobalSanitizer]
1396     Vals.push_back(addToStrtab(GV.getName()));
1397     Vals.push_back(GV.getName().size());
1398     Vals.push_back(VE.getTypeID(GV.getValueType()));
1399     Vals.push_back(GV.getType()->getAddressSpace() << 2 | 2 | GV.isConstant());
1400     Vals.push_back(GV.isDeclaration() ? 0 :
1401                    (VE.getValueID(GV.getInitializer()) + 1));
1402     Vals.push_back(getEncodedLinkage(GV));
1403     Vals.push_back(getEncodedAlign(GV.getAlign()));
1404     Vals.push_back(GV.hasSection() ? SectionMap[std::string(GV.getSection())]
1405                                    : 0);
1406     if (GV.isThreadLocal() ||
1407         GV.getVisibility() != GlobalValue::DefaultVisibility ||
1408         GV.getUnnamedAddr() != GlobalValue::UnnamedAddr::None ||
1409         GV.isExternallyInitialized() ||
1410         GV.getDLLStorageClass() != GlobalValue::DefaultStorageClass ||
1411         GV.hasComdat() || GV.hasAttributes() || GV.isDSOLocal() ||
1412         GV.hasPartition() || GV.hasSanitizerMetadata()) {
1413       Vals.push_back(getEncodedVisibility(GV));
1414       Vals.push_back(getEncodedThreadLocalMode(GV));
1415       Vals.push_back(getEncodedUnnamedAddr(GV));
1416       Vals.push_back(GV.isExternallyInitialized());
1417       Vals.push_back(getEncodedDLLStorageClass(GV));
1418       Vals.push_back(GV.hasComdat() ? VE.getComdatID(GV.getComdat()) : 0);
1419
1420       auto AL = GV.getAttributesAsList(AttributeList::FunctionIndex);
1421       Vals.push_back(VE.getAttributeListID(AL));
1422
1423       Vals.push_back(GV.isDSOLocal());
1424       Vals.push_back(addToStrtab(GV.getPartition()));
1425       Vals.push_back(GV.getPartition().size());
1426
1427       Vals.push_back((GV.hasSanitizerMetadata() ? serializeSanitizerMetadata(
1428                                                       GV.getSanitizerMetadata())
1429                                                 : 0));
1430     } else {
1431       AbbrevToUse = SimpleGVarAbbrev;
1432     }
1433
1434     Stream.EmitRecord(bitc::MODULE_CODE_GLOBALVAR, Vals, AbbrevToUse);
1435     Vals.clear();
1436   }
1437
1438   // Emit the function proto information.
1439   for (const Function &F : M) {
1440     // FUNCTION:  [strtab offset, strtab size, type, callingconv, isproto,
1441     //             linkage, paramattrs, alignment, section, visibility, gc,
1442     //             unnamed_addr, prologuedata, dllstorageclass, comdat,
1443     //             prefixdata, personalityfn, DSO_Local, addrspace]
1444     Vals.push_back(addToStrtab(F.getName()));
1445     Vals.push_back(F.getName().size());
1446     Vals.push_back(VE.getTypeID(F.getFunctionType()));
1447     Vals.push_back(F.getCallingConv());
1448     Vals.push_back(F.isDeclaration());
1449     Vals.push_back(getEncodedLinkage(F));
1450     Vals.push_back(VE.getAttributeListID(F.getAttributes()));
1451     Vals.push_back(getEncodedAlign(F.getAlign()));
1452     Vals.push_back(F.hasSection() ? SectionMap[std::string(F.getSection())]
1453                                   : 0);
1454     Vals.push_back(getEncodedVisibility(F));
1455     Vals.push_back(F.hasGC() ? GCMap[F.getGC()] : 0);
1456     Vals.push_back(getEncodedUnnamedAddr(F));
1457     Vals.push_back(F.hasPrologueData() ? (VE.getValueID(F.getPrologueData()) + 1)
1458                                        : 0);
1459     Vals.push_back(getEncodedDLLStorageClass(F));
1460     Vals.push_back(F.hasComdat() ? VE.getComdatID(F.getComdat()) : 0);
1461     Vals.push_back(F.hasPrefixData() ? (VE.getValueID(F.getPrefixData()) + 1)
1462                                      : 0);
1463     Vals.push_back(
1464         F.hasPersonalityFn() ? (VE.getValueID(F.getPersonalityFn()) + 1) : 0);
1465
1466     Vals.push_back(F.isDSOLocal());
1467     Vals.push_back(F.getAddressSpace());
1468     Vals.push_back(addToStrtab(F.getPartition()));
1469     Vals.push_back(F.getPartition().size());
1470
1471     unsigned AbbrevToUse = 0;
1472     Stream.EmitRecord(bitc::MODULE_CODE_FUNCTION, Vals, AbbrevToUse);
1473     Vals.clear();
1474   }
1475
1476   // Emit the alias information.
1477   for (const GlobalAlias &A : M.aliases()) {
1478     // ALIAS: [strtab offset, strtab size, alias type, aliasee val#, linkage,
1479     //         visibility, dllstorageclass, threadlocal, unnamed_addr,
1480     //         DSO_Local]
1481     Vals.push_back(addToStrtab(A.getName()));
1482     Vals.push_back(A.getName().size());
1483     Vals.push_back(VE.getTypeID(A.getValueType()));
1484     Vals.push_back(A.getType()->getAddressSpace());
1485     Vals.push_back(VE.getValueID(A.getAliasee()));
1486     Vals.push_back(getEncodedLinkage(A));
1487     Vals.push_back(getEncodedVisibility(A));
1488     Vals.push_back(getEncodedDLLStorageClass(A));
1489     Vals.push_back(getEncodedThreadLocalMode(A));
1490     Vals.push_back(getEncodedUnnamedAddr(A));
1491     Vals.push_back(A.isDSOLocal());
1492     Vals.push_back(addToStrtab(A.getPartition()));
1493     Vals.push_back(A.getPartition().size());
1494
1495     unsigned AbbrevToUse = 0;
1496     Stream.EmitRecord(bitc::MODULE_CODE_ALIAS, Vals, AbbrevToUse);
1497     Vals.clear();
1498   }
1499
1500   // Emit the ifunc information.
1501   for (const GlobalIFunc &I : M.ifuncs()) {
1502     // IFUNC: [strtab offset, strtab size, ifunc type, address space, resolver
1503     //         val#, linkage, visibility, DSO_Local]
1504     Vals.push_back(addToStrtab(I.getName()));
1505     Vals.push_back(I.getName().size());
1506     Vals.push_back(VE.getTypeID(I.getValueType()));
1507     Vals.push_back(I.getType()->getAddressSpace());
1508     Vals.push_back(VE.getValueID(I.getResolver()));
1509     Vals.push_back(getEncodedLinkage(I));
1510     Vals.push_back(getEncodedVisibility(I));
1511     Vals.push_back(I.isDSOLocal());
1512     Vals.push_back(addToStrtab(I.getPartition()));
1513     Vals.push_back(I.getPartition().size());
1514     Stream.EmitRecord(bitc::MODULE_CODE_IFUNC, Vals);
1515     Vals.clear();
1516   }
1517
1518   writeValueSymbolTableForwardDecl();
1519 }
1520
1521 static uint64_t getOptimizationFlags(const Value *V) {
1522   uint64_t Flags = 0;
1523
1524   if (const auto *OBO = dyn_cast<OverflowingBinaryOperator>(V)) {
1525     if (OBO->hasNoSignedWrap())
1526       Flags |= 1 << bitc::OBO_NO_SIGNED_WRAP;
1527     if (OBO->hasNoUnsignedWrap())
1528       Flags |= 1 << bitc::OBO_NO_UNSIGNED_WRAP;
1529   } else if (const auto *PEO = dyn_cast<PossiblyExactOperator>(V)) {
1530     if (PEO->isExact())
1531       Flags |= 1 << bitc::PEO_EXACT;
1532   } else if (const auto *FPMO = dyn_cast<FPMathOperator>(V)) {
1533     if (FPMO->hasAllowReassoc())
1534       Flags |= bitc::AllowReassoc;
1535     if (FPMO->hasNoNaNs())
1536       Flags |= bitc::NoNaNs;
1537     if (FPMO->hasNoInfs())
1538       Flags |= bitc::NoInfs;
1539     if (FPMO->hasNoSignedZeros())
1540       Flags |= bitc::NoSignedZeros;
1541     if (FPMO->hasAllowReciprocal())
1542       Flags |= bitc::AllowReciprocal;
1543     if (FPMO->hasAllowContract())
1544       Flags |= bitc::AllowContract;
1545     if (FPMO->hasApproxFunc())
1546       Flags |= bitc::ApproxFunc;
1547   }
1548
1549   return Flags;
1550 }
1551
1552 void ModuleBitcodeWriter::writeValueAsMetadata(
1553     const ValueAsMetadata *MD, SmallVectorImpl<uint64_t> &Record) {
1554   // Mimic an MDNode with a value as one operand.
1555   Value *V = MD->getValue();
1556   Record.push_back(VE.getTypeID(V->getType()));
1557   Record.push_back(VE.getValueID(V));
1558   Stream.EmitRecord(bitc::METADATA_VALUE, Record, 0);
1559   Record.clear();
1560 }
1561
1562 void ModuleBitcodeWriter::writeMDTuple(const MDTuple *N,
1563                                        SmallVectorImpl<uint64_t> &Record,
1564                                        unsigned Abbrev) {
1565   for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
1566     Metadata *MD = N->getOperand(i);
1567     assert(!(MD && isa<LocalAsMetadata>(MD)) &&
1568            "Unexpected function-local metadata");
1569     Record.push_back(VE.getMetadataOrNullID(MD));
1570   }
1571   Stream.EmitRecord(N->isDistinct() ? bitc::METADATA_DISTINCT_NODE
1572                                     : bitc::METADATA_NODE,
1573                     Record, Abbrev);
1574   Record.clear();
1575 }
1576
1577 unsigned ModuleBitcodeWriter::createDILocationAbbrev() {
1578   // Assume the column is usually under 128, and always output the inlined-at
1579   // location (it's never more expensive than building an array size 1).
1580   auto Abbv = std::make_shared<BitCodeAbbrev>();
1581   Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_LOCATION));
1582   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1583   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1584   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1585   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1586   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1587   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1588   return Stream.EmitAbbrev(std::move(Abbv));
1589 }
1590
1591 void ModuleBitcodeWriter::writeDILocation(const DILocation *N,
1592                                           SmallVectorImpl<uint64_t> &Record,
1593                                           unsigned &Abbrev) {
1594   if (!Abbrev)
1595     Abbrev = createDILocationAbbrev();
1596
1597   Record.push_back(N->isDistinct());
1598   Record.push_back(N->getLine());
1599   Record.push_back(N->getColumn());
1600   Record.push_back(VE.getMetadataID(N->getScope()));
1601   Record.push_back(VE.getMetadataOrNullID(N->getInlinedAt()));
1602   Record.push_back(N->isImplicitCode());
1603
1604   Stream.EmitRecord(bitc::METADATA_LOCATION, Record, Abbrev);
1605   Record.clear();
1606 }
1607
1608 unsigned ModuleBitcodeWriter::createGenericDINodeAbbrev() {
1609   // Assume the column is usually under 128, and always output the inlined-at
1610   // location (it's never more expensive than building an array size 1).
1611   auto Abbv = std::make_shared<BitCodeAbbrev>();
1612   Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_GENERIC_DEBUG));
1613   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1614   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1615   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1616   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1617   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1618   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1619   return Stream.EmitAbbrev(std::move(Abbv));
1620 }
1621
1622 void ModuleBitcodeWriter::writeGenericDINode(const GenericDINode *N,
1623                                              SmallVectorImpl<uint64_t> &Record,
1624                                              unsigned &Abbrev) {
1625   if (!Abbrev)
1626     Abbrev = createGenericDINodeAbbrev();
1627
1628   Record.push_back(N->isDistinct());
1629   Record.push_back(N->getTag());
1630   Record.push_back(0); // Per-tag version field; unused for now.
1631
1632   for (auto &I : N->operands())
1633     Record.push_back(VE.getMetadataOrNullID(I));
1634
1635   Stream.EmitRecord(bitc::METADATA_GENERIC_DEBUG, Record, Abbrev);
1636   Record.clear();
1637 }
1638
1639 void ModuleBitcodeWriter::writeDISubrange(const DISubrange *N,
1640                                           SmallVectorImpl<uint64_t> &Record,
1641                                           unsigned Abbrev) {
1642   const uint64_t Version = 2 << 1;
1643   Record.push_back((uint64_t)N->isDistinct() | Version);
1644   Record.push_back(VE.getMetadataOrNullID(N->getRawCountNode()));
1645   Record.push_back(VE.getMetadataOrNullID(N->getRawLowerBound()));
1646   Record.push_back(VE.getMetadataOrNullID(N->getRawUpperBound()));
1647   Record.push_back(VE.getMetadataOrNullID(N->getRawStride()));
1648
1649   Stream.EmitRecord(bitc::METADATA_SUBRANGE, Record, Abbrev);
1650   Record.clear();
1651 }
1652
1653 void ModuleBitcodeWriter::writeDIGenericSubrange(
1654     const DIGenericSubrange *N, SmallVectorImpl<uint64_t> &Record,
1655     unsigned Abbrev) {
1656   Record.push_back((uint64_t)N->isDistinct());
1657   Record.push_back(VE.getMetadataOrNullID(N->getRawCountNode()));
1658   Record.push_back(VE.getMetadataOrNullID(N->getRawLowerBound()));
1659   Record.push_back(VE.getMetadataOrNullID(N->getRawUpperBound()));
1660   Record.push_back(VE.getMetadataOrNullID(N->getRawStride()));
1661
1662   Stream.EmitRecord(bitc::METADATA_GENERIC_SUBRANGE, Record, Abbrev);
1663   Record.clear();
1664 }
1665
1666 static void emitSignedInt64(SmallVectorImpl<uint64_t> &Vals, uint64_t V) {
1667   if ((int64_t)V >= 0)
1668     Vals.push_back(V << 1);
1669   else
1670     Vals.push_back((-V << 1) | 1);
1671 }
1672
1673 static void emitWideAPInt(SmallVectorImpl<uint64_t> &Vals, const APInt &A) {
1674   // We have an arbitrary precision integer value to write whose
1675   // bit width is > 64. However, in canonical unsigned integer
1676   // format it is likely that the high bits are going to be zero.
1677   // So, we only write the number of active words.
1678   unsigned NumWords = A.getActiveWords();
1679   const uint64_t *RawData = A.getRawData();
1680   for (unsigned i = 0; i < NumWords; i++)
1681     emitSignedInt64(Vals, RawData[i]);
1682 }
1683
1684 void ModuleBitcodeWriter::writeDIEnumerator(const DIEnumerator *N,
1685                                             SmallVectorImpl<uint64_t> &Record,
1686                                             unsigned Abbrev) {
1687   const uint64_t IsBigInt = 1 << 2;
1688   Record.push_back(IsBigInt | (N->isUnsigned() << 1) | N->isDistinct());
1689   Record.push_back(N->getValue().getBitWidth());
1690   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
1691   emitWideAPInt(Record, N->getValue());
1692
1693   Stream.EmitRecord(bitc::METADATA_ENUMERATOR, Record, Abbrev);
1694   Record.clear();
1695 }
1696
1697 void ModuleBitcodeWriter::writeDIBasicType(const DIBasicType *N,
1698                                            SmallVectorImpl<uint64_t> &Record,
1699                                            unsigned Abbrev) {
1700   Record.push_back(N->isDistinct());
1701   Record.push_back(N->getTag());
1702   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
1703   Record.push_back(N->getSizeInBits());
1704   Record.push_back(N->getAlignInBits());
1705   Record.push_back(N->getEncoding());
1706   Record.push_back(N->getFlags());
1707
1708   Stream.EmitRecord(bitc::METADATA_BASIC_TYPE, Record, Abbrev);
1709   Record.clear();
1710 }
1711
1712 void ModuleBitcodeWriter::writeDIStringType(const DIStringType *N,
1713                                             SmallVectorImpl<uint64_t> &Record,
1714                                             unsigned Abbrev) {
1715   Record.push_back(N->isDistinct());
1716   Record.push_back(N->getTag());
1717   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
1718   Record.push_back(VE.getMetadataOrNullID(N->getStringLength()));
1719   Record.push_back(VE.getMetadataOrNullID(N->getStringLengthExp()));
1720   Record.push_back(VE.getMetadataOrNullID(N->getStringLocationExp()));
1721   Record.push_back(N->getSizeInBits());
1722   Record.push_back(N->getAlignInBits());
1723   Record.push_back(N->getEncoding());
1724
1725   Stream.EmitRecord(bitc::METADATA_STRING_TYPE, Record, Abbrev);
1726   Record.clear();
1727 }
1728
1729 void ModuleBitcodeWriter::writeDIDerivedType(const DIDerivedType *N,
1730                                              SmallVectorImpl<uint64_t> &Record,
1731                                              unsigned Abbrev) {
1732   Record.push_back(N->isDistinct());
1733   Record.push_back(N->getTag());
1734   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
1735   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
1736   Record.push_back(N->getLine());
1737   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
1738   Record.push_back(VE.getMetadataOrNullID(N->getBaseType()));
1739   Record.push_back(N->getSizeInBits());
1740   Record.push_back(N->getAlignInBits());
1741   Record.push_back(N->getOffsetInBits());
1742   Record.push_back(N->getFlags());
1743   Record.push_back(VE.getMetadataOrNullID(N->getExtraData()));
1744
1745   // DWARF address space is encoded as N->getDWARFAddressSpace() + 1. 0 means
1746   // that there is no DWARF address space associated with DIDerivedType.
1747   if (const auto &DWARFAddressSpace = N->getDWARFAddressSpace())
1748     Record.push_back(*DWARFAddressSpace + 1);
1749   else
1750     Record.push_back(0);
1751
1752   Record.push_back(VE.getMetadataOrNullID(N->getAnnotations().get()));
1753
1754   Stream.EmitRecord(bitc::METADATA_DERIVED_TYPE, Record, Abbrev);
1755   Record.clear();
1756 }
1757
1758 void ModuleBitcodeWriter::writeDICompositeType(
1759     const DICompositeType *N, SmallVectorImpl<uint64_t> &Record,
1760     unsigned Abbrev) {
1761   const unsigned IsNotUsedInOldTypeRef = 0x2;
1762   Record.push_back(IsNotUsedInOldTypeRef | (unsigned)N->isDistinct());
1763   Record.push_back(N->getTag());
1764   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
1765   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
1766   Record.push_back(N->getLine());
1767   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
1768   Record.push_back(VE.getMetadataOrNullID(N->getBaseType()));
1769   Record.push_back(N->getSizeInBits());
1770   Record.push_back(N->getAlignInBits());
1771   Record.push_back(N->getOffsetInBits());
1772   Record.push_back(N->getFlags());
1773   Record.push_back(VE.getMetadataOrNullID(N->getElements().get()));
1774   Record.push_back(N->getRuntimeLang());
1775   Record.push_back(VE.getMetadataOrNullID(N->getVTableHolder()));
1776   Record.push_back(VE.getMetadataOrNullID(N->getTemplateParams().get()));
1777   Record.push_back(VE.getMetadataOrNullID(N->getRawIdentifier()));
1778   Record.push_back(VE.getMetadataOrNullID(N->getDiscriminator()));
1779   Record.push_back(VE.getMetadataOrNullID(N->getRawDataLocation()));
1780   Record.push_back(VE.getMetadataOrNullID(N->getRawAssociated()));
1781   Record.push_back(VE.getMetadataOrNullID(N->getRawAllocated()));
1782   Record.push_back(VE.getMetadataOrNullID(N->getRawRank()));
1783   Record.push_back(VE.getMetadataOrNullID(N->getAnnotations().get()));
1784
1785   Stream.EmitRecord(bitc::METADATA_COMPOSITE_TYPE, Record, Abbrev);
1786   Record.clear();
1787 }
1788
1789 void ModuleBitcodeWriter::writeDISubroutineType(
1790     const DISubroutineType *N, SmallVectorImpl<uint64_t> &Record,
1791     unsigned Abbrev) {
1792   const unsigned HasNoOldTypeRefs = 0x2;
1793   Record.push_back(HasNoOldTypeRefs | (unsigned)N->isDistinct());
1794   Record.push_back(N->getFlags());
1795   Record.push_back(VE.getMetadataOrNullID(N->getTypeArray().get()));
1796   Record.push_back(N->getCC());
1797
1798   Stream.EmitRecord(bitc::METADATA_SUBROUTINE_TYPE, Record, Abbrev);
1799   Record.clear();
1800 }
1801
1802 void ModuleBitcodeWriter::writeDIFile(const DIFile *N,
1803                                       SmallVectorImpl<uint64_t> &Record,
1804                                       unsigned Abbrev) {
1805   Record.push_back(N->isDistinct());
1806   Record.push_back(VE.getMetadataOrNullID(N->getRawFilename()));
1807   Record.push_back(VE.getMetadataOrNullID(N->getRawDirectory()));
1808   if (N->getRawChecksum()) {
1809     Record.push_back(N->getRawChecksum()->Kind);
1810     Record.push_back(VE.getMetadataOrNullID(N->getRawChecksum()->Value));
1811   } else {
1812     // Maintain backwards compatibility with the old internal representation of
1813     // CSK_None in ChecksumKind by writing nulls here when Checksum is None.
1814     Record.push_back(0);
1815     Record.push_back(VE.getMetadataOrNullID(nullptr));
1816   }
1817   auto Source = N->getRawSource();
1818   if (Source)
1819     Record.push_back(VE.getMetadataOrNullID(Source));
1820
1821   Stream.EmitRecord(bitc::METADATA_FILE, Record, Abbrev);
1822   Record.clear();
1823 }
1824
1825 void ModuleBitcodeWriter::writeDICompileUnit(const DICompileUnit *N,
1826                                              SmallVectorImpl<uint64_t> &Record,
1827                                              unsigned Abbrev) {
1828   assert(N->isDistinct() && "Expected distinct compile units");
1829   Record.push_back(/* IsDistinct */ true);
1830   Record.push_back(N->getSourceLanguage());
1831   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
1832   Record.push_back(VE.getMetadataOrNullID(N->getRawProducer()));
1833   Record.push_back(N->isOptimized());
1834   Record.push_back(VE.getMetadataOrNullID(N->getRawFlags()));
1835   Record.push_back(N->getRuntimeVersion());
1836   Record.push_back(VE.getMetadataOrNullID(N->getRawSplitDebugFilename()));
1837   Record.push_back(N->getEmissionKind());
1838   Record.push_back(VE.getMetadataOrNullID(N->getEnumTypes().get()));
1839   Record.push_back(VE.getMetadataOrNullID(N->getRetainedTypes().get()));
1840   Record.push_back(/* subprograms */ 0);
1841   Record.push_back(VE.getMetadataOrNullID(N->getGlobalVariables().get()));
1842   Record.push_back(VE.getMetadataOrNullID(N->getImportedEntities().get()));
1843   Record.push_back(N->getDWOId());
1844   Record.push_back(VE.getMetadataOrNullID(N->getMacros().get()));
1845   Record.push_back(N->getSplitDebugInlining());
1846   Record.push_back(N->getDebugInfoForProfiling());
1847   Record.push_back((unsigned)N->getNameTableKind());
1848   Record.push_back(N->getRangesBaseAddress());
1849   Record.push_back(VE.getMetadataOrNullID(N->getRawSysRoot()));
1850   Record.push_back(VE.getMetadataOrNullID(N->getRawSDK()));
1851
1852   Stream.EmitRecord(bitc::METADATA_COMPILE_UNIT, Record, Abbrev);
1853   Record.clear();
1854 }
1855
1856 void ModuleBitcodeWriter::writeDISubprogram(const DISubprogram *N,
1857                                             SmallVectorImpl<uint64_t> &Record,
1858                                             unsigned Abbrev) {
1859   const uint64_t HasUnitFlag = 1 << 1;
1860   const uint64_t HasSPFlagsFlag = 1 << 2;
1861   Record.push_back(uint64_t(N->isDistinct()) | HasUnitFlag | HasSPFlagsFlag);
1862   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
1863   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
1864   Record.push_back(VE.getMetadataOrNullID(N->getRawLinkageName()));
1865   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
1866   Record.push_back(N->getLine());
1867   Record.push_back(VE.getMetadataOrNullID(N->getType()));
1868   Record.push_back(N->getScopeLine());
1869   Record.push_back(VE.getMetadataOrNullID(N->getContainingType()));
1870   Record.push_back(N->getSPFlags());
1871   Record.push_back(N->getVirtualIndex());
1872   Record.push_back(N->getFlags());
1873   Record.push_back(VE.getMetadataOrNullID(N->getRawUnit()));
1874   Record.push_back(VE.getMetadataOrNullID(N->getTemplateParams().get()));
1875   Record.push_back(VE.getMetadataOrNullID(N->getDeclaration()));
1876   Record.push_back(VE.getMetadataOrNullID(N->getRetainedNodes().get()));
1877   Record.push_back(N->getThisAdjustment());
1878   Record.push_back(VE.getMetadataOrNullID(N->getThrownTypes().get()));
1879   Record.push_back(VE.getMetadataOrNullID(N->getAnnotations().get()));
1880   Record.push_back(VE.getMetadataOrNullID(N->getRawTargetFuncName()));
1881
1882   Stream.EmitRecord(bitc::METADATA_SUBPROGRAM, Record, Abbrev);
1883   Record.clear();
1884 }
1885
1886 void ModuleBitcodeWriter::writeDILexicalBlock(const DILexicalBlock *N,
1887                                               SmallVectorImpl<uint64_t> &Record,
1888                                               unsigned Abbrev) {
1889   Record.push_back(N->isDistinct());
1890   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
1891   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
1892   Record.push_back(N->getLine());
1893   Record.push_back(N->getColumn());
1894
1895   Stream.EmitRecord(bitc::METADATA_LEXICAL_BLOCK, Record, Abbrev);
1896   Record.clear();
1897 }
1898
1899 void ModuleBitcodeWriter::writeDILexicalBlockFile(
1900     const DILexicalBlockFile *N, SmallVectorImpl<uint64_t> &Record,
1901     unsigned Abbrev) {
1902   Record.push_back(N->isDistinct());
1903   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
1904   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
1905   Record.push_back(N->getDiscriminator());
1906
1907   Stream.EmitRecord(bitc::METADATA_LEXICAL_BLOCK_FILE, Record, Abbrev);
1908   Record.clear();
1909 }
1910
1911 void ModuleBitcodeWriter::writeDICommonBlock(const DICommonBlock *N,
1912                                              SmallVectorImpl<uint64_t> &Record,
1913                                              unsigned Abbrev) {
1914   Record.push_back(N->isDistinct());
1915   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
1916   Record.push_back(VE.getMetadataOrNullID(N->getDecl()));
1917   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
1918   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
1919   Record.push_back(N->getLineNo());
1920
1921   Stream.EmitRecord(bitc::METADATA_COMMON_BLOCK, Record, Abbrev);
1922   Record.clear();
1923 }
1924
1925 void ModuleBitcodeWriter::writeDINamespace(const DINamespace *N,
1926                                            SmallVectorImpl<uint64_t> &Record,
1927                                            unsigned Abbrev) {
1928   Record.push_back(N->isDistinct() | N->getExportSymbols() << 1);
1929   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
1930   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
1931
1932   Stream.EmitRecord(bitc::METADATA_NAMESPACE, Record, Abbrev);
1933   Record.clear();
1934 }
1935
1936 void ModuleBitcodeWriter::writeDIMacro(const DIMacro *N,
1937                                        SmallVectorImpl<uint64_t> &Record,
1938                                        unsigned Abbrev) {
1939   Record.push_back(N->isDistinct());
1940   Record.push_back(N->getMacinfoType());
1941   Record.push_back(N->getLine());
1942   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
1943   Record.push_back(VE.getMetadataOrNullID(N->getRawValue()));
1944
1945   Stream.EmitRecord(bitc::METADATA_MACRO, Record, Abbrev);
1946   Record.clear();
1947 }
1948
1949 void ModuleBitcodeWriter::writeDIMacroFile(const DIMacroFile *N,
1950                                            SmallVectorImpl<uint64_t> &Record,
1951                                            unsigned Abbrev) {
1952   Record.push_back(N->isDistinct());
1953   Record.push_back(N->getMacinfoType());
1954   Record.push_back(N->getLine());
1955   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
1956   Record.push_back(VE.getMetadataOrNullID(N->getElements().get()));
1957
1958   Stream.EmitRecord(bitc::METADATA_MACRO_FILE, Record, Abbrev);
1959   Record.clear();
1960 }
1961
1962 void ModuleBitcodeWriter::writeDIArgList(const DIArgList *N,
1963                                          SmallVectorImpl<uint64_t> &Record,
1964                                          unsigned Abbrev) {
1965   Record.reserve(N->getArgs().size());
1966   for (ValueAsMetadata *MD : N->getArgs())
1967     Record.push_back(VE.getMetadataID(MD));
1968
1969   Stream.EmitRecord(bitc::METADATA_ARG_LIST, Record, Abbrev);
1970   Record.clear();
1971 }
1972
1973 void ModuleBitcodeWriter::writeDIModule(const DIModule *N,
1974                                         SmallVectorImpl<uint64_t> &Record,
1975                                         unsigned Abbrev) {
1976   Record.push_back(N->isDistinct());
1977   for (auto &I : N->operands())
1978     Record.push_back(VE.getMetadataOrNullID(I));
1979   Record.push_back(N->getLineNo());
1980   Record.push_back(N->getIsDecl());
1981
1982   Stream.EmitRecord(bitc::METADATA_MODULE, Record, Abbrev);
1983   Record.clear();
1984 }
1985
1986 void ModuleBitcodeWriter::writeDIAssignID(const DIAssignID *N,
1987                                           SmallVectorImpl<uint64_t> &Record,
1988                                           unsigned Abbrev) {
1989   // There are no arguments for this metadata type.
1990   Record.push_back(N->isDistinct());
1991   Stream.EmitRecord(bitc::METADATA_ASSIGN_ID, Record, Abbrev);
1992   Record.clear();
1993 }
1994
1995 void ModuleBitcodeWriter::writeDITemplateTypeParameter(
1996     const DITemplateTypeParameter *N, SmallVectorImpl<uint64_t> &Record,
1997     unsigned Abbrev) {
1998   Record.push_back(N->isDistinct());
1999   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
2000   Record.push_back(VE.getMetadataOrNullID(N->getType()));
2001   Record.push_back(N->isDefault());
2002
2003   Stream.EmitRecord(bitc::METADATA_TEMPLATE_TYPE, Record, Abbrev);
2004   Record.clear();
2005 }
2006
2007 void ModuleBitcodeWriter::writeDITemplateValueParameter(
2008     const DITemplateValueParameter *N, SmallVectorImpl<uint64_t> &Record,
2009     unsigned Abbrev) {
2010   Record.push_back(N->isDistinct());
2011   Record.push_back(N->getTag());
2012   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
2013   Record.push_back(VE.getMetadataOrNullID(N->getType()));
2014   Record.push_back(N->isDefault());
2015   Record.push_back(VE.getMetadataOrNullID(N->getValue()));
2016
2017   Stream.EmitRecord(bitc::METADATA_TEMPLATE_VALUE, Record, Abbrev);
2018   Record.clear();
2019 }
2020
2021 void ModuleBitcodeWriter::writeDIGlobalVariable(
2022     const DIGlobalVariable *N, SmallVectorImpl<uint64_t> &Record,
2023     unsigned Abbrev) {
2024   const uint64_t Version = 2 << 1;
2025   Record.push_back((uint64_t)N->isDistinct() | Version);
2026   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
2027   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
2028   Record.push_back(VE.getMetadataOrNullID(N->getRawLinkageName()));
2029   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
2030   Record.push_back(N->getLine());
2031   Record.push_back(VE.getMetadataOrNullID(N->getType()));
2032   Record.push_back(N->isLocalToUnit());
2033   Record.push_back(N->isDefinition());
2034   Record.push_back(VE.getMetadataOrNullID(N->getStaticDataMemberDeclaration()));
2035   Record.push_back(VE.getMetadataOrNullID(N->getTemplateParams()));
2036   Record.push_back(N->getAlignInBits());
2037   Record.push_back(VE.getMetadataOrNullID(N->getAnnotations().get()));
2038
2039   Stream.EmitRecord(bitc::METADATA_GLOBAL_VAR, Record, Abbrev);
2040   Record.clear();
2041 }
2042
2043 void ModuleBitcodeWriter::writeDILocalVariable(
2044     const DILocalVariable *N, SmallVectorImpl<uint64_t> &Record,
2045     unsigned Abbrev) {
2046   // In order to support all possible bitcode formats in BitcodeReader we need
2047   // to distinguish the following cases:
2048   // 1) Record has no artificial tag (Record[1]),
2049   //   has no obsolete inlinedAt field (Record[9]).
2050   //   In this case Record size will be 8, HasAlignment flag is false.
2051   // 2) Record has artificial tag (Record[1]),
2052   //   has no obsolete inlignedAt field (Record[9]).
2053   //   In this case Record size will be 9, HasAlignment flag is false.
2054   // 3) Record has both artificial tag (Record[1]) and
2055   //   obsolete inlignedAt field (Record[9]).
2056   //   In this case Record size will be 10, HasAlignment flag is false.
2057   // 4) Record has neither artificial tag, nor inlignedAt field, but
2058   //   HasAlignment flag is true and Record[8] contains alignment value.
2059   const uint64_t HasAlignmentFlag = 1 << 1;
2060   Record.push_back((uint64_t)N->isDistinct() | HasAlignmentFlag);
2061   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
2062   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
2063   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
2064   Record.push_back(N->getLine());
2065   Record.push_back(VE.getMetadataOrNullID(N->getType()));
2066   Record.push_back(N->getArg());
2067   Record.push_back(N->getFlags());
2068   Record.push_back(N->getAlignInBits());
2069   Record.push_back(VE.getMetadataOrNullID(N->getAnnotations().get()));
2070
2071   Stream.EmitRecord(bitc::METADATA_LOCAL_VAR, Record, Abbrev);
2072   Record.clear();
2073 }
2074
2075 void ModuleBitcodeWriter::writeDILabel(
2076     const DILabel *N, SmallVectorImpl<uint64_t> &Record,
2077     unsigned Abbrev) {
2078   Record.push_back((uint64_t)N->isDistinct());
2079   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
2080   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
2081   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
2082   Record.push_back(N->getLine());
2083
2084   Stream.EmitRecord(bitc::METADATA_LABEL, Record, Abbrev);
2085   Record.clear();
2086 }
2087
2088 void ModuleBitcodeWriter::writeDIExpression(const DIExpression *N,
2089                                             SmallVectorImpl<uint64_t> &Record,
2090                                             unsigned Abbrev) {
2091   Record.reserve(N->getElements().size() + 1);
2092   const uint64_t Version = 3 << 1;
2093   Record.push_back((uint64_t)N->isDistinct() | Version);
2094   Record.append(N->elements_begin(), N->elements_end());
2095
2096   Stream.EmitRecord(bitc::METADATA_EXPRESSION, Record, Abbrev);
2097   Record.clear();
2098 }
2099
2100 void ModuleBitcodeWriter::writeDIGlobalVariableExpression(
2101     const DIGlobalVariableExpression *N, SmallVectorImpl<uint64_t> &Record,
2102     unsigned Abbrev) {
2103   Record.push_back(N->isDistinct());
2104   Record.push_back(VE.getMetadataOrNullID(N->getVariable()));
2105   Record.push_back(VE.getMetadataOrNullID(N->getExpression()));
2106
2107   Stream.EmitRecord(bitc::METADATA_GLOBAL_VAR_EXPR, Record, Abbrev);
2108   Record.clear();
2109 }
2110
2111 void ModuleBitcodeWriter::writeDIObjCProperty(const DIObjCProperty *N,
2112                                               SmallVectorImpl<uint64_t> &Record,
2113                                               unsigned Abbrev) {
2114   Record.push_back(N->isDistinct());
2115   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
2116   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
2117   Record.push_back(N->getLine());
2118   Record.push_back(VE.getMetadataOrNullID(N->getRawSetterName()));
2119   Record.push_back(VE.getMetadataOrNullID(N->getRawGetterName()));
2120   Record.push_back(N->getAttributes());
2121   Record.push_back(VE.getMetadataOrNullID(N->getType()));
2122
2123   Stream.EmitRecord(bitc::METADATA_OBJC_PROPERTY, Record, Abbrev);
2124   Record.clear();
2125 }
2126
2127 void ModuleBitcodeWriter::writeDIImportedEntity(
2128     const DIImportedEntity *N, SmallVectorImpl<uint64_t> &Record,
2129     unsigned Abbrev) {
2130   Record.push_back(N->isDistinct());
2131   Record.push_back(N->getTag());
2132   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
2133   Record.push_back(VE.getMetadataOrNullID(N->getEntity()));
2134   Record.push_back(N->getLine());
2135   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
2136   Record.push_back(VE.getMetadataOrNullID(N->getRawFile()));
2137   Record.push_back(VE.getMetadataOrNullID(N->getElements().get()));
2138
2139   Stream.EmitRecord(bitc::METADATA_IMPORTED_ENTITY, Record, Abbrev);
2140   Record.clear();
2141 }
2142
2143 unsigned ModuleBitcodeWriter::createNamedMetadataAbbrev() {
2144   auto Abbv = std::make_shared<BitCodeAbbrev>();
2145   Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_NAME));
2146   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2147   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
2148   return Stream.EmitAbbrev(std::move(Abbv));
2149 }
2150
2151 void ModuleBitcodeWriter::writeNamedMetadata(
2152     SmallVectorImpl<uint64_t> &Record) {
2153   if (M.named_metadata_empty())
2154     return;
2155
2156   unsigned Abbrev = createNamedMetadataAbbrev();
2157   for (const NamedMDNode &NMD : M.named_metadata()) {
2158     // Write name.
2159     StringRef Str = NMD.getName();
2160     Record.append(Str.bytes_begin(), Str.bytes_end());
2161     Stream.EmitRecord(bitc::METADATA_NAME, Record, Abbrev);
2162     Record.clear();
2163
2164     // Write named metadata operands.
2165     for (const MDNode *N : NMD.operands())
2166       Record.push_back(VE.getMetadataID(N));
2167     Stream.EmitRecord(bitc::METADATA_NAMED_NODE, Record, 0);
2168     Record.clear();
2169   }
2170 }
2171
2172 unsigned ModuleBitcodeWriter::createMetadataStringsAbbrev() {
2173   auto Abbv = std::make_shared<BitCodeAbbrev>();
2174   Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_STRINGS));
2175   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // # of strings
2176   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // offset to chars
2177   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
2178   return Stream.EmitAbbrev(std::move(Abbv));
2179 }
2180
2181 /// Write out a record for MDString.
2182 ///
2183 /// All the metadata strings in a metadata block are emitted in a single
2184 /// record.  The sizes and strings themselves are shoved into a blob.
2185 void ModuleBitcodeWriter::writeMetadataStrings(
2186     ArrayRef<const Metadata *> Strings, SmallVectorImpl<uint64_t> &Record) {
2187   if (Strings.empty())
2188     return;
2189
2190   // Start the record with the number of strings.
2191   Record.push_back(bitc::METADATA_STRINGS);
2192   Record.push_back(Strings.size());
2193
2194   // Emit the sizes of the strings in the blob.
2195   SmallString<256> Blob;
2196   {
2197     BitstreamWriter W(Blob);
2198     for (const Metadata *MD : Strings)
2199       W.EmitVBR(cast<MDString>(MD)->getLength(), 6);
2200     W.FlushToWord();
2201   }
2202
2203   // Add the offset to the strings to the record.
2204   Record.push_back(Blob.size());
2205
2206   // Add the strings to the blob.
2207   for (const Metadata *MD : Strings)
2208     Blob.append(cast<MDString>(MD)->getString());
2209
2210   // Emit the final record.
2211   Stream.EmitRecordWithBlob(createMetadataStringsAbbrev(), Record, Blob);
2212   Record.clear();
2213 }
2214
2215 // Generates an enum to use as an index in the Abbrev array of Metadata record.
2216 enum MetadataAbbrev : unsigned {
2217 #define HANDLE_MDNODE_LEAF(CLASS) CLASS##AbbrevID,
2218 #include "llvm/IR/Metadata.def"
2219   LastPlusOne
2220 };
2221
2222 void ModuleBitcodeWriter::writeMetadataRecords(
2223     ArrayRef<const Metadata *> MDs, SmallVectorImpl<uint64_t> &Record,
2224     std::vector<unsigned> *MDAbbrevs, std::vector<uint64_t> *IndexPos) {
2225   if (MDs.empty())
2226     return;
2227
2228   // Initialize MDNode abbreviations.
2229 #define HANDLE_MDNODE_LEAF(CLASS) unsigned CLASS##Abbrev = 0;
2230 #include "llvm/IR/Metadata.def"
2231
2232   for (const Metadata *MD : MDs) {
2233     if (IndexPos)
2234       IndexPos->push_back(Stream.GetCurrentBitNo());
2235     if (const MDNode *N = dyn_cast<MDNode>(MD)) {
2236       assert(N->isResolved() && "Expected forward references to be resolved");
2237
2238       switch (N->getMetadataID()) {
2239       default:
2240         llvm_unreachable("Invalid MDNode subclass");
2241 #define HANDLE_MDNODE_LEAF(CLASS)                                              \
2242   case Metadata::CLASS##Kind:                                                  \
2243     if (MDAbbrevs)                                                             \
2244       write##CLASS(cast<CLASS>(N), Record,                                     \
2245                    (*MDAbbrevs)[MetadataAbbrev::CLASS##AbbrevID]);             \
2246     else                                                                       \
2247       write##CLASS(cast<CLASS>(N), Record, CLASS##Abbrev);                     \
2248     continue;
2249 #include "llvm/IR/Metadata.def"
2250       }
2251     }
2252     writeValueAsMetadata(cast<ValueAsMetadata>(MD), Record);
2253   }
2254 }
2255
2256 void ModuleBitcodeWriter::writeModuleMetadata() {
2257   if (!VE.hasMDs() && M.named_metadata_empty())
2258     return;
2259
2260   Stream.EnterSubblock(bitc::METADATA_BLOCK_ID, 4);
2261   SmallVector<uint64_t, 64> Record;
2262
2263   // Emit all abbrevs upfront, so that the reader can jump in the middle of the
2264   // block and load any metadata.
2265   std::vector<unsigned> MDAbbrevs;
2266
2267   MDAbbrevs.resize(MetadataAbbrev::LastPlusOne);
2268   MDAbbrevs[MetadataAbbrev::DILocationAbbrevID] = createDILocationAbbrev();
2269   MDAbbrevs[MetadataAbbrev::GenericDINodeAbbrevID] =
2270       createGenericDINodeAbbrev();
2271
2272   auto Abbv = std::make_shared<BitCodeAbbrev>();
2273   Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_INDEX_OFFSET));
2274   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
2275   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
2276   unsigned OffsetAbbrev = Stream.EmitAbbrev(std::move(Abbv));
2277
2278   Abbv = std::make_shared<BitCodeAbbrev>();
2279   Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_INDEX));
2280   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2281   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
2282   unsigned IndexAbbrev = Stream.EmitAbbrev(std::move(Abbv));
2283
2284   // Emit MDStrings together upfront.
2285   writeMetadataStrings(VE.getMDStrings(), Record);
2286
2287   // We only emit an index for the metadata record if we have more than a given
2288   // (naive) threshold of metadatas, otherwise it is not worth it.
2289   if (VE.getNonMDStrings().size() > IndexThreshold) {
2290     // Write a placeholder value in for the offset of the metadata index,
2291     // which is written after the records, so that it can include
2292     // the offset of each entry. The placeholder offset will be
2293     // updated after all records are emitted.
2294     uint64_t Vals[] = {0, 0};
2295     Stream.EmitRecord(bitc::METADATA_INDEX_OFFSET, Vals, OffsetAbbrev);
2296   }
2297
2298   // Compute and save the bit offset to the current position, which will be
2299   // patched when we emit the index later. We can simply subtract the 64-bit
2300   // fixed size from the current bit number to get the location to backpatch.
2301   uint64_t IndexOffsetRecordBitPos = Stream.GetCurrentBitNo();
2302
2303   // This index will contain the bitpos for each individual record.
2304   std::vector<uint64_t> IndexPos;
2305   IndexPos.reserve(VE.getNonMDStrings().size());
2306
2307   // Write all the records
2308   writeMetadataRecords(VE.getNonMDStrings(), Record, &MDAbbrevs, &IndexPos);
2309
2310   if (VE.getNonMDStrings().size() > IndexThreshold) {
2311     // Now that we have emitted all the records we will emit the index. But
2312     // first
2313     // backpatch the forward reference so that the reader can skip the records
2314     // efficiently.
2315     Stream.BackpatchWord64(IndexOffsetRecordBitPos - 64,
2316                            Stream.GetCurrentBitNo() - IndexOffsetRecordBitPos);
2317
2318     // Delta encode the index.
2319     uint64_t PreviousValue = IndexOffsetRecordBitPos;
2320     for (auto &Elt : IndexPos) {
2321       auto EltDelta = Elt - PreviousValue;
2322       PreviousValue = Elt;
2323       Elt = EltDelta;
2324     }
2325     // Emit the index record.
2326     Stream.EmitRecord(bitc::METADATA_INDEX, IndexPos, IndexAbbrev);
2327     IndexPos.clear();
2328   }
2329
2330   // Write the named metadata now.
2331   writeNamedMetadata(Record);
2332
2333   auto AddDeclAttachedMetadata = [&](const GlobalObject &GO) {
2334     SmallVector<uint64_t, 4> Record;
2335     Record.push_back(VE.getValueID(&GO));
2336     pushGlobalMetadataAttachment(Record, GO);
2337     Stream.EmitRecord(bitc::METADATA_GLOBAL_DECL_ATTACHMENT, Record);
2338   };
2339   for (const Function &F : M)
2340     if (F.isDeclaration() && F.hasMetadata())
2341       AddDeclAttachedMetadata(F);
2342   // FIXME: Only store metadata for declarations here, and move data for global
2343   // variable definitions to a separate block (PR28134).
2344   for (const GlobalVariable &GV : M.globals())
2345     if (GV.hasMetadata())
2346       AddDeclAttachedMetadata(GV);
2347
2348   Stream.ExitBlock();
2349 }
2350
2351 void ModuleBitcodeWriter::writeFunctionMetadata(const Function &F) {
2352   if (!VE.hasMDs())
2353     return;
2354
2355   Stream.EnterSubblock(bitc::METADATA_BLOCK_ID, 3);
2356   SmallVector<uint64_t, 64> Record;
2357   writeMetadataStrings(VE.getMDStrings(), Record);
2358   writeMetadataRecords(VE.getNonMDStrings(), Record);
2359   Stream.ExitBlock();
2360 }
2361
2362 void ModuleBitcodeWriter::pushGlobalMetadataAttachment(
2363     SmallVectorImpl<uint64_t> &Record, const GlobalObject &GO) {
2364   // [n x [id, mdnode]]
2365   SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
2366   GO.getAllMetadata(MDs);
2367   for (const auto &I : MDs) {
2368     Record.push_back(I.first);
2369     Record.push_back(VE.getMetadataID(I.second));
2370   }
2371 }
2372
2373 void ModuleBitcodeWriter::writeFunctionMetadataAttachment(const Function &F) {
2374   Stream.EnterSubblock(bitc::METADATA_ATTACHMENT_ID, 3);
2375
2376   SmallVector<uint64_t, 64> Record;
2377
2378   if (F.hasMetadata()) {
2379     pushGlobalMetadataAttachment(Record, F);
2380     Stream.EmitRecord(bitc::METADATA_ATTACHMENT, Record, 0);
2381     Record.clear();
2382   }
2383
2384   // Write metadata attachments
2385   // METADATA_ATTACHMENT - [m x [value, [n x [id, mdnode]]]
2386   SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
2387   for (const BasicBlock &BB : F)
2388     for (const Instruction &I : BB) {
2389       MDs.clear();
2390       I.getAllMetadataOtherThanDebugLoc(MDs);
2391
2392       // If no metadata, ignore instruction.
2393       if (MDs.empty()) continue;
2394
2395       Record.push_back(VE.getInstructionID(&I));
2396
2397       for (unsigned i = 0, e = MDs.size(); i != e; ++i) {
2398         Record.push_back(MDs[i].first);
2399         Record.push_back(VE.getMetadataID(MDs[i].second));
2400       }
2401       Stream.EmitRecord(bitc::METADATA_ATTACHMENT, Record, 0);
2402       Record.clear();
2403     }
2404
2405   Stream.ExitBlock();
2406 }
2407
2408 void ModuleBitcodeWriter::writeModuleMetadataKinds() {
2409   SmallVector<uint64_t, 64> Record;
2410
2411   // Write metadata kinds
2412   // METADATA_KIND - [n x [id, name]]
2413   SmallVector<StringRef, 8> Names;
2414   M.getMDKindNames(Names);
2415
2416   if (Names.empty()) return;
2417
2418   Stream.EnterSubblock(bitc::METADATA_KIND_BLOCK_ID, 3);
2419
2420   for (unsigned MDKindID = 0, e = Names.size(); MDKindID != e; ++MDKindID) {
2421     Record.push_back(MDKindID);
2422     StringRef KName = Names[MDKindID];
2423     Record.append(KName.begin(), KName.end());
2424
2425     Stream.EmitRecord(bitc::METADATA_KIND, Record, 0);
2426     Record.clear();
2427   }
2428
2429   Stream.ExitBlock();
2430 }
2431
2432 void ModuleBitcodeWriter::writeOperandBundleTags() {
2433   // Write metadata kinds
2434   //
2435   // OPERAND_BUNDLE_TAGS_BLOCK_ID : N x OPERAND_BUNDLE_TAG
2436   //
2437   // OPERAND_BUNDLE_TAG - [strchr x N]
2438
2439   SmallVector<StringRef, 8> Tags;
2440   M.getOperandBundleTags(Tags);
2441
2442   if (Tags.empty())
2443     return;
2444
2445   Stream.EnterSubblock(bitc::OPERAND_BUNDLE_TAGS_BLOCK_ID, 3);
2446
2447   SmallVector<uint64_t, 64> Record;
2448
2449   for (auto Tag : Tags) {
2450     Record.append(Tag.begin(), Tag.end());
2451
2452     Stream.EmitRecord(bitc::OPERAND_BUNDLE_TAG, Record, 0);
2453     Record.clear();
2454   }
2455
2456   Stream.ExitBlock();
2457 }
2458
2459 void ModuleBitcodeWriter::writeSyncScopeNames() {
2460   SmallVector<StringRef, 8> SSNs;
2461   M.getContext().getSyncScopeNames(SSNs);
2462   if (SSNs.empty())
2463     return;
2464
2465   Stream.EnterSubblock(bitc::SYNC_SCOPE_NAMES_BLOCK_ID, 2);
2466
2467   SmallVector<uint64_t, 64> Record;
2468   for (auto SSN : SSNs) {
2469     Record.append(SSN.begin(), SSN.end());
2470     Stream.EmitRecord(bitc::SYNC_SCOPE_NAME, Record, 0);
2471     Record.clear();
2472   }
2473
2474   Stream.ExitBlock();
2475 }
2476
2477 void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal,
2478                                          bool isGlobal) {
2479   if (FirstVal == LastVal) return;
2480
2481   Stream.EnterSubblock(bitc::CONSTANTS_BLOCK_ID, 4);
2482
2483   unsigned AggregateAbbrev = 0;
2484   unsigned String8Abbrev = 0;
2485   unsigned CString7Abbrev = 0;
2486   unsigned CString6Abbrev = 0;
2487   // If this is a constant pool for the module, emit module-specific abbrevs.
2488   if (isGlobal) {
2489     // Abbrev for CST_CODE_AGGREGATE.
2490     auto Abbv = std::make_shared<BitCodeAbbrev>();
2491     Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_AGGREGATE));
2492     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2493     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, Log2_32_Ceil(LastVal+1)));
2494     AggregateAbbrev = Stream.EmitAbbrev(std::move(Abbv));
2495
2496     // Abbrev for CST_CODE_STRING.
2497     Abbv = std::make_shared<BitCodeAbbrev>();
2498     Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_STRING));
2499     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2500     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
2501     String8Abbrev = Stream.EmitAbbrev(std::move(Abbv));
2502     // Abbrev for CST_CODE_CSTRING.
2503     Abbv = std::make_shared<BitCodeAbbrev>();
2504     Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_CSTRING));
2505     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2506     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
2507     CString7Abbrev = Stream.EmitAbbrev(std::move(Abbv));
2508     // Abbrev for CST_CODE_CSTRING.
2509     Abbv = std::make_shared<BitCodeAbbrev>();
2510     Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_CSTRING));
2511     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2512     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
2513     CString6Abbrev = Stream.EmitAbbrev(std::move(Abbv));
2514   }
2515
2516   SmallVector<uint64_t, 64> Record;
2517
2518   const ValueEnumerator::ValueList &Vals = VE.getValues();
2519   Type *LastTy = nullptr;
2520   for (unsigned i = FirstVal; i != LastVal; ++i) {
2521     const Value *V = Vals[i].first;
2522     // If we need to switch types, do so now.
2523     if (V->getType() != LastTy) {
2524       LastTy = V->getType();
2525       Record.push_back(VE.getTypeID(LastTy));
2526       Stream.EmitRecord(bitc::CST_CODE_SETTYPE, Record,
2527                         CONSTANTS_SETTYPE_ABBREV);
2528       Record.clear();
2529     }
2530
2531     if (const InlineAsm *IA = dyn_cast<InlineAsm>(V)) {
2532       Record.push_back(VE.getTypeID(IA->getFunctionType()));
2533       Record.push_back(
2534           unsigned(IA->hasSideEffects()) | unsigned(IA->isAlignStack()) << 1 |
2535           unsigned(IA->getDialect() & 1) << 2 | unsigned(IA->canThrow()) << 3);
2536
2537       // Add the asm string.
2538       const std::string &AsmStr = IA->getAsmString();
2539       Record.push_back(AsmStr.size());
2540       Record.append(AsmStr.begin(), AsmStr.end());
2541
2542       // Add the constraint string.
2543       const std::string &ConstraintStr = IA->getConstraintString();
2544       Record.push_back(ConstraintStr.size());
2545       Record.append(ConstraintStr.begin(), ConstraintStr.end());
2546       Stream.EmitRecord(bitc::CST_CODE_INLINEASM, Record);
2547       Record.clear();
2548       continue;
2549     }
2550     const Constant *C = cast<Constant>(V);
2551     unsigned Code = -1U;
2552     unsigned AbbrevToUse = 0;
2553     if (C->isNullValue()) {
2554       Code = bitc::CST_CODE_NULL;
2555     } else if (isa<PoisonValue>(C)) {
2556       Code = bitc::CST_CODE_POISON;
2557     } else if (isa<UndefValue>(C)) {
2558       Code = bitc::CST_CODE_UNDEF;
2559     } else if (const ConstantInt *IV = dyn_cast<ConstantInt>(C)) {
2560       if (IV->getBitWidth() <= 64) {
2561         uint64_t V = IV->getSExtValue();
2562         emitSignedInt64(Record, V);
2563         Code = bitc::CST_CODE_INTEGER;
2564         AbbrevToUse = CONSTANTS_INTEGER_ABBREV;
2565       } else {                             // Wide integers, > 64 bits in size.
2566         emitWideAPInt(Record, IV->getValue());
2567         Code = bitc::CST_CODE_WIDE_INTEGER;
2568       }
2569     } else if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C)) {
2570       Code = bitc::CST_CODE_FLOAT;
2571       Type *Ty = CFP->getType();
2572       if (Ty->isHalfTy() || Ty->isBFloatTy() || Ty->isFloatTy() ||
2573           Ty->isDoubleTy()) {
2574         Record.push_back(CFP->getValueAPF().bitcastToAPInt().getZExtValue());
2575       } else if (Ty->isX86_FP80Ty()) {
2576         // api needed to prevent premature destruction
2577         // bits are not in the same order as a normal i80 APInt, compensate.
2578         APInt api = CFP->getValueAPF().bitcastToAPInt();
2579         const uint64_t *p = api.getRawData();
2580         Record.push_back((p[1] << 48) | (p[0] >> 16));
2581         Record.push_back(p[0] & 0xffffLL);
2582       } else if (Ty->isFP128Ty() || Ty->isPPC_FP128Ty()) {
2583         APInt api = CFP->getValueAPF().bitcastToAPInt();
2584         const uint64_t *p = api.getRawData();
2585         Record.push_back(p[0]);
2586         Record.push_back(p[1]);
2587       } else {
2588         assert(0 && "Unknown FP type!");
2589       }
2590     } else if (isa<ConstantDataSequential>(C) &&
2591                cast<ConstantDataSequential>(C)->isString()) {
2592       const ConstantDataSequential *Str = cast<ConstantDataSequential>(C);
2593       // Emit constant strings specially.
2594       unsigned NumElts = Str->getNumElements();
2595       // If this is a null-terminated string, use the denser CSTRING encoding.
2596       if (Str->isCString()) {
2597         Code = bitc::CST_CODE_CSTRING;
2598         --NumElts;  // Don't encode the null, which isn't allowed by char6.
2599       } else {
2600         Code = bitc::CST_CODE_STRING;
2601         AbbrevToUse = String8Abbrev;
2602       }
2603       bool isCStr7 = Code == bitc::CST_CODE_CSTRING;
2604       bool isCStrChar6 = Code == bitc::CST_CODE_CSTRING;
2605       for (unsigned i = 0; i != NumElts; ++i) {
2606         unsigned char V = Str->getElementAsInteger(i);
2607         Record.push_back(V);
2608         isCStr7 &= (V & 128) == 0;
2609         if (isCStrChar6)
2610           isCStrChar6 = BitCodeAbbrevOp::isChar6(V);
2611       }
2612
2613       if (isCStrChar6)
2614         AbbrevToUse = CString6Abbrev;
2615       else if (isCStr7)
2616         AbbrevToUse = CString7Abbrev;
2617     } else if (const ConstantDataSequential *CDS =
2618                   dyn_cast<ConstantDataSequential>(C)) {
2619       Code = bitc::CST_CODE_DATA;
2620       Type *EltTy = CDS->getElementType();
2621       if (isa<IntegerType>(EltTy)) {
2622         for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i)
2623           Record.push_back(CDS->getElementAsInteger(i));
2624       } else {
2625         for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i)
2626           Record.push_back(
2627               CDS->getElementAsAPFloat(i).bitcastToAPInt().getLimitedValue());
2628       }
2629     } else if (isa<ConstantAggregate>(C)) {
2630       Code = bitc::CST_CODE_AGGREGATE;
2631       for (const Value *Op : C->operands())
2632         Record.push_back(VE.getValueID(Op));
2633       AbbrevToUse = AggregateAbbrev;
2634     } else if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
2635       switch (CE->getOpcode()) {
2636       default:
2637         if (Instruction::isCast(CE->getOpcode())) {
2638           Code = bitc::CST_CODE_CE_CAST;
2639           Record.push_back(getEncodedCastOpcode(CE->getOpcode()));
2640           Record.push_back(VE.getTypeID(C->getOperand(0)->getType()));
2641           Record.push_back(VE.getValueID(C->getOperand(0)));
2642           AbbrevToUse = CONSTANTS_CE_CAST_Abbrev;
2643         } else {
2644           assert(CE->getNumOperands() == 2 && "Unknown constant expr!");
2645           Code = bitc::CST_CODE_CE_BINOP;
2646           Record.push_back(getEncodedBinaryOpcode(CE->getOpcode()));
2647           Record.push_back(VE.getValueID(C->getOperand(0)));
2648           Record.push_back(VE.getValueID(C->getOperand(1)));
2649           uint64_t Flags = getOptimizationFlags(CE);
2650           if (Flags != 0)
2651             Record.push_back(Flags);
2652         }
2653         break;
2654       case Instruction::FNeg: {
2655         assert(CE->getNumOperands() == 1 && "Unknown constant expr!");
2656         Code = bitc::CST_CODE_CE_UNOP;
2657         Record.push_back(getEncodedUnaryOpcode(CE->getOpcode()));
2658         Record.push_back(VE.getValueID(C->getOperand(0)));
2659         uint64_t Flags = getOptimizationFlags(CE);
2660         if (Flags != 0)
2661           Record.push_back(Flags);
2662         break;
2663       }
2664       case Instruction::GetElementPtr: {
2665         Code = bitc::CST_CODE_CE_GEP;
2666         const auto *GO = cast<GEPOperator>(C);
2667         Record.push_back(VE.getTypeID(GO->getSourceElementType()));
2668         if (std::optional<unsigned> Idx = GO->getInRangeIndex()) {
2669           Code = bitc::CST_CODE_CE_GEP_WITH_INRANGE_INDEX;
2670           Record.push_back((*Idx << 1) | GO->isInBounds());
2671         } else if (GO->isInBounds())
2672           Code = bitc::CST_CODE_CE_INBOUNDS_GEP;
2673         for (unsigned i = 0, e = CE->getNumOperands(); i != e; ++i) {
2674           Record.push_back(VE.getTypeID(C->getOperand(i)->getType()));
2675           Record.push_back(VE.getValueID(C->getOperand(i)));
2676         }
2677         break;
2678       }
2679       case Instruction::ExtractElement:
2680         Code = bitc::CST_CODE_CE_EXTRACTELT;
2681         Record.push_back(VE.getTypeID(C->getOperand(0)->getType()));
2682         Record.push_back(VE.getValueID(C->getOperand(0)));
2683         Record.push_back(VE.getTypeID(C->getOperand(1)->getType()));
2684         Record.push_back(VE.getValueID(C->getOperand(1)));
2685         break;
2686       case Instruction::InsertElement:
2687         Code = bitc::CST_CODE_CE_INSERTELT;
2688         Record.push_back(VE.getValueID(C->getOperand(0)));
2689         Record.push_back(VE.getValueID(C->getOperand(1)));
2690         Record.push_back(VE.getTypeID(C->getOperand(2)->getType()));
2691         Record.push_back(VE.getValueID(C->getOperand(2)));
2692         break;
2693       case Instruction::ShuffleVector:
2694         // If the return type and argument types are the same, this is a
2695         // standard shufflevector instruction.  If the types are different,
2696         // then the shuffle is widening or truncating the input vectors, and
2697         // the argument type must also be encoded.
2698         if (C->getType() == C->getOperand(0)->getType()) {
2699           Code = bitc::CST_CODE_CE_SHUFFLEVEC;
2700         } else {
2701           Code = bitc::CST_CODE_CE_SHUFVEC_EX;
2702           Record.push_back(VE.getTypeID(C->getOperand(0)->getType()));
2703         }
2704         Record.push_back(VE.getValueID(C->getOperand(0)));
2705         Record.push_back(VE.getValueID(C->getOperand(1)));
2706         Record.push_back(VE.getValueID(CE->getShuffleMaskForBitcode()));
2707         break;
2708       case Instruction::ICmp:
2709       case Instruction::FCmp:
2710         Code = bitc::CST_CODE_CE_CMP;
2711         Record.push_back(VE.getTypeID(C->getOperand(0)->getType()));
2712         Record.push_back(VE.getValueID(C->getOperand(0)));
2713         Record.push_back(VE.getValueID(C->getOperand(1)));
2714         Record.push_back(CE->getPredicate());
2715         break;
2716       }
2717     } else if (const BlockAddress *BA = dyn_cast<BlockAddress>(C)) {
2718       Code = bitc::CST_CODE_BLOCKADDRESS;
2719       Record.push_back(VE.getTypeID(BA->getFunction()->getType()));
2720       Record.push_back(VE.getValueID(BA->getFunction()));
2721       Record.push_back(VE.getGlobalBasicBlockID(BA->getBasicBlock()));
2722     } else if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C)) {
2723       Code = bitc::CST_CODE_DSO_LOCAL_EQUIVALENT;
2724       Record.push_back(VE.getTypeID(Equiv->getGlobalValue()->getType()));
2725       Record.push_back(VE.getValueID(Equiv->getGlobalValue()));
2726     } else if (const auto *NC = dyn_cast<NoCFIValue>(C)) {
2727       Code = bitc::CST_CODE_NO_CFI_VALUE;
2728       Record.push_back(VE.getTypeID(NC->getGlobalValue()->getType()));
2729       Record.push_back(VE.getValueID(NC->getGlobalValue()));
2730     } else {
2731 #ifndef NDEBUG
2732       C->dump();
2733 #endif
2734       llvm_unreachable("Unknown constant!");
2735     }
2736     Stream.EmitRecord(Code, Record, AbbrevToUse);
2737     Record.clear();
2738   }
2739
2740   Stream.ExitBlock();
2741 }
2742
2743 void ModuleBitcodeWriter::writeModuleConstants() {
2744   const ValueEnumerator::ValueList &Vals = VE.getValues();
2745
2746   // Find the first constant to emit, which is the first non-globalvalue value.
2747   // We know globalvalues have been emitted by WriteModuleInfo.
2748   for (unsigned i = 0, e = Vals.size(); i != e; ++i) {
2749     if (!isa<GlobalValue>(Vals[i].first)) {
2750       writeConstants(i, Vals.size(), true);
2751       return;
2752     }
2753   }
2754 }
2755
2756 /// pushValueAndType - The file has to encode both the value and type id for
2757 /// many values, because we need to know what type to create for forward
2758 /// references.  However, most operands are not forward references, so this type
2759 /// field is not needed.
2760 ///
2761 /// This function adds V's value ID to Vals.  If the value ID is higher than the
2762 /// instruction ID, then it is a forward reference, and it also includes the
2763 /// type ID.  The value ID that is written is encoded relative to the InstID.
2764 bool ModuleBitcodeWriter::pushValueAndType(const Value *V, unsigned InstID,
2765                                            SmallVectorImpl<unsigned> &Vals) {
2766   unsigned ValID = VE.getValueID(V);
2767   // Make encoding relative to the InstID.
2768   Vals.push_back(InstID - ValID);
2769   if (ValID >= InstID) {
2770     Vals.push_back(VE.getTypeID(V->getType()));
2771     return true;
2772   }
2773   return false;
2774 }
2775
2776 void ModuleBitcodeWriter::writeOperandBundles(const CallBase &CS,
2777                                               unsigned InstID) {
2778   SmallVector<unsigned, 64> Record;
2779   LLVMContext &C = CS.getContext();
2780
2781   for (unsigned i = 0, e = CS.getNumOperandBundles(); i != e; ++i) {
2782     const auto &Bundle = CS.getOperandBundleAt(i);
2783     Record.push_back(C.getOperandBundleTagID(Bundle.getTagName()));
2784
2785     for (auto &Input : Bundle.Inputs)
2786       pushValueAndType(Input, InstID, Record);
2787
2788     Stream.EmitRecord(bitc::FUNC_CODE_OPERAND_BUNDLE, Record);
2789     Record.clear();
2790   }
2791 }
2792
2793 /// pushValue - Like pushValueAndType, but where the type of the value is
2794 /// omitted (perhaps it was already encoded in an earlier operand).
2795 void ModuleBitcodeWriter::pushValue(const Value *V, unsigned InstID,
2796                                     SmallVectorImpl<unsigned> &Vals) {
2797   unsigned ValID = VE.getValueID(V);
2798   Vals.push_back(InstID - ValID);
2799 }
2800
2801 void ModuleBitcodeWriter::pushValueSigned(const Value *V, unsigned InstID,
2802                                           SmallVectorImpl<uint64_t> &Vals) {
2803   unsigned ValID = VE.getValueID(V);
2804   int64_t diff = ((int32_t)InstID - (int32_t)ValID);
2805   emitSignedInt64(Vals, diff);
2806 }
2807
2808 /// WriteInstruction - Emit an instruction to the specified stream.
2809 void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
2810                                            unsigned InstID,
2811                                            SmallVectorImpl<unsigned> &Vals) {
2812   unsigned Code = 0;
2813   unsigned AbbrevToUse = 0;
2814   VE.setInstructionID(&I);
2815   switch (I.getOpcode()) {
2816   default:
2817     if (Instruction::isCast(I.getOpcode())) {
2818       Code = bitc::FUNC_CODE_INST_CAST;
2819       if (!pushValueAndType(I.getOperand(0), InstID, Vals))
2820         AbbrevToUse = FUNCTION_INST_CAST_ABBREV;
2821       Vals.push_back(VE.getTypeID(I.getType()));
2822       Vals.push_back(getEncodedCastOpcode(I.getOpcode()));
2823     } else {
2824       assert(isa<BinaryOperator>(I) && "Unknown instruction!");
2825       Code = bitc::FUNC_CODE_INST_BINOP;
2826       if (!pushValueAndType(I.getOperand(0), InstID, Vals))
2827         AbbrevToUse = FUNCTION_INST_BINOP_ABBREV;
2828       pushValue(I.getOperand(1), InstID, Vals);
2829       Vals.push_back(getEncodedBinaryOpcode(I.getOpcode()));
2830       uint64_t Flags = getOptimizationFlags(&I);
2831       if (Flags != 0) {
2832         if (AbbrevToUse == FUNCTION_INST_BINOP_ABBREV)
2833           AbbrevToUse = FUNCTION_INST_BINOP_FLAGS_ABBREV;
2834         Vals.push_back(Flags);
2835       }
2836     }
2837     break;
2838   case Instruction::FNeg: {
2839     Code = bitc::FUNC_CODE_INST_UNOP;
2840     if (!pushValueAndType(I.getOperand(0), InstID, Vals))
2841       AbbrevToUse = FUNCTION_INST_UNOP_ABBREV;
2842     Vals.push_back(getEncodedUnaryOpcode(I.getOpcode()));
2843     uint64_t Flags = getOptimizationFlags(&I);
2844     if (Flags != 0) {
2845       if (AbbrevToUse == FUNCTION_INST_UNOP_ABBREV)
2846         AbbrevToUse = FUNCTION_INST_UNOP_FLAGS_ABBREV;
2847       Vals.push_back(Flags);
2848     }
2849     break;
2850   }
2851   case Instruction::GetElementPtr: {
2852     Code = bitc::FUNC_CODE_INST_GEP;
2853     AbbrevToUse = FUNCTION_INST_GEP_ABBREV;
2854     auto &GEPInst = cast<GetElementPtrInst>(I);
2855     Vals.push_back(GEPInst.isInBounds());
2856     Vals.push_back(VE.getTypeID(GEPInst.getSourceElementType()));
2857     for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i)
2858       pushValueAndType(I.getOperand(i), InstID, Vals);
2859     break;
2860   }
2861   case Instruction::ExtractValue: {
2862     Code = bitc::FUNC_CODE_INST_EXTRACTVAL;
2863     pushValueAndType(I.getOperand(0), InstID, Vals);
2864     const ExtractValueInst *EVI = cast<ExtractValueInst>(&I);
2865     Vals.append(EVI->idx_begin(), EVI->idx_end());
2866     break;
2867   }
2868   case Instruction::InsertValue: {
2869     Code = bitc::FUNC_CODE_INST_INSERTVAL;
2870     pushValueAndType(I.getOperand(0), InstID, Vals);
2871     pushValueAndType(I.getOperand(1), InstID, Vals);
2872     const InsertValueInst *IVI = cast<InsertValueInst>(&I);
2873     Vals.append(IVI->idx_begin(), IVI->idx_end());
2874     break;
2875   }
2876   case Instruction::Select: {
2877     Code = bitc::FUNC_CODE_INST_VSELECT;
2878     pushValueAndType(I.getOperand(1), InstID, Vals);
2879     pushValue(I.getOperand(2), InstID, Vals);
2880     pushValueAndType(I.getOperand(0), InstID, Vals);
2881     uint64_t Flags = getOptimizationFlags(&I);
2882     if (Flags != 0)
2883       Vals.push_back(Flags);
2884     break;
2885   }
2886   case Instruction::ExtractElement:
2887     Code = bitc::FUNC_CODE_INST_EXTRACTELT;
2888     pushValueAndType(I.getOperand(0), InstID, Vals);
2889     pushValueAndType(I.getOperand(1), InstID, Vals);
2890     break;
2891   case Instruction::InsertElement:
2892     Code = bitc::FUNC_CODE_INST_INSERTELT;
2893     pushValueAndType(I.getOperand(0), InstID, Vals);
2894     pushValue(I.getOperand(1), InstID, Vals);
2895     pushValueAndType(I.getOperand(2), InstID, Vals);
2896     break;
2897   case Instruction::ShuffleVector:
2898     Code = bitc::FUNC_CODE_INST_SHUFFLEVEC;
2899     pushValueAndType(I.getOperand(0), InstID, Vals);
2900     pushValue(I.getOperand(1), InstID, Vals);
2901     pushValue(cast<ShuffleVectorInst>(I).getShuffleMaskForBitcode(), InstID,
2902               Vals);
2903     break;
2904   case Instruction::ICmp:
2905   case Instruction::FCmp: {
2906     // compare returning Int1Ty or vector of Int1Ty
2907     Code = bitc::FUNC_CODE_INST_CMP2;
2908     pushValueAndType(I.getOperand(0), InstID, Vals);
2909     pushValue(I.getOperand(1), InstID, Vals);
2910     Vals.push_back(cast<CmpInst>(I).getPredicate());
2911     uint64_t Flags = getOptimizationFlags(&I);
2912     if (Flags != 0)
2913       Vals.push_back(Flags);
2914     break;
2915   }
2916
2917   case Instruction::Ret:
2918     {
2919       Code = bitc::FUNC_CODE_INST_RET;
2920       unsigned NumOperands = I.getNumOperands();
2921       if (NumOperands == 0)
2922         AbbrevToUse = FUNCTION_INST_RET_VOID_ABBREV;
2923       else if (NumOperands == 1) {
2924         if (!pushValueAndType(I.getOperand(0), InstID, Vals))
2925           AbbrevToUse = FUNCTION_INST_RET_VAL_ABBREV;
2926       } else {
2927         for (unsigned i = 0, e = NumOperands; i != e; ++i)
2928           pushValueAndType(I.getOperand(i), InstID, Vals);
2929       }
2930     }
2931     break;
2932   case Instruction::Br:
2933     {
2934       Code = bitc::FUNC_CODE_INST_BR;
2935       const BranchInst &II = cast<BranchInst>(I);
2936       Vals.push_back(VE.getValueID(II.getSuccessor(0)));
2937       if (II.isConditional()) {
2938         Vals.push_back(VE.getValueID(II.getSuccessor(1)));
2939         pushValue(II.getCondition(), InstID, Vals);
2940       }
2941     }
2942     break;
2943   case Instruction::Switch:
2944     {
2945       Code = bitc::FUNC_CODE_INST_SWITCH;
2946       const SwitchInst &SI = cast<SwitchInst>(I);
2947       Vals.push_back(VE.getTypeID(SI.getCondition()->getType()));
2948       pushValue(SI.getCondition(), InstID, Vals);
2949       Vals.push_back(VE.getValueID(SI.getDefaultDest()));
2950       for (auto Case : SI.cases()) {
2951         Vals.push_back(VE.getValueID(Case.getCaseValue()));
2952         Vals.push_back(VE.getValueID(Case.getCaseSuccessor()));
2953       }
2954     }
2955     break;
2956   case Instruction::IndirectBr:
2957     Code = bitc::FUNC_CODE_INST_INDIRECTBR;
2958     Vals.push_back(VE.getTypeID(I.getOperand(0)->getType()));
2959     // Encode the address operand as relative, but not the basic blocks.
2960     pushValue(I.getOperand(0), InstID, Vals);
2961     for (unsigned i = 1, e = I.getNumOperands(); i != e; ++i)
2962       Vals.push_back(VE.getValueID(I.getOperand(i)));
2963     break;
2964
2965   case Instruction::Invoke: {
2966     const InvokeInst *II = cast<InvokeInst>(&I);
2967     const Value *Callee = II->getCalledOperand();
2968     FunctionType *FTy = II->getFunctionType();
2969
2970     if (II->hasOperandBundles())
2971       writeOperandBundles(*II, InstID);
2972
2973     Code = bitc::FUNC_CODE_INST_INVOKE;
2974
2975     Vals.push_back(VE.getAttributeListID(II->getAttributes()));
2976     Vals.push_back(II->getCallingConv() | 1 << 13);
2977     Vals.push_back(VE.getValueID(II->getNormalDest()));
2978     Vals.push_back(VE.getValueID(II->getUnwindDest()));
2979     Vals.push_back(VE.getTypeID(FTy));
2980     pushValueAndType(Callee, InstID, Vals);
2981
2982     // Emit value #'s for the fixed parameters.
2983     for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
2984       pushValue(I.getOperand(i), InstID, Vals); // fixed param.
2985
2986     // Emit type/value pairs for varargs params.
2987     if (FTy->isVarArg()) {
2988       for (unsigned i = FTy->getNumParams(), e = II->arg_size(); i != e; ++i)
2989         pushValueAndType(I.getOperand(i), InstID, Vals); // vararg
2990     }
2991     break;
2992   }
2993   case Instruction::Resume:
2994     Code = bitc::FUNC_CODE_INST_RESUME;
2995     pushValueAndType(I.getOperand(0), InstID, Vals);
2996     break;
2997   case Instruction::CleanupRet: {
2998     Code = bitc::FUNC_CODE_INST_CLEANUPRET;
2999     const auto &CRI = cast<CleanupReturnInst>(I);
3000     pushValue(CRI.getCleanupPad(), InstID, Vals);
3001     if (CRI.hasUnwindDest())
3002       Vals.push_back(VE.getValueID(CRI.getUnwindDest()));
3003     break;
3004   }
3005   case Instruction::CatchRet: {
3006     Code = bitc::FUNC_CODE_INST_CATCHRET;
3007     const auto &CRI = cast<CatchReturnInst>(I);
3008     pushValue(CRI.getCatchPad(), InstID, Vals);
3009     Vals.push_back(VE.getValueID(CRI.getSuccessor()));
3010     break;
3011   }
3012   case Instruction::CleanupPad:
3013   case Instruction::CatchPad: {
3014     const auto &FuncletPad = cast<FuncletPadInst>(I);
3015     Code = isa<CatchPadInst>(FuncletPad) ? bitc::FUNC_CODE_INST_CATCHPAD
3016                                          : bitc::FUNC_CODE_INST_CLEANUPPAD;
3017     pushValue(FuncletPad.getParentPad(), InstID, Vals);
3018
3019     unsigned NumArgOperands = FuncletPad.arg_size();
3020     Vals.push_back(NumArgOperands);
3021     for (unsigned Op = 0; Op != NumArgOperands; ++Op)
3022       pushValueAndType(FuncletPad.getArgOperand(Op), InstID, Vals);
3023     break;
3024   }
3025   case Instruction::CatchSwitch: {
3026     Code = bitc::FUNC_CODE_INST_CATCHSWITCH;
3027     const auto &CatchSwitch = cast<CatchSwitchInst>(I);
3028
3029     pushValue(CatchSwitch.getParentPad(), InstID, Vals);
3030
3031     unsigned NumHandlers = CatchSwitch.getNumHandlers();
3032     Vals.push_back(NumHandlers);
3033     for (const BasicBlock *CatchPadBB : CatchSwitch.handlers())
3034       Vals.push_back(VE.getValueID(CatchPadBB));
3035
3036     if (CatchSwitch.hasUnwindDest())
3037       Vals.push_back(VE.getValueID(CatchSwitch.getUnwindDest()));
3038     break;
3039   }
3040   case Instruction::CallBr: {
3041     const CallBrInst *CBI = cast<CallBrInst>(&I);
3042     const Value *Callee = CBI->getCalledOperand();
3043     FunctionType *FTy = CBI->getFunctionType();
3044
3045     if (CBI->hasOperandBundles())
3046       writeOperandBundles(*CBI, InstID);
3047
3048     Code = bitc::FUNC_CODE_INST_CALLBR;
3049
3050     Vals.push_back(VE.getAttributeListID(CBI->getAttributes()));
3051
3052     Vals.push_back(CBI->getCallingConv() << bitc::CALL_CCONV |
3053                    1 << bitc::CALL_EXPLICIT_TYPE);
3054
3055     Vals.push_back(VE.getValueID(CBI->getDefaultDest()));
3056     Vals.push_back(CBI->getNumIndirectDests());
3057     for (unsigned i = 0, e = CBI->getNumIndirectDests(); i != e; ++i)
3058       Vals.push_back(VE.getValueID(CBI->getIndirectDest(i)));
3059
3060     Vals.push_back(VE.getTypeID(FTy));
3061     pushValueAndType(Callee, InstID, Vals);
3062
3063     // Emit value #'s for the fixed parameters.
3064     for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3065       pushValue(I.getOperand(i), InstID, Vals); // fixed param.
3066
3067     // Emit type/value pairs for varargs params.
3068     if (FTy->isVarArg()) {
3069       for (unsigned i = FTy->getNumParams(), e = CBI->arg_size(); i != e; ++i)
3070         pushValueAndType(I.getOperand(i), InstID, Vals); // vararg
3071     }
3072     break;
3073   }
3074   case Instruction::Unreachable:
3075     Code = bitc::FUNC_CODE_INST_UNREACHABLE;
3076     AbbrevToUse = FUNCTION_INST_UNREACHABLE_ABBREV;
3077     break;
3078
3079   case Instruction::PHI: {
3080     const PHINode &PN = cast<PHINode>(I);
3081     Code = bitc::FUNC_CODE_INST_PHI;
3082     // With the newer instruction encoding, forward references could give
3083     // negative valued IDs.  This is most common for PHIs, so we use
3084     // signed VBRs.
3085     SmallVector<uint64_t, 128> Vals64;
3086     Vals64.push_back(VE.getTypeID(PN.getType()));
3087     for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) {
3088       pushValueSigned(PN.getIncomingValue(i), InstID, Vals64);
3089       Vals64.push_back(VE.getValueID(PN.getIncomingBlock(i)));
3090     }
3091
3092     uint64_t Flags = getOptimizationFlags(&I);
3093     if (Flags != 0)
3094       Vals64.push_back(Flags);
3095
3096     // Emit a Vals64 vector and exit.
3097     Stream.EmitRecord(Code, Vals64, AbbrevToUse);
3098     Vals64.clear();
3099     return;
3100   }
3101
3102   case Instruction::LandingPad: {
3103     const LandingPadInst &LP = cast<LandingPadInst>(I);
3104     Code = bitc::FUNC_CODE_INST_LANDINGPAD;
3105     Vals.push_back(VE.getTypeID(LP.getType()));
3106     Vals.push_back(LP.isCleanup());
3107     Vals.push_back(LP.getNumClauses());
3108     for (unsigned I = 0, E = LP.getNumClauses(); I != E; ++I) {
3109       if (LP.isCatch(I))
3110         Vals.push_back(LandingPadInst::Catch);
3111       else
3112         Vals.push_back(LandingPadInst::Filter);
3113       pushValueAndType(LP.getClause(I), InstID, Vals);
3114     }
3115     break;
3116   }
3117
3118   case Instruction::Alloca: {
3119     Code = bitc::FUNC_CODE_INST_ALLOCA;
3120     const AllocaInst &AI = cast<AllocaInst>(I);
3121     Vals.push_back(VE.getTypeID(AI.getAllocatedType()));
3122     Vals.push_back(VE.getTypeID(I.getOperand(0)->getType()));
3123     Vals.push_back(VE.getValueID(I.getOperand(0))); // size.
3124     using APV = AllocaPackedValues;
3125     unsigned Record = 0;
3126     unsigned EncodedAlign = getEncodedAlign(AI.getAlign());
3127     Bitfield::set<APV::AlignLower>(
3128         Record, EncodedAlign & ((1 << APV::AlignLower::Bits) - 1));
3129     Bitfield::set<APV::AlignUpper>(Record,
3130                                    EncodedAlign >> APV::AlignLower::Bits);
3131     Bitfield::set<APV::UsedWithInAlloca>(Record, AI.isUsedWithInAlloca());
3132     Bitfield::set<APV::ExplicitType>(Record, true);
3133     Bitfield::set<APV::SwiftError>(Record, AI.isSwiftError());
3134     Vals.push_back(Record);
3135
3136     unsigned AS = AI.getAddressSpace();
3137     if (AS != M.getDataLayout().getAllocaAddrSpace())
3138       Vals.push_back(AS);
3139     break;
3140   }
3141
3142   case Instruction::Load:
3143     if (cast<LoadInst>(I).isAtomic()) {
3144       Code = bitc::FUNC_CODE_INST_LOADATOMIC;
3145       pushValueAndType(I.getOperand(0), InstID, Vals);
3146     } else {
3147       Code = bitc::FUNC_CODE_INST_LOAD;
3148       if (!pushValueAndType(I.getOperand(0), InstID, Vals)) // ptr
3149         AbbrevToUse = FUNCTION_INST_LOAD_ABBREV;
3150     }
3151     Vals.push_back(VE.getTypeID(I.getType()));
3152     Vals.push_back(getEncodedAlign(cast<LoadInst>(I).getAlign()));
3153     Vals.push_back(cast<LoadInst>(I).isVolatile());
3154     if (cast<LoadInst>(I).isAtomic()) {
3155       Vals.push_back(getEncodedOrdering(cast<LoadInst>(I).getOrdering()));
3156       Vals.push_back(getEncodedSyncScopeID(cast<LoadInst>(I).getSyncScopeID()));
3157     }
3158     break;
3159   case Instruction::Store:
3160     if (cast<StoreInst>(I).isAtomic())
3161       Code = bitc::FUNC_CODE_INST_STOREATOMIC;
3162     else
3163       Code = bitc::FUNC_CODE_INST_STORE;
3164     pushValueAndType(I.getOperand(1), InstID, Vals); // ptrty + ptr
3165     pushValueAndType(I.getOperand(0), InstID, Vals); // valty + val
3166     Vals.push_back(getEncodedAlign(cast<StoreInst>(I).getAlign()));
3167     Vals.push_back(cast<StoreInst>(I).isVolatile());
3168     if (cast<StoreInst>(I).isAtomic()) {
3169       Vals.push_back(getEncodedOrdering(cast<StoreInst>(I).getOrdering()));
3170       Vals.push_back(
3171           getEncodedSyncScopeID(cast<StoreInst>(I).getSyncScopeID()));
3172     }
3173     break;
3174   case Instruction::AtomicCmpXchg:
3175     Code = bitc::FUNC_CODE_INST_CMPXCHG;
3176     pushValueAndType(I.getOperand(0), InstID, Vals); // ptrty + ptr
3177     pushValueAndType(I.getOperand(1), InstID, Vals); // cmp.
3178     pushValue(I.getOperand(2), InstID, Vals);        // newval.
3179     Vals.push_back(cast<AtomicCmpXchgInst>(I).isVolatile());
3180     Vals.push_back(
3181         getEncodedOrdering(cast<AtomicCmpXchgInst>(I).getSuccessOrdering()));
3182     Vals.push_back(
3183         getEncodedSyncScopeID(cast<AtomicCmpXchgInst>(I).getSyncScopeID()));
3184     Vals.push_back(
3185         getEncodedOrdering(cast<AtomicCmpXchgInst>(I).getFailureOrdering()));
3186     Vals.push_back(cast<AtomicCmpXchgInst>(I).isWeak());
3187     Vals.push_back(getEncodedAlign(cast<AtomicCmpXchgInst>(I).getAlign()));
3188     break;
3189   case Instruction::AtomicRMW:
3190     Code = bitc::FUNC_CODE_INST_ATOMICRMW;
3191     pushValueAndType(I.getOperand(0), InstID, Vals); // ptrty + ptr
3192     pushValueAndType(I.getOperand(1), InstID, Vals); // valty + val
3193     Vals.push_back(
3194         getEncodedRMWOperation(cast<AtomicRMWInst>(I).getOperation()));
3195     Vals.push_back(cast<AtomicRMWInst>(I).isVolatile());
3196     Vals.push_back(getEncodedOrdering(cast<AtomicRMWInst>(I).getOrdering()));
3197     Vals.push_back(
3198         getEncodedSyncScopeID(cast<AtomicRMWInst>(I).getSyncScopeID()));
3199     Vals.push_back(getEncodedAlign(cast<AtomicRMWInst>(I).getAlign()));
3200     break;
3201   case Instruction::Fence:
3202     Code = bitc::FUNC_CODE_INST_FENCE;
3203     Vals.push_back(getEncodedOrdering(cast<FenceInst>(I).getOrdering()));
3204     Vals.push_back(getEncodedSyncScopeID(cast<FenceInst>(I).getSyncScopeID()));
3205     break;
3206   case Instruction::Call: {
3207     const CallInst &CI = cast<CallInst>(I);
3208     FunctionType *FTy = CI.getFunctionType();
3209
3210     if (CI.hasOperandBundles())
3211       writeOperandBundles(CI, InstID);
3212
3213     Code = bitc::FUNC_CODE_INST_CALL;
3214
3215     Vals.push_back(VE.getAttributeListID(CI.getAttributes()));
3216
3217     unsigned Flags = getOptimizationFlags(&I);
3218     Vals.push_back(CI.getCallingConv() << bitc::CALL_CCONV |
3219                    unsigned(CI.isTailCall()) << bitc::CALL_TAIL |
3220                    unsigned(CI.isMustTailCall()) << bitc::CALL_MUSTTAIL |
3221                    1 << bitc::CALL_EXPLICIT_TYPE |
3222                    unsigned(CI.isNoTailCall()) << bitc::CALL_NOTAIL |
3223                    unsigned(Flags != 0) << bitc::CALL_FMF);
3224     if (Flags != 0)
3225       Vals.push_back(Flags);
3226
3227     Vals.push_back(VE.getTypeID(FTy));
3228     pushValueAndType(CI.getCalledOperand(), InstID, Vals); // Callee
3229
3230     // Emit value #'s for the fixed parameters.
3231     for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i) {
3232       // Check for labels (can happen with asm labels).
3233       if (FTy->getParamType(i)->isLabelTy())
3234         Vals.push_back(VE.getValueID(CI.getArgOperand(i)));
3235       else
3236         pushValue(CI.getArgOperand(i), InstID, Vals); // fixed param.
3237     }
3238
3239     // Emit type/value pairs for varargs params.
3240     if (FTy->isVarArg()) {
3241       for (unsigned i = FTy->getNumParams(), e = CI.arg_size(); i != e; ++i)
3242         pushValueAndType(CI.getArgOperand(i), InstID, Vals); // varargs
3243     }
3244     break;
3245   }
3246   case Instruction::VAArg:
3247     Code = bitc::FUNC_CODE_INST_VAARG;
3248     Vals.push_back(VE.getTypeID(I.getOperand(0)->getType()));   // valistty
3249     pushValue(I.getOperand(0), InstID, Vals);                   // valist.
3250     Vals.push_back(VE.getTypeID(I.getType())); // restype.
3251     break;
3252   case Instruction::Freeze:
3253     Code = bitc::FUNC_CODE_INST_FREEZE;
3254     pushValueAndType(I.getOperand(0), InstID, Vals);
3255     break;
3256   }
3257
3258   Stream.EmitRecord(Code, Vals, AbbrevToUse);
3259   Vals.clear();
3260 }
3261
3262 /// Write a GlobalValue VST to the module. The purpose of this data structure is
3263 /// to allow clients to efficiently find the function body.
3264 void ModuleBitcodeWriter::writeGlobalValueSymbolTable(
3265   DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) {
3266   // Get the offset of the VST we are writing, and backpatch it into
3267   // the VST forward declaration record.
3268   uint64_t VSTOffset = Stream.GetCurrentBitNo();
3269   // The BitcodeStartBit was the stream offset of the identification block.
3270   VSTOffset -= bitcodeStartBit();
3271   assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned");
3272   // Note that we add 1 here because the offset is relative to one word
3273   // before the start of the identification block, which was historically
3274   // always the start of the regular bitcode header.
3275   Stream.BackpatchWord(VSTOffsetPlaceholder, VSTOffset / 32 + 1);
3276
3277   Stream.EnterSubblock(bitc::VALUE_SYMTAB_BLOCK_ID, 4);
3278
3279   auto Abbv = std::make_shared<BitCodeAbbrev>();
3280   Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY));
3281   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id
3282   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset
3283   unsigned FnEntryAbbrev = Stream.EmitAbbrev(std::move(Abbv));
3284
3285   for (const Function &F : M) {
3286     uint64_t Record[2];
3287
3288     if (F.isDeclaration())
3289       continue;
3290
3291     Record[0] = VE.getValueID(&F);
3292
3293     // Save the word offset of the function (from the start of the
3294     // actual bitcode written to the stream).
3295     uint64_t BitcodeIndex = FunctionToBitcodeIndex[&F] - bitcodeStartBit();
3296     assert((BitcodeIndex & 31) == 0 && "function block not 32-bit aligned");
3297     // Note that we add 1 here because the offset is relative to one word
3298     // before the start of the identification block, which was historically
3299     // always the start of the regular bitcode header.
3300     Record[1] = BitcodeIndex / 32 + 1;
3301
3302     Stream.EmitRecord(bitc::VST_CODE_FNENTRY, Record, FnEntryAbbrev);
3303   }
3304
3305   Stream.ExitBlock();
3306 }
3307
3308 /// Emit names for arguments, instructions and basic blocks in a function.
3309 void ModuleBitcodeWriter::writeFunctionLevelValueSymbolTable(
3310     const ValueSymbolTable &VST) {
3311   if (VST.empty())
3312     return;
3313
3314   Stream.EnterSubblock(bitc::VALUE_SYMTAB_BLOCK_ID, 4);
3315
3316   // FIXME: Set up the abbrev, we know how many values there are!
3317   // FIXME: We know if the type names can use 7-bit ascii.
3318   SmallVector<uint64_t, 64> NameVals;
3319
3320   for (const ValueName &Name : VST) {
3321     // Figure out the encoding to use for the name.
3322     StringEncoding Bits = getStringEncoding(Name.getKey());
3323
3324     unsigned AbbrevToUse = VST_ENTRY_8_ABBREV;
3325     NameVals.push_back(VE.getValueID(Name.getValue()));
3326
3327     // VST_CODE_ENTRY:   [valueid, namechar x N]
3328     // VST_CODE_BBENTRY: [bbid, namechar x N]
3329     unsigned Code;
3330     if (isa<BasicBlock>(Name.getValue())) {
3331       Code = bitc::VST_CODE_BBENTRY;
3332       if (Bits == SE_Char6)
3333         AbbrevToUse = VST_BBENTRY_6_ABBREV;
3334     } else {
3335       Code = bitc::VST_CODE_ENTRY;
3336       if (Bits == SE_Char6)
3337         AbbrevToUse = VST_ENTRY_6_ABBREV;
3338       else if (Bits == SE_Fixed7)
3339         AbbrevToUse = VST_ENTRY_7_ABBREV;
3340     }
3341
3342     for (const auto P : Name.getKey())
3343       NameVals.push_back((unsigned char)P);
3344
3345     // Emit the finished record.
3346     Stream.EmitRecord(Code, NameVals, AbbrevToUse);
3347     NameVals.clear();
3348   }
3349
3350   Stream.ExitBlock();
3351 }
3352
3353 void ModuleBitcodeWriter::writeUseList(UseListOrder &&Order) {
3354   assert(Order.Shuffle.size() >= 2 && "Shuffle too small");
3355   unsigned Code;
3356   if (isa<BasicBlock>(Order.V))
3357     Code = bitc::USELIST_CODE_BB;
3358   else
3359     Code = bitc::USELIST_CODE_DEFAULT;
3360
3361   SmallVector<uint64_t, 64> Record(Order.Shuffle.begin(), Order.Shuffle.end());
3362   Record.push_back(VE.getValueID(Order.V));
3363   Stream.EmitRecord(Code, Record);
3364 }
3365
3366 void ModuleBitcodeWriter::writeUseListBlock(const Function *F) {
3367   assert(VE.shouldPreserveUseListOrder() &&
3368          "Expected to be preserving use-list order");
3369
3370   auto hasMore = [&]() {
3371     return !VE.UseListOrders.empty() && VE.UseListOrders.back().F == F;
3372   };
3373   if (!hasMore())
3374     // Nothing to do.
3375     return;
3376
3377   Stream.EnterSubblock(bitc::USELIST_BLOCK_ID, 3);
3378   while (hasMore()) {
3379     writeUseList(std::move(VE.UseListOrders.back()));
3380     VE.UseListOrders.pop_back();
3381   }
3382   Stream.ExitBlock();
3383 }
3384
3385 /// Emit a function body to the module stream.
3386 void ModuleBitcodeWriter::writeFunction(
3387     const Function &F,
3388     DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) {
3389   // Save the bitcode index of the start of this function block for recording
3390   // in the VST.
3391   FunctionToBitcodeIndex[&F] = Stream.GetCurrentBitNo();
3392
3393   Stream.EnterSubblock(bitc::FUNCTION_BLOCK_ID, 4);
3394   VE.incorporateFunction(F);
3395
3396   SmallVector<unsigned, 64> Vals;
3397
3398   // Emit the number of basic blocks, so the reader can create them ahead of
3399   // time.
3400   Vals.push_back(VE.getBasicBlocks().size());
3401   Stream.EmitRecord(bitc::FUNC_CODE_DECLAREBLOCKS, Vals);
3402   Vals.clear();
3403
3404   // If there are function-local constants, emit them now.
3405   unsigned CstStart, CstEnd;
3406   VE.getFunctionConstantRange(CstStart, CstEnd);
3407   writeConstants(CstStart, CstEnd, false);
3408
3409   // If there is function-local metadata, emit it now.
3410   writeFunctionMetadata(F);
3411
3412   // Keep a running idea of what the instruction ID is.
3413   unsigned InstID = CstEnd;
3414
3415   bool NeedsMetadataAttachment = F.hasMetadata();
3416
3417   DILocation *LastDL = nullptr;
3418   SmallSetVector<Function *, 4> BlockAddressUsers;
3419
3420   // Finally, emit all the instructions, in order.
3421   for (const BasicBlock &BB : F) {
3422     for (const Instruction &I : BB) {
3423       writeInstruction(I, InstID, Vals);
3424
3425       if (!I.getType()->isVoidTy())
3426         ++InstID;
3427
3428       // If the instruction has metadata, write a metadata attachment later.
3429       NeedsMetadataAttachment |= I.hasMetadataOtherThanDebugLoc();
3430
3431       // If the instruction has a debug location, emit it.
3432       DILocation *DL = I.getDebugLoc();
3433       if (!DL)
3434         continue;
3435
3436       if (DL == LastDL) {
3437         // Just repeat the same debug loc as last time.
3438         Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_LOC_AGAIN, Vals);
3439         continue;
3440       }
3441
3442       Vals.push_back(DL->getLine());
3443       Vals.push_back(DL->getColumn());
3444       Vals.push_back(VE.getMetadataOrNullID(DL->getScope()));
3445       Vals.push_back(VE.getMetadataOrNullID(DL->getInlinedAt()));
3446       Vals.push_back(DL->isImplicitCode());
3447       Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_LOC, Vals);
3448       Vals.clear();
3449
3450       LastDL = DL;
3451     }
3452
3453     if (BlockAddress *BA = BlockAddress::lookup(&BB)) {
3454       SmallVector<Value *> Worklist{BA};
3455       SmallPtrSet<Value *, 8> Visited{BA};
3456       while (!Worklist.empty()) {
3457         Value *V = Worklist.pop_back_val();
3458         for (User *U : V->users()) {
3459           if (auto *I = dyn_cast<Instruction>(U)) {
3460             Function *P = I->getFunction();
3461             if (P != &F)
3462               BlockAddressUsers.insert(P);
3463           } else if (isa<Constant>(U) && !isa<GlobalValue>(U) &&
3464                      Visited.insert(U).second)
3465             Worklist.push_back(U);
3466         }
3467       }
3468     }
3469   }
3470
3471   if (!BlockAddressUsers.empty()) {
3472     Vals.resize(BlockAddressUsers.size());
3473     for (auto I : llvm::enumerate(BlockAddressUsers))
3474       Vals[I.index()] = VE.getValueID(I.value());
3475     Stream.EmitRecord(bitc::FUNC_CODE_BLOCKADDR_USERS, Vals);
3476     Vals.clear();
3477   }
3478
3479   // Emit names for all the instructions etc.
3480   if (auto *Symtab = F.getValueSymbolTable())
3481     writeFunctionLevelValueSymbolTable(*Symtab);
3482
3483   if (NeedsMetadataAttachment)
3484     writeFunctionMetadataAttachment(F);
3485   if (VE.shouldPreserveUseListOrder())
3486     writeUseListBlock(&F);
3487   VE.purgeFunction();
3488   Stream.ExitBlock();
3489 }
3490
3491 // Emit blockinfo, which defines the standard abbreviations etc.
3492 void ModuleBitcodeWriter::writeBlockInfo() {
3493   // We only want to emit block info records for blocks that have multiple
3494   // instances: CONSTANTS_BLOCK, FUNCTION_BLOCK and VALUE_SYMTAB_BLOCK.
3495   // Other blocks can define their abbrevs inline.
3496   Stream.EnterBlockInfoBlock();
3497
3498   { // 8-bit fixed-width VST_CODE_ENTRY/VST_CODE_BBENTRY strings.
3499     auto Abbv = std::make_shared<BitCodeAbbrev>();
3500     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3));
3501     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3502     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3503     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
3504     if (Stream.EmitBlockInfoAbbrev(bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3505         VST_ENTRY_8_ABBREV)
3506       llvm_unreachable("Unexpected abbrev ordering!");
3507   }
3508
3509   { // 7-bit fixed width VST_CODE_ENTRY strings.
3510     auto Abbv = std::make_shared<BitCodeAbbrev>();
3511     Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_ENTRY));
3512     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3513     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3514     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
3515     if (Stream.EmitBlockInfoAbbrev(bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3516         VST_ENTRY_7_ABBREV)
3517       llvm_unreachable("Unexpected abbrev ordering!");
3518   }
3519   { // 6-bit char6 VST_CODE_ENTRY strings.
3520     auto Abbv = std::make_shared<BitCodeAbbrev>();
3521     Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_ENTRY));
3522     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3523     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3524     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
3525     if (Stream.EmitBlockInfoAbbrev(bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3526         VST_ENTRY_6_ABBREV)
3527       llvm_unreachable("Unexpected abbrev ordering!");
3528   }
3529   { // 6-bit char6 VST_CODE_BBENTRY strings.
3530     auto Abbv = std::make_shared<BitCodeAbbrev>();
3531     Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_BBENTRY));
3532     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3533     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3534     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
3535     if (Stream.EmitBlockInfoAbbrev(bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3536         VST_BBENTRY_6_ABBREV)
3537       llvm_unreachable("Unexpected abbrev ordering!");
3538   }
3539
3540   { // SETTYPE abbrev for CONSTANTS_BLOCK.
3541     auto Abbv = std::make_shared<BitCodeAbbrev>();
3542     Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_SETTYPE));
3543     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
3544                               VE.computeBitsRequiredForTypeIndicies()));
3545     if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3546         CONSTANTS_SETTYPE_ABBREV)
3547       llvm_unreachable("Unexpected abbrev ordering!");
3548   }
3549
3550   { // INTEGER abbrev for CONSTANTS_BLOCK.
3551     auto Abbv = std::make_shared<BitCodeAbbrev>();
3552     Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_INTEGER));
3553     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3554     if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3555         CONSTANTS_INTEGER_ABBREV)
3556       llvm_unreachable("Unexpected abbrev ordering!");
3557   }
3558
3559   { // CE_CAST abbrev for CONSTANTS_BLOCK.
3560     auto Abbv = std::make_shared<BitCodeAbbrev>();
3561     Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_CE_CAST));
3562     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4));  // cast opc
3563     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,       // typeid
3564                               VE.computeBitsRequiredForTypeIndicies()));
3565     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));    // value id
3566
3567     if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3568         CONSTANTS_CE_CAST_Abbrev)
3569       llvm_unreachable("Unexpected abbrev ordering!");
3570   }
3571   { // NULL abbrev for CONSTANTS_BLOCK.
3572     auto Abbv = std::make_shared<BitCodeAbbrev>();
3573     Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_NULL));
3574     if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3575         CONSTANTS_NULL_Abbrev)
3576       llvm_unreachable("Unexpected abbrev ordering!");
3577   }
3578
3579   // FIXME: This should only use space for first class types!
3580
3581   { // INST_LOAD abbrev for FUNCTION_BLOCK.
3582     auto Abbv = std::make_shared<BitCodeAbbrev>();
3583     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_LOAD));
3584     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Ptr
3585     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,    // dest ty
3586                               VE.computeBitsRequiredForTypeIndicies()));
3587     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // Align
3588     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile
3589     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
3590         FUNCTION_INST_LOAD_ABBREV)
3591       llvm_unreachable("Unexpected abbrev ordering!");
3592   }
3593   { // INST_UNOP abbrev for FUNCTION_BLOCK.
3594     auto Abbv = std::make_shared<BitCodeAbbrev>();
3595     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
3596     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS
3597     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
3598     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
3599         FUNCTION_INST_UNOP_ABBREV)
3600       llvm_unreachable("Unexpected abbrev ordering!");
3601   }
3602   { // INST_UNOP_FLAGS abbrev for FUNCTION_BLOCK.
3603     auto Abbv = std::make_shared<BitCodeAbbrev>();
3604     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
3605     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS
3606     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
3607     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
3608     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
3609         FUNCTION_INST_UNOP_FLAGS_ABBREV)
3610       llvm_unreachable("Unexpected abbrev ordering!");
3611   }
3612   { // INST_BINOP abbrev for FUNCTION_BLOCK.
3613     auto Abbv = std::make_shared<BitCodeAbbrev>();
3614     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
3615     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS
3616     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // RHS
3617     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
3618     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
3619         FUNCTION_INST_BINOP_ABBREV)
3620       llvm_unreachable("Unexpected abbrev ordering!");
3621   }
3622   { // INST_BINOP_FLAGS abbrev for FUNCTION_BLOCK.
3623     auto Abbv = std::make_shared<BitCodeAbbrev>();
3624     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
3625     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS
3626     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // RHS
3627     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
3628     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
3629     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
3630         FUNCTION_INST_BINOP_FLAGS_ABBREV)
3631       llvm_unreachable("Unexpected abbrev ordering!");
3632   }
3633   { // INST_CAST abbrev for FUNCTION_BLOCK.
3634     auto Abbv = std::make_shared<BitCodeAbbrev>();
3635     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
3636     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));    // OpVal
3637     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,       // dest ty
3638                               VE.computeBitsRequiredForTypeIndicies()));
3639     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4));  // opc
3640     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
3641         FUNCTION_INST_CAST_ABBREV)
3642       llvm_unreachable("Unexpected abbrev ordering!");
3643   }
3644
3645   { // INST_RET abbrev for FUNCTION_BLOCK.
3646     auto Abbv = std::make_shared<BitCodeAbbrev>();
3647     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET));
3648     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
3649         FUNCTION_INST_RET_VOID_ABBREV)
3650       llvm_unreachable("Unexpected abbrev ordering!");
3651   }
3652   { // INST_RET abbrev for FUNCTION_BLOCK.
3653     auto Abbv = std::make_shared<BitCodeAbbrev>();
3654     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET));
3655     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // ValID
3656     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
3657         FUNCTION_INST_RET_VAL_ABBREV)
3658       llvm_unreachable("Unexpected abbrev ordering!");
3659   }
3660   { // INST_UNREACHABLE abbrev for FUNCTION_BLOCK.
3661     auto Abbv = std::make_shared<BitCodeAbbrev>();
3662     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNREACHABLE));
3663     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
3664         FUNCTION_INST_UNREACHABLE_ABBREV)
3665       llvm_unreachable("Unexpected abbrev ordering!");
3666   }
3667   {
3668     auto Abbv = std::make_shared<BitCodeAbbrev>();
3669     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_GEP));
3670     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
3671     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // dest ty
3672                               Log2_32_Ceil(VE.getTypes().size() + 1)));
3673     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3674     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
3675     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
3676         FUNCTION_INST_GEP_ABBREV)
3677       llvm_unreachable("Unexpected abbrev ordering!");
3678   }
3679
3680   Stream.ExitBlock();
3681 }
3682
3683 /// Write the module path strings, currently only used when generating
3684 /// a combined index file.
3685 void IndexBitcodeWriter::writeModStrings() {
3686   Stream.EnterSubblock(bitc::MODULE_STRTAB_BLOCK_ID, 3);
3687
3688   // TODO: See which abbrev sizes we actually need to emit
3689
3690   // 8-bit fixed-width MST_ENTRY strings.
3691   auto Abbv = std::make_shared<BitCodeAbbrev>();
3692   Abbv->Add(BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
3693   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3694   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3695   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
3696   unsigned Abbrev8Bit = Stream.EmitAbbrev(std::move(Abbv));
3697
3698   // 7-bit fixed width MST_ENTRY strings.
3699   Abbv = std::make_shared<BitCodeAbbrev>();
3700   Abbv->Add(BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
3701   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3702   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3703   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
3704   unsigned Abbrev7Bit = Stream.EmitAbbrev(std::move(Abbv));
3705
3706   // 6-bit char6 MST_ENTRY strings.
3707   Abbv = std::make_shared<BitCodeAbbrev>();
3708   Abbv->Add(BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
3709   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3710   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3711   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
3712   unsigned Abbrev6Bit = Stream.EmitAbbrev(std::move(Abbv));
3713
3714   // Module Hash, 160 bits SHA1. Optionally, emitted after each MST_CODE_ENTRY.
3715   Abbv = std::make_shared<BitCodeAbbrev>();
3716   Abbv->Add(BitCodeAbbrevOp(bitc::MST_CODE_HASH));
3717   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
3718   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
3719   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
3720   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
3721   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
3722   unsigned AbbrevHash = Stream.EmitAbbrev(std::move(Abbv));
3723
3724   SmallVector<unsigned, 64> Vals;
3725   forEachModule(
3726       [&](const StringMapEntry<std::pair<uint64_t, ModuleHash>> &MPSE) {
3727         StringRef Key = MPSE.getKey();
3728         const auto &Value = MPSE.getValue();
3729         StringEncoding Bits = getStringEncoding(Key);
3730         unsigned AbbrevToUse = Abbrev8Bit;
3731         if (Bits == SE_Char6)
3732           AbbrevToUse = Abbrev6Bit;
3733         else if (Bits == SE_Fixed7)
3734           AbbrevToUse = Abbrev7Bit;
3735
3736         Vals.push_back(Value.first);
3737         Vals.append(Key.begin(), Key.end());
3738
3739         // Emit the finished record.
3740         Stream.EmitRecord(bitc::MST_CODE_ENTRY, Vals, AbbrevToUse);
3741
3742         // Emit an optional hash for the module now
3743         const auto &Hash = Value.second;
3744         if (llvm::any_of(Hash, [](uint32_t H) { return H; })) {
3745           Vals.assign(Hash.begin(), Hash.end());
3746           // Emit the hash record.
3747           Stream.EmitRecord(bitc::MST_CODE_HASH, Vals, AbbrevHash);
3748         }
3749
3750         Vals.clear();
3751       });
3752   Stream.ExitBlock();
3753 }
3754
3755 /// Write the function type metadata related records that need to appear before
3756 /// a function summary entry (whether per-module or combined).
3757 template <typename Fn>
3758 static void writeFunctionTypeMetadataRecords(BitstreamWriter &Stream,
3759                                              FunctionSummary *FS,
3760                                              Fn GetValueID) {
3761   if (!FS->type_tests().empty())
3762     Stream.EmitRecord(bitc::FS_TYPE_TESTS, FS->type_tests());
3763
3764   SmallVector<uint64_t, 64> Record;
3765
3766   auto WriteVFuncIdVec = [&](uint64_t Ty,
3767                              ArrayRef<FunctionSummary::VFuncId> VFs) {
3768     if (VFs.empty())
3769       return;
3770     Record.clear();
3771     for (auto &VF : VFs) {
3772       Record.push_back(VF.GUID);
3773       Record.push_back(VF.Offset);
3774     }
3775     Stream.EmitRecord(Ty, Record);
3776   };
3777
3778   WriteVFuncIdVec(bitc::FS_TYPE_TEST_ASSUME_VCALLS,
3779                   FS->type_test_assume_vcalls());
3780   WriteVFuncIdVec(bitc::FS_TYPE_CHECKED_LOAD_VCALLS,
3781                   FS->type_checked_load_vcalls());
3782
3783   auto WriteConstVCallVec = [&](uint64_t Ty,
3784                                 ArrayRef<FunctionSummary::ConstVCall> VCs) {
3785     for (auto &VC : VCs) {
3786       Record.clear();
3787       Record.push_back(VC.VFunc.GUID);
3788       Record.push_back(VC.VFunc.Offset);
3789       llvm::append_range(Record, VC.Args);
3790       Stream.EmitRecord(Ty, Record);
3791     }
3792   };
3793
3794   WriteConstVCallVec(bitc::FS_TYPE_TEST_ASSUME_CONST_VCALL,
3795                      FS->type_test_assume_const_vcalls());
3796   WriteConstVCallVec(bitc::FS_TYPE_CHECKED_LOAD_CONST_VCALL,
3797                      FS->type_checked_load_const_vcalls());
3798
3799   auto WriteRange = [&](ConstantRange Range) {
3800     Range = Range.sextOrTrunc(FunctionSummary::ParamAccess::RangeWidth);
3801     assert(Range.getLower().getNumWords() == 1);
3802     assert(Range.getUpper().getNumWords() == 1);
3803     emitSignedInt64(Record, *Range.getLower().getRawData());
3804     emitSignedInt64(Record, *Range.getUpper().getRawData());
3805   };
3806
3807   if (!FS->paramAccesses().empty()) {
3808     Record.clear();
3809     for (auto &Arg : FS->paramAccesses()) {
3810       size_t UndoSize = Record.size();
3811       Record.push_back(Arg.ParamNo);
3812       WriteRange(Arg.Use);
3813       Record.push_back(Arg.Calls.size());
3814       for (auto &Call : Arg.Calls) {
3815         Record.push_back(Call.ParamNo);
3816         std::optional<unsigned> ValueID = GetValueID(Call.Callee);
3817         if (!ValueID) {
3818           // If ValueID is unknown we can't drop just this call, we must drop
3819           // entire parameter.
3820           Record.resize(UndoSize);
3821           break;
3822         }
3823         Record.push_back(*ValueID);
3824         WriteRange(Call.Offsets);
3825       }
3826     }
3827     if (!Record.empty())
3828       Stream.EmitRecord(bitc::FS_PARAM_ACCESS, Record);
3829   }
3830 }
3831
3832 /// Collect type IDs from type tests used by function.
3833 static void
3834 getReferencedTypeIds(FunctionSummary *FS,
3835                      std::set<GlobalValue::GUID> &ReferencedTypeIds) {
3836   if (!FS->type_tests().empty())
3837     for (auto &TT : FS->type_tests())
3838       ReferencedTypeIds.insert(TT);
3839
3840   auto GetReferencedTypesFromVFuncIdVec =
3841       [&](ArrayRef<FunctionSummary::VFuncId> VFs) {
3842         for (auto &VF : VFs)
3843           ReferencedTypeIds.insert(VF.GUID);
3844       };
3845
3846   GetReferencedTypesFromVFuncIdVec(FS->type_test_assume_vcalls());
3847   GetReferencedTypesFromVFuncIdVec(FS->type_checked_load_vcalls());
3848
3849   auto GetReferencedTypesFromConstVCallVec =
3850       [&](ArrayRef<FunctionSummary::ConstVCall> VCs) {
3851         for (auto &VC : VCs)
3852           ReferencedTypeIds.insert(VC.VFunc.GUID);
3853       };
3854
3855   GetReferencedTypesFromConstVCallVec(FS->type_test_assume_const_vcalls());
3856   GetReferencedTypesFromConstVCallVec(FS->type_checked_load_const_vcalls());
3857 }
3858
3859 static void writeWholeProgramDevirtResolutionByArg(
3860     SmallVector<uint64_t, 64> &NameVals, const std::vector<uint64_t> &args,
3861     const WholeProgramDevirtResolution::ByArg &ByArg) {
3862   NameVals.push_back(args.size());
3863   llvm::append_range(NameVals, args);
3864
3865   NameVals.push_back(ByArg.TheKind);
3866   NameVals.push_back(ByArg.Info);
3867   NameVals.push_back(ByArg.Byte);
3868   NameVals.push_back(ByArg.Bit);
3869 }
3870
3871 static void writeWholeProgramDevirtResolution(
3872     SmallVector<uint64_t, 64> &NameVals, StringTableBuilder &StrtabBuilder,
3873     uint64_t Id, const WholeProgramDevirtResolution &Wpd) {
3874   NameVals.push_back(Id);
3875
3876   NameVals.push_back(Wpd.TheKind);
3877   NameVals.push_back(StrtabBuilder.add(Wpd.SingleImplName));
3878   NameVals.push_back(Wpd.SingleImplName.size());
3879
3880   NameVals.push_back(Wpd.ResByArg.size());
3881   for (auto &A : Wpd.ResByArg)
3882     writeWholeProgramDevirtResolutionByArg(NameVals, A.first, A.second);
3883 }
3884
3885 static void writeTypeIdSummaryRecord(SmallVector<uint64_t, 64> &NameVals,
3886                                      StringTableBuilder &StrtabBuilder,
3887                                      const std::string &Id,
3888                                      const TypeIdSummary &Summary) {
3889   NameVals.push_back(StrtabBuilder.add(Id));
3890   NameVals.push_back(Id.size());
3891
3892   NameVals.push_back(Summary.TTRes.TheKind);
3893   NameVals.push_back(Summary.TTRes.SizeM1BitWidth);
3894   NameVals.push_back(Summary.TTRes.AlignLog2);
3895   NameVals.push_back(Summary.TTRes.SizeM1);
3896   NameVals.push_back(Summary.TTRes.BitMask);
3897   NameVals.push_back(Summary.TTRes.InlineBits);
3898
3899   for (auto &W : Summary.WPDRes)
3900     writeWholeProgramDevirtResolution(NameVals, StrtabBuilder, W.first,
3901                                       W.second);
3902 }
3903
3904 static void writeTypeIdCompatibleVtableSummaryRecord(
3905     SmallVector<uint64_t, 64> &NameVals, StringTableBuilder &StrtabBuilder,
3906     const std::string &Id, const TypeIdCompatibleVtableInfo &Summary,
3907     ValueEnumerator &VE) {
3908   NameVals.push_back(StrtabBuilder.add(Id));
3909   NameVals.push_back(Id.size());
3910
3911   for (auto &P : Summary) {
3912     NameVals.push_back(P.AddressPointOffset);
3913     NameVals.push_back(VE.getValueID(P.VTableVI.getValue()));
3914   }
3915 }
3916
3917 static void writeFunctionHeapProfileRecords(
3918     BitstreamWriter &Stream, FunctionSummary *FS, unsigned CallsiteAbbrev,
3919     unsigned AllocAbbrev, bool PerModule,
3920     std::function<unsigned(const ValueInfo &VI)> GetValueID,
3921     std::function<unsigned(unsigned)> GetStackIndex) {
3922   SmallVector<uint64_t> Record;
3923
3924   for (auto &CI : FS->callsites()) {
3925     Record.clear();
3926     // Per module callsite clones should always have a single entry of
3927     // value 0.
3928     assert(!PerModule || (CI.Clones.size() == 1 && CI.Clones[0] == 0));
3929     Record.push_back(GetValueID(CI.Callee));
3930     if (!PerModule) {
3931       Record.push_back(CI.StackIdIndices.size());
3932       Record.push_back(CI.Clones.size());
3933     }
3934     for (auto Id : CI.StackIdIndices)
3935       Record.push_back(GetStackIndex(Id));
3936     if (!PerModule) {
3937       for (auto V : CI.Clones)
3938         Record.push_back(V);
3939     }
3940     Stream.EmitRecord(PerModule ? bitc::FS_PERMODULE_CALLSITE_INFO
3941                                 : bitc::FS_COMBINED_CALLSITE_INFO,
3942                       Record, CallsiteAbbrev);
3943   }
3944
3945   for (auto &AI : FS->allocs()) {
3946     Record.clear();
3947     // Per module alloc versions should always have a single entry of
3948     // value 0.
3949     assert(!PerModule || (AI.Versions.size() == 1 && AI.Versions[0] == 0));
3950     if (!PerModule) {
3951       Record.push_back(AI.MIBs.size());
3952       Record.push_back(AI.Versions.size());
3953     }
3954     for (auto &MIB : AI.MIBs) {
3955       Record.push_back((uint8_t)MIB.AllocType);
3956       Record.push_back(MIB.StackIdIndices.size());
3957       for (auto Id : MIB.StackIdIndices)
3958         Record.push_back(GetStackIndex(Id));
3959     }
3960     if (!PerModule) {
3961       for (auto V : AI.Versions)
3962         Record.push_back(V);
3963     }
3964     Stream.EmitRecord(PerModule ? bitc::FS_PERMODULE_ALLOC_INFO
3965                                 : bitc::FS_COMBINED_ALLOC_INFO,
3966                       Record, AllocAbbrev);
3967   }
3968 }
3969
3970 // Helper to emit a single function summary record.
3971 void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord(
3972     SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
3973     unsigned ValueID, unsigned FSCallsAbbrev, unsigned FSCallsProfileAbbrev,
3974     unsigned CallsiteAbbrev, unsigned AllocAbbrev, const Function &F) {
3975   NameVals.push_back(ValueID);
3976
3977   FunctionSummary *FS = cast<FunctionSummary>(Summary);
3978
3979   writeFunctionTypeMetadataRecords(
3980       Stream, FS, [&](const ValueInfo &VI) -> std::optional<unsigned> {
3981         return {VE.getValueID(VI.getValue())};
3982       });
3983
3984   writeFunctionHeapProfileRecords(
3985       Stream, FS, CallsiteAbbrev, AllocAbbrev,
3986       /*PerModule*/ true,
3987       /*GetValueId*/ [&](const ValueInfo &VI) { return getValueId(VI); },
3988       /*GetStackIndex*/ [&](unsigned I) { return I; });
3989
3990   auto SpecialRefCnts = FS->specialRefCounts();
3991   NameVals.push_back(getEncodedGVSummaryFlags(FS->flags()));
3992   NameVals.push_back(FS->instCount());
3993   NameVals.push_back(getEncodedFFlags(FS->fflags()));
3994   NameVals.push_back(FS->refs().size());
3995   NameVals.push_back(SpecialRefCnts.first);  // rorefcnt
3996   NameVals.push_back(SpecialRefCnts.second); // worefcnt
3997
3998   for (auto &RI : FS->refs())
3999     NameVals.push_back(VE.getValueID(RI.getValue()));
4000
4001   bool HasProfileData =
4002       F.hasProfileData() || ForceSummaryEdgesCold != FunctionSummary::FSHT_None;
4003   for (auto &ECI : FS->calls()) {
4004     NameVals.push_back(getValueId(ECI.first));
4005     if (HasProfileData)
4006       NameVals.push_back(static_cast<uint8_t>(ECI.second.Hotness));
4007     else if (WriteRelBFToSummary)
4008       NameVals.push_back(ECI.second.RelBlockFreq);
4009   }
4010
4011   unsigned FSAbbrev = (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev);
4012   unsigned Code =
4013       (HasProfileData ? bitc::FS_PERMODULE_PROFILE
4014                       : (WriteRelBFToSummary ? bitc::FS_PERMODULE_RELBF
4015                                              : bitc::FS_PERMODULE));
4016
4017   // Emit the finished record.
4018   Stream.EmitRecord(Code, NameVals, FSAbbrev);
4019   NameVals.clear();
4020 }
4021
4022 // Collect the global value references in the given variable's initializer,
4023 // and emit them in a summary record.
4024 void ModuleBitcodeWriterBase::writeModuleLevelReferences(
4025     const GlobalVariable &V, SmallVector<uint64_t, 64> &NameVals,
4026     unsigned FSModRefsAbbrev, unsigned FSModVTableRefsAbbrev) {
4027   auto VI = Index->getValueInfo(V.getGUID());
4028   if (!VI || VI.getSummaryList().empty()) {
4029     // Only declarations should not have a summary (a declaration might however
4030     // have a summary if the def was in module level asm).
4031     assert(V.isDeclaration());
4032     return;
4033   }
4034   auto *Summary = VI.getSummaryList()[0].get();
4035   NameVals.push_back(VE.getValueID(&V));
4036   GlobalVarSummary *VS = cast<GlobalVarSummary>(Summary);
4037   NameVals.push_back(getEncodedGVSummaryFlags(VS->flags()));
4038   NameVals.push_back(getEncodedGVarFlags(VS->varflags()));
4039
4040   auto VTableFuncs = VS->vTableFuncs();
4041   if (!VTableFuncs.empty())
4042     NameVals.push_back(VS->refs().size());
4043
4044   unsigned SizeBeforeRefs = NameVals.size();
4045   for (auto &RI : VS->refs())
4046     NameVals.push_back(VE.getValueID(RI.getValue()));
4047   // Sort the refs for determinism output, the vector returned by FS->refs() has
4048   // been initialized from a DenseSet.
4049   llvm::sort(drop_begin(NameVals, SizeBeforeRefs));
4050
4051   if (VTableFuncs.empty())
4052     Stream.EmitRecord(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS, NameVals,
4053                       FSModRefsAbbrev);
4054   else {
4055     // VTableFuncs pairs should already be sorted by offset.
4056     for (auto &P : VTableFuncs) {
4057       NameVals.push_back(VE.getValueID(P.FuncVI.getValue()));
4058       NameVals.push_back(P.VTableOffset);
4059     }
4060
4061     Stream.EmitRecord(bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS, NameVals,
4062                       FSModVTableRefsAbbrev);
4063   }
4064   NameVals.clear();
4065 }
4066
4067 /// Emit the per-module summary section alongside the rest of
4068 /// the module's bitcode.
4069 void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() {
4070   // By default we compile with ThinLTO if the module has a summary, but the
4071   // client can request full LTO with a module flag.
4072   bool IsThinLTO = true;
4073   if (auto *MD =
4074           mdconst::extract_or_null<ConstantInt>(M.getModuleFlag("ThinLTO")))
4075     IsThinLTO = MD->getZExtValue();
4076   Stream.EnterSubblock(IsThinLTO ? bitc::GLOBALVAL_SUMMARY_BLOCK_ID
4077                                  : bitc::FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID,
4078                        4);
4079
4080   Stream.EmitRecord(
4081       bitc::FS_VERSION,
4082       ArrayRef<uint64_t>{ModuleSummaryIndex::BitcodeSummaryVersion});
4083
4084   // Write the index flags.
4085   uint64_t Flags = 0;
4086   // Bits 1-3 are set only in the combined index, skip them.
4087   if (Index->enableSplitLTOUnit())
4088     Flags |= 0x8;
4089   Stream.EmitRecord(bitc::FS_FLAGS, ArrayRef<uint64_t>{Flags});
4090
4091   if (Index->begin() == Index->end()) {
4092     Stream.ExitBlock();
4093     return;
4094   }
4095
4096   for (const auto &GVI : valueIds()) {
4097     Stream.EmitRecord(bitc::FS_VALUE_GUID,
4098                       ArrayRef<uint64_t>{GVI.second, GVI.first});
4099   }
4100
4101   if (!Index->stackIds().empty()) {
4102     auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
4103     StackIdAbbv->Add(BitCodeAbbrevOp(bitc::FS_STACK_IDS));
4104     // numids x stackid
4105     StackIdAbbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4106     StackIdAbbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4107     unsigned StackIdAbbvId = Stream.EmitAbbrev(std::move(StackIdAbbv));
4108     Stream.EmitRecord(bitc::FS_STACK_IDS, Index->stackIds(), StackIdAbbvId);
4109   }
4110
4111   // Abbrev for FS_PERMODULE_PROFILE.
4112   auto Abbv = std::make_shared<BitCodeAbbrev>();
4113   Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE));
4114   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // valueid
4115   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // flags
4116   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // instcount
4117   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // fflags
4118   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // numrefs
4119   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // rorefcnt
4120   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // worefcnt
4121   // numrefs x valueid, n x (valueid, hotness)
4122   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4123   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4124   unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4125
4126   // Abbrev for FS_PERMODULE or FS_PERMODULE_RELBF.
4127   Abbv = std::make_shared<BitCodeAbbrev>();
4128   if (WriteRelBFToSummary)
4129     Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_RELBF));
4130   else
4131     Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE));
4132   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // valueid
4133   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // flags
4134   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // instcount
4135   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // fflags
4136   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // numrefs
4137   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // rorefcnt
4138   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // worefcnt
4139   // numrefs x valueid, n x (valueid [, rel_block_freq])
4140   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4141   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4142   unsigned FSCallsAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4143
4144   // Abbrev for FS_PERMODULE_GLOBALVAR_INIT_REFS.
4145   Abbv = std::make_shared<BitCodeAbbrev>();
4146   Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS));
4147   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4148   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4149   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));  // valueids
4150   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4151   unsigned FSModRefsAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4152
4153   // Abbrev for FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS.
4154   Abbv = std::make_shared<BitCodeAbbrev>();
4155   Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS));
4156   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4157   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4158   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4159   // numrefs x valueid, n x (valueid , offset)
4160   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4161   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4162   unsigned FSModVTableRefsAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4163
4164   // Abbrev for FS_ALIAS.
4165   Abbv = std::make_shared<BitCodeAbbrev>();
4166   Abbv->Add(BitCodeAbbrevOp(bitc::FS_ALIAS));
4167   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // valueid
4168   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // flags
4169   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // valueid
4170   unsigned FSAliasAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4171
4172   // Abbrev for FS_TYPE_ID_METADATA
4173   Abbv = std::make_shared<BitCodeAbbrev>();
4174   Abbv->Add(BitCodeAbbrevOp(bitc::FS_TYPE_ID_METADATA));
4175   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // typeid strtab index
4176   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // typeid length
4177   // n x (valueid , offset)
4178   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4179   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4180   unsigned TypeIdCompatibleVtableAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4181
4182   Abbv = std::make_shared<BitCodeAbbrev>();
4183   Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_CALLSITE_INFO));
4184   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4185   // n x stackidindex
4186   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4187   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4188   unsigned CallsiteAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4189
4190   Abbv = std::make_shared<BitCodeAbbrev>();
4191   Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_ALLOC_INFO));
4192   // n x (alloc type, numstackids, numstackids x stackidindex)
4193   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4194   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4195   unsigned AllocAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4196
4197   SmallVector<uint64_t, 64> NameVals;
4198   // Iterate over the list of functions instead of the Index to
4199   // ensure the ordering is stable.
4200   for (const Function &F : M) {
4201     // Summary emission does not support anonymous functions, they have to
4202     // renamed using the anonymous function renaming pass.
4203     if (!F.hasName())
4204       report_fatal_error("Unexpected anonymous function when writing summary");
4205
4206     ValueInfo VI = Index->getValueInfo(F.getGUID());
4207     if (!VI || VI.getSummaryList().empty()) {
4208       // Only declarations should not have a summary (a declaration might
4209       // however have a summary if the def was in module level asm).
4210       assert(F.isDeclaration());
4211       continue;
4212     }
4213     auto *Summary = VI.getSummaryList()[0].get();
4214     writePerModuleFunctionSummaryRecord(NameVals, Summary, VE.getValueID(&F),
4215                                         FSCallsAbbrev, FSCallsProfileAbbrev,
4216                                         CallsiteAbbrev, AllocAbbrev, F);
4217   }
4218
4219   // Capture references from GlobalVariable initializers, which are outside
4220   // of a function scope.
4221   for (const GlobalVariable &G : M.globals())
4222     writeModuleLevelReferences(G, NameVals, FSModRefsAbbrev,
4223                                FSModVTableRefsAbbrev);
4224
4225   for (const GlobalAlias &A : M.aliases()) {
4226     auto *Aliasee = A.getAliaseeObject();
4227     // Skip ifunc and nameless functions which don't have an entry in the
4228     // summary.
4229     if (!Aliasee->hasName() || isa<GlobalIFunc>(Aliasee))
4230       continue;
4231     auto AliasId = VE.getValueID(&A);
4232     auto AliaseeId = VE.getValueID(Aliasee);
4233     NameVals.push_back(AliasId);
4234     auto *Summary = Index->getGlobalValueSummary(A);
4235     AliasSummary *AS = cast<AliasSummary>(Summary);
4236     NameVals.push_back(getEncodedGVSummaryFlags(AS->flags()));
4237     NameVals.push_back(AliaseeId);
4238     Stream.EmitRecord(bitc::FS_ALIAS, NameVals, FSAliasAbbrev);
4239     NameVals.clear();
4240   }
4241
4242   for (auto &S : Index->typeIdCompatibleVtableMap()) {
4243     writeTypeIdCompatibleVtableSummaryRecord(NameVals, StrtabBuilder, S.first,
4244                                              S.second, VE);
4245     Stream.EmitRecord(bitc::FS_TYPE_ID_METADATA, NameVals,
4246                       TypeIdCompatibleVtableAbbrev);
4247     NameVals.clear();
4248   }
4249
4250   if (Index->getBlockCount())
4251     Stream.EmitRecord(bitc::FS_BLOCK_COUNT,
4252                       ArrayRef<uint64_t>{Index->getBlockCount()});
4253
4254   Stream.ExitBlock();
4255 }
4256
4257 /// Emit the combined summary section into the combined index file.
4258 void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
4259   Stream.EnterSubblock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID, 4);
4260   Stream.EmitRecord(
4261       bitc::FS_VERSION,
4262       ArrayRef<uint64_t>{ModuleSummaryIndex::BitcodeSummaryVersion});
4263
4264   // Write the index flags.
4265   Stream.EmitRecord(bitc::FS_FLAGS, ArrayRef<uint64_t>{Index.getFlags()});
4266
4267   for (const auto &GVI : valueIds()) {
4268     Stream.EmitRecord(bitc::FS_VALUE_GUID,
4269                       ArrayRef<uint64_t>{GVI.second, GVI.first});
4270   }
4271
4272   if (!StackIdIndices.empty()) {
4273     auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
4274     StackIdAbbv->Add(BitCodeAbbrevOp(bitc::FS_STACK_IDS));
4275     // numids x stackid
4276     StackIdAbbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4277     StackIdAbbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4278     unsigned StackIdAbbvId = Stream.EmitAbbrev(std::move(StackIdAbbv));
4279     // Write the stack ids used by this index, which will be a subset of those in
4280     // the full index in the case of distributed indexes.
4281     std::vector<uint64_t> StackIds;
4282     for (auto &I : StackIdIndices)
4283       StackIds.push_back(Index.getStackIdAtIndex(I));
4284     Stream.EmitRecord(bitc::FS_STACK_IDS, StackIds, StackIdAbbvId);
4285   }
4286
4287   // Abbrev for FS_COMBINED.
4288   auto Abbv = std::make_shared<BitCodeAbbrev>();
4289   Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED));
4290   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // valueid
4291   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // modid
4292   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // flags
4293   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // instcount
4294   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // fflags
4295   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // entrycount
4296   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // numrefs
4297   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // rorefcnt
4298   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // worefcnt
4299   // numrefs x valueid, n x (valueid)
4300   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4301   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4302   unsigned FSCallsAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4303
4304   // Abbrev for FS_COMBINED_PROFILE.
4305   Abbv = std::make_shared<BitCodeAbbrev>();
4306   Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_PROFILE));
4307   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // valueid
4308   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // modid
4309   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // flags
4310   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // instcount
4311   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // fflags
4312   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // entrycount
4313   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // numrefs
4314   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // rorefcnt
4315   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // worefcnt
4316   // numrefs x valueid, n x (valueid, hotness)
4317   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4318   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4319   unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4320
4321   // Abbrev for FS_COMBINED_GLOBALVAR_INIT_REFS.
4322   Abbv = std::make_shared<BitCodeAbbrev>();
4323   Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS));
4324   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // valueid
4325   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // modid
4326   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // flags
4327   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));    // valueids
4328   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4329   unsigned FSModRefsAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4330
4331   // Abbrev for FS_COMBINED_ALIAS.
4332   Abbv = std::make_shared<BitCodeAbbrev>();
4333   Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_ALIAS));
4334   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // valueid
4335   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // modid
4336   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // flags
4337   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // valueid
4338   unsigned FSAliasAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4339
4340   Abbv = std::make_shared<BitCodeAbbrev>();
4341   Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_CALLSITE_INFO));
4342   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4343   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numstackindices
4344   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver
4345   // numstackindices x stackidindex, numver x version
4346   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4347   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4348   unsigned CallsiteAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4349
4350   Abbv = std::make_shared<BitCodeAbbrev>();
4351   Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_ALLOC_INFO));
4352   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // nummib
4353   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver
4354   // nummib x (alloc type, numstackids, numstackids x stackidindex),
4355   // numver x version
4356   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4357   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4358   unsigned AllocAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4359
4360   // The aliases are emitted as a post-pass, and will point to the value
4361   // id of the aliasee. Save them in a vector for post-processing.
4362   SmallVector<AliasSummary *, 64> Aliases;
4363
4364   // Save the value id for each summary for alias emission.
4365   DenseMap<const GlobalValueSummary *, unsigned> SummaryToValueIdMap;
4366
4367   SmallVector<uint64_t, 64> NameVals;
4368
4369   // Set that will be populated during call to writeFunctionTypeMetadataRecords
4370   // with the type ids referenced by this index file.
4371   std::set<GlobalValue::GUID> ReferencedTypeIds;
4372
4373   // For local linkage, we also emit the original name separately
4374   // immediately after the record.
4375   auto MaybeEmitOriginalName = [&](GlobalValueSummary &S) {
4376     // We don't need to emit the original name if we are writing the index for
4377     // distributed backends (in which case ModuleToSummariesForIndex is
4378     // non-null). The original name is only needed during the thin link, since
4379     // for SamplePGO the indirect call targets for local functions have
4380     // have the original name annotated in profile.
4381     // Continue to emit it when writing out the entire combined index, which is
4382     // used in testing the thin link via llvm-lto.
4383     if (ModuleToSummariesForIndex || !GlobalValue::isLocalLinkage(S.linkage()))
4384       return;
4385     NameVals.push_back(S.getOriginalName());
4386     Stream.EmitRecord(bitc::FS_COMBINED_ORIGINAL_NAME, NameVals);
4387     NameVals.clear();
4388   };
4389
4390   std::set<GlobalValue::GUID> DefOrUseGUIDs;
4391   forEachSummary([&](GVInfo I, bool IsAliasee) {
4392     GlobalValueSummary *S = I.second;
4393     assert(S);
4394     DefOrUseGUIDs.insert(I.first);
4395     for (const ValueInfo &VI : S->refs())
4396       DefOrUseGUIDs.insert(VI.getGUID());
4397
4398     auto ValueId = getValueId(I.first);
4399     assert(ValueId);
4400     SummaryToValueIdMap[S] = *ValueId;
4401
4402     // If this is invoked for an aliasee, we want to record the above
4403     // mapping, but then not emit a summary entry (if the aliasee is
4404     // to be imported, we will invoke this separately with IsAliasee=false).
4405     if (IsAliasee)
4406       return;
4407
4408     if (auto *AS = dyn_cast<AliasSummary>(S)) {
4409       // Will process aliases as a post-pass because the reader wants all
4410       // global to be loaded first.
4411       Aliases.push_back(AS);
4412       return;
4413     }
4414
4415     if (auto *VS = dyn_cast<GlobalVarSummary>(S)) {
4416       NameVals.push_back(*ValueId);
4417       NameVals.push_back(Index.getModuleId(VS->modulePath()));
4418       NameVals.push_back(getEncodedGVSummaryFlags(VS->flags()));
4419       NameVals.push_back(getEncodedGVarFlags(VS->varflags()));
4420       for (auto &RI : VS->refs()) {
4421         auto RefValueId = getValueId(RI.getGUID());
4422         if (!RefValueId)
4423           continue;
4424         NameVals.push_back(*RefValueId);
4425       }
4426
4427       // Emit the finished record.
4428       Stream.EmitRecord(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS, NameVals,
4429                         FSModRefsAbbrev);
4430       NameVals.clear();
4431       MaybeEmitOriginalName(*S);
4432       return;
4433     }
4434
4435     auto GetValueId = [&](const ValueInfo &VI) -> std::optional<unsigned> {
4436       if (!VI)
4437         return std::nullopt;
4438       return getValueId(VI.getGUID());
4439     };
4440
4441     auto *FS = cast<FunctionSummary>(S);
4442     writeFunctionTypeMetadataRecords(Stream, FS, GetValueId);
4443     getReferencedTypeIds(FS, ReferencedTypeIds);
4444
4445     writeFunctionHeapProfileRecords(
4446         Stream, FS, CallsiteAbbrev, AllocAbbrev,
4447         /*PerModule*/ false,
4448         /*GetValueId*/ [&](const ValueInfo &VI) -> unsigned {
4449           std::optional<unsigned> ValueID = GetValueId(VI);
4450           // This can happen in shared index files for distributed ThinLTO if
4451           // the callee function summary is not included. Record 0 which we
4452           // will have to deal with conservatively when doing any kind of
4453           // validation in the ThinLTO backends.
4454           if (!ValueID)
4455             return 0;
4456           return *ValueID;
4457         },
4458         /*GetStackIndex*/ [&](unsigned I) {
4459           // Get the corresponding index into the list of StackIdIndices
4460           // actually being written for this combined index (which may be a
4461           // subset in the case of distributed indexes).
4462           auto Lower = llvm::lower_bound(StackIdIndices, I);
4463           return std::distance(StackIdIndices.begin(), Lower);
4464         });
4465
4466     NameVals.push_back(*ValueId);
4467     NameVals.push_back(Index.getModuleId(FS->modulePath()));
4468     NameVals.push_back(getEncodedGVSummaryFlags(FS->flags()));
4469     NameVals.push_back(FS->instCount());
4470     NameVals.push_back(getEncodedFFlags(FS->fflags()));
4471     NameVals.push_back(FS->entryCount());
4472
4473     // Fill in below
4474     NameVals.push_back(0); // numrefs
4475     NameVals.push_back(0); // rorefcnt
4476     NameVals.push_back(0); // worefcnt
4477
4478     unsigned Count = 0, RORefCnt = 0, WORefCnt = 0;
4479     for (auto &RI : FS->refs()) {
4480       auto RefValueId = getValueId(RI.getGUID());
4481       if (!RefValueId)
4482         continue;
4483       NameVals.push_back(*RefValueId);
4484       if (RI.isReadOnly())
4485         RORefCnt++;
4486       else if (RI.isWriteOnly())
4487         WORefCnt++;
4488       Count++;
4489     }
4490     NameVals[6] = Count;
4491     NameVals[7] = RORefCnt;
4492     NameVals[8] = WORefCnt;
4493
4494     bool HasProfileData = false;
4495     for (auto &EI : FS->calls()) {
4496       HasProfileData |=
4497           EI.second.getHotness() != CalleeInfo::HotnessType::Unknown;
4498       if (HasProfileData)
4499         break;
4500     }
4501
4502     for (auto &EI : FS->calls()) {
4503       // If this GUID doesn't have a value id, it doesn't have a function
4504       // summary and we don't need to record any calls to it.
4505       std::optional<unsigned> CallValueId = GetValueId(EI.first);
4506       if (!CallValueId)
4507         continue;
4508       NameVals.push_back(*CallValueId);
4509       if (HasProfileData)
4510         NameVals.push_back(static_cast<uint8_t>(EI.second.Hotness));
4511     }
4512
4513     unsigned FSAbbrev = (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev);
4514     unsigned Code =
4515         (HasProfileData ? bitc::FS_COMBINED_PROFILE : bitc::FS_COMBINED);
4516
4517     // Emit the finished record.
4518     Stream.EmitRecord(Code, NameVals, FSAbbrev);
4519     NameVals.clear();
4520     MaybeEmitOriginalName(*S);
4521   });
4522
4523   for (auto *AS : Aliases) {
4524     auto AliasValueId = SummaryToValueIdMap[AS];
4525     assert(AliasValueId);
4526     NameVals.push_back(AliasValueId);
4527     NameVals.push_back(Index.getModuleId(AS->modulePath()));
4528     NameVals.push_back(getEncodedGVSummaryFlags(AS->flags()));
4529     auto AliaseeValueId = SummaryToValueIdMap[&AS->getAliasee()];
4530     assert(AliaseeValueId);
4531     NameVals.push_back(AliaseeValueId);
4532
4533     // Emit the finished record.
4534     Stream.EmitRecord(bitc::FS_COMBINED_ALIAS, NameVals, FSAliasAbbrev);
4535     NameVals.clear();
4536     MaybeEmitOriginalName(*AS);
4537
4538     if (auto *FS = dyn_cast<FunctionSummary>(&AS->getAliasee()))
4539       getReferencedTypeIds(FS, ReferencedTypeIds);
4540   }
4541
4542   if (!Index.cfiFunctionDefs().empty()) {
4543     for (auto &S : Index.cfiFunctionDefs()) {
4544       if (DefOrUseGUIDs.count(
4545               GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(S)))) {
4546         NameVals.push_back(StrtabBuilder.add(S));
4547         NameVals.push_back(S.size());
4548       }
4549     }
4550     if (!NameVals.empty()) {
4551       Stream.EmitRecord(bitc::FS_CFI_FUNCTION_DEFS, NameVals);
4552       NameVals.clear();
4553     }
4554   }
4555
4556   if (!Index.cfiFunctionDecls().empty()) {
4557     for (auto &S : Index.cfiFunctionDecls()) {
4558       if (DefOrUseGUIDs.count(
4559               GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(S)))) {
4560         NameVals.push_back(StrtabBuilder.add(S));
4561         NameVals.push_back(S.size());
4562       }
4563     }
4564     if (!NameVals.empty()) {
4565       Stream.EmitRecord(bitc::FS_CFI_FUNCTION_DECLS, NameVals);
4566       NameVals.clear();
4567     }
4568   }
4569
4570   // Walk the GUIDs that were referenced, and write the
4571   // corresponding type id records.
4572   for (auto &T : ReferencedTypeIds) {
4573     auto TidIter = Index.typeIds().equal_range(T);
4574     for (auto It = TidIter.first; It != TidIter.second; ++It) {
4575       writeTypeIdSummaryRecord(NameVals, StrtabBuilder, It->second.first,
4576                                It->second.second);
4577       Stream.EmitRecord(bitc::FS_TYPE_ID, NameVals);
4578       NameVals.clear();
4579     }
4580   }
4581
4582   if (Index.getBlockCount())
4583     Stream.EmitRecord(bitc::FS_BLOCK_COUNT,
4584                       ArrayRef<uint64_t>{Index.getBlockCount()});
4585
4586   Stream.ExitBlock();
4587 }
4588
4589 /// Create the "IDENTIFICATION_BLOCK_ID" containing a single string with the
4590 /// current llvm version, and a record for the epoch number.
4591 static void writeIdentificationBlock(BitstreamWriter &Stream) {
4592   Stream.EnterSubblock(bitc::IDENTIFICATION_BLOCK_ID, 5);
4593
4594   // Write the "user readable" string identifying the bitcode producer
4595   auto Abbv = std::make_shared<BitCodeAbbrev>();
4596   Abbv->Add(BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_STRING));
4597   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4598   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
4599   auto StringAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4600   writeStringRecord(Stream, bitc::IDENTIFICATION_CODE_STRING,
4601                     "LLVM" LLVM_VERSION_STRING, StringAbbrev);
4602
4603   // Write the epoch version
4604   Abbv = std::make_shared<BitCodeAbbrev>();
4605   Abbv->Add(BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_EPOCH));
4606   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4607   auto EpochAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4608   constexpr std::array<unsigned, 1> Vals = {{bitc::BITCODE_CURRENT_EPOCH}};
4609   Stream.EmitRecord(bitc::IDENTIFICATION_CODE_EPOCH, Vals, EpochAbbrev);
4610   Stream.ExitBlock();
4611 }
4612
4613 void ModuleBitcodeWriter::writeModuleHash(size_t BlockStartPos) {
4614   // Emit the module's hash.
4615   // MODULE_CODE_HASH: [5*i32]
4616   if (GenerateHash) {
4617     uint32_t Vals[5];
4618     Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&(Buffer)[BlockStartPos],
4619                                     Buffer.size() - BlockStartPos));
4620     std::array<uint8_t, 20> Hash = Hasher.result();
4621     for (int Pos = 0; Pos < 20; Pos += 4) {
4622       Vals[Pos / 4] = support::endian::read32be(Hash.data() + Pos);
4623     }
4624
4625     // Emit the finished record.
4626     Stream.EmitRecord(bitc::MODULE_CODE_HASH, Vals);
4627
4628     if (ModHash)
4629       // Save the written hash value.
4630       llvm::copy(Vals, std::begin(*ModHash));
4631   }
4632 }
4633
4634 void ModuleBitcodeWriter::write() {
4635   writeIdentificationBlock(Stream);
4636
4637   Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3);
4638   size_t BlockStartPos = Buffer.size();
4639
4640   writeModuleVersion();
4641
4642   // Emit blockinfo, which defines the standard abbreviations etc.
4643   writeBlockInfo();
4644
4645   // Emit information describing all of the types in the module.
4646   writeTypeTable();
4647
4648   // Emit information about attribute groups.
4649   writeAttributeGroupTable();
4650
4651   // Emit information about parameter attributes.
4652   writeAttributeTable();
4653
4654   writeComdats();
4655
4656   // Emit top-level description of module, including target triple, inline asm,
4657   // descriptors for global variables, and function prototype info.
4658   writeModuleInfo();
4659
4660   // Emit constants.
4661   writeModuleConstants();
4662
4663   // Emit metadata kind names.
4664   writeModuleMetadataKinds();
4665
4666   // Emit metadata.
4667   writeModuleMetadata();
4668
4669   // Emit module-level use-lists.
4670   if (VE.shouldPreserveUseListOrder())
4671     writeUseListBlock(nullptr);
4672
4673   writeOperandBundleTags();
4674   writeSyncScopeNames();
4675
4676   // Emit function bodies.
4677   DenseMap<const Function *, uint64_t> FunctionToBitcodeIndex;
4678   for (const Function &F : M)
4679     if (!F.isDeclaration())
4680       writeFunction(F, FunctionToBitcodeIndex);
4681
4682   // Need to write after the above call to WriteFunction which populates
4683   // the summary information in the index.
4684   if (Index)
4685     writePerModuleGlobalValueSummary();
4686
4687   writeGlobalValueSymbolTable(FunctionToBitcodeIndex);
4688
4689   writeModuleHash(BlockStartPos);
4690
4691   Stream.ExitBlock();
4692 }
4693
4694 static void writeInt32ToBuffer(uint32_t Value, SmallVectorImpl<char> &Buffer,
4695                                uint32_t &Position) {
4696   support::endian::write32le(&Buffer[Position], Value);
4697   Position += 4;
4698 }
4699
4700 /// If generating a bc file on darwin, we have to emit a
4701 /// header and trailer to make it compatible with the system archiver.  To do
4702 /// this we emit the following header, and then emit a trailer that pads the
4703 /// file out to be a multiple of 16 bytes.
4704 ///
4705 /// struct bc_header {
4706 ///   uint32_t Magic;         // 0x0B17C0DE
4707 ///   uint32_t Version;       // Version, currently always 0.
4708 ///   uint32_t BitcodeOffset; // Offset to traditional bitcode file.
4709 ///   uint32_t BitcodeSize;   // Size of traditional bitcode file.
4710 ///   uint32_t CPUType;       // CPU specifier.
4711 ///   ... potentially more later ...
4712 /// };
4713 static void emitDarwinBCHeaderAndTrailer(SmallVectorImpl<char> &Buffer,
4714                                          const Triple &TT) {
4715   unsigned CPUType = ~0U;
4716
4717   // Match x86_64-*, i[3-9]86-*, powerpc-*, powerpc64-*, arm-*, thumb-*,
4718   // armv[0-9]-*, thumbv[0-9]-*, armv5te-*, or armv6t2-*. The CPUType is a magic
4719   // number from /usr/include/mach/machine.h.  It is ok to reproduce the
4720   // specific constants here because they are implicitly part of the Darwin ABI.
4721   enum {
4722     DARWIN_CPU_ARCH_ABI64      = 0x01000000,
4723     DARWIN_CPU_TYPE_X86        = 7,
4724     DARWIN_CPU_TYPE_ARM        = 12,
4725     DARWIN_CPU_TYPE_POWERPC    = 18
4726   };
4727
4728   Triple::ArchType Arch = TT.getArch();
4729   if (Arch == Triple::x86_64)
4730     CPUType = DARWIN_CPU_TYPE_X86 | DARWIN_CPU_ARCH_ABI64;
4731   else if (Arch == Triple::x86)
4732     CPUType = DARWIN_CPU_TYPE_X86;
4733   else if (Arch == Triple::ppc)
4734     CPUType = DARWIN_CPU_TYPE_POWERPC;
4735   else if (Arch == Triple::ppc64)
4736     CPUType = DARWIN_CPU_TYPE_POWERPC | DARWIN_CPU_ARCH_ABI64;
4737   else if (Arch == Triple::arm || Arch == Triple::thumb)
4738     CPUType = DARWIN_CPU_TYPE_ARM;
4739
4740   // Traditional Bitcode starts after header.
4741   assert(Buffer.size() >= BWH_HeaderSize &&
4742          "Expected header size to be reserved");
4743   unsigned BCOffset = BWH_HeaderSize;
4744   unsigned BCSize = Buffer.size() - BWH_HeaderSize;
4745
4746   // Write the magic and version.
4747   unsigned Position = 0;
4748   writeInt32ToBuffer(0x0B17C0DE, Buffer, Position);
4749   writeInt32ToBuffer(0, Buffer, Position); // Version.
4750   writeInt32ToBuffer(BCOffset, Buffer, Position);
4751   writeInt32ToBuffer(BCSize, Buffer, Position);
4752   writeInt32ToBuffer(CPUType, Buffer, Position);
4753
4754   // If the file is not a multiple of 16 bytes, insert dummy padding.
4755   while (Buffer.size() & 15)
4756     Buffer.push_back(0);
4757 }
4758
4759 /// Helper to write the header common to all bitcode files.
4760 static void writeBitcodeHeader(BitstreamWriter &Stream) {
4761   // Emit the file header.
4762   Stream.Emit((unsigned)'B', 8);
4763   Stream.Emit((unsigned)'C', 8);
4764   Stream.Emit(0x0, 4);
4765   Stream.Emit(0xC, 4);
4766   Stream.Emit(0xE, 4);
4767   Stream.Emit(0xD, 4);
4768 }
4769
4770 BitcodeWriter::BitcodeWriter(SmallVectorImpl<char> &Buffer, raw_fd_stream *FS)
4771     : Buffer(Buffer), Stream(new BitstreamWriter(Buffer, FS, FlushThreshold)) {
4772   writeBitcodeHeader(*Stream);
4773 }
4774
4775 BitcodeWriter::~BitcodeWriter() { assert(WroteStrtab); }
4776
4777 void BitcodeWriter::writeBlob(unsigned Block, unsigned Record, StringRef Blob) {
4778   Stream->EnterSubblock(Block, 3);
4779
4780   auto Abbv = std::make_shared<BitCodeAbbrev>();
4781   Abbv->Add(BitCodeAbbrevOp(Record));
4782   Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
4783   auto AbbrevNo = Stream->EmitAbbrev(std::move(Abbv));
4784
4785   Stream->EmitRecordWithBlob(AbbrevNo, ArrayRef<uint64_t>{Record}, Blob);
4786
4787   Stream->ExitBlock();
4788 }
4789
4790 void BitcodeWriter::writeSymtab() {
4791   assert(!WroteStrtab && !WroteSymtab);
4792
4793   // If any module has module-level inline asm, we will require a registered asm
4794   // parser for the target so that we can create an accurate symbol table for
4795   // the module.
4796   for (Module *M : Mods) {
4797     if (M->getModuleInlineAsm().empty())
4798       continue;
4799
4800     std::string Err;
4801     const Triple TT(M->getTargetTriple());
4802     const Target *T = TargetRegistry::lookupTarget(TT.str(), Err);
4803     if (!T || !T->hasMCAsmParser())
4804       return;
4805   }
4806
4807   WroteSymtab = true;
4808   SmallVector<char, 0> Symtab;
4809   // The irsymtab::build function may be unable to create a symbol table if the
4810   // module is malformed (e.g. it contains an invalid alias). Writing a symbol
4811   // table is not required for correctness, but we still want to be able to
4812   // write malformed modules to bitcode files, so swallow the error.
4813   if (Error E = irsymtab::build(Mods, Symtab, StrtabBuilder, Alloc)) {
4814     consumeError(std::move(E));
4815     return;
4816   }
4817
4818   writeBlob(bitc::SYMTAB_BLOCK_ID, bitc::SYMTAB_BLOB,
4819             {Symtab.data(), Symtab.size()});
4820 }
4821
4822 void BitcodeWriter::writeStrtab() {
4823   assert(!WroteStrtab);
4824
4825   std::vector<char> Strtab;
4826   StrtabBuilder.finalizeInOrder();
4827   Strtab.resize(StrtabBuilder.getSize());
4828   StrtabBuilder.write((uint8_t *)Strtab.data());
4829
4830   writeBlob(bitc::STRTAB_BLOCK_ID, bitc::STRTAB_BLOB,
4831             {Strtab.data(), Strtab.size()});
4832
4833   WroteStrtab = true;
4834 }
4835
4836 void BitcodeWriter::copyStrtab(StringRef Strtab) {
4837   writeBlob(bitc::STRTAB_BLOCK_ID, bitc::STRTAB_BLOB, Strtab);
4838   WroteStrtab = true;
4839 }
4840
4841 void BitcodeWriter::writeModule(const Module &M,
4842                                 bool ShouldPreserveUseListOrder,
4843                                 const ModuleSummaryIndex *Index,
4844                                 bool GenerateHash, ModuleHash *ModHash) {
4845   assert(!WroteStrtab);
4846
4847   // The Mods vector is used by irsymtab::build, which requires non-const
4848   // Modules in case it needs to materialize metadata. But the bitcode writer
4849   // requires that the module is materialized, so we can cast to non-const here,
4850   // after checking that it is in fact materialized.
4851   assert(M.isMaterialized());
4852   Mods.push_back(const_cast<Module *>(&M));
4853
4854   ModuleBitcodeWriter ModuleWriter(M, Buffer, StrtabBuilder, *Stream,
4855                                    ShouldPreserveUseListOrder, Index,
4856                                    GenerateHash, ModHash);
4857   ModuleWriter.write();
4858 }
4859
4860 void BitcodeWriter::writeIndex(
4861     const ModuleSummaryIndex *Index,
4862     const std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex) {
4863   IndexBitcodeWriter IndexWriter(*Stream, StrtabBuilder, *Index,
4864                                  ModuleToSummariesForIndex);
4865   IndexWriter.write();
4866 }
4867
4868 /// Write the specified module to the specified output stream.
4869 void llvm::WriteBitcodeToFile(const Module &M, raw_ostream &Out,
4870                               bool ShouldPreserveUseListOrder,
4871                               const ModuleSummaryIndex *Index,
4872                               bool GenerateHash, ModuleHash *ModHash) {
4873   SmallVector<char, 0> Buffer;
4874   Buffer.reserve(256*1024);
4875
4876   // If this is darwin or another generic macho target, reserve space for the
4877   // header.
4878   Triple TT(M.getTargetTriple());
4879   if (TT.isOSDarwin() || TT.isOSBinFormatMachO())
4880     Buffer.insert(Buffer.begin(), BWH_HeaderSize, 0);
4881
4882   BitcodeWriter Writer(Buffer, dyn_cast<raw_fd_stream>(&Out));
4883   Writer.writeModule(M, ShouldPreserveUseListOrder, Index, GenerateHash,
4884                      ModHash);
4885   Writer.writeSymtab();
4886   Writer.writeStrtab();
4887
4888   if (TT.isOSDarwin() || TT.isOSBinFormatMachO())
4889     emitDarwinBCHeaderAndTrailer(Buffer, TT);
4890
4891   // Write the generated bitstream to "Out".
4892   if (!Buffer.empty())
4893     Out.write((char *)&Buffer.front(), Buffer.size());
4894 }
4895
4896 void IndexBitcodeWriter::write() {
4897   Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3);
4898
4899   writeModuleVersion();
4900
4901   // Write the module paths in the combined index.
4902   writeModStrings();
4903
4904   // Write the summary combined index records.
4905   writeCombinedGlobalValueSummary();
4906
4907   Stream.ExitBlock();
4908 }
4909
4910 // Write the specified module summary index to the given raw output stream,
4911 // where it will be written in a new bitcode block. This is used when
4912 // writing the combined index file for ThinLTO. When writing a subset of the
4913 // index for a distributed backend, provide a \p ModuleToSummariesForIndex map.
4914 void llvm::writeIndexToFile(
4915     const ModuleSummaryIndex &Index, raw_ostream &Out,
4916     const std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex) {
4917   SmallVector<char, 0> Buffer;
4918   Buffer.reserve(256 * 1024);
4919
4920   BitcodeWriter Writer(Buffer);
4921   Writer.writeIndex(&Index, ModuleToSummariesForIndex);
4922   Writer.writeStrtab();
4923
4924   Out.write((char *)&Buffer.front(), Buffer.size());
4925 }
4926
4927 namespace {
4928
4929 /// Class to manage the bitcode writing for a thin link bitcode file.
4930 class ThinLinkBitcodeWriter : public ModuleBitcodeWriterBase {
4931   /// ModHash is for use in ThinLTO incremental build, generated while writing
4932   /// the module bitcode file.
4933   const ModuleHash *ModHash;
4934
4935 public:
4936   ThinLinkBitcodeWriter(const Module &M, StringTableBuilder &StrtabBuilder,
4937                         BitstreamWriter &Stream,
4938                         const ModuleSummaryIndex &Index,
4939                         const ModuleHash &ModHash)
4940       : ModuleBitcodeWriterBase(M, StrtabBuilder, Stream,
4941                                 /*ShouldPreserveUseListOrder=*/false, &Index),
4942         ModHash(&ModHash) {}
4943
4944   void write();
4945
4946 private:
4947   void writeSimplifiedModuleInfo();
4948 };
4949
4950 } // end anonymous namespace
4951
4952 // This function writes a simpilified module info for thin link bitcode file.
4953 // It only contains the source file name along with the name(the offset and
4954 // size in strtab) and linkage for global values. For the global value info
4955 // entry, in order to keep linkage at offset 5, there are three zeros used
4956 // as padding.
4957 void ThinLinkBitcodeWriter::writeSimplifiedModuleInfo() {
4958   SmallVector<unsigned, 64> Vals;
4959   // Emit the module's source file name.
4960   {
4961     StringEncoding Bits = getStringEncoding(M.getSourceFileName());
4962     BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8);
4963     if (Bits == SE_Char6)
4964       AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6);
4965     else if (Bits == SE_Fixed7)
4966       AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7);
4967
4968     // MODULE_CODE_SOURCE_FILENAME: [namechar x N]
4969     auto Abbv = std::make_shared<BitCodeAbbrev>();
4970     Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME));
4971     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4972     Abbv->Add(AbbrevOpToUse);
4973     unsigned FilenameAbbrev = Stream.EmitAbbrev(std::move(Abbv));
4974
4975     for (const auto P : M.getSourceFileName())
4976       Vals.push_back((unsigned char)P);
4977
4978     Stream.EmitRecord(bitc::MODULE_CODE_SOURCE_FILENAME, Vals, FilenameAbbrev);
4979     Vals.clear();
4980   }
4981
4982   // Emit the global variable information.
4983   for (const GlobalVariable &GV : M.globals()) {
4984     // GLOBALVAR: [strtab offset, strtab size, 0, 0, 0, linkage]
4985     Vals.push_back(StrtabBuilder.add(GV.getName()));
4986     Vals.push_back(GV.getName().size());
4987     Vals.push_back(0);
4988     Vals.push_back(0);
4989     Vals.push_back(0);
4990     Vals.push_back(getEncodedLinkage(GV));
4991
4992     Stream.EmitRecord(bitc::MODULE_CODE_GLOBALVAR, Vals);
4993     Vals.clear();
4994   }
4995
4996   // Emit the function proto information.
4997   for (const Function &F : M) {
4998     // FUNCTION:  [strtab offset, strtab size, 0, 0, 0, linkage]
4999     Vals.push_back(StrtabBuilder.add(F.getName()));
5000     Vals.push_back(F.getName().size());
5001     Vals.push_back(0);
5002     Vals.push_back(0);
5003     Vals.push_back(0);
5004     Vals.push_back(getEncodedLinkage(F));
5005
5006     Stream.EmitRecord(bitc::MODULE_CODE_FUNCTION, Vals);
5007     Vals.clear();
5008   }
5009
5010   // Emit the alias information.
5011   for (const GlobalAlias &A : M.aliases()) {
5012     // ALIAS: [strtab offset, strtab size, 0, 0, 0, linkage]
5013     Vals.push_back(StrtabBuilder.add(A.getName()));
5014     Vals.push_back(A.getName().size());
5015     Vals.push_back(0);
5016     Vals.push_back(0);
5017     Vals.push_back(0);
5018     Vals.push_back(getEncodedLinkage(A));
5019
5020     Stream.EmitRecord(bitc::MODULE_CODE_ALIAS, Vals);
5021     Vals.clear();
5022   }
5023
5024   // Emit the ifunc information.
5025   for (const GlobalIFunc &I : M.ifuncs()) {
5026     // IFUNC: [strtab offset, strtab size, 0, 0, 0, linkage]
5027     Vals.push_back(StrtabBuilder.add(I.getName()));
5028     Vals.push_back(I.getName().size());
5029     Vals.push_back(0);
5030     Vals.push_back(0);
5031     Vals.push_back(0);
5032     Vals.push_back(getEncodedLinkage(I));
5033
5034     Stream.EmitRecord(bitc::MODULE_CODE_IFUNC, Vals);
5035     Vals.clear();
5036   }
5037 }
5038
5039 void ThinLinkBitcodeWriter::write() {
5040   Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3);
5041
5042   writeModuleVersion();
5043
5044   writeSimplifiedModuleInfo();
5045
5046   writePerModuleGlobalValueSummary();
5047
5048   // Write module hash.
5049   Stream.EmitRecord(bitc::MODULE_CODE_HASH, ArrayRef<uint32_t>(*ModHash));
5050
5051   Stream.ExitBlock();
5052 }
5053
5054 void BitcodeWriter::writeThinLinkBitcode(const Module &M,
5055                                          const ModuleSummaryIndex &Index,
5056                                          const ModuleHash &ModHash) {
5057   assert(!WroteStrtab);
5058
5059   // The Mods vector is used by irsymtab::build, which requires non-const
5060   // Modules in case it needs to materialize metadata. But the bitcode writer
5061   // requires that the module is materialized, so we can cast to non-const here,
5062   // after checking that it is in fact materialized.
5063   assert(M.isMaterialized());
5064   Mods.push_back(const_cast<Module *>(&M));
5065
5066   ThinLinkBitcodeWriter ThinLinkWriter(M, StrtabBuilder, *Stream, Index,
5067                                        ModHash);
5068   ThinLinkWriter.write();
5069 }
5070
5071 // Write the specified thin link bitcode file to the given raw output stream,
5072 // where it will be written in a new bitcode block. This is used when
5073 // writing the per-module index file for ThinLTO.
5074 void llvm::writeThinLinkBitcodeToFile(const Module &M, raw_ostream &Out,
5075                                       const ModuleSummaryIndex &Index,
5076                                       const ModuleHash &ModHash) {
5077   SmallVector<char, 0> Buffer;
5078   Buffer.reserve(256 * 1024);
5079
5080   BitcodeWriter Writer(Buffer);
5081   Writer.writeThinLinkBitcode(M, Index, ModHash);
5082   Writer.writeSymtab();
5083   Writer.writeStrtab();
5084
5085   Out.write((char *)&Buffer.front(), Buffer.size());
5086 }
5087
5088 static const char *getSectionNameForBitcode(const Triple &T) {
5089   switch (T.getObjectFormat()) {
5090   case Triple::MachO:
5091     return "__LLVM,__bitcode";
5092   case Triple::COFF:
5093   case Triple::ELF:
5094   case Triple::Wasm:
5095   case Triple::UnknownObjectFormat:
5096     return ".llvmbc";
5097   case Triple::GOFF:
5098     llvm_unreachable("GOFF is not yet implemented");
5099     break;
5100   case Triple::SPIRV:
5101     llvm_unreachable("SPIRV is not yet implemented");
5102     break;
5103   case Triple::XCOFF:
5104     llvm_unreachable("XCOFF is not yet implemented");
5105     break;
5106   case Triple::DXContainer:
5107     llvm_unreachable("DXContainer is not yet implemented");
5108     break;
5109   }
5110   llvm_unreachable("Unimplemented ObjectFormatType");
5111 }
5112
5113 static const char *getSectionNameForCommandline(const Triple &T) {
5114   switch (T.getObjectFormat()) {
5115   case Triple::MachO:
5116     return "__LLVM,__cmdline";
5117   case Triple::COFF:
5118   case Triple::ELF:
5119   case Triple::Wasm:
5120   case Triple::UnknownObjectFormat:
5121     return ".llvmcmd";
5122   case Triple::GOFF:
5123     llvm_unreachable("GOFF is not yet implemented");
5124     break;
5125   case Triple::SPIRV:
5126     llvm_unreachable("SPIRV is not yet implemented");
5127     break;
5128   case Triple::XCOFF:
5129     llvm_unreachable("XCOFF is not yet implemented");
5130     break;
5131   case Triple::DXContainer:
5132     llvm_unreachable("DXC is not yet implemented");
5133     break;
5134   }
5135   llvm_unreachable("Unimplemented ObjectFormatType");
5136 }
5137
5138 void llvm::embedBitcodeInModule(llvm::Module &M, llvm::MemoryBufferRef Buf,
5139                                 bool EmbedBitcode, bool EmbedCmdline,
5140                                 const std::vector<uint8_t> &CmdArgs) {
5141   // Save llvm.compiler.used and remove it.
5142   SmallVector<Constant *, 2> UsedArray;
5143   SmallVector<GlobalValue *, 4> UsedGlobals;
5144   Type *UsedElementType = Type::getInt8Ty(M.getContext())->getPointerTo(0);
5145   GlobalVariable *Used = collectUsedGlobalVariables(M, UsedGlobals, true);
5146   for (auto *GV : UsedGlobals) {
5147     if (GV->getName() != "llvm.embedded.module" &&
5148         GV->getName() != "llvm.cmdline")
5149       UsedArray.push_back(
5150           ConstantExpr::getPointerBitCastOrAddrSpaceCast(GV, UsedElementType));
5151   }
5152   if (Used)
5153     Used->eraseFromParent();
5154
5155   // Embed the bitcode for the llvm module.
5156   std::string Data;
5157   ArrayRef<uint8_t> ModuleData;
5158   Triple T(M.getTargetTriple());
5159
5160   if (EmbedBitcode) {
5161     if (Buf.getBufferSize() == 0 ||
5162         !isBitcode((const unsigned char *)Buf.getBufferStart(),
5163                    (const unsigned char *)Buf.getBufferEnd())) {
5164       // If the input is LLVM Assembly, bitcode is produced by serializing
5165       // the module. Use-lists order need to be preserved in this case.
5166       llvm::raw_string_ostream OS(Data);
5167       llvm::WriteBitcodeToFile(M, OS, /* ShouldPreserveUseListOrder */ true);
5168       ModuleData =
5169           ArrayRef<uint8_t>((const uint8_t *)OS.str().data(), OS.str().size());
5170     } else
5171       // If the input is LLVM bitcode, write the input byte stream directly.
5172       ModuleData = ArrayRef<uint8_t>((const uint8_t *)Buf.getBufferStart(),
5173                                      Buf.getBufferSize());
5174   }
5175   llvm::Constant *ModuleConstant =
5176       llvm::ConstantDataArray::get(M.getContext(), ModuleData);
5177   llvm::GlobalVariable *GV = new llvm::GlobalVariable(
5178       M, ModuleConstant->getType(), true, llvm::GlobalValue::PrivateLinkage,
5179       ModuleConstant);
5180   GV->setSection(getSectionNameForBitcode(T));
5181   // Set alignment to 1 to prevent padding between two contributions from input
5182   // sections after linking.
5183   GV->setAlignment(Align(1));
5184   UsedArray.push_back(
5185       ConstantExpr::getPointerBitCastOrAddrSpaceCast(GV, UsedElementType));
5186   if (llvm::GlobalVariable *Old =
5187           M.getGlobalVariable("llvm.embedded.module", true)) {
5188     assert(Old->hasZeroLiveUses() &&
5189            "llvm.embedded.module can only be used once in llvm.compiler.used");
5190     GV->takeName(Old);
5191     Old->eraseFromParent();
5192   } else {
5193     GV->setName("llvm.embedded.module");
5194   }
5195
5196   // Skip if only bitcode needs to be embedded.
5197   if (EmbedCmdline) {
5198     // Embed command-line options.
5199     ArrayRef<uint8_t> CmdData(const_cast<uint8_t *>(CmdArgs.data()),
5200                               CmdArgs.size());
5201     llvm::Constant *CmdConstant =
5202         llvm::ConstantDataArray::get(M.getContext(), CmdData);
5203     GV = new llvm::GlobalVariable(M, CmdConstant->getType(), true,
5204                                   llvm::GlobalValue::PrivateLinkage,
5205                                   CmdConstant);
5206     GV->setSection(getSectionNameForCommandline(T));
5207     GV->setAlignment(Align(1));
5208     UsedArray.push_back(
5209         ConstantExpr::getPointerBitCastOrAddrSpaceCast(GV, UsedElementType));
5210     if (llvm::GlobalVariable *Old = M.getGlobalVariable("llvm.cmdline", true)) {
5211       assert(Old->hasZeroLiveUses() &&
5212              "llvm.cmdline can only be used once in llvm.compiler.used");
5213       GV->takeName(Old);
5214       Old->eraseFromParent();
5215     } else {
5216       GV->setName("llvm.cmdline");
5217     }
5218   }
5219
5220   if (UsedArray.empty())
5221     return;
5222
5223   // Recreate llvm.compiler.used.
5224   ArrayType *ATy = ArrayType::get(UsedElementType, UsedArray.size());
5225   auto *NewUsed = new GlobalVariable(
5226       M, ATy, false, llvm::GlobalValue::AppendingLinkage,
5227       llvm::ConstantArray::get(ATy, UsedArray), "llvm.compiler.used");
5228   NewUsed->setSection("llvm.metadata");
5229 }