clang-tools-extra/clangd/index/YAMLSerialization.cpp

   1 //===-- YAMLSerialization.cpp ------------------------------------*- C++-*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // A YAML index file is a sequence of tagged entries.
  10 // Each entry either encodes a Symbol or the list of references to a symbol
  11 // (a "ref bundle").
  12 //
  13 //===----------------------------------------------------------------------===//
  14
  15 #include "Headers.h"
  16 #include "index/Ref.h"
  17 #include "index/Relation.h"
  18 #include "index/Serialization.h"
  19 #include "index/Symbol.h"
  20 #include "index/SymbolLocation.h"
  21 #include "index/SymbolOrigin.h"
  22 #include "clang/Tooling/CompilationDatabase.h"
  23 #include "llvm/ADT/StringRef.h"
  24 #include "llvm/Support/Allocator.h"
  25 #include "llvm/Support/StringSaver.h"
  26 #include "llvm/Support/YAMLTraits.h"
  27 #include "llvm/Support/raw_ostream.h"
  28 #include <cstdint>
  29 #include <optional>
  30
  31 namespace {
  32 struct YIncludeHeaderWithReferences;
  33 }
  34
  35 LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Symbol::IncludeHeaderWithReferences)
  36 LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Ref)
  37 LLVM_YAML_IS_SEQUENCE_VECTOR(YIncludeHeaderWithReferences)
  38
  39 namespace {
  40 using RefBundle =
  41     std::pair<clang::clangd::SymbolID, std::vector<clang::clangd::Ref>>;
  42 // This is a pale imitation of std::variant<Symbol, RefBundle, Relation>
  43 struct VariantEntry {
  44   std::optional<clang::clangd::Symbol> Symbol;
  45   std::optional<RefBundle> Refs;
  46   std::optional<clang::clangd::Relation> Relation;
  47   std::optional<clang::clangd::IncludeGraphNode> Source;
  48   std::optional<clang::tooling::CompileCommand> Cmd;
  49 };
  50 // A class helps YAML to serialize the 32-bit encoded position (Line&Column),
  51 // as YAMLIO can't directly map bitfields.
  52 struct YPosition {
  53   uint32_t Line;
  54   uint32_t Column;
  55 };
  56 // A class helps YAML to serialize the IncludeHeaderWithReferences as YAMLIO
  57 // can't directly map bitfields.
  58 struct YIncludeHeaderWithReferences {
  59   llvm::StringRef IncludeHeader;
  60   uint32_t References;
  61   clang::clangd::Symbol::IncludeDirective SupportedDirectives;
  62
  63   YIncludeHeaderWithReferences() = default;
  64
  65   YIncludeHeaderWithReferences(
  66       llvm::StringRef IncludeHeader, uint32_t References,
  67       clang::clangd::Symbol::IncludeDirective SupportedDirectives)
  68       : IncludeHeader(IncludeHeader), References(References),
  69         SupportedDirectives(SupportedDirectives) {}
  70 };
  71
  72 // avoid ODR violation of specialization for non-owned CompileCommand
  73 struct CompileCommandYAML : clang::tooling::CompileCommand {};
  74
  75 } // namespace
  76 namespace llvm {
  77 namespace yaml {
  78
  79 using clang::clangd::FileDigest;
  80 using clang::clangd::IncludeGraph;
  81 using clang::clangd::IncludeGraphNode;
  82 using clang::clangd::Ref;
  83 using clang::clangd::RefKind;
  84 using clang::clangd::Relation;
  85 using clang::clangd::RelationKind;
  86 using clang::clangd::Symbol;
  87 using clang::clangd::SymbolID;
  88 using clang::clangd::SymbolLocation;
  89 using clang::index::SymbolInfo;
  90 using clang::index::SymbolKind;
  91 using clang::index::SymbolLanguage;
  92 using clang::tooling::CompileCommand;
  93
  94 // Helper to (de)serialize the SymbolID. We serialize it as a hex string.
  95 struct NormalizedSymbolID {
  96   NormalizedSymbolID(IO &) {}
  97   NormalizedSymbolID(IO &, const SymbolID &ID) {
  98     llvm::raw_string_ostream OS(HexString);
  99     OS << ID;
 100   }
 101
 102   SymbolID denormalize(IO &I) {
 103     auto ID = SymbolID::fromStr(HexString);
 104     if (!ID) {
 105       I.setError(llvm::toString(ID.takeError()));
 106       return SymbolID();
 107     }
 108     return *ID;
 109   }
 110
 111   std::string HexString;
 112 };
 113
 114 struct NormalizedSymbolFlag {
 115   NormalizedSymbolFlag(IO &) {}
 116   NormalizedSymbolFlag(IO &, Symbol::SymbolFlag F) {
 117     Flag = static_cast<uint8_t>(F);
 118   }
 119
 120   Symbol::SymbolFlag denormalize(IO &) {
 121     return static_cast<Symbol::SymbolFlag>(Flag);
 122   }
 123
 124   uint8_t Flag = 0;
 125 };
 126
 127 template <> struct MappingTraits<YPosition> {
 128   static void mapping(IO &IO, YPosition &Value) {
 129     IO.mapRequired("Line", Value.Line);
 130     IO.mapRequired("Column", Value.Column);
 131   }
 132 };
 133
 134 struct NormalizedPosition {
 135   using Position = clang::clangd::SymbolLocation::Position;
 136   NormalizedPosition(IO &) {}
 137   NormalizedPosition(IO &, const Position &Pos) {
 138     P.Line = Pos.line();
 139     P.Column = Pos.column();
 140   }
 141
 142   Position denormalize(IO &) {
 143     Position Pos;
 144     Pos.setLine(P.Line);
 145     Pos.setColumn(P.Column);
 146     return Pos;
 147   }
 148   YPosition P;
 149 };
 150
 151 struct NormalizedFileURI {
 152   NormalizedFileURI(IO &) {}
 153   NormalizedFileURI(IO &, const char *FileURI) { URI = FileURI; }
 154
 155   const char *denormalize(IO &IO) {
 156     assert(IO.getContext() &&
 157            "Expecting an UniqueStringSaver to allocate data");
 158     return static_cast<llvm::UniqueStringSaver *>(IO.getContext())
 159         ->save(URI)
 160         .data();
 161   }
 162
 163   std::string URI;
 164 };
 165
 166 template <> struct MappingTraits<SymbolLocation> {
 167   static void mapping(IO &IO, SymbolLocation &Value) {
 168     MappingNormalization<NormalizedFileURI, const char *> NFile(IO,
 169                                                                 Value.FileURI);
 170     IO.mapRequired("FileURI", NFile->URI);
 171     MappingNormalization<NormalizedPosition, SymbolLocation::Position> NStart(
 172         IO, Value.Start);
 173     IO.mapRequired("Start", NStart->P);
 174     MappingNormalization<NormalizedPosition, SymbolLocation::Position> NEnd(
 175         IO, Value.End);
 176     IO.mapRequired("End", NEnd->P);
 177   }
 178 };
 179
 180 template <> struct MappingTraits<SymbolInfo> {
 181   static void mapping(IO &IO, SymbolInfo &SymInfo) {
 182     // FIXME: expose other fields?
 183     IO.mapRequired("Kind", SymInfo.Kind);
 184     IO.mapRequired("Lang", SymInfo.Lang);
 185   }
 186 };
 187
 188 template <> struct ScalarBitSetTraits<clang::clangd::Symbol::IncludeDirective> {
 189   static void bitset(IO &IO, clang::clangd::Symbol::IncludeDirective &Value) {
 190     IO.bitSetCase(Value, "Include", clang::clangd::Symbol::Include);
 191     IO.bitSetCase(Value, "Import", clang::clangd::Symbol::Import);
 192   }
 193 };
 194
 195 template <> struct MappingTraits<YIncludeHeaderWithReferences> {
 196   static void mapping(IO &IO, YIncludeHeaderWithReferences &Inc) {
 197     IO.mapRequired("Header", Inc.IncludeHeader);
 198     IO.mapRequired("References", Inc.References);
 199     IO.mapOptional("Directives", Inc.SupportedDirectives,
 200                    clang::clangd::Symbol::Include);
 201   }
 202 };
 203
 204 struct NormalizedIncludeHeaders {
 205   using IncludeHeader = clang::clangd::Symbol::IncludeHeaderWithReferences;
 206   NormalizedIncludeHeaders(IO &) {}
 207   NormalizedIncludeHeaders(
 208       IO &, const llvm::SmallVector<IncludeHeader, 1> &IncludeHeaders) {
 209     for (auto &I : IncludeHeaders) {
 210       Headers.emplace_back(I.IncludeHeader, I.References,
 211                            I.supportedDirectives());
 212     }
 213   }
 214
 215   llvm::SmallVector<IncludeHeader, 1> denormalize(IO &) {
 216     llvm::SmallVector<IncludeHeader, 1> Result;
 217     for (auto &H : Headers)
 218       Result.emplace_back(H.IncludeHeader, H.References, H.SupportedDirectives);
 219     return Result;
 220   }
 221   llvm::SmallVector<YIncludeHeaderWithReferences, 1> Headers;
 222 };
 223
 224 template <> struct MappingTraits<Symbol> {
 225   static void mapping(IO &IO, Symbol &Sym) {
 226     MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, Sym.ID);
 227     MappingNormalization<NormalizedSymbolFlag, Symbol::SymbolFlag> NSymbolFlag(
 228         IO, Sym.Flags);
 229     MappingNormalization<
 230         NormalizedIncludeHeaders,
 231         llvm::SmallVector<Symbol::IncludeHeaderWithReferences, 1>>
 232         NIncludeHeaders(IO, Sym.IncludeHeaders);
 233     IO.mapRequired("ID", NSymbolID->HexString);
 234     IO.mapRequired("Name", Sym.Name);
 235     IO.mapRequired("Scope", Sym.Scope);
 236     IO.mapRequired("SymInfo", Sym.SymInfo);
 237     IO.mapOptional("CanonicalDeclaration", Sym.CanonicalDeclaration,
 238                    SymbolLocation());
 239     IO.mapOptional("Definition", Sym.Definition, SymbolLocation());
 240     IO.mapOptional("References", Sym.References, 0u);
 241     IO.mapOptional("Flags", NSymbolFlag->Flag);
 242     IO.mapOptional("Signature", Sym.Signature);
 243     IO.mapOptional("TemplateSpecializationArgs",
 244                    Sym.TemplateSpecializationArgs);
 245     IO.mapOptional("CompletionSnippetSuffix", Sym.CompletionSnippetSuffix);
 246     IO.mapOptional("Documentation", Sym.Documentation);
 247     IO.mapOptional("ReturnType", Sym.ReturnType);
 248     IO.mapOptional("Type", Sym.Type);
 249     IO.mapOptional("IncludeHeaders", NIncludeHeaders->Headers);
 250   }
 251 };
 252
 253 template <> struct ScalarEnumerationTraits<SymbolLanguage> {
 254   static void enumeration(IO &IO, SymbolLanguage &Value) {
 255     IO.enumCase(Value, "C", SymbolLanguage::C);
 256     IO.enumCase(Value, "Cpp", SymbolLanguage::CXX);
 257     IO.enumCase(Value, "ObjC", SymbolLanguage::ObjC);
 258     IO.enumCase(Value, "Swift", SymbolLanguage::Swift);
 259   }
 260 };
 261
 262 template <> struct ScalarEnumerationTraits<SymbolKind> {
 263   static void enumeration(IO &IO, SymbolKind &Value) {
 264 #define DEFINE_ENUM(name) IO.enumCase(Value, #name, SymbolKind::name)
 265
 266     DEFINE_ENUM(Unknown);
 267     DEFINE_ENUM(Function);
 268     DEFINE_ENUM(Module);
 269     DEFINE_ENUM(Namespace);
 270     DEFINE_ENUM(NamespaceAlias);
 271     DEFINE_ENUM(Macro);
 272     DEFINE_ENUM(Enum);
 273     DEFINE_ENUM(Struct);
 274     DEFINE_ENUM(Class);
 275     DEFINE_ENUM(Protocol);
 276     DEFINE_ENUM(Extension);
 277     DEFINE_ENUM(Union);
 278     DEFINE_ENUM(TypeAlias);
 279     DEFINE_ENUM(Function);
 280     DEFINE_ENUM(Variable);
 281     DEFINE_ENUM(Field);
 282     DEFINE_ENUM(EnumConstant);
 283     DEFINE_ENUM(InstanceMethod);
 284     DEFINE_ENUM(ClassMethod);
 285     DEFINE_ENUM(StaticMethod);
 286     DEFINE_ENUM(InstanceProperty);
 287     DEFINE_ENUM(ClassProperty);
 288     DEFINE_ENUM(StaticProperty);
 289     DEFINE_ENUM(Constructor);
 290     DEFINE_ENUM(Destructor);
 291     DEFINE_ENUM(ConversionFunction);
 292     DEFINE_ENUM(Parameter);
 293     DEFINE_ENUM(Using);
 294
 295 #undef DEFINE_ENUM
 296   }
 297 };
 298
 299 template <> struct MappingTraits<RefBundle> {
 300   static void mapping(IO &IO, RefBundle &Refs) {
 301     MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO,
 302                                                                  Refs.first);
 303     IO.mapRequired("ID", NSymbolID->HexString);
 304     IO.mapRequired("References", Refs.second);
 305   }
 306 };
 307
 308 struct NormalizedRefKind {
 309   NormalizedRefKind(IO &) {}
 310   NormalizedRefKind(IO &, RefKind O) { Kind = static_cast<uint8_t>(O); }
 311
 312   RefKind denormalize(IO &) { return static_cast<RefKind>(Kind); }
 313
 314   uint8_t Kind = 0;
 315 };
 316
 317 template <> struct MappingTraits<Ref> {
 318   static void mapping(IO &IO, Ref &R) {
 319     MappingNormalization<NormalizedRefKind, RefKind> NKind(IO, R.Kind);
 320     IO.mapRequired("Kind", NKind->Kind);
 321     IO.mapRequired("Location", R.Location);
 322   }
 323 };
 324
 325 struct NormalizedSymbolRole {
 326   NormalizedSymbolRole(IO &) {}
 327   NormalizedSymbolRole(IO &IO, RelationKind R) {
 328     Kind = static_cast<uint8_t>(R);
 329   }
 330
 331   RelationKind denormalize(IO &IO) { return static_cast<RelationKind>(Kind); }
 332
 333   uint8_t Kind = 0;
 334 };
 335
 336 template <> struct MappingTraits<SymbolID> {
 337   static void mapping(IO &IO, SymbolID &ID) {
 338     MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, ID);
 339     IO.mapRequired("ID", NSymbolID->HexString);
 340   }
 341 };
 342
 343 template <> struct MappingTraits<Relation> {
 344   static void mapping(IO &IO, Relation &Relation) {
 345     MappingNormalization<NormalizedSymbolRole, RelationKind> NRole(
 346         IO, Relation.Predicate);
 347     IO.mapRequired("Subject", Relation.Subject);
 348     IO.mapRequired("Predicate", NRole->Kind);
 349     IO.mapRequired("Object", Relation.Object);
 350   }
 351 };
 352
 353 struct NormalizedSourceFlag {
 354   NormalizedSourceFlag(IO &) {}
 355   NormalizedSourceFlag(IO &, IncludeGraphNode::SourceFlag O) {
 356     Flag = static_cast<uint8_t>(O);
 357   }
 358
 359   IncludeGraphNode::SourceFlag denormalize(IO &) {
 360     return static_cast<IncludeGraphNode::SourceFlag>(Flag);
 361   }
 362
 363   uint8_t Flag = 0;
 364 };
 365
 366 struct NormalizedFileDigest {
 367   NormalizedFileDigest(IO &) {}
 368   NormalizedFileDigest(IO &, const FileDigest &Digest) {
 369     HexString = llvm::toHex(Digest);
 370   }
 371
 372   FileDigest denormalize(IO &I) {
 373     FileDigest Digest;
 374     if (HexString.size() == Digest.size() * 2 &&
 375         llvm::all_of(HexString, llvm::isHexDigit)) {
 376       memcpy(Digest.data(), llvm::fromHex(HexString).data(), Digest.size());
 377     } else {
 378       I.setError(std::string("Bad hex file digest: ") + HexString);
 379     }
 380     return Digest;
 381   }
 382
 383   std::string HexString;
 384 };
 385
 386 template <> struct MappingTraits<IncludeGraphNode> {
 387   static void mapping(IO &IO, IncludeGraphNode &Node) {
 388     IO.mapRequired("URI", Node.URI);
 389     MappingNormalization<NormalizedSourceFlag, IncludeGraphNode::SourceFlag>
 390         NSourceFlag(IO, Node.Flags);
 391     IO.mapRequired("Flags", NSourceFlag->Flag);
 392     MappingNormalization<NormalizedFileDigest, FileDigest> NDigest(IO,
 393                                                                    Node.Digest);
 394     IO.mapRequired("Digest", NDigest->HexString);
 395     IO.mapRequired("DirectIncludes", Node.DirectIncludes);
 396   }
 397 };
 398
 399 template <> struct MappingTraits<CompileCommandYAML> {
 400   static void mapping(IO &IO, CompileCommandYAML &Cmd) {
 401     IO.mapRequired("Directory", Cmd.Directory);
 402     IO.mapRequired("CommandLine", Cmd.CommandLine);
 403   }
 404 };
 405
 406 template <> struct MappingTraits<VariantEntry> {
 407   static void mapping(IO &IO, VariantEntry &Variant) {
 408     if (IO.mapTag("!Symbol", Variant.Symbol.has_value())) {
 409       if (!IO.outputting())
 410         Variant.Symbol.emplace();
 411       MappingTraits<Symbol>::mapping(IO, *Variant.Symbol);
 412     } else if (IO.mapTag("!Refs", Variant.Refs.has_value())) {
 413       if (!IO.outputting())
 414         Variant.Refs.emplace();
 415       MappingTraits<RefBundle>::mapping(IO, *Variant.Refs);
 416     } else if (IO.mapTag("!Relations", Variant.Relation.has_value())) {
 417       if (!IO.outputting())
 418         Variant.Relation.emplace();
 419       MappingTraits<Relation>::mapping(IO, *Variant.Relation);
 420     } else if (IO.mapTag("!Source", Variant.Source.has_value())) {
 421       if (!IO.outputting())
 422         Variant.Source.emplace();
 423       MappingTraits<IncludeGraphNode>::mapping(IO, *Variant.Source);
 424     } else if (IO.mapTag("!Cmd", Variant.Cmd.has_value())) {
 425       if (!IO.outputting())
 426         Variant.Cmd.emplace();
 427       MappingTraits<CompileCommandYAML>::mapping(
 428           IO, static_cast<CompileCommandYAML &>(*Variant.Cmd));
 429     }
 430   }
 431 };
 432
 433 } // namespace yaml
 434 } // namespace llvm
 435
 436 namespace clang {
 437 namespace clangd {
 438
 439 void writeYAML(const IndexFileOut &O, llvm::raw_ostream &OS) {
 440   llvm::yaml::Output Yout(OS);
 441   for (const auto &Sym : *O.Symbols) {
 442     VariantEntry Entry;
 443     Entry.Symbol = Sym;
 444     Yout << Entry;
 445   }
 446   if (O.Refs)
 447     for (auto &Sym : *O.Refs) {
 448       VariantEntry Entry;
 449       Entry.Refs = Sym;
 450       Yout << Entry;
 451     }
 452   if (O.Relations)
 453     for (auto &R : *O.Relations) {
 454       VariantEntry Entry;
 455       Entry.Relation = R;
 456       Yout << Entry;
 457     }
 458   if (O.Sources) {
 459     for (const auto &Source : *O.Sources) {
 460       VariantEntry Entry;
 461       Entry.Source = Source.getValue();
 462       Yout << Entry;
 463     }
 464   }
 465   if (O.Cmd) {
 466     VariantEntry Entry;
 467     Entry.Cmd = *O.Cmd;
 468     Yout << Entry;
 469   }
 470 }
 471
 472 llvm::Expected<IndexFileIn> readYAML(llvm::StringRef Data,
 473                                      SymbolOrigin Origin) {
 474   SymbolSlab::Builder Symbols;
 475   RefSlab::Builder Refs;
 476   RelationSlab::Builder Relations;
 477   llvm::BumpPtrAllocator
 478       Arena; // store the underlying data of Position::FileURI.
 479   llvm::UniqueStringSaver Strings(Arena);
 480   llvm::yaml::Input Yin(Data, &Strings);
 481   IncludeGraph Sources;
 482   std::optional<tooling::CompileCommand> Cmd;
 483   while (Yin.setCurrentDocument()) {
 484     llvm::yaml::EmptyContext Ctx;
 485     VariantEntry Variant;
 486     yamlize(Yin, Variant, true, Ctx);
 487     if (Yin.error())
 488       return llvm::errorCodeToError(Yin.error());
 489
 490     if (Variant.Symbol) {
 491       Variant.Symbol->Origin = Origin;
 492       Symbols.insert(*Variant.Symbol);
 493     }
 494     if (Variant.Refs)
 495       for (const auto &Ref : Variant.Refs->second)
 496         Refs.insert(Variant.Refs->first, Ref);
 497     if (Variant.Relation)
 498       Relations.insert(*Variant.Relation);
 499     if (Variant.Source) {
 500       auto &IGN = *Variant.Source;
 501       auto Entry = Sources.try_emplace(IGN.URI).first;
 502       Entry->getValue() = std::move(IGN);
 503       // Fixup refs to refer to map keys which will live on
 504       Entry->getValue().URI = Entry->getKey();
 505       for (auto &Include : Entry->getValue().DirectIncludes)
 506         Include = Sources.try_emplace(Include).first->getKey();
 507     }
 508     if (Variant.Cmd)
 509       Cmd = *Variant.Cmd;
 510     Yin.nextDocument();
 511   }
 512
 513   IndexFileIn Result;
 514   Result.Symbols.emplace(std::move(Symbols).build());
 515   Result.Refs.emplace(std::move(Refs).build());
 516   Result.Relations.emplace(std::move(Relations).build());
 517   if (Sources.size())
 518     Result.Sources = std::move(Sources);
 519   Result.Cmd = std::move(Cmd);
 520   return std::move(Result);
 521 }
 522
 523 std::string toYAML(const Symbol &S) {
 524   std::string Buf;
 525   {
 526     llvm::raw_string_ostream OS(Buf);
 527     llvm::yaml::Output Yout(OS);
 528     Symbol Sym = S; // copy: Yout<< requires mutability.
 529     Yout << Sym;
 530   }
 531   return Buf;
 532 }
 533
 534 std::string toYAML(const std::pair<SymbolID, llvm::ArrayRef<Ref>> &Data) {
 535   RefBundle Refs = {Data.first, Data.second};
 536   std::string Buf;
 537   {
 538     llvm::raw_string_ostream OS(Buf);
 539     llvm::yaml::Output Yout(OS);
 540     Yout << Refs;
 541   }
 542   return Buf;
 543 }
 544
 545 std::string toYAML(const Relation &R) {
 546   std::string Buf;
 547   {
 548     llvm::raw_string_ostream OS(Buf);
 549     llvm::yaml::Output Yout(OS);
 550     Relation Rel = R; // copy: Yout<< requires mutability.
 551     Yout << Rel;
 552   }
 553   return Buf;
 554 }
 555
 556 std::string toYAML(const Ref &R) {
 557   std::string Buf;
 558   {
 559     llvm::raw_string_ostream OS(Buf);
 560     llvm::yaml::Output Yout(OS);
 561     Ref Reference = R; // copy: Yout<< requires mutability.
 562     Yout << Reference;
 563   }
 564   return Buf;
 565 }
 566
 567 } // namespace clangd
 568 } // namespace clang