clang-tools-extra/clangd/index/YAMLSerialization.cpp

   1 //===-- YAMLSerialization.cpp ------------------------------------*- C++-*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // A YAML index file is a sequence of tagged entries.
  10 // Each entry either encodes a Symbol or the list of references to a symbol
  11 // (a "ref bundle").
  12 //
  13 //===----------------------------------------------------------------------===//
  14
  15 #include "Headers.h"
  16 #include "index/Ref.h"
  17 #include "index/Relation.h"
  18 #include "index/Serialization.h"
  19 #include "index/Symbol.h"
  20 #include "index/SymbolLocation.h"
  21 #include "index/SymbolOrigin.h"
  22 #include "clang/Tooling/CompilationDatabase.h"
  23 #include "llvm/ADT/Optional.h"
  24 #include "llvm/ADT/StringRef.h"
  25 #include "llvm/Support/Allocator.h"
  26 #include "llvm/Support/StringSaver.h"
  27 #include "llvm/Support/YAMLTraits.h"
  28 #include "llvm/Support/raw_ostream.h"
  29 #include <cstdint>
  30
  31 LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Symbol::IncludeHeaderWithReferences)
  32 LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Ref)
  33
  34 namespace {
  35 using RefBundle =
  36     std::pair<clang::clangd::SymbolID, std::vector<clang::clangd::Ref>>;
  37 // This is a pale imitation of std::variant<Symbol, RefBundle, Relation>
  38 struct VariantEntry {
  39   llvm::Optional<clang::clangd::Symbol> Symbol;
  40   llvm::Optional<RefBundle> Refs;
  41   llvm::Optional<clang::clangd::Relation> Relation;
  42   llvm::Optional<clang::clangd::IncludeGraphNode> Source;
  43   llvm::Optional<clang::tooling::CompileCommand> Cmd;
  44 };
  45 // A class helps YAML to serialize the 32-bit encoded position (Line&Column),
  46 // as YAMLIO can't directly map bitfields.
  47 struct YPosition {
  48   uint32_t Line;
  49   uint32_t Column;
  50 };
  51
  52 // avoid ODR violation of specialization for non-owned CompileCommand
  53 struct CompileCommandYAML : clang::tooling::CompileCommand {};
  54
  55 } // namespace
  56 namespace llvm {
  57 namespace yaml {
  58
  59 using clang::clangd::FileDigest;
  60 using clang::clangd::IncludeGraph;
  61 using clang::clangd::IncludeGraphNode;
  62 using clang::clangd::Ref;
  63 using clang::clangd::RefKind;
  64 using clang::clangd::Relation;
  65 using clang::clangd::RelationKind;
  66 using clang::clangd::Symbol;
  67 using clang::clangd::SymbolID;
  68 using clang::clangd::SymbolLocation;
  69 using clang::index::SymbolInfo;
  70 using clang::index::SymbolKind;
  71 using clang::index::SymbolLanguage;
  72 using clang::tooling::CompileCommand;
  73
  74 // Helper to (de)serialize the SymbolID. We serialize it as a hex string.
  75 struct NormalizedSymbolID {
  76   NormalizedSymbolID(IO &) {}
  77   NormalizedSymbolID(IO &, const SymbolID &ID) {
  78     llvm::raw_string_ostream OS(HexString);
  79     OS << ID;
  80   }
  81
  82   SymbolID denormalize(IO &I) {
  83     auto ID = SymbolID::fromStr(HexString);
  84     if (!ID) {
  85       I.setError(llvm::toString(ID.takeError()));
  86       return SymbolID();
  87     }
  88     return *ID;
  89   }
  90
  91   std::string HexString;
  92 };
  93
  94 struct NormalizedSymbolFlag {
  95   NormalizedSymbolFlag(IO &) {}
  96   NormalizedSymbolFlag(IO &, Symbol::SymbolFlag F) {
  97     Flag = static_cast<uint8_t>(F);
  98   }
  99
 100   Symbol::SymbolFlag denormalize(IO &) {
 101     return static_cast<Symbol::SymbolFlag>(Flag);
 102   }
 103
 104   uint8_t Flag = 0;
 105 };
 106
 107 template <> struct MappingTraits<YPosition> {
 108   static void mapping(IO &IO, YPosition &Value) {
 109     IO.mapRequired("Line", Value.Line);
 110     IO.mapRequired("Column", Value.Column);
 111   }
 112 };
 113
 114 struct NormalizedPosition {
 115   using Position = clang::clangd::SymbolLocation::Position;
 116   NormalizedPosition(IO &) {}
 117   NormalizedPosition(IO &, const Position &Pos) {
 118     P.Line = Pos.line();
 119     P.Column = Pos.column();
 120   }
 121
 122   Position denormalize(IO &) {
 123     Position Pos;
 124     Pos.setLine(P.Line);
 125     Pos.setColumn(P.Column);
 126     return Pos;
 127   }
 128   YPosition P;
 129 };
 130
 131 struct NormalizedFileURI {
 132   NormalizedFileURI(IO &) {}
 133   NormalizedFileURI(IO &, const char *FileURI) { URI = FileURI; }
 134
 135   const char *denormalize(IO &IO) {
 136     assert(IO.getContext() &&
 137            "Expecting an UniqueStringSaver to allocate data");
 138     return static_cast<llvm::UniqueStringSaver *>(IO.getContext())
 139         ->save(URI)
 140         .data();
 141   }
 142
 143   std::string URI;
 144 };
 145
 146 template <> struct MappingTraits<SymbolLocation> {
 147   static void mapping(IO &IO, SymbolLocation &Value) {
 148     MappingNormalization<NormalizedFileURI, const char *> NFile(IO,
 149                                                                 Value.FileURI);
 150     IO.mapRequired("FileURI", NFile->URI);
 151     MappingNormalization<NormalizedPosition, SymbolLocation::Position> NStart(
 152         IO, Value.Start);
 153     IO.mapRequired("Start", NStart->P);
 154     MappingNormalization<NormalizedPosition, SymbolLocation::Position> NEnd(
 155         IO, Value.End);
 156     IO.mapRequired("End", NEnd->P);
 157   }
 158 };
 159
 160 template <> struct MappingTraits<SymbolInfo> {
 161   static void mapping(IO &IO, SymbolInfo &SymInfo) {
 162     // FIXME: expose other fields?
 163     IO.mapRequired("Kind", SymInfo.Kind);
 164     IO.mapRequired("Lang", SymInfo.Lang);
 165   }
 166 };
 167
 168 template <>
 169 struct MappingTraits<clang::clangd::Symbol::IncludeHeaderWithReferences> {
 170   static void mapping(IO &IO,
 171                       clang::clangd::Symbol::IncludeHeaderWithReferences &Inc) {
 172     IO.mapRequired("Header", Inc.IncludeHeader);
 173     IO.mapRequired("References", Inc.References);
 174   }
 175 };
 176
 177 template <> struct MappingTraits<Symbol> {
 178   static void mapping(IO &IO, Symbol &Sym) {
 179     MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, Sym.ID);
 180     MappingNormalization<NormalizedSymbolFlag, Symbol::SymbolFlag> NSymbolFlag(
 181         IO, Sym.Flags);
 182     IO.mapRequired("ID", NSymbolID->HexString);
 183     IO.mapRequired("Name", Sym.Name);
 184     IO.mapRequired("Scope", Sym.Scope);
 185     IO.mapRequired("SymInfo", Sym.SymInfo);
 186     IO.mapOptional("CanonicalDeclaration", Sym.CanonicalDeclaration,
 187                    SymbolLocation());
 188     IO.mapOptional("Definition", Sym.Definition, SymbolLocation());
 189     IO.mapOptional("References", Sym.References, 0u);
 190     IO.mapOptional("Flags", NSymbolFlag->Flag);
 191     IO.mapOptional("Signature", Sym.Signature);
 192     IO.mapOptional("TemplateSpecializationArgs",
 193                    Sym.TemplateSpecializationArgs);
 194     IO.mapOptional("CompletionSnippetSuffix", Sym.CompletionSnippetSuffix);
 195     IO.mapOptional("Documentation", Sym.Documentation);
 196     IO.mapOptional("ReturnType", Sym.ReturnType);
 197     IO.mapOptional("Type", Sym.Type);
 198     IO.mapOptional("IncludeHeaders", Sym.IncludeHeaders);
 199   }
 200 };
 201
 202 template <> struct ScalarEnumerationTraits<SymbolLanguage> {
 203   static void enumeration(IO &IO, SymbolLanguage &Value) {
 204     IO.enumCase(Value, "C", SymbolLanguage::C);
 205     IO.enumCase(Value, "Cpp", SymbolLanguage::CXX);
 206     IO.enumCase(Value, "ObjC", SymbolLanguage::ObjC);
 207     IO.enumCase(Value, "Swift", SymbolLanguage::Swift);
 208   }
 209 };
 210
 211 template <> struct ScalarEnumerationTraits<SymbolKind> {
 212   static void enumeration(IO &IO, SymbolKind &Value) {
 213 #define DEFINE_ENUM(name) IO.enumCase(Value, #name, SymbolKind::name)
 214
 215     DEFINE_ENUM(Unknown);
 216     DEFINE_ENUM(Function);
 217     DEFINE_ENUM(Module);
 218     DEFINE_ENUM(Namespace);
 219     DEFINE_ENUM(NamespaceAlias);
 220     DEFINE_ENUM(Macro);
 221     DEFINE_ENUM(Enum);
 222     DEFINE_ENUM(Struct);
 223     DEFINE_ENUM(Class);
 224     DEFINE_ENUM(Protocol);
 225     DEFINE_ENUM(Extension);
 226     DEFINE_ENUM(Union);
 227     DEFINE_ENUM(TypeAlias);
 228     DEFINE_ENUM(Function);
 229     DEFINE_ENUM(Variable);
 230     DEFINE_ENUM(Field);
 231     DEFINE_ENUM(EnumConstant);
 232     DEFINE_ENUM(InstanceMethod);
 233     DEFINE_ENUM(ClassMethod);
 234     DEFINE_ENUM(StaticMethod);
 235     DEFINE_ENUM(InstanceProperty);
 236     DEFINE_ENUM(ClassProperty);
 237     DEFINE_ENUM(StaticProperty);
 238     DEFINE_ENUM(Constructor);
 239     DEFINE_ENUM(Destructor);
 240     DEFINE_ENUM(ConversionFunction);
 241     DEFINE_ENUM(Parameter);
 242     DEFINE_ENUM(Using);
 243
 244 #undef DEFINE_ENUM
 245   }
 246 };
 247
 248 template <> struct MappingTraits<RefBundle> {
 249   static void mapping(IO &IO, RefBundle &Refs) {
 250     MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO,
 251                                                                  Refs.first);
 252     IO.mapRequired("ID", NSymbolID->HexString);
 253     IO.mapRequired("References", Refs.second);
 254   }
 255 };
 256
 257 struct NormalizedRefKind {
 258   NormalizedRefKind(IO &) {}
 259   NormalizedRefKind(IO &, RefKind O) { Kind = static_cast<uint8_t>(O); }
 260
 261   RefKind denormalize(IO &) { return static_cast<RefKind>(Kind); }
 262
 263   uint8_t Kind = 0;
 264 };
 265
 266 template <> struct MappingTraits<Ref> {
 267   static void mapping(IO &IO, Ref &R) {
 268     MappingNormalization<NormalizedRefKind, RefKind> NKind(IO, R.Kind);
 269     IO.mapRequired("Kind", NKind->Kind);
 270     IO.mapRequired("Location", R.Location);
 271   }
 272 };
 273
 274 struct NormalizedSymbolRole {
 275   NormalizedSymbolRole(IO &) {}
 276   NormalizedSymbolRole(IO &IO, RelationKind R) {
 277     Kind = static_cast<uint8_t>(R);
 278   }
 279
 280   RelationKind denormalize(IO &IO) { return static_cast<RelationKind>(Kind); }
 281
 282   uint8_t Kind = 0;
 283 };
 284
 285 template <> struct MappingTraits<SymbolID> {
 286   static void mapping(IO &IO, SymbolID &ID) {
 287     MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, ID);
 288     IO.mapRequired("ID", NSymbolID->HexString);
 289   }
 290 };
 291
 292 template <> struct MappingTraits<Relation> {
 293   static void mapping(IO &IO, Relation &Relation) {
 294     MappingNormalization<NormalizedSymbolRole, RelationKind> NRole(
 295         IO, Relation.Predicate);
 296     IO.mapRequired("Subject", Relation.Subject);
 297     IO.mapRequired("Predicate", NRole->Kind);
 298     IO.mapRequired("Object", Relation.Object);
 299   }
 300 };
 301
 302 struct NormalizedSourceFlag {
 303   NormalizedSourceFlag(IO &) {}
 304   NormalizedSourceFlag(IO &, IncludeGraphNode::SourceFlag O) {
 305     Flag = static_cast<uint8_t>(O);
 306   }
 307
 308   IncludeGraphNode::SourceFlag denormalize(IO &) {
 309     return static_cast<IncludeGraphNode::SourceFlag>(Flag);
 310   }
 311
 312   uint8_t Flag = 0;
 313 };
 314
 315 struct NormalizedFileDigest {
 316   NormalizedFileDigest(IO &) {}
 317   NormalizedFileDigest(IO &, const FileDigest &Digest) {
 318     HexString = llvm::toHex(Digest);
 319   }
 320
 321   FileDigest denormalize(IO &I) {
 322     FileDigest Digest;
 323     if (HexString.size() == Digest.size() * 2 &&
 324         llvm::all_of(HexString, llvm::isHexDigit)) {
 325       memcpy(Digest.data(), llvm::fromHex(HexString).data(), Digest.size());
 326     } else {
 327       I.setError(std::string("Bad hex file digest: ") + HexString);
 328     }
 329     return Digest;
 330   }
 331
 332   std::string HexString;
 333 };
 334
 335 template <> struct MappingTraits<IncludeGraphNode> {
 336   static void mapping(IO &IO, IncludeGraphNode &Node) {
 337     IO.mapRequired("URI", Node.URI);
 338     MappingNormalization<NormalizedSourceFlag, IncludeGraphNode::SourceFlag>
 339         NSourceFlag(IO, Node.Flags);
 340     IO.mapRequired("Flags", NSourceFlag->Flag);
 341     MappingNormalization<NormalizedFileDigest, FileDigest> NDigest(IO,
 342                                                                    Node.Digest);
 343     IO.mapRequired("Digest", NDigest->HexString);
 344     IO.mapRequired("DirectIncludes", Node.DirectIncludes);
 345   }
 346 };
 347
 348 template <> struct MappingTraits<CompileCommandYAML> {
 349   static void mapping(IO &IO, CompileCommandYAML &Cmd) {
 350     IO.mapRequired("Directory", Cmd.Directory);
 351     IO.mapRequired("CommandLine", Cmd.CommandLine);
 352   }
 353 };
 354
 355 template <> struct MappingTraits<VariantEntry> {
 356   static void mapping(IO &IO, VariantEntry &Variant) {
 357     if (IO.mapTag("!Symbol", Variant.Symbol.has_value())) {
 358       if (!IO.outputting())
 359         Variant.Symbol.emplace();
 360       MappingTraits<Symbol>::mapping(IO, *Variant.Symbol);
 361     } else if (IO.mapTag("!Refs", Variant.Refs.has_value())) {
 362       if (!IO.outputting())
 363         Variant.Refs.emplace();
 364       MappingTraits<RefBundle>::mapping(IO, *Variant.Refs);
 365     } else if (IO.mapTag("!Relations", Variant.Relation.has_value())) {
 366       if (!IO.outputting())
 367         Variant.Relation.emplace();
 368       MappingTraits<Relation>::mapping(IO, *Variant.Relation);
 369     } else if (IO.mapTag("!Source", Variant.Source.has_value())) {
 370       if (!IO.outputting())
 371         Variant.Source.emplace();
 372       MappingTraits<IncludeGraphNode>::mapping(IO, *Variant.Source);
 373     } else if (IO.mapTag("!Cmd", Variant.Cmd.has_value())) {
 374       if (!IO.outputting())
 375         Variant.Cmd.emplace();
 376       MappingTraits<CompileCommandYAML>::mapping(
 377           IO, static_cast<CompileCommandYAML &>(*Variant.Cmd));
 378     }
 379   }
 380 };
 381
 382 } // namespace yaml
 383 } // namespace llvm
 384
 385 namespace clang {
 386 namespace clangd {
 387
 388 void writeYAML(const IndexFileOut &O, llvm::raw_ostream &OS) {
 389   llvm::yaml::Output Yout(OS);
 390   for (const auto &Sym : *O.Symbols) {
 391     VariantEntry Entry;
 392     Entry.Symbol = Sym;
 393     Yout << Entry;
 394   }
 395   if (O.Refs)
 396     for (auto &Sym : *O.Refs) {
 397       VariantEntry Entry;
 398       Entry.Refs = Sym;
 399       Yout << Entry;
 400     }
 401   if (O.Relations)
 402     for (auto &R : *O.Relations) {
 403       VariantEntry Entry;
 404       Entry.Relation = R;
 405       Yout << Entry;
 406     }
 407   if (O.Sources) {
 408     for (const auto &Source : *O.Sources) {
 409       VariantEntry Entry;
 410       Entry.Source = Source.getValue();
 411       Yout << Entry;
 412     }
 413   }
 414   if (O.Cmd) {
 415     VariantEntry Entry;
 416     Entry.Cmd = *O.Cmd;
 417     Yout << Entry;
 418   }
 419 }
 420
 421 llvm::Expected<IndexFileIn> readYAML(llvm::StringRef Data,
 422                                      SymbolOrigin Origin) {
 423   SymbolSlab::Builder Symbols;
 424   RefSlab::Builder Refs;
 425   RelationSlab::Builder Relations;
 426   llvm::BumpPtrAllocator
 427       Arena; // store the underlying data of Position::FileURI.
 428   llvm::UniqueStringSaver Strings(Arena);
 429   llvm::yaml::Input Yin(Data, &Strings);
 430   IncludeGraph Sources;
 431   llvm::Optional<tooling::CompileCommand> Cmd;
 432   while (Yin.setCurrentDocument()) {
 433     llvm::yaml::EmptyContext Ctx;
 434     VariantEntry Variant;
 435     yamlize(Yin, Variant, true, Ctx);
 436     if (Yin.error())
 437       return llvm::errorCodeToError(Yin.error());
 438
 439     if (Variant.Symbol) {
 440       Variant.Symbol->Origin = Origin;
 441       Symbols.insert(*Variant.Symbol);
 442     }
 443     if (Variant.Refs)
 444       for (const auto &Ref : Variant.Refs->second)
 445         Refs.insert(Variant.Refs->first, Ref);
 446     if (Variant.Relation)
 447       Relations.insert(*Variant.Relation);
 448     if (Variant.Source) {
 449       auto &IGN = *Variant.Source;
 450       auto Entry = Sources.try_emplace(IGN.URI).first;
 451       Entry->getValue() = std::move(IGN);
 452       // Fixup refs to refer to map keys which will live on
 453       Entry->getValue().URI = Entry->getKey();
 454       for (auto &Include : Entry->getValue().DirectIncludes)
 455         Include = Sources.try_emplace(Include).first->getKey();
 456     }
 457     if (Variant.Cmd)
 458       Cmd = *Variant.Cmd;
 459     Yin.nextDocument();
 460   }
 461
 462   IndexFileIn Result;
 463   Result.Symbols.emplace(std::move(Symbols).build());
 464   Result.Refs.emplace(std::move(Refs).build());
 465   Result.Relations.emplace(std::move(Relations).build());
 466   if (Sources.size())
 467     Result.Sources = std::move(Sources);
 468   Result.Cmd = std::move(Cmd);
 469   return std::move(Result);
 470 }
 471
 472 std::string toYAML(const Symbol &S) {
 473   std::string Buf;
 474   {
 475     llvm::raw_string_ostream OS(Buf);
 476     llvm::yaml::Output Yout(OS);
 477     Symbol Sym = S; // copy: Yout<< requires mutability.
 478     Yout << Sym;
 479   }
 480   return Buf;
 481 }
 482
 483 std::string toYAML(const std::pair<SymbolID, llvm::ArrayRef<Ref>> &Data) {
 484   RefBundle Refs = {Data.first, Data.second};
 485   std::string Buf;
 486   {
 487     llvm::raw_string_ostream OS(Buf);
 488     llvm::yaml::Output Yout(OS);
 489     Yout << Refs;
 490   }
 491   return Buf;
 492 }
 493
 494 std::string toYAML(const Relation &R) {
 495   std::string Buf;
 496   {
 497     llvm::raw_string_ostream OS(Buf);
 498     llvm::yaml::Output Yout(OS);
 499     Relation Rel = R; // copy: Yout<< requires mutability.
 500     Yout << Rel;
 501   }
 502   return Buf;
 503 }
 504
 505 std::string toYAML(const Ref &R) {
 506   std::string Buf;
 507   {
 508     llvm::raw_string_ostream OS(Buf);
 509     llvm::yaml::Output Yout(OS);
 510     Ref Reference = R; // copy: Yout<< requires mutability.
 511     Yout << Reference;
 512   }
 513   return Buf;
 514 }
 515
 516 } // namespace clangd
 517 } // namespace clang