1 //===-- YAMLSerialization.cpp ------------------------------------*- C++-*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // A YAML index file is a sequence of tagged entries.
10 // Each entry either encodes a Symbol or the list of references to a symbol
11 // (a "ref bundle").
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "Index.h"
16 #include "Relation.h"
17 #include "Serialization.h"
18 #include "SymbolLocation.h"
19 #include "SymbolOrigin.h"
20 #include "dex/Dex.h"
21 #include "support/Logger.h"
22 #include "support/Trace.h"
23 #include "llvm/ADT/Optional.h"
24 #include "llvm/ADT/SmallVector.h"
25 #include "llvm/ADT/StringRef.h"
26 #include "llvm/Support/Allocator.h"
27 #include "llvm/Support/Errc.h"
28 #include "llvm/Support/MemoryBuffer.h"
29 #include "llvm/Support/StringSaver.h"
30 #include "llvm/Support/YAMLTraits.h"
31 #include "llvm/Support/raw_ostream.h"
32 #include <cstdint>
33 
34 LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Symbol::IncludeHeaderWithReferences)
35 LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Ref)
36 
37 namespace {
38 using RefBundle =
39     std::pair<clang::clangd::SymbolID, std::vector<clang::clangd::Ref>>;
40 // This is a pale imitation of std::variant<Symbol, RefBundle, Relation>
41 struct VariantEntry {
42   llvm::Optional<clang::clangd::Symbol> Symbol;
43   llvm::Optional<RefBundle> Refs;
44   llvm::Optional<clang::clangd::Relation> Relation;
45   llvm::Optional<clang::clangd::IncludeGraphNode> Source;
46   llvm::Optional<clang::tooling::CompileCommand> Cmd;
47 };
48 // A class helps YAML to serialize the 32-bit encoded position (Line&Column),
49 // as YAMLIO can't directly map bitfields.
50 struct YPosition {
51   uint32_t Line;
52   uint32_t Column;
53 };
54 
55 // avoid ODR violation of specialization for non-owned CompileCommand
56 struct CompileCommandYAML : clang::tooling::CompileCommand {};
57 
58 } // namespace
59 namespace llvm {
60 namespace yaml {
61 
62 using clang::clangd::FileDigest;
63 using clang::clangd::IncludeGraph;
64 using clang::clangd::IncludeGraphNode;
65 using clang::clangd::Ref;
66 using clang::clangd::RefKind;
67 using clang::clangd::Relation;
68 using clang::clangd::RelationKind;
69 using clang::clangd::Symbol;
70 using clang::clangd::SymbolID;
71 using clang::clangd::SymbolLocation;
72 using clang::clangd::SymbolOrigin;
73 using clang::index::SymbolInfo;
74 using clang::index::SymbolKind;
75 using clang::index::SymbolLanguage;
76 using clang::tooling::CompileCommand;
77 
78 // Helper to (de)serialize the SymbolID. We serialize it as a hex string.
79 struct NormalizedSymbolID {
80   NormalizedSymbolID(IO &) {}
81   NormalizedSymbolID(IO &, const SymbolID &ID) {
82     llvm::raw_string_ostream OS(HexString);
83     OS << ID;
84   }
85 
86   SymbolID denormalize(IO &I) {
87     auto ID = SymbolID::fromStr(HexString);
88     if (!ID) {
89       I.setError(llvm::toString(ID.takeError()));
90       return SymbolID();
91     }
92     return *ID;
93   }
94 
95   std::string HexString;
96 };
97 
98 struct NormalizedSymbolFlag {
99   NormalizedSymbolFlag(IO &) {}
100   NormalizedSymbolFlag(IO &, Symbol::SymbolFlag F) {
101     Flag = static_cast<uint8_t>(F);
102   }
103 
104   Symbol::SymbolFlag denormalize(IO &) {
105     return static_cast<Symbol::SymbolFlag>(Flag);
106   }
107 
108   uint8_t Flag = 0;
109 };
110 
111 struct NormalizedSymbolOrigin {
112   NormalizedSymbolOrigin(IO &) {}
113   NormalizedSymbolOrigin(IO &, SymbolOrigin O) {
114     Origin = static_cast<uint8_t>(O);
115   }
116 
117   SymbolOrigin denormalize(IO &) { return static_cast<SymbolOrigin>(Origin); }
118 
119   uint8_t Origin = 0;
120 };
121 
122 template <> struct MappingTraits<YPosition> {
123   static void mapping(IO &IO, YPosition &Value) {
124     IO.mapRequired("Line", Value.Line);
125     IO.mapRequired("Column", Value.Column);
126   }
127 };
128 
129 struct NormalizedPosition {
130   using Position = clang::clangd::SymbolLocation::Position;
131   NormalizedPosition(IO &) {}
132   NormalizedPosition(IO &, const Position &Pos) {
133     P.Line = Pos.line();
134     P.Column = Pos.column();
135   }
136 
137   Position denormalize(IO &) {
138     Position Pos;
139     Pos.setLine(P.Line);
140     Pos.setColumn(P.Column);
141     return Pos;
142   }
143   YPosition P;
144 };
145 
146 struct NormalizedFileURI {
147   NormalizedFileURI(IO &) {}
148   NormalizedFileURI(IO &, const char *FileURI) { URI = FileURI; }
149 
150   const char *denormalize(IO &IO) {
151     assert(IO.getContext() &&
152            "Expecting an UniqueStringSaver to allocate data");
153     return static_cast<llvm::UniqueStringSaver *>(IO.getContext())
154         ->save(URI)
155         .data();
156   }
157 
158   std::string URI;
159 };
160 
161 template <> struct MappingTraits<SymbolLocation> {
162   static void mapping(IO &IO, SymbolLocation &Value) {
163     MappingNormalization<NormalizedFileURI, const char *> NFile(IO,
164                                                                 Value.FileURI);
165     IO.mapRequired("FileURI", NFile->URI);
166     MappingNormalization<NormalizedPosition, SymbolLocation::Position> NStart(
167         IO, Value.Start);
168     IO.mapRequired("Start", NStart->P);
169     MappingNormalization<NormalizedPosition, SymbolLocation::Position> NEnd(
170         IO, Value.End);
171     IO.mapRequired("End", NEnd->P);
172   }
173 };
174 
175 template <> struct MappingTraits<SymbolInfo> {
176   static void mapping(IO &io, SymbolInfo &SymInfo) {
177     // FIXME: expose other fields?
178     io.mapRequired("Kind", SymInfo.Kind);
179     io.mapRequired("Lang", SymInfo.Lang);
180   }
181 };
182 
183 template <>
184 struct MappingTraits<clang::clangd::Symbol::IncludeHeaderWithReferences> {
185   static void mapping(IO &io,
186                       clang::clangd::Symbol::IncludeHeaderWithReferences &Inc) {
187     io.mapRequired("Header", Inc.IncludeHeader);
188     io.mapRequired("References", Inc.References);
189   }
190 };
191 
192 template <> struct MappingTraits<Symbol> {
193   static void mapping(IO &IO, Symbol &Sym) {
194     MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, Sym.ID);
195     MappingNormalization<NormalizedSymbolFlag, Symbol::SymbolFlag> NSymbolFlag(
196         IO, Sym.Flags);
197     MappingNormalization<NormalizedSymbolOrigin, SymbolOrigin> NSymbolOrigin(
198         IO, Sym.Origin);
199     IO.mapRequired("ID", NSymbolID->HexString);
200     IO.mapRequired("Name", Sym.Name);
201     IO.mapRequired("Scope", Sym.Scope);
202     IO.mapRequired("SymInfo", Sym.SymInfo);
203     IO.mapOptional("CanonicalDeclaration", Sym.CanonicalDeclaration,
204                    SymbolLocation());
205     IO.mapOptional("Definition", Sym.Definition, SymbolLocation());
206     IO.mapOptional("References", Sym.References, 0u);
207     IO.mapOptional("Origin", NSymbolOrigin->Origin);
208     IO.mapOptional("Flags", NSymbolFlag->Flag);
209     IO.mapOptional("Signature", Sym.Signature);
210     IO.mapOptional("TemplateSpecializationArgs",
211                    Sym.TemplateSpecializationArgs);
212     IO.mapOptional("CompletionSnippetSuffix", Sym.CompletionSnippetSuffix);
213     IO.mapOptional("Documentation", Sym.Documentation);
214     IO.mapOptional("ReturnType", Sym.ReturnType);
215     IO.mapOptional("Type", Sym.Type);
216     IO.mapOptional("IncludeHeaders", Sym.IncludeHeaders);
217   }
218 };
219 
220 template <> struct ScalarEnumerationTraits<SymbolLanguage> {
221   static void enumeration(IO &IO, SymbolLanguage &Value) {
222     IO.enumCase(Value, "C", SymbolLanguage::C);
223     IO.enumCase(Value, "Cpp", SymbolLanguage::CXX);
224     IO.enumCase(Value, "ObjC", SymbolLanguage::ObjC);
225     IO.enumCase(Value, "Swift", SymbolLanguage::Swift);
226   }
227 };
228 
229 template <> struct ScalarEnumerationTraits<SymbolKind> {
230   static void enumeration(IO &IO, SymbolKind &Value) {
231 #define DEFINE_ENUM(name) IO.enumCase(Value, #name, SymbolKind::name)
232 
233     DEFINE_ENUM(Unknown);
234     DEFINE_ENUM(Function);
235     DEFINE_ENUM(Module);
236     DEFINE_ENUM(Namespace);
237     DEFINE_ENUM(NamespaceAlias);
238     DEFINE_ENUM(Macro);
239     DEFINE_ENUM(Enum);
240     DEFINE_ENUM(Struct);
241     DEFINE_ENUM(Class);
242     DEFINE_ENUM(Protocol);
243     DEFINE_ENUM(Extension);
244     DEFINE_ENUM(Union);
245     DEFINE_ENUM(TypeAlias);
246     DEFINE_ENUM(Function);
247     DEFINE_ENUM(Variable);
248     DEFINE_ENUM(Field);
249     DEFINE_ENUM(EnumConstant);
250     DEFINE_ENUM(InstanceMethod);
251     DEFINE_ENUM(ClassMethod);
252     DEFINE_ENUM(StaticMethod);
253     DEFINE_ENUM(InstanceProperty);
254     DEFINE_ENUM(ClassProperty);
255     DEFINE_ENUM(StaticProperty);
256     DEFINE_ENUM(Constructor);
257     DEFINE_ENUM(Destructor);
258     DEFINE_ENUM(ConversionFunction);
259     DEFINE_ENUM(Parameter);
260     DEFINE_ENUM(Using);
261 
262 #undef DEFINE_ENUM
263   }
264 };
265 
266 template <> struct MappingTraits<RefBundle> {
267   static void mapping(IO &IO, RefBundle &Refs) {
268     MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO,
269                                                                  Refs.first);
270     IO.mapRequired("ID", NSymbolID->HexString);
271     IO.mapRequired("References", Refs.second);
272   }
273 };
274 
275 struct NormalizedRefKind {
276   NormalizedRefKind(IO &) {}
277   NormalizedRefKind(IO &, RefKind O) { Kind = static_cast<uint8_t>(O); }
278 
279   RefKind denormalize(IO &) { return static_cast<RefKind>(Kind); }
280 
281   uint8_t Kind = 0;
282 };
283 
284 template <> struct MappingTraits<Ref> {
285   static void mapping(IO &IO, Ref &R) {
286     MappingNormalization<NormalizedRefKind, RefKind> NKind(IO, R.Kind);
287     IO.mapRequired("Kind", NKind->Kind);
288     IO.mapRequired("Location", R.Location);
289   }
290 };
291 
292 struct NormalizedSymbolRole {
293   NormalizedSymbolRole(IO &) {}
294   NormalizedSymbolRole(IO &IO, RelationKind R) {
295     Kind = static_cast<uint8_t>(R);
296   }
297 
298   RelationKind denormalize(IO &IO) { return static_cast<RelationKind>(Kind); }
299 
300   uint8_t Kind = 0;
301 };
302 
303 template <> struct MappingTraits<SymbolID> {
304   static void mapping(IO &IO, SymbolID &ID) {
305     MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, ID);
306     IO.mapRequired("ID", NSymbolID->HexString);
307   }
308 };
309 
310 template <> struct MappingTraits<Relation> {
311   static void mapping(IO &IO, Relation &Relation) {
312     MappingNormalization<NormalizedSymbolRole, RelationKind> NRole(
313         IO, Relation.Predicate);
314     IO.mapRequired("Subject", Relation.Subject);
315     IO.mapRequired("Predicate", NRole->Kind);
316     IO.mapRequired("Object", Relation.Object);
317   }
318 };
319 
320 struct NormalizedSourceFlag {
321   NormalizedSourceFlag(IO &) {}
322   NormalizedSourceFlag(IO &, IncludeGraphNode::SourceFlag O) {
323     Flag = static_cast<uint8_t>(O);
324   }
325 
326   IncludeGraphNode::SourceFlag denormalize(IO &) {
327     return static_cast<IncludeGraphNode::SourceFlag>(Flag);
328   }
329 
330   uint8_t Flag = 0;
331 };
332 
333 struct NormalizedFileDigest {
334   NormalizedFileDigest(IO &) {}
335   NormalizedFileDigest(IO &, const FileDigest &Digest) {
336     HexString = llvm::toHex(Digest);
337   }
338 
339   FileDigest denormalize(IO &I) {
340     FileDigest Digest;
341     if (HexString.size() == Digest.size() * 2 &&
342         llvm::all_of(HexString, llvm::isHexDigit)) {
343       memcpy(Digest.data(), llvm::fromHex(HexString).data(), Digest.size());
344     } else {
345       I.setError(std::string("Bad hex file digest: ") + HexString);
346     }
347     return Digest;
348   }
349 
350   std::string HexString;
351 };
352 
353 template <> struct MappingTraits<IncludeGraphNode> {
354   static void mapping(IO &IO, IncludeGraphNode &Node) {
355     IO.mapRequired("URI", Node.URI);
356     MappingNormalization<NormalizedSourceFlag, IncludeGraphNode::SourceFlag>
357         NSourceFlag(IO, Node.Flags);
358     IO.mapRequired("Flags", NSourceFlag->Flag);
359     MappingNormalization<NormalizedFileDigest, FileDigest> NDigest(IO,
360                                                                    Node.Digest);
361     IO.mapRequired("Digest", NDigest->HexString);
362     IO.mapRequired("DirectIncludes", Node.DirectIncludes);
363   }
364 };
365 
366 template <> struct MappingTraits<CompileCommandYAML> {
367   static void mapping(IO &IO, CompileCommandYAML &Cmd) {
368     IO.mapRequired("Directory", Cmd.Directory);
369     IO.mapRequired("CommandLine", Cmd.CommandLine);
370   }
371 };
372 
373 template <> struct MappingTraits<VariantEntry> {
374   static void mapping(IO &IO, VariantEntry &Variant) {
375     if (IO.mapTag("!Symbol", Variant.Symbol.hasValue())) {
376       if (!IO.outputting())
377         Variant.Symbol.emplace();
378       MappingTraits<Symbol>::mapping(IO, *Variant.Symbol);
379     } else if (IO.mapTag("!Refs", Variant.Refs.hasValue())) {
380       if (!IO.outputting())
381         Variant.Refs.emplace();
382       MappingTraits<RefBundle>::mapping(IO, *Variant.Refs);
383     } else if (IO.mapTag("!Relations", Variant.Relation.hasValue())) {
384       if (!IO.outputting())
385         Variant.Relation.emplace();
386       MappingTraits<Relation>::mapping(IO, *Variant.Relation);
387     } else if (IO.mapTag("!Source", Variant.Source.hasValue())) {
388       if (!IO.outputting())
389         Variant.Source.emplace();
390       MappingTraits<IncludeGraphNode>::mapping(IO, *Variant.Source);
391     } else if (IO.mapTag("!Cmd", Variant.Cmd.hasValue())) {
392       if (!IO.outputting())
393         Variant.Cmd.emplace();
394       MappingTraits<CompileCommandYAML>::mapping(
395           IO, static_cast<CompileCommandYAML &>(*Variant.Cmd));
396     }
397   }
398 };
399 
400 } // namespace yaml
401 } // namespace llvm
402 
403 namespace clang {
404 namespace clangd {
405 
406 void writeYAML(const IndexFileOut &O, llvm::raw_ostream &OS) {
407   llvm::yaml::Output Yout(OS);
408   for (const auto &Sym : *O.Symbols) {
409     VariantEntry Entry;
410     Entry.Symbol = Sym;
411     Yout << Entry;
412   }
413   if (O.Refs)
414     for (auto &Sym : *O.Refs) {
415       VariantEntry Entry;
416       Entry.Refs = Sym;
417       Yout << Entry;
418     }
419   if (O.Relations)
420     for (auto &R : *O.Relations) {
421       VariantEntry Entry;
422       Entry.Relation = R;
423       Yout << Entry;
424     }
425   if (O.Sources) {
426     for (const auto &Source : *O.Sources) {
427       VariantEntry Entry;
428       Entry.Source = Source.getValue();
429       Yout << Entry;
430     }
431   }
432   if (O.Cmd) {
433     VariantEntry Entry;
434     Entry.Cmd = *O.Cmd;
435     Yout << Entry;
436   }
437 }
438 
439 llvm::Expected<IndexFileIn> readYAML(llvm::StringRef Data) {
440   SymbolSlab::Builder Symbols;
441   RefSlab::Builder Refs;
442   RelationSlab::Builder Relations;
443   llvm::BumpPtrAllocator
444       Arena; // store the underlying data of Position::FileURI.
445   llvm::UniqueStringSaver Strings(Arena);
446   llvm::yaml::Input Yin(Data, &Strings);
447   IncludeGraph Sources;
448   llvm::Optional<tooling::CompileCommand> Cmd;
449   while (Yin.setCurrentDocument()) {
450     llvm::yaml::EmptyContext Ctx;
451     VariantEntry Variant;
452     yamlize(Yin, Variant, true, Ctx);
453     if (Yin.error())
454       return llvm::errorCodeToError(Yin.error());
455 
456     if (Variant.Symbol)
457       Symbols.insert(*Variant.Symbol);
458     if (Variant.Refs)
459       for (const auto &Ref : Variant.Refs->second)
460         Refs.insert(Variant.Refs->first, Ref);
461     if (Variant.Relation)
462       Relations.insert(*Variant.Relation);
463     if (Variant.Source) {
464       auto &IGN = Variant.Source.getValue();
465       auto Entry = Sources.try_emplace(IGN.URI).first;
466       Entry->getValue() = std::move(IGN);
467       // Fixup refs to refer to map keys which will live on
468       Entry->getValue().URI = Entry->getKey();
469       for (auto &Include : Entry->getValue().DirectIncludes)
470         Include = Sources.try_emplace(Include).first->getKey();
471     }
472     if (Variant.Cmd)
473       Cmd = *Variant.Cmd;
474     Yin.nextDocument();
475   }
476 
477   IndexFileIn Result;
478   Result.Symbols.emplace(std::move(Symbols).build());
479   Result.Refs.emplace(std::move(Refs).build());
480   Result.Relations.emplace(std::move(Relations).build());
481   if (Sources.size())
482     Result.Sources = std::move(Sources);
483   Result.Cmd = std::move(Cmd);
484   return std::move(Result);
485 }
486 
487 std::string toYAML(const Symbol &S) {
488   std::string Buf;
489   {
490     llvm::raw_string_ostream OS(Buf);
491     llvm::yaml::Output Yout(OS);
492     Symbol Sym = S; // copy: Yout<< requires mutability.
493     Yout << Sym;
494   }
495   return Buf;
496 }
497 
498 std::string toYAML(const std::pair<SymbolID, llvm::ArrayRef<Ref>> &Data) {
499   RefBundle Refs = {Data.first, Data.second};
500   std::string Buf;
501   {
502     llvm::raw_string_ostream OS(Buf);
503     llvm::yaml::Output Yout(OS);
504     Yout << Refs;
505   }
506   return Buf;
507 }
508 
509 std::string toYAML(const Relation &R) {
510   std::string Buf;
511   {
512     llvm::raw_string_ostream OS(Buf);
513     llvm::yaml::Output Yout(OS);
514     Relation Rel = R; // copy: Yout<< requires mutability.
515     Yout << Rel;
516   }
517   return Buf;
518 }
519 
520 std::string toYAML(const Ref &R) {
521   std::string Buf;
522   {
523     llvm::raw_string_ostream OS(Buf);
524     llvm::yaml::Output Yout(OS);
525     Ref Reference = R; // copy: Yout<< requires mutability.
526     Yout << Reference;
527   }
528   return Buf;
529 }
530 
531 llvm::Expected<clangd::Symbol>
532 symbolFromYAML(StringRef YAML, llvm::UniqueStringSaver *Strings) {
533   clangd::Symbol Deserialized;
534   llvm::yaml::Input YAMLInput(YAML, Strings);
535   if (YAMLInput.error())
536     return error("Unable to deserialize Symbol from YAML: {0}", YAML);
537   YAMLInput >> Deserialized;
538   return Deserialized;
539 }
540 
541 llvm::Expected<clangd::Ref> refFromYAML(StringRef YAML,
542                                         llvm::UniqueStringSaver *Strings) {
543   clangd::Ref Deserialized;
544   llvm::yaml::Input YAMLInput(YAML, Strings);
545   if (YAMLInput.error())
546     return error("Unable to deserialize Symbol from YAML: {0}", YAML);
547   YAMLInput >> Deserialized;
548   return Deserialized;
549 }
550 
551 } // namespace clangd
552 } // namespace clang
553