1 //===-- YAMLSerialization.cpp ------------------------------------*- C++-*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // A YAML index file is a sequence of tagged entries.
10 // Each entry either encodes a Symbol or the list of references to a symbol
11 // (a "ref bundle").
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "Headers.h"
16 #include "index/Ref.h"
17 #include "index/Relation.h"
18 #include "index/Serialization.h"
19 #include "index/Symbol.h"
20 #include "index/SymbolLocation.h"
21 #include "index/SymbolOrigin.h"
22 #include "clang/Tooling/CompilationDatabase.h"
23 #include "llvm/ADT/Optional.h"
24 #include "llvm/ADT/StringRef.h"
25 #include "llvm/Support/Allocator.h"
26 #include "llvm/Support/StringSaver.h"
27 #include "llvm/Support/YAMLTraits.h"
28 #include "llvm/Support/raw_ostream.h"
29 #include <cstdint>
30 
31 LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Symbol::IncludeHeaderWithReferences)
32 LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Ref)
33 
34 namespace {
35 using RefBundle =
36     std::pair<clang::clangd::SymbolID, std::vector<clang::clangd::Ref>>;
37 // This is a pale imitation of std::variant<Symbol, RefBundle, Relation>
38 struct VariantEntry {
39   llvm::Optional<clang::clangd::Symbol> Symbol;
40   llvm::Optional<RefBundle> Refs;
41   llvm::Optional<clang::clangd::Relation> Relation;
42   llvm::Optional<clang::clangd::IncludeGraphNode> Source;
43   llvm::Optional<clang::tooling::CompileCommand> Cmd;
44 };
45 // A class helps YAML to serialize the 32-bit encoded position (Line&Column),
46 // as YAMLIO can't directly map bitfields.
47 struct YPosition {
48   uint32_t Line;
49   uint32_t Column;
50 };
51 
52 // avoid ODR violation of specialization for non-owned CompileCommand
53 struct CompileCommandYAML : clang::tooling::CompileCommand {};
54 
55 } // namespace
56 namespace llvm {
57 namespace yaml {
58 
59 using clang::clangd::FileDigest;
60 using clang::clangd::IncludeGraph;
61 using clang::clangd::IncludeGraphNode;
62 using clang::clangd::Ref;
63 using clang::clangd::RefKind;
64 using clang::clangd::Relation;
65 using clang::clangd::RelationKind;
66 using clang::clangd::Symbol;
67 using clang::clangd::SymbolID;
68 using clang::clangd::SymbolLocation;
69 using clang::index::SymbolInfo;
70 using clang::index::SymbolKind;
71 using clang::index::SymbolLanguage;
72 using clang::tooling::CompileCommand;
73 
74 // Helper to (de)serialize the SymbolID. We serialize it as a hex string.
75 struct NormalizedSymbolID {
76   NormalizedSymbolID(IO &) {}
77   NormalizedSymbolID(IO &, const SymbolID &ID) {
78     llvm::raw_string_ostream OS(HexString);
79     OS << ID;
80   }
81 
82   SymbolID denormalize(IO &I) {
83     auto ID = SymbolID::fromStr(HexString);
84     if (!ID) {
85       I.setError(llvm::toString(ID.takeError()));
86       return SymbolID();
87     }
88     return *ID;
89   }
90 
91   std::string HexString;
92 };
93 
94 struct NormalizedSymbolFlag {
95   NormalizedSymbolFlag(IO &) {}
96   NormalizedSymbolFlag(IO &, Symbol::SymbolFlag F) {
97     Flag = static_cast<uint8_t>(F);
98   }
99 
100   Symbol::SymbolFlag denormalize(IO &) {
101     return static_cast<Symbol::SymbolFlag>(Flag);
102   }
103 
104   uint8_t Flag = 0;
105 };
106 
107 template <> struct MappingTraits<YPosition> {
108   static void mapping(IO &IO, YPosition &Value) {
109     IO.mapRequired("Line", Value.Line);
110     IO.mapRequired("Column", Value.Column);
111   }
112 };
113 
114 struct NormalizedPosition {
115   using Position = clang::clangd::SymbolLocation::Position;
116   NormalizedPosition(IO &) {}
117   NormalizedPosition(IO &, const Position &Pos) {
118     P.Line = Pos.line();
119     P.Column = Pos.column();
120   }
121 
122   Position denormalize(IO &) {
123     Position Pos;
124     Pos.setLine(P.Line);
125     Pos.setColumn(P.Column);
126     return Pos;
127   }
128   YPosition P;
129 };
130 
131 struct NormalizedFileURI {
132   NormalizedFileURI(IO &) {}
133   NormalizedFileURI(IO &, const char *FileURI) { URI = FileURI; }
134 
135   const char *denormalize(IO &IO) {
136     assert(IO.getContext() &&
137            "Expecting an UniqueStringSaver to allocate data");
138     return static_cast<llvm::UniqueStringSaver *>(IO.getContext())
139         ->save(URI)
140         .data();
141   }
142 
143   std::string URI;
144 };
145 
146 template <> struct MappingTraits<SymbolLocation> {
147   static void mapping(IO &IO, SymbolLocation &Value) {
148     MappingNormalization<NormalizedFileURI, const char *> NFile(IO,
149                                                                 Value.FileURI);
150     IO.mapRequired("FileURI", NFile->URI);
151     MappingNormalization<NormalizedPosition, SymbolLocation::Position> NStart(
152         IO, Value.Start);
153     IO.mapRequired("Start", NStart->P);
154     MappingNormalization<NormalizedPosition, SymbolLocation::Position> NEnd(
155         IO, Value.End);
156     IO.mapRequired("End", NEnd->P);
157   }
158 };
159 
160 template <> struct MappingTraits<SymbolInfo> {
161   static void mapping(IO &IO, SymbolInfo &SymInfo) {
162     // FIXME: expose other fields?
163     IO.mapRequired("Kind", SymInfo.Kind);
164     IO.mapRequired("Lang", SymInfo.Lang);
165   }
166 };
167 
168 template <>
169 struct MappingTraits<clang::clangd::Symbol::IncludeHeaderWithReferences> {
170   static void mapping(IO &IO,
171                       clang::clangd::Symbol::IncludeHeaderWithReferences &Inc) {
172     IO.mapRequired("Header", Inc.IncludeHeader);
173     IO.mapRequired("References", Inc.References);
174   }
175 };
176 
177 template <> struct MappingTraits<Symbol> {
178   static void mapping(IO &IO, Symbol &Sym) {
179     MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, Sym.ID);
180     MappingNormalization<NormalizedSymbolFlag, Symbol::SymbolFlag> NSymbolFlag(
181         IO, Sym.Flags);
182     IO.mapRequired("ID", NSymbolID->HexString);
183     IO.mapRequired("Name", Sym.Name);
184     IO.mapRequired("Scope", Sym.Scope);
185     IO.mapRequired("SymInfo", Sym.SymInfo);
186     IO.mapOptional("CanonicalDeclaration", Sym.CanonicalDeclaration,
187                    SymbolLocation());
188     IO.mapOptional("Definition", Sym.Definition, SymbolLocation());
189     IO.mapOptional("References", Sym.References, 0u);
190     IO.mapOptional("Flags", NSymbolFlag->Flag);
191     IO.mapOptional("Signature", Sym.Signature);
192     IO.mapOptional("TemplateSpecializationArgs",
193                    Sym.TemplateSpecializationArgs);
194     IO.mapOptional("CompletionSnippetSuffix", Sym.CompletionSnippetSuffix);
195     IO.mapOptional("Documentation", Sym.Documentation);
196     IO.mapOptional("ReturnType", Sym.ReturnType);
197     IO.mapOptional("Type", Sym.Type);
198     IO.mapOptional("IncludeHeaders", Sym.IncludeHeaders);
199   }
200 };
201 
202 template <> struct ScalarEnumerationTraits<SymbolLanguage> {
203   static void enumeration(IO &IO, SymbolLanguage &Value) {
204     IO.enumCase(Value, "C", SymbolLanguage::C);
205     IO.enumCase(Value, "Cpp", SymbolLanguage::CXX);
206     IO.enumCase(Value, "ObjC", SymbolLanguage::ObjC);
207     IO.enumCase(Value, "Swift", SymbolLanguage::Swift);
208   }
209 };
210 
211 template <> struct ScalarEnumerationTraits<SymbolKind> {
212   static void enumeration(IO &IO, SymbolKind &Value) {
213 #define DEFINE_ENUM(name) IO.enumCase(Value, #name, SymbolKind::name)
214 
215     DEFINE_ENUM(Unknown);
216     DEFINE_ENUM(Function);
217     DEFINE_ENUM(Module);
218     DEFINE_ENUM(Namespace);
219     DEFINE_ENUM(NamespaceAlias);
220     DEFINE_ENUM(Macro);
221     DEFINE_ENUM(Enum);
222     DEFINE_ENUM(Struct);
223     DEFINE_ENUM(Class);
224     DEFINE_ENUM(Protocol);
225     DEFINE_ENUM(Extension);
226     DEFINE_ENUM(Union);
227     DEFINE_ENUM(TypeAlias);
228     DEFINE_ENUM(Function);
229     DEFINE_ENUM(Variable);
230     DEFINE_ENUM(Field);
231     DEFINE_ENUM(EnumConstant);
232     DEFINE_ENUM(InstanceMethod);
233     DEFINE_ENUM(ClassMethod);
234     DEFINE_ENUM(StaticMethod);
235     DEFINE_ENUM(InstanceProperty);
236     DEFINE_ENUM(ClassProperty);
237     DEFINE_ENUM(StaticProperty);
238     DEFINE_ENUM(Constructor);
239     DEFINE_ENUM(Destructor);
240     DEFINE_ENUM(ConversionFunction);
241     DEFINE_ENUM(Parameter);
242     DEFINE_ENUM(Using);
243 
244 #undef DEFINE_ENUM
245   }
246 };
247 
248 template <> struct MappingTraits<RefBundle> {
249   static void mapping(IO &IO, RefBundle &Refs) {
250     MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO,
251                                                                  Refs.first);
252     IO.mapRequired("ID", NSymbolID->HexString);
253     IO.mapRequired("References", Refs.second);
254   }
255 };
256 
257 struct NormalizedRefKind {
258   NormalizedRefKind(IO &) {}
259   NormalizedRefKind(IO &, RefKind O) { Kind = static_cast<uint8_t>(O); }
260 
261   RefKind denormalize(IO &) { return static_cast<RefKind>(Kind); }
262 
263   uint8_t Kind = 0;
264 };
265 
266 template <> struct MappingTraits<Ref> {
267   static void mapping(IO &IO, Ref &R) {
268     MappingNormalization<NormalizedRefKind, RefKind> NKind(IO, R.Kind);
269     IO.mapRequired("Kind", NKind->Kind);
270     IO.mapRequired("Location", R.Location);
271   }
272 };
273 
274 struct NormalizedSymbolRole {
275   NormalizedSymbolRole(IO &) {}
276   NormalizedSymbolRole(IO &IO, RelationKind R) {
277     Kind = static_cast<uint8_t>(R);
278   }
279 
280   RelationKind denormalize(IO &IO) { return static_cast<RelationKind>(Kind); }
281 
282   uint8_t Kind = 0;
283 };
284 
285 template <> struct MappingTraits<SymbolID> {
286   static void mapping(IO &IO, SymbolID &ID) {
287     MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, ID);
288     IO.mapRequired("ID", NSymbolID->HexString);
289   }
290 };
291 
292 template <> struct MappingTraits<Relation> {
293   static void mapping(IO &IO, Relation &Relation) {
294     MappingNormalization<NormalizedSymbolRole, RelationKind> NRole(
295         IO, Relation.Predicate);
296     IO.mapRequired("Subject", Relation.Subject);
297     IO.mapRequired("Predicate", NRole->Kind);
298     IO.mapRequired("Object", Relation.Object);
299   }
300 };
301 
302 struct NormalizedSourceFlag {
303   NormalizedSourceFlag(IO &) {}
304   NormalizedSourceFlag(IO &, IncludeGraphNode::SourceFlag O) {
305     Flag = static_cast<uint8_t>(O);
306   }
307 
308   IncludeGraphNode::SourceFlag denormalize(IO &) {
309     return static_cast<IncludeGraphNode::SourceFlag>(Flag);
310   }
311 
312   uint8_t Flag = 0;
313 };
314 
315 struct NormalizedFileDigest {
316   NormalizedFileDigest(IO &) {}
317   NormalizedFileDigest(IO &, const FileDigest &Digest) {
318     HexString = llvm::toHex(Digest);
319   }
320 
321   FileDigest denormalize(IO &I) {
322     FileDigest Digest;
323     if (HexString.size() == Digest.size() * 2 &&
324         llvm::all_of(HexString, llvm::isHexDigit)) {
325       memcpy(Digest.data(), llvm::fromHex(HexString).data(), Digest.size());
326     } else {
327       I.setError(std::string("Bad hex file digest: ") + HexString);
328     }
329     return Digest;
330   }
331 
332   std::string HexString;
333 };
334 
335 template <> struct MappingTraits<IncludeGraphNode> {
336   static void mapping(IO &IO, IncludeGraphNode &Node) {
337     IO.mapRequired("URI", Node.URI);
338     MappingNormalization<NormalizedSourceFlag, IncludeGraphNode::SourceFlag>
339         NSourceFlag(IO, Node.Flags);
340     IO.mapRequired("Flags", NSourceFlag->Flag);
341     MappingNormalization<NormalizedFileDigest, FileDigest> NDigest(IO,
342                                                                    Node.Digest);
343     IO.mapRequired("Digest", NDigest->HexString);
344     IO.mapRequired("DirectIncludes", Node.DirectIncludes);
345   }
346 };
347 
348 template <> struct MappingTraits<CompileCommandYAML> {
349   static void mapping(IO &IO, CompileCommandYAML &Cmd) {
350     IO.mapRequired("Directory", Cmd.Directory);
351     IO.mapRequired("CommandLine", Cmd.CommandLine);
352   }
353 };
354 
355 template <> struct MappingTraits<VariantEntry> {
356   static void mapping(IO &IO, VariantEntry &Variant) {
357     if (IO.mapTag("!Symbol", Variant.Symbol.has_value())) {
358       if (!IO.outputting())
359         Variant.Symbol.emplace();
360       MappingTraits<Symbol>::mapping(IO, *Variant.Symbol);
361     } else if (IO.mapTag("!Refs", Variant.Refs.has_value())) {
362       if (!IO.outputting())
363         Variant.Refs.emplace();
364       MappingTraits<RefBundle>::mapping(IO, *Variant.Refs);
365     } else if (IO.mapTag("!Relations", Variant.Relation.has_value())) {
366       if (!IO.outputting())
367         Variant.Relation.emplace();
368       MappingTraits<Relation>::mapping(IO, *Variant.Relation);
369     } else if (IO.mapTag("!Source", Variant.Source.has_value())) {
370       if (!IO.outputting())
371         Variant.Source.emplace();
372       MappingTraits<IncludeGraphNode>::mapping(IO, *Variant.Source);
373     } else if (IO.mapTag("!Cmd", Variant.Cmd.has_value())) {
374       if (!IO.outputting())
375         Variant.Cmd.emplace();
376       MappingTraits<CompileCommandYAML>::mapping(
377           IO, static_cast<CompileCommandYAML &>(*Variant.Cmd));
378     }
379   }
380 };
381 
382 } // namespace yaml
383 } // namespace llvm
384 
385 namespace clang {
386 namespace clangd {
387 
388 void writeYAML(const IndexFileOut &O, llvm::raw_ostream &OS) {
389   llvm::yaml::Output Yout(OS);
390   for (const auto &Sym : *O.Symbols) {
391     VariantEntry Entry;
392     Entry.Symbol = Sym;
393     Yout << Entry;
394   }
395   if (O.Refs)
396     for (auto &Sym : *O.Refs) {
397       VariantEntry Entry;
398       Entry.Refs = Sym;
399       Yout << Entry;
400     }
401   if (O.Relations)
402     for (auto &R : *O.Relations) {
403       VariantEntry Entry;
404       Entry.Relation = R;
405       Yout << Entry;
406     }
407   if (O.Sources) {
408     for (const auto &Source : *O.Sources) {
409       VariantEntry Entry;
410       Entry.Source = Source.getValue();
411       Yout << Entry;
412     }
413   }
414   if (O.Cmd) {
415     VariantEntry Entry;
416     Entry.Cmd = *O.Cmd;
417     Yout << Entry;
418   }
419 }
420 
421 llvm::Expected<IndexFileIn> readYAML(llvm::StringRef Data,
422                                      SymbolOrigin Origin) {
423   SymbolSlab::Builder Symbols;
424   RefSlab::Builder Refs;
425   RelationSlab::Builder Relations;
426   llvm::BumpPtrAllocator
427       Arena; // store the underlying data of Position::FileURI.
428   llvm::UniqueStringSaver Strings(Arena);
429   llvm::yaml::Input Yin(Data, &Strings);
430   IncludeGraph Sources;
431   llvm::Optional<tooling::CompileCommand> Cmd;
432   while (Yin.setCurrentDocument()) {
433     llvm::yaml::EmptyContext Ctx;
434     VariantEntry Variant;
435     yamlize(Yin, Variant, true, Ctx);
436     if (Yin.error())
437       return llvm::errorCodeToError(Yin.error());
438 
439     if (Variant.Symbol) {
440       Variant.Symbol->Origin = Origin;
441       Symbols.insert(*Variant.Symbol);
442     }
443     if (Variant.Refs)
444       for (const auto &Ref : Variant.Refs->second)
445         Refs.insert(Variant.Refs->first, Ref);
446     if (Variant.Relation)
447       Relations.insert(*Variant.Relation);
448     if (Variant.Source) {
449       auto &IGN = *Variant.Source;
450       auto Entry = Sources.try_emplace(IGN.URI).first;
451       Entry->getValue() = std::move(IGN);
452       // Fixup refs to refer to map keys which will live on
453       Entry->getValue().URI = Entry->getKey();
454       for (auto &Include : Entry->getValue().DirectIncludes)
455         Include = Sources.try_emplace(Include).first->getKey();
456     }
457     if (Variant.Cmd)
458       Cmd = *Variant.Cmd;
459     Yin.nextDocument();
460   }
461 
462   IndexFileIn Result;
463   Result.Symbols.emplace(std::move(Symbols).build());
464   Result.Refs.emplace(std::move(Refs).build());
465   Result.Relations.emplace(std::move(Relations).build());
466   if (Sources.size())
467     Result.Sources = std::move(Sources);
468   Result.Cmd = std::move(Cmd);
469   return std::move(Result);
470 }
471 
472 std::string toYAML(const Symbol &S) {
473   std::string Buf;
474   {
475     llvm::raw_string_ostream OS(Buf);
476     llvm::yaml::Output Yout(OS);
477     Symbol Sym = S; // copy: Yout<< requires mutability.
478     Yout << Sym;
479   }
480   return Buf;
481 }
482 
483 std::string toYAML(const std::pair<SymbolID, llvm::ArrayRef<Ref>> &Data) {
484   RefBundle Refs = {Data.first, Data.second};
485   std::string Buf;
486   {
487     llvm::raw_string_ostream OS(Buf);
488     llvm::yaml::Output Yout(OS);
489     Yout << Refs;
490   }
491   return Buf;
492 }
493 
494 std::string toYAML(const Relation &R) {
495   std::string Buf;
496   {
497     llvm::raw_string_ostream OS(Buf);
498     llvm::yaml::Output Yout(OS);
499     Relation Rel = R; // copy: Yout<< requires mutability.
500     Yout << Rel;
501   }
502   return Buf;
503 }
504 
505 std::string toYAML(const Ref &R) {
506   std::string Buf;
507   {
508     llvm::raw_string_ostream OS(Buf);
509     llvm::yaml::Output Yout(OS);
510     Ref Reference = R; // copy: Yout<< requires mutability.
511     Yout << Reference;
512   }
513   return Buf;
514 }
515 
516 } // namespace clangd
517 } // namespace clang
518