1 //===-- SerializationTests.cpp - Binary and YAML serialization unit tests -===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "Headers.h"
10 #include "RIFF.h"
11 #include "index/Serialization.h"
12 #include "support/Logger.h"
13 #include "clang/Tooling/CompilationDatabase.h"
14 #include "llvm/ADT/StringExtras.h"
15 #include "llvm/Support/Compression.h"
16 #include "llvm/Support/Error.h"
17 #include "llvm/Support/ScopedPrinter.h"
18 #include "gmock/gmock.h"
19 #include "gtest/gtest.h"
20 #ifdef LLVM_ON_UNIX
21 #include <sys/resource.h>
22 #endif
23 
24 using ::testing::ElementsAre;
25 using ::testing::Pair;
26 using ::testing::UnorderedElementsAre;
27 using ::testing::UnorderedElementsAreArray;
28 
29 namespace clang {
30 namespace clangd {
31 namespace {
32 
33 const char *YAML = R"(
34 ---
35 !Symbol
36 ID: 057557CEBF6E6B2D
37 Name:   'Foo1'
38 Scope:   'clang::'
39 SymInfo:
40   Kind:            Function
41   Lang:            Cpp
42 CanonicalDeclaration:
43   FileURI:        file:///path/foo.h
44   Start:
45     Line: 1
46     Column: 0
47   End:
48     Line: 1
49     Column: 1
50 Flags:    129
51 Documentation:    'Foo doc'
52 ReturnType:    'int'
53 IncludeHeaders:
54   - Header:    'include1'
55     References:    7
56   - Header:    'include2'
57     References:    3
58 ...
59 ---
60 !Symbol
61 ID: 057557CEBF6E6B2E
62 Name:   'Foo2'
63 Scope:   'clang::'
64 SymInfo:
65   Kind:            Function
66   Lang:            Cpp
67 CanonicalDeclaration:
68   FileURI:        file:///path/bar.h
69   Start:
70     Line: 1
71     Column: 0
72   End:
73     Line: 1
74     Column: 1
75 Flags:    2
76 Signature:    '-sig'
77 CompletionSnippetSuffix:    '-snippet'
78 ...
79 !Refs
80 ID: 057557CEBF6E6B2D
81 References:
82   - Kind: 4
83     Location:
84       FileURI:    file:///path/foo.cc
85       Start:
86         Line: 5
87         Column: 3
88       End:
89         Line: 5
90         Column: 8
91 ...
92 --- !Relations
93 Subject:
94   ID:              6481EE7AF2841756
95 Predicate:       0
96 Object:
97   ID:              6512AEC512EA3A2D
98 ...
99 --- !Cmd
100 Directory:       'testdir'
101 CommandLine:
102   - 'cmd1'
103   - 'cmd2'
104 ...
105 --- !Source
106 URI:             'file:///path/source1.cpp'
107 Flags:           1
108 Digest:          EED8F5EAF25C453C
109 DirectIncludes:
110   - 'file:///path/inc1.h'
111   - 'file:///path/inc2.h'
112 ...
113 )";
114 
115 MATCHER_P(id, I, "") { return arg.ID == cantFail(SymbolID::fromStr(I)); }
116 MATCHER_P(qName, Name, "") { return (arg.Scope + arg.Name).str() == Name; }
117 MATCHER_P2(IncludeHeaderWithRef, IncludeHeader, References, "") {
118   return (arg.IncludeHeader == IncludeHeader) && (arg.References == References);
119 }
120 
121 auto readIndexFile(llvm::StringRef Text) {
122   return readIndexFile(Text, SymbolOrigin::Static);
123 }
124 
125 TEST(SerializationTest, NoCrashOnEmptyYAML) {
126   EXPECT_TRUE(bool(readIndexFile("")));
127 }
128 
129 TEST(SerializationTest, YAMLConversions) {
130   auto ParsedYAML = readIndexFile(YAML);
131   ASSERT_TRUE(bool(ParsedYAML)) << ParsedYAML.takeError();
132   ASSERT_TRUE(bool(ParsedYAML->Symbols));
133   EXPECT_THAT(
134       *ParsedYAML->Symbols,
135       UnorderedElementsAre(id("057557CEBF6E6B2D"), id("057557CEBF6E6B2E")));
136 
137   auto Sym1 = *ParsedYAML->Symbols->find(
138       cantFail(SymbolID::fromStr("057557CEBF6E6B2D")));
139   auto Sym2 = *ParsedYAML->Symbols->find(
140       cantFail(SymbolID::fromStr("057557CEBF6E6B2E")));
141 
142   EXPECT_THAT(Sym1, qName("clang::Foo1"));
143   EXPECT_EQ(Sym1.Signature, "");
144   EXPECT_EQ(Sym1.Documentation, "Foo doc");
145   EXPECT_EQ(Sym1.ReturnType, "int");
146   EXPECT_EQ(StringRef(Sym1.CanonicalDeclaration.FileURI), "file:///path/foo.h");
147   EXPECT_EQ(Sym1.Origin, SymbolOrigin::Static);
148   EXPECT_EQ(static_cast<uint8_t>(Sym1.Flags), 129);
149   EXPECT_TRUE(Sym1.Flags & Symbol::IndexedForCodeCompletion);
150   EXPECT_FALSE(Sym1.Flags & Symbol::Deprecated);
151   EXPECT_THAT(Sym1.IncludeHeaders,
152               UnorderedElementsAre(IncludeHeaderWithRef("include1", 7u),
153                                    IncludeHeaderWithRef("include2", 3u)));
154 
155   EXPECT_THAT(Sym2, qName("clang::Foo2"));
156   EXPECT_EQ(Sym2.Signature, "-sig");
157   EXPECT_EQ(Sym2.ReturnType, "");
158   EXPECT_EQ(llvm::StringRef(Sym2.CanonicalDeclaration.FileURI),
159             "file:///path/bar.h");
160   EXPECT_FALSE(Sym2.Flags & Symbol::IndexedForCodeCompletion);
161   EXPECT_TRUE(Sym2.Flags & Symbol::Deprecated);
162 
163   ASSERT_TRUE(bool(ParsedYAML->Refs));
164   EXPECT_THAT(
165       *ParsedYAML->Refs,
166       UnorderedElementsAre(Pair(cantFail(SymbolID::fromStr("057557CEBF6E6B2D")),
167                                 ::testing::SizeIs(1))));
168   auto Ref1 = ParsedYAML->Refs->begin()->second.front();
169   EXPECT_EQ(Ref1.Kind, RefKind::Reference);
170   EXPECT_EQ(StringRef(Ref1.Location.FileURI), "file:///path/foo.cc");
171 
172   SymbolID Base = cantFail(SymbolID::fromStr("6481EE7AF2841756"));
173   SymbolID Derived = cantFail(SymbolID::fromStr("6512AEC512EA3A2D"));
174   ASSERT_TRUE(bool(ParsedYAML->Relations));
175   EXPECT_THAT(
176       *ParsedYAML->Relations,
177       UnorderedElementsAre(Relation{Base, RelationKind::BaseOf, Derived}));
178 
179   ASSERT_TRUE(bool(ParsedYAML->Cmd));
180   auto &Cmd = *ParsedYAML->Cmd;
181   ASSERT_EQ(Cmd.Directory, "testdir");
182   EXPECT_THAT(Cmd.CommandLine, ElementsAre("cmd1", "cmd2"));
183 
184   ASSERT_TRUE(bool(ParsedYAML->Sources));
185   const auto *URI = "file:///path/source1.cpp";
186   ASSERT_TRUE(ParsedYAML->Sources->count(URI));
187   auto IGNDeserialized = ParsedYAML->Sources->lookup(URI);
188   EXPECT_EQ(llvm::toHex(IGNDeserialized.Digest), "EED8F5EAF25C453C");
189   EXPECT_THAT(IGNDeserialized.DirectIncludes,
190               ElementsAre("file:///path/inc1.h", "file:///path/inc2.h"));
191   EXPECT_EQ(IGNDeserialized.URI, URI);
192   EXPECT_EQ(IGNDeserialized.Flags, IncludeGraphNode::SourceFlag(1));
193 }
194 
195 std::vector<std::string> yamlFromSymbols(const SymbolSlab &Slab) {
196   std::vector<std::string> Result;
197   for (const auto &Sym : Slab)
198     Result.push_back(toYAML(Sym));
199   return Result;
200 }
201 std::vector<std::string> yamlFromRefs(const RefSlab &Slab) {
202   std::vector<std::string> Result;
203   for (const auto &Refs : Slab)
204     Result.push_back(toYAML(Refs));
205   return Result;
206 }
207 
208 std::vector<std::string> yamlFromRelations(const RelationSlab &Slab) {
209   std::vector<std::string> Result;
210   for (const auto &Rel : Slab)
211     Result.push_back(toYAML(Rel));
212   return Result;
213 }
214 
215 TEST(SerializationTest, BinaryConversions) {
216   auto In = readIndexFile(YAML);
217   EXPECT_TRUE(bool(In)) << In.takeError();
218 
219   // Write to binary format, and parse again.
220   IndexFileOut Out(*In);
221   Out.Format = IndexFileFormat::RIFF;
222   std::string Serialized = llvm::to_string(Out);
223 
224   auto In2 = readIndexFile(Serialized);
225   ASSERT_TRUE(bool(In2)) << In.takeError();
226   ASSERT_TRUE(In2->Symbols);
227   ASSERT_TRUE(In2->Refs);
228   ASSERT_TRUE(In2->Relations);
229 
230   // Assert the YAML serializations match, for nice comparisons and diffs.
231   EXPECT_THAT(yamlFromSymbols(*In2->Symbols),
232               UnorderedElementsAreArray(yamlFromSymbols(*In->Symbols)));
233   EXPECT_THAT(yamlFromRefs(*In2->Refs),
234               UnorderedElementsAreArray(yamlFromRefs(*In->Refs)));
235   EXPECT_THAT(yamlFromRelations(*In2->Relations),
236               UnorderedElementsAreArray(yamlFromRelations(*In->Relations)));
237 }
238 
239 TEST(SerializationTest, SrcsTest) {
240   auto In = readIndexFile(YAML);
241   EXPECT_TRUE(bool(In)) << In.takeError();
242 
243   std::string TestContent("TestContent");
244   IncludeGraphNode IGN;
245   IGN.Digest = digest(TestContent);
246   IGN.DirectIncludes = {"inc1", "inc2"};
247   IGN.URI = "URI";
248   IGN.Flags |= IncludeGraphNode::SourceFlag::IsTU;
249   IGN.Flags |= IncludeGraphNode::SourceFlag::HadErrors;
250   IncludeGraph Sources;
251   Sources[IGN.URI] = IGN;
252   // Write to binary format, and parse again.
253   IndexFileOut Out(*In);
254   Out.Format = IndexFileFormat::RIFF;
255   Out.Sources = &Sources;
256   {
257     std::string Serialized = llvm::to_string(Out);
258 
259     auto In = readIndexFile(Serialized);
260     ASSERT_TRUE(bool(In)) << In.takeError();
261     ASSERT_TRUE(In->Symbols);
262     ASSERT_TRUE(In->Refs);
263     ASSERT_TRUE(In->Sources);
264     ASSERT_TRUE(In->Sources->count(IGN.URI));
265     // Assert the YAML serializations match, for nice comparisons and diffs.
266     EXPECT_THAT(yamlFromSymbols(*In->Symbols),
267                 UnorderedElementsAreArray(yamlFromSymbols(*In->Symbols)));
268     EXPECT_THAT(yamlFromRefs(*In->Refs),
269                 UnorderedElementsAreArray(yamlFromRefs(*In->Refs)));
270     auto IGNDeserialized = In->Sources->lookup(IGN.URI);
271     EXPECT_EQ(IGNDeserialized.Digest, IGN.Digest);
272     EXPECT_EQ(IGNDeserialized.DirectIncludes, IGN.DirectIncludes);
273     EXPECT_EQ(IGNDeserialized.URI, IGN.URI);
274     EXPECT_EQ(IGNDeserialized.Flags, IGN.Flags);
275   }
276 }
277 
278 TEST(SerializationTest, CmdlTest) {
279   auto In = readIndexFile(YAML);
280   EXPECT_TRUE(bool(In)) << In.takeError();
281 
282   tooling::CompileCommand Cmd;
283   Cmd.Directory = "testdir";
284   Cmd.CommandLine.push_back("cmd1");
285   Cmd.CommandLine.push_back("cmd2");
286   Cmd.Filename = "ignored";
287   Cmd.Heuristic = "ignored";
288   Cmd.Output = "ignored";
289 
290   IndexFileOut Out(*In);
291   Out.Format = IndexFileFormat::RIFF;
292   Out.Cmd = &Cmd;
293   {
294     std::string Serialized = llvm::to_string(Out);
295 
296     auto In = readIndexFile(Serialized);
297     ASSERT_TRUE(bool(In)) << In.takeError();
298     ASSERT_TRUE(In->Cmd);
299 
300     const tooling::CompileCommand &SerializedCmd = *In->Cmd;
301     EXPECT_EQ(SerializedCmd.CommandLine, Cmd.CommandLine);
302     EXPECT_EQ(SerializedCmd.Directory, Cmd.Directory);
303     EXPECT_NE(SerializedCmd.Filename, Cmd.Filename);
304     EXPECT_NE(SerializedCmd.Heuristic, Cmd.Heuristic);
305     EXPECT_NE(SerializedCmd.Output, Cmd.Output);
306   }
307 }
308 
309 // rlimit is part of POSIX. RLIMIT_AS does not exist in OpenBSD.
310 // Sanitizers use a lot of address space, so we can't apply strict limits.
311 #if LLVM_ON_UNIX && defined(RLIMIT_AS) && !LLVM_ADDRESS_SANITIZER_BUILD &&     \
312     !LLVM_MEMORY_SANITIZER_BUILD
313 class ScopedMemoryLimit {
314   struct rlimit OriginalLimit;
315   bool Succeeded = false;
316 
317 public:
318   ScopedMemoryLimit(rlim_t Bytes) {
319     if (!getrlimit(RLIMIT_AS, &OriginalLimit)) {
320       struct rlimit NewLimit = OriginalLimit;
321       NewLimit.rlim_cur = Bytes;
322       Succeeded = !setrlimit(RLIMIT_AS, &NewLimit);
323     }
324     if (!Succeeded)
325       log("Failed to set rlimit");
326   }
327 
328   ~ScopedMemoryLimit() {
329     if (Succeeded)
330       setrlimit(RLIMIT_AS, &OriginalLimit);
331   }
332 };
333 #else
334 class ScopedMemoryLimit {
335 public:
336   ScopedMemoryLimit(unsigned Bytes) { log("rlimit unsupported"); }
337 };
338 #endif
339 
340 // Test that our deserialization detects invalid array sizes without allocating.
341 // If this detection fails, the test should allocate a huge array and crash.
342 TEST(SerializationTest, NoCrashOnBadArraySize) {
343   // This test is tricky because we need to construct a subtly invalid file.
344   // First, create a valid serialized file.
345   auto In = readIndexFile(YAML);
346   ASSERT_FALSE(!In) << In.takeError();
347   IndexFileOut Out(*In);
348   Out.Format = IndexFileFormat::RIFF;
349   std::string Serialized = llvm::to_string(Out);
350 
351   // Low-level parse it again and find the `srcs` chunk we're going to corrupt.
352   auto Parsed = riff::readFile(Serialized);
353   ASSERT_FALSE(!Parsed) << Parsed.takeError();
354   auto Srcs = llvm::find_if(Parsed->Chunks, [](riff::Chunk C) {
355     return C.ID == riff::fourCC("srcs");
356   });
357   ASSERT_NE(Srcs, Parsed->Chunks.end());
358 
359   // Srcs consists of a sequence of IncludeGraphNodes. In our case, just one.
360   // The node has:
361   //  - 1 byte: flags (1)
362   //  - varint(stringID): URI
363   //  - 8 byte: file digest
364   //  - varint: DirectIncludes.length
365   //  - repeated varint(stringID): DirectIncludes
366   // We want to set DirectIncludes.length to a huge number.
367   // The offset isn't trivial to find, so we use the file digest.
368   std::string FileDigest = llvm::fromHex("EED8F5EAF25C453C");
369   unsigned Pos = Srcs->Data.find_first_of(FileDigest);
370   ASSERT_NE(Pos, StringRef::npos) << "Couldn't locate file digest";
371   Pos += FileDigest.size();
372 
373   // Varints are little-endian base-128 numbers, where the top-bit of each byte
374   // indicates whether there are more. ffffffff0f -> 0xffffffff.
375   std::string CorruptSrcs =
376       (Srcs->Data.take_front(Pos) + llvm::fromHex("ffffffff0f") +
377        "some_random_garbage")
378           .str();
379   Srcs->Data = CorruptSrcs;
380 
381   // Try to crash rather than hang on large allocation.
382   ScopedMemoryLimit MemLimit(1000 * 1024 * 1024); // 1GB
383 
384   std::string CorruptFile = llvm::to_string(*Parsed);
385   auto CorruptParsed = readIndexFile(CorruptFile);
386   ASSERT_TRUE(!CorruptParsed);
387   EXPECT_EQ(llvm::toString(CorruptParsed.takeError()),
388             "malformed or truncated include uri");
389 }
390 
391 // Check we detect invalid string table size size without allocating it first.
392 // If this detection fails, the test should allocate a huge array and crash.
393 TEST(SerializationTest, NoCrashOnBadStringTableSize) {
394   if (!llvm::compression::zlib::isAvailable()) {
395     log("skipping test, no zlib");
396     return;
397   }
398 
399   // First, create a valid serialized file.
400   auto In = readIndexFile(YAML);
401   ASSERT_FALSE(!In) << In.takeError();
402   IndexFileOut Out(*In);
403   Out.Format = IndexFileFormat::RIFF;
404   std::string Serialized = llvm::to_string(Out);
405 
406   // Low-level parse it again, we're going to replace the `stri` chunk.
407   auto Parsed = riff::readFile(Serialized);
408   ASSERT_FALSE(!Parsed) << Parsed.takeError();
409   auto Stri = llvm::find_if(Parsed->Chunks, [](riff::Chunk C) {
410     return C.ID == riff::fourCC("stri");
411   });
412   ASSERT_NE(Stri, Parsed->Chunks.end());
413 
414   // stri consists of an 8 byte uncompressed-size, and then compressed data.
415   // We'll claim our small amount of data expands to 4GB
416   std::string CorruptStri =
417       (llvm::fromHex("ffffffff") + Stri->Data.drop_front(4)).str();
418   Stri->Data = CorruptStri;
419   std::string FileDigest = llvm::fromHex("EED8F5EAF25C453C");
420 
421   // Try to crash rather than hang on large allocation.
422   ScopedMemoryLimit MemLimit(1000 * 1024 * 1024); // 1GB
423 
424   std::string CorruptFile = llvm::to_string(*Parsed);
425   auto CorruptParsed = readIndexFile(CorruptFile);
426   ASSERT_TRUE(!CorruptParsed);
427   EXPECT_THAT(llvm::toString(CorruptParsed.takeError()),
428               testing::HasSubstr("bytes is implausible"));
429 }
430 
431 } // namespace
432 } // namespace clangd
433 } // namespace clang
434