1 //===- JSONCompilationDatabase.cpp ----------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 //  This file contains the implementation of the JSONCompilationDatabase.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "clang/Tooling/JSONCompilationDatabase.h"
14 #include "clang/Basic/LLVM.h"
15 #include "clang/Tooling/CompilationDatabase.h"
16 #include "clang/Tooling/CompilationDatabasePluginRegistry.h"
17 #include "llvm/ADT/Optional.h"
18 #include "llvm/ADT/SmallString.h"
19 #include "llvm/ADT/SmallVector.h"
20 #include "llvm/ADT/StringRef.h"
21 #include "llvm/ADT/Triple.h"
22 #include "llvm/Support/Allocator.h"
23 #include "llvm/Support/Casting.h"
24 #include "llvm/Support/CommandLine.h"
25 #include "llvm/Support/ErrorOr.h"
26 #include "llvm/Support/Host.h"
27 #include "llvm/Support/MemoryBuffer.h"
28 #include "llvm/Support/Path.h"
29 #include "llvm/Support/StringSaver.h"
30 #include "llvm/Support/YAMLParser.h"
31 #include "llvm/Support/raw_ostream.h"
32 #include <cassert>
33 #include <memory>
34 #include <string>
35 #include <system_error>
36 #include <tuple>
37 #include <utility>
38 #include <vector>
39 
40 using namespace clang;
41 using namespace tooling;
42 
43 namespace {
44 
45 /// A parser for escaped strings of command line arguments.
46 ///
47 /// Assumes \-escaping for quoted arguments (see the documentation of
48 /// unescapeCommandLine(...)).
49 class CommandLineArgumentParser {
50  public:
51   CommandLineArgumentParser(StringRef CommandLine)
52       : Input(CommandLine), Position(Input.begin()-1) {}
53 
54   std::vector<std::string> parse() {
55     bool HasMoreInput = true;
56     while (HasMoreInput && nextNonWhitespace()) {
57       std::string Argument;
58       HasMoreInput = parseStringInto(Argument);
59       CommandLine.push_back(Argument);
60     }
61     return CommandLine;
62   }
63 
64  private:
65   // All private methods return true if there is more input available.
66 
67   bool parseStringInto(std::string &String) {
68     do {
69       if (*Position == '"') {
70         if (!parseDoubleQuotedStringInto(String)) return false;
71       } else if (*Position == '\'') {
72         if (!parseSingleQuotedStringInto(String)) return false;
73       } else {
74         if (!parseFreeStringInto(String)) return false;
75       }
76     } while (*Position != ' ');
77     return true;
78   }
79 
80   bool parseDoubleQuotedStringInto(std::string &String) {
81     if (!next()) return false;
82     while (*Position != '"') {
83       if (!skipEscapeCharacter()) return false;
84       String.push_back(*Position);
85       if (!next()) return false;
86     }
87     return next();
88   }
89 
90   bool parseSingleQuotedStringInto(std::string &String) {
91     if (!next()) return false;
92     while (*Position != '\'') {
93       String.push_back(*Position);
94       if (!next()) return false;
95     }
96     return next();
97   }
98 
99   bool parseFreeStringInto(std::string &String) {
100     do {
101       if (!skipEscapeCharacter()) return false;
102       String.push_back(*Position);
103       if (!next()) return false;
104     } while (*Position != ' ' && *Position != '"' && *Position != '\'');
105     return true;
106   }
107 
108   bool skipEscapeCharacter() {
109     if (*Position == '\\') {
110       return next();
111     }
112     return true;
113   }
114 
115   bool nextNonWhitespace() {
116     do {
117       if (!next()) return false;
118     } while (*Position == ' ');
119     return true;
120   }
121 
122   bool next() {
123     ++Position;
124     return Position != Input.end();
125   }
126 
127   const StringRef Input;
128   StringRef::iterator Position;
129   std::vector<std::string> CommandLine;
130 };
131 
132 std::vector<std::string> unescapeCommandLine(JSONCommandLineSyntax Syntax,
133                                              StringRef EscapedCommandLine) {
134   if (Syntax == JSONCommandLineSyntax::AutoDetect) {
135     Syntax = JSONCommandLineSyntax::Gnu;
136     llvm::Triple Triple(llvm::sys::getProcessTriple());
137     if (Triple.getOS() == llvm::Triple::OSType::Win32) {
138       // Assume Windows command line parsing on Win32 unless the triple
139       // explicitly tells us otherwise.
140       if (!Triple.hasEnvironment() ||
141           Triple.getEnvironment() == llvm::Triple::EnvironmentType::MSVC)
142         Syntax = JSONCommandLineSyntax::Windows;
143     }
144   }
145 
146   if (Syntax == JSONCommandLineSyntax::Windows) {
147     llvm::BumpPtrAllocator Alloc;
148     llvm::StringSaver Saver(Alloc);
149     llvm::SmallVector<const char *, 64> T;
150     llvm::cl::TokenizeWindowsCommandLine(EscapedCommandLine, Saver, T);
151     std::vector<std::string> Result(T.begin(), T.end());
152     return Result;
153   }
154   assert(Syntax == JSONCommandLineSyntax::Gnu);
155   CommandLineArgumentParser parser(EscapedCommandLine);
156   return parser.parse();
157 }
158 
159 // This plugin locates a nearby compile_command.json file, and also infers
160 // compile commands for files not present in the database.
161 class JSONCompilationDatabasePlugin : public CompilationDatabasePlugin {
162   std::unique_ptr<CompilationDatabase>
163   loadFromDirectory(StringRef Directory, std::string &ErrorMessage) override {
164     SmallString<1024> JSONDatabasePath(Directory);
165     llvm::sys::path::append(JSONDatabasePath, "compile_commands.json");
166     auto Base = JSONCompilationDatabase::loadFromFile(
167         JSONDatabasePath, ErrorMessage, JSONCommandLineSyntax::AutoDetect);
168     return Base ? inferMissingCompileCommands(std::move(Base)) : nullptr;
169   }
170 };
171 
172 } // namespace
173 
174 // Register the JSONCompilationDatabasePlugin with the
175 // CompilationDatabasePluginRegistry using this statically initialized variable.
176 static CompilationDatabasePluginRegistry::Add<JSONCompilationDatabasePlugin>
177 X("json-compilation-database", "Reads JSON formatted compilation databases");
178 
179 namespace clang {
180 namespace tooling {
181 
182 // This anchor is used to force the linker to link in the generated object file
183 // and thus register the JSONCompilationDatabasePlugin.
184 volatile int JSONAnchorSource = 0;
185 
186 } // namespace tooling
187 } // namespace clang
188 
189 std::unique_ptr<JSONCompilationDatabase>
190 JSONCompilationDatabase::loadFromFile(StringRef FilePath,
191                                       std::string &ErrorMessage,
192                                       JSONCommandLineSyntax Syntax) {
193   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> DatabaseBuffer =
194       llvm::MemoryBuffer::getFile(FilePath);
195   if (std::error_code Result = DatabaseBuffer.getError()) {
196     ErrorMessage = "Error while opening JSON database: " + Result.message();
197     return nullptr;
198   }
199   std::unique_ptr<JSONCompilationDatabase> Database(
200       new JSONCompilationDatabase(std::move(*DatabaseBuffer), Syntax));
201   if (!Database->parse(ErrorMessage))
202     return nullptr;
203   return Database;
204 }
205 
206 std::unique_ptr<JSONCompilationDatabase>
207 JSONCompilationDatabase::loadFromBuffer(StringRef DatabaseString,
208                                         std::string &ErrorMessage,
209                                         JSONCommandLineSyntax Syntax) {
210   std::unique_ptr<llvm::MemoryBuffer> DatabaseBuffer(
211       llvm::MemoryBuffer::getMemBuffer(DatabaseString));
212   std::unique_ptr<JSONCompilationDatabase> Database(
213       new JSONCompilationDatabase(std::move(DatabaseBuffer), Syntax));
214   if (!Database->parse(ErrorMessage))
215     return nullptr;
216   return Database;
217 }
218 
219 std::vector<CompileCommand>
220 JSONCompilationDatabase::getCompileCommands(StringRef FilePath) const {
221   SmallString<128> NativeFilePath;
222   llvm::sys::path::native(FilePath, NativeFilePath);
223 
224   std::string Error;
225   llvm::raw_string_ostream ES(Error);
226   StringRef Match = MatchTrie.findEquivalent(NativeFilePath, ES);
227   if (Match.empty())
228     return {};
229   const auto CommandsRefI = IndexByFile.find(Match);
230   if (CommandsRefI == IndexByFile.end())
231     return {};
232   std::vector<CompileCommand> Commands;
233   getCommands(CommandsRefI->getValue(), Commands);
234   return Commands;
235 }
236 
237 std::vector<std::string>
238 JSONCompilationDatabase::getAllFiles() const {
239   std::vector<std::string> Result;
240   for (const auto &CommandRef : IndexByFile)
241     Result.push_back(CommandRef.first().str());
242   return Result;
243 }
244 
245 std::vector<CompileCommand>
246 JSONCompilationDatabase::getAllCompileCommands() const {
247   std::vector<CompileCommand> Commands;
248   getCommands(AllCommands, Commands);
249   return Commands;
250 }
251 
252 static std::vector<std::string>
253 nodeToCommandLine(JSONCommandLineSyntax Syntax,
254                   const std::vector<llvm::yaml::ScalarNode *> &Nodes) {
255   SmallString<1024> Storage;
256   if (Nodes.size() == 1)
257     return unescapeCommandLine(Syntax, Nodes[0]->getValue(Storage));
258   std::vector<std::string> Arguments;
259   for (const auto *Node : Nodes)
260     Arguments.push_back(Node->getValue(Storage));
261   return Arguments;
262 }
263 
264 void JSONCompilationDatabase::getCommands(
265     ArrayRef<CompileCommandRef> CommandsRef,
266     std::vector<CompileCommand> &Commands) const {
267   for (const auto &CommandRef : CommandsRef) {
268     SmallString<8> DirectoryStorage;
269     SmallString<32> FilenameStorage;
270     SmallString<32> OutputStorage;
271     auto Output = std::get<3>(CommandRef);
272     Commands.emplace_back(
273         std::get<0>(CommandRef)->getValue(DirectoryStorage),
274         std::get<1>(CommandRef)->getValue(FilenameStorage),
275         nodeToCommandLine(Syntax, std::get<2>(CommandRef)),
276         Output ? Output->getValue(OutputStorage) : "");
277   }
278 }
279 
280 bool JSONCompilationDatabase::parse(std::string &ErrorMessage) {
281   llvm::yaml::document_iterator I = YAMLStream.begin();
282   if (I == YAMLStream.end()) {
283     ErrorMessage = "Error while parsing YAML.";
284     return false;
285   }
286   llvm::yaml::Node *Root = I->getRoot();
287   if (!Root) {
288     ErrorMessage = "Error while parsing YAML.";
289     return false;
290   }
291   auto *Array = dyn_cast<llvm::yaml::SequenceNode>(Root);
292   if (!Array) {
293     ErrorMessage = "Expected array.";
294     return false;
295   }
296   for (auto &NextObject : *Array) {
297     auto *Object = dyn_cast<llvm::yaml::MappingNode>(&NextObject);
298     if (!Object) {
299       ErrorMessage = "Expected object.";
300       return false;
301     }
302     llvm::yaml::ScalarNode *Directory = nullptr;
303     llvm::Optional<std::vector<llvm::yaml::ScalarNode *>> Command;
304     llvm::yaml::ScalarNode *File = nullptr;
305     llvm::yaml::ScalarNode *Output = nullptr;
306     for (auto& NextKeyValue : *Object) {
307       auto *KeyString = dyn_cast<llvm::yaml::ScalarNode>(NextKeyValue.getKey());
308       if (!KeyString) {
309         ErrorMessage = "Expected strings as key.";
310         return false;
311       }
312       SmallString<10> KeyStorage;
313       StringRef KeyValue = KeyString->getValue(KeyStorage);
314       llvm::yaml::Node *Value = NextKeyValue.getValue();
315       if (!Value) {
316         ErrorMessage = "Expected value.";
317         return false;
318       }
319       auto *ValueString = dyn_cast<llvm::yaml::ScalarNode>(Value);
320       auto *SequenceString = dyn_cast<llvm::yaml::SequenceNode>(Value);
321       if (KeyValue == "arguments" && !SequenceString) {
322         ErrorMessage = "Expected sequence as value.";
323         return false;
324       } else if (KeyValue != "arguments" && !ValueString) {
325         ErrorMessage = "Expected string as value.";
326         return false;
327       }
328       if (KeyValue == "directory") {
329         Directory = ValueString;
330       } else if (KeyValue == "arguments") {
331         Command = std::vector<llvm::yaml::ScalarNode *>();
332         for (auto &Argument : *SequenceString) {
333           auto *Scalar = dyn_cast<llvm::yaml::ScalarNode>(&Argument);
334           if (!Scalar) {
335             ErrorMessage = "Only strings are allowed in 'arguments'.";
336             return false;
337           }
338           Command->push_back(Scalar);
339         }
340       } else if (KeyValue == "command") {
341         if (!Command)
342           Command = std::vector<llvm::yaml::ScalarNode *>(1, ValueString);
343       } else if (KeyValue == "file") {
344         File = ValueString;
345       } else if (KeyValue == "output") {
346         Output = ValueString;
347       } else {
348         ErrorMessage = ("Unknown key: \"" +
349                         KeyString->getRawValue() + "\"").str();
350         return false;
351       }
352     }
353     if (!File) {
354       ErrorMessage = "Missing key: \"file\".";
355       return false;
356     }
357     if (!Command) {
358       ErrorMessage = "Missing key: \"command\" or \"arguments\".";
359       return false;
360     }
361     if (!Directory) {
362       ErrorMessage = "Missing key: \"directory\".";
363       return false;
364     }
365     SmallString<8> FileStorage;
366     StringRef FileName = File->getValue(FileStorage);
367     SmallString<128> NativeFilePath;
368     if (llvm::sys::path::is_relative(FileName)) {
369       SmallString<8> DirectoryStorage;
370       SmallString<128> AbsolutePath(
371           Directory->getValue(DirectoryStorage));
372       llvm::sys::path::append(AbsolutePath, FileName);
373       llvm::sys::path::native(AbsolutePath, NativeFilePath);
374     } else {
375       llvm::sys::path::native(FileName, NativeFilePath);
376     }
377     auto Cmd = CompileCommandRef(Directory, File, *Command, Output);
378     IndexByFile[NativeFilePath].push_back(Cmd);
379     AllCommands.push_back(Cmd);
380     MatchTrie.insert(NativeFilePath);
381   }
382   return true;
383 }
384