1 //===--- QueryDriverDatabase.cpp ---------------------------------*- C++-*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 // Some compiler drivers have implicit search mechanism for system headers.
9 // This compilation database implementation tries to extract that information by
10 // executing the driver in verbose mode. gcc-compatible drivers print something
11 // like:
12 // ....
13 // ....
14 // #include <...> search starts here:
15 //  /usr/lib/gcc/x86_64-linux-gnu/7/include
16 //  /usr/local/include
17 //  /usr/lib/gcc/x86_64-linux-gnu/7/include-fixed
18 //  /usr/include/x86_64-linux-gnu
19 //  /usr/include
20 // End of search list.
21 // ....
22 // ....
23 // This component parses that output and adds each path to command line args
24 // provided by Base, after prepending them with -isystem. Therefore current
25 // implementation would not work with a driver that is not gcc-compatible.
26 //
27 // First argument of the command line received from underlying compilation
28 // database is used as compiler driver path. Due to this arbitrary binary
29 // execution, this mechanism is not used by default and only executes binaries
30 // in the paths that are explicitly included by the user.
31 
32 #include "GlobalCompilationDatabase.h"
33 #include "support/Logger.h"
34 #include "support/Path.h"
35 #include "support/Trace.h"
36 #include "clang/Basic/Diagnostic.h"
37 #include "clang/Basic/TargetInfo.h"
38 #include "clang/Basic/TargetOptions.h"
39 #include "clang/Driver/Types.h"
40 #include "clang/Tooling/CompilationDatabase.h"
41 #include "llvm/ADT/ScopeExit.h"
42 #include "llvm/ADT/SmallString.h"
43 #include "llvm/ADT/StringExtras.h"
44 #include "llvm/ADT/StringRef.h"
45 #include "llvm/Support/FileSystem.h"
46 #include "llvm/Support/MemoryBuffer.h"
47 #include "llvm/Support/Path.h"
48 #include "llvm/Support/Program.h"
49 #include "llvm/Support/Regex.h"
50 #include "llvm/Support/ScopedPrinter.h"
51 #include <algorithm>
52 #include <map>
53 #include <string>
54 #include <vector>
55 
56 namespace clang {
57 namespace clangd {
58 namespace {
59 
60 struct DriverInfo {
61   std::vector<std::string> SystemIncludes;
62   std::string Target;
63 };
64 
65 bool isValidTarget(llvm::StringRef Triple) {
66   std::shared_ptr<TargetOptions> TargetOpts(new TargetOptions);
67   TargetOpts->Triple = Triple.str();
68   DiagnosticsEngine Diags(new DiagnosticIDs, new DiagnosticOptions,
69                           new IgnoringDiagConsumer);
70   IntrusiveRefCntPtr<TargetInfo> Target =
71       TargetInfo::CreateTargetInfo(Diags, TargetOpts);
72   return bool(Target);
73 }
74 
75 llvm::Optional<DriverInfo> parseDriverOutput(llvm::StringRef Output) {
76   DriverInfo Info;
77   const char SIS[] = "#include <...> search starts here:";
78   const char SIE[] = "End of search list.";
79   const char TS[] = "Target: ";
80   llvm::SmallVector<llvm::StringRef> Lines;
81   Output.split(Lines, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
82 
83   enum {
84     Initial,            // Initial state: searching for target or includes list.
85     IncludesExtracting, // Includes extracting.
86     Done                // Includes and target extraction done.
87   } State = Initial;
88   bool SeenIncludes = false;
89   bool SeenTarget = false;
90   for (auto *It = Lines.begin(); State != Done && It != Lines.end(); ++It) {
91     auto Line = *It;
92     switch (State) {
93     case Initial:
94       if (!SeenIncludes && Line.trim() == SIS) {
95         SeenIncludes = true;
96         State = IncludesExtracting;
97       } else if (!SeenTarget && Line.trim().startswith(TS)) {
98         SeenTarget = true;
99         llvm::StringRef TargetLine = Line.trim();
100         TargetLine.consume_front(TS);
101         // Only detect targets that clang understands
102         if (!isValidTarget(TargetLine)) {
103           elog("System include extraction: invalid target \"{0}\", ignoring",
104                TargetLine);
105         } else {
106           Info.Target = TargetLine.str();
107           vlog("System include extraction: target extracted: \"{0}\"",
108                TargetLine);
109         }
110       }
111       break;
112     case IncludesExtracting:
113       if (Line.trim() == SIE) {
114         State = SeenTarget ? Done : Initial;
115       } else {
116         Info.SystemIncludes.push_back(Line.trim().str());
117         vlog("System include extraction: adding {0}", Line);
118       }
119       break;
120     default:
121       llvm_unreachable("Impossible state of the driver output parser");
122       break;
123     }
124   }
125   if (!SeenIncludes) {
126     elog("System include extraction: start marker not found: {0}", Output);
127     return llvm::None;
128   }
129   if (State == IncludesExtracting) {
130     elog("System include extraction: end marker missing: {0}", Output);
131     return llvm::None;
132   }
133   return std::move(Info);
134 }
135 
136 llvm::Optional<DriverInfo>
137 extractSystemIncludesAndTarget(llvm::SmallString<128> Driver,
138                                llvm::StringRef Lang,
139                                llvm::ArrayRef<std::string> CommandLine,
140                                const llvm::Regex &QueryDriverRegex) {
141   trace::Span Tracer("Extract system includes and target");
142 
143   if (!llvm::sys::path::is_absolute(Driver)) {
144     assert(llvm::none_of(
145         Driver, [](char C) { return llvm::sys::path::is_separator(C); }));
146     auto DriverProgram = llvm::sys::findProgramByName(Driver);
147     if (DriverProgram) {
148       vlog("System include extraction: driver {0} expanded to {1}", Driver,
149            *DriverProgram);
150       Driver = *DriverProgram;
151     } else {
152       elog("System include extraction: driver {0} not found in PATH", Driver);
153       return llvm::None;
154     }
155   }
156 
157   SPAN_ATTACH(Tracer, "driver", Driver);
158   SPAN_ATTACH(Tracer, "lang", Lang);
159 
160   if (!QueryDriverRegex.match(Driver)) {
161     vlog("System include extraction: not allowed driver {0}", Driver);
162     return llvm::None;
163   }
164 
165   llvm::SmallString<128> StdErrPath;
166   if (auto EC = llvm::sys::fs::createTemporaryFile("system-includes", "clangd",
167                                                    StdErrPath)) {
168     elog("System include extraction: failed to create temporary file with "
169          "error {0}",
170          EC.message());
171     return llvm::None;
172   }
173   auto CleanUp = llvm::make_scope_exit(
174       [&StdErrPath]() { llvm::sys::fs::remove(StdErrPath); });
175 
176   llvm::Optional<llvm::StringRef> Redirects[] = {{""}, {""}, StdErrPath.str()};
177 
178   llvm::SmallVector<llvm::StringRef> Args = {Driver, "-E", "-x",
179                                              Lang,   "-",  "-v"};
180 
181   // These flags will be preserved
182   const llvm::StringRef FlagsToPreserve[] = {
183       "-nostdinc", "--no-standard-includes", "-nostdinc++", "-nobuiltininc"};
184   // Preserves these flags and their values, either as separate args or with an
185   // equalsbetween them
186   const llvm::StringRef ArgsToPreserve[] = {"--sysroot", "-isysroot"};
187 
188   for (size_t I = 0, E = CommandLine.size(); I < E; ++I) {
189     llvm::StringRef Arg = CommandLine[I];
190     if (llvm::any_of(FlagsToPreserve,
191                      [&Arg](llvm::StringRef S) { return S == Arg; })) {
192       Args.push_back(Arg);
193     } else {
194       const auto *Found =
195           llvm::find_if(ArgsToPreserve, [&Arg](llvm::StringRef S) {
196             return Arg.startswith(S);
197           });
198       if (Found == std::end(ArgsToPreserve))
199         continue;
200       Arg = Arg.drop_front(Found->size());
201       if (Arg.empty() && I + 1 < E) {
202         Args.push_back(CommandLine[I]);
203         Args.push_back(CommandLine[++I]);
204       } else if (Arg.startswith("=")) {
205         Args.push_back(CommandLine[I]);
206       }
207     }
208   }
209 
210   std::string ErrMsg;
211   if (int RC = llvm::sys::ExecuteAndWait(Driver, Args, /*Env=*/llvm::None,
212                                          Redirects, /*SecondsToWait=*/0,
213                                          /*MemoryLimit=*/0, &ErrMsg)) {
214     elog("System include extraction: driver execution failed with return code: "
215          "{0} - '{1}'. Args: [{2}]",
216          llvm::to_string(RC), ErrMsg, printArgv(Args));
217     return llvm::None;
218   }
219 
220   auto BufOrError = llvm::MemoryBuffer::getFile(StdErrPath);
221   if (!BufOrError) {
222     elog("System include extraction: failed to read {0} with error {1}",
223          StdErrPath, BufOrError.getError().message());
224     return llvm::None;
225   }
226 
227   llvm::Optional<DriverInfo> Info =
228       parseDriverOutput(BufOrError->get()->getBuffer());
229   if (!Info)
230     return llvm::None;
231   log("System includes extractor: successfully executed {0}\n\tgot includes: "
232       "\"{1}\"\n\tgot target: \"{2}\"",
233       Driver, llvm::join(Info->SystemIncludes, ", "), Info->Target);
234   return Info;
235 }
236 
237 tooling::CompileCommand &
238 addSystemIncludes(tooling::CompileCommand &Cmd,
239                   llvm::ArrayRef<std::string> SystemIncludes) {
240   for (llvm::StringRef Include : SystemIncludes) {
241     // FIXME(kadircet): This doesn't work when we have "--driver-mode=cl"
242     Cmd.CommandLine.push_back("-isystem");
243     Cmd.CommandLine.push_back(Include.str());
244   }
245   return Cmd;
246 }
247 
248 tooling::CompileCommand &setTarget(tooling::CompileCommand &Cmd,
249                                    const std::string &Target) {
250   if (!Target.empty()) {
251     // We do not want to override existing target with extracted one.
252     for (llvm::StringRef Arg : Cmd.CommandLine) {
253       if (Arg == "-target" || Arg.startswith("--target="))
254         return Cmd;
255     }
256     Cmd.CommandLine.push_back("--target=" + Target);
257   }
258   return Cmd;
259 }
260 
261 /// Converts a glob containing only ** or * into a regex.
262 std::string convertGlobToRegex(llvm::StringRef Glob) {
263   std::string RegText;
264   llvm::raw_string_ostream RegStream(RegText);
265   RegStream << '^';
266   for (size_t I = 0, E = Glob.size(); I < E; ++I) {
267     if (Glob[I] == '*') {
268       if (I + 1 < E && Glob[I + 1] == '*') {
269         // Double star, accept any sequence.
270         RegStream << ".*";
271         // Also skip the second star.
272         ++I;
273       } else {
274         // Single star, accept any sequence without a slash.
275         RegStream << "[^/]*";
276       }
277     } else if (llvm::sys::path::is_separator(Glob[I]) &&
278                llvm::sys::path::is_separator('/') &&
279                llvm::sys::path::is_separator('\\')) {
280       RegStream << R"([/\\])"; // Accept either slash on windows.
281     } else {
282       RegStream << llvm::Regex::escape(Glob.substr(I, 1));
283     }
284   }
285   RegStream << '$';
286   RegStream.flush();
287   return RegText;
288 }
289 
290 /// Converts a glob containing only ** or * into a regex.
291 llvm::Regex convertGlobsToRegex(llvm::ArrayRef<std::string> Globs) {
292   assert(!Globs.empty() && "Globs cannot be empty!");
293   std::vector<std::string> RegTexts;
294   RegTexts.reserve(Globs.size());
295   for (llvm::StringRef Glob : Globs)
296     RegTexts.push_back(convertGlobToRegex(Glob));
297 
298   // Tempting to pass IgnoreCase, but we don't know the FS sensitivity.
299   llvm::Regex Reg(llvm::join(RegTexts, "|"));
300   assert(Reg.isValid(RegTexts.front()) &&
301          "Created an invalid regex from globs");
302   return Reg;
303 }
304 
305 /// Extracts system includes from a trusted driver by parsing the output of
306 /// include search path and appends them to the commands coming from underlying
307 /// compilation database.
308 class QueryDriverDatabase : public DelegatingCDB {
309 public:
310   QueryDriverDatabase(llvm::ArrayRef<std::string> QueryDriverGlobs,
311                       std::unique_ptr<GlobalCompilationDatabase> Base)
312       : DelegatingCDB(std::move(Base)),
313         QueryDriverRegex(convertGlobsToRegex(QueryDriverGlobs)) {}
314 
315   llvm::Optional<tooling::CompileCommand>
316   getCompileCommand(PathRef File) const override {
317     auto Cmd = DelegatingCDB::getCompileCommand(File);
318     if (!Cmd || Cmd->CommandLine.empty())
319       return Cmd;
320 
321     llvm::StringRef Lang;
322     for (size_t I = 0, E = Cmd->CommandLine.size(); I < E; ++I) {
323       llvm::StringRef Arg = Cmd->CommandLine[I];
324       if (Arg == "-x" && I + 1 < E)
325         Lang = Cmd->CommandLine[I + 1];
326       else if (Arg.startswith("-x"))
327         Lang = Arg.drop_front(2).trim();
328     }
329     if (Lang.empty()) {
330       llvm::StringRef Ext = llvm::sys::path::extension(File).trim('.');
331       auto Type = driver::types::lookupTypeForExtension(Ext);
332       if (Type == driver::types::TY_INVALID) {
333         elog("System include extraction: invalid file type for {0}", Ext);
334         return {};
335       }
336       Lang = driver::types::getTypeName(Type);
337     }
338 
339     llvm::SmallString<128> Driver(Cmd->CommandLine.front());
340     if (llvm::any_of(Driver,
341                        [](char C) { return llvm::sys::path::is_separator(C); }))
342       // Driver is a not a single executable name but instead a path (either
343       // relative or absolute).
344       llvm::sys::fs::make_absolute(Cmd->Directory, Driver);
345 
346     if (auto Info =
347             QueriedDrivers.get(/*Key=*/(Driver + ":" + Lang).str(), [&] {
348               return extractSystemIncludesAndTarget(
349                   Driver, Lang, Cmd->CommandLine, QueryDriverRegex);
350             })) {
351       setTarget(addSystemIncludes(*Cmd, Info->SystemIncludes), Info->Target);
352     }
353     return Cmd;
354   }
355 
356 private:
357   // Caches includes extracted from a driver. Key is driver:lang.
358   Memoize<llvm::StringMap<llvm::Optional<DriverInfo>>> QueriedDrivers;
359   llvm::Regex QueryDriverRegex;
360 };
361 } // namespace
362 
363 std::unique_ptr<GlobalCompilationDatabase>
364 getQueryDriverDatabase(llvm::ArrayRef<std::string> QueryDriverGlobs,
365                        std::unique_ptr<GlobalCompilationDatabase> Base) {
366   assert(Base && "Null base to SystemIncludeExtractor");
367   if (QueryDriverGlobs.empty())
368     return Base;
369   return std::make_unique<QueryDriverDatabase>(QueryDriverGlobs,
370                                                std::move(Base));
371 }
372 
373 } // namespace clangd
374 } // namespace clang
375