1 //===--- QueryDriverDatabase.cpp ---------------------------------*- C++-*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 // Some compiler drivers have implicit search mechanism for system headers.
9 // This compilation database implementation tries to extract that information by
10 // executing the driver in verbose mode. gcc-compatible drivers print something
11 // like:
12 // ....
13 // ....
14 // #include <...> search starts here:
15 //  /usr/lib/gcc/x86_64-linux-gnu/7/include
16 //  /usr/local/include
17 //  /usr/lib/gcc/x86_64-linux-gnu/7/include-fixed
18 //  /usr/include/x86_64-linux-gnu
19 //  /usr/include
20 // End of search list.
21 // ....
22 // ....
23 // This component parses that output and adds each path to command line args
24 // provided by Base, after prepending them with -isystem. Therefore current
25 // implementation would not work with a driver that is not gcc-compatible.
26 //
27 // First argument of the command line received from underlying compilation
28 // database is used as compiler driver path. Due to this arbitrary binary
29 // execution, this mechanism is not used by default and only executes binaries
30 // in the paths that are explicitly included by the user.
31 
32 #include "GlobalCompilationDatabase.h"
33 #include "support/Logger.h"
34 #include "support/Path.h"
35 #include "support/Trace.h"
36 #include "clang/Basic/Diagnostic.h"
37 #include "clang/Basic/TargetInfo.h"
38 #include "clang/Basic/TargetOptions.h"
39 #include "clang/Driver/Types.h"
40 #include "clang/Tooling/CompilationDatabase.h"
41 #include "llvm/ADT/DenseMap.h"
42 #include "llvm/ADT/ScopeExit.h"
43 #include "llvm/ADT/SmallString.h"
44 #include "llvm/ADT/StringExtras.h"
45 #include "llvm/ADT/StringRef.h"
46 #include "llvm/ADT/iterator_range.h"
47 #include "llvm/Support/FileSystem.h"
48 #include "llvm/Support/MemoryBuffer.h"
49 #include "llvm/Support/Path.h"
50 #include "llvm/Support/Program.h"
51 #include "llvm/Support/Regex.h"
52 #include "llvm/Support/ScopedPrinter.h"
53 #include <algorithm>
54 #include <map>
55 #include <string>
56 #include <vector>
57 
58 namespace clang {
59 namespace clangd {
60 namespace {
61 
62 struct DriverInfo {
63   std::vector<std::string> SystemIncludes;
64   std::string Target;
65 };
66 
67 bool isValidTarget(llvm::StringRef Triple) {
68   std::shared_ptr<TargetOptions> TargetOpts(new TargetOptions);
69   TargetOpts->Triple = Triple.str();
70   DiagnosticsEngine Diags(new DiagnosticIDs, new DiagnosticOptions,
71                           new IgnoringDiagConsumer);
72   IntrusiveRefCntPtr<TargetInfo> Target =
73       TargetInfo::CreateTargetInfo(Diags, TargetOpts);
74   return bool(Target);
75 }
76 
77 llvm::Optional<DriverInfo> parseDriverOutput(llvm::StringRef Output) {
78   DriverInfo Info;
79   const char SIS[] = "#include <...> search starts here:";
80   const char SIE[] = "End of search list.";
81   const char TS[] = "Target: ";
82   llvm::SmallVector<llvm::StringRef> Lines;
83   Output.split(Lines, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
84 
85   enum {
86     Initial,            // Initial state: searching for target or includes list.
87     IncludesExtracting, // Includes extracting.
88     Done                // Includes and target extraction done.
89   } State = Initial;
90   bool SeenIncludes = false;
91   bool SeenTarget = false;
92   for (auto *It = Lines.begin(); State != Done && It != Lines.end(); ++It) {
93     auto Line = *It;
94     switch (State) {
95     case Initial:
96       if (!SeenIncludes && Line.trim() == SIS) {
97         SeenIncludes = true;
98         State = IncludesExtracting;
99       } else if (!SeenTarget && Line.trim().startswith(TS)) {
100         SeenTarget = true;
101         llvm::StringRef TargetLine = Line.trim();
102         TargetLine.consume_front(TS);
103         // Only detect targets that clang understands
104         if (!isValidTarget(TargetLine)) {
105           elog("System include extraction: invalid target \"{0}\", ignoring",
106                TargetLine);
107         } else {
108           Info.Target = TargetLine.str();
109           vlog("System include extraction: target extracted: \"{0}\"",
110                TargetLine);
111         }
112       }
113       break;
114     case IncludesExtracting:
115       if (Line.trim() == SIE) {
116         State = SeenTarget ? Done : Initial;
117       } else {
118         Info.SystemIncludes.push_back(Line.trim().str());
119         vlog("System include extraction: adding {0}", Line);
120       }
121       break;
122     default:
123       llvm_unreachable("Impossible state of the driver output parser");
124       break;
125     }
126   }
127   if (!SeenIncludes) {
128     elog("System include extraction: start marker not found: {0}", Output);
129     return llvm::None;
130   }
131   if (State == IncludesExtracting) {
132     elog("System include extraction: end marker missing: {0}", Output);
133     return llvm::None;
134   }
135   return std::move(Info);
136 }
137 
138 llvm::Optional<DriverInfo>
139 extractSystemIncludesAndTarget(llvm::SmallString<128> Driver,
140                                llvm::StringRef Lang,
141                                llvm::ArrayRef<std::string> CommandLine,
142                                const llvm::Regex &QueryDriverRegex) {
143   trace::Span Tracer("Extract system includes and target");
144 
145   if (!llvm::sys::path::is_absolute(Driver)) {
146     assert(llvm::none_of(
147         Driver, [](char C) { return llvm::sys::path::is_separator(C); }));
148     auto DriverProgram = llvm::sys::findProgramByName(Driver);
149     if (DriverProgram) {
150       vlog("System include extraction: driver {0} expanded to {1}", Driver,
151            *DriverProgram);
152       Driver = *DriverProgram;
153     } else {
154       elog("System include extraction: driver {0} not found in PATH", Driver);
155       return llvm::None;
156     }
157   }
158 
159   SPAN_ATTACH(Tracer, "driver", Driver);
160   SPAN_ATTACH(Tracer, "lang", Lang);
161 
162   if (!QueryDriverRegex.match(Driver)) {
163     vlog("System include extraction: not allowed driver {0}", Driver);
164     return llvm::None;
165   }
166 
167   if (!llvm::sys::fs::exists(Driver)) {
168     elog("System include extraction: {0} does not exist.", Driver);
169     return llvm::None;
170   }
171   if (!llvm::sys::fs::can_execute(Driver)) {
172     elog("System include extraction: {0} is not executable.", Driver);
173     return llvm::None;
174   }
175 
176   llvm::SmallString<128> StdErrPath;
177   if (auto EC = llvm::sys::fs::createTemporaryFile("system-includes", "clangd",
178                                                    StdErrPath)) {
179     elog("System include extraction: failed to create temporary file with "
180          "error {0}",
181          EC.message());
182     return llvm::None;
183   }
184   auto CleanUp = llvm::make_scope_exit(
185       [&StdErrPath]() { llvm::sys::fs::remove(StdErrPath); });
186 
187   llvm::Optional<llvm::StringRef> Redirects[] = {
188       {""}, {""}, llvm::StringRef(StdErrPath)};
189 
190   llvm::SmallVector<llvm::StringRef> Args = {Driver, "-E", "-x",
191                                              Lang,   "-",  "-v"};
192 
193   // These flags will be preserved
194   const llvm::StringRef FlagsToPreserve[] = {
195       "-nostdinc", "--no-standard-includes", "-nostdinc++", "-nobuiltininc"};
196   // Preserves these flags and their values, either as separate args or with an
197   // equalsbetween them
198   const llvm::StringRef ArgsToPreserve[] = {"--sysroot", "-isysroot"};
199 
200   for (size_t I = 0, E = CommandLine.size(); I < E; ++I) {
201     llvm::StringRef Arg = CommandLine[I];
202     if (llvm::any_of(FlagsToPreserve,
203                      [&Arg](llvm::StringRef S) { return S == Arg; })) {
204       Args.push_back(Arg);
205     } else {
206       const auto *Found =
207           llvm::find_if(ArgsToPreserve, [&Arg](llvm::StringRef S) {
208             return Arg.startswith(S);
209           });
210       if (Found == std::end(ArgsToPreserve))
211         continue;
212       Arg = Arg.drop_front(Found->size());
213       if (Arg.empty() && I + 1 < E) {
214         Args.push_back(CommandLine[I]);
215         Args.push_back(CommandLine[++I]);
216       } else if (Arg.startswith("=")) {
217         Args.push_back(CommandLine[I]);
218       }
219     }
220   }
221 
222   if (int RC = llvm::sys::ExecuteAndWait(Driver, Args, /*Env=*/llvm::None,
223                                          Redirects)) {
224     elog("System include extraction: driver execution failed with return code: "
225          "{0}. Args: [{1}]",
226          llvm::to_string(RC), printArgv(Args));
227     return llvm::None;
228   }
229 
230   auto BufOrError = llvm::MemoryBuffer::getFile(StdErrPath);
231   if (!BufOrError) {
232     elog("System include extraction: failed to read {0} with error {1}",
233          StdErrPath, BufOrError.getError().message());
234     return llvm::None;
235   }
236 
237   llvm::Optional<DriverInfo> Info =
238       parseDriverOutput(BufOrError->get()->getBuffer());
239   if (!Info)
240     return llvm::None;
241   log("System includes extractor: successfully executed {0}\n\tgot includes: "
242       "\"{1}\"\n\tgot target: \"{2}\"",
243       Driver, llvm::join(Info->SystemIncludes, ", "), Info->Target);
244   return Info;
245 }
246 
247 tooling::CompileCommand &
248 addSystemIncludes(tooling::CompileCommand &Cmd,
249                   llvm::ArrayRef<std::string> SystemIncludes) {
250   for (llvm::StringRef Include : SystemIncludes) {
251     // FIXME(kadircet): This doesn't work when we have "--driver-mode=cl"
252     Cmd.CommandLine.push_back("-isystem");
253     Cmd.CommandLine.push_back(Include.str());
254   }
255   return Cmd;
256 }
257 
258 tooling::CompileCommand &setTarget(tooling::CompileCommand &Cmd,
259                                    const std::string &Target) {
260   if (!Target.empty()) {
261     // We do not want to override existing target with extracted one.
262     for (llvm::StringRef Arg : Cmd.CommandLine) {
263       if (Arg == "-target" || Arg.startswith("--target="))
264         return Cmd;
265     }
266     Cmd.CommandLine.push_back("--target=" + Target);
267   }
268   return Cmd;
269 }
270 
271 /// Converts a glob containing only ** or * into a regex.
272 std::string convertGlobToRegex(llvm::StringRef Glob) {
273   std::string RegText;
274   llvm::raw_string_ostream RegStream(RegText);
275   RegStream << '^';
276   for (size_t I = 0, E = Glob.size(); I < E; ++I) {
277     if (Glob[I] == '*') {
278       if (I + 1 < E && Glob[I + 1] == '*') {
279         // Double star, accept any sequence.
280         RegStream << ".*";
281         // Also skip the second star.
282         ++I;
283       } else {
284         // Single star, accept any sequence without a slash.
285         RegStream << "[^/]*";
286       }
287     } else {
288       RegStream << llvm::Regex::escape(Glob.substr(I, 1));
289     }
290   }
291   RegStream << '$';
292   RegStream.flush();
293   return RegText;
294 }
295 
296 /// Converts a glob containing only ** or * into a regex.
297 llvm::Regex convertGlobsToRegex(llvm::ArrayRef<std::string> Globs) {
298   assert(!Globs.empty() && "Globs cannot be empty!");
299   std::vector<std::string> RegTexts;
300   RegTexts.reserve(Globs.size());
301   for (llvm::StringRef Glob : Globs)
302     RegTexts.push_back(convertGlobToRegex(Glob));
303 
304   llvm::Regex Reg(llvm::join(RegTexts, "|"));
305   assert(Reg.isValid(RegTexts.front()) &&
306          "Created an invalid regex from globs");
307   return Reg;
308 }
309 
310 /// Extracts system includes from a trusted driver by parsing the output of
311 /// include search path and appends them to the commands coming from underlying
312 /// compilation database.
313 class QueryDriverDatabase : public DelegatingCDB {
314 public:
315   QueryDriverDatabase(llvm::ArrayRef<std::string> QueryDriverGlobs,
316                       std::unique_ptr<GlobalCompilationDatabase> Base)
317       : DelegatingCDB(std::move(Base)),
318         QueryDriverRegex(convertGlobsToRegex(QueryDriverGlobs)) {}
319 
320   llvm::Optional<tooling::CompileCommand>
321   getCompileCommand(PathRef File) const override {
322     auto Cmd = DelegatingCDB::getCompileCommand(File);
323     if (!Cmd || Cmd->CommandLine.empty())
324       return Cmd;
325 
326     llvm::StringRef Lang;
327     for (size_t I = 0, E = Cmd->CommandLine.size(); I < E; ++I) {
328       llvm::StringRef Arg = Cmd->CommandLine[I];
329       if (Arg == "-x" && I + 1 < E)
330         Lang = Cmd->CommandLine[I + 1];
331       else if (Arg.startswith("-x"))
332         Lang = Arg.drop_front(2).trim();
333     }
334     if (Lang.empty()) {
335       llvm::StringRef Ext = llvm::sys::path::extension(File).trim('.');
336       auto Type = driver::types::lookupTypeForExtension(Ext);
337       if (Type == driver::types::TY_INVALID) {
338         elog("System include extraction: invalid file type for {0}", Ext);
339         return {};
340       }
341       Lang = driver::types::getTypeName(Type);
342     }
343 
344     llvm::SmallString<128> Driver(Cmd->CommandLine.front());
345     if (llvm::any_of(Driver,
346                        [](char C) { return llvm::sys::path::is_separator(C); }))
347       // Driver is a not a single executable name but instead a path (either
348       // relative or absolute).
349       llvm::sys::fs::make_absolute(Cmd->Directory, Driver);
350 
351     if (auto Info =
352             QueriedDrivers.get(/*Key=*/(Driver + ":" + Lang).str(), [&] {
353               return extractSystemIncludesAndTarget(
354                   Driver, Lang, Cmd->CommandLine, QueryDriverRegex);
355             })) {
356       setTarget(addSystemIncludes(*Cmd, Info->SystemIncludes), Info->Target);
357     }
358     return Cmd;
359   }
360 
361 private:
362   // Caches includes extracted from a driver. Key is driver:lang.
363   Memoize<llvm::StringMap<llvm::Optional<DriverInfo>>> QueriedDrivers;
364   llvm::Regex QueryDriverRegex;
365 };
366 } // namespace
367 
368 std::unique_ptr<GlobalCompilationDatabase>
369 getQueryDriverDatabase(llvm::ArrayRef<std::string> QueryDriverGlobs,
370                        std::unique_ptr<GlobalCompilationDatabase> Base) {
371   assert(Base && "Null base to SystemIncludeExtractor");
372   if (QueryDriverGlobs.empty())
373     return Base;
374   return std::make_unique<QueryDriverDatabase>(QueryDriverGlobs,
375                                                std::move(Base));
376 }
377 
378 } // namespace clangd
379 } // namespace clang
380