1 //===--- QueryDriverDatabase.cpp ---------------------------------*- C++-*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 // Some compiler drivers have implicit search mechanism for system headers.
9 // This compilation database implementation tries to extract that information by
10 // executing the driver in verbose mode. gcc-compatible drivers print something
11 // like:
12 // ....
13 // ....
14 // #include <...> search starts here:
15 //  /usr/lib/gcc/x86_64-linux-gnu/7/include
16 //  /usr/local/include
17 //  /usr/lib/gcc/x86_64-linux-gnu/7/include-fixed
18 //  /usr/include/x86_64-linux-gnu
19 //  /usr/include
20 // End of search list.
21 // ....
22 // ....
23 // This component parses that output and adds each path to command line args
24 // provided by Base, after prepending them with -isystem. Therefore current
25 // implementation would not work with a driver that is not gcc-compatible.
26 //
27 // First argument of the command line received from underlying compilation
28 // database is used as compiler driver path. Due to this arbitrary binary
29 // execution, this mechanism is not used by default and only executes binaries
30 // in the paths that are explicitly whitelisted by the user.
31 
32 #include "GlobalCompilationDatabase.h"
33 #include "Logger.h"
34 #include "Path.h"
35 #include "Trace.h"
36 #include "clang/Driver/Types.h"
37 #include "clang/Tooling/CompilationDatabase.h"
38 #include "llvm/ADT/DenseMap.h"
39 #include "llvm/ADT/ScopeExit.h"
40 #include "llvm/ADT/SmallString.h"
41 #include "llvm/ADT/StringExtras.h"
42 #include "llvm/ADT/StringRef.h"
43 #include "llvm/ADT/iterator_range.h"
44 #include "llvm/Support/FileSystem.h"
45 #include "llvm/Support/MemoryBuffer.h"
46 #include "llvm/Support/Path.h"
47 #include "llvm/Support/Program.h"
48 #include "llvm/Support/Regex.h"
49 #include "llvm/Support/ScopedPrinter.h"
50 #include <algorithm>
51 #include <map>
52 #include <string>
53 #include <vector>
54 
55 namespace clang {
56 namespace clangd {
57 namespace {
58 
59 std::vector<std::string> parseDriverOutput(llvm::StringRef Output) {
60   std::vector<std::string> SystemIncludes;
61   constexpr char const *SIS = "#include <...> search starts here:";
62   constexpr char const *SIE = "End of search list.";
63   llvm::SmallVector<llvm::StringRef, 8> Lines;
64   Output.split(Lines, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
65 
66   auto StartIt = std::find(Lines.begin(), Lines.end(), SIS);
67   if (StartIt == Lines.end()) {
68     elog("System include extraction: start marker not found: {0}", Output);
69     return {};
70   }
71   ++StartIt;
72   const auto EndIt = std::find(StartIt, Lines.end(), SIE);
73   if (EndIt == Lines.end()) {
74     elog("System include extraction: end marker missing: {0}", Output);
75     return {};
76   }
77 
78   for (llvm::StringRef Line : llvm::make_range(StartIt, EndIt)) {
79     SystemIncludes.push_back(Line.trim().str());
80     vlog("System include extraction: adding {0}", Line);
81   }
82   return SystemIncludes;
83 }
84 
85 std::vector<std::string> extractSystemIncludes(PathRef Driver,
86                                                llvm::StringRef Lang,
87                                                llvm::Regex &QueryDriverRegex) {
88   trace::Span Tracer("Extract system includes");
89   SPAN_ATTACH(Tracer, "driver", Driver);
90   SPAN_ATTACH(Tracer, "lang", Lang);
91 
92   if (!QueryDriverRegex.match(Driver)) {
93     vlog("System include extraction: not whitelisted driver {0}", Driver);
94     return {};
95   }
96 
97   if (!llvm::sys::fs::exists(Driver)) {
98     elog("System include extraction: {0} does not exist.", Driver);
99     return {};
100   }
101   if (!llvm::sys::fs::can_execute(Driver)) {
102     elog("System include extraction: {0} is not executable.", Driver);
103     return {};
104   }
105 
106   llvm::SmallString<128> StdErrPath;
107   if (auto EC = llvm::sys::fs::createTemporaryFile("system-includes", "clangd",
108                                                    StdErrPath)) {
109     elog("System include extraction: failed to create temporary file with "
110          "error {0}",
111          EC.message());
112     return {};
113   }
114   auto CleanUp = llvm::make_scope_exit(
115       [&StdErrPath]() { llvm::sys::fs::remove(StdErrPath); });
116 
117   llvm::Optional<llvm::StringRef> Redirects[] = {
118       {""}, {""}, llvm::StringRef(StdErrPath)};
119 
120   // Should we also preserve flags like "-sysroot", "-nostdinc" ?
121   const llvm::StringRef Args[] = {Driver, "-E", "-x", Lang, "-", "-v"};
122 
123   if (int RC = llvm::sys::ExecuteAndWait(Driver, Args, /*Env=*/llvm::None,
124                                          Redirects)) {
125     elog("System include extraction: driver execution failed with return code: "
126          "{0}",
127          llvm::to_string(RC));
128     return {};
129   }
130 
131   auto BufOrError = llvm::MemoryBuffer::getFile(StdErrPath);
132   if (!BufOrError) {
133     elog("System include extraction: failed to read {0} with error {1}",
134          StdErrPath, BufOrError.getError().message());
135     return {};
136   }
137 
138   auto Includes = parseDriverOutput(BufOrError->get()->getBuffer());
139   log("System include extractor: succesfully executed {0}, got includes: "
140       "\"{1}\"",
141       Driver, llvm::join(Includes, ", "));
142   return Includes;
143 }
144 
145 tooling::CompileCommand &
146 addSystemIncludes(tooling::CompileCommand &Cmd,
147                   llvm::ArrayRef<std::string> SystemIncludes) {
148   for (llvm::StringRef Include : SystemIncludes) {
149     // FIXME(kadircet): This doesn't work when we have "--driver-mode=cl"
150     Cmd.CommandLine.push_back("-isystem");
151     Cmd.CommandLine.push_back(Include.str());
152   }
153   return Cmd;
154 }
155 
156 /// Converts a glob containing only ** or * into a regex.
157 std::string convertGlobToRegex(llvm::StringRef Glob) {
158   std::string RegText;
159   llvm::raw_string_ostream RegStream(RegText);
160   RegStream << '^';
161   for (size_t I = 0, E = Glob.size(); I < E; ++I) {
162     if (Glob[I] == '*') {
163       if (I + 1 < E && Glob[I + 1] == '*') {
164         // Double star, accept any sequence.
165         RegStream << ".*";
166         // Also skip the second star.
167         ++I;
168       } else {
169         // Single star, accept any sequence without a slash.
170         RegStream << "[^/]*";
171       }
172     } else {
173       RegStream << llvm::Regex::escape(Glob.substr(I, 1));
174     }
175   }
176   RegStream << '$';
177   RegStream.flush();
178   return RegText;
179 }
180 
181 /// Converts a glob containing only ** or * into a regex.
182 llvm::Regex convertGlobsToRegex(llvm::ArrayRef<std::string> Globs) {
183   assert(!Globs.empty() && "Globs cannot be empty!");
184   std::vector<std::string> RegTexts;
185   RegTexts.reserve(Globs.size());
186   for (llvm::StringRef Glob : Globs)
187     RegTexts.push_back(convertGlobToRegex(Glob));
188 
189   llvm::Regex Reg(llvm::join(RegTexts, "|"));
190   assert(Reg.isValid(RegTexts.front()) &&
191          "Created an invalid regex from globs");
192   return Reg;
193 }
194 
195 /// Extracts system includes from a trusted driver by parsing the output of
196 /// include search path and appends them to the commands coming from underlying
197 /// compilation database.
198 class QueryDriverDatabase : public GlobalCompilationDatabase {
199 public:
200   QueryDriverDatabase(llvm::ArrayRef<std::string> QueryDriverGlobs,
201                       std::unique_ptr<GlobalCompilationDatabase> Base)
202       : QueryDriverRegex(convertGlobsToRegex(QueryDriverGlobs)),
203         Base(std::move(Base)) {
204     assert(this->Base);
205     BaseChanged =
206         this->Base->watch([this](const std::vector<std::string> &Changes) {
207           OnCommandChanged.broadcast(Changes);
208         });
209   }
210 
211   llvm::Optional<tooling::CompileCommand>
212   getCompileCommand(PathRef File) const override {
213     auto Cmd = Base->getCompileCommand(File);
214     if (!Cmd || Cmd->CommandLine.empty())
215       return Cmd;
216 
217     llvm::StringRef Lang;
218     for (size_t I = 0, E = Cmd->CommandLine.size(); I < E; ++I) {
219       llvm::StringRef Arg = Cmd->CommandLine[I];
220       if (Arg == "-x" && I + 1 < E)
221         Lang = Cmd->CommandLine[I + 1];
222       else if (Arg.startswith("-x"))
223         Lang = Arg.drop_front(2).trim();
224     }
225     if (Lang.empty()) {
226       llvm::StringRef Ext = llvm::sys::path::extension(File).trim('.');
227       auto Type = driver::types::lookupTypeForExtension(Ext);
228       if (Type == driver::types::TY_INVALID) {
229         elog("System include extraction: invalid file type for {0}", Ext);
230         return {};
231       }
232       Lang = driver::types::getTypeName(Type);
233     }
234 
235     llvm::SmallString<128> Driver(Cmd->CommandLine.front());
236     llvm::sys::fs::make_absolute(Cmd->Directory, Driver);
237     auto Key = std::make_pair(Driver.str(), Lang);
238 
239     std::vector<std::string> SystemIncludes;
240     {
241       std::lock_guard<std::mutex> Lock(Mu);
242 
243       auto It = DriverToIncludesCache.find(Key);
244       if (It != DriverToIncludesCache.end())
245         SystemIncludes = It->second;
246       else
247         DriverToIncludesCache[Key] = SystemIncludes =
248             extractSystemIncludes(Key.first, Key.second, QueryDriverRegex);
249     }
250 
251     return addSystemIncludes(*Cmd, SystemIncludes);
252   }
253 
254   llvm::Optional<ProjectInfo> getProjectInfo(PathRef File) const override {
255     return Base->getProjectInfo(File);
256   }
257 
258 private:
259   mutable std::mutex Mu;
260   // Caches includes extracted from a driver.
261   mutable std::map<std::pair<std::string, std::string>,
262                    std::vector<std::string>>
263       DriverToIncludesCache;
264   mutable llvm::Regex QueryDriverRegex;
265 
266   std::unique_ptr<GlobalCompilationDatabase> Base;
267   CommandChanged::Subscription BaseChanged;
268 };
269 } // namespace
270 
271 std::unique_ptr<GlobalCompilationDatabase>
272 getQueryDriverDatabase(llvm::ArrayRef<std::string> QueryDriverGlobs,
273                        std::unique_ptr<GlobalCompilationDatabase> Base) {
274   assert(Base && "Null base to SystemIncludeExtractor");
275   if (QueryDriverGlobs.empty())
276     return Base;
277   return llvm::make_unique<QueryDriverDatabase>(QueryDriverGlobs,
278                                                 std::move(Base));
279 }
280 
281 } // namespace clangd
282 } // namespace clang
283