1 //===--- Tooling.cpp - Running clang standalone tools ---------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 //  This file implements functions to run clang tools standalone instead
11 //  of running them as a plugin.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "clang/Tooling/Tooling.h"
16 #include "clang/AST/ASTConsumer.h"
17 #include "clang/Driver/Compilation.h"
18 #include "clang/Driver/Driver.h"
19 #include "clang/Driver/Tool.h"
20 #include "clang/Driver/ToolChain.h"
21 #include "clang/Frontend/ASTUnit.h"
22 #include "clang/Frontend/CompilerInstance.h"
23 #include "clang/Frontend/FrontendDiagnostic.h"
24 #include "clang/Frontend/TextDiagnosticPrinter.h"
25 #include "clang/Tooling/ArgumentsAdjusters.h"
26 #include "clang/Tooling/CompilationDatabase.h"
27 #include "llvm/ADT/STLExtras.h"
28 #include "llvm/Config/llvm-config.h"
29 #include "llvm/Option/Option.h"
30 #include "llvm/Support/Debug.h"
31 #include "llvm/Support/FileSystem.h"
32 #include "llvm/Support/Host.h"
33 #include "llvm/Support/raw_ostream.h"
34 #include <utility>
35 
36 #define DEBUG_TYPE "clang-tooling"
37 
38 namespace clang {
39 namespace tooling {
40 
41 ToolAction::~ToolAction() {}
42 
43 FrontendActionFactory::~FrontendActionFactory() {}
44 
45 // FIXME: This file contains structural duplication with other parts of the
46 // code that sets up a compiler to run tools on it, and we should refactor
47 // it to be based on the same framework.
48 
49 /// \brief Builds a clang driver initialized for running clang tools.
50 static clang::driver::Driver *newDriver(
51     clang::DiagnosticsEngine *Diagnostics, const char *BinaryName,
52     IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
53   clang::driver::Driver *CompilerDriver = new clang::driver::Driver(
54       BinaryName, llvm::sys::getDefaultTargetTriple(), *Diagnostics, VFS);
55   CompilerDriver->setTitle("clang_based_tool");
56   return CompilerDriver;
57 }
58 
59 /// \brief Retrieves the clang CC1 specific flags out of the compilation's jobs.
60 ///
61 /// Returns NULL on error.
62 static const llvm::opt::ArgStringList *getCC1Arguments(
63     clang::DiagnosticsEngine *Diagnostics,
64     clang::driver::Compilation *Compilation) {
65   // We expect to get back exactly one Command job, if we didn't something
66   // failed. Extract that job from the Compilation.
67   const clang::driver::JobList &Jobs = Compilation->getJobs();
68   if (Jobs.size() != 1 || !isa<clang::driver::Command>(*Jobs.begin())) {
69     SmallString<256> error_msg;
70     llvm::raw_svector_ostream error_stream(error_msg);
71     Jobs.Print(error_stream, "; ", true);
72     Diagnostics->Report(clang::diag::err_fe_expected_compiler_job)
73         << error_stream.str();
74     return nullptr;
75   }
76 
77   // The one job we find should be to invoke clang again.
78   const clang::driver::Command &Cmd =
79       cast<clang::driver::Command>(*Jobs.begin());
80   if (StringRef(Cmd.getCreator().getName()) != "clang") {
81     Diagnostics->Report(clang::diag::err_fe_expected_clang_command);
82     return nullptr;
83   }
84 
85   return &Cmd.getArguments();
86 }
87 
88 /// \brief Returns a clang build invocation initialized from the CC1 flags.
89 clang::CompilerInvocation *newInvocation(
90     clang::DiagnosticsEngine *Diagnostics,
91     const llvm::opt::ArgStringList &CC1Args) {
92   assert(!CC1Args.empty() && "Must at least contain the program name!");
93   clang::CompilerInvocation *Invocation = new clang::CompilerInvocation;
94   clang::CompilerInvocation::CreateFromArgs(
95       *Invocation, CC1Args.data() + 1, CC1Args.data() + CC1Args.size(),
96       *Diagnostics);
97   Invocation->getFrontendOpts().DisableFree = false;
98   Invocation->getCodeGenOpts().DisableFree = false;
99   Invocation->getDependencyOutputOpts() = DependencyOutputOptions();
100   return Invocation;
101 }
102 
103 bool runToolOnCode(clang::FrontendAction *ToolAction, const Twine &Code,
104                    const Twine &FileName,
105                    std::shared_ptr<PCHContainerOperations> PCHContainerOps) {
106   return runToolOnCodeWithArgs(ToolAction, Code, std::vector<std::string>(),
107                                FileName, "clang-tool", PCHContainerOps);
108 }
109 
110 static std::vector<std::string>
111 getSyntaxOnlyToolArgs(const Twine &ToolName,
112                       const std::vector<std::string> &ExtraArgs,
113                       StringRef FileName) {
114   std::vector<std::string> Args;
115   Args.push_back(ToolName.str());
116   Args.push_back("-fsyntax-only");
117   Args.insert(Args.end(), ExtraArgs.begin(), ExtraArgs.end());
118   Args.push_back(FileName.str());
119   return Args;
120 }
121 
122 bool runToolOnCodeWithArgs(
123     clang::FrontendAction *ToolAction, const Twine &Code,
124     const std::vector<std::string> &Args, const Twine &FileName,
125     const Twine &ToolName,
126     std::shared_ptr<PCHContainerOperations> PCHContainerOps,
127     const FileContentMappings &VirtualMappedFiles) {
128 
129   SmallString<16> FileNameStorage;
130   StringRef FileNameRef = FileName.toNullTerminatedStringRef(FileNameStorage);
131   llvm::IntrusiveRefCntPtr<vfs::OverlayFileSystem> OverlayFileSystem(
132       new vfs::OverlayFileSystem(vfs::getRealFileSystem()));
133   llvm::IntrusiveRefCntPtr<vfs::InMemoryFileSystem> InMemoryFileSystem(
134       new vfs::InMemoryFileSystem);
135   OverlayFileSystem->pushOverlay(InMemoryFileSystem);
136   llvm::IntrusiveRefCntPtr<FileManager> Files(
137       new FileManager(FileSystemOptions(), OverlayFileSystem));
138   ToolInvocation Invocation(getSyntaxOnlyToolArgs(ToolName, Args, FileNameRef),
139                             ToolAction, Files.get(), PCHContainerOps);
140 
141   SmallString<1024> CodeStorage;
142   InMemoryFileSystem->addFile(FileNameRef, 0,
143                               llvm::MemoryBuffer::getMemBuffer(
144                                   Code.toNullTerminatedStringRef(CodeStorage)));
145 
146   for (auto &FilenameWithContent : VirtualMappedFiles) {
147     InMemoryFileSystem->addFile(
148         FilenameWithContent.first, 0,
149         llvm::MemoryBuffer::getMemBuffer(FilenameWithContent.second));
150   }
151 
152   return Invocation.run();
153 }
154 
155 std::string getAbsolutePath(StringRef File) {
156   StringRef RelativePath(File);
157   // FIXME: Should '.\\' be accepted on Win32?
158   if (RelativePath.startswith("./")) {
159     RelativePath = RelativePath.substr(strlen("./"));
160   }
161 
162   SmallString<1024> AbsolutePath = RelativePath;
163   std::error_code EC = llvm::sys::fs::make_absolute(AbsolutePath);
164   assert(!EC);
165   (void)EC;
166   llvm::sys::path::native(AbsolutePath);
167   return AbsolutePath.str();
168 }
169 
170 void addTargetAndModeForProgramName(std::vector<std::string> &CommandLine,
171                                     StringRef InvokedAs) {
172   if (!CommandLine.empty() && !InvokedAs.empty()) {
173     bool AlreadyHasTarget = false;
174     bool AlreadyHasMode = false;
175     // Skip CommandLine[0].
176     for (auto Token = ++CommandLine.begin(); Token != CommandLine.end();
177          ++Token) {
178       StringRef TokenRef(*Token);
179       AlreadyHasTarget |=
180           (TokenRef == "-target" || TokenRef.startswith("-target="));
181       AlreadyHasMode |= (TokenRef == "--driver-mode" ||
182                          TokenRef.startswith("--driver-mode="));
183     }
184     auto TargetMode =
185         clang::driver::ToolChain::getTargetAndModeFromProgramName(InvokedAs);
186     if (!AlreadyHasMode && !TargetMode.second.empty()) {
187       CommandLine.insert(++CommandLine.begin(), TargetMode.second);
188     }
189     if (!AlreadyHasTarget && !TargetMode.first.empty()) {
190       CommandLine.insert(++CommandLine.begin(), {"-target", TargetMode.first});
191     }
192   }
193 }
194 
195 namespace {
196 
197 class SingleFrontendActionFactory : public FrontendActionFactory {
198   FrontendAction *Action;
199 
200 public:
201   SingleFrontendActionFactory(FrontendAction *Action) : Action(Action) {}
202 
203   FrontendAction *create() override { return Action; }
204 };
205 
206 }
207 
208 ToolInvocation::ToolInvocation(
209     std::vector<std::string> CommandLine, ToolAction *Action,
210     FileManager *Files, std::shared_ptr<PCHContainerOperations> PCHContainerOps)
211     : CommandLine(std::move(CommandLine)), Action(Action), OwnsAction(false),
212       Files(Files), PCHContainerOps(std::move(PCHContainerOps)),
213       DiagConsumer(nullptr) {}
214 
215 ToolInvocation::ToolInvocation(
216     std::vector<std::string> CommandLine, FrontendAction *FAction,
217     FileManager *Files, std::shared_ptr<PCHContainerOperations> PCHContainerOps)
218     : CommandLine(std::move(CommandLine)),
219       Action(new SingleFrontendActionFactory(FAction)), OwnsAction(true),
220       Files(Files), PCHContainerOps(std::move(PCHContainerOps)),
221       DiagConsumer(nullptr) {}
222 
223 ToolInvocation::~ToolInvocation() {
224   if (OwnsAction)
225     delete Action;
226 }
227 
228 void ToolInvocation::mapVirtualFile(StringRef FilePath, StringRef Content) {
229   SmallString<1024> PathStorage;
230   llvm::sys::path::native(FilePath, PathStorage);
231   MappedFileContents[PathStorage] = Content;
232 }
233 
234 bool ToolInvocation::run() {
235   std::vector<const char*> Argv;
236   for (const std::string &Str : CommandLine)
237     Argv.push_back(Str.c_str());
238   const char *const BinaryName = Argv[0];
239   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
240   TextDiagnosticPrinter DiagnosticPrinter(
241       llvm::errs(), &*DiagOpts);
242   DiagnosticsEngine Diagnostics(
243       IntrusiveRefCntPtr<clang::DiagnosticIDs>(new DiagnosticIDs()), &*DiagOpts,
244       DiagConsumer ? DiagConsumer : &DiagnosticPrinter, false);
245 
246   const std::unique_ptr<clang::driver::Driver> Driver(
247       newDriver(&Diagnostics, BinaryName, Files->getVirtualFileSystem()));
248   // Since the input might only be virtual, don't check whether it exists.
249   Driver->setCheckInputsExist(false);
250   const std::unique_ptr<clang::driver::Compilation> Compilation(
251       Driver->BuildCompilation(llvm::makeArrayRef(Argv)));
252   const llvm::opt::ArgStringList *const CC1Args = getCC1Arguments(
253       &Diagnostics, Compilation.get());
254   if (!CC1Args) {
255     return false;
256   }
257   std::unique_ptr<clang::CompilerInvocation> Invocation(
258       newInvocation(&Diagnostics, *CC1Args));
259   // FIXME: remove this when all users have migrated!
260   for (const auto &It : MappedFileContents) {
261     // Inject the code as the given file name into the preprocessor options.
262     std::unique_ptr<llvm::MemoryBuffer> Input =
263         llvm::MemoryBuffer::getMemBuffer(It.getValue());
264     Invocation->getPreprocessorOpts().addRemappedFile(It.getKey(),
265                                                       Input.release());
266   }
267   return runInvocation(BinaryName, Compilation.get(), Invocation.release(),
268                        PCHContainerOps);
269 }
270 
271 bool ToolInvocation::runInvocation(
272     const char *BinaryName, clang::driver::Compilation *Compilation,
273     clang::CompilerInvocation *Invocation,
274     std::shared_ptr<PCHContainerOperations> PCHContainerOps) {
275   // Show the invocation, with -v.
276   if (Invocation->getHeaderSearchOpts().Verbose) {
277     llvm::errs() << "clang Invocation:\n";
278     Compilation->getJobs().Print(llvm::errs(), "\n", true);
279     llvm::errs() << "\n";
280   }
281 
282   return Action->runInvocation(Invocation, Files, PCHContainerOps,
283                                DiagConsumer);
284 }
285 
286 bool FrontendActionFactory::runInvocation(
287     CompilerInvocation *Invocation, FileManager *Files,
288     std::shared_ptr<PCHContainerOperations> PCHContainerOps,
289     DiagnosticConsumer *DiagConsumer) {
290   // Create a compiler instance to handle the actual work.
291   clang::CompilerInstance Compiler(PCHContainerOps);
292   Compiler.setInvocation(Invocation);
293   Compiler.setFileManager(Files);
294 
295   // The FrontendAction can have lifetime requirements for Compiler or its
296   // members, and we need to ensure it's deleted earlier than Compiler. So we
297   // pass it to an std::unique_ptr declared after the Compiler variable.
298   std::unique_ptr<FrontendAction> ScopedToolAction(create());
299 
300   // Create the compiler's actual diagnostics engine.
301   Compiler.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
302   if (!Compiler.hasDiagnostics())
303     return false;
304 
305   Compiler.createSourceManager(*Files);
306 
307   const bool Success = Compiler.ExecuteAction(*ScopedToolAction);
308 
309   Files->clearStatCaches();
310   return Success;
311 }
312 
313 ClangTool::ClangTool(const CompilationDatabase &Compilations,
314                      ArrayRef<std::string> SourcePaths,
315                      std::shared_ptr<PCHContainerOperations> PCHContainerOps)
316     : Compilations(Compilations), SourcePaths(SourcePaths),
317       PCHContainerOps(std::move(PCHContainerOps)),
318       OverlayFileSystem(new vfs::OverlayFileSystem(vfs::getRealFileSystem())),
319       InMemoryFileSystem(new vfs::InMemoryFileSystem),
320       Files(new FileManager(FileSystemOptions(), OverlayFileSystem)),
321       DiagConsumer(nullptr) {
322   OverlayFileSystem->pushOverlay(InMemoryFileSystem);
323   appendArgumentsAdjuster(getClangStripOutputAdjuster());
324   appendArgumentsAdjuster(getClangSyntaxOnlyAdjuster());
325 }
326 
327 ClangTool::~ClangTool() {}
328 
329 void ClangTool::mapVirtualFile(StringRef FilePath, StringRef Content) {
330   MappedFileContents.push_back(std::make_pair(FilePath, Content));
331 }
332 
333 void ClangTool::appendArgumentsAdjuster(ArgumentsAdjuster Adjuster) {
334   if (ArgsAdjuster)
335     ArgsAdjuster = combineAdjusters(ArgsAdjuster, Adjuster);
336   else
337     ArgsAdjuster = Adjuster;
338 }
339 
340 void ClangTool::clearArgumentsAdjusters() {
341   ArgsAdjuster = nullptr;
342 }
343 
344 static void injectResourceDir(CommandLineArguments &Args, const char *Argv0,
345                               void *MainAddr) {
346   // Allow users to override the resource dir.
347   for (StringRef Arg : Args)
348     if (Arg.startswith("-resource-dir"))
349       return;
350 
351   // If there's no override in place add our resource dir.
352   Args.push_back("-resource-dir=" +
353                  CompilerInvocation::GetResourcesPath(Argv0, MainAddr));
354 }
355 
356 int ClangTool::run(ToolAction *Action) {
357   // Exists solely for the purpose of lookup of the resource path.
358   // This just needs to be some symbol in the binary.
359   static int StaticSymbol;
360 
361   llvm::SmallString<128> InitialDirectory;
362   if (std::error_code EC = llvm::sys::fs::current_path(InitialDirectory))
363     llvm::report_fatal_error("Cannot detect current path: " +
364                              Twine(EC.message()));
365 
366   // First insert all absolute paths into the in-memory VFS. These are global
367   // for all compile commands.
368   if (SeenWorkingDirectories.insert("/").second)
369     for (const auto &MappedFile : MappedFileContents)
370       if (llvm::sys::path::is_absolute(MappedFile.first))
371         InMemoryFileSystem->addFile(
372             MappedFile.first, 0,
373             llvm::MemoryBuffer::getMemBuffer(MappedFile.second));
374 
375   bool ProcessingFailed = false;
376   for (const auto &SourcePath : SourcePaths) {
377     std::string File(getAbsolutePath(SourcePath));
378 
379     // Currently implementations of CompilationDatabase::getCompileCommands can
380     // change the state of the file system (e.g.  prepare generated headers), so
381     // this method needs to run right before we invoke the tool, as the next
382     // file may require a different (incompatible) state of the file system.
383     //
384     // FIXME: Make the compilation database interface more explicit about the
385     // requirements to the order of invocation of its members.
386     std::vector<CompileCommand> CompileCommandsForFile =
387         Compilations.getCompileCommands(File);
388     if (CompileCommandsForFile.empty()) {
389       // FIXME: There are two use cases here: doing a fuzzy
390       // "find . -name '*.cc' |xargs tool" match, where as a user I don't care
391       // about the .cc files that were not found, and the use case where I
392       // specify all files I want to run over explicitly, where this should
393       // be an error. We'll want to add an option for this.
394       llvm::errs() << "Skipping " << File << ". Compile command not found.\n";
395       continue;
396     }
397     for (CompileCommand &CompileCommand : CompileCommandsForFile) {
398       // FIXME: chdir is thread hostile; on the other hand, creating the same
399       // behavior as chdir is complex: chdir resolves the path once, thus
400       // guaranteeing that all subsequent relative path operations work
401       // on the same path the original chdir resulted in. This makes a
402       // difference for example on network filesystems, where symlinks might be
403       // switched during runtime of the tool. Fixing this depends on having a
404       // file system abstraction that allows openat() style interactions.
405       if (OverlayFileSystem->setCurrentWorkingDirectory(
406               CompileCommand.Directory))
407         llvm::report_fatal_error("Cannot chdir into \"" +
408                                  Twine(CompileCommand.Directory) + "\n!");
409 
410       // Now fill the in-memory VFS with the relative file mappings so it will
411       // have the correct relative paths. We never remove mappings but that
412       // should be fine.
413       if (SeenWorkingDirectories.insert(CompileCommand.Directory).second)
414         for (const auto &MappedFile : MappedFileContents)
415           if (!llvm::sys::path::is_absolute(MappedFile.first))
416             InMemoryFileSystem->addFile(
417                 MappedFile.first, 0,
418                 llvm::MemoryBuffer::getMemBuffer(MappedFile.second));
419 
420       std::vector<std::string> CommandLine = CompileCommand.CommandLine;
421       if (ArgsAdjuster)
422         CommandLine = ArgsAdjuster(CommandLine, CompileCommand.Filename);
423       assert(!CommandLine.empty());
424 
425       // Add the resource dir based on the binary of this tool. argv[0] in the
426       // compilation database may refer to a different compiler and we want to
427       // pick up the very same standard library that compiler is using. The
428       // builtin headers in the resource dir need to match the exact clang
429       // version the tool is using.
430       // FIXME: On linux, GetMainExecutable is independent of the value of the
431       // first argument, thus allowing ClangTool and runToolOnCode to just
432       // pass in made-up names here. Make sure this works on other platforms.
433       injectResourceDir(CommandLine, "clang_tool", &StaticSymbol);
434 
435       // FIXME: We need a callback mechanism for the tool writer to output a
436       // customized message for each file.
437       DEBUG({ llvm::dbgs() << "Processing: " << File << ".\n"; });
438       ToolInvocation Invocation(std::move(CommandLine), Action, Files.get(),
439                                 PCHContainerOps);
440       Invocation.setDiagnosticConsumer(DiagConsumer);
441 
442       if (!Invocation.run()) {
443         // FIXME: Diagnostics should be used instead.
444         llvm::errs() << "Error while processing " << File << ".\n";
445         ProcessingFailed = true;
446       }
447       // Return to the initial directory to correctly resolve next file by
448       // relative path.
449       if (OverlayFileSystem->setCurrentWorkingDirectory(InitialDirectory.c_str()))
450         llvm::report_fatal_error("Cannot chdir into \"" +
451                                  Twine(InitialDirectory) + "\n!");
452     }
453   }
454   return ProcessingFailed ? 1 : 0;
455 }
456 
457 namespace {
458 
459 class ASTBuilderAction : public ToolAction {
460   std::vector<std::unique_ptr<ASTUnit>> &ASTs;
461 
462 public:
463   ASTBuilderAction(std::vector<std::unique_ptr<ASTUnit>> &ASTs) : ASTs(ASTs) {}
464 
465   bool runInvocation(CompilerInvocation *Invocation, FileManager *Files,
466                      std::shared_ptr<PCHContainerOperations> PCHContainerOps,
467                      DiagnosticConsumer *DiagConsumer) override {
468     std::unique_ptr<ASTUnit> AST = ASTUnit::LoadFromCompilerInvocation(
469         Invocation, PCHContainerOps,
470         CompilerInstance::createDiagnostics(&Invocation->getDiagnosticOpts(),
471                                             DiagConsumer,
472                                             /*ShouldOwnClient=*/false),
473         Files);
474     if (!AST)
475       return false;
476 
477     ASTs.push_back(std::move(AST));
478     return true;
479   }
480 };
481 
482 }
483 
484 int ClangTool::buildASTs(std::vector<std::unique_ptr<ASTUnit>> &ASTs) {
485   ASTBuilderAction Action(ASTs);
486   return run(&Action);
487 }
488 
489 std::unique_ptr<ASTUnit>
490 buildASTFromCode(const Twine &Code, const Twine &FileName,
491                  std::shared_ptr<PCHContainerOperations> PCHContainerOps) {
492   return buildASTFromCodeWithArgs(Code, std::vector<std::string>(), FileName,
493                                   "clang-tool", PCHContainerOps);
494 }
495 
496 std::unique_ptr<ASTUnit> buildASTFromCodeWithArgs(
497     const Twine &Code, const std::vector<std::string> &Args,
498     const Twine &FileName, const Twine &ToolName,
499     std::shared_ptr<PCHContainerOperations> PCHContainerOps) {
500   SmallString<16> FileNameStorage;
501   StringRef FileNameRef = FileName.toNullTerminatedStringRef(FileNameStorage);
502 
503   std::vector<std::unique_ptr<ASTUnit>> ASTs;
504   ASTBuilderAction Action(ASTs);
505   llvm::IntrusiveRefCntPtr<vfs::OverlayFileSystem> OverlayFileSystem(
506       new vfs::OverlayFileSystem(vfs::getRealFileSystem()));
507   llvm::IntrusiveRefCntPtr<vfs::InMemoryFileSystem> InMemoryFileSystem(
508       new vfs::InMemoryFileSystem);
509   OverlayFileSystem->pushOverlay(InMemoryFileSystem);
510   llvm::IntrusiveRefCntPtr<FileManager> Files(
511       new FileManager(FileSystemOptions(), OverlayFileSystem));
512   ToolInvocation Invocation(getSyntaxOnlyToolArgs(ToolName, Args, FileNameRef),
513                             &Action, Files.get(), PCHContainerOps);
514 
515   SmallString<1024> CodeStorage;
516   InMemoryFileSystem->addFile(FileNameRef, 0,
517                               llvm::MemoryBuffer::getMemBuffer(
518                                   Code.toNullTerminatedStringRef(CodeStorage)));
519   if (!Invocation.run())
520     return nullptr;
521 
522   assert(ASTs.size() == 1);
523   return std::move(ASTs[0]);
524 }
525 
526 } // end namespace tooling
527 } // end namespace clang
528