1 //===--- Tooling.cpp - Running clang standalone tools ---------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 //  This file implements functions to run clang tools standalone instead
11 //  of running them as a plugin.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "clang/Tooling/Tooling.h"
16 #include "clang/AST/ASTConsumer.h"
17 #include "clang/Driver/Compilation.h"
18 #include "clang/Driver/Driver.h"
19 #include "clang/Driver/Tool.h"
20 #include "clang/Frontend/ASTUnit.h"
21 #include "clang/Frontend/CompilerInstance.h"
22 #include "clang/Frontend/FrontendDiagnostic.h"
23 #include "clang/Frontend/TextDiagnosticPrinter.h"
24 #include "clang/Tooling/ArgumentsAdjusters.h"
25 #include "clang/Tooling/CompilationDatabase.h"
26 #include "llvm/ADT/STLExtras.h"
27 #include "llvm/Config/llvm-config.h"
28 #include "llvm/Option/Option.h"
29 #include "llvm/Support/Debug.h"
30 #include "llvm/Support/FileSystem.h"
31 #include "llvm/Support/Host.h"
32 #include "llvm/Support/raw_ostream.h"
33 
34 // For chdir, see the comment in ClangTool::run for more information.
35 #ifdef LLVM_ON_WIN32
36 #  include <direct.h>
37 #else
38 #  include <unistd.h>
39 #endif
40 
41 #define DEBUG_TYPE "clang-tooling"
42 
43 namespace clang {
44 namespace tooling {
45 
46 ToolAction::~ToolAction() {}
47 
48 FrontendActionFactory::~FrontendActionFactory() {}
49 
50 // FIXME: This file contains structural duplication with other parts of the
51 // code that sets up a compiler to run tools on it, and we should refactor
52 // it to be based on the same framework.
53 
54 /// \brief Builds a clang driver initialized for running clang tools.
55 static clang::driver::Driver *newDriver(clang::DiagnosticsEngine *Diagnostics,
56                                         const char *BinaryName) {
57   clang::driver::Driver *CompilerDriver = new clang::driver::Driver(
58       BinaryName, llvm::sys::getDefaultTargetTriple(), *Diagnostics);
59   CompilerDriver->setTitle("clang_based_tool");
60   return CompilerDriver;
61 }
62 
63 /// \brief Retrieves the clang CC1 specific flags out of the compilation's jobs.
64 ///
65 /// Returns NULL on error.
66 static const llvm::opt::ArgStringList *getCC1Arguments(
67     clang::DiagnosticsEngine *Diagnostics,
68     clang::driver::Compilation *Compilation) {
69   // We expect to get back exactly one Command job, if we didn't something
70   // failed. Extract that job from the Compilation.
71   const clang::driver::JobList &Jobs = Compilation->getJobs();
72   if (Jobs.size() != 1 || !isa<clang::driver::Command>(*Jobs.begin())) {
73     SmallString<256> error_msg;
74     llvm::raw_svector_ostream error_stream(error_msg);
75     Jobs.Print(error_stream, "; ", true);
76     Diagnostics->Report(clang::diag::err_fe_expected_compiler_job)
77         << error_stream.str();
78     return nullptr;
79   }
80 
81   // The one job we find should be to invoke clang again.
82   const clang::driver::Command &Cmd =
83       cast<clang::driver::Command>(*Jobs.begin());
84   if (StringRef(Cmd.getCreator().getName()) != "clang") {
85     Diagnostics->Report(clang::diag::err_fe_expected_clang_command);
86     return nullptr;
87   }
88 
89   return &Cmd.getArguments();
90 }
91 
92 /// \brief Returns a clang build invocation initialized from the CC1 flags.
93 clang::CompilerInvocation *newInvocation(
94     clang::DiagnosticsEngine *Diagnostics,
95     const llvm::opt::ArgStringList &CC1Args) {
96   assert(!CC1Args.empty() && "Must at least contain the program name!");
97   clang::CompilerInvocation *Invocation = new clang::CompilerInvocation;
98   clang::CompilerInvocation::CreateFromArgs(
99       *Invocation, CC1Args.data() + 1, CC1Args.data() + CC1Args.size(),
100       *Diagnostics);
101   Invocation->getFrontendOpts().DisableFree = false;
102   Invocation->getCodeGenOpts().DisableFree = false;
103   Invocation->getDependencyOutputOpts() = DependencyOutputOptions();
104   return Invocation;
105 }
106 
107 bool runToolOnCode(clang::FrontendAction *ToolAction, const Twine &Code,
108                    const Twine &FileName,
109                    std::shared_ptr<PCHContainerOperations> PCHContainerOps) {
110   return runToolOnCodeWithArgs(ToolAction, Code, std::vector<std::string>(),
111                                FileName, PCHContainerOps);
112 }
113 
114 static std::vector<std::string>
115 getSyntaxOnlyToolArgs(const std::vector<std::string> &ExtraArgs,
116                       StringRef FileName) {
117   std::vector<std::string> Args;
118   Args.push_back("clang-tool");
119   Args.push_back("-fsyntax-only");
120   Args.insert(Args.end(), ExtraArgs.begin(), ExtraArgs.end());
121   Args.push_back(FileName.str());
122   return Args;
123 }
124 
125 bool runToolOnCodeWithArgs(
126     clang::FrontendAction *ToolAction, const Twine &Code,
127     const std::vector<std::string> &Args, const Twine &FileName,
128     std::shared_ptr<PCHContainerOperations> PCHContainerOps,
129     const FileContentMappings &VirtualMappedFiles) {
130 
131   SmallString<16> FileNameStorage;
132   StringRef FileNameRef = FileName.toNullTerminatedStringRef(FileNameStorage);
133   llvm::IntrusiveRefCntPtr<FileManager> Files(
134       new FileManager(FileSystemOptions()));
135   ToolInvocation Invocation(getSyntaxOnlyToolArgs(Args, FileNameRef),
136                             ToolAction, Files.get(), PCHContainerOps);
137 
138   SmallString<1024> CodeStorage;
139   Invocation.mapVirtualFile(FileNameRef,
140                             Code.toNullTerminatedStringRef(CodeStorage));
141 
142   for (auto &FilenameWithContent : VirtualMappedFiles) {
143     Invocation.mapVirtualFile(FilenameWithContent.first,
144                               FilenameWithContent.second);
145   }
146 
147   return Invocation.run();
148 }
149 
150 std::string getAbsolutePath(StringRef File) {
151   StringRef RelativePath(File);
152   // FIXME: Should '.\\' be accepted on Win32?
153   if (RelativePath.startswith("./")) {
154     RelativePath = RelativePath.substr(strlen("./"));
155   }
156 
157   SmallString<1024> AbsolutePath = RelativePath;
158   std::error_code EC = llvm::sys::fs::make_absolute(AbsolutePath);
159   assert(!EC);
160   (void)EC;
161   llvm::sys::path::native(AbsolutePath);
162   return AbsolutePath.str();
163 }
164 
165 namespace {
166 
167 class SingleFrontendActionFactory : public FrontendActionFactory {
168   FrontendAction *Action;
169 
170 public:
171   SingleFrontendActionFactory(FrontendAction *Action) : Action(Action) {}
172 
173   FrontendAction *create() override { return Action; }
174 };
175 
176 }
177 
178 ToolInvocation::ToolInvocation(
179     std::vector<std::string> CommandLine, ToolAction *Action,
180     FileManager *Files, std::shared_ptr<PCHContainerOperations> PCHContainerOps)
181     : CommandLine(std::move(CommandLine)), Action(Action), OwnsAction(false),
182       Files(Files), PCHContainerOps(PCHContainerOps), DiagConsumer(nullptr) {}
183 
184 ToolInvocation::ToolInvocation(
185     std::vector<std::string> CommandLine, FrontendAction *FAction,
186     FileManager *Files, std::shared_ptr<PCHContainerOperations> PCHContainerOps)
187     : CommandLine(std::move(CommandLine)),
188       Action(new SingleFrontendActionFactory(FAction)), OwnsAction(true),
189       Files(Files), PCHContainerOps(PCHContainerOps), DiagConsumer(nullptr) {}
190 
191 ToolInvocation::~ToolInvocation() {
192   if (OwnsAction)
193     delete Action;
194 }
195 
196 void ToolInvocation::mapVirtualFile(StringRef FilePath, StringRef Content) {
197   SmallString<1024> PathStorage;
198   llvm::sys::path::native(FilePath, PathStorage);
199   MappedFileContents[PathStorage] = Content;
200 }
201 
202 bool ToolInvocation::run() {
203   std::vector<const char*> Argv;
204   for (const std::string &Str : CommandLine)
205     Argv.push_back(Str.c_str());
206   const char *const BinaryName = Argv[0];
207   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
208   TextDiagnosticPrinter DiagnosticPrinter(
209       llvm::errs(), &*DiagOpts);
210   DiagnosticsEngine Diagnostics(
211       IntrusiveRefCntPtr<clang::DiagnosticIDs>(new DiagnosticIDs()), &*DiagOpts,
212       DiagConsumer ? DiagConsumer : &DiagnosticPrinter, false);
213 
214   const std::unique_ptr<clang::driver::Driver> Driver(
215       newDriver(&Diagnostics, BinaryName));
216   // Since the input might only be virtual, don't check whether it exists.
217   Driver->setCheckInputsExist(false);
218   const std::unique_ptr<clang::driver::Compilation> Compilation(
219       Driver->BuildCompilation(llvm::makeArrayRef(Argv)));
220   const llvm::opt::ArgStringList *const CC1Args = getCC1Arguments(
221       &Diagnostics, Compilation.get());
222   if (!CC1Args) {
223     return false;
224   }
225   std::unique_ptr<clang::CompilerInvocation> Invocation(
226       newInvocation(&Diagnostics, *CC1Args));
227   for (const auto &It : MappedFileContents) {
228     // Inject the code as the given file name into the preprocessor options.
229     std::unique_ptr<llvm::MemoryBuffer> Input =
230         llvm::MemoryBuffer::getMemBuffer(It.getValue());
231     Invocation->getPreprocessorOpts().addRemappedFile(It.getKey(),
232                                                       Input.release());
233   }
234   return runInvocation(BinaryName, Compilation.get(), Invocation.release(),
235                        PCHContainerOps);
236 }
237 
238 bool ToolInvocation::runInvocation(
239     const char *BinaryName, clang::driver::Compilation *Compilation,
240     clang::CompilerInvocation *Invocation,
241     std::shared_ptr<PCHContainerOperations> PCHContainerOps) {
242   // Show the invocation, with -v.
243   if (Invocation->getHeaderSearchOpts().Verbose) {
244     llvm::errs() << "clang Invocation:\n";
245     Compilation->getJobs().Print(llvm::errs(), "\n", true);
246     llvm::errs() << "\n";
247   }
248 
249   return Action->runInvocation(Invocation, Files, PCHContainerOps,
250                                DiagConsumer);
251 }
252 
253 bool FrontendActionFactory::runInvocation(
254     CompilerInvocation *Invocation, FileManager *Files,
255     std::shared_ptr<PCHContainerOperations> PCHContainerOps,
256     DiagnosticConsumer *DiagConsumer) {
257   // Create a compiler instance to handle the actual work.
258   clang::CompilerInstance Compiler(PCHContainerOps);
259   Compiler.setInvocation(Invocation);
260   Compiler.setFileManager(Files);
261 
262   // The FrontendAction can have lifetime requirements for Compiler or its
263   // members, and we need to ensure it's deleted earlier than Compiler. So we
264   // pass it to an std::unique_ptr declared after the Compiler variable.
265   std::unique_ptr<FrontendAction> ScopedToolAction(create());
266 
267   // Create the compiler's actual diagnostics engine.
268   Compiler.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
269   if (!Compiler.hasDiagnostics())
270     return false;
271 
272   Compiler.createSourceManager(*Files);
273 
274   const bool Success = Compiler.ExecuteAction(*ScopedToolAction);
275 
276   Files->clearStatCaches();
277   return Success;
278 }
279 
280 ClangTool::ClangTool(const CompilationDatabase &Compilations,
281                      ArrayRef<std::string> SourcePaths,
282                      std::shared_ptr<PCHContainerOperations> PCHContainerOps)
283     : Compilations(Compilations), SourcePaths(SourcePaths),
284       PCHContainerOps(PCHContainerOps),
285       Files(new FileManager(FileSystemOptions())), DiagConsumer(nullptr) {
286   appendArgumentsAdjuster(getClangStripOutputAdjuster());
287   appendArgumentsAdjuster(getClangSyntaxOnlyAdjuster());
288 }
289 
290 ClangTool::~ClangTool() {}
291 
292 void ClangTool::mapVirtualFile(StringRef FilePath, StringRef Content) {
293   MappedFileContents.push_back(std::make_pair(FilePath, Content));
294 }
295 
296 void ClangTool::appendArgumentsAdjuster(ArgumentsAdjuster Adjuster) {
297   if (ArgsAdjuster)
298     ArgsAdjuster = combineAdjusters(ArgsAdjuster, Adjuster);
299   else
300     ArgsAdjuster = Adjuster;
301 }
302 
303 void ClangTool::clearArgumentsAdjusters() {
304   ArgsAdjuster = nullptr;
305 }
306 
307 int ClangTool::run(ToolAction *Action) {
308   // Exists solely for the purpose of lookup of the resource path.
309   // This just needs to be some symbol in the binary.
310   static int StaticSymbol;
311   // The driver detects the builtin header path based on the path of the
312   // executable.
313   // FIXME: On linux, GetMainExecutable is independent of the value of the
314   // first argument, thus allowing ClangTool and runToolOnCode to just
315   // pass in made-up names here. Make sure this works on other platforms.
316   std::string MainExecutable =
317       llvm::sys::fs::getMainExecutable("clang_tool", &StaticSymbol);
318 
319   llvm::SmallString<128> InitialDirectory;
320   if (std::error_code EC = llvm::sys::fs::current_path(InitialDirectory))
321     llvm::report_fatal_error("Cannot detect current path: " +
322                              Twine(EC.message()));
323   bool ProcessingFailed = false;
324   for (const auto &SourcePath : SourcePaths) {
325     std::string File(getAbsolutePath(SourcePath));
326 
327     // Currently implementations of CompilationDatabase::getCompileCommands can
328     // change the state of the file system (e.g.  prepare generated headers), so
329     // this method needs to run right before we invoke the tool, as the next
330     // file may require a different (incompatible) state of the file system.
331     //
332     // FIXME: Make the compilation database interface more explicit about the
333     // requirements to the order of invocation of its members.
334     std::vector<CompileCommand> CompileCommandsForFile =
335         Compilations.getCompileCommands(File);
336     if (CompileCommandsForFile.empty()) {
337       // FIXME: There are two use cases here: doing a fuzzy
338       // "find . -name '*.cc' |xargs tool" match, where as a user I don't care
339       // about the .cc files that were not found, and the use case where I
340       // specify all files I want to run over explicitly, where this should
341       // be an error. We'll want to add an option for this.
342       llvm::errs() << "Skipping " << File << ". Compile command not found.\n";
343       continue;
344     }
345     for (CompileCommand &CompileCommand : CompileCommandsForFile) {
346       // FIXME: chdir is thread hostile; on the other hand, creating the same
347       // behavior as chdir is complex: chdir resolves the path once, thus
348       // guaranteeing that all subsequent relative path operations work
349       // on the same path the original chdir resulted in. This makes a
350       // difference for example on network filesystems, where symlinks might be
351       // switched during runtime of the tool. Fixing this depends on having a
352       // file system abstraction that allows openat() style interactions.
353       if (chdir(CompileCommand.Directory.c_str()))
354         llvm::report_fatal_error("Cannot chdir into \"" +
355                                  Twine(CompileCommand.Directory) + "\n!");
356       std::vector<std::string> CommandLine = CompileCommand.CommandLine;
357       if (ArgsAdjuster)
358         CommandLine = ArgsAdjuster(CommandLine);
359       assert(!CommandLine.empty());
360       CommandLine[0] = MainExecutable;
361       // FIXME: We need a callback mechanism for the tool writer to output a
362       // customized message for each file.
363       DEBUG({ llvm::dbgs() << "Processing: " << File << ".\n"; });
364       ToolInvocation Invocation(std::move(CommandLine), Action, Files.get(),
365                                 PCHContainerOps);
366       Invocation.setDiagnosticConsumer(DiagConsumer);
367       for (const auto &MappedFile : MappedFileContents)
368         Invocation.mapVirtualFile(MappedFile.first, MappedFile.second);
369       if (!Invocation.run()) {
370         // FIXME: Diagnostics should be used instead.
371         llvm::errs() << "Error while processing " << File << ".\n";
372         ProcessingFailed = true;
373       }
374       // Return to the initial directory to correctly resolve next file by
375       // relative path.
376       if (chdir(InitialDirectory.c_str()))
377         llvm::report_fatal_error("Cannot chdir into \"" +
378                                  Twine(InitialDirectory) + "\n!");
379     }
380   }
381   return ProcessingFailed ? 1 : 0;
382 }
383 
384 namespace {
385 
386 class ASTBuilderAction : public ToolAction {
387   std::vector<std::unique_ptr<ASTUnit>> &ASTs;
388 
389 public:
390   ASTBuilderAction(std::vector<std::unique_ptr<ASTUnit>> &ASTs) : ASTs(ASTs) {}
391 
392   bool runInvocation(CompilerInvocation *Invocation, FileManager *Files,
393                      std::shared_ptr<PCHContainerOperations> PCHContainerOps,
394                      DiagnosticConsumer *DiagConsumer) override {
395     // FIXME: This should use the provided FileManager.
396     std::unique_ptr<ASTUnit> AST = ASTUnit::LoadFromCompilerInvocation(
397         Invocation, PCHContainerOps,
398         CompilerInstance::createDiagnostics(&Invocation->getDiagnosticOpts(),
399                                             DiagConsumer,
400                                             /*ShouldOwnClient=*/false));
401     if (!AST)
402       return false;
403 
404     ASTs.push_back(std::move(AST));
405     return true;
406   }
407 };
408 
409 }
410 
411 int ClangTool::buildASTs(std::vector<std::unique_ptr<ASTUnit>> &ASTs) {
412   ASTBuilderAction Action(ASTs);
413   return run(&Action);
414 }
415 
416 std::unique_ptr<ASTUnit>
417 buildASTFromCode(const Twine &Code, const Twine &FileName,
418                  std::shared_ptr<PCHContainerOperations> PCHContainerOps) {
419   return buildASTFromCodeWithArgs(Code, std::vector<std::string>(), FileName,
420                                   PCHContainerOps);
421 }
422 
423 std::unique_ptr<ASTUnit> buildASTFromCodeWithArgs(
424     const Twine &Code, const std::vector<std::string> &Args,
425     const Twine &FileName,
426     std::shared_ptr<PCHContainerOperations> PCHContainerOps) {
427   SmallString<16> FileNameStorage;
428   StringRef FileNameRef = FileName.toNullTerminatedStringRef(FileNameStorage);
429 
430   std::vector<std::unique_ptr<ASTUnit>> ASTs;
431   ASTBuilderAction Action(ASTs);
432   ToolInvocation Invocation(getSyntaxOnlyToolArgs(Args, FileNameRef), &Action,
433                             nullptr, PCHContainerOps);
434 
435   SmallString<1024> CodeStorage;
436   Invocation.mapVirtualFile(FileNameRef,
437                             Code.toNullTerminatedStringRef(CodeStorage));
438   if (!Invocation.run())
439     return nullptr;
440 
441   assert(ASTs.size() == 1);
442   return std::move(ASTs[0]);
443 }
444 
445 } // end namespace tooling
446 } // end namespace clang
447