1 //===- DependencyScanningWorker.cpp - clang-scan-deps worker --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
10 #include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
11 #include "clang/Frontend/CompilerInstance.h"
12 #include "clang/Frontend/CompilerInvocation.h"
13 #include "clang/Frontend/FrontendActions.h"
14 #include "clang/Frontend/TextDiagnosticPrinter.h"
15 #include "clang/Frontend/Utils.h"
16 #include "clang/Lex/PreprocessorOptions.h"
17 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
18 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
19 #include "clang/Tooling/Tooling.h"
20 
21 using namespace clang;
22 using namespace tooling;
23 using namespace dependencies;
24 
25 namespace {
26 
27 /// Forwards the gatherered dependencies to the consumer.
28 class DependencyConsumerForwarder : public DependencyFileGenerator {
29 public:
30   DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts,
31                               DependencyConsumer &C)
32       : DependencyFileGenerator(*Opts), Opts(std::move(Opts)), C(C) {}
33 
34   void finishedMainFile(DiagnosticsEngine &Diags) override {
35     C.handleDependencyOutputOpts(*Opts);
36     llvm::SmallString<256> CanonPath;
37     for (const auto &File : getDependencies()) {
38       CanonPath = File;
39       llvm::sys::path::remove_dots(CanonPath, /*remove_dot_dot=*/true);
40       C.handleFileDependency(CanonPath);
41     }
42   }
43 
44 private:
45   std::unique_ptr<DependencyOutputOptions> Opts;
46   DependencyConsumer &C;
47 };
48 
49 /// A listener that collects the names and paths to imported modules.
50 class ImportCollectingListener : public ASTReaderListener {
51   using PrebuiltModuleFilesT =
52       decltype(HeaderSearchOptions::PrebuiltModuleFiles);
53 
54 public:
55   ImportCollectingListener(PrebuiltModuleFilesT &PrebuiltModuleFiles)
56       : PrebuiltModuleFiles(PrebuiltModuleFiles) {}
57 
58   bool needsImportVisitation() const override { return true; }
59 
60   void visitImport(StringRef ModuleName, StringRef Filename) override {
61     PrebuiltModuleFiles[std::string(ModuleName)] = std::string(Filename);
62   }
63 
64 private:
65   PrebuiltModuleFilesT &PrebuiltModuleFiles;
66 };
67 
68 /// Transform arbitrary file name into an object-like file name.
69 static std::string makeObjFileName(StringRef FileName) {
70   SmallString<128> ObjFileName(FileName);
71   llvm::sys::path::replace_extension(ObjFileName, "o");
72   return std::string(ObjFileName.str());
73 }
74 
75 /// Deduce the dependency target based on the output file and input files.
76 static std::string
77 deduceDepTarget(const std::string &OutputFile,
78                 const SmallVectorImpl<FrontendInputFile> &InputFiles) {
79   if (OutputFile != "-")
80     return OutputFile;
81 
82   if (InputFiles.empty() || !InputFiles.front().isFile())
83     return "clang-scan-deps\\ dependency";
84 
85   return makeObjFileName(InputFiles.front().getFile());
86 }
87 
88 /// A clang tool that runs the preprocessor in a mode that's optimized for
89 /// dependency scanning for the given compiler invocation.
90 class DependencyScanningAction : public tooling::ToolAction {
91 public:
92   DependencyScanningAction(
93       StringRef WorkingDirectory, DependencyConsumer &Consumer,
94       llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
95       ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings,
96       ScanningOutputFormat Format)
97       : WorkingDirectory(WorkingDirectory), Consumer(Consumer),
98         DepFS(std::move(DepFS)), PPSkipMappings(PPSkipMappings),
99         Format(Format) {}
100 
101   bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
102                      FileManager *FileMgr,
103                      std::shared_ptr<PCHContainerOperations> PCHContainerOps,
104                      DiagnosticConsumer *DiagConsumer) override {
105     // Make a deep copy of the original Clang invocation.
106     CompilerInvocation OriginalInvocation(*Invocation);
107 
108     // Create a compiler instance to handle the actual work.
109     CompilerInstance Compiler(std::move(PCHContainerOps));
110     Compiler.setInvocation(std::move(Invocation));
111 
112     // Don't print 'X warnings and Y errors generated'.
113     Compiler.getDiagnosticOpts().ShowCarets = false;
114     // Don't write out diagnostic file.
115     Compiler.getDiagnosticOpts().DiagnosticSerializationFile.clear();
116     // Don't treat warnings as errors.
117     Compiler.getDiagnosticOpts().Warnings.push_back("no-error");
118     // Create the compiler's actual diagnostics engine.
119     Compiler.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
120     if (!Compiler.hasDiagnostics())
121       return false;
122 
123     Compiler.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath = true;
124 
125     // Use the dependency scanning optimized file system if we can.
126     if (DepFS) {
127       const CompilerInvocation &CI = Compiler.getInvocation();
128       // Add any filenames that were explicity passed in the build settings and
129       // that might be opened, as we want to ensure we don't run source
130       // minimization on them.
131       DepFS->IgnoredFiles.clear();
132       for (const auto &Entry : CI.getHeaderSearchOpts().UserEntries)
133         DepFS->IgnoredFiles.insert(Entry.Path);
134       for (const auto &Entry : CI.getHeaderSearchOpts().VFSOverlayFiles)
135         DepFS->IgnoredFiles.insert(Entry);
136 
137       // Support for virtual file system overlays on top of the caching
138       // filesystem.
139       FileMgr->setVirtualFileSystem(createVFSFromCompilerInvocation(
140           CI, Compiler.getDiagnostics(), DepFS));
141 
142       // Pass the skip mappings which should speed up excluded conditional block
143       // skipping in the preprocessor.
144       if (PPSkipMappings)
145         Compiler.getPreprocessorOpts()
146             .ExcludedConditionalDirectiveSkipMappings = PPSkipMappings;
147     }
148 
149     FileMgr->getFileSystemOpts().WorkingDir = std::string(WorkingDirectory);
150     Compiler.setFileManager(FileMgr);
151     Compiler.createSourceManager(*FileMgr);
152 
153     if (!Compiler.getPreprocessorOpts().ImplicitPCHInclude.empty()) {
154       // Collect the modules that were prebuilt as part of the PCH and pass them
155       // to the compiler. This will prevent the implicit build to create
156       // duplicate modules and force reuse of existing prebuilt module files
157       // instead.
158       ImportCollectingListener Listener(
159           Compiler.getHeaderSearchOpts().PrebuiltModuleFiles);
160       ASTReader::readASTFileControlBlock(
161           Compiler.getPreprocessorOpts().ImplicitPCHInclude,
162           Compiler.getFileManager(), Compiler.getPCHContainerReader(),
163           /*FindModuleFileExtensions=*/false, Listener,
164           /*ValidateDiagnosticOptions=*/false);
165     }
166 
167     // Create the dependency collector that will collect the produced
168     // dependencies.
169     //
170     // This also moves the existing dependency output options from the
171     // invocation to the collector. The options in the invocation are reset,
172     // which ensures that the compiler won't create new dependency collectors,
173     // and thus won't write out the extra '.d' files to disk.
174     auto Opts = std::make_unique<DependencyOutputOptions>();
175     std::swap(*Opts, Compiler.getInvocation().getDependencyOutputOpts());
176     // We need at least one -MT equivalent for the generator of make dependency
177     // files to work.
178     if (Opts->Targets.empty())
179       Opts->Targets = {deduceDepTarget(Compiler.getFrontendOpts().OutputFile,
180                                        Compiler.getFrontendOpts().Inputs)};
181     Opts->IncludeSystemHeaders = true;
182 
183     switch (Format) {
184     case ScanningOutputFormat::Make:
185       Compiler.addDependencyCollector(
186           std::make_shared<DependencyConsumerForwarder>(std::move(Opts),
187                                                         Consumer));
188       break;
189     case ScanningOutputFormat::Full:
190       Compiler.addDependencyCollector(std::make_shared<ModuleDepCollector>(
191           std::move(Opts), Compiler, Consumer, std::move(OriginalInvocation)));
192       break;
193     }
194 
195     // Consider different header search and diagnostic options to create
196     // different modules. This avoids the unsound aliasing of module PCMs.
197     //
198     // TODO: Implement diagnostic bucketing and header search pruning to reduce
199     // the impact of strict context hashing.
200     Compiler.getHeaderSearchOpts().ModulesStrictContextHash = true;
201 
202     auto Action = std::make_unique<ReadPCHAndPreprocessAction>();
203     const bool Result = Compiler.ExecuteAction(*Action);
204     if (!DepFS)
205       FileMgr->clearStatCache();
206     return Result;
207   }
208 
209 private:
210   StringRef WorkingDirectory;
211   DependencyConsumer &Consumer;
212   llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
213   ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings;
214   ScanningOutputFormat Format;
215 };
216 
217 } // end anonymous namespace
218 
219 DependencyScanningWorker::DependencyScanningWorker(
220     DependencyScanningService &Service)
221     : Format(Service.getFormat()) {
222   DiagOpts = new DiagnosticOptions();
223 
224   PCHContainerOps = std::make_shared<PCHContainerOperations>();
225   PCHContainerOps->registerReader(
226       std::make_unique<ObjectFilePCHContainerReader>());
227   // We don't need to write object files, but the current PCH implementation
228   // requires the writer to be registered as well.
229   PCHContainerOps->registerWriter(
230       std::make_unique<ObjectFilePCHContainerWriter>());
231 
232   RealFS = llvm::vfs::createPhysicalFileSystem();
233   if (Service.canSkipExcludedPPRanges())
234     PPSkipMappings =
235         std::make_unique<ExcludedPreprocessorDirectiveSkipMapping>();
236   if (Service.getMode() == ScanningMode::MinimizedSourcePreprocessing)
237     DepFS = new DependencyScanningWorkerFilesystem(
238         Service.getSharedCache(), RealFS, PPSkipMappings.get());
239   if (Service.canReuseFileManager())
240     Files = new FileManager(FileSystemOptions(), RealFS);
241 }
242 
243 static llvm::Error runWithDiags(
244     DiagnosticOptions *DiagOpts,
245     llvm::function_ref<bool(DiagnosticConsumer &DC)> BodyShouldSucceed) {
246   // Capture the emitted diagnostics and report them to the client
247   // in the case of a failure.
248   std::string DiagnosticOutput;
249   llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
250   TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts);
251 
252   if (BodyShouldSucceed(DiagPrinter))
253     return llvm::Error::success();
254   return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(),
255                                              llvm::inconvertibleErrorCode());
256 }
257 
258 llvm::Error DependencyScanningWorker::computeDependencies(
259     const std::string &Input, StringRef WorkingDirectory,
260     const CompilationDatabase &CDB, DependencyConsumer &Consumer) {
261   RealFS->setCurrentWorkingDirectory(WorkingDirectory);
262   return runWithDiags(DiagOpts.get(), [&](DiagnosticConsumer &DC) {
263     /// Create the tool that uses the underlying file system to ensure that any
264     /// file system requests that are made by the driver do not go through the
265     /// dependency scanning filesystem.
266     tooling::ClangTool Tool(CDB, Input, PCHContainerOps, RealFS, Files);
267     Tool.clearArgumentsAdjusters();
268     Tool.setRestoreWorkingDir(false);
269     Tool.setPrintErrorMessage(false);
270     Tool.setDiagnosticConsumer(&DC);
271     DependencyScanningAction Action(WorkingDirectory, Consumer, DepFS,
272                                     PPSkipMappings.get(), Format);
273     return !Tool.run(&Action);
274   });
275 }
276