1 //===- DependencyScanningWorker.cpp - clang-scan-deps worker --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
10 #include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
11 #include "clang/Frontend/CompilerInstance.h"
12 #include "clang/Frontend/CompilerInvocation.h"
13 #include "clang/Frontend/FrontendActions.h"
14 #include "clang/Frontend/TextDiagnosticPrinter.h"
15 #include "clang/Frontend/Utils.h"
16 #include "clang/Lex/PreprocessorOptions.h"
17 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
18 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
19 #include "clang/Tooling/Tooling.h"
20 
21 using namespace clang;
22 using namespace tooling;
23 using namespace dependencies;
24 
25 namespace {
26 
27 /// Forwards the gatherered dependencies to the consumer.
28 class DependencyConsumerForwarder : public DependencyFileGenerator {
29 public:
30   DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts,
31                               DependencyConsumer &C)
32       : DependencyFileGenerator(*Opts), Opts(std::move(Opts)), C(C) {}
33 
34   void finishedMainFile(DiagnosticsEngine &Diags) override {
35     C.handleDependencyOutputOpts(*Opts);
36     llvm::SmallString<256> CanonPath;
37     for (const auto &File : getDependencies()) {
38       CanonPath = File;
39       llvm::sys::path::remove_dots(CanonPath, /*remove_dot_dot=*/true);
40       C.handleFileDependency(CanonPath);
41     }
42   }
43 
44 private:
45   std::unique_ptr<DependencyOutputOptions> Opts;
46   DependencyConsumer &C;
47 };
48 
49 using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
50 
51 /// A listener that collects the imported modules and optionally the input
52 /// files.
53 class PrebuiltModuleListener : public ASTReaderListener {
54 public:
55   PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles,
56                          llvm::StringSet<> &InputFiles, bool VisitInputFiles,
57                          llvm::SmallVector<std::string> &NewModuleFiles)
58       : PrebuiltModuleFiles(PrebuiltModuleFiles), InputFiles(InputFiles),
59         VisitInputFiles(VisitInputFiles), NewModuleFiles(NewModuleFiles) {}
60 
61   bool needsImportVisitation() const override { return true; }
62   bool needsInputFileVisitation() override { return VisitInputFiles; }
63   bool needsSystemInputFileVisitation() override { return VisitInputFiles; }
64 
65   void visitImport(StringRef ModuleName, StringRef Filename) override {
66     if (PrebuiltModuleFiles.insert({ModuleName.str(), Filename.str()}).second)
67       NewModuleFiles.push_back(Filename.str());
68   }
69 
70   bool visitInputFile(StringRef Filename, bool isSystem, bool isOverridden,
71                       bool isExplicitModule) override {
72     InputFiles.insert(Filename);
73     return true;
74   }
75 
76 private:
77   PrebuiltModuleFilesT &PrebuiltModuleFiles;
78   llvm::StringSet<> &InputFiles;
79   bool VisitInputFiles;
80   llvm::SmallVector<std::string> &NewModuleFiles;
81 };
82 
83 /// Visit the given prebuilt module and collect all of the modules it
84 /// transitively imports and contributing input files.
85 static void visitPrebuiltModule(StringRef PrebuiltModuleFilename,
86                                 CompilerInstance &CI,
87                                 PrebuiltModuleFilesT &ModuleFiles,
88                                 llvm::StringSet<> &InputFiles,
89                                 bool VisitInputFiles) {
90   // List of module files to be processed.
91   llvm::SmallVector<std::string> Worklist{PrebuiltModuleFilename.str()};
92   PrebuiltModuleListener Listener(ModuleFiles, InputFiles, VisitInputFiles,
93                                   Worklist);
94 
95   while (!Worklist.empty())
96     ASTReader::readASTFileControlBlock(
97         Worklist.pop_back_val(), CI.getFileManager(),
98         CI.getPCHContainerReader(),
99         /*FindModuleFileExtensions=*/false, Listener,
100         /*ValidateDiagnosticOptions=*/false);
101 }
102 
103 /// Transform arbitrary file name into an object-like file name.
104 static std::string makeObjFileName(StringRef FileName) {
105   SmallString<128> ObjFileName(FileName);
106   llvm::sys::path::replace_extension(ObjFileName, "o");
107   return std::string(ObjFileName.str());
108 }
109 
110 /// Deduce the dependency target based on the output file and input files.
111 static std::string
112 deduceDepTarget(const std::string &OutputFile,
113                 const SmallVectorImpl<FrontendInputFile> &InputFiles) {
114   if (OutputFile != "-")
115     return OutputFile;
116 
117   if (InputFiles.empty() || !InputFiles.front().isFile())
118     return "clang-scan-deps\\ dependency";
119 
120   return makeObjFileName(InputFiles.front().getFile());
121 }
122 
123 /// Sanitize diagnostic options for dependency scan.
124 static void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
125   // Don't print 'X warnings and Y errors generated'.
126   DiagOpts.ShowCarets = false;
127   // Don't write out diagnostic file.
128   DiagOpts.DiagnosticSerializationFile.clear();
129   // Don't treat warnings as errors.
130   DiagOpts.Warnings.push_back("no-error");
131 }
132 
133 /// A clang tool that runs the preprocessor in a mode that's optimized for
134 /// dependency scanning for the given compiler invocation.
135 class DependencyScanningAction : public tooling::ToolAction {
136 public:
137   DependencyScanningAction(
138       StringRef WorkingDirectory, DependencyConsumer &Consumer,
139       llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
140       ExcludedPreprocessorDirectiveSkipMapping &PPSkipMappings,
141       ScanningOutputFormat Format, bool OptimizeArgs,
142       llvm::Optional<StringRef> ModuleName = None)
143       : WorkingDirectory(WorkingDirectory), Consumer(Consumer),
144         DepFS(std::move(DepFS)), PPSkipMappings(PPSkipMappings), Format(Format),
145         OptimizeArgs(OptimizeArgs), ModuleName(ModuleName) {}
146 
147   bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
148                      FileManager *FileMgr,
149                      std::shared_ptr<PCHContainerOperations> PCHContainerOps,
150                      DiagnosticConsumer *DiagConsumer) override {
151     // Make a deep copy of the original Clang invocation.
152     CompilerInvocation OriginalInvocation(*Invocation);
153 
154     // Create a compiler instance to handle the actual work.
155     CompilerInstance ScanInstance(std::move(PCHContainerOps));
156     ScanInstance.setInvocation(std::move(Invocation));
157 
158     // Create the compiler's actual diagnostics engine.
159     sanitizeDiagOpts(ScanInstance.getDiagnosticOpts());
160     ScanInstance.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
161     if (!ScanInstance.hasDiagnostics())
162       return false;
163 
164     ScanInstance.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
165         true;
166 
167     ScanInstance.getFrontendOpts().GenerateGlobalModuleIndex = false;
168     ScanInstance.getFrontendOpts().UseGlobalModuleIndex = false;
169 
170     FileMgr->getFileSystemOpts().WorkingDir = std::string(WorkingDirectory);
171     ScanInstance.setFileManager(FileMgr);
172     ScanInstance.createSourceManager(*FileMgr);
173 
174     llvm::StringSet<> PrebuiltModulesInputFiles;
175     // Store the list of prebuilt module files into header search options. This
176     // will prevent the implicit build to create duplicate modules and will
177     // force reuse of the existing prebuilt module files instead.
178     if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
179       visitPrebuiltModule(
180           ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance,
181           ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
182           PrebuiltModulesInputFiles, /*VisitInputFiles=*/DepFS != nullptr);
183 
184     // Use the dependency scanning optimized file system if requested to do so.
185     if (DepFS) {
186       DepFS->enableMinimizationOfAllFiles();
187       // Don't minimize any files that contributed to prebuilt modules. The
188       // implicit build validates the modules by comparing the reported sizes of
189       // their inputs to the current state of the filesystem. Minimization would
190       // throw this mechanism off.
191       for (const auto &File : PrebuiltModulesInputFiles)
192         DepFS->disableMinimization(File.getKey());
193       // Don't minimize any files that were explicitly passed in the build
194       // settings and that might be opened.
195       for (const auto &E : ScanInstance.getHeaderSearchOpts().UserEntries)
196         DepFS->disableMinimization(E.Path);
197       for (const auto &F : ScanInstance.getHeaderSearchOpts().VFSOverlayFiles)
198         DepFS->disableMinimization(F);
199 
200       // Support for virtual file system overlays on top of the caching
201       // filesystem.
202       FileMgr->setVirtualFileSystem(createVFSFromCompilerInvocation(
203           ScanInstance.getInvocation(), ScanInstance.getDiagnostics(), DepFS));
204 
205       // Pass the skip mappings which should speed up excluded conditional block
206       // skipping in the preprocessor.
207       ScanInstance.getPreprocessorOpts()
208           .ExcludedConditionalDirectiveSkipMappings = &PPSkipMappings;
209     }
210 
211     // Create the dependency collector that will collect the produced
212     // dependencies.
213     //
214     // This also moves the existing dependency output options from the
215     // invocation to the collector. The options in the invocation are reset,
216     // which ensures that the compiler won't create new dependency collectors,
217     // and thus won't write out the extra '.d' files to disk.
218     auto Opts = std::make_unique<DependencyOutputOptions>();
219     std::swap(*Opts, ScanInstance.getInvocation().getDependencyOutputOpts());
220     // We need at least one -MT equivalent for the generator of make dependency
221     // files to work.
222     if (Opts->Targets.empty())
223       Opts->Targets = {
224           deduceDepTarget(ScanInstance.getFrontendOpts().OutputFile,
225                           ScanInstance.getFrontendOpts().Inputs)};
226     Opts->IncludeSystemHeaders = true;
227 
228     switch (Format) {
229     case ScanningOutputFormat::Make:
230       ScanInstance.addDependencyCollector(
231           std::make_shared<DependencyConsumerForwarder>(std::move(Opts),
232                                                         Consumer));
233       break;
234     case ScanningOutputFormat::Full:
235       ScanInstance.addDependencyCollector(std::make_shared<ModuleDepCollector>(
236           std::move(Opts), ScanInstance, Consumer,
237           std::move(OriginalInvocation), OptimizeArgs));
238       break;
239     }
240 
241     // Consider different header search and diagnostic options to create
242     // different modules. This avoids the unsound aliasing of module PCMs.
243     //
244     // TODO: Implement diagnostic bucketing to reduce the impact of strict
245     // context hashing.
246     ScanInstance.getHeaderSearchOpts().ModulesStrictContextHash = true;
247 
248     std::unique_ptr<FrontendAction> Action;
249 
250     if (ModuleName.hasValue())
251       Action = std::make_unique<GetDependenciesByModuleNameAction>(*ModuleName);
252     else
253       Action = std::make_unique<ReadPCHAndPreprocessAction>();
254 
255     const bool Result = ScanInstance.ExecuteAction(*Action);
256     if (!DepFS)
257       FileMgr->clearStatCache();
258     return Result;
259   }
260 
261 private:
262   StringRef WorkingDirectory;
263   DependencyConsumer &Consumer;
264   llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
265   ExcludedPreprocessorDirectiveSkipMapping &PPSkipMappings;
266   ScanningOutputFormat Format;
267   bool OptimizeArgs;
268   llvm::Optional<StringRef> ModuleName;
269 };
270 
271 } // end anonymous namespace
272 
273 DependencyScanningWorker::DependencyScanningWorker(
274     DependencyScanningService &Service)
275     : Format(Service.getFormat()), OptimizeArgs(Service.canOptimizeArgs()) {
276   PCHContainerOps = std::make_shared<PCHContainerOperations>();
277   PCHContainerOps->registerReader(
278       std::make_unique<ObjectFilePCHContainerReader>());
279   // We don't need to write object files, but the current PCH implementation
280   // requires the writer to be registered as well.
281   PCHContainerOps->registerWriter(
282       std::make_unique<ObjectFilePCHContainerWriter>());
283 
284   auto OverlayFS = llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(
285       llvm::vfs::createPhysicalFileSystem());
286   InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
287   OverlayFS->pushOverlay(InMemoryFS);
288   RealFS = OverlayFS;
289 
290   if (Service.getMode() == ScanningMode::MinimizedSourcePreprocessing)
291     DepFS = new DependencyScanningWorkerFilesystem(Service.getSharedCache(),
292                                                    RealFS, PPSkipMappings);
293   if (Service.canReuseFileManager())
294     Files = new FileManager(FileSystemOptions(), RealFS);
295 }
296 
297 static llvm::Error
298 runWithDiags(DiagnosticOptions *DiagOpts,
299              llvm::function_ref<bool(DiagnosticConsumer &, DiagnosticOptions &)>
300                  BodyShouldSucceed) {
301   sanitizeDiagOpts(*DiagOpts);
302 
303   // Capture the emitted diagnostics and report them to the client
304   // in the case of a failure.
305   std::string DiagnosticOutput;
306   llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
307   TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts);
308 
309   if (BodyShouldSucceed(DiagPrinter, *DiagOpts))
310     return llvm::Error::success();
311   return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(),
312                                              llvm::inconvertibleErrorCode());
313 }
314 
315 llvm::Error DependencyScanningWorker::computeDependencies(
316     StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
317     DependencyConsumer &Consumer, llvm::Optional<StringRef> ModuleName) {
318   // Reset what might have been modified in the previous worker invocation.
319   RealFS->setCurrentWorkingDirectory(WorkingDirectory);
320   if (Files)
321     Files->setVirtualFileSystem(RealFS);
322 
323   llvm::IntrusiveRefCntPtr<FileManager> CurrentFiles =
324       Files ? Files : new FileManager(FileSystemOptions(), RealFS);
325 
326   Optional<std::vector<std::string>> ModifiedCommandLine;
327   if (ModuleName.hasValue()) {
328     ModifiedCommandLine = CommandLine;
329     InMemoryFS->addFile(*ModuleName, 0, llvm::MemoryBuffer::getMemBuffer(""));
330     ModifiedCommandLine->emplace_back(*ModuleName);
331   }
332 
333   const std::vector<std::string> &FinalCommandLine =
334       ModifiedCommandLine ? *ModifiedCommandLine : CommandLine;
335 
336   std::vector<const char *> FinalCCommandLine(CommandLine.size(), nullptr);
337   llvm::transform(CommandLine, FinalCCommandLine.begin(),
338                   [](const std::string &Str) { return Str.c_str(); });
339 
340   return runWithDiags(CreateAndPopulateDiagOpts(FinalCCommandLine).release(),
341                       [&](DiagnosticConsumer &DC, DiagnosticOptions &DiagOpts) {
342                         DependencyScanningAction Action(
343                             WorkingDirectory, Consumer, DepFS, PPSkipMappings,
344                             Format, OptimizeArgs, ModuleName);
345                         // Create an invocation that uses the underlying file
346                         // system to ensure that any file system requests that
347                         // are made by the driver do not go through the
348                         // dependency scanning filesystem.
349                         ToolInvocation Invocation(FinalCommandLine, &Action,
350                                                   CurrentFiles.get(),
351                                                   PCHContainerOps);
352                         Invocation.setDiagnosticConsumer(&DC);
353                         Invocation.setDiagnosticOptions(&DiagOpts);
354                         return Invocation.run();
355                       });
356 }
357