1 //===- DependencyScanningWorker.cpp - clang-scan-deps worker --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
10 #include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
11 #include "clang/Frontend/CompilerInstance.h"
12 #include "clang/Frontend/CompilerInvocation.h"
13 #include "clang/Frontend/FrontendActions.h"
14 #include "clang/Frontend/TextDiagnosticPrinter.h"
15 #include "clang/Frontend/Utils.h"
16 #include "clang/Lex/PreprocessorOptions.h"
17 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
18 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
19 #include "clang/Tooling/Tooling.h"
20 
21 using namespace clang;
22 using namespace tooling;
23 using namespace dependencies;
24 
25 namespace {
26 
27 /// Forwards the gatherered dependencies to the consumer.
28 class DependencyConsumerForwarder : public DependencyFileGenerator {
29 public:
30   DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts,
31                               DependencyConsumer &C)
32       : DependencyFileGenerator(*Opts), Opts(std::move(Opts)), C(C) {}
33 
34   void finishedMainFile(DiagnosticsEngine &Diags) override {
35     C.handleDependencyOutputOpts(*Opts);
36     llvm::SmallString<256> CanonPath;
37     for (const auto &File : getDependencies()) {
38       CanonPath = File;
39       llvm::sys::path::remove_dots(CanonPath, /*remove_dot_dot=*/true);
40       C.handleFileDependency(CanonPath);
41     }
42   }
43 
44 private:
45   std::unique_ptr<DependencyOutputOptions> Opts;
46   DependencyConsumer &C;
47 };
48 
49 using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
50 
51 /// A listener that collects the imported modules and optionally the input
52 /// files.
53 class PrebuiltModuleListener : public ASTReaderListener {
54 public:
55   PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles,
56                          llvm::StringSet<> &InputFiles, bool VisitInputFiles,
57                          llvm::SmallVector<std::string> &NewModuleFiles)
58       : PrebuiltModuleFiles(PrebuiltModuleFiles), InputFiles(InputFiles),
59         VisitInputFiles(VisitInputFiles), NewModuleFiles(NewModuleFiles) {}
60 
61   bool needsImportVisitation() const override { return true; }
62   bool needsInputFileVisitation() override { return VisitInputFiles; }
63   bool needsSystemInputFileVisitation() override { return VisitInputFiles; }
64 
65   void visitImport(StringRef ModuleName, StringRef Filename) override {
66     if (PrebuiltModuleFiles.insert({ModuleName.str(), Filename.str()}).second)
67       NewModuleFiles.push_back(Filename.str());
68   }
69 
70   bool visitInputFile(StringRef Filename, bool isSystem, bool isOverridden,
71                       bool isExplicitModule) override {
72     InputFiles.insert(Filename);
73     return true;
74   }
75 
76 private:
77   PrebuiltModuleFilesT &PrebuiltModuleFiles;
78   llvm::StringSet<> &InputFiles;
79   bool VisitInputFiles;
80   llvm::SmallVector<std::string> &NewModuleFiles;
81 };
82 
83 /// Visit the given prebuilt module and collect all of the modules it
84 /// transitively imports and contributing input files.
85 static void visitPrebuiltModule(StringRef PrebuiltModuleFilename,
86                                 CompilerInstance &CI,
87                                 PrebuiltModuleFilesT &ModuleFiles,
88                                 llvm::StringSet<> &InputFiles,
89                                 bool VisitInputFiles) {
90   // List of module files to be processed.
91   llvm::SmallVector<std::string> Worklist{PrebuiltModuleFilename.str()};
92   PrebuiltModuleListener Listener(ModuleFiles, InputFiles, VisitInputFiles,
93                                   Worklist);
94 
95   while (!Worklist.empty())
96     ASTReader::readASTFileControlBlock(
97         Worklist.pop_back_val(), CI.getFileManager(),
98         CI.getPCHContainerReader(),
99         /*FindModuleFileExtensions=*/false, Listener,
100         /*ValidateDiagnosticOptions=*/false);
101 }
102 
103 /// Transform arbitrary file name into an object-like file name.
104 static std::string makeObjFileName(StringRef FileName) {
105   SmallString<128> ObjFileName(FileName);
106   llvm::sys::path::replace_extension(ObjFileName, "o");
107   return std::string(ObjFileName.str());
108 }
109 
110 /// Deduce the dependency target based on the output file and input files.
111 static std::string
112 deduceDepTarget(const std::string &OutputFile,
113                 const SmallVectorImpl<FrontendInputFile> &InputFiles) {
114   if (OutputFile != "-")
115     return OutputFile;
116 
117   if (InputFiles.empty() || !InputFiles.front().isFile())
118     return "clang-scan-deps\\ dependency";
119 
120   return makeObjFileName(InputFiles.front().getFile());
121 }
122 
123 /// Sanitize diagnostic options for dependency scan.
124 static void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
125   // Don't print 'X warnings and Y errors generated'.
126   DiagOpts.ShowCarets = false;
127   // Don't write out diagnostic file.
128   DiagOpts.DiagnosticSerializationFile.clear();
129   // Don't treat warnings as errors.
130   DiagOpts.Warnings.push_back("no-error");
131 }
132 
133 /// A clang tool that runs the preprocessor in a mode that's optimized for
134 /// dependency scanning for the given compiler invocation.
135 class DependencyScanningAction : public tooling::ToolAction {
136 public:
137   DependencyScanningAction(
138       StringRef WorkingDirectory, DependencyConsumer &Consumer,
139       llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
140       ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings,
141       ScanningOutputFormat Format, bool OptimizeArgs,
142       llvm::Optional<StringRef> ModuleName = None)
143       : WorkingDirectory(WorkingDirectory), Consumer(Consumer),
144         DepFS(std::move(DepFS)), PPSkipMappings(PPSkipMappings), Format(Format),
145         OptimizeArgs(OptimizeArgs), ModuleName(ModuleName) {}
146 
147   bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
148                      FileManager *FileMgr,
149                      std::shared_ptr<PCHContainerOperations> PCHContainerOps,
150                      DiagnosticConsumer *DiagConsumer) override {
151     // Make a deep copy of the original Clang invocation.
152     CompilerInvocation OriginalInvocation(*Invocation);
153 
154     // Create a compiler instance to handle the actual work.
155     CompilerInstance ScanInstance(std::move(PCHContainerOps));
156     ScanInstance.setInvocation(std::move(Invocation));
157 
158     // Create the compiler's actual diagnostics engine.
159     sanitizeDiagOpts(ScanInstance.getDiagnosticOpts());
160     ScanInstance.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
161     if (!ScanInstance.hasDiagnostics())
162       return false;
163 
164     ScanInstance.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
165         true;
166 
167     ScanInstance.getFrontendOpts().GenerateGlobalModuleIndex = false;
168     ScanInstance.getFrontendOpts().UseGlobalModuleIndex = false;
169 
170     FileMgr->getFileSystemOpts().WorkingDir = std::string(WorkingDirectory);
171     ScanInstance.setFileManager(FileMgr);
172     ScanInstance.createSourceManager(*FileMgr);
173 
174     llvm::StringSet<> PrebuiltModulesInputFiles;
175     // Store the list of prebuilt module files into header search options. This
176     // will prevent the implicit build to create duplicate modules and will
177     // force reuse of the existing prebuilt module files instead.
178     if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
179       visitPrebuiltModule(
180           ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance,
181           ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
182           PrebuiltModulesInputFiles, /*VisitInputFiles=*/DepFS != nullptr);
183 
184     // Use the dependency scanning optimized file system if requested to do so.
185     if (DepFS) {
186       DepFS->enableMinimizationOfAllFiles();
187       // Don't minimize any files that contributed to prebuilt modules. The
188       // implicit build validates the modules by comparing the reported sizes of
189       // their inputs to the current state of the filesystem. Minimization would
190       // throw this mechanism off.
191       for (const auto &File : PrebuiltModulesInputFiles)
192         DepFS->disableMinimization(File.getKey());
193       // Don't minimize any files that were explicitly passed in the build
194       // settings and that might be opened.
195       for (const auto &E : ScanInstance.getHeaderSearchOpts().UserEntries)
196         DepFS->disableMinimization(E.Path);
197       for (const auto &F : ScanInstance.getHeaderSearchOpts().VFSOverlayFiles)
198         DepFS->disableMinimization(F);
199 
200       // Support for virtual file system overlays on top of the caching
201       // filesystem.
202       FileMgr->setVirtualFileSystem(createVFSFromCompilerInvocation(
203           ScanInstance.getInvocation(), ScanInstance.getDiagnostics(), DepFS));
204 
205       // Pass the skip mappings which should speed up excluded conditional block
206       // skipping in the preprocessor.
207       if (PPSkipMappings)
208         ScanInstance.getPreprocessorOpts()
209             .ExcludedConditionalDirectiveSkipMappings = PPSkipMappings;
210     }
211 
212     // Create the dependency collector that will collect the produced
213     // dependencies.
214     //
215     // This also moves the existing dependency output options from the
216     // invocation to the collector. The options in the invocation are reset,
217     // which ensures that the compiler won't create new dependency collectors,
218     // and thus won't write out the extra '.d' files to disk.
219     auto Opts = std::make_unique<DependencyOutputOptions>();
220     std::swap(*Opts, ScanInstance.getInvocation().getDependencyOutputOpts());
221     // We need at least one -MT equivalent for the generator of make dependency
222     // files to work.
223     if (Opts->Targets.empty())
224       Opts->Targets = {
225           deduceDepTarget(ScanInstance.getFrontendOpts().OutputFile,
226                           ScanInstance.getFrontendOpts().Inputs)};
227     Opts->IncludeSystemHeaders = true;
228 
229     switch (Format) {
230     case ScanningOutputFormat::Make:
231       ScanInstance.addDependencyCollector(
232           std::make_shared<DependencyConsumerForwarder>(std::move(Opts),
233                                                         Consumer));
234       break;
235     case ScanningOutputFormat::Full:
236       ScanInstance.addDependencyCollector(std::make_shared<ModuleDepCollector>(
237           std::move(Opts), ScanInstance, Consumer,
238           std::move(OriginalInvocation), OptimizeArgs));
239       break;
240     }
241 
242     // Consider different header search and diagnostic options to create
243     // different modules. This avoids the unsound aliasing of module PCMs.
244     //
245     // TODO: Implement diagnostic bucketing to reduce the impact of strict
246     // context hashing.
247     ScanInstance.getHeaderSearchOpts().ModulesStrictContextHash = true;
248 
249     std::unique_ptr<FrontendAction> Action;
250 
251     if (ModuleName.hasValue())
252       Action = std::make_unique<GetDependenciesByModuleNameAction>(*ModuleName);
253     else
254       Action = std::make_unique<ReadPCHAndPreprocessAction>();
255 
256     const bool Result = ScanInstance.ExecuteAction(*Action);
257     if (!DepFS)
258       FileMgr->clearStatCache();
259     return Result;
260   }
261 
262 private:
263   StringRef WorkingDirectory;
264   DependencyConsumer &Consumer;
265   llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
266   ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings;
267   ScanningOutputFormat Format;
268   bool OptimizeArgs;
269   llvm::Optional<StringRef> ModuleName;
270 };
271 
272 } // end anonymous namespace
273 
274 DependencyScanningWorker::DependencyScanningWorker(
275     DependencyScanningService &Service)
276     : Format(Service.getFormat()), OptimizeArgs(Service.canOptimizeArgs()) {
277   PCHContainerOps = std::make_shared<PCHContainerOperations>();
278   PCHContainerOps->registerReader(
279       std::make_unique<ObjectFilePCHContainerReader>());
280   // We don't need to write object files, but the current PCH implementation
281   // requires the writer to be registered as well.
282   PCHContainerOps->registerWriter(
283       std::make_unique<ObjectFilePCHContainerWriter>());
284 
285   auto OverlayFS = llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(
286       llvm::vfs::createPhysicalFileSystem());
287   InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
288   OverlayFS->pushOverlay(InMemoryFS);
289   RealFS = OverlayFS;
290 
291   if (Service.canSkipExcludedPPRanges())
292     PPSkipMappings =
293         std::make_unique<ExcludedPreprocessorDirectiveSkipMapping>();
294   if (Service.getMode() == ScanningMode::MinimizedSourcePreprocessing)
295     DepFS = new DependencyScanningWorkerFilesystem(
296         Service.getSharedCache(), RealFS, PPSkipMappings.get());
297   if (Service.canReuseFileManager())
298     Files = new FileManager(FileSystemOptions(), RealFS);
299 }
300 
301 static llvm::Error
302 runWithDiags(DiagnosticOptions *DiagOpts,
303              llvm::function_ref<bool(DiagnosticConsumer &, DiagnosticOptions &)>
304                  BodyShouldSucceed) {
305   sanitizeDiagOpts(*DiagOpts);
306 
307   // Capture the emitted diagnostics and report them to the client
308   // in the case of a failure.
309   std::string DiagnosticOutput;
310   llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
311   TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts);
312 
313   if (BodyShouldSucceed(DiagPrinter, *DiagOpts))
314     return llvm::Error::success();
315   return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(),
316                                              llvm::inconvertibleErrorCode());
317 }
318 
319 llvm::Error DependencyScanningWorker::computeDependencies(
320     StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
321     DependencyConsumer &Consumer, llvm::Optional<StringRef> ModuleName) {
322   // Reset what might have been modified in the previous worker invocation.
323   RealFS->setCurrentWorkingDirectory(WorkingDirectory);
324   if (Files)
325     Files->setVirtualFileSystem(RealFS);
326 
327   llvm::IntrusiveRefCntPtr<FileManager> CurrentFiles =
328       Files ? Files : new FileManager(FileSystemOptions(), RealFS);
329 
330   Optional<std::vector<std::string>> ModifiedCommandLine;
331   if (ModuleName.hasValue()) {
332     ModifiedCommandLine = CommandLine;
333     InMemoryFS->addFile(*ModuleName, 0, llvm::MemoryBuffer::getMemBuffer(""));
334     ModifiedCommandLine->emplace_back(*ModuleName);
335   }
336 
337   const std::vector<std::string> &FinalCommandLine =
338       ModifiedCommandLine ? *ModifiedCommandLine : CommandLine;
339 
340   std::vector<const char *> FinalCCommandLine(CommandLine.size(), nullptr);
341   llvm::transform(CommandLine, FinalCCommandLine.begin(),
342                   [](const std::string &Str) { return Str.c_str(); });
343 
344   return runWithDiags(CreateAndPopulateDiagOpts(FinalCCommandLine).release(),
345                       [&](DiagnosticConsumer &DC, DiagnosticOptions &DiagOpts) {
346                         DependencyScanningAction Action(
347                             WorkingDirectory, Consumer, DepFS,
348                             PPSkipMappings.get(), Format, OptimizeArgs,
349                             ModuleName);
350                         // Create an invocation that uses the underlying file
351                         // system to ensure that any file system requests that
352                         // are made by the driver do not go through the
353                         // dependency scanning filesystem.
354                         ToolInvocation Invocation(FinalCommandLine, &Action,
355                                                   CurrentFiles.get(),
356                                                   PCHContainerOps);
357                         Invocation.setDiagnosticConsumer(&DC);
358                         Invocation.setDiagnosticOptions(&DiagOpts);
359                         return Invocation.run();
360                       });
361 }
362