1 //===- DependencyScanningWorker.cpp - clang-scan-deps worker --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
10 #include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
11 #include "clang/Frontend/CompilerInstance.h"
12 #include "clang/Frontend/CompilerInvocation.h"
13 #include "clang/Frontend/FrontendActions.h"
14 #include "clang/Frontend/TextDiagnosticPrinter.h"
15 #include "clang/Frontend/Utils.h"
16 #include "clang/Lex/PreprocessorOptions.h"
17 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
18 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
19 #include "clang/Tooling/Tooling.h"
20 
21 using namespace clang;
22 using namespace tooling;
23 using namespace dependencies;
24 
25 namespace {
26 
27 /// Forwards the gatherered dependencies to the consumer.
28 class DependencyConsumerForwarder : public DependencyFileGenerator {
29 public:
30   DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts,
31                               DependencyConsumer &C)
32       : DependencyFileGenerator(*Opts), Opts(std::move(Opts)), C(C) {}
33 
34   void finishedMainFile(DiagnosticsEngine &Diags) override {
35     C.handleDependencyOutputOpts(*Opts);
36     llvm::SmallString<256> CanonPath;
37     for (const auto &File : getDependencies()) {
38       CanonPath = File;
39       llvm::sys::path::remove_dots(CanonPath, /*remove_dot_dot=*/true);
40       C.handleFileDependency(CanonPath);
41     }
42   }
43 
44 private:
45   std::unique_ptr<DependencyOutputOptions> Opts;
46   DependencyConsumer &C;
47 };
48 
49 /// A listener that collects the imported modules and optionally the input
50 /// files.
51 class PrebuiltModuleListener : public ASTReaderListener {
52 public:
53   PrebuiltModuleListener(llvm::StringMap<std::string> &PrebuiltModuleFiles,
54                          llvm::StringSet<> &InputFiles, bool VisitInputFiles)
55       : PrebuiltModuleFiles(PrebuiltModuleFiles), InputFiles(InputFiles),
56         VisitInputFiles(VisitInputFiles) {}
57 
58   bool needsImportVisitation() const override { return true; }
59   bool needsInputFileVisitation() override { return VisitInputFiles; }
60   bool needsSystemInputFileVisitation() override { return VisitInputFiles; }
61 
62   void visitImport(StringRef ModuleName, StringRef Filename) override {
63     PrebuiltModuleFiles.insert({ModuleName, Filename.str()});
64   }
65 
66   bool visitInputFile(StringRef Filename, bool isSystem, bool isOverridden,
67                       bool isExplicitModule) override {
68     InputFiles.insert(Filename);
69     return true;
70   }
71 
72 private:
73   llvm::StringMap<std::string> &PrebuiltModuleFiles;
74   llvm::StringSet<> &InputFiles;
75   bool VisitInputFiles;
76 };
77 
78 using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
79 
80 /// Visit the given prebuilt module and collect all of the modules it
81 /// transitively imports and contributing input files.
82 static void visitPrebuiltModule(StringRef PrebuiltModuleFilename,
83                                 CompilerInstance &CI,
84                                 PrebuiltModuleFilesT &ModuleFiles,
85                                 llvm::StringSet<> &InputFiles,
86                                 bool VisitInputFiles) {
87   // Maps the names of modules that weren't yet visited to their PCM path.
88   llvm::StringMap<std::string> ModuleFilesWorklist;
89   // Contains PCM paths of all visited modules.
90   llvm::StringSet<> VisitedModuleFiles;
91 
92   PrebuiltModuleListener Listener(ModuleFilesWorklist, InputFiles,
93                                   VisitInputFiles);
94 
95   auto GatherModuleFileInfo = [&](StringRef ASTFile) {
96     ASTReader::readASTFileControlBlock(
97         ASTFile, CI.getFileManager(), CI.getPCHContainerReader(),
98         /*FindModuleFileExtensions=*/false, Listener,
99         /*ValidateDiagnosticOptions=*/false);
100   };
101 
102   GatherModuleFileInfo(PrebuiltModuleFilename);
103   while (!ModuleFilesWorklist.empty()) {
104     auto WorklistItemIt = ModuleFilesWorklist.begin();
105 
106     if (!VisitedModuleFiles.contains(WorklistItemIt->getValue())) {
107       VisitedModuleFiles.insert(WorklistItemIt->getValue());
108       GatherModuleFileInfo(WorklistItemIt->getValue());
109       ModuleFiles[WorklistItemIt->getKey().str()] = WorklistItemIt->getValue();
110     }
111 
112     ModuleFilesWorklist.erase(WorklistItemIt);
113   }
114 }
115 
116 /// Transform arbitrary file name into an object-like file name.
117 static std::string makeObjFileName(StringRef FileName) {
118   SmallString<128> ObjFileName(FileName);
119   llvm::sys::path::replace_extension(ObjFileName, "o");
120   return std::string(ObjFileName.str());
121 }
122 
123 /// Deduce the dependency target based on the output file and input files.
124 static std::string
125 deduceDepTarget(const std::string &OutputFile,
126                 const SmallVectorImpl<FrontendInputFile> &InputFiles) {
127   if (OutputFile != "-")
128     return OutputFile;
129 
130   if (InputFiles.empty() || !InputFiles.front().isFile())
131     return "clang-scan-deps\\ dependency";
132 
133   return makeObjFileName(InputFiles.front().getFile());
134 }
135 
136 /// Sanitize diagnostic options for dependency scan.
137 static void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
138   // Don't print 'X warnings and Y errors generated'.
139   DiagOpts.ShowCarets = false;
140   // Don't write out diagnostic file.
141   DiagOpts.DiagnosticSerializationFile.clear();
142   // Don't treat warnings as errors.
143   DiagOpts.Warnings.push_back("no-error");
144 }
145 
146 /// A clang tool that runs the preprocessor in a mode that's optimized for
147 /// dependency scanning for the given compiler invocation.
148 class DependencyScanningAction : public tooling::ToolAction {
149 public:
150   DependencyScanningAction(
151       StringRef WorkingDirectory, DependencyConsumer &Consumer,
152       llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
153       ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings,
154       ScanningOutputFormat Format, llvm::Optional<StringRef> ModuleName = None)
155       : WorkingDirectory(WorkingDirectory), Consumer(Consumer),
156         DepFS(std::move(DepFS)), PPSkipMappings(PPSkipMappings), Format(Format),
157         ModuleName(ModuleName) {}
158 
159   bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
160                      FileManager *FileMgr,
161                      std::shared_ptr<PCHContainerOperations> PCHContainerOps,
162                      DiagnosticConsumer *DiagConsumer) override {
163     // Make a deep copy of the original Clang invocation.
164     CompilerInvocation OriginalInvocation(*Invocation);
165 
166     // Create a compiler instance to handle the actual work.
167     CompilerInstance Compiler(std::move(PCHContainerOps));
168     Compiler.setInvocation(std::move(Invocation));
169 
170     // Create the compiler's actual diagnostics engine.
171     sanitizeDiagOpts(Compiler.getDiagnosticOpts());
172     Compiler.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
173     if (!Compiler.hasDiagnostics())
174       return false;
175 
176     Compiler.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath = true;
177 
178     FileMgr->getFileSystemOpts().WorkingDir = std::string(WorkingDirectory);
179     Compiler.setFileManager(FileMgr);
180     Compiler.createSourceManager(*FileMgr);
181 
182     llvm::StringSet<> PrebuiltModulesInputFiles;
183     // Store the list of prebuilt module files into header search options. This
184     // will prevent the implicit build to create duplicate modules and will
185     // force reuse of the existing prebuilt module files instead.
186     if (!Compiler.getPreprocessorOpts().ImplicitPCHInclude.empty())
187       visitPrebuiltModule(
188           Compiler.getPreprocessorOpts().ImplicitPCHInclude, Compiler,
189           Compiler.getHeaderSearchOpts().PrebuiltModuleFiles,
190           PrebuiltModulesInputFiles, /*VisitInputFiles=*/DepFS != nullptr);
191 
192     // Use the dependency scanning optimized file system if requested to do so.
193     if (DepFS) {
194       const CompilerInvocation &CI = Compiler.getInvocation();
195       DepFS->clearIgnoredFiles();
196       // Ignore any files that contributed to prebuilt modules. The implicit
197       // build validates the modules by comparing the reported sizes of their
198       // inputs to the current state of the filesystem. Minimization would throw
199       // this mechanism off.
200       for (const auto &File : PrebuiltModulesInputFiles)
201         DepFS->ignoreFile(File.getKey());
202       // Add any filenames that were explicity passed in the build settings and
203       // that might be opened, as we want to ensure we don't run source
204       // minimization on them.
205       for (const auto &Entry : CI.getHeaderSearchOpts().UserEntries)
206         DepFS->ignoreFile(Entry.Path);
207       for (const auto &Entry : CI.getHeaderSearchOpts().VFSOverlayFiles)
208         DepFS->ignoreFile(Entry);
209 
210       // Support for virtual file system overlays on top of the caching
211       // filesystem.
212       FileMgr->setVirtualFileSystem(createVFSFromCompilerInvocation(
213           CI, Compiler.getDiagnostics(), DepFS));
214 
215       // Pass the skip mappings which should speed up excluded conditional block
216       // skipping in the preprocessor.
217       if (PPSkipMappings)
218         Compiler.getPreprocessorOpts()
219             .ExcludedConditionalDirectiveSkipMappings = PPSkipMappings;
220     }
221 
222     // Create the dependency collector that will collect the produced
223     // dependencies.
224     //
225     // This also moves the existing dependency output options from the
226     // invocation to the collector. The options in the invocation are reset,
227     // which ensures that the compiler won't create new dependency collectors,
228     // and thus won't write out the extra '.d' files to disk.
229     auto Opts = std::make_unique<DependencyOutputOptions>();
230     std::swap(*Opts, Compiler.getInvocation().getDependencyOutputOpts());
231     // We need at least one -MT equivalent for the generator of make dependency
232     // files to work.
233     if (Opts->Targets.empty())
234       Opts->Targets = {deduceDepTarget(Compiler.getFrontendOpts().OutputFile,
235                                        Compiler.getFrontendOpts().Inputs)};
236     Opts->IncludeSystemHeaders = true;
237 
238     switch (Format) {
239     case ScanningOutputFormat::Make:
240       Compiler.addDependencyCollector(
241           std::make_shared<DependencyConsumerForwarder>(std::move(Opts),
242                                                         Consumer));
243       break;
244     case ScanningOutputFormat::Full:
245       Compiler.addDependencyCollector(std::make_shared<ModuleDepCollector>(
246           std::move(Opts), Compiler, Consumer, std::move(OriginalInvocation)));
247       break;
248     }
249 
250     // Consider different header search and diagnostic options to create
251     // different modules. This avoids the unsound aliasing of module PCMs.
252     //
253     // TODO: Implement diagnostic bucketing and header search pruning to reduce
254     // the impact of strict context hashing.
255     Compiler.getHeaderSearchOpts().ModulesStrictContextHash = true;
256 
257     std::unique_ptr<FrontendAction> Action;
258 
259     if (ModuleName.hasValue())
260       Action = std::make_unique<GetDependenciesByModuleNameAction>(*ModuleName);
261     else
262       Action = std::make_unique<ReadPCHAndPreprocessAction>();
263 
264     const bool Result = Compiler.ExecuteAction(*Action);
265     if (!DepFS)
266       FileMgr->clearStatCache();
267     return Result;
268   }
269 
270 private:
271   StringRef WorkingDirectory;
272   DependencyConsumer &Consumer;
273   llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
274   ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings;
275   ScanningOutputFormat Format;
276   llvm::Optional<StringRef> ModuleName;
277 };
278 
279 } // end anonymous namespace
280 
281 DependencyScanningWorker::DependencyScanningWorker(
282     DependencyScanningService &Service)
283     : Format(Service.getFormat()) {
284   PCHContainerOps = std::make_shared<PCHContainerOperations>();
285   PCHContainerOps->registerReader(
286       std::make_unique<ObjectFilePCHContainerReader>());
287   // We don't need to write object files, but the current PCH implementation
288   // requires the writer to be registered as well.
289   PCHContainerOps->registerWriter(
290       std::make_unique<ObjectFilePCHContainerWriter>());
291 
292   auto OverlayFS = llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(
293       llvm::vfs::createPhysicalFileSystem());
294   InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
295   OverlayFS->pushOverlay(InMemoryFS);
296   RealFS = OverlayFS;
297 
298   if (Service.canSkipExcludedPPRanges())
299     PPSkipMappings =
300         std::make_unique<ExcludedPreprocessorDirectiveSkipMapping>();
301   if (Service.getMode() == ScanningMode::MinimizedSourcePreprocessing)
302     DepFS = new DependencyScanningWorkerFilesystem(
303         Service.getSharedCache(), RealFS, PPSkipMappings.get());
304   if (Service.canReuseFileManager())
305     Files = new FileManager(FileSystemOptions(), RealFS);
306 }
307 
308 static llvm::Error
309 runWithDiags(DiagnosticOptions *DiagOpts,
310              llvm::function_ref<bool(DiagnosticConsumer &, DiagnosticOptions &)>
311                  BodyShouldSucceed) {
312   sanitizeDiagOpts(*DiagOpts);
313 
314   // Capture the emitted diagnostics and report them to the client
315   // in the case of a failure.
316   std::string DiagnosticOutput;
317   llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
318   TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts);
319 
320   if (BodyShouldSucceed(DiagPrinter, *DiagOpts))
321     return llvm::Error::success();
322   return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(),
323                                              llvm::inconvertibleErrorCode());
324 }
325 
326 llvm::Error DependencyScanningWorker::computeDependencies(
327     StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
328     DependencyConsumer &Consumer, llvm::Optional<StringRef> ModuleName) {
329   // Reset what might have been modified in the previous worker invocation.
330   RealFS->setCurrentWorkingDirectory(WorkingDirectory);
331   if (Files)
332     Files->setVirtualFileSystem(RealFS);
333 
334   llvm::IntrusiveRefCntPtr<FileManager> CurrentFiles =
335       Files ? Files : new FileManager(FileSystemOptions(), RealFS);
336 
337   Optional<std::vector<std::string>> ModifiedCommandLine;
338   if (ModuleName.hasValue()) {
339     ModifiedCommandLine = CommandLine;
340     InMemoryFS->addFile(*ModuleName, 0, llvm::MemoryBuffer::getMemBuffer(""));
341     ModifiedCommandLine->emplace_back(*ModuleName);
342   }
343 
344   const std::vector<std::string> &FinalCommandLine =
345       ModifiedCommandLine ? *ModifiedCommandLine : CommandLine;
346 
347   std::vector<const char *> FinalCCommandLine(CommandLine.size(), nullptr);
348   llvm::transform(CommandLine, FinalCCommandLine.begin(),
349                   [](const std::string &Str) { return Str.c_str(); });
350 
351   return runWithDiags(CreateAndPopulateDiagOpts(FinalCCommandLine).release(),
352                       [&](DiagnosticConsumer &DC, DiagnosticOptions &DiagOpts) {
353                         DependencyScanningAction Action(
354                             WorkingDirectory, Consumer, DepFS,
355                             PPSkipMappings.get(), Format, ModuleName);
356                         // Create an invocation that uses the underlying file
357                         // system to ensure that any file system requests that
358                         // are made by the driver do not go through the
359                         // dependency scanning filesystem.
360                         ToolInvocation Invocation(FinalCommandLine, &Action,
361                                                   CurrentFiles.get(),
362                                                   PCHContainerOps);
363                         Invocation.setDiagnosticConsumer(&DC);
364                         Invocation.setDiagnosticOptions(&DiagOpts);
365                         return Invocation.run();
366                       });
367 }
368