1 //===- DependencyScanningWorker.cpp - clang-scan-deps worker --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
10 #include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
11 #include "clang/Frontend/CompilerInstance.h"
12 #include "clang/Frontend/CompilerInvocation.h"
13 #include "clang/Frontend/FrontendActions.h"
14 #include "clang/Frontend/TextDiagnosticPrinter.h"
15 #include "clang/Frontend/Utils.h"
16 #include "clang/Lex/PreprocessorOptions.h"
17 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
18 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
19 #include "clang/Tooling/Tooling.h"
20 
21 using namespace clang;
22 using namespace tooling;
23 using namespace dependencies;
24 
25 namespace {
26 
27 /// Forwards the gatherered dependencies to the consumer.
28 class DependencyConsumerForwarder : public DependencyFileGenerator {
29 public:
30   DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts,
31                               DependencyConsumer &C)
32       : DependencyFileGenerator(*Opts), Opts(std::move(Opts)), C(C) {}
33 
34   void finishedMainFile(DiagnosticsEngine &Diags) override {
35     C.handleDependencyOutputOpts(*Opts);
36     llvm::SmallString<256> CanonPath;
37     for (const auto &File : getDependencies()) {
38       CanonPath = File;
39       llvm::sys::path::remove_dots(CanonPath, /*remove_dot_dot=*/true);
40       C.handleFileDependency(CanonPath);
41     }
42   }
43 
44 private:
45   std::unique_ptr<DependencyOutputOptions> Opts;
46   DependencyConsumer &C;
47 };
48 
49 /// A listener that collects the imported modules and optionally the input
50 /// files.
51 class PrebuiltModuleListener : public ASTReaderListener {
52 public:
53   PrebuiltModuleListener(llvm::StringMap<std::string> &PrebuiltModuleFiles,
54                          llvm::StringSet<> &InputFiles, bool VisitInputFiles)
55       : PrebuiltModuleFiles(PrebuiltModuleFiles), InputFiles(InputFiles),
56         VisitInputFiles(VisitInputFiles) {}
57 
58   bool needsImportVisitation() const override { return true; }
59   bool needsInputFileVisitation() override { return VisitInputFiles; }
60   bool needsSystemInputFileVisitation() override { return VisitInputFiles; }
61 
62   void visitImport(StringRef ModuleName, StringRef Filename) override {
63     PrebuiltModuleFiles.insert({ModuleName, Filename.str()});
64   }
65 
66   bool visitInputFile(StringRef Filename, bool isSystem, bool isOverridden,
67                       bool isExplicitModule) override {
68     InputFiles.insert(Filename);
69     return true;
70   }
71 
72 private:
73   llvm::StringMap<std::string> &PrebuiltModuleFiles;
74   llvm::StringSet<> &InputFiles;
75   bool VisitInputFiles;
76 };
77 
78 using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
79 
80 /// Visit the given prebuilt module and collect all of the modules it
81 /// transitively imports and contributing input files.
82 static void visitPrebuiltModule(StringRef PrebuiltModuleFilename,
83                                 CompilerInstance &CI,
84                                 PrebuiltModuleFilesT &ModuleFiles,
85                                 llvm::StringSet<> &InputFiles,
86                                 bool VisitInputFiles) {
87   // Maps the names of modules that weren't yet visited to their PCM path.
88   llvm::StringMap<std::string> ModuleFilesWorklist;
89   // Contains PCM paths of all visited modules.
90   llvm::StringSet<> VisitedModuleFiles;
91 
92   PrebuiltModuleListener Listener(ModuleFilesWorklist, InputFiles,
93                                   VisitInputFiles);
94 
95   auto GatherModuleFileInfo = [&](StringRef ASTFile) {
96     ASTReader::readASTFileControlBlock(
97         ASTFile, CI.getFileManager(), CI.getPCHContainerReader(),
98         /*FindModuleFileExtensions=*/false, Listener,
99         /*ValidateDiagnosticOptions=*/false);
100   };
101 
102   GatherModuleFileInfo(PrebuiltModuleFilename);
103   while (!ModuleFilesWorklist.empty()) {
104     auto WorklistItemIt = ModuleFilesWorklist.begin();
105 
106     if (!VisitedModuleFiles.contains(WorklistItemIt->getValue())) {
107       VisitedModuleFiles.insert(WorklistItemIt->getValue());
108       GatherModuleFileInfo(WorklistItemIt->getValue());
109       ModuleFiles[WorklistItemIt->getKey().str()] = WorklistItemIt->getValue();
110     }
111 
112     ModuleFilesWorklist.erase(WorklistItemIt);
113   }
114 }
115 
116 /// Transform arbitrary file name into an object-like file name.
117 static std::string makeObjFileName(StringRef FileName) {
118   SmallString<128> ObjFileName(FileName);
119   llvm::sys::path::replace_extension(ObjFileName, "o");
120   return std::string(ObjFileName.str());
121 }
122 
123 /// Deduce the dependency target based on the output file and input files.
124 static std::string
125 deduceDepTarget(const std::string &OutputFile,
126                 const SmallVectorImpl<FrontendInputFile> &InputFiles) {
127   if (OutputFile != "-")
128     return OutputFile;
129 
130   if (InputFiles.empty() || !InputFiles.front().isFile())
131     return "clang-scan-deps\\ dependency";
132 
133   return makeObjFileName(InputFiles.front().getFile());
134 }
135 
136 /// Sanitize diagnostic options for dependency scan.
137 static void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
138   // Don't print 'X warnings and Y errors generated'.
139   DiagOpts.ShowCarets = false;
140   // Don't write out diagnostic file.
141   DiagOpts.DiagnosticSerializationFile.clear();
142   // Don't treat warnings as errors.
143   DiagOpts.Warnings.push_back("no-error");
144 }
145 
146 /// A clang tool that runs the preprocessor in a mode that's optimized for
147 /// dependency scanning for the given compiler invocation.
148 class DependencyScanningAction : public tooling::ToolAction {
149 public:
150   DependencyScanningAction(
151       StringRef WorkingDirectory, DependencyConsumer &Consumer,
152       llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
153       ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings,
154       ScanningOutputFormat Format, bool OptimizeArgs,
155       llvm::Optional<StringRef> ModuleName = None)
156       : WorkingDirectory(WorkingDirectory), Consumer(Consumer),
157         DepFS(std::move(DepFS)), PPSkipMappings(PPSkipMappings), Format(Format),
158         OptimizeArgs(OptimizeArgs), ModuleName(ModuleName) {}
159 
160   bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
161                      FileManager *FileMgr,
162                      std::shared_ptr<PCHContainerOperations> PCHContainerOps,
163                      DiagnosticConsumer *DiagConsumer) override {
164     // Make a deep copy of the original Clang invocation.
165     CompilerInvocation OriginalInvocation(*Invocation);
166 
167     // Create a compiler instance to handle the actual work.
168     CompilerInstance ScanInstance(std::move(PCHContainerOps));
169     ScanInstance.setInvocation(std::move(Invocation));
170 
171     // Create the compiler's actual diagnostics engine.
172     sanitizeDiagOpts(ScanInstance.getDiagnosticOpts());
173     ScanInstance.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
174     if (!ScanInstance.hasDiagnostics())
175       return false;
176 
177     ScanInstance.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
178         true;
179 
180     FileMgr->getFileSystemOpts().WorkingDir = std::string(WorkingDirectory);
181     ScanInstance.setFileManager(FileMgr);
182     ScanInstance.createSourceManager(*FileMgr);
183 
184     llvm::StringSet<> PrebuiltModulesInputFiles;
185     // Store the list of prebuilt module files into header search options. This
186     // will prevent the implicit build to create duplicate modules and will
187     // force reuse of the existing prebuilt module files instead.
188     if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
189       visitPrebuiltModule(
190           ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance,
191           ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
192           PrebuiltModulesInputFiles, /*VisitInputFiles=*/DepFS != nullptr);
193 
194     // Use the dependency scanning optimized file system if requested to do so.
195     if (DepFS) {
196       DepFS->clearIgnoredFiles();
197       // Ignore any files that contributed to prebuilt modules. The implicit
198       // build validates the modules by comparing the reported sizes of their
199       // inputs to the current state of the filesystem. Minimization would throw
200       // this mechanism off.
201       for (const auto &File : PrebuiltModulesInputFiles)
202         DepFS->ignoreFile(File.getKey());
203       // Add any filenames that were explicity passed in the build settings and
204       // that might be opened, as we want to ensure we don't run source
205       // minimization on them.
206       for (const auto &E : ScanInstance.getHeaderSearchOpts().UserEntries)
207         DepFS->ignoreFile(E.Path);
208       for (const auto &F : ScanInstance.getHeaderSearchOpts().VFSOverlayFiles)
209         DepFS->ignoreFile(F);
210 
211       // Support for virtual file system overlays on top of the caching
212       // filesystem.
213       FileMgr->setVirtualFileSystem(createVFSFromCompilerInvocation(
214           ScanInstance.getInvocation(), ScanInstance.getDiagnostics(), DepFS));
215 
216       // Pass the skip mappings which should speed up excluded conditional block
217       // skipping in the preprocessor.
218       if (PPSkipMappings)
219         ScanInstance.getPreprocessorOpts()
220             .ExcludedConditionalDirectiveSkipMappings = PPSkipMappings;
221     }
222 
223     // Create the dependency collector that will collect the produced
224     // dependencies.
225     //
226     // This also moves the existing dependency output options from the
227     // invocation to the collector. The options in the invocation are reset,
228     // which ensures that the compiler won't create new dependency collectors,
229     // and thus won't write out the extra '.d' files to disk.
230     auto Opts = std::make_unique<DependencyOutputOptions>();
231     std::swap(*Opts, ScanInstance.getInvocation().getDependencyOutputOpts());
232     // We need at least one -MT equivalent for the generator of make dependency
233     // files to work.
234     if (Opts->Targets.empty())
235       Opts->Targets = {
236           deduceDepTarget(ScanInstance.getFrontendOpts().OutputFile,
237                           ScanInstance.getFrontendOpts().Inputs)};
238     Opts->IncludeSystemHeaders = true;
239 
240     switch (Format) {
241     case ScanningOutputFormat::Make:
242       ScanInstance.addDependencyCollector(
243           std::make_shared<DependencyConsumerForwarder>(std::move(Opts),
244                                                         Consumer));
245       break;
246     case ScanningOutputFormat::Full:
247       ScanInstance.addDependencyCollector(std::make_shared<ModuleDepCollector>(
248           std::move(Opts), ScanInstance, Consumer,
249           std::move(OriginalInvocation), OptimizeArgs));
250       break;
251     }
252 
253     // Consider different header search and diagnostic options to create
254     // different modules. This avoids the unsound aliasing of module PCMs.
255     //
256     // TODO: Implement diagnostic bucketing to reduce the impact of strict
257     // context hashing.
258     ScanInstance.getHeaderSearchOpts().ModulesStrictContextHash = true;
259 
260     std::unique_ptr<FrontendAction> Action;
261 
262     if (ModuleName.hasValue())
263       Action = std::make_unique<GetDependenciesByModuleNameAction>(*ModuleName);
264     else
265       Action = std::make_unique<ReadPCHAndPreprocessAction>();
266 
267     const bool Result = ScanInstance.ExecuteAction(*Action);
268     if (!DepFS)
269       FileMgr->clearStatCache();
270     return Result;
271   }
272 
273 private:
274   StringRef WorkingDirectory;
275   DependencyConsumer &Consumer;
276   llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
277   ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings;
278   ScanningOutputFormat Format;
279   bool OptimizeArgs;
280   llvm::Optional<StringRef> ModuleName;
281 };
282 
283 } // end anonymous namespace
284 
285 DependencyScanningWorker::DependencyScanningWorker(
286     DependencyScanningService &Service)
287     : Format(Service.getFormat()), OptimizeArgs(Service.canOptimizeArgs()) {
288   PCHContainerOps = std::make_shared<PCHContainerOperations>();
289   PCHContainerOps->registerReader(
290       std::make_unique<ObjectFilePCHContainerReader>());
291   // We don't need to write object files, but the current PCH implementation
292   // requires the writer to be registered as well.
293   PCHContainerOps->registerWriter(
294       std::make_unique<ObjectFilePCHContainerWriter>());
295 
296   auto OverlayFS = llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(
297       llvm::vfs::createPhysicalFileSystem());
298   InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
299   OverlayFS->pushOverlay(InMemoryFS);
300   RealFS = OverlayFS;
301 
302   if (Service.canSkipExcludedPPRanges())
303     PPSkipMappings =
304         std::make_unique<ExcludedPreprocessorDirectiveSkipMapping>();
305   if (Service.getMode() == ScanningMode::MinimizedSourcePreprocessing)
306     DepFS = new DependencyScanningWorkerFilesystem(
307         Service.getSharedCache(), RealFS, PPSkipMappings.get());
308   if (Service.canReuseFileManager())
309     Files = new FileManager(FileSystemOptions(), RealFS);
310 }
311 
312 static llvm::Error
313 runWithDiags(DiagnosticOptions *DiagOpts,
314              llvm::function_ref<bool(DiagnosticConsumer &, DiagnosticOptions &)>
315                  BodyShouldSucceed) {
316   sanitizeDiagOpts(*DiagOpts);
317 
318   // Capture the emitted diagnostics and report them to the client
319   // in the case of a failure.
320   std::string DiagnosticOutput;
321   llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
322   TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts);
323 
324   if (BodyShouldSucceed(DiagPrinter, *DiagOpts))
325     return llvm::Error::success();
326   return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(),
327                                              llvm::inconvertibleErrorCode());
328 }
329 
330 llvm::Error DependencyScanningWorker::computeDependencies(
331     StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
332     DependencyConsumer &Consumer, llvm::Optional<StringRef> ModuleName) {
333   // Reset what might have been modified in the previous worker invocation.
334   RealFS->setCurrentWorkingDirectory(WorkingDirectory);
335   if (Files)
336     Files->setVirtualFileSystem(RealFS);
337 
338   llvm::IntrusiveRefCntPtr<FileManager> CurrentFiles =
339       Files ? Files : new FileManager(FileSystemOptions(), RealFS);
340 
341   Optional<std::vector<std::string>> ModifiedCommandLine;
342   if (ModuleName.hasValue()) {
343     ModifiedCommandLine = CommandLine;
344     InMemoryFS->addFile(*ModuleName, 0, llvm::MemoryBuffer::getMemBuffer(""));
345     ModifiedCommandLine->emplace_back(*ModuleName);
346   }
347 
348   const std::vector<std::string> &FinalCommandLine =
349       ModifiedCommandLine ? *ModifiedCommandLine : CommandLine;
350 
351   std::vector<const char *> FinalCCommandLine(CommandLine.size(), nullptr);
352   llvm::transform(CommandLine, FinalCCommandLine.begin(),
353                   [](const std::string &Str) { return Str.c_str(); });
354 
355   return runWithDiags(CreateAndPopulateDiagOpts(FinalCCommandLine).release(),
356                       [&](DiagnosticConsumer &DC, DiagnosticOptions &DiagOpts) {
357                         DependencyScanningAction Action(
358                             WorkingDirectory, Consumer, DepFS,
359                             PPSkipMappings.get(), Format, OptimizeArgs,
360                             ModuleName);
361                         // Create an invocation that uses the underlying file
362                         // system to ensure that any file system requests that
363                         // are made by the driver do not go through the
364                         // dependency scanning filesystem.
365                         ToolInvocation Invocation(FinalCommandLine, &Action,
366                                                   CurrentFiles.get(),
367                                                   PCHContainerOps);
368                         Invocation.setDiagnosticConsumer(&DC);
369                         Invocation.setDiagnosticOptions(&DiagOpts);
370                         return Invocation.run();
371                       });
372 }
373