1 //===- DependencyScanningWorker.cpp - clang-scan-deps worker --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
10 #include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
11 #include "clang/Frontend/CompilerInstance.h"
12 #include "clang/Frontend/CompilerInvocation.h"
13 #include "clang/Frontend/FrontendActions.h"
14 #include "clang/Frontend/TextDiagnosticPrinter.h"
15 #include "clang/Frontend/Utils.h"
16 #include "clang/Lex/PreprocessorOptions.h"
17 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
18 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
19 #include "clang/Tooling/Tooling.h"
20 
21 using namespace clang;
22 using namespace tooling;
23 using namespace dependencies;
24 
25 namespace {
26 
27 /// Forwards the gatherered dependencies to the consumer.
28 class DependencyConsumerForwarder : public DependencyFileGenerator {
29 public:
30   DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts,
31                               DependencyConsumer &C)
32       : DependencyFileGenerator(*Opts), Opts(std::move(Opts)), C(C) {}
33 
34   void finishedMainFile(DiagnosticsEngine &Diags) override {
35     C.handleDependencyOutputOpts(*Opts);
36     llvm::SmallString<256> CanonPath;
37     for (const auto &File : getDependencies()) {
38       CanonPath = File;
39       llvm::sys::path::remove_dots(CanonPath, /*remove_dot_dot=*/true);
40       C.handleFileDependency(CanonPath);
41     }
42   }
43 
44 private:
45   std::unique_ptr<DependencyOutputOptions> Opts;
46   DependencyConsumer &C;
47 };
48 
49 /// A listener that collects the imported modules and optionally the input
50 /// files.
51 class PrebuiltModuleListener : public ASTReaderListener {
52 public:
53   PrebuiltModuleListener(llvm::StringMap<std::string> &PrebuiltModuleFiles,
54                          llvm::StringSet<> &InputFiles, bool VisitInputFiles)
55       : PrebuiltModuleFiles(PrebuiltModuleFiles), InputFiles(InputFiles),
56         VisitInputFiles(VisitInputFiles) {}
57 
58   bool needsImportVisitation() const override { return true; }
59   bool needsInputFileVisitation() override { return VisitInputFiles; }
60   bool needsSystemInputFileVisitation() override { return VisitInputFiles; }
61 
62   void visitImport(StringRef ModuleName, StringRef Filename) override {
63     PrebuiltModuleFiles.insert({ModuleName, Filename.str()});
64   }
65 
66   bool visitInputFile(StringRef Filename, bool isSystem, bool isOverridden,
67                       bool isExplicitModule) override {
68     InputFiles.insert(Filename);
69     return true;
70   }
71 
72 private:
73   llvm::StringMap<std::string> &PrebuiltModuleFiles;
74   llvm::StringSet<> &InputFiles;
75   bool VisitInputFiles;
76 };
77 
78 using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
79 
80 /// Visit the given prebuilt module and collect all of the modules it
81 /// transitively imports and contributing input files.
82 static void visitPrebuiltModule(StringRef PrebuiltModuleFilename,
83                                 CompilerInstance &CI,
84                                 PrebuiltModuleFilesT &ModuleFiles,
85                                 llvm::StringSet<> &InputFiles,
86                                 bool VisitInputFiles) {
87   // Maps the names of modules that weren't yet visited to their PCM path.
88   llvm::StringMap<std::string> ModuleFilesWorklist;
89   // Contains PCM paths of all visited modules.
90   llvm::StringSet<> VisitedModuleFiles;
91 
92   PrebuiltModuleListener Listener(ModuleFilesWorklist, InputFiles,
93                                   VisitInputFiles);
94 
95   auto GatherModuleFileInfo = [&](StringRef ASTFile) {
96     ASTReader::readASTFileControlBlock(
97         ASTFile, CI.getFileManager(), CI.getPCHContainerReader(),
98         /*FindModuleFileExtensions=*/false, Listener,
99         /*ValidateDiagnosticOptions=*/false);
100   };
101 
102   GatherModuleFileInfo(PrebuiltModuleFilename);
103   while (!ModuleFilesWorklist.empty()) {
104     auto WorklistItemIt = ModuleFilesWorklist.begin();
105 
106     if (!VisitedModuleFiles.contains(WorklistItemIt->getValue())) {
107       VisitedModuleFiles.insert(WorklistItemIt->getValue());
108       GatherModuleFileInfo(WorklistItemIt->getValue());
109       ModuleFiles[WorklistItemIt->getKey().str()] = WorklistItemIt->getValue();
110     }
111 
112     ModuleFilesWorklist.erase(WorklistItemIt);
113   }
114 }
115 
116 /// Transform arbitrary file name into an object-like file name.
117 static std::string makeObjFileName(StringRef FileName) {
118   SmallString<128> ObjFileName(FileName);
119   llvm::sys::path::replace_extension(ObjFileName, "o");
120   return std::string(ObjFileName.str());
121 }
122 
123 /// Deduce the dependency target based on the output file and input files.
124 static std::string
125 deduceDepTarget(const std::string &OutputFile,
126                 const SmallVectorImpl<FrontendInputFile> &InputFiles) {
127   if (OutputFile != "-")
128     return OutputFile;
129 
130   if (InputFiles.empty() || !InputFiles.front().isFile())
131     return "clang-scan-deps\\ dependency";
132 
133   return makeObjFileName(InputFiles.front().getFile());
134 }
135 
136 /// Sanitize diagnostic options for dependency scan.
137 static void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
138   // Don't print 'X warnings and Y errors generated'.
139   DiagOpts.ShowCarets = false;
140   // Don't write out diagnostic file.
141   DiagOpts.DiagnosticSerializationFile.clear();
142   // Don't treat warnings as errors.
143   DiagOpts.Warnings.push_back("no-error");
144 }
145 
146 /// A clang tool that runs the preprocessor in a mode that's optimized for
147 /// dependency scanning for the given compiler invocation.
148 class DependencyScanningAction : public tooling::ToolAction {
149 public:
150   DependencyScanningAction(
151       StringRef WorkingDirectory, DependencyConsumer &Consumer,
152       llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
153       ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings,
154       ScanningOutputFormat Format, bool OptimizeArgs,
155       llvm::Optional<StringRef> ModuleName = None)
156       : WorkingDirectory(WorkingDirectory), Consumer(Consumer),
157         DepFS(std::move(DepFS)), PPSkipMappings(PPSkipMappings), Format(Format),
158         OptimizeArgs(OptimizeArgs), ModuleName(ModuleName) {}
159 
160   bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
161                      FileManager *FileMgr,
162                      std::shared_ptr<PCHContainerOperations> PCHContainerOps,
163                      DiagnosticConsumer *DiagConsumer) override {
164     // Make a deep copy of the original Clang invocation.
165     CompilerInvocation OriginalInvocation(*Invocation);
166 
167     // Create a compiler instance to handle the actual work.
168     CompilerInstance ScanInstance(std::move(PCHContainerOps));
169     ScanInstance.setInvocation(std::move(Invocation));
170 
171     // Create the compiler's actual diagnostics engine.
172     sanitizeDiagOpts(ScanInstance.getDiagnosticOpts());
173     ScanInstance.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
174     if (!ScanInstance.hasDiagnostics())
175       return false;
176 
177     ScanInstance.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
178         true;
179 
180     ScanInstance.getFrontendOpts().GenerateGlobalModuleIndex = false;
181     ScanInstance.getFrontendOpts().UseGlobalModuleIndex = false;
182 
183     FileMgr->getFileSystemOpts().WorkingDir = std::string(WorkingDirectory);
184     ScanInstance.setFileManager(FileMgr);
185     ScanInstance.createSourceManager(*FileMgr);
186 
187     llvm::StringSet<> PrebuiltModulesInputFiles;
188     // Store the list of prebuilt module files into header search options. This
189     // will prevent the implicit build to create duplicate modules and will
190     // force reuse of the existing prebuilt module files instead.
191     if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
192       visitPrebuiltModule(
193           ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance,
194           ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
195           PrebuiltModulesInputFiles, /*VisitInputFiles=*/DepFS != nullptr);
196 
197     // Use the dependency scanning optimized file system if requested to do so.
198     if (DepFS) {
199       DepFS->enableMinimizationOfAllFiles();
200       // Don't minimize any files that contributed to prebuilt modules. The
201       // implicit build validates the modules by comparing the reported sizes of
202       // their inputs to the current state of the filesystem. Minimization would
203       // throw this mechanism off.
204       for (const auto &File : PrebuiltModulesInputFiles)
205         DepFS->disableMinimization(File.getKey());
206       // Don't minimize any files that were explicitly passed in the build
207       // settings and that might be opened.
208       for (const auto &E : ScanInstance.getHeaderSearchOpts().UserEntries)
209         DepFS->disableMinimization(E.Path);
210       for (const auto &F : ScanInstance.getHeaderSearchOpts().VFSOverlayFiles)
211         DepFS->disableMinimization(F);
212 
213       // Support for virtual file system overlays on top of the caching
214       // filesystem.
215       FileMgr->setVirtualFileSystem(createVFSFromCompilerInvocation(
216           ScanInstance.getInvocation(), ScanInstance.getDiagnostics(), DepFS));
217 
218       // Pass the skip mappings which should speed up excluded conditional block
219       // skipping in the preprocessor.
220       if (PPSkipMappings)
221         ScanInstance.getPreprocessorOpts()
222             .ExcludedConditionalDirectiveSkipMappings = PPSkipMappings;
223     }
224 
225     // Create the dependency collector that will collect the produced
226     // dependencies.
227     //
228     // This also moves the existing dependency output options from the
229     // invocation to the collector. The options in the invocation are reset,
230     // which ensures that the compiler won't create new dependency collectors,
231     // and thus won't write out the extra '.d' files to disk.
232     auto Opts = std::make_unique<DependencyOutputOptions>();
233     std::swap(*Opts, ScanInstance.getInvocation().getDependencyOutputOpts());
234     // We need at least one -MT equivalent for the generator of make dependency
235     // files to work.
236     if (Opts->Targets.empty())
237       Opts->Targets = {
238           deduceDepTarget(ScanInstance.getFrontendOpts().OutputFile,
239                           ScanInstance.getFrontendOpts().Inputs)};
240     Opts->IncludeSystemHeaders = true;
241 
242     switch (Format) {
243     case ScanningOutputFormat::Make:
244       ScanInstance.addDependencyCollector(
245           std::make_shared<DependencyConsumerForwarder>(std::move(Opts),
246                                                         Consumer));
247       break;
248     case ScanningOutputFormat::Full:
249       ScanInstance.addDependencyCollector(std::make_shared<ModuleDepCollector>(
250           std::move(Opts), ScanInstance, Consumer,
251           std::move(OriginalInvocation), OptimizeArgs));
252       break;
253     }
254 
255     // Consider different header search and diagnostic options to create
256     // different modules. This avoids the unsound aliasing of module PCMs.
257     //
258     // TODO: Implement diagnostic bucketing to reduce the impact of strict
259     // context hashing.
260     ScanInstance.getHeaderSearchOpts().ModulesStrictContextHash = true;
261 
262     std::unique_ptr<FrontendAction> Action;
263 
264     if (ModuleName.hasValue())
265       Action = std::make_unique<GetDependenciesByModuleNameAction>(*ModuleName);
266     else
267       Action = std::make_unique<ReadPCHAndPreprocessAction>();
268 
269     const bool Result = ScanInstance.ExecuteAction(*Action);
270     if (!DepFS)
271       FileMgr->clearStatCache();
272     return Result;
273   }
274 
275 private:
276   StringRef WorkingDirectory;
277   DependencyConsumer &Consumer;
278   llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
279   ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings;
280   ScanningOutputFormat Format;
281   bool OptimizeArgs;
282   llvm::Optional<StringRef> ModuleName;
283 };
284 
285 } // end anonymous namespace
286 
287 DependencyScanningWorker::DependencyScanningWorker(
288     DependencyScanningService &Service)
289     : Format(Service.getFormat()), OptimizeArgs(Service.canOptimizeArgs()) {
290   PCHContainerOps = std::make_shared<PCHContainerOperations>();
291   PCHContainerOps->registerReader(
292       std::make_unique<ObjectFilePCHContainerReader>());
293   // We don't need to write object files, but the current PCH implementation
294   // requires the writer to be registered as well.
295   PCHContainerOps->registerWriter(
296       std::make_unique<ObjectFilePCHContainerWriter>());
297 
298   auto OverlayFS = llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(
299       llvm::vfs::createPhysicalFileSystem());
300   InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
301   OverlayFS->pushOverlay(InMemoryFS);
302   RealFS = OverlayFS;
303 
304   if (Service.canSkipExcludedPPRanges())
305     PPSkipMappings =
306         std::make_unique<ExcludedPreprocessorDirectiveSkipMapping>();
307   if (Service.getMode() == ScanningMode::MinimizedSourcePreprocessing)
308     DepFS = new DependencyScanningWorkerFilesystem(
309         Service.getSharedCache(), RealFS, PPSkipMappings.get());
310   if (Service.canReuseFileManager())
311     Files = new FileManager(FileSystemOptions(), RealFS);
312 }
313 
314 static llvm::Error
315 runWithDiags(DiagnosticOptions *DiagOpts,
316              llvm::function_ref<bool(DiagnosticConsumer &, DiagnosticOptions &)>
317                  BodyShouldSucceed) {
318   sanitizeDiagOpts(*DiagOpts);
319 
320   // Capture the emitted diagnostics and report them to the client
321   // in the case of a failure.
322   std::string DiagnosticOutput;
323   llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
324   TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts);
325 
326   if (BodyShouldSucceed(DiagPrinter, *DiagOpts))
327     return llvm::Error::success();
328   return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(),
329                                              llvm::inconvertibleErrorCode());
330 }
331 
332 llvm::Error DependencyScanningWorker::computeDependencies(
333     StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
334     DependencyConsumer &Consumer, llvm::Optional<StringRef> ModuleName) {
335   // Reset what might have been modified in the previous worker invocation.
336   RealFS->setCurrentWorkingDirectory(WorkingDirectory);
337   if (Files)
338     Files->setVirtualFileSystem(RealFS);
339 
340   llvm::IntrusiveRefCntPtr<FileManager> CurrentFiles =
341       Files ? Files : new FileManager(FileSystemOptions(), RealFS);
342 
343   Optional<std::vector<std::string>> ModifiedCommandLine;
344   if (ModuleName.hasValue()) {
345     ModifiedCommandLine = CommandLine;
346     InMemoryFS->addFile(*ModuleName, 0, llvm::MemoryBuffer::getMemBuffer(""));
347     ModifiedCommandLine->emplace_back(*ModuleName);
348   }
349 
350   const std::vector<std::string> &FinalCommandLine =
351       ModifiedCommandLine ? *ModifiedCommandLine : CommandLine;
352 
353   std::vector<const char *> FinalCCommandLine(CommandLine.size(), nullptr);
354   llvm::transform(CommandLine, FinalCCommandLine.begin(),
355                   [](const std::string &Str) { return Str.c_str(); });
356 
357   return runWithDiags(CreateAndPopulateDiagOpts(FinalCCommandLine).release(),
358                       [&](DiagnosticConsumer &DC, DiagnosticOptions &DiagOpts) {
359                         DependencyScanningAction Action(
360                             WorkingDirectory, Consumer, DepFS,
361                             PPSkipMappings.get(), Format, OptimizeArgs,
362                             ModuleName);
363                         // Create an invocation that uses the underlying file
364                         // system to ensure that any file system requests that
365                         // are made by the driver do not go through the
366                         // dependency scanning filesystem.
367                         ToolInvocation Invocation(FinalCommandLine, &Action,
368                                                   CurrentFiles.get(),
369                                                   PCHContainerOps);
370                         Invocation.setDiagnosticConsumer(&DC);
371                         Invocation.setDiagnosticOptions(&DiagOpts);
372                         return Invocation.run();
373                       });
374 }
375