1 //===- DependencyScanningWorker.cpp - clang-scan-deps worker --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
10 #include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
11 #include "clang/Frontend/CompilerInstance.h"
12 #include "clang/Frontend/CompilerInvocation.h"
13 #include "clang/Frontend/FrontendActions.h"
14 #include "clang/Frontend/TextDiagnosticPrinter.h"
15 #include "clang/Frontend/Utils.h"
16 #include "clang/Lex/PreprocessorOptions.h"
17 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
18 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
19 #include "clang/Tooling/Tooling.h"
20 
21 using namespace clang;
22 using namespace tooling;
23 using namespace dependencies;
24 
25 namespace {
26 
27 /// Forwards the gatherered dependencies to the consumer.
28 class DependencyConsumerForwarder : public DependencyFileGenerator {
29 public:
30   DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts,
31                               DependencyConsumer &C)
32       : DependencyFileGenerator(*Opts), Opts(std::move(Opts)), C(C) {}
33 
34   void finishedMainFile(DiagnosticsEngine &Diags) override {
35     C.handleDependencyOutputOpts(*Opts);
36     llvm::SmallString<256> CanonPath;
37     for (const auto &File : getDependencies()) {
38       CanonPath = File;
39       llvm::sys::path::remove_dots(CanonPath, /*remove_dot_dot=*/true);
40       C.handleFileDependency(CanonPath);
41     }
42   }
43 
44 private:
45   std::unique_ptr<DependencyOutputOptions> Opts;
46   DependencyConsumer &C;
47 };
48 
49 /// A listener that collects the imported modules and optionally the input
50 /// files.
51 class PrebuiltModuleListener : public ASTReaderListener {
52 public:
53   PrebuiltModuleListener(llvm::StringMap<std::string> &PrebuiltModuleFiles,
54                          llvm::StringSet<> &InputFiles, bool VisitInputFiles)
55       : PrebuiltModuleFiles(PrebuiltModuleFiles), InputFiles(InputFiles),
56         VisitInputFiles(VisitInputFiles) {}
57 
58   bool needsImportVisitation() const override { return true; }
59   bool needsInputFileVisitation() override { return VisitInputFiles; }
60   bool needsSystemInputFileVisitation() override { return VisitInputFiles; }
61 
62   void visitImport(StringRef ModuleName, StringRef Filename) override {
63     PrebuiltModuleFiles.insert({ModuleName, Filename.str()});
64   }
65 
66   bool visitInputFile(StringRef Filename, bool isSystem, bool isOverridden,
67                       bool isExplicitModule) override {
68     InputFiles.insert(Filename);
69     return true;
70   }
71 
72 private:
73   llvm::StringMap<std::string> &PrebuiltModuleFiles;
74   llvm::StringSet<> &InputFiles;
75   bool VisitInputFiles;
76 };
77 
78 using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
79 
80 /// Visit the given prebuilt module and collect all of the modules it
81 /// transitively imports and contributing input files.
82 static void visitPrebuiltModule(StringRef PrebuiltModuleFilename,
83                                 CompilerInstance &CI,
84                                 PrebuiltModuleFilesT &ModuleFiles,
85                                 llvm::StringSet<> &InputFiles,
86                                 bool VisitInputFiles) {
87   // Maps the names of modules that weren't yet visited to their PCM path.
88   llvm::StringMap<std::string> ModuleFilesWorklist;
89   // Contains PCM paths of all visited modules.
90   llvm::StringSet<> VisitedModuleFiles;
91 
92   PrebuiltModuleListener Listener(ModuleFilesWorklist, InputFiles,
93                                   VisitInputFiles);
94 
95   auto GatherModuleFileInfo = [&](StringRef ASTFile) {
96     ASTReader::readASTFileControlBlock(
97         ASTFile, CI.getFileManager(), CI.getPCHContainerReader(),
98         /*FindModuleFileExtensions=*/false, Listener,
99         /*ValidateDiagnosticOptions=*/false);
100   };
101 
102   GatherModuleFileInfo(PrebuiltModuleFilename);
103   while (!ModuleFilesWorklist.empty()) {
104     auto WorklistItemIt = ModuleFilesWorklist.begin();
105 
106     if (!VisitedModuleFiles.contains(WorklistItemIt->getValue())) {
107       VisitedModuleFiles.insert(WorklistItemIt->getValue());
108       GatherModuleFileInfo(WorklistItemIt->getValue());
109       ModuleFiles[WorklistItemIt->getKey().str()] = WorklistItemIt->getValue();
110     }
111 
112     ModuleFilesWorklist.erase(WorklistItemIt);
113   }
114 }
115 
116 /// Transform arbitrary file name into an object-like file name.
117 static std::string makeObjFileName(StringRef FileName) {
118   SmallString<128> ObjFileName(FileName);
119   llvm::sys::path::replace_extension(ObjFileName, "o");
120   return std::string(ObjFileName.str());
121 }
122 
123 /// Deduce the dependency target based on the output file and input files.
124 static std::string
125 deduceDepTarget(const std::string &OutputFile,
126                 const SmallVectorImpl<FrontendInputFile> &InputFiles) {
127   if (OutputFile != "-")
128     return OutputFile;
129 
130   if (InputFiles.empty() || !InputFiles.front().isFile())
131     return "clang-scan-deps\\ dependency";
132 
133   return makeObjFileName(InputFiles.front().getFile());
134 }
135 
136 /// A clang tool that runs the preprocessor in a mode that's optimized for
137 /// dependency scanning for the given compiler invocation.
138 class DependencyScanningAction : public tooling::ToolAction {
139 public:
140   DependencyScanningAction(
141       StringRef WorkingDirectory, DependencyConsumer &Consumer,
142       llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
143       ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings,
144       ScanningOutputFormat Format, llvm::Optional<StringRef> ModuleName = None,
145       llvm::Optional<llvm::MemoryBufferRef> FakeMemBuffer = None)
146       : WorkingDirectory(WorkingDirectory), Consumer(Consumer),
147         DepFS(std::move(DepFS)), PPSkipMappings(PPSkipMappings), Format(Format),
148         ModuleName(ModuleName), FakeMemBuffer(FakeMemBuffer) {}
149 
150   bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
151                      FileManager *FileMgr,
152                      std::shared_ptr<PCHContainerOperations> PCHContainerOps,
153                      DiagnosticConsumer *DiagConsumer) override {
154     // Make a deep copy of the original Clang invocation.
155     CompilerInvocation OriginalInvocation(*Invocation);
156 
157     // Create a compiler instance to handle the actual work.
158     CompilerInstance Compiler(std::move(PCHContainerOps));
159     Compiler.setInvocation(std::move(Invocation));
160 
161     // Don't print 'X warnings and Y errors generated'.
162     Compiler.getDiagnosticOpts().ShowCarets = false;
163     // Don't write out diagnostic file.
164     Compiler.getDiagnosticOpts().DiagnosticSerializationFile.clear();
165     // Don't treat warnings as errors.
166     Compiler.getDiagnosticOpts().Warnings.push_back("no-error");
167     // Create the compiler's actual diagnostics engine.
168     Compiler.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
169     if (!Compiler.hasDiagnostics())
170       return false;
171 
172     Compiler.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath = true;
173 
174     FileMgr->getFileSystemOpts().WorkingDir = std::string(WorkingDirectory);
175     Compiler.setFileManager(FileMgr);
176     Compiler.createSourceManager(*FileMgr);
177 
178     llvm::StringSet<> PrebuiltModulesInputFiles;
179     // Store the list of prebuilt module files into header search options. This
180     // will prevent the implicit build to create duplicate modules and will
181     // force reuse of the existing prebuilt module files instead.
182     if (!Compiler.getPreprocessorOpts().ImplicitPCHInclude.empty())
183       visitPrebuiltModule(
184           Compiler.getPreprocessorOpts().ImplicitPCHInclude, Compiler,
185           Compiler.getHeaderSearchOpts().PrebuiltModuleFiles,
186           PrebuiltModulesInputFiles, /*VisitInputFiles=*/DepFS != nullptr);
187 
188     // Use the dependency scanning optimized file system if requested to do so.
189     if (DepFS) {
190       const CompilerInvocation &CI = Compiler.getInvocation();
191       DepFS->clearIgnoredFiles();
192       // Ignore any files that contributed to prebuilt modules. The implicit
193       // build validates the modules by comparing the reported sizes of their
194       // inputs to the current state of the filesystem. Minimization would throw
195       // this mechanism off.
196       for (const auto &File : PrebuiltModulesInputFiles)
197         DepFS->ignoreFile(File.getKey());
198       // Add any filenames that were explicity passed in the build settings and
199       // that might be opened, as we want to ensure we don't run source
200       // minimization on them.
201       for (const auto &Entry : CI.getHeaderSearchOpts().UserEntries)
202         DepFS->ignoreFile(Entry.Path);
203       for (const auto &Entry : CI.getHeaderSearchOpts().VFSOverlayFiles)
204         DepFS->ignoreFile(Entry);
205 
206       // Support for virtual file system overlays on top of the caching
207       // filesystem.
208       FileMgr->setVirtualFileSystem(createVFSFromCompilerInvocation(
209           CI, Compiler.getDiagnostics(), DepFS));
210 
211       // Pass the skip mappings which should speed up excluded conditional block
212       // skipping in the preprocessor.
213       if (PPSkipMappings)
214         Compiler.getPreprocessorOpts()
215             .ExcludedConditionalDirectiveSkipMappings = PPSkipMappings;
216     }
217 
218     if (ModuleName.hasValue()) {
219       SmallString<128> FullPath(*ModuleName);
220       llvm::sys::fs::make_absolute(WorkingDirectory, FullPath);
221       SourceManager &SrcMgr = Compiler.getSourceManager();
222       FileMgr->getVirtualFile(FullPath.c_str(), FakeMemBuffer->getBufferSize(),
223                               0);
224       FileID MainFileID = SrcMgr.createFileID(*FakeMemBuffer);
225       SrcMgr.setMainFileID(MainFileID);
226     }
227 
228     // Create the dependency collector that will collect the produced
229     // dependencies.
230     //
231     // This also moves the existing dependency output options from the
232     // invocation to the collector. The options in the invocation are reset,
233     // which ensures that the compiler won't create new dependency collectors,
234     // and thus won't write out the extra '.d' files to disk.
235     auto Opts = std::make_unique<DependencyOutputOptions>();
236     std::swap(*Opts, Compiler.getInvocation().getDependencyOutputOpts());
237     // We need at least one -MT equivalent for the generator of make dependency
238     // files to work.
239     if (Opts->Targets.empty())
240       Opts->Targets = {deduceDepTarget(Compiler.getFrontendOpts().OutputFile,
241                                        Compiler.getFrontendOpts().Inputs)};
242     Opts->IncludeSystemHeaders = true;
243 
244     switch (Format) {
245     case ScanningOutputFormat::Make:
246       Compiler.addDependencyCollector(
247           std::make_shared<DependencyConsumerForwarder>(std::move(Opts),
248                                                         Consumer));
249       break;
250     case ScanningOutputFormat::Full:
251       Compiler.addDependencyCollector(std::make_shared<ModuleDepCollector>(
252           std::move(Opts), Compiler, Consumer, std::move(OriginalInvocation)));
253       break;
254     }
255 
256     // Consider different header search and diagnostic options to create
257     // different modules. This avoids the unsound aliasing of module PCMs.
258     //
259     // TODO: Implement diagnostic bucketing and header search pruning to reduce
260     // the impact of strict context hashing.
261     Compiler.getHeaderSearchOpts().ModulesStrictContextHash = true;
262 
263     std::unique_ptr<FrontendAction> Action;
264 
265     if (ModuleName.hasValue())
266       Action = std::make_unique<GetDependenciesByModuleNameAction>(*ModuleName);
267     else
268       Action = std::make_unique<ReadPCHAndPreprocessAction>();
269 
270     const bool Result = Compiler.ExecuteAction(*Action);
271     if (!DepFS)
272       FileMgr->clearStatCache();
273     return Result;
274   }
275 
276 private:
277   StringRef WorkingDirectory;
278   DependencyConsumer &Consumer;
279   llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
280   ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings;
281   ScanningOutputFormat Format;
282   llvm::Optional<StringRef> ModuleName;
283   llvm::Optional<llvm::MemoryBufferRef> FakeMemBuffer;
284 };
285 
286 } // end anonymous namespace
287 
288 DependencyScanningWorker::DependencyScanningWorker(
289     DependencyScanningService &Service)
290     : Format(Service.getFormat()) {
291   DiagOpts = new DiagnosticOptions();
292 
293   PCHContainerOps = std::make_shared<PCHContainerOperations>();
294   PCHContainerOps->registerReader(
295       std::make_unique<ObjectFilePCHContainerReader>());
296   // We don't need to write object files, but the current PCH implementation
297   // requires the writer to be registered as well.
298   PCHContainerOps->registerWriter(
299       std::make_unique<ObjectFilePCHContainerWriter>());
300 
301   RealFS = llvm::vfs::createPhysicalFileSystem();
302   if (Service.canSkipExcludedPPRanges())
303     PPSkipMappings =
304         std::make_unique<ExcludedPreprocessorDirectiveSkipMapping>();
305   if (Service.getMode() == ScanningMode::MinimizedSourcePreprocessing)
306     DepFS = new DependencyScanningWorkerFilesystem(
307         Service.getSharedCache(), RealFS, PPSkipMappings.get());
308   if (Service.canReuseFileManager())
309     Files = new FileManager(FileSystemOptions(), RealFS);
310 }
311 
312 static llvm::Error runWithDiags(
313     DiagnosticOptions *DiagOpts,
314     llvm::function_ref<bool(DiagnosticConsumer &DC)> BodyShouldSucceed) {
315   // Capture the emitted diagnostics and report them to the client
316   // in the case of a failure.
317   std::string DiagnosticOutput;
318   llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
319   TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts);
320 
321   if (BodyShouldSucceed(DiagPrinter))
322     return llvm::Error::success();
323   return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(),
324                                              llvm::inconvertibleErrorCode());
325 }
326 
327 llvm::Error DependencyScanningWorker::computeDependencies(
328     const std::string &Input, StringRef WorkingDirectory,
329     const CompilationDatabase &CDB, DependencyConsumer &Consumer,
330     llvm::Optional<StringRef> ModuleName) {
331   RealFS->setCurrentWorkingDirectory(WorkingDirectory);
332   std::unique_ptr<llvm::MemoryBuffer> FakeMemBuffer =
333       ModuleName.hasValue() ? llvm::MemoryBuffer::getMemBuffer(" ") : nullptr;
334   return runWithDiags(DiagOpts.get(), [&](DiagnosticConsumer &DC) {
335     /// Create the tool that uses the underlying file system to ensure that any
336     /// file system requests that are made by the driver do not go through the
337     /// dependency scanning filesystem.
338     SmallString<128> FullPath;
339     tooling::ClangTool Tool(CDB,
340                             ModuleName.hasValue() ? ModuleName->str() : Input,
341                             PCHContainerOps, RealFS, Files);
342     Tool.clearArgumentsAdjusters();
343     Tool.setRestoreWorkingDir(false);
344     Tool.setPrintErrorMessage(false);
345     Tool.setDiagnosticConsumer(&DC);
346     DependencyScanningAction Action(
347         WorkingDirectory, Consumer, DepFS, PPSkipMappings.get(), Format,
348         ModuleName,
349         FakeMemBuffer
350             ? llvm::Optional<llvm::MemoryBufferRef>(*FakeMemBuffer.get())
351             : None);
352 
353     if (ModuleName.hasValue()) {
354       Tool.mapVirtualFile(*ModuleName, FakeMemBuffer->getBuffer());
355       FullPath = *ModuleName;
356       llvm::sys::fs::make_absolute(WorkingDirectory, FullPath);
357       Tool.appendArgumentsAdjuster(
358           [&](const tooling::CommandLineArguments &Args, StringRef FileName) {
359             tooling::CommandLineArguments AdjustedArgs(Args);
360             AdjustedArgs.push_back(std::string(FullPath));
361             return AdjustedArgs;
362           });
363     }
364 
365     return !Tool.run(&Action);
366   });
367 }
368