1 //===- DependencyScanningWorker.cpp - clang-scan-deps worker --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
10 #include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
11 #include "clang/Frontend/CompilerInstance.h"
12 #include "clang/Frontend/CompilerInvocation.h"
13 #include "clang/Frontend/FrontendActions.h"
14 #include "clang/Frontend/TextDiagnosticPrinter.h"
15 #include "clang/Frontend/Utils.h"
16 #include "clang/Lex/PreprocessorOptions.h"
17 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
18 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
19 #include "clang/Tooling/Tooling.h"
20 
21 using namespace clang;
22 using namespace tooling;
23 using namespace dependencies;
24 
25 namespace {
26 
27 /// Forwards the gatherered dependencies to the consumer.
28 class DependencyConsumerForwarder : public DependencyFileGenerator {
29 public:
30   DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts,
31                               DependencyConsumer &C)
32       : DependencyFileGenerator(*Opts), Opts(std::move(Opts)), C(C) {}
33 
34   void finishedMainFile(DiagnosticsEngine &Diags) override {
35     C.handleDependencyOutputOpts(*Opts);
36     llvm::SmallString<256> CanonPath;
37     for (const auto &File : getDependencies()) {
38       CanonPath = File;
39       llvm::sys::path::remove_dots(CanonPath, /*remove_dot_dot=*/true);
40       C.handleFileDependency(CanonPath);
41     }
42   }
43 
44 private:
45   std::unique_ptr<DependencyOutputOptions> Opts;
46   DependencyConsumer &C;
47 };
48 
49 using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
50 
51 /// A listener that collects the imported modules and optionally the input
52 /// files.
53 class PrebuiltModuleListener : public ASTReaderListener {
54 public:
55   PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles,
56                          llvm::StringSet<> &InputFiles, bool VisitInputFiles,
57                          llvm::SmallVector<std::string> &NewModuleFiles)
58       : PrebuiltModuleFiles(PrebuiltModuleFiles), InputFiles(InputFiles),
59         VisitInputFiles(VisitInputFiles), NewModuleFiles(NewModuleFiles) {}
60 
61   bool needsImportVisitation() const override { return true; }
62   bool needsInputFileVisitation() override { return VisitInputFiles; }
63   bool needsSystemInputFileVisitation() override { return VisitInputFiles; }
64 
65   void visitImport(StringRef ModuleName, StringRef Filename) override {
66     if (PrebuiltModuleFiles.insert({ModuleName.str(), Filename.str()}).second)
67       NewModuleFiles.push_back(Filename.str());
68   }
69 
70   bool visitInputFile(StringRef Filename, bool isSystem, bool isOverridden,
71                       bool isExplicitModule) override {
72     InputFiles.insert(Filename);
73     return true;
74   }
75 
76 private:
77   PrebuiltModuleFilesT &PrebuiltModuleFiles;
78   llvm::StringSet<> &InputFiles;
79   bool VisitInputFiles;
80   llvm::SmallVector<std::string> &NewModuleFiles;
81 };
82 
83 /// Visit the given prebuilt module and collect all of the modules it
84 /// transitively imports and contributing input files.
85 static void visitPrebuiltModule(StringRef PrebuiltModuleFilename,
86                                 CompilerInstance &CI,
87                                 PrebuiltModuleFilesT &ModuleFiles,
88                                 llvm::StringSet<> &InputFiles,
89                                 bool VisitInputFiles) {
90   // List of module files to be processed.
91   llvm::SmallVector<std::string> Worklist{PrebuiltModuleFilename.str()};
92   PrebuiltModuleListener Listener(ModuleFiles, InputFiles, VisitInputFiles,
93                                   Worklist);
94 
95   while (!Worklist.empty())
96     ASTReader::readASTFileControlBlock(
97         Worklist.pop_back_val(), CI.getFileManager(),
98         CI.getPCHContainerReader(),
99         /*FindModuleFileExtensions=*/false, Listener,
100         /*ValidateDiagnosticOptions=*/false);
101 }
102 
103 /// Transform arbitrary file name into an object-like file name.
104 static std::string makeObjFileName(StringRef FileName) {
105   SmallString<128> ObjFileName(FileName);
106   llvm::sys::path::replace_extension(ObjFileName, "o");
107   return std::string(ObjFileName.str());
108 }
109 
110 /// Deduce the dependency target based on the output file and input files.
111 static std::string
112 deduceDepTarget(const std::string &OutputFile,
113                 const SmallVectorImpl<FrontendInputFile> &InputFiles) {
114   if (OutputFile != "-")
115     return OutputFile;
116 
117   if (InputFiles.empty() || !InputFiles.front().isFile())
118     return "clang-scan-deps\\ dependency";
119 
120   return makeObjFileName(InputFiles.front().getFile());
121 }
122 
123 /// Sanitize diagnostic options for dependency scan.
124 static void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
125   // Don't print 'X warnings and Y errors generated'.
126   DiagOpts.ShowCarets = false;
127   // Don't write out diagnostic file.
128   DiagOpts.DiagnosticSerializationFile.clear();
129   // Don't treat warnings as errors.
130   DiagOpts.Warnings.push_back("no-error");
131 }
132 
133 /// A clang tool that runs the preprocessor in a mode that's optimized for
134 /// dependency scanning for the given compiler invocation.
135 class DependencyScanningAction : public tooling::ToolAction {
136 public:
137   DependencyScanningAction(
138       StringRef WorkingDirectory, DependencyConsumer &Consumer,
139       llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
140       ScanningOutputFormat Format, bool OptimizeArgs,
141       llvm::Optional<StringRef> ModuleName = None)
142       : WorkingDirectory(WorkingDirectory), Consumer(Consumer),
143         DepFS(std::move(DepFS)), Format(Format), OptimizeArgs(OptimizeArgs),
144         ModuleName(ModuleName) {}
145 
146   bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
147                      FileManager *FileMgr,
148                      std::shared_ptr<PCHContainerOperations> PCHContainerOps,
149                      DiagnosticConsumer *DiagConsumer) override {
150     // Make a deep copy of the original Clang invocation.
151     CompilerInvocation OriginalInvocation(*Invocation);
152 
153     // Create a compiler instance to handle the actual work.
154     CompilerInstance ScanInstance(std::move(PCHContainerOps));
155     ScanInstance.setInvocation(std::move(Invocation));
156 
157     // Create the compiler's actual diagnostics engine.
158     sanitizeDiagOpts(ScanInstance.getDiagnosticOpts());
159     ScanInstance.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
160     if (!ScanInstance.hasDiagnostics())
161       return false;
162 
163     ScanInstance.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
164         true;
165 
166     ScanInstance.getFrontendOpts().GenerateGlobalModuleIndex = false;
167     ScanInstance.getFrontendOpts().UseGlobalModuleIndex = false;
168 
169     FileMgr->getFileSystemOpts().WorkingDir = std::string(WorkingDirectory);
170     ScanInstance.setFileManager(FileMgr);
171     ScanInstance.createSourceManager(*FileMgr);
172 
173     llvm::StringSet<> PrebuiltModulesInputFiles;
174     // Store the list of prebuilt module files into header search options. This
175     // will prevent the implicit build to create duplicate modules and will
176     // force reuse of the existing prebuilt module files instead.
177     if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
178       visitPrebuiltModule(
179           ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance,
180           ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
181           PrebuiltModulesInputFiles, /*VisitInputFiles=*/DepFS != nullptr);
182 
183     // Use the dependency scanning optimized file system if requested to do so.
184     if (DepFS) {
185       // Support for virtual file system overlays on top of the caching
186       // filesystem.
187       FileMgr->setVirtualFileSystem(createVFSFromCompilerInvocation(
188           ScanInstance.getInvocation(), ScanInstance.getDiagnostics(), DepFS));
189 
190       llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> LocalDepFS =
191           DepFS;
192       ScanInstance.getPreprocessorOpts().DependencyDirectivesForFile =
193           [LocalDepFS = std::move(LocalDepFS)](FileEntryRef File)
194           -> Optional<ArrayRef<dependency_directives_scan::Directive>> {
195         if (llvm::ErrorOr<EntryRef> Entry =
196                 LocalDepFS->getOrCreateFileSystemEntry(File.getName()))
197           return Entry->getDirectiveTokens();
198         return None;
199       };
200     }
201 
202     // Create the dependency collector that will collect the produced
203     // dependencies.
204     //
205     // This also moves the existing dependency output options from the
206     // invocation to the collector. The options in the invocation are reset,
207     // which ensures that the compiler won't create new dependency collectors,
208     // and thus won't write out the extra '.d' files to disk.
209     auto Opts = std::make_unique<DependencyOutputOptions>();
210     std::swap(*Opts, ScanInstance.getInvocation().getDependencyOutputOpts());
211     // We need at least one -MT equivalent for the generator of make dependency
212     // files to work.
213     if (Opts->Targets.empty())
214       Opts->Targets = {
215           deduceDepTarget(ScanInstance.getFrontendOpts().OutputFile,
216                           ScanInstance.getFrontendOpts().Inputs)};
217     Opts->IncludeSystemHeaders = true;
218 
219     switch (Format) {
220     case ScanningOutputFormat::Make:
221       ScanInstance.addDependencyCollector(
222           std::make_shared<DependencyConsumerForwarder>(std::move(Opts),
223                                                         Consumer));
224       break;
225     case ScanningOutputFormat::Full:
226       ScanInstance.addDependencyCollector(std::make_shared<ModuleDepCollector>(
227           std::move(Opts), ScanInstance, Consumer,
228           std::move(OriginalInvocation), OptimizeArgs));
229       break;
230     }
231 
232     // Consider different header search and diagnostic options to create
233     // different modules. This avoids the unsound aliasing of module PCMs.
234     //
235     // TODO: Implement diagnostic bucketing to reduce the impact of strict
236     // context hashing.
237     ScanInstance.getHeaderSearchOpts().ModulesStrictContextHash = true;
238 
239     std::unique_ptr<FrontendAction> Action;
240 
241     if (ModuleName.hasValue())
242       Action = std::make_unique<GetDependenciesByModuleNameAction>(*ModuleName);
243     else
244       Action = std::make_unique<ReadPCHAndPreprocessAction>();
245 
246     const bool Result = ScanInstance.ExecuteAction(*Action);
247     if (!DepFS)
248       FileMgr->clearStatCache();
249     return Result;
250   }
251 
252 private:
253   StringRef WorkingDirectory;
254   DependencyConsumer &Consumer;
255   llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
256   ScanningOutputFormat Format;
257   bool OptimizeArgs;
258   llvm::Optional<StringRef> ModuleName;
259 };
260 
261 } // end anonymous namespace
262 
263 DependencyScanningWorker::DependencyScanningWorker(
264     DependencyScanningService &Service)
265     : Format(Service.getFormat()), OptimizeArgs(Service.canOptimizeArgs()) {
266   PCHContainerOps = std::make_shared<PCHContainerOperations>();
267   PCHContainerOps->registerReader(
268       std::make_unique<ObjectFilePCHContainerReader>());
269   // We don't need to write object files, but the current PCH implementation
270   // requires the writer to be registered as well.
271   PCHContainerOps->registerWriter(
272       std::make_unique<ObjectFilePCHContainerWriter>());
273 
274   auto OverlayFS = llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(
275       llvm::vfs::createPhysicalFileSystem());
276   InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
277   OverlayFS->pushOverlay(InMemoryFS);
278   RealFS = OverlayFS;
279 
280   if (Service.getMode() == ScanningMode::DependencyDirectivesScan)
281     DepFS = new DependencyScanningWorkerFilesystem(Service.getSharedCache(),
282                                                    RealFS);
283   if (Service.canReuseFileManager())
284     Files = new FileManager(FileSystemOptions(), RealFS);
285 }
286 
287 static llvm::Error
288 runWithDiags(DiagnosticOptions *DiagOpts,
289              llvm::function_ref<bool(DiagnosticConsumer &, DiagnosticOptions &)>
290                  BodyShouldSucceed) {
291   sanitizeDiagOpts(*DiagOpts);
292 
293   // Capture the emitted diagnostics and report them to the client
294   // in the case of a failure.
295   std::string DiagnosticOutput;
296   llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
297   TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts);
298 
299   if (BodyShouldSucceed(DiagPrinter, *DiagOpts))
300     return llvm::Error::success();
301   return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(),
302                                              llvm::inconvertibleErrorCode());
303 }
304 
305 llvm::Error DependencyScanningWorker::computeDependencies(
306     StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
307     DependencyConsumer &Consumer, llvm::Optional<StringRef> ModuleName) {
308   // Reset what might have been modified in the previous worker invocation.
309   RealFS->setCurrentWorkingDirectory(WorkingDirectory);
310   if (Files)
311     Files->setVirtualFileSystem(RealFS);
312 
313   llvm::IntrusiveRefCntPtr<FileManager> CurrentFiles =
314       Files ? Files : new FileManager(FileSystemOptions(), RealFS);
315 
316   Optional<std::vector<std::string>> ModifiedCommandLine;
317   if (ModuleName.hasValue()) {
318     ModifiedCommandLine = CommandLine;
319     InMemoryFS->addFile(*ModuleName, 0, llvm::MemoryBuffer::getMemBuffer(""));
320     ModifiedCommandLine->emplace_back(*ModuleName);
321   }
322 
323   const std::vector<std::string> &FinalCommandLine =
324       ModifiedCommandLine ? *ModifiedCommandLine : CommandLine;
325 
326   std::vector<const char *> FinalCCommandLine(CommandLine.size(), nullptr);
327   llvm::transform(CommandLine, FinalCCommandLine.begin(),
328                   [](const std::string &Str) { return Str.c_str(); });
329 
330   return runWithDiags(CreateAndPopulateDiagOpts(FinalCCommandLine).release(),
331                       [&](DiagnosticConsumer &DC, DiagnosticOptions &DiagOpts) {
332                         DependencyScanningAction Action(
333                             WorkingDirectory, Consumer, DepFS, Format,
334                             OptimizeArgs, ModuleName);
335                         // Create an invocation that uses the underlying file
336                         // system to ensure that any file system requests that
337                         // are made by the driver do not go through the
338                         // dependency scanning filesystem.
339                         ToolInvocation Invocation(FinalCommandLine, &Action,
340                                                   CurrentFiles.get(),
341                                                   PCHContainerOps);
342                         Invocation.setDiagnosticConsumer(&DC);
343                         Invocation.setDiagnosticOptions(&DiagOpts);
344                         return Invocation.run();
345                       });
346 }
347