1 //===- ModuleDepCollector.cpp - Callbacks to collect deps -------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
10 
11 #include "clang/Frontend/CompilerInstance.h"
12 #include "clang/Lex/Preprocessor.h"
13 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
14 #include "llvm/Support/StringSaver.h"
15 
16 using namespace clang;
17 using namespace tooling;
18 using namespace dependencies;
19 
20 static void optimizeHeaderSearchOpts(HeaderSearchOptions &Opts,
21                                      ASTReader &Reader,
22                                      const serialization::ModuleFile &MF) {
23   // Only preserve search paths that were used during the dependency scan.
24   std::vector<HeaderSearchOptions::Entry> Entries = Opts.UserEntries;
25   Opts.UserEntries.clear();
26   for (unsigned I = 0; I < Entries.size(); ++I)
27     if (MF.SearchPathUsage[I])
28       Opts.UserEntries.push_back(Entries[I]);
29 }
30 
31 CompilerInvocation ModuleDepCollector::makeInvocationForModuleBuildWithoutPaths(
32     const ModuleDeps &Deps,
33     llvm::function_ref<void(CompilerInvocation &)> Optimize) const {
34   // Make a deep copy of the original Clang invocation.
35   CompilerInvocation CI(OriginalInvocation);
36 
37   CI.getLangOpts()->resetNonModularOptions();
38   CI.getPreprocessorOpts().resetNonModularOptions();
39 
40   // Remove options incompatible with explicit module build or are likely to
41   // differ between identical modules discovered from different translation
42   // units.
43   CI.getFrontendOpts().Inputs.clear();
44   CI.getFrontendOpts().OutputFile.clear();
45   CI.getCodeGenOpts().MainFileName.clear();
46   CI.getCodeGenOpts().DwarfDebugFlags.clear();
47 
48   CI.getFrontendOpts().ProgramAction = frontend::GenerateModule;
49   CI.getLangOpts()->ModuleName = Deps.ID.ModuleName;
50   CI.getFrontendOpts().IsSystemModule = Deps.IsSystem;
51 
52   CI.getLangOpts()->ImplicitModules = false;
53 
54   // Report the prebuilt modules this module uses.
55   for (const auto &PrebuiltModule : Deps.PrebuiltModuleDeps)
56     CI.getFrontendOpts().ModuleFiles.push_back(PrebuiltModule.PCMFile);
57 
58   Optimize(CI);
59 
60   // The original invocation probably didn't have strict context hash enabled.
61   // We will use the context hash of this invocation to distinguish between
62   // multiple incompatible versions of the same module and will use it when
63   // reporting dependencies to the clients. Let's make sure we're using
64   // **strict** context hash in order to prevent accidental sharing of
65   // incompatible modules (e.g. with differences in search paths).
66   CI.getHeaderSearchOpts().ModulesStrictContextHash = true;
67 
68   return CI;
69 }
70 
71 static std::vector<std::string>
72 serializeCompilerInvocation(const CompilerInvocation &CI) {
73   // Set up string allocator.
74   llvm::BumpPtrAllocator Alloc;
75   llvm::StringSaver Strings(Alloc);
76   auto SA = [&Strings](const Twine &Arg) { return Strings.save(Arg).data(); };
77 
78   // Synthesize full command line from the CompilerInvocation, including "-cc1".
79   SmallVector<const char *, 32> Args{"-cc1"};
80   CI.generateCC1CommandLine(Args, SA);
81 
82   // Convert arguments to the return type.
83   return std::vector<std::string>{Args.begin(), Args.end()};
84 }
85 
86 std::vector<std::string> ModuleDeps::getCanonicalCommandLine(
87     std::function<StringRef(ModuleID)> LookupPCMPath,
88     std::function<const ModuleDeps &(ModuleID)> LookupModuleDeps) const {
89   CompilerInvocation CI(BuildInvocation);
90   FrontendOptions &FrontendOpts = CI.getFrontendOpts();
91 
92   InputKind ModuleMapInputKind(FrontendOpts.DashX.getLanguage(),
93                                InputKind::Format::ModuleMap);
94   FrontendOpts.Inputs.emplace_back(ClangModuleMapFile, ModuleMapInputKind);
95   FrontendOpts.OutputFile = std::string(LookupPCMPath(ID));
96 
97   dependencies::detail::collectPCMAndModuleMapPaths(
98       ClangModuleDeps, LookupPCMPath, LookupModuleDeps,
99       FrontendOpts.ModuleFiles, FrontendOpts.ModuleMapFiles);
100 
101   return serializeCompilerInvocation(CI);
102 }
103 
104 std::vector<std::string>
105 ModuleDeps::getCanonicalCommandLineWithoutModulePaths() const {
106   return serializeCompilerInvocation(BuildInvocation);
107 }
108 
109 void dependencies::detail::collectPCMAndModuleMapPaths(
110     llvm::ArrayRef<ModuleID> Modules,
111     std::function<StringRef(ModuleID)> LookupPCMPath,
112     std::function<const ModuleDeps &(ModuleID)> LookupModuleDeps,
113     std::vector<std::string> &PCMPaths, std::vector<std::string> &ModMapPaths) {
114   llvm::StringSet<> AlreadyAdded;
115 
116   std::function<void(llvm::ArrayRef<ModuleID>)> AddArgs =
117       [&](llvm::ArrayRef<ModuleID> Modules) {
118         for (const ModuleID &MID : Modules) {
119           if (!AlreadyAdded.insert(MID.ModuleName + MID.ContextHash).second)
120             continue;
121           const ModuleDeps &M = LookupModuleDeps(MID);
122           // Depth first traversal.
123           AddArgs(M.ClangModuleDeps);
124           PCMPaths.push_back(LookupPCMPath(MID).str());
125           if (!M.ClangModuleMapFile.empty())
126             ModMapPaths.push_back(M.ClangModuleMapFile);
127         }
128       };
129 
130   AddArgs(Modules);
131 }
132 
133 void ModuleDepCollectorPP::FileChanged(SourceLocation Loc,
134                                        FileChangeReason Reason,
135                                        SrcMgr::CharacteristicKind FileType,
136                                        FileID PrevFID) {
137   if (Reason != PPCallbacks::EnterFile)
138     return;
139 
140   // This has to be delayed as the context hash can change at the start of
141   // `CompilerInstance::ExecuteAction`.
142   if (MDC.ContextHash.empty()) {
143     MDC.ContextHash = MDC.ScanInstance.getInvocation().getModuleHash();
144     MDC.Consumer.handleContextHash(MDC.ContextHash);
145   }
146 
147   SourceManager &SM = MDC.ScanInstance.getSourceManager();
148 
149   // Dependency generation really does want to go all the way to the
150   // file entry for a source location to find out what is depended on.
151   // We do not want #line markers to affect dependency generation!
152   if (Optional<StringRef> Filename =
153           SM.getNonBuiltinFilenameForID(SM.getFileID(SM.getExpansionLoc(Loc))))
154     MDC.FileDeps.push_back(
155         std::string(llvm::sys::path::remove_leading_dotslash(*Filename)));
156 }
157 
158 void ModuleDepCollectorPP::InclusionDirective(
159     SourceLocation HashLoc, const Token &IncludeTok, StringRef FileName,
160     bool IsAngled, CharSourceRange FilenameRange, const FileEntry *File,
161     StringRef SearchPath, StringRef RelativePath, const Module *Imported,
162     SrcMgr::CharacteristicKind FileType) {
163   if (!File && !Imported) {
164     // This is a non-modular include that HeaderSearch failed to find. Add it
165     // here as `FileChanged` will never see it.
166     MDC.FileDeps.push_back(std::string(FileName));
167   }
168   handleImport(Imported);
169 }
170 
171 void ModuleDepCollectorPP::moduleImport(SourceLocation ImportLoc,
172                                         ModuleIdPath Path,
173                                         const Module *Imported) {
174   handleImport(Imported);
175 }
176 
177 void ModuleDepCollectorPP::handleImport(const Module *Imported) {
178   if (!Imported)
179     return;
180 
181   const Module *TopLevelModule = Imported->getTopLevelModule();
182 
183   if (MDC.isPrebuiltModule(TopLevelModule))
184     DirectPrebuiltModularDeps.insert(TopLevelModule);
185   else
186     DirectModularDeps.insert(TopLevelModule);
187 }
188 
189 void ModuleDepCollectorPP::EndOfMainFile() {
190   FileID MainFileID = MDC.ScanInstance.getSourceManager().getMainFileID();
191   MDC.MainFile = std::string(MDC.ScanInstance.getSourceManager()
192                                  .getFileEntryForID(MainFileID)
193                                  ->getName());
194 
195   if (!MDC.ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
196     MDC.FileDeps.push_back(
197         MDC.ScanInstance.getPreprocessorOpts().ImplicitPCHInclude);
198 
199   for (const Module *M : DirectModularDeps) {
200     // A top-level module might not be actually imported as a module when
201     // -fmodule-name is used to compile a translation unit that imports this
202     // module. In that case it can be skipped. The appropriate header
203     // dependencies will still be reported as expected.
204     if (!M->getASTFile())
205       continue;
206     handleTopLevelModule(M);
207   }
208 
209   MDC.Consumer.handleDependencyOutputOpts(*MDC.Opts);
210 
211   for (auto &&I : MDC.ModularDeps)
212     MDC.Consumer.handleModuleDependency(I.second);
213 
214   for (auto &&I : MDC.FileDeps)
215     MDC.Consumer.handleFileDependency(I);
216 
217   for (auto &&I : DirectPrebuiltModularDeps)
218     MDC.Consumer.handlePrebuiltModuleDependency(PrebuiltModuleDep{I});
219 }
220 
221 ModuleID ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
222   assert(M == M->getTopLevelModule() && "Expected top level module!");
223 
224   // If this module has been handled already, just return its ID.
225   auto ModI = MDC.ModularDeps.insert({M, ModuleDeps{}});
226   if (!ModI.second)
227     return ModI.first->second.ID;
228 
229   ModuleDeps &MD = ModI.first->second;
230 
231   MD.ID.ModuleName = M->getFullModuleName();
232   MD.ImportedByMainFile = DirectModularDeps.contains(M);
233   MD.ImplicitModulePCMPath = std::string(M->getASTFile()->getName());
234   MD.IsSystem = M->IsSystem;
235 
236   const FileEntry *ModuleMap = MDC.ScanInstance.getPreprocessor()
237                                    .getHeaderSearchInfo()
238                                    .getModuleMap()
239                                    .getModuleMapFileForUniquing(M);
240   MD.ClangModuleMapFile = std::string(ModuleMap ? ModuleMap->getName() : "");
241 
242   serialization::ModuleFile *MF =
243       MDC.ScanInstance.getASTReader()->getModuleManager().lookup(
244           M->getASTFile());
245   MDC.ScanInstance.getASTReader()->visitInputFiles(
246       *MF, true, true, [&](const serialization::InputFile &IF, bool isSystem) {
247         // __inferred_module.map is the result of the way in which an implicit
248         // module build handles inferred modules. It adds an overlay VFS with
249         // this file in the proper directory and relies on the rest of Clang to
250         // handle it like normal. With explicitly built modules we don't need
251         // to play VFS tricks, so replace it with the correct module map.
252         if (IF.getFile()->getName().endswith("__inferred_module.map")) {
253           MD.FileDeps.insert(ModuleMap->getName());
254           return;
255         }
256         MD.FileDeps.insert(IF.getFile()->getName());
257       });
258 
259   // Add direct prebuilt module dependencies now, so that we can use them when
260   // creating a CompilerInvocation and computing context hash for this
261   // ModuleDeps instance.
262   llvm::DenseSet<const Module *> SeenModules;
263   addAllSubmodulePrebuiltDeps(M, MD, SeenModules);
264 
265   MD.BuildInvocation = MDC.makeInvocationForModuleBuildWithoutPaths(
266       MD, [&](CompilerInvocation &BuildInvocation) {
267         if (MDC.OptimizeArgs)
268           optimizeHeaderSearchOpts(BuildInvocation.getHeaderSearchOpts(),
269                                    *MDC.ScanInstance.getASTReader(), *MF);
270       });
271   MD.ID.ContextHash = MD.BuildInvocation.getModuleHash();
272 
273   llvm::DenseSet<const Module *> AddedModules;
274   addAllSubmoduleDeps(M, MD, AddedModules);
275 
276   return MD.ID;
277 }
278 
279 void ModuleDepCollectorPP::addAllSubmodulePrebuiltDeps(
280     const Module *M, ModuleDeps &MD,
281     llvm::DenseSet<const Module *> &SeenSubmodules) {
282   addModulePrebuiltDeps(M, MD, SeenSubmodules);
283 
284   for (const Module *SubM : M->submodules())
285     addAllSubmodulePrebuiltDeps(SubM, MD, SeenSubmodules);
286 }
287 
288 void ModuleDepCollectorPP::addModulePrebuiltDeps(
289     const Module *M, ModuleDeps &MD,
290     llvm::DenseSet<const Module *> &SeenSubmodules) {
291   for (const Module *Import : M->Imports)
292     if (Import->getTopLevelModule() != M->getTopLevelModule())
293       if (MDC.isPrebuiltModule(Import->getTopLevelModule()))
294         if (SeenSubmodules.insert(Import->getTopLevelModule()).second)
295           MD.PrebuiltModuleDeps.emplace_back(Import->getTopLevelModule());
296 }
297 
298 void ModuleDepCollectorPP::addAllSubmoduleDeps(
299     const Module *M, ModuleDeps &MD,
300     llvm::DenseSet<const Module *> &AddedModules) {
301   addModuleDep(M, MD, AddedModules);
302 
303   for (const Module *SubM : M->submodules())
304     addAllSubmoduleDeps(SubM, MD, AddedModules);
305 }
306 
307 void ModuleDepCollectorPP::addModuleDep(
308     const Module *M, ModuleDeps &MD,
309     llvm::DenseSet<const Module *> &AddedModules) {
310   for (const Module *Import : M->Imports) {
311     if (Import->getTopLevelModule() != M->getTopLevelModule() &&
312         !MDC.isPrebuiltModule(Import)) {
313       ModuleID ImportID = handleTopLevelModule(Import->getTopLevelModule());
314       if (AddedModules.insert(Import->getTopLevelModule()).second)
315         MD.ClangModuleDeps.push_back(ImportID);
316     }
317   }
318 }
319 
320 ModuleDepCollector::ModuleDepCollector(
321     std::unique_ptr<DependencyOutputOptions> Opts,
322     CompilerInstance &ScanInstance, DependencyConsumer &C,
323     CompilerInvocation &&OriginalCI, bool OptimizeArgs)
324     : ScanInstance(ScanInstance), Consumer(C), Opts(std::move(Opts)),
325       OriginalInvocation(std::move(OriginalCI)), OptimizeArgs(OptimizeArgs) {}
326 
327 void ModuleDepCollector::attachToPreprocessor(Preprocessor &PP) {
328   PP.addPPCallbacks(std::make_unique<ModuleDepCollectorPP>(*this));
329 }
330 
331 void ModuleDepCollector::attachToASTReader(ASTReader &R) {}
332 
333 bool ModuleDepCollector::isPrebuiltModule(const Module *M) {
334   std::string Name(M->getTopLevelModuleName());
335   const auto &PrebuiltModuleFiles =
336       ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles;
337   auto PrebuiltModuleFileIt = PrebuiltModuleFiles.find(Name);
338   if (PrebuiltModuleFileIt == PrebuiltModuleFiles.end())
339     return false;
340   assert("Prebuilt module came from the expected AST file" &&
341          PrebuiltModuleFileIt->second == M->getASTFile()->getName());
342   return true;
343 }
344