1 //===- ModuleDepCollector.cpp - Callbacks to collect deps -------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
10 
11 #include "clang/Frontend/CompilerInstance.h"
12 #include "clang/Lex/Preprocessor.h"
13 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
14 #include "llvm/Support/StringSaver.h"
15 
16 using namespace clang;
17 using namespace tooling;
18 using namespace dependencies;
19 
20 static void optimizeHeaderSearchOpts(HeaderSearchOptions &Opts,
21                                      ASTReader &Reader,
22                                      const serialization::ModuleFile &MF) {
23   // Only preserve search paths that were used during the dependency scan.
24   std::vector<HeaderSearchOptions::Entry> Entries = Opts.UserEntries;
25   Opts.UserEntries.clear();
26   for (unsigned I = 0; I < Entries.size(); ++I)
27     if (MF.SearchPathUsage[I])
28       Opts.UserEntries.push_back(Entries[I]);
29 }
30 
31 CompilerInvocation ModuleDepCollector::makeInvocationForModuleBuildWithoutPaths(
32     const ModuleDeps &Deps,
33     llvm::function_ref<void(CompilerInvocation &)> Optimize) const {
34   // Make a deep copy of the original Clang invocation.
35   CompilerInvocation CI(OriginalInvocation);
36 
37   CI.getLangOpts()->resetNonModularOptions();
38   CI.getPreprocessorOpts().resetNonModularOptions();
39 
40   // Remove options incompatible with explicit module build.
41   CI.getFrontendOpts().Inputs.clear();
42   CI.getFrontendOpts().OutputFile.clear();
43 
44   CI.getFrontendOpts().ProgramAction = frontend::GenerateModule;
45   CI.getLangOpts()->ModuleName = Deps.ID.ModuleName;
46   CI.getFrontendOpts().IsSystemModule = Deps.IsSystem;
47 
48   CI.getLangOpts()->ImplicitModules = false;
49 
50   // Report the prebuilt modules this module uses.
51   for (const auto &PrebuiltModule : Deps.PrebuiltModuleDeps) {
52     CI.getFrontendOpts().ModuleFiles.push_back(PrebuiltModule.PCMFile);
53     CI.getFrontendOpts().ModuleMapFiles.push_back(PrebuiltModule.ModuleMapFile);
54   }
55 
56   Optimize(CI);
57 
58   // The original invocation probably didn't have strict context hash enabled.
59   // We will use the context hash of this invocation to distinguish between
60   // multiple incompatible versions of the same module and will use it when
61   // reporting dependencies to the clients. Let's make sure we're using
62   // **strict** context hash in order to prevent accidental sharing of
63   // incompatible modules (e.g. with differences in search paths).
64   CI.getHeaderSearchOpts().ModulesStrictContextHash = true;
65 
66   return CI;
67 }
68 
69 static std::vector<std::string>
70 serializeCompilerInvocation(const CompilerInvocation &CI) {
71   // Set up string allocator.
72   llvm::BumpPtrAllocator Alloc;
73   llvm::StringSaver Strings(Alloc);
74   auto SA = [&Strings](const Twine &Arg) { return Strings.save(Arg).data(); };
75 
76   // Synthesize full command line from the CompilerInvocation, including "-cc1".
77   SmallVector<const char *, 32> Args{"-cc1"};
78   CI.generateCC1CommandLine(Args, SA);
79 
80   // Convert arguments to the return type.
81   return std::vector<std::string>{Args.begin(), Args.end()};
82 }
83 
84 std::vector<std::string> ModuleDeps::getCanonicalCommandLine(
85     std::function<StringRef(ModuleID)> LookupPCMPath,
86     std::function<const ModuleDeps &(ModuleID)> LookupModuleDeps) const {
87   CompilerInvocation CI(BuildInvocation);
88   FrontendOptions &FrontendOpts = CI.getFrontendOpts();
89 
90   InputKind ModuleMapInputKind(FrontendOpts.DashX.getLanguage(),
91                                InputKind::Format::ModuleMap);
92   FrontendOpts.Inputs.emplace_back(ClangModuleMapFile, ModuleMapInputKind);
93   FrontendOpts.OutputFile = std::string(LookupPCMPath(ID));
94 
95   dependencies::detail::collectPCMAndModuleMapPaths(
96       ClangModuleDeps, LookupPCMPath, LookupModuleDeps,
97       FrontendOpts.ModuleFiles, FrontendOpts.ModuleMapFiles);
98 
99   return serializeCompilerInvocation(CI);
100 }
101 
102 std::vector<std::string>
103 ModuleDeps::getCanonicalCommandLineWithoutModulePaths() const {
104   return serializeCompilerInvocation(BuildInvocation);
105 }
106 
107 void dependencies::detail::collectPCMAndModuleMapPaths(
108     llvm::ArrayRef<ModuleID> Modules,
109     std::function<StringRef(ModuleID)> LookupPCMPath,
110     std::function<const ModuleDeps &(ModuleID)> LookupModuleDeps,
111     std::vector<std::string> &PCMPaths, std::vector<std::string> &ModMapPaths) {
112   llvm::StringSet<> AlreadyAdded;
113 
114   std::function<void(llvm::ArrayRef<ModuleID>)> AddArgs =
115       [&](llvm::ArrayRef<ModuleID> Modules) {
116         for (const ModuleID &MID : Modules) {
117           if (!AlreadyAdded.insert(MID.ModuleName + MID.ContextHash).second)
118             continue;
119           const ModuleDeps &M = LookupModuleDeps(MID);
120           // Depth first traversal.
121           AddArgs(M.ClangModuleDeps);
122           PCMPaths.push_back(LookupPCMPath(MID).str());
123           if (!M.ClangModuleMapFile.empty())
124             ModMapPaths.push_back(M.ClangModuleMapFile);
125         }
126       };
127 
128   AddArgs(Modules);
129 }
130 
131 void ModuleDepCollectorPP::FileChanged(SourceLocation Loc,
132                                        FileChangeReason Reason,
133                                        SrcMgr::CharacteristicKind FileType,
134                                        FileID PrevFID) {
135   if (Reason != PPCallbacks::EnterFile)
136     return;
137 
138   // This has to be delayed as the context hash can change at the start of
139   // `CompilerInstance::ExecuteAction`.
140   if (MDC.ContextHash.empty()) {
141     MDC.ContextHash = MDC.ScanInstance.getInvocation().getModuleHash();
142     MDC.Consumer.handleContextHash(MDC.ContextHash);
143   }
144 
145   SourceManager &SM = MDC.ScanInstance.getSourceManager();
146 
147   // Dependency generation really does want to go all the way to the
148   // file entry for a source location to find out what is depended on.
149   // We do not want #line markers to affect dependency generation!
150   if (Optional<StringRef> Filename =
151           SM.getNonBuiltinFilenameForID(SM.getFileID(SM.getExpansionLoc(Loc))))
152     MDC.FileDeps.push_back(
153         std::string(llvm::sys::path::remove_leading_dotslash(*Filename)));
154 }
155 
156 void ModuleDepCollectorPP::InclusionDirective(
157     SourceLocation HashLoc, const Token &IncludeTok, StringRef FileName,
158     bool IsAngled, CharSourceRange FilenameRange, const FileEntry *File,
159     StringRef SearchPath, StringRef RelativePath, const Module *Imported,
160     SrcMgr::CharacteristicKind FileType) {
161   if (!File && !Imported) {
162     // This is a non-modular include that HeaderSearch failed to find. Add it
163     // here as `FileChanged` will never see it.
164     MDC.FileDeps.push_back(std::string(FileName));
165   }
166   handleImport(Imported);
167 }
168 
169 void ModuleDepCollectorPP::moduleImport(SourceLocation ImportLoc,
170                                         ModuleIdPath Path,
171                                         const Module *Imported) {
172   handleImport(Imported);
173 }
174 
175 void ModuleDepCollectorPP::handleImport(const Module *Imported) {
176   if (!Imported)
177     return;
178 
179   const Module *TopLevelModule = Imported->getTopLevelModule();
180 
181   if (MDC.isPrebuiltModule(TopLevelModule))
182     DirectPrebuiltModularDeps.insert(TopLevelModule);
183   else
184     DirectModularDeps.insert(TopLevelModule);
185 }
186 
187 void ModuleDepCollectorPP::EndOfMainFile() {
188   FileID MainFileID = MDC.ScanInstance.getSourceManager().getMainFileID();
189   MDC.MainFile = std::string(MDC.ScanInstance.getSourceManager()
190                                  .getFileEntryForID(MainFileID)
191                                  ->getName());
192 
193   if (!MDC.ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
194     MDC.FileDeps.push_back(
195         MDC.ScanInstance.getPreprocessorOpts().ImplicitPCHInclude);
196 
197   for (const Module *M : DirectModularDeps) {
198     // A top-level module might not be actually imported as a module when
199     // -fmodule-name is used to compile a translation unit that imports this
200     // module. In that case it can be skipped. The appropriate header
201     // dependencies will still be reported as expected.
202     if (!M->getASTFile())
203       continue;
204     handleTopLevelModule(M);
205   }
206 
207   MDC.Consumer.handleDependencyOutputOpts(*MDC.Opts);
208 
209   for (auto &&I : MDC.ModularDeps)
210     MDC.Consumer.handleModuleDependency(I.second);
211 
212   for (auto &&I : MDC.FileDeps)
213     MDC.Consumer.handleFileDependency(I);
214 
215   for (auto &&I : DirectPrebuiltModularDeps)
216     MDC.Consumer.handlePrebuiltModuleDependency(PrebuiltModuleDep{I});
217 }
218 
219 ModuleID ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
220   assert(M == M->getTopLevelModule() && "Expected top level module!");
221 
222   // If this module has been handled already, just return its ID.
223   auto ModI = MDC.ModularDeps.insert({M, ModuleDeps{}});
224   if (!ModI.second)
225     return ModI.first->second.ID;
226 
227   ModuleDeps &MD = ModI.first->second;
228 
229   MD.ID.ModuleName = M->getFullModuleName();
230   MD.ImportedByMainFile = DirectModularDeps.contains(M);
231   MD.ImplicitModulePCMPath = std::string(M->getASTFile()->getName());
232   MD.IsSystem = M->IsSystem;
233 
234   const FileEntry *ModuleMap = MDC.ScanInstance.getPreprocessor()
235                                    .getHeaderSearchInfo()
236                                    .getModuleMap()
237                                    .getModuleMapFileForUniquing(M);
238   MD.ClangModuleMapFile = std::string(ModuleMap ? ModuleMap->getName() : "");
239 
240   serialization::ModuleFile *MF =
241       MDC.ScanInstance.getASTReader()->getModuleManager().lookup(
242           M->getASTFile());
243   MDC.ScanInstance.getASTReader()->visitInputFiles(
244       *MF, true, true, [&](const serialization::InputFile &IF, bool isSystem) {
245         // __inferred_module.map is the result of the way in which an implicit
246         // module build handles inferred modules. It adds an overlay VFS with
247         // this file in the proper directory and relies on the rest of Clang to
248         // handle it like normal. With explicitly built modules we don't need
249         // to play VFS tricks, so replace it with the correct module map.
250         if (IF.getFile()->getName().endswith("__inferred_module.map")) {
251           MD.FileDeps.insert(ModuleMap->getName());
252           return;
253         }
254         MD.FileDeps.insert(IF.getFile()->getName());
255       });
256 
257   // Add direct prebuilt module dependencies now, so that we can use them when
258   // creating a CompilerInvocation and computing context hash for this
259   // ModuleDeps instance.
260   llvm::DenseSet<const Module *> SeenModules;
261   addAllSubmodulePrebuiltDeps(M, MD, SeenModules);
262 
263   MD.BuildInvocation = MDC.makeInvocationForModuleBuildWithoutPaths(
264       MD, [&](CompilerInvocation &BuildInvocation) {
265         if (MDC.OptimizeArgs)
266           optimizeHeaderSearchOpts(BuildInvocation.getHeaderSearchOpts(),
267                                    *MDC.ScanInstance.getASTReader(), *MF);
268       });
269   MD.ID.ContextHash = MD.BuildInvocation.getModuleHash();
270 
271   llvm::DenseSet<const Module *> AddedModules;
272   addAllSubmoduleDeps(M, MD, AddedModules);
273 
274   return MD.ID;
275 }
276 
277 void ModuleDepCollectorPP::addAllSubmodulePrebuiltDeps(
278     const Module *M, ModuleDeps &MD,
279     llvm::DenseSet<const Module *> &SeenSubmodules) {
280   addModulePrebuiltDeps(M, MD, SeenSubmodules);
281 
282   for (const Module *SubM : M->submodules())
283     addAllSubmodulePrebuiltDeps(SubM, MD, SeenSubmodules);
284 }
285 
286 void ModuleDepCollectorPP::addModulePrebuiltDeps(
287     const Module *M, ModuleDeps &MD,
288     llvm::DenseSet<const Module *> &SeenSubmodules) {
289   for (const Module *Import : M->Imports)
290     if (Import->getTopLevelModule() != M->getTopLevelModule())
291       if (MDC.isPrebuiltModule(Import->getTopLevelModule()))
292         if (SeenSubmodules.insert(Import->getTopLevelModule()).second)
293           MD.PrebuiltModuleDeps.emplace_back(Import->getTopLevelModule());
294 }
295 
296 void ModuleDepCollectorPP::addAllSubmoduleDeps(
297     const Module *M, ModuleDeps &MD,
298     llvm::DenseSet<const Module *> &AddedModules) {
299   addModuleDep(M, MD, AddedModules);
300 
301   for (const Module *SubM : M->submodules())
302     addAllSubmoduleDeps(SubM, MD, AddedModules);
303 }
304 
305 void ModuleDepCollectorPP::addModuleDep(
306     const Module *M, ModuleDeps &MD,
307     llvm::DenseSet<const Module *> &AddedModules) {
308   for (const Module *Import : M->Imports) {
309     if (Import->getTopLevelModule() != M->getTopLevelModule() &&
310         !MDC.isPrebuiltModule(Import)) {
311       ModuleID ImportID = handleTopLevelModule(Import->getTopLevelModule());
312       if (AddedModules.insert(Import->getTopLevelModule()).second)
313         MD.ClangModuleDeps.push_back(ImportID);
314     }
315   }
316 }
317 
318 ModuleDepCollector::ModuleDepCollector(
319     std::unique_ptr<DependencyOutputOptions> Opts,
320     CompilerInstance &ScanInstance, DependencyConsumer &C,
321     CompilerInvocation &&OriginalCI, bool OptimizeArgs)
322     : ScanInstance(ScanInstance), Consumer(C), Opts(std::move(Opts)),
323       OriginalInvocation(std::move(OriginalCI)), OptimizeArgs(OptimizeArgs) {}
324 
325 void ModuleDepCollector::attachToPreprocessor(Preprocessor &PP) {
326   PP.addPPCallbacks(std::make_unique<ModuleDepCollectorPP>(*this));
327 }
328 
329 void ModuleDepCollector::attachToASTReader(ASTReader &R) {}
330 
331 bool ModuleDepCollector::isPrebuiltModule(const Module *M) {
332   std::string Name(M->getTopLevelModuleName());
333   const auto &PrebuiltModuleFiles =
334       ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles;
335   auto PrebuiltModuleFileIt = PrebuiltModuleFiles.find(Name);
336   if (PrebuiltModuleFileIt == PrebuiltModuleFiles.end())
337     return false;
338   assert("Prebuilt module came from the expected AST file" &&
339          PrebuiltModuleFileIt->second == M->getASTFile()->getName());
340   return true;
341 }
342