1 //===- ModuleDepCollector.cpp - Callbacks to collect deps -------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
10 
11 #include "clang/Frontend/CompilerInstance.h"
12 #include "clang/Lex/Preprocessor.h"
13 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
14 #include "llvm/Support/StringSaver.h"
15 
16 using namespace clang;
17 using namespace tooling;
18 using namespace dependencies;
19 
20 static void optimizeHeaderSearchOpts(HeaderSearchOptions &Opts,
21                                      ASTReader &Reader,
22                                      const serialization::ModuleFile &MF) {
23   // Only preserve search paths that were used during the dependency scan.
24   std::vector<HeaderSearchOptions::Entry> Entries = Opts.UserEntries;
25   Opts.UserEntries.clear();
26   for (unsigned I = 0; I < Entries.size(); ++I)
27     if (MF.SearchPathUsage[I])
28       Opts.UserEntries.push_back(Entries[I]);
29 }
30 
31 CompilerInvocation ModuleDepCollector::makeInvocationForModuleBuildWithoutPaths(
32     const ModuleDeps &Deps,
33     llvm::function_ref<void(CompilerInvocation &)> Optimize) const {
34   // Make a deep copy of the original Clang invocation.
35   CompilerInvocation CI(OriginalInvocation);
36 
37   CI.getLangOpts()->resetNonModularOptions();
38   CI.getPreprocessorOpts().resetNonModularOptions();
39 
40   // Remove options incompatible with explicit module build.
41   CI.getFrontendOpts().Inputs.clear();
42   CI.getFrontendOpts().OutputFile.clear();
43 
44   CI.getFrontendOpts().ProgramAction = frontend::GenerateModule;
45   CI.getLangOpts()->ModuleName = Deps.ID.ModuleName;
46   CI.getFrontendOpts().IsSystemModule = Deps.IsSystem;
47 
48   CI.getLangOpts()->ImplicitModules = false;
49 
50   // Report the prebuilt modules this module uses.
51   for (const auto &PrebuiltModule : Deps.PrebuiltModuleDeps) {
52     CI.getFrontendOpts().ModuleFiles.push_back(PrebuiltModule.PCMFile);
53     CI.getFrontendOpts().ModuleMapFiles.push_back(PrebuiltModule.ModuleMapFile);
54   }
55 
56   Optimize(CI);
57 
58   // The original invocation probably didn't have strict context hash enabled.
59   // We will use the context hash of this invocation to distinguish between
60   // multiple incompatible versions of the same module and will use it when
61   // reporting dependencies to the clients. Let's make sure we're using
62   // **strict** context hash in order to prevent accidental sharing of
63   // incompatible modules (e.g. with differences in search paths).
64   CI.getHeaderSearchOpts().ModulesStrictContextHash = true;
65 
66   return CI;
67 }
68 
69 static std::vector<std::string>
70 serializeCompilerInvocation(const CompilerInvocation &CI) {
71   // Set up string allocator.
72   llvm::BumpPtrAllocator Alloc;
73   llvm::StringSaver Strings(Alloc);
74   auto SA = [&Strings](const Twine &Arg) { return Strings.save(Arg).data(); };
75 
76   // Synthesize full command line from the CompilerInvocation, including "-cc1".
77   SmallVector<const char *, 32> Args{"-cc1"};
78   CI.generateCC1CommandLine(Args, SA);
79 
80   // Convert arguments to the return type.
81   return std::vector<std::string>{Args.begin(), Args.end()};
82 }
83 
84 std::vector<std::string> ModuleDeps::getCanonicalCommandLine(
85     std::function<StringRef(ModuleID)> LookupPCMPath,
86     std::function<const ModuleDeps &(ModuleID)> LookupModuleDeps) const {
87   CompilerInvocation CI(Invocation);
88   FrontendOptions &FrontendOpts = CI.getFrontendOpts();
89 
90   InputKind ModuleMapInputKind(FrontendOpts.DashX.getLanguage(),
91                                InputKind::Format::ModuleMap);
92   FrontendOpts.Inputs.emplace_back(ClangModuleMapFile, ModuleMapInputKind);
93   FrontendOpts.OutputFile = std::string(LookupPCMPath(ID));
94 
95   dependencies::detail::collectPCMAndModuleMapPaths(
96       ClangModuleDeps, LookupPCMPath, LookupModuleDeps,
97       FrontendOpts.ModuleFiles, FrontendOpts.ModuleMapFiles);
98 
99   return serializeCompilerInvocation(CI);
100 }
101 
102 std::vector<std::string>
103 ModuleDeps::getCanonicalCommandLineWithoutModulePaths() const {
104   return serializeCompilerInvocation(Invocation);
105 }
106 
107 void dependencies::detail::collectPCMAndModuleMapPaths(
108     llvm::ArrayRef<ModuleID> Modules,
109     std::function<StringRef(ModuleID)> LookupPCMPath,
110     std::function<const ModuleDeps &(ModuleID)> LookupModuleDeps,
111     std::vector<std::string> &PCMPaths, std::vector<std::string> &ModMapPaths) {
112   llvm::StringSet<> AlreadyAdded;
113 
114   std::function<void(llvm::ArrayRef<ModuleID>)> AddArgs =
115       [&](llvm::ArrayRef<ModuleID> Modules) {
116         for (const ModuleID &MID : Modules) {
117           if (!AlreadyAdded.insert(MID.ModuleName + MID.ContextHash).second)
118             continue;
119           const ModuleDeps &M = LookupModuleDeps(MID);
120           // Depth first traversal.
121           AddArgs(M.ClangModuleDeps);
122           PCMPaths.push_back(LookupPCMPath(MID).str());
123           if (!M.ClangModuleMapFile.empty())
124             ModMapPaths.push_back(M.ClangModuleMapFile);
125         }
126       };
127 
128   AddArgs(Modules);
129 }
130 
131 void ModuleDepCollectorPP::FileChanged(SourceLocation Loc,
132                                        FileChangeReason Reason,
133                                        SrcMgr::CharacteristicKind FileType,
134                                        FileID PrevFID) {
135   if (Reason != PPCallbacks::EnterFile)
136     return;
137 
138   // This has to be delayed as the context hash can change at the start of
139   // `CompilerInstance::ExecuteAction`.
140   if (MDC.ContextHash.empty()) {
141     MDC.ContextHash = Instance.getInvocation().getModuleHash();
142     MDC.Consumer.handleContextHash(MDC.ContextHash);
143   }
144 
145   SourceManager &SM = Instance.getSourceManager();
146 
147   // Dependency generation really does want to go all the way to the
148   // file entry for a source location to find out what is depended on.
149   // We do not want #line markers to affect dependency generation!
150   if (Optional<StringRef> Filename =
151           SM.getNonBuiltinFilenameForID(SM.getFileID(SM.getExpansionLoc(Loc))))
152     MDC.FileDeps.push_back(
153         std::string(llvm::sys::path::remove_leading_dotslash(*Filename)));
154 }
155 
156 void ModuleDepCollectorPP::InclusionDirective(
157     SourceLocation HashLoc, const Token &IncludeTok, StringRef FileName,
158     bool IsAngled, CharSourceRange FilenameRange, const FileEntry *File,
159     StringRef SearchPath, StringRef RelativePath, const Module *Imported,
160     SrcMgr::CharacteristicKind FileType) {
161   if (!File && !Imported) {
162     // This is a non-modular include that HeaderSearch failed to find. Add it
163     // here as `FileChanged` will never see it.
164     MDC.FileDeps.push_back(std::string(FileName));
165   }
166   handleImport(Imported);
167 }
168 
169 void ModuleDepCollectorPP::moduleImport(SourceLocation ImportLoc,
170                                         ModuleIdPath Path,
171                                         const Module *Imported) {
172   handleImport(Imported);
173 }
174 
175 void ModuleDepCollectorPP::handleImport(const Module *Imported) {
176   if (!Imported)
177     return;
178 
179   const Module *TopLevelModule = Imported->getTopLevelModule();
180 
181   if (MDC.isPrebuiltModule(TopLevelModule))
182     DirectPrebuiltModularDeps.insert(TopLevelModule);
183   else
184     DirectModularDeps.insert(TopLevelModule);
185 }
186 
187 void ModuleDepCollectorPP::EndOfMainFile() {
188   FileID MainFileID = Instance.getSourceManager().getMainFileID();
189   MDC.MainFile = std::string(
190       Instance.getSourceManager().getFileEntryForID(MainFileID)->getName());
191 
192   if (!Instance.getPreprocessorOpts().ImplicitPCHInclude.empty())
193     MDC.FileDeps.push_back(Instance.getPreprocessorOpts().ImplicitPCHInclude);
194 
195   for (const Module *M : DirectModularDeps) {
196     // A top-level module might not be actually imported as a module when
197     // -fmodule-name is used to compile a translation unit that imports this
198     // module. In that case it can be skipped. The appropriate header
199     // dependencies will still be reported as expected.
200     if (!M->getASTFile())
201       continue;
202     handleTopLevelModule(M);
203   }
204 
205   MDC.Consumer.handleDependencyOutputOpts(*MDC.Opts);
206 
207   for (auto &&I : MDC.ModularDeps)
208     MDC.Consumer.handleModuleDependency(I.second);
209 
210   for (auto &&I : MDC.FileDeps)
211     MDC.Consumer.handleFileDependency(I);
212 
213   for (auto &&I : DirectPrebuiltModularDeps)
214     MDC.Consumer.handlePrebuiltModuleDependency(PrebuiltModuleDep{I});
215 }
216 
217 ModuleID ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
218   assert(M == M->getTopLevelModule() && "Expected top level module!");
219 
220   // If this module has been handled already, just return its ID.
221   auto ModI = MDC.ModularDeps.insert({M, ModuleDeps{}});
222   if (!ModI.second)
223     return ModI.first->second.ID;
224 
225   ModuleDeps &MD = ModI.first->second;
226 
227   MD.ID.ModuleName = M->getFullModuleName();
228   MD.ImportedByMainFile = DirectModularDeps.contains(M);
229   MD.ImplicitModulePCMPath = std::string(M->getASTFile()->getName());
230   MD.IsSystem = M->IsSystem;
231 
232   const FileEntry *ModuleMap = Instance.getPreprocessor()
233                                    .getHeaderSearchInfo()
234                                    .getModuleMap()
235                                    .getModuleMapFileForUniquing(M);
236   MD.ClangModuleMapFile = std::string(ModuleMap ? ModuleMap->getName() : "");
237 
238   serialization::ModuleFile *MF =
239       MDC.Instance.getASTReader()->getModuleManager().lookup(M->getASTFile());
240   MDC.Instance.getASTReader()->visitInputFiles(
241       *MF, true, true, [&](const serialization::InputFile &IF, bool isSystem) {
242         // __inferred_module.map is the result of the way in which an implicit
243         // module build handles inferred modules. It adds an overlay VFS with
244         // this file in the proper directory and relies on the rest of Clang to
245         // handle it like normal. With explicitly built modules we don't need
246         // to play VFS tricks, so replace it with the correct module map.
247         if (IF.getFile()->getName().endswith("__inferred_module.map")) {
248           MD.FileDeps.insert(ModuleMap->getName());
249           return;
250         }
251         MD.FileDeps.insert(IF.getFile()->getName());
252       });
253 
254   // Add direct prebuilt module dependencies now, so that we can use them when
255   // creating a CompilerInvocation and computing context hash for this
256   // ModuleDeps instance.
257   llvm::DenseSet<const Module *> SeenModules;
258   addAllSubmodulePrebuiltDeps(M, MD, SeenModules);
259 
260   MD.Invocation = MDC.makeInvocationForModuleBuildWithoutPaths(
261       MD, [&](CompilerInvocation &CI) {
262         if (MDC.OptimizeArgs)
263           optimizeHeaderSearchOpts(CI.getHeaderSearchOpts(),
264                                    *MDC.Instance.getASTReader(), *MF);
265       });
266   MD.ID.ContextHash = MD.Invocation.getModuleHash();
267 
268   llvm::DenseSet<const Module *> AddedModules;
269   addAllSubmoduleDeps(M, MD, AddedModules);
270 
271   return MD.ID;
272 }
273 
274 void ModuleDepCollectorPP::addAllSubmodulePrebuiltDeps(
275     const Module *M, ModuleDeps &MD,
276     llvm::DenseSet<const Module *> &SeenSubmodules) {
277   addModulePrebuiltDeps(M, MD, SeenSubmodules);
278 
279   for (const Module *SubM : M->submodules())
280     addAllSubmodulePrebuiltDeps(SubM, MD, SeenSubmodules);
281 }
282 
283 void ModuleDepCollectorPP::addModulePrebuiltDeps(
284     const Module *M, ModuleDeps &MD,
285     llvm::DenseSet<const Module *> &SeenSubmodules) {
286   for (const Module *Import : M->Imports)
287     if (Import->getTopLevelModule() != M->getTopLevelModule())
288       if (MDC.isPrebuiltModule(Import->getTopLevelModule()))
289         if (SeenSubmodules.insert(Import->getTopLevelModule()).second)
290           MD.PrebuiltModuleDeps.emplace_back(Import->getTopLevelModule());
291 }
292 
293 void ModuleDepCollectorPP::addAllSubmoduleDeps(
294     const Module *M, ModuleDeps &MD,
295     llvm::DenseSet<const Module *> &AddedModules) {
296   addModuleDep(M, MD, AddedModules);
297 
298   for (const Module *SubM : M->submodules())
299     addAllSubmoduleDeps(SubM, MD, AddedModules);
300 }
301 
302 void ModuleDepCollectorPP::addModuleDep(
303     const Module *M, ModuleDeps &MD,
304     llvm::DenseSet<const Module *> &AddedModules) {
305   for (const Module *Import : M->Imports) {
306     if (Import->getTopLevelModule() != M->getTopLevelModule() &&
307         !MDC.isPrebuiltModule(Import)) {
308       ModuleID ImportID = handleTopLevelModule(Import->getTopLevelModule());
309       if (AddedModules.insert(Import->getTopLevelModule()).second)
310         MD.ClangModuleDeps.push_back(ImportID);
311     }
312   }
313 }
314 
315 ModuleDepCollector::ModuleDepCollector(
316     std::unique_ptr<DependencyOutputOptions> Opts, CompilerInstance &I,
317     DependencyConsumer &C, CompilerInvocation &&OriginalCI, bool OptimizeArgs)
318     : Instance(I), Consumer(C), Opts(std::move(Opts)),
319       OriginalInvocation(std::move(OriginalCI)), OptimizeArgs(OptimizeArgs) {}
320 
321 void ModuleDepCollector::attachToPreprocessor(Preprocessor &PP) {
322   PP.addPPCallbacks(std::make_unique<ModuleDepCollectorPP>(Instance, *this));
323 }
324 
325 void ModuleDepCollector::attachToASTReader(ASTReader &R) {}
326 
327 bool ModuleDepCollector::isPrebuiltModule(const Module *M) {
328   std::string Name(M->getTopLevelModuleName());
329   const auto &PrebuiltModuleFiles =
330       Instance.getHeaderSearchOpts().PrebuiltModuleFiles;
331   auto PrebuiltModuleFileIt = PrebuiltModuleFiles.find(Name);
332   if (PrebuiltModuleFileIt == PrebuiltModuleFiles.end())
333     return false;
334   assert("Prebuilt module came from the expected AST file" &&
335          PrebuiltModuleFileIt->second == M->getASTFile()->getName());
336   return true;
337 }
338