1 //===- ModuleDepCollector.cpp - Callbacks to collect deps -------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
10 
11 #include "clang/Frontend/CompilerInstance.h"
12 #include "clang/Lex/Preprocessor.h"
13 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
14 #include "llvm/Support/StringSaver.h"
15 
16 using namespace clang;
17 using namespace tooling;
18 using namespace dependencies;
19 
20 static void optimizeHeaderSearchOpts(HeaderSearchOptions &Opts,
21                                      ASTReader &Reader,
22                                      const serialization::ModuleFile &MF) {
23   // Only preserve search paths that were used during the dependency scan.
24   std::vector<HeaderSearchOptions::Entry> Entries = Opts.UserEntries;
25   Opts.UserEntries.clear();
26   for (unsigned I = 0; I < Entries.size(); ++I)
27     if (MF.SearchPathUsage[I])
28       Opts.UserEntries.push_back(Entries[I]);
29 }
30 
31 CompilerInvocation ModuleDepCollector::makeInvocationForModuleBuildWithoutPaths(
32     const ModuleDeps &Deps,
33     llvm::function_ref<void(CompilerInvocation &)> Optimize) const {
34   // Make a deep copy of the original Clang invocation.
35   CompilerInvocation CI(OriginalInvocation);
36 
37   CI.getLangOpts()->resetNonModularOptions();
38   CI.getPreprocessorOpts().resetNonModularOptions();
39 
40   // Remove options incompatible with explicit module build.
41   CI.getFrontendOpts().Inputs.clear();
42   CI.getFrontendOpts().OutputFile.clear();
43 
44   CI.getFrontendOpts().ProgramAction = frontend::GenerateModule;
45   CI.getLangOpts()->ModuleName = Deps.ID.ModuleName;
46   CI.getFrontendOpts().IsSystemModule = Deps.IsSystem;
47 
48   CI.getLangOpts()->ImplicitModules = false;
49 
50   // Report the prebuilt modules this module uses.
51   for (const auto &PrebuiltModule : Deps.PrebuiltModuleDeps) {
52     CI.getFrontendOpts().ModuleFiles.push_back(PrebuiltModule.PCMFile);
53     CI.getFrontendOpts().ModuleMapFiles.push_back(PrebuiltModule.ModuleMapFile);
54   }
55 
56   Optimize(CI);
57 
58   // The original invocation probably didn't have strict context hash enabled.
59   // We will use the context hash of this invocation to distinguish between
60   // multiple incompatible versions of the same module and will use it when
61   // reporting dependencies to the clients. Let's make sure we're using
62   // **strict** context hash in order to prevent accidental sharing of
63   // incompatible modules (e.g. with differences in search paths).
64   CI.getHeaderSearchOpts().ModulesStrictContextHash = true;
65 
66   return CI;
67 }
68 
69 static std::vector<std::string>
70 serializeCompilerInvocation(const CompilerInvocation &CI) {
71   // Set up string allocator.
72   llvm::BumpPtrAllocator Alloc;
73   llvm::StringSaver Strings(Alloc);
74   auto SA = [&Strings](const Twine &Arg) { return Strings.save(Arg).data(); };
75 
76   // Synthesize full command line from the CompilerInvocation, including "-cc1".
77   SmallVector<const char *, 32> Args{"-cc1"};
78   CI.generateCC1CommandLine(Args, SA);
79 
80   // Convert arguments to the return type.
81   return std::vector<std::string>{Args.begin(), Args.end()};
82 }
83 
84 std::vector<std::string> ModuleDeps::getCanonicalCommandLine(
85     std::function<StringRef(ModuleID)> LookupPCMPath,
86     std::function<const ModuleDeps &(ModuleID)> LookupModuleDeps) const {
87   CompilerInvocation CI(Invocation);
88   FrontendOptions &FrontendOpts = CI.getFrontendOpts();
89 
90   InputKind ModuleMapInputKind(FrontendOpts.DashX.getLanguage(),
91                                InputKind::Format::ModuleMap);
92   FrontendOpts.Inputs.emplace_back(ClangModuleMapFile, ModuleMapInputKind);
93   FrontendOpts.OutputFile = std::string(LookupPCMPath(ID));
94 
95   dependencies::detail::collectPCMAndModuleMapPaths(
96       ClangModuleDeps, LookupPCMPath, LookupModuleDeps,
97       FrontendOpts.ModuleFiles, FrontendOpts.ModuleMapFiles);
98 
99   return serializeCompilerInvocation(CI);
100 }
101 
102 std::vector<std::string>
103 ModuleDeps::getCanonicalCommandLineWithoutModulePaths() const {
104   return serializeCompilerInvocation(Invocation);
105 }
106 
107 void dependencies::detail::collectPCMAndModuleMapPaths(
108     llvm::ArrayRef<ModuleID> Modules,
109     std::function<StringRef(ModuleID)> LookupPCMPath,
110     std::function<const ModuleDeps &(ModuleID)> LookupModuleDeps,
111     std::vector<std::string> &PCMPaths, std::vector<std::string> &ModMapPaths) {
112   llvm::StringSet<> AlreadyAdded;
113 
114   std::function<void(llvm::ArrayRef<ModuleID>)> AddArgs =
115       [&](llvm::ArrayRef<ModuleID> Modules) {
116         for (const ModuleID &MID : Modules) {
117           if (!AlreadyAdded.insert(MID.ModuleName + MID.ContextHash).second)
118             continue;
119           const ModuleDeps &M = LookupModuleDeps(MID);
120           // Depth first traversal.
121           AddArgs(M.ClangModuleDeps);
122           PCMPaths.push_back(LookupPCMPath(MID).str());
123           if (!M.ClangModuleMapFile.empty())
124             ModMapPaths.push_back(M.ClangModuleMapFile);
125         }
126       };
127 
128   AddArgs(Modules);
129 }
130 
131 void ModuleDepCollectorPP::FileChanged(SourceLocation Loc,
132                                        FileChangeReason Reason,
133                                        SrcMgr::CharacteristicKind FileType,
134                                        FileID PrevFID) {
135   if (Reason != PPCallbacks::EnterFile)
136     return;
137 
138   // This has to be delayed as the context hash can change at the start of
139   // `CompilerInstance::ExecuteAction`.
140   if (MDC.ContextHash.empty()) {
141     MDC.ContextHash = MDC.Instance.getInvocation().getModuleHash();
142     MDC.Consumer.handleContextHash(MDC.ContextHash);
143   }
144 
145   SourceManager &SM = MDC.Instance.getSourceManager();
146 
147   // Dependency generation really does want to go all the way to the
148   // file entry for a source location to find out what is depended on.
149   // We do not want #line markers to affect dependency generation!
150   if (Optional<StringRef> Filename =
151           SM.getNonBuiltinFilenameForID(SM.getFileID(SM.getExpansionLoc(Loc))))
152     MDC.FileDeps.push_back(
153         std::string(llvm::sys::path::remove_leading_dotslash(*Filename)));
154 }
155 
156 void ModuleDepCollectorPP::InclusionDirective(
157     SourceLocation HashLoc, const Token &IncludeTok, StringRef FileName,
158     bool IsAngled, CharSourceRange FilenameRange, const FileEntry *File,
159     StringRef SearchPath, StringRef RelativePath, const Module *Imported,
160     SrcMgr::CharacteristicKind FileType) {
161   if (!File && !Imported) {
162     // This is a non-modular include that HeaderSearch failed to find. Add it
163     // here as `FileChanged` will never see it.
164     MDC.FileDeps.push_back(std::string(FileName));
165   }
166   handleImport(Imported);
167 }
168 
169 void ModuleDepCollectorPP::moduleImport(SourceLocation ImportLoc,
170                                         ModuleIdPath Path,
171                                         const Module *Imported) {
172   handleImport(Imported);
173 }
174 
175 void ModuleDepCollectorPP::handleImport(const Module *Imported) {
176   if (!Imported)
177     return;
178 
179   const Module *TopLevelModule = Imported->getTopLevelModule();
180 
181   if (MDC.isPrebuiltModule(TopLevelModule))
182     DirectPrebuiltModularDeps.insert(TopLevelModule);
183   else
184     DirectModularDeps.insert(TopLevelModule);
185 }
186 
187 void ModuleDepCollectorPP::EndOfMainFile() {
188   FileID MainFileID = MDC.Instance.getSourceManager().getMainFileID();
189   MDC.MainFile = std::string(
190       MDC.Instance.getSourceManager().getFileEntryForID(MainFileID)->getName());
191 
192   if (!MDC.Instance.getPreprocessorOpts().ImplicitPCHInclude.empty())
193     MDC.FileDeps.push_back(
194         MDC.Instance.getPreprocessorOpts().ImplicitPCHInclude);
195 
196   for (const Module *M : DirectModularDeps) {
197     // A top-level module might not be actually imported as a module when
198     // -fmodule-name is used to compile a translation unit that imports this
199     // module. In that case it can be skipped. The appropriate header
200     // dependencies will still be reported as expected.
201     if (!M->getASTFile())
202       continue;
203     handleTopLevelModule(M);
204   }
205 
206   MDC.Consumer.handleDependencyOutputOpts(*MDC.Opts);
207 
208   for (auto &&I : MDC.ModularDeps)
209     MDC.Consumer.handleModuleDependency(I.second);
210 
211   for (auto &&I : MDC.FileDeps)
212     MDC.Consumer.handleFileDependency(I);
213 
214   for (auto &&I : DirectPrebuiltModularDeps)
215     MDC.Consumer.handlePrebuiltModuleDependency(PrebuiltModuleDep{I});
216 }
217 
218 ModuleID ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
219   assert(M == M->getTopLevelModule() && "Expected top level module!");
220 
221   // If this module has been handled already, just return its ID.
222   auto ModI = MDC.ModularDeps.insert({M, ModuleDeps{}});
223   if (!ModI.second)
224     return ModI.first->second.ID;
225 
226   ModuleDeps &MD = ModI.first->second;
227 
228   MD.ID.ModuleName = M->getFullModuleName();
229   MD.ImportedByMainFile = DirectModularDeps.contains(M);
230   MD.ImplicitModulePCMPath = std::string(M->getASTFile()->getName());
231   MD.IsSystem = M->IsSystem;
232 
233   const FileEntry *ModuleMap = MDC.Instance.getPreprocessor()
234                                    .getHeaderSearchInfo()
235                                    .getModuleMap()
236                                    .getModuleMapFileForUniquing(M);
237   MD.ClangModuleMapFile = std::string(ModuleMap ? ModuleMap->getName() : "");
238 
239   serialization::ModuleFile *MF =
240       MDC.Instance.getASTReader()->getModuleManager().lookup(M->getASTFile());
241   MDC.Instance.getASTReader()->visitInputFiles(
242       *MF, true, true, [&](const serialization::InputFile &IF, bool isSystem) {
243         // __inferred_module.map is the result of the way in which an implicit
244         // module build handles inferred modules. It adds an overlay VFS with
245         // this file in the proper directory and relies on the rest of Clang to
246         // handle it like normal. With explicitly built modules we don't need
247         // to play VFS tricks, so replace it with the correct module map.
248         if (IF.getFile()->getName().endswith("__inferred_module.map")) {
249           MD.FileDeps.insert(ModuleMap->getName());
250           return;
251         }
252         MD.FileDeps.insert(IF.getFile()->getName());
253       });
254 
255   // Add direct prebuilt module dependencies now, so that we can use them when
256   // creating a CompilerInvocation and computing context hash for this
257   // ModuleDeps instance.
258   llvm::DenseSet<const Module *> SeenModules;
259   addAllSubmodulePrebuiltDeps(M, MD, SeenModules);
260 
261   MD.Invocation = MDC.makeInvocationForModuleBuildWithoutPaths(
262       MD, [&](CompilerInvocation &CI) {
263         if (MDC.OptimizeArgs)
264           optimizeHeaderSearchOpts(CI.getHeaderSearchOpts(),
265                                    *MDC.Instance.getASTReader(), *MF);
266       });
267   MD.ID.ContextHash = MD.Invocation.getModuleHash();
268 
269   llvm::DenseSet<const Module *> AddedModules;
270   addAllSubmoduleDeps(M, MD, AddedModules);
271 
272   return MD.ID;
273 }
274 
275 void ModuleDepCollectorPP::addAllSubmodulePrebuiltDeps(
276     const Module *M, ModuleDeps &MD,
277     llvm::DenseSet<const Module *> &SeenSubmodules) {
278   addModulePrebuiltDeps(M, MD, SeenSubmodules);
279 
280   for (const Module *SubM : M->submodules())
281     addAllSubmodulePrebuiltDeps(SubM, MD, SeenSubmodules);
282 }
283 
284 void ModuleDepCollectorPP::addModulePrebuiltDeps(
285     const Module *M, ModuleDeps &MD,
286     llvm::DenseSet<const Module *> &SeenSubmodules) {
287   for (const Module *Import : M->Imports)
288     if (Import->getTopLevelModule() != M->getTopLevelModule())
289       if (MDC.isPrebuiltModule(Import->getTopLevelModule()))
290         if (SeenSubmodules.insert(Import->getTopLevelModule()).second)
291           MD.PrebuiltModuleDeps.emplace_back(Import->getTopLevelModule());
292 }
293 
294 void ModuleDepCollectorPP::addAllSubmoduleDeps(
295     const Module *M, ModuleDeps &MD,
296     llvm::DenseSet<const Module *> &AddedModules) {
297   addModuleDep(M, MD, AddedModules);
298 
299   for (const Module *SubM : M->submodules())
300     addAllSubmoduleDeps(SubM, MD, AddedModules);
301 }
302 
303 void ModuleDepCollectorPP::addModuleDep(
304     const Module *M, ModuleDeps &MD,
305     llvm::DenseSet<const Module *> &AddedModules) {
306   for (const Module *Import : M->Imports) {
307     if (Import->getTopLevelModule() != M->getTopLevelModule() &&
308         !MDC.isPrebuiltModule(Import)) {
309       ModuleID ImportID = handleTopLevelModule(Import->getTopLevelModule());
310       if (AddedModules.insert(Import->getTopLevelModule()).second)
311         MD.ClangModuleDeps.push_back(ImportID);
312     }
313   }
314 }
315 
316 ModuleDepCollector::ModuleDepCollector(
317     std::unique_ptr<DependencyOutputOptions> Opts, CompilerInstance &I,
318     DependencyConsumer &C, CompilerInvocation &&OriginalCI, bool OptimizeArgs)
319     : Instance(I), Consumer(C), Opts(std::move(Opts)),
320       OriginalInvocation(std::move(OriginalCI)), OptimizeArgs(OptimizeArgs) {}
321 
322 void ModuleDepCollector::attachToPreprocessor(Preprocessor &PP) {
323   PP.addPPCallbacks(std::make_unique<ModuleDepCollectorPP>(*this));
324 }
325 
326 void ModuleDepCollector::attachToASTReader(ASTReader &R) {}
327 
328 bool ModuleDepCollector::isPrebuiltModule(const Module *M) {
329   std::string Name(M->getTopLevelModuleName());
330   const auto &PrebuiltModuleFiles =
331       Instance.getHeaderSearchOpts().PrebuiltModuleFiles;
332   auto PrebuiltModuleFileIt = PrebuiltModuleFiles.find(Name);
333   if (PrebuiltModuleFileIt == PrebuiltModuleFiles.end())
334     return false;
335   assert("Prebuilt module came from the expected AST file" &&
336          PrebuiltModuleFileIt->second == M->getASTFile()->getName());
337   return true;
338 }
339