1 //===-- ClangModulesDeclVendor.cpp ------------------------------*- C++ -*-===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 // C Includes
11 // C++ Includes
12 #include <mutex>
13 
14 // Other libraries and framework includes
15 #include "clang/Basic/TargetInfo.h"
16 #include "clang/Frontend/CompilerInstance.h"
17 #include "clang/Frontend/FrontendActions.h"
18 #include "clang/Lex/Preprocessor.h"
19 #include "clang/Lex/PreprocessorOptions.h"
20 #include "clang/Parse/Parser.h"
21 #include "clang/Sema/Lookup.h"
22 #include "clang/Serialization/ASTReader.h"
23 #include "llvm/Support/FileSystem.h"
24 #include "llvm/Support/Path.h"
25 #include "llvm/Support/Threading.h"
26 
27 // Project includes
28 #include "ClangModulesDeclVendor.h"
29 
30 #include "lldb/Core/ModuleList.h"
31 #include "lldb/Host/Host.h"
32 #include "lldb/Host/HostInfo.h"
33 #include "lldb/Symbol/CompileUnit.h"
34 #include "lldb/Target/Target.h"
35 #include "lldb/Utility/FileSpec.h"
36 #include "lldb/Utility/LLDBAssert.h"
37 #include "lldb/Utility/Log.h"
38 #include "lldb/Utility/StreamString.h"
39 
40 using namespace lldb_private;
41 
42 namespace {
43 // Any Clang compiler requires a consumer for diagnostics.  This one stores
44 // them as strings so we can provide them to the user in case a module failed
45 // to load.
46 class StoringDiagnosticConsumer : public clang::DiagnosticConsumer {
47 public:
48   StoringDiagnosticConsumer();
49 
50   void HandleDiagnostic(clang::DiagnosticsEngine::Level DiagLevel,
51                         const clang::Diagnostic &info) override;
52 
53   void ClearDiagnostics();
54 
55   void DumpDiagnostics(Stream &error_stream);
56 
57 private:
58   typedef std::pair<clang::DiagnosticsEngine::Level, std::string>
59       IDAndDiagnostic;
60   std::vector<IDAndDiagnostic> m_diagnostics;
61   Log *m_log;
62 };
63 
64 // The private implementation of our ClangModulesDeclVendor.  Contains all the
65 // Clang state required to load modules.
66 class ClangModulesDeclVendorImpl : public ClangModulesDeclVendor {
67 public:
68   ClangModulesDeclVendorImpl(
69       llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_engine,
70       std::shared_ptr<clang::CompilerInvocation> compiler_invocation,
71       std::unique_ptr<clang::CompilerInstance> compiler_instance,
72       std::unique_ptr<clang::Parser> parser);
73 
74   ~ClangModulesDeclVendorImpl() override = default;
75 
76   bool AddModule(ModulePath &path, ModuleVector *exported_modules,
77                  Stream &error_stream) override;
78 
79   bool AddModulesForCompileUnit(CompileUnit &cu, ModuleVector &exported_modules,
80                                 Stream &error_stream) override;
81 
82   uint32_t FindDecls(const ConstString &name, bool append, uint32_t max_matches,
83                      std::vector<clang::NamedDecl *> &decls) override;
84 
85   void ForEachMacro(const ModuleVector &modules,
86                     std::function<bool(const std::string &)> handler) override;
87 
88   clang::ExternalASTMerger::ImporterSource GetImporterSource() override;
89 private:
90   void
91   ReportModuleExportsHelper(std::set<ClangModulesDeclVendor::ModuleID> &exports,
92                             clang::Module *module);
93 
94   void ReportModuleExports(ModuleVector &exports, clang::Module *module);
95 
96   clang::ModuleLoadResult DoGetModule(clang::ModuleIdPath path,
97                                       bool make_visible);
98 
99   bool m_enabled = false;
100 
101   llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> m_diagnostics_engine;
102   std::shared_ptr<clang::CompilerInvocation> m_compiler_invocation;
103   std::unique_ptr<clang::CompilerInstance> m_compiler_instance;
104   std::unique_ptr<clang::Parser> m_parser;
105   size_t m_source_location_index =
106       0; // used to give name components fake SourceLocations
107 
108   typedef std::vector<ConstString> ImportedModule;
109   typedef std::map<ImportedModule, clang::Module *> ImportedModuleMap;
110   typedef std::set<ModuleID> ImportedModuleSet;
111   ImportedModuleMap m_imported_modules;
112   ImportedModuleSet m_user_imported_modules;
113   const clang::ExternalASTMerger::OriginMap m_origin_map;
114 };
115 } // anonymous namespace
116 
117 StoringDiagnosticConsumer::StoringDiagnosticConsumer() {
118   m_log = lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_EXPRESSIONS);
119 }
120 
121 void StoringDiagnosticConsumer::HandleDiagnostic(
122     clang::DiagnosticsEngine::Level DiagLevel, const clang::Diagnostic &info) {
123   llvm::SmallVector<char, 256> diagnostic_string;
124 
125   info.FormatDiagnostic(diagnostic_string);
126 
127   m_diagnostics.push_back(
128       IDAndDiagnostic(DiagLevel, std::string(diagnostic_string.data(),
129                                              diagnostic_string.size())));
130 }
131 
132 void StoringDiagnosticConsumer::ClearDiagnostics() { m_diagnostics.clear(); }
133 
134 void StoringDiagnosticConsumer::DumpDiagnostics(Stream &error_stream) {
135   for (IDAndDiagnostic &diag : m_diagnostics) {
136     switch (diag.first) {
137     default:
138       error_stream.PutCString(diag.second);
139       error_stream.PutChar('\n');
140       break;
141     case clang::DiagnosticsEngine::Level::Ignored:
142       break;
143     }
144   }
145 }
146 
147 static FileSpec GetResourceDir() {
148   static FileSpec g_cached_resource_dir;
149 
150   static llvm::once_flag g_once_flag;
151 
152   llvm::call_once(g_once_flag, []() {
153     HostInfo::GetLLDBPath(lldb::ePathTypeClangDir, g_cached_resource_dir);
154   });
155 
156   return g_cached_resource_dir;
157 }
158 
159 ClangModulesDeclVendor::ClangModulesDeclVendor() {}
160 
161 ClangModulesDeclVendor::~ClangModulesDeclVendor() {}
162 
163 ClangModulesDeclVendorImpl::ClangModulesDeclVendorImpl(
164     llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_engine,
165     std::shared_ptr<clang::CompilerInvocation> compiler_invocation,
166     std::unique_ptr<clang::CompilerInstance> compiler_instance,
167     std::unique_ptr<clang::Parser> parser)
168     : m_diagnostics_engine(std::move(diagnostics_engine)),
169       m_compiler_invocation(std::move(compiler_invocation)),
170       m_compiler_instance(std::move(compiler_instance)),
171       m_parser(std::move(parser)), m_origin_map() {}
172 
173 void ClangModulesDeclVendorImpl::ReportModuleExportsHelper(
174     std::set<ClangModulesDeclVendor::ModuleID> &exports,
175     clang::Module *module) {
176   if (exports.count(reinterpret_cast<ClangModulesDeclVendor::ModuleID>(module)))
177     return;
178 
179   exports.insert(reinterpret_cast<ClangModulesDeclVendor::ModuleID>(module));
180 
181   llvm::SmallVector<clang::Module *, 2> sub_exports;
182 
183   module->getExportedModules(sub_exports);
184 
185   for (clang::Module *module : sub_exports) {
186     ReportModuleExportsHelper(exports, module);
187   }
188 }
189 
190 void ClangModulesDeclVendorImpl::ReportModuleExports(
191     ClangModulesDeclVendor::ModuleVector &exports, clang::Module *module) {
192   std::set<ClangModulesDeclVendor::ModuleID> exports_set;
193 
194   ReportModuleExportsHelper(exports_set, module);
195 
196   for (ModuleID module : exports_set) {
197     exports.push_back(module);
198   }
199 }
200 
201 bool ClangModulesDeclVendorImpl::AddModule(ModulePath &path,
202                                            ModuleVector *exported_modules,
203                                            Stream &error_stream) {
204   // Fail early.
205 
206   if (m_compiler_instance->hadModuleLoaderFatalFailure()) {
207     error_stream.PutCString("error: Couldn't load a module because the module "
208                             "loader is in a fatal state.\n");
209     return false;
210   }
211 
212   // Check if we've already imported this module.
213 
214   std::vector<ConstString> imported_module;
215 
216   for (ConstString path_component : path) {
217     imported_module.push_back(path_component);
218   }
219 
220   {
221     ImportedModuleMap::iterator mi = m_imported_modules.find(imported_module);
222 
223     if (mi != m_imported_modules.end()) {
224       if (exported_modules) {
225         ReportModuleExports(*exported_modules, mi->second);
226       }
227       return true;
228     }
229   }
230 
231   if (!m_compiler_instance->getPreprocessor()
232            .getHeaderSearchInfo()
233            .lookupModule(path[0].GetStringRef())) {
234     error_stream.Printf("error: Header search couldn't locate module %s\n",
235                         path[0].AsCString());
236     return false;
237   }
238 
239   llvm::SmallVector<std::pair<clang::IdentifierInfo *, clang::SourceLocation>,
240                     4>
241       clang_path;
242 
243   {
244     clang::SourceManager &source_manager =
245         m_compiler_instance->getASTContext().getSourceManager();
246 
247     for (ConstString path_component : path) {
248       clang_path.push_back(std::make_pair(
249           &m_compiler_instance->getASTContext().Idents.get(
250               path_component.GetStringRef()),
251           source_manager.getLocForStartOfFile(source_manager.getMainFileID())
252               .getLocWithOffset(m_source_location_index++)));
253     }
254   }
255 
256   StoringDiagnosticConsumer *diagnostic_consumer =
257       static_cast<StoringDiagnosticConsumer *>(
258           m_compiler_instance->getDiagnostics().getClient());
259 
260   diagnostic_consumer->ClearDiagnostics();
261 
262   clang::Module *top_level_module = DoGetModule(clang_path.front(), false);
263 
264   if (!top_level_module) {
265     diagnostic_consumer->DumpDiagnostics(error_stream);
266     error_stream.Printf("error: Couldn't load top-level module %s\n",
267                         path[0].AsCString());
268     return false;
269   }
270 
271   clang::Module *submodule = top_level_module;
272 
273   for (size_t ci = 1; ci < path.size(); ++ci) {
274     llvm::StringRef component = path[ci].GetStringRef();
275     submodule = submodule->findSubmodule(component.str());
276     if (!submodule) {
277       diagnostic_consumer->DumpDiagnostics(error_stream);
278       error_stream.Printf("error: Couldn't load submodule %s\n",
279                           component.str().c_str());
280       return false;
281     }
282   }
283 
284   clang::Module *requested_module = DoGetModule(clang_path, true);
285 
286   if (requested_module != nullptr) {
287     if (exported_modules) {
288       ReportModuleExports(*exported_modules, requested_module);
289     }
290 
291     m_imported_modules[imported_module] = requested_module;
292 
293     m_enabled = true;
294 
295     return true;
296   }
297 
298   return false;
299 }
300 
301 bool ClangModulesDeclVendor::LanguageSupportsClangModules(
302     lldb::LanguageType language) {
303   switch (language) {
304   default:
305     return false;
306   // C++ and friends to be added
307   case lldb::LanguageType::eLanguageTypeC:
308   case lldb::LanguageType::eLanguageTypeC11:
309   case lldb::LanguageType::eLanguageTypeC89:
310   case lldb::LanguageType::eLanguageTypeC99:
311   case lldb::LanguageType::eLanguageTypeObjC:
312     return true;
313   }
314 }
315 
316 bool ClangModulesDeclVendorImpl::AddModulesForCompileUnit(
317     CompileUnit &cu, ClangModulesDeclVendor::ModuleVector &exported_modules,
318     Stream &error_stream) {
319   if (LanguageSupportsClangModules(cu.GetLanguage())) {
320     std::vector<ConstString> imported_modules = cu.GetImportedModules();
321 
322     for (ConstString imported_module : imported_modules) {
323       std::vector<ConstString> path;
324 
325       path.push_back(imported_module);
326 
327       if (!AddModule(path, &exported_modules, error_stream)) {
328         return false;
329       }
330     }
331 
332     return true;
333   }
334 
335   return true;
336 }
337 
338 // ClangImporter::lookupValue
339 
340 uint32_t
341 ClangModulesDeclVendorImpl::FindDecls(const ConstString &name, bool append,
342                                       uint32_t max_matches,
343                                       std::vector<clang::NamedDecl *> &decls) {
344   if (!m_enabled) {
345     return 0;
346   }
347 
348   if (!append)
349     decls.clear();
350 
351   clang::IdentifierInfo &ident =
352       m_compiler_instance->getASTContext().Idents.get(name.GetStringRef());
353 
354   clang::LookupResult lookup_result(
355       m_compiler_instance->getSema(), clang::DeclarationName(&ident),
356       clang::SourceLocation(), clang::Sema::LookupOrdinaryName);
357 
358   m_compiler_instance->getSema().LookupName(
359       lookup_result,
360       m_compiler_instance->getSema().getScopeForContext(
361           m_compiler_instance->getASTContext().getTranslationUnitDecl()));
362 
363   uint32_t num_matches = 0;
364 
365   for (clang::NamedDecl *named_decl : lookup_result) {
366     if (num_matches >= max_matches)
367       return num_matches;
368 
369     decls.push_back(named_decl);
370     ++num_matches;
371   }
372 
373   return num_matches;
374 }
375 
376 void ClangModulesDeclVendorImpl::ForEachMacro(
377     const ClangModulesDeclVendor::ModuleVector &modules,
378     std::function<bool(const std::string &)> handler) {
379   if (!m_enabled) {
380     return;
381   }
382 
383   typedef std::map<ModuleID, ssize_t> ModulePriorityMap;
384   ModulePriorityMap module_priorities;
385 
386   ssize_t priority = 0;
387 
388   for (ModuleID module : modules) {
389     module_priorities[module] = priority++;
390   }
391 
392   if (m_compiler_instance->getPreprocessor().getExternalSource()) {
393     m_compiler_instance->getPreprocessor()
394         .getExternalSource()
395         ->ReadDefinedMacros();
396   }
397 
398   for (clang::Preprocessor::macro_iterator
399            mi = m_compiler_instance->getPreprocessor().macro_begin(),
400            me = m_compiler_instance->getPreprocessor().macro_end();
401        mi != me; ++mi) {
402     const clang::IdentifierInfo *ii = nullptr;
403 
404     {
405       if (clang::IdentifierInfoLookup *lookup =
406               m_compiler_instance->getPreprocessor()
407                   .getIdentifierTable()
408                   .getExternalIdentifierLookup()) {
409         lookup->get(mi->first->getName());
410       }
411       if (!ii) {
412         ii = mi->first;
413       }
414     }
415 
416     ssize_t found_priority = -1;
417     clang::MacroInfo *macro_info = nullptr;
418 
419     for (clang::ModuleMacro *module_macro :
420          m_compiler_instance->getPreprocessor().getLeafModuleMacros(ii)) {
421       clang::Module *module = module_macro->getOwningModule();
422 
423       {
424         ModulePriorityMap::iterator pi =
425             module_priorities.find(reinterpret_cast<ModuleID>(module));
426 
427         if (pi != module_priorities.end() && pi->second > found_priority) {
428           macro_info = module_macro->getMacroInfo();
429           found_priority = pi->second;
430         }
431       }
432 
433       clang::Module *top_level_module = module->getTopLevelModule();
434 
435       if (top_level_module != module) {
436         ModulePriorityMap::iterator pi = module_priorities.find(
437             reinterpret_cast<ModuleID>(top_level_module));
438 
439         if ((pi != module_priorities.end()) && pi->second > found_priority) {
440           macro_info = module_macro->getMacroInfo();
441           found_priority = pi->second;
442         }
443       }
444     }
445 
446     if (macro_info) {
447       std::string macro_expansion = "#define ";
448       macro_expansion.append(mi->first->getName().str());
449 
450       {
451         if (macro_info->isFunctionLike()) {
452           macro_expansion.append("(");
453 
454           bool first_arg = true;
455 
456           for (auto pi = macro_info->param_begin(),
457                     pe = macro_info->param_end();
458                pi != pe; ++pi) {
459             if (!first_arg) {
460               macro_expansion.append(", ");
461             } else {
462               first_arg = false;
463             }
464 
465             macro_expansion.append((*pi)->getName().str());
466           }
467 
468           if (macro_info->isC99Varargs()) {
469             if (first_arg) {
470               macro_expansion.append("...");
471             } else {
472               macro_expansion.append(", ...");
473             }
474           } else if (macro_info->isGNUVarargs()) {
475             macro_expansion.append("...");
476           }
477 
478           macro_expansion.append(")");
479         }
480 
481         macro_expansion.append(" ");
482 
483         bool first_token = true;
484 
485         for (clang::MacroInfo::tokens_iterator ti = macro_info->tokens_begin(),
486                                                te = macro_info->tokens_end();
487              ti != te; ++ti) {
488           if (!first_token) {
489             macro_expansion.append(" ");
490           } else {
491             first_token = false;
492           }
493 
494           if (ti->isLiteral()) {
495             if (const char *literal_data = ti->getLiteralData()) {
496               std::string token_str(literal_data, ti->getLength());
497               macro_expansion.append(token_str);
498             } else {
499               bool invalid = false;
500               const char *literal_source =
501                   m_compiler_instance->getSourceManager().getCharacterData(
502                       ti->getLocation(), &invalid);
503 
504               if (invalid) {
505                 lldbassert(0 && "Unhandled token kind");
506                 macro_expansion.append("<unknown literal value>");
507               } else {
508                 macro_expansion.append(
509                     std::string(literal_source, ti->getLength()));
510               }
511             }
512           } else if (const char *punctuator_spelling =
513                          clang::tok::getPunctuatorSpelling(ti->getKind())) {
514             macro_expansion.append(punctuator_spelling);
515           } else if (const char *keyword_spelling =
516                          clang::tok::getKeywordSpelling(ti->getKind())) {
517             macro_expansion.append(keyword_spelling);
518           } else {
519             switch (ti->getKind()) {
520             case clang::tok::TokenKind::identifier:
521               macro_expansion.append(ti->getIdentifierInfo()->getName().str());
522               break;
523             case clang::tok::TokenKind::raw_identifier:
524               macro_expansion.append(ti->getRawIdentifier().str());
525               break;
526             default:
527               macro_expansion.append(ti->getName());
528               break;
529             }
530           }
531         }
532 
533         if (handler(macro_expansion)) {
534           return;
535         }
536       }
537     }
538   }
539 }
540 
541 clang::ModuleLoadResult
542 ClangModulesDeclVendorImpl::DoGetModule(clang::ModuleIdPath path,
543                                         bool make_visible) {
544   clang::Module::NameVisibilityKind visibility =
545       make_visible ? clang::Module::AllVisible : clang::Module::Hidden;
546 
547   const bool is_inclusion_directive = false;
548 
549   return m_compiler_instance->loadModule(path.front().second, path, visibility,
550                                          is_inclusion_directive);
551 }
552 
553 clang::ExternalASTMerger::ImporterSource
554 ClangModulesDeclVendorImpl::GetImporterSource() {
555   return {m_compiler_instance->getASTContext(),
556           m_compiler_instance->getFileManager(), m_origin_map};
557 }
558 
559 static const char *ModuleImportBufferName = "LLDBModulesMemoryBuffer";
560 
561 lldb_private::ClangModulesDeclVendor *
562 ClangModulesDeclVendor::Create(Target &target) {
563   // FIXME we should insure programmatically that the expression parser's
564   // compiler and the modules runtime's
565   // compiler are both initialized in the same way – preferably by the same
566   // code.
567 
568   if (!target.GetPlatform()->SupportsModules())
569     return nullptr;
570 
571   const ArchSpec &arch = target.GetArchitecture();
572 
573   std::vector<std::string> compiler_invocation_arguments = {
574       "clang",
575       "-fmodules",
576       "-fimplicit-module-maps",
577       "-fcxx-modules",
578       "-fsyntax-only",
579       "-femit-all-decls",
580       "-target",
581       arch.GetTriple().str(),
582       "-fmodules-validate-system-headers",
583       "-Werror=non-modular-include-in-framework-module"};
584 
585   target.GetPlatform()->AddClangModuleCompilationOptions(
586       &target, compiler_invocation_arguments);
587 
588   compiler_invocation_arguments.push_back(ModuleImportBufferName);
589 
590   // Add additional search paths with { "-I", path } or { "-F", path } here.
591 
592   {
593     llvm::SmallString<128> path;
594     auto props = ModuleList::GetGlobalModuleListProperties();
595     props.GetClangModulesCachePath().GetPath(path);
596     std::string module_cache_argument("-fmodules-cache-path=");
597     module_cache_argument.append(path.str());
598     compiler_invocation_arguments.push_back(module_cache_argument);
599   }
600 
601   FileSpecList &module_search_paths = target.GetClangModuleSearchPaths();
602 
603   for (size_t spi = 0, spe = module_search_paths.GetSize(); spi < spe; ++spi) {
604     const FileSpec &search_path = module_search_paths.GetFileSpecAtIndex(spi);
605 
606     std::string search_path_argument = "-I";
607     search_path_argument.append(search_path.GetPath());
608 
609     compiler_invocation_arguments.push_back(search_path_argument);
610   }
611 
612   {
613     FileSpec clang_resource_dir = GetResourceDir();
614 
615     if (llvm::sys::fs::is_directory(clang_resource_dir.GetPath())) {
616       compiler_invocation_arguments.push_back("-resource-dir");
617       compiler_invocation_arguments.push_back(clang_resource_dir.GetPath());
618     }
619   }
620 
621   llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_engine =
622       clang::CompilerInstance::createDiagnostics(new clang::DiagnosticOptions,
623                                                  new StoringDiagnosticConsumer);
624 
625   std::vector<const char *> compiler_invocation_argument_cstrs;
626 
627   for (const std::string &arg : compiler_invocation_arguments) {
628     compiler_invocation_argument_cstrs.push_back(arg.c_str());
629   }
630 
631   std::shared_ptr<clang::CompilerInvocation> invocation =
632       clang::createInvocationFromCommandLine(compiler_invocation_argument_cstrs,
633                                              diagnostics_engine);
634 
635   if (!invocation)
636     return nullptr;
637 
638   std::unique_ptr<llvm::MemoryBuffer> source_buffer =
639       llvm::MemoryBuffer::getMemBuffer(
640           "extern int __lldb __attribute__((unavailable));",
641           ModuleImportBufferName);
642 
643   invocation->getPreprocessorOpts().addRemappedFile(ModuleImportBufferName,
644                                                     source_buffer.release());
645 
646   std::unique_ptr<clang::CompilerInstance> instance(
647       new clang::CompilerInstance);
648 
649   instance->setDiagnostics(diagnostics_engine.get());
650   instance->setInvocation(invocation);
651 
652   std::unique_ptr<clang::FrontendAction> action(new clang::SyntaxOnlyAction);
653 
654   instance->setTarget(clang::TargetInfo::CreateTargetInfo(
655       *diagnostics_engine, instance->getInvocation().TargetOpts));
656 
657   if (!instance->hasTarget())
658     return nullptr;
659 
660   instance->getTarget().adjust(instance->getLangOpts());
661 
662   if (!action->BeginSourceFile(*instance,
663                                instance->getFrontendOpts().Inputs[0]))
664     return nullptr;
665 
666   instance->getPreprocessor().enableIncrementalProcessing();
667 
668   instance->createModuleManager();
669 
670   instance->createSema(action->getTranslationUnitKind(), nullptr);
671 
672   const bool skipFunctionBodies = false;
673   std::unique_ptr<clang::Parser> parser(new clang::Parser(
674       instance->getPreprocessor(), instance->getSema(), skipFunctionBodies));
675 
676   instance->getPreprocessor().EnterMainSourceFile();
677   parser->Initialize();
678 
679   clang::Parser::DeclGroupPtrTy parsed;
680 
681   while (!parser->ParseTopLevelDecl(parsed))
682     ;
683 
684   return new ClangModulesDeclVendorImpl(std::move(diagnostics_engine),
685                                         std::move(invocation),
686                                         std::move(instance), std::move(parser));
687 }
688