1 //===-- ClangModulesDeclVendor.cpp ------------------------------*- C++ -*-===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 // C Includes
11 // C++ Includes
12 #include <mutex>
13 
14 // Other libraries and framework includes
15 #include "clang/Basic/TargetInfo.h"
16 #include "clang/Frontend/CompilerInstance.h"
17 #include "clang/Frontend/FrontendActions.h"
18 #include "clang/Lex/Preprocessor.h"
19 #include "clang/Lex/PreprocessorOptions.h"
20 #include "clang/Parse/Parser.h"
21 #include "clang/Sema/Lookup.h"
22 #include "clang/Serialization/ASTReader.h"
23 #include "llvm/Support/Path.h"
24 
25 // Project includes
26 #include "ClangModulesDeclVendor.h"
27 
28 #include "lldb/Core/Log.h"
29 #include "lldb/Core/StreamString.h"
30 #include "lldb/Host/FileSpec.h"
31 #include "lldb/Host/Host.h"
32 #include "lldb/Host/HostInfo.h"
33 #include "lldb/Symbol/CompileUnit.h"
34 #include "lldb/Target/Target.h"
35 #include "lldb/Utility/LLDBAssert.h"
36 
37 using namespace lldb_private;
38 
39 namespace {
40 // Any Clang compiler requires a consumer for diagnostics.  This one stores them
41 // as strings
42 // so we can provide them to the user in case a module failed to load.
43 class StoringDiagnosticConsumer : public clang::DiagnosticConsumer {
44 public:
45   StoringDiagnosticConsumer();
46 
47   void HandleDiagnostic(clang::DiagnosticsEngine::Level DiagLevel,
48                         const clang::Diagnostic &info) override;
49 
50   void ClearDiagnostics();
51 
52   void DumpDiagnostics(Stream &error_stream);
53 
54 private:
55   typedef std::pair<clang::DiagnosticsEngine::Level, std::string>
56       IDAndDiagnostic;
57   std::vector<IDAndDiagnostic> m_diagnostics;
58   Log *m_log;
59 };
60 
61 // The private implementation of our ClangModulesDeclVendor.  Contains all the
62 // Clang state required
63 // to load modules.
64 class ClangModulesDeclVendorImpl : public ClangModulesDeclVendor {
65 public:
66   ClangModulesDeclVendorImpl(
67       llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> &diagnostics_engine,
68       llvm::IntrusiveRefCntPtr<clang::CompilerInvocation> &compiler_invocation,
69       std::unique_ptr<clang::CompilerInstance> &&compiler_instance,
70       std::unique_ptr<clang::Parser> &&parser);
71 
72   ~ClangModulesDeclVendorImpl() override = default;
73 
74   bool AddModule(ModulePath &path, ModuleVector *exported_modules,
75                  Stream &error_stream) override;
76 
77   bool AddModulesForCompileUnit(CompileUnit &cu, ModuleVector &exported_modules,
78                                 Stream &error_stream) override;
79 
80   uint32_t FindDecls(const ConstString &name, bool append, uint32_t max_matches,
81                      std::vector<clang::NamedDecl *> &decls) override;
82 
83   void ForEachMacro(const ModuleVector &modules,
84                     std::function<bool(const std::string &)> handler) override;
85 
86 private:
87   void
88   ReportModuleExportsHelper(std::set<ClangModulesDeclVendor::ModuleID> &exports,
89                             clang::Module *module);
90 
91   void ReportModuleExports(ModuleVector &exports, clang::Module *module);
92 
93   clang::ModuleLoadResult DoGetModule(clang::ModuleIdPath path,
94                                       bool make_visible);
95 
96   bool m_enabled = false;
97 
98   llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> m_diagnostics_engine;
99   llvm::IntrusiveRefCntPtr<clang::CompilerInvocation> m_compiler_invocation;
100   std::unique_ptr<clang::CompilerInstance> m_compiler_instance;
101   std::unique_ptr<clang::Parser> m_parser;
102   size_t m_source_location_index =
103       0; // used to give name components fake SourceLocations
104 
105   typedef std::vector<ConstString> ImportedModule;
106   typedef std::map<ImportedModule, clang::Module *> ImportedModuleMap;
107   typedef std::set<ModuleID> ImportedModuleSet;
108   ImportedModuleMap m_imported_modules;
109   ImportedModuleSet m_user_imported_modules;
110 };
111 } // anonymous namespace
112 
113 StoringDiagnosticConsumer::StoringDiagnosticConsumer() {
114   m_log = lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_EXPRESSIONS);
115 }
116 
117 void StoringDiagnosticConsumer::HandleDiagnostic(
118     clang::DiagnosticsEngine::Level DiagLevel, const clang::Diagnostic &info) {
119   llvm::SmallVector<char, 256> diagnostic_string;
120 
121   info.FormatDiagnostic(diagnostic_string);
122 
123   m_diagnostics.push_back(
124       IDAndDiagnostic(DiagLevel, std::string(diagnostic_string.data(),
125                                              diagnostic_string.size())));
126 }
127 
128 void StoringDiagnosticConsumer::ClearDiagnostics() { m_diagnostics.clear(); }
129 
130 void StoringDiagnosticConsumer::DumpDiagnostics(Stream &error_stream) {
131   for (IDAndDiagnostic &diag : m_diagnostics) {
132     switch (diag.first) {
133     default:
134       error_stream.PutCString(diag.second.c_str());
135       error_stream.PutChar('\n');
136       break;
137     case clang::DiagnosticsEngine::Level::Ignored:
138       break;
139     }
140   }
141 }
142 
143 static FileSpec GetResourceDir() {
144   static FileSpec g_cached_resource_dir;
145 
146   static std::once_flag g_once_flag;
147 
148   std::call_once(g_once_flag, []() {
149     HostInfo::GetLLDBPath(lldb::ePathTypeClangDir, g_cached_resource_dir);
150   });
151 
152   return g_cached_resource_dir;
153 }
154 
155 ClangModulesDeclVendor::ClangModulesDeclVendor() {}
156 
157 ClangModulesDeclVendor::~ClangModulesDeclVendor() {}
158 
159 ClangModulesDeclVendorImpl::ClangModulesDeclVendorImpl(
160     llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> &diagnostics_engine,
161     llvm::IntrusiveRefCntPtr<clang::CompilerInvocation> &compiler_invocation,
162     std::unique_ptr<clang::CompilerInstance> &&compiler_instance,
163     std::unique_ptr<clang::Parser> &&parser)
164     : ClangModulesDeclVendor(), m_diagnostics_engine(diagnostics_engine),
165       m_compiler_invocation(compiler_invocation),
166       m_compiler_instance(std::move(compiler_instance)),
167       m_parser(std::move(parser)), m_imported_modules() {}
168 
169 void ClangModulesDeclVendorImpl::ReportModuleExportsHelper(
170     std::set<ClangModulesDeclVendor::ModuleID> &exports,
171     clang::Module *module) {
172   if (exports.count(reinterpret_cast<ClangModulesDeclVendor::ModuleID>(module)))
173     return;
174 
175   exports.insert(reinterpret_cast<ClangModulesDeclVendor::ModuleID>(module));
176 
177   llvm::SmallVector<clang::Module *, 2> sub_exports;
178 
179   module->getExportedModules(sub_exports);
180 
181   for (clang::Module *module : sub_exports) {
182     ReportModuleExportsHelper(exports, module);
183   }
184 }
185 
186 void ClangModulesDeclVendorImpl::ReportModuleExports(
187     ClangModulesDeclVendor::ModuleVector &exports, clang::Module *module) {
188   std::set<ClangModulesDeclVendor::ModuleID> exports_set;
189 
190   ReportModuleExportsHelper(exports_set, module);
191 
192   for (ModuleID module : exports_set) {
193     exports.push_back(module);
194   }
195 }
196 
197 bool ClangModulesDeclVendorImpl::AddModule(ModulePath &path,
198                                            ModuleVector *exported_modules,
199                                            Stream &error_stream) {
200   // Fail early.
201 
202   if (m_compiler_instance->hadModuleLoaderFatalFailure()) {
203     error_stream.PutCString("error: Couldn't load a module because the module "
204                             "loader is in a fatal state.\n");
205     return false;
206   }
207 
208   // Check if we've already imported this module.
209 
210   std::vector<ConstString> imported_module;
211 
212   for (ConstString path_component : path) {
213     imported_module.push_back(path_component);
214   }
215 
216   {
217     ImportedModuleMap::iterator mi = m_imported_modules.find(imported_module);
218 
219     if (mi != m_imported_modules.end()) {
220       if (exported_modules) {
221         ReportModuleExports(*exported_modules, mi->second);
222       }
223       return true;
224     }
225   }
226 
227   if (!m_compiler_instance->getPreprocessor()
228            .getHeaderSearchInfo()
229            .lookupModule(path[0].GetStringRef())) {
230     error_stream.Printf("error: Header search couldn't locate module %s\n",
231                         path[0].AsCString());
232     return false;
233   }
234 
235   llvm::SmallVector<std::pair<clang::IdentifierInfo *, clang::SourceLocation>,
236                     4>
237       clang_path;
238 
239   {
240     clang::SourceManager &source_manager =
241         m_compiler_instance->getASTContext().getSourceManager();
242 
243     for (ConstString path_component : path) {
244       clang_path.push_back(std::make_pair(
245           &m_compiler_instance->getASTContext().Idents.get(
246               path_component.GetStringRef()),
247           source_manager.getLocForStartOfFile(source_manager.getMainFileID())
248               .getLocWithOffset(m_source_location_index++)));
249     }
250   }
251 
252   StoringDiagnosticConsumer *diagnostic_consumer =
253       static_cast<StoringDiagnosticConsumer *>(
254           m_compiler_instance->getDiagnostics().getClient());
255 
256   diagnostic_consumer->ClearDiagnostics();
257 
258   clang::Module *top_level_module = DoGetModule(clang_path.front(), false);
259 
260   if (!top_level_module) {
261     diagnostic_consumer->DumpDiagnostics(error_stream);
262     error_stream.Printf("error: Couldn't load top-level module %s\n",
263                         path[0].AsCString());
264     return false;
265   }
266 
267   clang::Module *submodule = top_level_module;
268 
269   for (size_t ci = 1; ci < path.size(); ++ci) {
270     llvm::StringRef component = path[ci].GetStringRef();
271     submodule = submodule->findSubmodule(component.str());
272     if (!submodule) {
273       diagnostic_consumer->DumpDiagnostics(error_stream);
274       error_stream.Printf("error: Couldn't load submodule %s\n",
275                           component.str().c_str());
276       return false;
277     }
278   }
279 
280   clang::Module *requested_module = DoGetModule(clang_path, true);
281 
282   if (requested_module != nullptr) {
283     if (exported_modules) {
284       ReportModuleExports(*exported_modules, requested_module);
285     }
286 
287     m_imported_modules[imported_module] = requested_module;
288 
289     m_enabled = true;
290 
291     return true;
292   }
293 
294   return false;
295 }
296 
297 bool ClangModulesDeclVendor::LanguageSupportsClangModules(
298     lldb::LanguageType language) {
299   switch (language) {
300   default:
301     return false;
302   // C++ and friends to be added
303   case lldb::LanguageType::eLanguageTypeC:
304   case lldb::LanguageType::eLanguageTypeC11:
305   case lldb::LanguageType::eLanguageTypeC89:
306   case lldb::LanguageType::eLanguageTypeC99:
307   case lldb::LanguageType::eLanguageTypeObjC:
308     return true;
309   }
310 }
311 
312 bool ClangModulesDeclVendorImpl::AddModulesForCompileUnit(
313     CompileUnit &cu, ClangModulesDeclVendor::ModuleVector &exported_modules,
314     Stream &error_stream) {
315   if (LanguageSupportsClangModules(cu.GetLanguage())) {
316     std::vector<ConstString> imported_modules = cu.GetImportedModules();
317 
318     for (ConstString imported_module : imported_modules) {
319       std::vector<ConstString> path;
320 
321       path.push_back(imported_module);
322 
323       if (!AddModule(path, &exported_modules, error_stream)) {
324         return false;
325       }
326     }
327 
328     return true;
329   }
330 
331   return true;
332 }
333 
334 // ClangImporter::lookupValue
335 
336 uint32_t
337 ClangModulesDeclVendorImpl::FindDecls(const ConstString &name, bool append,
338                                       uint32_t max_matches,
339                                       std::vector<clang::NamedDecl *> &decls) {
340   if (!m_enabled) {
341     return 0;
342   }
343 
344   if (!append)
345     decls.clear();
346 
347   clang::IdentifierInfo &ident =
348       m_compiler_instance->getASTContext().Idents.get(name.GetStringRef());
349 
350   clang::LookupResult lookup_result(
351       m_compiler_instance->getSema(), clang::DeclarationName(&ident),
352       clang::SourceLocation(), clang::Sema::LookupOrdinaryName);
353 
354   m_compiler_instance->getSema().LookupName(
355       lookup_result,
356       m_compiler_instance->getSema().getScopeForContext(
357           m_compiler_instance->getASTContext().getTranslationUnitDecl()));
358 
359   uint32_t num_matches = 0;
360 
361   for (clang::NamedDecl *named_decl : lookup_result) {
362     if (num_matches >= max_matches)
363       return num_matches;
364 
365     decls.push_back(named_decl);
366     ++num_matches;
367   }
368 
369   return num_matches;
370 }
371 
372 void ClangModulesDeclVendorImpl::ForEachMacro(
373     const ClangModulesDeclVendor::ModuleVector &modules,
374     std::function<bool(const std::string &)> handler) {
375   if (!m_enabled) {
376     return;
377   }
378 
379   typedef std::map<ModuleID, ssize_t> ModulePriorityMap;
380   ModulePriorityMap module_priorities;
381 
382   ssize_t priority = 0;
383 
384   for (ModuleID module : modules) {
385     module_priorities[module] = priority++;
386   }
387 
388   if (m_compiler_instance->getPreprocessor().getExternalSource()) {
389     m_compiler_instance->getPreprocessor()
390         .getExternalSource()
391         ->ReadDefinedMacros();
392   }
393 
394   for (clang::Preprocessor::macro_iterator
395            mi = m_compiler_instance->getPreprocessor().macro_begin(),
396            me = m_compiler_instance->getPreprocessor().macro_end();
397        mi != me; ++mi) {
398     const clang::IdentifierInfo *ii = nullptr;
399 
400     {
401       if (clang::IdentifierInfoLookup *lookup =
402               m_compiler_instance->getPreprocessor()
403                   .getIdentifierTable()
404                   .getExternalIdentifierLookup()) {
405         lookup->get(mi->first->getName());
406       }
407       if (!ii) {
408         ii = mi->first;
409       }
410     }
411 
412     ssize_t found_priority = -1;
413     clang::MacroInfo *macro_info = nullptr;
414 
415     for (clang::ModuleMacro *module_macro :
416          m_compiler_instance->getPreprocessor().getLeafModuleMacros(ii)) {
417       clang::Module *module = module_macro->getOwningModule();
418 
419       {
420         ModulePriorityMap::iterator pi =
421             module_priorities.find(reinterpret_cast<ModuleID>(module));
422 
423         if (pi != module_priorities.end() && pi->second > found_priority) {
424           macro_info = module_macro->getMacroInfo();
425           found_priority = pi->second;
426         }
427       }
428 
429       clang::Module *top_level_module = module->getTopLevelModule();
430 
431       if (top_level_module != module) {
432         ModulePriorityMap::iterator pi = module_priorities.find(
433             reinterpret_cast<ModuleID>(top_level_module));
434 
435         if ((pi != module_priorities.end()) && pi->second > found_priority) {
436           macro_info = module_macro->getMacroInfo();
437           found_priority = pi->second;
438         }
439       }
440     }
441 
442     if (macro_info) {
443       std::string macro_expansion = "#define ";
444       macro_expansion.append(mi->first->getName().str().c_str());
445 
446       {
447         if (macro_info->isFunctionLike()) {
448           macro_expansion.append("(");
449 
450           bool first_arg = true;
451 
452           for (clang::MacroInfo::arg_iterator ai = macro_info->arg_begin(),
453                                               ae = macro_info->arg_end();
454                ai != ae; ++ai) {
455             if (!first_arg) {
456               macro_expansion.append(", ");
457             } else {
458               first_arg = false;
459             }
460 
461             macro_expansion.append((*ai)->getName().str());
462           }
463 
464           if (macro_info->isC99Varargs()) {
465             if (first_arg) {
466               macro_expansion.append("...");
467             } else {
468               macro_expansion.append(", ...");
469             }
470           } else if (macro_info->isGNUVarargs()) {
471             macro_expansion.append("...");
472           }
473 
474           macro_expansion.append(")");
475         }
476 
477         macro_expansion.append(" ");
478 
479         bool first_token = true;
480 
481         for (clang::MacroInfo::tokens_iterator ti = macro_info->tokens_begin(),
482                                                te = macro_info->tokens_end();
483              ti != te; ++ti) {
484           if (!first_token) {
485             macro_expansion.append(" ");
486           } else {
487             first_token = false;
488           }
489 
490           if (ti->isLiteral()) {
491             if (const char *literal_data = ti->getLiteralData()) {
492               std::string token_str(literal_data, ti->getLength());
493               macro_expansion.append(token_str);
494             } else {
495               bool invalid = false;
496               const char *literal_source =
497                   m_compiler_instance->getSourceManager().getCharacterData(
498                       ti->getLocation(), &invalid);
499 
500               if (invalid) {
501                 lldbassert(!"Unhandled token kind");
502                 macro_expansion.append("<unknown literal value>");
503               } else {
504                 macro_expansion.append(
505                     std::string(literal_source, ti->getLength()));
506               }
507             }
508           } else if (const char *punctuator_spelling =
509                          clang::tok::getPunctuatorSpelling(ti->getKind())) {
510             macro_expansion.append(punctuator_spelling);
511           } else if (const char *keyword_spelling =
512                          clang::tok::getKeywordSpelling(ti->getKind())) {
513             macro_expansion.append(keyword_spelling);
514           } else {
515             switch (ti->getKind()) {
516             case clang::tok::TokenKind::identifier:
517               macro_expansion.append(ti->getIdentifierInfo()->getName().str());
518               break;
519             case clang::tok::TokenKind::raw_identifier:
520               macro_expansion.append(ti->getRawIdentifier().str());
521               break;
522             default:
523               macro_expansion.append(ti->getName());
524               break;
525             }
526           }
527         }
528 
529         if (handler(macro_expansion)) {
530           return;
531         }
532       }
533     }
534   }
535 }
536 
537 clang::ModuleLoadResult
538 ClangModulesDeclVendorImpl::DoGetModule(clang::ModuleIdPath path,
539                                         bool make_visible) {
540   clang::Module::NameVisibilityKind visibility =
541       make_visible ? clang::Module::AllVisible : clang::Module::Hidden;
542 
543   const bool is_inclusion_directive = false;
544 
545   return m_compiler_instance->loadModule(path.front().second, path, visibility,
546                                          is_inclusion_directive);
547 }
548 
549 static const char *ModuleImportBufferName = "LLDBModulesMemoryBuffer";
550 
551 lldb_private::ClangModulesDeclVendor *
552 ClangModulesDeclVendor::Create(Target &target) {
553   // FIXME we should insure programmatically that the expression parser's
554   // compiler and the modules runtime's
555   // compiler are both initialized in the same way – preferably by the same
556   // code.
557 
558   if (!target.GetPlatform()->SupportsModules())
559     return nullptr;
560 
561   const ArchSpec &arch = target.GetArchitecture();
562 
563   std::vector<std::string> compiler_invocation_arguments = {
564       "clang",
565       "-fmodules",
566       "-fimplicit-module-maps",
567       "-fcxx-modules",
568       "-fsyntax-only",
569       "-femit-all-decls",
570       "-target",
571       arch.GetTriple().str(),
572       "-fmodules-validate-system-headers",
573       "-Werror=non-modular-include-in-framework-module"};
574 
575   target.GetPlatform()->AddClangModuleCompilationOptions(
576       &target, compiler_invocation_arguments);
577 
578   compiler_invocation_arguments.push_back(ModuleImportBufferName);
579 
580   // Add additional search paths with { "-I", path } or { "-F", path } here.
581 
582   {
583     llvm::SmallString<128> DefaultModuleCache;
584     const bool erased_on_reboot = false;
585     llvm::sys::path::system_temp_directory(erased_on_reboot,
586                                            DefaultModuleCache);
587     llvm::sys::path::append(DefaultModuleCache, "org.llvm.clang");
588     llvm::sys::path::append(DefaultModuleCache, "ModuleCache");
589     std::string module_cache_argument("-fmodules-cache-path=");
590     module_cache_argument.append(DefaultModuleCache.str().str());
591     compiler_invocation_arguments.push_back(module_cache_argument);
592   }
593 
594   FileSpecList &module_search_paths = target.GetClangModuleSearchPaths();
595 
596   for (size_t spi = 0, spe = module_search_paths.GetSize(); spi < spe; ++spi) {
597     const FileSpec &search_path = module_search_paths.GetFileSpecAtIndex(spi);
598 
599     std::string search_path_argument = "-I";
600     search_path_argument.append(search_path.GetPath());
601 
602     compiler_invocation_arguments.push_back(search_path_argument);
603   }
604 
605   {
606     FileSpec clang_resource_dir = GetResourceDir();
607 
608     if (clang_resource_dir.IsDirectory()) {
609       compiler_invocation_arguments.push_back("-resource-dir");
610       compiler_invocation_arguments.push_back(clang_resource_dir.GetPath());
611     }
612   }
613 
614   llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_engine =
615       clang::CompilerInstance::createDiagnostics(new clang::DiagnosticOptions,
616                                                  new StoringDiagnosticConsumer);
617 
618   std::vector<const char *> compiler_invocation_argument_cstrs;
619 
620   for (const std::string &arg : compiler_invocation_arguments) {
621     compiler_invocation_argument_cstrs.push_back(arg.c_str());
622   }
623 
624   llvm::IntrusiveRefCntPtr<clang::CompilerInvocation> invocation(
625       clang::createInvocationFromCommandLine(compiler_invocation_argument_cstrs,
626                                              diagnostics_engine));
627 
628   if (!invocation)
629     return nullptr;
630 
631   std::unique_ptr<llvm::MemoryBuffer> source_buffer =
632       llvm::MemoryBuffer::getMemBuffer(
633           "extern int __lldb __attribute__((unavailable));",
634           ModuleImportBufferName);
635 
636   invocation->getPreprocessorOpts().addRemappedFile(ModuleImportBufferName,
637                                                     source_buffer.release());
638 
639   std::unique_ptr<clang::CompilerInstance> instance(
640       new clang::CompilerInstance);
641 
642   instance->setDiagnostics(diagnostics_engine.get());
643   instance->setInvocation(invocation.get());
644 
645   std::unique_ptr<clang::FrontendAction> action(new clang::SyntaxOnlyAction);
646 
647   instance->setTarget(clang::TargetInfo::CreateTargetInfo(
648       *diagnostics_engine, instance->getInvocation().TargetOpts));
649 
650   if (!instance->hasTarget())
651     return nullptr;
652 
653   instance->getTarget().adjust(instance->getLangOpts());
654 
655   if (!action->BeginSourceFile(*instance,
656                                instance->getFrontendOpts().Inputs[0]))
657     return nullptr;
658 
659   instance->getPreprocessor().enableIncrementalProcessing();
660 
661   instance->createModuleManager();
662 
663   instance->createSema(action->getTranslationUnitKind(), nullptr);
664 
665   const bool skipFunctionBodies = false;
666   std::unique_ptr<clang::Parser> parser(new clang::Parser(
667       instance->getPreprocessor(), instance->getSema(), skipFunctionBodies));
668 
669   instance->getPreprocessor().EnterMainSourceFile();
670   parser->Initialize();
671 
672   clang::Parser::DeclGroupPtrTy parsed;
673 
674   while (!parser->ParseTopLevelDecl(parsed))
675     ;
676 
677   return new ClangModulesDeclVendorImpl(diagnostics_engine, invocation,
678                                         std::move(instance), std::move(parser));
679 }
680