1 //===-- ClangModulesDeclVendor.cpp ------------------------------*- C++ -*-===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 // C Includes
11 // C++ Includes
12 #include <mutex>
13 
14 // Other libraries and framework includes
15 #include "clang/Basic/TargetInfo.h"
16 #include "clang/Frontend/CompilerInstance.h"
17 #include "clang/Frontend/FrontendActions.h"
18 #include "clang/Lex/Preprocessor.h"
19 #include "clang/Lex/PreprocessorOptions.h"
20 #include "clang/Parse/Parser.h"
21 #include "clang/Sema/Lookup.h"
22 #include "clang/Serialization/ASTReader.h"
23 #include "llvm/Support/FileSystem.h"
24 #include "llvm/Support/Path.h"
25 #include "llvm/Support/Threading.h"
26 
27 // Project includes
28 #include "ClangModulesDeclVendor.h"
29 
30 #include "lldb/Core/ModuleList.h"
31 #include "lldb/Host/Host.h"
32 #include "lldb/Host/HostInfo.h"
33 #include "lldb/Symbol/CompileUnit.h"
34 #include "lldb/Target/Target.h"
35 #include "lldb/Utility/FileSpec.h"
36 #include "lldb/Utility/LLDBAssert.h"
37 #include "lldb/Utility/Log.h"
38 #include "lldb/Utility/StreamString.h"
39 
40 using namespace lldb_private;
41 
42 namespace {
43 // Any Clang compiler requires a consumer for diagnostics.  This one stores them
44 // as strings
45 // so we can provide them to the user in case a module failed to load.
46 class StoringDiagnosticConsumer : public clang::DiagnosticConsumer {
47 public:
48   StoringDiagnosticConsumer();
49 
50   void HandleDiagnostic(clang::DiagnosticsEngine::Level DiagLevel,
51                         const clang::Diagnostic &info) override;
52 
53   void ClearDiagnostics();
54 
55   void DumpDiagnostics(Stream &error_stream);
56 
57 private:
58   typedef std::pair<clang::DiagnosticsEngine::Level, std::string>
59       IDAndDiagnostic;
60   std::vector<IDAndDiagnostic> m_diagnostics;
61   Log *m_log;
62 };
63 
64 // The private implementation of our ClangModulesDeclVendor.  Contains all the
65 // Clang state required
66 // to load modules.
67 class ClangModulesDeclVendorImpl : public ClangModulesDeclVendor {
68 public:
69   ClangModulesDeclVendorImpl(
70       llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_engine,
71       std::shared_ptr<clang::CompilerInvocation> compiler_invocation,
72       std::unique_ptr<clang::CompilerInstance> compiler_instance,
73       std::unique_ptr<clang::Parser> parser);
74 
75   ~ClangModulesDeclVendorImpl() override = default;
76 
77   bool AddModule(ModulePath &path, ModuleVector *exported_modules,
78                  Stream &error_stream) override;
79 
80   bool AddModulesForCompileUnit(CompileUnit &cu, ModuleVector &exported_modules,
81                                 Stream &error_stream) override;
82 
83   uint32_t FindDecls(const ConstString &name, bool append, uint32_t max_matches,
84                      std::vector<clang::NamedDecl *> &decls) override;
85 
86   void ForEachMacro(const ModuleVector &modules,
87                     std::function<bool(const std::string &)> handler) override;
88 
89   clang::ExternalASTMerger::ImporterSource GetImporterSource() override;
90 private:
91   void
92   ReportModuleExportsHelper(std::set<ClangModulesDeclVendor::ModuleID> &exports,
93                             clang::Module *module);
94 
95   void ReportModuleExports(ModuleVector &exports, clang::Module *module);
96 
97   clang::ModuleLoadResult DoGetModule(clang::ModuleIdPath path,
98                                       bool make_visible);
99 
100   bool m_enabled = false;
101 
102   llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> m_diagnostics_engine;
103   std::shared_ptr<clang::CompilerInvocation> m_compiler_invocation;
104   std::unique_ptr<clang::CompilerInstance> m_compiler_instance;
105   std::unique_ptr<clang::Parser> m_parser;
106   size_t m_source_location_index =
107       0; // used to give name components fake SourceLocations
108 
109   typedef std::vector<ConstString> ImportedModule;
110   typedef std::map<ImportedModule, clang::Module *> ImportedModuleMap;
111   typedef std::set<ModuleID> ImportedModuleSet;
112   ImportedModuleMap m_imported_modules;
113   ImportedModuleSet m_user_imported_modules;
114   const clang::ExternalASTMerger::OriginMap m_origin_map;
115 };
116 } // anonymous namespace
117 
118 StoringDiagnosticConsumer::StoringDiagnosticConsumer() {
119   m_log = lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_EXPRESSIONS);
120 }
121 
122 void StoringDiagnosticConsumer::HandleDiagnostic(
123     clang::DiagnosticsEngine::Level DiagLevel, const clang::Diagnostic &info) {
124   llvm::SmallVector<char, 256> diagnostic_string;
125 
126   info.FormatDiagnostic(diagnostic_string);
127 
128   m_diagnostics.push_back(
129       IDAndDiagnostic(DiagLevel, std::string(diagnostic_string.data(),
130                                              diagnostic_string.size())));
131 }
132 
133 void StoringDiagnosticConsumer::ClearDiagnostics() { m_diagnostics.clear(); }
134 
135 void StoringDiagnosticConsumer::DumpDiagnostics(Stream &error_stream) {
136   for (IDAndDiagnostic &diag : m_diagnostics) {
137     switch (diag.first) {
138     default:
139       error_stream.PutCString(diag.second);
140       error_stream.PutChar('\n');
141       break;
142     case clang::DiagnosticsEngine::Level::Ignored:
143       break;
144     }
145   }
146 }
147 
148 static FileSpec GetResourceDir() {
149   static FileSpec g_cached_resource_dir;
150 
151   static llvm::once_flag g_once_flag;
152 
153   llvm::call_once(g_once_flag, []() {
154     HostInfo::GetLLDBPath(lldb::ePathTypeClangDir, g_cached_resource_dir);
155   });
156 
157   return g_cached_resource_dir;
158 }
159 
160 ClangModulesDeclVendor::ClangModulesDeclVendor() {}
161 
162 ClangModulesDeclVendor::~ClangModulesDeclVendor() {}
163 
164 ClangModulesDeclVendorImpl::ClangModulesDeclVendorImpl(
165     llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_engine,
166     std::shared_ptr<clang::CompilerInvocation> compiler_invocation,
167     std::unique_ptr<clang::CompilerInstance> compiler_instance,
168     std::unique_ptr<clang::Parser> parser)
169     : m_diagnostics_engine(std::move(diagnostics_engine)),
170       m_compiler_invocation(std::move(compiler_invocation)),
171       m_compiler_instance(std::move(compiler_instance)),
172       m_parser(std::move(parser)), m_origin_map() {}
173 
174 void ClangModulesDeclVendorImpl::ReportModuleExportsHelper(
175     std::set<ClangModulesDeclVendor::ModuleID> &exports,
176     clang::Module *module) {
177   if (exports.count(reinterpret_cast<ClangModulesDeclVendor::ModuleID>(module)))
178     return;
179 
180   exports.insert(reinterpret_cast<ClangModulesDeclVendor::ModuleID>(module));
181 
182   llvm::SmallVector<clang::Module *, 2> sub_exports;
183 
184   module->getExportedModules(sub_exports);
185 
186   for (clang::Module *module : sub_exports) {
187     ReportModuleExportsHelper(exports, module);
188   }
189 }
190 
191 void ClangModulesDeclVendorImpl::ReportModuleExports(
192     ClangModulesDeclVendor::ModuleVector &exports, clang::Module *module) {
193   std::set<ClangModulesDeclVendor::ModuleID> exports_set;
194 
195   ReportModuleExportsHelper(exports_set, module);
196 
197   for (ModuleID module : exports_set) {
198     exports.push_back(module);
199   }
200 }
201 
202 bool ClangModulesDeclVendorImpl::AddModule(ModulePath &path,
203                                            ModuleVector *exported_modules,
204                                            Stream &error_stream) {
205   // Fail early.
206 
207   if (m_compiler_instance->hadModuleLoaderFatalFailure()) {
208     error_stream.PutCString("error: Couldn't load a module because the module "
209                             "loader is in a fatal state.\n");
210     return false;
211   }
212 
213   // Check if we've already imported this module.
214 
215   std::vector<ConstString> imported_module;
216 
217   for (ConstString path_component : path) {
218     imported_module.push_back(path_component);
219   }
220 
221   {
222     ImportedModuleMap::iterator mi = m_imported_modules.find(imported_module);
223 
224     if (mi != m_imported_modules.end()) {
225       if (exported_modules) {
226         ReportModuleExports(*exported_modules, mi->second);
227       }
228       return true;
229     }
230   }
231 
232   if (!m_compiler_instance->getPreprocessor()
233            .getHeaderSearchInfo()
234            .lookupModule(path[0].GetStringRef())) {
235     error_stream.Printf("error: Header search couldn't locate module %s\n",
236                         path[0].AsCString());
237     return false;
238   }
239 
240   llvm::SmallVector<std::pair<clang::IdentifierInfo *, clang::SourceLocation>,
241                     4>
242       clang_path;
243 
244   {
245     clang::SourceManager &source_manager =
246         m_compiler_instance->getASTContext().getSourceManager();
247 
248     for (ConstString path_component : path) {
249       clang_path.push_back(std::make_pair(
250           &m_compiler_instance->getASTContext().Idents.get(
251               path_component.GetStringRef()),
252           source_manager.getLocForStartOfFile(source_manager.getMainFileID())
253               .getLocWithOffset(m_source_location_index++)));
254     }
255   }
256 
257   StoringDiagnosticConsumer *diagnostic_consumer =
258       static_cast<StoringDiagnosticConsumer *>(
259           m_compiler_instance->getDiagnostics().getClient());
260 
261   diagnostic_consumer->ClearDiagnostics();
262 
263   clang::Module *top_level_module = DoGetModule(clang_path.front(), false);
264 
265   if (!top_level_module) {
266     diagnostic_consumer->DumpDiagnostics(error_stream);
267     error_stream.Printf("error: Couldn't load top-level module %s\n",
268                         path[0].AsCString());
269     return false;
270   }
271 
272   clang::Module *submodule = top_level_module;
273 
274   for (size_t ci = 1; ci < path.size(); ++ci) {
275     llvm::StringRef component = path[ci].GetStringRef();
276     submodule = submodule->findSubmodule(component.str());
277     if (!submodule) {
278       diagnostic_consumer->DumpDiagnostics(error_stream);
279       error_stream.Printf("error: Couldn't load submodule %s\n",
280                           component.str().c_str());
281       return false;
282     }
283   }
284 
285   clang::Module *requested_module = DoGetModule(clang_path, true);
286 
287   if (requested_module != nullptr) {
288     if (exported_modules) {
289       ReportModuleExports(*exported_modules, requested_module);
290     }
291 
292     m_imported_modules[imported_module] = requested_module;
293 
294     m_enabled = true;
295 
296     return true;
297   }
298 
299   return false;
300 }
301 
302 bool ClangModulesDeclVendor::LanguageSupportsClangModules(
303     lldb::LanguageType language) {
304   switch (language) {
305   default:
306     return false;
307   // C++ and friends to be added
308   case lldb::LanguageType::eLanguageTypeC:
309   case lldb::LanguageType::eLanguageTypeC11:
310   case lldb::LanguageType::eLanguageTypeC89:
311   case lldb::LanguageType::eLanguageTypeC99:
312   case lldb::LanguageType::eLanguageTypeObjC:
313     return true;
314   }
315 }
316 
317 bool ClangModulesDeclVendorImpl::AddModulesForCompileUnit(
318     CompileUnit &cu, ClangModulesDeclVendor::ModuleVector &exported_modules,
319     Stream &error_stream) {
320   if (LanguageSupportsClangModules(cu.GetLanguage())) {
321     std::vector<ConstString> imported_modules = cu.GetImportedModules();
322 
323     for (ConstString imported_module : imported_modules) {
324       std::vector<ConstString> path;
325 
326       path.push_back(imported_module);
327 
328       if (!AddModule(path, &exported_modules, error_stream)) {
329         return false;
330       }
331     }
332 
333     return true;
334   }
335 
336   return true;
337 }
338 
339 // ClangImporter::lookupValue
340 
341 uint32_t
342 ClangModulesDeclVendorImpl::FindDecls(const ConstString &name, bool append,
343                                       uint32_t max_matches,
344                                       std::vector<clang::NamedDecl *> &decls) {
345   if (!m_enabled) {
346     return 0;
347   }
348 
349   if (!append)
350     decls.clear();
351 
352   clang::IdentifierInfo &ident =
353       m_compiler_instance->getASTContext().Idents.get(name.GetStringRef());
354 
355   clang::LookupResult lookup_result(
356       m_compiler_instance->getSema(), clang::DeclarationName(&ident),
357       clang::SourceLocation(), clang::Sema::LookupOrdinaryName);
358 
359   m_compiler_instance->getSema().LookupName(
360       lookup_result,
361       m_compiler_instance->getSema().getScopeForContext(
362           m_compiler_instance->getASTContext().getTranslationUnitDecl()));
363 
364   uint32_t num_matches = 0;
365 
366   for (clang::NamedDecl *named_decl : lookup_result) {
367     if (num_matches >= max_matches)
368       return num_matches;
369 
370     decls.push_back(named_decl);
371     ++num_matches;
372   }
373 
374   return num_matches;
375 }
376 
377 void ClangModulesDeclVendorImpl::ForEachMacro(
378     const ClangModulesDeclVendor::ModuleVector &modules,
379     std::function<bool(const std::string &)> handler) {
380   if (!m_enabled) {
381     return;
382   }
383 
384   typedef std::map<ModuleID, ssize_t> ModulePriorityMap;
385   ModulePriorityMap module_priorities;
386 
387   ssize_t priority = 0;
388 
389   for (ModuleID module : modules) {
390     module_priorities[module] = priority++;
391   }
392 
393   if (m_compiler_instance->getPreprocessor().getExternalSource()) {
394     m_compiler_instance->getPreprocessor()
395         .getExternalSource()
396         ->ReadDefinedMacros();
397   }
398 
399   for (clang::Preprocessor::macro_iterator
400            mi = m_compiler_instance->getPreprocessor().macro_begin(),
401            me = m_compiler_instance->getPreprocessor().macro_end();
402        mi != me; ++mi) {
403     const clang::IdentifierInfo *ii = nullptr;
404 
405     {
406       if (clang::IdentifierInfoLookup *lookup =
407               m_compiler_instance->getPreprocessor()
408                   .getIdentifierTable()
409                   .getExternalIdentifierLookup()) {
410         lookup->get(mi->first->getName());
411       }
412       if (!ii) {
413         ii = mi->first;
414       }
415     }
416 
417     ssize_t found_priority = -1;
418     clang::MacroInfo *macro_info = nullptr;
419 
420     for (clang::ModuleMacro *module_macro :
421          m_compiler_instance->getPreprocessor().getLeafModuleMacros(ii)) {
422       clang::Module *module = module_macro->getOwningModule();
423 
424       {
425         ModulePriorityMap::iterator pi =
426             module_priorities.find(reinterpret_cast<ModuleID>(module));
427 
428         if (pi != module_priorities.end() && pi->second > found_priority) {
429           macro_info = module_macro->getMacroInfo();
430           found_priority = pi->second;
431         }
432       }
433 
434       clang::Module *top_level_module = module->getTopLevelModule();
435 
436       if (top_level_module != module) {
437         ModulePriorityMap::iterator pi = module_priorities.find(
438             reinterpret_cast<ModuleID>(top_level_module));
439 
440         if ((pi != module_priorities.end()) && pi->second > found_priority) {
441           macro_info = module_macro->getMacroInfo();
442           found_priority = pi->second;
443         }
444       }
445     }
446 
447     if (macro_info) {
448       std::string macro_expansion = "#define ";
449       macro_expansion.append(mi->first->getName().str());
450 
451       {
452         if (macro_info->isFunctionLike()) {
453           macro_expansion.append("(");
454 
455           bool first_arg = true;
456 
457           for (auto pi = macro_info->param_begin(),
458                     pe = macro_info->param_end();
459                pi != pe; ++pi) {
460             if (!first_arg) {
461               macro_expansion.append(", ");
462             } else {
463               first_arg = false;
464             }
465 
466             macro_expansion.append((*pi)->getName().str());
467           }
468 
469           if (macro_info->isC99Varargs()) {
470             if (first_arg) {
471               macro_expansion.append("...");
472             } else {
473               macro_expansion.append(", ...");
474             }
475           } else if (macro_info->isGNUVarargs()) {
476             macro_expansion.append("...");
477           }
478 
479           macro_expansion.append(")");
480         }
481 
482         macro_expansion.append(" ");
483 
484         bool first_token = true;
485 
486         for (clang::MacroInfo::tokens_iterator ti = macro_info->tokens_begin(),
487                                                te = macro_info->tokens_end();
488              ti != te; ++ti) {
489           if (!first_token) {
490             macro_expansion.append(" ");
491           } else {
492             first_token = false;
493           }
494 
495           if (ti->isLiteral()) {
496             if (const char *literal_data = ti->getLiteralData()) {
497               std::string token_str(literal_data, ti->getLength());
498               macro_expansion.append(token_str);
499             } else {
500               bool invalid = false;
501               const char *literal_source =
502                   m_compiler_instance->getSourceManager().getCharacterData(
503                       ti->getLocation(), &invalid);
504 
505               if (invalid) {
506                 lldbassert(0 && "Unhandled token kind");
507                 macro_expansion.append("<unknown literal value>");
508               } else {
509                 macro_expansion.append(
510                     std::string(literal_source, ti->getLength()));
511               }
512             }
513           } else if (const char *punctuator_spelling =
514                          clang::tok::getPunctuatorSpelling(ti->getKind())) {
515             macro_expansion.append(punctuator_spelling);
516           } else if (const char *keyword_spelling =
517                          clang::tok::getKeywordSpelling(ti->getKind())) {
518             macro_expansion.append(keyword_spelling);
519           } else {
520             switch (ti->getKind()) {
521             case clang::tok::TokenKind::identifier:
522               macro_expansion.append(ti->getIdentifierInfo()->getName().str());
523               break;
524             case clang::tok::TokenKind::raw_identifier:
525               macro_expansion.append(ti->getRawIdentifier().str());
526               break;
527             default:
528               macro_expansion.append(ti->getName());
529               break;
530             }
531           }
532         }
533 
534         if (handler(macro_expansion)) {
535           return;
536         }
537       }
538     }
539   }
540 }
541 
542 clang::ModuleLoadResult
543 ClangModulesDeclVendorImpl::DoGetModule(clang::ModuleIdPath path,
544                                         bool make_visible) {
545   clang::Module::NameVisibilityKind visibility =
546       make_visible ? clang::Module::AllVisible : clang::Module::Hidden;
547 
548   const bool is_inclusion_directive = false;
549 
550   return m_compiler_instance->loadModule(path.front().second, path, visibility,
551                                          is_inclusion_directive);
552 }
553 
554 clang::ExternalASTMerger::ImporterSource
555 ClangModulesDeclVendorImpl::GetImporterSource() {
556   return {m_compiler_instance->getASTContext(),
557           m_compiler_instance->getFileManager(), m_origin_map};
558 }
559 
560 static const char *ModuleImportBufferName = "LLDBModulesMemoryBuffer";
561 
562 lldb_private::ClangModulesDeclVendor *
563 ClangModulesDeclVendor::Create(Target &target) {
564   // FIXME we should insure programmatically that the expression parser's
565   // compiler and the modules runtime's
566   // compiler are both initialized in the same way – preferably by the same
567   // code.
568 
569   if (!target.GetPlatform()->SupportsModules())
570     return nullptr;
571 
572   const ArchSpec &arch = target.GetArchitecture();
573 
574   std::vector<std::string> compiler_invocation_arguments = {
575       "clang",
576       "-fmodules",
577       "-fimplicit-module-maps",
578       "-fcxx-modules",
579       "-fsyntax-only",
580       "-femit-all-decls",
581       "-target",
582       arch.GetTriple().str(),
583       "-fmodules-validate-system-headers",
584       "-Werror=non-modular-include-in-framework-module"};
585 
586   target.GetPlatform()->AddClangModuleCompilationOptions(
587       &target, compiler_invocation_arguments);
588 
589   compiler_invocation_arguments.push_back(ModuleImportBufferName);
590 
591   // Add additional search paths with { "-I", path } or { "-F", path } here.
592 
593   {
594     llvm::SmallString<128> path;
595     auto props = ModuleList::GetGlobalModuleListProperties();
596     props.GetClangModulesCachePath().GetPath(path);
597     std::string module_cache_argument("-fmodules-cache-path=");
598     module_cache_argument.append(path.str());
599     compiler_invocation_arguments.push_back(module_cache_argument);
600   }
601 
602   FileSpecList &module_search_paths = target.GetClangModuleSearchPaths();
603 
604   for (size_t spi = 0, spe = module_search_paths.GetSize(); spi < spe; ++spi) {
605     const FileSpec &search_path = module_search_paths.GetFileSpecAtIndex(spi);
606 
607     std::string search_path_argument = "-I";
608     search_path_argument.append(search_path.GetPath());
609 
610     compiler_invocation_arguments.push_back(search_path_argument);
611   }
612 
613   {
614     FileSpec clang_resource_dir = GetResourceDir();
615 
616     if (llvm::sys::fs::is_directory(clang_resource_dir.GetPath())) {
617       compiler_invocation_arguments.push_back("-resource-dir");
618       compiler_invocation_arguments.push_back(clang_resource_dir.GetPath());
619     }
620   }
621 
622   llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_engine =
623       clang::CompilerInstance::createDiagnostics(new clang::DiagnosticOptions,
624                                                  new StoringDiagnosticConsumer);
625 
626   std::vector<const char *> compiler_invocation_argument_cstrs;
627 
628   for (const std::string &arg : compiler_invocation_arguments) {
629     compiler_invocation_argument_cstrs.push_back(arg.c_str());
630   }
631 
632   std::shared_ptr<clang::CompilerInvocation> invocation =
633       clang::createInvocationFromCommandLine(compiler_invocation_argument_cstrs,
634                                              diagnostics_engine);
635 
636   if (!invocation)
637     return nullptr;
638 
639   std::unique_ptr<llvm::MemoryBuffer> source_buffer =
640       llvm::MemoryBuffer::getMemBuffer(
641           "extern int __lldb __attribute__((unavailable));",
642           ModuleImportBufferName);
643 
644   invocation->getPreprocessorOpts().addRemappedFile(ModuleImportBufferName,
645                                                     source_buffer.release());
646 
647   std::unique_ptr<clang::CompilerInstance> instance(
648       new clang::CompilerInstance);
649 
650   instance->setDiagnostics(diagnostics_engine.get());
651   instance->setInvocation(invocation);
652 
653   std::unique_ptr<clang::FrontendAction> action(new clang::SyntaxOnlyAction);
654 
655   instance->setTarget(clang::TargetInfo::CreateTargetInfo(
656       *diagnostics_engine, instance->getInvocation().TargetOpts));
657 
658   if (!instance->hasTarget())
659     return nullptr;
660 
661   instance->getTarget().adjust(instance->getLangOpts());
662 
663   if (!action->BeginSourceFile(*instance,
664                                instance->getFrontendOpts().Inputs[0]))
665     return nullptr;
666 
667   instance->getPreprocessor().enableIncrementalProcessing();
668 
669   instance->createModuleManager();
670 
671   instance->createSema(action->getTranslationUnitKind(), nullptr);
672 
673   const bool skipFunctionBodies = false;
674   std::unique_ptr<clang::Parser> parser(new clang::Parser(
675       instance->getPreprocessor(), instance->getSema(), skipFunctionBodies));
676 
677   instance->getPreprocessor().EnterMainSourceFile();
678   parser->Initialize();
679 
680   clang::Parser::DeclGroupPtrTy parsed;
681 
682   while (!parser->ParseTopLevelDecl(parsed))
683     ;
684 
685   return new ClangModulesDeclVendorImpl(std::move(diagnostics_engine),
686                                         std::move(invocation),
687                                         std::move(instance), std::move(parser));
688 }
689