1 //===-- ClangModulesDeclVendor.cpp ------------------------------*- C++ -*-===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include <mutex>
11 
12 #include "clang/Basic/TargetInfo.h"
13 #include "clang/Frontend/CompilerInstance.h"
14 #include "clang/Frontend/FrontendActions.h"
15 #include "clang/Lex/Preprocessor.h"
16 #include "clang/Lex/PreprocessorOptions.h"
17 #include "clang/Parse/Parser.h"
18 #include "clang/Sema/Lookup.h"
19 #include "clang/Serialization/ASTReader.h"
20 #include "llvm/Support/FileSystem.h"
21 #include "llvm/Support/Path.h"
22 #include "llvm/Support/Threading.h"
23 
24 #include "ClangHost.h"
25 #include "ClangModulesDeclVendor.h"
26 
27 #include "lldb/Core/ModuleList.h"
28 #include "lldb/Host/Host.h"
29 #include "lldb/Host/HostInfo.h"
30 #include "lldb/Symbol/CompileUnit.h"
31 #include "lldb/Target/Target.h"
32 #include "lldb/Utility/FileSpec.h"
33 #include "lldb/Utility/LLDBAssert.h"
34 #include "lldb/Utility/Log.h"
35 #include "lldb/Utility/StreamString.h"
36 
37 using namespace lldb_private;
38 
39 namespace {
40 // Any Clang compiler requires a consumer for diagnostics.  This one stores
41 // them as strings so we can provide them to the user in case a module failed
42 // to load.
43 class StoringDiagnosticConsumer : public clang::DiagnosticConsumer {
44 public:
45   StoringDiagnosticConsumer();
46 
47   void HandleDiagnostic(clang::DiagnosticsEngine::Level DiagLevel,
48                         const clang::Diagnostic &info) override;
49 
50   void ClearDiagnostics();
51 
52   void DumpDiagnostics(Stream &error_stream);
53 
54 private:
55   typedef std::pair<clang::DiagnosticsEngine::Level, std::string>
56       IDAndDiagnostic;
57   std::vector<IDAndDiagnostic> m_diagnostics;
58   Log *m_log;
59 };
60 
61 // The private implementation of our ClangModulesDeclVendor.  Contains all the
62 // Clang state required to load modules.
63 class ClangModulesDeclVendorImpl : public ClangModulesDeclVendor {
64 public:
65   ClangModulesDeclVendorImpl(
66       llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_engine,
67       std::shared_ptr<clang::CompilerInvocation> compiler_invocation,
68       std::unique_ptr<clang::CompilerInstance> compiler_instance,
69       std::unique_ptr<clang::Parser> parser);
70 
71   ~ClangModulesDeclVendorImpl() override = default;
72 
73   bool AddModule(ModulePath &path, ModuleVector *exported_modules,
74                  Stream &error_stream) override;
75 
76   bool AddModulesForCompileUnit(CompileUnit &cu, ModuleVector &exported_modules,
77                                 Stream &error_stream) override;
78 
79   uint32_t FindDecls(const ConstString &name, bool append, uint32_t max_matches,
80                      std::vector<clang::NamedDecl *> &decls) override;
81 
82   void ForEachMacro(const ModuleVector &modules,
83                     std::function<bool(const std::string &)> handler) override;
84 
85   clang::ExternalASTMerger::ImporterSource GetImporterSource() override;
86 private:
87   void
88   ReportModuleExportsHelper(std::set<ClangModulesDeclVendor::ModuleID> &exports,
89                             clang::Module *module);
90 
91   void ReportModuleExports(ModuleVector &exports, clang::Module *module);
92 
93   clang::ModuleLoadResult DoGetModule(clang::ModuleIdPath path,
94                                       bool make_visible);
95 
96   bool m_enabled = false;
97 
98   llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> m_diagnostics_engine;
99   std::shared_ptr<clang::CompilerInvocation> m_compiler_invocation;
100   std::unique_ptr<clang::CompilerInstance> m_compiler_instance;
101   std::unique_ptr<clang::Parser> m_parser;
102   size_t m_source_location_index =
103       0; // used to give name components fake SourceLocations
104 
105   typedef std::vector<ConstString> ImportedModule;
106   typedef std::map<ImportedModule, clang::Module *> ImportedModuleMap;
107   typedef std::set<ModuleID> ImportedModuleSet;
108   ImportedModuleMap m_imported_modules;
109   ImportedModuleSet m_user_imported_modules;
110   const clang::ExternalASTMerger::OriginMap m_origin_map;
111 };
112 } // anonymous namespace
113 
114 StoringDiagnosticConsumer::StoringDiagnosticConsumer() {
115   m_log = lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_EXPRESSIONS);
116 }
117 
118 void StoringDiagnosticConsumer::HandleDiagnostic(
119     clang::DiagnosticsEngine::Level DiagLevel, const clang::Diagnostic &info) {
120   llvm::SmallVector<char, 256> diagnostic_string;
121 
122   info.FormatDiagnostic(diagnostic_string);
123 
124   m_diagnostics.push_back(
125       IDAndDiagnostic(DiagLevel, std::string(diagnostic_string.data(),
126                                              diagnostic_string.size())));
127 }
128 
129 void StoringDiagnosticConsumer::ClearDiagnostics() { m_diagnostics.clear(); }
130 
131 void StoringDiagnosticConsumer::DumpDiagnostics(Stream &error_stream) {
132   for (IDAndDiagnostic &diag : m_diagnostics) {
133     switch (diag.first) {
134     default:
135       error_stream.PutCString(diag.second);
136       error_stream.PutChar('\n');
137       break;
138     case clang::DiagnosticsEngine::Level::Ignored:
139       break;
140     }
141   }
142 }
143 
144 ClangModulesDeclVendor::ClangModulesDeclVendor() {}
145 
146 ClangModulesDeclVendor::~ClangModulesDeclVendor() {}
147 
148 ClangModulesDeclVendorImpl::ClangModulesDeclVendorImpl(
149     llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_engine,
150     std::shared_ptr<clang::CompilerInvocation> compiler_invocation,
151     std::unique_ptr<clang::CompilerInstance> compiler_instance,
152     std::unique_ptr<clang::Parser> parser)
153     : m_diagnostics_engine(std::move(diagnostics_engine)),
154       m_compiler_invocation(std::move(compiler_invocation)),
155       m_compiler_instance(std::move(compiler_instance)),
156       m_parser(std::move(parser)), m_origin_map() {}
157 
158 void ClangModulesDeclVendorImpl::ReportModuleExportsHelper(
159     std::set<ClangModulesDeclVendor::ModuleID> &exports,
160     clang::Module *module) {
161   if (exports.count(reinterpret_cast<ClangModulesDeclVendor::ModuleID>(module)))
162     return;
163 
164   exports.insert(reinterpret_cast<ClangModulesDeclVendor::ModuleID>(module));
165 
166   llvm::SmallVector<clang::Module *, 2> sub_exports;
167 
168   module->getExportedModules(sub_exports);
169 
170   for (clang::Module *module : sub_exports) {
171     ReportModuleExportsHelper(exports, module);
172   }
173 }
174 
175 void ClangModulesDeclVendorImpl::ReportModuleExports(
176     ClangModulesDeclVendor::ModuleVector &exports, clang::Module *module) {
177   std::set<ClangModulesDeclVendor::ModuleID> exports_set;
178 
179   ReportModuleExportsHelper(exports_set, module);
180 
181   for (ModuleID module : exports_set) {
182     exports.push_back(module);
183   }
184 }
185 
186 bool ClangModulesDeclVendorImpl::AddModule(ModulePath &path,
187                                            ModuleVector *exported_modules,
188                                            Stream &error_stream) {
189   // Fail early.
190 
191   if (m_compiler_instance->hadModuleLoaderFatalFailure()) {
192     error_stream.PutCString("error: Couldn't load a module because the module "
193                             "loader is in a fatal state.\n");
194     return false;
195   }
196 
197   // Check if we've already imported this module.
198 
199   std::vector<ConstString> imported_module;
200 
201   for (ConstString path_component : path) {
202     imported_module.push_back(path_component);
203   }
204 
205   {
206     ImportedModuleMap::iterator mi = m_imported_modules.find(imported_module);
207 
208     if (mi != m_imported_modules.end()) {
209       if (exported_modules) {
210         ReportModuleExports(*exported_modules, mi->second);
211       }
212       return true;
213     }
214   }
215 
216   if (!m_compiler_instance->getPreprocessor()
217            .getHeaderSearchInfo()
218            .lookupModule(path[0].GetStringRef())) {
219     error_stream.Printf("error: Header search couldn't locate module %s\n",
220                         path[0].AsCString());
221     return false;
222   }
223 
224   llvm::SmallVector<std::pair<clang::IdentifierInfo *, clang::SourceLocation>,
225                     4>
226       clang_path;
227 
228   {
229     clang::SourceManager &source_manager =
230         m_compiler_instance->getASTContext().getSourceManager();
231 
232     for (ConstString path_component : path) {
233       clang_path.push_back(std::make_pair(
234           &m_compiler_instance->getASTContext().Idents.get(
235               path_component.GetStringRef()),
236           source_manager.getLocForStartOfFile(source_manager.getMainFileID())
237               .getLocWithOffset(m_source_location_index++)));
238     }
239   }
240 
241   StoringDiagnosticConsumer *diagnostic_consumer =
242       static_cast<StoringDiagnosticConsumer *>(
243           m_compiler_instance->getDiagnostics().getClient());
244 
245   diagnostic_consumer->ClearDiagnostics();
246 
247   clang::Module *top_level_module = DoGetModule(clang_path.front(), false);
248 
249   if (!top_level_module) {
250     diagnostic_consumer->DumpDiagnostics(error_stream);
251     error_stream.Printf("error: Couldn't load top-level module %s\n",
252                         path[0].AsCString());
253     return false;
254   }
255 
256   clang::Module *submodule = top_level_module;
257 
258   for (size_t ci = 1; ci < path.size(); ++ci) {
259     llvm::StringRef component = path[ci].GetStringRef();
260     submodule = submodule->findSubmodule(component.str());
261     if (!submodule) {
262       diagnostic_consumer->DumpDiagnostics(error_stream);
263       error_stream.Printf("error: Couldn't load submodule %s\n",
264                           component.str().c_str());
265       return false;
266     }
267   }
268 
269   clang::Module *requested_module = DoGetModule(clang_path, true);
270 
271   if (requested_module != nullptr) {
272     if (exported_modules) {
273       ReportModuleExports(*exported_modules, requested_module);
274     }
275 
276     m_imported_modules[imported_module] = requested_module;
277 
278     m_enabled = true;
279 
280     return true;
281   }
282 
283   return false;
284 }
285 
286 bool ClangModulesDeclVendor::LanguageSupportsClangModules(
287     lldb::LanguageType language) {
288   switch (language) {
289   default:
290     return false;
291   // C++ and friends to be added
292   case lldb::LanguageType::eLanguageTypeC:
293   case lldb::LanguageType::eLanguageTypeC11:
294   case lldb::LanguageType::eLanguageTypeC89:
295   case lldb::LanguageType::eLanguageTypeC99:
296   case lldb::LanguageType::eLanguageTypeObjC:
297     return true;
298   }
299 }
300 
301 bool ClangModulesDeclVendorImpl::AddModulesForCompileUnit(
302     CompileUnit &cu, ClangModulesDeclVendor::ModuleVector &exported_modules,
303     Stream &error_stream) {
304   if (LanguageSupportsClangModules(cu.GetLanguage())) {
305     std::vector<ConstString> imported_modules = cu.GetImportedModules();
306 
307     for (ConstString imported_module : imported_modules) {
308       std::vector<ConstString> path;
309 
310       path.push_back(imported_module);
311 
312       if (!AddModule(path, &exported_modules, error_stream)) {
313         return false;
314       }
315     }
316 
317     return true;
318   }
319 
320   return true;
321 }
322 
323 // ClangImporter::lookupValue
324 
325 uint32_t
326 ClangModulesDeclVendorImpl::FindDecls(const ConstString &name, bool append,
327                                       uint32_t max_matches,
328                                       std::vector<clang::NamedDecl *> &decls) {
329   if (!m_enabled) {
330     return 0;
331   }
332 
333   if (!append)
334     decls.clear();
335 
336   clang::IdentifierInfo &ident =
337       m_compiler_instance->getASTContext().Idents.get(name.GetStringRef());
338 
339   clang::LookupResult lookup_result(
340       m_compiler_instance->getSema(), clang::DeclarationName(&ident),
341       clang::SourceLocation(), clang::Sema::LookupOrdinaryName);
342 
343   m_compiler_instance->getSema().LookupName(
344       lookup_result,
345       m_compiler_instance->getSema().getScopeForContext(
346           m_compiler_instance->getASTContext().getTranslationUnitDecl()));
347 
348   uint32_t num_matches = 0;
349 
350   for (clang::NamedDecl *named_decl : lookup_result) {
351     if (num_matches >= max_matches)
352       return num_matches;
353 
354     decls.push_back(named_decl);
355     ++num_matches;
356   }
357 
358   return num_matches;
359 }
360 
361 void ClangModulesDeclVendorImpl::ForEachMacro(
362     const ClangModulesDeclVendor::ModuleVector &modules,
363     std::function<bool(const std::string &)> handler) {
364   if (!m_enabled) {
365     return;
366   }
367 
368   typedef std::map<ModuleID, ssize_t> ModulePriorityMap;
369   ModulePriorityMap module_priorities;
370 
371   ssize_t priority = 0;
372 
373   for (ModuleID module : modules) {
374     module_priorities[module] = priority++;
375   }
376 
377   if (m_compiler_instance->getPreprocessor().getExternalSource()) {
378     m_compiler_instance->getPreprocessor()
379         .getExternalSource()
380         ->ReadDefinedMacros();
381   }
382 
383   for (clang::Preprocessor::macro_iterator
384            mi = m_compiler_instance->getPreprocessor().macro_begin(),
385            me = m_compiler_instance->getPreprocessor().macro_end();
386        mi != me; ++mi) {
387     const clang::IdentifierInfo *ii = nullptr;
388 
389     {
390       if (clang::IdentifierInfoLookup *lookup =
391               m_compiler_instance->getPreprocessor()
392                   .getIdentifierTable()
393                   .getExternalIdentifierLookup()) {
394         lookup->get(mi->first->getName());
395       }
396       if (!ii) {
397         ii = mi->first;
398       }
399     }
400 
401     ssize_t found_priority = -1;
402     clang::MacroInfo *macro_info = nullptr;
403 
404     for (clang::ModuleMacro *module_macro :
405          m_compiler_instance->getPreprocessor().getLeafModuleMacros(ii)) {
406       clang::Module *module = module_macro->getOwningModule();
407 
408       {
409         ModulePriorityMap::iterator pi =
410             module_priorities.find(reinterpret_cast<ModuleID>(module));
411 
412         if (pi != module_priorities.end() && pi->second > found_priority) {
413           macro_info = module_macro->getMacroInfo();
414           found_priority = pi->second;
415         }
416       }
417 
418       clang::Module *top_level_module = module->getTopLevelModule();
419 
420       if (top_level_module != module) {
421         ModulePriorityMap::iterator pi = module_priorities.find(
422             reinterpret_cast<ModuleID>(top_level_module));
423 
424         if ((pi != module_priorities.end()) && pi->second > found_priority) {
425           macro_info = module_macro->getMacroInfo();
426           found_priority = pi->second;
427         }
428       }
429     }
430 
431     if (macro_info) {
432       std::string macro_expansion = "#define ";
433       macro_expansion.append(mi->first->getName().str());
434 
435       {
436         if (macro_info->isFunctionLike()) {
437           macro_expansion.append("(");
438 
439           bool first_arg = true;
440 
441           for (auto pi = macro_info->param_begin(),
442                     pe = macro_info->param_end();
443                pi != pe; ++pi) {
444             if (!first_arg) {
445               macro_expansion.append(", ");
446             } else {
447               first_arg = false;
448             }
449 
450             macro_expansion.append((*pi)->getName().str());
451           }
452 
453           if (macro_info->isC99Varargs()) {
454             if (first_arg) {
455               macro_expansion.append("...");
456             } else {
457               macro_expansion.append(", ...");
458             }
459           } else if (macro_info->isGNUVarargs()) {
460             macro_expansion.append("...");
461           }
462 
463           macro_expansion.append(")");
464         }
465 
466         macro_expansion.append(" ");
467 
468         bool first_token = true;
469 
470         for (clang::MacroInfo::tokens_iterator ti = macro_info->tokens_begin(),
471                                                te = macro_info->tokens_end();
472              ti != te; ++ti) {
473           if (!first_token) {
474             macro_expansion.append(" ");
475           } else {
476             first_token = false;
477           }
478 
479           if (ti->isLiteral()) {
480             if (const char *literal_data = ti->getLiteralData()) {
481               std::string token_str(literal_data, ti->getLength());
482               macro_expansion.append(token_str);
483             } else {
484               bool invalid = false;
485               const char *literal_source =
486                   m_compiler_instance->getSourceManager().getCharacterData(
487                       ti->getLocation(), &invalid);
488 
489               if (invalid) {
490                 lldbassert(0 && "Unhandled token kind");
491                 macro_expansion.append("<unknown literal value>");
492               } else {
493                 macro_expansion.append(
494                     std::string(literal_source, ti->getLength()));
495               }
496             }
497           } else if (const char *punctuator_spelling =
498                          clang::tok::getPunctuatorSpelling(ti->getKind())) {
499             macro_expansion.append(punctuator_spelling);
500           } else if (const char *keyword_spelling =
501                          clang::tok::getKeywordSpelling(ti->getKind())) {
502             macro_expansion.append(keyword_spelling);
503           } else {
504             switch (ti->getKind()) {
505             case clang::tok::TokenKind::identifier:
506               macro_expansion.append(ti->getIdentifierInfo()->getName().str());
507               break;
508             case clang::tok::TokenKind::raw_identifier:
509               macro_expansion.append(ti->getRawIdentifier().str());
510               break;
511             default:
512               macro_expansion.append(ti->getName());
513               break;
514             }
515           }
516         }
517 
518         if (handler(macro_expansion)) {
519           return;
520         }
521       }
522     }
523   }
524 }
525 
526 clang::ModuleLoadResult
527 ClangModulesDeclVendorImpl::DoGetModule(clang::ModuleIdPath path,
528                                         bool make_visible) {
529   clang::Module::NameVisibilityKind visibility =
530       make_visible ? clang::Module::AllVisible : clang::Module::Hidden;
531 
532   const bool is_inclusion_directive = false;
533 
534   return m_compiler_instance->loadModule(path.front().second, path, visibility,
535                                          is_inclusion_directive);
536 }
537 
538 clang::ExternalASTMerger::ImporterSource
539 ClangModulesDeclVendorImpl::GetImporterSource() {
540   return {m_compiler_instance->getASTContext(),
541           m_compiler_instance->getFileManager(), m_origin_map};
542 }
543 
544 static const char *ModuleImportBufferName = "LLDBModulesMemoryBuffer";
545 
546 lldb_private::ClangModulesDeclVendor *
547 ClangModulesDeclVendor::Create(Target &target) {
548   // FIXME we should insure programmatically that the expression parser's
549   // compiler and the modules runtime's
550   // compiler are both initialized in the same way – preferably by the same
551   // code.
552 
553   if (!target.GetPlatform()->SupportsModules())
554     return nullptr;
555 
556   const ArchSpec &arch = target.GetArchitecture();
557 
558   std::vector<std::string> compiler_invocation_arguments = {
559       "clang",
560       "-fmodules",
561       "-fimplicit-module-maps",
562       "-fcxx-modules",
563       "-fsyntax-only",
564       "-femit-all-decls",
565       "-target",
566       arch.GetTriple().str(),
567       "-fmodules-validate-system-headers",
568       "-Werror=non-modular-include-in-framework-module"};
569 
570   target.GetPlatform()->AddClangModuleCompilationOptions(
571       &target, compiler_invocation_arguments);
572 
573   compiler_invocation_arguments.push_back(ModuleImportBufferName);
574 
575   // Add additional search paths with { "-I", path } or { "-F", path } here.
576 
577   {
578     llvm::SmallString<128> path;
579     auto props = ModuleList::GetGlobalModuleListProperties();
580     props.GetClangModulesCachePath().GetPath(path);
581     std::string module_cache_argument("-fmodules-cache-path=");
582     module_cache_argument.append(path.str());
583     compiler_invocation_arguments.push_back(module_cache_argument);
584   }
585 
586   FileSpecList &module_search_paths = target.GetClangModuleSearchPaths();
587 
588   for (size_t spi = 0, spe = module_search_paths.GetSize(); spi < spe; ++spi) {
589     const FileSpec &search_path = module_search_paths.GetFileSpecAtIndex(spi);
590 
591     std::string search_path_argument = "-I";
592     search_path_argument.append(search_path.GetPath());
593 
594     compiler_invocation_arguments.push_back(search_path_argument);
595   }
596 
597   {
598     FileSpec clang_resource_dir = GetClangResourceDir();
599 
600     if (FileSystem::Instance().IsDirectory(clang_resource_dir.GetPath())) {
601       compiler_invocation_arguments.push_back("-resource-dir");
602       compiler_invocation_arguments.push_back(clang_resource_dir.GetPath());
603     }
604   }
605 
606   llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_engine =
607       clang::CompilerInstance::createDiagnostics(new clang::DiagnosticOptions,
608                                                  new StoringDiagnosticConsumer);
609 
610   std::vector<const char *> compiler_invocation_argument_cstrs;
611 
612   for (const std::string &arg : compiler_invocation_arguments) {
613     compiler_invocation_argument_cstrs.push_back(arg.c_str());
614   }
615 
616   std::shared_ptr<clang::CompilerInvocation> invocation =
617       clang::createInvocationFromCommandLine(compiler_invocation_argument_cstrs,
618                                              diagnostics_engine);
619 
620   if (!invocation)
621     return nullptr;
622 
623   std::unique_ptr<llvm::MemoryBuffer> source_buffer =
624       llvm::MemoryBuffer::getMemBuffer(
625           "extern int __lldb __attribute__((unavailable));",
626           ModuleImportBufferName);
627 
628   invocation->getPreprocessorOpts().addRemappedFile(ModuleImportBufferName,
629                                                     source_buffer.release());
630 
631   std::unique_ptr<clang::CompilerInstance> instance(
632       new clang::CompilerInstance);
633 
634   instance->setDiagnostics(diagnostics_engine.get());
635   instance->setInvocation(invocation);
636 
637   std::unique_ptr<clang::FrontendAction> action(new clang::SyntaxOnlyAction);
638 
639   instance->setTarget(clang::TargetInfo::CreateTargetInfo(
640       *diagnostics_engine, instance->getInvocation().TargetOpts));
641 
642   if (!instance->hasTarget())
643     return nullptr;
644 
645   instance->getTarget().adjust(instance->getLangOpts());
646 
647   if (!action->BeginSourceFile(*instance,
648                                instance->getFrontendOpts().Inputs[0]))
649     return nullptr;
650 
651   instance->getPreprocessor().enableIncrementalProcessing();
652 
653   instance->createModuleManager();
654 
655   instance->createSema(action->getTranslationUnitKind(), nullptr);
656 
657   const bool skipFunctionBodies = false;
658   std::unique_ptr<clang::Parser> parser(new clang::Parser(
659       instance->getPreprocessor(), instance->getSema(), skipFunctionBodies));
660 
661   instance->getPreprocessor().EnterMainSourceFile();
662   parser->Initialize();
663 
664   clang::Parser::DeclGroupPtrTy parsed;
665 
666   while (!parser->ParseTopLevelDecl(parsed))
667     ;
668 
669   return new ClangModulesDeclVendorImpl(std::move(diagnostics_engine),
670                                         std::move(invocation),
671                                         std::move(instance), std::move(parser));
672 }
673