1 //===-- ClangModulesDeclVendor.cpp ------------------------------*- C++ -*-===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 // C Includes
11 // C++ Includes
12 #include <mutex>
13 
14 // Other libraries and framework includes
15 #include "clang/Basic/TargetInfo.h"
16 #include "clang/Frontend/CompilerInstance.h"
17 #include "clang/Frontend/FrontendActions.h"
18 #include "clang/Lex/Preprocessor.h"
19 #include "clang/Lex/PreprocessorOptions.h"
20 #include "clang/Parse/Parser.h"
21 #include "clang/Sema/Lookup.h"
22 #include "clang/Serialization/ASTReader.h"
23 #include "llvm/Support/Path.h"
24 #include "llvm/Support/Threading.h"
25 
26 // Project includes
27 #include "ClangModulesDeclVendor.h"
28 
29 #include "lldb/Core/Log.h"
30 #include "lldb/Host/FileSpec.h"
31 #include "lldb/Host/Host.h"
32 #include "lldb/Host/HostInfo.h"
33 #include "lldb/Symbol/CompileUnit.h"
34 #include "lldb/Target/Target.h"
35 #include "lldb/Utility/LLDBAssert.h"
36 #include "lldb/Utility/StreamString.h"
37 
38 using namespace lldb_private;
39 
40 namespace {
41 // Any Clang compiler requires a consumer for diagnostics.  This one stores them
42 // as strings
43 // so we can provide them to the user in case a module failed to load.
44 class StoringDiagnosticConsumer : public clang::DiagnosticConsumer {
45 public:
46   StoringDiagnosticConsumer();
47 
48   void HandleDiagnostic(clang::DiagnosticsEngine::Level DiagLevel,
49                         const clang::Diagnostic &info) override;
50 
51   void ClearDiagnostics();
52 
53   void DumpDiagnostics(Stream &error_stream);
54 
55 private:
56   typedef std::pair<clang::DiagnosticsEngine::Level, std::string>
57       IDAndDiagnostic;
58   std::vector<IDAndDiagnostic> m_diagnostics;
59   Log *m_log;
60 };
61 
62 // The private implementation of our ClangModulesDeclVendor.  Contains all the
63 // Clang state required
64 // to load modules.
65 class ClangModulesDeclVendorImpl : public ClangModulesDeclVendor {
66 public:
67   ClangModulesDeclVendorImpl(
68       llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_engine,
69       std::shared_ptr<clang::CompilerInvocation> compiler_invocation,
70       std::unique_ptr<clang::CompilerInstance> compiler_instance,
71       std::unique_ptr<clang::Parser> parser);
72 
73   ~ClangModulesDeclVendorImpl() override = default;
74 
75   bool AddModule(ModulePath &path, ModuleVector *exported_modules,
76                  Stream &error_stream) override;
77 
78   bool AddModulesForCompileUnit(CompileUnit &cu, ModuleVector &exported_modules,
79                                 Stream &error_stream) override;
80 
81   uint32_t FindDecls(const ConstString &name, bool append, uint32_t max_matches,
82                      std::vector<clang::NamedDecl *> &decls) override;
83 
84   void ForEachMacro(const ModuleVector &modules,
85                     std::function<bool(const std::string &)> handler) override;
86 
87 private:
88   void
89   ReportModuleExportsHelper(std::set<ClangModulesDeclVendor::ModuleID> &exports,
90                             clang::Module *module);
91 
92   void ReportModuleExports(ModuleVector &exports, clang::Module *module);
93 
94   clang::ModuleLoadResult DoGetModule(clang::ModuleIdPath path,
95                                       bool make_visible);
96 
97   bool m_enabled = false;
98 
99   llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> m_diagnostics_engine;
100   std::shared_ptr<clang::CompilerInvocation> m_compiler_invocation;
101   std::unique_ptr<clang::CompilerInstance> m_compiler_instance;
102   std::unique_ptr<clang::Parser> m_parser;
103   size_t m_source_location_index =
104       0; // used to give name components fake SourceLocations
105 
106   typedef std::vector<ConstString> ImportedModule;
107   typedef std::map<ImportedModule, clang::Module *> ImportedModuleMap;
108   typedef std::set<ModuleID> ImportedModuleSet;
109   ImportedModuleMap m_imported_modules;
110   ImportedModuleSet m_user_imported_modules;
111 };
112 } // anonymous namespace
113 
114 StoringDiagnosticConsumer::StoringDiagnosticConsumer() {
115   m_log = lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_EXPRESSIONS);
116 }
117 
118 void StoringDiagnosticConsumer::HandleDiagnostic(
119     clang::DiagnosticsEngine::Level DiagLevel, const clang::Diagnostic &info) {
120   llvm::SmallVector<char, 256> diagnostic_string;
121 
122   info.FormatDiagnostic(diagnostic_string);
123 
124   m_diagnostics.push_back(
125       IDAndDiagnostic(DiagLevel, std::string(diagnostic_string.data(),
126                                              diagnostic_string.size())));
127 }
128 
129 void StoringDiagnosticConsumer::ClearDiagnostics() { m_diagnostics.clear(); }
130 
131 void StoringDiagnosticConsumer::DumpDiagnostics(Stream &error_stream) {
132   for (IDAndDiagnostic &diag : m_diagnostics) {
133     switch (diag.first) {
134     default:
135       error_stream.PutCString(diag.second);
136       error_stream.PutChar('\n');
137       break;
138     case clang::DiagnosticsEngine::Level::Ignored:
139       break;
140     }
141   }
142 }
143 
144 static FileSpec GetResourceDir() {
145   static FileSpec g_cached_resource_dir;
146 
147   static llvm::once_flag g_once_flag;
148 
149   llvm::call_once(g_once_flag, []() {
150     HostInfo::GetLLDBPath(lldb::ePathTypeClangDir, g_cached_resource_dir);
151   });
152 
153   return g_cached_resource_dir;
154 }
155 
156 ClangModulesDeclVendor::ClangModulesDeclVendor() {}
157 
158 ClangModulesDeclVendor::~ClangModulesDeclVendor() {}
159 
160 ClangModulesDeclVendorImpl::ClangModulesDeclVendorImpl(
161     llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_engine,
162     std::shared_ptr<clang::CompilerInvocation> compiler_invocation,
163     std::unique_ptr<clang::CompilerInstance> compiler_instance,
164     std::unique_ptr<clang::Parser> parser)
165     : m_diagnostics_engine(std::move(diagnostics_engine)),
166       m_compiler_invocation(std::move(compiler_invocation)),
167       m_compiler_instance(std::move(compiler_instance)),
168       m_parser(std::move(parser)) {}
169 
170 void ClangModulesDeclVendorImpl::ReportModuleExportsHelper(
171     std::set<ClangModulesDeclVendor::ModuleID> &exports,
172     clang::Module *module) {
173   if (exports.count(reinterpret_cast<ClangModulesDeclVendor::ModuleID>(module)))
174     return;
175 
176   exports.insert(reinterpret_cast<ClangModulesDeclVendor::ModuleID>(module));
177 
178   llvm::SmallVector<clang::Module *, 2> sub_exports;
179 
180   module->getExportedModules(sub_exports);
181 
182   for (clang::Module *module : sub_exports) {
183     ReportModuleExportsHelper(exports, module);
184   }
185 }
186 
187 void ClangModulesDeclVendorImpl::ReportModuleExports(
188     ClangModulesDeclVendor::ModuleVector &exports, clang::Module *module) {
189   std::set<ClangModulesDeclVendor::ModuleID> exports_set;
190 
191   ReportModuleExportsHelper(exports_set, module);
192 
193   for (ModuleID module : exports_set) {
194     exports.push_back(module);
195   }
196 }
197 
198 bool ClangModulesDeclVendorImpl::AddModule(ModulePath &path,
199                                            ModuleVector *exported_modules,
200                                            Stream &error_stream) {
201   // Fail early.
202 
203   if (m_compiler_instance->hadModuleLoaderFatalFailure()) {
204     error_stream.PutCString("error: Couldn't load a module because the module "
205                             "loader is in a fatal state.\n");
206     return false;
207   }
208 
209   // Check if we've already imported this module.
210 
211   std::vector<ConstString> imported_module;
212 
213   for (ConstString path_component : path) {
214     imported_module.push_back(path_component);
215   }
216 
217   {
218     ImportedModuleMap::iterator mi = m_imported_modules.find(imported_module);
219 
220     if (mi != m_imported_modules.end()) {
221       if (exported_modules) {
222         ReportModuleExports(*exported_modules, mi->second);
223       }
224       return true;
225     }
226   }
227 
228   if (!m_compiler_instance->getPreprocessor()
229            .getHeaderSearchInfo()
230            .lookupModule(path[0].GetStringRef())) {
231     error_stream.Printf("error: Header search couldn't locate module %s\n",
232                         path[0].AsCString());
233     return false;
234   }
235 
236   llvm::SmallVector<std::pair<clang::IdentifierInfo *, clang::SourceLocation>,
237                     4>
238       clang_path;
239 
240   {
241     clang::SourceManager &source_manager =
242         m_compiler_instance->getASTContext().getSourceManager();
243 
244     for (ConstString path_component : path) {
245       clang_path.push_back(std::make_pair(
246           &m_compiler_instance->getASTContext().Idents.get(
247               path_component.GetStringRef()),
248           source_manager.getLocForStartOfFile(source_manager.getMainFileID())
249               .getLocWithOffset(m_source_location_index++)));
250     }
251   }
252 
253   StoringDiagnosticConsumer *diagnostic_consumer =
254       static_cast<StoringDiagnosticConsumer *>(
255           m_compiler_instance->getDiagnostics().getClient());
256 
257   diagnostic_consumer->ClearDiagnostics();
258 
259   clang::Module *top_level_module = DoGetModule(clang_path.front(), false);
260 
261   if (!top_level_module) {
262     diagnostic_consumer->DumpDiagnostics(error_stream);
263     error_stream.Printf("error: Couldn't load top-level module %s\n",
264                         path[0].AsCString());
265     return false;
266   }
267 
268   clang::Module *submodule = top_level_module;
269 
270   for (size_t ci = 1; ci < path.size(); ++ci) {
271     llvm::StringRef component = path[ci].GetStringRef();
272     submodule = submodule->findSubmodule(component.str());
273     if (!submodule) {
274       diagnostic_consumer->DumpDiagnostics(error_stream);
275       error_stream.Printf("error: Couldn't load submodule %s\n",
276                           component.str().c_str());
277       return false;
278     }
279   }
280 
281   clang::Module *requested_module = DoGetModule(clang_path, true);
282 
283   if (requested_module != nullptr) {
284     if (exported_modules) {
285       ReportModuleExports(*exported_modules, requested_module);
286     }
287 
288     m_imported_modules[imported_module] = requested_module;
289 
290     m_enabled = true;
291 
292     return true;
293   }
294 
295   return false;
296 }
297 
298 bool ClangModulesDeclVendor::LanguageSupportsClangModules(
299     lldb::LanguageType language) {
300   switch (language) {
301   default:
302     return false;
303   // C++ and friends to be added
304   case lldb::LanguageType::eLanguageTypeC:
305   case lldb::LanguageType::eLanguageTypeC11:
306   case lldb::LanguageType::eLanguageTypeC89:
307   case lldb::LanguageType::eLanguageTypeC99:
308   case lldb::LanguageType::eLanguageTypeObjC:
309     return true;
310   }
311 }
312 
313 bool ClangModulesDeclVendorImpl::AddModulesForCompileUnit(
314     CompileUnit &cu, ClangModulesDeclVendor::ModuleVector &exported_modules,
315     Stream &error_stream) {
316   if (LanguageSupportsClangModules(cu.GetLanguage())) {
317     std::vector<ConstString> imported_modules = cu.GetImportedModules();
318 
319     for (ConstString imported_module : imported_modules) {
320       std::vector<ConstString> path;
321 
322       path.push_back(imported_module);
323 
324       if (!AddModule(path, &exported_modules, error_stream)) {
325         return false;
326       }
327     }
328 
329     return true;
330   }
331 
332   return true;
333 }
334 
335 // ClangImporter::lookupValue
336 
337 uint32_t
338 ClangModulesDeclVendorImpl::FindDecls(const ConstString &name, bool append,
339                                       uint32_t max_matches,
340                                       std::vector<clang::NamedDecl *> &decls) {
341   if (!m_enabled) {
342     return 0;
343   }
344 
345   if (!append)
346     decls.clear();
347 
348   clang::IdentifierInfo &ident =
349       m_compiler_instance->getASTContext().Idents.get(name.GetStringRef());
350 
351   clang::LookupResult lookup_result(
352       m_compiler_instance->getSema(), clang::DeclarationName(&ident),
353       clang::SourceLocation(), clang::Sema::LookupOrdinaryName);
354 
355   m_compiler_instance->getSema().LookupName(
356       lookup_result,
357       m_compiler_instance->getSema().getScopeForContext(
358           m_compiler_instance->getASTContext().getTranslationUnitDecl()));
359 
360   uint32_t num_matches = 0;
361 
362   for (clang::NamedDecl *named_decl : lookup_result) {
363     if (num_matches >= max_matches)
364       return num_matches;
365 
366     decls.push_back(named_decl);
367     ++num_matches;
368   }
369 
370   return num_matches;
371 }
372 
373 void ClangModulesDeclVendorImpl::ForEachMacro(
374     const ClangModulesDeclVendor::ModuleVector &modules,
375     std::function<bool(const std::string &)> handler) {
376   if (!m_enabled) {
377     return;
378   }
379 
380   typedef std::map<ModuleID, ssize_t> ModulePriorityMap;
381   ModulePriorityMap module_priorities;
382 
383   ssize_t priority = 0;
384 
385   for (ModuleID module : modules) {
386     module_priorities[module] = priority++;
387   }
388 
389   if (m_compiler_instance->getPreprocessor().getExternalSource()) {
390     m_compiler_instance->getPreprocessor()
391         .getExternalSource()
392         ->ReadDefinedMacros();
393   }
394 
395   for (clang::Preprocessor::macro_iterator
396            mi = m_compiler_instance->getPreprocessor().macro_begin(),
397            me = m_compiler_instance->getPreprocessor().macro_end();
398        mi != me; ++mi) {
399     const clang::IdentifierInfo *ii = nullptr;
400 
401     {
402       if (clang::IdentifierInfoLookup *lookup =
403               m_compiler_instance->getPreprocessor()
404                   .getIdentifierTable()
405                   .getExternalIdentifierLookup()) {
406         lookup->get(mi->first->getName());
407       }
408       if (!ii) {
409         ii = mi->first;
410       }
411     }
412 
413     ssize_t found_priority = -1;
414     clang::MacroInfo *macro_info = nullptr;
415 
416     for (clang::ModuleMacro *module_macro :
417          m_compiler_instance->getPreprocessor().getLeafModuleMacros(ii)) {
418       clang::Module *module = module_macro->getOwningModule();
419 
420       {
421         ModulePriorityMap::iterator pi =
422             module_priorities.find(reinterpret_cast<ModuleID>(module));
423 
424         if (pi != module_priorities.end() && pi->second > found_priority) {
425           macro_info = module_macro->getMacroInfo();
426           found_priority = pi->second;
427         }
428       }
429 
430       clang::Module *top_level_module = module->getTopLevelModule();
431 
432       if (top_level_module != module) {
433         ModulePriorityMap::iterator pi = module_priorities.find(
434             reinterpret_cast<ModuleID>(top_level_module));
435 
436         if ((pi != module_priorities.end()) && pi->second > found_priority) {
437           macro_info = module_macro->getMacroInfo();
438           found_priority = pi->second;
439         }
440       }
441     }
442 
443     if (macro_info) {
444       std::string macro_expansion = "#define ";
445       macro_expansion.append(mi->first->getName().str());
446 
447       {
448         if (macro_info->isFunctionLike()) {
449           macro_expansion.append("(");
450 
451           bool first_arg = true;
452 
453           for (clang::MacroInfo::arg_iterator ai = macro_info->arg_begin(),
454                                               ae = macro_info->arg_end();
455                ai != ae; ++ai) {
456             if (!first_arg) {
457               macro_expansion.append(", ");
458             } else {
459               first_arg = false;
460             }
461 
462             macro_expansion.append((*ai)->getName().str());
463           }
464 
465           if (macro_info->isC99Varargs()) {
466             if (first_arg) {
467               macro_expansion.append("...");
468             } else {
469               macro_expansion.append(", ...");
470             }
471           } else if (macro_info->isGNUVarargs()) {
472             macro_expansion.append("...");
473           }
474 
475           macro_expansion.append(")");
476         }
477 
478         macro_expansion.append(" ");
479 
480         bool first_token = true;
481 
482         for (clang::MacroInfo::tokens_iterator ti = macro_info->tokens_begin(),
483                                                te = macro_info->tokens_end();
484              ti != te; ++ti) {
485           if (!first_token) {
486             macro_expansion.append(" ");
487           } else {
488             first_token = false;
489           }
490 
491           if (ti->isLiteral()) {
492             if (const char *literal_data = ti->getLiteralData()) {
493               std::string token_str(literal_data, ti->getLength());
494               macro_expansion.append(token_str);
495             } else {
496               bool invalid = false;
497               const char *literal_source =
498                   m_compiler_instance->getSourceManager().getCharacterData(
499                       ti->getLocation(), &invalid);
500 
501               if (invalid) {
502                 lldbassert(0 && "Unhandled token kind");
503                 macro_expansion.append("<unknown literal value>");
504               } else {
505                 macro_expansion.append(
506                     std::string(literal_source, ti->getLength()));
507               }
508             }
509           } else if (const char *punctuator_spelling =
510                          clang::tok::getPunctuatorSpelling(ti->getKind())) {
511             macro_expansion.append(punctuator_spelling);
512           } else if (const char *keyword_spelling =
513                          clang::tok::getKeywordSpelling(ti->getKind())) {
514             macro_expansion.append(keyword_spelling);
515           } else {
516             switch (ti->getKind()) {
517             case clang::tok::TokenKind::identifier:
518               macro_expansion.append(ti->getIdentifierInfo()->getName().str());
519               break;
520             case clang::tok::TokenKind::raw_identifier:
521               macro_expansion.append(ti->getRawIdentifier().str());
522               break;
523             default:
524               macro_expansion.append(ti->getName());
525               break;
526             }
527           }
528         }
529 
530         if (handler(macro_expansion)) {
531           return;
532         }
533       }
534     }
535   }
536 }
537 
538 clang::ModuleLoadResult
539 ClangModulesDeclVendorImpl::DoGetModule(clang::ModuleIdPath path,
540                                         bool make_visible) {
541   clang::Module::NameVisibilityKind visibility =
542       make_visible ? clang::Module::AllVisible : clang::Module::Hidden;
543 
544   const bool is_inclusion_directive = false;
545 
546   return m_compiler_instance->loadModule(path.front().second, path, visibility,
547                                          is_inclusion_directive);
548 }
549 
550 static const char *ModuleImportBufferName = "LLDBModulesMemoryBuffer";
551 
552 lldb_private::ClangModulesDeclVendor *
553 ClangModulesDeclVendor::Create(Target &target) {
554   // FIXME we should insure programmatically that the expression parser's
555   // compiler and the modules runtime's
556   // compiler are both initialized in the same way – preferably by the same
557   // code.
558 
559   if (!target.GetPlatform()->SupportsModules())
560     return nullptr;
561 
562   const ArchSpec &arch = target.GetArchitecture();
563 
564   std::vector<std::string> compiler_invocation_arguments = {
565       "clang",
566       "-fmodules",
567       "-fimplicit-module-maps",
568       "-fcxx-modules",
569       "-fsyntax-only",
570       "-femit-all-decls",
571       "-target",
572       arch.GetTriple().str(),
573       "-fmodules-validate-system-headers",
574       "-Werror=non-modular-include-in-framework-module"};
575 
576   target.GetPlatform()->AddClangModuleCompilationOptions(
577       &target, compiler_invocation_arguments);
578 
579   compiler_invocation_arguments.push_back(ModuleImportBufferName);
580 
581   // Add additional search paths with { "-I", path } or { "-F", path } here.
582 
583   {
584     llvm::SmallString<128> DefaultModuleCache;
585     const bool erased_on_reboot = false;
586     llvm::sys::path::system_temp_directory(erased_on_reboot,
587                                            DefaultModuleCache);
588     llvm::sys::path::append(DefaultModuleCache, "org.llvm.clang");
589     llvm::sys::path::append(DefaultModuleCache, "ModuleCache");
590     std::string module_cache_argument("-fmodules-cache-path=");
591     module_cache_argument.append(DefaultModuleCache.str().str());
592     compiler_invocation_arguments.push_back(module_cache_argument);
593   }
594 
595   FileSpecList &module_search_paths = target.GetClangModuleSearchPaths();
596 
597   for (size_t spi = 0, spe = module_search_paths.GetSize(); spi < spe; ++spi) {
598     const FileSpec &search_path = module_search_paths.GetFileSpecAtIndex(spi);
599 
600     std::string search_path_argument = "-I";
601     search_path_argument.append(search_path.GetPath());
602 
603     compiler_invocation_arguments.push_back(search_path_argument);
604   }
605 
606   {
607     FileSpec clang_resource_dir = GetResourceDir();
608 
609     if (clang_resource_dir.IsDirectory()) {
610       compiler_invocation_arguments.push_back("-resource-dir");
611       compiler_invocation_arguments.push_back(clang_resource_dir.GetPath());
612     }
613   }
614 
615   llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_engine =
616       clang::CompilerInstance::createDiagnostics(new clang::DiagnosticOptions,
617                                                  new StoringDiagnosticConsumer);
618 
619   std::vector<const char *> compiler_invocation_argument_cstrs;
620 
621   for (const std::string &arg : compiler_invocation_arguments) {
622     compiler_invocation_argument_cstrs.push_back(arg.c_str());
623   }
624 
625   std::shared_ptr<clang::CompilerInvocation> invocation =
626       clang::createInvocationFromCommandLine(compiler_invocation_argument_cstrs,
627                                              diagnostics_engine);
628 
629   if (!invocation)
630     return nullptr;
631 
632   std::unique_ptr<llvm::MemoryBuffer> source_buffer =
633       llvm::MemoryBuffer::getMemBuffer(
634           "extern int __lldb __attribute__((unavailable));",
635           ModuleImportBufferName);
636 
637   invocation->getPreprocessorOpts().addRemappedFile(ModuleImportBufferName,
638                                                     source_buffer.release());
639 
640   std::unique_ptr<clang::CompilerInstance> instance(
641       new clang::CompilerInstance);
642 
643   instance->setDiagnostics(diagnostics_engine.get());
644   instance->setInvocation(invocation);
645 
646   std::unique_ptr<clang::FrontendAction> action(new clang::SyntaxOnlyAction);
647 
648   instance->setTarget(clang::TargetInfo::CreateTargetInfo(
649       *diagnostics_engine, instance->getInvocation().TargetOpts));
650 
651   if (!instance->hasTarget())
652     return nullptr;
653 
654   instance->getTarget().adjust(instance->getLangOpts());
655 
656   if (!action->BeginSourceFile(*instance,
657                                instance->getFrontendOpts().Inputs[0]))
658     return nullptr;
659 
660   instance->getPreprocessor().enableIncrementalProcessing();
661 
662   instance->createModuleManager();
663 
664   instance->createSema(action->getTranslationUnitKind(), nullptr);
665 
666   const bool skipFunctionBodies = false;
667   std::unique_ptr<clang::Parser> parser(new clang::Parser(
668       instance->getPreprocessor(), instance->getSema(), skipFunctionBodies));
669 
670   instance->getPreprocessor().EnterMainSourceFile();
671   parser->Initialize();
672 
673   clang::Parser::DeclGroupPtrTy parsed;
674 
675   while (!parser->ParseTopLevelDecl(parsed))
676     ;
677 
678   return new ClangModulesDeclVendorImpl(std::move(diagnostics_engine),
679                                         std::move(invocation),
680                                         std::move(instance), std::move(parser));
681 }
682