1 //===-- ClangModulesDeclVendor.cpp ------------------------------*- C++ -*-===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 // C Includes
11 // C++ Includes
12 #include <mutex>
13 
14 // Other libraries and framework includes
15 #include "clang/Basic/TargetInfo.h"
16 #include "clang/Frontend/CompilerInstance.h"
17 #include "clang/Frontend/FrontendActions.h"
18 #include "clang/Lex/Preprocessor.h"
19 #include "clang/Lex/PreprocessorOptions.h"
20 #include "clang/Parse/Parser.h"
21 #include "clang/Sema/Lookup.h"
22 #include "clang/Serialization/ASTReader.h"
23 #include "llvm/Support/FileSystem.h"
24 #include "llvm/Support/Path.h"
25 #include "llvm/Support/Threading.h"
26 
27 // Project includes
28 #include "ClangModulesDeclVendor.h"
29 
30 #include "lldb/Host/Host.h"
31 #include "lldb/Host/HostInfo.h"
32 #include "lldb/Symbol/CompileUnit.h"
33 #include "lldb/Target/Target.h"
34 #include "lldb/Utility/FileSpec.h"
35 #include "lldb/Utility/LLDBAssert.h"
36 #include "lldb/Utility/Log.h"
37 #include "lldb/Utility/StreamString.h"
38 
39 using namespace lldb_private;
40 
41 namespace {
42 // Any Clang compiler requires a consumer for diagnostics.  This one stores them
43 // as strings
44 // so we can provide them to the user in case a module failed to load.
45 class StoringDiagnosticConsumer : public clang::DiagnosticConsumer {
46 public:
47   StoringDiagnosticConsumer();
48 
49   void HandleDiagnostic(clang::DiagnosticsEngine::Level DiagLevel,
50                         const clang::Diagnostic &info) override;
51 
52   void ClearDiagnostics();
53 
54   void DumpDiagnostics(Stream &error_stream);
55 
56 private:
57   typedef std::pair<clang::DiagnosticsEngine::Level, std::string>
58       IDAndDiagnostic;
59   std::vector<IDAndDiagnostic> m_diagnostics;
60   Log *m_log;
61 };
62 
63 // The private implementation of our ClangModulesDeclVendor.  Contains all the
64 // Clang state required
65 // to load modules.
66 class ClangModulesDeclVendorImpl : public ClangModulesDeclVendor {
67 public:
68   ClangModulesDeclVendorImpl(
69       llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_engine,
70       std::shared_ptr<clang::CompilerInvocation> compiler_invocation,
71       std::unique_ptr<clang::CompilerInstance> compiler_instance,
72       std::unique_ptr<clang::Parser> parser);
73 
74   ~ClangModulesDeclVendorImpl() override = default;
75 
76   bool AddModule(ModulePath &path, ModuleVector *exported_modules,
77                  Stream &error_stream) override;
78 
79   bool AddModulesForCompileUnit(CompileUnit &cu, ModuleVector &exported_modules,
80                                 Stream &error_stream) override;
81 
82   uint32_t FindDecls(const ConstString &name, bool append, uint32_t max_matches,
83                      std::vector<clang::NamedDecl *> &decls) override;
84 
85   void ForEachMacro(const ModuleVector &modules,
86                     std::function<bool(const std::string &)> handler) override;
87 
88 private:
89   void
90   ReportModuleExportsHelper(std::set<ClangModulesDeclVendor::ModuleID> &exports,
91                             clang::Module *module);
92 
93   void ReportModuleExports(ModuleVector &exports, clang::Module *module);
94 
95   clang::ModuleLoadResult DoGetModule(clang::ModuleIdPath path,
96                                       bool make_visible);
97 
98   bool m_enabled = false;
99 
100   llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> m_diagnostics_engine;
101   std::shared_ptr<clang::CompilerInvocation> m_compiler_invocation;
102   std::unique_ptr<clang::CompilerInstance> m_compiler_instance;
103   std::unique_ptr<clang::Parser> m_parser;
104   size_t m_source_location_index =
105       0; // used to give name components fake SourceLocations
106 
107   typedef std::vector<ConstString> ImportedModule;
108   typedef std::map<ImportedModule, clang::Module *> ImportedModuleMap;
109   typedef std::set<ModuleID> ImportedModuleSet;
110   ImportedModuleMap m_imported_modules;
111   ImportedModuleSet m_user_imported_modules;
112 };
113 } // anonymous namespace
114 
115 StoringDiagnosticConsumer::StoringDiagnosticConsumer() {
116   m_log = lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_EXPRESSIONS);
117 }
118 
119 void StoringDiagnosticConsumer::HandleDiagnostic(
120     clang::DiagnosticsEngine::Level DiagLevel, const clang::Diagnostic &info) {
121   llvm::SmallVector<char, 256> diagnostic_string;
122 
123   info.FormatDiagnostic(diagnostic_string);
124 
125   m_diagnostics.push_back(
126       IDAndDiagnostic(DiagLevel, std::string(diagnostic_string.data(),
127                                              diagnostic_string.size())));
128 }
129 
130 void StoringDiagnosticConsumer::ClearDiagnostics() { m_diagnostics.clear(); }
131 
132 void StoringDiagnosticConsumer::DumpDiagnostics(Stream &error_stream) {
133   for (IDAndDiagnostic &diag : m_diagnostics) {
134     switch (diag.first) {
135     default:
136       error_stream.PutCString(diag.second);
137       error_stream.PutChar('\n');
138       break;
139     case clang::DiagnosticsEngine::Level::Ignored:
140       break;
141     }
142   }
143 }
144 
145 static FileSpec GetResourceDir() {
146   static FileSpec g_cached_resource_dir;
147 
148   static llvm::once_flag g_once_flag;
149 
150   llvm::call_once(g_once_flag, []() {
151     HostInfo::GetLLDBPath(lldb::ePathTypeClangDir, g_cached_resource_dir);
152   });
153 
154   return g_cached_resource_dir;
155 }
156 
157 ClangModulesDeclVendor::ClangModulesDeclVendor() {}
158 
159 ClangModulesDeclVendor::~ClangModulesDeclVendor() {}
160 
161 ClangModulesDeclVendorImpl::ClangModulesDeclVendorImpl(
162     llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_engine,
163     std::shared_ptr<clang::CompilerInvocation> compiler_invocation,
164     std::unique_ptr<clang::CompilerInstance> compiler_instance,
165     std::unique_ptr<clang::Parser> parser)
166     : m_diagnostics_engine(std::move(diagnostics_engine)),
167       m_compiler_invocation(std::move(compiler_invocation)),
168       m_compiler_instance(std::move(compiler_instance)),
169       m_parser(std::move(parser)) {}
170 
171 void ClangModulesDeclVendorImpl::ReportModuleExportsHelper(
172     std::set<ClangModulesDeclVendor::ModuleID> &exports,
173     clang::Module *module) {
174   if (exports.count(reinterpret_cast<ClangModulesDeclVendor::ModuleID>(module)))
175     return;
176 
177   exports.insert(reinterpret_cast<ClangModulesDeclVendor::ModuleID>(module));
178 
179   llvm::SmallVector<clang::Module *, 2> sub_exports;
180 
181   module->getExportedModules(sub_exports);
182 
183   for (clang::Module *module : sub_exports) {
184     ReportModuleExportsHelper(exports, module);
185   }
186 }
187 
188 void ClangModulesDeclVendorImpl::ReportModuleExports(
189     ClangModulesDeclVendor::ModuleVector &exports, clang::Module *module) {
190   std::set<ClangModulesDeclVendor::ModuleID> exports_set;
191 
192   ReportModuleExportsHelper(exports_set, module);
193 
194   for (ModuleID module : exports_set) {
195     exports.push_back(module);
196   }
197 }
198 
199 bool ClangModulesDeclVendorImpl::AddModule(ModulePath &path,
200                                            ModuleVector *exported_modules,
201                                            Stream &error_stream) {
202   // Fail early.
203 
204   if (m_compiler_instance->hadModuleLoaderFatalFailure()) {
205     error_stream.PutCString("error: Couldn't load a module because the module "
206                             "loader is in a fatal state.\n");
207     return false;
208   }
209 
210   // Check if we've already imported this module.
211 
212   std::vector<ConstString> imported_module;
213 
214   for (ConstString path_component : path) {
215     imported_module.push_back(path_component);
216   }
217 
218   {
219     ImportedModuleMap::iterator mi = m_imported_modules.find(imported_module);
220 
221     if (mi != m_imported_modules.end()) {
222       if (exported_modules) {
223         ReportModuleExports(*exported_modules, mi->second);
224       }
225       return true;
226     }
227   }
228 
229   if (!m_compiler_instance->getPreprocessor()
230            .getHeaderSearchInfo()
231            .lookupModule(path[0].GetStringRef())) {
232     error_stream.Printf("error: Header search couldn't locate module %s\n",
233                         path[0].AsCString());
234     return false;
235   }
236 
237   llvm::SmallVector<std::pair<clang::IdentifierInfo *, clang::SourceLocation>,
238                     4>
239       clang_path;
240 
241   {
242     clang::SourceManager &source_manager =
243         m_compiler_instance->getASTContext().getSourceManager();
244 
245     for (ConstString path_component : path) {
246       clang_path.push_back(std::make_pair(
247           &m_compiler_instance->getASTContext().Idents.get(
248               path_component.GetStringRef()),
249           source_manager.getLocForStartOfFile(source_manager.getMainFileID())
250               .getLocWithOffset(m_source_location_index++)));
251     }
252   }
253 
254   StoringDiagnosticConsumer *diagnostic_consumer =
255       static_cast<StoringDiagnosticConsumer *>(
256           m_compiler_instance->getDiagnostics().getClient());
257 
258   diagnostic_consumer->ClearDiagnostics();
259 
260   clang::Module *top_level_module = DoGetModule(clang_path.front(), false);
261 
262   if (!top_level_module) {
263     diagnostic_consumer->DumpDiagnostics(error_stream);
264     error_stream.Printf("error: Couldn't load top-level module %s\n",
265                         path[0].AsCString());
266     return false;
267   }
268 
269   clang::Module *submodule = top_level_module;
270 
271   for (size_t ci = 1; ci < path.size(); ++ci) {
272     llvm::StringRef component = path[ci].GetStringRef();
273     submodule = submodule->findSubmodule(component.str());
274     if (!submodule) {
275       diagnostic_consumer->DumpDiagnostics(error_stream);
276       error_stream.Printf("error: Couldn't load submodule %s\n",
277                           component.str().c_str());
278       return false;
279     }
280   }
281 
282   clang::Module *requested_module = DoGetModule(clang_path, true);
283 
284   if (requested_module != nullptr) {
285     if (exported_modules) {
286       ReportModuleExports(*exported_modules, requested_module);
287     }
288 
289     m_imported_modules[imported_module] = requested_module;
290 
291     m_enabled = true;
292 
293     return true;
294   }
295 
296   return false;
297 }
298 
299 bool ClangModulesDeclVendor::LanguageSupportsClangModules(
300     lldb::LanguageType language) {
301   switch (language) {
302   default:
303     return false;
304   // C++ and friends to be added
305   case lldb::LanguageType::eLanguageTypeC:
306   case lldb::LanguageType::eLanguageTypeC11:
307   case lldb::LanguageType::eLanguageTypeC89:
308   case lldb::LanguageType::eLanguageTypeC99:
309   case lldb::LanguageType::eLanguageTypeObjC:
310     return true;
311   }
312 }
313 
314 bool ClangModulesDeclVendorImpl::AddModulesForCompileUnit(
315     CompileUnit &cu, ClangModulesDeclVendor::ModuleVector &exported_modules,
316     Stream &error_stream) {
317   if (LanguageSupportsClangModules(cu.GetLanguage())) {
318     std::vector<ConstString> imported_modules = cu.GetImportedModules();
319 
320     for (ConstString imported_module : imported_modules) {
321       std::vector<ConstString> path;
322 
323       path.push_back(imported_module);
324 
325       if (!AddModule(path, &exported_modules, error_stream)) {
326         return false;
327       }
328     }
329 
330     return true;
331   }
332 
333   return true;
334 }
335 
336 // ClangImporter::lookupValue
337 
338 uint32_t
339 ClangModulesDeclVendorImpl::FindDecls(const ConstString &name, bool append,
340                                       uint32_t max_matches,
341                                       std::vector<clang::NamedDecl *> &decls) {
342   if (!m_enabled) {
343     return 0;
344   }
345 
346   if (!append)
347     decls.clear();
348 
349   clang::IdentifierInfo &ident =
350       m_compiler_instance->getASTContext().Idents.get(name.GetStringRef());
351 
352   clang::LookupResult lookup_result(
353       m_compiler_instance->getSema(), clang::DeclarationName(&ident),
354       clang::SourceLocation(), clang::Sema::LookupOrdinaryName);
355 
356   m_compiler_instance->getSema().LookupName(
357       lookup_result,
358       m_compiler_instance->getSema().getScopeForContext(
359           m_compiler_instance->getASTContext().getTranslationUnitDecl()));
360 
361   uint32_t num_matches = 0;
362 
363   for (clang::NamedDecl *named_decl : lookup_result) {
364     if (num_matches >= max_matches)
365       return num_matches;
366 
367     decls.push_back(named_decl);
368     ++num_matches;
369   }
370 
371   return num_matches;
372 }
373 
374 void ClangModulesDeclVendorImpl::ForEachMacro(
375     const ClangModulesDeclVendor::ModuleVector &modules,
376     std::function<bool(const std::string &)> handler) {
377   if (!m_enabled) {
378     return;
379   }
380 
381   typedef std::map<ModuleID, ssize_t> ModulePriorityMap;
382   ModulePriorityMap module_priorities;
383 
384   ssize_t priority = 0;
385 
386   for (ModuleID module : modules) {
387     module_priorities[module] = priority++;
388   }
389 
390   if (m_compiler_instance->getPreprocessor().getExternalSource()) {
391     m_compiler_instance->getPreprocessor()
392         .getExternalSource()
393         ->ReadDefinedMacros();
394   }
395 
396   for (clang::Preprocessor::macro_iterator
397            mi = m_compiler_instance->getPreprocessor().macro_begin(),
398            me = m_compiler_instance->getPreprocessor().macro_end();
399        mi != me; ++mi) {
400     const clang::IdentifierInfo *ii = nullptr;
401 
402     {
403       if (clang::IdentifierInfoLookup *lookup =
404               m_compiler_instance->getPreprocessor()
405                   .getIdentifierTable()
406                   .getExternalIdentifierLookup()) {
407         lookup->get(mi->first->getName());
408       }
409       if (!ii) {
410         ii = mi->first;
411       }
412     }
413 
414     ssize_t found_priority = -1;
415     clang::MacroInfo *macro_info = nullptr;
416 
417     for (clang::ModuleMacro *module_macro :
418          m_compiler_instance->getPreprocessor().getLeafModuleMacros(ii)) {
419       clang::Module *module = module_macro->getOwningModule();
420 
421       {
422         ModulePriorityMap::iterator pi =
423             module_priorities.find(reinterpret_cast<ModuleID>(module));
424 
425         if (pi != module_priorities.end() && pi->second > found_priority) {
426           macro_info = module_macro->getMacroInfo();
427           found_priority = pi->second;
428         }
429       }
430 
431       clang::Module *top_level_module = module->getTopLevelModule();
432 
433       if (top_level_module != module) {
434         ModulePriorityMap::iterator pi = module_priorities.find(
435             reinterpret_cast<ModuleID>(top_level_module));
436 
437         if ((pi != module_priorities.end()) && pi->second > found_priority) {
438           macro_info = module_macro->getMacroInfo();
439           found_priority = pi->second;
440         }
441       }
442     }
443 
444     if (macro_info) {
445       std::string macro_expansion = "#define ";
446       macro_expansion.append(mi->first->getName().str());
447 
448       {
449         if (macro_info->isFunctionLike()) {
450           macro_expansion.append("(");
451 
452           bool first_arg = true;
453 
454           for (clang::MacroInfo::arg_iterator ai = macro_info->arg_begin(),
455                                               ae = macro_info->arg_end();
456                ai != ae; ++ai) {
457             if (!first_arg) {
458               macro_expansion.append(", ");
459             } else {
460               first_arg = false;
461             }
462 
463             macro_expansion.append((*ai)->getName().str());
464           }
465 
466           if (macro_info->isC99Varargs()) {
467             if (first_arg) {
468               macro_expansion.append("...");
469             } else {
470               macro_expansion.append(", ...");
471             }
472           } else if (macro_info->isGNUVarargs()) {
473             macro_expansion.append("...");
474           }
475 
476           macro_expansion.append(")");
477         }
478 
479         macro_expansion.append(" ");
480 
481         bool first_token = true;
482 
483         for (clang::MacroInfo::tokens_iterator ti = macro_info->tokens_begin(),
484                                                te = macro_info->tokens_end();
485              ti != te; ++ti) {
486           if (!first_token) {
487             macro_expansion.append(" ");
488           } else {
489             first_token = false;
490           }
491 
492           if (ti->isLiteral()) {
493             if (const char *literal_data = ti->getLiteralData()) {
494               std::string token_str(literal_data, ti->getLength());
495               macro_expansion.append(token_str);
496             } else {
497               bool invalid = false;
498               const char *literal_source =
499                   m_compiler_instance->getSourceManager().getCharacterData(
500                       ti->getLocation(), &invalid);
501 
502               if (invalid) {
503                 lldbassert(0 && "Unhandled token kind");
504                 macro_expansion.append("<unknown literal value>");
505               } else {
506                 macro_expansion.append(
507                     std::string(literal_source, ti->getLength()));
508               }
509             }
510           } else if (const char *punctuator_spelling =
511                          clang::tok::getPunctuatorSpelling(ti->getKind())) {
512             macro_expansion.append(punctuator_spelling);
513           } else if (const char *keyword_spelling =
514                          clang::tok::getKeywordSpelling(ti->getKind())) {
515             macro_expansion.append(keyword_spelling);
516           } else {
517             switch (ti->getKind()) {
518             case clang::tok::TokenKind::identifier:
519               macro_expansion.append(ti->getIdentifierInfo()->getName().str());
520               break;
521             case clang::tok::TokenKind::raw_identifier:
522               macro_expansion.append(ti->getRawIdentifier().str());
523               break;
524             default:
525               macro_expansion.append(ti->getName());
526               break;
527             }
528           }
529         }
530 
531         if (handler(macro_expansion)) {
532           return;
533         }
534       }
535     }
536   }
537 }
538 
539 clang::ModuleLoadResult
540 ClangModulesDeclVendorImpl::DoGetModule(clang::ModuleIdPath path,
541                                         bool make_visible) {
542   clang::Module::NameVisibilityKind visibility =
543       make_visible ? clang::Module::AllVisible : clang::Module::Hidden;
544 
545   const bool is_inclusion_directive = false;
546 
547   return m_compiler_instance->loadModule(path.front().second, path, visibility,
548                                          is_inclusion_directive);
549 }
550 
551 static const char *ModuleImportBufferName = "LLDBModulesMemoryBuffer";
552 
553 lldb_private::ClangModulesDeclVendor *
554 ClangModulesDeclVendor::Create(Target &target) {
555   // FIXME we should insure programmatically that the expression parser's
556   // compiler and the modules runtime's
557   // compiler are both initialized in the same way – preferably by the same
558   // code.
559 
560   if (!target.GetPlatform()->SupportsModules())
561     return nullptr;
562 
563   const ArchSpec &arch = target.GetArchitecture();
564 
565   std::vector<std::string> compiler_invocation_arguments = {
566       "clang",
567       "-fmodules",
568       "-fimplicit-module-maps",
569       "-fcxx-modules",
570       "-fsyntax-only",
571       "-femit-all-decls",
572       "-target",
573       arch.GetTriple().str(),
574       "-fmodules-validate-system-headers",
575       "-Werror=non-modular-include-in-framework-module"};
576 
577   target.GetPlatform()->AddClangModuleCompilationOptions(
578       &target, compiler_invocation_arguments);
579 
580   compiler_invocation_arguments.push_back(ModuleImportBufferName);
581 
582   // Add additional search paths with { "-I", path } or { "-F", path } here.
583 
584   {
585     llvm::SmallString<128> DefaultModuleCache;
586     const bool erased_on_reboot = false;
587     llvm::sys::path::system_temp_directory(erased_on_reboot,
588                                            DefaultModuleCache);
589     llvm::sys::path::append(DefaultModuleCache, "org.llvm.clang");
590     llvm::sys::path::append(DefaultModuleCache, "ModuleCache");
591     std::string module_cache_argument("-fmodules-cache-path=");
592     module_cache_argument.append(DefaultModuleCache.str().str());
593     compiler_invocation_arguments.push_back(module_cache_argument);
594   }
595 
596   FileSpecList &module_search_paths = target.GetClangModuleSearchPaths();
597 
598   for (size_t spi = 0, spe = module_search_paths.GetSize(); spi < spe; ++spi) {
599     const FileSpec &search_path = module_search_paths.GetFileSpecAtIndex(spi);
600 
601     std::string search_path_argument = "-I";
602     search_path_argument.append(search_path.GetPath());
603 
604     compiler_invocation_arguments.push_back(search_path_argument);
605   }
606 
607   {
608     FileSpec clang_resource_dir = GetResourceDir();
609 
610     if (llvm::sys::fs::is_directory(clang_resource_dir.GetPath())) {
611       compiler_invocation_arguments.push_back("-resource-dir");
612       compiler_invocation_arguments.push_back(clang_resource_dir.GetPath());
613     }
614   }
615 
616   llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_engine =
617       clang::CompilerInstance::createDiagnostics(new clang::DiagnosticOptions,
618                                                  new StoringDiagnosticConsumer);
619 
620   std::vector<const char *> compiler_invocation_argument_cstrs;
621 
622   for (const std::string &arg : compiler_invocation_arguments) {
623     compiler_invocation_argument_cstrs.push_back(arg.c_str());
624   }
625 
626   std::shared_ptr<clang::CompilerInvocation> invocation =
627       clang::createInvocationFromCommandLine(compiler_invocation_argument_cstrs,
628                                              diagnostics_engine);
629 
630   if (!invocation)
631     return nullptr;
632 
633   std::unique_ptr<llvm::MemoryBuffer> source_buffer =
634       llvm::MemoryBuffer::getMemBuffer(
635           "extern int __lldb __attribute__((unavailable));",
636           ModuleImportBufferName);
637 
638   invocation->getPreprocessorOpts().addRemappedFile(ModuleImportBufferName,
639                                                     source_buffer.release());
640 
641   std::unique_ptr<clang::CompilerInstance> instance(
642       new clang::CompilerInstance);
643 
644   instance->setDiagnostics(diagnostics_engine.get());
645   instance->setInvocation(invocation);
646 
647   std::unique_ptr<clang::FrontendAction> action(new clang::SyntaxOnlyAction);
648 
649   instance->setTarget(clang::TargetInfo::CreateTargetInfo(
650       *diagnostics_engine, instance->getInvocation().TargetOpts));
651 
652   if (!instance->hasTarget())
653     return nullptr;
654 
655   instance->getTarget().adjust(instance->getLangOpts());
656 
657   if (!action->BeginSourceFile(*instance,
658                                instance->getFrontendOpts().Inputs[0]))
659     return nullptr;
660 
661   instance->getPreprocessor().enableIncrementalProcessing();
662 
663   instance->createModuleManager();
664 
665   instance->createSema(action->getTranslationUnitKind(), nullptr);
666 
667   const bool skipFunctionBodies = false;
668   std::unique_ptr<clang::Parser> parser(new clang::Parser(
669       instance->getPreprocessor(), instance->getSema(), skipFunctionBodies));
670 
671   instance->getPreprocessor().EnterMainSourceFile();
672   parser->Initialize();
673 
674   clang::Parser::DeclGroupPtrTy parsed;
675 
676   while (!parser->ParseTopLevelDecl(parsed))
677     ;
678 
679   return new ClangModulesDeclVendorImpl(std::move(diagnostics_engine),
680                                         std::move(invocation),
681                                         std::move(instance), std::move(parser));
682 }
683