1 //===-- ClangModulesDeclVendor.cpp ------------------------------*- C++ -*-===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 // C Includes
11 // C++ Includes
12 #include <mutex>
13 
14 // Other libraries and framework includes
15 #include "clang/Basic/TargetInfo.h"
16 #include "clang/Frontend/CompilerInstance.h"
17 #include "clang/Frontend/FrontendActions.h"
18 #include "clang/Lex/Preprocessor.h"
19 #include "clang/Lex/PreprocessorOptions.h"
20 #include "clang/Parse/Parser.h"
21 #include "clang/Sema/Lookup.h"
22 #include "clang/Serialization/ASTReader.h"
23 #include "llvm/Support/FileSystem.h"
24 #include "llvm/Support/Path.h"
25 #include "llvm/Support/Threading.h"
26 
27 // Project includes
28 #include "ClangHost.h"
29 #include "ClangModulesDeclVendor.h"
30 
31 #include "lldb/Core/ModuleList.h"
32 #include "lldb/Host/Host.h"
33 #include "lldb/Host/HostInfo.h"
34 #include "lldb/Symbol/CompileUnit.h"
35 #include "lldb/Target/Target.h"
36 #include "lldb/Utility/FileSpec.h"
37 #include "lldb/Utility/LLDBAssert.h"
38 #include "lldb/Utility/Log.h"
39 #include "lldb/Utility/StreamString.h"
40 
41 using namespace lldb_private;
42 
43 namespace {
44 // Any Clang compiler requires a consumer for diagnostics.  This one stores
45 // them as strings so we can provide them to the user in case a module failed
46 // to load.
47 class StoringDiagnosticConsumer : public clang::DiagnosticConsumer {
48 public:
49   StoringDiagnosticConsumer();
50 
51   void HandleDiagnostic(clang::DiagnosticsEngine::Level DiagLevel,
52                         const clang::Diagnostic &info) override;
53 
54   void ClearDiagnostics();
55 
56   void DumpDiagnostics(Stream &error_stream);
57 
58 private:
59   typedef std::pair<clang::DiagnosticsEngine::Level, std::string>
60       IDAndDiagnostic;
61   std::vector<IDAndDiagnostic> m_diagnostics;
62   Log *m_log;
63 };
64 
65 // The private implementation of our ClangModulesDeclVendor.  Contains all the
66 // Clang state required to load modules.
67 class ClangModulesDeclVendorImpl : public ClangModulesDeclVendor {
68 public:
69   ClangModulesDeclVendorImpl(
70       llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_engine,
71       std::shared_ptr<clang::CompilerInvocation> compiler_invocation,
72       std::unique_ptr<clang::CompilerInstance> compiler_instance,
73       std::unique_ptr<clang::Parser> parser);
74 
75   ~ClangModulesDeclVendorImpl() override = default;
76 
77   bool AddModule(ModulePath &path, ModuleVector *exported_modules,
78                  Stream &error_stream) override;
79 
80   bool AddModulesForCompileUnit(CompileUnit &cu, ModuleVector &exported_modules,
81                                 Stream &error_stream) override;
82 
83   uint32_t FindDecls(const ConstString &name, bool append, uint32_t max_matches,
84                      std::vector<clang::NamedDecl *> &decls) override;
85 
86   void ForEachMacro(const ModuleVector &modules,
87                     std::function<bool(const std::string &)> handler) override;
88 
89   clang::ExternalASTMerger::ImporterSource GetImporterSource() override;
90 private:
91   void
92   ReportModuleExportsHelper(std::set<ClangModulesDeclVendor::ModuleID> &exports,
93                             clang::Module *module);
94 
95   void ReportModuleExports(ModuleVector &exports, clang::Module *module);
96 
97   clang::ModuleLoadResult DoGetModule(clang::ModuleIdPath path,
98                                       bool make_visible);
99 
100   bool m_enabled = false;
101 
102   llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> m_diagnostics_engine;
103   std::shared_ptr<clang::CompilerInvocation> m_compiler_invocation;
104   std::unique_ptr<clang::CompilerInstance> m_compiler_instance;
105   std::unique_ptr<clang::Parser> m_parser;
106   size_t m_source_location_index =
107       0; // used to give name components fake SourceLocations
108 
109   typedef std::vector<ConstString> ImportedModule;
110   typedef std::map<ImportedModule, clang::Module *> ImportedModuleMap;
111   typedef std::set<ModuleID> ImportedModuleSet;
112   ImportedModuleMap m_imported_modules;
113   ImportedModuleSet m_user_imported_modules;
114   const clang::ExternalASTMerger::OriginMap m_origin_map;
115 };
116 } // anonymous namespace
117 
118 StoringDiagnosticConsumer::StoringDiagnosticConsumer() {
119   m_log = lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_EXPRESSIONS);
120 }
121 
122 void StoringDiagnosticConsumer::HandleDiagnostic(
123     clang::DiagnosticsEngine::Level DiagLevel, const clang::Diagnostic &info) {
124   llvm::SmallVector<char, 256> diagnostic_string;
125 
126   info.FormatDiagnostic(diagnostic_string);
127 
128   m_diagnostics.push_back(
129       IDAndDiagnostic(DiagLevel, std::string(diagnostic_string.data(),
130                                              diagnostic_string.size())));
131 }
132 
133 void StoringDiagnosticConsumer::ClearDiagnostics() { m_diagnostics.clear(); }
134 
135 void StoringDiagnosticConsumer::DumpDiagnostics(Stream &error_stream) {
136   for (IDAndDiagnostic &diag : m_diagnostics) {
137     switch (diag.first) {
138     default:
139       error_stream.PutCString(diag.second);
140       error_stream.PutChar('\n');
141       break;
142     case clang::DiagnosticsEngine::Level::Ignored:
143       break;
144     }
145   }
146 }
147 
148 ClangModulesDeclVendor::ClangModulesDeclVendor() {}
149 
150 ClangModulesDeclVendor::~ClangModulesDeclVendor() {}
151 
152 ClangModulesDeclVendorImpl::ClangModulesDeclVendorImpl(
153     llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_engine,
154     std::shared_ptr<clang::CompilerInvocation> compiler_invocation,
155     std::unique_ptr<clang::CompilerInstance> compiler_instance,
156     std::unique_ptr<clang::Parser> parser)
157     : m_diagnostics_engine(std::move(diagnostics_engine)),
158       m_compiler_invocation(std::move(compiler_invocation)),
159       m_compiler_instance(std::move(compiler_instance)),
160       m_parser(std::move(parser)), m_origin_map() {}
161 
162 void ClangModulesDeclVendorImpl::ReportModuleExportsHelper(
163     std::set<ClangModulesDeclVendor::ModuleID> &exports,
164     clang::Module *module) {
165   if (exports.count(reinterpret_cast<ClangModulesDeclVendor::ModuleID>(module)))
166     return;
167 
168   exports.insert(reinterpret_cast<ClangModulesDeclVendor::ModuleID>(module));
169 
170   llvm::SmallVector<clang::Module *, 2> sub_exports;
171 
172   module->getExportedModules(sub_exports);
173 
174   for (clang::Module *module : sub_exports) {
175     ReportModuleExportsHelper(exports, module);
176   }
177 }
178 
179 void ClangModulesDeclVendorImpl::ReportModuleExports(
180     ClangModulesDeclVendor::ModuleVector &exports, clang::Module *module) {
181   std::set<ClangModulesDeclVendor::ModuleID> exports_set;
182 
183   ReportModuleExportsHelper(exports_set, module);
184 
185   for (ModuleID module : exports_set) {
186     exports.push_back(module);
187   }
188 }
189 
190 bool ClangModulesDeclVendorImpl::AddModule(ModulePath &path,
191                                            ModuleVector *exported_modules,
192                                            Stream &error_stream) {
193   // Fail early.
194 
195   if (m_compiler_instance->hadModuleLoaderFatalFailure()) {
196     error_stream.PutCString("error: Couldn't load a module because the module "
197                             "loader is in a fatal state.\n");
198     return false;
199   }
200 
201   // Check if we've already imported this module.
202 
203   std::vector<ConstString> imported_module;
204 
205   for (ConstString path_component : path) {
206     imported_module.push_back(path_component);
207   }
208 
209   {
210     ImportedModuleMap::iterator mi = m_imported_modules.find(imported_module);
211 
212     if (mi != m_imported_modules.end()) {
213       if (exported_modules) {
214         ReportModuleExports(*exported_modules, mi->second);
215       }
216       return true;
217     }
218   }
219 
220   if (!m_compiler_instance->getPreprocessor()
221            .getHeaderSearchInfo()
222            .lookupModule(path[0].GetStringRef())) {
223     error_stream.Printf("error: Header search couldn't locate module %s\n",
224                         path[0].AsCString());
225     return false;
226   }
227 
228   llvm::SmallVector<std::pair<clang::IdentifierInfo *, clang::SourceLocation>,
229                     4>
230       clang_path;
231 
232   {
233     clang::SourceManager &source_manager =
234         m_compiler_instance->getASTContext().getSourceManager();
235 
236     for (ConstString path_component : path) {
237       clang_path.push_back(std::make_pair(
238           &m_compiler_instance->getASTContext().Idents.get(
239               path_component.GetStringRef()),
240           source_manager.getLocForStartOfFile(source_manager.getMainFileID())
241               .getLocWithOffset(m_source_location_index++)));
242     }
243   }
244 
245   StoringDiagnosticConsumer *diagnostic_consumer =
246       static_cast<StoringDiagnosticConsumer *>(
247           m_compiler_instance->getDiagnostics().getClient());
248 
249   diagnostic_consumer->ClearDiagnostics();
250 
251   clang::Module *top_level_module = DoGetModule(clang_path.front(), false);
252 
253   if (!top_level_module) {
254     diagnostic_consumer->DumpDiagnostics(error_stream);
255     error_stream.Printf("error: Couldn't load top-level module %s\n",
256                         path[0].AsCString());
257     return false;
258   }
259 
260   clang::Module *submodule = top_level_module;
261 
262   for (size_t ci = 1; ci < path.size(); ++ci) {
263     llvm::StringRef component = path[ci].GetStringRef();
264     submodule = submodule->findSubmodule(component.str());
265     if (!submodule) {
266       diagnostic_consumer->DumpDiagnostics(error_stream);
267       error_stream.Printf("error: Couldn't load submodule %s\n",
268                           component.str().c_str());
269       return false;
270     }
271   }
272 
273   clang::Module *requested_module = DoGetModule(clang_path, true);
274 
275   if (requested_module != nullptr) {
276     if (exported_modules) {
277       ReportModuleExports(*exported_modules, requested_module);
278     }
279 
280     m_imported_modules[imported_module] = requested_module;
281 
282     m_enabled = true;
283 
284     return true;
285   }
286 
287   return false;
288 }
289 
290 bool ClangModulesDeclVendor::LanguageSupportsClangModules(
291     lldb::LanguageType language) {
292   switch (language) {
293   default:
294     return false;
295   // C++ and friends to be added
296   case lldb::LanguageType::eLanguageTypeC:
297   case lldb::LanguageType::eLanguageTypeC11:
298   case lldb::LanguageType::eLanguageTypeC89:
299   case lldb::LanguageType::eLanguageTypeC99:
300   case lldb::LanguageType::eLanguageTypeObjC:
301     return true;
302   }
303 }
304 
305 bool ClangModulesDeclVendorImpl::AddModulesForCompileUnit(
306     CompileUnit &cu, ClangModulesDeclVendor::ModuleVector &exported_modules,
307     Stream &error_stream) {
308   if (LanguageSupportsClangModules(cu.GetLanguage())) {
309     std::vector<ConstString> imported_modules = cu.GetImportedModules();
310 
311     for (ConstString imported_module : imported_modules) {
312       std::vector<ConstString> path;
313 
314       path.push_back(imported_module);
315 
316       if (!AddModule(path, &exported_modules, error_stream)) {
317         return false;
318       }
319     }
320 
321     return true;
322   }
323 
324   return true;
325 }
326 
327 // ClangImporter::lookupValue
328 
329 uint32_t
330 ClangModulesDeclVendorImpl::FindDecls(const ConstString &name, bool append,
331                                       uint32_t max_matches,
332                                       std::vector<clang::NamedDecl *> &decls) {
333   if (!m_enabled) {
334     return 0;
335   }
336 
337   if (!append)
338     decls.clear();
339 
340   clang::IdentifierInfo &ident =
341       m_compiler_instance->getASTContext().Idents.get(name.GetStringRef());
342 
343   clang::LookupResult lookup_result(
344       m_compiler_instance->getSema(), clang::DeclarationName(&ident),
345       clang::SourceLocation(), clang::Sema::LookupOrdinaryName);
346 
347   m_compiler_instance->getSema().LookupName(
348       lookup_result,
349       m_compiler_instance->getSema().getScopeForContext(
350           m_compiler_instance->getASTContext().getTranslationUnitDecl()));
351 
352   uint32_t num_matches = 0;
353 
354   for (clang::NamedDecl *named_decl : lookup_result) {
355     if (num_matches >= max_matches)
356       return num_matches;
357 
358     decls.push_back(named_decl);
359     ++num_matches;
360   }
361 
362   return num_matches;
363 }
364 
365 void ClangModulesDeclVendorImpl::ForEachMacro(
366     const ClangModulesDeclVendor::ModuleVector &modules,
367     std::function<bool(const std::string &)> handler) {
368   if (!m_enabled) {
369     return;
370   }
371 
372   typedef std::map<ModuleID, ssize_t> ModulePriorityMap;
373   ModulePriorityMap module_priorities;
374 
375   ssize_t priority = 0;
376 
377   for (ModuleID module : modules) {
378     module_priorities[module] = priority++;
379   }
380 
381   if (m_compiler_instance->getPreprocessor().getExternalSource()) {
382     m_compiler_instance->getPreprocessor()
383         .getExternalSource()
384         ->ReadDefinedMacros();
385   }
386 
387   for (clang::Preprocessor::macro_iterator
388            mi = m_compiler_instance->getPreprocessor().macro_begin(),
389            me = m_compiler_instance->getPreprocessor().macro_end();
390        mi != me; ++mi) {
391     const clang::IdentifierInfo *ii = nullptr;
392 
393     {
394       if (clang::IdentifierInfoLookup *lookup =
395               m_compiler_instance->getPreprocessor()
396                   .getIdentifierTable()
397                   .getExternalIdentifierLookup()) {
398         lookup->get(mi->first->getName());
399       }
400       if (!ii) {
401         ii = mi->first;
402       }
403     }
404 
405     ssize_t found_priority = -1;
406     clang::MacroInfo *macro_info = nullptr;
407 
408     for (clang::ModuleMacro *module_macro :
409          m_compiler_instance->getPreprocessor().getLeafModuleMacros(ii)) {
410       clang::Module *module = module_macro->getOwningModule();
411 
412       {
413         ModulePriorityMap::iterator pi =
414             module_priorities.find(reinterpret_cast<ModuleID>(module));
415 
416         if (pi != module_priorities.end() && pi->second > found_priority) {
417           macro_info = module_macro->getMacroInfo();
418           found_priority = pi->second;
419         }
420       }
421 
422       clang::Module *top_level_module = module->getTopLevelModule();
423 
424       if (top_level_module != module) {
425         ModulePriorityMap::iterator pi = module_priorities.find(
426             reinterpret_cast<ModuleID>(top_level_module));
427 
428         if ((pi != module_priorities.end()) && pi->second > found_priority) {
429           macro_info = module_macro->getMacroInfo();
430           found_priority = pi->second;
431         }
432       }
433     }
434 
435     if (macro_info) {
436       std::string macro_expansion = "#define ";
437       macro_expansion.append(mi->first->getName().str());
438 
439       {
440         if (macro_info->isFunctionLike()) {
441           macro_expansion.append("(");
442 
443           bool first_arg = true;
444 
445           for (auto pi = macro_info->param_begin(),
446                     pe = macro_info->param_end();
447                pi != pe; ++pi) {
448             if (!first_arg) {
449               macro_expansion.append(", ");
450             } else {
451               first_arg = false;
452             }
453 
454             macro_expansion.append((*pi)->getName().str());
455           }
456 
457           if (macro_info->isC99Varargs()) {
458             if (first_arg) {
459               macro_expansion.append("...");
460             } else {
461               macro_expansion.append(", ...");
462             }
463           } else if (macro_info->isGNUVarargs()) {
464             macro_expansion.append("...");
465           }
466 
467           macro_expansion.append(")");
468         }
469 
470         macro_expansion.append(" ");
471 
472         bool first_token = true;
473 
474         for (clang::MacroInfo::tokens_iterator ti = macro_info->tokens_begin(),
475                                                te = macro_info->tokens_end();
476              ti != te; ++ti) {
477           if (!first_token) {
478             macro_expansion.append(" ");
479           } else {
480             first_token = false;
481           }
482 
483           if (ti->isLiteral()) {
484             if (const char *literal_data = ti->getLiteralData()) {
485               std::string token_str(literal_data, ti->getLength());
486               macro_expansion.append(token_str);
487             } else {
488               bool invalid = false;
489               const char *literal_source =
490                   m_compiler_instance->getSourceManager().getCharacterData(
491                       ti->getLocation(), &invalid);
492 
493               if (invalid) {
494                 lldbassert(0 && "Unhandled token kind");
495                 macro_expansion.append("<unknown literal value>");
496               } else {
497                 macro_expansion.append(
498                     std::string(literal_source, ti->getLength()));
499               }
500             }
501           } else if (const char *punctuator_spelling =
502                          clang::tok::getPunctuatorSpelling(ti->getKind())) {
503             macro_expansion.append(punctuator_spelling);
504           } else if (const char *keyword_spelling =
505                          clang::tok::getKeywordSpelling(ti->getKind())) {
506             macro_expansion.append(keyword_spelling);
507           } else {
508             switch (ti->getKind()) {
509             case clang::tok::TokenKind::identifier:
510               macro_expansion.append(ti->getIdentifierInfo()->getName().str());
511               break;
512             case clang::tok::TokenKind::raw_identifier:
513               macro_expansion.append(ti->getRawIdentifier().str());
514               break;
515             default:
516               macro_expansion.append(ti->getName());
517               break;
518             }
519           }
520         }
521 
522         if (handler(macro_expansion)) {
523           return;
524         }
525       }
526     }
527   }
528 }
529 
530 clang::ModuleLoadResult
531 ClangModulesDeclVendorImpl::DoGetModule(clang::ModuleIdPath path,
532                                         bool make_visible) {
533   clang::Module::NameVisibilityKind visibility =
534       make_visible ? clang::Module::AllVisible : clang::Module::Hidden;
535 
536   const bool is_inclusion_directive = false;
537 
538   return m_compiler_instance->loadModule(path.front().second, path, visibility,
539                                          is_inclusion_directive);
540 }
541 
542 clang::ExternalASTMerger::ImporterSource
543 ClangModulesDeclVendorImpl::GetImporterSource() {
544   return {m_compiler_instance->getASTContext(),
545           m_compiler_instance->getFileManager(), m_origin_map};
546 }
547 
548 static const char *ModuleImportBufferName = "LLDBModulesMemoryBuffer";
549 
550 lldb_private::ClangModulesDeclVendor *
551 ClangModulesDeclVendor::Create(Target &target) {
552   // FIXME we should insure programmatically that the expression parser's
553   // compiler and the modules runtime's
554   // compiler are both initialized in the same way – preferably by the same
555   // code.
556 
557   if (!target.GetPlatform()->SupportsModules())
558     return nullptr;
559 
560   const ArchSpec &arch = target.GetArchitecture();
561 
562   std::vector<std::string> compiler_invocation_arguments = {
563       "clang",
564       "-fmodules",
565       "-fimplicit-module-maps",
566       "-fcxx-modules",
567       "-fsyntax-only",
568       "-femit-all-decls",
569       "-target",
570       arch.GetTriple().str(),
571       "-fmodules-validate-system-headers",
572       "-Werror=non-modular-include-in-framework-module"};
573 
574   target.GetPlatform()->AddClangModuleCompilationOptions(
575       &target, compiler_invocation_arguments);
576 
577   compiler_invocation_arguments.push_back(ModuleImportBufferName);
578 
579   // Add additional search paths with { "-I", path } or { "-F", path } here.
580 
581   {
582     llvm::SmallString<128> path;
583     auto props = ModuleList::GetGlobalModuleListProperties();
584     props.GetClangModulesCachePath().GetPath(path);
585     std::string module_cache_argument("-fmodules-cache-path=");
586     module_cache_argument.append(path.str());
587     compiler_invocation_arguments.push_back(module_cache_argument);
588   }
589 
590   FileSpecList &module_search_paths = target.GetClangModuleSearchPaths();
591 
592   for (size_t spi = 0, spe = module_search_paths.GetSize(); spi < spe; ++spi) {
593     const FileSpec &search_path = module_search_paths.GetFileSpecAtIndex(spi);
594 
595     std::string search_path_argument = "-I";
596     search_path_argument.append(search_path.GetPath());
597 
598     compiler_invocation_arguments.push_back(search_path_argument);
599   }
600 
601   {
602     FileSpec clang_resource_dir = GetClangResourceDir();
603 
604     if (llvm::sys::fs::is_directory(clang_resource_dir.GetPath())) {
605       compiler_invocation_arguments.push_back("-resource-dir");
606       compiler_invocation_arguments.push_back(clang_resource_dir.GetPath());
607     }
608   }
609 
610   llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_engine =
611       clang::CompilerInstance::createDiagnostics(new clang::DiagnosticOptions,
612                                                  new StoringDiagnosticConsumer);
613 
614   std::vector<const char *> compiler_invocation_argument_cstrs;
615 
616   for (const std::string &arg : compiler_invocation_arguments) {
617     compiler_invocation_argument_cstrs.push_back(arg.c_str());
618   }
619 
620   std::shared_ptr<clang::CompilerInvocation> invocation =
621       clang::createInvocationFromCommandLine(compiler_invocation_argument_cstrs,
622                                              diagnostics_engine);
623 
624   if (!invocation)
625     return nullptr;
626 
627   std::unique_ptr<llvm::MemoryBuffer> source_buffer =
628       llvm::MemoryBuffer::getMemBuffer(
629           "extern int __lldb __attribute__((unavailable));",
630           ModuleImportBufferName);
631 
632   invocation->getPreprocessorOpts().addRemappedFile(ModuleImportBufferName,
633                                                     source_buffer.release());
634 
635   std::unique_ptr<clang::CompilerInstance> instance(
636       new clang::CompilerInstance);
637 
638   instance->setDiagnostics(diagnostics_engine.get());
639   instance->setInvocation(invocation);
640 
641   std::unique_ptr<clang::FrontendAction> action(new clang::SyntaxOnlyAction);
642 
643   instance->setTarget(clang::TargetInfo::CreateTargetInfo(
644       *diagnostics_engine, instance->getInvocation().TargetOpts));
645 
646   if (!instance->hasTarget())
647     return nullptr;
648 
649   instance->getTarget().adjust(instance->getLangOpts());
650 
651   if (!action->BeginSourceFile(*instance,
652                                instance->getFrontendOpts().Inputs[0]))
653     return nullptr;
654 
655   instance->getPreprocessor().enableIncrementalProcessing();
656 
657   instance->createModuleManager();
658 
659   instance->createSema(action->getTranslationUnitKind(), nullptr);
660 
661   const bool skipFunctionBodies = false;
662   std::unique_ptr<clang::Parser> parser(new clang::Parser(
663       instance->getPreprocessor(), instance->getSema(), skipFunctionBodies));
664 
665   instance->getPreprocessor().EnterMainSourceFile();
666   parser->Initialize();
667 
668   clang::Parser::DeclGroupPtrTy parsed;
669 
670   while (!parser->ParseTopLevelDecl(parsed))
671     ;
672 
673   return new ClangModulesDeclVendorImpl(std::move(diagnostics_engine),
674                                         std::move(invocation),
675                                         std::move(instance), std::move(parser));
676 }
677