1 //===-- ClangModulesDeclVendor.cpp ------------------------------*- C++ -*-===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 10 // C Includes 11 // C++ Includes 12 #include <mutex> 13 14 // Other libraries and framework includes 15 #include "clang/Basic/TargetInfo.h" 16 #include "clang/Frontend/CompilerInstance.h" 17 #include "clang/Frontend/FrontendActions.h" 18 #include "clang/Lex/Preprocessor.h" 19 #include "clang/Lex/PreprocessorOptions.h" 20 #include "clang/Parse/Parser.h" 21 #include "clang/Sema/Lookup.h" 22 #include "clang/Serialization/ASTReader.h" 23 #include "llvm/Support/Path.h" 24 25 // Project includes 26 #include "ClangModulesDeclVendor.h" 27 28 #include "lldb/Core/Log.h" 29 #include "lldb/Core/StreamString.h" 30 #include "lldb/Host/FileSpec.h" 31 #include "lldb/Host/Host.h" 32 #include "lldb/Host/HostInfo.h" 33 #include "lldb/Symbol/CompileUnit.h" 34 #include "lldb/Target/Target.h" 35 #include "lldb/Utility/LLDBAssert.h" 36 37 using namespace lldb_private; 38 39 namespace { 40 // Any Clang compiler requires a consumer for diagnostics. This one stores them as strings 41 // so we can provide them to the user in case a module failed to load. 42 class StoringDiagnosticConsumer : public clang::DiagnosticConsumer 43 { 44 public: 45 StoringDiagnosticConsumer (); 46 47 void 48 HandleDiagnostic(clang::DiagnosticsEngine::Level DiagLevel, 49 const clang::Diagnostic &info) override; 50 51 void 52 ClearDiagnostics (); 53 54 void 55 DumpDiagnostics (Stream &error_stream); 56 57 private: 58 typedef std::pair<clang::DiagnosticsEngine::Level, std::string> IDAndDiagnostic; 59 std::vector<IDAndDiagnostic> m_diagnostics; 60 Log * m_log; 61 }; 62 63 // The private implementation of our ClangModulesDeclVendor. Contains all the Clang state required 64 // to load modules. 65 class ClangModulesDeclVendorImpl : public ClangModulesDeclVendor 66 { 67 public: 68 ClangModulesDeclVendorImpl(llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> &diagnostics_engine, 69 llvm::IntrusiveRefCntPtr<clang::CompilerInvocation> &compiler_invocation, 70 std::unique_ptr<clang::CompilerInstance> &&compiler_instance, 71 std::unique_ptr<clang::Parser> &&parser); 72 73 ~ClangModulesDeclVendorImpl() override = default; 74 75 bool 76 AddModule(ModulePath &path, 77 ModuleVector *exported_modules, 78 Stream &error_stream) override; 79 80 bool 81 AddModulesForCompileUnit(CompileUnit &cu, 82 ModuleVector &exported_modules, 83 Stream &error_stream) override; 84 85 uint32_t 86 FindDecls(const ConstString &name, 87 bool append, 88 uint32_t max_matches, 89 std::vector <clang::NamedDecl*> &decls) override; 90 91 void 92 ForEachMacro(const ModuleVector &modules, 93 std::function<bool (const std::string &)> handler) override; 94 95 private: 96 void 97 ReportModuleExportsHelper (std::set<ClangModulesDeclVendor::ModuleID> &exports, 98 clang::Module *module); 99 100 void 101 ReportModuleExports (ModuleVector &exports, 102 clang::Module *module); 103 104 clang::ModuleLoadResult 105 DoGetModule(clang::ModuleIdPath path, bool make_visible); 106 107 bool m_enabled = false; 108 109 llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> m_diagnostics_engine; 110 llvm::IntrusiveRefCntPtr<clang::CompilerInvocation> m_compiler_invocation; 111 std::unique_ptr<clang::CompilerInstance> m_compiler_instance; 112 std::unique_ptr<clang::Parser> m_parser; 113 size_t m_source_location_index = 0; // used to give name components fake SourceLocations 114 115 typedef std::vector<ConstString> ImportedModule; 116 typedef std::map<ImportedModule, clang::Module *> ImportedModuleMap; 117 typedef std::set<ModuleID> ImportedModuleSet; 118 ImportedModuleMap m_imported_modules; 119 ImportedModuleSet m_user_imported_modules; 120 }; 121 } // anonymous namespace 122 123 StoringDiagnosticConsumer::StoringDiagnosticConsumer () 124 { 125 m_log = lldb_private::GetLogIfAllCategoriesSet (LIBLLDB_LOG_EXPRESSIONS); 126 } 127 128 void 129 StoringDiagnosticConsumer::HandleDiagnostic (clang::DiagnosticsEngine::Level DiagLevel, const clang::Diagnostic &info) 130 { 131 llvm::SmallVector<char, 256> diagnostic_string; 132 133 info.FormatDiagnostic(diagnostic_string); 134 135 m_diagnostics.push_back(IDAndDiagnostic(DiagLevel, std::string(diagnostic_string.data(), diagnostic_string.size()))); 136 } 137 138 void 139 StoringDiagnosticConsumer::ClearDiagnostics () 140 { 141 m_diagnostics.clear(); 142 } 143 144 void 145 StoringDiagnosticConsumer::DumpDiagnostics (Stream &error_stream) 146 { 147 for (IDAndDiagnostic &diag : m_diagnostics) 148 { 149 switch (diag.first) 150 { 151 default: 152 error_stream.PutCString(diag.second.c_str()); 153 error_stream.PutChar('\n'); 154 break; 155 case clang::DiagnosticsEngine::Level::Ignored: 156 break; 157 } 158 } 159 } 160 161 static FileSpec 162 GetResourceDir () 163 { 164 static FileSpec g_cached_resource_dir; 165 166 static std::once_flag g_once_flag; 167 168 std::call_once(g_once_flag, [](){ 169 HostInfo::GetLLDBPath (lldb::ePathTypeClangDir, g_cached_resource_dir); 170 }); 171 172 return g_cached_resource_dir; 173 } 174 175 ClangModulesDeclVendor::ClangModulesDeclVendor() 176 { 177 } 178 179 ClangModulesDeclVendor::~ClangModulesDeclVendor() 180 { 181 } 182 183 ClangModulesDeclVendorImpl::ClangModulesDeclVendorImpl(llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> &diagnostics_engine, 184 llvm::IntrusiveRefCntPtr<clang::CompilerInvocation> &compiler_invocation, 185 std::unique_ptr<clang::CompilerInstance> &&compiler_instance, 186 std::unique_ptr<clang::Parser> &&parser) : 187 ClangModulesDeclVendor(), 188 m_diagnostics_engine(diagnostics_engine), 189 m_compiler_invocation(compiler_invocation), 190 m_compiler_instance(std::move(compiler_instance)), 191 m_parser(std::move(parser)), 192 m_imported_modules() 193 { 194 } 195 196 void 197 ClangModulesDeclVendorImpl::ReportModuleExportsHelper (std::set<ClangModulesDeclVendor::ModuleID> &exports, 198 clang::Module *module) 199 { 200 if (exports.count(reinterpret_cast<ClangModulesDeclVendor::ModuleID>(module))) 201 return; 202 203 exports.insert(reinterpret_cast<ClangModulesDeclVendor::ModuleID>(module)); 204 205 llvm::SmallVector<clang::Module*, 2> sub_exports; 206 207 module->getExportedModules(sub_exports); 208 209 for (clang::Module *module : sub_exports) 210 { 211 ReportModuleExportsHelper(exports, module); 212 } 213 } 214 215 void 216 ClangModulesDeclVendorImpl::ReportModuleExports (ClangModulesDeclVendor::ModuleVector &exports, 217 clang::Module *module) 218 { 219 std::set<ClangModulesDeclVendor::ModuleID> exports_set; 220 221 ReportModuleExportsHelper(exports_set, module); 222 223 for (ModuleID module : exports_set) 224 { 225 exports.push_back(module); 226 } 227 } 228 229 bool 230 ClangModulesDeclVendorImpl::AddModule(ModulePath &path, 231 ModuleVector *exported_modules, 232 Stream &error_stream) 233 { 234 // Fail early. 235 236 if (m_compiler_instance->hadModuleLoaderFatalFailure()) 237 { 238 error_stream.PutCString("error: Couldn't load a module because the module loader is in a fatal state.\n"); 239 return false; 240 } 241 242 // Check if we've already imported this module. 243 244 std::vector<ConstString> imported_module; 245 246 for (ConstString path_component : path) 247 { 248 imported_module.push_back(path_component); 249 } 250 251 { 252 ImportedModuleMap::iterator mi = m_imported_modules.find(imported_module); 253 254 if (mi != m_imported_modules.end()) 255 { 256 if (exported_modules) 257 { 258 ReportModuleExports(*exported_modules, mi->second); 259 } 260 return true; 261 } 262 } 263 264 if (!m_compiler_instance->getPreprocessor().getHeaderSearchInfo().lookupModule(path[0].GetStringRef())) 265 { 266 error_stream.Printf("error: Header search couldn't locate module %s\n", path[0].AsCString()); 267 return false; 268 } 269 270 llvm::SmallVector<std::pair<clang::IdentifierInfo *, clang::SourceLocation>, 4> clang_path; 271 272 { 273 clang::SourceManager &source_manager = m_compiler_instance->getASTContext().getSourceManager(); 274 275 for (ConstString path_component : path) 276 { 277 clang_path.push_back(std::make_pair(&m_compiler_instance->getASTContext().Idents.get(path_component.GetStringRef()), 278 source_manager.getLocForStartOfFile(source_manager.getMainFileID()).getLocWithOffset(m_source_location_index++))); 279 } 280 } 281 282 StoringDiagnosticConsumer *diagnostic_consumer = static_cast<StoringDiagnosticConsumer *>(m_compiler_instance->getDiagnostics().getClient()); 283 284 diagnostic_consumer->ClearDiagnostics(); 285 286 clang::Module *top_level_module = DoGetModule(clang_path.front(), false); 287 288 if (!top_level_module) 289 { 290 diagnostic_consumer->DumpDiagnostics(error_stream); 291 error_stream.Printf("error: Couldn't load top-level module %s\n", path[0].AsCString()); 292 return false; 293 } 294 295 clang::Module *submodule = top_level_module; 296 297 for (size_t ci = 1; ci < path.size(); ++ci) 298 { 299 llvm::StringRef component = path[ci].GetStringRef(); 300 submodule = submodule->findSubmodule(component.str()); 301 if (!submodule) 302 { 303 diagnostic_consumer->DumpDiagnostics(error_stream); 304 error_stream.Printf("error: Couldn't load submodule %s\n", component.str().c_str()); 305 return false; 306 } 307 } 308 309 clang::Module *requested_module = DoGetModule(clang_path, true); 310 311 if (requested_module != nullptr) 312 { 313 if (exported_modules) 314 { 315 ReportModuleExports(*exported_modules, requested_module); 316 } 317 318 m_imported_modules[imported_module] = requested_module; 319 320 m_enabled = true; 321 322 return true; 323 } 324 325 return false; 326 } 327 328 bool 329 ClangModulesDeclVendor::LanguageSupportsClangModules (lldb::LanguageType language) 330 { 331 switch (language) 332 { 333 default: 334 return false; 335 // C++ and friends to be added 336 case lldb::LanguageType::eLanguageTypeC: 337 case lldb::LanguageType::eLanguageTypeC11: 338 case lldb::LanguageType::eLanguageTypeC89: 339 case lldb::LanguageType::eLanguageTypeC99: 340 case lldb::LanguageType::eLanguageTypeObjC: 341 return true; 342 } 343 } 344 345 bool 346 ClangModulesDeclVendorImpl::AddModulesForCompileUnit(CompileUnit &cu, 347 ClangModulesDeclVendor::ModuleVector &exported_modules, 348 Stream &error_stream) 349 { 350 if (LanguageSupportsClangModules(cu.GetLanguage())) 351 { 352 std::vector<ConstString> imported_modules = cu.GetImportedModules(); 353 354 for (ConstString imported_module : imported_modules) 355 { 356 std::vector<ConstString> path; 357 358 path.push_back(imported_module); 359 360 if (!AddModule(path, &exported_modules, error_stream)) 361 { 362 return false; 363 } 364 } 365 366 return true; 367 } 368 369 return true; 370 } 371 372 // ClangImporter::lookupValue 373 374 uint32_t 375 ClangModulesDeclVendorImpl::FindDecls (const ConstString &name, 376 bool append, 377 uint32_t max_matches, 378 std::vector <clang::NamedDecl*> &decls) 379 { 380 if (!m_enabled) 381 { 382 return 0; 383 } 384 385 if (!append) 386 decls.clear(); 387 388 clang::IdentifierInfo &ident = m_compiler_instance->getASTContext().Idents.get(name.GetStringRef()); 389 390 clang::LookupResult lookup_result(m_compiler_instance->getSema(), 391 clang::DeclarationName(&ident), 392 clang::SourceLocation(), 393 clang::Sema::LookupOrdinaryName); 394 395 m_compiler_instance->getSema().LookupName(lookup_result, m_compiler_instance->getSema().getScopeForContext(m_compiler_instance->getASTContext().getTranslationUnitDecl())); 396 397 uint32_t num_matches = 0; 398 399 for (clang::NamedDecl *named_decl : lookup_result) 400 { 401 if (num_matches >= max_matches) 402 return num_matches; 403 404 decls.push_back(named_decl); 405 ++num_matches; 406 } 407 408 return num_matches; 409 } 410 411 void 412 ClangModulesDeclVendorImpl::ForEachMacro(const ClangModulesDeclVendor::ModuleVector &modules, 413 std::function<bool (const std::string &)> handler) 414 { 415 if (!m_enabled) 416 { 417 return; 418 } 419 420 typedef std::map<ModuleID, ssize_t> ModulePriorityMap; 421 ModulePriorityMap module_priorities; 422 423 ssize_t priority = 0; 424 425 for (ModuleID module : modules) 426 { 427 module_priorities[module] = priority++; 428 } 429 430 if (m_compiler_instance->getPreprocessor().getExternalSource()) 431 { 432 m_compiler_instance->getPreprocessor().getExternalSource()->ReadDefinedMacros(); 433 } 434 435 for (clang::Preprocessor::macro_iterator mi = m_compiler_instance->getPreprocessor().macro_begin(), 436 me = m_compiler_instance->getPreprocessor().macro_end(); 437 mi != me; 438 ++mi) 439 { 440 const clang::IdentifierInfo *ii = nullptr; 441 442 { 443 if (clang::IdentifierInfoLookup *lookup = m_compiler_instance->getPreprocessor().getIdentifierTable().getExternalIdentifierLookup()) 444 { 445 lookup->get(mi->first->getName()); 446 } 447 if (!ii) 448 { 449 ii = mi->first; 450 } 451 } 452 453 ssize_t found_priority = -1; 454 clang::MacroInfo *macro_info = nullptr; 455 456 for (clang::ModuleMacro *module_macro : m_compiler_instance->getPreprocessor().getLeafModuleMacros(ii)) 457 { 458 clang::Module *module = module_macro->getOwningModule(); 459 460 { 461 ModulePriorityMap::iterator pi = module_priorities.find(reinterpret_cast<ModuleID>(module)); 462 463 if (pi != module_priorities.end() && pi->second > found_priority) 464 { 465 macro_info = module_macro->getMacroInfo(); 466 found_priority = pi->second; 467 } 468 } 469 470 clang::Module *top_level_module = module->getTopLevelModule(); 471 472 if (top_level_module != module) 473 { 474 ModulePriorityMap::iterator pi = module_priorities.find(reinterpret_cast<ModuleID>(top_level_module)); 475 476 if ((pi != module_priorities.end()) && pi->second > found_priority) 477 { 478 macro_info = module_macro->getMacroInfo(); 479 found_priority = pi->second; 480 } 481 } 482 } 483 484 if (macro_info) 485 { 486 std::string macro_expansion = "#define "; 487 macro_expansion.append(mi->first->getName().str().c_str()); 488 489 { 490 if (macro_info->isFunctionLike()) 491 { 492 macro_expansion.append("("); 493 494 bool first_arg = true; 495 496 for (clang::MacroInfo::arg_iterator ai = macro_info->arg_begin(), 497 ae = macro_info->arg_end(); 498 ai != ae; 499 ++ai) 500 { 501 if (!first_arg) 502 { 503 macro_expansion.append(", "); 504 } 505 else 506 { 507 first_arg = false; 508 } 509 510 macro_expansion.append((*ai)->getName().str()); 511 } 512 513 if (macro_info->isC99Varargs()) 514 { 515 if (first_arg) 516 { 517 macro_expansion.append("..."); 518 } 519 else 520 { 521 macro_expansion.append(", ..."); 522 } 523 } 524 else if (macro_info->isGNUVarargs()) 525 { 526 macro_expansion.append("..."); 527 } 528 529 macro_expansion.append(")"); 530 } 531 532 macro_expansion.append(" "); 533 534 bool first_token = true; 535 536 for (clang::MacroInfo::tokens_iterator ti = macro_info->tokens_begin(), 537 te = macro_info->tokens_end(); 538 ti != te; 539 ++ti) 540 { 541 if (!first_token) 542 { 543 macro_expansion.append(" "); 544 } 545 else 546 { 547 first_token = false; 548 } 549 550 if (ti->isLiteral()) 551 { 552 if (const char *literal_data = ti->getLiteralData()) 553 { 554 std::string token_str(literal_data, ti->getLength()); 555 macro_expansion.append(token_str); 556 } 557 else 558 { 559 bool invalid = false; 560 const char *literal_source = m_compiler_instance->getSourceManager().getCharacterData(ti->getLocation(), &invalid); 561 562 if (invalid) 563 { 564 lldbassert(!"Unhandled token kind"); 565 macro_expansion.append("<unknown literal value>"); 566 } 567 else 568 { 569 macro_expansion.append(std::string(literal_source, ti->getLength())); 570 } 571 } 572 } 573 else if (const char *punctuator_spelling = clang::tok::getPunctuatorSpelling(ti->getKind())) 574 { 575 macro_expansion.append(punctuator_spelling); 576 } 577 else if (const char *keyword_spelling = clang::tok::getKeywordSpelling(ti->getKind())) 578 { 579 macro_expansion.append(keyword_spelling); 580 } 581 else 582 { 583 switch (ti->getKind()) 584 { 585 case clang::tok::TokenKind::identifier: 586 macro_expansion.append(ti->getIdentifierInfo()->getName().str()); 587 break; 588 case clang::tok::TokenKind::raw_identifier: 589 macro_expansion.append(ti->getRawIdentifier().str()); 590 break; 591 default: 592 macro_expansion.append(ti->getName()); 593 break; 594 } 595 } 596 } 597 598 if (handler(macro_expansion)) 599 { 600 return; 601 } 602 } 603 } 604 } 605 } 606 607 clang::ModuleLoadResult 608 ClangModulesDeclVendorImpl::DoGetModule(clang::ModuleIdPath path, 609 bool make_visible) 610 { 611 clang::Module::NameVisibilityKind visibility = make_visible ? clang::Module::AllVisible : clang::Module::Hidden; 612 613 const bool is_inclusion_directive = false; 614 615 return m_compiler_instance->loadModule(path.front().second, path, visibility, is_inclusion_directive); 616 } 617 618 static const char *ModuleImportBufferName = "LLDBModulesMemoryBuffer"; 619 620 lldb_private::ClangModulesDeclVendor * 621 ClangModulesDeclVendor::Create(Target &target) 622 { 623 // FIXME we should insure programmatically that the expression parser's compiler and the modules runtime's 624 // compiler are both initialized in the same way – preferably by the same code. 625 626 if (!target.GetPlatform()->SupportsModules()) 627 return nullptr; 628 629 const ArchSpec &arch = target.GetArchitecture(); 630 631 std::vector<std::string> compiler_invocation_arguments = 632 { 633 "clang", 634 "-fmodules", 635 "-fimplicit-module-maps", 636 "-fcxx-modules", 637 "-fsyntax-only", 638 "-femit-all-decls", 639 "-target", arch.GetTriple().str(), 640 "-fmodules-validate-system-headers", 641 "-Werror=non-modular-include-in-framework-module" 642 }; 643 644 target.GetPlatform()->AddClangModuleCompilationOptions(&target, compiler_invocation_arguments); 645 646 compiler_invocation_arguments.push_back(ModuleImportBufferName); 647 648 // Add additional search paths with { "-I", path } or { "-F", path } here. 649 650 { 651 llvm::SmallString<128> DefaultModuleCache; 652 const bool erased_on_reboot = false; 653 llvm::sys::path::system_temp_directory(erased_on_reboot, DefaultModuleCache); 654 llvm::sys::path::append(DefaultModuleCache, "org.llvm.clang"); 655 llvm::sys::path::append(DefaultModuleCache, "ModuleCache"); 656 std::string module_cache_argument("-fmodules-cache-path="); 657 module_cache_argument.append(DefaultModuleCache.str().str()); 658 compiler_invocation_arguments.push_back(module_cache_argument); 659 } 660 661 FileSpecList &module_search_paths = target.GetClangModuleSearchPaths(); 662 663 for (size_t spi = 0, spe = module_search_paths.GetSize(); spi < spe; ++spi) 664 { 665 const FileSpec &search_path = module_search_paths.GetFileSpecAtIndex(spi); 666 667 std::string search_path_argument = "-I"; 668 search_path_argument.append(search_path.GetPath()); 669 670 compiler_invocation_arguments.push_back(search_path_argument); 671 } 672 673 { 674 FileSpec clang_resource_dir = GetResourceDir(); 675 676 if (clang_resource_dir.IsDirectory()) 677 { 678 compiler_invocation_arguments.push_back("-resource-dir"); 679 compiler_invocation_arguments.push_back(clang_resource_dir.GetPath()); 680 } 681 } 682 683 llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_engine = clang::CompilerInstance::createDiagnostics(new clang::DiagnosticOptions, 684 new StoringDiagnosticConsumer); 685 686 std::vector<const char *> compiler_invocation_argument_cstrs; 687 688 for (const std::string &arg : compiler_invocation_arguments) { 689 compiler_invocation_argument_cstrs.push_back(arg.c_str()); 690 } 691 692 llvm::IntrusiveRefCntPtr<clang::CompilerInvocation> invocation(clang::createInvocationFromCommandLine(compiler_invocation_argument_cstrs, diagnostics_engine)); 693 694 if (!invocation) 695 return nullptr; 696 697 std::unique_ptr<llvm::MemoryBuffer> source_buffer = llvm::MemoryBuffer::getMemBuffer("extern int __lldb __attribute__((unavailable));", 698 ModuleImportBufferName); 699 700 invocation->getPreprocessorOpts().addRemappedFile(ModuleImportBufferName, source_buffer.release()); 701 702 std::unique_ptr<clang::CompilerInstance> instance(new clang::CompilerInstance); 703 704 instance->setDiagnostics(diagnostics_engine.get()); 705 instance->setInvocation(invocation.get()); 706 707 std::unique_ptr<clang::FrontendAction> action(new clang::SyntaxOnlyAction); 708 709 instance->setTarget(clang::TargetInfo::CreateTargetInfo(*diagnostics_engine, instance->getInvocation().TargetOpts)); 710 711 if (!instance->hasTarget()) 712 return nullptr; 713 714 instance->getTarget().adjust(instance->getLangOpts()); 715 716 if (!action->BeginSourceFile(*instance, instance->getFrontendOpts().Inputs[0])) 717 return nullptr; 718 719 instance->getPreprocessor().enableIncrementalProcessing(); 720 721 instance->createModuleManager(); 722 723 instance->createSema(action->getTranslationUnitKind(), nullptr); 724 725 const bool skipFunctionBodies = false; 726 std::unique_ptr<clang::Parser> parser(new clang::Parser(instance->getPreprocessor(), instance->getSema(), skipFunctionBodies)); 727 728 instance->getPreprocessor().EnterMainSourceFile(); 729 parser->Initialize(); 730 731 clang::Parser::DeclGroupPtrTy parsed; 732 733 while (!parser->ParseTopLevelDecl(parsed)); 734 735 return new ClangModulesDeclVendorImpl (diagnostics_engine, invocation, std::move(instance), std::move(parser)); 736 } 737