1 //===-- ClangModulesDeclVendor.cpp ------------------------------*- C++ -*-===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 10 #include <mutex> // std::once 11 12 #include "ClangModulesDeclVendor.h" 13 14 #include "lldb/Core/Log.h" 15 #include "lldb/Core/StreamString.h" 16 #include "lldb/Host/FileSpec.h" 17 #include "lldb/Host/Host.h" 18 #include "lldb/Host/HostInfo.h" 19 #include "lldb/Symbol/CompileUnit.h" 20 #include "lldb/Target/Target.h" 21 #include "lldb/Utility/LLDBAssert.h" 22 23 #include "clang/Basic/TargetInfo.h" 24 #include "clang/Frontend/CompilerInstance.h" 25 #include "clang/Frontend/FrontendActions.h" 26 #include "clang/Lex/Preprocessor.h" 27 #include "clang/Parse/Parser.h" 28 #include "clang/Sema/Lookup.h" 29 #include "clang/Serialization/ASTReader.h" 30 31 32 using namespace lldb_private; 33 34 namespace { 35 // Any Clang compiler requires a consumer for diagnostics. This one stores them as strings 36 // so we can provide them to the user in case a module failed to load. 37 class StoringDiagnosticConsumer : public clang::DiagnosticConsumer 38 { 39 public: 40 StoringDiagnosticConsumer (); 41 void 42 HandleDiagnostic (clang::DiagnosticsEngine::Level DiagLevel, const clang::Diagnostic &info); 43 44 void 45 ClearDiagnostics (); 46 47 void 48 DumpDiagnostics (Stream &error_stream); 49 private: 50 typedef std::pair<clang::DiagnosticsEngine::Level, std::string> IDAndDiagnostic; 51 std::vector<IDAndDiagnostic> m_diagnostics; 52 Log * m_log; 53 }; 54 55 // The private implementation of our ClangModulesDeclVendor. Contains all the Clang state required 56 // to load modules. 57 class ClangModulesDeclVendorImpl : public ClangModulesDeclVendor 58 { 59 public: 60 ClangModulesDeclVendorImpl(llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> &diagnostics_engine, 61 llvm::IntrusiveRefCntPtr<clang::CompilerInvocation> &compiler_invocation, 62 std::unique_ptr<clang::CompilerInstance> &&compiler_instance, 63 std::unique_ptr<clang::Parser> &&parser); 64 65 virtual bool 66 AddModule(ModulePath &path, 67 ModuleVector *exported_modules, 68 Stream &error_stream) override; 69 70 virtual bool 71 AddModulesForCompileUnit(CompileUnit &cu, 72 ModuleVector &exported_modules, 73 Stream &error_stream) override; 74 75 virtual uint32_t 76 FindDecls (const ConstString &name, 77 bool append, 78 uint32_t max_matches, 79 std::vector <clang::NamedDecl*> &decls) override; 80 81 virtual void 82 ForEachMacro(const ModuleVector &modules, 83 std::function<bool (const std::string &)> handler) override; 84 85 ~ClangModulesDeclVendorImpl(); 86 87 private: 88 void 89 ReportModuleExportsHelper (std::set<ClangModulesDeclVendor::ModuleID> &exports, 90 clang::Module *module); 91 92 void 93 ReportModuleExports (ModuleVector &exports, 94 clang::Module *module); 95 96 clang::ModuleLoadResult 97 DoGetModule(clang::ModuleIdPath path, bool make_visible); 98 99 bool m_enabled = false; 100 101 llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> m_diagnostics_engine; 102 llvm::IntrusiveRefCntPtr<clang::CompilerInvocation> m_compiler_invocation; 103 std::unique_ptr<clang::CompilerInstance> m_compiler_instance; 104 std::unique_ptr<clang::Parser> m_parser; 105 size_t m_source_location_index = 0; // used to give name components fake SourceLocations 106 107 typedef std::vector<ConstString> ImportedModule; 108 typedef std::map<ImportedModule, clang::Module *> ImportedModuleMap; 109 typedef std::set<ModuleID> ImportedModuleSet; 110 ImportedModuleMap m_imported_modules; 111 ImportedModuleSet m_user_imported_modules; 112 }; 113 } 114 115 StoringDiagnosticConsumer::StoringDiagnosticConsumer () 116 { 117 m_log = lldb_private::GetLogIfAllCategoriesSet (LIBLLDB_LOG_EXPRESSIONS); 118 } 119 120 void 121 StoringDiagnosticConsumer::HandleDiagnostic (clang::DiagnosticsEngine::Level DiagLevel, const clang::Diagnostic &info) 122 { 123 llvm::SmallVector<char, 256> diagnostic_string; 124 125 info.FormatDiagnostic(diagnostic_string); 126 127 m_diagnostics.push_back(IDAndDiagnostic(DiagLevel, std::string(diagnostic_string.data(), diagnostic_string.size()))); 128 } 129 130 void 131 StoringDiagnosticConsumer::ClearDiagnostics () 132 { 133 m_diagnostics.clear(); 134 } 135 136 void 137 StoringDiagnosticConsumer::DumpDiagnostics (Stream &error_stream) 138 { 139 for (IDAndDiagnostic &diag : m_diagnostics) 140 { 141 switch (diag.first) 142 { 143 default: 144 error_stream.PutCString(diag.second.c_str()); 145 error_stream.PutChar('\n'); 146 break; 147 case clang::DiagnosticsEngine::Level::Ignored: 148 break; 149 } 150 } 151 } 152 153 static FileSpec 154 GetResourceDir () 155 { 156 static FileSpec g_cached_resource_dir; 157 158 static std::once_flag g_once_flag; 159 160 std::call_once(g_once_flag, [](){ 161 HostInfo::GetLLDBPath (lldb::ePathTypeClangDir, g_cached_resource_dir); 162 }); 163 164 return g_cached_resource_dir; 165 } 166 167 168 ClangModulesDeclVendor::ClangModulesDeclVendor() 169 { 170 } 171 172 ClangModulesDeclVendor::~ClangModulesDeclVendor() 173 { 174 } 175 176 ClangModulesDeclVendorImpl::ClangModulesDeclVendorImpl(llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> &diagnostics_engine, 177 llvm::IntrusiveRefCntPtr<clang::CompilerInvocation> &compiler_invocation, 178 std::unique_ptr<clang::CompilerInstance> &&compiler_instance, 179 std::unique_ptr<clang::Parser> &&parser) : 180 ClangModulesDeclVendor(), 181 m_diagnostics_engine(diagnostics_engine), 182 m_compiler_invocation(compiler_invocation), 183 m_compiler_instance(std::move(compiler_instance)), 184 m_parser(std::move(parser)), 185 m_imported_modules() 186 { 187 } 188 189 void 190 ClangModulesDeclVendorImpl::ReportModuleExportsHelper (std::set<ClangModulesDeclVendor::ModuleID> &exports, 191 clang::Module *module) 192 { 193 if (exports.count(reinterpret_cast<ClangModulesDeclVendor::ModuleID>(module))) 194 return; 195 196 exports.insert(reinterpret_cast<ClangModulesDeclVendor::ModuleID>(module)); 197 198 llvm::SmallVector<clang::Module*, 2> sub_exports; 199 200 module->getExportedModules(sub_exports); 201 202 for (clang::Module *module : sub_exports) 203 { 204 ReportModuleExportsHelper(exports, module); 205 } 206 } 207 208 void 209 ClangModulesDeclVendorImpl::ReportModuleExports (ClangModulesDeclVendor::ModuleVector &exports, 210 clang::Module *module) 211 { 212 std::set<ClangModulesDeclVendor::ModuleID> exports_set; 213 214 ReportModuleExportsHelper(exports_set, module); 215 216 for (ModuleID module : exports_set) 217 { 218 exports.push_back(module); 219 } 220 } 221 222 bool 223 ClangModulesDeclVendorImpl::AddModule(ModulePath &path, 224 ModuleVector *exported_modules, 225 Stream &error_stream) 226 { 227 // Fail early. 228 229 if (m_compiler_instance->hadModuleLoaderFatalFailure()) 230 { 231 error_stream.PutCString("error: Couldn't load a module because the module loader is in a fatal state.\n"); 232 return false; 233 } 234 235 // Check if we've already imported this module. 236 237 std::vector<ConstString> imported_module; 238 239 for (ConstString path_component : path) 240 { 241 imported_module.push_back(path_component); 242 } 243 244 { 245 ImportedModuleMap::iterator mi = m_imported_modules.find(imported_module); 246 247 if (mi != m_imported_modules.end()) 248 { 249 if (exported_modules) 250 { 251 ReportModuleExports(*exported_modules, mi->second); 252 } 253 return true; 254 } 255 } 256 257 if (!m_compiler_instance->getPreprocessor().getHeaderSearchInfo().lookupModule(path[0].GetStringRef())) 258 { 259 error_stream.Printf("error: Header search couldn't locate module %s\n", path[0].AsCString()); 260 return false; 261 } 262 263 llvm::SmallVector<std::pair<clang::IdentifierInfo *, clang::SourceLocation>, 4> clang_path; 264 265 { 266 clang::SourceManager &source_manager = m_compiler_instance->getASTContext().getSourceManager(); 267 268 for (ConstString path_component : path) 269 { 270 clang_path.push_back(std::make_pair(&m_compiler_instance->getASTContext().Idents.get(path_component.GetStringRef()), 271 source_manager.getLocForStartOfFile(source_manager.getMainFileID()).getLocWithOffset(m_source_location_index++))); 272 } 273 } 274 275 StoringDiagnosticConsumer *diagnostic_consumer = static_cast<StoringDiagnosticConsumer *>(m_compiler_instance->getDiagnostics().getClient()); 276 277 diagnostic_consumer->ClearDiagnostics(); 278 279 clang::Module *top_level_module = DoGetModule(clang_path.front(), false); 280 281 if (!top_level_module) 282 { 283 diagnostic_consumer->DumpDiagnostics(error_stream); 284 error_stream.Printf("error: Couldn't load top-level module %s\n", path[0].AsCString()); 285 return false; 286 } 287 288 clang::Module *submodule = top_level_module; 289 290 for (size_t ci = 1; ci < path.size(); ++ci) 291 { 292 llvm::StringRef component = path[ci].GetStringRef(); 293 submodule = submodule->findSubmodule(component.str()); 294 if (!submodule) 295 { 296 diagnostic_consumer->DumpDiagnostics(error_stream); 297 error_stream.Printf("error: Couldn't load submodule %s\n", component.str().c_str()); 298 return false; 299 } 300 } 301 302 clang::Module *requested_module = DoGetModule(clang_path, true); 303 304 if (requested_module != nullptr) 305 { 306 if (exported_modules) 307 { 308 ReportModuleExports(*exported_modules, requested_module); 309 } 310 311 m_imported_modules[imported_module] = requested_module; 312 313 m_enabled = true; 314 315 return true; 316 } 317 318 return false; 319 } 320 321 322 bool 323 ClangModulesDeclVendor::LanguageSupportsClangModules (lldb::LanguageType language) 324 { 325 switch (language) 326 { 327 default: 328 return false; 329 // C++ and friends to be added 330 case lldb::LanguageType::eLanguageTypeC: 331 case lldb::LanguageType::eLanguageTypeC11: 332 case lldb::LanguageType::eLanguageTypeC89: 333 case lldb::LanguageType::eLanguageTypeC99: 334 case lldb::LanguageType::eLanguageTypeObjC: 335 return true; 336 } 337 } 338 339 bool 340 ClangModulesDeclVendorImpl::AddModulesForCompileUnit(CompileUnit &cu, 341 ClangModulesDeclVendor::ModuleVector &exported_modules, 342 Stream &error_stream) 343 { 344 if (LanguageSupportsClangModules(cu.GetLanguage())) 345 { 346 std::vector<ConstString> imported_modules = cu.GetImportedModules(); 347 348 for (ConstString imported_module : imported_modules) 349 { 350 std::vector<ConstString> path; 351 352 path.push_back(imported_module); 353 354 if (!AddModule(path, &exported_modules, error_stream)) 355 { 356 return false; 357 } 358 } 359 360 return true; 361 } 362 363 return true; 364 } 365 366 // ClangImporter::lookupValue 367 368 uint32_t 369 ClangModulesDeclVendorImpl::FindDecls (const ConstString &name, 370 bool append, 371 uint32_t max_matches, 372 std::vector <clang::NamedDecl*> &decls) 373 { 374 if (!m_enabled) 375 { 376 return 0; 377 } 378 379 if (!append) 380 decls.clear(); 381 382 clang::IdentifierInfo &ident = m_compiler_instance->getASTContext().Idents.get(name.GetStringRef()); 383 384 clang::LookupResult lookup_result(m_compiler_instance->getSema(), 385 clang::DeclarationName(&ident), 386 clang::SourceLocation(), 387 clang::Sema::LookupOrdinaryName); 388 389 m_compiler_instance->getSema().LookupName(lookup_result, m_compiler_instance->getSema().getScopeForContext(m_compiler_instance->getASTContext().getTranslationUnitDecl())); 390 391 uint32_t num_matches = 0; 392 393 for (clang::NamedDecl *named_decl : lookup_result) 394 { 395 if (num_matches >= max_matches) 396 return num_matches; 397 398 decls.push_back(named_decl); 399 ++num_matches; 400 } 401 402 return num_matches; 403 } 404 405 void 406 ClangModulesDeclVendorImpl::ForEachMacro(const ClangModulesDeclVendor::ModuleVector &modules, 407 std::function<bool (const std::string &)> handler) 408 { 409 if (!m_enabled) 410 { 411 return; 412 } 413 414 typedef std::map<ModuleID, ssize_t> ModulePriorityMap; 415 ModulePriorityMap module_priorities; 416 417 ssize_t priority = 0; 418 419 for (ModuleID module : modules) 420 { 421 module_priorities[module] = priority++; 422 } 423 424 if (m_compiler_instance->getPreprocessor().getExternalSource()) 425 { 426 m_compiler_instance->getPreprocessor().getExternalSource()->ReadDefinedMacros(); 427 } 428 429 for (clang::Preprocessor::macro_iterator mi = m_compiler_instance->getPreprocessor().macro_begin(), 430 me = m_compiler_instance->getPreprocessor().macro_end(); 431 mi != me; 432 ++mi) 433 { 434 const clang::IdentifierInfo *ii = nullptr; 435 436 { 437 if (clang::IdentifierInfoLookup *lookup = m_compiler_instance->getPreprocessor().getIdentifierTable().getExternalIdentifierLookup()) 438 { 439 lookup->get(mi->first->getName()); 440 } 441 if (!ii) 442 { 443 ii = mi->first; 444 } 445 } 446 447 ssize_t found_priority = -1; 448 clang::MacroInfo *macro_info = nullptr; 449 450 for (clang::ModuleMacro *module_macro : m_compiler_instance->getPreprocessor().getLeafModuleMacros(ii)) 451 { 452 clang::Module *module = module_macro->getOwningModule(); 453 454 { 455 ModulePriorityMap::iterator pi = module_priorities.find(reinterpret_cast<ModuleID>(module)); 456 457 if (pi != module_priorities.end() && pi->second > found_priority) 458 { 459 macro_info = module_macro->getMacroInfo(); 460 found_priority = pi->second; 461 } 462 } 463 464 clang::Module *top_level_module = module->getTopLevelModule(); 465 466 if (top_level_module != module) 467 { 468 ModulePriorityMap::iterator pi = module_priorities.find(reinterpret_cast<ModuleID>(top_level_module)); 469 470 if ((pi != module_priorities.end()) && pi->second > found_priority) 471 { 472 macro_info = module_macro->getMacroInfo(); 473 found_priority = pi->second; 474 } 475 } 476 } 477 478 if (macro_info) 479 { 480 std::string macro_expansion = "#define "; 481 macro_expansion.append(mi->first->getName().str().c_str()); 482 483 { 484 if (macro_info->isFunctionLike()) 485 { 486 macro_expansion.append("("); 487 488 bool first_arg = true; 489 490 for (clang::MacroInfo::arg_iterator ai = macro_info->arg_begin(), 491 ae = macro_info->arg_end(); 492 ai != ae; 493 ++ai) 494 { 495 if (!first_arg) 496 { 497 macro_expansion.append(", "); 498 } 499 else 500 { 501 first_arg = false; 502 } 503 504 macro_expansion.append((*ai)->getName().str()); 505 } 506 507 if (macro_info->isC99Varargs()) 508 { 509 if (first_arg) 510 { 511 macro_expansion.append("..."); 512 } 513 else 514 { 515 macro_expansion.append(", ..."); 516 } 517 } 518 else if (macro_info->isGNUVarargs()) 519 { 520 macro_expansion.append("..."); 521 } 522 523 macro_expansion.append(")"); 524 } 525 526 macro_expansion.append(" "); 527 528 bool first_token = true; 529 530 for (clang::MacroInfo::tokens_iterator ti = macro_info->tokens_begin(), 531 te = macro_info->tokens_end(); 532 ti != te; 533 ++ti) 534 { 535 if (!first_token) 536 { 537 macro_expansion.append(" "); 538 } 539 else 540 { 541 first_token = false; 542 } 543 544 if (ti->isLiteral()) 545 { 546 if (const char *literal_data = ti->getLiteralData()) 547 { 548 std::string token_str(literal_data, ti->getLength()); 549 macro_expansion.append(token_str); 550 } 551 else 552 { 553 bool invalid = false; 554 const char *literal_source = m_compiler_instance->getSourceManager().getCharacterData(ti->getLocation(), &invalid); 555 556 if (invalid) 557 { 558 lldbassert(!"Unhandled token kind"); 559 macro_expansion.append("<unknown literal value>"); 560 } 561 else 562 { 563 macro_expansion.append(std::string(literal_source, ti->getLength())); 564 } 565 } 566 } 567 else if (const char *punctuator_spelling = clang::tok::getPunctuatorSpelling(ti->getKind())) 568 { 569 macro_expansion.append(punctuator_spelling); 570 } 571 else if (const char *keyword_spelling = clang::tok::getKeywordSpelling(ti->getKind())) 572 { 573 macro_expansion.append(keyword_spelling); 574 } 575 else 576 { 577 switch (ti->getKind()) 578 { 579 case clang::tok::TokenKind::identifier: 580 macro_expansion.append(ti->getIdentifierInfo()->getName().str()); 581 break; 582 case clang::tok::TokenKind::raw_identifier: 583 macro_expansion.append(ti->getRawIdentifier().str()); 584 default: 585 macro_expansion.append(ti->getName()); 586 break; 587 } 588 } 589 } 590 591 if (handler(macro_expansion)) 592 { 593 return; 594 } 595 } 596 } 597 } 598 } 599 600 ClangModulesDeclVendorImpl::~ClangModulesDeclVendorImpl() 601 { 602 } 603 604 clang::ModuleLoadResult 605 ClangModulesDeclVendorImpl::DoGetModule(clang::ModuleIdPath path, 606 bool make_visible) 607 { 608 clang::Module::NameVisibilityKind visibility = make_visible ? clang::Module::AllVisible : clang::Module::Hidden; 609 610 const bool is_inclusion_directive = false; 611 612 return m_compiler_instance->loadModule(path.front().second, path, visibility, is_inclusion_directive); 613 } 614 615 static const char *ModuleImportBufferName = "LLDBModulesMemoryBuffer"; 616 617 lldb_private::ClangModulesDeclVendor * 618 ClangModulesDeclVendor::Create(Target &target) 619 { 620 // FIXME we should insure programmatically that the expression parser's compiler and the modules runtime's 621 // compiler are both initialized in the same way – preferably by the same code. 622 623 if (!target.GetPlatform()->SupportsModules()) 624 return nullptr; 625 626 const ArchSpec &arch = target.GetArchitecture(); 627 628 std::vector<std::string> compiler_invocation_arguments = 629 { 630 "-fmodules", 631 "-fcxx-modules", 632 "-fsyntax-only", 633 "-femit-all-decls", 634 "-target", arch.GetTriple().str(), 635 "-fmodules-validate-system-headers", 636 "-Werror=non-modular-include-in-framework-module" 637 }; 638 639 target.GetPlatform()->AddClangModuleCompilationOptions(&target, compiler_invocation_arguments); 640 641 compiler_invocation_arguments.push_back(ModuleImportBufferName); 642 643 // Add additional search paths with { "-I", path } or { "-F", path } here. 644 645 { 646 llvm::SmallString<128> DefaultModuleCache; 647 const bool erased_on_reboot = false; 648 llvm::sys::path::system_temp_directory(erased_on_reboot, DefaultModuleCache); 649 llvm::sys::path::append(DefaultModuleCache, "org.llvm.clang"); 650 llvm::sys::path::append(DefaultModuleCache, "ModuleCache"); 651 std::string module_cache_argument("-fmodules-cache-path="); 652 module_cache_argument.append(DefaultModuleCache.str().str()); 653 compiler_invocation_arguments.push_back(module_cache_argument); 654 } 655 656 FileSpecList &module_search_paths = target.GetClangModuleSearchPaths(); 657 658 for (size_t spi = 0, spe = module_search_paths.GetSize(); spi < spe; ++spi) 659 { 660 const FileSpec &search_path = module_search_paths.GetFileSpecAtIndex(spi); 661 662 std::string search_path_argument = "-I"; 663 search_path_argument.append(search_path.GetPath()); 664 665 compiler_invocation_arguments.push_back(search_path_argument); 666 } 667 668 { 669 FileSpec clang_resource_dir = GetResourceDir(); 670 671 if (clang_resource_dir.IsDirectory()) 672 { 673 compiler_invocation_arguments.push_back("-resource-dir"); 674 compiler_invocation_arguments.push_back(clang_resource_dir.GetPath()); 675 } 676 } 677 678 llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_engine = clang::CompilerInstance::createDiagnostics(new clang::DiagnosticOptions, 679 new StoringDiagnosticConsumer); 680 681 std::vector<const char *> compiler_invocation_argument_cstrs; 682 683 for (const std::string &arg : compiler_invocation_arguments) { 684 compiler_invocation_argument_cstrs.push_back(arg.c_str()); 685 } 686 687 llvm::IntrusiveRefCntPtr<clang::CompilerInvocation> invocation(clang::createInvocationFromCommandLine(compiler_invocation_argument_cstrs, diagnostics_engine)); 688 689 if (!invocation) 690 return nullptr; 691 692 std::unique_ptr<llvm::MemoryBuffer> source_buffer = llvm::MemoryBuffer::getMemBuffer("extern int __lldb __attribute__((unavailable));", 693 ModuleImportBufferName); 694 695 invocation->getPreprocessorOpts().addRemappedFile(ModuleImportBufferName, source_buffer.release()); 696 697 std::unique_ptr<clang::CompilerInstance> instance(new clang::CompilerInstance); 698 699 instance->setDiagnostics(diagnostics_engine.get()); 700 instance->setInvocation(invocation.get()); 701 702 std::unique_ptr<clang::FrontendAction> action(new clang::SyntaxOnlyAction); 703 704 instance->setTarget(clang::TargetInfo::CreateTargetInfo(*diagnostics_engine, instance->getInvocation().TargetOpts)); 705 706 if (!instance->hasTarget()) 707 return nullptr; 708 709 instance->getTarget().adjust(instance->getLangOpts()); 710 711 if (!action->BeginSourceFile(*instance, instance->getFrontendOpts().Inputs[0])) 712 return nullptr; 713 714 instance->getPreprocessor().enableIncrementalProcessing(); 715 716 instance->createModuleManager(); 717 718 instance->createSema(action->getTranslationUnitKind(), nullptr); 719 720 const bool skipFunctionBodies = false; 721 std::unique_ptr<clang::Parser> parser(new clang::Parser(instance->getPreprocessor(), instance->getSema(), skipFunctionBodies)); 722 723 instance->getPreprocessor().EnterMainSourceFile(); 724 parser->Initialize(); 725 726 clang::Parser::DeclGroupPtrTy parsed; 727 728 while (!parser->ParseTopLevelDecl(parsed)); 729 730 return new ClangModulesDeclVendorImpl (diagnostics_engine, invocation, std::move(instance), std::move(parser)); 731 } 732