1 //===-- ClangModulesDeclVendor.cpp ------------------------------*- C++ -*-===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 10 // C Includes 11 // C++ Includes 12 #include <mutex> 13 14 // Other libraries and framework includes 15 #include "clang/Basic/TargetInfo.h" 16 #include "clang/Frontend/CompilerInstance.h" 17 #include "clang/Frontend/FrontendActions.h" 18 #include "clang/Lex/Preprocessor.h" 19 #include "clang/Parse/Parser.h" 20 #include "clang/Sema/Lookup.h" 21 #include "clang/Serialization/ASTReader.h" 22 23 // Project includes 24 #include "ClangModulesDeclVendor.h" 25 26 #include "lldb/Core/Log.h" 27 #include "lldb/Core/StreamString.h" 28 #include "lldb/Host/FileSpec.h" 29 #include "lldb/Host/Host.h" 30 #include "lldb/Host/HostInfo.h" 31 #include "lldb/Symbol/CompileUnit.h" 32 #include "lldb/Target/Target.h" 33 #include "lldb/Utility/LLDBAssert.h" 34 35 using namespace lldb_private; 36 37 namespace { 38 // Any Clang compiler requires a consumer for diagnostics. This one stores them as strings 39 // so we can provide them to the user in case a module failed to load. 40 class StoringDiagnosticConsumer : public clang::DiagnosticConsumer 41 { 42 public: 43 StoringDiagnosticConsumer (); 44 45 void 46 HandleDiagnostic(clang::DiagnosticsEngine::Level DiagLevel, 47 const clang::Diagnostic &info) override; 48 49 void 50 ClearDiagnostics (); 51 52 void 53 DumpDiagnostics (Stream &error_stream); 54 55 private: 56 typedef std::pair<clang::DiagnosticsEngine::Level, std::string> IDAndDiagnostic; 57 std::vector<IDAndDiagnostic> m_diagnostics; 58 Log * m_log; 59 }; 60 61 // The private implementation of our ClangModulesDeclVendor. Contains all the Clang state required 62 // to load modules. 63 class ClangModulesDeclVendorImpl : public ClangModulesDeclVendor 64 { 65 public: 66 ClangModulesDeclVendorImpl(llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> &diagnostics_engine, 67 llvm::IntrusiveRefCntPtr<clang::CompilerInvocation> &compiler_invocation, 68 std::unique_ptr<clang::CompilerInstance> &&compiler_instance, 69 std::unique_ptr<clang::Parser> &&parser); 70 71 ~ClangModulesDeclVendorImpl() override = default; 72 73 bool 74 AddModule(ModulePath &path, 75 ModuleVector *exported_modules, 76 Stream &error_stream) override; 77 78 bool 79 AddModulesForCompileUnit(CompileUnit &cu, 80 ModuleVector &exported_modules, 81 Stream &error_stream) override; 82 83 uint32_t 84 FindDecls(const ConstString &name, 85 bool append, 86 uint32_t max_matches, 87 std::vector <clang::NamedDecl*> &decls) override; 88 89 void 90 ForEachMacro(const ModuleVector &modules, 91 std::function<bool (const std::string &)> handler) override; 92 93 private: 94 void 95 ReportModuleExportsHelper (std::set<ClangModulesDeclVendor::ModuleID> &exports, 96 clang::Module *module); 97 98 void 99 ReportModuleExports (ModuleVector &exports, 100 clang::Module *module); 101 102 clang::ModuleLoadResult 103 DoGetModule(clang::ModuleIdPath path, bool make_visible); 104 105 bool m_enabled = false; 106 107 llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> m_diagnostics_engine; 108 llvm::IntrusiveRefCntPtr<clang::CompilerInvocation> m_compiler_invocation; 109 std::unique_ptr<clang::CompilerInstance> m_compiler_instance; 110 std::unique_ptr<clang::Parser> m_parser; 111 size_t m_source_location_index = 0; // used to give name components fake SourceLocations 112 113 typedef std::vector<ConstString> ImportedModule; 114 typedef std::map<ImportedModule, clang::Module *> ImportedModuleMap; 115 typedef std::set<ModuleID> ImportedModuleSet; 116 ImportedModuleMap m_imported_modules; 117 ImportedModuleSet m_user_imported_modules; 118 }; 119 } // anonymous namespace 120 121 StoringDiagnosticConsumer::StoringDiagnosticConsumer () 122 { 123 m_log = lldb_private::GetLogIfAllCategoriesSet (LIBLLDB_LOG_EXPRESSIONS); 124 } 125 126 void 127 StoringDiagnosticConsumer::HandleDiagnostic (clang::DiagnosticsEngine::Level DiagLevel, const clang::Diagnostic &info) 128 { 129 llvm::SmallVector<char, 256> diagnostic_string; 130 131 info.FormatDiagnostic(diagnostic_string); 132 133 m_diagnostics.push_back(IDAndDiagnostic(DiagLevel, std::string(diagnostic_string.data(), diagnostic_string.size()))); 134 } 135 136 void 137 StoringDiagnosticConsumer::ClearDiagnostics () 138 { 139 m_diagnostics.clear(); 140 } 141 142 void 143 StoringDiagnosticConsumer::DumpDiagnostics (Stream &error_stream) 144 { 145 for (IDAndDiagnostic &diag : m_diagnostics) 146 { 147 switch (diag.first) 148 { 149 default: 150 error_stream.PutCString(diag.second.c_str()); 151 error_stream.PutChar('\n'); 152 break; 153 case clang::DiagnosticsEngine::Level::Ignored: 154 break; 155 } 156 } 157 } 158 159 static FileSpec 160 GetResourceDir () 161 { 162 static FileSpec g_cached_resource_dir; 163 164 static std::once_flag g_once_flag; 165 166 std::call_once(g_once_flag, [](){ 167 HostInfo::GetLLDBPath (lldb::ePathTypeClangDir, g_cached_resource_dir); 168 }); 169 170 return g_cached_resource_dir; 171 } 172 173 ClangModulesDeclVendor::ClangModulesDeclVendor() 174 { 175 } 176 177 ClangModulesDeclVendor::~ClangModulesDeclVendor() 178 { 179 } 180 181 ClangModulesDeclVendorImpl::ClangModulesDeclVendorImpl(llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> &diagnostics_engine, 182 llvm::IntrusiveRefCntPtr<clang::CompilerInvocation> &compiler_invocation, 183 std::unique_ptr<clang::CompilerInstance> &&compiler_instance, 184 std::unique_ptr<clang::Parser> &&parser) : 185 ClangModulesDeclVendor(), 186 m_diagnostics_engine(diagnostics_engine), 187 m_compiler_invocation(compiler_invocation), 188 m_compiler_instance(std::move(compiler_instance)), 189 m_parser(std::move(parser)), 190 m_imported_modules() 191 { 192 } 193 194 void 195 ClangModulesDeclVendorImpl::ReportModuleExportsHelper (std::set<ClangModulesDeclVendor::ModuleID> &exports, 196 clang::Module *module) 197 { 198 if (exports.count(reinterpret_cast<ClangModulesDeclVendor::ModuleID>(module))) 199 return; 200 201 exports.insert(reinterpret_cast<ClangModulesDeclVendor::ModuleID>(module)); 202 203 llvm::SmallVector<clang::Module*, 2> sub_exports; 204 205 module->getExportedModules(sub_exports); 206 207 for (clang::Module *module : sub_exports) 208 { 209 ReportModuleExportsHelper(exports, module); 210 } 211 } 212 213 void 214 ClangModulesDeclVendorImpl::ReportModuleExports (ClangModulesDeclVendor::ModuleVector &exports, 215 clang::Module *module) 216 { 217 std::set<ClangModulesDeclVendor::ModuleID> exports_set; 218 219 ReportModuleExportsHelper(exports_set, module); 220 221 for (ModuleID module : exports_set) 222 { 223 exports.push_back(module); 224 } 225 } 226 227 bool 228 ClangModulesDeclVendorImpl::AddModule(ModulePath &path, 229 ModuleVector *exported_modules, 230 Stream &error_stream) 231 { 232 // Fail early. 233 234 if (m_compiler_instance->hadModuleLoaderFatalFailure()) 235 { 236 error_stream.PutCString("error: Couldn't load a module because the module loader is in a fatal state.\n"); 237 return false; 238 } 239 240 // Check if we've already imported this module. 241 242 std::vector<ConstString> imported_module; 243 244 for (ConstString path_component : path) 245 { 246 imported_module.push_back(path_component); 247 } 248 249 { 250 ImportedModuleMap::iterator mi = m_imported_modules.find(imported_module); 251 252 if (mi != m_imported_modules.end()) 253 { 254 if (exported_modules) 255 { 256 ReportModuleExports(*exported_modules, mi->second); 257 } 258 return true; 259 } 260 } 261 262 if (!m_compiler_instance->getPreprocessor().getHeaderSearchInfo().lookupModule(path[0].GetStringRef())) 263 { 264 error_stream.Printf("error: Header search couldn't locate module %s\n", path[0].AsCString()); 265 return false; 266 } 267 268 llvm::SmallVector<std::pair<clang::IdentifierInfo *, clang::SourceLocation>, 4> clang_path; 269 270 { 271 clang::SourceManager &source_manager = m_compiler_instance->getASTContext().getSourceManager(); 272 273 for (ConstString path_component : path) 274 { 275 clang_path.push_back(std::make_pair(&m_compiler_instance->getASTContext().Idents.get(path_component.GetStringRef()), 276 source_manager.getLocForStartOfFile(source_manager.getMainFileID()).getLocWithOffset(m_source_location_index++))); 277 } 278 } 279 280 StoringDiagnosticConsumer *diagnostic_consumer = static_cast<StoringDiagnosticConsumer *>(m_compiler_instance->getDiagnostics().getClient()); 281 282 diagnostic_consumer->ClearDiagnostics(); 283 284 clang::Module *top_level_module = DoGetModule(clang_path.front(), false); 285 286 if (!top_level_module) 287 { 288 diagnostic_consumer->DumpDiagnostics(error_stream); 289 error_stream.Printf("error: Couldn't load top-level module %s\n", path[0].AsCString()); 290 return false; 291 } 292 293 clang::Module *submodule = top_level_module; 294 295 for (size_t ci = 1; ci < path.size(); ++ci) 296 { 297 llvm::StringRef component = path[ci].GetStringRef(); 298 submodule = submodule->findSubmodule(component.str()); 299 if (!submodule) 300 { 301 diagnostic_consumer->DumpDiagnostics(error_stream); 302 error_stream.Printf("error: Couldn't load submodule %s\n", component.str().c_str()); 303 return false; 304 } 305 } 306 307 clang::Module *requested_module = DoGetModule(clang_path, true); 308 309 if (requested_module != nullptr) 310 { 311 if (exported_modules) 312 { 313 ReportModuleExports(*exported_modules, requested_module); 314 } 315 316 m_imported_modules[imported_module] = requested_module; 317 318 m_enabled = true; 319 320 return true; 321 } 322 323 return false; 324 } 325 326 bool 327 ClangModulesDeclVendor::LanguageSupportsClangModules (lldb::LanguageType language) 328 { 329 switch (language) 330 { 331 default: 332 return false; 333 // C++ and friends to be added 334 case lldb::LanguageType::eLanguageTypeC: 335 case lldb::LanguageType::eLanguageTypeC11: 336 case lldb::LanguageType::eLanguageTypeC89: 337 case lldb::LanguageType::eLanguageTypeC99: 338 case lldb::LanguageType::eLanguageTypeObjC: 339 return true; 340 } 341 } 342 343 bool 344 ClangModulesDeclVendorImpl::AddModulesForCompileUnit(CompileUnit &cu, 345 ClangModulesDeclVendor::ModuleVector &exported_modules, 346 Stream &error_stream) 347 { 348 if (LanguageSupportsClangModules(cu.GetLanguage())) 349 { 350 std::vector<ConstString> imported_modules = cu.GetImportedModules(); 351 352 for (ConstString imported_module : imported_modules) 353 { 354 std::vector<ConstString> path; 355 356 path.push_back(imported_module); 357 358 if (!AddModule(path, &exported_modules, error_stream)) 359 { 360 return false; 361 } 362 } 363 364 return true; 365 } 366 367 return true; 368 } 369 370 // ClangImporter::lookupValue 371 372 uint32_t 373 ClangModulesDeclVendorImpl::FindDecls (const ConstString &name, 374 bool append, 375 uint32_t max_matches, 376 std::vector <clang::NamedDecl*> &decls) 377 { 378 if (!m_enabled) 379 { 380 return 0; 381 } 382 383 if (!append) 384 decls.clear(); 385 386 clang::IdentifierInfo &ident = m_compiler_instance->getASTContext().Idents.get(name.GetStringRef()); 387 388 clang::LookupResult lookup_result(m_compiler_instance->getSema(), 389 clang::DeclarationName(&ident), 390 clang::SourceLocation(), 391 clang::Sema::LookupOrdinaryName); 392 393 m_compiler_instance->getSema().LookupName(lookup_result, m_compiler_instance->getSema().getScopeForContext(m_compiler_instance->getASTContext().getTranslationUnitDecl())); 394 395 uint32_t num_matches = 0; 396 397 for (clang::NamedDecl *named_decl : lookup_result) 398 { 399 if (num_matches >= max_matches) 400 return num_matches; 401 402 decls.push_back(named_decl); 403 ++num_matches; 404 } 405 406 return num_matches; 407 } 408 409 void 410 ClangModulesDeclVendorImpl::ForEachMacro(const ClangModulesDeclVendor::ModuleVector &modules, 411 std::function<bool (const std::string &)> handler) 412 { 413 if (!m_enabled) 414 { 415 return; 416 } 417 418 typedef std::map<ModuleID, ssize_t> ModulePriorityMap; 419 ModulePriorityMap module_priorities; 420 421 ssize_t priority = 0; 422 423 for (ModuleID module : modules) 424 { 425 module_priorities[module] = priority++; 426 } 427 428 if (m_compiler_instance->getPreprocessor().getExternalSource()) 429 { 430 m_compiler_instance->getPreprocessor().getExternalSource()->ReadDefinedMacros(); 431 } 432 433 for (clang::Preprocessor::macro_iterator mi = m_compiler_instance->getPreprocessor().macro_begin(), 434 me = m_compiler_instance->getPreprocessor().macro_end(); 435 mi != me; 436 ++mi) 437 { 438 const clang::IdentifierInfo *ii = nullptr; 439 440 { 441 if (clang::IdentifierInfoLookup *lookup = m_compiler_instance->getPreprocessor().getIdentifierTable().getExternalIdentifierLookup()) 442 { 443 lookup->get(mi->first->getName()); 444 } 445 if (!ii) 446 { 447 ii = mi->first; 448 } 449 } 450 451 ssize_t found_priority = -1; 452 clang::MacroInfo *macro_info = nullptr; 453 454 for (clang::ModuleMacro *module_macro : m_compiler_instance->getPreprocessor().getLeafModuleMacros(ii)) 455 { 456 clang::Module *module = module_macro->getOwningModule(); 457 458 { 459 ModulePriorityMap::iterator pi = module_priorities.find(reinterpret_cast<ModuleID>(module)); 460 461 if (pi != module_priorities.end() && pi->second > found_priority) 462 { 463 macro_info = module_macro->getMacroInfo(); 464 found_priority = pi->second; 465 } 466 } 467 468 clang::Module *top_level_module = module->getTopLevelModule(); 469 470 if (top_level_module != module) 471 { 472 ModulePriorityMap::iterator pi = module_priorities.find(reinterpret_cast<ModuleID>(top_level_module)); 473 474 if ((pi != module_priorities.end()) && pi->second > found_priority) 475 { 476 macro_info = module_macro->getMacroInfo(); 477 found_priority = pi->second; 478 } 479 } 480 } 481 482 if (macro_info) 483 { 484 std::string macro_expansion = "#define "; 485 macro_expansion.append(mi->first->getName().str().c_str()); 486 487 { 488 if (macro_info->isFunctionLike()) 489 { 490 macro_expansion.append("("); 491 492 bool first_arg = true; 493 494 for (clang::MacroInfo::arg_iterator ai = macro_info->arg_begin(), 495 ae = macro_info->arg_end(); 496 ai != ae; 497 ++ai) 498 { 499 if (!first_arg) 500 { 501 macro_expansion.append(", "); 502 } 503 else 504 { 505 first_arg = false; 506 } 507 508 macro_expansion.append((*ai)->getName().str()); 509 } 510 511 if (macro_info->isC99Varargs()) 512 { 513 if (first_arg) 514 { 515 macro_expansion.append("..."); 516 } 517 else 518 { 519 macro_expansion.append(", ..."); 520 } 521 } 522 else if (macro_info->isGNUVarargs()) 523 { 524 macro_expansion.append("..."); 525 } 526 527 macro_expansion.append(")"); 528 } 529 530 macro_expansion.append(" "); 531 532 bool first_token = true; 533 534 for (clang::MacroInfo::tokens_iterator ti = macro_info->tokens_begin(), 535 te = macro_info->tokens_end(); 536 ti != te; 537 ++ti) 538 { 539 if (!first_token) 540 { 541 macro_expansion.append(" "); 542 } 543 else 544 { 545 first_token = false; 546 } 547 548 if (ti->isLiteral()) 549 { 550 if (const char *literal_data = ti->getLiteralData()) 551 { 552 std::string token_str(literal_data, ti->getLength()); 553 macro_expansion.append(token_str); 554 } 555 else 556 { 557 bool invalid = false; 558 const char *literal_source = m_compiler_instance->getSourceManager().getCharacterData(ti->getLocation(), &invalid); 559 560 if (invalid) 561 { 562 lldbassert(!"Unhandled token kind"); 563 macro_expansion.append("<unknown literal value>"); 564 } 565 else 566 { 567 macro_expansion.append(std::string(literal_source, ti->getLength())); 568 } 569 } 570 } 571 else if (const char *punctuator_spelling = clang::tok::getPunctuatorSpelling(ti->getKind())) 572 { 573 macro_expansion.append(punctuator_spelling); 574 } 575 else if (const char *keyword_spelling = clang::tok::getKeywordSpelling(ti->getKind())) 576 { 577 macro_expansion.append(keyword_spelling); 578 } 579 else 580 { 581 switch (ti->getKind()) 582 { 583 case clang::tok::TokenKind::identifier: 584 macro_expansion.append(ti->getIdentifierInfo()->getName().str()); 585 break; 586 case clang::tok::TokenKind::raw_identifier: 587 macro_expansion.append(ti->getRawIdentifier().str()); 588 break; 589 default: 590 macro_expansion.append(ti->getName()); 591 break; 592 } 593 } 594 } 595 596 if (handler(macro_expansion)) 597 { 598 return; 599 } 600 } 601 } 602 } 603 } 604 605 clang::ModuleLoadResult 606 ClangModulesDeclVendorImpl::DoGetModule(clang::ModuleIdPath path, 607 bool make_visible) 608 { 609 clang::Module::NameVisibilityKind visibility = make_visible ? clang::Module::AllVisible : clang::Module::Hidden; 610 611 const bool is_inclusion_directive = false; 612 613 return m_compiler_instance->loadModule(path.front().second, path, visibility, is_inclusion_directive); 614 } 615 616 static const char *ModuleImportBufferName = "LLDBModulesMemoryBuffer"; 617 618 lldb_private::ClangModulesDeclVendor * 619 ClangModulesDeclVendor::Create(Target &target) 620 { 621 // FIXME we should insure programmatically that the expression parser's compiler and the modules runtime's 622 // compiler are both initialized in the same way – preferably by the same code. 623 624 if (!target.GetPlatform()->SupportsModules()) 625 return nullptr; 626 627 const ArchSpec &arch = target.GetArchitecture(); 628 629 std::vector<std::string> compiler_invocation_arguments = 630 { 631 "clang", 632 "-fmodules", 633 "-fimplicit-module-maps", 634 "-fcxx-modules", 635 "-fsyntax-only", 636 "-femit-all-decls", 637 "-target", arch.GetTriple().str(), 638 "-fmodules-validate-system-headers", 639 "-Werror=non-modular-include-in-framework-module" 640 }; 641 642 target.GetPlatform()->AddClangModuleCompilationOptions(&target, compiler_invocation_arguments); 643 644 compiler_invocation_arguments.push_back(ModuleImportBufferName); 645 646 // Add additional search paths with { "-I", path } or { "-F", path } here. 647 648 { 649 llvm::SmallString<128> DefaultModuleCache; 650 const bool erased_on_reboot = false; 651 llvm::sys::path::system_temp_directory(erased_on_reboot, DefaultModuleCache); 652 llvm::sys::path::append(DefaultModuleCache, "org.llvm.clang"); 653 llvm::sys::path::append(DefaultModuleCache, "ModuleCache"); 654 std::string module_cache_argument("-fmodules-cache-path="); 655 module_cache_argument.append(DefaultModuleCache.str().str()); 656 compiler_invocation_arguments.push_back(module_cache_argument); 657 } 658 659 FileSpecList &module_search_paths = target.GetClangModuleSearchPaths(); 660 661 for (size_t spi = 0, spe = module_search_paths.GetSize(); spi < spe; ++spi) 662 { 663 const FileSpec &search_path = module_search_paths.GetFileSpecAtIndex(spi); 664 665 std::string search_path_argument = "-I"; 666 search_path_argument.append(search_path.GetPath()); 667 668 compiler_invocation_arguments.push_back(search_path_argument); 669 } 670 671 { 672 FileSpec clang_resource_dir = GetResourceDir(); 673 674 if (clang_resource_dir.IsDirectory()) 675 { 676 compiler_invocation_arguments.push_back("-resource-dir"); 677 compiler_invocation_arguments.push_back(clang_resource_dir.GetPath()); 678 } 679 } 680 681 llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_engine = clang::CompilerInstance::createDiagnostics(new clang::DiagnosticOptions, 682 new StoringDiagnosticConsumer); 683 684 std::vector<const char *> compiler_invocation_argument_cstrs; 685 686 for (const std::string &arg : compiler_invocation_arguments) { 687 compiler_invocation_argument_cstrs.push_back(arg.c_str()); 688 } 689 690 llvm::IntrusiveRefCntPtr<clang::CompilerInvocation> invocation(clang::createInvocationFromCommandLine(compiler_invocation_argument_cstrs, diagnostics_engine)); 691 692 if (!invocation) 693 return nullptr; 694 695 std::unique_ptr<llvm::MemoryBuffer> source_buffer = llvm::MemoryBuffer::getMemBuffer("extern int __lldb __attribute__((unavailable));", 696 ModuleImportBufferName); 697 698 invocation->getPreprocessorOpts().addRemappedFile(ModuleImportBufferName, source_buffer.release()); 699 700 std::unique_ptr<clang::CompilerInstance> instance(new clang::CompilerInstance); 701 702 instance->setDiagnostics(diagnostics_engine.get()); 703 instance->setInvocation(invocation.get()); 704 705 std::unique_ptr<clang::FrontendAction> action(new clang::SyntaxOnlyAction); 706 707 instance->setTarget(clang::TargetInfo::CreateTargetInfo(*diagnostics_engine, instance->getInvocation().TargetOpts)); 708 709 if (!instance->hasTarget()) 710 return nullptr; 711 712 instance->getTarget().adjust(instance->getLangOpts()); 713 714 if (!action->BeginSourceFile(*instance, instance->getFrontendOpts().Inputs[0])) 715 return nullptr; 716 717 instance->getPreprocessor().enableIncrementalProcessing(); 718 719 instance->createModuleManager(); 720 721 instance->createSema(action->getTranslationUnitKind(), nullptr); 722 723 const bool skipFunctionBodies = false; 724 std::unique_ptr<clang::Parser> parser(new clang::Parser(instance->getPreprocessor(), instance->getSema(), skipFunctionBodies)); 725 726 instance->getPreprocessor().EnterMainSourceFile(); 727 parser->Initialize(); 728 729 clang::Parser::DeclGroupPtrTy parsed; 730 731 while (!parser->ParseTopLevelDecl(parsed)); 732 733 return new ClangModulesDeclVendorImpl (diagnostics_engine, invocation, std::move(instance), std::move(parser)); 734 } 735