1 //===---- ELF_x86_64.cpp -JIT linker implementation for ELF/x86-64 ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // ELF/x86-64 jit-link implementation. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h" 14 #include "llvm/ExecutionEngine/JITLink/JITLink.h" 15 #include "llvm/ExecutionEngine/JITLink/x86_64.h" 16 #include "llvm/Object/ELFObjectFile.h" 17 #include "llvm/Support/Endian.h" 18 19 #include "DefineExternalSectionStartAndEndSymbols.h" 20 #include "EHFrameSupportImpl.h" 21 #include "JITLinkGeneric.h" 22 #include "PerGraphGOTAndPLTStubsBuilder.h" 23 24 #define DEBUG_TYPE "jitlink" 25 26 using namespace llvm; 27 using namespace llvm::jitlink; 28 using namespace llvm::jitlink::ELF_x86_64_Edges; 29 30 namespace { 31 32 constexpr StringRef ELFGOTSectionName = "$__GOT"; 33 constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_"; 34 35 class PerGraphGOTAndPLTStubsBuilder_ELF_x86_64 36 : public PerGraphGOTAndPLTStubsBuilder< 37 PerGraphGOTAndPLTStubsBuilder_ELF_x86_64> { 38 public: 39 static const uint8_t NullGOTEntryContent[8]; 40 static const uint8_t StubContent[6]; 41 42 using PerGraphGOTAndPLTStubsBuilder< 43 PerGraphGOTAndPLTStubsBuilder_ELF_x86_64>::PerGraphGOTAndPLTStubsBuilder; 44 45 bool isGOTEdgeToFix(Edge &E) const { 46 if (E.getKind() == GOTOFF64) { 47 // We need to make sure that the GOT section exists, but don't otherwise 48 // need to fix up this edge. 49 getGOTSection(); 50 return false; 51 } 52 53 return E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad || 54 E.getKind() == PCRel64GOT || E.getKind() == GOT64; 55 } 56 57 Symbol &createGOTEntry(Symbol &Target) { 58 auto &GOTEntryBlock = G.createContentBlock( 59 getGOTSection(), getGOTEntryBlockContent(), 0, 8, 0); 60 GOTEntryBlock.addEdge(Pointer64, 0, Target, 0); 61 return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); 62 } 63 64 void fixGOTEdge(Edge &E, Symbol &GOTEntry) { 65 // If this is a PCRel32GOT/PCRel64GOT then change it to an ordinary 66 // PCRel32/PCRel64. If it is a PCRel32GOTLoad then leave it as-is for now: 67 // We will use the kind to check for GOT optimization opportunities in the 68 // optimizeMachO_x86_64_GOTAndStubs pass below. 69 // If it's a GOT64 leave it as is. 70 switch (E.getKind()) { 71 case PCRel32GOT: 72 E.setKind(PCRel32); 73 break; 74 case PCRel64GOT: 75 E.setKind(PCRel64); 76 break; 77 case GOT64: 78 break; 79 case PCRel32GOTLoad: 80 break; 81 default: 82 llvm_unreachable("Unexpected GOT edge kind"); 83 } 84 85 E.setTarget(GOTEntry); 86 // Leave the edge addend as-is. 87 } 88 89 bool isExternalBranchEdge(Edge &E) { 90 return E.getKind() == Branch32 && !E.getTarget().isDefined(); 91 } 92 93 Symbol &createPLTStub(Symbol &Target) { 94 auto &StubContentBlock = 95 G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0); 96 // Re-use GOT entries for stub targets. 97 auto &GOTEntrySymbol = getGOTEntry(Target); 98 StubContentBlock.addEdge(PCRel32, 2, GOTEntrySymbol, -4); 99 return G.addAnonymousSymbol(StubContentBlock, 0, 6, true, false); 100 } 101 102 void fixPLTEdge(Edge &E, Symbol &Stub) { 103 assert(E.getKind() == Branch32 && "Not a Branch32 edge?"); 104 105 // Set the edge kind to Branch32ToStub. We will use this to check for stub 106 // optimization opportunities in the optimize ELF_x86_64_GOTAndStubs pass 107 // below. 108 E.setKind(Branch32ToStub); 109 E.setTarget(Stub); 110 } 111 112 private: 113 Section &getGOTSection() const { 114 if (!GOTSection) 115 GOTSection = &G.createSection(ELFGOTSectionName, sys::Memory::MF_READ); 116 return *GOTSection; 117 } 118 119 Section &getStubsSection() const { 120 if (!StubsSection) { 121 auto StubsProt = static_cast<sys::Memory::ProtectionFlags>( 122 sys::Memory::MF_READ | sys::Memory::MF_EXEC); 123 StubsSection = &G.createSection("$__STUBS", StubsProt); 124 } 125 return *StubsSection; 126 } 127 128 ArrayRef<char> getGOTEntryBlockContent() { 129 return {reinterpret_cast<const char *>(NullGOTEntryContent), 130 sizeof(NullGOTEntryContent)}; 131 } 132 133 ArrayRef<char> getStubBlockContent() { 134 return {reinterpret_cast<const char *>(StubContent), sizeof(StubContent)}; 135 } 136 137 mutable Section *GOTSection = nullptr; 138 mutable Section *StubsSection = nullptr; 139 }; 140 141 const char *const DwarfSectionNames[] = { 142 #define HANDLE_DWARF_SECTION(ENUM_NAME, ELF_NAME, CMDLINE_NAME, OPTION) \ 143 ELF_NAME, 144 #include "llvm/BinaryFormat/Dwarf.def" 145 #undef HANDLE_DWARF_SECTION 146 }; 147 148 } // namespace 149 150 const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::NullGOTEntryContent[8] = 151 {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; 152 const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::StubContent[6] = { 153 0xFF, 0x25, 0x00, 0x00, 0x00, 0x00}; 154 155 static const char *CommonSectionName = "__common"; 156 static Error optimizeELF_x86_64_GOTAndStubs(LinkGraph &G) { 157 LLVM_DEBUG(dbgs() << "Optimizing GOT entries and stubs:\n"); 158 159 for (auto *B : G.blocks()) 160 for (auto &E : B->edges()) 161 if (E.getKind() == PCRel32GOTLoad) { 162 // Replace GOT load with LEA only for MOVQ instructions. 163 constexpr uint8_t MOVQRIPRel[] = {0x48, 0x8b}; 164 if (E.getOffset() < 3 || 165 strncmp(B->getContent().data() + E.getOffset() - 3, 166 reinterpret_cast<const char *>(MOVQRIPRel), 2) != 0) 167 continue; 168 169 auto &GOTBlock = E.getTarget().getBlock(); 170 assert(GOTBlock.getSize() == G.getPointerSize() && 171 "GOT entry block should be pointer sized"); 172 assert(GOTBlock.edges_size() == 1 && 173 "GOT entry should only have one outgoing edge"); 174 175 auto &GOTTarget = GOTBlock.edges().begin()->getTarget(); 176 JITTargetAddress EdgeAddr = B->getAddress() + E.getOffset(); 177 JITTargetAddress TargetAddr = GOTTarget.getAddress(); 178 179 int64_t Displacement = TargetAddr - EdgeAddr + 4; 180 if (Displacement >= std::numeric_limits<int32_t>::min() && 181 Displacement <= std::numeric_limits<int32_t>::max()) { 182 // Change the edge kind as we don't go through GOT anymore. This is 183 // for formal correctness only. Technically, the two relocation kinds 184 // are resolved the same way. 185 E.setKind(PCRel32); 186 E.setTarget(GOTTarget); 187 auto *BlockData = reinterpret_cast<uint8_t *>( 188 const_cast<char *>(B->getContent().data())); 189 BlockData[E.getOffset() - 2] = 0x8d; 190 LLVM_DEBUG({ 191 dbgs() << " Replaced GOT load wih LEA:\n "; 192 printEdge(dbgs(), *B, E, getELFX86RelocationKindName(E.getKind())); 193 dbgs() << "\n"; 194 }); 195 } 196 } else if (E.getKind() == Branch32ToStub) { 197 auto &StubBlock = E.getTarget().getBlock(); 198 assert( 199 StubBlock.getSize() == 200 sizeof(PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::StubContent) && 201 "Stub block should be stub sized"); 202 assert(StubBlock.edges_size() == 1 && 203 "Stub block should only have one outgoing edge"); 204 205 auto &GOTBlock = StubBlock.edges().begin()->getTarget().getBlock(); 206 assert(GOTBlock.getSize() == G.getPointerSize() && 207 "GOT block should be pointer sized"); 208 assert(GOTBlock.edges_size() == 1 && 209 "GOT block should only have one outgoing edge"); 210 211 auto &GOTTarget = GOTBlock.edges().begin()->getTarget(); 212 JITTargetAddress EdgeAddr = B->getAddress() + E.getOffset(); 213 JITTargetAddress TargetAddr = GOTTarget.getAddress(); 214 215 int64_t Displacement = TargetAddr - EdgeAddr + 4; 216 if (Displacement >= std::numeric_limits<int32_t>::min() && 217 Displacement <= std::numeric_limits<int32_t>::max()) { 218 E.setKind(Branch32); 219 E.setTarget(GOTTarget); 220 LLVM_DEBUG({ 221 dbgs() << " Replaced stub branch with direct branch:\n "; 222 printEdge(dbgs(), *B, E, getELFX86RelocationKindName(E.getKind())); 223 dbgs() << "\n"; 224 }); 225 } 226 } 227 228 return Error::success(); 229 } 230 231 static bool isDwarfSection(StringRef SectionName) { 232 return llvm::is_contained(DwarfSectionNames, SectionName); 233 } 234 235 namespace llvm { 236 namespace jitlink { 237 238 // This should become a template as the ELFFile is so a lot of this could become 239 // generic 240 class ELFLinkGraphBuilder_x86_64 { 241 242 private: 243 Section *CommonSection = nullptr; 244 245 // TODO hack to get this working 246 // Find a better way 247 using SymbolTable = object::ELFFile<object::ELF64LE>::Elf_Shdr; 248 // For now we just assume 249 using SymbolMap = std::map<int32_t, Symbol *>; 250 SymbolMap JITSymbolTable; 251 252 Section &getCommonSection() { 253 if (!CommonSection) { 254 auto Prot = static_cast<sys::Memory::ProtectionFlags>( 255 sys::Memory::MF_READ | sys::Memory::MF_WRITE); 256 CommonSection = &G->createSection(CommonSectionName, Prot); 257 } 258 return *CommonSection; 259 } 260 261 static Expected<ELF_x86_64_Edges::ELFX86RelocationKind> 262 getRelocationKind(const uint32_t Type) { 263 switch (Type) { 264 case ELF::R_X86_64_PC32: 265 return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32; 266 case ELF::R_X86_64_PC64: 267 case ELF::R_X86_64_GOTPC64: 268 return ELF_x86_64_Edges::ELFX86RelocationKind::Delta64; 269 case ELF::R_X86_64_64: 270 return ELF_x86_64_Edges::ELFX86RelocationKind::Pointer64; 271 case ELF::R_X86_64_GOTPCREL: 272 case ELF::R_X86_64_GOTPCRELX: 273 case ELF::R_X86_64_REX_GOTPCRELX: 274 return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32GOTLoad; 275 case ELF::R_X86_64_GOTPCREL64: 276 return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel64GOT; 277 case ELF::R_X86_64_GOT64: 278 return ELF_x86_64_Edges::ELFX86RelocationKind::GOT64; 279 case ELF::R_X86_64_GOTOFF64: 280 return ELF_x86_64_Edges::ELFX86RelocationKind::GOTOFF64; 281 case ELF::R_X86_64_PLT32: 282 return ELF_x86_64_Edges::ELFX86RelocationKind::Branch32; 283 } 284 return make_error<JITLinkError>("Unsupported x86-64 relocation:" + 285 formatv("{0:d}", Type)); 286 } 287 288 std::unique_ptr<LinkGraph> G; 289 // This could be a template 290 const object::ELFFile<object::ELF64LE> &Obj; 291 object::ELFFile<object::ELF64LE>::Elf_Shdr_Range sections; 292 SymbolTable SymTab; 293 294 bool isRelocatable() { return Obj.getHeader().e_type == llvm::ELF::ET_REL; } 295 296 support::endianness 297 getEndianness(const object::ELFFile<object::ELF64LE> &Obj) { 298 return Obj.isLE() ? support::little : support::big; 299 } 300 301 // This could also just become part of a template 302 unsigned getPointerSize(const object::ELFFile<object::ELF64LE> &Obj) { 303 return Obj.getHeader().getFileClass() == ELF::ELFCLASS64 ? 8 : 4; 304 } 305 306 // We don't technically need this right now 307 // But for now going to keep it as it helps me to debug things 308 309 Error createNormalizedSymbols() { 310 LLVM_DEBUG(dbgs() << "Creating normalized symbols...\n"); 311 312 for (auto SecRef : sections) { 313 if (SecRef.sh_type != ELF::SHT_SYMTAB && 314 SecRef.sh_type != ELF::SHT_DYNSYM) 315 continue; 316 317 auto Symbols = Obj.symbols(&SecRef); 318 // TODO: Currently I use this function to test things 319 // I also want to leave it to see if its common between MACH and elf 320 // so for now I just want to continue even if there is an error 321 if (errorToBool(Symbols.takeError())) 322 continue; 323 324 auto StrTabSec = Obj.getSection(SecRef.sh_link); 325 if (!StrTabSec) 326 return StrTabSec.takeError(); 327 auto StringTable = Obj.getStringTable(**StrTabSec); 328 if (!StringTable) 329 return StringTable.takeError(); 330 331 for (auto SymRef : *Symbols) { 332 Optional<StringRef> Name; 333 334 if (auto NameOrErr = SymRef.getName(*StringTable)) 335 Name = *NameOrErr; 336 else 337 return NameOrErr.takeError(); 338 339 LLVM_DEBUG({ 340 dbgs() << " value = " << formatv("{0:x16}", SymRef.getValue()) 341 << ", type = " << formatv("{0:x2}", SymRef.getType()) 342 << ", binding = " << formatv("{0:x2}", SymRef.getBinding()) 343 << ", size = " 344 << formatv("{0:x16}", static_cast<uint64_t>(SymRef.st_size)) 345 << ", info = " << formatv("{0:x2}", SymRef.st_info) 346 << " :" << (Name ? *Name : "<anonymous symbol>") << "\n"; 347 }); 348 } 349 } 350 return Error::success(); 351 } 352 353 Error createNormalizedSections() { 354 LLVM_DEBUG(dbgs() << "Creating normalized sections...\n"); 355 for (auto &SecRef : sections) { 356 auto Name = Obj.getSectionName(SecRef); 357 if (!Name) 358 return Name.takeError(); 359 360 // Skip Dwarf sections. 361 if (isDwarfSection(*Name)) { 362 LLVM_DEBUG({ 363 dbgs() << *Name 364 << " is a debug section: No graph section will be created.\n"; 365 }); 366 continue; 367 } 368 369 sys::Memory::ProtectionFlags Prot; 370 if (SecRef.sh_flags & ELF::SHF_EXECINSTR) { 371 Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | 372 sys::Memory::MF_EXEC); 373 } else { 374 Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | 375 sys::Memory::MF_WRITE); 376 } 377 uint64_t Address = SecRef.sh_addr; 378 uint64_t Size = SecRef.sh_size; 379 uint64_t Flags = SecRef.sh_flags; 380 uint64_t Alignment = SecRef.sh_addralign; 381 const char *Data = nullptr; 382 // for now we just use this to skip the "undefined" section, probably need 383 // to revist 384 if (Size == 0) 385 continue; 386 387 // FIXME: Use flags. 388 (void)Flags; 389 390 LLVM_DEBUG({ 391 dbgs() << " " << *Name << ": " << formatv("{0:x16}", Address) << " -- " 392 << formatv("{0:x16}", Address + Size) << ", align: " << Alignment 393 << " Flags: " << formatv("{0:x}", Flags) << "\n"; 394 }); 395 396 if (SecRef.sh_type != ELF::SHT_NOBITS) { 397 // .sections() already checks that the data is not beyond the end of 398 // file 399 auto contents = Obj.getSectionContentsAsArray<char>(SecRef); 400 if (!contents) 401 return contents.takeError(); 402 403 Data = contents->data(); 404 // TODO protection flags. 405 // for now everything is 406 auto §ion = G->createSection(*Name, Prot); 407 // Do this here because we have it, but move it into graphify later 408 G->createContentBlock(section, ArrayRef<char>(Data, Size), Address, 409 Alignment, 0); 410 if (SecRef.sh_type == ELF::SHT_SYMTAB) 411 // TODO: Dynamic? 412 SymTab = SecRef; 413 } else { 414 auto &Section = G->createSection(*Name, Prot); 415 G->createZeroFillBlock(Section, Size, Address, Alignment, 0); 416 } 417 } 418 419 return Error::success(); 420 } 421 422 Error addRelocations() { 423 LLVM_DEBUG(dbgs() << "Adding relocations\n"); 424 // TODO a partern is forming of iterate some sections but only give me 425 // ones I am interested, i should abstract that concept some where 426 for (auto &SecRef : sections) { 427 if (SecRef.sh_type != ELF::SHT_RELA && SecRef.sh_type != ELF::SHT_REL) 428 continue; 429 // TODO can the elf obj file do this for me? 430 if (SecRef.sh_type == ELF::SHT_REL) 431 return make_error<llvm::StringError>("Shouldn't have REL in x64", 432 llvm::inconvertibleErrorCode()); 433 434 auto RelSectName = Obj.getSectionName(SecRef); 435 if (!RelSectName) 436 return RelSectName.takeError(); 437 438 LLVM_DEBUG({ 439 dbgs() << "Adding relocations from section " << *RelSectName << "\n"; 440 }); 441 442 auto UpdateSection = Obj.getSection(SecRef.sh_info); 443 if (!UpdateSection) 444 return UpdateSection.takeError(); 445 446 auto UpdateSectionName = Obj.getSectionName(**UpdateSection); 447 if (!UpdateSectionName) 448 return UpdateSectionName.takeError(); 449 450 // Don't process relocations for debug sections. 451 if (isDwarfSection(*UpdateSectionName)) { 452 LLVM_DEBUG({ 453 dbgs() << " Target is dwarf section " << *UpdateSectionName 454 << ". Skipping.\n"; 455 }); 456 continue; 457 } else 458 LLVM_DEBUG({ 459 dbgs() << " For target section " << *UpdateSectionName << "\n"; 460 }); 461 462 auto JITSection = G->findSectionByName(*UpdateSectionName); 463 if (!JITSection) 464 return make_error<llvm::StringError>( 465 "Refencing a a section that wasn't added to graph" + 466 *UpdateSectionName, 467 llvm::inconvertibleErrorCode()); 468 469 auto Relocations = Obj.relas(SecRef); 470 if (!Relocations) 471 return Relocations.takeError(); 472 473 for (const auto &Rela : *Relocations) { 474 auto Type = Rela.getType(false); 475 476 LLVM_DEBUG({ 477 dbgs() << "Relocation Type: " << Type << "\n" 478 << "Name: " << Obj.getRelocationTypeName(Type) << "\n"; 479 }); 480 auto SymbolIndex = Rela.getSymbol(false); 481 auto Symbol = Obj.getRelocationSymbol(Rela, &SymTab); 482 if (!Symbol) 483 return Symbol.takeError(); 484 485 auto BlockToFix = *(JITSection->blocks().begin()); 486 auto *TargetSymbol = JITSymbolTable[SymbolIndex]; 487 488 if (!TargetSymbol) { 489 return make_error<llvm::StringError>( 490 "Could not find symbol at given index, did you add it to " 491 "JITSymbolTable? index: " + std::to_string(SymbolIndex) 492 + ", shndx: " + std::to_string((*Symbol)->st_shndx) + 493 " Size of table: " + std::to_string(JITSymbolTable.size()), 494 llvm::inconvertibleErrorCode()); 495 } 496 uint64_t Addend = Rela.r_addend; 497 JITTargetAddress FixupAddress = 498 (*UpdateSection)->sh_addr + Rela.r_offset; 499 500 LLVM_DEBUG({ 501 dbgs() << "Processing relocation at " 502 << format("0x%016" PRIx64, FixupAddress) << "\n"; 503 }); 504 auto Kind = getRelocationKind(Type); 505 if (!Kind) 506 return Kind.takeError(); 507 508 LLVM_DEBUG({ 509 Edge GE(*Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol, 510 Addend); 511 printEdge(dbgs(), *BlockToFix, GE, 512 getELFX86RelocationKindName(*Kind)); 513 dbgs() << "\n"; 514 }); 515 BlockToFix->addEdge(*Kind, FixupAddress - BlockToFix->getAddress(), 516 *TargetSymbol, Addend); 517 } 518 } 519 return Error::success(); 520 } 521 522 Error graphifyRegularSymbols() { 523 524 // TODO: ELF supports beyond SHN_LORESERVE, 525 // need to perf test how a vector vs map handles those cases 526 527 std::vector<std::vector<object::ELFFile<object::ELF64LE>::Elf_Shdr_Range *>> 528 SecIndexToSymbols; 529 530 LLVM_DEBUG(dbgs() << "Creating graph symbols...\n"); 531 532 for (auto SecRef : sections) { 533 534 if (SecRef.sh_type != ELF::SHT_SYMTAB && 535 SecRef.sh_type != ELF::SHT_DYNSYM) 536 continue; 537 auto Symbols = Obj.symbols(&SecRef); 538 if (!Symbols) 539 return Symbols.takeError(); 540 541 auto StrTabSec = Obj.getSection(SecRef.sh_link); 542 if (!StrTabSec) 543 return StrTabSec.takeError(); 544 auto StringTable = Obj.getStringTable(**StrTabSec); 545 if (!StringTable) 546 return StringTable.takeError(); 547 auto Name = Obj.getSectionName(SecRef); 548 if (!Name) 549 return Name.takeError(); 550 551 LLVM_DEBUG(dbgs() << "Processing symbol section " << *Name << ":\n"); 552 553 auto Section = G->findSectionByName(*Name); 554 if (!Section) 555 return make_error<llvm::StringError>("Could not find a section " + 556 *Name, 557 llvm::inconvertibleErrorCode()); 558 // we only have one for now 559 auto blocks = Section->blocks(); 560 if (blocks.empty()) 561 return make_error<llvm::StringError>("Section has no block", 562 llvm::inconvertibleErrorCode()); 563 int SymbolIndex = -1; 564 for (auto SymRef : *Symbols) { 565 ++SymbolIndex; 566 auto Type = SymRef.getType(); 567 568 if (Type == ELF::STT_FILE || SymbolIndex == 0) 569 continue; 570 // these should do it for now 571 // if(Type != ELF::STT_NOTYPE && 572 // Type != ELF::STT_OBJECT && 573 // Type != ELF::STT_FUNC && 574 // Type != ELF::STT_SECTION && 575 // Type != ELF::STT_COMMON) { 576 // continue; 577 // } 578 auto Name = SymRef.getName(*StringTable); 579 // I am not sure on If this is going to hold as an invariant. Revisit. 580 if (!Name) 581 return Name.takeError(); 582 583 if (SymRef.isCommon()) { 584 // Symbols in SHN_COMMON refer to uninitialized data. The st_value 585 // field holds alignment constraints. 586 Symbol &S = 587 G->addCommonSymbol(*Name, Scope::Default, getCommonSection(), 0, 588 SymRef.st_size, SymRef.getValue(), false); 589 JITSymbolTable[SymbolIndex] = &S; 590 continue; 591 } 592 593 // Map Visibility and Binding to Scope and Linkage: 594 Linkage L = Linkage::Strong; 595 Scope S = Scope::Default; 596 597 switch (SymRef.getBinding()) { 598 case ELF::STB_LOCAL: 599 S = Scope::Local; 600 break; 601 case ELF::STB_GLOBAL: 602 // Nothing to do here. 603 break; 604 case ELF::STB_WEAK: 605 L = Linkage::Weak; 606 break; 607 default: 608 return make_error<StringError>("Unrecognized symbol binding for " + 609 *Name, 610 inconvertibleErrorCode()); 611 } 612 613 switch (SymRef.getVisibility()) { 614 case ELF::STV_DEFAULT: 615 case ELF::STV_PROTECTED: 616 // FIXME: Make STV_DEFAULT symbols pre-emptible? This probably needs 617 // Orc support. 618 // Otherwise nothing to do here. 619 break; 620 case ELF::STV_HIDDEN: 621 // Default scope -> Hidden scope. No effect on local scope. 622 if (S == Scope::Default) 623 S = Scope::Hidden; 624 break; 625 case ELF::STV_INTERNAL: 626 return make_error<StringError>("Unrecognized symbol visibility for " + 627 *Name, 628 inconvertibleErrorCode()); 629 } 630 631 if (SymRef.isDefined() && 632 (Type == ELF::STT_NOTYPE || Type == ELF::STT_FUNC || 633 Type == ELF::STT_OBJECT || Type == ELF::STT_SECTION)) { 634 635 auto DefinedSection = Obj.getSection(SymRef.st_shndx); 636 if (!DefinedSection) 637 return DefinedSection.takeError(); 638 auto sectName = Obj.getSectionName(**DefinedSection); 639 if (!sectName) 640 return Name.takeError(); 641 642 // Skip debug section symbols. 643 if (isDwarfSection(*sectName)) 644 continue; 645 646 auto JitSection = G->findSectionByName(*sectName); 647 if (!JitSection) 648 return make_error<llvm::StringError>( 649 "Could not find the JitSection " + *sectName, 650 llvm::inconvertibleErrorCode()); 651 auto bs = JitSection->blocks(); 652 if (bs.empty()) 653 return make_error<llvm::StringError>( 654 "Section has no block", llvm::inconvertibleErrorCode()); 655 656 auto *B = *bs.begin(); 657 LLVM_DEBUG({ dbgs() << " " << *Name << " at index " << SymbolIndex << "\n"; }); 658 if (SymRef.getType() == ELF::STT_SECTION) 659 *Name = *sectName; 660 auto &Sym = G->addDefinedSymbol( 661 *B, SymRef.getValue(), *Name, SymRef.st_size, L, S, 662 SymRef.getType() == ELF::STT_FUNC, false); 663 JITSymbolTable[SymbolIndex] = &Sym; 664 } else if (SymRef.isUndefined() && SymRef.isExternal()) { 665 auto &Sym = G->addExternalSymbol(*Name, SymRef.st_size, L); 666 JITSymbolTable[SymbolIndex] = &Sym; 667 } else 668 LLVM_DEBUG({ 669 dbgs() 670 << "Not creating graph symbol for normalized symbol at index " 671 << SymbolIndex << ", \"" << *Name << "\"\n"; 672 }); 673 674 // TODO: The following has to be implmented. 675 // leaving commented out to save time for future patchs 676 /* 677 G->addAbsoluteSymbol(*Name, SymRef.getValue(), SymRef.st_size, 678 Linkage::Strong, Scope::Default, false); 679 */ 680 } 681 } 682 return Error::success(); 683 } 684 685 public: 686 ELFLinkGraphBuilder_x86_64(StringRef FileName, 687 const object::ELFFile<object::ELF64LE> &Obj) 688 : G(std::make_unique<LinkGraph>( 689 FileName.str(), Triple("x86_64-unknown-linux"), getPointerSize(Obj), 690 getEndianness(Obj), getELFX86RelocationKindName)), 691 Obj(Obj) {} 692 693 Expected<std::unique_ptr<LinkGraph>> buildGraph() { 694 // Sanity check: we only operate on relocatable objects. 695 if (!isRelocatable()) 696 return make_error<JITLinkError>("Object is not a relocatable ELF"); 697 698 auto Secs = Obj.sections(); 699 700 if (!Secs) { 701 return Secs.takeError(); 702 } 703 sections = *Secs; 704 705 if (auto Err = createNormalizedSections()) 706 return std::move(Err); 707 708 if (auto Err = createNormalizedSymbols()) 709 return std::move(Err); 710 711 if (auto Err = graphifyRegularSymbols()) 712 return std::move(Err); 713 714 if (auto Err = addRelocations()) 715 return std::move(Err); 716 717 return std::move(G); 718 } 719 }; 720 721 class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> { 722 friend class JITLinker<ELFJITLinker_x86_64>; 723 724 public: 725 ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx, 726 std::unique_ptr<LinkGraph> G, 727 PassConfiguration PassConfig) 728 : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) { 729 getPassConfig().PostAllocationPasses.push_back( 730 [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); }); 731 } 732 733 private: 734 Symbol *GOTSymbol = nullptr; 735 736 Error getOrCreateGOTSymbol(LinkGraph &G) { 737 auto DefineExternalGOTSymbolIfPresent = 738 createDefineExternalSectionStartAndEndSymbolsPass( 739 [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc { 740 if (Sym.getName() == ELFGOTSymbolName) 741 if (auto *GOTSection = G.findSectionByName(ELFGOTSectionName)) { 742 GOTSymbol = &Sym; 743 return {*GOTSection, true}; 744 } 745 return {}; 746 }); 747 748 // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an 749 // external. 750 if (auto Err = DefineExternalGOTSymbolIfPresent(G)) 751 return Err; 752 753 // If we succeeded then we're done. 754 if (GOTSymbol) 755 return Error::success(); 756 757 // Otherwise look for a GOT section: If it already has a start symbol we'll 758 // record it, otherwise we'll create our own. 759 // If there's a GOT section but we didn't find an external GOT symbol... 760 if (auto *GOTSection = G.findSectionByName(ELFGOTSectionName)) { 761 762 // Check for an existing defined symbol. 763 for (auto *Sym : GOTSection->symbols()) 764 if (Sym->getName() == ELFGOTSymbolName) { 765 GOTSymbol = Sym; 766 return Error::success(); 767 } 768 769 // If there's no defined symbol then create one. 770 SectionRange SR(*GOTSection); 771 if (SR.empty()) 772 GOTSymbol = &G.addAbsoluteSymbol(ELFGOTSymbolName, 0, 0, 773 Linkage::Strong, Scope::Local, true); 774 else 775 GOTSymbol = 776 &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0, 777 Linkage::Strong, Scope::Local, false, true); 778 } 779 780 return Error::success(); 781 } 782 783 Error applyFixup(LinkGraph &G, Block &B, const Edge &E, 784 char *BlockWorkingMem) const { 785 using namespace ELF_x86_64_Edges; 786 using namespace llvm::support; 787 char *FixupPtr = BlockWorkingMem + E.getOffset(); 788 JITTargetAddress FixupAddress = B.getAddress() + E.getOffset(); 789 switch (E.getKind()) { 790 case ELFX86RelocationKind::Branch32: 791 case ELFX86RelocationKind::Branch32ToStub: 792 case ELFX86RelocationKind::PCRel32: 793 case ELFX86RelocationKind::PCRel32GOTLoad: { 794 int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; 795 if (LLVM_LIKELY(x86_64::isInRangeForImmS32(Value))) 796 *(little32_t *)FixupPtr = Value; 797 else 798 return makeTargetOutOfRangeError(G, B, E); 799 break; 800 } 801 case ELFX86RelocationKind::PCRel64: { 802 int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; 803 *(little64_t *)FixupPtr = Value; 804 break; 805 } 806 case ELFX86RelocationKind::Pointer64: { 807 int64_t Value = E.getTarget().getAddress() + E.getAddend(); 808 *(ulittle64_t *)FixupPtr = Value; 809 break; 810 } 811 case ELFX86RelocationKind::Delta32: { 812 int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; 813 if (LLVM_LIKELY(x86_64::isInRangeForImmS32(Value))) 814 *(little32_t *)FixupPtr = Value; 815 else 816 return makeTargetOutOfRangeError(G, B, E); 817 break; 818 } 819 case ELFX86RelocationKind::Delta64: { 820 int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; 821 *(little64_t *)FixupPtr = Value; 822 break; 823 } 824 case ELFX86RelocationKind::NegDelta32: { 825 int64_t Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); 826 if (LLVM_LIKELY(x86_64::isInRangeForImmS32(Value))) 827 *(little32_t *)FixupPtr = Value; 828 else 829 return makeTargetOutOfRangeError(G, B, E); 830 break; 831 } 832 case ELFX86RelocationKind::NegDelta64: { 833 int64_t Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); 834 *(little64_t *)FixupPtr = Value; 835 break; 836 } 837 case ELFX86RelocationKind::GOT64: 838 case ELFX86RelocationKind::GOTOFF64: { 839 // GOT64: Offset of GOT entry within GOT. 840 // GOTOFF64: Offset from GOT base to target. 841 // The expressions are the same in both cases, but in the GOT64 case the 842 // edge will have been fixed to point at the GOT entry, and in the 843 // GOTOFF64 case it will still point at the original target. 844 assert(GOTSymbol && "No GOT section symbol"); 845 int64_t Value = 846 E.getTarget().getAddress() - GOTSymbol->getAddress() + E.getAddend(); 847 *(little64_t *)FixupPtr = Value; 848 break; 849 } 850 default: 851 LLVM_DEBUG({ 852 dbgs() << "Bad edge: " << getELFX86RelocationKindName(E.getKind()) 853 << "\n"; 854 }); 855 llvm_unreachable("Unsupported relocation"); 856 } 857 return Error::success(); 858 } 859 }; 860 861 Expected<std::unique_ptr<LinkGraph>> 862 createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) { 863 LLVM_DEBUG({ 864 dbgs() << "Building jitlink graph for new input " 865 << ObjectBuffer.getBufferIdentifier() << "...\n"; 866 }); 867 868 auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer); 869 if (!ELFObj) 870 return ELFObj.takeError(); 871 872 auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj); 873 return ELFLinkGraphBuilder_x86_64((*ELFObj)->getFileName(), 874 ELFObjFile.getELFFile()) 875 .buildGraph(); 876 } 877 878 static SectionRangeSymbolDesc 879 identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) { 880 constexpr StringRef StartSymbolPrefix = "__start"; 881 constexpr StringRef EndSymbolPrefix = "__end"; 882 883 auto SymName = Sym.getName(); 884 if (SymName.startswith(StartSymbolPrefix)) { 885 if (auto *Sec = 886 G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size()))) 887 return {*Sec, true}; 888 } else if (SymName.startswith(EndSymbolPrefix)) { 889 if (auto *Sec = 890 G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size()))) 891 return {*Sec, false}; 892 } 893 return {}; 894 } 895 896 void link_ELF_x86_64(std::unique_ptr<LinkGraph> G, 897 std::unique_ptr<JITLinkContext> Ctx) { 898 PassConfiguration Config; 899 900 if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) { 901 902 Config.PrePrunePasses.push_back(EHFrameSplitter(".eh_frame")); 903 Config.PrePrunePasses.push_back(EHFrameEdgeFixer( 904 ".eh_frame", G->getPointerSize(), Delta64, Delta32, NegDelta32)); 905 Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame")); 906 907 // Construct a JITLinker and run the link function. 908 // Add a mark-live pass. 909 if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple())) 910 Config.PrePrunePasses.push_back(std::move(MarkLive)); 911 else 912 Config.PrePrunePasses.push_back(markAllSymbolsLive); 913 914 // Add an in-place GOT/Stubs pass. 915 Config.PostPrunePasses.push_back( 916 PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::asPass); 917 918 // Resolve any external section start / end symbols. 919 Config.PostAllocationPasses.push_back( 920 createDefineExternalSectionStartAndEndSymbolsPass( 921 identifyELFSectionStartAndEndSymbols)); 922 923 // Add GOT/Stubs optimizer pass. 924 Config.PreFixupPasses.push_back(optimizeELF_x86_64_GOTAndStubs); 925 } 926 927 if (auto Err = Ctx->modifyPassConfig(*G, Config)) 928 return Ctx->notifyFailed(std::move(Err)); 929 930 ELFJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config)); 931 } 932 const char *getELFX86RelocationKindName(Edge::Kind R) { 933 switch (R) { 934 case Branch32: 935 return "Branch32"; 936 case Branch32ToStub: 937 return "Branch32ToStub"; 938 case Pointer32: 939 return "Pointer32"; 940 case Pointer64: 941 return "Pointer64"; 942 case Pointer64Anon: 943 return "Pointer64Anon"; 944 case PCRel32: 945 return "PCRel32"; 946 case PCRel32Minus1: 947 return "PCRel32Minus1"; 948 case PCRel32Minus2: 949 return "PCRel32Minus2"; 950 case PCRel32Minus4: 951 return "PCRel32Minus4"; 952 case PCRel32Anon: 953 return "PCRel32Anon"; 954 case PCRel32Minus1Anon: 955 return "PCRel32Minus1Anon"; 956 case PCRel32Minus2Anon: 957 return "PCRel32Minus2Anon"; 958 case PCRel32Minus4Anon: 959 return "PCRel32Minus4Anon"; 960 case PCRel32GOTLoad: 961 return "PCRel32GOTLoad"; 962 case PCRel32GOT: 963 return "PCRel32GOT"; 964 case PCRel32TLV: 965 return "PCRel32TLV"; 966 case Delta32: 967 return "Delta32"; 968 case Delta64: 969 return "Delta64"; 970 case NegDelta32: 971 return "NegDelta32"; 972 case NegDelta64: 973 return "NegDelta64"; 974 } 975 return getGenericEdgeKindName(static_cast<Edge::Kind>(R)); 976 } 977 } // end namespace jitlink 978 } // end namespace llvm 979