1 //===---- ELF_x86_64.cpp -JIT linker implementation for ELF/x86-64 ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // ELF/x86-64 jit-link implementation. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h" 14 #include "llvm/ExecutionEngine/JITLink/JITLink.h" 15 #include "llvm/ExecutionEngine/JITLink/x86_64.h" 16 #include "llvm/Object/ELFObjectFile.h" 17 #include "llvm/Support/Endian.h" 18 19 #include "DefineExternalSectionStartAndEndSymbols.h" 20 #include "EHFrameSupportImpl.h" 21 #include "JITLinkGeneric.h" 22 #include "PerGraphGOTAndPLTStubsBuilder.h" 23 24 #define DEBUG_TYPE "jitlink" 25 26 using namespace llvm; 27 using namespace llvm::jitlink; 28 using namespace llvm::jitlink::ELF_x86_64_Edges; 29 30 namespace { 31 32 constexpr StringRef ELFGOTSectionName = "$__GOT"; 33 constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_"; 34 35 class PerGraphGOTAndPLTStubsBuilder_ELF_x86_64 36 : public PerGraphGOTAndPLTStubsBuilder< 37 PerGraphGOTAndPLTStubsBuilder_ELF_x86_64> { 38 public: 39 static const uint8_t NullGOTEntryContent[8]; 40 static const uint8_t StubContent[6]; 41 42 using PerGraphGOTAndPLTStubsBuilder< 43 PerGraphGOTAndPLTStubsBuilder_ELF_x86_64>::PerGraphGOTAndPLTStubsBuilder; 44 45 bool isGOTEdgeToFix(Edge &E) const { 46 if (E.getKind() == GOTOFF64) { 47 // We need to make sure that the GOT section exists, but don't otherwise 48 // need to fix up this edge. 49 getGOTSection(); 50 return false; 51 } 52 53 return E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad || 54 E.getKind() == PCRel64GOT || E.getKind() == GOT64; 55 } 56 57 Symbol &createGOTEntry(Symbol &Target) { 58 auto &GOTEntryBlock = G.createContentBlock( 59 getGOTSection(), getGOTEntryBlockContent(), 0, 8, 0); 60 GOTEntryBlock.addEdge(Pointer64, 0, Target, 0); 61 return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); 62 } 63 64 void fixGOTEdge(Edge &E, Symbol &GOTEntry) { 65 // If this is a PCRel32GOT/PCRel64GOT then change it to an ordinary 66 // PCRel32/PCRel64. If it is a PCRel32GOTLoad then leave it as-is for now: 67 // We will use the kind to check for GOT optimization opportunities in the 68 // optimizeMachO_x86_64_GOTAndStubs pass below. 69 // If it's a GOT64 leave it as is. 70 switch (E.getKind()) { 71 case PCRel32GOT: 72 E.setKind(PCRel32); 73 break; 74 case PCRel64GOT: 75 E.setKind(PCRel64); 76 break; 77 case GOT64: 78 break; 79 case PCRel32GOTLoad: 80 break; 81 default: 82 llvm_unreachable("Unexpected GOT edge kind"); 83 } 84 85 E.setTarget(GOTEntry); 86 // Leave the edge addend as-is. 87 } 88 89 bool isExternalBranchEdge(Edge &E) { 90 return E.getKind() == Branch32 && !E.getTarget().isDefined(); 91 } 92 93 Symbol &createPLTStub(Symbol &Target) { 94 auto &StubContentBlock = 95 G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0); 96 // Re-use GOT entries for stub targets. 97 auto &GOTEntrySymbol = getGOTEntry(Target); 98 StubContentBlock.addEdge(PCRel32, 2, GOTEntrySymbol, -4); 99 return G.addAnonymousSymbol(StubContentBlock, 0, 6, true, false); 100 } 101 102 void fixPLTEdge(Edge &E, Symbol &Stub) { 103 assert(E.getKind() == Branch32 && "Not a Branch32 edge?"); 104 105 // Set the edge kind to Branch32ToStub. We will use this to check for stub 106 // optimization opportunities in the optimize ELF_x86_64_GOTAndStubs pass 107 // below. 108 E.setKind(Branch32ToStub); 109 E.setTarget(Stub); 110 } 111 112 private: 113 Section &getGOTSection() const { 114 if (!GOTSection) 115 GOTSection = &G.createSection(ELFGOTSectionName, sys::Memory::MF_READ); 116 return *GOTSection; 117 } 118 119 Section &getStubsSection() const { 120 if (!StubsSection) { 121 auto StubsProt = static_cast<sys::Memory::ProtectionFlags>( 122 sys::Memory::MF_READ | sys::Memory::MF_EXEC); 123 StubsSection = &G.createSection("$__STUBS", StubsProt); 124 } 125 return *StubsSection; 126 } 127 128 StringRef getGOTEntryBlockContent() { 129 return StringRef(reinterpret_cast<const char *>(NullGOTEntryContent), 130 sizeof(NullGOTEntryContent)); 131 } 132 133 StringRef getStubBlockContent() { 134 return StringRef(reinterpret_cast<const char *>(StubContent), 135 sizeof(StubContent)); 136 } 137 138 mutable Section *GOTSection = nullptr; 139 mutable Section *StubsSection = nullptr; 140 }; 141 142 const char *const DwarfSectionNames[] = { 143 #define HANDLE_DWARF_SECTION(ENUM_NAME, ELF_NAME, CMDLINE_NAME, OPTION) \ 144 ELF_NAME, 145 #include "llvm/BinaryFormat/Dwarf.def" 146 #undef HANDLE_DWARF_SECTION 147 }; 148 149 } // namespace 150 151 const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::NullGOTEntryContent[8] = 152 {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; 153 const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::StubContent[6] = { 154 0xFF, 0x25, 0x00, 0x00, 0x00, 0x00}; 155 156 static const char *CommonSectionName = "__common"; 157 static Error optimizeELF_x86_64_GOTAndStubs(LinkGraph &G) { 158 LLVM_DEBUG(dbgs() << "Optimizing GOT entries and stubs:\n"); 159 160 for (auto *B : G.blocks()) 161 for (auto &E : B->edges()) 162 if (E.getKind() == PCRel32GOTLoad) { 163 // Replace GOT load with LEA only for MOVQ instructions. 164 constexpr uint8_t MOVQRIPRel[] = {0x48, 0x8b}; 165 if (E.getOffset() < 3 || 166 strncmp(B->getContent().data() + E.getOffset() - 3, 167 reinterpret_cast<const char *>(MOVQRIPRel), 2) != 0) 168 continue; 169 170 auto &GOTBlock = E.getTarget().getBlock(); 171 assert(GOTBlock.getSize() == G.getPointerSize() && 172 "GOT entry block should be pointer sized"); 173 assert(GOTBlock.edges_size() == 1 && 174 "GOT entry should only have one outgoing edge"); 175 176 auto &GOTTarget = GOTBlock.edges().begin()->getTarget(); 177 JITTargetAddress EdgeAddr = B->getAddress() + E.getOffset(); 178 JITTargetAddress TargetAddr = GOTTarget.getAddress(); 179 180 int64_t Displacement = TargetAddr - EdgeAddr + 4; 181 if (Displacement >= std::numeric_limits<int32_t>::min() && 182 Displacement <= std::numeric_limits<int32_t>::max()) { 183 // Change the edge kind as we don't go through GOT anymore. This is 184 // for formal correctness only. Technically, the two relocation kinds 185 // are resolved the same way. 186 E.setKind(PCRel32); 187 E.setTarget(GOTTarget); 188 auto *BlockData = reinterpret_cast<uint8_t *>( 189 const_cast<char *>(B->getContent().data())); 190 BlockData[E.getOffset() - 2] = 0x8d; 191 LLVM_DEBUG({ 192 dbgs() << " Replaced GOT load wih LEA:\n "; 193 printEdge(dbgs(), *B, E, getELFX86RelocationKindName(E.getKind())); 194 dbgs() << "\n"; 195 }); 196 } 197 } else if (E.getKind() == Branch32ToStub) { 198 auto &StubBlock = E.getTarget().getBlock(); 199 assert( 200 StubBlock.getSize() == 201 sizeof(PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::StubContent) && 202 "Stub block should be stub sized"); 203 assert(StubBlock.edges_size() == 1 && 204 "Stub block should only have one outgoing edge"); 205 206 auto &GOTBlock = StubBlock.edges().begin()->getTarget().getBlock(); 207 assert(GOTBlock.getSize() == G.getPointerSize() && 208 "GOT block should be pointer sized"); 209 assert(GOTBlock.edges_size() == 1 && 210 "GOT block should only have one outgoing edge"); 211 212 auto &GOTTarget = GOTBlock.edges().begin()->getTarget(); 213 JITTargetAddress EdgeAddr = B->getAddress() + E.getOffset(); 214 JITTargetAddress TargetAddr = GOTTarget.getAddress(); 215 216 int64_t Displacement = TargetAddr - EdgeAddr + 4; 217 if (Displacement >= std::numeric_limits<int32_t>::min() && 218 Displacement <= std::numeric_limits<int32_t>::max()) { 219 E.setKind(Branch32); 220 E.setTarget(GOTTarget); 221 LLVM_DEBUG({ 222 dbgs() << " Replaced stub branch with direct branch:\n "; 223 printEdge(dbgs(), *B, E, getELFX86RelocationKindName(E.getKind())); 224 dbgs() << "\n"; 225 }); 226 } 227 } 228 229 return Error::success(); 230 } 231 232 static bool isDwarfSection(StringRef SectionName) { 233 return llvm::is_contained(DwarfSectionNames, SectionName); 234 } 235 236 namespace llvm { 237 namespace jitlink { 238 239 // This should become a template as the ELFFile is so a lot of this could become 240 // generic 241 class ELFLinkGraphBuilder_x86_64 { 242 243 private: 244 Section *CommonSection = nullptr; 245 246 // TODO hack to get this working 247 // Find a better way 248 using SymbolTable = object::ELFFile<object::ELF64LE>::Elf_Shdr; 249 // For now we just assume 250 using SymbolMap = std::map<int32_t, Symbol *>; 251 SymbolMap JITSymbolTable; 252 253 Section &getCommonSection() { 254 if (!CommonSection) { 255 auto Prot = static_cast<sys::Memory::ProtectionFlags>( 256 sys::Memory::MF_READ | sys::Memory::MF_WRITE); 257 CommonSection = &G->createSection(CommonSectionName, Prot); 258 } 259 return *CommonSection; 260 } 261 262 static Expected<ELF_x86_64_Edges::ELFX86RelocationKind> 263 getRelocationKind(const uint32_t Type) { 264 switch (Type) { 265 case ELF::R_X86_64_PC32: 266 return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32; 267 case ELF::R_X86_64_PC64: 268 case ELF::R_X86_64_GOTPC64: 269 return ELF_x86_64_Edges::ELFX86RelocationKind::Delta64; 270 case ELF::R_X86_64_64: 271 return ELF_x86_64_Edges::ELFX86RelocationKind::Pointer64; 272 case ELF::R_X86_64_GOTPCREL: 273 case ELF::R_X86_64_GOTPCRELX: 274 case ELF::R_X86_64_REX_GOTPCRELX: 275 return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32GOTLoad; 276 case ELF::R_X86_64_GOTPCREL64: 277 return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel64GOT; 278 case ELF::R_X86_64_GOT64: 279 return ELF_x86_64_Edges::ELFX86RelocationKind::GOT64; 280 case ELF::R_X86_64_GOTOFF64: 281 return ELF_x86_64_Edges::ELFX86RelocationKind::GOTOFF64; 282 case ELF::R_X86_64_PLT32: 283 return ELF_x86_64_Edges::ELFX86RelocationKind::Branch32; 284 } 285 return make_error<JITLinkError>("Unsupported x86-64 relocation:" + 286 formatv("{0:d}", Type)); 287 } 288 289 std::unique_ptr<LinkGraph> G; 290 // This could be a template 291 const object::ELFFile<object::ELF64LE> &Obj; 292 object::ELFFile<object::ELF64LE>::Elf_Shdr_Range sections; 293 SymbolTable SymTab; 294 295 bool isRelocatable() { return Obj.getHeader().e_type == llvm::ELF::ET_REL; } 296 297 support::endianness 298 getEndianness(const object::ELFFile<object::ELF64LE> &Obj) { 299 return Obj.isLE() ? support::little : support::big; 300 } 301 302 // This could also just become part of a template 303 unsigned getPointerSize(const object::ELFFile<object::ELF64LE> &Obj) { 304 return Obj.getHeader().getFileClass() == ELF::ELFCLASS64 ? 8 : 4; 305 } 306 307 // We don't technically need this right now 308 // But for now going to keep it as it helps me to debug things 309 310 Error createNormalizedSymbols() { 311 LLVM_DEBUG(dbgs() << "Creating normalized symbols...\n"); 312 313 for (auto SecRef : sections) { 314 if (SecRef.sh_type != ELF::SHT_SYMTAB && 315 SecRef.sh_type != ELF::SHT_DYNSYM) 316 continue; 317 318 auto Symbols = Obj.symbols(&SecRef); 319 // TODO: Currently I use this function to test things 320 // I also want to leave it to see if its common between MACH and elf 321 // so for now I just want to continue even if there is an error 322 if (errorToBool(Symbols.takeError())) 323 continue; 324 325 auto StrTabSec = Obj.getSection(SecRef.sh_link); 326 if (!StrTabSec) 327 return StrTabSec.takeError(); 328 auto StringTable = Obj.getStringTable(**StrTabSec); 329 if (!StringTable) 330 return StringTable.takeError(); 331 332 for (auto SymRef : *Symbols) { 333 Optional<StringRef> Name; 334 335 if (auto NameOrErr = SymRef.getName(*StringTable)) 336 Name = *NameOrErr; 337 else 338 return NameOrErr.takeError(); 339 340 LLVM_DEBUG({ 341 dbgs() << " value = " << formatv("{0:x16}", SymRef.getValue()) 342 << ", type = " << formatv("{0:x2}", SymRef.getType()) 343 << ", binding = " << formatv("{0:x2}", SymRef.getBinding()) 344 << ", size = " 345 << formatv("{0:x16}", static_cast<uint64_t>(SymRef.st_size)) 346 << ", info = " << formatv("{0:x2}", SymRef.st_info) 347 << " :" << (Name ? *Name : "<anonymous symbol>") << "\n"; 348 }); 349 } 350 } 351 return Error::success(); 352 } 353 354 Error createNormalizedSections() { 355 LLVM_DEBUG(dbgs() << "Creating normalized sections...\n"); 356 for (auto &SecRef : sections) { 357 auto Name = Obj.getSectionName(SecRef); 358 if (!Name) 359 return Name.takeError(); 360 361 // Skip Dwarf sections. 362 if (isDwarfSection(*Name)) { 363 LLVM_DEBUG({ 364 dbgs() << *Name 365 << " is a debug section: No graph section will be created.\n"; 366 }); 367 continue; 368 } 369 370 sys::Memory::ProtectionFlags Prot; 371 if (SecRef.sh_flags & ELF::SHF_EXECINSTR) { 372 Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | 373 sys::Memory::MF_EXEC); 374 } else { 375 Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | 376 sys::Memory::MF_WRITE); 377 } 378 uint64_t Address = SecRef.sh_addr; 379 uint64_t Size = SecRef.sh_size; 380 uint64_t Flags = SecRef.sh_flags; 381 uint64_t Alignment = SecRef.sh_addralign; 382 const char *Data = nullptr; 383 // for now we just use this to skip the "undefined" section, probably need 384 // to revist 385 if (Size == 0) 386 continue; 387 388 // FIXME: Use flags. 389 (void)Flags; 390 391 LLVM_DEBUG({ 392 dbgs() << " " << *Name << ": " << formatv("{0:x16}", Address) << " -- " 393 << formatv("{0:x16}", Address + Size) << ", align: " << Alignment 394 << " Flags: " << formatv("{0:x}", Flags) << "\n"; 395 }); 396 397 if (SecRef.sh_type != ELF::SHT_NOBITS) { 398 // .sections() already checks that the data is not beyond the end of 399 // file 400 auto contents = Obj.getSectionContentsAsArray<char>(SecRef); 401 if (!contents) 402 return contents.takeError(); 403 404 Data = contents->data(); 405 // TODO protection flags. 406 // for now everything is 407 auto §ion = G->createSection(*Name, Prot); 408 // Do this here because we have it, but move it into graphify later 409 G->createContentBlock(section, StringRef(Data, Size), Address, 410 Alignment, 0); 411 if (SecRef.sh_type == ELF::SHT_SYMTAB) 412 // TODO: Dynamic? 413 SymTab = SecRef; 414 } else { 415 auto &Section = G->createSection(*Name, Prot); 416 G->createZeroFillBlock(Section, Size, Address, Alignment, 0); 417 } 418 } 419 420 return Error::success(); 421 } 422 423 Error addRelocations() { 424 LLVM_DEBUG(dbgs() << "Adding relocations\n"); 425 // TODO a partern is forming of iterate some sections but only give me 426 // ones I am interested, i should abstract that concept some where 427 for (auto &SecRef : sections) { 428 if (SecRef.sh_type != ELF::SHT_RELA && SecRef.sh_type != ELF::SHT_REL) 429 continue; 430 // TODO can the elf obj file do this for me? 431 if (SecRef.sh_type == ELF::SHT_REL) 432 return make_error<llvm::StringError>("Shouldn't have REL in x64", 433 llvm::inconvertibleErrorCode()); 434 435 auto RelSectName = Obj.getSectionName(SecRef); 436 if (!RelSectName) 437 return RelSectName.takeError(); 438 439 LLVM_DEBUG({ 440 dbgs() << "Adding relocations from section " << *RelSectName << "\n"; 441 }); 442 443 auto UpdateSection = Obj.getSection(SecRef.sh_info); 444 if (!UpdateSection) 445 return UpdateSection.takeError(); 446 447 auto UpdateSectionName = Obj.getSectionName(**UpdateSection); 448 if (!UpdateSectionName) 449 return UpdateSectionName.takeError(); 450 451 // Don't process relocations for debug sections. 452 if (isDwarfSection(*UpdateSectionName)) { 453 LLVM_DEBUG({ 454 dbgs() << " Target is dwarf section " << *UpdateSectionName 455 << ". Skipping.\n"; 456 }); 457 continue; 458 } else 459 LLVM_DEBUG({ 460 dbgs() << " For target section " << *UpdateSectionName << "\n"; 461 }); 462 463 auto JITSection = G->findSectionByName(*UpdateSectionName); 464 if (!JITSection) 465 return make_error<llvm::StringError>( 466 "Refencing a a section that wasn't added to graph" + 467 *UpdateSectionName, 468 llvm::inconvertibleErrorCode()); 469 470 auto Relocations = Obj.relas(SecRef); 471 if (!Relocations) 472 return Relocations.takeError(); 473 474 for (const auto &Rela : *Relocations) { 475 auto Type = Rela.getType(false); 476 477 LLVM_DEBUG({ 478 dbgs() << "Relocation Type: " << Type << "\n" 479 << "Name: " << Obj.getRelocationTypeName(Type) << "\n"; 480 }); 481 auto SymbolIndex = Rela.getSymbol(false); 482 auto Symbol = Obj.getRelocationSymbol(Rela, &SymTab); 483 if (!Symbol) 484 return Symbol.takeError(); 485 486 auto BlockToFix = *(JITSection->blocks().begin()); 487 auto *TargetSymbol = JITSymbolTable[SymbolIndex]; 488 489 if (!TargetSymbol) { 490 return make_error<llvm::StringError>( 491 "Could not find symbol at given index, did you add it to " 492 "JITSymbolTable? index: " + std::to_string(SymbolIndex) 493 + ", shndx: " + std::to_string((*Symbol)->st_shndx) + 494 " Size of table: " + std::to_string(JITSymbolTable.size()), 495 llvm::inconvertibleErrorCode()); 496 } 497 uint64_t Addend = Rela.r_addend; 498 JITTargetAddress FixupAddress = 499 (*UpdateSection)->sh_addr + Rela.r_offset; 500 501 LLVM_DEBUG({ 502 dbgs() << "Processing relocation at " 503 << format("0x%016" PRIx64, FixupAddress) << "\n"; 504 }); 505 auto Kind = getRelocationKind(Type); 506 if (!Kind) 507 return Kind.takeError(); 508 509 LLVM_DEBUG({ 510 Edge GE(*Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol, 511 Addend); 512 printEdge(dbgs(), *BlockToFix, GE, 513 getELFX86RelocationKindName(*Kind)); 514 dbgs() << "\n"; 515 }); 516 BlockToFix->addEdge(*Kind, FixupAddress - BlockToFix->getAddress(), 517 *TargetSymbol, Addend); 518 } 519 } 520 return Error::success(); 521 } 522 523 Error graphifyRegularSymbols() { 524 525 // TODO: ELF supports beyond SHN_LORESERVE, 526 // need to perf test how a vector vs map handles those cases 527 528 std::vector<std::vector<object::ELFFile<object::ELF64LE>::Elf_Shdr_Range *>> 529 SecIndexToSymbols; 530 531 LLVM_DEBUG(dbgs() << "Creating graph symbols...\n"); 532 533 for (auto SecRef : sections) { 534 535 if (SecRef.sh_type != ELF::SHT_SYMTAB && 536 SecRef.sh_type != ELF::SHT_DYNSYM) 537 continue; 538 auto Symbols = Obj.symbols(&SecRef); 539 if (!Symbols) 540 return Symbols.takeError(); 541 542 auto StrTabSec = Obj.getSection(SecRef.sh_link); 543 if (!StrTabSec) 544 return StrTabSec.takeError(); 545 auto StringTable = Obj.getStringTable(**StrTabSec); 546 if (!StringTable) 547 return StringTable.takeError(); 548 auto Name = Obj.getSectionName(SecRef); 549 if (!Name) 550 return Name.takeError(); 551 552 LLVM_DEBUG(dbgs() << "Processing symbol section " << *Name << ":\n"); 553 554 auto Section = G->findSectionByName(*Name); 555 if (!Section) 556 return make_error<llvm::StringError>("Could not find a section " + 557 *Name, 558 llvm::inconvertibleErrorCode()); 559 // we only have one for now 560 auto blocks = Section->blocks(); 561 if (blocks.empty()) 562 return make_error<llvm::StringError>("Section has no block", 563 llvm::inconvertibleErrorCode()); 564 int SymbolIndex = -1; 565 for (auto SymRef : *Symbols) { 566 ++SymbolIndex; 567 auto Type = SymRef.getType(); 568 569 if (Type == ELF::STT_FILE || SymbolIndex == 0) 570 continue; 571 // these should do it for now 572 // if(Type != ELF::STT_NOTYPE && 573 // Type != ELF::STT_OBJECT && 574 // Type != ELF::STT_FUNC && 575 // Type != ELF::STT_SECTION && 576 // Type != ELF::STT_COMMON) { 577 // continue; 578 // } 579 auto Name = SymRef.getName(*StringTable); 580 // I am not sure on If this is going to hold as an invariant. Revisit. 581 if (!Name) 582 return Name.takeError(); 583 584 if (SymRef.isCommon()) { 585 // Symbols in SHN_COMMON refer to uninitialized data. The st_value 586 // field holds alignment constraints. 587 Symbol &S = 588 G->addCommonSymbol(*Name, Scope::Default, getCommonSection(), 0, 589 SymRef.st_size, SymRef.getValue(), false); 590 JITSymbolTable[SymbolIndex] = &S; 591 continue; 592 } 593 594 // Map Visibility and Binding to Scope and Linkage: 595 Linkage L = Linkage::Strong; 596 Scope S = Scope::Default; 597 598 switch (SymRef.getBinding()) { 599 case ELF::STB_LOCAL: 600 S = Scope::Local; 601 break; 602 case ELF::STB_GLOBAL: 603 // Nothing to do here. 604 break; 605 case ELF::STB_WEAK: 606 L = Linkage::Weak; 607 break; 608 default: 609 return make_error<StringError>("Unrecognized symbol binding for " + 610 *Name, 611 inconvertibleErrorCode()); 612 } 613 614 switch (SymRef.getVisibility()) { 615 case ELF::STV_DEFAULT: 616 case ELF::STV_PROTECTED: 617 // FIXME: Make STV_DEFAULT symbols pre-emptible? This probably needs 618 // Orc support. 619 // Otherwise nothing to do here. 620 break; 621 case ELF::STV_HIDDEN: 622 // Default scope -> Hidden scope. No effect on local scope. 623 if (S == Scope::Default) 624 S = Scope::Hidden; 625 break; 626 case ELF::STV_INTERNAL: 627 return make_error<StringError>("Unrecognized symbol visibility for " + 628 *Name, 629 inconvertibleErrorCode()); 630 } 631 632 if (SymRef.isDefined() && 633 (Type == ELF::STT_NOTYPE || Type == ELF::STT_FUNC || 634 Type == ELF::STT_OBJECT || Type == ELF::STT_SECTION)) { 635 636 auto DefinedSection = Obj.getSection(SymRef.st_shndx); 637 if (!DefinedSection) 638 return DefinedSection.takeError(); 639 auto sectName = Obj.getSectionName(**DefinedSection); 640 if (!sectName) 641 return Name.takeError(); 642 643 // Skip debug section symbols. 644 if (isDwarfSection(*sectName)) 645 continue; 646 647 auto JitSection = G->findSectionByName(*sectName); 648 if (!JitSection) 649 return make_error<llvm::StringError>( 650 "Could not find the JitSection " + *sectName, 651 llvm::inconvertibleErrorCode()); 652 auto bs = JitSection->blocks(); 653 if (bs.empty()) 654 return make_error<llvm::StringError>( 655 "Section has no block", llvm::inconvertibleErrorCode()); 656 657 auto *B = *bs.begin(); 658 LLVM_DEBUG({ dbgs() << " " << *Name << " at index " << SymbolIndex << "\n"; }); 659 if (SymRef.getType() == ELF::STT_SECTION) 660 *Name = *sectName; 661 auto &Sym = G->addDefinedSymbol( 662 *B, SymRef.getValue(), *Name, SymRef.st_size, L, S, 663 SymRef.getType() == ELF::STT_FUNC, false); 664 JITSymbolTable[SymbolIndex] = &Sym; 665 } else if (SymRef.isUndefined() && SymRef.isExternal()) { 666 auto &Sym = G->addExternalSymbol(*Name, SymRef.st_size, L); 667 JITSymbolTable[SymbolIndex] = &Sym; 668 } else 669 LLVM_DEBUG({ 670 dbgs() 671 << "Not creating graph symbol for normalized symbol at index " 672 << SymbolIndex << ", \"" << *Name << "\"\n"; 673 }); 674 675 // TODO: The following has to be implmented. 676 // leaving commented out to save time for future patchs 677 /* 678 G->addAbsoluteSymbol(*Name, SymRef.getValue(), SymRef.st_size, 679 Linkage::Strong, Scope::Default, false); 680 */ 681 } 682 } 683 return Error::success(); 684 } 685 686 public: 687 ELFLinkGraphBuilder_x86_64(StringRef FileName, 688 const object::ELFFile<object::ELF64LE> &Obj) 689 : G(std::make_unique<LinkGraph>( 690 FileName.str(), Triple("x86_64-unknown-linux"), getPointerSize(Obj), 691 getEndianness(Obj), getELFX86RelocationKindName)), 692 Obj(Obj) {} 693 694 Expected<std::unique_ptr<LinkGraph>> buildGraph() { 695 // Sanity check: we only operate on relocatable objects. 696 if (!isRelocatable()) 697 return make_error<JITLinkError>("Object is not a relocatable ELF"); 698 699 auto Secs = Obj.sections(); 700 701 if (!Secs) { 702 return Secs.takeError(); 703 } 704 sections = *Secs; 705 706 if (auto Err = createNormalizedSections()) 707 return std::move(Err); 708 709 if (auto Err = createNormalizedSymbols()) 710 return std::move(Err); 711 712 if (auto Err = graphifyRegularSymbols()) 713 return std::move(Err); 714 715 if (auto Err = addRelocations()) 716 return std::move(Err); 717 718 return std::move(G); 719 } 720 }; 721 722 class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> { 723 friend class JITLinker<ELFJITLinker_x86_64>; 724 725 public: 726 ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx, 727 std::unique_ptr<LinkGraph> G, 728 PassConfiguration PassConfig) 729 : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) { 730 getPassConfig().PostAllocationPasses.push_back( 731 [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); }); 732 } 733 734 private: 735 Symbol *GOTSymbol = nullptr; 736 737 Error getOrCreateGOTSymbol(LinkGraph &G) { 738 auto DefineExternalGOTSymbolIfPresent = 739 createDefineExternalSectionStartAndEndSymbolsPass( 740 [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc { 741 if (Sym.getName() == ELFGOTSymbolName) 742 if (auto *GOTSection = G.findSectionByName(ELFGOTSectionName)) { 743 GOTSymbol = &Sym; 744 return {*GOTSection, true}; 745 } 746 return {}; 747 }); 748 749 // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an 750 // external. 751 if (auto Err = DefineExternalGOTSymbolIfPresent(G)) 752 return Err; 753 754 // If we succeeded then we're done. 755 if (GOTSymbol) 756 return Error::success(); 757 758 // Otherwise look for a GOT section: If it already has a start symbol we'll 759 // record it, otherwise we'll create our own. 760 // If there's a GOT section but we didn't find an external GOT symbol... 761 if (auto *GOTSection = G.findSectionByName(ELFGOTSectionName)) { 762 763 // Check for an existing defined symbol. 764 for (auto *Sym : GOTSection->symbols()) 765 if (Sym->getName() == ELFGOTSymbolName) { 766 GOTSymbol = Sym; 767 return Error::success(); 768 } 769 770 // If there's no defined symbol then create one. 771 SectionRange SR(*GOTSection); 772 if (SR.empty()) 773 GOTSymbol = &G.addAbsoluteSymbol(ELFGOTSymbolName, 0, 0, 774 Linkage::Strong, Scope::Local, true); 775 else 776 GOTSymbol = 777 &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0, 778 Linkage::Strong, Scope::Local, false, true); 779 } 780 781 return Error::success(); 782 } 783 784 Error applyFixup(LinkGraph &G, Block &B, const Edge &E, 785 char *BlockWorkingMem) const { 786 using namespace ELF_x86_64_Edges; 787 using namespace llvm::support; 788 char *FixupPtr = BlockWorkingMem + E.getOffset(); 789 JITTargetAddress FixupAddress = B.getAddress() + E.getOffset(); 790 switch (E.getKind()) { 791 case ELFX86RelocationKind::Branch32: 792 case ELFX86RelocationKind::Branch32ToStub: 793 case ELFX86RelocationKind::PCRel32: 794 case ELFX86RelocationKind::PCRel32GOTLoad: { 795 int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; 796 if (LLVM_LIKELY(x86_64::isInRangeForImmS32(Value))) 797 *(little32_t *)FixupPtr = Value; 798 else 799 return makeTargetOutOfRangeError(G, B, E); 800 break; 801 } 802 case ELFX86RelocationKind::PCRel64: { 803 int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; 804 *(little64_t *)FixupPtr = Value; 805 break; 806 } 807 case ELFX86RelocationKind::Pointer64: { 808 int64_t Value = E.getTarget().getAddress() + E.getAddend(); 809 *(ulittle64_t *)FixupPtr = Value; 810 break; 811 } 812 case ELFX86RelocationKind::Delta32: { 813 int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; 814 if (LLVM_LIKELY(x86_64::isInRangeForImmS32(Value))) 815 *(little32_t *)FixupPtr = Value; 816 else 817 return makeTargetOutOfRangeError(G, B, E); 818 break; 819 } 820 case ELFX86RelocationKind::Delta64: { 821 int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; 822 *(little64_t *)FixupPtr = Value; 823 break; 824 } 825 case ELFX86RelocationKind::NegDelta32: { 826 int64_t Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); 827 if (LLVM_LIKELY(x86_64::isInRangeForImmS32(Value))) 828 *(little32_t *)FixupPtr = Value; 829 else 830 return makeTargetOutOfRangeError(G, B, E); 831 break; 832 } 833 case ELFX86RelocationKind::NegDelta64: { 834 int64_t Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); 835 *(little64_t *)FixupPtr = Value; 836 break; 837 } 838 case ELFX86RelocationKind::GOT64: 839 case ELFX86RelocationKind::GOTOFF64: { 840 // GOT64: Offset of GOT entry within GOT. 841 // GOTOFF64: Offset from GOT base to target. 842 // The expressions are the same in both cases, but in the GOT64 case the 843 // edge will have been fixed to point at the GOT entry, and in the 844 // GOTOFF64 case it will still point at the original target. 845 assert(GOTSymbol && "No GOT section symbol"); 846 int64_t Value = 847 E.getTarget().getAddress() - GOTSymbol->getAddress() + E.getAddend(); 848 *(little64_t *)FixupPtr = Value; 849 break; 850 } 851 default: 852 LLVM_DEBUG({ 853 dbgs() << "Bad edge: " << getELFX86RelocationKindName(E.getKind()) 854 << "\n"; 855 }); 856 llvm_unreachable("Unsupported relocation"); 857 } 858 return Error::success(); 859 } 860 }; 861 862 Expected<std::unique_ptr<LinkGraph>> 863 createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) { 864 LLVM_DEBUG({ 865 dbgs() << "Building jitlink graph for new input " 866 << ObjectBuffer.getBufferIdentifier() << "...\n"; 867 }); 868 869 auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer); 870 if (!ELFObj) 871 return ELFObj.takeError(); 872 873 auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj); 874 return ELFLinkGraphBuilder_x86_64((*ELFObj)->getFileName(), 875 ELFObjFile.getELFFile()) 876 .buildGraph(); 877 } 878 879 static SectionRangeSymbolDesc 880 identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) { 881 constexpr StringRef StartSymbolPrefix = "__start"; 882 constexpr StringRef EndSymbolPrefix = "__end"; 883 884 auto SymName = Sym.getName(); 885 if (SymName.startswith(StartSymbolPrefix)) { 886 if (auto *Sec = 887 G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size()))) 888 return {*Sec, true}; 889 } else if (SymName.startswith(EndSymbolPrefix)) { 890 if (auto *Sec = 891 G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size()))) 892 return {*Sec, false}; 893 } 894 return {}; 895 } 896 897 void link_ELF_x86_64(std::unique_ptr<LinkGraph> G, 898 std::unique_ptr<JITLinkContext> Ctx) { 899 PassConfiguration Config; 900 901 if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) { 902 903 Config.PrePrunePasses.push_back(EHFrameSplitter(".eh_frame")); 904 Config.PrePrunePasses.push_back(EHFrameEdgeFixer( 905 ".eh_frame", G->getPointerSize(), Delta64, Delta32, NegDelta32)); 906 Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame")); 907 908 // Construct a JITLinker and run the link function. 909 // Add a mark-live pass. 910 if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple())) 911 Config.PrePrunePasses.push_back(std::move(MarkLive)); 912 else 913 Config.PrePrunePasses.push_back(markAllSymbolsLive); 914 915 // Add an in-place GOT/Stubs pass. 916 Config.PostPrunePasses.push_back( 917 PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::asPass); 918 919 // Resolve any external section start / end symbols. 920 Config.PostAllocationPasses.push_back( 921 createDefineExternalSectionStartAndEndSymbolsPass( 922 identifyELFSectionStartAndEndSymbols)); 923 924 // Add GOT/Stubs optimizer pass. 925 Config.PreFixupPasses.push_back(optimizeELF_x86_64_GOTAndStubs); 926 } 927 928 if (auto Err = Ctx->modifyPassConfig(*G, Config)) 929 return Ctx->notifyFailed(std::move(Err)); 930 931 ELFJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config)); 932 } 933 const char *getELFX86RelocationKindName(Edge::Kind R) { 934 switch (R) { 935 case Branch32: 936 return "Branch32"; 937 case Branch32ToStub: 938 return "Branch32ToStub"; 939 case Pointer32: 940 return "Pointer32"; 941 case Pointer64: 942 return "Pointer64"; 943 case Pointer64Anon: 944 return "Pointer64Anon"; 945 case PCRel32: 946 return "PCRel32"; 947 case PCRel32Minus1: 948 return "PCRel32Minus1"; 949 case PCRel32Minus2: 950 return "PCRel32Minus2"; 951 case PCRel32Minus4: 952 return "PCRel32Minus4"; 953 case PCRel32Anon: 954 return "PCRel32Anon"; 955 case PCRel32Minus1Anon: 956 return "PCRel32Minus1Anon"; 957 case PCRel32Minus2Anon: 958 return "PCRel32Minus2Anon"; 959 case PCRel32Minus4Anon: 960 return "PCRel32Minus4Anon"; 961 case PCRel32GOTLoad: 962 return "PCRel32GOTLoad"; 963 case PCRel32GOT: 964 return "PCRel32GOT"; 965 case PCRel32TLV: 966 return "PCRel32TLV"; 967 case Delta32: 968 return "Delta32"; 969 case Delta64: 970 return "Delta64"; 971 case NegDelta32: 972 return "NegDelta32"; 973 case NegDelta64: 974 return "NegDelta64"; 975 } 976 return getGenericEdgeKindName(static_cast<Edge::Kind>(R)); 977 } 978 } // end namespace jitlink 979 } // end namespace llvm 980