1 //===- ARM64.cpp ----------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "InputFiles.h" 10 #include "Symbols.h" 11 #include "SyntheticSections.h" 12 #include "Target.h" 13 14 #include "lld/Common/ErrorHandler.h" 15 #include "llvm/ADT/SmallVector.h" 16 #include "llvm/ADT/StringRef.h" 17 #include "llvm/BinaryFormat/MachO.h" 18 #include "llvm/Support/Endian.h" 19 #include "llvm/Support/MathExtras.h" 20 21 using namespace llvm::MachO; 22 using namespace llvm::support::endian; 23 using namespace lld; 24 using namespace lld::macho; 25 26 namespace { 27 28 struct ARM64 : TargetInfo { 29 ARM64(); 30 31 int64_t getEmbeddedAddend(MemoryBufferRef, const section_64 &, 32 const relocation_info) const override; 33 void relocateOne(uint8_t *loc, const Reloc &, uint64_t va, 34 uint64_t pc) const override; 35 36 void writeStub(uint8_t *buf, const macho::Symbol &) const override; 37 void writeStubHelperHeader(uint8_t *buf) const override; 38 void writeStubHelperEntry(uint8_t *buf, const DylibSymbol &, 39 uint64_t entryAddr) const override; 40 41 void relaxGotLoad(uint8_t *loc, uint8_t type) const override; 42 const RelocAttrs &getRelocAttrs(uint8_t type) const override; 43 uint64_t getPageSize() const override { return 16 * 1024; } 44 }; 45 46 } // namespace 47 48 // Random notes on reloc types: 49 // ADDEND always pairs with BRANCH26, PAGE21, or PAGEOFF12 50 // POINTER_TO_GOT: ld64 supports a 4-byte pc-relative form as well as an 8-byte 51 // absolute version of this relocation. The semantics of the absolute relocation 52 // are weird -- it results in the value of the GOT slot being written, instead 53 // of the address. Let's not support it unless we find a real-world use case. 54 55 const RelocAttrs &ARM64::getRelocAttrs(uint8_t type) const { 56 static const std::array<RelocAttrs, 11> relocAttrsArray{{ 57 #define B(x) RelocAttrBits::x 58 {"UNSIGNED", B(UNSIGNED) | B(ABSOLUTE) | B(EXTERN) | B(LOCAL) | 59 B(DYSYM8) | B(BYTE4) | B(BYTE8)}, 60 {"SUBTRACTOR", B(SUBTRAHEND) | B(BYTE4) | B(BYTE8)}, 61 {"BRANCH26", B(PCREL) | B(EXTERN) | B(BRANCH) | B(BYTE4)}, 62 {"PAGE21", B(PCREL) | B(EXTERN) | B(BYTE4)}, 63 {"PAGEOFF12", B(ABSOLUTE) | B(EXTERN) | B(BYTE4)}, 64 {"GOT_LOAD_PAGE21", B(PCREL) | B(EXTERN) | B(GOT) | B(BYTE4)}, 65 {"GOT_LOAD_PAGEOFF12", 66 B(ABSOLUTE) | B(EXTERN) | B(GOT) | B(LOAD) | B(BYTE4)}, 67 {"POINTER_TO_GOT", B(PCREL) | B(EXTERN) | B(GOT) | B(POINTER) | B(BYTE4)}, 68 {"TLVP_LOAD_PAGE21", B(PCREL) | B(EXTERN) | B(TLV) | B(BYTE4)}, 69 {"TLVP_LOAD_PAGEOFF12", 70 B(ABSOLUTE) | B(EXTERN) | B(TLV) | B(LOAD) | B(BYTE4)}, 71 {"ADDEND", B(ADDEND)}, 72 #undef B 73 }}; 74 assert(type < relocAttrsArray.size() && "invalid relocation type"); 75 if (type >= relocAttrsArray.size()) 76 return invalidRelocAttrs; 77 return relocAttrsArray[type]; 78 } 79 80 int64_t ARM64::getEmbeddedAddend(MemoryBufferRef mb, const section_64 &sec, 81 const relocation_info rel) const { 82 if (rel.r_type != ARM64_RELOC_UNSIGNED && 83 rel.r_type != ARM64_RELOC_SUBTRACTOR) { 84 // All other reloc types should use the ADDEND relocation to store their 85 // addends. 86 // TODO(gkm): extract embedded addend just so we can assert that it is 0 87 return 0; 88 } 89 90 auto *buf = reinterpret_cast<const uint8_t *>(mb.getBufferStart()); 91 const uint8_t *loc = buf + sec.offset + rel.r_address; 92 switch (rel.r_length) { 93 case 2: 94 return static_cast<int32_t>(read32le(loc)); 95 case 3: 96 return read64le(loc); 97 default: 98 llvm_unreachable("invalid r_length"); 99 } 100 } 101 102 inline uint64_t bitField(uint64_t value, int right, int width, int left) { 103 return ((value >> right) & ((1 << width) - 1)) << left; 104 } 105 106 // 25 0 107 // +-----------+---------------------------------------------------+ 108 // | | imm26 | 109 // +-----------+---------------------------------------------------+ 110 111 inline uint64_t encodeBranch26(const Reloc &r, uint64_t base, uint64_t va) { 112 checkInt(r, va, 28); 113 // Since branch destinations are 4-byte aligned, the 2 least- 114 // significant bits are 0. They are right shifted off the end. 115 return (base | bitField(va, 2, 26, 0)); 116 } 117 118 inline uint64_t encodeBranch26(SymbolDiagnostic d, uint64_t base, uint64_t va) { 119 checkInt(d, va, 28); 120 return (base | bitField(va, 2, 26, 0)); 121 } 122 123 // 30 29 23 5 124 // +-+---+---------+-------------------------------------+---------+ 125 // | |ilo| | immhi | | 126 // +-+---+---------+-------------------------------------+---------+ 127 128 inline uint64_t encodePage21(const Reloc &r, uint64_t base, uint64_t va) { 129 checkInt(r, va, 35); 130 return (base | bitField(va, 12, 2, 29) | bitField(va, 14, 19, 5)); 131 } 132 133 inline uint64_t encodePage21(SymbolDiagnostic d, uint64_t base, uint64_t va) { 134 checkInt(d, va, 35); 135 return (base | bitField(va, 12, 2, 29) | bitField(va, 14, 19, 5)); 136 } 137 138 // 21 10 139 // +-------------------+-----------------------+-------------------+ 140 // | | imm12 | | 141 // +-------------------+-----------------------+-------------------+ 142 143 inline uint64_t encodePageOff12(uint32_t base, uint64_t va) { 144 int scale = 0; 145 if ((base & 0x3b00'0000) == 0x3900'0000) { // load/store 146 scale = base >> 30; 147 if (scale == 0 && (base & 0x0480'0000) == 0x0480'0000) // 128-bit variant 148 scale = 4; 149 } 150 151 // TODO(gkm): extract embedded addend and warn if != 0 152 // uint64_t addend = ((base & 0x003FFC00) >> 10); 153 return (base | bitField(va, scale, 12 - scale, 10)); 154 } 155 156 inline uint64_t pageBits(uint64_t address) { 157 const uint64_t pageMask = ~0xfffull; 158 return address & pageMask; 159 } 160 161 // For instruction relocations (load, store, add), the base 162 // instruction is pre-populated in the text section. A pre-populated 163 // instruction has opcode & register-operand bits set, with immediate 164 // operands zeroed. We read it from text, OR-in the immediate 165 // operands, then write-back the completed instruction. 166 167 void ARM64::relocateOne(uint8_t *loc, const Reloc &r, uint64_t value, 168 uint64_t pc) const { 169 uint32_t base = ((r.length == 2) ? read32le(loc) : 0); 170 value += r.addend; 171 switch (r.type) { 172 case ARM64_RELOC_BRANCH26: 173 value = encodeBranch26(r, base, value - pc); 174 break; 175 case ARM64_RELOC_SUBTRACTOR: 176 case ARM64_RELOC_UNSIGNED: 177 if (r.length == 2) 178 checkInt(r, value, 32); 179 break; 180 case ARM64_RELOC_POINTER_TO_GOT: 181 if (r.pcrel) 182 value -= pc; 183 checkInt(r, value, 32); 184 break; 185 case ARM64_RELOC_PAGE21: 186 case ARM64_RELOC_GOT_LOAD_PAGE21: 187 case ARM64_RELOC_TLVP_LOAD_PAGE21: { 188 assert(r.pcrel); 189 value = encodePage21(r, base, pageBits(value) - pageBits(pc)); 190 break; 191 } 192 case ARM64_RELOC_PAGEOFF12: 193 case ARM64_RELOC_GOT_LOAD_PAGEOFF12: 194 case ARM64_RELOC_TLVP_LOAD_PAGEOFF12: 195 assert(!r.pcrel); 196 value = encodePageOff12(base, value); 197 break; 198 default: 199 llvm_unreachable("unexpected relocation type"); 200 } 201 202 switch (r.length) { 203 case 2: 204 write32le(loc, value); 205 break; 206 case 3: 207 write64le(loc, value); 208 break; 209 default: 210 llvm_unreachable("invalid r_length"); 211 } 212 } 213 214 static constexpr uint32_t stubCode[] = { 215 0x90000010, // 00: adrp x16, __la_symbol_ptr@page 216 0xf9400210, // 04: ldr x16, [x16, __la_symbol_ptr@pageoff] 217 0xd61f0200, // 08: br x16 218 }; 219 220 void ARM64::writeStub(uint8_t *buf8, const macho::Symbol &sym) const { 221 auto *buf32 = reinterpret_cast<uint32_t *>(buf8); 222 uint64_t pcPageBits = 223 pageBits(in.stubs->addr + sym.stubsIndex * sizeof(stubCode)); 224 uint64_t lazyPointerVA = in.lazyPointers->addr + sym.stubsIndex * WordSize; 225 buf32[0] = encodePage21({&sym, "stub"}, stubCode[0], 226 pageBits(lazyPointerVA) - pcPageBits); 227 buf32[1] = encodePageOff12(stubCode[1], lazyPointerVA); 228 buf32[2] = stubCode[2]; 229 } 230 231 static constexpr uint32_t stubHelperHeaderCode[] = { 232 0x90000011, // 00: adrp x17, _dyld_private@page 233 0x91000231, // 04: add x17, x17, _dyld_private@pageoff 234 0xa9bf47f0, // 08: stp x16/x17, [sp, #-16]! 235 0x90000010, // 0c: adrp x16, dyld_stub_binder@page 236 0xf9400210, // 10: ldr x16, [x16, dyld_stub_binder@pageoff] 237 0xd61f0200, // 14: br x16 238 }; 239 240 void ARM64::writeStubHelperHeader(uint8_t *buf8) const { 241 auto *buf32 = reinterpret_cast<uint32_t *>(buf8); 242 auto pcPageBits = [](int i) { 243 return pageBits(in.stubHelper->addr + i * sizeof(uint32_t)); 244 }; 245 uint64_t loaderVA = in.imageLoaderCache->getVA(); 246 SymbolDiagnostic d = {nullptr, "stub header helper"}; 247 buf32[0] = encodePage21(d, stubHelperHeaderCode[0], 248 pageBits(loaderVA) - pcPageBits(0)); 249 buf32[1] = encodePageOff12(stubHelperHeaderCode[1], loaderVA); 250 buf32[2] = stubHelperHeaderCode[2]; 251 uint64_t binderVA = 252 in.got->addr + in.stubHelper->stubBinder->gotIndex * WordSize; 253 buf32[3] = encodePage21(d, stubHelperHeaderCode[3], 254 pageBits(binderVA) - pcPageBits(3)); 255 buf32[4] = encodePageOff12(stubHelperHeaderCode[4], binderVA); 256 buf32[5] = stubHelperHeaderCode[5]; 257 } 258 259 static constexpr uint32_t stubHelperEntryCode[] = { 260 0x18000050, // 00: ldr w16, l0 261 0x14000000, // 04: b stubHelperHeader 262 0x00000000, // 08: l0: .long 0 263 }; 264 265 void ARM64::writeStubHelperEntry(uint8_t *buf8, const DylibSymbol &sym, 266 uint64_t entryVA) const { 267 auto *buf32 = reinterpret_cast<uint32_t *>(buf8); 268 auto pcVA = [entryVA](int i) { return entryVA + i * sizeof(uint32_t); }; 269 uint64_t stubHelperHeaderVA = in.stubHelper->addr; 270 buf32[0] = stubHelperEntryCode[0]; 271 buf32[1] = encodeBranch26({&sym, "stub helper"}, stubHelperEntryCode[1], 272 stubHelperHeaderVA - pcVA(1)); 273 buf32[2] = sym.lazyBindOffset; 274 } 275 276 void ARM64::relaxGotLoad(uint8_t *loc, uint8_t type) const { 277 // The instruction format comments below are quoted from 278 // Arm® Architecture Reference Manual 279 // Armv8, for Armv8-A architecture profile 280 // ARM DDI 0487G.a (ID011921) 281 uint32_t instruction = read32le(loc); 282 // C6.2.132 LDR (immediate) 283 // LDR <Xt>, [<Xn|SP>{, #<pimm>}] 284 if ((instruction & 0xffc00000) != 0xf9400000) 285 error(getRelocAttrs(type).name + " reloc requires LDR instruction"); 286 assert(((instruction >> 10) & 0xfff) == 0 && 287 "non-zero embedded LDR immediate"); 288 // C6.2.4 ADD (immediate) 289 // ADD <Xd|SP>, <Xn|SP>, #<imm>{, <shift>} 290 instruction = ((instruction & 0x001fffff) | 0x91000000); 291 write32le(loc, instruction); 292 } 293 294 ARM64::ARM64() { 295 cpuType = CPU_TYPE_ARM64; 296 cpuSubtype = CPU_SUBTYPE_ARM64_ALL; 297 298 stubSize = sizeof(stubCode); 299 stubHelperHeaderSize = sizeof(stubHelperHeaderCode); 300 stubHelperEntrySize = sizeof(stubHelperEntryCode); 301 } 302 303 TargetInfo *macho::createARM64TargetInfo() { 304 static ARM64 t; 305 return &t; 306 } 307