1 //===- ARM64.cpp ----------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "Arch/ARM64Common.h" 10 #include "InputFiles.h" 11 #include "Symbols.h" 12 #include "SyntheticSections.h" 13 #include "Target.h" 14 15 #include "lld/Common/ErrorHandler.h" 16 #include "mach-o/compact_unwind_encoding.h" 17 #include "llvm/ADT/SmallVector.h" 18 #include "llvm/ADT/StringRef.h" 19 #include "llvm/BinaryFormat/MachO.h" 20 #include "llvm/Support/Endian.h" 21 #include "llvm/Support/MathExtras.h" 22 23 using namespace llvm; 24 using namespace llvm::MachO; 25 using namespace llvm::support::endian; 26 using namespace lld; 27 using namespace lld::macho; 28 29 namespace { 30 31 struct ARM64 : ARM64Common { 32 ARM64(); 33 void writeStub(uint8_t *buf, const Symbol &) const override; 34 void writeStubHelperHeader(uint8_t *buf) const override; 35 void writeStubHelperEntry(uint8_t *buf, const Symbol &, 36 uint64_t entryAddr) const override; 37 const RelocAttrs &getRelocAttrs(uint8_t type) const override; 38 void populateThunk(InputSection *thunk, Symbol *funcSym) override; 39 void applyOptimizationHints(uint8_t *, const ConcatInputSection *, 40 ArrayRef<uint64_t>) const override; 41 }; 42 43 } // namespace 44 45 // Random notes on reloc types: 46 // ADDEND always pairs with BRANCH26, PAGE21, or PAGEOFF12 47 // POINTER_TO_GOT: ld64 supports a 4-byte pc-relative form as well as an 8-byte 48 // absolute version of this relocation. The semantics of the absolute relocation 49 // are weird -- it results in the value of the GOT slot being written, instead 50 // of the address. Let's not support it unless we find a real-world use case. 51 52 const RelocAttrs &ARM64::getRelocAttrs(uint8_t type) const { 53 static const std::array<RelocAttrs, 11> relocAttrsArray{{ 54 #define B(x) RelocAttrBits::x 55 {"UNSIGNED", 56 B(UNSIGNED) | B(ABSOLUTE) | B(EXTERN) | B(LOCAL) | B(BYTE4) | B(BYTE8)}, 57 {"SUBTRACTOR", B(SUBTRAHEND) | B(EXTERN) | B(BYTE4) | B(BYTE8)}, 58 {"BRANCH26", B(PCREL) | B(EXTERN) | B(BRANCH) | B(BYTE4)}, 59 {"PAGE21", B(PCREL) | B(EXTERN) | B(BYTE4)}, 60 {"PAGEOFF12", B(ABSOLUTE) | B(EXTERN) | B(BYTE4)}, 61 {"GOT_LOAD_PAGE21", B(PCREL) | B(EXTERN) | B(GOT) | B(BYTE4)}, 62 {"GOT_LOAD_PAGEOFF12", 63 B(ABSOLUTE) | B(EXTERN) | B(GOT) | B(LOAD) | B(BYTE4)}, 64 {"POINTER_TO_GOT", B(PCREL) | B(EXTERN) | B(GOT) | B(POINTER) | B(BYTE4)}, 65 {"TLVP_LOAD_PAGE21", B(PCREL) | B(EXTERN) | B(TLV) | B(BYTE4)}, 66 {"TLVP_LOAD_PAGEOFF12", 67 B(ABSOLUTE) | B(EXTERN) | B(TLV) | B(LOAD) | B(BYTE4)}, 68 {"ADDEND", B(ADDEND)}, 69 #undef B 70 }}; 71 assert(type < relocAttrsArray.size() && "invalid relocation type"); 72 if (type >= relocAttrsArray.size()) 73 return invalidRelocAttrs; 74 return relocAttrsArray[type]; 75 } 76 77 static constexpr uint32_t stubCode[] = { 78 0x90000010, // 00: adrp x16, __la_symbol_ptr@page 79 0xf9400210, // 04: ldr x16, [x16, __la_symbol_ptr@pageoff] 80 0xd61f0200, // 08: br x16 81 }; 82 83 void ARM64::writeStub(uint8_t *buf8, const Symbol &sym) const { 84 ::writeStub<LP64>(buf8, stubCode, sym); 85 } 86 87 static constexpr uint32_t stubHelperHeaderCode[] = { 88 0x90000011, // 00: adrp x17, _dyld_private@page 89 0x91000231, // 04: add x17, x17, _dyld_private@pageoff 90 0xa9bf47f0, // 08: stp x16/x17, [sp, #-16]! 91 0x90000010, // 0c: adrp x16, dyld_stub_binder@page 92 0xf9400210, // 10: ldr x16, [x16, dyld_stub_binder@pageoff] 93 0xd61f0200, // 14: br x16 94 }; 95 96 void ARM64::writeStubHelperHeader(uint8_t *buf8) const { 97 ::writeStubHelperHeader<LP64>(buf8, stubHelperHeaderCode); 98 } 99 100 static constexpr uint32_t stubHelperEntryCode[] = { 101 0x18000050, // 00: ldr w16, l0 102 0x14000000, // 04: b stubHelperHeader 103 0x00000000, // 08: l0: .long 0 104 }; 105 106 void ARM64::writeStubHelperEntry(uint8_t *buf8, const Symbol &sym, 107 uint64_t entryVA) const { 108 ::writeStubHelperEntry(buf8, stubHelperEntryCode, sym, entryVA); 109 } 110 111 // A thunk is the relaxed variation of stubCode. We don't need the 112 // extra indirection through a lazy pointer because the target address 113 // is known at link time. 114 static constexpr uint32_t thunkCode[] = { 115 0x90000010, // 00: adrp x16, <thunk.ptr>@page 116 0x91000210, // 04: add x16, [x16,<thunk.ptr>@pageoff] 117 0xd61f0200, // 08: br x16 118 }; 119 120 void ARM64::populateThunk(InputSection *thunk, Symbol *funcSym) { 121 thunk->align = 4; 122 thunk->data = {reinterpret_cast<const uint8_t *>(thunkCode), 123 sizeof(thunkCode)}; 124 thunk->relocs.push_back({/*type=*/ARM64_RELOC_PAGEOFF12, 125 /*pcrel=*/false, /*length=*/2, 126 /*offset=*/4, /*addend=*/0, 127 /*referent=*/funcSym}); 128 thunk->relocs.push_back({/*type=*/ARM64_RELOC_PAGE21, 129 /*pcrel=*/true, /*length=*/2, 130 /*offset=*/0, /*addend=*/0, 131 /*referent=*/funcSym}); 132 } 133 134 ARM64::ARM64() : ARM64Common(LP64()) { 135 cpuType = CPU_TYPE_ARM64; 136 cpuSubtype = CPU_SUBTYPE_ARM64_ALL; 137 138 stubSize = sizeof(stubCode); 139 thunkSize = sizeof(thunkCode); 140 141 // Branch immediate is two's complement 26 bits, which is implicitly 142 // multiplied by 4 (since all functions are 4-aligned: The branch range 143 // is -4*(2**(26-1))..4*(2**(26-1) - 1). 144 backwardBranchRange = 128 * 1024 * 1024; 145 forwardBranchRange = backwardBranchRange - 4; 146 147 modeDwarfEncoding = UNWIND_ARM64_MODE_DWARF; 148 subtractorRelocType = ARM64_RELOC_SUBTRACTOR; 149 unsignedRelocType = ARM64_RELOC_UNSIGNED; 150 151 stubHelperHeaderSize = sizeof(stubHelperHeaderCode); 152 stubHelperEntrySize = sizeof(stubHelperEntryCode); 153 } 154 155 namespace { 156 struct Adrp { 157 uint32_t destRegister; 158 }; 159 160 struct Add { 161 uint8_t destRegister; 162 uint8_t srcRegister; 163 uint32_t addend; 164 }; 165 166 struct PerformedReloc { 167 const Reloc &rel; 168 uint64_t referentVA; 169 }; 170 171 class OptimizationHintContext { 172 public: 173 OptimizationHintContext(uint8_t *buf, const ConcatInputSection *isec, 174 ArrayRef<uint64_t> relocTargets) 175 : buf(buf), isec(isec), relocTargets(relocTargets), 176 relocIt(isec->relocs.rbegin()) {} 177 178 void applyAdrpAdd(const OptimizationHint &); 179 void applyAdrpAdrp(const OptimizationHint &); 180 181 private: 182 uint8_t *buf; 183 const ConcatInputSection *isec; 184 ArrayRef<uint64_t> relocTargets; 185 std::vector<Reloc>::const_reverse_iterator relocIt; 186 187 uint64_t getRelocTarget(const Reloc &); 188 189 Optional<PerformedReloc> findPrimaryReloc(uint64_t offset); 190 Optional<PerformedReloc> findReloc(uint64_t offset); 191 }; 192 } // namespace 193 194 static bool parseAdrp(uint32_t insn, Adrp &adrp) { 195 if ((insn & 0x9f000000) != 0x90000000) 196 return false; 197 adrp.destRegister = insn & 0x1f; 198 return true; 199 } 200 201 static bool parseAdd(uint32_t insn, Add &add) { 202 if ((insn & 0xffc00000) != 0x91000000) 203 return false; 204 add.destRegister = insn & 0x1f; 205 add.srcRegister = (insn >> 5) & 0x1f; 206 add.addend = (insn >> 10) & 0xfff; 207 return true; 208 } 209 210 static void writeAdr(void *loc, uint32_t dest, int32_t delta) { 211 uint32_t opcode = 0x10000000; 212 uint32_t immHi = (delta & 0x001ffffc) << 3; 213 uint32_t immLo = (delta & 0x00000003) << 29; 214 write32le(loc, opcode | immHi | immLo | dest); 215 } 216 217 static void writeNop(void *loc) { write32le(loc, 0xd503201f); } 218 219 uint64_t OptimizationHintContext::getRelocTarget(const Reloc &reloc) { 220 size_t relocIdx = &reloc - isec->relocs.data(); 221 return relocTargets[relocIdx]; 222 } 223 224 // Optimization hints are sorted in a monotonically increasing order by their 225 // first address as are relocations (albeit in decreasing order), so if we keep 226 // a pointer around to the last found relocation, we don't have to do a full 227 // binary search every time. 228 Optional<PerformedReloc> 229 OptimizationHintContext::findPrimaryReloc(uint64_t offset) { 230 const auto end = isec->relocs.rend(); 231 while (relocIt != end && relocIt->offset < offset) 232 ++relocIt; 233 if (relocIt == end || relocIt->offset != offset) 234 return None; 235 return PerformedReloc{*relocIt, getRelocTarget(*relocIt)}; 236 } 237 238 // The second and third addresses of optimization hints have no such 239 // monotonicity as the first, so we search the entire range of relocations. 240 Optional<PerformedReloc> OptimizationHintContext::findReloc(uint64_t offset) { 241 // Optimization hints often apply to successive relocations, so we check for 242 // that first before doing a full binary search. 243 auto end = isec->relocs.rend(); 244 if (relocIt < end - 1 && (relocIt + 1)->offset == offset) 245 return PerformedReloc{*(relocIt + 1), getRelocTarget(*(relocIt + 1))}; 246 247 auto reloc = lower_bound(isec->relocs, offset, 248 [](const Reloc &reloc, uint64_t offset) { 249 return offset < reloc.offset; 250 }); 251 252 if (reloc == isec->relocs.end() || reloc->offset != offset) 253 return None; 254 return PerformedReloc{*reloc, getRelocTarget(*reloc)}; 255 } 256 257 // Transforms a pair of adrp+add instructions into an adr instruction if the 258 // target is within the +/- 1 MiB range allowed by the adr's 21 bit signed 259 // immediate offset. 260 // 261 // adrp xN, _foo@PAGE 262 // add xM, xN, _foo@PAGEOFF 263 // -> 264 // adr xM, _foo 265 // nop 266 void OptimizationHintContext::applyAdrpAdd(const OptimizationHint &hint) { 267 uint32_t ins1 = read32le(buf + hint.offset0); 268 uint32_t ins2 = read32le(buf + hint.offset0 + hint.delta[0]); 269 Adrp adrp; 270 if (!parseAdrp(ins1, adrp)) 271 return; 272 Add add; 273 if (!parseAdd(ins2, add)) 274 return; 275 if (adrp.destRegister != add.srcRegister) 276 return; 277 278 Optional<PerformedReloc> rel1 = findPrimaryReloc(hint.offset0); 279 Optional<PerformedReloc> rel2 = findReloc(hint.offset0 + hint.delta[0]); 280 if (!rel1 || !rel2) 281 return; 282 if (rel1->referentVA != rel2->referentVA) 283 return; 284 int64_t delta = rel1->referentVA - rel1->rel.offset - isec->getVA(); 285 if (delta >= (1 << 20) || delta < -(1 << 20)) 286 return; 287 288 writeAdr(buf + hint.offset0, add.destRegister, delta); 289 writeNop(buf + hint.offset0 + hint.delta[0]); 290 } 291 292 // Transforms two adrp instructions into a single adrp if their referent 293 // addresses are located on the same 4096 byte page. 294 // 295 // adrp xN, _foo@PAGE 296 // adrp xN, _bar@PAGE 297 // -> 298 // adrp xN, _foo@PAGE 299 // nop 300 void OptimizationHintContext::applyAdrpAdrp(const OptimizationHint &hint) { 301 uint32_t ins1 = read32le(buf + hint.offset0); 302 uint32_t ins2 = read32le(buf + hint.offset0 + hint.delta[0]); 303 Adrp adrp1, adrp2; 304 if (!parseAdrp(ins1, adrp1) || !parseAdrp(ins2, adrp2)) 305 return; 306 if (adrp1.destRegister != adrp2.destRegister) 307 return; 308 309 Optional<PerformedReloc> rel1 = findPrimaryReloc(hint.offset0); 310 Optional<PerformedReloc> rel2 = findReloc(hint.offset0 + hint.delta[0]); 311 if (!rel1 || !rel2) 312 return; 313 if ((rel1->referentVA & ~0xfffULL) != (rel2->referentVA & ~0xfffULL)) 314 return; 315 316 writeNop(buf + hint.offset0 + hint.delta[0]); 317 } 318 319 void ARM64::applyOptimizationHints(uint8_t *buf, const ConcatInputSection *isec, 320 ArrayRef<uint64_t> relocTargets) const { 321 assert(isec); 322 assert(relocTargets.size() == isec->relocs.size()); 323 324 // Note: Some of these optimizations might not be valid when shared regions 325 // are in use. Will need to revisit this if splitSegInfo is added. 326 327 OptimizationHintContext ctx1(buf, isec, relocTargets); 328 for (const OptimizationHint &hint : isec->optimizationHints) { 329 switch (hint.type) { 330 case LOH_ARM64_ADRP_ADRP: 331 // This is done in another pass because the other optimization hints 332 // might cause its targets to be turned into NOPs. 333 break; 334 case LOH_ARM64_ADRP_LDR: 335 case LOH_ARM64_ADRP_ADD_LDR: 336 case LOH_ARM64_ADRP_LDR_GOT_LDR: 337 case LOH_ARM64_ADRP_ADD_STR: 338 case LOH_ARM64_ADRP_LDR_GOT_STR: 339 // TODO: Implement these 340 break; 341 case LOH_ARM64_ADRP_ADD: 342 ctx1.applyAdrpAdd(hint); 343 break; 344 case LOH_ARM64_ADRP_LDR_GOT: 345 // TODO: Implement this as well 346 break; 347 } 348 } 349 350 OptimizationHintContext ctx2(buf, isec, relocTargets); 351 for (const OptimizationHint &hint : isec->optimizationHints) 352 if (hint.type == LOH_ARM64_ADRP_ADRP) 353 ctx2.applyAdrpAdrp(hint); 354 } 355 356 TargetInfo *macho::createARM64TargetInfo() { 357 static ARM64 t; 358 return &t; 359 } 360