1 //===- AArch64.cpp --------------------------------------------------------===// 2 // 3 // The LLVM Linker 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 10 #include "Symbols.h" 11 #include "SyntheticSections.h" 12 #include "Target.h" 13 #include "Thunks.h" 14 #include "lld/Common/ErrorHandler.h" 15 #include "llvm/Object/ELF.h" 16 #include "llvm/Support/Endian.h" 17 18 using namespace llvm; 19 using namespace llvm::support::endian; 20 using namespace llvm::ELF; 21 using namespace lld; 22 using namespace lld::elf; 23 24 // Page(Expr) is the page address of the expression Expr, defined 25 // as (Expr & ~0xFFF). (This applies even if the machine page size 26 // supported by the platform has a different value.) 27 uint64_t elf::getAArch64Page(uint64_t Expr) { 28 return Expr & ~static_cast<uint64_t>(0xFFF); 29 } 30 31 namespace { 32 class AArch64 final : public TargetInfo { 33 public: 34 AArch64(); 35 RelExpr getRelExpr(RelType Type, const Symbol &S, 36 const uint8_t *Loc) const override; 37 RelType getDynRel(RelType Type) const override; 38 void writeGotPlt(uint8_t *Buf, const Symbol &S) const override; 39 void writePltHeader(uint8_t *Buf) const override; 40 void writePlt(uint8_t *Buf, uint64_t GotPltEntryAddr, uint64_t PltEntryAddr, 41 int32_t Index, unsigned RelOff) const override; 42 bool needsThunk(RelExpr Expr, RelType Type, const InputFile *File, 43 uint64_t BranchAddr, const Symbol &S) const override; 44 uint32_t getThunkSectionSpacing() const override; 45 bool inBranchRange(RelType Type, uint64_t Src, uint64_t Dst) const override; 46 bool usesOnlyLowPageBits(RelType Type) const override; 47 void relocateOne(uint8_t *Loc, RelType Type, uint64_t Val) const override; 48 RelExpr adjustRelaxExpr(RelType Type, const uint8_t *Data, 49 RelExpr Expr) const override; 50 void relaxTlsGdToLe(uint8_t *Loc, RelType Type, uint64_t Val) const override; 51 void relaxTlsGdToIe(uint8_t *Loc, RelType Type, uint64_t Val) const override; 52 void relaxTlsIeToLe(uint8_t *Loc, RelType Type, uint64_t Val) const override; 53 }; 54 } // namespace 55 56 AArch64::AArch64() { 57 CopyRel = R_AARCH64_COPY; 58 RelativeRel = R_AARCH64_RELATIVE; 59 IRelativeRel = R_AARCH64_IRELATIVE; 60 GotRel = R_AARCH64_GLOB_DAT; 61 NoneRel = R_AARCH64_NONE; 62 PltRel = R_AARCH64_JUMP_SLOT; 63 TlsDescRel = R_AARCH64_TLSDESC; 64 TlsGotRel = R_AARCH64_TLS_TPREL64; 65 GotEntrySize = 8; 66 GotPltEntrySize = 8; 67 PltEntrySize = 16; 68 PltHeaderSize = 32; 69 DefaultMaxPageSize = 65536; 70 71 // Align to the 2 MiB page size (known as a superpage or huge page). 72 // FreeBSD automatically promotes 2 MiB-aligned allocations. 73 DefaultImageBase = 0x200000; 74 75 // It doesn't seem to be documented anywhere, but tls on aarch64 uses variant 76 // 1 of the tls structures and the tcb size is 16. 77 TcbSize = 16; 78 NeedsThunks = true; 79 } 80 81 RelExpr AArch64::getRelExpr(RelType Type, const Symbol &S, 82 const uint8_t *Loc) const { 83 switch (Type) { 84 case R_AARCH64_TLSDESC_ADR_PAGE21: 85 return R_TLSDESC_PAGE; 86 case R_AARCH64_TLSDESC_LD64_LO12: 87 case R_AARCH64_TLSDESC_ADD_LO12: 88 return R_TLSDESC; 89 case R_AARCH64_TLSDESC_CALL: 90 return R_TLSDESC_CALL; 91 case R_AARCH64_TLSLE_ADD_TPREL_HI12: 92 case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC: 93 case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC: 94 case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC: 95 case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC: 96 case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC: 97 case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC: 98 return R_TLS; 99 case R_AARCH64_CALL26: 100 case R_AARCH64_CONDBR19: 101 case R_AARCH64_JUMP26: 102 case R_AARCH64_TSTBR14: 103 return R_PLT_PC; 104 case R_AARCH64_PREL16: 105 case R_AARCH64_PREL32: 106 case R_AARCH64_PREL64: 107 case R_AARCH64_ADR_PREL_LO21: 108 case R_AARCH64_LD_PREL_LO19: 109 return R_PC; 110 case R_AARCH64_ADR_PREL_PG_HI21: 111 return R_PAGE_PC; 112 case R_AARCH64_LD64_GOT_LO12_NC: 113 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 114 return R_GOT; 115 case R_AARCH64_ADR_GOT_PAGE: 116 case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21: 117 return R_GOT_PAGE_PC; 118 case R_AARCH64_NONE: 119 return R_NONE; 120 default: 121 return R_ABS; 122 } 123 } 124 125 RelExpr AArch64::adjustRelaxExpr(RelType Type, const uint8_t *Data, 126 RelExpr Expr) const { 127 if (Expr == R_RELAX_TLS_GD_TO_IE) { 128 if (Type == R_AARCH64_TLSDESC_ADR_PAGE21) 129 return R_RELAX_TLS_GD_TO_IE_PAGE_PC; 130 return R_RELAX_TLS_GD_TO_IE_ABS; 131 } 132 return Expr; 133 } 134 135 bool AArch64::usesOnlyLowPageBits(RelType Type) const { 136 switch (Type) { 137 default: 138 return false; 139 case R_AARCH64_ADD_ABS_LO12_NC: 140 case R_AARCH64_LD64_GOT_LO12_NC: 141 case R_AARCH64_LDST128_ABS_LO12_NC: 142 case R_AARCH64_LDST16_ABS_LO12_NC: 143 case R_AARCH64_LDST32_ABS_LO12_NC: 144 case R_AARCH64_LDST64_ABS_LO12_NC: 145 case R_AARCH64_LDST8_ABS_LO12_NC: 146 case R_AARCH64_TLSDESC_ADD_LO12: 147 case R_AARCH64_TLSDESC_LD64_LO12: 148 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 149 return true; 150 } 151 } 152 153 RelType AArch64::getDynRel(RelType Type) const { 154 if (Type == R_AARCH64_ABS32 || Type == R_AARCH64_ABS64) 155 return Type; 156 return R_AARCH64_NONE; 157 } 158 159 void AArch64::writeGotPlt(uint8_t *Buf, const Symbol &) const { 160 write64le(Buf, In.Plt->getVA()); 161 } 162 163 void AArch64::writePltHeader(uint8_t *Buf) const { 164 const uint8_t PltData[] = { 165 0xf0, 0x7b, 0xbf, 0xa9, // stp x16, x30, [sp,#-16]! 166 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[2])) 167 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[2]))] 168 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.plt.got[2])) 169 0x20, 0x02, 0x1f, 0xd6, // br x17 170 0x1f, 0x20, 0x03, 0xd5, // nop 171 0x1f, 0x20, 0x03, 0xd5, // nop 172 0x1f, 0x20, 0x03, 0xd5 // nop 173 }; 174 memcpy(Buf, PltData, sizeof(PltData)); 175 176 uint64_t Got = In.GotPlt->getVA(); 177 uint64_t Plt = In.Plt->getVA(); 178 relocateOne(Buf + 4, R_AARCH64_ADR_PREL_PG_HI21, 179 getAArch64Page(Got + 16) - getAArch64Page(Plt + 4)); 180 relocateOne(Buf + 8, R_AARCH64_LDST64_ABS_LO12_NC, Got + 16); 181 relocateOne(Buf + 12, R_AARCH64_ADD_ABS_LO12_NC, Got + 16); 182 } 183 184 void AArch64::writePlt(uint8_t *Buf, uint64_t GotPltEntryAddr, 185 uint64_t PltEntryAddr, int32_t Index, 186 unsigned RelOff) const { 187 const uint8_t Inst[] = { 188 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[n])) 189 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[n]))] 190 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.plt.got[n])) 191 0x20, 0x02, 0x1f, 0xd6 // br x17 192 }; 193 memcpy(Buf, Inst, sizeof(Inst)); 194 195 relocateOne(Buf, R_AARCH64_ADR_PREL_PG_HI21, 196 getAArch64Page(GotPltEntryAddr) - getAArch64Page(PltEntryAddr)); 197 relocateOne(Buf + 4, R_AARCH64_LDST64_ABS_LO12_NC, GotPltEntryAddr); 198 relocateOne(Buf + 8, R_AARCH64_ADD_ABS_LO12_NC, GotPltEntryAddr); 199 } 200 201 bool AArch64::needsThunk(RelExpr Expr, RelType Type, const InputFile *File, 202 uint64_t BranchAddr, const Symbol &S) const { 203 // ELF for the ARM 64-bit architecture, section Call and Jump relocations 204 // only permits range extension thunks for R_AARCH64_CALL26 and 205 // R_AARCH64_JUMP26 relocation types. 206 if (Type != R_AARCH64_CALL26 && Type != R_AARCH64_JUMP26) 207 return false; 208 uint64_t Dst = (Expr == R_PLT_PC) ? S.getPltVA() : S.getVA(); 209 return !inBranchRange(Type, BranchAddr, Dst); 210 } 211 212 uint32_t AArch64::getThunkSectionSpacing() const { 213 // See comment in Arch/ARM.cpp for a more detailed explanation of 214 // getThunkSectionSpacing(). For AArch64 the only branches we are permitted to 215 // Thunk have a range of +/- 128 MiB 216 return (128 * 1024 * 1024) - 0x30000; 217 } 218 219 bool AArch64::inBranchRange(RelType Type, uint64_t Src, uint64_t Dst) const { 220 if (Type != R_AARCH64_CALL26 && Type != R_AARCH64_JUMP26) 221 return true; 222 // The AArch64 call and unconditional branch instructions have a range of 223 // +/- 128 MiB. 224 uint64_t Range = 128 * 1024 * 1024; 225 if (Dst > Src) { 226 // Immediate of branch is signed. 227 Range -= 4; 228 return Dst - Src <= Range; 229 } 230 return Src - Dst <= Range; 231 } 232 233 static void write32AArch64Addr(uint8_t *L, uint64_t Imm) { 234 uint32_t ImmLo = (Imm & 0x3) << 29; 235 uint32_t ImmHi = (Imm & 0x1FFFFC) << 3; 236 uint64_t Mask = (0x3 << 29) | (0x1FFFFC << 3); 237 write32le(L, (read32le(L) & ~Mask) | ImmLo | ImmHi); 238 } 239 240 // Return the bits [Start, End] from Val shifted Start bits. 241 // For instance, getBits(0xF0, 4, 8) returns 0xF. 242 static uint64_t getBits(uint64_t Val, int Start, int End) { 243 uint64_t Mask = ((uint64_t)1 << (End + 1 - Start)) - 1; 244 return (Val >> Start) & Mask; 245 } 246 247 static void or32le(uint8_t *P, int32_t V) { write32le(P, read32le(P) | V); } 248 249 // Update the immediate field in a AARCH64 ldr, str, and add instruction. 250 static void or32AArch64Imm(uint8_t *L, uint64_t Imm) { 251 or32le(L, (Imm & 0xFFF) << 10); 252 } 253 254 void AArch64::relocateOne(uint8_t *Loc, RelType Type, uint64_t Val) const { 255 switch (Type) { 256 case R_AARCH64_ABS16: 257 case R_AARCH64_PREL16: 258 checkIntUInt(Loc, Val, 16, Type); 259 write16le(Loc, Val); 260 break; 261 case R_AARCH64_ABS32: 262 case R_AARCH64_PREL32: 263 checkIntUInt(Loc, Val, 32, Type); 264 write32le(Loc, Val); 265 break; 266 case R_AARCH64_ABS64: 267 case R_AARCH64_GLOB_DAT: 268 case R_AARCH64_PREL64: 269 write64le(Loc, Val); 270 break; 271 case R_AARCH64_ADD_ABS_LO12_NC: 272 or32AArch64Imm(Loc, Val); 273 break; 274 case R_AARCH64_ADR_GOT_PAGE: 275 case R_AARCH64_ADR_PREL_PG_HI21: 276 case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21: 277 case R_AARCH64_TLSDESC_ADR_PAGE21: 278 checkInt(Loc, Val, 33, Type); 279 write32AArch64Addr(Loc, Val >> 12); 280 break; 281 case R_AARCH64_ADR_PREL_LO21: 282 checkInt(Loc, Val, 21, Type); 283 write32AArch64Addr(Loc, Val); 284 break; 285 case R_AARCH64_JUMP26: 286 // Normally we would just write the bits of the immediate field, however 287 // when patching instructions for the cpu errata fix -fix-cortex-a53-843419 288 // we want to replace a non-branch instruction with a branch immediate 289 // instruction. By writing all the bits of the instruction including the 290 // opcode and the immediate (0 001 | 01 imm26) we can do this 291 // transformation by placing a R_AARCH64_JUMP26 relocation at the offset of 292 // the instruction we want to patch. 293 write32le(Loc, 0x14000000); 294 LLVM_FALLTHROUGH; 295 case R_AARCH64_CALL26: 296 checkInt(Loc, Val, 28, Type); 297 or32le(Loc, (Val & 0x0FFFFFFC) >> 2); 298 break; 299 case R_AARCH64_CONDBR19: 300 case R_AARCH64_LD_PREL_LO19: 301 checkAlignment(Loc, Val, 4, Type); 302 checkInt(Loc, Val, 21, Type); 303 or32le(Loc, (Val & 0x1FFFFC) << 3); 304 break; 305 case R_AARCH64_LDST8_ABS_LO12_NC: 306 case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC: 307 or32AArch64Imm(Loc, getBits(Val, 0, 11)); 308 break; 309 case R_AARCH64_LDST16_ABS_LO12_NC: 310 case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC: 311 checkAlignment(Loc, Val, 2, Type); 312 or32AArch64Imm(Loc, getBits(Val, 1, 11)); 313 break; 314 case R_AARCH64_LDST32_ABS_LO12_NC: 315 case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC: 316 checkAlignment(Loc, Val, 4, Type); 317 or32AArch64Imm(Loc, getBits(Val, 2, 11)); 318 break; 319 case R_AARCH64_LDST64_ABS_LO12_NC: 320 case R_AARCH64_LD64_GOT_LO12_NC: 321 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 322 case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC: 323 case R_AARCH64_TLSDESC_LD64_LO12: 324 checkAlignment(Loc, Val, 8, Type); 325 or32AArch64Imm(Loc, getBits(Val, 3, 11)); 326 break; 327 case R_AARCH64_LDST128_ABS_LO12_NC: 328 case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC: 329 checkAlignment(Loc, Val, 16, Type); 330 or32AArch64Imm(Loc, getBits(Val, 4, 11)); 331 break; 332 case R_AARCH64_MOVW_UABS_G0_NC: 333 or32le(Loc, (Val & 0xFFFF) << 5); 334 break; 335 case R_AARCH64_MOVW_UABS_G1_NC: 336 or32le(Loc, (Val & 0xFFFF0000) >> 11); 337 break; 338 case R_AARCH64_MOVW_UABS_G2_NC: 339 or32le(Loc, (Val & 0xFFFF00000000) >> 27); 340 break; 341 case R_AARCH64_MOVW_UABS_G3: 342 or32le(Loc, (Val & 0xFFFF000000000000) >> 43); 343 break; 344 case R_AARCH64_TSTBR14: 345 checkInt(Loc, Val, 16, Type); 346 or32le(Loc, (Val & 0xFFFC) << 3); 347 break; 348 case R_AARCH64_TLSLE_ADD_TPREL_HI12: 349 checkUInt(Loc, Val, 24, Type); 350 or32AArch64Imm(Loc, Val >> 12); 351 break; 352 case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC: 353 case R_AARCH64_TLSDESC_ADD_LO12: 354 or32AArch64Imm(Loc, Val); 355 break; 356 default: 357 error(getErrorLocation(Loc) + "unrecognized reloc " + Twine(Type)); 358 } 359 } 360 361 void AArch64::relaxTlsGdToLe(uint8_t *Loc, RelType Type, uint64_t Val) const { 362 // TLSDESC Global-Dynamic relocation are in the form: 363 // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21] 364 // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12] 365 // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12] 366 // .tlsdesccall [R_AARCH64_TLSDESC_CALL] 367 // blr x1 368 // And it can optimized to: 369 // movz x0, #0x0, lsl #16 370 // movk x0, #0x10 371 // nop 372 // nop 373 checkUInt(Loc, Val, 32, Type); 374 375 switch (Type) { 376 case R_AARCH64_TLSDESC_ADD_LO12: 377 case R_AARCH64_TLSDESC_CALL: 378 write32le(Loc, 0xd503201f); // nop 379 return; 380 case R_AARCH64_TLSDESC_ADR_PAGE21: 381 write32le(Loc, 0xd2a00000 | (((Val >> 16) & 0xffff) << 5)); // movz 382 return; 383 case R_AARCH64_TLSDESC_LD64_LO12: 384 write32le(Loc, 0xf2800000 | ((Val & 0xffff) << 5)); // movk 385 return; 386 default: 387 llvm_unreachable("unsupported relocation for TLS GD to LE relaxation"); 388 } 389 } 390 391 void AArch64::relaxTlsGdToIe(uint8_t *Loc, RelType Type, uint64_t Val) const { 392 // TLSDESC Global-Dynamic relocation are in the form: 393 // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21] 394 // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12] 395 // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12] 396 // .tlsdesccall [R_AARCH64_TLSDESC_CALL] 397 // blr x1 398 // And it can optimized to: 399 // adrp x0, :gottprel:v 400 // ldr x0, [x0, :gottprel_lo12:v] 401 // nop 402 // nop 403 404 switch (Type) { 405 case R_AARCH64_TLSDESC_ADD_LO12: 406 case R_AARCH64_TLSDESC_CALL: 407 write32le(Loc, 0xd503201f); // nop 408 break; 409 case R_AARCH64_TLSDESC_ADR_PAGE21: 410 write32le(Loc, 0x90000000); // adrp 411 relocateOne(Loc, R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21, Val); 412 break; 413 case R_AARCH64_TLSDESC_LD64_LO12: 414 write32le(Loc, 0xf9400000); // ldr 415 relocateOne(Loc, R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC, Val); 416 break; 417 default: 418 llvm_unreachable("unsupported relocation for TLS GD to LE relaxation"); 419 } 420 } 421 422 void AArch64::relaxTlsIeToLe(uint8_t *Loc, RelType Type, uint64_t Val) const { 423 checkUInt(Loc, Val, 32, Type); 424 425 if (Type == R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21) { 426 // Generate MOVZ. 427 uint32_t RegNo = read32le(Loc) & 0x1f; 428 write32le(Loc, (0xd2a00000 | RegNo) | (((Val >> 16) & 0xffff) << 5)); 429 return; 430 } 431 if (Type == R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC) { 432 // Generate MOVK. 433 uint32_t RegNo = read32le(Loc) & 0x1f; 434 write32le(Loc, (0xf2800000 | RegNo) | ((Val & 0xffff) << 5)); 435 return; 436 } 437 llvm_unreachable("invalid relocation for TLS IE to LE relaxation"); 438 } 439 440 TargetInfo *elf::getAArch64TargetInfo() { 441 static AArch64 Target; 442 return &Target; 443 } 444