1 //===- AArch64.cpp --------------------------------------------------------===// 2 // 3 // The LLVM Linker 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 10 #include "Symbols.h" 11 #include "SyntheticSections.h" 12 #include "Target.h" 13 #include "Thunks.h" 14 #include "lld/Common/ErrorHandler.h" 15 #include "llvm/Object/ELF.h" 16 #include "llvm/Support/Endian.h" 17 18 using namespace llvm; 19 using namespace llvm::support::endian; 20 using namespace llvm::ELF; 21 using namespace lld; 22 using namespace lld::elf; 23 24 // Page(Expr) is the page address of the expression Expr, defined 25 // as (Expr & ~0xFFF). (This applies even if the machine page size 26 // supported by the platform has a different value.) 27 uint64_t elf::getAArch64Page(uint64_t Expr) { 28 return Expr & ~static_cast<uint64_t>(0xFFF); 29 } 30 31 namespace { 32 class AArch64 final : public TargetInfo { 33 public: 34 AArch64(); 35 RelExpr getRelExpr(RelType Type, const Symbol &S, 36 const uint8_t *Loc) const override; 37 RelType getDynRel(RelType Type) const override; 38 void writeGotPlt(uint8_t *Buf, const Symbol &S) const override; 39 void writePltHeader(uint8_t *Buf) const override; 40 void writePlt(uint8_t *Buf, uint64_t GotPltEntryAddr, uint64_t PltEntryAddr, 41 int32_t Index, unsigned RelOff) const override; 42 bool needsThunk(RelExpr Expr, RelType Type, const InputFile *File, 43 uint64_t BranchAddr, const Symbol &S) const override; 44 uint32_t getThunkSectionSpacing() const override; 45 bool inBranchRange(RelType Type, uint64_t Src, uint64_t Dst) const override; 46 bool usesOnlyLowPageBits(RelType Type) const override; 47 void relocateOne(uint8_t *Loc, RelType Type, uint64_t Val) const override; 48 RelExpr adjustRelaxExpr(RelType Type, const uint8_t *Data, 49 RelExpr Expr) const override; 50 void relaxTlsGdToLe(uint8_t *Loc, RelType Type, uint64_t Val) const override; 51 void relaxTlsGdToIe(uint8_t *Loc, RelType Type, uint64_t Val) const override; 52 void relaxTlsIeToLe(uint8_t *Loc, RelType Type, uint64_t Val) const override; 53 }; 54 } // namespace 55 56 AArch64::AArch64() { 57 CopyRel = R_AARCH64_COPY; 58 RelativeRel = R_AARCH64_RELATIVE; 59 IRelativeRel = R_AARCH64_IRELATIVE; 60 GotRel = R_AARCH64_GLOB_DAT; 61 NoneRel = R_AARCH64_NONE; 62 PltRel = R_AARCH64_JUMP_SLOT; 63 TlsDescRel = R_AARCH64_TLSDESC; 64 TlsGotRel = R_AARCH64_TLS_TPREL64; 65 GotEntrySize = 8; 66 GotPltEntrySize = 8; 67 PltEntrySize = 16; 68 PltHeaderSize = 32; 69 DefaultMaxPageSize = 65536; 70 71 // Align to the 2 MiB page size (known as a superpage or huge page). 72 // FreeBSD automatically promotes 2 MiB-aligned allocations. 73 DefaultImageBase = 0x200000; 74 75 NeedsThunks = true; 76 } 77 78 RelExpr AArch64::getRelExpr(RelType Type, const Symbol &S, 79 const uint8_t *Loc) const { 80 switch (Type) { 81 case R_AARCH64_TLSDESC_ADR_PAGE21: 82 return R_AARCH64_TLSDESC_PAGE; 83 case R_AARCH64_TLSDESC_LD64_LO12: 84 case R_AARCH64_TLSDESC_ADD_LO12: 85 return R_TLSDESC; 86 case R_AARCH64_TLSDESC_CALL: 87 return R_TLSDESC_CALL; 88 case R_AARCH64_TLSLE_ADD_TPREL_HI12: 89 case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC: 90 case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC: 91 case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC: 92 case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC: 93 case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC: 94 case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC: 95 return R_TLS; 96 case R_AARCH64_CALL26: 97 case R_AARCH64_CONDBR19: 98 case R_AARCH64_JUMP26: 99 case R_AARCH64_TSTBR14: 100 return R_PLT_PC; 101 case R_AARCH64_PREL16: 102 case R_AARCH64_PREL32: 103 case R_AARCH64_PREL64: 104 case R_AARCH64_ADR_PREL_LO21: 105 case R_AARCH64_LD_PREL_LO19: 106 return R_PC; 107 case R_AARCH64_ADR_PREL_PG_HI21: 108 return R_AARCH64_PAGE_PC; 109 case R_AARCH64_LD64_GOT_LO12_NC: 110 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 111 return R_GOT; 112 case R_AARCH64_ADR_GOT_PAGE: 113 case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21: 114 return R_AARCH64_GOT_PAGE_PC; 115 case R_AARCH64_NONE: 116 return R_NONE; 117 default: 118 return R_ABS; 119 } 120 } 121 122 RelExpr AArch64::adjustRelaxExpr(RelType Type, const uint8_t *Data, 123 RelExpr Expr) const { 124 if (Expr == R_RELAX_TLS_GD_TO_IE) { 125 if (Type == R_AARCH64_TLSDESC_ADR_PAGE21) 126 return R_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC; 127 return R_RELAX_TLS_GD_TO_IE_ABS; 128 } 129 return Expr; 130 } 131 132 bool AArch64::usesOnlyLowPageBits(RelType Type) const { 133 switch (Type) { 134 default: 135 return false; 136 case R_AARCH64_ADD_ABS_LO12_NC: 137 case R_AARCH64_LD64_GOT_LO12_NC: 138 case R_AARCH64_LDST128_ABS_LO12_NC: 139 case R_AARCH64_LDST16_ABS_LO12_NC: 140 case R_AARCH64_LDST32_ABS_LO12_NC: 141 case R_AARCH64_LDST64_ABS_LO12_NC: 142 case R_AARCH64_LDST8_ABS_LO12_NC: 143 case R_AARCH64_TLSDESC_ADD_LO12: 144 case R_AARCH64_TLSDESC_LD64_LO12: 145 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 146 return true; 147 } 148 } 149 150 RelType AArch64::getDynRel(RelType Type) const { 151 if (Type == R_AARCH64_ABS32 || Type == R_AARCH64_ABS64) 152 return Type; 153 return R_AARCH64_NONE; 154 } 155 156 void AArch64::writeGotPlt(uint8_t *Buf, const Symbol &) const { 157 write64le(Buf, In.Plt->getVA()); 158 } 159 160 void AArch64::writePltHeader(uint8_t *Buf) const { 161 const uint8_t PltData[] = { 162 0xf0, 0x7b, 0xbf, 0xa9, // stp x16, x30, [sp,#-16]! 163 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[2])) 164 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[2]))] 165 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.plt.got[2])) 166 0x20, 0x02, 0x1f, 0xd6, // br x17 167 0x1f, 0x20, 0x03, 0xd5, // nop 168 0x1f, 0x20, 0x03, 0xd5, // nop 169 0x1f, 0x20, 0x03, 0xd5 // nop 170 }; 171 memcpy(Buf, PltData, sizeof(PltData)); 172 173 uint64_t Got = In.GotPlt->getVA(); 174 uint64_t Plt = In.Plt->getVA(); 175 relocateOne(Buf + 4, R_AARCH64_ADR_PREL_PG_HI21, 176 getAArch64Page(Got + 16) - getAArch64Page(Plt + 4)); 177 relocateOne(Buf + 8, R_AARCH64_LDST64_ABS_LO12_NC, Got + 16); 178 relocateOne(Buf + 12, R_AARCH64_ADD_ABS_LO12_NC, Got + 16); 179 } 180 181 void AArch64::writePlt(uint8_t *Buf, uint64_t GotPltEntryAddr, 182 uint64_t PltEntryAddr, int32_t Index, 183 unsigned RelOff) const { 184 const uint8_t Inst[] = { 185 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[n])) 186 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[n]))] 187 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.plt.got[n])) 188 0x20, 0x02, 0x1f, 0xd6 // br x17 189 }; 190 memcpy(Buf, Inst, sizeof(Inst)); 191 192 relocateOne(Buf, R_AARCH64_ADR_PREL_PG_HI21, 193 getAArch64Page(GotPltEntryAddr) - getAArch64Page(PltEntryAddr)); 194 relocateOne(Buf + 4, R_AARCH64_LDST64_ABS_LO12_NC, GotPltEntryAddr); 195 relocateOne(Buf + 8, R_AARCH64_ADD_ABS_LO12_NC, GotPltEntryAddr); 196 } 197 198 bool AArch64::needsThunk(RelExpr Expr, RelType Type, const InputFile *File, 199 uint64_t BranchAddr, const Symbol &S) const { 200 // ELF for the ARM 64-bit architecture, section Call and Jump relocations 201 // only permits range extension thunks for R_AARCH64_CALL26 and 202 // R_AARCH64_JUMP26 relocation types. 203 if (Type != R_AARCH64_CALL26 && Type != R_AARCH64_JUMP26) 204 return false; 205 uint64_t Dst = (Expr == R_PLT_PC) ? S.getPltVA() : S.getVA(); 206 return !inBranchRange(Type, BranchAddr, Dst); 207 } 208 209 uint32_t AArch64::getThunkSectionSpacing() const { 210 // See comment in Arch/ARM.cpp for a more detailed explanation of 211 // getThunkSectionSpacing(). For AArch64 the only branches we are permitted to 212 // Thunk have a range of +/- 128 MiB 213 return (128 * 1024 * 1024) - 0x30000; 214 } 215 216 bool AArch64::inBranchRange(RelType Type, uint64_t Src, uint64_t Dst) const { 217 if (Type != R_AARCH64_CALL26 && Type != R_AARCH64_JUMP26) 218 return true; 219 // The AArch64 call and unconditional branch instructions have a range of 220 // +/- 128 MiB. 221 uint64_t Range = 128 * 1024 * 1024; 222 if (Dst > Src) { 223 // Immediate of branch is signed. 224 Range -= 4; 225 return Dst - Src <= Range; 226 } 227 return Src - Dst <= Range; 228 } 229 230 static void write32AArch64Addr(uint8_t *L, uint64_t Imm) { 231 uint32_t ImmLo = (Imm & 0x3) << 29; 232 uint32_t ImmHi = (Imm & 0x1FFFFC) << 3; 233 uint64_t Mask = (0x3 << 29) | (0x1FFFFC << 3); 234 write32le(L, (read32le(L) & ~Mask) | ImmLo | ImmHi); 235 } 236 237 // Return the bits [Start, End] from Val shifted Start bits. 238 // For instance, getBits(0xF0, 4, 8) returns 0xF. 239 static uint64_t getBits(uint64_t Val, int Start, int End) { 240 uint64_t Mask = ((uint64_t)1 << (End + 1 - Start)) - 1; 241 return (Val >> Start) & Mask; 242 } 243 244 static void or32le(uint8_t *P, int32_t V) { write32le(P, read32le(P) | V); } 245 246 // Update the immediate field in a AARCH64 ldr, str, and add instruction. 247 static void or32AArch64Imm(uint8_t *L, uint64_t Imm) { 248 or32le(L, (Imm & 0xFFF) << 10); 249 } 250 251 void AArch64::relocateOne(uint8_t *Loc, RelType Type, uint64_t Val) const { 252 switch (Type) { 253 case R_AARCH64_ABS16: 254 case R_AARCH64_PREL16: 255 checkIntUInt(Loc, Val, 16, Type); 256 write16le(Loc, Val); 257 break; 258 case R_AARCH64_ABS32: 259 case R_AARCH64_PREL32: 260 checkIntUInt(Loc, Val, 32, Type); 261 write32le(Loc, Val); 262 break; 263 case R_AARCH64_ABS64: 264 case R_AARCH64_GLOB_DAT: 265 case R_AARCH64_PREL64: 266 write64le(Loc, Val); 267 break; 268 case R_AARCH64_ADD_ABS_LO12_NC: 269 or32AArch64Imm(Loc, Val); 270 break; 271 case R_AARCH64_ADR_GOT_PAGE: 272 case R_AARCH64_ADR_PREL_PG_HI21: 273 case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21: 274 case R_AARCH64_TLSDESC_ADR_PAGE21: 275 checkInt(Loc, Val, 33, Type); 276 write32AArch64Addr(Loc, Val >> 12); 277 break; 278 case R_AARCH64_ADR_PREL_LO21: 279 checkInt(Loc, Val, 21, Type); 280 write32AArch64Addr(Loc, Val); 281 break; 282 case R_AARCH64_JUMP26: 283 // Normally we would just write the bits of the immediate field, however 284 // when patching instructions for the cpu errata fix -fix-cortex-a53-843419 285 // we want to replace a non-branch instruction with a branch immediate 286 // instruction. By writing all the bits of the instruction including the 287 // opcode and the immediate (0 001 | 01 imm26) we can do this 288 // transformation by placing a R_AARCH64_JUMP26 relocation at the offset of 289 // the instruction we want to patch. 290 write32le(Loc, 0x14000000); 291 LLVM_FALLTHROUGH; 292 case R_AARCH64_CALL26: 293 checkInt(Loc, Val, 28, Type); 294 or32le(Loc, (Val & 0x0FFFFFFC) >> 2); 295 break; 296 case R_AARCH64_CONDBR19: 297 case R_AARCH64_LD_PREL_LO19: 298 checkAlignment(Loc, Val, 4, Type); 299 checkInt(Loc, Val, 21, Type); 300 or32le(Loc, (Val & 0x1FFFFC) << 3); 301 break; 302 case R_AARCH64_LDST8_ABS_LO12_NC: 303 case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC: 304 or32AArch64Imm(Loc, getBits(Val, 0, 11)); 305 break; 306 case R_AARCH64_LDST16_ABS_LO12_NC: 307 case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC: 308 checkAlignment(Loc, Val, 2, Type); 309 or32AArch64Imm(Loc, getBits(Val, 1, 11)); 310 break; 311 case R_AARCH64_LDST32_ABS_LO12_NC: 312 case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC: 313 checkAlignment(Loc, Val, 4, Type); 314 or32AArch64Imm(Loc, getBits(Val, 2, 11)); 315 break; 316 case R_AARCH64_LDST64_ABS_LO12_NC: 317 case R_AARCH64_LD64_GOT_LO12_NC: 318 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 319 case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC: 320 case R_AARCH64_TLSDESC_LD64_LO12: 321 checkAlignment(Loc, Val, 8, Type); 322 or32AArch64Imm(Loc, getBits(Val, 3, 11)); 323 break; 324 case R_AARCH64_LDST128_ABS_LO12_NC: 325 case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC: 326 checkAlignment(Loc, Val, 16, Type); 327 or32AArch64Imm(Loc, getBits(Val, 4, 11)); 328 break; 329 case R_AARCH64_MOVW_UABS_G0_NC: 330 or32le(Loc, (Val & 0xFFFF) << 5); 331 break; 332 case R_AARCH64_MOVW_UABS_G1_NC: 333 or32le(Loc, (Val & 0xFFFF0000) >> 11); 334 break; 335 case R_AARCH64_MOVW_UABS_G2_NC: 336 or32le(Loc, (Val & 0xFFFF00000000) >> 27); 337 break; 338 case R_AARCH64_MOVW_UABS_G3: 339 or32le(Loc, (Val & 0xFFFF000000000000) >> 43); 340 break; 341 case R_AARCH64_TSTBR14: 342 checkInt(Loc, Val, 16, Type); 343 or32le(Loc, (Val & 0xFFFC) << 3); 344 break; 345 case R_AARCH64_TLSLE_ADD_TPREL_HI12: 346 checkUInt(Loc, Val, 24, Type); 347 or32AArch64Imm(Loc, Val >> 12); 348 break; 349 case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC: 350 case R_AARCH64_TLSDESC_ADD_LO12: 351 or32AArch64Imm(Loc, Val); 352 break; 353 default: 354 error(getErrorLocation(Loc) + "unrecognized reloc " + Twine(Type)); 355 } 356 } 357 358 void AArch64::relaxTlsGdToLe(uint8_t *Loc, RelType Type, uint64_t Val) const { 359 // TLSDESC Global-Dynamic relocation are in the form: 360 // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21] 361 // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12] 362 // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12] 363 // .tlsdesccall [R_AARCH64_TLSDESC_CALL] 364 // blr x1 365 // And it can optimized to: 366 // movz x0, #0x0, lsl #16 367 // movk x0, #0x10 368 // nop 369 // nop 370 checkUInt(Loc, Val, 32, Type); 371 372 switch (Type) { 373 case R_AARCH64_TLSDESC_ADD_LO12: 374 case R_AARCH64_TLSDESC_CALL: 375 write32le(Loc, 0xd503201f); // nop 376 return; 377 case R_AARCH64_TLSDESC_ADR_PAGE21: 378 write32le(Loc, 0xd2a00000 | (((Val >> 16) & 0xffff) << 5)); // movz 379 return; 380 case R_AARCH64_TLSDESC_LD64_LO12: 381 write32le(Loc, 0xf2800000 | ((Val & 0xffff) << 5)); // movk 382 return; 383 default: 384 llvm_unreachable("unsupported relocation for TLS GD to LE relaxation"); 385 } 386 } 387 388 void AArch64::relaxTlsGdToIe(uint8_t *Loc, RelType Type, uint64_t Val) const { 389 // TLSDESC Global-Dynamic relocation are in the form: 390 // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21] 391 // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12] 392 // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12] 393 // .tlsdesccall [R_AARCH64_TLSDESC_CALL] 394 // blr x1 395 // And it can optimized to: 396 // adrp x0, :gottprel:v 397 // ldr x0, [x0, :gottprel_lo12:v] 398 // nop 399 // nop 400 401 switch (Type) { 402 case R_AARCH64_TLSDESC_ADD_LO12: 403 case R_AARCH64_TLSDESC_CALL: 404 write32le(Loc, 0xd503201f); // nop 405 break; 406 case R_AARCH64_TLSDESC_ADR_PAGE21: 407 write32le(Loc, 0x90000000); // adrp 408 relocateOne(Loc, R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21, Val); 409 break; 410 case R_AARCH64_TLSDESC_LD64_LO12: 411 write32le(Loc, 0xf9400000); // ldr 412 relocateOne(Loc, R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC, Val); 413 break; 414 default: 415 llvm_unreachable("unsupported relocation for TLS GD to LE relaxation"); 416 } 417 } 418 419 void AArch64::relaxTlsIeToLe(uint8_t *Loc, RelType Type, uint64_t Val) const { 420 checkUInt(Loc, Val, 32, Type); 421 422 if (Type == R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21) { 423 // Generate MOVZ. 424 uint32_t RegNo = read32le(Loc) & 0x1f; 425 write32le(Loc, (0xd2a00000 | RegNo) | (((Val >> 16) & 0xffff) << 5)); 426 return; 427 } 428 if (Type == R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC) { 429 // Generate MOVK. 430 uint32_t RegNo = read32le(Loc) & 0x1f; 431 write32le(Loc, (0xf2800000 | RegNo) | ((Val & 0xffff) << 5)); 432 return; 433 } 434 llvm_unreachable("invalid relocation for TLS IE to LE relaxation"); 435 } 436 437 TargetInfo *elf::getAArch64TargetInfo() { 438 static AArch64 Target; 439 return &Target; 440 } 441