1 //===- AArch64.cpp --------------------------------------------------------===// 2 // 3 // The LLVM Linker 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 10 #include "Error.h" 11 #include "Symbols.h" 12 #include "SyntheticSections.h" 13 #include "Target.h" 14 #include "Thunks.h" 15 #include "llvm/Object/ELF.h" 16 #include "llvm/Support/Endian.h" 17 18 using namespace llvm; 19 using namespace llvm::support::endian; 20 using namespace llvm::ELF; 21 using namespace lld; 22 using namespace lld::elf; 23 24 // Page(Expr) is the page address of the expression Expr, defined 25 // as (Expr & ~0xFFF). (This applies even if the machine page size 26 // supported by the platform has a different value.) 27 uint64_t elf::getAArch64Page(uint64_t Expr) { 28 return Expr & ~static_cast<uint64_t>(0xFFF); 29 } 30 31 namespace { 32 class AArch64 final : public TargetInfo { 33 public: 34 AArch64(); 35 RelExpr getRelExpr(RelType Type, const SymbolBody &S, 36 const uint8_t *Loc) const override; 37 bool isPicRel(RelType Type) const override; 38 void writeGotPlt(uint8_t *Buf, const SymbolBody &S) const override; 39 void writePltHeader(uint8_t *Buf) const override; 40 void writePlt(uint8_t *Buf, uint64_t GotPltEntryAddr, uint64_t PltEntryAddr, 41 int32_t Index, unsigned RelOff) const override; 42 bool usesOnlyLowPageBits(RelType Type) const override; 43 void relocateOne(uint8_t *Loc, RelType Type, uint64_t Val) const override; 44 RelExpr adjustRelaxExpr(RelType Type, const uint8_t *Data, 45 RelExpr Expr) const override; 46 void relaxTlsGdToLe(uint8_t *Loc, RelType Type, uint64_t Val) const override; 47 void relaxTlsGdToIe(uint8_t *Loc, RelType Type, uint64_t Val) const override; 48 void relaxTlsIeToLe(uint8_t *Loc, RelType Type, uint64_t Val) const override; 49 }; 50 } // namespace 51 52 AArch64::AArch64() { 53 CopyRel = R_AARCH64_COPY; 54 RelativeRel = R_AARCH64_RELATIVE; 55 IRelativeRel = R_AARCH64_IRELATIVE; 56 GotRel = R_AARCH64_GLOB_DAT; 57 PltRel = R_AARCH64_JUMP_SLOT; 58 TlsDescRel = R_AARCH64_TLSDESC; 59 TlsGotRel = R_AARCH64_TLS_TPREL64; 60 GotEntrySize = 8; 61 GotPltEntrySize = 8; 62 PltEntrySize = 16; 63 PltHeaderSize = 32; 64 DefaultMaxPageSize = 65536; 65 66 // It doesn't seem to be documented anywhere, but tls on aarch64 uses variant 67 // 1 of the tls structures and the tcb size is 16. 68 TcbSize = 16; 69 } 70 71 RelExpr AArch64::getRelExpr(RelType Type, const SymbolBody &S, 72 const uint8_t *Loc) const { 73 switch (Type) { 74 case R_AARCH64_TLSDESC_ADR_PAGE21: 75 return R_TLSDESC_PAGE; 76 case R_AARCH64_TLSDESC_LD64_LO12: 77 case R_AARCH64_TLSDESC_ADD_LO12: 78 return R_TLSDESC; 79 case R_AARCH64_TLSDESC_CALL: 80 return R_TLSDESC_CALL; 81 case R_AARCH64_TLSLE_ADD_TPREL_HI12: 82 case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC: 83 return R_TLS; 84 case R_AARCH64_CALL26: 85 case R_AARCH64_CONDBR19: 86 case R_AARCH64_JUMP26: 87 case R_AARCH64_TSTBR14: 88 return R_PLT_PC; 89 case R_AARCH64_PREL16: 90 case R_AARCH64_PREL32: 91 case R_AARCH64_PREL64: 92 case R_AARCH64_ADR_PREL_LO21: 93 case R_AARCH64_LD_PREL_LO19: 94 return R_PC; 95 case R_AARCH64_ADR_PREL_PG_HI21: 96 return R_PAGE_PC; 97 case R_AARCH64_LD64_GOT_LO12_NC: 98 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 99 return R_GOT; 100 case R_AARCH64_ADR_GOT_PAGE: 101 case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21: 102 return R_GOT_PAGE_PC; 103 case R_AARCH64_NONE: 104 return R_NONE; 105 default: 106 return R_ABS; 107 } 108 } 109 110 RelExpr AArch64::adjustRelaxExpr(RelType Type, const uint8_t *Data, 111 RelExpr Expr) const { 112 if (Expr == R_RELAX_TLS_GD_TO_IE) { 113 if (Type == R_AARCH64_TLSDESC_ADR_PAGE21) 114 return R_RELAX_TLS_GD_TO_IE_PAGE_PC; 115 return R_RELAX_TLS_GD_TO_IE_ABS; 116 } 117 return Expr; 118 } 119 120 bool AArch64::usesOnlyLowPageBits(RelType Type) const { 121 switch (Type) { 122 default: 123 return false; 124 case R_AARCH64_ADD_ABS_LO12_NC: 125 case R_AARCH64_LD64_GOT_LO12_NC: 126 case R_AARCH64_LDST128_ABS_LO12_NC: 127 case R_AARCH64_LDST16_ABS_LO12_NC: 128 case R_AARCH64_LDST32_ABS_LO12_NC: 129 case R_AARCH64_LDST64_ABS_LO12_NC: 130 case R_AARCH64_LDST8_ABS_LO12_NC: 131 case R_AARCH64_TLSDESC_ADD_LO12: 132 case R_AARCH64_TLSDESC_LD64_LO12: 133 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 134 return true; 135 } 136 } 137 138 bool AArch64::isPicRel(RelType Type) const { 139 return Type == R_AARCH64_ABS32 || Type == R_AARCH64_ABS64; 140 } 141 142 void AArch64::writeGotPlt(uint8_t *Buf, const SymbolBody &) const { 143 write64le(Buf, InX::Plt->getVA()); 144 } 145 146 void AArch64::writePltHeader(uint8_t *Buf) const { 147 const uint8_t PltData[] = { 148 0xf0, 0x7b, 0xbf, 0xa9, // stp x16, x30, [sp,#-16]! 149 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[2])) 150 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[2]))] 151 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.plt.got[2])) 152 0x20, 0x02, 0x1f, 0xd6, // br x17 153 0x1f, 0x20, 0x03, 0xd5, // nop 154 0x1f, 0x20, 0x03, 0xd5, // nop 155 0x1f, 0x20, 0x03, 0xd5 // nop 156 }; 157 memcpy(Buf, PltData, sizeof(PltData)); 158 159 uint64_t Got = InX::GotPlt->getVA(); 160 uint64_t Plt = InX::Plt->getVA(); 161 relocateOne(Buf + 4, R_AARCH64_ADR_PREL_PG_HI21, 162 getAArch64Page(Got + 16) - getAArch64Page(Plt + 4)); 163 relocateOne(Buf + 8, R_AARCH64_LDST64_ABS_LO12_NC, Got + 16); 164 relocateOne(Buf + 12, R_AARCH64_ADD_ABS_LO12_NC, Got + 16); 165 } 166 167 void AArch64::writePlt(uint8_t *Buf, uint64_t GotPltEntryAddr, 168 uint64_t PltEntryAddr, int32_t Index, 169 unsigned RelOff) const { 170 const uint8_t Inst[] = { 171 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[n])) 172 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[n]))] 173 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.plt.got[n])) 174 0x20, 0x02, 0x1f, 0xd6 // br x17 175 }; 176 memcpy(Buf, Inst, sizeof(Inst)); 177 178 relocateOne(Buf, R_AARCH64_ADR_PREL_PG_HI21, 179 getAArch64Page(GotPltEntryAddr) - getAArch64Page(PltEntryAddr)); 180 relocateOne(Buf + 4, R_AARCH64_LDST64_ABS_LO12_NC, GotPltEntryAddr); 181 relocateOne(Buf + 8, R_AARCH64_ADD_ABS_LO12_NC, GotPltEntryAddr); 182 } 183 184 static void write32AArch64Addr(uint8_t *L, uint64_t Imm) { 185 uint32_t ImmLo = (Imm & 0x3) << 29; 186 uint32_t ImmHi = (Imm & 0x1FFFFC) << 3; 187 uint64_t Mask = (0x3 << 29) | (0x1FFFFC << 3); 188 write32le(L, (read32le(L) & ~Mask) | ImmLo | ImmHi); 189 } 190 191 // Return the bits [Start, End] from Val shifted Start bits. 192 // For instance, getBits(0xF0, 4, 8) returns 0xF. 193 static uint64_t getBits(uint64_t Val, int Start, int End) { 194 uint64_t Mask = ((uint64_t)1 << (End + 1 - Start)) - 1; 195 return (Val >> Start) & Mask; 196 } 197 198 static void or32le(uint8_t *P, int32_t V) { write32le(P, read32le(P) | V); } 199 200 // Update the immediate field in a AARCH64 ldr, str, and add instruction. 201 static void or32AArch64Imm(uint8_t *L, uint64_t Imm) { 202 or32le(L, (Imm & 0xFFF) << 10); 203 } 204 205 void AArch64::relocateOne(uint8_t *Loc, RelType Type, uint64_t Val) const { 206 switch (Type) { 207 case R_AARCH64_ABS16: 208 case R_AARCH64_PREL16: 209 checkIntUInt<16>(Loc, Val, Type); 210 write16le(Loc, Val); 211 break; 212 case R_AARCH64_ABS32: 213 case R_AARCH64_PREL32: 214 checkIntUInt<32>(Loc, Val, Type); 215 write32le(Loc, Val); 216 break; 217 case R_AARCH64_ABS64: 218 case R_AARCH64_GLOB_DAT: 219 case R_AARCH64_PREL64: 220 write64le(Loc, Val); 221 break; 222 case R_AARCH64_ADD_ABS_LO12_NC: 223 or32AArch64Imm(Loc, Val); 224 break; 225 case R_AARCH64_ADR_GOT_PAGE: 226 case R_AARCH64_ADR_PREL_PG_HI21: 227 case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21: 228 case R_AARCH64_TLSDESC_ADR_PAGE21: 229 checkInt<33>(Loc, Val, Type); 230 write32AArch64Addr(Loc, Val >> 12); 231 break; 232 case R_AARCH64_ADR_PREL_LO21: 233 checkInt<21>(Loc, Val, Type); 234 write32AArch64Addr(Loc, Val); 235 break; 236 case R_AARCH64_JUMP26: 237 // Normally we would just write the bits of the immediate field, however 238 // when patching instructions for the cpu errata fix -fix-cortex-a53-843419 239 // we want to replace a non-branch instruction with a branch immediate 240 // instruction. By writing all the bits of the instruction including the 241 // opcode and the immediate (0 001 | 01 imm26) we can do this 242 // transformation by placing a R_AARCH64_JUMP26 relocation at the offset of 243 // the instruction we want to patch. 244 write32le(Loc, 0x14000000); 245 LLVM_FALLTHROUGH; 246 case R_AARCH64_CALL26: 247 checkInt<28>(Loc, Val, Type); 248 or32le(Loc, (Val & 0x0FFFFFFC) >> 2); 249 break; 250 case R_AARCH64_CONDBR19: 251 case R_AARCH64_LD_PREL_LO19: 252 checkAlignment<4>(Loc, Val, Type); 253 checkInt<21>(Loc, Val, Type); 254 or32le(Loc, (Val & 0x1FFFFC) << 3); 255 break; 256 case R_AARCH64_LD64_GOT_LO12_NC: 257 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 258 case R_AARCH64_TLSDESC_LD64_LO12: 259 checkAlignment<8>(Loc, Val, Type); 260 or32le(Loc, (Val & 0xFF8) << 7); 261 break; 262 case R_AARCH64_LDST8_ABS_LO12_NC: 263 or32AArch64Imm(Loc, getBits(Val, 0, 11)); 264 break; 265 case R_AARCH64_LDST16_ABS_LO12_NC: 266 checkAlignment<2>(Loc, Val, Type); 267 or32AArch64Imm(Loc, getBits(Val, 1, 11)); 268 break; 269 case R_AARCH64_LDST32_ABS_LO12_NC: 270 checkAlignment<4>(Loc, Val, Type); 271 or32AArch64Imm(Loc, getBits(Val, 2, 11)); 272 break; 273 case R_AARCH64_LDST64_ABS_LO12_NC: 274 checkAlignment<8>(Loc, Val, Type); 275 or32AArch64Imm(Loc, getBits(Val, 3, 11)); 276 break; 277 case R_AARCH64_LDST128_ABS_LO12_NC: 278 checkAlignment<16>(Loc, Val, Type); 279 or32AArch64Imm(Loc, getBits(Val, 4, 11)); 280 break; 281 case R_AARCH64_MOVW_UABS_G0_NC: 282 or32le(Loc, (Val & 0xFFFF) << 5); 283 break; 284 case R_AARCH64_MOVW_UABS_G1_NC: 285 or32le(Loc, (Val & 0xFFFF0000) >> 11); 286 break; 287 case R_AARCH64_MOVW_UABS_G2_NC: 288 or32le(Loc, (Val & 0xFFFF00000000) >> 27); 289 break; 290 case R_AARCH64_MOVW_UABS_G3: 291 or32le(Loc, (Val & 0xFFFF000000000000) >> 43); 292 break; 293 case R_AARCH64_TSTBR14: 294 checkInt<16>(Loc, Val, Type); 295 or32le(Loc, (Val & 0xFFFC) << 3); 296 break; 297 case R_AARCH64_TLSLE_ADD_TPREL_HI12: 298 checkInt<24>(Loc, Val, Type); 299 or32AArch64Imm(Loc, Val >> 12); 300 break; 301 case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC: 302 case R_AARCH64_TLSDESC_ADD_LO12: 303 or32AArch64Imm(Loc, Val); 304 break; 305 default: 306 error(getErrorLocation(Loc) + "unrecognized reloc " + Twine(Type)); 307 } 308 } 309 310 void AArch64::relaxTlsGdToLe(uint8_t *Loc, RelType Type, uint64_t Val) const { 311 // TLSDESC Global-Dynamic relocation are in the form: 312 // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21] 313 // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12] 314 // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12] 315 // .tlsdesccall [R_AARCH64_TLSDESC_CALL] 316 // blr x1 317 // And it can optimized to: 318 // movz x0, #0x0, lsl #16 319 // movk x0, #0x10 320 // nop 321 // nop 322 checkUInt<32>(Loc, Val, Type); 323 324 switch (Type) { 325 case R_AARCH64_TLSDESC_ADD_LO12: 326 case R_AARCH64_TLSDESC_CALL: 327 write32le(Loc, 0xd503201f); // nop 328 return; 329 case R_AARCH64_TLSDESC_ADR_PAGE21: 330 write32le(Loc, 0xd2a00000 | (((Val >> 16) & 0xffff) << 5)); // movz 331 return; 332 case R_AARCH64_TLSDESC_LD64_LO12: 333 write32le(Loc, 0xf2800000 | ((Val & 0xffff) << 5)); // movk 334 return; 335 default: 336 llvm_unreachable("unsupported relocation for TLS GD to LE relaxation"); 337 } 338 } 339 340 void AArch64::relaxTlsGdToIe(uint8_t *Loc, RelType Type, uint64_t Val) const { 341 // TLSDESC Global-Dynamic relocation are in the form: 342 // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21] 343 // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12] 344 // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12] 345 // .tlsdesccall [R_AARCH64_TLSDESC_CALL] 346 // blr x1 347 // And it can optimized to: 348 // adrp x0, :gottprel:v 349 // ldr x0, [x0, :gottprel_lo12:v] 350 // nop 351 // nop 352 353 switch (Type) { 354 case R_AARCH64_TLSDESC_ADD_LO12: 355 case R_AARCH64_TLSDESC_CALL: 356 write32le(Loc, 0xd503201f); // nop 357 break; 358 case R_AARCH64_TLSDESC_ADR_PAGE21: 359 write32le(Loc, 0x90000000); // adrp 360 relocateOne(Loc, R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21, Val); 361 break; 362 case R_AARCH64_TLSDESC_LD64_LO12: 363 write32le(Loc, 0xf9400000); // ldr 364 relocateOne(Loc, R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC, Val); 365 break; 366 default: 367 llvm_unreachable("unsupported relocation for TLS GD to LE relaxation"); 368 } 369 } 370 371 void AArch64::relaxTlsIeToLe(uint8_t *Loc, RelType Type, uint64_t Val) const { 372 checkUInt<32>(Loc, Val, Type); 373 374 if (Type == R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21) { 375 // Generate MOVZ. 376 uint32_t RegNo = read32le(Loc) & 0x1f; 377 write32le(Loc, (0xd2a00000 | RegNo) | (((Val >> 16) & 0xffff) << 5)); 378 return; 379 } 380 if (Type == R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC) { 381 // Generate MOVK. 382 uint32_t RegNo = read32le(Loc) & 0x1f; 383 write32le(Loc, (0xf2800000 | RegNo) | ((Val & 0xffff) << 5)); 384 return; 385 } 386 llvm_unreachable("invalid relocation for TLS IE to LE relaxation"); 387 } 388 389 TargetInfo *elf::getAArch64TargetInfo() { 390 static AArch64 Target; 391 return &Target; 392 } 393