1 //===- AArch64.cpp --------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "Symbols.h" 10 #include "SyntheticSections.h" 11 #include "Target.h" 12 #include "Thunks.h" 13 #include "lld/Common/ErrorHandler.h" 14 #include "llvm/Object/ELF.h" 15 #include "llvm/Support/Endian.h" 16 17 using namespace llvm; 18 using namespace llvm::support::endian; 19 using namespace llvm::ELF; 20 using namespace lld; 21 using namespace lld::elf; 22 23 // Page(Expr) is the page address of the expression Expr, defined 24 // as (Expr & ~0xFFF). (This applies even if the machine page size 25 // supported by the platform has a different value.) 26 uint64_t elf::getAArch64Page(uint64_t expr) { 27 return expr & ~static_cast<uint64_t>(0xFFF); 28 } 29 30 namespace { 31 class AArch64 : public TargetInfo { 32 public: 33 AArch64(); 34 RelExpr getRelExpr(RelType type, const Symbol &s, 35 const uint8_t *loc) const override; 36 RelType getDynRel(RelType type) const override; 37 void writeGotPlt(uint8_t *buf, const Symbol &s) const override; 38 void writePltHeader(uint8_t *buf) const override; 39 void writePlt(uint8_t *buf, uint64_t gotPltEntryAddr, uint64_t pltEntryAddr, 40 int32_t index, unsigned relOff) const override; 41 bool needsThunk(RelExpr expr, RelType type, const InputFile *file, 42 uint64_t branchAddr, const Symbol &s) const override; 43 uint32_t getThunkSectionSpacing() const override; 44 bool inBranchRange(RelType type, uint64_t src, uint64_t dst) const override; 45 bool usesOnlyLowPageBits(RelType type) const override; 46 void relocateOne(uint8_t *loc, RelType type, uint64_t val) const override; 47 RelExpr adjustRelaxExpr(RelType type, const uint8_t *data, 48 RelExpr expr) const override; 49 void relaxTlsGdToLe(uint8_t *loc, RelType type, uint64_t val) const override; 50 void relaxTlsGdToIe(uint8_t *loc, RelType type, uint64_t val) const override; 51 void relaxTlsIeToLe(uint8_t *loc, RelType type, uint64_t val) const override; 52 }; 53 } // namespace 54 55 AArch64::AArch64() { 56 copyRel = R_AARCH64_COPY; 57 relativeRel = R_AARCH64_RELATIVE; 58 iRelativeRel = R_AARCH64_IRELATIVE; 59 gotRel = R_AARCH64_GLOB_DAT; 60 noneRel = R_AARCH64_NONE; 61 pltRel = R_AARCH64_JUMP_SLOT; 62 symbolicRel = R_AARCH64_ABS64; 63 tlsDescRel = R_AARCH64_TLSDESC; 64 tlsGotRel = R_AARCH64_TLS_TPREL64; 65 pltEntrySize = 16; 66 pltHeaderSize = 32; 67 defaultMaxPageSize = 65536; 68 69 // Align to the 2 MiB page size (known as a superpage or huge page). 70 // FreeBSD automatically promotes 2 MiB-aligned allocations. 71 defaultImageBase = 0x200000; 72 73 needsThunks = true; 74 } 75 76 RelExpr AArch64::getRelExpr(RelType type, const Symbol &s, 77 const uint8_t *loc) const { 78 switch (type) { 79 case R_AARCH64_ABS16: 80 case R_AARCH64_ABS32: 81 case R_AARCH64_ABS64: 82 case R_AARCH64_ADD_ABS_LO12_NC: 83 case R_AARCH64_LDST128_ABS_LO12_NC: 84 case R_AARCH64_LDST16_ABS_LO12_NC: 85 case R_AARCH64_LDST32_ABS_LO12_NC: 86 case R_AARCH64_LDST64_ABS_LO12_NC: 87 case R_AARCH64_LDST8_ABS_LO12_NC: 88 case R_AARCH64_MOVW_SABS_G0: 89 case R_AARCH64_MOVW_SABS_G1: 90 case R_AARCH64_MOVW_SABS_G2: 91 case R_AARCH64_MOVW_UABS_G0: 92 case R_AARCH64_MOVW_UABS_G0_NC: 93 case R_AARCH64_MOVW_UABS_G1: 94 case R_AARCH64_MOVW_UABS_G1_NC: 95 case R_AARCH64_MOVW_UABS_G2: 96 case R_AARCH64_MOVW_UABS_G2_NC: 97 case R_AARCH64_MOVW_UABS_G3: 98 return R_ABS; 99 case R_AARCH64_TLSDESC_ADR_PAGE21: 100 return R_AARCH64_TLSDESC_PAGE; 101 case R_AARCH64_TLSDESC_LD64_LO12: 102 case R_AARCH64_TLSDESC_ADD_LO12: 103 return R_TLSDESC; 104 case R_AARCH64_TLSDESC_CALL: 105 return R_TLSDESC_CALL; 106 case R_AARCH64_TLSLE_ADD_TPREL_HI12: 107 case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC: 108 case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC: 109 case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC: 110 case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC: 111 case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC: 112 case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC: 113 case R_AARCH64_TLSLE_MOVW_TPREL_G0: 114 case R_AARCH64_TLSLE_MOVW_TPREL_G0_NC: 115 case R_AARCH64_TLSLE_MOVW_TPREL_G1: 116 case R_AARCH64_TLSLE_MOVW_TPREL_G1_NC: 117 case R_AARCH64_TLSLE_MOVW_TPREL_G2: 118 return R_TLS; 119 case R_AARCH64_CALL26: 120 case R_AARCH64_CONDBR19: 121 case R_AARCH64_JUMP26: 122 case R_AARCH64_TSTBR14: 123 return R_PLT_PC; 124 case R_AARCH64_PREL16: 125 case R_AARCH64_PREL32: 126 case R_AARCH64_PREL64: 127 case R_AARCH64_ADR_PREL_LO21: 128 case R_AARCH64_LD_PREL_LO19: 129 case R_AARCH64_MOVW_PREL_G0: 130 case R_AARCH64_MOVW_PREL_G0_NC: 131 case R_AARCH64_MOVW_PREL_G1: 132 case R_AARCH64_MOVW_PREL_G1_NC: 133 case R_AARCH64_MOVW_PREL_G2: 134 case R_AARCH64_MOVW_PREL_G2_NC: 135 case R_AARCH64_MOVW_PREL_G3: 136 return R_PC; 137 case R_AARCH64_ADR_PREL_PG_HI21: 138 case R_AARCH64_ADR_PREL_PG_HI21_NC: 139 return R_AARCH64_PAGE_PC; 140 case R_AARCH64_LD64_GOT_LO12_NC: 141 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 142 return R_GOT; 143 case R_AARCH64_ADR_GOT_PAGE: 144 case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21: 145 return R_AARCH64_GOT_PAGE_PC; 146 case R_AARCH64_NONE: 147 return R_NONE; 148 default: 149 error(getErrorLocation(loc) + "unknown relocation (" + Twine(type) + 150 ") against symbol " + toString(s)); 151 return R_NONE; 152 } 153 } 154 155 RelExpr AArch64::adjustRelaxExpr(RelType type, const uint8_t *data, 156 RelExpr expr) const { 157 if (expr == R_RELAX_TLS_GD_TO_IE) { 158 if (type == R_AARCH64_TLSDESC_ADR_PAGE21) 159 return R_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC; 160 return R_RELAX_TLS_GD_TO_IE_ABS; 161 } 162 return expr; 163 } 164 165 bool AArch64::usesOnlyLowPageBits(RelType type) const { 166 switch (type) { 167 default: 168 return false; 169 case R_AARCH64_ADD_ABS_LO12_NC: 170 case R_AARCH64_LD64_GOT_LO12_NC: 171 case R_AARCH64_LDST128_ABS_LO12_NC: 172 case R_AARCH64_LDST16_ABS_LO12_NC: 173 case R_AARCH64_LDST32_ABS_LO12_NC: 174 case R_AARCH64_LDST64_ABS_LO12_NC: 175 case R_AARCH64_LDST8_ABS_LO12_NC: 176 case R_AARCH64_TLSDESC_ADD_LO12: 177 case R_AARCH64_TLSDESC_LD64_LO12: 178 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 179 return true; 180 } 181 } 182 183 RelType AArch64::getDynRel(RelType type) const { 184 if (type == R_AARCH64_ABS64) 185 return type; 186 return R_AARCH64_NONE; 187 } 188 189 void AArch64::writeGotPlt(uint8_t *buf, const Symbol &) const { 190 write64le(buf, in.plt->getVA()); 191 } 192 193 void AArch64::writePltHeader(uint8_t *buf) const { 194 const uint8_t pltData[] = { 195 0xf0, 0x7b, 0xbf, 0xa9, // stp x16, x30, [sp,#-16]! 196 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[2])) 197 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[2]))] 198 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.plt.got[2])) 199 0x20, 0x02, 0x1f, 0xd6, // br x17 200 0x1f, 0x20, 0x03, 0xd5, // nop 201 0x1f, 0x20, 0x03, 0xd5, // nop 202 0x1f, 0x20, 0x03, 0xd5 // nop 203 }; 204 memcpy(buf, pltData, sizeof(pltData)); 205 206 uint64_t got = in.gotPlt->getVA(); 207 uint64_t plt = in.plt->getVA(); 208 relocateOne(buf + 4, R_AARCH64_ADR_PREL_PG_HI21, 209 getAArch64Page(got + 16) - getAArch64Page(plt + 4)); 210 relocateOne(buf + 8, R_AARCH64_LDST64_ABS_LO12_NC, got + 16); 211 relocateOne(buf + 12, R_AARCH64_ADD_ABS_LO12_NC, got + 16); 212 } 213 214 void AArch64::writePlt(uint8_t *buf, uint64_t gotPltEntryAddr, 215 uint64_t pltEntryAddr, int32_t index, 216 unsigned relOff) const { 217 const uint8_t inst[] = { 218 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[n])) 219 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[n]))] 220 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.plt.got[n])) 221 0x20, 0x02, 0x1f, 0xd6 // br x17 222 }; 223 memcpy(buf, inst, sizeof(inst)); 224 225 relocateOne(buf, R_AARCH64_ADR_PREL_PG_HI21, 226 getAArch64Page(gotPltEntryAddr) - getAArch64Page(pltEntryAddr)); 227 relocateOne(buf + 4, R_AARCH64_LDST64_ABS_LO12_NC, gotPltEntryAddr); 228 relocateOne(buf + 8, R_AARCH64_ADD_ABS_LO12_NC, gotPltEntryAddr); 229 } 230 231 bool AArch64::needsThunk(RelExpr expr, RelType type, const InputFile *file, 232 uint64_t branchAddr, const Symbol &s) const { 233 // ELF for the ARM 64-bit architecture, section Call and Jump relocations 234 // only permits range extension thunks for R_AARCH64_CALL26 and 235 // R_AARCH64_JUMP26 relocation types. 236 if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26) 237 return false; 238 uint64_t dst = (expr == R_PLT_PC) ? s.getPltVA() : s.getVA(); 239 return !inBranchRange(type, branchAddr, dst); 240 } 241 242 uint32_t AArch64::getThunkSectionSpacing() const { 243 // See comment in Arch/ARM.cpp for a more detailed explanation of 244 // getThunkSectionSpacing(). For AArch64 the only branches we are permitted to 245 // Thunk have a range of +/- 128 MiB 246 return (128 * 1024 * 1024) - 0x30000; 247 } 248 249 bool AArch64::inBranchRange(RelType type, uint64_t src, uint64_t dst) const { 250 if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26) 251 return true; 252 // The AArch64 call and unconditional branch instructions have a range of 253 // +/- 128 MiB. 254 uint64_t range = 128 * 1024 * 1024; 255 if (dst > src) { 256 // Immediate of branch is signed. 257 range -= 4; 258 return dst - src <= range; 259 } 260 return src - dst <= range; 261 } 262 263 static void write32AArch64Addr(uint8_t *l, uint64_t imm) { 264 uint32_t immLo = (imm & 0x3) << 29; 265 uint32_t immHi = (imm & 0x1FFFFC) << 3; 266 uint64_t mask = (0x3 << 29) | (0x1FFFFC << 3); 267 write32le(l, (read32le(l) & ~mask) | immLo | immHi); 268 } 269 270 // Return the bits [Start, End] from Val shifted Start bits. 271 // For instance, getBits(0xF0, 4, 8) returns 0xF. 272 static uint64_t getBits(uint64_t val, int start, int end) { 273 uint64_t mask = ((uint64_t)1 << (end + 1 - start)) - 1; 274 return (val >> start) & mask; 275 } 276 277 static void or32le(uint8_t *p, int32_t v) { write32le(p, read32le(p) | v); } 278 279 // Update the immediate field in a AARCH64 ldr, str, and add instruction. 280 static void or32AArch64Imm(uint8_t *l, uint64_t imm) { 281 or32le(l, (imm & 0xFFF) << 10); 282 } 283 284 // Update the immediate field in an AArch64 movk, movn or movz instruction 285 // for a signed relocation, and update the opcode of a movn or movz instruction 286 // to match the sign of the operand. 287 static void writeSMovWImm(uint8_t *loc, uint32_t imm) { 288 uint32_t inst = read32le(loc); 289 // Opcode field is bits 30, 29, with 10 = movz, 00 = movn and 11 = movk. 290 if (!(inst & (1 << 29))) { 291 // movn or movz. 292 if (imm & 0x10000) { 293 // Change opcode to movn, which takes an inverted operand. 294 imm ^= 0xFFFF; 295 inst &= ~(1 << 30); 296 } else { 297 // Change opcode to movz. 298 inst |= 1 << 30; 299 } 300 } 301 write32le(loc, inst | ((imm & 0xFFFF) << 5)); 302 } 303 304 void AArch64::relocateOne(uint8_t *loc, RelType type, uint64_t val) const { 305 switch (type) { 306 case R_AARCH64_ABS16: 307 case R_AARCH64_PREL16: 308 checkIntUInt(loc, val, 16, type); 309 write16le(loc, val); 310 break; 311 case R_AARCH64_ABS32: 312 case R_AARCH64_PREL32: 313 checkIntUInt(loc, val, 32, type); 314 write32le(loc, val); 315 break; 316 case R_AARCH64_ABS64: 317 case R_AARCH64_PREL64: 318 write64le(loc, val); 319 break; 320 case R_AARCH64_ADD_ABS_LO12_NC: 321 or32AArch64Imm(loc, val); 322 break; 323 case R_AARCH64_ADR_GOT_PAGE: 324 case R_AARCH64_ADR_PREL_PG_HI21: 325 case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21: 326 case R_AARCH64_TLSDESC_ADR_PAGE21: 327 checkInt(loc, val, 33, type); 328 LLVM_FALLTHROUGH; 329 case R_AARCH64_ADR_PREL_PG_HI21_NC: 330 write32AArch64Addr(loc, val >> 12); 331 break; 332 case R_AARCH64_ADR_PREL_LO21: 333 checkInt(loc, val, 21, type); 334 write32AArch64Addr(loc, val); 335 break; 336 case R_AARCH64_JUMP26: 337 // Normally we would just write the bits of the immediate field, however 338 // when patching instructions for the cpu errata fix -fix-cortex-a53-843419 339 // we want to replace a non-branch instruction with a branch immediate 340 // instruction. By writing all the bits of the instruction including the 341 // opcode and the immediate (0 001 | 01 imm26) we can do this 342 // transformation by placing a R_AARCH64_JUMP26 relocation at the offset of 343 // the instruction we want to patch. 344 write32le(loc, 0x14000000); 345 LLVM_FALLTHROUGH; 346 case R_AARCH64_CALL26: 347 checkInt(loc, val, 28, type); 348 or32le(loc, (val & 0x0FFFFFFC) >> 2); 349 break; 350 case R_AARCH64_CONDBR19: 351 case R_AARCH64_LD_PREL_LO19: 352 checkAlignment(loc, val, 4, type); 353 checkInt(loc, val, 21, type); 354 or32le(loc, (val & 0x1FFFFC) << 3); 355 break; 356 case R_AARCH64_LDST8_ABS_LO12_NC: 357 case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC: 358 or32AArch64Imm(loc, getBits(val, 0, 11)); 359 break; 360 case R_AARCH64_LDST16_ABS_LO12_NC: 361 case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC: 362 checkAlignment(loc, val, 2, type); 363 or32AArch64Imm(loc, getBits(val, 1, 11)); 364 break; 365 case R_AARCH64_LDST32_ABS_LO12_NC: 366 case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC: 367 checkAlignment(loc, val, 4, type); 368 or32AArch64Imm(loc, getBits(val, 2, 11)); 369 break; 370 case R_AARCH64_LDST64_ABS_LO12_NC: 371 case R_AARCH64_LD64_GOT_LO12_NC: 372 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 373 case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC: 374 case R_AARCH64_TLSDESC_LD64_LO12: 375 checkAlignment(loc, val, 8, type); 376 or32AArch64Imm(loc, getBits(val, 3, 11)); 377 break; 378 case R_AARCH64_LDST128_ABS_LO12_NC: 379 case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC: 380 checkAlignment(loc, val, 16, type); 381 or32AArch64Imm(loc, getBits(val, 4, 11)); 382 break; 383 case R_AARCH64_MOVW_UABS_G0: 384 checkUInt(loc, val, 16, type); 385 LLVM_FALLTHROUGH; 386 case R_AARCH64_MOVW_UABS_G0_NC: 387 or32le(loc, (val & 0xFFFF) << 5); 388 break; 389 case R_AARCH64_MOVW_UABS_G1: 390 checkUInt(loc, val, 32, type); 391 LLVM_FALLTHROUGH; 392 case R_AARCH64_MOVW_UABS_G1_NC: 393 or32le(loc, (val & 0xFFFF0000) >> 11); 394 break; 395 case R_AARCH64_MOVW_UABS_G2: 396 checkUInt(loc, val, 48, type); 397 LLVM_FALLTHROUGH; 398 case R_AARCH64_MOVW_UABS_G2_NC: 399 or32le(loc, (val & 0xFFFF00000000) >> 27); 400 break; 401 case R_AARCH64_MOVW_UABS_G3: 402 or32le(loc, (val & 0xFFFF000000000000) >> 43); 403 break; 404 case R_AARCH64_MOVW_PREL_G0: 405 case R_AARCH64_MOVW_SABS_G0: 406 case R_AARCH64_TLSLE_MOVW_TPREL_G0: 407 checkInt(loc, val, 17, type); 408 LLVM_FALLTHROUGH; 409 case R_AARCH64_MOVW_PREL_G0_NC: 410 case R_AARCH64_TLSLE_MOVW_TPREL_G0_NC: 411 writeSMovWImm(loc, val); 412 break; 413 case R_AARCH64_MOVW_PREL_G1: 414 case R_AARCH64_MOVW_SABS_G1: 415 case R_AARCH64_TLSLE_MOVW_TPREL_G1: 416 checkInt(loc, val, 33, type); 417 LLVM_FALLTHROUGH; 418 case R_AARCH64_MOVW_PREL_G1_NC: 419 case R_AARCH64_TLSLE_MOVW_TPREL_G1_NC: 420 writeSMovWImm(loc, val >> 16); 421 break; 422 case R_AARCH64_MOVW_PREL_G2: 423 case R_AARCH64_MOVW_SABS_G2: 424 case R_AARCH64_TLSLE_MOVW_TPREL_G2: 425 checkInt(loc, val, 49, type); 426 LLVM_FALLTHROUGH; 427 case R_AARCH64_MOVW_PREL_G2_NC: 428 writeSMovWImm(loc, val >> 32); 429 break; 430 case R_AARCH64_MOVW_PREL_G3: 431 writeSMovWImm(loc, val >> 48); 432 break; 433 case R_AARCH64_TSTBR14: 434 checkInt(loc, val, 16, type); 435 or32le(loc, (val & 0xFFFC) << 3); 436 break; 437 case R_AARCH64_TLSLE_ADD_TPREL_HI12: 438 checkUInt(loc, val, 24, type); 439 or32AArch64Imm(loc, val >> 12); 440 break; 441 case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC: 442 case R_AARCH64_TLSDESC_ADD_LO12: 443 or32AArch64Imm(loc, val); 444 break; 445 default: 446 llvm_unreachable("unknown relocation"); 447 } 448 } 449 450 void AArch64::relaxTlsGdToLe(uint8_t *loc, RelType type, uint64_t val) const { 451 // TLSDESC Global-Dynamic relocation are in the form: 452 // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21] 453 // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12] 454 // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12] 455 // .tlsdesccall [R_AARCH64_TLSDESC_CALL] 456 // blr x1 457 // And it can optimized to: 458 // movz x0, #0x0, lsl #16 459 // movk x0, #0x10 460 // nop 461 // nop 462 checkUInt(loc, val, 32, type); 463 464 switch (type) { 465 case R_AARCH64_TLSDESC_ADD_LO12: 466 case R_AARCH64_TLSDESC_CALL: 467 write32le(loc, 0xd503201f); // nop 468 return; 469 case R_AARCH64_TLSDESC_ADR_PAGE21: 470 write32le(loc, 0xd2a00000 | (((val >> 16) & 0xffff) << 5)); // movz 471 return; 472 case R_AARCH64_TLSDESC_LD64_LO12: 473 write32le(loc, 0xf2800000 | ((val & 0xffff) << 5)); // movk 474 return; 475 default: 476 llvm_unreachable("unsupported relocation for TLS GD to LE relaxation"); 477 } 478 } 479 480 void AArch64::relaxTlsGdToIe(uint8_t *loc, RelType type, uint64_t val) const { 481 // TLSDESC Global-Dynamic relocation are in the form: 482 // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21] 483 // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12] 484 // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12] 485 // .tlsdesccall [R_AARCH64_TLSDESC_CALL] 486 // blr x1 487 // And it can optimized to: 488 // adrp x0, :gottprel:v 489 // ldr x0, [x0, :gottprel_lo12:v] 490 // nop 491 // nop 492 493 switch (type) { 494 case R_AARCH64_TLSDESC_ADD_LO12: 495 case R_AARCH64_TLSDESC_CALL: 496 write32le(loc, 0xd503201f); // nop 497 break; 498 case R_AARCH64_TLSDESC_ADR_PAGE21: 499 write32le(loc, 0x90000000); // adrp 500 relocateOne(loc, R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21, val); 501 break; 502 case R_AARCH64_TLSDESC_LD64_LO12: 503 write32le(loc, 0xf9400000); // ldr 504 relocateOne(loc, R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC, val); 505 break; 506 default: 507 llvm_unreachable("unsupported relocation for TLS GD to LE relaxation"); 508 } 509 } 510 511 void AArch64::relaxTlsIeToLe(uint8_t *loc, RelType type, uint64_t val) const { 512 checkUInt(loc, val, 32, type); 513 514 if (type == R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21) { 515 // Generate MOVZ. 516 uint32_t regNo = read32le(loc) & 0x1f; 517 write32le(loc, (0xd2a00000 | regNo) | (((val >> 16) & 0xffff) << 5)); 518 return; 519 } 520 if (type == R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC) { 521 // Generate MOVK. 522 uint32_t regNo = read32le(loc) & 0x1f; 523 write32le(loc, (0xf2800000 | regNo) | ((val & 0xffff) << 5)); 524 return; 525 } 526 llvm_unreachable("invalid relocation for TLS IE to LE relaxation"); 527 } 528 529 // AArch64 may use security features in variant PLT sequences. These are: 530 // Pointer Authentication (PAC), introduced in armv8.3-a and Branch Target 531 // Indicator (BTI) introduced in armv8.5-a. The additional instructions used 532 // in the variant Plt sequences are encoded in the Hint space so they can be 533 // deployed on older architectures, which treat the instructions as a nop. 534 // PAC and BTI can be combined leading to the following combinations: 535 // writePltHeader 536 // writePltHeaderBti (no PAC Header needed) 537 // writePlt 538 // writePltBti (BTI only) 539 // writePltPac (PAC only) 540 // writePltBtiPac (BTI and PAC) 541 // 542 // When PAC is enabled the dynamic loader encrypts the address that it places 543 // in the .got.plt using the pacia1716 instruction which encrypts the value in 544 // x17 using the modifier in x16. The static linker places autia1716 before the 545 // indirect branch to x17 to authenticate the address in x17 with the modifier 546 // in x16. This makes it more difficult for an attacker to modify the value in 547 // the .got.plt. 548 // 549 // When BTI is enabled all indirect branches must land on a bti instruction. 550 // The static linker must place a bti instruction at the start of any PLT entry 551 // that may be the target of an indirect branch. As the PLT entries call the 552 // lazy resolver indirectly this must have a bti instruction at start. In 553 // general a bti instruction is not needed for a PLT entry as indirect calls 554 // are resolved to the function address and not the PLT entry for the function. 555 // There are a small number of cases where the PLT address can escape, such as 556 // taking the address of a function or ifunc via a non got-generating 557 // relocation, and a shared library refers to that symbol. 558 // 559 // We use the bti c variant of the instruction which permits indirect branches 560 // (br) via x16/x17 and indirect function calls (blr) via any register. The ABI 561 // guarantees that all indirect branches from code requiring BTI protection 562 // will go via x16/x17 563 564 namespace { 565 class AArch64BtiPac final : public AArch64 { 566 public: 567 AArch64BtiPac(); 568 void writePltHeader(uint8_t *buf) const override; 569 void writePlt(uint8_t *buf, uint64_t gotPltEntryAddr, uint64_t pltEntryAddr, 570 int32_t index, unsigned relOff) const override; 571 572 private: 573 bool btiHeader; // bti instruction needed in PLT Header 574 bool btiEntry; // bti instruction needed in PLT Entry 575 bool pacEntry; // autia1716 instruction needed in PLT Entry 576 }; 577 } // namespace 578 579 AArch64BtiPac::AArch64BtiPac() { 580 btiHeader = (config->andFeatures & GNU_PROPERTY_AARCH64_FEATURE_1_BTI); 581 // A BTI (Branch Target Indicator) Plt Entry is only required if the 582 // address of the PLT entry can be taken by the program, which permits an 583 // indirect jump to the PLT entry. This can happen when the address 584 // of the PLT entry for a function is canonicalised due to the address of 585 // the function in an executable being taken by a shared library. 586 // FIXME: There is a potential optimization to omit the BTI if we detect 587 // that the address of the PLT entry isn't taken. 588 btiEntry = btiHeader && !config->shared; 589 pacEntry = (config->andFeatures & GNU_PROPERTY_AARCH64_FEATURE_1_PAC); 590 591 if (btiEntry || pacEntry) 592 pltEntrySize = 24; 593 } 594 595 void AArch64BtiPac::writePltHeader(uint8_t *buf) const { 596 const uint8_t btiData[] = { 0x5f, 0x24, 0x03, 0xd5 }; // bti c 597 const uint8_t pltData[] = { 598 0xf0, 0x7b, 0xbf, 0xa9, // stp x16, x30, [sp,#-16]! 599 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[2])) 600 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[2]))] 601 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.plt.got[2])) 602 0x20, 0x02, 0x1f, 0xd6, // br x17 603 0x1f, 0x20, 0x03, 0xd5, // nop 604 0x1f, 0x20, 0x03, 0xd5 // nop 605 }; 606 const uint8_t nopData[] = { 0x1f, 0x20, 0x03, 0xd5 }; // nop 607 608 uint64_t got = in.gotPlt->getVA(); 609 uint64_t plt = in.plt->getVA(); 610 611 if (btiHeader) { 612 // PltHeader is called indirectly by plt[N]. Prefix pltData with a BTI C 613 // instruction. 614 memcpy(buf, btiData, sizeof(btiData)); 615 buf += sizeof(btiData); 616 plt += sizeof(btiData); 617 } 618 memcpy(buf, pltData, sizeof(pltData)); 619 620 relocateOne(buf + 4, R_AARCH64_ADR_PREL_PG_HI21, 621 getAArch64Page(got + 16) - getAArch64Page(plt + 8)); 622 relocateOne(buf + 8, R_AARCH64_LDST64_ABS_LO12_NC, got + 16); 623 relocateOne(buf + 12, R_AARCH64_ADD_ABS_LO12_NC, got + 16); 624 if (!btiHeader) 625 // We didn't add the BTI c instruction so round out size with NOP. 626 memcpy(buf + sizeof(pltData), nopData, sizeof(nopData)); 627 } 628 629 void AArch64BtiPac::writePlt(uint8_t *buf, uint64_t gotPltEntryAddr, 630 uint64_t pltEntryAddr, int32_t index, 631 unsigned relOff) const { 632 // The PLT entry is of the form: 633 // [btiData] addrInst (pacBr | stdBr) [nopData] 634 const uint8_t btiData[] = { 0x5f, 0x24, 0x03, 0xd5 }; // bti c 635 const uint8_t addrInst[] = { 636 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[n])) 637 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[n]))] 638 0x10, 0x02, 0x00, 0x91 // add x16, x16, Offset(&(.plt.got[n])) 639 }; 640 const uint8_t pacBr[] = { 641 0x9f, 0x21, 0x03, 0xd5, // autia1716 642 0x20, 0x02, 0x1f, 0xd6 // br x17 643 }; 644 const uint8_t stdBr[] = { 645 0x20, 0x02, 0x1f, 0xd6, // br x17 646 0x1f, 0x20, 0x03, 0xd5 // nop 647 }; 648 const uint8_t nopData[] = { 0x1f, 0x20, 0x03, 0xd5 }; // nop 649 650 if (btiEntry) { 651 memcpy(buf, btiData, sizeof(btiData)); 652 buf += sizeof(btiData); 653 pltEntryAddr += sizeof(btiData); 654 } 655 656 memcpy(buf, addrInst, sizeof(addrInst)); 657 relocateOne(buf, R_AARCH64_ADR_PREL_PG_HI21, 658 getAArch64Page(gotPltEntryAddr) - 659 getAArch64Page(pltEntryAddr)); 660 relocateOne(buf + 4, R_AARCH64_LDST64_ABS_LO12_NC, gotPltEntryAddr); 661 relocateOne(buf + 8, R_AARCH64_ADD_ABS_LO12_NC, gotPltEntryAddr); 662 663 if (pacEntry) 664 memcpy(buf + sizeof(addrInst), pacBr, sizeof(pacBr)); 665 else 666 memcpy(buf + sizeof(addrInst), stdBr, sizeof(stdBr)); 667 if (!btiEntry) 668 // We didn't add the BTI c instruction so round out size with NOP. 669 memcpy(buf + sizeof(addrInst) + sizeof(stdBr), nopData, sizeof(nopData)); 670 } 671 672 static TargetInfo *getTargetInfo() { 673 if (config->andFeatures & (GNU_PROPERTY_AARCH64_FEATURE_1_BTI | 674 GNU_PROPERTY_AARCH64_FEATURE_1_PAC)) { 675 static AArch64BtiPac t; 676 return &t; 677 } 678 static AArch64 t; 679 return &t; 680 } 681 682 TargetInfo *elf::getAArch64TargetInfo() { return getTargetInfo(); } 683