1 //===- X86_64.cpp ---------------------------------------------------------===//
2 //
3 // The LLVM Linker
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9
10 #include "InputFiles.h"
11 #include "Symbols.h"
12 #include "SyntheticSections.h"
13 #include "Target.h"
14 #include "lld/Common/ErrorHandler.h"
15 #include "llvm/Object/ELF.h"
16 #include "llvm/Support/Endian.h"
17
18 using namespace llvm;
19 using namespace llvm::object;
20 using namespace llvm::support::endian;
21 using namespace llvm::ELF;
22 using namespace lld;
23 using namespace lld::elf;
24
25 namespace {
26 template <class ELFT> class X86_64 : public TargetInfo {
27 public:
28 X86_64();
29 RelExpr getRelExpr(RelType Type, const Symbol &S,
30 const uint8_t *Loc) const override;
31 RelType getDynRel(RelType Type) const override;
32 void writeGotPltHeader(uint8_t *Buf) const override;
33 void writeGotPlt(uint8_t *Buf, const Symbol &S) const override;
34 void writePltHeader(uint8_t *Buf) const override;
35 void writePlt(uint8_t *Buf, uint64_t GotPltEntryAddr, uint64_t PltEntryAddr,
36 int32_t Index, unsigned RelOff) const override;
37 void relocateOne(uint8_t *Loc, RelType Type, uint64_t Val) const override;
38
39 RelExpr adjustRelaxExpr(RelType Type, const uint8_t *Data,
40 RelExpr Expr) const override;
41 void relaxGot(uint8_t *Loc, uint64_t Val) const override;
42 void relaxTlsGdToIe(uint8_t *Loc, RelType Type, uint64_t Val) const override;
43 void relaxTlsGdToLe(uint8_t *Loc, RelType Type, uint64_t Val) const override;
44 void relaxTlsIeToLe(uint8_t *Loc, RelType Type, uint64_t Val) const override;
45 void relaxTlsLdToLe(uint8_t *Loc, RelType Type, uint64_t Val) const override;
46 bool adjustPrologueForCrossSplitStack(uint8_t *Loc, uint8_t *End,
47 uint8_t StOther) const override;
48
49 private:
50 void relaxGotNoPic(uint8_t *Loc, uint64_t Val, uint8_t Op,
51 uint8_t ModRm) const;
52 };
53 } // namespace
54
X86_64()55 template <class ELFT> X86_64<ELFT>::X86_64() {
56 CopyRel = R_X86_64_COPY;
57 GotRel = R_X86_64_GLOB_DAT;
58 NoneRel = R_X86_64_NONE;
59 PltRel = R_X86_64_JUMP_SLOT;
60 RelativeRel = R_X86_64_RELATIVE;
61 IRelativeRel = R_X86_64_IRELATIVE;
62 TlsGotRel = R_X86_64_TPOFF64;
63 TlsModuleIndexRel = R_X86_64_DTPMOD64;
64 TlsOffsetRel = R_X86_64_DTPOFF64;
65 GotEntrySize = 8;
66 GotPltEntrySize = 8;
67 PltEntrySize = 16;
68 PltHeaderSize = 16;
69 TlsGdRelaxSkip = 2;
70 TrapInstr = {0xcc, 0xcc, 0xcc, 0xcc}; // 0xcc = INT3
71
72 // Align to the large page size (known as a superpage or huge page).
73 // FreeBSD automatically promotes large, superpage-aligned allocations.
74 DefaultImageBase = 0x200000;
75 }
76
77 template <class ELFT>
getRelExpr(RelType Type,const Symbol & S,const uint8_t * Loc) const78 RelExpr X86_64<ELFT>::getRelExpr(RelType Type, const Symbol &S,
79 const uint8_t *Loc) const {
80 if (Type == R_X86_64_GOTTPOFF)
81 Config->HasStaticTlsModel = true;
82
83 switch (Type) {
84 case R_X86_64_8:
85 case R_X86_64_16:
86 case R_X86_64_32:
87 case R_X86_64_32S:
88 case R_X86_64_64:
89 case R_X86_64_DTPOFF32:
90 case R_X86_64_DTPOFF64:
91 return R_ABS;
92 case R_X86_64_TPOFF32:
93 return R_TLS;
94 case R_X86_64_TLSLD:
95 return R_TLSLD_PC;
96 case R_X86_64_TLSGD:
97 return R_TLSGD_PC;
98 case R_X86_64_SIZE32:
99 case R_X86_64_SIZE64:
100 return R_SIZE;
101 case R_X86_64_PLT32:
102 return R_PLT_PC;
103 case R_X86_64_PC32:
104 case R_X86_64_PC64:
105 return R_PC;
106 case R_X86_64_GOT32:
107 case R_X86_64_GOT64:
108 return R_GOT_FROM_END;
109 case R_X86_64_GOTPCREL:
110 case R_X86_64_GOTPCRELX:
111 case R_X86_64_REX_GOTPCRELX:
112 case R_X86_64_GOTTPOFF:
113 return R_GOT_PC;
114 case R_X86_64_GOTOFF64:
115 return R_GOTREL_FROM_END;
116 case R_X86_64_GOTPC32:
117 case R_X86_64_GOTPC64:
118 return R_GOTONLY_PC_FROM_END;
119 case R_X86_64_NONE:
120 return R_NONE;
121 default:
122 return R_INVALID;
123 }
124 }
125
writeGotPltHeader(uint8_t * Buf) const126 template <class ELFT> void X86_64<ELFT>::writeGotPltHeader(uint8_t *Buf) const {
127 // The first entry holds the value of _DYNAMIC. It is not clear why that is
128 // required, but it is documented in the psabi and the glibc dynamic linker
129 // seems to use it (note that this is relevant for linking ld.so, not any
130 // other program).
131 write64le(Buf, In.Dynamic->getVA());
132 }
133
134 template <class ELFT>
writeGotPlt(uint8_t * Buf,const Symbol & S) const135 void X86_64<ELFT>::writeGotPlt(uint8_t *Buf, const Symbol &S) const {
136 // See comments in X86::writeGotPlt.
137 write64le(Buf, S.getPltVA() + 6);
138 }
139
writePltHeader(uint8_t * Buf) const140 template <class ELFT> void X86_64<ELFT>::writePltHeader(uint8_t *Buf) const {
141 const uint8_t PltData[] = {
142 0xff, 0x35, 0, 0, 0, 0, // pushq GOTPLT+8(%rip)
143 0xff, 0x25, 0, 0, 0, 0, // jmp *GOTPLT+16(%rip)
144 0x0f, 0x1f, 0x40, 0x00, // nop
145 };
146 memcpy(Buf, PltData, sizeof(PltData));
147 uint64_t GotPlt = In.GotPlt->getVA();
148 uint64_t Plt = In.Plt->getVA();
149 write32le(Buf + 2, GotPlt - Plt + 2); // GOTPLT+8
150 write32le(Buf + 8, GotPlt - Plt + 4); // GOTPLT+16
151 }
152
153 template <class ELFT>
writePlt(uint8_t * Buf,uint64_t GotPltEntryAddr,uint64_t PltEntryAddr,int32_t Index,unsigned RelOff) const154 void X86_64<ELFT>::writePlt(uint8_t *Buf, uint64_t GotPltEntryAddr,
155 uint64_t PltEntryAddr, int32_t Index,
156 unsigned RelOff) const {
157 const uint8_t Inst[] = {
158 0xff, 0x25, 0, 0, 0, 0, // jmpq *got(%rip)
159 0x68, 0, 0, 0, 0, // pushq <relocation index>
160 0xe9, 0, 0, 0, 0, // jmpq plt[0]
161 };
162 memcpy(Buf, Inst, sizeof(Inst));
163
164 write32le(Buf + 2, GotPltEntryAddr - PltEntryAddr - 6);
165 write32le(Buf + 7, Index);
166 write32le(Buf + 12, -getPltEntryOffset(Index) - 16);
167 }
168
getDynRel(RelType Type) const169 template <class ELFT> RelType X86_64<ELFT>::getDynRel(RelType Type) const {
170 if (Type == R_X86_64_64 || Type == R_X86_64_PC64 || Type == R_X86_64_SIZE32 ||
171 Type == R_X86_64_SIZE64)
172 return Type;
173 return R_X86_64_NONE;
174 }
175
176 template <class ELFT>
relaxTlsGdToLe(uint8_t * Loc,RelType Type,uint64_t Val) const177 void X86_64<ELFT>::relaxTlsGdToLe(uint8_t *Loc, RelType Type,
178 uint64_t Val) const {
179 // Convert
180 // .byte 0x66
181 // leaq x@tlsgd(%rip), %rdi
182 // .word 0x6666
183 // rex64
184 // call __tls_get_addr@plt
185 // to
186 // mov %fs:0x0,%rax
187 // lea x@tpoff,%rax
188 const uint8_t Inst[] = {
189 0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00, 0x00, // mov %fs:0x0,%rax
190 0x48, 0x8d, 0x80, 0, 0, 0, 0, // lea x@tpoff,%rax
191 };
192 memcpy(Loc - 4, Inst, sizeof(Inst));
193
194 // The original code used a pc relative relocation and so we have to
195 // compensate for the -4 in had in the addend.
196 write32le(Loc + 8, Val + 4);
197 }
198
199 template <class ELFT>
relaxTlsGdToIe(uint8_t * Loc,RelType Type,uint64_t Val) const200 void X86_64<ELFT>::relaxTlsGdToIe(uint8_t *Loc, RelType Type,
201 uint64_t Val) const {
202 // Convert
203 // .byte 0x66
204 // leaq x@tlsgd(%rip), %rdi
205 // .word 0x6666
206 // rex64
207 // call __tls_get_addr@plt
208 // to
209 // mov %fs:0x0,%rax
210 // addq x@tpoff,%rax
211 const uint8_t Inst[] = {
212 0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00, 0x00, // mov %fs:0x0,%rax
213 0x48, 0x03, 0x05, 0, 0, 0, 0, // addq x@tpoff,%rax
214 };
215 memcpy(Loc - 4, Inst, sizeof(Inst));
216
217 // Both code sequences are PC relatives, but since we are moving the constant
218 // forward by 8 bytes we have to subtract the value by 8.
219 write32le(Loc + 8, Val - 8);
220 }
221
222 // In some conditions, R_X86_64_GOTTPOFF relocation can be optimized to
223 // R_X86_64_TPOFF32 so that it does not use GOT.
224 template <class ELFT>
relaxTlsIeToLe(uint8_t * Loc,RelType Type,uint64_t Val) const225 void X86_64<ELFT>::relaxTlsIeToLe(uint8_t *Loc, RelType Type,
226 uint64_t Val) const {
227 uint8_t *Inst = Loc - 3;
228 uint8_t Reg = Loc[-1] >> 3;
229 uint8_t *RegSlot = Loc - 1;
230
231 // Note that ADD with RSP or R12 is converted to ADD instead of LEA
232 // because LEA with these registers needs 4 bytes to encode and thus
233 // wouldn't fit the space.
234
235 if (memcmp(Inst, "\x48\x03\x25", 3) == 0) {
236 // "addq foo@gottpoff(%rip),%rsp" -> "addq $foo,%rsp"
237 memcpy(Inst, "\x48\x81\xc4", 3);
238 } else if (memcmp(Inst, "\x4c\x03\x25", 3) == 0) {
239 // "addq foo@gottpoff(%rip),%r12" -> "addq $foo,%r12"
240 memcpy(Inst, "\x49\x81\xc4", 3);
241 } else if (memcmp(Inst, "\x4c\x03", 2) == 0) {
242 // "addq foo@gottpoff(%rip),%r[8-15]" -> "leaq foo(%r[8-15]),%r[8-15]"
243 memcpy(Inst, "\x4d\x8d", 2);
244 *RegSlot = 0x80 | (Reg << 3) | Reg;
245 } else if (memcmp(Inst, "\x48\x03", 2) == 0) {
246 // "addq foo@gottpoff(%rip),%reg -> "leaq foo(%reg),%reg"
247 memcpy(Inst, "\x48\x8d", 2);
248 *RegSlot = 0x80 | (Reg << 3) | Reg;
249 } else if (memcmp(Inst, "\x4c\x8b", 2) == 0) {
250 // "movq foo@gottpoff(%rip),%r[8-15]" -> "movq $foo,%r[8-15]"
251 memcpy(Inst, "\x49\xc7", 2);
252 *RegSlot = 0xc0 | Reg;
253 } else if (memcmp(Inst, "\x48\x8b", 2) == 0) {
254 // "movq foo@gottpoff(%rip),%reg" -> "movq $foo,%reg"
255 memcpy(Inst, "\x48\xc7", 2);
256 *RegSlot = 0xc0 | Reg;
257 } else {
258 error(getErrorLocation(Loc - 3) +
259 "R_X86_64_GOTTPOFF must be used in MOVQ or ADDQ instructions only");
260 }
261
262 // The original code used a PC relative relocation.
263 // Need to compensate for the -4 it had in the addend.
264 write32le(Loc, Val + 4);
265 }
266
267 template <class ELFT>
relaxTlsLdToLe(uint8_t * Loc,RelType Type,uint64_t Val) const268 void X86_64<ELFT>::relaxTlsLdToLe(uint8_t *Loc, RelType Type,
269 uint64_t Val) const {
270 if (Type == R_X86_64_DTPOFF64) {
271 write64le(Loc, Val);
272 return;
273 }
274 if (Type == R_X86_64_DTPOFF32) {
275 write32le(Loc, Val);
276 return;
277 }
278
279 const uint8_t Inst[] = {
280 0x66, 0x66, // .word 0x6666
281 0x66, // .byte 0x66
282 0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00, 0x00, // mov %fs:0,%rax
283 };
284
285 if (Loc[4] == 0xe8) {
286 // Convert
287 // leaq bar@tlsld(%rip), %rdi # 48 8d 3d <Loc>
288 // callq __tls_get_addr@PLT # e8 <disp32>
289 // leaq bar@dtpoff(%rax), %rcx
290 // to
291 // .word 0x6666
292 // .byte 0x66
293 // mov %fs:0,%rax
294 // leaq bar@tpoff(%rax), %rcx
295 memcpy(Loc - 3, Inst, sizeof(Inst));
296 return;
297 }
298
299 if (Loc[4] == 0xff && Loc[5] == 0x15) {
300 // Convert
301 // leaq x@tlsld(%rip),%rdi # 48 8d 3d <Loc>
302 // call *__tls_get_addr@GOTPCREL(%rip) # ff 15 <disp32>
303 // to
304 // .long 0x66666666
305 // movq %fs:0,%rax
306 // See "Table 11.9: LD -> LE Code Transition (LP64)" in
307 // https://raw.githubusercontent.com/wiki/hjl-tools/x86-psABI/x86-64-psABI-1.0.pdf
308 Loc[-3] = 0x66;
309 memcpy(Loc - 2, Inst, sizeof(Inst));
310 return;
311 }
312
313 error(getErrorLocation(Loc - 3) +
314 "expected R_X86_64_PLT32 or R_X86_64_GOTPCRELX after R_X86_64_TLSLD");
315 }
316
317 template <class ELFT>
relocateOne(uint8_t * Loc,RelType Type,uint64_t Val) const318 void X86_64<ELFT>::relocateOne(uint8_t *Loc, RelType Type, uint64_t Val) const {
319 switch (Type) {
320 case R_X86_64_8:
321 checkUInt(Loc, Val, 8, Type);
322 *Loc = Val;
323 break;
324 case R_X86_64_16:
325 checkUInt(Loc, Val, 16, Type);
326 write16le(Loc, Val);
327 break;
328 case R_X86_64_32:
329 checkUInt(Loc, Val, 32, Type);
330 write32le(Loc, Val);
331 break;
332 case R_X86_64_32S:
333 case R_X86_64_TPOFF32:
334 case R_X86_64_GOT32:
335 case R_X86_64_GOTPC32:
336 case R_X86_64_GOTPCREL:
337 case R_X86_64_GOTPCRELX:
338 case R_X86_64_REX_GOTPCRELX:
339 case R_X86_64_PC32:
340 case R_X86_64_GOTTPOFF:
341 case R_X86_64_PLT32:
342 case R_X86_64_TLSGD:
343 case R_X86_64_TLSLD:
344 case R_X86_64_DTPOFF32:
345 case R_X86_64_SIZE32:
346 checkInt(Loc, Val, 32, Type);
347 write32le(Loc, Val);
348 break;
349 case R_X86_64_64:
350 case R_X86_64_DTPOFF64:
351 case R_X86_64_GLOB_DAT:
352 case R_X86_64_PC64:
353 case R_X86_64_SIZE64:
354 case R_X86_64_GOT64:
355 case R_X86_64_GOTOFF64:
356 case R_X86_64_GOTPC64:
357 write64le(Loc, Val);
358 break;
359 default:
360 error(getErrorLocation(Loc) + "unrecognized reloc " + Twine(Type));
361 }
362 }
363
364 template <class ELFT>
adjustRelaxExpr(RelType Type,const uint8_t * Data,RelExpr RelExpr) const365 RelExpr X86_64<ELFT>::adjustRelaxExpr(RelType Type, const uint8_t *Data,
366 RelExpr RelExpr) const {
367 if (Type != R_X86_64_GOTPCRELX && Type != R_X86_64_REX_GOTPCRELX)
368 return RelExpr;
369 const uint8_t Op = Data[-2];
370 const uint8_t ModRm = Data[-1];
371
372 // FIXME: When PIC is disabled and foo is defined locally in the
373 // lower 32 bit address space, memory operand in mov can be converted into
374 // immediate operand. Otherwise, mov must be changed to lea. We support only
375 // latter relaxation at this moment.
376 if (Op == 0x8b)
377 return R_RELAX_GOT_PC;
378
379 // Relax call and jmp.
380 if (Op == 0xff && (ModRm == 0x15 || ModRm == 0x25))
381 return R_RELAX_GOT_PC;
382
383 // Relaxation of test, adc, add, and, cmp, or, sbb, sub, xor.
384 // If PIC then no relaxation is available.
385 // We also don't relax test/binop instructions without REX byte,
386 // they are 32bit operations and not common to have.
387 assert(Type == R_X86_64_REX_GOTPCRELX);
388 return Config->Pic ? RelExpr : R_RELAX_GOT_PC_NOPIC;
389 }
390
391 // A subset of relaxations can only be applied for no-PIC. This method
392 // handles such relaxations. Instructions encoding information was taken from:
393 // "Intel 64 and IA-32 Architectures Software Developer's Manual V2"
394 // (http://www.intel.com/content/dam/www/public/us/en/documents/manuals/
395 // 64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf)
396 template <class ELFT>
relaxGotNoPic(uint8_t * Loc,uint64_t Val,uint8_t Op,uint8_t ModRm) const397 void X86_64<ELFT>::relaxGotNoPic(uint8_t *Loc, uint64_t Val, uint8_t Op,
398 uint8_t ModRm) const {
399 const uint8_t Rex = Loc[-3];
400 // Convert "test %reg, foo@GOTPCREL(%rip)" to "test $foo, %reg".
401 if (Op == 0x85) {
402 // See "TEST-Logical Compare" (4-428 Vol. 2B),
403 // TEST r/m64, r64 uses "full" ModR / M byte (no opcode extension).
404
405 // ModR/M byte has form XX YYY ZZZ, where
406 // YYY is MODRM.reg(register 2), ZZZ is MODRM.rm(register 1).
407 // XX has different meanings:
408 // 00: The operand's memory address is in reg1.
409 // 01: The operand's memory address is reg1 + a byte-sized displacement.
410 // 10: The operand's memory address is reg1 + a word-sized displacement.
411 // 11: The operand is reg1 itself.
412 // If an instruction requires only one operand, the unused reg2 field
413 // holds extra opcode bits rather than a register code
414 // 0xC0 == 11 000 000 binary.
415 // 0x38 == 00 111 000 binary.
416 // We transfer reg2 to reg1 here as operand.
417 // See "2.1.3 ModR/M and SIB Bytes" (Vol. 2A 2-3).
418 Loc[-1] = 0xc0 | (ModRm & 0x38) >> 3; // ModR/M byte.
419
420 // Change opcode from TEST r/m64, r64 to TEST r/m64, imm32
421 // See "TEST-Logical Compare" (4-428 Vol. 2B).
422 Loc[-2] = 0xf7;
423
424 // Move R bit to the B bit in REX byte.
425 // REX byte is encoded as 0100WRXB, where
426 // 0100 is 4bit fixed pattern.
427 // REX.W When 1, a 64-bit operand size is used. Otherwise, when 0, the
428 // default operand size is used (which is 32-bit for most but not all
429 // instructions).
430 // REX.R This 1-bit value is an extension to the MODRM.reg field.
431 // REX.X This 1-bit value is an extension to the SIB.index field.
432 // REX.B This 1-bit value is an extension to the MODRM.rm field or the
433 // SIB.base field.
434 // See "2.2.1.2 More on REX Prefix Fields " (2-8 Vol. 2A).
435 Loc[-3] = (Rex & ~0x4) | (Rex & 0x4) >> 2;
436 write32le(Loc, Val);
437 return;
438 }
439
440 // If we are here then we need to relax the adc, add, and, cmp, or, sbb, sub
441 // or xor operations.
442
443 // Convert "binop foo@GOTPCREL(%rip), %reg" to "binop $foo, %reg".
444 // Logic is close to one for test instruction above, but we also
445 // write opcode extension here, see below for details.
446 Loc[-1] = 0xc0 | (ModRm & 0x38) >> 3 | (Op & 0x3c); // ModR/M byte.
447
448 // Primary opcode is 0x81, opcode extension is one of:
449 // 000b = ADD, 001b is OR, 010b is ADC, 011b is SBB,
450 // 100b is AND, 101b is SUB, 110b is XOR, 111b is CMP.
451 // This value was wrote to MODRM.reg in a line above.
452 // See "3.2 INSTRUCTIONS (A-M)" (Vol. 2A 3-15),
453 // "INSTRUCTION SET REFERENCE, N-Z" (Vol. 2B 4-1) for
454 // descriptions about each operation.
455 Loc[-2] = 0x81;
456 Loc[-3] = (Rex & ~0x4) | (Rex & 0x4) >> 2;
457 write32le(Loc, Val);
458 }
459
460 template <class ELFT>
relaxGot(uint8_t * Loc,uint64_t Val) const461 void X86_64<ELFT>::relaxGot(uint8_t *Loc, uint64_t Val) const {
462 const uint8_t Op = Loc[-2];
463 const uint8_t ModRm = Loc[-1];
464
465 // Convert "mov foo@GOTPCREL(%rip),%reg" to "lea foo(%rip),%reg".
466 if (Op == 0x8b) {
467 Loc[-2] = 0x8d;
468 write32le(Loc, Val);
469 return;
470 }
471
472 if (Op != 0xff) {
473 // We are relaxing a rip relative to an absolute, so compensate
474 // for the old -4 addend.
475 assert(!Config->Pic);
476 relaxGotNoPic(Loc, Val + 4, Op, ModRm);
477 return;
478 }
479
480 // Convert call/jmp instructions.
481 if (ModRm == 0x15) {
482 // ABI says we can convert "call *foo@GOTPCREL(%rip)" to "nop; call foo".
483 // Instead we convert to "addr32 call foo" where addr32 is an instruction
484 // prefix. That makes result expression to be a single instruction.
485 Loc[-2] = 0x67; // addr32 prefix
486 Loc[-1] = 0xe8; // call
487 write32le(Loc, Val);
488 return;
489 }
490
491 // Convert "jmp *foo@GOTPCREL(%rip)" to "jmp foo; nop".
492 // jmp doesn't return, so it is fine to use nop here, it is just a stub.
493 assert(ModRm == 0x25);
494 Loc[-2] = 0xe9; // jmp
495 Loc[3] = 0x90; // nop
496 write32le(Loc - 1, Val + 1);
497 }
498
499 // This anonymous namespace works around a warning bug in
500 // old versions of gcc. See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=56480
501 namespace {
502
503 // A split-stack prologue starts by checking the amount of stack remaining
504 // in one of two ways:
505 // A) Comparing of the stack pointer to a field in the tcb.
506 // B) Or a load of a stack pointer offset with an lea to r10 or r11.
507 template <>
adjustPrologueForCrossSplitStack(uint8_t * Loc,uint8_t * End,uint8_t StOther) const508 bool X86_64<ELF64LE>::adjustPrologueForCrossSplitStack(uint8_t *Loc,
509 uint8_t *End,
510 uint8_t StOther) const {
511 if (Loc + 8 >= End)
512 return false;
513
514 // Replace "cmp %fs:0x70,%rsp" and subsequent branch
515 // with "stc, nopl 0x0(%rax,%rax,1)"
516 if (memcmp(Loc, "\x64\x48\x3b\x24\x25", 5) == 0) {
517 memcpy(Loc, "\xf9\x0f\x1f\x84\x00\x00\x00\x00", 8);
518 return true;
519 }
520
521 // Adjust "lea X(%rsp),%rYY" to lea "(X - 0x4000)(%rsp),%rYY" where rYY could
522 // be r10 or r11. The lea instruction feeds a subsequent compare which checks
523 // if there is X available stack space. Making X larger effectively reserves
524 // that much additional space. The stack grows downward so subtract the value.
525 if (memcmp(Loc, "\x4c\x8d\x94\x24", 4) == 0 ||
526 memcmp(Loc, "\x4c\x8d\x9c\x24", 4) == 0) {
527 // The offset bytes are encoded four bytes after the start of the
528 // instruction.
529 write32le(Loc + 4, read32le(Loc + 4) - 0x4000);
530 return true;
531 }
532 return false;
533 }
534
535 template <>
adjustPrologueForCrossSplitStack(uint8_t * Loc,uint8_t * End,uint8_t StOther) const536 bool X86_64<ELF32LE>::adjustPrologueForCrossSplitStack(uint8_t *Loc,
537 uint8_t *End,
538 uint8_t StOther) const {
539 llvm_unreachable("Target doesn't support split stacks.");
540 }
541
542 } // namespace
543
544 // These nonstandard PLT entries are to migtigate Spectre v2 security
545 // vulnerability. In order to mitigate Spectre v2, we want to avoid indirect
546 // branch instructions such as `jmp *GOTPLT(%rip)`. So, in the following PLT
547 // entries, we use a CALL followed by MOV and RET to do the same thing as an
548 // indirect jump. That instruction sequence is so-called "retpoline".
549 //
550 // We have two types of retpoline PLTs as a size optimization. If `-z now`
551 // is specified, all dynamic symbols are resolved at load-time. Thus, when
552 // that option is given, we can omit code for symbol lazy resolution.
553 namespace {
554 template <class ELFT> class Retpoline : public X86_64<ELFT> {
555 public:
556 Retpoline();
557 void writeGotPlt(uint8_t *Buf, const Symbol &S) const override;
558 void writePltHeader(uint8_t *Buf) const override;
559 void writePlt(uint8_t *Buf, uint64_t GotPltEntryAddr, uint64_t PltEntryAddr,
560 int32_t Index, unsigned RelOff) const override;
561 };
562
563 template <class ELFT> class RetpolineZNow : public X86_64<ELFT> {
564 public:
565 RetpolineZNow();
writeGotPlt(uint8_t * Buf,const Symbol & S) const566 void writeGotPlt(uint8_t *Buf, const Symbol &S) const override {}
567 void writePltHeader(uint8_t *Buf) const override;
568 void writePlt(uint8_t *Buf, uint64_t GotPltEntryAddr, uint64_t PltEntryAddr,
569 int32_t Index, unsigned RelOff) const override;
570 };
571 } // namespace
572
Retpoline()573 template <class ELFT> Retpoline<ELFT>::Retpoline() {
574 TargetInfo::PltHeaderSize = 48;
575 TargetInfo::PltEntrySize = 32;
576 }
577
578 template <class ELFT>
writeGotPlt(uint8_t * Buf,const Symbol & S) const579 void Retpoline<ELFT>::writeGotPlt(uint8_t *Buf, const Symbol &S) const {
580 write64le(Buf, S.getPltVA() + 17);
581 }
582
writePltHeader(uint8_t * Buf) const583 template <class ELFT> void Retpoline<ELFT>::writePltHeader(uint8_t *Buf) const {
584 const uint8_t Insn[] = {
585 0xff, 0x35, 0, 0, 0, 0, // 0: pushq GOTPLT+8(%rip)
586 0x4c, 0x8b, 0x1d, 0, 0, 0, 0, // 6: mov GOTPLT+16(%rip), %r11
587 0xe8, 0x0e, 0x00, 0x00, 0x00, // d: callq next
588 0xf3, 0x90, // 12: loop: pause
589 0x0f, 0xae, 0xe8, // 14: lfence
590 0xeb, 0xf9, // 17: jmp loop
591 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // 19: int3; .align 16
592 0x4c, 0x89, 0x1c, 0x24, // 20: next: mov %r11, (%rsp)
593 0xc3, // 24: ret
594 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // 25: int3; padding
595 0xcc, 0xcc, 0xcc, 0xcc, // 2c: int3; padding
596 };
597 memcpy(Buf, Insn, sizeof(Insn));
598
599 uint64_t GotPlt = In.GotPlt->getVA();
600 uint64_t Plt = In.Plt->getVA();
601 write32le(Buf + 2, GotPlt - Plt - 6 + 8);
602 write32le(Buf + 9, GotPlt - Plt - 13 + 16);
603 }
604
605 template <class ELFT>
writePlt(uint8_t * Buf,uint64_t GotPltEntryAddr,uint64_t PltEntryAddr,int32_t Index,unsigned RelOff) const606 void Retpoline<ELFT>::writePlt(uint8_t *Buf, uint64_t GotPltEntryAddr,
607 uint64_t PltEntryAddr, int32_t Index,
608 unsigned RelOff) const {
609 const uint8_t Insn[] = {
610 0x4c, 0x8b, 0x1d, 0, 0, 0, 0, // 0: mov foo@GOTPLT(%rip), %r11
611 0xe8, 0, 0, 0, 0, // 7: callq plt+0x20
612 0xe9, 0, 0, 0, 0, // c: jmp plt+0x12
613 0x68, 0, 0, 0, 0, // 11: pushq <relocation index>
614 0xe9, 0, 0, 0, 0, // 16: jmp plt+0
615 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // 1b: int3; padding
616 };
617 memcpy(Buf, Insn, sizeof(Insn));
618
619 uint64_t Off = getPltEntryOffset(Index);
620
621 write32le(Buf + 3, GotPltEntryAddr - PltEntryAddr - 7);
622 write32le(Buf + 8, -Off - 12 + 32);
623 write32le(Buf + 13, -Off - 17 + 18);
624 write32le(Buf + 18, Index);
625 write32le(Buf + 23, -Off - 27);
626 }
627
RetpolineZNow()628 template <class ELFT> RetpolineZNow<ELFT>::RetpolineZNow() {
629 TargetInfo::PltHeaderSize = 32;
630 TargetInfo::PltEntrySize = 16;
631 }
632
633 template <class ELFT>
writePltHeader(uint8_t * Buf) const634 void RetpolineZNow<ELFT>::writePltHeader(uint8_t *Buf) const {
635 const uint8_t Insn[] = {
636 0xe8, 0x0b, 0x00, 0x00, 0x00, // 0: call next
637 0xf3, 0x90, // 5: loop: pause
638 0x0f, 0xae, 0xe8, // 7: lfence
639 0xeb, 0xf9, // a: jmp loop
640 0xcc, 0xcc, 0xcc, 0xcc, // c: int3; .align 16
641 0x4c, 0x89, 0x1c, 0x24, // 10: next: mov %r11, (%rsp)
642 0xc3, // 14: ret
643 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // 15: int3; padding
644 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // 1a: int3; padding
645 0xcc, // 1f: int3; padding
646 };
647 memcpy(Buf, Insn, sizeof(Insn));
648 }
649
650 template <class ELFT>
writePlt(uint8_t * Buf,uint64_t GotPltEntryAddr,uint64_t PltEntryAddr,int32_t Index,unsigned RelOff) const651 void RetpolineZNow<ELFT>::writePlt(uint8_t *Buf, uint64_t GotPltEntryAddr,
652 uint64_t PltEntryAddr, int32_t Index,
653 unsigned RelOff) const {
654 const uint8_t Insn[] = {
655 0x4c, 0x8b, 0x1d, 0, 0, 0, 0, // mov foo@GOTPLT(%rip), %r11
656 0xe9, 0, 0, 0, 0, // jmp plt+0
657 0xcc, 0xcc, 0xcc, 0xcc, // int3; padding
658 };
659 memcpy(Buf, Insn, sizeof(Insn));
660
661 write32le(Buf + 3, GotPltEntryAddr - PltEntryAddr - 7);
662 write32le(Buf + 8, -getPltEntryOffset(Index) - 12);
663 }
664
getTargetInfo()665 template <class ELFT> static TargetInfo *getTargetInfo() {
666 if (Config->ZRetpolineplt) {
667 if (Config->ZNow) {
668 static RetpolineZNow<ELFT> T;
669 return &T;
670 }
671 static Retpoline<ELFT> T;
672 return &T;
673 }
674
675 static X86_64<ELFT> T;
676 return &T;
677 }
678
getX32TargetInfo()679 TargetInfo *elf::getX32TargetInfo() { return getTargetInfo<ELF32LE>(); }
getX86_64TargetInfo()680 TargetInfo *elf::getX86_64TargetInfo() { return getTargetInfo<ELF64LE>(); }
681