1 //===- RISCV.cpp ----------------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "InputFiles.h"
10 #include "OutputSections.h"
11 #include "Symbols.h"
12 #include "SyntheticSections.h"
13 #include "Target.h"
14 #include "llvm/Support/TimeProfiler.h"
15
16 using namespace llvm;
17 using namespace llvm::object;
18 using namespace llvm::support::endian;
19 using namespace llvm::ELF;
20 using namespace lld;
21 using namespace lld::elf;
22
23 namespace {
24
25 class RISCV final : public TargetInfo {
26 public:
27 RISCV();
28 uint32_t calcEFlags() const override;
29 int64_t getImplicitAddend(const uint8_t *buf, RelType type) const override;
30 void writeGotHeader(uint8_t *buf) const override;
31 void writeGotPlt(uint8_t *buf, const Symbol &s) const override;
32 void writeIgotPlt(uint8_t *buf, const Symbol &s) const override;
33 void writePltHeader(uint8_t *buf) const override;
34 void writePlt(uint8_t *buf, const Symbol &sym,
35 uint64_t pltEntryAddr) const override;
36 RelType getDynRel(RelType type) const override;
37 RelExpr getRelExpr(RelType type, const Symbol &s,
38 const uint8_t *loc) const override;
39 void relocate(uint8_t *loc, const Relocation &rel,
40 uint64_t val) const override;
41 bool relaxOnce(int pass) const override;
42 };
43
44 } // end anonymous namespace
45
46 const uint64_t dtpOffset = 0x800;
47
48 enum Op {
49 ADDI = 0x13,
50 AUIPC = 0x17,
51 JALR = 0x67,
52 LD = 0x3003,
53 LW = 0x2003,
54 SRLI = 0x5013,
55 SUB = 0x40000033,
56 };
57
58 enum Reg {
59 X_RA = 1,
60 X_TP = 4,
61 X_T0 = 5,
62 X_T1 = 6,
63 X_T2 = 7,
64 X_T3 = 28,
65 };
66
hi20(uint32_t val)67 static uint32_t hi20(uint32_t val) { return (val + 0x800) >> 12; }
lo12(uint32_t val)68 static uint32_t lo12(uint32_t val) { return val & 4095; }
69
itype(uint32_t op,uint32_t rd,uint32_t rs1,uint32_t imm)70 static uint32_t itype(uint32_t op, uint32_t rd, uint32_t rs1, uint32_t imm) {
71 return op | (rd << 7) | (rs1 << 15) | (imm << 20);
72 }
rtype(uint32_t op,uint32_t rd,uint32_t rs1,uint32_t rs2)73 static uint32_t rtype(uint32_t op, uint32_t rd, uint32_t rs1, uint32_t rs2) {
74 return op | (rd << 7) | (rs1 << 15) | (rs2 << 20);
75 }
utype(uint32_t op,uint32_t rd,uint32_t imm)76 static uint32_t utype(uint32_t op, uint32_t rd, uint32_t imm) {
77 return op | (rd << 7) | (imm << 12);
78 }
79
80 // Extract bits v[begin:end], where range is inclusive, and begin must be < 63.
extractBits(uint64_t v,uint32_t begin,uint32_t end)81 static uint32_t extractBits(uint64_t v, uint32_t begin, uint32_t end) {
82 return (v & ((1ULL << (begin + 1)) - 1)) >> end;
83 }
84
setLO12_I(uint32_t insn,uint32_t imm)85 static uint32_t setLO12_I(uint32_t insn, uint32_t imm) {
86 return (insn & 0xfffff) | (imm << 20);
87 }
setLO12_S(uint32_t insn,uint32_t imm)88 static uint32_t setLO12_S(uint32_t insn, uint32_t imm) {
89 return (insn & 0x1fff07f) | (extractBits(imm, 11, 5) << 25) |
90 (extractBits(imm, 4, 0) << 7);
91 }
92
RISCV()93 RISCV::RISCV() {
94 copyRel = R_RISCV_COPY;
95 pltRel = R_RISCV_JUMP_SLOT;
96 relativeRel = R_RISCV_RELATIVE;
97 iRelativeRel = R_RISCV_IRELATIVE;
98 if (config->is64) {
99 symbolicRel = R_RISCV_64;
100 tlsModuleIndexRel = R_RISCV_TLS_DTPMOD64;
101 tlsOffsetRel = R_RISCV_TLS_DTPREL64;
102 tlsGotRel = R_RISCV_TLS_TPREL64;
103 } else {
104 symbolicRel = R_RISCV_32;
105 tlsModuleIndexRel = R_RISCV_TLS_DTPMOD32;
106 tlsOffsetRel = R_RISCV_TLS_DTPREL32;
107 tlsGotRel = R_RISCV_TLS_TPREL32;
108 }
109 gotRel = symbolicRel;
110
111 // .got[0] = _DYNAMIC
112 gotHeaderEntriesNum = 1;
113
114 // .got.plt[0] = _dl_runtime_resolve, .got.plt[1] = link_map
115 gotPltHeaderEntriesNum = 2;
116
117 pltHeaderSize = 32;
118 pltEntrySize = 16;
119 ipltEntrySize = 16;
120 }
121
getEFlags(InputFile * f)122 static uint32_t getEFlags(InputFile *f) {
123 if (config->is64)
124 return cast<ObjFile<ELF64LE>>(f)->getObj().getHeader().e_flags;
125 return cast<ObjFile<ELF32LE>>(f)->getObj().getHeader().e_flags;
126 }
127
calcEFlags() const128 uint32_t RISCV::calcEFlags() const {
129 // If there are only binary input files (from -b binary), use a
130 // value of 0 for the ELF header flags.
131 if (ctx->objectFiles.empty())
132 return 0;
133
134 uint32_t target = getEFlags(ctx->objectFiles.front());
135
136 for (InputFile *f : ctx->objectFiles) {
137 uint32_t eflags = getEFlags(f);
138 if (eflags & EF_RISCV_RVC)
139 target |= EF_RISCV_RVC;
140
141 if ((eflags & EF_RISCV_FLOAT_ABI) != (target & EF_RISCV_FLOAT_ABI))
142 error(toString(f) +
143 ": cannot link object files with different floating-point ABI");
144
145 if ((eflags & EF_RISCV_RVE) != (target & EF_RISCV_RVE))
146 error(toString(f) +
147 ": cannot link object files with different EF_RISCV_RVE");
148 }
149
150 return target;
151 }
152
getImplicitAddend(const uint8_t * buf,RelType type) const153 int64_t RISCV::getImplicitAddend(const uint8_t *buf, RelType type) const {
154 switch (type) {
155 default:
156 internalLinkerError(getErrorLocation(buf),
157 "cannot read addend for relocation " + toString(type));
158 return 0;
159 case R_RISCV_32:
160 case R_RISCV_TLS_DTPMOD32:
161 case R_RISCV_TLS_DTPREL32:
162 return SignExtend64<32>(read32le(buf));
163 case R_RISCV_64:
164 return read64le(buf);
165 case R_RISCV_RELATIVE:
166 case R_RISCV_IRELATIVE:
167 return config->is64 ? read64le(buf) : read32le(buf);
168 case R_RISCV_NONE:
169 case R_RISCV_JUMP_SLOT:
170 // These relocations are defined as not having an implicit addend.
171 return 0;
172 }
173 }
174
writeGotHeader(uint8_t * buf) const175 void RISCV::writeGotHeader(uint8_t *buf) const {
176 if (config->is64)
177 write64le(buf, mainPart->dynamic->getVA());
178 else
179 write32le(buf, mainPart->dynamic->getVA());
180 }
181
writeGotPlt(uint8_t * buf,const Symbol & s) const182 void RISCV::writeGotPlt(uint8_t *buf, const Symbol &s) const {
183 if (config->is64)
184 write64le(buf, in.plt->getVA());
185 else
186 write32le(buf, in.plt->getVA());
187 }
188
writeIgotPlt(uint8_t * buf,const Symbol & s) const189 void RISCV::writeIgotPlt(uint8_t *buf, const Symbol &s) const {
190 if (config->writeAddends) {
191 if (config->is64)
192 write64le(buf, s.getVA());
193 else
194 write32le(buf, s.getVA());
195 }
196 }
197
writePltHeader(uint8_t * buf) const198 void RISCV::writePltHeader(uint8_t *buf) const {
199 // 1: auipc t2, %pcrel_hi(.got.plt)
200 // sub t1, t1, t3
201 // l[wd] t3, %pcrel_lo(1b)(t2); t3 = _dl_runtime_resolve
202 // addi t1, t1, -pltHeaderSize-12; t1 = &.plt[i] - &.plt[0]
203 // addi t0, t2, %pcrel_lo(1b)
204 // srli t1, t1, (rv64?1:2); t1 = &.got.plt[i] - &.got.plt[0]
205 // l[wd] t0, Wordsize(t0); t0 = link_map
206 // jr t3
207 uint32_t offset = in.gotPlt->getVA() - in.plt->getVA();
208 uint32_t load = config->is64 ? LD : LW;
209 write32le(buf + 0, utype(AUIPC, X_T2, hi20(offset)));
210 write32le(buf + 4, rtype(SUB, X_T1, X_T1, X_T3));
211 write32le(buf + 8, itype(load, X_T3, X_T2, lo12(offset)));
212 write32le(buf + 12, itype(ADDI, X_T1, X_T1, -target->pltHeaderSize - 12));
213 write32le(buf + 16, itype(ADDI, X_T0, X_T2, lo12(offset)));
214 write32le(buf + 20, itype(SRLI, X_T1, X_T1, config->is64 ? 1 : 2));
215 write32le(buf + 24, itype(load, X_T0, X_T0, config->wordsize));
216 write32le(buf + 28, itype(JALR, 0, X_T3, 0));
217 }
218
writePlt(uint8_t * buf,const Symbol & sym,uint64_t pltEntryAddr) const219 void RISCV::writePlt(uint8_t *buf, const Symbol &sym,
220 uint64_t pltEntryAddr) const {
221 // 1: auipc t3, %pcrel_hi([email protected])
222 // l[wd] t3, %pcrel_lo(1b)(t3)
223 // jalr t1, t3
224 // nop
225 uint32_t offset = sym.getGotPltVA() - pltEntryAddr;
226 write32le(buf + 0, utype(AUIPC, X_T3, hi20(offset)));
227 write32le(buf + 4, itype(config->is64 ? LD : LW, X_T3, X_T3, lo12(offset)));
228 write32le(buf + 8, itype(JALR, X_T1, X_T3, 0));
229 write32le(buf + 12, itype(ADDI, 0, 0, 0));
230 }
231
getDynRel(RelType type) const232 RelType RISCV::getDynRel(RelType type) const {
233 return type == target->symbolicRel ? type
234 : static_cast<RelType>(R_RISCV_NONE);
235 }
236
getRelExpr(const RelType type,const Symbol & s,const uint8_t * loc) const237 RelExpr RISCV::getRelExpr(const RelType type, const Symbol &s,
238 const uint8_t *loc) const {
239 switch (type) {
240 case R_RISCV_NONE:
241 return R_NONE;
242 case R_RISCV_32:
243 case R_RISCV_64:
244 case R_RISCV_HI20:
245 case R_RISCV_LO12_I:
246 case R_RISCV_LO12_S:
247 case R_RISCV_RVC_LUI:
248 return R_ABS;
249 case R_RISCV_ADD8:
250 case R_RISCV_ADD16:
251 case R_RISCV_ADD32:
252 case R_RISCV_ADD64:
253 case R_RISCV_SET6:
254 case R_RISCV_SET8:
255 case R_RISCV_SET16:
256 case R_RISCV_SET32:
257 case R_RISCV_SUB6:
258 case R_RISCV_SUB8:
259 case R_RISCV_SUB16:
260 case R_RISCV_SUB32:
261 case R_RISCV_SUB64:
262 return R_RISCV_ADD;
263 case R_RISCV_JAL:
264 case R_RISCV_BRANCH:
265 case R_RISCV_PCREL_HI20:
266 case R_RISCV_RVC_BRANCH:
267 case R_RISCV_RVC_JUMP:
268 case R_RISCV_32_PCREL:
269 return R_PC;
270 case R_RISCV_CALL:
271 case R_RISCV_CALL_PLT:
272 return R_PLT_PC;
273 case R_RISCV_GOT_HI20:
274 return R_GOT_PC;
275 case R_RISCV_PCREL_LO12_I:
276 case R_RISCV_PCREL_LO12_S:
277 return R_RISCV_PC_INDIRECT;
278 case R_RISCV_TLS_GD_HI20:
279 return R_TLSGD_PC;
280 case R_RISCV_TLS_GOT_HI20:
281 config->hasTlsIe = true;
282 return R_GOT_PC;
283 case R_RISCV_TPREL_HI20:
284 case R_RISCV_TPREL_LO12_I:
285 case R_RISCV_TPREL_LO12_S:
286 return R_TPREL;
287 case R_RISCV_ALIGN:
288 return R_RELAX_HINT;
289 case R_RISCV_TPREL_ADD:
290 case R_RISCV_RELAX:
291 return config->relax ? R_RELAX_HINT : R_NONE;
292 default:
293 error(getErrorLocation(loc) + "unknown relocation (" + Twine(type) +
294 ") against symbol " + toString(s));
295 return R_NONE;
296 }
297 }
298
relocate(uint8_t * loc,const Relocation & rel,uint64_t val) const299 void RISCV::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
300 const unsigned bits = config->wordsize * 8;
301
302 switch (rel.type) {
303 case R_RISCV_32:
304 write32le(loc, val);
305 return;
306 case R_RISCV_64:
307 write64le(loc, val);
308 return;
309
310 case R_RISCV_RVC_BRANCH: {
311 checkInt(loc, val, 9, rel);
312 checkAlignment(loc, val, 2, rel);
313 uint16_t insn = read16le(loc) & 0xE383;
314 uint16_t imm8 = extractBits(val, 8, 8) << 12;
315 uint16_t imm4_3 = extractBits(val, 4, 3) << 10;
316 uint16_t imm7_6 = extractBits(val, 7, 6) << 5;
317 uint16_t imm2_1 = extractBits(val, 2, 1) << 3;
318 uint16_t imm5 = extractBits(val, 5, 5) << 2;
319 insn |= imm8 | imm4_3 | imm7_6 | imm2_1 | imm5;
320
321 write16le(loc, insn);
322 return;
323 }
324
325 case R_RISCV_RVC_JUMP: {
326 checkInt(loc, val, 12, rel);
327 checkAlignment(loc, val, 2, rel);
328 uint16_t insn = read16le(loc) & 0xE003;
329 uint16_t imm11 = extractBits(val, 11, 11) << 12;
330 uint16_t imm4 = extractBits(val, 4, 4) << 11;
331 uint16_t imm9_8 = extractBits(val, 9, 8) << 9;
332 uint16_t imm10 = extractBits(val, 10, 10) << 8;
333 uint16_t imm6 = extractBits(val, 6, 6) << 7;
334 uint16_t imm7 = extractBits(val, 7, 7) << 6;
335 uint16_t imm3_1 = extractBits(val, 3, 1) << 3;
336 uint16_t imm5 = extractBits(val, 5, 5) << 2;
337 insn |= imm11 | imm4 | imm9_8 | imm10 | imm6 | imm7 | imm3_1 | imm5;
338
339 write16le(loc, insn);
340 return;
341 }
342
343 case R_RISCV_RVC_LUI: {
344 int64_t imm = SignExtend64(val + 0x800, bits) >> 12;
345 checkInt(loc, imm, 6, rel);
346 if (imm == 0) { // `c.lui rd, 0` is illegal, convert to `c.li rd, 0`
347 write16le(loc, (read16le(loc) & 0x0F83) | 0x4000);
348 } else {
349 uint16_t imm17 = extractBits(val + 0x800, 17, 17) << 12;
350 uint16_t imm16_12 = extractBits(val + 0x800, 16, 12) << 2;
351 write16le(loc, (read16le(loc) & 0xEF83) | imm17 | imm16_12);
352 }
353 return;
354 }
355
356 case R_RISCV_JAL: {
357 checkInt(loc, val, 21, rel);
358 checkAlignment(loc, val, 2, rel);
359
360 uint32_t insn = read32le(loc) & 0xFFF;
361 uint32_t imm20 = extractBits(val, 20, 20) << 31;
362 uint32_t imm10_1 = extractBits(val, 10, 1) << 21;
363 uint32_t imm11 = extractBits(val, 11, 11) << 20;
364 uint32_t imm19_12 = extractBits(val, 19, 12) << 12;
365 insn |= imm20 | imm10_1 | imm11 | imm19_12;
366
367 write32le(loc, insn);
368 return;
369 }
370
371 case R_RISCV_BRANCH: {
372 checkInt(loc, val, 13, rel);
373 checkAlignment(loc, val, 2, rel);
374
375 uint32_t insn = read32le(loc) & 0x1FFF07F;
376 uint32_t imm12 = extractBits(val, 12, 12) << 31;
377 uint32_t imm10_5 = extractBits(val, 10, 5) << 25;
378 uint32_t imm4_1 = extractBits(val, 4, 1) << 8;
379 uint32_t imm11 = extractBits(val, 11, 11) << 7;
380 insn |= imm12 | imm10_5 | imm4_1 | imm11;
381
382 write32le(loc, insn);
383 return;
384 }
385
386 // auipc + jalr pair
387 case R_RISCV_CALL:
388 case R_RISCV_CALL_PLT: {
389 int64_t hi = SignExtend64(val + 0x800, bits) >> 12;
390 checkInt(loc, hi, 20, rel);
391 if (isInt<20>(hi)) {
392 relocateNoSym(loc, R_RISCV_PCREL_HI20, val);
393 relocateNoSym(loc + 4, R_RISCV_PCREL_LO12_I, val);
394 }
395 return;
396 }
397
398 case R_RISCV_GOT_HI20:
399 case R_RISCV_PCREL_HI20:
400 case R_RISCV_TLS_GD_HI20:
401 case R_RISCV_TLS_GOT_HI20:
402 case R_RISCV_TPREL_HI20:
403 case R_RISCV_HI20: {
404 uint64_t hi = val + 0x800;
405 checkInt(loc, SignExtend64(hi, bits) >> 12, 20, rel);
406 write32le(loc, (read32le(loc) & 0xFFF) | (hi & 0xFFFFF000));
407 return;
408 }
409
410 case R_RISCV_PCREL_LO12_I:
411 case R_RISCV_TPREL_LO12_I:
412 case R_RISCV_LO12_I: {
413 uint64_t hi = (val + 0x800) >> 12;
414 uint64_t lo = val - (hi << 12);
415 write32le(loc, setLO12_I(read32le(loc), lo & 0xfff));
416 return;
417 }
418
419 case R_RISCV_PCREL_LO12_S:
420 case R_RISCV_TPREL_LO12_S:
421 case R_RISCV_LO12_S: {
422 uint64_t hi = (val + 0x800) >> 12;
423 uint64_t lo = val - (hi << 12);
424 write32le(loc, setLO12_S(read32le(loc), lo));
425 return;
426 }
427
428 case R_RISCV_ADD8:
429 *loc += val;
430 return;
431 case R_RISCV_ADD16:
432 write16le(loc, read16le(loc) + val);
433 return;
434 case R_RISCV_ADD32:
435 write32le(loc, read32le(loc) + val);
436 return;
437 case R_RISCV_ADD64:
438 write64le(loc, read64le(loc) + val);
439 return;
440 case R_RISCV_SUB6:
441 *loc = (*loc & 0xc0) | (((*loc & 0x3f) - val) & 0x3f);
442 return;
443 case R_RISCV_SUB8:
444 *loc -= val;
445 return;
446 case R_RISCV_SUB16:
447 write16le(loc, read16le(loc) - val);
448 return;
449 case R_RISCV_SUB32:
450 write32le(loc, read32le(loc) - val);
451 return;
452 case R_RISCV_SUB64:
453 write64le(loc, read64le(loc) - val);
454 return;
455 case R_RISCV_SET6:
456 *loc = (*loc & 0xc0) | (val & 0x3f);
457 return;
458 case R_RISCV_SET8:
459 *loc = val;
460 return;
461 case R_RISCV_SET16:
462 write16le(loc, val);
463 return;
464 case R_RISCV_SET32:
465 case R_RISCV_32_PCREL:
466 write32le(loc, val);
467 return;
468
469 case R_RISCV_TLS_DTPREL32:
470 write32le(loc, val - dtpOffset);
471 break;
472 case R_RISCV_TLS_DTPREL64:
473 write64le(loc, val - dtpOffset);
474 break;
475
476 case R_RISCV_RELAX:
477 return; // Ignored (for now)
478
479 default:
480 llvm_unreachable("unknown relocation");
481 }
482 }
483
484 namespace {
485 struct SymbolAnchor {
486 uint64_t offset;
487 Defined *d;
488 bool end; // true for the anchor of st_value+st_size
489 };
490 } // namespace
491
492 struct elf::RISCVRelaxAux {
493 // This records symbol start and end offsets which will be adjusted according
494 // to the nearest relocDeltas element.
495 SmallVector<SymbolAnchor, 0> anchors;
496 // For relocations[i], the actual offset is r_offset - (i ? relocDeltas[i-1] :
497 // 0).
498 std::unique_ptr<uint32_t[]> relocDeltas;
499 // For relocations[i], the actual type is relocTypes[i].
500 std::unique_ptr<RelType[]> relocTypes;
501 SmallVector<uint32_t, 0> writes;
502 };
503
initSymbolAnchors()504 static void initSymbolAnchors() {
505 SmallVector<InputSection *, 0> storage;
506 for (OutputSection *osec : outputSections) {
507 if (!(osec->flags & SHF_EXECINSTR))
508 continue;
509 for (InputSection *sec : getInputSections(*osec, storage)) {
510 sec->relaxAux = make<RISCVRelaxAux>();
511 if (sec->relocations.size()) {
512 sec->relaxAux->relocDeltas =
513 std::make_unique<uint32_t[]>(sec->relocations.size());
514 sec->relaxAux->relocTypes =
515 std::make_unique<RelType[]>(sec->relocations.size());
516 }
517 }
518 }
519 // Store anchors (st_value and st_value+st_size) for symbols relative to text
520 // sections.
521 for (InputFile *file : ctx->objectFiles)
522 for (Symbol *sym : file->getSymbols()) {
523 auto *d = dyn_cast<Defined>(sym);
524 if (!d || d->file != file)
525 continue;
526 if (auto *sec = dyn_cast_or_null<InputSection>(d->section))
527 if (sec->flags & SHF_EXECINSTR && sec->relaxAux) {
528 // If sec is discarded, relaxAux will be nullptr.
529 sec->relaxAux->anchors.push_back({d->value, d, false});
530 sec->relaxAux->anchors.push_back({d->value + d->size, d, true});
531 }
532 }
533 // Sort anchors by offset so that we can find the closest relocation
534 // efficiently. For a zero size symbol, ensure that its start anchor precedes
535 // its end anchor. For two symbols with anchors at the same offset, their
536 // order does not matter.
537 for (OutputSection *osec : outputSections) {
538 if (!(osec->flags & SHF_EXECINSTR))
539 continue;
540 for (InputSection *sec : getInputSections(*osec, storage)) {
541 llvm::sort(sec->relaxAux->anchors, [](auto &a, auto &b) {
542 return std::make_pair(a.offset, a.end) <
543 std::make_pair(b.offset, b.end);
544 });
545 }
546 }
547 }
548
549 // Relax R_RISCV_CALL/R_RISCV_CALL_PLT auipc+jalr to c.j, c.jal, or jal.
relaxCall(const InputSection & sec,size_t i,uint64_t loc,Relocation & r,uint32_t & remove)550 static void relaxCall(const InputSection &sec, size_t i, uint64_t loc,
551 Relocation &r, uint32_t &remove) {
552 const bool rvc = config->eflags & EF_RISCV_RVC;
553 const Symbol &sym = *r.sym;
554 const uint64_t insnPair = read64le(sec.rawData.data() + r.offset);
555 const uint32_t rd = extractBits(insnPair, 32 + 11, 32 + 7);
556 const uint64_t dest =
557 (r.expr == R_PLT_PC ? sym.getPltVA() : sym.getVA()) + r.addend;
558 const int64_t displace = dest - loc;
559
560 if (rvc && isInt<12>(displace) && rd == 0) {
561 sec.relaxAux->relocTypes[i] = R_RISCV_RVC_JUMP;
562 sec.relaxAux->writes.push_back(0xa001); // c.j
563 remove = 6;
564 } else if (rvc && isInt<12>(displace) && rd == X_RA &&
565 !config->is64) { // RV32C only
566 sec.relaxAux->relocTypes[i] = R_RISCV_RVC_JUMP;
567 sec.relaxAux->writes.push_back(0x2001); // c.jal
568 remove = 6;
569 } else if (isInt<21>(displace)) {
570 sec.relaxAux->relocTypes[i] = R_RISCV_JAL;
571 sec.relaxAux->writes.push_back(0x6f | rd << 7); // jal
572 remove = 4;
573 }
574 }
575
576 // Relax local-exec TLS when hi20 is zero.
relaxTlsLe(const InputSection & sec,size_t i,uint64_t loc,Relocation & r,uint32_t & remove)577 static void relaxTlsLe(const InputSection &sec, size_t i, uint64_t loc,
578 Relocation &r, uint32_t &remove) {
579 uint64_t val = r.sym->getVA(r.addend);
580 if (hi20(val) != 0)
581 return;
582 uint32_t insn = read32le(sec.rawData.data() + r.offset);
583 switch (r.type) {
584 case R_RISCV_TPREL_HI20:
585 case R_RISCV_TPREL_ADD:
586 // Remove lui rd, %tprel_hi(x) and add rd, rd, tp, %tprel_add(x).
587 sec.relaxAux->relocTypes[i] = R_RISCV_RELAX;
588 remove = 4;
589 break;
590 case R_RISCV_TPREL_LO12_I:
591 // addi rd, rd, %tprel_lo(x) => addi rd, tp, st_value(x)
592 sec.relaxAux->relocTypes[i] = R_RISCV_32;
593 insn = (insn & ~(31 << 15)) | (X_TP << 15);
594 sec.relaxAux->writes.push_back(setLO12_I(insn, val));
595 break;
596 case R_RISCV_TPREL_LO12_S:
597 // sw rs, %tprel_lo(x)(rd) => sw rs, st_value(x)(rd)
598 sec.relaxAux->relocTypes[i] = R_RISCV_32;
599 insn = (insn & ~(31 << 15)) | (X_TP << 15);
600 sec.relaxAux->writes.push_back(setLO12_S(insn, val));
601 break;
602 }
603 }
604
relax(InputSection & sec)605 static bool relax(InputSection &sec) {
606 const uint64_t secAddr = sec.getVA();
607 auto &aux = *sec.relaxAux;
608 bool changed = false;
609
610 // Get st_value delta for symbols relative to this section from the previous
611 // iteration.
612 DenseMap<const Defined *, uint64_t> valueDelta;
613 ArrayRef<SymbolAnchor> sa = makeArrayRef(aux.anchors);
614 uint32_t delta = 0;
615 for (auto it : llvm::enumerate(sec.relocations)) {
616 for (; sa.size() && sa[0].offset <= it.value().offset; sa = sa.slice(1))
617 if (!sa[0].end)
618 valueDelta[sa[0].d] = delta;
619 delta = aux.relocDeltas[it.index()];
620 }
621 for (const SymbolAnchor &sa : sa)
622 if (!sa.end)
623 valueDelta[sa.d] = delta;
624 sa = makeArrayRef(aux.anchors);
625 delta = 0;
626
627 std::fill_n(aux.relocTypes.get(), sec.relocations.size(), R_RISCV_NONE);
628 aux.writes.clear();
629 for (auto it : llvm::enumerate(sec.relocations)) {
630 Relocation &r = it.value();
631 const size_t i = it.index();
632 const uint64_t loc = secAddr + r.offset - delta;
633 uint32_t &cur = aux.relocDeltas[i], remove = 0;
634 switch (r.type) {
635 case R_RISCV_ALIGN: {
636 const uint64_t nextLoc = loc + r.addend;
637 const uint64_t align = PowerOf2Ceil(r.addend + 2);
638 // All bytes beyond the alignment boundary should be removed.
639 remove = nextLoc - ((loc + align - 1) & -align);
640 assert(static_cast<int32_t>(remove) >= 0 &&
641 "R_RISCV_ALIGN needs expanding the content");
642 break;
643 }
644 case R_RISCV_CALL:
645 case R_RISCV_CALL_PLT:
646 if (i + 1 != sec.relocations.size() &&
647 sec.relocations[i + 1].type == R_RISCV_RELAX)
648 relaxCall(sec, i, loc, r, remove);
649 break;
650 case R_RISCV_TPREL_HI20:
651 case R_RISCV_TPREL_ADD:
652 case R_RISCV_TPREL_LO12_I:
653 case R_RISCV_TPREL_LO12_S:
654 if (i + 1 != sec.relocations.size() &&
655 sec.relocations[i + 1].type == R_RISCV_RELAX)
656 relaxTlsLe(sec, i, loc, r, remove);
657 break;
658 }
659
660 // For all anchors whose offsets are <= r.offset, they are preceded by
661 // the previous relocation whose `relocDeltas` value equals `delta`.
662 // Decrease their st_value and update their st_size.
663 for (; sa.size() && sa[0].offset <= r.offset; sa = sa.slice(1)) {
664 if (sa[0].end)
665 sa[0].d->size = sa[0].offset - delta - sa[0].d->value;
666 else
667 sa[0].d->value -= delta - valueDelta.find(sa[0].d)->second;
668 }
669 delta += remove;
670 if (delta != cur) {
671 cur = delta;
672 changed = true;
673 }
674 }
675
676 for (const SymbolAnchor &a : sa) {
677 if (a.end)
678 a.d->size = a.offset - delta - a.d->value;
679 else
680 a.d->value -= delta - valueDelta.find(a.d)->second;
681 }
682 // Inform assignAddresses that the size has changed.
683 if (!isUInt<16>(delta))
684 fatal("section size decrease is too large");
685 sec.bytesDropped = delta;
686 return changed;
687 }
688
689 // When relaxing just R_RISCV_ALIGN, relocDeltas is usually changed only once in
690 // the absence of a linker script. For call and load/store R_RISCV_RELAX, code
691 // shrinkage may reduce displacement and make more relocations eligible for
692 // relaxation. Code shrinkage may increase displacement to a call/load/store
693 // target at a higher fixed address, invalidating an earlier relaxation. Any
694 // change in section sizes can have cascading effect and require another
695 // relaxation pass.
relaxOnce(int pass) const696 bool RISCV::relaxOnce(int pass) const {
697 llvm::TimeTraceScope timeScope("RISC-V relaxOnce");
698 if (config->relocatable)
699 return false;
700
701 if (pass == 0)
702 initSymbolAnchors();
703
704 SmallVector<InputSection *, 0> storage;
705 bool changed = false;
706 for (OutputSection *osec : outputSections) {
707 if (!(osec->flags & SHF_EXECINSTR))
708 continue;
709 for (InputSection *sec : getInputSections(*osec, storage))
710 changed |= relax(*sec);
711 }
712 return changed;
713 }
714
riscvFinalizeRelax(int passes)715 void elf::riscvFinalizeRelax(int passes) {
716 llvm::TimeTraceScope timeScope("Finalize RISC-V relaxation");
717 log("relaxation passes: " + Twine(passes));
718 SmallVector<InputSection *, 0> storage;
719 for (OutputSection *osec : outputSections) {
720 if (!(osec->flags & SHF_EXECINSTR))
721 continue;
722 for (InputSection *sec : getInputSections(*osec, storage)) {
723 RISCVRelaxAux &aux = *sec->relaxAux;
724 if (!aux.relocDeltas)
725 continue;
726
727 auto &rels = sec->relocations;
728 ArrayRef<uint8_t> old = sec->rawData;
729 size_t newSize =
730 old.size() - aux.relocDeltas[sec->relocations.size() - 1];
731 size_t writesIdx = 0;
732 uint8_t *p = context().bAlloc.Allocate<uint8_t>(newSize);
733 uint64_t offset = 0;
734 int64_t delta = 0;
735 sec->rawData = makeArrayRef(p, newSize);
736 sec->bytesDropped = 0;
737
738 // Update section content: remove NOPs for R_RISCV_ALIGN and rewrite
739 // instructions for relaxed relocations.
740 for (size_t i = 0, e = rels.size(); i != e; ++i) {
741 uint32_t remove = aux.relocDeltas[i] - delta;
742 delta = aux.relocDeltas[i];
743 if (remove == 0 && aux.relocTypes[i] == R_RISCV_NONE)
744 continue;
745
746 // Copy from last location to the current relocated location.
747 const Relocation &r = rels[i];
748 uint64_t size = r.offset - offset;
749 memcpy(p, old.data() + offset, size);
750 p += size;
751
752 // For R_RISCV_ALIGN, we will place `offset` in a location (among NOPs)
753 // to satisfy the alignment requirement. If both `remove` and r.addend
754 // are multiples of 4, it is as if we have skipped some NOPs. Otherwise
755 // we are in the middle of a 4-byte NOP, and we need to rewrite the NOP
756 // sequence.
757 int64_t skip = 0;
758 if (r.type == R_RISCV_ALIGN) {
759 if (remove % 4 || r.addend % 4) {
760 skip = r.addend - remove;
761 int64_t j = 0;
762 for (; j + 4 <= skip; j += 4)
763 write32le(p + j, 0x00000013); // nop
764 if (j != skip) {
765 assert(j + 2 == skip);
766 write16le(p + j, 0x0001); // c.nop
767 }
768 }
769 } else if (RelType newType = aux.relocTypes[i]) {
770 switch (newType) {
771 case R_RISCV_RELAX:
772 // Used by relaxTlsLe to indicate the relocation is ignored.
773 break;
774 case R_RISCV_RVC_JUMP:
775 skip = 2;
776 write16le(p, aux.writes[writesIdx++]);
777 break;
778 case R_RISCV_JAL:
779 skip = 4;
780 write32le(p, aux.writes[writesIdx++]);
781 break;
782 case R_RISCV_32:
783 // Used by relaxTlsLe to write a uint32_t then suppress the handling
784 // in relocateAlloc.
785 skip = 4;
786 write32le(p, aux.writes[writesIdx++]);
787 aux.relocTypes[i] = R_RISCV_NONE;
788 break;
789 default:
790 llvm_unreachable("unsupported type");
791 }
792 }
793
794 p += skip;
795 offset = r.offset + skip + remove;
796 }
797 memcpy(p, old.data() + offset, old.size() - offset);
798
799 // Subtract the previous relocDeltas value from the relocation offset.
800 // For a pair of R_RISCV_CALL/R_RISCV_RELAX with the same offset, decrease
801 // their r_offset by the same delta.
802 delta = 0;
803 for (size_t i = 0, e = rels.size(); i != e;) {
804 uint64_t cur = rels[i].offset;
805 do {
806 rels[i].offset -= delta;
807 if (aux.relocTypes[i] != R_RISCV_NONE)
808 rels[i].type = aux.relocTypes[i];
809 } while (++i != e && rels[i].offset == cur);
810 delta = aux.relocDeltas[i - 1];
811 }
812 }
813 }
814 }
815
getRISCVTargetInfo()816 TargetInfo *elf::getRISCVTargetInfo() {
817 static RISCV target;
818 return ⌖
819 }
820