1 //! This module defines riscv64-specific machine instruction types.
2
3 use super::lower::isle::generated_code::{VecAMode, VecElementWidth, VecOpMasking};
4 use crate::binemit::{Addend, CodeOffset, Reloc};
5 pub use crate::ir::condcodes::IntCC;
6 use crate::ir::types::{self, F16, F32, F64, F128, I8, I8X16, I16, I32, I64, I128};
7
8 pub use crate::ir::{ExternalName, MemFlags, Type};
9 use crate::isa::{CallConv, FunctionAlignment};
10 use crate::machinst::*;
11 use crate::{CodegenError, CodegenResult, settings};
12
13 pub use crate::ir::condcodes::FloatCC;
14
15 use alloc::boxed::Box;
16 use alloc::string::{String, ToString};
17 use alloc::vec::Vec;
18 use core::fmt::Write;
19 use regalloc2::RegClass;
20 use smallvec::{SmallVec, smallvec};
21
22 pub mod regs;
23 pub use self::regs::*;
24 pub mod imms;
25 pub use self::imms::*;
26 pub mod args;
27 pub use self::args::*;
28 pub mod emit;
29 pub use self::emit::*;
30 pub mod vector;
31 pub use self::vector::*;
32 pub mod encode;
33 pub use self::encode::*;
34 pub mod unwind;
35
36 use crate::isa::riscv64::abi::Riscv64MachineDeps;
37
38 #[cfg(test)]
39 mod emit_tests;
40
41 use core::fmt::{Display, Formatter};
42
43 pub(crate) type VecU8 = Vec<u8>;
44
45 //=============================================================================
46 // Instructions (top level): definition
47
48 pub use crate::isa::riscv64::lower::isle::generated_code::{
49 AluOPRRI, AluOPRRR, AtomicOP, CSR, CsrImmOP, CsrRegOP, FClassResult, FFlagsException, FRM,
50 FpuOPRR, FpuOPRRR, FpuOPRRRR, LoadOP, MInst as Inst, StoreOP,
51 };
52 use crate::isa::riscv64::lower::isle::generated_code::{CjOp, MInst, VecAluOpRRImm5, VecAluOpRRR};
53
54 /// Additional information for `return_call[_ind]` instructions, left out of
55 /// line to lower the size of the `Inst` enum.
56 #[derive(Clone, Debug)]
57 pub struct ReturnCallInfo<T> {
58 pub dest: T,
59 pub uses: CallArgList,
60 pub new_stack_arg_size: u32,
61 }
62
63 /// A conditional branch target.
64 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
65 pub enum CondBrTarget {
66 /// An unresolved reference to a Label, as passed into
67 /// `lower_branch_group()`.
68 Label(MachLabel),
69 /// No jump; fall through to the next instruction.
70 Fallthrough,
71 }
72
73 impl CondBrTarget {
74 /// Return the target's label, if it is a label-based target.
as_label(self) -> Option<MachLabel>75 pub(crate) fn as_label(self) -> Option<MachLabel> {
76 match self {
77 CondBrTarget::Label(l) => Some(l),
78 _ => None,
79 }
80 }
81
is_fallthrouh(&self) -> bool82 pub(crate) fn is_fallthrouh(&self) -> bool {
83 self == &CondBrTarget::Fallthrough
84 }
85 }
86
87 impl Display for CondBrTarget {
fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result88 fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result {
89 match self {
90 CondBrTarget::Label(l) => write!(f, "{}", l.to_string()),
91 CondBrTarget::Fallthrough => write!(f, "0"),
92 }
93 }
94 }
95
enc_auipc(rd: Writable<Reg>, imm: Imm20) -> u3296 pub(crate) fn enc_auipc(rd: Writable<Reg>, imm: Imm20) -> u32 {
97 let x = 0b0010111 | reg_to_gpr_num(rd.to_reg()) << 7 | imm.bits() << 12;
98 x
99 }
100
enc_jalr(rd: Writable<Reg>, base: Reg, offset: Imm12) -> u32101 pub(crate) fn enc_jalr(rd: Writable<Reg>, base: Reg, offset: Imm12) -> u32 {
102 let x = 0b1100111
103 | reg_to_gpr_num(rd.to_reg()) << 7
104 | 0b000 << 12
105 | reg_to_gpr_num(base) << 15
106 | offset.bits() << 20;
107 x
108 }
109
110 /// rd and src must have the same length.
gen_moves(rd: &[Writable<Reg>], src: &[Reg]) -> SmallInstVec<Inst>111 pub(crate) fn gen_moves(rd: &[Writable<Reg>], src: &[Reg]) -> SmallInstVec<Inst> {
112 assert!(rd.len() == src.len());
113 assert!(rd.len() > 0);
114 let mut insts = SmallInstVec::new();
115 for (dst, src) in rd.iter().zip(src.iter()) {
116 let ty = Inst::canonical_type_for_rc(dst.to_reg().class());
117 insts.push(Inst::gen_move(*dst, *src, ty));
118 }
119 insts
120 }
121
122 impl Inst {
123 /// RISC-V can have multiple instruction sizes. 2 bytes for compressed
124 /// instructions, 4 for regular instructions, 6 and 8 byte instructions
125 /// are also being considered.
126 const UNCOMPRESSED_INSTRUCTION_SIZE: i32 = 4;
127
128 #[inline]
load_imm12(rd: Writable<Reg>, imm: Imm12) -> Inst129 pub(crate) fn load_imm12(rd: Writable<Reg>, imm: Imm12) -> Inst {
130 Inst::AluRRImm12 {
131 alu_op: AluOPRRI::Addi,
132 rd,
133 rs: zero_reg(),
134 imm12: imm,
135 }
136 }
137
138 /// Immediates can be loaded using lui and addi instructions.
load_const_imm(rd: Writable<Reg>, value: u64) -> Option<SmallInstVec<Inst>>139 fn load_const_imm(rd: Writable<Reg>, value: u64) -> Option<SmallInstVec<Inst>> {
140 Inst::generate_imm(value).map(|(imm20, imm12)| {
141 let mut insts = SmallVec::new();
142
143 let imm20_is_zero = imm20.as_i32() == 0;
144 let imm12_is_zero = imm12.as_i16() == 0;
145
146 let rs = if !imm20_is_zero {
147 insts.push(Inst::Lui { rd, imm: imm20 });
148 rd.to_reg()
149 } else {
150 zero_reg()
151 };
152
153 // We also need to emit the addi if the value is 0, otherwise we just
154 // won't produce any instructions.
155 if !imm12_is_zero || (imm20_is_zero && imm12_is_zero) {
156 insts.push(Inst::AluRRImm12 {
157 alu_op: AluOPRRI::Addi,
158 rd,
159 rs,
160 imm12,
161 })
162 }
163
164 insts
165 })
166 }
167
load_constant_u32(rd: Writable<Reg>, value: u64) -> SmallInstVec<Inst>168 pub(crate) fn load_constant_u32(rd: Writable<Reg>, value: u64) -> SmallInstVec<Inst> {
169 let insts = Inst::load_const_imm(rd, value);
170 insts.unwrap_or_else(|| {
171 smallvec![Inst::LoadInlineConst {
172 rd,
173 ty: I32,
174 imm: value
175 }]
176 })
177 }
178
load_constant_u64(rd: Writable<Reg>, value: u64) -> SmallInstVec<Inst>179 pub fn load_constant_u64(rd: Writable<Reg>, value: u64) -> SmallInstVec<Inst> {
180 let insts = Inst::load_const_imm(rd, value);
181 insts.unwrap_or_else(|| {
182 smallvec![Inst::LoadInlineConst {
183 rd,
184 ty: I64,
185 imm: value
186 }]
187 })
188 }
189
construct_auipc_and_jalr( link: Option<Writable<Reg>>, tmp: Writable<Reg>, offset: i64, ) -> [Inst; 2]190 pub(crate) fn construct_auipc_and_jalr(
191 link: Option<Writable<Reg>>,
192 tmp: Writable<Reg>,
193 offset: i64,
194 ) -> [Inst; 2] {
195 Inst::generate_imm(offset as u64)
196 .map(|(imm20, imm12)| {
197 let a = Inst::Auipc {
198 rd: tmp,
199 imm: imm20,
200 };
201 let b = Inst::Jalr {
202 rd: link.unwrap_or(writable_zero_reg()),
203 base: tmp.to_reg(),
204 offset: imm12,
205 };
206 [a, b]
207 })
208 .expect("code range is too big.")
209 }
210
211 /// Generic constructor for a load (zero-extending where appropriate).
gen_load(into_reg: Writable<Reg>, mem: AMode, ty: Type, flags: MemFlags) -> Inst212 pub fn gen_load(into_reg: Writable<Reg>, mem: AMode, ty: Type, flags: MemFlags) -> Inst {
213 if ty.is_vector() {
214 Inst::VecLoad {
215 eew: VecElementWidth::from_type(ty),
216 to: into_reg,
217 from: VecAMode::UnitStride { base: mem },
218 flags,
219 mask: VecOpMasking::Disabled,
220 vstate: VState::from_type(ty),
221 }
222 } else {
223 Inst::Load {
224 rd: into_reg,
225 op: LoadOP::from_type(ty),
226 from: mem,
227 flags,
228 }
229 }
230 }
231
232 /// Generic constructor for a store.
gen_store(mem: AMode, from_reg: Reg, ty: Type, flags: MemFlags) -> Inst233 pub fn gen_store(mem: AMode, from_reg: Reg, ty: Type, flags: MemFlags) -> Inst {
234 if ty.is_vector() {
235 Inst::VecStore {
236 eew: VecElementWidth::from_type(ty),
237 to: VecAMode::UnitStride { base: mem },
238 from: from_reg,
239 flags,
240 mask: VecOpMasking::Disabled,
241 vstate: VState::from_type(ty),
242 }
243 } else {
244 Inst::Store {
245 src: from_reg,
246 op: StoreOP::from_type(ty),
247 to: mem,
248 flags,
249 }
250 }
251 }
252 }
253
254 //=============================================================================
255
vec_mask_operands(mask: &mut VecOpMasking, collector: &mut impl OperandVisitor)256 fn vec_mask_operands(mask: &mut VecOpMasking, collector: &mut impl OperandVisitor) {
257 match mask {
258 VecOpMasking::Enabled { reg } => {
259 collector.reg_fixed_use(reg, pv_reg(0).into());
260 }
261 VecOpMasking::Disabled => {}
262 }
263 }
vec_mask_late_operands(mask: &mut VecOpMasking, collector: &mut impl OperandVisitor)264 fn vec_mask_late_operands(mask: &mut VecOpMasking, collector: &mut impl OperandVisitor) {
265 match mask {
266 VecOpMasking::Enabled { reg } => {
267 collector.reg_fixed_late_use(reg, pv_reg(0).into());
268 }
269 VecOpMasking::Disabled => {}
270 }
271 }
272
riscv64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor)273 fn riscv64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
274 match inst {
275 Inst::Nop0 | Inst::Nop4 => {}
276 Inst::BrTable {
277 index, tmp1, tmp2, ..
278 } => {
279 collector.reg_use(index);
280 collector.reg_early_def(tmp1);
281 collector.reg_early_def(tmp2);
282 }
283 Inst::Auipc { rd, .. } => collector.reg_def(rd),
284 Inst::Lui { rd, .. } => collector.reg_def(rd),
285 Inst::Fli { rd, .. } => collector.reg_def(rd),
286 Inst::LoadInlineConst { rd, .. } => collector.reg_def(rd),
287 Inst::AluRRR { rd, rs1, rs2, .. } => {
288 collector.reg_use(rs1);
289 collector.reg_use(rs2);
290 collector.reg_def(rd);
291 }
292 Inst::FpuRRR { rd, rs1, rs2, .. } => {
293 collector.reg_use(rs1);
294 collector.reg_use(rs2);
295 collector.reg_def(rd);
296 }
297 Inst::AluRRImm12 { rd, rs, .. } => {
298 collector.reg_use(rs);
299 collector.reg_def(rd);
300 }
301 Inst::CsrReg { rd, rs, .. } => {
302 collector.reg_use(rs);
303 collector.reg_def(rd);
304 }
305 Inst::CsrImm { rd, .. } => {
306 collector.reg_def(rd);
307 }
308 Inst::Load { rd, from, .. } => {
309 from.get_operands(collector);
310 collector.reg_def(rd);
311 }
312 Inst::Store { to, src, .. } => {
313 to.get_operands(collector);
314 collector.reg_use(src);
315 }
316
317 Inst::Args { args } => {
318 for ArgPair { vreg, preg } in args {
319 collector.reg_fixed_def(vreg, *preg);
320 }
321 }
322 Inst::Rets { rets } => {
323 for RetPair { vreg, preg } in rets {
324 collector.reg_fixed_use(vreg, *preg);
325 }
326 }
327 Inst::Ret { .. } => {}
328
329 Inst::Extend { rd, rn, .. } => {
330 collector.reg_use(rn);
331 collector.reg_def(rd);
332 }
333 Inst::Call { info, .. } => {
334 let CallInfo { uses, defs, .. } = &mut **info;
335 for CallArgPair { vreg, preg } in uses {
336 collector.reg_fixed_use(vreg, *preg);
337 }
338 for CallRetPair { vreg, location } in defs {
339 match location {
340 RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg),
341 RetLocation::Stack(..) => collector.any_def(vreg),
342 }
343 }
344 collector.reg_clobbers(info.clobbers);
345 if let Some(try_call_info) = &mut info.try_call_info {
346 try_call_info.collect_operands(collector);
347 }
348 }
349 Inst::CallInd { info } => {
350 let CallInfo {
351 dest, uses, defs, ..
352 } = &mut **info;
353 collector.reg_use(dest);
354 for CallArgPair { vreg, preg } in uses {
355 collector.reg_fixed_use(vreg, *preg);
356 }
357 for CallRetPair { vreg, location } in defs {
358 match location {
359 RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg),
360 RetLocation::Stack(..) => collector.any_def(vreg),
361 }
362 }
363 collector.reg_clobbers(info.clobbers);
364 if let Some(try_call_info) = &mut info.try_call_info {
365 try_call_info.collect_operands(collector);
366 }
367 }
368 Inst::ReturnCall { info } => {
369 for CallArgPair { vreg, preg } in &mut info.uses {
370 collector.reg_fixed_use(vreg, *preg);
371 }
372 }
373 Inst::ReturnCallInd { info } => {
374 // TODO(https://github.com/bytecodealliance/regalloc2/issues/145):
375 // This shouldn't be a fixed register constraint.
376 collector.reg_fixed_use(&mut info.dest, x_reg(5));
377
378 for CallArgPair { vreg, preg } in &mut info.uses {
379 collector.reg_fixed_use(vreg, *preg);
380 }
381 }
382 Inst::Jal { .. } => {
383 // JAL technically has a rd register, but we currently always
384 // hardcode it to x0.
385 }
386 Inst::CondBr {
387 kind: IntegerCompare { rs1, rs2, .. },
388 ..
389 } => {
390 collector.reg_use(rs1);
391 collector.reg_use(rs2);
392 }
393 Inst::LoadExtNameGot { rd, .. }
394 | Inst::LoadExtNameNear { rd, .. }
395 | Inst::LoadExtNameFar { rd, .. } => {
396 collector.reg_def(rd);
397 }
398 Inst::ElfTlsGetAddr { rd, .. } => {
399 // x10 is a0 which is both the first argument and the first return value.
400 collector.reg_fixed_def(rd, a0());
401 let mut clobbers =
402 Riscv64MachineDeps::get_regs_clobbered_by_call(CallConv::SystemV, false);
403 clobbers.remove(px_reg(10));
404 collector.reg_clobbers(clobbers);
405 }
406 Inst::LoadAddr { rd, mem } => {
407 mem.get_operands(collector);
408 collector.reg_early_def(rd);
409 }
410
411 Inst::Mov { rd, rm, .. } => {
412 collector.reg_use(rm);
413 collector.reg_def(rd);
414 }
415 Inst::MovFromPReg { rd, rm } => {
416 debug_assert!([px_reg(2), px_reg(8)].contains(rm));
417 collector.reg_def(rd);
418 }
419 Inst::Fence { .. } => {}
420 Inst::EBreak => {}
421 Inst::Udf { .. } => {}
422 Inst::FpuRR { rd, rs, .. } => {
423 collector.reg_use(rs);
424 collector.reg_def(rd);
425 }
426 Inst::FpuRRRR {
427 rd, rs1, rs2, rs3, ..
428 } => {
429 collector.reg_use(rs1);
430 collector.reg_use(rs2);
431 collector.reg_use(rs3);
432 collector.reg_def(rd);
433 }
434
435 Inst::Jalr { rd, base, .. } => {
436 collector.reg_use(base);
437 collector.reg_def(rd);
438 }
439 Inst::Atomic { rd, addr, src, .. } => {
440 collector.reg_use(addr);
441 collector.reg_use(src);
442 collector.reg_def(rd);
443 }
444 Inst::Select {
445 dst,
446 condition: IntegerCompare { rs1, rs2, .. },
447 x,
448 y,
449 ..
450 } => {
451 // Mark the condition registers as late use so that they don't overlap with the destination
452 // register. We may potentially write to the destination register before evaluating the
453 // condition.
454 collector.reg_late_use(rs1);
455 collector.reg_late_use(rs2);
456
457 for reg in x.regs_mut() {
458 collector.reg_use(reg);
459 }
460 for reg in y.regs_mut() {
461 collector.reg_use(reg);
462 }
463
464 // If there's more than one destination register then use
465 // `reg_early_def` to prevent destination registers from overlapping
466 // with any operands. This ensures that the lowering doesn't have to
467 // deal with a situation such as when the input registers need to be
468 // swapped when moved to the destination.
469 //
470 // When there's only one destination register though don't use an
471 // early def because once the register is written no other inputs
472 // are read so it's ok for the destination to overlap the sources.
473 // The condition registers are already marked as late use so they
474 // won't overlap with the destination.
475 match dst.regs_mut() {
476 [reg] => collector.reg_def(reg),
477 regs => {
478 for d in regs {
479 collector.reg_early_def(d);
480 }
481 }
482 }
483 }
484 Inst::AtomicCas {
485 offset,
486 t0,
487 dst,
488 e,
489 addr,
490 v,
491 ..
492 } => {
493 collector.reg_use(offset);
494 collector.reg_use(e);
495 collector.reg_use(addr);
496 collector.reg_use(v);
497 collector.reg_early_def(t0);
498 collector.reg_early_def(dst);
499 }
500
501 Inst::RawData { .. } => {}
502 Inst::AtomicStore { src, p, .. } => {
503 collector.reg_use(src);
504 collector.reg_use(p);
505 }
506 Inst::AtomicLoad { rd, p, .. } => {
507 collector.reg_use(p);
508 collector.reg_def(rd);
509 }
510 Inst::AtomicRmwLoop {
511 offset,
512 dst,
513 p,
514 x,
515 t0,
516 ..
517 } => {
518 collector.reg_use(offset);
519 collector.reg_use(p);
520 collector.reg_use(x);
521 collector.reg_early_def(t0);
522 collector.reg_early_def(dst);
523 }
524 Inst::TrapIf { rs1, rs2, .. } => {
525 collector.reg_use(rs1);
526 collector.reg_use(rs2);
527 }
528 Inst::Unwind { .. } => {}
529 Inst::DummyUse { reg } => {
530 collector.reg_use(reg);
531 }
532 Inst::Popcnt {
533 sum, step, rs, tmp, ..
534 } => {
535 collector.reg_use(rs);
536 collector.reg_early_def(tmp);
537 collector.reg_early_def(step);
538 collector.reg_early_def(sum);
539 }
540 Inst::Cltz {
541 sum, step, tmp, rs, ..
542 } => {
543 collector.reg_use(rs);
544 collector.reg_early_def(tmp);
545 collector.reg_early_def(step);
546 collector.reg_early_def(sum);
547 }
548 Inst::Brev8 {
549 rs,
550 rd,
551 step,
552 tmp,
553 tmp2,
554 ..
555 } => {
556 collector.reg_use(rs);
557 collector.reg_early_def(step);
558 collector.reg_early_def(tmp);
559 collector.reg_early_def(tmp2);
560 collector.reg_early_def(rd);
561 }
562 Inst::StackProbeLoop { .. } => {
563 // StackProbeLoop has a tmp register and StackProbeLoop used at gen_prologue.
564 // t3 will do the job. (t3 is caller-save register and not used directly by compiler like writable_spilltmp_reg)
565 // gen_prologue is called at emit stage.
566 // no need let reg alloc know.
567 }
568 Inst::VecAluRRRR {
569 op,
570 vd,
571 vd_src,
572 vs1,
573 vs2,
574 mask,
575 ..
576 } => {
577 debug_assert_eq!(vd_src.class(), RegClass::Vector);
578 debug_assert_eq!(vd.to_reg().class(), RegClass::Vector);
579 debug_assert_eq!(vs2.class(), RegClass::Vector);
580 debug_assert_eq!(vs1.class(), op.vs1_regclass());
581
582 collector.reg_late_use(vs1);
583 collector.reg_late_use(vs2);
584 collector.reg_use(vd_src);
585 collector.reg_reuse_def(vd, 2); // `vd` == `vd_src`.
586 vec_mask_late_operands(mask, collector);
587 }
588 Inst::VecAluRRRImm5 {
589 op,
590 vd,
591 vd_src,
592 vs2,
593 mask,
594 ..
595 } => {
596 debug_assert_eq!(vd_src.class(), RegClass::Vector);
597 debug_assert_eq!(vd.to_reg().class(), RegClass::Vector);
598 debug_assert_eq!(vs2.class(), RegClass::Vector);
599
600 // If the operation forbids source/destination overlap we need to
601 // ensure that the source and destination registers are different.
602 if op.forbids_overlaps(mask) {
603 collector.reg_late_use(vs2);
604 collector.reg_use(vd_src);
605 collector.reg_reuse_def(vd, 1); // `vd` == `vd_src`.
606 vec_mask_late_operands(mask, collector);
607 } else {
608 collector.reg_use(vs2);
609 collector.reg_use(vd_src);
610 collector.reg_reuse_def(vd, 1); // `vd` == `vd_src`.
611 vec_mask_operands(mask, collector);
612 }
613 }
614 Inst::VecAluRRR {
615 op,
616 vd,
617 vs1,
618 vs2,
619 mask,
620 ..
621 } => {
622 debug_assert_eq!(vd.to_reg().class(), RegClass::Vector);
623 debug_assert_eq!(vs2.class(), RegClass::Vector);
624 debug_assert_eq!(vs1.class(), op.vs1_regclass());
625
626 collector.reg_use(vs1);
627 collector.reg_use(vs2);
628
629 // If the operation forbids source/destination overlap, then we must
630 // register it as an early_def. This encodes the constraint that
631 // these must not overlap.
632 if op.forbids_overlaps(mask) {
633 collector.reg_early_def(vd);
634 } else {
635 collector.reg_def(vd);
636 }
637
638 vec_mask_operands(mask, collector);
639 }
640 Inst::VecAluRRImm5 {
641 op, vd, vs2, mask, ..
642 } => {
643 debug_assert_eq!(vd.to_reg().class(), RegClass::Vector);
644 debug_assert_eq!(vs2.class(), RegClass::Vector);
645
646 collector.reg_use(vs2);
647
648 // If the operation forbids source/destination overlap, then we must
649 // register it as an early_def. This encodes the constraint that
650 // these must not overlap.
651 if op.forbids_overlaps(mask) {
652 collector.reg_early_def(vd);
653 } else {
654 collector.reg_def(vd);
655 }
656
657 vec_mask_operands(mask, collector);
658 }
659 Inst::VecAluRR {
660 op, vd, vs, mask, ..
661 } => {
662 debug_assert_eq!(vd.to_reg().class(), op.dst_regclass());
663 debug_assert_eq!(vs.class(), op.src_regclass());
664
665 collector.reg_use(vs);
666
667 // If the operation forbids source/destination overlap, then we must
668 // register it as an early_def. This encodes the constraint that
669 // these must not overlap.
670 if op.forbids_overlaps(mask) {
671 collector.reg_early_def(vd);
672 } else {
673 collector.reg_def(vd);
674 }
675
676 vec_mask_operands(mask, collector);
677 }
678 Inst::VecAluRImm5 { op, vd, mask, .. } => {
679 debug_assert_eq!(vd.to_reg().class(), RegClass::Vector);
680 debug_assert!(!op.forbids_overlaps(mask));
681
682 collector.reg_def(vd);
683 vec_mask_operands(mask, collector);
684 }
685 Inst::VecSetState { rd, .. } => {
686 collector.reg_def(rd);
687 }
688 Inst::VecLoad { to, from, mask, .. } => {
689 from.get_operands(collector);
690 collector.reg_def(to);
691 vec_mask_operands(mask, collector);
692 }
693 Inst::VecStore { to, from, mask, .. } => {
694 to.get_operands(collector);
695 collector.reg_use(from);
696 vec_mask_operands(mask, collector);
697 }
698 Inst::EmitIsland { .. } => {}
699 Inst::LabelAddress { dst, .. } => {
700 collector.reg_def(dst);
701 }
702 Inst::SequencePoint { .. } => {}
703 }
704 }
705
706 impl MachInst for Inst {
707 type LabelUse = LabelUse;
708 type ABIMachineSpec = Riscv64MachineDeps;
709
710 // https://github.com/riscv/riscv-isa-manual/issues/850
711 // all zero will cause invalid opcode.
712 const TRAP_OPCODE: &'static [u8] = &[0; 4];
713
gen_dummy_use(reg: Reg) -> Self714 fn gen_dummy_use(reg: Reg) -> Self {
715 Inst::DummyUse { reg }
716 }
717
canonical_type_for_rc(rc: RegClass) -> Type718 fn canonical_type_for_rc(rc: RegClass) -> Type {
719 match rc {
720 regalloc2::RegClass::Int => I64,
721 regalloc2::RegClass::Float => F64,
722 regalloc2::RegClass::Vector => I8X16,
723 }
724 }
725
is_safepoint(&self) -> bool726 fn is_safepoint(&self) -> bool {
727 match self {
728 Inst::Call { .. } | Inst::CallInd { .. } => true,
729 _ => false,
730 }
731 }
732
get_operands(&mut self, collector: &mut impl OperandVisitor)733 fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
734 riscv64_get_operands(self, collector);
735 }
736
is_move(&self) -> Option<(Writable<Reg>, Reg)>737 fn is_move(&self) -> Option<(Writable<Reg>, Reg)> {
738 match self {
739 Inst::Mov { rd, rm, .. } => Some((*rd, *rm)),
740 _ => None,
741 }
742 }
743
is_included_in_clobbers(&self) -> bool744 fn is_included_in_clobbers(&self) -> bool {
745 match self {
746 &Inst::Args { .. } => false,
747 _ => true,
748 }
749 }
750
is_trap(&self) -> bool751 fn is_trap(&self) -> bool {
752 match self {
753 Self::Udf { .. } => true,
754 _ => false,
755 }
756 }
757
is_args(&self) -> bool758 fn is_args(&self) -> bool {
759 match self {
760 Self::Args { .. } => true,
761 _ => false,
762 }
763 }
764
call_type(&self) -> CallType765 fn call_type(&self) -> CallType {
766 match self {
767 Inst::Call { .. } | Inst::CallInd { .. } | Inst::ElfTlsGetAddr { .. } => {
768 CallType::Regular
769 }
770
771 Inst::ReturnCall { .. } | Inst::ReturnCallInd { .. } => CallType::TailCall,
772
773 _ => CallType::None,
774 }
775 }
776
is_term(&self) -> MachTerminator777 fn is_term(&self) -> MachTerminator {
778 match self {
779 &Inst::Jal { .. } => MachTerminator::Branch,
780 &Inst::CondBr { .. } => MachTerminator::Branch,
781 &Inst::Jalr { .. } => MachTerminator::Branch,
782 &Inst::Rets { .. } => MachTerminator::Ret,
783 &Inst::BrTable { .. } => MachTerminator::Branch,
784 &Inst::ReturnCall { .. } | &Inst::ReturnCallInd { .. } => MachTerminator::RetCall,
785 &Inst::Call { ref info } if info.try_call_info.is_some() => MachTerminator::Branch,
786 &Inst::CallInd { ref info } if info.try_call_info.is_some() => MachTerminator::Branch,
787 _ => MachTerminator::None,
788 }
789 }
790
is_mem_access(&self) -> bool791 fn is_mem_access(&self) -> bool {
792 panic!("TODO FILL ME OUT")
793 }
794
gen_move(to_reg: Writable<Reg>, from_reg: Reg, ty: Type) -> Inst795 fn gen_move(to_reg: Writable<Reg>, from_reg: Reg, ty: Type) -> Inst {
796 let x = Inst::Mov {
797 rd: to_reg,
798 rm: from_reg,
799 ty,
800 };
801 x
802 }
803
gen_nop(preferred_size: usize) -> Inst804 fn gen_nop(preferred_size: usize) -> Inst {
805 if preferred_size == 0 {
806 return Inst::Nop0;
807 }
808 // We can't give a NOP (or any insn) < 4 bytes.
809 assert!(preferred_size >= 4);
810 Inst::Nop4
811 }
812
gen_nop_units() -> Vec<Vec<u8>>813 fn gen_nop_units() -> Vec<Vec<u8>> {
814 vec![vec![0x13, 0x00, 0x00, 0x00]]
815 }
816
rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])>817 fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {
818 match ty {
819 I8 => Ok((&[RegClass::Int], &[I8])),
820 I16 => Ok((&[RegClass::Int], &[I16])),
821 I32 => Ok((&[RegClass::Int], &[I32])),
822 I64 => Ok((&[RegClass::Int], &[I64])),
823 F16 => Ok((&[RegClass::Float], &[F16])),
824 F32 => Ok((&[RegClass::Float], &[F32])),
825 F64 => Ok((&[RegClass::Float], &[F64])),
826 // FIXME(#8312): Add support for Q extension
827 F128 | I128 => Ok((&[RegClass::Int, RegClass::Int], &[I64, I64])),
828 _ if ty.is_vector() => {
829 debug_assert!(ty.bits() <= 512);
830
831 // Here we only need to return a SIMD type with the same size as `ty`.
832 // We use these types for spills and reloads, so prefer types with lanes <= 31
833 // since that fits in the immediate field of `vsetivli`.
834 const SIMD_TYPES: [[Type; 1]; 6] = [
835 [types::I8X2],
836 [types::I8X4],
837 [types::I8X8],
838 [types::I8X16],
839 [types::I16X16],
840 [types::I32X16],
841 ];
842 let idx = (ty.bytes().ilog2() - 1) as usize;
843 let ty = &SIMD_TYPES[idx][..];
844
845 Ok((&[RegClass::Vector], ty))
846 }
847 _ => Err(CodegenError::Unsupported(format!(
848 "Unexpected SSA-value type: {ty}"
849 ))),
850 }
851 }
852
gen_jump(target: MachLabel) -> Inst853 fn gen_jump(target: MachLabel) -> Inst {
854 Inst::Jal { label: target }
855 }
856
worst_case_size() -> CodeOffset857 fn worst_case_size() -> CodeOffset {
858 // Our worst case size is determined by the riscv64_worst_case_instruction_size test
859 84
860 }
861
ref_type_regclass(_settings: &settings::Flags) -> RegClass862 fn ref_type_regclass(_settings: &settings::Flags) -> RegClass {
863 RegClass::Int
864 }
865
function_alignment() -> FunctionAlignment866 fn function_alignment() -> FunctionAlignment {
867 FunctionAlignment {
868 minimum: 2,
869 preferred: 4,
870 }
871 }
872 }
873
874 //=============================================================================
875 // Pretty-printing of instructions.
reg_name(reg: Reg) -> String876 pub fn reg_name(reg: Reg) -> String {
877 match reg.to_real_reg() {
878 Some(real) => match real.class() {
879 RegClass::Int => match real.hw_enc() {
880 0 => "zero".into(),
881 1 => "ra".into(),
882 2 => "sp".into(),
883 3 => "gp".into(),
884 4 => "tp".into(),
885 5..=7 => format!("t{}", real.hw_enc() - 5),
886 8 => "fp".into(),
887 9 => "s1".into(),
888 10..=17 => format!("a{}", real.hw_enc() - 10),
889 18..=27 => format!("s{}", real.hw_enc() - 16),
890 28..=31 => format!("t{}", real.hw_enc() - 25),
891 _ => unreachable!(),
892 },
893 RegClass::Float => match real.hw_enc() {
894 0..=7 => format!("ft{}", real.hw_enc() - 0),
895 8..=9 => format!("fs{}", real.hw_enc() - 8),
896 10..=17 => format!("fa{}", real.hw_enc() - 10),
897 18..=27 => format!("fs{}", real.hw_enc() - 16),
898 28..=31 => format!("ft{}", real.hw_enc() - 20),
899 _ => unreachable!(),
900 },
901 RegClass::Vector => format!("v{}", real.hw_enc()),
902 },
903 None => {
904 format!("{reg:?}")
905 }
906 }
907 }
908
pretty_print_try_call(info: &TryCallInfo) -> String909 fn pretty_print_try_call(info: &TryCallInfo) -> String {
910 format!(
911 "; j {:?}; catch [{}]",
912 info.continuation,
913 info.pretty_print_dests()
914 )
915 }
916
917 impl Inst {
print_with_state(&self, _state: &mut EmitState) -> String918 fn print_with_state(&self, _state: &mut EmitState) -> String {
919 let format_reg = |reg: Reg| -> String { reg_name(reg) };
920
921 let format_vec_amode = |amode: &VecAMode| -> String {
922 match amode {
923 VecAMode::UnitStride { base } => base.to_string(),
924 }
925 };
926
927 let format_mask = |mask: &VecOpMasking| -> String {
928 match mask {
929 VecOpMasking::Enabled { reg } => format!(",{}.t", format_reg(*reg)),
930 VecOpMasking::Disabled => format!(""),
931 }
932 };
933
934 let format_regs = |regs: &[Reg]| -> String {
935 let mut x = if regs.len() > 1 {
936 String::from("[")
937 } else {
938 String::default()
939 };
940 regs.iter().for_each(|i| {
941 x.push_str(format_reg(*i).as_str());
942 if *i != *regs.last().unwrap() {
943 x.push_str(",");
944 }
945 });
946 if regs.len() > 1 {
947 x.push_str("]");
948 }
949 x
950 };
951 let format_labels = |labels: &[MachLabel]| -> String {
952 if labels.len() == 0 {
953 return String::from("[_]");
954 }
955 let mut x = String::from("[");
956 labels.iter().for_each(|l| {
957 x.push_str(
958 format!(
959 "{:?}{}",
960 l,
961 if l != labels.last().unwrap() { "," } else { "" },
962 )
963 .as_str(),
964 );
965 });
966 x.push_str("]");
967 x
968 };
969
970 fn format_frm(rounding_mode: FRM) -> String {
971 format!(",{}", rounding_mode.to_static_str())
972 }
973
974 match self {
975 &Inst::Nop0 => {
976 format!("##zero length nop")
977 }
978 &Inst::Nop4 => {
979 format!("##fixed 4-size nop")
980 }
981 &Inst::StackProbeLoop {
982 guard_size,
983 probe_count,
984 tmp,
985 } => {
986 let tmp = format_reg(tmp.to_reg());
987 format!(
988 "inline_stack_probe##guard_size={guard_size} probe_count={probe_count} tmp={tmp}"
989 )
990 }
991 &Inst::AtomicStore { src, ty, p } => {
992 let src = format_reg(src);
993 let p = format_reg(p);
994 format!("atomic_store.{ty} {src},({p})")
995 }
996 &Inst::DummyUse { reg } => {
997 let reg = format_reg(reg);
998 format!("dummy_use {reg}")
999 }
1000
1001 &Inst::AtomicLoad { rd, ty, p } => {
1002 let p = format_reg(p);
1003 let rd = format_reg(rd.to_reg());
1004 format!("atomic_load.{ty} {rd},({p})")
1005 }
1006 &Inst::AtomicRmwLoop {
1007 offset,
1008 op,
1009 dst,
1010 ty,
1011 p,
1012 x,
1013 t0,
1014 } => {
1015 let offset = format_reg(offset);
1016 let p = format_reg(p);
1017 let x = format_reg(x);
1018 let t0 = format_reg(t0.to_reg());
1019 let dst = format_reg(dst.to_reg());
1020 format!("atomic_rmw.{ty} {op} {dst},{x},({p})##t0={t0} offset={offset}")
1021 }
1022
1023 &Inst::RawData { ref data } => match data.len() {
1024 4 => {
1025 let mut bytes = [0; 4];
1026 for i in 0..bytes.len() {
1027 bytes[i] = data[i];
1028 }
1029 format!(".4byte 0x{:x}", u32::from_le_bytes(bytes))
1030 }
1031 8 => {
1032 let mut bytes = [0; 8];
1033 for i in 0..bytes.len() {
1034 bytes[i] = data[i];
1035 }
1036 format!(".8byte 0x{:x}", u64::from_le_bytes(bytes))
1037 }
1038 _ => {
1039 format!(".data {data:?}")
1040 }
1041 },
1042 &Inst::Unwind { ref inst } => {
1043 format!("unwind {inst:?}")
1044 }
1045 &Inst::Brev8 {
1046 rs,
1047 ty,
1048 step,
1049 tmp,
1050 tmp2,
1051 rd,
1052 } => {
1053 let rs = format_reg(rs);
1054 let step = format_reg(step.to_reg());
1055 let tmp = format_reg(tmp.to_reg());
1056 let tmp2 = format_reg(tmp2.to_reg());
1057 let rd = format_reg(rd.to_reg());
1058 format!("brev8 {rd},{rs}##tmp={tmp} tmp2={tmp2} step={step} ty={ty}")
1059 }
1060 &Inst::Popcnt {
1061 sum,
1062 step,
1063 rs,
1064 tmp,
1065 ty,
1066 } => {
1067 let rs = format_reg(rs);
1068 let tmp = format_reg(tmp.to_reg());
1069 let step = format_reg(step.to_reg());
1070 let sum = format_reg(sum.to_reg());
1071 format!("popcnt {sum},{rs}##ty={ty} tmp={tmp} step={step}")
1072 }
1073 &Inst::Cltz {
1074 sum,
1075 step,
1076 rs,
1077 tmp,
1078 ty,
1079 leading,
1080 } => {
1081 let rs = format_reg(rs);
1082 let tmp = format_reg(tmp.to_reg());
1083 let step = format_reg(step.to_reg());
1084 let sum = format_reg(sum.to_reg());
1085 format!(
1086 "{} {},{}##ty={} tmp={} step={}",
1087 if leading { "clz" } else { "ctz" },
1088 sum,
1089 rs,
1090 ty,
1091 tmp,
1092 step
1093 )
1094 }
1095 &Inst::AtomicCas {
1096 offset,
1097 t0,
1098 dst,
1099 e,
1100 addr,
1101 v,
1102 ty,
1103 } => {
1104 let offset = format_reg(offset);
1105 let e = format_reg(e);
1106 let addr = format_reg(addr);
1107 let v = format_reg(v);
1108 let t0 = format_reg(t0.to_reg());
1109 let dst = format_reg(dst.to_reg());
1110 format!("atomic_cas.{ty} {dst},{e},{v},({addr})##t0={t0} offset={offset}",)
1111 }
1112 &Inst::BrTable {
1113 index,
1114 tmp1,
1115 tmp2,
1116 ref targets,
1117 } => {
1118 format!(
1119 "{} {},{}##tmp1={},tmp2={}",
1120 "br_table",
1121 format_reg(index),
1122 format_labels(&targets[..]),
1123 format_reg(tmp1.to_reg()),
1124 format_reg(tmp2.to_reg()),
1125 )
1126 }
1127 &Inst::Auipc { rd, imm } => {
1128 format!("{} {},{}", "auipc", format_reg(rd.to_reg()), imm.as_i32(),)
1129 }
1130 &Inst::Jalr { rd, base, offset } => {
1131 let base = format_reg(base);
1132 let rd = format_reg(rd.to_reg());
1133 format!("{} {},{}({})", "jalr", rd, offset.as_i16(), base)
1134 }
1135 &Inst::Lui { rd, ref imm } => {
1136 format!("{} {},{}", "lui", format_reg(rd.to_reg()), imm.as_i32())
1137 }
1138 &Inst::Fli { rd, width, imm } => {
1139 let rd_s = format_reg(rd.to_reg());
1140 let imm_s = imm.format();
1141 format!("fli.{width} {rd_s},{imm_s}")
1142 }
1143 &Inst::LoadInlineConst { rd, imm, .. } => {
1144 let rd = format_reg(rd.to_reg());
1145 let mut buf = String::new();
1146 write!(&mut buf, "auipc {rd},0; ").unwrap();
1147 write!(&mut buf, "ld {rd},12({rd}); ").unwrap();
1148 write!(&mut buf, "j {}; ", Inst::UNCOMPRESSED_INSTRUCTION_SIZE + 8).unwrap();
1149 write!(&mut buf, ".8byte 0x{imm:x}").unwrap();
1150 buf
1151 }
1152 &Inst::AluRRR {
1153 alu_op,
1154 rd,
1155 rs1,
1156 rs2,
1157 } => {
1158 let rs1_s = format_reg(rs1);
1159 let rs2_s = format_reg(rs2);
1160 let rd_s = format_reg(rd.to_reg());
1161 match alu_op {
1162 AluOPRRR::Adduw if rs2 == zero_reg() => {
1163 format!("zext.w {rd_s},{rs1_s}")
1164 }
1165 _ => {
1166 format!("{} {},{},{}", alu_op.op_name(), rd_s, rs1_s, rs2_s)
1167 }
1168 }
1169 }
1170 &Inst::FpuRR {
1171 alu_op,
1172 width,
1173 frm,
1174 rd,
1175 rs,
1176 } => {
1177 let rs = format_reg(rs);
1178 let rd = format_reg(rd.to_reg());
1179 let frm = if alu_op.has_frm() {
1180 format_frm(frm)
1181 } else {
1182 String::new()
1183 };
1184 format!("{} {rd},{rs}{frm}", alu_op.op_name(width))
1185 }
1186 &Inst::FpuRRR {
1187 alu_op,
1188 width,
1189 rd,
1190 rs1,
1191 rs2,
1192 frm,
1193 } => {
1194 let rs1 = format_reg(rs1);
1195 let rs2 = format_reg(rs2);
1196 let rd = format_reg(rd.to_reg());
1197 let frm = if alu_op.has_frm() {
1198 format_frm(frm)
1199 } else {
1200 String::new()
1201 };
1202
1203 let rs1_is_rs2 = rs1 == rs2;
1204 match alu_op {
1205 FpuOPRRR::Fsgnj if rs1_is_rs2 => format!("fmv.{width} {rd},{rs1}"),
1206 FpuOPRRR::Fsgnjn if rs1_is_rs2 => format!("fneg.{width} {rd},{rs1}"),
1207 FpuOPRRR::Fsgnjx if rs1_is_rs2 => format!("fabs.{width} {rd},{rs1}"),
1208 _ => format!("{} {rd},{rs1},{rs2}{frm}", alu_op.op_name(width)),
1209 }
1210 }
1211 &Inst::FpuRRRR {
1212 alu_op,
1213 rd,
1214 rs1,
1215 rs2,
1216 rs3,
1217 frm,
1218 width,
1219 } => {
1220 let rs1 = format_reg(rs1);
1221 let rs2 = format_reg(rs2);
1222 let rs3 = format_reg(rs3);
1223 let rd = format_reg(rd.to_reg());
1224 let frm = format_frm(frm);
1225 let op_name = alu_op.op_name(width);
1226 format!("{op_name} {rd},{rs1},{rs2},{rs3}{frm}")
1227 }
1228 &Inst::AluRRImm12 {
1229 alu_op,
1230 rd,
1231 rs,
1232 ref imm12,
1233 } => {
1234 let rs_s = format_reg(rs);
1235 let rd = format_reg(rd.to_reg());
1236
1237 // Some of these special cases are better known as
1238 // their pseudo-instruction version, so prefer printing those.
1239 match (alu_op, rs, imm12) {
1240 (AluOPRRI::Addi, rs, _) if rs == zero_reg() => {
1241 return format!("li {},{}", rd, imm12.as_i16());
1242 }
1243 (AluOPRRI::Addiw, _, imm12) if imm12.as_i16() == 0 => {
1244 return format!("sext.w {rd},{rs_s}");
1245 }
1246 (AluOPRRI::Xori, _, imm12) if imm12.as_i16() == -1 => {
1247 return format!("not {rd},{rs_s}");
1248 }
1249 (AluOPRRI::SltiU, _, imm12) if imm12.as_i16() == 1 => {
1250 return format!("seqz {rd},{rs_s}");
1251 }
1252 (alu_op, _, _) if alu_op.option_funct12().is_some() => {
1253 format!("{} {},{}", alu_op.op_name(), rd, rs_s)
1254 }
1255 (alu_op, _, imm12) => {
1256 format!("{} {},{},{}", alu_op.op_name(), rd, rs_s, imm12.as_i16())
1257 }
1258 }
1259 }
1260 &Inst::CsrReg { op, rd, rs, csr } => {
1261 let rs_s = format_reg(rs);
1262 let rd_s = format_reg(rd.to_reg());
1263
1264 match (op, csr, rd) {
1265 (CsrRegOP::CsrRW, CSR::Frm, rd) if rd.to_reg() == zero_reg() => {
1266 format!("fsrm {rs_s}")
1267 }
1268 _ => {
1269 format!("{op} {rd_s},{csr},{rs_s}")
1270 }
1271 }
1272 }
1273 &Inst::CsrImm { op, rd, csr, imm } => {
1274 let rd_s = format_reg(rd.to_reg());
1275
1276 match (op, csr, rd) {
1277 (CsrImmOP::CsrRWI, CSR::Frm, rd) if rd.to_reg() != zero_reg() => {
1278 format!("fsrmi {rd_s},{imm}")
1279 }
1280 _ => {
1281 format!("{op} {rd_s},{csr},{imm}")
1282 }
1283 }
1284 }
1285 &Inst::Load {
1286 rd,
1287 op,
1288 from,
1289 flags: _flags,
1290 } => {
1291 let base = from.to_string();
1292 let rd = format_reg(rd.to_reg());
1293 format!("{} {},{}", op.op_name(), rd, base,)
1294 }
1295 &Inst::Store {
1296 to,
1297 src,
1298 op,
1299 flags: _flags,
1300 } => {
1301 let base = to.to_string();
1302 let src = format_reg(src);
1303 format!("{} {},{}", op.op_name(), src, base,)
1304 }
1305 &Inst::Args { ref args } => {
1306 let mut s = "args".to_string();
1307 for arg in args {
1308 let preg = format_reg(arg.preg);
1309 let def = format_reg(arg.vreg.to_reg());
1310 write!(&mut s, " {def}={preg}").unwrap();
1311 }
1312 s
1313 }
1314 &Inst::Rets { ref rets } => {
1315 let mut s = "rets".to_string();
1316 for ret in rets {
1317 let preg = format_reg(ret.preg);
1318 let vreg = format_reg(ret.vreg);
1319 write!(&mut s, " {vreg}={preg}").unwrap();
1320 }
1321 s
1322 }
1323 &Inst::Ret {} => "ret".to_string(),
1324
1325 &MInst::Extend {
1326 rd,
1327 rn,
1328 signed,
1329 from_bits,
1330 ..
1331 } => {
1332 let rn = format_reg(rn);
1333 let rd = format_reg(rd.to_reg());
1334 return if signed == false && from_bits == 8 {
1335 format!("andi {rd},{rn}")
1336 } else {
1337 let op = if signed { "srai" } else { "srli" };
1338 let shift_bits = (64 - from_bits) as i16;
1339 format!("slli {rd},{rn},{shift_bits}; {op} {rd},{rd},{shift_bits}")
1340 };
1341 }
1342 &MInst::Call { ref info } => {
1343 let try_call = info
1344 .try_call_info
1345 .as_ref()
1346 .map(|tci| pretty_print_try_call(tci))
1347 .unwrap_or_default();
1348 format!("call {}{try_call}", info.dest.display(None))
1349 }
1350 &MInst::CallInd { ref info } => {
1351 let rd = format_reg(info.dest);
1352 let try_call = info
1353 .try_call_info
1354 .as_ref()
1355 .map(|tci| pretty_print_try_call(tci))
1356 .unwrap_or_default();
1357 format!("callind {rd}{try_call}")
1358 }
1359 &MInst::ReturnCall { ref info } => {
1360 let mut s = format!(
1361 "return_call {:?} new_stack_arg_size:{}",
1362 info.dest, info.new_stack_arg_size
1363 );
1364 for ret in &info.uses {
1365 let preg = format_reg(ret.preg);
1366 let vreg = format_reg(ret.vreg);
1367 write!(&mut s, " {vreg}={preg}").unwrap();
1368 }
1369 s
1370 }
1371 &MInst::ReturnCallInd { ref info } => {
1372 let callee = format_reg(info.dest);
1373 let mut s = format!(
1374 "return_call_ind {callee} new_stack_arg_size:{}",
1375 info.new_stack_arg_size
1376 );
1377 for ret in &info.uses {
1378 let preg = format_reg(ret.preg);
1379 let vreg = format_reg(ret.vreg);
1380 write!(&mut s, " {vreg}={preg}").unwrap();
1381 }
1382 s
1383 }
1384 &MInst::TrapIf {
1385 rs1,
1386 rs2,
1387 cc,
1388 trap_code,
1389 } => {
1390 let rs1 = format_reg(rs1);
1391 let rs2 = format_reg(rs2);
1392 format!("trap_if {trap_code}##({rs1} {cc} {rs2})")
1393 }
1394 &MInst::Jal { label } => {
1395 format!("j {}", label.to_string())
1396 }
1397 &MInst::CondBr {
1398 taken,
1399 not_taken,
1400 kind,
1401 ..
1402 } => {
1403 let rs1 = format_reg(kind.rs1);
1404 let rs2 = format_reg(kind.rs2);
1405 if not_taken.is_fallthrouh() && taken.as_label().is_none() {
1406 format!("{} {},{},0", kind.op_name(), rs1, rs2)
1407 } else {
1408 let x = format!(
1409 "{} {},{},taken({}),not_taken({})",
1410 kind.op_name(),
1411 rs1,
1412 rs2,
1413 taken,
1414 not_taken
1415 );
1416 x
1417 }
1418 }
1419 &MInst::Atomic {
1420 op,
1421 rd,
1422 addr,
1423 src,
1424 amo,
1425 } => {
1426 let op_name = op.op_name(amo);
1427 let addr = format_reg(addr);
1428 let src = format_reg(src);
1429 let rd = format_reg(rd.to_reg());
1430 if op.is_load() {
1431 format!("{op_name} {rd},({addr})")
1432 } else {
1433 format!("{op_name} {rd},{src},({addr})")
1434 }
1435 }
1436 &MInst::LoadExtNameGot { rd, ref name } => {
1437 let rd = format_reg(rd.to_reg());
1438 format!("load_ext_name_got {rd},{}", name.display(None))
1439 }
1440 &MInst::LoadExtNameNear {
1441 rd,
1442 ref name,
1443 offset,
1444 } => {
1445 let rd = format_reg(rd.to_reg());
1446 format!("load_ext_name_near {rd},{}{offset:+}", name.display(None))
1447 }
1448 &MInst::LoadExtNameFar {
1449 rd,
1450 ref name,
1451 offset,
1452 } => {
1453 let rd = format_reg(rd.to_reg());
1454 format!("load_ext_name_far {rd},{}{offset:+}", name.display(None))
1455 }
1456 &Inst::ElfTlsGetAddr { rd, ref name } => {
1457 let rd = format_reg(rd.to_reg());
1458 format!("elf_tls_get_addr {rd},{}", name.display(None))
1459 }
1460 &MInst::LoadAddr { ref rd, ref mem } => {
1461 let rs = mem.to_string();
1462 let rd = format_reg(rd.to_reg());
1463 format!("load_addr {rd},{rs}")
1464 }
1465 &MInst::Mov { rd, rm, ty } => {
1466 let rm = format_reg(rm);
1467 let rd = format_reg(rd.to_reg());
1468
1469 let op = match ty {
1470 F16 => "fmv.h",
1471 F32 => "fmv.s",
1472 F64 => "fmv.d",
1473 ty if ty.is_vector() => "vmv1r.v",
1474 _ => "mv",
1475 };
1476
1477 format!("{op} {rd},{rm}")
1478 }
1479 &MInst::MovFromPReg { rd, rm } => {
1480 let rd = format_reg(rd.to_reg());
1481 debug_assert!([px_reg(2), px_reg(8)].contains(&rm));
1482 let rm = reg_name(Reg::from(rm));
1483 format!("mv {rd},{rm}")
1484 }
1485 &MInst::Fence { pred, succ } => {
1486 format!(
1487 "fence {},{}",
1488 Inst::fence_req_to_string(pred),
1489 Inst::fence_req_to_string(succ),
1490 )
1491 }
1492 &MInst::Select {
1493 ref dst,
1494 condition,
1495 ref x,
1496 ref y,
1497 } => {
1498 let c_rs1 = format_reg(condition.rs1);
1499 let c_rs2 = format_reg(condition.rs2);
1500 let x = format_regs(x.regs());
1501 let y = format_regs(y.regs());
1502 let dst = dst.map(|r| r.to_reg());
1503 let dst = format_regs(dst.regs());
1504 format!(
1505 "select {},{},{}##condition=({} {} {})",
1506 dst,
1507 x,
1508 y,
1509 c_rs1,
1510 condition.kind.to_static_str(),
1511 c_rs2
1512 )
1513 }
1514 &MInst::Udf { trap_code } => format!("udf##trap_code={trap_code}"),
1515 &MInst::EBreak {} => String::from("ebreak"),
1516 &Inst::VecAluRRRR {
1517 op,
1518 vd,
1519 vd_src,
1520 vs1,
1521 vs2,
1522 ref mask,
1523 ref vstate,
1524 } => {
1525 let vs1_s = format_reg(vs1);
1526 let vs2_s = format_reg(vs2);
1527 let vd_src_s = format_reg(vd_src);
1528 let vd_s = format_reg(vd.to_reg());
1529 let mask = format_mask(mask);
1530
1531 let vd_fmt = if vd_s != vd_src_s {
1532 format!("{vd_s},{vd_src_s}")
1533 } else {
1534 vd_s
1535 };
1536
1537 // Note: vs2 and vs1 here are opposite to the standard scalar ordering.
1538 // This is noted in Section 10.1 of the RISC-V Vector spec.
1539 format!("{op} {vd_fmt},{vs2_s},{vs1_s}{mask} {vstate}")
1540 }
1541 &Inst::VecAluRRRImm5 {
1542 op,
1543 vd,
1544 imm,
1545 vs2,
1546 ref mask,
1547 ref vstate,
1548 ..
1549 } => {
1550 let vs2_s = format_reg(vs2);
1551 let vd_s = format_reg(vd.to_reg());
1552 let mask = format_mask(mask);
1553
1554 // Some opcodes interpret the immediate as unsigned, lets show the
1555 // correct number here.
1556 let imm_s = if op.imm_is_unsigned() {
1557 format!("{}", imm.bits())
1558 } else {
1559 format!("{imm}")
1560 };
1561
1562 format!("{op} {vd_s},{vs2_s},{imm_s}{mask} {vstate}")
1563 }
1564 &Inst::VecAluRRR {
1565 op,
1566 vd,
1567 vs1,
1568 vs2,
1569 ref mask,
1570 ref vstate,
1571 } => {
1572 let vs1_s = format_reg(vs1);
1573 let vs2_s = format_reg(vs2);
1574 let vd_s = format_reg(vd.to_reg());
1575 let mask = format_mask(mask);
1576
1577 // Note: vs2 and vs1 here are opposite to the standard scalar ordering.
1578 // This is noted in Section 10.1 of the RISC-V Vector spec.
1579 match (op, vs2, vs1) {
1580 (VecAluOpRRR::VrsubVX, _, vs1) if vs1 == zero_reg() => {
1581 format!("vneg.v {vd_s},{vs2_s}{mask} {vstate}")
1582 }
1583 (VecAluOpRRR::VfsgnjnVV, vs2, vs1) if vs2 == vs1 => {
1584 format!("vfneg.v {vd_s},{vs2_s}{mask} {vstate}")
1585 }
1586 (VecAluOpRRR::VfsgnjxVV, vs2, vs1) if vs2 == vs1 => {
1587 format!("vfabs.v {vd_s},{vs2_s}{mask} {vstate}")
1588 }
1589 (VecAluOpRRR::VmnandMM, vs2, vs1) if vs2 == vs1 => {
1590 format!("vmnot.m {vd_s},{vs2_s}{mask} {vstate}")
1591 }
1592 _ => format!("{op} {vd_s},{vs2_s},{vs1_s}{mask} {vstate}"),
1593 }
1594 }
1595 &Inst::VecAluRRImm5 {
1596 op,
1597 vd,
1598 imm,
1599 vs2,
1600 ref mask,
1601 ref vstate,
1602 } => {
1603 let vs2_s = format_reg(vs2);
1604 let vd_s = format_reg(vd.to_reg());
1605 let mask = format_mask(mask);
1606
1607 // Some opcodes interpret the immediate as unsigned, lets show the
1608 // correct number here.
1609 let imm_s = if op.imm_is_unsigned() {
1610 format!("{}", imm.bits())
1611 } else {
1612 format!("{imm}")
1613 };
1614
1615 match (op, imm) {
1616 (VecAluOpRRImm5::VxorVI, imm) if imm == Imm5::maybe_from_i8(-1).unwrap() => {
1617 format!("vnot.v {vd_s},{vs2_s}{mask} {vstate}")
1618 }
1619 _ => format!("{op} {vd_s},{vs2_s},{imm_s}{mask} {vstate}"),
1620 }
1621 }
1622 &Inst::VecAluRR {
1623 op,
1624 vd,
1625 vs,
1626 ref mask,
1627 ref vstate,
1628 } => {
1629 let vs_s = format_reg(vs);
1630 let vd_s = format_reg(vd.to_reg());
1631 let mask = format_mask(mask);
1632
1633 format!("{op} {vd_s},{vs_s}{mask} {vstate}")
1634 }
1635 &Inst::VecAluRImm5 {
1636 op,
1637 vd,
1638 imm,
1639 ref mask,
1640 ref vstate,
1641 } => {
1642 let vd_s = format_reg(vd.to_reg());
1643 let mask = format_mask(mask);
1644
1645 format!("{op} {vd_s},{imm}{mask} {vstate}")
1646 }
1647 &Inst::VecSetState { rd, ref vstate } => {
1648 let rd_s = format_reg(rd.to_reg());
1649 assert!(vstate.avl.is_static());
1650 format!("vsetivli {}, {}, {}", rd_s, vstate.avl, vstate.vtype)
1651 }
1652 Inst::VecLoad {
1653 eew,
1654 to,
1655 from,
1656 mask,
1657 vstate,
1658 ..
1659 } => {
1660 let base = format_vec_amode(from);
1661 let vd = format_reg(to.to_reg());
1662 let mask = format_mask(mask);
1663
1664 format!("vl{eew}.v {vd},{base}{mask} {vstate}")
1665 }
1666 Inst::VecStore {
1667 eew,
1668 to,
1669 from,
1670 mask,
1671 vstate,
1672 ..
1673 } => {
1674 let dst = format_vec_amode(to);
1675 let vs3 = format_reg(*from);
1676 let mask = format_mask(mask);
1677
1678 format!("vs{eew}.v {vs3},{dst}{mask} {vstate}")
1679 }
1680 Inst::EmitIsland { needed_space } => {
1681 format!("emit_island {needed_space}")
1682 }
1683
1684 Inst::LabelAddress { dst, label } => {
1685 let dst = format_reg(dst.to_reg());
1686 format!("label_address {dst}, {label:?}")
1687 }
1688
1689 Inst::SequencePoint {} => {
1690 format!("sequence_point")
1691 }
1692 }
1693 }
1694 }
1695
1696 /// Different forms of label references for different instruction formats.
1697 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
1698 pub enum LabelUse {
1699 /// 20-bit branch offset (unconditional branches). PC-rel, offset is
1700 /// imm << 1. Immediate is 20 signed bits. Use in Jal instructions.
1701 Jal20,
1702
1703 /// The unconditional jump instructions all use PC-relative
1704 /// addressing to help support position independent code. The JALR
1705 /// instruction was defined to enable a two-instruction sequence to
1706 /// jump anywhere in a 32-bit absolute address range. A LUI
1707 /// instruction can first load rs1 with the upper 20 bits of a
1708 /// target address, then JALR can add in the lower bits. Similarly,
1709 /// AUIPC then JALR can jump anywhere in a 32-bit pc-relative
1710 /// address range.
1711 PCRel32,
1712
1713 /// All branch instructions use the B-type instruction format. The
1714 /// 12-bit B-immediate encodes signed offsets in multiples of 2, and
1715 /// is added to the current pc to give the target address. The
1716 /// conditional branch range is ±4 KiB.
1717 B12,
1718
1719 /// Equivalent to the `R_RISCV_PCREL_HI20` relocation, Allows setting
1720 /// the immediate field of an `auipc` instruction.
1721 PCRelHi20,
1722
1723 /// Similar to the `R_RISCV_PCREL_LO12_I` relocation but pointing to
1724 /// the final address, instead of the `PCREL_HI20` label. Allows setting
1725 /// the immediate field of I Type instructions such as `addi` or `lw`.
1726 ///
1727 /// Since we currently don't support offsets in labels, this relocation has
1728 /// an implicit offset of 4.
1729 PCRelLo12I,
1730
1731 /// 11-bit PC-relative jump offset. Equivalent to the `RVC_JUMP` relocation
1732 RVCJump,
1733 }
1734
1735 impl MachInstLabelUse for LabelUse {
1736 /// Alignment for veneer code. Every Riscv64 instruction must be
1737 /// 4-byte-aligned.
1738 const ALIGN: CodeOffset = 4;
1739
1740 /// Maximum PC-relative range (positive), inclusive.
max_pos_range(self) -> CodeOffset1741 fn max_pos_range(self) -> CodeOffset {
1742 match self {
1743 LabelUse::Jal20 => ((1 << 19) - 1) * 2,
1744 LabelUse::PCRelLo12I | LabelUse::PCRelHi20 | LabelUse::PCRel32 => {
1745 Inst::imm_max() as CodeOffset
1746 }
1747 LabelUse::B12 => ((1 << 11) - 1) * 2,
1748 LabelUse::RVCJump => ((1 << 10) - 1) * 2,
1749 }
1750 }
1751
1752 /// Maximum PC-relative range (negative).
max_neg_range(self) -> CodeOffset1753 fn max_neg_range(self) -> CodeOffset {
1754 match self {
1755 LabelUse::PCRel32 => Inst::imm_min().abs() as CodeOffset,
1756 _ => self.max_pos_range() + 2,
1757 }
1758 }
1759
1760 /// Size of window into code needed to do the patch.
patch_size(self) -> CodeOffset1761 fn patch_size(self) -> CodeOffset {
1762 match self {
1763 LabelUse::RVCJump => 2,
1764 LabelUse::Jal20 | LabelUse::B12 | LabelUse::PCRelHi20 | LabelUse::PCRelLo12I => 4,
1765 LabelUse::PCRel32 => 8,
1766 }
1767 }
1768
1769 /// Perform the patch.
patch(self, buffer: &mut [u8], use_offset: CodeOffset, label_offset: CodeOffset)1770 fn patch(self, buffer: &mut [u8], use_offset: CodeOffset, label_offset: CodeOffset) {
1771 assert!(use_offset % 2 == 0);
1772 assert!(label_offset % 2 == 0);
1773 let offset = (label_offset as i64) - (use_offset as i64);
1774
1775 // re-check range
1776 assert!(
1777 offset >= -(self.max_neg_range() as i64) && offset <= (self.max_pos_range() as i64),
1778 "{self:?} offset '{offset}' use_offset:'{use_offset}' label_offset:'{label_offset}' must not exceed max range.",
1779 );
1780 self.patch_raw_offset(buffer, offset);
1781 }
1782
1783 /// Is a veneer supported for this label reference type?
supports_veneer(self) -> bool1784 fn supports_veneer(self) -> bool {
1785 match self {
1786 Self::Jal20 | Self::B12 | Self::RVCJump => true,
1787 _ => false,
1788 }
1789 }
1790
1791 /// How large is the veneer, if supported?
veneer_size(self) -> CodeOffset1792 fn veneer_size(self) -> CodeOffset {
1793 match self {
1794 Self::B12 | Self::Jal20 | Self::RVCJump => 8,
1795 _ => unreachable!(),
1796 }
1797 }
1798
worst_case_veneer_size() -> CodeOffset1799 fn worst_case_veneer_size() -> CodeOffset {
1800 8
1801 }
1802
1803 /// Generate a veneer into the buffer, given that this veneer is at `veneer_offset`, and return
1804 /// an offset and label-use for the veneer's use of the original label.
generate_veneer( self, buffer: &mut [u8], veneer_offset: CodeOffset, ) -> (CodeOffset, LabelUse)1805 fn generate_veneer(
1806 self,
1807 buffer: &mut [u8],
1808 veneer_offset: CodeOffset,
1809 ) -> (CodeOffset, LabelUse) {
1810 let base = writable_spilltmp_reg();
1811 {
1812 let x = enc_auipc(base, Imm20::ZERO).to_le_bytes();
1813 buffer[0] = x[0];
1814 buffer[1] = x[1];
1815 buffer[2] = x[2];
1816 buffer[3] = x[3];
1817 }
1818 {
1819 let x = enc_jalr(writable_zero_reg(), base.to_reg(), Imm12::ZERO).to_le_bytes();
1820 buffer[4] = x[0];
1821 buffer[5] = x[1];
1822 buffer[6] = x[2];
1823 buffer[7] = x[3];
1824 }
1825 (veneer_offset, Self::PCRel32)
1826 }
1827
from_reloc(reloc: Reloc, addend: Addend) -> Option<LabelUse>1828 fn from_reloc(reloc: Reloc, addend: Addend) -> Option<LabelUse> {
1829 match (reloc, addend) {
1830 (Reloc::RiscvCallPlt, _) => Some(Self::PCRel32),
1831 _ => None,
1832 }
1833 }
1834 }
1835
1836 impl LabelUse {
1837 #[expect(dead_code, reason = "in case it's needed in the future")]
offset_in_range(self, offset: i64) -> bool1838 fn offset_in_range(self, offset: i64) -> bool {
1839 let min = -(self.max_neg_range() as i64);
1840 let max = self.max_pos_range() as i64;
1841 offset >= min && offset <= max
1842 }
1843
patch_raw_offset(self, buffer: &mut [u8], offset: i64)1844 fn patch_raw_offset(self, buffer: &mut [u8], offset: i64) {
1845 let insn = match self {
1846 LabelUse::RVCJump => u16::from_le_bytes(buffer[..2].try_into().unwrap()) as u32,
1847 _ => u32::from_le_bytes(buffer[..4].try_into().unwrap()),
1848 };
1849
1850 match self {
1851 LabelUse::Jal20 => {
1852 let offset = offset as u32;
1853 let v = ((offset >> 12 & 0b1111_1111) << 12)
1854 | ((offset >> 11 & 0b1) << 20)
1855 | ((offset >> 1 & 0b11_1111_1111) << 21)
1856 | ((offset >> 20 & 0b1) << 31);
1857 buffer[0..4].clone_from_slice(&u32::to_le_bytes(insn | v));
1858 }
1859 LabelUse::PCRel32 => {
1860 let insn2 = u32::from_le_bytes([buffer[4], buffer[5], buffer[6], buffer[7]]);
1861 Inst::generate_imm(offset as u64)
1862 .map(|(imm20, imm12)| {
1863 // Encode the OR-ed-in value with zero_reg(). The
1864 // register parameter must be in the original
1865 // encoded instruction and or'ing in zeroes does not
1866 // change it.
1867 buffer[0..4].clone_from_slice(&u32::to_le_bytes(
1868 insn | enc_auipc(writable_zero_reg(), imm20),
1869 ));
1870 buffer[4..8].clone_from_slice(&u32::to_le_bytes(
1871 insn2 | enc_jalr(writable_zero_reg(), zero_reg(), imm12),
1872 ));
1873 })
1874 // expect make sure we handled.
1875 .expect("we have check the range before,this is a compiler error.");
1876 }
1877
1878 LabelUse::B12 => {
1879 let offset = offset as u32;
1880 let v = ((offset >> 11 & 0b1) << 7)
1881 | ((offset >> 1 & 0b1111) << 8)
1882 | ((offset >> 5 & 0b11_1111) << 25)
1883 | ((offset >> 12 & 0b1) << 31);
1884 buffer[0..4].clone_from_slice(&u32::to_le_bytes(insn | v));
1885 }
1886
1887 LabelUse::PCRelHi20 => {
1888 // See https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses
1889 //
1890 // We need to add 0x800 to ensure that we land at the next page as soon as it goes out of range for the
1891 // Lo12 relocation. That relocation is signed and has a maximum range of -2048..2047. So when we get an
1892 // offset of 2048, we need to land at the next page and subtract instead.
1893 let offset = offset as u32;
1894 let hi20 = offset.wrapping_add(0x800) >> 12;
1895 let insn = (insn & 0xFFF) | (hi20 << 12);
1896 buffer[0..4].clone_from_slice(&u32::to_le_bytes(insn));
1897 }
1898
1899 LabelUse::PCRelLo12I => {
1900 // `offset` is the offset from the current instruction to the target address.
1901 //
1902 // However we are trying to compute the offset to the target address from the previous instruction.
1903 // The previous instruction should be the one that contains the PCRelHi20 relocation and
1904 // stores/references the program counter (`auipc` usually).
1905 //
1906 // Since we are trying to compute the offset from the previous instruction, we can
1907 // represent it as offset = target_address - (current_instruction_address - 4)
1908 // which is equivalent to offset = target_address - current_instruction_address + 4.
1909 //
1910 // Thus we need to add 4 to the offset here.
1911 let lo12 = (offset + 4) as u32 & 0xFFF;
1912 let insn = (insn & 0xFFFFF) | (lo12 << 20);
1913 buffer[0..4].clone_from_slice(&u32::to_le_bytes(insn));
1914 }
1915 LabelUse::RVCJump => {
1916 debug_assert!(offset & 1 == 0);
1917
1918 // We currently only support this for the C.J operation, so assert that is the opcode in
1919 // the buffer.
1920 debug_assert_eq!(insn & 0xFFFF, 0xA001);
1921
1922 buffer[0..2].clone_from_slice(&u16::to_le_bytes(encode_cj_type(
1923 CjOp::CJ,
1924 Imm12::from_i16(i16::try_from(offset).unwrap()),
1925 )));
1926 }
1927 }
1928 }
1929 }
1930
1931 #[cfg(test)]
1932 mod test {
1933 use super::*;
1934 #[test]
label_use_max_range()1935 fn label_use_max_range() {
1936 assert!(LabelUse::B12.max_neg_range() == LabelUse::B12.max_pos_range() + 2);
1937 assert!(LabelUse::Jal20.max_neg_range() == LabelUse::Jal20.max_pos_range() + 2);
1938 assert!(LabelUse::PCRel32.max_pos_range() == (Inst::imm_max() as CodeOffset));
1939 assert!(LabelUse::PCRel32.max_neg_range() == (Inst::imm_min().abs() as CodeOffset));
1940 assert!(LabelUse::B12.max_pos_range() == ((1 << 11) - 1) * 2);
1941 }
1942 }
1943