1 //! Assembler library implementation for Aarch64.
2 
3 use super::{address::Address, regs};
4 use crate::masm::{RoundingMode, ShiftKind};
5 use crate::{masm::OperandSize, reg::Reg};
6 use cranelift_codegen::isa::aarch64::inst::FPUOpRI::{UShr32, UShr64};
7 use cranelift_codegen::isa::aarch64::inst::{
8     FPULeftShiftImm, FPUOp1, FPUOp2, FPUOpRI, FPUOpRIMod, FPURightShiftImm, FpuRoundMode, ImmLogic,
9     ImmShift, ScalarSize,
10 };
11 use cranelift_codegen::{
12     ir::{MemFlags, SourceLoc},
13     isa::aarch64::inst::{
14         self,
15         emit::{EmitInfo, EmitState},
16         ALUOp, ALUOp3, AMode, ExtendOp, Imm12, Inst, PairAMode,
17     },
18     settings, Final, MachBuffer, MachBufferFinalized, MachInstEmit, MachInstEmitState, MachLabel,
19     Writable,
20 };
21 
22 impl From<OperandSize> for inst::OperandSize {
23     fn from(size: OperandSize) -> Self {
24         match size {
25             OperandSize::S32 => Self::Size32,
26             OperandSize::S64 => Self::Size64,
27             s => panic!("Invalid operand size {:?}", s),
28         }
29     }
30 }
31 
32 impl Into<ScalarSize> for OperandSize {
33     fn into(self) -> ScalarSize {
34         match self {
35             OperandSize::S8 => ScalarSize::Size8,
36             OperandSize::S16 => ScalarSize::Size16,
37             OperandSize::S32 => ScalarSize::Size32,
38             OperandSize::S64 => ScalarSize::Size64,
39             OperandSize::S128 => ScalarSize::Size128,
40         }
41     }
42 }
43 
44 /// Low level assembler implementation for Aarch64.
45 pub(crate) struct Assembler {
46     /// The machine instruction buffer.
47     buffer: MachBuffer<Inst>,
48     /// Constant emission information.
49     emit_info: EmitInfo,
50     /// Emission state.
51     emit_state: EmitState,
52 }
53 
54 impl Assembler {
55     /// Create a new Aarch64 assembler.
56     pub fn new(shared_flags: settings::Flags) -> Self {
57         Self {
58             buffer: MachBuffer::<Inst>::new(),
59             emit_state: Default::default(),
60             emit_info: EmitInfo::new(shared_flags),
61         }
62     }
63 }
64 
65 impl Assembler {
66     /// Return the emitted code.
67     pub fn finalize(mut self, loc: Option<SourceLoc>) -> MachBufferFinalized<Final> {
68         let constants = Default::default();
69         let stencil = self
70             .buffer
71             .finish(&constants, self.emit_state.ctrl_plane_mut());
72         stencil.apply_base_srcloc(loc.unwrap_or_default())
73     }
74 
75     fn emit(&mut self, inst: Inst) {
76         inst.emit(&mut self.buffer, &self.emit_info, &mut self.emit_state);
77     }
78 
79     /// Load a constant into a register.
80     pub fn load_constant(&mut self, imm: u64, rd: Reg) {
81         let writable = Writable::from_reg(rd.into());
82         Inst::load_constant(writable, imm, &mut |_| writable)
83             .into_iter()
84             .for_each(|i| self.emit(i));
85     }
86 
87     /// Store a pair of registers.
88     pub fn stp(&mut self, xt1: Reg, xt2: Reg, addr: Address) {
89         let mem: PairAMode = addr.try_into().unwrap();
90         self.emit(Inst::StoreP64 {
91             rt: xt1.into(),
92             rt2: xt2.into(),
93             mem,
94             flags: MemFlags::trusted(),
95         });
96     }
97 
98     /// Store a register.
99     pub fn str(&mut self, reg: Reg, addr: Address, size: OperandSize) {
100         let mem: AMode = addr.try_into().unwrap();
101         let flags = MemFlags::trusted();
102 
103         use OperandSize::*;
104         let inst = match size {
105             S64 => Inst::Store64 {
106                 rd: reg.into(),
107                 mem,
108                 flags,
109             },
110             S32 => Inst::Store32 {
111                 rd: reg.into(),
112                 mem,
113                 flags,
114             },
115 
116             _ => unreachable!(),
117         };
118 
119         self.emit(inst);
120     }
121 
122     /// Load a register.
123     pub fn ldr(&mut self, addr: Address, rd: Reg, size: OperandSize) {
124         use OperandSize::*;
125         let writable_reg = Writable::from_reg(rd.into());
126         let mem: AMode = addr.try_into().unwrap();
127         let flags = MemFlags::trusted();
128 
129         let inst = match size {
130             S64 => Inst::ULoad64 {
131                 rd: writable_reg,
132                 mem,
133                 flags,
134             },
135             S32 => Inst::ULoad32 {
136                 rd: writable_reg,
137                 mem,
138                 flags,
139             },
140             _ => unreachable!(),
141         };
142 
143         self.emit(inst);
144     }
145 
146     /// Load a pair of registers.
147     pub fn ldp(&mut self, xt1: Reg, xt2: Reg, addr: Address) {
148         let writable_xt1 = Writable::from_reg(xt1.into());
149         let writable_xt2 = Writable::from_reg(xt2.into());
150         let mem = addr.try_into().unwrap();
151 
152         self.emit(Inst::LoadP64 {
153             rt: writable_xt1,
154             rt2: writable_xt2,
155             mem,
156             flags: MemFlags::trusted(),
157         });
158     }
159 
160     /// Register to register move.
161     pub fn mov_rr(&mut self, rm: Reg, rd: Reg, size: OperandSize) {
162         let writable_rd = Writable::from_reg(rd.into());
163         self.emit(Inst::Mov {
164             size: size.into(),
165             rd: writable_rd,
166             rm: rm.into(),
167         });
168     }
169 
170     /// Floating point register to register move.
171     pub fn fmov_rr(&mut self, rn: Reg, rd: Reg, size: OperandSize) {
172         let writable = Writable::from_reg(rd.into());
173         let inst = match size {
174             OperandSize::S32 => Inst::FpuMove32 {
175                 rd: writable,
176                 rn: rn.into(),
177             },
178             OperandSize::S64 => Inst::FpuMove64 {
179                 rd: writable,
180                 rn: rn.into(),
181             },
182             _ => unreachable!(),
183         };
184 
185         self.emit(inst);
186     }
187 
188     pub fn mov_to_fpu(&mut self, rn: Reg, rd: Reg, size: OperandSize) {
189         let writable_rd = Writable::from_reg(rd.into());
190         self.emit(Inst::MovToFpu {
191             size: size.into(),
192             rd: writable_rd,
193             rn: rn.into(),
194         });
195     }
196 
197     /// Add with three registers.
198     pub fn add_rrr(&mut self, rm: Reg, rn: Reg, rd: Reg, size: OperandSize) {
199         self.emit_alu_rrr_extend(ALUOp::Add, rm, rn, rd, size);
200     }
201 
202     /// Add immediate and register.
203     pub fn add_ir(&mut self, imm: u64, rn: Reg, rd: Reg, size: OperandSize) {
204         let alu_op = ALUOp::Add;
205         if let Some(imm) = Imm12::maybe_from_u64(imm) {
206             self.emit_alu_rri(alu_op, imm, rn, rd, size);
207         } else {
208             let scratch = regs::scratch();
209             self.load_constant(imm, scratch);
210             self.emit_alu_rrr_extend(alu_op, scratch, rn, rd, size);
211         }
212     }
213 
214     /// Subtract with three registers.
215     pub fn sub_rrr(&mut self, rm: Reg, rn: Reg, rd: Reg, size: OperandSize) {
216         self.emit_alu_rrr_extend(ALUOp::Sub, rm, rn, rd, size);
217     }
218 
219     /// Subtract immediate and register.
220     pub fn sub_ir(&mut self, imm: u64, rn: Reg, rd: Reg, size: OperandSize) {
221         let alu_op = ALUOp::Sub;
222         if let Some(imm) = Imm12::maybe_from_u64(imm) {
223             self.emit_alu_rri(alu_op, imm, rn, rd, size);
224         } else {
225             let scratch = regs::scratch();
226             self.load_constant(imm, scratch);
227             self.emit_alu_rrr_extend(alu_op, scratch, rn, rd, size);
228         }
229     }
230 
231     /// Multiply with three registers.
232     pub fn mul_rrr(&mut self, rm: Reg, rn: Reg, rd: Reg, size: OperandSize) {
233         self.emit_alu_rrrr(ALUOp3::MAdd, rm, rn, rd, regs::zero(), size);
234     }
235 
236     /// Multiply immediate and register.
237     pub fn mul_ir(&mut self, imm: u64, rn: Reg, rd: Reg, size: OperandSize) {
238         let scratch = regs::scratch();
239         self.load_constant(imm, scratch);
240         self.emit_alu_rrrr(ALUOp3::MAdd, scratch, rn, rd, regs::zero(), size);
241     }
242 
243     /// And with three registers.
244     pub fn and_rrr(&mut self, rm: Reg, rn: Reg, rd: Reg, size: OperandSize) {
245         self.emit_alu_rrr(ALUOp::And, rm, rn, rd, size);
246     }
247 
248     /// And immediate and register.
249     pub fn and_ir(&mut self, imm: u64, rn: Reg, rd: Reg, size: OperandSize) {
250         let alu_op = ALUOp::And;
251         let cl_size: inst::OperandSize = size.into();
252         if let Some(imm) = ImmLogic::maybe_from_u64(imm, cl_size.to_ty()) {
253             self.emit_alu_rri_logic(alu_op, imm, rn, rd, size);
254         } else {
255             let scratch = regs::scratch();
256             self.load_constant(imm, scratch);
257             self.emit_alu_rrr(alu_op, scratch, rn, rd, size);
258         }
259     }
260 
261     /// Or with three registers.
262     pub fn or_rrr(&mut self, rm: Reg, rn: Reg, rd: Reg, size: OperandSize) {
263         self.emit_alu_rrr(ALUOp::Orr, rm, rn, rd, size);
264     }
265 
266     /// Or immediate and register.
267     pub fn or_ir(&mut self, imm: u64, rn: Reg, rd: Reg, size: OperandSize) {
268         let alu_op = ALUOp::Orr;
269         let cl_size: inst::OperandSize = size.into();
270         if let Some(imm) = ImmLogic::maybe_from_u64(imm, cl_size.to_ty()) {
271             self.emit_alu_rri_logic(alu_op, imm, rn, rd, size);
272         } else {
273             let scratch = regs::scratch();
274             self.load_constant(imm, scratch);
275             self.emit_alu_rrr(alu_op, scratch, rn, rd, size);
276         }
277     }
278 
279     /// Xor with three registers.
280     pub fn xor_rrr(&mut self, rm: Reg, rn: Reg, rd: Reg, size: OperandSize) {
281         self.emit_alu_rrr(ALUOp::Eor, rm, rn, rd, size);
282     }
283 
284     /// Xor immediate and register.
285     pub fn xor_ir(&mut self, imm: u64, rn: Reg, rd: Reg, size: OperandSize) {
286         let alu_op = ALUOp::Eor;
287         let cl_size: inst::OperandSize = size.into();
288         if let Some(imm) = ImmLogic::maybe_from_u64(imm, cl_size.to_ty()) {
289             self.emit_alu_rri_logic(alu_op, imm, rn, rd, size);
290         } else {
291             let scratch = regs::scratch();
292             self.load_constant(imm, scratch);
293             self.emit_alu_rrr(alu_op, scratch, rn, rd, size);
294         }
295     }
296 
297     /// Shift with three registers.
298     pub fn shift_rrr(&mut self, rm: Reg, rn: Reg, rd: Reg, kind: ShiftKind, size: OperandSize) {
299         let shift_op = self.shift_kind_to_alu_op(kind, rm, size);
300         self.emit_alu_rrr(shift_op, rm, rn, rd, size);
301     }
302 
303     /// Shift immediate and register.
304     pub fn shift_ir(&mut self, imm: u64, rn: Reg, rd: Reg, kind: ShiftKind, size: OperandSize) {
305         let shift_op = self.shift_kind_to_alu_op(kind, rn, size);
306 
307         if let Some(imm) = ImmShift::maybe_from_u64(imm) {
308             self.emit_alu_rri_shift(shift_op, imm, rn, rd, size);
309         } else {
310             let scratch = regs::scratch();
311             self.load_constant(imm, scratch);
312             self.emit_alu_rrr(shift_op, scratch, rn, rd, size);
313         }
314     }
315 
316     /// Float add with three registers.
317     pub fn fadd_rrr(&mut self, rm: Reg, rn: Reg, rd: Reg, size: OperandSize) {
318         self.emit_fpu_rrr(FPUOp2::Add, rm, rn, rd, size);
319     }
320 
321     /// Float sub with three registers.
322     pub fn fsub_rrr(&mut self, rm: Reg, rn: Reg, rd: Reg, size: OperandSize) {
323         self.emit_fpu_rrr(FPUOp2::Sub, rm, rn, rd, size);
324     }
325 
326     /// Float multiply with three registers.
327     pub fn fmul_rrr(&mut self, rm: Reg, rn: Reg, rd: Reg, size: OperandSize) {
328         self.emit_fpu_rrr(FPUOp2::Mul, rm, rn, rd, size);
329     }
330 
331     /// Float division with three registers.
332     pub fn fdiv_rrr(&mut self, rm: Reg, rn: Reg, rd: Reg, size: OperandSize) {
333         self.emit_fpu_rrr(FPUOp2::Div, rm, rn, rd, size);
334     }
335 
336     /// Float max with three registers.
337     pub fn fmax_rrr(&mut self, rm: Reg, rn: Reg, rd: Reg, size: OperandSize) {
338         self.emit_fpu_rrr(FPUOp2::Max, rm, rn, rd, size);
339     }
340 
341     /// Float min with three registers.
342     pub fn fmin_rrr(&mut self, rm: Reg, rn: Reg, rd: Reg, size: OperandSize) {
343         self.emit_fpu_rrr(FPUOp2::Min, rm, rn, rd, size);
344     }
345 
346     /// Float neg with two registers.
347     pub fn fneg_rr(&mut self, rn: Reg, rd: Reg, size: OperandSize) {
348         self.emit_fpu_rr(FPUOp1::Neg, rn, rd, size);
349     }
350 
351     /// Float abs with two registers.
352     pub fn fabs_rr(&mut self, rn: Reg, rd: Reg, size: OperandSize) {
353         self.emit_fpu_rr(FPUOp1::Abs, rn, rd, size);
354     }
355 
356     /// Float sqrt with two registers.
357     pub fn fsqrt_rr(&mut self, rn: Reg, rd: Reg, size: OperandSize) {
358         self.emit_fpu_rr(FPUOp1::Sqrt, rn, rd, size);
359     }
360 
361     /// Float round (ceil, trunc, floor) with two registers.
362     pub fn fround_rr(&mut self, rn: Reg, rd: Reg, mode: RoundingMode, size: OperandSize) {
363         let fpu_mode = match (mode, size) {
364             (RoundingMode::Nearest, OperandSize::S32) => FpuRoundMode::Nearest32,
365             (RoundingMode::Up, OperandSize::S32) => FpuRoundMode::Plus32,
366             (RoundingMode::Down, OperandSize::S32) => FpuRoundMode::Minus32,
367             (RoundingMode::Zero, OperandSize::S32) => FpuRoundMode::Zero32,
368             (RoundingMode::Nearest, OperandSize::S64) => FpuRoundMode::Nearest64,
369             (RoundingMode::Up, OperandSize::S64) => FpuRoundMode::Plus64,
370             (RoundingMode::Down, OperandSize::S64) => FpuRoundMode::Minus64,
371             (RoundingMode::Zero, OperandSize::S64) => FpuRoundMode::Zero64,
372             (m, o) => panic!("Invalid rounding mode or operand size {:?}, {:?}", m, o),
373         };
374         self.emit_fpu_round(fpu_mode, rn, rd)
375     }
376 
377     /// Float unsigned shift right with two registers and an immediate.
378     pub fn fushr_rri(&mut self, rn: Reg, rd: Reg, amount: u8, size: OperandSize) {
379         let imm = FPURightShiftImm {
380             amount: amount,
381             lane_size_in_bits: size.num_bits(),
382         };
383         let ushr = match size {
384             OperandSize::S32 => UShr32(imm),
385             OperandSize::S64 => UShr64(imm),
386             _ => unreachable!(),
387         };
388         self.emit_fpu_rri(ushr, rn, rd)
389     }
390 
391     /// Float unsigned shift left and insert with three registers
392     /// and an immediate.
393     pub fn fsli_rri_mod(&mut self, ri: Reg, rn: Reg, rd: Reg, amount: u8, size: OperandSize) {
394         let imm = FPULeftShiftImm {
395             amount: amount,
396             lane_size_in_bits: size.num_bits(),
397         };
398         let sli = match size {
399             OperandSize::S32 => FPUOpRIMod::Sli32(imm),
400             OperandSize::S64 => FPUOpRIMod::Sli64(imm),
401             _ => unreachable!(),
402         };
403         self.emit_fpu_rri_mod(sli, ri, rn, rd)
404     }
405 
406     /// Return instruction.
407     pub fn ret(&mut self) {
408         self.emit(Inst::Ret {});
409     }
410 
411     // Helpers for ALU operations.
412 
413     fn emit_alu_rri(&mut self, op: ALUOp, imm: Imm12, rn: Reg, rd: Reg, size: OperandSize) {
414         self.emit(Inst::AluRRImm12 {
415             alu_op: op,
416             size: size.into(),
417             rd: Writable::from_reg(rd.into()),
418             rn: rn.into(),
419             imm12: imm,
420         });
421     }
422 
423     fn emit_alu_rri_logic(
424         &mut self,
425         op: ALUOp,
426         imm: ImmLogic,
427         rn: Reg,
428         rd: Reg,
429         size: OperandSize,
430     ) {
431         self.emit(Inst::AluRRImmLogic {
432             alu_op: op,
433             size: size.into(),
434             rd: Writable::from_reg(rd.into()),
435             rn: rn.into(),
436             imml: imm,
437         });
438     }
439 
440     fn emit_alu_rri_shift(
441         &mut self,
442         op: ALUOp,
443         imm: ImmShift,
444         rn: Reg,
445         rd: Reg,
446         size: OperandSize,
447     ) {
448         self.emit(Inst::AluRRImmShift {
449             alu_op: op,
450             size: size.into(),
451             rd: Writable::from_reg(rd.into()),
452             rn: rn.into(),
453             immshift: imm,
454         });
455     }
456 
457     fn emit_alu_rrr(&mut self, op: ALUOp, rm: Reg, rn: Reg, rd: Reg, size: OperandSize) {
458         self.emit(Inst::AluRRR {
459             alu_op: op,
460             size: size.into(),
461             rd: Writable::from_reg(rd.into()),
462             rn: rn.into(),
463             rm: rm.into(),
464         });
465     }
466 
467     fn emit_alu_rrr_extend(&mut self, op: ALUOp, rm: Reg, rn: Reg, rd: Reg, size: OperandSize) {
468         self.emit(Inst::AluRRRExtend {
469             alu_op: op,
470             size: size.into(),
471             rd: Writable::from_reg(rd.into()),
472             rn: rn.into(),
473             rm: rm.into(),
474             extendop: ExtendOp::UXTX,
475         });
476     }
477 
478     fn emit_alu_rrrr(&mut self, op: ALUOp3, rm: Reg, rn: Reg, rd: Reg, ra: Reg, size: OperandSize) {
479         self.emit(Inst::AluRRRR {
480             alu_op: op,
481             size: size.into(),
482             rd: Writable::from_reg(rd.into()),
483             rn: rn.into(),
484             rm: rm.into(),
485             ra: ra.into(),
486         });
487     }
488 
489     fn emit_fpu_rrr(&mut self, op: FPUOp2, rm: Reg, rn: Reg, rd: Reg, size: OperandSize) {
490         self.emit(Inst::FpuRRR {
491             fpu_op: op,
492             size: size.into(),
493             rd: Writable::from_reg(rd.into()),
494             rn: rn.into(),
495             rm: rm.into(),
496         });
497     }
498 
499     fn emit_fpu_rri(&mut self, op: FPUOpRI, rn: Reg, rd: Reg) {
500         self.emit(Inst::FpuRRI {
501             fpu_op: op,
502             rd: Writable::from_reg(rd.into()),
503             rn: rn.into(),
504         });
505     }
506 
507     fn emit_fpu_rri_mod(&mut self, op: FPUOpRIMod, ri: Reg, rn: Reg, rd: Reg) {
508         self.emit(Inst::FpuRRIMod {
509             fpu_op: op,
510             rd: Writable::from_reg(rd.into()),
511             ri: ri.into(),
512             rn: rn.into(),
513         });
514     }
515 
516     fn emit_fpu_rr(&mut self, op: FPUOp1, rn: Reg, rd: Reg, size: OperandSize) {
517         self.emit(Inst::FpuRR {
518             fpu_op: op,
519             size: size.into(),
520             rd: Writable::from_reg(rd.into()),
521             rn: rn.into(),
522         });
523     }
524 
525     fn emit_fpu_round(&mut self, op: FpuRoundMode, rn: Reg, rd: Reg) {
526         self.emit(Inst::FpuRound {
527             op: op,
528             rd: Writable::from_reg(rd.into()),
529             rn: rn.into(),
530         });
531     }
532 
533     // Convert ShiftKind to ALUOp. If kind == Rotl, then emulate it by emitting
534     // the negation of the given reg r, and returns ALUOp::RotR.
535     fn shift_kind_to_alu_op(&mut self, kind: ShiftKind, r: Reg, size: OperandSize) -> ALUOp {
536         match kind {
537             ShiftKind::Shl => ALUOp::Lsl,
538             ShiftKind::ShrS => ALUOp::Asr,
539             ShiftKind::ShrU => ALUOp::Lsr,
540             ShiftKind::Rotr => ALUOp::RotR,
541             ShiftKind::Rotl => {
542                 // neg(r) is sub(zero, r).
543                 self.emit_alu_rrr(ALUOp::Sub, regs::zero(), r, r, size);
544                 ALUOp::RotR
545             }
546         }
547     }
548 
549     /// Get a label from the underlying machine code buffer.
550     pub fn get_label(&mut self) -> MachLabel {
551         self.buffer.get_label()
552     }
553 
554     /// Get a mutable reference to underlying
555     /// machine buffer.
556     pub fn buffer_mut(&mut self) -> &mut MachBuffer<Inst> {
557         &mut self.buffer
558     }
559 
560     /// Get a reference to the underlying machine buffer.
561     pub fn buffer(&self) -> &MachBuffer<Inst> {
562         &self.buffer
563     }
564 }
565