1 //! Assembler library implementation for Aarch64. 2 use super::regs; 3 use crate::CallingConvention; 4 use crate::aarch64::regs::zero; 5 use crate::masm::{ 6 DivKind, Extend, ExtendKind, FloatCmpKind, Imm, IntCmpKind, RemKind, RoundingMode, ShiftKind, 7 Signed, TRUSTED_FLAGS, TruncKind, 8 }; 9 use crate::{ 10 constant_pool::ConstantPool, 11 masm::OperandSize, 12 reg::{Reg, WritableReg, writable}, 13 }; 14 15 use cranelift_codegen::isa::aarch64; 16 use cranelift_codegen::isa::aarch64::inst::emit::{enc_arith_rrr, enc_move_wide, enc_movk}; 17 use cranelift_codegen::isa::aarch64::inst::{ 18 ASIMDFPModImm, FpuToIntOp, MoveWideConst, NZCV, UImm5, 19 }; 20 use cranelift_codegen::{ 21 Final, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit, MachInstEmitState, MachLabel, 22 Writable, 23 ir::{ExternalName, MemFlags, SourceLoc, TrapCode, UserExternalNameRef}, 24 isa::aarch64::inst::{ 25 self, ALUOp, ALUOp3, AMode, BitOp, BranchTarget, Cond, CondBrKind, ExtendOp, 26 FPULeftShiftImm, FPUOp1, FPUOp2, 27 FPUOpRI::{self, UShr32, UShr64}, 28 FPUOpRIMod, FPURightShiftImm, FpuRoundMode, Imm12, ImmLogic, ImmShift, Inst, IntToFpuOp, 29 PairAMode, ScalarSize, VecLanesOp, VecMisc2, VectorSize, 30 emit::{EmitInfo, EmitState}, 31 }, 32 settings, 33 }; 34 use cranelift_codegen::{PatchRegion, VCodeConstant}; 35 use regalloc2::RegClass; 36 use wasmtime_core::math::{f32_cvt_to_int_bounds, f64_cvt_to_int_bounds}; 37 38 impl From<OperandSize> for inst::OperandSize { 39 fn from(size: OperandSize) -> Self { 40 match size { 41 OperandSize::S32 => Self::Size32, 42 OperandSize::S64 => Self::Size64, 43 s => panic!("Invalid operand size {s:?}"), 44 } 45 } 46 } 47 48 impl From<IntCmpKind> for Cond { 49 fn from(value: IntCmpKind) -> Self { 50 match value { 51 IntCmpKind::Eq => Cond::Eq, 52 IntCmpKind::Ne => Cond::Ne, 53 IntCmpKind::LtS => Cond::Lt, 54 IntCmpKind::LtU => Cond::Lo, 55 IntCmpKind::GtS => Cond::Gt, 56 IntCmpKind::GtU => Cond::Hi, 57 IntCmpKind::LeS => Cond::Le, 58 IntCmpKind::LeU => Cond::Ls, 59 IntCmpKind::GeS => Cond::Ge, 60 IntCmpKind::GeU => Cond::Hs, 61 } 62 } 63 } 64 65 impl From<FloatCmpKind> for Cond { 66 fn from(value: FloatCmpKind) -> Self { 67 match value { 68 FloatCmpKind::Eq => Cond::Eq, 69 FloatCmpKind::Ne => Cond::Ne, 70 FloatCmpKind::Lt => Cond::Mi, 71 FloatCmpKind::Gt => Cond::Gt, 72 FloatCmpKind::Le => Cond::Ls, 73 FloatCmpKind::Ge => Cond::Ge, 74 } 75 } 76 } 77 78 impl From<OperandSize> for ScalarSize { 79 fn from(size: OperandSize) -> ScalarSize { 80 match size { 81 OperandSize::S8 => ScalarSize::Size8, 82 OperandSize::S16 => ScalarSize::Size16, 83 OperandSize::S32 => ScalarSize::Size32, 84 OperandSize::S64 => ScalarSize::Size64, 85 OperandSize::S128 => ScalarSize::Size128, 86 } 87 } 88 } 89 90 impl From<ShiftKind> for ALUOp { 91 fn from(kind: ShiftKind) -> Self { 92 match kind { 93 ShiftKind::Shl => ALUOp::Lsl, 94 ShiftKind::ShrS => ALUOp::Asr, 95 ShiftKind::ShrU => ALUOp::Lsr, 96 ShiftKind::Rotr => ALUOp::Extr, 97 ShiftKind::Rotl => ALUOp::Extr, 98 } 99 } 100 } 101 102 /// Low level assembler implementation for Aarch64. 103 pub(crate) struct Assembler { 104 /// The machine instruction buffer. 105 buffer: MachBuffer<Inst>, 106 /// Constant emission information. 107 emit_info: EmitInfo, 108 /// Emission state. 109 emit_state: EmitState, 110 /// Constant pool. 111 pool: ConstantPool, 112 } 113 114 impl Assembler { 115 /// Create a new Aarch64 assembler. 116 pub fn new(shared_flags: settings::Flags, isa_flags: aarch64::settings::Flags) -> Self { 117 Self { 118 buffer: MachBuffer::<Inst>::new(), 119 emit_state: Default::default(), 120 emit_info: EmitInfo::new(shared_flags, isa_flags), 121 pool: ConstantPool::new(), 122 } 123 } 124 } 125 126 impl Assembler { 127 /// Return the emitted code. 128 pub fn finalize(mut self, loc: Option<SourceLoc>) -> MachBufferFinalized<Final> { 129 let stencil = self 130 .buffer 131 .finish(&self.pool.constants(), self.emit_state.ctrl_plane_mut()); 132 stencil.apply_base_srcloc(loc.unwrap_or_default()) 133 } 134 135 fn emit(&mut self, inst: Inst) { 136 self.emit_with_island(inst, Inst::worst_case_size()); 137 } 138 139 fn emit_with_island(&mut self, inst: Inst, needed_space: u32) { 140 if self.buffer.island_needed(needed_space) { 141 let label = self.buffer.get_label(); 142 let jmp = Inst::Jump { 143 dest: BranchTarget::Label(label), 144 }; 145 jmp.emit(&mut self.buffer, &self.emit_info, &mut self.emit_state); 146 self.buffer 147 .emit_island(needed_space, self.emit_state.ctrl_plane_mut()); 148 self.buffer 149 .bind_label(label, self.emit_state.ctrl_plane_mut()); 150 } 151 inst.emit(&mut self.buffer, &self.emit_info, &mut self.emit_state); 152 } 153 154 /// Adds a constant to the constant pool, returning its address. 155 pub fn add_constant(&mut self, constant: &[u8]) -> VCodeConstant { 156 let handle = self.pool.register(constant, &mut self.buffer); 157 handle 158 } 159 160 /// Store a pair of registers. 161 pub fn stp(&mut self, xt1: Reg, xt2: Reg, mem: PairAMode) { 162 self.emit(Inst::StoreP64 { 163 rt: xt1.into(), 164 rt2: xt2.into(), 165 mem, 166 flags: MemFlags::trusted(), 167 }); 168 } 169 170 /// Store a register. 171 pub fn str(&mut self, reg: Reg, mem: AMode, size: OperandSize, flags: MemFlags) { 172 use OperandSize::*; 173 let inst = match (reg.is_int(), size) { 174 (_, S8) => Inst::Store8 { 175 rd: reg.into(), 176 mem, 177 flags, 178 }, 179 (_, S16) => Inst::Store16 { 180 rd: reg.into(), 181 mem, 182 flags, 183 }, 184 (true, S32) => Inst::Store32 { 185 rd: reg.into(), 186 mem, 187 flags, 188 }, 189 (false, S32) => Inst::FpuStore32 { 190 rd: reg.into(), 191 mem, 192 flags, 193 }, 194 (true, S64) => Inst::Store64 { 195 rd: reg.into(), 196 mem, 197 flags, 198 }, 199 (false, S64) => Inst::FpuStore64 { 200 rd: reg.into(), 201 mem, 202 flags, 203 }, 204 (_, S128) => Inst::FpuStore128 { 205 rd: reg.into(), 206 mem, 207 flags, 208 }, 209 }; 210 211 self.emit(inst); 212 } 213 214 /// Load a signed register. 215 pub fn sload(&mut self, mem: AMode, rd: WritableReg, size: OperandSize, flags: MemFlags) { 216 self.ldr(mem, rd, size, true, flags); 217 } 218 219 /// Load an unsigned register. 220 pub fn uload(&mut self, mem: AMode, rd: WritableReg, size: OperandSize, flags: MemFlags) { 221 self.ldr(mem, rd, size, false, flags); 222 } 223 224 /// Load address into a register. 225 fn ldr( 226 &mut self, 227 mem: AMode, 228 rd: WritableReg, 229 size: OperandSize, 230 signed: bool, 231 flags: MemFlags, 232 ) { 233 use OperandSize::*; 234 let writable_reg = rd.map(Into::into); 235 236 let inst = match (rd.to_reg().is_int(), signed, size) { 237 (_, false, S8) => Inst::ULoad8 { 238 rd: writable_reg, 239 mem, 240 flags, 241 }, 242 (_, true, S8) => Inst::SLoad8 { 243 rd: writable_reg, 244 mem, 245 flags, 246 }, 247 (_, false, S16) => Inst::ULoad16 { 248 rd: writable_reg, 249 mem, 250 flags, 251 }, 252 (_, true, S16) => Inst::SLoad16 { 253 rd: writable_reg, 254 mem, 255 flags, 256 }, 257 (true, false, S32) => Inst::ULoad32 { 258 rd: writable_reg, 259 mem, 260 flags, 261 }, 262 (false, _, S32) => Inst::FpuLoad32 { 263 rd: writable_reg, 264 mem, 265 flags, 266 }, 267 (true, true, S32) => Inst::SLoad32 { 268 rd: writable_reg, 269 mem, 270 flags, 271 }, 272 (true, _, S64) => Inst::ULoad64 { 273 rd: writable_reg, 274 mem, 275 flags, 276 }, 277 (false, _, S64) => Inst::FpuLoad64 { 278 rd: writable_reg, 279 mem, 280 flags, 281 }, 282 (_, _, S128) => Inst::FpuLoad128 { 283 rd: writable_reg, 284 mem, 285 flags, 286 }, 287 }; 288 289 self.emit(inst); 290 } 291 292 /// Load a pair of registers. 293 pub fn ldp(&mut self, xt1: Reg, xt2: Reg, mem: PairAMode) { 294 let writable_xt1 = Writable::from_reg(xt1.into()); 295 let writable_xt2 = Writable::from_reg(xt2.into()); 296 297 self.emit(Inst::LoadP64 { 298 rt: writable_xt1, 299 rt2: writable_xt2, 300 mem, 301 flags: MemFlags::trusted(), 302 }); 303 } 304 305 /// Emit a series of instructions to move an arbitrary 64-bit immediate 306 /// into the destination register. 307 /// The emitted instructions will depend on the destination register class. 308 pub fn mov_ir(&mut self, rd: WritableReg, imm: Imm, size: OperandSize) { 309 match rd.to_reg().class() { 310 RegClass::Int => { 311 Inst::load_constant(rd.map(Into::into), imm.unwrap_as_u64()) 312 .into_iter() 313 .for_each(|i| self.emit(i)); 314 } 315 RegClass::Float => { 316 match ASIMDFPModImm::maybe_from_u64(imm.unwrap_as_u64(), size.into()) { 317 Some(imm) => { 318 self.emit(Inst::FpuMoveFPImm { 319 rd: rd.map(Into::into), 320 imm, 321 size: size.into(), 322 }); 323 } 324 _ => { 325 let constant = self.add_constant(&imm.to_bytes()); 326 let addr = AMode::Const { addr: constant }; 327 self.uload(addr, rd, size, TRUSTED_FLAGS); 328 } 329 } 330 } 331 _ => unreachable!(), 332 } 333 } 334 335 /// Register to register move. 336 pub fn mov_rr(&mut self, rm: Reg, rd: WritableReg, size: OperandSize) { 337 let writable_rd = rd.map(Into::into); 338 self.emit(Inst::Mov { 339 size: size.into(), 340 rd: writable_rd, 341 rm: rm.into(), 342 }); 343 } 344 345 /// Floating point register to register move. 346 pub fn fmov_rr(&mut self, rn: Reg, rd: WritableReg, size: OperandSize) { 347 let writable = rd.map(Into::into); 348 let inst = match size { 349 OperandSize::S32 => Inst::FpuMove32 { 350 rd: writable, 351 rn: rn.into(), 352 }, 353 OperandSize::S64 => Inst::FpuMove64 { 354 rd: writable, 355 rn: rn.into(), 356 }, 357 _ => unreachable!(), 358 }; 359 360 self.emit(inst); 361 } 362 363 pub fn mov_to_fpu(&mut self, rn: Reg, rd: WritableReg, size: OperandSize) { 364 let writable_rd = rd.map(Into::into); 365 self.emit(Inst::MovToFpu { 366 size: size.into(), 367 rd: writable_rd, 368 rn: rn.into(), 369 }); 370 } 371 372 pub fn mov_from_vec(&mut self, rn: Reg, rd: WritableReg, idx: u8, size: OperandSize) { 373 self.emit(Inst::MovFromVec { 374 rd: rd.map(Into::into), 375 rn: rn.into(), 376 idx, 377 size: size.into(), 378 }); 379 } 380 381 /// Add immediate and register. 382 pub fn add_ir(&mut self, imm: Imm12, rn: Reg, rd: WritableReg, size: OperandSize) { 383 self.alu_rri(ALUOp::Add, imm, rn, rd, size); 384 } 385 386 /// Add immediate and register, setting overflow flags. 387 pub fn adds_ir(&mut self, imm: Imm12, rn: Reg, rd: WritableReg, size: OperandSize) { 388 self.alu_rri(ALUOp::AddS, imm, rn, rd, size); 389 } 390 391 /// Add with three registers. 392 pub fn add_rrr(&mut self, rm: Reg, rn: Reg, rd: WritableReg, size: OperandSize) { 393 self.alu_rrr_extend(ALUOp::Add, rm, rn, rd, size); 394 } 395 396 /// Add with three registers, setting overflow flags. 397 pub fn adds_rrr(&mut self, rm: Reg, rn: Reg, rd: WritableReg, size: OperandSize) { 398 self.alu_rrr_extend(ALUOp::AddS, rm, rn, rd, size); 399 } 400 401 /// Add across Vector. 402 pub fn addv(&mut self, rn: Reg, rd: WritableReg, size: VectorSize) { 403 self.emit(Inst::VecLanes { 404 op: VecLanesOp::Addv, 405 rd: rd.map(Into::into), 406 rn: rn.into(), 407 size, 408 }); 409 } 410 411 /// Subtract immediate and register. 412 pub fn sub_ir(&mut self, imm: Imm12, rn: Reg, rd: WritableReg, size: OperandSize) { 413 self.alu_rri(ALUOp::Sub, imm, rn, rd, size); 414 } 415 416 /// Subtract immediate and register, setting flags. 417 pub fn subs_ir(&mut self, imm: Imm12, rn: Reg, size: OperandSize) { 418 self.alu_rri(ALUOp::SubS, imm, rn, writable!(regs::zero()), size); 419 } 420 421 /// Subtract with three registers. 422 pub fn sub_rrr(&mut self, rm: Reg, rn: Reg, rd: WritableReg, size: OperandSize) { 423 self.alu_rrr_extend(ALUOp::Sub, rm, rn, rd, size); 424 } 425 426 /// Subtract with three registers, setting flags. 427 pub fn subs_rrr(&mut self, rm: Reg, rn: Reg, size: OperandSize) { 428 self.alu_rrr_extend(ALUOp::SubS, rm, rn, writable!(regs::zero()), size); 429 } 430 431 /// Multiply with three registers. 432 pub fn mul_rrr(&mut self, rm: Reg, rn: Reg, rd: WritableReg, size: OperandSize) { 433 self.alu_rrrr(ALUOp3::MAdd, rm, rn, rd, regs::zero(), size); 434 } 435 436 /// Signed/unsigned division with three registers. 437 pub fn div_rrr( 438 &mut self, 439 divisor: Reg, 440 dividend: Reg, 441 dest: Writable<Reg>, 442 kind: DivKind, 443 size: OperandSize, 444 ) { 445 // Check for division by 0. 446 self.trapz(divisor, TrapCode::INTEGER_DIVISION_BY_ZERO, size); 447 448 // check for overflow 449 if kind == DivKind::Signed { 450 // Check for divisor overflow. 451 self.alu_rri( 452 ALUOp::AddS, 453 Imm12::maybe_from_u64(1).expect("1 to fit in 12 bits"), 454 divisor, 455 writable!(zero()), 456 size, 457 ); 458 459 // Check if the dividend is 1. 460 self.emit(Inst::CCmpImm { 461 size: size.into(), 462 rn: dividend.into(), 463 imm: UImm5::maybe_from_u8(1).expect("1 fits in 5 bits"), 464 nzcv: NZCV::new(false, false, false, false), 465 cond: Cond::Eq, 466 }); 467 468 // Finally, trap if the previous operation overflowed. 469 self.trapif(Cond::Vs, TrapCode::INTEGER_OVERFLOW); 470 } 471 472 // `cranelift-codegen` doesn't support emitting sdiv for anything but I64, 473 // we therefore sign-extend the operand. 474 // see: https://github.com/bytecodealliance/wasmtime/issues/9766 475 let size = if size == OperandSize::S32 && kind == DivKind::Signed { 476 self.extend( 477 divisor, 478 writable!(divisor), 479 ExtendKind::Signed(Extend::<Signed>::I64Extend32), 480 ); 481 self.extend( 482 dividend, 483 writable!(dividend), 484 ExtendKind::Signed(Extend::<Signed>::I64Extend32), 485 ); 486 OperandSize::S64 487 } else { 488 size 489 }; 490 491 let op = match kind { 492 DivKind::Signed => ALUOp::SDiv, 493 DivKind::Unsigned => ALUOp::UDiv, 494 }; 495 496 self.alu_rrr(op, divisor, dividend, dest.map(Into::into), size); 497 } 498 499 /// Signed/unsigned remainder operation with three registers. 500 pub fn rem_rrr( 501 &mut self, 502 divisor: Reg, 503 dividend: Reg, 504 dest: Writable<Reg>, 505 scratch: WritableReg, 506 kind: RemKind, 507 size: OperandSize, 508 ) { 509 // Check for division by 0 510 self.trapz(divisor, TrapCode::INTEGER_DIVISION_BY_ZERO, size); 511 512 // `cranelift-codegen` doesn't support emitting sdiv for anything but I64, 513 // we therefore sign-extend the operand. 514 // see: https://github.com/bytecodealliance/wasmtime/issues/9766 515 let size = if size == OperandSize::S32 && kind.is_signed() { 516 self.extend( 517 divisor, 518 writable!(divisor), 519 ExtendKind::Signed(Extend::<Signed>::I64Extend32), 520 ); 521 self.extend( 522 dividend, 523 writable!(dividend), 524 ExtendKind::Signed(Extend::<Signed>::I64Extend32), 525 ); 526 OperandSize::S64 527 } else { 528 size 529 }; 530 531 let op = match kind { 532 RemKind::Signed => ALUOp::SDiv, 533 RemKind::Unsigned => ALUOp::UDiv, 534 }; 535 536 self.alu_rrr(op, divisor, dividend, scratch, size); 537 538 self.alu_rrrr( 539 ALUOp3::MSub, 540 scratch.to_reg(), 541 divisor, 542 dest.map(Into::into), 543 dividend, 544 size, 545 ); 546 } 547 548 /// And with three registers. 549 pub fn and_rrr(&mut self, rm: Reg, rn: Reg, rd: WritableReg, size: OperandSize) { 550 self.alu_rrr(ALUOp::And, rm, rn, rd, size); 551 } 552 553 /// And immediate and register. 554 pub fn and_ir(&mut self, imm: ImmLogic, rn: Reg, rd: WritableReg, size: OperandSize) { 555 self.alu_rri_logic(ALUOp::And, imm, rn, rd, size); 556 } 557 558 /// Or with three registers. 559 pub fn or_rrr(&mut self, rm: Reg, rn: Reg, rd: WritableReg, size: OperandSize) { 560 self.alu_rrr(ALUOp::Orr, rm, rn, rd, size); 561 } 562 563 /// Or immediate and register. 564 pub fn or_ir(&mut self, imm: ImmLogic, rn: Reg, rd: WritableReg, size: OperandSize) { 565 self.alu_rri_logic(ALUOp::Orr, imm, rn, rd, size); 566 } 567 568 /// Xor with three registers. 569 pub fn xor_rrr(&mut self, rm: Reg, rn: Reg, rd: WritableReg, size: OperandSize) { 570 self.alu_rrr(ALUOp::Eor, rm, rn, rd, size); 571 } 572 573 /// Xor immediate and register. 574 pub fn xor_ir(&mut self, imm: ImmLogic, rn: Reg, rd: WritableReg, size: OperandSize) { 575 self.alu_rri_logic(ALUOp::Eor, imm, rn, rd, size); 576 } 577 578 /// Shift with three registers. 579 pub fn shift_rrr( 580 &mut self, 581 rm: Reg, 582 rn: Reg, 583 rd: WritableReg, 584 kind: ShiftKind, 585 size: OperandSize, 586 ) { 587 let shift_op: ALUOp = kind.into(); 588 // In the case of rotate left, we negate the register containing the 589 // shift value. 590 if kind == ShiftKind::Rotl { 591 self.alu_rrr(ALUOp::Sub, rm, regs::zero(), writable!(rm), size); 592 self.alu_rrr(shift_op, rm, rn, rd, size); 593 } else { 594 self.alu_rrr(shift_op, rm, rn, rd, size); 595 } 596 } 597 598 /// Shift immediate and register. 599 pub fn shift_ir( 600 &mut self, 601 imm: ImmShift, 602 rn: Reg, 603 rd: WritableReg, 604 kind: ShiftKind, 605 size: OperandSize, 606 ) { 607 let shift_op: ALUOp = kind.into(); 608 // In the case of rotate left, we emit rotate right with type_size - 609 // value. 610 if kind == ShiftKind::Rotl { 611 let value_size = size.num_bits(); 612 let mut imm_val = value_size.wrapping_sub(imm.value()); 613 imm_val &= value_size - 1; 614 let negated_imm = ImmShift::maybe_from_u64(imm_val as u64).unwrap(); 615 616 self.alu_rri_shift(shift_op, negated_imm, rn, rd, size); 617 } else { 618 self.alu_rri_shift(shift_op, imm, rn, rd, size); 619 } 620 } 621 622 /// Count Leading Zeros. 623 pub fn clz(&mut self, rn: Reg, rd: WritableReg, size: OperandSize) { 624 self.bit_rr(BitOp::Clz, rn, rd, size); 625 } 626 627 /// Reverse Bits reverses the bit order in a register. 628 pub fn rbit(&mut self, rn: Reg, rd: WritableReg, size: OperandSize) { 629 self.bit_rr(BitOp::RBit, rn, rd, size); 630 } 631 632 /// Float add with three registers. 633 pub fn fadd_rrr(&mut self, rm: Reg, rn: Reg, rd: WritableReg, size: OperandSize) { 634 self.fpu_rrr(FPUOp2::Add, rm, rn, rd, size); 635 } 636 637 /// Float sub with three registers. 638 pub fn fsub_rrr(&mut self, rm: Reg, rn: Reg, rd: WritableReg, size: OperandSize) { 639 self.fpu_rrr(FPUOp2::Sub, rm, rn, rd, size); 640 } 641 642 /// Float multiply with three registers. 643 pub fn fmul_rrr(&mut self, rm: Reg, rn: Reg, rd: WritableReg, size: OperandSize) { 644 self.fpu_rrr(FPUOp2::Mul, rm, rn, rd, size); 645 } 646 647 /// Float division with three registers. 648 pub fn fdiv_rrr(&mut self, rm: Reg, rn: Reg, rd: WritableReg, size: OperandSize) { 649 self.fpu_rrr(FPUOp2::Div, rm, rn, rd, size); 650 } 651 652 /// Float max with three registers. 653 pub fn fmax_rrr(&mut self, rm: Reg, rn: Reg, rd: WritableReg, size: OperandSize) { 654 self.fpu_rrr(FPUOp2::Max, rm, rn, rd, size); 655 } 656 657 /// Float min with three registers. 658 pub fn fmin_rrr(&mut self, rm: Reg, rn: Reg, rd: WritableReg, size: OperandSize) { 659 self.fpu_rrr(FPUOp2::Min, rm, rn, rd, size); 660 } 661 662 /// Float neg with two registers. 663 pub fn fneg_rr(&mut self, rn: Reg, rd: WritableReg, size: OperandSize) { 664 self.fpu_rr(FPUOp1::Neg, rn, rd, size); 665 } 666 667 /// Float abs with two registers. 668 pub fn fabs_rr(&mut self, rn: Reg, rd: WritableReg, size: OperandSize) { 669 self.fpu_rr(FPUOp1::Abs, rn, rd, size); 670 } 671 672 /// Float sqrt with two registers. 673 pub fn fsqrt_rr(&mut self, rn: Reg, rd: WritableReg, size: OperandSize) { 674 self.fpu_rr(FPUOp1::Sqrt, rn, rd, size); 675 } 676 677 /// Float round (ceil, trunc, floor) with two registers. 678 pub fn fround_rr(&mut self, rn: Reg, rd: WritableReg, mode: RoundingMode, size: OperandSize) { 679 let fpu_mode = match (mode, size) { 680 (RoundingMode::Nearest, OperandSize::S32) => FpuRoundMode::Nearest32, 681 (RoundingMode::Up, OperandSize::S32) => FpuRoundMode::Plus32, 682 (RoundingMode::Down, OperandSize::S32) => FpuRoundMode::Minus32, 683 (RoundingMode::Zero, OperandSize::S32) => FpuRoundMode::Zero32, 684 (RoundingMode::Nearest, OperandSize::S64) => FpuRoundMode::Nearest64, 685 (RoundingMode::Up, OperandSize::S64) => FpuRoundMode::Plus64, 686 (RoundingMode::Down, OperandSize::S64) => FpuRoundMode::Minus64, 687 (RoundingMode::Zero, OperandSize::S64) => FpuRoundMode::Zero64, 688 (m, o) => panic!("Invalid rounding mode or operand size {m:?}, {o:?}"), 689 }; 690 self.fpu_round(fpu_mode, rn, rd) 691 } 692 693 /// Float unsigned shift right with two registers and an immediate. 694 pub fn fushr_rri(&mut self, rn: Reg, rd: WritableReg, amount: u8, size: OperandSize) { 695 let imm = FPURightShiftImm { 696 amount, 697 lane_size_in_bits: size.num_bits(), 698 }; 699 let ushr = match size { 700 OperandSize::S32 => UShr32(imm), 701 OperandSize::S64 => UShr64(imm), 702 _ => unreachable!(), 703 }; 704 self.fpu_rri(ushr, rn, rd) 705 } 706 707 /// Float unsigned shift left and insert with three registers 708 /// and an immediate. 709 pub fn fsli_rri_mod( 710 &mut self, 711 ri: Reg, 712 rn: Reg, 713 rd: WritableReg, 714 amount: u8, 715 size: OperandSize, 716 ) { 717 let imm = FPULeftShiftImm { 718 amount, 719 lane_size_in_bits: size.num_bits(), 720 }; 721 let sli = match size { 722 OperandSize::S32 => FPUOpRIMod::Sli32(imm), 723 OperandSize::S64 => FPUOpRIMod::Sli64(imm), 724 _ => unreachable!(), 725 }; 726 self.fpu_rri_mod(sli, ri, rn, rd) 727 } 728 729 /// Float compare. 730 pub fn fcmp(&mut self, rn: Reg, rm: Reg, size: OperandSize) { 731 self.emit(Inst::FpuCmp { 732 size: size.into(), 733 rn: rn.into(), 734 rm: rm.into(), 735 }) 736 } 737 738 /// Convert an signed integer to a float. 739 pub fn cvt_sint_to_float( 740 &mut self, 741 rn: Reg, 742 rd: WritableReg, 743 src_size: OperandSize, 744 dst_size: OperandSize, 745 ) { 746 let op = match (src_size, dst_size) { 747 (OperandSize::S32, OperandSize::S32) => IntToFpuOp::I32ToF32, 748 (OperandSize::S64, OperandSize::S32) => IntToFpuOp::I64ToF32, 749 (OperandSize::S32, OperandSize::S64) => IntToFpuOp::I32ToF64, 750 (OperandSize::S64, OperandSize::S64) => IntToFpuOp::I64ToF64, 751 _ => unreachable!(), 752 }; 753 754 self.emit(Inst::IntToFpu { 755 op, 756 rd: rd.map(Into::into), 757 rn: rn.into(), 758 }); 759 } 760 761 /// Convert an unsigned integer to a float. 762 pub fn cvt_uint_to_float( 763 &mut self, 764 rn: Reg, 765 rd: WritableReg, 766 src_size: OperandSize, 767 dst_size: OperandSize, 768 ) { 769 let op = match (src_size, dst_size) { 770 (OperandSize::S32, OperandSize::S32) => IntToFpuOp::U32ToF32, 771 (OperandSize::S64, OperandSize::S32) => IntToFpuOp::U64ToF32, 772 (OperandSize::S32, OperandSize::S64) => IntToFpuOp::U32ToF64, 773 (OperandSize::S64, OperandSize::S64) => IntToFpuOp::U64ToF64, 774 _ => unreachable!(), 775 }; 776 777 self.emit(Inst::IntToFpu { 778 op, 779 rd: rd.map(Into::into), 780 rn: rn.into(), 781 }); 782 } 783 784 /// Change precision of float. 785 pub fn cvt_float_to_float( 786 &mut self, 787 rn: Reg, 788 rd: WritableReg, 789 src_size: OperandSize, 790 dst_size: OperandSize, 791 ) { 792 let (fpu_op, size) = match (src_size, dst_size) { 793 (OperandSize::S32, OperandSize::S64) => (FPUOp1::Cvt32To64, ScalarSize::Size32), 794 (OperandSize::S64, OperandSize::S32) => (FPUOp1::Cvt64To32, ScalarSize::Size64), 795 _ => unimplemented!(), 796 }; 797 self.emit(Inst::FpuRR { 798 fpu_op, 799 size, 800 rd: rd.map(Into::into), 801 rn: rn.into(), 802 }); 803 } 804 805 /// Return instruction. 806 pub fn ret(&mut self) { 807 self.emit(Inst::Ret {}); 808 } 809 810 /// An unconditional branch. 811 pub fn jmp(&mut self, target: MachLabel) { 812 self.emit(Inst::Jump { 813 dest: BranchTarget::Label(target), 814 }); 815 } 816 817 /// A conditional branch. 818 pub fn jmp_if(&mut self, kind: Cond, taken: MachLabel) { 819 self.emit(Inst::CondBr { 820 taken: BranchTarget::Label(taken), 821 not_taken: BranchTarget::ResolvedOffset(4), 822 kind: CondBrKind::Cond(kind), 823 }); 824 } 825 826 /// Emits a jump table sequence. 827 pub fn jmp_table( 828 &mut self, 829 targets: &[MachLabel], 830 default: MachLabel, 831 index: Reg, 832 tmp1: Reg, 833 tmp2: Reg, 834 ) { 835 self.emit_with_island( 836 Inst::JTSequence { 837 default, 838 targets: Box::new(targets.to_vec()), 839 ridx: index.into(), 840 rtmp1: Writable::from_reg(tmp1.into()), 841 rtmp2: Writable::from_reg(tmp2.into()), 842 }, 843 // number of bytes needed for the jumptable sequence: 844 // 4 bytes per instruction, with 8 instructions base + the size of 845 // the jumptable more. 846 (4 * (8 + targets.len())).try_into().unwrap(), 847 ); 848 } 849 850 /// Conditional Set sets the destination register to 1 if the condition 851 /// is true, and otherwise sets it to 0. 852 pub fn cset(&mut self, rd: WritableReg, cond: Cond) { 853 self.emit(Inst::CSet { 854 rd: rd.map(Into::into), 855 cond, 856 }); 857 } 858 859 /// If the condition is true, `csel` writes rn to rd. If the 860 /// condition is false, it writes rm to rd 861 pub fn csel(&mut self, rn: Reg, rm: Reg, rd: WritableReg, cond: Cond) { 862 self.emit(Inst::CSel { 863 rd: rd.map(Into::into), 864 rn: rn.into(), 865 rm: rm.into(), 866 cond, 867 }); 868 } 869 870 /// If the condition is true, `csel` writes rn to rd. If the 871 /// condition is false, it writes rm to rd 872 pub fn fpu_csel(&mut self, rn: Reg, rm: Reg, rd: WritableReg, cond: Cond, size: OperandSize) { 873 match size { 874 OperandSize::S32 => { 875 self.emit(Inst::FpuCSel32 { 876 rd: rd.map(Into::into), 877 rn: rn.into(), 878 rm: rm.into(), 879 cond, 880 }); 881 } 882 OperandSize::S64 => { 883 self.emit(Inst::FpuCSel64 { 884 rd: rd.map(Into::into), 885 rn: rn.into(), 886 rm: rm.into(), 887 cond, 888 }); 889 } 890 _ => todo!(), 891 } 892 } 893 894 /// Population count per byte. 895 pub fn cnt(&mut self, rd: WritableReg) { 896 self.emit(Inst::VecMisc { 897 op: VecMisc2::Cnt, 898 rd: rd.map(Into::into), 899 rn: rd.to_reg().into(), 900 size: VectorSize::Size8x8, 901 }); 902 } 903 904 pub fn extend(&mut self, rn: Reg, rd: WritableReg, kind: ExtendKind) { 905 self.emit(Inst::Extend { 906 rd: rd.map(Into::into), 907 rn: rn.into(), 908 signed: kind.signed(), 909 from_bits: kind.from_bits(), 910 to_bits: kind.to_bits(), 911 }) 912 } 913 914 /// Bitwise AND (shifted register), setting flags. 915 pub fn ands_rr(&mut self, rn: Reg, rm: Reg, size: OperandSize) { 916 self.alu_rrr(ALUOp::AndS, rm, rn, writable!(regs::zero()), size); 917 } 918 919 /// Permanently Undefined. 920 pub fn udf(&mut self, code: TrapCode) { 921 self.emit(Inst::Udf { trap_code: code }); 922 } 923 924 /// Conditional trap. 925 pub fn trapif(&mut self, cc: Cond, code: TrapCode) { 926 self.emit(Inst::TrapIf { 927 kind: CondBrKind::Cond(cc), 928 trap_code: code, 929 }); 930 } 931 932 /// Trap if `rn` is zero. 933 pub fn trapz(&mut self, rn: Reg, code: TrapCode, size: OperandSize) { 934 self.emit(Inst::TrapIf { 935 kind: CondBrKind::Zero(rn.into(), size.into()), 936 trap_code: code, 937 }); 938 } 939 940 // Helpers for ALU operations. 941 942 fn alu_rri(&mut self, op: ALUOp, imm: Imm12, rn: Reg, rd: WritableReg, size: OperandSize) { 943 self.emit(Inst::AluRRImm12 { 944 alu_op: op, 945 size: size.into(), 946 rd: rd.map(Into::into), 947 rn: rn.into(), 948 imm12: imm, 949 }); 950 } 951 952 fn alu_rri_logic( 953 &mut self, 954 op: ALUOp, 955 imm: ImmLogic, 956 rn: Reg, 957 rd: WritableReg, 958 size: OperandSize, 959 ) { 960 self.emit(Inst::AluRRImmLogic { 961 alu_op: op, 962 size: size.into(), 963 rd: rd.map(Into::into), 964 rn: rn.into(), 965 imml: imm, 966 }); 967 } 968 969 fn alu_rri_shift( 970 &mut self, 971 op: ALUOp, 972 imm: ImmShift, 973 rn: Reg, 974 rd: WritableReg, 975 size: OperandSize, 976 ) { 977 self.emit(Inst::AluRRImmShift { 978 alu_op: op, 979 size: size.into(), 980 rd: rd.map(Into::into), 981 rn: rn.into(), 982 immshift: imm, 983 }); 984 } 985 986 fn alu_rrr(&mut self, op: ALUOp, rm: Reg, rn: Reg, rd: WritableReg, size: OperandSize) { 987 self.emit(Inst::AluRRR { 988 alu_op: op, 989 size: size.into(), 990 rd: rd.map(Into::into), 991 rn: rn.into(), 992 rm: rm.into(), 993 }); 994 } 995 996 fn alu_rrr_extend(&mut self, op: ALUOp, rm: Reg, rn: Reg, rd: WritableReg, size: OperandSize) { 997 self.emit(Inst::AluRRRExtend { 998 alu_op: op, 999 size: size.into(), 1000 rd: rd.map(Into::into), 1001 rn: rn.into(), 1002 rm: rm.into(), 1003 extendop: ExtendOp::UXTX, 1004 }); 1005 } 1006 1007 fn alu_rrrr( 1008 &mut self, 1009 op: ALUOp3, 1010 rm: Reg, 1011 rn: Reg, 1012 rd: WritableReg, 1013 ra: Reg, 1014 size: OperandSize, 1015 ) { 1016 self.emit(Inst::AluRRRR { 1017 alu_op: op, 1018 size: size.into(), 1019 rd: rd.map(Into::into), 1020 rn: rn.into(), 1021 rm: rm.into(), 1022 ra: ra.into(), 1023 }); 1024 } 1025 1026 fn fpu_rrr(&mut self, op: FPUOp2, rm: Reg, rn: Reg, rd: WritableReg, size: OperandSize) { 1027 self.emit(Inst::FpuRRR { 1028 fpu_op: op, 1029 size: size.into(), 1030 rd: rd.map(Into::into), 1031 rn: rn.into(), 1032 rm: rm.into(), 1033 }); 1034 } 1035 1036 fn fpu_rri(&mut self, op: FPUOpRI, rn: Reg, rd: WritableReg) { 1037 self.emit(Inst::FpuRRI { 1038 fpu_op: op, 1039 rd: rd.map(Into::into), 1040 rn: rn.into(), 1041 }); 1042 } 1043 1044 fn fpu_rri_mod(&mut self, op: FPUOpRIMod, ri: Reg, rn: Reg, rd: WritableReg) { 1045 self.emit(Inst::FpuRRIMod { 1046 fpu_op: op, 1047 rd: rd.map(Into::into), 1048 ri: ri.into(), 1049 rn: rn.into(), 1050 }); 1051 } 1052 1053 fn fpu_rr(&mut self, op: FPUOp1, rn: Reg, rd: WritableReg, size: OperandSize) { 1054 self.emit(Inst::FpuRR { 1055 fpu_op: op, 1056 size: size.into(), 1057 rd: rd.map(Into::into), 1058 rn: rn.into(), 1059 }); 1060 } 1061 1062 fn fpu_round(&mut self, op: FpuRoundMode, rn: Reg, rd: WritableReg) { 1063 self.emit(Inst::FpuRound { 1064 op, 1065 rd: rd.map(Into::into), 1066 rn: rn.into(), 1067 }); 1068 } 1069 1070 fn bit_rr(&mut self, op: BitOp, rn: Reg, rd: WritableReg, size: OperandSize) { 1071 self.emit(Inst::BitRR { 1072 op, 1073 size: size.into(), 1074 rd: rd.map(Into::into), 1075 rn: rn.into(), 1076 }); 1077 } 1078 1079 /// Get a label from the underlying machine code buffer. 1080 pub fn get_label(&mut self) -> MachLabel { 1081 self.buffer.get_label() 1082 } 1083 1084 /// Get a mutable reference to underlying 1085 /// machine buffer. 1086 pub fn buffer_mut(&mut self) -> &mut MachBuffer<Inst> { 1087 &mut self.buffer 1088 } 1089 1090 /// Get a reference to the underlying machine buffer. 1091 pub fn buffer(&self) -> &MachBuffer<Inst> { 1092 &self.buffer 1093 } 1094 1095 /// Emit a direct call to a function defined locally and 1096 /// referenced to by `name`. 1097 pub fn call_with_name(&mut self, name: UserExternalNameRef, call_conv: CallingConvention) { 1098 self.emit(Inst::Call { 1099 info: Box::new(cranelift_codegen::CallInfo::empty( 1100 ExternalName::user(name), 1101 call_conv.into(), 1102 )), 1103 }) 1104 } 1105 1106 /// Emit an indirect call to a function whose address is 1107 /// stored the `callee` register. 1108 pub fn call_with_reg(&mut self, callee: Reg, call_conv: CallingConvention) { 1109 self.emit(Inst::CallInd { 1110 info: Box::new(cranelift_codegen::CallInfo::empty( 1111 callee.into(), 1112 call_conv.into(), 1113 )), 1114 }) 1115 } 1116 1117 /// Load the min value for an integer of size out_size, as a floating-point 1118 /// of size `in-size`, into register `rd`. 1119 fn min_fp_value( 1120 &mut self, 1121 signed: bool, 1122 in_size: OperandSize, 1123 out_size: OperandSize, 1124 rd: Writable<Reg>, 1125 ) { 1126 match in_size { 1127 OperandSize::S32 => { 1128 let (min, _) = f32_cvt_to_int_bounds(signed, out_size.num_bits().into()); 1129 self.mov_ir(rd, Imm::f32(min.to_bits()), in_size); 1130 } 1131 OperandSize::S64 => { 1132 let (min, _) = f64_cvt_to_int_bounds(signed, out_size.num_bits().into()); 1133 self.mov_ir(rd, Imm::f64(min.to_bits()), in_size); 1134 } 1135 s => unreachable!("unsupported floating-point size: {}bit", s.num_bits()), 1136 }; 1137 } 1138 1139 /// Load the max value for an integer of size out_size, as a floating-point 1140 /// of size `in_size`, into register `rd`. 1141 fn max_fp_value( 1142 &mut self, 1143 signed: bool, 1144 in_size: OperandSize, 1145 out_size: OperandSize, 1146 rd: Writable<Reg>, 1147 ) { 1148 match in_size { 1149 OperandSize::S32 => { 1150 let (_, max) = f32_cvt_to_int_bounds(signed, out_size.num_bits().into()); 1151 self.mov_ir(rd, Imm::f32(max.to_bits()), in_size); 1152 } 1153 OperandSize::S64 => { 1154 let (_, max) = f64_cvt_to_int_bounds(signed, out_size.num_bits().into()); 1155 self.mov_ir(rd, Imm::f64(max.to_bits()), in_size); 1156 } 1157 s => unreachable!("unsupported floating-point size: {}bit", s.num_bits()), 1158 }; 1159 } 1160 1161 /// Emit instructions to check if the value in `rn` is NaN. 1162 fn check_nan(&mut self, rn: Reg, size: OperandSize) { 1163 self.fcmp(rn, rn, size); 1164 self.trapif(Cond::Vs, TrapCode::BAD_CONVERSION_TO_INTEGER); 1165 } 1166 1167 /// Convert the floating point of size `src_size` stored in `src`, into a integer of size 1168 /// `dst_size`, storing the result in `dst`. 1169 pub fn fpu_to_int( 1170 &mut self, 1171 dst: Writable<Reg>, 1172 src: Reg, 1173 tmp_reg: WritableReg, 1174 src_size: OperandSize, 1175 dst_size: OperandSize, 1176 kind: TruncKind, 1177 signed: bool, 1178 ) { 1179 if kind.is_unchecked() { 1180 // Confusingly, when `kind` is `Unchecked` is when we actually need to perform the checks: 1181 // - check if fp is NaN 1182 // - check bounds 1183 self.check_nan(src, src_size); 1184 1185 self.min_fp_value(signed, src_size, dst_size, tmp_reg); 1186 self.fcmp(src, tmp_reg.to_reg(), src_size); 1187 self.trapif(Cond::Le, TrapCode::INTEGER_OVERFLOW); 1188 1189 self.max_fp_value(signed, src_size, dst_size, tmp_reg); 1190 self.fcmp(src, tmp_reg.to_reg(), src_size); 1191 self.trapif(Cond::Ge, TrapCode::INTEGER_OVERFLOW); 1192 } 1193 1194 self.cvt_fpu_to_int(dst, src, src_size, dst_size, signed) 1195 } 1196 1197 /// Select and emit the appropriate `fcvt*` instruction 1198 pub fn cvt_fpu_to_int( 1199 &mut self, 1200 dst: Writable<Reg>, 1201 src: Reg, 1202 src_size: OperandSize, 1203 dst_size: OperandSize, 1204 signed: bool, 1205 ) { 1206 let op = match (src_size, dst_size, signed) { 1207 (OperandSize::S32, OperandSize::S32, false) => FpuToIntOp::F32ToU32, 1208 (OperandSize::S32, OperandSize::S32, true) => FpuToIntOp::F32ToI32, 1209 (OperandSize::S32, OperandSize::S64, false) => FpuToIntOp::F32ToU64, 1210 (OperandSize::S32, OperandSize::S64, true) => FpuToIntOp::F32ToI64, 1211 (OperandSize::S64, OperandSize::S32, false) => FpuToIntOp::F64ToU32, 1212 (OperandSize::S64, OperandSize::S32, true) => FpuToIntOp::F64ToI32, 1213 (OperandSize::S64, OperandSize::S64, false) => FpuToIntOp::F64ToU64, 1214 (OperandSize::S64, OperandSize::S64, true) => FpuToIntOp::F64ToI64, 1215 (fsize, int_size, signed) => unimplemented!( 1216 "unsupported conversion: f{} to {}{}", 1217 fsize.num_bits(), 1218 if signed { "i" } else { "u" }, 1219 int_size.num_bits(), 1220 ), 1221 }; 1222 1223 self.emit(Inst::FpuToInt { 1224 op, 1225 rd: dst.map(Into::into), 1226 rn: src.into(), 1227 }); 1228 } 1229 } 1230 1231 /// Captures the region in a MachBuffer where an add-with-immediate instruction would be emitted, 1232 /// but the immediate is not yet known. 1233 pub(crate) struct PatchableAddToReg { 1234 /// The region to be patched in the [`MachBuffer`]. It contains 1235 /// space for 3 32-bit instructions, i.e. it's 12 bytes long. 1236 region: PatchRegion, 1237 1238 // The destination register for the add instruction. 1239 reg: Writable<Reg>, 1240 1241 // The temporary register used to hold the immediate value. 1242 tmp: Writable<Reg>, 1243 } 1244 1245 impl PatchableAddToReg { 1246 /// Create a new [`PatchableAddToReg`] by capturing a region in the output 1247 /// buffer containing an instruction sequence that loads an immediate into a 1248 /// register `tmp`, then adds it to a register `reg`. The [`MachBuffer`] 1249 /// will have that instruction sequence written to the region, though the 1250 /// immediate loaded into `tmp` will be `0` until the `::finalize` method is 1251 /// called. 1252 pub(crate) fn new(reg: Writable<Reg>, tmp: Writable<Reg>, buf: &mut MachBuffer<Inst>) -> Self { 1253 let insns = Self::add_immediate_instruction_sequence(reg, tmp, 0); 1254 let open = buf.start_patchable(); 1255 buf.put_data(&insns); 1256 let region = buf.end_patchable(open); 1257 1258 Self { region, reg, tmp } 1259 } 1260 1261 fn add_immediate_instruction_sequence( 1262 reg: Writable<Reg>, 1263 tmp: Writable<Reg>, 1264 imm: i32, 1265 ) -> [u8; 12] { 1266 let imm_hi = imm as u64 & 0xffff_0000; 1267 let imm_hi = MoveWideConst::maybe_from_u64(imm_hi).unwrap(); 1268 1269 let imm_lo = imm as u64 & 0x0000_ffff; 1270 let imm_lo = MoveWideConst::maybe_from_u64(imm_lo).unwrap(); 1271 1272 let size = OperandSize::S64.into(); 1273 1274 let tmp = tmp.map(Into::into); 1275 let rd = reg.map(Into::into); 1276 1277 // This is "movz to bits 16-31 of 64 bit reg tmp and zero the rest" 1278 let mov_insn = enc_move_wide(inst::MoveWideOp::MovZ, tmp, imm_hi, size); 1279 1280 // This is "movk to bits 0-15 of 64 bit reg tmp" 1281 let movk_insn = enc_movk(tmp, imm_lo, size); 1282 1283 // This is "add tmp to rd". The opcodes are somewhat buried in the 1284 // instruction encoder so we just repeat them here. 1285 let add_bits_31_21: u32 = 0b00001011_000 | (size.sf_bit() << 10); 1286 let add_bits_15_10: u32 = 0; 1287 let add_insn = enc_arith_rrr( 1288 add_bits_31_21, 1289 add_bits_15_10, 1290 rd, 1291 rd.to_reg(), 1292 tmp.to_reg(), 1293 ); 1294 1295 let mut buf = [0u8; 12]; 1296 buf[0..4].copy_from_slice(&mov_insn.to_le_bytes()); 1297 buf[4..8].copy_from_slice(&movk_insn.to_le_bytes()); 1298 buf[8..12].copy_from_slice(&add_insn.to_le_bytes()); 1299 buf 1300 } 1301 1302 /// Patch the [`MachBuffer`] with the known constant to be added to the register. The final 1303 /// value is passed in as an i32, but the instruction encoding is fixed when 1304 /// [`PatchableAddToReg::new`] is called. 1305 pub(crate) fn finalize(self, val: i32, buffer: &mut MachBuffer<Inst>) { 1306 let insns = Self::add_immediate_instruction_sequence(self.reg, self.tmp, val); 1307 let slice = self.region.patch(buffer); 1308 assert_eq!(slice.len(), insns.len()); 1309 slice.copy_from_slice(&insns); 1310 } 1311 } 1312