1 //! This module is the central place for machine code emission. 2 //! It defines an implementation of wasmparser's Visitor trait 3 //! for `CodeGen`; which defines a visitor per op-code, 4 //! which validates and dispatches to the corresponding 5 //! machine code emitter. 6 7 use crate::abi::RetArea; 8 use crate::codegen::{control_index, Callee, CodeGen, ControlStackFrame, FnCall}; 9 use crate::masm::{ 10 DivKind, ExtendKind, FloatCmpKind, IntCmpKind, MacroAssembler, MemMoveDirection, MulWideKind, 11 OperandSize, RegImm, RemKind, RoundingMode, SPOffset, ShiftKind, TruncKind, 12 }; 13 use crate::reg::{writable, Reg}; 14 use crate::stack::{TypedReg, Val}; 15 use regalloc2::RegClass; 16 use smallvec::SmallVec; 17 use wasmparser::{BlockType, BrTable, Ieee32, Ieee64, MemArg, VisitOperator, V128}; 18 use wasmtime_cranelift::TRAP_INDIRECT_CALL_TO_NULL; 19 use wasmtime_environ::{ 20 FuncIndex, GlobalIndex, MemoryIndex, TableIndex, TypeIndex, WasmHeapType, WasmValType, 21 FUNCREF_INIT_BIT, 22 }; 23 24 /// A macro to define unsupported WebAssembly operators. 25 /// 26 /// This macro calls itself recursively; 27 /// 1. It no-ops when matching a supported operator. 28 /// 2. Defines the visitor function and panics when 29 /// matching an unsupported operator. 30 macro_rules! def_unsupported { 31 ($( @$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident $ann:tt)*) => { 32 $( 33 def_unsupported!( 34 emit 35 $op 36 37 fn $visit(&mut self $($(,$arg: $argty)*)?) -> Self::Output { 38 $($(let _ = $arg;)*)? 39 40 self.found_unsupported_instruction = Some(stringify!($op)); 41 } 42 ); 43 )* 44 }; 45 46 (emit I32Const $($rest:tt)*) => {}; 47 (emit I64Const $($rest:tt)*) => {}; 48 (emit F32Const $($rest:tt)*) => {}; 49 (emit F64Const $($rest:tt)*) => {}; 50 (emit V128Const $($rest:tt)*) => {}; 51 (emit F32Add $($rest:tt)*) => {}; 52 (emit F64Add $($rest:tt)*) => {}; 53 (emit F32Sub $($rest:tt)*) => {}; 54 (emit F64Sub $($rest:tt)*) => {}; 55 (emit F32Mul $($rest:tt)*) => {}; 56 (emit F64Mul $($rest:tt)*) => {}; 57 (emit F32Div $($rest:tt)*) => {}; 58 (emit F64Div $($rest:tt)*) => {}; 59 (emit F32Min $($rest:tt)*) => {}; 60 (emit F64Min $($rest:tt)*) => {}; 61 (emit F32Max $($rest:tt)*) => {}; 62 (emit F64Max $($rest:tt)*) => {}; 63 (emit F32Copysign $($rest:tt)*) => {}; 64 (emit F64Copysign $($rest:tt)*) => {}; 65 (emit F32Abs $($rest:tt)*) => {}; 66 (emit F64Abs $($rest:tt)*) => {}; 67 (emit F32Neg $($rest:tt)*) => {}; 68 (emit F64Neg $($rest:tt)*) => {}; 69 (emit F32Floor $($rest:tt)*) => {}; 70 (emit F64Floor $($rest:tt)*) => {}; 71 (emit F32Ceil $($rest:tt)*) => {}; 72 (emit F64Ceil $($rest:tt)*) => {}; 73 (emit F32Nearest $($rest:tt)*) => {}; 74 (emit F64Nearest $($rest:tt)*) => {}; 75 (emit F32Trunc $($rest:tt)*) => {}; 76 (emit F64Trunc $($rest:tt)*) => {}; 77 (emit F32Sqrt $($rest:tt)*) => {}; 78 (emit F64Sqrt $($rest:tt)*) => {}; 79 (emit F32Eq $($rest:tt)*) => {}; 80 (emit F64Eq $($rest:tt)*) => {}; 81 (emit F32Ne $($rest:tt)*) => {}; 82 (emit F64Ne $($rest:tt)*) => {}; 83 (emit F32Lt $($rest:tt)*) => {}; 84 (emit F64Lt $($rest:tt)*) => {}; 85 (emit F32Gt $($rest:tt)*) => {}; 86 (emit F64Gt $($rest:tt)*) => {}; 87 (emit F32Le $($rest:tt)*) => {}; 88 (emit F64Le $($rest:tt)*) => {}; 89 (emit F32Ge $($rest:tt)*) => {}; 90 (emit F64Ge $($rest:tt)*) => {}; 91 (emit F32ConvertI32S $($rest:tt)*) => {}; 92 (emit F32ConvertI32U $($rest:tt)*) => {}; 93 (emit F32ConvertI64S $($rest:tt)*) => {}; 94 (emit F32ConvertI64U $($rest:tt)*) => {}; 95 (emit F64ConvertI32S $($rest:tt)*) => {}; 96 (emit F64ConvertI32U $($rest:tt)*) => {}; 97 (emit F64ConvertI64S $($rest:tt)*) => {}; 98 (emit F64ConvertI64U $($rest:tt)*) => {}; 99 (emit F32ReinterpretI32 $($rest:tt)*) => {}; 100 (emit F64ReinterpretI64 $($rest:tt)*) => {}; 101 (emit F32DemoteF64 $($rest:tt)*) => {}; 102 (emit F64PromoteF32 $($rest:tt)*) => {}; 103 (emit I32Add $($rest:tt)*) => {}; 104 (emit I64Add $($rest:tt)*) => {}; 105 (emit I32Sub $($rest:tt)*) => {}; 106 (emit I32Mul $($rest:tt)*) => {}; 107 (emit I32DivS $($rest:tt)*) => {}; 108 (emit I32DivU $($rest:tt)*) => {}; 109 (emit I64DivS $($rest:tt)*) => {}; 110 (emit I64DivU $($rest:tt)*) => {}; 111 (emit I64RemU $($rest:tt)*) => {}; 112 (emit I64RemS $($rest:tt)*) => {}; 113 (emit I32RemU $($rest:tt)*) => {}; 114 (emit I32RemS $($rest:tt)*) => {}; 115 (emit I64Mul $($rest:tt)*) => {}; 116 (emit I64Sub $($rest:tt)*) => {}; 117 (emit I32Eq $($rest:tt)*) => {}; 118 (emit I64Eq $($rest:tt)*) => {}; 119 (emit I32Ne $($rest:tt)*) => {}; 120 (emit I64Ne $($rest:tt)*) => {}; 121 (emit I32LtS $($rest:tt)*) => {}; 122 (emit I64LtS $($rest:tt)*) => {}; 123 (emit I32LtU $($rest:tt)*) => {}; 124 (emit I64LtU $($rest:tt)*) => {}; 125 (emit I32LeS $($rest:tt)*) => {}; 126 (emit I64LeS $($rest:tt)*) => {}; 127 (emit I32LeU $($rest:tt)*) => {}; 128 (emit I64LeU $($rest:tt)*) => {}; 129 (emit I32GtS $($rest:tt)*) => {}; 130 (emit I64GtS $($rest:tt)*) => {}; 131 (emit I32GtU $($rest:tt)*) => {}; 132 (emit I64GtU $($rest:tt)*) => {}; 133 (emit I32GeS $($rest:tt)*) => {}; 134 (emit I64GeS $($rest:tt)*) => {}; 135 (emit I32GeU $($rest:tt)*) => {}; 136 (emit I64GeU $($rest:tt)*) => {}; 137 (emit I32Eqz $($rest:tt)*) => {}; 138 (emit I64Eqz $($rest:tt)*) => {}; 139 (emit I32And $($rest:tt)*) => {}; 140 (emit I64And $($rest:tt)*) => {}; 141 (emit I32Or $($rest:tt)*) => {}; 142 (emit I64Or $($rest:tt)*) => {}; 143 (emit I32Xor $($rest:tt)*) => {}; 144 (emit I64Xor $($rest:tt)*) => {}; 145 (emit I32Shl $($rest:tt)*) => {}; 146 (emit I64Shl $($rest:tt)*) => {}; 147 (emit I32ShrS $($rest:tt)*) => {}; 148 (emit I64ShrS $($rest:tt)*) => {}; 149 (emit I32ShrU $($rest:tt)*) => {}; 150 (emit I64ShrU $($rest:tt)*) => {}; 151 (emit I32Rotl $($rest:tt)*) => {}; 152 (emit I64Rotl $($rest:tt)*) => {}; 153 (emit I32Rotr $($rest:tt)*) => {}; 154 (emit I64Rotr $($rest:tt)*) => {}; 155 (emit I32Clz $($rest:tt)*) => {}; 156 (emit I64Clz $($rest:tt)*) => {}; 157 (emit I32Ctz $($rest:tt)*) => {}; 158 (emit I64Ctz $($rest:tt)*) => {}; 159 (emit I32Popcnt $($rest:tt)*) => {}; 160 (emit I64Popcnt $($rest:tt)*) => {}; 161 (emit I32WrapI64 $($rest:tt)*) => {}; 162 (emit I64ExtendI32S $($rest:tt)*) => {}; 163 (emit I64ExtendI32U $($rest:tt)*) => {}; 164 (emit I32Extend8S $($rest:tt)*) => {}; 165 (emit I32Extend16S $($rest:tt)*) => {}; 166 (emit I64Extend8S $($rest:tt)*) => {}; 167 (emit I64Extend16S $($rest:tt)*) => {}; 168 (emit I64Extend32S $($rest:tt)*) => {}; 169 (emit I32TruncF32S $($rest:tt)*) => {}; 170 (emit I32TruncF32U $($rest:tt)*) => {}; 171 (emit I32TruncF64S $($rest:tt)*) => {}; 172 (emit I32TruncF64U $($rest:tt)*) => {}; 173 (emit I64TruncF32S $($rest:tt)*) => {}; 174 (emit I64TruncF32U $($rest:tt)*) => {}; 175 (emit I64TruncF64S $($rest:tt)*) => {}; 176 (emit I64TruncF64U $($rest:tt)*) => {}; 177 (emit I32ReinterpretF32 $($rest:tt)*) => {}; 178 (emit I64ReinterpretF64 $($rest:tt)*) => {}; 179 (emit LocalGet $($rest:tt)*) => {}; 180 (emit LocalSet $($rest:tt)*) => {}; 181 (emit Call $($rest:tt)*) => {}; 182 (emit End $($rest:tt)*) => {}; 183 (emit Nop $($rest:tt)*) => {}; 184 (emit If $($rest:tt)*) => {}; 185 (emit Else $($rest:tt)*) => {}; 186 (emit Block $($rest:tt)*) => {}; 187 (emit Loop $($rest:tt)*) => {}; 188 (emit Br $($rest:tt)*) => {}; 189 (emit BrIf $($rest:tt)*) => {}; 190 (emit Return $($rest:tt)*) => {}; 191 (emit Unreachable $($rest:tt)*) => {}; 192 (emit LocalTee $($rest:tt)*) => {}; 193 (emit GlobalGet $($rest:tt)*) => {}; 194 (emit GlobalSet $($rest:tt)*) => {}; 195 (emit Select $($rest:tt)*) => {}; 196 (emit Drop $($rest:tt)*) => {}; 197 (emit BrTable $($rest:tt)*) => {}; 198 (emit CallIndirect $($rest:tt)*) => {}; 199 (emit TableInit $($rest:tt)*) => {}; 200 (emit TableCopy $($rest:tt)*) => {}; 201 (emit TableGet $($rest:tt)*) => {}; 202 (emit TableSet $($rest:tt)*) => {}; 203 (emit TableGrow $($rest:tt)*) => {}; 204 (emit TableSize $($rest:tt)*) => {}; 205 (emit TableFill $($rest:tt)*) => {}; 206 (emit ElemDrop $($rest:tt)*) => {}; 207 (emit MemoryInit $($rest:tt)*) => {}; 208 (emit MemoryCopy $($rest:tt)*) => {}; 209 (emit DataDrop $($rest:tt)*) => {}; 210 (emit MemoryFill $($rest:tt)*) => {}; 211 (emit MemorySize $($rest:tt)*) => {}; 212 (emit MemoryGrow $($rest:tt)*) => {}; 213 (emit I32Load $($rest:tt)*) => {}; 214 (emit I32Load8S $($rest:tt)*) => {}; 215 (emit I32Load8U $($rest:tt)*) => {}; 216 (emit I32Load16S $($rest:tt)*) => {}; 217 (emit I32Load16U $($rest:tt)*) => {}; 218 (emit I64Load8S $($rest:tt)*) => {}; 219 (emit I64Load8U $($rest:tt)*) => {}; 220 (emit I64Load16S $($rest:tt)*) => {}; 221 (emit I64Load16U $($rest:tt)*) => {}; 222 (emit I64Load32S $($rest:tt)*) => {}; 223 (emit I64Load32U $($rest:tt)*) => {}; 224 (emit I64Load $($rest:tt)*) => {}; 225 (emit I32Store $($rest:tt)*) => {}; 226 (emit I32Store8 $($rest:tt)*) => {}; 227 (emit I32Store16 $($rest:tt)*) => {}; 228 (emit I64Store $($rest:tt)*) => {}; 229 (emit I64Store8 $($rest:tt)*) => {}; 230 (emit I64Store16 $($rest:tt)*) => {}; 231 (emit I64Store32 $($rest:tt)*) => {}; 232 (emit F32Load $($rest:tt)*) => {}; 233 (emit F32Store $($rest:tt)*) => {}; 234 (emit F64Load $($rest:tt)*) => {}; 235 (emit F64Store $($rest:tt)*) => {}; 236 (emit I32TruncSatF32S $($rest:tt)*) => {}; 237 (emit I32TruncSatF32U $($rest:tt)*) => {}; 238 (emit I32TruncSatF64S $($rest:tt)*) => {}; 239 (emit I32TruncSatF64U $($rest:tt)*) => {}; 240 (emit I64TruncSatF32S $($rest:tt)*) => {}; 241 (emit I64TruncSatF32U $($rest:tt)*) => {}; 242 (emit I64TruncSatF64S $($rest:tt)*) => {}; 243 (emit I64TruncSatF64U $($rest:tt)*) => {}; 244 (emit V128Load $($rest:tt)*) => {}; 245 (emit V128Store $($rest:tt)*) => {}; 246 (emit I64Add128 $($rest:tt)*) => {}; 247 (emit I64Sub128 $($rest:tt)*) => {}; 248 (emit I64MulWideS $($rest:tt)*) => {}; 249 (emit I64MulWideU $($rest:tt)*) => {}; 250 251 (emit $unsupported:tt $($rest:tt)*) => {$($rest)*}; 252 } 253 254 impl<'a, 'translation, 'data, M> VisitOperator<'a> for CodeGen<'a, 'translation, 'data, M> 255 where 256 M: MacroAssembler, 257 { 258 type Output = (); 259 260 fn visit_i32_const(&mut self, val: i32) { 261 self.context.stack.push(Val::i32(val)); 262 } 263 264 fn visit_i64_const(&mut self, val: i64) { 265 self.context.stack.push(Val::i64(val)); 266 } 267 268 fn visit_f32_const(&mut self, val: Ieee32) { 269 self.context.stack.push(Val::f32(val)); 270 } 271 272 fn visit_f64_const(&mut self, val: Ieee64) { 273 self.context.stack.push(Val::f64(val)); 274 } 275 276 fn visit_v128_const(&mut self, val: V128) { 277 self.context.stack.push(Val::v128(val.i128())) 278 } 279 280 fn visit_f32_add(&mut self) { 281 self.context.binop( 282 self.masm, 283 OperandSize::S32, 284 &mut |masm: &mut M, dst, src, size| { 285 masm.float_add(writable!(dst), dst, src, size); 286 TypedReg::f32(dst) 287 }, 288 ); 289 } 290 291 fn visit_f64_add(&mut self) { 292 self.context.binop( 293 self.masm, 294 OperandSize::S64, 295 &mut |masm: &mut M, dst, src, size| { 296 masm.float_add(writable!(dst), dst, src, size); 297 TypedReg::f64(dst) 298 }, 299 ); 300 } 301 302 fn visit_f32_sub(&mut self) { 303 self.context.binop( 304 self.masm, 305 OperandSize::S32, 306 &mut |masm: &mut M, dst, src, size| { 307 masm.float_sub(writable!(dst), dst, src, size); 308 TypedReg::f32(dst) 309 }, 310 ); 311 } 312 313 fn visit_f64_sub(&mut self) { 314 self.context.binop( 315 self.masm, 316 OperandSize::S64, 317 &mut |masm: &mut M, dst, src, size| { 318 masm.float_sub(writable!(dst), dst, src, size); 319 TypedReg::f64(dst) 320 }, 321 ); 322 } 323 324 fn visit_f32_mul(&mut self) { 325 self.context.binop( 326 self.masm, 327 OperandSize::S32, 328 &mut |masm: &mut M, dst, src, size| { 329 masm.float_mul(writable!(dst), dst, src, size); 330 TypedReg::f32(dst) 331 }, 332 ); 333 } 334 335 fn visit_f64_mul(&mut self) { 336 self.context.binop( 337 self.masm, 338 OperandSize::S64, 339 &mut |masm: &mut M, dst, src, size| { 340 masm.float_mul(writable!(dst), dst, src, size); 341 TypedReg::f64(dst) 342 }, 343 ); 344 } 345 346 fn visit_f32_div(&mut self) { 347 self.context.binop( 348 self.masm, 349 OperandSize::S32, 350 &mut |masm: &mut M, dst, src, size| { 351 masm.float_div(writable!(dst), dst, src, size); 352 TypedReg::f32(dst) 353 }, 354 ); 355 } 356 357 fn visit_f64_div(&mut self) { 358 self.context.binop( 359 self.masm, 360 OperandSize::S64, 361 &mut |masm: &mut M, dst, src, size| { 362 masm.float_div(writable!(dst), dst, src, size); 363 TypedReg::f64(dst) 364 }, 365 ); 366 } 367 368 fn visit_f32_min(&mut self) { 369 self.context.binop( 370 self.masm, 371 OperandSize::S32, 372 &mut |masm: &mut M, dst, src, size| { 373 masm.float_min(writable!(dst), dst, src, size); 374 TypedReg::f32(dst) 375 }, 376 ); 377 } 378 379 fn visit_f64_min(&mut self) { 380 self.context.binop( 381 self.masm, 382 OperandSize::S64, 383 &mut |masm: &mut M, dst, src, size| { 384 masm.float_min(writable!(dst), dst, src, size); 385 TypedReg::f64(dst) 386 }, 387 ); 388 } 389 390 fn visit_f32_max(&mut self) { 391 self.context.binop( 392 self.masm, 393 OperandSize::S32, 394 &mut |masm: &mut M, dst, src, size| { 395 masm.float_max(writable!(dst), dst, src, size); 396 TypedReg::f32(dst) 397 }, 398 ); 399 } 400 401 fn visit_f64_max(&mut self) { 402 self.context.binop( 403 self.masm, 404 OperandSize::S64, 405 &mut |masm: &mut M, dst, src, size| { 406 masm.float_max(writable!(dst), dst, src, size); 407 TypedReg::f64(dst) 408 }, 409 ); 410 } 411 412 fn visit_f32_copysign(&mut self) { 413 self.context.binop( 414 self.masm, 415 OperandSize::S32, 416 &mut |masm: &mut M, dst, src, size| { 417 masm.float_copysign(writable!(dst), dst, src, size); 418 TypedReg::f32(dst) 419 }, 420 ); 421 } 422 423 fn visit_f64_copysign(&mut self) { 424 self.context.binop( 425 self.masm, 426 OperandSize::S64, 427 &mut |masm: &mut M, dst, src, size| { 428 masm.float_copysign(writable!(dst), dst, src, size); 429 TypedReg::f64(dst) 430 }, 431 ); 432 } 433 434 fn visit_f32_abs(&mut self) { 435 self.context 436 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 437 masm.float_abs(writable!(reg), size); 438 TypedReg::f32(reg) 439 }); 440 } 441 442 fn visit_f64_abs(&mut self) { 443 self.context 444 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 445 masm.float_abs(writable!(reg), size); 446 TypedReg::f64(reg) 447 }); 448 } 449 450 fn visit_f32_neg(&mut self) { 451 self.context 452 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 453 masm.float_neg(writable!(reg), size); 454 TypedReg::f32(reg) 455 }); 456 } 457 458 fn visit_f64_neg(&mut self) { 459 self.context 460 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 461 masm.float_neg(writable!(reg), size); 462 TypedReg::f64(reg) 463 }); 464 } 465 466 fn visit_f32_floor(&mut self) { 467 self.masm.float_round( 468 RoundingMode::Down, 469 &mut self.env, 470 &mut self.context, 471 OperandSize::S32, 472 |env, cx, masm| { 473 let builtin = env.builtins.floor_f32::<M::ABI>(); 474 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 475 }, 476 ); 477 } 478 479 fn visit_f64_floor(&mut self) { 480 self.masm.float_round( 481 RoundingMode::Down, 482 &mut self.env, 483 &mut self.context, 484 OperandSize::S64, 485 |env, cx, masm| { 486 let builtin = env.builtins.floor_f64::<M::ABI>(); 487 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 488 }, 489 ); 490 } 491 492 fn visit_f32_ceil(&mut self) { 493 self.masm.float_round( 494 RoundingMode::Up, 495 &mut self.env, 496 &mut self.context, 497 OperandSize::S32, 498 |env, cx, masm| { 499 let builtin = env.builtins.ceil_f32::<M::ABI>(); 500 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 501 }, 502 ); 503 } 504 505 fn visit_f64_ceil(&mut self) { 506 self.masm.float_round( 507 RoundingMode::Up, 508 &mut self.env, 509 &mut self.context, 510 OperandSize::S64, 511 |env, cx, masm| { 512 let builtin = env.builtins.ceil_f64::<M::ABI>(); 513 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 514 }, 515 ); 516 } 517 518 fn visit_f32_nearest(&mut self) { 519 self.masm.float_round( 520 RoundingMode::Nearest, 521 &mut self.env, 522 &mut self.context, 523 OperandSize::S32, 524 |env, cx, masm| { 525 let builtin = env.builtins.nearest_f32::<M::ABI>(); 526 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 527 }, 528 ); 529 } 530 531 fn visit_f64_nearest(&mut self) { 532 self.masm.float_round( 533 RoundingMode::Nearest, 534 &mut self.env, 535 &mut self.context, 536 OperandSize::S64, 537 |env, cx, masm| { 538 let builtin = env.builtins.nearest_f64::<M::ABI>(); 539 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 540 }, 541 ); 542 } 543 544 fn visit_f32_trunc(&mut self) { 545 self.masm.float_round( 546 RoundingMode::Zero, 547 &mut self.env, 548 &mut self.context, 549 OperandSize::S32, 550 |env, cx, masm| { 551 let builtin = env.builtins.trunc_f32::<M::ABI>(); 552 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 553 }, 554 ); 555 } 556 557 fn visit_f64_trunc(&mut self) { 558 self.masm.float_round( 559 RoundingMode::Zero, 560 &mut self.env, 561 &mut self.context, 562 OperandSize::S64, 563 |env, cx, masm| { 564 let builtin = env.builtins.trunc_f64::<M::ABI>(); 565 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 566 }, 567 ); 568 } 569 570 fn visit_f32_sqrt(&mut self) { 571 self.context 572 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 573 masm.float_sqrt(writable!(reg), reg, size); 574 TypedReg::f32(reg) 575 }); 576 } 577 578 fn visit_f64_sqrt(&mut self) { 579 self.context 580 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 581 masm.float_sqrt(writable!(reg), reg, size); 582 TypedReg::f64(reg) 583 }); 584 } 585 586 fn visit_f32_eq(&mut self) { 587 self.context.float_cmp_op( 588 self.masm, 589 OperandSize::S32, 590 &mut |masm: &mut M, dst, src1, src2, size| { 591 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Eq, size); 592 }, 593 ); 594 } 595 596 fn visit_f64_eq(&mut self) { 597 self.context.float_cmp_op( 598 self.masm, 599 OperandSize::S64, 600 &mut |masm: &mut M, dst, src1, src2, size| { 601 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Eq, size); 602 }, 603 ); 604 } 605 606 fn visit_f32_ne(&mut self) { 607 self.context.float_cmp_op( 608 self.masm, 609 OperandSize::S32, 610 &mut |masm: &mut M, dst, src1, src2, size| { 611 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ne, size); 612 }, 613 ); 614 } 615 616 fn visit_f64_ne(&mut self) { 617 self.context.float_cmp_op( 618 self.masm, 619 OperandSize::S64, 620 &mut |masm: &mut M, dst, src1, src2, size| { 621 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ne, size); 622 }, 623 ); 624 } 625 626 fn visit_f32_lt(&mut self) { 627 self.context.float_cmp_op( 628 self.masm, 629 OperandSize::S32, 630 &mut |masm: &mut M, dst, src1, src2, size| { 631 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Lt, size); 632 }, 633 ); 634 } 635 636 fn visit_f64_lt(&mut self) { 637 self.context.float_cmp_op( 638 self.masm, 639 OperandSize::S64, 640 &mut |masm: &mut M, dst, src1, src2, size| { 641 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Lt, size); 642 }, 643 ); 644 } 645 646 fn visit_f32_gt(&mut self) { 647 self.context.float_cmp_op( 648 self.masm, 649 OperandSize::S32, 650 &mut |masm: &mut M, dst, src1, src2, size| { 651 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Gt, size); 652 }, 653 ); 654 } 655 656 fn visit_f64_gt(&mut self) { 657 self.context.float_cmp_op( 658 self.masm, 659 OperandSize::S64, 660 &mut |masm: &mut M, dst, src1, src2, size| { 661 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Gt, size); 662 }, 663 ); 664 } 665 666 fn visit_f32_le(&mut self) { 667 self.context.float_cmp_op( 668 self.masm, 669 OperandSize::S32, 670 &mut |masm: &mut M, dst, src1, src2, size| { 671 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Le, size); 672 }, 673 ); 674 } 675 676 fn visit_f64_le(&mut self) { 677 self.context.float_cmp_op( 678 self.masm, 679 OperandSize::S64, 680 &mut |masm: &mut M, dst, src1, src2, size| { 681 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Le, size); 682 }, 683 ); 684 } 685 686 fn visit_f32_ge(&mut self) { 687 self.context.float_cmp_op( 688 self.masm, 689 OperandSize::S32, 690 &mut |masm: &mut M, dst, src1, src2, size| { 691 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ge, size); 692 }, 693 ); 694 } 695 696 fn visit_f64_ge(&mut self) { 697 self.context.float_cmp_op( 698 self.masm, 699 OperandSize::S64, 700 &mut |masm: &mut M, dst, src1, src2, size| { 701 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ge, size); 702 }, 703 ); 704 } 705 706 fn visit_f32_convert_i32_s(&mut self) { 707 self.context 708 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 709 masm.signed_convert(writable!(dst), src, OperandSize::S32, dst_size); 710 }); 711 } 712 713 fn visit_f32_convert_i32_u(&mut self) { 714 self.context.convert_op_with_tmp_reg( 715 self.masm, 716 WasmValType::F32, 717 RegClass::Int, 718 |masm, dst, src, tmp_gpr, dst_size| { 719 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S32, dst_size); 720 }, 721 ); 722 } 723 724 fn visit_f32_convert_i64_s(&mut self) { 725 self.context 726 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 727 masm.signed_convert(writable!(dst), src, OperandSize::S64, dst_size); 728 }); 729 } 730 731 fn visit_f32_convert_i64_u(&mut self) { 732 self.context.convert_op_with_tmp_reg( 733 self.masm, 734 WasmValType::F32, 735 RegClass::Int, 736 |masm, dst, src, tmp_gpr, dst_size| { 737 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S64, dst_size); 738 }, 739 ); 740 } 741 742 fn visit_f64_convert_i32_s(&mut self) { 743 self.context 744 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 745 masm.signed_convert(writable!(dst), src, OperandSize::S32, dst_size); 746 }); 747 } 748 749 fn visit_f64_convert_i32_u(&mut self) { 750 self.context.convert_op_with_tmp_reg( 751 self.masm, 752 WasmValType::F64, 753 RegClass::Int, 754 |masm, dst, src, tmp_gpr, dst_size| { 755 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S32, dst_size); 756 }, 757 ); 758 } 759 760 fn visit_f64_convert_i64_s(&mut self) { 761 self.context 762 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 763 masm.signed_convert(writable!(dst), src, OperandSize::S64, dst_size); 764 }); 765 } 766 767 fn visit_f64_convert_i64_u(&mut self) { 768 self.context.convert_op_with_tmp_reg( 769 self.masm, 770 WasmValType::F64, 771 RegClass::Int, 772 |masm, dst, src, tmp_gpr, dst_size| { 773 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S64, dst_size); 774 }, 775 ); 776 } 777 778 fn visit_f32_reinterpret_i32(&mut self) { 779 self.context 780 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, size| { 781 masm.reinterpret_int_as_float(writable!(dst), src.into(), size); 782 }); 783 } 784 785 fn visit_f64_reinterpret_i64(&mut self) { 786 self.context 787 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, size| { 788 masm.reinterpret_int_as_float(writable!(dst), src.into(), size); 789 }); 790 } 791 792 fn visit_f32_demote_f64(&mut self) { 793 self.context 794 .unop(self.masm, OperandSize::S64, &mut |masm, reg, _size| { 795 masm.demote(writable!(reg), reg); 796 TypedReg::f32(reg) 797 }); 798 } 799 800 fn visit_f64_promote_f32(&mut self) { 801 self.context 802 .unop(self.masm, OperandSize::S32, &mut |masm, reg, _size| { 803 masm.promote(writable!(reg), reg); 804 TypedReg::f64(reg) 805 }); 806 } 807 808 fn visit_i32_add(&mut self) { 809 self.context.i32_binop(self.masm, |masm, dst, src, size| { 810 masm.add(writable!(dst), dst, src, size); 811 TypedReg::i32(dst) 812 }); 813 } 814 815 fn visit_i64_add(&mut self) { 816 self.context.i64_binop(self.masm, |masm, dst, src, size| { 817 masm.add(writable!(dst), dst, src, size); 818 TypedReg::i64(dst) 819 }); 820 } 821 822 fn visit_i32_sub(&mut self) { 823 self.context.i32_binop(self.masm, |masm, dst, src, size| { 824 masm.sub(writable!(dst), dst, src, size); 825 TypedReg::i32(dst) 826 }); 827 } 828 829 fn visit_i64_sub(&mut self) { 830 self.context.i64_binop(self.masm, |masm, dst, src, size| { 831 masm.sub(writable!(dst), dst, src, size); 832 TypedReg::i64(dst) 833 }); 834 } 835 836 fn visit_i32_mul(&mut self) { 837 self.context.i32_binop(self.masm, |masm, dst, src, size| { 838 masm.mul(writable!(dst), dst, src, size); 839 TypedReg::i32(dst) 840 }); 841 } 842 843 fn visit_i64_mul(&mut self) { 844 self.context.i64_binop(self.masm, |masm, dst, src, size| { 845 masm.mul(writable!(dst), dst, src, size); 846 TypedReg::i64(dst) 847 }); 848 } 849 850 fn visit_i32_div_s(&mut self) { 851 use DivKind::*; 852 use OperandSize::*; 853 854 self.masm.div(&mut self.context, Signed, S32); 855 } 856 857 fn visit_i32_div_u(&mut self) { 858 use DivKind::*; 859 use OperandSize::*; 860 861 self.masm.div(&mut self.context, Unsigned, S32); 862 } 863 864 fn visit_i64_div_s(&mut self) { 865 use DivKind::*; 866 use OperandSize::*; 867 868 self.masm.div(&mut self.context, Signed, S64); 869 } 870 871 fn visit_i64_div_u(&mut self) { 872 use DivKind::*; 873 use OperandSize::*; 874 875 self.masm.div(&mut self.context, Unsigned, S64); 876 } 877 878 fn visit_i32_rem_s(&mut self) { 879 use OperandSize::*; 880 use RemKind::*; 881 882 self.masm.rem(&mut self.context, Signed, S32); 883 } 884 885 fn visit_i32_rem_u(&mut self) { 886 use OperandSize::*; 887 use RemKind::*; 888 889 self.masm.rem(&mut self.context, Unsigned, S32); 890 } 891 892 fn visit_i64_rem_s(&mut self) { 893 use OperandSize::*; 894 use RemKind::*; 895 896 self.masm.rem(&mut self.context, Signed, S64); 897 } 898 899 fn visit_i64_rem_u(&mut self) { 900 use OperandSize::*; 901 use RemKind::*; 902 903 self.masm.rem(&mut self.context, Unsigned, S64); 904 } 905 906 fn visit_i32_eq(&mut self) { 907 self.cmp_i32s(IntCmpKind::Eq); 908 } 909 910 fn visit_i64_eq(&mut self) { 911 self.cmp_i64s(IntCmpKind::Eq); 912 } 913 914 fn visit_i32_ne(&mut self) { 915 self.cmp_i32s(IntCmpKind::Ne); 916 } 917 918 fn visit_i64_ne(&mut self) { 919 self.cmp_i64s(IntCmpKind::Ne); 920 } 921 922 fn visit_i32_lt_s(&mut self) { 923 self.cmp_i32s(IntCmpKind::LtS); 924 } 925 926 fn visit_i64_lt_s(&mut self) { 927 self.cmp_i64s(IntCmpKind::LtS); 928 } 929 930 fn visit_i32_lt_u(&mut self) { 931 self.cmp_i32s(IntCmpKind::LtU); 932 } 933 934 fn visit_i64_lt_u(&mut self) { 935 self.cmp_i64s(IntCmpKind::LtU); 936 } 937 938 fn visit_i32_le_s(&mut self) { 939 self.cmp_i32s(IntCmpKind::LeS); 940 } 941 942 fn visit_i64_le_s(&mut self) { 943 self.cmp_i64s(IntCmpKind::LeS); 944 } 945 946 fn visit_i32_le_u(&mut self) { 947 self.cmp_i32s(IntCmpKind::LeU); 948 } 949 950 fn visit_i64_le_u(&mut self) { 951 self.cmp_i64s(IntCmpKind::LeU); 952 } 953 954 fn visit_i32_gt_s(&mut self) { 955 self.cmp_i32s(IntCmpKind::GtS); 956 } 957 958 fn visit_i64_gt_s(&mut self) { 959 self.cmp_i64s(IntCmpKind::GtS); 960 } 961 962 fn visit_i32_gt_u(&mut self) { 963 self.cmp_i32s(IntCmpKind::GtU); 964 } 965 966 fn visit_i64_gt_u(&mut self) { 967 self.cmp_i64s(IntCmpKind::GtU); 968 } 969 970 fn visit_i32_ge_s(&mut self) { 971 self.cmp_i32s(IntCmpKind::GeS); 972 } 973 974 fn visit_i64_ge_s(&mut self) { 975 self.cmp_i64s(IntCmpKind::GeS); 976 } 977 978 fn visit_i32_ge_u(&mut self) { 979 self.cmp_i32s(IntCmpKind::GeU); 980 } 981 982 fn visit_i64_ge_u(&mut self) { 983 self.cmp_i64s(IntCmpKind::GeU); 984 } 985 986 fn visit_i32_eqz(&mut self) { 987 use OperandSize::*; 988 989 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 990 masm.cmp_with_set(writable!(reg.into()), RegImm::i32(0), IntCmpKind::Eq, size); 991 TypedReg::i32(reg) 992 }); 993 } 994 995 fn visit_i64_eqz(&mut self) { 996 use OperandSize::*; 997 998 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 999 masm.cmp_with_set(writable!(reg.into()), RegImm::i64(0), IntCmpKind::Eq, size); 1000 TypedReg::i32(reg) // Return value for `i64.eqz` is an `i32`. 1001 }); 1002 } 1003 1004 fn visit_i32_clz(&mut self) { 1005 use OperandSize::*; 1006 1007 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 1008 masm.clz(writable!(reg), reg, size); 1009 TypedReg::i32(reg) 1010 }); 1011 } 1012 1013 fn visit_i64_clz(&mut self) { 1014 use OperandSize::*; 1015 1016 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 1017 masm.clz(writable!(reg), reg, size); 1018 TypedReg::i64(reg) 1019 }); 1020 } 1021 1022 fn visit_i32_ctz(&mut self) { 1023 use OperandSize::*; 1024 1025 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 1026 masm.ctz(writable!(reg), reg, size); 1027 TypedReg::i32(reg) 1028 }); 1029 } 1030 1031 fn visit_i64_ctz(&mut self) { 1032 use OperandSize::*; 1033 1034 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 1035 masm.ctz(writable!(reg), reg, size); 1036 TypedReg::i64(reg) 1037 }); 1038 } 1039 1040 fn visit_i32_and(&mut self) { 1041 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1042 masm.and(writable!(dst), dst, src, size); 1043 TypedReg::i32(dst) 1044 }); 1045 } 1046 1047 fn visit_i64_and(&mut self) { 1048 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1049 masm.and(writable!(dst), dst, src, size); 1050 TypedReg::i64(dst) 1051 }); 1052 } 1053 1054 fn visit_i32_or(&mut self) { 1055 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1056 masm.or(writable!(dst), dst, src, size); 1057 TypedReg::i32(dst) 1058 }); 1059 } 1060 1061 fn visit_i64_or(&mut self) { 1062 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1063 masm.or(writable!(dst), dst, src, size); 1064 TypedReg::i64(dst) 1065 }); 1066 } 1067 1068 fn visit_i32_xor(&mut self) { 1069 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1070 masm.xor(writable!(dst), dst, src, size); 1071 TypedReg::i32(dst) 1072 }); 1073 } 1074 1075 fn visit_i64_xor(&mut self) { 1076 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1077 masm.xor(writable!(dst), dst, src, size); 1078 TypedReg::i64(dst) 1079 }); 1080 } 1081 1082 fn visit_i32_shl(&mut self) { 1083 use ShiftKind::*; 1084 1085 self.context.i32_shift(self.masm, Shl); 1086 } 1087 1088 fn visit_i64_shl(&mut self) { 1089 use ShiftKind::*; 1090 1091 self.context.i64_shift(self.masm, Shl); 1092 } 1093 1094 fn visit_i32_shr_s(&mut self) { 1095 use ShiftKind::*; 1096 1097 self.context.i32_shift(self.masm, ShrS); 1098 } 1099 1100 fn visit_i64_shr_s(&mut self) { 1101 use ShiftKind::*; 1102 1103 self.context.i64_shift(self.masm, ShrS); 1104 } 1105 1106 fn visit_i32_shr_u(&mut self) { 1107 use ShiftKind::*; 1108 1109 self.context.i32_shift(self.masm, ShrU); 1110 } 1111 1112 fn visit_i64_shr_u(&mut self) { 1113 use ShiftKind::*; 1114 1115 self.context.i64_shift(self.masm, ShrU); 1116 } 1117 1118 fn visit_i32_rotl(&mut self) { 1119 use ShiftKind::*; 1120 1121 self.context.i32_shift(self.masm, Rotl); 1122 } 1123 1124 fn visit_i64_rotl(&mut self) { 1125 use ShiftKind::*; 1126 1127 self.context.i64_shift(self.masm, Rotl); 1128 } 1129 1130 fn visit_i32_rotr(&mut self) { 1131 use ShiftKind::*; 1132 1133 self.context.i32_shift(self.masm, Rotr); 1134 } 1135 1136 fn visit_i64_rotr(&mut self) { 1137 use ShiftKind::*; 1138 1139 self.context.i64_shift(self.masm, Rotr); 1140 } 1141 1142 fn visit_end(&mut self) { 1143 if !self.context.reachable { 1144 self.handle_unreachable_end(); 1145 } else { 1146 let mut control = self.control_frames.pop().unwrap(); 1147 control.emit_end(self.masm, &mut self.context); 1148 } 1149 } 1150 1151 fn visit_i32_popcnt(&mut self) { 1152 use OperandSize::*; 1153 self.masm.popcnt(&mut self.context, S32); 1154 } 1155 1156 fn visit_i64_popcnt(&mut self) { 1157 use OperandSize::*; 1158 1159 self.masm.popcnt(&mut self.context, S64); 1160 } 1161 1162 fn visit_i32_wrap_i64(&mut self) { 1163 use OperandSize::*; 1164 1165 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1166 masm.wrap(writable!(reg), reg); 1167 TypedReg::i32(reg) 1168 }); 1169 } 1170 1171 fn visit_i64_extend_i32_s(&mut self) { 1172 use OperandSize::*; 1173 1174 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1175 masm.extend(writable!(reg), reg, ExtendKind::I64ExtendI32S); 1176 TypedReg::i64(reg) 1177 }); 1178 } 1179 1180 fn visit_i64_extend_i32_u(&mut self) { 1181 use OperandSize::*; 1182 1183 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1184 masm.extend(writable!(reg), reg, ExtendKind::I64ExtendI32U); 1185 TypedReg::i64(reg) 1186 }); 1187 } 1188 1189 fn visit_i32_extend8_s(&mut self) { 1190 use OperandSize::*; 1191 1192 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1193 masm.extend(writable!(reg), reg, ExtendKind::I32Extend8S); 1194 TypedReg::i32(reg) 1195 }); 1196 } 1197 1198 fn visit_i32_extend16_s(&mut self) { 1199 use OperandSize::*; 1200 1201 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1202 masm.extend(writable!(reg), reg, ExtendKind::I32Extend16S); 1203 TypedReg::i32(reg) 1204 }); 1205 } 1206 1207 fn visit_i64_extend8_s(&mut self) { 1208 use OperandSize::*; 1209 1210 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1211 masm.extend(writable!(reg), reg, ExtendKind::I64Extend8S); 1212 TypedReg::i64(reg) 1213 }); 1214 } 1215 1216 fn visit_i64_extend16_s(&mut self) { 1217 use OperandSize::*; 1218 1219 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1220 masm.extend(writable!(reg), reg, ExtendKind::I64Extend16S); 1221 TypedReg::i64(reg) 1222 }); 1223 } 1224 1225 fn visit_i64_extend32_s(&mut self) { 1226 use OperandSize::*; 1227 1228 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1229 masm.extend(writable!(reg), reg, ExtendKind::I64Extend32S); 1230 TypedReg::i64(reg) 1231 }); 1232 } 1233 1234 fn visit_i32_trunc_f32_s(&mut self) { 1235 use OperandSize::*; 1236 1237 self.context 1238 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1239 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Unchecked); 1240 }); 1241 } 1242 1243 fn visit_i32_trunc_f32_u(&mut self) { 1244 use OperandSize::*; 1245 1246 self.context.convert_op_with_tmp_reg( 1247 self.masm, 1248 WasmValType::I32, 1249 RegClass::Float, 1250 |masm, dst, src, tmp_fpr, dst_size| { 1251 masm.unsigned_truncate( 1252 writable!(dst), 1253 src, 1254 tmp_fpr, 1255 S32, 1256 dst_size, 1257 TruncKind::Unchecked, 1258 ); 1259 }, 1260 ); 1261 } 1262 1263 fn visit_i32_trunc_f64_s(&mut self) { 1264 use OperandSize::*; 1265 1266 self.context 1267 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1268 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Unchecked); 1269 }); 1270 } 1271 1272 fn visit_i32_trunc_f64_u(&mut self) { 1273 use OperandSize::*; 1274 1275 self.context.convert_op_with_tmp_reg( 1276 self.masm, 1277 WasmValType::I32, 1278 RegClass::Float, 1279 |masm, dst, src, tmp_fpr, dst_size| { 1280 masm.unsigned_truncate( 1281 writable!(dst), 1282 src, 1283 tmp_fpr, 1284 S64, 1285 dst_size, 1286 TruncKind::Unchecked, 1287 ); 1288 }, 1289 ); 1290 } 1291 1292 fn visit_i64_trunc_f32_s(&mut self) { 1293 use OperandSize::*; 1294 1295 self.context 1296 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1297 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Unchecked); 1298 }); 1299 } 1300 1301 fn visit_i64_trunc_f32_u(&mut self) { 1302 use OperandSize::*; 1303 1304 self.context.convert_op_with_tmp_reg( 1305 self.masm, 1306 WasmValType::I64, 1307 RegClass::Float, 1308 |masm, dst, src, tmp_fpr, dst_size| { 1309 masm.unsigned_truncate( 1310 writable!(dst), 1311 src, 1312 tmp_fpr, 1313 S32, 1314 dst_size, 1315 TruncKind::Unchecked, 1316 ); 1317 }, 1318 ); 1319 } 1320 1321 fn visit_i64_trunc_f64_s(&mut self) { 1322 use OperandSize::*; 1323 1324 self.context 1325 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1326 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Unchecked); 1327 }); 1328 } 1329 1330 fn visit_i64_trunc_f64_u(&mut self) { 1331 use OperandSize::*; 1332 1333 self.context.convert_op_with_tmp_reg( 1334 self.masm, 1335 WasmValType::I64, 1336 RegClass::Float, 1337 |masm, dst, src, tmp_fpr, dst_size| { 1338 masm.unsigned_truncate( 1339 writable!(dst), 1340 src, 1341 tmp_fpr, 1342 S64, 1343 dst_size, 1344 TruncKind::Unchecked, 1345 ); 1346 }, 1347 ); 1348 } 1349 1350 fn visit_i32_reinterpret_f32(&mut self) { 1351 self.context 1352 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, size| { 1353 masm.reinterpret_float_as_int(writable!(dst), src.into(), size); 1354 }); 1355 } 1356 1357 fn visit_i64_reinterpret_f64(&mut self) { 1358 self.context 1359 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, size| { 1360 masm.reinterpret_float_as_int(writable!(dst), src.into(), size); 1361 }); 1362 } 1363 1364 fn visit_local_get(&mut self, index: u32) { 1365 use WasmValType::*; 1366 let context = &mut self.context; 1367 let slot = context.frame.get_wasm_local(index); 1368 match slot.ty { 1369 I32 | I64 | F32 | F64 | V128 => context.stack.push(Val::local(index, slot.ty)), 1370 Ref(rt) => match rt.heap_type { 1371 WasmHeapType::Func => context.stack.push(Val::local(index, slot.ty)), 1372 WasmHeapType::Extern => { 1373 self.found_unsupported_instruction = 1374 Some("unsupported local.get of externref local"); 1375 } 1376 ht => unimplemented!("Support for WasmHeapType: {ht}"), 1377 }, 1378 } 1379 } 1380 1381 fn visit_local_set(&mut self, index: u32) { 1382 let src = self.emit_set_local(index); 1383 self.context.free_reg(src); 1384 } 1385 1386 fn visit_call(&mut self, index: u32) { 1387 let callee = self.env.callee_from_index(FuncIndex::from_u32(index)); 1388 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee) 1389 } 1390 1391 fn visit_call_indirect(&mut self, type_index: u32, table_index: u32) { 1392 // Spill now because `emit_lazy_init_funcref` and the `FnCall::emit` 1393 // invocations will both trigger spills since they both call functions. 1394 // However, the machine instructions for the spill emitted by 1395 // `emit_lazy_funcref` will be jumped over if the funcref was previously 1396 // initialized which may result in the machine stack becoming 1397 // unbalanced. 1398 self.context.spill(self.masm); 1399 1400 let type_index = TypeIndex::from_u32(type_index); 1401 let table_index = TableIndex::from_u32(table_index); 1402 1403 self.emit_lazy_init_funcref(table_index); 1404 1405 // Perform the indirect call. 1406 // This code assumes that [`Self::emit_lazy_init_funcref`] will 1407 // push the funcref to the value stack. 1408 let funcref_ptr = self.context.stack.peek().map(|v| v.unwrap_reg()).unwrap(); 1409 self.masm 1410 .trapz(funcref_ptr.into(), TRAP_INDIRECT_CALL_TO_NULL); 1411 self.emit_typecheck_funcref(funcref_ptr.into(), type_index); 1412 1413 let callee = self.env.funcref(type_index); 1414 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee) 1415 } 1416 1417 fn visit_table_init(&mut self, elem: u32, table: u32) { 1418 debug_assert!(self.context.stack.len() >= 3); 1419 let at = self.context.stack.len() - 3; 1420 1421 self.context 1422 .stack 1423 .insert_many(at, &[table.try_into().unwrap(), elem.try_into().unwrap()]); 1424 1425 let builtin = self.env.builtins.table_init::<M::ABI, M::Ptr>(); 1426 FnCall::emit::<M>( 1427 &mut self.env, 1428 self.masm, 1429 &mut self.context, 1430 Callee::Builtin(builtin.clone()), 1431 ) 1432 } 1433 1434 fn visit_table_copy(&mut self, dst: u32, src: u32) { 1435 debug_assert!(self.context.stack.len() >= 3); 1436 let at = self.context.stack.len() - 3; 1437 self.context 1438 .stack 1439 .insert_many(at, &[dst.try_into().unwrap(), src.try_into().unwrap()]); 1440 1441 let builtin = self.env.builtins.table_copy::<M::ABI, M::Ptr>(); 1442 FnCall::emit::<M>( 1443 &mut self.env, 1444 self.masm, 1445 &mut self.context, 1446 Callee::Builtin(builtin), 1447 ) 1448 } 1449 1450 fn visit_table_get(&mut self, table: u32) { 1451 let table_index = TableIndex::from_u32(table); 1452 let table = self.env.table(table_index); 1453 let heap_type = table.ref_type.heap_type; 1454 1455 match heap_type { 1456 WasmHeapType::Func => self.emit_lazy_init_funcref(table_index), 1457 WasmHeapType::Extern => { 1458 self.found_unsupported_instruction = 1459 Some("unsupported table.get of externref table"); 1460 } 1461 t => { 1462 unimplemented!("Support for WasmHeapType: {t}") 1463 } 1464 } 1465 } 1466 1467 fn visit_table_grow(&mut self, table: u32) { 1468 let table_index = TableIndex::from_u32(table); 1469 let table_ty = self.env.table(table_index); 1470 let builtin = match table_ty.ref_type.heap_type { 1471 WasmHeapType::Func => self.env.builtins.table_grow_func_ref::<M::ABI, M::Ptr>(), 1472 ty => unimplemented!("Support for HeapType: {ty}"), 1473 }; 1474 1475 let len = self.context.stack.len(); 1476 // table.grow` requires at least 2 elements on the value stack. 1477 debug_assert!(len >= 2); 1478 let at = len - 2; 1479 1480 // The table_grow builtin expects the parameters in a different 1481 // order. 1482 // The value stack at this point should contain: 1483 // [ init_value | delta ] (stack top) 1484 // but the builtin function expects the init value as the last 1485 // argument. 1486 self.context.stack.inner_mut().swap(len - 1, len - 2); 1487 self.context 1488 .stack 1489 .insert_many(at, &[table.try_into().unwrap()]); 1490 1491 FnCall::emit::<M>( 1492 &mut self.env, 1493 self.masm, 1494 &mut self.context, 1495 Callee::Builtin(builtin.clone()), 1496 ) 1497 } 1498 1499 fn visit_table_size(&mut self, table: u32) { 1500 let table_index = TableIndex::from_u32(table); 1501 let table_data = self.env.resolve_table_data(table_index); 1502 self.emit_compute_table_size(&table_data); 1503 } 1504 1505 fn visit_table_fill(&mut self, table: u32) { 1506 let table_index = TableIndex::from_u32(table); 1507 let table_ty = self.env.table(table_index); 1508 let builtin = match table_ty.ref_type.heap_type { 1509 WasmHeapType::Func => self.env.builtins.table_fill_func_ref::<M::ABI, M::Ptr>(), 1510 ty => unimplemented!("Support for heap type: {ty}"), 1511 }; 1512 1513 let len = self.context.stack.len(); 1514 debug_assert!(len >= 3); 1515 let at = len - 3; 1516 self.context 1517 .stack 1518 .insert_many(at, &[table.try_into().unwrap()]); 1519 FnCall::emit::<M>( 1520 &mut self.env, 1521 self.masm, 1522 &mut self.context, 1523 Callee::Builtin(builtin.clone()), 1524 ) 1525 } 1526 1527 fn visit_table_set(&mut self, table: u32) { 1528 let ptr_type = self.env.ptr_type(); 1529 let table_index = TableIndex::from_u32(table); 1530 let table_data = self.env.resolve_table_data(table_index); 1531 let table = self.env.table(table_index); 1532 match table.ref_type.heap_type { 1533 WasmHeapType::Func => { 1534 assert!( 1535 self.tunables.table_lazy_init, 1536 "unsupported table eager init" 1537 ); 1538 let value = self.context.pop_to_reg(self.masm, None); 1539 let index = self.context.pop_to_reg(self.masm, None); 1540 let base = self.context.any_gpr(self.masm); 1541 let elem_addr = self.emit_compute_table_elem_addr(index.into(), base, &table_data); 1542 // Set the initialized bit. 1543 self.masm.or( 1544 writable!(value.into()), 1545 value.into(), 1546 RegImm::i64(FUNCREF_INIT_BIT as i64), 1547 ptr_type.into(), 1548 ); 1549 1550 self.masm.store_ptr(value.into(), elem_addr); 1551 1552 self.context.free_reg(value); 1553 self.context.free_reg(index); 1554 self.context.free_reg(base); 1555 } 1556 ty => unimplemented!("Support for WasmHeapType: {ty}"), 1557 }; 1558 } 1559 1560 fn visit_elem_drop(&mut self, index: u32) { 1561 let elem_drop = self.env.builtins.elem_drop::<M::ABI, M::Ptr>(); 1562 self.context.stack.extend([index.try_into().unwrap()]); 1563 FnCall::emit::<M>( 1564 &mut self.env, 1565 self.masm, 1566 &mut self.context, 1567 Callee::Builtin(elem_drop), 1568 ) 1569 } 1570 1571 fn visit_memory_init(&mut self, data_index: u32, mem: u32) { 1572 debug_assert!(self.context.stack.len() >= 3); 1573 let at = self.context.stack.len() - 3; 1574 self.context.stack.insert_many( 1575 at, 1576 &[mem.try_into().unwrap(), data_index.try_into().unwrap()], 1577 ); 1578 let builtin = self.env.builtins.memory_init::<M::ABI, M::Ptr>(); 1579 FnCall::emit::<M>( 1580 &mut self.env, 1581 self.masm, 1582 &mut self.context, 1583 Callee::Builtin(builtin), 1584 ) 1585 } 1586 1587 fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) { 1588 // At this point, the stack is expected to contain: 1589 // [ dst_offset, src_offset, len ] 1590 // The following code inserts the missing params, so that stack contains: 1591 // [ vmctx, dst_mem, dst_offset, src_mem, src_offset, len ] 1592 // Which is the order expected by the builtin function. 1593 debug_assert!(self.context.stack.len() >= 3); 1594 let at = self.context.stack.len() - 2; 1595 self.context 1596 .stack 1597 .insert_many(at, &[src_mem.try_into().unwrap()]); 1598 1599 // One element was inserted above, so instead of 3, we use 4. 1600 let at = self.context.stack.len() - 4; 1601 self.context 1602 .stack 1603 .insert_many(at, &[dst_mem.try_into().unwrap()]); 1604 1605 let builtin = self.env.builtins.memory_copy::<M::ABI, M::Ptr>(); 1606 1607 FnCall::emit::<M>( 1608 &mut self.env, 1609 self.masm, 1610 &mut self.context, 1611 Callee::Builtin(builtin), 1612 ) 1613 } 1614 1615 fn visit_memory_fill(&mut self, mem: u32) { 1616 debug_assert!(self.context.stack.len() >= 3); 1617 let at = self.context.stack.len() - 3; 1618 1619 self.context 1620 .stack 1621 .insert_many(at, &[mem.try_into().unwrap()]); 1622 1623 let builtin = self.env.builtins.memory_fill::<M::ABI, M::Ptr>(); 1624 FnCall::emit::<M>( 1625 &mut self.env, 1626 self.masm, 1627 &mut self.context, 1628 Callee::Builtin(builtin), 1629 ) 1630 } 1631 1632 fn visit_memory_size(&mut self, mem: u32) { 1633 let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem)); 1634 self.emit_compute_memory_size(&heap); 1635 } 1636 1637 fn visit_memory_grow(&mut self, mem: u32) { 1638 debug_assert!(self.context.stack.len() >= 1); 1639 // The stack at this point contains: [ delta ] 1640 // The desired state is 1641 // [ vmctx, delta, index ] 1642 self.context.stack.extend([mem.try_into().unwrap()]); 1643 1644 let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem)); 1645 let builtin = self.env.builtins.memory32_grow::<M::ABI, M::Ptr>(); 1646 FnCall::emit::<M>( 1647 &mut self.env, 1648 self.masm, 1649 &mut self.context, 1650 Callee::Builtin(builtin), 1651 ); 1652 1653 // The memory32_grow builtin returns a pointer type, therefore we must 1654 // ensure that the return type is representative of the address space of 1655 // the heap type. 1656 match (self.env.ptr_type(), heap.index_type()) { 1657 (WasmValType::I64, WasmValType::I64) => {} 1658 // When the heap type is smaller than the pointer type, we adjust 1659 // the result of the memory32_grow builtin. 1660 (WasmValType::I64, WasmValType::I32) => { 1661 let top: Reg = self.context.pop_to_reg(self.masm, None).into(); 1662 self.masm.wrap(writable!(top.into()), top.into()); 1663 self.context.stack.push(TypedReg::i32(top).into()); 1664 } 1665 _ => unimplemented!("Support for 32-bit platforms"), 1666 } 1667 } 1668 1669 fn visit_data_drop(&mut self, data_index: u32) { 1670 self.context.stack.extend([data_index.try_into().unwrap()]); 1671 1672 let builtin = self.env.builtins.data_drop::<M::ABI, M::Ptr>(); 1673 FnCall::emit::<M>( 1674 &mut self.env, 1675 self.masm, 1676 &mut self.context, 1677 Callee::Builtin(builtin), 1678 ) 1679 } 1680 1681 fn visit_nop(&mut self) {} 1682 1683 fn visit_if(&mut self, blockty: BlockType) { 1684 self.control_frames.push(ControlStackFrame::r#if( 1685 self.env.resolve_block_sig(blockty), 1686 self.masm, 1687 &mut self.context, 1688 )); 1689 } 1690 1691 fn visit_else(&mut self) { 1692 if !self.context.reachable { 1693 self.handle_unreachable_else(); 1694 } else { 1695 let control = self 1696 .control_frames 1697 .last_mut() 1698 .unwrap_or_else(|| panic!("Expected active control stack frame for else")); 1699 control.emit_else(self.masm, &mut self.context); 1700 } 1701 } 1702 1703 fn visit_block(&mut self, blockty: BlockType) { 1704 self.control_frames.push(ControlStackFrame::block( 1705 self.env.resolve_block_sig(blockty), 1706 self.masm, 1707 &mut self.context, 1708 )); 1709 } 1710 1711 fn visit_loop(&mut self, blockty: BlockType) { 1712 self.control_frames.push(ControlStackFrame::r#loop( 1713 self.env.resolve_block_sig(blockty), 1714 self.masm, 1715 &mut self.context, 1716 )); 1717 1718 // Emit fuel check right after binding the loop header. 1719 if self.tunables.consume_fuel { 1720 self.emit_fuel_check(); 1721 } 1722 } 1723 1724 fn visit_br(&mut self, depth: u32) { 1725 let index = control_index(depth, self.control_frames.len()); 1726 let frame = &mut self.control_frames[index]; 1727 self.context 1728 .unconditional_jump(frame, self.masm, |masm, cx, frame| { 1729 frame 1730 .pop_abi_results::<M, _>(cx, masm, |results, _, _| results.ret_area().copied()); 1731 }); 1732 } 1733 1734 fn visit_br_if(&mut self, depth: u32) { 1735 let index = control_index(depth, self.control_frames.len()); 1736 let frame = &mut self.control_frames[index]; 1737 frame.set_as_target(); 1738 1739 let top = { 1740 let top = self.context.without::<TypedReg, M, _>( 1741 frame.results::<M>().regs(), 1742 self.masm, 1743 |ctx, masm| ctx.pop_to_reg(masm, None), 1744 ); 1745 // Explicitly save any live registers and locals before setting up 1746 // the branch state. 1747 // In some cases, calculating the `top` value above, will result in 1748 // a spill, thus the following one will result in a no-op. 1749 self.context.spill(self.masm); 1750 frame.top_abi_results::<M, _>( 1751 &mut self.context, 1752 self.masm, 1753 |results, context, masm| { 1754 // In the case of `br_if` there's a possibility that we'll 1755 // exit early from the block or fallthrough, for 1756 // a fallthrough, we cannot rely on the pre-computed return area; 1757 // it must be recalculated so that any values that are 1758 // generated are correctly placed near the current stack 1759 // pointer. 1760 results.on_stack().then(|| { 1761 let stack_consumed = context.stack.sizeof(results.stack_operands_len()); 1762 let base = masm.sp_offset().as_u32() - stack_consumed; 1763 let offs = base + results.size(); 1764 RetArea::sp(SPOffset::from_u32(offs)) 1765 }) 1766 }, 1767 ); 1768 top 1769 }; 1770 1771 // Emit instructions to balance the machine stack if the frame has 1772 // a different offset. 1773 let current_sp_offset = self.masm.sp_offset(); 1774 let results_size = frame.results::<M>().size(); 1775 let state = frame.stack_state(); 1776 let (label, cmp, needs_cleanup) = if current_sp_offset > state.target_offset { 1777 (self.masm.get_label(), IntCmpKind::Eq, true) 1778 } else { 1779 (*frame.label(), IntCmpKind::Ne, false) 1780 }; 1781 1782 self.masm 1783 .branch(cmp, top.reg.into(), top.reg.into(), label, OperandSize::S32); 1784 self.context.free_reg(top); 1785 1786 if needs_cleanup { 1787 // Emit instructions to balance the stack and jump if not falling 1788 // through. 1789 self.masm.memmove( 1790 current_sp_offset, 1791 state.target_offset, 1792 results_size, 1793 MemMoveDirection::LowToHigh, 1794 ); 1795 self.masm.ensure_sp_for_jump(state.target_offset); 1796 self.masm.jmp(*frame.label()); 1797 1798 // Restore sp_offset to what it was for falling through and emit 1799 // fallthrough label. 1800 self.masm.reset_stack_pointer(current_sp_offset); 1801 self.masm.bind(label); 1802 } 1803 } 1804 1805 fn visit_br_table(&mut self, targets: BrTable<'a>) { 1806 // +1 to account for the default target. 1807 let len = targets.len() + 1; 1808 // SmallVec<[_; 5]> to match the binary emission layer (e.g 1809 // see `JmpTableSeq'), but here we use 5 instead since we 1810 // bundle the default target as the last element in the array. 1811 let labels: SmallVec<[_; 5]> = (0..len).map(|_| self.masm.get_label()).collect(); 1812 1813 let default_index = control_index(targets.default(), self.control_frames.len()); 1814 let default_frame = &mut self.control_frames[default_index]; 1815 let default_result = default_frame.results::<M>(); 1816 1817 let (index, tmp) = { 1818 let index_and_tmp = self.context.without::<(TypedReg, _), M, _>( 1819 default_result.regs(), 1820 self.masm, 1821 |cx, masm| (cx.pop_to_reg(masm, None), cx.any_gpr(masm)), 1822 ); 1823 1824 // Materialize any constants or locals into their result representation, 1825 // so that when reachability is restored, they are correctly located. 1826 default_frame.top_abi_results::<M, _>(&mut self.context, self.masm, |results, _, _| { 1827 results.ret_area().copied() 1828 }); 1829 index_and_tmp 1830 }; 1831 1832 self.masm.jmp_table(&labels, index.into(), tmp); 1833 // Save the original stack pointer offset; we will reset the stack 1834 // pointer to this offset after jumping to each of the targets. Each 1835 // jump might adjust the stack according to the base offset of the 1836 // target. 1837 let current_sp = self.masm.sp_offset(); 1838 1839 for (t, l) in targets 1840 .targets() 1841 .into_iter() 1842 .chain(std::iter::once(Ok(targets.default()))) 1843 .zip(labels.iter()) 1844 { 1845 let control_index = control_index(t.unwrap(), self.control_frames.len()); 1846 let frame = &mut self.control_frames[control_index]; 1847 // Reset the stack pointer to its original offset. This is needed 1848 // because each jump will potentially adjust the stack pointer 1849 // according to the base offset of the target. 1850 self.masm.reset_stack_pointer(current_sp); 1851 1852 // NB: We don't perform any result handling as it was 1853 // already taken care of above before jumping to the 1854 // jump table. 1855 self.masm.bind(*l); 1856 // Ensure that the stack pointer is correctly positioned before 1857 // jumping to the jump table code. 1858 let state = frame.stack_state(); 1859 self.masm.ensure_sp_for_jump(state.target_offset); 1860 self.masm.jmp(*frame.label()); 1861 frame.set_as_target(); 1862 } 1863 // Finally reset the stack pointer to the original location. 1864 // The reachability analysis, will ensure it's correctly located 1865 // once reachability is restored. 1866 self.masm.reset_stack_pointer(current_sp); 1867 self.context.reachable = false; 1868 self.context.free_reg(index.reg); 1869 self.context.free_reg(tmp); 1870 } 1871 1872 fn visit_return(&mut self) { 1873 // Grab the outermost frame, which is the function's body 1874 // frame. We don't rely on [`codegen::control_index`] since 1875 // this frame is implicit and we know that it should exist at 1876 // index 0. 1877 let outermost = &mut self.control_frames[0]; 1878 self.context 1879 .unconditional_jump(outermost, self.masm, |masm, cx, frame| { 1880 frame 1881 .pop_abi_results::<M, _>(cx, masm, |results, _, _| results.ret_area().copied()); 1882 }); 1883 } 1884 1885 fn visit_unreachable(&mut self) { 1886 self.masm.unreachable(); 1887 self.context.reachable = false; 1888 // Set the implicit outermost frame as target to perform the necessary 1889 // stack clean up. 1890 let outermost = &mut self.control_frames[0]; 1891 outermost.set_as_target(); 1892 } 1893 1894 fn visit_local_tee(&mut self, index: u32) { 1895 let typed_reg = self.emit_set_local(index); 1896 self.context.stack.push(typed_reg.into()); 1897 } 1898 1899 fn visit_global_get(&mut self, global_index: u32) { 1900 let index = GlobalIndex::from_u32(global_index); 1901 let (ty, addr) = self.emit_get_global_addr(index); 1902 let dst = self.context.reg_for_type(ty, self.masm); 1903 self.masm.load(addr, writable!(dst), ty.into()); 1904 self.context.stack.push(Val::reg(dst, ty)); 1905 } 1906 1907 fn visit_global_set(&mut self, global_index: u32) { 1908 let index = GlobalIndex::from_u32(global_index); 1909 let (ty, addr) = self.emit_get_global_addr(index); 1910 1911 let typed_reg = self.context.pop_to_reg(self.masm, None); 1912 self.context.free_reg(typed_reg.reg); 1913 self.masm.store(typed_reg.reg.into(), addr, ty.into()); 1914 } 1915 1916 fn visit_drop(&mut self) { 1917 self.context.drop_last(1, |regalloc, val| match val { 1918 Val::Reg(tr) => regalloc.free(tr.reg.into()), 1919 Val::Memory(m) => self.masm.free_stack(m.slot.size), 1920 _ => {} 1921 }); 1922 } 1923 1924 fn visit_select(&mut self) { 1925 let cond = self.context.pop_to_reg(self.masm, None); 1926 let val2 = self.context.pop_to_reg(self.masm, None); 1927 let val1 = self.context.pop_to_reg(self.masm, None); 1928 self.masm 1929 .cmp(cond.reg.into(), RegImm::i32(0), OperandSize::S32); 1930 // Conditionally move val1 to val2 if the comparison is 1931 // not zero. 1932 self.masm.cmov( 1933 writable!(val2.into()), 1934 val1.into(), 1935 IntCmpKind::Ne, 1936 val1.ty.into(), 1937 ); 1938 self.context.stack.push(val2.into()); 1939 self.context.free_reg(val1.reg); 1940 self.context.free_reg(cond); 1941 } 1942 1943 fn visit_i32_load(&mut self, memarg: MemArg) { 1944 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S32, None); 1945 } 1946 1947 fn visit_i32_load8_s(&mut self, memarg: MemArg) { 1948 self.emit_wasm_load( 1949 &memarg, 1950 WasmValType::I32, 1951 OperandSize::S8, 1952 Some(ExtendKind::I32Extend8S), 1953 ); 1954 } 1955 1956 fn visit_i32_load8_u(&mut self, memarg: MemArg) { 1957 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S8, None); 1958 } 1959 1960 fn visit_i32_load16_s(&mut self, memarg: MemArg) { 1961 self.emit_wasm_load( 1962 &memarg, 1963 WasmValType::I32, 1964 OperandSize::S16, 1965 Some(ExtendKind::I32Extend16S), 1966 ) 1967 } 1968 1969 fn visit_i32_load16_u(&mut self, memarg: MemArg) { 1970 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S16, None) 1971 } 1972 1973 fn visit_i32_store(&mut self, memarg: MemArg) { 1974 self.emit_wasm_store(&memarg, OperandSize::S32); 1975 } 1976 1977 fn visit_i32_store8(&mut self, memarg: MemArg) { 1978 self.emit_wasm_store(&memarg, OperandSize::S8) 1979 } 1980 1981 fn visit_i32_store16(&mut self, memarg: MemArg) { 1982 self.emit_wasm_store(&memarg, OperandSize::S16) 1983 } 1984 1985 fn visit_i64_load8_s(&mut self, memarg: MemArg) { 1986 self.emit_wasm_load( 1987 &memarg, 1988 WasmValType::I64, 1989 OperandSize::S8, 1990 Some(ExtendKind::I64Extend8S), 1991 ) 1992 } 1993 1994 fn visit_i64_load8_u(&mut self, memarg: MemArg) { 1995 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S8, None) 1996 } 1997 1998 fn visit_i64_load16_u(&mut self, memarg: MemArg) { 1999 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S16, None) 2000 } 2001 2002 fn visit_i64_load16_s(&mut self, memarg: MemArg) { 2003 self.emit_wasm_load( 2004 &memarg, 2005 WasmValType::I64, 2006 OperandSize::S16, 2007 Some(ExtendKind::I64Extend16S), 2008 ) 2009 } 2010 2011 fn visit_i64_load32_u(&mut self, memarg: MemArg) { 2012 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S32, None) 2013 } 2014 2015 fn visit_i64_load32_s(&mut self, memarg: MemArg) { 2016 self.emit_wasm_load( 2017 &memarg, 2018 WasmValType::I64, 2019 OperandSize::S32, 2020 Some(ExtendKind::I64Extend32S), 2021 ) 2022 } 2023 2024 fn visit_i64_load(&mut self, memarg: MemArg) { 2025 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S64, None) 2026 } 2027 2028 fn visit_i64_store(&mut self, memarg: MemArg) -> Self::Output { 2029 self.emit_wasm_store(&memarg, OperandSize::S64) 2030 } 2031 2032 fn visit_i64_store8(&mut self, memarg: MemArg) -> Self::Output { 2033 self.emit_wasm_store(&memarg, OperandSize::S8) 2034 } 2035 2036 fn visit_i64_store16(&mut self, memarg: MemArg) -> Self::Output { 2037 self.emit_wasm_store(&memarg, OperandSize::S16) 2038 } 2039 2040 fn visit_i64_store32(&mut self, memarg: MemArg) -> Self::Output { 2041 self.emit_wasm_store(&memarg, OperandSize::S32) 2042 } 2043 2044 fn visit_f32_load(&mut self, memarg: MemArg) { 2045 self.emit_wasm_load(&memarg, WasmValType::F32, OperandSize::S32, None) 2046 } 2047 2048 fn visit_f32_store(&mut self, memarg: MemArg) { 2049 self.emit_wasm_store(&memarg, OperandSize::S32) 2050 } 2051 2052 fn visit_f64_load(&mut self, memarg: MemArg) { 2053 self.emit_wasm_load(&memarg, WasmValType::F64, OperandSize::S64, None) 2054 } 2055 2056 fn visit_f64_store(&mut self, memarg: MemArg) { 2057 self.emit_wasm_store(&memarg, OperandSize::S64) 2058 } 2059 2060 fn visit_v128_load(&mut self, memarg: MemArg) { 2061 self.emit_wasm_load(&memarg, WasmValType::V128, OperandSize::S128, None) 2062 } 2063 2064 fn visit_v128_store(&mut self, memarg: MemArg) { 2065 self.emit_wasm_store(&memarg, OperandSize::S128) 2066 } 2067 2068 fn visit_i32_trunc_sat_f32_s(&mut self) { 2069 use OperandSize::*; 2070 2071 self.context 2072 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2073 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Checked); 2074 }); 2075 } 2076 2077 fn visit_i32_trunc_sat_f32_u(&mut self) { 2078 use OperandSize::*; 2079 2080 self.context.convert_op_with_tmp_reg( 2081 self.masm, 2082 WasmValType::I32, 2083 RegClass::Float, 2084 |masm, dst, src, tmp_fpr, dst_size| { 2085 masm.unsigned_truncate( 2086 writable!(dst), 2087 src, 2088 tmp_fpr, 2089 S32, 2090 dst_size, 2091 TruncKind::Checked, 2092 ); 2093 }, 2094 ); 2095 } 2096 2097 fn visit_i32_trunc_sat_f64_s(&mut self) { 2098 use OperandSize::*; 2099 2100 self.context 2101 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2102 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Checked); 2103 }); 2104 } 2105 2106 fn visit_i32_trunc_sat_f64_u(&mut self) { 2107 use OperandSize::*; 2108 2109 self.context.convert_op_with_tmp_reg( 2110 self.masm, 2111 WasmValType::I32, 2112 RegClass::Float, 2113 |masm, dst, src, tmp_fpr, dst_size| { 2114 masm.unsigned_truncate( 2115 writable!(dst), 2116 src, 2117 tmp_fpr, 2118 S64, 2119 dst_size, 2120 TruncKind::Checked, 2121 ); 2122 }, 2123 ); 2124 } 2125 2126 fn visit_i64_trunc_sat_f32_s(&mut self) { 2127 use OperandSize::*; 2128 2129 self.context 2130 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2131 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Checked); 2132 }); 2133 } 2134 2135 fn visit_i64_trunc_sat_f32_u(&mut self) { 2136 use OperandSize::*; 2137 2138 self.context.convert_op_with_tmp_reg( 2139 self.masm, 2140 WasmValType::I64, 2141 RegClass::Float, 2142 |masm, dst, src, tmp_fpr, dst_size| { 2143 masm.unsigned_truncate( 2144 writable!(dst), 2145 src, 2146 tmp_fpr, 2147 S32, 2148 dst_size, 2149 TruncKind::Checked, 2150 ); 2151 }, 2152 ); 2153 } 2154 2155 fn visit_i64_trunc_sat_f64_s(&mut self) { 2156 use OperandSize::*; 2157 2158 self.context 2159 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2160 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Checked); 2161 }); 2162 } 2163 2164 fn visit_i64_trunc_sat_f64_u(&mut self) { 2165 use OperandSize::*; 2166 2167 self.context.convert_op_with_tmp_reg( 2168 self.masm, 2169 WasmValType::I64, 2170 RegClass::Float, 2171 |masm, dst, src, tmp_fpr, dst_size| { 2172 masm.unsigned_truncate( 2173 writable!(dst), 2174 src, 2175 tmp_fpr, 2176 S64, 2177 dst_size, 2178 TruncKind::Checked, 2179 ); 2180 }, 2181 ); 2182 } 2183 2184 fn visit_i64_add128(&mut self) { 2185 self.context 2186 .binop128(self.masm, |masm, lhs_lo, lhs_hi, rhs_lo, rhs_hi| { 2187 masm.add128( 2188 writable!(lhs_lo), 2189 writable!(lhs_hi), 2190 lhs_lo, 2191 lhs_hi, 2192 rhs_lo, 2193 rhs_hi, 2194 ); 2195 (TypedReg::i64(lhs_lo), TypedReg::i64(lhs_hi)) 2196 }); 2197 } 2198 2199 fn visit_i64_sub128(&mut self) { 2200 self.context 2201 .binop128(self.masm, |masm, lhs_lo, lhs_hi, rhs_lo, rhs_hi| { 2202 masm.sub128( 2203 writable!(lhs_lo), 2204 writable!(lhs_hi), 2205 lhs_lo, 2206 lhs_hi, 2207 rhs_lo, 2208 rhs_hi, 2209 ); 2210 (TypedReg::i64(lhs_lo), TypedReg::i64(lhs_hi)) 2211 }); 2212 } 2213 2214 fn visit_i64_mul_wide_s(&mut self) { 2215 self.masm.mul_wide(&mut self.context, MulWideKind::Signed); 2216 } 2217 2218 fn visit_i64_mul_wide_u(&mut self) { 2219 self.masm.mul_wide(&mut self.context, MulWideKind::Unsigned); 2220 } 2221 2222 wasmparser::for_each_operator!(def_unsupported); 2223 } 2224 2225 impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M> 2226 where 2227 M: MacroAssembler, 2228 { 2229 fn cmp_i32s(&mut self, kind: IntCmpKind) { 2230 self.context.i32_binop(self.masm, |masm, dst, src, size| { 2231 masm.cmp_with_set(writable!(dst), src, kind, size); 2232 TypedReg::i32(dst) 2233 }); 2234 } 2235 2236 fn cmp_i64s(&mut self, kind: IntCmpKind) { 2237 self.context 2238 .i64_binop(self.masm, move |masm, dst, src, size| { 2239 masm.cmp_with_set(writable!(dst), src, kind, size); 2240 TypedReg::i32(dst) // Return value for comparisons is an `i32`. 2241 }); 2242 } 2243 } 2244 2245 impl From<WasmValType> for OperandSize { 2246 fn from(ty: WasmValType) -> OperandSize { 2247 match ty { 2248 WasmValType::I32 | WasmValType::F32 => OperandSize::S32, 2249 WasmValType::I64 | WasmValType::F64 => OperandSize::S64, 2250 WasmValType::V128 => OperandSize::S128, 2251 WasmValType::Ref(rt) => { 2252 match rt.heap_type { 2253 // TODO: Hardcoded size, assuming 64-bit support only. Once 2254 // Wasmtime supports 32-bit architectures, this will need 2255 // to be updated in such a way that the calculation of the 2256 // OperandSize will depend on the target's pointer size. 2257 WasmHeapType::Func => OperandSize::S64, 2258 WasmHeapType::Extern => OperandSize::S64, 2259 t => unimplemented!("Support for WasmHeapType: {t}"), 2260 } 2261 } 2262 } 2263 } 2264 } 2265