1 //! This module is the central place for machine code emission. 2 //! It defines an implementation of wasmparser's Visitor trait 3 //! for `CodeGen`; which defines a visitor per op-code, 4 //! which validates and dispatches to the corresponding 5 //! machine code emitter. 6 7 use crate::abi::RetArea; 8 use crate::codegen::{control_index, Callee, CodeGen, ControlStackFrame, FnCall}; 9 use crate::masm::{ 10 DivKind, ExtendKind, FloatCmpKind, IntCmpKind, MacroAssembler, MemMoveDirection, OperandSize, 11 RegImm, RemKind, RoundingMode, SPOffset, ShiftKind, TruncKind, 12 }; 13 use crate::stack::{TypedReg, Val}; 14 use cranelift_codegen::ir::TrapCode; 15 use regalloc2::RegClass; 16 use smallvec::SmallVec; 17 use wasmparser::{BlockType, BrTable, Ieee32, Ieee64, MemArg, VisitOperator}; 18 use wasmtime_environ::{ 19 FuncIndex, GlobalIndex, MemoryIndex, TableIndex, TableStyle, TypeIndex, WasmHeapType, 20 WasmValType, FUNCREF_INIT_BIT, 21 }; 22 23 /// A macro to define unsupported WebAssembly operators. 24 /// 25 /// This macro calls itself recursively; 26 /// 1. It no-ops when matching a supported operator. 27 /// 2. Defines the visitor function and panics when 28 /// matching an unsupported operator. 29 macro_rules! def_unsupported { 30 ($( @$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident)*) => { 31 $( 32 def_unsupported!( 33 emit 34 $op 35 36 fn $visit(&mut self $($(,$arg: $argty)*)?) -> Self::Output { 37 $($(let _ = $arg;)*)? 38 todo!(stringify!($op)) 39 } 40 ); 41 )* 42 }; 43 44 (emit I32Const $($rest:tt)*) => {}; 45 (emit I64Const $($rest:tt)*) => {}; 46 (emit F32Const $($rest:tt)*) => {}; 47 (emit F64Const $($rest:tt)*) => {}; 48 (emit F32Add $($rest:tt)*) => {}; 49 (emit F64Add $($rest:tt)*) => {}; 50 (emit F32Sub $($rest:tt)*) => {}; 51 (emit F64Sub $($rest:tt)*) => {}; 52 (emit F32Mul $($rest:tt)*) => {}; 53 (emit F64Mul $($rest:tt)*) => {}; 54 (emit F32Div $($rest:tt)*) => {}; 55 (emit F64Div $($rest:tt)*) => {}; 56 (emit F32Min $($rest:tt)*) => {}; 57 (emit F64Min $($rest:tt)*) => {}; 58 (emit F32Max $($rest:tt)*) => {}; 59 (emit F64Max $($rest:tt)*) => {}; 60 (emit F32Copysign $($rest:tt)*) => {}; 61 (emit F64Copysign $($rest:tt)*) => {}; 62 (emit F32Abs $($rest:tt)*) => {}; 63 (emit F64Abs $($rest:tt)*) => {}; 64 (emit F32Neg $($rest:tt)*) => {}; 65 (emit F64Neg $($rest:tt)*) => {}; 66 (emit F32Floor $($rest:tt)*) => {}; 67 (emit F64Floor $($rest:tt)*) => {}; 68 (emit F32Ceil $($rest:tt)*) => {}; 69 (emit F64Ceil $($rest:tt)*) => {}; 70 (emit F32Nearest $($rest:tt)*) => {}; 71 (emit F64Nearest $($rest:tt)*) => {}; 72 (emit F32Trunc $($rest:tt)*) => {}; 73 (emit F64Trunc $($rest:tt)*) => {}; 74 (emit F32Sqrt $($rest:tt)*) => {}; 75 (emit F64Sqrt $($rest:tt)*) => {}; 76 (emit F32Eq $($rest:tt)*) => {}; 77 (emit F64Eq $($rest:tt)*) => {}; 78 (emit F32Ne $($rest:tt)*) => {}; 79 (emit F64Ne $($rest:tt)*) => {}; 80 (emit F32Lt $($rest:tt)*) => {}; 81 (emit F64Lt $($rest:tt)*) => {}; 82 (emit F32Gt $($rest:tt)*) => {}; 83 (emit F64Gt $($rest:tt)*) => {}; 84 (emit F32Le $($rest:tt)*) => {}; 85 (emit F64Le $($rest:tt)*) => {}; 86 (emit F32Ge $($rest:tt)*) => {}; 87 (emit F64Ge $($rest:tt)*) => {}; 88 (emit F32ConvertI32S $($rest:tt)*) => {}; 89 (emit F32ConvertI32U $($rest:tt)*) => {}; 90 (emit F32ConvertI64S $($rest:tt)*) => {}; 91 (emit F32ConvertI64U $($rest:tt)*) => {}; 92 (emit F64ConvertI32S $($rest:tt)*) => {}; 93 (emit F64ConvertI32U $($rest:tt)*) => {}; 94 (emit F64ConvertI64S $($rest:tt)*) => {}; 95 (emit F64ConvertI64U $($rest:tt)*) => {}; 96 (emit F32ReinterpretI32 $($rest:tt)*) => {}; 97 (emit F64ReinterpretI64 $($rest:tt)*) => {}; 98 (emit F32DemoteF64 $($rest:tt)*) => {}; 99 (emit F64PromoteF32 $($rest:tt)*) => {}; 100 (emit I32Add $($rest:tt)*) => {}; 101 (emit I64Add $($rest:tt)*) => {}; 102 (emit I32Sub $($rest:tt)*) => {}; 103 (emit I32Mul $($rest:tt)*) => {}; 104 (emit I32DivS $($rest:tt)*) => {}; 105 (emit I32DivU $($rest:tt)*) => {}; 106 (emit I64DivS $($rest:tt)*) => {}; 107 (emit I64DivU $($rest:tt)*) => {}; 108 (emit I64RemU $($rest:tt)*) => {}; 109 (emit I64RemS $($rest:tt)*) => {}; 110 (emit I32RemU $($rest:tt)*) => {}; 111 (emit I32RemS $($rest:tt)*) => {}; 112 (emit I64Mul $($rest:tt)*) => {}; 113 (emit I64Sub $($rest:tt)*) => {}; 114 (emit I32Eq $($rest:tt)*) => {}; 115 (emit I64Eq $($rest:tt)*) => {}; 116 (emit I32Ne $($rest:tt)*) => {}; 117 (emit I64Ne $($rest:tt)*) => {}; 118 (emit I32LtS $($rest:tt)*) => {}; 119 (emit I64LtS $($rest:tt)*) => {}; 120 (emit I32LtU $($rest:tt)*) => {}; 121 (emit I64LtU $($rest:tt)*) => {}; 122 (emit I32LeS $($rest:tt)*) => {}; 123 (emit I64LeS $($rest:tt)*) => {}; 124 (emit I32LeU $($rest:tt)*) => {}; 125 (emit I64LeU $($rest:tt)*) => {}; 126 (emit I32GtS $($rest:tt)*) => {}; 127 (emit I64GtS $($rest:tt)*) => {}; 128 (emit I32GtU $($rest:tt)*) => {}; 129 (emit I64GtU $($rest:tt)*) => {}; 130 (emit I32GeS $($rest:tt)*) => {}; 131 (emit I64GeS $($rest:tt)*) => {}; 132 (emit I32GeU $($rest:tt)*) => {}; 133 (emit I64GeU $($rest:tt)*) => {}; 134 (emit I32Eqz $($rest:tt)*) => {}; 135 (emit I64Eqz $($rest:tt)*) => {}; 136 (emit I32And $($rest:tt)*) => {}; 137 (emit I64And $($rest:tt)*) => {}; 138 (emit I32Or $($rest:tt)*) => {}; 139 (emit I64Or $($rest:tt)*) => {}; 140 (emit I32Xor $($rest:tt)*) => {}; 141 (emit I64Xor $($rest:tt)*) => {}; 142 (emit I32Shl $($rest:tt)*) => {}; 143 (emit I64Shl $($rest:tt)*) => {}; 144 (emit I32ShrS $($rest:tt)*) => {}; 145 (emit I64ShrS $($rest:tt)*) => {}; 146 (emit I32ShrU $($rest:tt)*) => {}; 147 (emit I64ShrU $($rest:tt)*) => {}; 148 (emit I32Rotl $($rest:tt)*) => {}; 149 (emit I64Rotl $($rest:tt)*) => {}; 150 (emit I32Rotr $($rest:tt)*) => {}; 151 (emit I64Rotr $($rest:tt)*) => {}; 152 (emit I32Clz $($rest:tt)*) => {}; 153 (emit I64Clz $($rest:tt)*) => {}; 154 (emit I32Ctz $($rest:tt)*) => {}; 155 (emit I64Ctz $($rest:tt)*) => {}; 156 (emit I32Popcnt $($rest:tt)*) => {}; 157 (emit I64Popcnt $($rest:tt)*) => {}; 158 (emit I32WrapI64 $($rest:tt)*) => {}; 159 (emit I64ExtendI32S $($rest:tt)*) => {}; 160 (emit I64ExtendI32U $($rest:tt)*) => {}; 161 (emit I32Extend8S $($rest:tt)*) => {}; 162 (emit I32Extend16S $($rest:tt)*) => {}; 163 (emit I64Extend8S $($rest:tt)*) => {}; 164 (emit I64Extend16S $($rest:tt)*) => {}; 165 (emit I64Extend32S $($rest:tt)*) => {}; 166 (emit I32TruncF32S $($rest:tt)*) => {}; 167 (emit I32TruncF32U $($rest:tt)*) => {}; 168 (emit I32TruncF64S $($rest:tt)*) => {}; 169 (emit I32TruncF64U $($rest:tt)*) => {}; 170 (emit I64TruncF32S $($rest:tt)*) => {}; 171 (emit I64TruncF32U $($rest:tt)*) => {}; 172 (emit I64TruncF64S $($rest:tt)*) => {}; 173 (emit I64TruncF64U $($rest:tt)*) => {}; 174 (emit I32ReinterpretF32 $($rest:tt)*) => {}; 175 (emit I64ReinterpretF64 $($rest:tt)*) => {}; 176 (emit LocalGet $($rest:tt)*) => {}; 177 (emit LocalSet $($rest:tt)*) => {}; 178 (emit Call $($rest:tt)*) => {}; 179 (emit End $($rest:tt)*) => {}; 180 (emit Nop $($rest:tt)*) => {}; 181 (emit If $($rest:tt)*) => {}; 182 (emit Else $($rest:tt)*) => {}; 183 (emit Block $($rest:tt)*) => {}; 184 (emit Loop $($rest:tt)*) => {}; 185 (emit Br $($rest:tt)*) => {}; 186 (emit BrIf $($rest:tt)*) => {}; 187 (emit Return $($rest:tt)*) => {}; 188 (emit Unreachable $($rest:tt)*) => {}; 189 (emit LocalTee $($rest:tt)*) => {}; 190 (emit GlobalGet $($rest:tt)*) => {}; 191 (emit GlobalSet $($rest:tt)*) => {}; 192 (emit Select $($rest:tt)*) => {}; 193 (emit Drop $($rest:tt)*) => {}; 194 (emit BrTable $($rest:tt)*) => {}; 195 (emit CallIndirect $($rest:tt)*) => {}; 196 (emit TableInit $($rest:tt)*) => {}; 197 (emit TableCopy $($rest:tt)*) => {}; 198 (emit TableGet $($rest:tt)*) => {}; 199 (emit TableSet $($rest:tt)*) => {}; 200 (emit TableGrow $($rest:tt)*) => {}; 201 (emit TableSize $($rest:tt)*) => {}; 202 (emit TableFill $($rest:tt)*) => {}; 203 (emit ElemDrop $($rest:tt)*) => {}; 204 (emit MemoryInit $($rest:tt)*) => {}; 205 (emit MemoryCopy $($rest:tt)*) => {}; 206 (emit DataDrop $($rest:tt)*) => {}; 207 (emit MemoryFill $($rest:tt)*) => {}; 208 (emit MemorySize $($rest:tt)*) => {}; 209 (emit MemoryGrow $($rest:tt)*) => {}; 210 (emit I32Load $($rest:tt)*) => {}; 211 (emit I32Load8S $($rest:tt)*) => {}; 212 (emit I32Load8U $($rest:tt)*) => {}; 213 (emit I32Load16S $($rest:tt)*) => {}; 214 (emit I32Load16U $($rest:tt)*) => {}; 215 (emit I64Load8S $($rest:tt)*) => {}; 216 (emit I64Load8U $($rest:tt)*) => {}; 217 (emit I64Load16S $($rest:tt)*) => {}; 218 (emit I64Load16U $($rest:tt)*) => {}; 219 (emit I64Load32S $($rest:tt)*) => {}; 220 (emit I64Load32U $($rest:tt)*) => {}; 221 (emit I64Load $($rest:tt)*) => {}; 222 (emit I32Store $($rest:tt)*) => {}; 223 (emit I32Store $($rest:tt)*) => {}; 224 (emit I32Store8 $($rest:tt)*) => {}; 225 (emit I32Store16 $($rest:tt)*) => {}; 226 (emit I64Store $($rest:tt)*) => {}; 227 (emit I64Store8 $($rest:tt)*) => {}; 228 (emit I64Store16 $($rest:tt)*) => {}; 229 (emit I64Store32 $($rest:tt)*) => {}; 230 (emit F32Load $($rest:tt)*) => {}; 231 (emit F32Store $($rest:tt)*) => {}; 232 (emit F64Load $($rest:tt)*) => {}; 233 (emit F64Store $($rest:tt)*) => {}; 234 (emit I32TruncSatF32S $($rest:tt)*) => {}; 235 (emit I32TruncSatF32U $($rest:tt)*) => {}; 236 (emit I32TruncSatF64S $($rest:tt)*) => {}; 237 (emit I32TruncSatF64U $($rest:tt)*) => {}; 238 (emit I64TruncSatF32S $($rest:tt)*) => {}; 239 (emit I64TruncSatF32U $($rest:tt)*) => {}; 240 (emit I64TruncSatF64S $($rest:tt)*) => {}; 241 (emit I64TruncSatF64U $($rest:tt)*) => {}; 242 243 244 (emit $unsupported:tt $($rest:tt)*) => {$($rest)*}; 245 } 246 247 impl<'a, 'translation, 'data, M> VisitOperator<'a> for CodeGen<'a, 'translation, 'data, M> 248 where 249 M: MacroAssembler, 250 { 251 type Output = (); 252 253 fn visit_i32_const(&mut self, val: i32) { 254 self.context.stack.push(Val::i32(val)); 255 } 256 257 fn visit_i64_const(&mut self, val: i64) { 258 self.context.stack.push(Val::i64(val)); 259 } 260 261 fn visit_f32_const(&mut self, val: Ieee32) { 262 self.context.stack.push(Val::f32(val)); 263 } 264 265 fn visit_f64_const(&mut self, val: Ieee64) { 266 self.context.stack.push(Val::f64(val)); 267 } 268 269 fn visit_f32_add(&mut self) { 270 self.context.binop( 271 self.masm, 272 OperandSize::S32, 273 &mut |masm: &mut M, dst, src, size| { 274 masm.float_add(dst, dst, src, size); 275 TypedReg::f32(dst) 276 }, 277 ); 278 } 279 280 fn visit_f64_add(&mut self) { 281 self.context.binop( 282 self.masm, 283 OperandSize::S64, 284 &mut |masm: &mut M, dst, src, size| { 285 masm.float_add(dst, dst, src, size); 286 TypedReg::f64(dst) 287 }, 288 ); 289 } 290 291 fn visit_f32_sub(&mut self) { 292 self.context.binop( 293 self.masm, 294 OperandSize::S32, 295 &mut |masm: &mut M, dst, src, size| { 296 masm.float_sub(dst, dst, src, size); 297 TypedReg::f32(dst) 298 }, 299 ); 300 } 301 302 fn visit_f64_sub(&mut self) { 303 self.context.binop( 304 self.masm, 305 OperandSize::S64, 306 &mut |masm: &mut M, dst, src, size| { 307 masm.float_sub(dst, dst, src, size); 308 TypedReg::f64(dst) 309 }, 310 ); 311 } 312 313 fn visit_f32_mul(&mut self) { 314 self.context.binop( 315 self.masm, 316 OperandSize::S32, 317 &mut |masm: &mut M, dst, src, size| { 318 masm.float_mul(dst, dst, src, size); 319 TypedReg::f32(dst) 320 }, 321 ); 322 } 323 324 fn visit_f64_mul(&mut self) { 325 self.context.binop( 326 self.masm, 327 OperandSize::S64, 328 &mut |masm: &mut M, dst, src, size| { 329 masm.float_mul(dst, dst, src, size); 330 TypedReg::f64(dst) 331 }, 332 ); 333 } 334 335 fn visit_f32_div(&mut self) { 336 self.context.binop( 337 self.masm, 338 OperandSize::S32, 339 &mut |masm: &mut M, dst, src, size| { 340 masm.float_div(dst, dst, src, size); 341 TypedReg::f32(dst) 342 }, 343 ); 344 } 345 346 fn visit_f64_div(&mut self) { 347 self.context.binop( 348 self.masm, 349 OperandSize::S64, 350 &mut |masm: &mut M, dst, src, size| { 351 masm.float_div(dst, dst, src, size); 352 TypedReg::f64(dst) 353 }, 354 ); 355 } 356 357 fn visit_f32_min(&mut self) { 358 self.context.binop( 359 self.masm, 360 OperandSize::S32, 361 &mut |masm: &mut M, dst, src, size| { 362 masm.float_min(dst, dst, src, size); 363 TypedReg::f32(dst) 364 }, 365 ); 366 } 367 368 fn visit_f64_min(&mut self) { 369 self.context.binop( 370 self.masm, 371 OperandSize::S64, 372 &mut |masm: &mut M, dst, src, size| { 373 masm.float_min(dst, dst, src, size); 374 TypedReg::f64(dst) 375 }, 376 ); 377 } 378 379 fn visit_f32_max(&mut self) { 380 self.context.binop( 381 self.masm, 382 OperandSize::S32, 383 &mut |masm: &mut M, dst, src, size| { 384 masm.float_max(dst, dst, src, size); 385 TypedReg::f32(dst) 386 }, 387 ); 388 } 389 390 fn visit_f64_max(&mut self) { 391 self.context.binop( 392 self.masm, 393 OperandSize::S64, 394 &mut |masm: &mut M, dst, src, size| { 395 masm.float_max(dst, dst, src, size); 396 TypedReg::f64(dst) 397 }, 398 ); 399 } 400 401 fn visit_f32_copysign(&mut self) { 402 self.context.binop( 403 self.masm, 404 OperandSize::S32, 405 &mut |masm: &mut M, dst, src, size| { 406 masm.float_copysign(dst, dst, src, size); 407 TypedReg::f32(dst) 408 }, 409 ); 410 } 411 412 fn visit_f64_copysign(&mut self) { 413 self.context.binop( 414 self.masm, 415 OperandSize::S64, 416 &mut |masm: &mut M, dst, src, size| { 417 masm.float_copysign(dst, dst, src, size); 418 TypedReg::f64(dst) 419 }, 420 ); 421 } 422 423 fn visit_f32_abs(&mut self) { 424 self.context 425 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 426 masm.float_abs(reg, size); 427 TypedReg::f32(reg) 428 }); 429 } 430 431 fn visit_f64_abs(&mut self) { 432 self.context 433 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 434 masm.float_abs(reg, size); 435 TypedReg::f64(reg) 436 }); 437 } 438 439 fn visit_f32_neg(&mut self) { 440 self.context 441 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 442 masm.float_neg(reg, size); 443 TypedReg::f32(reg) 444 }); 445 } 446 447 fn visit_f64_neg(&mut self) { 448 self.context 449 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 450 masm.float_neg(reg, size); 451 TypedReg::f64(reg) 452 }); 453 } 454 455 fn visit_f32_floor(&mut self) { 456 self.masm.float_round( 457 RoundingMode::Down, 458 &mut self.env, 459 &mut self.context, 460 OperandSize::S32, 461 |env, cx, masm| { 462 let builtin = env.builtins.floor_f32::<M::ABI>(); 463 FnCall::emit::<M, M::Ptr>(masm, cx, Callee::Builtin(builtin)); 464 }, 465 ); 466 } 467 468 fn visit_f64_floor(&mut self) { 469 self.masm.float_round( 470 RoundingMode::Down, 471 &mut self.env, 472 &mut self.context, 473 OperandSize::S64, 474 |env, cx, masm| { 475 let builtin = env.builtins.floor_f64::<M::ABI>(); 476 FnCall::emit::<M, M::Ptr>(masm, cx, Callee::Builtin(builtin)); 477 }, 478 ); 479 } 480 481 fn visit_f32_ceil(&mut self) { 482 self.masm.float_round( 483 RoundingMode::Up, 484 &mut self.env, 485 &mut self.context, 486 OperandSize::S32, 487 |env, cx, masm| { 488 let builtin = env.builtins.ceil_f32::<M::ABI>(); 489 FnCall::emit::<M, M::Ptr>(masm, cx, Callee::Builtin(builtin)); 490 }, 491 ); 492 } 493 494 fn visit_f64_ceil(&mut self) { 495 self.masm.float_round( 496 RoundingMode::Up, 497 &mut self.env, 498 &mut self.context, 499 OperandSize::S64, 500 |env, cx, masm| { 501 let builtin = env.builtins.ceil_f64::<M::ABI>(); 502 FnCall::emit::<M, M::Ptr>(masm, cx, Callee::Builtin(builtin)); 503 }, 504 ); 505 } 506 507 fn visit_f32_nearest(&mut self) { 508 self.masm.float_round( 509 RoundingMode::Nearest, 510 &mut self.env, 511 &mut self.context, 512 OperandSize::S32, 513 |env, cx, masm| { 514 let builtin = env.builtins.nearest_f32::<M::ABI>(); 515 FnCall::emit::<M, M::Ptr>(masm, cx, Callee::Builtin(builtin)) 516 }, 517 ); 518 } 519 520 fn visit_f64_nearest(&mut self) { 521 self.masm.float_round( 522 RoundingMode::Nearest, 523 &mut self.env, 524 &mut self.context, 525 OperandSize::S64, 526 |env, cx, masm| { 527 let builtin = env.builtins.nearest_f64::<M::ABI>(); 528 FnCall::emit::<M, M::Ptr>(masm, cx, Callee::Builtin(builtin)); 529 }, 530 ); 531 } 532 533 fn visit_f32_trunc(&mut self) { 534 self.masm.float_round( 535 RoundingMode::Zero, 536 &mut self.env, 537 &mut self.context, 538 OperandSize::S32, 539 |env, cx, masm| { 540 let builtin = env.builtins.trunc_f32::<M::ABI>(); 541 FnCall::emit::<M, M::Ptr>(masm, cx, Callee::Builtin(builtin)); 542 }, 543 ); 544 } 545 546 fn visit_f64_trunc(&mut self) { 547 self.masm.float_round( 548 RoundingMode::Zero, 549 &mut self.env, 550 &mut self.context, 551 OperandSize::S64, 552 |env, cx, masm| { 553 let builtin = env.builtins.trunc_f64::<M::ABI>(); 554 FnCall::emit::<M, M::Ptr>(masm, cx, Callee::Builtin(builtin)); 555 }, 556 ); 557 } 558 559 fn visit_f32_sqrt(&mut self) { 560 self.context 561 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 562 masm.float_sqrt(reg, reg, size); 563 TypedReg::f32(reg) 564 }); 565 } 566 567 fn visit_f64_sqrt(&mut self) { 568 self.context 569 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 570 masm.float_sqrt(reg, reg, size); 571 TypedReg::f64(reg) 572 }); 573 } 574 575 fn visit_f32_eq(&mut self) { 576 self.context.float_cmp_op( 577 self.masm, 578 OperandSize::S32, 579 &mut |masm: &mut M, dst, src1, src2, size| { 580 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Eq, size); 581 }, 582 ); 583 } 584 585 fn visit_f64_eq(&mut self) { 586 self.context.float_cmp_op( 587 self.masm, 588 OperandSize::S64, 589 &mut |masm: &mut M, dst, src1, src2, size| { 590 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Eq, size); 591 }, 592 ); 593 } 594 595 fn visit_f32_ne(&mut self) { 596 self.context.float_cmp_op( 597 self.masm, 598 OperandSize::S32, 599 &mut |masm: &mut M, dst, src1, src2, size| { 600 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Ne, size); 601 }, 602 ); 603 } 604 605 fn visit_f64_ne(&mut self) { 606 self.context.float_cmp_op( 607 self.masm, 608 OperandSize::S64, 609 &mut |masm: &mut M, dst, src1, src2, size| { 610 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Ne, size); 611 }, 612 ); 613 } 614 615 fn visit_f32_lt(&mut self) { 616 self.context.float_cmp_op( 617 self.masm, 618 OperandSize::S32, 619 &mut |masm: &mut M, dst, src1, src2, size| { 620 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Lt, size); 621 }, 622 ); 623 } 624 625 fn visit_f64_lt(&mut self) { 626 self.context.float_cmp_op( 627 self.masm, 628 OperandSize::S64, 629 &mut |masm: &mut M, dst, src1, src2, size| { 630 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Lt, size); 631 }, 632 ); 633 } 634 635 fn visit_f32_gt(&mut self) { 636 self.context.float_cmp_op( 637 self.masm, 638 OperandSize::S32, 639 &mut |masm: &mut M, dst, src1, src2, size| { 640 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Gt, size); 641 }, 642 ); 643 } 644 645 fn visit_f64_gt(&mut self) { 646 self.context.float_cmp_op( 647 self.masm, 648 OperandSize::S64, 649 &mut |masm: &mut M, dst, src1, src2, size| { 650 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Gt, size); 651 }, 652 ); 653 } 654 655 fn visit_f32_le(&mut self) { 656 self.context.float_cmp_op( 657 self.masm, 658 OperandSize::S32, 659 &mut |masm: &mut M, dst, src1, src2, size| { 660 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Le, size); 661 }, 662 ); 663 } 664 665 fn visit_f64_le(&mut self) { 666 self.context.float_cmp_op( 667 self.masm, 668 OperandSize::S64, 669 &mut |masm: &mut M, dst, src1, src2, size| { 670 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Le, size); 671 }, 672 ); 673 } 674 675 fn visit_f32_ge(&mut self) { 676 self.context.float_cmp_op( 677 self.masm, 678 OperandSize::S32, 679 &mut |masm: &mut M, dst, src1, src2, size| { 680 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Ge, size); 681 }, 682 ); 683 } 684 685 fn visit_f64_ge(&mut self) { 686 self.context.float_cmp_op( 687 self.masm, 688 OperandSize::S64, 689 &mut |masm: &mut M, dst, src1, src2, size| { 690 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Ge, size); 691 }, 692 ); 693 } 694 695 fn visit_f32_convert_i32_s(&mut self) { 696 self.context 697 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 698 masm.signed_convert(src, dst, OperandSize::S32, dst_size); 699 }); 700 } 701 702 fn visit_f32_convert_i32_u(&mut self) { 703 self.context.convert_op_with_tmp_reg( 704 self.masm, 705 WasmValType::F32, 706 RegClass::Int, 707 |masm, dst, src, tmp_gpr, dst_size| { 708 masm.unsigned_convert(src, dst, tmp_gpr, OperandSize::S32, dst_size); 709 }, 710 ); 711 } 712 713 fn visit_f32_convert_i64_s(&mut self) { 714 self.context 715 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 716 masm.signed_convert(src, dst, OperandSize::S64, dst_size); 717 }); 718 } 719 720 fn visit_f32_convert_i64_u(&mut self) { 721 self.context.convert_op_with_tmp_reg( 722 self.masm, 723 WasmValType::F32, 724 RegClass::Int, 725 |masm, dst, src, tmp_gpr, dst_size| { 726 masm.unsigned_convert(src, dst, tmp_gpr, OperandSize::S64, dst_size); 727 }, 728 ); 729 } 730 731 fn visit_f64_convert_i32_s(&mut self) { 732 self.context 733 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 734 masm.signed_convert(src, dst, OperandSize::S32, dst_size); 735 }); 736 } 737 738 fn visit_f64_convert_i32_u(&mut self) { 739 self.context.convert_op_with_tmp_reg( 740 self.masm, 741 WasmValType::F64, 742 RegClass::Int, 743 |masm, dst, src, tmp_gpr, dst_size| { 744 masm.unsigned_convert(src, dst, tmp_gpr, OperandSize::S32, dst_size); 745 }, 746 ); 747 } 748 749 fn visit_f64_convert_i64_s(&mut self) { 750 self.context 751 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 752 masm.signed_convert(src, dst, OperandSize::S64, dst_size); 753 }); 754 } 755 756 fn visit_f64_convert_i64_u(&mut self) { 757 self.context.convert_op_with_tmp_reg( 758 self.masm, 759 WasmValType::F64, 760 RegClass::Int, 761 |masm, dst, src, tmp_gpr, dst_size| { 762 masm.unsigned_convert(src, dst, tmp_gpr, OperandSize::S64, dst_size); 763 }, 764 ); 765 } 766 767 fn visit_f32_reinterpret_i32(&mut self) { 768 self.context 769 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, size| { 770 masm.reinterpret_int_as_float(src.into(), dst, size); 771 }); 772 } 773 774 fn visit_f64_reinterpret_i64(&mut self) { 775 self.context 776 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, size| { 777 masm.reinterpret_int_as_float(src.into(), dst, size); 778 }); 779 } 780 781 fn visit_f32_demote_f64(&mut self) { 782 self.context 783 .unop(self.masm, OperandSize::S64, &mut |masm, reg, _size| { 784 masm.demote(reg, reg); 785 TypedReg::f32(reg) 786 }); 787 } 788 789 fn visit_f64_promote_f32(&mut self) { 790 self.context 791 .unop(self.masm, OperandSize::S32, &mut |masm, reg, _size| { 792 masm.promote(reg, reg); 793 TypedReg::f64(reg) 794 }); 795 } 796 797 fn visit_i32_add(&mut self) { 798 self.context.i32_binop(self.masm, |masm, dst, src, size| { 799 masm.add(dst, dst, src, size); 800 TypedReg::i32(dst) 801 }); 802 } 803 804 fn visit_i64_add(&mut self) { 805 self.context.i64_binop(self.masm, |masm, dst, src, size| { 806 masm.add(dst, dst, src, size); 807 TypedReg::i64(dst) 808 }); 809 } 810 811 fn visit_i32_sub(&mut self) { 812 self.context.i32_binop(self.masm, |masm, dst, src, size| { 813 masm.sub(dst, dst, src, size); 814 TypedReg::i32(dst) 815 }); 816 } 817 818 fn visit_i64_sub(&mut self) { 819 self.context.i64_binop(self.masm, |masm, dst, src, size| { 820 masm.sub(dst, dst, src, size); 821 TypedReg::i64(dst) 822 }); 823 } 824 825 fn visit_i32_mul(&mut self) { 826 self.context.i32_binop(self.masm, |masm, dst, src, size| { 827 masm.mul(dst, dst, src, size); 828 TypedReg::i32(dst) 829 }); 830 } 831 832 fn visit_i64_mul(&mut self) { 833 self.context.i64_binop(self.masm, |masm, dst, src, size| { 834 masm.mul(dst, dst, src, size); 835 TypedReg::i64(dst) 836 }); 837 } 838 839 fn visit_i32_div_s(&mut self) { 840 use DivKind::*; 841 use OperandSize::*; 842 843 self.masm.div(&mut self.context, Signed, S32); 844 } 845 846 fn visit_i32_div_u(&mut self) { 847 use DivKind::*; 848 use OperandSize::*; 849 850 self.masm.div(&mut self.context, Unsigned, S32); 851 } 852 853 fn visit_i64_div_s(&mut self) { 854 use DivKind::*; 855 use OperandSize::*; 856 857 self.masm.div(&mut self.context, Signed, S64); 858 } 859 860 fn visit_i64_div_u(&mut self) { 861 use DivKind::*; 862 use OperandSize::*; 863 864 self.masm.div(&mut self.context, Unsigned, S64); 865 } 866 867 fn visit_i32_rem_s(&mut self) { 868 use OperandSize::*; 869 use RemKind::*; 870 871 self.masm.rem(&mut self.context, Signed, S32); 872 } 873 874 fn visit_i32_rem_u(&mut self) { 875 use OperandSize::*; 876 use RemKind::*; 877 878 self.masm.rem(&mut self.context, Unsigned, S32); 879 } 880 881 fn visit_i64_rem_s(&mut self) { 882 use OperandSize::*; 883 use RemKind::*; 884 885 self.masm.rem(&mut self.context, Signed, S64); 886 } 887 888 fn visit_i64_rem_u(&mut self) { 889 use OperandSize::*; 890 use RemKind::*; 891 892 self.masm.rem(&mut self.context, Unsigned, S64); 893 } 894 895 fn visit_i32_eq(&mut self) { 896 self.cmp_i32s(IntCmpKind::Eq); 897 } 898 899 fn visit_i64_eq(&mut self) { 900 self.cmp_i64s(IntCmpKind::Eq); 901 } 902 903 fn visit_i32_ne(&mut self) { 904 self.cmp_i32s(IntCmpKind::Ne); 905 } 906 907 fn visit_i64_ne(&mut self) { 908 self.cmp_i64s(IntCmpKind::Ne); 909 } 910 911 fn visit_i32_lt_s(&mut self) { 912 self.cmp_i32s(IntCmpKind::LtS); 913 } 914 915 fn visit_i64_lt_s(&mut self) { 916 self.cmp_i64s(IntCmpKind::LtS); 917 } 918 919 fn visit_i32_lt_u(&mut self) { 920 self.cmp_i32s(IntCmpKind::LtU); 921 } 922 923 fn visit_i64_lt_u(&mut self) { 924 self.cmp_i64s(IntCmpKind::LtU); 925 } 926 927 fn visit_i32_le_s(&mut self) { 928 self.cmp_i32s(IntCmpKind::LeS); 929 } 930 931 fn visit_i64_le_s(&mut self) { 932 self.cmp_i64s(IntCmpKind::LeS); 933 } 934 935 fn visit_i32_le_u(&mut self) { 936 self.cmp_i32s(IntCmpKind::LeU); 937 } 938 939 fn visit_i64_le_u(&mut self) { 940 self.cmp_i64s(IntCmpKind::LeU); 941 } 942 943 fn visit_i32_gt_s(&mut self) { 944 self.cmp_i32s(IntCmpKind::GtS); 945 } 946 947 fn visit_i64_gt_s(&mut self) { 948 self.cmp_i64s(IntCmpKind::GtS); 949 } 950 951 fn visit_i32_gt_u(&mut self) { 952 self.cmp_i32s(IntCmpKind::GtU); 953 } 954 955 fn visit_i64_gt_u(&mut self) { 956 self.cmp_i64s(IntCmpKind::GtU); 957 } 958 959 fn visit_i32_ge_s(&mut self) { 960 self.cmp_i32s(IntCmpKind::GeS); 961 } 962 963 fn visit_i64_ge_s(&mut self) { 964 self.cmp_i64s(IntCmpKind::GeS); 965 } 966 967 fn visit_i32_ge_u(&mut self) { 968 self.cmp_i32s(IntCmpKind::GeU); 969 } 970 971 fn visit_i64_ge_u(&mut self) { 972 self.cmp_i64s(IntCmpKind::GeU); 973 } 974 975 fn visit_i32_eqz(&mut self) { 976 use OperandSize::*; 977 978 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 979 masm.cmp_with_set(RegImm::i32(0), reg.into(), IntCmpKind::Eq, size); 980 TypedReg::i32(reg) 981 }); 982 } 983 984 fn visit_i64_eqz(&mut self) { 985 use OperandSize::*; 986 987 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 988 masm.cmp_with_set(RegImm::i64(0), reg.into(), IntCmpKind::Eq, size); 989 TypedReg::i32(reg) // Return value for `i64.eqz` is an `i32`. 990 }); 991 } 992 993 fn visit_i32_clz(&mut self) { 994 use OperandSize::*; 995 996 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 997 masm.clz(reg, reg, size); 998 TypedReg::i32(reg) 999 }); 1000 } 1001 1002 fn visit_i64_clz(&mut self) { 1003 use OperandSize::*; 1004 1005 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 1006 masm.clz(reg, reg, size); 1007 TypedReg::i64(reg) 1008 }); 1009 } 1010 1011 fn visit_i32_ctz(&mut self) { 1012 use OperandSize::*; 1013 1014 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 1015 masm.ctz(reg, reg, size); 1016 TypedReg::i32(reg) 1017 }); 1018 } 1019 1020 fn visit_i64_ctz(&mut self) { 1021 use OperandSize::*; 1022 1023 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 1024 masm.ctz(reg, reg, size); 1025 TypedReg::i64(reg) 1026 }); 1027 } 1028 1029 fn visit_i32_and(&mut self) { 1030 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1031 masm.and(dst, dst, src, size); 1032 TypedReg::i32(dst) 1033 }); 1034 } 1035 1036 fn visit_i64_and(&mut self) { 1037 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1038 masm.and(dst, dst, src, size); 1039 TypedReg::i64(dst) 1040 }); 1041 } 1042 1043 fn visit_i32_or(&mut self) { 1044 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1045 masm.or(dst, dst, src, size); 1046 TypedReg::i32(dst) 1047 }); 1048 } 1049 1050 fn visit_i64_or(&mut self) { 1051 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1052 masm.or(dst, dst, src, size); 1053 TypedReg::i64(dst) 1054 }); 1055 } 1056 1057 fn visit_i32_xor(&mut self) { 1058 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1059 masm.xor(dst, dst, src, size); 1060 TypedReg::i32(dst) 1061 }); 1062 } 1063 1064 fn visit_i64_xor(&mut self) { 1065 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1066 masm.xor(dst, dst, src, size); 1067 TypedReg::i64(dst) 1068 }); 1069 } 1070 1071 fn visit_i32_shl(&mut self) { 1072 use OperandSize::*; 1073 use ShiftKind::*; 1074 1075 self.masm.shift(&mut self.context, Shl, S32); 1076 } 1077 1078 fn visit_i64_shl(&mut self) { 1079 use OperandSize::*; 1080 use ShiftKind::*; 1081 1082 self.masm.shift(&mut self.context, Shl, S64); 1083 } 1084 1085 fn visit_i32_shr_s(&mut self) { 1086 use OperandSize::*; 1087 use ShiftKind::*; 1088 1089 self.masm.shift(&mut self.context, ShrS, S32); 1090 } 1091 1092 fn visit_i64_shr_s(&mut self) { 1093 use OperandSize::*; 1094 use ShiftKind::*; 1095 1096 self.masm.shift(&mut self.context, ShrS, S64); 1097 } 1098 1099 fn visit_i32_shr_u(&mut self) { 1100 use OperandSize::*; 1101 use ShiftKind::*; 1102 1103 self.masm.shift(&mut self.context, ShrU, S32); 1104 } 1105 1106 fn visit_i64_shr_u(&mut self) { 1107 use OperandSize::*; 1108 use ShiftKind::*; 1109 1110 self.masm.shift(&mut self.context, ShrU, S64); 1111 } 1112 1113 fn visit_i32_rotl(&mut self) { 1114 use OperandSize::*; 1115 use ShiftKind::*; 1116 1117 self.masm.shift(&mut self.context, Rotl, S32); 1118 } 1119 1120 fn visit_i64_rotl(&mut self) { 1121 use OperandSize::*; 1122 use ShiftKind::*; 1123 1124 self.masm.shift(&mut self.context, Rotl, S64); 1125 } 1126 1127 fn visit_i32_rotr(&mut self) { 1128 use OperandSize::*; 1129 use ShiftKind::*; 1130 1131 self.masm.shift(&mut self.context, Rotr, S32); 1132 } 1133 1134 fn visit_i64_rotr(&mut self) { 1135 use OperandSize::*; 1136 use ShiftKind::*; 1137 1138 self.masm.shift(&mut self.context, Rotr, S64); 1139 } 1140 1141 fn visit_end(&mut self) { 1142 if !self.context.reachable { 1143 self.handle_unreachable_end(); 1144 } else { 1145 let mut control = self.control_frames.pop().unwrap(); 1146 control.emit_end(self.masm, &mut self.context); 1147 } 1148 } 1149 1150 fn visit_i32_popcnt(&mut self) { 1151 use OperandSize::*; 1152 self.masm.popcnt(&mut self.context, S32); 1153 } 1154 1155 fn visit_i64_popcnt(&mut self) { 1156 use OperandSize::*; 1157 1158 self.masm.popcnt(&mut self.context, S64); 1159 } 1160 1161 fn visit_i32_wrap_i64(&mut self) { 1162 use OperandSize::*; 1163 1164 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1165 masm.wrap(reg, reg); 1166 TypedReg::i32(reg) 1167 }); 1168 } 1169 1170 fn visit_i64_extend_i32_s(&mut self) { 1171 use OperandSize::*; 1172 1173 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1174 masm.extend(reg, reg, ExtendKind::I64ExtendI32S); 1175 TypedReg::i64(reg) 1176 }); 1177 } 1178 1179 fn visit_i64_extend_i32_u(&mut self) { 1180 use OperandSize::*; 1181 1182 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1183 masm.extend(reg, reg, ExtendKind::I64ExtendI32U); 1184 TypedReg::i64(reg) 1185 }); 1186 } 1187 1188 fn visit_i32_extend8_s(&mut self) { 1189 use OperandSize::*; 1190 1191 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1192 masm.extend(reg, reg, ExtendKind::I32Extend8S); 1193 TypedReg::i32(reg) 1194 }); 1195 } 1196 1197 fn visit_i32_extend16_s(&mut self) { 1198 use OperandSize::*; 1199 1200 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1201 masm.extend(reg, reg, ExtendKind::I32Extend16S); 1202 TypedReg::i32(reg) 1203 }); 1204 } 1205 1206 fn visit_i64_extend8_s(&mut self) { 1207 use OperandSize::*; 1208 1209 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1210 masm.extend(reg, reg, ExtendKind::I64Extend8S); 1211 TypedReg::i64(reg) 1212 }); 1213 } 1214 1215 fn visit_i64_extend16_s(&mut self) { 1216 use OperandSize::*; 1217 1218 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1219 masm.extend(reg, reg, ExtendKind::I64Extend16S); 1220 TypedReg::i64(reg) 1221 }); 1222 } 1223 1224 fn visit_i64_extend32_s(&mut self) { 1225 use OperandSize::*; 1226 1227 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1228 masm.extend(reg, reg, ExtendKind::I64Extend32S); 1229 TypedReg::i64(reg) 1230 }); 1231 } 1232 1233 fn visit_i32_trunc_f32_s(&mut self) { 1234 use OperandSize::*; 1235 1236 self.context 1237 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1238 masm.signed_truncate(src, dst, S32, dst_size, TruncKind::Unchecked); 1239 }); 1240 } 1241 1242 fn visit_i32_trunc_f32_u(&mut self) { 1243 use OperandSize::*; 1244 1245 self.context.convert_op_with_tmp_reg( 1246 self.masm, 1247 WasmValType::I32, 1248 RegClass::Float, 1249 |masm, dst, src, tmp_fpr, dst_size| { 1250 masm.unsigned_truncate(src, dst, tmp_fpr, S32, dst_size, TruncKind::Unchecked); 1251 }, 1252 ); 1253 } 1254 1255 fn visit_i32_trunc_f64_s(&mut self) { 1256 use OperandSize::*; 1257 1258 self.context 1259 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1260 masm.signed_truncate(src, dst, S64, dst_size, TruncKind::Unchecked); 1261 }); 1262 } 1263 1264 fn visit_i32_trunc_f64_u(&mut self) { 1265 use OperandSize::*; 1266 1267 self.context.convert_op_with_tmp_reg( 1268 self.masm, 1269 WasmValType::I32, 1270 RegClass::Float, 1271 |masm, dst, src, tmp_fpr, dst_size| { 1272 masm.unsigned_truncate(src, dst, tmp_fpr, S64, dst_size, TruncKind::Unchecked); 1273 }, 1274 ); 1275 } 1276 1277 fn visit_i64_trunc_f32_s(&mut self) { 1278 use OperandSize::*; 1279 1280 self.context 1281 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1282 masm.signed_truncate(src, dst, S32, dst_size, TruncKind::Unchecked); 1283 }); 1284 } 1285 1286 fn visit_i64_trunc_f32_u(&mut self) { 1287 use OperandSize::*; 1288 1289 self.context.convert_op_with_tmp_reg( 1290 self.masm, 1291 WasmValType::I64, 1292 RegClass::Float, 1293 |masm, dst, src, tmp_fpr, dst_size| { 1294 masm.unsigned_truncate(src, dst, tmp_fpr, S32, dst_size, TruncKind::Unchecked); 1295 }, 1296 ); 1297 } 1298 1299 fn visit_i64_trunc_f64_s(&mut self) { 1300 use OperandSize::*; 1301 1302 self.context 1303 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1304 masm.signed_truncate(src, dst, S64, dst_size, TruncKind::Unchecked); 1305 }); 1306 } 1307 1308 fn visit_i64_trunc_f64_u(&mut self) { 1309 use OperandSize::*; 1310 1311 self.context.convert_op_with_tmp_reg( 1312 self.masm, 1313 WasmValType::I64, 1314 RegClass::Float, 1315 |masm, dst, src, tmp_fpr, dst_size| { 1316 masm.unsigned_truncate(src, dst, tmp_fpr, S64, dst_size, TruncKind::Unchecked); 1317 }, 1318 ); 1319 } 1320 1321 fn visit_i32_reinterpret_f32(&mut self) { 1322 self.context 1323 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, size| { 1324 masm.reinterpret_float_as_int(src.into(), dst, size); 1325 }); 1326 } 1327 1328 fn visit_i64_reinterpret_f64(&mut self) { 1329 self.context 1330 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, size| { 1331 masm.reinterpret_float_as_int(src.into(), dst, size); 1332 }); 1333 } 1334 1335 fn visit_local_get(&mut self, index: u32) { 1336 use WasmValType::*; 1337 let context = &mut self.context; 1338 let slot = context.frame.get_wasm_local(index); 1339 match slot.ty { 1340 I32 | I64 | F32 | F64 => context.stack.push(Val::local(index, slot.ty)), 1341 Ref(rt) => match rt.heap_type { 1342 WasmHeapType::Func => context.stack.push(Val::local(index, slot.ty)), 1343 ht => unimplemented!("Support for WasmHeapType: {ht}"), 1344 }, 1345 t => unimplemented!("Support local type: {t}"), 1346 } 1347 } 1348 1349 fn visit_local_set(&mut self, index: u32) { 1350 let src = self.emit_set_local(index); 1351 self.context.free_reg(src); 1352 } 1353 1354 fn visit_call(&mut self, index: u32) { 1355 FnCall::emit::<M, M::Ptr>( 1356 self.masm, 1357 &mut self.context, 1358 self.env 1359 .callee_from_index::<M::ABI>(FuncIndex::from_u32(index)), 1360 ) 1361 } 1362 1363 fn visit_call_indirect(&mut self, type_index: u32, table_index: u32, _: u8) { 1364 // Spill now because `emit_lazy_init_funcref` and the `FnCall::emit` 1365 // invocations will both trigger spills since they both call functions. 1366 // However, the machine instructions for the spill emitted by 1367 // `emit_lazy_funcref` will be jumped over if the funcref was previously 1368 // initialized which may result in the machine stack becoming 1369 // unbalanced. 1370 self.context.spill(self.masm); 1371 1372 let type_index = TypeIndex::from_u32(type_index); 1373 let table_index = TableIndex::from_u32(table_index); 1374 1375 self.emit_lazy_init_funcref(table_index); 1376 1377 // Perform the indirect call. 1378 // This code assumes that [`Self::emit_lazy_init_funcref`] will 1379 // push the funcref to the value stack. 1380 match self.env.translation.module.table_plans[table_index].style { 1381 TableStyle::CallerChecksSignature => { 1382 let funcref_ptr = self.context.stack.peek().map(|v| v.unwrap_reg()).unwrap(); 1383 self.masm 1384 .trapz(funcref_ptr.into(), TrapCode::IndirectCallToNull); 1385 self.emit_typecheck_funcref(funcref_ptr.into(), type_index); 1386 } 1387 } 1388 1389 FnCall::emit::<M, M::Ptr>( 1390 self.masm, 1391 &mut self.context, 1392 self.env.funcref::<M::ABI>(type_index), 1393 ) 1394 } 1395 1396 fn visit_table_init(&mut self, elem: u32, table: u32) { 1397 debug_assert!(self.context.stack.len() >= 3); 1398 let at = self.context.stack.len() - 3; 1399 1400 self.context 1401 .stack 1402 .insert_many(at, &[table.try_into().unwrap(), elem.try_into().unwrap()]); 1403 1404 let builtin = self.env.builtins.table_init::<M::ABI, M::Ptr>(); 1405 FnCall::emit::<M, M::Ptr>( 1406 self.masm, 1407 &mut self.context, 1408 Callee::Builtin(builtin.clone()), 1409 ) 1410 } 1411 1412 fn visit_table_copy(&mut self, dst: u32, src: u32) { 1413 debug_assert!(self.context.stack.len() >= 3); 1414 let at = self.context.stack.len() - 3; 1415 self.context 1416 .stack 1417 .insert_many(at, &[dst.try_into().unwrap(), src.try_into().unwrap()]); 1418 1419 let builtin = self.env.builtins.table_copy::<M::ABI, M::Ptr>(); 1420 FnCall::emit::<M, M::Ptr>(self.masm, &mut self.context, Callee::Builtin(builtin)) 1421 } 1422 1423 fn visit_table_get(&mut self, table: u32) { 1424 let table_index = TableIndex::from_u32(table); 1425 let plan = self.env.table_plan(table_index); 1426 let heap_type = plan.table.wasm_ty.heap_type; 1427 let style = &plan.style; 1428 1429 match heap_type { 1430 WasmHeapType::Func => match style { 1431 TableStyle::CallerChecksSignature => self.emit_lazy_init_funcref(table_index), 1432 }, 1433 t => unimplemented!("Support for WasmHeapType: {t}"), 1434 } 1435 } 1436 1437 fn visit_table_grow(&mut self, table: u32) { 1438 let table_index = TableIndex::from_u32(table); 1439 let table_plan = self.env.table_plan(table_index); 1440 let builtin = match table_plan.table.wasm_ty.heap_type { 1441 WasmHeapType::Func => self.env.builtins.table_grow_func_ref::<M::ABI, M::Ptr>(), 1442 ty => unimplemented!("Support for HeapType: {ty}"), 1443 }; 1444 1445 let len = self.context.stack.len(); 1446 // table.grow` requires at least 2 elements on the value stack. 1447 debug_assert!(len >= 2); 1448 let at = len - 2; 1449 1450 // The table_grow builtin expects the parameters in a different 1451 // order. 1452 // The value stack at this point should contain: 1453 // [ init_value | delta ] (stack top) 1454 // but the builtin function expects the init value as the last 1455 // argument. 1456 self.context.stack.inner_mut().swap(len - 1, len - 2); 1457 self.context 1458 .stack 1459 .insert_many(at, &[table.try_into().unwrap()]); 1460 1461 FnCall::emit::<M, M::Ptr>( 1462 self.masm, 1463 &mut self.context, 1464 Callee::Builtin(builtin.clone()), 1465 ) 1466 } 1467 1468 fn visit_table_size(&mut self, table: u32) { 1469 let table_index = TableIndex::from_u32(table); 1470 let table_data = self.env.resolve_table_data(table_index); 1471 self.masm.table_size(&table_data, &mut self.context); 1472 } 1473 1474 fn visit_table_fill(&mut self, table: u32) { 1475 let table_index = TableIndex::from_u32(table); 1476 let table_plan = self.env.table_plan(table_index); 1477 let builtin = match table_plan.table.wasm_ty.heap_type { 1478 WasmHeapType::Func => self.env.builtins.table_fill_func_ref::<M::ABI, M::Ptr>(), 1479 ty => unimplemented!("Support for heap type: {ty}"), 1480 }; 1481 1482 let len = self.context.stack.len(); 1483 debug_assert!(len >= 3); 1484 let at = len - 3; 1485 self.context 1486 .stack 1487 .insert_many(at, &[table.try_into().unwrap()]); 1488 FnCall::emit::<M, M::Ptr>( 1489 self.masm, 1490 &mut self.context, 1491 Callee::Builtin(builtin.clone()), 1492 ) 1493 } 1494 1495 fn visit_table_set(&mut self, table: u32) { 1496 let ptr_type = self.env.ptr_type(); 1497 let table_index = TableIndex::from_u32(table); 1498 let table_data = self.env.resolve_table_data(table_index); 1499 let plan = self.env.table_plan(table_index); 1500 match plan.table.wasm_ty.heap_type { 1501 WasmHeapType::Func => match plan.style { 1502 TableStyle::CallerChecksSignature => { 1503 let value = self.context.pop_to_reg(self.masm, None); 1504 let index = self.context.pop_to_reg(self.masm, None); 1505 let base = self.context.any_gpr(self.masm); 1506 let elem_addr = self.masm.table_elem_address( 1507 index.into(), 1508 base, 1509 &table_data, 1510 &mut self.context, 1511 ); 1512 1513 // Set the initialized bit. 1514 self.masm.or( 1515 value.into(), 1516 value.into(), 1517 RegImm::i64(FUNCREF_INIT_BIT as i64), 1518 ptr_type.into(), 1519 ); 1520 1521 self.masm.store_ptr(value.into(), elem_addr); 1522 1523 self.context.free_reg(value); 1524 self.context.free_reg(index); 1525 self.context.free_reg(base); 1526 } 1527 }, 1528 ty => unimplemented!("Support for WasmHeapType: {ty}"), 1529 }; 1530 } 1531 1532 fn visit_elem_drop(&mut self, index: u32) { 1533 let elem_drop = self.env.builtins.elem_drop::<M::ABI, M::Ptr>(); 1534 self.context.stack.extend([index.try_into().unwrap()]); 1535 FnCall::emit::<M, M::Ptr>(self.masm, &mut self.context, Callee::Builtin(elem_drop)) 1536 } 1537 1538 fn visit_memory_init(&mut self, data_index: u32, mem: u32) { 1539 debug_assert!(self.context.stack.len() >= 3); 1540 let at = self.context.stack.len() - 3; 1541 self.context.stack.insert_many( 1542 at, 1543 &[mem.try_into().unwrap(), data_index.try_into().unwrap()], 1544 ); 1545 let builtin = self.env.builtins.memory_init::<M::ABI, M::Ptr>(); 1546 FnCall::emit::<M, M::Ptr>(self.masm, &mut self.context, Callee::Builtin(builtin)) 1547 } 1548 1549 fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) { 1550 // At this point, the stack is expected to contain: 1551 // [ dst_offset, src_offset, len ] 1552 // The following code inserts the missing params, so that stack contains: 1553 // [ vmctx, dst_mem, dst_offset, src_mem, src_offset, len ] 1554 // Which is the order expected by the builtin function. 1555 debug_assert!(self.context.stack.len() >= 3); 1556 let at = self.context.stack.len() - 2; 1557 self.context 1558 .stack 1559 .insert_many(at, &[src_mem.try_into().unwrap()]); 1560 1561 // One element was inserted above, so instead of 3, we use 4. 1562 let at = self.context.stack.len() - 4; 1563 self.context 1564 .stack 1565 .insert_many(at, &[dst_mem.try_into().unwrap()]); 1566 1567 let builtin = self.env.builtins.memory_copy::<M::ABI, M::Ptr>(); 1568 1569 FnCall::emit::<M, M::Ptr>(self.masm, &mut self.context, Callee::Builtin(builtin)) 1570 } 1571 1572 fn visit_memory_fill(&mut self, mem: u32) { 1573 debug_assert!(self.context.stack.len() >= 3); 1574 let at = self.context.stack.len() - 3; 1575 1576 self.context 1577 .stack 1578 .insert_many(at, &[mem.try_into().unwrap()]); 1579 1580 let builtin = self.env.builtins.memory_fill::<M::ABI, M::Ptr>(); 1581 FnCall::emit::<M, M::Ptr>(self.masm, &mut self.context, Callee::Builtin(builtin)) 1582 } 1583 1584 fn visit_memory_size(&mut self, mem: u32, _: u8) { 1585 let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem)); 1586 self.masm.memory_size(&heap, &mut self.context); 1587 } 1588 1589 fn visit_memory_grow(&mut self, mem: u32, _: u8) { 1590 debug_assert!(self.context.stack.len() >= 1); 1591 // The stack at this point contains: [ delta ] 1592 // The desired state is 1593 // [ vmctx, delta, index ] 1594 self.context.stack.extend([mem.try_into().unwrap()]); 1595 1596 let builtin = self.env.builtins.memory32_grow::<M::ABI, M::Ptr>(); 1597 FnCall::emit::<M, M::Ptr>(self.masm, &mut self.context, Callee::Builtin(builtin)) 1598 } 1599 1600 fn visit_data_drop(&mut self, data_index: u32) { 1601 self.context.stack.extend([data_index.try_into().unwrap()]); 1602 1603 let builtin = self.env.builtins.data_drop::<M::ABI, M::Ptr>(); 1604 FnCall::emit::<M, M::Ptr>(self.masm, &mut self.context, Callee::Builtin(builtin)) 1605 } 1606 1607 fn visit_nop(&mut self) {} 1608 1609 fn visit_if(&mut self, blockty: BlockType) { 1610 self.control_frames.push(ControlStackFrame::r#if( 1611 self.env.resolve_block_sig(blockty), 1612 self.masm, 1613 &mut self.context, 1614 )); 1615 } 1616 1617 fn visit_else(&mut self) { 1618 if !self.context.reachable { 1619 self.handle_unreachable_else(); 1620 } else { 1621 let control = self 1622 .control_frames 1623 .last_mut() 1624 .unwrap_or_else(|| panic!("Expected active control stack frame for else")); 1625 control.emit_else(self.masm, &mut self.context); 1626 } 1627 } 1628 1629 fn visit_block(&mut self, blockty: BlockType) { 1630 self.control_frames.push(ControlStackFrame::block( 1631 self.env.resolve_block_sig(blockty), 1632 self.masm, 1633 &mut self.context, 1634 )); 1635 } 1636 1637 fn visit_loop(&mut self, blockty: BlockType) { 1638 self.control_frames.push(ControlStackFrame::r#loop( 1639 self.env.resolve_block_sig(blockty), 1640 self.masm, 1641 &mut self.context, 1642 )); 1643 } 1644 1645 fn visit_br(&mut self, depth: u32) { 1646 let index = control_index(depth, self.control_frames.len()); 1647 let frame = &mut self.control_frames[index]; 1648 self.context 1649 .unconditional_jump(frame, self.masm, |masm, cx, frame| { 1650 frame 1651 .pop_abi_results::<M, _>(cx, masm, |results, _, _| results.ret_area().copied()); 1652 }); 1653 } 1654 1655 fn visit_br_if(&mut self, depth: u32) { 1656 let index = control_index(depth, self.control_frames.len()); 1657 let frame = &mut self.control_frames[index]; 1658 frame.set_as_target(); 1659 1660 let top = { 1661 let top = self.context.without::<TypedReg, M, _>( 1662 frame.results::<M>().regs(), 1663 self.masm, 1664 |ctx, masm| ctx.pop_to_reg(masm, None), 1665 ); 1666 frame.top_abi_results::<M, _>( 1667 &mut self.context, 1668 self.masm, 1669 |results, context, masm| { 1670 // In the case of `br_if` theres a possibility that we'll 1671 // exit early from the block or falltrough, for 1672 // a falltrough, we cannot rely on the pre-computed return area; 1673 // it must be recalculated so that any values that are 1674 // generated are correctly placed near the current stack 1675 // pointer. 1676 results.on_stack().then(|| { 1677 let stack_consumed = context.stack.sizeof(results.stack_operands_len()); 1678 let base = masm.sp_offset().as_u32() - stack_consumed; 1679 let offs = base + results.size(); 1680 RetArea::sp(SPOffset::from_u32(offs)) 1681 }) 1682 }, 1683 ); 1684 top 1685 }; 1686 1687 // Emit instructions to balance the machine stack if the frame has 1688 // a different offset. 1689 let current_sp_offset = self.masm.sp_offset(); 1690 let results_size = frame.results::<M>().size(); 1691 let state = frame.stack_state(); 1692 let (label, cmp, needs_cleanup) = if current_sp_offset > state.target_offset { 1693 (self.masm.get_label(), IntCmpKind::Eq, true) 1694 } else { 1695 (*frame.label(), IntCmpKind::Ne, false) 1696 }; 1697 1698 self.masm 1699 .branch(cmp, top.reg.into(), top.reg.into(), label, OperandSize::S32); 1700 self.context.free_reg(top); 1701 1702 if needs_cleanup { 1703 // Emit instructions to balance the stack and jump if not falling 1704 // through. 1705 self.masm.memmove( 1706 current_sp_offset, 1707 state.target_offset, 1708 results_size, 1709 MemMoveDirection::LowToHigh, 1710 ); 1711 self.masm.ensure_sp_for_jump(state.target_offset); 1712 self.masm.jmp(*frame.label()); 1713 1714 // Restore sp_offset to what it was for falling through and emit 1715 // fallthrough label. 1716 self.masm.reset_stack_pointer(current_sp_offset); 1717 self.masm.bind(label); 1718 } 1719 } 1720 1721 fn visit_br_table(&mut self, targets: BrTable<'a>) { 1722 // +1 to account for the default target. 1723 let len = targets.len() + 1; 1724 // SmallVec<[_; 5]> to match the binary emission layer (e.g 1725 // see `JmpTableSeq'), but here we use 5 instead since we 1726 // bundle the default target as the last element in the array. 1727 let labels: SmallVec<[_; 5]> = (0..len).map(|_| self.masm.get_label()).collect(); 1728 1729 let default_index = control_index(targets.default(), self.control_frames.len()); 1730 let default_frame = &mut self.control_frames[default_index]; 1731 let default_result = default_frame.results::<M>(); 1732 1733 let (index, tmp) = { 1734 let index_and_tmp = self.context.without::<(TypedReg, _), M, _>( 1735 default_result.regs(), 1736 self.masm, 1737 |cx, masm| (cx.pop_to_reg(masm, None), cx.any_gpr(masm)), 1738 ); 1739 1740 // Materialize any constants or locals into their result representation, 1741 // so that when reachability is restored, they are correctly located. 1742 default_frame.top_abi_results::<M, _>(&mut self.context, self.masm, |results, _, _| { 1743 results.ret_area().copied() 1744 }); 1745 index_and_tmp 1746 }; 1747 1748 self.masm.jmp_table(&labels, index.into(), tmp); 1749 // Save the original stack pointer offset; we will reset the stack 1750 // pointer to this offset after jumping to each of the targets. Each 1751 // jump might adjust the stack according to the base offset of the 1752 // target. 1753 let current_sp = self.masm.sp_offset(); 1754 1755 for (t, l) in targets 1756 .targets() 1757 .into_iter() 1758 .chain(std::iter::once(Ok(targets.default()))) 1759 .zip(labels.iter()) 1760 { 1761 let control_index = control_index(t.unwrap(), self.control_frames.len()); 1762 let frame = &mut self.control_frames[control_index]; 1763 // Reset the stack pointer to its original offset. This is needed 1764 // because each jump will potentially adjust the stack pointer 1765 // according to the base offset of the target. 1766 self.masm.reset_stack_pointer(current_sp); 1767 1768 // NB: We don't perform any result handling as it was 1769 // already taken care of above before jumping to the 1770 // jump table. 1771 self.masm.bind(*l); 1772 // Ensure that the stack pointer is correctly positioned before 1773 // jumping to the jump table code. 1774 let state = frame.stack_state(); 1775 self.masm.ensure_sp_for_jump(state.target_offset); 1776 self.masm.jmp(*frame.label()); 1777 frame.set_as_target(); 1778 } 1779 // Finally reset the stack pointer to the original location. 1780 // The reachability analysis, will ensure it's correctly located 1781 // once reachability is restored. 1782 self.masm.reset_stack_pointer(current_sp); 1783 self.context.reachable = false; 1784 self.context.free_reg(index.reg); 1785 self.context.free_reg(tmp); 1786 } 1787 1788 fn visit_return(&mut self) { 1789 // Grab the outermost frame, which is the function's body 1790 // frame. We don't rely on [`codegen::control_index`] since 1791 // this frame is implicit and we know that it should exist at 1792 // index 0. 1793 let outermost = &mut self.control_frames[0]; 1794 self.context 1795 .unconditional_jump(outermost, self.masm, |masm, cx, frame| { 1796 frame 1797 .pop_abi_results::<M, _>(cx, masm, |results, _, _| results.ret_area().copied()); 1798 }); 1799 } 1800 1801 fn visit_unreachable(&mut self) { 1802 self.masm.unreachable(); 1803 self.context.reachable = false; 1804 // Set the implicit outermost frame as target to perform the necessary 1805 // stack clean up. 1806 let outermost = &mut self.control_frames[0]; 1807 outermost.set_as_target(); 1808 } 1809 1810 fn visit_local_tee(&mut self, index: u32) { 1811 let typed_reg = self.emit_set_local(index); 1812 self.context.stack.push(typed_reg.into()); 1813 } 1814 1815 fn visit_global_get(&mut self, global_index: u32) { 1816 let index = GlobalIndex::from_u32(global_index); 1817 let (ty, addr) = self.emit_get_global_addr(index); 1818 let dst = self.context.reg_for_type(ty, self.masm); 1819 self.masm.load(addr, dst, ty.into()); 1820 self.context.stack.push(Val::reg(dst, ty)); 1821 } 1822 1823 fn visit_global_set(&mut self, global_index: u32) { 1824 let index = GlobalIndex::from_u32(global_index); 1825 let (ty, addr) = self.emit_get_global_addr(index); 1826 1827 let typed_reg = self.context.pop_to_reg(self.masm, None); 1828 self.context.free_reg(typed_reg.reg); 1829 self.masm.store(typed_reg.reg.into(), addr, ty.into()); 1830 } 1831 1832 fn visit_drop(&mut self) { 1833 self.context.drop_last(1, |regalloc, val| match val { 1834 Val::Reg(tr) => regalloc.free(tr.reg.into()), 1835 Val::Memory(m) => self.masm.free_stack(m.slot.size), 1836 _ => {} 1837 }); 1838 } 1839 1840 fn visit_select(&mut self) { 1841 let cond = self.context.pop_to_reg(self.masm, None); 1842 let val2 = self.context.pop_to_reg(self.masm, None); 1843 let val1 = self.context.pop_to_reg(self.masm, None); 1844 self.masm 1845 .cmp(RegImm::i32(0), cond.reg.into(), OperandSize::S32); 1846 // Conditionally move val1 to val2 if the the comparision is 1847 // not zero. 1848 self.masm 1849 .cmov(val1.into(), val2.into(), IntCmpKind::Ne, val1.ty.into()); 1850 self.context.stack.push(val2.into()); 1851 self.context.free_reg(val1.reg); 1852 self.context.free_reg(cond); 1853 } 1854 1855 fn visit_i32_load(&mut self, memarg: MemArg) { 1856 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S32, None); 1857 } 1858 1859 fn visit_i32_load8_s(&mut self, memarg: MemArg) { 1860 self.emit_wasm_load( 1861 &memarg, 1862 WasmValType::I32, 1863 OperandSize::S8, 1864 Some(ExtendKind::I32Extend8S), 1865 ); 1866 } 1867 1868 fn visit_i32_load8_u(&mut self, memarg: MemArg) { 1869 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S8, None); 1870 } 1871 1872 fn visit_i32_load16_s(&mut self, memarg: MemArg) { 1873 self.emit_wasm_load( 1874 &memarg, 1875 WasmValType::I32, 1876 OperandSize::S16, 1877 Some(ExtendKind::I32Extend16S), 1878 ) 1879 } 1880 1881 fn visit_i32_load16_u(&mut self, memarg: MemArg) { 1882 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S16, None) 1883 } 1884 1885 fn visit_i32_store(&mut self, memarg: MemArg) { 1886 self.emit_wasm_store(&memarg, OperandSize::S32); 1887 } 1888 1889 fn visit_i32_store8(&mut self, memarg: MemArg) { 1890 self.emit_wasm_store(&memarg, OperandSize::S8) 1891 } 1892 1893 fn visit_i32_store16(&mut self, memarg: MemArg) { 1894 self.emit_wasm_store(&memarg, OperandSize::S16) 1895 } 1896 1897 fn visit_i64_load8_s(&mut self, memarg: MemArg) { 1898 self.emit_wasm_load( 1899 &memarg, 1900 WasmValType::I64, 1901 OperandSize::S8, 1902 Some(ExtendKind::I64Extend8S), 1903 ) 1904 } 1905 1906 fn visit_i64_load8_u(&mut self, memarg: MemArg) { 1907 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S8, None) 1908 } 1909 1910 fn visit_i64_load16_u(&mut self, memarg: MemArg) { 1911 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S16, None) 1912 } 1913 1914 fn visit_i64_load16_s(&mut self, memarg: MemArg) { 1915 self.emit_wasm_load( 1916 &memarg, 1917 WasmValType::I64, 1918 OperandSize::S16, 1919 Some(ExtendKind::I64Extend16S), 1920 ) 1921 } 1922 1923 fn visit_i64_load32_u(&mut self, memarg: MemArg) { 1924 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S32, None) 1925 } 1926 1927 fn visit_i64_load32_s(&mut self, memarg: MemArg) { 1928 self.emit_wasm_load( 1929 &memarg, 1930 WasmValType::I64, 1931 OperandSize::S32, 1932 Some(ExtendKind::I64Extend32S), 1933 ) 1934 } 1935 1936 fn visit_i64_load(&mut self, memarg: MemArg) { 1937 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S64, None) 1938 } 1939 1940 fn visit_i64_store(&mut self, memarg: MemArg) -> Self::Output { 1941 self.emit_wasm_store(&memarg, OperandSize::S64) 1942 } 1943 1944 fn visit_i64_store8(&mut self, memarg: MemArg) -> Self::Output { 1945 self.emit_wasm_store(&memarg, OperandSize::S8) 1946 } 1947 1948 fn visit_i64_store16(&mut self, memarg: MemArg) -> Self::Output { 1949 self.emit_wasm_store(&memarg, OperandSize::S16) 1950 } 1951 1952 fn visit_i64_store32(&mut self, memarg: MemArg) -> Self::Output { 1953 self.emit_wasm_store(&memarg, OperandSize::S32) 1954 } 1955 1956 fn visit_f32_load(&mut self, memarg: MemArg) { 1957 self.emit_wasm_load(&memarg, WasmValType::F32, OperandSize::S32, None) 1958 } 1959 1960 fn visit_f32_store(&mut self, memarg: MemArg) { 1961 self.emit_wasm_store(&memarg, OperandSize::S32) 1962 } 1963 1964 fn visit_f64_load(&mut self, memarg: MemArg) { 1965 self.emit_wasm_load(&memarg, WasmValType::F64, OperandSize::S64, None) 1966 } 1967 1968 fn visit_f64_store(&mut self, memarg: MemArg) { 1969 self.emit_wasm_store(&memarg, OperandSize::S64) 1970 } 1971 1972 fn visit_i32_trunc_sat_f32_s(&mut self) { 1973 use OperandSize::*; 1974 1975 self.context 1976 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1977 masm.signed_truncate(src, dst, S32, dst_size, TruncKind::Checked); 1978 }); 1979 } 1980 1981 fn visit_i32_trunc_sat_f32_u(&mut self) { 1982 use OperandSize::*; 1983 1984 self.context.convert_op_with_tmp_reg( 1985 self.masm, 1986 WasmValType::I32, 1987 RegClass::Float, 1988 |masm, dst, src, tmp_fpr, dst_size| { 1989 masm.unsigned_truncate(src, dst, tmp_fpr, S32, dst_size, TruncKind::Checked); 1990 }, 1991 ); 1992 } 1993 1994 fn visit_i32_trunc_sat_f64_s(&mut self) { 1995 use OperandSize::*; 1996 1997 self.context 1998 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1999 masm.signed_truncate(src, dst, S64, dst_size, TruncKind::Checked); 2000 }); 2001 } 2002 2003 fn visit_i32_trunc_sat_f64_u(&mut self) { 2004 use OperandSize::*; 2005 2006 self.context.convert_op_with_tmp_reg( 2007 self.masm, 2008 WasmValType::I32, 2009 RegClass::Float, 2010 |masm, dst, src, tmp_fpr, dst_size| { 2011 masm.unsigned_truncate(src, dst, tmp_fpr, S64, dst_size, TruncKind::Checked); 2012 }, 2013 ); 2014 } 2015 2016 fn visit_i64_trunc_sat_f32_s(&mut self) { 2017 use OperandSize::*; 2018 2019 self.context 2020 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2021 masm.signed_truncate(src, dst, S32, dst_size, TruncKind::Checked); 2022 }); 2023 } 2024 2025 fn visit_i64_trunc_sat_f32_u(&mut self) { 2026 use OperandSize::*; 2027 2028 self.context.convert_op_with_tmp_reg( 2029 self.masm, 2030 WasmValType::I64, 2031 RegClass::Float, 2032 |masm, dst, src, tmp_fpr, dst_size| { 2033 masm.unsigned_truncate(src, dst, tmp_fpr, S32, dst_size, TruncKind::Checked); 2034 }, 2035 ); 2036 } 2037 2038 fn visit_i64_trunc_sat_f64_s(&mut self) { 2039 use OperandSize::*; 2040 2041 self.context 2042 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2043 masm.signed_truncate(src, dst, S64, dst_size, TruncKind::Checked); 2044 }); 2045 } 2046 2047 fn visit_i64_trunc_sat_f64_u(&mut self) { 2048 use OperandSize::*; 2049 2050 self.context.convert_op_with_tmp_reg( 2051 self.masm, 2052 WasmValType::I64, 2053 RegClass::Float, 2054 |masm, dst, src, tmp_fpr, dst_size| { 2055 masm.unsigned_truncate(src, dst, tmp_fpr, S64, dst_size, TruncKind::Checked); 2056 }, 2057 ); 2058 } 2059 2060 wasmparser::for_each_operator!(def_unsupported); 2061 } 2062 2063 impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M> 2064 where 2065 M: MacroAssembler, 2066 { 2067 fn cmp_i32s(&mut self, kind: IntCmpKind) { 2068 self.context.i32_binop(self.masm, |masm, dst, src, size| { 2069 masm.cmp_with_set(src, dst, kind, size); 2070 TypedReg::i32(dst) 2071 }); 2072 } 2073 2074 fn cmp_i64s(&mut self, kind: IntCmpKind) { 2075 self.context 2076 .i64_binop(self.masm, move |masm, dst, src, size| { 2077 masm.cmp_with_set(src, dst, kind, size); 2078 TypedReg::i32(dst) // Return value for comparisons is an `i32`. 2079 }); 2080 } 2081 } 2082 2083 impl From<WasmValType> for OperandSize { 2084 fn from(ty: WasmValType) -> OperandSize { 2085 match ty { 2086 WasmValType::I32 | WasmValType::F32 => OperandSize::S32, 2087 WasmValType::I64 | WasmValType::F64 => OperandSize::S64, 2088 WasmValType::Ref(rt) => { 2089 match rt.heap_type { 2090 // TODO: Harcoded size, assuming 64-bit support only. Once 2091 // Wasmtime supports 32-bit architectures, this will need 2092 // to be updated in such a way that the calculation of the 2093 // OperandSize will depend on the target's pointer size. 2094 WasmHeapType::Func => OperandSize::S64, 2095 t => unimplemented!("Support for WasmHeapType: {t}"), 2096 } 2097 } 2098 ty => unimplemented!("Support for WasmValType {ty}"), 2099 } 2100 } 2101 } 2102