1 //! This module is the central place for machine code emission. 2 //! It defines an implementation of wasmparser's Visitor trait 3 //! for `CodeGen`; which defines a visitor per op-code, 4 //! which validates and dispatches to the corresponding 5 //! machine code emitter. 6 7 use crate::abi::RetArea; 8 use crate::codegen::{control_index, Callee, CodeGen, ControlStackFrame, FnCall}; 9 use crate::masm::{ 10 DivKind, ExtendKind, FloatCmpKind, IntCmpKind, MacroAssembler, MemMoveDirection, OperandSize, 11 RegImm, RemKind, RoundingMode, SPOffset, ShiftKind, TruncKind, 12 }; 13 use crate::reg::Reg; 14 use crate::stack::{TypedReg, Val}; 15 use cranelift_codegen::ir::TrapCode; 16 use regalloc2::RegClass; 17 use smallvec::SmallVec; 18 use wasmparser::{BlockType, BrTable, Ieee32, Ieee64, MemArg, VisitOperator, V128}; 19 use wasmtime_environ::{ 20 FuncIndex, GlobalIndex, MemoryIndex, TableIndex, TableStyle, TypeIndex, WasmHeapType, 21 WasmValType, FUNCREF_INIT_BIT, 22 }; 23 24 /// A macro to define unsupported WebAssembly operators. 25 /// 26 /// This macro calls itself recursively; 27 /// 1. It no-ops when matching a supported operator. 28 /// 2. Defines the visitor function and panics when 29 /// matching an unsupported operator. 30 macro_rules! def_unsupported { 31 ($( @$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident)*) => { 32 $( 33 def_unsupported!( 34 emit 35 $op 36 37 fn $visit(&mut self $($(,$arg: $argty)*)?) -> Self::Output { 38 $($(let _ = $arg;)*)? 39 todo!(stringify!($op)) 40 } 41 ); 42 )* 43 }; 44 45 (emit I32Const $($rest:tt)*) => {}; 46 (emit I64Const $($rest:tt)*) => {}; 47 (emit F32Const $($rest:tt)*) => {}; 48 (emit F64Const $($rest:tt)*) => {}; 49 (emit V128Const $($rest:tt)*) => {}; 50 (emit F32Add $($rest:tt)*) => {}; 51 (emit F64Add $($rest:tt)*) => {}; 52 (emit F32Sub $($rest:tt)*) => {}; 53 (emit F64Sub $($rest:tt)*) => {}; 54 (emit F32Mul $($rest:tt)*) => {}; 55 (emit F64Mul $($rest:tt)*) => {}; 56 (emit F32Div $($rest:tt)*) => {}; 57 (emit F64Div $($rest:tt)*) => {}; 58 (emit F32Min $($rest:tt)*) => {}; 59 (emit F64Min $($rest:tt)*) => {}; 60 (emit F32Max $($rest:tt)*) => {}; 61 (emit F64Max $($rest:tt)*) => {}; 62 (emit F32Copysign $($rest:tt)*) => {}; 63 (emit F64Copysign $($rest:tt)*) => {}; 64 (emit F32Abs $($rest:tt)*) => {}; 65 (emit F64Abs $($rest:tt)*) => {}; 66 (emit F32Neg $($rest:tt)*) => {}; 67 (emit F64Neg $($rest:tt)*) => {}; 68 (emit F32Floor $($rest:tt)*) => {}; 69 (emit F64Floor $($rest:tt)*) => {}; 70 (emit F32Ceil $($rest:tt)*) => {}; 71 (emit F64Ceil $($rest:tt)*) => {}; 72 (emit F32Nearest $($rest:tt)*) => {}; 73 (emit F64Nearest $($rest:tt)*) => {}; 74 (emit F32Trunc $($rest:tt)*) => {}; 75 (emit F64Trunc $($rest:tt)*) => {}; 76 (emit F32Sqrt $($rest:tt)*) => {}; 77 (emit F64Sqrt $($rest:tt)*) => {}; 78 (emit F32Eq $($rest:tt)*) => {}; 79 (emit F64Eq $($rest:tt)*) => {}; 80 (emit F32Ne $($rest:tt)*) => {}; 81 (emit F64Ne $($rest:tt)*) => {}; 82 (emit F32Lt $($rest:tt)*) => {}; 83 (emit F64Lt $($rest:tt)*) => {}; 84 (emit F32Gt $($rest:tt)*) => {}; 85 (emit F64Gt $($rest:tt)*) => {}; 86 (emit F32Le $($rest:tt)*) => {}; 87 (emit F64Le $($rest:tt)*) => {}; 88 (emit F32Ge $($rest:tt)*) => {}; 89 (emit F64Ge $($rest:tt)*) => {}; 90 (emit F32ConvertI32S $($rest:tt)*) => {}; 91 (emit F32ConvertI32U $($rest:tt)*) => {}; 92 (emit F32ConvertI64S $($rest:tt)*) => {}; 93 (emit F32ConvertI64U $($rest:tt)*) => {}; 94 (emit F64ConvertI32S $($rest:tt)*) => {}; 95 (emit F64ConvertI32U $($rest:tt)*) => {}; 96 (emit F64ConvertI64S $($rest:tt)*) => {}; 97 (emit F64ConvertI64U $($rest:tt)*) => {}; 98 (emit F32ReinterpretI32 $($rest:tt)*) => {}; 99 (emit F64ReinterpretI64 $($rest:tt)*) => {}; 100 (emit F32DemoteF64 $($rest:tt)*) => {}; 101 (emit F64PromoteF32 $($rest:tt)*) => {}; 102 (emit I32Add $($rest:tt)*) => {}; 103 (emit I64Add $($rest:tt)*) => {}; 104 (emit I32Sub $($rest:tt)*) => {}; 105 (emit I32Mul $($rest:tt)*) => {}; 106 (emit I32DivS $($rest:tt)*) => {}; 107 (emit I32DivU $($rest:tt)*) => {}; 108 (emit I64DivS $($rest:tt)*) => {}; 109 (emit I64DivU $($rest:tt)*) => {}; 110 (emit I64RemU $($rest:tt)*) => {}; 111 (emit I64RemS $($rest:tt)*) => {}; 112 (emit I32RemU $($rest:tt)*) => {}; 113 (emit I32RemS $($rest:tt)*) => {}; 114 (emit I64Mul $($rest:tt)*) => {}; 115 (emit I64Sub $($rest:tt)*) => {}; 116 (emit I32Eq $($rest:tt)*) => {}; 117 (emit I64Eq $($rest:tt)*) => {}; 118 (emit I32Ne $($rest:tt)*) => {}; 119 (emit I64Ne $($rest:tt)*) => {}; 120 (emit I32LtS $($rest:tt)*) => {}; 121 (emit I64LtS $($rest:tt)*) => {}; 122 (emit I32LtU $($rest:tt)*) => {}; 123 (emit I64LtU $($rest:tt)*) => {}; 124 (emit I32LeS $($rest:tt)*) => {}; 125 (emit I64LeS $($rest:tt)*) => {}; 126 (emit I32LeU $($rest:tt)*) => {}; 127 (emit I64LeU $($rest:tt)*) => {}; 128 (emit I32GtS $($rest:tt)*) => {}; 129 (emit I64GtS $($rest:tt)*) => {}; 130 (emit I32GtU $($rest:tt)*) => {}; 131 (emit I64GtU $($rest:tt)*) => {}; 132 (emit I32GeS $($rest:tt)*) => {}; 133 (emit I64GeS $($rest:tt)*) => {}; 134 (emit I32GeU $($rest:tt)*) => {}; 135 (emit I64GeU $($rest:tt)*) => {}; 136 (emit I32Eqz $($rest:tt)*) => {}; 137 (emit I64Eqz $($rest:tt)*) => {}; 138 (emit I32And $($rest:tt)*) => {}; 139 (emit I64And $($rest:tt)*) => {}; 140 (emit I32Or $($rest:tt)*) => {}; 141 (emit I64Or $($rest:tt)*) => {}; 142 (emit I32Xor $($rest:tt)*) => {}; 143 (emit I64Xor $($rest:tt)*) => {}; 144 (emit I32Shl $($rest:tt)*) => {}; 145 (emit I64Shl $($rest:tt)*) => {}; 146 (emit I32ShrS $($rest:tt)*) => {}; 147 (emit I64ShrS $($rest:tt)*) => {}; 148 (emit I32ShrU $($rest:tt)*) => {}; 149 (emit I64ShrU $($rest:tt)*) => {}; 150 (emit I32Rotl $($rest:tt)*) => {}; 151 (emit I64Rotl $($rest:tt)*) => {}; 152 (emit I32Rotr $($rest:tt)*) => {}; 153 (emit I64Rotr $($rest:tt)*) => {}; 154 (emit I32Clz $($rest:tt)*) => {}; 155 (emit I64Clz $($rest:tt)*) => {}; 156 (emit I32Ctz $($rest:tt)*) => {}; 157 (emit I64Ctz $($rest:tt)*) => {}; 158 (emit I32Popcnt $($rest:tt)*) => {}; 159 (emit I64Popcnt $($rest:tt)*) => {}; 160 (emit I32WrapI64 $($rest:tt)*) => {}; 161 (emit I64ExtendI32S $($rest:tt)*) => {}; 162 (emit I64ExtendI32U $($rest:tt)*) => {}; 163 (emit I32Extend8S $($rest:tt)*) => {}; 164 (emit I32Extend16S $($rest:tt)*) => {}; 165 (emit I64Extend8S $($rest:tt)*) => {}; 166 (emit I64Extend16S $($rest:tt)*) => {}; 167 (emit I64Extend32S $($rest:tt)*) => {}; 168 (emit I32TruncF32S $($rest:tt)*) => {}; 169 (emit I32TruncF32U $($rest:tt)*) => {}; 170 (emit I32TruncF64S $($rest:tt)*) => {}; 171 (emit I32TruncF64U $($rest:tt)*) => {}; 172 (emit I64TruncF32S $($rest:tt)*) => {}; 173 (emit I64TruncF32U $($rest:tt)*) => {}; 174 (emit I64TruncF64S $($rest:tt)*) => {}; 175 (emit I64TruncF64U $($rest:tt)*) => {}; 176 (emit I32ReinterpretF32 $($rest:tt)*) => {}; 177 (emit I64ReinterpretF64 $($rest:tt)*) => {}; 178 (emit LocalGet $($rest:tt)*) => {}; 179 (emit LocalSet $($rest:tt)*) => {}; 180 (emit Call $($rest:tt)*) => {}; 181 (emit End $($rest:tt)*) => {}; 182 (emit Nop $($rest:tt)*) => {}; 183 (emit If $($rest:tt)*) => {}; 184 (emit Else $($rest:tt)*) => {}; 185 (emit Block $($rest:tt)*) => {}; 186 (emit Loop $($rest:tt)*) => {}; 187 (emit Br $($rest:tt)*) => {}; 188 (emit BrIf $($rest:tt)*) => {}; 189 (emit Return $($rest:tt)*) => {}; 190 (emit Unreachable $($rest:tt)*) => {}; 191 (emit LocalTee $($rest:tt)*) => {}; 192 (emit GlobalGet $($rest:tt)*) => {}; 193 (emit GlobalSet $($rest:tt)*) => {}; 194 (emit Select $($rest:tt)*) => {}; 195 (emit Drop $($rest:tt)*) => {}; 196 (emit BrTable $($rest:tt)*) => {}; 197 (emit CallIndirect $($rest:tt)*) => {}; 198 (emit TableInit $($rest:tt)*) => {}; 199 (emit TableCopy $($rest:tt)*) => {}; 200 (emit TableGet $($rest:tt)*) => {}; 201 (emit TableSet $($rest:tt)*) => {}; 202 (emit TableGrow $($rest:tt)*) => {}; 203 (emit TableSize $($rest:tt)*) => {}; 204 (emit TableFill $($rest:tt)*) => {}; 205 (emit ElemDrop $($rest:tt)*) => {}; 206 (emit MemoryInit $($rest:tt)*) => {}; 207 (emit MemoryCopy $($rest:tt)*) => {}; 208 (emit DataDrop $($rest:tt)*) => {}; 209 (emit MemoryFill $($rest:tt)*) => {}; 210 (emit MemorySize $($rest:tt)*) => {}; 211 (emit MemoryGrow $($rest:tt)*) => {}; 212 (emit I32Load $($rest:tt)*) => {}; 213 (emit I32Load8S $($rest:tt)*) => {}; 214 (emit I32Load8U $($rest:tt)*) => {}; 215 (emit I32Load16S $($rest:tt)*) => {}; 216 (emit I32Load16U $($rest:tt)*) => {}; 217 (emit I64Load8S $($rest:tt)*) => {}; 218 (emit I64Load8U $($rest:tt)*) => {}; 219 (emit I64Load16S $($rest:tt)*) => {}; 220 (emit I64Load16U $($rest:tt)*) => {}; 221 (emit I64Load32S $($rest:tt)*) => {}; 222 (emit I64Load32U $($rest:tt)*) => {}; 223 (emit I64Load $($rest:tt)*) => {}; 224 (emit I32Store $($rest:tt)*) => {}; 225 (emit I32Store $($rest:tt)*) => {}; 226 (emit I32Store8 $($rest:tt)*) => {}; 227 (emit I32Store16 $($rest:tt)*) => {}; 228 (emit I64Store $($rest:tt)*) => {}; 229 (emit I64Store8 $($rest:tt)*) => {}; 230 (emit I64Store16 $($rest:tt)*) => {}; 231 (emit I64Store32 $($rest:tt)*) => {}; 232 (emit F32Load $($rest:tt)*) => {}; 233 (emit F32Store $($rest:tt)*) => {}; 234 (emit F64Load $($rest:tt)*) => {}; 235 (emit F64Store $($rest:tt)*) => {}; 236 (emit I32TruncSatF32S $($rest:tt)*) => {}; 237 (emit I32TruncSatF32U $($rest:tt)*) => {}; 238 (emit I32TruncSatF64S $($rest:tt)*) => {}; 239 (emit I32TruncSatF64U $($rest:tt)*) => {}; 240 (emit I64TruncSatF32S $($rest:tt)*) => {}; 241 (emit I64TruncSatF32U $($rest:tt)*) => {}; 242 (emit I64TruncSatF64S $($rest:tt)*) => {}; 243 (emit I64TruncSatF64U $($rest:tt)*) => {}; 244 (emit V128Load $($rest:tt)*) => {}; 245 (emit V128Store $($rest:tt)*) => {}; 246 247 (emit $unsupported:tt $($rest:tt)*) => {$($rest)*}; 248 } 249 250 impl<'a, 'translation, 'data, M> VisitOperator<'a> for CodeGen<'a, 'translation, 'data, M> 251 where 252 M: MacroAssembler, 253 { 254 type Output = (); 255 256 fn visit_i32_const(&mut self, val: i32) { 257 self.context.stack.push(Val::i32(val)); 258 } 259 260 fn visit_i64_const(&mut self, val: i64) { 261 self.context.stack.push(Val::i64(val)); 262 } 263 264 fn visit_f32_const(&mut self, val: Ieee32) { 265 self.context.stack.push(Val::f32(val)); 266 } 267 268 fn visit_f64_const(&mut self, val: Ieee64) { 269 self.context.stack.push(Val::f64(val)); 270 } 271 272 fn visit_v128_const(&mut self, val: V128) { 273 self.context.stack.push(Val::v128(val.i128())) 274 } 275 276 fn visit_f32_add(&mut self) { 277 self.context.binop( 278 self.masm, 279 OperandSize::S32, 280 &mut |masm: &mut M, dst, src, size| { 281 masm.float_add(dst, dst, src, size); 282 TypedReg::f32(dst) 283 }, 284 ); 285 } 286 287 fn visit_f64_add(&mut self) { 288 self.context.binop( 289 self.masm, 290 OperandSize::S64, 291 &mut |masm: &mut M, dst, src, size| { 292 masm.float_add(dst, dst, src, size); 293 TypedReg::f64(dst) 294 }, 295 ); 296 } 297 298 fn visit_f32_sub(&mut self) { 299 self.context.binop( 300 self.masm, 301 OperandSize::S32, 302 &mut |masm: &mut M, dst, src, size| { 303 masm.float_sub(dst, dst, src, size); 304 TypedReg::f32(dst) 305 }, 306 ); 307 } 308 309 fn visit_f64_sub(&mut self) { 310 self.context.binop( 311 self.masm, 312 OperandSize::S64, 313 &mut |masm: &mut M, dst, src, size| { 314 masm.float_sub(dst, dst, src, size); 315 TypedReg::f64(dst) 316 }, 317 ); 318 } 319 320 fn visit_f32_mul(&mut self) { 321 self.context.binop( 322 self.masm, 323 OperandSize::S32, 324 &mut |masm: &mut M, dst, src, size| { 325 masm.float_mul(dst, dst, src, size); 326 TypedReg::f32(dst) 327 }, 328 ); 329 } 330 331 fn visit_f64_mul(&mut self) { 332 self.context.binop( 333 self.masm, 334 OperandSize::S64, 335 &mut |masm: &mut M, dst, src, size| { 336 masm.float_mul(dst, dst, src, size); 337 TypedReg::f64(dst) 338 }, 339 ); 340 } 341 342 fn visit_f32_div(&mut self) { 343 self.context.binop( 344 self.masm, 345 OperandSize::S32, 346 &mut |masm: &mut M, dst, src, size| { 347 masm.float_div(dst, dst, src, size); 348 TypedReg::f32(dst) 349 }, 350 ); 351 } 352 353 fn visit_f64_div(&mut self) { 354 self.context.binop( 355 self.masm, 356 OperandSize::S64, 357 &mut |masm: &mut M, dst, src, size| { 358 masm.float_div(dst, dst, src, size); 359 TypedReg::f64(dst) 360 }, 361 ); 362 } 363 364 fn visit_f32_min(&mut self) { 365 self.context.binop( 366 self.masm, 367 OperandSize::S32, 368 &mut |masm: &mut M, dst, src, size| { 369 masm.float_min(dst, dst, src, size); 370 TypedReg::f32(dst) 371 }, 372 ); 373 } 374 375 fn visit_f64_min(&mut self) { 376 self.context.binop( 377 self.masm, 378 OperandSize::S64, 379 &mut |masm: &mut M, dst, src, size| { 380 masm.float_min(dst, dst, src, size); 381 TypedReg::f64(dst) 382 }, 383 ); 384 } 385 386 fn visit_f32_max(&mut self) { 387 self.context.binop( 388 self.masm, 389 OperandSize::S32, 390 &mut |masm: &mut M, dst, src, size| { 391 masm.float_max(dst, dst, src, size); 392 TypedReg::f32(dst) 393 }, 394 ); 395 } 396 397 fn visit_f64_max(&mut self) { 398 self.context.binop( 399 self.masm, 400 OperandSize::S64, 401 &mut |masm: &mut M, dst, src, size| { 402 masm.float_max(dst, dst, src, size); 403 TypedReg::f64(dst) 404 }, 405 ); 406 } 407 408 fn visit_f32_copysign(&mut self) { 409 self.context.binop( 410 self.masm, 411 OperandSize::S32, 412 &mut |masm: &mut M, dst, src, size| { 413 masm.float_copysign(dst, dst, src, size); 414 TypedReg::f32(dst) 415 }, 416 ); 417 } 418 419 fn visit_f64_copysign(&mut self) { 420 self.context.binop( 421 self.masm, 422 OperandSize::S64, 423 &mut |masm: &mut M, dst, src, size| { 424 masm.float_copysign(dst, dst, src, size); 425 TypedReg::f64(dst) 426 }, 427 ); 428 } 429 430 fn visit_f32_abs(&mut self) { 431 self.context 432 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 433 masm.float_abs(reg, size); 434 TypedReg::f32(reg) 435 }); 436 } 437 438 fn visit_f64_abs(&mut self) { 439 self.context 440 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 441 masm.float_abs(reg, size); 442 TypedReg::f64(reg) 443 }); 444 } 445 446 fn visit_f32_neg(&mut self) { 447 self.context 448 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 449 masm.float_neg(reg, size); 450 TypedReg::f32(reg) 451 }); 452 } 453 454 fn visit_f64_neg(&mut self) { 455 self.context 456 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 457 masm.float_neg(reg, size); 458 TypedReg::f64(reg) 459 }); 460 } 461 462 fn visit_f32_floor(&mut self) { 463 self.masm.float_round( 464 RoundingMode::Down, 465 &mut self.env, 466 &mut self.context, 467 OperandSize::S32, 468 |env, cx, masm| { 469 let builtin = env.builtins.floor_f32::<M::ABI>(); 470 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 471 }, 472 ); 473 } 474 475 fn visit_f64_floor(&mut self) { 476 self.masm.float_round( 477 RoundingMode::Down, 478 &mut self.env, 479 &mut self.context, 480 OperandSize::S64, 481 |env, cx, masm| { 482 let builtin = env.builtins.floor_f64::<M::ABI>(); 483 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 484 }, 485 ); 486 } 487 488 fn visit_f32_ceil(&mut self) { 489 self.masm.float_round( 490 RoundingMode::Up, 491 &mut self.env, 492 &mut self.context, 493 OperandSize::S32, 494 |env, cx, masm| { 495 let builtin = env.builtins.ceil_f32::<M::ABI>(); 496 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 497 }, 498 ); 499 } 500 501 fn visit_f64_ceil(&mut self) { 502 self.masm.float_round( 503 RoundingMode::Up, 504 &mut self.env, 505 &mut self.context, 506 OperandSize::S64, 507 |env, cx, masm| { 508 let builtin = env.builtins.ceil_f64::<M::ABI>(); 509 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 510 }, 511 ); 512 } 513 514 fn visit_f32_nearest(&mut self) { 515 self.masm.float_round( 516 RoundingMode::Nearest, 517 &mut self.env, 518 &mut self.context, 519 OperandSize::S32, 520 |env, cx, masm| { 521 let builtin = env.builtins.nearest_f32::<M::ABI>(); 522 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 523 }, 524 ); 525 } 526 527 fn visit_f64_nearest(&mut self) { 528 self.masm.float_round( 529 RoundingMode::Nearest, 530 &mut self.env, 531 &mut self.context, 532 OperandSize::S64, 533 |env, cx, masm| { 534 let builtin = env.builtins.nearest_f64::<M::ABI>(); 535 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 536 }, 537 ); 538 } 539 540 fn visit_f32_trunc(&mut self) { 541 self.masm.float_round( 542 RoundingMode::Zero, 543 &mut self.env, 544 &mut self.context, 545 OperandSize::S32, 546 |env, cx, masm| { 547 let builtin = env.builtins.trunc_f32::<M::ABI>(); 548 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 549 }, 550 ); 551 } 552 553 fn visit_f64_trunc(&mut self) { 554 self.masm.float_round( 555 RoundingMode::Zero, 556 &mut self.env, 557 &mut self.context, 558 OperandSize::S64, 559 |env, cx, masm| { 560 let builtin = env.builtins.trunc_f64::<M::ABI>(); 561 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 562 }, 563 ); 564 } 565 566 fn visit_f32_sqrt(&mut self) { 567 self.context 568 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 569 masm.float_sqrt(reg, reg, size); 570 TypedReg::f32(reg) 571 }); 572 } 573 574 fn visit_f64_sqrt(&mut self) { 575 self.context 576 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 577 masm.float_sqrt(reg, reg, size); 578 TypedReg::f64(reg) 579 }); 580 } 581 582 fn visit_f32_eq(&mut self) { 583 self.context.float_cmp_op( 584 self.masm, 585 OperandSize::S32, 586 &mut |masm: &mut M, dst, src1, src2, size| { 587 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Eq, size); 588 }, 589 ); 590 } 591 592 fn visit_f64_eq(&mut self) { 593 self.context.float_cmp_op( 594 self.masm, 595 OperandSize::S64, 596 &mut |masm: &mut M, dst, src1, src2, size| { 597 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Eq, size); 598 }, 599 ); 600 } 601 602 fn visit_f32_ne(&mut self) { 603 self.context.float_cmp_op( 604 self.masm, 605 OperandSize::S32, 606 &mut |masm: &mut M, dst, src1, src2, size| { 607 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Ne, size); 608 }, 609 ); 610 } 611 612 fn visit_f64_ne(&mut self) { 613 self.context.float_cmp_op( 614 self.masm, 615 OperandSize::S64, 616 &mut |masm: &mut M, dst, src1, src2, size| { 617 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Ne, size); 618 }, 619 ); 620 } 621 622 fn visit_f32_lt(&mut self) { 623 self.context.float_cmp_op( 624 self.masm, 625 OperandSize::S32, 626 &mut |masm: &mut M, dst, src1, src2, size| { 627 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Lt, size); 628 }, 629 ); 630 } 631 632 fn visit_f64_lt(&mut self) { 633 self.context.float_cmp_op( 634 self.masm, 635 OperandSize::S64, 636 &mut |masm: &mut M, dst, src1, src2, size| { 637 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Lt, size); 638 }, 639 ); 640 } 641 642 fn visit_f32_gt(&mut self) { 643 self.context.float_cmp_op( 644 self.masm, 645 OperandSize::S32, 646 &mut |masm: &mut M, dst, src1, src2, size| { 647 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Gt, size); 648 }, 649 ); 650 } 651 652 fn visit_f64_gt(&mut self) { 653 self.context.float_cmp_op( 654 self.masm, 655 OperandSize::S64, 656 &mut |masm: &mut M, dst, src1, src2, size| { 657 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Gt, size); 658 }, 659 ); 660 } 661 662 fn visit_f32_le(&mut self) { 663 self.context.float_cmp_op( 664 self.masm, 665 OperandSize::S32, 666 &mut |masm: &mut M, dst, src1, src2, size| { 667 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Le, size); 668 }, 669 ); 670 } 671 672 fn visit_f64_le(&mut self) { 673 self.context.float_cmp_op( 674 self.masm, 675 OperandSize::S64, 676 &mut |masm: &mut M, dst, src1, src2, size| { 677 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Le, size); 678 }, 679 ); 680 } 681 682 fn visit_f32_ge(&mut self) { 683 self.context.float_cmp_op( 684 self.masm, 685 OperandSize::S32, 686 &mut |masm: &mut M, dst, src1, src2, size| { 687 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Ge, size); 688 }, 689 ); 690 } 691 692 fn visit_f64_ge(&mut self) { 693 self.context.float_cmp_op( 694 self.masm, 695 OperandSize::S64, 696 &mut |masm: &mut M, dst, src1, src2, size| { 697 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Ge, size); 698 }, 699 ); 700 } 701 702 fn visit_f32_convert_i32_s(&mut self) { 703 self.context 704 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 705 masm.signed_convert(src, dst, OperandSize::S32, dst_size); 706 }); 707 } 708 709 fn visit_f32_convert_i32_u(&mut self) { 710 self.context.convert_op_with_tmp_reg( 711 self.masm, 712 WasmValType::F32, 713 RegClass::Int, 714 |masm, dst, src, tmp_gpr, dst_size| { 715 masm.unsigned_convert(src, dst, tmp_gpr, OperandSize::S32, dst_size); 716 }, 717 ); 718 } 719 720 fn visit_f32_convert_i64_s(&mut self) { 721 self.context 722 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 723 masm.signed_convert(src, dst, OperandSize::S64, dst_size); 724 }); 725 } 726 727 fn visit_f32_convert_i64_u(&mut self) { 728 self.context.convert_op_with_tmp_reg( 729 self.masm, 730 WasmValType::F32, 731 RegClass::Int, 732 |masm, dst, src, tmp_gpr, dst_size| { 733 masm.unsigned_convert(src, dst, tmp_gpr, OperandSize::S64, dst_size); 734 }, 735 ); 736 } 737 738 fn visit_f64_convert_i32_s(&mut self) { 739 self.context 740 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 741 masm.signed_convert(src, dst, OperandSize::S32, dst_size); 742 }); 743 } 744 745 fn visit_f64_convert_i32_u(&mut self) { 746 self.context.convert_op_with_tmp_reg( 747 self.masm, 748 WasmValType::F64, 749 RegClass::Int, 750 |masm, dst, src, tmp_gpr, dst_size| { 751 masm.unsigned_convert(src, dst, tmp_gpr, OperandSize::S32, dst_size); 752 }, 753 ); 754 } 755 756 fn visit_f64_convert_i64_s(&mut self) { 757 self.context 758 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 759 masm.signed_convert(src, dst, OperandSize::S64, dst_size); 760 }); 761 } 762 763 fn visit_f64_convert_i64_u(&mut self) { 764 self.context.convert_op_with_tmp_reg( 765 self.masm, 766 WasmValType::F64, 767 RegClass::Int, 768 |masm, dst, src, tmp_gpr, dst_size| { 769 masm.unsigned_convert(src, dst, tmp_gpr, OperandSize::S64, dst_size); 770 }, 771 ); 772 } 773 774 fn visit_f32_reinterpret_i32(&mut self) { 775 self.context 776 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, size| { 777 masm.reinterpret_int_as_float(src.into(), dst, size); 778 }); 779 } 780 781 fn visit_f64_reinterpret_i64(&mut self) { 782 self.context 783 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, size| { 784 masm.reinterpret_int_as_float(src.into(), dst, size); 785 }); 786 } 787 788 fn visit_f32_demote_f64(&mut self) { 789 self.context 790 .unop(self.masm, OperandSize::S64, &mut |masm, reg, _size| { 791 masm.demote(reg, reg); 792 TypedReg::f32(reg) 793 }); 794 } 795 796 fn visit_f64_promote_f32(&mut self) { 797 self.context 798 .unop(self.masm, OperandSize::S32, &mut |masm, reg, _size| { 799 masm.promote(reg, reg); 800 TypedReg::f64(reg) 801 }); 802 } 803 804 fn visit_i32_add(&mut self) { 805 self.context.i32_binop(self.masm, |masm, dst, src, size| { 806 masm.add(dst, dst, src, size); 807 TypedReg::i32(dst) 808 }); 809 } 810 811 fn visit_i64_add(&mut self) { 812 self.context.i64_binop(self.masm, |masm, dst, src, size| { 813 masm.add(dst, dst, src, size); 814 TypedReg::i64(dst) 815 }); 816 } 817 818 fn visit_i32_sub(&mut self) { 819 self.context.i32_binop(self.masm, |masm, dst, src, size| { 820 masm.sub(dst, dst, src, size); 821 TypedReg::i32(dst) 822 }); 823 } 824 825 fn visit_i64_sub(&mut self) { 826 self.context.i64_binop(self.masm, |masm, dst, src, size| { 827 masm.sub(dst, dst, src, size); 828 TypedReg::i64(dst) 829 }); 830 } 831 832 fn visit_i32_mul(&mut self) { 833 self.context.i32_binop(self.masm, |masm, dst, src, size| { 834 masm.mul(dst, dst, src, size); 835 TypedReg::i32(dst) 836 }); 837 } 838 839 fn visit_i64_mul(&mut self) { 840 self.context.i64_binop(self.masm, |masm, dst, src, size| { 841 masm.mul(dst, dst, src, size); 842 TypedReg::i64(dst) 843 }); 844 } 845 846 fn visit_i32_div_s(&mut self) { 847 use DivKind::*; 848 use OperandSize::*; 849 850 self.masm.div(&mut self.context, Signed, S32); 851 } 852 853 fn visit_i32_div_u(&mut self) { 854 use DivKind::*; 855 use OperandSize::*; 856 857 self.masm.div(&mut self.context, Unsigned, S32); 858 } 859 860 fn visit_i64_div_s(&mut self) { 861 use DivKind::*; 862 use OperandSize::*; 863 864 self.masm.div(&mut self.context, Signed, S64); 865 } 866 867 fn visit_i64_div_u(&mut self) { 868 use DivKind::*; 869 use OperandSize::*; 870 871 self.masm.div(&mut self.context, Unsigned, S64); 872 } 873 874 fn visit_i32_rem_s(&mut self) { 875 use OperandSize::*; 876 use RemKind::*; 877 878 self.masm.rem(&mut self.context, Signed, S32); 879 } 880 881 fn visit_i32_rem_u(&mut self) { 882 use OperandSize::*; 883 use RemKind::*; 884 885 self.masm.rem(&mut self.context, Unsigned, S32); 886 } 887 888 fn visit_i64_rem_s(&mut self) { 889 use OperandSize::*; 890 use RemKind::*; 891 892 self.masm.rem(&mut self.context, Signed, S64); 893 } 894 895 fn visit_i64_rem_u(&mut self) { 896 use OperandSize::*; 897 use RemKind::*; 898 899 self.masm.rem(&mut self.context, Unsigned, S64); 900 } 901 902 fn visit_i32_eq(&mut self) { 903 self.cmp_i32s(IntCmpKind::Eq); 904 } 905 906 fn visit_i64_eq(&mut self) { 907 self.cmp_i64s(IntCmpKind::Eq); 908 } 909 910 fn visit_i32_ne(&mut self) { 911 self.cmp_i32s(IntCmpKind::Ne); 912 } 913 914 fn visit_i64_ne(&mut self) { 915 self.cmp_i64s(IntCmpKind::Ne); 916 } 917 918 fn visit_i32_lt_s(&mut self) { 919 self.cmp_i32s(IntCmpKind::LtS); 920 } 921 922 fn visit_i64_lt_s(&mut self) { 923 self.cmp_i64s(IntCmpKind::LtS); 924 } 925 926 fn visit_i32_lt_u(&mut self) { 927 self.cmp_i32s(IntCmpKind::LtU); 928 } 929 930 fn visit_i64_lt_u(&mut self) { 931 self.cmp_i64s(IntCmpKind::LtU); 932 } 933 934 fn visit_i32_le_s(&mut self) { 935 self.cmp_i32s(IntCmpKind::LeS); 936 } 937 938 fn visit_i64_le_s(&mut self) { 939 self.cmp_i64s(IntCmpKind::LeS); 940 } 941 942 fn visit_i32_le_u(&mut self) { 943 self.cmp_i32s(IntCmpKind::LeU); 944 } 945 946 fn visit_i64_le_u(&mut self) { 947 self.cmp_i64s(IntCmpKind::LeU); 948 } 949 950 fn visit_i32_gt_s(&mut self) { 951 self.cmp_i32s(IntCmpKind::GtS); 952 } 953 954 fn visit_i64_gt_s(&mut self) { 955 self.cmp_i64s(IntCmpKind::GtS); 956 } 957 958 fn visit_i32_gt_u(&mut self) { 959 self.cmp_i32s(IntCmpKind::GtU); 960 } 961 962 fn visit_i64_gt_u(&mut self) { 963 self.cmp_i64s(IntCmpKind::GtU); 964 } 965 966 fn visit_i32_ge_s(&mut self) { 967 self.cmp_i32s(IntCmpKind::GeS); 968 } 969 970 fn visit_i64_ge_s(&mut self) { 971 self.cmp_i64s(IntCmpKind::GeS); 972 } 973 974 fn visit_i32_ge_u(&mut self) { 975 self.cmp_i32s(IntCmpKind::GeU); 976 } 977 978 fn visit_i64_ge_u(&mut self) { 979 self.cmp_i64s(IntCmpKind::GeU); 980 } 981 982 fn visit_i32_eqz(&mut self) { 983 use OperandSize::*; 984 985 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 986 masm.cmp_with_set(RegImm::i32(0), reg.into(), IntCmpKind::Eq, size); 987 TypedReg::i32(reg) 988 }); 989 } 990 991 fn visit_i64_eqz(&mut self) { 992 use OperandSize::*; 993 994 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 995 masm.cmp_with_set(RegImm::i64(0), reg.into(), IntCmpKind::Eq, size); 996 TypedReg::i32(reg) // Return value for `i64.eqz` is an `i32`. 997 }); 998 } 999 1000 fn visit_i32_clz(&mut self) { 1001 use OperandSize::*; 1002 1003 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 1004 masm.clz(reg, reg, size); 1005 TypedReg::i32(reg) 1006 }); 1007 } 1008 1009 fn visit_i64_clz(&mut self) { 1010 use OperandSize::*; 1011 1012 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 1013 masm.clz(reg, reg, size); 1014 TypedReg::i64(reg) 1015 }); 1016 } 1017 1018 fn visit_i32_ctz(&mut self) { 1019 use OperandSize::*; 1020 1021 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 1022 masm.ctz(reg, reg, size); 1023 TypedReg::i32(reg) 1024 }); 1025 } 1026 1027 fn visit_i64_ctz(&mut self) { 1028 use OperandSize::*; 1029 1030 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 1031 masm.ctz(reg, reg, size); 1032 TypedReg::i64(reg) 1033 }); 1034 } 1035 1036 fn visit_i32_and(&mut self) { 1037 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1038 masm.and(dst, dst, src, size); 1039 TypedReg::i32(dst) 1040 }); 1041 } 1042 1043 fn visit_i64_and(&mut self) { 1044 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1045 masm.and(dst, dst, src, size); 1046 TypedReg::i64(dst) 1047 }); 1048 } 1049 1050 fn visit_i32_or(&mut self) { 1051 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1052 masm.or(dst, dst, src, size); 1053 TypedReg::i32(dst) 1054 }); 1055 } 1056 1057 fn visit_i64_or(&mut self) { 1058 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1059 masm.or(dst, dst, src, size); 1060 TypedReg::i64(dst) 1061 }); 1062 } 1063 1064 fn visit_i32_xor(&mut self) { 1065 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1066 masm.xor(dst, dst, src, size); 1067 TypedReg::i32(dst) 1068 }); 1069 } 1070 1071 fn visit_i64_xor(&mut self) { 1072 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1073 masm.xor(dst, dst, src, size); 1074 TypedReg::i64(dst) 1075 }); 1076 } 1077 1078 fn visit_i32_shl(&mut self) { 1079 use ShiftKind::*; 1080 1081 self.context.i32_shift(self.masm, Shl); 1082 } 1083 1084 fn visit_i64_shl(&mut self) { 1085 use ShiftKind::*; 1086 1087 self.context.i64_shift(self.masm, Shl); 1088 } 1089 1090 fn visit_i32_shr_s(&mut self) { 1091 use ShiftKind::*; 1092 1093 self.context.i32_shift(self.masm, ShrS); 1094 } 1095 1096 fn visit_i64_shr_s(&mut self) { 1097 use ShiftKind::*; 1098 1099 self.context.i64_shift(self.masm, ShrS); 1100 } 1101 1102 fn visit_i32_shr_u(&mut self) { 1103 use ShiftKind::*; 1104 1105 self.context.i32_shift(self.masm, ShrU); 1106 } 1107 1108 fn visit_i64_shr_u(&mut self) { 1109 use ShiftKind::*; 1110 1111 self.context.i64_shift(self.masm, ShrU); 1112 } 1113 1114 fn visit_i32_rotl(&mut self) { 1115 use ShiftKind::*; 1116 1117 self.context.i32_shift(self.masm, Rotl); 1118 } 1119 1120 fn visit_i64_rotl(&mut self) { 1121 use ShiftKind::*; 1122 1123 self.context.i64_shift(self.masm, Rotl); 1124 } 1125 1126 fn visit_i32_rotr(&mut self) { 1127 use ShiftKind::*; 1128 1129 self.context.i32_shift(self.masm, Rotr); 1130 } 1131 1132 fn visit_i64_rotr(&mut self) { 1133 use ShiftKind::*; 1134 1135 self.context.i64_shift(self.masm, Rotr); 1136 } 1137 1138 fn visit_end(&mut self) { 1139 if !self.context.reachable { 1140 self.handle_unreachable_end(); 1141 } else { 1142 let mut control = self.control_frames.pop().unwrap(); 1143 control.emit_end(self.masm, &mut self.context); 1144 } 1145 } 1146 1147 fn visit_i32_popcnt(&mut self) { 1148 use OperandSize::*; 1149 self.masm.popcnt(&mut self.context, S32); 1150 } 1151 1152 fn visit_i64_popcnt(&mut self) { 1153 use OperandSize::*; 1154 1155 self.masm.popcnt(&mut self.context, S64); 1156 } 1157 1158 fn visit_i32_wrap_i64(&mut self) { 1159 use OperandSize::*; 1160 1161 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1162 masm.wrap(reg, reg); 1163 TypedReg::i32(reg) 1164 }); 1165 } 1166 1167 fn visit_i64_extend_i32_s(&mut self) { 1168 use OperandSize::*; 1169 1170 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1171 masm.extend(reg, reg, ExtendKind::I64ExtendI32S); 1172 TypedReg::i64(reg) 1173 }); 1174 } 1175 1176 fn visit_i64_extend_i32_u(&mut self) { 1177 use OperandSize::*; 1178 1179 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1180 masm.extend(reg, reg, ExtendKind::I64ExtendI32U); 1181 TypedReg::i64(reg) 1182 }); 1183 } 1184 1185 fn visit_i32_extend8_s(&mut self) { 1186 use OperandSize::*; 1187 1188 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1189 masm.extend(reg, reg, ExtendKind::I32Extend8S); 1190 TypedReg::i32(reg) 1191 }); 1192 } 1193 1194 fn visit_i32_extend16_s(&mut self) { 1195 use OperandSize::*; 1196 1197 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1198 masm.extend(reg, reg, ExtendKind::I32Extend16S); 1199 TypedReg::i32(reg) 1200 }); 1201 } 1202 1203 fn visit_i64_extend8_s(&mut self) { 1204 use OperandSize::*; 1205 1206 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1207 masm.extend(reg, reg, ExtendKind::I64Extend8S); 1208 TypedReg::i64(reg) 1209 }); 1210 } 1211 1212 fn visit_i64_extend16_s(&mut self) { 1213 use OperandSize::*; 1214 1215 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1216 masm.extend(reg, reg, ExtendKind::I64Extend16S); 1217 TypedReg::i64(reg) 1218 }); 1219 } 1220 1221 fn visit_i64_extend32_s(&mut self) { 1222 use OperandSize::*; 1223 1224 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1225 masm.extend(reg, reg, ExtendKind::I64Extend32S); 1226 TypedReg::i64(reg) 1227 }); 1228 } 1229 1230 fn visit_i32_trunc_f32_s(&mut self) { 1231 use OperandSize::*; 1232 1233 self.context 1234 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1235 masm.signed_truncate(src, dst, S32, dst_size, TruncKind::Unchecked); 1236 }); 1237 } 1238 1239 fn visit_i32_trunc_f32_u(&mut self) { 1240 use OperandSize::*; 1241 1242 self.context.convert_op_with_tmp_reg( 1243 self.masm, 1244 WasmValType::I32, 1245 RegClass::Float, 1246 |masm, dst, src, tmp_fpr, dst_size| { 1247 masm.unsigned_truncate(src, dst, tmp_fpr, S32, dst_size, TruncKind::Unchecked); 1248 }, 1249 ); 1250 } 1251 1252 fn visit_i32_trunc_f64_s(&mut self) { 1253 use OperandSize::*; 1254 1255 self.context 1256 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1257 masm.signed_truncate(src, dst, S64, dst_size, TruncKind::Unchecked); 1258 }); 1259 } 1260 1261 fn visit_i32_trunc_f64_u(&mut self) { 1262 use OperandSize::*; 1263 1264 self.context.convert_op_with_tmp_reg( 1265 self.masm, 1266 WasmValType::I32, 1267 RegClass::Float, 1268 |masm, dst, src, tmp_fpr, dst_size| { 1269 masm.unsigned_truncate(src, dst, tmp_fpr, S64, dst_size, TruncKind::Unchecked); 1270 }, 1271 ); 1272 } 1273 1274 fn visit_i64_trunc_f32_s(&mut self) { 1275 use OperandSize::*; 1276 1277 self.context 1278 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1279 masm.signed_truncate(src, dst, S32, dst_size, TruncKind::Unchecked); 1280 }); 1281 } 1282 1283 fn visit_i64_trunc_f32_u(&mut self) { 1284 use OperandSize::*; 1285 1286 self.context.convert_op_with_tmp_reg( 1287 self.masm, 1288 WasmValType::I64, 1289 RegClass::Float, 1290 |masm, dst, src, tmp_fpr, dst_size| { 1291 masm.unsigned_truncate(src, dst, tmp_fpr, S32, dst_size, TruncKind::Unchecked); 1292 }, 1293 ); 1294 } 1295 1296 fn visit_i64_trunc_f64_s(&mut self) { 1297 use OperandSize::*; 1298 1299 self.context 1300 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1301 masm.signed_truncate(src, dst, S64, dst_size, TruncKind::Unchecked); 1302 }); 1303 } 1304 1305 fn visit_i64_trunc_f64_u(&mut self) { 1306 use OperandSize::*; 1307 1308 self.context.convert_op_with_tmp_reg( 1309 self.masm, 1310 WasmValType::I64, 1311 RegClass::Float, 1312 |masm, dst, src, tmp_fpr, dst_size| { 1313 masm.unsigned_truncate(src, dst, tmp_fpr, S64, dst_size, TruncKind::Unchecked); 1314 }, 1315 ); 1316 } 1317 1318 fn visit_i32_reinterpret_f32(&mut self) { 1319 self.context 1320 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, size| { 1321 masm.reinterpret_float_as_int(src.into(), dst, size); 1322 }); 1323 } 1324 1325 fn visit_i64_reinterpret_f64(&mut self) { 1326 self.context 1327 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, size| { 1328 masm.reinterpret_float_as_int(src.into(), dst, size); 1329 }); 1330 } 1331 1332 fn visit_local_get(&mut self, index: u32) { 1333 use WasmValType::*; 1334 let context = &mut self.context; 1335 let slot = context.frame.get_wasm_local(index); 1336 match slot.ty { 1337 I32 | I64 | F32 | F64 | V128 => context.stack.push(Val::local(index, slot.ty)), 1338 Ref(rt) => match rt.heap_type { 1339 WasmHeapType::Func => context.stack.push(Val::local(index, slot.ty)), 1340 ht => unimplemented!("Support for WasmHeapType: {ht}"), 1341 }, 1342 } 1343 } 1344 1345 fn visit_local_set(&mut self, index: u32) { 1346 let src = self.emit_set_local(index); 1347 self.context.free_reg(src); 1348 } 1349 1350 fn visit_call(&mut self, index: u32) { 1351 let callee = self.env.callee_from_index(FuncIndex::from_u32(index)); 1352 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee) 1353 } 1354 1355 fn visit_call_indirect(&mut self, type_index: u32, table_index: u32) { 1356 // Spill now because `emit_lazy_init_funcref` and the `FnCall::emit` 1357 // invocations will both trigger spills since they both call functions. 1358 // However, the machine instructions for the spill emitted by 1359 // `emit_lazy_funcref` will be jumped over if the funcref was previously 1360 // initialized which may result in the machine stack becoming 1361 // unbalanced. 1362 self.context.spill(self.masm); 1363 1364 let type_index = TypeIndex::from_u32(type_index); 1365 let table_index = TableIndex::from_u32(table_index); 1366 1367 self.emit_lazy_init_funcref(table_index); 1368 1369 // Perform the indirect call. 1370 // This code assumes that [`Self::emit_lazy_init_funcref`] will 1371 // push the funcref to the value stack. 1372 match self.env.translation.module.table_plans[table_index].style { 1373 TableStyle::CallerChecksSignature { lazy_init: true } => { 1374 let funcref_ptr = self.context.stack.peek().map(|v| v.unwrap_reg()).unwrap(); 1375 self.masm 1376 .trapz(funcref_ptr.into(), TrapCode::IndirectCallToNull); 1377 self.emit_typecheck_funcref(funcref_ptr.into(), type_index); 1378 } 1379 _ => unimplemented!("Support for eager table init"), 1380 } 1381 1382 let callee = self.env.funcref(type_index); 1383 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee) 1384 } 1385 1386 fn visit_table_init(&mut self, elem: u32, table: u32) { 1387 debug_assert!(self.context.stack.len() >= 3); 1388 let at = self.context.stack.len() - 3; 1389 1390 self.context 1391 .stack 1392 .insert_many(at, &[table.try_into().unwrap(), elem.try_into().unwrap()]); 1393 1394 let builtin = self.env.builtins.table_init::<M::ABI, M::Ptr>(); 1395 FnCall::emit::<M>( 1396 &mut self.env, 1397 self.masm, 1398 &mut self.context, 1399 Callee::Builtin(builtin.clone()), 1400 ) 1401 } 1402 1403 fn visit_table_copy(&mut self, dst: u32, src: u32) { 1404 debug_assert!(self.context.stack.len() >= 3); 1405 let at = self.context.stack.len() - 3; 1406 self.context 1407 .stack 1408 .insert_many(at, &[dst.try_into().unwrap(), src.try_into().unwrap()]); 1409 1410 let builtin = self.env.builtins.table_copy::<M::ABI, M::Ptr>(); 1411 FnCall::emit::<M>( 1412 &mut self.env, 1413 self.masm, 1414 &mut self.context, 1415 Callee::Builtin(builtin), 1416 ) 1417 } 1418 1419 fn visit_table_get(&mut self, table: u32) { 1420 let table_index = TableIndex::from_u32(table); 1421 let plan = self.env.table_plan(table_index); 1422 let heap_type = plan.table.wasm_ty.heap_type; 1423 let style = &plan.style; 1424 1425 match heap_type { 1426 WasmHeapType::Func => match style { 1427 TableStyle::CallerChecksSignature { lazy_init: true } => { 1428 self.emit_lazy_init_funcref(table_index) 1429 } 1430 _ => unimplemented!("Support for eager table init"), 1431 }, 1432 t => unimplemented!("Support for WasmHeapType: {t}"), 1433 } 1434 } 1435 1436 fn visit_table_grow(&mut self, table: u32) { 1437 let table_index = TableIndex::from_u32(table); 1438 let table_plan = self.env.table_plan(table_index); 1439 let builtin = match table_plan.table.wasm_ty.heap_type { 1440 WasmHeapType::Func => self.env.builtins.table_grow_func_ref::<M::ABI, M::Ptr>(), 1441 ty => unimplemented!("Support for HeapType: {ty}"), 1442 }; 1443 1444 let len = self.context.stack.len(); 1445 // table.grow` requires at least 2 elements on the value stack. 1446 debug_assert!(len >= 2); 1447 let at = len - 2; 1448 1449 // The table_grow builtin expects the parameters in a different 1450 // order. 1451 // The value stack at this point should contain: 1452 // [ init_value | delta ] (stack top) 1453 // but the builtin function expects the init value as the last 1454 // argument. 1455 self.context.stack.inner_mut().swap(len - 1, len - 2); 1456 self.context 1457 .stack 1458 .insert_many(at, &[table.try_into().unwrap()]); 1459 1460 FnCall::emit::<M>( 1461 &mut self.env, 1462 self.masm, 1463 &mut self.context, 1464 Callee::Builtin(builtin.clone()), 1465 ) 1466 } 1467 1468 fn visit_table_size(&mut self, table: u32) { 1469 let table_index = TableIndex::from_u32(table); 1470 let table_data = self.env.resolve_table_data(table_index); 1471 self.emit_compute_table_size(&table_data); 1472 } 1473 1474 fn visit_table_fill(&mut self, table: u32) { 1475 let table_index = TableIndex::from_u32(table); 1476 let table_plan = self.env.table_plan(table_index); 1477 let builtin = match table_plan.table.wasm_ty.heap_type { 1478 WasmHeapType::Func => self.env.builtins.table_fill_func_ref::<M::ABI, M::Ptr>(), 1479 ty => unimplemented!("Support for heap type: {ty}"), 1480 }; 1481 1482 let len = self.context.stack.len(); 1483 debug_assert!(len >= 3); 1484 let at = len - 3; 1485 self.context 1486 .stack 1487 .insert_many(at, &[table.try_into().unwrap()]); 1488 FnCall::emit::<M>( 1489 &mut self.env, 1490 self.masm, 1491 &mut self.context, 1492 Callee::Builtin(builtin.clone()), 1493 ) 1494 } 1495 1496 fn visit_table_set(&mut self, table: u32) { 1497 let ptr_type = self.env.ptr_type(); 1498 let table_index = TableIndex::from_u32(table); 1499 let table_data = self.env.resolve_table_data(table_index); 1500 let plan = self.env.table_plan(table_index); 1501 match plan.table.wasm_ty.heap_type { 1502 WasmHeapType::Func => match plan.style { 1503 TableStyle::CallerChecksSignature { lazy_init: true } => { 1504 let value = self.context.pop_to_reg(self.masm, None); 1505 let index = self.context.pop_to_reg(self.masm, None); 1506 let base = self.context.any_gpr(self.masm); 1507 let elem_addr = 1508 self.emit_compute_table_elem_addr(index.into(), base, &table_data); 1509 // Set the initialized bit. 1510 self.masm.or( 1511 value.into(), 1512 value.into(), 1513 RegImm::i64(FUNCREF_INIT_BIT as i64), 1514 ptr_type.into(), 1515 ); 1516 1517 self.masm.store_ptr(value.into(), elem_addr); 1518 1519 self.context.free_reg(value); 1520 self.context.free_reg(index); 1521 self.context.free_reg(base); 1522 } 1523 _ => unimplemented!("Support for eager table init"), 1524 }, 1525 ty => unimplemented!("Support for WasmHeapType: {ty}"), 1526 }; 1527 } 1528 1529 fn visit_elem_drop(&mut self, index: u32) { 1530 let elem_drop = self.env.builtins.elem_drop::<M::ABI, M::Ptr>(); 1531 self.context.stack.extend([index.try_into().unwrap()]); 1532 FnCall::emit::<M>( 1533 &mut self.env, 1534 self.masm, 1535 &mut self.context, 1536 Callee::Builtin(elem_drop), 1537 ) 1538 } 1539 1540 fn visit_memory_init(&mut self, data_index: u32, mem: u32) { 1541 debug_assert!(self.context.stack.len() >= 3); 1542 let at = self.context.stack.len() - 3; 1543 self.context.stack.insert_many( 1544 at, 1545 &[mem.try_into().unwrap(), data_index.try_into().unwrap()], 1546 ); 1547 let builtin = self.env.builtins.memory_init::<M::ABI, M::Ptr>(); 1548 FnCall::emit::<M>( 1549 &mut self.env, 1550 self.masm, 1551 &mut self.context, 1552 Callee::Builtin(builtin), 1553 ) 1554 } 1555 1556 fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) { 1557 // At this point, the stack is expected to contain: 1558 // [ dst_offset, src_offset, len ] 1559 // The following code inserts the missing params, so that stack contains: 1560 // [ vmctx, dst_mem, dst_offset, src_mem, src_offset, len ] 1561 // Which is the order expected by the builtin function. 1562 debug_assert!(self.context.stack.len() >= 3); 1563 let at = self.context.stack.len() - 2; 1564 self.context 1565 .stack 1566 .insert_many(at, &[src_mem.try_into().unwrap()]); 1567 1568 // One element was inserted above, so instead of 3, we use 4. 1569 let at = self.context.stack.len() - 4; 1570 self.context 1571 .stack 1572 .insert_many(at, &[dst_mem.try_into().unwrap()]); 1573 1574 let builtin = self.env.builtins.memory_copy::<M::ABI, M::Ptr>(); 1575 1576 FnCall::emit::<M>( 1577 &mut self.env, 1578 self.masm, 1579 &mut self.context, 1580 Callee::Builtin(builtin), 1581 ) 1582 } 1583 1584 fn visit_memory_fill(&mut self, mem: u32) { 1585 debug_assert!(self.context.stack.len() >= 3); 1586 let at = self.context.stack.len() - 3; 1587 1588 self.context 1589 .stack 1590 .insert_many(at, &[mem.try_into().unwrap()]); 1591 1592 let builtin = self.env.builtins.memory_fill::<M::ABI, M::Ptr>(); 1593 FnCall::emit::<M>( 1594 &mut self.env, 1595 self.masm, 1596 &mut self.context, 1597 Callee::Builtin(builtin), 1598 ) 1599 } 1600 1601 fn visit_memory_size(&mut self, mem: u32) { 1602 let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem)); 1603 self.emit_compute_memory_size(&heap); 1604 } 1605 1606 fn visit_memory_grow(&mut self, mem: u32) { 1607 debug_assert!(self.context.stack.len() >= 1); 1608 // The stack at this point contains: [ delta ] 1609 // The desired state is 1610 // [ vmctx, delta, index ] 1611 self.context.stack.extend([mem.try_into().unwrap()]); 1612 1613 let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem)); 1614 let builtin = self.env.builtins.memory32_grow::<M::ABI, M::Ptr>(); 1615 FnCall::emit::<M>( 1616 &mut self.env, 1617 self.masm, 1618 &mut self.context, 1619 Callee::Builtin(builtin), 1620 ); 1621 1622 // The memory32_grow builtin returns a pointer type, therefore we must 1623 // ensure that the return type is representative of the address space of 1624 // the heap type. 1625 match (self.env.ptr_type(), heap.ty) { 1626 (WasmValType::I64, WasmValType::I64) => {} 1627 // When the heap type is smaller than the pointer type, we adjust 1628 // the result of the memory32_grow builtin. 1629 (WasmValType::I64, WasmValType::I32) => { 1630 let top: Reg = self.context.pop_to_reg(self.masm, None).into(); 1631 self.masm.wrap(top.into(), top.into()); 1632 self.context.stack.push(TypedReg::i32(top).into()); 1633 } 1634 _ => unimplemented!("Support for 32-bit platforms"), 1635 } 1636 } 1637 1638 fn visit_data_drop(&mut self, data_index: u32) { 1639 self.context.stack.extend([data_index.try_into().unwrap()]); 1640 1641 let builtin = self.env.builtins.data_drop::<M::ABI, M::Ptr>(); 1642 FnCall::emit::<M>( 1643 &mut self.env, 1644 self.masm, 1645 &mut self.context, 1646 Callee::Builtin(builtin), 1647 ) 1648 } 1649 1650 fn visit_nop(&mut self) {} 1651 1652 fn visit_if(&mut self, blockty: BlockType) { 1653 self.control_frames.push(ControlStackFrame::r#if( 1654 self.env.resolve_block_sig(blockty), 1655 self.masm, 1656 &mut self.context, 1657 )); 1658 } 1659 1660 fn visit_else(&mut self) { 1661 if !self.context.reachable { 1662 self.handle_unreachable_else(); 1663 } else { 1664 let control = self 1665 .control_frames 1666 .last_mut() 1667 .unwrap_or_else(|| panic!("Expected active control stack frame for else")); 1668 control.emit_else(self.masm, &mut self.context); 1669 } 1670 } 1671 1672 fn visit_block(&mut self, blockty: BlockType) { 1673 self.control_frames.push(ControlStackFrame::block( 1674 self.env.resolve_block_sig(blockty), 1675 self.masm, 1676 &mut self.context, 1677 )); 1678 } 1679 1680 fn visit_loop(&mut self, blockty: BlockType) { 1681 self.control_frames.push(ControlStackFrame::r#loop( 1682 self.env.resolve_block_sig(blockty), 1683 self.masm, 1684 &mut self.context, 1685 )); 1686 } 1687 1688 fn visit_br(&mut self, depth: u32) { 1689 let index = control_index(depth, self.control_frames.len()); 1690 let frame = &mut self.control_frames[index]; 1691 self.context 1692 .unconditional_jump(frame, self.masm, |masm, cx, frame| { 1693 frame 1694 .pop_abi_results::<M, _>(cx, masm, |results, _, _| results.ret_area().copied()); 1695 }); 1696 } 1697 1698 fn visit_br_if(&mut self, depth: u32) { 1699 let index = control_index(depth, self.control_frames.len()); 1700 let frame = &mut self.control_frames[index]; 1701 frame.set_as_target(); 1702 1703 let top = { 1704 let top = self.context.without::<TypedReg, M, _>( 1705 frame.results::<M>().regs(), 1706 self.masm, 1707 |ctx, masm| ctx.pop_to_reg(masm, None), 1708 ); 1709 // Explicitly save any live registers and locals before setting up 1710 // the branch state. 1711 // In some cases, calculating the `top` value above, will result in 1712 // a spill, thus the following one will result in a no-op. 1713 self.context.spill(self.masm); 1714 frame.top_abi_results::<M, _>( 1715 &mut self.context, 1716 self.masm, 1717 |results, context, masm| { 1718 // In the case of `br_if` there's a possibility that we'll 1719 // exit early from the block or fallthrough, for 1720 // a fallthrough, we cannot rely on the pre-computed return area; 1721 // it must be recalculated so that any values that are 1722 // generated are correctly placed near the current stack 1723 // pointer. 1724 results.on_stack().then(|| { 1725 let stack_consumed = context.stack.sizeof(results.stack_operands_len()); 1726 let base = masm.sp_offset().as_u32() - stack_consumed; 1727 let offs = base + results.size(); 1728 RetArea::sp(SPOffset::from_u32(offs)) 1729 }) 1730 }, 1731 ); 1732 top 1733 }; 1734 1735 // Emit instructions to balance the machine stack if the frame has 1736 // a different offset. 1737 let current_sp_offset = self.masm.sp_offset(); 1738 let results_size = frame.results::<M>().size(); 1739 let state = frame.stack_state(); 1740 let (label, cmp, needs_cleanup) = if current_sp_offset > state.target_offset { 1741 (self.masm.get_label(), IntCmpKind::Eq, true) 1742 } else { 1743 (*frame.label(), IntCmpKind::Ne, false) 1744 }; 1745 1746 self.masm 1747 .branch(cmp, top.reg.into(), top.reg.into(), label, OperandSize::S32); 1748 self.context.free_reg(top); 1749 1750 if needs_cleanup { 1751 // Emit instructions to balance the stack and jump if not falling 1752 // through. 1753 self.masm.memmove( 1754 current_sp_offset, 1755 state.target_offset, 1756 results_size, 1757 MemMoveDirection::LowToHigh, 1758 ); 1759 self.masm.ensure_sp_for_jump(state.target_offset); 1760 self.masm.jmp(*frame.label()); 1761 1762 // Restore sp_offset to what it was for falling through and emit 1763 // fallthrough label. 1764 self.masm.reset_stack_pointer(current_sp_offset); 1765 self.masm.bind(label); 1766 } 1767 } 1768 1769 fn visit_br_table(&mut self, targets: BrTable<'a>) { 1770 // +1 to account for the default target. 1771 let len = targets.len() + 1; 1772 // SmallVec<[_; 5]> to match the binary emission layer (e.g 1773 // see `JmpTableSeq'), but here we use 5 instead since we 1774 // bundle the default target as the last element in the array. 1775 let labels: SmallVec<[_; 5]> = (0..len).map(|_| self.masm.get_label()).collect(); 1776 1777 let default_index = control_index(targets.default(), self.control_frames.len()); 1778 let default_frame = &mut self.control_frames[default_index]; 1779 let default_result = default_frame.results::<M>(); 1780 1781 let (index, tmp) = { 1782 let index_and_tmp = self.context.without::<(TypedReg, _), M, _>( 1783 default_result.regs(), 1784 self.masm, 1785 |cx, masm| (cx.pop_to_reg(masm, None), cx.any_gpr(masm)), 1786 ); 1787 1788 // Materialize any constants or locals into their result representation, 1789 // so that when reachability is restored, they are correctly located. 1790 default_frame.top_abi_results::<M, _>(&mut self.context, self.masm, |results, _, _| { 1791 results.ret_area().copied() 1792 }); 1793 index_and_tmp 1794 }; 1795 1796 self.masm.jmp_table(&labels, index.into(), tmp); 1797 // Save the original stack pointer offset; we will reset the stack 1798 // pointer to this offset after jumping to each of the targets. Each 1799 // jump might adjust the stack according to the base offset of the 1800 // target. 1801 let current_sp = self.masm.sp_offset(); 1802 1803 for (t, l) in targets 1804 .targets() 1805 .into_iter() 1806 .chain(std::iter::once(Ok(targets.default()))) 1807 .zip(labels.iter()) 1808 { 1809 let control_index = control_index(t.unwrap(), self.control_frames.len()); 1810 let frame = &mut self.control_frames[control_index]; 1811 // Reset the stack pointer to its original offset. This is needed 1812 // because each jump will potentially adjust the stack pointer 1813 // according to the base offset of the target. 1814 self.masm.reset_stack_pointer(current_sp); 1815 1816 // NB: We don't perform any result handling as it was 1817 // already taken care of above before jumping to the 1818 // jump table. 1819 self.masm.bind(*l); 1820 // Ensure that the stack pointer is correctly positioned before 1821 // jumping to the jump table code. 1822 let state = frame.stack_state(); 1823 self.masm.ensure_sp_for_jump(state.target_offset); 1824 self.masm.jmp(*frame.label()); 1825 frame.set_as_target(); 1826 } 1827 // Finally reset the stack pointer to the original location. 1828 // The reachability analysis, will ensure it's correctly located 1829 // once reachability is restored. 1830 self.masm.reset_stack_pointer(current_sp); 1831 self.context.reachable = false; 1832 self.context.free_reg(index.reg); 1833 self.context.free_reg(tmp); 1834 } 1835 1836 fn visit_return(&mut self) { 1837 // Grab the outermost frame, which is the function's body 1838 // frame. We don't rely on [`codegen::control_index`] since 1839 // this frame is implicit and we know that it should exist at 1840 // index 0. 1841 let outermost = &mut self.control_frames[0]; 1842 self.context 1843 .unconditional_jump(outermost, self.masm, |masm, cx, frame| { 1844 frame 1845 .pop_abi_results::<M, _>(cx, masm, |results, _, _| results.ret_area().copied()); 1846 }); 1847 } 1848 1849 fn visit_unreachable(&mut self) { 1850 self.masm.unreachable(); 1851 self.context.reachable = false; 1852 // Set the implicit outermost frame as target to perform the necessary 1853 // stack clean up. 1854 let outermost = &mut self.control_frames[0]; 1855 outermost.set_as_target(); 1856 } 1857 1858 fn visit_local_tee(&mut self, index: u32) { 1859 let typed_reg = self.emit_set_local(index); 1860 self.context.stack.push(typed_reg.into()); 1861 } 1862 1863 fn visit_global_get(&mut self, global_index: u32) { 1864 let index = GlobalIndex::from_u32(global_index); 1865 let (ty, addr) = self.emit_get_global_addr(index); 1866 let dst = self.context.reg_for_type(ty, self.masm); 1867 self.masm.load(addr, dst, ty.into()); 1868 self.context.stack.push(Val::reg(dst, ty)); 1869 } 1870 1871 fn visit_global_set(&mut self, global_index: u32) { 1872 let index = GlobalIndex::from_u32(global_index); 1873 let (ty, addr) = self.emit_get_global_addr(index); 1874 1875 let typed_reg = self.context.pop_to_reg(self.masm, None); 1876 self.context.free_reg(typed_reg.reg); 1877 self.masm.store(typed_reg.reg.into(), addr, ty.into()); 1878 } 1879 1880 fn visit_drop(&mut self) { 1881 self.context.drop_last(1, |regalloc, val| match val { 1882 Val::Reg(tr) => regalloc.free(tr.reg.into()), 1883 Val::Memory(m) => self.masm.free_stack(m.slot.size), 1884 _ => {} 1885 }); 1886 } 1887 1888 fn visit_select(&mut self) { 1889 let cond = self.context.pop_to_reg(self.masm, None); 1890 let val2 = self.context.pop_to_reg(self.masm, None); 1891 let val1 = self.context.pop_to_reg(self.masm, None); 1892 self.masm 1893 .cmp(cond.reg.into(), RegImm::i32(0), OperandSize::S32); 1894 // Conditionally move val1 to val2 if the comparison is 1895 // not zero. 1896 self.masm 1897 .cmov(val1.into(), val2.into(), IntCmpKind::Ne, val1.ty.into()); 1898 self.context.stack.push(val2.into()); 1899 self.context.free_reg(val1.reg); 1900 self.context.free_reg(cond); 1901 } 1902 1903 fn visit_i32_load(&mut self, memarg: MemArg) { 1904 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S32, None); 1905 } 1906 1907 fn visit_i32_load8_s(&mut self, memarg: MemArg) { 1908 self.emit_wasm_load( 1909 &memarg, 1910 WasmValType::I32, 1911 OperandSize::S8, 1912 Some(ExtendKind::I32Extend8S), 1913 ); 1914 } 1915 1916 fn visit_i32_load8_u(&mut self, memarg: MemArg) { 1917 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S8, None); 1918 } 1919 1920 fn visit_i32_load16_s(&mut self, memarg: MemArg) { 1921 self.emit_wasm_load( 1922 &memarg, 1923 WasmValType::I32, 1924 OperandSize::S16, 1925 Some(ExtendKind::I32Extend16S), 1926 ) 1927 } 1928 1929 fn visit_i32_load16_u(&mut self, memarg: MemArg) { 1930 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S16, None) 1931 } 1932 1933 fn visit_i32_store(&mut self, memarg: MemArg) { 1934 self.emit_wasm_store(&memarg, OperandSize::S32); 1935 } 1936 1937 fn visit_i32_store8(&mut self, memarg: MemArg) { 1938 self.emit_wasm_store(&memarg, OperandSize::S8) 1939 } 1940 1941 fn visit_i32_store16(&mut self, memarg: MemArg) { 1942 self.emit_wasm_store(&memarg, OperandSize::S16) 1943 } 1944 1945 fn visit_i64_load8_s(&mut self, memarg: MemArg) { 1946 self.emit_wasm_load( 1947 &memarg, 1948 WasmValType::I64, 1949 OperandSize::S8, 1950 Some(ExtendKind::I64Extend8S), 1951 ) 1952 } 1953 1954 fn visit_i64_load8_u(&mut self, memarg: MemArg) { 1955 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S8, None) 1956 } 1957 1958 fn visit_i64_load16_u(&mut self, memarg: MemArg) { 1959 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S16, None) 1960 } 1961 1962 fn visit_i64_load16_s(&mut self, memarg: MemArg) { 1963 self.emit_wasm_load( 1964 &memarg, 1965 WasmValType::I64, 1966 OperandSize::S16, 1967 Some(ExtendKind::I64Extend16S), 1968 ) 1969 } 1970 1971 fn visit_i64_load32_u(&mut self, memarg: MemArg) { 1972 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S32, None) 1973 } 1974 1975 fn visit_i64_load32_s(&mut self, memarg: MemArg) { 1976 self.emit_wasm_load( 1977 &memarg, 1978 WasmValType::I64, 1979 OperandSize::S32, 1980 Some(ExtendKind::I64Extend32S), 1981 ) 1982 } 1983 1984 fn visit_i64_load(&mut self, memarg: MemArg) { 1985 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S64, None) 1986 } 1987 1988 fn visit_i64_store(&mut self, memarg: MemArg) -> Self::Output { 1989 self.emit_wasm_store(&memarg, OperandSize::S64) 1990 } 1991 1992 fn visit_i64_store8(&mut self, memarg: MemArg) -> Self::Output { 1993 self.emit_wasm_store(&memarg, OperandSize::S8) 1994 } 1995 1996 fn visit_i64_store16(&mut self, memarg: MemArg) -> Self::Output { 1997 self.emit_wasm_store(&memarg, OperandSize::S16) 1998 } 1999 2000 fn visit_i64_store32(&mut self, memarg: MemArg) -> Self::Output { 2001 self.emit_wasm_store(&memarg, OperandSize::S32) 2002 } 2003 2004 fn visit_f32_load(&mut self, memarg: MemArg) { 2005 self.emit_wasm_load(&memarg, WasmValType::F32, OperandSize::S32, None) 2006 } 2007 2008 fn visit_f32_store(&mut self, memarg: MemArg) { 2009 self.emit_wasm_store(&memarg, OperandSize::S32) 2010 } 2011 2012 fn visit_f64_load(&mut self, memarg: MemArg) { 2013 self.emit_wasm_load(&memarg, WasmValType::F64, OperandSize::S64, None) 2014 } 2015 2016 fn visit_f64_store(&mut self, memarg: MemArg) { 2017 self.emit_wasm_store(&memarg, OperandSize::S64) 2018 } 2019 2020 fn visit_v128_load(&mut self, memarg: MemArg) { 2021 self.emit_wasm_load(&memarg, WasmValType::V128, OperandSize::S128, None) 2022 } 2023 2024 fn visit_v128_store(&mut self, memarg: MemArg) { 2025 self.emit_wasm_store(&memarg, OperandSize::S128) 2026 } 2027 2028 fn visit_i32_trunc_sat_f32_s(&mut self) { 2029 use OperandSize::*; 2030 2031 self.context 2032 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2033 masm.signed_truncate(src, dst, S32, dst_size, TruncKind::Checked); 2034 }); 2035 } 2036 2037 fn visit_i32_trunc_sat_f32_u(&mut self) { 2038 use OperandSize::*; 2039 2040 self.context.convert_op_with_tmp_reg( 2041 self.masm, 2042 WasmValType::I32, 2043 RegClass::Float, 2044 |masm, dst, src, tmp_fpr, dst_size| { 2045 masm.unsigned_truncate(src, dst, tmp_fpr, S32, dst_size, TruncKind::Checked); 2046 }, 2047 ); 2048 } 2049 2050 fn visit_i32_trunc_sat_f64_s(&mut self) { 2051 use OperandSize::*; 2052 2053 self.context 2054 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2055 masm.signed_truncate(src, dst, S64, dst_size, TruncKind::Checked); 2056 }); 2057 } 2058 2059 fn visit_i32_trunc_sat_f64_u(&mut self) { 2060 use OperandSize::*; 2061 2062 self.context.convert_op_with_tmp_reg( 2063 self.masm, 2064 WasmValType::I32, 2065 RegClass::Float, 2066 |masm, dst, src, tmp_fpr, dst_size| { 2067 masm.unsigned_truncate(src, dst, tmp_fpr, S64, dst_size, TruncKind::Checked); 2068 }, 2069 ); 2070 } 2071 2072 fn visit_i64_trunc_sat_f32_s(&mut self) { 2073 use OperandSize::*; 2074 2075 self.context 2076 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2077 masm.signed_truncate(src, dst, S32, dst_size, TruncKind::Checked); 2078 }); 2079 } 2080 2081 fn visit_i64_trunc_sat_f32_u(&mut self) { 2082 use OperandSize::*; 2083 2084 self.context.convert_op_with_tmp_reg( 2085 self.masm, 2086 WasmValType::I64, 2087 RegClass::Float, 2088 |masm, dst, src, tmp_fpr, dst_size| { 2089 masm.unsigned_truncate(src, dst, tmp_fpr, S32, dst_size, TruncKind::Checked); 2090 }, 2091 ); 2092 } 2093 2094 fn visit_i64_trunc_sat_f64_s(&mut self) { 2095 use OperandSize::*; 2096 2097 self.context 2098 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2099 masm.signed_truncate(src, dst, S64, dst_size, TruncKind::Checked); 2100 }); 2101 } 2102 2103 fn visit_i64_trunc_sat_f64_u(&mut self) { 2104 use OperandSize::*; 2105 2106 self.context.convert_op_with_tmp_reg( 2107 self.masm, 2108 WasmValType::I64, 2109 RegClass::Float, 2110 |masm, dst, src, tmp_fpr, dst_size| { 2111 masm.unsigned_truncate(src, dst, tmp_fpr, S64, dst_size, TruncKind::Checked); 2112 }, 2113 ); 2114 } 2115 2116 wasmparser::for_each_operator!(def_unsupported); 2117 } 2118 2119 impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M> 2120 where 2121 M: MacroAssembler, 2122 { 2123 fn cmp_i32s(&mut self, kind: IntCmpKind) { 2124 self.context.i32_binop(self.masm, |masm, dst, src, size| { 2125 masm.cmp_with_set(src, dst, kind, size); 2126 TypedReg::i32(dst) 2127 }); 2128 } 2129 2130 fn cmp_i64s(&mut self, kind: IntCmpKind) { 2131 self.context 2132 .i64_binop(self.masm, move |masm, dst, src, size| { 2133 masm.cmp_with_set(src, dst, kind, size); 2134 TypedReg::i32(dst) // Return value for comparisons is an `i32`. 2135 }); 2136 } 2137 } 2138 2139 impl From<WasmValType> for OperandSize { 2140 fn from(ty: WasmValType) -> OperandSize { 2141 match ty { 2142 WasmValType::I32 | WasmValType::F32 => OperandSize::S32, 2143 WasmValType::I64 | WasmValType::F64 => OperandSize::S64, 2144 WasmValType::V128 => OperandSize::S128, 2145 WasmValType::Ref(rt) => { 2146 match rt.heap_type { 2147 // TODO: Hardcoded size, assuming 64-bit support only. Once 2148 // Wasmtime supports 32-bit architectures, this will need 2149 // to be updated in such a way that the calculation of the 2150 // OperandSize will depend on the target's pointer size. 2151 WasmHeapType::Func => OperandSize::S64, 2152 t => unimplemented!("Support for WasmHeapType: {t}"), 2153 } 2154 } 2155 } 2156 } 2157 } 2158