1 //! This module is the central place for machine code emission. 2 //! It defines an implementation of wasmparser's Visitor trait 3 //! for `CodeGen`; which defines a visitor per op-code, 4 //! which validates and dispatches to the corresponding 5 //! machine code emitter. 6 7 use crate::abi::RetArea; 8 use crate::codegen::{control_index, Callee, CodeGen, ControlStackFrame, FnCall}; 9 use crate::masm::{ 10 DivKind, ExtendKind, FloatCmpKind, IntCmpKind, MacroAssembler, MemMoveDirection, OperandSize, 11 RegImm, RemKind, RoundingMode, SPOffset, ShiftKind, TruncKind, 12 }; 13 use crate::reg::{writable, Reg}; 14 use crate::stack::{TypedReg, Val}; 15 use regalloc2::RegClass; 16 use smallvec::SmallVec; 17 use wasmparser::{BlockType, BrTable, Ieee32, Ieee64, MemArg, VisitOperator, V128}; 18 use wasmtime_cranelift::TRAP_INDIRECT_CALL_TO_NULL; 19 use wasmtime_environ::{ 20 FuncIndex, GlobalIndex, MemoryIndex, TableIndex, TableStyle, TypeIndex, WasmHeapType, 21 WasmValType, FUNCREF_INIT_BIT, 22 }; 23 24 /// A macro to define unsupported WebAssembly operators. 25 /// 26 /// This macro calls itself recursively; 27 /// 1. It no-ops when matching a supported operator. 28 /// 2. Defines the visitor function and panics when 29 /// matching an unsupported operator. 30 macro_rules! def_unsupported { 31 ($( @$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident $ann:tt)*) => { 32 $( 33 def_unsupported!( 34 emit 35 $op 36 37 fn $visit(&mut self $($(,$arg: $argty)*)?) -> Self::Output { 38 $($(let _ = $arg;)*)? 39 40 self.found_unsupported_instruction = Some(stringify!($op)); 41 } 42 ); 43 )* 44 }; 45 46 (emit I32Const $($rest:tt)*) => {}; 47 (emit I64Const $($rest:tt)*) => {}; 48 (emit F32Const $($rest:tt)*) => {}; 49 (emit F64Const $($rest:tt)*) => {}; 50 (emit V128Const $($rest:tt)*) => {}; 51 (emit F32Add $($rest:tt)*) => {}; 52 (emit F64Add $($rest:tt)*) => {}; 53 (emit F32Sub $($rest:tt)*) => {}; 54 (emit F64Sub $($rest:tt)*) => {}; 55 (emit F32Mul $($rest:tt)*) => {}; 56 (emit F64Mul $($rest:tt)*) => {}; 57 (emit F32Div $($rest:tt)*) => {}; 58 (emit F64Div $($rest:tt)*) => {}; 59 (emit F32Min $($rest:tt)*) => {}; 60 (emit F64Min $($rest:tt)*) => {}; 61 (emit F32Max $($rest:tt)*) => {}; 62 (emit F64Max $($rest:tt)*) => {}; 63 (emit F32Copysign $($rest:tt)*) => {}; 64 (emit F64Copysign $($rest:tt)*) => {}; 65 (emit F32Abs $($rest:tt)*) => {}; 66 (emit F64Abs $($rest:tt)*) => {}; 67 (emit F32Neg $($rest:tt)*) => {}; 68 (emit F64Neg $($rest:tt)*) => {}; 69 (emit F32Floor $($rest:tt)*) => {}; 70 (emit F64Floor $($rest:tt)*) => {}; 71 (emit F32Ceil $($rest:tt)*) => {}; 72 (emit F64Ceil $($rest:tt)*) => {}; 73 (emit F32Nearest $($rest:tt)*) => {}; 74 (emit F64Nearest $($rest:tt)*) => {}; 75 (emit F32Trunc $($rest:tt)*) => {}; 76 (emit F64Trunc $($rest:tt)*) => {}; 77 (emit F32Sqrt $($rest:tt)*) => {}; 78 (emit F64Sqrt $($rest:tt)*) => {}; 79 (emit F32Eq $($rest:tt)*) => {}; 80 (emit F64Eq $($rest:tt)*) => {}; 81 (emit F32Ne $($rest:tt)*) => {}; 82 (emit F64Ne $($rest:tt)*) => {}; 83 (emit F32Lt $($rest:tt)*) => {}; 84 (emit F64Lt $($rest:tt)*) => {}; 85 (emit F32Gt $($rest:tt)*) => {}; 86 (emit F64Gt $($rest:tt)*) => {}; 87 (emit F32Le $($rest:tt)*) => {}; 88 (emit F64Le $($rest:tt)*) => {}; 89 (emit F32Ge $($rest:tt)*) => {}; 90 (emit F64Ge $($rest:tt)*) => {}; 91 (emit F32ConvertI32S $($rest:tt)*) => {}; 92 (emit F32ConvertI32U $($rest:tt)*) => {}; 93 (emit F32ConvertI64S $($rest:tt)*) => {}; 94 (emit F32ConvertI64U $($rest:tt)*) => {}; 95 (emit F64ConvertI32S $($rest:tt)*) => {}; 96 (emit F64ConvertI32U $($rest:tt)*) => {}; 97 (emit F64ConvertI64S $($rest:tt)*) => {}; 98 (emit F64ConvertI64U $($rest:tt)*) => {}; 99 (emit F32ReinterpretI32 $($rest:tt)*) => {}; 100 (emit F64ReinterpretI64 $($rest:tt)*) => {}; 101 (emit F32DemoteF64 $($rest:tt)*) => {}; 102 (emit F64PromoteF32 $($rest:tt)*) => {}; 103 (emit I32Add $($rest:tt)*) => {}; 104 (emit I64Add $($rest:tt)*) => {}; 105 (emit I32Sub $($rest:tt)*) => {}; 106 (emit I32Mul $($rest:tt)*) => {}; 107 (emit I32DivS $($rest:tt)*) => {}; 108 (emit I32DivU $($rest:tt)*) => {}; 109 (emit I64DivS $($rest:tt)*) => {}; 110 (emit I64DivU $($rest:tt)*) => {}; 111 (emit I64RemU $($rest:tt)*) => {}; 112 (emit I64RemS $($rest:tt)*) => {}; 113 (emit I32RemU $($rest:tt)*) => {}; 114 (emit I32RemS $($rest:tt)*) => {}; 115 (emit I64Mul $($rest:tt)*) => {}; 116 (emit I64Sub $($rest:tt)*) => {}; 117 (emit I32Eq $($rest:tt)*) => {}; 118 (emit I64Eq $($rest:tt)*) => {}; 119 (emit I32Ne $($rest:tt)*) => {}; 120 (emit I64Ne $($rest:tt)*) => {}; 121 (emit I32LtS $($rest:tt)*) => {}; 122 (emit I64LtS $($rest:tt)*) => {}; 123 (emit I32LtU $($rest:tt)*) => {}; 124 (emit I64LtU $($rest:tt)*) => {}; 125 (emit I32LeS $($rest:tt)*) => {}; 126 (emit I64LeS $($rest:tt)*) => {}; 127 (emit I32LeU $($rest:tt)*) => {}; 128 (emit I64LeU $($rest:tt)*) => {}; 129 (emit I32GtS $($rest:tt)*) => {}; 130 (emit I64GtS $($rest:tt)*) => {}; 131 (emit I32GtU $($rest:tt)*) => {}; 132 (emit I64GtU $($rest:tt)*) => {}; 133 (emit I32GeS $($rest:tt)*) => {}; 134 (emit I64GeS $($rest:tt)*) => {}; 135 (emit I32GeU $($rest:tt)*) => {}; 136 (emit I64GeU $($rest:tt)*) => {}; 137 (emit I32Eqz $($rest:tt)*) => {}; 138 (emit I64Eqz $($rest:tt)*) => {}; 139 (emit I32And $($rest:tt)*) => {}; 140 (emit I64And $($rest:tt)*) => {}; 141 (emit I32Or $($rest:tt)*) => {}; 142 (emit I64Or $($rest:tt)*) => {}; 143 (emit I32Xor $($rest:tt)*) => {}; 144 (emit I64Xor $($rest:tt)*) => {}; 145 (emit I32Shl $($rest:tt)*) => {}; 146 (emit I64Shl $($rest:tt)*) => {}; 147 (emit I32ShrS $($rest:tt)*) => {}; 148 (emit I64ShrS $($rest:tt)*) => {}; 149 (emit I32ShrU $($rest:tt)*) => {}; 150 (emit I64ShrU $($rest:tt)*) => {}; 151 (emit I32Rotl $($rest:tt)*) => {}; 152 (emit I64Rotl $($rest:tt)*) => {}; 153 (emit I32Rotr $($rest:tt)*) => {}; 154 (emit I64Rotr $($rest:tt)*) => {}; 155 (emit I32Clz $($rest:tt)*) => {}; 156 (emit I64Clz $($rest:tt)*) => {}; 157 (emit I32Ctz $($rest:tt)*) => {}; 158 (emit I64Ctz $($rest:tt)*) => {}; 159 (emit I32Popcnt $($rest:tt)*) => {}; 160 (emit I64Popcnt $($rest:tt)*) => {}; 161 (emit I32WrapI64 $($rest:tt)*) => {}; 162 (emit I64ExtendI32S $($rest:tt)*) => {}; 163 (emit I64ExtendI32U $($rest:tt)*) => {}; 164 (emit I32Extend8S $($rest:tt)*) => {}; 165 (emit I32Extend16S $($rest:tt)*) => {}; 166 (emit I64Extend8S $($rest:tt)*) => {}; 167 (emit I64Extend16S $($rest:tt)*) => {}; 168 (emit I64Extend32S $($rest:tt)*) => {}; 169 (emit I32TruncF32S $($rest:tt)*) => {}; 170 (emit I32TruncF32U $($rest:tt)*) => {}; 171 (emit I32TruncF64S $($rest:tt)*) => {}; 172 (emit I32TruncF64U $($rest:tt)*) => {}; 173 (emit I64TruncF32S $($rest:tt)*) => {}; 174 (emit I64TruncF32U $($rest:tt)*) => {}; 175 (emit I64TruncF64S $($rest:tt)*) => {}; 176 (emit I64TruncF64U $($rest:tt)*) => {}; 177 (emit I32ReinterpretF32 $($rest:tt)*) => {}; 178 (emit I64ReinterpretF64 $($rest:tt)*) => {}; 179 (emit LocalGet $($rest:tt)*) => {}; 180 (emit LocalSet $($rest:tt)*) => {}; 181 (emit Call $($rest:tt)*) => {}; 182 (emit End $($rest:tt)*) => {}; 183 (emit Nop $($rest:tt)*) => {}; 184 (emit If $($rest:tt)*) => {}; 185 (emit Else $($rest:tt)*) => {}; 186 (emit Block $($rest:tt)*) => {}; 187 (emit Loop $($rest:tt)*) => {}; 188 (emit Br $($rest:tt)*) => {}; 189 (emit BrIf $($rest:tt)*) => {}; 190 (emit Return $($rest:tt)*) => {}; 191 (emit Unreachable $($rest:tt)*) => {}; 192 (emit LocalTee $($rest:tt)*) => {}; 193 (emit GlobalGet $($rest:tt)*) => {}; 194 (emit GlobalSet $($rest:tt)*) => {}; 195 (emit Select $($rest:tt)*) => {}; 196 (emit Drop $($rest:tt)*) => {}; 197 (emit BrTable $($rest:tt)*) => {}; 198 (emit CallIndirect $($rest:tt)*) => {}; 199 (emit TableInit $($rest:tt)*) => {}; 200 (emit TableCopy $($rest:tt)*) => {}; 201 (emit TableGet $($rest:tt)*) => {}; 202 (emit TableSet $($rest:tt)*) => {}; 203 (emit TableGrow $($rest:tt)*) => {}; 204 (emit TableSize $($rest:tt)*) => {}; 205 (emit TableFill $($rest:tt)*) => {}; 206 (emit ElemDrop $($rest:tt)*) => {}; 207 (emit MemoryInit $($rest:tt)*) => {}; 208 (emit MemoryCopy $($rest:tt)*) => {}; 209 (emit DataDrop $($rest:tt)*) => {}; 210 (emit MemoryFill $($rest:tt)*) => {}; 211 (emit MemorySize $($rest:tt)*) => {}; 212 (emit MemoryGrow $($rest:tt)*) => {}; 213 (emit I32Load $($rest:tt)*) => {}; 214 (emit I32Load8S $($rest:tt)*) => {}; 215 (emit I32Load8U $($rest:tt)*) => {}; 216 (emit I32Load16S $($rest:tt)*) => {}; 217 (emit I32Load16U $($rest:tt)*) => {}; 218 (emit I64Load8S $($rest:tt)*) => {}; 219 (emit I64Load8U $($rest:tt)*) => {}; 220 (emit I64Load16S $($rest:tt)*) => {}; 221 (emit I64Load16U $($rest:tt)*) => {}; 222 (emit I64Load32S $($rest:tt)*) => {}; 223 (emit I64Load32U $($rest:tt)*) => {}; 224 (emit I64Load $($rest:tt)*) => {}; 225 (emit I32Store $($rest:tt)*) => {}; 226 (emit I32Store8 $($rest:tt)*) => {}; 227 (emit I32Store16 $($rest:tt)*) => {}; 228 (emit I64Store $($rest:tt)*) => {}; 229 (emit I64Store8 $($rest:tt)*) => {}; 230 (emit I64Store16 $($rest:tt)*) => {}; 231 (emit I64Store32 $($rest:tt)*) => {}; 232 (emit F32Load $($rest:tt)*) => {}; 233 (emit F32Store $($rest:tt)*) => {}; 234 (emit F64Load $($rest:tt)*) => {}; 235 (emit F64Store $($rest:tt)*) => {}; 236 (emit I32TruncSatF32S $($rest:tt)*) => {}; 237 (emit I32TruncSatF32U $($rest:tt)*) => {}; 238 (emit I32TruncSatF64S $($rest:tt)*) => {}; 239 (emit I32TruncSatF64U $($rest:tt)*) => {}; 240 (emit I64TruncSatF32S $($rest:tt)*) => {}; 241 (emit I64TruncSatF32U $($rest:tt)*) => {}; 242 (emit I64TruncSatF64S $($rest:tt)*) => {}; 243 (emit I64TruncSatF64U $($rest:tt)*) => {}; 244 (emit V128Load $($rest:tt)*) => {}; 245 (emit V128Store $($rest:tt)*) => {}; 246 247 (emit $unsupported:tt $($rest:tt)*) => {$($rest)*}; 248 } 249 250 impl<'a, 'translation, 'data, M> VisitOperator<'a> for CodeGen<'a, 'translation, 'data, M> 251 where 252 M: MacroAssembler, 253 { 254 type Output = (); 255 256 fn visit_i32_const(&mut self, val: i32) { 257 self.context.stack.push(Val::i32(val)); 258 } 259 260 fn visit_i64_const(&mut self, val: i64) { 261 self.context.stack.push(Val::i64(val)); 262 } 263 264 fn visit_f32_const(&mut self, val: Ieee32) { 265 self.context.stack.push(Val::f32(val)); 266 } 267 268 fn visit_f64_const(&mut self, val: Ieee64) { 269 self.context.stack.push(Val::f64(val)); 270 } 271 272 fn visit_v128_const(&mut self, val: V128) { 273 self.context.stack.push(Val::v128(val.i128())) 274 } 275 276 fn visit_f32_add(&mut self) { 277 self.context.binop( 278 self.masm, 279 OperandSize::S32, 280 &mut |masm: &mut M, dst, src, size| { 281 masm.float_add(writable!(dst), dst, src, size); 282 TypedReg::f32(dst) 283 }, 284 ); 285 } 286 287 fn visit_f64_add(&mut self) { 288 self.context.binop( 289 self.masm, 290 OperandSize::S64, 291 &mut |masm: &mut M, dst, src, size| { 292 masm.float_add(writable!(dst), dst, src, size); 293 TypedReg::f64(dst) 294 }, 295 ); 296 } 297 298 fn visit_f32_sub(&mut self) { 299 self.context.binop( 300 self.masm, 301 OperandSize::S32, 302 &mut |masm: &mut M, dst, src, size| { 303 masm.float_sub(writable!(dst), dst, src, size); 304 TypedReg::f32(dst) 305 }, 306 ); 307 } 308 309 fn visit_f64_sub(&mut self) { 310 self.context.binop( 311 self.masm, 312 OperandSize::S64, 313 &mut |masm: &mut M, dst, src, size| { 314 masm.float_sub(writable!(dst), dst, src, size); 315 TypedReg::f64(dst) 316 }, 317 ); 318 } 319 320 fn visit_f32_mul(&mut self) { 321 self.context.binop( 322 self.masm, 323 OperandSize::S32, 324 &mut |masm: &mut M, dst, src, size| { 325 masm.float_mul(writable!(dst), dst, src, size); 326 TypedReg::f32(dst) 327 }, 328 ); 329 } 330 331 fn visit_f64_mul(&mut self) { 332 self.context.binop( 333 self.masm, 334 OperandSize::S64, 335 &mut |masm: &mut M, dst, src, size| { 336 masm.float_mul(writable!(dst), dst, src, size); 337 TypedReg::f64(dst) 338 }, 339 ); 340 } 341 342 fn visit_f32_div(&mut self) { 343 self.context.binop( 344 self.masm, 345 OperandSize::S32, 346 &mut |masm: &mut M, dst, src, size| { 347 masm.float_div(writable!(dst), dst, src, size); 348 TypedReg::f32(dst) 349 }, 350 ); 351 } 352 353 fn visit_f64_div(&mut self) { 354 self.context.binop( 355 self.masm, 356 OperandSize::S64, 357 &mut |masm: &mut M, dst, src, size| { 358 masm.float_div(writable!(dst), dst, src, size); 359 TypedReg::f64(dst) 360 }, 361 ); 362 } 363 364 fn visit_f32_min(&mut self) { 365 self.context.binop( 366 self.masm, 367 OperandSize::S32, 368 &mut |masm: &mut M, dst, src, size| { 369 masm.float_min(writable!(dst), dst, src, size); 370 TypedReg::f32(dst) 371 }, 372 ); 373 } 374 375 fn visit_f64_min(&mut self) { 376 self.context.binop( 377 self.masm, 378 OperandSize::S64, 379 &mut |masm: &mut M, dst, src, size| { 380 masm.float_min(writable!(dst), dst, src, size); 381 TypedReg::f64(dst) 382 }, 383 ); 384 } 385 386 fn visit_f32_max(&mut self) { 387 self.context.binop( 388 self.masm, 389 OperandSize::S32, 390 &mut |masm: &mut M, dst, src, size| { 391 masm.float_max(writable!(dst), dst, src, size); 392 TypedReg::f32(dst) 393 }, 394 ); 395 } 396 397 fn visit_f64_max(&mut self) { 398 self.context.binop( 399 self.masm, 400 OperandSize::S64, 401 &mut |masm: &mut M, dst, src, size| { 402 masm.float_max(writable!(dst), dst, src, size); 403 TypedReg::f64(dst) 404 }, 405 ); 406 } 407 408 fn visit_f32_copysign(&mut self) { 409 self.context.binop( 410 self.masm, 411 OperandSize::S32, 412 &mut |masm: &mut M, dst, src, size| { 413 masm.float_copysign(writable!(dst), dst, src, size); 414 TypedReg::f32(dst) 415 }, 416 ); 417 } 418 419 fn visit_f64_copysign(&mut self) { 420 self.context.binop( 421 self.masm, 422 OperandSize::S64, 423 &mut |masm: &mut M, dst, src, size| { 424 masm.float_copysign(writable!(dst), dst, src, size); 425 TypedReg::f64(dst) 426 }, 427 ); 428 } 429 430 fn visit_f32_abs(&mut self) { 431 self.context 432 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 433 masm.float_abs(writable!(reg), size); 434 TypedReg::f32(reg) 435 }); 436 } 437 438 fn visit_f64_abs(&mut self) { 439 self.context 440 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 441 masm.float_abs(writable!(reg), size); 442 TypedReg::f64(reg) 443 }); 444 } 445 446 fn visit_f32_neg(&mut self) { 447 self.context 448 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 449 masm.float_neg(writable!(reg), size); 450 TypedReg::f32(reg) 451 }); 452 } 453 454 fn visit_f64_neg(&mut self) { 455 self.context 456 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 457 masm.float_neg(writable!(reg), size); 458 TypedReg::f64(reg) 459 }); 460 } 461 462 fn visit_f32_floor(&mut self) { 463 self.masm.float_round( 464 RoundingMode::Down, 465 &mut self.env, 466 &mut self.context, 467 OperandSize::S32, 468 |env, cx, masm| { 469 let builtin = env.builtins.floor_f32::<M::ABI>(); 470 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 471 }, 472 ); 473 } 474 475 fn visit_f64_floor(&mut self) { 476 self.masm.float_round( 477 RoundingMode::Down, 478 &mut self.env, 479 &mut self.context, 480 OperandSize::S64, 481 |env, cx, masm| { 482 let builtin = env.builtins.floor_f64::<M::ABI>(); 483 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 484 }, 485 ); 486 } 487 488 fn visit_f32_ceil(&mut self) { 489 self.masm.float_round( 490 RoundingMode::Up, 491 &mut self.env, 492 &mut self.context, 493 OperandSize::S32, 494 |env, cx, masm| { 495 let builtin = env.builtins.ceil_f32::<M::ABI>(); 496 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 497 }, 498 ); 499 } 500 501 fn visit_f64_ceil(&mut self) { 502 self.masm.float_round( 503 RoundingMode::Up, 504 &mut self.env, 505 &mut self.context, 506 OperandSize::S64, 507 |env, cx, masm| { 508 let builtin = env.builtins.ceil_f64::<M::ABI>(); 509 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 510 }, 511 ); 512 } 513 514 fn visit_f32_nearest(&mut self) { 515 self.masm.float_round( 516 RoundingMode::Nearest, 517 &mut self.env, 518 &mut self.context, 519 OperandSize::S32, 520 |env, cx, masm| { 521 let builtin = env.builtins.nearest_f32::<M::ABI>(); 522 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 523 }, 524 ); 525 } 526 527 fn visit_f64_nearest(&mut self) { 528 self.masm.float_round( 529 RoundingMode::Nearest, 530 &mut self.env, 531 &mut self.context, 532 OperandSize::S64, 533 |env, cx, masm| { 534 let builtin = env.builtins.nearest_f64::<M::ABI>(); 535 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 536 }, 537 ); 538 } 539 540 fn visit_f32_trunc(&mut self) { 541 self.masm.float_round( 542 RoundingMode::Zero, 543 &mut self.env, 544 &mut self.context, 545 OperandSize::S32, 546 |env, cx, masm| { 547 let builtin = env.builtins.trunc_f32::<M::ABI>(); 548 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 549 }, 550 ); 551 } 552 553 fn visit_f64_trunc(&mut self) { 554 self.masm.float_round( 555 RoundingMode::Zero, 556 &mut self.env, 557 &mut self.context, 558 OperandSize::S64, 559 |env, cx, masm| { 560 let builtin = env.builtins.trunc_f64::<M::ABI>(); 561 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 562 }, 563 ); 564 } 565 566 fn visit_f32_sqrt(&mut self) { 567 self.context 568 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 569 masm.float_sqrt(writable!(reg), reg, size); 570 TypedReg::f32(reg) 571 }); 572 } 573 574 fn visit_f64_sqrt(&mut self) { 575 self.context 576 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 577 masm.float_sqrt(writable!(reg), reg, size); 578 TypedReg::f64(reg) 579 }); 580 } 581 582 fn visit_f32_eq(&mut self) { 583 self.context.float_cmp_op( 584 self.masm, 585 OperandSize::S32, 586 &mut |masm: &mut M, dst, src1, src2, size| { 587 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Eq, size); 588 }, 589 ); 590 } 591 592 fn visit_f64_eq(&mut self) { 593 self.context.float_cmp_op( 594 self.masm, 595 OperandSize::S64, 596 &mut |masm: &mut M, dst, src1, src2, size| { 597 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Eq, size); 598 }, 599 ); 600 } 601 602 fn visit_f32_ne(&mut self) { 603 self.context.float_cmp_op( 604 self.masm, 605 OperandSize::S32, 606 &mut |masm: &mut M, dst, src1, src2, size| { 607 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ne, size); 608 }, 609 ); 610 } 611 612 fn visit_f64_ne(&mut self) { 613 self.context.float_cmp_op( 614 self.masm, 615 OperandSize::S64, 616 &mut |masm: &mut M, dst, src1, src2, size| { 617 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ne, size); 618 }, 619 ); 620 } 621 622 fn visit_f32_lt(&mut self) { 623 self.context.float_cmp_op( 624 self.masm, 625 OperandSize::S32, 626 &mut |masm: &mut M, dst, src1, src2, size| { 627 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Lt, size); 628 }, 629 ); 630 } 631 632 fn visit_f64_lt(&mut self) { 633 self.context.float_cmp_op( 634 self.masm, 635 OperandSize::S64, 636 &mut |masm: &mut M, dst, src1, src2, size| { 637 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Lt, size); 638 }, 639 ); 640 } 641 642 fn visit_f32_gt(&mut self) { 643 self.context.float_cmp_op( 644 self.masm, 645 OperandSize::S32, 646 &mut |masm: &mut M, dst, src1, src2, size| { 647 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Gt, size); 648 }, 649 ); 650 } 651 652 fn visit_f64_gt(&mut self) { 653 self.context.float_cmp_op( 654 self.masm, 655 OperandSize::S64, 656 &mut |masm: &mut M, dst, src1, src2, size| { 657 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Gt, size); 658 }, 659 ); 660 } 661 662 fn visit_f32_le(&mut self) { 663 self.context.float_cmp_op( 664 self.masm, 665 OperandSize::S32, 666 &mut |masm: &mut M, dst, src1, src2, size| { 667 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Le, size); 668 }, 669 ); 670 } 671 672 fn visit_f64_le(&mut self) { 673 self.context.float_cmp_op( 674 self.masm, 675 OperandSize::S64, 676 &mut |masm: &mut M, dst, src1, src2, size| { 677 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Le, size); 678 }, 679 ); 680 } 681 682 fn visit_f32_ge(&mut self) { 683 self.context.float_cmp_op( 684 self.masm, 685 OperandSize::S32, 686 &mut |masm: &mut M, dst, src1, src2, size| { 687 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ge, size); 688 }, 689 ); 690 } 691 692 fn visit_f64_ge(&mut self) { 693 self.context.float_cmp_op( 694 self.masm, 695 OperandSize::S64, 696 &mut |masm: &mut M, dst, src1, src2, size| { 697 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ge, size); 698 }, 699 ); 700 } 701 702 fn visit_f32_convert_i32_s(&mut self) { 703 self.context 704 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 705 masm.signed_convert(writable!(dst), src, OperandSize::S32, dst_size); 706 }); 707 } 708 709 fn visit_f32_convert_i32_u(&mut self) { 710 self.context.convert_op_with_tmp_reg( 711 self.masm, 712 WasmValType::F32, 713 RegClass::Int, 714 |masm, dst, src, tmp_gpr, dst_size| { 715 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S32, dst_size); 716 }, 717 ); 718 } 719 720 fn visit_f32_convert_i64_s(&mut self) { 721 self.context 722 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 723 masm.signed_convert(writable!(dst), src, OperandSize::S64, dst_size); 724 }); 725 } 726 727 fn visit_f32_convert_i64_u(&mut self) { 728 self.context.convert_op_with_tmp_reg( 729 self.masm, 730 WasmValType::F32, 731 RegClass::Int, 732 |masm, dst, src, tmp_gpr, dst_size| { 733 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S64, dst_size); 734 }, 735 ); 736 } 737 738 fn visit_f64_convert_i32_s(&mut self) { 739 self.context 740 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 741 masm.signed_convert(writable!(dst), src, OperandSize::S32, dst_size); 742 }); 743 } 744 745 fn visit_f64_convert_i32_u(&mut self) { 746 self.context.convert_op_with_tmp_reg( 747 self.masm, 748 WasmValType::F64, 749 RegClass::Int, 750 |masm, dst, src, tmp_gpr, dst_size| { 751 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S32, dst_size); 752 }, 753 ); 754 } 755 756 fn visit_f64_convert_i64_s(&mut self) { 757 self.context 758 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 759 masm.signed_convert(writable!(dst), src, OperandSize::S64, dst_size); 760 }); 761 } 762 763 fn visit_f64_convert_i64_u(&mut self) { 764 self.context.convert_op_with_tmp_reg( 765 self.masm, 766 WasmValType::F64, 767 RegClass::Int, 768 |masm, dst, src, tmp_gpr, dst_size| { 769 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S64, dst_size); 770 }, 771 ); 772 } 773 774 fn visit_f32_reinterpret_i32(&mut self) { 775 self.context 776 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, size| { 777 masm.reinterpret_int_as_float(writable!(dst), src.into(), size); 778 }); 779 } 780 781 fn visit_f64_reinterpret_i64(&mut self) { 782 self.context 783 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, size| { 784 masm.reinterpret_int_as_float(writable!(dst), src.into(), size); 785 }); 786 } 787 788 fn visit_f32_demote_f64(&mut self) { 789 self.context 790 .unop(self.masm, OperandSize::S64, &mut |masm, reg, _size| { 791 masm.demote(writable!(reg), reg); 792 TypedReg::f32(reg) 793 }); 794 } 795 796 fn visit_f64_promote_f32(&mut self) { 797 self.context 798 .unop(self.masm, OperandSize::S32, &mut |masm, reg, _size| { 799 masm.promote(writable!(reg), reg); 800 TypedReg::f64(reg) 801 }); 802 } 803 804 fn visit_i32_add(&mut self) { 805 self.context.i32_binop(self.masm, |masm, dst, src, size| { 806 masm.add(writable!(dst), dst, src, size); 807 TypedReg::i32(dst) 808 }); 809 } 810 811 fn visit_i64_add(&mut self) { 812 self.context.i64_binop(self.masm, |masm, dst, src, size| { 813 masm.add(writable!(dst), dst, src, size); 814 TypedReg::i64(dst) 815 }); 816 } 817 818 fn visit_i32_sub(&mut self) { 819 self.context.i32_binop(self.masm, |masm, dst, src, size| { 820 masm.sub(writable!(dst), dst, src, size); 821 TypedReg::i32(dst) 822 }); 823 } 824 825 fn visit_i64_sub(&mut self) { 826 self.context.i64_binop(self.masm, |masm, dst, src, size| { 827 masm.sub(writable!(dst), dst, src, size); 828 TypedReg::i64(dst) 829 }); 830 } 831 832 fn visit_i32_mul(&mut self) { 833 self.context.i32_binop(self.masm, |masm, dst, src, size| { 834 masm.mul(writable!(dst), dst, src, size); 835 TypedReg::i32(dst) 836 }); 837 } 838 839 fn visit_i64_mul(&mut self) { 840 self.context.i64_binop(self.masm, |masm, dst, src, size| { 841 masm.mul(writable!(dst), dst, src, size); 842 TypedReg::i64(dst) 843 }); 844 } 845 846 fn visit_i32_div_s(&mut self) { 847 use DivKind::*; 848 use OperandSize::*; 849 850 self.masm.div(&mut self.context, Signed, S32); 851 } 852 853 fn visit_i32_div_u(&mut self) { 854 use DivKind::*; 855 use OperandSize::*; 856 857 self.masm.div(&mut self.context, Unsigned, S32); 858 } 859 860 fn visit_i64_div_s(&mut self) { 861 use DivKind::*; 862 use OperandSize::*; 863 864 self.masm.div(&mut self.context, Signed, S64); 865 } 866 867 fn visit_i64_div_u(&mut self) { 868 use DivKind::*; 869 use OperandSize::*; 870 871 self.masm.div(&mut self.context, Unsigned, S64); 872 } 873 874 fn visit_i32_rem_s(&mut self) { 875 use OperandSize::*; 876 use RemKind::*; 877 878 self.masm.rem(&mut self.context, Signed, S32); 879 } 880 881 fn visit_i32_rem_u(&mut self) { 882 use OperandSize::*; 883 use RemKind::*; 884 885 self.masm.rem(&mut self.context, Unsigned, S32); 886 } 887 888 fn visit_i64_rem_s(&mut self) { 889 use OperandSize::*; 890 use RemKind::*; 891 892 self.masm.rem(&mut self.context, Signed, S64); 893 } 894 895 fn visit_i64_rem_u(&mut self) { 896 use OperandSize::*; 897 use RemKind::*; 898 899 self.masm.rem(&mut self.context, Unsigned, S64); 900 } 901 902 fn visit_i32_eq(&mut self) { 903 self.cmp_i32s(IntCmpKind::Eq); 904 } 905 906 fn visit_i64_eq(&mut self) { 907 self.cmp_i64s(IntCmpKind::Eq); 908 } 909 910 fn visit_i32_ne(&mut self) { 911 self.cmp_i32s(IntCmpKind::Ne); 912 } 913 914 fn visit_i64_ne(&mut self) { 915 self.cmp_i64s(IntCmpKind::Ne); 916 } 917 918 fn visit_i32_lt_s(&mut self) { 919 self.cmp_i32s(IntCmpKind::LtS); 920 } 921 922 fn visit_i64_lt_s(&mut self) { 923 self.cmp_i64s(IntCmpKind::LtS); 924 } 925 926 fn visit_i32_lt_u(&mut self) { 927 self.cmp_i32s(IntCmpKind::LtU); 928 } 929 930 fn visit_i64_lt_u(&mut self) { 931 self.cmp_i64s(IntCmpKind::LtU); 932 } 933 934 fn visit_i32_le_s(&mut self) { 935 self.cmp_i32s(IntCmpKind::LeS); 936 } 937 938 fn visit_i64_le_s(&mut self) { 939 self.cmp_i64s(IntCmpKind::LeS); 940 } 941 942 fn visit_i32_le_u(&mut self) { 943 self.cmp_i32s(IntCmpKind::LeU); 944 } 945 946 fn visit_i64_le_u(&mut self) { 947 self.cmp_i64s(IntCmpKind::LeU); 948 } 949 950 fn visit_i32_gt_s(&mut self) { 951 self.cmp_i32s(IntCmpKind::GtS); 952 } 953 954 fn visit_i64_gt_s(&mut self) { 955 self.cmp_i64s(IntCmpKind::GtS); 956 } 957 958 fn visit_i32_gt_u(&mut self) { 959 self.cmp_i32s(IntCmpKind::GtU); 960 } 961 962 fn visit_i64_gt_u(&mut self) { 963 self.cmp_i64s(IntCmpKind::GtU); 964 } 965 966 fn visit_i32_ge_s(&mut self) { 967 self.cmp_i32s(IntCmpKind::GeS); 968 } 969 970 fn visit_i64_ge_s(&mut self) { 971 self.cmp_i64s(IntCmpKind::GeS); 972 } 973 974 fn visit_i32_ge_u(&mut self) { 975 self.cmp_i32s(IntCmpKind::GeU); 976 } 977 978 fn visit_i64_ge_u(&mut self) { 979 self.cmp_i64s(IntCmpKind::GeU); 980 } 981 982 fn visit_i32_eqz(&mut self) { 983 use OperandSize::*; 984 985 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 986 masm.cmp_with_set(writable!(reg.into()), RegImm::i32(0), IntCmpKind::Eq, size); 987 TypedReg::i32(reg) 988 }); 989 } 990 991 fn visit_i64_eqz(&mut self) { 992 use OperandSize::*; 993 994 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 995 masm.cmp_with_set(writable!(reg.into()), RegImm::i64(0), IntCmpKind::Eq, size); 996 TypedReg::i32(reg) // Return value for `i64.eqz` is an `i32`. 997 }); 998 } 999 1000 fn visit_i32_clz(&mut self) { 1001 use OperandSize::*; 1002 1003 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 1004 masm.clz(writable!(reg), reg, size); 1005 TypedReg::i32(reg) 1006 }); 1007 } 1008 1009 fn visit_i64_clz(&mut self) { 1010 use OperandSize::*; 1011 1012 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 1013 masm.clz(writable!(reg), reg, size); 1014 TypedReg::i64(reg) 1015 }); 1016 } 1017 1018 fn visit_i32_ctz(&mut self) { 1019 use OperandSize::*; 1020 1021 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 1022 masm.ctz(writable!(reg), reg, size); 1023 TypedReg::i32(reg) 1024 }); 1025 } 1026 1027 fn visit_i64_ctz(&mut self) { 1028 use OperandSize::*; 1029 1030 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 1031 masm.ctz(writable!(reg), reg, size); 1032 TypedReg::i64(reg) 1033 }); 1034 } 1035 1036 fn visit_i32_and(&mut self) { 1037 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1038 masm.and(writable!(dst), dst, src, size); 1039 TypedReg::i32(dst) 1040 }); 1041 } 1042 1043 fn visit_i64_and(&mut self) { 1044 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1045 masm.and(writable!(dst), dst, src, size); 1046 TypedReg::i64(dst) 1047 }); 1048 } 1049 1050 fn visit_i32_or(&mut self) { 1051 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1052 masm.or(writable!(dst), dst, src, size); 1053 TypedReg::i32(dst) 1054 }); 1055 } 1056 1057 fn visit_i64_or(&mut self) { 1058 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1059 masm.or(writable!(dst), dst, src, size); 1060 TypedReg::i64(dst) 1061 }); 1062 } 1063 1064 fn visit_i32_xor(&mut self) { 1065 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1066 masm.xor(writable!(dst), dst, src, size); 1067 TypedReg::i32(dst) 1068 }); 1069 } 1070 1071 fn visit_i64_xor(&mut self) { 1072 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1073 masm.xor(writable!(dst), dst, src, size); 1074 TypedReg::i64(dst) 1075 }); 1076 } 1077 1078 fn visit_i32_shl(&mut self) { 1079 use ShiftKind::*; 1080 1081 self.context.i32_shift(self.masm, Shl); 1082 } 1083 1084 fn visit_i64_shl(&mut self) { 1085 use ShiftKind::*; 1086 1087 self.context.i64_shift(self.masm, Shl); 1088 } 1089 1090 fn visit_i32_shr_s(&mut self) { 1091 use ShiftKind::*; 1092 1093 self.context.i32_shift(self.masm, ShrS); 1094 } 1095 1096 fn visit_i64_shr_s(&mut self) { 1097 use ShiftKind::*; 1098 1099 self.context.i64_shift(self.masm, ShrS); 1100 } 1101 1102 fn visit_i32_shr_u(&mut self) { 1103 use ShiftKind::*; 1104 1105 self.context.i32_shift(self.masm, ShrU); 1106 } 1107 1108 fn visit_i64_shr_u(&mut self) { 1109 use ShiftKind::*; 1110 1111 self.context.i64_shift(self.masm, ShrU); 1112 } 1113 1114 fn visit_i32_rotl(&mut self) { 1115 use ShiftKind::*; 1116 1117 self.context.i32_shift(self.masm, Rotl); 1118 } 1119 1120 fn visit_i64_rotl(&mut self) { 1121 use ShiftKind::*; 1122 1123 self.context.i64_shift(self.masm, Rotl); 1124 } 1125 1126 fn visit_i32_rotr(&mut self) { 1127 use ShiftKind::*; 1128 1129 self.context.i32_shift(self.masm, Rotr); 1130 } 1131 1132 fn visit_i64_rotr(&mut self) { 1133 use ShiftKind::*; 1134 1135 self.context.i64_shift(self.masm, Rotr); 1136 } 1137 1138 fn visit_end(&mut self) { 1139 if !self.context.reachable { 1140 self.handle_unreachable_end(); 1141 } else { 1142 let mut control = self.control_frames.pop().unwrap(); 1143 control.emit_end(self.masm, &mut self.context); 1144 } 1145 } 1146 1147 fn visit_i32_popcnt(&mut self) { 1148 use OperandSize::*; 1149 self.masm.popcnt(&mut self.context, S32); 1150 } 1151 1152 fn visit_i64_popcnt(&mut self) { 1153 use OperandSize::*; 1154 1155 self.masm.popcnt(&mut self.context, S64); 1156 } 1157 1158 fn visit_i32_wrap_i64(&mut self) { 1159 use OperandSize::*; 1160 1161 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1162 masm.wrap(writable!(reg), reg); 1163 TypedReg::i32(reg) 1164 }); 1165 } 1166 1167 fn visit_i64_extend_i32_s(&mut self) { 1168 use OperandSize::*; 1169 1170 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1171 masm.extend(writable!(reg), reg, ExtendKind::I64ExtendI32S); 1172 TypedReg::i64(reg) 1173 }); 1174 } 1175 1176 fn visit_i64_extend_i32_u(&mut self) { 1177 use OperandSize::*; 1178 1179 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1180 masm.extend(writable!(reg), reg, ExtendKind::I64ExtendI32U); 1181 TypedReg::i64(reg) 1182 }); 1183 } 1184 1185 fn visit_i32_extend8_s(&mut self) { 1186 use OperandSize::*; 1187 1188 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1189 masm.extend(writable!(reg), reg, ExtendKind::I32Extend8S); 1190 TypedReg::i32(reg) 1191 }); 1192 } 1193 1194 fn visit_i32_extend16_s(&mut self) { 1195 use OperandSize::*; 1196 1197 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1198 masm.extend(writable!(reg), reg, ExtendKind::I32Extend16S); 1199 TypedReg::i32(reg) 1200 }); 1201 } 1202 1203 fn visit_i64_extend8_s(&mut self) { 1204 use OperandSize::*; 1205 1206 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1207 masm.extend(writable!(reg), reg, ExtendKind::I64Extend8S); 1208 TypedReg::i64(reg) 1209 }); 1210 } 1211 1212 fn visit_i64_extend16_s(&mut self) { 1213 use OperandSize::*; 1214 1215 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1216 masm.extend(writable!(reg), reg, ExtendKind::I64Extend16S); 1217 TypedReg::i64(reg) 1218 }); 1219 } 1220 1221 fn visit_i64_extend32_s(&mut self) { 1222 use OperandSize::*; 1223 1224 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1225 masm.extend(writable!(reg), reg, ExtendKind::I64Extend32S); 1226 TypedReg::i64(reg) 1227 }); 1228 } 1229 1230 fn visit_i32_trunc_f32_s(&mut self) { 1231 use OperandSize::*; 1232 1233 self.context 1234 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1235 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Unchecked); 1236 }); 1237 } 1238 1239 fn visit_i32_trunc_f32_u(&mut self) { 1240 use OperandSize::*; 1241 1242 self.context.convert_op_with_tmp_reg( 1243 self.masm, 1244 WasmValType::I32, 1245 RegClass::Float, 1246 |masm, dst, src, tmp_fpr, dst_size| { 1247 masm.unsigned_truncate( 1248 writable!(dst), 1249 src, 1250 tmp_fpr, 1251 S32, 1252 dst_size, 1253 TruncKind::Unchecked, 1254 ); 1255 }, 1256 ); 1257 } 1258 1259 fn visit_i32_trunc_f64_s(&mut self) { 1260 use OperandSize::*; 1261 1262 self.context 1263 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1264 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Unchecked); 1265 }); 1266 } 1267 1268 fn visit_i32_trunc_f64_u(&mut self) { 1269 use OperandSize::*; 1270 1271 self.context.convert_op_with_tmp_reg( 1272 self.masm, 1273 WasmValType::I32, 1274 RegClass::Float, 1275 |masm, dst, src, tmp_fpr, dst_size| { 1276 masm.unsigned_truncate( 1277 writable!(dst), 1278 src, 1279 tmp_fpr, 1280 S64, 1281 dst_size, 1282 TruncKind::Unchecked, 1283 ); 1284 }, 1285 ); 1286 } 1287 1288 fn visit_i64_trunc_f32_s(&mut self) { 1289 use OperandSize::*; 1290 1291 self.context 1292 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1293 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Unchecked); 1294 }); 1295 } 1296 1297 fn visit_i64_trunc_f32_u(&mut self) { 1298 use OperandSize::*; 1299 1300 self.context.convert_op_with_tmp_reg( 1301 self.masm, 1302 WasmValType::I64, 1303 RegClass::Float, 1304 |masm, dst, src, tmp_fpr, dst_size| { 1305 masm.unsigned_truncate( 1306 writable!(dst), 1307 src, 1308 tmp_fpr, 1309 S32, 1310 dst_size, 1311 TruncKind::Unchecked, 1312 ); 1313 }, 1314 ); 1315 } 1316 1317 fn visit_i64_trunc_f64_s(&mut self) { 1318 use OperandSize::*; 1319 1320 self.context 1321 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1322 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Unchecked); 1323 }); 1324 } 1325 1326 fn visit_i64_trunc_f64_u(&mut self) { 1327 use OperandSize::*; 1328 1329 self.context.convert_op_with_tmp_reg( 1330 self.masm, 1331 WasmValType::I64, 1332 RegClass::Float, 1333 |masm, dst, src, tmp_fpr, dst_size| { 1334 masm.unsigned_truncate( 1335 writable!(dst), 1336 src, 1337 tmp_fpr, 1338 S64, 1339 dst_size, 1340 TruncKind::Unchecked, 1341 ); 1342 }, 1343 ); 1344 } 1345 1346 fn visit_i32_reinterpret_f32(&mut self) { 1347 self.context 1348 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, size| { 1349 masm.reinterpret_float_as_int(writable!(dst), src.into(), size); 1350 }); 1351 } 1352 1353 fn visit_i64_reinterpret_f64(&mut self) { 1354 self.context 1355 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, size| { 1356 masm.reinterpret_float_as_int(writable!(dst), src.into(), size); 1357 }); 1358 } 1359 1360 fn visit_local_get(&mut self, index: u32) { 1361 use WasmValType::*; 1362 let context = &mut self.context; 1363 let slot = context.frame.get_wasm_local(index); 1364 match slot.ty { 1365 I32 | I64 | F32 | F64 | V128 => context.stack.push(Val::local(index, slot.ty)), 1366 Ref(rt) => match rt.heap_type { 1367 WasmHeapType::Func => context.stack.push(Val::local(index, slot.ty)), 1368 WasmHeapType::Extern => { 1369 self.found_unsupported_instruction = 1370 Some("unsupported local.get of externref local"); 1371 } 1372 ht => unimplemented!("Support for WasmHeapType: {ht}"), 1373 }, 1374 } 1375 } 1376 1377 fn visit_local_set(&mut self, index: u32) { 1378 let src = self.emit_set_local(index); 1379 self.context.free_reg(src); 1380 } 1381 1382 fn visit_call(&mut self, index: u32) { 1383 let callee = self.env.callee_from_index(FuncIndex::from_u32(index)); 1384 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee) 1385 } 1386 1387 fn visit_call_indirect(&mut self, type_index: u32, table_index: u32) { 1388 // Spill now because `emit_lazy_init_funcref` and the `FnCall::emit` 1389 // invocations will both trigger spills since they both call functions. 1390 // However, the machine instructions for the spill emitted by 1391 // `emit_lazy_funcref` will be jumped over if the funcref was previously 1392 // initialized which may result in the machine stack becoming 1393 // unbalanced. 1394 self.context.spill(self.masm); 1395 1396 let type_index = TypeIndex::from_u32(type_index); 1397 let table_index = TableIndex::from_u32(table_index); 1398 1399 self.emit_lazy_init_funcref(table_index); 1400 1401 // Perform the indirect call. 1402 // This code assumes that [`Self::emit_lazy_init_funcref`] will 1403 // push the funcref to the value stack. 1404 match self.env.translation.module.table_plans[table_index].style { 1405 TableStyle::CallerChecksSignature { lazy_init: true } => { 1406 let funcref_ptr = self.context.stack.peek().map(|v| v.unwrap_reg()).unwrap(); 1407 self.masm 1408 .trapz(funcref_ptr.into(), TRAP_INDIRECT_CALL_TO_NULL); 1409 self.emit_typecheck_funcref(funcref_ptr.into(), type_index); 1410 } 1411 _ => unimplemented!("Support for eager table init"), 1412 } 1413 1414 let callee = self.env.funcref(type_index); 1415 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee) 1416 } 1417 1418 fn visit_table_init(&mut self, elem: u32, table: u32) { 1419 debug_assert!(self.context.stack.len() >= 3); 1420 let at = self.context.stack.len() - 3; 1421 1422 self.context 1423 .stack 1424 .insert_many(at, &[table.try_into().unwrap(), elem.try_into().unwrap()]); 1425 1426 let builtin = self.env.builtins.table_init::<M::ABI, M::Ptr>(); 1427 FnCall::emit::<M>( 1428 &mut self.env, 1429 self.masm, 1430 &mut self.context, 1431 Callee::Builtin(builtin.clone()), 1432 ) 1433 } 1434 1435 fn visit_table_copy(&mut self, dst: u32, src: u32) { 1436 debug_assert!(self.context.stack.len() >= 3); 1437 let at = self.context.stack.len() - 3; 1438 self.context 1439 .stack 1440 .insert_many(at, &[dst.try_into().unwrap(), src.try_into().unwrap()]); 1441 1442 let builtin = self.env.builtins.table_copy::<M::ABI, M::Ptr>(); 1443 FnCall::emit::<M>( 1444 &mut self.env, 1445 self.masm, 1446 &mut self.context, 1447 Callee::Builtin(builtin), 1448 ) 1449 } 1450 1451 fn visit_table_get(&mut self, table: u32) { 1452 let table_index = TableIndex::from_u32(table); 1453 let plan = self.env.table_plan(table_index); 1454 let heap_type = plan.table.ref_type.heap_type; 1455 let style = &plan.style; 1456 1457 match heap_type { 1458 WasmHeapType::Func => match style { 1459 TableStyle::CallerChecksSignature { lazy_init: true } => { 1460 self.emit_lazy_init_funcref(table_index) 1461 } 1462 _ => unimplemented!("Support for eager table init"), 1463 }, 1464 WasmHeapType::Extern => { 1465 self.found_unsupported_instruction = 1466 Some("unsupported table.get of externref table"); 1467 } 1468 t => { 1469 unimplemented!("Support for WasmHeapType: {t}") 1470 } 1471 } 1472 } 1473 1474 fn visit_table_grow(&mut self, table: u32) { 1475 let table_index = TableIndex::from_u32(table); 1476 let table_plan = self.env.table_plan(table_index); 1477 let builtin = match table_plan.table.ref_type.heap_type { 1478 WasmHeapType::Func => self.env.builtins.table_grow_func_ref::<M::ABI, M::Ptr>(), 1479 ty => unimplemented!("Support for HeapType: {ty}"), 1480 }; 1481 1482 let len = self.context.stack.len(); 1483 // table.grow` requires at least 2 elements on the value stack. 1484 debug_assert!(len >= 2); 1485 let at = len - 2; 1486 1487 // The table_grow builtin expects the parameters in a different 1488 // order. 1489 // The value stack at this point should contain: 1490 // [ init_value | delta ] (stack top) 1491 // but the builtin function expects the init value as the last 1492 // argument. 1493 self.context.stack.inner_mut().swap(len - 1, len - 2); 1494 self.context 1495 .stack 1496 .insert_many(at, &[table.try_into().unwrap()]); 1497 1498 FnCall::emit::<M>( 1499 &mut self.env, 1500 self.masm, 1501 &mut self.context, 1502 Callee::Builtin(builtin.clone()), 1503 ) 1504 } 1505 1506 fn visit_table_size(&mut self, table: u32) { 1507 let table_index = TableIndex::from_u32(table); 1508 let table_data = self.env.resolve_table_data(table_index); 1509 self.emit_compute_table_size(&table_data); 1510 } 1511 1512 fn visit_table_fill(&mut self, table: u32) { 1513 let table_index = TableIndex::from_u32(table); 1514 let table_plan = self.env.table_plan(table_index); 1515 let builtin = match table_plan.table.ref_type.heap_type { 1516 WasmHeapType::Func => self.env.builtins.table_fill_func_ref::<M::ABI, M::Ptr>(), 1517 ty => unimplemented!("Support for heap type: {ty}"), 1518 }; 1519 1520 let len = self.context.stack.len(); 1521 debug_assert!(len >= 3); 1522 let at = len - 3; 1523 self.context 1524 .stack 1525 .insert_many(at, &[table.try_into().unwrap()]); 1526 FnCall::emit::<M>( 1527 &mut self.env, 1528 self.masm, 1529 &mut self.context, 1530 Callee::Builtin(builtin.clone()), 1531 ) 1532 } 1533 1534 fn visit_table_set(&mut self, table: u32) { 1535 let ptr_type = self.env.ptr_type(); 1536 let table_index = TableIndex::from_u32(table); 1537 let table_data = self.env.resolve_table_data(table_index); 1538 let plan = self.env.table_plan(table_index); 1539 match plan.table.ref_type.heap_type { 1540 WasmHeapType::Func => match plan.style { 1541 TableStyle::CallerChecksSignature { lazy_init: true } => { 1542 let value = self.context.pop_to_reg(self.masm, None); 1543 let index = self.context.pop_to_reg(self.masm, None); 1544 let base = self.context.any_gpr(self.masm); 1545 let elem_addr = 1546 self.emit_compute_table_elem_addr(index.into(), base, &table_data); 1547 // Set the initialized bit. 1548 self.masm.or( 1549 writable!(value.into()), 1550 value.into(), 1551 RegImm::i64(FUNCREF_INIT_BIT as i64), 1552 ptr_type.into(), 1553 ); 1554 1555 self.masm.store_ptr(value.into(), elem_addr); 1556 1557 self.context.free_reg(value); 1558 self.context.free_reg(index); 1559 self.context.free_reg(base); 1560 } 1561 _ => unimplemented!("Support for eager table init"), 1562 }, 1563 ty => unimplemented!("Support for WasmHeapType: {ty}"), 1564 }; 1565 } 1566 1567 fn visit_elem_drop(&mut self, index: u32) { 1568 let elem_drop = self.env.builtins.elem_drop::<M::ABI, M::Ptr>(); 1569 self.context.stack.extend([index.try_into().unwrap()]); 1570 FnCall::emit::<M>( 1571 &mut self.env, 1572 self.masm, 1573 &mut self.context, 1574 Callee::Builtin(elem_drop), 1575 ) 1576 } 1577 1578 fn visit_memory_init(&mut self, data_index: u32, mem: u32) { 1579 debug_assert!(self.context.stack.len() >= 3); 1580 let at = self.context.stack.len() - 3; 1581 self.context.stack.insert_many( 1582 at, 1583 &[mem.try_into().unwrap(), data_index.try_into().unwrap()], 1584 ); 1585 let builtin = self.env.builtins.memory_init::<M::ABI, M::Ptr>(); 1586 FnCall::emit::<M>( 1587 &mut self.env, 1588 self.masm, 1589 &mut self.context, 1590 Callee::Builtin(builtin), 1591 ) 1592 } 1593 1594 fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) { 1595 // At this point, the stack is expected to contain: 1596 // [ dst_offset, src_offset, len ] 1597 // The following code inserts the missing params, so that stack contains: 1598 // [ vmctx, dst_mem, dst_offset, src_mem, src_offset, len ] 1599 // Which is the order expected by the builtin function. 1600 debug_assert!(self.context.stack.len() >= 3); 1601 let at = self.context.stack.len() - 2; 1602 self.context 1603 .stack 1604 .insert_many(at, &[src_mem.try_into().unwrap()]); 1605 1606 // One element was inserted above, so instead of 3, we use 4. 1607 let at = self.context.stack.len() - 4; 1608 self.context 1609 .stack 1610 .insert_many(at, &[dst_mem.try_into().unwrap()]); 1611 1612 let builtin = self.env.builtins.memory_copy::<M::ABI, M::Ptr>(); 1613 1614 FnCall::emit::<M>( 1615 &mut self.env, 1616 self.masm, 1617 &mut self.context, 1618 Callee::Builtin(builtin), 1619 ) 1620 } 1621 1622 fn visit_memory_fill(&mut self, mem: u32) { 1623 debug_assert!(self.context.stack.len() >= 3); 1624 let at = self.context.stack.len() - 3; 1625 1626 self.context 1627 .stack 1628 .insert_many(at, &[mem.try_into().unwrap()]); 1629 1630 let builtin = self.env.builtins.memory_fill::<M::ABI, M::Ptr>(); 1631 FnCall::emit::<M>( 1632 &mut self.env, 1633 self.masm, 1634 &mut self.context, 1635 Callee::Builtin(builtin), 1636 ) 1637 } 1638 1639 fn visit_memory_size(&mut self, mem: u32) { 1640 let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem)); 1641 self.emit_compute_memory_size(&heap); 1642 } 1643 1644 fn visit_memory_grow(&mut self, mem: u32) { 1645 debug_assert!(self.context.stack.len() >= 1); 1646 // The stack at this point contains: [ delta ] 1647 // The desired state is 1648 // [ vmctx, delta, index ] 1649 self.context.stack.extend([mem.try_into().unwrap()]); 1650 1651 let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem)); 1652 let builtin = self.env.builtins.memory32_grow::<M::ABI, M::Ptr>(); 1653 FnCall::emit::<M>( 1654 &mut self.env, 1655 self.masm, 1656 &mut self.context, 1657 Callee::Builtin(builtin), 1658 ); 1659 1660 // The memory32_grow builtin returns a pointer type, therefore we must 1661 // ensure that the return type is representative of the address space of 1662 // the heap type. 1663 match (self.env.ptr_type(), heap.ty) { 1664 (WasmValType::I64, WasmValType::I64) => {} 1665 // When the heap type is smaller than the pointer type, we adjust 1666 // the result of the memory32_grow builtin. 1667 (WasmValType::I64, WasmValType::I32) => { 1668 let top: Reg = self.context.pop_to_reg(self.masm, None).into(); 1669 self.masm.wrap(writable!(top.into()), top.into()); 1670 self.context.stack.push(TypedReg::i32(top).into()); 1671 } 1672 _ => unimplemented!("Support for 32-bit platforms"), 1673 } 1674 } 1675 1676 fn visit_data_drop(&mut self, data_index: u32) { 1677 self.context.stack.extend([data_index.try_into().unwrap()]); 1678 1679 let builtin = self.env.builtins.data_drop::<M::ABI, M::Ptr>(); 1680 FnCall::emit::<M>( 1681 &mut self.env, 1682 self.masm, 1683 &mut self.context, 1684 Callee::Builtin(builtin), 1685 ) 1686 } 1687 1688 fn visit_nop(&mut self) {} 1689 1690 fn visit_if(&mut self, blockty: BlockType) { 1691 self.control_frames.push(ControlStackFrame::r#if( 1692 self.env.resolve_block_sig(blockty), 1693 self.masm, 1694 &mut self.context, 1695 )); 1696 } 1697 1698 fn visit_else(&mut self) { 1699 if !self.context.reachable { 1700 self.handle_unreachable_else(); 1701 } else { 1702 let control = self 1703 .control_frames 1704 .last_mut() 1705 .unwrap_or_else(|| panic!("Expected active control stack frame for else")); 1706 control.emit_else(self.masm, &mut self.context); 1707 } 1708 } 1709 1710 fn visit_block(&mut self, blockty: BlockType) { 1711 self.control_frames.push(ControlStackFrame::block( 1712 self.env.resolve_block_sig(blockty), 1713 self.masm, 1714 &mut self.context, 1715 )); 1716 } 1717 1718 fn visit_loop(&mut self, blockty: BlockType) { 1719 self.control_frames.push(ControlStackFrame::r#loop( 1720 self.env.resolve_block_sig(blockty), 1721 self.masm, 1722 &mut self.context, 1723 )); 1724 } 1725 1726 fn visit_br(&mut self, depth: u32) { 1727 let index = control_index(depth, self.control_frames.len()); 1728 let frame = &mut self.control_frames[index]; 1729 self.context 1730 .unconditional_jump(frame, self.masm, |masm, cx, frame| { 1731 frame 1732 .pop_abi_results::<M, _>(cx, masm, |results, _, _| results.ret_area().copied()); 1733 }); 1734 } 1735 1736 fn visit_br_if(&mut self, depth: u32) { 1737 let index = control_index(depth, self.control_frames.len()); 1738 let frame = &mut self.control_frames[index]; 1739 frame.set_as_target(); 1740 1741 let top = { 1742 let top = self.context.without::<TypedReg, M, _>( 1743 frame.results::<M>().regs(), 1744 self.masm, 1745 |ctx, masm| ctx.pop_to_reg(masm, None), 1746 ); 1747 // Explicitly save any live registers and locals before setting up 1748 // the branch state. 1749 // In some cases, calculating the `top` value above, will result in 1750 // a spill, thus the following one will result in a no-op. 1751 self.context.spill(self.masm); 1752 frame.top_abi_results::<M, _>( 1753 &mut self.context, 1754 self.masm, 1755 |results, context, masm| { 1756 // In the case of `br_if` there's a possibility that we'll 1757 // exit early from the block or fallthrough, for 1758 // a fallthrough, we cannot rely on the pre-computed return area; 1759 // it must be recalculated so that any values that are 1760 // generated are correctly placed near the current stack 1761 // pointer. 1762 results.on_stack().then(|| { 1763 let stack_consumed = context.stack.sizeof(results.stack_operands_len()); 1764 let base = masm.sp_offset().as_u32() - stack_consumed; 1765 let offs = base + results.size(); 1766 RetArea::sp(SPOffset::from_u32(offs)) 1767 }) 1768 }, 1769 ); 1770 top 1771 }; 1772 1773 // Emit instructions to balance the machine stack if the frame has 1774 // a different offset. 1775 let current_sp_offset = self.masm.sp_offset(); 1776 let results_size = frame.results::<M>().size(); 1777 let state = frame.stack_state(); 1778 let (label, cmp, needs_cleanup) = if current_sp_offset > state.target_offset { 1779 (self.masm.get_label(), IntCmpKind::Eq, true) 1780 } else { 1781 (*frame.label(), IntCmpKind::Ne, false) 1782 }; 1783 1784 self.masm 1785 .branch(cmp, top.reg.into(), top.reg.into(), label, OperandSize::S32); 1786 self.context.free_reg(top); 1787 1788 if needs_cleanup { 1789 // Emit instructions to balance the stack and jump if not falling 1790 // through. 1791 self.masm.memmove( 1792 current_sp_offset, 1793 state.target_offset, 1794 results_size, 1795 MemMoveDirection::LowToHigh, 1796 ); 1797 self.masm.ensure_sp_for_jump(state.target_offset); 1798 self.masm.jmp(*frame.label()); 1799 1800 // Restore sp_offset to what it was for falling through and emit 1801 // fallthrough label. 1802 self.masm.reset_stack_pointer(current_sp_offset); 1803 self.masm.bind(label); 1804 } 1805 } 1806 1807 fn visit_br_table(&mut self, targets: BrTable<'a>) { 1808 // +1 to account for the default target. 1809 let len = targets.len() + 1; 1810 // SmallVec<[_; 5]> to match the binary emission layer (e.g 1811 // see `JmpTableSeq'), but here we use 5 instead since we 1812 // bundle the default target as the last element in the array. 1813 let labels: SmallVec<[_; 5]> = (0..len).map(|_| self.masm.get_label()).collect(); 1814 1815 let default_index = control_index(targets.default(), self.control_frames.len()); 1816 let default_frame = &mut self.control_frames[default_index]; 1817 let default_result = default_frame.results::<M>(); 1818 1819 let (index, tmp) = { 1820 let index_and_tmp = self.context.without::<(TypedReg, _), M, _>( 1821 default_result.regs(), 1822 self.masm, 1823 |cx, masm| (cx.pop_to_reg(masm, None), cx.any_gpr(masm)), 1824 ); 1825 1826 // Materialize any constants or locals into their result representation, 1827 // so that when reachability is restored, they are correctly located. 1828 default_frame.top_abi_results::<M, _>(&mut self.context, self.masm, |results, _, _| { 1829 results.ret_area().copied() 1830 }); 1831 index_and_tmp 1832 }; 1833 1834 self.masm.jmp_table(&labels, index.into(), tmp); 1835 // Save the original stack pointer offset; we will reset the stack 1836 // pointer to this offset after jumping to each of the targets. Each 1837 // jump might adjust the stack according to the base offset of the 1838 // target. 1839 let current_sp = self.masm.sp_offset(); 1840 1841 for (t, l) in targets 1842 .targets() 1843 .into_iter() 1844 .chain(std::iter::once(Ok(targets.default()))) 1845 .zip(labels.iter()) 1846 { 1847 let control_index = control_index(t.unwrap(), self.control_frames.len()); 1848 let frame = &mut self.control_frames[control_index]; 1849 // Reset the stack pointer to its original offset. This is needed 1850 // because each jump will potentially adjust the stack pointer 1851 // according to the base offset of the target. 1852 self.masm.reset_stack_pointer(current_sp); 1853 1854 // NB: We don't perform any result handling as it was 1855 // already taken care of above before jumping to the 1856 // jump table. 1857 self.masm.bind(*l); 1858 // Ensure that the stack pointer is correctly positioned before 1859 // jumping to the jump table code. 1860 let state = frame.stack_state(); 1861 self.masm.ensure_sp_for_jump(state.target_offset); 1862 self.masm.jmp(*frame.label()); 1863 frame.set_as_target(); 1864 } 1865 // Finally reset the stack pointer to the original location. 1866 // The reachability analysis, will ensure it's correctly located 1867 // once reachability is restored. 1868 self.masm.reset_stack_pointer(current_sp); 1869 self.context.reachable = false; 1870 self.context.free_reg(index.reg); 1871 self.context.free_reg(tmp); 1872 } 1873 1874 fn visit_return(&mut self) { 1875 // Grab the outermost frame, which is the function's body 1876 // frame. We don't rely on [`codegen::control_index`] since 1877 // this frame is implicit and we know that it should exist at 1878 // index 0. 1879 let outermost = &mut self.control_frames[0]; 1880 self.context 1881 .unconditional_jump(outermost, self.masm, |masm, cx, frame| { 1882 frame 1883 .pop_abi_results::<M, _>(cx, masm, |results, _, _| results.ret_area().copied()); 1884 }); 1885 } 1886 1887 fn visit_unreachable(&mut self) { 1888 self.masm.unreachable(); 1889 self.context.reachable = false; 1890 // Set the implicit outermost frame as target to perform the necessary 1891 // stack clean up. 1892 let outermost = &mut self.control_frames[0]; 1893 outermost.set_as_target(); 1894 } 1895 1896 fn visit_local_tee(&mut self, index: u32) { 1897 let typed_reg = self.emit_set_local(index); 1898 self.context.stack.push(typed_reg.into()); 1899 } 1900 1901 fn visit_global_get(&mut self, global_index: u32) { 1902 let index = GlobalIndex::from_u32(global_index); 1903 let (ty, addr) = self.emit_get_global_addr(index); 1904 let dst = self.context.reg_for_type(ty, self.masm); 1905 self.masm.load(addr, writable!(dst), ty.into()); 1906 self.context.stack.push(Val::reg(dst, ty)); 1907 } 1908 1909 fn visit_global_set(&mut self, global_index: u32) { 1910 let index = GlobalIndex::from_u32(global_index); 1911 let (ty, addr) = self.emit_get_global_addr(index); 1912 1913 let typed_reg = self.context.pop_to_reg(self.masm, None); 1914 self.context.free_reg(typed_reg.reg); 1915 self.masm.store(typed_reg.reg.into(), addr, ty.into()); 1916 } 1917 1918 fn visit_drop(&mut self) { 1919 self.context.drop_last(1, |regalloc, val| match val { 1920 Val::Reg(tr) => regalloc.free(tr.reg.into()), 1921 Val::Memory(m) => self.masm.free_stack(m.slot.size), 1922 _ => {} 1923 }); 1924 } 1925 1926 fn visit_select(&mut self) { 1927 let cond = self.context.pop_to_reg(self.masm, None); 1928 let val2 = self.context.pop_to_reg(self.masm, None); 1929 let val1 = self.context.pop_to_reg(self.masm, None); 1930 self.masm 1931 .cmp(cond.reg.into(), RegImm::i32(0), OperandSize::S32); 1932 // Conditionally move val1 to val2 if the comparison is 1933 // not zero. 1934 self.masm.cmov( 1935 writable!(val2.into()), 1936 val1.into(), 1937 IntCmpKind::Ne, 1938 val1.ty.into(), 1939 ); 1940 self.context.stack.push(val2.into()); 1941 self.context.free_reg(val1.reg); 1942 self.context.free_reg(cond); 1943 } 1944 1945 fn visit_i32_load(&mut self, memarg: MemArg) { 1946 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S32, None); 1947 } 1948 1949 fn visit_i32_load8_s(&mut self, memarg: MemArg) { 1950 self.emit_wasm_load( 1951 &memarg, 1952 WasmValType::I32, 1953 OperandSize::S8, 1954 Some(ExtendKind::I32Extend8S), 1955 ); 1956 } 1957 1958 fn visit_i32_load8_u(&mut self, memarg: MemArg) { 1959 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S8, None); 1960 } 1961 1962 fn visit_i32_load16_s(&mut self, memarg: MemArg) { 1963 self.emit_wasm_load( 1964 &memarg, 1965 WasmValType::I32, 1966 OperandSize::S16, 1967 Some(ExtendKind::I32Extend16S), 1968 ) 1969 } 1970 1971 fn visit_i32_load16_u(&mut self, memarg: MemArg) { 1972 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S16, None) 1973 } 1974 1975 fn visit_i32_store(&mut self, memarg: MemArg) { 1976 self.emit_wasm_store(&memarg, OperandSize::S32); 1977 } 1978 1979 fn visit_i32_store8(&mut self, memarg: MemArg) { 1980 self.emit_wasm_store(&memarg, OperandSize::S8) 1981 } 1982 1983 fn visit_i32_store16(&mut self, memarg: MemArg) { 1984 self.emit_wasm_store(&memarg, OperandSize::S16) 1985 } 1986 1987 fn visit_i64_load8_s(&mut self, memarg: MemArg) { 1988 self.emit_wasm_load( 1989 &memarg, 1990 WasmValType::I64, 1991 OperandSize::S8, 1992 Some(ExtendKind::I64Extend8S), 1993 ) 1994 } 1995 1996 fn visit_i64_load8_u(&mut self, memarg: MemArg) { 1997 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S8, None) 1998 } 1999 2000 fn visit_i64_load16_u(&mut self, memarg: MemArg) { 2001 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S16, None) 2002 } 2003 2004 fn visit_i64_load16_s(&mut self, memarg: MemArg) { 2005 self.emit_wasm_load( 2006 &memarg, 2007 WasmValType::I64, 2008 OperandSize::S16, 2009 Some(ExtendKind::I64Extend16S), 2010 ) 2011 } 2012 2013 fn visit_i64_load32_u(&mut self, memarg: MemArg) { 2014 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S32, None) 2015 } 2016 2017 fn visit_i64_load32_s(&mut self, memarg: MemArg) { 2018 self.emit_wasm_load( 2019 &memarg, 2020 WasmValType::I64, 2021 OperandSize::S32, 2022 Some(ExtendKind::I64Extend32S), 2023 ) 2024 } 2025 2026 fn visit_i64_load(&mut self, memarg: MemArg) { 2027 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S64, None) 2028 } 2029 2030 fn visit_i64_store(&mut self, memarg: MemArg) -> Self::Output { 2031 self.emit_wasm_store(&memarg, OperandSize::S64) 2032 } 2033 2034 fn visit_i64_store8(&mut self, memarg: MemArg) -> Self::Output { 2035 self.emit_wasm_store(&memarg, OperandSize::S8) 2036 } 2037 2038 fn visit_i64_store16(&mut self, memarg: MemArg) -> Self::Output { 2039 self.emit_wasm_store(&memarg, OperandSize::S16) 2040 } 2041 2042 fn visit_i64_store32(&mut self, memarg: MemArg) -> Self::Output { 2043 self.emit_wasm_store(&memarg, OperandSize::S32) 2044 } 2045 2046 fn visit_f32_load(&mut self, memarg: MemArg) { 2047 self.emit_wasm_load(&memarg, WasmValType::F32, OperandSize::S32, None) 2048 } 2049 2050 fn visit_f32_store(&mut self, memarg: MemArg) { 2051 self.emit_wasm_store(&memarg, OperandSize::S32) 2052 } 2053 2054 fn visit_f64_load(&mut self, memarg: MemArg) { 2055 self.emit_wasm_load(&memarg, WasmValType::F64, OperandSize::S64, None) 2056 } 2057 2058 fn visit_f64_store(&mut self, memarg: MemArg) { 2059 self.emit_wasm_store(&memarg, OperandSize::S64) 2060 } 2061 2062 fn visit_v128_load(&mut self, memarg: MemArg) { 2063 self.emit_wasm_load(&memarg, WasmValType::V128, OperandSize::S128, None) 2064 } 2065 2066 fn visit_v128_store(&mut self, memarg: MemArg) { 2067 self.emit_wasm_store(&memarg, OperandSize::S128) 2068 } 2069 2070 fn visit_i32_trunc_sat_f32_s(&mut self) { 2071 use OperandSize::*; 2072 2073 self.context 2074 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2075 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Checked); 2076 }); 2077 } 2078 2079 fn visit_i32_trunc_sat_f32_u(&mut self) { 2080 use OperandSize::*; 2081 2082 self.context.convert_op_with_tmp_reg( 2083 self.masm, 2084 WasmValType::I32, 2085 RegClass::Float, 2086 |masm, dst, src, tmp_fpr, dst_size| { 2087 masm.unsigned_truncate( 2088 writable!(dst), 2089 src, 2090 tmp_fpr, 2091 S32, 2092 dst_size, 2093 TruncKind::Checked, 2094 ); 2095 }, 2096 ); 2097 } 2098 2099 fn visit_i32_trunc_sat_f64_s(&mut self) { 2100 use OperandSize::*; 2101 2102 self.context 2103 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2104 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Checked); 2105 }); 2106 } 2107 2108 fn visit_i32_trunc_sat_f64_u(&mut self) { 2109 use OperandSize::*; 2110 2111 self.context.convert_op_with_tmp_reg( 2112 self.masm, 2113 WasmValType::I32, 2114 RegClass::Float, 2115 |masm, dst, src, tmp_fpr, dst_size| { 2116 masm.unsigned_truncate( 2117 writable!(dst), 2118 src, 2119 tmp_fpr, 2120 S64, 2121 dst_size, 2122 TruncKind::Checked, 2123 ); 2124 }, 2125 ); 2126 } 2127 2128 fn visit_i64_trunc_sat_f32_s(&mut self) { 2129 use OperandSize::*; 2130 2131 self.context 2132 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2133 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Checked); 2134 }); 2135 } 2136 2137 fn visit_i64_trunc_sat_f32_u(&mut self) { 2138 use OperandSize::*; 2139 2140 self.context.convert_op_with_tmp_reg( 2141 self.masm, 2142 WasmValType::I64, 2143 RegClass::Float, 2144 |masm, dst, src, tmp_fpr, dst_size| { 2145 masm.unsigned_truncate( 2146 writable!(dst), 2147 src, 2148 tmp_fpr, 2149 S32, 2150 dst_size, 2151 TruncKind::Checked, 2152 ); 2153 }, 2154 ); 2155 } 2156 2157 fn visit_i64_trunc_sat_f64_s(&mut self) { 2158 use OperandSize::*; 2159 2160 self.context 2161 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2162 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Checked); 2163 }); 2164 } 2165 2166 fn visit_i64_trunc_sat_f64_u(&mut self) { 2167 use OperandSize::*; 2168 2169 self.context.convert_op_with_tmp_reg( 2170 self.masm, 2171 WasmValType::I64, 2172 RegClass::Float, 2173 |masm, dst, src, tmp_fpr, dst_size| { 2174 masm.unsigned_truncate( 2175 writable!(dst), 2176 src, 2177 tmp_fpr, 2178 S64, 2179 dst_size, 2180 TruncKind::Checked, 2181 ); 2182 }, 2183 ); 2184 } 2185 2186 wasmparser::for_each_operator!(def_unsupported); 2187 } 2188 2189 impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M> 2190 where 2191 M: MacroAssembler, 2192 { 2193 fn cmp_i32s(&mut self, kind: IntCmpKind) { 2194 self.context.i32_binop(self.masm, |masm, dst, src, size| { 2195 masm.cmp_with_set(writable!(dst), src, kind, size); 2196 TypedReg::i32(dst) 2197 }); 2198 } 2199 2200 fn cmp_i64s(&mut self, kind: IntCmpKind) { 2201 self.context 2202 .i64_binop(self.masm, move |masm, dst, src, size| { 2203 masm.cmp_with_set(writable!(dst), src, kind, size); 2204 TypedReg::i32(dst) // Return value for comparisons is an `i32`. 2205 }); 2206 } 2207 } 2208 2209 impl From<WasmValType> for OperandSize { 2210 fn from(ty: WasmValType) -> OperandSize { 2211 match ty { 2212 WasmValType::I32 | WasmValType::F32 => OperandSize::S32, 2213 WasmValType::I64 | WasmValType::F64 => OperandSize::S64, 2214 WasmValType::V128 => OperandSize::S128, 2215 WasmValType::Ref(rt) => { 2216 match rt.heap_type { 2217 // TODO: Hardcoded size, assuming 64-bit support only. Once 2218 // Wasmtime supports 32-bit architectures, this will need 2219 // to be updated in such a way that the calculation of the 2220 // OperandSize will depend on the target's pointer size. 2221 WasmHeapType::Func => OperandSize::S64, 2222 WasmHeapType::Extern => OperandSize::S64, 2223 t => unimplemented!("Support for WasmHeapType: {t}"), 2224 } 2225 } 2226 } 2227 } 2228 } 2229