1 //! This module is the central place for machine code emission. 2 //! It defines an implementation of wasmparser's Visitor trait 3 //! for `CodeGen`; which defines a visitor per op-code, 4 //! which validates and dispatches to the corresponding 5 //! machine code emitter. 6 7 use crate::abi::RetArea; 8 use crate::codegen::{control_index, Callee, CodeGen, ControlStackFrame, Emission, FnCall}; 9 use crate::masm::{ 10 DivKind, ExtendKind, FloatCmpKind, IntCmpKind, MacroAssembler, MemMoveDirection, MulWideKind, 11 OperandSize, RegImm, RemKind, RoundingMode, SPOffset, ShiftKind, TruncKind, 12 }; 13 use crate::reg::{writable, Reg}; 14 use crate::stack::{TypedReg, Val}; 15 use regalloc2::RegClass; 16 use smallvec::SmallVec; 17 use wasmparser::{ 18 BlockType, BrTable, Ieee32, Ieee64, MemArg, VisitOperator, VisitSimdOperator, V128, 19 }; 20 use wasmtime_cranelift::TRAP_INDIRECT_CALL_TO_NULL; 21 use wasmtime_environ::{ 22 FuncIndex, GlobalIndex, MemoryIndex, TableIndex, TypeIndex, WasmHeapType, WasmValType, 23 FUNCREF_INIT_BIT, 24 }; 25 26 /// A macro to define unsupported WebAssembly operators. 27 /// 28 /// This macro calls itself recursively; 29 /// 1. It no-ops when matching a supported operator. 30 /// 2. Defines the visitor function and panics when 31 /// matching an unsupported operator. 32 macro_rules! def_unsupported { 33 ($( @$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident $ann:tt)*) => { 34 $( 35 def_unsupported!( 36 emit 37 $op 38 39 fn $visit(&mut self $($(,$arg: $argty)*)?) -> Self::Output { 40 $($(let _ = $arg;)*)? 41 42 self.found_unsupported_instruction = Some(stringify!($op)); 43 } 44 ); 45 )* 46 }; 47 48 (emit I32Const $($rest:tt)*) => {}; 49 (emit I64Const $($rest:tt)*) => {}; 50 (emit F32Const $($rest:tt)*) => {}; 51 (emit F64Const $($rest:tt)*) => {}; 52 (emit V128Const $($rest:tt)*) => {}; 53 (emit F32Add $($rest:tt)*) => {}; 54 (emit F64Add $($rest:tt)*) => {}; 55 (emit F32Sub $($rest:tt)*) => {}; 56 (emit F64Sub $($rest:tt)*) => {}; 57 (emit F32Mul $($rest:tt)*) => {}; 58 (emit F64Mul $($rest:tt)*) => {}; 59 (emit F32Div $($rest:tt)*) => {}; 60 (emit F64Div $($rest:tt)*) => {}; 61 (emit F32Min $($rest:tt)*) => {}; 62 (emit F64Min $($rest:tt)*) => {}; 63 (emit F32Max $($rest:tt)*) => {}; 64 (emit F64Max $($rest:tt)*) => {}; 65 (emit F32Copysign $($rest:tt)*) => {}; 66 (emit F64Copysign $($rest:tt)*) => {}; 67 (emit F32Abs $($rest:tt)*) => {}; 68 (emit F64Abs $($rest:tt)*) => {}; 69 (emit F32Neg $($rest:tt)*) => {}; 70 (emit F64Neg $($rest:tt)*) => {}; 71 (emit F32Floor $($rest:tt)*) => {}; 72 (emit F64Floor $($rest:tt)*) => {}; 73 (emit F32Ceil $($rest:tt)*) => {}; 74 (emit F64Ceil $($rest:tt)*) => {}; 75 (emit F32Nearest $($rest:tt)*) => {}; 76 (emit F64Nearest $($rest:tt)*) => {}; 77 (emit F32Trunc $($rest:tt)*) => {}; 78 (emit F64Trunc $($rest:tt)*) => {}; 79 (emit F32Sqrt $($rest:tt)*) => {}; 80 (emit F64Sqrt $($rest:tt)*) => {}; 81 (emit F32Eq $($rest:tt)*) => {}; 82 (emit F64Eq $($rest:tt)*) => {}; 83 (emit F32Ne $($rest:tt)*) => {}; 84 (emit F64Ne $($rest:tt)*) => {}; 85 (emit F32Lt $($rest:tt)*) => {}; 86 (emit F64Lt $($rest:tt)*) => {}; 87 (emit F32Gt $($rest:tt)*) => {}; 88 (emit F64Gt $($rest:tt)*) => {}; 89 (emit F32Le $($rest:tt)*) => {}; 90 (emit F64Le $($rest:tt)*) => {}; 91 (emit F32Ge $($rest:tt)*) => {}; 92 (emit F64Ge $($rest:tt)*) => {}; 93 (emit F32ConvertI32S $($rest:tt)*) => {}; 94 (emit F32ConvertI32U $($rest:tt)*) => {}; 95 (emit F32ConvertI64S $($rest:tt)*) => {}; 96 (emit F32ConvertI64U $($rest:tt)*) => {}; 97 (emit F64ConvertI32S $($rest:tt)*) => {}; 98 (emit F64ConvertI32U $($rest:tt)*) => {}; 99 (emit F64ConvertI64S $($rest:tt)*) => {}; 100 (emit F64ConvertI64U $($rest:tt)*) => {}; 101 (emit F32ReinterpretI32 $($rest:tt)*) => {}; 102 (emit F64ReinterpretI64 $($rest:tt)*) => {}; 103 (emit F32DemoteF64 $($rest:tt)*) => {}; 104 (emit F64PromoteF32 $($rest:tt)*) => {}; 105 (emit I32Add $($rest:tt)*) => {}; 106 (emit I64Add $($rest:tt)*) => {}; 107 (emit I32Sub $($rest:tt)*) => {}; 108 (emit I32Mul $($rest:tt)*) => {}; 109 (emit I32DivS $($rest:tt)*) => {}; 110 (emit I32DivU $($rest:tt)*) => {}; 111 (emit I64DivS $($rest:tt)*) => {}; 112 (emit I64DivU $($rest:tt)*) => {}; 113 (emit I64RemU $($rest:tt)*) => {}; 114 (emit I64RemS $($rest:tt)*) => {}; 115 (emit I32RemU $($rest:tt)*) => {}; 116 (emit I32RemS $($rest:tt)*) => {}; 117 (emit I64Mul $($rest:tt)*) => {}; 118 (emit I64Sub $($rest:tt)*) => {}; 119 (emit I32Eq $($rest:tt)*) => {}; 120 (emit I64Eq $($rest:tt)*) => {}; 121 (emit I32Ne $($rest:tt)*) => {}; 122 (emit I64Ne $($rest:tt)*) => {}; 123 (emit I32LtS $($rest:tt)*) => {}; 124 (emit I64LtS $($rest:tt)*) => {}; 125 (emit I32LtU $($rest:tt)*) => {}; 126 (emit I64LtU $($rest:tt)*) => {}; 127 (emit I32LeS $($rest:tt)*) => {}; 128 (emit I64LeS $($rest:tt)*) => {}; 129 (emit I32LeU $($rest:tt)*) => {}; 130 (emit I64LeU $($rest:tt)*) => {}; 131 (emit I32GtS $($rest:tt)*) => {}; 132 (emit I64GtS $($rest:tt)*) => {}; 133 (emit I32GtU $($rest:tt)*) => {}; 134 (emit I64GtU $($rest:tt)*) => {}; 135 (emit I32GeS $($rest:tt)*) => {}; 136 (emit I64GeS $($rest:tt)*) => {}; 137 (emit I32GeU $($rest:tt)*) => {}; 138 (emit I64GeU $($rest:tt)*) => {}; 139 (emit I32Eqz $($rest:tt)*) => {}; 140 (emit I64Eqz $($rest:tt)*) => {}; 141 (emit I32And $($rest:tt)*) => {}; 142 (emit I64And $($rest:tt)*) => {}; 143 (emit I32Or $($rest:tt)*) => {}; 144 (emit I64Or $($rest:tt)*) => {}; 145 (emit I32Xor $($rest:tt)*) => {}; 146 (emit I64Xor $($rest:tt)*) => {}; 147 (emit I32Shl $($rest:tt)*) => {}; 148 (emit I64Shl $($rest:tt)*) => {}; 149 (emit I32ShrS $($rest:tt)*) => {}; 150 (emit I64ShrS $($rest:tt)*) => {}; 151 (emit I32ShrU $($rest:tt)*) => {}; 152 (emit I64ShrU $($rest:tt)*) => {}; 153 (emit I32Rotl $($rest:tt)*) => {}; 154 (emit I64Rotl $($rest:tt)*) => {}; 155 (emit I32Rotr $($rest:tt)*) => {}; 156 (emit I64Rotr $($rest:tt)*) => {}; 157 (emit I32Clz $($rest:tt)*) => {}; 158 (emit I64Clz $($rest:tt)*) => {}; 159 (emit I32Ctz $($rest:tt)*) => {}; 160 (emit I64Ctz $($rest:tt)*) => {}; 161 (emit I32Popcnt $($rest:tt)*) => {}; 162 (emit I64Popcnt $($rest:tt)*) => {}; 163 (emit I32WrapI64 $($rest:tt)*) => {}; 164 (emit I64ExtendI32S $($rest:tt)*) => {}; 165 (emit I64ExtendI32U $($rest:tt)*) => {}; 166 (emit I32Extend8S $($rest:tt)*) => {}; 167 (emit I32Extend16S $($rest:tt)*) => {}; 168 (emit I64Extend8S $($rest:tt)*) => {}; 169 (emit I64Extend16S $($rest:tt)*) => {}; 170 (emit I64Extend32S $($rest:tt)*) => {}; 171 (emit I32TruncF32S $($rest:tt)*) => {}; 172 (emit I32TruncF32U $($rest:tt)*) => {}; 173 (emit I32TruncF64S $($rest:tt)*) => {}; 174 (emit I32TruncF64U $($rest:tt)*) => {}; 175 (emit I64TruncF32S $($rest:tt)*) => {}; 176 (emit I64TruncF32U $($rest:tt)*) => {}; 177 (emit I64TruncF64S $($rest:tt)*) => {}; 178 (emit I64TruncF64U $($rest:tt)*) => {}; 179 (emit I32ReinterpretF32 $($rest:tt)*) => {}; 180 (emit I64ReinterpretF64 $($rest:tt)*) => {}; 181 (emit LocalGet $($rest:tt)*) => {}; 182 (emit LocalSet $($rest:tt)*) => {}; 183 (emit Call $($rest:tt)*) => {}; 184 (emit End $($rest:tt)*) => {}; 185 (emit Nop $($rest:tt)*) => {}; 186 (emit If $($rest:tt)*) => {}; 187 (emit Else $($rest:tt)*) => {}; 188 (emit Block $($rest:tt)*) => {}; 189 (emit Loop $($rest:tt)*) => {}; 190 (emit Br $($rest:tt)*) => {}; 191 (emit BrIf $($rest:tt)*) => {}; 192 (emit Return $($rest:tt)*) => {}; 193 (emit Unreachable $($rest:tt)*) => {}; 194 (emit LocalTee $($rest:tt)*) => {}; 195 (emit GlobalGet $($rest:tt)*) => {}; 196 (emit GlobalSet $($rest:tt)*) => {}; 197 (emit Select $($rest:tt)*) => {}; 198 (emit Drop $($rest:tt)*) => {}; 199 (emit BrTable $($rest:tt)*) => {}; 200 (emit CallIndirect $($rest:tt)*) => {}; 201 (emit TableInit $($rest:tt)*) => {}; 202 (emit TableCopy $($rest:tt)*) => {}; 203 (emit TableGet $($rest:tt)*) => {}; 204 (emit TableSet $($rest:tt)*) => {}; 205 (emit TableGrow $($rest:tt)*) => {}; 206 (emit TableSize $($rest:tt)*) => {}; 207 (emit TableFill $($rest:tt)*) => {}; 208 (emit ElemDrop $($rest:tt)*) => {}; 209 (emit MemoryInit $($rest:tt)*) => {}; 210 (emit MemoryCopy $($rest:tt)*) => {}; 211 (emit DataDrop $($rest:tt)*) => {}; 212 (emit MemoryFill $($rest:tt)*) => {}; 213 (emit MemorySize $($rest:tt)*) => {}; 214 (emit MemoryGrow $($rest:tt)*) => {}; 215 (emit I32Load $($rest:tt)*) => {}; 216 (emit I32Load8S $($rest:tt)*) => {}; 217 (emit I32Load8U $($rest:tt)*) => {}; 218 (emit I32Load16S $($rest:tt)*) => {}; 219 (emit I32Load16U $($rest:tt)*) => {}; 220 (emit I64Load8S $($rest:tt)*) => {}; 221 (emit I64Load8U $($rest:tt)*) => {}; 222 (emit I64Load16S $($rest:tt)*) => {}; 223 (emit I64Load16U $($rest:tt)*) => {}; 224 (emit I64Load32S $($rest:tt)*) => {}; 225 (emit I64Load32U $($rest:tt)*) => {}; 226 (emit I64Load $($rest:tt)*) => {}; 227 (emit I32Store $($rest:tt)*) => {}; 228 (emit I32Store8 $($rest:tt)*) => {}; 229 (emit I32Store16 $($rest:tt)*) => {}; 230 (emit I64Store $($rest:tt)*) => {}; 231 (emit I64Store8 $($rest:tt)*) => {}; 232 (emit I64Store16 $($rest:tt)*) => {}; 233 (emit I64Store32 $($rest:tt)*) => {}; 234 (emit F32Load $($rest:tt)*) => {}; 235 (emit F32Store $($rest:tt)*) => {}; 236 (emit F64Load $($rest:tt)*) => {}; 237 (emit F64Store $($rest:tt)*) => {}; 238 (emit I32TruncSatF32S $($rest:tt)*) => {}; 239 (emit I32TruncSatF32U $($rest:tt)*) => {}; 240 (emit I32TruncSatF64S $($rest:tt)*) => {}; 241 (emit I32TruncSatF64U $($rest:tt)*) => {}; 242 (emit I64TruncSatF32S $($rest:tt)*) => {}; 243 (emit I64TruncSatF32U $($rest:tt)*) => {}; 244 (emit I64TruncSatF64S $($rest:tt)*) => {}; 245 (emit I64TruncSatF64U $($rest:tt)*) => {}; 246 (emit V128Load $($rest:tt)*) => {}; 247 (emit V128Store $($rest:tt)*) => {}; 248 (emit I64Add128 $($rest:tt)*) => {}; 249 (emit I64Sub128 $($rest:tt)*) => {}; 250 (emit I64MulWideS $($rest:tt)*) => {}; 251 (emit I64MulWideU $($rest:tt)*) => {}; 252 253 (emit $unsupported:tt $($rest:tt)*) => {$($rest)*}; 254 } 255 256 impl<'a, 'translation, 'data, M> VisitOperator<'a> for CodeGen<'a, 'translation, 'data, M, Emission> 257 where 258 M: MacroAssembler, 259 { 260 type Output = (); 261 262 fn visit_i32_const(&mut self, val: i32) { 263 self.context.stack.push(Val::i32(val)); 264 } 265 266 fn visit_i64_const(&mut self, val: i64) { 267 self.context.stack.push(Val::i64(val)); 268 } 269 270 fn visit_f32_const(&mut self, val: Ieee32) { 271 self.context.stack.push(Val::f32(val)); 272 } 273 274 fn visit_f64_const(&mut self, val: Ieee64) { 275 self.context.stack.push(Val::f64(val)); 276 } 277 278 fn visit_f32_add(&mut self) { 279 self.context.binop( 280 self.masm, 281 OperandSize::S32, 282 &mut |masm: &mut M, dst, src, size| { 283 masm.float_add(writable!(dst), dst, src, size); 284 TypedReg::f32(dst) 285 }, 286 ); 287 } 288 289 fn visit_f64_add(&mut self) { 290 self.context.binop( 291 self.masm, 292 OperandSize::S64, 293 &mut |masm: &mut M, dst, src, size| { 294 masm.float_add(writable!(dst), dst, src, size); 295 TypedReg::f64(dst) 296 }, 297 ); 298 } 299 300 fn visit_f32_sub(&mut self) { 301 self.context.binop( 302 self.masm, 303 OperandSize::S32, 304 &mut |masm: &mut M, dst, src, size| { 305 masm.float_sub(writable!(dst), dst, src, size); 306 TypedReg::f32(dst) 307 }, 308 ); 309 } 310 311 fn visit_f64_sub(&mut self) { 312 self.context.binop( 313 self.masm, 314 OperandSize::S64, 315 &mut |masm: &mut M, dst, src, size| { 316 masm.float_sub(writable!(dst), dst, src, size); 317 TypedReg::f64(dst) 318 }, 319 ); 320 } 321 322 fn visit_f32_mul(&mut self) { 323 self.context.binop( 324 self.masm, 325 OperandSize::S32, 326 &mut |masm: &mut M, dst, src, size| { 327 masm.float_mul(writable!(dst), dst, src, size); 328 TypedReg::f32(dst) 329 }, 330 ); 331 } 332 333 fn visit_f64_mul(&mut self) { 334 self.context.binop( 335 self.masm, 336 OperandSize::S64, 337 &mut |masm: &mut M, dst, src, size| { 338 masm.float_mul(writable!(dst), dst, src, size); 339 TypedReg::f64(dst) 340 }, 341 ); 342 } 343 344 fn visit_f32_div(&mut self) { 345 self.context.binop( 346 self.masm, 347 OperandSize::S32, 348 &mut |masm: &mut M, dst, src, size| { 349 masm.float_div(writable!(dst), dst, src, size); 350 TypedReg::f32(dst) 351 }, 352 ); 353 } 354 355 fn visit_f64_div(&mut self) { 356 self.context.binop( 357 self.masm, 358 OperandSize::S64, 359 &mut |masm: &mut M, dst, src, size| { 360 masm.float_div(writable!(dst), dst, src, size); 361 TypedReg::f64(dst) 362 }, 363 ); 364 } 365 366 fn visit_f32_min(&mut self) { 367 self.context.binop( 368 self.masm, 369 OperandSize::S32, 370 &mut |masm: &mut M, dst, src, size| { 371 masm.float_min(writable!(dst), dst, src, size); 372 TypedReg::f32(dst) 373 }, 374 ); 375 } 376 377 fn visit_f64_min(&mut self) { 378 self.context.binop( 379 self.masm, 380 OperandSize::S64, 381 &mut |masm: &mut M, dst, src, size| { 382 masm.float_min(writable!(dst), dst, src, size); 383 TypedReg::f64(dst) 384 }, 385 ); 386 } 387 388 fn visit_f32_max(&mut self) { 389 self.context.binop( 390 self.masm, 391 OperandSize::S32, 392 &mut |masm: &mut M, dst, src, size| { 393 masm.float_max(writable!(dst), dst, src, size); 394 TypedReg::f32(dst) 395 }, 396 ); 397 } 398 399 fn visit_f64_max(&mut self) { 400 self.context.binop( 401 self.masm, 402 OperandSize::S64, 403 &mut |masm: &mut M, dst, src, size| { 404 masm.float_max(writable!(dst), dst, src, size); 405 TypedReg::f64(dst) 406 }, 407 ); 408 } 409 410 fn visit_f32_copysign(&mut self) { 411 self.context.binop( 412 self.masm, 413 OperandSize::S32, 414 &mut |masm: &mut M, dst, src, size| { 415 masm.float_copysign(writable!(dst), dst, src, size); 416 TypedReg::f32(dst) 417 }, 418 ); 419 } 420 421 fn visit_f64_copysign(&mut self) { 422 self.context.binop( 423 self.masm, 424 OperandSize::S64, 425 &mut |masm: &mut M, dst, src, size| { 426 masm.float_copysign(writable!(dst), dst, src, size); 427 TypedReg::f64(dst) 428 }, 429 ); 430 } 431 432 fn visit_f32_abs(&mut self) { 433 self.context 434 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 435 masm.float_abs(writable!(reg), size); 436 TypedReg::f32(reg) 437 }); 438 } 439 440 fn visit_f64_abs(&mut self) { 441 self.context 442 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 443 masm.float_abs(writable!(reg), size); 444 TypedReg::f64(reg) 445 }); 446 } 447 448 fn visit_f32_neg(&mut self) { 449 self.context 450 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 451 masm.float_neg(writable!(reg), size); 452 TypedReg::f32(reg) 453 }); 454 } 455 456 fn visit_f64_neg(&mut self) { 457 self.context 458 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 459 masm.float_neg(writable!(reg), size); 460 TypedReg::f64(reg) 461 }); 462 } 463 464 fn visit_f32_floor(&mut self) { 465 self.masm.float_round( 466 RoundingMode::Down, 467 &mut self.env, 468 &mut self.context, 469 OperandSize::S32, 470 |env, cx, masm| { 471 let builtin = env.builtins.floor_f32::<M::ABI>(); 472 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 473 }, 474 ); 475 } 476 477 fn visit_f64_floor(&mut self) { 478 self.masm.float_round( 479 RoundingMode::Down, 480 &mut self.env, 481 &mut self.context, 482 OperandSize::S64, 483 |env, cx, masm| { 484 let builtin = env.builtins.floor_f64::<M::ABI>(); 485 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 486 }, 487 ); 488 } 489 490 fn visit_f32_ceil(&mut self) { 491 self.masm.float_round( 492 RoundingMode::Up, 493 &mut self.env, 494 &mut self.context, 495 OperandSize::S32, 496 |env, cx, masm| { 497 let builtin = env.builtins.ceil_f32::<M::ABI>(); 498 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 499 }, 500 ); 501 } 502 503 fn visit_f64_ceil(&mut self) { 504 self.masm.float_round( 505 RoundingMode::Up, 506 &mut self.env, 507 &mut self.context, 508 OperandSize::S64, 509 |env, cx, masm| { 510 let builtin = env.builtins.ceil_f64::<M::ABI>(); 511 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 512 }, 513 ); 514 } 515 516 fn visit_f32_nearest(&mut self) { 517 self.masm.float_round( 518 RoundingMode::Nearest, 519 &mut self.env, 520 &mut self.context, 521 OperandSize::S32, 522 |env, cx, masm| { 523 let builtin = env.builtins.nearest_f32::<M::ABI>(); 524 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 525 }, 526 ); 527 } 528 529 fn visit_f64_nearest(&mut self) { 530 self.masm.float_round( 531 RoundingMode::Nearest, 532 &mut self.env, 533 &mut self.context, 534 OperandSize::S64, 535 |env, cx, masm| { 536 let builtin = env.builtins.nearest_f64::<M::ABI>(); 537 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 538 }, 539 ); 540 } 541 542 fn visit_f32_trunc(&mut self) { 543 self.masm.float_round( 544 RoundingMode::Zero, 545 &mut self.env, 546 &mut self.context, 547 OperandSize::S32, 548 |env, cx, masm| { 549 let builtin = env.builtins.trunc_f32::<M::ABI>(); 550 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 551 }, 552 ); 553 } 554 555 fn visit_f64_trunc(&mut self) { 556 self.masm.float_round( 557 RoundingMode::Zero, 558 &mut self.env, 559 &mut self.context, 560 OperandSize::S64, 561 |env, cx, masm| { 562 let builtin = env.builtins.trunc_f64::<M::ABI>(); 563 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 564 }, 565 ); 566 } 567 568 fn visit_f32_sqrt(&mut self) { 569 self.context 570 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 571 masm.float_sqrt(writable!(reg), reg, size); 572 TypedReg::f32(reg) 573 }); 574 } 575 576 fn visit_f64_sqrt(&mut self) { 577 self.context 578 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 579 masm.float_sqrt(writable!(reg), reg, size); 580 TypedReg::f64(reg) 581 }); 582 } 583 584 fn visit_f32_eq(&mut self) { 585 self.context.float_cmp_op( 586 self.masm, 587 OperandSize::S32, 588 &mut |masm: &mut M, dst, src1, src2, size| { 589 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Eq, size); 590 }, 591 ); 592 } 593 594 fn visit_f64_eq(&mut self) { 595 self.context.float_cmp_op( 596 self.masm, 597 OperandSize::S64, 598 &mut |masm: &mut M, dst, src1, src2, size| { 599 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Eq, size); 600 }, 601 ); 602 } 603 604 fn visit_f32_ne(&mut self) { 605 self.context.float_cmp_op( 606 self.masm, 607 OperandSize::S32, 608 &mut |masm: &mut M, dst, src1, src2, size| { 609 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ne, size); 610 }, 611 ); 612 } 613 614 fn visit_f64_ne(&mut self) { 615 self.context.float_cmp_op( 616 self.masm, 617 OperandSize::S64, 618 &mut |masm: &mut M, dst, src1, src2, size| { 619 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ne, size); 620 }, 621 ); 622 } 623 624 fn visit_f32_lt(&mut self) { 625 self.context.float_cmp_op( 626 self.masm, 627 OperandSize::S32, 628 &mut |masm: &mut M, dst, src1, src2, size| { 629 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Lt, size); 630 }, 631 ); 632 } 633 634 fn visit_f64_lt(&mut self) { 635 self.context.float_cmp_op( 636 self.masm, 637 OperandSize::S64, 638 &mut |masm: &mut M, dst, src1, src2, size| { 639 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Lt, size); 640 }, 641 ); 642 } 643 644 fn visit_f32_gt(&mut self) { 645 self.context.float_cmp_op( 646 self.masm, 647 OperandSize::S32, 648 &mut |masm: &mut M, dst, src1, src2, size| { 649 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Gt, size); 650 }, 651 ); 652 } 653 654 fn visit_f64_gt(&mut self) { 655 self.context.float_cmp_op( 656 self.masm, 657 OperandSize::S64, 658 &mut |masm: &mut M, dst, src1, src2, size| { 659 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Gt, size); 660 }, 661 ); 662 } 663 664 fn visit_f32_le(&mut self) { 665 self.context.float_cmp_op( 666 self.masm, 667 OperandSize::S32, 668 &mut |masm: &mut M, dst, src1, src2, size| { 669 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Le, size); 670 }, 671 ); 672 } 673 674 fn visit_f64_le(&mut self) { 675 self.context.float_cmp_op( 676 self.masm, 677 OperandSize::S64, 678 &mut |masm: &mut M, dst, src1, src2, size| { 679 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Le, size); 680 }, 681 ); 682 } 683 684 fn visit_f32_ge(&mut self) { 685 self.context.float_cmp_op( 686 self.masm, 687 OperandSize::S32, 688 &mut |masm: &mut M, dst, src1, src2, size| { 689 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ge, size); 690 }, 691 ); 692 } 693 694 fn visit_f64_ge(&mut self) { 695 self.context.float_cmp_op( 696 self.masm, 697 OperandSize::S64, 698 &mut |masm: &mut M, dst, src1, src2, size| { 699 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ge, size); 700 }, 701 ); 702 } 703 704 fn visit_f32_convert_i32_s(&mut self) { 705 self.context 706 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 707 masm.signed_convert(writable!(dst), src, OperandSize::S32, dst_size); 708 }); 709 } 710 711 fn visit_f32_convert_i32_u(&mut self) { 712 self.context.convert_op_with_tmp_reg( 713 self.masm, 714 WasmValType::F32, 715 RegClass::Int, 716 |masm, dst, src, tmp_gpr, dst_size| { 717 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S32, dst_size); 718 }, 719 ); 720 } 721 722 fn visit_f32_convert_i64_s(&mut self) { 723 self.context 724 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 725 masm.signed_convert(writable!(dst), src, OperandSize::S64, dst_size); 726 }); 727 } 728 729 fn visit_f32_convert_i64_u(&mut self) { 730 self.context.convert_op_with_tmp_reg( 731 self.masm, 732 WasmValType::F32, 733 RegClass::Int, 734 |masm, dst, src, tmp_gpr, dst_size| { 735 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S64, dst_size); 736 }, 737 ); 738 } 739 740 fn visit_f64_convert_i32_s(&mut self) { 741 self.context 742 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 743 masm.signed_convert(writable!(dst), src, OperandSize::S32, dst_size); 744 }); 745 } 746 747 fn visit_f64_convert_i32_u(&mut self) { 748 self.context.convert_op_with_tmp_reg( 749 self.masm, 750 WasmValType::F64, 751 RegClass::Int, 752 |masm, dst, src, tmp_gpr, dst_size| { 753 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S32, dst_size); 754 }, 755 ); 756 } 757 758 fn visit_f64_convert_i64_s(&mut self) { 759 self.context 760 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 761 masm.signed_convert(writable!(dst), src, OperandSize::S64, dst_size); 762 }); 763 } 764 765 fn visit_f64_convert_i64_u(&mut self) { 766 self.context.convert_op_with_tmp_reg( 767 self.masm, 768 WasmValType::F64, 769 RegClass::Int, 770 |masm, dst, src, tmp_gpr, dst_size| { 771 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S64, dst_size); 772 }, 773 ); 774 } 775 776 fn visit_f32_reinterpret_i32(&mut self) { 777 self.context 778 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, size| { 779 masm.reinterpret_int_as_float(writable!(dst), src.into(), size); 780 }); 781 } 782 783 fn visit_f64_reinterpret_i64(&mut self) { 784 self.context 785 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, size| { 786 masm.reinterpret_int_as_float(writable!(dst), src.into(), size); 787 }); 788 } 789 790 fn visit_f32_demote_f64(&mut self) { 791 self.context 792 .unop(self.masm, OperandSize::S64, &mut |masm, reg, _size| { 793 masm.demote(writable!(reg), reg); 794 TypedReg::f32(reg) 795 }); 796 } 797 798 fn visit_f64_promote_f32(&mut self) { 799 self.context 800 .unop(self.masm, OperandSize::S32, &mut |masm, reg, _size| { 801 masm.promote(writable!(reg), reg); 802 TypedReg::f64(reg) 803 }); 804 } 805 806 fn visit_i32_add(&mut self) { 807 self.context.i32_binop(self.masm, |masm, dst, src, size| { 808 masm.add(writable!(dst), dst, src, size); 809 TypedReg::i32(dst) 810 }); 811 } 812 813 fn visit_i64_add(&mut self) { 814 self.context.i64_binop(self.masm, |masm, dst, src, size| { 815 masm.add(writable!(dst), dst, src, size); 816 TypedReg::i64(dst) 817 }); 818 } 819 820 fn visit_i32_sub(&mut self) { 821 self.context.i32_binop(self.masm, |masm, dst, src, size| { 822 masm.sub(writable!(dst), dst, src, size); 823 TypedReg::i32(dst) 824 }); 825 } 826 827 fn visit_i64_sub(&mut self) { 828 self.context.i64_binop(self.masm, |masm, dst, src, size| { 829 masm.sub(writable!(dst), dst, src, size); 830 TypedReg::i64(dst) 831 }); 832 } 833 834 fn visit_i32_mul(&mut self) { 835 self.context.i32_binop(self.masm, |masm, dst, src, size| { 836 masm.mul(writable!(dst), dst, src, size); 837 TypedReg::i32(dst) 838 }); 839 } 840 841 fn visit_i64_mul(&mut self) { 842 self.context.i64_binop(self.masm, |masm, dst, src, size| { 843 masm.mul(writable!(dst), dst, src, size); 844 TypedReg::i64(dst) 845 }); 846 } 847 848 fn visit_i32_div_s(&mut self) { 849 use DivKind::*; 850 use OperandSize::*; 851 852 self.masm.div(&mut self.context, Signed, S32); 853 } 854 855 fn visit_i32_div_u(&mut self) { 856 use DivKind::*; 857 use OperandSize::*; 858 859 self.masm.div(&mut self.context, Unsigned, S32); 860 } 861 862 fn visit_i64_div_s(&mut self) { 863 use DivKind::*; 864 use OperandSize::*; 865 866 self.masm.div(&mut self.context, Signed, S64); 867 } 868 869 fn visit_i64_div_u(&mut self) { 870 use DivKind::*; 871 use OperandSize::*; 872 873 self.masm.div(&mut self.context, Unsigned, S64); 874 } 875 876 fn visit_i32_rem_s(&mut self) { 877 use OperandSize::*; 878 use RemKind::*; 879 880 self.masm.rem(&mut self.context, Signed, S32); 881 } 882 883 fn visit_i32_rem_u(&mut self) { 884 use OperandSize::*; 885 use RemKind::*; 886 887 self.masm.rem(&mut self.context, Unsigned, S32); 888 } 889 890 fn visit_i64_rem_s(&mut self) { 891 use OperandSize::*; 892 use RemKind::*; 893 894 self.masm.rem(&mut self.context, Signed, S64); 895 } 896 897 fn visit_i64_rem_u(&mut self) { 898 use OperandSize::*; 899 use RemKind::*; 900 901 self.masm.rem(&mut self.context, Unsigned, S64); 902 } 903 904 fn visit_i32_eq(&mut self) { 905 self.cmp_i32s(IntCmpKind::Eq); 906 } 907 908 fn visit_i64_eq(&mut self) { 909 self.cmp_i64s(IntCmpKind::Eq); 910 } 911 912 fn visit_i32_ne(&mut self) { 913 self.cmp_i32s(IntCmpKind::Ne); 914 } 915 916 fn visit_i64_ne(&mut self) { 917 self.cmp_i64s(IntCmpKind::Ne); 918 } 919 920 fn visit_i32_lt_s(&mut self) { 921 self.cmp_i32s(IntCmpKind::LtS); 922 } 923 924 fn visit_i64_lt_s(&mut self) { 925 self.cmp_i64s(IntCmpKind::LtS); 926 } 927 928 fn visit_i32_lt_u(&mut self) { 929 self.cmp_i32s(IntCmpKind::LtU); 930 } 931 932 fn visit_i64_lt_u(&mut self) { 933 self.cmp_i64s(IntCmpKind::LtU); 934 } 935 936 fn visit_i32_le_s(&mut self) { 937 self.cmp_i32s(IntCmpKind::LeS); 938 } 939 940 fn visit_i64_le_s(&mut self) { 941 self.cmp_i64s(IntCmpKind::LeS); 942 } 943 944 fn visit_i32_le_u(&mut self) { 945 self.cmp_i32s(IntCmpKind::LeU); 946 } 947 948 fn visit_i64_le_u(&mut self) { 949 self.cmp_i64s(IntCmpKind::LeU); 950 } 951 952 fn visit_i32_gt_s(&mut self) { 953 self.cmp_i32s(IntCmpKind::GtS); 954 } 955 956 fn visit_i64_gt_s(&mut self) { 957 self.cmp_i64s(IntCmpKind::GtS); 958 } 959 960 fn visit_i32_gt_u(&mut self) { 961 self.cmp_i32s(IntCmpKind::GtU); 962 } 963 964 fn visit_i64_gt_u(&mut self) { 965 self.cmp_i64s(IntCmpKind::GtU); 966 } 967 968 fn visit_i32_ge_s(&mut self) { 969 self.cmp_i32s(IntCmpKind::GeS); 970 } 971 972 fn visit_i64_ge_s(&mut self) { 973 self.cmp_i64s(IntCmpKind::GeS); 974 } 975 976 fn visit_i32_ge_u(&mut self) { 977 self.cmp_i32s(IntCmpKind::GeU); 978 } 979 980 fn visit_i64_ge_u(&mut self) { 981 self.cmp_i64s(IntCmpKind::GeU); 982 } 983 984 fn visit_i32_eqz(&mut self) { 985 use OperandSize::*; 986 987 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 988 masm.cmp_with_set(writable!(reg.into()), RegImm::i32(0), IntCmpKind::Eq, size); 989 TypedReg::i32(reg) 990 }); 991 } 992 993 fn visit_i64_eqz(&mut self) { 994 use OperandSize::*; 995 996 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 997 masm.cmp_with_set(writable!(reg.into()), RegImm::i64(0), IntCmpKind::Eq, size); 998 TypedReg::i32(reg) // Return value for `i64.eqz` is an `i32`. 999 }); 1000 } 1001 1002 fn visit_i32_clz(&mut self) { 1003 use OperandSize::*; 1004 1005 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 1006 masm.clz(writable!(reg), reg, size); 1007 TypedReg::i32(reg) 1008 }); 1009 } 1010 1011 fn visit_i64_clz(&mut self) { 1012 use OperandSize::*; 1013 1014 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 1015 masm.clz(writable!(reg), reg, size); 1016 TypedReg::i64(reg) 1017 }); 1018 } 1019 1020 fn visit_i32_ctz(&mut self) { 1021 use OperandSize::*; 1022 1023 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 1024 masm.ctz(writable!(reg), reg, size); 1025 TypedReg::i32(reg) 1026 }); 1027 } 1028 1029 fn visit_i64_ctz(&mut self) { 1030 use OperandSize::*; 1031 1032 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 1033 masm.ctz(writable!(reg), reg, size); 1034 TypedReg::i64(reg) 1035 }); 1036 } 1037 1038 fn visit_i32_and(&mut self) { 1039 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1040 masm.and(writable!(dst), dst, src, size); 1041 TypedReg::i32(dst) 1042 }); 1043 } 1044 1045 fn visit_i64_and(&mut self) { 1046 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1047 masm.and(writable!(dst), dst, src, size); 1048 TypedReg::i64(dst) 1049 }); 1050 } 1051 1052 fn visit_i32_or(&mut self) { 1053 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1054 masm.or(writable!(dst), dst, src, size); 1055 TypedReg::i32(dst) 1056 }); 1057 } 1058 1059 fn visit_i64_or(&mut self) { 1060 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1061 masm.or(writable!(dst), dst, src, size); 1062 TypedReg::i64(dst) 1063 }); 1064 } 1065 1066 fn visit_i32_xor(&mut self) { 1067 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1068 masm.xor(writable!(dst), dst, src, size); 1069 TypedReg::i32(dst) 1070 }); 1071 } 1072 1073 fn visit_i64_xor(&mut self) { 1074 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1075 masm.xor(writable!(dst), dst, src, size); 1076 TypedReg::i64(dst) 1077 }); 1078 } 1079 1080 fn visit_i32_shl(&mut self) { 1081 use ShiftKind::*; 1082 1083 self.context.i32_shift(self.masm, Shl); 1084 } 1085 1086 fn visit_i64_shl(&mut self) { 1087 use ShiftKind::*; 1088 1089 self.context.i64_shift(self.masm, Shl); 1090 } 1091 1092 fn visit_i32_shr_s(&mut self) { 1093 use ShiftKind::*; 1094 1095 self.context.i32_shift(self.masm, ShrS); 1096 } 1097 1098 fn visit_i64_shr_s(&mut self) { 1099 use ShiftKind::*; 1100 1101 self.context.i64_shift(self.masm, ShrS); 1102 } 1103 1104 fn visit_i32_shr_u(&mut self) { 1105 use ShiftKind::*; 1106 1107 self.context.i32_shift(self.masm, ShrU); 1108 } 1109 1110 fn visit_i64_shr_u(&mut self) { 1111 use ShiftKind::*; 1112 1113 self.context.i64_shift(self.masm, ShrU); 1114 } 1115 1116 fn visit_i32_rotl(&mut self) { 1117 use ShiftKind::*; 1118 1119 self.context.i32_shift(self.masm, Rotl); 1120 } 1121 1122 fn visit_i64_rotl(&mut self) { 1123 use ShiftKind::*; 1124 1125 self.context.i64_shift(self.masm, Rotl); 1126 } 1127 1128 fn visit_i32_rotr(&mut self) { 1129 use ShiftKind::*; 1130 1131 self.context.i32_shift(self.masm, Rotr); 1132 } 1133 1134 fn visit_i64_rotr(&mut self) { 1135 use ShiftKind::*; 1136 1137 self.context.i64_shift(self.masm, Rotr); 1138 } 1139 1140 fn visit_end(&mut self) { 1141 if !self.context.reachable { 1142 self.handle_unreachable_end(); 1143 } else { 1144 let mut control = self.control_frames.pop().unwrap(); 1145 control.emit_end(self.masm, &mut self.context); 1146 } 1147 } 1148 1149 fn visit_i32_popcnt(&mut self) { 1150 use OperandSize::*; 1151 self.masm.popcnt(&mut self.context, S32); 1152 } 1153 1154 fn visit_i64_popcnt(&mut self) { 1155 use OperandSize::*; 1156 1157 self.masm.popcnt(&mut self.context, S64); 1158 } 1159 1160 fn visit_i32_wrap_i64(&mut self) { 1161 use OperandSize::*; 1162 1163 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1164 masm.wrap(writable!(reg), reg); 1165 TypedReg::i32(reg) 1166 }); 1167 } 1168 1169 fn visit_i64_extend_i32_s(&mut self) { 1170 use OperandSize::*; 1171 1172 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1173 masm.extend(writable!(reg), reg, ExtendKind::I64ExtendI32S); 1174 TypedReg::i64(reg) 1175 }); 1176 } 1177 1178 fn visit_i64_extend_i32_u(&mut self) { 1179 use OperandSize::*; 1180 1181 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1182 masm.extend(writable!(reg), reg, ExtendKind::I64ExtendI32U); 1183 TypedReg::i64(reg) 1184 }); 1185 } 1186 1187 fn visit_i32_extend8_s(&mut self) { 1188 use OperandSize::*; 1189 1190 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1191 masm.extend(writable!(reg), reg, ExtendKind::I32Extend8S); 1192 TypedReg::i32(reg) 1193 }); 1194 } 1195 1196 fn visit_i32_extend16_s(&mut self) { 1197 use OperandSize::*; 1198 1199 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1200 masm.extend(writable!(reg), reg, ExtendKind::I32Extend16S); 1201 TypedReg::i32(reg) 1202 }); 1203 } 1204 1205 fn visit_i64_extend8_s(&mut self) { 1206 use OperandSize::*; 1207 1208 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1209 masm.extend(writable!(reg), reg, ExtendKind::I64Extend8S); 1210 TypedReg::i64(reg) 1211 }); 1212 } 1213 1214 fn visit_i64_extend16_s(&mut self) { 1215 use OperandSize::*; 1216 1217 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1218 masm.extend(writable!(reg), reg, ExtendKind::I64Extend16S); 1219 TypedReg::i64(reg) 1220 }); 1221 } 1222 1223 fn visit_i64_extend32_s(&mut self) { 1224 use OperandSize::*; 1225 1226 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1227 masm.extend(writable!(reg), reg, ExtendKind::I64Extend32S); 1228 TypedReg::i64(reg) 1229 }); 1230 } 1231 1232 fn visit_i32_trunc_f32_s(&mut self) { 1233 use OperandSize::*; 1234 1235 self.context 1236 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1237 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Unchecked); 1238 }); 1239 } 1240 1241 fn visit_i32_trunc_f32_u(&mut self) { 1242 use OperandSize::*; 1243 1244 self.context.convert_op_with_tmp_reg( 1245 self.masm, 1246 WasmValType::I32, 1247 RegClass::Float, 1248 |masm, dst, src, tmp_fpr, dst_size| { 1249 masm.unsigned_truncate( 1250 writable!(dst), 1251 src, 1252 tmp_fpr, 1253 S32, 1254 dst_size, 1255 TruncKind::Unchecked, 1256 ); 1257 }, 1258 ); 1259 } 1260 1261 fn visit_i32_trunc_f64_s(&mut self) { 1262 use OperandSize::*; 1263 1264 self.context 1265 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1266 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Unchecked); 1267 }); 1268 } 1269 1270 fn visit_i32_trunc_f64_u(&mut self) { 1271 use OperandSize::*; 1272 1273 self.context.convert_op_with_tmp_reg( 1274 self.masm, 1275 WasmValType::I32, 1276 RegClass::Float, 1277 |masm, dst, src, tmp_fpr, dst_size| { 1278 masm.unsigned_truncate( 1279 writable!(dst), 1280 src, 1281 tmp_fpr, 1282 S64, 1283 dst_size, 1284 TruncKind::Unchecked, 1285 ); 1286 }, 1287 ); 1288 } 1289 1290 fn visit_i64_trunc_f32_s(&mut self) { 1291 use OperandSize::*; 1292 1293 self.context 1294 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1295 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Unchecked); 1296 }); 1297 } 1298 1299 fn visit_i64_trunc_f32_u(&mut self) { 1300 use OperandSize::*; 1301 1302 self.context.convert_op_with_tmp_reg( 1303 self.masm, 1304 WasmValType::I64, 1305 RegClass::Float, 1306 |masm, dst, src, tmp_fpr, dst_size| { 1307 masm.unsigned_truncate( 1308 writable!(dst), 1309 src, 1310 tmp_fpr, 1311 S32, 1312 dst_size, 1313 TruncKind::Unchecked, 1314 ); 1315 }, 1316 ); 1317 } 1318 1319 fn visit_i64_trunc_f64_s(&mut self) { 1320 use OperandSize::*; 1321 1322 self.context 1323 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1324 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Unchecked); 1325 }); 1326 } 1327 1328 fn visit_i64_trunc_f64_u(&mut self) { 1329 use OperandSize::*; 1330 1331 self.context.convert_op_with_tmp_reg( 1332 self.masm, 1333 WasmValType::I64, 1334 RegClass::Float, 1335 |masm, dst, src, tmp_fpr, dst_size| { 1336 masm.unsigned_truncate( 1337 writable!(dst), 1338 src, 1339 tmp_fpr, 1340 S64, 1341 dst_size, 1342 TruncKind::Unchecked, 1343 ); 1344 }, 1345 ); 1346 } 1347 1348 fn visit_i32_reinterpret_f32(&mut self) { 1349 self.context 1350 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, size| { 1351 masm.reinterpret_float_as_int(writable!(dst), src.into(), size); 1352 }); 1353 } 1354 1355 fn visit_i64_reinterpret_f64(&mut self) { 1356 self.context 1357 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, size| { 1358 masm.reinterpret_float_as_int(writable!(dst), src.into(), size); 1359 }); 1360 } 1361 1362 fn visit_local_get(&mut self, index: u32) { 1363 use WasmValType::*; 1364 let context = &mut self.context; 1365 let slot = context.frame.get_wasm_local(index); 1366 match slot.ty { 1367 I32 | I64 | F32 | F64 | V128 => context.stack.push(Val::local(index, slot.ty)), 1368 Ref(rt) => match rt.heap_type { 1369 WasmHeapType::Func => context.stack.push(Val::local(index, slot.ty)), 1370 WasmHeapType::Extern => { 1371 self.found_unsupported_instruction = 1372 Some("unsupported local.get of externref local"); 1373 } 1374 ht => unimplemented!("Support for WasmHeapType: {ht}"), 1375 }, 1376 } 1377 } 1378 1379 fn visit_local_set(&mut self, index: u32) { 1380 let src = self.emit_set_local(index); 1381 self.context.free_reg(src); 1382 } 1383 1384 fn visit_call(&mut self, index: u32) { 1385 let callee = self.env.callee_from_index(FuncIndex::from_u32(index)); 1386 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee) 1387 } 1388 1389 fn visit_call_indirect(&mut self, type_index: u32, table_index: u32) { 1390 // Spill now because `emit_lazy_init_funcref` and the `FnCall::emit` 1391 // invocations will both trigger spills since they both call functions. 1392 // However, the machine instructions for the spill emitted by 1393 // `emit_lazy_funcref` will be jumped over if the funcref was previously 1394 // initialized which may result in the machine stack becoming 1395 // unbalanced. 1396 self.context.spill(self.masm); 1397 1398 let type_index = TypeIndex::from_u32(type_index); 1399 let table_index = TableIndex::from_u32(table_index); 1400 1401 self.emit_lazy_init_funcref(table_index); 1402 1403 // Perform the indirect call. 1404 // This code assumes that [`Self::emit_lazy_init_funcref`] will 1405 // push the funcref to the value stack. 1406 let funcref_ptr = self.context.stack.peek().map(|v| v.unwrap_reg()).unwrap(); 1407 self.masm 1408 .trapz(funcref_ptr.into(), TRAP_INDIRECT_CALL_TO_NULL); 1409 self.emit_typecheck_funcref(funcref_ptr.into(), type_index); 1410 1411 let callee = self.env.funcref(type_index); 1412 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee) 1413 } 1414 1415 fn visit_table_init(&mut self, elem: u32, table: u32) { 1416 debug_assert!(self.context.stack.len() >= 3); 1417 let at = self.context.stack.len() - 3; 1418 1419 self.context 1420 .stack 1421 .insert_many(at, &[table.try_into().unwrap(), elem.try_into().unwrap()]); 1422 1423 let builtin = self.env.builtins.table_init::<M::ABI, M::Ptr>(); 1424 FnCall::emit::<M>( 1425 &mut self.env, 1426 self.masm, 1427 &mut self.context, 1428 Callee::Builtin(builtin.clone()), 1429 ) 1430 } 1431 1432 fn visit_table_copy(&mut self, dst: u32, src: u32) { 1433 debug_assert!(self.context.stack.len() >= 3); 1434 let at = self.context.stack.len() - 3; 1435 self.context 1436 .stack 1437 .insert_many(at, &[dst.try_into().unwrap(), src.try_into().unwrap()]); 1438 1439 let builtin = self.env.builtins.table_copy::<M::ABI, M::Ptr>(); 1440 FnCall::emit::<M>( 1441 &mut self.env, 1442 self.masm, 1443 &mut self.context, 1444 Callee::Builtin(builtin), 1445 ) 1446 } 1447 1448 fn visit_table_get(&mut self, table: u32) { 1449 let table_index = TableIndex::from_u32(table); 1450 let table = self.env.table(table_index); 1451 let heap_type = table.ref_type.heap_type; 1452 1453 match heap_type { 1454 WasmHeapType::Func => self.emit_lazy_init_funcref(table_index), 1455 WasmHeapType::Extern => { 1456 self.found_unsupported_instruction = 1457 Some("unsupported table.get of externref table"); 1458 } 1459 t => { 1460 unimplemented!("Support for WasmHeapType: {t}") 1461 } 1462 } 1463 } 1464 1465 fn visit_table_grow(&mut self, table: u32) { 1466 let table_index = TableIndex::from_u32(table); 1467 let table_ty = self.env.table(table_index); 1468 let builtin = match table_ty.ref_type.heap_type { 1469 WasmHeapType::Func => self.env.builtins.table_grow_func_ref::<M::ABI, M::Ptr>(), 1470 ty => unimplemented!("Support for HeapType: {ty}"), 1471 }; 1472 1473 let len = self.context.stack.len(); 1474 // table.grow` requires at least 2 elements on the value stack. 1475 debug_assert!(len >= 2); 1476 let at = len - 2; 1477 1478 // The table_grow builtin expects the parameters in a different 1479 // order. 1480 // The value stack at this point should contain: 1481 // [ init_value | delta ] (stack top) 1482 // but the builtin function expects the init value as the last 1483 // argument. 1484 self.context.stack.inner_mut().swap(len - 1, len - 2); 1485 self.context 1486 .stack 1487 .insert_many(at, &[table.try_into().unwrap()]); 1488 1489 FnCall::emit::<M>( 1490 &mut self.env, 1491 self.masm, 1492 &mut self.context, 1493 Callee::Builtin(builtin.clone()), 1494 ) 1495 } 1496 1497 fn visit_table_size(&mut self, table: u32) { 1498 let table_index = TableIndex::from_u32(table); 1499 let table_data = self.env.resolve_table_data(table_index); 1500 self.emit_compute_table_size(&table_data); 1501 } 1502 1503 fn visit_table_fill(&mut self, table: u32) { 1504 let table_index = TableIndex::from_u32(table); 1505 let table_ty = self.env.table(table_index); 1506 let builtin = match table_ty.ref_type.heap_type { 1507 WasmHeapType::Func => self.env.builtins.table_fill_func_ref::<M::ABI, M::Ptr>(), 1508 ty => unimplemented!("Support for heap type: {ty}"), 1509 }; 1510 1511 let len = self.context.stack.len(); 1512 debug_assert!(len >= 3); 1513 let at = len - 3; 1514 self.context 1515 .stack 1516 .insert_many(at, &[table.try_into().unwrap()]); 1517 FnCall::emit::<M>( 1518 &mut self.env, 1519 self.masm, 1520 &mut self.context, 1521 Callee::Builtin(builtin.clone()), 1522 ) 1523 } 1524 1525 fn visit_table_set(&mut self, table: u32) { 1526 let ptr_type = self.env.ptr_type(); 1527 let table_index = TableIndex::from_u32(table); 1528 let table_data = self.env.resolve_table_data(table_index); 1529 let table = self.env.table(table_index); 1530 match table.ref_type.heap_type { 1531 WasmHeapType::Func => { 1532 assert!( 1533 self.tunables.table_lazy_init, 1534 "unsupported table eager init" 1535 ); 1536 let value = self.context.pop_to_reg(self.masm, None); 1537 let index = self.context.pop_to_reg(self.masm, None); 1538 let base = self.context.any_gpr(self.masm); 1539 let elem_addr = self.emit_compute_table_elem_addr(index.into(), base, &table_data); 1540 // Set the initialized bit. 1541 self.masm.or( 1542 writable!(value.into()), 1543 value.into(), 1544 RegImm::i64(FUNCREF_INIT_BIT as i64), 1545 ptr_type.into(), 1546 ); 1547 1548 self.masm.store_ptr(value.into(), elem_addr); 1549 1550 self.context.free_reg(value); 1551 self.context.free_reg(index); 1552 self.context.free_reg(base); 1553 } 1554 ty => unimplemented!("Support for WasmHeapType: {ty}"), 1555 }; 1556 } 1557 1558 fn visit_elem_drop(&mut self, index: u32) { 1559 let elem_drop = self.env.builtins.elem_drop::<M::ABI, M::Ptr>(); 1560 self.context.stack.extend([index.try_into().unwrap()]); 1561 FnCall::emit::<M>( 1562 &mut self.env, 1563 self.masm, 1564 &mut self.context, 1565 Callee::Builtin(elem_drop), 1566 ) 1567 } 1568 1569 fn visit_memory_init(&mut self, data_index: u32, mem: u32) { 1570 debug_assert!(self.context.stack.len() >= 3); 1571 let at = self.context.stack.len() - 3; 1572 self.context.stack.insert_many( 1573 at, 1574 &[mem.try_into().unwrap(), data_index.try_into().unwrap()], 1575 ); 1576 let builtin = self.env.builtins.memory_init::<M::ABI, M::Ptr>(); 1577 FnCall::emit::<M>( 1578 &mut self.env, 1579 self.masm, 1580 &mut self.context, 1581 Callee::Builtin(builtin), 1582 ) 1583 } 1584 1585 fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) { 1586 // At this point, the stack is expected to contain: 1587 // [ dst_offset, src_offset, len ] 1588 // The following code inserts the missing params, so that stack contains: 1589 // [ vmctx, dst_mem, dst_offset, src_mem, src_offset, len ] 1590 // Which is the order expected by the builtin function. 1591 debug_assert!(self.context.stack.len() >= 3); 1592 let at = self.context.stack.len() - 2; 1593 self.context 1594 .stack 1595 .insert_many(at, &[src_mem.try_into().unwrap()]); 1596 1597 // One element was inserted above, so instead of 3, we use 4. 1598 let at = self.context.stack.len() - 4; 1599 self.context 1600 .stack 1601 .insert_many(at, &[dst_mem.try_into().unwrap()]); 1602 1603 let builtin = self.env.builtins.memory_copy::<M::ABI, M::Ptr>(); 1604 1605 FnCall::emit::<M>( 1606 &mut self.env, 1607 self.masm, 1608 &mut self.context, 1609 Callee::Builtin(builtin), 1610 ) 1611 } 1612 1613 fn visit_memory_fill(&mut self, mem: u32) { 1614 debug_assert!(self.context.stack.len() >= 3); 1615 let at = self.context.stack.len() - 3; 1616 1617 self.context 1618 .stack 1619 .insert_many(at, &[mem.try_into().unwrap()]); 1620 1621 let builtin = self.env.builtins.memory_fill::<M::ABI, M::Ptr>(); 1622 FnCall::emit::<M>( 1623 &mut self.env, 1624 self.masm, 1625 &mut self.context, 1626 Callee::Builtin(builtin), 1627 ) 1628 } 1629 1630 fn visit_memory_size(&mut self, mem: u32) { 1631 let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem)); 1632 self.emit_compute_memory_size(&heap); 1633 } 1634 1635 fn visit_memory_grow(&mut self, mem: u32) { 1636 debug_assert!(self.context.stack.len() >= 1); 1637 // The stack at this point contains: [ delta ] 1638 // The desired state is 1639 // [ vmctx, delta, index ] 1640 self.context.stack.extend([mem.try_into().unwrap()]); 1641 1642 let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem)); 1643 let builtin = self.env.builtins.memory32_grow::<M::ABI, M::Ptr>(); 1644 FnCall::emit::<M>( 1645 &mut self.env, 1646 self.masm, 1647 &mut self.context, 1648 Callee::Builtin(builtin), 1649 ); 1650 1651 // The memory32_grow builtin returns a pointer type, therefore we must 1652 // ensure that the return type is representative of the address space of 1653 // the heap type. 1654 match (self.env.ptr_type(), heap.index_type()) { 1655 (WasmValType::I64, WasmValType::I64) => {} 1656 // When the heap type is smaller than the pointer type, we adjust 1657 // the result of the memory32_grow builtin. 1658 (WasmValType::I64, WasmValType::I32) => { 1659 let top: Reg = self.context.pop_to_reg(self.masm, None).into(); 1660 self.masm.wrap(writable!(top.into()), top.into()); 1661 self.context.stack.push(TypedReg::i32(top).into()); 1662 } 1663 _ => unimplemented!("Support for 32-bit platforms"), 1664 } 1665 } 1666 1667 fn visit_data_drop(&mut self, data_index: u32) { 1668 self.context.stack.extend([data_index.try_into().unwrap()]); 1669 1670 let builtin = self.env.builtins.data_drop::<M::ABI, M::Ptr>(); 1671 FnCall::emit::<M>( 1672 &mut self.env, 1673 self.masm, 1674 &mut self.context, 1675 Callee::Builtin(builtin), 1676 ) 1677 } 1678 1679 fn visit_nop(&mut self) {} 1680 1681 fn visit_if(&mut self, blockty: BlockType) { 1682 self.control_frames.push(ControlStackFrame::r#if( 1683 self.env.resolve_block_sig(blockty), 1684 self.masm, 1685 &mut self.context, 1686 )); 1687 } 1688 1689 fn visit_else(&mut self) { 1690 if !self.context.reachable { 1691 self.handle_unreachable_else(); 1692 } else { 1693 let control = self 1694 .control_frames 1695 .last_mut() 1696 .unwrap_or_else(|| panic!("Expected active control stack frame for else")); 1697 control.emit_else(self.masm, &mut self.context); 1698 } 1699 } 1700 1701 fn visit_block(&mut self, blockty: BlockType) { 1702 self.control_frames.push(ControlStackFrame::block( 1703 self.env.resolve_block_sig(blockty), 1704 self.masm, 1705 &mut self.context, 1706 )); 1707 } 1708 1709 fn visit_loop(&mut self, blockty: BlockType) { 1710 self.control_frames.push(ControlStackFrame::r#loop( 1711 self.env.resolve_block_sig(blockty), 1712 self.masm, 1713 &mut self.context, 1714 )); 1715 1716 // Emit fuel check right after binding the loop header. 1717 if self.tunables.consume_fuel { 1718 self.emit_fuel_check(); 1719 } 1720 } 1721 1722 fn visit_br(&mut self, depth: u32) { 1723 let index = control_index(depth, self.control_frames.len()); 1724 let frame = &mut self.control_frames[index]; 1725 self.context 1726 .unconditional_jump(frame, self.masm, |masm, cx, frame| { 1727 frame 1728 .pop_abi_results::<M, _>(cx, masm, |results, _, _| results.ret_area().copied()); 1729 }); 1730 } 1731 1732 fn visit_br_if(&mut self, depth: u32) { 1733 let index = control_index(depth, self.control_frames.len()); 1734 let frame = &mut self.control_frames[index]; 1735 frame.set_as_target(); 1736 1737 let top = { 1738 let top = self.context.without::<TypedReg, M, _>( 1739 frame.results::<M>().regs(), 1740 self.masm, 1741 |ctx, masm| ctx.pop_to_reg(masm, None), 1742 ); 1743 // Explicitly save any live registers and locals before setting up 1744 // the branch state. 1745 // In some cases, calculating the `top` value above, will result in 1746 // a spill, thus the following one will result in a no-op. 1747 self.context.spill(self.masm); 1748 frame.top_abi_results::<M, _>( 1749 &mut self.context, 1750 self.masm, 1751 |results, context, masm| { 1752 // In the case of `br_if` there's a possibility that we'll 1753 // exit early from the block or fallthrough, for 1754 // a fallthrough, we cannot rely on the pre-computed return area; 1755 // it must be recalculated so that any values that are 1756 // generated are correctly placed near the current stack 1757 // pointer. 1758 results.on_stack().then(|| { 1759 let stack_consumed = context.stack.sizeof(results.stack_operands_len()); 1760 let base = masm.sp_offset().as_u32() - stack_consumed; 1761 let offs = base + results.size(); 1762 RetArea::sp(SPOffset::from_u32(offs)) 1763 }) 1764 }, 1765 ); 1766 top 1767 }; 1768 1769 // Emit instructions to balance the machine stack if the frame has 1770 // a different offset. 1771 let current_sp_offset = self.masm.sp_offset(); 1772 let results_size = frame.results::<M>().size(); 1773 let state = frame.stack_state(); 1774 let (label, cmp, needs_cleanup) = if current_sp_offset > state.target_offset { 1775 (self.masm.get_label(), IntCmpKind::Eq, true) 1776 } else { 1777 (*frame.label(), IntCmpKind::Ne, false) 1778 }; 1779 1780 self.masm 1781 .branch(cmp, top.reg.into(), top.reg.into(), label, OperandSize::S32); 1782 self.context.free_reg(top); 1783 1784 if needs_cleanup { 1785 // Emit instructions to balance the stack and jump if not falling 1786 // through. 1787 self.masm.memmove( 1788 current_sp_offset, 1789 state.target_offset, 1790 results_size, 1791 MemMoveDirection::LowToHigh, 1792 ); 1793 self.masm.ensure_sp_for_jump(state.target_offset); 1794 self.masm.jmp(*frame.label()); 1795 1796 // Restore sp_offset to what it was for falling through and emit 1797 // fallthrough label. 1798 self.masm.reset_stack_pointer(current_sp_offset); 1799 self.masm.bind(label); 1800 } 1801 } 1802 1803 fn visit_br_table(&mut self, targets: BrTable<'a>) { 1804 // +1 to account for the default target. 1805 let len = targets.len() + 1; 1806 // SmallVec<[_; 5]> to match the binary emission layer (e.g 1807 // see `JmpTableSeq'), but here we use 5 instead since we 1808 // bundle the default target as the last element in the array. 1809 let labels: SmallVec<[_; 5]> = (0..len).map(|_| self.masm.get_label()).collect(); 1810 1811 let default_index = control_index(targets.default(), self.control_frames.len()); 1812 let default_frame = &mut self.control_frames[default_index]; 1813 let default_result = default_frame.results::<M>(); 1814 1815 let (index, tmp) = { 1816 let index_and_tmp = self.context.without::<(TypedReg, _), M, _>( 1817 default_result.regs(), 1818 self.masm, 1819 |cx, masm| (cx.pop_to_reg(masm, None), cx.any_gpr(masm)), 1820 ); 1821 1822 // Materialize any constants or locals into their result representation, 1823 // so that when reachability is restored, they are correctly located. 1824 default_frame.top_abi_results::<M, _>(&mut self.context, self.masm, |results, _, _| { 1825 results.ret_area().copied() 1826 }); 1827 index_and_tmp 1828 }; 1829 1830 self.masm.jmp_table(&labels, index.into(), tmp); 1831 // Save the original stack pointer offset; we will reset the stack 1832 // pointer to this offset after jumping to each of the targets. Each 1833 // jump might adjust the stack according to the base offset of the 1834 // target. 1835 let current_sp = self.masm.sp_offset(); 1836 1837 for (t, l) in targets 1838 .targets() 1839 .into_iter() 1840 .chain(std::iter::once(Ok(targets.default()))) 1841 .zip(labels.iter()) 1842 { 1843 let control_index = control_index(t.unwrap(), self.control_frames.len()); 1844 let frame = &mut self.control_frames[control_index]; 1845 // Reset the stack pointer to its original offset. This is needed 1846 // because each jump will potentially adjust the stack pointer 1847 // according to the base offset of the target. 1848 self.masm.reset_stack_pointer(current_sp); 1849 1850 // NB: We don't perform any result handling as it was 1851 // already taken care of above before jumping to the 1852 // jump table. 1853 self.masm.bind(*l); 1854 // Ensure that the stack pointer is correctly positioned before 1855 // jumping to the jump table code. 1856 let state = frame.stack_state(); 1857 self.masm.ensure_sp_for_jump(state.target_offset); 1858 self.masm.jmp(*frame.label()); 1859 frame.set_as_target(); 1860 } 1861 // Finally reset the stack pointer to the original location. 1862 // The reachability analysis, will ensure it's correctly located 1863 // once reachability is restored. 1864 self.masm.reset_stack_pointer(current_sp); 1865 self.context.reachable = false; 1866 self.context.free_reg(index.reg); 1867 self.context.free_reg(tmp); 1868 } 1869 1870 fn visit_return(&mut self) { 1871 // Grab the outermost frame, which is the function's body 1872 // frame. We don't rely on [`codegen::control_index`] since 1873 // this frame is implicit and we know that it should exist at 1874 // index 0. 1875 let outermost = &mut self.control_frames[0]; 1876 self.context 1877 .unconditional_jump(outermost, self.masm, |masm, cx, frame| { 1878 frame 1879 .pop_abi_results::<M, _>(cx, masm, |results, _, _| results.ret_area().copied()); 1880 }); 1881 } 1882 1883 fn visit_unreachable(&mut self) { 1884 self.masm.unreachable(); 1885 self.context.reachable = false; 1886 // Set the implicit outermost frame as target to perform the necessary 1887 // stack clean up. 1888 let outermost = &mut self.control_frames[0]; 1889 outermost.set_as_target(); 1890 } 1891 1892 fn visit_local_tee(&mut self, index: u32) { 1893 let typed_reg = self.emit_set_local(index); 1894 self.context.stack.push(typed_reg.into()); 1895 } 1896 1897 fn visit_global_get(&mut self, global_index: u32) { 1898 let index = GlobalIndex::from_u32(global_index); 1899 let (ty, addr) = self.emit_get_global_addr(index); 1900 let dst = self.context.reg_for_type(ty, self.masm); 1901 self.masm.load(addr, writable!(dst), ty.into()); 1902 self.context.stack.push(Val::reg(dst, ty)); 1903 } 1904 1905 fn visit_global_set(&mut self, global_index: u32) { 1906 let index = GlobalIndex::from_u32(global_index); 1907 let (ty, addr) = self.emit_get_global_addr(index); 1908 1909 let typed_reg = self.context.pop_to_reg(self.masm, None); 1910 self.context.free_reg(typed_reg.reg); 1911 self.masm.store(typed_reg.reg.into(), addr, ty.into()); 1912 } 1913 1914 fn visit_drop(&mut self) { 1915 self.context.drop_last(1, |regalloc, val| match val { 1916 Val::Reg(tr) => regalloc.free(tr.reg.into()), 1917 Val::Memory(m) => self.masm.free_stack(m.slot.size), 1918 _ => {} 1919 }); 1920 } 1921 1922 fn visit_select(&mut self) { 1923 let cond = self.context.pop_to_reg(self.masm, None); 1924 let val2 = self.context.pop_to_reg(self.masm, None); 1925 let val1 = self.context.pop_to_reg(self.masm, None); 1926 self.masm 1927 .cmp(cond.reg.into(), RegImm::i32(0), OperandSize::S32); 1928 // Conditionally move val1 to val2 if the comparison is 1929 // not zero. 1930 self.masm.cmov( 1931 writable!(val2.into()), 1932 val1.into(), 1933 IntCmpKind::Ne, 1934 val1.ty.into(), 1935 ); 1936 self.context.stack.push(val2.into()); 1937 self.context.free_reg(val1.reg); 1938 self.context.free_reg(cond); 1939 } 1940 1941 fn visit_i32_load(&mut self, memarg: MemArg) { 1942 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S32, None); 1943 } 1944 1945 fn visit_i32_load8_s(&mut self, memarg: MemArg) { 1946 self.emit_wasm_load( 1947 &memarg, 1948 WasmValType::I32, 1949 OperandSize::S8, 1950 Some(ExtendKind::I32Extend8S), 1951 ); 1952 } 1953 1954 fn visit_i32_load8_u(&mut self, memarg: MemArg) { 1955 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S8, None); 1956 } 1957 1958 fn visit_i32_load16_s(&mut self, memarg: MemArg) { 1959 self.emit_wasm_load( 1960 &memarg, 1961 WasmValType::I32, 1962 OperandSize::S16, 1963 Some(ExtendKind::I32Extend16S), 1964 ) 1965 } 1966 1967 fn visit_i32_load16_u(&mut self, memarg: MemArg) { 1968 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S16, None) 1969 } 1970 1971 fn visit_i32_store(&mut self, memarg: MemArg) { 1972 self.emit_wasm_store(&memarg, OperandSize::S32); 1973 } 1974 1975 fn visit_i32_store8(&mut self, memarg: MemArg) { 1976 self.emit_wasm_store(&memarg, OperandSize::S8) 1977 } 1978 1979 fn visit_i32_store16(&mut self, memarg: MemArg) { 1980 self.emit_wasm_store(&memarg, OperandSize::S16) 1981 } 1982 1983 fn visit_i64_load8_s(&mut self, memarg: MemArg) { 1984 self.emit_wasm_load( 1985 &memarg, 1986 WasmValType::I64, 1987 OperandSize::S8, 1988 Some(ExtendKind::I64Extend8S), 1989 ) 1990 } 1991 1992 fn visit_i64_load8_u(&mut self, memarg: MemArg) { 1993 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S8, None) 1994 } 1995 1996 fn visit_i64_load16_u(&mut self, memarg: MemArg) { 1997 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S16, None) 1998 } 1999 2000 fn visit_i64_load16_s(&mut self, memarg: MemArg) { 2001 self.emit_wasm_load( 2002 &memarg, 2003 WasmValType::I64, 2004 OperandSize::S16, 2005 Some(ExtendKind::I64Extend16S), 2006 ) 2007 } 2008 2009 fn visit_i64_load32_u(&mut self, memarg: MemArg) { 2010 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S32, None) 2011 } 2012 2013 fn visit_i64_load32_s(&mut self, memarg: MemArg) { 2014 self.emit_wasm_load( 2015 &memarg, 2016 WasmValType::I64, 2017 OperandSize::S32, 2018 Some(ExtendKind::I64Extend32S), 2019 ) 2020 } 2021 2022 fn visit_i64_load(&mut self, memarg: MemArg) { 2023 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S64, None) 2024 } 2025 2026 fn visit_i64_store(&mut self, memarg: MemArg) -> Self::Output { 2027 self.emit_wasm_store(&memarg, OperandSize::S64) 2028 } 2029 2030 fn visit_i64_store8(&mut self, memarg: MemArg) -> Self::Output { 2031 self.emit_wasm_store(&memarg, OperandSize::S8) 2032 } 2033 2034 fn visit_i64_store16(&mut self, memarg: MemArg) -> Self::Output { 2035 self.emit_wasm_store(&memarg, OperandSize::S16) 2036 } 2037 2038 fn visit_i64_store32(&mut self, memarg: MemArg) -> Self::Output { 2039 self.emit_wasm_store(&memarg, OperandSize::S32) 2040 } 2041 2042 fn visit_f32_load(&mut self, memarg: MemArg) { 2043 self.emit_wasm_load(&memarg, WasmValType::F32, OperandSize::S32, None) 2044 } 2045 2046 fn visit_f32_store(&mut self, memarg: MemArg) { 2047 self.emit_wasm_store(&memarg, OperandSize::S32) 2048 } 2049 2050 fn visit_f64_load(&mut self, memarg: MemArg) { 2051 self.emit_wasm_load(&memarg, WasmValType::F64, OperandSize::S64, None) 2052 } 2053 2054 fn visit_f64_store(&mut self, memarg: MemArg) { 2055 self.emit_wasm_store(&memarg, OperandSize::S64) 2056 } 2057 2058 fn visit_i32_trunc_sat_f32_s(&mut self) { 2059 use OperandSize::*; 2060 2061 self.context 2062 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2063 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Checked); 2064 }); 2065 } 2066 2067 fn visit_i32_trunc_sat_f32_u(&mut self) { 2068 use OperandSize::*; 2069 2070 self.context.convert_op_with_tmp_reg( 2071 self.masm, 2072 WasmValType::I32, 2073 RegClass::Float, 2074 |masm, dst, src, tmp_fpr, dst_size| { 2075 masm.unsigned_truncate( 2076 writable!(dst), 2077 src, 2078 tmp_fpr, 2079 S32, 2080 dst_size, 2081 TruncKind::Checked, 2082 ); 2083 }, 2084 ); 2085 } 2086 2087 fn visit_i32_trunc_sat_f64_s(&mut self) { 2088 use OperandSize::*; 2089 2090 self.context 2091 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2092 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Checked); 2093 }); 2094 } 2095 2096 fn visit_i32_trunc_sat_f64_u(&mut self) { 2097 use OperandSize::*; 2098 2099 self.context.convert_op_with_tmp_reg( 2100 self.masm, 2101 WasmValType::I32, 2102 RegClass::Float, 2103 |masm, dst, src, tmp_fpr, dst_size| { 2104 masm.unsigned_truncate( 2105 writable!(dst), 2106 src, 2107 tmp_fpr, 2108 S64, 2109 dst_size, 2110 TruncKind::Checked, 2111 ); 2112 }, 2113 ); 2114 } 2115 2116 fn visit_i64_trunc_sat_f32_s(&mut self) { 2117 use OperandSize::*; 2118 2119 self.context 2120 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2121 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Checked); 2122 }); 2123 } 2124 2125 fn visit_i64_trunc_sat_f32_u(&mut self) { 2126 use OperandSize::*; 2127 2128 self.context.convert_op_with_tmp_reg( 2129 self.masm, 2130 WasmValType::I64, 2131 RegClass::Float, 2132 |masm, dst, src, tmp_fpr, dst_size| { 2133 masm.unsigned_truncate( 2134 writable!(dst), 2135 src, 2136 tmp_fpr, 2137 S32, 2138 dst_size, 2139 TruncKind::Checked, 2140 ); 2141 }, 2142 ); 2143 } 2144 2145 fn visit_i64_trunc_sat_f64_s(&mut self) { 2146 use OperandSize::*; 2147 2148 self.context 2149 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2150 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Checked); 2151 }); 2152 } 2153 2154 fn visit_i64_trunc_sat_f64_u(&mut self) { 2155 use OperandSize::*; 2156 2157 self.context.convert_op_with_tmp_reg( 2158 self.masm, 2159 WasmValType::I64, 2160 RegClass::Float, 2161 |masm, dst, src, tmp_fpr, dst_size| { 2162 masm.unsigned_truncate( 2163 writable!(dst), 2164 src, 2165 tmp_fpr, 2166 S64, 2167 dst_size, 2168 TruncKind::Checked, 2169 ); 2170 }, 2171 ); 2172 } 2173 2174 fn visit_i64_add128(&mut self) { 2175 self.context 2176 .binop128(self.masm, |masm, lhs_lo, lhs_hi, rhs_lo, rhs_hi| { 2177 masm.add128( 2178 writable!(lhs_lo), 2179 writable!(lhs_hi), 2180 lhs_lo, 2181 lhs_hi, 2182 rhs_lo, 2183 rhs_hi, 2184 ); 2185 (TypedReg::i64(lhs_lo), TypedReg::i64(lhs_hi)) 2186 }); 2187 } 2188 2189 fn visit_i64_sub128(&mut self) { 2190 self.context 2191 .binop128(self.masm, |masm, lhs_lo, lhs_hi, rhs_lo, rhs_hi| { 2192 masm.sub128( 2193 writable!(lhs_lo), 2194 writable!(lhs_hi), 2195 lhs_lo, 2196 lhs_hi, 2197 rhs_lo, 2198 rhs_hi, 2199 ); 2200 (TypedReg::i64(lhs_lo), TypedReg::i64(lhs_hi)) 2201 }); 2202 } 2203 2204 fn visit_i64_mul_wide_s(&mut self) { 2205 self.masm.mul_wide(&mut self.context, MulWideKind::Signed); 2206 } 2207 2208 fn visit_i64_mul_wide_u(&mut self) { 2209 self.masm.mul_wide(&mut self.context, MulWideKind::Unsigned); 2210 } 2211 2212 wasmparser::for_each_visit_operator!(def_unsupported); 2213 } 2214 2215 impl<'a, 'translation, 'data, M> VisitSimdOperator<'a> 2216 for CodeGen<'a, 'translation, 'data, M, Emission> 2217 where 2218 M: MacroAssembler, 2219 { 2220 fn visit_v128_const(&mut self, val: V128) { 2221 self.context.stack.push(Val::v128(val.i128())) 2222 } 2223 2224 fn visit_v128_load(&mut self, memarg: MemArg) { 2225 self.emit_wasm_load(&memarg, WasmValType::V128, OperandSize::S128, None) 2226 } 2227 2228 fn visit_v128_store(&mut self, memarg: MemArg) { 2229 self.emit_wasm_store(&memarg, OperandSize::S128) 2230 } 2231 2232 wasmparser::for_each_visit_simd_operator!(def_unsupported); 2233 } 2234 2235 impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Emission> 2236 where 2237 M: MacroAssembler, 2238 { 2239 fn cmp_i32s(&mut self, kind: IntCmpKind) { 2240 self.context.i32_binop(self.masm, |masm, dst, src, size| { 2241 masm.cmp_with_set(writable!(dst), src, kind, size); 2242 TypedReg::i32(dst) 2243 }); 2244 } 2245 2246 fn cmp_i64s(&mut self, kind: IntCmpKind) { 2247 self.context 2248 .i64_binop(self.masm, move |masm, dst, src, size| { 2249 masm.cmp_with_set(writable!(dst), src, kind, size); 2250 TypedReg::i32(dst) // Return value for comparisons is an `i32`. 2251 }); 2252 } 2253 } 2254 2255 impl From<WasmValType> for OperandSize { 2256 fn from(ty: WasmValType) -> OperandSize { 2257 match ty { 2258 WasmValType::I32 | WasmValType::F32 => OperandSize::S32, 2259 WasmValType::I64 | WasmValType::F64 => OperandSize::S64, 2260 WasmValType::V128 => OperandSize::S128, 2261 WasmValType::Ref(rt) => { 2262 match rt.heap_type { 2263 // TODO: Hardcoded size, assuming 64-bit support only. Once 2264 // Wasmtime supports 32-bit architectures, this will need 2265 // to be updated in such a way that the calculation of the 2266 // OperandSize will depend on the target's pointer size. 2267 WasmHeapType::Func => OperandSize::S64, 2268 WasmHeapType::Extern => OperandSize::S64, 2269 t => unimplemented!("Support for WasmHeapType: {t}"), 2270 } 2271 } 2272 } 2273 } 2274 } 2275