1 //! This module is the central place for machine code emission. 2 //! It defines an implementation of wasmparser's Visitor trait 3 //! for `CodeGen`; which defines a visitor per op-code, 4 //! which validates and dispatches to the corresponding 5 //! machine code emitter. 6 7 use crate::abi::RetArea; 8 use crate::codegen::{control_index, Callee, CodeGen, ControlStackFrame, Emission, FnCall}; 9 use crate::masm::{ 10 DivKind, ExtendKind, FloatCmpKind, IntCmpKind, MacroAssembler, MemMoveDirection, MulWideKind, 11 OperandSize, RegImm, RemKind, RoundingMode, SPOffset, ShiftKind, TruncKind, 12 }; 13 use crate::reg::{writable, Reg}; 14 use crate::stack::{TypedReg, Val}; 15 use regalloc2::RegClass; 16 use smallvec::SmallVec; 17 use wasmparser::{ 18 BlockType, BrTable, Ieee32, Ieee64, MemArg, VisitOperator, VisitSimdOperator, V128, 19 }; 20 use wasmtime_cranelift::TRAP_INDIRECT_CALL_TO_NULL; 21 use wasmtime_environ::{ 22 FuncIndex, GlobalIndex, MemoryIndex, TableIndex, TypeIndex, WasmHeapType, WasmValType, 23 FUNCREF_INIT_BIT, 24 }; 25 26 /// A macro to define unsupported WebAssembly operators. 27 /// 28 /// This macro calls itself recursively; 29 /// 1. It no-ops when matching a supported operator. 30 /// 2. Defines the visitor function and panics when 31 /// matching an unsupported operator. 32 macro_rules! def_unsupported { 33 ($( @$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident $ann:tt)*) => { 34 $( 35 def_unsupported!( 36 emit 37 $op 38 39 fn $visit(&mut self $($(,$arg: $argty)*)?) -> Self::Output { 40 $($(let _ = $arg;)*)? 41 42 self.found_unsupported_instruction = Some(stringify!($op)); 43 } 44 ); 45 )* 46 }; 47 48 (emit I32Const $($rest:tt)*) => {}; 49 (emit I64Const $($rest:tt)*) => {}; 50 (emit F32Const $($rest:tt)*) => {}; 51 (emit F64Const $($rest:tt)*) => {}; 52 (emit V128Const $($rest:tt)*) => {}; 53 (emit F32Add $($rest:tt)*) => {}; 54 (emit F64Add $($rest:tt)*) => {}; 55 (emit F32Sub $($rest:tt)*) => {}; 56 (emit F64Sub $($rest:tt)*) => {}; 57 (emit F32Mul $($rest:tt)*) => {}; 58 (emit F64Mul $($rest:tt)*) => {}; 59 (emit F32Div $($rest:tt)*) => {}; 60 (emit F64Div $($rest:tt)*) => {}; 61 (emit F32Min $($rest:tt)*) => {}; 62 (emit F64Min $($rest:tt)*) => {}; 63 (emit F32Max $($rest:tt)*) => {}; 64 (emit F64Max $($rest:tt)*) => {}; 65 (emit F32Copysign $($rest:tt)*) => {}; 66 (emit F64Copysign $($rest:tt)*) => {}; 67 (emit F32Abs $($rest:tt)*) => {}; 68 (emit F64Abs $($rest:tt)*) => {}; 69 (emit F32Neg $($rest:tt)*) => {}; 70 (emit F64Neg $($rest:tt)*) => {}; 71 (emit F32Floor $($rest:tt)*) => {}; 72 (emit F64Floor $($rest:tt)*) => {}; 73 (emit F32Ceil $($rest:tt)*) => {}; 74 (emit F64Ceil $($rest:tt)*) => {}; 75 (emit F32Nearest $($rest:tt)*) => {}; 76 (emit F64Nearest $($rest:tt)*) => {}; 77 (emit F32Trunc $($rest:tt)*) => {}; 78 (emit F64Trunc $($rest:tt)*) => {}; 79 (emit F32Sqrt $($rest:tt)*) => {}; 80 (emit F64Sqrt $($rest:tt)*) => {}; 81 (emit F32Eq $($rest:tt)*) => {}; 82 (emit F64Eq $($rest:tt)*) => {}; 83 (emit F32Ne $($rest:tt)*) => {}; 84 (emit F64Ne $($rest:tt)*) => {}; 85 (emit F32Lt $($rest:tt)*) => {}; 86 (emit F64Lt $($rest:tt)*) => {}; 87 (emit F32Gt $($rest:tt)*) => {}; 88 (emit F64Gt $($rest:tt)*) => {}; 89 (emit F32Le $($rest:tt)*) => {}; 90 (emit F64Le $($rest:tt)*) => {}; 91 (emit F32Ge $($rest:tt)*) => {}; 92 (emit F64Ge $($rest:tt)*) => {}; 93 (emit F32ConvertI32S $($rest:tt)*) => {}; 94 (emit F32ConvertI32U $($rest:tt)*) => {}; 95 (emit F32ConvertI64S $($rest:tt)*) => {}; 96 (emit F32ConvertI64U $($rest:tt)*) => {}; 97 (emit F64ConvertI32S $($rest:tt)*) => {}; 98 (emit F64ConvertI32U $($rest:tt)*) => {}; 99 (emit F64ConvertI64S $($rest:tt)*) => {}; 100 (emit F64ConvertI64U $($rest:tt)*) => {}; 101 (emit F32ReinterpretI32 $($rest:tt)*) => {}; 102 (emit F64ReinterpretI64 $($rest:tt)*) => {}; 103 (emit F32DemoteF64 $($rest:tt)*) => {}; 104 (emit F64PromoteF32 $($rest:tt)*) => {}; 105 (emit I32Add $($rest:tt)*) => {}; 106 (emit I64Add $($rest:tt)*) => {}; 107 (emit I32Sub $($rest:tt)*) => {}; 108 (emit I32Mul $($rest:tt)*) => {}; 109 (emit I32DivS $($rest:tt)*) => {}; 110 (emit I32DivU $($rest:tt)*) => {}; 111 (emit I64DivS $($rest:tt)*) => {}; 112 (emit I64DivU $($rest:tt)*) => {}; 113 (emit I64RemU $($rest:tt)*) => {}; 114 (emit I64RemS $($rest:tt)*) => {}; 115 (emit I32RemU $($rest:tt)*) => {}; 116 (emit I32RemS $($rest:tt)*) => {}; 117 (emit I64Mul $($rest:tt)*) => {}; 118 (emit I64Sub $($rest:tt)*) => {}; 119 (emit I32Eq $($rest:tt)*) => {}; 120 (emit I64Eq $($rest:tt)*) => {}; 121 (emit I32Ne $($rest:tt)*) => {}; 122 (emit I64Ne $($rest:tt)*) => {}; 123 (emit I32LtS $($rest:tt)*) => {}; 124 (emit I64LtS $($rest:tt)*) => {}; 125 (emit I32LtU $($rest:tt)*) => {}; 126 (emit I64LtU $($rest:tt)*) => {}; 127 (emit I32LeS $($rest:tt)*) => {}; 128 (emit I64LeS $($rest:tt)*) => {}; 129 (emit I32LeU $($rest:tt)*) => {}; 130 (emit I64LeU $($rest:tt)*) => {}; 131 (emit I32GtS $($rest:tt)*) => {}; 132 (emit I64GtS $($rest:tt)*) => {}; 133 (emit I32GtU $($rest:tt)*) => {}; 134 (emit I64GtU $($rest:tt)*) => {}; 135 (emit I32GeS $($rest:tt)*) => {}; 136 (emit I64GeS $($rest:tt)*) => {}; 137 (emit I32GeU $($rest:tt)*) => {}; 138 (emit I64GeU $($rest:tt)*) => {}; 139 (emit I32Eqz $($rest:tt)*) => {}; 140 (emit I64Eqz $($rest:tt)*) => {}; 141 (emit I32And $($rest:tt)*) => {}; 142 (emit I64And $($rest:tt)*) => {}; 143 (emit I32Or $($rest:tt)*) => {}; 144 (emit I64Or $($rest:tt)*) => {}; 145 (emit I32Xor $($rest:tt)*) => {}; 146 (emit I64Xor $($rest:tt)*) => {}; 147 (emit I32Shl $($rest:tt)*) => {}; 148 (emit I64Shl $($rest:tt)*) => {}; 149 (emit I32ShrS $($rest:tt)*) => {}; 150 (emit I64ShrS $($rest:tt)*) => {}; 151 (emit I32ShrU $($rest:tt)*) => {}; 152 (emit I64ShrU $($rest:tt)*) => {}; 153 (emit I32Rotl $($rest:tt)*) => {}; 154 (emit I64Rotl $($rest:tt)*) => {}; 155 (emit I32Rotr $($rest:tt)*) => {}; 156 (emit I64Rotr $($rest:tt)*) => {}; 157 (emit I32Clz $($rest:tt)*) => {}; 158 (emit I64Clz $($rest:tt)*) => {}; 159 (emit I32Ctz $($rest:tt)*) => {}; 160 (emit I64Ctz $($rest:tt)*) => {}; 161 (emit I32Popcnt $($rest:tt)*) => {}; 162 (emit I64Popcnt $($rest:tt)*) => {}; 163 (emit I32WrapI64 $($rest:tt)*) => {}; 164 (emit I64ExtendI32S $($rest:tt)*) => {}; 165 (emit I64ExtendI32U $($rest:tt)*) => {}; 166 (emit I32Extend8S $($rest:tt)*) => {}; 167 (emit I32Extend16S $($rest:tt)*) => {}; 168 (emit I64Extend8S $($rest:tt)*) => {}; 169 (emit I64Extend16S $($rest:tt)*) => {}; 170 (emit I64Extend32S $($rest:tt)*) => {}; 171 (emit I32TruncF32S $($rest:tt)*) => {}; 172 (emit I32TruncF32U $($rest:tt)*) => {}; 173 (emit I32TruncF64S $($rest:tt)*) => {}; 174 (emit I32TruncF64U $($rest:tt)*) => {}; 175 (emit I64TruncF32S $($rest:tt)*) => {}; 176 (emit I64TruncF32U $($rest:tt)*) => {}; 177 (emit I64TruncF64S $($rest:tt)*) => {}; 178 (emit I64TruncF64U $($rest:tt)*) => {}; 179 (emit I32ReinterpretF32 $($rest:tt)*) => {}; 180 (emit I64ReinterpretF64 $($rest:tt)*) => {}; 181 (emit LocalGet $($rest:tt)*) => {}; 182 (emit LocalSet $($rest:tt)*) => {}; 183 (emit Call $($rest:tt)*) => {}; 184 (emit End $($rest:tt)*) => {}; 185 (emit Nop $($rest:tt)*) => {}; 186 (emit If $($rest:tt)*) => {}; 187 (emit Else $($rest:tt)*) => {}; 188 (emit Block $($rest:tt)*) => {}; 189 (emit Loop $($rest:tt)*) => {}; 190 (emit Br $($rest:tt)*) => {}; 191 (emit BrIf $($rest:tt)*) => {}; 192 (emit Return $($rest:tt)*) => {}; 193 (emit Unreachable $($rest:tt)*) => {}; 194 (emit LocalTee $($rest:tt)*) => {}; 195 (emit GlobalGet $($rest:tt)*) => {}; 196 (emit GlobalSet $($rest:tt)*) => {}; 197 (emit Select $($rest:tt)*) => {}; 198 (emit Drop $($rest:tt)*) => {}; 199 (emit BrTable $($rest:tt)*) => {}; 200 (emit CallIndirect $($rest:tt)*) => {}; 201 (emit TableInit $($rest:tt)*) => {}; 202 (emit TableCopy $($rest:tt)*) => {}; 203 (emit TableGet $($rest:tt)*) => {}; 204 (emit TableSet $($rest:tt)*) => {}; 205 (emit TableGrow $($rest:tt)*) => {}; 206 (emit TableSize $($rest:tt)*) => {}; 207 (emit TableFill $($rest:tt)*) => {}; 208 (emit ElemDrop $($rest:tt)*) => {}; 209 (emit MemoryInit $($rest:tt)*) => {}; 210 (emit MemoryCopy $($rest:tt)*) => {}; 211 (emit DataDrop $($rest:tt)*) => {}; 212 (emit MemoryFill $($rest:tt)*) => {}; 213 (emit MemorySize $($rest:tt)*) => {}; 214 (emit MemoryGrow $($rest:tt)*) => {}; 215 (emit I32Load $($rest:tt)*) => {}; 216 (emit I32Load8S $($rest:tt)*) => {}; 217 (emit I32Load8U $($rest:tt)*) => {}; 218 (emit I32Load16S $($rest:tt)*) => {}; 219 (emit I32Load16U $($rest:tt)*) => {}; 220 (emit I64Load8S $($rest:tt)*) => {}; 221 (emit I64Load8U $($rest:tt)*) => {}; 222 (emit I64Load16S $($rest:tt)*) => {}; 223 (emit I64Load16U $($rest:tt)*) => {}; 224 (emit I64Load32S $($rest:tt)*) => {}; 225 (emit I64Load32U $($rest:tt)*) => {}; 226 (emit I64Load $($rest:tt)*) => {}; 227 (emit I32Store $($rest:tt)*) => {}; 228 (emit I32Store8 $($rest:tt)*) => {}; 229 (emit I32Store16 $($rest:tt)*) => {}; 230 (emit I64Store $($rest:tt)*) => {}; 231 (emit I64Store8 $($rest:tt)*) => {}; 232 (emit I64Store16 $($rest:tt)*) => {}; 233 (emit I64Store32 $($rest:tt)*) => {}; 234 (emit F32Load $($rest:tt)*) => {}; 235 (emit F32Store $($rest:tt)*) => {}; 236 (emit F64Load $($rest:tt)*) => {}; 237 (emit F64Store $($rest:tt)*) => {}; 238 (emit I32TruncSatF32S $($rest:tt)*) => {}; 239 (emit I32TruncSatF32U $($rest:tt)*) => {}; 240 (emit I32TruncSatF64S $($rest:tt)*) => {}; 241 (emit I32TruncSatF64U $($rest:tt)*) => {}; 242 (emit I64TruncSatF32S $($rest:tt)*) => {}; 243 (emit I64TruncSatF32U $($rest:tt)*) => {}; 244 (emit I64TruncSatF64S $($rest:tt)*) => {}; 245 (emit I64TruncSatF64U $($rest:tt)*) => {}; 246 (emit V128Load $($rest:tt)*) => {}; 247 (emit V128Store $($rest:tt)*) => {}; 248 (emit I64Add128 $($rest:tt)*) => {}; 249 (emit I64Sub128 $($rest:tt)*) => {}; 250 (emit I64MulWideS $($rest:tt)*) => {}; 251 (emit I64MulWideU $($rest:tt)*) => {}; 252 253 (emit $unsupported:tt $($rest:tt)*) => {$($rest)*}; 254 } 255 256 impl<'a, 'translation, 'data, M> VisitOperator<'a> for CodeGen<'a, 'translation, 'data, M, Emission> 257 where 258 M: MacroAssembler, 259 { 260 type Output = (); 261 262 fn visit_i32_const(&mut self, val: i32) { 263 self.context.stack.push(Val::i32(val)); 264 } 265 266 fn visit_i64_const(&mut self, val: i64) { 267 self.context.stack.push(Val::i64(val)); 268 } 269 270 fn visit_f32_const(&mut self, val: Ieee32) { 271 self.context.stack.push(Val::f32(val)); 272 } 273 274 fn visit_f64_const(&mut self, val: Ieee64) { 275 self.context.stack.push(Val::f64(val)); 276 } 277 278 fn visit_f32_add(&mut self) { 279 self.context.binop( 280 self.masm, 281 OperandSize::S32, 282 &mut |masm: &mut M, dst, src, size| { 283 masm.float_add(writable!(dst), dst, src, size); 284 TypedReg::f32(dst) 285 }, 286 ); 287 } 288 289 fn visit_f64_add(&mut self) { 290 self.context.binop( 291 self.masm, 292 OperandSize::S64, 293 &mut |masm: &mut M, dst, src, size| { 294 masm.float_add(writable!(dst), dst, src, size); 295 TypedReg::f64(dst) 296 }, 297 ); 298 } 299 300 fn visit_f32_sub(&mut self) { 301 self.context.binop( 302 self.masm, 303 OperandSize::S32, 304 &mut |masm: &mut M, dst, src, size| { 305 masm.float_sub(writable!(dst), dst, src, size); 306 TypedReg::f32(dst) 307 }, 308 ); 309 } 310 311 fn visit_f64_sub(&mut self) { 312 self.context.binop( 313 self.masm, 314 OperandSize::S64, 315 &mut |masm: &mut M, dst, src, size| { 316 masm.float_sub(writable!(dst), dst, src, size); 317 TypedReg::f64(dst) 318 }, 319 ); 320 } 321 322 fn visit_f32_mul(&mut self) { 323 self.context.binop( 324 self.masm, 325 OperandSize::S32, 326 &mut |masm: &mut M, dst, src, size| { 327 masm.float_mul(writable!(dst), dst, src, size); 328 TypedReg::f32(dst) 329 }, 330 ); 331 } 332 333 fn visit_f64_mul(&mut self) { 334 self.context.binop( 335 self.masm, 336 OperandSize::S64, 337 &mut |masm: &mut M, dst, src, size| { 338 masm.float_mul(writable!(dst), dst, src, size); 339 TypedReg::f64(dst) 340 }, 341 ); 342 } 343 344 fn visit_f32_div(&mut self) { 345 self.context.binop( 346 self.masm, 347 OperandSize::S32, 348 &mut |masm: &mut M, dst, src, size| { 349 masm.float_div(writable!(dst), dst, src, size); 350 TypedReg::f32(dst) 351 }, 352 ); 353 } 354 355 fn visit_f64_div(&mut self) { 356 self.context.binop( 357 self.masm, 358 OperandSize::S64, 359 &mut |masm: &mut M, dst, src, size| { 360 masm.float_div(writable!(dst), dst, src, size); 361 TypedReg::f64(dst) 362 }, 363 ); 364 } 365 366 fn visit_f32_min(&mut self) { 367 self.context.binop( 368 self.masm, 369 OperandSize::S32, 370 &mut |masm: &mut M, dst, src, size| { 371 masm.float_min(writable!(dst), dst, src, size); 372 TypedReg::f32(dst) 373 }, 374 ); 375 } 376 377 fn visit_f64_min(&mut self) { 378 self.context.binop( 379 self.masm, 380 OperandSize::S64, 381 &mut |masm: &mut M, dst, src, size| { 382 masm.float_min(writable!(dst), dst, src, size); 383 TypedReg::f64(dst) 384 }, 385 ); 386 } 387 388 fn visit_f32_max(&mut self) { 389 self.context.binop( 390 self.masm, 391 OperandSize::S32, 392 &mut |masm: &mut M, dst, src, size| { 393 masm.float_max(writable!(dst), dst, src, size); 394 TypedReg::f32(dst) 395 }, 396 ); 397 } 398 399 fn visit_f64_max(&mut self) { 400 self.context.binop( 401 self.masm, 402 OperandSize::S64, 403 &mut |masm: &mut M, dst, src, size| { 404 masm.float_max(writable!(dst), dst, src, size); 405 TypedReg::f64(dst) 406 }, 407 ); 408 } 409 410 fn visit_f32_copysign(&mut self) { 411 self.context.binop( 412 self.masm, 413 OperandSize::S32, 414 &mut |masm: &mut M, dst, src, size| { 415 masm.float_copysign(writable!(dst), dst, src, size); 416 TypedReg::f32(dst) 417 }, 418 ); 419 } 420 421 fn visit_f64_copysign(&mut self) { 422 self.context.binop( 423 self.masm, 424 OperandSize::S64, 425 &mut |masm: &mut M, dst, src, size| { 426 masm.float_copysign(writable!(dst), dst, src, size); 427 TypedReg::f64(dst) 428 }, 429 ); 430 } 431 432 fn visit_f32_abs(&mut self) { 433 self.context 434 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 435 masm.float_abs(writable!(reg), size); 436 TypedReg::f32(reg) 437 }); 438 } 439 440 fn visit_f64_abs(&mut self) { 441 self.context 442 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 443 masm.float_abs(writable!(reg), size); 444 TypedReg::f64(reg) 445 }); 446 } 447 448 fn visit_f32_neg(&mut self) { 449 self.context 450 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 451 masm.float_neg(writable!(reg), size); 452 TypedReg::f32(reg) 453 }); 454 } 455 456 fn visit_f64_neg(&mut self) { 457 self.context 458 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 459 masm.float_neg(writable!(reg), size); 460 TypedReg::f64(reg) 461 }); 462 } 463 464 fn visit_f32_floor(&mut self) { 465 self.masm.float_round( 466 RoundingMode::Down, 467 &mut self.env, 468 &mut self.context, 469 OperandSize::S32, 470 |env, cx, masm| { 471 let builtin = env.builtins.floor_f32::<M::ABI>(); 472 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 473 }, 474 ); 475 } 476 477 fn visit_f64_floor(&mut self) { 478 self.masm.float_round( 479 RoundingMode::Down, 480 &mut self.env, 481 &mut self.context, 482 OperandSize::S64, 483 |env, cx, masm| { 484 let builtin = env.builtins.floor_f64::<M::ABI>(); 485 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 486 }, 487 ); 488 } 489 490 fn visit_f32_ceil(&mut self) { 491 self.masm.float_round( 492 RoundingMode::Up, 493 &mut self.env, 494 &mut self.context, 495 OperandSize::S32, 496 |env, cx, masm| { 497 let builtin = env.builtins.ceil_f32::<M::ABI>(); 498 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 499 }, 500 ); 501 } 502 503 fn visit_f64_ceil(&mut self) { 504 self.masm.float_round( 505 RoundingMode::Up, 506 &mut self.env, 507 &mut self.context, 508 OperandSize::S64, 509 |env, cx, masm| { 510 let builtin = env.builtins.ceil_f64::<M::ABI>(); 511 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 512 }, 513 ); 514 } 515 516 fn visit_f32_nearest(&mut self) { 517 self.masm.float_round( 518 RoundingMode::Nearest, 519 &mut self.env, 520 &mut self.context, 521 OperandSize::S32, 522 |env, cx, masm| { 523 let builtin = env.builtins.nearest_f32::<M::ABI>(); 524 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 525 }, 526 ); 527 } 528 529 fn visit_f64_nearest(&mut self) { 530 self.masm.float_round( 531 RoundingMode::Nearest, 532 &mut self.env, 533 &mut self.context, 534 OperandSize::S64, 535 |env, cx, masm| { 536 let builtin = env.builtins.nearest_f64::<M::ABI>(); 537 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 538 }, 539 ); 540 } 541 542 fn visit_f32_trunc(&mut self) { 543 self.masm.float_round( 544 RoundingMode::Zero, 545 &mut self.env, 546 &mut self.context, 547 OperandSize::S32, 548 |env, cx, masm| { 549 let builtin = env.builtins.trunc_f32::<M::ABI>(); 550 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 551 }, 552 ); 553 } 554 555 fn visit_f64_trunc(&mut self) { 556 self.masm.float_round( 557 RoundingMode::Zero, 558 &mut self.env, 559 &mut self.context, 560 OperandSize::S64, 561 |env, cx, masm| { 562 let builtin = env.builtins.trunc_f64::<M::ABI>(); 563 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 564 }, 565 ); 566 } 567 568 fn visit_f32_sqrt(&mut self) { 569 self.context 570 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 571 masm.float_sqrt(writable!(reg), reg, size); 572 TypedReg::f32(reg) 573 }); 574 } 575 576 fn visit_f64_sqrt(&mut self) { 577 self.context 578 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 579 masm.float_sqrt(writable!(reg), reg, size); 580 TypedReg::f64(reg) 581 }); 582 } 583 584 fn visit_f32_eq(&mut self) { 585 self.context.float_cmp_op( 586 self.masm, 587 OperandSize::S32, 588 &mut |masm: &mut M, dst, src1, src2, size| { 589 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Eq, size); 590 }, 591 ); 592 } 593 594 fn visit_f64_eq(&mut self) { 595 self.context.float_cmp_op( 596 self.masm, 597 OperandSize::S64, 598 &mut |masm: &mut M, dst, src1, src2, size| { 599 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Eq, size); 600 }, 601 ); 602 } 603 604 fn visit_f32_ne(&mut self) { 605 self.context.float_cmp_op( 606 self.masm, 607 OperandSize::S32, 608 &mut |masm: &mut M, dst, src1, src2, size| { 609 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ne, size); 610 }, 611 ); 612 } 613 614 fn visit_f64_ne(&mut self) { 615 self.context.float_cmp_op( 616 self.masm, 617 OperandSize::S64, 618 &mut |masm: &mut M, dst, src1, src2, size| { 619 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ne, size); 620 }, 621 ); 622 } 623 624 fn visit_f32_lt(&mut self) { 625 self.context.float_cmp_op( 626 self.masm, 627 OperandSize::S32, 628 &mut |masm: &mut M, dst, src1, src2, size| { 629 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Lt, size); 630 }, 631 ); 632 } 633 634 fn visit_f64_lt(&mut self) { 635 self.context.float_cmp_op( 636 self.masm, 637 OperandSize::S64, 638 &mut |masm: &mut M, dst, src1, src2, size| { 639 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Lt, size); 640 }, 641 ); 642 } 643 644 fn visit_f32_gt(&mut self) { 645 self.context.float_cmp_op( 646 self.masm, 647 OperandSize::S32, 648 &mut |masm: &mut M, dst, src1, src2, size| { 649 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Gt, size); 650 }, 651 ); 652 } 653 654 fn visit_f64_gt(&mut self) { 655 self.context.float_cmp_op( 656 self.masm, 657 OperandSize::S64, 658 &mut |masm: &mut M, dst, src1, src2, size| { 659 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Gt, size); 660 }, 661 ); 662 } 663 664 fn visit_f32_le(&mut self) { 665 self.context.float_cmp_op( 666 self.masm, 667 OperandSize::S32, 668 &mut |masm: &mut M, dst, src1, src2, size| { 669 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Le, size); 670 }, 671 ); 672 } 673 674 fn visit_f64_le(&mut self) { 675 self.context.float_cmp_op( 676 self.masm, 677 OperandSize::S64, 678 &mut |masm: &mut M, dst, src1, src2, size| { 679 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Le, size); 680 }, 681 ); 682 } 683 684 fn visit_f32_ge(&mut self) { 685 self.context.float_cmp_op( 686 self.masm, 687 OperandSize::S32, 688 &mut |masm: &mut M, dst, src1, src2, size| { 689 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ge, size); 690 }, 691 ); 692 } 693 694 fn visit_f64_ge(&mut self) { 695 self.context.float_cmp_op( 696 self.masm, 697 OperandSize::S64, 698 &mut |masm: &mut M, dst, src1, src2, size| { 699 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ge, size); 700 }, 701 ); 702 } 703 704 fn visit_f32_convert_i32_s(&mut self) { 705 self.context 706 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 707 masm.signed_convert(writable!(dst), src, OperandSize::S32, dst_size); 708 }); 709 } 710 711 fn visit_f32_convert_i32_u(&mut self) { 712 self.context.convert_op_with_tmp_reg( 713 self.masm, 714 WasmValType::F32, 715 RegClass::Int, 716 |masm, dst, src, tmp_gpr, dst_size| { 717 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S32, dst_size); 718 }, 719 ); 720 } 721 722 fn visit_f32_convert_i64_s(&mut self) { 723 self.context 724 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 725 masm.signed_convert(writable!(dst), src, OperandSize::S64, dst_size); 726 }); 727 } 728 729 fn visit_f32_convert_i64_u(&mut self) { 730 self.context.convert_op_with_tmp_reg( 731 self.masm, 732 WasmValType::F32, 733 RegClass::Int, 734 |masm, dst, src, tmp_gpr, dst_size| { 735 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S64, dst_size); 736 }, 737 ); 738 } 739 740 fn visit_f64_convert_i32_s(&mut self) { 741 self.context 742 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 743 masm.signed_convert(writable!(dst), src, OperandSize::S32, dst_size); 744 }); 745 } 746 747 fn visit_f64_convert_i32_u(&mut self) { 748 self.context.convert_op_with_tmp_reg( 749 self.masm, 750 WasmValType::F64, 751 RegClass::Int, 752 |masm, dst, src, tmp_gpr, dst_size| { 753 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S32, dst_size); 754 }, 755 ); 756 } 757 758 fn visit_f64_convert_i64_s(&mut self) { 759 self.context 760 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 761 masm.signed_convert(writable!(dst), src, OperandSize::S64, dst_size); 762 }); 763 } 764 765 fn visit_f64_convert_i64_u(&mut self) { 766 self.context.convert_op_with_tmp_reg( 767 self.masm, 768 WasmValType::F64, 769 RegClass::Int, 770 |masm, dst, src, tmp_gpr, dst_size| { 771 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S64, dst_size); 772 }, 773 ); 774 } 775 776 fn visit_f32_reinterpret_i32(&mut self) { 777 self.context 778 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, size| { 779 masm.reinterpret_int_as_float(writable!(dst), src.into(), size); 780 }); 781 } 782 783 fn visit_f64_reinterpret_i64(&mut self) { 784 self.context 785 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, size| { 786 masm.reinterpret_int_as_float(writable!(dst), src.into(), size); 787 }); 788 } 789 790 fn visit_f32_demote_f64(&mut self) { 791 self.context 792 .unop(self.masm, OperandSize::S64, &mut |masm, reg, _size| { 793 masm.demote(writable!(reg), reg); 794 TypedReg::f32(reg) 795 }); 796 } 797 798 fn visit_f64_promote_f32(&mut self) { 799 self.context 800 .unop(self.masm, OperandSize::S32, &mut |masm, reg, _size| { 801 masm.promote(writable!(reg), reg); 802 TypedReg::f64(reg) 803 }); 804 } 805 806 fn visit_i32_add(&mut self) { 807 self.context.i32_binop(self.masm, |masm, dst, src, size| { 808 masm.add(writable!(dst), dst, src, size); 809 TypedReg::i32(dst) 810 }); 811 } 812 813 fn visit_i64_add(&mut self) { 814 self.context.i64_binop(self.masm, |masm, dst, src, size| { 815 masm.add(writable!(dst), dst, src, size); 816 TypedReg::i64(dst) 817 }); 818 } 819 820 fn visit_i32_sub(&mut self) { 821 self.context.i32_binop(self.masm, |masm, dst, src, size| { 822 masm.sub(writable!(dst), dst, src, size); 823 TypedReg::i32(dst) 824 }); 825 } 826 827 fn visit_i64_sub(&mut self) { 828 self.context.i64_binop(self.masm, |masm, dst, src, size| { 829 masm.sub(writable!(dst), dst, src, size); 830 TypedReg::i64(dst) 831 }); 832 } 833 834 fn visit_i32_mul(&mut self) { 835 self.context.i32_binop(self.masm, |masm, dst, src, size| { 836 masm.mul(writable!(dst), dst, src, size); 837 TypedReg::i32(dst) 838 }); 839 } 840 841 fn visit_i64_mul(&mut self) { 842 self.context.i64_binop(self.masm, |masm, dst, src, size| { 843 masm.mul(writable!(dst), dst, src, size); 844 TypedReg::i64(dst) 845 }); 846 } 847 848 fn visit_i32_div_s(&mut self) { 849 use DivKind::*; 850 use OperandSize::*; 851 852 self.masm.div(&mut self.context, Signed, S32); 853 } 854 855 fn visit_i32_div_u(&mut self) { 856 use DivKind::*; 857 use OperandSize::*; 858 859 self.masm.div(&mut self.context, Unsigned, S32); 860 } 861 862 fn visit_i64_div_s(&mut self) { 863 use DivKind::*; 864 use OperandSize::*; 865 866 self.masm.div(&mut self.context, Signed, S64); 867 } 868 869 fn visit_i64_div_u(&mut self) { 870 use DivKind::*; 871 use OperandSize::*; 872 873 self.masm.div(&mut self.context, Unsigned, S64); 874 } 875 876 fn visit_i32_rem_s(&mut self) { 877 use OperandSize::*; 878 use RemKind::*; 879 880 self.masm.rem(&mut self.context, Signed, S32); 881 } 882 883 fn visit_i32_rem_u(&mut self) { 884 use OperandSize::*; 885 use RemKind::*; 886 887 self.masm.rem(&mut self.context, Unsigned, S32); 888 } 889 890 fn visit_i64_rem_s(&mut self) { 891 use OperandSize::*; 892 use RemKind::*; 893 894 self.masm.rem(&mut self.context, Signed, S64); 895 } 896 897 fn visit_i64_rem_u(&mut self) { 898 use OperandSize::*; 899 use RemKind::*; 900 901 self.masm.rem(&mut self.context, Unsigned, S64); 902 } 903 904 fn visit_i32_eq(&mut self) { 905 self.cmp_i32s(IntCmpKind::Eq); 906 } 907 908 fn visit_i64_eq(&mut self) { 909 self.cmp_i64s(IntCmpKind::Eq); 910 } 911 912 fn visit_i32_ne(&mut self) { 913 self.cmp_i32s(IntCmpKind::Ne); 914 } 915 916 fn visit_i64_ne(&mut self) { 917 self.cmp_i64s(IntCmpKind::Ne); 918 } 919 920 fn visit_i32_lt_s(&mut self) { 921 self.cmp_i32s(IntCmpKind::LtS); 922 } 923 924 fn visit_i64_lt_s(&mut self) { 925 self.cmp_i64s(IntCmpKind::LtS); 926 } 927 928 fn visit_i32_lt_u(&mut self) { 929 self.cmp_i32s(IntCmpKind::LtU); 930 } 931 932 fn visit_i64_lt_u(&mut self) { 933 self.cmp_i64s(IntCmpKind::LtU); 934 } 935 936 fn visit_i32_le_s(&mut self) { 937 self.cmp_i32s(IntCmpKind::LeS); 938 } 939 940 fn visit_i64_le_s(&mut self) { 941 self.cmp_i64s(IntCmpKind::LeS); 942 } 943 944 fn visit_i32_le_u(&mut self) { 945 self.cmp_i32s(IntCmpKind::LeU); 946 } 947 948 fn visit_i64_le_u(&mut self) { 949 self.cmp_i64s(IntCmpKind::LeU); 950 } 951 952 fn visit_i32_gt_s(&mut self) { 953 self.cmp_i32s(IntCmpKind::GtS); 954 } 955 956 fn visit_i64_gt_s(&mut self) { 957 self.cmp_i64s(IntCmpKind::GtS); 958 } 959 960 fn visit_i32_gt_u(&mut self) { 961 self.cmp_i32s(IntCmpKind::GtU); 962 } 963 964 fn visit_i64_gt_u(&mut self) { 965 self.cmp_i64s(IntCmpKind::GtU); 966 } 967 968 fn visit_i32_ge_s(&mut self) { 969 self.cmp_i32s(IntCmpKind::GeS); 970 } 971 972 fn visit_i64_ge_s(&mut self) { 973 self.cmp_i64s(IntCmpKind::GeS); 974 } 975 976 fn visit_i32_ge_u(&mut self) { 977 self.cmp_i32s(IntCmpKind::GeU); 978 } 979 980 fn visit_i64_ge_u(&mut self) { 981 self.cmp_i64s(IntCmpKind::GeU); 982 } 983 984 fn visit_i32_eqz(&mut self) { 985 use OperandSize::*; 986 987 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 988 masm.cmp_with_set(writable!(reg.into()), RegImm::i32(0), IntCmpKind::Eq, size); 989 TypedReg::i32(reg) 990 }); 991 } 992 993 fn visit_i64_eqz(&mut self) { 994 use OperandSize::*; 995 996 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 997 masm.cmp_with_set(writable!(reg.into()), RegImm::i64(0), IntCmpKind::Eq, size); 998 TypedReg::i32(reg) // Return value for `i64.eqz` is an `i32`. 999 }); 1000 } 1001 1002 fn visit_i32_clz(&mut self) { 1003 use OperandSize::*; 1004 1005 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 1006 masm.clz(writable!(reg), reg, size); 1007 TypedReg::i32(reg) 1008 }); 1009 } 1010 1011 fn visit_i64_clz(&mut self) { 1012 use OperandSize::*; 1013 1014 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 1015 masm.clz(writable!(reg), reg, size); 1016 TypedReg::i64(reg) 1017 }); 1018 } 1019 1020 fn visit_i32_ctz(&mut self) { 1021 use OperandSize::*; 1022 1023 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 1024 masm.ctz(writable!(reg), reg, size); 1025 TypedReg::i32(reg) 1026 }); 1027 } 1028 1029 fn visit_i64_ctz(&mut self) { 1030 use OperandSize::*; 1031 1032 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 1033 masm.ctz(writable!(reg), reg, size); 1034 TypedReg::i64(reg) 1035 }); 1036 } 1037 1038 fn visit_i32_and(&mut self) { 1039 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1040 masm.and(writable!(dst), dst, src, size); 1041 TypedReg::i32(dst) 1042 }); 1043 } 1044 1045 fn visit_i64_and(&mut self) { 1046 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1047 masm.and(writable!(dst), dst, src, size); 1048 TypedReg::i64(dst) 1049 }); 1050 } 1051 1052 fn visit_i32_or(&mut self) { 1053 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1054 masm.or(writable!(dst), dst, src, size); 1055 TypedReg::i32(dst) 1056 }); 1057 } 1058 1059 fn visit_i64_or(&mut self) { 1060 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1061 masm.or(writable!(dst), dst, src, size); 1062 TypedReg::i64(dst) 1063 }); 1064 } 1065 1066 fn visit_i32_xor(&mut self) { 1067 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1068 masm.xor(writable!(dst), dst, src, size); 1069 TypedReg::i32(dst) 1070 }); 1071 } 1072 1073 fn visit_i64_xor(&mut self) { 1074 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1075 masm.xor(writable!(dst), dst, src, size); 1076 TypedReg::i64(dst) 1077 }); 1078 } 1079 1080 fn visit_i32_shl(&mut self) { 1081 use ShiftKind::*; 1082 1083 self.context.i32_shift(self.masm, Shl); 1084 } 1085 1086 fn visit_i64_shl(&mut self) { 1087 use ShiftKind::*; 1088 1089 self.context.i64_shift(self.masm, Shl); 1090 } 1091 1092 fn visit_i32_shr_s(&mut self) { 1093 use ShiftKind::*; 1094 1095 self.context.i32_shift(self.masm, ShrS); 1096 } 1097 1098 fn visit_i64_shr_s(&mut self) { 1099 use ShiftKind::*; 1100 1101 self.context.i64_shift(self.masm, ShrS); 1102 } 1103 1104 fn visit_i32_shr_u(&mut self) { 1105 use ShiftKind::*; 1106 1107 self.context.i32_shift(self.masm, ShrU); 1108 } 1109 1110 fn visit_i64_shr_u(&mut self) { 1111 use ShiftKind::*; 1112 1113 self.context.i64_shift(self.masm, ShrU); 1114 } 1115 1116 fn visit_i32_rotl(&mut self) { 1117 use ShiftKind::*; 1118 1119 self.context.i32_shift(self.masm, Rotl); 1120 } 1121 1122 fn visit_i64_rotl(&mut self) { 1123 use ShiftKind::*; 1124 1125 self.context.i64_shift(self.masm, Rotl); 1126 } 1127 1128 fn visit_i32_rotr(&mut self) { 1129 use ShiftKind::*; 1130 1131 self.context.i32_shift(self.masm, Rotr); 1132 } 1133 1134 fn visit_i64_rotr(&mut self) { 1135 use ShiftKind::*; 1136 1137 self.context.i64_shift(self.masm, Rotr); 1138 } 1139 1140 fn visit_end(&mut self) { 1141 if !self.context.reachable { 1142 self.handle_unreachable_end(); 1143 } else { 1144 let mut control = self.control_frames.pop().unwrap(); 1145 control.emit_end(self.masm, &mut self.context); 1146 } 1147 } 1148 1149 fn visit_i32_popcnt(&mut self) { 1150 use OperandSize::*; 1151 self.masm.popcnt(&mut self.context, S32); 1152 } 1153 1154 fn visit_i64_popcnt(&mut self) { 1155 use OperandSize::*; 1156 1157 self.masm.popcnt(&mut self.context, S64); 1158 } 1159 1160 fn visit_i32_wrap_i64(&mut self) { 1161 use OperandSize::*; 1162 1163 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1164 masm.wrap(writable!(reg), reg); 1165 TypedReg::i32(reg) 1166 }); 1167 } 1168 1169 fn visit_i64_extend_i32_s(&mut self) { 1170 use OperandSize::*; 1171 1172 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1173 masm.extend(writable!(reg), reg, ExtendKind::I64ExtendI32S); 1174 TypedReg::i64(reg) 1175 }); 1176 } 1177 1178 fn visit_i64_extend_i32_u(&mut self) { 1179 use OperandSize::*; 1180 1181 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1182 masm.extend(writable!(reg), reg, ExtendKind::I64ExtendI32U); 1183 TypedReg::i64(reg) 1184 }); 1185 } 1186 1187 fn visit_i32_extend8_s(&mut self) { 1188 use OperandSize::*; 1189 1190 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1191 masm.extend(writable!(reg), reg, ExtendKind::I32Extend8S); 1192 TypedReg::i32(reg) 1193 }); 1194 } 1195 1196 fn visit_i32_extend16_s(&mut self) { 1197 use OperandSize::*; 1198 1199 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1200 masm.extend(writable!(reg), reg, ExtendKind::I32Extend16S); 1201 TypedReg::i32(reg) 1202 }); 1203 } 1204 1205 fn visit_i64_extend8_s(&mut self) { 1206 use OperandSize::*; 1207 1208 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1209 masm.extend(writable!(reg), reg, ExtendKind::I64Extend8S); 1210 TypedReg::i64(reg) 1211 }); 1212 } 1213 1214 fn visit_i64_extend16_s(&mut self) { 1215 use OperandSize::*; 1216 1217 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1218 masm.extend(writable!(reg), reg, ExtendKind::I64Extend16S); 1219 TypedReg::i64(reg) 1220 }); 1221 } 1222 1223 fn visit_i64_extend32_s(&mut self) { 1224 use OperandSize::*; 1225 1226 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1227 masm.extend(writable!(reg), reg, ExtendKind::I64Extend32S); 1228 TypedReg::i64(reg) 1229 }); 1230 } 1231 1232 fn visit_i32_trunc_f32_s(&mut self) { 1233 use OperandSize::*; 1234 1235 self.context 1236 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1237 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Unchecked); 1238 }); 1239 } 1240 1241 fn visit_i32_trunc_f32_u(&mut self) { 1242 use OperandSize::*; 1243 1244 self.context.convert_op_with_tmp_reg( 1245 self.masm, 1246 WasmValType::I32, 1247 RegClass::Float, 1248 |masm, dst, src, tmp_fpr, dst_size| { 1249 masm.unsigned_truncate( 1250 writable!(dst), 1251 src, 1252 tmp_fpr, 1253 S32, 1254 dst_size, 1255 TruncKind::Unchecked, 1256 ); 1257 }, 1258 ); 1259 } 1260 1261 fn visit_i32_trunc_f64_s(&mut self) { 1262 use OperandSize::*; 1263 1264 self.context 1265 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1266 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Unchecked); 1267 }); 1268 } 1269 1270 fn visit_i32_trunc_f64_u(&mut self) { 1271 use OperandSize::*; 1272 1273 self.context.convert_op_with_tmp_reg( 1274 self.masm, 1275 WasmValType::I32, 1276 RegClass::Float, 1277 |masm, dst, src, tmp_fpr, dst_size| { 1278 masm.unsigned_truncate( 1279 writable!(dst), 1280 src, 1281 tmp_fpr, 1282 S64, 1283 dst_size, 1284 TruncKind::Unchecked, 1285 ); 1286 }, 1287 ); 1288 } 1289 1290 fn visit_i64_trunc_f32_s(&mut self) { 1291 use OperandSize::*; 1292 1293 self.context 1294 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1295 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Unchecked); 1296 }); 1297 } 1298 1299 fn visit_i64_trunc_f32_u(&mut self) { 1300 use OperandSize::*; 1301 1302 self.context.convert_op_with_tmp_reg( 1303 self.masm, 1304 WasmValType::I64, 1305 RegClass::Float, 1306 |masm, dst, src, tmp_fpr, dst_size| { 1307 masm.unsigned_truncate( 1308 writable!(dst), 1309 src, 1310 tmp_fpr, 1311 S32, 1312 dst_size, 1313 TruncKind::Unchecked, 1314 ); 1315 }, 1316 ); 1317 } 1318 1319 fn visit_i64_trunc_f64_s(&mut self) { 1320 use OperandSize::*; 1321 1322 self.context 1323 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1324 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Unchecked); 1325 }); 1326 } 1327 1328 fn visit_i64_trunc_f64_u(&mut self) { 1329 use OperandSize::*; 1330 1331 self.context.convert_op_with_tmp_reg( 1332 self.masm, 1333 WasmValType::I64, 1334 RegClass::Float, 1335 |masm, dst, src, tmp_fpr, dst_size| { 1336 masm.unsigned_truncate( 1337 writable!(dst), 1338 src, 1339 tmp_fpr, 1340 S64, 1341 dst_size, 1342 TruncKind::Unchecked, 1343 ); 1344 }, 1345 ); 1346 } 1347 1348 fn visit_i32_reinterpret_f32(&mut self) { 1349 self.context 1350 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, size| { 1351 masm.reinterpret_float_as_int(writable!(dst), src.into(), size); 1352 }); 1353 } 1354 1355 fn visit_i64_reinterpret_f64(&mut self) { 1356 self.context 1357 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, size| { 1358 masm.reinterpret_float_as_int(writable!(dst), src.into(), size); 1359 }); 1360 } 1361 1362 fn visit_local_get(&mut self, index: u32) { 1363 use WasmValType::*; 1364 let context = &mut self.context; 1365 let slot = context.frame.get_wasm_local(index); 1366 match slot.ty { 1367 I32 | I64 | F32 | F64 | V128 => context.stack.push(Val::local(index, slot.ty)), 1368 Ref(rt) => match rt.heap_type { 1369 WasmHeapType::Func => context.stack.push(Val::local(index, slot.ty)), 1370 WasmHeapType::Extern => { 1371 self.found_unsupported_instruction = 1372 Some("unsupported local.get of externref local"); 1373 } 1374 ht => unimplemented!("Support for WasmHeapType: {ht}"), 1375 }, 1376 } 1377 } 1378 1379 fn visit_local_set(&mut self, index: u32) { 1380 let src = self.emit_set_local(index); 1381 self.context.free_reg(src); 1382 } 1383 1384 fn visit_call(&mut self, index: u32) { 1385 let callee = self.env.callee_from_index(FuncIndex::from_u32(index)); 1386 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee) 1387 } 1388 1389 fn visit_call_indirect(&mut self, type_index: u32, table_index: u32) { 1390 // Spill now because `emit_lazy_init_funcref` and the `FnCall::emit` 1391 // invocations will both trigger spills since they both call functions. 1392 // However, the machine instructions for the spill emitted by 1393 // `emit_lazy_funcref` will be jumped over if the funcref was previously 1394 // initialized which may result in the machine stack becoming 1395 // unbalanced. 1396 self.context.spill(self.masm); 1397 1398 let type_index = TypeIndex::from_u32(type_index); 1399 let table_index = TableIndex::from_u32(table_index); 1400 1401 self.emit_lazy_init_funcref(table_index); 1402 1403 // Perform the indirect call. 1404 // This code assumes that [`Self::emit_lazy_init_funcref`] will 1405 // push the funcref to the value stack. 1406 let funcref_ptr = self.context.stack.peek().map(|v| v.unwrap_reg()).unwrap(); 1407 self.masm 1408 .trapz(funcref_ptr.into(), TRAP_INDIRECT_CALL_TO_NULL); 1409 self.emit_typecheck_funcref(funcref_ptr.into(), type_index); 1410 1411 let callee = self.env.funcref(type_index); 1412 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee) 1413 } 1414 1415 fn visit_table_init(&mut self, elem: u32, table: u32) { 1416 debug_assert!(self.context.stack.len() >= 3); 1417 let at = self.context.stack.len() - 3; 1418 1419 self.context 1420 .stack 1421 .insert_many(at, &[table.try_into().unwrap(), elem.try_into().unwrap()]); 1422 1423 let builtin = self.env.builtins.table_init::<M::ABI, M::Ptr>(); 1424 FnCall::emit::<M>( 1425 &mut self.env, 1426 self.masm, 1427 &mut self.context, 1428 Callee::Builtin(builtin.clone()), 1429 ); 1430 self.context.pop_and_free(self.masm); 1431 } 1432 1433 fn visit_table_copy(&mut self, dst: u32, src: u32) { 1434 debug_assert!(self.context.stack.len() >= 3); 1435 let at = self.context.stack.len() - 3; 1436 self.context 1437 .stack 1438 .insert_many(at, &[dst.try_into().unwrap(), src.try_into().unwrap()]); 1439 1440 let builtin = self.env.builtins.table_copy::<M::ABI, M::Ptr>(); 1441 FnCall::emit::<M>( 1442 &mut self.env, 1443 self.masm, 1444 &mut self.context, 1445 Callee::Builtin(builtin), 1446 ); 1447 self.context.pop_and_free(self.masm); 1448 } 1449 1450 fn visit_table_get(&mut self, table: u32) { 1451 let table_index = TableIndex::from_u32(table); 1452 let table = self.env.table(table_index); 1453 let heap_type = table.ref_type.heap_type; 1454 1455 match heap_type { 1456 WasmHeapType::Func => self.emit_lazy_init_funcref(table_index), 1457 WasmHeapType::Extern => { 1458 self.found_unsupported_instruction = 1459 Some("unsupported table.get of externref table"); 1460 } 1461 t => { 1462 unimplemented!("Support for WasmHeapType: {t}") 1463 } 1464 } 1465 } 1466 1467 fn visit_table_grow(&mut self, table: u32) { 1468 let table_index = TableIndex::from_u32(table); 1469 let table_ty = self.env.table(table_index); 1470 let builtin = match table_ty.ref_type.heap_type { 1471 WasmHeapType::Func => self.env.builtins.table_grow_func_ref::<M::ABI, M::Ptr>(), 1472 ty => unimplemented!("Support for HeapType: {ty}"), 1473 }; 1474 1475 let len = self.context.stack.len(); 1476 // table.grow` requires at least 2 elements on the value stack. 1477 debug_assert!(len >= 2); 1478 let at = len - 2; 1479 1480 // The table_grow builtin expects the parameters in a different 1481 // order. 1482 // The value stack at this point should contain: 1483 // [ init_value | delta ] (stack top) 1484 // but the builtin function expects the init value as the last 1485 // argument. 1486 self.context.stack.inner_mut().swap(len - 1, len - 2); 1487 self.context 1488 .stack 1489 .insert_many(at, &[table.try_into().unwrap()]); 1490 1491 FnCall::emit::<M>( 1492 &mut self.env, 1493 self.masm, 1494 &mut self.context, 1495 Callee::Builtin(builtin.clone()), 1496 ); 1497 } 1498 1499 fn visit_table_size(&mut self, table: u32) { 1500 let table_index = TableIndex::from_u32(table); 1501 let table_data = self.env.resolve_table_data(table_index); 1502 self.emit_compute_table_size(&table_data); 1503 } 1504 1505 fn visit_table_fill(&mut self, table: u32) { 1506 let table_index = TableIndex::from_u32(table); 1507 let table_ty = self.env.table(table_index); 1508 let builtin = match table_ty.ref_type.heap_type { 1509 WasmHeapType::Func => self.env.builtins.table_fill_func_ref::<M::ABI, M::Ptr>(), 1510 ty => unimplemented!("Support for heap type: {ty}"), 1511 }; 1512 1513 let len = self.context.stack.len(); 1514 debug_assert!(len >= 3); 1515 let at = len - 3; 1516 self.context 1517 .stack 1518 .insert_many(at, &[table.try_into().unwrap()]); 1519 FnCall::emit::<M>( 1520 &mut self.env, 1521 self.masm, 1522 &mut self.context, 1523 Callee::Builtin(builtin.clone()), 1524 ); 1525 self.context.pop_and_free(self.masm); 1526 } 1527 1528 fn visit_table_set(&mut self, table: u32) { 1529 let ptr_type = self.env.ptr_type(); 1530 let table_index = TableIndex::from_u32(table); 1531 let table_data = self.env.resolve_table_data(table_index); 1532 let table = self.env.table(table_index); 1533 match table.ref_type.heap_type { 1534 WasmHeapType::Func => { 1535 assert!( 1536 self.tunables.table_lazy_init, 1537 "unsupported table eager init" 1538 ); 1539 let value = self.context.pop_to_reg(self.masm, None); 1540 let index = self.context.pop_to_reg(self.masm, None); 1541 let base = self.context.any_gpr(self.masm); 1542 let elem_addr = self.emit_compute_table_elem_addr(index.into(), base, &table_data); 1543 // Set the initialized bit. 1544 self.masm.or( 1545 writable!(value.into()), 1546 value.into(), 1547 RegImm::i64(FUNCREF_INIT_BIT as i64), 1548 ptr_type.into(), 1549 ); 1550 1551 self.masm.store_ptr(value.into(), elem_addr); 1552 1553 self.context.free_reg(value); 1554 self.context.free_reg(index); 1555 self.context.free_reg(base); 1556 } 1557 ty => unimplemented!("Support for WasmHeapType: {ty}"), 1558 }; 1559 } 1560 1561 fn visit_elem_drop(&mut self, index: u32) { 1562 let elem_drop = self.env.builtins.elem_drop::<M::ABI, M::Ptr>(); 1563 self.context.stack.extend([index.try_into().unwrap()]); 1564 FnCall::emit::<M>( 1565 &mut self.env, 1566 self.masm, 1567 &mut self.context, 1568 Callee::Builtin(elem_drop), 1569 ) 1570 } 1571 1572 fn visit_memory_init(&mut self, data_index: u32, mem: u32) { 1573 debug_assert!(self.context.stack.len() >= 3); 1574 let at = self.context.stack.len() - 3; 1575 self.context.stack.insert_many( 1576 at, 1577 &[mem.try_into().unwrap(), data_index.try_into().unwrap()], 1578 ); 1579 let builtin = self.env.builtins.memory_init::<M::ABI, M::Ptr>(); 1580 FnCall::emit::<M>( 1581 &mut self.env, 1582 self.masm, 1583 &mut self.context, 1584 Callee::Builtin(builtin), 1585 ); 1586 self.context.pop_and_free(self.masm); 1587 } 1588 1589 fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) { 1590 // At this point, the stack is expected to contain: 1591 // [ dst_offset, src_offset, len ] 1592 // The following code inserts the missing params, so that stack contains: 1593 // [ vmctx, dst_mem, dst_offset, src_mem, src_offset, len ] 1594 // Which is the order expected by the builtin function. 1595 debug_assert!(self.context.stack.len() >= 3); 1596 let at = self.context.stack.len() - 2; 1597 self.context 1598 .stack 1599 .insert_many(at, &[src_mem.try_into().unwrap()]); 1600 1601 // One element was inserted above, so instead of 3, we use 4. 1602 let at = self.context.stack.len() - 4; 1603 self.context 1604 .stack 1605 .insert_many(at, &[dst_mem.try_into().unwrap()]); 1606 1607 let builtin = self.env.builtins.memory_copy::<M::ABI, M::Ptr>(); 1608 1609 FnCall::emit::<M>( 1610 &mut self.env, 1611 self.masm, 1612 &mut self.context, 1613 Callee::Builtin(builtin), 1614 ); 1615 self.context.pop_and_free(self.masm); 1616 } 1617 1618 fn visit_memory_fill(&mut self, mem: u32) { 1619 debug_assert!(self.context.stack.len() >= 3); 1620 let at = self.context.stack.len() - 3; 1621 1622 self.context 1623 .stack 1624 .insert_many(at, &[mem.try_into().unwrap()]); 1625 1626 let builtin = self.env.builtins.memory_fill::<M::ABI, M::Ptr>(); 1627 FnCall::emit::<M>( 1628 &mut self.env, 1629 self.masm, 1630 &mut self.context, 1631 Callee::Builtin(builtin), 1632 ); 1633 self.context.pop_and_free(self.masm); 1634 } 1635 1636 fn visit_memory_size(&mut self, mem: u32) { 1637 let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem)); 1638 self.emit_compute_memory_size(&heap); 1639 } 1640 1641 fn visit_memory_grow(&mut self, mem: u32) { 1642 debug_assert!(self.context.stack.len() >= 1); 1643 // The stack at this point contains: [ delta ] 1644 // The desired state is 1645 // [ vmctx, delta, index ] 1646 self.context.stack.extend([mem.try_into().unwrap()]); 1647 1648 let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem)); 1649 let builtin = self.env.builtins.memory32_grow::<M::ABI, M::Ptr>(); 1650 FnCall::emit::<M>( 1651 &mut self.env, 1652 self.masm, 1653 &mut self.context, 1654 Callee::Builtin(builtin), 1655 ); 1656 1657 // The memory32_grow builtin returns a pointer type, therefore we must 1658 // ensure that the return type is representative of the address space of 1659 // the heap type. 1660 match (self.env.ptr_type(), heap.index_type()) { 1661 (WasmValType::I64, WasmValType::I64) => {} 1662 // When the heap type is smaller than the pointer type, we adjust 1663 // the result of the memory32_grow builtin. 1664 (WasmValType::I64, WasmValType::I32) => { 1665 let top: Reg = self.context.pop_to_reg(self.masm, None).into(); 1666 self.masm.wrap(writable!(top.into()), top.into()); 1667 self.context.stack.push(TypedReg::i32(top).into()); 1668 } 1669 _ => unimplemented!("Support for 32-bit platforms"), 1670 } 1671 } 1672 1673 fn visit_data_drop(&mut self, data_index: u32) { 1674 self.context.stack.extend([data_index.try_into().unwrap()]); 1675 1676 let builtin = self.env.builtins.data_drop::<M::ABI, M::Ptr>(); 1677 FnCall::emit::<M>( 1678 &mut self.env, 1679 self.masm, 1680 &mut self.context, 1681 Callee::Builtin(builtin), 1682 ) 1683 } 1684 1685 fn visit_nop(&mut self) {} 1686 1687 fn visit_if(&mut self, blockty: BlockType) { 1688 self.control_frames.push(ControlStackFrame::r#if( 1689 self.env.resolve_block_sig(blockty), 1690 self.masm, 1691 &mut self.context, 1692 )); 1693 } 1694 1695 fn visit_else(&mut self) { 1696 if !self.context.reachable { 1697 self.handle_unreachable_else(); 1698 } else { 1699 let control = self 1700 .control_frames 1701 .last_mut() 1702 .unwrap_or_else(|| panic!("Expected active control stack frame for else")); 1703 control.emit_else(self.masm, &mut self.context); 1704 } 1705 } 1706 1707 fn visit_block(&mut self, blockty: BlockType) { 1708 self.control_frames.push(ControlStackFrame::block( 1709 self.env.resolve_block_sig(blockty), 1710 self.masm, 1711 &mut self.context, 1712 )); 1713 } 1714 1715 fn visit_loop(&mut self, blockty: BlockType) { 1716 self.control_frames.push(ControlStackFrame::r#loop( 1717 self.env.resolve_block_sig(blockty), 1718 self.masm, 1719 &mut self.context, 1720 )); 1721 1722 // Emit fuel check right after binding the loop header. 1723 if self.tunables.consume_fuel { 1724 self.emit_fuel_check(); 1725 } 1726 } 1727 1728 fn visit_br(&mut self, depth: u32) { 1729 let index = control_index(depth, self.control_frames.len()); 1730 let frame = &mut self.control_frames[index]; 1731 self.context 1732 .unconditional_jump(frame, self.masm, |masm, cx, frame| { 1733 frame 1734 .pop_abi_results::<M, _>(cx, masm, |results, _, _| results.ret_area().copied()); 1735 }); 1736 } 1737 1738 fn visit_br_if(&mut self, depth: u32) { 1739 let index = control_index(depth, self.control_frames.len()); 1740 let frame = &mut self.control_frames[index]; 1741 frame.set_as_target(); 1742 1743 let top = { 1744 let top = self.context.without::<TypedReg, M, _>( 1745 frame.results::<M>().regs(), 1746 self.masm, 1747 |ctx, masm| ctx.pop_to_reg(masm, None), 1748 ); 1749 // Explicitly save any live registers and locals before setting up 1750 // the branch state. 1751 // In some cases, calculating the `top` value above, will result in 1752 // a spill, thus the following one will result in a no-op. 1753 self.context.spill(self.masm); 1754 frame.top_abi_results::<M, _>( 1755 &mut self.context, 1756 self.masm, 1757 |results, context, masm| { 1758 // In the case of `br_if` there's a possibility that we'll 1759 // exit early from the block or fallthrough, for 1760 // a fallthrough, we cannot rely on the pre-computed return area; 1761 // it must be recalculated so that any values that are 1762 // generated are correctly placed near the current stack 1763 // pointer. 1764 results.on_stack().then(|| { 1765 let stack_consumed = context.stack.sizeof(results.stack_operands_len()); 1766 let base = masm.sp_offset().as_u32() - stack_consumed; 1767 let offs = base + results.size(); 1768 RetArea::sp(SPOffset::from_u32(offs)) 1769 }) 1770 }, 1771 ); 1772 top 1773 }; 1774 1775 // Emit instructions to balance the machine stack if the frame has 1776 // a different offset. 1777 let current_sp_offset = self.masm.sp_offset(); 1778 let results_size = frame.results::<M>().size(); 1779 let state = frame.stack_state(); 1780 let (label, cmp, needs_cleanup) = if current_sp_offset > state.target_offset { 1781 (self.masm.get_label(), IntCmpKind::Eq, true) 1782 } else { 1783 (*frame.label(), IntCmpKind::Ne, false) 1784 }; 1785 1786 self.masm 1787 .branch(cmp, top.reg.into(), top.reg.into(), label, OperandSize::S32); 1788 self.context.free_reg(top); 1789 1790 if needs_cleanup { 1791 // Emit instructions to balance the stack and jump if not falling 1792 // through. 1793 self.masm.memmove( 1794 current_sp_offset, 1795 state.target_offset, 1796 results_size, 1797 MemMoveDirection::LowToHigh, 1798 ); 1799 self.masm.ensure_sp_for_jump(state.target_offset); 1800 self.masm.jmp(*frame.label()); 1801 1802 // Restore sp_offset to what it was for falling through and emit 1803 // fallthrough label. 1804 self.masm.reset_stack_pointer(current_sp_offset); 1805 self.masm.bind(label); 1806 } 1807 } 1808 1809 fn visit_br_table(&mut self, targets: BrTable<'a>) { 1810 // +1 to account for the default target. 1811 let len = targets.len() + 1; 1812 // SmallVec<[_; 5]> to match the binary emission layer (e.g 1813 // see `JmpTableSeq'), but here we use 5 instead since we 1814 // bundle the default target as the last element in the array. 1815 let labels: SmallVec<[_; 5]> = (0..len).map(|_| self.masm.get_label()).collect(); 1816 1817 let default_index = control_index(targets.default(), self.control_frames.len()); 1818 let default_frame = &mut self.control_frames[default_index]; 1819 let default_result = default_frame.results::<M>(); 1820 1821 let (index, tmp) = { 1822 let index_and_tmp = self.context.without::<(TypedReg, _), M, _>( 1823 default_result.regs(), 1824 self.masm, 1825 |cx, masm| (cx.pop_to_reg(masm, None), cx.any_gpr(masm)), 1826 ); 1827 1828 // Materialize any constants or locals into their result representation, 1829 // so that when reachability is restored, they are correctly located. 1830 default_frame.top_abi_results::<M, _>(&mut self.context, self.masm, |results, _, _| { 1831 results.ret_area().copied() 1832 }); 1833 index_and_tmp 1834 }; 1835 1836 self.masm.jmp_table(&labels, index.into(), tmp); 1837 // Save the original stack pointer offset; we will reset the stack 1838 // pointer to this offset after jumping to each of the targets. Each 1839 // jump might adjust the stack according to the base offset of the 1840 // target. 1841 let current_sp = self.masm.sp_offset(); 1842 1843 for (t, l) in targets 1844 .targets() 1845 .into_iter() 1846 .chain(std::iter::once(Ok(targets.default()))) 1847 .zip(labels.iter()) 1848 { 1849 let control_index = control_index(t.unwrap(), self.control_frames.len()); 1850 let frame = &mut self.control_frames[control_index]; 1851 // Reset the stack pointer to its original offset. This is needed 1852 // because each jump will potentially adjust the stack pointer 1853 // according to the base offset of the target. 1854 self.masm.reset_stack_pointer(current_sp); 1855 1856 // NB: We don't perform any result handling as it was 1857 // already taken care of above before jumping to the 1858 // jump table. 1859 self.masm.bind(*l); 1860 // Ensure that the stack pointer is correctly positioned before 1861 // jumping to the jump table code. 1862 let state = frame.stack_state(); 1863 self.masm.ensure_sp_for_jump(state.target_offset); 1864 self.masm.jmp(*frame.label()); 1865 frame.set_as_target(); 1866 } 1867 // Finally reset the stack pointer to the original location. 1868 // The reachability analysis, will ensure it's correctly located 1869 // once reachability is restored. 1870 self.masm.reset_stack_pointer(current_sp); 1871 self.context.reachable = false; 1872 self.context.free_reg(index.reg); 1873 self.context.free_reg(tmp); 1874 } 1875 1876 fn visit_return(&mut self) { 1877 // Grab the outermost frame, which is the function's body 1878 // frame. We don't rely on [`codegen::control_index`] since 1879 // this frame is implicit and we know that it should exist at 1880 // index 0. 1881 let outermost = &mut self.control_frames[0]; 1882 self.context 1883 .unconditional_jump(outermost, self.masm, |masm, cx, frame| { 1884 frame 1885 .pop_abi_results::<M, _>(cx, masm, |results, _, _| results.ret_area().copied()); 1886 }); 1887 } 1888 1889 fn visit_unreachable(&mut self) { 1890 self.masm.unreachable(); 1891 self.context.reachable = false; 1892 // Set the implicit outermost frame as target to perform the necessary 1893 // stack clean up. 1894 let outermost = &mut self.control_frames[0]; 1895 outermost.set_as_target(); 1896 } 1897 1898 fn visit_local_tee(&mut self, index: u32) { 1899 let typed_reg = self.emit_set_local(index); 1900 self.context.stack.push(typed_reg.into()); 1901 } 1902 1903 fn visit_global_get(&mut self, global_index: u32) { 1904 let index = GlobalIndex::from_u32(global_index); 1905 let (ty, addr) = self.emit_get_global_addr(index); 1906 let dst = self.context.reg_for_type(ty, self.masm); 1907 self.masm.load(addr, writable!(dst), ty.into()); 1908 self.context.stack.push(Val::reg(dst, ty)); 1909 } 1910 1911 fn visit_global_set(&mut self, global_index: u32) { 1912 let index = GlobalIndex::from_u32(global_index); 1913 let (ty, addr) = self.emit_get_global_addr(index); 1914 1915 let typed_reg = self.context.pop_to_reg(self.masm, None); 1916 self.context.free_reg(typed_reg.reg); 1917 self.masm.store(typed_reg.reg.into(), addr, ty.into()); 1918 } 1919 1920 fn visit_drop(&mut self) { 1921 self.context.drop_last(1, |regalloc, val| match val { 1922 Val::Reg(tr) => regalloc.free(tr.reg.into()), 1923 Val::Memory(m) => self.masm.free_stack(m.slot.size), 1924 _ => {} 1925 }); 1926 } 1927 1928 fn visit_select(&mut self) { 1929 let cond = self.context.pop_to_reg(self.masm, None); 1930 let val2 = self.context.pop_to_reg(self.masm, None); 1931 let val1 = self.context.pop_to_reg(self.masm, None); 1932 self.masm 1933 .cmp(cond.reg.into(), RegImm::i32(0), OperandSize::S32); 1934 // Conditionally move val1 to val2 if the comparison is 1935 // not zero. 1936 self.masm.cmov( 1937 writable!(val2.into()), 1938 val1.into(), 1939 IntCmpKind::Ne, 1940 val1.ty.into(), 1941 ); 1942 self.context.stack.push(val2.into()); 1943 self.context.free_reg(val1.reg); 1944 self.context.free_reg(cond); 1945 } 1946 1947 fn visit_i32_load(&mut self, memarg: MemArg) { 1948 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S32, None); 1949 } 1950 1951 fn visit_i32_load8_s(&mut self, memarg: MemArg) { 1952 self.emit_wasm_load( 1953 &memarg, 1954 WasmValType::I32, 1955 OperandSize::S8, 1956 Some(ExtendKind::I32Extend8S), 1957 ); 1958 } 1959 1960 fn visit_i32_load8_u(&mut self, memarg: MemArg) { 1961 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S8, None); 1962 } 1963 1964 fn visit_i32_load16_s(&mut self, memarg: MemArg) { 1965 self.emit_wasm_load( 1966 &memarg, 1967 WasmValType::I32, 1968 OperandSize::S16, 1969 Some(ExtendKind::I32Extend16S), 1970 ) 1971 } 1972 1973 fn visit_i32_load16_u(&mut self, memarg: MemArg) { 1974 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S16, None) 1975 } 1976 1977 fn visit_i32_store(&mut self, memarg: MemArg) { 1978 self.emit_wasm_store(&memarg, OperandSize::S32); 1979 } 1980 1981 fn visit_i32_store8(&mut self, memarg: MemArg) { 1982 self.emit_wasm_store(&memarg, OperandSize::S8) 1983 } 1984 1985 fn visit_i32_store16(&mut self, memarg: MemArg) { 1986 self.emit_wasm_store(&memarg, OperandSize::S16) 1987 } 1988 1989 fn visit_i64_load8_s(&mut self, memarg: MemArg) { 1990 self.emit_wasm_load( 1991 &memarg, 1992 WasmValType::I64, 1993 OperandSize::S8, 1994 Some(ExtendKind::I64Extend8S), 1995 ) 1996 } 1997 1998 fn visit_i64_load8_u(&mut self, memarg: MemArg) { 1999 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S8, None) 2000 } 2001 2002 fn visit_i64_load16_u(&mut self, memarg: MemArg) { 2003 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S16, None) 2004 } 2005 2006 fn visit_i64_load16_s(&mut self, memarg: MemArg) { 2007 self.emit_wasm_load( 2008 &memarg, 2009 WasmValType::I64, 2010 OperandSize::S16, 2011 Some(ExtendKind::I64Extend16S), 2012 ) 2013 } 2014 2015 fn visit_i64_load32_u(&mut self, memarg: MemArg) { 2016 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S32, None) 2017 } 2018 2019 fn visit_i64_load32_s(&mut self, memarg: MemArg) { 2020 self.emit_wasm_load( 2021 &memarg, 2022 WasmValType::I64, 2023 OperandSize::S32, 2024 Some(ExtendKind::I64Extend32S), 2025 ) 2026 } 2027 2028 fn visit_i64_load(&mut self, memarg: MemArg) { 2029 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S64, None) 2030 } 2031 2032 fn visit_i64_store(&mut self, memarg: MemArg) -> Self::Output { 2033 self.emit_wasm_store(&memarg, OperandSize::S64) 2034 } 2035 2036 fn visit_i64_store8(&mut self, memarg: MemArg) -> Self::Output { 2037 self.emit_wasm_store(&memarg, OperandSize::S8) 2038 } 2039 2040 fn visit_i64_store16(&mut self, memarg: MemArg) -> Self::Output { 2041 self.emit_wasm_store(&memarg, OperandSize::S16) 2042 } 2043 2044 fn visit_i64_store32(&mut self, memarg: MemArg) -> Self::Output { 2045 self.emit_wasm_store(&memarg, OperandSize::S32) 2046 } 2047 2048 fn visit_f32_load(&mut self, memarg: MemArg) { 2049 self.emit_wasm_load(&memarg, WasmValType::F32, OperandSize::S32, None) 2050 } 2051 2052 fn visit_f32_store(&mut self, memarg: MemArg) { 2053 self.emit_wasm_store(&memarg, OperandSize::S32) 2054 } 2055 2056 fn visit_f64_load(&mut self, memarg: MemArg) { 2057 self.emit_wasm_load(&memarg, WasmValType::F64, OperandSize::S64, None) 2058 } 2059 2060 fn visit_f64_store(&mut self, memarg: MemArg) { 2061 self.emit_wasm_store(&memarg, OperandSize::S64) 2062 } 2063 2064 fn visit_i32_trunc_sat_f32_s(&mut self) { 2065 use OperandSize::*; 2066 2067 self.context 2068 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2069 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Checked); 2070 }); 2071 } 2072 2073 fn visit_i32_trunc_sat_f32_u(&mut self) { 2074 use OperandSize::*; 2075 2076 self.context.convert_op_with_tmp_reg( 2077 self.masm, 2078 WasmValType::I32, 2079 RegClass::Float, 2080 |masm, dst, src, tmp_fpr, dst_size| { 2081 masm.unsigned_truncate( 2082 writable!(dst), 2083 src, 2084 tmp_fpr, 2085 S32, 2086 dst_size, 2087 TruncKind::Checked, 2088 ); 2089 }, 2090 ); 2091 } 2092 2093 fn visit_i32_trunc_sat_f64_s(&mut self) { 2094 use OperandSize::*; 2095 2096 self.context 2097 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2098 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Checked); 2099 }); 2100 } 2101 2102 fn visit_i32_trunc_sat_f64_u(&mut self) { 2103 use OperandSize::*; 2104 2105 self.context.convert_op_with_tmp_reg( 2106 self.masm, 2107 WasmValType::I32, 2108 RegClass::Float, 2109 |masm, dst, src, tmp_fpr, dst_size| { 2110 masm.unsigned_truncate( 2111 writable!(dst), 2112 src, 2113 tmp_fpr, 2114 S64, 2115 dst_size, 2116 TruncKind::Checked, 2117 ); 2118 }, 2119 ); 2120 } 2121 2122 fn visit_i64_trunc_sat_f32_s(&mut self) { 2123 use OperandSize::*; 2124 2125 self.context 2126 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2127 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Checked); 2128 }); 2129 } 2130 2131 fn visit_i64_trunc_sat_f32_u(&mut self) { 2132 use OperandSize::*; 2133 2134 self.context.convert_op_with_tmp_reg( 2135 self.masm, 2136 WasmValType::I64, 2137 RegClass::Float, 2138 |masm, dst, src, tmp_fpr, dst_size| { 2139 masm.unsigned_truncate( 2140 writable!(dst), 2141 src, 2142 tmp_fpr, 2143 S32, 2144 dst_size, 2145 TruncKind::Checked, 2146 ); 2147 }, 2148 ); 2149 } 2150 2151 fn visit_i64_trunc_sat_f64_s(&mut self) { 2152 use OperandSize::*; 2153 2154 self.context 2155 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2156 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Checked); 2157 }); 2158 } 2159 2160 fn visit_i64_trunc_sat_f64_u(&mut self) { 2161 use OperandSize::*; 2162 2163 self.context.convert_op_with_tmp_reg( 2164 self.masm, 2165 WasmValType::I64, 2166 RegClass::Float, 2167 |masm, dst, src, tmp_fpr, dst_size| { 2168 masm.unsigned_truncate( 2169 writable!(dst), 2170 src, 2171 tmp_fpr, 2172 S64, 2173 dst_size, 2174 TruncKind::Checked, 2175 ); 2176 }, 2177 ); 2178 } 2179 2180 fn visit_i64_add128(&mut self) { 2181 self.context 2182 .binop128(self.masm, |masm, lhs_lo, lhs_hi, rhs_lo, rhs_hi| { 2183 masm.add128( 2184 writable!(lhs_lo), 2185 writable!(lhs_hi), 2186 lhs_lo, 2187 lhs_hi, 2188 rhs_lo, 2189 rhs_hi, 2190 ); 2191 (TypedReg::i64(lhs_lo), TypedReg::i64(lhs_hi)) 2192 }); 2193 } 2194 2195 fn visit_i64_sub128(&mut self) { 2196 self.context 2197 .binop128(self.masm, |masm, lhs_lo, lhs_hi, rhs_lo, rhs_hi| { 2198 masm.sub128( 2199 writable!(lhs_lo), 2200 writable!(lhs_hi), 2201 lhs_lo, 2202 lhs_hi, 2203 rhs_lo, 2204 rhs_hi, 2205 ); 2206 (TypedReg::i64(lhs_lo), TypedReg::i64(lhs_hi)) 2207 }); 2208 } 2209 2210 fn visit_i64_mul_wide_s(&mut self) { 2211 self.masm.mul_wide(&mut self.context, MulWideKind::Signed); 2212 } 2213 2214 fn visit_i64_mul_wide_u(&mut self) { 2215 self.masm.mul_wide(&mut self.context, MulWideKind::Unsigned); 2216 } 2217 2218 wasmparser::for_each_visit_operator!(def_unsupported); 2219 } 2220 2221 impl<'a, 'translation, 'data, M> VisitSimdOperator<'a> 2222 for CodeGen<'a, 'translation, 'data, M, Emission> 2223 where 2224 M: MacroAssembler, 2225 { 2226 fn visit_v128_const(&mut self, val: V128) { 2227 self.context.stack.push(Val::v128(val.i128())) 2228 } 2229 2230 fn visit_v128_load(&mut self, memarg: MemArg) { 2231 self.emit_wasm_load(&memarg, WasmValType::V128, OperandSize::S128, None) 2232 } 2233 2234 fn visit_v128_store(&mut self, memarg: MemArg) { 2235 self.emit_wasm_store(&memarg, OperandSize::S128) 2236 } 2237 2238 wasmparser::for_each_visit_simd_operator!(def_unsupported); 2239 } 2240 2241 impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Emission> 2242 where 2243 M: MacroAssembler, 2244 { 2245 fn cmp_i32s(&mut self, kind: IntCmpKind) { 2246 self.context.i32_binop(self.masm, |masm, dst, src, size| { 2247 masm.cmp_with_set(writable!(dst), src, kind, size); 2248 TypedReg::i32(dst) 2249 }); 2250 } 2251 2252 fn cmp_i64s(&mut self, kind: IntCmpKind) { 2253 self.context 2254 .i64_binop(self.masm, move |masm, dst, src, size| { 2255 masm.cmp_with_set(writable!(dst), src, kind, size); 2256 TypedReg::i32(dst) // Return value for comparisons is an `i32`. 2257 }); 2258 } 2259 } 2260 2261 impl From<WasmValType> for OperandSize { 2262 fn from(ty: WasmValType) -> OperandSize { 2263 match ty { 2264 WasmValType::I32 | WasmValType::F32 => OperandSize::S32, 2265 WasmValType::I64 | WasmValType::F64 => OperandSize::S64, 2266 WasmValType::V128 => OperandSize::S128, 2267 WasmValType::Ref(rt) => { 2268 match rt.heap_type { 2269 // TODO: Hardcoded size, assuming 64-bit support only. Once 2270 // Wasmtime supports 32-bit architectures, this will need 2271 // to be updated in such a way that the calculation of the 2272 // OperandSize will depend on the target's pointer size. 2273 WasmHeapType::Func => OperandSize::S64, 2274 WasmHeapType::Extern => OperandSize::S64, 2275 t => unimplemented!("Support for WasmHeapType: {t}"), 2276 } 2277 } 2278 } 2279 } 2280 } 2281