1 //! This module is the central place for machine code emission. 2 //! It defines an implementation of wasmparser's Visitor trait 3 //! for `CodeGen`; which defines a visitor per op-code, 4 //! which validates and dispatches to the corresponding 5 //! machine code emitter. 6 7 use crate::abi::RetArea; 8 use crate::codegen::{control_index, Callee, CodeGen, ControlStackFrame, FnCall}; 9 use crate::masm::{ 10 DivKind, ExtendKind, FloatCmpKind, IntCmpKind, MacroAssembler, MemMoveDirection, OperandSize, 11 RegImm, RemKind, RoundingMode, SPOffset, ShiftKind, TruncKind, 12 }; 13 use crate::reg::Reg; 14 use crate::stack::{TypedReg, Val}; 15 use cranelift_codegen::ir::TrapCode; 16 use regalloc2::RegClass; 17 use smallvec::SmallVec; 18 use wasmparser::{BlockType, BrTable, Ieee32, Ieee64, MemArg, VisitOperator}; 19 use wasmtime_environ::{ 20 FuncIndex, GlobalIndex, MemoryIndex, TableIndex, TableStyle, TypeIndex, WasmHeapType, 21 WasmValType, FUNCREF_INIT_BIT, 22 }; 23 24 /// A macro to define unsupported WebAssembly operators. 25 /// 26 /// This macro calls itself recursively; 27 /// 1. It no-ops when matching a supported operator. 28 /// 2. Defines the visitor function and panics when 29 /// matching an unsupported operator. 30 macro_rules! def_unsupported { 31 ($( @$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident)*) => { 32 $( 33 def_unsupported!( 34 emit 35 $op 36 37 fn $visit(&mut self $($(,$arg: $argty)*)?) -> Self::Output { 38 $($(let _ = $arg;)*)? 39 todo!(stringify!($op)) 40 } 41 ); 42 )* 43 }; 44 45 (emit I32Const $($rest:tt)*) => {}; 46 (emit I64Const $($rest:tt)*) => {}; 47 (emit F32Const $($rest:tt)*) => {}; 48 (emit F64Const $($rest:tt)*) => {}; 49 (emit F32Add $($rest:tt)*) => {}; 50 (emit F64Add $($rest:tt)*) => {}; 51 (emit F32Sub $($rest:tt)*) => {}; 52 (emit F64Sub $($rest:tt)*) => {}; 53 (emit F32Mul $($rest:tt)*) => {}; 54 (emit F64Mul $($rest:tt)*) => {}; 55 (emit F32Div $($rest:tt)*) => {}; 56 (emit F64Div $($rest:tt)*) => {}; 57 (emit F32Min $($rest:tt)*) => {}; 58 (emit F64Min $($rest:tt)*) => {}; 59 (emit F32Max $($rest:tt)*) => {}; 60 (emit F64Max $($rest:tt)*) => {}; 61 (emit F32Copysign $($rest:tt)*) => {}; 62 (emit F64Copysign $($rest:tt)*) => {}; 63 (emit F32Abs $($rest:tt)*) => {}; 64 (emit F64Abs $($rest:tt)*) => {}; 65 (emit F32Neg $($rest:tt)*) => {}; 66 (emit F64Neg $($rest:tt)*) => {}; 67 (emit F32Floor $($rest:tt)*) => {}; 68 (emit F64Floor $($rest:tt)*) => {}; 69 (emit F32Ceil $($rest:tt)*) => {}; 70 (emit F64Ceil $($rest:tt)*) => {}; 71 (emit F32Nearest $($rest:tt)*) => {}; 72 (emit F64Nearest $($rest:tt)*) => {}; 73 (emit F32Trunc $($rest:tt)*) => {}; 74 (emit F64Trunc $($rest:tt)*) => {}; 75 (emit F32Sqrt $($rest:tt)*) => {}; 76 (emit F64Sqrt $($rest:tt)*) => {}; 77 (emit F32Eq $($rest:tt)*) => {}; 78 (emit F64Eq $($rest:tt)*) => {}; 79 (emit F32Ne $($rest:tt)*) => {}; 80 (emit F64Ne $($rest:tt)*) => {}; 81 (emit F32Lt $($rest:tt)*) => {}; 82 (emit F64Lt $($rest:tt)*) => {}; 83 (emit F32Gt $($rest:tt)*) => {}; 84 (emit F64Gt $($rest:tt)*) => {}; 85 (emit F32Le $($rest:tt)*) => {}; 86 (emit F64Le $($rest:tt)*) => {}; 87 (emit F32Ge $($rest:tt)*) => {}; 88 (emit F64Ge $($rest:tt)*) => {}; 89 (emit F32ConvertI32S $($rest:tt)*) => {}; 90 (emit F32ConvertI32U $($rest:tt)*) => {}; 91 (emit F32ConvertI64S $($rest:tt)*) => {}; 92 (emit F32ConvertI64U $($rest:tt)*) => {}; 93 (emit F64ConvertI32S $($rest:tt)*) => {}; 94 (emit F64ConvertI32U $($rest:tt)*) => {}; 95 (emit F64ConvertI64S $($rest:tt)*) => {}; 96 (emit F64ConvertI64U $($rest:tt)*) => {}; 97 (emit F32ReinterpretI32 $($rest:tt)*) => {}; 98 (emit F64ReinterpretI64 $($rest:tt)*) => {}; 99 (emit F32DemoteF64 $($rest:tt)*) => {}; 100 (emit F64PromoteF32 $($rest:tt)*) => {}; 101 (emit I32Add $($rest:tt)*) => {}; 102 (emit I64Add $($rest:tt)*) => {}; 103 (emit I32Sub $($rest:tt)*) => {}; 104 (emit I32Mul $($rest:tt)*) => {}; 105 (emit I32DivS $($rest:tt)*) => {}; 106 (emit I32DivU $($rest:tt)*) => {}; 107 (emit I64DivS $($rest:tt)*) => {}; 108 (emit I64DivU $($rest:tt)*) => {}; 109 (emit I64RemU $($rest:tt)*) => {}; 110 (emit I64RemS $($rest:tt)*) => {}; 111 (emit I32RemU $($rest:tt)*) => {}; 112 (emit I32RemS $($rest:tt)*) => {}; 113 (emit I64Mul $($rest:tt)*) => {}; 114 (emit I64Sub $($rest:tt)*) => {}; 115 (emit I32Eq $($rest:tt)*) => {}; 116 (emit I64Eq $($rest:tt)*) => {}; 117 (emit I32Ne $($rest:tt)*) => {}; 118 (emit I64Ne $($rest:tt)*) => {}; 119 (emit I32LtS $($rest:tt)*) => {}; 120 (emit I64LtS $($rest:tt)*) => {}; 121 (emit I32LtU $($rest:tt)*) => {}; 122 (emit I64LtU $($rest:tt)*) => {}; 123 (emit I32LeS $($rest:tt)*) => {}; 124 (emit I64LeS $($rest:tt)*) => {}; 125 (emit I32LeU $($rest:tt)*) => {}; 126 (emit I64LeU $($rest:tt)*) => {}; 127 (emit I32GtS $($rest:tt)*) => {}; 128 (emit I64GtS $($rest:tt)*) => {}; 129 (emit I32GtU $($rest:tt)*) => {}; 130 (emit I64GtU $($rest:tt)*) => {}; 131 (emit I32GeS $($rest:tt)*) => {}; 132 (emit I64GeS $($rest:tt)*) => {}; 133 (emit I32GeU $($rest:tt)*) => {}; 134 (emit I64GeU $($rest:tt)*) => {}; 135 (emit I32Eqz $($rest:tt)*) => {}; 136 (emit I64Eqz $($rest:tt)*) => {}; 137 (emit I32And $($rest:tt)*) => {}; 138 (emit I64And $($rest:tt)*) => {}; 139 (emit I32Or $($rest:tt)*) => {}; 140 (emit I64Or $($rest:tt)*) => {}; 141 (emit I32Xor $($rest:tt)*) => {}; 142 (emit I64Xor $($rest:tt)*) => {}; 143 (emit I32Shl $($rest:tt)*) => {}; 144 (emit I64Shl $($rest:tt)*) => {}; 145 (emit I32ShrS $($rest:tt)*) => {}; 146 (emit I64ShrS $($rest:tt)*) => {}; 147 (emit I32ShrU $($rest:tt)*) => {}; 148 (emit I64ShrU $($rest:tt)*) => {}; 149 (emit I32Rotl $($rest:tt)*) => {}; 150 (emit I64Rotl $($rest:tt)*) => {}; 151 (emit I32Rotr $($rest:tt)*) => {}; 152 (emit I64Rotr $($rest:tt)*) => {}; 153 (emit I32Clz $($rest:tt)*) => {}; 154 (emit I64Clz $($rest:tt)*) => {}; 155 (emit I32Ctz $($rest:tt)*) => {}; 156 (emit I64Ctz $($rest:tt)*) => {}; 157 (emit I32Popcnt $($rest:tt)*) => {}; 158 (emit I64Popcnt $($rest:tt)*) => {}; 159 (emit I32WrapI64 $($rest:tt)*) => {}; 160 (emit I64ExtendI32S $($rest:tt)*) => {}; 161 (emit I64ExtendI32U $($rest:tt)*) => {}; 162 (emit I32Extend8S $($rest:tt)*) => {}; 163 (emit I32Extend16S $($rest:tt)*) => {}; 164 (emit I64Extend8S $($rest:tt)*) => {}; 165 (emit I64Extend16S $($rest:tt)*) => {}; 166 (emit I64Extend32S $($rest:tt)*) => {}; 167 (emit I32TruncF32S $($rest:tt)*) => {}; 168 (emit I32TruncF32U $($rest:tt)*) => {}; 169 (emit I32TruncF64S $($rest:tt)*) => {}; 170 (emit I32TruncF64U $($rest:tt)*) => {}; 171 (emit I64TruncF32S $($rest:tt)*) => {}; 172 (emit I64TruncF32U $($rest:tt)*) => {}; 173 (emit I64TruncF64S $($rest:tt)*) => {}; 174 (emit I64TruncF64U $($rest:tt)*) => {}; 175 (emit I32ReinterpretF32 $($rest:tt)*) => {}; 176 (emit I64ReinterpretF64 $($rest:tt)*) => {}; 177 (emit LocalGet $($rest:tt)*) => {}; 178 (emit LocalSet $($rest:tt)*) => {}; 179 (emit Call $($rest:tt)*) => {}; 180 (emit End $($rest:tt)*) => {}; 181 (emit Nop $($rest:tt)*) => {}; 182 (emit If $($rest:tt)*) => {}; 183 (emit Else $($rest:tt)*) => {}; 184 (emit Block $($rest:tt)*) => {}; 185 (emit Loop $($rest:tt)*) => {}; 186 (emit Br $($rest:tt)*) => {}; 187 (emit BrIf $($rest:tt)*) => {}; 188 (emit Return $($rest:tt)*) => {}; 189 (emit Unreachable $($rest:tt)*) => {}; 190 (emit LocalTee $($rest:tt)*) => {}; 191 (emit GlobalGet $($rest:tt)*) => {}; 192 (emit GlobalSet $($rest:tt)*) => {}; 193 (emit Select $($rest:tt)*) => {}; 194 (emit Drop $($rest:tt)*) => {}; 195 (emit BrTable $($rest:tt)*) => {}; 196 (emit CallIndirect $($rest:tt)*) => {}; 197 (emit TableInit $($rest:tt)*) => {}; 198 (emit TableCopy $($rest:tt)*) => {}; 199 (emit TableGet $($rest:tt)*) => {}; 200 (emit TableSet $($rest:tt)*) => {}; 201 (emit TableGrow $($rest:tt)*) => {}; 202 (emit TableSize $($rest:tt)*) => {}; 203 (emit TableFill $($rest:tt)*) => {}; 204 (emit ElemDrop $($rest:tt)*) => {}; 205 (emit MemoryInit $($rest:tt)*) => {}; 206 (emit MemoryCopy $($rest:tt)*) => {}; 207 (emit DataDrop $($rest:tt)*) => {}; 208 (emit MemoryFill $($rest:tt)*) => {}; 209 (emit MemorySize $($rest:tt)*) => {}; 210 (emit MemoryGrow $($rest:tt)*) => {}; 211 (emit I32Load $($rest:tt)*) => {}; 212 (emit I32Load8S $($rest:tt)*) => {}; 213 (emit I32Load8U $($rest:tt)*) => {}; 214 (emit I32Load16S $($rest:tt)*) => {}; 215 (emit I32Load16U $($rest:tt)*) => {}; 216 (emit I64Load8S $($rest:tt)*) => {}; 217 (emit I64Load8U $($rest:tt)*) => {}; 218 (emit I64Load16S $($rest:tt)*) => {}; 219 (emit I64Load16U $($rest:tt)*) => {}; 220 (emit I64Load32S $($rest:tt)*) => {}; 221 (emit I64Load32U $($rest:tt)*) => {}; 222 (emit I64Load $($rest:tt)*) => {}; 223 (emit I32Store $($rest:tt)*) => {}; 224 (emit I32Store $($rest:tt)*) => {}; 225 (emit I32Store8 $($rest:tt)*) => {}; 226 (emit I32Store16 $($rest:tt)*) => {}; 227 (emit I64Store $($rest:tt)*) => {}; 228 (emit I64Store8 $($rest:tt)*) => {}; 229 (emit I64Store16 $($rest:tt)*) => {}; 230 (emit I64Store32 $($rest:tt)*) => {}; 231 (emit F32Load $($rest:tt)*) => {}; 232 (emit F32Store $($rest:tt)*) => {}; 233 (emit F64Load $($rest:tt)*) => {}; 234 (emit F64Store $($rest:tt)*) => {}; 235 (emit I32TruncSatF32S $($rest:tt)*) => {}; 236 (emit I32TruncSatF32U $($rest:tt)*) => {}; 237 (emit I32TruncSatF64S $($rest:tt)*) => {}; 238 (emit I32TruncSatF64U $($rest:tt)*) => {}; 239 (emit I64TruncSatF32S $($rest:tt)*) => {}; 240 (emit I64TruncSatF32U $($rest:tt)*) => {}; 241 (emit I64TruncSatF64S $($rest:tt)*) => {}; 242 (emit I64TruncSatF64U $($rest:tt)*) => {}; 243 244 245 (emit $unsupported:tt $($rest:tt)*) => {$($rest)*}; 246 } 247 248 impl<'a, 'translation, 'data, M> VisitOperator<'a> for CodeGen<'a, 'translation, 'data, M> 249 where 250 M: MacroAssembler, 251 { 252 type Output = (); 253 254 fn visit_i32_const(&mut self, val: i32) { 255 self.context.stack.push(Val::i32(val)); 256 } 257 258 fn visit_i64_const(&mut self, val: i64) { 259 self.context.stack.push(Val::i64(val)); 260 } 261 262 fn visit_f32_const(&mut self, val: Ieee32) { 263 self.context.stack.push(Val::f32(val)); 264 } 265 266 fn visit_f64_const(&mut self, val: Ieee64) { 267 self.context.stack.push(Val::f64(val)); 268 } 269 270 fn visit_f32_add(&mut self) { 271 self.context.binop( 272 self.masm, 273 OperandSize::S32, 274 &mut |masm: &mut M, dst, src, size| { 275 masm.float_add(dst, dst, src, size); 276 TypedReg::f32(dst) 277 }, 278 ); 279 } 280 281 fn visit_f64_add(&mut self) { 282 self.context.binop( 283 self.masm, 284 OperandSize::S64, 285 &mut |masm: &mut M, dst, src, size| { 286 masm.float_add(dst, dst, src, size); 287 TypedReg::f64(dst) 288 }, 289 ); 290 } 291 292 fn visit_f32_sub(&mut self) { 293 self.context.binop( 294 self.masm, 295 OperandSize::S32, 296 &mut |masm: &mut M, dst, src, size| { 297 masm.float_sub(dst, dst, src, size); 298 TypedReg::f32(dst) 299 }, 300 ); 301 } 302 303 fn visit_f64_sub(&mut self) { 304 self.context.binop( 305 self.masm, 306 OperandSize::S64, 307 &mut |masm: &mut M, dst, src, size| { 308 masm.float_sub(dst, dst, src, size); 309 TypedReg::f64(dst) 310 }, 311 ); 312 } 313 314 fn visit_f32_mul(&mut self) { 315 self.context.binop( 316 self.masm, 317 OperandSize::S32, 318 &mut |masm: &mut M, dst, src, size| { 319 masm.float_mul(dst, dst, src, size); 320 TypedReg::f32(dst) 321 }, 322 ); 323 } 324 325 fn visit_f64_mul(&mut self) { 326 self.context.binop( 327 self.masm, 328 OperandSize::S64, 329 &mut |masm: &mut M, dst, src, size| { 330 masm.float_mul(dst, dst, src, size); 331 TypedReg::f64(dst) 332 }, 333 ); 334 } 335 336 fn visit_f32_div(&mut self) { 337 self.context.binop( 338 self.masm, 339 OperandSize::S32, 340 &mut |masm: &mut M, dst, src, size| { 341 masm.float_div(dst, dst, src, size); 342 TypedReg::f32(dst) 343 }, 344 ); 345 } 346 347 fn visit_f64_div(&mut self) { 348 self.context.binop( 349 self.masm, 350 OperandSize::S64, 351 &mut |masm: &mut M, dst, src, size| { 352 masm.float_div(dst, dst, src, size); 353 TypedReg::f64(dst) 354 }, 355 ); 356 } 357 358 fn visit_f32_min(&mut self) { 359 self.context.binop( 360 self.masm, 361 OperandSize::S32, 362 &mut |masm: &mut M, dst, src, size| { 363 masm.float_min(dst, dst, src, size); 364 TypedReg::f32(dst) 365 }, 366 ); 367 } 368 369 fn visit_f64_min(&mut self) { 370 self.context.binop( 371 self.masm, 372 OperandSize::S64, 373 &mut |masm: &mut M, dst, src, size| { 374 masm.float_min(dst, dst, src, size); 375 TypedReg::f64(dst) 376 }, 377 ); 378 } 379 380 fn visit_f32_max(&mut self) { 381 self.context.binop( 382 self.masm, 383 OperandSize::S32, 384 &mut |masm: &mut M, dst, src, size| { 385 masm.float_max(dst, dst, src, size); 386 TypedReg::f32(dst) 387 }, 388 ); 389 } 390 391 fn visit_f64_max(&mut self) { 392 self.context.binop( 393 self.masm, 394 OperandSize::S64, 395 &mut |masm: &mut M, dst, src, size| { 396 masm.float_max(dst, dst, src, size); 397 TypedReg::f64(dst) 398 }, 399 ); 400 } 401 402 fn visit_f32_copysign(&mut self) { 403 self.context.binop( 404 self.masm, 405 OperandSize::S32, 406 &mut |masm: &mut M, dst, src, size| { 407 masm.float_copysign(dst, dst, src, size); 408 TypedReg::f32(dst) 409 }, 410 ); 411 } 412 413 fn visit_f64_copysign(&mut self) { 414 self.context.binop( 415 self.masm, 416 OperandSize::S64, 417 &mut |masm: &mut M, dst, src, size| { 418 masm.float_copysign(dst, dst, src, size); 419 TypedReg::f64(dst) 420 }, 421 ); 422 } 423 424 fn visit_f32_abs(&mut self) { 425 self.context 426 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 427 masm.float_abs(reg, size); 428 TypedReg::f32(reg) 429 }); 430 } 431 432 fn visit_f64_abs(&mut self) { 433 self.context 434 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 435 masm.float_abs(reg, size); 436 TypedReg::f64(reg) 437 }); 438 } 439 440 fn visit_f32_neg(&mut self) { 441 self.context 442 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 443 masm.float_neg(reg, size); 444 TypedReg::f32(reg) 445 }); 446 } 447 448 fn visit_f64_neg(&mut self) { 449 self.context 450 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 451 masm.float_neg(reg, size); 452 TypedReg::f64(reg) 453 }); 454 } 455 456 fn visit_f32_floor(&mut self) { 457 self.masm.float_round( 458 RoundingMode::Down, 459 &mut self.env, 460 &mut self.context, 461 OperandSize::S32, 462 |env, cx, masm| { 463 let builtin = env.builtins.floor_f32::<M::ABI>(); 464 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 465 }, 466 ); 467 } 468 469 fn visit_f64_floor(&mut self) { 470 self.masm.float_round( 471 RoundingMode::Down, 472 &mut self.env, 473 &mut self.context, 474 OperandSize::S64, 475 |env, cx, masm| { 476 let builtin = env.builtins.floor_f64::<M::ABI>(); 477 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 478 }, 479 ); 480 } 481 482 fn visit_f32_ceil(&mut self) { 483 self.masm.float_round( 484 RoundingMode::Up, 485 &mut self.env, 486 &mut self.context, 487 OperandSize::S32, 488 |env, cx, masm| { 489 let builtin = env.builtins.ceil_f32::<M::ABI>(); 490 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 491 }, 492 ); 493 } 494 495 fn visit_f64_ceil(&mut self) { 496 self.masm.float_round( 497 RoundingMode::Up, 498 &mut self.env, 499 &mut self.context, 500 OperandSize::S64, 501 |env, cx, masm| { 502 let builtin = env.builtins.ceil_f64::<M::ABI>(); 503 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 504 }, 505 ); 506 } 507 508 fn visit_f32_nearest(&mut self) { 509 self.masm.float_round( 510 RoundingMode::Nearest, 511 &mut self.env, 512 &mut self.context, 513 OperandSize::S32, 514 |env, cx, masm| { 515 let builtin = env.builtins.nearest_f32::<M::ABI>(); 516 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 517 }, 518 ); 519 } 520 521 fn visit_f64_nearest(&mut self) { 522 self.masm.float_round( 523 RoundingMode::Nearest, 524 &mut self.env, 525 &mut self.context, 526 OperandSize::S64, 527 |env, cx, masm| { 528 let builtin = env.builtins.nearest_f64::<M::ABI>(); 529 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 530 }, 531 ); 532 } 533 534 fn visit_f32_trunc(&mut self) { 535 self.masm.float_round( 536 RoundingMode::Zero, 537 &mut self.env, 538 &mut self.context, 539 OperandSize::S32, 540 |env, cx, masm| { 541 let builtin = env.builtins.trunc_f32::<M::ABI>(); 542 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 543 }, 544 ); 545 } 546 547 fn visit_f64_trunc(&mut self) { 548 self.masm.float_round( 549 RoundingMode::Zero, 550 &mut self.env, 551 &mut self.context, 552 OperandSize::S64, 553 |env, cx, masm| { 554 let builtin = env.builtins.trunc_f64::<M::ABI>(); 555 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 556 }, 557 ); 558 } 559 560 fn visit_f32_sqrt(&mut self) { 561 self.context 562 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 563 masm.float_sqrt(reg, reg, size); 564 TypedReg::f32(reg) 565 }); 566 } 567 568 fn visit_f64_sqrt(&mut self) { 569 self.context 570 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 571 masm.float_sqrt(reg, reg, size); 572 TypedReg::f64(reg) 573 }); 574 } 575 576 fn visit_f32_eq(&mut self) { 577 self.context.float_cmp_op( 578 self.masm, 579 OperandSize::S32, 580 &mut |masm: &mut M, dst, src1, src2, size| { 581 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Eq, size); 582 }, 583 ); 584 } 585 586 fn visit_f64_eq(&mut self) { 587 self.context.float_cmp_op( 588 self.masm, 589 OperandSize::S64, 590 &mut |masm: &mut M, dst, src1, src2, size| { 591 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Eq, size); 592 }, 593 ); 594 } 595 596 fn visit_f32_ne(&mut self) { 597 self.context.float_cmp_op( 598 self.masm, 599 OperandSize::S32, 600 &mut |masm: &mut M, dst, src1, src2, size| { 601 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Ne, size); 602 }, 603 ); 604 } 605 606 fn visit_f64_ne(&mut self) { 607 self.context.float_cmp_op( 608 self.masm, 609 OperandSize::S64, 610 &mut |masm: &mut M, dst, src1, src2, size| { 611 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Ne, size); 612 }, 613 ); 614 } 615 616 fn visit_f32_lt(&mut self) { 617 self.context.float_cmp_op( 618 self.masm, 619 OperandSize::S32, 620 &mut |masm: &mut M, dst, src1, src2, size| { 621 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Lt, size); 622 }, 623 ); 624 } 625 626 fn visit_f64_lt(&mut self) { 627 self.context.float_cmp_op( 628 self.masm, 629 OperandSize::S64, 630 &mut |masm: &mut M, dst, src1, src2, size| { 631 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Lt, size); 632 }, 633 ); 634 } 635 636 fn visit_f32_gt(&mut self) { 637 self.context.float_cmp_op( 638 self.masm, 639 OperandSize::S32, 640 &mut |masm: &mut M, dst, src1, src2, size| { 641 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Gt, size); 642 }, 643 ); 644 } 645 646 fn visit_f64_gt(&mut self) { 647 self.context.float_cmp_op( 648 self.masm, 649 OperandSize::S64, 650 &mut |masm: &mut M, dst, src1, src2, size| { 651 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Gt, size); 652 }, 653 ); 654 } 655 656 fn visit_f32_le(&mut self) { 657 self.context.float_cmp_op( 658 self.masm, 659 OperandSize::S32, 660 &mut |masm: &mut M, dst, src1, src2, size| { 661 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Le, size); 662 }, 663 ); 664 } 665 666 fn visit_f64_le(&mut self) { 667 self.context.float_cmp_op( 668 self.masm, 669 OperandSize::S64, 670 &mut |masm: &mut M, dst, src1, src2, size| { 671 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Le, size); 672 }, 673 ); 674 } 675 676 fn visit_f32_ge(&mut self) { 677 self.context.float_cmp_op( 678 self.masm, 679 OperandSize::S32, 680 &mut |masm: &mut M, dst, src1, src2, size| { 681 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Ge, size); 682 }, 683 ); 684 } 685 686 fn visit_f64_ge(&mut self) { 687 self.context.float_cmp_op( 688 self.masm, 689 OperandSize::S64, 690 &mut |masm: &mut M, dst, src1, src2, size| { 691 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Ge, size); 692 }, 693 ); 694 } 695 696 fn visit_f32_convert_i32_s(&mut self) { 697 self.context 698 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 699 masm.signed_convert(src, dst, OperandSize::S32, dst_size); 700 }); 701 } 702 703 fn visit_f32_convert_i32_u(&mut self) { 704 self.context.convert_op_with_tmp_reg( 705 self.masm, 706 WasmValType::F32, 707 RegClass::Int, 708 |masm, dst, src, tmp_gpr, dst_size| { 709 masm.unsigned_convert(src, dst, tmp_gpr, OperandSize::S32, dst_size); 710 }, 711 ); 712 } 713 714 fn visit_f32_convert_i64_s(&mut self) { 715 self.context 716 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 717 masm.signed_convert(src, dst, OperandSize::S64, dst_size); 718 }); 719 } 720 721 fn visit_f32_convert_i64_u(&mut self) { 722 self.context.convert_op_with_tmp_reg( 723 self.masm, 724 WasmValType::F32, 725 RegClass::Int, 726 |masm, dst, src, tmp_gpr, dst_size| { 727 masm.unsigned_convert(src, dst, tmp_gpr, OperandSize::S64, dst_size); 728 }, 729 ); 730 } 731 732 fn visit_f64_convert_i32_s(&mut self) { 733 self.context 734 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 735 masm.signed_convert(src, dst, OperandSize::S32, dst_size); 736 }); 737 } 738 739 fn visit_f64_convert_i32_u(&mut self) { 740 self.context.convert_op_with_tmp_reg( 741 self.masm, 742 WasmValType::F64, 743 RegClass::Int, 744 |masm, dst, src, tmp_gpr, dst_size| { 745 masm.unsigned_convert(src, dst, tmp_gpr, OperandSize::S32, dst_size); 746 }, 747 ); 748 } 749 750 fn visit_f64_convert_i64_s(&mut self) { 751 self.context 752 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 753 masm.signed_convert(src, dst, OperandSize::S64, dst_size); 754 }); 755 } 756 757 fn visit_f64_convert_i64_u(&mut self) { 758 self.context.convert_op_with_tmp_reg( 759 self.masm, 760 WasmValType::F64, 761 RegClass::Int, 762 |masm, dst, src, tmp_gpr, dst_size| { 763 masm.unsigned_convert(src, dst, tmp_gpr, OperandSize::S64, dst_size); 764 }, 765 ); 766 } 767 768 fn visit_f32_reinterpret_i32(&mut self) { 769 self.context 770 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, size| { 771 masm.reinterpret_int_as_float(src.into(), dst, size); 772 }); 773 } 774 775 fn visit_f64_reinterpret_i64(&mut self) { 776 self.context 777 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, size| { 778 masm.reinterpret_int_as_float(src.into(), dst, size); 779 }); 780 } 781 782 fn visit_f32_demote_f64(&mut self) { 783 self.context 784 .unop(self.masm, OperandSize::S64, &mut |masm, reg, _size| { 785 masm.demote(reg, reg); 786 TypedReg::f32(reg) 787 }); 788 } 789 790 fn visit_f64_promote_f32(&mut self) { 791 self.context 792 .unop(self.masm, OperandSize::S32, &mut |masm, reg, _size| { 793 masm.promote(reg, reg); 794 TypedReg::f64(reg) 795 }); 796 } 797 798 fn visit_i32_add(&mut self) { 799 self.context.i32_binop(self.masm, |masm, dst, src, size| { 800 masm.add(dst, dst, src, size); 801 TypedReg::i32(dst) 802 }); 803 } 804 805 fn visit_i64_add(&mut self) { 806 self.context.i64_binop(self.masm, |masm, dst, src, size| { 807 masm.add(dst, dst, src, size); 808 TypedReg::i64(dst) 809 }); 810 } 811 812 fn visit_i32_sub(&mut self) { 813 self.context.i32_binop(self.masm, |masm, dst, src, size| { 814 masm.sub(dst, dst, src, size); 815 TypedReg::i32(dst) 816 }); 817 } 818 819 fn visit_i64_sub(&mut self) { 820 self.context.i64_binop(self.masm, |masm, dst, src, size| { 821 masm.sub(dst, dst, src, size); 822 TypedReg::i64(dst) 823 }); 824 } 825 826 fn visit_i32_mul(&mut self) { 827 self.context.i32_binop(self.masm, |masm, dst, src, size| { 828 masm.mul(dst, dst, src, size); 829 TypedReg::i32(dst) 830 }); 831 } 832 833 fn visit_i64_mul(&mut self) { 834 self.context.i64_binop(self.masm, |masm, dst, src, size| { 835 masm.mul(dst, dst, src, size); 836 TypedReg::i64(dst) 837 }); 838 } 839 840 fn visit_i32_div_s(&mut self) { 841 use DivKind::*; 842 use OperandSize::*; 843 844 self.masm.div(&mut self.context, Signed, S32); 845 } 846 847 fn visit_i32_div_u(&mut self) { 848 use DivKind::*; 849 use OperandSize::*; 850 851 self.masm.div(&mut self.context, Unsigned, S32); 852 } 853 854 fn visit_i64_div_s(&mut self) { 855 use DivKind::*; 856 use OperandSize::*; 857 858 self.masm.div(&mut self.context, Signed, S64); 859 } 860 861 fn visit_i64_div_u(&mut self) { 862 use DivKind::*; 863 use OperandSize::*; 864 865 self.masm.div(&mut self.context, Unsigned, S64); 866 } 867 868 fn visit_i32_rem_s(&mut self) { 869 use OperandSize::*; 870 use RemKind::*; 871 872 self.masm.rem(&mut self.context, Signed, S32); 873 } 874 875 fn visit_i32_rem_u(&mut self) { 876 use OperandSize::*; 877 use RemKind::*; 878 879 self.masm.rem(&mut self.context, Unsigned, S32); 880 } 881 882 fn visit_i64_rem_s(&mut self) { 883 use OperandSize::*; 884 use RemKind::*; 885 886 self.masm.rem(&mut self.context, Signed, S64); 887 } 888 889 fn visit_i64_rem_u(&mut self) { 890 use OperandSize::*; 891 use RemKind::*; 892 893 self.masm.rem(&mut self.context, Unsigned, S64); 894 } 895 896 fn visit_i32_eq(&mut self) { 897 self.cmp_i32s(IntCmpKind::Eq); 898 } 899 900 fn visit_i64_eq(&mut self) { 901 self.cmp_i64s(IntCmpKind::Eq); 902 } 903 904 fn visit_i32_ne(&mut self) { 905 self.cmp_i32s(IntCmpKind::Ne); 906 } 907 908 fn visit_i64_ne(&mut self) { 909 self.cmp_i64s(IntCmpKind::Ne); 910 } 911 912 fn visit_i32_lt_s(&mut self) { 913 self.cmp_i32s(IntCmpKind::LtS); 914 } 915 916 fn visit_i64_lt_s(&mut self) { 917 self.cmp_i64s(IntCmpKind::LtS); 918 } 919 920 fn visit_i32_lt_u(&mut self) { 921 self.cmp_i32s(IntCmpKind::LtU); 922 } 923 924 fn visit_i64_lt_u(&mut self) { 925 self.cmp_i64s(IntCmpKind::LtU); 926 } 927 928 fn visit_i32_le_s(&mut self) { 929 self.cmp_i32s(IntCmpKind::LeS); 930 } 931 932 fn visit_i64_le_s(&mut self) { 933 self.cmp_i64s(IntCmpKind::LeS); 934 } 935 936 fn visit_i32_le_u(&mut self) { 937 self.cmp_i32s(IntCmpKind::LeU); 938 } 939 940 fn visit_i64_le_u(&mut self) { 941 self.cmp_i64s(IntCmpKind::LeU); 942 } 943 944 fn visit_i32_gt_s(&mut self) { 945 self.cmp_i32s(IntCmpKind::GtS); 946 } 947 948 fn visit_i64_gt_s(&mut self) { 949 self.cmp_i64s(IntCmpKind::GtS); 950 } 951 952 fn visit_i32_gt_u(&mut self) { 953 self.cmp_i32s(IntCmpKind::GtU); 954 } 955 956 fn visit_i64_gt_u(&mut self) { 957 self.cmp_i64s(IntCmpKind::GtU); 958 } 959 960 fn visit_i32_ge_s(&mut self) { 961 self.cmp_i32s(IntCmpKind::GeS); 962 } 963 964 fn visit_i64_ge_s(&mut self) { 965 self.cmp_i64s(IntCmpKind::GeS); 966 } 967 968 fn visit_i32_ge_u(&mut self) { 969 self.cmp_i32s(IntCmpKind::GeU); 970 } 971 972 fn visit_i64_ge_u(&mut self) { 973 self.cmp_i64s(IntCmpKind::GeU); 974 } 975 976 fn visit_i32_eqz(&mut self) { 977 use OperandSize::*; 978 979 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 980 masm.cmp_with_set(RegImm::i32(0), reg.into(), IntCmpKind::Eq, size); 981 TypedReg::i32(reg) 982 }); 983 } 984 985 fn visit_i64_eqz(&mut self) { 986 use OperandSize::*; 987 988 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 989 masm.cmp_with_set(RegImm::i64(0), reg.into(), IntCmpKind::Eq, size); 990 TypedReg::i32(reg) // Return value for `i64.eqz` is an `i32`. 991 }); 992 } 993 994 fn visit_i32_clz(&mut self) { 995 use OperandSize::*; 996 997 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 998 masm.clz(reg, reg, size); 999 TypedReg::i32(reg) 1000 }); 1001 } 1002 1003 fn visit_i64_clz(&mut self) { 1004 use OperandSize::*; 1005 1006 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 1007 masm.clz(reg, reg, size); 1008 TypedReg::i64(reg) 1009 }); 1010 } 1011 1012 fn visit_i32_ctz(&mut self) { 1013 use OperandSize::*; 1014 1015 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 1016 masm.ctz(reg, reg, size); 1017 TypedReg::i32(reg) 1018 }); 1019 } 1020 1021 fn visit_i64_ctz(&mut self) { 1022 use OperandSize::*; 1023 1024 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 1025 masm.ctz(reg, reg, size); 1026 TypedReg::i64(reg) 1027 }); 1028 } 1029 1030 fn visit_i32_and(&mut self) { 1031 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1032 masm.and(dst, dst, src, size); 1033 TypedReg::i32(dst) 1034 }); 1035 } 1036 1037 fn visit_i64_and(&mut self) { 1038 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1039 masm.and(dst, dst, src, size); 1040 TypedReg::i64(dst) 1041 }); 1042 } 1043 1044 fn visit_i32_or(&mut self) { 1045 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1046 masm.or(dst, dst, src, size); 1047 TypedReg::i32(dst) 1048 }); 1049 } 1050 1051 fn visit_i64_or(&mut self) { 1052 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1053 masm.or(dst, dst, src, size); 1054 TypedReg::i64(dst) 1055 }); 1056 } 1057 1058 fn visit_i32_xor(&mut self) { 1059 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1060 masm.xor(dst, dst, src, size); 1061 TypedReg::i32(dst) 1062 }); 1063 } 1064 1065 fn visit_i64_xor(&mut self) { 1066 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1067 masm.xor(dst, dst, src, size); 1068 TypedReg::i64(dst) 1069 }); 1070 } 1071 1072 fn visit_i32_shl(&mut self) { 1073 use OperandSize::*; 1074 use ShiftKind::*; 1075 1076 self.masm.shift(&mut self.context, Shl, S32); 1077 } 1078 1079 fn visit_i64_shl(&mut self) { 1080 use OperandSize::*; 1081 use ShiftKind::*; 1082 1083 self.masm.shift(&mut self.context, Shl, S64); 1084 } 1085 1086 fn visit_i32_shr_s(&mut self) { 1087 use OperandSize::*; 1088 use ShiftKind::*; 1089 1090 self.masm.shift(&mut self.context, ShrS, S32); 1091 } 1092 1093 fn visit_i64_shr_s(&mut self) { 1094 use OperandSize::*; 1095 use ShiftKind::*; 1096 1097 self.masm.shift(&mut self.context, ShrS, S64); 1098 } 1099 1100 fn visit_i32_shr_u(&mut self) { 1101 use OperandSize::*; 1102 use ShiftKind::*; 1103 1104 self.masm.shift(&mut self.context, ShrU, S32); 1105 } 1106 1107 fn visit_i64_shr_u(&mut self) { 1108 use OperandSize::*; 1109 use ShiftKind::*; 1110 1111 self.masm.shift(&mut self.context, ShrU, S64); 1112 } 1113 1114 fn visit_i32_rotl(&mut self) { 1115 use OperandSize::*; 1116 use ShiftKind::*; 1117 1118 self.masm.shift(&mut self.context, Rotl, S32); 1119 } 1120 1121 fn visit_i64_rotl(&mut self) { 1122 use OperandSize::*; 1123 use ShiftKind::*; 1124 1125 self.masm.shift(&mut self.context, Rotl, S64); 1126 } 1127 1128 fn visit_i32_rotr(&mut self) { 1129 use OperandSize::*; 1130 use ShiftKind::*; 1131 1132 self.masm.shift(&mut self.context, Rotr, S32); 1133 } 1134 1135 fn visit_i64_rotr(&mut self) { 1136 use OperandSize::*; 1137 use ShiftKind::*; 1138 1139 self.masm.shift(&mut self.context, Rotr, S64); 1140 } 1141 1142 fn visit_end(&mut self) { 1143 if !self.context.reachable { 1144 self.handle_unreachable_end(); 1145 } else { 1146 let mut control = self.control_frames.pop().unwrap(); 1147 control.emit_end(self.masm, &mut self.context); 1148 } 1149 } 1150 1151 fn visit_i32_popcnt(&mut self) { 1152 use OperandSize::*; 1153 self.masm.popcnt(&mut self.context, S32); 1154 } 1155 1156 fn visit_i64_popcnt(&mut self) { 1157 use OperandSize::*; 1158 1159 self.masm.popcnt(&mut self.context, S64); 1160 } 1161 1162 fn visit_i32_wrap_i64(&mut self) { 1163 use OperandSize::*; 1164 1165 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1166 masm.wrap(reg, reg); 1167 TypedReg::i32(reg) 1168 }); 1169 } 1170 1171 fn visit_i64_extend_i32_s(&mut self) { 1172 use OperandSize::*; 1173 1174 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1175 masm.extend(reg, reg, ExtendKind::I64ExtendI32S); 1176 TypedReg::i64(reg) 1177 }); 1178 } 1179 1180 fn visit_i64_extend_i32_u(&mut self) { 1181 use OperandSize::*; 1182 1183 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1184 masm.extend(reg, reg, ExtendKind::I64ExtendI32U); 1185 TypedReg::i64(reg) 1186 }); 1187 } 1188 1189 fn visit_i32_extend8_s(&mut self) { 1190 use OperandSize::*; 1191 1192 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1193 masm.extend(reg, reg, ExtendKind::I32Extend8S); 1194 TypedReg::i32(reg) 1195 }); 1196 } 1197 1198 fn visit_i32_extend16_s(&mut self) { 1199 use OperandSize::*; 1200 1201 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1202 masm.extend(reg, reg, ExtendKind::I32Extend16S); 1203 TypedReg::i32(reg) 1204 }); 1205 } 1206 1207 fn visit_i64_extend8_s(&mut self) { 1208 use OperandSize::*; 1209 1210 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1211 masm.extend(reg, reg, ExtendKind::I64Extend8S); 1212 TypedReg::i64(reg) 1213 }); 1214 } 1215 1216 fn visit_i64_extend16_s(&mut self) { 1217 use OperandSize::*; 1218 1219 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1220 masm.extend(reg, reg, ExtendKind::I64Extend16S); 1221 TypedReg::i64(reg) 1222 }); 1223 } 1224 1225 fn visit_i64_extend32_s(&mut self) { 1226 use OperandSize::*; 1227 1228 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1229 masm.extend(reg, reg, ExtendKind::I64Extend32S); 1230 TypedReg::i64(reg) 1231 }); 1232 } 1233 1234 fn visit_i32_trunc_f32_s(&mut self) { 1235 use OperandSize::*; 1236 1237 self.context 1238 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1239 masm.signed_truncate(src, dst, S32, dst_size, TruncKind::Unchecked); 1240 }); 1241 } 1242 1243 fn visit_i32_trunc_f32_u(&mut self) { 1244 use OperandSize::*; 1245 1246 self.context.convert_op_with_tmp_reg( 1247 self.masm, 1248 WasmValType::I32, 1249 RegClass::Float, 1250 |masm, dst, src, tmp_fpr, dst_size| { 1251 masm.unsigned_truncate(src, dst, tmp_fpr, S32, dst_size, TruncKind::Unchecked); 1252 }, 1253 ); 1254 } 1255 1256 fn visit_i32_trunc_f64_s(&mut self) { 1257 use OperandSize::*; 1258 1259 self.context 1260 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1261 masm.signed_truncate(src, dst, S64, dst_size, TruncKind::Unchecked); 1262 }); 1263 } 1264 1265 fn visit_i32_trunc_f64_u(&mut self) { 1266 use OperandSize::*; 1267 1268 self.context.convert_op_with_tmp_reg( 1269 self.masm, 1270 WasmValType::I32, 1271 RegClass::Float, 1272 |masm, dst, src, tmp_fpr, dst_size| { 1273 masm.unsigned_truncate(src, dst, tmp_fpr, S64, dst_size, TruncKind::Unchecked); 1274 }, 1275 ); 1276 } 1277 1278 fn visit_i64_trunc_f32_s(&mut self) { 1279 use OperandSize::*; 1280 1281 self.context 1282 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1283 masm.signed_truncate(src, dst, S32, dst_size, TruncKind::Unchecked); 1284 }); 1285 } 1286 1287 fn visit_i64_trunc_f32_u(&mut self) { 1288 use OperandSize::*; 1289 1290 self.context.convert_op_with_tmp_reg( 1291 self.masm, 1292 WasmValType::I64, 1293 RegClass::Float, 1294 |masm, dst, src, tmp_fpr, dst_size| { 1295 masm.unsigned_truncate(src, dst, tmp_fpr, S32, dst_size, TruncKind::Unchecked); 1296 }, 1297 ); 1298 } 1299 1300 fn visit_i64_trunc_f64_s(&mut self) { 1301 use OperandSize::*; 1302 1303 self.context 1304 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1305 masm.signed_truncate(src, dst, S64, dst_size, TruncKind::Unchecked); 1306 }); 1307 } 1308 1309 fn visit_i64_trunc_f64_u(&mut self) { 1310 use OperandSize::*; 1311 1312 self.context.convert_op_with_tmp_reg( 1313 self.masm, 1314 WasmValType::I64, 1315 RegClass::Float, 1316 |masm, dst, src, tmp_fpr, dst_size| { 1317 masm.unsigned_truncate(src, dst, tmp_fpr, S64, dst_size, TruncKind::Unchecked); 1318 }, 1319 ); 1320 } 1321 1322 fn visit_i32_reinterpret_f32(&mut self) { 1323 self.context 1324 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, size| { 1325 masm.reinterpret_float_as_int(src.into(), dst, size); 1326 }); 1327 } 1328 1329 fn visit_i64_reinterpret_f64(&mut self) { 1330 self.context 1331 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, size| { 1332 masm.reinterpret_float_as_int(src.into(), dst, size); 1333 }); 1334 } 1335 1336 fn visit_local_get(&mut self, index: u32) { 1337 use WasmValType::*; 1338 let context = &mut self.context; 1339 let slot = context.frame.get_wasm_local(index); 1340 match slot.ty { 1341 I32 | I64 | F32 | F64 => context.stack.push(Val::local(index, slot.ty)), 1342 Ref(rt) => match rt.heap_type { 1343 WasmHeapType::Func => context.stack.push(Val::local(index, slot.ty)), 1344 ht => unimplemented!("Support for WasmHeapType: {ht}"), 1345 }, 1346 t => unimplemented!("Support local type: {t}"), 1347 } 1348 } 1349 1350 fn visit_local_set(&mut self, index: u32) { 1351 let src = self.emit_set_local(index); 1352 self.context.free_reg(src); 1353 } 1354 1355 fn visit_call(&mut self, index: u32) { 1356 let callee = self.env.callee_from_index(FuncIndex::from_u32(index)); 1357 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee) 1358 } 1359 1360 fn visit_call_indirect(&mut self, type_index: u32, table_index: u32) { 1361 // Spill now because `emit_lazy_init_funcref` and the `FnCall::emit` 1362 // invocations will both trigger spills since they both call functions. 1363 // However, the machine instructions for the spill emitted by 1364 // `emit_lazy_funcref` will be jumped over if the funcref was previously 1365 // initialized which may result in the machine stack becoming 1366 // unbalanced. 1367 self.context.spill(self.masm); 1368 1369 let type_index = TypeIndex::from_u32(type_index); 1370 let table_index = TableIndex::from_u32(table_index); 1371 1372 self.emit_lazy_init_funcref(table_index); 1373 1374 // Perform the indirect call. 1375 // This code assumes that [`Self::emit_lazy_init_funcref`] will 1376 // push the funcref to the value stack. 1377 match self.env.translation.module.table_plans[table_index].style { 1378 TableStyle::CallerChecksSignature { lazy_init: true } => { 1379 let funcref_ptr = self.context.stack.peek().map(|v| v.unwrap_reg()).unwrap(); 1380 self.masm 1381 .trapz(funcref_ptr.into(), TrapCode::IndirectCallToNull); 1382 self.emit_typecheck_funcref(funcref_ptr.into(), type_index); 1383 } 1384 _ => unimplemented!("Support for eager table init"), 1385 } 1386 1387 let callee = self.env.funcref(type_index); 1388 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee) 1389 } 1390 1391 fn visit_table_init(&mut self, elem: u32, table: u32) { 1392 debug_assert!(self.context.stack.len() >= 3); 1393 let at = self.context.stack.len() - 3; 1394 1395 self.context 1396 .stack 1397 .insert_many(at, &[table.try_into().unwrap(), elem.try_into().unwrap()]); 1398 1399 let builtin = self.env.builtins.table_init::<M::ABI, M::Ptr>(); 1400 FnCall::emit::<M>( 1401 &mut self.env, 1402 self.masm, 1403 &mut self.context, 1404 Callee::Builtin(builtin.clone()), 1405 ) 1406 } 1407 1408 fn visit_table_copy(&mut self, dst: u32, src: u32) { 1409 debug_assert!(self.context.stack.len() >= 3); 1410 let at = self.context.stack.len() - 3; 1411 self.context 1412 .stack 1413 .insert_many(at, &[dst.try_into().unwrap(), src.try_into().unwrap()]); 1414 1415 let builtin = self.env.builtins.table_copy::<M::ABI, M::Ptr>(); 1416 FnCall::emit::<M>( 1417 &mut self.env, 1418 self.masm, 1419 &mut self.context, 1420 Callee::Builtin(builtin), 1421 ) 1422 } 1423 1424 fn visit_table_get(&mut self, table: u32) { 1425 let table_index = TableIndex::from_u32(table); 1426 let plan = self.env.table_plan(table_index); 1427 let heap_type = plan.table.wasm_ty.heap_type; 1428 let style = &plan.style; 1429 1430 match heap_type { 1431 WasmHeapType::Func => match style { 1432 TableStyle::CallerChecksSignature { lazy_init: true } => { 1433 self.emit_lazy_init_funcref(table_index) 1434 } 1435 _ => unimplemented!("Support for eager table init"), 1436 }, 1437 t => unimplemented!("Support for WasmHeapType: {t}"), 1438 } 1439 } 1440 1441 fn visit_table_grow(&mut self, table: u32) { 1442 let table_index = TableIndex::from_u32(table); 1443 let table_plan = self.env.table_plan(table_index); 1444 let builtin = match table_plan.table.wasm_ty.heap_type { 1445 WasmHeapType::Func => self.env.builtins.table_grow_func_ref::<M::ABI, M::Ptr>(), 1446 ty => unimplemented!("Support for HeapType: {ty}"), 1447 }; 1448 1449 let len = self.context.stack.len(); 1450 // table.grow` requires at least 2 elements on the value stack. 1451 debug_assert!(len >= 2); 1452 let at = len - 2; 1453 1454 // The table_grow builtin expects the parameters in a different 1455 // order. 1456 // The value stack at this point should contain: 1457 // [ init_value | delta ] (stack top) 1458 // but the builtin function expects the init value as the last 1459 // argument. 1460 self.context.stack.inner_mut().swap(len - 1, len - 2); 1461 self.context 1462 .stack 1463 .insert_many(at, &[table.try_into().unwrap()]); 1464 1465 FnCall::emit::<M>( 1466 &mut self.env, 1467 self.masm, 1468 &mut self.context, 1469 Callee::Builtin(builtin.clone()), 1470 ) 1471 } 1472 1473 fn visit_table_size(&mut self, table: u32) { 1474 let table_index = TableIndex::from_u32(table); 1475 let table_data = self.env.resolve_table_data(table_index); 1476 self.emit_compute_table_size(&table_data); 1477 } 1478 1479 fn visit_table_fill(&mut self, table: u32) { 1480 let table_index = TableIndex::from_u32(table); 1481 let table_plan = self.env.table_plan(table_index); 1482 let builtin = match table_plan.table.wasm_ty.heap_type { 1483 WasmHeapType::Func => self.env.builtins.table_fill_func_ref::<M::ABI, M::Ptr>(), 1484 ty => unimplemented!("Support for heap type: {ty}"), 1485 }; 1486 1487 let len = self.context.stack.len(); 1488 debug_assert!(len >= 3); 1489 let at = len - 3; 1490 self.context 1491 .stack 1492 .insert_many(at, &[table.try_into().unwrap()]); 1493 FnCall::emit::<M>( 1494 &mut self.env, 1495 self.masm, 1496 &mut self.context, 1497 Callee::Builtin(builtin.clone()), 1498 ) 1499 } 1500 1501 fn visit_table_set(&mut self, table: u32) { 1502 let ptr_type = self.env.ptr_type(); 1503 let table_index = TableIndex::from_u32(table); 1504 let table_data = self.env.resolve_table_data(table_index); 1505 let plan = self.env.table_plan(table_index); 1506 match plan.table.wasm_ty.heap_type { 1507 WasmHeapType::Func => match plan.style { 1508 TableStyle::CallerChecksSignature { lazy_init: true } => { 1509 let value = self.context.pop_to_reg(self.masm, None); 1510 let index = self.context.pop_to_reg(self.masm, None); 1511 let base = self.context.any_gpr(self.masm); 1512 let elem_addr = 1513 self.emit_compute_table_elem_addr(index.into(), base, &table_data); 1514 // Set the initialized bit. 1515 self.masm.or( 1516 value.into(), 1517 value.into(), 1518 RegImm::i64(FUNCREF_INIT_BIT as i64), 1519 ptr_type.into(), 1520 ); 1521 1522 self.masm.store_ptr(value.into(), elem_addr); 1523 1524 self.context.free_reg(value); 1525 self.context.free_reg(index); 1526 self.context.free_reg(base); 1527 } 1528 _ => unimplemented!("Support for eager table init"), 1529 }, 1530 ty => unimplemented!("Support for WasmHeapType: {ty}"), 1531 }; 1532 } 1533 1534 fn visit_elem_drop(&mut self, index: u32) { 1535 let elem_drop = self.env.builtins.elem_drop::<M::ABI, M::Ptr>(); 1536 self.context.stack.extend([index.try_into().unwrap()]); 1537 FnCall::emit::<M>( 1538 &mut self.env, 1539 self.masm, 1540 &mut self.context, 1541 Callee::Builtin(elem_drop), 1542 ) 1543 } 1544 1545 fn visit_memory_init(&mut self, data_index: u32, mem: u32) { 1546 debug_assert!(self.context.stack.len() >= 3); 1547 let at = self.context.stack.len() - 3; 1548 self.context.stack.insert_many( 1549 at, 1550 &[mem.try_into().unwrap(), data_index.try_into().unwrap()], 1551 ); 1552 let builtin = self.env.builtins.memory_init::<M::ABI, M::Ptr>(); 1553 FnCall::emit::<M>( 1554 &mut self.env, 1555 self.masm, 1556 &mut self.context, 1557 Callee::Builtin(builtin), 1558 ) 1559 } 1560 1561 fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) { 1562 // At this point, the stack is expected to contain: 1563 // [ dst_offset, src_offset, len ] 1564 // The following code inserts the missing params, so that stack contains: 1565 // [ vmctx, dst_mem, dst_offset, src_mem, src_offset, len ] 1566 // Which is the order expected by the builtin function. 1567 debug_assert!(self.context.stack.len() >= 3); 1568 let at = self.context.stack.len() - 2; 1569 self.context 1570 .stack 1571 .insert_many(at, &[src_mem.try_into().unwrap()]); 1572 1573 // One element was inserted above, so instead of 3, we use 4. 1574 let at = self.context.stack.len() - 4; 1575 self.context 1576 .stack 1577 .insert_many(at, &[dst_mem.try_into().unwrap()]); 1578 1579 let builtin = self.env.builtins.memory_copy::<M::ABI, M::Ptr>(); 1580 1581 FnCall::emit::<M>( 1582 &mut self.env, 1583 self.masm, 1584 &mut self.context, 1585 Callee::Builtin(builtin), 1586 ) 1587 } 1588 1589 fn visit_memory_fill(&mut self, mem: u32) { 1590 debug_assert!(self.context.stack.len() >= 3); 1591 let at = self.context.stack.len() - 3; 1592 1593 self.context 1594 .stack 1595 .insert_many(at, &[mem.try_into().unwrap()]); 1596 1597 let builtin = self.env.builtins.memory_fill::<M::ABI, M::Ptr>(); 1598 FnCall::emit::<M>( 1599 &mut self.env, 1600 self.masm, 1601 &mut self.context, 1602 Callee::Builtin(builtin), 1603 ) 1604 } 1605 1606 fn visit_memory_size(&mut self, mem: u32) { 1607 let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem)); 1608 self.emit_compute_memory_size(&heap); 1609 } 1610 1611 fn visit_memory_grow(&mut self, mem: u32) { 1612 debug_assert!(self.context.stack.len() >= 1); 1613 // The stack at this point contains: [ delta ] 1614 // The desired state is 1615 // [ vmctx, delta, index ] 1616 self.context.stack.extend([mem.try_into().unwrap()]); 1617 1618 let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem)); 1619 let builtin = self.env.builtins.memory32_grow::<M::ABI, M::Ptr>(); 1620 FnCall::emit::<M>( 1621 &mut self.env, 1622 self.masm, 1623 &mut self.context, 1624 Callee::Builtin(builtin), 1625 ); 1626 1627 // The memory32_grow builtin returns a pointer type, therefore we must 1628 // ensure that the return type is representative of the address space of 1629 // the heap type. 1630 match (self.env.ptr_type(), heap.ty) { 1631 (WasmValType::I64, WasmValType::I64) => {} 1632 // When the heap type is smaller than the pointer type, we adjust 1633 // the result of the memory32_grow builtin. 1634 (WasmValType::I64, WasmValType::I32) => { 1635 let top: Reg = self.context.pop_to_reg(self.masm, None).into(); 1636 self.masm.wrap(top.into(), top.into()); 1637 self.context.stack.push(TypedReg::i32(top).into()); 1638 } 1639 _ => unimplemented!("Support for 32-bit platforms"), 1640 } 1641 } 1642 1643 fn visit_data_drop(&mut self, data_index: u32) { 1644 self.context.stack.extend([data_index.try_into().unwrap()]); 1645 1646 let builtin = self.env.builtins.data_drop::<M::ABI, M::Ptr>(); 1647 FnCall::emit::<M>( 1648 &mut self.env, 1649 self.masm, 1650 &mut self.context, 1651 Callee::Builtin(builtin), 1652 ) 1653 } 1654 1655 fn visit_nop(&mut self) {} 1656 1657 fn visit_if(&mut self, blockty: BlockType) { 1658 self.control_frames.push(ControlStackFrame::r#if( 1659 self.env.resolve_block_sig(blockty), 1660 self.masm, 1661 &mut self.context, 1662 )); 1663 } 1664 1665 fn visit_else(&mut self) { 1666 if !self.context.reachable { 1667 self.handle_unreachable_else(); 1668 } else { 1669 let control = self 1670 .control_frames 1671 .last_mut() 1672 .unwrap_or_else(|| panic!("Expected active control stack frame for else")); 1673 control.emit_else(self.masm, &mut self.context); 1674 } 1675 } 1676 1677 fn visit_block(&mut self, blockty: BlockType) { 1678 self.control_frames.push(ControlStackFrame::block( 1679 self.env.resolve_block_sig(blockty), 1680 self.masm, 1681 &mut self.context, 1682 )); 1683 } 1684 1685 fn visit_loop(&mut self, blockty: BlockType) { 1686 self.control_frames.push(ControlStackFrame::r#loop( 1687 self.env.resolve_block_sig(blockty), 1688 self.masm, 1689 &mut self.context, 1690 )); 1691 } 1692 1693 fn visit_br(&mut self, depth: u32) { 1694 let index = control_index(depth, self.control_frames.len()); 1695 let frame = &mut self.control_frames[index]; 1696 self.context 1697 .unconditional_jump(frame, self.masm, |masm, cx, frame| { 1698 frame 1699 .pop_abi_results::<M, _>(cx, masm, |results, _, _| results.ret_area().copied()); 1700 }); 1701 } 1702 1703 fn visit_br_if(&mut self, depth: u32) { 1704 let index = control_index(depth, self.control_frames.len()); 1705 let frame = &mut self.control_frames[index]; 1706 frame.set_as_target(); 1707 1708 let top = { 1709 let top = self.context.without::<TypedReg, M, _>( 1710 frame.results::<M>().regs(), 1711 self.masm, 1712 |ctx, masm| ctx.pop_to_reg(masm, None), 1713 ); 1714 // Explicitly save any live registers and locals before setting up 1715 // the branch state. 1716 // In some cases, calculating the `top` value above, will result in 1717 // a spill, thus the following one will result in a no-op. 1718 self.context.spill(self.masm); 1719 frame.top_abi_results::<M, _>( 1720 &mut self.context, 1721 self.masm, 1722 |results, context, masm| { 1723 // In the case of `br_if` there's a possibility that we'll 1724 // exit early from the block or fallthrough, for 1725 // a fallthrough, we cannot rely on the pre-computed return area; 1726 // it must be recalculated so that any values that are 1727 // generated are correctly placed near the current stack 1728 // pointer. 1729 results.on_stack().then(|| { 1730 let stack_consumed = context.stack.sizeof(results.stack_operands_len()); 1731 let base = masm.sp_offset().as_u32() - stack_consumed; 1732 let offs = base + results.size(); 1733 RetArea::sp(SPOffset::from_u32(offs)) 1734 }) 1735 }, 1736 ); 1737 top 1738 }; 1739 1740 // Emit instructions to balance the machine stack if the frame has 1741 // a different offset. 1742 let current_sp_offset = self.masm.sp_offset(); 1743 let results_size = frame.results::<M>().size(); 1744 let state = frame.stack_state(); 1745 let (label, cmp, needs_cleanup) = if current_sp_offset > state.target_offset { 1746 (self.masm.get_label(), IntCmpKind::Eq, true) 1747 } else { 1748 (*frame.label(), IntCmpKind::Ne, false) 1749 }; 1750 1751 self.masm 1752 .branch(cmp, top.reg.into(), top.reg.into(), label, OperandSize::S32); 1753 self.context.free_reg(top); 1754 1755 if needs_cleanup { 1756 // Emit instructions to balance the stack and jump if not falling 1757 // through. 1758 self.masm.memmove( 1759 current_sp_offset, 1760 state.target_offset, 1761 results_size, 1762 MemMoveDirection::LowToHigh, 1763 ); 1764 self.masm.ensure_sp_for_jump(state.target_offset); 1765 self.masm.jmp(*frame.label()); 1766 1767 // Restore sp_offset to what it was for falling through and emit 1768 // fallthrough label. 1769 self.masm.reset_stack_pointer(current_sp_offset); 1770 self.masm.bind(label); 1771 } 1772 } 1773 1774 fn visit_br_table(&mut self, targets: BrTable<'a>) { 1775 // +1 to account for the default target. 1776 let len = targets.len() + 1; 1777 // SmallVec<[_; 5]> to match the binary emission layer (e.g 1778 // see `JmpTableSeq'), but here we use 5 instead since we 1779 // bundle the default target as the last element in the array. 1780 let labels: SmallVec<[_; 5]> = (0..len).map(|_| self.masm.get_label()).collect(); 1781 1782 let default_index = control_index(targets.default(), self.control_frames.len()); 1783 let default_frame = &mut self.control_frames[default_index]; 1784 let default_result = default_frame.results::<M>(); 1785 1786 let (index, tmp) = { 1787 let index_and_tmp = self.context.without::<(TypedReg, _), M, _>( 1788 default_result.regs(), 1789 self.masm, 1790 |cx, masm| (cx.pop_to_reg(masm, None), cx.any_gpr(masm)), 1791 ); 1792 1793 // Materialize any constants or locals into their result representation, 1794 // so that when reachability is restored, they are correctly located. 1795 default_frame.top_abi_results::<M, _>(&mut self.context, self.masm, |results, _, _| { 1796 results.ret_area().copied() 1797 }); 1798 index_and_tmp 1799 }; 1800 1801 self.masm.jmp_table(&labels, index.into(), tmp); 1802 // Save the original stack pointer offset; we will reset the stack 1803 // pointer to this offset after jumping to each of the targets. Each 1804 // jump might adjust the stack according to the base offset of the 1805 // target. 1806 let current_sp = self.masm.sp_offset(); 1807 1808 for (t, l) in targets 1809 .targets() 1810 .into_iter() 1811 .chain(std::iter::once(Ok(targets.default()))) 1812 .zip(labels.iter()) 1813 { 1814 let control_index = control_index(t.unwrap(), self.control_frames.len()); 1815 let frame = &mut self.control_frames[control_index]; 1816 // Reset the stack pointer to its original offset. This is needed 1817 // because each jump will potentially adjust the stack pointer 1818 // according to the base offset of the target. 1819 self.masm.reset_stack_pointer(current_sp); 1820 1821 // NB: We don't perform any result handling as it was 1822 // already taken care of above before jumping to the 1823 // jump table. 1824 self.masm.bind(*l); 1825 // Ensure that the stack pointer is correctly positioned before 1826 // jumping to the jump table code. 1827 let state = frame.stack_state(); 1828 self.masm.ensure_sp_for_jump(state.target_offset); 1829 self.masm.jmp(*frame.label()); 1830 frame.set_as_target(); 1831 } 1832 // Finally reset the stack pointer to the original location. 1833 // The reachability analysis, will ensure it's correctly located 1834 // once reachability is restored. 1835 self.masm.reset_stack_pointer(current_sp); 1836 self.context.reachable = false; 1837 self.context.free_reg(index.reg); 1838 self.context.free_reg(tmp); 1839 } 1840 1841 fn visit_return(&mut self) { 1842 // Grab the outermost frame, which is the function's body 1843 // frame. We don't rely on [`codegen::control_index`] since 1844 // this frame is implicit and we know that it should exist at 1845 // index 0. 1846 let outermost = &mut self.control_frames[0]; 1847 self.context 1848 .unconditional_jump(outermost, self.masm, |masm, cx, frame| { 1849 frame 1850 .pop_abi_results::<M, _>(cx, masm, |results, _, _| results.ret_area().copied()); 1851 }); 1852 } 1853 1854 fn visit_unreachable(&mut self) { 1855 self.masm.unreachable(); 1856 self.context.reachable = false; 1857 // Set the implicit outermost frame as target to perform the necessary 1858 // stack clean up. 1859 let outermost = &mut self.control_frames[0]; 1860 outermost.set_as_target(); 1861 } 1862 1863 fn visit_local_tee(&mut self, index: u32) { 1864 let typed_reg = self.emit_set_local(index); 1865 self.context.stack.push(typed_reg.into()); 1866 } 1867 1868 fn visit_global_get(&mut self, global_index: u32) { 1869 let index = GlobalIndex::from_u32(global_index); 1870 let (ty, addr) = self.emit_get_global_addr(index); 1871 let dst = self.context.reg_for_type(ty, self.masm); 1872 self.masm.load(addr, dst, ty.into()); 1873 self.context.stack.push(Val::reg(dst, ty)); 1874 } 1875 1876 fn visit_global_set(&mut self, global_index: u32) { 1877 let index = GlobalIndex::from_u32(global_index); 1878 let (ty, addr) = self.emit_get_global_addr(index); 1879 1880 let typed_reg = self.context.pop_to_reg(self.masm, None); 1881 self.context.free_reg(typed_reg.reg); 1882 self.masm.store(typed_reg.reg.into(), addr, ty.into()); 1883 } 1884 1885 fn visit_drop(&mut self) { 1886 self.context.drop_last(1, |regalloc, val| match val { 1887 Val::Reg(tr) => regalloc.free(tr.reg.into()), 1888 Val::Memory(m) => self.masm.free_stack(m.slot.size), 1889 _ => {} 1890 }); 1891 } 1892 1893 fn visit_select(&mut self) { 1894 let cond = self.context.pop_to_reg(self.masm, None); 1895 let val2 = self.context.pop_to_reg(self.masm, None); 1896 let val1 = self.context.pop_to_reg(self.masm, None); 1897 self.masm 1898 .cmp(cond.reg.into(), RegImm::i32(0), OperandSize::S32); 1899 // Conditionally move val1 to val2 if the comparison is 1900 // not zero. 1901 self.masm 1902 .cmov(val1.into(), val2.into(), IntCmpKind::Ne, val1.ty.into()); 1903 self.context.stack.push(val2.into()); 1904 self.context.free_reg(val1.reg); 1905 self.context.free_reg(cond); 1906 } 1907 1908 fn visit_i32_load(&mut self, memarg: MemArg) { 1909 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S32, None); 1910 } 1911 1912 fn visit_i32_load8_s(&mut self, memarg: MemArg) { 1913 self.emit_wasm_load( 1914 &memarg, 1915 WasmValType::I32, 1916 OperandSize::S8, 1917 Some(ExtendKind::I32Extend8S), 1918 ); 1919 } 1920 1921 fn visit_i32_load8_u(&mut self, memarg: MemArg) { 1922 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S8, None); 1923 } 1924 1925 fn visit_i32_load16_s(&mut self, memarg: MemArg) { 1926 self.emit_wasm_load( 1927 &memarg, 1928 WasmValType::I32, 1929 OperandSize::S16, 1930 Some(ExtendKind::I32Extend16S), 1931 ) 1932 } 1933 1934 fn visit_i32_load16_u(&mut self, memarg: MemArg) { 1935 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S16, None) 1936 } 1937 1938 fn visit_i32_store(&mut self, memarg: MemArg) { 1939 self.emit_wasm_store(&memarg, OperandSize::S32); 1940 } 1941 1942 fn visit_i32_store8(&mut self, memarg: MemArg) { 1943 self.emit_wasm_store(&memarg, OperandSize::S8) 1944 } 1945 1946 fn visit_i32_store16(&mut self, memarg: MemArg) { 1947 self.emit_wasm_store(&memarg, OperandSize::S16) 1948 } 1949 1950 fn visit_i64_load8_s(&mut self, memarg: MemArg) { 1951 self.emit_wasm_load( 1952 &memarg, 1953 WasmValType::I64, 1954 OperandSize::S8, 1955 Some(ExtendKind::I64Extend8S), 1956 ) 1957 } 1958 1959 fn visit_i64_load8_u(&mut self, memarg: MemArg) { 1960 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S8, None) 1961 } 1962 1963 fn visit_i64_load16_u(&mut self, memarg: MemArg) { 1964 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S16, None) 1965 } 1966 1967 fn visit_i64_load16_s(&mut self, memarg: MemArg) { 1968 self.emit_wasm_load( 1969 &memarg, 1970 WasmValType::I64, 1971 OperandSize::S16, 1972 Some(ExtendKind::I64Extend16S), 1973 ) 1974 } 1975 1976 fn visit_i64_load32_u(&mut self, memarg: MemArg) { 1977 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S32, None) 1978 } 1979 1980 fn visit_i64_load32_s(&mut self, memarg: MemArg) { 1981 self.emit_wasm_load( 1982 &memarg, 1983 WasmValType::I64, 1984 OperandSize::S32, 1985 Some(ExtendKind::I64Extend32S), 1986 ) 1987 } 1988 1989 fn visit_i64_load(&mut self, memarg: MemArg) { 1990 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S64, None) 1991 } 1992 1993 fn visit_i64_store(&mut self, memarg: MemArg) -> Self::Output { 1994 self.emit_wasm_store(&memarg, OperandSize::S64) 1995 } 1996 1997 fn visit_i64_store8(&mut self, memarg: MemArg) -> Self::Output { 1998 self.emit_wasm_store(&memarg, OperandSize::S8) 1999 } 2000 2001 fn visit_i64_store16(&mut self, memarg: MemArg) -> Self::Output { 2002 self.emit_wasm_store(&memarg, OperandSize::S16) 2003 } 2004 2005 fn visit_i64_store32(&mut self, memarg: MemArg) -> Self::Output { 2006 self.emit_wasm_store(&memarg, OperandSize::S32) 2007 } 2008 2009 fn visit_f32_load(&mut self, memarg: MemArg) { 2010 self.emit_wasm_load(&memarg, WasmValType::F32, OperandSize::S32, None) 2011 } 2012 2013 fn visit_f32_store(&mut self, memarg: MemArg) { 2014 self.emit_wasm_store(&memarg, OperandSize::S32) 2015 } 2016 2017 fn visit_f64_load(&mut self, memarg: MemArg) { 2018 self.emit_wasm_load(&memarg, WasmValType::F64, OperandSize::S64, None) 2019 } 2020 2021 fn visit_f64_store(&mut self, memarg: MemArg) { 2022 self.emit_wasm_store(&memarg, OperandSize::S64) 2023 } 2024 2025 fn visit_i32_trunc_sat_f32_s(&mut self) { 2026 use OperandSize::*; 2027 2028 self.context 2029 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2030 masm.signed_truncate(src, dst, S32, dst_size, TruncKind::Checked); 2031 }); 2032 } 2033 2034 fn visit_i32_trunc_sat_f32_u(&mut self) { 2035 use OperandSize::*; 2036 2037 self.context.convert_op_with_tmp_reg( 2038 self.masm, 2039 WasmValType::I32, 2040 RegClass::Float, 2041 |masm, dst, src, tmp_fpr, dst_size| { 2042 masm.unsigned_truncate(src, dst, tmp_fpr, S32, dst_size, TruncKind::Checked); 2043 }, 2044 ); 2045 } 2046 2047 fn visit_i32_trunc_sat_f64_s(&mut self) { 2048 use OperandSize::*; 2049 2050 self.context 2051 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2052 masm.signed_truncate(src, dst, S64, dst_size, TruncKind::Checked); 2053 }); 2054 } 2055 2056 fn visit_i32_trunc_sat_f64_u(&mut self) { 2057 use OperandSize::*; 2058 2059 self.context.convert_op_with_tmp_reg( 2060 self.masm, 2061 WasmValType::I32, 2062 RegClass::Float, 2063 |masm, dst, src, tmp_fpr, dst_size| { 2064 masm.unsigned_truncate(src, dst, tmp_fpr, S64, dst_size, TruncKind::Checked); 2065 }, 2066 ); 2067 } 2068 2069 fn visit_i64_trunc_sat_f32_s(&mut self) { 2070 use OperandSize::*; 2071 2072 self.context 2073 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2074 masm.signed_truncate(src, dst, S32, dst_size, TruncKind::Checked); 2075 }); 2076 } 2077 2078 fn visit_i64_trunc_sat_f32_u(&mut self) { 2079 use OperandSize::*; 2080 2081 self.context.convert_op_with_tmp_reg( 2082 self.masm, 2083 WasmValType::I64, 2084 RegClass::Float, 2085 |masm, dst, src, tmp_fpr, dst_size| { 2086 masm.unsigned_truncate(src, dst, tmp_fpr, S32, dst_size, TruncKind::Checked); 2087 }, 2088 ); 2089 } 2090 2091 fn visit_i64_trunc_sat_f64_s(&mut self) { 2092 use OperandSize::*; 2093 2094 self.context 2095 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2096 masm.signed_truncate(src, dst, S64, dst_size, TruncKind::Checked); 2097 }); 2098 } 2099 2100 fn visit_i64_trunc_sat_f64_u(&mut self) { 2101 use OperandSize::*; 2102 2103 self.context.convert_op_with_tmp_reg( 2104 self.masm, 2105 WasmValType::I64, 2106 RegClass::Float, 2107 |masm, dst, src, tmp_fpr, dst_size| { 2108 masm.unsigned_truncate(src, dst, tmp_fpr, S64, dst_size, TruncKind::Checked); 2109 }, 2110 ); 2111 } 2112 2113 wasmparser::for_each_operator!(def_unsupported); 2114 } 2115 2116 impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M> 2117 where 2118 M: MacroAssembler, 2119 { 2120 fn cmp_i32s(&mut self, kind: IntCmpKind) { 2121 self.context.i32_binop(self.masm, |masm, dst, src, size| { 2122 masm.cmp_with_set(src, dst, kind, size); 2123 TypedReg::i32(dst) 2124 }); 2125 } 2126 2127 fn cmp_i64s(&mut self, kind: IntCmpKind) { 2128 self.context 2129 .i64_binop(self.masm, move |masm, dst, src, size| { 2130 masm.cmp_with_set(src, dst, kind, size); 2131 TypedReg::i32(dst) // Return value for comparisons is an `i32`. 2132 }); 2133 } 2134 } 2135 2136 impl From<WasmValType> for OperandSize { 2137 fn from(ty: WasmValType) -> OperandSize { 2138 match ty { 2139 WasmValType::I32 | WasmValType::F32 => OperandSize::S32, 2140 WasmValType::I64 | WasmValType::F64 => OperandSize::S64, 2141 WasmValType::Ref(rt) => { 2142 match rt.heap_type { 2143 // TODO: Hardcoded size, assuming 64-bit support only. Once 2144 // Wasmtime supports 32-bit architectures, this will need 2145 // to be updated in such a way that the calculation of the 2146 // OperandSize will depend on the target's pointer size. 2147 WasmHeapType::Func => OperandSize::S64, 2148 t => unimplemented!("Support for WasmHeapType: {t}"), 2149 } 2150 } 2151 ty => unimplemented!("Support for WasmValType {ty}"), 2152 } 2153 } 2154 } 2155