1 //! This module is the central place for machine code emission. 2 //! It defines an implementation of wasmparser's Visitor trait 3 //! for `CodeGen`; which defines a visitor per op-code, 4 //! which validates and dispatches to the corresponding 5 //! machine code emitter. 6 7 use crate::abi::RetArea; 8 use crate::codegen::{control_index, Callee, CodeGen, ControlStackFrame, FnCall}; 9 use crate::masm::{ 10 DivKind, ExtendKind, FloatCmpKind, IntCmpKind, MacroAssembler, MemMoveDirection, OperandSize, 11 RegImm, RemKind, RoundingMode, SPOffset, ShiftKind, TruncKind, 12 }; 13 use crate::reg::Reg; 14 use crate::stack::{TypedReg, Val}; 15 use cranelift_codegen::ir::TrapCode; 16 use regalloc2::RegClass; 17 use smallvec::SmallVec; 18 use wasmparser::{BlockType, BrTable, Ieee32, Ieee64, MemArg, VisitOperator, V128}; 19 use wasmtime_environ::{ 20 FuncIndex, GlobalIndex, MemoryIndex, TableIndex, TableStyle, TypeIndex, WasmHeapType, 21 WasmValType, FUNCREF_INIT_BIT, 22 }; 23 24 /// A macro to define unsupported WebAssembly operators. 25 /// 26 /// This macro calls itself recursively; 27 /// 1. It no-ops when matching a supported operator. 28 /// 2. Defines the visitor function and panics when 29 /// matching an unsupported operator. 30 macro_rules! def_unsupported { 31 ($( @$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident)*) => { 32 $( 33 def_unsupported!( 34 emit 35 $op 36 37 fn $visit(&mut self $($(,$arg: $argty)*)?) -> Self::Output { 38 $($(let _ = $arg;)*)? 39 40 self.found_unsupported_instruction = Some(stringify!($op)); 41 } 42 ); 43 )* 44 }; 45 46 (emit I32Const $($rest:tt)*) => {}; 47 (emit I64Const $($rest:tt)*) => {}; 48 (emit F32Const $($rest:tt)*) => {}; 49 (emit F64Const $($rest:tt)*) => {}; 50 (emit V128Const $($rest:tt)*) => {}; 51 (emit F32Add $($rest:tt)*) => {}; 52 (emit F64Add $($rest:tt)*) => {}; 53 (emit F32Sub $($rest:tt)*) => {}; 54 (emit F64Sub $($rest:tt)*) => {}; 55 (emit F32Mul $($rest:tt)*) => {}; 56 (emit F64Mul $($rest:tt)*) => {}; 57 (emit F32Div $($rest:tt)*) => {}; 58 (emit F64Div $($rest:tt)*) => {}; 59 (emit F32Min $($rest:tt)*) => {}; 60 (emit F64Min $($rest:tt)*) => {}; 61 (emit F32Max $($rest:tt)*) => {}; 62 (emit F64Max $($rest:tt)*) => {}; 63 (emit F32Copysign $($rest:tt)*) => {}; 64 (emit F64Copysign $($rest:tt)*) => {}; 65 (emit F32Abs $($rest:tt)*) => {}; 66 (emit F64Abs $($rest:tt)*) => {}; 67 (emit F32Neg $($rest:tt)*) => {}; 68 (emit F64Neg $($rest:tt)*) => {}; 69 (emit F32Floor $($rest:tt)*) => {}; 70 (emit F64Floor $($rest:tt)*) => {}; 71 (emit F32Ceil $($rest:tt)*) => {}; 72 (emit F64Ceil $($rest:tt)*) => {}; 73 (emit F32Nearest $($rest:tt)*) => {}; 74 (emit F64Nearest $($rest:tt)*) => {}; 75 (emit F32Trunc $($rest:tt)*) => {}; 76 (emit F64Trunc $($rest:tt)*) => {}; 77 (emit F32Sqrt $($rest:tt)*) => {}; 78 (emit F64Sqrt $($rest:tt)*) => {}; 79 (emit F32Eq $($rest:tt)*) => {}; 80 (emit F64Eq $($rest:tt)*) => {}; 81 (emit F32Ne $($rest:tt)*) => {}; 82 (emit F64Ne $($rest:tt)*) => {}; 83 (emit F32Lt $($rest:tt)*) => {}; 84 (emit F64Lt $($rest:tt)*) => {}; 85 (emit F32Gt $($rest:tt)*) => {}; 86 (emit F64Gt $($rest:tt)*) => {}; 87 (emit F32Le $($rest:tt)*) => {}; 88 (emit F64Le $($rest:tt)*) => {}; 89 (emit F32Ge $($rest:tt)*) => {}; 90 (emit F64Ge $($rest:tt)*) => {}; 91 (emit F32ConvertI32S $($rest:tt)*) => {}; 92 (emit F32ConvertI32U $($rest:tt)*) => {}; 93 (emit F32ConvertI64S $($rest:tt)*) => {}; 94 (emit F32ConvertI64U $($rest:tt)*) => {}; 95 (emit F64ConvertI32S $($rest:tt)*) => {}; 96 (emit F64ConvertI32U $($rest:tt)*) => {}; 97 (emit F64ConvertI64S $($rest:tt)*) => {}; 98 (emit F64ConvertI64U $($rest:tt)*) => {}; 99 (emit F32ReinterpretI32 $($rest:tt)*) => {}; 100 (emit F64ReinterpretI64 $($rest:tt)*) => {}; 101 (emit F32DemoteF64 $($rest:tt)*) => {}; 102 (emit F64PromoteF32 $($rest:tt)*) => {}; 103 (emit I32Add $($rest:tt)*) => {}; 104 (emit I64Add $($rest:tt)*) => {}; 105 (emit I32Sub $($rest:tt)*) => {}; 106 (emit I32Mul $($rest:tt)*) => {}; 107 (emit I32DivS $($rest:tt)*) => {}; 108 (emit I32DivU $($rest:tt)*) => {}; 109 (emit I64DivS $($rest:tt)*) => {}; 110 (emit I64DivU $($rest:tt)*) => {}; 111 (emit I64RemU $($rest:tt)*) => {}; 112 (emit I64RemS $($rest:tt)*) => {}; 113 (emit I32RemU $($rest:tt)*) => {}; 114 (emit I32RemS $($rest:tt)*) => {}; 115 (emit I64Mul $($rest:tt)*) => {}; 116 (emit I64Sub $($rest:tt)*) => {}; 117 (emit I32Eq $($rest:tt)*) => {}; 118 (emit I64Eq $($rest:tt)*) => {}; 119 (emit I32Ne $($rest:tt)*) => {}; 120 (emit I64Ne $($rest:tt)*) => {}; 121 (emit I32LtS $($rest:tt)*) => {}; 122 (emit I64LtS $($rest:tt)*) => {}; 123 (emit I32LtU $($rest:tt)*) => {}; 124 (emit I64LtU $($rest:tt)*) => {}; 125 (emit I32LeS $($rest:tt)*) => {}; 126 (emit I64LeS $($rest:tt)*) => {}; 127 (emit I32LeU $($rest:tt)*) => {}; 128 (emit I64LeU $($rest:tt)*) => {}; 129 (emit I32GtS $($rest:tt)*) => {}; 130 (emit I64GtS $($rest:tt)*) => {}; 131 (emit I32GtU $($rest:tt)*) => {}; 132 (emit I64GtU $($rest:tt)*) => {}; 133 (emit I32GeS $($rest:tt)*) => {}; 134 (emit I64GeS $($rest:tt)*) => {}; 135 (emit I32GeU $($rest:tt)*) => {}; 136 (emit I64GeU $($rest:tt)*) => {}; 137 (emit I32Eqz $($rest:tt)*) => {}; 138 (emit I64Eqz $($rest:tt)*) => {}; 139 (emit I32And $($rest:tt)*) => {}; 140 (emit I64And $($rest:tt)*) => {}; 141 (emit I32Or $($rest:tt)*) => {}; 142 (emit I64Or $($rest:tt)*) => {}; 143 (emit I32Xor $($rest:tt)*) => {}; 144 (emit I64Xor $($rest:tt)*) => {}; 145 (emit I32Shl $($rest:tt)*) => {}; 146 (emit I64Shl $($rest:tt)*) => {}; 147 (emit I32ShrS $($rest:tt)*) => {}; 148 (emit I64ShrS $($rest:tt)*) => {}; 149 (emit I32ShrU $($rest:tt)*) => {}; 150 (emit I64ShrU $($rest:tt)*) => {}; 151 (emit I32Rotl $($rest:tt)*) => {}; 152 (emit I64Rotl $($rest:tt)*) => {}; 153 (emit I32Rotr $($rest:tt)*) => {}; 154 (emit I64Rotr $($rest:tt)*) => {}; 155 (emit I32Clz $($rest:tt)*) => {}; 156 (emit I64Clz $($rest:tt)*) => {}; 157 (emit I32Ctz $($rest:tt)*) => {}; 158 (emit I64Ctz $($rest:tt)*) => {}; 159 (emit I32Popcnt $($rest:tt)*) => {}; 160 (emit I64Popcnt $($rest:tt)*) => {}; 161 (emit I32WrapI64 $($rest:tt)*) => {}; 162 (emit I64ExtendI32S $($rest:tt)*) => {}; 163 (emit I64ExtendI32U $($rest:tt)*) => {}; 164 (emit I32Extend8S $($rest:tt)*) => {}; 165 (emit I32Extend16S $($rest:tt)*) => {}; 166 (emit I64Extend8S $($rest:tt)*) => {}; 167 (emit I64Extend16S $($rest:tt)*) => {}; 168 (emit I64Extend32S $($rest:tt)*) => {}; 169 (emit I32TruncF32S $($rest:tt)*) => {}; 170 (emit I32TruncF32U $($rest:tt)*) => {}; 171 (emit I32TruncF64S $($rest:tt)*) => {}; 172 (emit I32TruncF64U $($rest:tt)*) => {}; 173 (emit I64TruncF32S $($rest:tt)*) => {}; 174 (emit I64TruncF32U $($rest:tt)*) => {}; 175 (emit I64TruncF64S $($rest:tt)*) => {}; 176 (emit I64TruncF64U $($rest:tt)*) => {}; 177 (emit I32ReinterpretF32 $($rest:tt)*) => {}; 178 (emit I64ReinterpretF64 $($rest:tt)*) => {}; 179 (emit LocalGet $($rest:tt)*) => {}; 180 (emit LocalSet $($rest:tt)*) => {}; 181 (emit Call $($rest:tt)*) => {}; 182 (emit End $($rest:tt)*) => {}; 183 (emit Nop $($rest:tt)*) => {}; 184 (emit If $($rest:tt)*) => {}; 185 (emit Else $($rest:tt)*) => {}; 186 (emit Block $($rest:tt)*) => {}; 187 (emit Loop $($rest:tt)*) => {}; 188 (emit Br $($rest:tt)*) => {}; 189 (emit BrIf $($rest:tt)*) => {}; 190 (emit Return $($rest:tt)*) => {}; 191 (emit Unreachable $($rest:tt)*) => {}; 192 (emit LocalTee $($rest:tt)*) => {}; 193 (emit GlobalGet $($rest:tt)*) => {}; 194 (emit GlobalSet $($rest:tt)*) => {}; 195 (emit Select $($rest:tt)*) => {}; 196 (emit Drop $($rest:tt)*) => {}; 197 (emit BrTable $($rest:tt)*) => {}; 198 (emit CallIndirect $($rest:tt)*) => {}; 199 (emit TableInit $($rest:tt)*) => {}; 200 (emit TableCopy $($rest:tt)*) => {}; 201 (emit TableGet $($rest:tt)*) => {}; 202 (emit TableSet $($rest:tt)*) => {}; 203 (emit TableGrow $($rest:tt)*) => {}; 204 (emit TableSize $($rest:tt)*) => {}; 205 (emit TableFill $($rest:tt)*) => {}; 206 (emit ElemDrop $($rest:tt)*) => {}; 207 (emit MemoryInit $($rest:tt)*) => {}; 208 (emit MemoryCopy $($rest:tt)*) => {}; 209 (emit DataDrop $($rest:tt)*) => {}; 210 (emit MemoryFill $($rest:tt)*) => {}; 211 (emit MemorySize $($rest:tt)*) => {}; 212 (emit MemoryGrow $($rest:tt)*) => {}; 213 (emit I32Load $($rest:tt)*) => {}; 214 (emit I32Load8S $($rest:tt)*) => {}; 215 (emit I32Load8U $($rest:tt)*) => {}; 216 (emit I32Load16S $($rest:tt)*) => {}; 217 (emit I32Load16U $($rest:tt)*) => {}; 218 (emit I64Load8S $($rest:tt)*) => {}; 219 (emit I64Load8U $($rest:tt)*) => {}; 220 (emit I64Load16S $($rest:tt)*) => {}; 221 (emit I64Load16U $($rest:tt)*) => {}; 222 (emit I64Load32S $($rest:tt)*) => {}; 223 (emit I64Load32U $($rest:tt)*) => {}; 224 (emit I64Load $($rest:tt)*) => {}; 225 (emit I32Store $($rest:tt)*) => {}; 226 (emit I32Store $($rest:tt)*) => {}; 227 (emit I32Store8 $($rest:tt)*) => {}; 228 (emit I32Store16 $($rest:tt)*) => {}; 229 (emit I64Store $($rest:tt)*) => {}; 230 (emit I64Store8 $($rest:tt)*) => {}; 231 (emit I64Store16 $($rest:tt)*) => {}; 232 (emit I64Store32 $($rest:tt)*) => {}; 233 (emit F32Load $($rest:tt)*) => {}; 234 (emit F32Store $($rest:tt)*) => {}; 235 (emit F64Load $($rest:tt)*) => {}; 236 (emit F64Store $($rest:tt)*) => {}; 237 (emit I32TruncSatF32S $($rest:tt)*) => {}; 238 (emit I32TruncSatF32U $($rest:tt)*) => {}; 239 (emit I32TruncSatF64S $($rest:tt)*) => {}; 240 (emit I32TruncSatF64U $($rest:tt)*) => {}; 241 (emit I64TruncSatF32S $($rest:tt)*) => {}; 242 (emit I64TruncSatF32U $($rest:tt)*) => {}; 243 (emit I64TruncSatF64S $($rest:tt)*) => {}; 244 (emit I64TruncSatF64U $($rest:tt)*) => {}; 245 (emit V128Load $($rest:tt)*) => {}; 246 (emit V128Store $($rest:tt)*) => {}; 247 248 (emit $unsupported:tt $($rest:tt)*) => {$($rest)*}; 249 } 250 251 impl<'a, 'translation, 'data, M> VisitOperator<'a> for CodeGen<'a, 'translation, 'data, M> 252 where 253 M: MacroAssembler, 254 { 255 type Output = (); 256 257 fn visit_i32_const(&mut self, val: i32) { 258 self.context.stack.push(Val::i32(val)); 259 } 260 261 fn visit_i64_const(&mut self, val: i64) { 262 self.context.stack.push(Val::i64(val)); 263 } 264 265 fn visit_f32_const(&mut self, val: Ieee32) { 266 self.context.stack.push(Val::f32(val)); 267 } 268 269 fn visit_f64_const(&mut self, val: Ieee64) { 270 self.context.stack.push(Val::f64(val)); 271 } 272 273 fn visit_v128_const(&mut self, val: V128) { 274 self.context.stack.push(Val::v128(val.i128())) 275 } 276 277 fn visit_f32_add(&mut self) { 278 self.context.binop( 279 self.masm, 280 OperandSize::S32, 281 &mut |masm: &mut M, dst, src, size| { 282 masm.float_add(dst, dst, src, size); 283 TypedReg::f32(dst) 284 }, 285 ); 286 } 287 288 fn visit_f64_add(&mut self) { 289 self.context.binop( 290 self.masm, 291 OperandSize::S64, 292 &mut |masm: &mut M, dst, src, size| { 293 masm.float_add(dst, dst, src, size); 294 TypedReg::f64(dst) 295 }, 296 ); 297 } 298 299 fn visit_f32_sub(&mut self) { 300 self.context.binop( 301 self.masm, 302 OperandSize::S32, 303 &mut |masm: &mut M, dst, src, size| { 304 masm.float_sub(dst, dst, src, size); 305 TypedReg::f32(dst) 306 }, 307 ); 308 } 309 310 fn visit_f64_sub(&mut self) { 311 self.context.binop( 312 self.masm, 313 OperandSize::S64, 314 &mut |masm: &mut M, dst, src, size| { 315 masm.float_sub(dst, dst, src, size); 316 TypedReg::f64(dst) 317 }, 318 ); 319 } 320 321 fn visit_f32_mul(&mut self) { 322 self.context.binop( 323 self.masm, 324 OperandSize::S32, 325 &mut |masm: &mut M, dst, src, size| { 326 masm.float_mul(dst, dst, src, size); 327 TypedReg::f32(dst) 328 }, 329 ); 330 } 331 332 fn visit_f64_mul(&mut self) { 333 self.context.binop( 334 self.masm, 335 OperandSize::S64, 336 &mut |masm: &mut M, dst, src, size| { 337 masm.float_mul(dst, dst, src, size); 338 TypedReg::f64(dst) 339 }, 340 ); 341 } 342 343 fn visit_f32_div(&mut self) { 344 self.context.binop( 345 self.masm, 346 OperandSize::S32, 347 &mut |masm: &mut M, dst, src, size| { 348 masm.float_div(dst, dst, src, size); 349 TypedReg::f32(dst) 350 }, 351 ); 352 } 353 354 fn visit_f64_div(&mut self) { 355 self.context.binop( 356 self.masm, 357 OperandSize::S64, 358 &mut |masm: &mut M, dst, src, size| { 359 masm.float_div(dst, dst, src, size); 360 TypedReg::f64(dst) 361 }, 362 ); 363 } 364 365 fn visit_f32_min(&mut self) { 366 self.context.binop( 367 self.masm, 368 OperandSize::S32, 369 &mut |masm: &mut M, dst, src, size| { 370 masm.float_min(dst, dst, src, size); 371 TypedReg::f32(dst) 372 }, 373 ); 374 } 375 376 fn visit_f64_min(&mut self) { 377 self.context.binop( 378 self.masm, 379 OperandSize::S64, 380 &mut |masm: &mut M, dst, src, size| { 381 masm.float_min(dst, dst, src, size); 382 TypedReg::f64(dst) 383 }, 384 ); 385 } 386 387 fn visit_f32_max(&mut self) { 388 self.context.binop( 389 self.masm, 390 OperandSize::S32, 391 &mut |masm: &mut M, dst, src, size| { 392 masm.float_max(dst, dst, src, size); 393 TypedReg::f32(dst) 394 }, 395 ); 396 } 397 398 fn visit_f64_max(&mut self) { 399 self.context.binop( 400 self.masm, 401 OperandSize::S64, 402 &mut |masm: &mut M, dst, src, size| { 403 masm.float_max(dst, dst, src, size); 404 TypedReg::f64(dst) 405 }, 406 ); 407 } 408 409 fn visit_f32_copysign(&mut self) { 410 self.context.binop( 411 self.masm, 412 OperandSize::S32, 413 &mut |masm: &mut M, dst, src, size| { 414 masm.float_copysign(dst, dst, src, size); 415 TypedReg::f32(dst) 416 }, 417 ); 418 } 419 420 fn visit_f64_copysign(&mut self) { 421 self.context.binop( 422 self.masm, 423 OperandSize::S64, 424 &mut |masm: &mut M, dst, src, size| { 425 masm.float_copysign(dst, dst, src, size); 426 TypedReg::f64(dst) 427 }, 428 ); 429 } 430 431 fn visit_f32_abs(&mut self) { 432 self.context 433 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 434 masm.float_abs(reg, size); 435 TypedReg::f32(reg) 436 }); 437 } 438 439 fn visit_f64_abs(&mut self) { 440 self.context 441 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 442 masm.float_abs(reg, size); 443 TypedReg::f64(reg) 444 }); 445 } 446 447 fn visit_f32_neg(&mut self) { 448 self.context 449 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 450 masm.float_neg(reg, size); 451 TypedReg::f32(reg) 452 }); 453 } 454 455 fn visit_f64_neg(&mut self) { 456 self.context 457 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 458 masm.float_neg(reg, size); 459 TypedReg::f64(reg) 460 }); 461 } 462 463 fn visit_f32_floor(&mut self) { 464 self.masm.float_round( 465 RoundingMode::Down, 466 &mut self.env, 467 &mut self.context, 468 OperandSize::S32, 469 |env, cx, masm| { 470 let builtin = env.builtins.floor_f32::<M::ABI>(); 471 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 472 }, 473 ); 474 } 475 476 fn visit_f64_floor(&mut self) { 477 self.masm.float_round( 478 RoundingMode::Down, 479 &mut self.env, 480 &mut self.context, 481 OperandSize::S64, 482 |env, cx, masm| { 483 let builtin = env.builtins.floor_f64::<M::ABI>(); 484 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 485 }, 486 ); 487 } 488 489 fn visit_f32_ceil(&mut self) { 490 self.masm.float_round( 491 RoundingMode::Up, 492 &mut self.env, 493 &mut self.context, 494 OperandSize::S32, 495 |env, cx, masm| { 496 let builtin = env.builtins.ceil_f32::<M::ABI>(); 497 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 498 }, 499 ); 500 } 501 502 fn visit_f64_ceil(&mut self) { 503 self.masm.float_round( 504 RoundingMode::Up, 505 &mut self.env, 506 &mut self.context, 507 OperandSize::S64, 508 |env, cx, masm| { 509 let builtin = env.builtins.ceil_f64::<M::ABI>(); 510 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 511 }, 512 ); 513 } 514 515 fn visit_f32_nearest(&mut self) { 516 self.masm.float_round( 517 RoundingMode::Nearest, 518 &mut self.env, 519 &mut self.context, 520 OperandSize::S32, 521 |env, cx, masm| { 522 let builtin = env.builtins.nearest_f32::<M::ABI>(); 523 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 524 }, 525 ); 526 } 527 528 fn visit_f64_nearest(&mut self) { 529 self.masm.float_round( 530 RoundingMode::Nearest, 531 &mut self.env, 532 &mut self.context, 533 OperandSize::S64, 534 |env, cx, masm| { 535 let builtin = env.builtins.nearest_f64::<M::ABI>(); 536 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 537 }, 538 ); 539 } 540 541 fn visit_f32_trunc(&mut self) { 542 self.masm.float_round( 543 RoundingMode::Zero, 544 &mut self.env, 545 &mut self.context, 546 OperandSize::S32, 547 |env, cx, masm| { 548 let builtin = env.builtins.trunc_f32::<M::ABI>(); 549 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 550 }, 551 ); 552 } 553 554 fn visit_f64_trunc(&mut self) { 555 self.masm.float_round( 556 RoundingMode::Zero, 557 &mut self.env, 558 &mut self.context, 559 OperandSize::S64, 560 |env, cx, masm| { 561 let builtin = env.builtins.trunc_f64::<M::ABI>(); 562 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)); 563 }, 564 ); 565 } 566 567 fn visit_f32_sqrt(&mut self) { 568 self.context 569 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 570 masm.float_sqrt(reg, reg, size); 571 TypedReg::f32(reg) 572 }); 573 } 574 575 fn visit_f64_sqrt(&mut self) { 576 self.context 577 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 578 masm.float_sqrt(reg, reg, size); 579 TypedReg::f64(reg) 580 }); 581 } 582 583 fn visit_f32_eq(&mut self) { 584 self.context.float_cmp_op( 585 self.masm, 586 OperandSize::S32, 587 &mut |masm: &mut M, dst, src1, src2, size| { 588 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Eq, size); 589 }, 590 ); 591 } 592 593 fn visit_f64_eq(&mut self) { 594 self.context.float_cmp_op( 595 self.masm, 596 OperandSize::S64, 597 &mut |masm: &mut M, dst, src1, src2, size| { 598 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Eq, size); 599 }, 600 ); 601 } 602 603 fn visit_f32_ne(&mut self) { 604 self.context.float_cmp_op( 605 self.masm, 606 OperandSize::S32, 607 &mut |masm: &mut M, dst, src1, src2, size| { 608 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Ne, size); 609 }, 610 ); 611 } 612 613 fn visit_f64_ne(&mut self) { 614 self.context.float_cmp_op( 615 self.masm, 616 OperandSize::S64, 617 &mut |masm: &mut M, dst, src1, src2, size| { 618 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Ne, size); 619 }, 620 ); 621 } 622 623 fn visit_f32_lt(&mut self) { 624 self.context.float_cmp_op( 625 self.masm, 626 OperandSize::S32, 627 &mut |masm: &mut M, dst, src1, src2, size| { 628 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Lt, size); 629 }, 630 ); 631 } 632 633 fn visit_f64_lt(&mut self) { 634 self.context.float_cmp_op( 635 self.masm, 636 OperandSize::S64, 637 &mut |masm: &mut M, dst, src1, src2, size| { 638 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Lt, size); 639 }, 640 ); 641 } 642 643 fn visit_f32_gt(&mut self) { 644 self.context.float_cmp_op( 645 self.masm, 646 OperandSize::S32, 647 &mut |masm: &mut M, dst, src1, src2, size| { 648 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Gt, size); 649 }, 650 ); 651 } 652 653 fn visit_f64_gt(&mut self) { 654 self.context.float_cmp_op( 655 self.masm, 656 OperandSize::S64, 657 &mut |masm: &mut M, dst, src1, src2, size| { 658 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Gt, size); 659 }, 660 ); 661 } 662 663 fn visit_f32_le(&mut self) { 664 self.context.float_cmp_op( 665 self.masm, 666 OperandSize::S32, 667 &mut |masm: &mut M, dst, src1, src2, size| { 668 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Le, size); 669 }, 670 ); 671 } 672 673 fn visit_f64_le(&mut self) { 674 self.context.float_cmp_op( 675 self.masm, 676 OperandSize::S64, 677 &mut |masm: &mut M, dst, src1, src2, size| { 678 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Le, size); 679 }, 680 ); 681 } 682 683 fn visit_f32_ge(&mut self) { 684 self.context.float_cmp_op( 685 self.masm, 686 OperandSize::S32, 687 &mut |masm: &mut M, dst, src1, src2, size| { 688 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Ge, size); 689 }, 690 ); 691 } 692 693 fn visit_f64_ge(&mut self) { 694 self.context.float_cmp_op( 695 self.masm, 696 OperandSize::S64, 697 &mut |masm: &mut M, dst, src1, src2, size| { 698 masm.float_cmp_with_set(src1, src2, dst, FloatCmpKind::Ge, size); 699 }, 700 ); 701 } 702 703 fn visit_f32_convert_i32_s(&mut self) { 704 self.context 705 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 706 masm.signed_convert(src, dst, OperandSize::S32, dst_size); 707 }); 708 } 709 710 fn visit_f32_convert_i32_u(&mut self) { 711 self.context.convert_op_with_tmp_reg( 712 self.masm, 713 WasmValType::F32, 714 RegClass::Int, 715 |masm, dst, src, tmp_gpr, dst_size| { 716 masm.unsigned_convert(src, dst, tmp_gpr, OperandSize::S32, dst_size); 717 }, 718 ); 719 } 720 721 fn visit_f32_convert_i64_s(&mut self) { 722 self.context 723 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 724 masm.signed_convert(src, dst, OperandSize::S64, dst_size); 725 }); 726 } 727 728 fn visit_f32_convert_i64_u(&mut self) { 729 self.context.convert_op_with_tmp_reg( 730 self.masm, 731 WasmValType::F32, 732 RegClass::Int, 733 |masm, dst, src, tmp_gpr, dst_size| { 734 masm.unsigned_convert(src, dst, tmp_gpr, OperandSize::S64, dst_size); 735 }, 736 ); 737 } 738 739 fn visit_f64_convert_i32_s(&mut self) { 740 self.context 741 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 742 masm.signed_convert(src, dst, OperandSize::S32, dst_size); 743 }); 744 } 745 746 fn visit_f64_convert_i32_u(&mut self) { 747 self.context.convert_op_with_tmp_reg( 748 self.masm, 749 WasmValType::F64, 750 RegClass::Int, 751 |masm, dst, src, tmp_gpr, dst_size| { 752 masm.unsigned_convert(src, dst, tmp_gpr, OperandSize::S32, dst_size); 753 }, 754 ); 755 } 756 757 fn visit_f64_convert_i64_s(&mut self) { 758 self.context 759 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 760 masm.signed_convert(src, dst, OperandSize::S64, dst_size); 761 }); 762 } 763 764 fn visit_f64_convert_i64_u(&mut self) { 765 self.context.convert_op_with_tmp_reg( 766 self.masm, 767 WasmValType::F64, 768 RegClass::Int, 769 |masm, dst, src, tmp_gpr, dst_size| { 770 masm.unsigned_convert(src, dst, tmp_gpr, OperandSize::S64, dst_size); 771 }, 772 ); 773 } 774 775 fn visit_f32_reinterpret_i32(&mut self) { 776 self.context 777 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, size| { 778 masm.reinterpret_int_as_float(src.into(), dst, size); 779 }); 780 } 781 782 fn visit_f64_reinterpret_i64(&mut self) { 783 self.context 784 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, size| { 785 masm.reinterpret_int_as_float(src.into(), dst, size); 786 }); 787 } 788 789 fn visit_f32_demote_f64(&mut self) { 790 self.context 791 .unop(self.masm, OperandSize::S64, &mut |masm, reg, _size| { 792 masm.demote(reg, reg); 793 TypedReg::f32(reg) 794 }); 795 } 796 797 fn visit_f64_promote_f32(&mut self) { 798 self.context 799 .unop(self.masm, OperandSize::S32, &mut |masm, reg, _size| { 800 masm.promote(reg, reg); 801 TypedReg::f64(reg) 802 }); 803 } 804 805 fn visit_i32_add(&mut self) { 806 self.context.i32_binop(self.masm, |masm, dst, src, size| { 807 masm.add(dst, dst, src, size); 808 TypedReg::i32(dst) 809 }); 810 } 811 812 fn visit_i64_add(&mut self) { 813 self.context.i64_binop(self.masm, |masm, dst, src, size| { 814 masm.add(dst, dst, src, size); 815 TypedReg::i64(dst) 816 }); 817 } 818 819 fn visit_i32_sub(&mut self) { 820 self.context.i32_binop(self.masm, |masm, dst, src, size| { 821 masm.sub(dst, dst, src, size); 822 TypedReg::i32(dst) 823 }); 824 } 825 826 fn visit_i64_sub(&mut self) { 827 self.context.i64_binop(self.masm, |masm, dst, src, size| { 828 masm.sub(dst, dst, src, size); 829 TypedReg::i64(dst) 830 }); 831 } 832 833 fn visit_i32_mul(&mut self) { 834 self.context.i32_binop(self.masm, |masm, dst, src, size| { 835 masm.mul(dst, dst, src, size); 836 TypedReg::i32(dst) 837 }); 838 } 839 840 fn visit_i64_mul(&mut self) { 841 self.context.i64_binop(self.masm, |masm, dst, src, size| { 842 masm.mul(dst, dst, src, size); 843 TypedReg::i64(dst) 844 }); 845 } 846 847 fn visit_i32_div_s(&mut self) { 848 use DivKind::*; 849 use OperandSize::*; 850 851 self.masm.div(&mut self.context, Signed, S32); 852 } 853 854 fn visit_i32_div_u(&mut self) { 855 use DivKind::*; 856 use OperandSize::*; 857 858 self.masm.div(&mut self.context, Unsigned, S32); 859 } 860 861 fn visit_i64_div_s(&mut self) { 862 use DivKind::*; 863 use OperandSize::*; 864 865 self.masm.div(&mut self.context, Signed, S64); 866 } 867 868 fn visit_i64_div_u(&mut self) { 869 use DivKind::*; 870 use OperandSize::*; 871 872 self.masm.div(&mut self.context, Unsigned, S64); 873 } 874 875 fn visit_i32_rem_s(&mut self) { 876 use OperandSize::*; 877 use RemKind::*; 878 879 self.masm.rem(&mut self.context, Signed, S32); 880 } 881 882 fn visit_i32_rem_u(&mut self) { 883 use OperandSize::*; 884 use RemKind::*; 885 886 self.masm.rem(&mut self.context, Unsigned, S32); 887 } 888 889 fn visit_i64_rem_s(&mut self) { 890 use OperandSize::*; 891 use RemKind::*; 892 893 self.masm.rem(&mut self.context, Signed, S64); 894 } 895 896 fn visit_i64_rem_u(&mut self) { 897 use OperandSize::*; 898 use RemKind::*; 899 900 self.masm.rem(&mut self.context, Unsigned, S64); 901 } 902 903 fn visit_i32_eq(&mut self) { 904 self.cmp_i32s(IntCmpKind::Eq); 905 } 906 907 fn visit_i64_eq(&mut self) { 908 self.cmp_i64s(IntCmpKind::Eq); 909 } 910 911 fn visit_i32_ne(&mut self) { 912 self.cmp_i32s(IntCmpKind::Ne); 913 } 914 915 fn visit_i64_ne(&mut self) { 916 self.cmp_i64s(IntCmpKind::Ne); 917 } 918 919 fn visit_i32_lt_s(&mut self) { 920 self.cmp_i32s(IntCmpKind::LtS); 921 } 922 923 fn visit_i64_lt_s(&mut self) { 924 self.cmp_i64s(IntCmpKind::LtS); 925 } 926 927 fn visit_i32_lt_u(&mut self) { 928 self.cmp_i32s(IntCmpKind::LtU); 929 } 930 931 fn visit_i64_lt_u(&mut self) { 932 self.cmp_i64s(IntCmpKind::LtU); 933 } 934 935 fn visit_i32_le_s(&mut self) { 936 self.cmp_i32s(IntCmpKind::LeS); 937 } 938 939 fn visit_i64_le_s(&mut self) { 940 self.cmp_i64s(IntCmpKind::LeS); 941 } 942 943 fn visit_i32_le_u(&mut self) { 944 self.cmp_i32s(IntCmpKind::LeU); 945 } 946 947 fn visit_i64_le_u(&mut self) { 948 self.cmp_i64s(IntCmpKind::LeU); 949 } 950 951 fn visit_i32_gt_s(&mut self) { 952 self.cmp_i32s(IntCmpKind::GtS); 953 } 954 955 fn visit_i64_gt_s(&mut self) { 956 self.cmp_i64s(IntCmpKind::GtS); 957 } 958 959 fn visit_i32_gt_u(&mut self) { 960 self.cmp_i32s(IntCmpKind::GtU); 961 } 962 963 fn visit_i64_gt_u(&mut self) { 964 self.cmp_i64s(IntCmpKind::GtU); 965 } 966 967 fn visit_i32_ge_s(&mut self) { 968 self.cmp_i32s(IntCmpKind::GeS); 969 } 970 971 fn visit_i64_ge_s(&mut self) { 972 self.cmp_i64s(IntCmpKind::GeS); 973 } 974 975 fn visit_i32_ge_u(&mut self) { 976 self.cmp_i32s(IntCmpKind::GeU); 977 } 978 979 fn visit_i64_ge_u(&mut self) { 980 self.cmp_i64s(IntCmpKind::GeU); 981 } 982 983 fn visit_i32_eqz(&mut self) { 984 use OperandSize::*; 985 986 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 987 masm.cmp_with_set(RegImm::i32(0), reg.into(), IntCmpKind::Eq, size); 988 TypedReg::i32(reg) 989 }); 990 } 991 992 fn visit_i64_eqz(&mut self) { 993 use OperandSize::*; 994 995 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 996 masm.cmp_with_set(RegImm::i64(0), reg.into(), IntCmpKind::Eq, size); 997 TypedReg::i32(reg) // Return value for `i64.eqz` is an `i32`. 998 }); 999 } 1000 1001 fn visit_i32_clz(&mut self) { 1002 use OperandSize::*; 1003 1004 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 1005 masm.clz(reg, reg, size); 1006 TypedReg::i32(reg) 1007 }); 1008 } 1009 1010 fn visit_i64_clz(&mut self) { 1011 use OperandSize::*; 1012 1013 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 1014 masm.clz(reg, reg, size); 1015 TypedReg::i64(reg) 1016 }); 1017 } 1018 1019 fn visit_i32_ctz(&mut self) { 1020 use OperandSize::*; 1021 1022 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 1023 masm.ctz(reg, reg, size); 1024 TypedReg::i32(reg) 1025 }); 1026 } 1027 1028 fn visit_i64_ctz(&mut self) { 1029 use OperandSize::*; 1030 1031 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 1032 masm.ctz(reg, reg, size); 1033 TypedReg::i64(reg) 1034 }); 1035 } 1036 1037 fn visit_i32_and(&mut self) { 1038 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1039 masm.and(dst, dst, src, size); 1040 TypedReg::i32(dst) 1041 }); 1042 } 1043 1044 fn visit_i64_and(&mut self) { 1045 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1046 masm.and(dst, dst, src, size); 1047 TypedReg::i64(dst) 1048 }); 1049 } 1050 1051 fn visit_i32_or(&mut self) { 1052 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1053 masm.or(dst, dst, src, size); 1054 TypedReg::i32(dst) 1055 }); 1056 } 1057 1058 fn visit_i64_or(&mut self) { 1059 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1060 masm.or(dst, dst, src, size); 1061 TypedReg::i64(dst) 1062 }); 1063 } 1064 1065 fn visit_i32_xor(&mut self) { 1066 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1067 masm.xor(dst, dst, src, size); 1068 TypedReg::i32(dst) 1069 }); 1070 } 1071 1072 fn visit_i64_xor(&mut self) { 1073 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1074 masm.xor(dst, dst, src, size); 1075 TypedReg::i64(dst) 1076 }); 1077 } 1078 1079 fn visit_i32_shl(&mut self) { 1080 use ShiftKind::*; 1081 1082 self.context.i32_shift(self.masm, Shl); 1083 } 1084 1085 fn visit_i64_shl(&mut self) { 1086 use ShiftKind::*; 1087 1088 self.context.i64_shift(self.masm, Shl); 1089 } 1090 1091 fn visit_i32_shr_s(&mut self) { 1092 use ShiftKind::*; 1093 1094 self.context.i32_shift(self.masm, ShrS); 1095 } 1096 1097 fn visit_i64_shr_s(&mut self) { 1098 use ShiftKind::*; 1099 1100 self.context.i64_shift(self.masm, ShrS); 1101 } 1102 1103 fn visit_i32_shr_u(&mut self) { 1104 use ShiftKind::*; 1105 1106 self.context.i32_shift(self.masm, ShrU); 1107 } 1108 1109 fn visit_i64_shr_u(&mut self) { 1110 use ShiftKind::*; 1111 1112 self.context.i64_shift(self.masm, ShrU); 1113 } 1114 1115 fn visit_i32_rotl(&mut self) { 1116 use ShiftKind::*; 1117 1118 self.context.i32_shift(self.masm, Rotl); 1119 } 1120 1121 fn visit_i64_rotl(&mut self) { 1122 use ShiftKind::*; 1123 1124 self.context.i64_shift(self.masm, Rotl); 1125 } 1126 1127 fn visit_i32_rotr(&mut self) { 1128 use ShiftKind::*; 1129 1130 self.context.i32_shift(self.masm, Rotr); 1131 } 1132 1133 fn visit_i64_rotr(&mut self) { 1134 use ShiftKind::*; 1135 1136 self.context.i64_shift(self.masm, Rotr); 1137 } 1138 1139 fn visit_end(&mut self) { 1140 if !self.context.reachable { 1141 self.handle_unreachable_end(); 1142 } else { 1143 let mut control = self.control_frames.pop().unwrap(); 1144 control.emit_end(self.masm, &mut self.context); 1145 } 1146 } 1147 1148 fn visit_i32_popcnt(&mut self) { 1149 use OperandSize::*; 1150 self.masm.popcnt(&mut self.context, S32); 1151 } 1152 1153 fn visit_i64_popcnt(&mut self) { 1154 use OperandSize::*; 1155 1156 self.masm.popcnt(&mut self.context, S64); 1157 } 1158 1159 fn visit_i32_wrap_i64(&mut self) { 1160 use OperandSize::*; 1161 1162 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1163 masm.wrap(reg, reg); 1164 TypedReg::i32(reg) 1165 }); 1166 } 1167 1168 fn visit_i64_extend_i32_s(&mut self) { 1169 use OperandSize::*; 1170 1171 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1172 masm.extend(reg, reg, ExtendKind::I64ExtendI32S); 1173 TypedReg::i64(reg) 1174 }); 1175 } 1176 1177 fn visit_i64_extend_i32_u(&mut self) { 1178 use OperandSize::*; 1179 1180 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1181 masm.extend(reg, reg, ExtendKind::I64ExtendI32U); 1182 TypedReg::i64(reg) 1183 }); 1184 } 1185 1186 fn visit_i32_extend8_s(&mut self) { 1187 use OperandSize::*; 1188 1189 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1190 masm.extend(reg, reg, ExtendKind::I32Extend8S); 1191 TypedReg::i32(reg) 1192 }); 1193 } 1194 1195 fn visit_i32_extend16_s(&mut self) { 1196 use OperandSize::*; 1197 1198 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1199 masm.extend(reg, reg, ExtendKind::I32Extend16S); 1200 TypedReg::i32(reg) 1201 }); 1202 } 1203 1204 fn visit_i64_extend8_s(&mut self) { 1205 use OperandSize::*; 1206 1207 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1208 masm.extend(reg, reg, ExtendKind::I64Extend8S); 1209 TypedReg::i64(reg) 1210 }); 1211 } 1212 1213 fn visit_i64_extend16_s(&mut self) { 1214 use OperandSize::*; 1215 1216 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1217 masm.extend(reg, reg, ExtendKind::I64Extend16S); 1218 TypedReg::i64(reg) 1219 }); 1220 } 1221 1222 fn visit_i64_extend32_s(&mut self) { 1223 use OperandSize::*; 1224 1225 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1226 masm.extend(reg, reg, ExtendKind::I64Extend32S); 1227 TypedReg::i64(reg) 1228 }); 1229 } 1230 1231 fn visit_i32_trunc_f32_s(&mut self) { 1232 use OperandSize::*; 1233 1234 self.context 1235 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1236 masm.signed_truncate(src, dst, S32, dst_size, TruncKind::Unchecked); 1237 }); 1238 } 1239 1240 fn visit_i32_trunc_f32_u(&mut self) { 1241 use OperandSize::*; 1242 1243 self.context.convert_op_with_tmp_reg( 1244 self.masm, 1245 WasmValType::I32, 1246 RegClass::Float, 1247 |masm, dst, src, tmp_fpr, dst_size| { 1248 masm.unsigned_truncate(src, dst, tmp_fpr, S32, dst_size, TruncKind::Unchecked); 1249 }, 1250 ); 1251 } 1252 1253 fn visit_i32_trunc_f64_s(&mut self) { 1254 use OperandSize::*; 1255 1256 self.context 1257 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1258 masm.signed_truncate(src, dst, S64, dst_size, TruncKind::Unchecked); 1259 }); 1260 } 1261 1262 fn visit_i32_trunc_f64_u(&mut self) { 1263 use OperandSize::*; 1264 1265 self.context.convert_op_with_tmp_reg( 1266 self.masm, 1267 WasmValType::I32, 1268 RegClass::Float, 1269 |masm, dst, src, tmp_fpr, dst_size| { 1270 masm.unsigned_truncate(src, dst, tmp_fpr, S64, dst_size, TruncKind::Unchecked); 1271 }, 1272 ); 1273 } 1274 1275 fn visit_i64_trunc_f32_s(&mut self) { 1276 use OperandSize::*; 1277 1278 self.context 1279 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1280 masm.signed_truncate(src, dst, S32, dst_size, TruncKind::Unchecked); 1281 }); 1282 } 1283 1284 fn visit_i64_trunc_f32_u(&mut self) { 1285 use OperandSize::*; 1286 1287 self.context.convert_op_with_tmp_reg( 1288 self.masm, 1289 WasmValType::I64, 1290 RegClass::Float, 1291 |masm, dst, src, tmp_fpr, dst_size| { 1292 masm.unsigned_truncate(src, dst, tmp_fpr, S32, dst_size, TruncKind::Unchecked); 1293 }, 1294 ); 1295 } 1296 1297 fn visit_i64_trunc_f64_s(&mut self) { 1298 use OperandSize::*; 1299 1300 self.context 1301 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1302 masm.signed_truncate(src, dst, S64, dst_size, TruncKind::Unchecked); 1303 }); 1304 } 1305 1306 fn visit_i64_trunc_f64_u(&mut self) { 1307 use OperandSize::*; 1308 1309 self.context.convert_op_with_tmp_reg( 1310 self.masm, 1311 WasmValType::I64, 1312 RegClass::Float, 1313 |masm, dst, src, tmp_fpr, dst_size| { 1314 masm.unsigned_truncate(src, dst, tmp_fpr, S64, dst_size, TruncKind::Unchecked); 1315 }, 1316 ); 1317 } 1318 1319 fn visit_i32_reinterpret_f32(&mut self) { 1320 self.context 1321 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, size| { 1322 masm.reinterpret_float_as_int(src.into(), dst, size); 1323 }); 1324 } 1325 1326 fn visit_i64_reinterpret_f64(&mut self) { 1327 self.context 1328 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, size| { 1329 masm.reinterpret_float_as_int(src.into(), dst, size); 1330 }); 1331 } 1332 1333 fn visit_local_get(&mut self, index: u32) { 1334 use WasmValType::*; 1335 let context = &mut self.context; 1336 let slot = context.frame.get_wasm_local(index); 1337 match slot.ty { 1338 I32 | I64 | F32 | F64 | V128 => context.stack.push(Val::local(index, slot.ty)), 1339 Ref(rt) => match rt.heap_type { 1340 WasmHeapType::Func => context.stack.push(Val::local(index, slot.ty)), 1341 WasmHeapType::Extern => { 1342 self.found_unsupported_instruction = 1343 Some("unsupported local.get of externref local"); 1344 } 1345 ht => unimplemented!("Support for WasmHeapType: {ht}"), 1346 }, 1347 } 1348 } 1349 1350 fn visit_local_set(&mut self, index: u32) { 1351 let src = self.emit_set_local(index); 1352 self.context.free_reg(src); 1353 } 1354 1355 fn visit_call(&mut self, index: u32) { 1356 let callee = self.env.callee_from_index(FuncIndex::from_u32(index)); 1357 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee) 1358 } 1359 1360 fn visit_call_indirect(&mut self, type_index: u32, table_index: u32) { 1361 // Spill now because `emit_lazy_init_funcref` and the `FnCall::emit` 1362 // invocations will both trigger spills since they both call functions. 1363 // However, the machine instructions for the spill emitted by 1364 // `emit_lazy_funcref` will be jumped over if the funcref was previously 1365 // initialized which may result in the machine stack becoming 1366 // unbalanced. 1367 self.context.spill(self.masm); 1368 1369 let type_index = TypeIndex::from_u32(type_index); 1370 let table_index = TableIndex::from_u32(table_index); 1371 1372 self.emit_lazy_init_funcref(table_index); 1373 1374 // Perform the indirect call. 1375 // This code assumes that [`Self::emit_lazy_init_funcref`] will 1376 // push the funcref to the value stack. 1377 match self.env.translation.module.table_plans[table_index].style { 1378 TableStyle::CallerChecksSignature { lazy_init: true } => { 1379 let funcref_ptr = self.context.stack.peek().map(|v| v.unwrap_reg()).unwrap(); 1380 self.masm 1381 .trapz(funcref_ptr.into(), TrapCode::IndirectCallToNull); 1382 self.emit_typecheck_funcref(funcref_ptr.into(), type_index); 1383 } 1384 _ => unimplemented!("Support for eager table init"), 1385 } 1386 1387 let callee = self.env.funcref(type_index); 1388 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee) 1389 } 1390 1391 fn visit_table_init(&mut self, elem: u32, table: u32) { 1392 debug_assert!(self.context.stack.len() >= 3); 1393 let at = self.context.stack.len() - 3; 1394 1395 self.context 1396 .stack 1397 .insert_many(at, &[table.try_into().unwrap(), elem.try_into().unwrap()]); 1398 1399 let builtin = self.env.builtins.table_init::<M::ABI, M::Ptr>(); 1400 FnCall::emit::<M>( 1401 &mut self.env, 1402 self.masm, 1403 &mut self.context, 1404 Callee::Builtin(builtin.clone()), 1405 ) 1406 } 1407 1408 fn visit_table_copy(&mut self, dst: u32, src: u32) { 1409 debug_assert!(self.context.stack.len() >= 3); 1410 let at = self.context.stack.len() - 3; 1411 self.context 1412 .stack 1413 .insert_many(at, &[dst.try_into().unwrap(), src.try_into().unwrap()]); 1414 1415 let builtin = self.env.builtins.table_copy::<M::ABI, M::Ptr>(); 1416 FnCall::emit::<M>( 1417 &mut self.env, 1418 self.masm, 1419 &mut self.context, 1420 Callee::Builtin(builtin), 1421 ) 1422 } 1423 1424 fn visit_table_get(&mut self, table: u32) { 1425 let table_index = TableIndex::from_u32(table); 1426 let plan = self.env.table_plan(table_index); 1427 let heap_type = plan.table.wasm_ty.heap_type; 1428 let style = &plan.style; 1429 1430 match heap_type { 1431 WasmHeapType::Func => match style { 1432 TableStyle::CallerChecksSignature { lazy_init: true } => { 1433 self.emit_lazy_init_funcref(table_index) 1434 } 1435 _ => unimplemented!("Support for eager table init"), 1436 }, 1437 WasmHeapType::Extern => { 1438 self.found_unsupported_instruction = 1439 Some("unsupported table.get of externref table"); 1440 } 1441 t => { 1442 unimplemented!("Support for WasmHeapType: {t}") 1443 } 1444 } 1445 } 1446 1447 fn visit_table_grow(&mut self, table: u32) { 1448 let table_index = TableIndex::from_u32(table); 1449 let table_plan = self.env.table_plan(table_index); 1450 let builtin = match table_plan.table.wasm_ty.heap_type { 1451 WasmHeapType::Func => self.env.builtins.table_grow_func_ref::<M::ABI, M::Ptr>(), 1452 ty => unimplemented!("Support for HeapType: {ty}"), 1453 }; 1454 1455 let len = self.context.stack.len(); 1456 // table.grow` requires at least 2 elements on the value stack. 1457 debug_assert!(len >= 2); 1458 let at = len - 2; 1459 1460 // The table_grow builtin expects the parameters in a different 1461 // order. 1462 // The value stack at this point should contain: 1463 // [ init_value | delta ] (stack top) 1464 // but the builtin function expects the init value as the last 1465 // argument. 1466 self.context.stack.inner_mut().swap(len - 1, len - 2); 1467 self.context 1468 .stack 1469 .insert_many(at, &[table.try_into().unwrap()]); 1470 1471 FnCall::emit::<M>( 1472 &mut self.env, 1473 self.masm, 1474 &mut self.context, 1475 Callee::Builtin(builtin.clone()), 1476 ) 1477 } 1478 1479 fn visit_table_size(&mut self, table: u32) { 1480 let table_index = TableIndex::from_u32(table); 1481 let table_data = self.env.resolve_table_data(table_index); 1482 self.emit_compute_table_size(&table_data); 1483 } 1484 1485 fn visit_table_fill(&mut self, table: u32) { 1486 let table_index = TableIndex::from_u32(table); 1487 let table_plan = self.env.table_plan(table_index); 1488 let builtin = match table_plan.table.wasm_ty.heap_type { 1489 WasmHeapType::Func => self.env.builtins.table_fill_func_ref::<M::ABI, M::Ptr>(), 1490 ty => unimplemented!("Support for heap type: {ty}"), 1491 }; 1492 1493 let len = self.context.stack.len(); 1494 debug_assert!(len >= 3); 1495 let at = len - 3; 1496 self.context 1497 .stack 1498 .insert_many(at, &[table.try_into().unwrap()]); 1499 FnCall::emit::<M>( 1500 &mut self.env, 1501 self.masm, 1502 &mut self.context, 1503 Callee::Builtin(builtin.clone()), 1504 ) 1505 } 1506 1507 fn visit_table_set(&mut self, table: u32) { 1508 let ptr_type = self.env.ptr_type(); 1509 let table_index = TableIndex::from_u32(table); 1510 let table_data = self.env.resolve_table_data(table_index); 1511 let plan = self.env.table_plan(table_index); 1512 match plan.table.wasm_ty.heap_type { 1513 WasmHeapType::Func => match plan.style { 1514 TableStyle::CallerChecksSignature { lazy_init: true } => { 1515 let value = self.context.pop_to_reg(self.masm, None); 1516 let index = self.context.pop_to_reg(self.masm, None); 1517 let base = self.context.any_gpr(self.masm); 1518 let elem_addr = 1519 self.emit_compute_table_elem_addr(index.into(), base, &table_data); 1520 // Set the initialized bit. 1521 self.masm.or( 1522 value.into(), 1523 value.into(), 1524 RegImm::i64(FUNCREF_INIT_BIT as i64), 1525 ptr_type.into(), 1526 ); 1527 1528 self.masm.store_ptr(value.into(), elem_addr); 1529 1530 self.context.free_reg(value); 1531 self.context.free_reg(index); 1532 self.context.free_reg(base); 1533 } 1534 _ => unimplemented!("Support for eager table init"), 1535 }, 1536 ty => unimplemented!("Support for WasmHeapType: {ty}"), 1537 }; 1538 } 1539 1540 fn visit_elem_drop(&mut self, index: u32) { 1541 let elem_drop = self.env.builtins.elem_drop::<M::ABI, M::Ptr>(); 1542 self.context.stack.extend([index.try_into().unwrap()]); 1543 FnCall::emit::<M>( 1544 &mut self.env, 1545 self.masm, 1546 &mut self.context, 1547 Callee::Builtin(elem_drop), 1548 ) 1549 } 1550 1551 fn visit_memory_init(&mut self, data_index: u32, mem: u32) { 1552 debug_assert!(self.context.stack.len() >= 3); 1553 let at = self.context.stack.len() - 3; 1554 self.context.stack.insert_many( 1555 at, 1556 &[mem.try_into().unwrap(), data_index.try_into().unwrap()], 1557 ); 1558 let builtin = self.env.builtins.memory_init::<M::ABI, M::Ptr>(); 1559 FnCall::emit::<M>( 1560 &mut self.env, 1561 self.masm, 1562 &mut self.context, 1563 Callee::Builtin(builtin), 1564 ) 1565 } 1566 1567 fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) { 1568 // At this point, the stack is expected to contain: 1569 // [ dst_offset, src_offset, len ] 1570 // The following code inserts the missing params, so that stack contains: 1571 // [ vmctx, dst_mem, dst_offset, src_mem, src_offset, len ] 1572 // Which is the order expected by the builtin function. 1573 debug_assert!(self.context.stack.len() >= 3); 1574 let at = self.context.stack.len() - 2; 1575 self.context 1576 .stack 1577 .insert_many(at, &[src_mem.try_into().unwrap()]); 1578 1579 // One element was inserted above, so instead of 3, we use 4. 1580 let at = self.context.stack.len() - 4; 1581 self.context 1582 .stack 1583 .insert_many(at, &[dst_mem.try_into().unwrap()]); 1584 1585 let builtin = self.env.builtins.memory_copy::<M::ABI, M::Ptr>(); 1586 1587 FnCall::emit::<M>( 1588 &mut self.env, 1589 self.masm, 1590 &mut self.context, 1591 Callee::Builtin(builtin), 1592 ) 1593 } 1594 1595 fn visit_memory_fill(&mut self, mem: u32) { 1596 debug_assert!(self.context.stack.len() >= 3); 1597 let at = self.context.stack.len() - 3; 1598 1599 self.context 1600 .stack 1601 .insert_many(at, &[mem.try_into().unwrap()]); 1602 1603 let builtin = self.env.builtins.memory_fill::<M::ABI, M::Ptr>(); 1604 FnCall::emit::<M>( 1605 &mut self.env, 1606 self.masm, 1607 &mut self.context, 1608 Callee::Builtin(builtin), 1609 ) 1610 } 1611 1612 fn visit_memory_size(&mut self, mem: u32) { 1613 let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem)); 1614 self.emit_compute_memory_size(&heap); 1615 } 1616 1617 fn visit_memory_grow(&mut self, mem: u32) { 1618 debug_assert!(self.context.stack.len() >= 1); 1619 // The stack at this point contains: [ delta ] 1620 // The desired state is 1621 // [ vmctx, delta, index ] 1622 self.context.stack.extend([mem.try_into().unwrap()]); 1623 1624 let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem)); 1625 let builtin = self.env.builtins.memory32_grow::<M::ABI, M::Ptr>(); 1626 FnCall::emit::<M>( 1627 &mut self.env, 1628 self.masm, 1629 &mut self.context, 1630 Callee::Builtin(builtin), 1631 ); 1632 1633 // The memory32_grow builtin returns a pointer type, therefore we must 1634 // ensure that the return type is representative of the address space of 1635 // the heap type. 1636 match (self.env.ptr_type(), heap.ty) { 1637 (WasmValType::I64, WasmValType::I64) => {} 1638 // When the heap type is smaller than the pointer type, we adjust 1639 // the result of the memory32_grow builtin. 1640 (WasmValType::I64, WasmValType::I32) => { 1641 let top: Reg = self.context.pop_to_reg(self.masm, None).into(); 1642 self.masm.wrap(top.into(), top.into()); 1643 self.context.stack.push(TypedReg::i32(top).into()); 1644 } 1645 _ => unimplemented!("Support for 32-bit platforms"), 1646 } 1647 } 1648 1649 fn visit_data_drop(&mut self, data_index: u32) { 1650 self.context.stack.extend([data_index.try_into().unwrap()]); 1651 1652 let builtin = self.env.builtins.data_drop::<M::ABI, M::Ptr>(); 1653 FnCall::emit::<M>( 1654 &mut self.env, 1655 self.masm, 1656 &mut self.context, 1657 Callee::Builtin(builtin), 1658 ) 1659 } 1660 1661 fn visit_nop(&mut self) {} 1662 1663 fn visit_if(&mut self, blockty: BlockType) { 1664 self.control_frames.push(ControlStackFrame::r#if( 1665 self.env.resolve_block_sig(blockty), 1666 self.masm, 1667 &mut self.context, 1668 )); 1669 } 1670 1671 fn visit_else(&mut self) { 1672 if !self.context.reachable { 1673 self.handle_unreachable_else(); 1674 } else { 1675 let control = self 1676 .control_frames 1677 .last_mut() 1678 .unwrap_or_else(|| panic!("Expected active control stack frame for else")); 1679 control.emit_else(self.masm, &mut self.context); 1680 } 1681 } 1682 1683 fn visit_block(&mut self, blockty: BlockType) { 1684 self.control_frames.push(ControlStackFrame::block( 1685 self.env.resolve_block_sig(blockty), 1686 self.masm, 1687 &mut self.context, 1688 )); 1689 } 1690 1691 fn visit_loop(&mut self, blockty: BlockType) { 1692 self.control_frames.push(ControlStackFrame::r#loop( 1693 self.env.resolve_block_sig(blockty), 1694 self.masm, 1695 &mut self.context, 1696 )); 1697 } 1698 1699 fn visit_br(&mut self, depth: u32) { 1700 let index = control_index(depth, self.control_frames.len()); 1701 let frame = &mut self.control_frames[index]; 1702 self.context 1703 .unconditional_jump(frame, self.masm, |masm, cx, frame| { 1704 frame 1705 .pop_abi_results::<M, _>(cx, masm, |results, _, _| results.ret_area().copied()); 1706 }); 1707 } 1708 1709 fn visit_br_if(&mut self, depth: u32) { 1710 let index = control_index(depth, self.control_frames.len()); 1711 let frame = &mut self.control_frames[index]; 1712 frame.set_as_target(); 1713 1714 let top = { 1715 let top = self.context.without::<TypedReg, M, _>( 1716 frame.results::<M>().regs(), 1717 self.masm, 1718 |ctx, masm| ctx.pop_to_reg(masm, None), 1719 ); 1720 // Explicitly save any live registers and locals before setting up 1721 // the branch state. 1722 // In some cases, calculating the `top` value above, will result in 1723 // a spill, thus the following one will result in a no-op. 1724 self.context.spill(self.masm); 1725 frame.top_abi_results::<M, _>( 1726 &mut self.context, 1727 self.masm, 1728 |results, context, masm| { 1729 // In the case of `br_if` there's a possibility that we'll 1730 // exit early from the block or fallthrough, for 1731 // a fallthrough, we cannot rely on the pre-computed return area; 1732 // it must be recalculated so that any values that are 1733 // generated are correctly placed near the current stack 1734 // pointer. 1735 results.on_stack().then(|| { 1736 let stack_consumed = context.stack.sizeof(results.stack_operands_len()); 1737 let base = masm.sp_offset().as_u32() - stack_consumed; 1738 let offs = base + results.size(); 1739 RetArea::sp(SPOffset::from_u32(offs)) 1740 }) 1741 }, 1742 ); 1743 top 1744 }; 1745 1746 // Emit instructions to balance the machine stack if the frame has 1747 // a different offset. 1748 let current_sp_offset = self.masm.sp_offset(); 1749 let results_size = frame.results::<M>().size(); 1750 let state = frame.stack_state(); 1751 let (label, cmp, needs_cleanup) = if current_sp_offset > state.target_offset { 1752 (self.masm.get_label(), IntCmpKind::Eq, true) 1753 } else { 1754 (*frame.label(), IntCmpKind::Ne, false) 1755 }; 1756 1757 self.masm 1758 .branch(cmp, top.reg.into(), top.reg.into(), label, OperandSize::S32); 1759 self.context.free_reg(top); 1760 1761 if needs_cleanup { 1762 // Emit instructions to balance the stack and jump if not falling 1763 // through. 1764 self.masm.memmove( 1765 current_sp_offset, 1766 state.target_offset, 1767 results_size, 1768 MemMoveDirection::LowToHigh, 1769 ); 1770 self.masm.ensure_sp_for_jump(state.target_offset); 1771 self.masm.jmp(*frame.label()); 1772 1773 // Restore sp_offset to what it was for falling through and emit 1774 // fallthrough label. 1775 self.masm.reset_stack_pointer(current_sp_offset); 1776 self.masm.bind(label); 1777 } 1778 } 1779 1780 fn visit_br_table(&mut self, targets: BrTable<'a>) { 1781 // +1 to account for the default target. 1782 let len = targets.len() + 1; 1783 // SmallVec<[_; 5]> to match the binary emission layer (e.g 1784 // see `JmpTableSeq'), but here we use 5 instead since we 1785 // bundle the default target as the last element in the array. 1786 let labels: SmallVec<[_; 5]> = (0..len).map(|_| self.masm.get_label()).collect(); 1787 1788 let default_index = control_index(targets.default(), self.control_frames.len()); 1789 let default_frame = &mut self.control_frames[default_index]; 1790 let default_result = default_frame.results::<M>(); 1791 1792 let (index, tmp) = { 1793 let index_and_tmp = self.context.without::<(TypedReg, _), M, _>( 1794 default_result.regs(), 1795 self.masm, 1796 |cx, masm| (cx.pop_to_reg(masm, None), cx.any_gpr(masm)), 1797 ); 1798 1799 // Materialize any constants or locals into their result representation, 1800 // so that when reachability is restored, they are correctly located. 1801 default_frame.top_abi_results::<M, _>(&mut self.context, self.masm, |results, _, _| { 1802 results.ret_area().copied() 1803 }); 1804 index_and_tmp 1805 }; 1806 1807 self.masm.jmp_table(&labels, index.into(), tmp); 1808 // Save the original stack pointer offset; we will reset the stack 1809 // pointer to this offset after jumping to each of the targets. Each 1810 // jump might adjust the stack according to the base offset of the 1811 // target. 1812 let current_sp = self.masm.sp_offset(); 1813 1814 for (t, l) in targets 1815 .targets() 1816 .into_iter() 1817 .chain(std::iter::once(Ok(targets.default()))) 1818 .zip(labels.iter()) 1819 { 1820 let control_index = control_index(t.unwrap(), self.control_frames.len()); 1821 let frame = &mut self.control_frames[control_index]; 1822 // Reset the stack pointer to its original offset. This is needed 1823 // because each jump will potentially adjust the stack pointer 1824 // according to the base offset of the target. 1825 self.masm.reset_stack_pointer(current_sp); 1826 1827 // NB: We don't perform any result handling as it was 1828 // already taken care of above before jumping to the 1829 // jump table. 1830 self.masm.bind(*l); 1831 // Ensure that the stack pointer is correctly positioned before 1832 // jumping to the jump table code. 1833 let state = frame.stack_state(); 1834 self.masm.ensure_sp_for_jump(state.target_offset); 1835 self.masm.jmp(*frame.label()); 1836 frame.set_as_target(); 1837 } 1838 // Finally reset the stack pointer to the original location. 1839 // The reachability analysis, will ensure it's correctly located 1840 // once reachability is restored. 1841 self.masm.reset_stack_pointer(current_sp); 1842 self.context.reachable = false; 1843 self.context.free_reg(index.reg); 1844 self.context.free_reg(tmp); 1845 } 1846 1847 fn visit_return(&mut self) { 1848 // Grab the outermost frame, which is the function's body 1849 // frame. We don't rely on [`codegen::control_index`] since 1850 // this frame is implicit and we know that it should exist at 1851 // index 0. 1852 let outermost = &mut self.control_frames[0]; 1853 self.context 1854 .unconditional_jump(outermost, self.masm, |masm, cx, frame| { 1855 frame 1856 .pop_abi_results::<M, _>(cx, masm, |results, _, _| results.ret_area().copied()); 1857 }); 1858 } 1859 1860 fn visit_unreachable(&mut self) { 1861 self.masm.unreachable(); 1862 self.context.reachable = false; 1863 // Set the implicit outermost frame as target to perform the necessary 1864 // stack clean up. 1865 let outermost = &mut self.control_frames[0]; 1866 outermost.set_as_target(); 1867 } 1868 1869 fn visit_local_tee(&mut self, index: u32) { 1870 let typed_reg = self.emit_set_local(index); 1871 self.context.stack.push(typed_reg.into()); 1872 } 1873 1874 fn visit_global_get(&mut self, global_index: u32) { 1875 let index = GlobalIndex::from_u32(global_index); 1876 let (ty, addr) = self.emit_get_global_addr(index); 1877 let dst = self.context.reg_for_type(ty, self.masm); 1878 self.masm.load(addr, dst, ty.into()); 1879 self.context.stack.push(Val::reg(dst, ty)); 1880 } 1881 1882 fn visit_global_set(&mut self, global_index: u32) { 1883 let index = GlobalIndex::from_u32(global_index); 1884 let (ty, addr) = self.emit_get_global_addr(index); 1885 1886 let typed_reg = self.context.pop_to_reg(self.masm, None); 1887 self.context.free_reg(typed_reg.reg); 1888 self.masm.store(typed_reg.reg.into(), addr, ty.into()); 1889 } 1890 1891 fn visit_drop(&mut self) { 1892 self.context.drop_last(1, |regalloc, val| match val { 1893 Val::Reg(tr) => regalloc.free(tr.reg.into()), 1894 Val::Memory(m) => self.masm.free_stack(m.slot.size), 1895 _ => {} 1896 }); 1897 } 1898 1899 fn visit_select(&mut self) { 1900 let cond = self.context.pop_to_reg(self.masm, None); 1901 let val2 = self.context.pop_to_reg(self.masm, None); 1902 let val1 = self.context.pop_to_reg(self.masm, None); 1903 self.masm 1904 .cmp(cond.reg.into(), RegImm::i32(0), OperandSize::S32); 1905 // Conditionally move val1 to val2 if the comparison is 1906 // not zero. 1907 self.masm 1908 .cmov(val1.into(), val2.into(), IntCmpKind::Ne, val1.ty.into()); 1909 self.context.stack.push(val2.into()); 1910 self.context.free_reg(val1.reg); 1911 self.context.free_reg(cond); 1912 } 1913 1914 fn visit_i32_load(&mut self, memarg: MemArg) { 1915 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S32, None); 1916 } 1917 1918 fn visit_i32_load8_s(&mut self, memarg: MemArg) { 1919 self.emit_wasm_load( 1920 &memarg, 1921 WasmValType::I32, 1922 OperandSize::S8, 1923 Some(ExtendKind::I32Extend8S), 1924 ); 1925 } 1926 1927 fn visit_i32_load8_u(&mut self, memarg: MemArg) { 1928 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S8, None); 1929 } 1930 1931 fn visit_i32_load16_s(&mut self, memarg: MemArg) { 1932 self.emit_wasm_load( 1933 &memarg, 1934 WasmValType::I32, 1935 OperandSize::S16, 1936 Some(ExtendKind::I32Extend16S), 1937 ) 1938 } 1939 1940 fn visit_i32_load16_u(&mut self, memarg: MemArg) { 1941 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S16, None) 1942 } 1943 1944 fn visit_i32_store(&mut self, memarg: MemArg) { 1945 self.emit_wasm_store(&memarg, OperandSize::S32); 1946 } 1947 1948 fn visit_i32_store8(&mut self, memarg: MemArg) { 1949 self.emit_wasm_store(&memarg, OperandSize::S8) 1950 } 1951 1952 fn visit_i32_store16(&mut self, memarg: MemArg) { 1953 self.emit_wasm_store(&memarg, OperandSize::S16) 1954 } 1955 1956 fn visit_i64_load8_s(&mut self, memarg: MemArg) { 1957 self.emit_wasm_load( 1958 &memarg, 1959 WasmValType::I64, 1960 OperandSize::S8, 1961 Some(ExtendKind::I64Extend8S), 1962 ) 1963 } 1964 1965 fn visit_i64_load8_u(&mut self, memarg: MemArg) { 1966 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S8, None) 1967 } 1968 1969 fn visit_i64_load16_u(&mut self, memarg: MemArg) { 1970 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S16, None) 1971 } 1972 1973 fn visit_i64_load16_s(&mut self, memarg: MemArg) { 1974 self.emit_wasm_load( 1975 &memarg, 1976 WasmValType::I64, 1977 OperandSize::S16, 1978 Some(ExtendKind::I64Extend16S), 1979 ) 1980 } 1981 1982 fn visit_i64_load32_u(&mut self, memarg: MemArg) { 1983 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S32, None) 1984 } 1985 1986 fn visit_i64_load32_s(&mut self, memarg: MemArg) { 1987 self.emit_wasm_load( 1988 &memarg, 1989 WasmValType::I64, 1990 OperandSize::S32, 1991 Some(ExtendKind::I64Extend32S), 1992 ) 1993 } 1994 1995 fn visit_i64_load(&mut self, memarg: MemArg) { 1996 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S64, None) 1997 } 1998 1999 fn visit_i64_store(&mut self, memarg: MemArg) -> Self::Output { 2000 self.emit_wasm_store(&memarg, OperandSize::S64) 2001 } 2002 2003 fn visit_i64_store8(&mut self, memarg: MemArg) -> Self::Output { 2004 self.emit_wasm_store(&memarg, OperandSize::S8) 2005 } 2006 2007 fn visit_i64_store16(&mut self, memarg: MemArg) -> Self::Output { 2008 self.emit_wasm_store(&memarg, OperandSize::S16) 2009 } 2010 2011 fn visit_i64_store32(&mut self, memarg: MemArg) -> Self::Output { 2012 self.emit_wasm_store(&memarg, OperandSize::S32) 2013 } 2014 2015 fn visit_f32_load(&mut self, memarg: MemArg) { 2016 self.emit_wasm_load(&memarg, WasmValType::F32, OperandSize::S32, None) 2017 } 2018 2019 fn visit_f32_store(&mut self, memarg: MemArg) { 2020 self.emit_wasm_store(&memarg, OperandSize::S32) 2021 } 2022 2023 fn visit_f64_load(&mut self, memarg: MemArg) { 2024 self.emit_wasm_load(&memarg, WasmValType::F64, OperandSize::S64, None) 2025 } 2026 2027 fn visit_f64_store(&mut self, memarg: MemArg) { 2028 self.emit_wasm_store(&memarg, OperandSize::S64) 2029 } 2030 2031 fn visit_v128_load(&mut self, memarg: MemArg) { 2032 self.emit_wasm_load(&memarg, WasmValType::V128, OperandSize::S128, None) 2033 } 2034 2035 fn visit_v128_store(&mut self, memarg: MemArg) { 2036 self.emit_wasm_store(&memarg, OperandSize::S128) 2037 } 2038 2039 fn visit_i32_trunc_sat_f32_s(&mut self) { 2040 use OperandSize::*; 2041 2042 self.context 2043 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2044 masm.signed_truncate(src, dst, S32, dst_size, TruncKind::Checked); 2045 }); 2046 } 2047 2048 fn visit_i32_trunc_sat_f32_u(&mut self) { 2049 use OperandSize::*; 2050 2051 self.context.convert_op_with_tmp_reg( 2052 self.masm, 2053 WasmValType::I32, 2054 RegClass::Float, 2055 |masm, dst, src, tmp_fpr, dst_size| { 2056 masm.unsigned_truncate(src, dst, tmp_fpr, S32, dst_size, TruncKind::Checked); 2057 }, 2058 ); 2059 } 2060 2061 fn visit_i32_trunc_sat_f64_s(&mut self) { 2062 use OperandSize::*; 2063 2064 self.context 2065 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2066 masm.signed_truncate(src, dst, S64, dst_size, TruncKind::Checked); 2067 }); 2068 } 2069 2070 fn visit_i32_trunc_sat_f64_u(&mut self) { 2071 use OperandSize::*; 2072 2073 self.context.convert_op_with_tmp_reg( 2074 self.masm, 2075 WasmValType::I32, 2076 RegClass::Float, 2077 |masm, dst, src, tmp_fpr, dst_size| { 2078 masm.unsigned_truncate(src, dst, tmp_fpr, S64, dst_size, TruncKind::Checked); 2079 }, 2080 ); 2081 } 2082 2083 fn visit_i64_trunc_sat_f32_s(&mut self) { 2084 use OperandSize::*; 2085 2086 self.context 2087 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2088 masm.signed_truncate(src, dst, S32, dst_size, TruncKind::Checked); 2089 }); 2090 } 2091 2092 fn visit_i64_trunc_sat_f32_u(&mut self) { 2093 use OperandSize::*; 2094 2095 self.context.convert_op_with_tmp_reg( 2096 self.masm, 2097 WasmValType::I64, 2098 RegClass::Float, 2099 |masm, dst, src, tmp_fpr, dst_size| { 2100 masm.unsigned_truncate(src, dst, tmp_fpr, S32, dst_size, TruncKind::Checked); 2101 }, 2102 ); 2103 } 2104 2105 fn visit_i64_trunc_sat_f64_s(&mut self) { 2106 use OperandSize::*; 2107 2108 self.context 2109 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2110 masm.signed_truncate(src, dst, S64, dst_size, TruncKind::Checked); 2111 }); 2112 } 2113 2114 fn visit_i64_trunc_sat_f64_u(&mut self) { 2115 use OperandSize::*; 2116 2117 self.context.convert_op_with_tmp_reg( 2118 self.masm, 2119 WasmValType::I64, 2120 RegClass::Float, 2121 |masm, dst, src, tmp_fpr, dst_size| { 2122 masm.unsigned_truncate(src, dst, tmp_fpr, S64, dst_size, TruncKind::Checked); 2123 }, 2124 ); 2125 } 2126 2127 wasmparser::for_each_operator!(def_unsupported); 2128 } 2129 2130 impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M> 2131 where 2132 M: MacroAssembler, 2133 { 2134 fn cmp_i32s(&mut self, kind: IntCmpKind) { 2135 self.context.i32_binop(self.masm, |masm, dst, src, size| { 2136 masm.cmp_with_set(src, dst, kind, size); 2137 TypedReg::i32(dst) 2138 }); 2139 } 2140 2141 fn cmp_i64s(&mut self, kind: IntCmpKind) { 2142 self.context 2143 .i64_binop(self.masm, move |masm, dst, src, size| { 2144 masm.cmp_with_set(src, dst, kind, size); 2145 TypedReg::i32(dst) // Return value for comparisons is an `i32`. 2146 }); 2147 } 2148 } 2149 2150 impl From<WasmValType> for OperandSize { 2151 fn from(ty: WasmValType) -> OperandSize { 2152 match ty { 2153 WasmValType::I32 | WasmValType::F32 => OperandSize::S32, 2154 WasmValType::I64 | WasmValType::F64 => OperandSize::S64, 2155 WasmValType::V128 => OperandSize::S128, 2156 WasmValType::Ref(rt) => { 2157 match rt.heap_type { 2158 // TODO: Hardcoded size, assuming 64-bit support only. Once 2159 // Wasmtime supports 32-bit architectures, this will need 2160 // to be updated in such a way that the calculation of the 2161 // OperandSize will depend on the target's pointer size. 2162 WasmHeapType::Func => OperandSize::S64, 2163 WasmHeapType::Extern => OperandSize::S64, 2164 t => unimplemented!("Support for WasmHeapType: {t}"), 2165 } 2166 } 2167 } 2168 } 2169 } 2170