1 //! This module is the central place for machine code emission. 2 //! It defines an implementation of wasmparser's Visitor trait 3 //! for `CodeGen`; which defines a visitor per op-code, 4 //! which validates and dispatches to the corresponding 5 //! machine code emitter. 6 7 use crate::abi::RetArea; 8 use crate::codegen::{ 9 control_index, Callee, CodeGen, CodeGenError, ControlStackFrame, Emission, FnCall, 10 }; 11 use crate::masm::{ 12 DivKind, ExtendKind, FloatCmpKind, IntCmpKind, MacroAssembler, MemMoveDirection, MulWideKind, 13 OperandSize, RegImm, RemKind, RoundingMode, SPOffset, ShiftKind, TruncKind, 14 }; 15 16 use crate::reg::{writable, Reg}; 17 use crate::stack::{TypedReg, Val}; 18 use anyhow::{anyhow, bail, ensure, Result}; 19 use regalloc2::RegClass; 20 use smallvec::{smallvec, SmallVec}; 21 use wasmparser::{ 22 BlockType, BrTable, Ieee32, Ieee64, MemArg, VisitOperator, VisitSimdOperator, V128, 23 }; 24 use wasmtime_cranelift::TRAP_INDIRECT_CALL_TO_NULL; 25 use wasmtime_environ::{ 26 FuncIndex, GlobalIndex, MemoryIndex, TableIndex, TypeIndex, WasmHeapType, WasmValType, 27 FUNCREF_INIT_BIT, 28 }; 29 30 /// A macro to define unsupported WebAssembly operators. 31 /// 32 /// This macro calls itself recursively; 33 /// 1. It no-ops when matching a supported operator. 34 /// 2. Defines the visitor function and panics when 35 /// matching an unsupported operator. 36 macro_rules! def_unsupported { 37 ($( @$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident $ann:tt)*) => { 38 $( 39 def_unsupported!( 40 emit 41 $op 42 43 fn $visit(&mut self $($(,$arg: $argty)*)?) -> Self::Output { 44 $($(let _ = $arg;)*)? 45 46 Err(anyhow!(CodeGenError::unimplemented_wasm_instruction())) 47 } 48 ); 49 )* 50 }; 51 52 (emit I32Const $($rest:tt)*) => {}; 53 (emit I64Const $($rest:tt)*) => {}; 54 (emit F32Const $($rest:tt)*) => {}; 55 (emit F64Const $($rest:tt)*) => {}; 56 (emit V128Const $($rest:tt)*) => {}; 57 (emit F32Add $($rest:tt)*) => {}; 58 (emit F64Add $($rest:tt)*) => {}; 59 (emit F32Sub $($rest:tt)*) => {}; 60 (emit F64Sub $($rest:tt)*) => {}; 61 (emit F32Mul $($rest:tt)*) => {}; 62 (emit F64Mul $($rest:tt)*) => {}; 63 (emit F32Div $($rest:tt)*) => {}; 64 (emit F64Div $($rest:tt)*) => {}; 65 (emit F32Min $($rest:tt)*) => {}; 66 (emit F64Min $($rest:tt)*) => {}; 67 (emit F32Max $($rest:tt)*) => {}; 68 (emit F64Max $($rest:tt)*) => {}; 69 (emit F32Copysign $($rest:tt)*) => {}; 70 (emit F64Copysign $($rest:tt)*) => {}; 71 (emit F32Abs $($rest:tt)*) => {}; 72 (emit F64Abs $($rest:tt)*) => {}; 73 (emit F32Neg $($rest:tt)*) => {}; 74 (emit F64Neg $($rest:tt)*) => {}; 75 (emit F32Floor $($rest:tt)*) => {}; 76 (emit F64Floor $($rest:tt)*) => {}; 77 (emit F32Ceil $($rest:tt)*) => {}; 78 (emit F64Ceil $($rest:tt)*) => {}; 79 (emit F32Nearest $($rest:tt)*) => {}; 80 (emit F64Nearest $($rest:tt)*) => {}; 81 (emit F32Trunc $($rest:tt)*) => {}; 82 (emit F64Trunc $($rest:tt)*) => {}; 83 (emit F32Sqrt $($rest:tt)*) => {}; 84 (emit F64Sqrt $($rest:tt)*) => {}; 85 (emit F32Eq $($rest:tt)*) => {}; 86 (emit F64Eq $($rest:tt)*) => {}; 87 (emit F32Ne $($rest:tt)*) => {}; 88 (emit F64Ne $($rest:tt)*) => {}; 89 (emit F32Lt $($rest:tt)*) => {}; 90 (emit F64Lt $($rest:tt)*) => {}; 91 (emit F32Gt $($rest:tt)*) => {}; 92 (emit F64Gt $($rest:tt)*) => {}; 93 (emit F32Le $($rest:tt)*) => {}; 94 (emit F64Le $($rest:tt)*) => {}; 95 (emit F32Ge $($rest:tt)*) => {}; 96 (emit F64Ge $($rest:tt)*) => {}; 97 (emit F32ConvertI32S $($rest:tt)*) => {}; 98 (emit F32ConvertI32U $($rest:tt)*) => {}; 99 (emit F32ConvertI64S $($rest:tt)*) => {}; 100 (emit F32ConvertI64U $($rest:tt)*) => {}; 101 (emit F64ConvertI32S $($rest:tt)*) => {}; 102 (emit F64ConvertI32U $($rest:tt)*) => {}; 103 (emit F64ConvertI64S $($rest:tt)*) => {}; 104 (emit F64ConvertI64U $($rest:tt)*) => {}; 105 (emit F32ReinterpretI32 $($rest:tt)*) => {}; 106 (emit F64ReinterpretI64 $($rest:tt)*) => {}; 107 (emit F32DemoteF64 $($rest:tt)*) => {}; 108 (emit F64PromoteF32 $($rest:tt)*) => {}; 109 (emit I32Add $($rest:tt)*) => {}; 110 (emit I64Add $($rest:tt)*) => {}; 111 (emit I32Sub $($rest:tt)*) => {}; 112 (emit I32Mul $($rest:tt)*) => {}; 113 (emit I32DivS $($rest:tt)*) => {}; 114 (emit I32DivU $($rest:tt)*) => {}; 115 (emit I64DivS $($rest:tt)*) => {}; 116 (emit I64DivU $($rest:tt)*) => {}; 117 (emit I64RemU $($rest:tt)*) => {}; 118 (emit I64RemS $($rest:tt)*) => {}; 119 (emit I32RemU $($rest:tt)*) => {}; 120 (emit I32RemS $($rest:tt)*) => {}; 121 (emit I64Mul $($rest:tt)*) => {}; 122 (emit I64Sub $($rest:tt)*) => {}; 123 (emit I32Eq $($rest:tt)*) => {}; 124 (emit I64Eq $($rest:tt)*) => {}; 125 (emit I32Ne $($rest:tt)*) => {}; 126 (emit I64Ne $($rest:tt)*) => {}; 127 (emit I32LtS $($rest:tt)*) => {}; 128 (emit I64LtS $($rest:tt)*) => {}; 129 (emit I32LtU $($rest:tt)*) => {}; 130 (emit I64LtU $($rest:tt)*) => {}; 131 (emit I32LeS $($rest:tt)*) => {}; 132 (emit I64LeS $($rest:tt)*) => {}; 133 (emit I32LeU $($rest:tt)*) => {}; 134 (emit I64LeU $($rest:tt)*) => {}; 135 (emit I32GtS $($rest:tt)*) => {}; 136 (emit I64GtS $($rest:tt)*) => {}; 137 (emit I32GtU $($rest:tt)*) => {}; 138 (emit I64GtU $($rest:tt)*) => {}; 139 (emit I32GeS $($rest:tt)*) => {}; 140 (emit I64GeS $($rest:tt)*) => {}; 141 (emit I32GeU $($rest:tt)*) => {}; 142 (emit I64GeU $($rest:tt)*) => {}; 143 (emit I32Eqz $($rest:tt)*) => {}; 144 (emit I64Eqz $($rest:tt)*) => {}; 145 (emit I32And $($rest:tt)*) => {}; 146 (emit I64And $($rest:tt)*) => {}; 147 (emit I32Or $($rest:tt)*) => {}; 148 (emit I64Or $($rest:tt)*) => {}; 149 (emit I32Xor $($rest:tt)*) => {}; 150 (emit I64Xor $($rest:tt)*) => {}; 151 (emit I32Shl $($rest:tt)*) => {}; 152 (emit I64Shl $($rest:tt)*) => {}; 153 (emit I32ShrS $($rest:tt)*) => {}; 154 (emit I64ShrS $($rest:tt)*) => {}; 155 (emit I32ShrU $($rest:tt)*) => {}; 156 (emit I64ShrU $($rest:tt)*) => {}; 157 (emit I32Rotl $($rest:tt)*) => {}; 158 (emit I64Rotl $($rest:tt)*) => {}; 159 (emit I32Rotr $($rest:tt)*) => {}; 160 (emit I64Rotr $($rest:tt)*) => {}; 161 (emit I32Clz $($rest:tt)*) => {}; 162 (emit I64Clz $($rest:tt)*) => {}; 163 (emit I32Ctz $($rest:tt)*) => {}; 164 (emit I64Ctz $($rest:tt)*) => {}; 165 (emit I32Popcnt $($rest:tt)*) => {}; 166 (emit I64Popcnt $($rest:tt)*) => {}; 167 (emit I32WrapI64 $($rest:tt)*) => {}; 168 (emit I64ExtendI32S $($rest:tt)*) => {}; 169 (emit I64ExtendI32U $($rest:tt)*) => {}; 170 (emit I32Extend8S $($rest:tt)*) => {}; 171 (emit I32Extend16S $($rest:tt)*) => {}; 172 (emit I64Extend8S $($rest:tt)*) => {}; 173 (emit I64Extend16S $($rest:tt)*) => {}; 174 (emit I64Extend32S $($rest:tt)*) => {}; 175 (emit I32TruncF32S $($rest:tt)*) => {}; 176 (emit I32TruncF32U $($rest:tt)*) => {}; 177 (emit I32TruncF64S $($rest:tt)*) => {}; 178 (emit I32TruncF64U $($rest:tt)*) => {}; 179 (emit I64TruncF32S $($rest:tt)*) => {}; 180 (emit I64TruncF32U $($rest:tt)*) => {}; 181 (emit I64TruncF64S $($rest:tt)*) => {}; 182 (emit I64TruncF64U $($rest:tt)*) => {}; 183 (emit I32ReinterpretF32 $($rest:tt)*) => {}; 184 (emit I64ReinterpretF64 $($rest:tt)*) => {}; 185 (emit LocalGet $($rest:tt)*) => {}; 186 (emit LocalSet $($rest:tt)*) => {}; 187 (emit Call $($rest:tt)*) => {}; 188 (emit End $($rest:tt)*) => {}; 189 (emit Nop $($rest:tt)*) => {}; 190 (emit If $($rest:tt)*) => {}; 191 (emit Else $($rest:tt)*) => {}; 192 (emit Block $($rest:tt)*) => {}; 193 (emit Loop $($rest:tt)*) => {}; 194 (emit Br $($rest:tt)*) => {}; 195 (emit BrIf $($rest:tt)*) => {}; 196 (emit Return $($rest:tt)*) => {}; 197 (emit Unreachable $($rest:tt)*) => {}; 198 (emit LocalTee $($rest:tt)*) => {}; 199 (emit GlobalGet $($rest:tt)*) => {}; 200 (emit GlobalSet $($rest:tt)*) => {}; 201 (emit Select $($rest:tt)*) => {}; 202 (emit Drop $($rest:tt)*) => {}; 203 (emit BrTable $($rest:tt)*) => {}; 204 (emit CallIndirect $($rest:tt)*) => {}; 205 (emit TableInit $($rest:tt)*) => {}; 206 (emit TableCopy $($rest:tt)*) => {}; 207 (emit TableGet $($rest:tt)*) => {}; 208 (emit TableSet $($rest:tt)*) => {}; 209 (emit TableGrow $($rest:tt)*) => {}; 210 (emit TableSize $($rest:tt)*) => {}; 211 (emit TableFill $($rest:tt)*) => {}; 212 (emit ElemDrop $($rest:tt)*) => {}; 213 (emit MemoryInit $($rest:tt)*) => {}; 214 (emit MemoryCopy $($rest:tt)*) => {}; 215 (emit DataDrop $($rest:tt)*) => {}; 216 (emit MemoryFill $($rest:tt)*) => {}; 217 (emit MemorySize $($rest:tt)*) => {}; 218 (emit MemoryGrow $($rest:tt)*) => {}; 219 (emit I32Load $($rest:tt)*) => {}; 220 (emit I32Load8S $($rest:tt)*) => {}; 221 (emit I32Load8U $($rest:tt)*) => {}; 222 (emit I32Load16S $($rest:tt)*) => {}; 223 (emit I32Load16U $($rest:tt)*) => {}; 224 (emit I64Load8S $($rest:tt)*) => {}; 225 (emit I64Load8U $($rest:tt)*) => {}; 226 (emit I64Load16S $($rest:tt)*) => {}; 227 (emit I64Load16U $($rest:tt)*) => {}; 228 (emit I64Load32S $($rest:tt)*) => {}; 229 (emit I64Load32U $($rest:tt)*) => {}; 230 (emit I64Load $($rest:tt)*) => {}; 231 (emit I32Store $($rest:tt)*) => {}; 232 (emit I32Store8 $($rest:tt)*) => {}; 233 (emit I32Store16 $($rest:tt)*) => {}; 234 (emit I64Store $($rest:tt)*) => {}; 235 (emit I64Store8 $($rest:tt)*) => {}; 236 (emit I64Store16 $($rest:tt)*) => {}; 237 (emit I64Store32 $($rest:tt)*) => {}; 238 (emit F32Load $($rest:tt)*) => {}; 239 (emit F32Store $($rest:tt)*) => {}; 240 (emit F64Load $($rest:tt)*) => {}; 241 (emit F64Store $($rest:tt)*) => {}; 242 (emit I32TruncSatF32S $($rest:tt)*) => {}; 243 (emit I32TruncSatF32U $($rest:tt)*) => {}; 244 (emit I32TruncSatF64S $($rest:tt)*) => {}; 245 (emit I32TruncSatF64U $($rest:tt)*) => {}; 246 (emit I64TruncSatF32S $($rest:tt)*) => {}; 247 (emit I64TruncSatF32U $($rest:tt)*) => {}; 248 (emit I64TruncSatF64S $($rest:tt)*) => {}; 249 (emit I64TruncSatF64U $($rest:tt)*) => {}; 250 (emit V128Load $($rest:tt)*) => {}; 251 (emit V128Store $($rest:tt)*) => {}; 252 (emit I64Add128 $($rest:tt)*) => {}; 253 (emit I64Sub128 $($rest:tt)*) => {}; 254 (emit I64MulWideS $($rest:tt)*) => {}; 255 (emit I64MulWideU $($rest:tt)*) => {}; 256 257 (emit $unsupported:tt $($rest:tt)*) => {$($rest)*}; 258 } 259 260 impl<'a, 'translation, 'data, M> VisitOperator<'a> for CodeGen<'a, 'translation, 'data, M, Emission> 261 where 262 M: MacroAssembler, 263 { 264 type Output = Result<()>; 265 266 fn visit_i32_const(&mut self, val: i32) -> Self::Output { 267 self.context.stack.push(Val::i32(val)); 268 269 Ok(()) 270 } 271 272 fn visit_i64_const(&mut self, val: i64) -> Self::Output { 273 self.context.stack.push(Val::i64(val)); 274 Ok(()) 275 } 276 277 fn visit_f32_const(&mut self, val: Ieee32) -> Self::Output { 278 self.context.stack.push(Val::f32(val)); 279 Ok(()) 280 } 281 282 fn visit_f64_const(&mut self, val: Ieee64) -> Self::Output { 283 self.context.stack.push(Val::f64(val)); 284 Ok(()) 285 } 286 287 fn visit_f32_add(&mut self) -> Self::Output { 288 self.context.binop( 289 self.masm, 290 OperandSize::S32, 291 &mut |masm: &mut M, dst, src, size| { 292 masm.float_add(writable!(dst), dst, src, size)?; 293 Ok(TypedReg::f32(dst)) 294 }, 295 ) 296 } 297 298 fn visit_f64_add(&mut self) -> Self::Output { 299 self.context.binop( 300 self.masm, 301 OperandSize::S64, 302 &mut |masm: &mut M, dst, src, size| { 303 masm.float_add(writable!(dst), dst, src, size)?; 304 Ok(TypedReg::f64(dst)) 305 }, 306 ) 307 } 308 309 fn visit_f32_sub(&mut self) -> Self::Output { 310 self.context.binop( 311 self.masm, 312 OperandSize::S32, 313 &mut |masm: &mut M, dst, src, size| { 314 masm.float_sub(writable!(dst), dst, src, size)?; 315 Ok(TypedReg::f32(dst)) 316 }, 317 ) 318 } 319 320 fn visit_f64_sub(&mut self) -> Self::Output { 321 self.context.binop( 322 self.masm, 323 OperandSize::S64, 324 &mut |masm: &mut M, dst, src, size| { 325 masm.float_sub(writable!(dst), dst, src, size)?; 326 Ok(TypedReg::f64(dst)) 327 }, 328 ) 329 } 330 331 fn visit_f32_mul(&mut self) -> Self::Output { 332 self.context.binop( 333 self.masm, 334 OperandSize::S32, 335 &mut |masm: &mut M, dst, src, size| { 336 masm.float_mul(writable!(dst), dst, src, size)?; 337 Ok(TypedReg::f32(dst)) 338 }, 339 ) 340 } 341 342 fn visit_f64_mul(&mut self) -> Self::Output { 343 self.context.binop( 344 self.masm, 345 OperandSize::S64, 346 &mut |masm: &mut M, dst, src, size| { 347 masm.float_mul(writable!(dst), dst, src, size)?; 348 Ok(TypedReg::f64(dst)) 349 }, 350 ) 351 } 352 353 fn visit_f32_div(&mut self) -> Self::Output { 354 self.context.binop( 355 self.masm, 356 OperandSize::S32, 357 &mut |masm: &mut M, dst, src, size| { 358 masm.float_div(writable!(dst), dst, src, size)?; 359 Ok(TypedReg::f32(dst)) 360 }, 361 ) 362 } 363 364 fn visit_f64_div(&mut self) -> Self::Output { 365 self.context.binop( 366 self.masm, 367 OperandSize::S64, 368 &mut |masm: &mut M, dst, src, size| { 369 masm.float_div(writable!(dst), dst, src, size)?; 370 Ok(TypedReg::f64(dst)) 371 }, 372 ) 373 } 374 375 fn visit_f32_min(&mut self) -> Self::Output { 376 self.context.binop( 377 self.masm, 378 OperandSize::S32, 379 &mut |masm: &mut M, dst, src, size| { 380 masm.float_min(writable!(dst), dst, src, size)?; 381 Ok(TypedReg::f32(dst)) 382 }, 383 ) 384 } 385 386 fn visit_f64_min(&mut self) -> Self::Output { 387 self.context.binop( 388 self.masm, 389 OperandSize::S64, 390 &mut |masm: &mut M, dst, src, size| { 391 masm.float_min(writable!(dst), dst, src, size)?; 392 Ok(TypedReg::f64(dst)) 393 }, 394 ) 395 } 396 397 fn visit_f32_max(&mut self) -> Self::Output { 398 self.context.binop( 399 self.masm, 400 OperandSize::S32, 401 &mut |masm: &mut M, dst, src, size| { 402 masm.float_max(writable!(dst), dst, src, size)?; 403 Ok(TypedReg::f32(dst)) 404 }, 405 ) 406 } 407 408 fn visit_f64_max(&mut self) -> Self::Output { 409 self.context.binop( 410 self.masm, 411 OperandSize::S64, 412 &mut |masm: &mut M, dst, src, size| { 413 masm.float_max(writable!(dst), dst, src, size)?; 414 Ok(TypedReg::f64(dst)) 415 }, 416 ) 417 } 418 419 fn visit_f32_copysign(&mut self) -> Self::Output { 420 self.context.binop( 421 self.masm, 422 OperandSize::S32, 423 &mut |masm: &mut M, dst, src, size| { 424 masm.float_copysign(writable!(dst), dst, src, size)?; 425 Ok(TypedReg::f32(dst)) 426 }, 427 ) 428 } 429 430 fn visit_f64_copysign(&mut self) -> Self::Output { 431 self.context.binop( 432 self.masm, 433 OperandSize::S64, 434 &mut |masm: &mut M, dst, src, size| { 435 masm.float_copysign(writable!(dst), dst, src, size)?; 436 Ok(TypedReg::f64(dst)) 437 }, 438 ) 439 } 440 441 fn visit_f32_abs(&mut self) -> Self::Output { 442 self.context 443 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 444 masm.float_abs(writable!(reg), size)?; 445 Ok(TypedReg::f32(reg)) 446 }) 447 } 448 449 fn visit_f64_abs(&mut self) -> Self::Output { 450 self.context 451 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 452 masm.float_abs(writable!(reg), size)?; 453 Ok(TypedReg::f64(reg)) 454 }) 455 } 456 457 fn visit_f32_neg(&mut self) -> Self::Output { 458 self.context 459 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 460 masm.float_neg(writable!(reg), size)?; 461 Ok(TypedReg::f32(reg)) 462 }) 463 } 464 465 fn visit_f64_neg(&mut self) -> Self::Output { 466 self.context 467 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 468 masm.float_neg(writable!(reg), size)?; 469 Ok(TypedReg::f64(reg)) 470 }) 471 } 472 473 fn visit_f32_floor(&mut self) -> Self::Output { 474 self.masm.float_round( 475 RoundingMode::Down, 476 &mut self.env, 477 &mut self.context, 478 OperandSize::S32, 479 |env, cx, masm| { 480 let builtin = env.builtins.floor_f32::<M::ABI>(); 481 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 482 }, 483 ) 484 } 485 486 fn visit_f64_floor(&mut self) -> Self::Output { 487 self.masm.float_round( 488 RoundingMode::Down, 489 &mut self.env, 490 &mut self.context, 491 OperandSize::S64, 492 |env, cx, masm| { 493 let builtin = env.builtins.floor_f64::<M::ABI>(); 494 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 495 }, 496 ) 497 } 498 499 fn visit_f32_ceil(&mut self) -> Self::Output { 500 self.masm.float_round( 501 RoundingMode::Up, 502 &mut self.env, 503 &mut self.context, 504 OperandSize::S32, 505 |env, cx, masm| { 506 let builtin = env.builtins.ceil_f32::<M::ABI>(); 507 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 508 }, 509 ) 510 } 511 512 fn visit_f64_ceil(&mut self) -> Self::Output { 513 self.masm.float_round( 514 RoundingMode::Up, 515 &mut self.env, 516 &mut self.context, 517 OperandSize::S64, 518 |env, cx, masm| { 519 let builtin = env.builtins.ceil_f64::<M::ABI>(); 520 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 521 }, 522 ) 523 } 524 525 fn visit_f32_nearest(&mut self) -> Self::Output { 526 self.masm.float_round( 527 RoundingMode::Nearest, 528 &mut self.env, 529 &mut self.context, 530 OperandSize::S32, 531 |env, cx, masm| { 532 let builtin = env.builtins.nearest_f32::<M::ABI>(); 533 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 534 }, 535 ) 536 } 537 538 fn visit_f64_nearest(&mut self) -> Self::Output { 539 self.masm.float_round( 540 RoundingMode::Nearest, 541 &mut self.env, 542 &mut self.context, 543 OperandSize::S64, 544 |env, cx, masm| { 545 let builtin = env.builtins.nearest_f64::<M::ABI>(); 546 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 547 }, 548 ) 549 } 550 551 fn visit_f32_trunc(&mut self) -> Self::Output { 552 self.masm.float_round( 553 RoundingMode::Zero, 554 &mut self.env, 555 &mut self.context, 556 OperandSize::S32, 557 |env, cx, masm| { 558 let builtin = env.builtins.trunc_f32::<M::ABI>(); 559 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 560 }, 561 ) 562 } 563 564 fn visit_f64_trunc(&mut self) -> Self::Output { 565 self.masm.float_round( 566 RoundingMode::Zero, 567 &mut self.env, 568 &mut self.context, 569 OperandSize::S64, 570 |env, cx, masm| { 571 let builtin = env.builtins.trunc_f64::<M::ABI>(); 572 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 573 }, 574 ) 575 } 576 577 fn visit_f32_sqrt(&mut self) -> Self::Output { 578 self.context 579 .unop(self.masm, OperandSize::S32, &mut |masm, reg, size| { 580 masm.float_sqrt(writable!(reg), reg, size)?; 581 Ok(TypedReg::f32(reg)) 582 }) 583 } 584 585 fn visit_f64_sqrt(&mut self) -> Self::Output { 586 self.context 587 .unop(self.masm, OperandSize::S64, &mut |masm, reg, size| { 588 masm.float_sqrt(writable!(reg), reg, size)?; 589 Ok(TypedReg::f64(reg)) 590 }) 591 } 592 593 fn visit_f32_eq(&mut self) -> Self::Output { 594 self.context.float_cmp_op( 595 self.masm, 596 OperandSize::S32, 597 &mut |masm: &mut M, dst, src1, src2, size| { 598 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Eq, size) 599 }, 600 ) 601 } 602 603 fn visit_f64_eq(&mut self) -> Self::Output { 604 self.context.float_cmp_op( 605 self.masm, 606 OperandSize::S64, 607 &mut |masm: &mut M, dst, src1, src2, size| { 608 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Eq, size) 609 }, 610 ) 611 } 612 613 fn visit_f32_ne(&mut self) -> Self::Output { 614 self.context.float_cmp_op( 615 self.masm, 616 OperandSize::S32, 617 &mut |masm: &mut M, dst, src1, src2, size| { 618 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ne, size) 619 }, 620 ) 621 } 622 623 fn visit_f64_ne(&mut self) -> Self::Output { 624 self.context.float_cmp_op( 625 self.masm, 626 OperandSize::S64, 627 &mut |masm: &mut M, dst, src1, src2, size| { 628 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ne, size) 629 }, 630 ) 631 } 632 633 fn visit_f32_lt(&mut self) -> Self::Output { 634 self.context.float_cmp_op( 635 self.masm, 636 OperandSize::S32, 637 &mut |masm: &mut M, dst, src1, src2, size| { 638 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Lt, size) 639 }, 640 ) 641 } 642 643 fn visit_f64_lt(&mut self) -> Self::Output { 644 self.context.float_cmp_op( 645 self.masm, 646 OperandSize::S64, 647 &mut |masm: &mut M, dst, src1, src2, size| { 648 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Lt, size) 649 }, 650 ) 651 } 652 653 fn visit_f32_gt(&mut self) -> Self::Output { 654 self.context.float_cmp_op( 655 self.masm, 656 OperandSize::S32, 657 &mut |masm: &mut M, dst, src1, src2, size| { 658 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Gt, size) 659 }, 660 ) 661 } 662 663 fn visit_f64_gt(&mut self) -> Self::Output { 664 self.context.float_cmp_op( 665 self.masm, 666 OperandSize::S64, 667 &mut |masm: &mut M, dst, src1, src2, size| { 668 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Gt, size) 669 }, 670 ) 671 } 672 673 fn visit_f32_le(&mut self) -> Self::Output { 674 self.context.float_cmp_op( 675 self.masm, 676 OperandSize::S32, 677 &mut |masm: &mut M, dst, src1, src2, size| { 678 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Le, size) 679 }, 680 ) 681 } 682 683 fn visit_f64_le(&mut self) -> Self::Output { 684 self.context.float_cmp_op( 685 self.masm, 686 OperandSize::S64, 687 &mut |masm: &mut M, dst, src1, src2, size| { 688 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Le, size) 689 }, 690 ) 691 } 692 693 fn visit_f32_ge(&mut self) -> Self::Output { 694 self.context.float_cmp_op( 695 self.masm, 696 OperandSize::S32, 697 &mut |masm: &mut M, dst, src1, src2, size| { 698 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ge, size) 699 }, 700 ) 701 } 702 703 fn visit_f64_ge(&mut self) -> Self::Output { 704 self.context.float_cmp_op( 705 self.masm, 706 OperandSize::S64, 707 &mut |masm: &mut M, dst, src1, src2, size| { 708 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ge, size) 709 }, 710 ) 711 } 712 713 fn visit_f32_convert_i32_s(&mut self) -> Self::Output { 714 self.context 715 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 716 masm.signed_convert(writable!(dst), src, OperandSize::S32, dst_size) 717 }) 718 } 719 720 fn visit_f32_convert_i32_u(&mut self) -> Self::Output { 721 self.context.convert_op_with_tmp_reg( 722 self.masm, 723 WasmValType::F32, 724 RegClass::Int, 725 |masm, dst, src, tmp_gpr, dst_size| { 726 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S32, dst_size) 727 }, 728 ) 729 } 730 731 fn visit_f32_convert_i64_s(&mut self) -> Self::Output { 732 self.context 733 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 734 masm.signed_convert(writable!(dst), src, OperandSize::S64, dst_size) 735 }) 736 } 737 738 fn visit_f32_convert_i64_u(&mut self) -> Self::Output { 739 self.context.convert_op_with_tmp_reg( 740 self.masm, 741 WasmValType::F32, 742 RegClass::Int, 743 |masm, dst, src, tmp_gpr, dst_size| { 744 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S64, dst_size) 745 }, 746 ) 747 } 748 749 fn visit_f64_convert_i32_s(&mut self) -> Self::Output { 750 self.context 751 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 752 masm.signed_convert(writable!(dst), src, OperandSize::S32, dst_size) 753 }) 754 } 755 756 fn visit_f64_convert_i32_u(&mut self) -> Self::Output { 757 self.context.convert_op_with_tmp_reg( 758 self.masm, 759 WasmValType::F64, 760 RegClass::Int, 761 |masm, dst, src, tmp_gpr, dst_size| { 762 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S32, dst_size) 763 }, 764 ) 765 } 766 767 fn visit_f64_convert_i64_s(&mut self) -> Self::Output { 768 self.context 769 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 770 masm.signed_convert(writable!(dst), src, OperandSize::S64, dst_size) 771 }) 772 } 773 774 fn visit_f64_convert_i64_u(&mut self) -> Self::Output { 775 self.context.convert_op_with_tmp_reg( 776 self.masm, 777 WasmValType::F64, 778 RegClass::Int, 779 |masm, dst, src, tmp_gpr, dst_size| { 780 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S64, dst_size) 781 }, 782 ) 783 } 784 785 fn visit_f32_reinterpret_i32(&mut self) -> Self::Output { 786 self.context 787 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, size| { 788 masm.reinterpret_int_as_float(writable!(dst), src.into(), size) 789 }) 790 } 791 792 fn visit_f64_reinterpret_i64(&mut self) -> Self::Output { 793 self.context 794 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, size| { 795 masm.reinterpret_int_as_float(writable!(dst), src.into(), size) 796 }) 797 } 798 799 fn visit_f32_demote_f64(&mut self) -> Self::Output { 800 self.context 801 .unop(self.masm, OperandSize::S64, &mut |masm, reg, _size| { 802 masm.demote(writable!(reg), reg)?; 803 Ok(TypedReg::f32(reg)) 804 }) 805 } 806 807 fn visit_f64_promote_f32(&mut self) -> Self::Output { 808 self.context 809 .unop(self.masm, OperandSize::S32, &mut |masm, reg, _size| { 810 masm.promote(writable!(reg), reg)?; 811 Ok(TypedReg::f64(reg)) 812 }) 813 } 814 815 fn visit_i32_add(&mut self) -> Self::Output { 816 self.context.i32_binop(self.masm, |masm, dst, src, size| { 817 masm.add(writable!(dst), dst, src, size)?; 818 Ok(TypedReg::i32(dst)) 819 }) 820 } 821 822 fn visit_i64_add(&mut self) -> Self::Output { 823 self.context.i64_binop(self.masm, |masm, dst, src, size| { 824 masm.add(writable!(dst), dst, src, size)?; 825 Ok(TypedReg::i64(dst)) 826 }) 827 } 828 829 fn visit_i32_sub(&mut self) -> Self::Output { 830 self.context.i32_binop(self.masm, |masm, dst, src, size| { 831 masm.sub(writable!(dst), dst, src, size)?; 832 Ok(TypedReg::i32(dst)) 833 }) 834 } 835 836 fn visit_i64_sub(&mut self) -> Self::Output { 837 self.context.i64_binop(self.masm, |masm, dst, src, size| { 838 masm.sub(writable!(dst), dst, src, size)?; 839 Ok(TypedReg::i64(dst)) 840 }) 841 } 842 843 fn visit_i32_mul(&mut self) -> Self::Output { 844 self.context.i32_binop(self.masm, |masm, dst, src, size| { 845 masm.mul(writable!(dst), dst, src, size)?; 846 Ok(TypedReg::i32(dst)) 847 }) 848 } 849 850 fn visit_i64_mul(&mut self) -> Self::Output { 851 self.context.i64_binop(self.masm, |masm, dst, src, size| { 852 masm.mul(writable!(dst), dst, src, size)?; 853 Ok(TypedReg::i64(dst)) 854 }) 855 } 856 857 fn visit_i32_div_s(&mut self) -> Self::Output { 858 use DivKind::*; 859 use OperandSize::*; 860 861 self.masm.div(&mut self.context, Signed, S32) 862 } 863 864 fn visit_i32_div_u(&mut self) -> Self::Output { 865 use DivKind::*; 866 use OperandSize::*; 867 868 self.masm.div(&mut self.context, Unsigned, S32) 869 } 870 871 fn visit_i64_div_s(&mut self) -> Self::Output { 872 use DivKind::*; 873 use OperandSize::*; 874 875 self.masm.div(&mut self.context, Signed, S64) 876 } 877 878 fn visit_i64_div_u(&mut self) -> Self::Output { 879 use DivKind::*; 880 use OperandSize::*; 881 882 self.masm.div(&mut self.context, Unsigned, S64) 883 } 884 885 fn visit_i32_rem_s(&mut self) -> Self::Output { 886 use OperandSize::*; 887 use RemKind::*; 888 889 self.masm.rem(&mut self.context, Signed, S32) 890 } 891 892 fn visit_i32_rem_u(&mut self) -> Self::Output { 893 use OperandSize::*; 894 use RemKind::*; 895 896 self.masm.rem(&mut self.context, Unsigned, S32) 897 } 898 899 fn visit_i64_rem_s(&mut self) -> Self::Output { 900 use OperandSize::*; 901 use RemKind::*; 902 903 self.masm.rem(&mut self.context, Signed, S64) 904 } 905 906 fn visit_i64_rem_u(&mut self) -> Self::Output { 907 use OperandSize::*; 908 use RemKind::*; 909 910 self.masm.rem(&mut self.context, Unsigned, S64) 911 } 912 913 fn visit_i32_eq(&mut self) -> Self::Output { 914 self.cmp_i32s(IntCmpKind::Eq) 915 } 916 917 fn visit_i64_eq(&mut self) -> Self::Output { 918 self.cmp_i64s(IntCmpKind::Eq) 919 } 920 921 fn visit_i32_ne(&mut self) -> Self::Output { 922 self.cmp_i32s(IntCmpKind::Ne) 923 } 924 925 fn visit_i64_ne(&mut self) -> Self::Output { 926 self.cmp_i64s(IntCmpKind::Ne) 927 } 928 929 fn visit_i32_lt_s(&mut self) -> Self::Output { 930 self.cmp_i32s(IntCmpKind::LtS) 931 } 932 933 fn visit_i64_lt_s(&mut self) -> Self::Output { 934 self.cmp_i64s(IntCmpKind::LtS) 935 } 936 937 fn visit_i32_lt_u(&mut self) -> Self::Output { 938 self.cmp_i32s(IntCmpKind::LtU) 939 } 940 941 fn visit_i64_lt_u(&mut self) -> Self::Output { 942 self.cmp_i64s(IntCmpKind::LtU) 943 } 944 945 fn visit_i32_le_s(&mut self) -> Self::Output { 946 self.cmp_i32s(IntCmpKind::LeS) 947 } 948 949 fn visit_i64_le_s(&mut self) -> Self::Output { 950 self.cmp_i64s(IntCmpKind::LeS) 951 } 952 953 fn visit_i32_le_u(&mut self) -> Self::Output { 954 self.cmp_i32s(IntCmpKind::LeU) 955 } 956 957 fn visit_i64_le_u(&mut self) -> Self::Output { 958 self.cmp_i64s(IntCmpKind::LeU) 959 } 960 961 fn visit_i32_gt_s(&mut self) -> Self::Output { 962 self.cmp_i32s(IntCmpKind::GtS) 963 } 964 965 fn visit_i64_gt_s(&mut self) -> Self::Output { 966 self.cmp_i64s(IntCmpKind::GtS) 967 } 968 969 fn visit_i32_gt_u(&mut self) -> Self::Output { 970 self.cmp_i32s(IntCmpKind::GtU) 971 } 972 973 fn visit_i64_gt_u(&mut self) -> Self::Output { 974 self.cmp_i64s(IntCmpKind::GtU) 975 } 976 977 fn visit_i32_ge_s(&mut self) -> Self::Output { 978 self.cmp_i32s(IntCmpKind::GeS) 979 } 980 981 fn visit_i64_ge_s(&mut self) -> Self::Output { 982 self.cmp_i64s(IntCmpKind::GeS) 983 } 984 985 fn visit_i32_ge_u(&mut self) -> Self::Output { 986 self.cmp_i32s(IntCmpKind::GeU) 987 } 988 989 fn visit_i64_ge_u(&mut self) -> Self::Output { 990 self.cmp_i64s(IntCmpKind::GeU) 991 } 992 993 fn visit_i32_eqz(&mut self) -> Self::Output { 994 use OperandSize::*; 995 996 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 997 masm.cmp_with_set(writable!(reg.into()), RegImm::i32(0), IntCmpKind::Eq, size)?; 998 Ok(TypedReg::i32(reg)) 999 }) 1000 } 1001 1002 fn visit_i64_eqz(&mut self) -> Self::Output { 1003 use OperandSize::*; 1004 1005 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 1006 masm.cmp_with_set(writable!(reg.into()), RegImm::i64(0), IntCmpKind::Eq, size)?; 1007 Ok(TypedReg::i32(reg)) // Return value for `i64.eqz` is an `i32`. 1008 }) 1009 } 1010 1011 fn visit_i32_clz(&mut self) -> Self::Output { 1012 use OperandSize::*; 1013 1014 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 1015 masm.clz(writable!(reg), reg, size)?; 1016 Ok(TypedReg::i32(reg)) 1017 }) 1018 } 1019 1020 fn visit_i64_clz(&mut self) -> Self::Output { 1021 use OperandSize::*; 1022 1023 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 1024 masm.clz(writable!(reg), reg, size)?; 1025 Ok(TypedReg::i64(reg)) 1026 }) 1027 } 1028 1029 fn visit_i32_ctz(&mut self) -> Self::Output { 1030 use OperandSize::*; 1031 1032 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 1033 masm.ctz(writable!(reg), reg, size)?; 1034 Ok(TypedReg::i32(reg)) 1035 }) 1036 } 1037 1038 fn visit_i64_ctz(&mut self) -> Self::Output { 1039 use OperandSize::*; 1040 1041 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 1042 masm.ctz(writable!(reg), reg, size)?; 1043 Ok(TypedReg::i64(reg)) 1044 }) 1045 } 1046 1047 fn visit_i32_and(&mut self) -> Self::Output { 1048 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1049 masm.and(writable!(dst), dst, src, size)?; 1050 Ok(TypedReg::i32(dst)) 1051 }) 1052 } 1053 1054 fn visit_i64_and(&mut self) -> Self::Output { 1055 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1056 masm.and(writable!(dst), dst, src, size)?; 1057 Ok(TypedReg::i64(dst)) 1058 }) 1059 } 1060 1061 fn visit_i32_or(&mut self) -> Self::Output { 1062 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1063 masm.or(writable!(dst), dst, src, size)?; 1064 Ok(TypedReg::i32(dst)) 1065 }) 1066 } 1067 1068 fn visit_i64_or(&mut self) -> Self::Output { 1069 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1070 masm.or(writable!(dst), dst, src, size)?; 1071 Ok(TypedReg::i64(dst)) 1072 }) 1073 } 1074 1075 fn visit_i32_xor(&mut self) -> Self::Output { 1076 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1077 masm.xor(writable!(dst), dst, src, size)?; 1078 Ok(TypedReg::i32(dst)) 1079 }) 1080 } 1081 1082 fn visit_i64_xor(&mut self) -> Self::Output { 1083 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1084 masm.xor(writable!(dst), dst, src, size)?; 1085 Ok(TypedReg::i64(dst)) 1086 }) 1087 } 1088 1089 fn visit_i32_shl(&mut self) -> Self::Output { 1090 use ShiftKind::*; 1091 1092 self.context.i32_shift(self.masm, Shl) 1093 } 1094 1095 fn visit_i64_shl(&mut self) -> Self::Output { 1096 use ShiftKind::*; 1097 1098 self.context.i64_shift(self.masm, Shl) 1099 } 1100 1101 fn visit_i32_shr_s(&mut self) -> Self::Output { 1102 use ShiftKind::*; 1103 1104 self.context.i32_shift(self.masm, ShrS) 1105 } 1106 1107 fn visit_i64_shr_s(&mut self) -> Self::Output { 1108 use ShiftKind::*; 1109 1110 self.context.i64_shift(self.masm, ShrS) 1111 } 1112 1113 fn visit_i32_shr_u(&mut self) -> Self::Output { 1114 use ShiftKind::*; 1115 1116 self.context.i32_shift(self.masm, ShrU) 1117 } 1118 1119 fn visit_i64_shr_u(&mut self) -> Self::Output { 1120 use ShiftKind::*; 1121 1122 self.context.i64_shift(self.masm, ShrU) 1123 } 1124 1125 fn visit_i32_rotl(&mut self) -> Self::Output { 1126 use ShiftKind::*; 1127 1128 self.context.i32_shift(self.masm, Rotl) 1129 } 1130 1131 fn visit_i64_rotl(&mut self) -> Self::Output { 1132 use ShiftKind::*; 1133 1134 self.context.i64_shift(self.masm, Rotl) 1135 } 1136 1137 fn visit_i32_rotr(&mut self) -> Self::Output { 1138 use ShiftKind::*; 1139 1140 self.context.i32_shift(self.masm, Rotr) 1141 } 1142 1143 fn visit_i64_rotr(&mut self) -> Self::Output { 1144 use ShiftKind::*; 1145 1146 self.context.i64_shift(self.masm, Rotr) 1147 } 1148 1149 fn visit_end(&mut self) -> Self::Output { 1150 if !self.context.reachable { 1151 self.handle_unreachable_end() 1152 } else { 1153 let mut control = self.pop_control_frame()?; 1154 control.emit_end(self.masm, &mut self.context) 1155 } 1156 } 1157 1158 fn visit_i32_popcnt(&mut self) -> Self::Output { 1159 use OperandSize::*; 1160 self.masm.popcnt(&mut self.context, S32) 1161 } 1162 1163 fn visit_i64_popcnt(&mut self) -> Self::Output { 1164 use OperandSize::*; 1165 1166 self.masm.popcnt(&mut self.context, S64) 1167 } 1168 1169 fn visit_i32_wrap_i64(&mut self) -> Self::Output { 1170 use OperandSize::*; 1171 1172 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1173 masm.wrap(writable!(reg), reg)?; 1174 Ok(TypedReg::i32(reg)) 1175 }) 1176 } 1177 1178 fn visit_i64_extend_i32_s(&mut self) -> Self::Output { 1179 use OperandSize::*; 1180 1181 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1182 masm.extend(writable!(reg), reg, ExtendKind::I64ExtendI32S)?; 1183 Ok(TypedReg::i64(reg)) 1184 }) 1185 } 1186 1187 fn visit_i64_extend_i32_u(&mut self) -> Self::Output { 1188 use OperandSize::*; 1189 1190 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1191 masm.extend(writable!(reg), reg, ExtendKind::I64ExtendI32U)?; 1192 Ok(TypedReg::i64(reg)) 1193 }) 1194 } 1195 1196 fn visit_i32_extend8_s(&mut self) -> Self::Output { 1197 use OperandSize::*; 1198 1199 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1200 masm.extend(writable!(reg), reg, ExtendKind::I32Extend8S)?; 1201 Ok(TypedReg::i32(reg)) 1202 }) 1203 } 1204 1205 fn visit_i32_extend16_s(&mut self) -> Self::Output { 1206 use OperandSize::*; 1207 1208 self.context.unop(self.masm, S32, &mut |masm, reg, _size| { 1209 masm.extend(writable!(reg), reg, ExtendKind::I32Extend16S)?; 1210 Ok(TypedReg::i32(reg)) 1211 }) 1212 } 1213 1214 fn visit_i64_extend8_s(&mut self) -> Self::Output { 1215 use OperandSize::*; 1216 1217 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1218 masm.extend(writable!(reg), reg, ExtendKind::I64Extend8S)?; 1219 Ok(TypedReg::i64(reg)) 1220 }) 1221 } 1222 1223 fn visit_i64_extend16_s(&mut self) -> Self::Output { 1224 use OperandSize::*; 1225 1226 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1227 masm.extend(writable!(reg), reg, ExtendKind::I64Extend16S)?; 1228 Ok(TypedReg::i64(reg)) 1229 }) 1230 } 1231 1232 fn visit_i64_extend32_s(&mut self) -> Self::Output { 1233 use OperandSize::*; 1234 1235 self.context.unop(self.masm, S64, &mut |masm, reg, _size| { 1236 masm.extend(writable!(reg), reg, ExtendKind::I64Extend32S)?; 1237 Ok(TypedReg::i64(reg)) 1238 }) 1239 } 1240 1241 fn visit_i32_trunc_f32_s(&mut self) -> Self::Output { 1242 use OperandSize::*; 1243 1244 self.context 1245 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1246 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Unchecked) 1247 }) 1248 } 1249 1250 fn visit_i32_trunc_f32_u(&mut self) -> Self::Output { 1251 use OperandSize::*; 1252 1253 self.masm 1254 .unsigned_truncate(&mut self.context, S32, S32, TruncKind::Unchecked) 1255 } 1256 1257 fn visit_i32_trunc_f64_s(&mut self) -> Self::Output { 1258 use OperandSize::*; 1259 1260 self.context 1261 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1262 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Unchecked) 1263 }) 1264 } 1265 1266 fn visit_i32_trunc_f64_u(&mut self) -> Self::Output { 1267 use OperandSize::*; 1268 self.masm 1269 .unsigned_truncate(&mut self.context, S64, S32, TruncKind::Unchecked) 1270 } 1271 1272 fn visit_i64_trunc_f32_s(&mut self) -> Self::Output { 1273 use OperandSize::*; 1274 1275 self.context 1276 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1277 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Unchecked) 1278 }) 1279 } 1280 1281 fn visit_i64_trunc_f32_u(&mut self) -> Self::Output { 1282 use OperandSize::*; 1283 1284 self.masm 1285 .unsigned_truncate(&mut self.context, S32, S64, TruncKind::Unchecked) 1286 } 1287 1288 fn visit_i64_trunc_f64_s(&mut self) -> Self::Output { 1289 use OperandSize::*; 1290 1291 self.context 1292 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1293 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Unchecked) 1294 }) 1295 } 1296 1297 fn visit_i64_trunc_f64_u(&mut self) -> Self::Output { 1298 use OperandSize::*; 1299 1300 self.masm 1301 .unsigned_truncate(&mut self.context, S64, S64, TruncKind::Unchecked) 1302 } 1303 1304 fn visit_i32_reinterpret_f32(&mut self) -> Self::Output { 1305 self.context 1306 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, size| { 1307 masm.reinterpret_float_as_int(writable!(dst), src.into(), size) 1308 }) 1309 } 1310 1311 fn visit_i64_reinterpret_f64(&mut self) -> Self::Output { 1312 self.context 1313 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, size| { 1314 masm.reinterpret_float_as_int(writable!(dst), src.into(), size) 1315 }) 1316 } 1317 1318 fn visit_local_get(&mut self, index: u32) -> Self::Output { 1319 use WasmValType::*; 1320 let context = &mut self.context; 1321 let slot = context.frame.get_wasm_local(index); 1322 match slot.ty { 1323 I32 | I64 | F32 | F64 | V128 => context.stack.push(Val::local(index, slot.ty)), 1324 Ref(rt) => match rt.heap_type { 1325 WasmHeapType::Func => context.stack.push(Val::local(index, slot.ty)), 1326 _ => bail!(CodeGenError::unsupported_wasm_type()), 1327 }, 1328 } 1329 1330 Ok(()) 1331 } 1332 1333 fn visit_local_set(&mut self, index: u32) -> Self::Output { 1334 let src = self.emit_set_local(index)?; 1335 self.context.free_reg(src); 1336 Ok(()) 1337 } 1338 1339 fn visit_call(&mut self, index: u32) -> Self::Output { 1340 let callee = self.env.callee_from_index(FuncIndex::from_u32(index)); 1341 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee)?; 1342 Ok(()) 1343 } 1344 1345 fn visit_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output { 1346 // Spill now because `emit_lazy_init_funcref` and the `FnCall::emit` 1347 // invocations will both trigger spills since they both call functions. 1348 // However, the machine instructions for the spill emitted by 1349 // `emit_lazy_funcref` will be jumped over if the funcref was previously 1350 // initialized which may result in the machine stack becoming 1351 // unbalanced. 1352 self.context.spill(self.masm)?; 1353 1354 let type_index = TypeIndex::from_u32(type_index); 1355 let table_index = TableIndex::from_u32(table_index); 1356 1357 self.emit_lazy_init_funcref(table_index)?; 1358 1359 // Perform the indirect call. 1360 // This code assumes that [`Self::emit_lazy_init_funcref`] will 1361 // push the funcref to the value stack. 1362 let funcref_ptr = self 1363 .context 1364 .stack 1365 .peek() 1366 .map(|v| v.unwrap_reg()) 1367 .ok_or_else(|| CodeGenError::missing_values_in_stack())?; 1368 self.masm 1369 .trapz(funcref_ptr.into(), TRAP_INDIRECT_CALL_TO_NULL)?; 1370 self.emit_typecheck_funcref(funcref_ptr.into(), type_index)?; 1371 1372 let callee = self.env.funcref(type_index); 1373 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee)?; 1374 Ok(()) 1375 } 1376 1377 fn visit_table_init(&mut self, elem: u32, table: u32) -> Self::Output { 1378 let at = self.context.stack.ensure_index_at(3)?; 1379 1380 self.context 1381 .stack 1382 .insert_many(at, &[table.try_into()?, elem.try_into()?]); 1383 1384 let builtin = self.env.builtins.table_init::<M::ABI, M::Ptr>(); 1385 FnCall::emit::<M>( 1386 &mut self.env, 1387 self.masm, 1388 &mut self.context, 1389 Callee::Builtin(builtin.clone()), 1390 )?; 1391 self.context.pop_and_free(self.masm) 1392 } 1393 1394 fn visit_table_copy(&mut self, dst: u32, src: u32) -> Self::Output { 1395 let at = self.context.stack.ensure_index_at(3)?; 1396 self.context 1397 .stack 1398 .insert_many(at, &[dst.try_into()?, src.try_into()?]); 1399 1400 let builtin = self.env.builtins.table_copy::<M::ABI, M::Ptr>(); 1401 FnCall::emit::<M>( 1402 &mut self.env, 1403 self.masm, 1404 &mut self.context, 1405 Callee::Builtin(builtin), 1406 )?; 1407 self.context.pop_and_free(self.masm) 1408 } 1409 1410 fn visit_table_get(&mut self, table: u32) -> Self::Output { 1411 let table_index = TableIndex::from_u32(table); 1412 let table = self.env.table(table_index); 1413 let heap_type = table.ref_type.heap_type; 1414 1415 match heap_type { 1416 WasmHeapType::Func => self.emit_lazy_init_funcref(table_index), 1417 _ => Err(anyhow!(CodeGenError::unsupported_wasm_type())), 1418 } 1419 } 1420 1421 fn visit_table_grow(&mut self, table: u32) -> Self::Output { 1422 let table_index = TableIndex::from_u32(table); 1423 let table_ty = self.env.table(table_index); 1424 let builtin = match table_ty.ref_type.heap_type { 1425 WasmHeapType::Func => self.env.builtins.table_grow_func_ref::<M::ABI, M::Ptr>(), 1426 _ => bail!(CodeGenError::unsupported_wasm_type()), 1427 }; 1428 1429 let len = self.context.stack.len(); 1430 // table.grow` requires at least 2 elements on the value stack. 1431 let at = self.context.stack.ensure_index_at(2)?; 1432 1433 // The table_grow builtin expects the parameters in a different 1434 // order. 1435 // The value stack at this point should contain: 1436 // [ init_value | delta ] (stack top) 1437 // but the builtin function expects the init value as the last 1438 // argument. 1439 self.context.stack.inner_mut().swap(len - 1, len - 2); 1440 self.context.stack.insert_many(at, &[table.try_into()?]); 1441 1442 FnCall::emit::<M>( 1443 &mut self.env, 1444 self.masm, 1445 &mut self.context, 1446 Callee::Builtin(builtin.clone()), 1447 )?; 1448 1449 Ok(()) 1450 } 1451 1452 fn visit_table_size(&mut self, table: u32) -> Self::Output { 1453 let table_index = TableIndex::from_u32(table); 1454 let table_data = self.env.resolve_table_data(table_index); 1455 self.emit_compute_table_size(&table_data) 1456 } 1457 1458 fn visit_table_fill(&mut self, table: u32) -> Self::Output { 1459 let table_index = TableIndex::from_u32(table); 1460 let table_ty = self.env.table(table_index); 1461 1462 ensure!( 1463 table_ty.ref_type.heap_type == WasmHeapType::Func, 1464 CodeGenError::unsupported_wasm_type() 1465 ); 1466 1467 let builtin = self.env.builtins.table_fill_func_ref::<M::ABI, M::Ptr>(); 1468 1469 let at = self.context.stack.ensure_index_at(3)?; 1470 1471 self.context.stack.insert_many(at, &[table.try_into()?]); 1472 FnCall::emit::<M>( 1473 &mut self.env, 1474 self.masm, 1475 &mut self.context, 1476 Callee::Builtin(builtin.clone()), 1477 )?; 1478 self.context.pop_and_free(self.masm) 1479 } 1480 1481 fn visit_table_set(&mut self, table: u32) -> Self::Output { 1482 let ptr_type = self.env.ptr_type(); 1483 let table_index = TableIndex::from_u32(table); 1484 let table_data = self.env.resolve_table_data(table_index); 1485 let table = self.env.table(table_index); 1486 match table.ref_type.heap_type { 1487 WasmHeapType::Func => { 1488 ensure!( 1489 self.tunables.table_lazy_init, 1490 CodeGenError::unsupported_table_eager_init() 1491 ); 1492 let value = self.context.pop_to_reg(self.masm, None)?; 1493 let index = self.context.pop_to_reg(self.masm, None)?; 1494 let base = self.context.any_gpr(self.masm)?; 1495 let elem_addr = 1496 self.emit_compute_table_elem_addr(index.into(), base, &table_data)?; 1497 // Set the initialized bit. 1498 self.masm.or( 1499 writable!(value.into()), 1500 value.into(), 1501 RegImm::i64(FUNCREF_INIT_BIT as i64), 1502 ptr_type.try_into()?, 1503 )?; 1504 1505 self.masm.store_ptr(value.into(), elem_addr)?; 1506 1507 self.context.free_reg(value); 1508 self.context.free_reg(index); 1509 self.context.free_reg(base); 1510 Ok(()) 1511 } 1512 _ => Err(anyhow!(CodeGenError::unsupported_wasm_type())), 1513 } 1514 } 1515 1516 fn visit_elem_drop(&mut self, index: u32) -> Self::Output { 1517 let elem_drop = self.env.builtins.elem_drop::<M::ABI, M::Ptr>(); 1518 self.context.stack.extend([index.try_into()?]); 1519 FnCall::emit::<M>( 1520 &mut self.env, 1521 self.masm, 1522 &mut self.context, 1523 Callee::Builtin(elem_drop), 1524 )?; 1525 Ok(()) 1526 } 1527 1528 fn visit_memory_init(&mut self, data_index: u32, mem: u32) -> Self::Output { 1529 let at = self.context.stack.ensure_index_at(3)?; 1530 self.context 1531 .stack 1532 .insert_many(at, &[mem.try_into()?, data_index.try_into()?]); 1533 let builtin = self.env.builtins.memory_init::<M::ABI, M::Ptr>(); 1534 FnCall::emit::<M>( 1535 &mut self.env, 1536 self.masm, 1537 &mut self.context, 1538 Callee::Builtin(builtin), 1539 )?; 1540 self.context.pop_and_free(self.masm) 1541 } 1542 1543 fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) -> Self::Output { 1544 // At this point, the stack is expected to contain: 1545 // [ dst_offset, src_offset, len ] 1546 // The following code inserts the missing params, so that stack contains: 1547 // [ vmctx, dst_mem, dst_offset, src_mem, src_offset, len ] 1548 // Which is the order expected by the builtin function. 1549 let _ = self.context.stack.ensure_index_at(3)?; 1550 let at = self.context.stack.ensure_index_at(2)?; 1551 self.context.stack.insert_many(at, &[src_mem.try_into()?]); 1552 1553 // One element was inserted above, so instead of 3, we use 4. 1554 let at = self.context.stack.ensure_index_at(4)?; 1555 self.context.stack.insert_many(at, &[dst_mem.try_into()?]); 1556 1557 let builtin = self.env.builtins.memory_copy::<M::ABI, M::Ptr>(); 1558 1559 FnCall::emit::<M>( 1560 &mut self.env, 1561 self.masm, 1562 &mut self.context, 1563 Callee::Builtin(builtin), 1564 )?; 1565 self.context.pop_and_free(self.masm) 1566 } 1567 1568 fn visit_memory_fill(&mut self, mem: u32) -> Self::Output { 1569 let at = self.context.stack.ensure_index_at(3)?; 1570 1571 self.context.stack.insert_many(at, &[mem.try_into()?]); 1572 1573 let builtin = self.env.builtins.memory_fill::<M::ABI, M::Ptr>(); 1574 FnCall::emit::<M>( 1575 &mut self.env, 1576 self.masm, 1577 &mut self.context, 1578 Callee::Builtin(builtin), 1579 )?; 1580 self.context.pop_and_free(self.masm) 1581 } 1582 1583 fn visit_memory_size(&mut self, mem: u32) -> Self::Output { 1584 let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem)); 1585 self.emit_compute_memory_size(&heap) 1586 } 1587 1588 fn visit_memory_grow(&mut self, mem: u32) -> Self::Output { 1589 let _ = self.context.stack.ensure_index_at(1)?; 1590 // The stack at this point contains: [ delta ] 1591 // The desired state is 1592 // [ vmctx, delta, index ] 1593 self.context.stack.extend([mem.try_into()?]); 1594 1595 let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem)); 1596 let builtin = self.env.builtins.memory32_grow::<M::ABI, M::Ptr>(); 1597 FnCall::emit::<M>( 1598 &mut self.env, 1599 self.masm, 1600 &mut self.context, 1601 Callee::Builtin(builtin), 1602 )?; 1603 1604 // The memory32_grow builtin returns a pointer type, therefore we must 1605 // ensure that the return type is representative of the address space of 1606 // the heap type. 1607 match (self.env.ptr_type(), heap.index_type()) { 1608 (WasmValType::I64, WasmValType::I64) => Ok(()), 1609 // When the heap type is smaller than the pointer type, we adjust 1610 // the result of the memory32_grow builtin. 1611 (WasmValType::I64, WasmValType::I32) => { 1612 let top: Reg = self.context.pop_to_reg(self.masm, None)?.into(); 1613 self.masm.wrap(writable!(top.into()), top.into())?; 1614 self.context.stack.push(TypedReg::i32(top).into()); 1615 Ok(()) 1616 } 1617 _ => Err(anyhow!(CodeGenError::unsupported_32_bit_platform())), 1618 } 1619 } 1620 1621 fn visit_data_drop(&mut self, data_index: u32) -> Self::Output { 1622 self.context.stack.extend([data_index.try_into()?]); 1623 1624 let builtin = self.env.builtins.data_drop::<M::ABI, M::Ptr>(); 1625 FnCall::emit::<M>( 1626 &mut self.env, 1627 self.masm, 1628 &mut self.context, 1629 Callee::Builtin(builtin), 1630 ) 1631 } 1632 1633 fn visit_nop(&mut self) -> Self::Output { 1634 Ok(()) 1635 } 1636 1637 fn visit_if(&mut self, blockty: BlockType) -> Self::Output { 1638 self.control_frames.push(ControlStackFrame::r#if( 1639 self.env.resolve_block_sig(blockty), 1640 self.masm, 1641 &mut self.context, 1642 )?); 1643 1644 Ok(()) 1645 } 1646 1647 fn visit_else(&mut self) -> Self::Output { 1648 if !self.context.reachable { 1649 self.handle_unreachable_else() 1650 } else { 1651 let control = self 1652 .control_frames 1653 .last_mut() 1654 .ok_or_else(|| CodeGenError::control_frame_expected())?; 1655 control.emit_else(self.masm, &mut self.context) 1656 } 1657 } 1658 1659 fn visit_block(&mut self, blockty: BlockType) -> Self::Output { 1660 self.control_frames.push(ControlStackFrame::block( 1661 self.env.resolve_block_sig(blockty), 1662 self.masm, 1663 &mut self.context, 1664 )?); 1665 1666 Ok(()) 1667 } 1668 1669 fn visit_loop(&mut self, blockty: BlockType) -> Self::Output { 1670 self.control_frames.push(ControlStackFrame::r#loop( 1671 self.env.resolve_block_sig(blockty), 1672 self.masm, 1673 &mut self.context, 1674 )?); 1675 1676 self.maybe_emit_epoch_check()?; 1677 self.maybe_emit_fuel_check() 1678 } 1679 1680 fn visit_br(&mut self, depth: u32) -> Self::Output { 1681 let index = control_index(depth, self.control_frames.len())?; 1682 let frame = &mut self.control_frames[index]; 1683 self.context 1684 .unconditional_jump(frame, self.masm, |masm, cx, frame| { 1685 frame.pop_abi_results::<M, _>(cx, masm, |results, _, _| { 1686 Ok(results.ret_area().copied()) 1687 }) 1688 }) 1689 } 1690 1691 fn visit_br_if(&mut self, depth: u32) -> Self::Output { 1692 let index = control_index(depth, self.control_frames.len())?; 1693 let frame = &mut self.control_frames[index]; 1694 frame.set_as_target(); 1695 1696 let top = { 1697 let top = self.context.without::<Result<TypedReg>, M, _>( 1698 frame.results::<M>().regs(), 1699 self.masm, 1700 |ctx, masm| ctx.pop_to_reg(masm, None), 1701 )??; 1702 // Explicitly save any live registers and locals before setting up 1703 // the branch state. 1704 // In some cases, calculating the `top` value above, will result in 1705 // a spill, thus the following one will result in a no-op. 1706 self.context.spill(self.masm)?; 1707 frame.top_abi_results::<M, _>( 1708 &mut self.context, 1709 self.masm, 1710 |results, context, masm| { 1711 // In the case of `br_if` there's a possibility that we'll 1712 // exit early from the block or fallthrough, for 1713 // a fallthrough, we cannot rely on the pre-computed return area; 1714 // it must be recalculated so that any values that are 1715 // generated are correctly placed near the current stack 1716 // pointer. 1717 if results.on_stack() { 1718 let stack_consumed = context.stack.sizeof(results.stack_operands_len()); 1719 let base = masm.sp_offset()?.as_u32() - stack_consumed; 1720 let offs = base + results.size(); 1721 Ok(Some(RetArea::sp(SPOffset::from_u32(offs)))) 1722 } else { 1723 Ok(None) 1724 } 1725 }, 1726 )?; 1727 top 1728 }; 1729 1730 // Emit instructions to balance the machine stack if the frame has 1731 // a different offset. 1732 let current_sp_offset = self.masm.sp_offset()?; 1733 let results_size = frame.results::<M>().size(); 1734 let state = frame.stack_state(); 1735 let (label, cmp, needs_cleanup) = if current_sp_offset > state.target_offset { 1736 (self.masm.get_label()?, IntCmpKind::Eq, true) 1737 } else { 1738 (*frame.label(), IntCmpKind::Ne, false) 1739 }; 1740 1741 self.masm 1742 .branch(cmp, top.reg.into(), top.reg.into(), label, OperandSize::S32)?; 1743 self.context.free_reg(top); 1744 1745 if needs_cleanup { 1746 // Emit instructions to balance the stack and jump if not falling 1747 // through. 1748 self.masm.memmove( 1749 current_sp_offset, 1750 state.target_offset, 1751 results_size, 1752 MemMoveDirection::LowToHigh, 1753 )?; 1754 self.masm.ensure_sp_for_jump(state.target_offset)?; 1755 self.masm.jmp(*frame.label())?; 1756 1757 // Restore sp_offset to what it was for falling through and emit 1758 // fallthrough label. 1759 self.masm.reset_stack_pointer(current_sp_offset)?; 1760 self.masm.bind(label)?; 1761 } 1762 1763 Ok(()) 1764 } 1765 1766 fn visit_br_table(&mut self, targets: BrTable<'a>) -> Self::Output { 1767 // +1 to account for the default target. 1768 let len = targets.len() + 1; 1769 // SmallVec<[_; 5]> to match the binary emission layer (e.g 1770 // see `JmpTableSeq'), but here we use 5 instead since we 1771 // bundle the default target as the last element in the array. 1772 let mut labels: SmallVec<[_; 5]> = smallvec![]; 1773 for _ in 0..len { 1774 labels.push(self.masm.get_label()?); 1775 } 1776 1777 let default_index = control_index(targets.default(), self.control_frames.len())?; 1778 let default_frame = &mut self.control_frames[default_index]; 1779 let default_result = default_frame.results::<M>(); 1780 1781 let (index, tmp) = { 1782 let index_and_tmp = self.context.without::<Result<(TypedReg, _)>, M, _>( 1783 default_result.regs(), 1784 self.masm, 1785 |cx, masm| Ok((cx.pop_to_reg(masm, None)?, cx.any_gpr(masm)?)), 1786 )??; 1787 1788 // Materialize any constants or locals into their result representation, 1789 // so that when reachability is restored, they are correctly located. 1790 default_frame.top_abi_results::<M, _>( 1791 &mut self.context, 1792 self.masm, 1793 |results, _, _| Ok(results.ret_area().copied()), 1794 )?; 1795 index_and_tmp 1796 }; 1797 1798 self.masm.jmp_table(&labels, index.into(), tmp)?; 1799 // Save the original stack pointer offset; we will reset the stack 1800 // pointer to this offset after jumping to each of the targets. Each 1801 // jump might adjust the stack according to the base offset of the 1802 // target. 1803 let current_sp = self.masm.sp_offset()?; 1804 1805 for (t, l) in targets 1806 .targets() 1807 .into_iter() 1808 .chain(std::iter::once(Ok(targets.default()))) 1809 .zip(labels.iter()) 1810 { 1811 let control_index = control_index(t?, self.control_frames.len())?; 1812 let frame = &mut self.control_frames[control_index]; 1813 // Reset the stack pointer to its original offset. This is needed 1814 // because each jump will potentially adjust the stack pointer 1815 // according to the base offset of the target. 1816 self.masm.reset_stack_pointer(current_sp)?; 1817 1818 // NB: We don't perform any result handling as it was 1819 // already taken care of above before jumping to the 1820 // jump table. 1821 self.masm.bind(*l)?; 1822 // Ensure that the stack pointer is correctly positioned before 1823 // jumping to the jump table code. 1824 let state = frame.stack_state(); 1825 self.masm.ensure_sp_for_jump(state.target_offset)?; 1826 self.masm.jmp(*frame.label())?; 1827 frame.set_as_target(); 1828 } 1829 // Finally reset the stack pointer to the original location. 1830 // The reachability analysis, will ensure it's correctly located 1831 // once reachability is restored. 1832 self.masm.reset_stack_pointer(current_sp)?; 1833 self.context.reachable = false; 1834 self.context.free_reg(index.reg); 1835 self.context.free_reg(tmp); 1836 1837 Ok(()) 1838 } 1839 1840 fn visit_return(&mut self) -> Self::Output { 1841 // Grab the outermost frame, which is the function's body 1842 // frame. We don't rely on [`codegen::control_index`] since 1843 // this frame is implicit and we know that it should exist at 1844 // index 0. 1845 let outermost = &mut self.control_frames[0]; 1846 self.context 1847 .unconditional_jump(outermost, self.masm, |masm, cx, frame| { 1848 frame.pop_abi_results::<M, _>(cx, masm, |results, _, _| { 1849 Ok(results.ret_area().copied()) 1850 }) 1851 }) 1852 } 1853 1854 fn visit_unreachable(&mut self) -> Self::Output { 1855 self.masm.unreachable()?; 1856 self.context.reachable = false; 1857 // Set the implicit outermost frame as target to perform the necessary 1858 // stack clean up. 1859 let outermost = &mut self.control_frames[0]; 1860 outermost.set_as_target(); 1861 1862 Ok(()) 1863 } 1864 1865 fn visit_local_tee(&mut self, index: u32) -> Self::Output { 1866 let typed_reg = self.emit_set_local(index)?; 1867 self.context.stack.push(typed_reg.into()); 1868 1869 Ok(()) 1870 } 1871 1872 fn visit_global_get(&mut self, global_index: u32) -> Self::Output { 1873 let index = GlobalIndex::from_u32(global_index); 1874 let (ty, addr) = self.emit_get_global_addr(index)?; 1875 let dst = self.context.reg_for_type(ty, self.masm)?; 1876 self.masm.load(addr, writable!(dst), ty.try_into()?)?; 1877 self.context.stack.push(Val::reg(dst, ty)); 1878 1879 Ok(()) 1880 } 1881 1882 fn visit_global_set(&mut self, global_index: u32) -> Self::Output { 1883 let index = GlobalIndex::from_u32(global_index); 1884 let (ty, addr) = self.emit_get_global_addr(index)?; 1885 1886 let typed_reg = self.context.pop_to_reg(self.masm, None)?; 1887 self.context.free_reg(typed_reg.reg); 1888 self.masm 1889 .store(typed_reg.reg.into(), addr, ty.try_into()?)?; 1890 1891 Ok(()) 1892 } 1893 1894 fn visit_drop(&mut self) -> Self::Output { 1895 self.context.drop_last(1, |regalloc, val| match val { 1896 Val::Reg(tr) => Ok(regalloc.free(tr.reg.into())), 1897 Val::Memory(m) => self.masm.free_stack(m.slot.size), 1898 _ => Ok(()), 1899 }) 1900 } 1901 1902 fn visit_select(&mut self) -> Self::Output { 1903 let cond = self.context.pop_to_reg(self.masm, None)?; 1904 let val2 = self.context.pop_to_reg(self.masm, None)?; 1905 let val1 = self.context.pop_to_reg(self.masm, None)?; 1906 self.masm 1907 .cmp(cond.reg.into(), RegImm::i32(0), OperandSize::S32)?; 1908 // Conditionally move val1 to val2 if the comparison is 1909 // not zero. 1910 self.masm.cmov( 1911 writable!(val2.into()), 1912 val1.into(), 1913 IntCmpKind::Ne, 1914 val1.ty.try_into()?, 1915 )?; 1916 self.context.stack.push(val2.into()); 1917 self.context.free_reg(val1.reg); 1918 self.context.free_reg(cond); 1919 1920 Ok(()) 1921 } 1922 1923 fn visit_i32_load(&mut self, memarg: MemArg) -> Self::Output { 1924 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S32, None) 1925 } 1926 1927 fn visit_i32_load8_s(&mut self, memarg: MemArg) -> Self::Output { 1928 self.emit_wasm_load( 1929 &memarg, 1930 WasmValType::I32, 1931 OperandSize::S8, 1932 Some(ExtendKind::I32Extend8S), 1933 ) 1934 } 1935 1936 fn visit_i32_load8_u(&mut self, memarg: MemArg) -> Self::Output { 1937 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S8, None) 1938 } 1939 1940 fn visit_i32_load16_s(&mut self, memarg: MemArg) -> Self::Output { 1941 self.emit_wasm_load( 1942 &memarg, 1943 WasmValType::I32, 1944 OperandSize::S16, 1945 Some(ExtendKind::I32Extend16S), 1946 ) 1947 } 1948 1949 fn visit_i32_load16_u(&mut self, memarg: MemArg) -> Self::Output { 1950 self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S16, None) 1951 } 1952 1953 fn visit_i32_store(&mut self, memarg: MemArg) -> Self::Output { 1954 self.emit_wasm_store(&memarg, OperandSize::S32) 1955 } 1956 1957 fn visit_i32_store8(&mut self, memarg: MemArg) -> Self::Output { 1958 self.emit_wasm_store(&memarg, OperandSize::S8) 1959 } 1960 1961 fn visit_i32_store16(&mut self, memarg: MemArg) -> Self::Output { 1962 self.emit_wasm_store(&memarg, OperandSize::S16) 1963 } 1964 1965 fn visit_i64_load8_s(&mut self, memarg: MemArg) -> Self::Output { 1966 self.emit_wasm_load( 1967 &memarg, 1968 WasmValType::I64, 1969 OperandSize::S8, 1970 Some(ExtendKind::I64Extend8S), 1971 ) 1972 } 1973 1974 fn visit_i64_load8_u(&mut self, memarg: MemArg) -> Self::Output { 1975 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S8, None) 1976 } 1977 1978 fn visit_i64_load16_u(&mut self, memarg: MemArg) -> Self::Output { 1979 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S16, None) 1980 } 1981 1982 fn visit_i64_load16_s(&mut self, memarg: MemArg) -> Self::Output { 1983 self.emit_wasm_load( 1984 &memarg, 1985 WasmValType::I64, 1986 OperandSize::S16, 1987 Some(ExtendKind::I64Extend16S), 1988 ) 1989 } 1990 1991 fn visit_i64_load32_u(&mut self, memarg: MemArg) -> Self::Output { 1992 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S32, None) 1993 } 1994 1995 fn visit_i64_load32_s(&mut self, memarg: MemArg) -> Self::Output { 1996 self.emit_wasm_load( 1997 &memarg, 1998 WasmValType::I64, 1999 OperandSize::S32, 2000 Some(ExtendKind::I64Extend32S), 2001 ) 2002 } 2003 2004 fn visit_i64_load(&mut self, memarg: MemArg) -> Self::Output { 2005 self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S64, None) 2006 } 2007 2008 fn visit_i64_store(&mut self, memarg: MemArg) -> Self::Output { 2009 self.emit_wasm_store(&memarg, OperandSize::S64) 2010 } 2011 2012 fn visit_i64_store8(&mut self, memarg: MemArg) -> Self::Output { 2013 self.emit_wasm_store(&memarg, OperandSize::S8) 2014 } 2015 2016 fn visit_i64_store16(&mut self, memarg: MemArg) -> Self::Output { 2017 self.emit_wasm_store(&memarg, OperandSize::S16) 2018 } 2019 2020 fn visit_i64_store32(&mut self, memarg: MemArg) -> Self::Output { 2021 self.emit_wasm_store(&memarg, OperandSize::S32) 2022 } 2023 2024 fn visit_f32_load(&mut self, memarg: MemArg) -> Self::Output { 2025 self.emit_wasm_load(&memarg, WasmValType::F32, OperandSize::S32, None) 2026 } 2027 2028 fn visit_f32_store(&mut self, memarg: MemArg) -> Self::Output { 2029 self.emit_wasm_store(&memarg, OperandSize::S32) 2030 } 2031 2032 fn visit_f64_load(&mut self, memarg: MemArg) -> Self::Output { 2033 self.emit_wasm_load(&memarg, WasmValType::F64, OperandSize::S64, None) 2034 } 2035 2036 fn visit_f64_store(&mut self, memarg: MemArg) -> Self::Output { 2037 self.emit_wasm_store(&memarg, OperandSize::S64) 2038 } 2039 2040 fn visit_i32_trunc_sat_f32_s(&mut self) -> Self::Output { 2041 use OperandSize::*; 2042 2043 self.context 2044 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2045 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Checked) 2046 }) 2047 } 2048 2049 fn visit_i32_trunc_sat_f32_u(&mut self) -> Self::Output { 2050 use OperandSize::*; 2051 2052 self.masm 2053 .unsigned_truncate(&mut self.context, S32, S32, TruncKind::Checked) 2054 } 2055 2056 fn visit_i32_trunc_sat_f64_s(&mut self) -> Self::Output { 2057 use OperandSize::*; 2058 2059 self.context 2060 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2061 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Checked) 2062 }) 2063 } 2064 2065 fn visit_i32_trunc_sat_f64_u(&mut self) -> Self::Output { 2066 use OperandSize::*; 2067 2068 self.masm 2069 .unsigned_truncate(&mut self.context, S64, S32, TruncKind::Checked) 2070 } 2071 2072 fn visit_i64_trunc_sat_f32_s(&mut self) -> Self::Output { 2073 use OperandSize::*; 2074 2075 self.context 2076 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2077 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Checked) 2078 }) 2079 } 2080 2081 fn visit_i64_trunc_sat_f32_u(&mut self) -> Self::Output { 2082 use OperandSize::*; 2083 2084 self.masm 2085 .unsigned_truncate(&mut self.context, S32, S64, TruncKind::Checked) 2086 } 2087 2088 fn visit_i64_trunc_sat_f64_s(&mut self) -> Self::Output { 2089 use OperandSize::*; 2090 2091 self.context 2092 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2093 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Checked) 2094 }) 2095 } 2096 2097 fn visit_i64_trunc_sat_f64_u(&mut self) -> Self::Output { 2098 use OperandSize::*; 2099 2100 self.masm 2101 .unsigned_truncate(&mut self.context, S64, S64, TruncKind::Checked) 2102 } 2103 2104 fn visit_i64_add128(&mut self) -> Self::Output { 2105 self.context 2106 .binop128(self.masm, |masm, lhs_lo, lhs_hi, rhs_lo, rhs_hi| { 2107 masm.add128( 2108 writable!(lhs_lo), 2109 writable!(lhs_hi), 2110 lhs_lo, 2111 lhs_hi, 2112 rhs_lo, 2113 rhs_hi, 2114 )?; 2115 Ok((TypedReg::i64(lhs_lo), TypedReg::i64(lhs_hi))) 2116 }) 2117 } 2118 2119 fn visit_i64_sub128(&mut self) -> Self::Output { 2120 self.context 2121 .binop128(self.masm, |masm, lhs_lo, lhs_hi, rhs_lo, rhs_hi| { 2122 masm.sub128( 2123 writable!(lhs_lo), 2124 writable!(lhs_hi), 2125 lhs_lo, 2126 lhs_hi, 2127 rhs_lo, 2128 rhs_hi, 2129 )?; 2130 Ok((TypedReg::i64(lhs_lo), TypedReg::i64(lhs_hi))) 2131 }) 2132 } 2133 2134 fn visit_i64_mul_wide_s(&mut self) -> Self::Output { 2135 self.masm.mul_wide(&mut self.context, MulWideKind::Signed) 2136 } 2137 2138 fn visit_i64_mul_wide_u(&mut self) -> Self::Output { 2139 self.masm.mul_wide(&mut self.context, MulWideKind::Unsigned) 2140 } 2141 2142 wasmparser::for_each_visit_operator!(def_unsupported); 2143 } 2144 2145 impl<'a, 'translation, 'data, M> VisitSimdOperator<'a> 2146 for CodeGen<'a, 'translation, 'data, M, Emission> 2147 where 2148 M: MacroAssembler, 2149 { 2150 fn visit_v128_const(&mut self, val: V128) -> Self::Output { 2151 self.context.stack.push(Val::v128(val.i128())); 2152 Ok(()) 2153 } 2154 2155 fn visit_v128_load(&mut self, memarg: MemArg) -> Self::Output { 2156 self.emit_wasm_load(&memarg, WasmValType::V128, OperandSize::S128, None) 2157 } 2158 2159 fn visit_v128_store(&mut self, memarg: MemArg) -> Self::Output { 2160 self.emit_wasm_store(&memarg, OperandSize::S128) 2161 } 2162 2163 wasmparser::for_each_visit_simd_operator!(def_unsupported); 2164 } 2165 2166 impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Emission> 2167 where 2168 M: MacroAssembler, 2169 { 2170 fn cmp_i32s(&mut self, kind: IntCmpKind) -> Result<()> { 2171 self.context.i32_binop(self.masm, |masm, dst, src, size| { 2172 masm.cmp_with_set(writable!(dst), src, kind, size)?; 2173 Ok(TypedReg::i32(dst)) 2174 }) 2175 } 2176 2177 fn cmp_i64s(&mut self, kind: IntCmpKind) -> Result<()> { 2178 self.context 2179 .i64_binop(self.masm, move |masm, dst, src, size| { 2180 masm.cmp_with_set(writable!(dst), src, kind, size)?; 2181 Ok(TypedReg::i32(dst)) // Return value for comparisons is an `i32`. 2182 }) 2183 } 2184 } 2185 2186 impl TryFrom<WasmValType> for OperandSize { 2187 type Error = anyhow::Error; 2188 fn try_from(ty: WasmValType) -> Result<OperandSize> { 2189 let ty = match ty { 2190 WasmValType::I32 | WasmValType::F32 => OperandSize::S32, 2191 WasmValType::I64 | WasmValType::F64 => OperandSize::S64, 2192 WasmValType::V128 => OperandSize::S128, 2193 WasmValType::Ref(rt) => { 2194 match rt.heap_type { 2195 // TODO: Hardcoded size, assuming 64-bit support only. Once 2196 // Wasmtime supports 32-bit architectures, this will need 2197 // to be updated in such a way that the calculation of the 2198 // OperandSize will depend on the target's pointer size. 2199 WasmHeapType::Func => OperandSize::S64, 2200 WasmHeapType::Extern => OperandSize::S64, 2201 _ => bail!(CodeGenError::unsupported_wasm_type()), 2202 } 2203 } 2204 }; 2205 Ok(ty) 2206 } 2207 } 2208