1 //! This module is the central place for machine code emission. 2 //! It defines an implementation of wasmparser's Visitor trait 3 //! for `CodeGen`; which defines a visitor per op-code, 4 //! which validates and dispatches to the corresponding 5 //! machine code emitter. 6 7 use crate::abi::RetArea; 8 use crate::codegen::{ 9 control_index, Callee, CodeGen, CodeGenError, ControlStackFrame, Emission, FnCall, 10 }; 11 use crate::masm::{ 12 DivKind, ExtendKind, FloatCmpKind, IntCmpKind, LoadKind, MacroAssembler, MemMoveDirection, 13 MemOpKind, MulWideKind, OperandSize, RegImm, RemKind, RmwOp, RoundingMode, SPOffset, ShiftKind, 14 SplatKind, SplatLoadKind, TruncKind, VectorExtendKind, 15 }; 16 17 use crate::reg::{writable, Reg}; 18 use crate::stack::{TypedReg, Val}; 19 use anyhow::{anyhow, bail, ensure, Result}; 20 use regalloc2::RegClass; 21 use smallvec::{smallvec, SmallVec}; 22 use wasmparser::{ 23 BlockType, BrTable, Ieee32, Ieee64, MemArg, VisitOperator, VisitSimdOperator, V128, 24 }; 25 use wasmtime_cranelift::TRAP_INDIRECT_CALL_TO_NULL; 26 use wasmtime_environ::{ 27 FuncIndex, GlobalIndex, MemoryIndex, TableIndex, TypeIndex, WasmHeapType, WasmValType, 28 FUNCREF_INIT_BIT, 29 }; 30 31 /// A macro to define unsupported WebAssembly operators. 32 /// 33 /// This macro calls itself recursively; 34 /// 1. It no-ops when matching a supported operator. 35 /// 2. Defines the visitor function and panics when 36 /// matching an unsupported operator. 37 macro_rules! def_unsupported { 38 ($( @$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident $ann:tt)*) => { 39 $( 40 def_unsupported!( 41 emit 42 $op 43 44 fn $visit(&mut self $($(,$arg: $argty)*)?) -> Self::Output { 45 $($(let _ = $arg;)*)? 46 47 Err(anyhow!(CodeGenError::unimplemented_wasm_instruction())) 48 } 49 ); 50 )* 51 }; 52 53 (emit I32Const $($rest:tt)*) => {}; 54 (emit I64Const $($rest:tt)*) => {}; 55 (emit F32Const $($rest:tt)*) => {}; 56 (emit F64Const $($rest:tt)*) => {}; 57 (emit V128Const $($rest:tt)*) => {}; 58 (emit F32Add $($rest:tt)*) => {}; 59 (emit F64Add $($rest:tt)*) => {}; 60 (emit F32Sub $($rest:tt)*) => {}; 61 (emit F64Sub $($rest:tt)*) => {}; 62 (emit F32Mul $($rest:tt)*) => {}; 63 (emit F64Mul $($rest:tt)*) => {}; 64 (emit F32Div $($rest:tt)*) => {}; 65 (emit F64Div $($rest:tt)*) => {}; 66 (emit F32Min $($rest:tt)*) => {}; 67 (emit F64Min $($rest:tt)*) => {}; 68 (emit F32Max $($rest:tt)*) => {}; 69 (emit F64Max $($rest:tt)*) => {}; 70 (emit F32Copysign $($rest:tt)*) => {}; 71 (emit F64Copysign $($rest:tt)*) => {}; 72 (emit F32Abs $($rest:tt)*) => {}; 73 (emit F64Abs $($rest:tt)*) => {}; 74 (emit F32Neg $($rest:tt)*) => {}; 75 (emit F64Neg $($rest:tt)*) => {}; 76 (emit F32Floor $($rest:tt)*) => {}; 77 (emit F64Floor $($rest:tt)*) => {}; 78 (emit F32Ceil $($rest:tt)*) => {}; 79 (emit F64Ceil $($rest:tt)*) => {}; 80 (emit F32Nearest $($rest:tt)*) => {}; 81 (emit F64Nearest $($rest:tt)*) => {}; 82 (emit F32Trunc $($rest:tt)*) => {}; 83 (emit F64Trunc $($rest:tt)*) => {}; 84 (emit F32Sqrt $($rest:tt)*) => {}; 85 (emit F64Sqrt $($rest:tt)*) => {}; 86 (emit F32Eq $($rest:tt)*) => {}; 87 (emit F64Eq $($rest:tt)*) => {}; 88 (emit F32Ne $($rest:tt)*) => {}; 89 (emit F64Ne $($rest:tt)*) => {}; 90 (emit F32Lt $($rest:tt)*) => {}; 91 (emit F64Lt $($rest:tt)*) => {}; 92 (emit F32Gt $($rest:tt)*) => {}; 93 (emit F64Gt $($rest:tt)*) => {}; 94 (emit F32Le $($rest:tt)*) => {}; 95 (emit F64Le $($rest:tt)*) => {}; 96 (emit F32Ge $($rest:tt)*) => {}; 97 (emit F64Ge $($rest:tt)*) => {}; 98 (emit F32ConvertI32S $($rest:tt)*) => {}; 99 (emit F32ConvertI32U $($rest:tt)*) => {}; 100 (emit F32ConvertI64S $($rest:tt)*) => {}; 101 (emit F32ConvertI64U $($rest:tt)*) => {}; 102 (emit F64ConvertI32S $($rest:tt)*) => {}; 103 (emit F64ConvertI32U $($rest:tt)*) => {}; 104 (emit F64ConvertI64S $($rest:tt)*) => {}; 105 (emit F64ConvertI64U $($rest:tt)*) => {}; 106 (emit F32ReinterpretI32 $($rest:tt)*) => {}; 107 (emit F64ReinterpretI64 $($rest:tt)*) => {}; 108 (emit F32DemoteF64 $($rest:tt)*) => {}; 109 (emit F64PromoteF32 $($rest:tt)*) => {}; 110 (emit I32Add $($rest:tt)*) => {}; 111 (emit I64Add $($rest:tt)*) => {}; 112 (emit I32Sub $($rest:tt)*) => {}; 113 (emit I32Mul $($rest:tt)*) => {}; 114 (emit I32DivS $($rest:tt)*) => {}; 115 (emit I32DivU $($rest:tt)*) => {}; 116 (emit I64DivS $($rest:tt)*) => {}; 117 (emit I64DivU $($rest:tt)*) => {}; 118 (emit I64RemU $($rest:tt)*) => {}; 119 (emit I64RemS $($rest:tt)*) => {}; 120 (emit I32RemU $($rest:tt)*) => {}; 121 (emit I32RemS $($rest:tt)*) => {}; 122 (emit I64Mul $($rest:tt)*) => {}; 123 (emit I64Sub $($rest:tt)*) => {}; 124 (emit I32Eq $($rest:tt)*) => {}; 125 (emit I64Eq $($rest:tt)*) => {}; 126 (emit I32Ne $($rest:tt)*) => {}; 127 (emit I64Ne $($rest:tt)*) => {}; 128 (emit I32LtS $($rest:tt)*) => {}; 129 (emit I64LtS $($rest:tt)*) => {}; 130 (emit I32LtU $($rest:tt)*) => {}; 131 (emit I64LtU $($rest:tt)*) => {}; 132 (emit I32LeS $($rest:tt)*) => {}; 133 (emit I64LeS $($rest:tt)*) => {}; 134 (emit I32LeU $($rest:tt)*) => {}; 135 (emit I64LeU $($rest:tt)*) => {}; 136 (emit I32GtS $($rest:tt)*) => {}; 137 (emit I64GtS $($rest:tt)*) => {}; 138 (emit I32GtU $($rest:tt)*) => {}; 139 (emit I64GtU $($rest:tt)*) => {}; 140 (emit I32GeS $($rest:tt)*) => {}; 141 (emit I64GeS $($rest:tt)*) => {}; 142 (emit I32GeU $($rest:tt)*) => {}; 143 (emit I64GeU $($rest:tt)*) => {}; 144 (emit I32Eqz $($rest:tt)*) => {}; 145 (emit I64Eqz $($rest:tt)*) => {}; 146 (emit I32And $($rest:tt)*) => {}; 147 (emit I64And $($rest:tt)*) => {}; 148 (emit I32Or $($rest:tt)*) => {}; 149 (emit I64Or $($rest:tt)*) => {}; 150 (emit I32Xor $($rest:tt)*) => {}; 151 (emit I64Xor $($rest:tt)*) => {}; 152 (emit I32Shl $($rest:tt)*) => {}; 153 (emit I64Shl $($rest:tt)*) => {}; 154 (emit I32ShrS $($rest:tt)*) => {}; 155 (emit I64ShrS $($rest:tt)*) => {}; 156 (emit I32ShrU $($rest:tt)*) => {}; 157 (emit I64ShrU $($rest:tt)*) => {}; 158 (emit I32Rotl $($rest:tt)*) => {}; 159 (emit I64Rotl $($rest:tt)*) => {}; 160 (emit I32Rotr $($rest:tt)*) => {}; 161 (emit I64Rotr $($rest:tt)*) => {}; 162 (emit I32Clz $($rest:tt)*) => {}; 163 (emit I64Clz $($rest:tt)*) => {}; 164 (emit I32Ctz $($rest:tt)*) => {}; 165 (emit I64Ctz $($rest:tt)*) => {}; 166 (emit I32Popcnt $($rest:tt)*) => {}; 167 (emit I64Popcnt $($rest:tt)*) => {}; 168 (emit I32WrapI64 $($rest:tt)*) => {}; 169 (emit I64ExtendI32S $($rest:tt)*) => {}; 170 (emit I64ExtendI32U $($rest:tt)*) => {}; 171 (emit I32Extend8S $($rest:tt)*) => {}; 172 (emit I32Extend16S $($rest:tt)*) => {}; 173 (emit I64Extend8S $($rest:tt)*) => {}; 174 (emit I64Extend16S $($rest:tt)*) => {}; 175 (emit I64Extend32S $($rest:tt)*) => {}; 176 (emit I32TruncF32S $($rest:tt)*) => {}; 177 (emit I32TruncF32U $($rest:tt)*) => {}; 178 (emit I32TruncF64S $($rest:tt)*) => {}; 179 (emit I32TruncF64U $($rest:tt)*) => {}; 180 (emit I64TruncF32S $($rest:tt)*) => {}; 181 (emit I64TruncF32U $($rest:tt)*) => {}; 182 (emit I64TruncF64S $($rest:tt)*) => {}; 183 (emit I64TruncF64U $($rest:tt)*) => {}; 184 (emit I32ReinterpretF32 $($rest:tt)*) => {}; 185 (emit I64ReinterpretF64 $($rest:tt)*) => {}; 186 (emit LocalGet $($rest:tt)*) => {}; 187 (emit LocalSet $($rest:tt)*) => {}; 188 (emit Call $($rest:tt)*) => {}; 189 (emit End $($rest:tt)*) => {}; 190 (emit Nop $($rest:tt)*) => {}; 191 (emit If $($rest:tt)*) => {}; 192 (emit Else $($rest:tt)*) => {}; 193 (emit Block $($rest:tt)*) => {}; 194 (emit Loop $($rest:tt)*) => {}; 195 (emit Br $($rest:tt)*) => {}; 196 (emit BrIf $($rest:tt)*) => {}; 197 (emit Return $($rest:tt)*) => {}; 198 (emit Unreachable $($rest:tt)*) => {}; 199 (emit LocalTee $($rest:tt)*) => {}; 200 (emit GlobalGet $($rest:tt)*) => {}; 201 (emit GlobalSet $($rest:tt)*) => {}; 202 (emit Select $($rest:tt)*) => {}; 203 (emit Drop $($rest:tt)*) => {}; 204 (emit BrTable $($rest:tt)*) => {}; 205 (emit CallIndirect $($rest:tt)*) => {}; 206 (emit TableInit $($rest:tt)*) => {}; 207 (emit TableCopy $($rest:tt)*) => {}; 208 (emit TableGet $($rest:tt)*) => {}; 209 (emit TableSet $($rest:tt)*) => {}; 210 (emit TableGrow $($rest:tt)*) => {}; 211 (emit TableSize $($rest:tt)*) => {}; 212 (emit TableFill $($rest:tt)*) => {}; 213 (emit ElemDrop $($rest:tt)*) => {}; 214 (emit MemoryInit $($rest:tt)*) => {}; 215 (emit MemoryCopy $($rest:tt)*) => {}; 216 (emit DataDrop $($rest:tt)*) => {}; 217 (emit MemoryFill $($rest:tt)*) => {}; 218 (emit MemorySize $($rest:tt)*) => {}; 219 (emit MemoryGrow $($rest:tt)*) => {}; 220 (emit I32Load $($rest:tt)*) => {}; 221 (emit I32Load8S $($rest:tt)*) => {}; 222 (emit I32Load8U $($rest:tt)*) => {}; 223 (emit I32Load16S $($rest:tt)*) => {}; 224 (emit I32Load16U $($rest:tt)*) => {}; 225 (emit I64Load8S $($rest:tt)*) => {}; 226 (emit I64Load8U $($rest:tt)*) => {}; 227 (emit I64Load16S $($rest:tt)*) => {}; 228 (emit I64Load16U $($rest:tt)*) => {}; 229 (emit I64Load32S $($rest:tt)*) => {}; 230 (emit I64Load32U $($rest:tt)*) => {}; 231 (emit I64Load $($rest:tt)*) => {}; 232 (emit I32Store $($rest:tt)*) => {}; 233 (emit I32Store8 $($rest:tt)*) => {}; 234 (emit I32Store16 $($rest:tt)*) => {}; 235 (emit I64Store $($rest:tt)*) => {}; 236 (emit I64Store8 $($rest:tt)*) => {}; 237 (emit I64Store16 $($rest:tt)*) => {}; 238 (emit I64Store32 $($rest:tt)*) => {}; 239 (emit F32Load $($rest:tt)*) => {}; 240 (emit F32Store $($rest:tt)*) => {}; 241 (emit F64Load $($rest:tt)*) => {}; 242 (emit F64Store $($rest:tt)*) => {}; 243 (emit I32TruncSatF32S $($rest:tt)*) => {}; 244 (emit I32TruncSatF32U $($rest:tt)*) => {}; 245 (emit I32TruncSatF64S $($rest:tt)*) => {}; 246 (emit I32TruncSatF64U $($rest:tt)*) => {}; 247 (emit I64TruncSatF32S $($rest:tt)*) => {}; 248 (emit I64TruncSatF32U $($rest:tt)*) => {}; 249 (emit I64TruncSatF64S $($rest:tt)*) => {}; 250 (emit I64TruncSatF64U $($rest:tt)*) => {}; 251 (emit V128Load $($rest:tt)*) => {}; 252 (emit V128Store $($rest:tt)*) => {}; 253 (emit I64Add128 $($rest:tt)*) => {}; 254 (emit I64Sub128 $($rest:tt)*) => {}; 255 (emit I64MulWideS $($rest:tt)*) => {}; 256 (emit I64MulWideU $($rest:tt)*) => {}; 257 (emit I32AtomicLoad8U $($rest:tt)*) => {}; 258 (emit I32AtomicLoad16U $($rest:tt)*) => {}; 259 (emit I32AtomicLoad $($rest:tt)*) => {}; 260 (emit I64AtomicLoad8U $($rest:tt)*) => {}; 261 (emit I64AtomicLoad16U $($rest:tt)*) => {}; 262 (emit I64AtomicLoad32U $($rest:tt)*) => {}; 263 (emit I64AtomicLoad $($rest:tt)*) => {}; 264 (emit V128Load8x8S $($rest:tt)*) => {}; 265 (emit V128Load8x8U $($rest:tt)*) => {}; 266 (emit V128Load16x4S $($rest:tt)*) => {}; 267 (emit V128Load16x4U $($rest:tt)*) => {}; 268 (emit V128Load32x2S $($rest:tt)*) => {}; 269 (emit V128Load32x2U $($rest:tt)*) => {}; 270 (emit V128Load8Splat $($rest:tt)*) => {}; 271 (emit V128Load16Splat $($rest:tt)*) => {}; 272 (emit V128Load32Splat $($rest:tt)*) => {}; 273 (emit V128Load64Splat $($rest:tt)*) => {}; 274 (emit I8x16Splat $($rest:tt)*) => {}; 275 (emit I16x8Splat $($rest:tt)*) => {}; 276 (emit I32x4Splat $($rest:tt)*) => {}; 277 (emit I64x2Splat $($rest:tt)*) => {}; 278 (emit F32x4Splat $($rest:tt)*) => {}; 279 (emit F64x2Splat $($rest:tt)*) => {}; 280 (emit I32AtomicStore8 $($rest:tt)*) => {}; 281 (emit I32AtomicStore16 $($rest:tt)*) => {}; 282 (emit I32AtomicStore $($rest:tt)*) => {}; 283 (emit I64AtomicStore8 $($rest:tt)*) => {}; 284 (emit I64AtomicStore16 $($rest:tt)*) => {}; 285 (emit I64AtomicStore32 $($rest:tt)*) => {}; 286 (emit I64AtomicStore $($rest:tt)*) => {}; 287 (emit I32AtomicRmw8AddU $($rest:tt)*) => {}; 288 (emit I32AtomicRmw16AddU $($rest:tt)*) => {}; 289 (emit I32AtomicRmwAdd $($rest:tt)*) => {}; 290 (emit I64AtomicRmw8AddU $($rest:tt)*) => {}; 291 (emit I64AtomicRmw16AddU $($rest:tt)*) => {}; 292 (emit I64AtomicRmw32AddU $($rest:tt)*) => {}; 293 (emit I64AtomicRmwAdd $($rest:tt)*) => {}; 294 (emit I8x16Shuffle $($rest:tt)*) => {}; 295 (emit I32AtomicRmw8SubU $($rest:tt)*) => {}; 296 (emit I32AtomicRmw16SubU $($rest:tt)*) => {}; 297 (emit I32AtomicRmwSub $($rest:tt)*) => {}; 298 (emit I64AtomicRmw8SubU $($rest:tt)*) => {}; 299 (emit I64AtomicRmw16SubU $($rest:tt)*) => {}; 300 (emit I64AtomicRmw32SubU $($rest:tt)*) => {}; 301 (emit I64AtomicRmwSub $($rest:tt)*) => {}; 302 303 (emit $unsupported:tt $($rest:tt)*) => {$($rest)*}; 304 } 305 306 impl<'a, 'translation, 'data, M> VisitOperator<'a> for CodeGen<'a, 'translation, 'data, M, Emission> 307 where 308 M: MacroAssembler, 309 { 310 type Output = Result<()>; 311 312 fn visit_i32_const(&mut self, val: i32) -> Self::Output { 313 self.context.stack.push(Val::i32(val)); 314 315 Ok(()) 316 } 317 318 fn visit_i64_const(&mut self, val: i64) -> Self::Output { 319 self.context.stack.push(Val::i64(val)); 320 Ok(()) 321 } 322 323 fn visit_f32_const(&mut self, val: Ieee32) -> Self::Output { 324 self.context.stack.push(Val::f32(val)); 325 Ok(()) 326 } 327 328 fn visit_f64_const(&mut self, val: Ieee64) -> Self::Output { 329 self.context.stack.push(Val::f64(val)); 330 Ok(()) 331 } 332 333 fn visit_f32_add(&mut self) -> Self::Output { 334 self.context.binop( 335 self.masm, 336 OperandSize::S32, 337 &mut |masm: &mut M, dst, src, size| { 338 masm.float_add(writable!(dst), dst, src, size)?; 339 Ok(TypedReg::f32(dst)) 340 }, 341 ) 342 } 343 344 fn visit_f64_add(&mut self) -> Self::Output { 345 self.context.binop( 346 self.masm, 347 OperandSize::S64, 348 &mut |masm: &mut M, dst, src, size| { 349 masm.float_add(writable!(dst), dst, src, size)?; 350 Ok(TypedReg::f64(dst)) 351 }, 352 ) 353 } 354 355 fn visit_f32_sub(&mut self) -> Self::Output { 356 self.context.binop( 357 self.masm, 358 OperandSize::S32, 359 &mut |masm: &mut M, dst, src, size| { 360 masm.float_sub(writable!(dst), dst, src, size)?; 361 Ok(TypedReg::f32(dst)) 362 }, 363 ) 364 } 365 366 fn visit_f64_sub(&mut self) -> Self::Output { 367 self.context.binop( 368 self.masm, 369 OperandSize::S64, 370 &mut |masm: &mut M, dst, src, size| { 371 masm.float_sub(writable!(dst), dst, src, size)?; 372 Ok(TypedReg::f64(dst)) 373 }, 374 ) 375 } 376 377 fn visit_f32_mul(&mut self) -> Self::Output { 378 self.context.binop( 379 self.masm, 380 OperandSize::S32, 381 &mut |masm: &mut M, dst, src, size| { 382 masm.float_mul(writable!(dst), dst, src, size)?; 383 Ok(TypedReg::f32(dst)) 384 }, 385 ) 386 } 387 388 fn visit_f64_mul(&mut self) -> Self::Output { 389 self.context.binop( 390 self.masm, 391 OperandSize::S64, 392 &mut |masm: &mut M, dst, src, size| { 393 masm.float_mul(writable!(dst), dst, src, size)?; 394 Ok(TypedReg::f64(dst)) 395 }, 396 ) 397 } 398 399 fn visit_f32_div(&mut self) -> Self::Output { 400 self.context.binop( 401 self.masm, 402 OperandSize::S32, 403 &mut |masm: &mut M, dst, src, size| { 404 masm.float_div(writable!(dst), dst, src, size)?; 405 Ok(TypedReg::f32(dst)) 406 }, 407 ) 408 } 409 410 fn visit_f64_div(&mut self) -> Self::Output { 411 self.context.binop( 412 self.masm, 413 OperandSize::S64, 414 &mut |masm: &mut M, dst, src, size| { 415 masm.float_div(writable!(dst), dst, src, size)?; 416 Ok(TypedReg::f64(dst)) 417 }, 418 ) 419 } 420 421 fn visit_f32_min(&mut self) -> Self::Output { 422 self.context.binop( 423 self.masm, 424 OperandSize::S32, 425 &mut |masm: &mut M, dst, src, size| { 426 masm.float_min(writable!(dst), dst, src, size)?; 427 Ok(TypedReg::f32(dst)) 428 }, 429 ) 430 } 431 432 fn visit_f64_min(&mut self) -> Self::Output { 433 self.context.binop( 434 self.masm, 435 OperandSize::S64, 436 &mut |masm: &mut M, dst, src, size| { 437 masm.float_min(writable!(dst), dst, src, size)?; 438 Ok(TypedReg::f64(dst)) 439 }, 440 ) 441 } 442 443 fn visit_f32_max(&mut self) -> Self::Output { 444 self.context.binop( 445 self.masm, 446 OperandSize::S32, 447 &mut |masm: &mut M, dst, src, size| { 448 masm.float_max(writable!(dst), dst, src, size)?; 449 Ok(TypedReg::f32(dst)) 450 }, 451 ) 452 } 453 454 fn visit_f64_max(&mut self) -> Self::Output { 455 self.context.binop( 456 self.masm, 457 OperandSize::S64, 458 &mut |masm: &mut M, dst, src, size| { 459 masm.float_max(writable!(dst), dst, src, size)?; 460 Ok(TypedReg::f64(dst)) 461 }, 462 ) 463 } 464 465 fn visit_f32_copysign(&mut self) -> Self::Output { 466 self.context.binop( 467 self.masm, 468 OperandSize::S32, 469 &mut |masm: &mut M, dst, src, size| { 470 masm.float_copysign(writable!(dst), dst, src, size)?; 471 Ok(TypedReg::f32(dst)) 472 }, 473 ) 474 } 475 476 fn visit_f64_copysign(&mut self) -> Self::Output { 477 self.context.binop( 478 self.masm, 479 OperandSize::S64, 480 &mut |masm: &mut M, dst, src, size| { 481 masm.float_copysign(writable!(dst), dst, src, size)?; 482 Ok(TypedReg::f64(dst)) 483 }, 484 ) 485 } 486 487 fn visit_f32_abs(&mut self) -> Self::Output { 488 self.context.unop(self.masm, &mut |masm, reg| { 489 masm.float_abs(writable!(reg), OperandSize::S32)?; 490 Ok(TypedReg::f32(reg)) 491 }) 492 } 493 494 fn visit_f64_abs(&mut self) -> Self::Output { 495 self.context.unop(self.masm, &mut |masm, reg| { 496 masm.float_abs(writable!(reg), OperandSize::S64)?; 497 Ok(TypedReg::f64(reg)) 498 }) 499 } 500 501 fn visit_f32_neg(&mut self) -> Self::Output { 502 self.context.unop(self.masm, &mut |masm, reg| { 503 masm.float_neg(writable!(reg), OperandSize::S32)?; 504 Ok(TypedReg::f32(reg)) 505 }) 506 } 507 508 fn visit_f64_neg(&mut self) -> Self::Output { 509 self.context.unop(self.masm, &mut |masm, reg| { 510 masm.float_neg(writable!(reg), OperandSize::S64)?; 511 Ok(TypedReg::f64(reg)) 512 }) 513 } 514 515 fn visit_f32_floor(&mut self) -> Self::Output { 516 self.masm.float_round( 517 RoundingMode::Down, 518 &mut self.env, 519 &mut self.context, 520 OperandSize::S32, 521 |env, cx, masm| { 522 let builtin = env.builtins.floor_f32::<M::ABI>()?; 523 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 524 }, 525 ) 526 } 527 528 fn visit_f64_floor(&mut self) -> Self::Output { 529 self.masm.float_round( 530 RoundingMode::Down, 531 &mut self.env, 532 &mut self.context, 533 OperandSize::S64, 534 |env, cx, masm| { 535 let builtin = env.builtins.floor_f64::<M::ABI>()?; 536 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 537 }, 538 ) 539 } 540 541 fn visit_f32_ceil(&mut self) -> Self::Output { 542 self.masm.float_round( 543 RoundingMode::Up, 544 &mut self.env, 545 &mut self.context, 546 OperandSize::S32, 547 |env, cx, masm| { 548 let builtin = env.builtins.ceil_f32::<M::ABI>()?; 549 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 550 }, 551 ) 552 } 553 554 fn visit_f64_ceil(&mut self) -> Self::Output { 555 self.masm.float_round( 556 RoundingMode::Up, 557 &mut self.env, 558 &mut self.context, 559 OperandSize::S64, 560 |env, cx, masm| { 561 let builtin = env.builtins.ceil_f64::<M::ABI>()?; 562 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 563 }, 564 ) 565 } 566 567 fn visit_f32_nearest(&mut self) -> Self::Output { 568 self.masm.float_round( 569 RoundingMode::Nearest, 570 &mut self.env, 571 &mut self.context, 572 OperandSize::S32, 573 |env, cx, masm| { 574 let builtin = env.builtins.nearest_f32::<M::ABI>()?; 575 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 576 }, 577 ) 578 } 579 580 fn visit_f64_nearest(&mut self) -> Self::Output { 581 self.masm.float_round( 582 RoundingMode::Nearest, 583 &mut self.env, 584 &mut self.context, 585 OperandSize::S64, 586 |env, cx, masm| { 587 let builtin = env.builtins.nearest_f64::<M::ABI>()?; 588 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 589 }, 590 ) 591 } 592 593 fn visit_f32_trunc(&mut self) -> Self::Output { 594 self.masm.float_round( 595 RoundingMode::Zero, 596 &mut self.env, 597 &mut self.context, 598 OperandSize::S32, 599 |env, cx, masm| { 600 let builtin = env.builtins.trunc_f32::<M::ABI>()?; 601 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 602 }, 603 ) 604 } 605 606 fn visit_f64_trunc(&mut self) -> Self::Output { 607 self.masm.float_round( 608 RoundingMode::Zero, 609 &mut self.env, 610 &mut self.context, 611 OperandSize::S64, 612 |env, cx, masm| { 613 let builtin = env.builtins.trunc_f64::<M::ABI>()?; 614 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 615 }, 616 ) 617 } 618 619 fn visit_f32_sqrt(&mut self) -> Self::Output { 620 self.context.unop(self.masm, &mut |masm, reg| { 621 masm.float_sqrt(writable!(reg), reg, OperandSize::S32)?; 622 Ok(TypedReg::f32(reg)) 623 }) 624 } 625 626 fn visit_f64_sqrt(&mut self) -> Self::Output { 627 self.context.unop(self.masm, &mut |masm, reg| { 628 masm.float_sqrt(writable!(reg), reg, OperandSize::S64)?; 629 Ok(TypedReg::f64(reg)) 630 }) 631 } 632 633 fn visit_f32_eq(&mut self) -> Self::Output { 634 self.context.float_cmp_op( 635 self.masm, 636 OperandSize::S32, 637 &mut |masm: &mut M, dst, src1, src2, size| { 638 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Eq, size) 639 }, 640 ) 641 } 642 643 fn visit_f64_eq(&mut self) -> Self::Output { 644 self.context.float_cmp_op( 645 self.masm, 646 OperandSize::S64, 647 &mut |masm: &mut M, dst, src1, src2, size| { 648 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Eq, size) 649 }, 650 ) 651 } 652 653 fn visit_f32_ne(&mut self) -> Self::Output { 654 self.context.float_cmp_op( 655 self.masm, 656 OperandSize::S32, 657 &mut |masm: &mut M, dst, src1, src2, size| { 658 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ne, size) 659 }, 660 ) 661 } 662 663 fn visit_f64_ne(&mut self) -> Self::Output { 664 self.context.float_cmp_op( 665 self.masm, 666 OperandSize::S64, 667 &mut |masm: &mut M, dst, src1, src2, size| { 668 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ne, size) 669 }, 670 ) 671 } 672 673 fn visit_f32_lt(&mut self) -> Self::Output { 674 self.context.float_cmp_op( 675 self.masm, 676 OperandSize::S32, 677 &mut |masm: &mut M, dst, src1, src2, size| { 678 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Lt, size) 679 }, 680 ) 681 } 682 683 fn visit_f64_lt(&mut self) -> Self::Output { 684 self.context.float_cmp_op( 685 self.masm, 686 OperandSize::S64, 687 &mut |masm: &mut M, dst, src1, src2, size| { 688 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Lt, size) 689 }, 690 ) 691 } 692 693 fn visit_f32_gt(&mut self) -> Self::Output { 694 self.context.float_cmp_op( 695 self.masm, 696 OperandSize::S32, 697 &mut |masm: &mut M, dst, src1, src2, size| { 698 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Gt, size) 699 }, 700 ) 701 } 702 703 fn visit_f64_gt(&mut self) -> Self::Output { 704 self.context.float_cmp_op( 705 self.masm, 706 OperandSize::S64, 707 &mut |masm: &mut M, dst, src1, src2, size| { 708 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Gt, size) 709 }, 710 ) 711 } 712 713 fn visit_f32_le(&mut self) -> Self::Output { 714 self.context.float_cmp_op( 715 self.masm, 716 OperandSize::S32, 717 &mut |masm: &mut M, dst, src1, src2, size| { 718 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Le, size) 719 }, 720 ) 721 } 722 723 fn visit_f64_le(&mut self) -> Self::Output { 724 self.context.float_cmp_op( 725 self.masm, 726 OperandSize::S64, 727 &mut |masm: &mut M, dst, src1, src2, size| { 728 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Le, size) 729 }, 730 ) 731 } 732 733 fn visit_f32_ge(&mut self) -> Self::Output { 734 self.context.float_cmp_op( 735 self.masm, 736 OperandSize::S32, 737 &mut |masm: &mut M, dst, src1, src2, size| { 738 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ge, size) 739 }, 740 ) 741 } 742 743 fn visit_f64_ge(&mut self) -> Self::Output { 744 self.context.float_cmp_op( 745 self.masm, 746 OperandSize::S64, 747 &mut |masm: &mut M, dst, src1, src2, size| { 748 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ge, size) 749 }, 750 ) 751 } 752 753 fn visit_f32_convert_i32_s(&mut self) -> Self::Output { 754 self.context 755 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 756 masm.signed_convert(writable!(dst), src, OperandSize::S32, dst_size) 757 }) 758 } 759 760 fn visit_f32_convert_i32_u(&mut self) -> Self::Output { 761 self.context.convert_op_with_tmp_reg( 762 self.masm, 763 WasmValType::F32, 764 RegClass::Int, 765 |masm, dst, src, tmp_gpr, dst_size| { 766 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S32, dst_size) 767 }, 768 ) 769 } 770 771 fn visit_f32_convert_i64_s(&mut self) -> Self::Output { 772 self.context 773 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 774 masm.signed_convert(writable!(dst), src, OperandSize::S64, dst_size) 775 }) 776 } 777 778 fn visit_f32_convert_i64_u(&mut self) -> Self::Output { 779 self.context.convert_op_with_tmp_reg( 780 self.masm, 781 WasmValType::F32, 782 RegClass::Int, 783 |masm, dst, src, tmp_gpr, dst_size| { 784 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S64, dst_size) 785 }, 786 ) 787 } 788 789 fn visit_f64_convert_i32_s(&mut self) -> Self::Output { 790 self.context 791 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 792 masm.signed_convert(writable!(dst), src, OperandSize::S32, dst_size) 793 }) 794 } 795 796 fn visit_f64_convert_i32_u(&mut self) -> Self::Output { 797 self.context.convert_op_with_tmp_reg( 798 self.masm, 799 WasmValType::F64, 800 RegClass::Int, 801 |masm, dst, src, tmp_gpr, dst_size| { 802 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S32, dst_size) 803 }, 804 ) 805 } 806 807 fn visit_f64_convert_i64_s(&mut self) -> Self::Output { 808 self.context 809 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 810 masm.signed_convert(writable!(dst), src, OperandSize::S64, dst_size) 811 }) 812 } 813 814 fn visit_f64_convert_i64_u(&mut self) -> Self::Output { 815 self.context.convert_op_with_tmp_reg( 816 self.masm, 817 WasmValType::F64, 818 RegClass::Int, 819 |masm, dst, src, tmp_gpr, dst_size| { 820 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S64, dst_size) 821 }, 822 ) 823 } 824 825 fn visit_f32_reinterpret_i32(&mut self) -> Self::Output { 826 self.context 827 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, size| { 828 masm.reinterpret_int_as_float(writable!(dst), src.into(), size) 829 }) 830 } 831 832 fn visit_f64_reinterpret_i64(&mut self) -> Self::Output { 833 self.context 834 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, size| { 835 masm.reinterpret_int_as_float(writable!(dst), src.into(), size) 836 }) 837 } 838 839 fn visit_f32_demote_f64(&mut self) -> Self::Output { 840 self.context.unop(self.masm, &mut |masm, reg| { 841 masm.demote(writable!(reg), reg)?; 842 Ok(TypedReg::f32(reg)) 843 }) 844 } 845 846 fn visit_f64_promote_f32(&mut self) -> Self::Output { 847 self.context.unop(self.masm, &mut |masm, reg| { 848 masm.promote(writable!(reg), reg)?; 849 Ok(TypedReg::f64(reg)) 850 }) 851 } 852 853 fn visit_i32_add(&mut self) -> Self::Output { 854 self.context.i32_binop(self.masm, |masm, dst, src, size| { 855 masm.add(writable!(dst), dst, src, size)?; 856 Ok(TypedReg::i32(dst)) 857 }) 858 } 859 860 fn visit_i64_add(&mut self) -> Self::Output { 861 self.context.i64_binop(self.masm, |masm, dst, src, size| { 862 masm.add(writable!(dst), dst, src, size)?; 863 Ok(TypedReg::i64(dst)) 864 }) 865 } 866 867 fn visit_i32_sub(&mut self) -> Self::Output { 868 self.context.i32_binop(self.masm, |masm, dst, src, size| { 869 masm.sub(writable!(dst), dst, src, size)?; 870 Ok(TypedReg::i32(dst)) 871 }) 872 } 873 874 fn visit_i64_sub(&mut self) -> Self::Output { 875 self.context.i64_binop(self.masm, |masm, dst, src, size| { 876 masm.sub(writable!(dst), dst, src, size)?; 877 Ok(TypedReg::i64(dst)) 878 }) 879 } 880 881 fn visit_i32_mul(&mut self) -> Self::Output { 882 self.context.i32_binop(self.masm, |masm, dst, src, size| { 883 masm.mul(writable!(dst), dst, src, size)?; 884 Ok(TypedReg::i32(dst)) 885 }) 886 } 887 888 fn visit_i64_mul(&mut self) -> Self::Output { 889 self.context.i64_binop(self.masm, |masm, dst, src, size| { 890 masm.mul(writable!(dst), dst, src, size)?; 891 Ok(TypedReg::i64(dst)) 892 }) 893 } 894 895 fn visit_i32_div_s(&mut self) -> Self::Output { 896 use DivKind::*; 897 use OperandSize::*; 898 899 self.masm.div(&mut self.context, Signed, S32) 900 } 901 902 fn visit_i32_div_u(&mut self) -> Self::Output { 903 use DivKind::*; 904 use OperandSize::*; 905 906 self.masm.div(&mut self.context, Unsigned, S32) 907 } 908 909 fn visit_i64_div_s(&mut self) -> Self::Output { 910 use DivKind::*; 911 use OperandSize::*; 912 913 self.masm.div(&mut self.context, Signed, S64) 914 } 915 916 fn visit_i64_div_u(&mut self) -> Self::Output { 917 use DivKind::*; 918 use OperandSize::*; 919 920 self.masm.div(&mut self.context, Unsigned, S64) 921 } 922 923 fn visit_i32_rem_s(&mut self) -> Self::Output { 924 use OperandSize::*; 925 use RemKind::*; 926 927 self.masm.rem(&mut self.context, Signed, S32) 928 } 929 930 fn visit_i32_rem_u(&mut self) -> Self::Output { 931 use OperandSize::*; 932 use RemKind::*; 933 934 self.masm.rem(&mut self.context, Unsigned, S32) 935 } 936 937 fn visit_i64_rem_s(&mut self) -> Self::Output { 938 use OperandSize::*; 939 use RemKind::*; 940 941 self.masm.rem(&mut self.context, Signed, S64) 942 } 943 944 fn visit_i64_rem_u(&mut self) -> Self::Output { 945 use OperandSize::*; 946 use RemKind::*; 947 948 self.masm.rem(&mut self.context, Unsigned, S64) 949 } 950 951 fn visit_i32_eq(&mut self) -> Self::Output { 952 self.cmp_i32s(IntCmpKind::Eq) 953 } 954 955 fn visit_i64_eq(&mut self) -> Self::Output { 956 self.cmp_i64s(IntCmpKind::Eq) 957 } 958 959 fn visit_i32_ne(&mut self) -> Self::Output { 960 self.cmp_i32s(IntCmpKind::Ne) 961 } 962 963 fn visit_i64_ne(&mut self) -> Self::Output { 964 self.cmp_i64s(IntCmpKind::Ne) 965 } 966 967 fn visit_i32_lt_s(&mut self) -> Self::Output { 968 self.cmp_i32s(IntCmpKind::LtS) 969 } 970 971 fn visit_i64_lt_s(&mut self) -> Self::Output { 972 self.cmp_i64s(IntCmpKind::LtS) 973 } 974 975 fn visit_i32_lt_u(&mut self) -> Self::Output { 976 self.cmp_i32s(IntCmpKind::LtU) 977 } 978 979 fn visit_i64_lt_u(&mut self) -> Self::Output { 980 self.cmp_i64s(IntCmpKind::LtU) 981 } 982 983 fn visit_i32_le_s(&mut self) -> Self::Output { 984 self.cmp_i32s(IntCmpKind::LeS) 985 } 986 987 fn visit_i64_le_s(&mut self) -> Self::Output { 988 self.cmp_i64s(IntCmpKind::LeS) 989 } 990 991 fn visit_i32_le_u(&mut self) -> Self::Output { 992 self.cmp_i32s(IntCmpKind::LeU) 993 } 994 995 fn visit_i64_le_u(&mut self) -> Self::Output { 996 self.cmp_i64s(IntCmpKind::LeU) 997 } 998 999 fn visit_i32_gt_s(&mut self) -> Self::Output { 1000 self.cmp_i32s(IntCmpKind::GtS) 1001 } 1002 1003 fn visit_i64_gt_s(&mut self) -> Self::Output { 1004 self.cmp_i64s(IntCmpKind::GtS) 1005 } 1006 1007 fn visit_i32_gt_u(&mut self) -> Self::Output { 1008 self.cmp_i32s(IntCmpKind::GtU) 1009 } 1010 1011 fn visit_i64_gt_u(&mut self) -> Self::Output { 1012 self.cmp_i64s(IntCmpKind::GtU) 1013 } 1014 1015 fn visit_i32_ge_s(&mut self) -> Self::Output { 1016 self.cmp_i32s(IntCmpKind::GeS) 1017 } 1018 1019 fn visit_i64_ge_s(&mut self) -> Self::Output { 1020 self.cmp_i64s(IntCmpKind::GeS) 1021 } 1022 1023 fn visit_i32_ge_u(&mut self) -> Self::Output { 1024 self.cmp_i32s(IntCmpKind::GeU) 1025 } 1026 1027 fn visit_i64_ge_u(&mut self) -> Self::Output { 1028 self.cmp_i64s(IntCmpKind::GeU) 1029 } 1030 1031 fn visit_i32_eqz(&mut self) -> Self::Output { 1032 use OperandSize::*; 1033 1034 self.context.unop(self.masm, &mut |masm, reg| { 1035 masm.cmp_with_set(writable!(reg.into()), RegImm::i32(0), IntCmpKind::Eq, S32)?; 1036 Ok(TypedReg::i32(reg)) 1037 }) 1038 } 1039 1040 fn visit_i64_eqz(&mut self) -> Self::Output { 1041 use OperandSize::*; 1042 1043 self.context.unop(self.masm, &mut |masm, reg| { 1044 masm.cmp_with_set(writable!(reg.into()), RegImm::i64(0), IntCmpKind::Eq, S64)?; 1045 Ok(TypedReg::i32(reg)) // Return value for `i64.eqz` is an `i32`. 1046 }) 1047 } 1048 1049 fn visit_i32_clz(&mut self) -> Self::Output { 1050 use OperandSize::*; 1051 1052 self.context.unop(self.masm, &mut |masm, reg| { 1053 masm.clz(writable!(reg), reg, S32)?; 1054 Ok(TypedReg::i32(reg)) 1055 }) 1056 } 1057 1058 fn visit_i64_clz(&mut self) -> Self::Output { 1059 use OperandSize::*; 1060 1061 self.context.unop(self.masm, &mut |masm, reg| { 1062 masm.clz(writable!(reg), reg, S64)?; 1063 Ok(TypedReg::i64(reg)) 1064 }) 1065 } 1066 1067 fn visit_i32_ctz(&mut self) -> Self::Output { 1068 use OperandSize::*; 1069 1070 self.context.unop(self.masm, &mut |masm, reg| { 1071 masm.ctz(writable!(reg), reg, S32)?; 1072 Ok(TypedReg::i32(reg)) 1073 }) 1074 } 1075 1076 fn visit_i64_ctz(&mut self) -> Self::Output { 1077 use OperandSize::*; 1078 1079 self.context.unop(self.masm, &mut |masm, reg| { 1080 masm.ctz(writable!(reg), reg, S64)?; 1081 Ok(TypedReg::i64(reg)) 1082 }) 1083 } 1084 1085 fn visit_i32_and(&mut self) -> Self::Output { 1086 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1087 masm.and(writable!(dst), dst, src, size)?; 1088 Ok(TypedReg::i32(dst)) 1089 }) 1090 } 1091 1092 fn visit_i64_and(&mut self) -> Self::Output { 1093 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1094 masm.and(writable!(dst), dst, src, size)?; 1095 Ok(TypedReg::i64(dst)) 1096 }) 1097 } 1098 1099 fn visit_i32_or(&mut self) -> Self::Output { 1100 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1101 masm.or(writable!(dst), dst, src, size)?; 1102 Ok(TypedReg::i32(dst)) 1103 }) 1104 } 1105 1106 fn visit_i64_or(&mut self) -> Self::Output { 1107 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1108 masm.or(writable!(dst), dst, src, size)?; 1109 Ok(TypedReg::i64(dst)) 1110 }) 1111 } 1112 1113 fn visit_i32_xor(&mut self) -> Self::Output { 1114 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1115 masm.xor(writable!(dst), dst, src, size)?; 1116 Ok(TypedReg::i32(dst)) 1117 }) 1118 } 1119 1120 fn visit_i64_xor(&mut self) -> Self::Output { 1121 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1122 masm.xor(writable!(dst), dst, src, size)?; 1123 Ok(TypedReg::i64(dst)) 1124 }) 1125 } 1126 1127 fn visit_i32_shl(&mut self) -> Self::Output { 1128 use ShiftKind::*; 1129 1130 self.context.i32_shift(self.masm, Shl) 1131 } 1132 1133 fn visit_i64_shl(&mut self) -> Self::Output { 1134 use ShiftKind::*; 1135 1136 self.context.i64_shift(self.masm, Shl) 1137 } 1138 1139 fn visit_i32_shr_s(&mut self) -> Self::Output { 1140 use ShiftKind::*; 1141 1142 self.context.i32_shift(self.masm, ShrS) 1143 } 1144 1145 fn visit_i64_shr_s(&mut self) -> Self::Output { 1146 use ShiftKind::*; 1147 1148 self.context.i64_shift(self.masm, ShrS) 1149 } 1150 1151 fn visit_i32_shr_u(&mut self) -> Self::Output { 1152 use ShiftKind::*; 1153 1154 self.context.i32_shift(self.masm, ShrU) 1155 } 1156 1157 fn visit_i64_shr_u(&mut self) -> Self::Output { 1158 use ShiftKind::*; 1159 1160 self.context.i64_shift(self.masm, ShrU) 1161 } 1162 1163 fn visit_i32_rotl(&mut self) -> Self::Output { 1164 use ShiftKind::*; 1165 1166 self.context.i32_shift(self.masm, Rotl) 1167 } 1168 1169 fn visit_i64_rotl(&mut self) -> Self::Output { 1170 use ShiftKind::*; 1171 1172 self.context.i64_shift(self.masm, Rotl) 1173 } 1174 1175 fn visit_i32_rotr(&mut self) -> Self::Output { 1176 use ShiftKind::*; 1177 1178 self.context.i32_shift(self.masm, Rotr) 1179 } 1180 1181 fn visit_i64_rotr(&mut self) -> Self::Output { 1182 use ShiftKind::*; 1183 1184 self.context.i64_shift(self.masm, Rotr) 1185 } 1186 1187 fn visit_end(&mut self) -> Self::Output { 1188 if !self.context.reachable { 1189 self.handle_unreachable_end() 1190 } else { 1191 let mut control = self.pop_control_frame()?; 1192 control.emit_end(self.masm, &mut self.context) 1193 } 1194 } 1195 1196 fn visit_i32_popcnt(&mut self) -> Self::Output { 1197 use OperandSize::*; 1198 self.masm.popcnt(&mut self.context, S32) 1199 } 1200 1201 fn visit_i64_popcnt(&mut self) -> Self::Output { 1202 use OperandSize::*; 1203 1204 self.masm.popcnt(&mut self.context, S64) 1205 } 1206 1207 fn visit_i32_wrap_i64(&mut self) -> Self::Output { 1208 self.context.unop(self.masm, &mut |masm, reg| { 1209 masm.wrap(writable!(reg), reg)?; 1210 Ok(TypedReg::i32(reg)) 1211 }) 1212 } 1213 1214 fn visit_i64_extend_i32_s(&mut self) -> Self::Output { 1215 self.context.unop(self.masm, &mut |masm, reg| { 1216 masm.extend(writable!(reg), reg, ExtendKind::I64Extend32S)?; 1217 Ok(TypedReg::i64(reg)) 1218 }) 1219 } 1220 1221 fn visit_i64_extend_i32_u(&mut self) -> Self::Output { 1222 self.context.unop(self.masm, &mut |masm, reg| { 1223 masm.extend(writable!(reg), reg, ExtendKind::I64Extend32U)?; 1224 Ok(TypedReg::i64(reg)) 1225 }) 1226 } 1227 1228 fn visit_i32_extend8_s(&mut self) -> Self::Output { 1229 self.context.unop(self.masm, &mut |masm, reg| { 1230 masm.extend(writable!(reg), reg, ExtendKind::I32Extend8S)?; 1231 Ok(TypedReg::i32(reg)) 1232 }) 1233 } 1234 1235 fn visit_i32_extend16_s(&mut self) -> Self::Output { 1236 self.context.unop(self.masm, &mut |masm, reg| { 1237 masm.extend(writable!(reg), reg, ExtendKind::I32Extend16S)?; 1238 Ok(TypedReg::i32(reg)) 1239 }) 1240 } 1241 1242 fn visit_i64_extend8_s(&mut self) -> Self::Output { 1243 self.context.unop(self.masm, &mut |masm, reg| { 1244 masm.extend(writable!(reg), reg, ExtendKind::I64Extend8S)?; 1245 Ok(TypedReg::i64(reg)) 1246 }) 1247 } 1248 1249 fn visit_i64_extend16_s(&mut self) -> Self::Output { 1250 self.context.unop(self.masm, &mut |masm, reg| { 1251 masm.extend(writable!(reg), reg, ExtendKind::I64Extend16S)?; 1252 Ok(TypedReg::i64(reg)) 1253 }) 1254 } 1255 1256 fn visit_i64_extend32_s(&mut self) -> Self::Output { 1257 self.context.unop(self.masm, &mut |masm, reg| { 1258 masm.extend(writable!(reg), reg, ExtendKind::I64Extend32S)?; 1259 Ok(TypedReg::i64(reg)) 1260 }) 1261 } 1262 1263 fn visit_i32_trunc_f32_s(&mut self) -> Self::Output { 1264 use OperandSize::*; 1265 1266 self.context 1267 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1268 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Unchecked) 1269 }) 1270 } 1271 1272 fn visit_i32_trunc_f32_u(&mut self) -> Self::Output { 1273 use OperandSize::*; 1274 1275 self.masm 1276 .unsigned_truncate(&mut self.context, S32, S32, TruncKind::Unchecked) 1277 } 1278 1279 fn visit_i32_trunc_f64_s(&mut self) -> Self::Output { 1280 use OperandSize::*; 1281 1282 self.context 1283 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1284 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Unchecked) 1285 }) 1286 } 1287 1288 fn visit_i32_trunc_f64_u(&mut self) -> Self::Output { 1289 use OperandSize::*; 1290 self.masm 1291 .unsigned_truncate(&mut self.context, S64, S32, TruncKind::Unchecked) 1292 } 1293 1294 fn visit_i64_trunc_f32_s(&mut self) -> Self::Output { 1295 use OperandSize::*; 1296 1297 self.context 1298 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1299 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Unchecked) 1300 }) 1301 } 1302 1303 fn visit_i64_trunc_f32_u(&mut self) -> Self::Output { 1304 use OperandSize::*; 1305 1306 self.masm 1307 .unsigned_truncate(&mut self.context, S32, S64, TruncKind::Unchecked) 1308 } 1309 1310 fn visit_i64_trunc_f64_s(&mut self) -> Self::Output { 1311 use OperandSize::*; 1312 1313 self.context 1314 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1315 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Unchecked) 1316 }) 1317 } 1318 1319 fn visit_i64_trunc_f64_u(&mut self) -> Self::Output { 1320 use OperandSize::*; 1321 1322 self.masm 1323 .unsigned_truncate(&mut self.context, S64, S64, TruncKind::Unchecked) 1324 } 1325 1326 fn visit_i32_reinterpret_f32(&mut self) -> Self::Output { 1327 self.context 1328 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, size| { 1329 masm.reinterpret_float_as_int(writable!(dst), src.into(), size) 1330 }) 1331 } 1332 1333 fn visit_i64_reinterpret_f64(&mut self) -> Self::Output { 1334 self.context 1335 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, size| { 1336 masm.reinterpret_float_as_int(writable!(dst), src.into(), size) 1337 }) 1338 } 1339 1340 fn visit_local_get(&mut self, index: u32) -> Self::Output { 1341 use WasmValType::*; 1342 let context = &mut self.context; 1343 let slot = context.frame.get_wasm_local(index); 1344 match slot.ty { 1345 I32 | I64 | F32 | F64 | V128 => context.stack.push(Val::local(index, slot.ty)), 1346 Ref(rt) => match rt.heap_type { 1347 WasmHeapType::Func => context.stack.push(Val::local(index, slot.ty)), 1348 _ => bail!(CodeGenError::unsupported_wasm_type()), 1349 }, 1350 } 1351 1352 Ok(()) 1353 } 1354 1355 fn visit_local_set(&mut self, index: u32) -> Self::Output { 1356 let src = self.emit_set_local(index)?; 1357 self.context.free_reg(src); 1358 Ok(()) 1359 } 1360 1361 fn visit_call(&mut self, index: u32) -> Self::Output { 1362 let callee = self.env.callee_from_index(FuncIndex::from_u32(index)); 1363 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee)?; 1364 Ok(()) 1365 } 1366 1367 fn visit_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output { 1368 // Spill now because `emit_lazy_init_funcref` and the `FnCall::emit` 1369 // invocations will both trigger spills since they both call functions. 1370 // However, the machine instructions for the spill emitted by 1371 // `emit_lazy_funcref` will be jumped over if the funcref was previously 1372 // initialized which may result in the machine stack becoming 1373 // unbalanced. 1374 self.context.spill(self.masm)?; 1375 1376 let type_index = TypeIndex::from_u32(type_index); 1377 let table_index = TableIndex::from_u32(table_index); 1378 1379 self.emit_lazy_init_funcref(table_index)?; 1380 1381 // Perform the indirect call. 1382 // This code assumes that [`Self::emit_lazy_init_funcref`] will 1383 // push the funcref to the value stack. 1384 let funcref_ptr = self 1385 .context 1386 .stack 1387 .peek() 1388 .map(|v| v.unwrap_reg()) 1389 .ok_or_else(|| CodeGenError::missing_values_in_stack())?; 1390 self.masm 1391 .trapz(funcref_ptr.into(), TRAP_INDIRECT_CALL_TO_NULL)?; 1392 self.emit_typecheck_funcref(funcref_ptr.into(), type_index)?; 1393 1394 let callee = self.env.funcref(type_index); 1395 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee)?; 1396 Ok(()) 1397 } 1398 1399 fn visit_table_init(&mut self, elem: u32, table: u32) -> Self::Output { 1400 let at = self.context.stack.ensure_index_at(3)?; 1401 1402 self.context 1403 .stack 1404 .insert_many(at, &[table.try_into()?, elem.try_into()?]); 1405 1406 let builtin = self.env.builtins.table_init::<M::ABI, M::Ptr>()?; 1407 FnCall::emit::<M>( 1408 &mut self.env, 1409 self.masm, 1410 &mut self.context, 1411 Callee::Builtin(builtin.clone()), 1412 )?; 1413 self.context.pop_and_free(self.masm) 1414 } 1415 1416 fn visit_table_copy(&mut self, dst: u32, src: u32) -> Self::Output { 1417 let at = self.context.stack.ensure_index_at(3)?; 1418 self.context 1419 .stack 1420 .insert_many(at, &[dst.try_into()?, src.try_into()?]); 1421 1422 let builtin = self.env.builtins.table_copy::<M::ABI, M::Ptr>()?; 1423 FnCall::emit::<M>( 1424 &mut self.env, 1425 self.masm, 1426 &mut self.context, 1427 Callee::Builtin(builtin), 1428 )?; 1429 self.context.pop_and_free(self.masm) 1430 } 1431 1432 fn visit_table_get(&mut self, table: u32) -> Self::Output { 1433 let table_index = TableIndex::from_u32(table); 1434 let table = self.env.table(table_index); 1435 let heap_type = table.ref_type.heap_type; 1436 1437 match heap_type { 1438 WasmHeapType::Func => self.emit_lazy_init_funcref(table_index), 1439 _ => Err(anyhow!(CodeGenError::unsupported_wasm_type())), 1440 } 1441 } 1442 1443 fn visit_table_grow(&mut self, table: u32) -> Self::Output { 1444 let table_index = TableIndex::from_u32(table); 1445 let table_ty = self.env.table(table_index); 1446 let builtin = match table_ty.ref_type.heap_type { 1447 WasmHeapType::Func => self.env.builtins.table_grow_func_ref::<M::ABI, M::Ptr>()?, 1448 _ => bail!(CodeGenError::unsupported_wasm_type()), 1449 }; 1450 1451 let len = self.context.stack.len(); 1452 // table.grow` requires at least 2 elements on the value stack. 1453 let at = self.context.stack.ensure_index_at(2)?; 1454 1455 // The table_grow builtin expects the parameters in a different 1456 // order. 1457 // The value stack at this point should contain: 1458 // [ init_value | delta ] (stack top) 1459 // but the builtin function expects the init value as the last 1460 // argument. 1461 self.context.stack.inner_mut().swap(len - 1, len - 2); 1462 self.context.stack.insert_many(at, &[table.try_into()?]); 1463 1464 FnCall::emit::<M>( 1465 &mut self.env, 1466 self.masm, 1467 &mut self.context, 1468 Callee::Builtin(builtin.clone()), 1469 )?; 1470 1471 Ok(()) 1472 } 1473 1474 fn visit_table_size(&mut self, table: u32) -> Self::Output { 1475 let table_index = TableIndex::from_u32(table); 1476 let table_data = self.env.resolve_table_data(table_index); 1477 self.emit_compute_table_size(&table_data) 1478 } 1479 1480 fn visit_table_fill(&mut self, table: u32) -> Self::Output { 1481 let table_index = TableIndex::from_u32(table); 1482 let table_ty = self.env.table(table_index); 1483 1484 ensure!( 1485 table_ty.ref_type.heap_type == WasmHeapType::Func, 1486 CodeGenError::unsupported_wasm_type() 1487 ); 1488 1489 let builtin = self.env.builtins.table_fill_func_ref::<M::ABI, M::Ptr>()?; 1490 1491 let at = self.context.stack.ensure_index_at(3)?; 1492 1493 self.context.stack.insert_many(at, &[table.try_into()?]); 1494 FnCall::emit::<M>( 1495 &mut self.env, 1496 self.masm, 1497 &mut self.context, 1498 Callee::Builtin(builtin.clone()), 1499 )?; 1500 self.context.pop_and_free(self.masm) 1501 } 1502 1503 fn visit_table_set(&mut self, table: u32) -> Self::Output { 1504 let ptr_type = self.env.ptr_type(); 1505 let table_index = TableIndex::from_u32(table); 1506 let table_data = self.env.resolve_table_data(table_index); 1507 let table = self.env.table(table_index); 1508 match table.ref_type.heap_type { 1509 WasmHeapType::Func => { 1510 ensure!( 1511 self.tunables.table_lazy_init, 1512 CodeGenError::unsupported_table_eager_init() 1513 ); 1514 let value = self.context.pop_to_reg(self.masm, None)?; 1515 let index = self.context.pop_to_reg(self.masm, None)?; 1516 let base = self.context.any_gpr(self.masm)?; 1517 let elem_addr = 1518 self.emit_compute_table_elem_addr(index.into(), base, &table_data)?; 1519 // Set the initialized bit. 1520 self.masm.or( 1521 writable!(value.into()), 1522 value.into(), 1523 RegImm::i64(FUNCREF_INIT_BIT as i64), 1524 ptr_type.try_into()?, 1525 )?; 1526 1527 self.masm.store_ptr(value.into(), elem_addr)?; 1528 1529 self.context.free_reg(value); 1530 self.context.free_reg(index); 1531 self.context.free_reg(base); 1532 Ok(()) 1533 } 1534 _ => Err(anyhow!(CodeGenError::unsupported_wasm_type())), 1535 } 1536 } 1537 1538 fn visit_elem_drop(&mut self, index: u32) -> Self::Output { 1539 let elem_drop = self.env.builtins.elem_drop::<M::ABI, M::Ptr>()?; 1540 self.context.stack.extend([index.try_into()?]); 1541 FnCall::emit::<M>( 1542 &mut self.env, 1543 self.masm, 1544 &mut self.context, 1545 Callee::Builtin(elem_drop), 1546 )?; 1547 Ok(()) 1548 } 1549 1550 fn visit_memory_init(&mut self, data_index: u32, mem: u32) -> Self::Output { 1551 let at = self.context.stack.ensure_index_at(3)?; 1552 self.context 1553 .stack 1554 .insert_many(at, &[mem.try_into()?, data_index.try_into()?]); 1555 let builtin = self.env.builtins.memory_init::<M::ABI, M::Ptr>()?; 1556 FnCall::emit::<M>( 1557 &mut self.env, 1558 self.masm, 1559 &mut self.context, 1560 Callee::Builtin(builtin), 1561 )?; 1562 self.context.pop_and_free(self.masm) 1563 } 1564 1565 fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) -> Self::Output { 1566 // At this point, the stack is expected to contain: 1567 // [ dst_offset, src_offset, len ] 1568 // The following code inserts the missing params, so that stack contains: 1569 // [ vmctx, dst_mem, dst_offset, src_mem, src_offset, len ] 1570 // Which is the order expected by the builtin function. 1571 let _ = self.context.stack.ensure_index_at(3)?; 1572 let at = self.context.stack.ensure_index_at(2)?; 1573 self.context.stack.insert_many(at, &[src_mem.try_into()?]); 1574 1575 // One element was inserted above, so instead of 3, we use 4. 1576 let at = self.context.stack.ensure_index_at(4)?; 1577 self.context.stack.insert_many(at, &[dst_mem.try_into()?]); 1578 1579 let builtin = self.env.builtins.memory_copy::<M::ABI, M::Ptr>()?; 1580 1581 FnCall::emit::<M>( 1582 &mut self.env, 1583 self.masm, 1584 &mut self.context, 1585 Callee::Builtin(builtin), 1586 )?; 1587 self.context.pop_and_free(self.masm) 1588 } 1589 1590 fn visit_memory_fill(&mut self, mem: u32) -> Self::Output { 1591 let at = self.context.stack.ensure_index_at(3)?; 1592 1593 self.context.stack.insert_many(at, &[mem.try_into()?]); 1594 1595 let builtin = self.env.builtins.memory_fill::<M::ABI, M::Ptr>()?; 1596 FnCall::emit::<M>( 1597 &mut self.env, 1598 self.masm, 1599 &mut self.context, 1600 Callee::Builtin(builtin), 1601 )?; 1602 self.context.pop_and_free(self.masm) 1603 } 1604 1605 fn visit_memory_size(&mut self, mem: u32) -> Self::Output { 1606 let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem)); 1607 self.emit_compute_memory_size(&heap) 1608 } 1609 1610 fn visit_memory_grow(&mut self, mem: u32) -> Self::Output { 1611 let _ = self.context.stack.ensure_index_at(1)?; 1612 // The stack at this point contains: [ delta ] 1613 // The desired state is 1614 // [ vmctx, delta, index ] 1615 self.context.stack.extend([mem.try_into()?]); 1616 1617 let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem)); 1618 let builtin = self.env.builtins.memory32_grow::<M::ABI, M::Ptr>()?; 1619 FnCall::emit::<M>( 1620 &mut self.env, 1621 self.masm, 1622 &mut self.context, 1623 Callee::Builtin(builtin), 1624 )?; 1625 1626 // The memory32_grow builtin returns a pointer type, therefore we must 1627 // ensure that the return type is representative of the address space of 1628 // the heap type. 1629 match (self.env.ptr_type(), heap.index_type()) { 1630 (WasmValType::I64, WasmValType::I64) => Ok(()), 1631 // When the heap type is smaller than the pointer type, we adjust 1632 // the result of the memory32_grow builtin. 1633 (WasmValType::I64, WasmValType::I32) => { 1634 let top: Reg = self.context.pop_to_reg(self.masm, None)?.into(); 1635 self.masm.wrap(writable!(top.into()), top.into())?; 1636 self.context.stack.push(TypedReg::i32(top).into()); 1637 Ok(()) 1638 } 1639 _ => Err(anyhow!(CodeGenError::unsupported_32_bit_platform())), 1640 } 1641 } 1642 1643 fn visit_data_drop(&mut self, data_index: u32) -> Self::Output { 1644 self.context.stack.extend([data_index.try_into()?]); 1645 1646 let builtin = self.env.builtins.data_drop::<M::ABI, M::Ptr>()?; 1647 FnCall::emit::<M>( 1648 &mut self.env, 1649 self.masm, 1650 &mut self.context, 1651 Callee::Builtin(builtin), 1652 ) 1653 } 1654 1655 fn visit_nop(&mut self) -> Self::Output { 1656 Ok(()) 1657 } 1658 1659 fn visit_if(&mut self, blockty: BlockType) -> Self::Output { 1660 self.control_frames.push(ControlStackFrame::r#if( 1661 self.env.resolve_block_sig(blockty), 1662 self.masm, 1663 &mut self.context, 1664 )?); 1665 1666 Ok(()) 1667 } 1668 1669 fn visit_else(&mut self) -> Self::Output { 1670 if !self.context.reachable { 1671 self.handle_unreachable_else() 1672 } else { 1673 let control = self 1674 .control_frames 1675 .last_mut() 1676 .ok_or_else(|| CodeGenError::control_frame_expected())?; 1677 control.emit_else(self.masm, &mut self.context) 1678 } 1679 } 1680 1681 fn visit_block(&mut self, blockty: BlockType) -> Self::Output { 1682 self.control_frames.push(ControlStackFrame::block( 1683 self.env.resolve_block_sig(blockty), 1684 self.masm, 1685 &mut self.context, 1686 )?); 1687 1688 Ok(()) 1689 } 1690 1691 fn visit_loop(&mut self, blockty: BlockType) -> Self::Output { 1692 self.control_frames.push(ControlStackFrame::r#loop( 1693 self.env.resolve_block_sig(blockty), 1694 self.masm, 1695 &mut self.context, 1696 )?); 1697 1698 self.maybe_emit_epoch_check()?; 1699 self.maybe_emit_fuel_check() 1700 } 1701 1702 fn visit_br(&mut self, depth: u32) -> Self::Output { 1703 let index = control_index(depth, self.control_frames.len())?; 1704 let frame = &mut self.control_frames[index]; 1705 self.context 1706 .unconditional_jump(frame, self.masm, |masm, cx, frame| { 1707 frame.pop_abi_results::<M, _>(cx, masm, |results, _, _| { 1708 Ok(results.ret_area().copied()) 1709 }) 1710 }) 1711 } 1712 1713 fn visit_br_if(&mut self, depth: u32) -> Self::Output { 1714 let index = control_index(depth, self.control_frames.len())?; 1715 let frame = &mut self.control_frames[index]; 1716 frame.set_as_target(); 1717 1718 let top = { 1719 let top = self.context.without::<Result<TypedReg>, M, _>( 1720 frame.results::<M>()?.regs(), 1721 self.masm, 1722 |ctx, masm| ctx.pop_to_reg(masm, None), 1723 )??; 1724 // Explicitly save any live registers and locals before setting up 1725 // the branch state. 1726 // In some cases, calculating the `top` value above, will result in 1727 // a spill, thus the following one will result in a no-op. 1728 self.context.spill(self.masm)?; 1729 frame.top_abi_results::<M, _>( 1730 &mut self.context, 1731 self.masm, 1732 |results, context, masm| { 1733 // In the case of `br_if` there's a possibility that we'll 1734 // exit early from the block or fallthrough, for 1735 // a fallthrough, we cannot rely on the pre-computed return area; 1736 // it must be recalculated so that any values that are 1737 // generated are correctly placed near the current stack 1738 // pointer. 1739 if results.on_stack() { 1740 let stack_consumed = context.stack.sizeof(results.stack_operands_len()); 1741 let base = masm.sp_offset()?.as_u32() - stack_consumed; 1742 let offs = base + results.size(); 1743 Ok(Some(RetArea::sp(SPOffset::from_u32(offs)))) 1744 } else { 1745 Ok(None) 1746 } 1747 }, 1748 )?; 1749 top 1750 }; 1751 1752 // Emit instructions to balance the machine stack if the frame has 1753 // a different offset. 1754 let current_sp_offset = self.masm.sp_offset()?; 1755 let results_size = frame.results::<M>()?.size(); 1756 let state = frame.stack_state(); 1757 let (label, cmp, needs_cleanup) = if current_sp_offset > state.target_offset { 1758 (self.masm.get_label()?, IntCmpKind::Eq, true) 1759 } else { 1760 (*frame.label(), IntCmpKind::Ne, false) 1761 }; 1762 1763 self.masm 1764 .branch(cmp, top.reg.into(), top.reg.into(), label, OperandSize::S32)?; 1765 self.context.free_reg(top); 1766 1767 if needs_cleanup { 1768 // Emit instructions to balance the stack and jump if not falling 1769 // through. 1770 self.masm.memmove( 1771 current_sp_offset, 1772 state.target_offset, 1773 results_size, 1774 MemMoveDirection::LowToHigh, 1775 )?; 1776 self.masm.ensure_sp_for_jump(state.target_offset)?; 1777 self.masm.jmp(*frame.label())?; 1778 1779 // Restore sp_offset to what it was for falling through and emit 1780 // fallthrough label. 1781 self.masm.reset_stack_pointer(current_sp_offset)?; 1782 self.masm.bind(label)?; 1783 } 1784 1785 Ok(()) 1786 } 1787 1788 fn visit_br_table(&mut self, targets: BrTable<'a>) -> Self::Output { 1789 // +1 to account for the default target. 1790 let len = targets.len() + 1; 1791 // SmallVec<[_; 5]> to match the binary emission layer (e.g 1792 // see `JmpTableSeq'), but here we use 5 instead since we 1793 // bundle the default target as the last element in the array. 1794 let mut labels: SmallVec<[_; 5]> = smallvec![]; 1795 for _ in 0..len { 1796 labels.push(self.masm.get_label()?); 1797 } 1798 1799 let default_index = control_index(targets.default(), self.control_frames.len())?; 1800 let default_frame = &mut self.control_frames[default_index]; 1801 let default_result = default_frame.results::<M>()?; 1802 1803 let (index, tmp) = { 1804 let index_and_tmp = self.context.without::<Result<(TypedReg, _)>, M, _>( 1805 default_result.regs(), 1806 self.masm, 1807 |cx, masm| Ok((cx.pop_to_reg(masm, None)?, cx.any_gpr(masm)?)), 1808 )??; 1809 1810 // Materialize any constants or locals into their result representation, 1811 // so that when reachability is restored, they are correctly located. 1812 default_frame.top_abi_results::<M, _>( 1813 &mut self.context, 1814 self.masm, 1815 |results, _, _| Ok(results.ret_area().copied()), 1816 )?; 1817 index_and_tmp 1818 }; 1819 1820 self.masm.jmp_table(&labels, index.into(), tmp)?; 1821 // Save the original stack pointer offset; we will reset the stack 1822 // pointer to this offset after jumping to each of the targets. Each 1823 // jump might adjust the stack according to the base offset of the 1824 // target. 1825 let current_sp = self.masm.sp_offset()?; 1826 1827 for (t, l) in targets 1828 .targets() 1829 .into_iter() 1830 .chain(std::iter::once(Ok(targets.default()))) 1831 .zip(labels.iter()) 1832 { 1833 let control_index = control_index(t?, self.control_frames.len())?; 1834 let frame = &mut self.control_frames[control_index]; 1835 // Reset the stack pointer to its original offset. This is needed 1836 // because each jump will potentially adjust the stack pointer 1837 // according to the base offset of the target. 1838 self.masm.reset_stack_pointer(current_sp)?; 1839 1840 // NB: We don't perform any result handling as it was 1841 // already taken care of above before jumping to the 1842 // jump table. 1843 self.masm.bind(*l)?; 1844 // Ensure that the stack pointer is correctly positioned before 1845 // jumping to the jump table code. 1846 let state = frame.stack_state(); 1847 self.masm.ensure_sp_for_jump(state.target_offset)?; 1848 self.masm.jmp(*frame.label())?; 1849 frame.set_as_target(); 1850 } 1851 // Finally reset the stack pointer to the original location. 1852 // The reachability analysis, will ensure it's correctly located 1853 // once reachability is restored. 1854 self.masm.reset_stack_pointer(current_sp)?; 1855 self.context.reachable = false; 1856 self.context.free_reg(index.reg); 1857 self.context.free_reg(tmp); 1858 1859 Ok(()) 1860 } 1861 1862 fn visit_return(&mut self) -> Self::Output { 1863 // Grab the outermost frame, which is the function's body 1864 // frame. We don't rely on [`codegen::control_index`] since 1865 // this frame is implicit and we know that it should exist at 1866 // index 0. 1867 let outermost = &mut self.control_frames[0]; 1868 self.context 1869 .unconditional_jump(outermost, self.masm, |masm, cx, frame| { 1870 frame.pop_abi_results::<M, _>(cx, masm, |results, _, _| { 1871 Ok(results.ret_area().copied()) 1872 }) 1873 }) 1874 } 1875 1876 fn visit_unreachable(&mut self) -> Self::Output { 1877 self.masm.unreachable()?; 1878 self.context.reachable = false; 1879 // Set the implicit outermost frame as target to perform the necessary 1880 // stack clean up. 1881 let outermost = &mut self.control_frames[0]; 1882 outermost.set_as_target(); 1883 1884 Ok(()) 1885 } 1886 1887 fn visit_local_tee(&mut self, index: u32) -> Self::Output { 1888 let typed_reg = self.emit_set_local(index)?; 1889 self.context.stack.push(typed_reg.into()); 1890 1891 Ok(()) 1892 } 1893 1894 fn visit_global_get(&mut self, global_index: u32) -> Self::Output { 1895 let index = GlobalIndex::from_u32(global_index); 1896 let (ty, addr) = self.emit_get_global_addr(index)?; 1897 let dst = self.context.reg_for_type(ty, self.masm)?; 1898 self.masm.load(addr, writable!(dst), ty.try_into()?)?; 1899 self.context.stack.push(Val::reg(dst, ty)); 1900 1901 Ok(()) 1902 } 1903 1904 fn visit_global_set(&mut self, global_index: u32) -> Self::Output { 1905 let index = GlobalIndex::from_u32(global_index); 1906 let (ty, addr) = self.emit_get_global_addr(index)?; 1907 1908 let typed_reg = self.context.pop_to_reg(self.masm, None)?; 1909 self.context.free_reg(typed_reg.reg); 1910 self.masm 1911 .store(typed_reg.reg.into(), addr, ty.try_into()?)?; 1912 1913 Ok(()) 1914 } 1915 1916 fn visit_drop(&mut self) -> Self::Output { 1917 self.context.drop_last(1, |regalloc, val| match val { 1918 Val::Reg(tr) => Ok(regalloc.free(tr.reg.into())), 1919 Val::Memory(m) => self.masm.free_stack(m.slot.size), 1920 _ => Ok(()), 1921 }) 1922 } 1923 1924 fn visit_select(&mut self) -> Self::Output { 1925 let cond = self.context.pop_to_reg(self.masm, None)?; 1926 let val2 = self.context.pop_to_reg(self.masm, None)?; 1927 let val1 = self.context.pop_to_reg(self.masm, None)?; 1928 self.masm 1929 .cmp(cond.reg.into(), RegImm::i32(0), OperandSize::S32)?; 1930 // Conditionally move val1 to val2 if the comparison is 1931 // not zero. 1932 self.masm.cmov( 1933 writable!(val2.into()), 1934 val1.into(), 1935 IntCmpKind::Ne, 1936 val1.ty.try_into()?, 1937 )?; 1938 self.context.stack.push(val2.into()); 1939 self.context.free_reg(val1.reg); 1940 self.context.free_reg(cond); 1941 1942 Ok(()) 1943 } 1944 1945 fn visit_i32_load(&mut self, memarg: MemArg) -> Self::Output { 1946 self.emit_wasm_load( 1947 &memarg, 1948 WasmValType::I32, 1949 LoadKind::Operand(OperandSize::S32), 1950 MemOpKind::Normal, 1951 ) 1952 } 1953 1954 fn visit_i32_load8_s(&mut self, memarg: MemArg) -> Self::Output { 1955 self.emit_wasm_load( 1956 &memarg, 1957 WasmValType::I32, 1958 LoadKind::ScalarExtend(ExtendKind::I32Extend8S), 1959 MemOpKind::Normal, 1960 ) 1961 } 1962 1963 fn visit_i32_load8_u(&mut self, memarg: MemArg) -> Self::Output { 1964 self.emit_wasm_load( 1965 &memarg, 1966 WasmValType::I32, 1967 LoadKind::ScalarExtend(ExtendKind::I32Extend8U), 1968 MemOpKind::Normal, 1969 ) 1970 } 1971 1972 fn visit_i32_load16_s(&mut self, memarg: MemArg) -> Self::Output { 1973 self.emit_wasm_load( 1974 &memarg, 1975 WasmValType::I32, 1976 LoadKind::ScalarExtend(ExtendKind::I32Extend16S), 1977 MemOpKind::Normal, 1978 ) 1979 } 1980 1981 fn visit_i32_load16_u(&mut self, memarg: MemArg) -> Self::Output { 1982 self.emit_wasm_load( 1983 &memarg, 1984 WasmValType::I32, 1985 LoadKind::ScalarExtend(ExtendKind::I32Extend16U), 1986 MemOpKind::Normal, 1987 ) 1988 } 1989 1990 fn visit_i32_store(&mut self, memarg: MemArg) -> Self::Output { 1991 self.emit_wasm_store(&memarg, OperandSize::S32, MemOpKind::Normal) 1992 } 1993 1994 fn visit_i32_store8(&mut self, memarg: MemArg) -> Self::Output { 1995 self.emit_wasm_store(&memarg, OperandSize::S8, MemOpKind::Normal) 1996 } 1997 1998 fn visit_i32_store16(&mut self, memarg: MemArg) -> Self::Output { 1999 self.emit_wasm_store(&memarg, OperandSize::S16, MemOpKind::Normal) 2000 } 2001 2002 fn visit_i64_load8_s(&mut self, memarg: MemArg) -> Self::Output { 2003 self.emit_wasm_load( 2004 &memarg, 2005 WasmValType::I64, 2006 LoadKind::ScalarExtend(ExtendKind::I64Extend8S), 2007 MemOpKind::Normal, 2008 ) 2009 } 2010 2011 fn visit_i64_load8_u(&mut self, memarg: MemArg) -> Self::Output { 2012 self.emit_wasm_load( 2013 &memarg, 2014 WasmValType::I64, 2015 LoadKind::ScalarExtend(ExtendKind::I64Extend8U), 2016 MemOpKind::Normal, 2017 ) 2018 } 2019 2020 fn visit_i64_load16_u(&mut self, memarg: MemArg) -> Self::Output { 2021 self.emit_wasm_load( 2022 &memarg, 2023 WasmValType::I64, 2024 LoadKind::ScalarExtend(ExtendKind::I64Extend16U), 2025 MemOpKind::Normal, 2026 ) 2027 } 2028 2029 fn visit_i64_load16_s(&mut self, memarg: MemArg) -> Self::Output { 2030 self.emit_wasm_load( 2031 &memarg, 2032 WasmValType::I64, 2033 LoadKind::ScalarExtend(ExtendKind::I64Extend16S), 2034 MemOpKind::Normal, 2035 ) 2036 } 2037 2038 fn visit_i64_load32_u(&mut self, memarg: MemArg) -> Self::Output { 2039 self.emit_wasm_load( 2040 &memarg, 2041 WasmValType::I64, 2042 LoadKind::ScalarExtend(ExtendKind::I64Extend32U), 2043 MemOpKind::Normal, 2044 ) 2045 } 2046 2047 fn visit_i64_load32_s(&mut self, memarg: MemArg) -> Self::Output { 2048 self.emit_wasm_load( 2049 &memarg, 2050 WasmValType::I64, 2051 LoadKind::ScalarExtend(ExtendKind::I64Extend32S), 2052 MemOpKind::Normal, 2053 ) 2054 } 2055 2056 fn visit_i64_load(&mut self, memarg: MemArg) -> Self::Output { 2057 self.emit_wasm_load( 2058 &memarg, 2059 WasmValType::I64, 2060 LoadKind::Operand(OperandSize::S64), 2061 MemOpKind::Normal, 2062 ) 2063 } 2064 2065 fn visit_i64_store(&mut self, memarg: MemArg) -> Self::Output { 2066 self.emit_wasm_store(&memarg, OperandSize::S64, MemOpKind::Normal) 2067 } 2068 2069 fn visit_i64_store8(&mut self, memarg: MemArg) -> Self::Output { 2070 self.emit_wasm_store(&memarg, OperandSize::S8, MemOpKind::Normal) 2071 } 2072 2073 fn visit_i64_store16(&mut self, memarg: MemArg) -> Self::Output { 2074 self.emit_wasm_store(&memarg, OperandSize::S16, MemOpKind::Normal) 2075 } 2076 2077 fn visit_i64_store32(&mut self, memarg: MemArg) -> Self::Output { 2078 self.emit_wasm_store(&memarg, OperandSize::S32, MemOpKind::Normal) 2079 } 2080 2081 fn visit_f32_load(&mut self, memarg: MemArg) -> Self::Output { 2082 self.emit_wasm_load( 2083 &memarg, 2084 WasmValType::F32, 2085 LoadKind::Operand(OperandSize::S32), 2086 MemOpKind::Normal, 2087 ) 2088 } 2089 2090 fn visit_f32_store(&mut self, memarg: MemArg) -> Self::Output { 2091 self.emit_wasm_store(&memarg, OperandSize::S32, MemOpKind::Normal) 2092 } 2093 2094 fn visit_f64_load(&mut self, memarg: MemArg) -> Self::Output { 2095 self.emit_wasm_load( 2096 &memarg, 2097 WasmValType::F64, 2098 LoadKind::Operand(OperandSize::S64), 2099 MemOpKind::Normal, 2100 ) 2101 } 2102 2103 fn visit_f64_store(&mut self, memarg: MemArg) -> Self::Output { 2104 self.emit_wasm_store(&memarg, OperandSize::S64, MemOpKind::Normal) 2105 } 2106 2107 fn visit_i32_trunc_sat_f32_s(&mut self) -> Self::Output { 2108 use OperandSize::*; 2109 2110 self.context 2111 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2112 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Checked) 2113 }) 2114 } 2115 2116 fn visit_i32_trunc_sat_f32_u(&mut self) -> Self::Output { 2117 use OperandSize::*; 2118 2119 self.masm 2120 .unsigned_truncate(&mut self.context, S32, S32, TruncKind::Checked) 2121 } 2122 2123 fn visit_i32_trunc_sat_f64_s(&mut self) -> Self::Output { 2124 use OperandSize::*; 2125 2126 self.context 2127 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2128 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Checked) 2129 }) 2130 } 2131 2132 fn visit_i32_trunc_sat_f64_u(&mut self) -> Self::Output { 2133 use OperandSize::*; 2134 2135 self.masm 2136 .unsigned_truncate(&mut self.context, S64, S32, TruncKind::Checked) 2137 } 2138 2139 fn visit_i64_trunc_sat_f32_s(&mut self) -> Self::Output { 2140 use OperandSize::*; 2141 2142 self.context 2143 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2144 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Checked) 2145 }) 2146 } 2147 2148 fn visit_i64_trunc_sat_f32_u(&mut self) -> Self::Output { 2149 use OperandSize::*; 2150 2151 self.masm 2152 .unsigned_truncate(&mut self.context, S32, S64, TruncKind::Checked) 2153 } 2154 2155 fn visit_i64_trunc_sat_f64_s(&mut self) -> Self::Output { 2156 use OperandSize::*; 2157 2158 self.context 2159 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2160 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Checked) 2161 }) 2162 } 2163 2164 fn visit_i64_trunc_sat_f64_u(&mut self) -> Self::Output { 2165 use OperandSize::*; 2166 2167 self.masm 2168 .unsigned_truncate(&mut self.context, S64, S64, TruncKind::Checked) 2169 } 2170 2171 fn visit_i64_add128(&mut self) -> Self::Output { 2172 self.context 2173 .binop128(self.masm, |masm, lhs_lo, lhs_hi, rhs_lo, rhs_hi| { 2174 masm.add128( 2175 writable!(lhs_lo), 2176 writable!(lhs_hi), 2177 lhs_lo, 2178 lhs_hi, 2179 rhs_lo, 2180 rhs_hi, 2181 )?; 2182 Ok((TypedReg::i64(lhs_lo), TypedReg::i64(lhs_hi))) 2183 }) 2184 } 2185 2186 fn visit_i64_sub128(&mut self) -> Self::Output { 2187 self.context 2188 .binop128(self.masm, |masm, lhs_lo, lhs_hi, rhs_lo, rhs_hi| { 2189 masm.sub128( 2190 writable!(lhs_lo), 2191 writable!(lhs_hi), 2192 lhs_lo, 2193 lhs_hi, 2194 rhs_lo, 2195 rhs_hi, 2196 )?; 2197 Ok((TypedReg::i64(lhs_lo), TypedReg::i64(lhs_hi))) 2198 }) 2199 } 2200 2201 fn visit_i64_mul_wide_s(&mut self) -> Self::Output { 2202 self.masm.mul_wide(&mut self.context, MulWideKind::Signed) 2203 } 2204 2205 fn visit_i64_mul_wide_u(&mut self) -> Self::Output { 2206 self.masm.mul_wide(&mut self.context, MulWideKind::Unsigned) 2207 } 2208 2209 fn visit_i32_atomic_load8_u(&mut self, memarg: MemArg) -> Self::Output { 2210 self.emit_wasm_load( 2211 &memarg, 2212 WasmValType::I32, 2213 LoadKind::ScalarExtend(ExtendKind::I32Extend8U), 2214 MemOpKind::Atomic, 2215 ) 2216 } 2217 2218 fn visit_i32_atomic_load16_u(&mut self, memarg: MemArg) -> Self::Output { 2219 self.emit_wasm_load( 2220 &memarg, 2221 WasmValType::I32, 2222 LoadKind::ScalarExtend(ExtendKind::I32Extend16U), 2223 MemOpKind::Atomic, 2224 ) 2225 } 2226 2227 fn visit_i32_atomic_load(&mut self, memarg: MemArg) -> Self::Output { 2228 self.emit_wasm_load( 2229 &memarg, 2230 WasmValType::I32, 2231 LoadKind::Operand(OperandSize::S32), 2232 MemOpKind::Atomic, 2233 ) 2234 } 2235 2236 fn visit_i64_atomic_load8_u(&mut self, memarg: MemArg) -> Self::Output { 2237 self.emit_wasm_load( 2238 &memarg, 2239 WasmValType::I64, 2240 LoadKind::ScalarExtend(ExtendKind::I64Extend8U), 2241 MemOpKind::Atomic, 2242 ) 2243 } 2244 2245 fn visit_i64_atomic_load16_u(&mut self, memarg: MemArg) -> Self::Output { 2246 self.emit_wasm_load( 2247 &memarg, 2248 WasmValType::I64, 2249 LoadKind::ScalarExtend(ExtendKind::I64Extend16U), 2250 MemOpKind::Atomic, 2251 ) 2252 } 2253 2254 fn visit_i64_atomic_load32_u(&mut self, memarg: MemArg) -> Self::Output { 2255 self.emit_wasm_load( 2256 &memarg, 2257 WasmValType::I64, 2258 LoadKind::ScalarExtend(ExtendKind::I64Extend32U), 2259 MemOpKind::Atomic, 2260 ) 2261 } 2262 2263 fn visit_i64_atomic_load(&mut self, memarg: MemArg) -> Self::Output { 2264 self.emit_wasm_load( 2265 &memarg, 2266 WasmValType::I64, 2267 LoadKind::Operand(OperandSize::S64), 2268 MemOpKind::Atomic, 2269 ) 2270 } 2271 2272 fn visit_i32_atomic_store(&mut self, memarg: MemArg) -> Self::Output { 2273 self.emit_wasm_store(&memarg, OperandSize::S32, MemOpKind::Atomic) 2274 } 2275 2276 fn visit_i64_atomic_store(&mut self, memarg: MemArg) -> Self::Output { 2277 self.emit_wasm_store(&memarg, OperandSize::S64, MemOpKind::Atomic) 2278 } 2279 2280 fn visit_i32_atomic_store8(&mut self, memarg: MemArg) -> Self::Output { 2281 self.emit_wasm_store(&memarg, OperandSize::S8, MemOpKind::Atomic) 2282 } 2283 2284 fn visit_i32_atomic_store16(&mut self, memarg: MemArg) -> Self::Output { 2285 self.emit_wasm_store(&memarg, OperandSize::S16, MemOpKind::Atomic) 2286 } 2287 2288 fn visit_i64_atomic_store8(&mut self, memarg: MemArg) -> Self::Output { 2289 self.emit_wasm_store(&memarg, OperandSize::S8, MemOpKind::Atomic) 2290 } 2291 2292 fn visit_i64_atomic_store16(&mut self, memarg: MemArg) -> Self::Output { 2293 self.emit_wasm_store(&memarg, OperandSize::S16, MemOpKind::Atomic) 2294 } 2295 2296 fn visit_i64_atomic_store32(&mut self, memarg: MemArg) -> Self::Output { 2297 self.emit_wasm_store(&memarg, OperandSize::S32, MemOpKind::Atomic) 2298 } 2299 2300 fn visit_i32_atomic_rmw8_add_u(&mut self, arg: MemArg) -> Self::Output { 2301 self.emit_atomic_rmw( 2302 &arg, 2303 RmwOp::Add, 2304 OperandSize::S8, 2305 Some(ExtendKind::I32Extend8U), 2306 ) 2307 } 2308 2309 fn visit_i32_atomic_rmw16_add_u(&mut self, arg: MemArg) -> Self::Output { 2310 self.emit_atomic_rmw( 2311 &arg, 2312 RmwOp::Add, 2313 OperandSize::S16, 2314 Some(ExtendKind::I32Extend16U), 2315 ) 2316 } 2317 2318 fn visit_i32_atomic_rmw_add(&mut self, arg: MemArg) -> Self::Output { 2319 self.emit_atomic_rmw(&arg, RmwOp::Add, OperandSize::S32, None) 2320 } 2321 2322 fn visit_i64_atomic_rmw8_add_u(&mut self, arg: MemArg) -> Self::Output { 2323 self.emit_atomic_rmw( 2324 &arg, 2325 RmwOp::Add, 2326 OperandSize::S8, 2327 Some(ExtendKind::I64Extend8U), 2328 ) 2329 } 2330 2331 fn visit_i64_atomic_rmw16_add_u(&mut self, arg: MemArg) -> Self::Output { 2332 self.emit_atomic_rmw( 2333 &arg, 2334 RmwOp::Add, 2335 OperandSize::S16, 2336 Some(ExtendKind::I64Extend16U), 2337 ) 2338 } 2339 2340 fn visit_i64_atomic_rmw32_add_u(&mut self, arg: MemArg) -> Self::Output { 2341 self.emit_atomic_rmw( 2342 &arg, 2343 RmwOp::Add, 2344 OperandSize::S32, 2345 Some(ExtendKind::I64Extend32U), 2346 ) 2347 } 2348 2349 fn visit_i64_atomic_rmw_add(&mut self, arg: MemArg) -> Self::Output { 2350 self.emit_atomic_rmw(&arg, RmwOp::Add, OperandSize::S64, None) 2351 } 2352 2353 fn visit_i32_atomic_rmw_sub(&mut self, arg: MemArg) -> Self::Output { 2354 self.emit_atomic_rmw(&arg, RmwOp::Sub, OperandSize::S32, None) 2355 } 2356 2357 fn visit_i64_atomic_rmw_sub(&mut self, arg: MemArg) -> Self::Output { 2358 self.emit_atomic_rmw(&arg, RmwOp::Sub, OperandSize::S64, None) 2359 } 2360 2361 fn visit_i32_atomic_rmw8_sub_u(&mut self, arg: MemArg) -> Self::Output { 2362 self.emit_atomic_rmw( 2363 &arg, 2364 RmwOp::Sub, 2365 OperandSize::S8, 2366 Some(ExtendKind::I32Extend8U), 2367 ) 2368 } 2369 2370 fn visit_i32_atomic_rmw16_sub_u(&mut self, arg: MemArg) -> Self::Output { 2371 self.emit_atomic_rmw( 2372 &arg, 2373 RmwOp::Sub, 2374 OperandSize::S16, 2375 Some(ExtendKind::I32Extend16U), 2376 ) 2377 } 2378 2379 fn visit_i64_atomic_rmw8_sub_u(&mut self, arg: MemArg) -> Self::Output { 2380 self.emit_atomic_rmw( 2381 &arg, 2382 RmwOp::Sub, 2383 OperandSize::S8, 2384 Some(ExtendKind::I64Extend8U), 2385 ) 2386 } 2387 2388 fn visit_i64_atomic_rmw16_sub_u(&mut self, arg: MemArg) -> Self::Output { 2389 self.emit_atomic_rmw( 2390 &arg, 2391 RmwOp::Sub, 2392 OperandSize::S16, 2393 Some(ExtendKind::I64Extend16U), 2394 ) 2395 } 2396 2397 fn visit_i64_atomic_rmw32_sub_u(&mut self, arg: MemArg) -> Self::Output { 2398 self.emit_atomic_rmw( 2399 &arg, 2400 RmwOp::Sub, 2401 OperandSize::S32, 2402 Some(ExtendKind::I64Extend32U), 2403 ) 2404 } 2405 2406 wasmparser::for_each_visit_operator!(def_unsupported); 2407 } 2408 2409 impl<'a, 'translation, 'data, M> VisitSimdOperator<'a> 2410 for CodeGen<'a, 'translation, 'data, M, Emission> 2411 where 2412 M: MacroAssembler, 2413 { 2414 fn visit_v128_const(&mut self, val: V128) -> Self::Output { 2415 self.context.stack.push(Val::v128(val.i128())); 2416 Ok(()) 2417 } 2418 2419 fn visit_v128_load(&mut self, memarg: MemArg) -> Self::Output { 2420 self.emit_wasm_load( 2421 &memarg, 2422 WasmValType::V128, 2423 LoadKind::Operand(OperandSize::S128), 2424 MemOpKind::Normal, 2425 ) 2426 } 2427 2428 fn visit_v128_store(&mut self, memarg: MemArg) -> Self::Output { 2429 self.emit_wasm_store(&memarg, OperandSize::S128, MemOpKind::Normal) 2430 } 2431 2432 fn visit_v128_load8x8_s(&mut self, memarg: MemArg) -> Self::Output { 2433 self.emit_wasm_load( 2434 &memarg, 2435 WasmValType::V128, 2436 LoadKind::VectorExtend(VectorExtendKind::V128Extend8x8S), 2437 MemOpKind::Normal, 2438 ) 2439 } 2440 2441 fn visit_v128_load8x8_u(&mut self, memarg: MemArg) -> Self::Output { 2442 self.emit_wasm_load( 2443 &memarg, 2444 WasmValType::V128, 2445 LoadKind::VectorExtend(VectorExtendKind::V128Extend8x8U), 2446 MemOpKind::Normal, 2447 ) 2448 } 2449 2450 fn visit_v128_load16x4_s(&mut self, memarg: MemArg) -> Self::Output { 2451 self.emit_wasm_load( 2452 &memarg, 2453 WasmValType::V128, 2454 LoadKind::VectorExtend(VectorExtendKind::V128Extend16x4S), 2455 MemOpKind::Normal, 2456 ) 2457 } 2458 2459 fn visit_v128_load16x4_u(&mut self, memarg: MemArg) -> Self::Output { 2460 self.emit_wasm_load( 2461 &memarg, 2462 WasmValType::V128, 2463 LoadKind::VectorExtend(VectorExtendKind::V128Extend16x4U), 2464 MemOpKind::Normal, 2465 ) 2466 } 2467 2468 fn visit_v128_load32x2_s(&mut self, memarg: MemArg) -> Self::Output { 2469 self.emit_wasm_load( 2470 &memarg, 2471 WasmValType::V128, 2472 LoadKind::VectorExtend(VectorExtendKind::V128Extend32x2S), 2473 MemOpKind::Normal, 2474 ) 2475 } 2476 2477 fn visit_v128_load32x2_u(&mut self, memarg: MemArg) -> Self::Output { 2478 self.emit_wasm_load( 2479 &memarg, 2480 WasmValType::V128, 2481 LoadKind::VectorExtend(VectorExtendKind::V128Extend32x2U), 2482 MemOpKind::Normal, 2483 ) 2484 } 2485 2486 fn visit_v128_load8_splat(&mut self, memarg: MemArg) -> Self::Output { 2487 self.emit_wasm_load( 2488 &memarg, 2489 WasmValType::V128, 2490 LoadKind::Splat(SplatLoadKind::S8), 2491 MemOpKind::Normal, 2492 ) 2493 } 2494 2495 fn visit_v128_load16_splat(&mut self, memarg: MemArg) -> Self::Output { 2496 self.emit_wasm_load( 2497 &memarg, 2498 WasmValType::V128, 2499 LoadKind::Splat(SplatLoadKind::S16), 2500 MemOpKind::Normal, 2501 ) 2502 } 2503 2504 fn visit_v128_load32_splat(&mut self, memarg: MemArg) -> Self::Output { 2505 self.emit_wasm_load( 2506 &memarg, 2507 WasmValType::V128, 2508 LoadKind::Splat(SplatLoadKind::S32), 2509 MemOpKind::Normal, 2510 ) 2511 } 2512 2513 fn visit_v128_load64_splat(&mut self, memarg: MemArg) -> Self::Output { 2514 self.emit_wasm_load( 2515 &memarg, 2516 WasmValType::V128, 2517 LoadKind::Splat(SplatLoadKind::S64), 2518 MemOpKind::Normal, 2519 ) 2520 } 2521 2522 fn visit_i8x16_splat(&mut self) -> Self::Output { 2523 self.masm.splat(&mut self.context, SplatKind::I8x16) 2524 } 2525 2526 fn visit_i16x8_splat(&mut self) -> Self::Output { 2527 self.masm.splat(&mut self.context, SplatKind::I16x8) 2528 } 2529 2530 fn visit_i32x4_splat(&mut self) -> Self::Output { 2531 self.masm.splat(&mut self.context, SplatKind::I32x4) 2532 } 2533 2534 fn visit_i64x2_splat(&mut self) -> Self::Output { 2535 self.masm.splat(&mut self.context, SplatKind::I64x2) 2536 } 2537 2538 fn visit_f32x4_splat(&mut self) -> Self::Output { 2539 self.masm.splat(&mut self.context, SplatKind::F32x4) 2540 } 2541 2542 fn visit_f64x2_splat(&mut self) -> Self::Output { 2543 self.masm.splat(&mut self.context, SplatKind::F64x2) 2544 } 2545 2546 fn visit_i8x16_shuffle(&mut self, lanes: [u8; 16]) -> Self::Output { 2547 let rhs = self.context.pop_to_reg(self.masm, None)?; 2548 let lhs = self.context.pop_to_reg(self.masm, None)?; 2549 self.masm 2550 .shuffle(writable!(lhs.into()), lhs.into(), rhs.into(), lanes)?; 2551 self.context.stack.push(TypedReg::v128(lhs.into()).into()); 2552 self.context.free_reg(rhs); 2553 Ok(()) 2554 } 2555 2556 wasmparser::for_each_visit_simd_operator!(def_unsupported); 2557 } 2558 2559 impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Emission> 2560 where 2561 M: MacroAssembler, 2562 { 2563 fn cmp_i32s(&mut self, kind: IntCmpKind) -> Result<()> { 2564 self.context.i32_binop(self.masm, |masm, dst, src, size| { 2565 masm.cmp_with_set(writable!(dst), src, kind, size)?; 2566 Ok(TypedReg::i32(dst)) 2567 }) 2568 } 2569 2570 fn cmp_i64s(&mut self, kind: IntCmpKind) -> Result<()> { 2571 self.context 2572 .i64_binop(self.masm, move |masm, dst, src, size| { 2573 masm.cmp_with_set(writable!(dst), src, kind, size)?; 2574 Ok(TypedReg::i32(dst)) // Return value for comparisons is an `i32`. 2575 }) 2576 } 2577 } 2578 2579 impl TryFrom<WasmValType> for OperandSize { 2580 type Error = anyhow::Error; 2581 fn try_from(ty: WasmValType) -> Result<OperandSize> { 2582 let ty = match ty { 2583 WasmValType::I32 | WasmValType::F32 => OperandSize::S32, 2584 WasmValType::I64 | WasmValType::F64 => OperandSize::S64, 2585 WasmValType::V128 => OperandSize::S128, 2586 WasmValType::Ref(rt) => { 2587 match rt.heap_type { 2588 // TODO: Hardcoded size, assuming 64-bit support only. Once 2589 // Wasmtime supports 32-bit architectures, this will need 2590 // to be updated in such a way that the calculation of the 2591 // OperandSize will depend on the target's pointer size. 2592 WasmHeapType::Func => OperandSize::S64, 2593 WasmHeapType::Extern => OperandSize::S64, 2594 _ => bail!(CodeGenError::unsupported_wasm_type()), 2595 } 2596 } 2597 }; 2598 Ok(ty) 2599 } 2600 } 2601