1 //! This module is the central place for machine code emission. 2 //! It defines an implementation of wasmparser's Visitor trait 3 //! for `CodeGen`; which defines a visitor per op-code, 4 //! which validates and dispatches to the corresponding 5 //! machine code emitter. 6 7 use crate::abi::RetArea; 8 use crate::codegen::{ 9 control_index, Callee, CodeGen, CodeGenError, ControlStackFrame, Emission, FnCall, 10 }; 11 use crate::masm::{ 12 DivKind, Extend, ExtractLaneKind, FloatCmpKind, IntCmpKind, LoadKind, MacroAssembler, 13 MemMoveDirection, MemOpKind, MulWideKind, OperandSize, RegImm, RemKind, RmwOp, RoundingMode, 14 SPOffset, ShiftKind, Signed, SplatKind, SplatLoadKind, TruncKind, VectorExtendKind, Zero, 15 }; 16 17 use crate::reg::{writable, Reg}; 18 use crate::stack::{TypedReg, Val}; 19 use anyhow::{anyhow, bail, ensure, Result}; 20 use regalloc2::RegClass; 21 use smallvec::{smallvec, SmallVec}; 22 use wasmparser::{ 23 BlockType, BrTable, Ieee32, Ieee64, MemArg, VisitOperator, VisitSimdOperator, V128, 24 }; 25 use wasmtime_cranelift::TRAP_INDIRECT_CALL_TO_NULL; 26 use wasmtime_environ::{ 27 FuncIndex, GlobalIndex, MemoryIndex, TableIndex, TypeIndex, WasmHeapType, WasmValType, 28 FUNCREF_INIT_BIT, 29 }; 30 31 /// A macro to define unsupported WebAssembly operators. 32 /// 33 /// This macro calls itself recursively; 34 /// 1. It no-ops when matching a supported operator. 35 /// 2. Defines the visitor function and panics when 36 /// matching an unsupported operator. 37 macro_rules! def_unsupported { 38 ($( @$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident $ann:tt)*) => { 39 $( 40 def_unsupported!( 41 emit 42 $op 43 44 fn $visit(&mut self $($(,$arg: $argty)*)?) -> Self::Output { 45 $($(let _ = $arg;)*)? 46 47 Err(anyhow!(CodeGenError::unimplemented_wasm_instruction())) 48 } 49 ); 50 )* 51 }; 52 53 (emit I32Const $($rest:tt)*) => {}; 54 (emit I64Const $($rest:tt)*) => {}; 55 (emit F32Const $($rest:tt)*) => {}; 56 (emit F64Const $($rest:tt)*) => {}; 57 (emit V128Const $($rest:tt)*) => {}; 58 (emit F32Add $($rest:tt)*) => {}; 59 (emit F64Add $($rest:tt)*) => {}; 60 (emit F32Sub $($rest:tt)*) => {}; 61 (emit F64Sub $($rest:tt)*) => {}; 62 (emit F32Mul $($rest:tt)*) => {}; 63 (emit F64Mul $($rest:tt)*) => {}; 64 (emit F32Div $($rest:tt)*) => {}; 65 (emit F64Div $($rest:tt)*) => {}; 66 (emit F32Min $($rest:tt)*) => {}; 67 (emit F64Min $($rest:tt)*) => {}; 68 (emit F32Max $($rest:tt)*) => {}; 69 (emit F64Max $($rest:tt)*) => {}; 70 (emit F32Copysign $($rest:tt)*) => {}; 71 (emit F64Copysign $($rest:tt)*) => {}; 72 (emit F32Abs $($rest:tt)*) => {}; 73 (emit F64Abs $($rest:tt)*) => {}; 74 (emit F32Neg $($rest:tt)*) => {}; 75 (emit F64Neg $($rest:tt)*) => {}; 76 (emit F32Floor $($rest:tt)*) => {}; 77 (emit F64Floor $($rest:tt)*) => {}; 78 (emit F32Ceil $($rest:tt)*) => {}; 79 (emit F64Ceil $($rest:tt)*) => {}; 80 (emit F32Nearest $($rest:tt)*) => {}; 81 (emit F64Nearest $($rest:tt)*) => {}; 82 (emit F32Trunc $($rest:tt)*) => {}; 83 (emit F64Trunc $($rest:tt)*) => {}; 84 (emit F32Sqrt $($rest:tt)*) => {}; 85 (emit F64Sqrt $($rest:tt)*) => {}; 86 (emit F32Eq $($rest:tt)*) => {}; 87 (emit F64Eq $($rest:tt)*) => {}; 88 (emit F32Ne $($rest:tt)*) => {}; 89 (emit F64Ne $($rest:tt)*) => {}; 90 (emit F32Lt $($rest:tt)*) => {}; 91 (emit F64Lt $($rest:tt)*) => {}; 92 (emit F32Gt $($rest:tt)*) => {}; 93 (emit F64Gt $($rest:tt)*) => {}; 94 (emit F32Le $($rest:tt)*) => {}; 95 (emit F64Le $($rest:tt)*) => {}; 96 (emit F32Ge $($rest:tt)*) => {}; 97 (emit F64Ge $($rest:tt)*) => {}; 98 (emit F32ConvertI32S $($rest:tt)*) => {}; 99 (emit F32ConvertI32U $($rest:tt)*) => {}; 100 (emit F32ConvertI64S $($rest:tt)*) => {}; 101 (emit F32ConvertI64U $($rest:tt)*) => {}; 102 (emit F64ConvertI32S $($rest:tt)*) => {}; 103 (emit F64ConvertI32U $($rest:tt)*) => {}; 104 (emit F64ConvertI64S $($rest:tt)*) => {}; 105 (emit F64ConvertI64U $($rest:tt)*) => {}; 106 (emit F32ReinterpretI32 $($rest:tt)*) => {}; 107 (emit F64ReinterpretI64 $($rest:tt)*) => {}; 108 (emit F32DemoteF64 $($rest:tt)*) => {}; 109 (emit F64PromoteF32 $($rest:tt)*) => {}; 110 (emit I32Add $($rest:tt)*) => {}; 111 (emit I64Add $($rest:tt)*) => {}; 112 (emit I32Sub $($rest:tt)*) => {}; 113 (emit I32Mul $($rest:tt)*) => {}; 114 (emit I32DivS $($rest:tt)*) => {}; 115 (emit I32DivU $($rest:tt)*) => {}; 116 (emit I64DivS $($rest:tt)*) => {}; 117 (emit I64DivU $($rest:tt)*) => {}; 118 (emit I64RemU $($rest:tt)*) => {}; 119 (emit I64RemS $($rest:tt)*) => {}; 120 (emit I32RemU $($rest:tt)*) => {}; 121 (emit I32RemS $($rest:tt)*) => {}; 122 (emit I64Mul $($rest:tt)*) => {}; 123 (emit I64Sub $($rest:tt)*) => {}; 124 (emit I32Eq $($rest:tt)*) => {}; 125 (emit I64Eq $($rest:tt)*) => {}; 126 (emit I32Ne $($rest:tt)*) => {}; 127 (emit I64Ne $($rest:tt)*) => {}; 128 (emit I32LtS $($rest:tt)*) => {}; 129 (emit I64LtS $($rest:tt)*) => {}; 130 (emit I32LtU $($rest:tt)*) => {}; 131 (emit I64LtU $($rest:tt)*) => {}; 132 (emit I32LeS $($rest:tt)*) => {}; 133 (emit I64LeS $($rest:tt)*) => {}; 134 (emit I32LeU $($rest:tt)*) => {}; 135 (emit I64LeU $($rest:tt)*) => {}; 136 (emit I32GtS $($rest:tt)*) => {}; 137 (emit I64GtS $($rest:tt)*) => {}; 138 (emit I32GtU $($rest:tt)*) => {}; 139 (emit I64GtU $($rest:tt)*) => {}; 140 (emit I32GeS $($rest:tt)*) => {}; 141 (emit I64GeS $($rest:tt)*) => {}; 142 (emit I32GeU $($rest:tt)*) => {}; 143 (emit I64GeU $($rest:tt)*) => {}; 144 (emit I32Eqz $($rest:tt)*) => {}; 145 (emit I64Eqz $($rest:tt)*) => {}; 146 (emit I32And $($rest:tt)*) => {}; 147 (emit I64And $($rest:tt)*) => {}; 148 (emit I32Or $($rest:tt)*) => {}; 149 (emit I64Or $($rest:tt)*) => {}; 150 (emit I32Xor $($rest:tt)*) => {}; 151 (emit I64Xor $($rest:tt)*) => {}; 152 (emit I32Shl $($rest:tt)*) => {}; 153 (emit I64Shl $($rest:tt)*) => {}; 154 (emit I32ShrS $($rest:tt)*) => {}; 155 (emit I64ShrS $($rest:tt)*) => {}; 156 (emit I32ShrU $($rest:tt)*) => {}; 157 (emit I64ShrU $($rest:tt)*) => {}; 158 (emit I32Rotl $($rest:tt)*) => {}; 159 (emit I64Rotl $($rest:tt)*) => {}; 160 (emit I32Rotr $($rest:tt)*) => {}; 161 (emit I64Rotr $($rest:tt)*) => {}; 162 (emit I32Clz $($rest:tt)*) => {}; 163 (emit I64Clz $($rest:tt)*) => {}; 164 (emit I32Ctz $($rest:tt)*) => {}; 165 (emit I64Ctz $($rest:tt)*) => {}; 166 (emit I32Popcnt $($rest:tt)*) => {}; 167 (emit I64Popcnt $($rest:tt)*) => {}; 168 (emit I32WrapI64 $($rest:tt)*) => {}; 169 (emit I64ExtendI32S $($rest:tt)*) => {}; 170 (emit I64ExtendI32U $($rest:tt)*) => {}; 171 (emit I32Extend8S $($rest:tt)*) => {}; 172 (emit I32Extend16S $($rest:tt)*) => {}; 173 (emit I64Extend8S $($rest:tt)*) => {}; 174 (emit I64Extend16S $($rest:tt)*) => {}; 175 (emit I64Extend32S $($rest:tt)*) => {}; 176 (emit I32TruncF32S $($rest:tt)*) => {}; 177 (emit I32TruncF32U $($rest:tt)*) => {}; 178 (emit I32TruncF64S $($rest:tt)*) => {}; 179 (emit I32TruncF64U $($rest:tt)*) => {}; 180 (emit I64TruncF32S $($rest:tt)*) => {}; 181 (emit I64TruncF32U $($rest:tt)*) => {}; 182 (emit I64TruncF64S $($rest:tt)*) => {}; 183 (emit I64TruncF64U $($rest:tt)*) => {}; 184 (emit I32ReinterpretF32 $($rest:tt)*) => {}; 185 (emit I64ReinterpretF64 $($rest:tt)*) => {}; 186 (emit LocalGet $($rest:tt)*) => {}; 187 (emit LocalSet $($rest:tt)*) => {}; 188 (emit Call $($rest:tt)*) => {}; 189 (emit End $($rest:tt)*) => {}; 190 (emit Nop $($rest:tt)*) => {}; 191 (emit If $($rest:tt)*) => {}; 192 (emit Else $($rest:tt)*) => {}; 193 (emit Block $($rest:tt)*) => {}; 194 (emit Loop $($rest:tt)*) => {}; 195 (emit Br $($rest:tt)*) => {}; 196 (emit BrIf $($rest:tt)*) => {}; 197 (emit Return $($rest:tt)*) => {}; 198 (emit Unreachable $($rest:tt)*) => {}; 199 (emit LocalTee $($rest:tt)*) => {}; 200 (emit GlobalGet $($rest:tt)*) => {}; 201 (emit GlobalSet $($rest:tt)*) => {}; 202 (emit Select $($rest:tt)*) => {}; 203 (emit Drop $($rest:tt)*) => {}; 204 (emit BrTable $($rest:tt)*) => {}; 205 (emit CallIndirect $($rest:tt)*) => {}; 206 (emit TableInit $($rest:tt)*) => {}; 207 (emit TableCopy $($rest:tt)*) => {}; 208 (emit TableGet $($rest:tt)*) => {}; 209 (emit TableSet $($rest:tt)*) => {}; 210 (emit TableGrow $($rest:tt)*) => {}; 211 (emit TableSize $($rest:tt)*) => {}; 212 (emit TableFill $($rest:tt)*) => {}; 213 (emit ElemDrop $($rest:tt)*) => {}; 214 (emit MemoryInit $($rest:tt)*) => {}; 215 (emit MemoryCopy $($rest:tt)*) => {}; 216 (emit DataDrop $($rest:tt)*) => {}; 217 (emit MemoryFill $($rest:tt)*) => {}; 218 (emit MemorySize $($rest:tt)*) => {}; 219 (emit MemoryGrow $($rest:tt)*) => {}; 220 (emit I32Load $($rest:tt)*) => {}; 221 (emit I32Load8S $($rest:tt)*) => {}; 222 (emit I32Load8U $($rest:tt)*) => {}; 223 (emit I32Load16S $($rest:tt)*) => {}; 224 (emit I32Load16U $($rest:tt)*) => {}; 225 (emit I64Load8S $($rest:tt)*) => {}; 226 (emit I64Load8U $($rest:tt)*) => {}; 227 (emit I64Load16S $($rest:tt)*) => {}; 228 (emit I64Load16U $($rest:tt)*) => {}; 229 (emit I64Load32S $($rest:tt)*) => {}; 230 (emit I64Load32U $($rest:tt)*) => {}; 231 (emit I64Load $($rest:tt)*) => {}; 232 (emit I32Store $($rest:tt)*) => {}; 233 (emit I32Store8 $($rest:tt)*) => {}; 234 (emit I32Store16 $($rest:tt)*) => {}; 235 (emit I64Store $($rest:tt)*) => {}; 236 (emit I64Store8 $($rest:tt)*) => {}; 237 (emit I64Store16 $($rest:tt)*) => {}; 238 (emit I64Store32 $($rest:tt)*) => {}; 239 (emit F32Load $($rest:tt)*) => {}; 240 (emit F32Store $($rest:tt)*) => {}; 241 (emit F64Load $($rest:tt)*) => {}; 242 (emit F64Store $($rest:tt)*) => {}; 243 (emit I32TruncSatF32S $($rest:tt)*) => {}; 244 (emit I32TruncSatF32U $($rest:tt)*) => {}; 245 (emit I32TruncSatF64S $($rest:tt)*) => {}; 246 (emit I32TruncSatF64U $($rest:tt)*) => {}; 247 (emit I64TruncSatF32S $($rest:tt)*) => {}; 248 (emit I64TruncSatF32U $($rest:tt)*) => {}; 249 (emit I64TruncSatF64S $($rest:tt)*) => {}; 250 (emit I64TruncSatF64U $($rest:tt)*) => {}; 251 (emit V128Load $($rest:tt)*) => {}; 252 (emit V128Store $($rest:tt)*) => {}; 253 (emit I64Add128 $($rest:tt)*) => {}; 254 (emit I64Sub128 $($rest:tt)*) => {}; 255 (emit I64MulWideS $($rest:tt)*) => {}; 256 (emit I64MulWideU $($rest:tt)*) => {}; 257 (emit I32AtomicLoad8U $($rest:tt)*) => {}; 258 (emit I32AtomicLoad16U $($rest:tt)*) => {}; 259 (emit I32AtomicLoad $($rest:tt)*) => {}; 260 (emit I64AtomicLoad8U $($rest:tt)*) => {}; 261 (emit I64AtomicLoad16U $($rest:tt)*) => {}; 262 (emit I64AtomicLoad32U $($rest:tt)*) => {}; 263 (emit I64AtomicLoad $($rest:tt)*) => {}; 264 (emit V128Load8x8S $($rest:tt)*) => {}; 265 (emit V128Load8x8U $($rest:tt)*) => {}; 266 (emit V128Load16x4S $($rest:tt)*) => {}; 267 (emit V128Load16x4U $($rest:tt)*) => {}; 268 (emit V128Load32x2S $($rest:tt)*) => {}; 269 (emit V128Load32x2U $($rest:tt)*) => {}; 270 (emit V128Load8Splat $($rest:tt)*) => {}; 271 (emit V128Load16Splat $($rest:tt)*) => {}; 272 (emit V128Load32Splat $($rest:tt)*) => {}; 273 (emit V128Load64Splat $($rest:tt)*) => {}; 274 (emit I8x16Splat $($rest:tt)*) => {}; 275 (emit I16x8Splat $($rest:tt)*) => {}; 276 (emit I32x4Splat $($rest:tt)*) => {}; 277 (emit I64x2Splat $($rest:tt)*) => {}; 278 (emit F32x4Splat $($rest:tt)*) => {}; 279 (emit F64x2Splat $($rest:tt)*) => {}; 280 (emit I32AtomicStore8 $($rest:tt)*) => {}; 281 (emit I32AtomicStore16 $($rest:tt)*) => {}; 282 (emit I32AtomicStore $($rest:tt)*) => {}; 283 (emit I64AtomicStore8 $($rest:tt)*) => {}; 284 (emit I64AtomicStore16 $($rest:tt)*) => {}; 285 (emit I64AtomicStore32 $($rest:tt)*) => {}; 286 (emit I64AtomicStore $($rest:tt)*) => {}; 287 (emit I32AtomicRmw8AddU $($rest:tt)*) => {}; 288 (emit I32AtomicRmw16AddU $($rest:tt)*) => {}; 289 (emit I32AtomicRmwAdd $($rest:tt)*) => {}; 290 (emit I64AtomicRmw8AddU $($rest:tt)*) => {}; 291 (emit I64AtomicRmw16AddU $($rest:tt)*) => {}; 292 (emit I64AtomicRmw32AddU $($rest:tt)*) => {}; 293 (emit I64AtomicRmwAdd $($rest:tt)*) => {}; 294 (emit I8x16Shuffle $($rest:tt)*) => {}; 295 (emit I8x16Swizzle $($rest:tt)*) => {}; 296 (emit I32AtomicRmw8SubU $($rest:tt)*) => {}; 297 (emit I32AtomicRmw16SubU $($rest:tt)*) => {}; 298 (emit I32AtomicRmwSub $($rest:tt)*) => {}; 299 (emit I64AtomicRmw8SubU $($rest:tt)*) => {}; 300 (emit I64AtomicRmw16SubU $($rest:tt)*) => {}; 301 (emit I64AtomicRmw32SubU $($rest:tt)*) => {}; 302 (emit I64AtomicRmwSub $($rest:tt)*) => {}; 303 (emit I32AtomicRmw8XchgU $($rest:tt)*) => {}; 304 (emit I32AtomicRmw16XchgU $($rest:tt)*) => {}; 305 (emit I32AtomicRmwXchg $($rest:tt)*) => {}; 306 (emit I64AtomicRmw8XchgU $($rest:tt)*) => {}; 307 (emit I64AtomicRmw16XchgU $($rest:tt)*) => {}; 308 (emit I64AtomicRmw32XchgU $($rest:tt)*) => {}; 309 (emit I64AtomicRmwXchg $($rest:tt)*) => {}; 310 (emit I8x16ExtractLaneS $($rest:tt)*) => {}; 311 (emit I8x16ExtractLaneU $($rest:tt)*) => {}; 312 (emit I16x8ExtractLaneS $($rest:tt)*) => {}; 313 (emit I16x8ExtractLaneU $($rest:tt)*) => {}; 314 (emit I32x4ExtractLane $($rest:tt)*) => {}; 315 (emit I64x2ExtractLane $($rest:tt)*) => {}; 316 (emit F32x4ExtractLane $($rest:tt)*) => {}; 317 (emit F64x2ExtractLane $($rest:tt)*) => {}; 318 (emit I32AtomicRmw8AndU $($rest:tt)*) => {}; 319 (emit I32AtomicRmw16AndU $($rest:tt)*) => {}; 320 (emit I32AtomicRmwAnd $($rest:tt)*) => {}; 321 (emit I64AtomicRmw8AndU $($rest:tt)*) => {}; 322 (emit I64AtomicRmw16AndU $($rest:tt)*) => {}; 323 (emit I64AtomicRmw32AndU $($rest:tt)*) => {}; 324 (emit I64AtomicRmwAnd $($rest:tt)*) => {}; 325 (emit I32AtomicRmw8OrU $($rest:tt)*) => {}; 326 (emit I32AtomicRmw16OrU $($rest:tt)*) => {}; 327 (emit I32AtomicRmwOr $($rest:tt)*) => {}; 328 (emit I64AtomicRmw8OrU $($rest:tt)*) => {}; 329 (emit I64AtomicRmw16OrU $($rest:tt)*) => {}; 330 (emit I64AtomicRmw32OrU $($rest:tt)*) => {}; 331 (emit I64AtomicRmwOr $($rest:tt)*) => {}; 332 (emit I32AtomicRmw8XorU $($rest:tt)*) => {}; 333 (emit I32AtomicRmw16XorU $($rest:tt)*) => {}; 334 (emit I32AtomicRmwXor $($rest:tt)*) => {}; 335 (emit I64AtomicRmw8XorU $($rest:tt)*) => {}; 336 (emit I64AtomicRmw16XorU $($rest:tt)*) => {}; 337 (emit I64AtomicRmw32XorU $($rest:tt)*) => {}; 338 (emit I64AtomicRmwXor $($rest:tt)*) => {}; 339 (emit I32AtomicRmw8CmpxchgU $($rest:tt)*) => {}; 340 (emit I32AtomicRmw16CmpxchgU $($rest:tt)*) => {}; 341 (emit I32AtomicRmwCmpxchg $($rest:tt)*) => {}; 342 (emit I64AtomicRmw8CmpxchgU $($rest:tt)*) => {}; 343 (emit I64AtomicRmw16CmpxchgU $($rest:tt)*) => {}; 344 (emit I64AtomicRmw32CmpxchgU $($rest:tt)*) => {}; 345 (emit I64AtomicRmwCmpxchg $($rest:tt)*) => {}; 346 347 (emit $unsupported:tt $($rest:tt)*) => {$($rest)*}; 348 } 349 350 impl<'a, 'translation, 'data, M> VisitOperator<'a> for CodeGen<'a, 'translation, 'data, M, Emission> 351 where 352 M: MacroAssembler, 353 { 354 type Output = Result<()>; 355 356 fn visit_i32_const(&mut self, val: i32) -> Self::Output { 357 self.context.stack.push(Val::i32(val)); 358 359 Ok(()) 360 } 361 362 fn visit_i64_const(&mut self, val: i64) -> Self::Output { 363 self.context.stack.push(Val::i64(val)); 364 Ok(()) 365 } 366 367 fn visit_f32_const(&mut self, val: Ieee32) -> Self::Output { 368 self.context.stack.push(Val::f32(val)); 369 Ok(()) 370 } 371 372 fn visit_f64_const(&mut self, val: Ieee64) -> Self::Output { 373 self.context.stack.push(Val::f64(val)); 374 Ok(()) 375 } 376 377 fn visit_f32_add(&mut self) -> Self::Output { 378 self.context.binop( 379 self.masm, 380 OperandSize::S32, 381 &mut |masm: &mut M, dst, src, size| { 382 masm.float_add(writable!(dst), dst, src, size)?; 383 Ok(TypedReg::f32(dst)) 384 }, 385 ) 386 } 387 388 fn visit_f64_add(&mut self) -> Self::Output { 389 self.context.binop( 390 self.masm, 391 OperandSize::S64, 392 &mut |masm: &mut M, dst, src, size| { 393 masm.float_add(writable!(dst), dst, src, size)?; 394 Ok(TypedReg::f64(dst)) 395 }, 396 ) 397 } 398 399 fn visit_f32_sub(&mut self) -> Self::Output { 400 self.context.binop( 401 self.masm, 402 OperandSize::S32, 403 &mut |masm: &mut M, dst, src, size| { 404 masm.float_sub(writable!(dst), dst, src, size)?; 405 Ok(TypedReg::f32(dst)) 406 }, 407 ) 408 } 409 410 fn visit_f64_sub(&mut self) -> Self::Output { 411 self.context.binop( 412 self.masm, 413 OperandSize::S64, 414 &mut |masm: &mut M, dst, src, size| { 415 masm.float_sub(writable!(dst), dst, src, size)?; 416 Ok(TypedReg::f64(dst)) 417 }, 418 ) 419 } 420 421 fn visit_f32_mul(&mut self) -> Self::Output { 422 self.context.binop( 423 self.masm, 424 OperandSize::S32, 425 &mut |masm: &mut M, dst, src, size| { 426 masm.float_mul(writable!(dst), dst, src, size)?; 427 Ok(TypedReg::f32(dst)) 428 }, 429 ) 430 } 431 432 fn visit_f64_mul(&mut self) -> Self::Output { 433 self.context.binop( 434 self.masm, 435 OperandSize::S64, 436 &mut |masm: &mut M, dst, src, size| { 437 masm.float_mul(writable!(dst), dst, src, size)?; 438 Ok(TypedReg::f64(dst)) 439 }, 440 ) 441 } 442 443 fn visit_f32_div(&mut self) -> Self::Output { 444 self.context.binop( 445 self.masm, 446 OperandSize::S32, 447 &mut |masm: &mut M, dst, src, size| { 448 masm.float_div(writable!(dst), dst, src, size)?; 449 Ok(TypedReg::f32(dst)) 450 }, 451 ) 452 } 453 454 fn visit_f64_div(&mut self) -> Self::Output { 455 self.context.binop( 456 self.masm, 457 OperandSize::S64, 458 &mut |masm: &mut M, dst, src, size| { 459 masm.float_div(writable!(dst), dst, src, size)?; 460 Ok(TypedReg::f64(dst)) 461 }, 462 ) 463 } 464 465 fn visit_f32_min(&mut self) -> Self::Output { 466 self.context.binop( 467 self.masm, 468 OperandSize::S32, 469 &mut |masm: &mut M, dst, src, size| { 470 masm.float_min(writable!(dst), dst, src, size)?; 471 Ok(TypedReg::f32(dst)) 472 }, 473 ) 474 } 475 476 fn visit_f64_min(&mut self) -> Self::Output { 477 self.context.binop( 478 self.masm, 479 OperandSize::S64, 480 &mut |masm: &mut M, dst, src, size| { 481 masm.float_min(writable!(dst), dst, src, size)?; 482 Ok(TypedReg::f64(dst)) 483 }, 484 ) 485 } 486 487 fn visit_f32_max(&mut self) -> Self::Output { 488 self.context.binop( 489 self.masm, 490 OperandSize::S32, 491 &mut |masm: &mut M, dst, src, size| { 492 masm.float_max(writable!(dst), dst, src, size)?; 493 Ok(TypedReg::f32(dst)) 494 }, 495 ) 496 } 497 498 fn visit_f64_max(&mut self) -> Self::Output { 499 self.context.binop( 500 self.masm, 501 OperandSize::S64, 502 &mut |masm: &mut M, dst, src, size| { 503 masm.float_max(writable!(dst), dst, src, size)?; 504 Ok(TypedReg::f64(dst)) 505 }, 506 ) 507 } 508 509 fn visit_f32_copysign(&mut self) -> Self::Output { 510 self.context.binop( 511 self.masm, 512 OperandSize::S32, 513 &mut |masm: &mut M, dst, src, size| { 514 masm.float_copysign(writable!(dst), dst, src, size)?; 515 Ok(TypedReg::f32(dst)) 516 }, 517 ) 518 } 519 520 fn visit_f64_copysign(&mut self) -> Self::Output { 521 self.context.binop( 522 self.masm, 523 OperandSize::S64, 524 &mut |masm: &mut M, dst, src, size| { 525 masm.float_copysign(writable!(dst), dst, src, size)?; 526 Ok(TypedReg::f64(dst)) 527 }, 528 ) 529 } 530 531 fn visit_f32_abs(&mut self) -> Self::Output { 532 self.context.unop(self.masm, &mut |masm, reg| { 533 masm.float_abs(writable!(reg), OperandSize::S32)?; 534 Ok(TypedReg::f32(reg)) 535 }) 536 } 537 538 fn visit_f64_abs(&mut self) -> Self::Output { 539 self.context.unop(self.masm, &mut |masm, reg| { 540 masm.float_abs(writable!(reg), OperandSize::S64)?; 541 Ok(TypedReg::f64(reg)) 542 }) 543 } 544 545 fn visit_f32_neg(&mut self) -> Self::Output { 546 self.context.unop(self.masm, &mut |masm, reg| { 547 masm.float_neg(writable!(reg), OperandSize::S32)?; 548 Ok(TypedReg::f32(reg)) 549 }) 550 } 551 552 fn visit_f64_neg(&mut self) -> Self::Output { 553 self.context.unop(self.masm, &mut |masm, reg| { 554 masm.float_neg(writable!(reg), OperandSize::S64)?; 555 Ok(TypedReg::f64(reg)) 556 }) 557 } 558 559 fn visit_f32_floor(&mut self) -> Self::Output { 560 self.masm.float_round( 561 RoundingMode::Down, 562 &mut self.env, 563 &mut self.context, 564 OperandSize::S32, 565 |env, cx, masm| { 566 let builtin = env.builtins.floor_f32::<M::ABI>()?; 567 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 568 }, 569 ) 570 } 571 572 fn visit_f64_floor(&mut self) -> Self::Output { 573 self.masm.float_round( 574 RoundingMode::Down, 575 &mut self.env, 576 &mut self.context, 577 OperandSize::S64, 578 |env, cx, masm| { 579 let builtin = env.builtins.floor_f64::<M::ABI>()?; 580 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 581 }, 582 ) 583 } 584 585 fn visit_f32_ceil(&mut self) -> Self::Output { 586 self.masm.float_round( 587 RoundingMode::Up, 588 &mut self.env, 589 &mut self.context, 590 OperandSize::S32, 591 |env, cx, masm| { 592 let builtin = env.builtins.ceil_f32::<M::ABI>()?; 593 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 594 }, 595 ) 596 } 597 598 fn visit_f64_ceil(&mut self) -> Self::Output { 599 self.masm.float_round( 600 RoundingMode::Up, 601 &mut self.env, 602 &mut self.context, 603 OperandSize::S64, 604 |env, cx, masm| { 605 let builtin = env.builtins.ceil_f64::<M::ABI>()?; 606 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 607 }, 608 ) 609 } 610 611 fn visit_f32_nearest(&mut self) -> Self::Output { 612 self.masm.float_round( 613 RoundingMode::Nearest, 614 &mut self.env, 615 &mut self.context, 616 OperandSize::S32, 617 |env, cx, masm| { 618 let builtin = env.builtins.nearest_f32::<M::ABI>()?; 619 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 620 }, 621 ) 622 } 623 624 fn visit_f64_nearest(&mut self) -> Self::Output { 625 self.masm.float_round( 626 RoundingMode::Nearest, 627 &mut self.env, 628 &mut self.context, 629 OperandSize::S64, 630 |env, cx, masm| { 631 let builtin = env.builtins.nearest_f64::<M::ABI>()?; 632 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 633 }, 634 ) 635 } 636 637 fn visit_f32_trunc(&mut self) -> Self::Output { 638 self.masm.float_round( 639 RoundingMode::Zero, 640 &mut self.env, 641 &mut self.context, 642 OperandSize::S32, 643 |env, cx, masm| { 644 let builtin = env.builtins.trunc_f32::<M::ABI>()?; 645 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 646 }, 647 ) 648 } 649 650 fn visit_f64_trunc(&mut self) -> Self::Output { 651 self.masm.float_round( 652 RoundingMode::Zero, 653 &mut self.env, 654 &mut self.context, 655 OperandSize::S64, 656 |env, cx, masm| { 657 let builtin = env.builtins.trunc_f64::<M::ABI>()?; 658 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 659 }, 660 ) 661 } 662 663 fn visit_f32_sqrt(&mut self) -> Self::Output { 664 self.context.unop(self.masm, &mut |masm, reg| { 665 masm.float_sqrt(writable!(reg), reg, OperandSize::S32)?; 666 Ok(TypedReg::f32(reg)) 667 }) 668 } 669 670 fn visit_f64_sqrt(&mut self) -> Self::Output { 671 self.context.unop(self.masm, &mut |masm, reg| { 672 masm.float_sqrt(writable!(reg), reg, OperandSize::S64)?; 673 Ok(TypedReg::f64(reg)) 674 }) 675 } 676 677 fn visit_f32_eq(&mut self) -> Self::Output { 678 self.context.float_cmp_op( 679 self.masm, 680 OperandSize::S32, 681 &mut |masm: &mut M, dst, src1, src2, size| { 682 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Eq, size) 683 }, 684 ) 685 } 686 687 fn visit_f64_eq(&mut self) -> Self::Output { 688 self.context.float_cmp_op( 689 self.masm, 690 OperandSize::S64, 691 &mut |masm: &mut M, dst, src1, src2, size| { 692 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Eq, size) 693 }, 694 ) 695 } 696 697 fn visit_f32_ne(&mut self) -> Self::Output { 698 self.context.float_cmp_op( 699 self.masm, 700 OperandSize::S32, 701 &mut |masm: &mut M, dst, src1, src2, size| { 702 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ne, size) 703 }, 704 ) 705 } 706 707 fn visit_f64_ne(&mut self) -> Self::Output { 708 self.context.float_cmp_op( 709 self.masm, 710 OperandSize::S64, 711 &mut |masm: &mut M, dst, src1, src2, size| { 712 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ne, size) 713 }, 714 ) 715 } 716 717 fn visit_f32_lt(&mut self) -> Self::Output { 718 self.context.float_cmp_op( 719 self.masm, 720 OperandSize::S32, 721 &mut |masm: &mut M, dst, src1, src2, size| { 722 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Lt, size) 723 }, 724 ) 725 } 726 727 fn visit_f64_lt(&mut self) -> Self::Output { 728 self.context.float_cmp_op( 729 self.masm, 730 OperandSize::S64, 731 &mut |masm: &mut M, dst, src1, src2, size| { 732 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Lt, size) 733 }, 734 ) 735 } 736 737 fn visit_f32_gt(&mut self) -> Self::Output { 738 self.context.float_cmp_op( 739 self.masm, 740 OperandSize::S32, 741 &mut |masm: &mut M, dst, src1, src2, size| { 742 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Gt, size) 743 }, 744 ) 745 } 746 747 fn visit_f64_gt(&mut self) -> Self::Output { 748 self.context.float_cmp_op( 749 self.masm, 750 OperandSize::S64, 751 &mut |masm: &mut M, dst, src1, src2, size| { 752 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Gt, size) 753 }, 754 ) 755 } 756 757 fn visit_f32_le(&mut self) -> Self::Output { 758 self.context.float_cmp_op( 759 self.masm, 760 OperandSize::S32, 761 &mut |masm: &mut M, dst, src1, src2, size| { 762 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Le, size) 763 }, 764 ) 765 } 766 767 fn visit_f64_le(&mut self) -> Self::Output { 768 self.context.float_cmp_op( 769 self.masm, 770 OperandSize::S64, 771 &mut |masm: &mut M, dst, src1, src2, size| { 772 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Le, size) 773 }, 774 ) 775 } 776 777 fn visit_f32_ge(&mut self) -> Self::Output { 778 self.context.float_cmp_op( 779 self.masm, 780 OperandSize::S32, 781 &mut |masm: &mut M, dst, src1, src2, size| { 782 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ge, size) 783 }, 784 ) 785 } 786 787 fn visit_f64_ge(&mut self) -> Self::Output { 788 self.context.float_cmp_op( 789 self.masm, 790 OperandSize::S64, 791 &mut |masm: &mut M, dst, src1, src2, size| { 792 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ge, size) 793 }, 794 ) 795 } 796 797 fn visit_f32_convert_i32_s(&mut self) -> Self::Output { 798 self.context 799 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 800 masm.signed_convert(writable!(dst), src, OperandSize::S32, dst_size) 801 }) 802 } 803 804 fn visit_f32_convert_i32_u(&mut self) -> Self::Output { 805 self.context.convert_op_with_tmp_reg( 806 self.masm, 807 WasmValType::F32, 808 RegClass::Int, 809 |masm, dst, src, tmp_gpr, dst_size| { 810 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S32, dst_size) 811 }, 812 ) 813 } 814 815 fn visit_f32_convert_i64_s(&mut self) -> Self::Output { 816 self.context 817 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 818 masm.signed_convert(writable!(dst), src, OperandSize::S64, dst_size) 819 }) 820 } 821 822 fn visit_f32_convert_i64_u(&mut self) -> Self::Output { 823 self.context.convert_op_with_tmp_reg( 824 self.masm, 825 WasmValType::F32, 826 RegClass::Int, 827 |masm, dst, src, tmp_gpr, dst_size| { 828 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S64, dst_size) 829 }, 830 ) 831 } 832 833 fn visit_f64_convert_i32_s(&mut self) -> Self::Output { 834 self.context 835 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 836 masm.signed_convert(writable!(dst), src, OperandSize::S32, dst_size) 837 }) 838 } 839 840 fn visit_f64_convert_i32_u(&mut self) -> Self::Output { 841 self.context.convert_op_with_tmp_reg( 842 self.masm, 843 WasmValType::F64, 844 RegClass::Int, 845 |masm, dst, src, tmp_gpr, dst_size| { 846 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S32, dst_size) 847 }, 848 ) 849 } 850 851 fn visit_f64_convert_i64_s(&mut self) -> Self::Output { 852 self.context 853 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 854 masm.signed_convert(writable!(dst), src, OperandSize::S64, dst_size) 855 }) 856 } 857 858 fn visit_f64_convert_i64_u(&mut self) -> Self::Output { 859 self.context.convert_op_with_tmp_reg( 860 self.masm, 861 WasmValType::F64, 862 RegClass::Int, 863 |masm, dst, src, tmp_gpr, dst_size| { 864 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S64, dst_size) 865 }, 866 ) 867 } 868 869 fn visit_f32_reinterpret_i32(&mut self) -> Self::Output { 870 self.context 871 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, size| { 872 masm.reinterpret_int_as_float(writable!(dst), src.into(), size) 873 }) 874 } 875 876 fn visit_f64_reinterpret_i64(&mut self) -> Self::Output { 877 self.context 878 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, size| { 879 masm.reinterpret_int_as_float(writable!(dst), src.into(), size) 880 }) 881 } 882 883 fn visit_f32_demote_f64(&mut self) -> Self::Output { 884 self.context.unop(self.masm, &mut |masm, reg| { 885 masm.demote(writable!(reg), reg)?; 886 Ok(TypedReg::f32(reg)) 887 }) 888 } 889 890 fn visit_f64_promote_f32(&mut self) -> Self::Output { 891 self.context.unop(self.masm, &mut |masm, reg| { 892 masm.promote(writable!(reg), reg)?; 893 Ok(TypedReg::f64(reg)) 894 }) 895 } 896 897 fn visit_i32_add(&mut self) -> Self::Output { 898 self.context.i32_binop(self.masm, |masm, dst, src, size| { 899 masm.add(writable!(dst), dst, src, size)?; 900 Ok(TypedReg::i32(dst)) 901 }) 902 } 903 904 fn visit_i64_add(&mut self) -> Self::Output { 905 self.context.i64_binop(self.masm, |masm, dst, src, size| { 906 masm.add(writable!(dst), dst, src, size)?; 907 Ok(TypedReg::i64(dst)) 908 }) 909 } 910 911 fn visit_i32_sub(&mut self) -> Self::Output { 912 self.context.i32_binop(self.masm, |masm, dst, src, size| { 913 masm.sub(writable!(dst), dst, src, size)?; 914 Ok(TypedReg::i32(dst)) 915 }) 916 } 917 918 fn visit_i64_sub(&mut self) -> Self::Output { 919 self.context.i64_binop(self.masm, |masm, dst, src, size| { 920 masm.sub(writable!(dst), dst, src, size)?; 921 Ok(TypedReg::i64(dst)) 922 }) 923 } 924 925 fn visit_i32_mul(&mut self) -> Self::Output { 926 self.context.i32_binop(self.masm, |masm, dst, src, size| { 927 masm.mul(writable!(dst), dst, src, size)?; 928 Ok(TypedReg::i32(dst)) 929 }) 930 } 931 932 fn visit_i64_mul(&mut self) -> Self::Output { 933 self.context.i64_binop(self.masm, |masm, dst, src, size| { 934 masm.mul(writable!(dst), dst, src, size)?; 935 Ok(TypedReg::i64(dst)) 936 }) 937 } 938 939 fn visit_i32_div_s(&mut self) -> Self::Output { 940 use DivKind::*; 941 use OperandSize::*; 942 943 self.masm.div(&mut self.context, Signed, S32) 944 } 945 946 fn visit_i32_div_u(&mut self) -> Self::Output { 947 use DivKind::*; 948 use OperandSize::*; 949 950 self.masm.div(&mut self.context, Unsigned, S32) 951 } 952 953 fn visit_i64_div_s(&mut self) -> Self::Output { 954 use DivKind::*; 955 use OperandSize::*; 956 957 self.masm.div(&mut self.context, Signed, S64) 958 } 959 960 fn visit_i64_div_u(&mut self) -> Self::Output { 961 use DivKind::*; 962 use OperandSize::*; 963 964 self.masm.div(&mut self.context, Unsigned, S64) 965 } 966 967 fn visit_i32_rem_s(&mut self) -> Self::Output { 968 use OperandSize::*; 969 use RemKind::*; 970 971 self.masm.rem(&mut self.context, Signed, S32) 972 } 973 974 fn visit_i32_rem_u(&mut self) -> Self::Output { 975 use OperandSize::*; 976 use RemKind::*; 977 978 self.masm.rem(&mut self.context, Unsigned, S32) 979 } 980 981 fn visit_i64_rem_s(&mut self) -> Self::Output { 982 use OperandSize::*; 983 use RemKind::*; 984 985 self.masm.rem(&mut self.context, Signed, S64) 986 } 987 988 fn visit_i64_rem_u(&mut self) -> Self::Output { 989 use OperandSize::*; 990 use RemKind::*; 991 992 self.masm.rem(&mut self.context, Unsigned, S64) 993 } 994 995 fn visit_i32_eq(&mut self) -> Self::Output { 996 self.cmp_i32s(IntCmpKind::Eq) 997 } 998 999 fn visit_i64_eq(&mut self) -> Self::Output { 1000 self.cmp_i64s(IntCmpKind::Eq) 1001 } 1002 1003 fn visit_i32_ne(&mut self) -> Self::Output { 1004 self.cmp_i32s(IntCmpKind::Ne) 1005 } 1006 1007 fn visit_i64_ne(&mut self) -> Self::Output { 1008 self.cmp_i64s(IntCmpKind::Ne) 1009 } 1010 1011 fn visit_i32_lt_s(&mut self) -> Self::Output { 1012 self.cmp_i32s(IntCmpKind::LtS) 1013 } 1014 1015 fn visit_i64_lt_s(&mut self) -> Self::Output { 1016 self.cmp_i64s(IntCmpKind::LtS) 1017 } 1018 1019 fn visit_i32_lt_u(&mut self) -> Self::Output { 1020 self.cmp_i32s(IntCmpKind::LtU) 1021 } 1022 1023 fn visit_i64_lt_u(&mut self) -> Self::Output { 1024 self.cmp_i64s(IntCmpKind::LtU) 1025 } 1026 1027 fn visit_i32_le_s(&mut self) -> Self::Output { 1028 self.cmp_i32s(IntCmpKind::LeS) 1029 } 1030 1031 fn visit_i64_le_s(&mut self) -> Self::Output { 1032 self.cmp_i64s(IntCmpKind::LeS) 1033 } 1034 1035 fn visit_i32_le_u(&mut self) -> Self::Output { 1036 self.cmp_i32s(IntCmpKind::LeU) 1037 } 1038 1039 fn visit_i64_le_u(&mut self) -> Self::Output { 1040 self.cmp_i64s(IntCmpKind::LeU) 1041 } 1042 1043 fn visit_i32_gt_s(&mut self) -> Self::Output { 1044 self.cmp_i32s(IntCmpKind::GtS) 1045 } 1046 1047 fn visit_i64_gt_s(&mut self) -> Self::Output { 1048 self.cmp_i64s(IntCmpKind::GtS) 1049 } 1050 1051 fn visit_i32_gt_u(&mut self) -> Self::Output { 1052 self.cmp_i32s(IntCmpKind::GtU) 1053 } 1054 1055 fn visit_i64_gt_u(&mut self) -> Self::Output { 1056 self.cmp_i64s(IntCmpKind::GtU) 1057 } 1058 1059 fn visit_i32_ge_s(&mut self) -> Self::Output { 1060 self.cmp_i32s(IntCmpKind::GeS) 1061 } 1062 1063 fn visit_i64_ge_s(&mut self) -> Self::Output { 1064 self.cmp_i64s(IntCmpKind::GeS) 1065 } 1066 1067 fn visit_i32_ge_u(&mut self) -> Self::Output { 1068 self.cmp_i32s(IntCmpKind::GeU) 1069 } 1070 1071 fn visit_i64_ge_u(&mut self) -> Self::Output { 1072 self.cmp_i64s(IntCmpKind::GeU) 1073 } 1074 1075 fn visit_i32_eqz(&mut self) -> Self::Output { 1076 use OperandSize::*; 1077 1078 self.context.unop(self.masm, &mut |masm, reg| { 1079 masm.cmp_with_set(writable!(reg.into()), RegImm::i32(0), IntCmpKind::Eq, S32)?; 1080 Ok(TypedReg::i32(reg)) 1081 }) 1082 } 1083 1084 fn visit_i64_eqz(&mut self) -> Self::Output { 1085 use OperandSize::*; 1086 1087 self.context.unop(self.masm, &mut |masm, reg| { 1088 masm.cmp_with_set(writable!(reg.into()), RegImm::i64(0), IntCmpKind::Eq, S64)?; 1089 Ok(TypedReg::i32(reg)) // Return value for `i64.eqz` is an `i32`. 1090 }) 1091 } 1092 1093 fn visit_i32_clz(&mut self) -> Self::Output { 1094 use OperandSize::*; 1095 1096 self.context.unop(self.masm, &mut |masm, reg| { 1097 masm.clz(writable!(reg), reg, S32)?; 1098 Ok(TypedReg::i32(reg)) 1099 }) 1100 } 1101 1102 fn visit_i64_clz(&mut self) -> Self::Output { 1103 use OperandSize::*; 1104 1105 self.context.unop(self.masm, &mut |masm, reg| { 1106 masm.clz(writable!(reg), reg, S64)?; 1107 Ok(TypedReg::i64(reg)) 1108 }) 1109 } 1110 1111 fn visit_i32_ctz(&mut self) -> Self::Output { 1112 use OperandSize::*; 1113 1114 self.context.unop(self.masm, &mut |masm, reg| { 1115 masm.ctz(writable!(reg), reg, S32)?; 1116 Ok(TypedReg::i32(reg)) 1117 }) 1118 } 1119 1120 fn visit_i64_ctz(&mut self) -> Self::Output { 1121 use OperandSize::*; 1122 1123 self.context.unop(self.masm, &mut |masm, reg| { 1124 masm.ctz(writable!(reg), reg, S64)?; 1125 Ok(TypedReg::i64(reg)) 1126 }) 1127 } 1128 1129 fn visit_i32_and(&mut self) -> Self::Output { 1130 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1131 masm.and(writable!(dst), dst, src, size)?; 1132 Ok(TypedReg::i32(dst)) 1133 }) 1134 } 1135 1136 fn visit_i64_and(&mut self) -> Self::Output { 1137 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1138 masm.and(writable!(dst), dst, src, size)?; 1139 Ok(TypedReg::i64(dst)) 1140 }) 1141 } 1142 1143 fn visit_i32_or(&mut self) -> Self::Output { 1144 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1145 masm.or(writable!(dst), dst, src, size)?; 1146 Ok(TypedReg::i32(dst)) 1147 }) 1148 } 1149 1150 fn visit_i64_or(&mut self) -> Self::Output { 1151 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1152 masm.or(writable!(dst), dst, src, size)?; 1153 Ok(TypedReg::i64(dst)) 1154 }) 1155 } 1156 1157 fn visit_i32_xor(&mut self) -> Self::Output { 1158 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1159 masm.xor(writable!(dst), dst, src, size)?; 1160 Ok(TypedReg::i32(dst)) 1161 }) 1162 } 1163 1164 fn visit_i64_xor(&mut self) -> Self::Output { 1165 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1166 masm.xor(writable!(dst), dst, src, size)?; 1167 Ok(TypedReg::i64(dst)) 1168 }) 1169 } 1170 1171 fn visit_i32_shl(&mut self) -> Self::Output { 1172 use ShiftKind::*; 1173 1174 self.context.i32_shift(self.masm, Shl) 1175 } 1176 1177 fn visit_i64_shl(&mut self) -> Self::Output { 1178 use ShiftKind::*; 1179 1180 self.context.i64_shift(self.masm, Shl) 1181 } 1182 1183 fn visit_i32_shr_s(&mut self) -> Self::Output { 1184 use ShiftKind::*; 1185 1186 self.context.i32_shift(self.masm, ShrS) 1187 } 1188 1189 fn visit_i64_shr_s(&mut self) -> Self::Output { 1190 use ShiftKind::*; 1191 1192 self.context.i64_shift(self.masm, ShrS) 1193 } 1194 1195 fn visit_i32_shr_u(&mut self) -> Self::Output { 1196 use ShiftKind::*; 1197 1198 self.context.i32_shift(self.masm, ShrU) 1199 } 1200 1201 fn visit_i64_shr_u(&mut self) -> Self::Output { 1202 use ShiftKind::*; 1203 1204 self.context.i64_shift(self.masm, ShrU) 1205 } 1206 1207 fn visit_i32_rotl(&mut self) -> Self::Output { 1208 use ShiftKind::*; 1209 1210 self.context.i32_shift(self.masm, Rotl) 1211 } 1212 1213 fn visit_i64_rotl(&mut self) -> Self::Output { 1214 use ShiftKind::*; 1215 1216 self.context.i64_shift(self.masm, Rotl) 1217 } 1218 1219 fn visit_i32_rotr(&mut self) -> Self::Output { 1220 use ShiftKind::*; 1221 1222 self.context.i32_shift(self.masm, Rotr) 1223 } 1224 1225 fn visit_i64_rotr(&mut self) -> Self::Output { 1226 use ShiftKind::*; 1227 1228 self.context.i64_shift(self.masm, Rotr) 1229 } 1230 1231 fn visit_end(&mut self) -> Self::Output { 1232 if !self.context.reachable { 1233 self.handle_unreachable_end() 1234 } else { 1235 let mut control = self.pop_control_frame()?; 1236 control.emit_end(self.masm, &mut self.context) 1237 } 1238 } 1239 1240 fn visit_i32_popcnt(&mut self) -> Self::Output { 1241 use OperandSize::*; 1242 self.masm.popcnt(&mut self.context, S32) 1243 } 1244 1245 fn visit_i64_popcnt(&mut self) -> Self::Output { 1246 use OperandSize::*; 1247 1248 self.masm.popcnt(&mut self.context, S64) 1249 } 1250 1251 fn visit_i32_wrap_i64(&mut self) -> Self::Output { 1252 self.context.unop(self.masm, &mut |masm, reg| { 1253 masm.wrap(writable!(reg), reg)?; 1254 Ok(TypedReg::i32(reg)) 1255 }) 1256 } 1257 1258 fn visit_i64_extend_i32_s(&mut self) -> Self::Output { 1259 self.context.unop(self.masm, &mut |masm, reg| { 1260 masm.extend(writable!(reg), reg, Extend::<Signed>::I64Extend32.into())?; 1261 Ok(TypedReg::i64(reg)) 1262 }) 1263 } 1264 1265 fn visit_i64_extend_i32_u(&mut self) -> Self::Output { 1266 self.context.unop(self.masm, &mut |masm, reg| { 1267 masm.extend(writable!(reg), reg, Extend::<Zero>::I64Extend32.into())?; 1268 Ok(TypedReg::i64(reg)) 1269 }) 1270 } 1271 1272 fn visit_i32_extend8_s(&mut self) -> Self::Output { 1273 self.context.unop(self.masm, &mut |masm, reg| { 1274 masm.extend(writable!(reg), reg, Extend::<Signed>::I32Extend8.into())?; 1275 Ok(TypedReg::i32(reg)) 1276 }) 1277 } 1278 1279 fn visit_i32_extend16_s(&mut self) -> Self::Output { 1280 self.context.unop(self.masm, &mut |masm, reg| { 1281 masm.extend(writable!(reg), reg, Extend::<Signed>::I32Extend16.into())?; 1282 Ok(TypedReg::i32(reg)) 1283 }) 1284 } 1285 1286 fn visit_i64_extend8_s(&mut self) -> Self::Output { 1287 self.context.unop(self.masm, &mut |masm, reg| { 1288 masm.extend(writable!(reg), reg, Extend::<Signed>::I64Extend8.into())?; 1289 Ok(TypedReg::i64(reg)) 1290 }) 1291 } 1292 1293 fn visit_i64_extend16_s(&mut self) -> Self::Output { 1294 self.context.unop(self.masm, &mut |masm, reg| { 1295 masm.extend(writable!(reg), reg, Extend::<Signed>::I64Extend16.into())?; 1296 Ok(TypedReg::i64(reg)) 1297 }) 1298 } 1299 1300 fn visit_i64_extend32_s(&mut self) -> Self::Output { 1301 self.context.unop(self.masm, &mut |masm, reg| { 1302 masm.extend(writable!(reg), reg, Extend::<Signed>::I64Extend32.into())?; 1303 Ok(TypedReg::i64(reg)) 1304 }) 1305 } 1306 1307 fn visit_i32_trunc_f32_s(&mut self) -> Self::Output { 1308 use OperandSize::*; 1309 1310 self.context 1311 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1312 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Unchecked) 1313 }) 1314 } 1315 1316 fn visit_i32_trunc_f32_u(&mut self) -> Self::Output { 1317 use OperandSize::*; 1318 1319 self.masm 1320 .unsigned_truncate(&mut self.context, S32, S32, TruncKind::Unchecked) 1321 } 1322 1323 fn visit_i32_trunc_f64_s(&mut self) -> Self::Output { 1324 use OperandSize::*; 1325 1326 self.context 1327 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1328 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Unchecked) 1329 }) 1330 } 1331 1332 fn visit_i32_trunc_f64_u(&mut self) -> Self::Output { 1333 use OperandSize::*; 1334 self.masm 1335 .unsigned_truncate(&mut self.context, S64, S32, TruncKind::Unchecked) 1336 } 1337 1338 fn visit_i64_trunc_f32_s(&mut self) -> Self::Output { 1339 use OperandSize::*; 1340 1341 self.context 1342 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1343 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Unchecked) 1344 }) 1345 } 1346 1347 fn visit_i64_trunc_f32_u(&mut self) -> Self::Output { 1348 use OperandSize::*; 1349 1350 self.masm 1351 .unsigned_truncate(&mut self.context, S32, S64, TruncKind::Unchecked) 1352 } 1353 1354 fn visit_i64_trunc_f64_s(&mut self) -> Self::Output { 1355 use OperandSize::*; 1356 1357 self.context 1358 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1359 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Unchecked) 1360 }) 1361 } 1362 1363 fn visit_i64_trunc_f64_u(&mut self) -> Self::Output { 1364 use OperandSize::*; 1365 1366 self.masm 1367 .unsigned_truncate(&mut self.context, S64, S64, TruncKind::Unchecked) 1368 } 1369 1370 fn visit_i32_reinterpret_f32(&mut self) -> Self::Output { 1371 self.context 1372 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, size| { 1373 masm.reinterpret_float_as_int(writable!(dst), src.into(), size) 1374 }) 1375 } 1376 1377 fn visit_i64_reinterpret_f64(&mut self) -> Self::Output { 1378 self.context 1379 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, size| { 1380 masm.reinterpret_float_as_int(writable!(dst), src.into(), size) 1381 }) 1382 } 1383 1384 fn visit_local_get(&mut self, index: u32) -> Self::Output { 1385 use WasmValType::*; 1386 let context = &mut self.context; 1387 let slot = context.frame.get_wasm_local(index); 1388 match slot.ty { 1389 I32 | I64 | F32 | F64 | V128 => context.stack.push(Val::local(index, slot.ty)), 1390 Ref(rt) => match rt.heap_type { 1391 WasmHeapType::Func => context.stack.push(Val::local(index, slot.ty)), 1392 _ => bail!(CodeGenError::unsupported_wasm_type()), 1393 }, 1394 } 1395 1396 Ok(()) 1397 } 1398 1399 fn visit_local_set(&mut self, index: u32) -> Self::Output { 1400 let src = self.emit_set_local(index)?; 1401 self.context.free_reg(src); 1402 Ok(()) 1403 } 1404 1405 fn visit_call(&mut self, index: u32) -> Self::Output { 1406 let callee = self.env.callee_from_index(FuncIndex::from_u32(index)); 1407 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee)?; 1408 Ok(()) 1409 } 1410 1411 fn visit_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output { 1412 // Spill now because `emit_lazy_init_funcref` and the `FnCall::emit` 1413 // invocations will both trigger spills since they both call functions. 1414 // However, the machine instructions for the spill emitted by 1415 // `emit_lazy_funcref` will be jumped over if the funcref was previously 1416 // initialized which may result in the machine stack becoming 1417 // unbalanced. 1418 self.context.spill(self.masm)?; 1419 1420 let type_index = TypeIndex::from_u32(type_index); 1421 let table_index = TableIndex::from_u32(table_index); 1422 1423 self.emit_lazy_init_funcref(table_index)?; 1424 1425 // Perform the indirect call. 1426 // This code assumes that [`Self::emit_lazy_init_funcref`] will 1427 // push the funcref to the value stack. 1428 let funcref_ptr = self 1429 .context 1430 .stack 1431 .peek() 1432 .map(|v| v.unwrap_reg()) 1433 .ok_or_else(|| CodeGenError::missing_values_in_stack())?; 1434 self.masm 1435 .trapz(funcref_ptr.into(), TRAP_INDIRECT_CALL_TO_NULL)?; 1436 self.emit_typecheck_funcref(funcref_ptr.into(), type_index)?; 1437 1438 let callee = self.env.funcref(type_index); 1439 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee)?; 1440 Ok(()) 1441 } 1442 1443 fn visit_table_init(&mut self, elem: u32, table: u32) -> Self::Output { 1444 let at = self.context.stack.ensure_index_at(3)?; 1445 1446 self.context 1447 .stack 1448 .insert_many(at, &[table.try_into()?, elem.try_into()?]); 1449 1450 let builtin = self.env.builtins.table_init::<M::ABI, M::Ptr>()?; 1451 FnCall::emit::<M>( 1452 &mut self.env, 1453 self.masm, 1454 &mut self.context, 1455 Callee::Builtin(builtin.clone()), 1456 )?; 1457 self.context.pop_and_free(self.masm) 1458 } 1459 1460 fn visit_table_copy(&mut self, dst: u32, src: u32) -> Self::Output { 1461 let at = self.context.stack.ensure_index_at(3)?; 1462 self.context 1463 .stack 1464 .insert_many(at, &[dst.try_into()?, src.try_into()?]); 1465 1466 let builtin = self.env.builtins.table_copy::<M::ABI, M::Ptr>()?; 1467 FnCall::emit::<M>( 1468 &mut self.env, 1469 self.masm, 1470 &mut self.context, 1471 Callee::Builtin(builtin), 1472 )?; 1473 self.context.pop_and_free(self.masm) 1474 } 1475 1476 fn visit_table_get(&mut self, table: u32) -> Self::Output { 1477 let table_index = TableIndex::from_u32(table); 1478 let table = self.env.table(table_index); 1479 let heap_type = table.ref_type.heap_type; 1480 1481 match heap_type { 1482 WasmHeapType::Func => self.emit_lazy_init_funcref(table_index), 1483 _ => Err(anyhow!(CodeGenError::unsupported_wasm_type())), 1484 } 1485 } 1486 1487 fn visit_table_grow(&mut self, table: u32) -> Self::Output { 1488 let table_index = TableIndex::from_u32(table); 1489 let table_ty = self.env.table(table_index); 1490 let builtin = match table_ty.ref_type.heap_type { 1491 WasmHeapType::Func => self.env.builtins.table_grow_func_ref::<M::ABI, M::Ptr>()?, 1492 _ => bail!(CodeGenError::unsupported_wasm_type()), 1493 }; 1494 1495 let len = self.context.stack.len(); 1496 // table.grow` requires at least 2 elements on the value stack. 1497 let at = self.context.stack.ensure_index_at(2)?; 1498 1499 // The table_grow builtin expects the parameters in a different 1500 // order. 1501 // The value stack at this point should contain: 1502 // [ init_value | delta ] (stack top) 1503 // but the builtin function expects the init value as the last 1504 // argument. 1505 self.context.stack.inner_mut().swap(len - 1, len - 2); 1506 self.context.stack.insert_many(at, &[table.try_into()?]); 1507 1508 FnCall::emit::<M>( 1509 &mut self.env, 1510 self.masm, 1511 &mut self.context, 1512 Callee::Builtin(builtin.clone()), 1513 )?; 1514 1515 Ok(()) 1516 } 1517 1518 fn visit_table_size(&mut self, table: u32) -> Self::Output { 1519 let table_index = TableIndex::from_u32(table); 1520 let table_data = self.env.resolve_table_data(table_index); 1521 self.emit_compute_table_size(&table_data) 1522 } 1523 1524 fn visit_table_fill(&mut self, table: u32) -> Self::Output { 1525 let table_index = TableIndex::from_u32(table); 1526 let table_ty = self.env.table(table_index); 1527 1528 ensure!( 1529 table_ty.ref_type.heap_type == WasmHeapType::Func, 1530 CodeGenError::unsupported_wasm_type() 1531 ); 1532 1533 let builtin = self.env.builtins.table_fill_func_ref::<M::ABI, M::Ptr>()?; 1534 1535 let at = self.context.stack.ensure_index_at(3)?; 1536 1537 self.context.stack.insert_many(at, &[table.try_into()?]); 1538 FnCall::emit::<M>( 1539 &mut self.env, 1540 self.masm, 1541 &mut self.context, 1542 Callee::Builtin(builtin.clone()), 1543 )?; 1544 self.context.pop_and_free(self.masm) 1545 } 1546 1547 fn visit_table_set(&mut self, table: u32) -> Self::Output { 1548 let ptr_type = self.env.ptr_type(); 1549 let table_index = TableIndex::from_u32(table); 1550 let table_data = self.env.resolve_table_data(table_index); 1551 let table = self.env.table(table_index); 1552 match table.ref_type.heap_type { 1553 WasmHeapType::Func => { 1554 ensure!( 1555 self.tunables.table_lazy_init, 1556 CodeGenError::unsupported_table_eager_init() 1557 ); 1558 let value = self.context.pop_to_reg(self.masm, None)?; 1559 let index = self.context.pop_to_reg(self.masm, None)?; 1560 let base = self.context.any_gpr(self.masm)?; 1561 let elem_addr = 1562 self.emit_compute_table_elem_addr(index.into(), base, &table_data)?; 1563 // Set the initialized bit. 1564 self.masm.or( 1565 writable!(value.into()), 1566 value.into(), 1567 RegImm::i64(FUNCREF_INIT_BIT as i64), 1568 ptr_type.try_into()?, 1569 )?; 1570 1571 self.masm.store_ptr(value.into(), elem_addr)?; 1572 1573 self.context.free_reg(value); 1574 self.context.free_reg(index); 1575 self.context.free_reg(base); 1576 Ok(()) 1577 } 1578 _ => Err(anyhow!(CodeGenError::unsupported_wasm_type())), 1579 } 1580 } 1581 1582 fn visit_elem_drop(&mut self, index: u32) -> Self::Output { 1583 let elem_drop = self.env.builtins.elem_drop::<M::ABI, M::Ptr>()?; 1584 self.context.stack.extend([index.try_into()?]); 1585 FnCall::emit::<M>( 1586 &mut self.env, 1587 self.masm, 1588 &mut self.context, 1589 Callee::Builtin(elem_drop), 1590 )?; 1591 Ok(()) 1592 } 1593 1594 fn visit_memory_init(&mut self, data_index: u32, mem: u32) -> Self::Output { 1595 let at = self.context.stack.ensure_index_at(3)?; 1596 self.context 1597 .stack 1598 .insert_many(at, &[mem.try_into()?, data_index.try_into()?]); 1599 let builtin = self.env.builtins.memory_init::<M::ABI, M::Ptr>()?; 1600 FnCall::emit::<M>( 1601 &mut self.env, 1602 self.masm, 1603 &mut self.context, 1604 Callee::Builtin(builtin), 1605 )?; 1606 self.context.pop_and_free(self.masm) 1607 } 1608 1609 fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) -> Self::Output { 1610 // At this point, the stack is expected to contain: 1611 // [ dst_offset, src_offset, len ] 1612 // The following code inserts the missing params, so that stack contains: 1613 // [ vmctx, dst_mem, dst_offset, src_mem, src_offset, len ] 1614 // Which is the order expected by the builtin function. 1615 let _ = self.context.stack.ensure_index_at(3)?; 1616 let at = self.context.stack.ensure_index_at(2)?; 1617 self.context.stack.insert_many(at, &[src_mem.try_into()?]); 1618 1619 // One element was inserted above, so instead of 3, we use 4. 1620 let at = self.context.stack.ensure_index_at(4)?; 1621 self.context.stack.insert_many(at, &[dst_mem.try_into()?]); 1622 1623 let builtin = self.env.builtins.memory_copy::<M::ABI, M::Ptr>()?; 1624 1625 FnCall::emit::<M>( 1626 &mut self.env, 1627 self.masm, 1628 &mut self.context, 1629 Callee::Builtin(builtin), 1630 )?; 1631 self.context.pop_and_free(self.masm) 1632 } 1633 1634 fn visit_memory_fill(&mut self, mem: u32) -> Self::Output { 1635 let at = self.context.stack.ensure_index_at(3)?; 1636 1637 self.context.stack.insert_many(at, &[mem.try_into()?]); 1638 1639 let builtin = self.env.builtins.memory_fill::<M::ABI, M::Ptr>()?; 1640 FnCall::emit::<M>( 1641 &mut self.env, 1642 self.masm, 1643 &mut self.context, 1644 Callee::Builtin(builtin), 1645 )?; 1646 self.context.pop_and_free(self.masm) 1647 } 1648 1649 fn visit_memory_size(&mut self, mem: u32) -> Self::Output { 1650 let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem)); 1651 self.emit_compute_memory_size(&heap) 1652 } 1653 1654 fn visit_memory_grow(&mut self, mem: u32) -> Self::Output { 1655 let _ = self.context.stack.ensure_index_at(1)?; 1656 // The stack at this point contains: [ delta ] 1657 // The desired state is 1658 // [ vmctx, delta, index ] 1659 self.context.stack.extend([mem.try_into()?]); 1660 1661 let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem)); 1662 let builtin = self.env.builtins.memory32_grow::<M::ABI, M::Ptr>()?; 1663 FnCall::emit::<M>( 1664 &mut self.env, 1665 self.masm, 1666 &mut self.context, 1667 Callee::Builtin(builtin), 1668 )?; 1669 1670 // The memory32_grow builtin returns a pointer type, therefore we must 1671 // ensure that the return type is representative of the address space of 1672 // the heap type. 1673 match (self.env.ptr_type(), heap.index_type()) { 1674 (WasmValType::I64, WasmValType::I64) => Ok(()), 1675 // When the heap type is smaller than the pointer type, we adjust 1676 // the result of the memory32_grow builtin. 1677 (WasmValType::I64, WasmValType::I32) => { 1678 let top: Reg = self.context.pop_to_reg(self.masm, None)?.into(); 1679 self.masm.wrap(writable!(top.into()), top.into())?; 1680 self.context.stack.push(TypedReg::i32(top).into()); 1681 Ok(()) 1682 } 1683 _ => Err(anyhow!(CodeGenError::unsupported_32_bit_platform())), 1684 } 1685 } 1686 1687 fn visit_data_drop(&mut self, data_index: u32) -> Self::Output { 1688 self.context.stack.extend([data_index.try_into()?]); 1689 1690 let builtin = self.env.builtins.data_drop::<M::ABI, M::Ptr>()?; 1691 FnCall::emit::<M>( 1692 &mut self.env, 1693 self.masm, 1694 &mut self.context, 1695 Callee::Builtin(builtin), 1696 ) 1697 } 1698 1699 fn visit_nop(&mut self) -> Self::Output { 1700 Ok(()) 1701 } 1702 1703 fn visit_if(&mut self, blockty: BlockType) -> Self::Output { 1704 self.control_frames.push(ControlStackFrame::r#if( 1705 self.env.resolve_block_sig(blockty), 1706 self.masm, 1707 &mut self.context, 1708 )?); 1709 1710 Ok(()) 1711 } 1712 1713 fn visit_else(&mut self) -> Self::Output { 1714 if !self.context.reachable { 1715 self.handle_unreachable_else() 1716 } else { 1717 let control = self 1718 .control_frames 1719 .last_mut() 1720 .ok_or_else(|| CodeGenError::control_frame_expected())?; 1721 control.emit_else(self.masm, &mut self.context) 1722 } 1723 } 1724 1725 fn visit_block(&mut self, blockty: BlockType) -> Self::Output { 1726 self.control_frames.push(ControlStackFrame::block( 1727 self.env.resolve_block_sig(blockty), 1728 self.masm, 1729 &mut self.context, 1730 )?); 1731 1732 Ok(()) 1733 } 1734 1735 fn visit_loop(&mut self, blockty: BlockType) -> Self::Output { 1736 self.control_frames.push(ControlStackFrame::r#loop( 1737 self.env.resolve_block_sig(blockty), 1738 self.masm, 1739 &mut self.context, 1740 )?); 1741 1742 self.maybe_emit_epoch_check()?; 1743 self.maybe_emit_fuel_check() 1744 } 1745 1746 fn visit_br(&mut self, depth: u32) -> Self::Output { 1747 let index = control_index(depth, self.control_frames.len())?; 1748 let frame = &mut self.control_frames[index]; 1749 self.context 1750 .unconditional_jump(frame, self.masm, |masm, cx, frame| { 1751 frame.pop_abi_results::<M, _>(cx, masm, |results, _, _| { 1752 Ok(results.ret_area().copied()) 1753 }) 1754 }) 1755 } 1756 1757 fn visit_br_if(&mut self, depth: u32) -> Self::Output { 1758 let index = control_index(depth, self.control_frames.len())?; 1759 let frame = &mut self.control_frames[index]; 1760 frame.set_as_target(); 1761 1762 let top = { 1763 let top = self.context.without::<Result<TypedReg>, M, _>( 1764 frame.results::<M>()?.regs(), 1765 self.masm, 1766 |ctx, masm| ctx.pop_to_reg(masm, None), 1767 )??; 1768 // Explicitly save any live registers and locals before setting up 1769 // the branch state. 1770 // In some cases, calculating the `top` value above, will result in 1771 // a spill, thus the following one will result in a no-op. 1772 self.context.spill(self.masm)?; 1773 frame.top_abi_results::<M, _>( 1774 &mut self.context, 1775 self.masm, 1776 |results, context, masm| { 1777 // In the case of `br_if` there's a possibility that we'll 1778 // exit early from the block or fallthrough, for 1779 // a fallthrough, we cannot rely on the pre-computed return area; 1780 // it must be recalculated so that any values that are 1781 // generated are correctly placed near the current stack 1782 // pointer. 1783 if results.on_stack() { 1784 let stack_consumed = context.stack.sizeof(results.stack_operands_len()); 1785 let base = masm.sp_offset()?.as_u32() - stack_consumed; 1786 let offs = base + results.size(); 1787 Ok(Some(RetArea::sp(SPOffset::from_u32(offs)))) 1788 } else { 1789 Ok(None) 1790 } 1791 }, 1792 )?; 1793 top 1794 }; 1795 1796 // Emit instructions to balance the machine stack if the frame has 1797 // a different offset. 1798 let current_sp_offset = self.masm.sp_offset()?; 1799 let results_size = frame.results::<M>()?.size(); 1800 let state = frame.stack_state(); 1801 let (label, cmp, needs_cleanup) = if current_sp_offset > state.target_offset { 1802 (self.masm.get_label()?, IntCmpKind::Eq, true) 1803 } else { 1804 (*frame.label(), IntCmpKind::Ne, false) 1805 }; 1806 1807 self.masm 1808 .branch(cmp, top.reg.into(), top.reg.into(), label, OperandSize::S32)?; 1809 self.context.free_reg(top); 1810 1811 if needs_cleanup { 1812 // Emit instructions to balance the stack and jump if not falling 1813 // through. 1814 self.masm.memmove( 1815 current_sp_offset, 1816 state.target_offset, 1817 results_size, 1818 MemMoveDirection::LowToHigh, 1819 )?; 1820 self.masm.ensure_sp_for_jump(state.target_offset)?; 1821 self.masm.jmp(*frame.label())?; 1822 1823 // Restore sp_offset to what it was for falling through and emit 1824 // fallthrough label. 1825 self.masm.reset_stack_pointer(current_sp_offset)?; 1826 self.masm.bind(label)?; 1827 } 1828 1829 Ok(()) 1830 } 1831 1832 fn visit_br_table(&mut self, targets: BrTable<'a>) -> Self::Output { 1833 // +1 to account for the default target. 1834 let len = targets.len() + 1; 1835 // SmallVec<[_; 5]> to match the binary emission layer (e.g 1836 // see `JmpTableSeq'), but here we use 5 instead since we 1837 // bundle the default target as the last element in the array. 1838 let mut labels: SmallVec<[_; 5]> = smallvec![]; 1839 for _ in 0..len { 1840 labels.push(self.masm.get_label()?); 1841 } 1842 1843 let default_index = control_index(targets.default(), self.control_frames.len())?; 1844 let default_frame = &mut self.control_frames[default_index]; 1845 let default_result = default_frame.results::<M>()?; 1846 1847 let (index, tmp) = { 1848 let index_and_tmp = self.context.without::<Result<(TypedReg, _)>, M, _>( 1849 default_result.regs(), 1850 self.masm, 1851 |cx, masm| Ok((cx.pop_to_reg(masm, None)?, cx.any_gpr(masm)?)), 1852 )??; 1853 1854 // Materialize any constants or locals into their result representation, 1855 // so that when reachability is restored, they are correctly located. 1856 default_frame.top_abi_results::<M, _>( 1857 &mut self.context, 1858 self.masm, 1859 |results, _, _| Ok(results.ret_area().copied()), 1860 )?; 1861 index_and_tmp 1862 }; 1863 1864 self.masm.jmp_table(&labels, index.into(), tmp)?; 1865 // Save the original stack pointer offset; we will reset the stack 1866 // pointer to this offset after jumping to each of the targets. Each 1867 // jump might adjust the stack according to the base offset of the 1868 // target. 1869 let current_sp = self.masm.sp_offset()?; 1870 1871 for (t, l) in targets 1872 .targets() 1873 .into_iter() 1874 .chain(std::iter::once(Ok(targets.default()))) 1875 .zip(labels.iter()) 1876 { 1877 let control_index = control_index(t?, self.control_frames.len())?; 1878 let frame = &mut self.control_frames[control_index]; 1879 // Reset the stack pointer to its original offset. This is needed 1880 // because each jump will potentially adjust the stack pointer 1881 // according to the base offset of the target. 1882 self.masm.reset_stack_pointer(current_sp)?; 1883 1884 // NB: We don't perform any result handling as it was 1885 // already taken care of above before jumping to the 1886 // jump table. 1887 self.masm.bind(*l)?; 1888 // Ensure that the stack pointer is correctly positioned before 1889 // jumping to the jump table code. 1890 let state = frame.stack_state(); 1891 self.masm.ensure_sp_for_jump(state.target_offset)?; 1892 self.masm.jmp(*frame.label())?; 1893 frame.set_as_target(); 1894 } 1895 // Finally reset the stack pointer to the original location. 1896 // The reachability analysis, will ensure it's correctly located 1897 // once reachability is restored. 1898 self.masm.reset_stack_pointer(current_sp)?; 1899 self.context.reachable = false; 1900 self.context.free_reg(index.reg); 1901 self.context.free_reg(tmp); 1902 1903 Ok(()) 1904 } 1905 1906 fn visit_return(&mut self) -> Self::Output { 1907 // Grab the outermost frame, which is the function's body 1908 // frame. We don't rely on [`codegen::control_index`] since 1909 // this frame is implicit and we know that it should exist at 1910 // index 0. 1911 let outermost = &mut self.control_frames[0]; 1912 self.context 1913 .unconditional_jump(outermost, self.masm, |masm, cx, frame| { 1914 frame.pop_abi_results::<M, _>(cx, masm, |results, _, _| { 1915 Ok(results.ret_area().copied()) 1916 }) 1917 }) 1918 } 1919 1920 fn visit_unreachable(&mut self) -> Self::Output { 1921 self.masm.unreachable()?; 1922 self.context.reachable = false; 1923 // Set the implicit outermost frame as target to perform the necessary 1924 // stack clean up. 1925 let outermost = &mut self.control_frames[0]; 1926 outermost.set_as_target(); 1927 1928 Ok(()) 1929 } 1930 1931 fn visit_local_tee(&mut self, index: u32) -> Self::Output { 1932 let typed_reg = self.emit_set_local(index)?; 1933 self.context.stack.push(typed_reg.into()); 1934 1935 Ok(()) 1936 } 1937 1938 fn visit_global_get(&mut self, global_index: u32) -> Self::Output { 1939 let index = GlobalIndex::from_u32(global_index); 1940 let (ty, base, offset) = self.emit_get_global_addr(index)?; 1941 let addr = self.masm.address_at_reg(base, offset)?; 1942 let dst = self.context.reg_for_type(ty, self.masm)?; 1943 self.masm.load(addr, writable!(dst), ty.try_into()?)?; 1944 self.context.stack.push(Val::reg(dst, ty)); 1945 1946 self.context.free_reg(base); 1947 1948 Ok(()) 1949 } 1950 1951 fn visit_global_set(&mut self, global_index: u32) -> Self::Output { 1952 let index = GlobalIndex::from_u32(global_index); 1953 let (ty, base, offset) = self.emit_get_global_addr(index)?; 1954 let addr = self.masm.address_at_reg(base, offset)?; 1955 1956 let typed_reg = self.context.pop_to_reg(self.masm, None)?; 1957 self.masm 1958 .store(typed_reg.reg.into(), addr, ty.try_into()?)?; 1959 self.context.free_reg(typed_reg.reg); 1960 self.context.free_reg(base); 1961 1962 Ok(()) 1963 } 1964 1965 fn visit_drop(&mut self) -> Self::Output { 1966 self.context.drop_last(1, |regalloc, val| match val { 1967 Val::Reg(tr) => Ok(regalloc.free(tr.reg.into())), 1968 Val::Memory(m) => self.masm.free_stack(m.slot.size), 1969 _ => Ok(()), 1970 }) 1971 } 1972 1973 fn visit_select(&mut self) -> Self::Output { 1974 let cond = self.context.pop_to_reg(self.masm, None)?; 1975 let val2 = self.context.pop_to_reg(self.masm, None)?; 1976 let val1 = self.context.pop_to_reg(self.masm, None)?; 1977 self.masm 1978 .cmp(cond.reg.into(), RegImm::i32(0), OperandSize::S32)?; 1979 // Conditionally move val1 to val2 if the comparison is 1980 // not zero. 1981 self.masm.cmov( 1982 writable!(val2.into()), 1983 val1.into(), 1984 IntCmpKind::Ne, 1985 val1.ty.try_into()?, 1986 )?; 1987 self.context.stack.push(val2.into()); 1988 self.context.free_reg(val1.reg); 1989 self.context.free_reg(cond); 1990 1991 Ok(()) 1992 } 1993 1994 fn visit_i32_load(&mut self, memarg: MemArg) -> Self::Output { 1995 self.emit_wasm_load( 1996 &memarg, 1997 WasmValType::I32, 1998 LoadKind::Operand(OperandSize::S32), 1999 MemOpKind::Normal, 2000 ) 2001 } 2002 2003 fn visit_i32_load8_s(&mut self, memarg: MemArg) -> Self::Output { 2004 self.emit_wasm_load( 2005 &memarg, 2006 WasmValType::I32, 2007 LoadKind::ScalarExtend(Extend::<Signed>::I32Extend8.into()), 2008 MemOpKind::Normal, 2009 ) 2010 } 2011 2012 fn visit_i32_load8_u(&mut self, memarg: MemArg) -> Self::Output { 2013 self.emit_wasm_load( 2014 &memarg, 2015 WasmValType::I32, 2016 LoadKind::ScalarExtend(Extend::<Zero>::I32Extend8.into()), 2017 MemOpKind::Normal, 2018 ) 2019 } 2020 2021 fn visit_i32_load16_s(&mut self, memarg: MemArg) -> Self::Output { 2022 self.emit_wasm_load( 2023 &memarg, 2024 WasmValType::I32, 2025 LoadKind::ScalarExtend(Extend::<Signed>::I32Extend16.into()), 2026 MemOpKind::Normal, 2027 ) 2028 } 2029 2030 fn visit_i32_load16_u(&mut self, memarg: MemArg) -> Self::Output { 2031 self.emit_wasm_load( 2032 &memarg, 2033 WasmValType::I32, 2034 LoadKind::ScalarExtend(Extend::<Zero>::I32Extend16.into()), 2035 MemOpKind::Normal, 2036 ) 2037 } 2038 2039 fn visit_i32_store(&mut self, memarg: MemArg) -> Self::Output { 2040 self.emit_wasm_store(&memarg, OperandSize::S32, MemOpKind::Normal) 2041 } 2042 2043 fn visit_i32_store8(&mut self, memarg: MemArg) -> Self::Output { 2044 self.emit_wasm_store(&memarg, OperandSize::S8, MemOpKind::Normal) 2045 } 2046 2047 fn visit_i32_store16(&mut self, memarg: MemArg) -> Self::Output { 2048 self.emit_wasm_store(&memarg, OperandSize::S16, MemOpKind::Normal) 2049 } 2050 2051 fn visit_i64_load8_s(&mut self, memarg: MemArg) -> Self::Output { 2052 self.emit_wasm_load( 2053 &memarg, 2054 WasmValType::I64, 2055 LoadKind::ScalarExtend(Extend::<Signed>::I64Extend8.into()), 2056 MemOpKind::Normal, 2057 ) 2058 } 2059 2060 fn visit_i64_load8_u(&mut self, memarg: MemArg) -> Self::Output { 2061 self.emit_wasm_load( 2062 &memarg, 2063 WasmValType::I64, 2064 LoadKind::ScalarExtend(Extend::<Zero>::I64Extend8.into()), 2065 MemOpKind::Normal, 2066 ) 2067 } 2068 2069 fn visit_i64_load16_u(&mut self, memarg: MemArg) -> Self::Output { 2070 self.emit_wasm_load( 2071 &memarg, 2072 WasmValType::I64, 2073 LoadKind::ScalarExtend(Extend::<Zero>::I64Extend16.into()), 2074 MemOpKind::Normal, 2075 ) 2076 } 2077 2078 fn visit_i64_load16_s(&mut self, memarg: MemArg) -> Self::Output { 2079 self.emit_wasm_load( 2080 &memarg, 2081 WasmValType::I64, 2082 LoadKind::ScalarExtend(Extend::<Signed>::I64Extend16.into()), 2083 MemOpKind::Normal, 2084 ) 2085 } 2086 2087 fn visit_i64_load32_u(&mut self, memarg: MemArg) -> Self::Output { 2088 self.emit_wasm_load( 2089 &memarg, 2090 WasmValType::I64, 2091 LoadKind::ScalarExtend(Extend::<Zero>::I64Extend32.into()), 2092 MemOpKind::Normal, 2093 ) 2094 } 2095 2096 fn visit_i64_load32_s(&mut self, memarg: MemArg) -> Self::Output { 2097 self.emit_wasm_load( 2098 &memarg, 2099 WasmValType::I64, 2100 LoadKind::ScalarExtend(Extend::<Signed>::I64Extend32.into()), 2101 MemOpKind::Normal, 2102 ) 2103 } 2104 2105 fn visit_i64_load(&mut self, memarg: MemArg) -> Self::Output { 2106 self.emit_wasm_load( 2107 &memarg, 2108 WasmValType::I64, 2109 LoadKind::Operand(OperandSize::S64), 2110 MemOpKind::Normal, 2111 ) 2112 } 2113 2114 fn visit_i64_store(&mut self, memarg: MemArg) -> Self::Output { 2115 self.emit_wasm_store(&memarg, OperandSize::S64, MemOpKind::Normal) 2116 } 2117 2118 fn visit_i64_store8(&mut self, memarg: MemArg) -> Self::Output { 2119 self.emit_wasm_store(&memarg, OperandSize::S8, MemOpKind::Normal) 2120 } 2121 2122 fn visit_i64_store16(&mut self, memarg: MemArg) -> Self::Output { 2123 self.emit_wasm_store(&memarg, OperandSize::S16, MemOpKind::Normal) 2124 } 2125 2126 fn visit_i64_store32(&mut self, memarg: MemArg) -> Self::Output { 2127 self.emit_wasm_store(&memarg, OperandSize::S32, MemOpKind::Normal) 2128 } 2129 2130 fn visit_f32_load(&mut self, memarg: MemArg) -> Self::Output { 2131 self.emit_wasm_load( 2132 &memarg, 2133 WasmValType::F32, 2134 LoadKind::Operand(OperandSize::S32), 2135 MemOpKind::Normal, 2136 ) 2137 } 2138 2139 fn visit_f32_store(&mut self, memarg: MemArg) -> Self::Output { 2140 self.emit_wasm_store(&memarg, OperandSize::S32, MemOpKind::Normal) 2141 } 2142 2143 fn visit_f64_load(&mut self, memarg: MemArg) -> Self::Output { 2144 self.emit_wasm_load( 2145 &memarg, 2146 WasmValType::F64, 2147 LoadKind::Operand(OperandSize::S64), 2148 MemOpKind::Normal, 2149 ) 2150 } 2151 2152 fn visit_f64_store(&mut self, memarg: MemArg) -> Self::Output { 2153 self.emit_wasm_store(&memarg, OperandSize::S64, MemOpKind::Normal) 2154 } 2155 2156 fn visit_i32_trunc_sat_f32_s(&mut self) -> Self::Output { 2157 use OperandSize::*; 2158 2159 self.context 2160 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2161 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Checked) 2162 }) 2163 } 2164 2165 fn visit_i32_trunc_sat_f32_u(&mut self) -> Self::Output { 2166 use OperandSize::*; 2167 2168 self.masm 2169 .unsigned_truncate(&mut self.context, S32, S32, TruncKind::Checked) 2170 } 2171 2172 fn visit_i32_trunc_sat_f64_s(&mut self) -> Self::Output { 2173 use OperandSize::*; 2174 2175 self.context 2176 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2177 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Checked) 2178 }) 2179 } 2180 2181 fn visit_i32_trunc_sat_f64_u(&mut self) -> Self::Output { 2182 use OperandSize::*; 2183 2184 self.masm 2185 .unsigned_truncate(&mut self.context, S64, S32, TruncKind::Checked) 2186 } 2187 2188 fn visit_i64_trunc_sat_f32_s(&mut self) -> Self::Output { 2189 use OperandSize::*; 2190 2191 self.context 2192 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2193 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Checked) 2194 }) 2195 } 2196 2197 fn visit_i64_trunc_sat_f32_u(&mut self) -> Self::Output { 2198 use OperandSize::*; 2199 2200 self.masm 2201 .unsigned_truncate(&mut self.context, S32, S64, TruncKind::Checked) 2202 } 2203 2204 fn visit_i64_trunc_sat_f64_s(&mut self) -> Self::Output { 2205 use OperandSize::*; 2206 2207 self.context 2208 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2209 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Checked) 2210 }) 2211 } 2212 2213 fn visit_i64_trunc_sat_f64_u(&mut self) -> Self::Output { 2214 use OperandSize::*; 2215 2216 self.masm 2217 .unsigned_truncate(&mut self.context, S64, S64, TruncKind::Checked) 2218 } 2219 2220 fn visit_i64_add128(&mut self) -> Self::Output { 2221 self.context 2222 .binop128(self.masm, |masm, lhs_lo, lhs_hi, rhs_lo, rhs_hi| { 2223 masm.add128( 2224 writable!(lhs_lo), 2225 writable!(lhs_hi), 2226 lhs_lo, 2227 lhs_hi, 2228 rhs_lo, 2229 rhs_hi, 2230 )?; 2231 Ok((TypedReg::i64(lhs_lo), TypedReg::i64(lhs_hi))) 2232 }) 2233 } 2234 2235 fn visit_i64_sub128(&mut self) -> Self::Output { 2236 self.context 2237 .binop128(self.masm, |masm, lhs_lo, lhs_hi, rhs_lo, rhs_hi| { 2238 masm.sub128( 2239 writable!(lhs_lo), 2240 writable!(lhs_hi), 2241 lhs_lo, 2242 lhs_hi, 2243 rhs_lo, 2244 rhs_hi, 2245 )?; 2246 Ok((TypedReg::i64(lhs_lo), TypedReg::i64(lhs_hi))) 2247 }) 2248 } 2249 2250 fn visit_i64_mul_wide_s(&mut self) -> Self::Output { 2251 self.masm.mul_wide(&mut self.context, MulWideKind::Signed) 2252 } 2253 2254 fn visit_i64_mul_wide_u(&mut self) -> Self::Output { 2255 self.masm.mul_wide(&mut self.context, MulWideKind::Unsigned) 2256 } 2257 2258 fn visit_i32_atomic_load8_u(&mut self, memarg: MemArg) -> Self::Output { 2259 self.emit_wasm_load( 2260 &memarg, 2261 WasmValType::I32, 2262 LoadKind::ScalarExtend(Extend::<Zero>::I32Extend8.into()), 2263 MemOpKind::Atomic, 2264 ) 2265 } 2266 2267 fn visit_i32_atomic_load16_u(&mut self, memarg: MemArg) -> Self::Output { 2268 self.emit_wasm_load( 2269 &memarg, 2270 WasmValType::I32, 2271 LoadKind::ScalarExtend(Extend::<Zero>::I32Extend16.into()), 2272 MemOpKind::Atomic, 2273 ) 2274 } 2275 2276 fn visit_i32_atomic_load(&mut self, memarg: MemArg) -> Self::Output { 2277 self.emit_wasm_load( 2278 &memarg, 2279 WasmValType::I32, 2280 LoadKind::Operand(OperandSize::S32), 2281 MemOpKind::Atomic, 2282 ) 2283 } 2284 2285 fn visit_i64_atomic_load8_u(&mut self, memarg: MemArg) -> Self::Output { 2286 self.emit_wasm_load( 2287 &memarg, 2288 WasmValType::I64, 2289 LoadKind::ScalarExtend(Extend::<Zero>::I64Extend8.into()), 2290 MemOpKind::Atomic, 2291 ) 2292 } 2293 2294 fn visit_i64_atomic_load16_u(&mut self, memarg: MemArg) -> Self::Output { 2295 self.emit_wasm_load( 2296 &memarg, 2297 WasmValType::I64, 2298 LoadKind::ScalarExtend(Extend::<Zero>::I64Extend16.into()), 2299 MemOpKind::Atomic, 2300 ) 2301 } 2302 2303 fn visit_i64_atomic_load32_u(&mut self, memarg: MemArg) -> Self::Output { 2304 self.emit_wasm_load( 2305 &memarg, 2306 WasmValType::I64, 2307 LoadKind::ScalarExtend(Extend::<Zero>::I64Extend32.into()), 2308 MemOpKind::Atomic, 2309 ) 2310 } 2311 2312 fn visit_i64_atomic_load(&mut self, memarg: MemArg) -> Self::Output { 2313 self.emit_wasm_load( 2314 &memarg, 2315 WasmValType::I64, 2316 LoadKind::Operand(OperandSize::S64), 2317 MemOpKind::Atomic, 2318 ) 2319 } 2320 2321 fn visit_i32_atomic_store(&mut self, memarg: MemArg) -> Self::Output { 2322 self.emit_wasm_store(&memarg, OperandSize::S32, MemOpKind::Atomic) 2323 } 2324 2325 fn visit_i64_atomic_store(&mut self, memarg: MemArg) -> Self::Output { 2326 self.emit_wasm_store(&memarg, OperandSize::S64, MemOpKind::Atomic) 2327 } 2328 2329 fn visit_i32_atomic_store8(&mut self, memarg: MemArg) -> Self::Output { 2330 self.emit_wasm_store(&memarg, OperandSize::S8, MemOpKind::Atomic) 2331 } 2332 2333 fn visit_i32_atomic_store16(&mut self, memarg: MemArg) -> Self::Output { 2334 self.emit_wasm_store(&memarg, OperandSize::S16, MemOpKind::Atomic) 2335 } 2336 2337 fn visit_i64_atomic_store8(&mut self, memarg: MemArg) -> Self::Output { 2338 self.emit_wasm_store(&memarg, OperandSize::S8, MemOpKind::Atomic) 2339 } 2340 2341 fn visit_i64_atomic_store16(&mut self, memarg: MemArg) -> Self::Output { 2342 self.emit_wasm_store(&memarg, OperandSize::S16, MemOpKind::Atomic) 2343 } 2344 2345 fn visit_i64_atomic_store32(&mut self, memarg: MemArg) -> Self::Output { 2346 self.emit_wasm_store(&memarg, OperandSize::S32, MemOpKind::Atomic) 2347 } 2348 2349 fn visit_i32_atomic_rmw8_add_u(&mut self, arg: MemArg) -> Self::Output { 2350 self.emit_atomic_rmw( 2351 &arg, 2352 RmwOp::Add, 2353 OperandSize::S8, 2354 Some(Extend::<Zero>::I32Extend8), 2355 ) 2356 } 2357 2358 fn visit_i32_atomic_rmw16_add_u(&mut self, arg: MemArg) -> Self::Output { 2359 self.emit_atomic_rmw( 2360 &arg, 2361 RmwOp::Add, 2362 OperandSize::S16, 2363 Some(Extend::<Zero>::I32Extend16), 2364 ) 2365 } 2366 2367 fn visit_i32_atomic_rmw_add(&mut self, arg: MemArg) -> Self::Output { 2368 self.emit_atomic_rmw(&arg, RmwOp::Add, OperandSize::S32, None) 2369 } 2370 2371 fn visit_i64_atomic_rmw8_add_u(&mut self, arg: MemArg) -> Self::Output { 2372 self.emit_atomic_rmw( 2373 &arg, 2374 RmwOp::Add, 2375 OperandSize::S8, 2376 Some(Extend::<Zero>::I64Extend8), 2377 ) 2378 } 2379 2380 fn visit_i64_atomic_rmw16_add_u(&mut self, arg: MemArg) -> Self::Output { 2381 self.emit_atomic_rmw( 2382 &arg, 2383 RmwOp::Add, 2384 OperandSize::S16, 2385 Some(Extend::<Zero>::I64Extend16), 2386 ) 2387 } 2388 2389 fn visit_i64_atomic_rmw32_add_u(&mut self, arg: MemArg) -> Self::Output { 2390 self.emit_atomic_rmw( 2391 &arg, 2392 RmwOp::Add, 2393 OperandSize::S32, 2394 Some(Extend::<Zero>::I64Extend32), 2395 ) 2396 } 2397 2398 fn visit_i64_atomic_rmw_add(&mut self, arg: MemArg) -> Self::Output { 2399 self.emit_atomic_rmw(&arg, RmwOp::Add, OperandSize::S64, None) 2400 } 2401 2402 fn visit_i32_atomic_rmw8_sub_u(&mut self, arg: MemArg) -> Self::Output { 2403 self.emit_atomic_rmw( 2404 &arg, 2405 RmwOp::Sub, 2406 OperandSize::S8, 2407 Some(Extend::<Zero>::I32Extend8), 2408 ) 2409 } 2410 fn visit_i32_atomic_rmw16_sub_u(&mut self, arg: MemArg) -> Self::Output { 2411 self.emit_atomic_rmw( 2412 &arg, 2413 RmwOp::Sub, 2414 OperandSize::S16, 2415 Some(Extend::<Zero>::I32Extend16), 2416 ) 2417 } 2418 2419 fn visit_i32_atomic_rmw_sub(&mut self, arg: MemArg) -> Self::Output { 2420 self.emit_atomic_rmw(&arg, RmwOp::Sub, OperandSize::S32, None) 2421 } 2422 2423 fn visit_i64_atomic_rmw8_sub_u(&mut self, arg: MemArg) -> Self::Output { 2424 self.emit_atomic_rmw( 2425 &arg, 2426 RmwOp::Sub, 2427 OperandSize::S8, 2428 Some(Extend::<Zero>::I64Extend8), 2429 ) 2430 } 2431 2432 fn visit_i64_atomic_rmw16_sub_u(&mut self, arg: MemArg) -> Self::Output { 2433 self.emit_atomic_rmw( 2434 &arg, 2435 RmwOp::Sub, 2436 OperandSize::S16, 2437 Some(Extend::<Zero>::I64Extend16), 2438 ) 2439 } 2440 2441 fn visit_i64_atomic_rmw32_sub_u(&mut self, arg: MemArg) -> Self::Output { 2442 self.emit_atomic_rmw( 2443 &arg, 2444 RmwOp::Sub, 2445 OperandSize::S32, 2446 Some(Extend::<Zero>::I64Extend32), 2447 ) 2448 } 2449 2450 fn visit_i64_atomic_rmw_sub(&mut self, arg: MemArg) -> Self::Output { 2451 self.emit_atomic_rmw(&arg, RmwOp::Sub, OperandSize::S64, None) 2452 } 2453 2454 fn visit_i32_atomic_rmw8_xchg_u(&mut self, arg: MemArg) -> Self::Output { 2455 self.emit_atomic_rmw( 2456 &arg, 2457 RmwOp::Xchg, 2458 OperandSize::S8, 2459 Some(Extend::<Zero>::I32Extend8), 2460 ) 2461 } 2462 2463 fn visit_i32_atomic_rmw16_xchg_u(&mut self, arg: MemArg) -> Self::Output { 2464 self.emit_atomic_rmw( 2465 &arg, 2466 RmwOp::Xchg, 2467 OperandSize::S16, 2468 Some(Extend::<Zero>::I32Extend16), 2469 ) 2470 } 2471 2472 fn visit_i32_atomic_rmw_xchg(&mut self, arg: MemArg) -> Self::Output { 2473 self.emit_atomic_rmw(&arg, RmwOp::Xchg, OperandSize::S32, None) 2474 } 2475 2476 fn visit_i64_atomic_rmw8_xchg_u(&mut self, arg: MemArg) -> Self::Output { 2477 self.emit_atomic_rmw( 2478 &arg, 2479 RmwOp::Xchg, 2480 OperandSize::S8, 2481 Some(Extend::<Zero>::I64Extend8), 2482 ) 2483 } 2484 2485 fn visit_i64_atomic_rmw16_xchg_u(&mut self, arg: MemArg) -> Self::Output { 2486 self.emit_atomic_rmw( 2487 &arg, 2488 RmwOp::Xchg, 2489 OperandSize::S16, 2490 Some(Extend::<Zero>::I64Extend16), 2491 ) 2492 } 2493 2494 fn visit_i64_atomic_rmw32_xchg_u(&mut self, arg: MemArg) -> Self::Output { 2495 self.emit_atomic_rmw( 2496 &arg, 2497 RmwOp::Xchg, 2498 OperandSize::S32, 2499 Some(Extend::<Zero>::I64Extend32), 2500 ) 2501 } 2502 2503 fn visit_i64_atomic_rmw_xchg(&mut self, arg: MemArg) -> Self::Output { 2504 self.emit_atomic_rmw(&arg, RmwOp::Xchg, OperandSize::S64, None) 2505 } 2506 2507 fn visit_i32_atomic_rmw8_and_u(&mut self, arg: MemArg) -> Self::Output { 2508 self.emit_atomic_rmw( 2509 &arg, 2510 RmwOp::And, 2511 OperandSize::S8, 2512 Some(Extend::<Zero>::I32Extend8), 2513 ) 2514 } 2515 2516 fn visit_i32_atomic_rmw16_and_u(&mut self, arg: MemArg) -> Self::Output { 2517 self.emit_atomic_rmw( 2518 &arg, 2519 RmwOp::And, 2520 OperandSize::S16, 2521 Some(Extend::<Zero>::I32Extend16), 2522 ) 2523 } 2524 2525 fn visit_i32_atomic_rmw_and(&mut self, arg: MemArg) -> Self::Output { 2526 self.emit_atomic_rmw(&arg, RmwOp::And, OperandSize::S32, None) 2527 } 2528 2529 fn visit_i64_atomic_rmw8_and_u(&mut self, arg: MemArg) -> Self::Output { 2530 self.emit_atomic_rmw( 2531 &arg, 2532 RmwOp::And, 2533 OperandSize::S8, 2534 Some(Extend::<Zero>::I64Extend8), 2535 ) 2536 } 2537 2538 fn visit_i64_atomic_rmw16_and_u(&mut self, arg: MemArg) -> Self::Output { 2539 self.emit_atomic_rmw( 2540 &arg, 2541 RmwOp::And, 2542 OperandSize::S16, 2543 Some(Extend::<Zero>::I64Extend16), 2544 ) 2545 } 2546 2547 fn visit_i64_atomic_rmw32_and_u(&mut self, arg: MemArg) -> Self::Output { 2548 self.emit_atomic_rmw( 2549 &arg, 2550 RmwOp::And, 2551 OperandSize::S32, 2552 Some(Extend::<Zero>::I64Extend32), 2553 ) 2554 } 2555 2556 fn visit_i64_atomic_rmw_and(&mut self, arg: MemArg) -> Self::Output { 2557 self.emit_atomic_rmw(&arg, RmwOp::And, OperandSize::S64, None) 2558 } 2559 2560 fn visit_i32_atomic_rmw8_or_u(&mut self, arg: MemArg) -> Self::Output { 2561 self.emit_atomic_rmw( 2562 &arg, 2563 RmwOp::Or, 2564 OperandSize::S8, 2565 Some(Extend::<Zero>::I32Extend8), 2566 ) 2567 } 2568 2569 fn visit_i32_atomic_rmw16_or_u(&mut self, arg: MemArg) -> Self::Output { 2570 self.emit_atomic_rmw( 2571 &arg, 2572 RmwOp::Or, 2573 OperandSize::S16, 2574 Some(Extend::<Zero>::I32Extend16), 2575 ) 2576 } 2577 2578 fn visit_i32_atomic_rmw_or(&mut self, arg: MemArg) -> Self::Output { 2579 self.emit_atomic_rmw(&arg, RmwOp::Or, OperandSize::S32, None) 2580 } 2581 2582 fn visit_i64_atomic_rmw8_or_u(&mut self, arg: MemArg) -> Self::Output { 2583 self.emit_atomic_rmw( 2584 &arg, 2585 RmwOp::Or, 2586 OperandSize::S8, 2587 Some(Extend::<Zero>::I64Extend8), 2588 ) 2589 } 2590 2591 fn visit_i64_atomic_rmw16_or_u(&mut self, arg: MemArg) -> Self::Output { 2592 self.emit_atomic_rmw( 2593 &arg, 2594 RmwOp::Or, 2595 OperandSize::S16, 2596 Some(Extend::<Zero>::I64Extend16), 2597 ) 2598 } 2599 2600 fn visit_i64_atomic_rmw32_or_u(&mut self, arg: MemArg) -> Self::Output { 2601 self.emit_atomic_rmw( 2602 &arg, 2603 RmwOp::Or, 2604 OperandSize::S32, 2605 Some(Extend::<Zero>::I64Extend32), 2606 ) 2607 } 2608 2609 fn visit_i64_atomic_rmw_or(&mut self, arg: MemArg) -> Self::Output { 2610 self.emit_atomic_rmw(&arg, RmwOp::Or, OperandSize::S64, None) 2611 } 2612 2613 fn visit_i32_atomic_rmw8_xor_u(&mut self, arg: MemArg) -> Self::Output { 2614 self.emit_atomic_rmw( 2615 &arg, 2616 RmwOp::Xor, 2617 OperandSize::S8, 2618 Some(Extend::<Zero>::I32Extend8), 2619 ) 2620 } 2621 2622 fn visit_i32_atomic_rmw16_xor_u(&mut self, arg: MemArg) -> Self::Output { 2623 self.emit_atomic_rmw( 2624 &arg, 2625 RmwOp::Xor, 2626 OperandSize::S16, 2627 Some(Extend::<Zero>::I32Extend16), 2628 ) 2629 } 2630 2631 fn visit_i32_atomic_rmw_xor(&mut self, arg: MemArg) -> Self::Output { 2632 self.emit_atomic_rmw(&arg, RmwOp::Xor, OperandSize::S32, None) 2633 } 2634 2635 fn visit_i64_atomic_rmw8_xor_u(&mut self, arg: MemArg) -> Self::Output { 2636 self.emit_atomic_rmw( 2637 &arg, 2638 RmwOp::Xor, 2639 OperandSize::S8, 2640 Some(Extend::<Zero>::I64Extend8), 2641 ) 2642 } 2643 2644 fn visit_i64_atomic_rmw16_xor_u(&mut self, arg: MemArg) -> Self::Output { 2645 self.emit_atomic_rmw( 2646 &arg, 2647 RmwOp::Xor, 2648 OperandSize::S16, 2649 Some(Extend::<Zero>::I64Extend16), 2650 ) 2651 } 2652 2653 fn visit_i64_atomic_rmw32_xor_u(&mut self, arg: MemArg) -> Self::Output { 2654 self.emit_atomic_rmw( 2655 &arg, 2656 RmwOp::Xor, 2657 OperandSize::S32, 2658 Some(Extend::<Zero>::I64Extend32), 2659 ) 2660 } 2661 2662 fn visit_i64_atomic_rmw_xor(&mut self, arg: MemArg) -> Self::Output { 2663 self.emit_atomic_rmw(&arg, RmwOp::Xor, OperandSize::S64, None) 2664 } 2665 2666 fn visit_i32_atomic_rmw8_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output { 2667 self.emit_atomic_cmpxchg(&arg, OperandSize::S8, Some(Extend::I32Extend8)) 2668 } 2669 2670 fn visit_i32_atomic_rmw16_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output { 2671 self.emit_atomic_cmpxchg(&arg, OperandSize::S16, Some(Extend::I32Extend16)) 2672 } 2673 2674 fn visit_i32_atomic_rmw_cmpxchg(&mut self, arg: MemArg) -> Self::Output { 2675 self.emit_atomic_cmpxchg(&arg, OperandSize::S32, None) 2676 } 2677 2678 fn visit_i64_atomic_rmw8_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output { 2679 self.emit_atomic_cmpxchg(&arg, OperandSize::S8, Some(Extend::I64Extend8)) 2680 } 2681 2682 fn visit_i64_atomic_rmw16_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output { 2683 self.emit_atomic_cmpxchg(&arg, OperandSize::S16, Some(Extend::I64Extend16)) 2684 } 2685 2686 fn visit_i64_atomic_rmw32_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output { 2687 self.emit_atomic_cmpxchg(&arg, OperandSize::S32, Some(Extend::I64Extend32)) 2688 } 2689 2690 fn visit_i64_atomic_rmw_cmpxchg(&mut self, arg: MemArg) -> Self::Output { 2691 self.emit_atomic_cmpxchg(&arg, OperandSize::S64, None) 2692 } 2693 2694 wasmparser::for_each_visit_operator!(def_unsupported); 2695 } 2696 2697 impl<'a, 'translation, 'data, M> VisitSimdOperator<'a> 2698 for CodeGen<'a, 'translation, 'data, M, Emission> 2699 where 2700 M: MacroAssembler, 2701 { 2702 fn visit_v128_const(&mut self, val: V128) -> Self::Output { 2703 self.context.stack.push(Val::v128(val.i128())); 2704 Ok(()) 2705 } 2706 2707 fn visit_v128_load(&mut self, memarg: MemArg) -> Self::Output { 2708 self.emit_wasm_load( 2709 &memarg, 2710 WasmValType::V128, 2711 LoadKind::Operand(OperandSize::S128), 2712 MemOpKind::Normal, 2713 ) 2714 } 2715 2716 fn visit_v128_store(&mut self, memarg: MemArg) -> Self::Output { 2717 self.emit_wasm_store(&memarg, OperandSize::S128, MemOpKind::Normal) 2718 } 2719 2720 fn visit_v128_load8x8_s(&mut self, memarg: MemArg) -> Self::Output { 2721 self.emit_wasm_load( 2722 &memarg, 2723 WasmValType::V128, 2724 LoadKind::VectorExtend(VectorExtendKind::V128Extend8x8S), 2725 MemOpKind::Normal, 2726 ) 2727 } 2728 2729 fn visit_v128_load8x8_u(&mut self, memarg: MemArg) -> Self::Output { 2730 self.emit_wasm_load( 2731 &memarg, 2732 WasmValType::V128, 2733 LoadKind::VectorExtend(VectorExtendKind::V128Extend8x8U), 2734 MemOpKind::Normal, 2735 ) 2736 } 2737 2738 fn visit_v128_load16x4_s(&mut self, memarg: MemArg) -> Self::Output { 2739 self.emit_wasm_load( 2740 &memarg, 2741 WasmValType::V128, 2742 LoadKind::VectorExtend(VectorExtendKind::V128Extend16x4S), 2743 MemOpKind::Normal, 2744 ) 2745 } 2746 2747 fn visit_v128_load16x4_u(&mut self, memarg: MemArg) -> Self::Output { 2748 self.emit_wasm_load( 2749 &memarg, 2750 WasmValType::V128, 2751 LoadKind::VectorExtend(VectorExtendKind::V128Extend16x4U), 2752 MemOpKind::Normal, 2753 ) 2754 } 2755 2756 fn visit_v128_load32x2_s(&mut self, memarg: MemArg) -> Self::Output { 2757 self.emit_wasm_load( 2758 &memarg, 2759 WasmValType::V128, 2760 LoadKind::VectorExtend(VectorExtendKind::V128Extend32x2S), 2761 MemOpKind::Normal, 2762 ) 2763 } 2764 2765 fn visit_v128_load32x2_u(&mut self, memarg: MemArg) -> Self::Output { 2766 self.emit_wasm_load( 2767 &memarg, 2768 WasmValType::V128, 2769 LoadKind::VectorExtend(VectorExtendKind::V128Extend32x2U), 2770 MemOpKind::Normal, 2771 ) 2772 } 2773 2774 fn visit_v128_load8_splat(&mut self, memarg: MemArg) -> Self::Output { 2775 self.emit_wasm_load( 2776 &memarg, 2777 WasmValType::V128, 2778 LoadKind::Splat(SplatLoadKind::S8), 2779 MemOpKind::Normal, 2780 ) 2781 } 2782 2783 fn visit_v128_load16_splat(&mut self, memarg: MemArg) -> Self::Output { 2784 self.emit_wasm_load( 2785 &memarg, 2786 WasmValType::V128, 2787 LoadKind::Splat(SplatLoadKind::S16), 2788 MemOpKind::Normal, 2789 ) 2790 } 2791 2792 fn visit_v128_load32_splat(&mut self, memarg: MemArg) -> Self::Output { 2793 self.emit_wasm_load( 2794 &memarg, 2795 WasmValType::V128, 2796 LoadKind::Splat(SplatLoadKind::S32), 2797 MemOpKind::Normal, 2798 ) 2799 } 2800 2801 fn visit_v128_load64_splat(&mut self, memarg: MemArg) -> Self::Output { 2802 self.emit_wasm_load( 2803 &memarg, 2804 WasmValType::V128, 2805 LoadKind::Splat(SplatLoadKind::S64), 2806 MemOpKind::Normal, 2807 ) 2808 } 2809 2810 fn visit_i8x16_splat(&mut self) -> Self::Output { 2811 self.masm.splat(&mut self.context, SplatKind::I8x16) 2812 } 2813 2814 fn visit_i16x8_splat(&mut self) -> Self::Output { 2815 self.masm.splat(&mut self.context, SplatKind::I16x8) 2816 } 2817 2818 fn visit_i32x4_splat(&mut self) -> Self::Output { 2819 self.masm.splat(&mut self.context, SplatKind::I32x4) 2820 } 2821 2822 fn visit_i64x2_splat(&mut self) -> Self::Output { 2823 self.masm.splat(&mut self.context, SplatKind::I64x2) 2824 } 2825 2826 fn visit_f32x4_splat(&mut self) -> Self::Output { 2827 self.masm.splat(&mut self.context, SplatKind::F32x4) 2828 } 2829 2830 fn visit_f64x2_splat(&mut self) -> Self::Output { 2831 self.masm.splat(&mut self.context, SplatKind::F64x2) 2832 } 2833 2834 fn visit_i8x16_shuffle(&mut self, lanes: [u8; 16]) -> Self::Output { 2835 let rhs = self.context.pop_to_reg(self.masm, None)?; 2836 let lhs = self.context.pop_to_reg(self.masm, None)?; 2837 self.masm 2838 .shuffle(writable!(lhs.into()), lhs.into(), rhs.into(), lanes)?; 2839 self.context.stack.push(TypedReg::v128(lhs.into()).into()); 2840 self.context.free_reg(rhs); 2841 Ok(()) 2842 } 2843 2844 fn visit_i8x16_swizzle(&mut self) -> Self::Output { 2845 let rhs = self.context.pop_to_reg(self.masm, None)?; 2846 let lhs = self.context.pop_to_reg(self.masm, None)?; 2847 self.masm 2848 .swizzle(writable!(lhs.into()), lhs.into(), rhs.into())?; 2849 self.context.stack.push(TypedReg::v128(lhs.into()).into()); 2850 self.context.free_reg(rhs); 2851 Ok(()) 2852 } 2853 2854 fn visit_i8x16_extract_lane_s(&mut self, lane: u8) -> Self::Output { 2855 self.context.extract_lane_op( 2856 self.masm, 2857 ExtractLaneKind::I8x16S, 2858 |masm, src, dst, kind| masm.extract_lane(src, dst, lane, kind), 2859 ) 2860 } 2861 2862 fn visit_i8x16_extract_lane_u(&mut self, lane: u8) -> Self::Output { 2863 self.context.extract_lane_op( 2864 self.masm, 2865 ExtractLaneKind::I8x16U, 2866 |masm, src, dst, kind| masm.extract_lane(src, dst, lane, kind), 2867 ) 2868 } 2869 2870 fn visit_i16x8_extract_lane_s(&mut self, lane: u8) -> Self::Output { 2871 self.context.extract_lane_op( 2872 self.masm, 2873 ExtractLaneKind::I16x8S, 2874 |masm, src, dst, kind| masm.extract_lane(src, dst, lane, kind), 2875 ) 2876 } 2877 2878 fn visit_i16x8_extract_lane_u(&mut self, lane: u8) -> Self::Output { 2879 self.context.extract_lane_op( 2880 self.masm, 2881 ExtractLaneKind::I16x8U, 2882 |masm, src, dst, kind| masm.extract_lane(src, dst, lane, kind), 2883 ) 2884 } 2885 2886 fn visit_i32x4_extract_lane(&mut self, lane: u8) -> Self::Output { 2887 self.context 2888 .extract_lane_op(self.masm, ExtractLaneKind::I32x4, |masm, src, dst, kind| { 2889 masm.extract_lane(src, dst, lane, kind) 2890 }) 2891 } 2892 2893 fn visit_i64x2_extract_lane(&mut self, lane: u8) -> Self::Output { 2894 self.context 2895 .extract_lane_op(self.masm, ExtractLaneKind::I64x2, |masm, src, dst, kind| { 2896 masm.extract_lane(src, dst, lane, kind) 2897 }) 2898 } 2899 2900 fn visit_f32x4_extract_lane(&mut self, lane: u8) -> Self::Output { 2901 self.context 2902 .extract_lane_op(self.masm, ExtractLaneKind::F32x4, |masm, src, dst, kind| { 2903 masm.extract_lane(src, dst, lane, kind) 2904 }) 2905 } 2906 2907 fn visit_f64x2_extract_lane(&mut self, lane: u8) -> Self::Output { 2908 self.context 2909 .extract_lane_op(self.masm, ExtractLaneKind::F64x2, |masm, src, dst, kind| { 2910 masm.extract_lane(src, dst, lane, kind) 2911 }) 2912 } 2913 2914 wasmparser::for_each_visit_simd_operator!(def_unsupported); 2915 } 2916 2917 impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Emission> 2918 where 2919 M: MacroAssembler, 2920 { 2921 fn cmp_i32s(&mut self, kind: IntCmpKind) -> Result<()> { 2922 self.context.i32_binop(self.masm, |masm, dst, src, size| { 2923 masm.cmp_with_set(writable!(dst), src, kind, size)?; 2924 Ok(TypedReg::i32(dst)) 2925 }) 2926 } 2927 2928 fn cmp_i64s(&mut self, kind: IntCmpKind) -> Result<()> { 2929 self.context 2930 .i64_binop(self.masm, move |masm, dst, src, size| { 2931 masm.cmp_with_set(writable!(dst), src, kind, size)?; 2932 Ok(TypedReg::i32(dst)) // Return value for comparisons is an `i32`. 2933 }) 2934 } 2935 } 2936 2937 impl TryFrom<WasmValType> for OperandSize { 2938 type Error = anyhow::Error; 2939 fn try_from(ty: WasmValType) -> Result<OperandSize> { 2940 let ty = match ty { 2941 WasmValType::I32 | WasmValType::F32 => OperandSize::S32, 2942 WasmValType::I64 | WasmValType::F64 => OperandSize::S64, 2943 WasmValType::V128 => OperandSize::S128, 2944 WasmValType::Ref(rt) => { 2945 match rt.heap_type { 2946 // TODO: Hardcoded size, assuming 64-bit support only. Once 2947 // Wasmtime supports 32-bit architectures, this will need 2948 // to be updated in such a way that the calculation of the 2949 // OperandSize will depend on the target's pointer size. 2950 WasmHeapType::Func => OperandSize::S64, 2951 WasmHeapType::Extern => OperandSize::S64, 2952 _ => bail!(CodeGenError::unsupported_wasm_type()), 2953 } 2954 } 2955 }; 2956 Ok(ty) 2957 } 2958 } 2959