1 //! This module is the central place for machine code emission. 2 //! It defines an implementation of wasmparser's Visitor trait 3 //! for `CodeGen`; which defines a visitor per op-code, 4 //! which validates and dispatches to the corresponding 5 //! machine code emitter. 6 7 use crate::abi::RetArea; 8 use crate::codegen::{ 9 Callee, CodeGen, CodeGenError, ConditionalBranch, ControlStackFrame, Emission, FnCall, 10 UnconditionalBranch, control_index, 11 }; 12 use crate::masm::{ 13 AtomicWaitKind, DivKind, Extend, ExtractLaneKind, FloatCmpKind, IntCmpKind, LoadKind, 14 MacroAssembler, MulWideKind, OperandSize, RegImm, RemKind, ReplaceLaneKind, RmwOp, 15 RoundingMode, SPOffset, ShiftKind, Signed, SplatKind, SplatLoadKind, StoreKind, TruncKind, 16 V128AbsKind, V128AddKind, V128ConvertKind, V128ExtAddKind, V128ExtMulKind, V128ExtendKind, 17 V128LoadExtendKind, V128MaxKind, V128MinKind, V128MulKind, V128NarrowKind, V128NegKind, 18 V128SubKind, V128TruncKind, VectorCompareKind, VectorEqualityKind, Zero, 19 }; 20 use crate::reg::{Reg, writable}; 21 use crate::stack::{TypedReg, Val}; 22 use crate::{Result, bail, ensure, format_err}; 23 use regalloc2::RegClass; 24 use smallvec::{SmallVec, smallvec}; 25 use wasmparser::{ 26 BlockType, BrTable, Ieee32, Ieee64, MemArg, V128, VisitOperator, VisitSimdOperator, 27 }; 28 use wasmtime_cranelift::TRAP_INDIRECT_CALL_TO_NULL; 29 use wasmtime_environ::{ 30 FUNCREF_INIT_BIT, FuncIndex, GlobalIndex, MemoryIndex, TableIndex, TypeIndex, WasmHeapType, 31 WasmValType, 32 }; 33 34 /// A macro to define unsupported WebAssembly operators. 35 /// 36 /// This macro calls itself recursively; 37 /// 1. It no-ops when matching a supported operator. 38 /// 2. Defines the visitor function and panics when 39 /// matching an unsupported operator. 40 macro_rules! def_unsupported { 41 ($( @$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident $ann:tt)*) => { 42 $( 43 def_unsupported!( 44 emit 45 $op 46 47 fn $visit(&mut self $($(,$arg: $argty)*)?) -> Self::Output { 48 $($(let _ = $arg;)*)? 49 50 Err(format_err!(CodeGenError::unimplemented_wasm_instruction())) 51 } 52 ); 53 )* 54 }; 55 56 (emit I32Const $($rest:tt)*) => {}; 57 (emit I64Const $($rest:tt)*) => {}; 58 (emit F32Const $($rest:tt)*) => {}; 59 (emit F64Const $($rest:tt)*) => {}; 60 (emit V128Const $($rest:tt)*) => {}; 61 (emit F32Add $($rest:tt)*) => {}; 62 (emit F64Add $($rest:tt)*) => {}; 63 (emit F32Sub $($rest:tt)*) => {}; 64 (emit F64Sub $($rest:tt)*) => {}; 65 (emit F32Mul $($rest:tt)*) => {}; 66 (emit F64Mul $($rest:tt)*) => {}; 67 (emit F32Div $($rest:tt)*) => {}; 68 (emit F64Div $($rest:tt)*) => {}; 69 (emit F32Min $($rest:tt)*) => {}; 70 (emit F64Min $($rest:tt)*) => {}; 71 (emit F32Max $($rest:tt)*) => {}; 72 (emit F64Max $($rest:tt)*) => {}; 73 (emit F32Copysign $($rest:tt)*) => {}; 74 (emit F64Copysign $($rest:tt)*) => {}; 75 (emit F32Abs $($rest:tt)*) => {}; 76 (emit F64Abs $($rest:tt)*) => {}; 77 (emit F32Neg $($rest:tt)*) => {}; 78 (emit F64Neg $($rest:tt)*) => {}; 79 (emit F32Floor $($rest:tt)*) => {}; 80 (emit F64Floor $($rest:tt)*) => {}; 81 (emit F32Ceil $($rest:tt)*) => {}; 82 (emit F64Ceil $($rest:tt)*) => {}; 83 (emit F32Nearest $($rest:tt)*) => {}; 84 (emit F64Nearest $($rest:tt)*) => {}; 85 (emit F32Trunc $($rest:tt)*) => {}; 86 (emit F64Trunc $($rest:tt)*) => {}; 87 (emit F32Sqrt $($rest:tt)*) => {}; 88 (emit F64Sqrt $($rest:tt)*) => {}; 89 (emit F32Eq $($rest:tt)*) => {}; 90 (emit F64Eq $($rest:tt)*) => {}; 91 (emit F32Ne $($rest:tt)*) => {}; 92 (emit F64Ne $($rest:tt)*) => {}; 93 (emit F32Lt $($rest:tt)*) => {}; 94 (emit F64Lt $($rest:tt)*) => {}; 95 (emit F32Gt $($rest:tt)*) => {}; 96 (emit F64Gt $($rest:tt)*) => {}; 97 (emit F32Le $($rest:tt)*) => {}; 98 (emit F64Le $($rest:tt)*) => {}; 99 (emit F32Ge $($rest:tt)*) => {}; 100 (emit F64Ge $($rest:tt)*) => {}; 101 (emit F32ConvertI32S $($rest:tt)*) => {}; 102 (emit F32ConvertI32U $($rest:tt)*) => {}; 103 (emit F32ConvertI64S $($rest:tt)*) => {}; 104 (emit F32ConvertI64U $($rest:tt)*) => {}; 105 (emit F64ConvertI32S $($rest:tt)*) => {}; 106 (emit F64ConvertI32U $($rest:tt)*) => {}; 107 (emit F64ConvertI64S $($rest:tt)*) => {}; 108 (emit F64ConvertI64U $($rest:tt)*) => {}; 109 (emit F32ReinterpretI32 $($rest:tt)*) => {}; 110 (emit F64ReinterpretI64 $($rest:tt)*) => {}; 111 (emit F32DemoteF64 $($rest:tt)*) => {}; 112 (emit F64PromoteF32 $($rest:tt)*) => {}; 113 (emit I32Add $($rest:tt)*) => {}; 114 (emit I64Add $($rest:tt)*) => {}; 115 (emit I32Sub $($rest:tt)*) => {}; 116 (emit I32Mul $($rest:tt)*) => {}; 117 (emit I32DivS $($rest:tt)*) => {}; 118 (emit I32DivU $($rest:tt)*) => {}; 119 (emit I64DivS $($rest:tt)*) => {}; 120 (emit I64DivU $($rest:tt)*) => {}; 121 (emit I64RemU $($rest:tt)*) => {}; 122 (emit I64RemS $($rest:tt)*) => {}; 123 (emit I32RemU $($rest:tt)*) => {}; 124 (emit I32RemS $($rest:tt)*) => {}; 125 (emit I64Mul $($rest:tt)*) => {}; 126 (emit I64Sub $($rest:tt)*) => {}; 127 (emit I32Eq $($rest:tt)*) => {}; 128 (emit I64Eq $($rest:tt)*) => {}; 129 (emit I32Ne $($rest:tt)*) => {}; 130 (emit I64Ne $($rest:tt)*) => {}; 131 (emit I32LtS $($rest:tt)*) => {}; 132 (emit I64LtS $($rest:tt)*) => {}; 133 (emit I32LtU $($rest:tt)*) => {}; 134 (emit I64LtU $($rest:tt)*) => {}; 135 (emit I32LeS $($rest:tt)*) => {}; 136 (emit I64LeS $($rest:tt)*) => {}; 137 (emit I32LeU $($rest:tt)*) => {}; 138 (emit I64LeU $($rest:tt)*) => {}; 139 (emit I32GtS $($rest:tt)*) => {}; 140 (emit I64GtS $($rest:tt)*) => {}; 141 (emit I32GtU $($rest:tt)*) => {}; 142 (emit I64GtU $($rest:tt)*) => {}; 143 (emit I32GeS $($rest:tt)*) => {}; 144 (emit I64GeS $($rest:tt)*) => {}; 145 (emit I32GeU $($rest:tt)*) => {}; 146 (emit I64GeU $($rest:tt)*) => {}; 147 (emit I32Eqz $($rest:tt)*) => {}; 148 (emit I64Eqz $($rest:tt)*) => {}; 149 (emit I32And $($rest:tt)*) => {}; 150 (emit I64And $($rest:tt)*) => {}; 151 (emit I32Or $($rest:tt)*) => {}; 152 (emit I64Or $($rest:tt)*) => {}; 153 (emit I32Xor $($rest:tt)*) => {}; 154 (emit I64Xor $($rest:tt)*) => {}; 155 (emit I32Shl $($rest:tt)*) => {}; 156 (emit I64Shl $($rest:tt)*) => {}; 157 (emit I32ShrS $($rest:tt)*) => {}; 158 (emit I64ShrS $($rest:tt)*) => {}; 159 (emit I32ShrU $($rest:tt)*) => {}; 160 (emit I64ShrU $($rest:tt)*) => {}; 161 (emit I32Rotl $($rest:tt)*) => {}; 162 (emit I64Rotl $($rest:tt)*) => {}; 163 (emit I32Rotr $($rest:tt)*) => {}; 164 (emit I64Rotr $($rest:tt)*) => {}; 165 (emit I32Clz $($rest:tt)*) => {}; 166 (emit I64Clz $($rest:tt)*) => {}; 167 (emit I32Ctz $($rest:tt)*) => {}; 168 (emit I64Ctz $($rest:tt)*) => {}; 169 (emit I32Popcnt $($rest:tt)*) => {}; 170 (emit I64Popcnt $($rest:tt)*) => {}; 171 (emit I32WrapI64 $($rest:tt)*) => {}; 172 (emit I64ExtendI32S $($rest:tt)*) => {}; 173 (emit I64ExtendI32U $($rest:tt)*) => {}; 174 (emit I32Extend8S $($rest:tt)*) => {}; 175 (emit I32Extend16S $($rest:tt)*) => {}; 176 (emit I64Extend8S $($rest:tt)*) => {}; 177 (emit I64Extend16S $($rest:tt)*) => {}; 178 (emit I64Extend32S $($rest:tt)*) => {}; 179 (emit I32TruncF32S $($rest:tt)*) => {}; 180 (emit I32TruncF32U $($rest:tt)*) => {}; 181 (emit I32TruncF64S $($rest:tt)*) => {}; 182 (emit I32TruncF64U $($rest:tt)*) => {}; 183 (emit I64TruncF32S $($rest:tt)*) => {}; 184 (emit I64TruncF32U $($rest:tt)*) => {}; 185 (emit I64TruncF64S $($rest:tt)*) => {}; 186 (emit I64TruncF64U $($rest:tt)*) => {}; 187 (emit I32ReinterpretF32 $($rest:tt)*) => {}; 188 (emit I64ReinterpretF64 $($rest:tt)*) => {}; 189 (emit LocalGet $($rest:tt)*) => {}; 190 (emit LocalSet $($rest:tt)*) => {}; 191 (emit Call $($rest:tt)*) => {}; 192 (emit End $($rest:tt)*) => {}; 193 (emit Nop $($rest:tt)*) => {}; 194 (emit If $($rest:tt)*) => {}; 195 (emit Else $($rest:tt)*) => {}; 196 (emit Block $($rest:tt)*) => {}; 197 (emit Loop $($rest:tt)*) => {}; 198 (emit Br $($rest:tt)*) => {}; 199 (emit BrIf $($rest:tt)*) => {}; 200 (emit Return $($rest:tt)*) => {}; 201 (emit Unreachable $($rest:tt)*) => {}; 202 (emit LocalTee $($rest:tt)*) => {}; 203 (emit GlobalGet $($rest:tt)*) => {}; 204 (emit GlobalSet $($rest:tt)*) => {}; 205 (emit Select $($rest:tt)*) => {}; 206 (emit Drop $($rest:tt)*) => {}; 207 (emit BrTable $($rest:tt)*) => {}; 208 (emit CallIndirect $($rest:tt)*) => {}; 209 (emit TableInit $($rest:tt)*) => {}; 210 (emit TableCopy $($rest:tt)*) => {}; 211 (emit TableGet $($rest:tt)*) => {}; 212 (emit TableSet $($rest:tt)*) => {}; 213 (emit TableGrow $($rest:tt)*) => {}; 214 (emit TableSize $($rest:tt)*) => {}; 215 (emit TableFill $($rest:tt)*) => {}; 216 (emit ElemDrop $($rest:tt)*) => {}; 217 (emit MemoryInit $($rest:tt)*) => {}; 218 (emit MemoryCopy $($rest:tt)*) => {}; 219 (emit DataDrop $($rest:tt)*) => {}; 220 (emit MemoryFill $($rest:tt)*) => {}; 221 (emit MemorySize $($rest:tt)*) => {}; 222 (emit MemoryGrow $($rest:tt)*) => {}; 223 (emit I32Load $($rest:tt)*) => {}; 224 (emit I32Load8S $($rest:tt)*) => {}; 225 (emit I32Load8U $($rest:tt)*) => {}; 226 (emit I32Load16S $($rest:tt)*) => {}; 227 (emit I32Load16U $($rest:tt)*) => {}; 228 (emit I64Load8S $($rest:tt)*) => {}; 229 (emit I64Load8U $($rest:tt)*) => {}; 230 (emit I64Load16S $($rest:tt)*) => {}; 231 (emit I64Load16U $($rest:tt)*) => {}; 232 (emit I64Load32S $($rest:tt)*) => {}; 233 (emit I64Load32U $($rest:tt)*) => {}; 234 (emit I64Load $($rest:tt)*) => {}; 235 (emit I32Store $($rest:tt)*) => {}; 236 (emit I32Store8 $($rest:tt)*) => {}; 237 (emit I32Store16 $($rest:tt)*) => {}; 238 (emit I64Store $($rest:tt)*) => {}; 239 (emit I64Store8 $($rest:tt)*) => {}; 240 (emit I64Store16 $($rest:tt)*) => {}; 241 (emit I64Store32 $($rest:tt)*) => {}; 242 (emit F32Load $($rest:tt)*) => {}; 243 (emit F32Store $($rest:tt)*) => {}; 244 (emit F64Load $($rest:tt)*) => {}; 245 (emit F64Store $($rest:tt)*) => {}; 246 (emit I32TruncSatF32S $($rest:tt)*) => {}; 247 (emit I32TruncSatF32U $($rest:tt)*) => {}; 248 (emit I32TruncSatF64S $($rest:tt)*) => {}; 249 (emit I32TruncSatF64U $($rest:tt)*) => {}; 250 (emit I64TruncSatF32S $($rest:tt)*) => {}; 251 (emit I64TruncSatF32U $($rest:tt)*) => {}; 252 (emit I64TruncSatF64S $($rest:tt)*) => {}; 253 (emit I64TruncSatF64U $($rest:tt)*) => {}; 254 (emit V128Load $($rest:tt)*) => {}; 255 (emit V128Store $($rest:tt)*) => {}; 256 (emit I64Add128 $($rest:tt)*) => {}; 257 (emit I64Sub128 $($rest:tt)*) => {}; 258 (emit I64MulWideS $($rest:tt)*) => {}; 259 (emit I64MulWideU $($rest:tt)*) => {}; 260 (emit I32AtomicLoad8U $($rest:tt)*) => {}; 261 (emit I32AtomicLoad16U $($rest:tt)*) => {}; 262 (emit I32AtomicLoad $($rest:tt)*) => {}; 263 (emit I64AtomicLoad8U $($rest:tt)*) => {}; 264 (emit I64AtomicLoad16U $($rest:tt)*) => {}; 265 (emit I64AtomicLoad32U $($rest:tt)*) => {}; 266 (emit I64AtomicLoad $($rest:tt)*) => {}; 267 (emit V128Load8x8S $($rest:tt)*) => {}; 268 (emit V128Load8x8U $($rest:tt)*) => {}; 269 (emit V128Load16x4S $($rest:tt)*) => {}; 270 (emit V128Load16x4U $($rest:tt)*) => {}; 271 (emit V128Load32x2S $($rest:tt)*) => {}; 272 (emit V128Load32x2U $($rest:tt)*) => {}; 273 (emit V128Load8Splat $($rest:tt)*) => {}; 274 (emit V128Load16Splat $($rest:tt)*) => {}; 275 (emit V128Load32Splat $($rest:tt)*) => {}; 276 (emit V128Load64Splat $($rest:tt)*) => {}; 277 (emit I8x16Splat $($rest:tt)*) => {}; 278 (emit I16x8Splat $($rest:tt)*) => {}; 279 (emit I32x4Splat $($rest:tt)*) => {}; 280 (emit I64x2Splat $($rest:tt)*) => {}; 281 (emit F32x4Splat $($rest:tt)*) => {}; 282 (emit F64x2Splat $($rest:tt)*) => {}; 283 (emit I32AtomicStore8 $($rest:tt)*) => {}; 284 (emit I32AtomicStore16 $($rest:tt)*) => {}; 285 (emit I32AtomicStore $($rest:tt)*) => {}; 286 (emit I64AtomicStore8 $($rest:tt)*) => {}; 287 (emit I64AtomicStore16 $($rest:tt)*) => {}; 288 (emit I64AtomicStore32 $($rest:tt)*) => {}; 289 (emit I64AtomicStore $($rest:tt)*) => {}; 290 (emit I32AtomicRmw8AddU $($rest:tt)*) => {}; 291 (emit I32AtomicRmw16AddU $($rest:tt)*) => {}; 292 (emit I32AtomicRmwAdd $($rest:tt)*) => {}; 293 (emit I64AtomicRmw8AddU $($rest:tt)*) => {}; 294 (emit I64AtomicRmw16AddU $($rest:tt)*) => {}; 295 (emit I64AtomicRmw32AddU $($rest:tt)*) => {}; 296 (emit I64AtomicRmwAdd $($rest:tt)*) => {}; 297 (emit I8x16Shuffle $($rest:tt)*) => {}; 298 (emit I8x16Swizzle $($rest:tt)*) => {}; 299 (emit I32AtomicRmw8SubU $($rest:tt)*) => {}; 300 (emit I32AtomicRmw16SubU $($rest:tt)*) => {}; 301 (emit I32AtomicRmwSub $($rest:tt)*) => {}; 302 (emit I64AtomicRmw8SubU $($rest:tt)*) => {}; 303 (emit I64AtomicRmw16SubU $($rest:tt)*) => {}; 304 (emit I64AtomicRmw32SubU $($rest:tt)*) => {}; 305 (emit I64AtomicRmwSub $($rest:tt)*) => {}; 306 (emit I32AtomicRmw8XchgU $($rest:tt)*) => {}; 307 (emit I32AtomicRmw16XchgU $($rest:tt)*) => {}; 308 (emit I32AtomicRmwXchg $($rest:tt)*) => {}; 309 (emit I64AtomicRmw8XchgU $($rest:tt)*) => {}; 310 (emit I64AtomicRmw16XchgU $($rest:tt)*) => {}; 311 (emit I64AtomicRmw32XchgU $($rest:tt)*) => {}; 312 (emit I64AtomicRmwXchg $($rest:tt)*) => {}; 313 (emit I8x16ExtractLaneS $($rest:tt)*) => {}; 314 (emit I8x16ExtractLaneU $($rest:tt)*) => {}; 315 (emit I16x8ExtractLaneS $($rest:tt)*) => {}; 316 (emit I16x8ExtractLaneU $($rest:tt)*) => {}; 317 (emit I32x4ExtractLane $($rest:tt)*) => {}; 318 (emit I64x2ExtractLane $($rest:tt)*) => {}; 319 (emit F32x4ExtractLane $($rest:tt)*) => {}; 320 (emit F64x2ExtractLane $($rest:tt)*) => {}; 321 (emit I32AtomicRmw8AndU $($rest:tt)*) => {}; 322 (emit I32AtomicRmw16AndU $($rest:tt)*) => {}; 323 (emit I32AtomicRmwAnd $($rest:tt)*) => {}; 324 (emit I64AtomicRmw8AndU $($rest:tt)*) => {}; 325 (emit I64AtomicRmw16AndU $($rest:tt)*) => {}; 326 (emit I64AtomicRmw32AndU $($rest:tt)*) => {}; 327 (emit I64AtomicRmwAnd $($rest:tt)*) => {}; 328 (emit I32AtomicRmw8OrU $($rest:tt)*) => {}; 329 (emit I32AtomicRmw16OrU $($rest:tt)*) => {}; 330 (emit I32AtomicRmwOr $($rest:tt)*) => {}; 331 (emit I64AtomicRmw8OrU $($rest:tt)*) => {}; 332 (emit I64AtomicRmw16OrU $($rest:tt)*) => {}; 333 (emit I64AtomicRmw32OrU $($rest:tt)*) => {}; 334 (emit I64AtomicRmwOr $($rest:tt)*) => {}; 335 (emit I32AtomicRmw8XorU $($rest:tt)*) => {}; 336 (emit I32AtomicRmw16XorU $($rest:tt)*) => {}; 337 (emit I32AtomicRmwXor $($rest:tt)*) => {}; 338 (emit I64AtomicRmw8XorU $($rest:tt)*) => {}; 339 (emit I64AtomicRmw16XorU $($rest:tt)*) => {}; 340 (emit I64AtomicRmw32XorU $($rest:tt)*) => {}; 341 (emit I64AtomicRmwXor $($rest:tt)*) => {}; 342 (emit I8x16ReplaceLane $($rest:tt)*) => {}; 343 (emit I16x8ReplaceLane $($rest:tt)*) => {}; 344 (emit I32x4ReplaceLane $($rest:tt)*) => {}; 345 (emit I64x2ReplaceLane $($rest:tt)*) => {}; 346 (emit F32x4ReplaceLane $($rest:tt)*) => {}; 347 (emit F64x2ReplaceLane $($rest:tt)*) => {}; 348 (emit I32AtomicRmw8CmpxchgU $($rest:tt)*) => {}; 349 (emit I32AtomicRmw16CmpxchgU $($rest:tt)*) => {}; 350 (emit I32AtomicRmwCmpxchg $($rest:tt)*) => {}; 351 (emit I64AtomicRmw8CmpxchgU $($rest:tt)*) => {}; 352 (emit I64AtomicRmw16CmpxchgU $($rest:tt)*) => {}; 353 (emit I64AtomicRmw32CmpxchgU $($rest:tt)*) => {}; 354 (emit I64AtomicRmwCmpxchg $($rest:tt)*) => {}; 355 (emit I8x16Eq $($rest:tt)*) => {}; 356 (emit I16x8Eq $($rest:tt)*) => {}; 357 (emit I32x4Eq $($rest:tt)*) => {}; 358 (emit I64x2Eq $($rest:tt)*) => {}; 359 (emit F32x4Eq $($rest:tt)*) => {}; 360 (emit F64x2Eq $($rest:tt)*) => {}; 361 (emit I8x16Ne $($rest:tt)*) => {}; 362 (emit I16x8Ne $($rest:tt)*) => {}; 363 (emit I32x4Ne $($rest:tt)*) => {}; 364 (emit I64x2Ne $($rest:tt)*) => {}; 365 (emit F32x4Ne $($rest:tt)*) => {}; 366 (emit F64x2Ne $($rest:tt)*) => {}; 367 (emit I8x16LtS $($rest:tt)*) => {}; 368 (emit I8x16LtU $($rest:tt)*) => {}; 369 (emit I16x8LtS $($rest:tt)*) => {}; 370 (emit I16x8LtU $($rest:tt)*) => {}; 371 (emit I32x4LtS $($rest:tt)*) => {}; 372 (emit I32x4LtU $($rest:tt)*) => {}; 373 (emit I64x2LtS $($rest:tt)*) => {}; 374 (emit F32x4Lt $($rest:tt)*) => {}; 375 (emit F64x2Lt $($rest:tt)*) => {}; 376 (emit I8x16LeS $($rest:tt)*) => {}; 377 (emit I8x16LeU $($rest:tt)*) => {}; 378 (emit I16x8LeS $($rest:tt)*) => {}; 379 (emit I16x8LeU $($rest:tt)*) => {}; 380 (emit I32x4LeS $($rest:tt)*) => {}; 381 (emit I32x4LeU $($rest:tt)*) => {}; 382 (emit I64x2LeS $($rest:tt)*) => {}; 383 (emit F32x4Le $($rest:tt)*) => {}; 384 (emit F64x2Le $($rest:tt)*) => {}; 385 (emit I8x16GtS $($rest:tt)*) => {}; 386 (emit I8x16GtU $($rest:tt)*) => {}; 387 (emit I16x8GtS $($rest:tt)*) => {}; 388 (emit I16x8GtU $($rest:tt)*) => {}; 389 (emit I32x4GtS $($rest:tt)*) => {}; 390 (emit I32x4GtU $($rest:tt)*) => {}; 391 (emit I64x2GtS $($rest:tt)*) => {}; 392 (emit F32x4Gt $($rest:tt)*) => {}; 393 (emit F64x2Gt $($rest:tt)*) => {}; 394 (emit I8x16GeS $($rest:tt)*) => {}; 395 (emit I8x16GeU $($rest:tt)*) => {}; 396 (emit I16x8GeS $($rest:tt)*) => {}; 397 (emit I16x8GeU $($rest:tt)*) => {}; 398 (emit I32x4GeS $($rest:tt)*) => {}; 399 (emit I32x4GeU $($rest:tt)*) => {}; 400 (emit I64x2GeS $($rest:tt)*) => {}; 401 (emit F32x4Ge $($rest:tt)*) => {}; 402 (emit F64x2Ge $($rest:tt)*) => {}; 403 (emit MemoryAtomicWait32 $($rest:tt)*) => {}; 404 (emit MemoryAtomicWait64 $($rest:tt)*) => {}; 405 (emit MemoryAtomicNotify $($rest:tt)*) => {}; 406 (emit AtomicFence $($rest:tt)*) => {}; 407 (emit V128Not $($rest:tt)*) => {}; 408 (emit V128And $($rest:tt)*) => {}; 409 (emit V128AndNot $($rest:tt)*) => {}; 410 (emit V128Or $($rest:tt)*) => {}; 411 (emit V128Xor $($rest:tt)*) => {}; 412 (emit V128Bitselect $($rest:tt)*) => {}; 413 (emit V128AnyTrue $($rest:tt)*) => {}; 414 (emit V128Load8Lane $($rest:tt)*) => {}; 415 (emit V128Load16Lane $($rest:tt)*) => {}; 416 (emit V128Load32Lane $($rest:tt)*) => {}; 417 (emit V128Load64Lane $($rest:tt)*) => {}; 418 (emit V128Store8Lane $($rest:tt)*) => {}; 419 (emit V128Store16Lane $($rest:tt)*) => {}; 420 (emit V128Store32Lane $($rest:tt)*) => {}; 421 (emit V128Store64Lane $($rest:tt)*) => {}; 422 (emit F32x4ConvertI32x4S $($rest:tt)*) => {}; 423 (emit F32x4ConvertI32x4U $($rest:tt)*) => {}; 424 (emit F64x2ConvertLowI32x4S $($rest:tt)*) => {}; 425 (emit F64x2ConvertLowI32x4U $($rest:tt)*) => {}; 426 (emit I8x16NarrowI16x8S $($rest:tt)*) => {}; 427 (emit I8x16NarrowI16x8U $($rest:tt)*) => {}; 428 (emit I16x8NarrowI32x4S $($rest:tt)*) => {}; 429 (emit I16x8NarrowI32x4U $($rest:tt)*) => {}; 430 (emit F32x4DemoteF64x2Zero $($rest:tt)*) => {}; 431 (emit F64x2PromoteLowF32x4 $($rest:tt)*) => {}; 432 (emit I16x8ExtendLowI8x16S $($rest:tt)*) => {}; 433 (emit I16x8ExtendHighI8x16S $($rest:tt)*) => {}; 434 (emit I16x8ExtendLowI8x16U $($rest:tt)*) => {}; 435 (emit I16x8ExtendHighI8x16U $($rest:tt)*) => {}; 436 (emit I32x4ExtendLowI16x8S $($rest:tt)*) => {}; 437 (emit I32x4ExtendHighI16x8S $($rest:tt)*) => {}; 438 (emit I32x4ExtendLowI16x8U $($rest:tt)*) => {}; 439 (emit I32x4ExtendHighI16x8U $($rest:tt)*) => {}; 440 (emit I64x2ExtendLowI32x4S $($rest:tt)*) => {}; 441 (emit I64x2ExtendHighI32x4S $($rest:tt)*) => {}; 442 (emit I64x2ExtendLowI32x4U $($rest:tt)*) => {}; 443 (emit I64x2ExtendHighI32x4U $($rest:tt)*) => {}; 444 (emit I8x16Add $($rest:tt)*) => {}; 445 (emit I16x8Add $($rest:tt)*) => {}; 446 (emit I32x4Add $($rest:tt)*) => {}; 447 (emit I64x2Add $($rest:tt)*) => {}; 448 (emit I8x16Sub $($rest:tt)*) => {}; 449 (emit I16x8Sub $($rest:tt)*) => {}; 450 (emit I32x4Sub $($rest:tt)*) => {}; 451 (emit I64x2Sub $($rest:tt)*) => {}; 452 (emit I16x8Mul $($rest:tt)*) => {}; 453 (emit I32x4Mul $($rest:tt)*) => {}; 454 (emit I64x2Mul $($rest:tt)*) => {}; 455 (emit I8x16AddSatS $($rest:tt)*) => {}; 456 (emit I16x8AddSatS $($rest:tt)*) => {}; 457 (emit I8x16AddSatU $($rest:tt)*) => {}; 458 (emit I16x8AddSatU $($rest:tt)*) => {}; 459 (emit I8x16SubSatS $($rest:tt)*) => {}; 460 (emit I16x8SubSatS $($rest:tt)*) => {}; 461 (emit I8x16SubSatU $($rest:tt)*) => {}; 462 (emit I16x8SubSatU $($rest:tt)*) => {}; 463 (emit I8x16Abs $($rest:tt)*) => {}; 464 (emit I16x8Abs $($rest:tt)*) => {}; 465 (emit I32x4Abs $($rest:tt)*) => {}; 466 (emit I64x2Abs $($rest:tt)*) => {}; 467 (emit F32x4Abs $($rest:tt)*) => {}; 468 (emit F64x2Abs $($rest:tt)*) => {}; 469 (emit I8x16Neg $($rest:tt)*) => {}; 470 (emit I16x8Neg $($rest:tt)*) => {}; 471 (emit I32x4Neg $($rest:tt)*) => {}; 472 (emit I64x2Neg $($rest:tt)*) => {}; 473 (emit I8x16Shl $($rest:tt)*) => {}; 474 (emit I16x8Shl $($rest:tt)*) => {}; 475 (emit I32x4Shl $($rest:tt)*) => {}; 476 (emit I64x2Shl $($rest:tt)*) => {}; 477 (emit I8x16ShrU $($rest:tt)*) => {}; 478 (emit I16x8ShrU $($rest:tt)*) => {}; 479 (emit I32x4ShrU $($rest:tt)*) => {}; 480 (emit I64x2ShrU $($rest:tt)*) => {}; 481 (emit I8x16ShrS $($rest:tt)*) => {}; 482 (emit I16x8ShrS $($rest:tt)*) => {}; 483 (emit I32x4ShrS $($rest:tt)*) => {}; 484 (emit I64x2ShrS $($rest:tt)*) => {}; 485 (emit I16x8Q15MulrSatS $($rest:tt)*) => {}; 486 (emit I8x16AllTrue $($rest:tt)*) => {}; 487 (emit I16x8AllTrue $($rest:tt)*) => {}; 488 (emit I32x4AllTrue $($rest:tt)*) => {}; 489 (emit I64x2AllTrue $($rest:tt)*) => {}; 490 (emit I8x16Bitmask $($rest:tt)*) => {}; 491 (emit I16x8Bitmask $($rest:tt)*) => {}; 492 (emit I32x4Bitmask $($rest:tt)*) => {}; 493 (emit I64x2Bitmask $($rest:tt)*) => {}; 494 (emit I32x4TruncSatF32x4S $($rest:tt)*) => {}; 495 (emit I32x4TruncSatF32x4U $($rest:tt)*) => {}; 496 (emit I32x4TruncSatF64x2SZero $($rest:tt)*) => {}; 497 (emit I32x4TruncSatF64x2UZero $($rest:tt)*) => {}; 498 (emit I8x16MinU $($rest:tt)*) => {}; 499 (emit I16x8MinU $($rest:tt)*) => {}; 500 (emit I32x4MinU $($rest:tt)*) => {}; 501 (emit I8x16MinS $($rest:tt)*) => {}; 502 (emit I16x8MinS $($rest:tt)*) => {}; 503 (emit I32x4MinS $($rest:tt)*) => {}; 504 (emit I8x16MaxU $($rest:tt)*) => {}; 505 (emit I16x8MaxU $($rest:tt)*) => {}; 506 (emit I32x4MaxU $($rest:tt)*) => {}; 507 (emit I8x16MaxS $($rest:tt)*) => {}; 508 (emit I16x8MaxS $($rest:tt)*) => {}; 509 (emit I32x4MaxS $($rest:tt)*) => {}; 510 (emit I16x8ExtMulLowI8x16S $($rest:tt)*) => {}; 511 (emit I32x4ExtMulLowI16x8S $($rest:tt)*) => {}; 512 (emit I64x2ExtMulLowI32x4S $($rest:tt)*) => {}; 513 (emit I16x8ExtMulHighI8x16S $($rest:tt)*) => {}; 514 (emit I32x4ExtMulHighI16x8S $($rest:tt)*) => {}; 515 (emit I64x2ExtMulHighI32x4S $($rest:tt)*) => {}; 516 (emit I16x8ExtMulLowI8x16U $($rest:tt)*) => {}; 517 (emit I32x4ExtMulLowI16x8U $($rest:tt)*) => {}; 518 (emit I64x2ExtMulLowI32x4U $($rest:tt)*) => {}; 519 (emit I16x8ExtMulHighI8x16U $($rest:tt)*) => {}; 520 (emit I32x4ExtMulHighI16x8U $($rest:tt)*) => {}; 521 (emit I64x2ExtMulHighI32x4U $($rest:tt)*) => {}; 522 (emit I16x8ExtAddPairwiseI8x16U $($rest:tt)*) => {}; 523 (emit I16x8ExtAddPairwiseI8x16S $($rest:tt)*) => {}; 524 (emit I32x4ExtAddPairwiseI16x8U $($rest:tt)*) => {}; 525 (emit I32x4ExtAddPairwiseI16x8S $($rest:tt)*) => {}; 526 (emit I32x4DotI16x8S $($rest:tt)*) => {}; 527 (emit I8x16Popcnt $($rest:tt)*) => {}; 528 (emit I8x16AvgrU $($rest:tt)*) => {}; 529 (emit I16x8AvgrU $($rest:tt)*) => {}; 530 (emit F32x4Add $($rest:tt)*) => {}; 531 (emit F64x2Add $($rest:tt)*) => {}; 532 (emit F32x4Sub $($rest:tt)*) => {}; 533 (emit F64x2Sub $($rest:tt)*) => {}; 534 (emit F32x4Mul $($rest:tt)*) => {}; 535 (emit F64x2Mul $($rest:tt)*) => {}; 536 (emit F32x4Div $($rest:tt)*) => {}; 537 (emit F64x2Div $($rest:tt)*) => {}; 538 (emit F32x4Neg $($rest:tt)*) => {}; 539 (emit F64x2Neg $($rest:tt)*) => {}; 540 (emit F32x4Sqrt $($rest:tt)*) => {}; 541 (emit F64x2Sqrt $($rest:tt)*) => {}; 542 (emit F32x4Ceil $($rest:tt)*) => {}; 543 (emit F64x2Ceil $($rest:tt)*) => {}; 544 (emit F32x4Floor $($rest:tt)*) => {}; 545 (emit F64x2Floor $($rest:tt)*) => {}; 546 (emit F32x4Nearest $($rest:tt)*) => {}; 547 (emit F64x2Nearest $($rest:tt)*) => {}; 548 (emit F32x4Trunc $($rest:tt)*) => {}; 549 (emit F64x2Trunc $($rest:tt)*) => {}; 550 (emit V128Load32Zero $($rest:tt)*) => {}; 551 (emit V128Load64Zero $($rest:tt)*) => {}; 552 (emit F32x4PMin $($rest:tt)*) => {}; 553 (emit F64x2PMin $($rest:tt)*) => {}; 554 (emit F32x4PMax $($rest:tt)*) => {}; 555 (emit F64x2PMax $($rest:tt)*) => {}; 556 (emit F32x4Min $($rest:tt)*) => {}; 557 (emit F64x2Min $($rest:tt)*) => {}; 558 (emit F32x4Max $($rest:tt)*) => {}; 559 (emit F64x2Max $($rest:tt)*) => {}; 560 561 (emit $unsupported:tt $($rest:tt)*) => {$($rest)*}; 562 } 563 564 impl<'a, 'translation, 'data, M> VisitOperator<'a> for CodeGen<'a, 'translation, 'data, M, Emission> 565 where 566 M: MacroAssembler, 567 { 568 type Output = Result<()>; 569 570 fn visit_i32_const(&mut self, val: i32) -> Self::Output { 571 self.context.stack.push(Val::i32(val)); 572 573 Ok(()) 574 } 575 576 fn visit_i64_const(&mut self, val: i64) -> Self::Output { 577 self.context.stack.push(Val::i64(val)); 578 Ok(()) 579 } 580 581 fn visit_f32_const(&mut self, val: Ieee32) -> Self::Output { 582 self.context.stack.push(Val::f32(val)); 583 Ok(()) 584 } 585 586 fn visit_f64_const(&mut self, val: Ieee64) -> Self::Output { 587 self.context.stack.push(Val::f64(val)); 588 Ok(()) 589 } 590 591 fn visit_f32_add(&mut self) -> Self::Output { 592 self.context.binop( 593 self.masm, 594 OperandSize::S32, 595 &mut |masm: &mut M, dst, src, size| { 596 masm.float_add(writable!(dst), dst, src, size)?; 597 Ok(TypedReg::f32(dst)) 598 }, 599 ) 600 } 601 602 fn visit_f64_add(&mut self) -> Self::Output { 603 self.context.binop( 604 self.masm, 605 OperandSize::S64, 606 &mut |masm: &mut M, dst, src, size| { 607 masm.float_add(writable!(dst), dst, src, size)?; 608 Ok(TypedReg::f64(dst)) 609 }, 610 ) 611 } 612 613 fn visit_f32_sub(&mut self) -> Self::Output { 614 self.context.binop( 615 self.masm, 616 OperandSize::S32, 617 &mut |masm: &mut M, dst, src, size| { 618 masm.float_sub(writable!(dst), dst, src, size)?; 619 Ok(TypedReg::f32(dst)) 620 }, 621 ) 622 } 623 624 fn visit_f64_sub(&mut self) -> Self::Output { 625 self.context.binop( 626 self.masm, 627 OperandSize::S64, 628 &mut |masm: &mut M, dst, src, size| { 629 masm.float_sub(writable!(dst), dst, src, size)?; 630 Ok(TypedReg::f64(dst)) 631 }, 632 ) 633 } 634 635 fn visit_f32_mul(&mut self) -> Self::Output { 636 self.context.binop( 637 self.masm, 638 OperandSize::S32, 639 &mut |masm: &mut M, dst, src, size| { 640 masm.float_mul(writable!(dst), dst, src, size)?; 641 Ok(TypedReg::f32(dst)) 642 }, 643 ) 644 } 645 646 fn visit_f64_mul(&mut self) -> Self::Output { 647 self.context.binop( 648 self.masm, 649 OperandSize::S64, 650 &mut |masm: &mut M, dst, src, size| { 651 masm.float_mul(writable!(dst), dst, src, size)?; 652 Ok(TypedReg::f64(dst)) 653 }, 654 ) 655 } 656 657 fn visit_f32_div(&mut self) -> Self::Output { 658 self.context.binop( 659 self.masm, 660 OperandSize::S32, 661 &mut |masm: &mut M, dst, src, size| { 662 masm.float_div(writable!(dst), dst, src, size)?; 663 Ok(TypedReg::f32(dst)) 664 }, 665 ) 666 } 667 668 fn visit_f64_div(&mut self) -> Self::Output { 669 self.context.binop( 670 self.masm, 671 OperandSize::S64, 672 &mut |masm: &mut M, dst, src, size| { 673 masm.float_div(writable!(dst), dst, src, size)?; 674 Ok(TypedReg::f64(dst)) 675 }, 676 ) 677 } 678 679 fn visit_f32_min(&mut self) -> Self::Output { 680 self.context.binop( 681 self.masm, 682 OperandSize::S32, 683 &mut |masm: &mut M, dst, src, size| { 684 masm.float_min(writable!(dst), dst, src, size)?; 685 Ok(TypedReg::f32(dst)) 686 }, 687 ) 688 } 689 690 fn visit_f64_min(&mut self) -> Self::Output { 691 self.context.binop( 692 self.masm, 693 OperandSize::S64, 694 &mut |masm: &mut M, dst, src, size| { 695 masm.float_min(writable!(dst), dst, src, size)?; 696 Ok(TypedReg::f64(dst)) 697 }, 698 ) 699 } 700 701 fn visit_f32_max(&mut self) -> Self::Output { 702 self.context.binop( 703 self.masm, 704 OperandSize::S32, 705 &mut |masm: &mut M, dst, src, size| { 706 masm.float_max(writable!(dst), dst, src, size)?; 707 Ok(TypedReg::f32(dst)) 708 }, 709 ) 710 } 711 712 fn visit_f64_max(&mut self) -> Self::Output { 713 self.context.binop( 714 self.masm, 715 OperandSize::S64, 716 &mut |masm: &mut M, dst, src, size| { 717 masm.float_max(writable!(dst), dst, src, size)?; 718 Ok(TypedReg::f64(dst)) 719 }, 720 ) 721 } 722 723 fn visit_f32_copysign(&mut self) -> Self::Output { 724 self.context.binop( 725 self.masm, 726 OperandSize::S32, 727 &mut |masm: &mut M, dst, src, size| { 728 masm.float_copysign(writable!(dst), dst, src, size)?; 729 Ok(TypedReg::f32(dst)) 730 }, 731 ) 732 } 733 734 fn visit_f64_copysign(&mut self) -> Self::Output { 735 self.context.binop( 736 self.masm, 737 OperandSize::S64, 738 &mut |masm: &mut M, dst, src, size| { 739 masm.float_copysign(writable!(dst), dst, src, size)?; 740 Ok(TypedReg::f64(dst)) 741 }, 742 ) 743 } 744 745 fn visit_f32_abs(&mut self) -> Self::Output { 746 self.context.unop(self.masm, |masm, reg| { 747 masm.float_abs(writable!(reg), OperandSize::S32)?; 748 Ok(TypedReg::f32(reg)) 749 }) 750 } 751 752 fn visit_f64_abs(&mut self) -> Self::Output { 753 self.context.unop(self.masm, |masm, reg| { 754 masm.float_abs(writable!(reg), OperandSize::S64)?; 755 Ok(TypedReg::f64(reg)) 756 }) 757 } 758 759 fn visit_f32_neg(&mut self) -> Self::Output { 760 self.context.unop(self.masm, |masm, reg| { 761 masm.float_neg(writable!(reg), OperandSize::S32)?; 762 Ok(TypedReg::f32(reg)) 763 }) 764 } 765 766 fn visit_f64_neg(&mut self) -> Self::Output { 767 self.context.unop(self.masm, |masm, reg| { 768 masm.float_neg(writable!(reg), OperandSize::S64)?; 769 Ok(TypedReg::f64(reg)) 770 }) 771 } 772 773 fn visit_f32_floor(&mut self) -> Self::Output { 774 self.masm.float_round( 775 RoundingMode::Down, 776 &mut self.env, 777 &mut self.context, 778 OperandSize::S32, 779 |env, cx, masm| { 780 let builtin = env.builtins.floor_f32::<M::ABI>()?; 781 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 782 }, 783 ) 784 } 785 786 fn visit_f64_floor(&mut self) -> Self::Output { 787 self.masm.float_round( 788 RoundingMode::Down, 789 &mut self.env, 790 &mut self.context, 791 OperandSize::S64, 792 |env, cx, masm| { 793 let builtin = env.builtins.floor_f64::<M::ABI>()?; 794 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 795 }, 796 ) 797 } 798 799 fn visit_f32_ceil(&mut self) -> Self::Output { 800 self.masm.float_round( 801 RoundingMode::Up, 802 &mut self.env, 803 &mut self.context, 804 OperandSize::S32, 805 |env, cx, masm| { 806 let builtin = env.builtins.ceil_f32::<M::ABI>()?; 807 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 808 }, 809 ) 810 } 811 812 fn visit_f64_ceil(&mut self) -> Self::Output { 813 self.masm.float_round( 814 RoundingMode::Up, 815 &mut self.env, 816 &mut self.context, 817 OperandSize::S64, 818 |env, cx, masm| { 819 let builtin = env.builtins.ceil_f64::<M::ABI>()?; 820 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 821 }, 822 ) 823 } 824 825 fn visit_f32_nearest(&mut self) -> Self::Output { 826 self.masm.float_round( 827 RoundingMode::Nearest, 828 &mut self.env, 829 &mut self.context, 830 OperandSize::S32, 831 |env, cx, masm| { 832 let builtin = env.builtins.nearest_f32::<M::ABI>()?; 833 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 834 }, 835 ) 836 } 837 838 fn visit_f64_nearest(&mut self) -> Self::Output { 839 self.masm.float_round( 840 RoundingMode::Nearest, 841 &mut self.env, 842 &mut self.context, 843 OperandSize::S64, 844 |env, cx, masm| { 845 let builtin = env.builtins.nearest_f64::<M::ABI>()?; 846 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 847 }, 848 ) 849 } 850 851 fn visit_f32_trunc(&mut self) -> Self::Output { 852 self.masm.float_round( 853 RoundingMode::Zero, 854 &mut self.env, 855 &mut self.context, 856 OperandSize::S32, 857 |env, cx, masm| { 858 let builtin = env.builtins.trunc_f32::<M::ABI>()?; 859 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 860 }, 861 ) 862 } 863 864 fn visit_f64_trunc(&mut self) -> Self::Output { 865 self.masm.float_round( 866 RoundingMode::Zero, 867 &mut self.env, 868 &mut self.context, 869 OperandSize::S64, 870 |env, cx, masm| { 871 let builtin = env.builtins.trunc_f64::<M::ABI>()?; 872 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 873 }, 874 ) 875 } 876 877 fn visit_f32_sqrt(&mut self) -> Self::Output { 878 self.context.unop(self.masm, |masm, reg| { 879 masm.float_sqrt(writable!(reg), reg, OperandSize::S32)?; 880 Ok(TypedReg::f32(reg)) 881 }) 882 } 883 884 fn visit_f64_sqrt(&mut self) -> Self::Output { 885 self.context.unop(self.masm, |masm, reg| { 886 masm.float_sqrt(writable!(reg), reg, OperandSize::S64)?; 887 Ok(TypedReg::f64(reg)) 888 }) 889 } 890 891 fn visit_f32_eq(&mut self) -> Self::Output { 892 self.context.float_cmp_op( 893 self.masm, 894 OperandSize::S32, 895 &mut |masm: &mut M, dst, src1, src2, size| { 896 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Eq, size) 897 }, 898 ) 899 } 900 901 fn visit_f64_eq(&mut self) -> Self::Output { 902 self.context.float_cmp_op( 903 self.masm, 904 OperandSize::S64, 905 &mut |masm: &mut M, dst, src1, src2, size| { 906 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Eq, size) 907 }, 908 ) 909 } 910 911 fn visit_f32_ne(&mut self) -> Self::Output { 912 self.context.float_cmp_op( 913 self.masm, 914 OperandSize::S32, 915 &mut |masm: &mut M, dst, src1, src2, size| { 916 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ne, size) 917 }, 918 ) 919 } 920 921 fn visit_f64_ne(&mut self) -> Self::Output { 922 self.context.float_cmp_op( 923 self.masm, 924 OperandSize::S64, 925 &mut |masm: &mut M, dst, src1, src2, size| { 926 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ne, size) 927 }, 928 ) 929 } 930 931 fn visit_f32_lt(&mut self) -> Self::Output { 932 self.context.float_cmp_op( 933 self.masm, 934 OperandSize::S32, 935 &mut |masm: &mut M, dst, src1, src2, size| { 936 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Lt, size) 937 }, 938 ) 939 } 940 941 fn visit_f64_lt(&mut self) -> Self::Output { 942 self.context.float_cmp_op( 943 self.masm, 944 OperandSize::S64, 945 &mut |masm: &mut M, dst, src1, src2, size| { 946 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Lt, size) 947 }, 948 ) 949 } 950 951 fn visit_f32_gt(&mut self) -> Self::Output { 952 self.context.float_cmp_op( 953 self.masm, 954 OperandSize::S32, 955 &mut |masm: &mut M, dst, src1, src2, size| { 956 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Gt, size) 957 }, 958 ) 959 } 960 961 fn visit_f64_gt(&mut self) -> Self::Output { 962 self.context.float_cmp_op( 963 self.masm, 964 OperandSize::S64, 965 &mut |masm: &mut M, dst, src1, src2, size| { 966 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Gt, size) 967 }, 968 ) 969 } 970 971 fn visit_f32_le(&mut self) -> Self::Output { 972 self.context.float_cmp_op( 973 self.masm, 974 OperandSize::S32, 975 &mut |masm: &mut M, dst, src1, src2, size| { 976 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Le, size) 977 }, 978 ) 979 } 980 981 fn visit_f64_le(&mut self) -> Self::Output { 982 self.context.float_cmp_op( 983 self.masm, 984 OperandSize::S64, 985 &mut |masm: &mut M, dst, src1, src2, size| { 986 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Le, size) 987 }, 988 ) 989 } 990 991 fn visit_f32_ge(&mut self) -> Self::Output { 992 self.context.float_cmp_op( 993 self.masm, 994 OperandSize::S32, 995 &mut |masm: &mut M, dst, src1, src2, size| { 996 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ge, size) 997 }, 998 ) 999 } 1000 1001 fn visit_f64_ge(&mut self) -> Self::Output { 1002 self.context.float_cmp_op( 1003 self.masm, 1004 OperandSize::S64, 1005 &mut |masm: &mut M, dst, src1, src2, size| { 1006 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ge, size) 1007 }, 1008 ) 1009 } 1010 1011 fn visit_f32_convert_i32_s(&mut self) -> Self::Output { 1012 self.context 1013 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 1014 masm.signed_convert(writable!(dst), src, OperandSize::S32, dst_size) 1015 }) 1016 } 1017 1018 fn visit_f32_convert_i32_u(&mut self) -> Self::Output { 1019 self.context.convert_op_with_tmp_reg( 1020 self.masm, 1021 WasmValType::F32, 1022 RegClass::Int, 1023 |masm, dst, src, tmp_gpr, dst_size| { 1024 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S32, dst_size) 1025 }, 1026 ) 1027 } 1028 1029 fn visit_f32_convert_i64_s(&mut self) -> Self::Output { 1030 self.context 1031 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 1032 masm.signed_convert(writable!(dst), src, OperandSize::S64, dst_size) 1033 }) 1034 } 1035 1036 fn visit_f32_convert_i64_u(&mut self) -> Self::Output { 1037 self.context.convert_op_with_tmp_reg( 1038 self.masm, 1039 WasmValType::F32, 1040 RegClass::Int, 1041 |masm, dst, src, tmp_gpr, dst_size| { 1042 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S64, dst_size) 1043 }, 1044 ) 1045 } 1046 1047 fn visit_f64_convert_i32_s(&mut self) -> Self::Output { 1048 self.context 1049 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 1050 masm.signed_convert(writable!(dst), src, OperandSize::S32, dst_size) 1051 }) 1052 } 1053 1054 fn visit_f64_convert_i32_u(&mut self) -> Self::Output { 1055 self.context.convert_op_with_tmp_reg( 1056 self.masm, 1057 WasmValType::F64, 1058 RegClass::Int, 1059 |masm, dst, src, tmp_gpr, dst_size| { 1060 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S32, dst_size) 1061 }, 1062 ) 1063 } 1064 1065 fn visit_f64_convert_i64_s(&mut self) -> Self::Output { 1066 self.context 1067 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 1068 masm.signed_convert(writable!(dst), src, OperandSize::S64, dst_size) 1069 }) 1070 } 1071 1072 fn visit_f64_convert_i64_u(&mut self) -> Self::Output { 1073 self.context.convert_op_with_tmp_reg( 1074 self.masm, 1075 WasmValType::F64, 1076 RegClass::Int, 1077 |masm, dst, src, tmp_gpr, dst_size| { 1078 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S64, dst_size) 1079 }, 1080 ) 1081 } 1082 1083 fn visit_f32_reinterpret_i32(&mut self) -> Self::Output { 1084 self.context 1085 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, size| { 1086 masm.reinterpret_int_as_float(writable!(dst), src, size) 1087 }) 1088 } 1089 1090 fn visit_f64_reinterpret_i64(&mut self) -> Self::Output { 1091 self.context 1092 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, size| { 1093 masm.reinterpret_int_as_float(writable!(dst), src, size) 1094 }) 1095 } 1096 1097 fn visit_f32_demote_f64(&mut self) -> Self::Output { 1098 self.context.unop(self.masm, |masm, reg| { 1099 masm.demote(writable!(reg), reg)?; 1100 Ok(TypedReg::f32(reg)) 1101 }) 1102 } 1103 1104 fn visit_f64_promote_f32(&mut self) -> Self::Output { 1105 self.context.unop(self.masm, |masm, reg| { 1106 masm.promote(writable!(reg), reg)?; 1107 Ok(TypedReg::f64(reg)) 1108 }) 1109 } 1110 1111 fn visit_i32_add(&mut self) -> Self::Output { 1112 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1113 masm.add(writable!(dst), dst, src, size)?; 1114 Ok(TypedReg::i32(dst)) 1115 }) 1116 } 1117 1118 fn visit_i64_add(&mut self) -> Self::Output { 1119 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1120 masm.add(writable!(dst), dst, src, size)?; 1121 Ok(TypedReg::i64(dst)) 1122 }) 1123 } 1124 1125 fn visit_i32_sub(&mut self) -> Self::Output { 1126 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1127 masm.sub(writable!(dst), dst, src, size)?; 1128 Ok(TypedReg::i32(dst)) 1129 }) 1130 } 1131 1132 fn visit_i64_sub(&mut self) -> Self::Output { 1133 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1134 masm.sub(writable!(dst), dst, src, size)?; 1135 Ok(TypedReg::i64(dst)) 1136 }) 1137 } 1138 1139 fn visit_i32_mul(&mut self) -> Self::Output { 1140 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1141 masm.mul(writable!(dst), dst, src, size)?; 1142 Ok(TypedReg::i32(dst)) 1143 }) 1144 } 1145 1146 fn visit_i64_mul(&mut self) -> Self::Output { 1147 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1148 masm.mul(writable!(dst), dst, src, size)?; 1149 Ok(TypedReg::i64(dst)) 1150 }) 1151 } 1152 1153 fn visit_i32_div_s(&mut self) -> Self::Output { 1154 use DivKind::*; 1155 use OperandSize::*; 1156 1157 self.masm.div(&mut self.context, Signed, S32) 1158 } 1159 1160 fn visit_i32_div_u(&mut self) -> Self::Output { 1161 use DivKind::*; 1162 use OperandSize::*; 1163 1164 self.masm.div(&mut self.context, Unsigned, S32) 1165 } 1166 1167 fn visit_i64_div_s(&mut self) -> Self::Output { 1168 use DivKind::*; 1169 use OperandSize::*; 1170 1171 self.masm.div(&mut self.context, Signed, S64) 1172 } 1173 1174 fn visit_i64_div_u(&mut self) -> Self::Output { 1175 use DivKind::*; 1176 use OperandSize::*; 1177 1178 self.masm.div(&mut self.context, Unsigned, S64) 1179 } 1180 1181 fn visit_i32_rem_s(&mut self) -> Self::Output { 1182 use OperandSize::*; 1183 use RemKind::*; 1184 1185 self.masm.rem(&mut self.context, Signed, S32) 1186 } 1187 1188 fn visit_i32_rem_u(&mut self) -> Self::Output { 1189 use OperandSize::*; 1190 use RemKind::*; 1191 1192 self.masm.rem(&mut self.context, Unsigned, S32) 1193 } 1194 1195 fn visit_i64_rem_s(&mut self) -> Self::Output { 1196 use OperandSize::*; 1197 use RemKind::*; 1198 1199 self.masm.rem(&mut self.context, Signed, S64) 1200 } 1201 1202 fn visit_i64_rem_u(&mut self) -> Self::Output { 1203 use OperandSize::*; 1204 use RemKind::*; 1205 1206 self.masm.rem(&mut self.context, Unsigned, S64) 1207 } 1208 1209 fn visit_i32_eq(&mut self) -> Self::Output { 1210 self.cmp_i32s(IntCmpKind::Eq) 1211 } 1212 1213 fn visit_i64_eq(&mut self) -> Self::Output { 1214 self.cmp_i64s(IntCmpKind::Eq) 1215 } 1216 1217 fn visit_i32_ne(&mut self) -> Self::Output { 1218 self.cmp_i32s(IntCmpKind::Ne) 1219 } 1220 1221 fn visit_i64_ne(&mut self) -> Self::Output { 1222 self.cmp_i64s(IntCmpKind::Ne) 1223 } 1224 1225 fn visit_i32_lt_s(&mut self) -> Self::Output { 1226 self.cmp_i32s(IntCmpKind::LtS) 1227 } 1228 1229 fn visit_i64_lt_s(&mut self) -> Self::Output { 1230 self.cmp_i64s(IntCmpKind::LtS) 1231 } 1232 1233 fn visit_i32_lt_u(&mut self) -> Self::Output { 1234 self.cmp_i32s(IntCmpKind::LtU) 1235 } 1236 1237 fn visit_i64_lt_u(&mut self) -> Self::Output { 1238 self.cmp_i64s(IntCmpKind::LtU) 1239 } 1240 1241 fn visit_i32_le_s(&mut self) -> Self::Output { 1242 self.cmp_i32s(IntCmpKind::LeS) 1243 } 1244 1245 fn visit_i64_le_s(&mut self) -> Self::Output { 1246 self.cmp_i64s(IntCmpKind::LeS) 1247 } 1248 1249 fn visit_i32_le_u(&mut self) -> Self::Output { 1250 self.cmp_i32s(IntCmpKind::LeU) 1251 } 1252 1253 fn visit_i64_le_u(&mut self) -> Self::Output { 1254 self.cmp_i64s(IntCmpKind::LeU) 1255 } 1256 1257 fn visit_i32_gt_s(&mut self) -> Self::Output { 1258 self.cmp_i32s(IntCmpKind::GtS) 1259 } 1260 1261 fn visit_i64_gt_s(&mut self) -> Self::Output { 1262 self.cmp_i64s(IntCmpKind::GtS) 1263 } 1264 1265 fn visit_i32_gt_u(&mut self) -> Self::Output { 1266 self.cmp_i32s(IntCmpKind::GtU) 1267 } 1268 1269 fn visit_i64_gt_u(&mut self) -> Self::Output { 1270 self.cmp_i64s(IntCmpKind::GtU) 1271 } 1272 1273 fn visit_i32_ge_s(&mut self) -> Self::Output { 1274 self.cmp_i32s(IntCmpKind::GeS) 1275 } 1276 1277 fn visit_i64_ge_s(&mut self) -> Self::Output { 1278 self.cmp_i64s(IntCmpKind::GeS) 1279 } 1280 1281 fn visit_i32_ge_u(&mut self) -> Self::Output { 1282 self.cmp_i32s(IntCmpKind::GeU) 1283 } 1284 1285 fn visit_i64_ge_u(&mut self) -> Self::Output { 1286 self.cmp_i64s(IntCmpKind::GeU) 1287 } 1288 1289 fn visit_i32_eqz(&mut self) -> Self::Output { 1290 use OperandSize::*; 1291 1292 self.context.unop(self.masm, |masm, reg| { 1293 masm.cmp_with_set(writable!(reg), RegImm::i32(0), IntCmpKind::Eq, S32)?; 1294 Ok(TypedReg::i32(reg)) 1295 }) 1296 } 1297 1298 fn visit_i64_eqz(&mut self) -> Self::Output { 1299 use OperandSize::*; 1300 1301 self.context.unop(self.masm, |masm, reg| { 1302 masm.cmp_with_set(writable!(reg), RegImm::i64(0), IntCmpKind::Eq, S64)?; 1303 Ok(TypedReg::i32(reg)) // Return value for `i64.eqz` is an `i32`. 1304 }) 1305 } 1306 1307 fn visit_i32_clz(&mut self) -> Self::Output { 1308 use OperandSize::*; 1309 1310 self.context.unop(self.masm, |masm, reg| { 1311 masm.clz(writable!(reg), reg, S32)?; 1312 Ok(TypedReg::i32(reg)) 1313 }) 1314 } 1315 1316 fn visit_i64_clz(&mut self) -> Self::Output { 1317 use OperandSize::*; 1318 1319 self.context.unop(self.masm, |masm, reg| { 1320 masm.clz(writable!(reg), reg, S64)?; 1321 Ok(TypedReg::i64(reg)) 1322 }) 1323 } 1324 1325 fn visit_i32_ctz(&mut self) -> Self::Output { 1326 use OperandSize::*; 1327 1328 self.context.unop(self.masm, |masm, reg| { 1329 masm.ctz(writable!(reg), reg, S32)?; 1330 Ok(TypedReg::i32(reg)) 1331 }) 1332 } 1333 1334 fn visit_i64_ctz(&mut self) -> Self::Output { 1335 use OperandSize::*; 1336 1337 self.context.unop(self.masm, |masm, reg| { 1338 masm.ctz(writable!(reg), reg, S64)?; 1339 Ok(TypedReg::i64(reg)) 1340 }) 1341 } 1342 1343 fn visit_i32_and(&mut self) -> Self::Output { 1344 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1345 masm.and(writable!(dst), dst, src, size)?; 1346 Ok(TypedReg::i32(dst)) 1347 }) 1348 } 1349 1350 fn visit_i64_and(&mut self) -> Self::Output { 1351 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1352 masm.and(writable!(dst), dst, src, size)?; 1353 Ok(TypedReg::i64(dst)) 1354 }) 1355 } 1356 1357 fn visit_i32_or(&mut self) -> Self::Output { 1358 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1359 masm.or(writable!(dst), dst, src, size)?; 1360 Ok(TypedReg::i32(dst)) 1361 }) 1362 } 1363 1364 fn visit_i64_or(&mut self) -> Self::Output { 1365 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1366 masm.or(writable!(dst), dst, src, size)?; 1367 Ok(TypedReg::i64(dst)) 1368 }) 1369 } 1370 1371 fn visit_i32_xor(&mut self) -> Self::Output { 1372 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1373 masm.xor(writable!(dst), dst, src, size)?; 1374 Ok(TypedReg::i32(dst)) 1375 }) 1376 } 1377 1378 fn visit_i64_xor(&mut self) -> Self::Output { 1379 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1380 masm.xor(writable!(dst), dst, src, size)?; 1381 Ok(TypedReg::i64(dst)) 1382 }) 1383 } 1384 1385 fn visit_i32_shl(&mut self) -> Self::Output { 1386 use ShiftKind::*; 1387 1388 self.context.i32_shift(self.masm, Shl) 1389 } 1390 1391 fn visit_i64_shl(&mut self) -> Self::Output { 1392 use ShiftKind::*; 1393 1394 self.context.i64_shift(self.masm, Shl) 1395 } 1396 1397 fn visit_i32_shr_s(&mut self) -> Self::Output { 1398 use ShiftKind::*; 1399 1400 self.context.i32_shift(self.masm, ShrS) 1401 } 1402 1403 fn visit_i64_shr_s(&mut self) -> Self::Output { 1404 use ShiftKind::*; 1405 1406 self.context.i64_shift(self.masm, ShrS) 1407 } 1408 1409 fn visit_i32_shr_u(&mut self) -> Self::Output { 1410 use ShiftKind::*; 1411 1412 self.context.i32_shift(self.masm, ShrU) 1413 } 1414 1415 fn visit_i64_shr_u(&mut self) -> Self::Output { 1416 use ShiftKind::*; 1417 1418 self.context.i64_shift(self.masm, ShrU) 1419 } 1420 1421 fn visit_i32_rotl(&mut self) -> Self::Output { 1422 use ShiftKind::*; 1423 1424 self.context.i32_shift(self.masm, Rotl) 1425 } 1426 1427 fn visit_i64_rotl(&mut self) -> Self::Output { 1428 use ShiftKind::*; 1429 1430 self.context.i64_shift(self.masm, Rotl) 1431 } 1432 1433 fn visit_i32_rotr(&mut self) -> Self::Output { 1434 use ShiftKind::*; 1435 1436 self.context.i32_shift(self.masm, Rotr) 1437 } 1438 1439 fn visit_i64_rotr(&mut self) -> Self::Output { 1440 use ShiftKind::*; 1441 1442 self.context.i64_shift(self.masm, Rotr) 1443 } 1444 1445 fn visit_end(&mut self) -> Self::Output { 1446 if !self.context.reachable { 1447 self.handle_unreachable_end() 1448 } else { 1449 let mut control = self.pop_control_frame()?; 1450 control.emit_end(self.masm, &mut self.context) 1451 } 1452 } 1453 1454 fn visit_i32_popcnt(&mut self) -> Self::Output { 1455 use OperandSize::*; 1456 self.masm.popcnt(&mut self.context, S32) 1457 } 1458 1459 fn visit_i64_popcnt(&mut self) -> Self::Output { 1460 use OperandSize::*; 1461 1462 self.masm.popcnt(&mut self.context, S64) 1463 } 1464 1465 fn visit_i32_wrap_i64(&mut self) -> Self::Output { 1466 self.context.unop(self.masm, |masm, reg| { 1467 masm.wrap(writable!(reg), reg)?; 1468 Ok(TypedReg::i32(reg)) 1469 }) 1470 } 1471 1472 fn visit_i64_extend_i32_s(&mut self) -> Self::Output { 1473 self.context.unop(self.masm, |masm, reg| { 1474 masm.extend(writable!(reg), reg, Extend::<Signed>::I64Extend32.into())?; 1475 Ok(TypedReg::i64(reg)) 1476 }) 1477 } 1478 1479 fn visit_i64_extend_i32_u(&mut self) -> Self::Output { 1480 self.context.unop(self.masm, |masm, reg| { 1481 masm.extend(writable!(reg), reg, Extend::<Zero>::I64Extend32.into())?; 1482 Ok(TypedReg::i64(reg)) 1483 }) 1484 } 1485 1486 fn visit_i32_extend8_s(&mut self) -> Self::Output { 1487 self.context.unop(self.masm, |masm, reg| { 1488 masm.extend(writable!(reg), reg, Extend::<Signed>::I32Extend8.into())?; 1489 Ok(TypedReg::i32(reg)) 1490 }) 1491 } 1492 1493 fn visit_i32_extend16_s(&mut self) -> Self::Output { 1494 self.context.unop(self.masm, |masm, reg| { 1495 masm.extend(writable!(reg), reg, Extend::<Signed>::I32Extend16.into())?; 1496 Ok(TypedReg::i32(reg)) 1497 }) 1498 } 1499 1500 fn visit_i64_extend8_s(&mut self) -> Self::Output { 1501 self.context.unop(self.masm, |masm, reg| { 1502 masm.extend(writable!(reg), reg, Extend::<Signed>::I64Extend8.into())?; 1503 Ok(TypedReg::i64(reg)) 1504 }) 1505 } 1506 1507 fn visit_i64_extend16_s(&mut self) -> Self::Output { 1508 self.context.unop(self.masm, |masm, reg| { 1509 masm.extend(writable!(reg), reg, Extend::<Signed>::I64Extend16.into())?; 1510 Ok(TypedReg::i64(reg)) 1511 }) 1512 } 1513 1514 fn visit_i64_extend32_s(&mut self) -> Self::Output { 1515 self.context.unop(self.masm, |masm, reg| { 1516 masm.extend(writable!(reg), reg, Extend::<Signed>::I64Extend32.into())?; 1517 Ok(TypedReg::i64(reg)) 1518 }) 1519 } 1520 1521 fn visit_i32_trunc_f32_s(&mut self) -> Self::Output { 1522 use OperandSize::*; 1523 1524 self.context 1525 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1526 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Unchecked) 1527 }) 1528 } 1529 1530 fn visit_i32_trunc_f32_u(&mut self) -> Self::Output { 1531 use OperandSize::*; 1532 1533 self.masm 1534 .unsigned_truncate(&mut self.context, S32, S32, TruncKind::Unchecked) 1535 } 1536 1537 fn visit_i32_trunc_f64_s(&mut self) -> Self::Output { 1538 use OperandSize::*; 1539 1540 self.context 1541 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1542 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Unchecked) 1543 }) 1544 } 1545 1546 fn visit_i32_trunc_f64_u(&mut self) -> Self::Output { 1547 use OperandSize::*; 1548 self.masm 1549 .unsigned_truncate(&mut self.context, S64, S32, TruncKind::Unchecked) 1550 } 1551 1552 fn visit_i64_trunc_f32_s(&mut self) -> Self::Output { 1553 use OperandSize::*; 1554 1555 self.context 1556 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1557 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Unchecked) 1558 }) 1559 } 1560 1561 fn visit_i64_trunc_f32_u(&mut self) -> Self::Output { 1562 use OperandSize::*; 1563 1564 self.masm 1565 .unsigned_truncate(&mut self.context, S32, S64, TruncKind::Unchecked) 1566 } 1567 1568 fn visit_i64_trunc_f64_s(&mut self) -> Self::Output { 1569 use OperandSize::*; 1570 1571 self.context 1572 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1573 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Unchecked) 1574 }) 1575 } 1576 1577 fn visit_i64_trunc_f64_u(&mut self) -> Self::Output { 1578 use OperandSize::*; 1579 1580 self.masm 1581 .unsigned_truncate(&mut self.context, S64, S64, TruncKind::Unchecked) 1582 } 1583 1584 fn visit_i32_reinterpret_f32(&mut self) -> Self::Output { 1585 self.context 1586 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, size| { 1587 masm.reinterpret_float_as_int(writable!(dst), src, size) 1588 }) 1589 } 1590 1591 fn visit_i64_reinterpret_f64(&mut self) -> Self::Output { 1592 self.context 1593 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, size| { 1594 masm.reinterpret_float_as_int(writable!(dst), src, size) 1595 }) 1596 } 1597 1598 fn visit_local_get(&mut self, index: u32) -> Self::Output { 1599 use WasmValType::*; 1600 let context = &mut self.context; 1601 let slot = context.frame.get_wasm_local(index); 1602 match slot.ty { 1603 I32 | I64 | F32 | F64 | V128 => context.stack.push(Val::local(index, slot.ty)), 1604 Ref(rt) => match rt.heap_type { 1605 WasmHeapType::Func => context.stack.push(Val::local(index, slot.ty)), 1606 _ => bail!(CodeGenError::unsupported_wasm_type()), 1607 }, 1608 } 1609 1610 Ok(()) 1611 } 1612 1613 fn visit_local_set(&mut self, index: u32) -> Self::Output { 1614 let src = self.emit_set_local(index)?; 1615 self.context.free_reg(src); 1616 Ok(()) 1617 } 1618 1619 fn visit_call(&mut self, index: u32) -> Self::Output { 1620 let callee = self.env.callee_from_index(FuncIndex::from_u32(index)); 1621 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee)?; 1622 Ok(()) 1623 } 1624 1625 fn visit_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output { 1626 // Spill now because `emit_lazy_init_funcref` and the `FnCall::emit` 1627 // invocations will both trigger spills since they both call functions. 1628 // However, the machine instructions for the spill emitted by 1629 // `emit_lazy_funcref` will be jumped over if the funcref was previously 1630 // initialized which may result in the machine stack becoming 1631 // unbalanced. 1632 self.context.spill(self.masm)?; 1633 1634 let type_index = TypeIndex::from_u32(type_index); 1635 let table_index = TableIndex::from_u32(table_index); 1636 1637 self.emit_lazy_init_funcref(table_index)?; 1638 1639 // Perform the indirect call. 1640 // This code assumes that [`Self::emit_lazy_init_funcref`] will 1641 // push the funcref to the value stack. 1642 let funcref_ptr = self 1643 .context 1644 .stack 1645 .peek() 1646 .map(|v| v.unwrap_reg()) 1647 .ok_or_else(|| CodeGenError::missing_values_in_stack())?; 1648 self.masm 1649 .trapz(funcref_ptr.into(), TRAP_INDIRECT_CALL_TO_NULL)?; 1650 self.emit_typecheck_funcref(funcref_ptr.into(), type_index)?; 1651 1652 let callee = self.env.funcref(type_index); 1653 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee)?; 1654 Ok(()) 1655 } 1656 1657 fn visit_table_init(&mut self, elem: u32, table: u32) -> Self::Output { 1658 let at = self.context.stack.ensure_index_at(3)?; 1659 1660 self.context 1661 .stack 1662 .insert_many(at, &[table.try_into()?, elem.try_into()?]); 1663 1664 let builtin = self.env.builtins.table_init::<M::ABI>()?; 1665 FnCall::emit::<M>( 1666 &mut self.env, 1667 self.masm, 1668 &mut self.context, 1669 Callee::Builtin(builtin.clone()), 1670 )?; 1671 self.context.pop_and_free(self.masm) 1672 } 1673 1674 fn visit_table_copy(&mut self, dst: u32, src: u32) -> Self::Output { 1675 let at = self.context.stack.ensure_index_at(3)?; 1676 self.context 1677 .stack 1678 .insert_many(at, &[dst.try_into()?, src.try_into()?]); 1679 1680 let builtin = self.env.builtins.table_copy::<M::ABI>()?; 1681 FnCall::emit::<M>( 1682 &mut self.env, 1683 self.masm, 1684 &mut self.context, 1685 Callee::Builtin(builtin), 1686 )?; 1687 self.context.pop_and_free(self.masm) 1688 } 1689 1690 fn visit_table_get(&mut self, table: u32) -> Self::Output { 1691 let table_index = TableIndex::from_u32(table); 1692 let table = self.env.table(table_index); 1693 let heap_type = table.ref_type.heap_type; 1694 1695 match heap_type { 1696 WasmHeapType::Func => self.emit_lazy_init_funcref(table_index), 1697 _ => Err(format_err!(CodeGenError::unsupported_wasm_type())), 1698 } 1699 } 1700 1701 fn visit_table_grow(&mut self, table: u32) -> Self::Output { 1702 let table_index = TableIndex::from_u32(table); 1703 let table_ty = self.env.table(table_index); 1704 let builtin = match table_ty.ref_type.heap_type { 1705 WasmHeapType::Func => self.env.builtins.table_grow_func_ref::<M::ABI>()?, 1706 _ => bail!(CodeGenError::unsupported_wasm_type()), 1707 }; 1708 1709 let len = self.context.stack.len(); 1710 // table.grow` requires at least 2 elements on the value stack. 1711 let at = self.context.stack.ensure_index_at(2)?; 1712 1713 // The table_grow builtin expects the parameters in a different 1714 // order. 1715 // The value stack at this point should contain: 1716 // [ init_value | delta ] (stack top) 1717 // but the builtin function expects the init value as the last 1718 // argument. 1719 self.context.stack.inner_mut().swap(len - 1, len - 2); 1720 1721 let builtin = self.prepare_builtin_defined_table_arg(table_index, at, builtin)?; 1722 1723 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, builtin)?; 1724 1725 Ok(()) 1726 } 1727 1728 fn visit_table_size(&mut self, table: u32) -> Self::Output { 1729 let table_index = TableIndex::from_u32(table); 1730 let table_data = self.env.resolve_table_data(table_index); 1731 self.emit_compute_table_size(&table_data) 1732 } 1733 1734 fn visit_table_fill(&mut self, table: u32) -> Self::Output { 1735 let table_index = TableIndex::from_u32(table); 1736 let table_ty = self.env.table(table_index); 1737 1738 ensure!( 1739 table_ty.ref_type.heap_type == WasmHeapType::Func, 1740 CodeGenError::unsupported_wasm_type() 1741 ); 1742 1743 let builtin = self.env.builtins.table_fill_func_ref::<M::ABI>()?; 1744 1745 let at = self.context.stack.ensure_index_at(3)?; 1746 1747 self.context.stack.insert_many(at, &[table.try_into()?]); 1748 FnCall::emit::<M>( 1749 &mut self.env, 1750 self.masm, 1751 &mut self.context, 1752 Callee::Builtin(builtin.clone()), 1753 )?; 1754 self.context.pop_and_free(self.masm) 1755 } 1756 1757 fn visit_table_set(&mut self, table: u32) -> Self::Output { 1758 let ptr_type = self.env.ptr_type(); 1759 let table_index = TableIndex::from_u32(table); 1760 let table_data = self.env.resolve_table_data(table_index); 1761 let table = self.env.table(table_index); 1762 match table.ref_type.heap_type { 1763 WasmHeapType::Func => { 1764 ensure!( 1765 self.tunables.table_lazy_init, 1766 CodeGenError::unsupported_table_eager_init() 1767 ); 1768 let value = self.context.pop_to_reg(self.masm, None)?; 1769 let index = self.context.pop_to_reg(self.masm, None)?; 1770 let base = self.context.any_gpr(self.masm)?; 1771 let elem_addr = 1772 self.emit_compute_table_elem_addr(index.into(), base, &table_data)?; 1773 // Set the initialized bit. 1774 self.masm.or( 1775 writable!(value.into()), 1776 value.into(), 1777 RegImm::i64(FUNCREF_INIT_BIT as i64), 1778 ptr_type.try_into()?, 1779 )?; 1780 1781 self.masm.store_ptr(value.into(), elem_addr)?; 1782 1783 self.context.free_reg(value); 1784 self.context.free_reg(index); 1785 self.context.free_reg(base); 1786 Ok(()) 1787 } 1788 _ => Err(format_err!(CodeGenError::unsupported_wasm_type())), 1789 } 1790 } 1791 1792 fn visit_elem_drop(&mut self, index: u32) -> Self::Output { 1793 let elem_drop = self.env.builtins.elem_drop::<M::ABI>()?; 1794 self.context.stack.extend([index.try_into()?]); 1795 FnCall::emit::<M>( 1796 &mut self.env, 1797 self.masm, 1798 &mut self.context, 1799 Callee::Builtin(elem_drop), 1800 )?; 1801 Ok(()) 1802 } 1803 1804 fn visit_memory_init(&mut self, data_index: u32, mem: u32) -> Self::Output { 1805 let at = self.context.stack.ensure_index_at(3)?; 1806 self.context 1807 .stack 1808 .insert_many(at, &[mem.try_into()?, data_index.try_into()?]); 1809 let builtin = self.env.builtins.memory_init::<M::ABI>()?; 1810 FnCall::emit::<M>( 1811 &mut self.env, 1812 self.masm, 1813 &mut self.context, 1814 Callee::Builtin(builtin), 1815 )?; 1816 self.context.pop_and_free(self.masm) 1817 } 1818 1819 fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) -> Self::Output { 1820 // At this point, the stack is expected to contain: 1821 // [ dst_offset, src_offset, len ] 1822 // The following code inserts the missing params, so that stack contains: 1823 // [ vmctx, dst_mem, dst_offset, src_mem, src_offset, len ] 1824 // Which is the order expected by the builtin function. 1825 let _ = self.context.stack.ensure_index_at(3)?; 1826 let at = self.context.stack.ensure_index_at(2)?; 1827 self.context.stack.insert_many(at, &[src_mem.try_into()?]); 1828 1829 // One element was inserted above, so instead of 3, we use 4. 1830 let at = self.context.stack.ensure_index_at(4)?; 1831 self.context.stack.insert_many(at, &[dst_mem.try_into()?]); 1832 1833 let builtin = self.env.builtins.memory_copy::<M::ABI>()?; 1834 1835 FnCall::emit::<M>( 1836 &mut self.env, 1837 self.masm, 1838 &mut self.context, 1839 Callee::Builtin(builtin), 1840 )?; 1841 self.context.pop_and_free(self.masm) 1842 } 1843 1844 fn visit_memory_fill(&mut self, mem: u32) -> Self::Output { 1845 let at = self.context.stack.ensure_index_at(3)?; 1846 let mem = MemoryIndex::from_u32(mem); 1847 1848 let builtin = self.env.builtins.memory_fill::<M::ABI>()?; 1849 let builtin = self.prepare_builtin_defined_memory_arg(mem, at, builtin)?; 1850 1851 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, builtin)?; 1852 self.context.pop_and_free(self.masm) 1853 } 1854 1855 fn visit_memory_size(&mut self, mem: u32) -> Self::Output { 1856 let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem)); 1857 self.emit_compute_memory_size(&heap) 1858 } 1859 1860 fn visit_memory_grow(&mut self, mem: u32) -> Self::Output { 1861 let at = self.context.stack.ensure_index_at(1)?; 1862 let mem = MemoryIndex::from_u32(mem); 1863 // The stack at this point contains: [ delta ] 1864 // The desired state is 1865 // [ vmctx, delta, index ] 1866 let builtin = self.env.builtins.memory_grow::<M::ABI>()?; 1867 let builtin = self.prepare_builtin_defined_memory_arg(mem, at + 1, builtin)?; 1868 1869 let heap = self.env.resolve_heap(mem); 1870 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, builtin)?; 1871 1872 // The memory32_grow builtin returns a pointer type, therefore we must 1873 // ensure that the return type is representative of the address space of 1874 // the heap type. 1875 match (self.env.ptr_type(), heap.index_type()) { 1876 (WasmValType::I64, WasmValType::I64) => Ok(()), 1877 // When the heap type is smaller than the pointer type, we adjust 1878 // the result of the memory32_grow builtin. 1879 (WasmValType::I64, WasmValType::I32) => { 1880 let top: Reg = self.context.pop_to_reg(self.masm, None)?.into(); 1881 self.masm.wrap(writable!(top), top)?; 1882 self.context.stack.push(TypedReg::i32(top).into()); 1883 Ok(()) 1884 } 1885 _ => Err(format_err!(CodeGenError::unsupported_32_bit_platform())), 1886 } 1887 } 1888 1889 fn visit_data_drop(&mut self, data_index: u32) -> Self::Output { 1890 self.context.stack.extend([data_index.try_into()?]); 1891 1892 let builtin = self.env.builtins.data_drop::<M::ABI>()?; 1893 FnCall::emit::<M>( 1894 &mut self.env, 1895 self.masm, 1896 &mut self.context, 1897 Callee::Builtin(builtin), 1898 ) 1899 } 1900 1901 fn visit_nop(&mut self) -> Self::Output { 1902 Ok(()) 1903 } 1904 1905 fn visit_if(&mut self, blockty: BlockType) -> Self::Output { 1906 self.control_frames.push(ControlStackFrame::r#if( 1907 self.env.resolve_block_sig(blockty)?, 1908 self.masm, 1909 &mut self.context, 1910 )?); 1911 1912 Ok(()) 1913 } 1914 1915 fn visit_else(&mut self) -> Self::Output { 1916 if !self.context.reachable { 1917 self.handle_unreachable_else() 1918 } else { 1919 let control = self 1920 .control_frames 1921 .last_mut() 1922 .ok_or_else(|| CodeGenError::control_frame_expected())?; 1923 control.emit_else(self.masm, &mut self.context) 1924 } 1925 } 1926 1927 fn visit_block(&mut self, blockty: BlockType) -> Self::Output { 1928 self.control_frames.push(ControlStackFrame::block( 1929 self.env.resolve_block_sig(blockty)?, 1930 self.masm, 1931 &mut self.context, 1932 )?); 1933 1934 Ok(()) 1935 } 1936 1937 fn visit_loop(&mut self, blockty: BlockType) -> Self::Output { 1938 self.control_frames.push(ControlStackFrame::r#loop( 1939 self.env.resolve_block_sig(blockty)?, 1940 self.masm, 1941 &mut self.context, 1942 )?); 1943 1944 self.maybe_emit_epoch_check()?; 1945 self.maybe_emit_fuel_check() 1946 } 1947 1948 fn visit_br(&mut self, depth: u32) -> Self::Output { 1949 let index = control_index(depth, self.control_frames.len())?; 1950 let frame = &mut self.control_frames[index]; 1951 self.context 1952 .br::<_, _, UnconditionalBranch>(frame, self.masm, |masm, cx, frame| { 1953 frame.pop_abi_results::<M, _>(cx, masm, |results, _, _| { 1954 Ok(results.ret_area().copied()) 1955 }) 1956 }) 1957 } 1958 1959 fn visit_br_if(&mut self, depth: u32) -> Self::Output { 1960 let index = control_index(depth, self.control_frames.len())?; 1961 let frame = &mut self.control_frames[index]; 1962 frame.set_as_target(); 1963 1964 let top = { 1965 let top = self.context.without::<Result<TypedReg>, M, _>( 1966 frame.results::<M>()?.regs(), 1967 self.masm, 1968 |ctx, masm| ctx.pop_to_reg(masm, None), 1969 )??; 1970 // Explicitly save any live registers and locals before setting up 1971 // the branch state. 1972 // In some cases, calculating the `top` value above, will result in 1973 // a spill, thus the following one will result in a no-op. 1974 self.context.spill(self.masm)?; 1975 frame.top_abi_results::<M, _>( 1976 &mut self.context, 1977 self.masm, 1978 |results, context, masm| { 1979 // In the case of `br_if` there's a possibility that we'll 1980 // exit early from the block or fallthrough, for 1981 // a fallthrough, we cannot rely on the pre-computed return area; 1982 // it must be recalculated so that any values that are 1983 // generated are correctly placed near the current stack 1984 // pointer. 1985 if results.on_stack() { 1986 let stack_consumed = context.stack.sizeof(results.stack_operands_len()); 1987 let base = masm.sp_offset()?.as_u32() - stack_consumed; 1988 let offs = base + results.size(); 1989 Ok(Some(RetArea::sp(SPOffset::from_u32(offs)))) 1990 } else { 1991 Ok(None) 1992 } 1993 }, 1994 )?; 1995 top 1996 }; 1997 1998 // Emit instructions to balance the machine stack. 1999 let current_sp_offset = self.masm.sp_offset()?; 2000 let unbalanced = frame.unbalanced(self.masm)?; 2001 let (label, cmp) = if unbalanced { 2002 (self.masm.get_label()?, IntCmpKind::Eq) 2003 } else { 2004 (*frame.label(), IntCmpKind::Ne) 2005 }; 2006 2007 self.masm 2008 .branch(cmp, top.reg, top.reg.into(), label, OperandSize::S32)?; 2009 self.context.free_reg(top); 2010 2011 if unbalanced { 2012 self.context 2013 .br::<_, _, ConditionalBranch>(frame, self.masm, |_, _, _| Ok(()))?; 2014 2015 // Restore sp_offset to what it was for falling through and emit 2016 // fallthrough label. 2017 self.masm.reset_stack_pointer(current_sp_offset)?; 2018 self.masm.bind(label)?; 2019 } 2020 2021 Ok(()) 2022 } 2023 2024 fn visit_br_table(&mut self, targets: BrTable<'a>) -> Self::Output { 2025 // +1 to account for the default target. 2026 let len = targets.len() + 1; 2027 // SmallVec<[_; 5]> to match the binary emission layer (e.g 2028 // see `JmpTableSeq'), but here we use 5 instead since we 2029 // bundle the default target as the last element in the array. 2030 let mut labels: SmallVec<[_; 5]> = smallvec![]; 2031 for _ in 0..len { 2032 labels.push(self.masm.get_label()?); 2033 } 2034 2035 // Find the innermost target and use it as the relative frame 2036 // for result handling below. 2037 // 2038 // This approach ensures that 2039 // 1. The stack pointer offset is correctly positioned 2040 // according to the expectations of the innermost block end 2041 // sequence. 2042 // 2. We meet the jump site invariants introduced by 2043 // `CodegenContext::br`, which take advantage of Wasm 2044 // semantics given that all jumps are "outward". 2045 let mut innermost = targets.default(); 2046 for target in targets.targets() { 2047 let target = target?; 2048 if target < innermost { 2049 innermost = target; 2050 } 2051 } 2052 2053 let innermost_index = control_index(innermost, self.control_frames.len())?; 2054 let innermost_frame = &mut self.control_frames[innermost_index]; 2055 let innermost_result = innermost_frame.results::<M>()?; 2056 2057 let (index, tmp) = { 2058 let index_and_tmp = self.context.without::<Result<(TypedReg, _)>, M, _>( 2059 innermost_result.regs(), 2060 self.masm, 2061 |cx, masm| Ok((cx.pop_to_reg(masm, None)?, cx.any_gpr(masm)?)), 2062 )??; 2063 2064 // Materialize any constants or locals into their result 2065 // representation, so that when reachability is restored, 2066 // they are correctly located. NB: the results are popped 2067 // in function of the innermost branch specified for 2068 // `br_table`, which implies that the machine stack will 2069 // be correctly balanced, by virtue of calling 2070 // `pop_abi_results`. 2071 2072 // It's possible that we need to balance the stack for the 2073 // rest of the targets, which will be done before emitting 2074 // the unconditional jump below. 2075 innermost_frame.pop_abi_results::<M, _>( 2076 &mut self.context, 2077 self.masm, 2078 |results, _, _| Ok(results.ret_area().copied()), 2079 )?; 2080 index_and_tmp 2081 }; 2082 2083 self.masm.jmp_table(&labels, index.into(), tmp)?; 2084 // Save the original stack pointer offset; we will reset the stack 2085 // pointer to this offset after jumping to each of the targets. Each 2086 // jump might adjust the stack according to the base offset of the 2087 // target. 2088 let current_sp = self.masm.sp_offset()?; 2089 2090 for (t, l) in targets 2091 .targets() 2092 .chain(std::iter::once(Ok(targets.default()))) 2093 .zip(labels.iter()) 2094 { 2095 let control_index = control_index(t?, self.control_frames.len())?; 2096 let frame = &mut self.control_frames[control_index]; 2097 // Reset the stack pointer to its original offset. This is needed 2098 // because each jump will potentially adjust the stack pointer 2099 // according to the base offset of the target. 2100 self.masm.reset_stack_pointer(current_sp)?; 2101 2102 // NB: We don't perform any result handling as it was 2103 // already taken care of above before jumping to the 2104 // jump table. 2105 self.masm.bind(*l)?; 2106 // Ensure that the stack pointer is correctly positioned before 2107 // jumping to the jump table code. 2108 self.context 2109 .br::<_, _, UnconditionalBranch>(frame, self.masm, |_, _, _| Ok(()))?; 2110 } 2111 // Finally reset the stack pointer to the original location. 2112 // The reachability analysis, will ensure it's correctly located 2113 // once reachability is restored. 2114 self.masm.reset_stack_pointer(current_sp)?; 2115 self.context.reachable = false; 2116 self.context.free_reg(index.reg); 2117 self.context.free_reg(tmp); 2118 2119 Ok(()) 2120 } 2121 2122 fn visit_return(&mut self) -> Self::Output { 2123 // Grab the outermost frame, which is the function's body 2124 // frame. We don't rely on [`codegen::control_index`] since 2125 // this frame is implicit and we know that it should exist at 2126 // index 0. 2127 let outermost = &mut self.control_frames[0]; 2128 self.context 2129 .br::<_, _, UnconditionalBranch>(outermost, self.masm, |masm, cx, frame| { 2130 frame.pop_abi_results::<M, _>(cx, masm, |results, _, _| { 2131 Ok(results.ret_area().copied()) 2132 }) 2133 }) 2134 } 2135 2136 fn visit_unreachable(&mut self) -> Self::Output { 2137 self.masm.unreachable()?; 2138 self.context.reachable = false; 2139 // Set the implicit outermost frame as target to perform the necessary 2140 // stack clean up. 2141 let outermost = &mut self.control_frames[0]; 2142 outermost.set_as_target(); 2143 2144 Ok(()) 2145 } 2146 2147 fn visit_local_tee(&mut self, index: u32) -> Self::Output { 2148 let typed_reg = self.emit_set_local(index)?; 2149 self.context.stack.push(typed_reg.into()); 2150 2151 Ok(()) 2152 } 2153 2154 fn visit_global_get(&mut self, global_index: u32) -> Self::Output { 2155 let index = GlobalIndex::from_u32(global_index); 2156 let (ty, base, offset) = self.emit_get_global_addr(index)?; 2157 let addr = self.masm.address_at_reg(base, offset)?; 2158 let dst = self.context.reg_for_type(ty, self.masm)?; 2159 self.masm.load(addr, writable!(dst), ty.try_into()?)?; 2160 self.context.stack.push(Val::reg(dst, ty)); 2161 2162 self.context.free_reg(base); 2163 2164 Ok(()) 2165 } 2166 2167 fn visit_global_set(&mut self, global_index: u32) -> Self::Output { 2168 let index = GlobalIndex::from_u32(global_index); 2169 let (ty, base, offset) = self.emit_get_global_addr(index)?; 2170 let addr = self.masm.address_at_reg(base, offset)?; 2171 2172 let typed_reg = self.context.pop_to_reg(self.masm, None)?; 2173 self.masm 2174 .store(typed_reg.reg.into(), addr, ty.try_into()?)?; 2175 self.context.free_reg(typed_reg.reg); 2176 self.context.free_reg(base); 2177 2178 Ok(()) 2179 } 2180 2181 fn visit_drop(&mut self) -> Self::Output { 2182 self.context.drop_last(1, |regalloc, val| match val { 2183 Val::Reg(tr) => Ok(regalloc.free(tr.reg)), 2184 Val::Memory(m) => self.masm.free_stack(m.slot.size), 2185 _ => Ok(()), 2186 }) 2187 } 2188 2189 fn visit_select(&mut self) -> Self::Output { 2190 let cond = self.context.pop_to_reg(self.masm, None)?; 2191 let val2 = self.context.pop_to_reg(self.masm, None)?; 2192 let val1 = self.context.pop_to_reg(self.masm, None)?; 2193 self.masm.cmp(cond.reg, RegImm::i32(0), OperandSize::S32)?; 2194 // Conditionally move val1 to val2 if the comparison is 2195 // not zero. 2196 self.masm.cmov( 2197 writable!(val2.into()), 2198 val1.into(), 2199 IntCmpKind::Ne, 2200 val1.ty.try_into()?, 2201 )?; 2202 self.context.stack.push(val2.into()); 2203 self.context.free_reg(val1.reg); 2204 self.context.free_reg(cond); 2205 2206 Ok(()) 2207 } 2208 2209 fn visit_i32_load(&mut self, memarg: MemArg) -> Self::Output { 2210 self.emit_wasm_load( 2211 &memarg, 2212 WasmValType::I32, 2213 LoadKind::Operand(OperandSize::S32), 2214 ) 2215 } 2216 2217 fn visit_i32_load8_s(&mut self, memarg: MemArg) -> Self::Output { 2218 self.emit_wasm_load( 2219 &memarg, 2220 WasmValType::I32, 2221 LoadKind::ScalarExtend(Extend::<Signed>::I32Extend8.into()), 2222 ) 2223 } 2224 2225 fn visit_i32_load8_u(&mut self, memarg: MemArg) -> Self::Output { 2226 self.emit_wasm_load( 2227 &memarg, 2228 WasmValType::I32, 2229 LoadKind::ScalarExtend(Extend::<Zero>::I32Extend8.into()), 2230 ) 2231 } 2232 2233 fn visit_i32_load16_s(&mut self, memarg: MemArg) -> Self::Output { 2234 self.emit_wasm_load( 2235 &memarg, 2236 WasmValType::I32, 2237 LoadKind::ScalarExtend(Extend::<Signed>::I32Extend16.into()), 2238 ) 2239 } 2240 2241 fn visit_i32_load16_u(&mut self, memarg: MemArg) -> Self::Output { 2242 self.emit_wasm_load( 2243 &memarg, 2244 WasmValType::I32, 2245 LoadKind::ScalarExtend(Extend::<Zero>::I32Extend16.into()), 2246 ) 2247 } 2248 2249 fn visit_i32_store(&mut self, memarg: MemArg) -> Self::Output { 2250 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S32)) 2251 } 2252 2253 fn visit_i32_store8(&mut self, memarg: MemArg) -> Self::Output { 2254 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S8)) 2255 } 2256 2257 fn visit_i32_store16(&mut self, memarg: MemArg) -> Self::Output { 2258 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S16)) 2259 } 2260 2261 fn visit_i64_load8_s(&mut self, memarg: MemArg) -> Self::Output { 2262 self.emit_wasm_load( 2263 &memarg, 2264 WasmValType::I64, 2265 LoadKind::ScalarExtend(Extend::<Signed>::I64Extend8.into()), 2266 ) 2267 } 2268 2269 fn visit_i64_load8_u(&mut self, memarg: MemArg) -> Self::Output { 2270 self.emit_wasm_load( 2271 &memarg, 2272 WasmValType::I64, 2273 LoadKind::ScalarExtend(Extend::<Zero>::I64Extend8.into()), 2274 ) 2275 } 2276 2277 fn visit_i64_load16_u(&mut self, memarg: MemArg) -> Self::Output { 2278 self.emit_wasm_load( 2279 &memarg, 2280 WasmValType::I64, 2281 LoadKind::ScalarExtend(Extend::<Zero>::I64Extend16.into()), 2282 ) 2283 } 2284 2285 fn visit_i64_load16_s(&mut self, memarg: MemArg) -> Self::Output { 2286 self.emit_wasm_load( 2287 &memarg, 2288 WasmValType::I64, 2289 LoadKind::ScalarExtend(Extend::<Signed>::I64Extend16.into()), 2290 ) 2291 } 2292 2293 fn visit_i64_load32_u(&mut self, memarg: MemArg) -> Self::Output { 2294 self.emit_wasm_load( 2295 &memarg, 2296 WasmValType::I64, 2297 LoadKind::ScalarExtend(Extend::<Zero>::I64Extend32.into()), 2298 ) 2299 } 2300 2301 fn visit_i64_load32_s(&mut self, memarg: MemArg) -> Self::Output { 2302 self.emit_wasm_load( 2303 &memarg, 2304 WasmValType::I64, 2305 LoadKind::ScalarExtend(Extend::<Signed>::I64Extend32.into()), 2306 ) 2307 } 2308 2309 fn visit_i64_load(&mut self, memarg: MemArg) -> Self::Output { 2310 self.emit_wasm_load( 2311 &memarg, 2312 WasmValType::I64, 2313 LoadKind::Operand(OperandSize::S64), 2314 ) 2315 } 2316 2317 fn visit_i64_store(&mut self, memarg: MemArg) -> Self::Output { 2318 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S64)) 2319 } 2320 2321 fn visit_i64_store8(&mut self, memarg: MemArg) -> Self::Output { 2322 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S8)) 2323 } 2324 2325 fn visit_i64_store16(&mut self, memarg: MemArg) -> Self::Output { 2326 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S16)) 2327 } 2328 2329 fn visit_i64_store32(&mut self, memarg: MemArg) -> Self::Output { 2330 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S32)) 2331 } 2332 2333 fn visit_f32_load(&mut self, memarg: MemArg) -> Self::Output { 2334 self.emit_wasm_load( 2335 &memarg, 2336 WasmValType::F32, 2337 LoadKind::Operand(OperandSize::S32), 2338 ) 2339 } 2340 2341 fn visit_f32_store(&mut self, memarg: MemArg) -> Self::Output { 2342 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S32)) 2343 } 2344 2345 fn visit_f64_load(&mut self, memarg: MemArg) -> Self::Output { 2346 self.emit_wasm_load( 2347 &memarg, 2348 WasmValType::F64, 2349 LoadKind::Operand(OperandSize::S64), 2350 ) 2351 } 2352 2353 fn visit_f64_store(&mut self, memarg: MemArg) -> Self::Output { 2354 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S64)) 2355 } 2356 2357 fn visit_i32_trunc_sat_f32_s(&mut self) -> Self::Output { 2358 use OperandSize::*; 2359 2360 self.context 2361 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2362 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Checked) 2363 }) 2364 } 2365 2366 fn visit_i32_trunc_sat_f32_u(&mut self) -> Self::Output { 2367 use OperandSize::*; 2368 2369 self.masm 2370 .unsigned_truncate(&mut self.context, S32, S32, TruncKind::Checked) 2371 } 2372 2373 fn visit_i32_trunc_sat_f64_s(&mut self) -> Self::Output { 2374 use OperandSize::*; 2375 2376 self.context 2377 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2378 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Checked) 2379 }) 2380 } 2381 2382 fn visit_i32_trunc_sat_f64_u(&mut self) -> Self::Output { 2383 use OperandSize::*; 2384 2385 self.masm 2386 .unsigned_truncate(&mut self.context, S64, S32, TruncKind::Checked) 2387 } 2388 2389 fn visit_i64_trunc_sat_f32_s(&mut self) -> Self::Output { 2390 use OperandSize::*; 2391 2392 self.context 2393 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2394 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Checked) 2395 }) 2396 } 2397 2398 fn visit_i64_trunc_sat_f32_u(&mut self) -> Self::Output { 2399 use OperandSize::*; 2400 2401 self.masm 2402 .unsigned_truncate(&mut self.context, S32, S64, TruncKind::Checked) 2403 } 2404 2405 fn visit_i64_trunc_sat_f64_s(&mut self) -> Self::Output { 2406 use OperandSize::*; 2407 2408 self.context 2409 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2410 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Checked) 2411 }) 2412 } 2413 2414 fn visit_i64_trunc_sat_f64_u(&mut self) -> Self::Output { 2415 use OperandSize::*; 2416 2417 self.masm 2418 .unsigned_truncate(&mut self.context, S64, S64, TruncKind::Checked) 2419 } 2420 2421 fn visit_i64_add128(&mut self) -> Self::Output { 2422 self.context 2423 .binop128(self.masm, |masm, lhs_lo, lhs_hi, rhs_lo, rhs_hi| { 2424 masm.add128( 2425 writable!(lhs_lo), 2426 writable!(lhs_hi), 2427 lhs_lo, 2428 lhs_hi, 2429 rhs_lo, 2430 rhs_hi, 2431 )?; 2432 Ok((TypedReg::i64(lhs_lo), TypedReg::i64(lhs_hi))) 2433 }) 2434 } 2435 2436 fn visit_i64_sub128(&mut self) -> Self::Output { 2437 self.context 2438 .binop128(self.masm, |masm, lhs_lo, lhs_hi, rhs_lo, rhs_hi| { 2439 masm.sub128( 2440 writable!(lhs_lo), 2441 writable!(lhs_hi), 2442 lhs_lo, 2443 lhs_hi, 2444 rhs_lo, 2445 rhs_hi, 2446 )?; 2447 Ok((TypedReg::i64(lhs_lo), TypedReg::i64(lhs_hi))) 2448 }) 2449 } 2450 2451 fn visit_i64_mul_wide_s(&mut self) -> Self::Output { 2452 self.masm.mul_wide(&mut self.context, MulWideKind::Signed) 2453 } 2454 2455 fn visit_i64_mul_wide_u(&mut self) -> Self::Output { 2456 self.masm.mul_wide(&mut self.context, MulWideKind::Unsigned) 2457 } 2458 2459 fn visit_i32_atomic_load8_u(&mut self, memarg: MemArg) -> Self::Output { 2460 self.emit_wasm_load( 2461 &memarg, 2462 WasmValType::I32, 2463 LoadKind::Atomic(OperandSize::S8, Some(Extend::<Zero>::I32Extend8.into())), 2464 ) 2465 } 2466 2467 fn visit_i32_atomic_load16_u(&mut self, memarg: MemArg) -> Self::Output { 2468 self.emit_wasm_load( 2469 &memarg, 2470 WasmValType::I32, 2471 LoadKind::Atomic(OperandSize::S16, Some(Extend::<Zero>::I32Extend16.into())), 2472 ) 2473 } 2474 2475 fn visit_i32_atomic_load(&mut self, memarg: MemArg) -> Self::Output { 2476 self.emit_wasm_load( 2477 &memarg, 2478 WasmValType::I32, 2479 LoadKind::Atomic(OperandSize::S32, None), 2480 ) 2481 } 2482 2483 fn visit_i64_atomic_load8_u(&mut self, memarg: MemArg) -> Self::Output { 2484 self.emit_wasm_load( 2485 &memarg, 2486 WasmValType::I64, 2487 LoadKind::Atomic(OperandSize::S8, Some(Extend::<Zero>::I64Extend8.into())), 2488 ) 2489 } 2490 2491 fn visit_i64_atomic_load16_u(&mut self, memarg: MemArg) -> Self::Output { 2492 self.emit_wasm_load( 2493 &memarg, 2494 WasmValType::I64, 2495 LoadKind::Atomic(OperandSize::S16, Some(Extend::<Zero>::I64Extend16.into())), 2496 ) 2497 } 2498 2499 fn visit_i64_atomic_load32_u(&mut self, memarg: MemArg) -> Self::Output { 2500 self.emit_wasm_load( 2501 &memarg, 2502 WasmValType::I64, 2503 LoadKind::Atomic(OperandSize::S32, Some(Extend::<Zero>::I64Extend32.into())), 2504 ) 2505 } 2506 2507 fn visit_i64_atomic_load(&mut self, memarg: MemArg) -> Self::Output { 2508 self.emit_wasm_load( 2509 &memarg, 2510 WasmValType::I64, 2511 LoadKind::Atomic(OperandSize::S64, None), 2512 ) 2513 } 2514 2515 fn visit_i32_atomic_store(&mut self, memarg: MemArg) -> Self::Output { 2516 self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S32)) 2517 } 2518 2519 fn visit_i64_atomic_store(&mut self, memarg: MemArg) -> Self::Output { 2520 self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S64)) 2521 } 2522 2523 fn visit_i32_atomic_store8(&mut self, memarg: MemArg) -> Self::Output { 2524 self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S8)) 2525 } 2526 2527 fn visit_i32_atomic_store16(&mut self, memarg: MemArg) -> Self::Output { 2528 self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S16)) 2529 } 2530 2531 fn visit_i64_atomic_store8(&mut self, memarg: MemArg) -> Self::Output { 2532 self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S8)) 2533 } 2534 2535 fn visit_i64_atomic_store16(&mut self, memarg: MemArg) -> Self::Output { 2536 self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S16)) 2537 } 2538 2539 fn visit_i64_atomic_store32(&mut self, memarg: MemArg) -> Self::Output { 2540 self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S32)) 2541 } 2542 2543 fn visit_i32_atomic_rmw8_add_u(&mut self, arg: MemArg) -> Self::Output { 2544 self.emit_atomic_rmw( 2545 &arg, 2546 RmwOp::Add, 2547 OperandSize::S8, 2548 Some(Extend::<Zero>::I32Extend8), 2549 ) 2550 } 2551 2552 fn visit_i32_atomic_rmw16_add_u(&mut self, arg: MemArg) -> Self::Output { 2553 self.emit_atomic_rmw( 2554 &arg, 2555 RmwOp::Add, 2556 OperandSize::S16, 2557 Some(Extend::<Zero>::I32Extend16), 2558 ) 2559 } 2560 2561 fn visit_i32_atomic_rmw_add(&mut self, arg: MemArg) -> Self::Output { 2562 self.emit_atomic_rmw(&arg, RmwOp::Add, OperandSize::S32, None) 2563 } 2564 2565 fn visit_i64_atomic_rmw8_add_u(&mut self, arg: MemArg) -> Self::Output { 2566 self.emit_atomic_rmw( 2567 &arg, 2568 RmwOp::Add, 2569 OperandSize::S8, 2570 Some(Extend::<Zero>::I64Extend8), 2571 ) 2572 } 2573 2574 fn visit_i64_atomic_rmw16_add_u(&mut self, arg: MemArg) -> Self::Output { 2575 self.emit_atomic_rmw( 2576 &arg, 2577 RmwOp::Add, 2578 OperandSize::S16, 2579 Some(Extend::<Zero>::I64Extend16), 2580 ) 2581 } 2582 2583 fn visit_i64_atomic_rmw32_add_u(&mut self, arg: MemArg) -> Self::Output { 2584 self.emit_atomic_rmw( 2585 &arg, 2586 RmwOp::Add, 2587 OperandSize::S32, 2588 Some(Extend::<Zero>::I64Extend32), 2589 ) 2590 } 2591 2592 fn visit_i64_atomic_rmw_add(&mut self, arg: MemArg) -> Self::Output { 2593 self.emit_atomic_rmw(&arg, RmwOp::Add, OperandSize::S64, None) 2594 } 2595 2596 fn visit_i32_atomic_rmw8_sub_u(&mut self, arg: MemArg) -> Self::Output { 2597 self.emit_atomic_rmw( 2598 &arg, 2599 RmwOp::Sub, 2600 OperandSize::S8, 2601 Some(Extend::<Zero>::I32Extend8), 2602 ) 2603 } 2604 fn visit_i32_atomic_rmw16_sub_u(&mut self, arg: MemArg) -> Self::Output { 2605 self.emit_atomic_rmw( 2606 &arg, 2607 RmwOp::Sub, 2608 OperandSize::S16, 2609 Some(Extend::<Zero>::I32Extend16), 2610 ) 2611 } 2612 2613 fn visit_i32_atomic_rmw_sub(&mut self, arg: MemArg) -> Self::Output { 2614 self.emit_atomic_rmw(&arg, RmwOp::Sub, OperandSize::S32, None) 2615 } 2616 2617 fn visit_i64_atomic_rmw8_sub_u(&mut self, arg: MemArg) -> Self::Output { 2618 self.emit_atomic_rmw( 2619 &arg, 2620 RmwOp::Sub, 2621 OperandSize::S8, 2622 Some(Extend::<Zero>::I64Extend8), 2623 ) 2624 } 2625 2626 fn visit_i64_atomic_rmw16_sub_u(&mut self, arg: MemArg) -> Self::Output { 2627 self.emit_atomic_rmw( 2628 &arg, 2629 RmwOp::Sub, 2630 OperandSize::S16, 2631 Some(Extend::<Zero>::I64Extend16), 2632 ) 2633 } 2634 2635 fn visit_i64_atomic_rmw32_sub_u(&mut self, arg: MemArg) -> Self::Output { 2636 self.emit_atomic_rmw( 2637 &arg, 2638 RmwOp::Sub, 2639 OperandSize::S32, 2640 Some(Extend::<Zero>::I64Extend32), 2641 ) 2642 } 2643 2644 fn visit_i64_atomic_rmw_sub(&mut self, arg: MemArg) -> Self::Output { 2645 self.emit_atomic_rmw(&arg, RmwOp::Sub, OperandSize::S64, None) 2646 } 2647 2648 fn visit_i32_atomic_rmw8_xchg_u(&mut self, arg: MemArg) -> Self::Output { 2649 self.emit_atomic_rmw( 2650 &arg, 2651 RmwOp::Xchg, 2652 OperandSize::S8, 2653 Some(Extend::<Zero>::I32Extend8), 2654 ) 2655 } 2656 2657 fn visit_i32_atomic_rmw16_xchg_u(&mut self, arg: MemArg) -> Self::Output { 2658 self.emit_atomic_rmw( 2659 &arg, 2660 RmwOp::Xchg, 2661 OperandSize::S16, 2662 Some(Extend::<Zero>::I32Extend16), 2663 ) 2664 } 2665 2666 fn visit_i32_atomic_rmw_xchg(&mut self, arg: MemArg) -> Self::Output { 2667 self.emit_atomic_rmw(&arg, RmwOp::Xchg, OperandSize::S32, None) 2668 } 2669 2670 fn visit_i64_atomic_rmw8_xchg_u(&mut self, arg: MemArg) -> Self::Output { 2671 self.emit_atomic_rmw( 2672 &arg, 2673 RmwOp::Xchg, 2674 OperandSize::S8, 2675 Some(Extend::<Zero>::I64Extend8), 2676 ) 2677 } 2678 2679 fn visit_i64_atomic_rmw16_xchg_u(&mut self, arg: MemArg) -> Self::Output { 2680 self.emit_atomic_rmw( 2681 &arg, 2682 RmwOp::Xchg, 2683 OperandSize::S16, 2684 Some(Extend::<Zero>::I64Extend16), 2685 ) 2686 } 2687 2688 fn visit_i64_atomic_rmw32_xchg_u(&mut self, arg: MemArg) -> Self::Output { 2689 self.emit_atomic_rmw( 2690 &arg, 2691 RmwOp::Xchg, 2692 OperandSize::S32, 2693 Some(Extend::<Zero>::I64Extend32), 2694 ) 2695 } 2696 2697 fn visit_i64_atomic_rmw_xchg(&mut self, arg: MemArg) -> Self::Output { 2698 self.emit_atomic_rmw(&arg, RmwOp::Xchg, OperandSize::S64, None) 2699 } 2700 2701 fn visit_i32_atomic_rmw8_and_u(&mut self, arg: MemArg) -> Self::Output { 2702 self.emit_atomic_rmw( 2703 &arg, 2704 RmwOp::And, 2705 OperandSize::S8, 2706 Some(Extend::<Zero>::I32Extend8), 2707 ) 2708 } 2709 2710 fn visit_i32_atomic_rmw16_and_u(&mut self, arg: MemArg) -> Self::Output { 2711 self.emit_atomic_rmw( 2712 &arg, 2713 RmwOp::And, 2714 OperandSize::S16, 2715 Some(Extend::<Zero>::I32Extend16), 2716 ) 2717 } 2718 2719 fn visit_i32_atomic_rmw_and(&mut self, arg: MemArg) -> Self::Output { 2720 self.emit_atomic_rmw(&arg, RmwOp::And, OperandSize::S32, None) 2721 } 2722 2723 fn visit_i64_atomic_rmw8_and_u(&mut self, arg: MemArg) -> Self::Output { 2724 self.emit_atomic_rmw( 2725 &arg, 2726 RmwOp::And, 2727 OperandSize::S8, 2728 Some(Extend::<Zero>::I64Extend8), 2729 ) 2730 } 2731 2732 fn visit_i64_atomic_rmw16_and_u(&mut self, arg: MemArg) -> Self::Output { 2733 self.emit_atomic_rmw( 2734 &arg, 2735 RmwOp::And, 2736 OperandSize::S16, 2737 Some(Extend::<Zero>::I64Extend16), 2738 ) 2739 } 2740 2741 fn visit_i64_atomic_rmw32_and_u(&mut self, arg: MemArg) -> Self::Output { 2742 self.emit_atomic_rmw( 2743 &arg, 2744 RmwOp::And, 2745 OperandSize::S32, 2746 Some(Extend::<Zero>::I64Extend32), 2747 ) 2748 } 2749 2750 fn visit_i64_atomic_rmw_and(&mut self, arg: MemArg) -> Self::Output { 2751 self.emit_atomic_rmw(&arg, RmwOp::And, OperandSize::S64, None) 2752 } 2753 2754 fn visit_i32_atomic_rmw8_or_u(&mut self, arg: MemArg) -> Self::Output { 2755 self.emit_atomic_rmw( 2756 &arg, 2757 RmwOp::Or, 2758 OperandSize::S8, 2759 Some(Extend::<Zero>::I32Extend8), 2760 ) 2761 } 2762 2763 fn visit_i32_atomic_rmw16_or_u(&mut self, arg: MemArg) -> Self::Output { 2764 self.emit_atomic_rmw( 2765 &arg, 2766 RmwOp::Or, 2767 OperandSize::S16, 2768 Some(Extend::<Zero>::I32Extend16), 2769 ) 2770 } 2771 2772 fn visit_i32_atomic_rmw_or(&mut self, arg: MemArg) -> Self::Output { 2773 self.emit_atomic_rmw(&arg, RmwOp::Or, OperandSize::S32, None) 2774 } 2775 2776 fn visit_i64_atomic_rmw8_or_u(&mut self, arg: MemArg) -> Self::Output { 2777 self.emit_atomic_rmw( 2778 &arg, 2779 RmwOp::Or, 2780 OperandSize::S8, 2781 Some(Extend::<Zero>::I64Extend8), 2782 ) 2783 } 2784 2785 fn visit_i64_atomic_rmw16_or_u(&mut self, arg: MemArg) -> Self::Output { 2786 self.emit_atomic_rmw( 2787 &arg, 2788 RmwOp::Or, 2789 OperandSize::S16, 2790 Some(Extend::<Zero>::I64Extend16), 2791 ) 2792 } 2793 2794 fn visit_i64_atomic_rmw32_or_u(&mut self, arg: MemArg) -> Self::Output { 2795 self.emit_atomic_rmw( 2796 &arg, 2797 RmwOp::Or, 2798 OperandSize::S32, 2799 Some(Extend::<Zero>::I64Extend32), 2800 ) 2801 } 2802 2803 fn visit_i64_atomic_rmw_or(&mut self, arg: MemArg) -> Self::Output { 2804 self.emit_atomic_rmw(&arg, RmwOp::Or, OperandSize::S64, None) 2805 } 2806 2807 fn visit_i32_atomic_rmw8_xor_u(&mut self, arg: MemArg) -> Self::Output { 2808 self.emit_atomic_rmw( 2809 &arg, 2810 RmwOp::Xor, 2811 OperandSize::S8, 2812 Some(Extend::<Zero>::I32Extend8), 2813 ) 2814 } 2815 2816 fn visit_i32_atomic_rmw16_xor_u(&mut self, arg: MemArg) -> Self::Output { 2817 self.emit_atomic_rmw( 2818 &arg, 2819 RmwOp::Xor, 2820 OperandSize::S16, 2821 Some(Extend::<Zero>::I32Extend16), 2822 ) 2823 } 2824 2825 fn visit_i32_atomic_rmw_xor(&mut self, arg: MemArg) -> Self::Output { 2826 self.emit_atomic_rmw(&arg, RmwOp::Xor, OperandSize::S32, None) 2827 } 2828 2829 fn visit_i64_atomic_rmw8_xor_u(&mut self, arg: MemArg) -> Self::Output { 2830 self.emit_atomic_rmw( 2831 &arg, 2832 RmwOp::Xor, 2833 OperandSize::S8, 2834 Some(Extend::<Zero>::I64Extend8), 2835 ) 2836 } 2837 2838 fn visit_i64_atomic_rmw16_xor_u(&mut self, arg: MemArg) -> Self::Output { 2839 self.emit_atomic_rmw( 2840 &arg, 2841 RmwOp::Xor, 2842 OperandSize::S16, 2843 Some(Extend::<Zero>::I64Extend16), 2844 ) 2845 } 2846 2847 fn visit_i64_atomic_rmw32_xor_u(&mut self, arg: MemArg) -> Self::Output { 2848 self.emit_atomic_rmw( 2849 &arg, 2850 RmwOp::Xor, 2851 OperandSize::S32, 2852 Some(Extend::<Zero>::I64Extend32), 2853 ) 2854 } 2855 2856 fn visit_i64_atomic_rmw_xor(&mut self, arg: MemArg) -> Self::Output { 2857 self.emit_atomic_rmw(&arg, RmwOp::Xor, OperandSize::S64, None) 2858 } 2859 2860 fn visit_i32_atomic_rmw8_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output { 2861 self.emit_atomic_cmpxchg(&arg, OperandSize::S8, Some(Extend::I32Extend8)) 2862 } 2863 2864 fn visit_i32_atomic_rmw16_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output { 2865 self.emit_atomic_cmpxchg(&arg, OperandSize::S16, Some(Extend::I32Extend16)) 2866 } 2867 2868 fn visit_i32_atomic_rmw_cmpxchg(&mut self, arg: MemArg) -> Self::Output { 2869 self.emit_atomic_cmpxchg(&arg, OperandSize::S32, None) 2870 } 2871 2872 fn visit_i64_atomic_rmw8_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output { 2873 self.emit_atomic_cmpxchg(&arg, OperandSize::S8, Some(Extend::I64Extend8)) 2874 } 2875 2876 fn visit_i64_atomic_rmw16_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output { 2877 self.emit_atomic_cmpxchg(&arg, OperandSize::S16, Some(Extend::I64Extend16)) 2878 } 2879 2880 fn visit_i64_atomic_rmw32_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output { 2881 self.emit_atomic_cmpxchg(&arg, OperandSize::S32, Some(Extend::I64Extend32)) 2882 } 2883 2884 fn visit_i64_atomic_rmw_cmpxchg(&mut self, arg: MemArg) -> Self::Output { 2885 self.emit_atomic_cmpxchg(&arg, OperandSize::S64, None) 2886 } 2887 2888 fn visit_memory_atomic_wait32(&mut self, arg: MemArg) -> Self::Output { 2889 self.emit_atomic_wait(&arg, AtomicWaitKind::Wait32) 2890 } 2891 2892 fn visit_memory_atomic_wait64(&mut self, arg: MemArg) -> Self::Output { 2893 self.emit_atomic_wait(&arg, AtomicWaitKind::Wait64) 2894 } 2895 2896 fn visit_memory_atomic_notify(&mut self, arg: MemArg) -> Self::Output { 2897 self.emit_atomic_notify(&arg) 2898 } 2899 2900 fn visit_atomic_fence(&mut self) -> Self::Output { 2901 self.masm.fence() 2902 } 2903 2904 wasmparser::for_each_visit_operator!(def_unsupported); 2905 } 2906 2907 impl<'a, 'translation, 'data, M> VisitSimdOperator<'a> 2908 for CodeGen<'a, 'translation, 'data, M, Emission> 2909 where 2910 M: MacroAssembler, 2911 { 2912 fn visit_v128_const(&mut self, val: V128) -> Self::Output { 2913 self.context.stack.push(Val::v128(val.i128())); 2914 Ok(()) 2915 } 2916 2917 fn visit_v128_load(&mut self, memarg: MemArg) -> Self::Output { 2918 self.emit_wasm_load( 2919 &memarg, 2920 WasmValType::V128, 2921 LoadKind::Operand(OperandSize::S128), 2922 ) 2923 } 2924 2925 fn visit_v128_store(&mut self, memarg: MemArg) -> Self::Output { 2926 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S128)) 2927 } 2928 2929 fn visit_v128_load8x8_s(&mut self, memarg: MemArg) -> Self::Output { 2930 self.emit_wasm_load( 2931 &memarg, 2932 WasmValType::V128, 2933 LoadKind::VectorExtend(V128LoadExtendKind::E8x8S), 2934 ) 2935 } 2936 2937 fn visit_v128_load8x8_u(&mut self, memarg: MemArg) -> Self::Output { 2938 self.emit_wasm_load( 2939 &memarg, 2940 WasmValType::V128, 2941 LoadKind::VectorExtend(V128LoadExtendKind::E8x8U), 2942 ) 2943 } 2944 2945 fn visit_v128_load16x4_s(&mut self, memarg: MemArg) -> Self::Output { 2946 self.emit_wasm_load( 2947 &memarg, 2948 WasmValType::V128, 2949 LoadKind::VectorExtend(V128LoadExtendKind::E16x4S), 2950 ) 2951 } 2952 2953 fn visit_v128_load16x4_u(&mut self, memarg: MemArg) -> Self::Output { 2954 self.emit_wasm_load( 2955 &memarg, 2956 WasmValType::V128, 2957 LoadKind::VectorExtend(V128LoadExtendKind::E16x4U), 2958 ) 2959 } 2960 2961 fn visit_v128_load32x2_s(&mut self, memarg: MemArg) -> Self::Output { 2962 self.emit_wasm_load( 2963 &memarg, 2964 WasmValType::V128, 2965 LoadKind::VectorExtend(V128LoadExtendKind::E32x2S), 2966 ) 2967 } 2968 2969 fn visit_v128_load32x2_u(&mut self, memarg: MemArg) -> Self::Output { 2970 self.emit_wasm_load( 2971 &memarg, 2972 WasmValType::V128, 2973 LoadKind::VectorExtend(V128LoadExtendKind::E32x2U), 2974 ) 2975 } 2976 2977 fn visit_v128_load8_splat(&mut self, memarg: MemArg) -> Self::Output { 2978 self.emit_wasm_load( 2979 &memarg, 2980 WasmValType::V128, 2981 LoadKind::Splat(SplatLoadKind::S8), 2982 ) 2983 } 2984 2985 fn visit_v128_load16_splat(&mut self, memarg: MemArg) -> Self::Output { 2986 self.emit_wasm_load( 2987 &memarg, 2988 WasmValType::V128, 2989 LoadKind::Splat(SplatLoadKind::S16), 2990 ) 2991 } 2992 2993 fn visit_v128_load32_splat(&mut self, memarg: MemArg) -> Self::Output { 2994 self.emit_wasm_load( 2995 &memarg, 2996 WasmValType::V128, 2997 LoadKind::Splat(SplatLoadKind::S32), 2998 ) 2999 } 3000 3001 fn visit_v128_load64_splat(&mut self, memarg: MemArg) -> Self::Output { 3002 self.emit_wasm_load( 3003 &memarg, 3004 WasmValType::V128, 3005 LoadKind::Splat(SplatLoadKind::S64), 3006 ) 3007 } 3008 3009 fn visit_i8x16_splat(&mut self) -> Self::Output { 3010 self.masm.splat(&mut self.context, SplatKind::I8x16) 3011 } 3012 3013 fn visit_i16x8_splat(&mut self) -> Self::Output { 3014 self.masm.splat(&mut self.context, SplatKind::I16x8) 3015 } 3016 3017 fn visit_i32x4_splat(&mut self) -> Self::Output { 3018 self.masm.splat(&mut self.context, SplatKind::I32x4) 3019 } 3020 3021 fn visit_i64x2_splat(&mut self) -> Self::Output { 3022 self.masm.splat(&mut self.context, SplatKind::I64x2) 3023 } 3024 3025 fn visit_f32x4_splat(&mut self) -> Self::Output { 3026 self.masm.splat(&mut self.context, SplatKind::F32x4) 3027 } 3028 3029 fn visit_f64x2_splat(&mut self) -> Self::Output { 3030 self.masm.splat(&mut self.context, SplatKind::F64x2) 3031 } 3032 3033 fn visit_i8x16_shuffle(&mut self, lanes: [u8; 16]) -> Self::Output { 3034 let rhs = self.context.pop_to_reg(self.masm, None)?; 3035 let lhs = self.context.pop_to_reg(self.masm, None)?; 3036 self.masm 3037 .shuffle(writable!(lhs.into()), lhs.into(), rhs.into(), lanes)?; 3038 self.context.stack.push(TypedReg::v128(lhs.into()).into()); 3039 self.context.free_reg(rhs); 3040 Ok(()) 3041 } 3042 3043 fn visit_i8x16_swizzle(&mut self) -> Self::Output { 3044 let rhs = self.context.pop_to_reg(self.masm, None)?; 3045 let lhs = self.context.pop_to_reg(self.masm, None)?; 3046 self.masm 3047 .swizzle(writable!(lhs.into()), lhs.into(), rhs.into())?; 3048 self.context.stack.push(TypedReg::v128(lhs.into()).into()); 3049 self.context.free_reg(rhs); 3050 Ok(()) 3051 } 3052 3053 fn visit_i8x16_extract_lane_s(&mut self, lane: u8) -> Self::Output { 3054 self.context.extract_lane_op( 3055 self.masm, 3056 ExtractLaneKind::I8x16S, 3057 |masm, src, dst, kind| masm.extract_lane(src, dst, lane, kind), 3058 ) 3059 } 3060 3061 fn visit_i8x16_extract_lane_u(&mut self, lane: u8) -> Self::Output { 3062 self.context.extract_lane_op( 3063 self.masm, 3064 ExtractLaneKind::I8x16U, 3065 |masm, src, dst, kind| masm.extract_lane(src, dst, lane, kind), 3066 ) 3067 } 3068 3069 fn visit_i16x8_extract_lane_s(&mut self, lane: u8) -> Self::Output { 3070 self.context.extract_lane_op( 3071 self.masm, 3072 ExtractLaneKind::I16x8S, 3073 |masm, src, dst, kind| masm.extract_lane(src, dst, lane, kind), 3074 ) 3075 } 3076 3077 fn visit_i16x8_extract_lane_u(&mut self, lane: u8) -> Self::Output { 3078 self.context.extract_lane_op( 3079 self.masm, 3080 ExtractLaneKind::I16x8U, 3081 |masm, src, dst, kind| masm.extract_lane(src, dst, lane, kind), 3082 ) 3083 } 3084 3085 fn visit_i32x4_extract_lane(&mut self, lane: u8) -> Self::Output { 3086 self.context 3087 .extract_lane_op(self.masm, ExtractLaneKind::I32x4, |masm, src, dst, kind| { 3088 masm.extract_lane(src, dst, lane, kind) 3089 }) 3090 } 3091 3092 fn visit_i64x2_extract_lane(&mut self, lane: u8) -> Self::Output { 3093 self.context 3094 .extract_lane_op(self.masm, ExtractLaneKind::I64x2, |masm, src, dst, kind| { 3095 masm.extract_lane(src, dst, lane, kind) 3096 }) 3097 } 3098 3099 fn visit_f32x4_extract_lane(&mut self, lane: u8) -> Self::Output { 3100 self.context 3101 .extract_lane_op(self.masm, ExtractLaneKind::F32x4, |masm, src, dst, kind| { 3102 masm.extract_lane(src, dst, lane, kind) 3103 }) 3104 } 3105 3106 fn visit_f64x2_extract_lane(&mut self, lane: u8) -> Self::Output { 3107 self.context 3108 .extract_lane_op(self.masm, ExtractLaneKind::F64x2, |masm, src, dst, kind| { 3109 masm.extract_lane(src, dst, lane, kind) 3110 }) 3111 } 3112 3113 fn visit_i8x16_eq(&mut self) -> Self::Output { 3114 self.context 3115 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3116 masm.v128_eq(writable!(dst), dst, src, VectorEqualityKind::I8x16)?; 3117 Ok(TypedReg::v128(dst)) 3118 }) 3119 } 3120 3121 fn visit_i16x8_eq(&mut self) -> Self::Output { 3122 self.context 3123 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3124 masm.v128_eq(writable!(dst), dst, src, VectorEqualityKind::I16x8)?; 3125 Ok(TypedReg::v128(dst)) 3126 }) 3127 } 3128 3129 fn visit_i32x4_eq(&mut self) -> Self::Output { 3130 self.context 3131 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3132 masm.v128_eq(writable!(dst), dst, src, VectorEqualityKind::I32x4)?; 3133 Ok(TypedReg::v128(dst)) 3134 }) 3135 } 3136 3137 fn visit_i64x2_eq(&mut self) -> Self::Output { 3138 self.context 3139 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3140 masm.v128_eq(writable!(dst), dst, src, VectorEqualityKind::I64x2)?; 3141 Ok(TypedReg::v128(dst)) 3142 }) 3143 } 3144 3145 fn visit_f32x4_eq(&mut self) -> Self::Output { 3146 self.context 3147 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3148 masm.v128_eq(writable!(dst), dst, src, VectorEqualityKind::F32x4)?; 3149 Ok(TypedReg::v128(dst)) 3150 }) 3151 } 3152 3153 fn visit_f64x2_eq(&mut self) -> Self::Output { 3154 self.context 3155 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3156 masm.v128_eq(writable!(dst), dst, src, VectorEqualityKind::F64x2)?; 3157 Ok(TypedReg::v128(dst)) 3158 }) 3159 } 3160 3161 fn visit_i8x16_ne(&mut self) -> Self::Output { 3162 self.context 3163 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3164 masm.v128_ne(writable!(dst), dst, src, VectorEqualityKind::I8x16)?; 3165 Ok(TypedReg::v128(dst)) 3166 }) 3167 } 3168 3169 fn visit_i16x8_ne(&mut self) -> Self::Output { 3170 self.context 3171 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3172 masm.v128_ne(writable!(dst), dst, src, VectorEqualityKind::I16x8)?; 3173 Ok(TypedReg::v128(dst)) 3174 }) 3175 } 3176 3177 fn visit_i32x4_ne(&mut self) -> Self::Output { 3178 self.context 3179 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3180 masm.v128_ne(writable!(dst), dst, src, VectorEqualityKind::I32x4)?; 3181 Ok(TypedReg::v128(dst)) 3182 }) 3183 } 3184 3185 fn visit_i64x2_ne(&mut self) -> Self::Output { 3186 self.context 3187 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3188 masm.v128_ne(writable!(dst), dst, src, VectorEqualityKind::I64x2)?; 3189 Ok(TypedReg::v128(dst)) 3190 }) 3191 } 3192 3193 fn visit_f32x4_ne(&mut self) -> Self::Output { 3194 self.context 3195 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3196 masm.v128_ne(writable!(dst), dst, src, VectorEqualityKind::F32x4)?; 3197 Ok(TypedReg::v128(dst)) 3198 }) 3199 } 3200 3201 fn visit_f64x2_ne(&mut self) -> Self::Output { 3202 self.context 3203 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3204 masm.v128_ne(writable!(dst), dst, src, VectorEqualityKind::F64x2)?; 3205 Ok(TypedReg::v128(dst)) 3206 }) 3207 } 3208 3209 fn visit_i8x16_lt_s(&mut self) -> Self::Output { 3210 self.context 3211 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3212 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I8x16S)?; 3213 Ok(TypedReg::v128(dst)) 3214 }) 3215 } 3216 3217 fn visit_i8x16_lt_u(&mut self) -> Self::Output { 3218 self.context 3219 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3220 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I8x16U)?; 3221 Ok(TypedReg::v128(dst)) 3222 }) 3223 } 3224 3225 fn visit_i16x8_lt_s(&mut self) -> Self::Output { 3226 self.context 3227 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3228 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I16x8S)?; 3229 Ok(TypedReg::v128(dst)) 3230 }) 3231 } 3232 3233 fn visit_i16x8_lt_u(&mut self) -> Self::Output { 3234 self.context 3235 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3236 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I16x8U)?; 3237 Ok(TypedReg::v128(dst)) 3238 }) 3239 } 3240 3241 fn visit_i32x4_lt_s(&mut self) -> Self::Output { 3242 self.context 3243 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3244 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I32x4S)?; 3245 Ok(TypedReg::v128(dst)) 3246 }) 3247 } 3248 3249 fn visit_i32x4_lt_u(&mut self) -> Self::Output { 3250 self.context 3251 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3252 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I32x4U)?; 3253 Ok(TypedReg::v128(dst)) 3254 }) 3255 } 3256 3257 fn visit_i64x2_lt_s(&mut self) -> Self::Output { 3258 self.context 3259 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3260 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I64x2S)?; 3261 Ok(TypedReg::v128(dst)) 3262 }) 3263 } 3264 3265 fn visit_f32x4_lt(&mut self) -> Self::Output { 3266 self.context 3267 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3268 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::F32x4)?; 3269 Ok(TypedReg::v128(dst)) 3270 }) 3271 } 3272 3273 fn visit_f64x2_lt(&mut self) -> Self::Output { 3274 self.context 3275 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3276 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::F64x2)?; 3277 Ok(TypedReg::v128(dst)) 3278 }) 3279 } 3280 3281 fn visit_i8x16_le_s(&mut self) -> Self::Output { 3282 self.context 3283 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3284 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I8x16S)?; 3285 Ok(TypedReg::v128(dst)) 3286 }) 3287 } 3288 3289 fn visit_i8x16_le_u(&mut self) -> Self::Output { 3290 self.context 3291 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3292 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I8x16U)?; 3293 Ok(TypedReg::v128(dst)) 3294 }) 3295 } 3296 3297 fn visit_i16x8_le_s(&mut self) -> Self::Output { 3298 self.context 3299 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3300 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I16x8S)?; 3301 Ok(TypedReg::v128(dst)) 3302 }) 3303 } 3304 3305 fn visit_i16x8_le_u(&mut self) -> Self::Output { 3306 self.context 3307 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3308 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I16x8U)?; 3309 Ok(TypedReg::v128(dst)) 3310 }) 3311 } 3312 3313 fn visit_i32x4_le_s(&mut self) -> Self::Output { 3314 self.context 3315 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3316 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I32x4S)?; 3317 Ok(TypedReg::v128(dst)) 3318 }) 3319 } 3320 3321 fn visit_i32x4_le_u(&mut self) -> Self::Output { 3322 self.context 3323 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3324 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I32x4U)?; 3325 Ok(TypedReg::v128(dst)) 3326 }) 3327 } 3328 3329 fn visit_i64x2_le_s(&mut self) -> Self::Output { 3330 self.context 3331 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3332 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I64x2S)?; 3333 Ok(TypedReg::v128(dst)) 3334 }) 3335 } 3336 3337 fn visit_f32x4_le(&mut self) -> Self::Output { 3338 self.context 3339 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3340 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::F32x4)?; 3341 Ok(TypedReg::v128(dst)) 3342 }) 3343 } 3344 3345 fn visit_f64x2_le(&mut self) -> Self::Output { 3346 self.context 3347 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3348 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::F64x2)?; 3349 Ok(TypedReg::v128(dst)) 3350 }) 3351 } 3352 3353 fn visit_i8x16_gt_s(&mut self) -> Self::Output { 3354 self.context 3355 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3356 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I8x16S)?; 3357 Ok(TypedReg::v128(dst)) 3358 }) 3359 } 3360 3361 fn visit_i8x16_gt_u(&mut self) -> Self::Output { 3362 self.context 3363 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3364 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I8x16U)?; 3365 Ok(TypedReg::v128(dst)) 3366 }) 3367 } 3368 3369 fn visit_i16x8_gt_s(&mut self) -> Self::Output { 3370 self.context 3371 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3372 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I16x8S)?; 3373 Ok(TypedReg::v128(dst)) 3374 }) 3375 } 3376 3377 fn visit_i16x8_gt_u(&mut self) -> Self::Output { 3378 self.context 3379 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3380 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I16x8U)?; 3381 Ok(TypedReg::v128(dst)) 3382 }) 3383 } 3384 3385 fn visit_i32x4_gt_s(&mut self) -> Self::Output { 3386 self.context 3387 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3388 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I32x4S)?; 3389 Ok(TypedReg::v128(dst)) 3390 }) 3391 } 3392 3393 fn visit_i32x4_gt_u(&mut self) -> Self::Output { 3394 self.context 3395 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3396 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I32x4U)?; 3397 Ok(TypedReg::v128(dst)) 3398 }) 3399 } 3400 3401 fn visit_i64x2_gt_s(&mut self) -> Self::Output { 3402 self.context 3403 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3404 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I64x2S)?; 3405 Ok(TypedReg::v128(dst)) 3406 }) 3407 } 3408 3409 fn visit_f32x4_gt(&mut self) -> Self::Output { 3410 self.context 3411 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3412 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::F32x4)?; 3413 Ok(TypedReg::v128(dst)) 3414 }) 3415 } 3416 3417 fn visit_f64x2_gt(&mut self) -> Self::Output { 3418 self.context 3419 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3420 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::F64x2)?; 3421 Ok(TypedReg::v128(dst)) 3422 }) 3423 } 3424 3425 fn visit_i8x16_ge_s(&mut self) -> Self::Output { 3426 self.context 3427 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3428 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I8x16S)?; 3429 Ok(TypedReg::v128(dst)) 3430 }) 3431 } 3432 3433 fn visit_i8x16_ge_u(&mut self) -> Self::Output { 3434 self.context 3435 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3436 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I8x16U)?; 3437 Ok(TypedReg::v128(dst)) 3438 }) 3439 } 3440 3441 fn visit_i16x8_ge_s(&mut self) -> Self::Output { 3442 self.context 3443 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3444 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I16x8S)?; 3445 Ok(TypedReg::v128(dst)) 3446 }) 3447 } 3448 3449 fn visit_i16x8_ge_u(&mut self) -> Self::Output { 3450 self.context 3451 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3452 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I16x8U)?; 3453 Ok(TypedReg::v128(dst)) 3454 }) 3455 } 3456 3457 fn visit_i32x4_ge_s(&mut self) -> Self::Output { 3458 self.context 3459 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3460 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I32x4S)?; 3461 Ok(TypedReg::v128(dst)) 3462 }) 3463 } 3464 3465 fn visit_i32x4_ge_u(&mut self) -> Self::Output { 3466 self.context 3467 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3468 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I32x4U)?; 3469 Ok(TypedReg::v128(dst)) 3470 }) 3471 } 3472 3473 fn visit_i64x2_ge_s(&mut self) -> Self::Output { 3474 self.context 3475 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3476 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I64x2S)?; 3477 Ok(TypedReg::v128(dst)) 3478 }) 3479 } 3480 3481 fn visit_f32x4_ge(&mut self) -> Self::Output { 3482 self.context 3483 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3484 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::F32x4)?; 3485 Ok(TypedReg::v128(dst)) 3486 }) 3487 } 3488 3489 fn visit_f64x2_ge(&mut self) -> Self::Output { 3490 self.context 3491 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3492 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::F64x2)?; 3493 Ok(TypedReg::v128(dst)) 3494 }) 3495 } 3496 3497 fn visit_i8x16_replace_lane(&mut self, lane: u8) -> Self::Output { 3498 self.context 3499 .replace_lane_op(self.masm, ReplaceLaneKind::I8x16, |masm, src, dst, kind| { 3500 masm.replace_lane(src, dst, lane, kind) 3501 }) 3502 } 3503 3504 fn visit_i16x8_replace_lane(&mut self, lane: u8) -> Self::Output { 3505 self.context 3506 .replace_lane_op(self.masm, ReplaceLaneKind::I16x8, |masm, src, dst, kind| { 3507 masm.replace_lane(src, dst, lane, kind) 3508 }) 3509 } 3510 3511 fn visit_i32x4_replace_lane(&mut self, lane: u8) -> Self::Output { 3512 self.context 3513 .replace_lane_op(self.masm, ReplaceLaneKind::I32x4, |masm, src, dst, kind| { 3514 masm.replace_lane(src, dst, lane, kind) 3515 }) 3516 } 3517 3518 fn visit_i64x2_replace_lane(&mut self, lane: u8) -> Self::Output { 3519 self.context 3520 .replace_lane_op(self.masm, ReplaceLaneKind::I64x2, |masm, src, dst, kind| { 3521 masm.replace_lane(src, dst, lane, kind) 3522 }) 3523 } 3524 3525 fn visit_f32x4_replace_lane(&mut self, lane: u8) -> Self::Output { 3526 self.context 3527 .replace_lane_op(self.masm, ReplaceLaneKind::F32x4, |masm, src, dst, kind| { 3528 masm.replace_lane(src, dst, lane, kind) 3529 }) 3530 } 3531 3532 fn visit_f64x2_replace_lane(&mut self, lane: u8) -> Self::Output { 3533 self.context 3534 .replace_lane_op(self.masm, ReplaceLaneKind::F64x2, |masm, src, dst, kind| { 3535 masm.replace_lane(src, dst, lane, kind) 3536 }) 3537 } 3538 3539 fn visit_v128_not(&mut self) -> Self::Output { 3540 self.context.unop(self.masm, |masm, reg| { 3541 masm.v128_not(writable!(reg))?; 3542 Ok(TypedReg::new(WasmValType::V128, reg)) 3543 }) 3544 } 3545 3546 fn visit_v128_and(&mut self) -> Self::Output { 3547 self.context 3548 .binop(self.masm, OperandSize::S128, |masm, dst, src, _size| { 3549 masm.v128_and(dst, src, writable!(dst))?; 3550 Ok(TypedReg::new(WasmValType::V128, dst)) 3551 }) 3552 } 3553 3554 fn visit_v128_andnot(&mut self) -> Self::Output { 3555 self.context 3556 .binop(self.masm, OperandSize::S128, |masm, dst, src, _size| { 3557 // careful here: and_not is *not* commutative: dst = !src1 & src2 3558 masm.v128_and_not(src, dst, writable!(dst))?; 3559 Ok(TypedReg::new(WasmValType::V128, dst)) 3560 }) 3561 } 3562 3563 fn visit_v128_or(&mut self) -> Self::Output { 3564 self.context 3565 .binop(self.masm, OperandSize::S128, |masm, dst, src, _size| { 3566 // careful here: and_not is *not* commutative: dst = !src1 & src2 3567 masm.v128_or(src, dst, writable!(dst))?; 3568 Ok(TypedReg::new(WasmValType::V128, dst)) 3569 }) 3570 } 3571 3572 fn visit_v128_xor(&mut self) -> Self::Output { 3573 self.context 3574 .binop(self.masm, OperandSize::S128, |masm, dst, src, _size| { 3575 // careful here: and_not is *not* commutative: dst = !src1 & src2 3576 masm.v128_xor(src, dst, writable!(dst))?; 3577 Ok(TypedReg::new(WasmValType::V128, dst)) 3578 }) 3579 } 3580 3581 fn visit_v128_bitselect(&mut self) -> Self::Output { 3582 let mask = self.context.pop_to_reg(self.masm, None)?; 3583 let op2 = self.context.pop_to_reg(self.masm, None)?; 3584 let op1 = self.context.pop_to_reg(self.masm, None)?; 3585 let dst = self.context.any_fpr(self.masm)?; 3586 3587 // careful here: bitselect is *not* commutative. 3588 self.masm 3589 .v128_bitselect(op1.reg, op2.reg, mask.reg, writable!(dst))?; 3590 3591 self.context 3592 .stack 3593 .push(TypedReg::new(WasmValType::V128, dst).into()); 3594 self.context.free_reg(op1); 3595 self.context.free_reg(op2); 3596 self.context.free_reg(mask); 3597 3598 Ok(()) 3599 } 3600 3601 fn visit_v128_any_true(&mut self) -> Self::Output { 3602 let src = self.context.pop_to_reg(self.masm, None)?; 3603 let dst = self.context.any_gpr(self.masm)?; 3604 3605 self.masm.v128_any_true(src.reg, writable!(dst))?; 3606 3607 self.context 3608 .stack 3609 .push(TypedReg::new(WasmValType::I32, dst).into()); 3610 self.context.free_reg(src); 3611 3612 Ok(()) 3613 } 3614 3615 fn visit_v128_load8_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output { 3616 self.emit_wasm_load( 3617 &arg, 3618 WasmValType::V128, 3619 LoadKind::vector_lane(lane, OperandSize::S8), 3620 ) 3621 } 3622 3623 fn visit_v128_load16_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output { 3624 self.emit_wasm_load( 3625 &arg, 3626 WasmValType::V128, 3627 LoadKind::vector_lane(lane, OperandSize::S16), 3628 ) 3629 } 3630 3631 fn visit_v128_load32_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output { 3632 self.emit_wasm_load( 3633 &arg, 3634 WasmValType::V128, 3635 LoadKind::vector_lane(lane, OperandSize::S32), 3636 ) 3637 } 3638 3639 fn visit_v128_load64_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output { 3640 self.emit_wasm_load( 3641 &arg, 3642 WasmValType::V128, 3643 LoadKind::vector_lane(lane, OperandSize::S64), 3644 ) 3645 } 3646 3647 fn visit_v128_store8_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output { 3648 self.emit_wasm_store(&arg, StoreKind::vector_lane(lane, OperandSize::S8)) 3649 } 3650 3651 fn visit_v128_store16_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output { 3652 self.emit_wasm_store(&arg, StoreKind::vector_lane(lane, OperandSize::S16)) 3653 } 3654 3655 fn visit_v128_store32_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output { 3656 self.emit_wasm_store(&arg, StoreKind::vector_lane(lane, OperandSize::S32)) 3657 } 3658 3659 fn visit_v128_store64_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output { 3660 self.emit_wasm_store(&arg, StoreKind::vector_lane(lane, OperandSize::S64)) 3661 } 3662 3663 fn visit_f32x4_convert_i32x4_s(&mut self) -> Self::Output { 3664 self.context.unop(self.masm, |masm, reg| { 3665 masm.v128_convert(reg, writable!(reg), V128ConvertKind::I32x4S)?; 3666 Ok(TypedReg::v128(reg)) 3667 }) 3668 } 3669 3670 fn visit_f32x4_convert_i32x4_u(&mut self) -> Self::Output { 3671 self.context.unop(self.masm, |masm, reg| { 3672 masm.v128_convert(reg, writable!(reg), V128ConvertKind::I32x4U)?; 3673 Ok(TypedReg::v128(reg)) 3674 }) 3675 } 3676 3677 fn visit_f64x2_convert_low_i32x4_s(&mut self) -> Self::Output { 3678 self.context.unop(self.masm, |masm, reg| { 3679 masm.v128_convert(reg, writable!(reg), V128ConvertKind::I32x4LowS)?; 3680 Ok(TypedReg::v128(reg)) 3681 }) 3682 } 3683 3684 fn visit_f64x2_convert_low_i32x4_u(&mut self) -> Self::Output { 3685 self.context.unop(self.masm, |masm, reg| { 3686 masm.v128_convert(reg, writable!(reg), V128ConvertKind::I32x4LowU)?; 3687 Ok(TypedReg::v128(reg)) 3688 }) 3689 } 3690 3691 fn visit_i8x16_narrow_i16x8_s(&mut self) -> Self::Output { 3692 self.context 3693 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3694 masm.v128_narrow(dst, src, writable!(dst), V128NarrowKind::I16x8S)?; 3695 Ok(TypedReg::v128(dst)) 3696 }) 3697 } 3698 3699 fn visit_i8x16_narrow_i16x8_u(&mut self) -> Self::Output { 3700 self.context 3701 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3702 masm.v128_narrow(dst, src, writable!(dst), V128NarrowKind::I16x8U)?; 3703 Ok(TypedReg::v128(dst)) 3704 }) 3705 } 3706 3707 fn visit_i16x8_narrow_i32x4_s(&mut self) -> Self::Output { 3708 self.context 3709 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3710 masm.v128_narrow(dst, src, writable!(dst), V128NarrowKind::I32x4S)?; 3711 Ok(TypedReg::v128(dst)) 3712 }) 3713 } 3714 3715 fn visit_i16x8_narrow_i32x4_u(&mut self) -> Self::Output { 3716 self.context 3717 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3718 masm.v128_narrow(dst, src, writable!(dst), V128NarrowKind::I32x4U)?; 3719 Ok(TypedReg::v128(dst)) 3720 }) 3721 } 3722 3723 fn visit_f32x4_demote_f64x2_zero(&mut self) -> Self::Output { 3724 self.context.unop(self.masm, |masm, reg| { 3725 masm.v128_demote(reg, writable!(reg))?; 3726 Ok(TypedReg::v128(reg)) 3727 }) 3728 } 3729 3730 fn visit_f64x2_promote_low_f32x4(&mut self) -> Self::Output { 3731 self.context.unop(self.masm, |masm, reg| { 3732 masm.v128_promote(reg, writable!(reg))?; 3733 Ok(TypedReg::v128(reg)) 3734 }) 3735 } 3736 3737 fn visit_i16x8_extend_low_i8x16_s(&mut self) -> Self::Output { 3738 self.context.unop(self.masm, |masm, reg| { 3739 masm.v128_extend(reg, writable!(reg), V128ExtendKind::LowI8x16S)?; 3740 Ok(TypedReg::v128(reg)) 3741 }) 3742 } 3743 3744 fn visit_i16x8_extend_high_i8x16_s(&mut self) -> Self::Output { 3745 self.context.unop(self.masm, |masm, reg| { 3746 masm.v128_extend(reg, writable!(reg), V128ExtendKind::HighI8x16S)?; 3747 Ok(TypedReg::v128(reg)) 3748 }) 3749 } 3750 3751 fn visit_i16x8_extend_low_i8x16_u(&mut self) -> Self::Output { 3752 self.context.unop(self.masm, |masm, reg| { 3753 masm.v128_extend(reg, writable!(reg), V128ExtendKind::LowI8x16U)?; 3754 Ok(TypedReg::v128(reg)) 3755 }) 3756 } 3757 3758 fn visit_i16x8_extend_high_i8x16_u(&mut self) -> Self::Output { 3759 self.context.unop(self.masm, |masm, reg| { 3760 masm.v128_extend(reg, writable!(reg), V128ExtendKind::HighI8x16U)?; 3761 Ok(TypedReg::v128(reg)) 3762 }) 3763 } 3764 3765 fn visit_i32x4_extend_low_i16x8_s(&mut self) -> Self::Output { 3766 self.context.unop(self.masm, |masm, reg| { 3767 masm.v128_extend(reg, writable!(reg), V128ExtendKind::LowI16x8S)?; 3768 Ok(TypedReg::v128(reg)) 3769 }) 3770 } 3771 3772 fn visit_i32x4_extend_high_i16x8_s(&mut self) -> Self::Output { 3773 self.context.unop(self.masm, |masm, reg| { 3774 masm.v128_extend(reg, writable!(reg), V128ExtendKind::HighI16x8S)?; 3775 Ok(TypedReg::v128(reg)) 3776 }) 3777 } 3778 3779 fn visit_i32x4_extend_low_i16x8_u(&mut self) -> Self::Output { 3780 self.context.unop(self.masm, |masm, reg| { 3781 masm.v128_extend(reg, writable!(reg), V128ExtendKind::LowI16x8U)?; 3782 Ok(TypedReg::v128(reg)) 3783 }) 3784 } 3785 3786 fn visit_i32x4_extend_high_i16x8_u(&mut self) -> Self::Output { 3787 self.context.unop(self.masm, |masm, reg| { 3788 masm.v128_extend(reg, writable!(reg), V128ExtendKind::HighI16x8U)?; 3789 Ok(TypedReg::v128(reg)) 3790 }) 3791 } 3792 3793 fn visit_i64x2_extend_low_i32x4_s(&mut self) -> Self::Output { 3794 self.context.unop(self.masm, |masm, reg| { 3795 masm.v128_extend(reg, writable!(reg), V128ExtendKind::LowI32x4S)?; 3796 Ok(TypedReg::v128(reg)) 3797 }) 3798 } 3799 3800 fn visit_i64x2_extend_high_i32x4_s(&mut self) -> Self::Output { 3801 self.context.unop(self.masm, |masm, reg| { 3802 masm.v128_extend(reg, writable!(reg), V128ExtendKind::HighI32x4S)?; 3803 Ok(TypedReg::v128(reg)) 3804 }) 3805 } 3806 3807 fn visit_i64x2_extend_low_i32x4_u(&mut self) -> Self::Output { 3808 self.context.unop(self.masm, |masm, reg| { 3809 masm.v128_extend(reg, writable!(reg), V128ExtendKind::LowI32x4U)?; 3810 Ok(TypedReg::v128(reg)) 3811 }) 3812 } 3813 3814 fn visit_i64x2_extend_high_i32x4_u(&mut self) -> Self::Output { 3815 self.context.unop(self.masm, |masm, reg| { 3816 masm.v128_extend(reg, writable!(reg), V128ExtendKind::HighI32x4U)?; 3817 Ok(TypedReg::v128(reg)) 3818 }) 3819 } 3820 3821 fn visit_i8x16_add(&mut self) -> Self::Output { 3822 self.context 3823 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3824 masm.v128_add(dst, src, writable!(dst), V128AddKind::I8x16)?; 3825 Ok(TypedReg::new(WasmValType::V128, dst)) 3826 }) 3827 } 3828 3829 fn visit_i16x8_add(&mut self) -> Self::Output { 3830 self.context 3831 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3832 masm.v128_add(dst, src, writable!(dst), V128AddKind::I16x8)?; 3833 Ok(TypedReg::new(WasmValType::V128, dst)) 3834 }) 3835 } 3836 3837 fn visit_i32x4_add(&mut self) -> Self::Output { 3838 self.context 3839 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3840 masm.v128_add(dst, src, writable!(dst), V128AddKind::I32x4)?; 3841 Ok(TypedReg::new(WasmValType::V128, dst)) 3842 }) 3843 } 3844 3845 fn visit_i64x2_add(&mut self) -> Self::Output { 3846 self.context 3847 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3848 masm.v128_add(dst, src, writable!(dst), V128AddKind::I64x2)?; 3849 Ok(TypedReg::new(WasmValType::V128, dst)) 3850 }) 3851 } 3852 3853 fn visit_i8x16_sub(&mut self) -> Self::Output { 3854 self.context 3855 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3856 masm.v128_sub(dst, src, writable!(dst), V128SubKind::I8x16)?; 3857 Ok(TypedReg::new(WasmValType::V128, dst)) 3858 }) 3859 } 3860 3861 fn visit_i16x8_sub(&mut self) -> Self::Output { 3862 self.context 3863 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3864 masm.v128_sub(dst, src, writable!(dst), V128SubKind::I16x8)?; 3865 Ok(TypedReg::new(WasmValType::V128, dst)) 3866 }) 3867 } 3868 3869 fn visit_i32x4_sub(&mut self) -> Self::Output { 3870 self.context 3871 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3872 masm.v128_sub(dst, src, writable!(dst), V128SubKind::I32x4)?; 3873 Ok(TypedReg::new(WasmValType::V128, dst)) 3874 }) 3875 } 3876 3877 fn visit_i64x2_sub(&mut self) -> Self::Output { 3878 self.context 3879 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3880 masm.v128_sub(dst, src, writable!(dst), V128SubKind::I64x2)?; 3881 Ok(TypedReg::new(WasmValType::V128, dst)) 3882 }) 3883 } 3884 3885 fn visit_i16x8_mul(&mut self) -> Self::Output { 3886 self.masm.v128_mul(&mut self.context, V128MulKind::I16x8) 3887 } 3888 3889 fn visit_i32x4_mul(&mut self) -> Self::Output { 3890 self.masm.v128_mul(&mut self.context, V128MulKind::I32x4) 3891 } 3892 3893 fn visit_i64x2_mul(&mut self) -> Self::Output { 3894 self.masm.v128_mul(&mut self.context, V128MulKind::I64x2) 3895 } 3896 3897 fn visit_i8x16_add_sat_s(&mut self) -> Self::Output { 3898 self.context 3899 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3900 masm.v128_add(dst, src, writable!(dst), V128AddKind::I8x16SatS)?; 3901 Ok(TypedReg::new(WasmValType::V128, dst)) 3902 }) 3903 } 3904 3905 fn visit_i16x8_add_sat_s(&mut self) -> Self::Output { 3906 self.context 3907 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3908 masm.v128_add(dst, src, writable!(dst), V128AddKind::I16x8SatS)?; 3909 Ok(TypedReg::new(WasmValType::V128, dst)) 3910 }) 3911 } 3912 3913 fn visit_i8x16_add_sat_u(&mut self) -> Self::Output { 3914 self.context 3915 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3916 masm.v128_add(dst, src, writable!(dst), V128AddKind::I8x16SatU)?; 3917 Ok(TypedReg::new(WasmValType::V128, dst)) 3918 }) 3919 } 3920 3921 fn visit_i16x8_add_sat_u(&mut self) -> Self::Output { 3922 self.context 3923 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3924 masm.v128_add(dst, src, writable!(dst), V128AddKind::I16x8SatU)?; 3925 Ok(TypedReg::new(WasmValType::V128, dst)) 3926 }) 3927 } 3928 3929 fn visit_i8x16_sub_sat_s(&mut self) -> Self::Output { 3930 self.context 3931 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3932 masm.v128_sub(dst, src, writable!(dst), V128SubKind::I8x16SatS)?; 3933 Ok(TypedReg::new(WasmValType::V128, dst)) 3934 }) 3935 } 3936 3937 fn visit_i16x8_sub_sat_s(&mut self) -> Self::Output { 3938 self.context 3939 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3940 masm.v128_sub(dst, src, writable!(dst), V128SubKind::I16x8SatS)?; 3941 Ok(TypedReg::new(WasmValType::V128, dst)) 3942 }) 3943 } 3944 3945 fn visit_i8x16_sub_sat_u(&mut self) -> Self::Output { 3946 self.context 3947 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3948 masm.v128_sub(dst, src, writable!(dst), V128SubKind::I8x16SatU)?; 3949 Ok(TypedReg::new(WasmValType::V128, dst)) 3950 }) 3951 } 3952 3953 fn visit_i16x8_sub_sat_u(&mut self) -> Self::Output { 3954 self.context 3955 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3956 masm.v128_sub(dst, src, writable!(dst), V128SubKind::I16x8SatU)?; 3957 Ok(TypedReg::new(WasmValType::V128, dst)) 3958 }) 3959 } 3960 3961 fn visit_i8x16_abs(&mut self) -> Self::Output { 3962 self.context.unop(self.masm, |masm, reg| { 3963 masm.v128_abs(reg, writable!(reg), V128AbsKind::I8x16)?; 3964 Ok(TypedReg::new(WasmValType::V128, reg)) 3965 }) 3966 } 3967 3968 fn visit_i16x8_abs(&mut self) -> Self::Output { 3969 self.context.unop(self.masm, |masm, reg| { 3970 masm.v128_abs(reg, writable!(reg), V128AbsKind::I16x8)?; 3971 Ok(TypedReg::new(WasmValType::V128, reg)) 3972 }) 3973 } 3974 3975 fn visit_i32x4_abs(&mut self) -> Self::Output { 3976 self.context.unop(self.masm, |masm, reg| { 3977 masm.v128_abs(reg, writable!(reg), V128AbsKind::I32x4)?; 3978 Ok(TypedReg::new(WasmValType::V128, reg)) 3979 }) 3980 } 3981 3982 fn visit_i64x2_abs(&mut self) -> Self::Output { 3983 self.context.unop(self.masm, |masm, reg| { 3984 masm.v128_abs(reg, writable!(reg), V128AbsKind::I64x2)?; 3985 Ok(TypedReg::new(WasmValType::V128, reg)) 3986 }) 3987 } 3988 3989 fn visit_f32x4_abs(&mut self) -> Self::Output { 3990 self.context.unop(self.masm, |masm, reg| { 3991 masm.v128_abs(reg, writable!(reg), V128AbsKind::F32x4)?; 3992 Ok(TypedReg::new(WasmValType::V128, reg)) 3993 }) 3994 } 3995 3996 fn visit_f64x2_abs(&mut self) -> Self::Output { 3997 self.context.unop(self.masm, |masm, reg| { 3998 masm.v128_abs(reg, writable!(reg), V128AbsKind::F64x2)?; 3999 Ok(TypedReg::new(WasmValType::V128, reg)) 4000 }) 4001 } 4002 4003 fn visit_i8x16_neg(&mut self) -> Self::Output { 4004 self.context.unop(self.masm, |masm, op| { 4005 masm.v128_neg(writable!(op), V128NegKind::I8x16)?; 4006 Ok(TypedReg::new(WasmValType::V128, op)) 4007 }) 4008 } 4009 4010 fn visit_i16x8_neg(&mut self) -> Self::Output { 4011 self.context.unop(self.masm, |masm, op| { 4012 masm.v128_neg(writable!(op), V128NegKind::I16x8)?; 4013 Ok(TypedReg::new(WasmValType::V128, op)) 4014 }) 4015 } 4016 4017 fn visit_i32x4_neg(&mut self) -> Self::Output { 4018 self.context.unop(self.masm, |masm, op| { 4019 masm.v128_neg(writable!(op), V128NegKind::I32x4)?; 4020 Ok(TypedReg::new(WasmValType::V128, op)) 4021 }) 4022 } 4023 4024 fn visit_i64x2_neg(&mut self) -> Self::Output { 4025 self.context.unop(self.masm, |masm, op| { 4026 masm.v128_neg(writable!(op), V128NegKind::I64x2)?; 4027 Ok(TypedReg::new(WasmValType::V128, op)) 4028 }) 4029 } 4030 4031 fn visit_i8x16_shl(&mut self) -> Self::Output { 4032 self.masm 4033 .v128_shift(&mut self.context, OperandSize::S8, ShiftKind::Shl) 4034 } 4035 4036 fn visit_i16x8_shl(&mut self) -> Self::Output { 4037 self.masm 4038 .v128_shift(&mut self.context, OperandSize::S16, ShiftKind::Shl) 4039 } 4040 4041 fn visit_i32x4_shl(&mut self) -> Self::Output { 4042 self.masm 4043 .v128_shift(&mut self.context, OperandSize::S32, ShiftKind::Shl) 4044 } 4045 4046 fn visit_i64x2_shl(&mut self) -> Self::Output { 4047 self.masm 4048 .v128_shift(&mut self.context, OperandSize::S64, ShiftKind::Shl) 4049 } 4050 4051 fn visit_i8x16_shr_u(&mut self) -> Self::Output { 4052 self.masm 4053 .v128_shift(&mut self.context, OperandSize::S8, ShiftKind::ShrU) 4054 } 4055 4056 fn visit_i16x8_shr_u(&mut self) -> Self::Output { 4057 self.masm 4058 .v128_shift(&mut self.context, OperandSize::S16, ShiftKind::ShrU) 4059 } 4060 4061 fn visit_i32x4_shr_u(&mut self) -> Self::Output { 4062 self.masm 4063 .v128_shift(&mut self.context, OperandSize::S32, ShiftKind::ShrU) 4064 } 4065 4066 fn visit_i64x2_shr_u(&mut self) -> Self::Output { 4067 self.masm 4068 .v128_shift(&mut self.context, OperandSize::S64, ShiftKind::ShrU) 4069 } 4070 4071 fn visit_i8x16_shr_s(&mut self) -> Self::Output { 4072 self.masm 4073 .v128_shift(&mut self.context, OperandSize::S8, ShiftKind::ShrS) 4074 } 4075 4076 fn visit_i16x8_shr_s(&mut self) -> Self::Output { 4077 self.masm 4078 .v128_shift(&mut self.context, OperandSize::S16, ShiftKind::ShrS) 4079 } 4080 4081 fn visit_i32x4_shr_s(&mut self) -> Self::Output { 4082 self.masm 4083 .v128_shift(&mut self.context, OperandSize::S32, ShiftKind::ShrS) 4084 } 4085 4086 fn visit_i64x2_shr_s(&mut self) -> Self::Output { 4087 self.masm 4088 .v128_shift(&mut self.context, OperandSize::S64, ShiftKind::ShrS) 4089 } 4090 4091 fn visit_i16x8_q15mulr_sat_s(&mut self) -> Self::Output { 4092 self.context 4093 .binop(self.masm, OperandSize::S16, |masm, dst, src, size| { 4094 masm.v128_q15mulr_sat_s(dst, src, writable!(dst), size)?; 4095 Ok(TypedReg::v128(dst)) 4096 }) 4097 } 4098 4099 fn visit_i8x16_min_s(&mut self) -> Self::Output { 4100 self.context 4101 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 4102 masm.v128_min(src, dst, writable!(dst), V128MinKind::I8x16S)?; 4103 Ok(TypedReg::v128(dst)) 4104 }) 4105 } 4106 4107 fn visit_i8x16_all_true(&mut self) -> Self::Output { 4108 self.context.v128_all_true_op(self.masm, |masm, src, dst| { 4109 masm.v128_all_true(src, writable!(dst), OperandSize::S8) 4110 }) 4111 } 4112 4113 fn visit_i16x8_all_true(&mut self) -> Self::Output { 4114 self.context.v128_all_true_op(self.masm, |masm, src, dst| { 4115 masm.v128_all_true(src, writable!(dst), OperandSize::S16) 4116 }) 4117 } 4118 4119 fn visit_i32x4_all_true(&mut self) -> Self::Output { 4120 self.context.v128_all_true_op(self.masm, |masm, src, dst| { 4121 masm.v128_all_true(src, writable!(dst), OperandSize::S32) 4122 }) 4123 } 4124 4125 fn visit_i64x2_all_true(&mut self) -> Self::Output { 4126 self.context.v128_all_true_op(self.masm, |masm, src, dst| { 4127 masm.v128_all_true(src, writable!(dst), OperandSize::S64) 4128 }) 4129 } 4130 4131 fn visit_i8x16_bitmask(&mut self) -> Self::Output { 4132 self.context.v128_bitmask_op(self.masm, |masm, src, dst| { 4133 masm.v128_bitmask(src, writable!(dst), OperandSize::S8) 4134 }) 4135 } 4136 4137 fn visit_i16x8_bitmask(&mut self) -> Self::Output { 4138 self.context.v128_bitmask_op(self.masm, |masm, src, dst| { 4139 masm.v128_bitmask(src, writable!(dst), OperandSize::S16) 4140 }) 4141 } 4142 4143 fn visit_i32x4_bitmask(&mut self) -> Self::Output { 4144 self.context.v128_bitmask_op(self.masm, |masm, src, dst| { 4145 masm.v128_bitmask(src, writable!(dst), OperandSize::S32) 4146 }) 4147 } 4148 4149 fn visit_i64x2_bitmask(&mut self) -> Self::Output { 4150 self.context.v128_bitmask_op(self.masm, |masm, src, dst| { 4151 masm.v128_bitmask(src, writable!(dst), OperandSize::S64) 4152 }) 4153 } 4154 4155 fn visit_i32x4_trunc_sat_f32x4_s(&mut self) -> Self::Output { 4156 self.masm 4157 .v128_trunc(&mut self.context, V128TruncKind::I32x4FromF32x4S) 4158 } 4159 4160 fn visit_i32x4_trunc_sat_f32x4_u(&mut self) -> Self::Output { 4161 self.masm 4162 .v128_trunc(&mut self.context, V128TruncKind::I32x4FromF32x4U) 4163 } 4164 4165 fn visit_i32x4_trunc_sat_f64x2_s_zero(&mut self) -> Self::Output { 4166 self.masm 4167 .v128_trunc(&mut self.context, V128TruncKind::I32x4FromF64x2SZero) 4168 } 4169 4170 fn visit_i32x4_trunc_sat_f64x2_u_zero(&mut self) -> Self::Output { 4171 self.masm 4172 .v128_trunc(&mut self.context, V128TruncKind::I32x4FromF64x2UZero) 4173 } 4174 4175 fn visit_i16x8_min_s(&mut self) -> Self::Output { 4176 self.context 4177 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 4178 masm.v128_min(src, dst, writable!(dst), V128MinKind::I16x8S)?; 4179 Ok(TypedReg::v128(dst)) 4180 }) 4181 } 4182 4183 fn visit_i32x4_dot_i16x8_s(&mut self) -> Self::Output { 4184 self.context 4185 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4186 masm.v128_dot(dst, src, writable!(dst))?; 4187 Ok(TypedReg::v128(dst)) 4188 }) 4189 } 4190 4191 fn visit_i8x16_popcnt(&mut self) -> Self::Output { 4192 self.masm.v128_popcnt(&mut self.context) 4193 } 4194 4195 fn visit_i8x16_avgr_u(&mut self) -> Self::Output { 4196 self.context 4197 .binop(self.masm, OperandSize::S8, |masm, dst, src, size| { 4198 masm.v128_avgr(dst, src, writable!(dst), size)?; 4199 Ok(TypedReg::v128(dst)) 4200 }) 4201 } 4202 4203 fn visit_i32x4_min_s(&mut self) -> Self::Output { 4204 self.context 4205 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4206 masm.v128_min(src, dst, writable!(dst), V128MinKind::I32x4S)?; 4207 Ok(TypedReg::v128(dst)) 4208 }) 4209 } 4210 4211 fn visit_i8x16_min_u(&mut self) -> Self::Output { 4212 self.context 4213 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 4214 masm.v128_min(src, dst, writable!(dst), V128MinKind::I8x16U)?; 4215 Ok(TypedReg::v128(dst)) 4216 }) 4217 } 4218 4219 fn visit_i16x8_avgr_u(&mut self) -> Self::Output { 4220 self.context 4221 .binop(self.masm, OperandSize::S16, |masm, dst, src, size| { 4222 masm.v128_avgr(dst, src, writable!(dst), size)?; 4223 Ok(TypedReg::v128(dst)) 4224 }) 4225 } 4226 4227 fn visit_i16x8_min_u(&mut self) -> Self::Output { 4228 self.context 4229 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 4230 masm.v128_min(src, dst, writable!(dst), V128MinKind::I16x8U)?; 4231 Ok(TypedReg::v128(dst)) 4232 }) 4233 } 4234 4235 fn visit_i32x4_min_u(&mut self) -> Self::Output { 4236 self.context 4237 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4238 masm.v128_min(src, dst, writable!(dst), V128MinKind::I32x4U)?; 4239 Ok(TypedReg::v128(dst)) 4240 }) 4241 } 4242 4243 fn visit_i8x16_max_s(&mut self) -> Self::Output { 4244 self.context 4245 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 4246 masm.v128_max(src, dst, writable!(dst), V128MaxKind::I8x16S)?; 4247 Ok(TypedReg::v128(dst)) 4248 }) 4249 } 4250 4251 fn visit_i16x8_max_s(&mut self) -> Self::Output { 4252 self.context 4253 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 4254 masm.v128_max(src, dst, writable!(dst), V128MaxKind::I16x8S)?; 4255 Ok(TypedReg::v128(dst)) 4256 }) 4257 } 4258 4259 fn visit_i32x4_max_s(&mut self) -> Self::Output { 4260 self.context 4261 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4262 masm.v128_max(src, dst, writable!(dst), V128MaxKind::I32x4S)?; 4263 Ok(TypedReg::v128(dst)) 4264 }) 4265 } 4266 4267 fn visit_i8x16_max_u(&mut self) -> Self::Output { 4268 self.context 4269 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 4270 masm.v128_max(src, dst, writable!(dst), V128MaxKind::I8x16U)?; 4271 Ok(TypedReg::v128(dst)) 4272 }) 4273 } 4274 4275 fn visit_i16x8_max_u(&mut self) -> Self::Output { 4276 self.context 4277 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 4278 masm.v128_max(src, dst, writable!(dst), V128MaxKind::I16x8U)?; 4279 Ok(TypedReg::v128(dst)) 4280 }) 4281 } 4282 4283 fn visit_i32x4_max_u(&mut self) -> Self::Output { 4284 self.context 4285 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4286 masm.v128_max(src, dst, writable!(dst), V128MaxKind::I32x4U)?; 4287 Ok(TypedReg::v128(dst)) 4288 }) 4289 } 4290 4291 fn visit_i16x8_extmul_low_i8x16_s(&mut self) -> Self::Output { 4292 self.masm 4293 .v128_extmul(&mut self.context, V128ExtMulKind::LowI8x16S) 4294 } 4295 4296 fn visit_i32x4_extmul_low_i16x8_s(&mut self) -> Self::Output { 4297 self.masm 4298 .v128_extmul(&mut self.context, V128ExtMulKind::LowI16x8S) 4299 } 4300 4301 fn visit_i64x2_extmul_low_i32x4_s(&mut self) -> Self::Output { 4302 self.masm 4303 .v128_extmul(&mut self.context, V128ExtMulKind::LowI32x4S) 4304 } 4305 4306 fn visit_i16x8_extmul_low_i8x16_u(&mut self) -> Self::Output { 4307 self.masm 4308 .v128_extmul(&mut self.context, V128ExtMulKind::LowI8x16U) 4309 } 4310 4311 fn visit_i32x4_extmul_low_i16x8_u(&mut self) -> Self::Output { 4312 self.masm 4313 .v128_extmul(&mut self.context, V128ExtMulKind::LowI16x8U) 4314 } 4315 4316 fn visit_i64x2_extmul_low_i32x4_u(&mut self) -> Self::Output { 4317 self.masm 4318 .v128_extmul(&mut self.context, V128ExtMulKind::LowI32x4U) 4319 } 4320 4321 fn visit_i16x8_extmul_high_i8x16_u(&mut self) -> Self::Output { 4322 self.masm 4323 .v128_extmul(&mut self.context, V128ExtMulKind::HighI8x16U) 4324 } 4325 4326 fn visit_i32x4_extmul_high_i16x8_u(&mut self) -> Self::Output { 4327 self.masm 4328 .v128_extmul(&mut self.context, V128ExtMulKind::HighI16x8U) 4329 } 4330 4331 fn visit_i64x2_extmul_high_i32x4_u(&mut self) -> Self::Output { 4332 self.masm 4333 .v128_extmul(&mut self.context, V128ExtMulKind::HighI32x4U) 4334 } 4335 4336 fn visit_i16x8_extmul_high_i8x16_s(&mut self) -> Self::Output { 4337 self.masm 4338 .v128_extmul(&mut self.context, V128ExtMulKind::HighI8x16S) 4339 } 4340 4341 fn visit_i32x4_extmul_high_i16x8_s(&mut self) -> Self::Output { 4342 self.masm 4343 .v128_extmul(&mut self.context, V128ExtMulKind::HighI16x8S) 4344 } 4345 4346 fn visit_i64x2_extmul_high_i32x4_s(&mut self) -> Self::Output { 4347 self.masm 4348 .v128_extmul(&mut self.context, V128ExtMulKind::HighI32x4S) 4349 } 4350 4351 fn visit_i16x8_extadd_pairwise_i8x16_s(&mut self) -> Self::Output { 4352 self.context.unop(self.masm, |masm, op| { 4353 masm.v128_extadd_pairwise(op, writable!(op), V128ExtAddKind::I8x16S)?; 4354 Ok(TypedReg::v128(op)) 4355 }) 4356 } 4357 4358 fn visit_i16x8_extadd_pairwise_i8x16_u(&mut self) -> Self::Output { 4359 self.context.unop(self.masm, |masm, op| { 4360 masm.v128_extadd_pairwise(op, writable!(op), V128ExtAddKind::I8x16U)?; 4361 Ok(TypedReg::v128(op)) 4362 }) 4363 } 4364 4365 fn visit_i32x4_extadd_pairwise_i16x8_s(&mut self) -> Self::Output { 4366 self.context.unop(self.masm, |masm, op| { 4367 masm.v128_extadd_pairwise(op, writable!(op), V128ExtAddKind::I16x8S)?; 4368 Ok(TypedReg::v128(op)) 4369 }) 4370 } 4371 4372 fn visit_i32x4_extadd_pairwise_i16x8_u(&mut self) -> Self::Output { 4373 self.context.unop(self.masm, |masm, op| { 4374 masm.v128_extadd_pairwise(op, writable!(op), V128ExtAddKind::I16x8U)?; 4375 Ok(TypedReg::v128(op)) 4376 }) 4377 } 4378 4379 fn visit_f32x4_add(&mut self) -> Self::Output { 4380 self.context 4381 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4382 masm.v128_add(dst, src, writable!(dst), V128AddKind::F32x4)?; 4383 Ok(TypedReg::v128(dst)) 4384 }) 4385 } 4386 4387 fn visit_f64x2_add(&mut self) -> Self::Output { 4388 self.context 4389 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 4390 masm.v128_add(dst, src, writable!(dst), V128AddKind::F64x2)?; 4391 Ok(TypedReg::v128(dst)) 4392 }) 4393 } 4394 4395 fn visit_f32x4_sub(&mut self) -> Self::Output { 4396 self.context 4397 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4398 masm.v128_sub(dst, src, writable!(dst), V128SubKind::F32x4)?; 4399 Ok(TypedReg::v128(dst)) 4400 }) 4401 } 4402 4403 fn visit_f64x2_sub(&mut self) -> Self::Output { 4404 self.context 4405 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 4406 masm.v128_sub(dst, src, writable!(dst), V128SubKind::F64x2)?; 4407 Ok(TypedReg::v128(dst)) 4408 }) 4409 } 4410 4411 fn visit_f32x4_mul(&mut self) -> Self::Output { 4412 self.masm.v128_mul(&mut self.context, V128MulKind::F32x4) 4413 } 4414 4415 fn visit_f64x2_mul(&mut self) -> Self::Output { 4416 self.masm.v128_mul(&mut self.context, V128MulKind::F64x2) 4417 } 4418 4419 fn visit_f32x4_div(&mut self) -> Self::Output { 4420 self.context 4421 .binop(self.masm, OperandSize::S32, |masm, dst, src, size| { 4422 masm.v128_div(dst, src, writable!(dst), size)?; 4423 Ok(TypedReg::v128(dst)) 4424 }) 4425 } 4426 4427 fn visit_f64x2_div(&mut self) -> Self::Output { 4428 self.context 4429 .binop(self.masm, OperandSize::S64, |masm, dst, src, size| { 4430 masm.v128_div(dst, src, writable!(dst), size)?; 4431 Ok(TypedReg::v128(dst)) 4432 }) 4433 } 4434 4435 fn visit_f32x4_neg(&mut self) -> Self::Output { 4436 self.context.unop(self.masm, |masm, reg| { 4437 masm.v128_neg(writable!(reg), V128NegKind::F32x4)?; 4438 Ok(TypedReg::v128(reg)) 4439 }) 4440 } 4441 4442 fn visit_f32x4_ceil(&mut self) -> Self::Output { 4443 self.context.unop(self.masm, |masm, reg| { 4444 masm.v128_ceil(reg, writable!(reg), OperandSize::S32)?; 4445 Ok(TypedReg::v128(reg)) 4446 }) 4447 } 4448 4449 fn visit_f64x2_neg(&mut self) -> Self::Output { 4450 self.context.unop(self.masm, |masm, reg| { 4451 masm.v128_neg(writable!(reg), V128NegKind::F64x2)?; 4452 Ok(TypedReg::v128(reg)) 4453 }) 4454 } 4455 4456 fn visit_f64x2_ceil(&mut self) -> Self::Output { 4457 self.context.unop(self.masm, |masm, reg| { 4458 masm.v128_ceil(reg, writable!(reg), OperandSize::S64)?; 4459 Ok(TypedReg::v128(reg)) 4460 }) 4461 } 4462 4463 fn visit_f32x4_sqrt(&mut self) -> Self::Output { 4464 self.context.unop(self.masm, |masm, reg| { 4465 masm.v128_sqrt(reg, writable!(reg), OperandSize::S32)?; 4466 Ok(TypedReg::v128(reg)) 4467 }) 4468 } 4469 4470 fn visit_f32x4_floor(&mut self) -> Self::Output { 4471 self.context.unop(self.masm, |masm, reg| { 4472 masm.v128_floor(reg, writable!(reg), OperandSize::S32)?; 4473 Ok(TypedReg::v128(reg)) 4474 }) 4475 } 4476 4477 fn visit_f64x2_sqrt(&mut self) -> Self::Output { 4478 self.context.unop(self.masm, |masm, reg| { 4479 masm.v128_sqrt(reg, writable!(reg), OperandSize::S64)?; 4480 Ok(TypedReg::v128(reg)) 4481 }) 4482 } 4483 4484 fn visit_f64x2_floor(&mut self) -> Self::Output { 4485 self.context.unop(self.masm, |masm, reg| { 4486 masm.v128_floor(reg, writable!(reg), OperandSize::S64)?; 4487 Ok(TypedReg::v128(reg)) 4488 }) 4489 } 4490 4491 fn visit_f32x4_nearest(&mut self) -> Self::Output { 4492 self.context.unop(self.masm, |masm, reg| { 4493 masm.v128_nearest(reg, writable!(reg), OperandSize::S32)?; 4494 Ok(TypedReg::v128(reg)) 4495 }) 4496 } 4497 4498 fn visit_f64x2_nearest(&mut self) -> Self::Output { 4499 self.context.unop(self.masm, |masm, reg| { 4500 masm.v128_nearest(reg, writable!(reg), OperandSize::S64)?; 4501 Ok(TypedReg::v128(reg)) 4502 }) 4503 } 4504 4505 fn visit_f32x4_trunc(&mut self) -> Self::Output { 4506 self.masm 4507 .v128_trunc(&mut self.context, V128TruncKind::F32x4) 4508 } 4509 4510 fn visit_f64x2_trunc(&mut self) -> Self::Output { 4511 self.masm 4512 .v128_trunc(&mut self.context, V128TruncKind::F64x2) 4513 } 4514 4515 fn visit_v128_load32_zero(&mut self, memarg: MemArg) -> Self::Output { 4516 self.emit_wasm_load( 4517 &memarg, 4518 WasmValType::V128, 4519 LoadKind::VectorZero(OperandSize::S32), 4520 ) 4521 } 4522 4523 fn visit_v128_load64_zero(&mut self, memarg: MemArg) -> Self::Output { 4524 self.emit_wasm_load( 4525 &memarg, 4526 WasmValType::V128, 4527 LoadKind::VectorZero(OperandSize::S64), 4528 ) 4529 } 4530 4531 fn visit_f32x4_pmin(&mut self) -> Self::Output { 4532 self.context 4533 .binop(self.masm, OperandSize::S32, |masm, dst, src, size| { 4534 masm.v128_pmin(dst, src, writable!(dst), size)?; 4535 Ok(TypedReg::v128(dst)) 4536 }) 4537 } 4538 4539 fn visit_f64x2_pmin(&mut self) -> Self::Output { 4540 self.context 4541 .binop(self.masm, OperandSize::S64, |masm, dst, src, size| { 4542 masm.v128_pmin(dst, src, writable!(dst), size)?; 4543 Ok(TypedReg::v128(dst)) 4544 }) 4545 } 4546 4547 fn visit_f32x4_pmax(&mut self) -> Self::Output { 4548 self.context 4549 .binop(self.masm, OperandSize::S32, |masm, dst, src, size| { 4550 masm.v128_pmax(dst, src, writable!(dst), size)?; 4551 Ok(TypedReg::v128(dst)) 4552 }) 4553 } 4554 4555 fn visit_f64x2_pmax(&mut self) -> Self::Output { 4556 self.context 4557 .binop(self.masm, OperandSize::S64, |masm, dst, src, size| { 4558 masm.v128_pmax(dst, src, writable!(dst), size)?; 4559 Ok(TypedReg::v128(dst)) 4560 }) 4561 } 4562 4563 fn visit_f32x4_min(&mut self) -> Self::Output { 4564 self.context 4565 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4566 masm.v128_min(dst, src, writable!(dst), V128MinKind::F32x4)?; 4567 Ok(TypedReg::v128(dst)) 4568 }) 4569 } 4570 4571 fn visit_f64x2_min(&mut self) -> Self::Output { 4572 self.context 4573 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 4574 masm.v128_min(dst, src, writable!(dst), V128MinKind::F64x2)?; 4575 Ok(TypedReg::v128(dst)) 4576 }) 4577 } 4578 4579 fn visit_f32x4_max(&mut self) -> Self::Output { 4580 self.context 4581 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4582 masm.v128_max(dst, src, writable!(dst), V128MaxKind::F32x4)?; 4583 Ok(TypedReg::v128(dst)) 4584 }) 4585 } 4586 4587 fn visit_f64x2_max(&mut self) -> Self::Output { 4588 self.context 4589 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 4590 masm.v128_max(dst, src, writable!(dst), V128MaxKind::F64x2)?; 4591 Ok(TypedReg::v128(dst)) 4592 }) 4593 } 4594 4595 wasmparser::for_each_visit_simd_operator!(def_unsupported); 4596 } 4597 4598 impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Emission> 4599 where 4600 M: MacroAssembler, 4601 { 4602 fn cmp_i32s(&mut self, kind: IntCmpKind) -> Result<()> { 4603 self.context.i32_binop(self.masm, |masm, dst, src, size| { 4604 masm.cmp_with_set(writable!(dst), src, kind, size)?; 4605 Ok(TypedReg::i32(dst)) 4606 }) 4607 } 4608 4609 fn cmp_i64s(&mut self, kind: IntCmpKind) -> Result<()> { 4610 self.context 4611 .i64_binop(self.masm, move |masm, dst, src, size| { 4612 masm.cmp_with_set(writable!(dst), src, kind, size)?; 4613 Ok(TypedReg::i32(dst)) // Return value for comparisons is an `i32`. 4614 }) 4615 } 4616 } 4617 4618 impl TryFrom<WasmValType> for OperandSize { 4619 type Error = crate::Error; 4620 fn try_from(ty: WasmValType) -> Result<OperandSize> { 4621 let ty = match ty { 4622 WasmValType::I32 | WasmValType::F32 => OperandSize::S32, 4623 WasmValType::I64 | WasmValType::F64 => OperandSize::S64, 4624 WasmValType::V128 => OperandSize::S128, 4625 WasmValType::Ref(rt) => { 4626 match rt.heap_type { 4627 // TODO: Hardcoded size, assuming 64-bit support only. Once 4628 // Wasmtime supports 32-bit architectures, this will need 4629 // to be updated in such a way that the calculation of the 4630 // OperandSize will depend on the target's pointer size. 4631 WasmHeapType::Func => OperandSize::S64, 4632 WasmHeapType::Extern => OperandSize::S64, 4633 _ => bail!(CodeGenError::unsupported_wasm_type()), 4634 } 4635 } 4636 }; 4637 Ok(ty) 4638 } 4639 } 4640