1 //! This module is the central place for machine code emission. 2 //! It defines an implementation of wasmparser's Visitor trait 3 //! for `CodeGen`; which defines a visitor per op-code, 4 //! which validates and dispatches to the corresponding 5 //! machine code emitter. 6 7 use crate::abi::RetArea; 8 use crate::codegen::{ 9 AtomicWaitKind, Callee, CodeGen, CodeGenError, ConditionalBranch, ControlStackFrame, Emission, 10 FnCall, UnconditionalBranch, control_index, 11 }; 12 use crate::masm::{ 13 DivKind, Extend, ExtractLaneKind, FloatCmpKind, IntCmpKind, LoadKind, MacroAssembler, 14 MulWideKind, OperandSize, RegImm, RemKind, ReplaceLaneKind, RmwOp, RoundingMode, SPOffset, 15 ShiftKind, Signed, SplatKind, SplatLoadKind, StoreKind, TruncKind, V128AbsKind, V128AddKind, 16 V128ConvertKind, V128ExtAddKind, V128ExtMulKind, V128ExtendKind, V128LoadExtendKind, 17 V128MaxKind, V128MinKind, V128MulKind, V128NarrowKind, V128NegKind, V128SubKind, V128TruncKind, 18 VectorCompareKind, VectorEqualityKind, Zero, 19 }; 20 21 use crate::reg::{Reg, writable}; 22 use crate::stack::{TypedReg, Val}; 23 use anyhow::{Result, anyhow, bail, ensure}; 24 use regalloc2::RegClass; 25 use smallvec::{SmallVec, smallvec}; 26 use wasmparser::{ 27 BlockType, BrTable, Ieee32, Ieee64, MemArg, V128, VisitOperator, VisitSimdOperator, 28 }; 29 use wasmtime_cranelift::TRAP_INDIRECT_CALL_TO_NULL; 30 use wasmtime_environ::{ 31 FUNCREF_INIT_BIT, FuncIndex, GlobalIndex, MemoryIndex, TableIndex, TypeIndex, WasmHeapType, 32 WasmValType, 33 }; 34 35 /// A macro to define unsupported WebAssembly operators. 36 /// 37 /// This macro calls itself recursively; 38 /// 1. It no-ops when matching a supported operator. 39 /// 2. Defines the visitor function and panics when 40 /// matching an unsupported operator. 41 macro_rules! def_unsupported { 42 ($( @$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident $ann:tt)*) => { 43 $( 44 def_unsupported!( 45 emit 46 $op 47 48 fn $visit(&mut self $($(,$arg: $argty)*)?) -> Self::Output { 49 $($(let _ = $arg;)*)? 50 51 Err(anyhow!(CodeGenError::unimplemented_wasm_instruction())) 52 } 53 ); 54 )* 55 }; 56 57 (emit I32Const $($rest:tt)*) => {}; 58 (emit I64Const $($rest:tt)*) => {}; 59 (emit F32Const $($rest:tt)*) => {}; 60 (emit F64Const $($rest:tt)*) => {}; 61 (emit V128Const $($rest:tt)*) => {}; 62 (emit F32Add $($rest:tt)*) => {}; 63 (emit F64Add $($rest:tt)*) => {}; 64 (emit F32Sub $($rest:tt)*) => {}; 65 (emit F64Sub $($rest:tt)*) => {}; 66 (emit F32Mul $($rest:tt)*) => {}; 67 (emit F64Mul $($rest:tt)*) => {}; 68 (emit F32Div $($rest:tt)*) => {}; 69 (emit F64Div $($rest:tt)*) => {}; 70 (emit F32Min $($rest:tt)*) => {}; 71 (emit F64Min $($rest:tt)*) => {}; 72 (emit F32Max $($rest:tt)*) => {}; 73 (emit F64Max $($rest:tt)*) => {}; 74 (emit F32Copysign $($rest:tt)*) => {}; 75 (emit F64Copysign $($rest:tt)*) => {}; 76 (emit F32Abs $($rest:tt)*) => {}; 77 (emit F64Abs $($rest:tt)*) => {}; 78 (emit F32Neg $($rest:tt)*) => {}; 79 (emit F64Neg $($rest:tt)*) => {}; 80 (emit F32Floor $($rest:tt)*) => {}; 81 (emit F64Floor $($rest:tt)*) => {}; 82 (emit F32Ceil $($rest:tt)*) => {}; 83 (emit F64Ceil $($rest:tt)*) => {}; 84 (emit F32Nearest $($rest:tt)*) => {}; 85 (emit F64Nearest $($rest:tt)*) => {}; 86 (emit F32Trunc $($rest:tt)*) => {}; 87 (emit F64Trunc $($rest:tt)*) => {}; 88 (emit F32Sqrt $($rest:tt)*) => {}; 89 (emit F64Sqrt $($rest:tt)*) => {}; 90 (emit F32Eq $($rest:tt)*) => {}; 91 (emit F64Eq $($rest:tt)*) => {}; 92 (emit F32Ne $($rest:tt)*) => {}; 93 (emit F64Ne $($rest:tt)*) => {}; 94 (emit F32Lt $($rest:tt)*) => {}; 95 (emit F64Lt $($rest:tt)*) => {}; 96 (emit F32Gt $($rest:tt)*) => {}; 97 (emit F64Gt $($rest:tt)*) => {}; 98 (emit F32Le $($rest:tt)*) => {}; 99 (emit F64Le $($rest:tt)*) => {}; 100 (emit F32Ge $($rest:tt)*) => {}; 101 (emit F64Ge $($rest:tt)*) => {}; 102 (emit F32ConvertI32S $($rest:tt)*) => {}; 103 (emit F32ConvertI32U $($rest:tt)*) => {}; 104 (emit F32ConvertI64S $($rest:tt)*) => {}; 105 (emit F32ConvertI64U $($rest:tt)*) => {}; 106 (emit F64ConvertI32S $($rest:tt)*) => {}; 107 (emit F64ConvertI32U $($rest:tt)*) => {}; 108 (emit F64ConvertI64S $($rest:tt)*) => {}; 109 (emit F64ConvertI64U $($rest:tt)*) => {}; 110 (emit F32ReinterpretI32 $($rest:tt)*) => {}; 111 (emit F64ReinterpretI64 $($rest:tt)*) => {}; 112 (emit F32DemoteF64 $($rest:tt)*) => {}; 113 (emit F64PromoteF32 $($rest:tt)*) => {}; 114 (emit I32Add $($rest:tt)*) => {}; 115 (emit I64Add $($rest:tt)*) => {}; 116 (emit I32Sub $($rest:tt)*) => {}; 117 (emit I32Mul $($rest:tt)*) => {}; 118 (emit I32DivS $($rest:tt)*) => {}; 119 (emit I32DivU $($rest:tt)*) => {}; 120 (emit I64DivS $($rest:tt)*) => {}; 121 (emit I64DivU $($rest:tt)*) => {}; 122 (emit I64RemU $($rest:tt)*) => {}; 123 (emit I64RemS $($rest:tt)*) => {}; 124 (emit I32RemU $($rest:tt)*) => {}; 125 (emit I32RemS $($rest:tt)*) => {}; 126 (emit I64Mul $($rest:tt)*) => {}; 127 (emit I64Sub $($rest:tt)*) => {}; 128 (emit I32Eq $($rest:tt)*) => {}; 129 (emit I64Eq $($rest:tt)*) => {}; 130 (emit I32Ne $($rest:tt)*) => {}; 131 (emit I64Ne $($rest:tt)*) => {}; 132 (emit I32LtS $($rest:tt)*) => {}; 133 (emit I64LtS $($rest:tt)*) => {}; 134 (emit I32LtU $($rest:tt)*) => {}; 135 (emit I64LtU $($rest:tt)*) => {}; 136 (emit I32LeS $($rest:tt)*) => {}; 137 (emit I64LeS $($rest:tt)*) => {}; 138 (emit I32LeU $($rest:tt)*) => {}; 139 (emit I64LeU $($rest:tt)*) => {}; 140 (emit I32GtS $($rest:tt)*) => {}; 141 (emit I64GtS $($rest:tt)*) => {}; 142 (emit I32GtU $($rest:tt)*) => {}; 143 (emit I64GtU $($rest:tt)*) => {}; 144 (emit I32GeS $($rest:tt)*) => {}; 145 (emit I64GeS $($rest:tt)*) => {}; 146 (emit I32GeU $($rest:tt)*) => {}; 147 (emit I64GeU $($rest:tt)*) => {}; 148 (emit I32Eqz $($rest:tt)*) => {}; 149 (emit I64Eqz $($rest:tt)*) => {}; 150 (emit I32And $($rest:tt)*) => {}; 151 (emit I64And $($rest:tt)*) => {}; 152 (emit I32Or $($rest:tt)*) => {}; 153 (emit I64Or $($rest:tt)*) => {}; 154 (emit I32Xor $($rest:tt)*) => {}; 155 (emit I64Xor $($rest:tt)*) => {}; 156 (emit I32Shl $($rest:tt)*) => {}; 157 (emit I64Shl $($rest:tt)*) => {}; 158 (emit I32ShrS $($rest:tt)*) => {}; 159 (emit I64ShrS $($rest:tt)*) => {}; 160 (emit I32ShrU $($rest:tt)*) => {}; 161 (emit I64ShrU $($rest:tt)*) => {}; 162 (emit I32Rotl $($rest:tt)*) => {}; 163 (emit I64Rotl $($rest:tt)*) => {}; 164 (emit I32Rotr $($rest:tt)*) => {}; 165 (emit I64Rotr $($rest:tt)*) => {}; 166 (emit I32Clz $($rest:tt)*) => {}; 167 (emit I64Clz $($rest:tt)*) => {}; 168 (emit I32Ctz $($rest:tt)*) => {}; 169 (emit I64Ctz $($rest:tt)*) => {}; 170 (emit I32Popcnt $($rest:tt)*) => {}; 171 (emit I64Popcnt $($rest:tt)*) => {}; 172 (emit I32WrapI64 $($rest:tt)*) => {}; 173 (emit I64ExtendI32S $($rest:tt)*) => {}; 174 (emit I64ExtendI32U $($rest:tt)*) => {}; 175 (emit I32Extend8S $($rest:tt)*) => {}; 176 (emit I32Extend16S $($rest:tt)*) => {}; 177 (emit I64Extend8S $($rest:tt)*) => {}; 178 (emit I64Extend16S $($rest:tt)*) => {}; 179 (emit I64Extend32S $($rest:tt)*) => {}; 180 (emit I32TruncF32S $($rest:tt)*) => {}; 181 (emit I32TruncF32U $($rest:tt)*) => {}; 182 (emit I32TruncF64S $($rest:tt)*) => {}; 183 (emit I32TruncF64U $($rest:tt)*) => {}; 184 (emit I64TruncF32S $($rest:tt)*) => {}; 185 (emit I64TruncF32U $($rest:tt)*) => {}; 186 (emit I64TruncF64S $($rest:tt)*) => {}; 187 (emit I64TruncF64U $($rest:tt)*) => {}; 188 (emit I32ReinterpretF32 $($rest:tt)*) => {}; 189 (emit I64ReinterpretF64 $($rest:tt)*) => {}; 190 (emit LocalGet $($rest:tt)*) => {}; 191 (emit LocalSet $($rest:tt)*) => {}; 192 (emit Call $($rest:tt)*) => {}; 193 (emit End $($rest:tt)*) => {}; 194 (emit Nop $($rest:tt)*) => {}; 195 (emit If $($rest:tt)*) => {}; 196 (emit Else $($rest:tt)*) => {}; 197 (emit Block $($rest:tt)*) => {}; 198 (emit Loop $($rest:tt)*) => {}; 199 (emit Br $($rest:tt)*) => {}; 200 (emit BrIf $($rest:tt)*) => {}; 201 (emit Return $($rest:tt)*) => {}; 202 (emit Unreachable $($rest:tt)*) => {}; 203 (emit LocalTee $($rest:tt)*) => {}; 204 (emit GlobalGet $($rest:tt)*) => {}; 205 (emit GlobalSet $($rest:tt)*) => {}; 206 (emit Select $($rest:tt)*) => {}; 207 (emit Drop $($rest:tt)*) => {}; 208 (emit BrTable $($rest:tt)*) => {}; 209 (emit CallIndirect $($rest:tt)*) => {}; 210 (emit TableInit $($rest:tt)*) => {}; 211 (emit TableCopy $($rest:tt)*) => {}; 212 (emit TableGet $($rest:tt)*) => {}; 213 (emit TableSet $($rest:tt)*) => {}; 214 (emit TableGrow $($rest:tt)*) => {}; 215 (emit TableSize $($rest:tt)*) => {}; 216 (emit TableFill $($rest:tt)*) => {}; 217 (emit ElemDrop $($rest:tt)*) => {}; 218 (emit MemoryInit $($rest:tt)*) => {}; 219 (emit MemoryCopy $($rest:tt)*) => {}; 220 (emit DataDrop $($rest:tt)*) => {}; 221 (emit MemoryFill $($rest:tt)*) => {}; 222 (emit MemorySize $($rest:tt)*) => {}; 223 (emit MemoryGrow $($rest:tt)*) => {}; 224 (emit I32Load $($rest:tt)*) => {}; 225 (emit I32Load8S $($rest:tt)*) => {}; 226 (emit I32Load8U $($rest:tt)*) => {}; 227 (emit I32Load16S $($rest:tt)*) => {}; 228 (emit I32Load16U $($rest:tt)*) => {}; 229 (emit I64Load8S $($rest:tt)*) => {}; 230 (emit I64Load8U $($rest:tt)*) => {}; 231 (emit I64Load16S $($rest:tt)*) => {}; 232 (emit I64Load16U $($rest:tt)*) => {}; 233 (emit I64Load32S $($rest:tt)*) => {}; 234 (emit I64Load32U $($rest:tt)*) => {}; 235 (emit I64Load $($rest:tt)*) => {}; 236 (emit I32Store $($rest:tt)*) => {}; 237 (emit I32Store8 $($rest:tt)*) => {}; 238 (emit I32Store16 $($rest:tt)*) => {}; 239 (emit I64Store $($rest:tt)*) => {}; 240 (emit I64Store8 $($rest:tt)*) => {}; 241 (emit I64Store16 $($rest:tt)*) => {}; 242 (emit I64Store32 $($rest:tt)*) => {}; 243 (emit F32Load $($rest:tt)*) => {}; 244 (emit F32Store $($rest:tt)*) => {}; 245 (emit F64Load $($rest:tt)*) => {}; 246 (emit F64Store $($rest:tt)*) => {}; 247 (emit I32TruncSatF32S $($rest:tt)*) => {}; 248 (emit I32TruncSatF32U $($rest:tt)*) => {}; 249 (emit I32TruncSatF64S $($rest:tt)*) => {}; 250 (emit I32TruncSatF64U $($rest:tt)*) => {}; 251 (emit I64TruncSatF32S $($rest:tt)*) => {}; 252 (emit I64TruncSatF32U $($rest:tt)*) => {}; 253 (emit I64TruncSatF64S $($rest:tt)*) => {}; 254 (emit I64TruncSatF64U $($rest:tt)*) => {}; 255 (emit V128Load $($rest:tt)*) => {}; 256 (emit V128Store $($rest:tt)*) => {}; 257 (emit I64Add128 $($rest:tt)*) => {}; 258 (emit I64Sub128 $($rest:tt)*) => {}; 259 (emit I64MulWideS $($rest:tt)*) => {}; 260 (emit I64MulWideU $($rest:tt)*) => {}; 261 (emit I32AtomicLoad8U $($rest:tt)*) => {}; 262 (emit I32AtomicLoad16U $($rest:tt)*) => {}; 263 (emit I32AtomicLoad $($rest:tt)*) => {}; 264 (emit I64AtomicLoad8U $($rest:tt)*) => {}; 265 (emit I64AtomicLoad16U $($rest:tt)*) => {}; 266 (emit I64AtomicLoad32U $($rest:tt)*) => {}; 267 (emit I64AtomicLoad $($rest:tt)*) => {}; 268 (emit V128Load8x8S $($rest:tt)*) => {}; 269 (emit V128Load8x8U $($rest:tt)*) => {}; 270 (emit V128Load16x4S $($rest:tt)*) => {}; 271 (emit V128Load16x4U $($rest:tt)*) => {}; 272 (emit V128Load32x2S $($rest:tt)*) => {}; 273 (emit V128Load32x2U $($rest:tt)*) => {}; 274 (emit V128Load8Splat $($rest:tt)*) => {}; 275 (emit V128Load16Splat $($rest:tt)*) => {}; 276 (emit V128Load32Splat $($rest:tt)*) => {}; 277 (emit V128Load64Splat $($rest:tt)*) => {}; 278 (emit I8x16Splat $($rest:tt)*) => {}; 279 (emit I16x8Splat $($rest:tt)*) => {}; 280 (emit I32x4Splat $($rest:tt)*) => {}; 281 (emit I64x2Splat $($rest:tt)*) => {}; 282 (emit F32x4Splat $($rest:tt)*) => {}; 283 (emit F64x2Splat $($rest:tt)*) => {}; 284 (emit I32AtomicStore8 $($rest:tt)*) => {}; 285 (emit I32AtomicStore16 $($rest:tt)*) => {}; 286 (emit I32AtomicStore $($rest:tt)*) => {}; 287 (emit I64AtomicStore8 $($rest:tt)*) => {}; 288 (emit I64AtomicStore16 $($rest:tt)*) => {}; 289 (emit I64AtomicStore32 $($rest:tt)*) => {}; 290 (emit I64AtomicStore $($rest:tt)*) => {}; 291 (emit I32AtomicRmw8AddU $($rest:tt)*) => {}; 292 (emit I32AtomicRmw16AddU $($rest:tt)*) => {}; 293 (emit I32AtomicRmwAdd $($rest:tt)*) => {}; 294 (emit I64AtomicRmw8AddU $($rest:tt)*) => {}; 295 (emit I64AtomicRmw16AddU $($rest:tt)*) => {}; 296 (emit I64AtomicRmw32AddU $($rest:tt)*) => {}; 297 (emit I64AtomicRmwAdd $($rest:tt)*) => {}; 298 (emit I8x16Shuffle $($rest:tt)*) => {}; 299 (emit I8x16Swizzle $($rest:tt)*) => {}; 300 (emit I32AtomicRmw8SubU $($rest:tt)*) => {}; 301 (emit I32AtomicRmw16SubU $($rest:tt)*) => {}; 302 (emit I32AtomicRmwSub $($rest:tt)*) => {}; 303 (emit I64AtomicRmw8SubU $($rest:tt)*) => {}; 304 (emit I64AtomicRmw16SubU $($rest:tt)*) => {}; 305 (emit I64AtomicRmw32SubU $($rest:tt)*) => {}; 306 (emit I64AtomicRmwSub $($rest:tt)*) => {}; 307 (emit I32AtomicRmw8XchgU $($rest:tt)*) => {}; 308 (emit I32AtomicRmw16XchgU $($rest:tt)*) => {}; 309 (emit I32AtomicRmwXchg $($rest:tt)*) => {}; 310 (emit I64AtomicRmw8XchgU $($rest:tt)*) => {}; 311 (emit I64AtomicRmw16XchgU $($rest:tt)*) => {}; 312 (emit I64AtomicRmw32XchgU $($rest:tt)*) => {}; 313 (emit I64AtomicRmwXchg $($rest:tt)*) => {}; 314 (emit I8x16ExtractLaneS $($rest:tt)*) => {}; 315 (emit I8x16ExtractLaneU $($rest:tt)*) => {}; 316 (emit I16x8ExtractLaneS $($rest:tt)*) => {}; 317 (emit I16x8ExtractLaneU $($rest:tt)*) => {}; 318 (emit I32x4ExtractLane $($rest:tt)*) => {}; 319 (emit I64x2ExtractLane $($rest:tt)*) => {}; 320 (emit F32x4ExtractLane $($rest:tt)*) => {}; 321 (emit F64x2ExtractLane $($rest:tt)*) => {}; 322 (emit I32AtomicRmw8AndU $($rest:tt)*) => {}; 323 (emit I32AtomicRmw16AndU $($rest:tt)*) => {}; 324 (emit I32AtomicRmwAnd $($rest:tt)*) => {}; 325 (emit I64AtomicRmw8AndU $($rest:tt)*) => {}; 326 (emit I64AtomicRmw16AndU $($rest:tt)*) => {}; 327 (emit I64AtomicRmw32AndU $($rest:tt)*) => {}; 328 (emit I64AtomicRmwAnd $($rest:tt)*) => {}; 329 (emit I32AtomicRmw8OrU $($rest:tt)*) => {}; 330 (emit I32AtomicRmw16OrU $($rest:tt)*) => {}; 331 (emit I32AtomicRmwOr $($rest:tt)*) => {}; 332 (emit I64AtomicRmw8OrU $($rest:tt)*) => {}; 333 (emit I64AtomicRmw16OrU $($rest:tt)*) => {}; 334 (emit I64AtomicRmw32OrU $($rest:tt)*) => {}; 335 (emit I64AtomicRmwOr $($rest:tt)*) => {}; 336 (emit I32AtomicRmw8XorU $($rest:tt)*) => {}; 337 (emit I32AtomicRmw16XorU $($rest:tt)*) => {}; 338 (emit I32AtomicRmwXor $($rest:tt)*) => {}; 339 (emit I64AtomicRmw8XorU $($rest:tt)*) => {}; 340 (emit I64AtomicRmw16XorU $($rest:tt)*) => {}; 341 (emit I64AtomicRmw32XorU $($rest:tt)*) => {}; 342 (emit I64AtomicRmwXor $($rest:tt)*) => {}; 343 (emit I8x16ReplaceLane $($rest:tt)*) => {}; 344 (emit I16x8ReplaceLane $($rest:tt)*) => {}; 345 (emit I32x4ReplaceLane $($rest:tt)*) => {}; 346 (emit I64x2ReplaceLane $($rest:tt)*) => {}; 347 (emit F32x4ReplaceLane $($rest:tt)*) => {}; 348 (emit F64x2ReplaceLane $($rest:tt)*) => {}; 349 (emit I32AtomicRmw8CmpxchgU $($rest:tt)*) => {}; 350 (emit I32AtomicRmw16CmpxchgU $($rest:tt)*) => {}; 351 (emit I32AtomicRmwCmpxchg $($rest:tt)*) => {}; 352 (emit I64AtomicRmw8CmpxchgU $($rest:tt)*) => {}; 353 (emit I64AtomicRmw16CmpxchgU $($rest:tt)*) => {}; 354 (emit I64AtomicRmw32CmpxchgU $($rest:tt)*) => {}; 355 (emit I64AtomicRmwCmpxchg $($rest:tt)*) => {}; 356 (emit I8x16Eq $($rest:tt)*) => {}; 357 (emit I16x8Eq $($rest:tt)*) => {}; 358 (emit I32x4Eq $($rest:tt)*) => {}; 359 (emit I64x2Eq $($rest:tt)*) => {}; 360 (emit F32x4Eq $($rest:tt)*) => {}; 361 (emit F64x2Eq $($rest:tt)*) => {}; 362 (emit I8x16Ne $($rest:tt)*) => {}; 363 (emit I16x8Ne $($rest:tt)*) => {}; 364 (emit I32x4Ne $($rest:tt)*) => {}; 365 (emit I64x2Ne $($rest:tt)*) => {}; 366 (emit F32x4Ne $($rest:tt)*) => {}; 367 (emit F64x2Ne $($rest:tt)*) => {}; 368 (emit I8x16LtS $($rest:tt)*) => {}; 369 (emit I8x16LtU $($rest:tt)*) => {}; 370 (emit I16x8LtS $($rest:tt)*) => {}; 371 (emit I16x8LtU $($rest:tt)*) => {}; 372 (emit I32x4LtS $($rest:tt)*) => {}; 373 (emit I32x4LtU $($rest:tt)*) => {}; 374 (emit I64x2LtS $($rest:tt)*) => {}; 375 (emit F32x4Lt $($rest:tt)*) => {}; 376 (emit F64x2Lt $($rest:tt)*) => {}; 377 (emit I8x16LeS $($rest:tt)*) => {}; 378 (emit I8x16LeU $($rest:tt)*) => {}; 379 (emit I16x8LeS $($rest:tt)*) => {}; 380 (emit I16x8LeU $($rest:tt)*) => {}; 381 (emit I32x4LeS $($rest:tt)*) => {}; 382 (emit I32x4LeU $($rest:tt)*) => {}; 383 (emit I64x2LeS $($rest:tt)*) => {}; 384 (emit F32x4Le $($rest:tt)*) => {}; 385 (emit F64x2Le $($rest:tt)*) => {}; 386 (emit I8x16GtS $($rest:tt)*) => {}; 387 (emit I8x16GtU $($rest:tt)*) => {}; 388 (emit I16x8GtS $($rest:tt)*) => {}; 389 (emit I16x8GtU $($rest:tt)*) => {}; 390 (emit I32x4GtS $($rest:tt)*) => {}; 391 (emit I32x4GtU $($rest:tt)*) => {}; 392 (emit I64x2GtS $($rest:tt)*) => {}; 393 (emit F32x4Gt $($rest:tt)*) => {}; 394 (emit F64x2Gt $($rest:tt)*) => {}; 395 (emit I8x16GeS $($rest:tt)*) => {}; 396 (emit I8x16GeU $($rest:tt)*) => {}; 397 (emit I16x8GeS $($rest:tt)*) => {}; 398 (emit I16x8GeU $($rest:tt)*) => {}; 399 (emit I32x4GeS $($rest:tt)*) => {}; 400 (emit I32x4GeU $($rest:tt)*) => {}; 401 (emit I64x2GeS $($rest:tt)*) => {}; 402 (emit F32x4Ge $($rest:tt)*) => {}; 403 (emit F64x2Ge $($rest:tt)*) => {}; 404 (emit MemoryAtomicWait32 $($rest:tt)*) => {}; 405 (emit MemoryAtomicWait64 $($rest:tt)*) => {}; 406 (emit MemoryAtomicNotify $($rest:tt)*) => {}; 407 (emit AtomicFence $($rest:tt)*) => {}; 408 (emit V128Not $($rest:tt)*) => {}; 409 (emit V128And $($rest:tt)*) => {}; 410 (emit V128AndNot $($rest:tt)*) => {}; 411 (emit V128Or $($rest:tt)*) => {}; 412 (emit V128Xor $($rest:tt)*) => {}; 413 (emit V128Bitselect $($rest:tt)*) => {}; 414 (emit V128AnyTrue $($rest:tt)*) => {}; 415 (emit V128Load8Lane $($rest:tt)*) => {}; 416 (emit V128Load16Lane $($rest:tt)*) => {}; 417 (emit V128Load32Lane $($rest:tt)*) => {}; 418 (emit V128Load64Lane $($rest:tt)*) => {}; 419 (emit V128Store8Lane $($rest:tt)*) => {}; 420 (emit V128Store16Lane $($rest:tt)*) => {}; 421 (emit V128Store32Lane $($rest:tt)*) => {}; 422 (emit V128Store64Lane $($rest:tt)*) => {}; 423 (emit F32x4ConvertI32x4S $($rest:tt)*) => {}; 424 (emit F32x4ConvertI32x4U $($rest:tt)*) => {}; 425 (emit F64x2ConvertLowI32x4S $($rest:tt)*) => {}; 426 (emit F64x2ConvertLowI32x4U $($rest:tt)*) => {}; 427 (emit I8x16NarrowI16x8S $($rest:tt)*) => {}; 428 (emit I8x16NarrowI16x8U $($rest:tt)*) => {}; 429 (emit I16x8NarrowI32x4S $($rest:tt)*) => {}; 430 (emit I16x8NarrowI32x4U $($rest:tt)*) => {}; 431 (emit F32x4DemoteF64x2Zero $($rest:tt)*) => {}; 432 (emit F64x2PromoteLowF32x4 $($rest:tt)*) => {}; 433 (emit I16x8ExtendLowI8x16S $($rest:tt)*) => {}; 434 (emit I16x8ExtendHighI8x16S $($rest:tt)*) => {}; 435 (emit I16x8ExtendLowI8x16U $($rest:tt)*) => {}; 436 (emit I16x8ExtendHighI8x16U $($rest:tt)*) => {}; 437 (emit I32x4ExtendLowI16x8S $($rest:tt)*) => {}; 438 (emit I32x4ExtendHighI16x8S $($rest:tt)*) => {}; 439 (emit I32x4ExtendLowI16x8U $($rest:tt)*) => {}; 440 (emit I32x4ExtendHighI16x8U $($rest:tt)*) => {}; 441 (emit I64x2ExtendLowI32x4S $($rest:tt)*) => {}; 442 (emit I64x2ExtendHighI32x4S $($rest:tt)*) => {}; 443 (emit I64x2ExtendLowI32x4U $($rest:tt)*) => {}; 444 (emit I64x2ExtendHighI32x4U $($rest:tt)*) => {}; 445 (emit I8x16Add $($rest:tt)*) => {}; 446 (emit I16x8Add $($rest:tt)*) => {}; 447 (emit I32x4Add $($rest:tt)*) => {}; 448 (emit I64x2Add $($rest:tt)*) => {}; 449 (emit I8x16Sub $($rest:tt)*) => {}; 450 (emit I16x8Sub $($rest:tt)*) => {}; 451 (emit I32x4Sub $($rest:tt)*) => {}; 452 (emit I64x2Sub $($rest:tt)*) => {}; 453 (emit I16x8Mul $($rest:tt)*) => {}; 454 (emit I32x4Mul $($rest:tt)*) => {}; 455 (emit I64x2Mul $($rest:tt)*) => {}; 456 (emit I8x16AddSatS $($rest:tt)*) => {}; 457 (emit I16x8AddSatS $($rest:tt)*) => {}; 458 (emit I8x16AddSatU $($rest:tt)*) => {}; 459 (emit I16x8AddSatU $($rest:tt)*) => {}; 460 (emit I8x16SubSatS $($rest:tt)*) => {}; 461 (emit I16x8SubSatS $($rest:tt)*) => {}; 462 (emit I8x16SubSatU $($rest:tt)*) => {}; 463 (emit I16x8SubSatU $($rest:tt)*) => {}; 464 (emit I8x16Abs $($rest:tt)*) => {}; 465 (emit I16x8Abs $($rest:tt)*) => {}; 466 (emit I32x4Abs $($rest:tt)*) => {}; 467 (emit I64x2Abs $($rest:tt)*) => {}; 468 (emit F32x4Abs $($rest:tt)*) => {}; 469 (emit F64x2Abs $($rest:tt)*) => {}; 470 (emit I8x16Neg $($rest:tt)*) => {}; 471 (emit I16x8Neg $($rest:tt)*) => {}; 472 (emit I32x4Neg $($rest:tt)*) => {}; 473 (emit I64x2Neg $($rest:tt)*) => {}; 474 (emit I8x16Shl $($rest:tt)*) => {}; 475 (emit I16x8Shl $($rest:tt)*) => {}; 476 (emit I32x4Shl $($rest:tt)*) => {}; 477 (emit I64x2Shl $($rest:tt)*) => {}; 478 (emit I8x16ShrU $($rest:tt)*) => {}; 479 (emit I16x8ShrU $($rest:tt)*) => {}; 480 (emit I32x4ShrU $($rest:tt)*) => {}; 481 (emit I64x2ShrU $($rest:tt)*) => {}; 482 (emit I8x16ShrS $($rest:tt)*) => {}; 483 (emit I16x8ShrS $($rest:tt)*) => {}; 484 (emit I32x4ShrS $($rest:tt)*) => {}; 485 (emit I64x2ShrS $($rest:tt)*) => {}; 486 (emit I16x8Q15MulrSatS $($rest:tt)*) => {}; 487 (emit I8x16AllTrue $($rest:tt)*) => {}; 488 (emit I16x8AllTrue $($rest:tt)*) => {}; 489 (emit I32x4AllTrue $($rest:tt)*) => {}; 490 (emit I64x2AllTrue $($rest:tt)*) => {}; 491 (emit I8x16Bitmask $($rest:tt)*) => {}; 492 (emit I16x8Bitmask $($rest:tt)*) => {}; 493 (emit I32x4Bitmask $($rest:tt)*) => {}; 494 (emit I64x2Bitmask $($rest:tt)*) => {}; 495 (emit I32x4TruncSatF32x4S $($rest:tt)*) => {}; 496 (emit I32x4TruncSatF32x4U $($rest:tt)*) => {}; 497 (emit I32x4TruncSatF64x2SZero $($rest:tt)*) => {}; 498 (emit I32x4TruncSatF64x2UZero $($rest:tt)*) => {}; 499 (emit I8x16MinU $($rest:tt)*) => {}; 500 (emit I16x8MinU $($rest:tt)*) => {}; 501 (emit I32x4MinU $($rest:tt)*) => {}; 502 (emit I8x16MinS $($rest:tt)*) => {}; 503 (emit I16x8MinS $($rest:tt)*) => {}; 504 (emit I32x4MinS $($rest:tt)*) => {}; 505 (emit I8x16MaxU $($rest:tt)*) => {}; 506 (emit I16x8MaxU $($rest:tt)*) => {}; 507 (emit I32x4MaxU $($rest:tt)*) => {}; 508 (emit I8x16MaxS $($rest:tt)*) => {}; 509 (emit I16x8MaxS $($rest:tt)*) => {}; 510 (emit I32x4MaxS $($rest:tt)*) => {}; 511 (emit I16x8ExtMulLowI8x16S $($rest:tt)*) => {}; 512 (emit I32x4ExtMulLowI16x8S $($rest:tt)*) => {}; 513 (emit I64x2ExtMulLowI32x4S $($rest:tt)*) => {}; 514 (emit I16x8ExtMulHighI8x16S $($rest:tt)*) => {}; 515 (emit I32x4ExtMulHighI16x8S $($rest:tt)*) => {}; 516 (emit I64x2ExtMulHighI32x4S $($rest:tt)*) => {}; 517 (emit I16x8ExtMulLowI8x16U $($rest:tt)*) => {}; 518 (emit I32x4ExtMulLowI16x8U $($rest:tt)*) => {}; 519 (emit I64x2ExtMulLowI32x4U $($rest:tt)*) => {}; 520 (emit I16x8ExtMulHighI8x16U $($rest:tt)*) => {}; 521 (emit I32x4ExtMulHighI16x8U $($rest:tt)*) => {}; 522 (emit I64x2ExtMulHighI32x4U $($rest:tt)*) => {}; 523 (emit I16x8ExtAddPairwiseI8x16U $($rest:tt)*) => {}; 524 (emit I16x8ExtAddPairwiseI8x16S $($rest:tt)*) => {}; 525 (emit I32x4ExtAddPairwiseI16x8U $($rest:tt)*) => {}; 526 (emit I32x4ExtAddPairwiseI16x8S $($rest:tt)*) => {}; 527 (emit I32x4DotI16x8S $($rest:tt)*) => {}; 528 (emit I8x16Popcnt $($rest:tt)*) => {}; 529 (emit I8x16AvgrU $($rest:tt)*) => {}; 530 (emit I16x8AvgrU $($rest:tt)*) => {}; 531 (emit F32x4Add $($rest:tt)*) => {}; 532 (emit F64x2Add $($rest:tt)*) => {}; 533 (emit F32x4Sub $($rest:tt)*) => {}; 534 (emit F64x2Sub $($rest:tt)*) => {}; 535 (emit F32x4Mul $($rest:tt)*) => {}; 536 (emit F64x2Mul $($rest:tt)*) => {}; 537 (emit F32x4Div $($rest:tt)*) => {}; 538 (emit F64x2Div $($rest:tt)*) => {}; 539 (emit F32x4Neg $($rest:tt)*) => {}; 540 (emit F64x2Neg $($rest:tt)*) => {}; 541 (emit F32x4Sqrt $($rest:tt)*) => {}; 542 (emit F64x2Sqrt $($rest:tt)*) => {}; 543 (emit F32x4Ceil $($rest:tt)*) => {}; 544 (emit F64x2Ceil $($rest:tt)*) => {}; 545 (emit F32x4Floor $($rest:tt)*) => {}; 546 (emit F64x2Floor $($rest:tt)*) => {}; 547 (emit F32x4Nearest $($rest:tt)*) => {}; 548 (emit F64x2Nearest $($rest:tt)*) => {}; 549 (emit F32x4Trunc $($rest:tt)*) => {}; 550 (emit F64x2Trunc $($rest:tt)*) => {}; 551 (emit V128Load32Zero $($rest:tt)*) => {}; 552 (emit V128Load64Zero $($rest:tt)*) => {}; 553 (emit F32x4PMin $($rest:tt)*) => {}; 554 (emit F64x2PMin $($rest:tt)*) => {}; 555 (emit F32x4PMax $($rest:tt)*) => {}; 556 (emit F64x2PMax $($rest:tt)*) => {}; 557 (emit F32x4Min $($rest:tt)*) => {}; 558 (emit F64x2Min $($rest:tt)*) => {}; 559 (emit F32x4Max $($rest:tt)*) => {}; 560 (emit F64x2Max $($rest:tt)*) => {}; 561 562 (emit $unsupported:tt $($rest:tt)*) => {$($rest)*}; 563 } 564 565 impl<'a, 'translation, 'data, M> VisitOperator<'a> for CodeGen<'a, 'translation, 'data, M, Emission> 566 where 567 M: MacroAssembler, 568 { 569 type Output = Result<()>; 570 571 fn visit_i32_const(&mut self, val: i32) -> Self::Output { 572 self.context.stack.push(Val::i32(val)); 573 574 Ok(()) 575 } 576 577 fn visit_i64_const(&mut self, val: i64) -> Self::Output { 578 self.context.stack.push(Val::i64(val)); 579 Ok(()) 580 } 581 582 fn visit_f32_const(&mut self, val: Ieee32) -> Self::Output { 583 self.context.stack.push(Val::f32(val)); 584 Ok(()) 585 } 586 587 fn visit_f64_const(&mut self, val: Ieee64) -> Self::Output { 588 self.context.stack.push(Val::f64(val)); 589 Ok(()) 590 } 591 592 fn visit_f32_add(&mut self) -> Self::Output { 593 self.context.binop( 594 self.masm, 595 OperandSize::S32, 596 &mut |masm: &mut M, dst, src, size| { 597 masm.float_add(writable!(dst), dst, src, size)?; 598 Ok(TypedReg::f32(dst)) 599 }, 600 ) 601 } 602 603 fn visit_f64_add(&mut self) -> Self::Output { 604 self.context.binop( 605 self.masm, 606 OperandSize::S64, 607 &mut |masm: &mut M, dst, src, size| { 608 masm.float_add(writable!(dst), dst, src, size)?; 609 Ok(TypedReg::f64(dst)) 610 }, 611 ) 612 } 613 614 fn visit_f32_sub(&mut self) -> Self::Output { 615 self.context.binop( 616 self.masm, 617 OperandSize::S32, 618 &mut |masm: &mut M, dst, src, size| { 619 masm.float_sub(writable!(dst), dst, src, size)?; 620 Ok(TypedReg::f32(dst)) 621 }, 622 ) 623 } 624 625 fn visit_f64_sub(&mut self) -> Self::Output { 626 self.context.binop( 627 self.masm, 628 OperandSize::S64, 629 &mut |masm: &mut M, dst, src, size| { 630 masm.float_sub(writable!(dst), dst, src, size)?; 631 Ok(TypedReg::f64(dst)) 632 }, 633 ) 634 } 635 636 fn visit_f32_mul(&mut self) -> Self::Output { 637 self.context.binop( 638 self.masm, 639 OperandSize::S32, 640 &mut |masm: &mut M, dst, src, size| { 641 masm.float_mul(writable!(dst), dst, src, size)?; 642 Ok(TypedReg::f32(dst)) 643 }, 644 ) 645 } 646 647 fn visit_f64_mul(&mut self) -> Self::Output { 648 self.context.binop( 649 self.masm, 650 OperandSize::S64, 651 &mut |masm: &mut M, dst, src, size| { 652 masm.float_mul(writable!(dst), dst, src, size)?; 653 Ok(TypedReg::f64(dst)) 654 }, 655 ) 656 } 657 658 fn visit_f32_div(&mut self) -> Self::Output { 659 self.context.binop( 660 self.masm, 661 OperandSize::S32, 662 &mut |masm: &mut M, dst, src, size| { 663 masm.float_div(writable!(dst), dst, src, size)?; 664 Ok(TypedReg::f32(dst)) 665 }, 666 ) 667 } 668 669 fn visit_f64_div(&mut self) -> Self::Output { 670 self.context.binop( 671 self.masm, 672 OperandSize::S64, 673 &mut |masm: &mut M, dst, src, size| { 674 masm.float_div(writable!(dst), dst, src, size)?; 675 Ok(TypedReg::f64(dst)) 676 }, 677 ) 678 } 679 680 fn visit_f32_min(&mut self) -> Self::Output { 681 self.context.binop( 682 self.masm, 683 OperandSize::S32, 684 &mut |masm: &mut M, dst, src, size| { 685 masm.float_min(writable!(dst), dst, src, size)?; 686 Ok(TypedReg::f32(dst)) 687 }, 688 ) 689 } 690 691 fn visit_f64_min(&mut self) -> Self::Output { 692 self.context.binop( 693 self.masm, 694 OperandSize::S64, 695 &mut |masm: &mut M, dst, src, size| { 696 masm.float_min(writable!(dst), dst, src, size)?; 697 Ok(TypedReg::f64(dst)) 698 }, 699 ) 700 } 701 702 fn visit_f32_max(&mut self) -> Self::Output { 703 self.context.binop( 704 self.masm, 705 OperandSize::S32, 706 &mut |masm: &mut M, dst, src, size| { 707 masm.float_max(writable!(dst), dst, src, size)?; 708 Ok(TypedReg::f32(dst)) 709 }, 710 ) 711 } 712 713 fn visit_f64_max(&mut self) -> Self::Output { 714 self.context.binop( 715 self.masm, 716 OperandSize::S64, 717 &mut |masm: &mut M, dst, src, size| { 718 masm.float_max(writable!(dst), dst, src, size)?; 719 Ok(TypedReg::f64(dst)) 720 }, 721 ) 722 } 723 724 fn visit_f32_copysign(&mut self) -> Self::Output { 725 self.context.binop( 726 self.masm, 727 OperandSize::S32, 728 &mut |masm: &mut M, dst, src, size| { 729 masm.float_copysign(writable!(dst), dst, src, size)?; 730 Ok(TypedReg::f32(dst)) 731 }, 732 ) 733 } 734 735 fn visit_f64_copysign(&mut self) -> Self::Output { 736 self.context.binop( 737 self.masm, 738 OperandSize::S64, 739 &mut |masm: &mut M, dst, src, size| { 740 masm.float_copysign(writable!(dst), dst, src, size)?; 741 Ok(TypedReg::f64(dst)) 742 }, 743 ) 744 } 745 746 fn visit_f32_abs(&mut self) -> Self::Output { 747 self.context.unop(self.masm, |masm, reg| { 748 masm.float_abs(writable!(reg), OperandSize::S32)?; 749 Ok(TypedReg::f32(reg)) 750 }) 751 } 752 753 fn visit_f64_abs(&mut self) -> Self::Output { 754 self.context.unop(self.masm, |masm, reg| { 755 masm.float_abs(writable!(reg), OperandSize::S64)?; 756 Ok(TypedReg::f64(reg)) 757 }) 758 } 759 760 fn visit_f32_neg(&mut self) -> Self::Output { 761 self.context.unop(self.masm, |masm, reg| { 762 masm.float_neg(writable!(reg), OperandSize::S32)?; 763 Ok(TypedReg::f32(reg)) 764 }) 765 } 766 767 fn visit_f64_neg(&mut self) -> Self::Output { 768 self.context.unop(self.masm, |masm, reg| { 769 masm.float_neg(writable!(reg), OperandSize::S64)?; 770 Ok(TypedReg::f64(reg)) 771 }) 772 } 773 774 fn visit_f32_floor(&mut self) -> Self::Output { 775 self.masm.float_round( 776 RoundingMode::Down, 777 &mut self.env, 778 &mut self.context, 779 OperandSize::S32, 780 |env, cx, masm| { 781 let builtin = env.builtins.floor_f32::<M::ABI, M::Ptr>()?; 782 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 783 }, 784 ) 785 } 786 787 fn visit_f64_floor(&mut self) -> Self::Output { 788 self.masm.float_round( 789 RoundingMode::Down, 790 &mut self.env, 791 &mut self.context, 792 OperandSize::S64, 793 |env, cx, masm| { 794 let builtin = env.builtins.floor_f64::<M::ABI, M::Ptr>()?; 795 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 796 }, 797 ) 798 } 799 800 fn visit_f32_ceil(&mut self) -> Self::Output { 801 self.masm.float_round( 802 RoundingMode::Up, 803 &mut self.env, 804 &mut self.context, 805 OperandSize::S32, 806 |env, cx, masm| { 807 let builtin = env.builtins.ceil_f32::<M::ABI, M::Ptr>()?; 808 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 809 }, 810 ) 811 } 812 813 fn visit_f64_ceil(&mut self) -> Self::Output { 814 self.masm.float_round( 815 RoundingMode::Up, 816 &mut self.env, 817 &mut self.context, 818 OperandSize::S64, 819 |env, cx, masm| { 820 let builtin = env.builtins.ceil_f64::<M::ABI, M::Ptr>()?; 821 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 822 }, 823 ) 824 } 825 826 fn visit_f32_nearest(&mut self) -> Self::Output { 827 self.masm.float_round( 828 RoundingMode::Nearest, 829 &mut self.env, 830 &mut self.context, 831 OperandSize::S32, 832 |env, cx, masm| { 833 let builtin = env.builtins.nearest_f32::<M::ABI, M::Ptr>()?; 834 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 835 }, 836 ) 837 } 838 839 fn visit_f64_nearest(&mut self) -> Self::Output { 840 self.masm.float_round( 841 RoundingMode::Nearest, 842 &mut self.env, 843 &mut self.context, 844 OperandSize::S64, 845 |env, cx, masm| { 846 let builtin = env.builtins.nearest_f64::<M::ABI, M::Ptr>()?; 847 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 848 }, 849 ) 850 } 851 852 fn visit_f32_trunc(&mut self) -> Self::Output { 853 self.masm.float_round( 854 RoundingMode::Zero, 855 &mut self.env, 856 &mut self.context, 857 OperandSize::S32, 858 |env, cx, masm| { 859 let builtin = env.builtins.trunc_f32::<M::ABI, M::Ptr>()?; 860 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 861 }, 862 ) 863 } 864 865 fn visit_f64_trunc(&mut self) -> Self::Output { 866 self.masm.float_round( 867 RoundingMode::Zero, 868 &mut self.env, 869 &mut self.context, 870 OperandSize::S64, 871 |env, cx, masm| { 872 let builtin = env.builtins.trunc_f64::<M::ABI, M::Ptr>()?; 873 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 874 }, 875 ) 876 } 877 878 fn visit_f32_sqrt(&mut self) -> Self::Output { 879 self.context.unop(self.masm, |masm, reg| { 880 masm.float_sqrt(writable!(reg), reg, OperandSize::S32)?; 881 Ok(TypedReg::f32(reg)) 882 }) 883 } 884 885 fn visit_f64_sqrt(&mut self) -> Self::Output { 886 self.context.unop(self.masm, |masm, reg| { 887 masm.float_sqrt(writable!(reg), reg, OperandSize::S64)?; 888 Ok(TypedReg::f64(reg)) 889 }) 890 } 891 892 fn visit_f32_eq(&mut self) -> Self::Output { 893 self.context.float_cmp_op( 894 self.masm, 895 OperandSize::S32, 896 &mut |masm: &mut M, dst, src1, src2, size| { 897 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Eq, size) 898 }, 899 ) 900 } 901 902 fn visit_f64_eq(&mut self) -> Self::Output { 903 self.context.float_cmp_op( 904 self.masm, 905 OperandSize::S64, 906 &mut |masm: &mut M, dst, src1, src2, size| { 907 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Eq, size) 908 }, 909 ) 910 } 911 912 fn visit_f32_ne(&mut self) -> Self::Output { 913 self.context.float_cmp_op( 914 self.masm, 915 OperandSize::S32, 916 &mut |masm: &mut M, dst, src1, src2, size| { 917 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ne, size) 918 }, 919 ) 920 } 921 922 fn visit_f64_ne(&mut self) -> Self::Output { 923 self.context.float_cmp_op( 924 self.masm, 925 OperandSize::S64, 926 &mut |masm: &mut M, dst, src1, src2, size| { 927 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ne, size) 928 }, 929 ) 930 } 931 932 fn visit_f32_lt(&mut self) -> Self::Output { 933 self.context.float_cmp_op( 934 self.masm, 935 OperandSize::S32, 936 &mut |masm: &mut M, dst, src1, src2, size| { 937 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Lt, size) 938 }, 939 ) 940 } 941 942 fn visit_f64_lt(&mut self) -> Self::Output { 943 self.context.float_cmp_op( 944 self.masm, 945 OperandSize::S64, 946 &mut |masm: &mut M, dst, src1, src2, size| { 947 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Lt, size) 948 }, 949 ) 950 } 951 952 fn visit_f32_gt(&mut self) -> Self::Output { 953 self.context.float_cmp_op( 954 self.masm, 955 OperandSize::S32, 956 &mut |masm: &mut M, dst, src1, src2, size| { 957 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Gt, size) 958 }, 959 ) 960 } 961 962 fn visit_f64_gt(&mut self) -> Self::Output { 963 self.context.float_cmp_op( 964 self.masm, 965 OperandSize::S64, 966 &mut |masm: &mut M, dst, src1, src2, size| { 967 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Gt, size) 968 }, 969 ) 970 } 971 972 fn visit_f32_le(&mut self) -> Self::Output { 973 self.context.float_cmp_op( 974 self.masm, 975 OperandSize::S32, 976 &mut |masm: &mut M, dst, src1, src2, size| { 977 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Le, size) 978 }, 979 ) 980 } 981 982 fn visit_f64_le(&mut self) -> Self::Output { 983 self.context.float_cmp_op( 984 self.masm, 985 OperandSize::S64, 986 &mut |masm: &mut M, dst, src1, src2, size| { 987 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Le, size) 988 }, 989 ) 990 } 991 992 fn visit_f32_ge(&mut self) -> Self::Output { 993 self.context.float_cmp_op( 994 self.masm, 995 OperandSize::S32, 996 &mut |masm: &mut M, dst, src1, src2, size| { 997 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ge, size) 998 }, 999 ) 1000 } 1001 1002 fn visit_f64_ge(&mut self) -> Self::Output { 1003 self.context.float_cmp_op( 1004 self.masm, 1005 OperandSize::S64, 1006 &mut |masm: &mut M, dst, src1, src2, size| { 1007 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ge, size) 1008 }, 1009 ) 1010 } 1011 1012 fn visit_f32_convert_i32_s(&mut self) -> Self::Output { 1013 self.context 1014 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 1015 masm.signed_convert(writable!(dst), src, OperandSize::S32, dst_size) 1016 }) 1017 } 1018 1019 fn visit_f32_convert_i32_u(&mut self) -> Self::Output { 1020 self.context.convert_op_with_tmp_reg( 1021 self.masm, 1022 WasmValType::F32, 1023 RegClass::Int, 1024 |masm, dst, src, tmp_gpr, dst_size| { 1025 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S32, dst_size) 1026 }, 1027 ) 1028 } 1029 1030 fn visit_f32_convert_i64_s(&mut self) -> Self::Output { 1031 self.context 1032 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 1033 masm.signed_convert(writable!(dst), src, OperandSize::S64, dst_size) 1034 }) 1035 } 1036 1037 fn visit_f32_convert_i64_u(&mut self) -> Self::Output { 1038 self.context.convert_op_with_tmp_reg( 1039 self.masm, 1040 WasmValType::F32, 1041 RegClass::Int, 1042 |masm, dst, src, tmp_gpr, dst_size| { 1043 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S64, dst_size) 1044 }, 1045 ) 1046 } 1047 1048 fn visit_f64_convert_i32_s(&mut self) -> Self::Output { 1049 self.context 1050 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 1051 masm.signed_convert(writable!(dst), src, OperandSize::S32, dst_size) 1052 }) 1053 } 1054 1055 fn visit_f64_convert_i32_u(&mut self) -> Self::Output { 1056 self.context.convert_op_with_tmp_reg( 1057 self.masm, 1058 WasmValType::F64, 1059 RegClass::Int, 1060 |masm, dst, src, tmp_gpr, dst_size| { 1061 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S32, dst_size) 1062 }, 1063 ) 1064 } 1065 1066 fn visit_f64_convert_i64_s(&mut self) -> Self::Output { 1067 self.context 1068 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 1069 masm.signed_convert(writable!(dst), src, OperandSize::S64, dst_size) 1070 }) 1071 } 1072 1073 fn visit_f64_convert_i64_u(&mut self) -> Self::Output { 1074 self.context.convert_op_with_tmp_reg( 1075 self.masm, 1076 WasmValType::F64, 1077 RegClass::Int, 1078 |masm, dst, src, tmp_gpr, dst_size| { 1079 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S64, dst_size) 1080 }, 1081 ) 1082 } 1083 1084 fn visit_f32_reinterpret_i32(&mut self) -> Self::Output { 1085 self.context 1086 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, size| { 1087 masm.reinterpret_int_as_float(writable!(dst), src.into(), size) 1088 }) 1089 } 1090 1091 fn visit_f64_reinterpret_i64(&mut self) -> Self::Output { 1092 self.context 1093 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, size| { 1094 masm.reinterpret_int_as_float(writable!(dst), src.into(), size) 1095 }) 1096 } 1097 1098 fn visit_f32_demote_f64(&mut self) -> Self::Output { 1099 self.context.unop(self.masm, |masm, reg| { 1100 masm.demote(writable!(reg), reg)?; 1101 Ok(TypedReg::f32(reg)) 1102 }) 1103 } 1104 1105 fn visit_f64_promote_f32(&mut self) -> Self::Output { 1106 self.context.unop(self.masm, |masm, reg| { 1107 masm.promote(writable!(reg), reg)?; 1108 Ok(TypedReg::f64(reg)) 1109 }) 1110 } 1111 1112 fn visit_i32_add(&mut self) -> Self::Output { 1113 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1114 masm.add(writable!(dst), dst, src, size)?; 1115 Ok(TypedReg::i32(dst)) 1116 }) 1117 } 1118 1119 fn visit_i64_add(&mut self) -> Self::Output { 1120 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1121 masm.add(writable!(dst), dst, src, size)?; 1122 Ok(TypedReg::i64(dst)) 1123 }) 1124 } 1125 1126 fn visit_i32_sub(&mut self) -> Self::Output { 1127 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1128 masm.sub(writable!(dst), dst, src, size)?; 1129 Ok(TypedReg::i32(dst)) 1130 }) 1131 } 1132 1133 fn visit_i64_sub(&mut self) -> Self::Output { 1134 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1135 masm.sub(writable!(dst), dst, src, size)?; 1136 Ok(TypedReg::i64(dst)) 1137 }) 1138 } 1139 1140 fn visit_i32_mul(&mut self) -> Self::Output { 1141 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1142 masm.mul(writable!(dst), dst, src, size)?; 1143 Ok(TypedReg::i32(dst)) 1144 }) 1145 } 1146 1147 fn visit_i64_mul(&mut self) -> Self::Output { 1148 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1149 masm.mul(writable!(dst), dst, src, size)?; 1150 Ok(TypedReg::i64(dst)) 1151 }) 1152 } 1153 1154 fn visit_i32_div_s(&mut self) -> Self::Output { 1155 use DivKind::*; 1156 use OperandSize::*; 1157 1158 self.masm.div(&mut self.context, Signed, S32) 1159 } 1160 1161 fn visit_i32_div_u(&mut self) -> Self::Output { 1162 use DivKind::*; 1163 use OperandSize::*; 1164 1165 self.masm.div(&mut self.context, Unsigned, S32) 1166 } 1167 1168 fn visit_i64_div_s(&mut self) -> Self::Output { 1169 use DivKind::*; 1170 use OperandSize::*; 1171 1172 self.masm.div(&mut self.context, Signed, S64) 1173 } 1174 1175 fn visit_i64_div_u(&mut self) -> Self::Output { 1176 use DivKind::*; 1177 use OperandSize::*; 1178 1179 self.masm.div(&mut self.context, Unsigned, S64) 1180 } 1181 1182 fn visit_i32_rem_s(&mut self) -> Self::Output { 1183 use OperandSize::*; 1184 use RemKind::*; 1185 1186 self.masm.rem(&mut self.context, Signed, S32) 1187 } 1188 1189 fn visit_i32_rem_u(&mut self) -> Self::Output { 1190 use OperandSize::*; 1191 use RemKind::*; 1192 1193 self.masm.rem(&mut self.context, Unsigned, S32) 1194 } 1195 1196 fn visit_i64_rem_s(&mut self) -> Self::Output { 1197 use OperandSize::*; 1198 use RemKind::*; 1199 1200 self.masm.rem(&mut self.context, Signed, S64) 1201 } 1202 1203 fn visit_i64_rem_u(&mut self) -> Self::Output { 1204 use OperandSize::*; 1205 use RemKind::*; 1206 1207 self.masm.rem(&mut self.context, Unsigned, S64) 1208 } 1209 1210 fn visit_i32_eq(&mut self) -> Self::Output { 1211 self.cmp_i32s(IntCmpKind::Eq) 1212 } 1213 1214 fn visit_i64_eq(&mut self) -> Self::Output { 1215 self.cmp_i64s(IntCmpKind::Eq) 1216 } 1217 1218 fn visit_i32_ne(&mut self) -> Self::Output { 1219 self.cmp_i32s(IntCmpKind::Ne) 1220 } 1221 1222 fn visit_i64_ne(&mut self) -> Self::Output { 1223 self.cmp_i64s(IntCmpKind::Ne) 1224 } 1225 1226 fn visit_i32_lt_s(&mut self) -> Self::Output { 1227 self.cmp_i32s(IntCmpKind::LtS) 1228 } 1229 1230 fn visit_i64_lt_s(&mut self) -> Self::Output { 1231 self.cmp_i64s(IntCmpKind::LtS) 1232 } 1233 1234 fn visit_i32_lt_u(&mut self) -> Self::Output { 1235 self.cmp_i32s(IntCmpKind::LtU) 1236 } 1237 1238 fn visit_i64_lt_u(&mut self) -> Self::Output { 1239 self.cmp_i64s(IntCmpKind::LtU) 1240 } 1241 1242 fn visit_i32_le_s(&mut self) -> Self::Output { 1243 self.cmp_i32s(IntCmpKind::LeS) 1244 } 1245 1246 fn visit_i64_le_s(&mut self) -> Self::Output { 1247 self.cmp_i64s(IntCmpKind::LeS) 1248 } 1249 1250 fn visit_i32_le_u(&mut self) -> Self::Output { 1251 self.cmp_i32s(IntCmpKind::LeU) 1252 } 1253 1254 fn visit_i64_le_u(&mut self) -> Self::Output { 1255 self.cmp_i64s(IntCmpKind::LeU) 1256 } 1257 1258 fn visit_i32_gt_s(&mut self) -> Self::Output { 1259 self.cmp_i32s(IntCmpKind::GtS) 1260 } 1261 1262 fn visit_i64_gt_s(&mut self) -> Self::Output { 1263 self.cmp_i64s(IntCmpKind::GtS) 1264 } 1265 1266 fn visit_i32_gt_u(&mut self) -> Self::Output { 1267 self.cmp_i32s(IntCmpKind::GtU) 1268 } 1269 1270 fn visit_i64_gt_u(&mut self) -> Self::Output { 1271 self.cmp_i64s(IntCmpKind::GtU) 1272 } 1273 1274 fn visit_i32_ge_s(&mut self) -> Self::Output { 1275 self.cmp_i32s(IntCmpKind::GeS) 1276 } 1277 1278 fn visit_i64_ge_s(&mut self) -> Self::Output { 1279 self.cmp_i64s(IntCmpKind::GeS) 1280 } 1281 1282 fn visit_i32_ge_u(&mut self) -> Self::Output { 1283 self.cmp_i32s(IntCmpKind::GeU) 1284 } 1285 1286 fn visit_i64_ge_u(&mut self) -> Self::Output { 1287 self.cmp_i64s(IntCmpKind::GeU) 1288 } 1289 1290 fn visit_i32_eqz(&mut self) -> Self::Output { 1291 use OperandSize::*; 1292 1293 self.context.unop(self.masm, |masm, reg| { 1294 masm.cmp_with_set(writable!(reg.into()), RegImm::i32(0), IntCmpKind::Eq, S32)?; 1295 Ok(TypedReg::i32(reg)) 1296 }) 1297 } 1298 1299 fn visit_i64_eqz(&mut self) -> Self::Output { 1300 use OperandSize::*; 1301 1302 self.context.unop(self.masm, |masm, reg| { 1303 masm.cmp_with_set(writable!(reg.into()), RegImm::i64(0), IntCmpKind::Eq, S64)?; 1304 Ok(TypedReg::i32(reg)) // Return value for `i64.eqz` is an `i32`. 1305 }) 1306 } 1307 1308 fn visit_i32_clz(&mut self) -> Self::Output { 1309 use OperandSize::*; 1310 1311 self.context.unop(self.masm, |masm, reg| { 1312 masm.clz(writable!(reg), reg, S32)?; 1313 Ok(TypedReg::i32(reg)) 1314 }) 1315 } 1316 1317 fn visit_i64_clz(&mut self) -> Self::Output { 1318 use OperandSize::*; 1319 1320 self.context.unop(self.masm, |masm, reg| { 1321 masm.clz(writable!(reg), reg, S64)?; 1322 Ok(TypedReg::i64(reg)) 1323 }) 1324 } 1325 1326 fn visit_i32_ctz(&mut self) -> Self::Output { 1327 use OperandSize::*; 1328 1329 self.context.unop(self.masm, |masm, reg| { 1330 masm.ctz(writable!(reg), reg, S32)?; 1331 Ok(TypedReg::i32(reg)) 1332 }) 1333 } 1334 1335 fn visit_i64_ctz(&mut self) -> Self::Output { 1336 use OperandSize::*; 1337 1338 self.context.unop(self.masm, |masm, reg| { 1339 masm.ctz(writable!(reg), reg, S64)?; 1340 Ok(TypedReg::i64(reg)) 1341 }) 1342 } 1343 1344 fn visit_i32_and(&mut self) -> Self::Output { 1345 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1346 masm.and(writable!(dst), dst, src, size)?; 1347 Ok(TypedReg::i32(dst)) 1348 }) 1349 } 1350 1351 fn visit_i64_and(&mut self) -> Self::Output { 1352 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1353 masm.and(writable!(dst), dst, src, size)?; 1354 Ok(TypedReg::i64(dst)) 1355 }) 1356 } 1357 1358 fn visit_i32_or(&mut self) -> Self::Output { 1359 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1360 masm.or(writable!(dst), dst, src, size)?; 1361 Ok(TypedReg::i32(dst)) 1362 }) 1363 } 1364 1365 fn visit_i64_or(&mut self) -> Self::Output { 1366 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1367 masm.or(writable!(dst), dst, src, size)?; 1368 Ok(TypedReg::i64(dst)) 1369 }) 1370 } 1371 1372 fn visit_i32_xor(&mut self) -> Self::Output { 1373 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1374 masm.xor(writable!(dst), dst, src, size)?; 1375 Ok(TypedReg::i32(dst)) 1376 }) 1377 } 1378 1379 fn visit_i64_xor(&mut self) -> Self::Output { 1380 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1381 masm.xor(writable!(dst), dst, src, size)?; 1382 Ok(TypedReg::i64(dst)) 1383 }) 1384 } 1385 1386 fn visit_i32_shl(&mut self) -> Self::Output { 1387 use ShiftKind::*; 1388 1389 self.context.i32_shift(self.masm, Shl) 1390 } 1391 1392 fn visit_i64_shl(&mut self) -> Self::Output { 1393 use ShiftKind::*; 1394 1395 self.context.i64_shift(self.masm, Shl) 1396 } 1397 1398 fn visit_i32_shr_s(&mut self) -> Self::Output { 1399 use ShiftKind::*; 1400 1401 self.context.i32_shift(self.masm, ShrS) 1402 } 1403 1404 fn visit_i64_shr_s(&mut self) -> Self::Output { 1405 use ShiftKind::*; 1406 1407 self.context.i64_shift(self.masm, ShrS) 1408 } 1409 1410 fn visit_i32_shr_u(&mut self) -> Self::Output { 1411 use ShiftKind::*; 1412 1413 self.context.i32_shift(self.masm, ShrU) 1414 } 1415 1416 fn visit_i64_shr_u(&mut self) -> Self::Output { 1417 use ShiftKind::*; 1418 1419 self.context.i64_shift(self.masm, ShrU) 1420 } 1421 1422 fn visit_i32_rotl(&mut self) -> Self::Output { 1423 use ShiftKind::*; 1424 1425 self.context.i32_shift(self.masm, Rotl) 1426 } 1427 1428 fn visit_i64_rotl(&mut self) -> Self::Output { 1429 use ShiftKind::*; 1430 1431 self.context.i64_shift(self.masm, Rotl) 1432 } 1433 1434 fn visit_i32_rotr(&mut self) -> Self::Output { 1435 use ShiftKind::*; 1436 1437 self.context.i32_shift(self.masm, Rotr) 1438 } 1439 1440 fn visit_i64_rotr(&mut self) -> Self::Output { 1441 use ShiftKind::*; 1442 1443 self.context.i64_shift(self.masm, Rotr) 1444 } 1445 1446 fn visit_end(&mut self) -> Self::Output { 1447 if !self.context.reachable { 1448 self.handle_unreachable_end() 1449 } else { 1450 let mut control = self.pop_control_frame()?; 1451 control.emit_end(self.masm, &mut self.context) 1452 } 1453 } 1454 1455 fn visit_i32_popcnt(&mut self) -> Self::Output { 1456 use OperandSize::*; 1457 self.masm.popcnt(&mut self.context, S32) 1458 } 1459 1460 fn visit_i64_popcnt(&mut self) -> Self::Output { 1461 use OperandSize::*; 1462 1463 self.masm.popcnt(&mut self.context, S64) 1464 } 1465 1466 fn visit_i32_wrap_i64(&mut self) -> Self::Output { 1467 self.context.unop(self.masm, |masm, reg| { 1468 masm.wrap(writable!(reg), reg)?; 1469 Ok(TypedReg::i32(reg)) 1470 }) 1471 } 1472 1473 fn visit_i64_extend_i32_s(&mut self) -> Self::Output { 1474 self.context.unop(self.masm, |masm, reg| { 1475 masm.extend(writable!(reg), reg, Extend::<Signed>::I64Extend32.into())?; 1476 Ok(TypedReg::i64(reg)) 1477 }) 1478 } 1479 1480 fn visit_i64_extend_i32_u(&mut self) -> Self::Output { 1481 self.context.unop(self.masm, |masm, reg| { 1482 masm.extend(writable!(reg), reg, Extend::<Zero>::I64Extend32.into())?; 1483 Ok(TypedReg::i64(reg)) 1484 }) 1485 } 1486 1487 fn visit_i32_extend8_s(&mut self) -> Self::Output { 1488 self.context.unop(self.masm, |masm, reg| { 1489 masm.extend(writable!(reg), reg, Extend::<Signed>::I32Extend8.into())?; 1490 Ok(TypedReg::i32(reg)) 1491 }) 1492 } 1493 1494 fn visit_i32_extend16_s(&mut self) -> Self::Output { 1495 self.context.unop(self.masm, |masm, reg| { 1496 masm.extend(writable!(reg), reg, Extend::<Signed>::I32Extend16.into())?; 1497 Ok(TypedReg::i32(reg)) 1498 }) 1499 } 1500 1501 fn visit_i64_extend8_s(&mut self) -> Self::Output { 1502 self.context.unop(self.masm, |masm, reg| { 1503 masm.extend(writable!(reg), reg, Extend::<Signed>::I64Extend8.into())?; 1504 Ok(TypedReg::i64(reg)) 1505 }) 1506 } 1507 1508 fn visit_i64_extend16_s(&mut self) -> Self::Output { 1509 self.context.unop(self.masm, |masm, reg| { 1510 masm.extend(writable!(reg), reg, Extend::<Signed>::I64Extend16.into())?; 1511 Ok(TypedReg::i64(reg)) 1512 }) 1513 } 1514 1515 fn visit_i64_extend32_s(&mut self) -> Self::Output { 1516 self.context.unop(self.masm, |masm, reg| { 1517 masm.extend(writable!(reg), reg, Extend::<Signed>::I64Extend32.into())?; 1518 Ok(TypedReg::i64(reg)) 1519 }) 1520 } 1521 1522 fn visit_i32_trunc_f32_s(&mut self) -> Self::Output { 1523 use OperandSize::*; 1524 1525 self.context 1526 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1527 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Unchecked) 1528 }) 1529 } 1530 1531 fn visit_i32_trunc_f32_u(&mut self) -> Self::Output { 1532 use OperandSize::*; 1533 1534 self.masm 1535 .unsigned_truncate(&mut self.context, S32, S32, TruncKind::Unchecked) 1536 } 1537 1538 fn visit_i32_trunc_f64_s(&mut self) -> Self::Output { 1539 use OperandSize::*; 1540 1541 self.context 1542 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1543 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Unchecked) 1544 }) 1545 } 1546 1547 fn visit_i32_trunc_f64_u(&mut self) -> Self::Output { 1548 use OperandSize::*; 1549 self.masm 1550 .unsigned_truncate(&mut self.context, S64, S32, TruncKind::Unchecked) 1551 } 1552 1553 fn visit_i64_trunc_f32_s(&mut self) -> Self::Output { 1554 use OperandSize::*; 1555 1556 self.context 1557 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1558 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Unchecked) 1559 }) 1560 } 1561 1562 fn visit_i64_trunc_f32_u(&mut self) -> Self::Output { 1563 use OperandSize::*; 1564 1565 self.masm 1566 .unsigned_truncate(&mut self.context, S32, S64, TruncKind::Unchecked) 1567 } 1568 1569 fn visit_i64_trunc_f64_s(&mut self) -> Self::Output { 1570 use OperandSize::*; 1571 1572 self.context 1573 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1574 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Unchecked) 1575 }) 1576 } 1577 1578 fn visit_i64_trunc_f64_u(&mut self) -> Self::Output { 1579 use OperandSize::*; 1580 1581 self.masm 1582 .unsigned_truncate(&mut self.context, S64, S64, TruncKind::Unchecked) 1583 } 1584 1585 fn visit_i32_reinterpret_f32(&mut self) -> Self::Output { 1586 self.context 1587 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, size| { 1588 masm.reinterpret_float_as_int(writable!(dst), src.into(), size) 1589 }) 1590 } 1591 1592 fn visit_i64_reinterpret_f64(&mut self) -> Self::Output { 1593 self.context 1594 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, size| { 1595 masm.reinterpret_float_as_int(writable!(dst), src.into(), size) 1596 }) 1597 } 1598 1599 fn visit_local_get(&mut self, index: u32) -> Self::Output { 1600 use WasmValType::*; 1601 let context = &mut self.context; 1602 let slot = context.frame.get_wasm_local(index); 1603 match slot.ty { 1604 I32 | I64 | F32 | F64 | V128 => context.stack.push(Val::local(index, slot.ty)), 1605 Ref(rt) => match rt.heap_type { 1606 WasmHeapType::Func => context.stack.push(Val::local(index, slot.ty)), 1607 _ => bail!(CodeGenError::unsupported_wasm_type()), 1608 }, 1609 } 1610 1611 Ok(()) 1612 } 1613 1614 fn visit_local_set(&mut self, index: u32) -> Self::Output { 1615 let src = self.emit_set_local(index)?; 1616 self.context.free_reg(src); 1617 Ok(()) 1618 } 1619 1620 fn visit_call(&mut self, index: u32) -> Self::Output { 1621 let callee = self.env.callee_from_index(FuncIndex::from_u32(index)); 1622 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee)?; 1623 Ok(()) 1624 } 1625 1626 fn visit_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output { 1627 // Spill now because `emit_lazy_init_funcref` and the `FnCall::emit` 1628 // invocations will both trigger spills since they both call functions. 1629 // However, the machine instructions for the spill emitted by 1630 // `emit_lazy_funcref` will be jumped over if the funcref was previously 1631 // initialized which may result in the machine stack becoming 1632 // unbalanced. 1633 self.context.spill(self.masm)?; 1634 1635 let type_index = TypeIndex::from_u32(type_index); 1636 let table_index = TableIndex::from_u32(table_index); 1637 1638 self.emit_lazy_init_funcref(table_index)?; 1639 1640 // Perform the indirect call. 1641 // This code assumes that [`Self::emit_lazy_init_funcref`] will 1642 // push the funcref to the value stack. 1643 let funcref_ptr = self 1644 .context 1645 .stack 1646 .peek() 1647 .map(|v| v.unwrap_reg()) 1648 .ok_or_else(|| CodeGenError::missing_values_in_stack())?; 1649 self.masm 1650 .trapz(funcref_ptr.into(), TRAP_INDIRECT_CALL_TO_NULL)?; 1651 self.emit_typecheck_funcref(funcref_ptr.into(), type_index)?; 1652 1653 let callee = self.env.funcref(type_index); 1654 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee)?; 1655 Ok(()) 1656 } 1657 1658 fn visit_table_init(&mut self, elem: u32, table: u32) -> Self::Output { 1659 let at = self.context.stack.ensure_index_at(3)?; 1660 1661 self.context 1662 .stack 1663 .insert_many(at, &[table.try_into()?, elem.try_into()?]); 1664 1665 let builtin = self.env.builtins.table_init::<M::ABI, M::Ptr>()?; 1666 FnCall::emit::<M>( 1667 &mut self.env, 1668 self.masm, 1669 &mut self.context, 1670 Callee::Builtin(builtin.clone()), 1671 )?; 1672 self.context.pop_and_free(self.masm) 1673 } 1674 1675 fn visit_table_copy(&mut self, dst: u32, src: u32) -> Self::Output { 1676 let at = self.context.stack.ensure_index_at(3)?; 1677 self.context 1678 .stack 1679 .insert_many(at, &[dst.try_into()?, src.try_into()?]); 1680 1681 let builtin = self.env.builtins.table_copy::<M::ABI, M::Ptr>()?; 1682 FnCall::emit::<M>( 1683 &mut self.env, 1684 self.masm, 1685 &mut self.context, 1686 Callee::Builtin(builtin), 1687 )?; 1688 self.context.pop_and_free(self.masm) 1689 } 1690 1691 fn visit_table_get(&mut self, table: u32) -> Self::Output { 1692 let table_index = TableIndex::from_u32(table); 1693 let table = self.env.table(table_index); 1694 let heap_type = table.ref_type.heap_type; 1695 1696 match heap_type { 1697 WasmHeapType::Func => self.emit_lazy_init_funcref(table_index), 1698 _ => Err(anyhow!(CodeGenError::unsupported_wasm_type())), 1699 } 1700 } 1701 1702 fn visit_table_grow(&mut self, table: u32) -> Self::Output { 1703 let table_index = TableIndex::from_u32(table); 1704 let table_ty = self.env.table(table_index); 1705 let builtin = match table_ty.ref_type.heap_type { 1706 WasmHeapType::Func => self.env.builtins.table_grow_func_ref::<M::ABI, M::Ptr>()?, 1707 _ => bail!(CodeGenError::unsupported_wasm_type()), 1708 }; 1709 1710 let len = self.context.stack.len(); 1711 // table.grow` requires at least 2 elements on the value stack. 1712 let at = self.context.stack.ensure_index_at(2)?; 1713 1714 // The table_grow builtin expects the parameters in a different 1715 // order. 1716 // The value stack at this point should contain: 1717 // [ init_value | delta ] (stack top) 1718 // but the builtin function expects the init value as the last 1719 // argument. 1720 self.context.stack.inner_mut().swap(len - 1, len - 2); 1721 self.context.stack.insert_many(at, &[table.try_into()?]); 1722 1723 FnCall::emit::<M>( 1724 &mut self.env, 1725 self.masm, 1726 &mut self.context, 1727 Callee::Builtin(builtin.clone()), 1728 )?; 1729 1730 Ok(()) 1731 } 1732 1733 fn visit_table_size(&mut self, table: u32) -> Self::Output { 1734 let table_index = TableIndex::from_u32(table); 1735 let table_data = self.env.resolve_table_data(table_index); 1736 self.emit_compute_table_size(&table_data) 1737 } 1738 1739 fn visit_table_fill(&mut self, table: u32) -> Self::Output { 1740 let table_index = TableIndex::from_u32(table); 1741 let table_ty = self.env.table(table_index); 1742 1743 ensure!( 1744 table_ty.ref_type.heap_type == WasmHeapType::Func, 1745 CodeGenError::unsupported_wasm_type() 1746 ); 1747 1748 let builtin = self.env.builtins.table_fill_func_ref::<M::ABI, M::Ptr>()?; 1749 1750 let at = self.context.stack.ensure_index_at(3)?; 1751 1752 self.context.stack.insert_many(at, &[table.try_into()?]); 1753 FnCall::emit::<M>( 1754 &mut self.env, 1755 self.masm, 1756 &mut self.context, 1757 Callee::Builtin(builtin.clone()), 1758 )?; 1759 self.context.pop_and_free(self.masm) 1760 } 1761 1762 fn visit_table_set(&mut self, table: u32) -> Self::Output { 1763 let ptr_type = self.env.ptr_type(); 1764 let table_index = TableIndex::from_u32(table); 1765 let table_data = self.env.resolve_table_data(table_index); 1766 let table = self.env.table(table_index); 1767 match table.ref_type.heap_type { 1768 WasmHeapType::Func => { 1769 ensure!( 1770 self.tunables.table_lazy_init, 1771 CodeGenError::unsupported_table_eager_init() 1772 ); 1773 let value = self.context.pop_to_reg(self.masm, None)?; 1774 let index = self.context.pop_to_reg(self.masm, None)?; 1775 let base = self.context.any_gpr(self.masm)?; 1776 let elem_addr = 1777 self.emit_compute_table_elem_addr(index.into(), base, &table_data)?; 1778 // Set the initialized bit. 1779 self.masm.or( 1780 writable!(value.into()), 1781 value.into(), 1782 RegImm::i64(FUNCREF_INIT_BIT as i64), 1783 ptr_type.try_into()?, 1784 )?; 1785 1786 self.masm.store_ptr(value.into(), elem_addr)?; 1787 1788 self.context.free_reg(value); 1789 self.context.free_reg(index); 1790 self.context.free_reg(base); 1791 Ok(()) 1792 } 1793 _ => Err(anyhow!(CodeGenError::unsupported_wasm_type())), 1794 } 1795 } 1796 1797 fn visit_elem_drop(&mut self, index: u32) -> Self::Output { 1798 let elem_drop = self.env.builtins.elem_drop::<M::ABI, M::Ptr>()?; 1799 self.context.stack.extend([index.try_into()?]); 1800 FnCall::emit::<M>( 1801 &mut self.env, 1802 self.masm, 1803 &mut self.context, 1804 Callee::Builtin(elem_drop), 1805 )?; 1806 Ok(()) 1807 } 1808 1809 fn visit_memory_init(&mut self, data_index: u32, mem: u32) -> Self::Output { 1810 let at = self.context.stack.ensure_index_at(3)?; 1811 self.context 1812 .stack 1813 .insert_many(at, &[mem.try_into()?, data_index.try_into()?]); 1814 let builtin = self.env.builtins.memory_init::<M::ABI, M::Ptr>()?; 1815 FnCall::emit::<M>( 1816 &mut self.env, 1817 self.masm, 1818 &mut self.context, 1819 Callee::Builtin(builtin), 1820 )?; 1821 self.context.pop_and_free(self.masm) 1822 } 1823 1824 fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) -> Self::Output { 1825 // At this point, the stack is expected to contain: 1826 // [ dst_offset, src_offset, len ] 1827 // The following code inserts the missing params, so that stack contains: 1828 // [ vmctx, dst_mem, dst_offset, src_mem, src_offset, len ] 1829 // Which is the order expected by the builtin function. 1830 let _ = self.context.stack.ensure_index_at(3)?; 1831 let at = self.context.stack.ensure_index_at(2)?; 1832 self.context.stack.insert_many(at, &[src_mem.try_into()?]); 1833 1834 // One element was inserted above, so instead of 3, we use 4. 1835 let at = self.context.stack.ensure_index_at(4)?; 1836 self.context.stack.insert_many(at, &[dst_mem.try_into()?]); 1837 1838 let builtin = self.env.builtins.memory_copy::<M::ABI, M::Ptr>()?; 1839 1840 FnCall::emit::<M>( 1841 &mut self.env, 1842 self.masm, 1843 &mut self.context, 1844 Callee::Builtin(builtin), 1845 )?; 1846 self.context.pop_and_free(self.masm) 1847 } 1848 1849 fn visit_memory_fill(&mut self, mem: u32) -> Self::Output { 1850 let at = self.context.stack.ensure_index_at(3)?; 1851 1852 self.context.stack.insert_many(at, &[mem.try_into()?]); 1853 1854 let builtin = self.env.builtins.memory_fill::<M::ABI, M::Ptr>()?; 1855 FnCall::emit::<M>( 1856 &mut self.env, 1857 self.masm, 1858 &mut self.context, 1859 Callee::Builtin(builtin), 1860 )?; 1861 self.context.pop_and_free(self.masm) 1862 } 1863 1864 fn visit_memory_size(&mut self, mem: u32) -> Self::Output { 1865 let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem)); 1866 self.emit_compute_memory_size(&heap) 1867 } 1868 1869 fn visit_memory_grow(&mut self, mem: u32) -> Self::Output { 1870 let _ = self.context.stack.ensure_index_at(1)?; 1871 // The stack at this point contains: [ delta ] 1872 // The desired state is 1873 // [ vmctx, delta, index ] 1874 self.context.stack.extend([mem.try_into()?]); 1875 1876 let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem)); 1877 let builtin = self.env.builtins.memory32_grow::<M::ABI, M::Ptr>()?; 1878 FnCall::emit::<M>( 1879 &mut self.env, 1880 self.masm, 1881 &mut self.context, 1882 Callee::Builtin(builtin), 1883 )?; 1884 1885 // The memory32_grow builtin returns a pointer type, therefore we must 1886 // ensure that the return type is representative of the address space of 1887 // the heap type. 1888 match (self.env.ptr_type(), heap.index_type()) { 1889 (WasmValType::I64, WasmValType::I64) => Ok(()), 1890 // When the heap type is smaller than the pointer type, we adjust 1891 // the result of the memory32_grow builtin. 1892 (WasmValType::I64, WasmValType::I32) => { 1893 let top: Reg = self.context.pop_to_reg(self.masm, None)?.into(); 1894 self.masm.wrap(writable!(top.into()), top.into())?; 1895 self.context.stack.push(TypedReg::i32(top).into()); 1896 Ok(()) 1897 } 1898 _ => Err(anyhow!(CodeGenError::unsupported_32_bit_platform())), 1899 } 1900 } 1901 1902 fn visit_data_drop(&mut self, data_index: u32) -> Self::Output { 1903 self.context.stack.extend([data_index.try_into()?]); 1904 1905 let builtin = self.env.builtins.data_drop::<M::ABI, M::Ptr>()?; 1906 FnCall::emit::<M>( 1907 &mut self.env, 1908 self.masm, 1909 &mut self.context, 1910 Callee::Builtin(builtin), 1911 ) 1912 } 1913 1914 fn visit_nop(&mut self) -> Self::Output { 1915 Ok(()) 1916 } 1917 1918 fn visit_if(&mut self, blockty: BlockType) -> Self::Output { 1919 self.control_frames.push(ControlStackFrame::r#if( 1920 self.env.resolve_block_sig(blockty)?, 1921 self.masm, 1922 &mut self.context, 1923 )?); 1924 1925 Ok(()) 1926 } 1927 1928 fn visit_else(&mut self) -> Self::Output { 1929 if !self.context.reachable { 1930 self.handle_unreachable_else() 1931 } else { 1932 let control = self 1933 .control_frames 1934 .last_mut() 1935 .ok_or_else(|| CodeGenError::control_frame_expected())?; 1936 control.emit_else(self.masm, &mut self.context) 1937 } 1938 } 1939 1940 fn visit_block(&mut self, blockty: BlockType) -> Self::Output { 1941 self.control_frames.push(ControlStackFrame::block( 1942 self.env.resolve_block_sig(blockty)?, 1943 self.masm, 1944 &mut self.context, 1945 )?); 1946 1947 Ok(()) 1948 } 1949 1950 fn visit_loop(&mut self, blockty: BlockType) -> Self::Output { 1951 self.control_frames.push(ControlStackFrame::r#loop( 1952 self.env.resolve_block_sig(blockty)?, 1953 self.masm, 1954 &mut self.context, 1955 )?); 1956 1957 self.maybe_emit_epoch_check()?; 1958 self.maybe_emit_fuel_check() 1959 } 1960 1961 fn visit_br(&mut self, depth: u32) -> Self::Output { 1962 let index = control_index(depth, self.control_frames.len())?; 1963 let frame = &mut self.control_frames[index]; 1964 self.context 1965 .br::<_, _, UnconditionalBranch>(frame, self.masm, |masm, cx, frame| { 1966 frame.pop_abi_results::<M, _>(cx, masm, |results, _, _| { 1967 Ok(results.ret_area().copied()) 1968 }) 1969 }) 1970 } 1971 1972 fn visit_br_if(&mut self, depth: u32) -> Self::Output { 1973 let index = control_index(depth, self.control_frames.len())?; 1974 let frame = &mut self.control_frames[index]; 1975 frame.set_as_target(); 1976 1977 let top = { 1978 let top = self.context.without::<Result<TypedReg>, M, _>( 1979 frame.results::<M>()?.regs(), 1980 self.masm, 1981 |ctx, masm| ctx.pop_to_reg(masm, None), 1982 )??; 1983 // Explicitly save any live registers and locals before setting up 1984 // the branch state. 1985 // In some cases, calculating the `top` value above, will result in 1986 // a spill, thus the following one will result in a no-op. 1987 self.context.spill(self.masm)?; 1988 frame.top_abi_results::<M, _>( 1989 &mut self.context, 1990 self.masm, 1991 |results, context, masm| { 1992 // In the case of `br_if` there's a possibility that we'll 1993 // exit early from the block or fallthrough, for 1994 // a fallthrough, we cannot rely on the pre-computed return area; 1995 // it must be recalculated so that any values that are 1996 // generated are correctly placed near the current stack 1997 // pointer. 1998 if results.on_stack() { 1999 let stack_consumed = context.stack.sizeof(results.stack_operands_len()); 2000 let base = masm.sp_offset()?.as_u32() - stack_consumed; 2001 let offs = base + results.size(); 2002 Ok(Some(RetArea::sp(SPOffset::from_u32(offs)))) 2003 } else { 2004 Ok(None) 2005 } 2006 }, 2007 )?; 2008 top 2009 }; 2010 2011 // Emit instructions to balance the machine stack. 2012 let current_sp_offset = self.masm.sp_offset()?; 2013 let unbalanced = frame.unbalanced(self.masm)?; 2014 let (label, cmp) = if unbalanced { 2015 (self.masm.get_label()?, IntCmpKind::Eq) 2016 } else { 2017 (*frame.label(), IntCmpKind::Ne) 2018 }; 2019 2020 self.masm 2021 .branch(cmp, top.reg.into(), top.reg.into(), label, OperandSize::S32)?; 2022 self.context.free_reg(top); 2023 2024 if unbalanced { 2025 self.context 2026 .br::<_, _, ConditionalBranch>(frame, self.masm, |_, _, _| Ok(()))?; 2027 2028 // Restore sp_offset to what it was for falling through and emit 2029 // fallthrough label. 2030 self.masm.reset_stack_pointer(current_sp_offset)?; 2031 self.masm.bind(label)?; 2032 } 2033 2034 Ok(()) 2035 } 2036 2037 fn visit_br_table(&mut self, targets: BrTable<'a>) -> Self::Output { 2038 // +1 to account for the default target. 2039 let len = targets.len() + 1; 2040 // SmallVec<[_; 5]> to match the binary emission layer (e.g 2041 // see `JmpTableSeq'), but here we use 5 instead since we 2042 // bundle the default target as the last element in the array. 2043 let mut labels: SmallVec<[_; 5]> = smallvec![]; 2044 for _ in 0..len { 2045 labels.push(self.masm.get_label()?); 2046 } 2047 2048 // Find the innermost target and use it as the relative frame 2049 // for result handling below. 2050 // 2051 // This approch ensures that 2052 // 1. The stack pointer offset is correctly positioned 2053 // according to the expectations of the innermost block end 2054 // sequence. 2055 // 2. We meet the jump site invariants introduced by 2056 // `CodegenContext::br`, which take advantage of Wasm 2057 // semantics given that all jumps are "outward". 2058 let mut innermost = targets.default(); 2059 for target in targets.targets() { 2060 let target = target?; 2061 if target < innermost { 2062 innermost = target; 2063 } 2064 } 2065 2066 let innermost_index = control_index(innermost, self.control_frames.len())?; 2067 let innermost_frame = &mut self.control_frames[innermost_index]; 2068 let innermost_result = innermost_frame.results::<M>()?; 2069 2070 let (index, tmp) = { 2071 let index_and_tmp = self.context.without::<Result<(TypedReg, _)>, M, _>( 2072 innermost_result.regs(), 2073 self.masm, 2074 |cx, masm| Ok((cx.pop_to_reg(masm, None)?, cx.any_gpr(masm)?)), 2075 )??; 2076 2077 // Materialize any constants or locals into their result 2078 // representation, so that when reachability is restored, 2079 // they are correctly located. NB: the results are popped 2080 // in function of the innermost branch specified for 2081 // `br_table`, which implies that the machine stack will 2082 // be correctly balanced, by virtue of calling 2083 // `pop_abi_results`. 2084 2085 // It's possible that we need to balance the stack for the 2086 // rest of the targets, which will be done before emitting 2087 // the unconditional jump below. 2088 innermost_frame.pop_abi_results::<M, _>( 2089 &mut self.context, 2090 self.masm, 2091 |results, _, _| Ok(results.ret_area().copied()), 2092 )?; 2093 index_and_tmp 2094 }; 2095 2096 self.masm.jmp_table(&labels, index.into(), tmp)?; 2097 // Save the original stack pointer offset; we will reset the stack 2098 // pointer to this offset after jumping to each of the targets. Each 2099 // jump might adjust the stack according to the base offset of the 2100 // target. 2101 let current_sp = self.masm.sp_offset()?; 2102 2103 for (t, l) in targets 2104 .targets() 2105 .into_iter() 2106 .chain(std::iter::once(Ok(targets.default()))) 2107 .zip(labels.iter()) 2108 { 2109 let control_index = control_index(t?, self.control_frames.len())?; 2110 let frame = &mut self.control_frames[control_index]; 2111 // Reset the stack pointer to its original offset. This is needed 2112 // because each jump will potentially adjust the stack pointer 2113 // according to the base offset of the target. 2114 self.masm.reset_stack_pointer(current_sp)?; 2115 2116 // NB: We don't perform any result handling as it was 2117 // already taken care of above before jumping to the 2118 // jump table. 2119 self.masm.bind(*l)?; 2120 // Ensure that the stack pointer is correctly positioned before 2121 // jumping to the jump table code. 2122 self.context 2123 .br::<_, _, UnconditionalBranch>(frame, self.masm, |_, _, _| Ok(()))?; 2124 } 2125 // Finally reset the stack pointer to the original location. 2126 // The reachability analysis, will ensure it's correctly located 2127 // once reachability is restored. 2128 self.masm.reset_stack_pointer(current_sp)?; 2129 self.context.reachable = false; 2130 self.context.free_reg(index.reg); 2131 self.context.free_reg(tmp); 2132 2133 Ok(()) 2134 } 2135 2136 fn visit_return(&mut self) -> Self::Output { 2137 // Grab the outermost frame, which is the function's body 2138 // frame. We don't rely on [`codegen::control_index`] since 2139 // this frame is implicit and we know that it should exist at 2140 // index 0. 2141 let outermost = &mut self.control_frames[0]; 2142 self.context 2143 .br::<_, _, UnconditionalBranch>(outermost, self.masm, |masm, cx, frame| { 2144 frame.pop_abi_results::<M, _>(cx, masm, |results, _, _| { 2145 Ok(results.ret_area().copied()) 2146 }) 2147 }) 2148 } 2149 2150 fn visit_unreachable(&mut self) -> Self::Output { 2151 self.masm.unreachable()?; 2152 self.context.reachable = false; 2153 // Set the implicit outermost frame as target to perform the necessary 2154 // stack clean up. 2155 let outermost = &mut self.control_frames[0]; 2156 outermost.set_as_target(); 2157 2158 Ok(()) 2159 } 2160 2161 fn visit_local_tee(&mut self, index: u32) -> Self::Output { 2162 let typed_reg = self.emit_set_local(index)?; 2163 self.context.stack.push(typed_reg.into()); 2164 2165 Ok(()) 2166 } 2167 2168 fn visit_global_get(&mut self, global_index: u32) -> Self::Output { 2169 let index = GlobalIndex::from_u32(global_index); 2170 let (ty, base, offset) = self.emit_get_global_addr(index)?; 2171 let addr = self.masm.address_at_reg(base, offset)?; 2172 let dst = self.context.reg_for_type(ty, self.masm)?; 2173 self.masm.load(addr, writable!(dst), ty.try_into()?)?; 2174 self.context.stack.push(Val::reg(dst, ty)); 2175 2176 self.context.free_reg(base); 2177 2178 Ok(()) 2179 } 2180 2181 fn visit_global_set(&mut self, global_index: u32) -> Self::Output { 2182 let index = GlobalIndex::from_u32(global_index); 2183 let (ty, base, offset) = self.emit_get_global_addr(index)?; 2184 let addr = self.masm.address_at_reg(base, offset)?; 2185 2186 let typed_reg = self.context.pop_to_reg(self.masm, None)?; 2187 self.masm 2188 .store(typed_reg.reg.into(), addr, ty.try_into()?)?; 2189 self.context.free_reg(typed_reg.reg); 2190 self.context.free_reg(base); 2191 2192 Ok(()) 2193 } 2194 2195 fn visit_drop(&mut self) -> Self::Output { 2196 self.context.drop_last(1, |regalloc, val| match val { 2197 Val::Reg(tr) => Ok(regalloc.free(tr.reg.into())), 2198 Val::Memory(m) => self.masm.free_stack(m.slot.size), 2199 _ => Ok(()), 2200 }) 2201 } 2202 2203 fn visit_select(&mut self) -> Self::Output { 2204 let cond = self.context.pop_to_reg(self.masm, None)?; 2205 let val2 = self.context.pop_to_reg(self.masm, None)?; 2206 let val1 = self.context.pop_to_reg(self.masm, None)?; 2207 self.masm 2208 .cmp(cond.reg.into(), RegImm::i32(0), OperandSize::S32)?; 2209 // Conditionally move val1 to val2 if the comparison is 2210 // not zero. 2211 self.masm.cmov( 2212 writable!(val2.into()), 2213 val1.into(), 2214 IntCmpKind::Ne, 2215 val1.ty.try_into()?, 2216 )?; 2217 self.context.stack.push(val2.into()); 2218 self.context.free_reg(val1.reg); 2219 self.context.free_reg(cond); 2220 2221 Ok(()) 2222 } 2223 2224 fn visit_i32_load(&mut self, memarg: MemArg) -> Self::Output { 2225 self.emit_wasm_load( 2226 &memarg, 2227 WasmValType::I32, 2228 LoadKind::Operand(OperandSize::S32), 2229 ) 2230 } 2231 2232 fn visit_i32_load8_s(&mut self, memarg: MemArg) -> Self::Output { 2233 self.emit_wasm_load( 2234 &memarg, 2235 WasmValType::I32, 2236 LoadKind::ScalarExtend(Extend::<Signed>::I32Extend8.into()), 2237 ) 2238 } 2239 2240 fn visit_i32_load8_u(&mut self, memarg: MemArg) -> Self::Output { 2241 self.emit_wasm_load( 2242 &memarg, 2243 WasmValType::I32, 2244 LoadKind::ScalarExtend(Extend::<Zero>::I32Extend8.into()), 2245 ) 2246 } 2247 2248 fn visit_i32_load16_s(&mut self, memarg: MemArg) -> Self::Output { 2249 self.emit_wasm_load( 2250 &memarg, 2251 WasmValType::I32, 2252 LoadKind::ScalarExtend(Extend::<Signed>::I32Extend16.into()), 2253 ) 2254 } 2255 2256 fn visit_i32_load16_u(&mut self, memarg: MemArg) -> Self::Output { 2257 self.emit_wasm_load( 2258 &memarg, 2259 WasmValType::I32, 2260 LoadKind::ScalarExtend(Extend::<Zero>::I32Extend16.into()), 2261 ) 2262 } 2263 2264 fn visit_i32_store(&mut self, memarg: MemArg) -> Self::Output { 2265 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S32)) 2266 } 2267 2268 fn visit_i32_store8(&mut self, memarg: MemArg) -> Self::Output { 2269 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S8)) 2270 } 2271 2272 fn visit_i32_store16(&mut self, memarg: MemArg) -> Self::Output { 2273 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S16)) 2274 } 2275 2276 fn visit_i64_load8_s(&mut self, memarg: MemArg) -> Self::Output { 2277 self.emit_wasm_load( 2278 &memarg, 2279 WasmValType::I64, 2280 LoadKind::ScalarExtend(Extend::<Signed>::I64Extend8.into()), 2281 ) 2282 } 2283 2284 fn visit_i64_load8_u(&mut self, memarg: MemArg) -> Self::Output { 2285 self.emit_wasm_load( 2286 &memarg, 2287 WasmValType::I64, 2288 LoadKind::ScalarExtend(Extend::<Zero>::I64Extend8.into()), 2289 ) 2290 } 2291 2292 fn visit_i64_load16_u(&mut self, memarg: MemArg) -> Self::Output { 2293 self.emit_wasm_load( 2294 &memarg, 2295 WasmValType::I64, 2296 LoadKind::ScalarExtend(Extend::<Zero>::I64Extend16.into()), 2297 ) 2298 } 2299 2300 fn visit_i64_load16_s(&mut self, memarg: MemArg) -> Self::Output { 2301 self.emit_wasm_load( 2302 &memarg, 2303 WasmValType::I64, 2304 LoadKind::ScalarExtend(Extend::<Signed>::I64Extend16.into()), 2305 ) 2306 } 2307 2308 fn visit_i64_load32_u(&mut self, memarg: MemArg) -> Self::Output { 2309 self.emit_wasm_load( 2310 &memarg, 2311 WasmValType::I64, 2312 LoadKind::ScalarExtend(Extend::<Zero>::I64Extend32.into()), 2313 ) 2314 } 2315 2316 fn visit_i64_load32_s(&mut self, memarg: MemArg) -> Self::Output { 2317 self.emit_wasm_load( 2318 &memarg, 2319 WasmValType::I64, 2320 LoadKind::ScalarExtend(Extend::<Signed>::I64Extend32.into()), 2321 ) 2322 } 2323 2324 fn visit_i64_load(&mut self, memarg: MemArg) -> Self::Output { 2325 self.emit_wasm_load( 2326 &memarg, 2327 WasmValType::I64, 2328 LoadKind::Operand(OperandSize::S64), 2329 ) 2330 } 2331 2332 fn visit_i64_store(&mut self, memarg: MemArg) -> Self::Output { 2333 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S64)) 2334 } 2335 2336 fn visit_i64_store8(&mut self, memarg: MemArg) -> Self::Output { 2337 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S8)) 2338 } 2339 2340 fn visit_i64_store16(&mut self, memarg: MemArg) -> Self::Output { 2341 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S16)) 2342 } 2343 2344 fn visit_i64_store32(&mut self, memarg: MemArg) -> Self::Output { 2345 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S32)) 2346 } 2347 2348 fn visit_f32_load(&mut self, memarg: MemArg) -> Self::Output { 2349 self.emit_wasm_load( 2350 &memarg, 2351 WasmValType::F32, 2352 LoadKind::Operand(OperandSize::S32), 2353 ) 2354 } 2355 2356 fn visit_f32_store(&mut self, memarg: MemArg) -> Self::Output { 2357 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S32)) 2358 } 2359 2360 fn visit_f64_load(&mut self, memarg: MemArg) -> Self::Output { 2361 self.emit_wasm_load( 2362 &memarg, 2363 WasmValType::F64, 2364 LoadKind::Operand(OperandSize::S64), 2365 ) 2366 } 2367 2368 fn visit_f64_store(&mut self, memarg: MemArg) -> Self::Output { 2369 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S64)) 2370 } 2371 2372 fn visit_i32_trunc_sat_f32_s(&mut self) -> Self::Output { 2373 use OperandSize::*; 2374 2375 self.context 2376 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2377 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Checked) 2378 }) 2379 } 2380 2381 fn visit_i32_trunc_sat_f32_u(&mut self) -> Self::Output { 2382 use OperandSize::*; 2383 2384 self.masm 2385 .unsigned_truncate(&mut self.context, S32, S32, TruncKind::Checked) 2386 } 2387 2388 fn visit_i32_trunc_sat_f64_s(&mut self) -> Self::Output { 2389 use OperandSize::*; 2390 2391 self.context 2392 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2393 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Checked) 2394 }) 2395 } 2396 2397 fn visit_i32_trunc_sat_f64_u(&mut self) -> Self::Output { 2398 use OperandSize::*; 2399 2400 self.masm 2401 .unsigned_truncate(&mut self.context, S64, S32, TruncKind::Checked) 2402 } 2403 2404 fn visit_i64_trunc_sat_f32_s(&mut self) -> Self::Output { 2405 use OperandSize::*; 2406 2407 self.context 2408 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2409 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Checked) 2410 }) 2411 } 2412 2413 fn visit_i64_trunc_sat_f32_u(&mut self) -> Self::Output { 2414 use OperandSize::*; 2415 2416 self.masm 2417 .unsigned_truncate(&mut self.context, S32, S64, TruncKind::Checked) 2418 } 2419 2420 fn visit_i64_trunc_sat_f64_s(&mut self) -> Self::Output { 2421 use OperandSize::*; 2422 2423 self.context 2424 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2425 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Checked) 2426 }) 2427 } 2428 2429 fn visit_i64_trunc_sat_f64_u(&mut self) -> Self::Output { 2430 use OperandSize::*; 2431 2432 self.masm 2433 .unsigned_truncate(&mut self.context, S64, S64, TruncKind::Checked) 2434 } 2435 2436 fn visit_i64_add128(&mut self) -> Self::Output { 2437 self.context 2438 .binop128(self.masm, |masm, lhs_lo, lhs_hi, rhs_lo, rhs_hi| { 2439 masm.add128( 2440 writable!(lhs_lo), 2441 writable!(lhs_hi), 2442 lhs_lo, 2443 lhs_hi, 2444 rhs_lo, 2445 rhs_hi, 2446 )?; 2447 Ok((TypedReg::i64(lhs_lo), TypedReg::i64(lhs_hi))) 2448 }) 2449 } 2450 2451 fn visit_i64_sub128(&mut self) -> Self::Output { 2452 self.context 2453 .binop128(self.masm, |masm, lhs_lo, lhs_hi, rhs_lo, rhs_hi| { 2454 masm.sub128( 2455 writable!(lhs_lo), 2456 writable!(lhs_hi), 2457 lhs_lo, 2458 lhs_hi, 2459 rhs_lo, 2460 rhs_hi, 2461 )?; 2462 Ok((TypedReg::i64(lhs_lo), TypedReg::i64(lhs_hi))) 2463 }) 2464 } 2465 2466 fn visit_i64_mul_wide_s(&mut self) -> Self::Output { 2467 self.masm.mul_wide(&mut self.context, MulWideKind::Signed) 2468 } 2469 2470 fn visit_i64_mul_wide_u(&mut self) -> Self::Output { 2471 self.masm.mul_wide(&mut self.context, MulWideKind::Unsigned) 2472 } 2473 2474 fn visit_i32_atomic_load8_u(&mut self, memarg: MemArg) -> Self::Output { 2475 self.emit_wasm_load( 2476 &memarg, 2477 WasmValType::I32, 2478 LoadKind::Atomic(OperandSize::S8, Some(Extend::<Zero>::I32Extend8.into())), 2479 ) 2480 } 2481 2482 fn visit_i32_atomic_load16_u(&mut self, memarg: MemArg) -> Self::Output { 2483 self.emit_wasm_load( 2484 &memarg, 2485 WasmValType::I32, 2486 LoadKind::Atomic(OperandSize::S16, Some(Extend::<Zero>::I32Extend16.into())), 2487 ) 2488 } 2489 2490 fn visit_i32_atomic_load(&mut self, memarg: MemArg) -> Self::Output { 2491 self.emit_wasm_load( 2492 &memarg, 2493 WasmValType::I32, 2494 LoadKind::Atomic(OperandSize::S32, None), 2495 ) 2496 } 2497 2498 fn visit_i64_atomic_load8_u(&mut self, memarg: MemArg) -> Self::Output { 2499 self.emit_wasm_load( 2500 &memarg, 2501 WasmValType::I64, 2502 LoadKind::Atomic(OperandSize::S8, Some(Extend::<Zero>::I64Extend8.into())), 2503 ) 2504 } 2505 2506 fn visit_i64_atomic_load16_u(&mut self, memarg: MemArg) -> Self::Output { 2507 self.emit_wasm_load( 2508 &memarg, 2509 WasmValType::I64, 2510 LoadKind::Atomic(OperandSize::S16, Some(Extend::<Zero>::I64Extend16.into())), 2511 ) 2512 } 2513 2514 fn visit_i64_atomic_load32_u(&mut self, memarg: MemArg) -> Self::Output { 2515 self.emit_wasm_load( 2516 &memarg, 2517 WasmValType::I64, 2518 LoadKind::Atomic(OperandSize::S32, Some(Extend::<Zero>::I64Extend32.into())), 2519 ) 2520 } 2521 2522 fn visit_i64_atomic_load(&mut self, memarg: MemArg) -> Self::Output { 2523 self.emit_wasm_load( 2524 &memarg, 2525 WasmValType::I64, 2526 LoadKind::Atomic(OperandSize::S64, None), 2527 ) 2528 } 2529 2530 fn visit_i32_atomic_store(&mut self, memarg: MemArg) -> Self::Output { 2531 self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S32)) 2532 } 2533 2534 fn visit_i64_atomic_store(&mut self, memarg: MemArg) -> Self::Output { 2535 self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S64)) 2536 } 2537 2538 fn visit_i32_atomic_store8(&mut self, memarg: MemArg) -> Self::Output { 2539 self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S8)) 2540 } 2541 2542 fn visit_i32_atomic_store16(&mut self, memarg: MemArg) -> Self::Output { 2543 self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S16)) 2544 } 2545 2546 fn visit_i64_atomic_store8(&mut self, memarg: MemArg) -> Self::Output { 2547 self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S8)) 2548 } 2549 2550 fn visit_i64_atomic_store16(&mut self, memarg: MemArg) -> Self::Output { 2551 self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S16)) 2552 } 2553 2554 fn visit_i64_atomic_store32(&mut self, memarg: MemArg) -> Self::Output { 2555 self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S32)) 2556 } 2557 2558 fn visit_i32_atomic_rmw8_add_u(&mut self, arg: MemArg) -> Self::Output { 2559 self.emit_atomic_rmw( 2560 &arg, 2561 RmwOp::Add, 2562 OperandSize::S8, 2563 Some(Extend::<Zero>::I32Extend8), 2564 ) 2565 } 2566 2567 fn visit_i32_atomic_rmw16_add_u(&mut self, arg: MemArg) -> Self::Output { 2568 self.emit_atomic_rmw( 2569 &arg, 2570 RmwOp::Add, 2571 OperandSize::S16, 2572 Some(Extend::<Zero>::I32Extend16), 2573 ) 2574 } 2575 2576 fn visit_i32_atomic_rmw_add(&mut self, arg: MemArg) -> Self::Output { 2577 self.emit_atomic_rmw(&arg, RmwOp::Add, OperandSize::S32, None) 2578 } 2579 2580 fn visit_i64_atomic_rmw8_add_u(&mut self, arg: MemArg) -> Self::Output { 2581 self.emit_atomic_rmw( 2582 &arg, 2583 RmwOp::Add, 2584 OperandSize::S8, 2585 Some(Extend::<Zero>::I64Extend8), 2586 ) 2587 } 2588 2589 fn visit_i64_atomic_rmw16_add_u(&mut self, arg: MemArg) -> Self::Output { 2590 self.emit_atomic_rmw( 2591 &arg, 2592 RmwOp::Add, 2593 OperandSize::S16, 2594 Some(Extend::<Zero>::I64Extend16), 2595 ) 2596 } 2597 2598 fn visit_i64_atomic_rmw32_add_u(&mut self, arg: MemArg) -> Self::Output { 2599 self.emit_atomic_rmw( 2600 &arg, 2601 RmwOp::Add, 2602 OperandSize::S32, 2603 Some(Extend::<Zero>::I64Extend32), 2604 ) 2605 } 2606 2607 fn visit_i64_atomic_rmw_add(&mut self, arg: MemArg) -> Self::Output { 2608 self.emit_atomic_rmw(&arg, RmwOp::Add, OperandSize::S64, None) 2609 } 2610 2611 fn visit_i32_atomic_rmw8_sub_u(&mut self, arg: MemArg) -> Self::Output { 2612 self.emit_atomic_rmw( 2613 &arg, 2614 RmwOp::Sub, 2615 OperandSize::S8, 2616 Some(Extend::<Zero>::I32Extend8), 2617 ) 2618 } 2619 fn visit_i32_atomic_rmw16_sub_u(&mut self, arg: MemArg) -> Self::Output { 2620 self.emit_atomic_rmw( 2621 &arg, 2622 RmwOp::Sub, 2623 OperandSize::S16, 2624 Some(Extend::<Zero>::I32Extend16), 2625 ) 2626 } 2627 2628 fn visit_i32_atomic_rmw_sub(&mut self, arg: MemArg) -> Self::Output { 2629 self.emit_atomic_rmw(&arg, RmwOp::Sub, OperandSize::S32, None) 2630 } 2631 2632 fn visit_i64_atomic_rmw8_sub_u(&mut self, arg: MemArg) -> Self::Output { 2633 self.emit_atomic_rmw( 2634 &arg, 2635 RmwOp::Sub, 2636 OperandSize::S8, 2637 Some(Extend::<Zero>::I64Extend8), 2638 ) 2639 } 2640 2641 fn visit_i64_atomic_rmw16_sub_u(&mut self, arg: MemArg) -> Self::Output { 2642 self.emit_atomic_rmw( 2643 &arg, 2644 RmwOp::Sub, 2645 OperandSize::S16, 2646 Some(Extend::<Zero>::I64Extend16), 2647 ) 2648 } 2649 2650 fn visit_i64_atomic_rmw32_sub_u(&mut self, arg: MemArg) -> Self::Output { 2651 self.emit_atomic_rmw( 2652 &arg, 2653 RmwOp::Sub, 2654 OperandSize::S32, 2655 Some(Extend::<Zero>::I64Extend32), 2656 ) 2657 } 2658 2659 fn visit_i64_atomic_rmw_sub(&mut self, arg: MemArg) -> Self::Output { 2660 self.emit_atomic_rmw(&arg, RmwOp::Sub, OperandSize::S64, None) 2661 } 2662 2663 fn visit_i32_atomic_rmw8_xchg_u(&mut self, arg: MemArg) -> Self::Output { 2664 self.emit_atomic_rmw( 2665 &arg, 2666 RmwOp::Xchg, 2667 OperandSize::S8, 2668 Some(Extend::<Zero>::I32Extend8), 2669 ) 2670 } 2671 2672 fn visit_i32_atomic_rmw16_xchg_u(&mut self, arg: MemArg) -> Self::Output { 2673 self.emit_atomic_rmw( 2674 &arg, 2675 RmwOp::Xchg, 2676 OperandSize::S16, 2677 Some(Extend::<Zero>::I32Extend16), 2678 ) 2679 } 2680 2681 fn visit_i32_atomic_rmw_xchg(&mut self, arg: MemArg) -> Self::Output { 2682 self.emit_atomic_rmw(&arg, RmwOp::Xchg, OperandSize::S32, None) 2683 } 2684 2685 fn visit_i64_atomic_rmw8_xchg_u(&mut self, arg: MemArg) -> Self::Output { 2686 self.emit_atomic_rmw( 2687 &arg, 2688 RmwOp::Xchg, 2689 OperandSize::S8, 2690 Some(Extend::<Zero>::I64Extend8), 2691 ) 2692 } 2693 2694 fn visit_i64_atomic_rmw16_xchg_u(&mut self, arg: MemArg) -> Self::Output { 2695 self.emit_atomic_rmw( 2696 &arg, 2697 RmwOp::Xchg, 2698 OperandSize::S16, 2699 Some(Extend::<Zero>::I64Extend16), 2700 ) 2701 } 2702 2703 fn visit_i64_atomic_rmw32_xchg_u(&mut self, arg: MemArg) -> Self::Output { 2704 self.emit_atomic_rmw( 2705 &arg, 2706 RmwOp::Xchg, 2707 OperandSize::S32, 2708 Some(Extend::<Zero>::I64Extend32), 2709 ) 2710 } 2711 2712 fn visit_i64_atomic_rmw_xchg(&mut self, arg: MemArg) -> Self::Output { 2713 self.emit_atomic_rmw(&arg, RmwOp::Xchg, OperandSize::S64, None) 2714 } 2715 2716 fn visit_i32_atomic_rmw8_and_u(&mut self, arg: MemArg) -> Self::Output { 2717 self.emit_atomic_rmw( 2718 &arg, 2719 RmwOp::And, 2720 OperandSize::S8, 2721 Some(Extend::<Zero>::I32Extend8), 2722 ) 2723 } 2724 2725 fn visit_i32_atomic_rmw16_and_u(&mut self, arg: MemArg) -> Self::Output { 2726 self.emit_atomic_rmw( 2727 &arg, 2728 RmwOp::And, 2729 OperandSize::S16, 2730 Some(Extend::<Zero>::I32Extend16), 2731 ) 2732 } 2733 2734 fn visit_i32_atomic_rmw_and(&mut self, arg: MemArg) -> Self::Output { 2735 self.emit_atomic_rmw(&arg, RmwOp::And, OperandSize::S32, None) 2736 } 2737 2738 fn visit_i64_atomic_rmw8_and_u(&mut self, arg: MemArg) -> Self::Output { 2739 self.emit_atomic_rmw( 2740 &arg, 2741 RmwOp::And, 2742 OperandSize::S8, 2743 Some(Extend::<Zero>::I64Extend8), 2744 ) 2745 } 2746 2747 fn visit_i64_atomic_rmw16_and_u(&mut self, arg: MemArg) -> Self::Output { 2748 self.emit_atomic_rmw( 2749 &arg, 2750 RmwOp::And, 2751 OperandSize::S16, 2752 Some(Extend::<Zero>::I64Extend16), 2753 ) 2754 } 2755 2756 fn visit_i64_atomic_rmw32_and_u(&mut self, arg: MemArg) -> Self::Output { 2757 self.emit_atomic_rmw( 2758 &arg, 2759 RmwOp::And, 2760 OperandSize::S32, 2761 Some(Extend::<Zero>::I64Extend32), 2762 ) 2763 } 2764 2765 fn visit_i64_atomic_rmw_and(&mut self, arg: MemArg) -> Self::Output { 2766 self.emit_atomic_rmw(&arg, RmwOp::And, OperandSize::S64, None) 2767 } 2768 2769 fn visit_i32_atomic_rmw8_or_u(&mut self, arg: MemArg) -> Self::Output { 2770 self.emit_atomic_rmw( 2771 &arg, 2772 RmwOp::Or, 2773 OperandSize::S8, 2774 Some(Extend::<Zero>::I32Extend8), 2775 ) 2776 } 2777 2778 fn visit_i32_atomic_rmw16_or_u(&mut self, arg: MemArg) -> Self::Output { 2779 self.emit_atomic_rmw( 2780 &arg, 2781 RmwOp::Or, 2782 OperandSize::S16, 2783 Some(Extend::<Zero>::I32Extend16), 2784 ) 2785 } 2786 2787 fn visit_i32_atomic_rmw_or(&mut self, arg: MemArg) -> Self::Output { 2788 self.emit_atomic_rmw(&arg, RmwOp::Or, OperandSize::S32, None) 2789 } 2790 2791 fn visit_i64_atomic_rmw8_or_u(&mut self, arg: MemArg) -> Self::Output { 2792 self.emit_atomic_rmw( 2793 &arg, 2794 RmwOp::Or, 2795 OperandSize::S8, 2796 Some(Extend::<Zero>::I64Extend8), 2797 ) 2798 } 2799 2800 fn visit_i64_atomic_rmw16_or_u(&mut self, arg: MemArg) -> Self::Output { 2801 self.emit_atomic_rmw( 2802 &arg, 2803 RmwOp::Or, 2804 OperandSize::S16, 2805 Some(Extend::<Zero>::I64Extend16), 2806 ) 2807 } 2808 2809 fn visit_i64_atomic_rmw32_or_u(&mut self, arg: MemArg) -> Self::Output { 2810 self.emit_atomic_rmw( 2811 &arg, 2812 RmwOp::Or, 2813 OperandSize::S32, 2814 Some(Extend::<Zero>::I64Extend32), 2815 ) 2816 } 2817 2818 fn visit_i64_atomic_rmw_or(&mut self, arg: MemArg) -> Self::Output { 2819 self.emit_atomic_rmw(&arg, RmwOp::Or, OperandSize::S64, None) 2820 } 2821 2822 fn visit_i32_atomic_rmw8_xor_u(&mut self, arg: MemArg) -> Self::Output { 2823 self.emit_atomic_rmw( 2824 &arg, 2825 RmwOp::Xor, 2826 OperandSize::S8, 2827 Some(Extend::<Zero>::I32Extend8), 2828 ) 2829 } 2830 2831 fn visit_i32_atomic_rmw16_xor_u(&mut self, arg: MemArg) -> Self::Output { 2832 self.emit_atomic_rmw( 2833 &arg, 2834 RmwOp::Xor, 2835 OperandSize::S16, 2836 Some(Extend::<Zero>::I32Extend16), 2837 ) 2838 } 2839 2840 fn visit_i32_atomic_rmw_xor(&mut self, arg: MemArg) -> Self::Output { 2841 self.emit_atomic_rmw(&arg, RmwOp::Xor, OperandSize::S32, None) 2842 } 2843 2844 fn visit_i64_atomic_rmw8_xor_u(&mut self, arg: MemArg) -> Self::Output { 2845 self.emit_atomic_rmw( 2846 &arg, 2847 RmwOp::Xor, 2848 OperandSize::S8, 2849 Some(Extend::<Zero>::I64Extend8), 2850 ) 2851 } 2852 2853 fn visit_i64_atomic_rmw16_xor_u(&mut self, arg: MemArg) -> Self::Output { 2854 self.emit_atomic_rmw( 2855 &arg, 2856 RmwOp::Xor, 2857 OperandSize::S16, 2858 Some(Extend::<Zero>::I64Extend16), 2859 ) 2860 } 2861 2862 fn visit_i64_atomic_rmw32_xor_u(&mut self, arg: MemArg) -> Self::Output { 2863 self.emit_atomic_rmw( 2864 &arg, 2865 RmwOp::Xor, 2866 OperandSize::S32, 2867 Some(Extend::<Zero>::I64Extend32), 2868 ) 2869 } 2870 2871 fn visit_i64_atomic_rmw_xor(&mut self, arg: MemArg) -> Self::Output { 2872 self.emit_atomic_rmw(&arg, RmwOp::Xor, OperandSize::S64, None) 2873 } 2874 2875 fn visit_i32_atomic_rmw8_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output { 2876 self.emit_atomic_cmpxchg(&arg, OperandSize::S8, Some(Extend::I32Extend8)) 2877 } 2878 2879 fn visit_i32_atomic_rmw16_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output { 2880 self.emit_atomic_cmpxchg(&arg, OperandSize::S16, Some(Extend::I32Extend16)) 2881 } 2882 2883 fn visit_i32_atomic_rmw_cmpxchg(&mut self, arg: MemArg) -> Self::Output { 2884 self.emit_atomic_cmpxchg(&arg, OperandSize::S32, None) 2885 } 2886 2887 fn visit_i64_atomic_rmw8_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output { 2888 self.emit_atomic_cmpxchg(&arg, OperandSize::S8, Some(Extend::I64Extend8)) 2889 } 2890 2891 fn visit_i64_atomic_rmw16_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output { 2892 self.emit_atomic_cmpxchg(&arg, OperandSize::S16, Some(Extend::I64Extend16)) 2893 } 2894 2895 fn visit_i64_atomic_rmw32_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output { 2896 self.emit_atomic_cmpxchg(&arg, OperandSize::S32, Some(Extend::I64Extend32)) 2897 } 2898 2899 fn visit_i64_atomic_rmw_cmpxchg(&mut self, arg: MemArg) -> Self::Output { 2900 self.emit_atomic_cmpxchg(&arg, OperandSize::S64, None) 2901 } 2902 2903 fn visit_memory_atomic_wait32(&mut self, arg: MemArg) -> Self::Output { 2904 self.emit_atomic_wait(&arg, AtomicWaitKind::Wait32) 2905 } 2906 2907 fn visit_memory_atomic_wait64(&mut self, arg: MemArg) -> Self::Output { 2908 self.emit_atomic_wait(&arg, AtomicWaitKind::Wait64) 2909 } 2910 2911 fn visit_memory_atomic_notify(&mut self, arg: MemArg) -> Self::Output { 2912 self.emit_atomic_notify(&arg) 2913 } 2914 2915 fn visit_atomic_fence(&mut self) -> Self::Output { 2916 self.masm.fence() 2917 } 2918 2919 wasmparser::for_each_visit_operator!(def_unsupported); 2920 } 2921 2922 impl<'a, 'translation, 'data, M> VisitSimdOperator<'a> 2923 for CodeGen<'a, 'translation, 'data, M, Emission> 2924 where 2925 M: MacroAssembler, 2926 { 2927 fn visit_v128_const(&mut self, val: V128) -> Self::Output { 2928 self.context.stack.push(Val::v128(val.i128())); 2929 Ok(()) 2930 } 2931 2932 fn visit_v128_load(&mut self, memarg: MemArg) -> Self::Output { 2933 self.emit_wasm_load( 2934 &memarg, 2935 WasmValType::V128, 2936 LoadKind::Operand(OperandSize::S128), 2937 ) 2938 } 2939 2940 fn visit_v128_store(&mut self, memarg: MemArg) -> Self::Output { 2941 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S128)) 2942 } 2943 2944 fn visit_v128_load8x8_s(&mut self, memarg: MemArg) -> Self::Output { 2945 self.emit_wasm_load( 2946 &memarg, 2947 WasmValType::V128, 2948 LoadKind::VectorExtend(V128LoadExtendKind::E8x8S), 2949 ) 2950 } 2951 2952 fn visit_v128_load8x8_u(&mut self, memarg: MemArg) -> Self::Output { 2953 self.emit_wasm_load( 2954 &memarg, 2955 WasmValType::V128, 2956 LoadKind::VectorExtend(V128LoadExtendKind::E8x8U), 2957 ) 2958 } 2959 2960 fn visit_v128_load16x4_s(&mut self, memarg: MemArg) -> Self::Output { 2961 self.emit_wasm_load( 2962 &memarg, 2963 WasmValType::V128, 2964 LoadKind::VectorExtend(V128LoadExtendKind::E16x4S), 2965 ) 2966 } 2967 2968 fn visit_v128_load16x4_u(&mut self, memarg: MemArg) -> Self::Output { 2969 self.emit_wasm_load( 2970 &memarg, 2971 WasmValType::V128, 2972 LoadKind::VectorExtend(V128LoadExtendKind::E16x4U), 2973 ) 2974 } 2975 2976 fn visit_v128_load32x2_s(&mut self, memarg: MemArg) -> Self::Output { 2977 self.emit_wasm_load( 2978 &memarg, 2979 WasmValType::V128, 2980 LoadKind::VectorExtend(V128LoadExtendKind::E32x2S), 2981 ) 2982 } 2983 2984 fn visit_v128_load32x2_u(&mut self, memarg: MemArg) -> Self::Output { 2985 self.emit_wasm_load( 2986 &memarg, 2987 WasmValType::V128, 2988 LoadKind::VectorExtend(V128LoadExtendKind::E32x2U), 2989 ) 2990 } 2991 2992 fn visit_v128_load8_splat(&mut self, memarg: MemArg) -> Self::Output { 2993 self.emit_wasm_load( 2994 &memarg, 2995 WasmValType::V128, 2996 LoadKind::Splat(SplatLoadKind::S8), 2997 ) 2998 } 2999 3000 fn visit_v128_load16_splat(&mut self, memarg: MemArg) -> Self::Output { 3001 self.emit_wasm_load( 3002 &memarg, 3003 WasmValType::V128, 3004 LoadKind::Splat(SplatLoadKind::S16), 3005 ) 3006 } 3007 3008 fn visit_v128_load32_splat(&mut self, memarg: MemArg) -> Self::Output { 3009 self.emit_wasm_load( 3010 &memarg, 3011 WasmValType::V128, 3012 LoadKind::Splat(SplatLoadKind::S32), 3013 ) 3014 } 3015 3016 fn visit_v128_load64_splat(&mut self, memarg: MemArg) -> Self::Output { 3017 self.emit_wasm_load( 3018 &memarg, 3019 WasmValType::V128, 3020 LoadKind::Splat(SplatLoadKind::S64), 3021 ) 3022 } 3023 3024 fn visit_i8x16_splat(&mut self) -> Self::Output { 3025 self.masm.splat(&mut self.context, SplatKind::I8x16) 3026 } 3027 3028 fn visit_i16x8_splat(&mut self) -> Self::Output { 3029 self.masm.splat(&mut self.context, SplatKind::I16x8) 3030 } 3031 3032 fn visit_i32x4_splat(&mut self) -> Self::Output { 3033 self.masm.splat(&mut self.context, SplatKind::I32x4) 3034 } 3035 3036 fn visit_i64x2_splat(&mut self) -> Self::Output { 3037 self.masm.splat(&mut self.context, SplatKind::I64x2) 3038 } 3039 3040 fn visit_f32x4_splat(&mut self) -> Self::Output { 3041 self.masm.splat(&mut self.context, SplatKind::F32x4) 3042 } 3043 3044 fn visit_f64x2_splat(&mut self) -> Self::Output { 3045 self.masm.splat(&mut self.context, SplatKind::F64x2) 3046 } 3047 3048 fn visit_i8x16_shuffle(&mut self, lanes: [u8; 16]) -> Self::Output { 3049 let rhs = self.context.pop_to_reg(self.masm, None)?; 3050 let lhs = self.context.pop_to_reg(self.masm, None)?; 3051 self.masm 3052 .shuffle(writable!(lhs.into()), lhs.into(), rhs.into(), lanes)?; 3053 self.context.stack.push(TypedReg::v128(lhs.into()).into()); 3054 self.context.free_reg(rhs); 3055 Ok(()) 3056 } 3057 3058 fn visit_i8x16_swizzle(&mut self) -> Self::Output { 3059 let rhs = self.context.pop_to_reg(self.masm, None)?; 3060 let lhs = self.context.pop_to_reg(self.masm, None)?; 3061 self.masm 3062 .swizzle(writable!(lhs.into()), lhs.into(), rhs.into())?; 3063 self.context.stack.push(TypedReg::v128(lhs.into()).into()); 3064 self.context.free_reg(rhs); 3065 Ok(()) 3066 } 3067 3068 fn visit_i8x16_extract_lane_s(&mut self, lane: u8) -> Self::Output { 3069 self.context.extract_lane_op( 3070 self.masm, 3071 ExtractLaneKind::I8x16S, 3072 |masm, src, dst, kind| masm.extract_lane(src, dst, lane, kind), 3073 ) 3074 } 3075 3076 fn visit_i8x16_extract_lane_u(&mut self, lane: u8) -> Self::Output { 3077 self.context.extract_lane_op( 3078 self.masm, 3079 ExtractLaneKind::I8x16U, 3080 |masm, src, dst, kind| masm.extract_lane(src, dst, lane, kind), 3081 ) 3082 } 3083 3084 fn visit_i16x8_extract_lane_s(&mut self, lane: u8) -> Self::Output { 3085 self.context.extract_lane_op( 3086 self.masm, 3087 ExtractLaneKind::I16x8S, 3088 |masm, src, dst, kind| masm.extract_lane(src, dst, lane, kind), 3089 ) 3090 } 3091 3092 fn visit_i16x8_extract_lane_u(&mut self, lane: u8) -> Self::Output { 3093 self.context.extract_lane_op( 3094 self.masm, 3095 ExtractLaneKind::I16x8U, 3096 |masm, src, dst, kind| masm.extract_lane(src, dst, lane, kind), 3097 ) 3098 } 3099 3100 fn visit_i32x4_extract_lane(&mut self, lane: u8) -> Self::Output { 3101 self.context 3102 .extract_lane_op(self.masm, ExtractLaneKind::I32x4, |masm, src, dst, kind| { 3103 masm.extract_lane(src, dst, lane, kind) 3104 }) 3105 } 3106 3107 fn visit_i64x2_extract_lane(&mut self, lane: u8) -> Self::Output { 3108 self.context 3109 .extract_lane_op(self.masm, ExtractLaneKind::I64x2, |masm, src, dst, kind| { 3110 masm.extract_lane(src, dst, lane, kind) 3111 }) 3112 } 3113 3114 fn visit_f32x4_extract_lane(&mut self, lane: u8) -> Self::Output { 3115 self.context 3116 .extract_lane_op(self.masm, ExtractLaneKind::F32x4, |masm, src, dst, kind| { 3117 masm.extract_lane(src, dst, lane, kind) 3118 }) 3119 } 3120 3121 fn visit_f64x2_extract_lane(&mut self, lane: u8) -> Self::Output { 3122 self.context 3123 .extract_lane_op(self.masm, ExtractLaneKind::F64x2, |masm, src, dst, kind| { 3124 masm.extract_lane(src, dst, lane, kind) 3125 }) 3126 } 3127 3128 fn visit_i8x16_eq(&mut self) -> Self::Output { 3129 self.context 3130 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3131 masm.v128_eq(writable!(dst), dst, src, VectorEqualityKind::I8x16)?; 3132 Ok(TypedReg::v128(dst)) 3133 }) 3134 } 3135 3136 fn visit_i16x8_eq(&mut self) -> Self::Output { 3137 self.context 3138 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3139 masm.v128_eq(writable!(dst), dst, src, VectorEqualityKind::I16x8)?; 3140 Ok(TypedReg::v128(dst)) 3141 }) 3142 } 3143 3144 fn visit_i32x4_eq(&mut self) -> Self::Output { 3145 self.context 3146 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3147 masm.v128_eq(writable!(dst), dst, src, VectorEqualityKind::I32x4)?; 3148 Ok(TypedReg::v128(dst)) 3149 }) 3150 } 3151 3152 fn visit_i64x2_eq(&mut self) -> Self::Output { 3153 self.context 3154 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3155 masm.v128_eq(writable!(dst), dst, src, VectorEqualityKind::I64x2)?; 3156 Ok(TypedReg::v128(dst)) 3157 }) 3158 } 3159 3160 fn visit_f32x4_eq(&mut self) -> Self::Output { 3161 self.context 3162 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3163 masm.v128_eq(writable!(dst), dst, src, VectorEqualityKind::F32x4)?; 3164 Ok(TypedReg::v128(dst)) 3165 }) 3166 } 3167 3168 fn visit_f64x2_eq(&mut self) -> Self::Output { 3169 self.context 3170 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3171 masm.v128_eq(writable!(dst), dst, src, VectorEqualityKind::F64x2)?; 3172 Ok(TypedReg::v128(dst)) 3173 }) 3174 } 3175 3176 fn visit_i8x16_ne(&mut self) -> Self::Output { 3177 self.context 3178 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3179 masm.v128_ne(writable!(dst), dst, src, VectorEqualityKind::I8x16)?; 3180 Ok(TypedReg::v128(dst)) 3181 }) 3182 } 3183 3184 fn visit_i16x8_ne(&mut self) -> Self::Output { 3185 self.context 3186 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3187 masm.v128_ne(writable!(dst), dst, src, VectorEqualityKind::I16x8)?; 3188 Ok(TypedReg::v128(dst)) 3189 }) 3190 } 3191 3192 fn visit_i32x4_ne(&mut self) -> Self::Output { 3193 self.context 3194 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3195 masm.v128_ne(writable!(dst), dst, src, VectorEqualityKind::I32x4)?; 3196 Ok(TypedReg::v128(dst)) 3197 }) 3198 } 3199 3200 fn visit_i64x2_ne(&mut self) -> Self::Output { 3201 self.context 3202 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3203 masm.v128_ne(writable!(dst), dst, src, VectorEqualityKind::I64x2)?; 3204 Ok(TypedReg::v128(dst)) 3205 }) 3206 } 3207 3208 fn visit_f32x4_ne(&mut self) -> Self::Output { 3209 self.context 3210 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3211 masm.v128_ne(writable!(dst), dst, src, VectorEqualityKind::F32x4)?; 3212 Ok(TypedReg::v128(dst)) 3213 }) 3214 } 3215 3216 fn visit_f64x2_ne(&mut self) -> Self::Output { 3217 self.context 3218 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3219 masm.v128_ne(writable!(dst), dst, src, VectorEqualityKind::F64x2)?; 3220 Ok(TypedReg::v128(dst)) 3221 }) 3222 } 3223 3224 fn visit_i8x16_lt_s(&mut self) -> Self::Output { 3225 self.context 3226 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3227 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I8x16S)?; 3228 Ok(TypedReg::v128(dst)) 3229 }) 3230 } 3231 3232 fn visit_i8x16_lt_u(&mut self) -> Self::Output { 3233 self.context 3234 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3235 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I8x16U)?; 3236 Ok(TypedReg::v128(dst)) 3237 }) 3238 } 3239 3240 fn visit_i16x8_lt_s(&mut self) -> Self::Output { 3241 self.context 3242 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3243 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I16x8S)?; 3244 Ok(TypedReg::v128(dst)) 3245 }) 3246 } 3247 3248 fn visit_i16x8_lt_u(&mut self) -> Self::Output { 3249 self.context 3250 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3251 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I16x8U)?; 3252 Ok(TypedReg::v128(dst)) 3253 }) 3254 } 3255 3256 fn visit_i32x4_lt_s(&mut self) -> Self::Output { 3257 self.context 3258 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3259 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I32x4S)?; 3260 Ok(TypedReg::v128(dst)) 3261 }) 3262 } 3263 3264 fn visit_i32x4_lt_u(&mut self) -> Self::Output { 3265 self.context 3266 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3267 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I32x4U)?; 3268 Ok(TypedReg::v128(dst)) 3269 }) 3270 } 3271 3272 fn visit_i64x2_lt_s(&mut self) -> Self::Output { 3273 self.context 3274 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3275 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I64x2S)?; 3276 Ok(TypedReg::v128(dst)) 3277 }) 3278 } 3279 3280 fn visit_f32x4_lt(&mut self) -> Self::Output { 3281 self.context 3282 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3283 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::F32x4)?; 3284 Ok(TypedReg::v128(dst)) 3285 }) 3286 } 3287 3288 fn visit_f64x2_lt(&mut self) -> Self::Output { 3289 self.context 3290 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3291 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::F64x2)?; 3292 Ok(TypedReg::v128(dst)) 3293 }) 3294 } 3295 3296 fn visit_i8x16_le_s(&mut self) -> Self::Output { 3297 self.context 3298 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3299 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I8x16S)?; 3300 Ok(TypedReg::v128(dst)) 3301 }) 3302 } 3303 3304 fn visit_i8x16_le_u(&mut self) -> Self::Output { 3305 self.context 3306 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3307 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I8x16U)?; 3308 Ok(TypedReg::v128(dst)) 3309 }) 3310 } 3311 3312 fn visit_i16x8_le_s(&mut self) -> Self::Output { 3313 self.context 3314 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3315 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I16x8S)?; 3316 Ok(TypedReg::v128(dst)) 3317 }) 3318 } 3319 3320 fn visit_i16x8_le_u(&mut self) -> Self::Output { 3321 self.context 3322 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3323 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I16x8U)?; 3324 Ok(TypedReg::v128(dst)) 3325 }) 3326 } 3327 3328 fn visit_i32x4_le_s(&mut self) -> Self::Output { 3329 self.context 3330 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3331 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I32x4S)?; 3332 Ok(TypedReg::v128(dst)) 3333 }) 3334 } 3335 3336 fn visit_i32x4_le_u(&mut self) -> Self::Output { 3337 self.context 3338 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3339 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I32x4U)?; 3340 Ok(TypedReg::v128(dst)) 3341 }) 3342 } 3343 3344 fn visit_i64x2_le_s(&mut self) -> Self::Output { 3345 self.context 3346 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3347 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I64x2S)?; 3348 Ok(TypedReg::v128(dst)) 3349 }) 3350 } 3351 3352 fn visit_f32x4_le(&mut self) -> Self::Output { 3353 self.context 3354 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3355 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::F32x4)?; 3356 Ok(TypedReg::v128(dst)) 3357 }) 3358 } 3359 3360 fn visit_f64x2_le(&mut self) -> Self::Output { 3361 self.context 3362 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3363 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::F64x2)?; 3364 Ok(TypedReg::v128(dst)) 3365 }) 3366 } 3367 3368 fn visit_i8x16_gt_s(&mut self) -> Self::Output { 3369 self.context 3370 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3371 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I8x16S)?; 3372 Ok(TypedReg::v128(dst)) 3373 }) 3374 } 3375 3376 fn visit_i8x16_gt_u(&mut self) -> Self::Output { 3377 self.context 3378 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3379 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I8x16U)?; 3380 Ok(TypedReg::v128(dst)) 3381 }) 3382 } 3383 3384 fn visit_i16x8_gt_s(&mut self) -> Self::Output { 3385 self.context 3386 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3387 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I16x8S)?; 3388 Ok(TypedReg::v128(dst)) 3389 }) 3390 } 3391 3392 fn visit_i16x8_gt_u(&mut self) -> Self::Output { 3393 self.context 3394 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3395 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I16x8U)?; 3396 Ok(TypedReg::v128(dst)) 3397 }) 3398 } 3399 3400 fn visit_i32x4_gt_s(&mut self) -> Self::Output { 3401 self.context 3402 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3403 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I32x4S)?; 3404 Ok(TypedReg::v128(dst)) 3405 }) 3406 } 3407 3408 fn visit_i32x4_gt_u(&mut self) -> Self::Output { 3409 self.context 3410 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3411 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I32x4U)?; 3412 Ok(TypedReg::v128(dst)) 3413 }) 3414 } 3415 3416 fn visit_i64x2_gt_s(&mut self) -> Self::Output { 3417 self.context 3418 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3419 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I64x2S)?; 3420 Ok(TypedReg::v128(dst)) 3421 }) 3422 } 3423 3424 fn visit_f32x4_gt(&mut self) -> Self::Output { 3425 self.context 3426 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3427 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::F32x4)?; 3428 Ok(TypedReg::v128(dst)) 3429 }) 3430 } 3431 3432 fn visit_f64x2_gt(&mut self) -> Self::Output { 3433 self.context 3434 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3435 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::F64x2)?; 3436 Ok(TypedReg::v128(dst)) 3437 }) 3438 } 3439 3440 fn visit_i8x16_ge_s(&mut self) -> Self::Output { 3441 self.context 3442 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3443 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I8x16S)?; 3444 Ok(TypedReg::v128(dst)) 3445 }) 3446 } 3447 3448 fn visit_i8x16_ge_u(&mut self) -> Self::Output { 3449 self.context 3450 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3451 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I8x16U)?; 3452 Ok(TypedReg::v128(dst)) 3453 }) 3454 } 3455 3456 fn visit_i16x8_ge_s(&mut self) -> Self::Output { 3457 self.context 3458 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3459 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I16x8S)?; 3460 Ok(TypedReg::v128(dst)) 3461 }) 3462 } 3463 3464 fn visit_i16x8_ge_u(&mut self) -> Self::Output { 3465 self.context 3466 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3467 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I16x8U)?; 3468 Ok(TypedReg::v128(dst)) 3469 }) 3470 } 3471 3472 fn visit_i32x4_ge_s(&mut self) -> Self::Output { 3473 self.context 3474 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3475 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I32x4S)?; 3476 Ok(TypedReg::v128(dst)) 3477 }) 3478 } 3479 3480 fn visit_i32x4_ge_u(&mut self) -> Self::Output { 3481 self.context 3482 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3483 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I32x4U)?; 3484 Ok(TypedReg::v128(dst)) 3485 }) 3486 } 3487 3488 fn visit_i64x2_ge_s(&mut self) -> Self::Output { 3489 self.context 3490 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3491 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I64x2S)?; 3492 Ok(TypedReg::v128(dst)) 3493 }) 3494 } 3495 3496 fn visit_f32x4_ge(&mut self) -> Self::Output { 3497 self.context 3498 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3499 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::F32x4)?; 3500 Ok(TypedReg::v128(dst)) 3501 }) 3502 } 3503 3504 fn visit_f64x2_ge(&mut self) -> Self::Output { 3505 self.context 3506 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3507 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::F64x2)?; 3508 Ok(TypedReg::v128(dst)) 3509 }) 3510 } 3511 3512 fn visit_i8x16_replace_lane(&mut self, lane: u8) -> Self::Output { 3513 self.context 3514 .replace_lane_op(self.masm, ReplaceLaneKind::I8x16, |masm, src, dst, kind| { 3515 masm.replace_lane(src, dst, lane, kind) 3516 }) 3517 } 3518 3519 fn visit_i16x8_replace_lane(&mut self, lane: u8) -> Self::Output { 3520 self.context 3521 .replace_lane_op(self.masm, ReplaceLaneKind::I16x8, |masm, src, dst, kind| { 3522 masm.replace_lane(src, dst, lane, kind) 3523 }) 3524 } 3525 3526 fn visit_i32x4_replace_lane(&mut self, lane: u8) -> Self::Output { 3527 self.context 3528 .replace_lane_op(self.masm, ReplaceLaneKind::I32x4, |masm, src, dst, kind| { 3529 masm.replace_lane(src, dst, lane, kind) 3530 }) 3531 } 3532 3533 fn visit_i64x2_replace_lane(&mut self, lane: u8) -> Self::Output { 3534 self.context 3535 .replace_lane_op(self.masm, ReplaceLaneKind::I64x2, |masm, src, dst, kind| { 3536 masm.replace_lane(src, dst, lane, kind) 3537 }) 3538 } 3539 3540 fn visit_f32x4_replace_lane(&mut self, lane: u8) -> Self::Output { 3541 self.context 3542 .replace_lane_op(self.masm, ReplaceLaneKind::F32x4, |masm, src, dst, kind| { 3543 masm.replace_lane(src, dst, lane, kind) 3544 }) 3545 } 3546 3547 fn visit_f64x2_replace_lane(&mut self, lane: u8) -> Self::Output { 3548 self.context 3549 .replace_lane_op(self.masm, ReplaceLaneKind::F64x2, |masm, src, dst, kind| { 3550 masm.replace_lane(src, dst, lane, kind) 3551 }) 3552 } 3553 3554 fn visit_v128_not(&mut self) -> Self::Output { 3555 self.context.unop(self.masm, |masm, reg| { 3556 masm.v128_not(writable!(reg))?; 3557 Ok(TypedReg::new(WasmValType::V128, reg)) 3558 }) 3559 } 3560 3561 fn visit_v128_and(&mut self) -> Self::Output { 3562 self.context 3563 .binop(self.masm, OperandSize::S128, |masm, dst, src, _size| { 3564 masm.v128_and(dst, src, writable!(dst))?; 3565 Ok(TypedReg::new(WasmValType::V128, dst)) 3566 }) 3567 } 3568 3569 fn visit_v128_andnot(&mut self) -> Self::Output { 3570 self.context 3571 .binop(self.masm, OperandSize::S128, |masm, dst, src, _size| { 3572 // careful here: and_not is *not* commutative: dst = !src1 & src2 3573 masm.v128_and_not(src, dst, writable!(dst))?; 3574 Ok(TypedReg::new(WasmValType::V128, dst)) 3575 }) 3576 } 3577 3578 fn visit_v128_or(&mut self) -> Self::Output { 3579 self.context 3580 .binop(self.masm, OperandSize::S128, |masm, dst, src, _size| { 3581 // careful here: and_not is *not* commutative: dst = !src1 & src2 3582 masm.v128_or(src, dst, writable!(dst))?; 3583 Ok(TypedReg::new(WasmValType::V128, dst)) 3584 }) 3585 } 3586 3587 fn visit_v128_xor(&mut self) -> Self::Output { 3588 self.context 3589 .binop(self.masm, OperandSize::S128, |masm, dst, src, _size| { 3590 // careful here: and_not is *not* commutative: dst = !src1 & src2 3591 masm.v128_xor(src, dst, writable!(dst))?; 3592 Ok(TypedReg::new(WasmValType::V128, dst)) 3593 }) 3594 } 3595 3596 fn visit_v128_bitselect(&mut self) -> Self::Output { 3597 let mask = self.context.pop_to_reg(self.masm, None)?; 3598 let op2 = self.context.pop_to_reg(self.masm, None)?; 3599 let op1 = self.context.pop_to_reg(self.masm, None)?; 3600 let dst = self.context.any_fpr(self.masm)?; 3601 3602 // careful here: bitselect is *not* commutative. 3603 self.masm 3604 .v128_bitselect(op1.reg, op2.reg, mask.reg, writable!(dst))?; 3605 3606 self.context 3607 .stack 3608 .push(TypedReg::new(WasmValType::V128, dst).into()); 3609 self.context.free_reg(op1); 3610 self.context.free_reg(op2); 3611 self.context.free_reg(mask); 3612 3613 Ok(()) 3614 } 3615 3616 fn visit_v128_any_true(&mut self) -> Self::Output { 3617 let src = self.context.pop_to_reg(self.masm, None)?; 3618 let dst = self.context.any_gpr(self.masm)?; 3619 3620 self.masm.v128_any_true(src.reg, writable!(dst))?; 3621 3622 self.context 3623 .stack 3624 .push(TypedReg::new(WasmValType::I32, dst).into()); 3625 self.context.free_reg(src); 3626 3627 Ok(()) 3628 } 3629 3630 fn visit_v128_load8_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output { 3631 self.emit_wasm_load( 3632 &arg, 3633 WasmValType::V128, 3634 LoadKind::vector_lane(lane, OperandSize::S8), 3635 ) 3636 } 3637 3638 fn visit_v128_load16_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output { 3639 self.emit_wasm_load( 3640 &arg, 3641 WasmValType::V128, 3642 LoadKind::vector_lane(lane, OperandSize::S16), 3643 ) 3644 } 3645 3646 fn visit_v128_load32_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output { 3647 self.emit_wasm_load( 3648 &arg, 3649 WasmValType::V128, 3650 LoadKind::vector_lane(lane, OperandSize::S32), 3651 ) 3652 } 3653 3654 fn visit_v128_load64_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output { 3655 self.emit_wasm_load( 3656 &arg, 3657 WasmValType::V128, 3658 LoadKind::vector_lane(lane, OperandSize::S64), 3659 ) 3660 } 3661 3662 fn visit_v128_store8_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output { 3663 self.emit_wasm_store(&arg, StoreKind::vector_lane(lane, OperandSize::S8)) 3664 } 3665 3666 fn visit_v128_store16_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output { 3667 self.emit_wasm_store(&arg, StoreKind::vector_lane(lane, OperandSize::S16)) 3668 } 3669 3670 fn visit_v128_store32_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output { 3671 self.emit_wasm_store(&arg, StoreKind::vector_lane(lane, OperandSize::S32)) 3672 } 3673 3674 fn visit_v128_store64_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output { 3675 self.emit_wasm_store(&arg, StoreKind::vector_lane(lane, OperandSize::S64)) 3676 } 3677 3678 fn visit_f32x4_convert_i32x4_s(&mut self) -> Self::Output { 3679 self.context.unop(self.masm, |masm, reg| { 3680 masm.v128_convert(reg, writable!(reg), V128ConvertKind::I32x4S)?; 3681 Ok(TypedReg::v128(reg)) 3682 }) 3683 } 3684 3685 fn visit_f32x4_convert_i32x4_u(&mut self) -> Self::Output { 3686 self.context.unop(self.masm, |masm, reg| { 3687 masm.v128_convert(reg, writable!(reg), V128ConvertKind::I32x4U)?; 3688 Ok(TypedReg::v128(reg)) 3689 }) 3690 } 3691 3692 fn visit_f64x2_convert_low_i32x4_s(&mut self) -> Self::Output { 3693 self.context.unop(self.masm, |masm, reg| { 3694 masm.v128_convert(reg, writable!(reg), V128ConvertKind::I32x4LowS)?; 3695 Ok(TypedReg::v128(reg)) 3696 }) 3697 } 3698 3699 fn visit_f64x2_convert_low_i32x4_u(&mut self) -> Self::Output { 3700 self.context.unop(self.masm, |masm, reg| { 3701 masm.v128_convert(reg, writable!(reg), V128ConvertKind::I32x4LowU)?; 3702 Ok(TypedReg::v128(reg)) 3703 }) 3704 } 3705 3706 fn visit_i8x16_narrow_i16x8_s(&mut self) -> Self::Output { 3707 self.context 3708 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3709 masm.v128_narrow(dst, src, writable!(dst), V128NarrowKind::I16x8S)?; 3710 Ok(TypedReg::v128(dst)) 3711 }) 3712 } 3713 3714 fn visit_i8x16_narrow_i16x8_u(&mut self) -> Self::Output { 3715 self.context 3716 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3717 masm.v128_narrow(dst, src, writable!(dst), V128NarrowKind::I16x8U)?; 3718 Ok(TypedReg::v128(dst)) 3719 }) 3720 } 3721 3722 fn visit_i16x8_narrow_i32x4_s(&mut self) -> Self::Output { 3723 self.context 3724 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3725 masm.v128_narrow(dst, src, writable!(dst), V128NarrowKind::I32x4S)?; 3726 Ok(TypedReg::v128(dst)) 3727 }) 3728 } 3729 3730 fn visit_i16x8_narrow_i32x4_u(&mut self) -> Self::Output { 3731 self.context 3732 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3733 masm.v128_narrow(dst, src, writable!(dst), V128NarrowKind::I32x4U)?; 3734 Ok(TypedReg::v128(dst)) 3735 }) 3736 } 3737 3738 fn visit_f32x4_demote_f64x2_zero(&mut self) -> Self::Output { 3739 self.context.unop(self.masm, |masm, reg| { 3740 masm.v128_demote(reg, writable!(reg))?; 3741 Ok(TypedReg::v128(reg)) 3742 }) 3743 } 3744 3745 fn visit_f64x2_promote_low_f32x4(&mut self) -> Self::Output { 3746 self.context.unop(self.masm, |masm, reg| { 3747 masm.v128_promote(reg, writable!(reg))?; 3748 Ok(TypedReg::v128(reg)) 3749 }) 3750 } 3751 3752 fn visit_i16x8_extend_low_i8x16_s(&mut self) -> Self::Output { 3753 self.context.unop(self.masm, |masm, reg| { 3754 masm.v128_extend(reg, writable!(reg), V128ExtendKind::LowI8x16S)?; 3755 Ok(TypedReg::v128(reg)) 3756 }) 3757 } 3758 3759 fn visit_i16x8_extend_high_i8x16_s(&mut self) -> Self::Output { 3760 self.context.unop(self.masm, |masm, reg| { 3761 masm.v128_extend(reg, writable!(reg), V128ExtendKind::HighI8x16S)?; 3762 Ok(TypedReg::v128(reg)) 3763 }) 3764 } 3765 3766 fn visit_i16x8_extend_low_i8x16_u(&mut self) -> Self::Output { 3767 self.context.unop(self.masm, |masm, reg| { 3768 masm.v128_extend(reg, writable!(reg), V128ExtendKind::LowI8x16U)?; 3769 Ok(TypedReg::v128(reg)) 3770 }) 3771 } 3772 3773 fn visit_i16x8_extend_high_i8x16_u(&mut self) -> Self::Output { 3774 self.context.unop(self.masm, |masm, reg| { 3775 masm.v128_extend(reg, writable!(reg), V128ExtendKind::HighI8x16U)?; 3776 Ok(TypedReg::v128(reg)) 3777 }) 3778 } 3779 3780 fn visit_i32x4_extend_low_i16x8_s(&mut self) -> Self::Output { 3781 self.context.unop(self.masm, |masm, reg| { 3782 masm.v128_extend(reg, writable!(reg), V128ExtendKind::LowI16x8S)?; 3783 Ok(TypedReg::v128(reg)) 3784 }) 3785 } 3786 3787 fn visit_i32x4_extend_high_i16x8_s(&mut self) -> Self::Output { 3788 self.context.unop(self.masm, |masm, reg| { 3789 masm.v128_extend(reg, writable!(reg), V128ExtendKind::HighI16x8S)?; 3790 Ok(TypedReg::v128(reg)) 3791 }) 3792 } 3793 3794 fn visit_i32x4_extend_low_i16x8_u(&mut self) -> Self::Output { 3795 self.context.unop(self.masm, |masm, reg| { 3796 masm.v128_extend(reg, writable!(reg), V128ExtendKind::LowI16x8U)?; 3797 Ok(TypedReg::v128(reg)) 3798 }) 3799 } 3800 3801 fn visit_i32x4_extend_high_i16x8_u(&mut self) -> Self::Output { 3802 self.context.unop(self.masm, |masm, reg| { 3803 masm.v128_extend(reg, writable!(reg), V128ExtendKind::HighI16x8U)?; 3804 Ok(TypedReg::v128(reg)) 3805 }) 3806 } 3807 3808 fn visit_i64x2_extend_low_i32x4_s(&mut self) -> Self::Output { 3809 self.context.unop(self.masm, |masm, reg| { 3810 masm.v128_extend(reg, writable!(reg), V128ExtendKind::LowI32x4S)?; 3811 Ok(TypedReg::v128(reg)) 3812 }) 3813 } 3814 3815 fn visit_i64x2_extend_high_i32x4_s(&mut self) -> Self::Output { 3816 self.context.unop(self.masm, |masm, reg| { 3817 masm.v128_extend(reg, writable!(reg), V128ExtendKind::HighI32x4S)?; 3818 Ok(TypedReg::v128(reg)) 3819 }) 3820 } 3821 3822 fn visit_i64x2_extend_low_i32x4_u(&mut self) -> Self::Output { 3823 self.context.unop(self.masm, |masm, reg| { 3824 masm.v128_extend(reg, writable!(reg), V128ExtendKind::LowI32x4U)?; 3825 Ok(TypedReg::v128(reg)) 3826 }) 3827 } 3828 3829 fn visit_i64x2_extend_high_i32x4_u(&mut self) -> Self::Output { 3830 self.context.unop(self.masm, |masm, reg| { 3831 masm.v128_extend(reg, writable!(reg), V128ExtendKind::HighI32x4U)?; 3832 Ok(TypedReg::v128(reg)) 3833 }) 3834 } 3835 3836 fn visit_i8x16_add(&mut self) -> Self::Output { 3837 self.context 3838 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3839 masm.v128_add(dst, src, writable!(dst), V128AddKind::I8x16)?; 3840 Ok(TypedReg::new(WasmValType::V128, dst)) 3841 }) 3842 } 3843 3844 fn visit_i16x8_add(&mut self) -> Self::Output { 3845 self.context 3846 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3847 masm.v128_add(dst, src, writable!(dst), V128AddKind::I16x8)?; 3848 Ok(TypedReg::new(WasmValType::V128, dst)) 3849 }) 3850 } 3851 3852 fn visit_i32x4_add(&mut self) -> Self::Output { 3853 self.context 3854 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3855 masm.v128_add(dst, src, writable!(dst), V128AddKind::I32x4)?; 3856 Ok(TypedReg::new(WasmValType::V128, dst)) 3857 }) 3858 } 3859 3860 fn visit_i64x2_add(&mut self) -> Self::Output { 3861 self.context 3862 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3863 masm.v128_add(dst, src, writable!(dst), V128AddKind::I64x2)?; 3864 Ok(TypedReg::new(WasmValType::V128, dst)) 3865 }) 3866 } 3867 3868 fn visit_i8x16_sub(&mut self) -> Self::Output { 3869 self.context 3870 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3871 masm.v128_sub(dst, src, writable!(dst), V128SubKind::I8x16)?; 3872 Ok(TypedReg::new(WasmValType::V128, dst)) 3873 }) 3874 } 3875 3876 fn visit_i16x8_sub(&mut self) -> Self::Output { 3877 self.context 3878 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3879 masm.v128_sub(dst, src, writable!(dst), V128SubKind::I16x8)?; 3880 Ok(TypedReg::new(WasmValType::V128, dst)) 3881 }) 3882 } 3883 3884 fn visit_i32x4_sub(&mut self) -> Self::Output { 3885 self.context 3886 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3887 masm.v128_sub(dst, src, writable!(dst), V128SubKind::I32x4)?; 3888 Ok(TypedReg::new(WasmValType::V128, dst)) 3889 }) 3890 } 3891 3892 fn visit_i64x2_sub(&mut self) -> Self::Output { 3893 self.context 3894 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3895 masm.v128_sub(dst, src, writable!(dst), V128SubKind::I64x2)?; 3896 Ok(TypedReg::new(WasmValType::V128, dst)) 3897 }) 3898 } 3899 3900 fn visit_i16x8_mul(&mut self) -> Self::Output { 3901 self.masm.v128_mul(&mut self.context, V128MulKind::I16x8) 3902 } 3903 3904 fn visit_i32x4_mul(&mut self) -> Self::Output { 3905 self.masm.v128_mul(&mut self.context, V128MulKind::I32x4) 3906 } 3907 3908 fn visit_i64x2_mul(&mut self) -> Self::Output { 3909 self.masm.v128_mul(&mut self.context, V128MulKind::I64x2) 3910 } 3911 3912 fn visit_i8x16_add_sat_s(&mut self) -> Self::Output { 3913 self.context 3914 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3915 masm.v128_add(dst, src, writable!(dst), V128AddKind::I8x16SatS)?; 3916 Ok(TypedReg::new(WasmValType::V128, dst)) 3917 }) 3918 } 3919 3920 fn visit_i16x8_add_sat_s(&mut self) -> Self::Output { 3921 self.context 3922 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3923 masm.v128_add(dst, src, writable!(dst), V128AddKind::I16x8SatS)?; 3924 Ok(TypedReg::new(WasmValType::V128, dst)) 3925 }) 3926 } 3927 3928 fn visit_i8x16_add_sat_u(&mut self) -> Self::Output { 3929 self.context 3930 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3931 masm.v128_add(dst, src, writable!(dst), V128AddKind::I8x16SatU)?; 3932 Ok(TypedReg::new(WasmValType::V128, dst)) 3933 }) 3934 } 3935 3936 fn visit_i16x8_add_sat_u(&mut self) -> Self::Output { 3937 self.context 3938 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3939 masm.v128_add(dst, src, writable!(dst), V128AddKind::I16x8SatU)?; 3940 Ok(TypedReg::new(WasmValType::V128, dst)) 3941 }) 3942 } 3943 3944 fn visit_i8x16_sub_sat_s(&mut self) -> Self::Output { 3945 self.context 3946 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3947 masm.v128_sub(dst, src, writable!(dst), V128SubKind::I8x16SatS)?; 3948 Ok(TypedReg::new(WasmValType::V128, dst)) 3949 }) 3950 } 3951 3952 fn visit_i16x8_sub_sat_s(&mut self) -> Self::Output { 3953 self.context 3954 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3955 masm.v128_sub(dst, src, writable!(dst), V128SubKind::I16x8SatS)?; 3956 Ok(TypedReg::new(WasmValType::V128, dst)) 3957 }) 3958 } 3959 3960 fn visit_i8x16_sub_sat_u(&mut self) -> Self::Output { 3961 self.context 3962 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3963 masm.v128_sub(dst, src, writable!(dst), V128SubKind::I8x16SatU)?; 3964 Ok(TypedReg::new(WasmValType::V128, dst)) 3965 }) 3966 } 3967 3968 fn visit_i16x8_sub_sat_u(&mut self) -> Self::Output { 3969 self.context 3970 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3971 masm.v128_sub(dst, src, writable!(dst), V128SubKind::I16x8SatU)?; 3972 Ok(TypedReg::new(WasmValType::V128, dst)) 3973 }) 3974 } 3975 3976 fn visit_i8x16_abs(&mut self) -> Self::Output { 3977 self.context.unop(self.masm, |masm, reg| { 3978 masm.v128_abs(reg, writable!(reg), V128AbsKind::I8x16)?; 3979 Ok(TypedReg::new(WasmValType::V128, reg)) 3980 }) 3981 } 3982 3983 fn visit_i16x8_abs(&mut self) -> Self::Output { 3984 self.context.unop(self.masm, |masm, reg| { 3985 masm.v128_abs(reg, writable!(reg), V128AbsKind::I16x8)?; 3986 Ok(TypedReg::new(WasmValType::V128, reg)) 3987 }) 3988 } 3989 3990 fn visit_i32x4_abs(&mut self) -> Self::Output { 3991 self.context.unop(self.masm, |masm, reg| { 3992 masm.v128_abs(reg, writable!(reg), V128AbsKind::I32x4)?; 3993 Ok(TypedReg::new(WasmValType::V128, reg)) 3994 }) 3995 } 3996 3997 fn visit_i64x2_abs(&mut self) -> Self::Output { 3998 self.context.unop(self.masm, |masm, reg| { 3999 masm.v128_abs(reg, writable!(reg), V128AbsKind::I64x2)?; 4000 Ok(TypedReg::new(WasmValType::V128, reg)) 4001 }) 4002 } 4003 4004 fn visit_f32x4_abs(&mut self) -> Self::Output { 4005 self.context.unop(self.masm, |masm, reg| { 4006 masm.v128_abs(reg, writable!(reg), V128AbsKind::F32x4)?; 4007 Ok(TypedReg::new(WasmValType::V128, reg)) 4008 }) 4009 } 4010 4011 fn visit_f64x2_abs(&mut self) -> Self::Output { 4012 self.context.unop(self.masm, |masm, reg| { 4013 masm.v128_abs(reg, writable!(reg), V128AbsKind::F64x2)?; 4014 Ok(TypedReg::new(WasmValType::V128, reg)) 4015 }) 4016 } 4017 4018 fn visit_i8x16_neg(&mut self) -> Self::Output { 4019 self.context.unop(self.masm, |masm, op| { 4020 masm.v128_neg(writable!(op), V128NegKind::I8x16)?; 4021 Ok(TypedReg::new(WasmValType::V128, op)) 4022 }) 4023 } 4024 4025 fn visit_i16x8_neg(&mut self) -> Self::Output { 4026 self.context.unop(self.masm, |masm, op| { 4027 masm.v128_neg(writable!(op), V128NegKind::I16x8)?; 4028 Ok(TypedReg::new(WasmValType::V128, op)) 4029 }) 4030 } 4031 4032 fn visit_i32x4_neg(&mut self) -> Self::Output { 4033 self.context.unop(self.masm, |masm, op| { 4034 masm.v128_neg(writable!(op), V128NegKind::I32x4)?; 4035 Ok(TypedReg::new(WasmValType::V128, op)) 4036 }) 4037 } 4038 4039 fn visit_i64x2_neg(&mut self) -> Self::Output { 4040 self.context.unop(self.masm, |masm, op| { 4041 masm.v128_neg(writable!(op), V128NegKind::I64x2)?; 4042 Ok(TypedReg::new(WasmValType::V128, op)) 4043 }) 4044 } 4045 4046 fn visit_i8x16_shl(&mut self) -> Self::Output { 4047 self.masm 4048 .v128_shift(&mut self.context, OperandSize::S8, ShiftKind::Shl) 4049 } 4050 4051 fn visit_i16x8_shl(&mut self) -> Self::Output { 4052 self.masm 4053 .v128_shift(&mut self.context, OperandSize::S16, ShiftKind::Shl) 4054 } 4055 4056 fn visit_i32x4_shl(&mut self) -> Self::Output { 4057 self.masm 4058 .v128_shift(&mut self.context, OperandSize::S32, ShiftKind::Shl) 4059 } 4060 4061 fn visit_i64x2_shl(&mut self) -> Self::Output { 4062 self.masm 4063 .v128_shift(&mut self.context, OperandSize::S64, ShiftKind::Shl) 4064 } 4065 4066 fn visit_i8x16_shr_u(&mut self) -> Self::Output { 4067 self.masm 4068 .v128_shift(&mut self.context, OperandSize::S8, ShiftKind::ShrU) 4069 } 4070 4071 fn visit_i16x8_shr_u(&mut self) -> Self::Output { 4072 self.masm 4073 .v128_shift(&mut self.context, OperandSize::S16, ShiftKind::ShrU) 4074 } 4075 4076 fn visit_i32x4_shr_u(&mut self) -> Self::Output { 4077 self.masm 4078 .v128_shift(&mut self.context, OperandSize::S32, ShiftKind::ShrU) 4079 } 4080 4081 fn visit_i64x2_shr_u(&mut self) -> Self::Output { 4082 self.masm 4083 .v128_shift(&mut self.context, OperandSize::S64, ShiftKind::ShrU) 4084 } 4085 4086 fn visit_i8x16_shr_s(&mut self) -> Self::Output { 4087 self.masm 4088 .v128_shift(&mut self.context, OperandSize::S8, ShiftKind::ShrS) 4089 } 4090 4091 fn visit_i16x8_shr_s(&mut self) -> Self::Output { 4092 self.masm 4093 .v128_shift(&mut self.context, OperandSize::S16, ShiftKind::ShrS) 4094 } 4095 4096 fn visit_i32x4_shr_s(&mut self) -> Self::Output { 4097 self.masm 4098 .v128_shift(&mut self.context, OperandSize::S32, ShiftKind::ShrS) 4099 } 4100 4101 fn visit_i64x2_shr_s(&mut self) -> Self::Output { 4102 self.masm 4103 .v128_shift(&mut self.context, OperandSize::S64, ShiftKind::ShrS) 4104 } 4105 4106 fn visit_i16x8_q15mulr_sat_s(&mut self) -> Self::Output { 4107 self.context 4108 .binop(self.masm, OperandSize::S16, |masm, dst, src, size| { 4109 masm.v128_q15mulr_sat_s(dst, src, writable!(dst), size)?; 4110 Ok(TypedReg::v128(dst)) 4111 }) 4112 } 4113 4114 fn visit_i8x16_min_s(&mut self) -> Self::Output { 4115 self.context 4116 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 4117 masm.v128_min(src, dst, writable!(dst), V128MinKind::I8x16S)?; 4118 Ok(TypedReg::v128(dst)) 4119 }) 4120 } 4121 4122 fn visit_i8x16_all_true(&mut self) -> Self::Output { 4123 self.context.v128_all_true_op(self.masm, |masm, src, dst| { 4124 masm.v128_all_true(src, writable!(dst), OperandSize::S8) 4125 }) 4126 } 4127 4128 fn visit_i16x8_all_true(&mut self) -> Self::Output { 4129 self.context.v128_all_true_op(self.masm, |masm, src, dst| { 4130 masm.v128_all_true(src, writable!(dst), OperandSize::S16) 4131 }) 4132 } 4133 4134 fn visit_i32x4_all_true(&mut self) -> Self::Output { 4135 self.context.v128_all_true_op(self.masm, |masm, src, dst| { 4136 masm.v128_all_true(src, writable!(dst), OperandSize::S32) 4137 }) 4138 } 4139 4140 fn visit_i64x2_all_true(&mut self) -> Self::Output { 4141 self.context.v128_all_true_op(self.masm, |masm, src, dst| { 4142 masm.v128_all_true(src, writable!(dst), OperandSize::S64) 4143 }) 4144 } 4145 4146 fn visit_i8x16_bitmask(&mut self) -> Self::Output { 4147 self.context.v128_bitmask_op(self.masm, |masm, src, dst| { 4148 masm.v128_bitmask(src, writable!(dst), OperandSize::S8) 4149 }) 4150 } 4151 4152 fn visit_i16x8_bitmask(&mut self) -> Self::Output { 4153 self.context.v128_bitmask_op(self.masm, |masm, src, dst| { 4154 masm.v128_bitmask(src, writable!(dst), OperandSize::S16) 4155 }) 4156 } 4157 4158 fn visit_i32x4_bitmask(&mut self) -> Self::Output { 4159 self.context.v128_bitmask_op(self.masm, |masm, src, dst| { 4160 masm.v128_bitmask(src, writable!(dst), OperandSize::S32) 4161 }) 4162 } 4163 4164 fn visit_i64x2_bitmask(&mut self) -> Self::Output { 4165 self.context.v128_bitmask_op(self.masm, |masm, src, dst| { 4166 masm.v128_bitmask(src, writable!(dst), OperandSize::S64) 4167 }) 4168 } 4169 4170 fn visit_i32x4_trunc_sat_f32x4_s(&mut self) -> Self::Output { 4171 self.masm 4172 .v128_trunc(&mut self.context, V128TruncKind::I32x4FromF32x4S) 4173 } 4174 4175 fn visit_i32x4_trunc_sat_f32x4_u(&mut self) -> Self::Output { 4176 self.masm 4177 .v128_trunc(&mut self.context, V128TruncKind::I32x4FromF32x4U) 4178 } 4179 4180 fn visit_i32x4_trunc_sat_f64x2_s_zero(&mut self) -> Self::Output { 4181 self.masm 4182 .v128_trunc(&mut self.context, V128TruncKind::I32x4FromF64x2SZero) 4183 } 4184 4185 fn visit_i32x4_trunc_sat_f64x2_u_zero(&mut self) -> Self::Output { 4186 self.masm 4187 .v128_trunc(&mut self.context, V128TruncKind::I32x4FromF64x2UZero) 4188 } 4189 4190 fn visit_i16x8_min_s(&mut self) -> Self::Output { 4191 self.context 4192 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 4193 masm.v128_min(src, dst, writable!(dst), V128MinKind::I16x8S)?; 4194 Ok(TypedReg::v128(dst)) 4195 }) 4196 } 4197 4198 fn visit_i32x4_dot_i16x8_s(&mut self) -> Self::Output { 4199 self.context 4200 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4201 masm.v128_dot(dst, src, writable!(dst))?; 4202 Ok(TypedReg::v128(dst)) 4203 }) 4204 } 4205 4206 fn visit_i8x16_popcnt(&mut self) -> Self::Output { 4207 self.masm.v128_popcnt(&mut self.context) 4208 } 4209 4210 fn visit_i8x16_avgr_u(&mut self) -> Self::Output { 4211 self.context 4212 .binop(self.masm, OperandSize::S8, |masm, dst, src, size| { 4213 masm.v128_avgr(dst, src, writable!(dst), size)?; 4214 Ok(TypedReg::v128(dst)) 4215 }) 4216 } 4217 4218 fn visit_i32x4_min_s(&mut self) -> Self::Output { 4219 self.context 4220 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4221 masm.v128_min(src, dst, writable!(dst), V128MinKind::I32x4S)?; 4222 Ok(TypedReg::v128(dst)) 4223 }) 4224 } 4225 4226 fn visit_i8x16_min_u(&mut self) -> Self::Output { 4227 self.context 4228 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 4229 masm.v128_min(src, dst, writable!(dst), V128MinKind::I8x16U)?; 4230 Ok(TypedReg::v128(dst)) 4231 }) 4232 } 4233 4234 fn visit_i16x8_avgr_u(&mut self) -> Self::Output { 4235 self.context 4236 .binop(self.masm, OperandSize::S16, |masm, dst, src, size| { 4237 masm.v128_avgr(dst, src, writable!(dst), size)?; 4238 Ok(TypedReg::v128(dst)) 4239 }) 4240 } 4241 4242 fn visit_i16x8_min_u(&mut self) -> Self::Output { 4243 self.context 4244 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 4245 masm.v128_min(src, dst, writable!(dst), V128MinKind::I16x8U)?; 4246 Ok(TypedReg::v128(dst)) 4247 }) 4248 } 4249 4250 fn visit_i32x4_min_u(&mut self) -> Self::Output { 4251 self.context 4252 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4253 masm.v128_min(src, dst, writable!(dst), V128MinKind::I32x4U)?; 4254 Ok(TypedReg::v128(dst)) 4255 }) 4256 } 4257 4258 fn visit_i8x16_max_s(&mut self) -> Self::Output { 4259 self.context 4260 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 4261 masm.v128_max(src, dst, writable!(dst), V128MaxKind::I8x16S)?; 4262 Ok(TypedReg::v128(dst)) 4263 }) 4264 } 4265 4266 fn visit_i16x8_max_s(&mut self) -> Self::Output { 4267 self.context 4268 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 4269 masm.v128_max(src, dst, writable!(dst), V128MaxKind::I16x8S)?; 4270 Ok(TypedReg::v128(dst)) 4271 }) 4272 } 4273 4274 fn visit_i32x4_max_s(&mut self) -> Self::Output { 4275 self.context 4276 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4277 masm.v128_max(src, dst, writable!(dst), V128MaxKind::I32x4S)?; 4278 Ok(TypedReg::v128(dst)) 4279 }) 4280 } 4281 4282 fn visit_i8x16_max_u(&mut self) -> Self::Output { 4283 self.context 4284 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 4285 masm.v128_max(src, dst, writable!(dst), V128MaxKind::I8x16U)?; 4286 Ok(TypedReg::v128(dst)) 4287 }) 4288 } 4289 4290 fn visit_i16x8_max_u(&mut self) -> Self::Output { 4291 self.context 4292 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 4293 masm.v128_max(src, dst, writable!(dst), V128MaxKind::I16x8U)?; 4294 Ok(TypedReg::v128(dst)) 4295 }) 4296 } 4297 4298 fn visit_i32x4_max_u(&mut self) -> Self::Output { 4299 self.context 4300 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4301 masm.v128_max(src, dst, writable!(dst), V128MaxKind::I32x4U)?; 4302 Ok(TypedReg::v128(dst)) 4303 }) 4304 } 4305 4306 fn visit_i16x8_extmul_low_i8x16_s(&mut self) -> Self::Output { 4307 self.masm 4308 .v128_extmul(&mut self.context, V128ExtMulKind::LowI8x16S) 4309 } 4310 4311 fn visit_i32x4_extmul_low_i16x8_s(&mut self) -> Self::Output { 4312 self.masm 4313 .v128_extmul(&mut self.context, V128ExtMulKind::LowI16x8S) 4314 } 4315 4316 fn visit_i64x2_extmul_low_i32x4_s(&mut self) -> Self::Output { 4317 self.masm 4318 .v128_extmul(&mut self.context, V128ExtMulKind::LowI32x4S) 4319 } 4320 4321 fn visit_i16x8_extmul_low_i8x16_u(&mut self) -> Self::Output { 4322 self.masm 4323 .v128_extmul(&mut self.context, V128ExtMulKind::LowI8x16U) 4324 } 4325 4326 fn visit_i32x4_extmul_low_i16x8_u(&mut self) -> Self::Output { 4327 self.masm 4328 .v128_extmul(&mut self.context, V128ExtMulKind::LowI16x8U) 4329 } 4330 4331 fn visit_i64x2_extmul_low_i32x4_u(&mut self) -> Self::Output { 4332 self.masm 4333 .v128_extmul(&mut self.context, V128ExtMulKind::LowI32x4U) 4334 } 4335 4336 fn visit_i16x8_extmul_high_i8x16_u(&mut self) -> Self::Output { 4337 self.masm 4338 .v128_extmul(&mut self.context, V128ExtMulKind::HighI8x16U) 4339 } 4340 4341 fn visit_i32x4_extmul_high_i16x8_u(&mut self) -> Self::Output { 4342 self.masm 4343 .v128_extmul(&mut self.context, V128ExtMulKind::HighI16x8U) 4344 } 4345 4346 fn visit_i64x2_extmul_high_i32x4_u(&mut self) -> Self::Output { 4347 self.masm 4348 .v128_extmul(&mut self.context, V128ExtMulKind::HighI32x4U) 4349 } 4350 4351 fn visit_i16x8_extmul_high_i8x16_s(&mut self) -> Self::Output { 4352 self.masm 4353 .v128_extmul(&mut self.context, V128ExtMulKind::HighI8x16S) 4354 } 4355 4356 fn visit_i32x4_extmul_high_i16x8_s(&mut self) -> Self::Output { 4357 self.masm 4358 .v128_extmul(&mut self.context, V128ExtMulKind::HighI16x8S) 4359 } 4360 4361 fn visit_i64x2_extmul_high_i32x4_s(&mut self) -> Self::Output { 4362 self.masm 4363 .v128_extmul(&mut self.context, V128ExtMulKind::HighI32x4S) 4364 } 4365 4366 fn visit_i16x8_extadd_pairwise_i8x16_s(&mut self) -> Self::Output { 4367 self.context.unop(self.masm, |masm, op| { 4368 masm.v128_extadd_pairwise(op, writable!(op), V128ExtAddKind::I8x16S)?; 4369 Ok(TypedReg::v128(op)) 4370 }) 4371 } 4372 4373 fn visit_i16x8_extadd_pairwise_i8x16_u(&mut self) -> Self::Output { 4374 self.context.unop(self.masm, |masm, op| { 4375 masm.v128_extadd_pairwise(op, writable!(op), V128ExtAddKind::I8x16U)?; 4376 Ok(TypedReg::v128(op)) 4377 }) 4378 } 4379 4380 fn visit_i32x4_extadd_pairwise_i16x8_s(&mut self) -> Self::Output { 4381 self.context.unop(self.masm, |masm, op| { 4382 masm.v128_extadd_pairwise(op, writable!(op), V128ExtAddKind::I16x8S)?; 4383 Ok(TypedReg::v128(op)) 4384 }) 4385 } 4386 4387 fn visit_i32x4_extadd_pairwise_i16x8_u(&mut self) -> Self::Output { 4388 self.context.unop(self.masm, |masm, op| { 4389 masm.v128_extadd_pairwise(op, writable!(op), V128ExtAddKind::I16x8U)?; 4390 Ok(TypedReg::v128(op)) 4391 }) 4392 } 4393 4394 fn visit_f32x4_add(&mut self) -> Self::Output { 4395 self.context 4396 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4397 masm.v128_add(dst, src, writable!(dst), V128AddKind::F32x4)?; 4398 Ok(TypedReg::v128(dst)) 4399 }) 4400 } 4401 4402 fn visit_f64x2_add(&mut self) -> Self::Output { 4403 self.context 4404 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 4405 masm.v128_add(dst, src, writable!(dst), V128AddKind::F64x2)?; 4406 Ok(TypedReg::v128(dst)) 4407 }) 4408 } 4409 4410 fn visit_f32x4_sub(&mut self) -> Self::Output { 4411 self.context 4412 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4413 masm.v128_sub(dst, src, writable!(dst), V128SubKind::F32x4)?; 4414 Ok(TypedReg::v128(dst)) 4415 }) 4416 } 4417 4418 fn visit_f64x2_sub(&mut self) -> Self::Output { 4419 self.context 4420 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 4421 masm.v128_sub(dst, src, writable!(dst), V128SubKind::F64x2)?; 4422 Ok(TypedReg::v128(dst)) 4423 }) 4424 } 4425 4426 fn visit_f32x4_mul(&mut self) -> Self::Output { 4427 self.masm.v128_mul(&mut self.context, V128MulKind::F32x4) 4428 } 4429 4430 fn visit_f64x2_mul(&mut self) -> Self::Output { 4431 self.masm.v128_mul(&mut self.context, V128MulKind::F64x2) 4432 } 4433 4434 fn visit_f32x4_div(&mut self) -> Self::Output { 4435 self.context 4436 .binop(self.masm, OperandSize::S32, |masm, dst, src, size| { 4437 masm.v128_div(dst, src, writable!(dst), size)?; 4438 Ok(TypedReg::v128(dst)) 4439 }) 4440 } 4441 4442 fn visit_f64x2_div(&mut self) -> Self::Output { 4443 self.context 4444 .binop(self.masm, OperandSize::S64, |masm, dst, src, size| { 4445 masm.v128_div(dst, src, writable!(dst), size)?; 4446 Ok(TypedReg::v128(dst)) 4447 }) 4448 } 4449 4450 fn visit_f32x4_neg(&mut self) -> Self::Output { 4451 self.context.unop(self.masm, |masm, reg| { 4452 masm.v128_neg(writable!(reg), V128NegKind::F32x4)?; 4453 Ok(TypedReg::v128(reg)) 4454 }) 4455 } 4456 4457 fn visit_f32x4_ceil(&mut self) -> Self::Output { 4458 self.context.unop(self.masm, |masm, reg| { 4459 masm.v128_ceil(reg, writable!(reg), OperandSize::S32)?; 4460 Ok(TypedReg::v128(reg)) 4461 }) 4462 } 4463 4464 fn visit_f64x2_neg(&mut self) -> Self::Output { 4465 self.context.unop(self.masm, |masm, reg| { 4466 masm.v128_neg(writable!(reg), V128NegKind::F64x2)?; 4467 Ok(TypedReg::v128(reg)) 4468 }) 4469 } 4470 4471 fn visit_f64x2_ceil(&mut self) -> Self::Output { 4472 self.context.unop(self.masm, |masm, reg| { 4473 masm.v128_ceil(reg, writable!(reg), OperandSize::S64)?; 4474 Ok(TypedReg::v128(reg)) 4475 }) 4476 } 4477 4478 fn visit_f32x4_sqrt(&mut self) -> Self::Output { 4479 self.context.unop(self.masm, |masm, reg| { 4480 masm.v128_sqrt(reg, writable!(reg), OperandSize::S32)?; 4481 Ok(TypedReg::v128(reg)) 4482 }) 4483 } 4484 4485 fn visit_f32x4_floor(&mut self) -> Self::Output { 4486 self.context.unop(self.masm, |masm, reg| { 4487 masm.v128_floor(reg, writable!(reg), OperandSize::S32)?; 4488 Ok(TypedReg::v128(reg)) 4489 }) 4490 } 4491 4492 fn visit_f64x2_sqrt(&mut self) -> Self::Output { 4493 self.context.unop(self.masm, |masm, reg| { 4494 masm.v128_sqrt(reg, writable!(reg), OperandSize::S64)?; 4495 Ok(TypedReg::v128(reg)) 4496 }) 4497 } 4498 4499 fn visit_f64x2_floor(&mut self) -> Self::Output { 4500 self.context.unop(self.masm, |masm, reg| { 4501 masm.v128_floor(reg, writable!(reg), OperandSize::S64)?; 4502 Ok(TypedReg::v128(reg)) 4503 }) 4504 } 4505 4506 fn visit_f32x4_nearest(&mut self) -> Self::Output { 4507 self.context.unop(self.masm, |masm, reg| { 4508 masm.v128_nearest(reg, writable!(reg), OperandSize::S32)?; 4509 Ok(TypedReg::v128(reg)) 4510 }) 4511 } 4512 4513 fn visit_f64x2_nearest(&mut self) -> Self::Output { 4514 self.context.unop(self.masm, |masm, reg| { 4515 masm.v128_nearest(reg, writable!(reg), OperandSize::S64)?; 4516 Ok(TypedReg::v128(reg)) 4517 }) 4518 } 4519 4520 fn visit_f32x4_trunc(&mut self) -> Self::Output { 4521 self.masm 4522 .v128_trunc(&mut self.context, V128TruncKind::F32x4) 4523 } 4524 4525 fn visit_f64x2_trunc(&mut self) -> Self::Output { 4526 self.masm 4527 .v128_trunc(&mut self.context, V128TruncKind::F64x2) 4528 } 4529 4530 fn visit_v128_load32_zero(&mut self, memarg: MemArg) -> Self::Output { 4531 self.emit_wasm_load( 4532 &memarg, 4533 WasmValType::V128, 4534 LoadKind::VectorZero(OperandSize::S32), 4535 ) 4536 } 4537 4538 fn visit_v128_load64_zero(&mut self, memarg: MemArg) -> Self::Output { 4539 self.emit_wasm_load( 4540 &memarg, 4541 WasmValType::V128, 4542 LoadKind::VectorZero(OperandSize::S64), 4543 ) 4544 } 4545 4546 fn visit_f32x4_pmin(&mut self) -> Self::Output { 4547 self.context 4548 .binop(self.masm, OperandSize::S32, |masm, dst, src, size| { 4549 masm.v128_pmin(dst, src, writable!(dst), size)?; 4550 Ok(TypedReg::v128(dst)) 4551 }) 4552 } 4553 4554 fn visit_f64x2_pmin(&mut self) -> Self::Output { 4555 self.context 4556 .binop(self.masm, OperandSize::S64, |masm, dst, src, size| { 4557 masm.v128_pmin(dst, src, writable!(dst), size)?; 4558 Ok(TypedReg::v128(dst)) 4559 }) 4560 } 4561 4562 fn visit_f32x4_pmax(&mut self) -> Self::Output { 4563 self.context 4564 .binop(self.masm, OperandSize::S32, |masm, dst, src, size| { 4565 masm.v128_pmax(dst, src, writable!(dst), size)?; 4566 Ok(TypedReg::v128(dst)) 4567 }) 4568 } 4569 4570 fn visit_f64x2_pmax(&mut self) -> Self::Output { 4571 self.context 4572 .binop(self.masm, OperandSize::S64, |masm, dst, src, size| { 4573 masm.v128_pmax(dst, src, writable!(dst), size)?; 4574 Ok(TypedReg::v128(dst)) 4575 }) 4576 } 4577 4578 fn visit_f32x4_min(&mut self) -> Self::Output { 4579 self.context 4580 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4581 masm.v128_min(dst, src, writable!(dst), V128MinKind::F32x4)?; 4582 Ok(TypedReg::v128(dst)) 4583 }) 4584 } 4585 4586 fn visit_f64x2_min(&mut self) -> Self::Output { 4587 self.context 4588 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 4589 masm.v128_min(dst, src, writable!(dst), V128MinKind::F64x2)?; 4590 Ok(TypedReg::v128(dst)) 4591 }) 4592 } 4593 4594 fn visit_f32x4_max(&mut self) -> Self::Output { 4595 self.context 4596 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4597 masm.v128_max(dst, src, writable!(dst), V128MaxKind::F32x4)?; 4598 Ok(TypedReg::v128(dst)) 4599 }) 4600 } 4601 4602 fn visit_f64x2_max(&mut self) -> Self::Output { 4603 self.context 4604 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 4605 masm.v128_max(dst, src, writable!(dst), V128MaxKind::F64x2)?; 4606 Ok(TypedReg::v128(dst)) 4607 }) 4608 } 4609 4610 wasmparser::for_each_visit_simd_operator!(def_unsupported); 4611 } 4612 4613 impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Emission> 4614 where 4615 M: MacroAssembler, 4616 { 4617 fn cmp_i32s(&mut self, kind: IntCmpKind) -> Result<()> { 4618 self.context.i32_binop(self.masm, |masm, dst, src, size| { 4619 masm.cmp_with_set(writable!(dst), src, kind, size)?; 4620 Ok(TypedReg::i32(dst)) 4621 }) 4622 } 4623 4624 fn cmp_i64s(&mut self, kind: IntCmpKind) -> Result<()> { 4625 self.context 4626 .i64_binop(self.masm, move |masm, dst, src, size| { 4627 masm.cmp_with_set(writable!(dst), src, kind, size)?; 4628 Ok(TypedReg::i32(dst)) // Return value for comparisons is an `i32`. 4629 }) 4630 } 4631 } 4632 4633 impl TryFrom<WasmValType> for OperandSize { 4634 type Error = anyhow::Error; 4635 fn try_from(ty: WasmValType) -> Result<OperandSize> { 4636 let ty = match ty { 4637 WasmValType::I32 | WasmValType::F32 => OperandSize::S32, 4638 WasmValType::I64 | WasmValType::F64 => OperandSize::S64, 4639 WasmValType::V128 => OperandSize::S128, 4640 WasmValType::Ref(rt) => { 4641 match rt.heap_type { 4642 // TODO: Hardcoded size, assuming 64-bit support only. Once 4643 // Wasmtime supports 32-bit architectures, this will need 4644 // to be updated in such a way that the calculation of the 4645 // OperandSize will depend on the target's pointer size. 4646 WasmHeapType::Func => OperandSize::S64, 4647 WasmHeapType::Extern => OperandSize::S64, 4648 _ => bail!(CodeGenError::unsupported_wasm_type()), 4649 } 4650 } 4651 }; 4652 Ok(ty) 4653 } 4654 } 4655