1 //! This module is the central place for machine code emission. 2 //! It defines an implementation of wasmparser's Visitor trait 3 //! for `CodeGen`; which defines a visitor per op-code, 4 //! which validates and dispatches to the corresponding 5 //! machine code emitter. 6 7 use crate::abi::RetArea; 8 use crate::codegen::{ 9 Callee, CodeGen, CodeGenError, ConditionalBranch, ControlStackFrame, Emission, FnCall, 10 UnconditionalBranch, control_index, 11 }; 12 use crate::masm::{ 13 AtomicWaitKind, DivKind, Extend, ExtractLaneKind, FloatCmpKind, IntCmpKind, LoadKind, 14 MacroAssembler, MulWideKind, OperandSize, RegImm, RemKind, ReplaceLaneKind, RmwOp, 15 RoundingMode, SPOffset, ShiftKind, Signed, SplatKind, SplatLoadKind, StoreKind, TruncKind, 16 V128AbsKind, V128AddKind, V128ConvertKind, V128ExtAddKind, V128ExtMulKind, V128ExtendKind, 17 V128LoadExtendKind, V128MaxKind, V128MinKind, V128MulKind, V128NarrowKind, V128NegKind, 18 V128SubKind, V128TruncKind, VectorCompareKind, VectorEqualityKind, Zero, 19 }; 20 use crate::reg::{Reg, writable}; 21 use crate::stack::{TypedReg, Val}; 22 use crate::{Result, bail, ensure, format_err}; 23 use regalloc2::RegClass; 24 use smallvec::{SmallVec, smallvec}; 25 use wasmparser::{ 26 BlockType, BrTable, HeapType, Ieee32, Ieee64, MemArg, V128, ValType, VisitOperator, 27 VisitSimdOperator, 28 }; 29 use wasmtime_cranelift::TRAP_INDIRECT_CALL_TO_NULL; 30 use wasmtime_environ::{ 31 FUNCREF_INIT_BIT, FuncIndex, GlobalIndex, IndexType, MemoryIndex, TableIndex, TypeIndex, 32 WasmHeapType, WasmValType, 33 }; 34 35 /// A macro to define unsupported WebAssembly operators. 36 /// 37 /// This macro calls itself recursively; 38 /// 1. It no-ops when matching a supported operator. 39 /// 2. Defines the visitor function and panics when 40 /// matching an unsupported operator. 41 macro_rules! def_unsupported { 42 ($( @$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident $ann:tt)*) => { 43 $( 44 def_unsupported!( 45 emit 46 $op 47 48 fn $visit(&mut self $($(,$arg: $argty)*)?) -> Self::Output { 49 $($(let _ = $arg;)*)? 50 51 Err(format_err!(CodeGenError::unimplemented_wasm_instruction())) 52 } 53 ); 54 )* 55 }; 56 57 (emit I32Const $($rest:tt)*) => {}; 58 (emit I64Const $($rest:tt)*) => {}; 59 (emit F32Const $($rest:tt)*) => {}; 60 (emit F64Const $($rest:tt)*) => {}; 61 (emit V128Const $($rest:tt)*) => {}; 62 (emit F32Add $($rest:tt)*) => {}; 63 (emit F64Add $($rest:tt)*) => {}; 64 (emit F32Sub $($rest:tt)*) => {}; 65 (emit F64Sub $($rest:tt)*) => {}; 66 (emit F32Mul $($rest:tt)*) => {}; 67 (emit F64Mul $($rest:tt)*) => {}; 68 (emit F32Div $($rest:tt)*) => {}; 69 (emit F64Div $($rest:tt)*) => {}; 70 (emit F32Min $($rest:tt)*) => {}; 71 (emit F64Min $($rest:tt)*) => {}; 72 (emit F32Max $($rest:tt)*) => {}; 73 (emit F64Max $($rest:tt)*) => {}; 74 (emit F32Copysign $($rest:tt)*) => {}; 75 (emit F64Copysign $($rest:tt)*) => {}; 76 (emit F32Abs $($rest:tt)*) => {}; 77 (emit F64Abs $($rest:tt)*) => {}; 78 (emit F32Neg $($rest:tt)*) => {}; 79 (emit F64Neg $($rest:tt)*) => {}; 80 (emit F32Floor $($rest:tt)*) => {}; 81 (emit F64Floor $($rest:tt)*) => {}; 82 (emit F32Ceil $($rest:tt)*) => {}; 83 (emit F64Ceil $($rest:tt)*) => {}; 84 (emit F32Nearest $($rest:tt)*) => {}; 85 (emit F64Nearest $($rest:tt)*) => {}; 86 (emit F32Trunc $($rest:tt)*) => {}; 87 (emit F64Trunc $($rest:tt)*) => {}; 88 (emit F32Sqrt $($rest:tt)*) => {}; 89 (emit F64Sqrt $($rest:tt)*) => {}; 90 (emit F32Eq $($rest:tt)*) => {}; 91 (emit F64Eq $($rest:tt)*) => {}; 92 (emit F32Ne $($rest:tt)*) => {}; 93 (emit F64Ne $($rest:tt)*) => {}; 94 (emit F32Lt $($rest:tt)*) => {}; 95 (emit F64Lt $($rest:tt)*) => {}; 96 (emit F32Gt $($rest:tt)*) => {}; 97 (emit F64Gt $($rest:tt)*) => {}; 98 (emit F32Le $($rest:tt)*) => {}; 99 (emit F64Le $($rest:tt)*) => {}; 100 (emit F32Ge $($rest:tt)*) => {}; 101 (emit F64Ge $($rest:tt)*) => {}; 102 (emit F32ConvertI32S $($rest:tt)*) => {}; 103 (emit F32ConvertI32U $($rest:tt)*) => {}; 104 (emit F32ConvertI64S $($rest:tt)*) => {}; 105 (emit F32ConvertI64U $($rest:tt)*) => {}; 106 (emit F64ConvertI32S $($rest:tt)*) => {}; 107 (emit F64ConvertI32U $($rest:tt)*) => {}; 108 (emit F64ConvertI64S $($rest:tt)*) => {}; 109 (emit F64ConvertI64U $($rest:tt)*) => {}; 110 (emit F32ReinterpretI32 $($rest:tt)*) => {}; 111 (emit F64ReinterpretI64 $($rest:tt)*) => {}; 112 (emit F32DemoteF64 $($rest:tt)*) => {}; 113 (emit F64PromoteF32 $($rest:tt)*) => {}; 114 (emit I32Add $($rest:tt)*) => {}; 115 (emit I64Add $($rest:tt)*) => {}; 116 (emit I32Sub $($rest:tt)*) => {}; 117 (emit I32Mul $($rest:tt)*) => {}; 118 (emit I32DivS $($rest:tt)*) => {}; 119 (emit I32DivU $($rest:tt)*) => {}; 120 (emit I64DivS $($rest:tt)*) => {}; 121 (emit I64DivU $($rest:tt)*) => {}; 122 (emit I64RemU $($rest:tt)*) => {}; 123 (emit I64RemS $($rest:tt)*) => {}; 124 (emit I32RemU $($rest:tt)*) => {}; 125 (emit I32RemS $($rest:tt)*) => {}; 126 (emit I64Mul $($rest:tt)*) => {}; 127 (emit I64Sub $($rest:tt)*) => {}; 128 (emit I32Eq $($rest:tt)*) => {}; 129 (emit I64Eq $($rest:tt)*) => {}; 130 (emit I32Ne $($rest:tt)*) => {}; 131 (emit I64Ne $($rest:tt)*) => {}; 132 (emit I32LtS $($rest:tt)*) => {}; 133 (emit I64LtS $($rest:tt)*) => {}; 134 (emit I32LtU $($rest:tt)*) => {}; 135 (emit I64LtU $($rest:tt)*) => {}; 136 (emit I32LeS $($rest:tt)*) => {}; 137 (emit I64LeS $($rest:tt)*) => {}; 138 (emit I32LeU $($rest:tt)*) => {}; 139 (emit I64LeU $($rest:tt)*) => {}; 140 (emit I32GtS $($rest:tt)*) => {}; 141 (emit I64GtS $($rest:tt)*) => {}; 142 (emit I32GtU $($rest:tt)*) => {}; 143 (emit I64GtU $($rest:tt)*) => {}; 144 (emit I32GeS $($rest:tt)*) => {}; 145 (emit I64GeS $($rest:tt)*) => {}; 146 (emit I32GeU $($rest:tt)*) => {}; 147 (emit I64GeU $($rest:tt)*) => {}; 148 (emit I32Eqz $($rest:tt)*) => {}; 149 (emit I64Eqz $($rest:tt)*) => {}; 150 (emit I32And $($rest:tt)*) => {}; 151 (emit I64And $($rest:tt)*) => {}; 152 (emit I32Or $($rest:tt)*) => {}; 153 (emit I64Or $($rest:tt)*) => {}; 154 (emit I32Xor $($rest:tt)*) => {}; 155 (emit I64Xor $($rest:tt)*) => {}; 156 (emit I32Shl $($rest:tt)*) => {}; 157 (emit I64Shl $($rest:tt)*) => {}; 158 (emit I32ShrS $($rest:tt)*) => {}; 159 (emit I64ShrS $($rest:tt)*) => {}; 160 (emit I32ShrU $($rest:tt)*) => {}; 161 (emit I64ShrU $($rest:tt)*) => {}; 162 (emit I32Rotl $($rest:tt)*) => {}; 163 (emit I64Rotl $($rest:tt)*) => {}; 164 (emit I32Rotr $($rest:tt)*) => {}; 165 (emit I64Rotr $($rest:tt)*) => {}; 166 (emit I32Clz $($rest:tt)*) => {}; 167 (emit I64Clz $($rest:tt)*) => {}; 168 (emit I32Ctz $($rest:tt)*) => {}; 169 (emit I64Ctz $($rest:tt)*) => {}; 170 (emit I32Popcnt $($rest:tt)*) => {}; 171 (emit I64Popcnt $($rest:tt)*) => {}; 172 (emit I32WrapI64 $($rest:tt)*) => {}; 173 (emit I64ExtendI32S $($rest:tt)*) => {}; 174 (emit I64ExtendI32U $($rest:tt)*) => {}; 175 (emit I32Extend8S $($rest:tt)*) => {}; 176 (emit I32Extend16S $($rest:tt)*) => {}; 177 (emit I64Extend8S $($rest:tt)*) => {}; 178 (emit I64Extend16S $($rest:tt)*) => {}; 179 (emit I64Extend32S $($rest:tt)*) => {}; 180 (emit I32TruncF32S $($rest:tt)*) => {}; 181 (emit I32TruncF32U $($rest:tt)*) => {}; 182 (emit I32TruncF64S $($rest:tt)*) => {}; 183 (emit I32TruncF64U $($rest:tt)*) => {}; 184 (emit I64TruncF32S $($rest:tt)*) => {}; 185 (emit I64TruncF32U $($rest:tt)*) => {}; 186 (emit I64TruncF64S $($rest:tt)*) => {}; 187 (emit I64TruncF64U $($rest:tt)*) => {}; 188 (emit I32ReinterpretF32 $($rest:tt)*) => {}; 189 (emit I64ReinterpretF64 $($rest:tt)*) => {}; 190 (emit LocalGet $($rest:tt)*) => {}; 191 (emit LocalSet $($rest:tt)*) => {}; 192 (emit Call $($rest:tt)*) => {}; 193 (emit End $($rest:tt)*) => {}; 194 (emit Nop $($rest:tt)*) => {}; 195 (emit If $($rest:tt)*) => {}; 196 (emit Else $($rest:tt)*) => {}; 197 (emit Block $($rest:tt)*) => {}; 198 (emit Loop $($rest:tt)*) => {}; 199 (emit Br $($rest:tt)*) => {}; 200 (emit BrIf $($rest:tt)*) => {}; 201 (emit Return $($rest:tt)*) => {}; 202 (emit Unreachable $($rest:tt)*) => {}; 203 (emit LocalTee $($rest:tt)*) => {}; 204 (emit GlobalGet $($rest:tt)*) => {}; 205 (emit GlobalSet $($rest:tt)*) => {}; 206 (emit Select $($rest:tt)*) => {}; 207 (emit TypedSelect $($rest:tt)*) => {}; 208 (emit RefNull $($rest:tt)*) => {}; 209 (emit RefIsNull $($rest:tt)*) => {}; 210 (emit RefFunc $($rest:tt)*) => {}; 211 (emit Drop $($rest:tt)*) => {}; 212 (emit BrTable $($rest:tt)*) => {}; 213 (emit CallIndirect $($rest:tt)*) => {}; 214 (emit TableInit $($rest:tt)*) => {}; 215 (emit TableCopy $($rest:tt)*) => {}; 216 (emit TableGet $($rest:tt)*) => {}; 217 (emit TableSet $($rest:tt)*) => {}; 218 (emit TableGrow $($rest:tt)*) => {}; 219 (emit TableSize $($rest:tt)*) => {}; 220 (emit TableFill $($rest:tt)*) => {}; 221 (emit ElemDrop $($rest:tt)*) => {}; 222 (emit MemoryInit $($rest:tt)*) => {}; 223 (emit MemoryCopy $($rest:tt)*) => {}; 224 (emit DataDrop $($rest:tt)*) => {}; 225 (emit MemoryFill $($rest:tt)*) => {}; 226 (emit MemorySize $($rest:tt)*) => {}; 227 (emit MemoryGrow $($rest:tt)*) => {}; 228 (emit I32Load $($rest:tt)*) => {}; 229 (emit I32Load8S $($rest:tt)*) => {}; 230 (emit I32Load8U $($rest:tt)*) => {}; 231 (emit I32Load16S $($rest:tt)*) => {}; 232 (emit I32Load16U $($rest:tt)*) => {}; 233 (emit I64Load8S $($rest:tt)*) => {}; 234 (emit I64Load8U $($rest:tt)*) => {}; 235 (emit I64Load16S $($rest:tt)*) => {}; 236 (emit I64Load16U $($rest:tt)*) => {}; 237 (emit I64Load32S $($rest:tt)*) => {}; 238 (emit I64Load32U $($rest:tt)*) => {}; 239 (emit I64Load $($rest:tt)*) => {}; 240 (emit I32Store $($rest:tt)*) => {}; 241 (emit I32Store8 $($rest:tt)*) => {}; 242 (emit I32Store16 $($rest:tt)*) => {}; 243 (emit I64Store $($rest:tt)*) => {}; 244 (emit I64Store8 $($rest:tt)*) => {}; 245 (emit I64Store16 $($rest:tt)*) => {}; 246 (emit I64Store32 $($rest:tt)*) => {}; 247 (emit F32Load $($rest:tt)*) => {}; 248 (emit F32Store $($rest:tt)*) => {}; 249 (emit F64Load $($rest:tt)*) => {}; 250 (emit F64Store $($rest:tt)*) => {}; 251 (emit I32TruncSatF32S $($rest:tt)*) => {}; 252 (emit I32TruncSatF32U $($rest:tt)*) => {}; 253 (emit I32TruncSatF64S $($rest:tt)*) => {}; 254 (emit I32TruncSatF64U $($rest:tt)*) => {}; 255 (emit I64TruncSatF32S $($rest:tt)*) => {}; 256 (emit I64TruncSatF32U $($rest:tt)*) => {}; 257 (emit I64TruncSatF64S $($rest:tt)*) => {}; 258 (emit I64TruncSatF64U $($rest:tt)*) => {}; 259 (emit V128Load $($rest:tt)*) => {}; 260 (emit V128Store $($rest:tt)*) => {}; 261 (emit I64Add128 $($rest:tt)*) => {}; 262 (emit I64Sub128 $($rest:tt)*) => {}; 263 (emit I64MulWideS $($rest:tt)*) => {}; 264 (emit I64MulWideU $($rest:tt)*) => {}; 265 (emit I32AtomicLoad8U $($rest:tt)*) => {}; 266 (emit I32AtomicLoad16U $($rest:tt)*) => {}; 267 (emit I32AtomicLoad $($rest:tt)*) => {}; 268 (emit I64AtomicLoad8U $($rest:tt)*) => {}; 269 (emit I64AtomicLoad16U $($rest:tt)*) => {}; 270 (emit I64AtomicLoad32U $($rest:tt)*) => {}; 271 (emit I64AtomicLoad $($rest:tt)*) => {}; 272 (emit V128Load8x8S $($rest:tt)*) => {}; 273 (emit V128Load8x8U $($rest:tt)*) => {}; 274 (emit V128Load16x4S $($rest:tt)*) => {}; 275 (emit V128Load16x4U $($rest:tt)*) => {}; 276 (emit V128Load32x2S $($rest:tt)*) => {}; 277 (emit V128Load32x2U $($rest:tt)*) => {}; 278 (emit V128Load8Splat $($rest:tt)*) => {}; 279 (emit V128Load16Splat $($rest:tt)*) => {}; 280 (emit V128Load32Splat $($rest:tt)*) => {}; 281 (emit V128Load64Splat $($rest:tt)*) => {}; 282 (emit I8x16Splat $($rest:tt)*) => {}; 283 (emit I16x8Splat $($rest:tt)*) => {}; 284 (emit I32x4Splat $($rest:tt)*) => {}; 285 (emit I64x2Splat $($rest:tt)*) => {}; 286 (emit F32x4Splat $($rest:tt)*) => {}; 287 (emit F64x2Splat $($rest:tt)*) => {}; 288 (emit I32AtomicStore8 $($rest:tt)*) => {}; 289 (emit I32AtomicStore16 $($rest:tt)*) => {}; 290 (emit I32AtomicStore $($rest:tt)*) => {}; 291 (emit I64AtomicStore8 $($rest:tt)*) => {}; 292 (emit I64AtomicStore16 $($rest:tt)*) => {}; 293 (emit I64AtomicStore32 $($rest:tt)*) => {}; 294 (emit I64AtomicStore $($rest:tt)*) => {}; 295 (emit I32AtomicRmw8AddU $($rest:tt)*) => {}; 296 (emit I32AtomicRmw16AddU $($rest:tt)*) => {}; 297 (emit I32AtomicRmwAdd $($rest:tt)*) => {}; 298 (emit I64AtomicRmw8AddU $($rest:tt)*) => {}; 299 (emit I64AtomicRmw16AddU $($rest:tt)*) => {}; 300 (emit I64AtomicRmw32AddU $($rest:tt)*) => {}; 301 (emit I64AtomicRmwAdd $($rest:tt)*) => {}; 302 (emit I8x16Shuffle $($rest:tt)*) => {}; 303 (emit I8x16Swizzle $($rest:tt)*) => {}; 304 (emit I32AtomicRmw8SubU $($rest:tt)*) => {}; 305 (emit I32AtomicRmw16SubU $($rest:tt)*) => {}; 306 (emit I32AtomicRmwSub $($rest:tt)*) => {}; 307 (emit I64AtomicRmw8SubU $($rest:tt)*) => {}; 308 (emit I64AtomicRmw16SubU $($rest:tt)*) => {}; 309 (emit I64AtomicRmw32SubU $($rest:tt)*) => {}; 310 (emit I64AtomicRmwSub $($rest:tt)*) => {}; 311 (emit I32AtomicRmw8XchgU $($rest:tt)*) => {}; 312 (emit I32AtomicRmw16XchgU $($rest:tt)*) => {}; 313 (emit I32AtomicRmwXchg $($rest:tt)*) => {}; 314 (emit I64AtomicRmw8XchgU $($rest:tt)*) => {}; 315 (emit I64AtomicRmw16XchgU $($rest:tt)*) => {}; 316 (emit I64AtomicRmw32XchgU $($rest:tt)*) => {}; 317 (emit I64AtomicRmwXchg $($rest:tt)*) => {}; 318 (emit I8x16ExtractLaneS $($rest:tt)*) => {}; 319 (emit I8x16ExtractLaneU $($rest:tt)*) => {}; 320 (emit I16x8ExtractLaneS $($rest:tt)*) => {}; 321 (emit I16x8ExtractLaneU $($rest:tt)*) => {}; 322 (emit I32x4ExtractLane $($rest:tt)*) => {}; 323 (emit I64x2ExtractLane $($rest:tt)*) => {}; 324 (emit F32x4ExtractLane $($rest:tt)*) => {}; 325 (emit F64x2ExtractLane $($rest:tt)*) => {}; 326 (emit I32AtomicRmw8AndU $($rest:tt)*) => {}; 327 (emit I32AtomicRmw16AndU $($rest:tt)*) => {}; 328 (emit I32AtomicRmwAnd $($rest:tt)*) => {}; 329 (emit I64AtomicRmw8AndU $($rest:tt)*) => {}; 330 (emit I64AtomicRmw16AndU $($rest:tt)*) => {}; 331 (emit I64AtomicRmw32AndU $($rest:tt)*) => {}; 332 (emit I64AtomicRmwAnd $($rest:tt)*) => {}; 333 (emit I32AtomicRmw8OrU $($rest:tt)*) => {}; 334 (emit I32AtomicRmw16OrU $($rest:tt)*) => {}; 335 (emit I32AtomicRmwOr $($rest:tt)*) => {}; 336 (emit I64AtomicRmw8OrU $($rest:tt)*) => {}; 337 (emit I64AtomicRmw16OrU $($rest:tt)*) => {}; 338 (emit I64AtomicRmw32OrU $($rest:tt)*) => {}; 339 (emit I64AtomicRmwOr $($rest:tt)*) => {}; 340 (emit I32AtomicRmw8XorU $($rest:tt)*) => {}; 341 (emit I32AtomicRmw16XorU $($rest:tt)*) => {}; 342 (emit I32AtomicRmwXor $($rest:tt)*) => {}; 343 (emit I64AtomicRmw8XorU $($rest:tt)*) => {}; 344 (emit I64AtomicRmw16XorU $($rest:tt)*) => {}; 345 (emit I64AtomicRmw32XorU $($rest:tt)*) => {}; 346 (emit I64AtomicRmwXor $($rest:tt)*) => {}; 347 (emit I8x16ReplaceLane $($rest:tt)*) => {}; 348 (emit I16x8ReplaceLane $($rest:tt)*) => {}; 349 (emit I32x4ReplaceLane $($rest:tt)*) => {}; 350 (emit I64x2ReplaceLane $($rest:tt)*) => {}; 351 (emit F32x4ReplaceLane $($rest:tt)*) => {}; 352 (emit F64x2ReplaceLane $($rest:tt)*) => {}; 353 (emit I32AtomicRmw8CmpxchgU $($rest:tt)*) => {}; 354 (emit I32AtomicRmw16CmpxchgU $($rest:tt)*) => {}; 355 (emit I32AtomicRmwCmpxchg $($rest:tt)*) => {}; 356 (emit I64AtomicRmw8CmpxchgU $($rest:tt)*) => {}; 357 (emit I64AtomicRmw16CmpxchgU $($rest:tt)*) => {}; 358 (emit I64AtomicRmw32CmpxchgU $($rest:tt)*) => {}; 359 (emit I64AtomicRmwCmpxchg $($rest:tt)*) => {}; 360 (emit I8x16Eq $($rest:tt)*) => {}; 361 (emit I16x8Eq $($rest:tt)*) => {}; 362 (emit I32x4Eq $($rest:tt)*) => {}; 363 (emit I64x2Eq $($rest:tt)*) => {}; 364 (emit F32x4Eq $($rest:tt)*) => {}; 365 (emit F64x2Eq $($rest:tt)*) => {}; 366 (emit I8x16Ne $($rest:tt)*) => {}; 367 (emit I16x8Ne $($rest:tt)*) => {}; 368 (emit I32x4Ne $($rest:tt)*) => {}; 369 (emit I64x2Ne $($rest:tt)*) => {}; 370 (emit F32x4Ne $($rest:tt)*) => {}; 371 (emit F64x2Ne $($rest:tt)*) => {}; 372 (emit I8x16LtS $($rest:tt)*) => {}; 373 (emit I8x16LtU $($rest:tt)*) => {}; 374 (emit I16x8LtS $($rest:tt)*) => {}; 375 (emit I16x8LtU $($rest:tt)*) => {}; 376 (emit I32x4LtS $($rest:tt)*) => {}; 377 (emit I32x4LtU $($rest:tt)*) => {}; 378 (emit I64x2LtS $($rest:tt)*) => {}; 379 (emit F32x4Lt $($rest:tt)*) => {}; 380 (emit F64x2Lt $($rest:tt)*) => {}; 381 (emit I8x16LeS $($rest:tt)*) => {}; 382 (emit I8x16LeU $($rest:tt)*) => {}; 383 (emit I16x8LeS $($rest:tt)*) => {}; 384 (emit I16x8LeU $($rest:tt)*) => {}; 385 (emit I32x4LeS $($rest:tt)*) => {}; 386 (emit I32x4LeU $($rest:tt)*) => {}; 387 (emit I64x2LeS $($rest:tt)*) => {}; 388 (emit F32x4Le $($rest:tt)*) => {}; 389 (emit F64x2Le $($rest:tt)*) => {}; 390 (emit I8x16GtS $($rest:tt)*) => {}; 391 (emit I8x16GtU $($rest:tt)*) => {}; 392 (emit I16x8GtS $($rest:tt)*) => {}; 393 (emit I16x8GtU $($rest:tt)*) => {}; 394 (emit I32x4GtS $($rest:tt)*) => {}; 395 (emit I32x4GtU $($rest:tt)*) => {}; 396 (emit I64x2GtS $($rest:tt)*) => {}; 397 (emit F32x4Gt $($rest:tt)*) => {}; 398 (emit F64x2Gt $($rest:tt)*) => {}; 399 (emit I8x16GeS $($rest:tt)*) => {}; 400 (emit I8x16GeU $($rest:tt)*) => {}; 401 (emit I16x8GeS $($rest:tt)*) => {}; 402 (emit I16x8GeU $($rest:tt)*) => {}; 403 (emit I32x4GeS $($rest:tt)*) => {}; 404 (emit I32x4GeU $($rest:tt)*) => {}; 405 (emit I64x2GeS $($rest:tt)*) => {}; 406 (emit F32x4Ge $($rest:tt)*) => {}; 407 (emit F64x2Ge $($rest:tt)*) => {}; 408 (emit MemoryAtomicWait32 $($rest:tt)*) => {}; 409 (emit MemoryAtomicWait64 $($rest:tt)*) => {}; 410 (emit MemoryAtomicNotify $($rest:tt)*) => {}; 411 (emit AtomicFence $($rest:tt)*) => {}; 412 (emit V128Not $($rest:tt)*) => {}; 413 (emit V128And $($rest:tt)*) => {}; 414 (emit V128AndNot $($rest:tt)*) => {}; 415 (emit V128Or $($rest:tt)*) => {}; 416 (emit V128Xor $($rest:tt)*) => {}; 417 (emit V128Bitselect $($rest:tt)*) => {}; 418 (emit V128AnyTrue $($rest:tt)*) => {}; 419 (emit V128Load8Lane $($rest:tt)*) => {}; 420 (emit V128Load16Lane $($rest:tt)*) => {}; 421 (emit V128Load32Lane $($rest:tt)*) => {}; 422 (emit V128Load64Lane $($rest:tt)*) => {}; 423 (emit V128Store8Lane $($rest:tt)*) => {}; 424 (emit V128Store16Lane $($rest:tt)*) => {}; 425 (emit V128Store32Lane $($rest:tt)*) => {}; 426 (emit V128Store64Lane $($rest:tt)*) => {}; 427 (emit F32x4ConvertI32x4S $($rest:tt)*) => {}; 428 (emit F32x4ConvertI32x4U $($rest:tt)*) => {}; 429 (emit F64x2ConvertLowI32x4S $($rest:tt)*) => {}; 430 (emit F64x2ConvertLowI32x4U $($rest:tt)*) => {}; 431 (emit I8x16NarrowI16x8S $($rest:tt)*) => {}; 432 (emit I8x16NarrowI16x8U $($rest:tt)*) => {}; 433 (emit I16x8NarrowI32x4S $($rest:tt)*) => {}; 434 (emit I16x8NarrowI32x4U $($rest:tt)*) => {}; 435 (emit F32x4DemoteF64x2Zero $($rest:tt)*) => {}; 436 (emit F64x2PromoteLowF32x4 $($rest:tt)*) => {}; 437 (emit I16x8ExtendLowI8x16S $($rest:tt)*) => {}; 438 (emit I16x8ExtendHighI8x16S $($rest:tt)*) => {}; 439 (emit I16x8ExtendLowI8x16U $($rest:tt)*) => {}; 440 (emit I16x8ExtendHighI8x16U $($rest:tt)*) => {}; 441 (emit I32x4ExtendLowI16x8S $($rest:tt)*) => {}; 442 (emit I32x4ExtendHighI16x8S $($rest:tt)*) => {}; 443 (emit I32x4ExtendLowI16x8U $($rest:tt)*) => {}; 444 (emit I32x4ExtendHighI16x8U $($rest:tt)*) => {}; 445 (emit I64x2ExtendLowI32x4S $($rest:tt)*) => {}; 446 (emit I64x2ExtendHighI32x4S $($rest:tt)*) => {}; 447 (emit I64x2ExtendLowI32x4U $($rest:tt)*) => {}; 448 (emit I64x2ExtendHighI32x4U $($rest:tt)*) => {}; 449 (emit I8x16Add $($rest:tt)*) => {}; 450 (emit I16x8Add $($rest:tt)*) => {}; 451 (emit I32x4Add $($rest:tt)*) => {}; 452 (emit I64x2Add $($rest:tt)*) => {}; 453 (emit I8x16Sub $($rest:tt)*) => {}; 454 (emit I16x8Sub $($rest:tt)*) => {}; 455 (emit I32x4Sub $($rest:tt)*) => {}; 456 (emit I64x2Sub $($rest:tt)*) => {}; 457 (emit I16x8Mul $($rest:tt)*) => {}; 458 (emit I32x4Mul $($rest:tt)*) => {}; 459 (emit I64x2Mul $($rest:tt)*) => {}; 460 (emit I8x16AddSatS $($rest:tt)*) => {}; 461 (emit I16x8AddSatS $($rest:tt)*) => {}; 462 (emit I8x16AddSatU $($rest:tt)*) => {}; 463 (emit I16x8AddSatU $($rest:tt)*) => {}; 464 (emit I8x16SubSatS $($rest:tt)*) => {}; 465 (emit I16x8SubSatS $($rest:tt)*) => {}; 466 (emit I8x16SubSatU $($rest:tt)*) => {}; 467 (emit I16x8SubSatU $($rest:tt)*) => {}; 468 (emit I8x16Abs $($rest:tt)*) => {}; 469 (emit I16x8Abs $($rest:tt)*) => {}; 470 (emit I32x4Abs $($rest:tt)*) => {}; 471 (emit I64x2Abs $($rest:tt)*) => {}; 472 (emit F32x4Abs $($rest:tt)*) => {}; 473 (emit F64x2Abs $($rest:tt)*) => {}; 474 (emit I8x16Neg $($rest:tt)*) => {}; 475 (emit I16x8Neg $($rest:tt)*) => {}; 476 (emit I32x4Neg $($rest:tt)*) => {}; 477 (emit I64x2Neg $($rest:tt)*) => {}; 478 (emit I8x16Shl $($rest:tt)*) => {}; 479 (emit I16x8Shl $($rest:tt)*) => {}; 480 (emit I32x4Shl $($rest:tt)*) => {}; 481 (emit I64x2Shl $($rest:tt)*) => {}; 482 (emit I8x16ShrU $($rest:tt)*) => {}; 483 (emit I16x8ShrU $($rest:tt)*) => {}; 484 (emit I32x4ShrU $($rest:tt)*) => {}; 485 (emit I64x2ShrU $($rest:tt)*) => {}; 486 (emit I8x16ShrS $($rest:tt)*) => {}; 487 (emit I16x8ShrS $($rest:tt)*) => {}; 488 (emit I32x4ShrS $($rest:tt)*) => {}; 489 (emit I64x2ShrS $($rest:tt)*) => {}; 490 (emit I16x8Q15MulrSatS $($rest:tt)*) => {}; 491 (emit I8x16AllTrue $($rest:tt)*) => {}; 492 (emit I16x8AllTrue $($rest:tt)*) => {}; 493 (emit I32x4AllTrue $($rest:tt)*) => {}; 494 (emit I64x2AllTrue $($rest:tt)*) => {}; 495 (emit I8x16Bitmask $($rest:tt)*) => {}; 496 (emit I16x8Bitmask $($rest:tt)*) => {}; 497 (emit I32x4Bitmask $($rest:tt)*) => {}; 498 (emit I64x2Bitmask $($rest:tt)*) => {}; 499 (emit I32x4TruncSatF32x4S $($rest:tt)*) => {}; 500 (emit I32x4TruncSatF32x4U $($rest:tt)*) => {}; 501 (emit I32x4TruncSatF64x2SZero $($rest:tt)*) => {}; 502 (emit I32x4TruncSatF64x2UZero $($rest:tt)*) => {}; 503 (emit I8x16MinU $($rest:tt)*) => {}; 504 (emit I16x8MinU $($rest:tt)*) => {}; 505 (emit I32x4MinU $($rest:tt)*) => {}; 506 (emit I8x16MinS $($rest:tt)*) => {}; 507 (emit I16x8MinS $($rest:tt)*) => {}; 508 (emit I32x4MinS $($rest:tt)*) => {}; 509 (emit I8x16MaxU $($rest:tt)*) => {}; 510 (emit I16x8MaxU $($rest:tt)*) => {}; 511 (emit I32x4MaxU $($rest:tt)*) => {}; 512 (emit I8x16MaxS $($rest:tt)*) => {}; 513 (emit I16x8MaxS $($rest:tt)*) => {}; 514 (emit I32x4MaxS $($rest:tt)*) => {}; 515 (emit I16x8ExtMulLowI8x16S $($rest:tt)*) => {}; 516 (emit I32x4ExtMulLowI16x8S $($rest:tt)*) => {}; 517 (emit I64x2ExtMulLowI32x4S $($rest:tt)*) => {}; 518 (emit I16x8ExtMulHighI8x16S $($rest:tt)*) => {}; 519 (emit I32x4ExtMulHighI16x8S $($rest:tt)*) => {}; 520 (emit I64x2ExtMulHighI32x4S $($rest:tt)*) => {}; 521 (emit I16x8ExtMulLowI8x16U $($rest:tt)*) => {}; 522 (emit I32x4ExtMulLowI16x8U $($rest:tt)*) => {}; 523 (emit I64x2ExtMulLowI32x4U $($rest:tt)*) => {}; 524 (emit I16x8ExtMulHighI8x16U $($rest:tt)*) => {}; 525 (emit I32x4ExtMulHighI16x8U $($rest:tt)*) => {}; 526 (emit I64x2ExtMulHighI32x4U $($rest:tt)*) => {}; 527 (emit I16x8ExtAddPairwiseI8x16U $($rest:tt)*) => {}; 528 (emit I16x8ExtAddPairwiseI8x16S $($rest:tt)*) => {}; 529 (emit I32x4ExtAddPairwiseI16x8U $($rest:tt)*) => {}; 530 (emit I32x4ExtAddPairwiseI16x8S $($rest:tt)*) => {}; 531 (emit I32x4DotI16x8S $($rest:tt)*) => {}; 532 (emit I8x16Popcnt $($rest:tt)*) => {}; 533 (emit I8x16AvgrU $($rest:tt)*) => {}; 534 (emit I16x8AvgrU $($rest:tt)*) => {}; 535 (emit F32x4Add $($rest:tt)*) => {}; 536 (emit F64x2Add $($rest:tt)*) => {}; 537 (emit F32x4Sub $($rest:tt)*) => {}; 538 (emit F64x2Sub $($rest:tt)*) => {}; 539 (emit F32x4Mul $($rest:tt)*) => {}; 540 (emit F64x2Mul $($rest:tt)*) => {}; 541 (emit F32x4Div $($rest:tt)*) => {}; 542 (emit F64x2Div $($rest:tt)*) => {}; 543 (emit F32x4Neg $($rest:tt)*) => {}; 544 (emit F64x2Neg $($rest:tt)*) => {}; 545 (emit F32x4Sqrt $($rest:tt)*) => {}; 546 (emit F64x2Sqrt $($rest:tt)*) => {}; 547 (emit F32x4Ceil $($rest:tt)*) => {}; 548 (emit F64x2Ceil $($rest:tt)*) => {}; 549 (emit F32x4Floor $($rest:tt)*) => {}; 550 (emit F64x2Floor $($rest:tt)*) => {}; 551 (emit F32x4Nearest $($rest:tt)*) => {}; 552 (emit F64x2Nearest $($rest:tt)*) => {}; 553 (emit F32x4Trunc $($rest:tt)*) => {}; 554 (emit F64x2Trunc $($rest:tt)*) => {}; 555 (emit V128Load32Zero $($rest:tt)*) => {}; 556 (emit V128Load64Zero $($rest:tt)*) => {}; 557 (emit F32x4PMin $($rest:tt)*) => {}; 558 (emit F64x2PMin $($rest:tt)*) => {}; 559 (emit F32x4PMax $($rest:tt)*) => {}; 560 (emit F64x2PMax $($rest:tt)*) => {}; 561 (emit F32x4Min $($rest:tt)*) => {}; 562 (emit F64x2Min $($rest:tt)*) => {}; 563 (emit F32x4Max $($rest:tt)*) => {}; 564 (emit F64x2Max $($rest:tt)*) => {}; 565 566 (emit $unsupported:tt $($rest:tt)*) => {$($rest)*}; 567 } 568 569 impl<'a, 'translation, 'data, M> VisitOperator<'a> for CodeGen<'a, 'translation, 'data, M, Emission> 570 where 571 M: MacroAssembler, 572 { 573 type Output = Result<()>; 574 visit_i32_const(&mut self, val: i32) -> Self::Output575 fn visit_i32_const(&mut self, val: i32) -> Self::Output { 576 self.context.stack.push(Val::i32(val)); 577 578 Ok(()) 579 } 580 visit_i64_const(&mut self, val: i64) -> Self::Output581 fn visit_i64_const(&mut self, val: i64) -> Self::Output { 582 self.context.stack.push(Val::i64(val)); 583 Ok(()) 584 } 585 visit_f32_const(&mut self, val: Ieee32) -> Self::Output586 fn visit_f32_const(&mut self, val: Ieee32) -> Self::Output { 587 self.context.stack.push(Val::f32(val)); 588 Ok(()) 589 } 590 visit_f64_const(&mut self, val: Ieee64) -> Self::Output591 fn visit_f64_const(&mut self, val: Ieee64) -> Self::Output { 592 self.context.stack.push(Val::f64(val)); 593 Ok(()) 594 } 595 visit_f32_add(&mut self) -> Self::Output596 fn visit_f32_add(&mut self) -> Self::Output { 597 self.context.binop( 598 self.masm, 599 OperandSize::S32, 600 &mut |masm: &mut M, dst, src, size| { 601 masm.float_add(writable!(dst), dst, src, size)?; 602 Ok(TypedReg::f32(dst)) 603 }, 604 ) 605 } 606 visit_f64_add(&mut self) -> Self::Output607 fn visit_f64_add(&mut self) -> Self::Output { 608 self.context.binop( 609 self.masm, 610 OperandSize::S64, 611 &mut |masm: &mut M, dst, src, size| { 612 masm.float_add(writable!(dst), dst, src, size)?; 613 Ok(TypedReg::f64(dst)) 614 }, 615 ) 616 } 617 visit_f32_sub(&mut self) -> Self::Output618 fn visit_f32_sub(&mut self) -> Self::Output { 619 self.context.binop( 620 self.masm, 621 OperandSize::S32, 622 &mut |masm: &mut M, dst, src, size| { 623 masm.float_sub(writable!(dst), dst, src, size)?; 624 Ok(TypedReg::f32(dst)) 625 }, 626 ) 627 } 628 visit_f64_sub(&mut self) -> Self::Output629 fn visit_f64_sub(&mut self) -> Self::Output { 630 self.context.binop( 631 self.masm, 632 OperandSize::S64, 633 &mut |masm: &mut M, dst, src, size| { 634 masm.float_sub(writable!(dst), dst, src, size)?; 635 Ok(TypedReg::f64(dst)) 636 }, 637 ) 638 } 639 visit_f32_mul(&mut self) -> Self::Output640 fn visit_f32_mul(&mut self) -> Self::Output { 641 self.context.binop( 642 self.masm, 643 OperandSize::S32, 644 &mut |masm: &mut M, dst, src, size| { 645 masm.float_mul(writable!(dst), dst, src, size)?; 646 Ok(TypedReg::f32(dst)) 647 }, 648 ) 649 } 650 visit_f64_mul(&mut self) -> Self::Output651 fn visit_f64_mul(&mut self) -> Self::Output { 652 self.context.binop( 653 self.masm, 654 OperandSize::S64, 655 &mut |masm: &mut M, dst, src, size| { 656 masm.float_mul(writable!(dst), dst, src, size)?; 657 Ok(TypedReg::f64(dst)) 658 }, 659 ) 660 } 661 visit_f32_div(&mut self) -> Self::Output662 fn visit_f32_div(&mut self) -> Self::Output { 663 self.context.binop( 664 self.masm, 665 OperandSize::S32, 666 &mut |masm: &mut M, dst, src, size| { 667 masm.float_div(writable!(dst), dst, src, size)?; 668 Ok(TypedReg::f32(dst)) 669 }, 670 ) 671 } 672 visit_f64_div(&mut self) -> Self::Output673 fn visit_f64_div(&mut self) -> Self::Output { 674 self.context.binop( 675 self.masm, 676 OperandSize::S64, 677 &mut |masm: &mut M, dst, src, size| { 678 masm.float_div(writable!(dst), dst, src, size)?; 679 Ok(TypedReg::f64(dst)) 680 }, 681 ) 682 } 683 visit_f32_min(&mut self) -> Self::Output684 fn visit_f32_min(&mut self) -> Self::Output { 685 self.context.binop( 686 self.masm, 687 OperandSize::S32, 688 &mut |masm: &mut M, dst, src, size| { 689 masm.float_min(writable!(dst), dst, src, size)?; 690 Ok(TypedReg::f32(dst)) 691 }, 692 ) 693 } 694 visit_f64_min(&mut self) -> Self::Output695 fn visit_f64_min(&mut self) -> Self::Output { 696 self.context.binop( 697 self.masm, 698 OperandSize::S64, 699 &mut |masm: &mut M, dst, src, size| { 700 masm.float_min(writable!(dst), dst, src, size)?; 701 Ok(TypedReg::f64(dst)) 702 }, 703 ) 704 } 705 visit_f32_max(&mut self) -> Self::Output706 fn visit_f32_max(&mut self) -> Self::Output { 707 self.context.binop( 708 self.masm, 709 OperandSize::S32, 710 &mut |masm: &mut M, dst, src, size| { 711 masm.float_max(writable!(dst), dst, src, size)?; 712 Ok(TypedReg::f32(dst)) 713 }, 714 ) 715 } 716 visit_f64_max(&mut self) -> Self::Output717 fn visit_f64_max(&mut self) -> Self::Output { 718 self.context.binop( 719 self.masm, 720 OperandSize::S64, 721 &mut |masm: &mut M, dst, src, size| { 722 masm.float_max(writable!(dst), dst, src, size)?; 723 Ok(TypedReg::f64(dst)) 724 }, 725 ) 726 } 727 visit_f32_copysign(&mut self) -> Self::Output728 fn visit_f32_copysign(&mut self) -> Self::Output { 729 self.context.binop( 730 self.masm, 731 OperandSize::S32, 732 &mut |masm: &mut M, dst, src, size| { 733 masm.float_copysign(writable!(dst), dst, src, size)?; 734 Ok(TypedReg::f32(dst)) 735 }, 736 ) 737 } 738 visit_f64_copysign(&mut self) -> Self::Output739 fn visit_f64_copysign(&mut self) -> Self::Output { 740 self.context.binop( 741 self.masm, 742 OperandSize::S64, 743 &mut |masm: &mut M, dst, src, size| { 744 masm.float_copysign(writable!(dst), dst, src, size)?; 745 Ok(TypedReg::f64(dst)) 746 }, 747 ) 748 } 749 visit_f32_abs(&mut self) -> Self::Output750 fn visit_f32_abs(&mut self) -> Self::Output { 751 self.context.unop(self.masm, |masm, reg| { 752 masm.float_abs(writable!(reg), OperandSize::S32)?; 753 Ok(TypedReg::f32(reg)) 754 }) 755 } 756 visit_f64_abs(&mut self) -> Self::Output757 fn visit_f64_abs(&mut self) -> Self::Output { 758 self.context.unop(self.masm, |masm, reg| { 759 masm.float_abs(writable!(reg), OperandSize::S64)?; 760 Ok(TypedReg::f64(reg)) 761 }) 762 } 763 visit_f32_neg(&mut self) -> Self::Output764 fn visit_f32_neg(&mut self) -> Self::Output { 765 self.context.unop(self.masm, |masm, reg| { 766 masm.float_neg(writable!(reg), OperandSize::S32)?; 767 Ok(TypedReg::f32(reg)) 768 }) 769 } 770 visit_f64_neg(&mut self) -> Self::Output771 fn visit_f64_neg(&mut self) -> Self::Output { 772 self.context.unop(self.masm, |masm, reg| { 773 masm.float_neg(writable!(reg), OperandSize::S64)?; 774 Ok(TypedReg::f64(reg)) 775 }) 776 } 777 visit_f32_floor(&mut self) -> Self::Output778 fn visit_f32_floor(&mut self) -> Self::Output { 779 self.masm.float_round( 780 RoundingMode::Down, 781 &mut self.env, 782 &mut self.context, 783 OperandSize::S32, 784 |env, cx, masm| { 785 let builtin = env.builtins.floor_f32::<M::ABI>()?; 786 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 787 }, 788 ) 789 } 790 visit_f64_floor(&mut self) -> Self::Output791 fn visit_f64_floor(&mut self) -> Self::Output { 792 self.masm.float_round( 793 RoundingMode::Down, 794 &mut self.env, 795 &mut self.context, 796 OperandSize::S64, 797 |env, cx, masm| { 798 let builtin = env.builtins.floor_f64::<M::ABI>()?; 799 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 800 }, 801 ) 802 } 803 visit_f32_ceil(&mut self) -> Self::Output804 fn visit_f32_ceil(&mut self) -> Self::Output { 805 self.masm.float_round( 806 RoundingMode::Up, 807 &mut self.env, 808 &mut self.context, 809 OperandSize::S32, 810 |env, cx, masm| { 811 let builtin = env.builtins.ceil_f32::<M::ABI>()?; 812 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 813 }, 814 ) 815 } 816 visit_f64_ceil(&mut self) -> Self::Output817 fn visit_f64_ceil(&mut self) -> Self::Output { 818 self.masm.float_round( 819 RoundingMode::Up, 820 &mut self.env, 821 &mut self.context, 822 OperandSize::S64, 823 |env, cx, masm| { 824 let builtin = env.builtins.ceil_f64::<M::ABI>()?; 825 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 826 }, 827 ) 828 } 829 visit_f32_nearest(&mut self) -> Self::Output830 fn visit_f32_nearest(&mut self) -> Self::Output { 831 self.masm.float_round( 832 RoundingMode::Nearest, 833 &mut self.env, 834 &mut self.context, 835 OperandSize::S32, 836 |env, cx, masm| { 837 let builtin = env.builtins.nearest_f32::<M::ABI>()?; 838 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 839 }, 840 ) 841 } 842 visit_f64_nearest(&mut self) -> Self::Output843 fn visit_f64_nearest(&mut self) -> Self::Output { 844 self.masm.float_round( 845 RoundingMode::Nearest, 846 &mut self.env, 847 &mut self.context, 848 OperandSize::S64, 849 |env, cx, masm| { 850 let builtin = env.builtins.nearest_f64::<M::ABI>()?; 851 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 852 }, 853 ) 854 } 855 visit_f32_trunc(&mut self) -> Self::Output856 fn visit_f32_trunc(&mut self) -> Self::Output { 857 self.masm.float_round( 858 RoundingMode::Zero, 859 &mut self.env, 860 &mut self.context, 861 OperandSize::S32, 862 |env, cx, masm| { 863 let builtin = env.builtins.trunc_f32::<M::ABI>()?; 864 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 865 }, 866 ) 867 } 868 visit_f64_trunc(&mut self) -> Self::Output869 fn visit_f64_trunc(&mut self) -> Self::Output { 870 self.masm.float_round( 871 RoundingMode::Zero, 872 &mut self.env, 873 &mut self.context, 874 OperandSize::S64, 875 |env, cx, masm| { 876 let builtin = env.builtins.trunc_f64::<M::ABI>()?; 877 FnCall::emit::<M>(env, masm, cx, Callee::Builtin(builtin)) 878 }, 879 ) 880 } 881 visit_f32_sqrt(&mut self) -> Self::Output882 fn visit_f32_sqrt(&mut self) -> Self::Output { 883 self.context.unop(self.masm, |masm, reg| { 884 masm.float_sqrt(writable!(reg), reg, OperandSize::S32)?; 885 Ok(TypedReg::f32(reg)) 886 }) 887 } 888 visit_f64_sqrt(&mut self) -> Self::Output889 fn visit_f64_sqrt(&mut self) -> Self::Output { 890 self.context.unop(self.masm, |masm, reg| { 891 masm.float_sqrt(writable!(reg), reg, OperandSize::S64)?; 892 Ok(TypedReg::f64(reg)) 893 }) 894 } 895 visit_f32_eq(&mut self) -> Self::Output896 fn visit_f32_eq(&mut self) -> Self::Output { 897 self.context.float_cmp_op( 898 self.masm, 899 OperandSize::S32, 900 &mut |masm: &mut M, dst, src1, src2, size| { 901 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Eq, size) 902 }, 903 ) 904 } 905 visit_f64_eq(&mut self) -> Self::Output906 fn visit_f64_eq(&mut self) -> Self::Output { 907 self.context.float_cmp_op( 908 self.masm, 909 OperandSize::S64, 910 &mut |masm: &mut M, dst, src1, src2, size| { 911 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Eq, size) 912 }, 913 ) 914 } 915 visit_f32_ne(&mut self) -> Self::Output916 fn visit_f32_ne(&mut self) -> Self::Output { 917 self.context.float_cmp_op( 918 self.masm, 919 OperandSize::S32, 920 &mut |masm: &mut M, dst, src1, src2, size| { 921 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ne, size) 922 }, 923 ) 924 } 925 visit_f64_ne(&mut self) -> Self::Output926 fn visit_f64_ne(&mut self) -> Self::Output { 927 self.context.float_cmp_op( 928 self.masm, 929 OperandSize::S64, 930 &mut |masm: &mut M, dst, src1, src2, size| { 931 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ne, size) 932 }, 933 ) 934 } 935 visit_f32_lt(&mut self) -> Self::Output936 fn visit_f32_lt(&mut self) -> Self::Output { 937 self.context.float_cmp_op( 938 self.masm, 939 OperandSize::S32, 940 &mut |masm: &mut M, dst, src1, src2, size| { 941 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Lt, size) 942 }, 943 ) 944 } 945 visit_f64_lt(&mut self) -> Self::Output946 fn visit_f64_lt(&mut self) -> Self::Output { 947 self.context.float_cmp_op( 948 self.masm, 949 OperandSize::S64, 950 &mut |masm: &mut M, dst, src1, src2, size| { 951 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Lt, size) 952 }, 953 ) 954 } 955 visit_f32_gt(&mut self) -> Self::Output956 fn visit_f32_gt(&mut self) -> Self::Output { 957 self.context.float_cmp_op( 958 self.masm, 959 OperandSize::S32, 960 &mut |masm: &mut M, dst, src1, src2, size| { 961 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Gt, size) 962 }, 963 ) 964 } 965 visit_f64_gt(&mut self) -> Self::Output966 fn visit_f64_gt(&mut self) -> Self::Output { 967 self.context.float_cmp_op( 968 self.masm, 969 OperandSize::S64, 970 &mut |masm: &mut M, dst, src1, src2, size| { 971 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Gt, size) 972 }, 973 ) 974 } 975 visit_f32_le(&mut self) -> Self::Output976 fn visit_f32_le(&mut self) -> Self::Output { 977 self.context.float_cmp_op( 978 self.masm, 979 OperandSize::S32, 980 &mut |masm: &mut M, dst, src1, src2, size| { 981 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Le, size) 982 }, 983 ) 984 } 985 visit_f64_le(&mut self) -> Self::Output986 fn visit_f64_le(&mut self) -> Self::Output { 987 self.context.float_cmp_op( 988 self.masm, 989 OperandSize::S64, 990 &mut |masm: &mut M, dst, src1, src2, size| { 991 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Le, size) 992 }, 993 ) 994 } 995 visit_f32_ge(&mut self) -> Self::Output996 fn visit_f32_ge(&mut self) -> Self::Output { 997 self.context.float_cmp_op( 998 self.masm, 999 OperandSize::S32, 1000 &mut |masm: &mut M, dst, src1, src2, size| { 1001 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ge, size) 1002 }, 1003 ) 1004 } 1005 visit_f64_ge(&mut self) -> Self::Output1006 fn visit_f64_ge(&mut self) -> Self::Output { 1007 self.context.float_cmp_op( 1008 self.masm, 1009 OperandSize::S64, 1010 &mut |masm: &mut M, dst, src1, src2, size| { 1011 masm.float_cmp_with_set(writable!(dst), src1, src2, FloatCmpKind::Ge, size) 1012 }, 1013 ) 1014 } 1015 visit_f32_convert_i32_s(&mut self) -> Self::Output1016 fn visit_f32_convert_i32_s(&mut self) -> Self::Output { 1017 self.context 1018 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 1019 masm.signed_convert(writable!(dst), src, OperandSize::S32, dst_size) 1020 }) 1021 } 1022 visit_f32_convert_i32_u(&mut self) -> Self::Output1023 fn visit_f32_convert_i32_u(&mut self) -> Self::Output { 1024 self.context.convert_op_with_tmp_reg( 1025 self.masm, 1026 WasmValType::F32, 1027 RegClass::Int, 1028 |masm, dst, src, tmp_gpr, dst_size| { 1029 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S32, dst_size) 1030 }, 1031 ) 1032 } 1033 visit_f32_convert_i64_s(&mut self) -> Self::Output1034 fn visit_f32_convert_i64_s(&mut self) -> Self::Output { 1035 self.context 1036 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, dst_size| { 1037 masm.signed_convert(writable!(dst), src, OperandSize::S64, dst_size) 1038 }) 1039 } 1040 visit_f32_convert_i64_u(&mut self) -> Self::Output1041 fn visit_f32_convert_i64_u(&mut self) -> Self::Output { 1042 self.context.convert_op_with_tmp_reg( 1043 self.masm, 1044 WasmValType::F32, 1045 RegClass::Int, 1046 |masm, dst, src, tmp_gpr, dst_size| { 1047 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S64, dst_size) 1048 }, 1049 ) 1050 } 1051 visit_f64_convert_i32_s(&mut self) -> Self::Output1052 fn visit_f64_convert_i32_s(&mut self) -> Self::Output { 1053 self.context 1054 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 1055 masm.signed_convert(writable!(dst), src, OperandSize::S32, dst_size) 1056 }) 1057 } 1058 visit_f64_convert_i32_u(&mut self) -> Self::Output1059 fn visit_f64_convert_i32_u(&mut self) -> Self::Output { 1060 self.context.convert_op_with_tmp_reg( 1061 self.masm, 1062 WasmValType::F64, 1063 RegClass::Int, 1064 |masm, dst, src, tmp_gpr, dst_size| { 1065 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S32, dst_size) 1066 }, 1067 ) 1068 } 1069 visit_f64_convert_i64_s(&mut self) -> Self::Output1070 fn visit_f64_convert_i64_s(&mut self) -> Self::Output { 1071 self.context 1072 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, dst_size| { 1073 masm.signed_convert(writable!(dst), src, OperandSize::S64, dst_size) 1074 }) 1075 } 1076 visit_f64_convert_i64_u(&mut self) -> Self::Output1077 fn visit_f64_convert_i64_u(&mut self) -> Self::Output { 1078 self.context.convert_op_with_tmp_reg( 1079 self.masm, 1080 WasmValType::F64, 1081 RegClass::Int, 1082 |masm, dst, src, tmp_gpr, dst_size| { 1083 masm.unsigned_convert(writable!(dst), src, tmp_gpr, OperandSize::S64, dst_size) 1084 }, 1085 ) 1086 } 1087 visit_f32_reinterpret_i32(&mut self) -> Self::Output1088 fn visit_f32_reinterpret_i32(&mut self) -> Self::Output { 1089 self.context 1090 .convert_op(self.masm, WasmValType::F32, |masm, dst, src, size| { 1091 masm.reinterpret_int_as_float(writable!(dst), src, size) 1092 }) 1093 } 1094 visit_f64_reinterpret_i64(&mut self) -> Self::Output1095 fn visit_f64_reinterpret_i64(&mut self) -> Self::Output { 1096 self.context 1097 .convert_op(self.masm, WasmValType::F64, |masm, dst, src, size| { 1098 masm.reinterpret_int_as_float(writable!(dst), src, size) 1099 }) 1100 } 1101 visit_f32_demote_f64(&mut self) -> Self::Output1102 fn visit_f32_demote_f64(&mut self) -> Self::Output { 1103 self.context.unop(self.masm, |masm, reg| { 1104 masm.demote(writable!(reg), reg)?; 1105 Ok(TypedReg::f32(reg)) 1106 }) 1107 } 1108 visit_f64_promote_f32(&mut self) -> Self::Output1109 fn visit_f64_promote_f32(&mut self) -> Self::Output { 1110 self.context.unop(self.masm, |masm, reg| { 1111 masm.promote(writable!(reg), reg)?; 1112 Ok(TypedReg::f64(reg)) 1113 }) 1114 } 1115 visit_i32_add(&mut self) -> Self::Output1116 fn visit_i32_add(&mut self) -> Self::Output { 1117 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1118 masm.add(writable!(dst), dst, src, size)?; 1119 Ok(TypedReg::i32(dst)) 1120 }) 1121 } 1122 visit_i64_add(&mut self) -> Self::Output1123 fn visit_i64_add(&mut self) -> Self::Output { 1124 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1125 masm.add(writable!(dst), dst, src, size)?; 1126 Ok(TypedReg::i64(dst)) 1127 }) 1128 } 1129 visit_i32_sub(&mut self) -> Self::Output1130 fn visit_i32_sub(&mut self) -> Self::Output { 1131 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1132 masm.sub(writable!(dst), dst, src, size)?; 1133 Ok(TypedReg::i32(dst)) 1134 }) 1135 } 1136 visit_i64_sub(&mut self) -> Self::Output1137 fn visit_i64_sub(&mut self) -> Self::Output { 1138 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1139 masm.sub(writable!(dst), dst, src, size)?; 1140 Ok(TypedReg::i64(dst)) 1141 }) 1142 } 1143 visit_i32_mul(&mut self) -> Self::Output1144 fn visit_i32_mul(&mut self) -> Self::Output { 1145 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1146 masm.mul(writable!(dst), dst, src, size)?; 1147 Ok(TypedReg::i32(dst)) 1148 }) 1149 } 1150 visit_i64_mul(&mut self) -> Self::Output1151 fn visit_i64_mul(&mut self) -> Self::Output { 1152 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1153 masm.mul(writable!(dst), dst, src, size)?; 1154 Ok(TypedReg::i64(dst)) 1155 }) 1156 } 1157 visit_i32_div_s(&mut self) -> Self::Output1158 fn visit_i32_div_s(&mut self) -> Self::Output { 1159 use DivKind::*; 1160 use OperandSize::*; 1161 1162 self.masm.div(&mut self.context, Signed, S32) 1163 } 1164 visit_i32_div_u(&mut self) -> Self::Output1165 fn visit_i32_div_u(&mut self) -> Self::Output { 1166 use DivKind::*; 1167 use OperandSize::*; 1168 1169 self.masm.div(&mut self.context, Unsigned, S32) 1170 } 1171 visit_i64_div_s(&mut self) -> Self::Output1172 fn visit_i64_div_s(&mut self) -> Self::Output { 1173 use DivKind::*; 1174 use OperandSize::*; 1175 1176 self.masm.div(&mut self.context, Signed, S64) 1177 } 1178 visit_i64_div_u(&mut self) -> Self::Output1179 fn visit_i64_div_u(&mut self) -> Self::Output { 1180 use DivKind::*; 1181 use OperandSize::*; 1182 1183 self.masm.div(&mut self.context, Unsigned, S64) 1184 } 1185 visit_i32_rem_s(&mut self) -> Self::Output1186 fn visit_i32_rem_s(&mut self) -> Self::Output { 1187 use OperandSize::*; 1188 use RemKind::*; 1189 1190 self.masm.rem(&mut self.context, Signed, S32) 1191 } 1192 visit_i32_rem_u(&mut self) -> Self::Output1193 fn visit_i32_rem_u(&mut self) -> Self::Output { 1194 use OperandSize::*; 1195 use RemKind::*; 1196 1197 self.masm.rem(&mut self.context, Unsigned, S32) 1198 } 1199 visit_i64_rem_s(&mut self) -> Self::Output1200 fn visit_i64_rem_s(&mut self) -> Self::Output { 1201 use OperandSize::*; 1202 use RemKind::*; 1203 1204 self.masm.rem(&mut self.context, Signed, S64) 1205 } 1206 visit_i64_rem_u(&mut self) -> Self::Output1207 fn visit_i64_rem_u(&mut self) -> Self::Output { 1208 use OperandSize::*; 1209 use RemKind::*; 1210 1211 self.masm.rem(&mut self.context, Unsigned, S64) 1212 } 1213 visit_i32_eq(&mut self) -> Self::Output1214 fn visit_i32_eq(&mut self) -> Self::Output { 1215 self.cmp_i32s(IntCmpKind::Eq) 1216 } 1217 visit_i64_eq(&mut self) -> Self::Output1218 fn visit_i64_eq(&mut self) -> Self::Output { 1219 self.cmp_i64s(IntCmpKind::Eq) 1220 } 1221 visit_i32_ne(&mut self) -> Self::Output1222 fn visit_i32_ne(&mut self) -> Self::Output { 1223 self.cmp_i32s(IntCmpKind::Ne) 1224 } 1225 visit_i64_ne(&mut self) -> Self::Output1226 fn visit_i64_ne(&mut self) -> Self::Output { 1227 self.cmp_i64s(IntCmpKind::Ne) 1228 } 1229 visit_i32_lt_s(&mut self) -> Self::Output1230 fn visit_i32_lt_s(&mut self) -> Self::Output { 1231 self.cmp_i32s(IntCmpKind::LtS) 1232 } 1233 visit_i64_lt_s(&mut self) -> Self::Output1234 fn visit_i64_lt_s(&mut self) -> Self::Output { 1235 self.cmp_i64s(IntCmpKind::LtS) 1236 } 1237 visit_i32_lt_u(&mut self) -> Self::Output1238 fn visit_i32_lt_u(&mut self) -> Self::Output { 1239 self.cmp_i32s(IntCmpKind::LtU) 1240 } 1241 visit_i64_lt_u(&mut self) -> Self::Output1242 fn visit_i64_lt_u(&mut self) -> Self::Output { 1243 self.cmp_i64s(IntCmpKind::LtU) 1244 } 1245 visit_i32_le_s(&mut self) -> Self::Output1246 fn visit_i32_le_s(&mut self) -> Self::Output { 1247 self.cmp_i32s(IntCmpKind::LeS) 1248 } 1249 visit_i64_le_s(&mut self) -> Self::Output1250 fn visit_i64_le_s(&mut self) -> Self::Output { 1251 self.cmp_i64s(IntCmpKind::LeS) 1252 } 1253 visit_i32_le_u(&mut self) -> Self::Output1254 fn visit_i32_le_u(&mut self) -> Self::Output { 1255 self.cmp_i32s(IntCmpKind::LeU) 1256 } 1257 visit_i64_le_u(&mut self) -> Self::Output1258 fn visit_i64_le_u(&mut self) -> Self::Output { 1259 self.cmp_i64s(IntCmpKind::LeU) 1260 } 1261 visit_i32_gt_s(&mut self) -> Self::Output1262 fn visit_i32_gt_s(&mut self) -> Self::Output { 1263 self.cmp_i32s(IntCmpKind::GtS) 1264 } 1265 visit_i64_gt_s(&mut self) -> Self::Output1266 fn visit_i64_gt_s(&mut self) -> Self::Output { 1267 self.cmp_i64s(IntCmpKind::GtS) 1268 } 1269 visit_i32_gt_u(&mut self) -> Self::Output1270 fn visit_i32_gt_u(&mut self) -> Self::Output { 1271 self.cmp_i32s(IntCmpKind::GtU) 1272 } 1273 visit_i64_gt_u(&mut self) -> Self::Output1274 fn visit_i64_gt_u(&mut self) -> Self::Output { 1275 self.cmp_i64s(IntCmpKind::GtU) 1276 } 1277 visit_i32_ge_s(&mut self) -> Self::Output1278 fn visit_i32_ge_s(&mut self) -> Self::Output { 1279 self.cmp_i32s(IntCmpKind::GeS) 1280 } 1281 visit_i64_ge_s(&mut self) -> Self::Output1282 fn visit_i64_ge_s(&mut self) -> Self::Output { 1283 self.cmp_i64s(IntCmpKind::GeS) 1284 } 1285 visit_i32_ge_u(&mut self) -> Self::Output1286 fn visit_i32_ge_u(&mut self) -> Self::Output { 1287 self.cmp_i32s(IntCmpKind::GeU) 1288 } 1289 visit_i64_ge_u(&mut self) -> Self::Output1290 fn visit_i64_ge_u(&mut self) -> Self::Output { 1291 self.cmp_i64s(IntCmpKind::GeU) 1292 } 1293 visit_i32_eqz(&mut self) -> Self::Output1294 fn visit_i32_eqz(&mut self) -> Self::Output { 1295 use OperandSize::*; 1296 1297 self.context.unop(self.masm, |masm, reg| { 1298 masm.cmp_with_set(writable!(reg), RegImm::i32(0), IntCmpKind::Eq, S32)?; 1299 Ok(TypedReg::i32(reg)) 1300 }) 1301 } 1302 visit_i64_eqz(&mut self) -> Self::Output1303 fn visit_i64_eqz(&mut self) -> Self::Output { 1304 use OperandSize::*; 1305 1306 self.context.unop(self.masm, |masm, reg| { 1307 masm.cmp_with_set(writable!(reg), RegImm::i64(0), IntCmpKind::Eq, S64)?; 1308 Ok(TypedReg::i32(reg)) // Return value for `i64.eqz` is an `i32`. 1309 }) 1310 } 1311 visit_i32_clz(&mut self) -> Self::Output1312 fn visit_i32_clz(&mut self) -> Self::Output { 1313 use OperandSize::*; 1314 1315 self.context.unop(self.masm, |masm, reg| { 1316 masm.clz(writable!(reg), reg, S32)?; 1317 Ok(TypedReg::i32(reg)) 1318 }) 1319 } 1320 visit_i64_clz(&mut self) -> Self::Output1321 fn visit_i64_clz(&mut self) -> Self::Output { 1322 use OperandSize::*; 1323 1324 self.context.unop(self.masm, |masm, reg| { 1325 masm.clz(writable!(reg), reg, S64)?; 1326 Ok(TypedReg::i64(reg)) 1327 }) 1328 } 1329 visit_i32_ctz(&mut self) -> Self::Output1330 fn visit_i32_ctz(&mut self) -> Self::Output { 1331 use OperandSize::*; 1332 1333 self.context.unop(self.masm, |masm, reg| { 1334 masm.ctz(writable!(reg), reg, S32)?; 1335 Ok(TypedReg::i32(reg)) 1336 }) 1337 } 1338 visit_i64_ctz(&mut self) -> Self::Output1339 fn visit_i64_ctz(&mut self) -> Self::Output { 1340 use OperandSize::*; 1341 1342 self.context.unop(self.masm, |masm, reg| { 1343 masm.ctz(writable!(reg), reg, S64)?; 1344 Ok(TypedReg::i64(reg)) 1345 }) 1346 } 1347 visit_i32_and(&mut self) -> Self::Output1348 fn visit_i32_and(&mut self) -> Self::Output { 1349 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1350 masm.and(writable!(dst), dst, src, size)?; 1351 Ok(TypedReg::i32(dst)) 1352 }) 1353 } 1354 visit_i64_and(&mut self) -> Self::Output1355 fn visit_i64_and(&mut self) -> Self::Output { 1356 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1357 masm.and(writable!(dst), dst, src, size)?; 1358 Ok(TypedReg::i64(dst)) 1359 }) 1360 } 1361 visit_i32_or(&mut self) -> Self::Output1362 fn visit_i32_or(&mut self) -> Self::Output { 1363 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1364 masm.or(writable!(dst), dst, src, size)?; 1365 Ok(TypedReg::i32(dst)) 1366 }) 1367 } 1368 visit_i64_or(&mut self) -> Self::Output1369 fn visit_i64_or(&mut self) -> Self::Output { 1370 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1371 masm.or(writable!(dst), dst, src, size)?; 1372 Ok(TypedReg::i64(dst)) 1373 }) 1374 } 1375 visit_i32_xor(&mut self) -> Self::Output1376 fn visit_i32_xor(&mut self) -> Self::Output { 1377 self.context.i32_binop(self.masm, |masm, dst, src, size| { 1378 masm.xor(writable!(dst), dst, src, size)?; 1379 Ok(TypedReg::i32(dst)) 1380 }) 1381 } 1382 visit_i64_xor(&mut self) -> Self::Output1383 fn visit_i64_xor(&mut self) -> Self::Output { 1384 self.context.i64_binop(self.masm, |masm, dst, src, size| { 1385 masm.xor(writable!(dst), dst, src, size)?; 1386 Ok(TypedReg::i64(dst)) 1387 }) 1388 } 1389 visit_i32_shl(&mut self) -> Self::Output1390 fn visit_i32_shl(&mut self) -> Self::Output { 1391 use ShiftKind::*; 1392 1393 self.context.i32_shift(self.masm, Shl) 1394 } 1395 visit_i64_shl(&mut self) -> Self::Output1396 fn visit_i64_shl(&mut self) -> Self::Output { 1397 use ShiftKind::*; 1398 1399 self.context.i64_shift(self.masm, Shl) 1400 } 1401 visit_i32_shr_s(&mut self) -> Self::Output1402 fn visit_i32_shr_s(&mut self) -> Self::Output { 1403 use ShiftKind::*; 1404 1405 self.context.i32_shift(self.masm, ShrS) 1406 } 1407 visit_i64_shr_s(&mut self) -> Self::Output1408 fn visit_i64_shr_s(&mut self) -> Self::Output { 1409 use ShiftKind::*; 1410 1411 self.context.i64_shift(self.masm, ShrS) 1412 } 1413 visit_i32_shr_u(&mut self) -> Self::Output1414 fn visit_i32_shr_u(&mut self) -> Self::Output { 1415 use ShiftKind::*; 1416 1417 self.context.i32_shift(self.masm, ShrU) 1418 } 1419 visit_i64_shr_u(&mut self) -> Self::Output1420 fn visit_i64_shr_u(&mut self) -> Self::Output { 1421 use ShiftKind::*; 1422 1423 self.context.i64_shift(self.masm, ShrU) 1424 } 1425 visit_i32_rotl(&mut self) -> Self::Output1426 fn visit_i32_rotl(&mut self) -> Self::Output { 1427 use ShiftKind::*; 1428 1429 self.context.i32_shift(self.masm, Rotl) 1430 } 1431 visit_i64_rotl(&mut self) -> Self::Output1432 fn visit_i64_rotl(&mut self) -> Self::Output { 1433 use ShiftKind::*; 1434 1435 self.context.i64_shift(self.masm, Rotl) 1436 } 1437 visit_i32_rotr(&mut self) -> Self::Output1438 fn visit_i32_rotr(&mut self) -> Self::Output { 1439 use ShiftKind::*; 1440 1441 self.context.i32_shift(self.masm, Rotr) 1442 } 1443 visit_i64_rotr(&mut self) -> Self::Output1444 fn visit_i64_rotr(&mut self) -> Self::Output { 1445 use ShiftKind::*; 1446 1447 self.context.i64_shift(self.masm, Rotr) 1448 } 1449 visit_end(&mut self) -> Self::Output1450 fn visit_end(&mut self) -> Self::Output { 1451 if !self.context.reachable { 1452 self.handle_unreachable_end() 1453 } else { 1454 let mut control = self.pop_control_frame()?; 1455 control.emit_end(self.masm, &mut self.context) 1456 } 1457 } 1458 visit_i32_popcnt(&mut self) -> Self::Output1459 fn visit_i32_popcnt(&mut self) -> Self::Output { 1460 use OperandSize::*; 1461 self.masm.popcnt(&mut self.context, S32) 1462 } 1463 visit_i64_popcnt(&mut self) -> Self::Output1464 fn visit_i64_popcnt(&mut self) -> Self::Output { 1465 use OperandSize::*; 1466 1467 self.masm.popcnt(&mut self.context, S64) 1468 } 1469 visit_i32_wrap_i64(&mut self) -> Self::Output1470 fn visit_i32_wrap_i64(&mut self) -> Self::Output { 1471 self.context.unop(self.masm, |masm, reg| { 1472 masm.wrap(writable!(reg), reg)?; 1473 Ok(TypedReg::i32(reg)) 1474 }) 1475 } 1476 visit_i64_extend_i32_s(&mut self) -> Self::Output1477 fn visit_i64_extend_i32_s(&mut self) -> Self::Output { 1478 self.context.unop(self.masm, |masm, reg| { 1479 masm.extend(writable!(reg), reg, Extend::<Signed>::I64Extend32.into())?; 1480 Ok(TypedReg::i64(reg)) 1481 }) 1482 } 1483 visit_i64_extend_i32_u(&mut self) -> Self::Output1484 fn visit_i64_extend_i32_u(&mut self) -> Self::Output { 1485 self.context.unop(self.masm, |masm, reg| { 1486 masm.extend(writable!(reg), reg, Extend::<Zero>::I64Extend32.into())?; 1487 Ok(TypedReg::i64(reg)) 1488 }) 1489 } 1490 visit_i32_extend8_s(&mut self) -> Self::Output1491 fn visit_i32_extend8_s(&mut self) -> Self::Output { 1492 self.context.unop(self.masm, |masm, reg| { 1493 masm.extend(writable!(reg), reg, Extend::<Signed>::I32Extend8.into())?; 1494 Ok(TypedReg::i32(reg)) 1495 }) 1496 } 1497 visit_i32_extend16_s(&mut self) -> Self::Output1498 fn visit_i32_extend16_s(&mut self) -> Self::Output { 1499 self.context.unop(self.masm, |masm, reg| { 1500 masm.extend(writable!(reg), reg, Extend::<Signed>::I32Extend16.into())?; 1501 Ok(TypedReg::i32(reg)) 1502 }) 1503 } 1504 visit_i64_extend8_s(&mut self) -> Self::Output1505 fn visit_i64_extend8_s(&mut self) -> Self::Output { 1506 self.context.unop(self.masm, |masm, reg| { 1507 masm.extend(writable!(reg), reg, Extend::<Signed>::I64Extend8.into())?; 1508 Ok(TypedReg::i64(reg)) 1509 }) 1510 } 1511 visit_i64_extend16_s(&mut self) -> Self::Output1512 fn visit_i64_extend16_s(&mut self) -> Self::Output { 1513 self.context.unop(self.masm, |masm, reg| { 1514 masm.extend(writable!(reg), reg, Extend::<Signed>::I64Extend16.into())?; 1515 Ok(TypedReg::i64(reg)) 1516 }) 1517 } 1518 visit_i64_extend32_s(&mut self) -> Self::Output1519 fn visit_i64_extend32_s(&mut self) -> Self::Output { 1520 self.context.unop(self.masm, |masm, reg| { 1521 masm.extend(writable!(reg), reg, Extend::<Signed>::I64Extend32.into())?; 1522 Ok(TypedReg::i64(reg)) 1523 }) 1524 } 1525 visit_i32_trunc_f32_s(&mut self) -> Self::Output1526 fn visit_i32_trunc_f32_s(&mut self) -> Self::Output { 1527 use OperandSize::*; 1528 1529 self.context 1530 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1531 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Unchecked) 1532 }) 1533 } 1534 visit_i32_trunc_f32_u(&mut self) -> Self::Output1535 fn visit_i32_trunc_f32_u(&mut self) -> Self::Output { 1536 use OperandSize::*; 1537 1538 self.masm 1539 .unsigned_truncate(&mut self.context, S32, S32, TruncKind::Unchecked) 1540 } 1541 visit_i32_trunc_f64_s(&mut self) -> Self::Output1542 fn visit_i32_trunc_f64_s(&mut self) -> Self::Output { 1543 use OperandSize::*; 1544 1545 self.context 1546 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 1547 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Unchecked) 1548 }) 1549 } 1550 visit_i32_trunc_f64_u(&mut self) -> Self::Output1551 fn visit_i32_trunc_f64_u(&mut self) -> Self::Output { 1552 use OperandSize::*; 1553 self.masm 1554 .unsigned_truncate(&mut self.context, S64, S32, TruncKind::Unchecked) 1555 } 1556 visit_i64_trunc_f32_s(&mut self) -> Self::Output1557 fn visit_i64_trunc_f32_s(&mut self) -> Self::Output { 1558 use OperandSize::*; 1559 1560 self.context 1561 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1562 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Unchecked) 1563 }) 1564 } 1565 visit_i64_trunc_f32_u(&mut self) -> Self::Output1566 fn visit_i64_trunc_f32_u(&mut self) -> Self::Output { 1567 use OperandSize::*; 1568 1569 self.masm 1570 .unsigned_truncate(&mut self.context, S32, S64, TruncKind::Unchecked) 1571 } 1572 visit_i64_trunc_f64_s(&mut self) -> Self::Output1573 fn visit_i64_trunc_f64_s(&mut self) -> Self::Output { 1574 use OperandSize::*; 1575 1576 self.context 1577 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 1578 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Unchecked) 1579 }) 1580 } 1581 visit_i64_trunc_f64_u(&mut self) -> Self::Output1582 fn visit_i64_trunc_f64_u(&mut self) -> Self::Output { 1583 use OperandSize::*; 1584 1585 self.masm 1586 .unsigned_truncate(&mut self.context, S64, S64, TruncKind::Unchecked) 1587 } 1588 visit_i32_reinterpret_f32(&mut self) -> Self::Output1589 fn visit_i32_reinterpret_f32(&mut self) -> Self::Output { 1590 self.context 1591 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, size| { 1592 masm.reinterpret_float_as_int(writable!(dst), src, size) 1593 }) 1594 } 1595 visit_i64_reinterpret_f64(&mut self) -> Self::Output1596 fn visit_i64_reinterpret_f64(&mut self) -> Self::Output { 1597 self.context 1598 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, size| { 1599 masm.reinterpret_float_as_int(writable!(dst), src, size) 1600 }) 1601 } 1602 visit_local_get(&mut self, index: u32) -> Self::Output1603 fn visit_local_get(&mut self, index: u32) -> Self::Output { 1604 use WasmValType::*; 1605 let context = &mut self.context; 1606 let slot = context.frame.get_wasm_local(index); 1607 match slot.ty { 1608 I32 | I64 | F32 | F64 | V128 => context.stack.push(Val::local(index, slot.ty)), 1609 Ref(rt) => match rt.heap_type { 1610 WasmHeapType::Func => context.stack.push(Val::local(index, slot.ty)), 1611 _ => bail!(CodeGenError::unsupported_wasm_type()), 1612 }, 1613 } 1614 1615 Ok(()) 1616 } 1617 visit_local_set(&mut self, index: u32) -> Self::Output1618 fn visit_local_set(&mut self, index: u32) -> Self::Output { 1619 let src = self.emit_set_local(index)?; 1620 self.context.free_reg(src); 1621 Ok(()) 1622 } 1623 visit_call(&mut self, index: u32) -> Self::Output1624 fn visit_call(&mut self, index: u32) -> Self::Output { 1625 let callee = self.env.callee_from_index(FuncIndex::from_u32(index)); 1626 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee)?; 1627 Ok(()) 1628 } 1629 visit_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output1630 fn visit_call_indirect(&mut self, type_index: u32, table_index: u32) -> Self::Output { 1631 // Spill now because `emit_lazy_init_funcref` and the `FnCall::emit` 1632 // invocations will both trigger spills since they both call functions. 1633 // However, the machine instructions for the spill emitted by 1634 // `emit_lazy_funcref` will be jumped over if the funcref was previously 1635 // initialized which may result in the machine stack becoming 1636 // unbalanced. 1637 self.context.spill(self.masm)?; 1638 1639 let type_index = TypeIndex::from_u32(type_index); 1640 let table_index = TableIndex::from_u32(table_index); 1641 1642 self.emit_lazy_init_funcref(table_index)?; 1643 1644 // Perform the indirect call. 1645 // This code assumes that [`Self::emit_lazy_init_funcref`] will 1646 // push the funcref to the value stack. 1647 let funcref_ptr = self 1648 .context 1649 .stack 1650 .peek() 1651 .map(|v| v.unwrap_reg()) 1652 .ok_or_else(|| CodeGenError::missing_values_in_stack())?; 1653 self.masm 1654 .trapz(funcref_ptr.into(), TRAP_INDIRECT_CALL_TO_NULL)?; 1655 self.emit_typecheck_funcref(funcref_ptr.into(), type_index)?; 1656 1657 let callee = self.env.funcref(type_index); 1658 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee)?; 1659 Ok(()) 1660 } 1661 visit_table_init(&mut self, elem: u32, table: u32) -> Self::Output1662 fn visit_table_init(&mut self, elem: u32, table: u32) -> Self::Output { 1663 let at = self.context.stack.ensure_index_at(3)?; 1664 1665 self.context 1666 .stack 1667 .insert_many(at, &[table.try_into()?, elem.try_into()?]); 1668 1669 let builtin = self.env.builtins.table_init::<M::ABI>()?; 1670 FnCall::emit::<M>( 1671 &mut self.env, 1672 self.masm, 1673 &mut self.context, 1674 Callee::Builtin(builtin.clone()), 1675 )?; 1676 self.context.pop_and_free(self.masm) 1677 } 1678 visit_table_copy(&mut self, dst: u32, src: u32) -> Self::Output1679 fn visit_table_copy(&mut self, dst: u32, src: u32) -> Self::Output { 1680 let at = self.context.stack.ensure_index_at(3)?; 1681 self.context 1682 .stack 1683 .insert_many(at, &[dst.try_into()?, src.try_into()?]); 1684 1685 let builtin = self.env.builtins.table_copy::<M::ABI>()?; 1686 FnCall::emit::<M>( 1687 &mut self.env, 1688 self.masm, 1689 &mut self.context, 1690 Callee::Builtin(builtin), 1691 )?; 1692 self.context.pop_and_free(self.masm) 1693 } 1694 visit_table_get(&mut self, table: u32) -> Self::Output1695 fn visit_table_get(&mut self, table: u32) -> Self::Output { 1696 let table_index = TableIndex::from_u32(table); 1697 let table = self.env.table(table_index); 1698 let heap_type = table.ref_type.heap_type; 1699 1700 match heap_type { 1701 WasmHeapType::Func => self.emit_lazy_init_funcref(table_index), 1702 _ => Err(format_err!(CodeGenError::unsupported_wasm_type())), 1703 } 1704 } 1705 visit_table_grow(&mut self, table: u32) -> Self::Output1706 fn visit_table_grow(&mut self, table: u32) -> Self::Output { 1707 let table_index = TableIndex::from_u32(table); 1708 let ptr_type = self.env.ptr_type(); 1709 let (heap_type, idx_type) = { 1710 let table_ty = self.env.table(table_index); 1711 (table_ty.ref_type.heap_type, table_ty.idx_type) 1712 }; 1713 let builtin = match heap_type { 1714 WasmHeapType::Func => self.env.builtins.table_grow_func_ref::<M::ABI>()?, 1715 _ => bail!(CodeGenError::unsupported_wasm_type()), 1716 }; 1717 1718 let len = self.context.stack.len(); 1719 // table.grow` requires at least 2 elements on the value stack. 1720 let at = self.context.stack.ensure_index_at(2)?; 1721 1722 // The table_grow builtin expects the parameters in a different 1723 // order. 1724 // The value stack at this point should contain: 1725 // [ init_value | delta ] (stack top) 1726 // but the builtin function expects the init value as the last 1727 // argument. 1728 self.context.stack.inner_mut().swap(len - 1, len - 2); 1729 1730 let builtin = self.prepare_builtin_defined_table_arg(table_index, at, builtin)?; 1731 1732 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, builtin)?; 1733 1734 // Similar to the memory.grow builtin, `table.grow` returns a 1735 // pointer, however, we need to ensure that the returned index 1736 // is representative of the address space for tables. 1737 match (ptr_type, idx_type) { 1738 (WasmValType::I64, IndexType::I64) => Ok(()), 1739 (WasmValType::I64, IndexType::I32) => { 1740 let top: Reg = self.context.pop_to_reg(self.masm, None)?.into(); 1741 self.masm.wrap(writable!(top), top)?; 1742 self.context.stack.push(TypedReg::i32(top).into()); 1743 Ok(()) 1744 } 1745 1746 _ => Err(format_err!(CodeGenError::unsupported_32_bit_platform())), 1747 } 1748 } 1749 visit_table_size(&mut self, table: u32) -> Self::Output1750 fn visit_table_size(&mut self, table: u32) -> Self::Output { 1751 let table_index = TableIndex::from_u32(table); 1752 let table_data = self.env.resolve_table_data(table_index); 1753 self.emit_compute_table_size(&table_data) 1754 } 1755 visit_table_fill(&mut self, table: u32) -> Self::Output1756 fn visit_table_fill(&mut self, table: u32) -> Self::Output { 1757 let table_index = TableIndex::from_u32(table); 1758 let table_ty = self.env.table(table_index); 1759 1760 ensure!( 1761 table_ty.ref_type.heap_type == WasmHeapType::Func, 1762 CodeGenError::unsupported_wasm_type() 1763 ); 1764 1765 let builtin = self.env.builtins.table_fill_func_ref::<M::ABI>()?; 1766 1767 let at = self.context.stack.ensure_index_at(3)?; 1768 1769 let callee = self.prepare_builtin_defined_table_arg(table_index, at, builtin)?; 1770 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, callee)?; 1771 1772 self.context.pop_and_free(self.masm) 1773 } 1774 visit_table_set(&mut self, table: u32) -> Self::Output1775 fn visit_table_set(&mut self, table: u32) -> Self::Output { 1776 let ptr_type = self.env.ptr_type(); 1777 let table_index = TableIndex::from_u32(table); 1778 let table_data = self.env.resolve_table_data(table_index); 1779 let table = self.env.table(table_index); 1780 match table.ref_type.heap_type { 1781 WasmHeapType::Func => { 1782 ensure!( 1783 self.tunables.table_lazy_init, 1784 CodeGenError::unsupported_table_eager_init() 1785 ); 1786 let value = self.context.pop_to_reg(self.masm, None)?; 1787 let index = self.context.pop_to_reg(self.masm, None)?; 1788 let base = self.context.any_gpr(self.masm)?; 1789 let elem_addr = 1790 self.emit_compute_table_elem_addr(index.into(), base, &table_data)?; 1791 // Set the initialized bit. 1792 self.masm.or( 1793 writable!(value.into()), 1794 value.into(), 1795 RegImm::i64(FUNCREF_INIT_BIT as i64), 1796 ptr_type.try_into()?, 1797 )?; 1798 1799 self.masm.store_ptr(value.into(), elem_addr)?; 1800 1801 self.context.free_reg(value); 1802 self.context.free_reg(index); 1803 self.context.free_reg(base); 1804 Ok(()) 1805 } 1806 _ => Err(format_err!(CodeGenError::unsupported_wasm_type())), 1807 } 1808 } 1809 visit_elem_drop(&mut self, index: u32) -> Self::Output1810 fn visit_elem_drop(&mut self, index: u32) -> Self::Output { 1811 let elem_drop = self.env.builtins.elem_drop::<M::ABI>()?; 1812 self.context.stack.extend([index.try_into()?]); 1813 FnCall::emit::<M>( 1814 &mut self.env, 1815 self.masm, 1816 &mut self.context, 1817 Callee::Builtin(elem_drop), 1818 )?; 1819 self.context.pop_and_free(self.masm) 1820 } 1821 visit_memory_init(&mut self, data_index: u32, mem: u32) -> Self::Output1822 fn visit_memory_init(&mut self, data_index: u32, mem: u32) -> Self::Output { 1823 let at = self.context.stack.ensure_index_at(3)?; 1824 self.context 1825 .stack 1826 .insert_many(at, &[mem.try_into()?, data_index.try_into()?]); 1827 let builtin = self.env.builtins.memory_init::<M::ABI>()?; 1828 FnCall::emit::<M>( 1829 &mut self.env, 1830 self.masm, 1831 &mut self.context, 1832 Callee::Builtin(builtin), 1833 )?; 1834 self.context.pop_and_free(self.masm) 1835 } 1836 visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) -> Self::Output1837 fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) -> Self::Output { 1838 // At this point, the stack is expected to contain: 1839 // [ dst_offset, src_offset, len ] 1840 // The following code inserts the missing params, so that stack contains: 1841 // [ vmctx, dst_mem, dst_offset, src_mem, src_offset, len ] 1842 // Which is the order expected by the builtin function. 1843 let _ = self.context.stack.ensure_index_at(3)?; 1844 let at = self.context.stack.ensure_index_at(2)?; 1845 self.context.stack.insert_many(at, &[src_mem.try_into()?]); 1846 1847 // One element was inserted above, so instead of 3, we use 4. 1848 let at = self.context.stack.ensure_index_at(4)?; 1849 self.context.stack.insert_many(at, &[dst_mem.try_into()?]); 1850 1851 let builtin = self.env.builtins.memory_copy::<M::ABI>()?; 1852 1853 FnCall::emit::<M>( 1854 &mut self.env, 1855 self.masm, 1856 &mut self.context, 1857 Callee::Builtin(builtin), 1858 )?; 1859 self.context.pop_and_free(self.masm) 1860 } 1861 visit_memory_fill(&mut self, mem: u32) -> Self::Output1862 fn visit_memory_fill(&mut self, mem: u32) -> Self::Output { 1863 let at = self.context.stack.ensure_index_at(3)?; 1864 let mem = MemoryIndex::from_u32(mem); 1865 1866 let builtin = self.env.builtins.memory_fill::<M::ABI>()?; 1867 let builtin = self.prepare_builtin_defined_memory_arg(mem, at, builtin)?; 1868 1869 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, builtin)?; 1870 self.context.pop_and_free(self.masm) 1871 } 1872 visit_memory_size(&mut self, mem: u32) -> Self::Output1873 fn visit_memory_size(&mut self, mem: u32) -> Self::Output { 1874 let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem)); 1875 self.emit_compute_memory_size(&heap) 1876 } 1877 visit_memory_grow(&mut self, mem: u32) -> Self::Output1878 fn visit_memory_grow(&mut self, mem: u32) -> Self::Output { 1879 let at = self.context.stack.ensure_index_at(1)?; 1880 let mem = MemoryIndex::from_u32(mem); 1881 // The stack at this point contains: [ delta ] 1882 // The desired state is 1883 // [ vmctx, delta, index ] 1884 let builtin = self.env.builtins.memory_grow::<M::ABI>()?; 1885 let builtin = self.prepare_builtin_defined_memory_arg(mem, at + 1, builtin)?; 1886 1887 let heap = self.env.resolve_heap(mem); 1888 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, builtin)?; 1889 1890 // The memory32_grow builtin returns a pointer type, therefore we must 1891 // ensure that the return type is representative of the address space of 1892 // the heap type. 1893 match (self.env.ptr_type(), heap.index_type()) { 1894 (WasmValType::I64, WasmValType::I64) => Ok(()), 1895 // When the heap type is smaller than the pointer type, we adjust 1896 // the result of the memory32_grow builtin. 1897 (WasmValType::I64, WasmValType::I32) => { 1898 let top: Reg = self.context.pop_to_reg(self.masm, None)?.into(); 1899 self.masm.wrap(writable!(top), top)?; 1900 self.context.stack.push(TypedReg::i32(top).into()); 1901 Ok(()) 1902 } 1903 _ => Err(format_err!(CodeGenError::unsupported_32_bit_platform())), 1904 } 1905 } 1906 visit_data_drop(&mut self, data_index: u32) -> Self::Output1907 fn visit_data_drop(&mut self, data_index: u32) -> Self::Output { 1908 self.context.stack.extend([data_index.try_into()?]); 1909 1910 let builtin = self.env.builtins.data_drop::<M::ABI>()?; 1911 FnCall::emit::<M>( 1912 &mut self.env, 1913 self.masm, 1914 &mut self.context, 1915 Callee::Builtin(builtin), 1916 )?; 1917 self.context.pop_and_free(self.masm) 1918 } 1919 visit_nop(&mut self) -> Self::Output1920 fn visit_nop(&mut self) -> Self::Output { 1921 Ok(()) 1922 } 1923 visit_if(&mut self, blockty: BlockType) -> Self::Output1924 fn visit_if(&mut self, blockty: BlockType) -> Self::Output { 1925 self.control_frames.push(ControlStackFrame::r#if( 1926 self.env.resolve_block_sig(blockty)?, 1927 self.masm, 1928 &mut self.context, 1929 )?); 1930 1931 Ok(()) 1932 } 1933 visit_else(&mut self) -> Self::Output1934 fn visit_else(&mut self) -> Self::Output { 1935 if !self.context.reachable { 1936 self.handle_unreachable_else() 1937 } else { 1938 let control = self 1939 .control_frames 1940 .last_mut() 1941 .ok_or_else(|| CodeGenError::control_frame_expected())?; 1942 control.emit_else(self.masm, &mut self.context) 1943 } 1944 } 1945 visit_block(&mut self, blockty: BlockType) -> Self::Output1946 fn visit_block(&mut self, blockty: BlockType) -> Self::Output { 1947 self.control_frames.push(ControlStackFrame::block( 1948 self.env.resolve_block_sig(blockty)?, 1949 self.masm, 1950 &mut self.context, 1951 )?); 1952 1953 Ok(()) 1954 } 1955 visit_loop(&mut self, blockty: BlockType) -> Self::Output1956 fn visit_loop(&mut self, blockty: BlockType) -> Self::Output { 1957 self.control_frames.push(ControlStackFrame::r#loop( 1958 self.env.resolve_block_sig(blockty)?, 1959 self.masm, 1960 &mut self.context, 1961 )?); 1962 1963 self.maybe_emit_epoch_check()?; 1964 self.maybe_emit_fuel_check() 1965 } 1966 visit_br(&mut self, depth: u32) -> Self::Output1967 fn visit_br(&mut self, depth: u32) -> Self::Output { 1968 let index = control_index(depth, self.control_frames.len())?; 1969 let frame = &mut self.control_frames[index]; 1970 self.context 1971 .br::<_, _, UnconditionalBranch>(frame, self.masm, |masm, cx, frame| { 1972 frame.pop_abi_results::<M, _>(cx, masm, |results, _, _| { 1973 Ok(results.ret_area().copied()) 1974 }) 1975 }) 1976 } 1977 visit_br_if(&mut self, depth: u32) -> Self::Output1978 fn visit_br_if(&mut self, depth: u32) -> Self::Output { 1979 let index = control_index(depth, self.control_frames.len())?; 1980 let frame = &mut self.control_frames[index]; 1981 frame.set_as_target(); 1982 1983 let top = { 1984 let top = self.context.without::<Result<TypedReg>, M, _>( 1985 frame.results::<M>()?.regs(), 1986 self.masm, 1987 |ctx, masm| ctx.pop_to_reg(masm, None), 1988 )??; 1989 // Explicitly save any live registers and locals before setting up 1990 // the branch state. 1991 // In some cases, calculating the `top` value above, will result in 1992 // a spill, thus the following one will result in a no-op. 1993 self.context.spill(self.masm)?; 1994 frame.top_abi_results::<M, _>( 1995 &mut self.context, 1996 self.masm, 1997 |results, context, masm| { 1998 // In the case of `br_if` there's a possibility that we'll 1999 // exit early from the block or fallthrough, for 2000 // a fallthrough, we cannot rely on the pre-computed return area; 2001 // it must be recalculated so that any values that are 2002 // generated are correctly placed near the current stack 2003 // pointer. 2004 if results.on_stack() { 2005 let stack_consumed = context.stack.sizeof(results.stack_operands_len()); 2006 let base = masm.sp_offset()?.as_u32() - stack_consumed; 2007 let offs = base + results.size(); 2008 Ok(Some(RetArea::sp(SPOffset::from_u32(offs)))) 2009 } else { 2010 Ok(None) 2011 } 2012 }, 2013 )?; 2014 top 2015 }; 2016 2017 // Emit instructions to balance the machine stack. 2018 let current_sp_offset = self.masm.sp_offset()?; 2019 let unbalanced = frame.unbalanced(self.masm)?; 2020 let (label, cmp) = if unbalanced { 2021 (self.masm.get_label()?, IntCmpKind::Eq) 2022 } else { 2023 (*frame.label(), IntCmpKind::Ne) 2024 }; 2025 2026 self.masm 2027 .branch(cmp, top.reg, top.reg.into(), label, OperandSize::S32)?; 2028 self.context.free_reg(top); 2029 2030 if unbalanced { 2031 self.context 2032 .br::<_, _, ConditionalBranch>(frame, self.masm, |_, _, _| Ok(()))?; 2033 2034 // Restore sp_offset to what it was for falling through and emit 2035 // fallthrough label. 2036 self.masm.reset_stack_pointer(current_sp_offset)?; 2037 self.masm.bind(label)?; 2038 } 2039 2040 Ok(()) 2041 } 2042 visit_br_table(&mut self, targets: BrTable<'a>) -> Self::Output2043 fn visit_br_table(&mut self, targets: BrTable<'a>) -> Self::Output { 2044 // +1 to account for the default target. 2045 let len = targets.len() + 1; 2046 // SmallVec<[_; 5]> to match the binary emission layer (e.g 2047 // see `JmpTableSeq'), but here we use 5 instead since we 2048 // bundle the default target as the last element in the array. 2049 let mut labels: SmallVec<[_; 5]> = smallvec![]; 2050 for _ in 0..len { 2051 labels.push(self.masm.get_label()?); 2052 } 2053 2054 // Find the innermost target and use it as the relative frame 2055 // for result handling below. 2056 // 2057 // This approach ensures that 2058 // 1. The stack pointer offset is correctly positioned 2059 // according to the expectations of the innermost block end 2060 // sequence. 2061 // 2. We meet the jump site invariants introduced by 2062 // `CodegenContext::br`, which take advantage of Wasm 2063 // semantics given that all jumps are "outward". 2064 let mut innermost = targets.default(); 2065 for target in targets.targets() { 2066 let target = target?; 2067 if target < innermost { 2068 innermost = target; 2069 } 2070 } 2071 2072 let innermost_index = control_index(innermost, self.control_frames.len())?; 2073 let innermost_frame = &mut self.control_frames[innermost_index]; 2074 let innermost_result = innermost_frame.results::<M>()?; 2075 2076 let (index, tmp) = { 2077 let index_and_tmp = self.context.without::<Result<(TypedReg, _)>, M, _>( 2078 innermost_result.regs(), 2079 self.masm, 2080 |cx, masm| Ok((cx.pop_to_reg(masm, None)?, cx.any_gpr(masm)?)), 2081 )??; 2082 2083 // Materialize any constants or locals into their result 2084 // representation, so that when reachability is restored, 2085 // they are correctly located. NB: the results are popped 2086 // in function of the innermost branch specified for 2087 // `br_table`, which implies that the machine stack will 2088 // be correctly balanced, by virtue of calling 2089 // `pop_abi_results`. 2090 2091 // It's possible that we need to balance the stack for the 2092 // rest of the targets, which will be done before emitting 2093 // the unconditional jump below. 2094 innermost_frame.pop_abi_results::<M, _>( 2095 &mut self.context, 2096 self.masm, 2097 |results, _, _| Ok(results.ret_area().copied()), 2098 )?; 2099 index_and_tmp 2100 }; 2101 2102 self.masm.jmp_table(&labels, index.into(), tmp)?; 2103 // Save the original stack pointer offset; we will reset the stack 2104 // pointer to this offset after jumping to each of the targets. Each 2105 // jump might adjust the stack according to the base offset of the 2106 // target. 2107 let current_sp = self.masm.sp_offset()?; 2108 2109 for (t, l) in targets 2110 .targets() 2111 .chain(std::iter::once(Ok(targets.default()))) 2112 .zip(labels.iter()) 2113 { 2114 let control_index = control_index(t?, self.control_frames.len())?; 2115 let frame = &mut self.control_frames[control_index]; 2116 // Reset the stack pointer to its original offset. This is needed 2117 // because each jump will potentially adjust the stack pointer 2118 // according to the base offset of the target. 2119 self.masm.reset_stack_pointer(current_sp)?; 2120 2121 // NB: We don't perform any result handling as it was 2122 // already taken care of above before jumping to the 2123 // jump table. 2124 self.masm.bind(*l)?; 2125 // Ensure that the stack pointer is correctly positioned before 2126 // jumping to the jump table code. 2127 self.context 2128 .br::<_, _, UnconditionalBranch>(frame, self.masm, |_, _, _| Ok(()))?; 2129 } 2130 // Finally reset the stack pointer to the original location. 2131 // The reachability analysis, will ensure it's correctly located 2132 // once reachability is restored. 2133 self.masm.reset_stack_pointer(current_sp)?; 2134 self.context.reachable = false; 2135 self.context.free_reg(index.reg); 2136 self.context.free_reg(tmp); 2137 2138 Ok(()) 2139 } 2140 visit_return(&mut self) -> Self::Output2141 fn visit_return(&mut self) -> Self::Output { 2142 // Grab the outermost frame, which is the function's body 2143 // frame. We don't rely on [`codegen::control_index`] since 2144 // this frame is implicit and we know that it should exist at 2145 // index 0. 2146 let outermost = &mut self.control_frames[0]; 2147 self.context 2148 .br::<_, _, UnconditionalBranch>(outermost, self.masm, |masm, cx, frame| { 2149 frame.pop_abi_results::<M, _>(cx, masm, |results, _, _| { 2150 Ok(results.ret_area().copied()) 2151 }) 2152 }) 2153 } 2154 visit_unreachable(&mut self) -> Self::Output2155 fn visit_unreachable(&mut self) -> Self::Output { 2156 self.masm.unreachable()?; 2157 self.context.reachable = false; 2158 // Set the implicit outermost frame as target to perform the necessary 2159 // stack clean up. 2160 let outermost = &mut self.control_frames[0]; 2161 outermost.set_as_target(); 2162 2163 Ok(()) 2164 } 2165 visit_local_tee(&mut self, index: u32) -> Self::Output2166 fn visit_local_tee(&mut self, index: u32) -> Self::Output { 2167 let typed_reg = self.emit_set_local(index)?; 2168 self.context.stack.push(typed_reg.into()); 2169 2170 Ok(()) 2171 } 2172 visit_global_get(&mut self, global_index: u32) -> Self::Output2173 fn visit_global_get(&mut self, global_index: u32) -> Self::Output { 2174 let index = GlobalIndex::from_u32(global_index); 2175 let (ty, base, offset) = self.emit_get_global_addr(index)?; 2176 let addr = self.masm.address_at_reg(base, offset)?; 2177 let dst = self.context.reg_for_type(ty, self.masm)?; 2178 self.masm.load(addr, writable!(dst), ty.try_into()?)?; 2179 self.context.stack.push(Val::reg(dst, ty)); 2180 2181 self.context.free_reg(base); 2182 2183 Ok(()) 2184 } 2185 visit_global_set(&mut self, global_index: u32) -> Self::Output2186 fn visit_global_set(&mut self, global_index: u32) -> Self::Output { 2187 let index = GlobalIndex::from_u32(global_index); 2188 let (ty, base, offset) = self.emit_get_global_addr(index)?; 2189 let addr = self.masm.address_at_reg(base, offset)?; 2190 2191 let typed_reg = self.context.pop_to_reg(self.masm, None)?; 2192 self.masm 2193 .store(typed_reg.reg.into(), addr, ty.try_into()?)?; 2194 self.context.free_reg(typed_reg.reg); 2195 self.context.free_reg(base); 2196 2197 Ok(()) 2198 } 2199 visit_drop(&mut self) -> Self::Output2200 fn visit_drop(&mut self) -> Self::Output { 2201 self.context.drop_last(1, |regalloc, val| match val { 2202 Val::Reg(tr) => Ok(regalloc.free(tr.reg)), 2203 Val::Memory(m) => self.masm.free_stack(m.slot.size), 2204 _ => Ok(()), 2205 }) 2206 } 2207 visit_select(&mut self) -> Self::Output2208 fn visit_select(&mut self) -> Self::Output { 2209 let cond = self.context.pop_to_reg(self.masm, None)?; 2210 let val2 = self.context.pop_to_reg(self.masm, None)?; 2211 let val1 = self.context.pop_to_reg(self.masm, None)?; 2212 self.masm.cmp(cond.reg, RegImm::i32(0), OperandSize::S32)?; 2213 // Conditionally move val1 to val2 if the comparison is 2214 // not zero. 2215 self.masm.cmov( 2216 writable!(val2.into()), 2217 val1.into(), 2218 IntCmpKind::Ne, 2219 val1.ty.try_into()?, 2220 )?; 2221 self.context.stack.push(val2.into()); 2222 self.context.free_reg(val1.reg); 2223 self.context.free_reg(cond); 2224 2225 Ok(()) 2226 } 2227 visit_typed_select(&mut self, _ty: ValType) -> Self::Output2228 fn visit_typed_select(&mut self, _ty: ValType) -> Self::Output { 2229 self.visit_select() 2230 } 2231 visit_ref_null(&mut self, hty: HeapType) -> Self::Output2232 fn visit_ref_null(&mut self, hty: HeapType) -> Self::Output { 2233 match hty { 2234 HeapType::FUNC => { 2235 let ptr_type = self.env.ptr_type(); 2236 match ptr_type { 2237 WasmValType::I64 => self.context.stack.push(Val::i64(0)), 2238 WasmValType::I32 => self.context.stack.push(Val::i32(0)), 2239 _ => bail!(CodeGenError::unsupported_wasm_type()), 2240 } 2241 Ok(()) 2242 } 2243 _ => Err(format_err!(CodeGenError::unsupported_wasm_type())), 2244 } 2245 } 2246 visit_ref_is_null(&mut self) -> Self::Output2247 fn visit_ref_is_null(&mut self) -> Self::Output { 2248 let (zero, size) = match self.env.ptr_type() { 2249 WasmValType::I64 => (RegImm::i64(0), OperandSize::S64), 2250 WasmValType::I32 => (RegImm::i32(0), OperandSize::S32), 2251 _ => bail!(CodeGenError::unsupported_wasm_type()), 2252 }; 2253 self.context.unop(self.masm, |masm, reg| { 2254 masm.cmp_with_set(writable!(reg), zero, IntCmpKind::Eq, size)?; 2255 Ok(TypedReg::i32(reg)) 2256 }) 2257 } 2258 visit_ref_func(&mut self, function_index: u32) -> Self::Output2259 fn visit_ref_func(&mut self, function_index: u32) -> Self::Output { 2260 let ref_func = self.env.builtins.ref_func::<M::ABI>()?; 2261 self.context.stack.extend([function_index.try_into()?]); 2262 FnCall::emit::<M>( 2263 &mut self.env, 2264 self.masm, 2265 &mut self.context, 2266 Callee::Builtin(ref_func), 2267 ) 2268 } 2269 visit_i32_load(&mut self, memarg: MemArg) -> Self::Output2270 fn visit_i32_load(&mut self, memarg: MemArg) -> Self::Output { 2271 self.emit_wasm_load( 2272 &memarg, 2273 WasmValType::I32, 2274 LoadKind::Operand(OperandSize::S32), 2275 ) 2276 } 2277 visit_i32_load8_s(&mut self, memarg: MemArg) -> Self::Output2278 fn visit_i32_load8_s(&mut self, memarg: MemArg) -> Self::Output { 2279 self.emit_wasm_load( 2280 &memarg, 2281 WasmValType::I32, 2282 LoadKind::ScalarExtend(Extend::<Signed>::I32Extend8.into()), 2283 ) 2284 } 2285 visit_i32_load8_u(&mut self, memarg: MemArg) -> Self::Output2286 fn visit_i32_load8_u(&mut self, memarg: MemArg) -> Self::Output { 2287 self.emit_wasm_load( 2288 &memarg, 2289 WasmValType::I32, 2290 LoadKind::ScalarExtend(Extend::<Zero>::I32Extend8.into()), 2291 ) 2292 } 2293 visit_i32_load16_s(&mut self, memarg: MemArg) -> Self::Output2294 fn visit_i32_load16_s(&mut self, memarg: MemArg) -> Self::Output { 2295 self.emit_wasm_load( 2296 &memarg, 2297 WasmValType::I32, 2298 LoadKind::ScalarExtend(Extend::<Signed>::I32Extend16.into()), 2299 ) 2300 } 2301 visit_i32_load16_u(&mut self, memarg: MemArg) -> Self::Output2302 fn visit_i32_load16_u(&mut self, memarg: MemArg) -> Self::Output { 2303 self.emit_wasm_load( 2304 &memarg, 2305 WasmValType::I32, 2306 LoadKind::ScalarExtend(Extend::<Zero>::I32Extend16.into()), 2307 ) 2308 } 2309 visit_i32_store(&mut self, memarg: MemArg) -> Self::Output2310 fn visit_i32_store(&mut self, memarg: MemArg) -> Self::Output { 2311 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S32)) 2312 } 2313 visit_i32_store8(&mut self, memarg: MemArg) -> Self::Output2314 fn visit_i32_store8(&mut self, memarg: MemArg) -> Self::Output { 2315 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S8)) 2316 } 2317 visit_i32_store16(&mut self, memarg: MemArg) -> Self::Output2318 fn visit_i32_store16(&mut self, memarg: MemArg) -> Self::Output { 2319 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S16)) 2320 } 2321 visit_i64_load8_s(&mut self, memarg: MemArg) -> Self::Output2322 fn visit_i64_load8_s(&mut self, memarg: MemArg) -> Self::Output { 2323 self.emit_wasm_load( 2324 &memarg, 2325 WasmValType::I64, 2326 LoadKind::ScalarExtend(Extend::<Signed>::I64Extend8.into()), 2327 ) 2328 } 2329 visit_i64_load8_u(&mut self, memarg: MemArg) -> Self::Output2330 fn visit_i64_load8_u(&mut self, memarg: MemArg) -> Self::Output { 2331 self.emit_wasm_load( 2332 &memarg, 2333 WasmValType::I64, 2334 LoadKind::ScalarExtend(Extend::<Zero>::I64Extend8.into()), 2335 ) 2336 } 2337 visit_i64_load16_u(&mut self, memarg: MemArg) -> Self::Output2338 fn visit_i64_load16_u(&mut self, memarg: MemArg) -> Self::Output { 2339 self.emit_wasm_load( 2340 &memarg, 2341 WasmValType::I64, 2342 LoadKind::ScalarExtend(Extend::<Zero>::I64Extend16.into()), 2343 ) 2344 } 2345 visit_i64_load16_s(&mut self, memarg: MemArg) -> Self::Output2346 fn visit_i64_load16_s(&mut self, memarg: MemArg) -> Self::Output { 2347 self.emit_wasm_load( 2348 &memarg, 2349 WasmValType::I64, 2350 LoadKind::ScalarExtend(Extend::<Signed>::I64Extend16.into()), 2351 ) 2352 } 2353 visit_i64_load32_u(&mut self, memarg: MemArg) -> Self::Output2354 fn visit_i64_load32_u(&mut self, memarg: MemArg) -> Self::Output { 2355 self.emit_wasm_load( 2356 &memarg, 2357 WasmValType::I64, 2358 LoadKind::ScalarExtend(Extend::<Zero>::I64Extend32.into()), 2359 ) 2360 } 2361 visit_i64_load32_s(&mut self, memarg: MemArg) -> Self::Output2362 fn visit_i64_load32_s(&mut self, memarg: MemArg) -> Self::Output { 2363 self.emit_wasm_load( 2364 &memarg, 2365 WasmValType::I64, 2366 LoadKind::ScalarExtend(Extend::<Signed>::I64Extend32.into()), 2367 ) 2368 } 2369 visit_i64_load(&mut self, memarg: MemArg) -> Self::Output2370 fn visit_i64_load(&mut self, memarg: MemArg) -> Self::Output { 2371 self.emit_wasm_load( 2372 &memarg, 2373 WasmValType::I64, 2374 LoadKind::Operand(OperandSize::S64), 2375 ) 2376 } 2377 visit_i64_store(&mut self, memarg: MemArg) -> Self::Output2378 fn visit_i64_store(&mut self, memarg: MemArg) -> Self::Output { 2379 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S64)) 2380 } 2381 visit_i64_store8(&mut self, memarg: MemArg) -> Self::Output2382 fn visit_i64_store8(&mut self, memarg: MemArg) -> Self::Output { 2383 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S8)) 2384 } 2385 visit_i64_store16(&mut self, memarg: MemArg) -> Self::Output2386 fn visit_i64_store16(&mut self, memarg: MemArg) -> Self::Output { 2387 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S16)) 2388 } 2389 visit_i64_store32(&mut self, memarg: MemArg) -> Self::Output2390 fn visit_i64_store32(&mut self, memarg: MemArg) -> Self::Output { 2391 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S32)) 2392 } 2393 visit_f32_load(&mut self, memarg: MemArg) -> Self::Output2394 fn visit_f32_load(&mut self, memarg: MemArg) -> Self::Output { 2395 self.emit_wasm_load( 2396 &memarg, 2397 WasmValType::F32, 2398 LoadKind::Operand(OperandSize::S32), 2399 ) 2400 } 2401 visit_f32_store(&mut self, memarg: MemArg) -> Self::Output2402 fn visit_f32_store(&mut self, memarg: MemArg) -> Self::Output { 2403 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S32)) 2404 } 2405 visit_f64_load(&mut self, memarg: MemArg) -> Self::Output2406 fn visit_f64_load(&mut self, memarg: MemArg) -> Self::Output { 2407 self.emit_wasm_load( 2408 &memarg, 2409 WasmValType::F64, 2410 LoadKind::Operand(OperandSize::S64), 2411 ) 2412 } 2413 visit_f64_store(&mut self, memarg: MemArg) -> Self::Output2414 fn visit_f64_store(&mut self, memarg: MemArg) -> Self::Output { 2415 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S64)) 2416 } 2417 visit_i32_trunc_sat_f32_s(&mut self) -> Self::Output2418 fn visit_i32_trunc_sat_f32_s(&mut self) -> Self::Output { 2419 use OperandSize::*; 2420 2421 self.context 2422 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2423 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Checked) 2424 }) 2425 } 2426 visit_i32_trunc_sat_f32_u(&mut self) -> Self::Output2427 fn visit_i32_trunc_sat_f32_u(&mut self) -> Self::Output { 2428 use OperandSize::*; 2429 2430 self.masm 2431 .unsigned_truncate(&mut self.context, S32, S32, TruncKind::Checked) 2432 } 2433 visit_i32_trunc_sat_f64_s(&mut self) -> Self::Output2434 fn visit_i32_trunc_sat_f64_s(&mut self) -> Self::Output { 2435 use OperandSize::*; 2436 2437 self.context 2438 .convert_op(self.masm, WasmValType::I32, |masm, dst, src, dst_size| { 2439 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Checked) 2440 }) 2441 } 2442 visit_i32_trunc_sat_f64_u(&mut self) -> Self::Output2443 fn visit_i32_trunc_sat_f64_u(&mut self) -> Self::Output { 2444 use OperandSize::*; 2445 2446 self.masm 2447 .unsigned_truncate(&mut self.context, S64, S32, TruncKind::Checked) 2448 } 2449 visit_i64_trunc_sat_f32_s(&mut self) -> Self::Output2450 fn visit_i64_trunc_sat_f32_s(&mut self) -> Self::Output { 2451 use OperandSize::*; 2452 2453 self.context 2454 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2455 masm.signed_truncate(writable!(dst), src, S32, dst_size, TruncKind::Checked) 2456 }) 2457 } 2458 visit_i64_trunc_sat_f32_u(&mut self) -> Self::Output2459 fn visit_i64_trunc_sat_f32_u(&mut self) -> Self::Output { 2460 use OperandSize::*; 2461 2462 self.masm 2463 .unsigned_truncate(&mut self.context, S32, S64, TruncKind::Checked) 2464 } 2465 visit_i64_trunc_sat_f64_s(&mut self) -> Self::Output2466 fn visit_i64_trunc_sat_f64_s(&mut self) -> Self::Output { 2467 use OperandSize::*; 2468 2469 self.context 2470 .convert_op(self.masm, WasmValType::I64, |masm, dst, src, dst_size| { 2471 masm.signed_truncate(writable!(dst), src, S64, dst_size, TruncKind::Checked) 2472 }) 2473 } 2474 visit_i64_trunc_sat_f64_u(&mut self) -> Self::Output2475 fn visit_i64_trunc_sat_f64_u(&mut self) -> Self::Output { 2476 use OperandSize::*; 2477 2478 self.masm 2479 .unsigned_truncate(&mut self.context, S64, S64, TruncKind::Checked) 2480 } 2481 visit_i64_add128(&mut self) -> Self::Output2482 fn visit_i64_add128(&mut self) -> Self::Output { 2483 self.context 2484 .binop128(self.masm, |masm, lhs_lo, lhs_hi, rhs_lo, rhs_hi| { 2485 masm.add128( 2486 writable!(lhs_lo), 2487 writable!(lhs_hi), 2488 lhs_lo, 2489 lhs_hi, 2490 rhs_lo, 2491 rhs_hi, 2492 )?; 2493 Ok((TypedReg::i64(lhs_lo), TypedReg::i64(lhs_hi))) 2494 }) 2495 } 2496 visit_i64_sub128(&mut self) -> Self::Output2497 fn visit_i64_sub128(&mut self) -> Self::Output { 2498 self.context 2499 .binop128(self.masm, |masm, lhs_lo, lhs_hi, rhs_lo, rhs_hi| { 2500 masm.sub128( 2501 writable!(lhs_lo), 2502 writable!(lhs_hi), 2503 lhs_lo, 2504 lhs_hi, 2505 rhs_lo, 2506 rhs_hi, 2507 )?; 2508 Ok((TypedReg::i64(lhs_lo), TypedReg::i64(lhs_hi))) 2509 }) 2510 } 2511 visit_i64_mul_wide_s(&mut self) -> Self::Output2512 fn visit_i64_mul_wide_s(&mut self) -> Self::Output { 2513 self.masm.mul_wide(&mut self.context, MulWideKind::Signed) 2514 } 2515 visit_i64_mul_wide_u(&mut self) -> Self::Output2516 fn visit_i64_mul_wide_u(&mut self) -> Self::Output { 2517 self.masm.mul_wide(&mut self.context, MulWideKind::Unsigned) 2518 } 2519 visit_i32_atomic_load8_u(&mut self, memarg: MemArg) -> Self::Output2520 fn visit_i32_atomic_load8_u(&mut self, memarg: MemArg) -> Self::Output { 2521 self.emit_wasm_load( 2522 &memarg, 2523 WasmValType::I32, 2524 LoadKind::Atomic(OperandSize::S8, Some(Extend::<Zero>::I32Extend8.into())), 2525 ) 2526 } 2527 visit_i32_atomic_load16_u(&mut self, memarg: MemArg) -> Self::Output2528 fn visit_i32_atomic_load16_u(&mut self, memarg: MemArg) -> Self::Output { 2529 self.emit_wasm_load( 2530 &memarg, 2531 WasmValType::I32, 2532 LoadKind::Atomic(OperandSize::S16, Some(Extend::<Zero>::I32Extend16.into())), 2533 ) 2534 } 2535 visit_i32_atomic_load(&mut self, memarg: MemArg) -> Self::Output2536 fn visit_i32_atomic_load(&mut self, memarg: MemArg) -> Self::Output { 2537 self.emit_wasm_load( 2538 &memarg, 2539 WasmValType::I32, 2540 LoadKind::Atomic(OperandSize::S32, None), 2541 ) 2542 } 2543 visit_i64_atomic_load8_u(&mut self, memarg: MemArg) -> Self::Output2544 fn visit_i64_atomic_load8_u(&mut self, memarg: MemArg) -> Self::Output { 2545 self.emit_wasm_load( 2546 &memarg, 2547 WasmValType::I64, 2548 LoadKind::Atomic(OperandSize::S8, Some(Extend::<Zero>::I64Extend8.into())), 2549 ) 2550 } 2551 visit_i64_atomic_load16_u(&mut self, memarg: MemArg) -> Self::Output2552 fn visit_i64_atomic_load16_u(&mut self, memarg: MemArg) -> Self::Output { 2553 self.emit_wasm_load( 2554 &memarg, 2555 WasmValType::I64, 2556 LoadKind::Atomic(OperandSize::S16, Some(Extend::<Zero>::I64Extend16.into())), 2557 ) 2558 } 2559 visit_i64_atomic_load32_u(&mut self, memarg: MemArg) -> Self::Output2560 fn visit_i64_atomic_load32_u(&mut self, memarg: MemArg) -> Self::Output { 2561 self.emit_wasm_load( 2562 &memarg, 2563 WasmValType::I64, 2564 LoadKind::Atomic(OperandSize::S32, Some(Extend::<Zero>::I64Extend32.into())), 2565 ) 2566 } 2567 visit_i64_atomic_load(&mut self, memarg: MemArg) -> Self::Output2568 fn visit_i64_atomic_load(&mut self, memarg: MemArg) -> Self::Output { 2569 self.emit_wasm_load( 2570 &memarg, 2571 WasmValType::I64, 2572 LoadKind::Atomic(OperandSize::S64, None), 2573 ) 2574 } 2575 visit_i32_atomic_store(&mut self, memarg: MemArg) -> Self::Output2576 fn visit_i32_atomic_store(&mut self, memarg: MemArg) -> Self::Output { 2577 self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S32)) 2578 } 2579 visit_i64_atomic_store(&mut self, memarg: MemArg) -> Self::Output2580 fn visit_i64_atomic_store(&mut self, memarg: MemArg) -> Self::Output { 2581 self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S64)) 2582 } 2583 visit_i32_atomic_store8(&mut self, memarg: MemArg) -> Self::Output2584 fn visit_i32_atomic_store8(&mut self, memarg: MemArg) -> Self::Output { 2585 self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S8)) 2586 } 2587 visit_i32_atomic_store16(&mut self, memarg: MemArg) -> Self::Output2588 fn visit_i32_atomic_store16(&mut self, memarg: MemArg) -> Self::Output { 2589 self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S16)) 2590 } 2591 visit_i64_atomic_store8(&mut self, memarg: MemArg) -> Self::Output2592 fn visit_i64_atomic_store8(&mut self, memarg: MemArg) -> Self::Output { 2593 self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S8)) 2594 } 2595 visit_i64_atomic_store16(&mut self, memarg: MemArg) -> Self::Output2596 fn visit_i64_atomic_store16(&mut self, memarg: MemArg) -> Self::Output { 2597 self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S16)) 2598 } 2599 visit_i64_atomic_store32(&mut self, memarg: MemArg) -> Self::Output2600 fn visit_i64_atomic_store32(&mut self, memarg: MemArg) -> Self::Output { 2601 self.emit_wasm_store(&memarg, StoreKind::Atomic(OperandSize::S32)) 2602 } 2603 visit_i32_atomic_rmw8_add_u(&mut self, arg: MemArg) -> Self::Output2604 fn visit_i32_atomic_rmw8_add_u(&mut self, arg: MemArg) -> Self::Output { 2605 self.emit_atomic_rmw( 2606 &arg, 2607 RmwOp::Add, 2608 OperandSize::S8, 2609 Some(Extend::<Zero>::I32Extend8), 2610 ) 2611 } 2612 visit_i32_atomic_rmw16_add_u(&mut self, arg: MemArg) -> Self::Output2613 fn visit_i32_atomic_rmw16_add_u(&mut self, arg: MemArg) -> Self::Output { 2614 self.emit_atomic_rmw( 2615 &arg, 2616 RmwOp::Add, 2617 OperandSize::S16, 2618 Some(Extend::<Zero>::I32Extend16), 2619 ) 2620 } 2621 visit_i32_atomic_rmw_add(&mut self, arg: MemArg) -> Self::Output2622 fn visit_i32_atomic_rmw_add(&mut self, arg: MemArg) -> Self::Output { 2623 self.emit_atomic_rmw(&arg, RmwOp::Add, OperandSize::S32, None) 2624 } 2625 visit_i64_atomic_rmw8_add_u(&mut self, arg: MemArg) -> Self::Output2626 fn visit_i64_atomic_rmw8_add_u(&mut self, arg: MemArg) -> Self::Output { 2627 self.emit_atomic_rmw( 2628 &arg, 2629 RmwOp::Add, 2630 OperandSize::S8, 2631 Some(Extend::<Zero>::I64Extend8), 2632 ) 2633 } 2634 visit_i64_atomic_rmw16_add_u(&mut self, arg: MemArg) -> Self::Output2635 fn visit_i64_atomic_rmw16_add_u(&mut self, arg: MemArg) -> Self::Output { 2636 self.emit_atomic_rmw( 2637 &arg, 2638 RmwOp::Add, 2639 OperandSize::S16, 2640 Some(Extend::<Zero>::I64Extend16), 2641 ) 2642 } 2643 visit_i64_atomic_rmw32_add_u(&mut self, arg: MemArg) -> Self::Output2644 fn visit_i64_atomic_rmw32_add_u(&mut self, arg: MemArg) -> Self::Output { 2645 self.emit_atomic_rmw( 2646 &arg, 2647 RmwOp::Add, 2648 OperandSize::S32, 2649 Some(Extend::<Zero>::I64Extend32), 2650 ) 2651 } 2652 visit_i64_atomic_rmw_add(&mut self, arg: MemArg) -> Self::Output2653 fn visit_i64_atomic_rmw_add(&mut self, arg: MemArg) -> Self::Output { 2654 self.emit_atomic_rmw(&arg, RmwOp::Add, OperandSize::S64, None) 2655 } 2656 visit_i32_atomic_rmw8_sub_u(&mut self, arg: MemArg) -> Self::Output2657 fn visit_i32_atomic_rmw8_sub_u(&mut self, arg: MemArg) -> Self::Output { 2658 self.emit_atomic_rmw( 2659 &arg, 2660 RmwOp::Sub, 2661 OperandSize::S8, 2662 Some(Extend::<Zero>::I32Extend8), 2663 ) 2664 } visit_i32_atomic_rmw16_sub_u(&mut self, arg: MemArg) -> Self::Output2665 fn visit_i32_atomic_rmw16_sub_u(&mut self, arg: MemArg) -> Self::Output { 2666 self.emit_atomic_rmw( 2667 &arg, 2668 RmwOp::Sub, 2669 OperandSize::S16, 2670 Some(Extend::<Zero>::I32Extend16), 2671 ) 2672 } 2673 visit_i32_atomic_rmw_sub(&mut self, arg: MemArg) -> Self::Output2674 fn visit_i32_atomic_rmw_sub(&mut self, arg: MemArg) -> Self::Output { 2675 self.emit_atomic_rmw(&arg, RmwOp::Sub, OperandSize::S32, None) 2676 } 2677 visit_i64_atomic_rmw8_sub_u(&mut self, arg: MemArg) -> Self::Output2678 fn visit_i64_atomic_rmw8_sub_u(&mut self, arg: MemArg) -> Self::Output { 2679 self.emit_atomic_rmw( 2680 &arg, 2681 RmwOp::Sub, 2682 OperandSize::S8, 2683 Some(Extend::<Zero>::I64Extend8), 2684 ) 2685 } 2686 visit_i64_atomic_rmw16_sub_u(&mut self, arg: MemArg) -> Self::Output2687 fn visit_i64_atomic_rmw16_sub_u(&mut self, arg: MemArg) -> Self::Output { 2688 self.emit_atomic_rmw( 2689 &arg, 2690 RmwOp::Sub, 2691 OperandSize::S16, 2692 Some(Extend::<Zero>::I64Extend16), 2693 ) 2694 } 2695 visit_i64_atomic_rmw32_sub_u(&mut self, arg: MemArg) -> Self::Output2696 fn visit_i64_atomic_rmw32_sub_u(&mut self, arg: MemArg) -> Self::Output { 2697 self.emit_atomic_rmw( 2698 &arg, 2699 RmwOp::Sub, 2700 OperandSize::S32, 2701 Some(Extend::<Zero>::I64Extend32), 2702 ) 2703 } 2704 visit_i64_atomic_rmw_sub(&mut self, arg: MemArg) -> Self::Output2705 fn visit_i64_atomic_rmw_sub(&mut self, arg: MemArg) -> Self::Output { 2706 self.emit_atomic_rmw(&arg, RmwOp::Sub, OperandSize::S64, None) 2707 } 2708 visit_i32_atomic_rmw8_xchg_u(&mut self, arg: MemArg) -> Self::Output2709 fn visit_i32_atomic_rmw8_xchg_u(&mut self, arg: MemArg) -> Self::Output { 2710 self.emit_atomic_rmw( 2711 &arg, 2712 RmwOp::Xchg, 2713 OperandSize::S8, 2714 Some(Extend::<Zero>::I32Extend8), 2715 ) 2716 } 2717 visit_i32_atomic_rmw16_xchg_u(&mut self, arg: MemArg) -> Self::Output2718 fn visit_i32_atomic_rmw16_xchg_u(&mut self, arg: MemArg) -> Self::Output { 2719 self.emit_atomic_rmw( 2720 &arg, 2721 RmwOp::Xchg, 2722 OperandSize::S16, 2723 Some(Extend::<Zero>::I32Extend16), 2724 ) 2725 } 2726 visit_i32_atomic_rmw_xchg(&mut self, arg: MemArg) -> Self::Output2727 fn visit_i32_atomic_rmw_xchg(&mut self, arg: MemArg) -> Self::Output { 2728 self.emit_atomic_rmw(&arg, RmwOp::Xchg, OperandSize::S32, None) 2729 } 2730 visit_i64_atomic_rmw8_xchg_u(&mut self, arg: MemArg) -> Self::Output2731 fn visit_i64_atomic_rmw8_xchg_u(&mut self, arg: MemArg) -> Self::Output { 2732 self.emit_atomic_rmw( 2733 &arg, 2734 RmwOp::Xchg, 2735 OperandSize::S8, 2736 Some(Extend::<Zero>::I64Extend8), 2737 ) 2738 } 2739 visit_i64_atomic_rmw16_xchg_u(&mut self, arg: MemArg) -> Self::Output2740 fn visit_i64_atomic_rmw16_xchg_u(&mut self, arg: MemArg) -> Self::Output { 2741 self.emit_atomic_rmw( 2742 &arg, 2743 RmwOp::Xchg, 2744 OperandSize::S16, 2745 Some(Extend::<Zero>::I64Extend16), 2746 ) 2747 } 2748 visit_i64_atomic_rmw32_xchg_u(&mut self, arg: MemArg) -> Self::Output2749 fn visit_i64_atomic_rmw32_xchg_u(&mut self, arg: MemArg) -> Self::Output { 2750 self.emit_atomic_rmw( 2751 &arg, 2752 RmwOp::Xchg, 2753 OperandSize::S32, 2754 Some(Extend::<Zero>::I64Extend32), 2755 ) 2756 } 2757 visit_i64_atomic_rmw_xchg(&mut self, arg: MemArg) -> Self::Output2758 fn visit_i64_atomic_rmw_xchg(&mut self, arg: MemArg) -> Self::Output { 2759 self.emit_atomic_rmw(&arg, RmwOp::Xchg, OperandSize::S64, None) 2760 } 2761 visit_i32_atomic_rmw8_and_u(&mut self, arg: MemArg) -> Self::Output2762 fn visit_i32_atomic_rmw8_and_u(&mut self, arg: MemArg) -> Self::Output { 2763 self.emit_atomic_rmw( 2764 &arg, 2765 RmwOp::And, 2766 OperandSize::S8, 2767 Some(Extend::<Zero>::I32Extend8), 2768 ) 2769 } 2770 visit_i32_atomic_rmw16_and_u(&mut self, arg: MemArg) -> Self::Output2771 fn visit_i32_atomic_rmw16_and_u(&mut self, arg: MemArg) -> Self::Output { 2772 self.emit_atomic_rmw( 2773 &arg, 2774 RmwOp::And, 2775 OperandSize::S16, 2776 Some(Extend::<Zero>::I32Extend16), 2777 ) 2778 } 2779 visit_i32_atomic_rmw_and(&mut self, arg: MemArg) -> Self::Output2780 fn visit_i32_atomic_rmw_and(&mut self, arg: MemArg) -> Self::Output { 2781 self.emit_atomic_rmw(&arg, RmwOp::And, OperandSize::S32, None) 2782 } 2783 visit_i64_atomic_rmw8_and_u(&mut self, arg: MemArg) -> Self::Output2784 fn visit_i64_atomic_rmw8_and_u(&mut self, arg: MemArg) -> Self::Output { 2785 self.emit_atomic_rmw( 2786 &arg, 2787 RmwOp::And, 2788 OperandSize::S8, 2789 Some(Extend::<Zero>::I64Extend8), 2790 ) 2791 } 2792 visit_i64_atomic_rmw16_and_u(&mut self, arg: MemArg) -> Self::Output2793 fn visit_i64_atomic_rmw16_and_u(&mut self, arg: MemArg) -> Self::Output { 2794 self.emit_atomic_rmw( 2795 &arg, 2796 RmwOp::And, 2797 OperandSize::S16, 2798 Some(Extend::<Zero>::I64Extend16), 2799 ) 2800 } 2801 visit_i64_atomic_rmw32_and_u(&mut self, arg: MemArg) -> Self::Output2802 fn visit_i64_atomic_rmw32_and_u(&mut self, arg: MemArg) -> Self::Output { 2803 self.emit_atomic_rmw( 2804 &arg, 2805 RmwOp::And, 2806 OperandSize::S32, 2807 Some(Extend::<Zero>::I64Extend32), 2808 ) 2809 } 2810 visit_i64_atomic_rmw_and(&mut self, arg: MemArg) -> Self::Output2811 fn visit_i64_atomic_rmw_and(&mut self, arg: MemArg) -> Self::Output { 2812 self.emit_atomic_rmw(&arg, RmwOp::And, OperandSize::S64, None) 2813 } 2814 visit_i32_atomic_rmw8_or_u(&mut self, arg: MemArg) -> Self::Output2815 fn visit_i32_atomic_rmw8_or_u(&mut self, arg: MemArg) -> Self::Output { 2816 self.emit_atomic_rmw( 2817 &arg, 2818 RmwOp::Or, 2819 OperandSize::S8, 2820 Some(Extend::<Zero>::I32Extend8), 2821 ) 2822 } 2823 visit_i32_atomic_rmw16_or_u(&mut self, arg: MemArg) -> Self::Output2824 fn visit_i32_atomic_rmw16_or_u(&mut self, arg: MemArg) -> Self::Output { 2825 self.emit_atomic_rmw( 2826 &arg, 2827 RmwOp::Or, 2828 OperandSize::S16, 2829 Some(Extend::<Zero>::I32Extend16), 2830 ) 2831 } 2832 visit_i32_atomic_rmw_or(&mut self, arg: MemArg) -> Self::Output2833 fn visit_i32_atomic_rmw_or(&mut self, arg: MemArg) -> Self::Output { 2834 self.emit_atomic_rmw(&arg, RmwOp::Or, OperandSize::S32, None) 2835 } 2836 visit_i64_atomic_rmw8_or_u(&mut self, arg: MemArg) -> Self::Output2837 fn visit_i64_atomic_rmw8_or_u(&mut self, arg: MemArg) -> Self::Output { 2838 self.emit_atomic_rmw( 2839 &arg, 2840 RmwOp::Or, 2841 OperandSize::S8, 2842 Some(Extend::<Zero>::I64Extend8), 2843 ) 2844 } 2845 visit_i64_atomic_rmw16_or_u(&mut self, arg: MemArg) -> Self::Output2846 fn visit_i64_atomic_rmw16_or_u(&mut self, arg: MemArg) -> Self::Output { 2847 self.emit_atomic_rmw( 2848 &arg, 2849 RmwOp::Or, 2850 OperandSize::S16, 2851 Some(Extend::<Zero>::I64Extend16), 2852 ) 2853 } 2854 visit_i64_atomic_rmw32_or_u(&mut self, arg: MemArg) -> Self::Output2855 fn visit_i64_atomic_rmw32_or_u(&mut self, arg: MemArg) -> Self::Output { 2856 self.emit_atomic_rmw( 2857 &arg, 2858 RmwOp::Or, 2859 OperandSize::S32, 2860 Some(Extend::<Zero>::I64Extend32), 2861 ) 2862 } 2863 visit_i64_atomic_rmw_or(&mut self, arg: MemArg) -> Self::Output2864 fn visit_i64_atomic_rmw_or(&mut self, arg: MemArg) -> Self::Output { 2865 self.emit_atomic_rmw(&arg, RmwOp::Or, OperandSize::S64, None) 2866 } 2867 visit_i32_atomic_rmw8_xor_u(&mut self, arg: MemArg) -> Self::Output2868 fn visit_i32_atomic_rmw8_xor_u(&mut self, arg: MemArg) -> Self::Output { 2869 self.emit_atomic_rmw( 2870 &arg, 2871 RmwOp::Xor, 2872 OperandSize::S8, 2873 Some(Extend::<Zero>::I32Extend8), 2874 ) 2875 } 2876 visit_i32_atomic_rmw16_xor_u(&mut self, arg: MemArg) -> Self::Output2877 fn visit_i32_atomic_rmw16_xor_u(&mut self, arg: MemArg) -> Self::Output { 2878 self.emit_atomic_rmw( 2879 &arg, 2880 RmwOp::Xor, 2881 OperandSize::S16, 2882 Some(Extend::<Zero>::I32Extend16), 2883 ) 2884 } 2885 visit_i32_atomic_rmw_xor(&mut self, arg: MemArg) -> Self::Output2886 fn visit_i32_atomic_rmw_xor(&mut self, arg: MemArg) -> Self::Output { 2887 self.emit_atomic_rmw(&arg, RmwOp::Xor, OperandSize::S32, None) 2888 } 2889 visit_i64_atomic_rmw8_xor_u(&mut self, arg: MemArg) -> Self::Output2890 fn visit_i64_atomic_rmw8_xor_u(&mut self, arg: MemArg) -> Self::Output { 2891 self.emit_atomic_rmw( 2892 &arg, 2893 RmwOp::Xor, 2894 OperandSize::S8, 2895 Some(Extend::<Zero>::I64Extend8), 2896 ) 2897 } 2898 visit_i64_atomic_rmw16_xor_u(&mut self, arg: MemArg) -> Self::Output2899 fn visit_i64_atomic_rmw16_xor_u(&mut self, arg: MemArg) -> Self::Output { 2900 self.emit_atomic_rmw( 2901 &arg, 2902 RmwOp::Xor, 2903 OperandSize::S16, 2904 Some(Extend::<Zero>::I64Extend16), 2905 ) 2906 } 2907 visit_i64_atomic_rmw32_xor_u(&mut self, arg: MemArg) -> Self::Output2908 fn visit_i64_atomic_rmw32_xor_u(&mut self, arg: MemArg) -> Self::Output { 2909 self.emit_atomic_rmw( 2910 &arg, 2911 RmwOp::Xor, 2912 OperandSize::S32, 2913 Some(Extend::<Zero>::I64Extend32), 2914 ) 2915 } 2916 visit_i64_atomic_rmw_xor(&mut self, arg: MemArg) -> Self::Output2917 fn visit_i64_atomic_rmw_xor(&mut self, arg: MemArg) -> Self::Output { 2918 self.emit_atomic_rmw(&arg, RmwOp::Xor, OperandSize::S64, None) 2919 } 2920 visit_i32_atomic_rmw8_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output2921 fn visit_i32_atomic_rmw8_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output { 2922 self.emit_atomic_cmpxchg(&arg, OperandSize::S8, Some(Extend::I32Extend8)) 2923 } 2924 visit_i32_atomic_rmw16_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output2925 fn visit_i32_atomic_rmw16_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output { 2926 self.emit_atomic_cmpxchg(&arg, OperandSize::S16, Some(Extend::I32Extend16)) 2927 } 2928 visit_i32_atomic_rmw_cmpxchg(&mut self, arg: MemArg) -> Self::Output2929 fn visit_i32_atomic_rmw_cmpxchg(&mut self, arg: MemArg) -> Self::Output { 2930 self.emit_atomic_cmpxchg(&arg, OperandSize::S32, None) 2931 } 2932 visit_i64_atomic_rmw8_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output2933 fn visit_i64_atomic_rmw8_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output { 2934 self.emit_atomic_cmpxchg(&arg, OperandSize::S8, Some(Extend::I64Extend8)) 2935 } 2936 visit_i64_atomic_rmw16_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output2937 fn visit_i64_atomic_rmw16_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output { 2938 self.emit_atomic_cmpxchg(&arg, OperandSize::S16, Some(Extend::I64Extend16)) 2939 } 2940 visit_i64_atomic_rmw32_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output2941 fn visit_i64_atomic_rmw32_cmpxchg_u(&mut self, arg: MemArg) -> Self::Output { 2942 self.emit_atomic_cmpxchg(&arg, OperandSize::S32, Some(Extend::I64Extend32)) 2943 } 2944 visit_i64_atomic_rmw_cmpxchg(&mut self, arg: MemArg) -> Self::Output2945 fn visit_i64_atomic_rmw_cmpxchg(&mut self, arg: MemArg) -> Self::Output { 2946 self.emit_atomic_cmpxchg(&arg, OperandSize::S64, None) 2947 } 2948 visit_memory_atomic_wait32(&mut self, arg: MemArg) -> Self::Output2949 fn visit_memory_atomic_wait32(&mut self, arg: MemArg) -> Self::Output { 2950 self.emit_atomic_wait(&arg, AtomicWaitKind::Wait32) 2951 } 2952 visit_memory_atomic_wait64(&mut self, arg: MemArg) -> Self::Output2953 fn visit_memory_atomic_wait64(&mut self, arg: MemArg) -> Self::Output { 2954 self.emit_atomic_wait(&arg, AtomicWaitKind::Wait64) 2955 } 2956 visit_memory_atomic_notify(&mut self, arg: MemArg) -> Self::Output2957 fn visit_memory_atomic_notify(&mut self, arg: MemArg) -> Self::Output { 2958 self.emit_atomic_notify(&arg) 2959 } 2960 visit_atomic_fence(&mut self) -> Self::Output2961 fn visit_atomic_fence(&mut self) -> Self::Output { 2962 self.masm.fence() 2963 } 2964 2965 wasmparser::for_each_visit_operator!(def_unsupported); 2966 } 2967 2968 impl<'a, 'translation, 'data, M> VisitSimdOperator<'a> 2969 for CodeGen<'a, 'translation, 'data, M, Emission> 2970 where 2971 M: MacroAssembler, 2972 { visit_v128_const(&mut self, val: V128) -> Self::Output2973 fn visit_v128_const(&mut self, val: V128) -> Self::Output { 2974 self.context.stack.push(Val::v128(val.i128())); 2975 Ok(()) 2976 } 2977 visit_v128_load(&mut self, memarg: MemArg) -> Self::Output2978 fn visit_v128_load(&mut self, memarg: MemArg) -> Self::Output { 2979 self.emit_wasm_load( 2980 &memarg, 2981 WasmValType::V128, 2982 LoadKind::Operand(OperandSize::S128), 2983 ) 2984 } 2985 visit_v128_store(&mut self, memarg: MemArg) -> Self::Output2986 fn visit_v128_store(&mut self, memarg: MemArg) -> Self::Output { 2987 self.emit_wasm_store(&memarg, StoreKind::Operand(OperandSize::S128)) 2988 } 2989 visit_v128_load8x8_s(&mut self, memarg: MemArg) -> Self::Output2990 fn visit_v128_load8x8_s(&mut self, memarg: MemArg) -> Self::Output { 2991 self.emit_wasm_load( 2992 &memarg, 2993 WasmValType::V128, 2994 LoadKind::VectorExtend(V128LoadExtendKind::E8x8S), 2995 ) 2996 } 2997 visit_v128_load8x8_u(&mut self, memarg: MemArg) -> Self::Output2998 fn visit_v128_load8x8_u(&mut self, memarg: MemArg) -> Self::Output { 2999 self.emit_wasm_load( 3000 &memarg, 3001 WasmValType::V128, 3002 LoadKind::VectorExtend(V128LoadExtendKind::E8x8U), 3003 ) 3004 } 3005 visit_v128_load16x4_s(&mut self, memarg: MemArg) -> Self::Output3006 fn visit_v128_load16x4_s(&mut self, memarg: MemArg) -> Self::Output { 3007 self.emit_wasm_load( 3008 &memarg, 3009 WasmValType::V128, 3010 LoadKind::VectorExtend(V128LoadExtendKind::E16x4S), 3011 ) 3012 } 3013 visit_v128_load16x4_u(&mut self, memarg: MemArg) -> Self::Output3014 fn visit_v128_load16x4_u(&mut self, memarg: MemArg) -> Self::Output { 3015 self.emit_wasm_load( 3016 &memarg, 3017 WasmValType::V128, 3018 LoadKind::VectorExtend(V128LoadExtendKind::E16x4U), 3019 ) 3020 } 3021 visit_v128_load32x2_s(&mut self, memarg: MemArg) -> Self::Output3022 fn visit_v128_load32x2_s(&mut self, memarg: MemArg) -> Self::Output { 3023 self.emit_wasm_load( 3024 &memarg, 3025 WasmValType::V128, 3026 LoadKind::VectorExtend(V128LoadExtendKind::E32x2S), 3027 ) 3028 } 3029 visit_v128_load32x2_u(&mut self, memarg: MemArg) -> Self::Output3030 fn visit_v128_load32x2_u(&mut self, memarg: MemArg) -> Self::Output { 3031 self.emit_wasm_load( 3032 &memarg, 3033 WasmValType::V128, 3034 LoadKind::VectorExtend(V128LoadExtendKind::E32x2U), 3035 ) 3036 } 3037 visit_v128_load8_splat(&mut self, memarg: MemArg) -> Self::Output3038 fn visit_v128_load8_splat(&mut self, memarg: MemArg) -> Self::Output { 3039 self.emit_wasm_load( 3040 &memarg, 3041 WasmValType::V128, 3042 LoadKind::Splat(SplatLoadKind::S8), 3043 ) 3044 } 3045 visit_v128_load16_splat(&mut self, memarg: MemArg) -> Self::Output3046 fn visit_v128_load16_splat(&mut self, memarg: MemArg) -> Self::Output { 3047 self.emit_wasm_load( 3048 &memarg, 3049 WasmValType::V128, 3050 LoadKind::Splat(SplatLoadKind::S16), 3051 ) 3052 } 3053 visit_v128_load32_splat(&mut self, memarg: MemArg) -> Self::Output3054 fn visit_v128_load32_splat(&mut self, memarg: MemArg) -> Self::Output { 3055 self.emit_wasm_load( 3056 &memarg, 3057 WasmValType::V128, 3058 LoadKind::Splat(SplatLoadKind::S32), 3059 ) 3060 } 3061 visit_v128_load64_splat(&mut self, memarg: MemArg) -> Self::Output3062 fn visit_v128_load64_splat(&mut self, memarg: MemArg) -> Self::Output { 3063 self.emit_wasm_load( 3064 &memarg, 3065 WasmValType::V128, 3066 LoadKind::Splat(SplatLoadKind::S64), 3067 ) 3068 } 3069 visit_i8x16_splat(&mut self) -> Self::Output3070 fn visit_i8x16_splat(&mut self) -> Self::Output { 3071 self.masm.splat(&mut self.context, SplatKind::I8x16) 3072 } 3073 visit_i16x8_splat(&mut self) -> Self::Output3074 fn visit_i16x8_splat(&mut self) -> Self::Output { 3075 self.masm.splat(&mut self.context, SplatKind::I16x8) 3076 } 3077 visit_i32x4_splat(&mut self) -> Self::Output3078 fn visit_i32x4_splat(&mut self) -> Self::Output { 3079 self.masm.splat(&mut self.context, SplatKind::I32x4) 3080 } 3081 visit_i64x2_splat(&mut self) -> Self::Output3082 fn visit_i64x2_splat(&mut self) -> Self::Output { 3083 self.masm.splat(&mut self.context, SplatKind::I64x2) 3084 } 3085 visit_f32x4_splat(&mut self) -> Self::Output3086 fn visit_f32x4_splat(&mut self) -> Self::Output { 3087 self.masm.splat(&mut self.context, SplatKind::F32x4) 3088 } 3089 visit_f64x2_splat(&mut self) -> Self::Output3090 fn visit_f64x2_splat(&mut self) -> Self::Output { 3091 self.masm.splat(&mut self.context, SplatKind::F64x2) 3092 } 3093 visit_i8x16_shuffle(&mut self, lanes: [u8; 16]) -> Self::Output3094 fn visit_i8x16_shuffle(&mut self, lanes: [u8; 16]) -> Self::Output { 3095 let rhs = self.context.pop_to_reg(self.masm, None)?; 3096 let lhs = self.context.pop_to_reg(self.masm, None)?; 3097 self.masm 3098 .shuffle(writable!(lhs.into()), lhs.into(), rhs.into(), lanes)?; 3099 self.context.stack.push(TypedReg::v128(lhs.into()).into()); 3100 self.context.free_reg(rhs); 3101 Ok(()) 3102 } 3103 visit_i8x16_swizzle(&mut self) -> Self::Output3104 fn visit_i8x16_swizzle(&mut self) -> Self::Output { 3105 let rhs = self.context.pop_to_reg(self.masm, None)?; 3106 let lhs = self.context.pop_to_reg(self.masm, None)?; 3107 self.masm 3108 .swizzle(writable!(lhs.into()), lhs.into(), rhs.into())?; 3109 self.context.stack.push(TypedReg::v128(lhs.into()).into()); 3110 self.context.free_reg(rhs); 3111 Ok(()) 3112 } 3113 visit_i8x16_extract_lane_s(&mut self, lane: u8) -> Self::Output3114 fn visit_i8x16_extract_lane_s(&mut self, lane: u8) -> Self::Output { 3115 self.context.extract_lane_op( 3116 self.masm, 3117 ExtractLaneKind::I8x16S, 3118 |masm, src, dst, kind| masm.extract_lane(src, dst, lane, kind), 3119 ) 3120 } 3121 visit_i8x16_extract_lane_u(&mut self, lane: u8) -> Self::Output3122 fn visit_i8x16_extract_lane_u(&mut self, lane: u8) -> Self::Output { 3123 self.context.extract_lane_op( 3124 self.masm, 3125 ExtractLaneKind::I8x16U, 3126 |masm, src, dst, kind| masm.extract_lane(src, dst, lane, kind), 3127 ) 3128 } 3129 visit_i16x8_extract_lane_s(&mut self, lane: u8) -> Self::Output3130 fn visit_i16x8_extract_lane_s(&mut self, lane: u8) -> Self::Output { 3131 self.context.extract_lane_op( 3132 self.masm, 3133 ExtractLaneKind::I16x8S, 3134 |masm, src, dst, kind| masm.extract_lane(src, dst, lane, kind), 3135 ) 3136 } 3137 visit_i16x8_extract_lane_u(&mut self, lane: u8) -> Self::Output3138 fn visit_i16x8_extract_lane_u(&mut self, lane: u8) -> Self::Output { 3139 self.context.extract_lane_op( 3140 self.masm, 3141 ExtractLaneKind::I16x8U, 3142 |masm, src, dst, kind| masm.extract_lane(src, dst, lane, kind), 3143 ) 3144 } 3145 visit_i32x4_extract_lane(&mut self, lane: u8) -> Self::Output3146 fn visit_i32x4_extract_lane(&mut self, lane: u8) -> Self::Output { 3147 self.context 3148 .extract_lane_op(self.masm, ExtractLaneKind::I32x4, |masm, src, dst, kind| { 3149 masm.extract_lane(src, dst, lane, kind) 3150 }) 3151 } 3152 visit_i64x2_extract_lane(&mut self, lane: u8) -> Self::Output3153 fn visit_i64x2_extract_lane(&mut self, lane: u8) -> Self::Output { 3154 self.context 3155 .extract_lane_op(self.masm, ExtractLaneKind::I64x2, |masm, src, dst, kind| { 3156 masm.extract_lane(src, dst, lane, kind) 3157 }) 3158 } 3159 visit_f32x4_extract_lane(&mut self, lane: u8) -> Self::Output3160 fn visit_f32x4_extract_lane(&mut self, lane: u8) -> Self::Output { 3161 self.context 3162 .extract_lane_op(self.masm, ExtractLaneKind::F32x4, |masm, src, dst, kind| { 3163 masm.extract_lane(src, dst, lane, kind) 3164 }) 3165 } 3166 visit_f64x2_extract_lane(&mut self, lane: u8) -> Self::Output3167 fn visit_f64x2_extract_lane(&mut self, lane: u8) -> Self::Output { 3168 self.context 3169 .extract_lane_op(self.masm, ExtractLaneKind::F64x2, |masm, src, dst, kind| { 3170 masm.extract_lane(src, dst, lane, kind) 3171 }) 3172 } 3173 visit_i8x16_eq(&mut self) -> Self::Output3174 fn visit_i8x16_eq(&mut self) -> Self::Output { 3175 self.context 3176 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3177 masm.v128_eq(writable!(dst), dst, src, VectorEqualityKind::I8x16)?; 3178 Ok(TypedReg::v128(dst)) 3179 }) 3180 } 3181 visit_i16x8_eq(&mut self) -> Self::Output3182 fn visit_i16x8_eq(&mut self) -> Self::Output { 3183 self.context 3184 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3185 masm.v128_eq(writable!(dst), dst, src, VectorEqualityKind::I16x8)?; 3186 Ok(TypedReg::v128(dst)) 3187 }) 3188 } 3189 visit_i32x4_eq(&mut self) -> Self::Output3190 fn visit_i32x4_eq(&mut self) -> Self::Output { 3191 self.context 3192 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3193 masm.v128_eq(writable!(dst), dst, src, VectorEqualityKind::I32x4)?; 3194 Ok(TypedReg::v128(dst)) 3195 }) 3196 } 3197 visit_i64x2_eq(&mut self) -> Self::Output3198 fn visit_i64x2_eq(&mut self) -> Self::Output { 3199 self.context 3200 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3201 masm.v128_eq(writable!(dst), dst, src, VectorEqualityKind::I64x2)?; 3202 Ok(TypedReg::v128(dst)) 3203 }) 3204 } 3205 visit_f32x4_eq(&mut self) -> Self::Output3206 fn visit_f32x4_eq(&mut self) -> Self::Output { 3207 self.context 3208 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3209 masm.v128_eq(writable!(dst), dst, src, VectorEqualityKind::F32x4)?; 3210 Ok(TypedReg::v128(dst)) 3211 }) 3212 } 3213 visit_f64x2_eq(&mut self) -> Self::Output3214 fn visit_f64x2_eq(&mut self) -> Self::Output { 3215 self.context 3216 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3217 masm.v128_eq(writable!(dst), dst, src, VectorEqualityKind::F64x2)?; 3218 Ok(TypedReg::v128(dst)) 3219 }) 3220 } 3221 visit_i8x16_ne(&mut self) -> Self::Output3222 fn visit_i8x16_ne(&mut self) -> Self::Output { 3223 self.context 3224 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3225 masm.v128_ne(writable!(dst), dst, src, VectorEqualityKind::I8x16)?; 3226 Ok(TypedReg::v128(dst)) 3227 }) 3228 } 3229 visit_i16x8_ne(&mut self) -> Self::Output3230 fn visit_i16x8_ne(&mut self) -> Self::Output { 3231 self.context 3232 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3233 masm.v128_ne(writable!(dst), dst, src, VectorEqualityKind::I16x8)?; 3234 Ok(TypedReg::v128(dst)) 3235 }) 3236 } 3237 visit_i32x4_ne(&mut self) -> Self::Output3238 fn visit_i32x4_ne(&mut self) -> Self::Output { 3239 self.context 3240 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3241 masm.v128_ne(writable!(dst), dst, src, VectorEqualityKind::I32x4)?; 3242 Ok(TypedReg::v128(dst)) 3243 }) 3244 } 3245 visit_i64x2_ne(&mut self) -> Self::Output3246 fn visit_i64x2_ne(&mut self) -> Self::Output { 3247 self.context 3248 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3249 masm.v128_ne(writable!(dst), dst, src, VectorEqualityKind::I64x2)?; 3250 Ok(TypedReg::v128(dst)) 3251 }) 3252 } 3253 visit_f32x4_ne(&mut self) -> Self::Output3254 fn visit_f32x4_ne(&mut self) -> Self::Output { 3255 self.context 3256 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3257 masm.v128_ne(writable!(dst), dst, src, VectorEqualityKind::F32x4)?; 3258 Ok(TypedReg::v128(dst)) 3259 }) 3260 } 3261 visit_f64x2_ne(&mut self) -> Self::Output3262 fn visit_f64x2_ne(&mut self) -> Self::Output { 3263 self.context 3264 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3265 masm.v128_ne(writable!(dst), dst, src, VectorEqualityKind::F64x2)?; 3266 Ok(TypedReg::v128(dst)) 3267 }) 3268 } 3269 visit_i8x16_lt_s(&mut self) -> Self::Output3270 fn visit_i8x16_lt_s(&mut self) -> Self::Output { 3271 self.context 3272 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3273 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I8x16S)?; 3274 Ok(TypedReg::v128(dst)) 3275 }) 3276 } 3277 visit_i8x16_lt_u(&mut self) -> Self::Output3278 fn visit_i8x16_lt_u(&mut self) -> Self::Output { 3279 self.context 3280 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3281 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I8x16U)?; 3282 Ok(TypedReg::v128(dst)) 3283 }) 3284 } 3285 visit_i16x8_lt_s(&mut self) -> Self::Output3286 fn visit_i16x8_lt_s(&mut self) -> Self::Output { 3287 self.context 3288 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3289 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I16x8S)?; 3290 Ok(TypedReg::v128(dst)) 3291 }) 3292 } 3293 visit_i16x8_lt_u(&mut self) -> Self::Output3294 fn visit_i16x8_lt_u(&mut self) -> Self::Output { 3295 self.context 3296 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3297 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I16x8U)?; 3298 Ok(TypedReg::v128(dst)) 3299 }) 3300 } 3301 visit_i32x4_lt_s(&mut self) -> Self::Output3302 fn visit_i32x4_lt_s(&mut self) -> Self::Output { 3303 self.context 3304 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3305 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I32x4S)?; 3306 Ok(TypedReg::v128(dst)) 3307 }) 3308 } 3309 visit_i32x4_lt_u(&mut self) -> Self::Output3310 fn visit_i32x4_lt_u(&mut self) -> Self::Output { 3311 self.context 3312 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3313 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I32x4U)?; 3314 Ok(TypedReg::v128(dst)) 3315 }) 3316 } 3317 visit_i64x2_lt_s(&mut self) -> Self::Output3318 fn visit_i64x2_lt_s(&mut self) -> Self::Output { 3319 self.context 3320 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3321 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::I64x2S)?; 3322 Ok(TypedReg::v128(dst)) 3323 }) 3324 } 3325 visit_f32x4_lt(&mut self) -> Self::Output3326 fn visit_f32x4_lt(&mut self) -> Self::Output { 3327 self.context 3328 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3329 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::F32x4)?; 3330 Ok(TypedReg::v128(dst)) 3331 }) 3332 } 3333 visit_f64x2_lt(&mut self) -> Self::Output3334 fn visit_f64x2_lt(&mut self) -> Self::Output { 3335 self.context 3336 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3337 masm.v128_lt(writable!(dst), dst, src, VectorCompareKind::F64x2)?; 3338 Ok(TypedReg::v128(dst)) 3339 }) 3340 } 3341 visit_i8x16_le_s(&mut self) -> Self::Output3342 fn visit_i8x16_le_s(&mut self) -> Self::Output { 3343 self.context 3344 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3345 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I8x16S)?; 3346 Ok(TypedReg::v128(dst)) 3347 }) 3348 } 3349 visit_i8x16_le_u(&mut self) -> Self::Output3350 fn visit_i8x16_le_u(&mut self) -> Self::Output { 3351 self.context 3352 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3353 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I8x16U)?; 3354 Ok(TypedReg::v128(dst)) 3355 }) 3356 } 3357 visit_i16x8_le_s(&mut self) -> Self::Output3358 fn visit_i16x8_le_s(&mut self) -> Self::Output { 3359 self.context 3360 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3361 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I16x8S)?; 3362 Ok(TypedReg::v128(dst)) 3363 }) 3364 } 3365 visit_i16x8_le_u(&mut self) -> Self::Output3366 fn visit_i16x8_le_u(&mut self) -> Self::Output { 3367 self.context 3368 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3369 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I16x8U)?; 3370 Ok(TypedReg::v128(dst)) 3371 }) 3372 } 3373 visit_i32x4_le_s(&mut self) -> Self::Output3374 fn visit_i32x4_le_s(&mut self) -> Self::Output { 3375 self.context 3376 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3377 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I32x4S)?; 3378 Ok(TypedReg::v128(dst)) 3379 }) 3380 } 3381 visit_i32x4_le_u(&mut self) -> Self::Output3382 fn visit_i32x4_le_u(&mut self) -> Self::Output { 3383 self.context 3384 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3385 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I32x4U)?; 3386 Ok(TypedReg::v128(dst)) 3387 }) 3388 } 3389 visit_i64x2_le_s(&mut self) -> Self::Output3390 fn visit_i64x2_le_s(&mut self) -> Self::Output { 3391 self.context 3392 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3393 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::I64x2S)?; 3394 Ok(TypedReg::v128(dst)) 3395 }) 3396 } 3397 visit_f32x4_le(&mut self) -> Self::Output3398 fn visit_f32x4_le(&mut self) -> Self::Output { 3399 self.context 3400 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3401 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::F32x4)?; 3402 Ok(TypedReg::v128(dst)) 3403 }) 3404 } 3405 visit_f64x2_le(&mut self) -> Self::Output3406 fn visit_f64x2_le(&mut self) -> Self::Output { 3407 self.context 3408 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3409 masm.v128_le(writable!(dst), dst, src, VectorCompareKind::F64x2)?; 3410 Ok(TypedReg::v128(dst)) 3411 }) 3412 } 3413 visit_i8x16_gt_s(&mut self) -> Self::Output3414 fn visit_i8x16_gt_s(&mut self) -> Self::Output { 3415 self.context 3416 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3417 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I8x16S)?; 3418 Ok(TypedReg::v128(dst)) 3419 }) 3420 } 3421 visit_i8x16_gt_u(&mut self) -> Self::Output3422 fn visit_i8x16_gt_u(&mut self) -> Self::Output { 3423 self.context 3424 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3425 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I8x16U)?; 3426 Ok(TypedReg::v128(dst)) 3427 }) 3428 } 3429 visit_i16x8_gt_s(&mut self) -> Self::Output3430 fn visit_i16x8_gt_s(&mut self) -> Self::Output { 3431 self.context 3432 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3433 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I16x8S)?; 3434 Ok(TypedReg::v128(dst)) 3435 }) 3436 } 3437 visit_i16x8_gt_u(&mut self) -> Self::Output3438 fn visit_i16x8_gt_u(&mut self) -> Self::Output { 3439 self.context 3440 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3441 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I16x8U)?; 3442 Ok(TypedReg::v128(dst)) 3443 }) 3444 } 3445 visit_i32x4_gt_s(&mut self) -> Self::Output3446 fn visit_i32x4_gt_s(&mut self) -> Self::Output { 3447 self.context 3448 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3449 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I32x4S)?; 3450 Ok(TypedReg::v128(dst)) 3451 }) 3452 } 3453 visit_i32x4_gt_u(&mut self) -> Self::Output3454 fn visit_i32x4_gt_u(&mut self) -> Self::Output { 3455 self.context 3456 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3457 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I32x4U)?; 3458 Ok(TypedReg::v128(dst)) 3459 }) 3460 } 3461 visit_i64x2_gt_s(&mut self) -> Self::Output3462 fn visit_i64x2_gt_s(&mut self) -> Self::Output { 3463 self.context 3464 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3465 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::I64x2S)?; 3466 Ok(TypedReg::v128(dst)) 3467 }) 3468 } 3469 visit_f32x4_gt(&mut self) -> Self::Output3470 fn visit_f32x4_gt(&mut self) -> Self::Output { 3471 self.context 3472 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3473 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::F32x4)?; 3474 Ok(TypedReg::v128(dst)) 3475 }) 3476 } 3477 visit_f64x2_gt(&mut self) -> Self::Output3478 fn visit_f64x2_gt(&mut self) -> Self::Output { 3479 self.context 3480 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3481 masm.v128_gt(writable!(dst), dst, src, VectorCompareKind::F64x2)?; 3482 Ok(TypedReg::v128(dst)) 3483 }) 3484 } 3485 visit_i8x16_ge_s(&mut self) -> Self::Output3486 fn visit_i8x16_ge_s(&mut self) -> Self::Output { 3487 self.context 3488 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3489 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I8x16S)?; 3490 Ok(TypedReg::v128(dst)) 3491 }) 3492 } 3493 visit_i8x16_ge_u(&mut self) -> Self::Output3494 fn visit_i8x16_ge_u(&mut self) -> Self::Output { 3495 self.context 3496 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3497 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I8x16U)?; 3498 Ok(TypedReg::v128(dst)) 3499 }) 3500 } 3501 visit_i16x8_ge_s(&mut self) -> Self::Output3502 fn visit_i16x8_ge_s(&mut self) -> Self::Output { 3503 self.context 3504 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3505 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I16x8S)?; 3506 Ok(TypedReg::v128(dst)) 3507 }) 3508 } 3509 visit_i16x8_ge_u(&mut self) -> Self::Output3510 fn visit_i16x8_ge_u(&mut self) -> Self::Output { 3511 self.context 3512 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3513 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I16x8U)?; 3514 Ok(TypedReg::v128(dst)) 3515 }) 3516 } 3517 visit_i32x4_ge_s(&mut self) -> Self::Output3518 fn visit_i32x4_ge_s(&mut self) -> Self::Output { 3519 self.context 3520 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3521 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I32x4S)?; 3522 Ok(TypedReg::v128(dst)) 3523 }) 3524 } 3525 visit_i32x4_ge_u(&mut self) -> Self::Output3526 fn visit_i32x4_ge_u(&mut self) -> Self::Output { 3527 self.context 3528 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3529 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I32x4U)?; 3530 Ok(TypedReg::v128(dst)) 3531 }) 3532 } 3533 visit_i64x2_ge_s(&mut self) -> Self::Output3534 fn visit_i64x2_ge_s(&mut self) -> Self::Output { 3535 self.context 3536 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3537 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::I64x2S)?; 3538 Ok(TypedReg::v128(dst)) 3539 }) 3540 } 3541 visit_f32x4_ge(&mut self) -> Self::Output3542 fn visit_f32x4_ge(&mut self) -> Self::Output { 3543 self.context 3544 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3545 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::F32x4)?; 3546 Ok(TypedReg::v128(dst)) 3547 }) 3548 } 3549 visit_f64x2_ge(&mut self) -> Self::Output3550 fn visit_f64x2_ge(&mut self) -> Self::Output { 3551 self.context 3552 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3553 masm.v128_ge(writable!(dst), dst, src, VectorCompareKind::F64x2)?; 3554 Ok(TypedReg::v128(dst)) 3555 }) 3556 } 3557 visit_i8x16_replace_lane(&mut self, lane: u8) -> Self::Output3558 fn visit_i8x16_replace_lane(&mut self, lane: u8) -> Self::Output { 3559 self.context 3560 .replace_lane_op(self.masm, ReplaceLaneKind::I8x16, |masm, src, dst, kind| { 3561 masm.replace_lane(src, dst, lane, kind) 3562 }) 3563 } 3564 visit_i16x8_replace_lane(&mut self, lane: u8) -> Self::Output3565 fn visit_i16x8_replace_lane(&mut self, lane: u8) -> Self::Output { 3566 self.context 3567 .replace_lane_op(self.masm, ReplaceLaneKind::I16x8, |masm, src, dst, kind| { 3568 masm.replace_lane(src, dst, lane, kind) 3569 }) 3570 } 3571 visit_i32x4_replace_lane(&mut self, lane: u8) -> Self::Output3572 fn visit_i32x4_replace_lane(&mut self, lane: u8) -> Self::Output { 3573 self.context 3574 .replace_lane_op(self.masm, ReplaceLaneKind::I32x4, |masm, src, dst, kind| { 3575 masm.replace_lane(src, dst, lane, kind) 3576 }) 3577 } 3578 visit_i64x2_replace_lane(&mut self, lane: u8) -> Self::Output3579 fn visit_i64x2_replace_lane(&mut self, lane: u8) -> Self::Output { 3580 self.context 3581 .replace_lane_op(self.masm, ReplaceLaneKind::I64x2, |masm, src, dst, kind| { 3582 masm.replace_lane(src, dst, lane, kind) 3583 }) 3584 } 3585 visit_f32x4_replace_lane(&mut self, lane: u8) -> Self::Output3586 fn visit_f32x4_replace_lane(&mut self, lane: u8) -> Self::Output { 3587 self.context 3588 .replace_lane_op(self.masm, ReplaceLaneKind::F32x4, |masm, src, dst, kind| { 3589 masm.replace_lane(src, dst, lane, kind) 3590 }) 3591 } 3592 visit_f64x2_replace_lane(&mut self, lane: u8) -> Self::Output3593 fn visit_f64x2_replace_lane(&mut self, lane: u8) -> Self::Output { 3594 self.context 3595 .replace_lane_op(self.masm, ReplaceLaneKind::F64x2, |masm, src, dst, kind| { 3596 masm.replace_lane(src, dst, lane, kind) 3597 }) 3598 } 3599 visit_v128_not(&mut self) -> Self::Output3600 fn visit_v128_not(&mut self) -> Self::Output { 3601 self.context.unop(self.masm, |masm, reg| { 3602 masm.v128_not(writable!(reg))?; 3603 Ok(TypedReg::new(WasmValType::V128, reg)) 3604 }) 3605 } 3606 visit_v128_and(&mut self) -> Self::Output3607 fn visit_v128_and(&mut self) -> Self::Output { 3608 self.context 3609 .binop(self.masm, OperandSize::S128, |masm, dst, src, _size| { 3610 masm.v128_and(dst, src, writable!(dst))?; 3611 Ok(TypedReg::new(WasmValType::V128, dst)) 3612 }) 3613 } 3614 visit_v128_andnot(&mut self) -> Self::Output3615 fn visit_v128_andnot(&mut self) -> Self::Output { 3616 self.context 3617 .binop(self.masm, OperandSize::S128, |masm, dst, src, _size| { 3618 // careful here: and_not is *not* commutative: dst = !src1 & src2 3619 masm.v128_and_not(src, dst, writable!(dst))?; 3620 Ok(TypedReg::new(WasmValType::V128, dst)) 3621 }) 3622 } 3623 visit_v128_or(&mut self) -> Self::Output3624 fn visit_v128_or(&mut self) -> Self::Output { 3625 self.context 3626 .binop(self.masm, OperandSize::S128, |masm, dst, src, _size| { 3627 // careful here: and_not is *not* commutative: dst = !src1 & src2 3628 masm.v128_or(src, dst, writable!(dst))?; 3629 Ok(TypedReg::new(WasmValType::V128, dst)) 3630 }) 3631 } 3632 visit_v128_xor(&mut self) -> Self::Output3633 fn visit_v128_xor(&mut self) -> Self::Output { 3634 self.context 3635 .binop(self.masm, OperandSize::S128, |masm, dst, src, _size| { 3636 // careful here: and_not is *not* commutative: dst = !src1 & src2 3637 masm.v128_xor(src, dst, writable!(dst))?; 3638 Ok(TypedReg::new(WasmValType::V128, dst)) 3639 }) 3640 } 3641 visit_v128_bitselect(&mut self) -> Self::Output3642 fn visit_v128_bitselect(&mut self) -> Self::Output { 3643 let mask = self.context.pop_to_reg(self.masm, None)?; 3644 let op2 = self.context.pop_to_reg(self.masm, None)?; 3645 let op1 = self.context.pop_to_reg(self.masm, None)?; 3646 let dst = self.context.any_fpr(self.masm)?; 3647 3648 // careful here: bitselect is *not* commutative. 3649 self.masm 3650 .v128_bitselect(op1.reg, op2.reg, mask.reg, writable!(dst))?; 3651 3652 self.context 3653 .stack 3654 .push(TypedReg::new(WasmValType::V128, dst).into()); 3655 self.context.free_reg(op1); 3656 self.context.free_reg(op2); 3657 self.context.free_reg(mask); 3658 3659 Ok(()) 3660 } 3661 visit_v128_any_true(&mut self) -> Self::Output3662 fn visit_v128_any_true(&mut self) -> Self::Output { 3663 let src = self.context.pop_to_reg(self.masm, None)?; 3664 let dst = self.context.any_gpr(self.masm)?; 3665 3666 self.masm.v128_any_true(src.reg, writable!(dst))?; 3667 3668 self.context 3669 .stack 3670 .push(TypedReg::new(WasmValType::I32, dst).into()); 3671 self.context.free_reg(src); 3672 3673 Ok(()) 3674 } 3675 visit_v128_load8_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output3676 fn visit_v128_load8_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output { 3677 self.emit_wasm_load( 3678 &arg, 3679 WasmValType::V128, 3680 LoadKind::vector_lane(lane, OperandSize::S8), 3681 ) 3682 } 3683 visit_v128_load16_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output3684 fn visit_v128_load16_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output { 3685 self.emit_wasm_load( 3686 &arg, 3687 WasmValType::V128, 3688 LoadKind::vector_lane(lane, OperandSize::S16), 3689 ) 3690 } 3691 visit_v128_load32_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output3692 fn visit_v128_load32_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output { 3693 self.emit_wasm_load( 3694 &arg, 3695 WasmValType::V128, 3696 LoadKind::vector_lane(lane, OperandSize::S32), 3697 ) 3698 } 3699 visit_v128_load64_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output3700 fn visit_v128_load64_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output { 3701 self.emit_wasm_load( 3702 &arg, 3703 WasmValType::V128, 3704 LoadKind::vector_lane(lane, OperandSize::S64), 3705 ) 3706 } 3707 visit_v128_store8_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output3708 fn visit_v128_store8_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output { 3709 self.emit_wasm_store(&arg, StoreKind::vector_lane(lane, OperandSize::S8)) 3710 } 3711 visit_v128_store16_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output3712 fn visit_v128_store16_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output { 3713 self.emit_wasm_store(&arg, StoreKind::vector_lane(lane, OperandSize::S16)) 3714 } 3715 visit_v128_store32_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output3716 fn visit_v128_store32_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output { 3717 self.emit_wasm_store(&arg, StoreKind::vector_lane(lane, OperandSize::S32)) 3718 } 3719 visit_v128_store64_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output3720 fn visit_v128_store64_lane(&mut self, arg: MemArg, lane: u8) -> Self::Output { 3721 self.emit_wasm_store(&arg, StoreKind::vector_lane(lane, OperandSize::S64)) 3722 } 3723 visit_f32x4_convert_i32x4_s(&mut self) -> Self::Output3724 fn visit_f32x4_convert_i32x4_s(&mut self) -> Self::Output { 3725 self.context.unop(self.masm, |masm, reg| { 3726 masm.v128_convert(reg, writable!(reg), V128ConvertKind::I32x4S)?; 3727 Ok(TypedReg::v128(reg)) 3728 }) 3729 } 3730 visit_f32x4_convert_i32x4_u(&mut self) -> Self::Output3731 fn visit_f32x4_convert_i32x4_u(&mut self) -> Self::Output { 3732 self.context.unop(self.masm, |masm, reg| { 3733 masm.v128_convert(reg, writable!(reg), V128ConvertKind::I32x4U)?; 3734 Ok(TypedReg::v128(reg)) 3735 }) 3736 } 3737 visit_f64x2_convert_low_i32x4_s(&mut self) -> Self::Output3738 fn visit_f64x2_convert_low_i32x4_s(&mut self) -> Self::Output { 3739 self.context.unop(self.masm, |masm, reg| { 3740 masm.v128_convert(reg, writable!(reg), V128ConvertKind::I32x4LowS)?; 3741 Ok(TypedReg::v128(reg)) 3742 }) 3743 } 3744 visit_f64x2_convert_low_i32x4_u(&mut self) -> Self::Output3745 fn visit_f64x2_convert_low_i32x4_u(&mut self) -> Self::Output { 3746 self.context.unop(self.masm, |masm, reg| { 3747 masm.v128_convert(reg, writable!(reg), V128ConvertKind::I32x4LowU)?; 3748 Ok(TypedReg::v128(reg)) 3749 }) 3750 } 3751 visit_i8x16_narrow_i16x8_s(&mut self) -> Self::Output3752 fn visit_i8x16_narrow_i16x8_s(&mut self) -> Self::Output { 3753 self.context 3754 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3755 masm.v128_narrow(dst, src, writable!(dst), V128NarrowKind::I16x8S)?; 3756 Ok(TypedReg::v128(dst)) 3757 }) 3758 } 3759 visit_i8x16_narrow_i16x8_u(&mut self) -> Self::Output3760 fn visit_i8x16_narrow_i16x8_u(&mut self) -> Self::Output { 3761 self.context 3762 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3763 masm.v128_narrow(dst, src, writable!(dst), V128NarrowKind::I16x8U)?; 3764 Ok(TypedReg::v128(dst)) 3765 }) 3766 } 3767 visit_i16x8_narrow_i32x4_s(&mut self) -> Self::Output3768 fn visit_i16x8_narrow_i32x4_s(&mut self) -> Self::Output { 3769 self.context 3770 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3771 masm.v128_narrow(dst, src, writable!(dst), V128NarrowKind::I32x4S)?; 3772 Ok(TypedReg::v128(dst)) 3773 }) 3774 } 3775 visit_i16x8_narrow_i32x4_u(&mut self) -> Self::Output3776 fn visit_i16x8_narrow_i32x4_u(&mut self) -> Self::Output { 3777 self.context 3778 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3779 masm.v128_narrow(dst, src, writable!(dst), V128NarrowKind::I32x4U)?; 3780 Ok(TypedReg::v128(dst)) 3781 }) 3782 } 3783 visit_f32x4_demote_f64x2_zero(&mut self) -> Self::Output3784 fn visit_f32x4_demote_f64x2_zero(&mut self) -> Self::Output { 3785 self.context.unop(self.masm, |masm, reg| { 3786 masm.v128_demote(reg, writable!(reg))?; 3787 Ok(TypedReg::v128(reg)) 3788 }) 3789 } 3790 visit_f64x2_promote_low_f32x4(&mut self) -> Self::Output3791 fn visit_f64x2_promote_low_f32x4(&mut self) -> Self::Output { 3792 self.context.unop(self.masm, |masm, reg| { 3793 masm.v128_promote(reg, writable!(reg))?; 3794 Ok(TypedReg::v128(reg)) 3795 }) 3796 } 3797 visit_i16x8_extend_low_i8x16_s(&mut self) -> Self::Output3798 fn visit_i16x8_extend_low_i8x16_s(&mut self) -> Self::Output { 3799 self.context.unop(self.masm, |masm, reg| { 3800 masm.v128_extend(reg, writable!(reg), V128ExtendKind::LowI8x16S)?; 3801 Ok(TypedReg::v128(reg)) 3802 }) 3803 } 3804 visit_i16x8_extend_high_i8x16_s(&mut self) -> Self::Output3805 fn visit_i16x8_extend_high_i8x16_s(&mut self) -> Self::Output { 3806 self.context.unop(self.masm, |masm, reg| { 3807 masm.v128_extend(reg, writable!(reg), V128ExtendKind::HighI8x16S)?; 3808 Ok(TypedReg::v128(reg)) 3809 }) 3810 } 3811 visit_i16x8_extend_low_i8x16_u(&mut self) -> Self::Output3812 fn visit_i16x8_extend_low_i8x16_u(&mut self) -> Self::Output { 3813 self.context.unop(self.masm, |masm, reg| { 3814 masm.v128_extend(reg, writable!(reg), V128ExtendKind::LowI8x16U)?; 3815 Ok(TypedReg::v128(reg)) 3816 }) 3817 } 3818 visit_i16x8_extend_high_i8x16_u(&mut self) -> Self::Output3819 fn visit_i16x8_extend_high_i8x16_u(&mut self) -> Self::Output { 3820 self.context.unop(self.masm, |masm, reg| { 3821 masm.v128_extend(reg, writable!(reg), V128ExtendKind::HighI8x16U)?; 3822 Ok(TypedReg::v128(reg)) 3823 }) 3824 } 3825 visit_i32x4_extend_low_i16x8_s(&mut self) -> Self::Output3826 fn visit_i32x4_extend_low_i16x8_s(&mut self) -> Self::Output { 3827 self.context.unop(self.masm, |masm, reg| { 3828 masm.v128_extend(reg, writable!(reg), V128ExtendKind::LowI16x8S)?; 3829 Ok(TypedReg::v128(reg)) 3830 }) 3831 } 3832 visit_i32x4_extend_high_i16x8_s(&mut self) -> Self::Output3833 fn visit_i32x4_extend_high_i16x8_s(&mut self) -> Self::Output { 3834 self.context.unop(self.masm, |masm, reg| { 3835 masm.v128_extend(reg, writable!(reg), V128ExtendKind::HighI16x8S)?; 3836 Ok(TypedReg::v128(reg)) 3837 }) 3838 } 3839 visit_i32x4_extend_low_i16x8_u(&mut self) -> Self::Output3840 fn visit_i32x4_extend_low_i16x8_u(&mut self) -> Self::Output { 3841 self.context.unop(self.masm, |masm, reg| { 3842 masm.v128_extend(reg, writable!(reg), V128ExtendKind::LowI16x8U)?; 3843 Ok(TypedReg::v128(reg)) 3844 }) 3845 } 3846 visit_i32x4_extend_high_i16x8_u(&mut self) -> Self::Output3847 fn visit_i32x4_extend_high_i16x8_u(&mut self) -> Self::Output { 3848 self.context.unop(self.masm, |masm, reg| { 3849 masm.v128_extend(reg, writable!(reg), V128ExtendKind::HighI16x8U)?; 3850 Ok(TypedReg::v128(reg)) 3851 }) 3852 } 3853 visit_i64x2_extend_low_i32x4_s(&mut self) -> Self::Output3854 fn visit_i64x2_extend_low_i32x4_s(&mut self) -> Self::Output { 3855 self.context.unop(self.masm, |masm, reg| { 3856 masm.v128_extend(reg, writable!(reg), V128ExtendKind::LowI32x4S)?; 3857 Ok(TypedReg::v128(reg)) 3858 }) 3859 } 3860 visit_i64x2_extend_high_i32x4_s(&mut self) -> Self::Output3861 fn visit_i64x2_extend_high_i32x4_s(&mut self) -> Self::Output { 3862 self.context.unop(self.masm, |masm, reg| { 3863 masm.v128_extend(reg, writable!(reg), V128ExtendKind::HighI32x4S)?; 3864 Ok(TypedReg::v128(reg)) 3865 }) 3866 } 3867 visit_i64x2_extend_low_i32x4_u(&mut self) -> Self::Output3868 fn visit_i64x2_extend_low_i32x4_u(&mut self) -> Self::Output { 3869 self.context.unop(self.masm, |masm, reg| { 3870 masm.v128_extend(reg, writable!(reg), V128ExtendKind::LowI32x4U)?; 3871 Ok(TypedReg::v128(reg)) 3872 }) 3873 } 3874 visit_i64x2_extend_high_i32x4_u(&mut self) -> Self::Output3875 fn visit_i64x2_extend_high_i32x4_u(&mut self) -> Self::Output { 3876 self.context.unop(self.masm, |masm, reg| { 3877 masm.v128_extend(reg, writable!(reg), V128ExtendKind::HighI32x4U)?; 3878 Ok(TypedReg::v128(reg)) 3879 }) 3880 } 3881 visit_i8x16_add(&mut self) -> Self::Output3882 fn visit_i8x16_add(&mut self) -> Self::Output { 3883 self.context 3884 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3885 masm.v128_add(dst, src, writable!(dst), V128AddKind::I8x16)?; 3886 Ok(TypedReg::new(WasmValType::V128, dst)) 3887 }) 3888 } 3889 visit_i16x8_add(&mut self) -> Self::Output3890 fn visit_i16x8_add(&mut self) -> Self::Output { 3891 self.context 3892 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3893 masm.v128_add(dst, src, writable!(dst), V128AddKind::I16x8)?; 3894 Ok(TypedReg::new(WasmValType::V128, dst)) 3895 }) 3896 } 3897 visit_i32x4_add(&mut self) -> Self::Output3898 fn visit_i32x4_add(&mut self) -> Self::Output { 3899 self.context 3900 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3901 masm.v128_add(dst, src, writable!(dst), V128AddKind::I32x4)?; 3902 Ok(TypedReg::new(WasmValType::V128, dst)) 3903 }) 3904 } 3905 visit_i64x2_add(&mut self) -> Self::Output3906 fn visit_i64x2_add(&mut self) -> Self::Output { 3907 self.context 3908 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3909 masm.v128_add(dst, src, writable!(dst), V128AddKind::I64x2)?; 3910 Ok(TypedReg::new(WasmValType::V128, dst)) 3911 }) 3912 } 3913 visit_i8x16_sub(&mut self) -> Self::Output3914 fn visit_i8x16_sub(&mut self) -> Self::Output { 3915 self.context 3916 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3917 masm.v128_sub(dst, src, writable!(dst), V128SubKind::I8x16)?; 3918 Ok(TypedReg::new(WasmValType::V128, dst)) 3919 }) 3920 } 3921 visit_i16x8_sub(&mut self) -> Self::Output3922 fn visit_i16x8_sub(&mut self) -> Self::Output { 3923 self.context 3924 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3925 masm.v128_sub(dst, src, writable!(dst), V128SubKind::I16x8)?; 3926 Ok(TypedReg::new(WasmValType::V128, dst)) 3927 }) 3928 } 3929 visit_i32x4_sub(&mut self) -> Self::Output3930 fn visit_i32x4_sub(&mut self) -> Self::Output { 3931 self.context 3932 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 3933 masm.v128_sub(dst, src, writable!(dst), V128SubKind::I32x4)?; 3934 Ok(TypedReg::new(WasmValType::V128, dst)) 3935 }) 3936 } 3937 visit_i64x2_sub(&mut self) -> Self::Output3938 fn visit_i64x2_sub(&mut self) -> Self::Output { 3939 self.context 3940 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 3941 masm.v128_sub(dst, src, writable!(dst), V128SubKind::I64x2)?; 3942 Ok(TypedReg::new(WasmValType::V128, dst)) 3943 }) 3944 } 3945 visit_i16x8_mul(&mut self) -> Self::Output3946 fn visit_i16x8_mul(&mut self) -> Self::Output { 3947 self.masm.v128_mul(&mut self.context, V128MulKind::I16x8) 3948 } 3949 visit_i32x4_mul(&mut self) -> Self::Output3950 fn visit_i32x4_mul(&mut self) -> Self::Output { 3951 self.masm.v128_mul(&mut self.context, V128MulKind::I32x4) 3952 } 3953 visit_i64x2_mul(&mut self) -> Self::Output3954 fn visit_i64x2_mul(&mut self) -> Self::Output { 3955 self.masm.v128_mul(&mut self.context, V128MulKind::I64x2) 3956 } 3957 visit_i8x16_add_sat_s(&mut self) -> Self::Output3958 fn visit_i8x16_add_sat_s(&mut self) -> Self::Output { 3959 self.context 3960 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3961 masm.v128_add(dst, src, writable!(dst), V128AddKind::I8x16SatS)?; 3962 Ok(TypedReg::new(WasmValType::V128, dst)) 3963 }) 3964 } 3965 visit_i16x8_add_sat_s(&mut self) -> Self::Output3966 fn visit_i16x8_add_sat_s(&mut self) -> Self::Output { 3967 self.context 3968 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3969 masm.v128_add(dst, src, writable!(dst), V128AddKind::I16x8SatS)?; 3970 Ok(TypedReg::new(WasmValType::V128, dst)) 3971 }) 3972 } 3973 visit_i8x16_add_sat_u(&mut self) -> Self::Output3974 fn visit_i8x16_add_sat_u(&mut self) -> Self::Output { 3975 self.context 3976 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3977 masm.v128_add(dst, src, writable!(dst), V128AddKind::I8x16SatU)?; 3978 Ok(TypedReg::new(WasmValType::V128, dst)) 3979 }) 3980 } 3981 visit_i16x8_add_sat_u(&mut self) -> Self::Output3982 fn visit_i16x8_add_sat_u(&mut self) -> Self::Output { 3983 self.context 3984 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 3985 masm.v128_add(dst, src, writable!(dst), V128AddKind::I16x8SatU)?; 3986 Ok(TypedReg::new(WasmValType::V128, dst)) 3987 }) 3988 } 3989 visit_i8x16_sub_sat_s(&mut self) -> Self::Output3990 fn visit_i8x16_sub_sat_s(&mut self) -> Self::Output { 3991 self.context 3992 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 3993 masm.v128_sub(dst, src, writable!(dst), V128SubKind::I8x16SatS)?; 3994 Ok(TypedReg::new(WasmValType::V128, dst)) 3995 }) 3996 } 3997 visit_i16x8_sub_sat_s(&mut self) -> Self::Output3998 fn visit_i16x8_sub_sat_s(&mut self) -> Self::Output { 3999 self.context 4000 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 4001 masm.v128_sub(dst, src, writable!(dst), V128SubKind::I16x8SatS)?; 4002 Ok(TypedReg::new(WasmValType::V128, dst)) 4003 }) 4004 } 4005 visit_i8x16_sub_sat_u(&mut self) -> Self::Output4006 fn visit_i8x16_sub_sat_u(&mut self) -> Self::Output { 4007 self.context 4008 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 4009 masm.v128_sub(dst, src, writable!(dst), V128SubKind::I8x16SatU)?; 4010 Ok(TypedReg::new(WasmValType::V128, dst)) 4011 }) 4012 } 4013 visit_i16x8_sub_sat_u(&mut self) -> Self::Output4014 fn visit_i16x8_sub_sat_u(&mut self) -> Self::Output { 4015 self.context 4016 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 4017 masm.v128_sub(dst, src, writable!(dst), V128SubKind::I16x8SatU)?; 4018 Ok(TypedReg::new(WasmValType::V128, dst)) 4019 }) 4020 } 4021 visit_i8x16_abs(&mut self) -> Self::Output4022 fn visit_i8x16_abs(&mut self) -> Self::Output { 4023 self.context.unop(self.masm, |masm, reg| { 4024 masm.v128_abs(reg, writable!(reg), V128AbsKind::I8x16)?; 4025 Ok(TypedReg::new(WasmValType::V128, reg)) 4026 }) 4027 } 4028 visit_i16x8_abs(&mut self) -> Self::Output4029 fn visit_i16x8_abs(&mut self) -> Self::Output { 4030 self.context.unop(self.masm, |masm, reg| { 4031 masm.v128_abs(reg, writable!(reg), V128AbsKind::I16x8)?; 4032 Ok(TypedReg::new(WasmValType::V128, reg)) 4033 }) 4034 } 4035 visit_i32x4_abs(&mut self) -> Self::Output4036 fn visit_i32x4_abs(&mut self) -> Self::Output { 4037 self.context.unop(self.masm, |masm, reg| { 4038 masm.v128_abs(reg, writable!(reg), V128AbsKind::I32x4)?; 4039 Ok(TypedReg::new(WasmValType::V128, reg)) 4040 }) 4041 } 4042 visit_i64x2_abs(&mut self) -> Self::Output4043 fn visit_i64x2_abs(&mut self) -> Self::Output { 4044 self.context.unop(self.masm, |masm, reg| { 4045 masm.v128_abs(reg, writable!(reg), V128AbsKind::I64x2)?; 4046 Ok(TypedReg::new(WasmValType::V128, reg)) 4047 }) 4048 } 4049 visit_f32x4_abs(&mut self) -> Self::Output4050 fn visit_f32x4_abs(&mut self) -> Self::Output { 4051 self.context.unop(self.masm, |masm, reg| { 4052 masm.v128_abs(reg, writable!(reg), V128AbsKind::F32x4)?; 4053 Ok(TypedReg::new(WasmValType::V128, reg)) 4054 }) 4055 } 4056 visit_f64x2_abs(&mut self) -> Self::Output4057 fn visit_f64x2_abs(&mut self) -> Self::Output { 4058 self.context.unop(self.masm, |masm, reg| { 4059 masm.v128_abs(reg, writable!(reg), V128AbsKind::F64x2)?; 4060 Ok(TypedReg::new(WasmValType::V128, reg)) 4061 }) 4062 } 4063 visit_i8x16_neg(&mut self) -> Self::Output4064 fn visit_i8x16_neg(&mut self) -> Self::Output { 4065 self.context.unop(self.masm, |masm, op| { 4066 masm.v128_neg(writable!(op), V128NegKind::I8x16)?; 4067 Ok(TypedReg::new(WasmValType::V128, op)) 4068 }) 4069 } 4070 visit_i16x8_neg(&mut self) -> Self::Output4071 fn visit_i16x8_neg(&mut self) -> Self::Output { 4072 self.context.unop(self.masm, |masm, op| { 4073 masm.v128_neg(writable!(op), V128NegKind::I16x8)?; 4074 Ok(TypedReg::new(WasmValType::V128, op)) 4075 }) 4076 } 4077 visit_i32x4_neg(&mut self) -> Self::Output4078 fn visit_i32x4_neg(&mut self) -> Self::Output { 4079 self.context.unop(self.masm, |masm, op| { 4080 masm.v128_neg(writable!(op), V128NegKind::I32x4)?; 4081 Ok(TypedReg::new(WasmValType::V128, op)) 4082 }) 4083 } 4084 visit_i64x2_neg(&mut self) -> Self::Output4085 fn visit_i64x2_neg(&mut self) -> Self::Output { 4086 self.context.unop(self.masm, |masm, op| { 4087 masm.v128_neg(writable!(op), V128NegKind::I64x2)?; 4088 Ok(TypedReg::new(WasmValType::V128, op)) 4089 }) 4090 } 4091 visit_i8x16_shl(&mut self) -> Self::Output4092 fn visit_i8x16_shl(&mut self) -> Self::Output { 4093 self.masm 4094 .v128_shift(&mut self.context, OperandSize::S8, ShiftKind::Shl) 4095 } 4096 visit_i16x8_shl(&mut self) -> Self::Output4097 fn visit_i16x8_shl(&mut self) -> Self::Output { 4098 self.masm 4099 .v128_shift(&mut self.context, OperandSize::S16, ShiftKind::Shl) 4100 } 4101 visit_i32x4_shl(&mut self) -> Self::Output4102 fn visit_i32x4_shl(&mut self) -> Self::Output { 4103 self.masm 4104 .v128_shift(&mut self.context, OperandSize::S32, ShiftKind::Shl) 4105 } 4106 visit_i64x2_shl(&mut self) -> Self::Output4107 fn visit_i64x2_shl(&mut self) -> Self::Output { 4108 self.masm 4109 .v128_shift(&mut self.context, OperandSize::S64, ShiftKind::Shl) 4110 } 4111 visit_i8x16_shr_u(&mut self) -> Self::Output4112 fn visit_i8x16_shr_u(&mut self) -> Self::Output { 4113 self.masm 4114 .v128_shift(&mut self.context, OperandSize::S8, ShiftKind::ShrU) 4115 } 4116 visit_i16x8_shr_u(&mut self) -> Self::Output4117 fn visit_i16x8_shr_u(&mut self) -> Self::Output { 4118 self.masm 4119 .v128_shift(&mut self.context, OperandSize::S16, ShiftKind::ShrU) 4120 } 4121 visit_i32x4_shr_u(&mut self) -> Self::Output4122 fn visit_i32x4_shr_u(&mut self) -> Self::Output { 4123 self.masm 4124 .v128_shift(&mut self.context, OperandSize::S32, ShiftKind::ShrU) 4125 } 4126 visit_i64x2_shr_u(&mut self) -> Self::Output4127 fn visit_i64x2_shr_u(&mut self) -> Self::Output { 4128 self.masm 4129 .v128_shift(&mut self.context, OperandSize::S64, ShiftKind::ShrU) 4130 } 4131 visit_i8x16_shr_s(&mut self) -> Self::Output4132 fn visit_i8x16_shr_s(&mut self) -> Self::Output { 4133 self.masm 4134 .v128_shift(&mut self.context, OperandSize::S8, ShiftKind::ShrS) 4135 } 4136 visit_i16x8_shr_s(&mut self) -> Self::Output4137 fn visit_i16x8_shr_s(&mut self) -> Self::Output { 4138 self.masm 4139 .v128_shift(&mut self.context, OperandSize::S16, ShiftKind::ShrS) 4140 } 4141 visit_i32x4_shr_s(&mut self) -> Self::Output4142 fn visit_i32x4_shr_s(&mut self) -> Self::Output { 4143 self.masm 4144 .v128_shift(&mut self.context, OperandSize::S32, ShiftKind::ShrS) 4145 } 4146 visit_i64x2_shr_s(&mut self) -> Self::Output4147 fn visit_i64x2_shr_s(&mut self) -> Self::Output { 4148 self.masm 4149 .v128_shift(&mut self.context, OperandSize::S64, ShiftKind::ShrS) 4150 } 4151 visit_i16x8_q15mulr_sat_s(&mut self) -> Self::Output4152 fn visit_i16x8_q15mulr_sat_s(&mut self) -> Self::Output { 4153 self.context 4154 .binop(self.masm, OperandSize::S16, |masm, dst, src, size| { 4155 masm.v128_q15mulr_sat_s(dst, src, writable!(dst), size)?; 4156 Ok(TypedReg::v128(dst)) 4157 }) 4158 } 4159 visit_i8x16_min_s(&mut self) -> Self::Output4160 fn visit_i8x16_min_s(&mut self) -> Self::Output { 4161 self.context 4162 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 4163 masm.v128_min(src, dst, writable!(dst), V128MinKind::I8x16S)?; 4164 Ok(TypedReg::v128(dst)) 4165 }) 4166 } 4167 visit_i8x16_all_true(&mut self) -> Self::Output4168 fn visit_i8x16_all_true(&mut self) -> Self::Output { 4169 self.context.v128_all_true_op(self.masm, |masm, src, dst| { 4170 masm.v128_all_true(src, writable!(dst), OperandSize::S8) 4171 }) 4172 } 4173 visit_i16x8_all_true(&mut self) -> Self::Output4174 fn visit_i16x8_all_true(&mut self) -> Self::Output { 4175 self.context.v128_all_true_op(self.masm, |masm, src, dst| { 4176 masm.v128_all_true(src, writable!(dst), OperandSize::S16) 4177 }) 4178 } 4179 visit_i32x4_all_true(&mut self) -> Self::Output4180 fn visit_i32x4_all_true(&mut self) -> Self::Output { 4181 self.context.v128_all_true_op(self.masm, |masm, src, dst| { 4182 masm.v128_all_true(src, writable!(dst), OperandSize::S32) 4183 }) 4184 } 4185 visit_i64x2_all_true(&mut self) -> Self::Output4186 fn visit_i64x2_all_true(&mut self) -> Self::Output { 4187 self.context.v128_all_true_op(self.masm, |masm, src, dst| { 4188 masm.v128_all_true(src, writable!(dst), OperandSize::S64) 4189 }) 4190 } 4191 visit_i8x16_bitmask(&mut self) -> Self::Output4192 fn visit_i8x16_bitmask(&mut self) -> Self::Output { 4193 self.context.v128_bitmask_op(self.masm, |masm, src, dst| { 4194 masm.v128_bitmask(src, writable!(dst), OperandSize::S8) 4195 }) 4196 } 4197 visit_i16x8_bitmask(&mut self) -> Self::Output4198 fn visit_i16x8_bitmask(&mut self) -> Self::Output { 4199 self.context.v128_bitmask_op(self.masm, |masm, src, dst| { 4200 masm.v128_bitmask(src, writable!(dst), OperandSize::S16) 4201 }) 4202 } 4203 visit_i32x4_bitmask(&mut self) -> Self::Output4204 fn visit_i32x4_bitmask(&mut self) -> Self::Output { 4205 self.context.v128_bitmask_op(self.masm, |masm, src, dst| { 4206 masm.v128_bitmask(src, writable!(dst), OperandSize::S32) 4207 }) 4208 } 4209 visit_i64x2_bitmask(&mut self) -> Self::Output4210 fn visit_i64x2_bitmask(&mut self) -> Self::Output { 4211 self.context.v128_bitmask_op(self.masm, |masm, src, dst| { 4212 masm.v128_bitmask(src, writable!(dst), OperandSize::S64) 4213 }) 4214 } 4215 visit_i32x4_trunc_sat_f32x4_s(&mut self) -> Self::Output4216 fn visit_i32x4_trunc_sat_f32x4_s(&mut self) -> Self::Output { 4217 self.masm 4218 .v128_trunc(&mut self.context, V128TruncKind::I32x4FromF32x4S) 4219 } 4220 visit_i32x4_trunc_sat_f32x4_u(&mut self) -> Self::Output4221 fn visit_i32x4_trunc_sat_f32x4_u(&mut self) -> Self::Output { 4222 self.masm 4223 .v128_trunc(&mut self.context, V128TruncKind::I32x4FromF32x4U) 4224 } 4225 visit_i32x4_trunc_sat_f64x2_s_zero(&mut self) -> Self::Output4226 fn visit_i32x4_trunc_sat_f64x2_s_zero(&mut self) -> Self::Output { 4227 self.masm 4228 .v128_trunc(&mut self.context, V128TruncKind::I32x4FromF64x2SZero) 4229 } 4230 visit_i32x4_trunc_sat_f64x2_u_zero(&mut self) -> Self::Output4231 fn visit_i32x4_trunc_sat_f64x2_u_zero(&mut self) -> Self::Output { 4232 self.masm 4233 .v128_trunc(&mut self.context, V128TruncKind::I32x4FromF64x2UZero) 4234 } 4235 visit_i16x8_min_s(&mut self) -> Self::Output4236 fn visit_i16x8_min_s(&mut self) -> Self::Output { 4237 self.context 4238 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 4239 masm.v128_min(src, dst, writable!(dst), V128MinKind::I16x8S)?; 4240 Ok(TypedReg::v128(dst)) 4241 }) 4242 } 4243 visit_i32x4_dot_i16x8_s(&mut self) -> Self::Output4244 fn visit_i32x4_dot_i16x8_s(&mut self) -> Self::Output { 4245 self.context 4246 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4247 masm.v128_dot(dst, src, writable!(dst))?; 4248 Ok(TypedReg::v128(dst)) 4249 }) 4250 } 4251 visit_i8x16_popcnt(&mut self) -> Self::Output4252 fn visit_i8x16_popcnt(&mut self) -> Self::Output { 4253 self.masm.v128_popcnt(&mut self.context) 4254 } 4255 visit_i8x16_avgr_u(&mut self) -> Self::Output4256 fn visit_i8x16_avgr_u(&mut self) -> Self::Output { 4257 self.context 4258 .binop(self.masm, OperandSize::S8, |masm, dst, src, size| { 4259 masm.v128_avgr(dst, src, writable!(dst), size)?; 4260 Ok(TypedReg::v128(dst)) 4261 }) 4262 } 4263 visit_i32x4_min_s(&mut self) -> Self::Output4264 fn visit_i32x4_min_s(&mut self) -> Self::Output { 4265 self.context 4266 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4267 masm.v128_min(src, dst, writable!(dst), V128MinKind::I32x4S)?; 4268 Ok(TypedReg::v128(dst)) 4269 }) 4270 } 4271 visit_i8x16_min_u(&mut self) -> Self::Output4272 fn visit_i8x16_min_u(&mut self) -> Self::Output { 4273 self.context 4274 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 4275 masm.v128_min(src, dst, writable!(dst), V128MinKind::I8x16U)?; 4276 Ok(TypedReg::v128(dst)) 4277 }) 4278 } 4279 visit_i16x8_avgr_u(&mut self) -> Self::Output4280 fn visit_i16x8_avgr_u(&mut self) -> Self::Output { 4281 self.context 4282 .binop(self.masm, OperandSize::S16, |masm, dst, src, size| { 4283 masm.v128_avgr(dst, src, writable!(dst), size)?; 4284 Ok(TypedReg::v128(dst)) 4285 }) 4286 } 4287 visit_i16x8_min_u(&mut self) -> Self::Output4288 fn visit_i16x8_min_u(&mut self) -> Self::Output { 4289 self.context 4290 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 4291 masm.v128_min(src, dst, writable!(dst), V128MinKind::I16x8U)?; 4292 Ok(TypedReg::v128(dst)) 4293 }) 4294 } 4295 visit_i32x4_min_u(&mut self) -> Self::Output4296 fn visit_i32x4_min_u(&mut self) -> Self::Output { 4297 self.context 4298 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4299 masm.v128_min(src, dst, writable!(dst), V128MinKind::I32x4U)?; 4300 Ok(TypedReg::v128(dst)) 4301 }) 4302 } 4303 visit_i8x16_max_s(&mut self) -> Self::Output4304 fn visit_i8x16_max_s(&mut self) -> Self::Output { 4305 self.context 4306 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 4307 masm.v128_max(src, dst, writable!(dst), V128MaxKind::I8x16S)?; 4308 Ok(TypedReg::v128(dst)) 4309 }) 4310 } 4311 visit_i16x8_max_s(&mut self) -> Self::Output4312 fn visit_i16x8_max_s(&mut self) -> Self::Output { 4313 self.context 4314 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 4315 masm.v128_max(src, dst, writable!(dst), V128MaxKind::I16x8S)?; 4316 Ok(TypedReg::v128(dst)) 4317 }) 4318 } 4319 visit_i32x4_max_s(&mut self) -> Self::Output4320 fn visit_i32x4_max_s(&mut self) -> Self::Output { 4321 self.context 4322 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4323 masm.v128_max(src, dst, writable!(dst), V128MaxKind::I32x4S)?; 4324 Ok(TypedReg::v128(dst)) 4325 }) 4326 } 4327 visit_i8x16_max_u(&mut self) -> Self::Output4328 fn visit_i8x16_max_u(&mut self) -> Self::Output { 4329 self.context 4330 .binop(self.masm, OperandSize::S8, |masm, dst, src, _size| { 4331 masm.v128_max(src, dst, writable!(dst), V128MaxKind::I8x16U)?; 4332 Ok(TypedReg::v128(dst)) 4333 }) 4334 } 4335 visit_i16x8_max_u(&mut self) -> Self::Output4336 fn visit_i16x8_max_u(&mut self) -> Self::Output { 4337 self.context 4338 .binop(self.masm, OperandSize::S16, |masm, dst, src, _size| { 4339 masm.v128_max(src, dst, writable!(dst), V128MaxKind::I16x8U)?; 4340 Ok(TypedReg::v128(dst)) 4341 }) 4342 } 4343 visit_i32x4_max_u(&mut self) -> Self::Output4344 fn visit_i32x4_max_u(&mut self) -> Self::Output { 4345 self.context 4346 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4347 masm.v128_max(src, dst, writable!(dst), V128MaxKind::I32x4U)?; 4348 Ok(TypedReg::v128(dst)) 4349 }) 4350 } 4351 visit_i16x8_extmul_low_i8x16_s(&mut self) -> Self::Output4352 fn visit_i16x8_extmul_low_i8x16_s(&mut self) -> Self::Output { 4353 self.masm 4354 .v128_extmul(&mut self.context, V128ExtMulKind::LowI8x16S) 4355 } 4356 visit_i32x4_extmul_low_i16x8_s(&mut self) -> Self::Output4357 fn visit_i32x4_extmul_low_i16x8_s(&mut self) -> Self::Output { 4358 self.masm 4359 .v128_extmul(&mut self.context, V128ExtMulKind::LowI16x8S) 4360 } 4361 visit_i64x2_extmul_low_i32x4_s(&mut self) -> Self::Output4362 fn visit_i64x2_extmul_low_i32x4_s(&mut self) -> Self::Output { 4363 self.masm 4364 .v128_extmul(&mut self.context, V128ExtMulKind::LowI32x4S) 4365 } 4366 visit_i16x8_extmul_low_i8x16_u(&mut self) -> Self::Output4367 fn visit_i16x8_extmul_low_i8x16_u(&mut self) -> Self::Output { 4368 self.masm 4369 .v128_extmul(&mut self.context, V128ExtMulKind::LowI8x16U) 4370 } 4371 visit_i32x4_extmul_low_i16x8_u(&mut self) -> Self::Output4372 fn visit_i32x4_extmul_low_i16x8_u(&mut self) -> Self::Output { 4373 self.masm 4374 .v128_extmul(&mut self.context, V128ExtMulKind::LowI16x8U) 4375 } 4376 visit_i64x2_extmul_low_i32x4_u(&mut self) -> Self::Output4377 fn visit_i64x2_extmul_low_i32x4_u(&mut self) -> Self::Output { 4378 self.masm 4379 .v128_extmul(&mut self.context, V128ExtMulKind::LowI32x4U) 4380 } 4381 visit_i16x8_extmul_high_i8x16_u(&mut self) -> Self::Output4382 fn visit_i16x8_extmul_high_i8x16_u(&mut self) -> Self::Output { 4383 self.masm 4384 .v128_extmul(&mut self.context, V128ExtMulKind::HighI8x16U) 4385 } 4386 visit_i32x4_extmul_high_i16x8_u(&mut self) -> Self::Output4387 fn visit_i32x4_extmul_high_i16x8_u(&mut self) -> Self::Output { 4388 self.masm 4389 .v128_extmul(&mut self.context, V128ExtMulKind::HighI16x8U) 4390 } 4391 visit_i64x2_extmul_high_i32x4_u(&mut self) -> Self::Output4392 fn visit_i64x2_extmul_high_i32x4_u(&mut self) -> Self::Output { 4393 self.masm 4394 .v128_extmul(&mut self.context, V128ExtMulKind::HighI32x4U) 4395 } 4396 visit_i16x8_extmul_high_i8x16_s(&mut self) -> Self::Output4397 fn visit_i16x8_extmul_high_i8x16_s(&mut self) -> Self::Output { 4398 self.masm 4399 .v128_extmul(&mut self.context, V128ExtMulKind::HighI8x16S) 4400 } 4401 visit_i32x4_extmul_high_i16x8_s(&mut self) -> Self::Output4402 fn visit_i32x4_extmul_high_i16x8_s(&mut self) -> Self::Output { 4403 self.masm 4404 .v128_extmul(&mut self.context, V128ExtMulKind::HighI16x8S) 4405 } 4406 visit_i64x2_extmul_high_i32x4_s(&mut self) -> Self::Output4407 fn visit_i64x2_extmul_high_i32x4_s(&mut self) -> Self::Output { 4408 self.masm 4409 .v128_extmul(&mut self.context, V128ExtMulKind::HighI32x4S) 4410 } 4411 visit_i16x8_extadd_pairwise_i8x16_s(&mut self) -> Self::Output4412 fn visit_i16x8_extadd_pairwise_i8x16_s(&mut self) -> Self::Output { 4413 self.context.unop(self.masm, |masm, op| { 4414 masm.v128_extadd_pairwise(op, writable!(op), V128ExtAddKind::I8x16S)?; 4415 Ok(TypedReg::v128(op)) 4416 }) 4417 } 4418 visit_i16x8_extadd_pairwise_i8x16_u(&mut self) -> Self::Output4419 fn visit_i16x8_extadd_pairwise_i8x16_u(&mut self) -> Self::Output { 4420 self.context.unop(self.masm, |masm, op| { 4421 masm.v128_extadd_pairwise(op, writable!(op), V128ExtAddKind::I8x16U)?; 4422 Ok(TypedReg::v128(op)) 4423 }) 4424 } 4425 visit_i32x4_extadd_pairwise_i16x8_s(&mut self) -> Self::Output4426 fn visit_i32x4_extadd_pairwise_i16x8_s(&mut self) -> Self::Output { 4427 self.context.unop(self.masm, |masm, op| { 4428 masm.v128_extadd_pairwise(op, writable!(op), V128ExtAddKind::I16x8S)?; 4429 Ok(TypedReg::v128(op)) 4430 }) 4431 } 4432 visit_i32x4_extadd_pairwise_i16x8_u(&mut self) -> Self::Output4433 fn visit_i32x4_extadd_pairwise_i16x8_u(&mut self) -> Self::Output { 4434 self.context.unop(self.masm, |masm, op| { 4435 masm.v128_extadd_pairwise(op, writable!(op), V128ExtAddKind::I16x8U)?; 4436 Ok(TypedReg::v128(op)) 4437 }) 4438 } 4439 visit_f32x4_add(&mut self) -> Self::Output4440 fn visit_f32x4_add(&mut self) -> Self::Output { 4441 self.context 4442 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4443 masm.v128_add(dst, src, writable!(dst), V128AddKind::F32x4)?; 4444 Ok(TypedReg::v128(dst)) 4445 }) 4446 } 4447 visit_f64x2_add(&mut self) -> Self::Output4448 fn visit_f64x2_add(&mut self) -> Self::Output { 4449 self.context 4450 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 4451 masm.v128_add(dst, src, writable!(dst), V128AddKind::F64x2)?; 4452 Ok(TypedReg::v128(dst)) 4453 }) 4454 } 4455 visit_f32x4_sub(&mut self) -> Self::Output4456 fn visit_f32x4_sub(&mut self) -> Self::Output { 4457 self.context 4458 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4459 masm.v128_sub(dst, src, writable!(dst), V128SubKind::F32x4)?; 4460 Ok(TypedReg::v128(dst)) 4461 }) 4462 } 4463 visit_f64x2_sub(&mut self) -> Self::Output4464 fn visit_f64x2_sub(&mut self) -> Self::Output { 4465 self.context 4466 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 4467 masm.v128_sub(dst, src, writable!(dst), V128SubKind::F64x2)?; 4468 Ok(TypedReg::v128(dst)) 4469 }) 4470 } 4471 visit_f32x4_mul(&mut self) -> Self::Output4472 fn visit_f32x4_mul(&mut self) -> Self::Output { 4473 self.masm.v128_mul(&mut self.context, V128MulKind::F32x4) 4474 } 4475 visit_f64x2_mul(&mut self) -> Self::Output4476 fn visit_f64x2_mul(&mut self) -> Self::Output { 4477 self.masm.v128_mul(&mut self.context, V128MulKind::F64x2) 4478 } 4479 visit_f32x4_div(&mut self) -> Self::Output4480 fn visit_f32x4_div(&mut self) -> Self::Output { 4481 self.context 4482 .binop(self.masm, OperandSize::S32, |masm, dst, src, size| { 4483 masm.v128_div(dst, src, writable!(dst), size)?; 4484 Ok(TypedReg::v128(dst)) 4485 }) 4486 } 4487 visit_f64x2_div(&mut self) -> Self::Output4488 fn visit_f64x2_div(&mut self) -> Self::Output { 4489 self.context 4490 .binop(self.masm, OperandSize::S64, |masm, dst, src, size| { 4491 masm.v128_div(dst, src, writable!(dst), size)?; 4492 Ok(TypedReg::v128(dst)) 4493 }) 4494 } 4495 visit_f32x4_neg(&mut self) -> Self::Output4496 fn visit_f32x4_neg(&mut self) -> Self::Output { 4497 self.context.unop(self.masm, |masm, reg| { 4498 masm.v128_neg(writable!(reg), V128NegKind::F32x4)?; 4499 Ok(TypedReg::v128(reg)) 4500 }) 4501 } 4502 visit_f32x4_ceil(&mut self) -> Self::Output4503 fn visit_f32x4_ceil(&mut self) -> Self::Output { 4504 self.context.unop(self.masm, |masm, reg| { 4505 masm.v128_ceil(reg, writable!(reg), OperandSize::S32)?; 4506 Ok(TypedReg::v128(reg)) 4507 }) 4508 } 4509 visit_f64x2_neg(&mut self) -> Self::Output4510 fn visit_f64x2_neg(&mut self) -> Self::Output { 4511 self.context.unop(self.masm, |masm, reg| { 4512 masm.v128_neg(writable!(reg), V128NegKind::F64x2)?; 4513 Ok(TypedReg::v128(reg)) 4514 }) 4515 } 4516 visit_f64x2_ceil(&mut self) -> Self::Output4517 fn visit_f64x2_ceil(&mut self) -> Self::Output { 4518 self.context.unop(self.masm, |masm, reg| { 4519 masm.v128_ceil(reg, writable!(reg), OperandSize::S64)?; 4520 Ok(TypedReg::v128(reg)) 4521 }) 4522 } 4523 visit_f32x4_sqrt(&mut self) -> Self::Output4524 fn visit_f32x4_sqrt(&mut self) -> Self::Output { 4525 self.context.unop(self.masm, |masm, reg| { 4526 masm.v128_sqrt(reg, writable!(reg), OperandSize::S32)?; 4527 Ok(TypedReg::v128(reg)) 4528 }) 4529 } 4530 visit_f32x4_floor(&mut self) -> Self::Output4531 fn visit_f32x4_floor(&mut self) -> Self::Output { 4532 self.context.unop(self.masm, |masm, reg| { 4533 masm.v128_floor(reg, writable!(reg), OperandSize::S32)?; 4534 Ok(TypedReg::v128(reg)) 4535 }) 4536 } 4537 visit_f64x2_sqrt(&mut self) -> Self::Output4538 fn visit_f64x2_sqrt(&mut self) -> Self::Output { 4539 self.context.unop(self.masm, |masm, reg| { 4540 masm.v128_sqrt(reg, writable!(reg), OperandSize::S64)?; 4541 Ok(TypedReg::v128(reg)) 4542 }) 4543 } 4544 visit_f64x2_floor(&mut self) -> Self::Output4545 fn visit_f64x2_floor(&mut self) -> Self::Output { 4546 self.context.unop(self.masm, |masm, reg| { 4547 masm.v128_floor(reg, writable!(reg), OperandSize::S64)?; 4548 Ok(TypedReg::v128(reg)) 4549 }) 4550 } 4551 visit_f32x4_nearest(&mut self) -> Self::Output4552 fn visit_f32x4_nearest(&mut self) -> Self::Output { 4553 self.context.unop(self.masm, |masm, reg| { 4554 masm.v128_nearest(reg, writable!(reg), OperandSize::S32)?; 4555 Ok(TypedReg::v128(reg)) 4556 }) 4557 } 4558 visit_f64x2_nearest(&mut self) -> Self::Output4559 fn visit_f64x2_nearest(&mut self) -> Self::Output { 4560 self.context.unop(self.masm, |masm, reg| { 4561 masm.v128_nearest(reg, writable!(reg), OperandSize::S64)?; 4562 Ok(TypedReg::v128(reg)) 4563 }) 4564 } 4565 visit_f32x4_trunc(&mut self) -> Self::Output4566 fn visit_f32x4_trunc(&mut self) -> Self::Output { 4567 self.masm 4568 .v128_trunc(&mut self.context, V128TruncKind::F32x4) 4569 } 4570 visit_f64x2_trunc(&mut self) -> Self::Output4571 fn visit_f64x2_trunc(&mut self) -> Self::Output { 4572 self.masm 4573 .v128_trunc(&mut self.context, V128TruncKind::F64x2) 4574 } 4575 visit_v128_load32_zero(&mut self, memarg: MemArg) -> Self::Output4576 fn visit_v128_load32_zero(&mut self, memarg: MemArg) -> Self::Output { 4577 self.emit_wasm_load( 4578 &memarg, 4579 WasmValType::V128, 4580 LoadKind::VectorZero(OperandSize::S32), 4581 ) 4582 } 4583 visit_v128_load64_zero(&mut self, memarg: MemArg) -> Self::Output4584 fn visit_v128_load64_zero(&mut self, memarg: MemArg) -> Self::Output { 4585 self.emit_wasm_load( 4586 &memarg, 4587 WasmValType::V128, 4588 LoadKind::VectorZero(OperandSize::S64), 4589 ) 4590 } 4591 visit_f32x4_pmin(&mut self) -> Self::Output4592 fn visit_f32x4_pmin(&mut self) -> Self::Output { 4593 self.context 4594 .binop(self.masm, OperandSize::S32, |masm, dst, src, size| { 4595 masm.v128_pmin(dst, src, writable!(dst), size)?; 4596 Ok(TypedReg::v128(dst)) 4597 }) 4598 } 4599 visit_f64x2_pmin(&mut self) -> Self::Output4600 fn visit_f64x2_pmin(&mut self) -> Self::Output { 4601 self.context 4602 .binop(self.masm, OperandSize::S64, |masm, dst, src, size| { 4603 masm.v128_pmin(dst, src, writable!(dst), size)?; 4604 Ok(TypedReg::v128(dst)) 4605 }) 4606 } 4607 visit_f32x4_pmax(&mut self) -> Self::Output4608 fn visit_f32x4_pmax(&mut self) -> Self::Output { 4609 self.context 4610 .binop(self.masm, OperandSize::S32, |masm, dst, src, size| { 4611 masm.v128_pmax(dst, src, writable!(dst), size)?; 4612 Ok(TypedReg::v128(dst)) 4613 }) 4614 } 4615 visit_f64x2_pmax(&mut self) -> Self::Output4616 fn visit_f64x2_pmax(&mut self) -> Self::Output { 4617 self.context 4618 .binop(self.masm, OperandSize::S64, |masm, dst, src, size| { 4619 masm.v128_pmax(dst, src, writable!(dst), size)?; 4620 Ok(TypedReg::v128(dst)) 4621 }) 4622 } 4623 visit_f32x4_min(&mut self) -> Self::Output4624 fn visit_f32x4_min(&mut self) -> Self::Output { 4625 self.context 4626 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4627 masm.v128_min(dst, src, writable!(dst), V128MinKind::F32x4)?; 4628 Ok(TypedReg::v128(dst)) 4629 }) 4630 } 4631 visit_f64x2_min(&mut self) -> Self::Output4632 fn visit_f64x2_min(&mut self) -> Self::Output { 4633 self.context 4634 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 4635 masm.v128_min(dst, src, writable!(dst), V128MinKind::F64x2)?; 4636 Ok(TypedReg::v128(dst)) 4637 }) 4638 } 4639 visit_f32x4_max(&mut self) -> Self::Output4640 fn visit_f32x4_max(&mut self) -> Self::Output { 4641 self.context 4642 .binop(self.masm, OperandSize::S32, |masm, dst, src, _size| { 4643 masm.v128_max(dst, src, writable!(dst), V128MaxKind::F32x4)?; 4644 Ok(TypedReg::v128(dst)) 4645 }) 4646 } 4647 visit_f64x2_max(&mut self) -> Self::Output4648 fn visit_f64x2_max(&mut self) -> Self::Output { 4649 self.context 4650 .binop(self.masm, OperandSize::S64, |masm, dst, src, _size| { 4651 masm.v128_max(dst, src, writable!(dst), V128MaxKind::F64x2)?; 4652 Ok(TypedReg::v128(dst)) 4653 }) 4654 } 4655 4656 wasmparser::for_each_visit_simd_operator!(def_unsupported); 4657 } 4658 4659 impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Emission> 4660 where 4661 M: MacroAssembler, 4662 { cmp_i32s(&mut self, kind: IntCmpKind) -> Result<()>4663 fn cmp_i32s(&mut self, kind: IntCmpKind) -> Result<()> { 4664 self.context.i32_binop(self.masm, |masm, dst, src, size| { 4665 masm.cmp_with_set(writable!(dst), src, kind, size)?; 4666 Ok(TypedReg::i32(dst)) 4667 }) 4668 } 4669 cmp_i64s(&mut self, kind: IntCmpKind) -> Result<()>4670 fn cmp_i64s(&mut self, kind: IntCmpKind) -> Result<()> { 4671 self.context 4672 .i64_binop(self.masm, move |masm, dst, src, size| { 4673 masm.cmp_with_set(writable!(dst), src, kind, size)?; 4674 Ok(TypedReg::i32(dst)) // Return value for comparisons is an `i32`. 4675 }) 4676 } 4677 } 4678 4679 impl TryFrom<WasmValType> for OperandSize { 4680 type Error = crate::Error; try_from(ty: WasmValType) -> Result<OperandSize>4681 fn try_from(ty: WasmValType) -> Result<OperandSize> { 4682 let ty = match ty { 4683 WasmValType::I32 | WasmValType::F32 => OperandSize::S32, 4684 WasmValType::I64 | WasmValType::F64 => OperandSize::S64, 4685 WasmValType::V128 => OperandSize::S128, 4686 WasmValType::Ref(rt) => { 4687 match rt.heap_type { 4688 // TODO: Hardcoded size, assuming 64-bit support only. Once 4689 // Wasmtime supports 32-bit architectures, this will need 4690 // to be updated in such a way that the calculation of the 4691 // OperandSize will depend on the target's pointer size. 4692 WasmHeapType::Func => OperandSize::S64, 4693 WasmHeapType::Extern => OperandSize::S64, 4694 _ => bail!(CodeGenError::unsupported_wasm_type()), 4695 } 4696 } 4697 }; 4698 Ok(ty) 4699 } 4700 } 4701