1// WebAssemblyInstrSIMD.td - WebAssembly SIMD codegen support -*- tablegen -*-//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// WebAssembly SIMD operand code-gen constructs.
11///
12//===----------------------------------------------------------------------===//
13
14// Instructions requiring HasSIMD128 and the simd128 prefix byte
15multiclass SIMD_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s,
16                  list<dag> pattern_r, string asmstr_r = "",
17                  string asmstr_s = "", bits<32> simdop = -1> {
18  defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s,
19              !if(!ge(simdop, 0x100),
20                  !or(0xfd0000, !and(0xffff, simdop)),
21                  !or(0xfd00, !and(0xff, simdop)))>,
22            Requires<[HasSIMD128]>;
23}
24
25defm "" : ARGUMENT<V128, v16i8>;
26defm "" : ARGUMENT<V128, v8i16>;
27defm "" : ARGUMENT<V128, v4i32>;
28defm "" : ARGUMENT<V128, v2i64>;
29defm "" : ARGUMENT<V128, v4f32>;
30defm "" : ARGUMENT<V128, v2f64>;
31
32// Constrained immediate argument types
33foreach SIZE = [8, 16] in
34def ImmI#SIZE : ImmLeaf<i32,
35  "return -(1 << ("#SIZE#" - 1)) <= Imm && Imm < (1 << ("#SIZE#" - 1));"
36>;
37foreach SIZE = [2, 4, 8, 16, 32] in
38def LaneIdx#SIZE : ImmLeaf<i32, "return 0 <= Imm && Imm < "#SIZE#";">;
39
40// Create vector with identical lanes: splat
41def splat2 : PatFrag<(ops node:$x), (build_vector $x, $x)>;
42def splat4 : PatFrag<(ops node:$x), (build_vector $x, $x, $x, $x)>;
43def splat8 : PatFrag<(ops node:$x), (build_vector $x, $x, $x, $x,
44                                                  $x, $x, $x, $x)>;
45def splat16 : PatFrag<(ops node:$x),
46                      (build_vector $x, $x, $x, $x, $x, $x, $x, $x,
47                                    $x, $x, $x, $x, $x, $x, $x, $x)>;
48
49class Vec {
50  ValueType vt;
51  ValueType int_vt;
52  ValueType lane_vt;
53  WebAssemblyRegClass lane_rc;
54  int lane_bits;
55  ImmLeaf lane_idx;
56  PatFrag splat;
57  string prefix;
58  Vec split;
59}
60
61def I8x16 : Vec {
62  let vt = v16i8;
63  let int_vt = vt;
64  let lane_vt = i32;
65  let lane_rc = I32;
66  let lane_bits = 8;
67  let lane_idx = LaneIdx16;
68  let splat = splat16;
69  let prefix = "i8x16";
70}
71
72def I16x8 : Vec {
73  let vt = v8i16;
74  let int_vt = vt;
75  let lane_vt = i32;
76  let lane_rc = I32;
77  let lane_bits = 16;
78  let lane_idx = LaneIdx8;
79  let splat = splat8;
80  let prefix = "i16x8";
81  let split = I8x16;
82}
83
84def I32x4 : Vec {
85  let vt = v4i32;
86  let int_vt = vt;
87  let lane_vt = i32;
88  let lane_rc = I32;
89  let lane_bits = 32;
90  let lane_idx = LaneIdx4;
91  let splat = splat4;
92  let prefix = "i32x4";
93  let split = I16x8;
94}
95
96def I64x2 : Vec {
97  let vt = v2i64;
98  let int_vt = vt;
99  let lane_vt = i64;
100  let lane_rc = I64;
101  let lane_bits = 64;
102  let lane_idx = LaneIdx2;
103  let splat = splat2;
104  let prefix = "i64x2";
105  let split = I32x4;
106}
107
108def F32x4 : Vec {
109  let vt = v4f32;
110  let int_vt = v4i32;
111  let lane_vt = f32;
112  let lane_rc = F32;
113  let lane_bits = 32;
114  let lane_idx = LaneIdx4;
115  let splat = splat4;
116  let prefix = "f32x4";
117}
118
119def F64x2 : Vec {
120  let vt = v2f64;
121  let int_vt = v2i64;
122  let lane_vt = f64;
123  let lane_rc = F64;
124  let lane_bits = 64;
125  let lane_idx = LaneIdx2;
126  let splat = splat2;
127  let prefix = "f64x2";
128}
129
130defvar AllVecs = [I8x16, I16x8, I32x4, I64x2, F32x4, F64x2];
131defvar IntVecs = [I8x16, I16x8, I32x4, I64x2];
132
133//===----------------------------------------------------------------------===//
134// Load and store
135//===----------------------------------------------------------------------===//
136
137// Load: v128.load
138let mayLoad = 1, UseNamedOperandTable = 1 in {
139defm LOAD_V128_A32 :
140  SIMD_I<(outs V128:$dst), (ins P2Align:$p2align, offset32_op:$off, I32:$addr),
141         (outs), (ins P2Align:$p2align, offset32_op:$off), [],
142         "v128.load\t$dst, ${off}(${addr})$p2align",
143         "v128.load\t$off$p2align", 0>;
144defm LOAD_V128_A64 :
145  SIMD_I<(outs V128:$dst), (ins P2Align:$p2align, offset64_op:$off, I64:$addr),
146         (outs), (ins P2Align:$p2align, offset64_op:$off), [],
147         "v128.load\t$dst, ${off}(${addr})$p2align",
148         "v128.load\t$off$p2align", 0>;
149}
150
151// Def load patterns from WebAssemblyInstrMemory.td for vector types
152foreach vec = AllVecs in {
153defm : LoadPatNoOffset<vec.vt, load, "LOAD_V128">;
154defm : LoadPatImmOff<vec.vt, load, regPlusImm, "LOAD_V128">;
155defm : LoadPatImmOff<vec.vt, load, or_is_add, "LOAD_V128">;
156defm : LoadPatOffsetOnly<vec.vt, load, "LOAD_V128">;
157defm : LoadPatGlobalAddrOffOnly<vec.vt, load, "LOAD_V128">;
158}
159
160// v128.loadX_splat
161multiclass SIMDLoadSplat<int size, bits<32> simdop> {
162  let mayLoad = 1, UseNamedOperandTable = 1 in {
163  defm LOAD#size#_SPLAT_A32 :
164    SIMD_I<(outs V128:$dst),
165           (ins P2Align:$p2align, offset32_op:$off, I32:$addr),
166           (outs),
167           (ins P2Align:$p2align, offset32_op:$off), [],
168           "v128.load"#size#"_splat\t$dst, ${off}(${addr})$p2align",
169           "v128.load"#size#"_splat\t$off$p2align", simdop>;
170  defm LOAD#size#_SPLAT_A64 :
171    SIMD_I<(outs V128:$dst),
172           (ins P2Align:$p2align, offset64_op:$off, I64:$addr),
173           (outs),
174           (ins P2Align:$p2align, offset64_op:$off), [],
175           "v128.load"#size#"_splat\t$dst, ${off}(${addr})$p2align",
176           "v128.load"#size#"_splat\t$off$p2align", simdop>;
177  }
178}
179
180defm "" : SIMDLoadSplat<8, 7>;
181defm "" : SIMDLoadSplat<16, 8>;
182defm "" : SIMDLoadSplat<32, 9>;
183defm "" : SIMDLoadSplat<64, 10>;
184
185def wasm_load_splat_t : SDTypeProfile<1, 1, [SDTCisPtrTy<1>]>;
186def wasm_load_splat : SDNode<"WebAssemblyISD::LOAD_SPLAT", wasm_load_splat_t,
187                             [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
188def load_splat : PatFrag<(ops node:$addr), (wasm_load_splat node:$addr)>;
189
190foreach vec = AllVecs in {
191defvar inst = "LOAD"#vec.lane_bits#"_SPLAT";
192defm : LoadPatNoOffset<vec.vt, load_splat, inst>;
193defm : LoadPatImmOff<vec.vt, load_splat, regPlusImm, inst>;
194defm : LoadPatImmOff<vec.vt, load_splat, or_is_add, inst>;
195defm : LoadPatOffsetOnly<vec.vt, load_splat, inst>;
196defm : LoadPatGlobalAddrOffOnly<vec.vt, load_splat, inst>;
197}
198
199// Load and extend
200multiclass SIMDLoadExtend<Vec vec, string loadPat, bits<32> simdop> {
201  defvar signed = vec.prefix#".load"#loadPat#"_s";
202  defvar unsigned = vec.prefix#".load"#loadPat#"_u";
203  let mayLoad = 1, UseNamedOperandTable = 1 in {
204  defm LOAD_EXTEND_S_#vec#_A32 :
205    SIMD_I<(outs V128:$dst),
206           (ins P2Align:$p2align, offset32_op:$off, I32:$addr),
207           (outs), (ins P2Align:$p2align, offset32_op:$off), [],
208           signed#"\t$dst, ${off}(${addr})$p2align",
209           signed#"\t$off$p2align", simdop>;
210  defm LOAD_EXTEND_U_#vec#_A32 :
211    SIMD_I<(outs V128:$dst),
212           (ins P2Align:$p2align, offset32_op:$off, I32:$addr),
213           (outs), (ins P2Align:$p2align, offset32_op:$off), [],
214           unsigned#"\t$dst, ${off}(${addr})$p2align",
215           unsigned#"\t$off$p2align", !add(simdop, 1)>;
216  defm LOAD_EXTEND_S_#vec#_A64 :
217    SIMD_I<(outs V128:$dst),
218           (ins P2Align:$p2align, offset64_op:$off, I64:$addr),
219           (outs), (ins P2Align:$p2align, offset64_op:$off), [],
220           signed#"\t$dst, ${off}(${addr})$p2align",
221           signed#"\t$off$p2align", simdop>;
222  defm LOAD_EXTEND_U_#vec#_A64 :
223    SIMD_I<(outs V128:$dst),
224           (ins P2Align:$p2align, offset64_op:$off, I64:$addr),
225           (outs), (ins P2Align:$p2align, offset64_op:$off), [],
226           unsigned#"\t$dst, ${off}(${addr})$p2align",
227           unsigned#"\t$off$p2align", !add(simdop, 1)>;
228  }
229}
230
231defm "" : SIMDLoadExtend<I16x8, "8x8", 1>;
232defm "" : SIMDLoadExtend<I32x4, "16x4", 3>;
233defm "" : SIMDLoadExtend<I64x2, "32x2", 5>;
234
235foreach vec = [I16x8, I32x4, I64x2] in
236foreach exts = [["sextloadvi", "_S"],
237                ["zextloadvi", "_U"],
238                ["extloadvi", "_U"]] in {
239defvar loadpat = !cast<PatFrag>(exts[0]#vec.split.lane_bits);
240defvar inst = "LOAD_EXTEND"#exts[1]#"_"#vec;
241defm : LoadPatNoOffset<vec.vt, loadpat, inst>;
242defm : LoadPatImmOff<vec.vt, loadpat, regPlusImm, inst>;
243defm : LoadPatImmOff<vec.vt, loadpat, or_is_add, inst>;
244defm : LoadPatOffsetOnly<vec.vt, loadpat, inst>;
245defm : LoadPatGlobalAddrOffOnly<vec.vt, loadpat, inst>;
246}
247
248// Load lane into zero vector
249multiclass SIMDLoadZero<Vec vec, bits<32> simdop> {
250  defvar name = "v128.load"#vec.lane_bits#"_zero";
251  let mayLoad = 1, UseNamedOperandTable = 1 in {
252  defm LOAD_ZERO_#vec#_A32 :
253    SIMD_I<(outs V128:$dst),
254           (ins P2Align:$p2align, offset32_op:$off, I32:$addr),
255           (outs), (ins P2Align:$p2align, offset32_op:$off), [],
256           name#"\t$dst, ${off}(${addr})$p2align",
257           name#"\t$off$p2align", simdop>;
258  defm LOAD_ZERO_#vec#_A64 :
259    SIMD_I<(outs V128:$dst),
260           (ins P2Align:$p2align, offset64_op:$off, I64:$addr),
261           (outs), (ins P2Align:$p2align, offset64_op:$off), [],
262           name#"\t$dst, ${off}(${addr})$p2align",
263           name#"\t$off$p2align", simdop>;
264  } // mayLoad = 1, UseNamedOperandTable = 1
265}
266
267defm "" : SIMDLoadZero<I32x4, 0x5c>;
268defm "" : SIMDLoadZero<I64x2, 0x5d>;
269
270// Use load_zero to load scalars into vectors as well where possible.
271// TODO: i32, i16, and i8 scalars
272def load_scalar :
273  PatFrag<(ops node:$addr), (scalar_to_vector (i64 (load $addr)))>;
274defm : LoadPatNoOffset<v2i64, load_scalar, "LOAD_ZERO_I64x2">;
275defm : LoadPatImmOff<v2i64, load_scalar, regPlusImm, "LOAD_ZERO_I64x2">;
276defm : LoadPatImmOff<v2i64, load_scalar, or_is_add, "LOAD_ZERO_I64x2">;
277defm : LoadPatOffsetOnly<v2i64, load_scalar, "LOAD_ZERO_I64x2">;
278defm : LoadPatGlobalAddrOffOnly<v2i64, load_scalar, "LOAD_ZERO_I64x2">;
279
280// TODO: f32x4 and f64x2 as well
281foreach vec = [I32x4, I64x2] in {
282  defvar inst = "LOAD_ZERO_"#vec;
283  defvar pat = PatFrag<(ops node:$ptr),
284    (vector_insert (vec.splat (vec.lane_vt 0)), (vec.lane_vt (load $ptr)), 0)>;
285  defm : LoadPatNoOffset<vec.vt, pat, inst>;
286  defm : LoadPatImmOff<vec.vt, pat, regPlusImm, inst>;
287  defm : LoadPatImmOff<vec.vt, pat, or_is_add, inst>;
288  defm : LoadPatOffsetOnly<vec.vt, pat, inst>;
289  defm : LoadPatGlobalAddrOffOnly<vec.vt, pat, inst>;
290}
291
292// Load lane
293multiclass SIMDLoadLane<Vec vec, bits<32> simdop> {
294  defvar name = "v128.load"#vec.lane_bits#"_lane";
295  let mayLoad = 1, UseNamedOperandTable = 1 in {
296  defm LOAD_LANE_#vec#_A32 :
297    SIMD_I<(outs V128:$dst),
298           (ins P2Align:$p2align, offset32_op:$off, vec_i8imm_op:$idx,
299                I32:$addr, V128:$vec),
300           (outs), (ins P2Align:$p2align, offset32_op:$off, vec_i8imm_op:$idx),
301           [], name#"\t$dst, ${off}(${addr})$p2align, $vec, $idx",
302           name#"\t$off$p2align, $idx", simdop>;
303  defm LOAD_LANE_#vec#_A64 :
304    SIMD_I<(outs V128:$dst),
305           (ins P2Align:$p2align, offset64_op:$off, vec_i8imm_op:$idx,
306                I64:$addr, V128:$vec),
307           (outs), (ins P2Align:$p2align, offset64_op:$off, vec_i8imm_op:$idx),
308           [], name#"\t$dst, ${off}(${addr})$p2align, $vec, $idx",
309           name#"\t$off$p2align, $idx", simdop>;
310  } // mayLoad = 1, UseNamedOperandTable = 1
311}
312
313defm "" : SIMDLoadLane<I8x16, 0x54>;
314defm "" : SIMDLoadLane<I16x8, 0x55>;
315defm "" : SIMDLoadLane<I32x4, 0x56>;
316defm "" : SIMDLoadLane<I64x2, 0x57>;
317
318// Select loads with no constant offset.
319multiclass LoadLanePatNoOffset<Vec vec, SDPatternOperator kind> {
320  defvar load_lane_a32 = !cast<NI>("LOAD_LANE_"#vec#"_A32");
321  defvar load_lane_a64 = !cast<NI>("LOAD_LANE_"#vec#"_A64");
322  def : Pat<(vec.vt (kind (i32 I32:$addr),
323              (vec.vt V128:$vec), (i32 vec.lane_idx:$idx))),
324            (load_lane_a32 0, 0, imm:$idx, $addr, $vec)>,
325        Requires<[HasAddr32]>;
326  def : Pat<(vec.vt (kind (i64 I64:$addr),
327              (vec.vt V128:$vec), (i32 vec.lane_idx:$idx))),
328            (load_lane_a64 0, 0, imm:$idx, $addr, $vec)>,
329        Requires<[HasAddr64]>;
330}
331
332def load8_lane :
333  PatFrag<(ops node:$ptr, node:$vec, node:$idx),
334          (vector_insert $vec, (i32 (extloadi8 $ptr)), $idx)>;
335def load16_lane :
336  PatFrag<(ops node:$ptr, node:$vec, node:$idx),
337          (vector_insert $vec, (i32 (extloadi16 $ptr)), $idx)>;
338def load32_lane :
339  PatFrag<(ops node:$ptr, node:$vec, node:$idx),
340          (vector_insert $vec, (i32 (load $ptr)), $idx)>;
341def load64_lane :
342  PatFrag<(ops node:$ptr, node:$vec, node:$idx),
343          (vector_insert $vec, (i64 (load $ptr)), $idx)>;
344// TODO: floating point lanes as well
345
346defm : LoadLanePatNoOffset<I8x16, load8_lane>;
347defm : LoadLanePatNoOffset<I16x8, load16_lane>;
348defm : LoadLanePatNoOffset<I32x4, load32_lane>;
349defm : LoadLanePatNoOffset<I64x2, load64_lane>;
350
351// TODO: Also support the other load patterns for load_lane once the instructions
352// are merged to the proposal.
353
354// Store: v128.store
355let mayStore = 1, UseNamedOperandTable = 1 in {
356defm STORE_V128_A32 :
357  SIMD_I<(outs), (ins P2Align:$p2align, offset32_op:$off, I32:$addr, V128:$vec),
358         (outs), (ins P2Align:$p2align, offset32_op:$off), [],
359         "v128.store\t${off}(${addr})$p2align, $vec",
360         "v128.store\t$off$p2align", 11>;
361defm STORE_V128_A64 :
362  SIMD_I<(outs), (ins P2Align:$p2align, offset64_op:$off, I64:$addr, V128:$vec),
363         (outs), (ins P2Align:$p2align, offset64_op:$off), [],
364         "v128.store\t${off}(${addr})$p2align, $vec",
365         "v128.store\t$off$p2align", 11>;
366}
367
368// Def store patterns from WebAssemblyInstrMemory.td for vector types
369foreach vec = AllVecs in {
370defm : StorePatNoOffset<vec.vt, store, "STORE_V128">;
371defm : StorePatImmOff<vec.vt, store, regPlusImm, "STORE_V128">;
372defm : StorePatImmOff<vec.vt, store, or_is_add, "STORE_V128">;
373defm : StorePatOffsetOnly<vec.vt, store, "STORE_V128">;
374defm : StorePatGlobalAddrOffOnly<vec.vt, store, "STORE_V128">;
375}
376
377// Store lane
378multiclass SIMDStoreLane<Vec vec, bits<32> simdop> {
379  defvar name = "v128.store"#vec.lane_bits#"_lane";
380  let mayStore = 1, UseNamedOperandTable = 1 in {
381  defm STORE_LANE_#vec#_A32 :
382    SIMD_I<(outs),
383           (ins P2Align:$p2align, offset32_op:$off, vec_i8imm_op:$idx,
384                I32:$addr, V128:$vec),
385           (outs), (ins P2Align:$p2align, offset32_op:$off, vec_i8imm_op:$idx),
386           [], name#"\t${off}(${addr})$p2align, $vec, $idx",
387           name#"\t$off$p2align, $idx", simdop>;
388  defm STORE_LANE_#vec#_A64 :
389    SIMD_I<(outs V128:$dst),
390           (ins P2Align:$p2align, offset64_op:$off, vec_i8imm_op:$idx,
391                I64:$addr, V128:$vec),
392           (outs), (ins P2Align:$p2align, offset64_op:$off, vec_i8imm_op:$idx),
393           [], name#"\t${off}(${addr})$p2align, $vec, $idx",
394           name#"\t$off$p2align, $idx", simdop>;
395  } // mayStore = 1, UseNamedOperandTable = 1
396}
397
398defm "" : SIMDStoreLane<I8x16, 0x58>;
399defm "" : SIMDStoreLane<I16x8, 0x59>;
400defm "" : SIMDStoreLane<I32x4, 0x5a>;
401defm "" : SIMDStoreLane<I64x2, 0x5b>;
402
403// Select stores with no constant offset.
404multiclass StoreLanePatNoOffset<Vec vec, SDPatternOperator kind> {
405  def : Pat<(kind (i32 I32:$addr), (vec.vt V128:$vec), (i32 vec.lane_idx:$idx)),
406            (!cast<NI>("STORE_LANE_"#vec#"_A32") 0, 0, imm:$idx, $addr, $vec)>,
407        Requires<[HasAddr32]>;
408  def : Pat<(kind (i64 I64:$addr), (vec.vt V128:$vec), (i32 vec.lane_idx:$idx)),
409            (!cast<NI>("STORE_LANE_"#vec#"_A64") 0, 0, imm:$idx, $addr, $vec)>,
410        Requires<[HasAddr64]>;
411}
412
413def store8_lane :
414  PatFrag<(ops node:$ptr, node:$vec, node:$idx),
415          (truncstorei8 (i32 (vector_extract $vec, $idx)), $ptr)>;
416def store16_lane :
417  PatFrag<(ops node:$ptr, node:$vec, node:$idx),
418          (truncstorei16 (i32 (vector_extract $vec, $idx)), $ptr)>;
419def store32_lane :
420  PatFrag<(ops node:$ptr, node:$vec, node:$idx),
421          (store (i32 (vector_extract $vec, $idx)), $ptr)>;
422def store64_lane :
423  PatFrag<(ops node:$ptr, node:$vec, node:$idx),
424          (store (i64 (vector_extract $vec, $idx)), $ptr)>;
425// TODO: floating point lanes as well
426
427let AddedComplexity = 1 in {
428defm : StoreLanePatNoOffset<I8x16, store8_lane>;
429defm : StoreLanePatNoOffset<I16x8, store16_lane>;
430defm : StoreLanePatNoOffset<I32x4, store32_lane>;
431defm : StoreLanePatNoOffset<I64x2, store64_lane>;
432}
433
434//===----------------------------------------------------------------------===//
435// Constructing SIMD values
436//===----------------------------------------------------------------------===//
437
438// Constant: v128.const
439multiclass ConstVec<Vec vec, dag ops, dag pat, string args> {
440  let isMoveImm = 1, isReMaterializable = 1 in
441  defm CONST_V128_#vec : SIMD_I<(outs V128:$dst), ops, (outs), ops,
442                                 [(set V128:$dst, (vec.vt pat))],
443                                 "v128.const\t$dst, "#args,
444                                 "v128.const\t"#args, 12>;
445}
446
447defm "" : ConstVec<I8x16,
448                   (ins vec_i8imm_op:$i0, vec_i8imm_op:$i1,
449                        vec_i8imm_op:$i2, vec_i8imm_op:$i3,
450                        vec_i8imm_op:$i4, vec_i8imm_op:$i5,
451                        vec_i8imm_op:$i6, vec_i8imm_op:$i7,
452                        vec_i8imm_op:$i8, vec_i8imm_op:$i9,
453                        vec_i8imm_op:$iA, vec_i8imm_op:$iB,
454                        vec_i8imm_op:$iC, vec_i8imm_op:$iD,
455                        vec_i8imm_op:$iE, vec_i8imm_op:$iF),
456                   (build_vector ImmI8:$i0, ImmI8:$i1, ImmI8:$i2, ImmI8:$i3,
457                                 ImmI8:$i4, ImmI8:$i5, ImmI8:$i6, ImmI8:$i7,
458                                 ImmI8:$i8, ImmI8:$i9, ImmI8:$iA, ImmI8:$iB,
459                                 ImmI8:$iC, ImmI8:$iD, ImmI8:$iE, ImmI8:$iF),
460                   !strconcat("$i0, $i1, $i2, $i3, $i4, $i5, $i6, $i7, ",
461                              "$i8, $i9, $iA, $iB, $iC, $iD, $iE, $iF")>;
462defm "" : ConstVec<I16x8,
463                   (ins vec_i16imm_op:$i0, vec_i16imm_op:$i1,
464                        vec_i16imm_op:$i2, vec_i16imm_op:$i3,
465                        vec_i16imm_op:$i4, vec_i16imm_op:$i5,
466                        vec_i16imm_op:$i6, vec_i16imm_op:$i7),
467                   (build_vector
468                     ImmI16:$i0, ImmI16:$i1, ImmI16:$i2, ImmI16:$i3,
469                     ImmI16:$i4, ImmI16:$i5, ImmI16:$i6, ImmI16:$i7),
470                   "$i0, $i1, $i2, $i3, $i4, $i5, $i6, $i7">;
471let IsCanonical = 1 in
472defm "" : ConstVec<I32x4,
473                   (ins vec_i32imm_op:$i0, vec_i32imm_op:$i1,
474                        vec_i32imm_op:$i2, vec_i32imm_op:$i3),
475                   (build_vector (i32 imm:$i0), (i32 imm:$i1),
476                                 (i32 imm:$i2), (i32 imm:$i3)),
477                   "$i0, $i1, $i2, $i3">;
478defm "" : ConstVec<I64x2,
479                   (ins vec_i64imm_op:$i0, vec_i64imm_op:$i1),
480                   (build_vector (i64 imm:$i0), (i64 imm:$i1)),
481                   "$i0, $i1">;
482defm "" : ConstVec<F32x4,
483                   (ins f32imm_op:$i0, f32imm_op:$i1,
484                        f32imm_op:$i2, f32imm_op:$i3),
485                   (build_vector (f32 fpimm:$i0), (f32 fpimm:$i1),
486                                 (f32 fpimm:$i2), (f32 fpimm:$i3)),
487                   "$i0, $i1, $i2, $i3">;
488defm "" : ConstVec<F64x2,
489                  (ins f64imm_op:$i0, f64imm_op:$i1),
490                  (build_vector (f64 fpimm:$i0), (f64 fpimm:$i1)),
491                  "$i0, $i1">;
492
493// Shuffle lanes: shuffle
494defm SHUFFLE :
495  SIMD_I<(outs V128:$dst),
496         (ins V128:$x, V128:$y,
497           vec_i8imm_op:$m0, vec_i8imm_op:$m1,
498           vec_i8imm_op:$m2, vec_i8imm_op:$m3,
499           vec_i8imm_op:$m4, vec_i8imm_op:$m5,
500           vec_i8imm_op:$m6, vec_i8imm_op:$m7,
501           vec_i8imm_op:$m8, vec_i8imm_op:$m9,
502           vec_i8imm_op:$mA, vec_i8imm_op:$mB,
503           vec_i8imm_op:$mC, vec_i8imm_op:$mD,
504           vec_i8imm_op:$mE, vec_i8imm_op:$mF),
505         (outs),
506         (ins
507           vec_i8imm_op:$m0, vec_i8imm_op:$m1,
508           vec_i8imm_op:$m2, vec_i8imm_op:$m3,
509           vec_i8imm_op:$m4, vec_i8imm_op:$m5,
510           vec_i8imm_op:$m6, vec_i8imm_op:$m7,
511           vec_i8imm_op:$m8, vec_i8imm_op:$m9,
512           vec_i8imm_op:$mA, vec_i8imm_op:$mB,
513           vec_i8imm_op:$mC, vec_i8imm_op:$mD,
514           vec_i8imm_op:$mE, vec_i8imm_op:$mF),
515         [],
516         "i8x16.shuffle\t$dst, $x, $y, "#
517           "$m0, $m1, $m2, $m3, $m4, $m5, $m6, $m7, "#
518           "$m8, $m9, $mA, $mB, $mC, $mD, $mE, $mF",
519         "i8x16.shuffle\t"#
520           "$m0, $m1, $m2, $m3, $m4, $m5, $m6, $m7, "#
521           "$m8, $m9, $mA, $mB, $mC, $mD, $mE, $mF",
522         13>;
523
524// Shuffles after custom lowering
525def wasm_shuffle_t : SDTypeProfile<1, 18, []>;
526def wasm_shuffle : SDNode<"WebAssemblyISD::SHUFFLE", wasm_shuffle_t>;
527foreach vec = AllVecs in {
528def : Pat<(vec.vt (wasm_shuffle (vec.vt V128:$x), (vec.vt V128:$y),
529            (i32 LaneIdx32:$m0), (i32 LaneIdx32:$m1),
530            (i32 LaneIdx32:$m2), (i32 LaneIdx32:$m3),
531            (i32 LaneIdx32:$m4), (i32 LaneIdx32:$m5),
532            (i32 LaneIdx32:$m6), (i32 LaneIdx32:$m7),
533            (i32 LaneIdx32:$m8), (i32 LaneIdx32:$m9),
534            (i32 LaneIdx32:$mA), (i32 LaneIdx32:$mB),
535            (i32 LaneIdx32:$mC), (i32 LaneIdx32:$mD),
536            (i32 LaneIdx32:$mE), (i32 LaneIdx32:$mF))),
537          (SHUFFLE $x, $y,
538            imm:$m0, imm:$m1, imm:$m2, imm:$m3,
539            imm:$m4, imm:$m5, imm:$m6, imm:$m7,
540            imm:$m8, imm:$m9, imm:$mA, imm:$mB,
541            imm:$mC, imm:$mD, imm:$mE, imm:$mF)>;
542}
543
544// Swizzle lanes: i8x16.swizzle
545def wasm_swizzle_t : SDTypeProfile<1, 2, []>;
546def wasm_swizzle : SDNode<"WebAssemblyISD::SWIZZLE", wasm_swizzle_t>;
547defm SWIZZLE :
548  SIMD_I<(outs V128:$dst), (ins V128:$src, V128:$mask), (outs), (ins),
549         [(set (v16i8 V128:$dst),
550           (wasm_swizzle (v16i8 V128:$src), (v16i8 V128:$mask)))],
551         "i8x16.swizzle\t$dst, $src, $mask", "i8x16.swizzle", 14>;
552
553def : Pat<(int_wasm_swizzle (v16i8 V128:$src), (v16i8 V128:$mask)),
554          (SWIZZLE $src, $mask)>;
555
556multiclass Splat<Vec vec, bits<32> simdop> {
557  defm SPLAT_#vec : SIMD_I<(outs V128:$dst), (ins vec.lane_rc:$x),
558                           (outs), (ins),
559                           [(set (vec.vt V128:$dst),
560                              (vec.splat vec.lane_rc:$x))],
561                           vec.prefix#".splat\t$dst, $x", vec.prefix#".splat",
562                           simdop>;
563}
564
565defm "" : Splat<I8x16, 15>;
566defm "" : Splat<I16x8, 16>;
567defm "" : Splat<I32x4, 17>;
568defm "" : Splat<I64x2, 18>;
569defm "" : Splat<F32x4, 19>;
570defm "" : Splat<F64x2, 20>;
571
572// scalar_to_vector leaves high lanes undefined, so can be a splat
573foreach vec = AllVecs in
574def : Pat<(vec.vt (scalar_to_vector (vec.lane_vt vec.lane_rc:$x))),
575          (!cast<Instruction>("SPLAT_"#vec) $x)>;
576
577//===----------------------------------------------------------------------===//
578// Accessing lanes
579//===----------------------------------------------------------------------===//
580
581// Extract lane as a scalar: extract_lane / extract_lane_s / extract_lane_u
582multiclass ExtractLane<Vec vec, bits<32> simdop, string suffix = ""> {
583  defm EXTRACT_LANE_#vec#suffix :
584      SIMD_I<(outs vec.lane_rc:$dst), (ins V128:$vec, vec_i8imm_op:$idx),
585             (outs), (ins vec_i8imm_op:$idx), [],
586             vec.prefix#".extract_lane"#suffix#"\t$dst, $vec, $idx",
587             vec.prefix#".extract_lane"#suffix#"\t$idx", simdop>;
588}
589
590defm "" : ExtractLane<I8x16, 21, "_s">;
591defm "" : ExtractLane<I8x16, 22, "_u">;
592defm "" : ExtractLane<I16x8, 24, "_s">;
593defm "" : ExtractLane<I16x8, 25, "_u">;
594defm "" : ExtractLane<I32x4, 27>;
595defm "" : ExtractLane<I64x2, 29>;
596defm "" : ExtractLane<F32x4, 31>;
597defm "" : ExtractLane<F64x2, 33>;
598
599def : Pat<(vector_extract (v16i8 V128:$vec), (i32 LaneIdx16:$idx)),
600          (EXTRACT_LANE_I8x16_u $vec, imm:$idx)>;
601def : Pat<(vector_extract (v8i16 V128:$vec), (i32 LaneIdx8:$idx)),
602          (EXTRACT_LANE_I16x8_u $vec, imm:$idx)>;
603def : Pat<(vector_extract (v4i32 V128:$vec), (i32 LaneIdx4:$idx)),
604          (EXTRACT_LANE_I32x4 $vec, imm:$idx)>;
605def : Pat<(vector_extract (v4f32 V128:$vec), (i32 LaneIdx4:$idx)),
606          (EXTRACT_LANE_F32x4 $vec, imm:$idx)>;
607def : Pat<(vector_extract (v2i64 V128:$vec), (i32 LaneIdx2:$idx)),
608          (EXTRACT_LANE_I64x2 $vec, imm:$idx)>;
609def : Pat<(vector_extract (v2f64 V128:$vec), (i32 LaneIdx2:$idx)),
610          (EXTRACT_LANE_F64x2 $vec, imm:$idx)>;
611
612def : Pat<
613  (sext_inreg (vector_extract (v16i8 V128:$vec), (i32 LaneIdx16:$idx)), i8),
614  (EXTRACT_LANE_I8x16_s $vec, imm:$idx)>;
615def : Pat<
616  (and (vector_extract (v16i8 V128:$vec), (i32 LaneIdx16:$idx)), (i32 0xff)),
617  (EXTRACT_LANE_I8x16_u $vec, imm:$idx)>;
618def : Pat<
619  (sext_inreg (vector_extract (v8i16 V128:$vec), (i32 LaneIdx8:$idx)), i16),
620  (EXTRACT_LANE_I16x8_s $vec, imm:$idx)>;
621def : Pat<
622  (and (vector_extract (v8i16 V128:$vec), (i32 LaneIdx8:$idx)), (i32 0xffff)),
623  (EXTRACT_LANE_I16x8_u $vec, imm:$idx)>;
624
625// Replace lane value: replace_lane
626multiclass ReplaceLane<Vec vec, bits<32> simdop> {
627  defm REPLACE_LANE_#vec :
628    SIMD_I<(outs V128:$dst), (ins V128:$vec, vec_i8imm_op:$idx, vec.lane_rc:$x),
629           (outs), (ins vec_i8imm_op:$idx),
630           [(set V128:$dst, (vector_insert
631             (vec.vt V128:$vec),
632             (vec.lane_vt vec.lane_rc:$x),
633             (i32 vec.lane_idx:$idx)))],
634           vec.prefix#".replace_lane\t$dst, $vec, $idx, $x",
635           vec.prefix#".replace_lane\t$idx", simdop>;
636}
637
638defm "" : ReplaceLane<I8x16, 23>;
639defm "" : ReplaceLane<I16x8, 26>;
640defm "" : ReplaceLane<I32x4, 28>;
641defm "" : ReplaceLane<I64x2, 30>;
642defm "" : ReplaceLane<F32x4, 32>;
643defm "" : ReplaceLane<F64x2, 34>;
644
645// Lower undef lane indices to zero
646def : Pat<(vector_insert (v16i8 V128:$vec), I32:$x, undef),
647          (REPLACE_LANE_I8x16 $vec, 0, $x)>;
648def : Pat<(vector_insert (v8i16 V128:$vec), I32:$x, undef),
649          (REPLACE_LANE_I16x8 $vec, 0, $x)>;
650def : Pat<(vector_insert (v4i32 V128:$vec), I32:$x, undef),
651          (REPLACE_LANE_I32x4 $vec, 0, $x)>;
652def : Pat<(vector_insert (v2i64 V128:$vec), I64:$x, undef),
653          (REPLACE_LANE_I64x2 $vec, 0, $x)>;
654def : Pat<(vector_insert (v4f32 V128:$vec), F32:$x, undef),
655          (REPLACE_LANE_F32x4 $vec, 0, $x)>;
656def : Pat<(vector_insert (v2f64 V128:$vec), F64:$x, undef),
657          (REPLACE_LANE_F64x2 $vec, 0, $x)>;
658
659//===----------------------------------------------------------------------===//
660// Comparisons
661//===----------------------------------------------------------------------===//
662
663multiclass SIMDCondition<Vec vec, string name, CondCode cond, bits<32> simdop> {
664  defm _#vec :
665    SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs), (outs), (ins),
666           [(set (vec.int_vt V128:$dst),
667             (setcc (vec.vt V128:$lhs), (vec.vt V128:$rhs), cond))],
668           vec.prefix#"."#name#"\t$dst, $lhs, $rhs",
669           vec.prefix#"."#name, simdop>;
670}
671
672multiclass SIMDConditionInt<string name, CondCode cond, bits<32> baseInst> {
673  defm "" : SIMDCondition<I8x16, name, cond, baseInst>;
674  defm "" : SIMDCondition<I16x8, name, cond, !add(baseInst, 10)>;
675  defm "" : SIMDCondition<I32x4, name, cond, !add(baseInst, 20)>;
676}
677
678multiclass SIMDConditionFP<string name, CondCode cond, bits<32> baseInst> {
679  defm "" : SIMDCondition<F32x4, name, cond, baseInst>;
680  defm "" : SIMDCondition<F64x2, name, cond, !add(baseInst, 6)>;
681}
682
683// Equality: eq
684let isCommutable = 1 in {
685defm EQ : SIMDConditionInt<"eq", SETEQ, 35>;
686defm EQ : SIMDCondition<I64x2, "eq", SETEQ, 214>;
687defm EQ : SIMDConditionFP<"eq", SETOEQ, 65>;
688} // isCommutable = 1
689
690// Non-equality: ne
691let isCommutable = 1 in {
692defm NE : SIMDConditionInt<"ne", SETNE, 36>;
693defm NE : SIMDCondition<I64x2, "ne", SETNE, 215>;
694defm NE : SIMDConditionFP<"ne", SETUNE, 66>;
695} // isCommutable = 1
696
697// Less than: lt_s / lt_u / lt
698defm LT_S : SIMDConditionInt<"lt_s", SETLT, 37>;
699defm LT_S : SIMDCondition<I64x2, "lt_s", SETLT, 216>;
700defm LT_U : SIMDConditionInt<"lt_u", SETULT, 38>;
701defm LT : SIMDConditionFP<"lt", SETOLT, 67>;
702
703// Greater than: gt_s / gt_u / gt
704defm GT_S : SIMDConditionInt<"gt_s", SETGT, 39>;
705defm GT_S : SIMDCondition<I64x2, "gt_s", SETGT, 217>;
706defm GT_U : SIMDConditionInt<"gt_u", SETUGT, 40>;
707defm GT : SIMDConditionFP<"gt", SETOGT, 68>;
708
709// Less than or equal: le_s / le_u / le
710defm LE_S : SIMDConditionInt<"le_s", SETLE, 41>;
711defm LE_S : SIMDCondition<I64x2, "le_s", SETLE, 218>;
712defm LE_U : SIMDConditionInt<"le_u", SETULE, 42>;
713defm LE : SIMDConditionFP<"le", SETOLE, 69>;
714
715// Greater than or equal: ge_s / ge_u / ge
716defm GE_S : SIMDConditionInt<"ge_s", SETGE, 43>;
717defm GE_S : SIMDCondition<I64x2, "ge_s", SETGE, 219>;
718defm GE_U : SIMDConditionInt<"ge_u", SETUGE, 44>;
719defm GE : SIMDConditionFP<"ge", SETOGE, 70>;
720
721// Lower float comparisons that don't care about NaN to standard WebAssembly
722// float comparisons. These instructions are generated with nnan and in the
723// target-independent expansion of unordered comparisons and ordered ne.
724foreach nodes = [[seteq, EQ_F32x4], [setne, NE_F32x4], [setlt, LT_F32x4],
725                 [setgt, GT_F32x4], [setle, LE_F32x4], [setge, GE_F32x4]] in
726def : Pat<(v4i32 (nodes[0] (v4f32 V128:$lhs), (v4f32 V128:$rhs))),
727          (nodes[1] $lhs, $rhs)>;
728
729foreach nodes = [[seteq, EQ_F64x2], [setne, NE_F64x2], [setlt, LT_F64x2],
730                 [setgt, GT_F64x2], [setle, LE_F64x2], [setge, GE_F64x2]] in
731def : Pat<(v2i64 (nodes[0] (v2f64 V128:$lhs), (v2f64 V128:$rhs))),
732          (nodes[1] $lhs, $rhs)>;
733
734//===----------------------------------------------------------------------===//
735// Bitwise operations
736//===----------------------------------------------------------------------===//
737
738multiclass SIMDBinary<Vec vec, SDPatternOperator node, string name, bits<32> simdop> {
739  defm _#vec : SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs),
740                      (outs), (ins),
741                      [(set (vec.vt V128:$dst),
742                        (node (vec.vt V128:$lhs), (vec.vt V128:$rhs)))],
743                      vec.prefix#"."#name#"\t$dst, $lhs, $rhs",
744                      vec.prefix#"."#name, simdop>;
745}
746
747multiclass SIMDBitwise<SDPatternOperator node, string name, bits<32> simdop,
748                       bit commutable = false> {
749  let isCommutable = commutable in
750  defm "" : SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs),
751                   (outs), (ins), [],
752                   "v128."#name#"\t$dst, $lhs, $rhs", "v128."#name, simdop>;
753  foreach vec = IntVecs in
754  def : Pat<(node (vec.vt V128:$lhs), (vec.vt V128:$rhs)),
755            (!cast<NI>(NAME) $lhs, $rhs)>;
756}
757
758multiclass SIMDUnary<Vec vec, SDPatternOperator node, string name, bits<32> simdop> {
759  defm _#vec : SIMD_I<(outs V128:$dst), (ins V128:$v), (outs), (ins),
760                      [(set (vec.vt V128:$dst),
761                        (vec.vt (node (vec.vt V128:$v))))],
762                      vec.prefix#"."#name#"\t$dst, $v",
763                      vec.prefix#"."#name, simdop>;
764}
765
766// Bitwise logic: v128.not
767defm NOT : SIMD_I<(outs V128:$dst), (ins V128:$v), (outs), (ins), [],
768                  "v128.not\t$dst, $v", "v128.not", 77>;
769foreach vec = IntVecs in
770def : Pat<(vnot (vec.vt V128:$v)), (NOT $v)>;
771
772// Bitwise logic: v128.and / v128.or / v128.xor
773defm AND : SIMDBitwise<and, "and", 78, true>;
774defm OR : SIMDBitwise<or, "or", 80, true>;
775defm XOR : SIMDBitwise<xor, "xor", 81, true>;
776
777// Bitwise logic: v128.andnot
778def andnot : PatFrag<(ops node:$left, node:$right), (and $left, (vnot $right))>;
779defm ANDNOT : SIMDBitwise<andnot, "andnot", 79>;
780
781// Bitwise select: v128.bitselect
782defm BITSELECT :
783  SIMD_I<(outs V128:$dst), (ins V128:$v1, V128:$v2, V128:$c), (outs), (ins), [],
784         "v128.bitselect\t$dst, $v1, $v2, $c", "v128.bitselect", 82>;
785
786foreach vec = AllVecs in
787def : Pat<(vec.vt (int_wasm_bitselect
788            (vec.vt V128:$v1), (vec.vt V128:$v2), (vec.vt V128:$c))),
789          (BITSELECT $v1, $v2, $c)>;
790
791// Bitselect is equivalent to (c & v1) | (~c & v2)
792foreach vec = IntVecs in
793def : Pat<(vec.vt (or (and (vec.vt V128:$c), (vec.vt V128:$v1)),
794            (and (vnot V128:$c), (vec.vt V128:$v2)))),
795          (BITSELECT $v1, $v2, $c)>;
796
797// Also implement vselect in terms of bitselect
798foreach vec = AllVecs in
799def : Pat<(vec.vt (vselect
800            (vec.int_vt V128:$c), (vec.vt V128:$v1), (vec.vt V128:$v2))),
801          (BITSELECT $v1, $v2, $c)>;
802
803// MVP select on v128 values
804defm SELECT_V128 :
805  I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs, I32:$cond), (outs), (ins), [],
806    "v128.select\t$dst, $lhs, $rhs, $cond", "v128.select", 0x1b>;
807
808foreach vec = AllVecs in {
809def : Pat<(select I32:$cond, (vec.vt V128:$lhs), (vec.vt V128:$rhs)),
810          (SELECT_V128 $lhs, $rhs, $cond)>;
811
812// ISD::SELECT requires its operand to conform to getBooleanContents, but
813// WebAssembly's select interprets any non-zero value as true, so we can fold
814// a setne with 0 into a select.
815def : Pat<(select
816            (i32 (setne I32:$cond, 0)), (vec.vt V128:$lhs), (vec.vt V128:$rhs)),
817          (SELECT_V128 $lhs, $rhs, $cond)>;
818
819// And again, this time with seteq instead of setne and the arms reversed.
820def : Pat<(select
821            (i32 (seteq I32:$cond, 0)), (vec.vt V128:$lhs), (vec.vt V128:$rhs)),
822          (SELECT_V128 $rhs, $lhs, $cond)>;
823} // foreach vec
824
825//===----------------------------------------------------------------------===//
826// Integer unary arithmetic
827//===----------------------------------------------------------------------===//
828
829multiclass SIMDUnaryInt<SDPatternOperator node, string name, bits<32> baseInst> {
830  defm "" : SIMDUnary<I8x16, node, name, baseInst>;
831  defm "" : SIMDUnary<I16x8, node, name, !add(baseInst, 32)>;
832  defm "" : SIMDUnary<I32x4, node, name, !add(baseInst, 64)>;
833  defm "" : SIMDUnary<I64x2, node, name, !add(baseInst, 96)>;
834}
835
836// Integer vector negation
837def ivneg : PatFrag<(ops node:$in), (sub immAllZerosV, $in)>;
838
839// Integer absolute value: abs
840defm ABS : SIMDUnaryInt<abs, "abs", 96>;
841
842// Integer negation: neg
843defm NEG : SIMDUnaryInt<ivneg, "neg", 97>;
844
845// Population count: popcnt
846defm POPCNT : SIMDUnary<I8x16, ctpop, "popcnt", 0x62>;
847
848// Any lane true: any_true
849defm ANYTRUE : SIMD_I<(outs I32:$dst), (ins V128:$vec), (outs), (ins), [],
850                      "v128.any_true\t$dst, $vec", "v128.any_true", 0x53>;
851
852foreach vec = IntVecs in
853def : Pat<(int_wasm_anytrue (vec.vt V128:$vec)), (ANYTRUE V128:$vec)>;
854
855// All lanes true: all_true
856multiclass SIMDAllTrue<Vec vec, bits<32> simdop> {
857  defm ALLTRUE_#vec : SIMD_I<(outs I32:$dst), (ins V128:$vec), (outs), (ins),
858                             [(set I32:$dst,
859                               (i32 (int_wasm_alltrue (vec.vt V128:$vec))))],
860                             vec.prefix#".all_true\t$dst, $vec",
861                             vec.prefix#".all_true", simdop>;
862}
863
864defm "" : SIMDAllTrue<I8x16, 0x63>;
865defm "" : SIMDAllTrue<I16x8, 0x83>;
866defm "" : SIMDAllTrue<I32x4, 0xa3>;
867defm "" : SIMDAllTrue<I64x2, 0xc3>;
868
869// Reductions already return 0 or 1, so and 1, setne 0, and seteq 1
870// can be folded out
871foreach reduction =
872  [["int_wasm_anytrue", "ANYTRUE", "I8x16"],
873   ["int_wasm_anytrue", "ANYTRUE", "I16x8"],
874   ["int_wasm_anytrue", "ANYTRUE", "I32x4"],
875   ["int_wasm_anytrue", "ANYTRUE", "I64x2"],
876   ["int_wasm_alltrue", "ALLTRUE_I8x16", "I8x16"],
877   ["int_wasm_alltrue", "ALLTRUE_I16x8", "I16x8"],
878   ["int_wasm_alltrue", "ALLTRUE_I32x4", "I32x4"],
879   ["int_wasm_alltrue", "ALLTRUE_I64x2", "I64x2"]] in {
880defvar intrinsic = !cast<Intrinsic>(reduction[0]);
881defvar inst = !cast<NI>(reduction[1]);
882defvar vec = !cast<Vec>(reduction[2]);
883def : Pat<(i32 (and (i32 (intrinsic (vec.vt V128:$x))), (i32 1))), (inst $x)>;
884def : Pat<(i32 (setne (i32 (intrinsic (vec.vt V128:$x))), (i32 0))), (inst $x)>;
885def : Pat<(i32 (seteq (i32 (intrinsic (vec.vt V128:$x))), (i32 1))), (inst $x)>;
886}
887
888multiclass SIMDBitmask<Vec vec, bits<32> simdop> {
889  defm _#vec : SIMD_I<(outs I32:$dst), (ins V128:$vec), (outs), (ins),
890                      [(set I32:$dst,
891                         (i32 (int_wasm_bitmask (vec.vt V128:$vec))))],
892                      vec.prefix#".bitmask\t$dst, $vec", vec.prefix#".bitmask",
893                      simdop>;
894}
895
896defm BITMASK : SIMDBitmask<I8x16, 100>;
897defm BITMASK : SIMDBitmask<I16x8, 132>;
898defm BITMASK : SIMDBitmask<I32x4, 164>;
899defm BITMASK : SIMDBitmask<I64x2, 196>;
900
901//===----------------------------------------------------------------------===//
902// Bit shifts
903//===----------------------------------------------------------------------===//
904
905multiclass SIMDShift<Vec vec, SDNode node, string name, bits<32> simdop> {
906  defm _#vec : SIMD_I<(outs V128:$dst), (ins V128:$vec, I32:$x), (outs), (ins),
907                      [(set (vec.vt V128:$dst), (node V128:$vec, I32:$x))],
908                      vec.prefix#"."#name#"\t$dst, $vec, $x",
909                      vec.prefix#"."#name, simdop>;
910}
911
912multiclass SIMDShiftInt<SDNode node, string name, bits<32> baseInst> {
913  defm "" : SIMDShift<I8x16, node, name, baseInst>;
914  defm "" : SIMDShift<I16x8, node, name, !add(baseInst, 32)>;
915  defm "" : SIMDShift<I32x4, node, name, !add(baseInst, 64)>;
916  defm "" : SIMDShift<I64x2, node, name, !add(baseInst, 96)>;
917}
918
919// WebAssembly SIMD shifts are nonstandard in that the shift amount is
920// an i32 rather than a vector, so they need custom nodes.
921def wasm_shift_t :
922  SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisSameAs<0, 1>, SDTCisVT<2, i32>]>;
923def wasm_shl : SDNode<"WebAssemblyISD::VEC_SHL", wasm_shift_t>;
924def wasm_shr_s : SDNode<"WebAssemblyISD::VEC_SHR_S", wasm_shift_t>;
925def wasm_shr_u : SDNode<"WebAssemblyISD::VEC_SHR_U", wasm_shift_t>;
926
927// Left shift by scalar: shl
928defm SHL : SIMDShiftInt<wasm_shl, "shl", 107>;
929
930// Right shift by scalar: shr_s / shr_u
931defm SHR_S : SIMDShiftInt<wasm_shr_s, "shr_s", 108>;
932defm SHR_U : SIMDShiftInt<wasm_shr_u, "shr_u", 109>;
933
934// Optimize away an explicit mask on a shift count.
935def : Pat<(wasm_shl (v16i8 V128:$lhs), (and I32:$rhs, 7)),
936          (SHL_I8x16 V128:$lhs, I32:$rhs)>;
937def : Pat<(wasm_shr_s (v16i8 V128:$lhs), (and I32:$rhs, 7)),
938          (SHR_S_I8x16 V128:$lhs, I32:$rhs)>;
939def : Pat<(wasm_shr_u (v16i8 V128:$lhs), (and I32:$rhs, 7)),
940          (SHR_U_I8x16 V128:$lhs, I32:$rhs)>;
941
942def : Pat<(wasm_shl (v8i16 V128:$lhs), (and I32:$rhs, 15)),
943          (SHL_I16x8 V128:$lhs, I32:$rhs)>;
944def : Pat<(wasm_shr_s (v8i16 V128:$lhs), (and I32:$rhs, 15)),
945          (SHR_S_I16x8 V128:$lhs, I32:$rhs)>;
946def : Pat<(wasm_shr_u (v8i16 V128:$lhs), (and I32:$rhs, 15)),
947          (SHR_U_I16x8 V128:$lhs, I32:$rhs)>;
948
949def : Pat<(wasm_shl (v4i32 V128:$lhs), (and I32:$rhs, 31)),
950          (SHL_I32x4 V128:$lhs, I32:$rhs)>;
951def : Pat<(wasm_shr_s (v4i32 V128:$lhs), (and I32:$rhs, 31)),
952          (SHR_S_I32x4 V128:$lhs, I32:$rhs)>;
953def : Pat<(wasm_shr_u (v4i32 V128:$lhs), (and I32:$rhs, 31)),
954          (SHR_U_I32x4 V128:$lhs, I32:$rhs)>;
955
956def : Pat<(wasm_shl (v2i64 V128:$lhs), (trunc (and I64:$rhs, 63))),
957          (SHL_I64x2 V128:$lhs, (I32_WRAP_I64 I64:$rhs))>;
958def : Pat<(wasm_shr_s (v2i64 V128:$lhs), (trunc (and I64:$rhs, 63))),
959          (SHR_S_I64x2 V128:$lhs, (I32_WRAP_I64 I64:$rhs))>;
960def : Pat<(wasm_shr_u (v2i64 V128:$lhs), (trunc (and I64:$rhs, 63))),
961          (SHR_U_I64x2 V128:$lhs, (I32_WRAP_I64 I64:$rhs))>;
962
963//===----------------------------------------------------------------------===//
964// Integer binary arithmetic
965//===----------------------------------------------------------------------===//
966
967multiclass SIMDBinaryIntNoI8x16<SDPatternOperator node, string name, bits<32> baseInst> {
968  defm "" : SIMDBinary<I16x8, node, name, !add(baseInst, 32)>;
969  defm "" : SIMDBinary<I32x4, node, name, !add(baseInst, 64)>;
970  defm "" : SIMDBinary<I64x2, node, name, !add(baseInst, 96)>;
971}
972
973multiclass SIMDBinaryIntSmall<SDPatternOperator node, string name, bits<32> baseInst> {
974  defm "" : SIMDBinary<I8x16, node, name, baseInst>;
975  defm "" : SIMDBinary<I16x8, node, name, !add(baseInst, 32)>;
976}
977
978multiclass SIMDBinaryIntNoI64x2<SDPatternOperator node, string name, bits<32> baseInst> {
979  defm "" : SIMDBinaryIntSmall<node, name, baseInst>;
980  defm "" : SIMDBinary<I32x4, node, name, !add(baseInst, 64)>;
981}
982
983multiclass SIMDBinaryInt<SDPatternOperator node, string name, bits<32> baseInst> {
984  defm "" : SIMDBinaryIntNoI64x2<node, name, baseInst>;
985  defm "" : SIMDBinary<I64x2, node, name, !add(baseInst, 96)>;
986}
987
988// Integer addition: add / add_sat_s / add_sat_u
989let isCommutable = 1 in {
990defm ADD : SIMDBinaryInt<add, "add", 110>;
991defm ADD_SAT_S : SIMDBinaryIntSmall<saddsat, "add_sat_s", 111>;
992defm ADD_SAT_U : SIMDBinaryIntSmall<uaddsat, "add_sat_u", 112>;
993} // isCommutable = 1
994
995// Integer subtraction: sub / sub_sat_s / sub_sat_u
996defm SUB : SIMDBinaryInt<sub, "sub", 113>;
997defm SUB_SAT_S :
998  SIMDBinaryIntSmall<int_wasm_sub_sat_signed, "sub_sat_s", 114>;
999defm SUB_SAT_U :
1000  SIMDBinaryIntSmall<int_wasm_sub_sat_unsigned, "sub_sat_u", 115>;
1001
1002// Integer multiplication: mul
1003let isCommutable = 1 in
1004defm MUL : SIMDBinaryIntNoI8x16<mul, "mul", 117>;
1005
1006// Integer min_s / min_u / max_s / max_u
1007let isCommutable = 1 in {
1008defm MIN_S : SIMDBinaryIntNoI64x2<smin, "min_s", 118>;
1009defm MIN_U : SIMDBinaryIntNoI64x2<umin, "min_u", 119>;
1010defm MAX_S : SIMDBinaryIntNoI64x2<smax, "max_s", 120>;
1011defm MAX_U : SIMDBinaryIntNoI64x2<umax, "max_u", 121>;
1012} // isCommutable = 1
1013
1014// Integer unsigned rounding average: avgr_u
1015let isCommutable = 1 in {
1016defm AVGR_U : SIMDBinary<I8x16, int_wasm_avgr_unsigned, "avgr_u", 123>;
1017defm AVGR_U : SIMDBinary<I16x8, int_wasm_avgr_unsigned, "avgr_u", 155>;
1018}
1019
1020def add_nuw : PatFrag<(ops node:$lhs, node:$rhs), (add $lhs, $rhs),
1021                      "return N->getFlags().hasNoUnsignedWrap();">;
1022
1023foreach vec = [I8x16, I16x8] in {
1024defvar inst = !cast<NI>("AVGR_U_"#vec);
1025def : Pat<(wasm_shr_u
1026            (add_nuw
1027              (add_nuw (vec.vt V128:$lhs), (vec.vt V128:$rhs)),
1028              (vec.splat (i32 1))),
1029            (i32 1)),
1030          (inst $lhs, $rhs)>;
1031}
1032
1033// Widening dot product: i32x4.dot_i16x8_s
1034let isCommutable = 1 in
1035defm DOT : SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs), (outs), (ins),
1036                  [(set V128:$dst, (int_wasm_dot V128:$lhs, V128:$rhs))],
1037                  "i32x4.dot_i16x8_s\t$dst, $lhs, $rhs", "i32x4.dot_i16x8_s",
1038                  186>;
1039
1040// Extending multiplication: extmul_{low,high}_P, extmul_high
1041def extend_t : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>]>;
1042def extend_low_s : SDNode<"WebAssemblyISD::EXTEND_LOW_S", extend_t>;
1043def extend_high_s : SDNode<"WebAssemblyISD::EXTEND_HIGH_S", extend_t>;
1044def extend_low_u : SDNode<"WebAssemblyISD::EXTEND_LOW_U", extend_t>;
1045def extend_high_u : SDNode<"WebAssemblyISD::EXTEND_HIGH_U", extend_t>;
1046
1047multiclass SIMDExtBinary<Vec vec, SDPatternOperator node, string name,
1048                         bits<32> simdop> {
1049  defm _#vec : SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs),
1050                      (outs), (ins),
1051                      [(set (vec.vt V128:$dst), (node
1052                         (vec.split.vt V128:$lhs),(vec.split.vt V128:$rhs)))],
1053                      vec.prefix#"."#name#"\t$dst, $lhs, $rhs",
1054                      vec.prefix#"."#name, simdop>;
1055}
1056
1057class ExtMulPat<SDNode extend> :
1058  PatFrag<(ops node:$lhs, node:$rhs),
1059          (mul (extend $lhs), (extend $rhs))> {}
1060
1061def extmul_low_s : ExtMulPat<extend_low_s>;
1062def extmul_high_s : ExtMulPat<extend_high_s>;
1063def extmul_low_u : ExtMulPat<extend_low_u>;
1064def extmul_high_u : ExtMulPat<extend_high_u>;
1065
1066defm EXTMUL_LOW_S :
1067  SIMDExtBinary<I16x8, extmul_low_s, "extmul_low_i8x16_s", 0x9c>;
1068defm EXTMUL_HIGH_S :
1069  SIMDExtBinary<I16x8, extmul_high_s, "extmul_high_i8x16_s", 0x9d>;
1070defm EXTMUL_LOW_U :
1071  SIMDExtBinary<I16x8, extmul_low_u, "extmul_low_i8x16_u", 0x9e>;
1072defm EXTMUL_HIGH_U :
1073  SIMDExtBinary<I16x8, extmul_high_u, "extmul_high_i8x16_u", 0x9f>;
1074
1075defm EXTMUL_LOW_S :
1076  SIMDExtBinary<I32x4, extmul_low_s, "extmul_low_i16x8_s", 0xbc>;
1077defm EXTMUL_HIGH_S :
1078  SIMDExtBinary<I32x4, extmul_high_s, "extmul_high_i16x8_s", 0xbd>;
1079defm EXTMUL_LOW_U :
1080  SIMDExtBinary<I32x4, extmul_low_u, "extmul_low_i16x8_u", 0xbe>;
1081defm EXTMUL_HIGH_U :
1082  SIMDExtBinary<I32x4, extmul_high_u, "extmul_high_i16x8_u", 0xbf>;
1083
1084defm EXTMUL_LOW_S :
1085  SIMDExtBinary<I64x2, extmul_low_s, "extmul_low_i32x4_s", 0xdc>;
1086defm EXTMUL_HIGH_S :
1087  SIMDExtBinary<I64x2, extmul_high_s, "extmul_high_i32x4_s", 0xdd>;
1088defm EXTMUL_LOW_U :
1089  SIMDExtBinary<I64x2, extmul_low_u, "extmul_low_i32x4_u", 0xde>;
1090defm EXTMUL_HIGH_U :
1091  SIMDExtBinary<I64x2, extmul_high_u, "extmul_high_i32x4_u", 0xdf>;
1092
1093//===----------------------------------------------------------------------===//
1094// Floating-point unary arithmetic
1095//===----------------------------------------------------------------------===//
1096
1097multiclass SIMDUnaryFP<SDNode node, string name, bits<32> baseInst> {
1098  defm "" : SIMDUnary<F32x4, node, name, baseInst>;
1099  defm "" : SIMDUnary<F64x2, node, name, !add(baseInst, 12)>;
1100}
1101
1102// Absolute value: abs
1103defm ABS : SIMDUnaryFP<fabs, "abs", 224>;
1104
1105// Negation: neg
1106defm NEG : SIMDUnaryFP<fneg, "neg", 225>;
1107
1108// Square root: sqrt
1109defm SQRT : SIMDUnaryFP<fsqrt, "sqrt", 227>;
1110
1111// Rounding: ceil, floor, trunc, nearest
1112defm CEIL : SIMDUnary<F32x4, fceil, "ceil", 0x67>;
1113defm FLOOR : SIMDUnary<F32x4, ffloor, "floor", 0x68>;
1114defm TRUNC: SIMDUnary<F32x4, ftrunc, "trunc", 0x69>;
1115defm NEAREST: SIMDUnary<F32x4, fnearbyint, "nearest", 0x6a>;
1116defm CEIL : SIMDUnary<F64x2, fceil, "ceil", 0x74>;
1117defm FLOOR : SIMDUnary<F64x2, ffloor, "floor", 0x75>;
1118defm TRUNC: SIMDUnary<F64x2, ftrunc, "trunc", 0x7a>;
1119defm NEAREST: SIMDUnary<F64x2, fnearbyint, "nearest", 0x94>;
1120
1121//===----------------------------------------------------------------------===//
1122// Floating-point binary arithmetic
1123//===----------------------------------------------------------------------===//
1124
1125multiclass SIMDBinaryFP<SDPatternOperator node, string name, bits<32> baseInst> {
1126  defm "" : SIMDBinary<F32x4, node, name, baseInst>;
1127  defm "" : SIMDBinary<F64x2, node, name, !add(baseInst, 12)>;
1128}
1129
1130// Addition: add
1131let isCommutable = 1 in
1132defm ADD : SIMDBinaryFP<fadd, "add", 228>;
1133
1134// Subtraction: sub
1135defm SUB : SIMDBinaryFP<fsub, "sub", 229>;
1136
1137// Multiplication: mul
1138let isCommutable = 1 in
1139defm MUL : SIMDBinaryFP<fmul, "mul", 230>;
1140
1141// Division: div
1142defm DIV : SIMDBinaryFP<fdiv, "div", 231>;
1143
1144// NaN-propagating minimum: min
1145defm MIN : SIMDBinaryFP<fminimum, "min", 232>;
1146
1147// NaN-propagating maximum: max
1148defm MAX : SIMDBinaryFP<fmaximum, "max", 233>;
1149
1150// Pseudo-minimum: pmin
1151def pmin : PatFrag<(ops node:$lhs, node:$rhs),
1152                   (vselect (setolt $rhs, $lhs), $rhs, $lhs)>;
1153defm PMIN : SIMDBinaryFP<pmin, "pmin", 234>;
1154
1155// Pseudo-maximum: pmax
1156def pmax : PatFrag<(ops node:$lhs, node:$rhs),
1157                   (vselect (setolt $lhs, $rhs), $rhs, $lhs)>;
1158defm PMAX : SIMDBinaryFP<pmax, "pmax", 235>;
1159
1160// Also match the pmin/pmax cases where the operands are int vectors (but the
1161// comparison is still a floating point comparison). This can happen when using
1162// the wasm_simd128.h intrinsics because v128_t is an integer vector.
1163foreach vec = [F32x4, F64x2] in {
1164defvar pmin = !cast<NI>("PMIN_"#vec);
1165defvar pmax = !cast<NI>("PMAX_"#vec);
1166def : Pat<(vec.int_vt (vselect
1167            (setolt (vec.vt (bitconvert V128:$rhs)),
1168                    (vec.vt (bitconvert V128:$lhs))),
1169            V128:$rhs, V128:$lhs)),
1170          (pmin $lhs, $rhs)>;
1171def : Pat<(vec.int_vt (vselect
1172            (setolt (vec.vt (bitconvert V128:$lhs)),
1173                    (vec.vt (bitconvert V128:$rhs))),
1174            V128:$rhs, V128:$lhs)),
1175          (pmax $lhs, $rhs)>;
1176}
1177
1178// And match the pmin/pmax LLVM intrinsics as well
1179def : Pat<(v4f32 (int_wasm_pmin (v4f32 V128:$lhs), (v4f32 V128:$rhs))),
1180          (PMIN_F32x4 V128:$lhs, V128:$rhs)>;
1181def : Pat<(v4f32 (int_wasm_pmax (v4f32 V128:$lhs), (v4f32 V128:$rhs))),
1182          (PMAX_F32x4 V128:$lhs, V128:$rhs)>;
1183def : Pat<(v2f64 (int_wasm_pmin (v2f64 V128:$lhs), (v2f64 V128:$rhs))),
1184          (PMIN_F64x2 V128:$lhs, V128:$rhs)>;
1185def : Pat<(v2f64 (int_wasm_pmax (v2f64 V128:$lhs), (v2f64 V128:$rhs))),
1186          (PMAX_F64x2 V128:$lhs, V128:$rhs)>;
1187
1188//===----------------------------------------------------------------------===//
1189// Conversions
1190//===----------------------------------------------------------------------===//
1191
1192multiclass SIMDConvert<Vec vec, Vec arg, SDPatternOperator op, string name,
1193                       bits<32> simdop> {
1194  defm op#_#vec :
1195    SIMD_I<(outs V128:$dst), (ins V128:$vec), (outs), (ins),
1196           [(set (vec.vt V128:$dst), (vec.vt (op (arg.vt V128:$vec))))],
1197           vec.prefix#"."#name#"\t$dst, $vec", vec.prefix#"."#name, simdop>;
1198}
1199
1200// Floating point to integer with saturation: trunc_sat
1201defm "" : SIMDConvert<I32x4, F32x4, fp_to_sint, "trunc_sat_f32x4_s", 248>;
1202defm "" : SIMDConvert<I32x4, F32x4, fp_to_uint, "trunc_sat_f32x4_u", 249>;
1203
1204// Support the saturating variety as well.
1205def trunc_s_sat32 : PatFrag<(ops node:$x), (fp_to_sint_sat $x, i32)>;
1206def trunc_u_sat32 : PatFrag<(ops node:$x), (fp_to_uint_sat $x, i32)>;
1207def : Pat<(v4i32 (trunc_s_sat32 (v4f32 V128:$src))), (fp_to_sint_I32x4 $src)>;
1208def : Pat<(v4i32 (trunc_u_sat32 (v4f32 V128:$src))), (fp_to_uint_I32x4 $src)>;
1209
1210def trunc_sat_zero_t : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>]>;
1211def trunc_sat_zero_s :
1212  SDNode<"WebAssemblyISD::TRUNC_SAT_ZERO_S", trunc_sat_zero_t>;
1213def trunc_sat_zero_u :
1214  SDNode<"WebAssemblyISD::TRUNC_SAT_ZERO_U", trunc_sat_zero_t>;
1215defm "" : SIMDConvert<I32x4, F64x2, trunc_sat_zero_s, "trunc_sat_zero_f64x2_s",
1216                      0xfc>;
1217defm "" : SIMDConvert<I32x4, F64x2, trunc_sat_zero_u, "trunc_sat_zero_f64x2_u",
1218                      0xfd>;
1219
1220// Integer to floating point: convert
1221def convert_low_t : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>]>;
1222def convert_low_s : SDNode<"WebAssemblyISD::CONVERT_LOW_S", convert_low_t>;
1223def convert_low_u : SDNode<"WebAssemblyISD::CONVERT_LOW_U", convert_low_t>;
1224defm "" : SIMDConvert<F32x4, I32x4, sint_to_fp, "convert_i32x4_s", 250>;
1225defm "" : SIMDConvert<F32x4, I32x4, uint_to_fp, "convert_i32x4_u", 251>;
1226defm "" : SIMDConvert<F64x2, I32x4, convert_low_s, "convert_low_i32x4_s", 0xfe>;
1227defm "" : SIMDConvert<F64x2, I32x4, convert_low_u, "convert_low_i32x4_u", 0xff>;
1228
1229// Extending operations
1230// TODO: refactor this to be uniform for i64x2 if the numbering is not changed.
1231multiclass SIMDExtend<Vec vec, bits<32> baseInst> {
1232  defm "" : SIMDConvert<vec, vec.split, extend_low_s,
1233                        "extend_low_"#vec.split.prefix#"_s", baseInst>;
1234  defm "" : SIMDConvert<vec, vec.split, extend_high_s,
1235                        "extend_high_"#vec.split.prefix#"_s", !add(baseInst, 1)>;
1236  defm "" : SIMDConvert<vec, vec.split, extend_low_u,
1237                        "extend_low_"#vec.split.prefix#"_u", !add(baseInst, 2)>;
1238  defm "" : SIMDConvert<vec, vec.split, extend_high_u,
1239                        "extend_high_"#vec.split.prefix#"_u", !add(baseInst, 3)>;
1240}
1241
1242defm "" : SIMDExtend<I16x8, 0x87>;
1243defm "" : SIMDExtend<I32x4, 0xa7>;
1244defm "" : SIMDExtend<I64x2, 0xc7>;
1245
1246// Narrowing operations
1247multiclass SIMDNarrow<Vec vec, bits<32> baseInst> {
1248  defvar name = vec.split.prefix#".narrow_"#vec.prefix;
1249  defm NARROW_S_#vec.split :
1250    SIMD_I<(outs V128:$dst), (ins V128:$low, V128:$high), (outs), (ins),
1251           [(set (vec.split.vt V128:$dst), (vec.split.vt (int_wasm_narrow_signed
1252             (vec.vt V128:$low), (vec.vt V128:$high))))],
1253           name#"_s\t$dst, $low, $high", name#"_s", baseInst>;
1254  defm NARROW_U_#vec.split :
1255    SIMD_I<(outs V128:$dst), (ins V128:$low, V128:$high), (outs), (ins),
1256           [(set (vec.split.vt V128:$dst), (vec.split.vt (int_wasm_narrow_unsigned
1257             (vec.vt V128:$low), (vec.vt V128:$high))))],
1258           name#"_u\t$dst, $low, $high", name#"_u", !add(baseInst, 1)>;
1259}
1260
1261defm "" : SIMDNarrow<I16x8, 101>;
1262defm "" : SIMDNarrow<I32x4, 133>;
1263
1264// Bitcasts are nops
1265// Matching bitcast t1 to t1 causes strange errors, so avoid repeating types
1266foreach t1 = AllVecs in
1267foreach t2 = AllVecs in
1268if !ne(t1, t2) then
1269def : Pat<(t1.vt (bitconvert (t2.vt V128:$v))), (t1.vt V128:$v)>;
1270
1271// Extended pairwise addition
1272defm "" : SIMDConvert<I16x8, I8x16, int_wasm_extadd_pairwise_signed,
1273                      "extadd_pairwise_i8x16_s", 0x7c>;
1274defm "" : SIMDConvert<I16x8, I8x16, int_wasm_extadd_pairwise_unsigned,
1275                      "extadd_pairwise_i8x16_u", 0x7d>;
1276defm "" : SIMDConvert<I32x4, I16x8, int_wasm_extadd_pairwise_signed,
1277                      "extadd_pairwise_i16x8_s", 0x7e>;
1278defm "" : SIMDConvert<I32x4, I16x8, int_wasm_extadd_pairwise_unsigned,
1279                      "extadd_pairwise_i16x8_u", 0x7f>;
1280
1281// f64x2 <-> f32x4 conversions
1282def demote_t : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>]>;
1283def demote_zero : SDNode<"WebAssemblyISD::DEMOTE_ZERO", demote_t>;
1284defm "" : SIMDConvert<F32x4, F64x2, demote_zero,
1285                      "demote_zero_f64x2", 0x5e>;
1286
1287def promote_t : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>]>;
1288def promote_low : SDNode<"WebAssemblyISD::PROMOTE_LOW", promote_t>;
1289defm "" : SIMDConvert<F64x2, F32x4, promote_low, "promote_low_f32x4", 0x5f>;
1290
1291//===----------------------------------------------------------------------===//
1292// Saturating Rounding Q-Format Multiplication
1293//===----------------------------------------------------------------------===//
1294
1295defm Q15MULR_SAT_S :
1296  SIMDBinary<I16x8, int_wasm_q15mulr_sat_signed, "q15mulr_sat_s", 0x82>;
1297