1// WebAssemblyInstrSIMD.td - WebAssembly SIMD codegen support -*- tablegen -*-//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9///
10/// \file
11/// WebAssembly SIMD operand code-gen constructs.
12///
13//===----------------------------------------------------------------------===//
14
15// Instructions requiring HasSIMD128 and the simd128 prefix byte
16multiclass SIMD_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s,
17                  list<dag> pattern_r, string asmstr_r = "",
18                  string asmstr_s = "", bits<32> simdop = -1> {
19  defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s,
20              !or(0xfd00, !and(0xff, simdop))>,
21            Requires<[HasSIMD128]>;
22}
23
24defm "" : ARGUMENT<V128, v16i8>;
25defm "" : ARGUMENT<V128, v8i16>;
26defm "" : ARGUMENT<V128, v4i32>;
27defm "" : ARGUMENT<V128, v2i64>;
28defm "" : ARGUMENT<V128, v4f32>;
29defm "" : ARGUMENT<V128, v2f64>;
30
31// Constrained immediate argument types
32foreach SIZE = [8, 16] in
33def ImmI#SIZE : ImmLeaf<i32,
34  "return ((uint64_t)Imm & ((1UL << "#SIZE#") - 1)) == (uint64_t)Imm;"
35>;
36foreach SIZE = [2, 4, 8, 16, 32] in
37def LaneIdx#SIZE : ImmLeaf<i32, "return 0 <= Imm && Imm < "#SIZE#";">;
38
39//===----------------------------------------------------------------------===//
40// Load and store
41//===----------------------------------------------------------------------===//
42
43// Load: v128.load
44multiclass SIMDLoad<ValueType vec_t> {
45  let mayLoad = 1 in
46  defm LOAD_#vec_t :
47    SIMD_I<(outs V128:$dst), (ins P2Align:$align, offset32_op:$off, I32:$addr),
48           (outs), (ins P2Align:$align, offset32_op:$off), [],
49           "v128.load\t$dst, ${off}(${addr})$align",
50           "v128.load\t$off$align", 0>;
51}
52
53foreach vec_t = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in {
54defm "" : SIMDLoad<vec_t>;
55
56// Def load and store patterns from WebAssemblyInstrMemory.td for vector types
57def : LoadPatNoOffset<vec_t, load, !cast<NI>("LOAD_"#vec_t)>;
58def : LoadPatImmOff<vec_t, load, regPlusImm, !cast<NI>("LOAD_"#vec_t)>;
59def : LoadPatImmOff<vec_t, load, or_is_add, !cast<NI>("LOAD_"#vec_t)>;
60def : LoadPatGlobalAddr<vec_t, load, !cast<NI>("LOAD_"#vec_t)>;
61def : LoadPatExternalSym<vec_t, load, !cast<NI>("LOAD_"#vec_t)>;
62def : LoadPatOffsetOnly<vec_t, load, !cast<NI>("LOAD_"#vec_t)>;
63def : LoadPatGlobalAddrOffOnly<vec_t, load, !cast<NI>("LOAD_"#vec_t)>;
64def : LoadPatExternSymOffOnly<vec_t, load, !cast<NI>("LOAD_"#vec_t)>;
65}
66
67// Store: v128.store
68multiclass SIMDStore<ValueType vec_t> {
69  let mayStore = 1 in
70  defm STORE_#vec_t :
71    SIMD_I<(outs), (ins P2Align:$align, offset32_op:$off, I32:$addr, V128:$vec),
72           (outs), (ins P2Align:$align, offset32_op:$off), [],
73           "v128.store\t${off}(${addr})$align, $vec",
74           "v128.store\t$off$align", 1>;
75}
76
77foreach vec_t = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in {
78defm "" : SIMDStore<vec_t>;
79
80// Def load and store patterns from WebAssemblyInstrMemory.td for vector types
81def : StorePatNoOffset<vec_t, store, !cast<NI>("STORE_"#vec_t)>;
82def : StorePatImmOff<vec_t, store, regPlusImm, !cast<NI>("STORE_"#vec_t)>;
83def : StorePatImmOff<vec_t, store, or_is_add, !cast<NI>("STORE_"#vec_t)>;
84def : StorePatGlobalAddr<vec_t, store, !cast<NI>("STORE_"#vec_t)>;
85def : StorePatExternalSym<vec_t, store, !cast<NI>("STORE_"#vec_t)>;
86def : StorePatOffsetOnly<vec_t, store, !cast<NI>("STORE_"#vec_t)>;
87def : StorePatGlobalAddrOffOnly<vec_t, store, !cast<NI>("STORE_"#vec_t)>;
88def : StorePatExternSymOffOnly<vec_t, store, !cast<NI>("STORE_"#vec_t)>;
89}
90
91//===----------------------------------------------------------------------===//
92// Constructing SIMD values
93//===----------------------------------------------------------------------===//
94
95// Constant: v128.const
96multiclass ConstVec<ValueType vec_t, dag ops, dag pat, string args> {
97  let isMoveImm = 1, isReMaterializable = 1 in
98  defm CONST_V128_#vec_t : SIMD_I<(outs V128:$dst), ops, (outs), ops,
99                                  [(set V128:$dst, (vec_t pat))],
100                                  "v128.const\t$dst, "#args,
101                                  "v128.const\t"#args, 2>;
102}
103
104defm "" : ConstVec<v16i8,
105                   (ins vec_i8imm_op:$i0, vec_i8imm_op:$i1,
106                        vec_i8imm_op:$i2, vec_i8imm_op:$i3,
107                        vec_i8imm_op:$i4, vec_i8imm_op:$i5,
108                        vec_i8imm_op:$i6, vec_i8imm_op:$i7,
109                        vec_i8imm_op:$i8, vec_i8imm_op:$i9,
110                        vec_i8imm_op:$iA, vec_i8imm_op:$iB,
111                        vec_i8imm_op:$iC, vec_i8imm_op:$iD,
112                        vec_i8imm_op:$iE, vec_i8imm_op:$iF),
113                   (build_vector ImmI8:$i0, ImmI8:$i1, ImmI8:$i2, ImmI8:$i3,
114                                 ImmI8:$i4, ImmI8:$i5, ImmI8:$i6, ImmI8:$i7,
115                                 ImmI8:$i8, ImmI8:$i9, ImmI8:$iA, ImmI8:$iB,
116                                 ImmI8:$iC, ImmI8:$iD, ImmI8:$iE, ImmI8:$iF),
117                   !strconcat("$i0, $i1, $i2, $i3, $i4, $i5, $i6, $i7, ",
118                              "$i8, $i9, $iA, $iB, $iC, $iD, $iE, $iF")>;
119defm "" : ConstVec<v8i16,
120                   (ins vec_i16imm_op:$i0, vec_i16imm_op:$i1,
121                        vec_i16imm_op:$i2, vec_i16imm_op:$i3,
122                        vec_i16imm_op:$i4, vec_i16imm_op:$i5,
123                        vec_i16imm_op:$i6, vec_i16imm_op:$i7),
124                   (build_vector
125                     ImmI16:$i0, ImmI16:$i1, ImmI16:$i2, ImmI16:$i3,
126                     ImmI16:$i4, ImmI16:$i5, ImmI16:$i6, ImmI16:$i7),
127                   "$i0, $i1, $i2, $i3, $i4, $i5, $i6, $i7">;
128defm "" : ConstVec<v4i32,
129                   (ins vec_i32imm_op:$i0, vec_i32imm_op:$i1,
130                        vec_i32imm_op:$i2, vec_i32imm_op:$i3),
131                   (build_vector (i32 imm:$i0), (i32 imm:$i1),
132                                 (i32 imm:$i2), (i32 imm:$i3)),
133                   "$i0, $i1, $i2, $i3">;
134defm "" : ConstVec<v2i64,
135                   (ins vec_i64imm_op:$i0, vec_i64imm_op:$i1),
136                   (build_vector (i64 imm:$i0), (i64 imm:$i1)),
137                   "$i0, $i1">;
138defm "" : ConstVec<v4f32,
139                   (ins f32imm_op:$i0, f32imm_op:$i1,
140                        f32imm_op:$i2, f32imm_op:$i3),
141                   (build_vector (f32 fpimm:$i0), (f32 fpimm:$i1),
142                                 (f32 fpimm:$i2), (f32 fpimm:$i3)),
143                   "$i0, $i1, $i2, $i3">;
144defm "" : ConstVec<v2f64,
145                  (ins f64imm_op:$i0, f64imm_op:$i1),
146                  (build_vector (f64 fpimm:$i0), (f64 fpimm:$i1)),
147                  "$i0, $i1">;
148
149// Shuffle lanes: shuffle
150defm SHUFFLE :
151  SIMD_I<(outs V128:$dst),
152         (ins V128:$x, V128:$y,
153           vec_i8imm_op:$m0, vec_i8imm_op:$m1,
154           vec_i8imm_op:$m2, vec_i8imm_op:$m3,
155           vec_i8imm_op:$m4, vec_i8imm_op:$m5,
156           vec_i8imm_op:$m6, vec_i8imm_op:$m7,
157           vec_i8imm_op:$m8, vec_i8imm_op:$m9,
158           vec_i8imm_op:$mA, vec_i8imm_op:$mB,
159           vec_i8imm_op:$mC, vec_i8imm_op:$mD,
160           vec_i8imm_op:$mE, vec_i8imm_op:$mF),
161         (outs),
162         (ins
163           vec_i8imm_op:$m0, vec_i8imm_op:$m1,
164           vec_i8imm_op:$m2, vec_i8imm_op:$m3,
165           vec_i8imm_op:$m4, vec_i8imm_op:$m5,
166           vec_i8imm_op:$m6, vec_i8imm_op:$m7,
167           vec_i8imm_op:$m8, vec_i8imm_op:$m9,
168           vec_i8imm_op:$mA, vec_i8imm_op:$mB,
169           vec_i8imm_op:$mC, vec_i8imm_op:$mD,
170           vec_i8imm_op:$mE, vec_i8imm_op:$mF),
171         [],
172         "v8x16.shuffle\t$dst, $x, $y, "#
173           "$m0, $m1, $m2, $m3, $m4, $m5, $m6, $m7, "#
174           "$m8, $m9, $mA, $mB, $mC, $mD, $mE, $mF",
175         "v8x16.shuffle\t"#
176           "$m0, $m1, $m2, $m3, $m4, $m5, $m6, $m7, "#
177           "$m8, $m9, $mA, $mB, $mC, $mD, $mE, $mF",
178         3>;
179
180// Shuffles after custom lowering
181def wasm_shuffle_t : SDTypeProfile<1, 18, []>;
182def wasm_shuffle : SDNode<"WebAssemblyISD::SHUFFLE", wasm_shuffle_t>;
183foreach vec_t = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in {
184def : Pat<(vec_t (wasm_shuffle (vec_t V128:$x), (vec_t V128:$y),
185            (i32 LaneIdx32:$m0), (i32 LaneIdx32:$m1),
186            (i32 LaneIdx32:$m2), (i32 LaneIdx32:$m3),
187            (i32 LaneIdx32:$m4), (i32 LaneIdx32:$m5),
188            (i32 LaneIdx32:$m6), (i32 LaneIdx32:$m7),
189            (i32 LaneIdx32:$m8), (i32 LaneIdx32:$m9),
190            (i32 LaneIdx32:$mA), (i32 LaneIdx32:$mB),
191            (i32 LaneIdx32:$mC), (i32 LaneIdx32:$mD),
192            (i32 LaneIdx32:$mE), (i32 LaneIdx32:$mF))),
193          (vec_t (SHUFFLE (vec_t V128:$x), (vec_t V128:$y),
194            (i32 LaneIdx32:$m0), (i32 LaneIdx32:$m1),
195            (i32 LaneIdx32:$m2), (i32 LaneIdx32:$m3),
196            (i32 LaneIdx32:$m4), (i32 LaneIdx32:$m5),
197            (i32 LaneIdx32:$m6), (i32 LaneIdx32:$m7),
198            (i32 LaneIdx32:$m8), (i32 LaneIdx32:$m9),
199            (i32 LaneIdx32:$mA), (i32 LaneIdx32:$mB),
200            (i32 LaneIdx32:$mC), (i32 LaneIdx32:$mD),
201            (i32 LaneIdx32:$mE), (i32 LaneIdx32:$mF)))>;
202}
203
204// Create vector with identical lanes: splat
205def splat2 : PatFrag<(ops node:$x), (build_vector node:$x, node:$x)>;
206def splat4 : PatFrag<(ops node:$x), (build_vector
207                       node:$x, node:$x, node:$x, node:$x)>;
208def splat8 : PatFrag<(ops node:$x), (build_vector
209                       node:$x, node:$x, node:$x, node:$x,
210                       node:$x, node:$x, node:$x, node:$x)>;
211def splat16 : PatFrag<(ops node:$x), (build_vector
212                        node:$x, node:$x, node:$x, node:$x,
213                        node:$x, node:$x, node:$x, node:$x,
214                        node:$x, node:$x, node:$x, node:$x,
215                        node:$x, node:$x, node:$x, node:$x)>;
216
217multiclass Splat<ValueType vec_t, string vec, WebAssemblyRegClass reg_t,
218                 PatFrag splat_pat, bits<32> simdop> {
219  // Prefer splats over v128.const for const splats (65 is lowest that works)
220  let AddedComplexity = 65 in
221  defm SPLAT_#vec_t : SIMD_I<(outs V128:$dst), (ins reg_t:$x), (outs), (ins),
222                             [(set (vec_t V128:$dst), (splat_pat reg_t:$x))],
223                             vec#".splat\t$dst, $x", vec#".splat", simdop>;
224}
225
226defm "" : Splat<v16i8, "i8x16", I32, splat16, 4>;
227defm "" : Splat<v8i16, "i16x8", I32, splat8, 8>;
228defm "" : Splat<v4i32, "i32x4", I32, splat4, 12>;
229defm "" : Splat<v2i64, "i64x2", I64, splat2, 15>;
230defm "" : Splat<v4f32, "f32x4", F32, splat4, 18>;
231defm "" : Splat<v2f64, "f64x2", F64, splat2, 21>;
232
233//===----------------------------------------------------------------------===//
234// Accessing lanes
235//===----------------------------------------------------------------------===//
236
237// Extract lane as a scalar: extract_lane / extract_lane_s / extract_lane_u
238multiclass ExtractLane<ValueType vec_t, string vec, ImmLeaf imm_t,
239                       WebAssemblyRegClass reg_t, bits<32> simdop,
240                       string suffix = "", SDNode extract = vector_extract> {
241  defm EXTRACT_LANE_#vec_t#suffix :
242      SIMD_I<(outs reg_t:$dst), (ins V128:$vec, vec_i8imm_op:$idx),
243             (outs), (ins vec_i8imm_op:$idx),
244             [(set reg_t:$dst, (extract (vec_t V128:$vec), (i32 imm_t:$idx)))],
245             vec#".extract_lane"#suffix#"\t$dst, $vec, $idx",
246             vec#".extract_lane"#suffix#"\t$idx", simdop>;
247}
248
249multiclass ExtractPat<ValueType lane_t, int mask> {
250  def _s : PatFrag<(ops node:$vec, node:$idx),
251                   (i32 (sext_inreg
252                     (i32 (vector_extract
253                       node:$vec,
254                       node:$idx
255                     )),
256                     lane_t
257                   ))>;
258  def _u : PatFrag<(ops node:$vec, node:$idx),
259                   (i32 (and
260                     (i32 (vector_extract
261                       node:$vec,
262                       node:$idx
263                     )),
264                     (i32 mask)
265                   ))>;
266}
267
268defm extract_i8x16 : ExtractPat<i8, 0xff>;
269defm extract_i16x8 : ExtractPat<i16, 0xffff>;
270
271multiclass ExtractLaneExtended<string sign, bits<32> baseInst> {
272  defm "" : ExtractLane<v16i8, "i8x16", LaneIdx16, I32, baseInst, sign,
273                        !cast<PatFrag>("extract_i8x16"#sign)>;
274  defm "" : ExtractLane<v8i16, "i16x8", LaneIdx8, I32, !add(baseInst, 4), sign,
275                        !cast<PatFrag>("extract_i16x8"#sign)>;
276}
277
278defm "" : ExtractLaneExtended<"_s", 5>;
279defm "" : ExtractLaneExtended<"_u", 6>;
280defm "" : ExtractLane<v4i32, "i32x4", LaneIdx4, I32, 13>;
281defm "" : ExtractLane<v2i64, "i64x2", LaneIdx2, I64, 16>;
282defm "" : ExtractLane<v4f32, "f32x4", LaneIdx4, F32, 19>;
283defm "" : ExtractLane<v2f64, "f64x2", LaneIdx2, F64, 22>;
284
285// Follow convention of making implicit expansions unsigned
286def : Pat<(i32 (vector_extract (v16i8 V128:$vec), (i32 LaneIdx16:$idx))),
287          (EXTRACT_LANE_v16i8_u V128:$vec, (i32 LaneIdx16:$idx))>;
288def : Pat<(i32 (vector_extract (v8i16 V128:$vec), (i32 LaneIdx8:$idx))),
289          (EXTRACT_LANE_v8i16_u V128:$vec, (i32 LaneIdx8:$idx))>;
290
291// Lower undef lane indices to zero
292def : Pat<(and (i32 (vector_extract (v16i8 V128:$vec), undef)), (i32 0xff)),
293          (EXTRACT_LANE_v16i8_u V128:$vec, 0)>;
294def : Pat<(and (i32 (vector_extract (v8i16 V128:$vec), undef)), (i32 0xffff)),
295          (EXTRACT_LANE_v8i16_u V128:$vec, 0)>;
296def : Pat<(i32 (vector_extract (v16i8 V128:$vec), undef)),
297          (EXTRACT_LANE_v16i8_u V128:$vec, 0)>;
298def : Pat<(i32 (vector_extract (v8i16 V128:$vec), undef)),
299          (EXTRACT_LANE_v8i16_u V128:$vec, 0)>;
300def : Pat<(sext_inreg (i32 (vector_extract (v16i8 V128:$vec), undef)), i8),
301          (EXTRACT_LANE_v16i8_s V128:$vec, 0)>;
302def : Pat<(sext_inreg (i32 (vector_extract (v8i16 V128:$vec), undef)), i16),
303          (EXTRACT_LANE_v8i16_s V128:$vec, 0)>;
304def : Pat<(vector_extract (v4i32 V128:$vec), undef),
305          (EXTRACT_LANE_v4i32 V128:$vec, 0)>;
306def : Pat<(vector_extract (v2i64 V128:$vec), undef),
307          (EXTRACT_LANE_v2i64 V128:$vec, 0)>;
308def : Pat<(vector_extract (v4f32 V128:$vec), undef),
309          (EXTRACT_LANE_v4f32 V128:$vec, 0)>;
310def : Pat<(vector_extract (v2f64 V128:$vec), undef),
311          (EXTRACT_LANE_v2f64 V128:$vec, 0)>;
312
313// Replace lane value: replace_lane
314multiclass ReplaceLane<ValueType vec_t, string vec, ImmLeaf imm_t,
315                       WebAssemblyRegClass reg_t, ValueType lane_t,
316                       bits<32> simdop> {
317  defm REPLACE_LANE_#vec_t :
318      SIMD_I<(outs V128:$dst), (ins V128:$vec, vec_i8imm_op:$idx, reg_t:$x),
319             (outs), (ins vec_i8imm_op:$idx),
320             [(set V128:$dst, (vector_insert
321               (vec_t V128:$vec), (lane_t reg_t:$x), (i32 imm_t:$idx)))],
322             vec#".replace_lane\t$dst, $vec, $idx, $x",
323             vec#".replace_lane\t$idx", simdop>;
324}
325
326defm "" : ReplaceLane<v16i8, "i8x16", LaneIdx16, I32, i32, 7>;
327defm "" : ReplaceLane<v8i16, "i16x8", LaneIdx8, I32, i32, 11>;
328defm "" : ReplaceLane<v4i32, "i32x4", LaneIdx4, I32, i32, 14>;
329defm "" : ReplaceLane<v2i64, "i64x2", LaneIdx2, I64, i64, 17>;
330defm "" : ReplaceLane<v4f32, "f32x4", LaneIdx4, F32, f32, 20>;
331defm "" : ReplaceLane<v2f64, "f64x2", LaneIdx2, F64, f64, 23>;
332
333// Lower undef lane indices to zero
334def : Pat<(vector_insert (v16i8 V128:$vec), I32:$x, undef),
335          (REPLACE_LANE_v16i8 V128:$vec, 0, I32:$x)>;
336def : Pat<(vector_insert (v8i16 V128:$vec), I32:$x, undef),
337          (REPLACE_LANE_v8i16 V128:$vec, 0, I32:$x)>;
338def : Pat<(vector_insert (v4i32 V128:$vec), I32:$x, undef),
339          (REPLACE_LANE_v4i32 V128:$vec, 0, I32:$x)>;
340def : Pat<(vector_insert (v2i64 V128:$vec), I64:$x, undef),
341          (REPLACE_LANE_v2i64 V128:$vec, 0, I64:$x)>;
342def : Pat<(vector_insert (v4f32 V128:$vec), F32:$x, undef),
343          (REPLACE_LANE_v4f32 V128:$vec, 0, F32:$x)>;
344def : Pat<(vector_insert (v2f64 V128:$vec), F64:$x, undef),
345          (REPLACE_LANE_v2f64 V128:$vec, 0, F64:$x)>;
346
347// Arbitrary other BUILD_VECTOR patterns
348def : Pat<(v16i8 (build_vector
349            (i32 I32:$x0), (i32 I32:$x1), (i32 I32:$x2), (i32 I32:$x3),
350            (i32 I32:$x4), (i32 I32:$x5), (i32 I32:$x6), (i32 I32:$x7),
351            (i32 I32:$x8), (i32 I32:$x9), (i32 I32:$x10), (i32 I32:$x11),
352            (i32 I32:$x12), (i32 I32:$x13), (i32 I32:$x14), (i32 I32:$x15)
353          )),
354          (v16i8 (REPLACE_LANE_v16i8
355            (v16i8 (REPLACE_LANE_v16i8
356              (v16i8 (REPLACE_LANE_v16i8
357                (v16i8 (REPLACE_LANE_v16i8
358                  (v16i8 (REPLACE_LANE_v16i8
359                    (v16i8 (REPLACE_LANE_v16i8
360                      (v16i8 (REPLACE_LANE_v16i8
361                        (v16i8 (REPLACE_LANE_v16i8
362                          (v16i8 (REPLACE_LANE_v16i8
363                            (v16i8 (REPLACE_LANE_v16i8
364                              (v16i8 (REPLACE_LANE_v16i8
365                                (v16i8 (REPLACE_LANE_v16i8
366                                  (v16i8 (REPLACE_LANE_v16i8
367                                    (v16i8 (REPLACE_LANE_v16i8
368                                      (v16i8 (REPLACE_LANE_v16i8
369                                        (v16i8 (SPLAT_v16i8 (i32 I32:$x0))),
370                                        1, I32:$x1
371                                      )),
372                                      2, I32:$x2
373                                    )),
374                                    3, I32:$x3
375                                  )),
376                                  4, I32:$x4
377                                )),
378                                5, I32:$x5
379                              )),
380                              6, I32:$x6
381                            )),
382                            7, I32:$x7
383                          )),
384                          8, I32:$x8
385                        )),
386                        9, I32:$x9
387                      )),
388                      10, I32:$x10
389                    )),
390                    11, I32:$x11
391                  )),
392                  12, I32:$x12
393                )),
394                13, I32:$x13
395              )),
396              14, I32:$x14
397            )),
398            15, I32:$x15
399          ))>;
400def : Pat<(v8i16 (build_vector
401            (i32 I32:$x0), (i32 I32:$x1), (i32 I32:$x2), (i32 I32:$x3),
402            (i32 I32:$x4), (i32 I32:$x5), (i32 I32:$x6), (i32 I32:$x7)
403          )),
404          (v8i16 (REPLACE_LANE_v8i16
405            (v8i16 (REPLACE_LANE_v8i16
406              (v8i16 (REPLACE_LANE_v8i16
407                (v8i16 (REPLACE_LANE_v8i16
408                  (v8i16 (REPLACE_LANE_v8i16
409                    (v8i16 (REPLACE_LANE_v8i16
410                      (v8i16 (REPLACE_LANE_v8i16
411                        (v8i16 (SPLAT_v8i16 (i32 I32:$x0))),
412                        1, I32:$x1
413                      )),
414                      2, I32:$x2
415                    )),
416                    3, I32:$x3
417                  )),
418                  4, I32:$x4
419                )),
420                5, I32:$x5
421              )),
422              6, I32:$x6
423            )),
424            7, I32:$x7
425          ))>;
426def : Pat<(v4i32 (build_vector
427            (i32 I32:$x0), (i32 I32:$x1), (i32 I32:$x2), (i32 I32:$x3)
428          )),
429          (v4i32 (REPLACE_LANE_v4i32
430            (v4i32 (REPLACE_LANE_v4i32
431              (v4i32 (REPLACE_LANE_v4i32
432                (v4i32 (SPLAT_v4i32 (i32 I32:$x0))),
433                1, I32:$x1
434              )),
435              2, I32:$x2
436            )),
437            3, I32:$x3
438          ))>;
439def : Pat<(v2i64 (build_vector (i64 I64:$x0), (i64 I64:$x1))),
440          (v2i64 (REPLACE_LANE_v2i64
441            (v2i64 (SPLAT_v2i64 (i64 I64:$x0))), 1, I64:$x1))>;
442def : Pat<(v4f32 (build_vector
443            (f32 F32:$x0), (f32 F32:$x1), (f32 F32:$x2), (f32 F32:$x3)
444          )),
445          (v4f32 (REPLACE_LANE_v4f32
446            (v4f32 (REPLACE_LANE_v4f32
447              (v4f32 (REPLACE_LANE_v4f32
448                (v4f32 (SPLAT_v4f32 (f32 F32:$x0))),
449                1, F32:$x1
450              )),
451              2, F32:$x2
452            )),
453            3, F32:$x3
454          ))>;
455def : Pat<(v2f64 (build_vector (f64 F64:$x0), (f64 F64:$x1))),
456          (v2f64 (REPLACE_LANE_v2f64
457            (v2f64 (SPLAT_v2f64 (f64 F64:$x0))), 1, F64:$x1))>;
458
459//===----------------------------------------------------------------------===//
460// Comparisons
461//===----------------------------------------------------------------------===//
462
463multiclass SIMDCondition<ValueType vec_t, ValueType out_t, string vec,
464                         string name, CondCode cond, bits<32> simdop> {
465  defm _#vec_t :
466    SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs), (outs), (ins),
467           [(set (out_t V128:$dst),
468             (setcc (vec_t V128:$lhs), (vec_t V128:$rhs), cond)
469           )],
470           vec#"."#name#"\t$dst, $lhs, $rhs", vec#"."#name, simdop>;
471}
472
473multiclass SIMDConditionInt<string name, CondCode cond, bits<32> baseInst> {
474  defm "" : SIMDCondition<v16i8, v16i8, "i8x16", name, cond, baseInst>;
475  defm "" : SIMDCondition<v8i16, v8i16, "i16x8", name, cond,
476                          !add(baseInst, 10)>;
477  defm "" : SIMDCondition<v4i32, v4i32, "i32x4", name, cond,
478                          !add(baseInst, 20)>;
479}
480
481multiclass SIMDConditionFP<string name, CondCode cond, bits<32> baseInst> {
482  defm "" : SIMDCondition<v4f32, v4i32, "f32x4", name, cond, baseInst>;
483  defm "" : SIMDCondition<v2f64, v2i64, "f64x2", name, cond,
484                          !add(baseInst, 6)>;
485}
486
487// Equality: eq
488let isCommutable = 1 in {
489defm EQ : SIMDConditionInt<"eq", SETEQ, 24>;
490defm EQ : SIMDConditionFP<"eq", SETOEQ, 64>;
491} // isCommutable = 1
492
493// Non-equality: ne
494let isCommutable = 1 in {
495defm NE : SIMDConditionInt<"ne", SETNE, 25>;
496defm NE : SIMDConditionFP<"ne", SETUNE, 65>;
497} // isCommutable = 1
498
499// Less than: lt_s / lt_u / lt
500defm LT_S : SIMDConditionInt<"lt_s", SETLT, 26>;
501defm LT_U : SIMDConditionInt<"lt_u", SETULT, 27>;
502defm LT : SIMDConditionFP<"lt", SETOLT, 66>;
503
504// Greater than: gt_s / gt_u / gt
505defm GT_S : SIMDConditionInt<"gt_s", SETGT, 28>;
506defm GT_U : SIMDConditionInt<"gt_u", SETUGT, 29>;
507defm GT : SIMDConditionFP<"gt", SETOGT, 67>;
508
509// Less than or equal: le_s / le_u / le
510defm LE_S : SIMDConditionInt<"le_s", SETLE, 30>;
511defm LE_U : SIMDConditionInt<"le_u", SETULE, 31>;
512defm LE : SIMDConditionFP<"le", SETOLE, 68>;
513
514// Greater than or equal: ge_s / ge_u / ge
515defm GE_S : SIMDConditionInt<"ge_s", SETGE, 32>;
516defm GE_U : SIMDConditionInt<"ge_u", SETUGE, 33>;
517defm GE : SIMDConditionFP<"ge", SETOGE, 69>;
518
519// Lower float comparisons that don't care about NaN to standard WebAssembly
520// float comparisons. These instructions are generated in the target-independent
521// expansion of unordered comparisons and ordered ne.
522def : Pat<(v4i32 (seteq (v4f32 V128:$lhs), (v4f32 V128:$rhs))),
523          (v4i32 (EQ_v4f32 (v4f32 V128:$lhs), (v4f32 V128:$rhs)))>;
524def : Pat<(v4i32 (setne (v4f32 V128:$lhs), (v4f32 V128:$rhs))),
525          (v4i32 (NE_v4f32 (v4f32 V128:$lhs), (v4f32 V128:$rhs)))>;
526def : Pat<(v2i64 (seteq (v2f64 V128:$lhs), (v2f64 V128:$rhs))),
527          (v2i64 (EQ_v2f64 (v2f64 V128:$lhs), (v2f64 V128:$rhs)))>;
528def : Pat<(v2i64 (setne (v2f64 V128:$lhs), (v2f64 V128:$rhs))),
529          (v2i64 (NE_v2f64 (v2f64 V128:$lhs), (v2f64 V128:$rhs)))>;
530
531//===----------------------------------------------------------------------===//
532// Bitwise operations
533//===----------------------------------------------------------------------===//
534
535multiclass SIMDBinary<ValueType vec_t, string vec, SDNode node, string name,
536                      bits<32> simdop> {
537  defm _#vec_t : SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs),
538                        (outs), (ins),
539                        [(set (vec_t V128:$dst),
540                          (node (vec_t V128:$lhs), (vec_t V128:$rhs))
541                        )],
542                        vec#"."#name#"\t$dst, $lhs, $rhs", vec#"."#name,
543                        simdop>;
544}
545
546multiclass SIMDBitwise<SDNode node, string name, bits<32> simdop> {
547  defm "" : SIMDBinary<v16i8, "v128", node, name, simdop>;
548  defm "" : SIMDBinary<v8i16, "v128", node, name, simdop>;
549  defm "" : SIMDBinary<v4i32, "v128", node, name, simdop>;
550  defm "" : SIMDBinary<v2i64, "v128", node, name, simdop>;
551}
552
553multiclass SIMDUnary<ValueType vec_t, string vec, SDNode node, string name,
554                     bits<32> simdop> {
555  defm _#vec_t : SIMD_I<(outs V128:$dst), (ins V128:$vec), (outs), (ins),
556                        [(set (vec_t V128:$dst),
557                          (vec_t (node (vec_t V128:$vec)))
558                        )],
559                        vec#"."#name#"\t$dst, $vec", vec#"."#name, simdop>;
560}
561
562// Bitwise logic: v128.and / v128.or / v128.xor
563let isCommutable = 1 in {
564defm AND : SIMDBitwise<and, "and", 76>;
565defm OR : SIMDBitwise<or, "or", 77>;
566defm XOR : SIMDBitwise<xor, "xor", 78>;
567} // isCommutable = 1
568
569// Bitwise logic: v128.not
570foreach vec_t = [v16i8, v8i16, v4i32, v2i64] in
571defm NOT: SIMDUnary<vec_t, "v128", vnot, "not", 79>;
572
573// Bitwise select: v128.bitselect
574foreach vec_t = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in
575  defm BITSELECT_#vec_t :
576    SIMD_I<(outs V128:$dst), (ins V128:$v1, V128:$v2, V128:$c), (outs), (ins),
577           [(set (vec_t V128:$dst),
578             (vec_t (int_wasm_bitselect
579               (vec_t V128:$c), (vec_t V128:$v1), (vec_t V128:$v2)
580             ))
581           )],
582           "v128.bitselect\t$dst, $v1, $v2, $c", "v128.bitselect", 80>;
583
584// Bitselect is equivalent to (c & v1) | (~c & v2)
585foreach vec_t = [v16i8, v8i16, v4i32, v2i64] in
586  def : Pat<(vec_t (or (and (vec_t V128:$c), (vec_t V128:$v1)),
587              (and (vnot V128:$c), (vec_t V128:$v2)))),
588            (!cast<Instruction>("BITSELECT_"#vec_t)
589              V128:$v1, V128:$v2, V128:$c)>;
590
591//===----------------------------------------------------------------------===//
592// Integer unary arithmetic
593//===----------------------------------------------------------------------===//
594
595multiclass SIMDUnaryInt<SDNode node, string name, bits<32> baseInst> {
596  defm "" : SIMDUnary<v16i8, "i8x16", node, name, baseInst>;
597  defm "" : SIMDUnary<v8i16, "i16x8", node, name, !add(baseInst, 17)>;
598  defm "" : SIMDUnary<v4i32, "i32x4", node, name, !add(baseInst, 34)>;
599  defm "" : SIMDUnary<v2i64, "i64x2", node, name, !add(baseInst, 51)>;
600}
601
602multiclass SIMDReduceVec<ValueType vec_t, string vec, SDNode op, string name,
603                         bits<32> simdop> {
604  defm _#vec_t : SIMD_I<(outs I32:$dst), (ins V128:$vec), (outs), (ins),
605                        [(set I32:$dst, (i32 (op (vec_t V128:$vec))))],
606                        vec#"."#name#"\t$dst, $vec", vec#"."#name, simdop>;
607}
608
609multiclass SIMDReduce<SDNode op, string name, bits<32> baseInst> {
610  defm "" : SIMDReduceVec<v16i8, "i8x16", op, name, baseInst>;
611  defm "" : SIMDReduceVec<v8i16, "i16x8", op, name, !add(baseInst, 17)>;
612  defm "" : SIMDReduceVec<v4i32, "i32x4", op, name, !add(baseInst, 34)>;
613  defm "" : SIMDReduceVec<v2i64, "i64x2", op, name, !add(baseInst, 51)>;
614}
615
616// Integer vector negation
617def ivneg : PatFrag<(ops node:$in), (sub immAllZerosV, node:$in)>;
618
619// Integer negation: neg
620defm NEG : SIMDUnaryInt<ivneg, "neg", 81>;
621
622// Any lane true: any_true
623defm ANYTRUE : SIMDReduce<int_wasm_anytrue, "any_true", 82>;
624
625// All lanes true: all_true
626defm ALLTRUE : SIMDReduce<int_wasm_alltrue, "all_true", 83>;
627
628//===----------------------------------------------------------------------===//
629// Bit shifts
630//===----------------------------------------------------------------------===//
631
632multiclass SIMDShift<ValueType vec_t, string vec, SDNode node, dag shift_vec,
633                     string name, bits<32> simdop> {
634  defm _#vec_t : SIMD_I<(outs V128:$dst), (ins V128:$vec, I32:$x),
635                        (outs), (ins),
636                        [(set (vec_t V128:$dst),
637                          (node V128:$vec, (vec_t shift_vec)))],
638                        vec#"."#name#"\t$dst, $vec, $x", vec#"."#name, simdop>;
639}
640
641multiclass SIMDShiftInt<SDNode node, string name, bits<32> baseInst> {
642  defm "" : SIMDShift<v16i8, "i8x16", node, (splat16 I32:$x), name, baseInst>;
643  defm "" : SIMDShift<v8i16, "i16x8", node, (splat8 I32:$x), name,
644                      !add(baseInst, 17)>;
645  defm "" : SIMDShift<v4i32, "i32x4", node, (splat4 I32:$x), name,
646                      !add(baseInst, 34)>;
647  defm "" : SIMDShift<v2i64, "i64x2", node, (splat2 (i64 (zext I32:$x))),
648                      name, !add(baseInst, 51)>;
649}
650
651// Left shift by scalar: shl
652defm SHL : SIMDShiftInt<shl, "shl", 84>;
653
654// Right shift by scalar: shr_s / shr_u
655defm SHR_S : SIMDShiftInt<sra, "shr_s", 85>;
656defm SHR_U : SIMDShiftInt<srl, "shr_u", 86>;
657
658// Truncate i64 shift operands to i32s
659foreach shifts = [[shl, SHL_v2i64], [sra, SHR_S_v2i64], [srl, SHR_U_v2i64]] in
660def : Pat<(v2i64 (shifts[0] (v2i64 V128:$vec), (v2i64 (splat2 I64:$x)))),
661          (v2i64 (shifts[1] (v2i64 V128:$vec), (I32_WRAP_I64 I64:$x)))>;
662
663// 2xi64 shifts with constant shift amounts are custom lowered to avoid wrapping
664def wasm_shift_t : SDTypeProfile<1, 2,
665  [SDTCisVec<0>, SDTCisSameAs<0, 1>, SDTCisVT<2, i32>]
666>;
667def wasm_shl : SDNode<"WebAssemblyISD::VEC_SHL", wasm_shift_t>;
668def wasm_shr_s : SDNode<"WebAssemblyISD::VEC_SHR_S", wasm_shift_t>;
669def wasm_shr_u : SDNode<"WebAssemblyISD::VEC_SHR_U", wasm_shift_t>;
670foreach shifts = [[wasm_shl, SHL_v2i64],
671                  [wasm_shr_s, SHR_S_v2i64],
672                  [wasm_shr_u, SHR_U_v2i64]] in
673def : Pat<(v2i64 (shifts[0] (v2i64 V128:$vec), I32:$x)),
674          (v2i64 (shifts[1] (v2i64 V128:$vec), I32:$x))>;
675
676//===----------------------------------------------------------------------===//
677// Integer binary arithmetic
678//===----------------------------------------------------------------------===//
679
680multiclass SIMDBinaryIntSmall<SDNode node, string name, bits<32> baseInst> {
681  defm "" : SIMDBinary<v16i8, "i8x16", node, name, baseInst>;
682  defm "" : SIMDBinary<v8i16, "i16x8", node, name, !add(baseInst, 17)>;
683}
684
685multiclass SIMDBinaryIntNoI64x2<SDNode node, string name, bits<32> baseInst> {
686  defm "" : SIMDBinaryIntSmall<node, name, baseInst>;
687  defm "" : SIMDBinary<v4i32, "i32x4", node, name, !add(baseInst, 34)>;
688}
689
690multiclass SIMDBinaryInt<SDNode node, string name, bits<32> baseInst> {
691  defm "" : SIMDBinaryIntNoI64x2<node, name, baseInst>;
692  defm "" : SIMDBinary<v2i64, "i64x2", node, name, !add(baseInst, 51)>;
693}
694
695// Integer addition: add / add_saturate_s / add_saturate_u
696let isCommutable = 1 in {
697defm ADD : SIMDBinaryInt<add, "add", 87>;
698defm ADD_SAT_S : SIMDBinaryIntSmall<saddsat, "add_saturate_s", 88>;
699defm ADD_SAT_U : SIMDBinaryIntSmall<uaddsat, "add_saturate_u", 89>;
700} // isCommutable = 1
701
702// Integer subtraction: sub / sub_saturate_s / sub_saturate_u
703defm SUB : SIMDBinaryInt<sub, "sub", 90>;
704defm SUB_SAT_S :
705  SIMDBinaryIntSmall<int_wasm_sub_saturate_signed, "sub_saturate_s", 91>;
706defm SUB_SAT_U :
707  SIMDBinaryIntSmall<int_wasm_sub_saturate_unsigned, "sub_saturate_u", 92>;
708
709// Integer multiplication: mul
710defm MUL : SIMDBinaryIntNoI64x2<mul, "mul", 93>;
711
712//===----------------------------------------------------------------------===//
713// Floating-point unary arithmetic
714//===----------------------------------------------------------------------===//
715
716multiclass SIMDUnaryFP<SDNode node, string name, bits<32> baseInst> {
717  defm "" : SIMDUnary<v4f32, "f32x4", node, name, baseInst>;
718  defm "" : SIMDUnary<v2f64, "f64x2", node, name, !add(baseInst, 11)>;
719}
720
721// Absolute value: abs
722defm ABS : SIMDUnaryFP<fabs, "abs", 149>;
723
724// Negation: neg
725defm NEG : SIMDUnaryFP<fneg, "neg", 150>;
726
727// Square root: sqrt
728defm SQRT : SIMDUnaryFP<fsqrt, "sqrt", 151>;
729
730//===----------------------------------------------------------------------===//
731// Floating-point binary arithmetic
732//===----------------------------------------------------------------------===//
733
734multiclass SIMDBinaryFP<SDNode node, string name, bits<32> baseInst> {
735  defm "" : SIMDBinary<v4f32, "f32x4", node, name, baseInst>;
736  defm "" : SIMDBinary<v2f64, "f64x2", node, name, !add(baseInst, 11)>;
737}
738
739// Addition: add
740let isCommutable = 1 in
741defm ADD : SIMDBinaryFP<fadd, "add", 154>;
742
743// Subtraction: sub
744defm SUB : SIMDBinaryFP<fsub, "sub", 155>;
745
746// Multiplication: mul
747let isCommutable = 1 in
748defm MUL : SIMDBinaryFP<fmul, "mul", 156>;
749
750// Division: div
751defm DIV : SIMDBinaryFP<fdiv, "div", 157>;
752
753// NaN-propagating minimum: min
754defm MIN : SIMDBinaryFP<fminimum, "min", 158>;
755
756// NaN-propagating maximum: max
757defm MAX : SIMDBinaryFP<fmaximum, "max", 159>;
758
759//===----------------------------------------------------------------------===//
760// Conversions
761//===----------------------------------------------------------------------===//
762
763multiclass SIMDConvert<ValueType vec_t, ValueType arg_t, SDNode op,
764                       string name, bits<32> simdop> {
765  defm op#_#vec_t#_#arg_t :
766    SIMD_I<(outs V128:$dst), (ins V128:$vec), (outs), (ins),
767           [(set (vec_t V128:$dst), (vec_t (op (arg_t V128:$vec))))],
768           name#"\t$dst, $vec", name, simdop>;
769}
770
771// Integer to floating point: convert_s / convert_u
772defm "" : SIMDConvert<v4f32, v4i32, sint_to_fp, "f32x4.convert_s/i32x4", 175>;
773defm "" : SIMDConvert<v4f32, v4i32, uint_to_fp, "f32x4.convert_u/i32x4", 176>;
774defm "" : SIMDConvert<v2f64, v2i64, sint_to_fp, "f64x2.convert_s/i64x2", 177>;
775defm "" : SIMDConvert<v2f64, v2i64, uint_to_fp, "f64x2.convert_u/i64x2", 178>;
776
777// Floating point to integer with saturation: trunc_sat_s / trunc_sat_u
778defm "" : SIMDConvert<v4i32, v4f32, fp_to_sint, "i32x4.trunc_sat_s/f32x4", 171>;
779defm "" : SIMDConvert<v4i32, v4f32, fp_to_uint, "i32x4.trunc_sat_u/f32x4", 172>;
780defm "" : SIMDConvert<v2i64, v2f64, fp_to_sint, "i64x2.trunc_sat_s/f64x2", 173>;
781defm "" : SIMDConvert<v2i64, v2f64, fp_to_uint, "i64x2.trunc_sat_u/f64x2", 174>;
782
783// Lower llvm.wasm.trunc.saturate.* to saturating instructions
784def : Pat<(v4i32 (int_wasm_trunc_saturate_signed (v4f32 V128:$src))),
785          (fp_to_sint_v4i32_v4f32 (v4f32 V128:$src))>;
786def : Pat<(v4i32 (int_wasm_trunc_saturate_unsigned (v4f32 V128:$src))),
787          (fp_to_uint_v4i32_v4f32 (v4f32 V128:$src))>;
788def : Pat<(v2i64 (int_wasm_trunc_saturate_signed (v2f64 V128:$src))),
789          (fp_to_sint_v2i64_v2f64 (v2f64 V128:$src))>;
790def : Pat<(v2i64 (int_wasm_trunc_saturate_unsigned (v2f64 V128:$src))),
791          (fp_to_uint_v2i64_v2f64 (v2f64 V128:$src))>;
792
793// Bitcasts are nops
794// Matching bitcast t1 to t1 causes strange errors, so avoid repeating types
795foreach t1 = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in
796foreach t2 = !foldl(
797  []<ValueType>, [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64],
798  acc, cur, !if(!eq(!cast<string>(t1), !cast<string>(cur)),
799    acc, !listconcat(acc, [cur])
800  )
801) in
802def : Pat<(t1 (bitconvert (t2 V128:$v))), (t1 V128:$v)>;
803