1//==- SystemZInstrVector.td - SystemZ Vector instructions ------*- tblgen-*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9//===----------------------------------------------------------------------===//
10// Move instructions
11//===----------------------------------------------------------------------===//
12
13let Predicates = [FeatureVector] in {
14  // Register move.
15  def VLR : UnaryVRRa<"vlr", 0xE756, null_frag, v128any, v128any>;
16  def VLR32 : UnaryAliasVRR<null_frag, v32sb, v32sb>;
17  def VLR64 : UnaryAliasVRR<null_frag, v64db, v64db>;
18
19  // Load GR from VR element.
20  def VLGV  : BinaryVRScGeneric<"vlgv", 0xE721>;
21  def VLGVB : BinaryVRSc<"vlgvb", 0xE721, null_frag, v128b, 0>;
22  def VLGVH : BinaryVRSc<"vlgvh", 0xE721, null_frag, v128h, 1>;
23  def VLGVF : BinaryVRSc<"vlgvf", 0xE721, null_frag, v128f, 2>;
24  def VLGVG : BinaryVRSc<"vlgvg", 0xE721, z_vector_extract, v128g, 3>;
25
26  // Load VR element from GR.
27  def VLVG  : TernaryVRSbGeneric<"vlvg", 0xE722>;
28  def VLVGB : TernaryVRSb<"vlvgb", 0xE722, z_vector_insert,
29                          v128b, v128b, GR32, 0>;
30  def VLVGH : TernaryVRSb<"vlvgh", 0xE722, z_vector_insert,
31                          v128h, v128h, GR32, 1>;
32  def VLVGF : TernaryVRSb<"vlvgf", 0xE722, z_vector_insert,
33                          v128f, v128f, GR32, 2>;
34  def VLVGG : TernaryVRSb<"vlvgg", 0xE722, z_vector_insert,
35                          v128g, v128g, GR64, 3>;
36
37  // Load VR from GRs disjoint.
38  def VLVGP : BinaryVRRf<"vlvgp", 0xE762, z_join_dwords, v128g>;
39  def VLVGP32 : BinaryAliasVRRf<GR32>;
40}
41
42// Extractions always assign to the full GR64, even if the element would
43// fit in the lower 32 bits.  Sub-i64 extracts therefore need to take a
44// subreg of the result.
45class VectorExtractSubreg<ValueType type, Instruction insn>
46  : Pat<(i32 (z_vector_extract (type VR128:$vec), shift12only:$index)),
47        (EXTRACT_SUBREG (insn VR128:$vec, shift12only:$index), subreg_l32)>;
48
49def : VectorExtractSubreg<v16i8, VLGVB>;
50def : VectorExtractSubreg<v8i16, VLGVH>;
51def : VectorExtractSubreg<v4i32, VLGVF>;
52
53//===----------------------------------------------------------------------===//
54// Immediate instructions
55//===----------------------------------------------------------------------===//
56
57let Predicates = [FeatureVector] in {
58  let isAsCheapAsAMove = 1, isMoveImm = 1, isReMaterializable = 1 in {
59
60    // Generate byte mask.
61    def VZERO : InherentVRIa<"vzero", 0xE744, 0>;
62    def VONE  : InherentVRIa<"vone", 0xE744, 0xffff>;
63    def VGBM  : UnaryVRIa<"vgbm", 0xE744, z_byte_mask, v128b, imm32zx16>;
64
65    // Generate mask.
66    def VGM  : BinaryVRIbGeneric<"vgm", 0xE746>;
67    def VGMB : BinaryVRIb<"vgmb", 0xE746, z_rotate_mask, v128b, 0>;
68    def VGMH : BinaryVRIb<"vgmh", 0xE746, z_rotate_mask, v128h, 1>;
69    def VGMF : BinaryVRIb<"vgmf", 0xE746, z_rotate_mask, v128f, 2>;
70    def VGMG : BinaryVRIb<"vgmg", 0xE746, z_rotate_mask, v128g, 3>;
71
72    // Replicate immediate.
73    def VREPI  : UnaryVRIaGeneric<"vrepi", 0xE745, imm32sx16>;
74    def VREPIB : UnaryVRIa<"vrepib", 0xE745, z_replicate, v128b, imm32sx16, 0>;
75    def VREPIH : UnaryVRIa<"vrepih", 0xE745, z_replicate, v128h, imm32sx16, 1>;
76    def VREPIF : UnaryVRIa<"vrepif", 0xE745, z_replicate, v128f, imm32sx16, 2>;
77    def VREPIG : UnaryVRIa<"vrepig", 0xE745, z_replicate, v128g, imm32sx16, 3>;
78  }
79
80  // Load element immediate.
81  //
82  // We want these instructions to be used ahead of VLVG* where possible.
83  // However, VLVG* takes a variable BD-format index whereas VLEI takes
84  // a plain immediate index.  This means that VLVG* has an extra "base"
85  // register operand and is 3 units more complex.  Bumping the complexity
86  // of the VLEI* instructions by 4 means that they are strictly better
87  // than VLVG* in cases where both forms match.
88  let AddedComplexity = 4 in {
89    def VLEIB : TernaryVRIa<"vleib", 0xE740, z_vector_insert,
90                            v128b, v128b, imm32sx16trunc, imm32zx4>;
91    def VLEIH : TernaryVRIa<"vleih", 0xE741, z_vector_insert,
92                            v128h, v128h, imm32sx16trunc, imm32zx3>;
93    def VLEIF : TernaryVRIa<"vleif", 0xE743, z_vector_insert,
94                            v128f, v128f, imm32sx16, imm32zx2>;
95    def VLEIG : TernaryVRIa<"vleig", 0xE742, z_vector_insert,
96                            v128g, v128g, imm64sx16, imm32zx1>;
97  }
98}
99
100//===----------------------------------------------------------------------===//
101// Loads
102//===----------------------------------------------------------------------===//
103
104let Predicates = [FeatureVector] in {
105  // Load.
106  def VL : UnaryVRX<"vl", 0xE706, null_frag, v128any, 16>;
107
108  // Load to block boundary.  The number of loaded bytes is only known
109  // at run time.  The instruction is really polymorphic, but v128b matches
110  // the return type of the associated intrinsic.
111  def VLBB : BinaryVRX<"vlbb", 0xE707, int_s390_vlbb, v128b, 0>;
112
113  // Load count to block boundary.
114  let Defs = [CC] in
115    def LCBB : InstRXE<0xE727, (outs GR32:$R1),
116                               (ins bdxaddr12only:$XBD2, imm32zx4:$M3),
117                       "lcbb\t$R1, $XBD2, $M3",
118                       [(set GR32:$R1, (int_s390_lcbb bdxaddr12only:$XBD2,
119                                                      imm32zx4:$M3))]>;
120
121  // Load with length.  The number of loaded bytes is only known at run time.
122  def VLL : BinaryVRSb<"vll", 0xE737, int_s390_vll, 0>;
123
124  // Load multiple.
125  def VLM : LoadMultipleVRSa<"vlm", 0xE736>;
126
127  // Load and replicate
128  def VLREP  : UnaryVRXGeneric<"vlrep", 0xE705>;
129  def VLREPB : UnaryVRX<"vlrepb", 0xE705, z_replicate_loadi8,  v128b, 1, 0>;
130  def VLREPH : UnaryVRX<"vlreph", 0xE705, z_replicate_loadi16, v128h, 2, 1>;
131  def VLREPF : UnaryVRX<"vlrepf", 0xE705, z_replicate_loadi32, v128f, 4, 2>;
132  def VLREPG : UnaryVRX<"vlrepg", 0xE705, z_replicate_loadi64, v128g, 8, 3>;
133  def : Pat<(v4f32 (z_replicate_loadf32 bdxaddr12only:$addr)),
134            (VLREPF bdxaddr12only:$addr)>;
135  def : Pat<(v2f64 (z_replicate_loadf64 bdxaddr12only:$addr)),
136            (VLREPG bdxaddr12only:$addr)>;
137
138  // Use VLREP to load subvectors.  These patterns use "12pair" because
139  // LEY and LDY offer full 20-bit displacement fields.  It's often better
140  // to use those instructions rather than force a 20-bit displacement
141  // into a GPR temporary.
142  let mayLoad = 1 in {
143    def VL32 : UnaryAliasVRX<load, v32sb, bdxaddr12pair>;
144    def VL64 : UnaryAliasVRX<load, v64db, bdxaddr12pair>;
145  }
146
147  // Load logical element and zero.
148  def VLLEZ  : UnaryVRXGeneric<"vllez", 0xE704>;
149  def VLLEZB : UnaryVRX<"vllezb", 0xE704, z_vllezi8,  v128b, 1, 0>;
150  def VLLEZH : UnaryVRX<"vllezh", 0xE704, z_vllezi16, v128h, 2, 1>;
151  def VLLEZF : UnaryVRX<"vllezf", 0xE704, z_vllezi32, v128f, 4, 2>;
152  def VLLEZG : UnaryVRX<"vllezg", 0xE704, z_vllezi64, v128g, 8, 3>;
153  def : Pat<(z_vllezf32 bdxaddr12only:$addr),
154            (VLLEZF bdxaddr12only:$addr)>;
155  def : Pat<(z_vllezf64 bdxaddr12only:$addr),
156            (VLLEZG bdxaddr12only:$addr)>;
157  let Predicates = [FeatureVectorEnhancements1] in {
158    def VLLEZLF : UnaryVRX<"vllezlf", 0xE704, z_vllezli32, v128f, 4, 6>;
159    def : Pat<(z_vllezlf32 bdxaddr12only:$addr),
160              (VLLEZLF bdxaddr12only:$addr)>;
161  }
162
163  // Load element.
164  def VLEB : TernaryVRX<"vleb", 0xE700, z_vlei8,  v128b, v128b, 1, imm32zx4>;
165  def VLEH : TernaryVRX<"vleh", 0xE701, z_vlei16, v128h, v128h, 2, imm32zx3>;
166  def VLEF : TernaryVRX<"vlef", 0xE703, z_vlei32, v128f, v128f, 4, imm32zx2>;
167  def VLEG : TernaryVRX<"vleg", 0xE702, z_vlei64, v128g, v128g, 8, imm32zx1>;
168  def : Pat<(z_vlef32 (v4f32 VR128:$val), bdxaddr12only:$addr, imm32zx2:$index),
169            (VLEF VR128:$val, bdxaddr12only:$addr, imm32zx2:$index)>;
170  def : Pat<(z_vlef64 (v2f64 VR128:$val), bdxaddr12only:$addr, imm32zx1:$index),
171            (VLEG VR128:$val, bdxaddr12only:$addr, imm32zx1:$index)>;
172
173  // Gather element.
174  def VGEF : TernaryVRV<"vgef", 0xE713, 4, imm32zx2>;
175  def VGEG : TernaryVRV<"vgeg", 0xE712, 8, imm32zx1>;
176}
177
178let Predicates = [FeatureVectorPackedDecimal] in {
179  // Load rightmost with length.  The number of loaded bytes is only known
180  // at run time.
181  def VLRL : BinaryVSI<"vlrl", 0xE635, int_s390_vlrl, 0>;
182  def VLRLR : BinaryVRSd<"vlrlr", 0xE637, int_s390_vlrl, 0>;
183}
184
185// Use replicating loads if we're inserting a single element into an
186// undefined vector.  This avoids a false dependency on the previous
187// register contents.
188multiclass ReplicatePeephole<Instruction vlrep, ValueType vectype,
189                             SDPatternOperator load, ValueType scalartype> {
190  def : Pat<(vectype (z_vector_insert
191                      (undef), (scalartype (load bdxaddr12only:$addr)), 0)),
192            (vlrep bdxaddr12only:$addr)>;
193  def : Pat<(vectype (scalar_to_vector
194                      (scalartype (load bdxaddr12only:$addr)))),
195            (vlrep bdxaddr12only:$addr)>;
196}
197defm : ReplicatePeephole<VLREPB, v16i8, anyextloadi8, i32>;
198defm : ReplicatePeephole<VLREPH, v8i16, anyextloadi16, i32>;
199defm : ReplicatePeephole<VLREPF, v4i32, load, i32>;
200defm : ReplicatePeephole<VLREPG, v2i64, load, i64>;
201defm : ReplicatePeephole<VLREPF, v4f32, load, f32>;
202defm : ReplicatePeephole<VLREPG, v2f64, load, f64>;
203
204//===----------------------------------------------------------------------===//
205// Stores
206//===----------------------------------------------------------------------===//
207
208let Predicates = [FeatureVector] in {
209  // Store.
210  def VST : StoreVRX<"vst", 0xE70E, null_frag, v128any, 16>;
211
212  // Store with length.  The number of stored bytes is only known at run time.
213  def VSTL : StoreLengthVRSb<"vstl", 0xE73F, int_s390_vstl, 0>;
214
215  // Store multiple.
216  def VSTM : StoreMultipleVRSa<"vstm", 0xE73E>;
217
218  // Store element.
219  def VSTEB : StoreBinaryVRX<"vsteb", 0xE708, z_vstei8,  v128b, 1, imm32zx4>;
220  def VSTEH : StoreBinaryVRX<"vsteh", 0xE709, z_vstei16, v128h, 2, imm32zx3>;
221  def VSTEF : StoreBinaryVRX<"vstef", 0xE70B, z_vstei32, v128f, 4, imm32zx2>;
222  def VSTEG : StoreBinaryVRX<"vsteg", 0xE70A, z_vstei64, v128g, 8, imm32zx1>;
223  def : Pat<(z_vstef32 (v4f32 VR128:$val), bdxaddr12only:$addr,
224                       imm32zx2:$index),
225            (VSTEF VR128:$val, bdxaddr12only:$addr, imm32zx2:$index)>;
226  def : Pat<(z_vstef64 (v2f64 VR128:$val), bdxaddr12only:$addr,
227                       imm32zx1:$index),
228            (VSTEG VR128:$val, bdxaddr12only:$addr, imm32zx1:$index)>;
229
230  // Use VSTE to store subvectors.  These patterns use "12pair" because
231  // STEY and STDY offer full 20-bit displacement fields.  It's often better
232  // to use those instructions rather than force a 20-bit displacement
233  // into a GPR temporary.
234  let mayStore = 1 in {
235    def VST32 : StoreAliasVRX<store, v32sb, bdxaddr12pair>;
236    def VST64 : StoreAliasVRX<store, v64db, bdxaddr12pair>;
237  }
238
239  // Scatter element.
240  def VSCEF : StoreBinaryVRV<"vscef", 0xE71B, 4, imm32zx2>;
241  def VSCEG : StoreBinaryVRV<"vsceg", 0xE71A, 8, imm32zx1>;
242}
243
244let Predicates = [FeatureVectorPackedDecimal] in {
245  // Store rightmost with length.  The number of stored bytes is only known
246  // at run time.
247  def VSTRL : StoreLengthVSI<"vstrl", 0xE63D, int_s390_vstrl, 0>;
248  def VSTRLR : StoreLengthVRSd<"vstrlr", 0xE63F, int_s390_vstrl, 0>;
249}
250
251//===----------------------------------------------------------------------===//
252// Selects and permutes
253//===----------------------------------------------------------------------===//
254
255let Predicates = [FeatureVector] in {
256  // Merge high.
257  def VMRH:   BinaryVRRcGeneric<"vmrh", 0xE761>;
258  def VMRHB : BinaryVRRc<"vmrhb", 0xE761, z_merge_high, v128b, v128b, 0>;
259  def VMRHH : BinaryVRRc<"vmrhh", 0xE761, z_merge_high, v128h, v128h, 1>;
260  def VMRHF : BinaryVRRc<"vmrhf", 0xE761, z_merge_high, v128f, v128f, 2>;
261  def VMRHG : BinaryVRRc<"vmrhg", 0xE761, z_merge_high, v128g, v128g, 3>;
262  def : BinaryRRWithType<VMRHF, VR128, z_merge_high, v4f32>;
263  def : BinaryRRWithType<VMRHG, VR128, z_merge_high, v2f64>;
264
265  // Merge low.
266  def VMRL:   BinaryVRRcGeneric<"vmrl", 0xE760>;
267  def VMRLB : BinaryVRRc<"vmrlb", 0xE760, z_merge_low, v128b, v128b, 0>;
268  def VMRLH : BinaryVRRc<"vmrlh", 0xE760, z_merge_low, v128h, v128h, 1>;
269  def VMRLF : BinaryVRRc<"vmrlf", 0xE760, z_merge_low, v128f, v128f, 2>;
270  def VMRLG : BinaryVRRc<"vmrlg", 0xE760, z_merge_low, v128g, v128g, 3>;
271  def : BinaryRRWithType<VMRLF, VR128, z_merge_low, v4f32>;
272  def : BinaryRRWithType<VMRLG, VR128, z_merge_low, v2f64>;
273
274  // Permute.
275  def VPERM : TernaryVRRe<"vperm", 0xE78C, z_permute, v128b, v128b>;
276
277  // Permute doubleword immediate.
278  def VPDI : TernaryVRRc<"vpdi", 0xE784, z_permute_dwords, v128g, v128g>;
279
280  // Bit Permute.
281  let Predicates = [FeatureVectorEnhancements1] in
282    def VBPERM : BinaryVRRc<"vbperm", 0xE785, int_s390_vbperm, v128g, v128b>;
283
284  // Replicate.
285  def VREP:   BinaryVRIcGeneric<"vrep", 0xE74D>;
286  def VREPB : BinaryVRIc<"vrepb", 0xE74D, z_splat, v128b, v128b, 0>;
287  def VREPH : BinaryVRIc<"vreph", 0xE74D, z_splat, v128h, v128h, 1>;
288  def VREPF : BinaryVRIc<"vrepf", 0xE74D, z_splat, v128f, v128f, 2>;
289  def VREPG : BinaryVRIc<"vrepg", 0xE74D, z_splat, v128g, v128g, 3>;
290  def : Pat<(v4f32 (z_splat VR128:$vec, imm32zx16:$index)),
291            (VREPF VR128:$vec, imm32zx16:$index)>;
292  def : Pat<(v2f64 (z_splat VR128:$vec, imm32zx16:$index)),
293            (VREPG VR128:$vec, imm32zx16:$index)>;
294
295  // Select.
296  def VSEL : TernaryVRRe<"vsel", 0xE78D, null_frag, v128any, v128any>;
297}
298
299//===----------------------------------------------------------------------===//
300// Widening and narrowing
301//===----------------------------------------------------------------------===//
302
303let Predicates = [FeatureVector] in {
304  // Pack
305  def VPK  : BinaryVRRcGeneric<"vpk", 0xE794>;
306  def VPKH : BinaryVRRc<"vpkh", 0xE794, z_pack, v128b, v128h, 1>;
307  def VPKF : BinaryVRRc<"vpkf", 0xE794, z_pack, v128h, v128f, 2>;
308  def VPKG : BinaryVRRc<"vpkg", 0xE794, z_pack, v128f, v128g, 3>;
309
310  // Pack saturate.
311  def  VPKS  : BinaryVRRbSPairGeneric<"vpks", 0xE797>;
312  defm VPKSH : BinaryVRRbSPair<"vpksh", 0xE797, int_s390_vpksh, z_packs_cc,
313                               v128b, v128h, 1>;
314  defm VPKSF : BinaryVRRbSPair<"vpksf", 0xE797, int_s390_vpksf, z_packs_cc,
315                               v128h, v128f, 2>;
316  defm VPKSG : BinaryVRRbSPair<"vpksg", 0xE797, int_s390_vpksg, z_packs_cc,
317                               v128f, v128g, 3>;
318
319  // Pack saturate logical.
320  def  VPKLS  : BinaryVRRbSPairGeneric<"vpkls", 0xE795>;
321  defm VPKLSH : BinaryVRRbSPair<"vpklsh", 0xE795, int_s390_vpklsh, z_packls_cc,
322                                v128b, v128h, 1>;
323  defm VPKLSF : BinaryVRRbSPair<"vpklsf", 0xE795, int_s390_vpklsf, z_packls_cc,
324                                v128h, v128f, 2>;
325  defm VPKLSG : BinaryVRRbSPair<"vpklsg", 0xE795, int_s390_vpklsg, z_packls_cc,
326                                v128f, v128g, 3>;
327
328  // Sign-extend to doubleword.
329  def VSEG  : UnaryVRRaGeneric<"vseg", 0xE75F>;
330  def VSEGB : UnaryVRRa<"vsegb", 0xE75F, z_vsei8,  v128g, v128g, 0>;
331  def VSEGH : UnaryVRRa<"vsegh", 0xE75F, z_vsei16, v128g, v128g, 1>;
332  def VSEGF : UnaryVRRa<"vsegf", 0xE75F, z_vsei32, v128g, v128g, 2>;
333  def : Pat<(z_vsei8_by_parts  (v16i8 VR128:$src)), (VSEGB VR128:$src)>;
334  def : Pat<(z_vsei16_by_parts (v8i16 VR128:$src)), (VSEGH VR128:$src)>;
335  def : Pat<(z_vsei32_by_parts (v4i32 VR128:$src)), (VSEGF VR128:$src)>;
336
337  // Unpack high.
338  def VUPH  : UnaryVRRaGeneric<"vuph", 0xE7D7>;
339  def VUPHB : UnaryVRRa<"vuphb", 0xE7D7, z_unpack_high, v128h, v128b, 0>;
340  def VUPHH : UnaryVRRa<"vuphh", 0xE7D7, z_unpack_high, v128f, v128h, 1>;
341  def VUPHF : UnaryVRRa<"vuphf", 0xE7D7, z_unpack_high, v128g, v128f, 2>;
342
343  // Unpack logical high.
344  def VUPLH  : UnaryVRRaGeneric<"vuplh", 0xE7D5>;
345  def VUPLHB : UnaryVRRa<"vuplhb", 0xE7D5, z_unpackl_high, v128h, v128b, 0>;
346  def VUPLHH : UnaryVRRa<"vuplhh", 0xE7D5, z_unpackl_high, v128f, v128h, 1>;
347  def VUPLHF : UnaryVRRa<"vuplhf", 0xE7D5, z_unpackl_high, v128g, v128f, 2>;
348
349  // Unpack low.
350  def VUPL   : UnaryVRRaGeneric<"vupl", 0xE7D6>;
351  def VUPLB  : UnaryVRRa<"vuplb",  0xE7D6, z_unpack_low, v128h, v128b, 0>;
352  def VUPLHW : UnaryVRRa<"vuplhw", 0xE7D6, z_unpack_low, v128f, v128h, 1>;
353  def VUPLF  : UnaryVRRa<"vuplf",  0xE7D6, z_unpack_low, v128g, v128f, 2>;
354
355  // Unpack logical low.
356  def VUPLL  : UnaryVRRaGeneric<"vupll", 0xE7D4>;
357  def VUPLLB : UnaryVRRa<"vupllb", 0xE7D4, z_unpackl_low, v128h, v128b, 0>;
358  def VUPLLH : UnaryVRRa<"vupllh", 0xE7D4, z_unpackl_low, v128f, v128h, 1>;
359  def VUPLLF : UnaryVRRa<"vupllf", 0xE7D4, z_unpackl_low, v128g, v128f, 2>;
360}
361
362//===----------------------------------------------------------------------===//
363// Instantiating generic operations for specific types.
364//===----------------------------------------------------------------------===//
365
366multiclass GenericVectorOps<ValueType type, ValueType inttype> {
367  let Predicates = [FeatureVector] in {
368    def : Pat<(type (load bdxaddr12only:$addr)),
369              (VL bdxaddr12only:$addr)>;
370    def : Pat<(store (type VR128:$src), bdxaddr12only:$addr),
371              (VST VR128:$src, bdxaddr12only:$addr)>;
372    def : Pat<(type (vselect (inttype VR128:$x), VR128:$y, VR128:$z)),
373              (VSEL VR128:$y, VR128:$z, VR128:$x)>;
374    def : Pat<(type (vselect (inttype (z_vnot VR128:$x)), VR128:$y, VR128:$z)),
375              (VSEL VR128:$z, VR128:$y, VR128:$x)>;
376  }
377}
378
379defm : GenericVectorOps<v16i8, v16i8>;
380defm : GenericVectorOps<v8i16, v8i16>;
381defm : GenericVectorOps<v4i32, v4i32>;
382defm : GenericVectorOps<v2i64, v2i64>;
383defm : GenericVectorOps<v4f32, v4i32>;
384defm : GenericVectorOps<v2f64, v2i64>;
385
386//===----------------------------------------------------------------------===//
387// Integer arithmetic
388//===----------------------------------------------------------------------===//
389
390let Predicates = [FeatureVector] in {
391  // Add.
392  def VA  : BinaryVRRcGeneric<"va", 0xE7F3>;
393  def VAB : BinaryVRRc<"vab", 0xE7F3, add, v128b, v128b, 0>;
394  def VAH : BinaryVRRc<"vah", 0xE7F3, add, v128h, v128h, 1>;
395  def VAF : BinaryVRRc<"vaf", 0xE7F3, add, v128f, v128f, 2>;
396  def VAG : BinaryVRRc<"vag", 0xE7F3, add, v128g, v128g, 3>;
397  def VAQ : BinaryVRRc<"vaq", 0xE7F3, int_s390_vaq, v128q, v128q, 4>;
398
399  // Add compute carry.
400  def VACC  : BinaryVRRcGeneric<"vacc", 0xE7F1>;
401  def VACCB : BinaryVRRc<"vaccb", 0xE7F1, int_s390_vaccb, v128b, v128b, 0>;
402  def VACCH : BinaryVRRc<"vacch", 0xE7F1, int_s390_vacch, v128h, v128h, 1>;
403  def VACCF : BinaryVRRc<"vaccf", 0xE7F1, int_s390_vaccf, v128f, v128f, 2>;
404  def VACCG : BinaryVRRc<"vaccg", 0xE7F1, int_s390_vaccg, v128g, v128g, 3>;
405  def VACCQ : BinaryVRRc<"vaccq", 0xE7F1, int_s390_vaccq, v128q, v128q, 4>;
406
407  // Add with carry.
408  def VAC  : TernaryVRRdGeneric<"vac", 0xE7BB>;
409  def VACQ : TernaryVRRd<"vacq", 0xE7BB, int_s390_vacq, v128q, v128q, 4>;
410
411  // Add with carry compute carry.
412  def VACCC  : TernaryVRRdGeneric<"vaccc", 0xE7B9>;
413  def VACCCQ : TernaryVRRd<"vacccq", 0xE7B9, int_s390_vacccq, v128q, v128q, 4>;
414
415  // And.
416  def VN : BinaryVRRc<"vn", 0xE768, null_frag, v128any, v128any>;
417
418  // And with complement.
419  def VNC : BinaryVRRc<"vnc", 0xE769, null_frag, v128any, v128any>;
420
421  // Average.
422  def VAVG  : BinaryVRRcGeneric<"vavg", 0xE7F2>;
423  def VAVGB : BinaryVRRc<"vavgb", 0xE7F2, int_s390_vavgb, v128b, v128b, 0>;
424  def VAVGH : BinaryVRRc<"vavgh", 0xE7F2, int_s390_vavgh, v128h, v128h, 1>;
425  def VAVGF : BinaryVRRc<"vavgf", 0xE7F2, int_s390_vavgf, v128f, v128f, 2>;
426  def VAVGG : BinaryVRRc<"vavgg", 0xE7F2, int_s390_vavgg, v128g, v128g, 3>;
427
428  // Average logical.
429  def VAVGL  : BinaryVRRcGeneric<"vavgl", 0xE7F0>;
430  def VAVGLB : BinaryVRRc<"vavglb", 0xE7F0, int_s390_vavglb, v128b, v128b, 0>;
431  def VAVGLH : BinaryVRRc<"vavglh", 0xE7F0, int_s390_vavglh, v128h, v128h, 1>;
432  def VAVGLF : BinaryVRRc<"vavglf", 0xE7F0, int_s390_vavglf, v128f, v128f, 2>;
433  def VAVGLG : BinaryVRRc<"vavglg", 0xE7F0, int_s390_vavglg, v128g, v128g, 3>;
434
435  // Checksum.
436  def VCKSM : BinaryVRRc<"vcksm", 0xE766, int_s390_vcksm, v128f, v128f>;
437
438  // Count leading zeros.
439  def VCLZ  : UnaryVRRaGeneric<"vclz", 0xE753>;
440  def VCLZB : UnaryVRRa<"vclzb", 0xE753, ctlz, v128b, v128b, 0>;
441  def VCLZH : UnaryVRRa<"vclzh", 0xE753, ctlz, v128h, v128h, 1>;
442  def VCLZF : UnaryVRRa<"vclzf", 0xE753, ctlz, v128f, v128f, 2>;
443  def VCLZG : UnaryVRRa<"vclzg", 0xE753, ctlz, v128g, v128g, 3>;
444
445  // Count trailing zeros.
446  def VCTZ  : UnaryVRRaGeneric<"vctz", 0xE752>;
447  def VCTZB : UnaryVRRa<"vctzb", 0xE752, cttz, v128b, v128b, 0>;
448  def VCTZH : UnaryVRRa<"vctzh", 0xE752, cttz, v128h, v128h, 1>;
449  def VCTZF : UnaryVRRa<"vctzf", 0xE752, cttz, v128f, v128f, 2>;
450  def VCTZG : UnaryVRRa<"vctzg", 0xE752, cttz, v128g, v128g, 3>;
451
452  // Not exclusive or.
453  let Predicates = [FeatureVectorEnhancements1] in
454    def VNX : BinaryVRRc<"vnx", 0xE76C, null_frag, v128any, v128any>;
455
456  // Exclusive or.
457  def VX : BinaryVRRc<"vx", 0xE76D, null_frag, v128any, v128any>;
458
459  // Galois field multiply sum.
460  def VGFM  : BinaryVRRcGeneric<"vgfm", 0xE7B4>;
461  def VGFMB : BinaryVRRc<"vgfmb", 0xE7B4, int_s390_vgfmb, v128h, v128b, 0>;
462  def VGFMH : BinaryVRRc<"vgfmh", 0xE7B4, int_s390_vgfmh, v128f, v128h, 1>;
463  def VGFMF : BinaryVRRc<"vgfmf", 0xE7B4, int_s390_vgfmf, v128g, v128f, 2>;
464  def VGFMG : BinaryVRRc<"vgfmg", 0xE7B4, int_s390_vgfmg, v128q, v128g, 3>;
465
466  // Galois field multiply sum and accumulate.
467  def VGFMA  : TernaryVRRdGeneric<"vgfma", 0xE7BC>;
468  def VGFMAB : TernaryVRRd<"vgfmab", 0xE7BC, int_s390_vgfmab, v128h, v128b, 0>;
469  def VGFMAH : TernaryVRRd<"vgfmah", 0xE7BC, int_s390_vgfmah, v128f, v128h, 1>;
470  def VGFMAF : TernaryVRRd<"vgfmaf", 0xE7BC, int_s390_vgfmaf, v128g, v128f, 2>;
471  def VGFMAG : TernaryVRRd<"vgfmag", 0xE7BC, int_s390_vgfmag, v128q, v128g, 3>;
472
473  // Load complement.
474  def VLC  : UnaryVRRaGeneric<"vlc", 0xE7DE>;
475  def VLCB : UnaryVRRa<"vlcb", 0xE7DE, z_vneg, v128b, v128b, 0>;
476  def VLCH : UnaryVRRa<"vlch", 0xE7DE, z_vneg, v128h, v128h, 1>;
477  def VLCF : UnaryVRRa<"vlcf", 0xE7DE, z_vneg, v128f, v128f, 2>;
478  def VLCG : UnaryVRRa<"vlcg", 0xE7DE, z_vneg, v128g, v128g, 3>;
479
480  // Load positive.
481  def VLP  : UnaryVRRaGeneric<"vlp", 0xE7DF>;
482  def VLPB : UnaryVRRa<"vlpb", 0xE7DF, z_viabs8,  v128b, v128b, 0>;
483  def VLPH : UnaryVRRa<"vlph", 0xE7DF, z_viabs16, v128h, v128h, 1>;
484  def VLPF : UnaryVRRa<"vlpf", 0xE7DF, z_viabs32, v128f, v128f, 2>;
485  def VLPG : UnaryVRRa<"vlpg", 0xE7DF, z_viabs64, v128g, v128g, 3>;
486
487  // Maximum.
488  def VMX  : BinaryVRRcGeneric<"vmx", 0xE7FF>;
489  def VMXB : BinaryVRRc<"vmxb", 0xE7FF, null_frag, v128b, v128b, 0>;
490  def VMXH : BinaryVRRc<"vmxh", 0xE7FF, null_frag, v128h, v128h, 1>;
491  def VMXF : BinaryVRRc<"vmxf", 0xE7FF, null_frag, v128f, v128f, 2>;
492  def VMXG : BinaryVRRc<"vmxg", 0xE7FF, null_frag, v128g, v128g, 3>;
493
494  // Maximum logical.
495  def VMXL  : BinaryVRRcGeneric<"vmxl", 0xE7FD>;
496  def VMXLB : BinaryVRRc<"vmxlb", 0xE7FD, null_frag, v128b, v128b, 0>;
497  def VMXLH : BinaryVRRc<"vmxlh", 0xE7FD, null_frag, v128h, v128h, 1>;
498  def VMXLF : BinaryVRRc<"vmxlf", 0xE7FD, null_frag, v128f, v128f, 2>;
499  def VMXLG : BinaryVRRc<"vmxlg", 0xE7FD, null_frag, v128g, v128g, 3>;
500
501  // Minimum.
502  def VMN  : BinaryVRRcGeneric<"vmn", 0xE7FE>;
503  def VMNB : BinaryVRRc<"vmnb", 0xE7FE, null_frag, v128b, v128b, 0>;
504  def VMNH : BinaryVRRc<"vmnh", 0xE7FE, null_frag, v128h, v128h, 1>;
505  def VMNF : BinaryVRRc<"vmnf", 0xE7FE, null_frag, v128f, v128f, 2>;
506  def VMNG : BinaryVRRc<"vmng", 0xE7FE, null_frag, v128g, v128g, 3>;
507
508  // Minimum logical.
509  def VMNL  : BinaryVRRcGeneric<"vmnl", 0xE7FC>;
510  def VMNLB : BinaryVRRc<"vmnlb", 0xE7FC, null_frag, v128b, v128b, 0>;
511  def VMNLH : BinaryVRRc<"vmnlh", 0xE7FC, null_frag, v128h, v128h, 1>;
512  def VMNLF : BinaryVRRc<"vmnlf", 0xE7FC, null_frag, v128f, v128f, 2>;
513  def VMNLG : BinaryVRRc<"vmnlg", 0xE7FC, null_frag, v128g, v128g, 3>;
514
515  // Multiply and add low.
516  def VMAL   : TernaryVRRdGeneric<"vmal", 0xE7AA>;
517  def VMALB  : TernaryVRRd<"vmalb",  0xE7AA, z_muladd, v128b, v128b, 0>;
518  def VMALHW : TernaryVRRd<"vmalhw", 0xE7AA, z_muladd, v128h, v128h, 1>;
519  def VMALF  : TernaryVRRd<"vmalf",  0xE7AA, z_muladd, v128f, v128f, 2>;
520
521  // Multiply and add high.
522  def VMAH  : TernaryVRRdGeneric<"vmah", 0xE7AB>;
523  def VMAHB : TernaryVRRd<"vmahb", 0xE7AB, int_s390_vmahb, v128b, v128b, 0>;
524  def VMAHH : TernaryVRRd<"vmahh", 0xE7AB, int_s390_vmahh, v128h, v128h, 1>;
525  def VMAHF : TernaryVRRd<"vmahf", 0xE7AB, int_s390_vmahf, v128f, v128f, 2>;
526
527  // Multiply and add logical high.
528  def VMALH  : TernaryVRRdGeneric<"vmalh", 0xE7A9>;
529  def VMALHB : TernaryVRRd<"vmalhb", 0xE7A9, int_s390_vmalhb, v128b, v128b, 0>;
530  def VMALHH : TernaryVRRd<"vmalhh", 0xE7A9, int_s390_vmalhh, v128h, v128h, 1>;
531  def VMALHF : TernaryVRRd<"vmalhf", 0xE7A9, int_s390_vmalhf, v128f, v128f, 2>;
532
533  // Multiply and add even.
534  def VMAE  : TernaryVRRdGeneric<"vmae", 0xE7AE>;
535  def VMAEB : TernaryVRRd<"vmaeb", 0xE7AE, int_s390_vmaeb, v128h, v128b, 0>;
536  def VMAEH : TernaryVRRd<"vmaeh", 0xE7AE, int_s390_vmaeh, v128f, v128h, 1>;
537  def VMAEF : TernaryVRRd<"vmaef", 0xE7AE, int_s390_vmaef, v128g, v128f, 2>;
538
539  // Multiply and add logical even.
540  def VMALE  : TernaryVRRdGeneric<"vmale", 0xE7AC>;
541  def VMALEB : TernaryVRRd<"vmaleb", 0xE7AC, int_s390_vmaleb, v128h, v128b, 0>;
542  def VMALEH : TernaryVRRd<"vmaleh", 0xE7AC, int_s390_vmaleh, v128f, v128h, 1>;
543  def VMALEF : TernaryVRRd<"vmalef", 0xE7AC, int_s390_vmalef, v128g, v128f, 2>;
544
545  // Multiply and add odd.
546  def VMAO  : TernaryVRRdGeneric<"vmao", 0xE7AF>;
547  def VMAOB : TernaryVRRd<"vmaob", 0xE7AF, int_s390_vmaob, v128h, v128b, 0>;
548  def VMAOH : TernaryVRRd<"vmaoh", 0xE7AF, int_s390_vmaoh, v128f, v128h, 1>;
549  def VMAOF : TernaryVRRd<"vmaof", 0xE7AF, int_s390_vmaof, v128g, v128f, 2>;
550
551  // Multiply and add logical odd.
552  def VMALO  : TernaryVRRdGeneric<"vmalo", 0xE7AD>;
553  def VMALOB : TernaryVRRd<"vmalob", 0xE7AD, int_s390_vmalob, v128h, v128b, 0>;
554  def VMALOH : TernaryVRRd<"vmaloh", 0xE7AD, int_s390_vmaloh, v128f, v128h, 1>;
555  def VMALOF : TernaryVRRd<"vmalof", 0xE7AD, int_s390_vmalof, v128g, v128f, 2>;
556
557  // Multiply high.
558  def VMH  : BinaryVRRcGeneric<"vmh", 0xE7A3>;
559  def VMHB : BinaryVRRc<"vmhb", 0xE7A3, int_s390_vmhb, v128b, v128b, 0>;
560  def VMHH : BinaryVRRc<"vmhh", 0xE7A3, int_s390_vmhh, v128h, v128h, 1>;
561  def VMHF : BinaryVRRc<"vmhf", 0xE7A3, int_s390_vmhf, v128f, v128f, 2>;
562
563  // Multiply logical high.
564  def VMLH  : BinaryVRRcGeneric<"vmlh", 0xE7A1>;
565  def VMLHB : BinaryVRRc<"vmlhb", 0xE7A1, int_s390_vmlhb, v128b, v128b, 0>;
566  def VMLHH : BinaryVRRc<"vmlhh", 0xE7A1, int_s390_vmlhh, v128h, v128h, 1>;
567  def VMLHF : BinaryVRRc<"vmlhf", 0xE7A1, int_s390_vmlhf, v128f, v128f, 2>;
568
569  // Multiply low.
570  def VML   : BinaryVRRcGeneric<"vml", 0xE7A2>;
571  def VMLB  : BinaryVRRc<"vmlb",  0xE7A2, mul, v128b, v128b, 0>;
572  def VMLHW : BinaryVRRc<"vmlhw", 0xE7A2, mul, v128h, v128h, 1>;
573  def VMLF  : BinaryVRRc<"vmlf",  0xE7A2, mul, v128f, v128f, 2>;
574
575  // Multiply even.
576  def VME  : BinaryVRRcGeneric<"vme", 0xE7A6>;
577  def VMEB : BinaryVRRc<"vmeb", 0xE7A6, int_s390_vmeb, v128h, v128b, 0>;
578  def VMEH : BinaryVRRc<"vmeh", 0xE7A6, int_s390_vmeh, v128f, v128h, 1>;
579  def VMEF : BinaryVRRc<"vmef", 0xE7A6, int_s390_vmef, v128g, v128f, 2>;
580
581  // Multiply logical even.
582  def VMLE  : BinaryVRRcGeneric<"vmle", 0xE7A4>;
583  def VMLEB : BinaryVRRc<"vmleb", 0xE7A4, int_s390_vmleb, v128h, v128b, 0>;
584  def VMLEH : BinaryVRRc<"vmleh", 0xE7A4, int_s390_vmleh, v128f, v128h, 1>;
585  def VMLEF : BinaryVRRc<"vmlef", 0xE7A4, int_s390_vmlef, v128g, v128f, 2>;
586
587  // Multiply odd.
588  def VMO  : BinaryVRRcGeneric<"vmo", 0xE7A7>;
589  def VMOB : BinaryVRRc<"vmob", 0xE7A7, int_s390_vmob, v128h, v128b, 0>;
590  def VMOH : BinaryVRRc<"vmoh", 0xE7A7, int_s390_vmoh, v128f, v128h, 1>;
591  def VMOF : BinaryVRRc<"vmof", 0xE7A7, int_s390_vmof, v128g, v128f, 2>;
592
593  // Multiply logical odd.
594  def VMLO  : BinaryVRRcGeneric<"vmlo", 0xE7A5>;
595  def VMLOB : BinaryVRRc<"vmlob", 0xE7A5, int_s390_vmlob, v128h, v128b, 0>;
596  def VMLOH : BinaryVRRc<"vmloh", 0xE7A5, int_s390_vmloh, v128f, v128h, 1>;
597  def VMLOF : BinaryVRRc<"vmlof", 0xE7A5, int_s390_vmlof, v128g, v128f, 2>;
598
599  // Multiply sum logical.
600  let Predicates = [FeatureVectorEnhancements1] in {
601    def VMSL  : QuaternaryVRRdGeneric<"vmsl", 0xE7B8>;
602    def VMSLG : QuaternaryVRRd<"vmslg", 0xE7B8, int_s390_vmslg,
603                               v128q, v128g, v128g, v128q, 3>;
604  }
605
606  // Nand.
607  let Predicates = [FeatureVectorEnhancements1] in
608    def VNN : BinaryVRRc<"vnn", 0xE76E, null_frag, v128any, v128any>;
609
610  // Nor.
611  def VNO : BinaryVRRc<"vno", 0xE76B, null_frag, v128any, v128any>;
612  def : InstAlias<"vnot\t$V1, $V2", (VNO VR128:$V1, VR128:$V2, VR128:$V2), 0>;
613
614  // Or.
615  def VO : BinaryVRRc<"vo", 0xE76A, null_frag, v128any, v128any>;
616
617  // Or with complement.
618  let Predicates = [FeatureVectorEnhancements1] in
619    def VOC : BinaryVRRc<"voc", 0xE76F, null_frag, v128any, v128any>;
620
621  // Population count.
622  def VPOPCT : UnaryVRRaGeneric<"vpopct", 0xE750>;
623  def : Pat<(v16i8 (z_popcnt VR128:$x)), (VPOPCT VR128:$x, 0)>;
624  let Predicates = [FeatureVectorEnhancements1] in {
625    def VPOPCTB : UnaryVRRa<"vpopctb", 0xE750, ctpop, v128b, v128b, 0>;
626    def VPOPCTH : UnaryVRRa<"vpopcth", 0xE750, ctpop, v128h, v128h, 1>;
627    def VPOPCTF : UnaryVRRa<"vpopctf", 0xE750, ctpop, v128f, v128f, 2>;
628    def VPOPCTG : UnaryVRRa<"vpopctg", 0xE750, ctpop, v128g, v128g, 3>;
629  }
630
631  // Element rotate left logical (with vector shift amount).
632  def VERLLV  : BinaryVRRcGeneric<"verllv", 0xE773>;
633  def VERLLVB : BinaryVRRc<"verllvb", 0xE773, int_s390_verllvb,
634                           v128b, v128b, 0>;
635  def VERLLVH : BinaryVRRc<"verllvh", 0xE773, int_s390_verllvh,
636                           v128h, v128h, 1>;
637  def VERLLVF : BinaryVRRc<"verllvf", 0xE773, int_s390_verllvf,
638                           v128f, v128f, 2>;
639  def VERLLVG : BinaryVRRc<"verllvg", 0xE773, int_s390_verllvg,
640                           v128g, v128g, 3>;
641
642  // Element rotate left logical (with scalar shift amount).
643  def VERLL  : BinaryVRSaGeneric<"verll", 0xE733>;
644  def VERLLB : BinaryVRSa<"verllb", 0xE733, int_s390_verllb, v128b, v128b, 0>;
645  def VERLLH : BinaryVRSa<"verllh", 0xE733, int_s390_verllh, v128h, v128h, 1>;
646  def VERLLF : BinaryVRSa<"verllf", 0xE733, int_s390_verllf, v128f, v128f, 2>;
647  def VERLLG : BinaryVRSa<"verllg", 0xE733, int_s390_verllg, v128g, v128g, 3>;
648
649  // Element rotate and insert under mask.
650  def VERIM  : QuaternaryVRIdGeneric<"verim", 0xE772>;
651  def VERIMB : QuaternaryVRId<"verimb", 0xE772, int_s390_verimb, v128b, v128b, 0>;
652  def VERIMH : QuaternaryVRId<"verimh", 0xE772, int_s390_verimh, v128h, v128h, 1>;
653  def VERIMF : QuaternaryVRId<"verimf", 0xE772, int_s390_verimf, v128f, v128f, 2>;
654  def VERIMG : QuaternaryVRId<"verimg", 0xE772, int_s390_verimg, v128g, v128g, 3>;
655
656  // Element shift left (with vector shift amount).
657  def VESLV  : BinaryVRRcGeneric<"veslv", 0xE770>;
658  def VESLVB : BinaryVRRc<"veslvb", 0xE770, z_vshl, v128b, v128b, 0>;
659  def VESLVH : BinaryVRRc<"veslvh", 0xE770, z_vshl, v128h, v128h, 1>;
660  def VESLVF : BinaryVRRc<"veslvf", 0xE770, z_vshl, v128f, v128f, 2>;
661  def VESLVG : BinaryVRRc<"veslvg", 0xE770, z_vshl, v128g, v128g, 3>;
662
663  // Element shift left (with scalar shift amount).
664  def VESL  : BinaryVRSaGeneric<"vesl", 0xE730>;
665  def VESLB : BinaryVRSa<"veslb", 0xE730, z_vshl_by_scalar, v128b, v128b, 0>;
666  def VESLH : BinaryVRSa<"veslh", 0xE730, z_vshl_by_scalar, v128h, v128h, 1>;
667  def VESLF : BinaryVRSa<"veslf", 0xE730, z_vshl_by_scalar, v128f, v128f, 2>;
668  def VESLG : BinaryVRSa<"veslg", 0xE730, z_vshl_by_scalar, v128g, v128g, 3>;
669
670  // Element shift right arithmetic (with vector shift amount).
671  def VESRAV  : BinaryVRRcGeneric<"vesrav", 0xE77A>;
672  def VESRAVB : BinaryVRRc<"vesravb", 0xE77A, z_vsra, v128b, v128b, 0>;
673  def VESRAVH : BinaryVRRc<"vesravh", 0xE77A, z_vsra, v128h, v128h, 1>;
674  def VESRAVF : BinaryVRRc<"vesravf", 0xE77A, z_vsra, v128f, v128f, 2>;
675  def VESRAVG : BinaryVRRc<"vesravg", 0xE77A, z_vsra, v128g, v128g, 3>;
676
677  // Element shift right arithmetic (with scalar shift amount).
678  def VESRA  : BinaryVRSaGeneric<"vesra", 0xE73A>;
679  def VESRAB : BinaryVRSa<"vesrab", 0xE73A, z_vsra_by_scalar, v128b, v128b, 0>;
680  def VESRAH : BinaryVRSa<"vesrah", 0xE73A, z_vsra_by_scalar, v128h, v128h, 1>;
681  def VESRAF : BinaryVRSa<"vesraf", 0xE73A, z_vsra_by_scalar, v128f, v128f, 2>;
682  def VESRAG : BinaryVRSa<"vesrag", 0xE73A, z_vsra_by_scalar, v128g, v128g, 3>;
683
684  // Element shift right logical (with vector shift amount).
685  def VESRLV  : BinaryVRRcGeneric<"vesrlv", 0xE778>;
686  def VESRLVB : BinaryVRRc<"vesrlvb", 0xE778, z_vsrl, v128b, v128b, 0>;
687  def VESRLVH : BinaryVRRc<"vesrlvh", 0xE778, z_vsrl, v128h, v128h, 1>;
688  def VESRLVF : BinaryVRRc<"vesrlvf", 0xE778, z_vsrl, v128f, v128f, 2>;
689  def VESRLVG : BinaryVRRc<"vesrlvg", 0xE778, z_vsrl, v128g, v128g, 3>;
690
691  // Element shift right logical (with scalar shift amount).
692  def VESRL  : BinaryVRSaGeneric<"vesrl", 0xE738>;
693  def VESRLB : BinaryVRSa<"vesrlb", 0xE738, z_vsrl_by_scalar, v128b, v128b, 0>;
694  def VESRLH : BinaryVRSa<"vesrlh", 0xE738, z_vsrl_by_scalar, v128h, v128h, 1>;
695  def VESRLF : BinaryVRSa<"vesrlf", 0xE738, z_vsrl_by_scalar, v128f, v128f, 2>;
696  def VESRLG : BinaryVRSa<"vesrlg", 0xE738, z_vsrl_by_scalar, v128g, v128g, 3>;
697
698  // Shift left.
699  def VSL : BinaryVRRc<"vsl", 0xE774, int_s390_vsl, v128b, v128b>;
700
701  // Shift left by byte.
702  def VSLB : BinaryVRRc<"vslb", 0xE775, int_s390_vslb, v128b, v128b>;
703
704  // Shift left double by byte.
705  def VSLDB : TernaryVRId<"vsldb", 0xE777, z_shl_double, v128b, v128b, 0>;
706  def : Pat<(int_s390_vsldb VR128:$x, VR128:$y, imm32zx8:$z),
707            (VSLDB VR128:$x, VR128:$y, imm32zx8:$z)>;
708
709  // Shift right arithmetic.
710  def VSRA : BinaryVRRc<"vsra", 0xE77E, int_s390_vsra, v128b, v128b>;
711
712  // Shift right arithmetic by byte.
713  def VSRAB : BinaryVRRc<"vsrab", 0xE77F, int_s390_vsrab, v128b, v128b>;
714
715  // Shift right logical.
716  def VSRL : BinaryVRRc<"vsrl", 0xE77C, int_s390_vsrl, v128b, v128b>;
717
718  // Shift right logical by byte.
719  def VSRLB : BinaryVRRc<"vsrlb", 0xE77D, int_s390_vsrlb, v128b, v128b>;
720
721  // Subtract.
722  def VS  : BinaryVRRcGeneric<"vs", 0xE7F7>;
723  def VSB : BinaryVRRc<"vsb", 0xE7F7, sub, v128b, v128b, 0>;
724  def VSH : BinaryVRRc<"vsh", 0xE7F7, sub, v128h, v128h, 1>;
725  def VSF : BinaryVRRc<"vsf", 0xE7F7, sub, v128f, v128f, 2>;
726  def VSG : BinaryVRRc<"vsg", 0xE7F7, sub, v128g, v128g, 3>;
727  def VSQ : BinaryVRRc<"vsq", 0xE7F7, int_s390_vsq, v128q, v128q, 4>;
728
729  // Subtract compute borrow indication.
730  def VSCBI  : BinaryVRRcGeneric<"vscbi", 0xE7F5>;
731  def VSCBIB : BinaryVRRc<"vscbib", 0xE7F5, int_s390_vscbib, v128b, v128b, 0>;
732  def VSCBIH : BinaryVRRc<"vscbih", 0xE7F5, int_s390_vscbih, v128h, v128h, 1>;
733  def VSCBIF : BinaryVRRc<"vscbif", 0xE7F5, int_s390_vscbif, v128f, v128f, 2>;
734  def VSCBIG : BinaryVRRc<"vscbig", 0xE7F5, int_s390_vscbig, v128g, v128g, 3>;
735  def VSCBIQ : BinaryVRRc<"vscbiq", 0xE7F5, int_s390_vscbiq, v128q, v128q, 4>;
736
737  // Subtract with borrow indication.
738  def VSBI  : TernaryVRRdGeneric<"vsbi", 0xE7BF>;
739  def VSBIQ : TernaryVRRd<"vsbiq", 0xE7BF, int_s390_vsbiq, v128q, v128q, 4>;
740
741  // Subtract with borrow compute borrow indication.
742  def VSBCBI  : TernaryVRRdGeneric<"vsbcbi", 0xE7BD>;
743  def VSBCBIQ : TernaryVRRd<"vsbcbiq", 0xE7BD, int_s390_vsbcbiq,
744                            v128q, v128q, 4>;
745
746  // Sum across doubleword.
747  def VSUMG  : BinaryVRRcGeneric<"vsumg", 0xE765>;
748  def VSUMGH : BinaryVRRc<"vsumgh", 0xE765, z_vsum, v128g, v128h, 1>;
749  def VSUMGF : BinaryVRRc<"vsumgf", 0xE765, z_vsum, v128g, v128f, 2>;
750
751  // Sum across quadword.
752  def VSUMQ  : BinaryVRRcGeneric<"vsumq", 0xE767>;
753  def VSUMQF : BinaryVRRc<"vsumqf", 0xE767, z_vsum, v128q, v128f, 2>;
754  def VSUMQG : BinaryVRRc<"vsumqg", 0xE767, z_vsum, v128q, v128g, 3>;
755
756  // Sum across word.
757  def VSUM  : BinaryVRRcGeneric<"vsum", 0xE764>;
758  def VSUMB : BinaryVRRc<"vsumb", 0xE764, z_vsum, v128f, v128b, 0>;
759  def VSUMH : BinaryVRRc<"vsumh", 0xE764, z_vsum, v128f, v128h, 1>;
760}
761
762// Instantiate the bitwise ops for type TYPE.
763multiclass BitwiseVectorOps<ValueType type> {
764  let Predicates = [FeatureVector] in {
765    def : Pat<(type (and VR128:$x, VR128:$y)), (VN VR128:$x, VR128:$y)>;
766    def : Pat<(type (and VR128:$x, (z_vnot VR128:$y))),
767              (VNC VR128:$x, VR128:$y)>;
768    def : Pat<(type (or VR128:$x, VR128:$y)), (VO VR128:$x, VR128:$y)>;
769    def : Pat<(type (xor VR128:$x, VR128:$y)), (VX VR128:$x, VR128:$y)>;
770    def : Pat<(type (or (and VR128:$x, VR128:$z),
771                        (and VR128:$y, (z_vnot VR128:$z)))),
772              (VSEL VR128:$x, VR128:$y, VR128:$z)>;
773    def : Pat<(type (z_vnot (or VR128:$x, VR128:$y))),
774              (VNO VR128:$x, VR128:$y)>;
775    def : Pat<(type (z_vnot VR128:$x)), (VNO VR128:$x, VR128:$x)>;
776  }
777  let Predicates = [FeatureVectorEnhancements1] in {
778    def : Pat<(type (z_vnot (xor VR128:$x, VR128:$y))),
779              (VNX VR128:$x, VR128:$y)>;
780    def : Pat<(type (z_vnot (and VR128:$x, VR128:$y))),
781              (VNN VR128:$x, VR128:$y)>;
782    def : Pat<(type (or VR128:$x, (z_vnot VR128:$y))),
783              (VOC VR128:$x, VR128:$y)>;
784  }
785}
786
787defm : BitwiseVectorOps<v16i8>;
788defm : BitwiseVectorOps<v8i16>;
789defm : BitwiseVectorOps<v4i32>;
790defm : BitwiseVectorOps<v2i64>;
791
792// Instantiate additional patterns for absolute-related expressions on
793// type TYPE.  LC is the negate instruction for TYPE and LP is the absolute
794// instruction.
795multiclass IntegerAbsoluteVectorOps<ValueType type, Instruction lc,
796                                    Instruction lp, int shift> {
797  let Predicates = [FeatureVector] in {
798    def : Pat<(type (vselect (type (z_vicmph_zero VR128:$x)),
799                             (z_vneg VR128:$x), VR128:$x)),
800              (lc (lp VR128:$x))>;
801    def : Pat<(type (vselect (type (z_vnot (z_vicmph_zero VR128:$x))),
802                             VR128:$x, (z_vneg VR128:$x))),
803              (lc (lp VR128:$x))>;
804    def : Pat<(type (vselect (type (z_vicmpl_zero VR128:$x)),
805                             VR128:$x, (z_vneg VR128:$x))),
806              (lc (lp VR128:$x))>;
807    def : Pat<(type (vselect (type (z_vnot (z_vicmpl_zero VR128:$x))),
808                             (z_vneg VR128:$x), VR128:$x)),
809              (lc (lp VR128:$x))>;
810    def : Pat<(type (or (and (z_vsra_by_scalar VR128:$x, (i32 shift)),
811                             (z_vneg VR128:$x)),
812                        (and (z_vnot (z_vsra_by_scalar VR128:$x, (i32 shift))),
813                             VR128:$x))),
814              (lp VR128:$x)>;
815    def : Pat<(type (or (and (z_vsra_by_scalar VR128:$x, (i32 shift)),
816                             VR128:$x),
817                        (and (z_vnot (z_vsra_by_scalar VR128:$x, (i32 shift))),
818                             (z_vneg VR128:$x)))),
819              (lc (lp VR128:$x))>;
820  }
821}
822
823defm : IntegerAbsoluteVectorOps<v16i8, VLCB, VLPB, 7>;
824defm : IntegerAbsoluteVectorOps<v8i16, VLCH, VLPH, 15>;
825defm : IntegerAbsoluteVectorOps<v4i32, VLCF, VLPF, 31>;
826defm : IntegerAbsoluteVectorOps<v2i64, VLCG, VLPG, 63>;
827
828// Instantiate minimum- and maximum-related patterns for TYPE.  CMPH is the
829// signed or unsigned "set if greater than" comparison instruction and
830// MIN and MAX are the associated minimum and maximum instructions.
831multiclass IntegerMinMaxVectorOps<ValueType type, SDPatternOperator cmph,
832                                  Instruction min, Instruction max> {
833  let Predicates = [FeatureVector] in {
834    def : Pat<(type (vselect (cmph VR128:$x, VR128:$y), VR128:$x, VR128:$y)),
835              (max VR128:$x, VR128:$y)>;
836    def : Pat<(type (vselect (cmph VR128:$x, VR128:$y), VR128:$y, VR128:$x)),
837              (min VR128:$x, VR128:$y)>;
838    def : Pat<(type (vselect (z_vnot (cmph VR128:$x, VR128:$y)),
839                             VR128:$x, VR128:$y)),
840              (min VR128:$x, VR128:$y)>;
841    def : Pat<(type (vselect (z_vnot (cmph VR128:$x, VR128:$y)),
842                             VR128:$y, VR128:$x)),
843              (max VR128:$x, VR128:$y)>;
844  }
845}
846
847// Signed min/max.
848defm : IntegerMinMaxVectorOps<v16i8, z_vicmph, VMNB, VMXB>;
849defm : IntegerMinMaxVectorOps<v8i16, z_vicmph, VMNH, VMXH>;
850defm : IntegerMinMaxVectorOps<v4i32, z_vicmph, VMNF, VMXF>;
851defm : IntegerMinMaxVectorOps<v2i64, z_vicmph, VMNG, VMXG>;
852
853// Unsigned min/max.
854defm : IntegerMinMaxVectorOps<v16i8, z_vicmphl, VMNLB, VMXLB>;
855defm : IntegerMinMaxVectorOps<v8i16, z_vicmphl, VMNLH, VMXLH>;
856defm : IntegerMinMaxVectorOps<v4i32, z_vicmphl, VMNLF, VMXLF>;
857defm : IntegerMinMaxVectorOps<v2i64, z_vicmphl, VMNLG, VMXLG>;
858
859//===----------------------------------------------------------------------===//
860// Integer comparison
861//===----------------------------------------------------------------------===//
862
863let Predicates = [FeatureVector] in {
864  // Element compare.
865  let Defs = [CC] in {
866    def VEC  : CompareVRRaGeneric<"vec", 0xE7DB>;
867    def VECB : CompareVRRa<"vecb", 0xE7DB, null_frag, v128b, 0>;
868    def VECH : CompareVRRa<"vech", 0xE7DB, null_frag, v128h, 1>;
869    def VECF : CompareVRRa<"vecf", 0xE7DB, null_frag, v128f, 2>;
870    def VECG : CompareVRRa<"vecg", 0xE7DB, null_frag, v128g, 3>;
871  }
872
873  // Element compare logical.
874  let Defs = [CC] in {
875    def VECL  : CompareVRRaGeneric<"vecl", 0xE7D9>;
876    def VECLB : CompareVRRa<"veclb", 0xE7D9, null_frag, v128b, 0>;
877    def VECLH : CompareVRRa<"veclh", 0xE7D9, null_frag, v128h, 1>;
878    def VECLF : CompareVRRa<"veclf", 0xE7D9, null_frag, v128f, 2>;
879    def VECLG : CompareVRRa<"veclg", 0xE7D9, null_frag, v128g, 3>;
880  }
881
882  // Compare equal.
883  def  VCEQ  : BinaryVRRbSPairGeneric<"vceq", 0xE7F8>;
884  defm VCEQB : BinaryVRRbSPair<"vceqb", 0xE7F8, z_vicmpe, z_vicmpes,
885                               v128b, v128b, 0>;
886  defm VCEQH : BinaryVRRbSPair<"vceqh", 0xE7F8, z_vicmpe, z_vicmpes,
887                               v128h, v128h, 1>;
888  defm VCEQF : BinaryVRRbSPair<"vceqf", 0xE7F8, z_vicmpe, z_vicmpes,
889                               v128f, v128f, 2>;
890  defm VCEQG : BinaryVRRbSPair<"vceqg", 0xE7F8, z_vicmpe, z_vicmpes,
891                               v128g, v128g, 3>;
892
893  // Compare high.
894  def  VCH  : BinaryVRRbSPairGeneric<"vch", 0xE7FB>;
895  defm VCHB : BinaryVRRbSPair<"vchb", 0xE7FB, z_vicmph, z_vicmphs,
896                              v128b, v128b, 0>;
897  defm VCHH : BinaryVRRbSPair<"vchh", 0xE7FB, z_vicmph, z_vicmphs,
898                              v128h, v128h, 1>;
899  defm VCHF : BinaryVRRbSPair<"vchf", 0xE7FB, z_vicmph, z_vicmphs,
900                              v128f, v128f, 2>;
901  defm VCHG : BinaryVRRbSPair<"vchg", 0xE7FB, z_vicmph, z_vicmphs,
902                              v128g, v128g, 3>;
903
904  // Compare high logical.
905  def  VCHL  : BinaryVRRbSPairGeneric<"vchl", 0xE7F9>;
906  defm VCHLB : BinaryVRRbSPair<"vchlb", 0xE7F9, z_vicmphl, z_vicmphls,
907                               v128b, v128b, 0>;
908  defm VCHLH : BinaryVRRbSPair<"vchlh", 0xE7F9, z_vicmphl, z_vicmphls,
909                               v128h, v128h, 1>;
910  defm VCHLF : BinaryVRRbSPair<"vchlf", 0xE7F9, z_vicmphl, z_vicmphls,
911                               v128f, v128f, 2>;
912  defm VCHLG : BinaryVRRbSPair<"vchlg", 0xE7F9, z_vicmphl, z_vicmphls,
913                               v128g, v128g, 3>;
914
915  // Test under mask.
916  let Defs = [CC] in
917    def VTM : CompareVRRa<"vtm", 0xE7D8, z_vtm, v128b, 0>;
918}
919
920//===----------------------------------------------------------------------===//
921// Floating-point arithmetic
922//===----------------------------------------------------------------------===//
923
924// See comments in SystemZInstrFP.td for the suppression flags and
925// rounding modes.
926multiclass VectorRounding<Instruction insn, TypedReg tr> {
927  def : FPConversion<insn, any_frint,      tr, tr, 0, 0>;
928  def : FPConversion<insn, any_fnearbyint, tr, tr, 4, 0>;
929  def : FPConversion<insn, any_ffloor,     tr, tr, 4, 7>;
930  def : FPConversion<insn, any_fceil,      tr, tr, 4, 6>;
931  def : FPConversion<insn, any_ftrunc,     tr, tr, 4, 5>;
932  def : FPConversion<insn, any_fround,     tr, tr, 4, 1>;
933}
934
935let Predicates = [FeatureVector] in {
936  // Add.
937  let Uses = [FPC], mayRaiseFPException = 1 in {
938    def VFA   : BinaryVRRcFloatGeneric<"vfa", 0xE7E3>;
939    def VFADB : BinaryVRRc<"vfadb", 0xE7E3, any_fadd, v128db, v128db, 3, 0>;
940    def WFADB : BinaryVRRc<"wfadb", 0xE7E3, any_fadd, v64db, v64db, 3, 8>;
941    let Predicates = [FeatureVectorEnhancements1] in {
942      def VFASB : BinaryVRRc<"vfasb", 0xE7E3, any_fadd, v128sb, v128sb, 2, 0>;
943      def WFASB : BinaryVRRc<"wfasb", 0xE7E3, any_fadd, v32sb, v32sb, 2, 8>;
944      def WFAXB : BinaryVRRc<"wfaxb", 0xE7E3, any_fadd, v128xb, v128xb, 4, 8>;
945    }
946  }
947
948  // Convert from fixed 64-bit.
949  let Uses = [FPC], mayRaiseFPException = 1 in {
950    def VCDG  : TernaryVRRaFloatGeneric<"vcdg", 0xE7C3>;
951    def VCDGB : TernaryVRRa<"vcdgb", 0xE7C3, null_frag, v128db, v128g, 3, 0>;
952    def WCDGB : TernaryVRRa<"wcdgb", 0xE7C3, null_frag, v64db, v64g, 3, 8>;
953  }
954  def : FPConversion<VCDGB, sint_to_fp, v128db, v128g, 0, 0>;
955
956  // Convert from logical 64-bit.
957  let Uses = [FPC], mayRaiseFPException = 1 in {
958    def VCDLG  : TernaryVRRaFloatGeneric<"vcdlg", 0xE7C1>;
959    def VCDLGB : TernaryVRRa<"vcdlgb", 0xE7C1, null_frag, v128db, v128g, 3, 0>;
960    def WCDLGB : TernaryVRRa<"wcdlgb", 0xE7C1, null_frag, v64db, v64g, 3, 8>;
961  }
962  def : FPConversion<VCDLGB, uint_to_fp, v128db, v128g, 0, 0>;
963
964  // Convert to fixed 64-bit.
965  let Uses = [FPC], mayRaiseFPException = 1 in {
966    def VCGD  : TernaryVRRaFloatGeneric<"vcgd", 0xE7C2>;
967    def VCGDB : TernaryVRRa<"vcgdb", 0xE7C2, null_frag, v128g, v128db, 3, 0>;
968    def WCGDB : TernaryVRRa<"wcgdb", 0xE7C2, null_frag, v64g, v64db, 3, 8>;
969  }
970  // Rounding mode should agree with SystemZInstrFP.td.
971  def : FPConversion<VCGDB, fp_to_sint, v128g, v128db, 0, 5>;
972
973  // Convert to logical 64-bit.
974  let Uses = [FPC], mayRaiseFPException = 1 in {
975    def VCLGD  : TernaryVRRaFloatGeneric<"vclgd", 0xE7C0>;
976    def VCLGDB : TernaryVRRa<"vclgdb", 0xE7C0, null_frag, v128g, v128db, 3, 0>;
977    def WCLGDB : TernaryVRRa<"wclgdb", 0xE7C0, null_frag, v64g, v64db, 3, 8>;
978  }
979  // Rounding mode should agree with SystemZInstrFP.td.
980  def : FPConversion<VCLGDB, fp_to_uint, v128g, v128db, 0, 5>;
981
982  // Divide.
983  let Uses = [FPC], mayRaiseFPException = 1 in {
984    def VFD   : BinaryVRRcFloatGeneric<"vfd", 0xE7E5>;
985    def VFDDB : BinaryVRRc<"vfddb", 0xE7E5, any_fdiv, v128db, v128db, 3, 0>;
986    def WFDDB : BinaryVRRc<"wfddb", 0xE7E5, any_fdiv, v64db, v64db, 3, 8>;
987    let Predicates = [FeatureVectorEnhancements1] in {
988      def VFDSB : BinaryVRRc<"vfdsb", 0xE7E5, any_fdiv, v128sb, v128sb, 2, 0>;
989      def WFDSB : BinaryVRRc<"wfdsb", 0xE7E5, any_fdiv, v32sb, v32sb, 2, 8>;
990      def WFDXB : BinaryVRRc<"wfdxb", 0xE7E5, any_fdiv, v128xb, v128xb, 4, 8>;
991    }
992  }
993
994  // Load FP integer.
995  let Uses = [FPC], mayRaiseFPException = 1 in {
996    def VFI   : TernaryVRRaFloatGeneric<"vfi", 0xE7C7>;
997    def VFIDB : TernaryVRRa<"vfidb", 0xE7C7, int_s390_vfidb, v128db, v128db, 3, 0>;
998    def WFIDB : TernaryVRRa<"wfidb", 0xE7C7, null_frag, v64db, v64db, 3, 8>;
999  }
1000  defm : VectorRounding<VFIDB, v128db>;
1001  defm : VectorRounding<WFIDB, v64db>;
1002  let Predicates = [FeatureVectorEnhancements1] in {
1003    let Uses = [FPC], mayRaiseFPException = 1 in {
1004      def VFISB : TernaryVRRa<"vfisb", 0xE7C7, int_s390_vfisb, v128sb, v128sb, 2, 0>;
1005      def WFISB : TernaryVRRa<"wfisb", 0xE7C7, null_frag, v32sb, v32sb, 2, 8>;
1006      def WFIXB : TernaryVRRa<"wfixb", 0xE7C7, null_frag, v128xb, v128xb, 4, 8>;
1007    }
1008    defm : VectorRounding<VFISB, v128sb>;
1009    defm : VectorRounding<WFISB, v32sb>;
1010    defm : VectorRounding<WFIXB, v128xb>;
1011  }
1012
1013  // Load lengthened.
1014  let Uses = [FPC], mayRaiseFPException = 1 in {
1015    def VLDE  : UnaryVRRaFloatGeneric<"vlde", 0xE7C4>;
1016    def VLDEB : UnaryVRRa<"vldeb", 0xE7C4, z_vextend, v128db, v128sb, 2, 0>;
1017    def WLDEB : UnaryVRRa<"wldeb", 0xE7C4, any_fpextend, v64db, v32sb, 2, 8>;
1018  }
1019  let Predicates = [FeatureVectorEnhancements1] in {
1020    let Uses = [FPC], mayRaiseFPException = 1 in {
1021      let isAsmParserOnly = 1 in {
1022        def VFLL  : UnaryVRRaFloatGeneric<"vfll", 0xE7C4>;
1023        def VFLLS : UnaryVRRa<"vflls", 0xE7C4, null_frag, v128db, v128sb, 2, 0>;
1024        def WFLLS : UnaryVRRa<"wflls", 0xE7C4, null_frag, v64db, v32sb, 2, 8>;
1025      }
1026      def WFLLD : UnaryVRRa<"wflld", 0xE7C4, any_fpextend, v128xb, v64db, 3, 8>;
1027    }
1028    def : Pat<(f128 (any_fpextend (f32 VR32:$src))),
1029              (WFLLD (WLDEB VR32:$src))>;
1030  }
1031
1032  // Load rounded.
1033  let Uses = [FPC], mayRaiseFPException = 1 in {
1034    def VLED  : TernaryVRRaFloatGeneric<"vled", 0xE7C5>;
1035    def VLEDB : TernaryVRRa<"vledb", 0xE7C5, null_frag, v128sb, v128db, 3, 0>;
1036    def WLEDB : TernaryVRRa<"wledb", 0xE7C5, null_frag, v32sb, v64db, 3, 8>;
1037  }
1038  def : Pat<(v4f32 (z_vround (v2f64 VR128:$src))), (VLEDB VR128:$src, 0, 0)>;
1039  def : FPConversion<WLEDB, any_fpround, v32sb, v64db, 0, 0>;
1040  let Predicates = [FeatureVectorEnhancements1] in {
1041    let Uses = [FPC], mayRaiseFPException = 1 in {
1042      let isAsmParserOnly = 1 in {
1043        def VFLR  : TernaryVRRaFloatGeneric<"vflr", 0xE7C5>;
1044        def VFLRD : TernaryVRRa<"vflrd", 0xE7C5, null_frag, v128sb, v128db, 3, 0>;
1045        def WFLRD : TernaryVRRa<"wflrd", 0xE7C5, null_frag, v32sb, v64db, 3, 8>;
1046      }
1047      def WFLRX : TernaryVRRa<"wflrx", 0xE7C5, null_frag, v64db, v128xb, 4, 8>;
1048    }
1049    def : FPConversion<WFLRX, any_fpround, v64db, v128xb, 0, 0>;
1050    def : Pat<(f32 (any_fpround (f128 VR128:$src))),
1051              (WLEDB (WFLRX VR128:$src, 0, 3), 0, 0)>;
1052  }
1053
1054  // Maximum.
1055  multiclass VectorMax<Instruction insn, TypedReg tr> {
1056    def : FPMinMax<insn, any_fmaxnum, tr, 4>;
1057    def : FPMinMax<insn, fmaximum, tr, 1>;
1058  }
1059  let Predicates = [FeatureVectorEnhancements1] in {
1060    let Uses = [FPC], mayRaiseFPException = 1 in {
1061      def VFMAX   : TernaryVRRcFloatGeneric<"vfmax", 0xE7EF>;
1062      def VFMAXDB : TernaryVRRcFloat<"vfmaxdb", 0xE7EF, int_s390_vfmaxdb,
1063                                     v128db, v128db, 3, 0>;
1064      def WFMAXDB : TernaryVRRcFloat<"wfmaxdb", 0xE7EF, null_frag,
1065                                     v64db, v64db, 3, 8>;
1066      def VFMAXSB : TernaryVRRcFloat<"vfmaxsb", 0xE7EF, int_s390_vfmaxsb,
1067                                     v128sb, v128sb, 2, 0>;
1068      def WFMAXSB : TernaryVRRcFloat<"wfmaxsb", 0xE7EF, null_frag,
1069                                     v32sb, v32sb, 2, 8>;
1070      def WFMAXXB : TernaryVRRcFloat<"wfmaxxb", 0xE7EF, null_frag,
1071                                     v128xb, v128xb, 4, 8>;
1072    }
1073    defm : VectorMax<VFMAXDB, v128db>;
1074    defm : VectorMax<WFMAXDB, v64db>;
1075    defm : VectorMax<VFMAXSB, v128sb>;
1076    defm : VectorMax<WFMAXSB, v32sb>;
1077    defm : VectorMax<WFMAXXB, v128xb>;
1078  }
1079
1080  // Minimum.
1081  multiclass VectorMin<Instruction insn, TypedReg tr> {
1082    def : FPMinMax<insn, any_fminnum, tr, 4>;
1083    def : FPMinMax<insn, fminimum, tr, 1>;
1084  }
1085  let Predicates = [FeatureVectorEnhancements1] in {
1086    let Uses = [FPC], mayRaiseFPException = 1 in {
1087      def VFMIN   : TernaryVRRcFloatGeneric<"vfmin", 0xE7EE>;
1088      def VFMINDB : TernaryVRRcFloat<"vfmindb", 0xE7EE, int_s390_vfmindb,
1089                                     v128db, v128db, 3, 0>;
1090      def WFMINDB : TernaryVRRcFloat<"wfmindb", 0xE7EE, null_frag,
1091                                     v64db, v64db, 3, 8>;
1092      def VFMINSB : TernaryVRRcFloat<"vfminsb", 0xE7EE, int_s390_vfminsb,
1093                                     v128sb, v128sb, 2, 0>;
1094      def WFMINSB : TernaryVRRcFloat<"wfminsb", 0xE7EE, null_frag,
1095                                     v32sb, v32sb, 2, 8>;
1096      def WFMINXB : TernaryVRRcFloat<"wfminxb", 0xE7EE, null_frag,
1097                                     v128xb, v128xb, 4, 8>;
1098    }
1099    defm : VectorMin<VFMINDB, v128db>;
1100    defm : VectorMin<WFMINDB, v64db>;
1101    defm : VectorMin<VFMINSB, v128sb>;
1102    defm : VectorMin<WFMINSB, v32sb>;
1103    defm : VectorMin<WFMINXB, v128xb>;
1104  }
1105
1106  // Multiply.
1107  let Uses = [FPC], mayRaiseFPException = 1 in {
1108    def VFM   : BinaryVRRcFloatGeneric<"vfm", 0xE7E7>;
1109    def VFMDB : BinaryVRRc<"vfmdb", 0xE7E7, any_fmul, v128db, v128db, 3, 0>;
1110    def WFMDB : BinaryVRRc<"wfmdb", 0xE7E7, any_fmul, v64db, v64db, 3, 8>;
1111    let Predicates = [FeatureVectorEnhancements1] in {
1112      def VFMSB : BinaryVRRc<"vfmsb", 0xE7E7, any_fmul, v128sb, v128sb, 2, 0>;
1113      def WFMSB : BinaryVRRc<"wfmsb", 0xE7E7, any_fmul, v32sb, v32sb, 2, 8>;
1114      def WFMXB : BinaryVRRc<"wfmxb", 0xE7E7, any_fmul, v128xb, v128xb, 4, 8>;
1115    }
1116  }
1117
1118  // Multiply and add.
1119  let Uses = [FPC], mayRaiseFPException = 1 in {
1120    def VFMA   : TernaryVRReFloatGeneric<"vfma", 0xE78F>;
1121    def VFMADB : TernaryVRRe<"vfmadb", 0xE78F, any_fma, v128db, v128db, 0, 3>;
1122    def WFMADB : TernaryVRRe<"wfmadb", 0xE78F, any_fma, v64db, v64db, 8, 3>;
1123    let Predicates = [FeatureVectorEnhancements1] in {
1124      def VFMASB : TernaryVRRe<"vfmasb", 0xE78F, any_fma, v128sb, v128sb, 0, 2>;
1125      def WFMASB : TernaryVRRe<"wfmasb", 0xE78F, any_fma, v32sb, v32sb, 8, 2>;
1126      def WFMAXB : TernaryVRRe<"wfmaxb", 0xE78F, any_fma, v128xb, v128xb, 8, 4>;
1127    }
1128  }
1129
1130  // Multiply and subtract.
1131  let Uses = [FPC], mayRaiseFPException = 1 in {
1132    def VFMS   : TernaryVRReFloatGeneric<"vfms", 0xE78E>;
1133    def VFMSDB : TernaryVRRe<"vfmsdb", 0xE78E, any_fms, v128db, v128db, 0, 3>;
1134    def WFMSDB : TernaryVRRe<"wfmsdb", 0xE78E, any_fms, v64db, v64db, 8, 3>;
1135    let Predicates = [FeatureVectorEnhancements1] in {
1136      def VFMSSB : TernaryVRRe<"vfmssb", 0xE78E, any_fms, v128sb, v128sb, 0, 2>;
1137      def WFMSSB : TernaryVRRe<"wfmssb", 0xE78E, any_fms, v32sb, v32sb, 8, 2>;
1138      def WFMSXB : TernaryVRRe<"wfmsxb", 0xE78E, any_fms, v128xb, v128xb, 8, 4>;
1139    }
1140  }
1141
1142  // Negative multiply and add.
1143  let Uses = [FPC], mayRaiseFPException = 1,
1144      Predicates = [FeatureVectorEnhancements1] in {
1145    def VFNMA   : TernaryVRReFloatGeneric<"vfnma", 0xE79F>;
1146    def VFNMADB : TernaryVRRe<"vfnmadb", 0xE79F, any_fnma, v128db, v128db, 0, 3>;
1147    def WFNMADB : TernaryVRRe<"wfnmadb", 0xE79F, any_fnma, v64db, v64db, 8, 3>;
1148    def VFNMASB : TernaryVRRe<"vfnmasb", 0xE79F, any_fnma, v128sb, v128sb, 0, 2>;
1149    def WFNMASB : TernaryVRRe<"wfnmasb", 0xE79F, any_fnma, v32sb, v32sb, 8, 2>;
1150    def WFNMAXB : TernaryVRRe<"wfnmaxb", 0xE79F, any_fnma, v128xb, v128xb, 8, 4>;
1151  }
1152
1153  // Negative multiply and subtract.
1154  let Uses = [FPC], mayRaiseFPException = 1,
1155      Predicates = [FeatureVectorEnhancements1] in {
1156    def VFNMS   : TernaryVRReFloatGeneric<"vfnms", 0xE79E>;
1157    def VFNMSDB : TernaryVRRe<"vfnmsdb", 0xE79E, any_fnms, v128db, v128db, 0, 3>;
1158    def WFNMSDB : TernaryVRRe<"wfnmsdb", 0xE79E, any_fnms, v64db, v64db, 8, 3>;
1159    def VFNMSSB : TernaryVRRe<"vfnmssb", 0xE79E, any_fnms, v128sb, v128sb, 0, 2>;
1160    def WFNMSSB : TernaryVRRe<"wfnmssb", 0xE79E, any_fnms, v32sb, v32sb, 8, 2>;
1161    def WFNMSXB : TernaryVRRe<"wfnmsxb", 0xE79E, any_fnms, v128xb, v128xb, 8, 4>;
1162  }
1163
1164  // Perform sign operation.
1165  def VFPSO   : BinaryVRRaFloatGeneric<"vfpso", 0xE7CC>;
1166  def VFPSODB : BinaryVRRa<"vfpsodb", 0xE7CC, null_frag, v128db, v128db, 3, 0>;
1167  def WFPSODB : BinaryVRRa<"wfpsodb", 0xE7CC, null_frag, v64db, v64db, 3, 8>;
1168  let Predicates = [FeatureVectorEnhancements1] in {
1169    def VFPSOSB : BinaryVRRa<"vfpsosb", 0xE7CC, null_frag, v128sb, v128sb, 2, 0>;
1170    def WFPSOSB : BinaryVRRa<"wfpsosb", 0xE7CC, null_frag, v32sb, v32sb, 2, 8>;
1171    def WFPSOXB : BinaryVRRa<"wfpsoxb", 0xE7CC, null_frag, v128xb, v128xb, 4, 8>;
1172  }
1173
1174  // Load complement.
1175  def VFLCDB : UnaryVRRa<"vflcdb", 0xE7CC, fneg, v128db, v128db, 3, 0, 0>;
1176  def WFLCDB : UnaryVRRa<"wflcdb", 0xE7CC, fneg, v64db, v64db, 3, 8, 0>;
1177  let Predicates = [FeatureVectorEnhancements1] in {
1178    def VFLCSB : UnaryVRRa<"vflcsb", 0xE7CC, fneg, v128sb, v128sb, 2, 0, 0>;
1179    def WFLCSB : UnaryVRRa<"wflcsb", 0xE7CC, fneg, v32sb, v32sb, 2, 8, 0>;
1180    def WFLCXB : UnaryVRRa<"wflcxb", 0xE7CC, fneg, v128xb, v128xb, 4, 8, 0>;
1181  }
1182
1183  // Load negative.
1184  def VFLNDB : UnaryVRRa<"vflndb", 0xE7CC, fnabs, v128db, v128db, 3, 0, 1>;
1185  def WFLNDB : UnaryVRRa<"wflndb", 0xE7CC, fnabs, v64db, v64db, 3, 8, 1>;
1186  let Predicates = [FeatureVectorEnhancements1] in {
1187    def VFLNSB : UnaryVRRa<"vflnsb", 0xE7CC, fnabs, v128sb, v128sb, 2, 0, 1>;
1188    def WFLNSB : UnaryVRRa<"wflnsb", 0xE7CC, fnabs, v32sb, v32sb, 2, 8, 1>;
1189    def WFLNXB : UnaryVRRa<"wflnxb", 0xE7CC, fnabs, v128xb, v128xb, 4, 8, 1>;
1190  }
1191
1192  // Load positive.
1193  def VFLPDB : UnaryVRRa<"vflpdb", 0xE7CC, fabs, v128db, v128db, 3, 0, 2>;
1194  def WFLPDB : UnaryVRRa<"wflpdb", 0xE7CC, fabs, v64db, v64db, 3, 8, 2>;
1195  let Predicates = [FeatureVectorEnhancements1] in {
1196    def VFLPSB : UnaryVRRa<"vflpsb", 0xE7CC, fabs, v128sb, v128sb, 2, 0, 2>;
1197    def WFLPSB : UnaryVRRa<"wflpsb", 0xE7CC, fabs, v32sb, v32sb, 2, 8, 2>;
1198    def WFLPXB : UnaryVRRa<"wflpxb", 0xE7CC, fabs, v128xb, v128xb, 4, 8, 2>;
1199  }
1200
1201  // Square root.
1202  let Uses = [FPC], mayRaiseFPException = 1 in {
1203    def VFSQ   : UnaryVRRaFloatGeneric<"vfsq", 0xE7CE>;
1204    def VFSQDB : UnaryVRRa<"vfsqdb", 0xE7CE, any_fsqrt, v128db, v128db, 3, 0>;
1205    def WFSQDB : UnaryVRRa<"wfsqdb", 0xE7CE, any_fsqrt, v64db, v64db, 3, 8>;
1206    let Predicates = [FeatureVectorEnhancements1] in {
1207      def VFSQSB : UnaryVRRa<"vfsqsb", 0xE7CE, any_fsqrt, v128sb, v128sb, 2, 0>;
1208      def WFSQSB : UnaryVRRa<"wfsqsb", 0xE7CE, any_fsqrt, v32sb, v32sb, 2, 8>;
1209      def WFSQXB : UnaryVRRa<"wfsqxb", 0xE7CE, any_fsqrt, v128xb, v128xb, 4, 8>;
1210    }
1211  }
1212
1213  // Subtract.
1214  let Uses = [FPC], mayRaiseFPException = 1 in {
1215    def VFS   : BinaryVRRcFloatGeneric<"vfs", 0xE7E2>;
1216    def VFSDB : BinaryVRRc<"vfsdb", 0xE7E2, any_fsub, v128db, v128db, 3, 0>;
1217    def WFSDB : BinaryVRRc<"wfsdb", 0xE7E2, any_fsub, v64db, v64db, 3, 8>;
1218    let Predicates = [FeatureVectorEnhancements1] in {
1219      def VFSSB : BinaryVRRc<"vfssb", 0xE7E2, any_fsub, v128sb, v128sb, 2, 0>;
1220      def WFSSB : BinaryVRRc<"wfssb", 0xE7E2, any_fsub, v32sb, v32sb, 2, 8>;
1221      def WFSXB : BinaryVRRc<"wfsxb", 0xE7E2, any_fsub, v128xb, v128xb, 4, 8>;
1222    }
1223  }
1224
1225  // Test data class immediate.
1226  let Defs = [CC] in {
1227    def VFTCI   : BinaryVRIeFloatGeneric<"vftci", 0xE74A>;
1228    def VFTCIDB : BinaryVRIe<"vftcidb", 0xE74A, z_vftci, v128g, v128db, 3, 0>;
1229    def WFTCIDB : BinaryVRIe<"wftcidb", 0xE74A, null_frag, v64g, v64db, 3, 8>;
1230    let Predicates = [FeatureVectorEnhancements1] in {
1231      def VFTCISB : BinaryVRIe<"vftcisb", 0xE74A, z_vftci, v128f, v128sb, 2, 0>;
1232      def WFTCISB : BinaryVRIe<"wftcisb", 0xE74A, null_frag, v32f, v32sb, 2, 8>;
1233      def WFTCIXB : BinaryVRIe<"wftcixb", 0xE74A, null_frag, v128q, v128xb, 4, 8>;
1234    }
1235  }
1236}
1237
1238//===----------------------------------------------------------------------===//
1239// Floating-point comparison
1240//===----------------------------------------------------------------------===//
1241
1242let Predicates = [FeatureVector] in {
1243  // Compare scalar.
1244  let Uses = [FPC], mayRaiseFPException = 1, Defs = [CC] in {
1245    def WFC   : CompareVRRaFloatGeneric<"wfc", 0xE7CB>;
1246    def WFCDB : CompareVRRa<"wfcdb", 0xE7CB, z_fcmp, v64db, 3>;
1247    let Predicates = [FeatureVectorEnhancements1] in {
1248      def WFCSB : CompareVRRa<"wfcsb", 0xE7CB, z_fcmp, v32sb, 2>;
1249      def WFCXB : CompareVRRa<"wfcxb", 0xE7CB, z_fcmp, v128xb, 4>;
1250    }
1251  }
1252
1253  // Compare and signal scalar.
1254  let Uses = [FPC], mayRaiseFPException = 1, Defs = [CC] in {
1255    def WFK   : CompareVRRaFloatGeneric<"wfk", 0xE7CA>;
1256    def WFKDB : CompareVRRa<"wfkdb", 0xE7CA, null_frag, v64db, 3>;
1257    let Predicates = [FeatureVectorEnhancements1] in {
1258      def WFKSB : CompareVRRa<"wfksb", 0xE7CA, null_frag, v32sb, 2>;
1259      def WFKXB : CompareVRRa<"wfkxb", 0xE7CA, null_frag, v128xb, 4>;
1260    }
1261  }
1262
1263  // Compare equal.
1264  let Uses = [FPC], mayRaiseFPException = 1 in {
1265    def  VFCE   : BinaryVRRcSPairFloatGeneric<"vfce", 0xE7E8>;
1266    defm VFCEDB : BinaryVRRcSPair<"vfcedb", 0xE7E8, z_vfcmpe, z_vfcmpes,
1267                                  v128g, v128db, 3, 0>;
1268    defm WFCEDB : BinaryVRRcSPair<"wfcedb", 0xE7E8, null_frag, null_frag,
1269                                  v64g, v64db, 3, 8>;
1270    let Predicates = [FeatureVectorEnhancements1] in {
1271      defm VFCESB : BinaryVRRcSPair<"vfcesb", 0xE7E8, z_vfcmpe, z_vfcmpes,
1272                                    v128f, v128sb, 2, 0>;
1273      defm WFCESB : BinaryVRRcSPair<"wfcesb", 0xE7E8, null_frag, null_frag,
1274                                    v32f, v32sb, 2, 8>;
1275      defm WFCEXB : BinaryVRRcSPair<"wfcexb", 0xE7E8, null_frag, null_frag,
1276                                    v128q, v128xb, 4, 8>;
1277    }
1278  }
1279
1280  // Compare and signal equal.
1281  let Uses = [FPC], mayRaiseFPException = 1,
1282      Predicates = [FeatureVectorEnhancements1] in {
1283    defm VFKEDB : BinaryVRRcSPair<"vfkedb", 0xE7E8, null_frag, null_frag,
1284                                  v128g, v128db, 3, 4>;
1285    defm WFKEDB : BinaryVRRcSPair<"wfkedb", 0xE7E8, null_frag, null_frag,
1286                                  v64g, v64db, 3, 12>;
1287    defm VFKESB : BinaryVRRcSPair<"vfkesb", 0xE7E8, null_frag, null_frag,
1288                                  v128f, v128sb, 2, 4>;
1289    defm WFKESB : BinaryVRRcSPair<"wfkesb", 0xE7E8, null_frag, null_frag,
1290                                  v32f, v32sb, 2, 12>;
1291    defm WFKEXB : BinaryVRRcSPair<"wfkexb", 0xE7E8, null_frag, null_frag,
1292                                  v128q, v128xb, 4, 12>;
1293  }
1294
1295  // Compare high.
1296  let Uses = [FPC], mayRaiseFPException = 1 in {
1297    def  VFCH   : BinaryVRRcSPairFloatGeneric<"vfch", 0xE7EB>;
1298    defm VFCHDB : BinaryVRRcSPair<"vfchdb", 0xE7EB, z_vfcmph, z_vfcmphs,
1299                                  v128g, v128db, 3, 0>;
1300    defm WFCHDB : BinaryVRRcSPair<"wfchdb", 0xE7EB, null_frag, null_frag,
1301                                  v64g, v64db, 3, 8>;
1302    let Predicates = [FeatureVectorEnhancements1] in {
1303      defm VFCHSB : BinaryVRRcSPair<"vfchsb", 0xE7EB, z_vfcmph, z_vfcmphs,
1304                                    v128f, v128sb, 2, 0>;
1305      defm WFCHSB : BinaryVRRcSPair<"wfchsb", 0xE7EB, null_frag, null_frag,
1306                                    v32f, v32sb, 2, 8>;
1307      defm WFCHXB : BinaryVRRcSPair<"wfchxb", 0xE7EB, null_frag, null_frag,
1308                                    v128q, v128xb, 4, 8>;
1309    }
1310  }
1311
1312  // Compare and signal high.
1313  let Uses = [FPC], mayRaiseFPException = 1,
1314      Predicates = [FeatureVectorEnhancements1] in {
1315    defm VFKHDB : BinaryVRRcSPair<"vfkhdb", 0xE7EB, null_frag, null_frag,
1316                                  v128g, v128db, 3, 4>;
1317    defm WFKHDB : BinaryVRRcSPair<"wfkhdb", 0xE7EB, null_frag, null_frag,
1318                                  v64g, v64db, 3, 12>;
1319    defm VFKHSB : BinaryVRRcSPair<"vfkhsb", 0xE7EB, null_frag, null_frag,
1320                                  v128f, v128sb, 2, 4>;
1321    defm WFKHSB : BinaryVRRcSPair<"wfkhsb", 0xE7EB, null_frag, null_frag,
1322                                  v32f, v32sb, 2, 12>;
1323    defm WFKHXB : BinaryVRRcSPair<"wfkhxb", 0xE7EB, null_frag, null_frag,
1324                                  v128q, v128xb, 4, 12>;
1325  }
1326
1327  // Compare high or equal.
1328  let Uses = [FPC], mayRaiseFPException = 1 in {
1329    def  VFCHE   : BinaryVRRcSPairFloatGeneric<"vfche", 0xE7EA>;
1330    defm VFCHEDB : BinaryVRRcSPair<"vfchedb", 0xE7EA, z_vfcmphe, z_vfcmphes,
1331                                   v128g, v128db, 3, 0>;
1332    defm WFCHEDB : BinaryVRRcSPair<"wfchedb", 0xE7EA, null_frag, null_frag,
1333                                   v64g, v64db, 3, 8>;
1334    let Predicates = [FeatureVectorEnhancements1] in {
1335      defm VFCHESB : BinaryVRRcSPair<"vfchesb", 0xE7EA, z_vfcmphe, z_vfcmphes,
1336                                     v128f, v128sb, 2, 0>;
1337      defm WFCHESB : BinaryVRRcSPair<"wfchesb", 0xE7EA, null_frag, null_frag,
1338                                     v32f, v32sb, 2, 8>;
1339      defm WFCHEXB : BinaryVRRcSPair<"wfchexb", 0xE7EA, null_frag, null_frag,
1340                                     v128q, v128xb, 4, 8>;
1341    }
1342  }
1343
1344  // Compare and signal high or equal.
1345  let Uses = [FPC], mayRaiseFPException = 1,
1346      Predicates = [FeatureVectorEnhancements1] in {
1347    defm VFKHEDB : BinaryVRRcSPair<"vfkhedb", 0xE7EA, null_frag, null_frag,
1348                                   v128g, v128db, 3, 4>;
1349    defm WFKHEDB : BinaryVRRcSPair<"wfkhedb", 0xE7EA, null_frag, null_frag,
1350                                   v64g, v64db, 3, 12>;
1351    defm VFKHESB : BinaryVRRcSPair<"vfkhesb", 0xE7EA, null_frag, null_frag,
1352                                   v128f, v128sb, 2, 4>;
1353    defm WFKHESB : BinaryVRRcSPair<"wfkhesb", 0xE7EA, null_frag, null_frag,
1354                                   v32f, v32sb, 2, 12>;
1355    defm WFKHEXB : BinaryVRRcSPair<"wfkhexb", 0xE7EA, null_frag, null_frag,
1356                                   v128q, v128xb, 4, 12>;
1357  }
1358}
1359
1360//===----------------------------------------------------------------------===//
1361// Conversions
1362//===----------------------------------------------------------------------===//
1363
1364def : Pat<(v16i8 (bitconvert (v8i16 VR128:$src))), (v16i8 VR128:$src)>;
1365def : Pat<(v16i8 (bitconvert (v4i32 VR128:$src))), (v16i8 VR128:$src)>;
1366def : Pat<(v16i8 (bitconvert (v2i64 VR128:$src))), (v16i8 VR128:$src)>;
1367def : Pat<(v16i8 (bitconvert (v4f32 VR128:$src))), (v16i8 VR128:$src)>;
1368def : Pat<(v16i8 (bitconvert (v2f64 VR128:$src))), (v16i8 VR128:$src)>;
1369def : Pat<(v16i8 (bitconvert (f128  VR128:$src))), (v16i8 VR128:$src)>;
1370
1371def : Pat<(v8i16 (bitconvert (v16i8 VR128:$src))), (v8i16 VR128:$src)>;
1372def : Pat<(v8i16 (bitconvert (v4i32 VR128:$src))), (v8i16 VR128:$src)>;
1373def : Pat<(v8i16 (bitconvert (v2i64 VR128:$src))), (v8i16 VR128:$src)>;
1374def : Pat<(v8i16 (bitconvert (v4f32 VR128:$src))), (v8i16 VR128:$src)>;
1375def : Pat<(v8i16 (bitconvert (v2f64 VR128:$src))), (v8i16 VR128:$src)>;
1376def : Pat<(v8i16 (bitconvert (f128  VR128:$src))), (v8i16 VR128:$src)>;
1377
1378def : Pat<(v4i32 (bitconvert (v16i8 VR128:$src))), (v4i32 VR128:$src)>;
1379def : Pat<(v4i32 (bitconvert (v8i16 VR128:$src))), (v4i32 VR128:$src)>;
1380def : Pat<(v4i32 (bitconvert (v2i64 VR128:$src))), (v4i32 VR128:$src)>;
1381def : Pat<(v4i32 (bitconvert (v4f32 VR128:$src))), (v4i32 VR128:$src)>;
1382def : Pat<(v4i32 (bitconvert (v2f64 VR128:$src))), (v4i32 VR128:$src)>;
1383def : Pat<(v4i32 (bitconvert (f128  VR128:$src))), (v4i32 VR128:$src)>;
1384
1385def : Pat<(v2i64 (bitconvert (v16i8 VR128:$src))), (v2i64 VR128:$src)>;
1386def : Pat<(v2i64 (bitconvert (v8i16 VR128:$src))), (v2i64 VR128:$src)>;
1387def : Pat<(v2i64 (bitconvert (v4i32 VR128:$src))), (v2i64 VR128:$src)>;
1388def : Pat<(v2i64 (bitconvert (v4f32 VR128:$src))), (v2i64 VR128:$src)>;
1389def : Pat<(v2i64 (bitconvert (v2f64 VR128:$src))), (v2i64 VR128:$src)>;
1390def : Pat<(v2i64 (bitconvert (f128  VR128:$src))), (v2i64 VR128:$src)>;
1391
1392def : Pat<(v4f32 (bitconvert (v16i8 VR128:$src))), (v4f32 VR128:$src)>;
1393def : Pat<(v4f32 (bitconvert (v8i16 VR128:$src))), (v4f32 VR128:$src)>;
1394def : Pat<(v4f32 (bitconvert (v4i32 VR128:$src))), (v4f32 VR128:$src)>;
1395def : Pat<(v4f32 (bitconvert (v2i64 VR128:$src))), (v4f32 VR128:$src)>;
1396def : Pat<(v4f32 (bitconvert (v2f64 VR128:$src))), (v4f32 VR128:$src)>;
1397def : Pat<(v4f32 (bitconvert (f128  VR128:$src))), (v4f32 VR128:$src)>;
1398
1399def : Pat<(v2f64 (bitconvert (v16i8 VR128:$src))), (v2f64 VR128:$src)>;
1400def : Pat<(v2f64 (bitconvert (v8i16 VR128:$src))), (v2f64 VR128:$src)>;
1401def : Pat<(v2f64 (bitconvert (v4i32 VR128:$src))), (v2f64 VR128:$src)>;
1402def : Pat<(v2f64 (bitconvert (v2i64 VR128:$src))), (v2f64 VR128:$src)>;
1403def : Pat<(v2f64 (bitconvert (v4f32 VR128:$src))), (v2f64 VR128:$src)>;
1404def : Pat<(v2f64 (bitconvert (f128  VR128:$src))), (v2f64 VR128:$src)>;
1405
1406def : Pat<(f128  (bitconvert (v16i8 VR128:$src))), (f128  VR128:$src)>;
1407def : Pat<(f128  (bitconvert (v8i16 VR128:$src))), (f128  VR128:$src)>;
1408def : Pat<(f128  (bitconvert (v4i32 VR128:$src))), (f128  VR128:$src)>;
1409def : Pat<(f128  (bitconvert (v2i64 VR128:$src))), (f128  VR128:$src)>;
1410def : Pat<(f128  (bitconvert (v4f32 VR128:$src))), (f128  VR128:$src)>;
1411def : Pat<(f128  (bitconvert (v2f64 VR128:$src))), (f128  VR128:$src)>;
1412
1413//===----------------------------------------------------------------------===//
1414// Replicating scalars
1415//===----------------------------------------------------------------------===//
1416
1417// Define patterns for replicating a scalar GR32 into a vector of type TYPE.
1418// INDEX is 8 minus the element size in bytes.
1419class VectorReplicateScalar<ValueType type, Instruction insn, bits<16> index>
1420  : Pat<(type (z_replicate GR32:$scalar)),
1421        (insn (VLVGP32 GR32:$scalar, GR32:$scalar), index)>;
1422
1423def : VectorReplicateScalar<v16i8, VREPB, 7>;
1424def : VectorReplicateScalar<v8i16, VREPH, 3>;
1425def : VectorReplicateScalar<v4i32, VREPF, 1>;
1426
1427// i64 replications are just a single isntruction.
1428def : Pat<(v2i64 (z_replicate GR64:$scalar)),
1429          (VLVGP GR64:$scalar, GR64:$scalar)>;
1430
1431//===----------------------------------------------------------------------===//
1432// Floating-point insertion and extraction
1433//===----------------------------------------------------------------------===//
1434
1435// Moving 32-bit values between GPRs and FPRs can be done using VLVGF
1436// and VLGVF.
1437let Predicates = [FeatureVector] in {
1438  def LEFR : UnaryAliasVRS<VR32, GR32>;
1439  def LFER : UnaryAliasVRS<GR64, VR32>;
1440  def : Pat<(f32 (bitconvert (i32 GR32:$src))), (LEFR GR32:$src)>;
1441  def : Pat<(i32 (bitconvert (f32 VR32:$src))),
1442            (EXTRACT_SUBREG (LFER VR32:$src), subreg_l32)>;
1443}
1444
1445// Floating-point values are stored in element 0 of the corresponding
1446// vector register.  Scalar to vector conversion is just a subreg and
1447// scalar replication can just replicate element 0 of the vector register.
1448multiclass ScalarToVectorFP<Instruction vrep, ValueType vt, RegisterOperand cls,
1449                            SubRegIndex subreg> {
1450  def : Pat<(vt (scalar_to_vector cls:$scalar)),
1451            (INSERT_SUBREG (vt (IMPLICIT_DEF)), cls:$scalar, subreg)>;
1452  def : Pat<(vt (z_replicate cls:$scalar)),
1453            (vrep (INSERT_SUBREG (vt (IMPLICIT_DEF)), cls:$scalar,
1454                                 subreg), 0)>;
1455}
1456defm : ScalarToVectorFP<VREPF, v4f32, FP32, subreg_h32>;
1457defm : ScalarToVectorFP<VREPG, v2f64, FP64, subreg_h64>;
1458
1459// Match v2f64 insertions.  The AddedComplexity counters the 3 added by
1460// TableGen for the base register operand in VLVG-based integer insertions
1461// and ensures that this version is strictly better.
1462let AddedComplexity = 4 in {
1463  def : Pat<(z_vector_insert (v2f64 VR128:$vec), FP64:$elt, 0),
1464            (VPDI (INSERT_SUBREG (v2f64 (IMPLICIT_DEF)), FP64:$elt,
1465                                 subreg_h64), VR128:$vec, 1)>;
1466  def : Pat<(z_vector_insert (v2f64 VR128:$vec), FP64:$elt, 1),
1467            (VPDI VR128:$vec, (INSERT_SUBREG (v2f64 (IMPLICIT_DEF)), FP64:$elt,
1468                                             subreg_h64), 0)>;
1469}
1470
1471// We extract floating-point element X by replicating (for elements other
1472// than 0) and then taking a high subreg.  The AddedComplexity counters the
1473// 3 added by TableGen for the base register operand in VLGV-based integer
1474// extractions and ensures that this version is strictly better.
1475let AddedComplexity = 4 in {
1476  def : Pat<(f32 (z_vector_extract (v4f32 VR128:$vec), 0)),
1477            (EXTRACT_SUBREG VR128:$vec, subreg_h32)>;
1478  def : Pat<(f32 (z_vector_extract (v4f32 VR128:$vec), imm32zx2:$index)),
1479            (EXTRACT_SUBREG (VREPF VR128:$vec, imm32zx2:$index), subreg_h32)>;
1480
1481  def : Pat<(f64 (z_vector_extract (v2f64 VR128:$vec), 0)),
1482            (EXTRACT_SUBREG VR128:$vec, subreg_h64)>;
1483  def : Pat<(f64 (z_vector_extract (v2f64 VR128:$vec), imm32zx1:$index)),
1484            (EXTRACT_SUBREG (VREPG VR128:$vec, imm32zx1:$index), subreg_h64)>;
1485}
1486
1487//===----------------------------------------------------------------------===//
1488// Support for 128-bit floating-point values in vector registers
1489//===----------------------------------------------------------------------===//
1490
1491let Predicates = [FeatureVectorEnhancements1] in {
1492  def : Pat<(f128 (load bdxaddr12only:$addr)),
1493            (VL bdxaddr12only:$addr)>;
1494  def : Pat<(store (f128 VR128:$src), bdxaddr12only:$addr),
1495            (VST VR128:$src, bdxaddr12only:$addr)>;
1496
1497  def : Pat<(f128 fpimm0), (VZERO)>;
1498  def : Pat<(f128 fpimmneg0), (WFLNXB (VZERO))>;
1499}
1500
1501//===----------------------------------------------------------------------===//
1502// String instructions
1503//===----------------------------------------------------------------------===//
1504
1505let Predicates = [FeatureVector] in {
1506  defm VFAE  : TernaryOptVRRbSPairGeneric<"vfae", 0xE782>;
1507  defm VFAEB : TernaryOptVRRbSPair<"vfaeb", 0xE782, int_s390_vfaeb,
1508                                   z_vfae_cc, v128b, v128b, 0>;
1509  defm VFAEH : TernaryOptVRRbSPair<"vfaeh", 0xE782, int_s390_vfaeh,
1510                                   z_vfae_cc, v128h, v128h, 1>;
1511  defm VFAEF : TernaryOptVRRbSPair<"vfaef", 0xE782, int_s390_vfaef,
1512                                   z_vfae_cc, v128f, v128f, 2>;
1513  defm VFAEZB : TernaryOptVRRbSPair<"vfaezb", 0xE782, int_s390_vfaezb,
1514                                    z_vfaez_cc, v128b, v128b, 0, 2>;
1515  defm VFAEZH : TernaryOptVRRbSPair<"vfaezh", 0xE782, int_s390_vfaezh,
1516                                    z_vfaez_cc, v128h, v128h, 1, 2>;
1517  defm VFAEZF : TernaryOptVRRbSPair<"vfaezf", 0xE782, int_s390_vfaezf,
1518                                    z_vfaez_cc, v128f, v128f, 2, 2>;
1519
1520  defm VFEE  : BinaryExtraVRRbSPairGeneric<"vfee", 0xE780>;
1521  defm VFEEB : BinaryExtraVRRbSPair<"vfeeb", 0xE780, int_s390_vfeeb,
1522                                    z_vfee_cc, v128b, v128b, 0>;
1523  defm VFEEH : BinaryExtraVRRbSPair<"vfeeh", 0xE780, int_s390_vfeeh,
1524                                    z_vfee_cc, v128h, v128h, 1>;
1525  defm VFEEF : BinaryExtraVRRbSPair<"vfeef", 0xE780, int_s390_vfeef,
1526                                    z_vfee_cc, v128f, v128f, 2>;
1527  defm VFEEZB : BinaryVRRbSPair<"vfeezb", 0xE780, int_s390_vfeezb,
1528                                z_vfeez_cc, v128b, v128b, 0, 2>;
1529  defm VFEEZH : BinaryVRRbSPair<"vfeezh", 0xE780, int_s390_vfeezh,
1530                                z_vfeez_cc, v128h, v128h, 1, 2>;
1531  defm VFEEZF : BinaryVRRbSPair<"vfeezf", 0xE780, int_s390_vfeezf,
1532                                z_vfeez_cc, v128f, v128f, 2, 2>;
1533
1534  defm VFENE  : BinaryExtraVRRbSPairGeneric<"vfene", 0xE781>;
1535  defm VFENEB : BinaryExtraVRRbSPair<"vfeneb", 0xE781, int_s390_vfeneb,
1536                                     z_vfene_cc, v128b, v128b, 0>;
1537  defm VFENEH : BinaryExtraVRRbSPair<"vfeneh", 0xE781, int_s390_vfeneh,
1538                                     z_vfene_cc, v128h, v128h, 1>;
1539  defm VFENEF : BinaryExtraVRRbSPair<"vfenef", 0xE781, int_s390_vfenef,
1540                                     z_vfene_cc, v128f, v128f, 2>;
1541  defm VFENEZB : BinaryVRRbSPair<"vfenezb", 0xE781, int_s390_vfenezb,
1542                                 z_vfenez_cc, v128b, v128b, 0, 2>;
1543  defm VFENEZH : BinaryVRRbSPair<"vfenezh", 0xE781, int_s390_vfenezh,
1544                                 z_vfenez_cc, v128h, v128h, 1, 2>;
1545  defm VFENEZF : BinaryVRRbSPair<"vfenezf", 0xE781, int_s390_vfenezf,
1546                                 z_vfenez_cc, v128f, v128f, 2, 2>;
1547
1548  defm VISTR  : UnaryExtraVRRaSPairGeneric<"vistr", 0xE75C>;
1549  defm VISTRB : UnaryExtraVRRaSPair<"vistrb", 0xE75C, int_s390_vistrb,
1550                                    z_vistr_cc, v128b, v128b, 0>;
1551  defm VISTRH : UnaryExtraVRRaSPair<"vistrh", 0xE75C, int_s390_vistrh,
1552                                    z_vistr_cc, v128h, v128h, 1>;
1553  defm VISTRF : UnaryExtraVRRaSPair<"vistrf", 0xE75C, int_s390_vistrf,
1554                                    z_vistr_cc, v128f, v128f, 2>;
1555
1556  defm VSTRC  : QuaternaryOptVRRdSPairGeneric<"vstrc", 0xE78A>;
1557  defm VSTRCB : QuaternaryOptVRRdSPair<"vstrcb", 0xE78A, int_s390_vstrcb,
1558                                       z_vstrc_cc, v128b, v128b, 0>;
1559  defm VSTRCH : QuaternaryOptVRRdSPair<"vstrch", 0xE78A, int_s390_vstrch,
1560                                       z_vstrc_cc, v128h, v128h, 1>;
1561  defm VSTRCF : QuaternaryOptVRRdSPair<"vstrcf", 0xE78A, int_s390_vstrcf,
1562                                       z_vstrc_cc, v128f, v128f, 2>;
1563  defm VSTRCZB : QuaternaryOptVRRdSPair<"vstrczb", 0xE78A, int_s390_vstrczb,
1564                                        z_vstrcz_cc, v128b, v128b, 0, 2>;
1565  defm VSTRCZH : QuaternaryOptVRRdSPair<"vstrczh", 0xE78A, int_s390_vstrczh,
1566                                        z_vstrcz_cc, v128h, v128h, 1, 2>;
1567  defm VSTRCZF : QuaternaryOptVRRdSPair<"vstrczf", 0xE78A, int_s390_vstrczf,
1568                                        z_vstrcz_cc, v128f, v128f, 2, 2>;
1569}
1570
1571//===----------------------------------------------------------------------===//
1572// Packed-decimal instructions
1573//===----------------------------------------------------------------------===//
1574
1575let Predicates = [FeatureVectorPackedDecimal] in {
1576  def VLIP : BinaryVRIh<"vlip", 0xE649>;
1577
1578  def VPKZ : BinaryVSI<"vpkz", 0xE634, null_frag, 0>;
1579  def VUPKZ : StoreLengthVSI<"vupkz", 0xE63C, null_frag, 0>;
1580
1581  let Defs = [CC] in {
1582    def VCVB : BinaryVRRi<"vcvb", 0xE650, GR32>;
1583    def VCVBG : BinaryVRRi<"vcvbg", 0xE652, GR64>;
1584    def VCVD : TernaryVRIi<"vcvd", 0xE658, GR32>;
1585    def VCVDG : TernaryVRIi<"vcvdg", 0xE65A, GR64>;
1586
1587    def VAP : QuaternaryVRIf<"vap", 0xE671>;
1588    def VSP : QuaternaryVRIf<"vsp", 0xE673>;
1589
1590    def VMP : QuaternaryVRIf<"vmp", 0xE678>;
1591    def VMSP : QuaternaryVRIf<"vmsp", 0xE679>;
1592
1593    def VDP : QuaternaryVRIf<"vdp", 0xE67A>;
1594    def VRP : QuaternaryVRIf<"vrp", 0xE67B>;
1595    def VSDP : QuaternaryVRIf<"vsdp", 0xE67E>;
1596
1597    def VSRP : QuaternaryVRIg<"vsrp", 0xE659>;
1598    def VPSOP : QuaternaryVRIg<"vpsop", 0xE65B>;
1599
1600    def VTP : TestVRRg<"vtp", 0xE65F>;
1601    def VCP : CompareVRRh<"vcp", 0xE677>;
1602  }
1603}
1604