1//===- PPCInstrVSX.td - The PowerPC VSX Extension --*- tablegen -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file describes the VSX extension to the PowerPC instruction set.
10//
11//===----------------------------------------------------------------------===//
12
13// *********************************** NOTE ***********************************
14// ** For POWER8 Little Endian, the VSX swap optimization relies on knowing  **
15// ** which VMX and VSX instructions are lane-sensitive and which are not.   **
16// ** A lane-sensitive instruction relies, implicitly or explicitly, on      **
17// ** whether lanes are numbered from left to right.  An instruction like    **
18// ** VADDFP is not lane-sensitive, because each lane of the result vector   **
19// ** relies only on the corresponding lane of the source vectors.  However, **
20// ** an instruction like VMULESB is lane-sensitive, because "even" and      **
21// ** "odd" lanes are different for big-endian and little-endian numbering.  **
22// **                                                                        **
23// ** When adding new VMX and VSX instructions, please consider whether they **
24// ** are lane-sensitive.  If so, they must be added to a switch statement   **
25// ** in PPCVSXSwapRemoval::gatherVectorInstructions().                      **
26// ****************************************************************************
27
28def PPCRegVSRCAsmOperand : AsmOperandClass {
29  let Name = "RegVSRC"; let PredicateMethod = "isVSRegNumber";
30}
31def vsrc : RegisterOperand<VSRC> {
32  let ParserMatchClass = PPCRegVSRCAsmOperand;
33}
34
35def PPCRegVSFRCAsmOperand : AsmOperandClass {
36  let Name = "RegVSFRC"; let PredicateMethod = "isVSRegNumber";
37}
38def vsfrc : RegisterOperand<VSFRC> {
39  let ParserMatchClass = PPCRegVSFRCAsmOperand;
40}
41
42def PPCRegVSSRCAsmOperand : AsmOperandClass {
43  let Name = "RegVSSRC"; let PredicateMethod = "isVSRegNumber";
44}
45def vssrc : RegisterOperand<VSSRC> {
46  let ParserMatchClass = PPCRegVSSRCAsmOperand;
47}
48
49def PPCRegSPILLTOVSRRCAsmOperand : AsmOperandClass {
50  let Name = "RegSPILLTOVSRRC"; let PredicateMethod = "isVSRegNumber";
51}
52
53def spilltovsrrc : RegisterOperand<SPILLTOVSRRC> {
54  let ParserMatchClass = PPCRegSPILLTOVSRRCAsmOperand;
55}
56
57def SDT_PPCldvsxlh : SDTypeProfile<1, 1, [
58  SDTCisVT<0, v4f32>, SDTCisPtrTy<1>
59]>;
60
61def SDT_PPCfpextlh : SDTypeProfile<1, 1, [
62  SDTCisVT<0, v2f64>, SDTCisVT<1, v4f32>
63]>;
64
65// Little-endian-specific nodes.
66def SDT_PPClxvd2x : SDTypeProfile<1, 1, [
67  SDTCisVT<0, v2f64>, SDTCisPtrTy<1>
68]>;
69def SDT_PPCstxvd2x : SDTypeProfile<0, 2, [
70  SDTCisVT<0, v2f64>, SDTCisPtrTy<1>
71]>;
72def SDT_PPCxxswapd : SDTypeProfile<1, 1, [
73  SDTCisSameAs<0, 1>
74]>;
75def SDTVecConv : SDTypeProfile<1, 2, [
76  SDTCisVec<0>, SDTCisVec<1>, SDTCisPtrTy<2>
77]>;
78def SDTVabsd : SDTypeProfile<1, 3, [
79  SDTCisVec<0>, SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, SDTCisVT<3, i32>
80]>;
81def SDT_PPCld_vec_be : SDTypeProfile<1, 1, [
82  SDTCisVec<0>, SDTCisPtrTy<1>
83]>;
84def SDT_PPCst_vec_be : SDTypeProfile<0, 2, [
85  SDTCisVec<0>, SDTCisPtrTy<1>
86]>;
87
88def PPClxvd2x  : SDNode<"PPCISD::LXVD2X", SDT_PPClxvd2x,
89                        [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
90def PPCstxvd2x : SDNode<"PPCISD::STXVD2X", SDT_PPCstxvd2x,
91                        [SDNPHasChain, SDNPMayStore]>;
92def PPCld_vec_be  : SDNode<"PPCISD::LOAD_VEC_BE", SDT_PPCld_vec_be,
93                        [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
94def PPCst_vec_be : SDNode<"PPCISD::STORE_VEC_BE", SDT_PPCst_vec_be,
95                        [SDNPHasChain, SDNPMayStore]>;
96def PPCxxswapd : SDNode<"PPCISD::XXSWAPD", SDT_PPCxxswapd, [SDNPHasChain]>;
97def PPCmfvsr : SDNode<"PPCISD::MFVSR", SDTUnaryOp, []>;
98def PPCmtvsra : SDNode<"PPCISD::MTVSRA", SDTUnaryOp, []>;
99def PPCmtvsrz : SDNode<"PPCISD::MTVSRZ", SDTUnaryOp, []>;
100def PPCsvec2fp : SDNode<"PPCISD::SINT_VEC_TO_FP", SDTVecConv, []>;
101def PPCuvec2fp: SDNode<"PPCISD::UINT_VEC_TO_FP", SDTVecConv, []>;
102def PPCswapNoChain : SDNode<"PPCISD::SWAP_NO_CHAIN", SDT_PPCxxswapd>;
103def PPCvabsd : SDNode<"PPCISD::VABSD", SDTVabsd, []>;
104
105def PPCfpextlh : SDNode<"PPCISD::FP_EXTEND_LH", SDT_PPCfpextlh, []>;
106def PPCldvsxlh : SDNode<"PPCISD::LD_VSX_LH", SDT_PPCldvsxlh,
107                        [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
108
109multiclass XX3Form_Rcr<bits<6> opcode, bits<7> xo, string asmbase,
110                    string asmstr, InstrItinClass itin, Intrinsic Int,
111                    ValueType OutTy, ValueType InTy> {
112  let BaseName = asmbase in {
113    def NAME : XX3Form_Rc<opcode, xo, (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
114                       !strconcat(asmbase, !strconcat(" ", asmstr)), itin,
115                       [(set OutTy:$XT, (Int InTy:$XA, InTy:$XB))]>;
116    let Defs = [CR6] in
117    def o    : XX3Form_Rc<opcode, xo, (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
118                       !strconcat(asmbase, !strconcat(". ", asmstr)), itin,
119                       [(set InTy:$XT,
120                                (InTy (PPCvcmp_o InTy:$XA, InTy:$XB, xo)))]>,
121                       isDOT;
122  }
123}
124
125// Instruction form with a single input register for instructions such as
126// XXPERMDI. The reason for defining this is that specifying multiple chained
127// operands (such as loads) to an instruction will perform both chained
128// operations rather than coalescing them into a single register - even though
129// the source memory location is the same. This simply forces the instruction
130// to use the same register for both inputs.
131// For example, an output DAG such as this:
132//   (XXPERMDI (LXSIBZX xoaddr:$src), (LXSIBZX xoaddr:$src ), 0))
133// would result in two load instructions emitted and used as separate inputs
134// to the XXPERMDI instruction.
135class XX3Form_2s<bits<6> opcode, bits<5> xo, dag OOL, dag IOL, string asmstr,
136                 InstrItinClass itin, list<dag> pattern>
137  : XX3Form_2<opcode, xo, OOL, IOL, asmstr, itin, pattern> {
138    let XB = XA;
139}
140
141def HasVSX : Predicate<"PPCSubTarget->hasVSX()">;
142def IsLittleEndian : Predicate<"PPCSubTarget->isLittleEndian()">;
143def IsBigEndian : Predicate<"!PPCSubTarget->isLittleEndian()">;
144def HasOnlySwappingMemOps : Predicate<"!PPCSubTarget->hasP9Vector()">;
145
146let Predicates = [HasVSX] in {
147let AddedComplexity = 400 in { // Prefer VSX patterns over non-VSX patterns.
148let hasSideEffects = 0 in { // VSX instructions don't have side effects.
149let Uses = [RM] in {
150
151  // Load indexed instructions
152  let mayLoad = 1, mayStore = 0 in {
153    let CodeSize = 3 in
154    def LXSDX : XX1Form_memOp<31, 588,
155                        (outs vsfrc:$XT), (ins memrr:$src),
156                        "lxsdx $XT, $src", IIC_LdStLFD,
157                        []>;
158
159    // Pseudo instruction XFLOADf64 will be expanded to LXSDX or LFDX later
160    let CodeSize = 3 in
161      def XFLOADf64  : PseudoXFormMemOp<(outs vsfrc:$XT), (ins memrr:$src),
162                              "#XFLOADf64",
163                              [(set f64:$XT, (load xoaddr:$src))]>;
164
165    let Predicates = [HasVSX, HasOnlySwappingMemOps] in
166    def LXVD2X : XX1Form_memOp<31, 844,
167                         (outs vsrc:$XT), (ins memrr:$src),
168                         "lxvd2x $XT, $src", IIC_LdStLFD,
169                         [(set v2f64:$XT, (int_ppc_vsx_lxvd2x xoaddr:$src))]>;
170
171    def LXVDSX : XX1Form_memOp<31, 332,
172                         (outs vsrc:$XT), (ins memrr:$src),
173                         "lxvdsx $XT, $src", IIC_LdStLFD, []>;
174
175    let Predicates = [HasVSX, HasOnlySwappingMemOps] in
176    def LXVW4X : XX1Form_memOp<31, 780,
177                         (outs vsrc:$XT), (ins memrr:$src),
178                         "lxvw4x $XT, $src", IIC_LdStLFD,
179                         []>;
180  } // mayLoad
181
182  // Store indexed instructions
183  let mayStore = 1, mayLoad = 0 in {
184    let CodeSize = 3 in
185    def STXSDX : XX1Form_memOp<31, 716,
186                        (outs), (ins vsfrc:$XT, memrr:$dst),
187                        "stxsdx $XT, $dst", IIC_LdStSTFD,
188                        []>;
189
190    // Pseudo instruction XFSTOREf64  will be expanded to STXSDX or STFDX later
191    let CodeSize = 3 in
192      def XFSTOREf64 : PseudoXFormMemOp<(outs), (ins vsfrc:$XT, memrr:$dst),
193                              "#XFSTOREf64",
194                              [(store f64:$XT, xoaddr:$dst)]>;
195
196    let Predicates = [HasVSX, HasOnlySwappingMemOps] in {
197    // The behaviour of this instruction is endianness-specific so we provide no
198    // pattern to match it without considering endianness.
199    def STXVD2X : XX1Form_memOp<31, 972,
200                         (outs), (ins vsrc:$XT, memrr:$dst),
201                         "stxvd2x $XT, $dst", IIC_LdStSTFD,
202                         []>;
203
204    def STXVW4X : XX1Form_memOp<31, 908,
205                         (outs), (ins vsrc:$XT, memrr:$dst),
206                         "stxvw4x $XT, $dst", IIC_LdStSTFD,
207                         []>;
208    }
209  } // mayStore
210
211  // Add/Mul Instructions
212  let isCommutable = 1 in {
213    def XSADDDP : XX3Form<60, 32,
214                          (outs vsfrc:$XT), (ins vsfrc:$XA, vsfrc:$XB),
215                          "xsadddp $XT, $XA, $XB", IIC_VecFP,
216                          [(set f64:$XT, (fadd f64:$XA, f64:$XB))]>;
217    def XSMULDP : XX3Form<60, 48,
218                          (outs vsfrc:$XT), (ins vsfrc:$XA, vsfrc:$XB),
219                          "xsmuldp $XT, $XA, $XB", IIC_VecFP,
220                          [(set f64:$XT, (fmul f64:$XA, f64:$XB))]>;
221
222    def XVADDDP : XX3Form<60, 96,
223                          (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
224                          "xvadddp $XT, $XA, $XB", IIC_VecFP,
225                          [(set v2f64:$XT, (fadd v2f64:$XA, v2f64:$XB))]>;
226
227    def XVADDSP : XX3Form<60, 64,
228                          (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
229                          "xvaddsp $XT, $XA, $XB", IIC_VecFP,
230                          [(set v4f32:$XT, (fadd v4f32:$XA, v4f32:$XB))]>;
231
232    def XVMULDP : XX3Form<60, 112,
233                          (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
234                          "xvmuldp $XT, $XA, $XB", IIC_VecFP,
235                          [(set v2f64:$XT, (fmul v2f64:$XA, v2f64:$XB))]>;
236
237    def XVMULSP : XX3Form<60, 80,
238                          (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
239                          "xvmulsp $XT, $XA, $XB", IIC_VecFP,
240                          [(set v4f32:$XT, (fmul v4f32:$XA, v4f32:$XB))]>;
241  }
242
243  // Subtract Instructions
244  def XSSUBDP : XX3Form<60, 40,
245                        (outs vsfrc:$XT), (ins vsfrc:$XA, vsfrc:$XB),
246                        "xssubdp $XT, $XA, $XB", IIC_VecFP,
247                        [(set f64:$XT, (fsub f64:$XA, f64:$XB))]>;
248
249  def XVSUBDP : XX3Form<60, 104,
250                        (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
251                        "xvsubdp $XT, $XA, $XB", IIC_VecFP,
252                        [(set v2f64:$XT, (fsub v2f64:$XA, v2f64:$XB))]>;
253  def XVSUBSP : XX3Form<60, 72,
254                        (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
255                        "xvsubsp $XT, $XA, $XB", IIC_VecFP,
256                        [(set v4f32:$XT, (fsub v4f32:$XA, v4f32:$XB))]>;
257
258  // FMA Instructions
259  let BaseName = "XSMADDADP" in {
260  let isCommutable = 1 in
261  def XSMADDADP : XX3Form<60, 33,
262                          (outs vsfrc:$XT), (ins vsfrc:$XTi, vsfrc:$XA, vsfrc:$XB),
263                          "xsmaddadp $XT, $XA, $XB", IIC_VecFP,
264                          [(set f64:$XT, (fma f64:$XA, f64:$XB, f64:$XTi))]>,
265                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
266                          AltVSXFMARel;
267  let IsVSXFMAAlt = 1 in
268  def XSMADDMDP : XX3Form<60, 41,
269                          (outs vsfrc:$XT), (ins vsfrc:$XTi, vsfrc:$XA, vsfrc:$XB),
270                          "xsmaddmdp $XT, $XA, $XB", IIC_VecFP, []>,
271                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
272                          AltVSXFMARel;
273  }
274
275  let BaseName = "XSMSUBADP" in {
276  let isCommutable = 1 in
277  def XSMSUBADP : XX3Form<60, 49,
278                          (outs vsfrc:$XT), (ins vsfrc:$XTi, vsfrc:$XA, vsfrc:$XB),
279                          "xsmsubadp $XT, $XA, $XB", IIC_VecFP,
280                          [(set f64:$XT, (fma f64:$XA, f64:$XB, (fneg f64:$XTi)))]>,
281                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
282                          AltVSXFMARel;
283  let IsVSXFMAAlt = 1 in
284  def XSMSUBMDP : XX3Form<60, 57,
285                          (outs vsfrc:$XT), (ins vsfrc:$XTi, vsfrc:$XA, vsfrc:$XB),
286                          "xsmsubmdp $XT, $XA, $XB", IIC_VecFP, []>,
287                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
288                          AltVSXFMARel;
289  }
290
291  let BaseName = "XSNMADDADP" in {
292  let isCommutable = 1 in
293  def XSNMADDADP : XX3Form<60, 161,
294                          (outs vsfrc:$XT), (ins vsfrc:$XTi, vsfrc:$XA, vsfrc:$XB),
295                          "xsnmaddadp $XT, $XA, $XB", IIC_VecFP,
296                          [(set f64:$XT, (fneg (fma f64:$XA, f64:$XB, f64:$XTi)))]>,
297                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
298                          AltVSXFMARel;
299  let IsVSXFMAAlt = 1 in
300  def XSNMADDMDP : XX3Form<60, 169,
301                          (outs vsfrc:$XT), (ins vsfrc:$XTi, vsfrc:$XA, vsfrc:$XB),
302                          "xsnmaddmdp $XT, $XA, $XB", IIC_VecFP, []>,
303                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
304                          AltVSXFMARel;
305  }
306
307  let BaseName = "XSNMSUBADP" in {
308  let isCommutable = 1 in
309  def XSNMSUBADP : XX3Form<60, 177,
310                          (outs vsfrc:$XT), (ins vsfrc:$XTi, vsfrc:$XA, vsfrc:$XB),
311                          "xsnmsubadp $XT, $XA, $XB", IIC_VecFP,
312                          [(set f64:$XT, (fneg (fma f64:$XA, f64:$XB, (fneg f64:$XTi))))]>,
313                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
314                          AltVSXFMARel;
315  let IsVSXFMAAlt = 1 in
316  def XSNMSUBMDP : XX3Form<60, 185,
317                          (outs vsfrc:$XT), (ins vsfrc:$XTi, vsfrc:$XA, vsfrc:$XB),
318                          "xsnmsubmdp $XT, $XA, $XB", IIC_VecFP, []>,
319                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
320                          AltVSXFMARel;
321  }
322
323  let BaseName = "XVMADDADP" in {
324  let isCommutable = 1 in
325  def XVMADDADP : XX3Form<60, 97,
326                          (outs vsrc:$XT), (ins vsrc:$XTi, vsrc:$XA, vsrc:$XB),
327                          "xvmaddadp $XT, $XA, $XB", IIC_VecFP,
328                          [(set v2f64:$XT, (fma v2f64:$XA, v2f64:$XB, v2f64:$XTi))]>,
329                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
330                          AltVSXFMARel;
331  let IsVSXFMAAlt = 1 in
332  def XVMADDMDP : XX3Form<60, 105,
333                          (outs vsrc:$XT), (ins vsrc:$XTi, vsrc:$XA, vsrc:$XB),
334                          "xvmaddmdp $XT, $XA, $XB", IIC_VecFP, []>,
335                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
336                          AltVSXFMARel;
337  }
338
339  let BaseName = "XVMADDASP" in {
340  let isCommutable = 1 in
341  def XVMADDASP : XX3Form<60, 65,
342                          (outs vsrc:$XT), (ins vsrc:$XTi, vsrc:$XA, vsrc:$XB),
343                          "xvmaddasp $XT, $XA, $XB", IIC_VecFP,
344                          [(set v4f32:$XT, (fma v4f32:$XA, v4f32:$XB, v4f32:$XTi))]>,
345                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
346                          AltVSXFMARel;
347  let IsVSXFMAAlt = 1 in
348  def XVMADDMSP : XX3Form<60, 73,
349                          (outs vsrc:$XT), (ins vsrc:$XTi, vsrc:$XA, vsrc:$XB),
350                          "xvmaddmsp $XT, $XA, $XB", IIC_VecFP, []>,
351                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
352                          AltVSXFMARel;
353  }
354
355  let BaseName = "XVMSUBADP" in {
356  let isCommutable = 1 in
357  def XVMSUBADP : XX3Form<60, 113,
358                          (outs vsrc:$XT), (ins vsrc:$XTi, vsrc:$XA, vsrc:$XB),
359                          "xvmsubadp $XT, $XA, $XB", IIC_VecFP,
360                          [(set v2f64:$XT, (fma v2f64:$XA, v2f64:$XB, (fneg v2f64:$XTi)))]>,
361                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
362                          AltVSXFMARel;
363  let IsVSXFMAAlt = 1 in
364  def XVMSUBMDP : XX3Form<60, 121,
365                          (outs vsrc:$XT), (ins vsrc:$XTi, vsrc:$XA, vsrc:$XB),
366                          "xvmsubmdp $XT, $XA, $XB", IIC_VecFP, []>,
367                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
368                          AltVSXFMARel;
369  }
370
371  let BaseName = "XVMSUBASP" in {
372  let isCommutable = 1 in
373  def XVMSUBASP : XX3Form<60, 81,
374                          (outs vsrc:$XT), (ins vsrc:$XTi, vsrc:$XA, vsrc:$XB),
375                          "xvmsubasp $XT, $XA, $XB", IIC_VecFP,
376                          [(set v4f32:$XT, (fma v4f32:$XA, v4f32:$XB, (fneg v4f32:$XTi)))]>,
377                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
378                          AltVSXFMARel;
379  let IsVSXFMAAlt = 1 in
380  def XVMSUBMSP : XX3Form<60, 89,
381                          (outs vsrc:$XT), (ins vsrc:$XTi, vsrc:$XA, vsrc:$XB),
382                          "xvmsubmsp $XT, $XA, $XB", IIC_VecFP, []>,
383                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
384                          AltVSXFMARel;
385  }
386
387  let BaseName = "XVNMADDADP" in {
388  let isCommutable = 1 in
389  def XVNMADDADP : XX3Form<60, 225,
390                          (outs vsrc:$XT), (ins vsrc:$XTi, vsrc:$XA, vsrc:$XB),
391                          "xvnmaddadp $XT, $XA, $XB", IIC_VecFP,
392                          [(set v2f64:$XT, (fneg (fma v2f64:$XA, v2f64:$XB, v2f64:$XTi)))]>,
393                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
394                          AltVSXFMARel;
395  let IsVSXFMAAlt = 1 in
396  def XVNMADDMDP : XX3Form<60, 233,
397                          (outs vsrc:$XT), (ins vsrc:$XTi, vsrc:$XA, vsrc:$XB),
398                          "xvnmaddmdp $XT, $XA, $XB", IIC_VecFP, []>,
399                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
400                          AltVSXFMARel;
401  }
402
403  let BaseName = "XVNMADDASP" in {
404  let isCommutable = 1 in
405  def XVNMADDASP : XX3Form<60, 193,
406                          (outs vsrc:$XT), (ins vsrc:$XTi, vsrc:$XA, vsrc:$XB),
407                          "xvnmaddasp $XT, $XA, $XB", IIC_VecFP,
408                          [(set v4f32:$XT, (fneg (fma v4f32:$XA, v4f32:$XB, v4f32:$XTi)))]>,
409                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
410                          AltVSXFMARel;
411  let IsVSXFMAAlt = 1 in
412  def XVNMADDMSP : XX3Form<60, 201,
413                          (outs vsrc:$XT), (ins vsrc:$XTi, vsrc:$XA, vsrc:$XB),
414                          "xvnmaddmsp $XT, $XA, $XB", IIC_VecFP, []>,
415                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
416                          AltVSXFMARel;
417  }
418
419  let BaseName = "XVNMSUBADP" in {
420  let isCommutable = 1 in
421  def XVNMSUBADP : XX3Form<60, 241,
422                          (outs vsrc:$XT), (ins vsrc:$XTi, vsrc:$XA, vsrc:$XB),
423                          "xvnmsubadp $XT, $XA, $XB", IIC_VecFP,
424                          [(set v2f64:$XT, (fneg (fma v2f64:$XA, v2f64:$XB, (fneg v2f64:$XTi))))]>,
425                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
426                          AltVSXFMARel;
427  let IsVSXFMAAlt = 1 in
428  def XVNMSUBMDP : XX3Form<60, 249,
429                          (outs vsrc:$XT), (ins vsrc:$XTi, vsrc:$XA, vsrc:$XB),
430                          "xvnmsubmdp $XT, $XA, $XB", IIC_VecFP, []>,
431                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
432                          AltVSXFMARel;
433  }
434
435  let BaseName = "XVNMSUBASP" in {
436  let isCommutable = 1 in
437  def XVNMSUBASP : XX3Form<60, 209,
438                          (outs vsrc:$XT), (ins vsrc:$XTi, vsrc:$XA, vsrc:$XB),
439                          "xvnmsubasp $XT, $XA, $XB", IIC_VecFP,
440                          [(set v4f32:$XT, (fneg (fma v4f32:$XA, v4f32:$XB, (fneg v4f32:$XTi))))]>,
441                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
442                          AltVSXFMARel;
443  let IsVSXFMAAlt = 1 in
444  def XVNMSUBMSP : XX3Form<60, 217,
445                          (outs vsrc:$XT), (ins vsrc:$XTi, vsrc:$XA, vsrc:$XB),
446                          "xvnmsubmsp $XT, $XA, $XB", IIC_VecFP, []>,
447                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
448                          AltVSXFMARel;
449  }
450
451  // Division Instructions
452  def XSDIVDP : XX3Form<60, 56,
453                        (outs vsfrc:$XT), (ins vsfrc:$XA, vsfrc:$XB),
454                        "xsdivdp $XT, $XA, $XB", IIC_FPDivD,
455                        [(set f64:$XT, (fdiv f64:$XA, f64:$XB))]>;
456  def XSSQRTDP : XX2Form<60, 75,
457                        (outs vsfrc:$XT), (ins vsfrc:$XB),
458                        "xssqrtdp $XT, $XB", IIC_FPSqrtD,
459                        [(set f64:$XT, (fsqrt f64:$XB))]>;
460
461  def XSREDP : XX2Form<60, 90,
462                        (outs vsfrc:$XT), (ins vsfrc:$XB),
463                        "xsredp $XT, $XB", IIC_VecFP,
464                        [(set f64:$XT, (PPCfre f64:$XB))]>;
465  def XSRSQRTEDP : XX2Form<60, 74,
466                           (outs vsfrc:$XT), (ins vsfrc:$XB),
467                           "xsrsqrtedp $XT, $XB", IIC_VecFP,
468                           [(set f64:$XT, (PPCfrsqrte f64:$XB))]>;
469
470  def XSTDIVDP : XX3Form_1<60, 61,
471                         (outs crrc:$crD), (ins vsfrc:$XA, vsfrc:$XB),
472                         "xstdivdp $crD, $XA, $XB", IIC_FPCompare, []>;
473  def XSTSQRTDP : XX2Form_1<60, 106,
474                          (outs crrc:$crD), (ins vsfrc:$XB),
475                          "xstsqrtdp $crD, $XB", IIC_FPCompare, []>;
476
477  def XVDIVDP : XX3Form<60, 120,
478                        (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
479                        "xvdivdp $XT, $XA, $XB", IIC_FPDivD,
480                        [(set v2f64:$XT, (fdiv v2f64:$XA, v2f64:$XB))]>;
481  def XVDIVSP : XX3Form<60, 88,
482                        (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
483                        "xvdivsp $XT, $XA, $XB", IIC_FPDivS,
484                        [(set v4f32:$XT, (fdiv v4f32:$XA, v4f32:$XB))]>;
485
486  def XVSQRTDP : XX2Form<60, 203,
487                        (outs vsrc:$XT), (ins vsrc:$XB),
488                        "xvsqrtdp $XT, $XB", IIC_FPSqrtD,
489                        [(set v2f64:$XT, (fsqrt v2f64:$XB))]>;
490  def XVSQRTSP : XX2Form<60, 139,
491                        (outs vsrc:$XT), (ins vsrc:$XB),
492                        "xvsqrtsp $XT, $XB", IIC_FPSqrtS,
493                        [(set v4f32:$XT, (fsqrt v4f32:$XB))]>;
494
495  def XVTDIVDP : XX3Form_1<60, 125,
496                         (outs crrc:$crD), (ins vsrc:$XA, vsrc:$XB),
497                         "xvtdivdp $crD, $XA, $XB", IIC_FPCompare, []>;
498  def XVTDIVSP : XX3Form_1<60, 93,
499                         (outs crrc:$crD), (ins vsrc:$XA, vsrc:$XB),
500                         "xvtdivsp $crD, $XA, $XB", IIC_FPCompare, []>;
501
502  def XVTSQRTDP : XX2Form_1<60, 234,
503                          (outs crrc:$crD), (ins vsrc:$XB),
504                          "xvtsqrtdp $crD, $XB", IIC_FPCompare, []>;
505  def XVTSQRTSP : XX2Form_1<60, 170,
506                          (outs crrc:$crD), (ins vsrc:$XB),
507                          "xvtsqrtsp $crD, $XB", IIC_FPCompare, []>;
508
509  def XVREDP : XX2Form<60, 218,
510                        (outs vsrc:$XT), (ins vsrc:$XB),
511                        "xvredp $XT, $XB", IIC_VecFP,
512                        [(set v2f64:$XT, (PPCfre v2f64:$XB))]>;
513  def XVRESP : XX2Form<60, 154,
514                        (outs vsrc:$XT), (ins vsrc:$XB),
515                        "xvresp $XT, $XB", IIC_VecFP,
516                        [(set v4f32:$XT, (PPCfre v4f32:$XB))]>;
517
518  def XVRSQRTEDP : XX2Form<60, 202,
519                           (outs vsrc:$XT), (ins vsrc:$XB),
520                           "xvrsqrtedp $XT, $XB", IIC_VecFP,
521                           [(set v2f64:$XT, (PPCfrsqrte v2f64:$XB))]>;
522  def XVRSQRTESP : XX2Form<60, 138,
523                           (outs vsrc:$XT), (ins vsrc:$XB),
524                           "xvrsqrtesp $XT, $XB", IIC_VecFP,
525                           [(set v4f32:$XT, (PPCfrsqrte v4f32:$XB))]>;
526
527  // Compare Instructions
528  def XSCMPODP : XX3Form_1<60, 43,
529                           (outs crrc:$crD), (ins vsfrc:$XA, vsfrc:$XB),
530                           "xscmpodp $crD, $XA, $XB", IIC_FPCompare, []>;
531  def XSCMPUDP : XX3Form_1<60, 35,
532                           (outs crrc:$crD), (ins vsfrc:$XA, vsfrc:$XB),
533                           "xscmpudp $crD, $XA, $XB", IIC_FPCompare, []>;
534
535  defm XVCMPEQDP : XX3Form_Rcr<60, 99,
536                             "xvcmpeqdp", "$XT, $XA, $XB", IIC_VecFPCompare,
537                             int_ppc_vsx_xvcmpeqdp, v2i64, v2f64>;
538  defm XVCMPEQSP : XX3Form_Rcr<60, 67,
539                             "xvcmpeqsp", "$XT, $XA, $XB", IIC_VecFPCompare,
540                             int_ppc_vsx_xvcmpeqsp, v4i32, v4f32>;
541  defm XVCMPGEDP : XX3Form_Rcr<60, 115,
542                             "xvcmpgedp", "$XT, $XA, $XB", IIC_VecFPCompare,
543                             int_ppc_vsx_xvcmpgedp, v2i64, v2f64>;
544  defm XVCMPGESP : XX3Form_Rcr<60, 83,
545                             "xvcmpgesp", "$XT, $XA, $XB", IIC_VecFPCompare,
546                             int_ppc_vsx_xvcmpgesp, v4i32, v4f32>;
547  defm XVCMPGTDP : XX3Form_Rcr<60, 107,
548                             "xvcmpgtdp", "$XT, $XA, $XB", IIC_VecFPCompare,
549                             int_ppc_vsx_xvcmpgtdp, v2i64, v2f64>;
550  defm XVCMPGTSP : XX3Form_Rcr<60, 75,
551                             "xvcmpgtsp", "$XT, $XA, $XB", IIC_VecFPCompare,
552                             int_ppc_vsx_xvcmpgtsp, v4i32, v4f32>;
553
554  // Move Instructions
555  def XSABSDP : XX2Form<60, 345,
556                      (outs vsfrc:$XT), (ins vsfrc:$XB),
557                      "xsabsdp $XT, $XB", IIC_VecFP,
558                      [(set f64:$XT, (fabs f64:$XB))]>;
559  def XSNABSDP : XX2Form<60, 361,
560                      (outs vsfrc:$XT), (ins vsfrc:$XB),
561                      "xsnabsdp $XT, $XB", IIC_VecFP,
562                      [(set f64:$XT, (fneg (fabs f64:$XB)))]>;
563  def XSNEGDP : XX2Form<60, 377,
564                      (outs vsfrc:$XT), (ins vsfrc:$XB),
565                      "xsnegdp $XT, $XB", IIC_VecFP,
566                      [(set f64:$XT, (fneg f64:$XB))]>;
567  def XSCPSGNDP : XX3Form<60, 176,
568                      (outs vsfrc:$XT), (ins vsfrc:$XA, vsfrc:$XB),
569                      "xscpsgndp $XT, $XA, $XB", IIC_VecFP,
570                      [(set f64:$XT, (fcopysign f64:$XB, f64:$XA))]>;
571
572  def XVABSDP : XX2Form<60, 473,
573                      (outs vsrc:$XT), (ins vsrc:$XB),
574                      "xvabsdp $XT, $XB", IIC_VecFP,
575                      [(set v2f64:$XT, (fabs v2f64:$XB))]>;
576
577  def XVABSSP : XX2Form<60, 409,
578                      (outs vsrc:$XT), (ins vsrc:$XB),
579                      "xvabssp $XT, $XB", IIC_VecFP,
580                      [(set v4f32:$XT, (fabs v4f32:$XB))]>;
581
582  def XVCPSGNDP : XX3Form<60, 240,
583                      (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
584                      "xvcpsgndp $XT, $XA, $XB", IIC_VecFP,
585                      [(set v2f64:$XT, (fcopysign v2f64:$XB, v2f64:$XA))]>;
586  def XVCPSGNSP : XX3Form<60, 208,
587                      (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
588                      "xvcpsgnsp $XT, $XA, $XB", IIC_VecFP,
589                      [(set v4f32:$XT, (fcopysign v4f32:$XB, v4f32:$XA))]>;
590
591  def XVNABSDP : XX2Form<60, 489,
592                      (outs vsrc:$XT), (ins vsrc:$XB),
593                      "xvnabsdp $XT, $XB", IIC_VecFP,
594                      [(set v2f64:$XT, (fneg (fabs v2f64:$XB)))]>;
595  def XVNABSSP : XX2Form<60, 425,
596                      (outs vsrc:$XT), (ins vsrc:$XB),
597                      "xvnabssp $XT, $XB", IIC_VecFP,
598                      [(set v4f32:$XT, (fneg (fabs v4f32:$XB)))]>;
599
600  def XVNEGDP : XX2Form<60, 505,
601                      (outs vsrc:$XT), (ins vsrc:$XB),
602                      "xvnegdp $XT, $XB", IIC_VecFP,
603                      [(set v2f64:$XT, (fneg v2f64:$XB))]>;
604  def XVNEGSP : XX2Form<60, 441,
605                      (outs vsrc:$XT), (ins vsrc:$XB),
606                      "xvnegsp $XT, $XB", IIC_VecFP,
607                      [(set v4f32:$XT, (fneg v4f32:$XB))]>;
608
609  // Conversion Instructions
610  def XSCVDPSP : XX2Form<60, 265,
611                      (outs vsfrc:$XT), (ins vsfrc:$XB),
612                      "xscvdpsp $XT, $XB", IIC_VecFP, []>;
613  def XSCVDPSXDS : XX2Form<60, 344,
614                      (outs vsfrc:$XT), (ins vsfrc:$XB),
615                      "xscvdpsxds $XT, $XB", IIC_VecFP,
616                      [(set f64:$XT, (PPCfctidz f64:$XB))]>;
617  let isCodeGenOnly = 1 in
618  def XSCVDPSXDSs : XX2Form<60, 344,
619                      (outs vssrc:$XT), (ins vssrc:$XB),
620                      "xscvdpsxds $XT, $XB", IIC_VecFP,
621                      [(set f32:$XT, (PPCfctidz f32:$XB))]>;
622  def XSCVDPSXWS : XX2Form<60, 88,
623                      (outs vsfrc:$XT), (ins vsfrc:$XB),
624                      "xscvdpsxws $XT, $XB", IIC_VecFP,
625                      [(set f64:$XT, (PPCfctiwz f64:$XB))]>;
626  let isCodeGenOnly = 1 in
627  def XSCVDPSXWSs : XX2Form<60, 88,
628                      (outs vssrc:$XT), (ins vssrc:$XB),
629                      "xscvdpsxws $XT, $XB", IIC_VecFP,
630                      [(set f32:$XT, (PPCfctiwz f32:$XB))]>;
631  def XSCVDPUXDS : XX2Form<60, 328,
632                      (outs vsfrc:$XT), (ins vsfrc:$XB),
633                      "xscvdpuxds $XT, $XB", IIC_VecFP,
634                      [(set f64:$XT, (PPCfctiduz f64:$XB))]>;
635  let isCodeGenOnly = 1 in
636  def XSCVDPUXDSs : XX2Form<60, 328,
637                      (outs vssrc:$XT), (ins vssrc:$XB),
638                      "xscvdpuxds $XT, $XB", IIC_VecFP,
639                      [(set f32:$XT, (PPCfctiduz f32:$XB))]>;
640  def XSCVDPUXWS : XX2Form<60, 72,
641                      (outs vsfrc:$XT), (ins vsfrc:$XB),
642                      "xscvdpuxws $XT, $XB", IIC_VecFP,
643                      [(set f64:$XT, (PPCfctiwuz f64:$XB))]>;
644  let isCodeGenOnly = 1 in
645  def XSCVDPUXWSs : XX2Form<60, 72,
646                      (outs vssrc:$XT), (ins vssrc:$XB),
647                      "xscvdpuxws $XT, $XB", IIC_VecFP,
648                      [(set f32:$XT, (PPCfctiwuz f32:$XB))]>;
649  def XSCVSPDP : XX2Form<60, 329,
650                      (outs vsfrc:$XT), (ins vsfrc:$XB),
651                      "xscvspdp $XT, $XB", IIC_VecFP, []>;
652  def XSCVSXDDP : XX2Form<60, 376,
653                      (outs vsfrc:$XT), (ins vsfrc:$XB),
654                      "xscvsxddp $XT, $XB", IIC_VecFP,
655                      [(set f64:$XT, (PPCfcfid f64:$XB))]>;
656  def XSCVUXDDP : XX2Form<60, 360,
657                      (outs vsfrc:$XT), (ins vsfrc:$XB),
658                      "xscvuxddp $XT, $XB", IIC_VecFP,
659                      [(set f64:$XT, (PPCfcfidu f64:$XB))]>;
660
661  def XVCVDPSP : XX2Form<60, 393,
662                      (outs vsrc:$XT), (ins vsrc:$XB),
663                      "xvcvdpsp $XT, $XB", IIC_VecFP,
664                      [(set v4f32:$XT, (int_ppc_vsx_xvcvdpsp v2f64:$XB))]>;
665  def XVCVDPSXDS : XX2Form<60, 472,
666                      (outs vsrc:$XT), (ins vsrc:$XB),
667                      "xvcvdpsxds $XT, $XB", IIC_VecFP,
668                      [(set v2i64:$XT, (fp_to_sint v2f64:$XB))]>;
669  def XVCVDPSXWS : XX2Form<60, 216,
670                      (outs vsrc:$XT), (ins vsrc:$XB),
671                      "xvcvdpsxws $XT, $XB", IIC_VecFP,
672                      [(set v4i32:$XT, (int_ppc_vsx_xvcvdpsxws v2f64:$XB))]>;
673  def XVCVDPUXDS : XX2Form<60, 456,
674                      (outs vsrc:$XT), (ins vsrc:$XB),
675                      "xvcvdpuxds $XT, $XB", IIC_VecFP,
676                      [(set v2i64:$XT, (fp_to_uint v2f64:$XB))]>;
677  def XVCVDPUXWS : XX2Form<60, 200,
678                      (outs vsrc:$XT), (ins vsrc:$XB),
679                      "xvcvdpuxws $XT, $XB", IIC_VecFP,
680                      [(set v4i32:$XT, (int_ppc_vsx_xvcvdpuxws v2f64:$XB))]>;
681
682  def XVCVSPDP : XX2Form<60, 457,
683                      (outs vsrc:$XT), (ins vsrc:$XB),
684                      "xvcvspdp $XT, $XB", IIC_VecFP,
685                      [(set v2f64:$XT, (int_ppc_vsx_xvcvspdp v4f32:$XB))]>;
686  def XVCVSPSXDS : XX2Form<60, 408,
687                      (outs vsrc:$XT), (ins vsrc:$XB),
688                      "xvcvspsxds $XT, $XB", IIC_VecFP, []>;
689  def XVCVSPSXWS : XX2Form<60, 152,
690                      (outs vsrc:$XT), (ins vsrc:$XB),
691                      "xvcvspsxws $XT, $XB", IIC_VecFP,
692                      [(set v4i32:$XT, (fp_to_sint v4f32:$XB))]>;
693  def XVCVSPUXDS : XX2Form<60, 392,
694                      (outs vsrc:$XT), (ins vsrc:$XB),
695                      "xvcvspuxds $XT, $XB", IIC_VecFP, []>;
696  def XVCVSPUXWS : XX2Form<60, 136,
697                      (outs vsrc:$XT), (ins vsrc:$XB),
698                      "xvcvspuxws $XT, $XB", IIC_VecFP,
699                      [(set v4i32:$XT, (fp_to_uint v4f32:$XB))]>;
700  def XVCVSXDDP : XX2Form<60, 504,
701                      (outs vsrc:$XT), (ins vsrc:$XB),
702                      "xvcvsxddp $XT, $XB", IIC_VecFP,
703                      [(set v2f64:$XT, (sint_to_fp v2i64:$XB))]>;
704  def XVCVSXDSP : XX2Form<60, 440,
705                      (outs vsrc:$XT), (ins vsrc:$XB),
706                      "xvcvsxdsp $XT, $XB", IIC_VecFP,
707                      [(set v4f32:$XT, (int_ppc_vsx_xvcvsxdsp v2i64:$XB))]>;
708  def XVCVSXWDP : XX2Form<60, 248,
709                      (outs vsrc:$XT), (ins vsrc:$XB),
710                      "xvcvsxwdp $XT, $XB", IIC_VecFP,
711                      [(set v2f64:$XT, (int_ppc_vsx_xvcvsxwdp v4i32:$XB))]>;
712  def XVCVSXWSP : XX2Form<60, 184,
713                      (outs vsrc:$XT), (ins vsrc:$XB),
714                      "xvcvsxwsp $XT, $XB", IIC_VecFP,
715                      [(set v4f32:$XT, (sint_to_fp v4i32:$XB))]>;
716  def XVCVUXDDP : XX2Form<60, 488,
717                      (outs vsrc:$XT), (ins vsrc:$XB),
718                      "xvcvuxddp $XT, $XB", IIC_VecFP,
719                      [(set v2f64:$XT, (uint_to_fp v2i64:$XB))]>;
720  def XVCVUXDSP : XX2Form<60, 424,
721                      (outs vsrc:$XT), (ins vsrc:$XB),
722                      "xvcvuxdsp $XT, $XB", IIC_VecFP,
723                      [(set v4f32:$XT, (int_ppc_vsx_xvcvuxdsp v2i64:$XB))]>;
724  def XVCVUXWDP : XX2Form<60, 232,
725                      (outs vsrc:$XT), (ins vsrc:$XB),
726                      "xvcvuxwdp $XT, $XB", IIC_VecFP,
727                      [(set v2f64:$XT, (int_ppc_vsx_xvcvuxwdp v4i32:$XB))]>;
728  def XVCVUXWSP : XX2Form<60, 168,
729                      (outs vsrc:$XT), (ins vsrc:$XB),
730                      "xvcvuxwsp $XT, $XB", IIC_VecFP,
731                      [(set v4f32:$XT, (uint_to_fp v4i32:$XB))]>;
732
733  // Rounding Instructions
734  def XSRDPI : XX2Form<60, 73,
735                      (outs vsfrc:$XT), (ins vsfrc:$XB),
736                      "xsrdpi $XT, $XB", IIC_VecFP,
737                      [(set f64:$XT, (fround f64:$XB))]>;
738  def XSRDPIC : XX2Form<60, 107,
739                      (outs vsfrc:$XT), (ins vsfrc:$XB),
740                      "xsrdpic $XT, $XB", IIC_VecFP,
741                      [(set f64:$XT, (fnearbyint f64:$XB))]>;
742  def XSRDPIM : XX2Form<60, 121,
743                      (outs vsfrc:$XT), (ins vsfrc:$XB),
744                      "xsrdpim $XT, $XB", IIC_VecFP,
745                      [(set f64:$XT, (ffloor f64:$XB))]>;
746  def XSRDPIP : XX2Form<60, 105,
747                      (outs vsfrc:$XT), (ins vsfrc:$XB),
748                      "xsrdpip $XT, $XB", IIC_VecFP,
749                      [(set f64:$XT, (fceil f64:$XB))]>;
750  def XSRDPIZ : XX2Form<60, 89,
751                      (outs vsfrc:$XT), (ins vsfrc:$XB),
752                      "xsrdpiz $XT, $XB", IIC_VecFP,
753                      [(set f64:$XT, (ftrunc f64:$XB))]>;
754
755  def XVRDPI : XX2Form<60, 201,
756                      (outs vsrc:$XT), (ins vsrc:$XB),
757                      "xvrdpi $XT, $XB", IIC_VecFP,
758                      [(set v2f64:$XT, (fround v2f64:$XB))]>;
759  def XVRDPIC : XX2Form<60, 235,
760                      (outs vsrc:$XT), (ins vsrc:$XB),
761                      "xvrdpic $XT, $XB", IIC_VecFP,
762                      [(set v2f64:$XT, (fnearbyint v2f64:$XB))]>;
763  def XVRDPIM : XX2Form<60, 249,
764                      (outs vsrc:$XT), (ins vsrc:$XB),
765                      "xvrdpim $XT, $XB", IIC_VecFP,
766                      [(set v2f64:$XT, (ffloor v2f64:$XB))]>;
767  def XVRDPIP : XX2Form<60, 233,
768                      (outs vsrc:$XT), (ins vsrc:$XB),
769                      "xvrdpip $XT, $XB", IIC_VecFP,
770                      [(set v2f64:$XT, (fceil v2f64:$XB))]>;
771  def XVRDPIZ : XX2Form<60, 217,
772                      (outs vsrc:$XT), (ins vsrc:$XB),
773                      "xvrdpiz $XT, $XB", IIC_VecFP,
774                      [(set v2f64:$XT, (ftrunc v2f64:$XB))]>;
775
776  def XVRSPI : XX2Form<60, 137,
777                      (outs vsrc:$XT), (ins vsrc:$XB),
778                      "xvrspi $XT, $XB", IIC_VecFP,
779                      [(set v4f32:$XT, (fround v4f32:$XB))]>;
780  def XVRSPIC : XX2Form<60, 171,
781                      (outs vsrc:$XT), (ins vsrc:$XB),
782                      "xvrspic $XT, $XB", IIC_VecFP,
783                      [(set v4f32:$XT, (fnearbyint v4f32:$XB))]>;
784  def XVRSPIM : XX2Form<60, 185,
785                      (outs vsrc:$XT), (ins vsrc:$XB),
786                      "xvrspim $XT, $XB", IIC_VecFP,
787                      [(set v4f32:$XT, (ffloor v4f32:$XB))]>;
788  def XVRSPIP : XX2Form<60, 169,
789                      (outs vsrc:$XT), (ins vsrc:$XB),
790                      "xvrspip $XT, $XB", IIC_VecFP,
791                      [(set v4f32:$XT, (fceil v4f32:$XB))]>;
792  def XVRSPIZ : XX2Form<60, 153,
793                      (outs vsrc:$XT), (ins vsrc:$XB),
794                      "xvrspiz $XT, $XB", IIC_VecFP,
795                      [(set v4f32:$XT, (ftrunc v4f32:$XB))]>;
796
797  // Max/Min Instructions
798  let isCommutable = 1 in {
799  def XSMAXDP : XX3Form<60, 160,
800                        (outs vsfrc:$XT), (ins vsfrc:$XA, vsfrc:$XB),
801                        "xsmaxdp $XT, $XA, $XB", IIC_VecFP,
802                        [(set vsfrc:$XT,
803                              (int_ppc_vsx_xsmaxdp vsfrc:$XA, vsfrc:$XB))]>;
804  def XSMINDP : XX3Form<60, 168,
805                        (outs vsfrc:$XT), (ins vsfrc:$XA, vsfrc:$XB),
806                        "xsmindp $XT, $XA, $XB", IIC_VecFP,
807                        [(set vsfrc:$XT,
808                              (int_ppc_vsx_xsmindp vsfrc:$XA, vsfrc:$XB))]>;
809
810  def XVMAXDP : XX3Form<60, 224,
811                        (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
812                        "xvmaxdp $XT, $XA, $XB", IIC_VecFP,
813                        [(set vsrc:$XT,
814                              (int_ppc_vsx_xvmaxdp vsrc:$XA, vsrc:$XB))]>;
815  def XVMINDP : XX3Form<60, 232,
816                        (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
817                        "xvmindp $XT, $XA, $XB", IIC_VecFP,
818                        [(set vsrc:$XT,
819                              (int_ppc_vsx_xvmindp vsrc:$XA, vsrc:$XB))]>;
820
821  def XVMAXSP : XX3Form<60, 192,
822                        (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
823                        "xvmaxsp $XT, $XA, $XB", IIC_VecFP,
824                        [(set vsrc:$XT,
825                              (int_ppc_vsx_xvmaxsp vsrc:$XA, vsrc:$XB))]>;
826  def XVMINSP : XX3Form<60, 200,
827                        (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
828                        "xvminsp $XT, $XA, $XB", IIC_VecFP,
829                        [(set vsrc:$XT,
830                              (int_ppc_vsx_xvminsp vsrc:$XA, vsrc:$XB))]>;
831  } // isCommutable
832} // Uses = [RM]
833
834  // Logical Instructions
835  let isCommutable = 1 in
836  def XXLAND : XX3Form<60, 130,
837                       (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
838                       "xxland $XT, $XA, $XB", IIC_VecGeneral,
839                       [(set v4i32:$XT, (and v4i32:$XA, v4i32:$XB))]>;
840  def XXLANDC : XX3Form<60, 138,
841                        (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
842                        "xxlandc $XT, $XA, $XB", IIC_VecGeneral,
843                        [(set v4i32:$XT, (and v4i32:$XA,
844                                              (vnot_ppc v4i32:$XB)))]>;
845  let isCommutable = 1 in {
846  def XXLNOR : XX3Form<60, 162,
847                       (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
848                       "xxlnor $XT, $XA, $XB", IIC_VecGeneral,
849                       [(set v4i32:$XT, (vnot_ppc (or v4i32:$XA,
850                                                   v4i32:$XB)))]>;
851  def XXLOR : XX3Form<60, 146,
852                      (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
853                      "xxlor $XT, $XA, $XB", IIC_VecGeneral,
854                      [(set v4i32:$XT, (or v4i32:$XA, v4i32:$XB))]>;
855  let isCodeGenOnly = 1 in
856  def XXLORf: XX3Form<60, 146,
857                      (outs vsfrc:$XT), (ins vsfrc:$XA, vsfrc:$XB),
858                      "xxlor $XT, $XA, $XB", IIC_VecGeneral, []>;
859  def XXLXOR : XX3Form<60, 154,
860                       (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
861                       "xxlxor $XT, $XA, $XB", IIC_VecGeneral,
862                       [(set v4i32:$XT, (xor v4i32:$XA, v4i32:$XB))]>;
863  } // isCommutable
864
865  let isCodeGenOnly = 1, isMoveImm = 1, isAsCheapAsAMove = 1,
866      isReMaterializable = 1 in {
867    def XXLXORz : XX3Form_SameOp<60, 154, (outs vsrc:$XT), (ins),
868                       "xxlxor $XT, $XT, $XT", IIC_VecGeneral,
869                       [(set v4i32:$XT, (v4i32 immAllZerosV))]>;
870    def XXLXORdpz : XX3Form_SameOp<60, 154,
871                         (outs vsfrc:$XT), (ins),
872                         "xxlxor $XT, $XT, $XT", IIC_VecGeneral,
873                         [(set f64:$XT, (fpimm0))]>;
874    def XXLXORspz : XX3Form_SameOp<60, 154,
875                         (outs vssrc:$XT), (ins),
876                         "xxlxor $XT, $XT, $XT", IIC_VecGeneral,
877                         [(set f32:$XT, (fpimm0))]>;
878  }
879
880  // Permutation Instructions
881  def XXMRGHW : XX3Form<60, 18,
882                       (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
883                       "xxmrghw $XT, $XA, $XB", IIC_VecPerm, []>;
884  def XXMRGLW : XX3Form<60, 50,
885                       (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
886                       "xxmrglw $XT, $XA, $XB", IIC_VecPerm, []>;
887
888  def XXPERMDI : XX3Form_2<60, 10,
889                       (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB, u2imm:$DM),
890                       "xxpermdi $XT, $XA, $XB, $DM", IIC_VecPerm,
891                       [(set v2i64:$XT, (PPCxxpermdi v2i64:$XA, v2i64:$XB,
892                         imm32SExt16:$DM))]>;
893  let isCodeGenOnly = 1 in
894  def XXPERMDIs : XX3Form_2s<60, 10, (outs vsrc:$XT), (ins vsfrc:$XA, u2imm:$DM),
895                             "xxpermdi $XT, $XA, $XA, $DM", IIC_VecPerm, []>;
896  def XXSEL : XX4Form<60, 3,
897                      (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB, vsrc:$XC),
898                      "xxsel $XT, $XA, $XB, $XC", IIC_VecPerm, []>;
899
900  def XXSLDWI : XX3Form_2<60, 2,
901                       (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB, u2imm:$SHW),
902                       "xxsldwi $XT, $XA, $XB, $SHW", IIC_VecPerm,
903                       [(set v4i32:$XT, (PPCvecshl v4i32:$XA, v4i32:$XB,
904                                                  imm32SExt16:$SHW))]>;
905
906  let isCodeGenOnly = 1 in
907  def XXSLDWIs : XX3Form_2s<60, 2,
908                       (outs vsrc:$XT), (ins vsfrc:$XA, u2imm:$SHW),
909                       "xxsldwi $XT, $XA, $XA, $SHW", IIC_VecPerm, []>;
910
911  def XXSPLTW : XX2Form_2<60, 164,
912                       (outs vsrc:$XT), (ins vsrc:$XB, u2imm:$UIM),
913                       "xxspltw $XT, $XB, $UIM", IIC_VecPerm,
914                       [(set v4i32:$XT,
915                             (PPCxxsplt v4i32:$XB, imm32SExt16:$UIM))]>;
916  let isCodeGenOnly = 1 in
917  def XXSPLTWs : XX2Form_2<60, 164,
918                       (outs vsrc:$XT), (ins vsfrc:$XB, u2imm:$UIM),
919                       "xxspltw $XT, $XB, $UIM", IIC_VecPerm, []>;
920
921} // hasSideEffects
922
923// SELECT_CC_* - Used to implement the SELECT_CC DAG operation.  Expanded after
924// instruction selection into a branch sequence.
925let PPC970_Single = 1 in {
926
927  def SELECT_CC_VSRC: PPCCustomInserterPseudo<(outs vsrc:$dst),
928                             (ins crrc:$cond, vsrc:$T, vsrc:$F, i32imm:$BROPC),
929                             "#SELECT_CC_VSRC",
930                             []>;
931  def SELECT_VSRC: PPCCustomInserterPseudo<(outs vsrc:$dst),
932                          (ins crbitrc:$cond, vsrc:$T, vsrc:$F),
933                          "#SELECT_VSRC",
934                          [(set v2f64:$dst,
935                                (select i1:$cond, v2f64:$T, v2f64:$F))]>;
936  def SELECT_CC_VSFRC: PPCCustomInserterPseudo<(outs f8rc:$dst),
937                              (ins crrc:$cond, f8rc:$T, f8rc:$F,
938                               i32imm:$BROPC), "#SELECT_CC_VSFRC",
939                              []>;
940  def SELECT_VSFRC: PPCCustomInserterPseudo<(outs f8rc:$dst),
941                           (ins crbitrc:$cond, f8rc:$T, f8rc:$F),
942                           "#SELECT_VSFRC",
943                           [(set f64:$dst,
944                                 (select i1:$cond, f64:$T, f64:$F))]>;
945  def SELECT_CC_VSSRC: PPCCustomInserterPseudo<(outs f4rc:$dst),
946                              (ins crrc:$cond, f4rc:$T, f4rc:$F,
947                               i32imm:$BROPC), "#SELECT_CC_VSSRC",
948                              []>;
949  def SELECT_VSSRC: PPCCustomInserterPseudo<(outs f4rc:$dst),
950                           (ins crbitrc:$cond, f4rc:$T, f4rc:$F),
951                           "#SELECT_VSSRC",
952                           [(set f32:$dst,
953                                 (select i1:$cond, f32:$T, f32:$F))]>;
954}
955} // AddedComplexity
956
957def : InstAlias<"xvmovdp $XT, $XB",
958                (XVCPSGNDP vsrc:$XT, vsrc:$XB, vsrc:$XB)>;
959def : InstAlias<"xvmovsp $XT, $XB",
960                (XVCPSGNSP vsrc:$XT, vsrc:$XB, vsrc:$XB)>;
961
962def : InstAlias<"xxspltd $XT, $XB, 0",
963                (XXPERMDI vsrc:$XT, vsrc:$XB, vsrc:$XB, 0)>;
964def : InstAlias<"xxspltd $XT, $XB, 1",
965                (XXPERMDI vsrc:$XT, vsrc:$XB, vsrc:$XB, 3)>;
966def : InstAlias<"xxmrghd $XT, $XA, $XB",
967                (XXPERMDI vsrc:$XT, vsrc:$XA, vsrc:$XB, 0)>;
968def : InstAlias<"xxmrgld $XT, $XA, $XB",
969                (XXPERMDI vsrc:$XT, vsrc:$XA, vsrc:$XB, 3)>;
970def : InstAlias<"xxswapd $XT, $XB",
971                (XXPERMDI vsrc:$XT, vsrc:$XB, vsrc:$XB, 2)>;
972def : InstAlias<"xxspltd $XT, $XB, 0",
973                (XXPERMDIs vsrc:$XT, vsfrc:$XB, 0)>;
974def : InstAlias<"xxspltd $XT, $XB, 1",
975                (XXPERMDIs vsrc:$XT, vsfrc:$XB, 3)>;
976def : InstAlias<"xxswapd $XT, $XB",
977                (XXPERMDIs vsrc:$XT, vsfrc:$XB, 2)>;
978
979let AddedComplexity = 400 in { // Prefer VSX patterns over non-VSX patterns.
980
981def : Pat<(v4i32 (vnot_ppc v4i32:$A)),
982          (v4i32 (XXLNOR $A, $A))>;
983def : Pat<(v4i32 (or (and (vnot_ppc v4i32:$C), v4i32:$A),
984                     (and v4i32:$B, v4i32:$C))),
985          (v4i32 (XXSEL $A, $B, $C))>;
986
987let Predicates = [IsBigEndian] in {
988def : Pat<(v2f64 (scalar_to_vector f64:$A)),
989          (v2f64 (SUBREG_TO_REG (i64 1), $A, sub_64))>;
990
991def : Pat<(f64 (extractelt v2f64:$S, 0)),
992          (f64 (EXTRACT_SUBREG $S, sub_64))>;
993def : Pat<(f64 (extractelt v2f64:$S, 1)),
994          (f64 (EXTRACT_SUBREG (XXPERMDI $S, $S, 2), sub_64))>;
995}
996
997let Predicates = [IsLittleEndian] in {
998def : Pat<(v2f64 (scalar_to_vector f64:$A)),
999          (v2f64 (XXPERMDI (SUBREG_TO_REG (i64 1), $A, sub_64),
1000                           (SUBREG_TO_REG (i64 1), $A, sub_64), 0))>;
1001
1002def : Pat<(f64 (extractelt v2f64:$S, 0)),
1003          (f64 (EXTRACT_SUBREG (XXPERMDI $S, $S, 2), sub_64))>;
1004def : Pat<(f64 (extractelt v2f64:$S, 1)),
1005          (f64 (EXTRACT_SUBREG $S, sub_64))>;
1006}
1007
1008// Additional fnmsub patterns: -a*c + b == -(a*c - b)
1009def : Pat<(fma (fneg f64:$A), f64:$C, f64:$B),
1010          (XSNMSUBADP $B, $C, $A)>;
1011def : Pat<(fma f64:$A, (fneg f64:$C), f64:$B),
1012          (XSNMSUBADP $B, $C, $A)>;
1013
1014def : Pat<(fma (fneg v2f64:$A), v2f64:$C, v2f64:$B),
1015          (XVNMSUBADP $B, $C, $A)>;
1016def : Pat<(fma v2f64:$A, (fneg v2f64:$C), v2f64:$B),
1017          (XVNMSUBADP $B, $C, $A)>;
1018
1019def : Pat<(fma (fneg v4f32:$A), v4f32:$C, v4f32:$B),
1020          (XVNMSUBASP $B, $C, $A)>;
1021def : Pat<(fma v4f32:$A, (fneg v4f32:$C), v4f32:$B),
1022          (XVNMSUBASP $B, $C, $A)>;
1023
1024def : Pat<(v2f64 (bitconvert v4f32:$A)),
1025          (COPY_TO_REGCLASS $A, VSRC)>;
1026def : Pat<(v2f64 (bitconvert v4i32:$A)),
1027          (COPY_TO_REGCLASS $A, VSRC)>;
1028def : Pat<(v2f64 (bitconvert v8i16:$A)),
1029          (COPY_TO_REGCLASS $A, VSRC)>;
1030def : Pat<(v2f64 (bitconvert v16i8:$A)),
1031          (COPY_TO_REGCLASS $A, VSRC)>;
1032
1033def : Pat<(v4f32 (bitconvert v2f64:$A)),
1034          (COPY_TO_REGCLASS $A, VRRC)>;
1035def : Pat<(v4i32 (bitconvert v2f64:$A)),
1036          (COPY_TO_REGCLASS $A, VRRC)>;
1037def : Pat<(v8i16 (bitconvert v2f64:$A)),
1038          (COPY_TO_REGCLASS $A, VRRC)>;
1039def : Pat<(v16i8 (bitconvert v2f64:$A)),
1040          (COPY_TO_REGCLASS $A, VRRC)>;
1041
1042def : Pat<(v2i64 (bitconvert v4f32:$A)),
1043          (COPY_TO_REGCLASS $A, VSRC)>;
1044def : Pat<(v2i64 (bitconvert v4i32:$A)),
1045          (COPY_TO_REGCLASS $A, VSRC)>;
1046def : Pat<(v2i64 (bitconvert v8i16:$A)),
1047          (COPY_TO_REGCLASS $A, VSRC)>;
1048def : Pat<(v2i64 (bitconvert v16i8:$A)),
1049          (COPY_TO_REGCLASS $A, VSRC)>;
1050
1051def : Pat<(v4f32 (bitconvert v2i64:$A)),
1052          (COPY_TO_REGCLASS $A, VRRC)>;
1053def : Pat<(v4i32 (bitconvert v2i64:$A)),
1054          (COPY_TO_REGCLASS $A, VRRC)>;
1055def : Pat<(v8i16 (bitconvert v2i64:$A)),
1056          (COPY_TO_REGCLASS $A, VRRC)>;
1057def : Pat<(v16i8 (bitconvert v2i64:$A)),
1058          (COPY_TO_REGCLASS $A, VRRC)>;
1059
1060def : Pat<(v2f64 (bitconvert v2i64:$A)),
1061          (COPY_TO_REGCLASS $A, VRRC)>;
1062def : Pat<(v2i64 (bitconvert v2f64:$A)),
1063          (COPY_TO_REGCLASS $A, VRRC)>;
1064
1065def : Pat<(v2f64 (bitconvert v1i128:$A)),
1066          (COPY_TO_REGCLASS $A, VRRC)>;
1067def : Pat<(v1i128 (bitconvert v2f64:$A)),
1068          (COPY_TO_REGCLASS $A, VRRC)>;
1069
1070def : Pat<(v2i64 (bitconvert f128:$A)),
1071          (COPY_TO_REGCLASS $A, VRRC)>;
1072def : Pat<(v4i32 (bitconvert f128:$A)),
1073          (COPY_TO_REGCLASS $A, VRRC)>;
1074def : Pat<(v8i16 (bitconvert f128:$A)),
1075          (COPY_TO_REGCLASS $A, VRRC)>;
1076def : Pat<(v16i8 (bitconvert f128:$A)),
1077          (COPY_TO_REGCLASS $A, VRRC)>;
1078
1079def : Pat<(v2f64 (PPCsvec2fp v4i32:$C, 0)),
1080          (v2f64 (XVCVSXWDP (v2i64 (XXMRGHW $C, $C))))>;
1081def : Pat<(v2f64 (PPCsvec2fp v4i32:$C, 1)),
1082          (v2f64 (XVCVSXWDP (v2i64 (XXMRGLW $C, $C))))>;
1083
1084def : Pat<(v2f64 (PPCuvec2fp v4i32:$C, 0)),
1085          (v2f64 (XVCVUXWDP (v2i64 (XXMRGHW $C, $C))))>;
1086def : Pat<(v2f64 (PPCuvec2fp v4i32:$C, 1)),
1087          (v2f64 (XVCVUXWDP (v2i64 (XXMRGLW $C, $C))))>;
1088
1089def : Pat<(v2f64 (PPCfpextlh v4f32:$C)), (XVCVSPDP (XXMRGHW $C, $C))>;
1090
1091// Loads.
1092let Predicates = [HasVSX, HasOnlySwappingMemOps] in {
1093  def : Pat<(v2f64 (PPClxvd2x xoaddr:$src)), (LXVD2X xoaddr:$src)>;
1094
1095  // Stores.
1096  def : Pat<(int_ppc_vsx_stxvd2x v2f64:$rS, xoaddr:$dst),
1097            (STXVD2X $rS, xoaddr:$dst)>;
1098  def : Pat<(PPCstxvd2x v2f64:$rS, xoaddr:$dst), (STXVD2X $rS, xoaddr:$dst)>;
1099}
1100
1101// Load vector big endian order
1102let Predicates = [IsLittleEndian, HasVSX] in {
1103  def : Pat<(v2f64 (PPCld_vec_be xoaddr:$src)), (LXVD2X xoaddr:$src)>;
1104  def : Pat<(PPCst_vec_be v2f64:$rS, xoaddr:$dst), (STXVD2X $rS, xoaddr:$dst)>;
1105  def : Pat<(v4f32 (PPCld_vec_be xoaddr:$src)), (LXVW4X xoaddr:$src)>;
1106  def : Pat<(PPCst_vec_be v4f32:$rS, xoaddr:$dst), (STXVW4X $rS, xoaddr:$dst)>;
1107  def : Pat<(v2i64 (PPCld_vec_be xoaddr:$src)), (LXVD2X xoaddr:$src)>;
1108  def : Pat<(PPCst_vec_be v2i64:$rS, xoaddr:$dst), (STXVD2X $rS, xoaddr:$dst)>;
1109  def : Pat<(v4i32 (PPCld_vec_be xoaddr:$src)), (LXVW4X xoaddr:$src)>;
1110  def : Pat<(PPCst_vec_be v4i32:$rS, xoaddr:$dst), (STXVW4X $rS, xoaddr:$dst)>;
1111}
1112
1113let Predicates = [IsBigEndian, HasVSX, HasOnlySwappingMemOps] in {
1114  def : Pat<(v2f64 (load xoaddr:$src)), (LXVD2X xoaddr:$src)>;
1115  def : Pat<(v2i64 (load xoaddr:$src)), (LXVD2X xoaddr:$src)>;
1116  def : Pat<(v4i32 (load xoaddr:$src)), (LXVW4X xoaddr:$src)>;
1117  def : Pat<(v4i32 (int_ppc_vsx_lxvw4x xoaddr:$src)), (LXVW4X xoaddr:$src)>;
1118  def : Pat<(store v2f64:$rS, xoaddr:$dst), (STXVD2X $rS, xoaddr:$dst)>;
1119  def : Pat<(store v2i64:$rS, xoaddr:$dst), (STXVD2X $rS, xoaddr:$dst)>;
1120  def : Pat<(store v4i32:$XT, xoaddr:$dst), (STXVW4X $XT, xoaddr:$dst)>;
1121  def : Pat<(int_ppc_vsx_stxvw4x v4i32:$rS, xoaddr:$dst),
1122            (STXVW4X $rS, xoaddr:$dst)>;
1123}
1124
1125// Permutes.
1126def : Pat<(v2f64 (PPCxxswapd v2f64:$src)), (XXPERMDI $src, $src, 2)>;
1127def : Pat<(v2i64 (PPCxxswapd v2i64:$src)), (XXPERMDI $src, $src, 2)>;
1128def : Pat<(v4f32 (PPCxxswapd v4f32:$src)), (XXPERMDI $src, $src, 2)>;
1129def : Pat<(v4i32 (PPCxxswapd v4i32:$src)), (XXPERMDI $src, $src, 2)>;
1130def : Pat<(v2f64 (PPCswapNoChain v2f64:$src)), (XXPERMDI $src, $src, 2)>;
1131
1132// PPCvecshl XT, XA, XA, 2 can be selected to both XXSLDWI XT,XA,XA,2 and
1133// XXSWAPD XT,XA (i.e. XXPERMDI XT,XA,XA,2), the later one is more profitable.
1134def : Pat<(v4i32 (PPCvecshl v4i32:$src, v4i32:$src, 2)), (XXPERMDI $src, $src, 2)>;
1135
1136// Selects.
1137def : Pat<(v2f64 (selectcc i1:$lhs, i1:$rhs, v2f64:$tval, v2f64:$fval, SETLT)),
1138          (SELECT_VSRC (CRANDC $lhs, $rhs), $tval, $fval)>;
1139def : Pat<(v2f64 (selectcc i1:$lhs, i1:$rhs, v2f64:$tval, v2f64:$fval, SETULT)),
1140          (SELECT_VSRC (CRANDC $rhs, $lhs), $tval, $fval)>;
1141def : Pat<(v2f64 (selectcc i1:$lhs, i1:$rhs, v2f64:$tval, v2f64:$fval, SETLE)),
1142          (SELECT_VSRC (CRORC  $lhs, $rhs), $tval, $fval)>;
1143def : Pat<(v2f64 (selectcc i1:$lhs, i1:$rhs, v2f64:$tval, v2f64:$fval, SETULE)),
1144          (SELECT_VSRC (CRORC  $rhs, $lhs), $tval, $fval)>;
1145def : Pat<(v2f64 (selectcc i1:$lhs, i1:$rhs, v2f64:$tval, v2f64:$fval, SETEQ)),
1146          (SELECT_VSRC (CREQV $lhs, $rhs), $tval, $fval)>;
1147def : Pat<(v2f64 (selectcc i1:$lhs, i1:$rhs, v2f64:$tval, v2f64:$fval, SETGE)),
1148          (SELECT_VSRC (CRORC  $rhs, $lhs), $tval, $fval)>;
1149def : Pat<(v2f64 (selectcc i1:$lhs, i1:$rhs, v2f64:$tval, v2f64:$fval, SETUGE)),
1150          (SELECT_VSRC (CRORC  $lhs, $rhs), $tval, $fval)>;
1151def : Pat<(v2f64 (selectcc i1:$lhs, i1:$rhs, v2f64:$tval, v2f64:$fval, SETGT)),
1152          (SELECT_VSRC (CRANDC $rhs, $lhs), $tval, $fval)>;
1153def : Pat<(v2f64 (selectcc i1:$lhs, i1:$rhs, v2f64:$tval, v2f64:$fval, SETUGT)),
1154          (SELECT_VSRC (CRANDC $lhs, $rhs), $tval, $fval)>;
1155def : Pat<(v2f64 (selectcc i1:$lhs, i1:$rhs, v2f64:$tval, v2f64:$fval, SETNE)),
1156          (SELECT_VSRC (CRXOR $lhs, $rhs), $tval, $fval)>;
1157
1158def : Pat<(f64 (selectcc i1:$lhs, i1:$rhs, f64:$tval, f64:$fval, SETLT)),
1159          (SELECT_VSFRC (CRANDC $lhs, $rhs), $tval, $fval)>;
1160def : Pat<(f64 (selectcc i1:$lhs, i1:$rhs, f64:$tval, f64:$fval, SETULT)),
1161          (SELECT_VSFRC (CRANDC $rhs, $lhs), $tval, $fval)>;
1162def : Pat<(f64 (selectcc i1:$lhs, i1:$rhs, f64:$tval, f64:$fval, SETLE)),
1163          (SELECT_VSFRC (CRORC  $lhs, $rhs), $tval, $fval)>;
1164def : Pat<(f64 (selectcc i1:$lhs, i1:$rhs, f64:$tval, f64:$fval, SETULE)),
1165          (SELECT_VSFRC (CRORC  $rhs, $lhs), $tval, $fval)>;
1166def : Pat<(f64 (selectcc i1:$lhs, i1:$rhs, f64:$tval, f64:$fval, SETEQ)),
1167          (SELECT_VSFRC (CREQV $lhs, $rhs), $tval, $fval)>;
1168def : Pat<(f64 (selectcc i1:$lhs, i1:$rhs, f64:$tval, f64:$fval, SETGE)),
1169          (SELECT_VSFRC (CRORC  $rhs, $lhs), $tval, $fval)>;
1170def : Pat<(f64 (selectcc i1:$lhs, i1:$rhs, f64:$tval, f64:$fval, SETUGE)),
1171          (SELECT_VSFRC (CRORC  $lhs, $rhs), $tval, $fval)>;
1172def : Pat<(f64 (selectcc i1:$lhs, i1:$rhs, f64:$tval, f64:$fval, SETGT)),
1173          (SELECT_VSFRC (CRANDC $rhs, $lhs), $tval, $fval)>;
1174def : Pat<(f64 (selectcc i1:$lhs, i1:$rhs, f64:$tval, f64:$fval, SETUGT)),
1175          (SELECT_VSFRC (CRANDC $lhs, $rhs), $tval, $fval)>;
1176def : Pat<(f64 (selectcc i1:$lhs, i1:$rhs, f64:$tval, f64:$fval, SETNE)),
1177          (SELECT_VSFRC (CRXOR $lhs, $rhs), $tval, $fval)>;
1178
1179// Divides.
1180def : Pat<(int_ppc_vsx_xvdivsp v4f32:$A, v4f32:$B),
1181          (XVDIVSP $A, $B)>;
1182def : Pat<(int_ppc_vsx_xvdivdp v2f64:$A, v2f64:$B),
1183          (XVDIVDP $A, $B)>;
1184
1185// Reciprocal estimate
1186def : Pat<(int_ppc_vsx_xvresp v4f32:$A),
1187          (XVRESP $A)>;
1188def : Pat<(int_ppc_vsx_xvredp v2f64:$A),
1189          (XVREDP $A)>;
1190
1191// Recip. square root estimate
1192def : Pat<(int_ppc_vsx_xvrsqrtesp v4f32:$A),
1193          (XVRSQRTESP $A)>;
1194def : Pat<(int_ppc_vsx_xvrsqrtedp v2f64:$A),
1195          (XVRSQRTEDP $A)>;
1196
1197// Vector selection
1198def : Pat<(v16i8 (vselect v16i8:$vA, v16i8:$vB, v16i8:$vC)),
1199          (COPY_TO_REGCLASS
1200                 (XXSEL (COPY_TO_REGCLASS $vC, VSRC),
1201                        (COPY_TO_REGCLASS $vB, VSRC),
1202                        (COPY_TO_REGCLASS $vA, VSRC)), VRRC)>;
1203def : Pat<(v8i16 (vselect v8i16:$vA, v8i16:$vB, v8i16:$vC)),
1204          (COPY_TO_REGCLASS
1205                 (XXSEL (COPY_TO_REGCLASS $vC, VSRC),
1206                        (COPY_TO_REGCLASS $vB, VSRC),
1207                        (COPY_TO_REGCLASS $vA, VSRC)), VRRC)>;
1208def : Pat<(vselect v4i32:$vA, v4i32:$vB, v4i32:$vC),
1209          (XXSEL $vC, $vB, $vA)>;
1210def : Pat<(vselect v2i64:$vA, v2i64:$vB, v2i64:$vC),
1211          (XXSEL $vC, $vB, $vA)>;
1212def : Pat<(vselect v4i32:$vA, v4f32:$vB, v4f32:$vC),
1213          (XXSEL $vC, $vB, $vA)>;
1214def : Pat<(vselect v2i64:$vA, v2f64:$vB, v2f64:$vC),
1215          (XXSEL $vC, $vB, $vA)>;
1216
1217def : Pat<(v4f32 (fmaxnum v4f32:$src1, v4f32:$src2)),
1218          (v4f32 (XVMAXSP $src1, $src2))>;
1219def : Pat<(v4f32 (fminnum v4f32:$src1, v4f32:$src2)),
1220          (v4f32 (XVMINSP $src1, $src2))>;
1221def : Pat<(v2f64 (fmaxnum v2f64:$src1, v2f64:$src2)),
1222          (v2f64 (XVMAXDP $src1, $src2))>;
1223def : Pat<(v2f64 (fminnum v2f64:$src1, v2f64:$src2)),
1224          (v2f64 (XVMINDP $src1, $src2))>;
1225
1226let Predicates = [IsLittleEndian] in {
1227def : Pat<(f64 (PPCfcfid (PPCmtvsra (i64 (vector_extract v2i64:$S, 0))))),
1228          (f64 (XSCVSXDDP (COPY_TO_REGCLASS (XXPERMDI $S, $S, 2), VSFRC)))>;
1229def : Pat<(f64 (PPCfcfid (PPCmtvsra (i64 (vector_extract v2i64:$S, 1))))),
1230          (f64 (XSCVSXDDP (COPY_TO_REGCLASS (f64 (COPY_TO_REGCLASS $S, VSRC)), VSFRC)))>;
1231def : Pat<(f64 (PPCfcfidu (PPCmtvsra (i64 (vector_extract v2i64:$S, 0))))),
1232          (f64 (XSCVUXDDP (COPY_TO_REGCLASS (XXPERMDI $S, $S, 2), VSFRC)))>;
1233def : Pat<(f64 (PPCfcfidu (PPCmtvsra (i64 (vector_extract v2i64:$S, 1))))),
1234          (f64 (XSCVUXDDP (COPY_TO_REGCLASS (f64 (COPY_TO_REGCLASS $S, VSRC)), VSFRC)))>;
1235} // IsLittleEndian
1236
1237let Predicates = [IsBigEndian] in {
1238def : Pat<(f64 (PPCfcfid (PPCmtvsra (i64 (vector_extract v2i64:$S, 0))))),
1239          (f64 (XSCVSXDDP (COPY_TO_REGCLASS $S, VSFRC)))>;
1240def : Pat<(f64 (PPCfcfid (PPCmtvsra (i64 (vector_extract v2i64:$S, 1))))),
1241          (f64 (XSCVSXDDP (COPY_TO_REGCLASS (XXPERMDI $S, $S, 2), VSFRC)))>;
1242def : Pat<(f64 (PPCfcfidu (PPCmtvsra (i64 (vector_extract v2i64:$S, 0))))),
1243          (f64 (XSCVUXDDP (COPY_TO_REGCLASS $S, VSFRC)))>;
1244def : Pat<(f64 (PPCfcfidu (PPCmtvsra (i64 (vector_extract v2i64:$S, 1))))),
1245          (f64 (XSCVUXDDP (COPY_TO_REGCLASS (XXPERMDI $S, $S, 2), VSFRC)))>;
1246} // IsBigEndian
1247
1248} // AddedComplexity
1249} // HasVSX
1250
1251def ScalarLoads {
1252  dag Li8 =       (i32 (extloadi8 xoaddr:$src));
1253  dag ZELi8 =     (i32 (zextloadi8 xoaddr:$src));
1254  dag ZELi8i64 =  (i64 (zextloadi8 xoaddr:$src));
1255  dag SELi8 =     (i32 (sext_inreg (extloadi8 xoaddr:$src), i8));
1256  dag SELi8i64 =  (i64 (sext_inreg (extloadi8 xoaddr:$src), i8));
1257
1258  dag Li16 =      (i32 (extloadi16 xoaddr:$src));
1259  dag ZELi16 =    (i32 (zextloadi16 xoaddr:$src));
1260  dag ZELi16i64 = (i64 (zextloadi16 xoaddr:$src));
1261  dag SELi16 =    (i32 (sextloadi16 xoaddr:$src));
1262  dag SELi16i64 = (i64 (sextloadi16 xoaddr:$src));
1263
1264  dag Li32 = (i32 (load xoaddr:$src));
1265}
1266
1267def DWToSPExtractConv {
1268  dag El0US1 = (f32 (PPCfcfidus
1269                    (f64 (PPCmtvsra (i64 (vector_extract v2i64:$S1, 0))))));
1270  dag El1US1 = (f32 (PPCfcfidus
1271                    (f64 (PPCmtvsra (i64 (vector_extract v2i64:$S1, 1))))));
1272  dag El0US2 = (f32 (PPCfcfidus
1273                    (f64 (PPCmtvsra (i64 (vector_extract v2i64:$S2, 0))))));
1274  dag El1US2 = (f32 (PPCfcfidus
1275                    (f64 (PPCmtvsra (i64 (vector_extract v2i64:$S2, 1))))));
1276  dag El0SS1 = (f32 (PPCfcfids
1277                    (f64 (PPCmtvsra (i64 (vector_extract v2i64:$S1, 0))))));
1278  dag El1SS1 = (f32 (PPCfcfids
1279                    (f64 (PPCmtvsra (i64 (vector_extract v2i64:$S1, 1))))));
1280  dag El0SS2 = (f32 (PPCfcfids
1281                    (f64 (PPCmtvsra (i64 (vector_extract v2i64:$S2, 0))))));
1282  dag El1SS2 = (f32 (PPCfcfids
1283                    (f64 (PPCmtvsra (i64 (vector_extract v2i64:$S2, 1))))));
1284  dag BVU = (v4f32 (build_vector El0US1, El1US1, El0US2, El1US2));
1285  dag BVS = (v4f32 (build_vector El0SS1, El1SS1, El0SS2, El1SS2));
1286}
1287
1288// The following VSX instructions were introduced in Power ISA 2.07
1289/* FIXME: if the operands are v2i64, these patterns will not match.
1290   we should define new patterns or otherwise match the same patterns
1291   when the elements are larger than i32.
1292*/
1293def HasP8Vector : Predicate<"PPCSubTarget->hasP8Vector()">;
1294def HasDirectMove : Predicate<"PPCSubTarget->hasDirectMove()">;
1295def NoP9Vector : Predicate<"!PPCSubTarget->hasP9Vector()">;
1296let Predicates = [HasP8Vector] in {
1297let AddedComplexity = 400 in { // Prefer VSX patterns over non-VSX patterns.
1298  let isCommutable = 1 in {
1299    def XXLEQV : XX3Form<60, 186,
1300                         (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
1301                         "xxleqv $XT, $XA, $XB", IIC_VecGeneral,
1302                         [(set v4i32:$XT, (vnot_ppc (xor v4i32:$XA, v4i32:$XB)))]>;
1303    def XXLNAND : XX3Form<60, 178,
1304                          (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
1305                          "xxlnand $XT, $XA, $XB", IIC_VecGeneral,
1306                          [(set v4i32:$XT, (vnot_ppc (and v4i32:$XA,
1307                                                    v4i32:$XB)))]>;
1308  } // isCommutable
1309
1310  def : Pat<(int_ppc_vsx_xxleqv v4i32:$A, v4i32:$B),
1311            (XXLEQV $A, $B)>;
1312
1313  let isCodeGenOnly = 1, isMoveImm = 1, isAsCheapAsAMove = 1,
1314      isReMaterializable = 1 in {
1315    def XXLEQVOnes : XX3Form_SameOp<60, 186, (outs vsrc:$XT), (ins),
1316                         "xxleqv $XT, $XT, $XT", IIC_VecGeneral,
1317                         [(set v4i32:$XT, (bitconvert (v16i8 immAllOnesV)))]>;
1318  }
1319
1320  def XXLORC : XX3Form<60, 170,
1321                       (outs vsrc:$XT), (ins vsrc:$XA, vsrc:$XB),
1322                       "xxlorc $XT, $XA, $XB", IIC_VecGeneral,
1323                       [(set v4i32:$XT, (or v4i32:$XA, (vnot_ppc v4i32:$XB)))]>;
1324
1325  // VSX scalar loads introduced in ISA 2.07
1326  let mayLoad = 1, mayStore = 0 in {
1327    let CodeSize = 3 in
1328    def LXSSPX : XX1Form_memOp<31, 524, (outs vssrc:$XT), (ins memrr:$src),
1329                         "lxsspx $XT, $src", IIC_LdStLFD, []>;
1330    def LXSIWAX : XX1Form_memOp<31, 76, (outs vsfrc:$XT), (ins memrr:$src),
1331                          "lxsiwax $XT, $src", IIC_LdStLFD, []>;
1332    def LXSIWZX : XX1Form_memOp<31, 12, (outs vsfrc:$XT), (ins memrr:$src),
1333                          "lxsiwzx $XT, $src", IIC_LdStLFD, []>;
1334
1335    // Pseudo instruction XFLOADf32 will be expanded to LXSSPX or LFSX later
1336    let CodeSize = 3 in
1337    def XFLOADf32  : PseudoXFormMemOp<(outs vssrc:$XT), (ins memrr:$src),
1338                            "#XFLOADf32",
1339                            [(set f32:$XT, (load xoaddr:$src))]>;
1340    // Pseudo instruction LIWAX will be expanded to LXSIWAX or LFIWAX later
1341    def LIWAX : PseudoXFormMemOp<(outs vsfrc:$XT), (ins memrr:$src),
1342                       "#LIWAX",
1343                       [(set f64:$XT, (PPClfiwax xoaddr:$src))]>;
1344    // Pseudo instruction LIWZX will be expanded to LXSIWZX or LFIWZX later
1345    def LIWZX : PseudoXFormMemOp<(outs vsfrc:$XT), (ins memrr:$src),
1346                       "#LIWZX",
1347                       [(set f64:$XT, (PPClfiwzx xoaddr:$src))]>;
1348  } // mayLoad
1349
1350  // VSX scalar stores introduced in ISA 2.07
1351  let mayStore = 1, mayLoad = 0 in {
1352    let CodeSize = 3 in
1353    def STXSSPX : XX1Form_memOp<31, 652, (outs), (ins vssrc:$XT, memrr:$dst),
1354                          "stxsspx $XT, $dst", IIC_LdStSTFD, []>;
1355    def STXSIWX : XX1Form_memOp<31, 140, (outs), (ins vsfrc:$XT, memrr:$dst),
1356                          "stxsiwx $XT, $dst", IIC_LdStSTFD, []>;
1357
1358    // Pseudo instruction XFSTOREf32 will be expanded to STXSSPX or STFSX later
1359    let CodeSize = 3 in
1360    def XFSTOREf32 : PseudoXFormMemOp<(outs), (ins vssrc:$XT, memrr:$dst),
1361                            "#XFSTOREf32",
1362                            [(store f32:$XT, xoaddr:$dst)]>;
1363    // Pseudo instruction STIWX will be expanded to STXSIWX or STFIWX later
1364    def STIWX : PseudoXFormMemOp<(outs), (ins vsfrc:$XT, memrr:$dst),
1365                       "#STIWX",
1366                      [(PPCstfiwx f64:$XT, xoaddr:$dst)]>;
1367  } // mayStore
1368
1369  def : Pat<(f64 (extloadf32 xoaddr:$src)),
1370            (COPY_TO_REGCLASS (XFLOADf32 xoaddr:$src), VSFRC)>;
1371  def : Pat<(f32 (fpround (f64 (extloadf32 xoaddr:$src)))),
1372            (f32 (XFLOADf32 xoaddr:$src))>;
1373  def : Pat<(f64 (fpextend f32:$src)),
1374            (COPY_TO_REGCLASS $src, VSFRC)>;
1375
1376  def : Pat<(f32 (selectcc i1:$lhs, i1:$rhs, f32:$tval, f32:$fval, SETLT)),
1377            (SELECT_VSSRC (CRANDC $lhs, $rhs), $tval, $fval)>;
1378  def : Pat<(f32 (selectcc i1:$lhs, i1:$rhs, f32:$tval, f32:$fval, SETULT)),
1379            (SELECT_VSSRC (CRANDC $rhs, $lhs), $tval, $fval)>;
1380  def : Pat<(f32 (selectcc i1:$lhs, i1:$rhs, f32:$tval, f32:$fval, SETLE)),
1381            (SELECT_VSSRC (CRORC  $lhs, $rhs), $tval, $fval)>;
1382  def : Pat<(f32 (selectcc i1:$lhs, i1:$rhs, f32:$tval, f32:$fval, SETULE)),
1383            (SELECT_VSSRC (CRORC  $rhs, $lhs), $tval, $fval)>;
1384  def : Pat<(f32 (selectcc i1:$lhs, i1:$rhs, f32:$tval, f32:$fval, SETEQ)),
1385            (SELECT_VSSRC (CREQV $lhs, $rhs), $tval, $fval)>;
1386  def : Pat<(f32 (selectcc i1:$lhs, i1:$rhs, f32:$tval, f32:$fval, SETGE)),
1387            (SELECT_VSSRC (CRORC  $rhs, $lhs), $tval, $fval)>;
1388  def : Pat<(f32 (selectcc i1:$lhs, i1:$rhs, f32:$tval, f32:$fval, SETUGE)),
1389            (SELECT_VSSRC (CRORC  $lhs, $rhs), $tval, $fval)>;
1390  def : Pat<(f32 (selectcc i1:$lhs, i1:$rhs, f32:$tval, f32:$fval, SETGT)),
1391            (SELECT_VSSRC (CRANDC $rhs, $lhs), $tval, $fval)>;
1392  def : Pat<(f32 (selectcc i1:$lhs, i1:$rhs, f32:$tval, f32:$fval, SETUGT)),
1393            (SELECT_VSSRC (CRANDC $lhs, $rhs), $tval, $fval)>;
1394  def : Pat<(f32 (selectcc i1:$lhs, i1:$rhs, f32:$tval, f32:$fval, SETNE)),
1395            (SELECT_VSSRC (CRXOR $lhs, $rhs), $tval, $fval)>;
1396
1397  // VSX Elementary Scalar FP arithmetic (SP)
1398  let isCommutable = 1 in {
1399    def XSADDSP : XX3Form<60, 0,
1400                          (outs vssrc:$XT), (ins vssrc:$XA, vssrc:$XB),
1401                          "xsaddsp $XT, $XA, $XB", IIC_VecFP,
1402                          [(set f32:$XT, (fadd f32:$XA, f32:$XB))]>;
1403    def XSMULSP : XX3Form<60, 16,
1404                          (outs vssrc:$XT), (ins vssrc:$XA, vssrc:$XB),
1405                          "xsmulsp $XT, $XA, $XB", IIC_VecFP,
1406                          [(set f32:$XT, (fmul f32:$XA, f32:$XB))]>;
1407  } // isCommutable
1408  def XSSUBSP : XX3Form<60, 8,
1409                        (outs vssrc:$XT), (ins vssrc:$XA, vssrc:$XB),
1410                        "xssubsp $XT, $XA, $XB", IIC_VecFP,
1411                        [(set f32:$XT, (fsub f32:$XA, f32:$XB))]>;
1412  def XSDIVSP : XX3Form<60, 24,
1413                        (outs vssrc:$XT), (ins vssrc:$XA, vssrc:$XB),
1414                        "xsdivsp $XT, $XA, $XB", IIC_FPDivS,
1415                        [(set f32:$XT, (fdiv f32:$XA, f32:$XB))]>;
1416  def XSRESP : XX2Form<60, 26,
1417                        (outs vssrc:$XT), (ins vssrc:$XB),
1418                        "xsresp $XT, $XB", IIC_VecFP,
1419                        [(set f32:$XT, (PPCfre f32:$XB))]>;
1420  def XSRSP : XX2Form<60, 281,
1421                        (outs vssrc:$XT), (ins vsfrc:$XB),
1422                        "xsrsp $XT, $XB", IIC_VecFP, []>;
1423  def XSSQRTSP : XX2Form<60, 11,
1424                        (outs vssrc:$XT), (ins vssrc:$XB),
1425                        "xssqrtsp $XT, $XB", IIC_FPSqrtS,
1426                        [(set f32:$XT, (fsqrt f32:$XB))]>;
1427  def XSRSQRTESP : XX2Form<60, 10,
1428                           (outs vssrc:$XT), (ins vssrc:$XB),
1429                           "xsrsqrtesp $XT, $XB", IIC_VecFP,
1430                           [(set f32:$XT, (PPCfrsqrte f32:$XB))]>;
1431
1432  // FMA Instructions
1433  let BaseName = "XSMADDASP" in {
1434  let isCommutable = 1 in
1435  def XSMADDASP : XX3Form<60, 1,
1436                          (outs vssrc:$XT),
1437                          (ins vssrc:$XTi, vssrc:$XA, vssrc:$XB),
1438                          "xsmaddasp $XT, $XA, $XB", IIC_VecFP,
1439                          [(set f32:$XT, (fma f32:$XA, f32:$XB, f32:$XTi))]>,
1440                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
1441                          AltVSXFMARel;
1442  let IsVSXFMAAlt = 1 in
1443  def XSMADDMSP : XX3Form<60, 9,
1444                          (outs vssrc:$XT),
1445                          (ins vssrc:$XTi, vssrc:$XA, vssrc:$XB),
1446                          "xsmaddmsp $XT, $XA, $XB", IIC_VecFP, []>,
1447                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
1448                          AltVSXFMARel;
1449  }
1450
1451  let BaseName = "XSMSUBASP" in {
1452  let isCommutable = 1 in
1453  def XSMSUBASP : XX3Form<60, 17,
1454                          (outs vssrc:$XT),
1455                          (ins vssrc:$XTi, vssrc:$XA, vssrc:$XB),
1456                          "xsmsubasp $XT, $XA, $XB", IIC_VecFP,
1457                          [(set f32:$XT, (fma f32:$XA, f32:$XB,
1458                                              (fneg f32:$XTi)))]>,
1459                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
1460                          AltVSXFMARel;
1461  let IsVSXFMAAlt = 1 in
1462  def XSMSUBMSP : XX3Form<60, 25,
1463                          (outs vssrc:$XT),
1464                          (ins vssrc:$XTi, vssrc:$XA, vssrc:$XB),
1465                          "xsmsubmsp $XT, $XA, $XB", IIC_VecFP, []>,
1466                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
1467                          AltVSXFMARel;
1468  }
1469
1470  let BaseName = "XSNMADDASP" in {
1471  let isCommutable = 1 in
1472  def XSNMADDASP : XX3Form<60, 129,
1473                          (outs vssrc:$XT),
1474                          (ins vssrc:$XTi, vssrc:$XA, vssrc:$XB),
1475                          "xsnmaddasp $XT, $XA, $XB", IIC_VecFP,
1476                          [(set f32:$XT, (fneg (fma f32:$XA, f32:$XB,
1477                                                    f32:$XTi)))]>,
1478                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
1479                          AltVSXFMARel;
1480  let IsVSXFMAAlt = 1 in
1481  def XSNMADDMSP : XX3Form<60, 137,
1482                          (outs vssrc:$XT),
1483                          (ins vssrc:$XTi, vssrc:$XA, vssrc:$XB),
1484                          "xsnmaddmsp $XT, $XA, $XB", IIC_VecFP, []>,
1485                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
1486                          AltVSXFMARel;
1487  }
1488
1489  let BaseName = "XSNMSUBASP" in {
1490  let isCommutable = 1 in
1491  def XSNMSUBASP : XX3Form<60, 145,
1492                          (outs vssrc:$XT),
1493                          (ins vssrc:$XTi, vssrc:$XA, vssrc:$XB),
1494                          "xsnmsubasp $XT, $XA, $XB", IIC_VecFP,
1495                          [(set f32:$XT, (fneg (fma f32:$XA, f32:$XB,
1496                                                    (fneg f32:$XTi))))]>,
1497                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
1498                          AltVSXFMARel;
1499  let IsVSXFMAAlt = 1 in
1500  def XSNMSUBMSP : XX3Form<60, 153,
1501                          (outs vssrc:$XT),
1502                          (ins vssrc:$XTi, vssrc:$XA, vssrc:$XB),
1503                          "xsnmsubmsp $XT, $XA, $XB", IIC_VecFP, []>,
1504                          RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">,
1505                          AltVSXFMARel;
1506  }
1507
1508  // Single Precision Conversions (FP <-> INT)
1509  def XSCVSXDSP : XX2Form<60, 312,
1510                      (outs vssrc:$XT), (ins vsfrc:$XB),
1511                      "xscvsxdsp $XT, $XB", IIC_VecFP,
1512                      [(set f32:$XT, (PPCfcfids f64:$XB))]>;
1513  def XSCVUXDSP : XX2Form<60, 296,
1514                      (outs vssrc:$XT), (ins vsfrc:$XB),
1515                      "xscvuxdsp $XT, $XB", IIC_VecFP,
1516                      [(set f32:$XT, (PPCfcfidus f64:$XB))]>;
1517
1518  // Conversions between vector and scalar single precision
1519  def XSCVDPSPN : XX2Form<60, 267, (outs vsrc:$XT), (ins vssrc:$XB),
1520                          "xscvdpspn $XT, $XB", IIC_VecFP, []>;
1521  def XSCVSPDPN : XX2Form<60, 331, (outs vssrc:$XT), (ins vsrc:$XB),
1522                          "xscvspdpn $XT, $XB", IIC_VecFP, []>;
1523
1524  let Predicates = [IsLittleEndian] in {
1525  def : Pat<DWToSPExtractConv.El0SS1,
1526            (f32 (XSCVSXDSP (COPY_TO_REGCLASS (XXPERMDI $S1, $S1, 2), VSFRC)))>;
1527  def : Pat<DWToSPExtractConv.El1SS1,
1528            (f32 (XSCVSXDSP (COPY_TO_REGCLASS
1529                              (f64 (COPY_TO_REGCLASS $S1, VSRC)), VSFRC)))>;
1530  def : Pat<DWToSPExtractConv.El0US1,
1531            (f32 (XSCVUXDSP (COPY_TO_REGCLASS (XXPERMDI $S1, $S1, 2), VSFRC)))>;
1532  def : Pat<DWToSPExtractConv.El1US1,
1533            (f32 (XSCVUXDSP (COPY_TO_REGCLASS
1534                              (f64 (COPY_TO_REGCLASS $S1, VSRC)), VSFRC)))>;
1535  }
1536
1537  let Predicates = [IsBigEndian] in {
1538  def : Pat<DWToSPExtractConv.El0SS1,
1539            (f32 (XSCVSXDSP (COPY_TO_REGCLASS $S1, VSFRC)))>;
1540  def : Pat<DWToSPExtractConv.El1SS1,
1541            (f32 (XSCVSXDSP (COPY_TO_REGCLASS (XXPERMDI $S1, $S1, 2), VSFRC)))>;
1542  def : Pat<DWToSPExtractConv.El0US1,
1543            (f32 (XSCVUXDSP (COPY_TO_REGCLASS $S1, VSFRC)))>;
1544  def : Pat<DWToSPExtractConv.El1US1,
1545            (f32 (XSCVUXDSP (COPY_TO_REGCLASS (XXPERMDI $S1, $S1, 2), VSFRC)))>;
1546  }
1547
1548  // Instructions for converting float to i64 feeding a store.
1549  let Predicates = [NoP9Vector] in {
1550  def : Pat<(PPCstore_scal_int_from_vsr
1551              (f64 (PPCcv_fp_to_sint_in_vsr f64:$src)), xoaddr:$dst, 8),
1552            (STXSDX (XSCVDPSXDS f64:$src), xoaddr:$dst)>;
1553  def : Pat<(PPCstore_scal_int_from_vsr
1554              (f64 (PPCcv_fp_to_uint_in_vsr f64:$src)), xoaddr:$dst, 8),
1555            (STXSDX (XSCVDPUXDS f64:$src), xoaddr:$dst)>;
1556  }
1557
1558  // Instructions for converting float to i32 feeding a store.
1559  def : Pat<(PPCstore_scal_int_from_vsr
1560              (f64 (PPCcv_fp_to_sint_in_vsr f64:$src)), xoaddr:$dst, 4),
1561            (STIWX (XSCVDPSXWS f64:$src), xoaddr:$dst)>;
1562  def : Pat<(PPCstore_scal_int_from_vsr
1563              (f64 (PPCcv_fp_to_uint_in_vsr f64:$src)), xoaddr:$dst, 4),
1564            (STIWX (XSCVDPUXWS f64:$src), xoaddr:$dst)>;
1565
1566  def : Pat<(v2i64 (smax v2i64:$src1, v2i64:$src2)),
1567            (v2i64 (VMAXSD (COPY_TO_REGCLASS $src1, VRRC),
1568                           (COPY_TO_REGCLASS $src2, VRRC)))>;
1569  def : Pat<(v2i64 (umax v2i64:$src1, v2i64:$src2)),
1570            (v2i64 (VMAXUD (COPY_TO_REGCLASS $src1, VRRC),
1571                           (COPY_TO_REGCLASS $src2, VRRC)))>;
1572  def : Pat<(v2i64 (smin v2i64:$src1, v2i64:$src2)),
1573            (v2i64 (VMINSD (COPY_TO_REGCLASS $src1, VRRC),
1574                           (COPY_TO_REGCLASS $src2, VRRC)))>;
1575  def : Pat<(v2i64 (umin v2i64:$src1, v2i64:$src2)),
1576            (v2i64 (VMINUD (COPY_TO_REGCLASS $src1, VRRC),
1577                           (COPY_TO_REGCLASS $src2, VRRC)))>;
1578} // AddedComplexity = 400
1579} // HasP8Vector
1580
1581let AddedComplexity = 400 in {
1582let Predicates = [HasDirectMove] in {
1583  // VSX direct move instructions
1584  def MFVSRD : XX1_RS6_RD5_XO<31, 51, (outs g8rc:$rA), (ins vsfrc:$XT),
1585                              "mfvsrd $rA, $XT", IIC_VecGeneral,
1586                              [(set i64:$rA, (PPCmfvsr f64:$XT))]>,
1587      Requires<[In64BitMode]>;
1588  let isCodeGenOnly = 1 in
1589  def MFVRD : XX1_RS6_RD5_XO<31, 51, (outs g8rc:$rA), (ins vsrc:$XT),
1590                             "mfvsrd $rA, $XT", IIC_VecGeneral,
1591                             []>,
1592      Requires<[In64BitMode]>;
1593  def MFVSRWZ : XX1_RS6_RD5_XO<31, 115, (outs gprc:$rA), (ins vsfrc:$XT),
1594                               "mfvsrwz $rA, $XT", IIC_VecGeneral,
1595                               [(set i32:$rA, (PPCmfvsr f64:$XT))]>;
1596  def MTVSRD : XX1_RS6_RD5_XO<31, 179, (outs vsfrc:$XT), (ins g8rc:$rA),
1597                              "mtvsrd $XT, $rA", IIC_VecGeneral,
1598                              [(set f64:$XT, (PPCmtvsra i64:$rA))]>,
1599      Requires<[In64BitMode]>;
1600  def MTVSRWA : XX1_RS6_RD5_XO<31, 211, (outs vsfrc:$XT), (ins gprc:$rA),
1601                               "mtvsrwa $XT, $rA", IIC_VecGeneral,
1602                               [(set f64:$XT, (PPCmtvsra i32:$rA))]>;
1603  def MTVSRWZ : XX1_RS6_RD5_XO<31, 243, (outs vsfrc:$XT), (ins gprc:$rA),
1604                               "mtvsrwz $XT, $rA", IIC_VecGeneral,
1605                               [(set f64:$XT, (PPCmtvsrz i32:$rA))]>;
1606} // HasDirectMove
1607
1608let Predicates = [IsISA3_0, HasDirectMove] in {
1609  def MTVSRWS: XX1_RS6_RD5_XO<31, 403, (outs vsrc:$XT), (ins gprc:$rA),
1610                              "mtvsrws $XT, $rA", IIC_VecGeneral, []>;
1611
1612  def MTVSRDD: XX1Form<31, 435, (outs vsrc:$XT), (ins g8rc_nox0:$rA, g8rc:$rB),
1613                       "mtvsrdd $XT, $rA, $rB", IIC_VecGeneral,
1614                       []>, Requires<[In64BitMode]>;
1615
1616  def MFVSRLD: XX1_RS6_RD5_XO<31, 307, (outs g8rc:$rA), (ins vsrc:$XT),
1617                              "mfvsrld $rA, $XT", IIC_VecGeneral,
1618                              []>, Requires<[In64BitMode]>;
1619
1620} // IsISA3_0, HasDirectMove
1621} // AddedComplexity = 400
1622
1623// We want to parse this from asm, but we don't want to emit this as it would
1624// be emitted with a VSX reg. So leave Emit = 0 here.
1625def : InstAlias<"mfvrd $rA, $XT",
1626                (MFVRD g8rc:$rA, vrrc:$XT), 0>;
1627def : InstAlias<"mffprd $rA, $src",
1628                (MFVSRD g8rc:$rA, f8rc:$src)>;
1629
1630/*  Direct moves of various widths from GPR's into VSR's. Each move lines
1631    the value up into element 0 (both BE and LE). Namely, entities smaller than
1632    a doubleword are shifted left and moved for BE. For LE, they're moved, then
1633    swapped to go into the least significant element of the VSR.
1634*/
1635def MovesToVSR {
1636  dag BE_BYTE_0 =
1637    (MTVSRD
1638      (RLDICR
1639        (INSERT_SUBREG (i64 (IMPLICIT_DEF)), $A, sub_32), 56, 7));
1640  dag BE_HALF_0 =
1641    (MTVSRD
1642      (RLDICR
1643        (INSERT_SUBREG (i64 (IMPLICIT_DEF)), $A, sub_32), 48, 15));
1644  dag BE_WORD_0 =
1645    (MTVSRD
1646      (RLDICR
1647        (INSERT_SUBREG (i64 (IMPLICIT_DEF)), $A, sub_32), 32, 31));
1648  dag BE_DWORD_0 = (MTVSRD $A);
1649
1650  dag LE_MTVSRW = (MTVSRD (INSERT_SUBREG (i64 (IMPLICIT_DEF)), $A, sub_32));
1651  dag LE_WORD_1 = (v2i64 (INSERT_SUBREG (v2i64 (IMPLICIT_DEF)),
1652                                        LE_MTVSRW, sub_64));
1653  dag LE_WORD_0 = (XXPERMDI LE_WORD_1, LE_WORD_1, 2);
1654  dag LE_DWORD_1 = (v2i64 (INSERT_SUBREG (v2i64 (IMPLICIT_DEF)),
1655                                         BE_DWORD_0, sub_64));
1656  dag LE_DWORD_0 = (XXPERMDI LE_DWORD_1, LE_DWORD_1, 2);
1657}
1658
1659/*  Patterns for extracting elements out of vectors. Integer elements are
1660    extracted using direct move operations. Patterns for extracting elements
1661    whose indices are not available at compile time are also provided with
1662    various _VARIABLE_ patterns.
1663    The numbering for the DAG's is for LE, but when used on BE, the correct
1664    LE element can just be used (i.e. LE_BYTE_2 == BE_BYTE_13).
1665*/
1666def VectorExtractions {
1667  // Doubleword extraction
1668  dag LE_DWORD_0 =
1669    (MFVSRD
1670      (EXTRACT_SUBREG
1671        (XXPERMDI (COPY_TO_REGCLASS $S, VSRC),
1672                  (COPY_TO_REGCLASS $S, VSRC), 2), sub_64));
1673  dag LE_DWORD_1 = (MFVSRD
1674                     (EXTRACT_SUBREG
1675                       (v2i64 (COPY_TO_REGCLASS $S, VSRC)), sub_64));
1676
1677  // Word extraction
1678  dag LE_WORD_0 = (MFVSRWZ (EXTRACT_SUBREG (XXPERMDI $S, $S, 2), sub_64));
1679  dag LE_WORD_1 = (MFVSRWZ (EXTRACT_SUBREG (XXSLDWI $S, $S, 1), sub_64));
1680  dag LE_WORD_2 = (MFVSRWZ (EXTRACT_SUBREG
1681                             (v2i64 (COPY_TO_REGCLASS $S, VSRC)), sub_64));
1682  dag LE_WORD_3 = (MFVSRWZ (EXTRACT_SUBREG (XXSLDWI $S, $S, 3), sub_64));
1683
1684  // Halfword extraction
1685  dag LE_HALF_0 = (i32 (EXTRACT_SUBREG (RLDICL LE_DWORD_0, 0, 48), sub_32));
1686  dag LE_HALF_1 = (i32 (EXTRACT_SUBREG (RLDICL LE_DWORD_0, 48, 48), sub_32));
1687  dag LE_HALF_2 = (i32 (EXTRACT_SUBREG (RLDICL LE_DWORD_0, 32, 48), sub_32));
1688  dag LE_HALF_3 = (i32 (EXTRACT_SUBREG (RLDICL LE_DWORD_0, 16, 48), sub_32));
1689  dag LE_HALF_4 = (i32 (EXTRACT_SUBREG (RLDICL LE_DWORD_1, 0, 48), sub_32));
1690  dag LE_HALF_5 = (i32 (EXTRACT_SUBREG (RLDICL LE_DWORD_1, 48, 48), sub_32));
1691  dag LE_HALF_6 = (i32 (EXTRACT_SUBREG (RLDICL LE_DWORD_1, 32, 48), sub_32));
1692  dag LE_HALF_7 = (i32 (EXTRACT_SUBREG (RLDICL LE_DWORD_1, 16, 48), sub_32));
1693
1694  // Byte extraction
1695  dag LE_BYTE_0 = (i32 (EXTRACT_SUBREG (RLDICL LE_DWORD_0, 0, 56), sub_32));
1696  dag LE_BYTE_1 = (i32 (EXTRACT_SUBREG (RLDICL LE_DWORD_0, 56, 56), sub_32));
1697  dag LE_BYTE_2 = (i32 (EXTRACT_SUBREG (RLDICL LE_DWORD_0, 48, 56), sub_32));
1698  dag LE_BYTE_3 = (i32 (EXTRACT_SUBREG (RLDICL LE_DWORD_0, 40, 56), sub_32));
1699  dag LE_BYTE_4 = (i32 (EXTRACT_SUBREG (RLDICL LE_DWORD_0, 32, 56), sub_32));
1700  dag LE_BYTE_5 = (i32 (EXTRACT_SUBREG (RLDICL LE_DWORD_0, 24, 56), sub_32));
1701  dag LE_BYTE_6 = (i32 (EXTRACT_SUBREG (RLDICL LE_DWORD_0, 16, 56), sub_32));
1702  dag LE_BYTE_7 = (i32 (EXTRACT_SUBREG (RLDICL LE_DWORD_0, 8, 56), sub_32));
1703  dag LE_BYTE_8 = (i32 (EXTRACT_SUBREG (RLDICL LE_DWORD_1, 0, 56), sub_32));
1704  dag LE_BYTE_9 = (i32 (EXTRACT_SUBREG (RLDICL LE_DWORD_1, 56, 56), sub_32));
1705  dag LE_BYTE_10 = (i32 (EXTRACT_SUBREG (RLDICL LE_DWORD_1, 48, 56), sub_32));
1706  dag LE_BYTE_11 = (i32 (EXTRACT_SUBREG (RLDICL LE_DWORD_1, 40, 56), sub_32));
1707  dag LE_BYTE_12 = (i32 (EXTRACT_SUBREG (RLDICL LE_DWORD_1, 32, 56), sub_32));
1708  dag LE_BYTE_13 = (i32 (EXTRACT_SUBREG (RLDICL LE_DWORD_1, 24, 56), sub_32));
1709  dag LE_BYTE_14 = (i32 (EXTRACT_SUBREG (RLDICL LE_DWORD_1, 16, 56), sub_32));
1710  dag LE_BYTE_15 = (i32 (EXTRACT_SUBREG (RLDICL LE_DWORD_1, 8, 56), sub_32));
1711
1712  /* Variable element number (BE and LE patterns must be specified separately)
1713     This is a rather involved process.
1714
1715     Conceptually, this is how the move is accomplished:
1716     1. Identify which doubleword contains the element
1717     2. Shift in the VMX register so that the correct doubleword is correctly
1718        lined up for the MFVSRD
1719     3. Perform the move so that the element (along with some extra stuff)
1720        is in the GPR
1721     4. Right shift within the GPR so that the element is right-justified
1722
1723     Of course, the index is an element number which has a different meaning
1724     on LE/BE so the patterns have to be specified separately.
1725
1726     Note: The final result will be the element right-justified with high
1727           order bits being arbitrarily defined (namely, whatever was in the
1728           vector register to the left of the value originally).
1729  */
1730
1731  /*  LE variable byte
1732      Number 1. above:
1733      - For elements 0-7, we shift left by 8 bytes since they're on the right
1734      - For elements 8-15, we need not shift (shift left by zero bytes)
1735      This is accomplished by inverting the bits of the index and AND-ing
1736      with 0x8 (i.e. clearing all bits of the index and inverting bit 60).
1737  */
1738  dag LE_VBYTE_PERM_VEC = (v16i8 (LVSL ZERO8, (ANDC8 (LI8 8), $Idx)));
1739
1740  //  Number 2. above:
1741  //  - Now that we set up the shift amount, we shift in the VMX register
1742  dag LE_VBYTE_PERMUTE = (v16i8 (VPERM $S, $S, LE_VBYTE_PERM_VEC));
1743
1744  //  Number 3. above:
1745  //  - The doubleword containing our element is moved to a GPR
1746  dag LE_MV_VBYTE = (MFVSRD
1747                      (EXTRACT_SUBREG
1748                        (v2i64 (COPY_TO_REGCLASS LE_VBYTE_PERMUTE, VSRC)),
1749                        sub_64));
1750
1751  /*  Number 4. above:
1752      - Truncate the element number to the range 0-7 (8-15 are symmetrical
1753        and out of range values are truncated accordingly)
1754      - Multiply by 8 as we need to shift right by the number of bits, not bytes
1755      - Shift right in the GPR by the calculated value
1756  */
1757  dag LE_VBYTE_SHIFT = (EXTRACT_SUBREG (RLDICR (AND8 (LI8 7), $Idx), 3, 60),
1758                                       sub_32);
1759  dag LE_VARIABLE_BYTE = (EXTRACT_SUBREG (SRD LE_MV_VBYTE, LE_VBYTE_SHIFT),
1760                                         sub_32);
1761
1762  /*  LE variable halfword
1763      Number 1. above:
1764      - For elements 0-3, we shift left by 8 since they're on the right
1765      - For elements 4-7, we need not shift (shift left by zero bytes)
1766      Similarly to the byte pattern, we invert the bits of the index, but we
1767      AND with 0x4 (i.e. clear all bits of the index and invert bit 61).
1768      Of course, the shift is still by 8 bytes, so we must multiply by 2.
1769  */
1770  dag LE_VHALF_PERM_VEC =
1771    (v16i8 (LVSL ZERO8, (RLDICR (ANDC8 (LI8 4), $Idx), 1, 62)));
1772
1773  //  Number 2. above:
1774  //  - Now that we set up the shift amount, we shift in the VMX register
1775  dag LE_VHALF_PERMUTE = (v16i8 (VPERM $S, $S, LE_VHALF_PERM_VEC));
1776
1777  //  Number 3. above:
1778  //  - The doubleword containing our element is moved to a GPR
1779  dag LE_MV_VHALF = (MFVSRD
1780                      (EXTRACT_SUBREG
1781                        (v2i64 (COPY_TO_REGCLASS LE_VHALF_PERMUTE, VSRC)),
1782                        sub_64));
1783
1784  /*  Number 4. above:
1785      - Truncate the element number to the range 0-3 (4-7 are symmetrical
1786        and out of range values are truncated accordingly)
1787      - Multiply by 16 as we need to shift right by the number of bits
1788      - Shift right in the GPR by the calculated value
1789  */
1790  dag LE_VHALF_SHIFT = (EXTRACT_SUBREG (RLDICR (AND8 (LI8 3), $Idx), 4, 59),
1791                                       sub_32);
1792  dag LE_VARIABLE_HALF = (EXTRACT_SUBREG (SRD LE_MV_VHALF, LE_VHALF_SHIFT),
1793                                         sub_32);
1794
1795  /*  LE variable word
1796      Number 1. above:
1797      - For elements 0-1, we shift left by 8 since they're on the right
1798      - For elements 2-3, we need not shift
1799  */
1800  dag LE_VWORD_PERM_VEC = (v16i8 (LVSL ZERO8,
1801                                       (RLDICR (ANDC8 (LI8 2), $Idx), 2, 61)));
1802
1803  //  Number 2. above:
1804  //  - Now that we set up the shift amount, we shift in the VMX register
1805  dag LE_VWORD_PERMUTE = (v16i8 (VPERM $S, $S, LE_VWORD_PERM_VEC));
1806
1807  //  Number 3. above:
1808  //  - The doubleword containing our element is moved to a GPR
1809  dag LE_MV_VWORD = (MFVSRD
1810                      (EXTRACT_SUBREG
1811                        (v2i64 (COPY_TO_REGCLASS LE_VWORD_PERMUTE, VSRC)),
1812                        sub_64));
1813
1814  /*  Number 4. above:
1815      - Truncate the element number to the range 0-1 (2-3 are symmetrical
1816        and out of range values are truncated accordingly)
1817      - Multiply by 32 as we need to shift right by the number of bits
1818      - Shift right in the GPR by the calculated value
1819  */
1820  dag LE_VWORD_SHIFT = (EXTRACT_SUBREG (RLDICR (AND8 (LI8 1), $Idx), 5, 58),
1821                                       sub_32);
1822  dag LE_VARIABLE_WORD = (EXTRACT_SUBREG (SRD LE_MV_VWORD, LE_VWORD_SHIFT),
1823                                         sub_32);
1824
1825  /*  LE variable doubleword
1826      Number 1. above:
1827      - For element 0, we shift left by 8 since it's on the right
1828      - For element 1, we need not shift
1829  */
1830  dag LE_VDWORD_PERM_VEC = (v16i8 (LVSL ZERO8,
1831                                        (RLDICR (ANDC8 (LI8 1), $Idx), 3, 60)));
1832
1833  //  Number 2. above:
1834  //  - Now that we set up the shift amount, we shift in the VMX register
1835  dag LE_VDWORD_PERMUTE = (v16i8 (VPERM $S, $S, LE_VDWORD_PERM_VEC));
1836
1837  // Number 3. above:
1838  //  - The doubleword containing our element is moved to a GPR
1839  //  - Number 4. is not needed for the doubleword as the value is 64-bits
1840  dag LE_VARIABLE_DWORD =
1841        (MFVSRD (EXTRACT_SUBREG
1842                  (v2i64 (COPY_TO_REGCLASS LE_VDWORD_PERMUTE, VSRC)),
1843                  sub_64));
1844
1845  /*  LE variable float
1846      - Shift the vector to line up the desired element to BE Word 0
1847      - Convert 32-bit float to a 64-bit single precision float
1848  */
1849  dag LE_VFLOAT_PERM_VEC = (v16i8 (LVSL ZERO8,
1850                                  (RLDICR (XOR8 (LI8 3), $Idx), 2, 61)));
1851  dag LE_VFLOAT_PERMUTE = (VPERM $S, $S, LE_VFLOAT_PERM_VEC);
1852  dag LE_VARIABLE_FLOAT = (XSCVSPDPN LE_VFLOAT_PERMUTE);
1853
1854  /*  LE variable double
1855      Same as the LE doubleword except there is no move.
1856  */
1857  dag LE_VDOUBLE_PERMUTE = (v16i8 (VPERM (v16i8 (COPY_TO_REGCLASS $S, VRRC)),
1858                                         (v16i8 (COPY_TO_REGCLASS $S, VRRC)),
1859                                         LE_VDWORD_PERM_VEC));
1860  dag LE_VARIABLE_DOUBLE = (COPY_TO_REGCLASS LE_VDOUBLE_PERMUTE, VSRC);
1861
1862  /*  BE variable byte
1863      The algorithm here is the same as the LE variable byte except:
1864      - The shift in the VMX register is by 0/8 for opposite element numbers so
1865        we simply AND the element number with 0x8
1866      - The order of elements after the move to GPR is reversed, so we invert
1867        the bits of the index prior to truncating to the range 0-7
1868  */
1869  dag BE_VBYTE_PERM_VEC = (v16i8 (LVSL ZERO8, (ANDIo8 $Idx, 8)));
1870  dag BE_VBYTE_PERMUTE = (v16i8 (VPERM $S, $S, BE_VBYTE_PERM_VEC));
1871  dag BE_MV_VBYTE = (MFVSRD
1872                      (EXTRACT_SUBREG
1873                        (v2i64 (COPY_TO_REGCLASS BE_VBYTE_PERMUTE, VSRC)),
1874                        sub_64));
1875  dag BE_VBYTE_SHIFT = (EXTRACT_SUBREG (RLDICR (ANDC8 (LI8 7), $Idx), 3, 60),
1876                                       sub_32);
1877  dag BE_VARIABLE_BYTE = (EXTRACT_SUBREG (SRD BE_MV_VBYTE, BE_VBYTE_SHIFT),
1878                                         sub_32);
1879
1880  /*  BE variable halfword
1881      The algorithm here is the same as the LE variable halfword except:
1882      - The shift in the VMX register is by 0/8 for opposite element numbers so
1883        we simply AND the element number with 0x4 and multiply by 2
1884      - The order of elements after the move to GPR is reversed, so we invert
1885        the bits of the index prior to truncating to the range 0-3
1886  */
1887  dag BE_VHALF_PERM_VEC = (v16i8 (LVSL ZERO8,
1888                                       (RLDICR (ANDIo8 $Idx, 4), 1, 62)));
1889  dag BE_VHALF_PERMUTE = (v16i8 (VPERM $S, $S, BE_VHALF_PERM_VEC));
1890  dag BE_MV_VHALF = (MFVSRD
1891                      (EXTRACT_SUBREG
1892                        (v2i64 (COPY_TO_REGCLASS BE_VHALF_PERMUTE, VSRC)),
1893                        sub_64));
1894  dag BE_VHALF_SHIFT = (EXTRACT_SUBREG (RLDICR (ANDC8 (LI8 3), $Idx), 4, 59),
1895                                       sub_32);
1896  dag BE_VARIABLE_HALF = (EXTRACT_SUBREG (SRD BE_MV_VHALF, BE_VHALF_SHIFT),
1897                                         sub_32);
1898
1899  /*  BE variable word
1900      The algorithm is the same as the LE variable word except:
1901      - The shift in the VMX register happens for opposite element numbers
1902      - The order of elements after the move to GPR is reversed, so we invert
1903        the bits of the index prior to truncating to the range 0-1
1904  */
1905  dag BE_VWORD_PERM_VEC = (v16i8 (LVSL ZERO8,
1906                                       (RLDICR (ANDIo8 $Idx, 2), 2, 61)));
1907  dag BE_VWORD_PERMUTE = (v16i8 (VPERM $S, $S, BE_VWORD_PERM_VEC));
1908  dag BE_MV_VWORD = (MFVSRD
1909                      (EXTRACT_SUBREG
1910                        (v2i64 (COPY_TO_REGCLASS BE_VWORD_PERMUTE, VSRC)),
1911                        sub_64));
1912  dag BE_VWORD_SHIFT = (EXTRACT_SUBREG (RLDICR (ANDC8 (LI8 1), $Idx), 5, 58),
1913                                       sub_32);
1914  dag BE_VARIABLE_WORD = (EXTRACT_SUBREG (SRD BE_MV_VWORD, BE_VWORD_SHIFT),
1915                                         sub_32);
1916
1917  /*  BE variable doubleword
1918      Same as the LE doubleword except we shift in the VMX register for opposite
1919      element indices.
1920  */
1921  dag BE_VDWORD_PERM_VEC = (v16i8 (LVSL ZERO8,
1922                                        (RLDICR (ANDIo8 $Idx, 1), 3, 60)));
1923  dag BE_VDWORD_PERMUTE = (v16i8 (VPERM $S, $S, BE_VDWORD_PERM_VEC));
1924  dag BE_VARIABLE_DWORD =
1925        (MFVSRD (EXTRACT_SUBREG
1926                  (v2i64 (COPY_TO_REGCLASS BE_VDWORD_PERMUTE, VSRC)),
1927                  sub_64));
1928
1929  /*  BE variable float
1930      - Shift the vector to line up the desired element to BE Word 0
1931      - Convert 32-bit float to a 64-bit single precision float
1932  */
1933  dag BE_VFLOAT_PERM_VEC = (v16i8 (LVSL ZERO8, (RLDICR $Idx, 2, 61)));
1934  dag BE_VFLOAT_PERMUTE = (VPERM $S, $S, BE_VFLOAT_PERM_VEC);
1935  dag BE_VARIABLE_FLOAT = (XSCVSPDPN BE_VFLOAT_PERMUTE);
1936
1937  /* BE variable double
1938      Same as the BE doubleword except there is no move.
1939  */
1940  dag BE_VDOUBLE_PERMUTE = (v16i8 (VPERM (v16i8 (COPY_TO_REGCLASS $S, VRRC)),
1941                                         (v16i8 (COPY_TO_REGCLASS $S, VRRC)),
1942                                         BE_VDWORD_PERM_VEC));
1943  dag BE_VARIABLE_DOUBLE = (COPY_TO_REGCLASS BE_VDOUBLE_PERMUTE, VSRC);
1944}
1945
1946def NoP9Altivec : Predicate<"!PPCSubTarget->hasP9Altivec()">;
1947let AddedComplexity = 400 in {
1948// v4f32 scalar <-> vector conversions (BE)
1949let Predicates = [IsBigEndian, HasP8Vector] in {
1950  def : Pat<(v4f32 (scalar_to_vector f32:$A)),
1951            (v4f32 (XSCVDPSPN $A))>;
1952  def : Pat<(f32 (vector_extract v4f32:$S, 0)),
1953            (f32 (XSCVSPDPN $S))>;
1954  def : Pat<(f32 (vector_extract v4f32:$S, 1)),
1955            (f32 (XSCVSPDPN (XXSLDWI $S, $S, 1)))>;
1956  def : Pat<(f32 (vector_extract v4f32:$S, 2)),
1957            (f32 (XSCVSPDPN (XXPERMDI $S, $S, 2)))>;
1958  def : Pat<(f32 (vector_extract v4f32:$S, 3)),
1959            (f32 (XSCVSPDPN (XXSLDWI $S, $S, 3)))>;
1960  def : Pat<(f32 (vector_extract v4f32:$S, i64:$Idx)),
1961            (f32 VectorExtractions.BE_VARIABLE_FLOAT)>;
1962} // IsBigEndian, HasP8Vector
1963
1964// Variable index vector_extract for v2f64 does not require P8Vector
1965let Predicates = [IsBigEndian, HasVSX] in
1966  def : Pat<(f64 (vector_extract v2f64:$S, i64:$Idx)),
1967            (f64 VectorExtractions.BE_VARIABLE_DOUBLE)>;
1968
1969let Predicates = [IsBigEndian, HasDirectMove] in {
1970  // v16i8 scalar <-> vector conversions (BE)
1971  def : Pat<(v16i8 (scalar_to_vector i32:$A)),
1972            (v16i8 (SUBREG_TO_REG (i64 1), MovesToVSR.BE_BYTE_0, sub_64))>;
1973  def : Pat<(v8i16 (scalar_to_vector i32:$A)),
1974            (v8i16 (SUBREG_TO_REG (i64 1), MovesToVSR.BE_HALF_0, sub_64))>;
1975  def : Pat<(v4i32 (scalar_to_vector i32:$A)),
1976            (v4i32 (SUBREG_TO_REG (i64 1), MovesToVSR.BE_WORD_0, sub_64))>;
1977  def : Pat<(v2i64 (scalar_to_vector i64:$A)),
1978            (v2i64 (SUBREG_TO_REG (i64 1), MovesToVSR.BE_DWORD_0, sub_64))>;
1979
1980  // v2i64 scalar <-> vector conversions (BE)
1981  def : Pat<(i64 (vector_extract v2i64:$S, 0)),
1982            (i64 VectorExtractions.LE_DWORD_1)>;
1983  def : Pat<(i64 (vector_extract v2i64:$S, 1)),
1984            (i64 VectorExtractions.LE_DWORD_0)>;
1985  def : Pat<(i64 (vector_extract v2i64:$S, i64:$Idx)),
1986            (i64 VectorExtractions.BE_VARIABLE_DWORD)>;
1987} // IsBigEndian, HasDirectMove
1988
1989let Predicates = [IsBigEndian, HasDirectMove, NoP9Altivec] in {
1990  def : Pat<(i32 (vector_extract v16i8:$S, 0)),
1991            (i32 VectorExtractions.LE_BYTE_15)>;
1992  def : Pat<(i32 (vector_extract v16i8:$S, 1)),
1993            (i32 VectorExtractions.LE_BYTE_14)>;
1994  def : Pat<(i32 (vector_extract v16i8:$S, 2)),
1995            (i32 VectorExtractions.LE_BYTE_13)>;
1996  def : Pat<(i32 (vector_extract v16i8:$S, 3)),
1997            (i32 VectorExtractions.LE_BYTE_12)>;
1998  def : Pat<(i32 (vector_extract v16i8:$S, 4)),
1999            (i32 VectorExtractions.LE_BYTE_11)>;
2000  def : Pat<(i32 (vector_extract v16i8:$S, 5)),
2001            (i32 VectorExtractions.LE_BYTE_10)>;
2002  def : Pat<(i32 (vector_extract v16i8:$S, 6)),
2003            (i32 VectorExtractions.LE_BYTE_9)>;
2004  def : Pat<(i32 (vector_extract v16i8:$S, 7)),
2005            (i32 VectorExtractions.LE_BYTE_8)>;
2006  def : Pat<(i32 (vector_extract v16i8:$S, 8)),
2007            (i32 VectorExtractions.LE_BYTE_7)>;
2008  def : Pat<(i32 (vector_extract v16i8:$S, 9)),
2009            (i32 VectorExtractions.LE_BYTE_6)>;
2010  def : Pat<(i32 (vector_extract v16i8:$S, 10)),
2011            (i32 VectorExtractions.LE_BYTE_5)>;
2012  def : Pat<(i32 (vector_extract v16i8:$S, 11)),
2013            (i32 VectorExtractions.LE_BYTE_4)>;
2014  def : Pat<(i32 (vector_extract v16i8:$S, 12)),
2015            (i32 VectorExtractions.LE_BYTE_3)>;
2016  def : Pat<(i32 (vector_extract v16i8:$S, 13)),
2017            (i32 VectorExtractions.LE_BYTE_2)>;
2018  def : Pat<(i32 (vector_extract v16i8:$S, 14)),
2019            (i32 VectorExtractions.LE_BYTE_1)>;
2020  def : Pat<(i32 (vector_extract v16i8:$S, 15)),
2021            (i32 VectorExtractions.LE_BYTE_0)>;
2022  def : Pat<(i32 (vector_extract v16i8:$S, i64:$Idx)),
2023            (i32 VectorExtractions.BE_VARIABLE_BYTE)>;
2024
2025  // v8i16 scalar <-> vector conversions (BE)
2026  def : Pat<(i32 (vector_extract v8i16:$S, 0)),
2027            (i32 VectorExtractions.LE_HALF_7)>;
2028  def : Pat<(i32 (vector_extract v8i16:$S, 1)),
2029            (i32 VectorExtractions.LE_HALF_6)>;
2030  def : Pat<(i32 (vector_extract v8i16:$S, 2)),
2031            (i32 VectorExtractions.LE_HALF_5)>;
2032  def : Pat<(i32 (vector_extract v8i16:$S, 3)),
2033            (i32 VectorExtractions.LE_HALF_4)>;
2034  def : Pat<(i32 (vector_extract v8i16:$S, 4)),
2035            (i32 VectorExtractions.LE_HALF_3)>;
2036  def : Pat<(i32 (vector_extract v8i16:$S, 5)),
2037            (i32 VectorExtractions.LE_HALF_2)>;
2038  def : Pat<(i32 (vector_extract v8i16:$S, 6)),
2039            (i32 VectorExtractions.LE_HALF_1)>;
2040  def : Pat<(i32 (vector_extract v8i16:$S, 7)),
2041            (i32 VectorExtractions.LE_HALF_0)>;
2042  def : Pat<(i32 (vector_extract v8i16:$S, i64:$Idx)),
2043            (i32 VectorExtractions.BE_VARIABLE_HALF)>;
2044
2045  // v4i32 scalar <-> vector conversions (BE)
2046  def : Pat<(i32 (vector_extract v4i32:$S, 0)),
2047            (i32 VectorExtractions.LE_WORD_3)>;
2048  def : Pat<(i32 (vector_extract v4i32:$S, 1)),
2049            (i32 VectorExtractions.LE_WORD_2)>;
2050  def : Pat<(i32 (vector_extract v4i32:$S, 2)),
2051            (i32 VectorExtractions.LE_WORD_1)>;
2052  def : Pat<(i32 (vector_extract v4i32:$S, 3)),
2053            (i32 VectorExtractions.LE_WORD_0)>;
2054  def : Pat<(i32 (vector_extract v4i32:$S, i64:$Idx)),
2055            (i32 VectorExtractions.BE_VARIABLE_WORD)>;
2056} // IsBigEndian, HasDirectMove, NoP9Altivec
2057
2058// v4f32 scalar <-> vector conversions (LE)
2059let Predicates = [IsLittleEndian, HasP8Vector] in {
2060  def : Pat<(v4f32 (scalar_to_vector f32:$A)),
2061            (v4f32 (XXSLDWI (XSCVDPSPN $A), (XSCVDPSPN $A), 1))>;
2062  def : Pat<(f32 (vector_extract v4f32:$S, 0)),
2063            (f32 (XSCVSPDPN (XXSLDWI $S, $S, 3)))>;
2064  def : Pat<(f32 (vector_extract v4f32:$S, 1)),
2065            (f32 (XSCVSPDPN (XXPERMDI $S, $S, 2)))>;
2066  def : Pat<(f32 (vector_extract v4f32:$S, 2)),
2067            (f32 (XSCVSPDPN (XXSLDWI $S, $S, 1)))>;
2068  def : Pat<(f32 (vector_extract v4f32:$S, 3)),
2069            (f32 (XSCVSPDPN $S))>;
2070  def : Pat<(f32 (vector_extract v4f32:$S, i64:$Idx)),
2071            (f32 VectorExtractions.LE_VARIABLE_FLOAT)>;
2072} // IsLittleEndian, HasP8Vector
2073
2074// Variable index vector_extract for v2f64 does not require P8Vector
2075let Predicates = [IsLittleEndian, HasVSX] in
2076  def : Pat<(f64 (vector_extract v2f64:$S, i64:$Idx)),
2077            (f64 VectorExtractions.LE_VARIABLE_DOUBLE)>;
2078
2079def : Pat<(int_ppc_vsx_stxvd2x_be v2f64:$rS, xoaddr:$dst),
2080            (STXVD2X $rS, xoaddr:$dst)>;
2081def : Pat<(int_ppc_vsx_stxvw4x_be v4i32:$rS, xoaddr:$dst),
2082            (STXVW4X $rS, xoaddr:$dst)>;
2083def : Pat<(v4i32 (int_ppc_vsx_lxvw4x_be xoaddr:$src)), (LXVW4X xoaddr:$src)>;
2084def : Pat<(v2f64 (int_ppc_vsx_lxvd2x_be xoaddr:$src)), (LXVD2X xoaddr:$src)>;
2085
2086// Variable index unsigned vector_extract on Power9
2087let Predicates = [HasP9Altivec, IsLittleEndian] in {
2088  def : Pat<(i64 (anyext (i32 (vector_extract v16i8:$S, i64:$Idx)))),
2089            (VEXTUBRX $Idx, $S)>;
2090
2091  def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, i64:$Idx)))),
2092            (VEXTUHRX (RLWINM8 $Idx, 1, 28, 30), $S)>;
2093  def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 0)))),
2094            (VEXTUHRX (LI8 0), $S)>;
2095  def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 1)))),
2096            (VEXTUHRX (LI8 2), $S)>;
2097  def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 2)))),
2098            (VEXTUHRX (LI8 4), $S)>;
2099  def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 3)))),
2100            (VEXTUHRX (LI8 6), $S)>;
2101  def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 4)))),
2102            (VEXTUHRX (LI8 8), $S)>;
2103  def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 5)))),
2104            (VEXTUHRX (LI8 10), $S)>;
2105  def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 6)))),
2106            (VEXTUHRX (LI8 12), $S)>;
2107  def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 7)))),
2108            (VEXTUHRX (LI8 14), $S)>;
2109
2110  def : Pat<(i64 (zext (i32 (vector_extract v4i32:$S, i64:$Idx)))),
2111            (VEXTUWRX (RLWINM8 $Idx, 2, 28, 29), $S)>;
2112  def : Pat<(i64 (zext (i32 (vector_extract v4i32:$S, 0)))),
2113            (VEXTUWRX (LI8 0), $S)>;
2114  def : Pat<(i64 (zext (i32 (vector_extract v4i32:$S, 1)))),
2115            (VEXTUWRX (LI8 4), $S)>;
2116  // For extracting LE word 2, MFVSRWZ is better than VEXTUWRX
2117  def : Pat<(i64 (zext (i32 (vector_extract v4i32:$S, 2)))),
2118            (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
2119	    (i32 VectorExtractions.LE_WORD_2), sub_32)>;
2120  def : Pat<(i64 (zext (i32 (vector_extract v4i32:$S, 3)))),
2121            (VEXTUWRX (LI8 12), $S)>;
2122
2123  def : Pat<(i64 (sext (i32 (vector_extract v4i32:$S, i64:$Idx)))),
2124            (EXTSW (VEXTUWRX (RLWINM8 $Idx, 2, 28, 29), $S))>;
2125  def : Pat<(i64 (sext (i32 (vector_extract v4i32:$S, 0)))),
2126            (EXTSW (VEXTUWRX (LI8 0), $S))>;
2127  def : Pat<(i64 (sext (i32 (vector_extract v4i32:$S, 1)))),
2128            (EXTSW (VEXTUWRX (LI8 4), $S))>;
2129  // For extracting LE word 2, MFVSRWZ is better than VEXTUWRX
2130  def : Pat<(i64 (sext (i32 (vector_extract v4i32:$S, 2)))),
2131            (EXTSW (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
2132	    (i32 VectorExtractions.LE_WORD_2), sub_32))>;
2133  def : Pat<(i64 (sext (i32 (vector_extract v4i32:$S, 3)))),
2134            (EXTSW (VEXTUWRX (LI8 12), $S))>;
2135
2136  def : Pat<(i32 (vector_extract v16i8:$S, i64:$Idx)),
2137            (i32 (EXTRACT_SUBREG (VEXTUBRX $Idx, $S), sub_32))>;
2138  def : Pat<(i32 (vector_extract v16i8:$S, 0)),
2139            (i32 (EXTRACT_SUBREG (VEXTUBRX (LI8 0), $S), sub_32))>;
2140  def : Pat<(i32 (vector_extract v16i8:$S, 1)),
2141            (i32 (EXTRACT_SUBREG (VEXTUBRX (LI8 1), $S), sub_32))>;
2142  def : Pat<(i32 (vector_extract v16i8:$S, 2)),
2143            (i32 (EXTRACT_SUBREG (VEXTUBRX (LI8 2), $S), sub_32))>;
2144  def : Pat<(i32 (vector_extract v16i8:$S, 3)),
2145            (i32 (EXTRACT_SUBREG (VEXTUBRX (LI8 3), $S), sub_32))>;
2146  def : Pat<(i32 (vector_extract v16i8:$S, 4)),
2147            (i32 (EXTRACT_SUBREG (VEXTUBRX (LI8 4), $S), sub_32))>;
2148  def : Pat<(i32 (vector_extract v16i8:$S, 5)),
2149            (i32 (EXTRACT_SUBREG (VEXTUBRX (LI8 5), $S), sub_32))>;
2150  def : Pat<(i32 (vector_extract v16i8:$S, 6)),
2151            (i32 (EXTRACT_SUBREG (VEXTUBRX (LI8 6), $S), sub_32))>;
2152  def : Pat<(i32 (vector_extract v16i8:$S, 7)),
2153            (i32 (EXTRACT_SUBREG (VEXTUBRX (LI8 7), $S), sub_32))>;
2154  def : Pat<(i32 (vector_extract v16i8:$S, 8)),
2155            (i32 (EXTRACT_SUBREG (VEXTUBRX (LI8 8), $S), sub_32))>;
2156  def : Pat<(i32 (vector_extract v16i8:$S, 9)),
2157            (i32 (EXTRACT_SUBREG (VEXTUBRX (LI8 9), $S), sub_32))>;
2158  def : Pat<(i32 (vector_extract v16i8:$S, 10)),
2159            (i32 (EXTRACT_SUBREG (VEXTUBRX (LI8 10), $S), sub_32))>;
2160  def : Pat<(i32 (vector_extract v16i8:$S, 11)),
2161            (i32 (EXTRACT_SUBREG (VEXTUBRX (LI8 11), $S), sub_32))>;
2162  def : Pat<(i32 (vector_extract v16i8:$S, 12)),
2163            (i32 (EXTRACT_SUBREG (VEXTUBRX (LI8 12), $S), sub_32))>;
2164  def : Pat<(i32 (vector_extract v16i8:$S, 13)),
2165            (i32 (EXTRACT_SUBREG (VEXTUBRX (LI8 13), $S), sub_32))>;
2166  def : Pat<(i32 (vector_extract v16i8:$S, 14)),
2167            (i32 (EXTRACT_SUBREG (VEXTUBRX (LI8 14), $S), sub_32))>;
2168  def : Pat<(i32 (vector_extract v16i8:$S, 15)),
2169            (i32 (EXTRACT_SUBREG (VEXTUBRX (LI8 15), $S), sub_32))>;
2170
2171  def : Pat<(i32 (vector_extract v8i16:$S, i64:$Idx)),
2172            (i32 (EXTRACT_SUBREG (VEXTUHRX
2173	    (RLWINM8 $Idx, 1, 28, 30), $S), sub_32))>;
2174  def : Pat<(i32 (vector_extract v8i16:$S, 0)),
2175            (i32 (EXTRACT_SUBREG (VEXTUHRX (LI8 0), $S), sub_32))>;
2176  def : Pat<(i32 (vector_extract v8i16:$S, 1)),
2177            (i32 (EXTRACT_SUBREG (VEXTUHRX (LI8 2), $S), sub_32))>;
2178  def : Pat<(i32 (vector_extract v8i16:$S, 2)),
2179            (i32 (EXTRACT_SUBREG (VEXTUHRX (LI8 4), $S), sub_32))>;
2180  def : Pat<(i32 (vector_extract v8i16:$S, 3)),
2181            (i32 (EXTRACT_SUBREG (VEXTUHRX (LI8 6), $S), sub_32))>;
2182  def : Pat<(i32 (vector_extract v8i16:$S, 4)),
2183            (i32 (EXTRACT_SUBREG (VEXTUHRX (LI8 8), $S), sub_32))>;
2184  def : Pat<(i32 (vector_extract v8i16:$S, 5)),
2185            (i32 (EXTRACT_SUBREG (VEXTUHRX (LI8 10), $S), sub_32))>;
2186  def : Pat<(i32 (vector_extract v8i16:$S, 6)),
2187            (i32 (EXTRACT_SUBREG (VEXTUHRX (LI8 12), $S), sub_32))>;
2188  def : Pat<(i32 (vector_extract v8i16:$S, 6)),
2189            (i32 (EXTRACT_SUBREG (VEXTUHRX (LI8 14), $S), sub_32))>;
2190
2191  def : Pat<(i32 (vector_extract v4i32:$S, i64:$Idx)),
2192            (i32 (EXTRACT_SUBREG (VEXTUWRX
2193	    (RLWINM8 $Idx, 2, 28, 29), $S), sub_32))>;
2194  def : Pat<(i32 (vector_extract v4i32:$S, 0)),
2195            (i32 (EXTRACT_SUBREG (VEXTUWRX (LI8 0), $S), sub_32))>;
2196  def : Pat<(i32 (vector_extract v4i32:$S, 1)),
2197            (i32 (EXTRACT_SUBREG (VEXTUWRX (LI8 4), $S), sub_32))>;
2198  // For extracting LE word 2, MFVSRWZ is better than VEXTUWRX
2199  def : Pat<(i32 (vector_extract v4i32:$S, 2)),
2200            (i32 VectorExtractions.LE_WORD_2)>;
2201  def : Pat<(i32 (vector_extract v4i32:$S, 3)),
2202            (i32 (EXTRACT_SUBREG (VEXTUWRX (LI8 12), $S), sub_32))>;
2203}
2204
2205let Predicates = [HasP9Altivec, IsBigEndian] in {
2206  def : Pat<(i64 (anyext (i32 (vector_extract v16i8:$S, i64:$Idx)))),
2207            (VEXTUBLX $Idx, $S)>;
2208
2209  def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, i64:$Idx)))),
2210            (VEXTUHLX (RLWINM8 $Idx, 1, 28, 30), $S)>;
2211  def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 0)))),
2212            (VEXTUHLX (LI8 0), $S)>;
2213  def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 1)))),
2214            (VEXTUHLX (LI8 2), $S)>;
2215  def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 2)))),
2216            (VEXTUHLX (LI8 4), $S)>;
2217  def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 3)))),
2218            (VEXTUHLX (LI8 6), $S)>;
2219  def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 4)))),
2220            (VEXTUHLX (LI8 8), $S)>;
2221  def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 5)))),
2222            (VEXTUHLX (LI8 10), $S)>;
2223  def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 6)))),
2224            (VEXTUHLX (LI8 12), $S)>;
2225  def : Pat<(i64 (anyext (i32 (vector_extract v8i16:$S, 7)))),
2226            (VEXTUHLX (LI8 14), $S)>;
2227
2228  def : Pat<(i64 (zext (i32 (vector_extract v4i32:$S, i64:$Idx)))),
2229            (VEXTUWLX (RLWINM8 $Idx, 2, 28, 29), $S)>;
2230  def : Pat<(i64 (zext (i32 (vector_extract v4i32:$S, 0)))),
2231            (VEXTUWLX (LI8 0), $S)>;
2232
2233  // For extracting BE word 1, MFVSRWZ is better than VEXTUWLX
2234  def : Pat<(i64 (zext (i32 (vector_extract v4i32:$S, 1)))),
2235            (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
2236	    (i32 VectorExtractions.LE_WORD_2), sub_32)>;
2237  def : Pat<(i64 (zext (i32 (vector_extract v4i32:$S, 2)))),
2238            (VEXTUWLX (LI8 8), $S)>;
2239  def : Pat<(i64 (zext (i32 (vector_extract v4i32:$S, 3)))),
2240            (VEXTUWLX (LI8 12), $S)>;
2241
2242  def : Pat<(i64 (sext (i32 (vector_extract v4i32:$S, i64:$Idx)))),
2243            (EXTSW (VEXTUWLX (RLWINM8 $Idx, 2, 28, 29), $S))>;
2244  def : Pat<(i64 (sext (i32 (vector_extract v4i32:$S, 0)))),
2245            (EXTSW (VEXTUWLX (LI8 0), $S))>;
2246  // For extracting BE word 1, MFVSRWZ is better than VEXTUWLX
2247  def : Pat<(i64 (sext (i32 (vector_extract v4i32:$S, 1)))),
2248            (EXTSW (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
2249	    (i32 VectorExtractions.LE_WORD_2), sub_32))>;
2250  def : Pat<(i64 (sext (i32 (vector_extract v4i32:$S, 2)))),
2251            (EXTSW (VEXTUWLX (LI8 8), $S))>;
2252  def : Pat<(i64 (sext (i32 (vector_extract v4i32:$S, 3)))),
2253            (EXTSW (VEXTUWLX (LI8 12), $S))>;
2254
2255  def : Pat<(i32 (vector_extract v16i8:$S, i64:$Idx)),
2256            (i32 (EXTRACT_SUBREG (VEXTUBLX $Idx, $S), sub_32))>;
2257  def : Pat<(i32 (vector_extract v16i8:$S, 0)),
2258            (i32 (EXTRACT_SUBREG (VEXTUBLX (LI8 0), $S), sub_32))>;
2259  def : Pat<(i32 (vector_extract v16i8:$S, 1)),
2260            (i32 (EXTRACT_SUBREG (VEXTUBLX (LI8 1), $S), sub_32))>;
2261  def : Pat<(i32 (vector_extract v16i8:$S, 2)),
2262            (i32 (EXTRACT_SUBREG (VEXTUBLX (LI8 2), $S), sub_32))>;
2263  def : Pat<(i32 (vector_extract v16i8:$S, 3)),
2264            (i32 (EXTRACT_SUBREG (VEXTUBLX (LI8 3), $S), sub_32))>;
2265  def : Pat<(i32 (vector_extract v16i8:$S, 4)),
2266            (i32 (EXTRACT_SUBREG (VEXTUBLX (LI8 4), $S), sub_32))>;
2267  def : Pat<(i32 (vector_extract v16i8:$S, 5)),
2268            (i32 (EXTRACT_SUBREG (VEXTUBLX (LI8 5), $S), sub_32))>;
2269  def : Pat<(i32 (vector_extract v16i8:$S, 6)),
2270            (i32 (EXTRACT_SUBREG (VEXTUBLX (LI8 6), $S), sub_32))>;
2271  def : Pat<(i32 (vector_extract v16i8:$S, 7)),
2272            (i32 (EXTRACT_SUBREG (VEXTUBLX (LI8 7), $S), sub_32))>;
2273  def : Pat<(i32 (vector_extract v16i8:$S, 8)),
2274            (i32 (EXTRACT_SUBREG (VEXTUBLX (LI8 8), $S), sub_32))>;
2275  def : Pat<(i32 (vector_extract v16i8:$S, 9)),
2276            (i32 (EXTRACT_SUBREG (VEXTUBLX (LI8 9), $S), sub_32))>;
2277  def : Pat<(i32 (vector_extract v16i8:$S, 10)),
2278            (i32 (EXTRACT_SUBREG (VEXTUBLX (LI8 10), $S), sub_32))>;
2279  def : Pat<(i32 (vector_extract v16i8:$S, 11)),
2280            (i32 (EXTRACT_SUBREG (VEXTUBLX (LI8 11), $S), sub_32))>;
2281  def : Pat<(i32 (vector_extract v16i8:$S, 12)),
2282            (i32 (EXTRACT_SUBREG (VEXTUBLX (LI8 12), $S), sub_32))>;
2283  def : Pat<(i32 (vector_extract v16i8:$S, 13)),
2284            (i32 (EXTRACT_SUBREG (VEXTUBLX (LI8 13), $S), sub_32))>;
2285  def : Pat<(i32 (vector_extract v16i8:$S, 14)),
2286            (i32 (EXTRACT_SUBREG (VEXTUBLX (LI8 14), $S), sub_32))>;
2287  def : Pat<(i32 (vector_extract v16i8:$S, 15)),
2288            (i32 (EXTRACT_SUBREG (VEXTUBLX (LI8 15), $S), sub_32))>;
2289
2290  def : Pat<(i32 (vector_extract v8i16:$S, i64:$Idx)),
2291            (i32 (EXTRACT_SUBREG (VEXTUHLX
2292	    (RLWINM8 $Idx, 1, 28, 30), $S), sub_32))>;
2293  def : Pat<(i32 (vector_extract v8i16:$S, 0)),
2294            (i32 (EXTRACT_SUBREG (VEXTUHLX (LI8 0), $S), sub_32))>;
2295  def : Pat<(i32 (vector_extract v8i16:$S, 1)),
2296            (i32 (EXTRACT_SUBREG (VEXTUHLX (LI8 2), $S), sub_32))>;
2297  def : Pat<(i32 (vector_extract v8i16:$S, 2)),
2298            (i32 (EXTRACT_SUBREG (VEXTUHLX (LI8 4), $S), sub_32))>;
2299  def : Pat<(i32 (vector_extract v8i16:$S, 3)),
2300            (i32 (EXTRACT_SUBREG (VEXTUHLX (LI8 6), $S), sub_32))>;
2301  def : Pat<(i32 (vector_extract v8i16:$S, 4)),
2302            (i32 (EXTRACT_SUBREG (VEXTUHLX (LI8 8), $S), sub_32))>;
2303  def : Pat<(i32 (vector_extract v8i16:$S, 5)),
2304            (i32 (EXTRACT_SUBREG (VEXTUHLX (LI8 10), $S), sub_32))>;
2305  def : Pat<(i32 (vector_extract v8i16:$S, 6)),
2306            (i32 (EXTRACT_SUBREG (VEXTUHLX (LI8 12), $S), sub_32))>;
2307  def : Pat<(i32 (vector_extract v8i16:$S, 6)),
2308            (i32 (EXTRACT_SUBREG (VEXTUHLX (LI8 14), $S), sub_32))>;
2309
2310  def : Pat<(i32 (vector_extract v4i32:$S, i64:$Idx)),
2311            (i32 (EXTRACT_SUBREG (VEXTUWLX
2312	    (RLWINM8 $Idx, 2, 28, 29), $S), sub_32))>;
2313  def : Pat<(i32 (vector_extract v4i32:$S, 0)),
2314            (i32 (EXTRACT_SUBREG (VEXTUWLX (LI8 0), $S), sub_32))>;
2315  // For extracting BE word 1, MFVSRWZ is better than VEXTUWLX
2316  def : Pat<(i32 (vector_extract v4i32:$S, 1)),
2317            (i32 VectorExtractions.LE_WORD_2)>;
2318  def : Pat<(i32 (vector_extract v4i32:$S, 2)),
2319            (i32 (EXTRACT_SUBREG (VEXTUWLX (LI8 8), $S), sub_32))>;
2320  def : Pat<(i32 (vector_extract v4i32:$S, 3)),
2321            (i32 (EXTRACT_SUBREG (VEXTUWLX (LI8 12), $S), sub_32))>;
2322}
2323
2324let Predicates = [IsLittleEndian, HasDirectMove] in {
2325  // v16i8 scalar <-> vector conversions (LE)
2326  def : Pat<(v16i8 (scalar_to_vector i32:$A)),
2327            (v16i8 (COPY_TO_REGCLASS MovesToVSR.LE_WORD_0, VSRC))>;
2328  def : Pat<(v8i16 (scalar_to_vector i32:$A)),
2329            (v8i16 (COPY_TO_REGCLASS MovesToVSR.LE_WORD_0, VSRC))>;
2330  def : Pat<(v4i32 (scalar_to_vector i32:$A)),
2331            (v4i32 MovesToVSR.LE_WORD_0)>;
2332  def : Pat<(v2i64 (scalar_to_vector i64:$A)),
2333            (v2i64 MovesToVSR.LE_DWORD_0)>;
2334  // v2i64 scalar <-> vector conversions (LE)
2335  def : Pat<(i64 (vector_extract v2i64:$S, 0)),
2336            (i64 VectorExtractions.LE_DWORD_0)>;
2337  def : Pat<(i64 (vector_extract v2i64:$S, 1)),
2338            (i64 VectorExtractions.LE_DWORD_1)>;
2339  def : Pat<(i64 (vector_extract v2i64:$S, i64:$Idx)),
2340            (i64 VectorExtractions.LE_VARIABLE_DWORD)>;
2341} // IsLittleEndian, HasDirectMove
2342
2343let Predicates = [IsLittleEndian, HasDirectMove, NoP9Altivec] in {
2344  def : Pat<(i32 (vector_extract v16i8:$S, 0)),
2345            (i32 VectorExtractions.LE_BYTE_0)>;
2346  def : Pat<(i32 (vector_extract v16i8:$S, 1)),
2347            (i32 VectorExtractions.LE_BYTE_1)>;
2348  def : Pat<(i32 (vector_extract v16i8:$S, 2)),
2349            (i32 VectorExtractions.LE_BYTE_2)>;
2350  def : Pat<(i32 (vector_extract v16i8:$S, 3)),
2351            (i32 VectorExtractions.LE_BYTE_3)>;
2352  def : Pat<(i32 (vector_extract v16i8:$S, 4)),
2353            (i32 VectorExtractions.LE_BYTE_4)>;
2354  def : Pat<(i32 (vector_extract v16i8:$S, 5)),
2355            (i32 VectorExtractions.LE_BYTE_5)>;
2356  def : Pat<(i32 (vector_extract v16i8:$S, 6)),
2357            (i32 VectorExtractions.LE_BYTE_6)>;
2358  def : Pat<(i32 (vector_extract v16i8:$S, 7)),
2359            (i32 VectorExtractions.LE_BYTE_7)>;
2360  def : Pat<(i32 (vector_extract v16i8:$S, 8)),
2361            (i32 VectorExtractions.LE_BYTE_8)>;
2362  def : Pat<(i32 (vector_extract v16i8:$S, 9)),
2363            (i32 VectorExtractions.LE_BYTE_9)>;
2364  def : Pat<(i32 (vector_extract v16i8:$S, 10)),
2365            (i32 VectorExtractions.LE_BYTE_10)>;
2366  def : Pat<(i32 (vector_extract v16i8:$S, 11)),
2367            (i32 VectorExtractions.LE_BYTE_11)>;
2368  def : Pat<(i32 (vector_extract v16i8:$S, 12)),
2369            (i32 VectorExtractions.LE_BYTE_12)>;
2370  def : Pat<(i32 (vector_extract v16i8:$S, 13)),
2371            (i32 VectorExtractions.LE_BYTE_13)>;
2372  def : Pat<(i32 (vector_extract v16i8:$S, 14)),
2373            (i32 VectorExtractions.LE_BYTE_14)>;
2374  def : Pat<(i32 (vector_extract v16i8:$S, 15)),
2375            (i32 VectorExtractions.LE_BYTE_15)>;
2376  def : Pat<(i32 (vector_extract v16i8:$S, i64:$Idx)),
2377            (i32 VectorExtractions.LE_VARIABLE_BYTE)>;
2378
2379  // v8i16 scalar <-> vector conversions (LE)
2380  def : Pat<(i32 (vector_extract v8i16:$S, 0)),
2381            (i32 VectorExtractions.LE_HALF_0)>;
2382  def : Pat<(i32 (vector_extract v8i16:$S, 1)),
2383            (i32 VectorExtractions.LE_HALF_1)>;
2384  def : Pat<(i32 (vector_extract v8i16:$S, 2)),
2385            (i32 VectorExtractions.LE_HALF_2)>;
2386  def : Pat<(i32 (vector_extract v8i16:$S, 3)),
2387            (i32 VectorExtractions.LE_HALF_3)>;
2388  def : Pat<(i32 (vector_extract v8i16:$S, 4)),
2389            (i32 VectorExtractions.LE_HALF_4)>;
2390  def : Pat<(i32 (vector_extract v8i16:$S, 5)),
2391            (i32 VectorExtractions.LE_HALF_5)>;
2392  def : Pat<(i32 (vector_extract v8i16:$S, 6)),
2393            (i32 VectorExtractions.LE_HALF_6)>;
2394  def : Pat<(i32 (vector_extract v8i16:$S, 7)),
2395            (i32 VectorExtractions.LE_HALF_7)>;
2396  def : Pat<(i32 (vector_extract v8i16:$S, i64:$Idx)),
2397            (i32 VectorExtractions.LE_VARIABLE_HALF)>;
2398
2399  // v4i32 scalar <-> vector conversions (LE)
2400  def : Pat<(i32 (vector_extract v4i32:$S, 0)),
2401            (i32 VectorExtractions.LE_WORD_0)>;
2402  def : Pat<(i32 (vector_extract v4i32:$S, 1)),
2403            (i32 VectorExtractions.LE_WORD_1)>;
2404  def : Pat<(i32 (vector_extract v4i32:$S, 2)),
2405            (i32 VectorExtractions.LE_WORD_2)>;
2406  def : Pat<(i32 (vector_extract v4i32:$S, 3)),
2407            (i32 VectorExtractions.LE_WORD_3)>;
2408  def : Pat<(i32 (vector_extract v4i32:$S, i64:$Idx)),
2409            (i32 VectorExtractions.LE_VARIABLE_WORD)>;
2410} // IsLittleEndian, HasDirectMove, NoP9Altivec
2411
2412let Predicates = [HasDirectMove, HasVSX] in {
2413// bitconvert f32 -> i32
2414// (convert to 32-bit fp single, shift right 1 word, move to GPR)
2415def : Pat<(i32 (bitconvert f32:$S)),
2416          (i32 (MFVSRWZ (EXTRACT_SUBREG
2417                          (XXSLDWI (XSCVDPSPN $S), (XSCVDPSPN $S), 3),
2418                          sub_64)))>;
2419// bitconvert i32 -> f32
2420// (move to FPR, shift left 1 word, convert to 64-bit fp single)
2421def : Pat<(f32 (bitconvert i32:$A)),
2422          (f32 (XSCVSPDPN
2423                 (XXSLDWI MovesToVSR.LE_WORD_1, MovesToVSR.LE_WORD_1, 1)))>;
2424
2425// bitconvert f64 -> i64
2426// (move to GPR, nothing else needed)
2427def : Pat<(i64 (bitconvert f64:$S)),
2428          (i64 (MFVSRD $S))>;
2429
2430// bitconvert i64 -> f64
2431// (move to FPR, nothing else needed)
2432def : Pat<(f64 (bitconvert i64:$S)),
2433          (f64 (MTVSRD $S))>;
2434}
2435
2436// Materialize a zero-vector of long long
2437def : Pat<(v2i64 immAllZerosV),
2438          (v2i64 (XXLXORz))>;
2439}
2440
2441def AlignValues {
2442  dag F32_TO_BE_WORD1 = (v4f32 (XXSLDWI (XSCVDPSPN $B), (XSCVDPSPN $B), 3));
2443  dag I32_TO_BE_WORD1 = (COPY_TO_REGCLASS (MTVSRWZ $B), VSRC);
2444}
2445
2446// The following VSX instructions were introduced in Power ISA 3.0
2447def HasP9Vector : Predicate<"PPCSubTarget->hasP9Vector()">;
2448let AddedComplexity = 400, Predicates = [HasP9Vector] in {
2449
2450  // [PO VRT XO VRB XO /]
2451  class X_VT5_XO5_VB5<bits<6> opcode, bits<5> xo2, bits<10> xo, string opc,
2452                      list<dag> pattern>
2453    : X_RD5_XO5_RS5<opcode, xo2, xo, (outs vrrc:$vT), (ins vrrc:$vB),
2454                    !strconcat(opc, " $vT, $vB"), IIC_VecFP, pattern>;
2455
2456  // [PO VRT XO VRB XO RO], Round to Odd version of [PO VRT XO VRB XO /]
2457  class X_VT5_XO5_VB5_Ro<bits<6> opcode, bits<5> xo2, bits<10> xo, string opc,
2458                         list<dag> pattern>
2459    : X_VT5_XO5_VB5<opcode, xo2, xo, opc, pattern>, isDOT;
2460
2461  // [PO VRT XO VRB XO /], but the VRB is only used the left 64 bits (or less),
2462  // So we use different operand class for VRB
2463  class X_VT5_XO5_VB5_TyVB<bits<6> opcode, bits<5> xo2, bits<10> xo, string opc,
2464                           RegisterOperand vbtype, list<dag> pattern>
2465    : X_RD5_XO5_RS5<opcode, xo2, xo, (outs vrrc:$vT), (ins vbtype:$vB),
2466                    !strconcat(opc, " $vT, $vB"), IIC_VecFP, pattern>;
2467
2468  // [PO VRT XO VRB XO /]
2469  class X_VT5_XO5_VB5_VSFR<bits<6> opcode, bits<5> xo2, bits<10> xo, string opc,
2470                      list<dag> pattern>
2471    : X_RD5_XO5_RS5<opcode, xo2, xo, (outs vfrc:$vT), (ins vrrc:$vB),
2472                    !strconcat(opc, " $vT, $vB"), IIC_VecFP, pattern>;
2473
2474  // [PO VRT XO VRB XO RO], Round to Odd version of [PO VRT XO VRB XO /]
2475  class X_VT5_XO5_VB5_VSFR_Ro<bits<6> opcode, bits<5> xo2, bits<10> xo, string opc,
2476                         list<dag> pattern>
2477    : X_VT5_XO5_VB5_VSFR<opcode, xo2, xo, opc, pattern>, isDOT;
2478
2479  // [PO T XO B XO BX /]
2480  class XX2_RT5_XO5_XB6<bits<6> opcode, bits<5> xo2, bits<9> xo, string opc,
2481                        list<dag> pattern>
2482    : XX2_RD5_XO5_RS6<opcode, xo2, xo, (outs g8rc:$rT), (ins vsfrc:$XB),
2483                      !strconcat(opc, " $rT, $XB"), IIC_VecFP, pattern>;
2484
2485  // [PO T XO B XO BX TX]
2486  class XX2_XT6_XO5_XB6<bits<6> opcode, bits<5> xo2, bits<9> xo, string opc,
2487                        RegisterOperand vtype, list<dag> pattern>
2488    : XX2_RD6_XO5_RS6<opcode, xo2, xo, (outs vtype:$XT), (ins vtype:$XB),
2489                      !strconcat(opc, " $XT, $XB"), IIC_VecFP, pattern>;
2490
2491  // [PO T A B XO AX BX TX], src and dest register use different operand class
2492  class XX3_XT5_XA5_XB5<bits<6> opcode, bits<8> xo, string opc,
2493                  RegisterOperand xty, RegisterOperand aty, RegisterOperand bty,
2494                  InstrItinClass itin, list<dag> pattern>
2495    : XX3Form<opcode, xo, (outs xty:$XT), (ins aty:$XA, bty:$XB),
2496              !strconcat(opc, " $XT, $XA, $XB"), itin, pattern>;
2497
2498  // [PO VRT VRA VRB XO /]
2499  class X_VT5_VA5_VB5<bits<6> opcode, bits<10> xo, string opc,
2500                      list<dag> pattern>
2501    : XForm_1<opcode, xo, (outs vrrc:$vT), (ins vrrc:$vA, vrrc:$vB),
2502              !strconcat(opc, " $vT, $vA, $vB"), IIC_VecFP, pattern>;
2503
2504  // [PO VRT VRA VRB XO RO], Round to Odd version of [PO VRT VRA VRB XO /]
2505  class X_VT5_VA5_VB5_Ro<bits<6> opcode, bits<10> xo, string opc,
2506                         list<dag> pattern>
2507    : X_VT5_VA5_VB5<opcode, xo, opc, pattern>, isDOT;
2508
2509  // [PO VRT VRA VRB XO /]
2510  class X_VT5_VA5_VB5_FMA<bits<6> opcode, bits<10> xo, string opc,
2511                          list<dag> pattern>
2512    : XForm_1<opcode, xo, (outs vrrc:$vT), (ins vrrc:$vTi, vrrc:$vA, vrrc:$vB),
2513              !strconcat(opc, " $vT, $vA, $vB"), IIC_VecFP, pattern>,
2514              RegConstraint<"$vTi = $vT">, NoEncode<"$vTi">;
2515
2516  // [PO VRT VRA VRB XO RO], Round to Odd version of [PO VRT VRA VRB XO /]
2517  class X_VT5_VA5_VB5_FMA_Ro<bits<6> opcode, bits<10> xo, string opc,
2518                          list<dag> pattern>
2519    : X_VT5_VA5_VB5_FMA<opcode, xo, opc, pattern>, isDOT;
2520
2521  //===--------------------------------------------------------------------===//
2522  // Quad-Precision Scalar Move Instructions:
2523
2524  // Copy Sign
2525  def XSCPSGNQP : X_VT5_VA5_VB5<63, 100, "xscpsgnqp",
2526                                [(set f128:$vT,
2527                                      (fcopysign f128:$vB, f128:$vA))]>;
2528
2529  // Absolute/Negative-Absolute/Negate
2530  def XSABSQP   : X_VT5_XO5_VB5<63,  0, 804, "xsabsqp",
2531                                [(set f128:$vT, (fabs f128:$vB))]>;
2532  def XSNABSQP  : X_VT5_XO5_VB5<63,  8, 804, "xsnabsqp",
2533                                [(set f128:$vT, (fneg (fabs f128:$vB)))]>;
2534  def XSNEGQP   : X_VT5_XO5_VB5<63, 16, 804, "xsnegqp",
2535                                [(set f128:$vT, (fneg f128:$vB))]>;
2536
2537  //===--------------------------------------------------------------------===//
2538  // Quad-Precision Scalar Floating-Point Arithmetic Instructions:
2539
2540  // Add/Divide/Multiply/Subtract
2541  let isCommutable = 1 in {
2542  def XSADDQP   : X_VT5_VA5_VB5   <63,   4, "xsaddqp",
2543                                   [(set f128:$vT, (fadd f128:$vA, f128:$vB))]>;
2544  def XSMULQP   : X_VT5_VA5_VB5   <63,  36, "xsmulqp",
2545                                   [(set f128:$vT, (fmul f128:$vA, f128:$vB))]>;
2546  }
2547  def XSSUBQP   : X_VT5_VA5_VB5   <63, 516, "xssubqp" ,
2548                                   [(set f128:$vT, (fsub f128:$vA, f128:$vB))]>;
2549  def XSDIVQP   : X_VT5_VA5_VB5   <63, 548, "xsdivqp",
2550                                   [(set f128:$vT, (fdiv f128:$vA, f128:$vB))]>;
2551  // Square-Root
2552  def XSSQRTQP  : X_VT5_XO5_VB5   <63, 27, 804, "xssqrtqp",
2553                                   [(set f128:$vT, (fsqrt f128:$vB))]>;
2554  // (Negative) Multiply-{Add/Subtract}
2555  def XSMADDQP : X_VT5_VA5_VB5_FMA <63, 388, "xsmaddqp",
2556                                    [(set f128:$vT,
2557                                          (fma f128:$vA, f128:$vB,
2558                                               f128:$vTi))]>;
2559  def XSMSUBQP  : X_VT5_VA5_VB5_FMA   <63, 420, "xsmsubqp"  ,
2560                                       [(set f128:$vT,
2561                                             (fma f128:$vA, f128:$vB,
2562                                                  (fneg f128:$vTi)))]>;
2563  def XSNMADDQP : X_VT5_VA5_VB5_FMA <63, 452, "xsnmaddqp",
2564                                     [(set f128:$vT,
2565                                           (fneg (fma f128:$vA, f128:$vB,
2566                                                      f128:$vTi)))]>;
2567  def XSNMSUBQP : X_VT5_VA5_VB5_FMA <63, 484, "xsnmsubqp",
2568                                     [(set f128:$vT,
2569                                           (fneg (fma f128:$vA, f128:$vB,
2570                                                      (fneg f128:$vTi))))]>;
2571
2572  let isCommutable = 1 in {
2573  def XSADDQPO : X_VT5_VA5_VB5_Ro<63, 4, "xsaddqpo",
2574                                  [(set f128:$vT,
2575                                  (int_ppc_addf128_round_to_odd
2576                                  f128:$vA, f128:$vB))]>;
2577  def XSMULQPO : X_VT5_VA5_VB5_Ro<63, 36, "xsmulqpo",
2578                                  [(set f128:$vT,
2579                                  (int_ppc_mulf128_round_to_odd
2580                                  f128:$vA, f128:$vB))]>;
2581  }
2582  def XSSUBQPO : X_VT5_VA5_VB5_Ro<63, 516, "xssubqpo",
2583                                  [(set f128:$vT,
2584                                  (int_ppc_subf128_round_to_odd
2585                                  f128:$vA, f128:$vB))]>;
2586  def XSDIVQPO : X_VT5_VA5_VB5_Ro<63, 548, "xsdivqpo",
2587                                  [(set f128:$vT,
2588                                  (int_ppc_divf128_round_to_odd
2589                                  f128:$vA, f128:$vB))]>;
2590  def XSSQRTQPO : X_VT5_XO5_VB5_Ro<63, 27, 804, "xssqrtqpo",
2591                                  [(set f128:$vT,
2592                                  (int_ppc_sqrtf128_round_to_odd f128:$vB))]>;
2593
2594
2595  def XSMADDQPO : X_VT5_VA5_VB5_FMA_Ro<63, 388, "xsmaddqpo",
2596                                      [(set f128:$vT,
2597                                      (int_ppc_fmaf128_round_to_odd
2598                                      f128:$vA,f128:$vB,f128:$vTi))]>;
2599
2600  def XSMSUBQPO : X_VT5_VA5_VB5_FMA_Ro<63, 420, "xsmsubqpo" ,
2601                                      [(set f128:$vT,
2602                                      (int_ppc_fmaf128_round_to_odd
2603                                      f128:$vA, f128:$vB, (fneg f128:$vTi)))]>;
2604  def XSNMADDQPO: X_VT5_VA5_VB5_FMA_Ro<63, 452, "xsnmaddqpo",
2605                                      [(set f128:$vT,
2606                                      (fneg (int_ppc_fmaf128_round_to_odd
2607                                      f128:$vA, f128:$vB, f128:$vTi)))]>;
2608  def XSNMSUBQPO: X_VT5_VA5_VB5_FMA_Ro<63, 484, "xsnmsubqpo",
2609                                      [(set f128:$vT,
2610                                      (fneg (int_ppc_fmaf128_round_to_odd
2611                                      f128:$vA, f128:$vB, (fneg f128:$vTi))))]>;
2612
2613  // Additional fnmsub patterns: -a*c + b == -(a*c - b)
2614  def : Pat<(fma (fneg f128:$A), f128:$C, f128:$B), (XSNMSUBQP $B, $C, $A)>;
2615  def : Pat<(fma f128:$A, (fneg f128:$C), f128:$B), (XSNMSUBQP $B, $C, $A)>;
2616
2617  //===--------------------------------------------------------------------===//
2618  // Quad/Double-Precision Compare Instructions:
2619
2620  // [PO BF // VRA VRB XO /]
2621  class X_BF3_VA5_VB5<bits<6> opcode, bits<10> xo, string opc,
2622                      list<dag> pattern>
2623    : XForm_17<opcode, xo, (outs crrc:$crD), (ins vrrc:$VA, vrrc:$VB),
2624               !strconcat(opc, " $crD, $VA, $VB"), IIC_FPCompare> {
2625    let Pattern = pattern;
2626  }
2627
2628  // QP Compare Ordered/Unordered
2629  def XSCMPOQP : X_BF3_VA5_VB5<63, 132, "xscmpoqp", []>;
2630  def XSCMPUQP : X_BF3_VA5_VB5<63, 644, "xscmpuqp", []>;
2631
2632  // DP/QP Compare Exponents
2633  def XSCMPEXPDP : XX3Form_1<60, 59,
2634                             (outs crrc:$crD), (ins vsfrc:$XA, vsfrc:$XB),
2635                             "xscmpexpdp $crD, $XA, $XB", IIC_FPCompare, []>;
2636  def XSCMPEXPQP : X_BF3_VA5_VB5<63, 164, "xscmpexpqp", []>;
2637
2638  // DP Compare ==, >=, >, !=
2639  // Use vsrc for XT, because the entire register of XT is set.
2640  // XT.dword[1] = 0x0000_0000_0000_0000
2641  def XSCMPEQDP : XX3_XT5_XA5_XB5<60,  3, "xscmpeqdp", vsrc, vsfrc, vsfrc,
2642                                  IIC_FPCompare, []>;
2643  def XSCMPGEDP : XX3_XT5_XA5_XB5<60, 19, "xscmpgedp", vsrc, vsfrc, vsfrc,
2644                                  IIC_FPCompare, []>;
2645  def XSCMPGTDP : XX3_XT5_XA5_XB5<60, 11, "xscmpgtdp", vsrc, vsfrc, vsfrc,
2646                                  IIC_FPCompare, []>;
2647
2648  //===--------------------------------------------------------------------===//
2649  // Quad-Precision Floating-Point Conversion Instructions:
2650
2651  // Convert DP -> QP
2652  def XSCVDPQP  : X_VT5_XO5_VB5_TyVB<63, 22, 836, "xscvdpqp", vfrc,
2653                                     [(set f128:$vT, (fpextend f64:$vB))]>;
2654
2655  // Round & Convert QP -> DP (dword[1] is set to zero)
2656  def XSCVQPDP  : X_VT5_XO5_VB5_VSFR<63, 20, 836, "xscvqpdp" , []>;
2657  def XSCVQPDPO : X_VT5_XO5_VB5_VSFR_Ro<63, 20, 836, "xscvqpdpo",
2658                                        [(set f64:$vT,
2659                                        (int_ppc_truncf128_round_to_odd
2660                                        f128:$vB))]>;
2661
2662  // Truncate & Convert QP -> (Un)Signed (D)Word (dword[1] is set to zero)
2663  def XSCVQPSDZ : X_VT5_XO5_VB5<63, 25, 836, "xscvqpsdz", []>;
2664  def XSCVQPSWZ : X_VT5_XO5_VB5<63,  9, 836, "xscvqpswz", []>;
2665  def XSCVQPUDZ : X_VT5_XO5_VB5<63, 17, 836, "xscvqpudz", []>;
2666  def XSCVQPUWZ : X_VT5_XO5_VB5<63,  1, 836, "xscvqpuwz", []>;
2667
2668  // Convert (Un)Signed DWord -> QP.
2669  def XSCVSDQP  : X_VT5_XO5_VB5_TyVB<63, 10, 836, "xscvsdqp", vfrc, []>;
2670  def : Pat<(f128 (sint_to_fp i64:$src)),
2671            (f128 (XSCVSDQP (COPY_TO_REGCLASS $src, VFRC)))>;
2672  def : Pat<(f128 (sint_to_fp (i64 (PPCmfvsr f64:$src)))),
2673            (f128 (XSCVSDQP $src))>;
2674  def : Pat<(f128 (sint_to_fp (i32 (PPCmfvsr f64:$src)))),
2675            (f128 (XSCVSDQP (VEXTSW2Ds $src)))>;
2676
2677  def XSCVUDQP  : X_VT5_XO5_VB5_TyVB<63,  2, 836, "xscvudqp", vfrc, []>;
2678  def : Pat<(f128 (uint_to_fp i64:$src)),
2679            (f128 (XSCVUDQP (COPY_TO_REGCLASS $src, VFRC)))>;
2680  def : Pat<(f128 (uint_to_fp (i64 (PPCmfvsr f64:$src)))),
2681            (f128 (XSCVUDQP $src))>;
2682
2683  // Convert (Un)Signed Word -> QP.
2684  def : Pat<(f128 (sint_to_fp i32:$src)),
2685            (f128 (XSCVSDQP (MTVSRWA $src)))>;
2686  def : Pat<(f128 (sint_to_fp (i32 (load xoaddr:$src)))),
2687            (f128 (XSCVSDQP (LIWAX xoaddr:$src)))>;
2688  def : Pat<(f128 (uint_to_fp i32:$src)),
2689            (f128 (XSCVUDQP (MTVSRWZ $src)))>;
2690  def : Pat<(f128 (uint_to_fp (i32 (load xoaddr:$src)))),
2691            (f128 (XSCVUDQP (LIWZX xoaddr:$src)))>;
2692
2693  //===--------------------------------------------------------------------===//
2694  // Round to Floating-Point Integer Instructions
2695
2696  // (Round &) Convert DP <-> HP
2697  // Note! xscvdphp's src and dest register both use the left 64 bits, so we use
2698  // vsfrc for src and dest register. xscvhpdp's src only use the left 16 bits,
2699  // but we still use vsfrc for it.
2700  def XSCVDPHP : XX2_XT6_XO5_XB6<60, 17, 347, "xscvdphp", vsfrc, []>;
2701  def XSCVHPDP : XX2_XT6_XO5_XB6<60, 16, 347, "xscvhpdp", vsfrc, []>;
2702
2703  // Vector HP -> SP
2704  def XVCVHPSP : XX2_XT6_XO5_XB6<60, 24, 475, "xvcvhpsp", vsrc, []>;
2705  def XVCVSPHP : XX2_XT6_XO5_XB6<60, 25, 475, "xvcvsphp", vsrc,
2706                                 [(set v4f32:$XT,
2707                                     (int_ppc_vsx_xvcvsphp v4f32:$XB))]>;
2708
2709  // Pattern for matching Vector HP -> Vector SP intrinsic. Defined as a
2710  // separate pattern so that it can convert the input register class from
2711  // VRRC(v8i16) to VSRC.
2712  def : Pat<(v4f32 (int_ppc_vsx_xvcvhpsp v8i16:$A)),
2713            (v4f32 (XVCVHPSP (COPY_TO_REGCLASS $A, VSRC)))>;
2714
2715  class Z23_VT5_R1_VB5_RMC2_EX1<bits<6> opcode, bits<8> xo, bit ex, string opc,
2716                                list<dag> pattern>
2717    : Z23Form_8<opcode, xo,
2718                (outs vrrc:$vT), (ins u1imm:$r, vrrc:$vB, u2imm:$rmc),
2719                !strconcat(opc, " $r, $vT, $vB, $rmc"), IIC_VecFP, pattern> {
2720    let RC = ex;
2721  }
2722
2723  // Round to Quad-Precision Integer [with Inexact]
2724  def XSRQPI   : Z23_VT5_R1_VB5_RMC2_EX1<63,  5, 0, "xsrqpi" , []>;
2725  def XSRQPIX  : Z23_VT5_R1_VB5_RMC2_EX1<63,  5, 1, "xsrqpix", []>;
2726
2727  // Use current rounding mode
2728  def : Pat<(f128 (fnearbyint f128:$vB)), (f128 (XSRQPI 0, $vB, 3))>;
2729  // Round to nearest, ties away from zero
2730  def : Pat<(f128 (fround f128:$vB)), (f128 (XSRQPI 0, $vB, 0))>;
2731  // Round towards Zero
2732  def : Pat<(f128 (ftrunc f128:$vB)), (f128 (XSRQPI 1, $vB, 1))>;
2733  // Round towards +Inf
2734  def : Pat<(f128 (fceil f128:$vB)), (f128 (XSRQPI 1, $vB, 2))>;
2735  // Round towards -Inf
2736  def : Pat<(f128 (ffloor f128:$vB)), (f128 (XSRQPI 1, $vB, 3))>;
2737
2738  // Use current rounding mode, [with Inexact]
2739  def : Pat<(f128 (frint f128:$vB)), (f128 (XSRQPIX 0, $vB, 3))>;
2740
2741  // Round Quad-Precision to Double-Extended Precision (fp80)
2742  def XSRQPXP  : Z23_VT5_R1_VB5_RMC2_EX1<63, 37, 0, "xsrqpxp", []>;
2743
2744  //===--------------------------------------------------------------------===//
2745  // Insert/Extract Instructions
2746
2747  // Insert Exponent DP/QP
2748  // XT NOTE: XT.dword[1] = 0xUUUU_UUUU_UUUU_UUUU
2749  def XSIEXPDP : XX1Form <60, 918, (outs vsrc:$XT), (ins g8rc:$rA, g8rc:$rB),
2750                          "xsiexpdp $XT, $rA, $rB", IIC_VecFP, []>;
2751  // vB NOTE: only vB.dword[0] is used, that's why we don't use
2752  //          X_VT5_VA5_VB5 form
2753  def XSIEXPQP : XForm_18<63, 868, (outs vrrc:$vT), (ins vrrc:$vA, vsfrc:$vB),
2754                          "xsiexpqp $vT, $vA, $vB", IIC_VecFP, []>;
2755
2756  def : Pat<(f128 (int_ppc_scalar_insert_exp_qp f128:$vA, i64:$vB)),
2757            (f128 (XSIEXPQP $vA, (MTVSRD $vB)))>;
2758
2759  // Extract Exponent/Significand DP/QP
2760  def XSXEXPDP : XX2_RT5_XO5_XB6<60,  0, 347, "xsxexpdp", []>;
2761  def XSXSIGDP : XX2_RT5_XO5_XB6<60,  1, 347, "xsxsigdp", []>;
2762
2763  def XSXEXPQP : X_VT5_XO5_VB5  <63,  2, 804, "xsxexpqp", []>;
2764  def XSXSIGQP : X_VT5_XO5_VB5  <63, 18, 804, "xsxsigqp", []>;
2765
2766  def : Pat<(i64 (int_ppc_scalar_extract_expq  f128:$vA)),
2767            (i64 (MFVSRD (EXTRACT_SUBREG
2768                           (v2i64 (XSXEXPQP $vA)), sub_64)))>;
2769
2770  // Vector Insert Word
2771  // XB NOTE: Only XB.dword[1] is used, but we use vsrc on XB.
2772  def XXINSERTW   :
2773    XX2_RD6_UIM5_RS6<60, 181, (outs vsrc:$XT),
2774                     (ins vsrc:$XTi, vsrc:$XB, u4imm:$UIM),
2775                     "xxinsertw $XT, $XB, $UIM", IIC_VecFP,
2776                     [(set v4i32:$XT, (PPCvecinsert v4i32:$XTi, v4i32:$XB,
2777                                                   imm32SExt16:$UIM))]>,
2778                     RegConstraint<"$XTi = $XT">, NoEncode<"$XTi">;
2779
2780  // Vector Extract Unsigned Word
2781  def XXEXTRACTUW : XX2_RD6_UIM5_RS6<60, 165,
2782                                  (outs vsfrc:$XT), (ins vsrc:$XB, u4imm:$UIMM),
2783                                  "xxextractuw $XT, $XB, $UIMM", IIC_VecFP, []>;
2784
2785  // Vector Insert Exponent DP/SP
2786  def XVIEXPDP : XX3_XT5_XA5_XB5<60, 248, "xviexpdp", vsrc, vsrc, vsrc,
2787    IIC_VecFP, [(set v2f64: $XT,(int_ppc_vsx_xviexpdp v2i64:$XA, v2i64:$XB))]>;
2788  def XVIEXPSP : XX3_XT5_XA5_XB5<60, 216, "xviexpsp", vsrc, vsrc, vsrc,
2789    IIC_VecFP, [(set v4f32: $XT,(int_ppc_vsx_xviexpsp v4i32:$XA, v4i32:$XB))]>;
2790
2791  // Vector Extract Exponent/Significand DP/SP
2792  def XVXEXPDP : XX2_XT6_XO5_XB6<60,  0, 475, "xvxexpdp", vsrc,
2793                                 [(set v2i64: $XT,
2794                                  (int_ppc_vsx_xvxexpdp v2f64:$XB))]>;
2795  def XVXEXPSP : XX2_XT6_XO5_XB6<60,  8, 475, "xvxexpsp", vsrc,
2796                                 [(set v4i32: $XT,
2797                                  (int_ppc_vsx_xvxexpsp v4f32:$XB))]>;
2798  def XVXSIGDP : XX2_XT6_XO5_XB6<60,  1, 475, "xvxsigdp", vsrc,
2799                                 [(set v2i64: $XT,
2800                                  (int_ppc_vsx_xvxsigdp v2f64:$XB))]>;
2801  def XVXSIGSP : XX2_XT6_XO5_XB6<60,  9, 475, "xvxsigsp", vsrc,
2802                                 [(set v4i32: $XT,
2803                                  (int_ppc_vsx_xvxsigsp v4f32:$XB))]>;
2804
2805  let AddedComplexity = 400, Predicates = [HasP9Vector] in {
2806  // Extra patterns expanding to vector Extract Word/Insert Word
2807  def : Pat<(v4i32 (int_ppc_vsx_xxinsertw v4i32:$A, v2i64:$B, imm:$IMM)),
2808            (v4i32 (XXINSERTW $A, $B, imm:$IMM))>;
2809  def : Pat<(v2i64 (int_ppc_vsx_xxextractuw v2i64:$A, imm:$IMM)),
2810            (v2i64 (COPY_TO_REGCLASS (XXEXTRACTUW $A, imm:$IMM), VSRC))>;
2811  } // AddedComplexity = 400, HasP9Vector
2812
2813  //===--------------------------------------------------------------------===//
2814
2815  // Test Data Class SP/DP/QP
2816  def XSTSTDCSP : XX2_BF3_DCMX7_RS6<60, 298,
2817                              (outs crrc:$BF), (ins u7imm:$DCMX, vsfrc:$XB),
2818                              "xststdcsp $BF, $XB, $DCMX", IIC_VecFP, []>;
2819  def XSTSTDCDP : XX2_BF3_DCMX7_RS6<60, 362,
2820                              (outs crrc:$BF), (ins u7imm:$DCMX, vsfrc:$XB),
2821                              "xststdcdp $BF, $XB, $DCMX", IIC_VecFP, []>;
2822  def XSTSTDCQP : X_BF3_DCMX7_RS5  <63, 708,
2823                              (outs crrc:$BF), (ins u7imm:$DCMX, vrrc:$vB),
2824                              "xststdcqp $BF, $vB, $DCMX", IIC_VecFP, []>;
2825
2826  // Vector Test Data Class SP/DP
2827  def XVTSTDCSP : XX2_RD6_DCMX7_RS6<60, 13, 5,
2828                              (outs vsrc:$XT), (ins u7imm:$DCMX, vsrc:$XB),
2829                              "xvtstdcsp $XT, $XB, $DCMX", IIC_VecFP,
2830                              [(set v4i32: $XT,
2831                               (int_ppc_vsx_xvtstdcsp v4f32:$XB, imm:$DCMX))]>;
2832  def XVTSTDCDP : XX2_RD6_DCMX7_RS6<60, 15, 5,
2833                              (outs vsrc:$XT), (ins u7imm:$DCMX, vsrc:$XB),
2834                              "xvtstdcdp $XT, $XB, $DCMX", IIC_VecFP,
2835                              [(set v2i64: $XT,
2836                               (int_ppc_vsx_xvtstdcdp v2f64:$XB, imm:$DCMX))]>;
2837
2838  //===--------------------------------------------------------------------===//
2839
2840  // Maximum/Minimum Type-C/Type-J DP
2841  // XT.dword[1] = 0xUUUU_UUUU_UUUU_UUUU, so we use vsrc for XT
2842  def XSMAXCDP : XX3_XT5_XA5_XB5<60, 128, "xsmaxcdp", vsrc, vsfrc, vsfrc,
2843                                 IIC_VecFP, []>;
2844  def XSMAXJDP : XX3_XT5_XA5_XB5<60, 144, "xsmaxjdp", vsrc, vsfrc, vsfrc,
2845                                 IIC_VecFP, []>;
2846  def XSMINCDP : XX3_XT5_XA5_XB5<60, 136, "xsmincdp", vsrc, vsfrc, vsfrc,
2847                                 IIC_VecFP, []>;
2848  def XSMINJDP : XX3_XT5_XA5_XB5<60, 152, "xsminjdp", vsrc, vsfrc, vsfrc,
2849                                 IIC_VecFP, []>;
2850
2851  //===--------------------------------------------------------------------===//
2852
2853  // Vector Byte-Reverse H/W/D/Q Word
2854  def XXBRH : XX2_XT6_XO5_XB6<60,  7, 475, "xxbrh", vsrc, []>;
2855  def XXBRW : XX2_XT6_XO5_XB6<60, 15, 475, "xxbrw", vsrc, []>;
2856  def XXBRD : XX2_XT6_XO5_XB6<60, 23, 475, "xxbrd", vsrc, []>;
2857  def XXBRQ : XX2_XT6_XO5_XB6<60, 31, 475, "xxbrq", vsrc, []>;
2858
2859  // Vector Reverse
2860  def : Pat<(v8i16 (PPCxxreverse v8i16 :$A)),
2861            (v8i16 (COPY_TO_REGCLASS (XXBRH (COPY_TO_REGCLASS $A, VSRC)), VRRC))>;
2862  def : Pat<(v4i32 (PPCxxreverse v4i32 :$A)),
2863            (v4i32 (XXBRW $A))>;
2864  def : Pat<(v2i64 (PPCxxreverse v2i64 :$A)),
2865            (v2i64 (XXBRD $A))>;
2866  def : Pat<(v1i128 (PPCxxreverse v1i128 :$A)),
2867            (v1i128 (COPY_TO_REGCLASS (XXBRQ (COPY_TO_REGCLASS $A, VSRC)), VRRC))>;
2868
2869  // Vector Permute
2870  def XXPERM  : XX3_XT5_XA5_XB5<60, 26, "xxperm" , vsrc, vsrc, vsrc,
2871                                IIC_VecPerm, []>;
2872  def XXPERMR : XX3_XT5_XA5_XB5<60, 58, "xxpermr", vsrc, vsrc, vsrc,
2873                                IIC_VecPerm, []>;
2874
2875  // Vector Splat Immediate Byte
2876  def XXSPLTIB : X_RD6_IMM8<60, 360, (outs vsrc:$XT), (ins u8imm:$IMM8),
2877                            "xxspltib $XT, $IMM8", IIC_VecPerm, []>;
2878
2879  //===--------------------------------------------------------------------===//
2880  // Vector/Scalar Load/Store Instructions
2881
2882  // When adding new D-Form loads/stores, be sure to update the ImmToIdxMap in
2883  // PPCRegisterInfo::PPCRegisterInfo and maybe save yourself some debugging.
2884  let mayLoad = 1, mayStore = 0 in {
2885  // Load Vector
2886  def LXV : DQ_RD6_RS5_DQ12<61, 1, (outs vsrc:$XT), (ins memrix16:$src),
2887                            "lxv $XT, $src", IIC_LdStLFD, []>;
2888  // Load DWord
2889  def LXSD  : DSForm_1<57, 2, (outs vfrc:$vD), (ins memrix:$src),
2890                       "lxsd $vD, $src", IIC_LdStLFD, []>;
2891  // Load SP from src, convert it to DP, and place in dword[0]
2892  def LXSSP : DSForm_1<57, 3, (outs vfrc:$vD), (ins memrix:$src),
2893                       "lxssp $vD, $src", IIC_LdStLFD, []>;
2894
2895  // [PO T RA RB XO TX] almost equal to [PO S RA RB XO SX], but has different
2896  // "out" and "in" dag
2897  class X_XT6_RA5_RB5<bits<6> opcode, bits<10> xo, string opc,
2898                      RegisterOperand vtype, list<dag> pattern>
2899    : XX1Form_memOp<opcode, xo, (outs vtype:$XT), (ins memrr:$src),
2900              !strconcat(opc, " $XT, $src"), IIC_LdStLFD, pattern>;
2901
2902  // Load as Integer Byte/Halfword & Zero Indexed
2903  def LXSIBZX : X_XT6_RA5_RB5<31, 781, "lxsibzx", vsfrc,
2904                              [(set f64:$XT, (PPClxsizx xoaddr:$src, 1))]>;
2905  def LXSIHZX : X_XT6_RA5_RB5<31, 813, "lxsihzx", vsfrc,
2906                              [(set f64:$XT, (PPClxsizx xoaddr:$src, 2))]>;
2907
2908  // Load Vector Halfword*8/Byte*16 Indexed
2909  def LXVH8X  : X_XT6_RA5_RB5<31, 812, "lxvh8x" , vsrc, []>;
2910  def LXVB16X : X_XT6_RA5_RB5<31, 876, "lxvb16x", vsrc, []>;
2911
2912  // Load Vector Indexed
2913  def LXVX    : X_XT6_RA5_RB5<31, 268, "lxvx"   , vsrc,
2914                [(set v2f64:$XT, (load xaddrX16:$src))]>;
2915  // Load Vector (Left-justified) with Length
2916  def LXVL : XX1Form_memOp<31, 269, (outs vsrc:$XT), (ins memr:$src, g8rc:$rB),
2917                   "lxvl $XT, $src, $rB", IIC_LdStLoad,
2918                   [(set v4i32:$XT, (int_ppc_vsx_lxvl addr:$src, i64:$rB))]>;
2919  def LXVLL : XX1Form_memOp<31,301, (outs vsrc:$XT), (ins memr:$src, g8rc:$rB),
2920                   "lxvll $XT, $src, $rB", IIC_LdStLoad,
2921                   [(set v4i32:$XT, (int_ppc_vsx_lxvll addr:$src, i64:$rB))]>;
2922
2923  // Load Vector Word & Splat Indexed
2924  def LXVWSX  : X_XT6_RA5_RB5<31, 364, "lxvwsx" , vsrc, []>;
2925  } // mayLoad
2926
2927  // When adding new D-Form loads/stores, be sure to update the ImmToIdxMap in
2928  // PPCRegisterInfo::PPCRegisterInfo and maybe save yourself some debugging.
2929  let mayStore = 1, mayLoad = 0 in {
2930  // Store Vector
2931  def STXV : DQ_RD6_RS5_DQ12<61, 5, (outs), (ins vsrc:$XT, memrix16:$dst),
2932                             "stxv $XT, $dst", IIC_LdStSTFD, []>;
2933  // Store DWord
2934  def STXSD  : DSForm_1<61, 2, (outs), (ins vfrc:$vS, memrix:$dst),
2935                        "stxsd $vS, $dst", IIC_LdStSTFD, []>;
2936  // Convert DP of dword[0] to SP, and Store to dst
2937  def STXSSP : DSForm_1<61, 3, (outs), (ins vfrc:$vS, memrix:$dst),
2938                        "stxssp $vS, $dst", IIC_LdStSTFD, []>;
2939
2940  // [PO S RA RB XO SX]
2941  class X_XS6_RA5_RB5<bits<6> opcode, bits<10> xo, string opc,
2942                      RegisterOperand vtype, list<dag> pattern>
2943    : XX1Form_memOp<opcode, xo, (outs), (ins vtype:$XT, memrr:$dst),
2944              !strconcat(opc, " $XT, $dst"), IIC_LdStSTFD, pattern>;
2945
2946  // Store as Integer Byte/Halfword Indexed
2947  def STXSIBX  : X_XS6_RA5_RB5<31,  909, "stxsibx" , vsfrc,
2948                               [(PPCstxsix f64:$XT, xoaddr:$dst, 1)]>;
2949  def STXSIHX  : X_XS6_RA5_RB5<31,  941, "stxsihx" , vsfrc,
2950                               [(PPCstxsix f64:$XT, xoaddr:$dst, 2)]>;
2951  let isCodeGenOnly = 1 in {
2952    def STXSIBXv  : X_XS6_RA5_RB5<31,  909, "stxsibx" , vsrc, []>;
2953    def STXSIHXv  : X_XS6_RA5_RB5<31,  941, "stxsihx" , vsrc, []>;
2954  }
2955
2956  // Store Vector Halfword*8/Byte*16 Indexed
2957  def STXVH8X  : X_XS6_RA5_RB5<31,  940, "stxvh8x" , vsrc, []>;
2958  def STXVB16X : X_XS6_RA5_RB5<31, 1004, "stxvb16x", vsrc, []>;
2959
2960  // Store Vector Indexed
2961  def STXVX    : X_XS6_RA5_RB5<31,  396, "stxvx"   , vsrc,
2962                 [(store v2f64:$XT, xaddrX16:$dst)]>;
2963
2964  // Store Vector (Left-justified) with Length
2965  def STXVL : XX1Form_memOp<31, 397, (outs),
2966                            (ins vsrc:$XT, memr:$dst, g8rc:$rB),
2967                            "stxvl $XT, $dst, $rB", IIC_LdStLoad,
2968                            [(int_ppc_vsx_stxvl v4i32:$XT, addr:$dst,
2969                              i64:$rB)]>;
2970  def STXVLL : XX1Form_memOp<31, 429, (outs),
2971                            (ins vsrc:$XT, memr:$dst, g8rc:$rB),
2972                            "stxvll $XT, $dst, $rB", IIC_LdStLoad,
2973                            [(int_ppc_vsx_stxvll v4i32:$XT, addr:$dst,
2974                              i64:$rB)]>;
2975  } // mayStore
2976
2977  let Predicates = [IsLittleEndian] in {
2978  def: Pat<(f32 (PPCfcfids (f64 (PPCmtvsra (i32 (extractelt v4i32:$A, 0)))))),
2979           (f32 (XSCVSPDPN (XVCVSXWSP (XXSPLTW $A, 3))))>;
2980  def: Pat<(f32 (PPCfcfids (f64 (PPCmtvsra (i32 (extractelt v4i32:$A, 1)))))),
2981           (f32 (XSCVSPDPN (XVCVSXWSP (XXSPLTW $A, 2))))>;
2982  def: Pat<(f32 (PPCfcfids (f64 (PPCmtvsra (i32 (extractelt v4i32:$A, 2)))))),
2983           (f32 (XSCVSPDPN (XVCVSXWSP (XXSPLTW $A, 1))))>;
2984  def: Pat<(f32 (PPCfcfids (f64 (PPCmtvsra (i32 (extractelt v4i32:$A, 3)))))),
2985           (f32 (XSCVSPDPN (XVCVSXWSP (XXSPLTW $A, 0))))>;
2986  def: Pat<(f64 (PPCfcfid (f64 (PPCmtvsra (i32 (extractelt v4i32:$A, 0)))))),
2987           (f64 (COPY_TO_REGCLASS (XVCVSXWDP (XXSPLTW $A, 3)), VSFRC))>;
2988  def: Pat<(f64 (PPCfcfid (f64 (PPCmtvsra (i32 (extractelt v4i32:$A, 1)))))),
2989           (f64 (COPY_TO_REGCLASS (XVCVSXWDP (XXSPLTW $A, 2)), VSFRC))>;
2990  def: Pat<(f64 (PPCfcfid (f64 (PPCmtvsra (i32 (extractelt v4i32:$A, 2)))))),
2991           (f64 (COPY_TO_REGCLASS (XVCVSXWDP (XXSPLTW $A, 1)), VSFRC))>;
2992  def: Pat<(f64 (PPCfcfid (f64 (PPCmtvsra (i32 (extractelt v4i32:$A, 3)))))),
2993           (f64 (COPY_TO_REGCLASS (XVCVSXWDP (XXSPLTW $A, 0)), VSFRC))>;
2994  }
2995
2996  let Predicates = [IsBigEndian] in {
2997  def: Pat<(f32 (PPCfcfids (f64 (PPCmtvsra (i32 (extractelt v4i32:$A, 0)))))),
2998           (f32 (XSCVSPDPN (XVCVSXWSP (XXSPLTW $A, 0))))>;
2999  def: Pat<(f32 (PPCfcfids (f64 (PPCmtvsra (i32 (extractelt v4i32:$A, 1)))))),
3000           (f32 (XSCVSPDPN (XVCVSXWSP (XXSPLTW $A, 1))))>;
3001  def: Pat<(f32 (PPCfcfids (f64 (PPCmtvsra (i32 (extractelt v4i32:$A, 2)))))),
3002           (f32 (XSCVSPDPN (XVCVSXWSP (XXSPLTW $A, 2))))>;
3003  def: Pat<(f32 (PPCfcfids (f64 (PPCmtvsra (i32 (extractelt v4i32:$A, 3)))))),
3004           (f32 (XSCVSPDPN (XVCVSXWSP (XXSPLTW $A, 3))))>;
3005  def: Pat<(f64 (PPCfcfid (f64 (PPCmtvsra (i32 (extractelt v4i32:$A, 0)))))),
3006           (f64 (COPY_TO_REGCLASS (XVCVSXWDP (XXSPLTW $A, 0)), VSFRC))>;
3007  def: Pat<(f64 (PPCfcfid (f64 (PPCmtvsra (i32 (extractelt v4i32:$A, 1)))))),
3008           (f64 (COPY_TO_REGCLASS (XVCVSXWDP (XXSPLTW $A, 1)), VSFRC))>;
3009  def: Pat<(f64 (PPCfcfid (f64 (PPCmtvsra (i32 (extractelt v4i32:$A, 2)))))),
3010           (f64 (COPY_TO_REGCLASS (XVCVSXWDP (XXSPLTW $A, 2)), VSFRC))>;
3011  def: Pat<(f64 (PPCfcfid (f64 (PPCmtvsra (i32 (extractelt v4i32:$A, 3)))))),
3012           (f64 (COPY_TO_REGCLASS (XVCVSXWDP (XXSPLTW $A, 3)), VSFRC))>;
3013  }
3014
3015  // Alternate patterns for PPCmtvsrz where the output is v8i16 or v16i8 instead
3016  // of f64
3017  def : Pat<(v8i16 (PPCmtvsrz i32:$A)),
3018            (v8i16 (SUBREG_TO_REG (i64 1), (MTVSRWZ $A), sub_64))>;
3019  def : Pat<(v16i8 (PPCmtvsrz i32:$A)),
3020            (v16i8 (SUBREG_TO_REG (i64 1), (MTVSRWZ $A), sub_64))>;
3021
3022  // Patterns for which instructions from ISA 3.0 are a better match
3023  let Predicates = [IsLittleEndian, HasP9Vector] in {
3024  def : Pat<(f32 (PPCfcfidus (f64 (PPCmtvsrz (i32 (extractelt v4i32:$A, 0)))))),
3025            (f32 (XSCVUXDSP (XXEXTRACTUW $A, 12)))>;
3026  def : Pat<(f32 (PPCfcfidus (f64 (PPCmtvsrz (i32 (extractelt v4i32:$A, 1)))))),
3027            (f32 (XSCVUXDSP (XXEXTRACTUW $A, 8)))>;
3028  def : Pat<(f32 (PPCfcfidus (f64 (PPCmtvsrz (i32 (extractelt v4i32:$A, 2)))))),
3029            (f32 (XSCVUXDSP (XXEXTRACTUW $A, 4)))>;
3030  def : Pat<(f32 (PPCfcfidus (f64 (PPCmtvsrz (i32 (extractelt v4i32:$A, 3)))))),
3031            (f32 (XSCVUXDSP (XXEXTRACTUW $A, 0)))>;
3032  def : Pat<(f64 (PPCfcfidu (f64 (PPCmtvsrz (i32 (extractelt v4i32:$A, 0)))))),
3033            (f64 (XSCVUXDDP (XXEXTRACTUW $A, 12)))>;
3034  def : Pat<(f64 (PPCfcfidu (f64 (PPCmtvsrz (i32 (extractelt v4i32:$A, 1)))))),
3035            (f64 (XSCVUXDDP (XXEXTRACTUW $A, 8)))>;
3036  def : Pat<(f64 (PPCfcfidu (f64 (PPCmtvsrz (i32 (extractelt v4i32:$A, 2)))))),
3037            (f64 (XSCVUXDDP (XXEXTRACTUW $A, 4)))>;
3038  def : Pat<(f64 (PPCfcfidu (f64 (PPCmtvsrz (i32 (extractelt v4i32:$A, 3)))))),
3039            (f64 (XSCVUXDDP (XXEXTRACTUW $A, 0)))>;
3040  def : Pat<(v4i32 (insertelt v4i32:$A, i32:$B, 0)),
3041            (v4i32 (XXINSERTW v4i32:$A, AlignValues.I32_TO_BE_WORD1, 12))>;
3042  def : Pat<(v4i32 (insertelt v4i32:$A, i32:$B, 1)),
3043            (v4i32 (XXINSERTW v4i32:$A, AlignValues.I32_TO_BE_WORD1, 8))>;
3044  def : Pat<(v4i32 (insertelt v4i32:$A, i32:$B, 2)),
3045            (v4i32 (XXINSERTW v4i32:$A, AlignValues.I32_TO_BE_WORD1, 4))>;
3046  def : Pat<(v4i32 (insertelt v4i32:$A, i32:$B, 3)),
3047            (v4i32 (XXINSERTW v4i32:$A, AlignValues.I32_TO_BE_WORD1, 0))>;
3048  def : Pat<(v4f32 (insertelt v4f32:$A, f32:$B, 0)),
3049            (v4f32 (XXINSERTW v4f32:$A, AlignValues.F32_TO_BE_WORD1, 12))>;
3050  def : Pat<(v4f32 (insertelt v4f32:$A, f32:$B, 1)),
3051            (v4f32 (XXINSERTW v4f32:$A, AlignValues.F32_TO_BE_WORD1, 8))>;
3052  def : Pat<(v4f32 (insertelt v4f32:$A, f32:$B, 2)),
3053            (v4f32 (XXINSERTW v4f32:$A, AlignValues.F32_TO_BE_WORD1, 4))>;
3054  def : Pat<(v4f32 (insertelt v4f32:$A, f32:$B, 3)),
3055            (v4f32 (XXINSERTW v4f32:$A, AlignValues.F32_TO_BE_WORD1, 0))>;
3056
3057  def : Pat<(v8i16 (PPCld_vec_be xoaddr:$src)),
3058            (COPY_TO_REGCLASS (LXVH8X xoaddr:$src), VRRC)>;
3059  def : Pat<(PPCst_vec_be v8i16:$rS, xoaddr:$dst),
3060            (STXVH8X (COPY_TO_REGCLASS $rS, VSRC), xoaddr:$dst)>;
3061
3062  def : Pat<(v16i8 (PPCld_vec_be xoaddr:$src)),
3063            (COPY_TO_REGCLASS (LXVB16X xoaddr:$src), VRRC)>;
3064  def : Pat<(PPCst_vec_be v16i8:$rS, xoaddr:$dst),
3065            (STXVB16X (COPY_TO_REGCLASS $rS, VSRC), xoaddr:$dst)>;
3066  } // IsLittleEndian, HasP9Vector
3067
3068  let Predicates = [IsBigEndian, HasP9Vector] in {
3069  def : Pat<(f32 (PPCfcfidus (f64 (PPCmtvsrz (i32 (extractelt v4i32:$A, 0)))))),
3070            (f32 (XSCVUXDSP (XXEXTRACTUW $A, 0)))>;
3071  def : Pat<(f32 (PPCfcfidus (f64 (PPCmtvsrz (i32 (extractelt v4i32:$A, 1)))))),
3072            (f32 (XSCVUXDSP (XXEXTRACTUW $A, 4)))>;
3073  def : Pat<(f32 (PPCfcfidus (f64 (PPCmtvsrz (i32 (extractelt v4i32:$A, 2)))))),
3074            (f32 (XSCVUXDSP (XXEXTRACTUW $A, 8)))>;
3075  def : Pat<(f32 (PPCfcfidus (f64 (PPCmtvsrz (i32 (extractelt v4i32:$A, 3)))))),
3076            (f32 (XSCVUXDSP (XXEXTRACTUW $A, 12)))>;
3077  def : Pat<(f64 (PPCfcfidu (f64 (PPCmtvsrz (i32 (extractelt v4i32:$A, 0)))))),
3078            (f64 (XSCVUXDDP (XXEXTRACTUW $A, 0)))>;
3079  def : Pat<(f64 (PPCfcfidu (f64 (PPCmtvsrz (i32 (extractelt v4i32:$A, 1)))))),
3080            (f64 (XSCVUXDDP (XXEXTRACTUW $A, 4)))>;
3081  def : Pat<(f64 (PPCfcfidu (f64 (PPCmtvsrz (i32 (extractelt v4i32:$A, 2)))))),
3082            (f64 (XSCVUXDDP (XXEXTRACTUW $A, 8)))>;
3083  def : Pat<(f64 (PPCfcfidu (f64 (PPCmtvsrz (i32 (extractelt v4i32:$A, 3)))))),
3084            (f64 (XSCVUXDDP (XXEXTRACTUW $A, 12)))>;
3085  def : Pat<(v4i32 (insertelt v4i32:$A, i32:$B, 0)),
3086            (v4i32 (XXINSERTW v4i32:$A, AlignValues.I32_TO_BE_WORD1, 0))>;
3087  def : Pat<(v4i32 (insertelt v4i32:$A, i32:$B, 1)),
3088            (v4i32 (XXINSERTW v4i32:$A, AlignValues.I32_TO_BE_WORD1, 4))>;
3089  def : Pat<(v4i32 (insertelt v4i32:$A, i32:$B, 2)),
3090            (v4i32 (XXINSERTW v4i32:$A, AlignValues.I32_TO_BE_WORD1, 8))>;
3091  def : Pat<(v4i32 (insertelt v4i32:$A, i32:$B, 3)),
3092            (v4i32 (XXINSERTW v4i32:$A, AlignValues.I32_TO_BE_WORD1, 12))>;
3093  def : Pat<(v4f32 (insertelt v4f32:$A, f32:$B, 0)),
3094            (v4f32 (XXINSERTW v4f32:$A, AlignValues.F32_TO_BE_WORD1, 0))>;
3095  def : Pat<(v4f32 (insertelt v4f32:$A, f32:$B, 1)),
3096            (v4f32 (XXINSERTW v4f32:$A, AlignValues.F32_TO_BE_WORD1, 4))>;
3097  def : Pat<(v4f32 (insertelt v4f32:$A, f32:$B, 2)),
3098            (v4f32 (XXINSERTW v4f32:$A, AlignValues.F32_TO_BE_WORD1, 8))>;
3099  def : Pat<(v4f32 (insertelt v4f32:$A, f32:$B, 3)),
3100            (v4f32 (XXINSERTW v4f32:$A, AlignValues.F32_TO_BE_WORD1, 12))>;
3101  } // IsBigEndian, HasP9Vector
3102
3103  // D-Form Load/Store
3104  def : Pat<(v4i32 (quadwOffsetLoad iaddrX16:$src)), (LXV memrix16:$src)>;
3105  def : Pat<(v4f32 (quadwOffsetLoad iaddrX16:$src)), (LXV memrix16:$src)>;
3106  def : Pat<(v2i64 (quadwOffsetLoad iaddrX16:$src)), (LXV memrix16:$src)>;
3107  def : Pat<(v2f64 (quadwOffsetLoad iaddrX16:$src)), (LXV memrix16:$src)>;
3108  def : Pat<(f128  (quadwOffsetLoad iaddrX16:$src)),
3109            (COPY_TO_REGCLASS (LXV memrix16:$src), VRRC)>;
3110  def : Pat<(v4i32 (int_ppc_vsx_lxvw4x iaddrX16:$src)), (LXV memrix16:$src)>;
3111  def : Pat<(v2f64 (int_ppc_vsx_lxvd2x iaddrX16:$src)), (LXV memrix16:$src)>;
3112
3113  def : Pat<(quadwOffsetStore v4f32:$rS, iaddrX16:$dst), (STXV $rS, memrix16:$dst)>;
3114  def : Pat<(quadwOffsetStore v4i32:$rS, iaddrX16:$dst), (STXV $rS, memrix16:$dst)>;
3115  def : Pat<(quadwOffsetStore v2f64:$rS, iaddrX16:$dst), (STXV $rS, memrix16:$dst)>;
3116  def : Pat<(quadwOffsetStore  f128:$rS, iaddrX16:$dst),
3117            (STXV (COPY_TO_REGCLASS $rS, VSRC), memrix16:$dst)>;
3118  def : Pat<(quadwOffsetStore v2i64:$rS, iaddrX16:$dst), (STXV $rS, memrix16:$dst)>;
3119  def : Pat<(int_ppc_vsx_stxvw4x v4i32:$rS, iaddrX16:$dst),
3120            (STXV $rS, memrix16:$dst)>;
3121  def : Pat<(int_ppc_vsx_stxvd2x v2f64:$rS, iaddrX16:$dst),
3122            (STXV $rS, memrix16:$dst)>;
3123
3124
3125  def : Pat<(v2f64 (nonQuadwOffsetLoad xoaddr:$src)), (LXVX xoaddr:$src)>;
3126  def : Pat<(v2i64 (nonQuadwOffsetLoad xoaddr:$src)), (LXVX xoaddr:$src)>;
3127  def : Pat<(v4f32 (nonQuadwOffsetLoad xoaddr:$src)), (LXVX xoaddr:$src)>;
3128  def : Pat<(v4i32 (nonQuadwOffsetLoad xoaddr:$src)), (LXVX xoaddr:$src)>;
3129  def : Pat<(v4i32 (int_ppc_vsx_lxvw4x xoaddr:$src)), (LXVX xoaddr:$src)>;
3130  def : Pat<(v2f64 (int_ppc_vsx_lxvd2x xoaddr:$src)), (LXVX xoaddr:$src)>;
3131  def : Pat<(f128  (nonQuadwOffsetLoad xoaddr:$src)),
3132            (COPY_TO_REGCLASS (LXVX xoaddr:$src), VRRC)>;
3133  def : Pat<(nonQuadwOffsetStore f128:$rS, xoaddr:$dst),
3134            (STXVX (COPY_TO_REGCLASS $rS, VSRC), xoaddr:$dst)>;
3135  def : Pat<(nonQuadwOffsetStore v2f64:$rS, xoaddr:$dst),
3136            (STXVX $rS, xoaddr:$dst)>;
3137  def : Pat<(nonQuadwOffsetStore v2i64:$rS, xoaddr:$dst),
3138            (STXVX $rS, xoaddr:$dst)>;
3139  def : Pat<(nonQuadwOffsetStore v4f32:$rS, xoaddr:$dst),
3140            (STXVX $rS, xoaddr:$dst)>;
3141  def : Pat<(nonQuadwOffsetStore v4i32:$rS, xoaddr:$dst),
3142            (STXVX $rS, xoaddr:$dst)>;
3143  def : Pat<(int_ppc_vsx_stxvw4x v4i32:$rS, xoaddr:$dst),
3144            (STXVX $rS, xoaddr:$dst)>;
3145  def : Pat<(int_ppc_vsx_stxvd2x v2f64:$rS, xoaddr:$dst),
3146            (STXVX $rS, xoaddr:$dst)>;
3147
3148  let AddedComplexity = 400 in {
3149    // LIWAX - This instruction is used for sign extending i32 -> i64.
3150    // LIWZX - This instruction will be emitted for i32, f32, and when
3151    //         zero-extending i32 to i64 (zext i32 -> i64).
3152    let Predicates = [IsLittleEndian] in {
3153
3154      def : Pat<(v2i64 (scalar_to_vector (i64 (sextloadi32 xoaddr:$src)))),
3155                (v2i64 (XXPERMDIs
3156                (COPY_TO_REGCLASS (LIWAX xoaddr:$src), VSRC), 2))>;
3157
3158      def : Pat<(v2i64 (scalar_to_vector (i64 (zextloadi32 xoaddr:$src)))),
3159                (v2i64 (XXPERMDIs
3160                (COPY_TO_REGCLASS (LIWZX xoaddr:$src), VSRC), 2))>;
3161
3162      def : Pat<(v4i32 (scalar_to_vector (i32 (load xoaddr:$src)))),
3163                (v4i32 (XXPERMDIs
3164                (COPY_TO_REGCLASS (LIWZX xoaddr:$src), VSRC), 2))>;
3165
3166      def : Pat<(v4f32 (scalar_to_vector (f32 (load xoaddr:$src)))),
3167                (v4f32 (XXPERMDIs
3168                (COPY_TO_REGCLASS (LIWZX xoaddr:$src), VSRC), 2))>;
3169    }
3170
3171    let Predicates = [IsBigEndian] in {
3172      def : Pat<(v2i64 (scalar_to_vector (i64 (sextloadi32 xoaddr:$src)))),
3173                (v2i64 (COPY_TO_REGCLASS (LIWAX xoaddr:$src), VSRC))>;
3174
3175      def : Pat<(v2i64 (scalar_to_vector (i64 (zextloadi32 xoaddr:$src)))),
3176                (v2i64 (COPY_TO_REGCLASS (LIWZX xoaddr:$src), VSRC))>;
3177
3178      def : Pat<(v4i32 (scalar_to_vector (i32 (load xoaddr:$src)))),
3179                (v4i32 (XXSLDWIs
3180                (COPY_TO_REGCLASS (LIWZX xoaddr:$src), VSRC), 1))>;
3181
3182      def : Pat<(v4f32 (scalar_to_vector (f32 (load xoaddr:$src)))),
3183                (v4f32 (XXSLDWIs
3184                (COPY_TO_REGCLASS (LIWZX xoaddr:$src), VSRC), 1))>;
3185    }
3186
3187  }
3188
3189  // Build vectors from i8 loads
3190  def : Pat<(v16i8 (scalar_to_vector ScalarLoads.Li8)),
3191            (v16i8 (VSPLTBs 7, (LXSIBZX xoaddr:$src)))>;
3192  def : Pat<(v8i16 (scalar_to_vector ScalarLoads.ZELi8)),
3193            (v8i16 (VSPLTHs 3, (LXSIBZX xoaddr:$src)))>;
3194  def : Pat<(v4i32 (scalar_to_vector ScalarLoads.ZELi8)),
3195           (v4i32 (XXSPLTWs (LXSIBZX xoaddr:$src), 1))>;
3196  def : Pat<(v2i64 (scalar_to_vector ScalarLoads.ZELi8i64)),
3197            (v2i64 (XXPERMDIs (LXSIBZX xoaddr:$src), 0))>;
3198  def : Pat<(v4i32 (scalar_to_vector ScalarLoads.SELi8)),
3199            (v4i32 (XXSPLTWs (VEXTSB2Ws (LXSIBZX xoaddr:$src)), 1))>;
3200  def : Pat<(v2i64 (scalar_to_vector ScalarLoads.SELi8i64)),
3201            (v2i64 (XXPERMDIs (VEXTSB2Ds (LXSIBZX xoaddr:$src)), 0))>;
3202
3203  // Build vectors from i16 loads
3204  def : Pat<(v8i16 (scalar_to_vector ScalarLoads.Li16)),
3205            (v8i16 (VSPLTHs 3, (LXSIHZX xoaddr:$src)))>;
3206  def : Pat<(v4i32 (scalar_to_vector ScalarLoads.ZELi16)),
3207            (v4i32 (XXSPLTWs (LXSIHZX xoaddr:$src), 1))>;
3208  def : Pat<(v2i64 (scalar_to_vector ScalarLoads.ZELi16i64)),
3209           (v2i64 (XXPERMDIs (LXSIHZX xoaddr:$src), 0))>;
3210  def : Pat<(v4i32 (scalar_to_vector ScalarLoads.SELi16)),
3211            (v4i32 (XXSPLTWs (VEXTSH2Ws (LXSIHZX xoaddr:$src)), 1))>;
3212  def : Pat<(v2i64 (scalar_to_vector ScalarLoads.SELi16i64)),
3213            (v2i64 (XXPERMDIs (VEXTSH2Ds (LXSIHZX xoaddr:$src)), 0))>;
3214
3215  let Predicates = [IsBigEndian, HasP9Vector] in {
3216  // Scalar stores of i8
3217  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 0)), xoaddr:$dst),
3218            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 9)), VSRC), xoaddr:$dst)>;
3219  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 1)), xoaddr:$dst),
3220            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 10)), VSRC), xoaddr:$dst)>;
3221  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 2)), xoaddr:$dst),
3222            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 11)), VSRC), xoaddr:$dst)>;
3223  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 3)), xoaddr:$dst),
3224            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 12)), VSRC), xoaddr:$dst)>;
3225  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 4)), xoaddr:$dst),
3226            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 13)), VSRC), xoaddr:$dst)>;
3227  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 5)), xoaddr:$dst),
3228            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 14)), VSRC), xoaddr:$dst)>;
3229  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 6)), xoaddr:$dst),
3230            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 15)), VSRC), xoaddr:$dst)>;
3231  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 7)), xoaddr:$dst),
3232            (STXSIBXv (COPY_TO_REGCLASS $S, VSRC), xoaddr:$dst)>;
3233  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 8)), xoaddr:$dst),
3234            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 1)), VSRC), xoaddr:$dst)>;
3235  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 9)), xoaddr:$dst),
3236            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 2)), VSRC), xoaddr:$dst)>;
3237  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 10)), xoaddr:$dst),
3238            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 3)), VSRC), xoaddr:$dst)>;
3239  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 11)), xoaddr:$dst),
3240            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 4)), VSRC), xoaddr:$dst)>;
3241  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 12)), xoaddr:$dst),
3242            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 5)), VSRC), xoaddr:$dst)>;
3243  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 13)), xoaddr:$dst),
3244            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 6)), VSRC), xoaddr:$dst)>;
3245  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 14)), xoaddr:$dst),
3246            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 7)), VSRC), xoaddr:$dst)>;
3247  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 15)), xoaddr:$dst),
3248            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 8)), VSRC), xoaddr:$dst)>;
3249
3250  // Scalar stores of i16
3251  def : Pat<(truncstorei16 (i32 (vector_extract v8i16:$S, 0)), xoaddr:$dst),
3252            (STXSIHXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 10)), VSRC), xoaddr:$dst)>;
3253  def : Pat<(truncstorei16 (i32 (vector_extract v8i16:$S, 1)), xoaddr:$dst),
3254            (STXSIHXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 12)), VSRC), xoaddr:$dst)>;
3255  def : Pat<(truncstorei16 (i32 (vector_extract v8i16:$S, 2)), xoaddr:$dst),
3256            (STXSIHXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 14)), VSRC), xoaddr:$dst)>;
3257  def : Pat<(truncstorei16 (i32 (vector_extract v8i16:$S, 3)), xoaddr:$dst),
3258            (STXSIHXv (COPY_TO_REGCLASS $S, VSRC), xoaddr:$dst)>;
3259  def : Pat<(truncstorei16 (i32 (vector_extract v8i16:$S, 4)), xoaddr:$dst),
3260            (STXSIHXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 2)), VSRC), xoaddr:$dst)>;
3261  def : Pat<(truncstorei16 (i32 (vector_extract v8i16:$S, 5)), xoaddr:$dst),
3262            (STXSIHXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 4)), VSRC), xoaddr:$dst)>;
3263  def : Pat<(truncstorei16 (i32 (vector_extract v8i16:$S, 6)), xoaddr:$dst),
3264            (STXSIHXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 6)), VSRC), xoaddr:$dst)>;
3265  def : Pat<(truncstorei16 (i32 (vector_extract v8i16:$S, 7)), xoaddr:$dst),
3266            (STXSIHXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 8)), VSRC), xoaddr:$dst)>;
3267  } // IsBigEndian, HasP9Vector
3268
3269  let Predicates = [IsLittleEndian, HasP9Vector] in {
3270  // Scalar stores of i8
3271  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 0)), xoaddr:$dst),
3272            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 8)), VSRC), xoaddr:$dst)>;
3273  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 1)), xoaddr:$dst),
3274            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 7)), VSRC), xoaddr:$dst)>;
3275  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 2)), xoaddr:$dst),
3276            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 6)), VSRC), xoaddr:$dst)>;
3277  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 3)), xoaddr:$dst),
3278            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 5)), VSRC), xoaddr:$dst)>;
3279  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 4)), xoaddr:$dst),
3280            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 4)), VSRC), xoaddr:$dst)>;
3281  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 5)), xoaddr:$dst),
3282            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 3)), VSRC), xoaddr:$dst)>;
3283  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 6)), xoaddr:$dst),
3284            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 2)), VSRC), xoaddr:$dst)>;
3285  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 7)), xoaddr:$dst),
3286            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 1)), VSRC), xoaddr:$dst)>;
3287  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 8)), xoaddr:$dst),
3288            (STXSIBXv (COPY_TO_REGCLASS $S, VSRC), xoaddr:$dst)>;
3289  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 9)), xoaddr:$dst),
3290            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 15)), VSRC), xoaddr:$dst)>;
3291  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 10)), xoaddr:$dst),
3292            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 14)), VSRC), xoaddr:$dst)>;
3293  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 11)), xoaddr:$dst),
3294            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 13)), VSRC), xoaddr:$dst)>;
3295  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 12)), xoaddr:$dst),
3296            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 12)), VSRC), xoaddr:$dst)>;
3297  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 13)), xoaddr:$dst),
3298            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 11)), VSRC), xoaddr:$dst)>;
3299  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 14)), xoaddr:$dst),
3300            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 10)), VSRC), xoaddr:$dst)>;
3301  def : Pat<(truncstorei8 (i32 (vector_extract v16i8:$S, 15)), xoaddr:$dst),
3302            (STXSIBXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 9)), VSRC), xoaddr:$dst)>;
3303
3304  // Scalar stores of i16
3305  def : Pat<(truncstorei16 (i32 (vector_extract v8i16:$S, 0)), xoaddr:$dst),
3306            (STXSIHXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 8)), VSRC), xoaddr:$dst)>;
3307  def : Pat<(truncstorei16 (i32 (vector_extract v8i16:$S, 1)), xoaddr:$dst),
3308            (STXSIHXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 6)), VSRC), xoaddr:$dst)>;
3309  def : Pat<(truncstorei16 (i32 (vector_extract v8i16:$S, 2)), xoaddr:$dst),
3310            (STXSIHXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 4)), VSRC), xoaddr:$dst)>;
3311  def : Pat<(truncstorei16 (i32 (vector_extract v8i16:$S, 3)), xoaddr:$dst),
3312            (STXSIHXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 2)), VSRC), xoaddr:$dst)>;
3313  def : Pat<(truncstorei16 (i32 (vector_extract v8i16:$S, 4)), xoaddr:$dst),
3314            (STXSIHXv (COPY_TO_REGCLASS $S, VSRC), xoaddr:$dst)>;
3315  def : Pat<(truncstorei16 (i32 (vector_extract v8i16:$S, 5)), xoaddr:$dst),
3316            (STXSIHXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 14)), VSRC), xoaddr:$dst)>;
3317  def : Pat<(truncstorei16 (i32 (vector_extract v8i16:$S, 6)), xoaddr:$dst),
3318            (STXSIHXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 12)), VSRC), xoaddr:$dst)>;
3319  def : Pat<(truncstorei16 (i32 (vector_extract v8i16:$S, 7)), xoaddr:$dst),
3320            (STXSIHXv (COPY_TO_REGCLASS (v16i8 (VSLDOI $S, $S, 10)), VSRC), xoaddr:$dst)>;
3321  } // IsLittleEndian, HasP9Vector
3322
3323
3324  // Vector sign extensions
3325  def : Pat<(f64 (PPCVexts f64:$A, 1)),
3326            (f64 (COPY_TO_REGCLASS (VEXTSB2Ds $A), VSFRC))>;
3327  def : Pat<(f64 (PPCVexts f64:$A, 2)),
3328            (f64 (COPY_TO_REGCLASS (VEXTSH2Ds $A), VSFRC))>;
3329
3330  def DFLOADf32  : PPCPostRAExpPseudo<(outs vssrc:$XT), (ins memrix:$src),
3331                          "#DFLOADf32",
3332                          [(set f32:$XT, (load iaddrX4:$src))]>;
3333  def DFLOADf64  : PPCPostRAExpPseudo<(outs vsfrc:$XT), (ins memrix:$src),
3334                          "#DFLOADf64",
3335                          [(set f64:$XT, (load iaddrX4:$src))]>;
3336  def DFSTOREf32 : PPCPostRAExpPseudo<(outs), (ins vssrc:$XT, memrix:$dst),
3337                          "#DFSTOREf32",
3338                          [(store f32:$XT, iaddrX4:$dst)]>;
3339  def DFSTOREf64 : PPCPostRAExpPseudo<(outs), (ins vsfrc:$XT, memrix:$dst),
3340                          "#DFSTOREf64",
3341                          [(store f64:$XT, iaddrX4:$dst)]>;
3342
3343  def : Pat<(f64 (extloadf32 iaddrX4:$src)),
3344            (COPY_TO_REGCLASS (DFLOADf32 iaddrX4:$src), VSFRC)>;
3345  def : Pat<(f32 (fpround (f64 (extloadf32 iaddrX4:$src)))),
3346            (f32 (DFLOADf32 iaddrX4:$src))>;
3347
3348  def : Pat<(v4f32 (PPCldvsxlh xaddr:$src)),
3349            (COPY_TO_REGCLASS (XFLOADf64 xaddr:$src), VSRC)>;
3350  def : Pat<(v4f32 (PPCldvsxlh iaddrX4:$src)),
3351            (COPY_TO_REGCLASS (DFLOADf64 iaddrX4:$src), VSRC)>;
3352
3353  let AddedComplexity = 400 in {
3354  // The following pseudoinstructions are used to ensure the utilization
3355  // of all 64 VSX registers.
3356    let Predicates = [IsLittleEndian, HasP9Vector] in {
3357      def : Pat<(v2i64 (scalar_to_vector (i64 (load iaddrX4:$src)))),
3358                (v2i64 (XXPERMDIs
3359                (COPY_TO_REGCLASS (DFLOADf64 iaddrX4:$src), VSRC), 2))>;
3360      def : Pat<(v2i64 (scalar_to_vector (i64 (load xaddrX4:$src)))),
3361                (v2i64 (XXPERMDIs
3362		(COPY_TO_REGCLASS (XFLOADf64 xaddrX4:$src), VSRC), 2))>;
3363
3364      def : Pat<(v2f64 (scalar_to_vector (f64 (load iaddrX4:$src)))),
3365                (v2f64 (XXPERMDIs
3366                (COPY_TO_REGCLASS (DFLOADf64 iaddrX4:$src), VSRC), 2))>;
3367      def : Pat<(v2f64 (scalar_to_vector (f64 (load xaddrX4:$src)))),
3368                (v2f64 (XXPERMDIs
3369                (COPY_TO_REGCLASS (XFLOADf64 xaddrX4:$src), VSRC), 2))>;
3370      def : Pat<(store (i64 (extractelt v2i64:$A, 0)), xaddrX4:$src),
3371                (XFSTOREf64 (EXTRACT_SUBREG (XXPERMDI $A, $A, 2),
3372                             sub_64), xaddrX4:$src)>;
3373      def : Pat<(store (f64 (extractelt v2f64:$A, 0)), xaddrX4:$src),
3374                (XFSTOREf64 (EXTRACT_SUBREG (XXPERMDI $A, $A, 2),
3375                             sub_64), xaddrX4:$src)>;
3376      def : Pat<(store (i64 (extractelt v2i64:$A, 1)), xaddrX4:$src),
3377                (XFSTOREf64 (EXTRACT_SUBREG $A, sub_64), xaddrX4:$src)>;
3378      def : Pat<(store (f64 (extractelt v2f64:$A, 1)), xaddrX4:$src),
3379                (XFSTOREf64 (EXTRACT_SUBREG $A, sub_64), xaddrX4:$src)>;
3380      def : Pat<(store (i64 (extractelt v2i64:$A, 0)), iaddrX4:$src),
3381                (DFSTOREf64 (EXTRACT_SUBREG (XXPERMDI $A, $A, 2),
3382                             sub_64), iaddrX4:$src)>;
3383      def : Pat<(store (f64 (extractelt v2f64:$A, 0)), iaddrX4:$src),
3384                (DFSTOREf64 (EXTRACT_SUBREG (XXPERMDI $A, $A, 2), sub_64),
3385                            iaddrX4:$src)>;
3386      def : Pat<(store (i64 (extractelt v2i64:$A, 1)), iaddrX4:$src),
3387                (DFSTOREf64 (EXTRACT_SUBREG $A, sub_64), iaddrX4:$src)>;
3388      def : Pat<(store (f64 (extractelt v2f64:$A, 1)), iaddrX4:$src),
3389                (DFSTOREf64 (EXTRACT_SUBREG $A, sub_64), iaddrX4:$src)>;
3390    } // IsLittleEndian, HasP9Vector
3391
3392    let Predicates = [IsBigEndian, HasP9Vector] in {
3393      def : Pat<(v2i64 (scalar_to_vector (i64 (load iaddrX4:$src)))),
3394                (v2i64 (COPY_TO_REGCLASS (DFLOADf64 iaddrX4:$src), VSRC))>;
3395      def : Pat<(v2i64 (scalar_to_vector (i64 (load xaddrX4:$src)))),
3396                (v2i64 (COPY_TO_REGCLASS (XFLOADf64 xaddrX4:$src), VSRC))>;
3397
3398      def : Pat<(v2f64 (scalar_to_vector (f64 (load iaddrX4:$src)))),
3399                (v2f64 (COPY_TO_REGCLASS (DFLOADf64 iaddrX4:$src), VSRC))>;
3400      def : Pat<(v2f64 (scalar_to_vector (f64 (load xaddrX4:$src)))),
3401                (v2f64 (COPY_TO_REGCLASS (XFLOADf64 xaddrX4:$src), VSRC))>;
3402      def : Pat<(store (i64 (extractelt v2i64:$A, 1)), xaddrX4:$src),
3403                (XFSTOREf64 (EXTRACT_SUBREG (XXPERMDI $A, $A, 2),
3404                             sub_64), xaddrX4:$src)>;
3405      def : Pat<(store (f64 (extractelt v2f64:$A, 1)), xaddrX4:$src),
3406                (XFSTOREf64 (EXTRACT_SUBREG (XXPERMDI $A, $A, 2),
3407                             sub_64), xaddrX4:$src)>;
3408      def : Pat<(store (i64 (extractelt v2i64:$A, 0)), xaddrX4:$src),
3409                (XFSTOREf64 (EXTRACT_SUBREG $A, sub_64), xaddrX4:$src)>;
3410      def : Pat<(store (f64 (extractelt v2f64:$A, 0)), xaddrX4:$src),
3411                (XFSTOREf64 (EXTRACT_SUBREG $A, sub_64), xaddrX4:$src)>;
3412      def : Pat<(store (i64 (extractelt v2i64:$A, 1)), iaddrX4:$src),
3413                (DFSTOREf64 (EXTRACT_SUBREG (XXPERMDI $A, $A, 2),
3414                             sub_64), iaddrX4:$src)>;
3415      def : Pat<(store (f64 (extractelt v2f64:$A, 1)), iaddrX4:$src),
3416                (DFSTOREf64 (EXTRACT_SUBREG (XXPERMDI $A, $A, 2),
3417                             sub_64), iaddrX4:$src)>;
3418      def : Pat<(store (i64 (extractelt v2i64:$A, 0)), iaddrX4:$src),
3419                (DFSTOREf64 (EXTRACT_SUBREG $A, sub_64), iaddrX4:$src)>;
3420      def : Pat<(store (f64 (extractelt v2f64:$A, 0)), iaddrX4:$src),
3421                (DFSTOREf64 (EXTRACT_SUBREG $A, sub_64), iaddrX4:$src)>;
3422    } // IsBigEndian, HasP9Vector
3423  }
3424
3425  let Predicates = [IsBigEndian, HasP9Vector] in {
3426
3427    // (Un)Signed DWord vector extract -> QP
3428    def : Pat<(f128 (sint_to_fp (i64 (extractelt v2i64:$src, 0)))),
3429              (f128 (XSCVSDQP (COPY_TO_REGCLASS $src, VFRC)))>;
3430    def : Pat<(f128 (sint_to_fp (i64 (extractelt v2i64:$src, 1)))),
3431              (f128 (XSCVSDQP
3432                      (EXTRACT_SUBREG (XXPERMDI $src, $src, 3), sub_64)))>;
3433    def : Pat<(f128 (uint_to_fp (i64 (extractelt v2i64:$src, 0)))),
3434              (f128 (XSCVUDQP (COPY_TO_REGCLASS $src, VFRC)))>;
3435    def : Pat<(f128 (uint_to_fp (i64 (extractelt v2i64:$src, 1)))),
3436              (f128 (XSCVUDQP
3437                      (EXTRACT_SUBREG (XXPERMDI $src, $src, 3), sub_64)))>;
3438
3439    // (Un)Signed Word vector extract -> QP
3440    def : Pat<(f128 (sint_to_fp (i32 (extractelt v4i32:$src, 1)))),
3441              (f128 (XSCVSDQP (EXTRACT_SUBREG (VEXTSW2D $src), sub_64)))>;
3442    foreach Idx = [0,2,3] in {
3443      def : Pat<(f128 (sint_to_fp (i32 (extractelt v4i32:$src, Idx)))),
3444                (f128 (XSCVSDQP (EXTRACT_SUBREG
3445                                (VEXTSW2D (VSPLTW Idx, $src)), sub_64)))>;
3446    }
3447    foreach Idx = 0-3 in {
3448      def : Pat<(f128 (uint_to_fp (i32 (extractelt v4i32:$src, Idx)))),
3449                (f128 (XSCVUDQP (XXEXTRACTUW $src, !shl(Idx, 2))))>;
3450    }
3451
3452    // (Un)Signed HWord vector extract -> QP
3453    foreach Idx = 0-7 in {
3454      def : Pat<(f128 (sint_to_fp
3455                        (i32 (sext_inreg
3456                               (vector_extract v8i16:$src, Idx), i16)))),
3457              (f128 (XSCVSDQP (EXTRACT_SUBREG
3458                                (VEXTSH2D (VEXTRACTUH !add(Idx, Idx), $src)),
3459                                sub_64)))>;
3460      // The SDAG adds the `and` since an `i16` is being extracted as an `i32`.
3461      def : Pat<(f128 (uint_to_fp
3462                        (and (i32 (vector_extract v8i16:$src, Idx)), 65535))),
3463                (f128 (XSCVUDQP (EXTRACT_SUBREG
3464                                  (VEXTRACTUH !add(Idx, Idx), $src), sub_64)))>;
3465    }
3466
3467    // (Un)Signed Byte vector extract -> QP
3468    foreach Idx = 0-15 in {
3469      def : Pat<(f128 (sint_to_fp
3470                        (i32 (sext_inreg (vector_extract v16i8:$src, Idx),
3471                                         i8)))),
3472                (f128 (XSCVSDQP (EXTRACT_SUBREG
3473                                  (VEXTSB2D (VEXTRACTUB Idx, $src)), sub_64)))>;
3474      def : Pat<(f128 (uint_to_fp
3475                        (and (i32 (vector_extract v16i8:$src, Idx)), 255))),
3476                (f128 (XSCVUDQP
3477                        (EXTRACT_SUBREG (VEXTRACTUB Idx, $src), sub_64)))>;
3478    }
3479
3480    // Unsiged int in vsx register -> QP
3481    def : Pat<(f128 (uint_to_fp (i32 (PPCmfvsr f64:$src)))),
3482              (f128 (XSCVUDQP
3483                      (XXEXTRACTUW (SUBREG_TO_REG (i64 1), $src, sub_64), 4)))>;
3484  } // IsBigEndian, HasP9Vector
3485
3486  let Predicates = [IsLittleEndian, HasP9Vector] in {
3487
3488    // (Un)Signed DWord vector extract -> QP
3489    def : Pat<(f128 (sint_to_fp (i64 (extractelt v2i64:$src, 0)))),
3490              (f128 (XSCVSDQP
3491                      (EXTRACT_SUBREG (XXPERMDI $src, $src, 3), sub_64)))>;
3492    def : Pat<(f128 (sint_to_fp (i64 (extractelt v2i64:$src, 1)))),
3493              (f128 (XSCVSDQP (COPY_TO_REGCLASS $src, VFRC)))>;
3494    def : Pat<(f128 (uint_to_fp (i64 (extractelt v2i64:$src, 0)))),
3495              (f128 (XSCVUDQP
3496                      (EXTRACT_SUBREG (XXPERMDI $src, $src, 3), sub_64)))>;
3497    def : Pat<(f128 (uint_to_fp (i64 (extractelt v2i64:$src, 1)))),
3498              (f128 (XSCVUDQP (COPY_TO_REGCLASS $src, VFRC)))>;
3499
3500    // (Un)Signed Word vector extract -> QP
3501    foreach Idx = [[0,3],[1,2],[3,0]] in {
3502      def : Pat<(f128 (sint_to_fp (i32 (extractelt v4i32:$src, !head(Idx))))),
3503                (f128 (XSCVSDQP (EXTRACT_SUBREG
3504                                  (VEXTSW2D (VSPLTW !head(!tail(Idx)), $src)),
3505                                  sub_64)))>;
3506    }
3507    def : Pat<(f128 (sint_to_fp (i32 (extractelt v4i32:$src, 2)))),
3508              (f128 (XSCVSDQP (EXTRACT_SUBREG (VEXTSW2D $src), sub_64)))>;
3509
3510    foreach Idx = [[0,12],[1,8],[2,4],[3,0]] in {
3511      def : Pat<(f128 (uint_to_fp (i32 (extractelt v4i32:$src, !head(Idx))))),
3512                (f128 (XSCVUDQP (XXEXTRACTUW $src, !head(!tail(Idx)))))>;
3513    }
3514
3515    // (Un)Signed HWord vector extract -> QP
3516    // The Nested foreach lists identifies the vector element and corresponding
3517    // register byte location.
3518    foreach Idx = [[0,14],[1,12],[2,10],[3,8],[4,6],[5,4],[6,2],[7,0]] in {
3519      def : Pat<(f128 (sint_to_fp
3520                        (i32 (sext_inreg
3521                               (vector_extract v8i16:$src, !head(Idx)), i16)))),
3522                (f128 (XSCVSDQP
3523                        (EXTRACT_SUBREG (VEXTSH2D
3524                                          (VEXTRACTUH !head(!tail(Idx)), $src)),
3525                                        sub_64)))>;
3526      def : Pat<(f128 (uint_to_fp
3527                        (and (i32 (vector_extract v8i16:$src, !head(Idx))),
3528                             65535))),
3529                (f128 (XSCVUDQP (EXTRACT_SUBREG
3530                                  (VEXTRACTUH !head(!tail(Idx)), $src), sub_64)))>;
3531    }
3532
3533    // (Un)Signed Byte vector extract -> QP
3534    foreach Idx = [[0,15],[1,14],[2,13],[3,12],[4,11],[5,10],[6,9],[7,8],[8,7],
3535                   [9,6],[10,5],[11,4],[12,3],[13,2],[14,1],[15,0]] in {
3536      def : Pat<(f128 (sint_to_fp
3537                        (i32 (sext_inreg
3538                               (vector_extract v16i8:$src, !head(Idx)), i8)))),
3539                (f128 (XSCVSDQP
3540                        (EXTRACT_SUBREG
3541                          (VEXTSB2D (VEXTRACTUB !head(!tail(Idx)), $src)),
3542                          sub_64)))>;
3543      def : Pat<(f128 (uint_to_fp
3544                        (and (i32 (vector_extract v16i8:$src, !head(Idx))),
3545                             255))),
3546                (f128 (XSCVUDQP
3547                        (EXTRACT_SUBREG
3548                          (VEXTRACTUB !head(!tail(Idx)), $src), sub_64)))>;
3549    }
3550
3551    // Unsiged int in vsx register -> QP
3552    def : Pat<(f128 (uint_to_fp (i32 (PPCmfvsr f64:$src)))),
3553              (f128 (XSCVUDQP
3554                      (XXEXTRACTUW (SUBREG_TO_REG (i64 1), $src, sub_64), 8)))>;
3555  } // IsLittleEndian, HasP9Vector
3556
3557  // Convert (Un)Signed DWord in memory -> QP
3558  def : Pat<(f128 (sint_to_fp (i64 (load xaddrX4:$src)))),
3559            (f128 (XSCVSDQP (LXSDX xaddrX4:$src)))>;
3560  def : Pat<(f128 (sint_to_fp (i64 (load iaddrX4:$src)))),
3561            (f128 (XSCVSDQP (LXSD iaddrX4:$src)))>;
3562  def : Pat<(f128 (uint_to_fp (i64 (load xaddrX4:$src)))),
3563            (f128 (XSCVUDQP (LXSDX xaddrX4:$src)))>;
3564  def : Pat<(f128 (uint_to_fp (i64 (load iaddrX4:$src)))),
3565            (f128 (XSCVUDQP (LXSD iaddrX4:$src)))>;
3566
3567  // Convert Unsigned HWord in memory -> QP
3568  def : Pat<(f128 (uint_to_fp ScalarLoads.ZELi16)),
3569            (f128 (XSCVUDQP (LXSIHZX xaddr:$src)))>;
3570
3571  // Convert Unsigned Byte in memory -> QP
3572  def : Pat<(f128 (uint_to_fp ScalarLoads.ZELi8)),
3573            (f128 (XSCVUDQP (LXSIBZX xoaddr:$src)))>;
3574
3575  // Truncate & Convert QP -> (Un)Signed (D)Word.
3576  def : Pat<(i64 (fp_to_sint f128:$src)), (i64 (MFVRD (XSCVQPSDZ $src)))>;
3577  def : Pat<(i64 (fp_to_uint f128:$src)), (i64 (MFVRD (XSCVQPUDZ $src)))>;
3578  def : Pat<(i32 (fp_to_sint f128:$src)),
3579            (i32 (MFVSRWZ (COPY_TO_REGCLASS (XSCVQPSWZ $src), VFRC)))>;
3580  def : Pat<(i32 (fp_to_uint f128:$src)),
3581            (i32 (MFVSRWZ (COPY_TO_REGCLASS (XSCVQPUWZ $src), VFRC)))>;
3582
3583  // Instructions for store(fptosi).
3584  // The 8-byte version is repeated here due to availability of D-Form STXSD.
3585  def : Pat<(PPCstore_scal_int_from_vsr
3586              (f64 (PPCcv_fp_to_sint_in_vsr f128:$src)), xaddrX4:$dst, 8),
3587            (STXSDX (COPY_TO_REGCLASS (XSCVQPSDZ f128:$src), VFRC),
3588                    xaddrX4:$dst)>;
3589  def : Pat<(PPCstore_scal_int_from_vsr
3590              (f64 (PPCcv_fp_to_sint_in_vsr f128:$src)), iaddrX4:$dst, 8),
3591            (STXSD (COPY_TO_REGCLASS (XSCVQPSDZ f128:$src), VFRC),
3592                   iaddrX4:$dst)>;
3593  def : Pat<(PPCstore_scal_int_from_vsr
3594              (f64 (PPCcv_fp_to_sint_in_vsr f128:$src)), xoaddr:$dst, 4),
3595            (STXSIWX (COPY_TO_REGCLASS (XSCVQPSWZ $src), VFRC), xoaddr:$dst)>;
3596  def : Pat<(PPCstore_scal_int_from_vsr
3597              (f64 (PPCcv_fp_to_sint_in_vsr f128:$src)), xoaddr:$dst, 2),
3598            (STXSIHX (COPY_TO_REGCLASS (XSCVQPSWZ $src), VFRC), xoaddr:$dst)>;
3599  def : Pat<(PPCstore_scal_int_from_vsr
3600              (f64 (PPCcv_fp_to_sint_in_vsr f128:$src)), xoaddr:$dst, 1),
3601            (STXSIBX (COPY_TO_REGCLASS (XSCVQPSWZ $src), VFRC), xoaddr:$dst)>;
3602  def : Pat<(PPCstore_scal_int_from_vsr
3603              (f64 (PPCcv_fp_to_sint_in_vsr f64:$src)), xaddrX4:$dst, 8),
3604            (STXSDX (XSCVDPSXDS f64:$src), xaddrX4:$dst)>;
3605  def : Pat<(PPCstore_scal_int_from_vsr
3606              (f64 (PPCcv_fp_to_sint_in_vsr f64:$src)), iaddrX4:$dst, 8),
3607            (STXSD (XSCVDPSXDS f64:$src), iaddrX4:$dst)>;
3608  def : Pat<(PPCstore_scal_int_from_vsr
3609              (f64 (PPCcv_fp_to_sint_in_vsr f64:$src)), xoaddr:$dst, 2),
3610            (STXSIHX (XSCVDPSXWS f64:$src), xoaddr:$dst)>;
3611  def : Pat<(PPCstore_scal_int_from_vsr
3612              (f64 (PPCcv_fp_to_sint_in_vsr f64:$src)), xoaddr:$dst, 1),
3613            (STXSIBX (XSCVDPSXWS f64:$src), xoaddr:$dst)>;
3614
3615  // Instructions for store(fptoui).
3616  def : Pat<(PPCstore_scal_int_from_vsr
3617              (f64 (PPCcv_fp_to_uint_in_vsr f128:$src)), xaddrX4:$dst, 8),
3618            (STXSDX (COPY_TO_REGCLASS (XSCVQPUDZ f128:$src), VFRC),
3619                    xaddrX4:$dst)>;
3620  def : Pat<(PPCstore_scal_int_from_vsr
3621              (f64 (PPCcv_fp_to_uint_in_vsr f128:$src)), iaddrX4:$dst, 8),
3622            (STXSD (COPY_TO_REGCLASS (XSCVQPUDZ f128:$src), VFRC),
3623                   iaddrX4:$dst)>;
3624  def : Pat<(PPCstore_scal_int_from_vsr
3625              (f64 (PPCcv_fp_to_uint_in_vsr f128:$src)), xoaddr:$dst, 4),
3626            (STXSIWX (COPY_TO_REGCLASS (XSCVQPUWZ $src), VFRC), xoaddr:$dst)>;
3627  def : Pat<(PPCstore_scal_int_from_vsr
3628              (f64 (PPCcv_fp_to_uint_in_vsr f128:$src)), xoaddr:$dst, 2),
3629            (STXSIHX (COPY_TO_REGCLASS (XSCVQPUWZ $src), VFRC), xoaddr:$dst)>;
3630  def : Pat<(PPCstore_scal_int_from_vsr
3631              (f64 (PPCcv_fp_to_uint_in_vsr f128:$src)), xoaddr:$dst, 1),
3632            (STXSIBX (COPY_TO_REGCLASS (XSCVQPUWZ $src), VFRC), xoaddr:$dst)>;
3633  def : Pat<(PPCstore_scal_int_from_vsr
3634              (f64 (PPCcv_fp_to_uint_in_vsr f64:$src)), xaddrX4:$dst, 8),
3635            (STXSDX (XSCVDPUXDS f64:$src), xaddrX4:$dst)>;
3636  def : Pat<(PPCstore_scal_int_from_vsr
3637              (f64 (PPCcv_fp_to_uint_in_vsr f64:$src)), iaddrX4:$dst, 8),
3638            (STXSD (XSCVDPUXDS f64:$src), iaddrX4:$dst)>;
3639  def : Pat<(PPCstore_scal_int_from_vsr
3640              (f64 (PPCcv_fp_to_uint_in_vsr f64:$src)), xoaddr:$dst, 2),
3641            (STXSIHX (XSCVDPUXWS f64:$src), xoaddr:$dst)>;
3642  def : Pat<(PPCstore_scal_int_from_vsr
3643              (f64 (PPCcv_fp_to_uint_in_vsr f64:$src)), xoaddr:$dst, 1),
3644            (STXSIBX (XSCVDPUXWS f64:$src), xoaddr:$dst)>;
3645
3646  // Round & Convert QP -> DP/SP
3647  def : Pat<(f64 (fpround f128:$src)), (f64 (XSCVQPDP $src))>;
3648  def : Pat<(f32 (fpround f128:$src)), (f32 (XSRSP (XSCVQPDPO $src)))>;
3649
3650  // Convert SP -> QP
3651  def : Pat<(f128 (fpextend f32:$src)),
3652            (f128 (XSCVDPQP (COPY_TO_REGCLASS $src, VFRC)))>;
3653
3654} // end HasP9Vector, AddedComplexity
3655
3656let AddedComplexity = 400 in {
3657  let Predicates = [IsISA3_0, HasP9Vector, HasDirectMove, IsBigEndian] in {
3658    def : Pat<(f128 (PPCbuild_fp128 i64:$rB, i64:$rA)),
3659              (f128 (COPY_TO_REGCLASS (MTVSRDD $rB, $rA), VRRC))>;
3660  }
3661  let Predicates = [IsISA3_0, HasP9Vector, HasDirectMove, IsLittleEndian] in {
3662    def : Pat<(f128 (PPCbuild_fp128 i64:$rA, i64:$rB)),
3663              (f128 (COPY_TO_REGCLASS (MTVSRDD $rB, $rA), VRRC))>;
3664  }
3665}
3666
3667let Predicates = [HasP9Vector] in {
3668  let mayStore = 1 in {
3669    def SPILLTOVSR_STX : PseudoXFormMemOp<(outs),
3670                                          (ins spilltovsrrc:$XT, memrr:$dst),
3671                                          "#SPILLTOVSR_STX", []>;
3672    def SPILLTOVSR_ST : PPCPostRAExpPseudo<(outs), (ins spilltovsrrc:$XT, memrix:$dst),
3673                              "#SPILLTOVSR_ST", []>;
3674  }
3675  let mayLoad = 1 in {
3676    def SPILLTOVSR_LDX : PseudoXFormMemOp<(outs spilltovsrrc:$XT),
3677                                          (ins memrr:$src),
3678                                          "#SPILLTOVSR_LDX", []>;
3679    def SPILLTOVSR_LD : PPCPostRAExpPseudo<(outs spilltovsrrc:$XT), (ins memrix:$src),
3680                              "#SPILLTOVSR_LD", []>;
3681
3682  }
3683}
3684// Integer extend helper dags 32 -> 64
3685def AnyExts {
3686  dag A = (INSERT_SUBREG (i64 (IMPLICIT_DEF)), $A, sub_32);
3687  dag B = (INSERT_SUBREG (i64 (IMPLICIT_DEF)), $B, sub_32);
3688  dag C = (INSERT_SUBREG (i64 (IMPLICIT_DEF)), $C, sub_32);
3689  dag D = (INSERT_SUBREG (i64 (IMPLICIT_DEF)), $D, sub_32);
3690}
3691
3692def DblToFlt {
3693  dag A0 = (f32 (fpround (f64 (extractelt v2f64:$A, 0))));
3694  dag A1 = (f32 (fpround (f64 (extractelt v2f64:$A, 1))));
3695  dag B0 = (f32 (fpround (f64 (extractelt v2f64:$B, 0))));
3696  dag B1 = (f32 (fpround (f64 (extractelt v2f64:$B, 1))));
3697}
3698
3699def ExtDbl {
3700  dag A0S = (i32 (PPCmfvsr (f64 (PPCfctiwz (f64 (extractelt v2f64:$A, 0))))));
3701  dag A1S = (i32 (PPCmfvsr (f64 (PPCfctiwz (f64 (extractelt v2f64:$A, 1))))));
3702  dag B0S = (i32 (PPCmfvsr (f64 (PPCfctiwz (f64 (extractelt v2f64:$B, 0))))));
3703  dag B1S = (i32 (PPCmfvsr (f64 (PPCfctiwz (f64 (extractelt v2f64:$B, 1))))));
3704  dag A0U = (i32 (PPCmfvsr (f64 (PPCfctiwuz (f64 (extractelt v2f64:$A, 0))))));
3705  dag A1U = (i32 (PPCmfvsr (f64 (PPCfctiwuz (f64 (extractelt v2f64:$A, 1))))));
3706  dag B0U = (i32 (PPCmfvsr (f64 (PPCfctiwuz (f64 (extractelt v2f64:$B, 0))))));
3707  dag B1U = (i32 (PPCmfvsr (f64 (PPCfctiwuz (f64 (extractelt v2f64:$B, 1))))));
3708}
3709
3710def ByteToWord {
3711  dag LE_A0 = (i32 (sext_inreg (i32 (vector_extract v16i8:$A, 0)), i8));
3712  dag LE_A1 = (i32 (sext_inreg (i32 (vector_extract v16i8:$A, 4)), i8));
3713  dag LE_A2 = (i32 (sext_inreg (i32 (vector_extract v16i8:$A, 8)), i8));
3714  dag LE_A3 = (i32 (sext_inreg (i32 (vector_extract v16i8:$A, 12)), i8));
3715  dag BE_A0 = (i32 (sext_inreg (i32 (vector_extract v16i8:$A, 3)), i8));
3716  dag BE_A1 = (i32 (sext_inreg (i32 (vector_extract v16i8:$A, 7)), i8));
3717  dag BE_A2 = (i32 (sext_inreg (i32 (vector_extract v16i8:$A, 11)), i8));
3718  dag BE_A3 = (i32 (sext_inreg (i32 (vector_extract v16i8:$A, 15)), i8));
3719}
3720
3721def ByteToDWord {
3722  dag LE_A0 = (i64 (sext_inreg
3723              (i64 (anyext (i32 (vector_extract v16i8:$A, 0)))), i8));
3724  dag LE_A1 = (i64 (sext_inreg
3725              (i64 (anyext (i32 (vector_extract v16i8:$A, 8)))), i8));
3726  dag BE_A0 = (i64 (sext_inreg
3727              (i64 (anyext (i32 (vector_extract v16i8:$A, 7)))), i8));
3728  dag BE_A1 = (i64 (sext_inreg
3729              (i64 (anyext (i32 (vector_extract v16i8:$A, 15)))), i8));
3730}
3731
3732def HWordToWord {
3733  dag LE_A0 = (i32 (sext_inreg (i32 (vector_extract v8i16:$A, 0)), i16));
3734  dag LE_A1 = (i32 (sext_inreg (i32 (vector_extract v8i16:$A, 2)), i16));
3735  dag LE_A2 = (i32 (sext_inreg (i32 (vector_extract v8i16:$A, 4)), i16));
3736  dag LE_A3 = (i32 (sext_inreg (i32 (vector_extract v8i16:$A, 6)), i16));
3737  dag BE_A0 = (i32 (sext_inreg (i32 (vector_extract v8i16:$A, 1)), i16));
3738  dag BE_A1 = (i32 (sext_inreg (i32 (vector_extract v8i16:$A, 3)), i16));
3739  dag BE_A2 = (i32 (sext_inreg (i32 (vector_extract v8i16:$A, 5)), i16));
3740  dag BE_A3 = (i32 (sext_inreg (i32 (vector_extract v8i16:$A, 7)), i16));
3741}
3742
3743def HWordToDWord {
3744  dag LE_A0 = (i64 (sext_inreg
3745              (i64 (anyext (i32 (vector_extract v8i16:$A, 0)))), i16));
3746  dag LE_A1 = (i64 (sext_inreg
3747              (i64 (anyext (i32 (vector_extract v8i16:$A, 4)))), i16));
3748  dag BE_A0 = (i64 (sext_inreg
3749              (i64 (anyext (i32 (vector_extract v8i16:$A, 3)))), i16));
3750  dag BE_A1 = (i64 (sext_inreg
3751              (i64 (anyext (i32 (vector_extract v8i16:$A, 7)))), i16));
3752}
3753
3754def WordToDWord {
3755  dag LE_A0 = (i64 (sext (i32 (vector_extract v4i32:$A, 0))));
3756  dag LE_A1 = (i64 (sext (i32 (vector_extract v4i32:$A, 2))));
3757  dag BE_A0 = (i64 (sext (i32 (vector_extract v4i32:$A, 1))));
3758  dag BE_A1 = (i64 (sext (i32 (vector_extract v4i32:$A, 3))));
3759}
3760
3761def FltToIntLoad {
3762  dag A = (i32 (PPCmfvsr (PPCfctiwz (f64 (extloadf32 xoaddr:$A)))));
3763}
3764def FltToUIntLoad {
3765  dag A = (i32 (PPCmfvsr (PPCfctiwuz (f64 (extloadf32 xoaddr:$A)))));
3766}
3767def FltToLongLoad {
3768  dag A = (i64 (PPCmfvsr (PPCfctidz (f64 (extloadf32 xoaddr:$A)))));
3769}
3770def FltToLongLoadP9 {
3771  dag A = (i64 (PPCmfvsr (PPCfctidz (f64 (extloadf32 iaddrX4:$A)))));
3772}
3773def FltToULongLoad {
3774  dag A = (i64 (PPCmfvsr (PPCfctiduz (f64 (extloadf32 xoaddr:$A)))));
3775}
3776def FltToULongLoadP9 {
3777  dag A = (i64 (PPCmfvsr (PPCfctiduz (f64 (extloadf32 iaddrX4:$A)))));
3778}
3779def FltToLong {
3780  dag A = (i64 (PPCmfvsr (f64 (PPCfctidz (fpextend f32:$A)))));
3781}
3782def FltToULong {
3783  dag A = (i64 (PPCmfvsr (f64 (PPCfctiduz (fpextend f32:$A)))));
3784}
3785def DblToInt {
3786  dag A = (i32 (PPCmfvsr (f64 (PPCfctiwz f64:$A))));
3787  dag B = (i32 (PPCmfvsr (f64 (PPCfctiwz f64:$B))));
3788  dag C = (i32 (PPCmfvsr (f64 (PPCfctiwz f64:$C))));
3789  dag D = (i32 (PPCmfvsr (f64 (PPCfctiwz f64:$D))));
3790}
3791def DblToUInt {
3792  dag A = (i32 (PPCmfvsr (f64 (PPCfctiwuz f64:$A))));
3793  dag B = (i32 (PPCmfvsr (f64 (PPCfctiwuz f64:$B))));
3794  dag C = (i32 (PPCmfvsr (f64 (PPCfctiwuz f64:$C))));
3795  dag D = (i32 (PPCmfvsr (f64 (PPCfctiwuz f64:$D))));
3796}
3797def DblToLong {
3798  dag A = (i64 (PPCmfvsr (f64 (PPCfctidz f64:$A))));
3799}
3800def DblToULong {
3801  dag A = (i64 (PPCmfvsr (f64 (PPCfctiduz f64:$A))));
3802}
3803def DblToIntLoad {
3804  dag A = (i32 (PPCmfvsr (PPCfctiwz (f64 (load xoaddr:$A)))));
3805}
3806def DblToIntLoadP9 {
3807  dag A = (i32 (PPCmfvsr (PPCfctiwz (f64 (load iaddrX4:$A)))));
3808}
3809def DblToUIntLoad {
3810  dag A = (i32 (PPCmfvsr (PPCfctiwuz (f64 (load xoaddr:$A)))));
3811}
3812def DblToUIntLoadP9 {
3813  dag A = (i32 (PPCmfvsr (PPCfctiwuz (f64 (load iaddrX4:$A)))));
3814}
3815def DblToLongLoad {
3816  dag A = (i64 (PPCmfvsr (PPCfctidz (f64 (load xoaddr:$A)))));
3817}
3818def DblToULongLoad {
3819  dag A = (i64 (PPCmfvsr (PPCfctiduz (f64 (load xoaddr:$A)))));
3820}
3821
3822// FP merge dags (for f32 -> v4f32)
3823def MrgFP {
3824  dag AC = (XVCVDPSP (XXPERMDI (COPY_TO_REGCLASS $A, VSRC),
3825                               (COPY_TO_REGCLASS $C, VSRC), 0));
3826  dag BD = (XVCVDPSP (XXPERMDI (COPY_TO_REGCLASS $B, VSRC),
3827                               (COPY_TO_REGCLASS $D, VSRC), 0));
3828  dag ABhToFlt = (XVCVDPSP (XXPERMDI $A, $B, 0));
3829  dag ABlToFlt = (XVCVDPSP (XXPERMDI $A, $B, 3));
3830  dag BAhToFlt = (XVCVDPSP (XXPERMDI $B, $A, 0));
3831  dag BAlToFlt = (XVCVDPSP (XXPERMDI $B, $A, 3));
3832}
3833
3834// Word-element merge dags - conversions from f64 to i32 merged into vectors.
3835def MrgWords {
3836  // For big endian, we merge low and hi doublewords (A, B).
3837  dag A0B0 = (v2f64 (XXPERMDI v2f64:$A, v2f64:$B, 0));
3838  dag A1B1 = (v2f64 (XXPERMDI v2f64:$A, v2f64:$B, 3));
3839  dag CVA1B1S = (v4i32 (XVCVDPSXWS A1B1));
3840  dag CVA0B0S = (v4i32 (XVCVDPSXWS A0B0));
3841  dag CVA1B1U = (v4i32 (XVCVDPUXWS A1B1));
3842  dag CVA0B0U = (v4i32 (XVCVDPUXWS A0B0));
3843
3844  // For little endian, we merge low and hi doublewords (B, A).
3845  dag B1A1 = (v2f64 (XXPERMDI v2f64:$B, v2f64:$A, 0));
3846  dag B0A0 = (v2f64 (XXPERMDI v2f64:$B, v2f64:$A, 3));
3847  dag CVB1A1S = (v4i32 (XVCVDPSXWS B1A1));
3848  dag CVB0A0S = (v4i32 (XVCVDPSXWS B0A0));
3849  dag CVB1A1U = (v4i32 (XVCVDPUXWS B1A1));
3850  dag CVB0A0U = (v4i32 (XVCVDPUXWS B0A0));
3851
3852  // For big endian, we merge hi doublewords of (A, C) and (B, D), convert
3853  // then merge.
3854  dag AC = (v2f64 (XXPERMDI (COPY_TO_REGCLASS f64:$A, VSRC),
3855                            (COPY_TO_REGCLASS f64:$C, VSRC), 0));
3856  dag BD = (v2f64 (XXPERMDI (COPY_TO_REGCLASS f64:$B, VSRC),
3857                            (COPY_TO_REGCLASS f64:$D, VSRC), 0));
3858  dag CVACS = (v4i32 (XVCVDPSXWS AC));
3859  dag CVBDS = (v4i32 (XVCVDPSXWS BD));
3860  dag CVACU = (v4i32 (XVCVDPUXWS AC));
3861  dag CVBDU = (v4i32 (XVCVDPUXWS BD));
3862
3863  // For little endian, we merge hi doublewords of (D, B) and (C, A), convert
3864  // then merge.
3865  dag DB = (v2f64 (XXPERMDI (COPY_TO_REGCLASS f64:$D, VSRC),
3866                            (COPY_TO_REGCLASS f64:$B, VSRC), 0));
3867  dag CA = (v2f64 (XXPERMDI (COPY_TO_REGCLASS f64:$C, VSRC),
3868                            (COPY_TO_REGCLASS f64:$A, VSRC), 0));
3869  dag CVDBS = (v4i32 (XVCVDPSXWS DB));
3870  dag CVCAS = (v4i32 (XVCVDPSXWS CA));
3871  dag CVDBU = (v4i32 (XVCVDPUXWS DB));
3872  dag CVCAU = (v4i32 (XVCVDPUXWS CA));
3873}
3874
3875// Patterns for BUILD_VECTOR nodes.
3876let AddedComplexity = 400 in {
3877
3878  let Predicates = [HasVSX] in {
3879    // Build vectors of floating point converted to i32.
3880    def : Pat<(v4i32 (build_vector DblToInt.A, DblToInt.A,
3881                                   DblToInt.A, DblToInt.A)),
3882              (v4i32 (XXSPLTW (COPY_TO_REGCLASS (XSCVDPSXWS $A), VSRC), 1))>;
3883    def : Pat<(v4i32 (build_vector DblToUInt.A, DblToUInt.A,
3884                                   DblToUInt.A, DblToUInt.A)),
3885              (v4i32 (XXSPLTW (COPY_TO_REGCLASS (XSCVDPUXWS $A), VSRC), 1))>;
3886    def : Pat<(v2i64 (build_vector DblToLong.A, DblToLong.A)),
3887              (v2i64 (XXPERMDI (COPY_TO_REGCLASS (XSCVDPSXDS $A), VSRC),
3888                               (COPY_TO_REGCLASS (XSCVDPSXDS $A), VSRC), 0))>;
3889    def : Pat<(v2i64 (build_vector DblToULong.A, DblToULong.A)),
3890              (v2i64 (XXPERMDI (COPY_TO_REGCLASS (XSCVDPUXDS $A), VSRC),
3891                               (COPY_TO_REGCLASS (XSCVDPUXDS $A), VSRC), 0))>;
3892    def : Pat<(v4i32 (scalar_to_vector FltToIntLoad.A)),
3893              (v4i32 (XXSPLTW (COPY_TO_REGCLASS
3894                                (XSCVDPSXWSs (XFLOADf32 xoaddr:$A)), VSRC), 1))>;
3895    def : Pat<(v4i32 (scalar_to_vector FltToUIntLoad.A)),
3896              (v4i32 (XXSPLTW (COPY_TO_REGCLASS
3897                                (XSCVDPUXWSs (XFLOADf32 xoaddr:$A)), VSRC), 1))>;
3898    def : Pat<(v4f32 (build_vector f32:$A, f32:$A, f32:$A, f32:$A)),
3899              (v4f32 (XXSPLTW (v4f32 (XSCVDPSPN $A)), 0))>;
3900
3901    // Build vectors of floating point converted to i64.
3902    def : Pat<(v2i64 (build_vector FltToLong.A, FltToLong.A)),
3903              (v2i64 (XXPERMDIs
3904                       (COPY_TO_REGCLASS (XSCVDPSXDSs $A), VSFRC), 0))>;
3905    def : Pat<(v2i64 (build_vector FltToULong.A, FltToULong.A)),
3906              (v2i64 (XXPERMDIs
3907                       (COPY_TO_REGCLASS (XSCVDPUXDSs $A), VSFRC), 0))>;
3908    def : Pat<(v2i64 (scalar_to_vector DblToLongLoad.A)),
3909              (v2i64 (XVCVDPSXDS (LXVDSX xoaddr:$A)))>;
3910    def : Pat<(v2i64 (scalar_to_vector DblToULongLoad.A)),
3911              (v2i64 (XVCVDPUXDS (LXVDSX xoaddr:$A)))>;
3912  }
3913
3914  let Predicates = [HasVSX, NoP9Vector] in {
3915    // Load-and-splat with fp-to-int conversion (using X-Form VSX/FP loads).
3916    def : Pat<(v4i32 (scalar_to_vector DblToIntLoad.A)),
3917              (v4i32 (XXSPLTW (COPY_TO_REGCLASS
3918                                (XSCVDPSXWS (XFLOADf64 xoaddr:$A)), VSRC), 1))>;
3919    def : Pat<(v4i32 (scalar_to_vector DblToUIntLoad.A)),
3920              (v4i32 (XXSPLTW (COPY_TO_REGCLASS
3921                                (XSCVDPUXWS (XFLOADf64 xoaddr:$A)), VSRC), 1))>;
3922    def : Pat<(v2i64 (scalar_to_vector FltToLongLoad.A)),
3923              (v2i64 (XXPERMDIs (XSCVDPSXDS (COPY_TO_REGCLASS
3924                                              (XFLOADf32 xoaddr:$A), VSFRC)), 0))>;
3925    def : Pat<(v2i64 (scalar_to_vector FltToULongLoad.A)),
3926              (v2i64 (XXPERMDIs (XSCVDPUXDS (COPY_TO_REGCLASS
3927                                              (XFLOADf32 xoaddr:$A), VSFRC)), 0))>;
3928  }
3929
3930  let Predicates = [IsBigEndian, HasP8Vector] in {
3931    def : Pat<DWToSPExtractConv.BVU,
3932              (v4f32 (VPKUDUM (XXSLDWI (XVCVUXDSP $S1), (XVCVUXDSP $S1), 3),
3933                              (XXSLDWI (XVCVUXDSP $S2), (XVCVUXDSP $S2), 3)))>;
3934    def : Pat<DWToSPExtractConv.BVS,
3935              (v4f32 (VPKUDUM (XXSLDWI (XVCVSXDSP $S1), (XVCVSXDSP $S1), 3),
3936                              (XXSLDWI (XVCVSXDSP $S2), (XVCVSXDSP $S2), 3)))>;
3937    def : Pat<(store (i32 (extractelt v4i32:$A, 1)), xoaddr:$src),
3938              (STIWX (EXTRACT_SUBREG $A, sub_64), xoaddr:$src)>;
3939    def : Pat<(store (f32 (extractelt v4f32:$A, 1)), xoaddr:$src),
3940              (STIWX (EXTRACT_SUBREG $A, sub_64), xoaddr:$src)>;
3941
3942    // Elements in a register on a BE system are in order <0, 1, 2, 3>.
3943    // The store instructions store the second word from the left.
3944    // So to align element zero, we need to modulo-left-shift by 3 words.
3945    // Similar logic applies for elements 2 and 3.
3946    foreach Idx = [ [0,3], [2,1], [3,2] ] in {
3947      def : Pat<(store (i32 (extractelt v4i32:$A, !head(Idx))), xoaddr:$src),
3948                (STIWX (EXTRACT_SUBREG (XXSLDWI $A, $A, !head(!tail(Idx))),
3949                                       sub_64), xoaddr:$src)>;
3950      def : Pat<(store (f32 (extractelt v4f32:$A, !head(Idx))), xoaddr:$src),
3951                (STIWX (EXTRACT_SUBREG (XXSLDWI $A, $A, !head(!tail(Idx))),
3952                                       sub_64), xoaddr:$src)>;
3953    }
3954  }
3955
3956  let Predicates = [HasP8Vector, IsBigEndian, NoP9Vector] in {
3957    def : Pat<(store (i64 (extractelt v2i64:$A, 0)), xoaddr:$src),
3958              (XFSTOREf64 (EXTRACT_SUBREG $A, sub_64), xoaddr:$src)>;
3959    def : Pat<(store (f64 (extractelt v2f64:$A, 0)), xoaddr:$src),
3960              (XFSTOREf64 (EXTRACT_SUBREG $A, sub_64), xoaddr:$src)>;
3961    def : Pat<(store (i64 (extractelt v2i64:$A, 1)), xoaddr:$src),
3962              (XFSTOREf64 (EXTRACT_SUBREG (XXPERMDI $A, $A, 2), sub_64),
3963                          xoaddr:$src)>;
3964    def : Pat<(store (f64 (extractelt v2f64:$A, 1)), xoaddr:$src),
3965              (XFSTOREf64 (EXTRACT_SUBREG (XXPERMDI $A, $A, 2), sub_64),
3966                          xoaddr:$src)>;
3967   }
3968
3969  // Big endian, available on all targets with VSX
3970  let Predicates = [IsBigEndian, HasVSX] in {
3971    def : Pat<(v2f64 (build_vector f64:$A, f64:$B)),
3972              (v2f64 (XXPERMDI
3973                        (COPY_TO_REGCLASS $A, VSRC),
3974                        (COPY_TO_REGCLASS $B, VSRC), 0))>;
3975
3976    def : Pat<(v4f32 (build_vector f32:$A, f32:$B, f32:$C, f32:$D)),
3977              (VMRGEW MrgFP.AC, MrgFP.BD)>;
3978    def : Pat<(v4f32 (build_vector DblToFlt.A0, DblToFlt.A1,
3979                                   DblToFlt.B0, DblToFlt.B1)),
3980              (v4f32 (VMRGEW MrgFP.ABhToFlt, MrgFP.ABlToFlt))>;
3981
3982    // Convert 4 doubles to a vector of ints.
3983    def : Pat<(v4i32 (build_vector DblToInt.A, DblToInt.B,
3984                                   DblToInt.C, DblToInt.D)),
3985              (v4i32 (VMRGEW MrgWords.CVACS, MrgWords.CVBDS))>;
3986    def : Pat<(v4i32 (build_vector DblToUInt.A, DblToUInt.B,
3987                                   DblToUInt.C, DblToUInt.D)),
3988              (v4i32 (VMRGEW MrgWords.CVACU, MrgWords.CVBDU))>;
3989    def : Pat<(v4i32 (build_vector ExtDbl.A0S, ExtDbl.A1S,
3990                                   ExtDbl.B0S, ExtDbl.B1S)),
3991              (v4i32 (VMRGEW MrgWords.CVA0B0S, MrgWords.CVA1B1S))>;
3992    def : Pat<(v4i32 (build_vector ExtDbl.A0U, ExtDbl.A1U,
3993                                   ExtDbl.B0U, ExtDbl.B1U)),
3994              (v4i32 (VMRGEW MrgWords.CVA0B0U, MrgWords.CVA1B1U))>;
3995  }
3996
3997  let Predicates = [IsLittleEndian, HasP8Vector] in {
3998    def : Pat<DWToSPExtractConv.BVU,
3999              (v4f32 (VPKUDUM (XXSLDWI (XVCVUXDSP $S2), (XVCVUXDSP $S2), 3),
4000                              (XXSLDWI (XVCVUXDSP $S1), (XVCVUXDSP $S1), 3)))>;
4001    def : Pat<DWToSPExtractConv.BVS,
4002              (v4f32 (VPKUDUM (XXSLDWI (XVCVSXDSP $S2), (XVCVSXDSP $S2), 3),
4003                              (XXSLDWI (XVCVSXDSP $S1), (XVCVSXDSP $S1), 3)))>;
4004    def : Pat<(store (i32 (extractelt v4i32:$A, 2)), xoaddr:$src),
4005              (STIWX (EXTRACT_SUBREG $A, sub_64), xoaddr:$src)>;
4006    def : Pat<(store (f32 (extractelt v4f32:$A, 2)), xoaddr:$src),
4007              (STIWX (EXTRACT_SUBREG $A, sub_64), xoaddr:$src)>;
4008
4009    // Elements in a register on a LE system are in order <3, 2, 1, 0>.
4010    // The store instructions store the second word from the left.
4011    // So to align element 3, we need to modulo-left-shift by 3 words.
4012    // Similar logic applies for elements 0 and 1.
4013    foreach Idx = [ [0,2], [1,1], [3,3] ] in {
4014      def : Pat<(store (i32 (extractelt v4i32:$A, !head(Idx))), xoaddr:$src),
4015                (STIWX (EXTRACT_SUBREG (XXSLDWI $A, $A, !head(!tail(Idx))),
4016                                       sub_64), xoaddr:$src)>;
4017      def : Pat<(store (f32 (extractelt v4f32:$A, !head(Idx))), xoaddr:$src),
4018                (STIWX (EXTRACT_SUBREG (XXSLDWI $A, $A, !head(!tail(Idx))),
4019                                       sub_64), xoaddr:$src)>;
4020    }
4021  }
4022
4023  let Predicates = [HasP8Vector, IsLittleEndian, NoP9Vector] in {
4024    def : Pat<(store (i64 (extractelt v2i64:$A, 0)), xoaddr:$src),
4025              (XFSTOREf64 (EXTRACT_SUBREG (XXPERMDI $A, $A, 2), sub_64),
4026                          xoaddr:$src)>;
4027    def : Pat<(store (f64 (extractelt v2f64:$A, 0)), xoaddr:$src),
4028              (XFSTOREf64 (EXTRACT_SUBREG (XXPERMDI $A, $A, 2), sub_64),
4029                          xoaddr:$src)>;
4030    def : Pat<(store (i64 (extractelt v2i64:$A, 1)), xoaddr:$src),
4031              (XFSTOREf64 (EXTRACT_SUBREG $A, sub_64), xoaddr:$src)>;
4032    def : Pat<(store (f64 (extractelt v2f64:$A, 1)), xoaddr:$src),
4033              (XFSTOREf64 (EXTRACT_SUBREG $A, sub_64), xoaddr:$src)>;
4034   }
4035
4036  let Predicates = [IsLittleEndian, HasVSX] in {
4037  // Little endian, available on all targets with VSX
4038    def : Pat<(v2f64 (build_vector f64:$A, f64:$B)),
4039              (v2f64 (XXPERMDI
4040                        (COPY_TO_REGCLASS $B, VSRC),
4041                        (COPY_TO_REGCLASS $A, VSRC), 0))>;
4042
4043    def : Pat<(v4f32 (build_vector f32:$D, f32:$C, f32:$B, f32:$A)),
4044              (VMRGEW MrgFP.AC, MrgFP.BD)>;
4045    def : Pat<(v4f32 (build_vector DblToFlt.A0, DblToFlt.A1,
4046                                   DblToFlt.B0, DblToFlt.B1)),
4047              (v4f32 (VMRGEW MrgFP.BAhToFlt, MrgFP.BAlToFlt))>;
4048
4049    // Convert 4 doubles to a vector of ints.
4050    def : Pat<(v4i32 (build_vector DblToInt.A, DblToInt.B,
4051                                   DblToInt.C, DblToInt.D)),
4052              (v4i32 (VMRGEW MrgWords.CVDBS, MrgWords.CVCAS))>;
4053    def : Pat<(v4i32 (build_vector DblToUInt.A, DblToUInt.B,
4054                                   DblToUInt.C, DblToUInt.D)),
4055              (v4i32 (VMRGEW MrgWords.CVDBU, MrgWords.CVCAU))>;
4056    def : Pat<(v4i32 (build_vector ExtDbl.A0S, ExtDbl.A1S,
4057                                   ExtDbl.B0S, ExtDbl.B1S)),
4058              (v4i32 (VMRGEW MrgWords.CVB1A1S, MrgWords.CVB0A0S))>;
4059    def : Pat<(v4i32 (build_vector ExtDbl.A0U, ExtDbl.A1U,
4060                                   ExtDbl.B0U, ExtDbl.B1U)),
4061              (v4i32 (VMRGEW MrgWords.CVB1A1U, MrgWords.CVB0A0U))>;
4062  }
4063
4064  let Predicates = [HasDirectMove] in {
4065    // Endianness-neutral constant splat on P8 and newer targets. The reason
4066    // for this pattern is that on targets with direct moves, we don't expand
4067    // BUILD_VECTOR nodes for v4i32.
4068    def : Pat<(v4i32 (build_vector immSExt5NonZero:$A, immSExt5NonZero:$A,
4069                                   immSExt5NonZero:$A, immSExt5NonZero:$A)),
4070              (v4i32 (VSPLTISW imm:$A))>;
4071  }
4072
4073  let Predicates = [IsBigEndian, HasDirectMove, NoP9Vector] in {
4074    // Big endian integer vectors using direct moves.
4075    def : Pat<(v2i64 (build_vector i64:$A, i64:$B)),
4076              (v2i64 (XXPERMDI
4077                        (COPY_TO_REGCLASS (MTVSRD $A), VSRC),
4078                        (COPY_TO_REGCLASS (MTVSRD $B), VSRC), 0))>;
4079    def : Pat<(v4i32 (build_vector i32:$A, i32:$B, i32:$C, i32:$D)),
4080              (XXPERMDI
4081                (COPY_TO_REGCLASS
4082                  (MTVSRD (RLDIMI AnyExts.B, AnyExts.A, 32, 0)), VSRC),
4083                (COPY_TO_REGCLASS
4084                  (MTVSRD (RLDIMI AnyExts.D, AnyExts.C, 32, 0)), VSRC), 0)>;
4085    def : Pat<(v4i32 (build_vector i32:$A, i32:$A, i32:$A, i32:$A)),
4086              (XXSPLTW (COPY_TO_REGCLASS (MTVSRWZ $A), VSRC), 1)>;
4087  }
4088
4089  let Predicates = [IsLittleEndian, HasDirectMove, NoP9Vector] in {
4090    // Little endian integer vectors using direct moves.
4091    def : Pat<(v2i64 (build_vector i64:$A, i64:$B)),
4092              (v2i64 (XXPERMDI
4093                        (COPY_TO_REGCLASS (MTVSRD $B), VSRC),
4094                        (COPY_TO_REGCLASS (MTVSRD $A), VSRC), 0))>;
4095    def : Pat<(v4i32 (build_vector i32:$A, i32:$B, i32:$C, i32:$D)),
4096              (XXPERMDI
4097                (COPY_TO_REGCLASS
4098                  (MTVSRD (RLDIMI AnyExts.C, AnyExts.D, 32, 0)), VSRC),
4099                (COPY_TO_REGCLASS
4100                  (MTVSRD (RLDIMI AnyExts.A, AnyExts.B, 32, 0)), VSRC), 0)>;
4101    def : Pat<(v4i32 (build_vector i32:$A, i32:$A, i32:$A, i32:$A)),
4102              (XXSPLTW (COPY_TO_REGCLASS (MTVSRWZ $A), VSRC), 1)>;
4103  }
4104
4105  let Predicates = [HasP8Vector] in {
4106    def : Pat<(v1i128 (bitconvert (v16i8 immAllOnesV))),
4107              (v1i128 (COPY_TO_REGCLASS(XXLEQVOnes), VSRC))>;
4108    def : Pat<(v2i64 (bitconvert (v16i8 immAllOnesV))),
4109              (v2i64 (COPY_TO_REGCLASS(XXLEQVOnes), VSRC))>;
4110    def : Pat<(v8i16 (bitconvert (v16i8 immAllOnesV))),
4111              (v8i16 (COPY_TO_REGCLASS(XXLEQVOnes), VSRC))>;
4112    def : Pat<(v16i8 (bitconvert (v16i8 immAllOnesV))),
4113              (v16i8 (COPY_TO_REGCLASS(XXLEQVOnes), VSRC))>;
4114  }
4115
4116  let Predicates = [HasP9Vector] in {
4117    // Endianness-neutral patterns for const splats with ISA 3.0 instructions.
4118    def : Pat<(v4i32 (scalar_to_vector i32:$A)),
4119              (v4i32 (MTVSRWS $A))>;
4120    def : Pat<(v4i32 (build_vector i32:$A, i32:$A, i32:$A, i32:$A)),
4121              (v4i32 (MTVSRWS $A))>;
4122    def : Pat<(v16i8 (build_vector immNonAllOneAnyExt8:$A, immNonAllOneAnyExt8:$A,
4123                                   immNonAllOneAnyExt8:$A, immNonAllOneAnyExt8:$A,
4124                                   immNonAllOneAnyExt8:$A, immNonAllOneAnyExt8:$A,
4125                                   immNonAllOneAnyExt8:$A, immNonAllOneAnyExt8:$A,
4126                                   immNonAllOneAnyExt8:$A, immNonAllOneAnyExt8:$A,
4127                                   immNonAllOneAnyExt8:$A, immNonAllOneAnyExt8:$A,
4128                                   immNonAllOneAnyExt8:$A, immNonAllOneAnyExt8:$A,
4129                                   immNonAllOneAnyExt8:$A, immNonAllOneAnyExt8:$A)),
4130              (v16i8 (COPY_TO_REGCLASS (XXSPLTIB imm:$A), VSRC))>;
4131    def : Pat<(v4i32 (scalar_to_vector FltToIntLoad.A)),
4132              (v4i32 (XVCVSPSXWS (LXVWSX xoaddr:$A)))>;
4133    def : Pat<(v4i32 (scalar_to_vector FltToUIntLoad.A)),
4134              (v4i32 (XVCVSPUXWS (LXVWSX xoaddr:$A)))>;
4135    def : Pat<(v4i32 (scalar_to_vector DblToIntLoadP9.A)),
4136              (v4i32 (XXSPLTW (COPY_TO_REGCLASS
4137                                (XSCVDPSXWS (DFLOADf64 iaddrX4:$A)), VSRC), 1))>;
4138    def : Pat<(v4i32 (scalar_to_vector DblToUIntLoadP9.A)),
4139              (v4i32 (XXSPLTW (COPY_TO_REGCLASS
4140                                (XSCVDPUXWS (DFLOADf64 iaddrX4:$A)), VSRC), 1))>;
4141    def : Pat<(v2i64 (scalar_to_vector FltToLongLoadP9.A)),
4142              (v2i64 (XXPERMDIs (XSCVDPSXDS (COPY_TO_REGCLASS
4143                                              (DFLOADf32 iaddrX4:$A),
4144                                              VSFRC)), 0))>;
4145    def : Pat<(v2i64 (scalar_to_vector FltToULongLoadP9.A)),
4146              (v2i64 (XXPERMDIs (XSCVDPUXDS (COPY_TO_REGCLASS
4147                                              (DFLOADf32 iaddrX4:$A),
4148                                              VSFRC)), 0))>;
4149  }
4150
4151  let Predicates = [IsISA3_0, HasDirectMove, IsBigEndian] in {
4152    def : Pat<(i64 (extractelt v2i64:$A, 1)),
4153              (i64 (MFVSRLD $A))>;
4154    // Better way to build integer vectors if we have MTVSRDD. Big endian.
4155    def : Pat<(v2i64 (build_vector i64:$rB, i64:$rA)),
4156              (v2i64 (MTVSRDD $rB, $rA))>;
4157    def : Pat<(v4i32 (build_vector i32:$A, i32:$B, i32:$C, i32:$D)),
4158              (MTVSRDD
4159                (RLDIMI AnyExts.B, AnyExts.A, 32, 0),
4160                (RLDIMI AnyExts.D, AnyExts.C, 32, 0))>;
4161  }
4162
4163  let Predicates = [IsISA3_0, HasDirectMove, IsLittleEndian] in {
4164    def : Pat<(i64 (extractelt v2i64:$A, 0)),
4165              (i64 (MFVSRLD $A))>;
4166    // Better way to build integer vectors if we have MTVSRDD. Little endian.
4167    def : Pat<(v2i64 (build_vector i64:$rA, i64:$rB)),
4168              (v2i64 (MTVSRDD $rB, $rA))>;
4169    def : Pat<(v4i32 (build_vector i32:$A, i32:$B, i32:$C, i32:$D)),
4170              (MTVSRDD
4171                (RLDIMI AnyExts.C, AnyExts.D, 32, 0),
4172                (RLDIMI AnyExts.A, AnyExts.B, 32, 0))>;
4173  }
4174  // P9 Altivec instructions that can be used to build vectors.
4175  // Adding them to PPCInstrVSX.td rather than PPCAltivecVSX.td to compete
4176  // with complexities of existing build vector patterns in this file.
4177  let Predicates = [HasP9Altivec, IsLittleEndian] in {
4178    def : Pat<(v2i64 (build_vector WordToDWord.LE_A0, WordToDWord.LE_A1)),
4179              (v2i64 (VEXTSW2D $A))>;
4180    def : Pat<(v2i64 (build_vector HWordToDWord.LE_A0, HWordToDWord.LE_A1)),
4181              (v2i64 (VEXTSH2D $A))>;
4182    def : Pat<(v4i32 (build_vector HWordToWord.LE_A0, HWordToWord.LE_A1,
4183                      HWordToWord.LE_A2, HWordToWord.LE_A3)),
4184              (v4i32 (VEXTSH2W $A))>;
4185    def : Pat<(v4i32 (build_vector ByteToWord.LE_A0, ByteToWord.LE_A1,
4186                      ByteToWord.LE_A2, ByteToWord.LE_A3)),
4187              (v4i32 (VEXTSB2W $A))>;
4188    def : Pat<(v2i64 (build_vector ByteToDWord.LE_A0, ByteToDWord.LE_A1)),
4189              (v2i64 (VEXTSB2D $A))>;
4190  }
4191
4192  let Predicates = [HasP9Altivec, IsBigEndian] in {
4193    def : Pat<(v2i64 (build_vector WordToDWord.BE_A0, WordToDWord.BE_A1)),
4194              (v2i64 (VEXTSW2D $A))>;
4195    def : Pat<(v2i64 (build_vector HWordToDWord.BE_A0, HWordToDWord.BE_A1)),
4196              (v2i64 (VEXTSH2D $A))>;
4197    def : Pat<(v4i32 (build_vector HWordToWord.BE_A0, HWordToWord.BE_A1,
4198                      HWordToWord.BE_A2, HWordToWord.BE_A3)),
4199              (v4i32 (VEXTSH2W $A))>;
4200    def : Pat<(v4i32 (build_vector ByteToWord.BE_A0, ByteToWord.BE_A1,
4201                      ByteToWord.BE_A2, ByteToWord.BE_A3)),
4202              (v4i32 (VEXTSB2W $A))>;
4203    def : Pat<(v2i64 (build_vector ByteToDWord.BE_A0, ByteToDWord.BE_A1)),
4204              (v2i64 (VEXTSB2D $A))>;
4205  }
4206
4207  let Predicates = [HasP9Altivec] in {
4208    def: Pat<(v2i64 (PPCSExtVElems v16i8:$A)),
4209              (v2i64 (VEXTSB2D $A))>;
4210    def: Pat<(v2i64 (PPCSExtVElems v8i16:$A)),
4211              (v2i64 (VEXTSH2D $A))>;
4212    def: Pat<(v2i64 (PPCSExtVElems v4i32:$A)),
4213              (v2i64 (VEXTSW2D $A))>;
4214    def: Pat<(v4i32 (PPCSExtVElems v16i8:$A)),
4215              (v4i32 (VEXTSB2W $A))>;
4216    def: Pat<(v4i32 (PPCSExtVElems v8i16:$A)),
4217              (v4i32 (VEXTSH2W $A))>;
4218  }
4219}
4220
4221// Put this P9Altivec related definition here since it's possible to be
4222// selected to VSX instruction xvnegsp, avoid possible undef.
4223let Predicates = [HasP9Altivec] in {
4224
4225  def : Pat<(v4i32 (PPCvabsd v4i32:$A, v4i32:$B, (i32 0))),
4226            (v4i32 (VABSDUW $A, $B))>;
4227
4228  def : Pat<(v8i16 (PPCvabsd v8i16:$A, v8i16:$B, (i32 0))),
4229            (v8i16 (VABSDUH $A, $B))>;
4230
4231  def : Pat<(v16i8 (PPCvabsd v16i8:$A, v16i8:$B, (i32 0))),
4232            (v16i8 (VABSDUB $A, $B))>;
4233
4234  // As PPCVABSD description, the last operand indicates whether do the
4235  // sign bit flip.
4236  def : Pat<(v4i32 (PPCvabsd v4i32:$A, v4i32:$B, (i32 1))),
4237            (v4i32 (VABSDUW (XVNEGSP $A), (XVNEGSP $B)))>;
4238}
4239