1 //===-- VVPISelLowering.cpp - VE DAG Lowering Implementation --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the lowering and legalization of vector instructions to
10 // VVP_*layer SDNodes.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "VECustomDAG.h"
15 #include "VEISelLowering.h"
16 
17 using namespace llvm;
18 
19 #define DEBUG_TYPE "ve-lower"
20 
21 SDValue VETargetLowering::splitMaskArithmetic(SDValue Op,
22                                               SelectionDAG &DAG) const {
23   VECustomDAG CDAG(DAG, Op);
24   SDValue AVL =
25       CDAG.getConstant(Op.getValueType().getVectorNumElements(), MVT::i32);
26   SDValue A = Op->getOperand(0);
27   SDValue B = Op->getOperand(1);
28   SDValue LoA = CDAG.getUnpack(MVT::v256i1, A, PackElem::Lo, AVL);
29   SDValue HiA = CDAG.getUnpack(MVT::v256i1, A, PackElem::Hi, AVL);
30   SDValue LoB = CDAG.getUnpack(MVT::v256i1, B, PackElem::Lo, AVL);
31   SDValue HiB = CDAG.getUnpack(MVT::v256i1, B, PackElem::Hi, AVL);
32   unsigned Opc = Op.getOpcode();
33   auto LoRes = CDAG.getNode(Opc, MVT::v256i1, {LoA, LoB});
34   auto HiRes = CDAG.getNode(Opc, MVT::v256i1, {HiA, HiB});
35   return CDAG.getPack(MVT::v512i1, LoRes, HiRes, AVL);
36 }
37 
38 SDValue VETargetLowering::lowerToVVP(SDValue Op, SelectionDAG &DAG) const {
39   // Can we represent this as a VVP node.
40   const unsigned Opcode = Op->getOpcode();
41   auto VVPOpcodeOpt = getVVPOpcode(Opcode);
42   if (!VVPOpcodeOpt.hasValue())
43     return SDValue();
44   unsigned VVPOpcode = VVPOpcodeOpt.getValue();
45   const bool FromVP = ISD::isVPOpcode(Opcode);
46 
47   // The representative and legalized vector type of this operation.
48   VECustomDAG CDAG(DAG, Op);
49   // Dispatch to complex lowering functions.
50   switch (VVPOpcode) {
51   case VEISD::VVP_LOAD:
52   case VEISD::VVP_STORE:
53     return lowerVVP_LOAD_STORE(Op, CDAG);
54   };
55 
56   EVT OpVecVT = Op.getValueType();
57   EVT LegalVecVT = getTypeToTransformTo(*DAG.getContext(), OpVecVT);
58   auto Packing = getTypePacking(LegalVecVT.getSimpleVT());
59 
60   SDValue AVL;
61   SDValue Mask;
62 
63   if (FromVP) {
64     // All upstream VP SDNodes always have a mask and avl.
65     auto MaskIdx = ISD::getVPMaskIdx(Opcode);
66     auto AVLIdx = ISD::getVPExplicitVectorLengthIdx(Opcode);
67     if (MaskIdx)
68       Mask = Op->getOperand(*MaskIdx);
69     if (AVLIdx)
70       AVL = Op->getOperand(*AVLIdx);
71   }
72 
73   // Materialize default mask and avl.
74   if (!AVL)
75     AVL = CDAG.getConstant(OpVecVT.getVectorNumElements(), MVT::i32);
76   if (!Mask)
77     Mask = CDAG.getConstantMask(Packing, true);
78 
79   if (isVVPBinaryOp(VVPOpcode)) {
80     assert(LegalVecVT.isSimple());
81     return CDAG.getNode(VVPOpcode, LegalVecVT,
82                         {Op->getOperand(0), Op->getOperand(1), Mask, AVL});
83   }
84   if (VVPOpcode == VEISD::VVP_SELECT) {
85     auto Mask = Op->getOperand(0);
86     auto OnTrue = Op->getOperand(1);
87     auto OnFalse = Op->getOperand(2);
88     return CDAG.getNode(VVPOpcode, LegalVecVT, {OnTrue, OnFalse, Mask, AVL});
89   }
90   if (VVPOpcode == VEISD::VVP_SETCC) {
91     auto LHS = Op->getOperand(0);
92     auto RHS = Op->getOperand(1);
93     auto Pred = Op->getOperand(2);
94     return CDAG.getNode(VVPOpcode, LegalVecVT, {LHS, RHS, Pred, Mask, AVL});
95   }
96   llvm_unreachable("lowerToVVP called for unexpected SDNode.");
97 }
98 
99 SDValue VETargetLowering::lowerVVP_LOAD_STORE(SDValue Op,
100                                               VECustomDAG &CDAG) const {
101   auto VVPOpc = *getVVPOpcode(Op->getOpcode());
102   const bool IsLoad = (VVPOpc == VEISD::VVP_LOAD);
103 
104   // Shares.
105   SDValue BasePtr = getMemoryPtr(Op);
106   SDValue Mask = getNodeMask(Op);
107   SDValue Chain = getNodeChain(Op);
108   SDValue AVL = getNodeAVL(Op);
109   // Store specific.
110   SDValue Data = getStoredValue(Op);
111   // Load specific.
112   SDValue PassThru = getNodePassthru(Op);
113 
114   auto DataVT = *getIdiomaticVectorType(Op.getNode());
115   auto Packing = getTypePacking(DataVT);
116 
117   assert(Packing == Packing::Normal && "TODO Packed load store isel");
118 
119   // TODO: Infer lower AVL from mask.
120   if (!AVL)
121     AVL = CDAG.getConstant(DataVT.getVectorNumElements(), MVT::i32);
122 
123   // Default to the all-true mask.
124   if (!Mask)
125     Mask = CDAG.getConstantMask(Packing, true);
126 
127   SDValue StrideV = getLoadStoreStride(Op, CDAG);
128   if (IsLoad) {
129     MVT LegalDataVT = getLegalVectorType(
130         Packing, DataVT.getVectorElementType().getSimpleVT());
131 
132     auto NewLoadV = CDAG.getNode(VEISD::VVP_LOAD, {LegalDataVT, MVT::Other},
133                                  {Chain, BasePtr, StrideV, Mask, AVL});
134 
135     if (!PassThru || PassThru->isUndef())
136       return NewLoadV;
137 
138     // Convert passthru to an explicit select node.
139     SDValue DataV = CDAG.getNode(VEISD::VVP_SELECT, DataVT,
140                                  {NewLoadV, PassThru, Mask, AVL});
141     SDValue NewLoadChainV = SDValue(NewLoadV.getNode(), 1);
142 
143     // Merge them back into one node.
144     return CDAG.getMergeValues({DataV, NewLoadChainV});
145   }
146 
147   // VVP_STORE
148   assert(VVPOpc == VEISD::VVP_STORE);
149   return CDAG.getNode(VEISD::VVP_STORE, Op.getNode()->getVTList(),
150                       {Chain, Data, BasePtr, StrideV, Mask, AVL});
151 }
152 
153 SDValue VETargetLowering::legalizeInternalVectorOp(SDValue Op,
154                                                    SelectionDAG &DAG) const {
155   VECustomDAG CDAG(DAG, Op);
156 
157   EVT IdiomVT = Op.getValueType();
158   if (isPackedVectorType(IdiomVT) &&
159       !supportsPackedMode(Op.getOpcode(), IdiomVT))
160     return splitVectorOp(Op, CDAG);
161 
162   // TODO: Implement odd/even splitting.
163   return legalizePackedAVL(Op, CDAG);
164 }
165 
166 SDValue VETargetLowering::splitVectorOp(SDValue Op, VECustomDAG &CDAG) const {
167   MVT ResVT = splitVectorType(Op.getValue(0).getSimpleValueType());
168 
169   auto AVLPos = getAVLPos(Op->getOpcode());
170   auto MaskPos = getMaskPos(Op->getOpcode());
171 
172   SDValue PackedMask = getNodeMask(Op);
173   auto AVLPair = getAnnotatedNodeAVL(Op);
174   SDValue PackedAVL = AVLPair.first;
175   assert(!AVLPair.second && "Expecting non pack-legalized oepration");
176 
177   // request the parts
178   SDValue PartOps[2];
179 
180   SDValue UpperPartAVL; // we will use this for packing things back together
181   for (PackElem Part : {PackElem::Hi, PackElem::Lo}) {
182     // VP ops already have an explicit mask and AVL. When expanding from non-VP
183     // attach those additional inputs here.
184     auto SplitTM = CDAG.getTargetSplitMask(PackedMask, PackedAVL, Part);
185 
186     if (Part == PackElem::Hi)
187       UpperPartAVL = SplitTM.AVL;
188 
189     // Attach non-predicating value operands
190     SmallVector<SDValue, 4> OpVec;
191     for (unsigned i = 0; i < Op.getNumOperands(); ++i) {
192       if (AVLPos && ((int)i) == *AVLPos)
193         continue;
194       if (MaskPos && ((int)i) == *MaskPos)
195         continue;
196 
197       // Value operand
198       auto PackedOperand = Op.getOperand(i);
199       auto UnpackedOpVT = splitVectorType(PackedOperand.getSimpleValueType());
200       SDValue PartV =
201           CDAG.getUnpack(UnpackedOpVT, PackedOperand, Part, SplitTM.AVL);
202       OpVec.push_back(PartV);
203     }
204 
205     // Add predicating args and generate part node.
206     OpVec.push_back(SplitTM.Mask);
207     OpVec.push_back(SplitTM.AVL);
208     // Emit legal VVP nodes.
209     PartOps[(int)Part] =
210         CDAG.getNode(Op.getOpcode(), ResVT, OpVec, Op->getFlags());
211   }
212 
213   // Re-package vectors.
214   return CDAG.getPack(Op.getValueType(), PartOps[(int)PackElem::Lo],
215                       PartOps[(int)PackElem::Hi], UpperPartAVL);
216 }
217 
218 SDValue VETargetLowering::legalizePackedAVL(SDValue Op,
219                                             VECustomDAG &CDAG) const {
220   LLVM_DEBUG(dbgs() << "::legalizePackedAVL\n";);
221   // Only required for VEC and VVP ops.
222   if (!isVVPOrVEC(Op->getOpcode()))
223     return Op;
224 
225   // Operation already has a legal AVL.
226   auto AVL = getNodeAVL(Op);
227   if (isLegalAVL(AVL))
228     return Op;
229 
230   // Half and round up EVL for 32bit element types.
231   SDValue LegalAVL = AVL;
232   if (isPackedVectorType(Op.getValueType())) {
233     assert(maySafelyIgnoreMask(Op) &&
234            "TODO Shift predication from EVL into Mask");
235 
236     if (auto *ConstAVL = dyn_cast<ConstantSDNode>(AVL)) {
237       LegalAVL = CDAG.getConstant((ConstAVL->getZExtValue() + 1) / 2, MVT::i32);
238     } else {
239       auto ConstOne = CDAG.getConstant(1, MVT::i32);
240       auto PlusOne = CDAG.getNode(ISD::ADD, MVT::i32, {AVL, ConstOne});
241       LegalAVL = CDAG.getNode(ISD::SRL, MVT::i32, {PlusOne, ConstOne});
242     }
243   }
244 
245   SDValue AnnotatedLegalAVL = CDAG.annotateLegalAVL(LegalAVL);
246 
247   // Copy the operand list.
248   int NumOp = Op->getNumOperands();
249   auto AVLPos = getAVLPos(Op->getOpcode());
250   std::vector<SDValue> FixedOperands;
251   for (int i = 0; i < NumOp; ++i) {
252     if (AVLPos && (i == *AVLPos)) {
253       FixedOperands.push_back(AnnotatedLegalAVL);
254       continue;
255     }
256     FixedOperands.push_back(Op->getOperand(i));
257   }
258 
259   // Clone the operation with fixed operands.
260   auto Flags = Op->getFlags();
261   SDValue NewN =
262       CDAG.getNode(Op->getOpcode(), Op->getVTList(), FixedOperands, Flags);
263   return NewN;
264 }
265