1 //===-- VVPISelLowering.cpp - VE DAG Lowering Implementation --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the lowering and legalization of vector instructions to
10 // VVP_*layer SDNodes.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "VECustomDAG.h"
15 #include "VEISelLowering.h"
16 
17 using namespace llvm;
18 
19 #define DEBUG_TYPE "ve-lower"
20 
21 SDValue VETargetLowering::splitMaskArithmetic(SDValue Op,
22                                               SelectionDAG &DAG) const {
23   VECustomDAG CDAG(DAG, Op);
24   SDValue AVL =
25       CDAG.getConstant(Op.getValueType().getVectorNumElements(), MVT::i32);
26   SDValue A = Op->getOperand(0);
27   SDValue B = Op->getOperand(1);
28   SDValue LoA = CDAG.getUnpack(MVT::v256i1, A, PackElem::Lo, AVL);
29   SDValue HiA = CDAG.getUnpack(MVT::v256i1, A, PackElem::Hi, AVL);
30   SDValue LoB = CDAG.getUnpack(MVT::v256i1, B, PackElem::Lo, AVL);
31   SDValue HiB = CDAG.getUnpack(MVT::v256i1, B, PackElem::Hi, AVL);
32   unsigned Opc = Op.getOpcode();
33   auto LoRes = CDAG.getNode(Opc, MVT::v256i1, {LoA, LoB});
34   auto HiRes = CDAG.getNode(Opc, MVT::v256i1, {HiA, HiB});
35   return CDAG.getPack(MVT::v512i1, LoRes, HiRes, AVL);
36 }
37 
38 SDValue VETargetLowering::lowerToVVP(SDValue Op, SelectionDAG &DAG) const {
39   // Can we represent this as a VVP node.
40   const unsigned Opcode = Op->getOpcode();
41   auto VVPOpcodeOpt = getVVPOpcode(Opcode);
42   if (!VVPOpcodeOpt.hasValue())
43     return SDValue();
44   unsigned VVPOpcode = VVPOpcodeOpt.getValue();
45   const bool FromVP = ISD::isVPOpcode(Opcode);
46 
47   // The representative and legalized vector type of this operation.
48   VECustomDAG CDAG(DAG, Op);
49   // Dispatch to complex lowering functions.
50   switch (VVPOpcode) {
51   case VEISD::VVP_LOAD:
52   case VEISD::VVP_STORE:
53     return lowerVVP_LOAD_STORE(Op, CDAG);
54   };
55 
56   EVT OpVecVT = Op.getValueType();
57   EVT LegalVecVT = getTypeToTransformTo(*DAG.getContext(), OpVecVT);
58   auto Packing = getTypePacking(LegalVecVT.getSimpleVT());
59 
60   SDValue AVL;
61   SDValue Mask;
62 
63   if (FromVP) {
64     // All upstream VP SDNodes always have a mask and avl.
65     auto MaskIdx = ISD::getVPMaskIdx(Opcode);
66     auto AVLIdx = ISD::getVPExplicitVectorLengthIdx(Opcode);
67     if (MaskIdx)
68       Mask = Op->getOperand(*MaskIdx);
69     if (AVLIdx)
70       AVL = Op->getOperand(*AVLIdx);
71   }
72 
73   // Materialize default mask and avl.
74   if (!AVL)
75     AVL = CDAG.getConstant(OpVecVT.getVectorNumElements(), MVT::i32);
76   if (!Mask)
77     Mask = CDAG.getConstantMask(Packing, true);
78 
79   if (isVVPBinaryOp(VVPOpcode)) {
80     assert(LegalVecVT.isSimple());
81     return CDAG.getNode(VVPOpcode, LegalVecVT,
82                         {Op->getOperand(0), Op->getOperand(1), Mask, AVL});
83   }
84   if (VVPOpcode == VEISD::VVP_SELECT) {
85     auto Mask = Op->getOperand(0);
86     auto OnTrue = Op->getOperand(1);
87     auto OnFalse = Op->getOperand(2);
88     return CDAG.getNode(VVPOpcode, LegalVecVT, {OnTrue, OnFalse, Mask, AVL});
89   }
90   if (VVPOpcode == VEISD::VVP_SETCC) {
91     auto LHS = Op->getOperand(0);
92     auto RHS = Op->getOperand(1);
93     auto Pred = Op->getOperand(2);
94     return CDAG.getNode(VVPOpcode, LegalVecVT, {LHS, RHS, Pred, Mask, AVL});
95   }
96   llvm_unreachable("lowerToVVP called for unexpected SDNode.");
97 }
98 
99 SDValue VETargetLowering::lowerVVP_LOAD_STORE(SDValue Op,
100                                               VECustomDAG &CDAG) const {
101   auto VVPOpc = *getVVPOpcode(Op->getOpcode());
102   const bool IsLoad = (VVPOpc == VEISD::VVP_LOAD);
103 
104   // Shares.
105   SDValue BasePtr = getMemoryPtr(Op);
106   SDValue Mask = getNodeMask(Op);
107   SDValue Chain = getNodeChain(Op);
108   SDValue AVL = getNodeAVL(Op);
109   // Store specific.
110   SDValue Data = getStoredValue(Op);
111   // Load specific.
112   SDValue PassThru = getNodePassthru(Op);
113 
114   auto DataVT = *getIdiomaticVectorType(Op.getNode());
115   auto Packing = getTypePacking(DataVT);
116 
117   // TODO: Infer lower AVL from mask.
118   if (!AVL)
119     AVL = CDAG.getConstant(DataVT.getVectorNumElements(), MVT::i32);
120 
121   // Default to the all-true mask.
122   if (!Mask)
123     Mask = CDAG.getConstantMask(Packing, true);
124 
125   SDValue StrideV = getLoadStoreStride(Op, CDAG);
126   if (IsLoad) {
127     MVT LegalDataVT = getLegalVectorType(
128         Packing, DataVT.getVectorElementType().getSimpleVT());
129 
130     auto NewLoadV = CDAG.getNode(VEISD::VVP_LOAD, {LegalDataVT, MVT::Other},
131                                  {Chain, BasePtr, StrideV, Mask, AVL});
132 
133     if (!PassThru || PassThru->isUndef())
134       return NewLoadV;
135 
136     // Convert passthru to an explicit select node.
137     SDValue DataV = CDAG.getNode(VEISD::VVP_SELECT, DataVT,
138                                  {NewLoadV, PassThru, Mask, AVL});
139     SDValue NewLoadChainV = SDValue(NewLoadV.getNode(), 1);
140 
141     // Merge them back into one node.
142     return CDAG.getMergeValues({DataV, NewLoadChainV});
143   }
144 
145   // VVP_STORE
146   assert(VVPOpc == VEISD::VVP_STORE);
147   return CDAG.getNode(VEISD::VVP_STORE, Op.getNode()->getVTList(),
148                       {Chain, Data, BasePtr, StrideV, Mask, AVL});
149 }
150 
151 SDValue VETargetLowering::splitPackedLoadStore(SDValue Op,
152                                                VECustomDAG &CDAG) const {
153   auto VVPOC = *getVVPOpcode(Op.getOpcode());
154   assert((VVPOC == VEISD::VVP_LOAD) || (VVPOC == VEISD::VVP_STORE));
155 
156   MVT DataVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT();
157   assert(getTypePacking(DataVT) == Packing::Dense &&
158          "Can only split packed load/store");
159   MVT SplitDataVT = splitVectorType(DataVT);
160 
161   SDValue PassThru = getNodePassthru(Op);
162   assert(!PassThru && "Should have been folded in lowering to VVP layer");
163 
164   // Analyze the operation
165   SDValue PackedMask = getNodeMask(Op);
166   SDValue PackedAVL = getAnnotatedNodeAVL(Op).first;
167   SDValue PackPtr = getMemoryPtr(Op);
168   SDValue PackData = getStoredValue(Op);
169   SDValue PackStride = getLoadStoreStride(Op, CDAG);
170 
171   unsigned ChainResIdx = PackData ? 0 : 1;
172 
173   SDValue PartOps[2];
174 
175   SDValue UpperPartAVL; // we will use this for packing things back together
176   for (PackElem Part : {PackElem::Hi, PackElem::Lo}) {
177     // VP ops already have an explicit mask and AVL. When expanding from non-VP
178     // attach those additional inputs here.
179     auto SplitTM = CDAG.getTargetSplitMask(PackedMask, PackedAVL, Part);
180 
181     // Keep track of the (higher) lvl.
182     if (Part == PackElem::Hi)
183       UpperPartAVL = SplitTM.AVL;
184 
185     // Attach non-predicating value operands
186     SmallVector<SDValue, 4> OpVec;
187 
188     // Chain
189     OpVec.push_back(getNodeChain(Op));
190 
191     // Data
192     if (PackData) {
193       SDValue PartData =
194           CDAG.getUnpack(SplitDataVT, PackData, Part, SplitTM.AVL);
195       OpVec.push_back(PartData);
196     }
197 
198     // Ptr & Stride
199     // Push (ptr + ElemBytes * <Part>, 2 * ElemBytes)
200     // Stride info
201     // EVT DataVT = LegalizeVectorType(getMemoryDataVT(Op), Op, DAG, Mode);
202     OpVec.push_back(CDAG.getSplitPtrOffset(PackPtr, PackStride, Part));
203     OpVec.push_back(CDAG.getSplitPtrStride(PackStride));
204 
205     // Add predicating args and generate part node
206     OpVec.push_back(SplitTM.Mask);
207     OpVec.push_back(SplitTM.AVL);
208 
209     if (PackData) {
210       // Store
211       PartOps[(int)Part] = CDAG.getNode(VVPOC, MVT::Other, OpVec);
212     } else {
213       // Load
214       PartOps[(int)Part] =
215           CDAG.getNode(VVPOC, {SplitDataVT, MVT::Other}, OpVec);
216     }
217   }
218 
219   // Merge the chains
220   SDValue LowChain = SDValue(PartOps[(int)PackElem::Lo].getNode(), ChainResIdx);
221   SDValue HiChain = SDValue(PartOps[(int)PackElem::Hi].getNode(), ChainResIdx);
222   SDValue FusedChains =
223       CDAG.getNode(ISD::TokenFactor, MVT::Other, {LowChain, HiChain});
224 
225   // Chain only [store]
226   if (PackData)
227     return FusedChains;
228 
229   // Re-pack into full packed vector result
230   MVT PackedVT =
231       getLegalVectorType(Packing::Dense, DataVT.getVectorElementType());
232   SDValue PackedVals = CDAG.getPack(PackedVT, PartOps[(int)PackElem::Lo],
233                                     PartOps[(int)PackElem::Hi], UpperPartAVL);
234 
235   return CDAG.getMergeValues({PackedVals, FusedChains});
236 }
237 
238 SDValue VETargetLowering::legalizeInternalLoadStoreOp(SDValue Op,
239                                                       VECustomDAG &CDAG) const {
240   LLVM_DEBUG(dbgs() << "::legalizeInternalLoadStoreOp\n";);
241   MVT DataVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT();
242 
243   // TODO: Recognize packable load,store.
244   if (isPackedVectorType(DataVT))
245     return splitPackedLoadStore(Op, CDAG);
246 
247   return legalizePackedAVL(Op, CDAG);
248 }
249 
250 SDValue VETargetLowering::legalizeInternalVectorOp(SDValue Op,
251                                                    SelectionDAG &DAG) const {
252   LLVM_DEBUG(dbgs() << "::legalizeInternalVectorOp\n";);
253   VECustomDAG CDAG(DAG, Op);
254 
255   // Dispatch to specialized legalization functions.
256   switch (Op->getOpcode()) {
257   case VEISD::VVP_LOAD:
258   case VEISD::VVP_STORE:
259     return legalizeInternalLoadStoreOp(Op, CDAG);
260   }
261 
262   EVT IdiomVT = Op.getValueType();
263   if (isPackedVectorType(IdiomVT) &&
264       !supportsPackedMode(Op.getOpcode(), IdiomVT))
265     return splitVectorOp(Op, CDAG);
266 
267   // TODO: Implement odd/even splitting.
268   return legalizePackedAVL(Op, CDAG);
269 }
270 
271 SDValue VETargetLowering::splitVectorOp(SDValue Op, VECustomDAG &CDAG) const {
272   MVT ResVT = splitVectorType(Op.getValue(0).getSimpleValueType());
273 
274   auto AVLPos = getAVLPos(Op->getOpcode());
275   auto MaskPos = getMaskPos(Op->getOpcode());
276 
277   SDValue PackedMask = getNodeMask(Op);
278   auto AVLPair = getAnnotatedNodeAVL(Op);
279   SDValue PackedAVL = AVLPair.first;
280   assert(!AVLPair.second && "Expecting non pack-legalized oepration");
281 
282   // request the parts
283   SDValue PartOps[2];
284 
285   SDValue UpperPartAVL; // we will use this for packing things back together
286   for (PackElem Part : {PackElem::Hi, PackElem::Lo}) {
287     // VP ops already have an explicit mask and AVL. When expanding from non-VP
288     // attach those additional inputs here.
289     auto SplitTM = CDAG.getTargetSplitMask(PackedMask, PackedAVL, Part);
290 
291     if (Part == PackElem::Hi)
292       UpperPartAVL = SplitTM.AVL;
293 
294     // Attach non-predicating value operands
295     SmallVector<SDValue, 4> OpVec;
296     for (unsigned i = 0; i < Op.getNumOperands(); ++i) {
297       if (AVLPos && ((int)i) == *AVLPos)
298         continue;
299       if (MaskPos && ((int)i) == *MaskPos)
300         continue;
301 
302       // Value operand
303       auto PackedOperand = Op.getOperand(i);
304       auto UnpackedOpVT = splitVectorType(PackedOperand.getSimpleValueType());
305       SDValue PartV =
306           CDAG.getUnpack(UnpackedOpVT, PackedOperand, Part, SplitTM.AVL);
307       OpVec.push_back(PartV);
308     }
309 
310     // Add predicating args and generate part node.
311     OpVec.push_back(SplitTM.Mask);
312     OpVec.push_back(SplitTM.AVL);
313     // Emit legal VVP nodes.
314     PartOps[(int)Part] =
315         CDAG.getNode(Op.getOpcode(), ResVT, OpVec, Op->getFlags());
316   }
317 
318   // Re-package vectors.
319   return CDAG.getPack(Op.getValueType(), PartOps[(int)PackElem::Lo],
320                       PartOps[(int)PackElem::Hi], UpperPartAVL);
321 }
322 
323 SDValue VETargetLowering::legalizePackedAVL(SDValue Op,
324                                             VECustomDAG &CDAG) const {
325   LLVM_DEBUG(dbgs() << "::legalizePackedAVL\n";);
326   // Only required for VEC and VVP ops.
327   if (!isVVPOrVEC(Op->getOpcode()))
328     return Op;
329 
330   // Operation already has a legal AVL.
331   auto AVL = getNodeAVL(Op);
332   if (isLegalAVL(AVL))
333     return Op;
334 
335   // Half and round up EVL for 32bit element types.
336   SDValue LegalAVL = AVL;
337   MVT IdiomVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT();
338   if (isPackedVectorType(IdiomVT)) {
339     assert(maySafelyIgnoreMask(Op) &&
340            "TODO Shift predication from EVL into Mask");
341 
342     if (auto *ConstAVL = dyn_cast<ConstantSDNode>(AVL)) {
343       LegalAVL = CDAG.getConstant((ConstAVL->getZExtValue() + 1) / 2, MVT::i32);
344     } else {
345       auto ConstOne = CDAG.getConstant(1, MVT::i32);
346       auto PlusOne = CDAG.getNode(ISD::ADD, MVT::i32, {AVL, ConstOne});
347       LegalAVL = CDAG.getNode(ISD::SRL, MVT::i32, {PlusOne, ConstOne});
348     }
349   }
350 
351   SDValue AnnotatedLegalAVL = CDAG.annotateLegalAVL(LegalAVL);
352 
353   // Copy the operand list.
354   int NumOp = Op->getNumOperands();
355   auto AVLPos = getAVLPos(Op->getOpcode());
356   std::vector<SDValue> FixedOperands;
357   for (int i = 0; i < NumOp; ++i) {
358     if (AVLPos && (i == *AVLPos)) {
359       FixedOperands.push_back(AnnotatedLegalAVL);
360       continue;
361     }
362     FixedOperands.push_back(Op->getOperand(i));
363   }
364 
365   // Clone the operation with fixed operands.
366   auto Flags = Op->getFlags();
367   SDValue NewN =
368       CDAG.getNode(Op->getOpcode(), Op->getVTList(), FixedOperands, Flags);
369   return NewN;
370 }
371