1 //===-- VVPISelLowering.cpp - VE DAG Lowering Implementation --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the lowering and legalization of vector instructions to
10 // VVP_*layer SDNodes.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "VECustomDAG.h"
15 #include "VEISelLowering.h"
16 
17 using namespace llvm;
18 
19 #define DEBUG_TYPE "ve-lower"
20 
21 SDValue VETargetLowering::splitMaskArithmetic(SDValue Op,
22                                               SelectionDAG &DAG) const {
23   VECustomDAG CDAG(DAG, Op);
24   SDValue AVL =
25       CDAG.getConstant(Op.getValueType().getVectorNumElements(), MVT::i32);
26   SDValue A = Op->getOperand(0);
27   SDValue B = Op->getOperand(1);
28   SDValue LoA = CDAG.getUnpack(MVT::v256i1, A, PackElem::Lo, AVL);
29   SDValue HiA = CDAG.getUnpack(MVT::v256i1, A, PackElem::Hi, AVL);
30   SDValue LoB = CDAG.getUnpack(MVT::v256i1, B, PackElem::Lo, AVL);
31   SDValue HiB = CDAG.getUnpack(MVT::v256i1, B, PackElem::Hi, AVL);
32   unsigned Opc = Op.getOpcode();
33   auto LoRes = CDAG.getNode(Opc, MVT::v256i1, {LoA, LoB});
34   auto HiRes = CDAG.getNode(Opc, MVT::v256i1, {HiA, HiB});
35   return CDAG.getPack(MVT::v512i1, LoRes, HiRes, AVL);
36 }
37 
38 SDValue VETargetLowering::lowerToVVP(SDValue Op, SelectionDAG &DAG) const {
39   // Can we represent this as a VVP node.
40   const unsigned Opcode = Op->getOpcode();
41   auto VVPOpcodeOpt = getVVPOpcode(Opcode);
42   if (!VVPOpcodeOpt.hasValue())
43     return SDValue();
44   unsigned VVPOpcode = VVPOpcodeOpt.getValue();
45   const bool FromVP = ISD::isVPOpcode(Opcode);
46 
47   // The representative and legalized vector type of this operation.
48   VECustomDAG CDAG(DAG, Op);
49   // Dispatch to complex lowering functions.
50   switch (VVPOpcode) {
51   case VEISD::VVP_LOAD:
52   case VEISD::VVP_STORE:
53     return lowerVVP_LOAD_STORE(Op, CDAG);
54   case VEISD::VVP_GATHER:
55   case VEISD::VVP_SCATTER:
56     return lowerVVP_GATHER_SCATTER(Op, CDAG);
57   }
58 
59   EVT OpVecVT = *getIdiomaticVectorType(Op.getNode());
60   EVT LegalVecVT = getTypeToTransformTo(*DAG.getContext(), OpVecVT);
61   auto Packing = getTypePacking(LegalVecVT.getSimpleVT());
62 
63   SDValue AVL;
64   SDValue Mask;
65 
66   if (FromVP) {
67     // All upstream VP SDNodes always have a mask and avl.
68     auto MaskIdx = ISD::getVPMaskIdx(Opcode);
69     auto AVLIdx = ISD::getVPExplicitVectorLengthIdx(Opcode);
70     if (MaskIdx)
71       Mask = Op->getOperand(*MaskIdx);
72     if (AVLIdx)
73       AVL = Op->getOperand(*AVLIdx);
74   }
75 
76   // Materialize default mask and avl.
77   if (!AVL)
78     AVL = CDAG.getConstant(OpVecVT.getVectorNumElements(), MVT::i32);
79   if (!Mask)
80     Mask = CDAG.getConstantMask(Packing, true);
81 
82   if (isVVPBinaryOp(VVPOpcode)) {
83     assert(LegalVecVT.isSimple());
84     return CDAG.getNode(VVPOpcode, LegalVecVT,
85                         {Op->getOperand(0), Op->getOperand(1), Mask, AVL});
86   }
87   if (isVVPReductionOp(VVPOpcode)) {
88     auto SrcHasStart = hasReductionStartParam(Op->getOpcode());
89     SDValue StartV = SrcHasStart ? Op->getOperand(0) : SDValue();
90     SDValue VectorV = Op->getOperand(SrcHasStart ? 1 : 0);
91     return CDAG.getLegalReductionOpVVP(VVPOpcode, Op.getValueType(), StartV,
92                                        VectorV, Mask, AVL, Op->getFlags());
93   }
94 
95   switch (VVPOpcode) {
96   default:
97     llvm_unreachable("lowerToVVP called for unexpected SDNode.");
98   case VEISD::VVP_FFMA: {
99     // VE has a swizzled operand order in FMA (compared to LLVM IR and
100     // SDNodes).
101     auto X = Op->getOperand(2);
102     auto Y = Op->getOperand(0);
103     auto Z = Op->getOperand(1);
104     return CDAG.getNode(VVPOpcode, LegalVecVT, {X, Y, Z, Mask, AVL});
105   }
106   case VEISD::VVP_SELECT: {
107     auto Mask = Op->getOperand(0);
108     auto OnTrue = Op->getOperand(1);
109     auto OnFalse = Op->getOperand(2);
110     return CDAG.getNode(VVPOpcode, LegalVecVT, {OnTrue, OnFalse, Mask, AVL});
111   }
112   case VEISD::VVP_SETCC: {
113     EVT LegalResVT = getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
114     auto LHS = Op->getOperand(0);
115     auto RHS = Op->getOperand(1);
116     auto Pred = Op->getOperand(2);
117     return CDAG.getNode(VVPOpcode, LegalResVT, {LHS, RHS, Pred, Mask, AVL});
118   }
119   }
120 }
121 
122 SDValue VETargetLowering::lowerVVP_LOAD_STORE(SDValue Op,
123                                               VECustomDAG &CDAG) const {
124   auto VVPOpc = *getVVPOpcode(Op->getOpcode());
125   const bool IsLoad = (VVPOpc == VEISD::VVP_LOAD);
126 
127   // Shares.
128   SDValue BasePtr = getMemoryPtr(Op);
129   SDValue Mask = getNodeMask(Op);
130   SDValue Chain = getNodeChain(Op);
131   SDValue AVL = getNodeAVL(Op);
132   // Store specific.
133   SDValue Data = getStoredValue(Op);
134   // Load specific.
135   SDValue PassThru = getNodePassthru(Op);
136 
137   SDValue StrideV = getLoadStoreStride(Op, CDAG);
138 
139   auto DataVT = *getIdiomaticVectorType(Op.getNode());
140   auto Packing = getTypePacking(DataVT);
141 
142   // TODO: Infer lower AVL from mask.
143   if (!AVL)
144     AVL = CDAG.getConstant(DataVT.getVectorNumElements(), MVT::i32);
145 
146   // Default to the all-true mask.
147   if (!Mask)
148     Mask = CDAG.getConstantMask(Packing, true);
149 
150   if (IsLoad) {
151     MVT LegalDataVT = getLegalVectorType(
152         Packing, DataVT.getVectorElementType().getSimpleVT());
153 
154     auto NewLoadV = CDAG.getNode(VEISD::VVP_LOAD, {LegalDataVT, MVT::Other},
155                                  {Chain, BasePtr, StrideV, Mask, AVL});
156 
157     if (!PassThru || PassThru->isUndef())
158       return NewLoadV;
159 
160     // Convert passthru to an explicit select node.
161     SDValue DataV = CDAG.getNode(VEISD::VVP_SELECT, DataVT,
162                                  {NewLoadV, PassThru, Mask, AVL});
163     SDValue NewLoadChainV = SDValue(NewLoadV.getNode(), 1);
164 
165     // Merge them back into one node.
166     return CDAG.getMergeValues({DataV, NewLoadChainV});
167   }
168 
169   // VVP_STORE
170   assert(VVPOpc == VEISD::VVP_STORE);
171   return CDAG.getNode(VEISD::VVP_STORE, Op.getNode()->getVTList(),
172                       {Chain, Data, BasePtr, StrideV, Mask, AVL});
173 }
174 
175 SDValue VETargetLowering::splitPackedLoadStore(SDValue Op,
176                                                VECustomDAG &CDAG) const {
177   auto VVPOC = *getVVPOpcode(Op.getOpcode());
178   assert((VVPOC == VEISD::VVP_LOAD) || (VVPOC == VEISD::VVP_STORE));
179 
180   MVT DataVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT();
181   assert(getTypePacking(DataVT) == Packing::Dense &&
182          "Can only split packed load/store");
183   MVT SplitDataVT = splitVectorType(DataVT);
184 
185   SDValue PassThru = getNodePassthru(Op);
186   assert(!PassThru && "Should have been folded in lowering to VVP layer");
187 
188   // Analyze the operation
189   SDValue PackedMask = getNodeMask(Op);
190   SDValue PackedAVL = getAnnotatedNodeAVL(Op).first;
191   SDValue PackPtr = getMemoryPtr(Op);
192   SDValue PackData = getStoredValue(Op);
193   SDValue PackStride = getLoadStoreStride(Op, CDAG);
194 
195   unsigned ChainResIdx = PackData ? 0 : 1;
196 
197   SDValue PartOps[2];
198 
199   SDValue UpperPartAVL; // we will use this for packing things back together
200   for (PackElem Part : {PackElem::Hi, PackElem::Lo}) {
201     // VP ops already have an explicit mask and AVL. When expanding from non-VP
202     // attach those additional inputs here.
203     auto SplitTM = CDAG.getTargetSplitMask(PackedMask, PackedAVL, Part);
204 
205     // Keep track of the (higher) lvl.
206     if (Part == PackElem::Hi)
207       UpperPartAVL = SplitTM.AVL;
208 
209     // Attach non-predicating value operands
210     SmallVector<SDValue, 4> OpVec;
211 
212     // Chain
213     OpVec.push_back(getNodeChain(Op));
214 
215     // Data
216     if (PackData) {
217       SDValue PartData =
218           CDAG.getUnpack(SplitDataVT, PackData, Part, SplitTM.AVL);
219       OpVec.push_back(PartData);
220     }
221 
222     // Ptr & Stride
223     // Push (ptr + ElemBytes * <Part>, 2 * ElemBytes)
224     // Stride info
225     // EVT DataVT = LegalizeVectorType(getMemoryDataVT(Op), Op, DAG, Mode);
226     OpVec.push_back(CDAG.getSplitPtrOffset(PackPtr, PackStride, Part));
227     OpVec.push_back(CDAG.getSplitPtrStride(PackStride));
228 
229     // Add predicating args and generate part node
230     OpVec.push_back(SplitTM.Mask);
231     OpVec.push_back(SplitTM.AVL);
232 
233     if (PackData) {
234       // Store
235       PartOps[(int)Part] = CDAG.getNode(VVPOC, MVT::Other, OpVec);
236     } else {
237       // Load
238       PartOps[(int)Part] =
239           CDAG.getNode(VVPOC, {SplitDataVT, MVT::Other}, OpVec);
240     }
241   }
242 
243   // Merge the chains
244   SDValue LowChain = SDValue(PartOps[(int)PackElem::Lo].getNode(), ChainResIdx);
245   SDValue HiChain = SDValue(PartOps[(int)PackElem::Hi].getNode(), ChainResIdx);
246   SDValue FusedChains =
247       CDAG.getNode(ISD::TokenFactor, MVT::Other, {LowChain, HiChain});
248 
249   // Chain only [store]
250   if (PackData)
251     return FusedChains;
252 
253   // Re-pack into full packed vector result
254   MVT PackedVT =
255       getLegalVectorType(Packing::Dense, DataVT.getVectorElementType());
256   SDValue PackedVals = CDAG.getPack(PackedVT, PartOps[(int)PackElem::Lo],
257                                     PartOps[(int)PackElem::Hi], UpperPartAVL);
258 
259   return CDAG.getMergeValues({PackedVals, FusedChains});
260 }
261 
262 SDValue VETargetLowering::lowerVVP_GATHER_SCATTER(SDValue Op,
263                                                   VECustomDAG &CDAG) const {
264   EVT DataVT = *getIdiomaticVectorType(Op.getNode());
265   auto Packing = getTypePacking(DataVT);
266   MVT LegalDataVT =
267       getLegalVectorType(Packing, DataVT.getVectorElementType().getSimpleVT());
268 
269   SDValue AVL = getAnnotatedNodeAVL(Op).first;
270   SDValue Index = getGatherScatterIndex(Op);
271   SDValue BasePtr = getMemoryPtr(Op);
272   SDValue Mask = getNodeMask(Op);
273   SDValue Chain = getNodeChain(Op);
274   SDValue Scale = getGatherScatterScale(Op);
275   SDValue PassThru = getNodePassthru(Op);
276   SDValue StoredValue = getStoredValue(Op);
277   if (PassThru && PassThru->isUndef())
278     PassThru = SDValue();
279 
280   bool IsScatter = (bool)StoredValue;
281 
282   // TODO: Infer lower AVL from mask.
283   if (!AVL)
284     AVL = CDAG.getConstant(DataVT.getVectorNumElements(), MVT::i32);
285 
286   // Default to the all-true mask.
287   if (!Mask)
288     Mask = CDAG.getConstantMask(Packing, true);
289 
290   SDValue AddressVec =
291       CDAG.getGatherScatterAddress(BasePtr, Scale, Index, Mask, AVL);
292   if (IsScatter)
293     return CDAG.getNode(VEISD::VVP_SCATTER, MVT::Other,
294                         {Chain, StoredValue, AddressVec, Mask, AVL});
295 
296   // Gather.
297   SDValue NewLoadV = CDAG.getNode(VEISD::VVP_GATHER, {LegalDataVT, MVT::Other},
298                                   {Chain, AddressVec, Mask, AVL});
299 
300   if (!PassThru)
301     return NewLoadV;
302 
303   // TODO: Use vvp_select
304   SDValue DataV = CDAG.getNode(VEISD::VVP_SELECT, LegalDataVT,
305                                {NewLoadV, PassThru, Mask, AVL});
306   SDValue NewLoadChainV = SDValue(NewLoadV.getNode(), 1);
307   return CDAG.getMergeValues({DataV, NewLoadChainV});
308 }
309 
310 SDValue VETargetLowering::legalizeInternalLoadStoreOp(SDValue Op,
311                                                       VECustomDAG &CDAG) const {
312   LLVM_DEBUG(dbgs() << "::legalizeInternalLoadStoreOp\n";);
313   MVT DataVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT();
314 
315   // TODO: Recognize packable load,store.
316   if (isPackedVectorType(DataVT))
317     return splitPackedLoadStore(Op, CDAG);
318 
319   return legalizePackedAVL(Op, CDAG);
320 }
321 
322 SDValue VETargetLowering::legalizeInternalVectorOp(SDValue Op,
323                                                    SelectionDAG &DAG) const {
324   LLVM_DEBUG(dbgs() << "::legalizeInternalVectorOp\n";);
325   VECustomDAG CDAG(DAG, Op);
326 
327   // Dispatch to specialized legalization functions.
328   switch (Op->getOpcode()) {
329   case VEISD::VVP_LOAD:
330   case VEISD::VVP_STORE:
331     return legalizeInternalLoadStoreOp(Op, CDAG);
332   }
333 
334   EVT IdiomVT = Op.getValueType();
335   if (isPackedVectorType(IdiomVT) &&
336       !supportsPackedMode(Op.getOpcode(), IdiomVT))
337     return splitVectorOp(Op, CDAG);
338 
339   // TODO: Implement odd/even splitting.
340   return legalizePackedAVL(Op, CDAG);
341 }
342 
343 SDValue VETargetLowering::splitVectorOp(SDValue Op, VECustomDAG &CDAG) const {
344   MVT ResVT = splitVectorType(Op.getValue(0).getSimpleValueType());
345 
346   auto AVLPos = getAVLPos(Op->getOpcode());
347   auto MaskPos = getMaskPos(Op->getOpcode());
348 
349   SDValue PackedMask = getNodeMask(Op);
350   auto AVLPair = getAnnotatedNodeAVL(Op);
351   SDValue PackedAVL = AVLPair.first;
352   assert(!AVLPair.second && "Expecting non pack-legalized oepration");
353 
354   // request the parts
355   SDValue PartOps[2];
356 
357   SDValue UpperPartAVL; // we will use this for packing things back together
358   for (PackElem Part : {PackElem::Hi, PackElem::Lo}) {
359     // VP ops already have an explicit mask and AVL. When expanding from non-VP
360     // attach those additional inputs here.
361     auto SplitTM = CDAG.getTargetSplitMask(PackedMask, PackedAVL, Part);
362 
363     if (Part == PackElem::Hi)
364       UpperPartAVL = SplitTM.AVL;
365 
366     // Attach non-predicating value operands
367     SmallVector<SDValue, 4> OpVec;
368     for (unsigned i = 0; i < Op.getNumOperands(); ++i) {
369       if (AVLPos && ((int)i) == *AVLPos)
370         continue;
371       if (MaskPos && ((int)i) == *MaskPos)
372         continue;
373 
374       // Value operand
375       auto PackedOperand = Op.getOperand(i);
376       auto UnpackedOpVT = splitVectorType(PackedOperand.getSimpleValueType());
377       SDValue PartV =
378           CDAG.getUnpack(UnpackedOpVT, PackedOperand, Part, SplitTM.AVL);
379       OpVec.push_back(PartV);
380     }
381 
382     // Add predicating args and generate part node.
383     OpVec.push_back(SplitTM.Mask);
384     OpVec.push_back(SplitTM.AVL);
385     // Emit legal VVP nodes.
386     PartOps[(int)Part] =
387         CDAG.getNode(Op.getOpcode(), ResVT, OpVec, Op->getFlags());
388   }
389 
390   // Re-package vectors.
391   return CDAG.getPack(Op.getValueType(), PartOps[(int)PackElem::Lo],
392                       PartOps[(int)PackElem::Hi], UpperPartAVL);
393 }
394 
395 SDValue VETargetLowering::legalizePackedAVL(SDValue Op,
396                                             VECustomDAG &CDAG) const {
397   LLVM_DEBUG(dbgs() << "::legalizePackedAVL\n";);
398   // Only required for VEC and VVP ops.
399   if (!isVVPOrVEC(Op->getOpcode()))
400     return Op;
401 
402   // Operation already has a legal AVL.
403   auto AVL = getNodeAVL(Op);
404   if (isLegalAVL(AVL))
405     return Op;
406 
407   // Half and round up EVL for 32bit element types.
408   SDValue LegalAVL = AVL;
409   MVT IdiomVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT();
410   if (isPackedVectorType(IdiomVT)) {
411     assert(maySafelyIgnoreMask(Op) &&
412            "TODO Shift predication from EVL into Mask");
413 
414     if (auto *ConstAVL = dyn_cast<ConstantSDNode>(AVL)) {
415       LegalAVL = CDAG.getConstant((ConstAVL->getZExtValue() + 1) / 2, MVT::i32);
416     } else {
417       auto ConstOne = CDAG.getConstant(1, MVT::i32);
418       auto PlusOne = CDAG.getNode(ISD::ADD, MVT::i32, {AVL, ConstOne});
419       LegalAVL = CDAG.getNode(ISD::SRL, MVT::i32, {PlusOne, ConstOne});
420     }
421   }
422 
423   SDValue AnnotatedLegalAVL = CDAG.annotateLegalAVL(LegalAVL);
424 
425   // Copy the operand list.
426   int NumOp = Op->getNumOperands();
427   auto AVLPos = getAVLPos(Op->getOpcode());
428   std::vector<SDValue> FixedOperands;
429   for (int i = 0; i < NumOp; ++i) {
430     if (AVLPos && (i == *AVLPos)) {
431       FixedOperands.push_back(AnnotatedLegalAVL);
432       continue;
433     }
434     FixedOperands.push_back(Op->getOperand(i));
435   }
436 
437   // Clone the operation with fixed operands.
438   auto Flags = Op->getFlags();
439   SDValue NewN =
440       CDAG.getNode(Op->getOpcode(), Op->getVTList(), FixedOperands, Flags);
441   return NewN;
442 }
443