1 //===-- VECustomDAG.h - VE Custom DAG Nodes ------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that VE uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "VECustomDAG.h"
15 
16 #ifndef DEBUG_TYPE
17 #define DEBUG_TYPE "vecustomdag"
18 #endif
19 
20 namespace llvm {
21 
22 bool isPackedVectorType(EVT SomeVT) {
23   if (!SomeVT.isVector())
24     return false;
25   return SomeVT.getVectorNumElements() > StandardVectorWidth;
26 }
27 
28 MVT splitVectorType(MVT VT) {
29   if (!VT.isVector())
30     return VT;
31   return MVT::getVectorVT(VT.getVectorElementType(), StandardVectorWidth);
32 }
33 
34 MVT getLegalVectorType(Packing P, MVT ElemVT) {
35   return MVT::getVectorVT(ElemVT, P == Packing::Normal ? StandardVectorWidth
36                                                        : PackedVectorWidth);
37 }
38 
39 Packing getTypePacking(EVT VT) {
40   assert(VT.isVector());
41   return isPackedVectorType(VT) ? Packing::Dense : Packing::Normal;
42 }
43 
44 bool isMaskType(EVT SomeVT) {
45   if (!SomeVT.isVector())
46     return false;
47   return SomeVT.getVectorElementType() == MVT::i1;
48 }
49 
50 bool isMaskArithmetic(SDValue Op) {
51   switch (Op.getOpcode()) {
52   default:
53     return false;
54   case ISD::AND:
55   case ISD::XOR:
56   case ISD::OR:
57     return isMaskType(Op.getValueType());
58   }
59 }
60 
61 /// \returns the VVP_* SDNode opcode corresponsing to \p OC.
62 Optional<unsigned> getVVPOpcode(unsigned Opcode) {
63   switch (Opcode) {
64   case ISD::MLOAD:
65     return VEISD::VVP_LOAD;
66   case ISD::MSTORE:
67     return VEISD::VVP_STORE;
68 #define HANDLE_VP_TO_VVP(VPOPC, VVPNAME)                                       \
69   case ISD::VPOPC:                                                             \
70     return VEISD::VVPNAME;
71 #define ADD_VVP_OP(VVPNAME, SDNAME)                                            \
72   case VEISD::VVPNAME:                                                         \
73   case ISD::SDNAME:                                                            \
74     return VEISD::VVPNAME;
75 #include "VVPNodes.def"
76   }
77   return None;
78 }
79 
80 bool maySafelyIgnoreMask(SDValue Op) {
81   auto VVPOpc = getVVPOpcode(Op->getOpcode());
82   auto Opc = VVPOpc.getValueOr(Op->getOpcode());
83 
84   switch (Opc) {
85   case VEISD::VVP_SDIV:
86   case VEISD::VVP_UDIV:
87   case VEISD::VVP_FDIV:
88   case VEISD::VVP_SELECT:
89     return false;
90 
91   default:
92     return true;
93   }
94 }
95 
96 bool supportsPackedMode(unsigned Opcode, EVT IdiomVT) {
97   bool IsPackedOp = isPackedVectorType(IdiomVT);
98   bool IsMaskOp = isMaskType(IdiomVT);
99   switch (Opcode) {
100   default:
101     return false;
102 
103   case VEISD::VEC_BROADCAST:
104     return true;
105 #define REGISTER_PACKED(VVP_NAME) case VEISD::VVP_NAME:
106 #include "VVPNodes.def"
107     return IsPackedOp && !IsMaskOp;
108   }
109 }
110 
111 bool isPackingSupportOpcode(unsigned Opc) {
112   switch (Opc) {
113   case VEISD::VEC_PACK:
114   case VEISD::VEC_UNPACK_LO:
115   case VEISD::VEC_UNPACK_HI:
116     return true;
117   }
118   return false;
119 }
120 
121 bool isVVPOrVEC(unsigned Opcode) {
122   switch (Opcode) {
123   case VEISD::VEC_BROADCAST:
124 #define ADD_VVP_OP(VVPNAME, ...) case VEISD::VVPNAME:
125 #include "VVPNodes.def"
126     return true;
127   }
128   return false;
129 }
130 
131 bool isVVPBinaryOp(unsigned VVPOpcode) {
132   switch (VVPOpcode) {
133 #define ADD_BINARY_VVP_OP(VVPNAME, ...)                                        \
134   case VEISD::VVPNAME:                                                         \
135     return true;
136 #include "VVPNodes.def"
137   }
138   return false;
139 }
140 
141 // Return the AVL operand position for this VVP or VEC Op.
142 Optional<int> getAVLPos(unsigned Opc) {
143   // This is only available for VP SDNodes
144   auto PosOpt = ISD::getVPExplicitVectorLengthIdx(Opc);
145   if (PosOpt)
146     return *PosOpt;
147 
148   // VVP Opcodes.
149   if (isVVPBinaryOp(Opc))
150     return 3;
151 
152   // VM Opcodes.
153   switch (Opc) {
154   case VEISD::VEC_BROADCAST:
155     return 1;
156   case VEISD::VVP_SELECT:
157     return 3;
158   }
159 
160   return None;
161 }
162 
163 Optional<int> getMaskPos(unsigned Opc) {
164   // This is only available for VP SDNodes
165   auto PosOpt = ISD::getVPMaskIdx(Opc);
166   if (PosOpt)
167     return *PosOpt;
168 
169   // VVP Opcodes.
170   if (isVVPBinaryOp(Opc))
171     return 2;
172 
173   // Other opcodes.
174   switch (Opc) {
175   case ISD::MSTORE:
176     return 4;
177   case ISD::MLOAD:
178     return 3;
179   case VEISD::VVP_SELECT:
180     return 2;
181   }
182 
183   return None;
184 }
185 
186 bool isLegalAVL(SDValue AVL) { return AVL->getOpcode() == VEISD::LEGALAVL; }
187 
188 /// Node Properties {
189 
190 SDValue getNodeChain(SDValue Op) {
191   if (MemSDNode *MemN = dyn_cast<MemSDNode>(Op.getNode()))
192     return MemN->getChain();
193 
194   switch (Op->getOpcode()) {
195   case VEISD::VVP_LOAD:
196   case VEISD::VVP_STORE:
197     return Op->getOperand(0);
198   }
199   return SDValue();
200 }
201 
202 SDValue getMemoryPtr(SDValue Op) {
203   if (auto *MemN = dyn_cast<MemSDNode>(Op.getNode()))
204     return MemN->getBasePtr();
205 
206   switch (Op->getOpcode()) {
207   case VEISD::VVP_LOAD:
208     return Op->getOperand(1);
209   case VEISD::VVP_STORE:
210     return Op->getOperand(2);
211   }
212   return SDValue();
213 }
214 
215 Optional<EVT> getIdiomaticVectorType(SDNode *Op) {
216   unsigned OC = Op->getOpcode();
217 
218   // For memory ops -> the transfered data type
219   if (auto MemN = dyn_cast<MemSDNode>(Op))
220     return MemN->getMemoryVT();
221 
222   switch (OC) {
223   // Standard ISD.
224   case ISD::SELECT: // not aliased with VVP_SELECT
225   case ISD::CONCAT_VECTORS:
226   case ISD::EXTRACT_SUBVECTOR:
227   case ISD::VECTOR_SHUFFLE:
228   case ISD::BUILD_VECTOR:
229   case ISD::SCALAR_TO_VECTOR:
230     return Op->getValueType(0);
231   }
232 
233   // Translate to VVP where possible.
234   if (auto VVPOpc = getVVPOpcode(OC))
235     OC = *VVPOpc;
236 
237   switch (OC) {
238   default:
239   case VEISD::VVP_SETCC:
240     return Op->getOperand(0).getValueType();
241 
242   case VEISD::VVP_SELECT:
243 #define ADD_BINARY_VVP_OP(VVP_NAME, ...) case VEISD::VVP_NAME:
244 #include "VVPNodes.def"
245     return Op->getValueType(0);
246 
247   case VEISD::VVP_LOAD:
248     return Op->getValueType(0);
249 
250   case VEISD::VVP_STORE:
251     return Op->getOperand(1)->getValueType(0);
252 
253   // VEC
254   case VEISD::VEC_BROADCAST:
255     return Op->getValueType(0);
256   }
257 }
258 
259 SDValue getLoadStoreStride(SDValue Op, VECustomDAG &CDAG) {
260   if (Op->getOpcode() == VEISD::VVP_STORE)
261     return Op->getOperand(3);
262   if (Op->getOpcode() == VEISD::VVP_LOAD)
263     return Op->getOperand(2);
264 
265   if (isa<MemSDNode>(Op.getNode())) {
266     // Regular MLOAD/MSTORE/LOAD/STORE
267     // No stride argument -> use the contiguous element size as stride.
268     uint64_t ElemStride = getIdiomaticVectorType(Op.getNode())
269                               ->getVectorElementType()
270                               .getStoreSize();
271     return CDAG.getConstant(ElemStride, MVT::i64);
272   }
273   return SDValue();
274 }
275 
276 SDValue getStoredValue(SDValue Op) {
277   switch (Op->getOpcode()) {
278   case VEISD::VVP_STORE:
279     return Op->getOperand(1);
280   }
281   if (auto *StoreN = dyn_cast<StoreSDNode>(Op.getNode()))
282     return StoreN->getValue();
283   if (auto *StoreN = dyn_cast<MaskedStoreSDNode>(Op.getNode()))
284     return StoreN->getValue();
285   if (auto *StoreN = dyn_cast<VPStoreSDNode>(Op.getNode()))
286     return StoreN->getValue();
287   return SDValue();
288 }
289 
290 SDValue getNodePassthru(SDValue Op) {
291   if (auto *N = dyn_cast<MaskedLoadSDNode>(Op.getNode()))
292     return N->getPassThru();
293   return SDValue();
294 }
295 
296 /// } Node Properties
297 
298 SDValue getNodeAVL(SDValue Op) {
299   auto PosOpt = getAVLPos(Op->getOpcode());
300   return PosOpt ? Op->getOperand(*PosOpt) : SDValue();
301 }
302 
303 SDValue getNodeMask(SDValue Op) {
304   auto PosOpt = getMaskPos(Op->getOpcode());
305   return PosOpt ? Op->getOperand(*PosOpt) : SDValue();
306 }
307 
308 std::pair<SDValue, bool> getAnnotatedNodeAVL(SDValue Op) {
309   SDValue AVL = getNodeAVL(Op);
310   if (!AVL)
311     return {SDValue(), true};
312   if (isLegalAVL(AVL))
313     return {AVL->getOperand(0), true};
314   return {AVL, false};
315 }
316 
317 SDValue VECustomDAG::getConstant(uint64_t Val, EVT VT, bool IsTarget,
318                                  bool IsOpaque) const {
319   return DAG.getConstant(Val, DL, VT, IsTarget, IsOpaque);
320 }
321 
322 SDValue VECustomDAG::getConstantMask(Packing Packing, bool AllTrue) const {
323   auto MaskVT = getLegalVectorType(Packing, MVT::i1);
324 
325   // VEISelDAGtoDAG will replace this pattern with the constant-true VM.
326   auto TrueVal = DAG.getConstant(-1, DL, MVT::i32);
327   auto AVL = getConstant(MaskVT.getVectorNumElements(), MVT::i32);
328   auto Res = getNode(VEISD::VEC_BROADCAST, MaskVT, {TrueVal, AVL});
329   if (AllTrue)
330     return Res;
331 
332   return DAG.getNOT(DL, Res, Res.getValueType());
333 }
334 
335 SDValue VECustomDAG::getMaskBroadcast(EVT ResultVT, SDValue Scalar,
336                                       SDValue AVL) const {
337   // Constant mask splat.
338   if (auto BcConst = dyn_cast<ConstantSDNode>(Scalar))
339     return getConstantMask(getTypePacking(ResultVT),
340                            BcConst->getSExtValue() != 0);
341 
342   // Expand the broadcast to a vector comparison.
343   auto ScalarBoolVT = Scalar.getSimpleValueType();
344   assert(ScalarBoolVT == MVT::i32);
345 
346   // Cast to i32 ty.
347   SDValue CmpElem = DAG.getSExtOrTrunc(Scalar, DL, MVT::i32);
348   unsigned ElemCount = ResultVT.getVectorNumElements();
349   MVT CmpVecTy = MVT::getVectorVT(ScalarBoolVT, ElemCount);
350 
351   // Broadcast to vector.
352   SDValue BCVec =
353       DAG.getNode(VEISD::VEC_BROADCAST, DL, CmpVecTy, {CmpElem, AVL});
354   SDValue ZeroVec =
355       getBroadcast(CmpVecTy, {DAG.getConstant(0, DL, ScalarBoolVT)}, AVL);
356 
357   MVT BoolVecTy = MVT::getVectorVT(MVT::i1, ElemCount);
358 
359   // Broadcast(Data) != Broadcast(0)
360   // TODO: Use a VVP operation for this.
361   return DAG.getSetCC(DL, BoolVecTy, BCVec, ZeroVec, ISD::CondCode::SETNE);
362 }
363 
364 SDValue VECustomDAG::getBroadcast(EVT ResultVT, SDValue Scalar,
365                                   SDValue AVL) const {
366   assert(ResultVT.isVector());
367   auto ScaVT = Scalar.getValueType();
368 
369   if (isMaskType(ResultVT))
370     return getMaskBroadcast(ResultVT, Scalar, AVL);
371 
372   if (isPackedVectorType(ResultVT)) {
373     // v512x packed mode broadcast
374     // Replicate the scalar reg (f32 or i32) onto the opposing half of the full
375     // scalar register. If it's an I64 type, assume that this has already
376     // happened.
377     if (ScaVT == MVT::f32) {
378       Scalar = getNode(VEISD::REPL_F32, MVT::i64, Scalar);
379     } else if (ScaVT == MVT::i32) {
380       Scalar = getNode(VEISD::REPL_I32, MVT::i64, Scalar);
381     }
382   }
383 
384   return getNode(VEISD::VEC_BROADCAST, ResultVT, {Scalar, AVL});
385 }
386 
387 SDValue VECustomDAG::annotateLegalAVL(SDValue AVL) const {
388   if (isLegalAVL(AVL))
389     return AVL;
390   return getNode(VEISD::LEGALAVL, AVL.getValueType(), AVL);
391 }
392 
393 SDValue VECustomDAG::getUnpack(EVT DestVT, SDValue Vec, PackElem Part,
394                                SDValue AVL) const {
395   assert(getAnnotatedNodeAVL(AVL).second && "Expected a pack-legalized AVL");
396 
397   // TODO: Peek through VEC_PACK and VEC_BROADCAST(REPL_<sth> ..) operands.
398   unsigned OC =
399       (Part == PackElem::Lo) ? VEISD::VEC_UNPACK_LO : VEISD::VEC_UNPACK_HI;
400   return DAG.getNode(OC, DL, DestVT, Vec, AVL);
401 }
402 
403 SDValue VECustomDAG::getPack(EVT DestVT, SDValue LoVec, SDValue HiVec,
404                              SDValue AVL) const {
405   assert(getAnnotatedNodeAVL(AVL).second && "Expected a pack-legalized AVL");
406 
407   // TODO: Peek through VEC_UNPACK_LO|HI operands.
408   return DAG.getNode(VEISD::VEC_PACK, DL, DestVT, LoVec, HiVec, AVL);
409 }
410 
411 VETargetMasks VECustomDAG::getTargetSplitMask(SDValue RawMask, SDValue RawAVL,
412                                               PackElem Part) const {
413   // Adjust AVL for this part
414   SDValue NewAVL;
415   SDValue OneV = getConstant(1, MVT::i32);
416   if (Part == PackElem::Hi)
417     NewAVL = getNode(ISD::ADD, MVT::i32, {RawAVL, OneV});
418   else
419     NewAVL = RawAVL;
420   NewAVL = getNode(ISD::SRL, MVT::i32, {NewAVL, OneV});
421 
422   NewAVL = annotateLegalAVL(NewAVL);
423 
424   // Legalize Mask (unpack or all-true)
425   SDValue NewMask;
426   if (!RawMask)
427     NewMask = getConstantMask(Packing::Normal, true);
428   else
429     NewMask = getUnpack(MVT::v256i1, RawMask, Part, NewAVL);
430 
431   return VETargetMasks(NewMask, NewAVL);
432 }
433 
434 } // namespace llvm
435