1 //===-- VECustomDAG.h - VE Custom DAG Nodes ------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that VE uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "VECustomDAG.h"
15 
16 #ifndef DEBUG_TYPE
17 #define DEBUG_TYPE "vecustomdag"
18 #endif
19 
20 namespace llvm {
21 
22 bool isPackedVectorType(EVT SomeVT) {
23   if (!SomeVT.isVector())
24     return false;
25   return SomeVT.getVectorNumElements() > StandardVectorWidth;
26 }
27 
28 MVT splitVectorType(MVT VT) {
29   if (!VT.isVector())
30     return VT;
31   return MVT::getVectorVT(VT.getVectorElementType(), StandardVectorWidth);
32 }
33 
34 MVT getLegalVectorType(Packing P, MVT ElemVT) {
35   return MVT::getVectorVT(ElemVT, P == Packing::Normal ? StandardVectorWidth
36                                                        : PackedVectorWidth);
37 }
38 
39 Packing getTypePacking(EVT VT) {
40   assert(VT.isVector());
41   return isPackedVectorType(VT) ? Packing::Dense : Packing::Normal;
42 }
43 
44 bool isMaskType(EVT SomeVT) {
45   if (!SomeVT.isVector())
46     return false;
47   return SomeVT.getVectorElementType() == MVT::i1;
48 }
49 
50 bool isMaskArithmetic(SDValue Op) {
51   switch (Op.getOpcode()) {
52   default:
53     return false;
54   case ISD::AND:
55   case ISD::XOR:
56   case ISD::OR:
57     return isMaskType(Op.getValueType());
58   }
59 }
60 
61 /// \returns the VVP_* SDNode opcode corresponsing to \p OC.
62 Optional<unsigned> getVVPOpcode(unsigned Opcode) {
63   switch (Opcode) {
64   case ISD::MLOAD:
65     return VEISD::VVP_LOAD;
66   case ISD::MSTORE:
67     return VEISD::VVP_STORE;
68 #define HANDLE_VP_TO_VVP(VPOPC, VVPNAME)                                       \
69   case ISD::VPOPC:                                                             \
70     return VEISD::VVPNAME;
71 #define ADD_VVP_OP(VVPNAME, SDNAME)                                            \
72   case VEISD::VVPNAME:                                                         \
73   case ISD::SDNAME:                                                            \
74     return VEISD::VVPNAME;
75 #include "VVPNodes.def"
76   }
77   return None;
78 }
79 
80 bool maySafelyIgnoreMask(SDValue Op) {
81   auto VVPOpc = getVVPOpcode(Op->getOpcode());
82   auto Opc = VVPOpc.getValueOr(Op->getOpcode());
83 
84   switch (Opc) {
85   case VEISD::VVP_SDIV:
86   case VEISD::VVP_UDIV:
87   case VEISD::VVP_FDIV:
88   case VEISD::VVP_SELECT:
89     return false;
90 
91   default:
92     return true;
93   }
94 }
95 
96 bool supportsPackedMode(unsigned Opcode, EVT IdiomVT) {
97   bool IsPackedOp = isPackedVectorType(IdiomVT);
98   bool IsMaskOp = isMaskType(IdiomVT);
99   switch (Opcode) {
100   default:
101     return false;
102 
103   case VEISD::VEC_BROADCAST:
104     return true;
105 #define REGISTER_PACKED(VVP_NAME) case VEISD::VVP_NAME:
106 #include "VVPNodes.def"
107     return IsPackedOp && !IsMaskOp;
108   }
109 }
110 
111 bool isPackingSupportOpcode(unsigned Opc) {
112   switch (Opc) {
113   case VEISD::VEC_PACK:
114   case VEISD::VEC_UNPACK_LO:
115   case VEISD::VEC_UNPACK_HI:
116     return true;
117   }
118   return false;
119 }
120 
121 bool isVVPOrVEC(unsigned Opcode) {
122   switch (Opcode) {
123   case VEISD::VEC_BROADCAST:
124 #define ADD_VVP_OP(VVPNAME, ...) case VEISD::VVPNAME:
125 #include "VVPNodes.def"
126     return true;
127   }
128   return false;
129 }
130 
131 bool isVVPBinaryOp(unsigned VVPOpcode) {
132   switch (VVPOpcode) {
133 #define ADD_BINARY_VVP_OP(VVPNAME, ...)                                        \
134   case VEISD::VVPNAME:                                                         \
135     return true;
136 #include "VVPNodes.def"
137   }
138   return false;
139 }
140 
141 // Return the AVL operand position for this VVP or VEC Op.
142 Optional<int> getAVLPos(unsigned Opc) {
143   // This is only available for VP SDNodes
144   auto PosOpt = ISD::getVPExplicitVectorLengthIdx(Opc);
145   if (PosOpt)
146     return *PosOpt;
147 
148   // VVP Opcodes.
149   if (isVVPBinaryOp(Opc))
150     return 3;
151 
152   // VM Opcodes.
153   switch (Opc) {
154   case VEISD::VEC_BROADCAST:
155     return 1;
156   case VEISD::VVP_SELECT:
157     return 3;
158   case VEISD::VVP_LOAD:
159     return 4;
160   case VEISD::VVP_STORE:
161     return 5;
162   }
163 
164   return None;
165 }
166 
167 Optional<int> getMaskPos(unsigned Opc) {
168   // This is only available for VP SDNodes
169   auto PosOpt = ISD::getVPMaskIdx(Opc);
170   if (PosOpt)
171     return *PosOpt;
172 
173   // VVP Opcodes.
174   if (isVVPBinaryOp(Opc))
175     return 2;
176 
177   // Other opcodes.
178   switch (Opc) {
179   case ISD::MSTORE:
180     return 4;
181   case ISD::MLOAD:
182     return 3;
183   case VEISD::VVP_SELECT:
184     return 2;
185   }
186 
187   return None;
188 }
189 
190 bool isLegalAVL(SDValue AVL) { return AVL->getOpcode() == VEISD::LEGALAVL; }
191 
192 /// Node Properties {
193 
194 SDValue getNodeChain(SDValue Op) {
195   if (MemSDNode *MemN = dyn_cast<MemSDNode>(Op.getNode()))
196     return MemN->getChain();
197 
198   switch (Op->getOpcode()) {
199   case VEISD::VVP_LOAD:
200   case VEISD::VVP_STORE:
201     return Op->getOperand(0);
202   }
203   return SDValue();
204 }
205 
206 SDValue getMemoryPtr(SDValue Op) {
207   if (auto *MemN = dyn_cast<MemSDNode>(Op.getNode()))
208     return MemN->getBasePtr();
209 
210   switch (Op->getOpcode()) {
211   case VEISD::VVP_LOAD:
212     return Op->getOperand(1);
213   case VEISD::VVP_STORE:
214     return Op->getOperand(2);
215   }
216   return SDValue();
217 }
218 
219 Optional<EVT> getIdiomaticVectorType(SDNode *Op) {
220   unsigned OC = Op->getOpcode();
221 
222   // For memory ops -> the transfered data type
223   if (auto MemN = dyn_cast<MemSDNode>(Op))
224     return MemN->getMemoryVT();
225 
226   switch (OC) {
227   // Standard ISD.
228   case ISD::SELECT: // not aliased with VVP_SELECT
229   case ISD::CONCAT_VECTORS:
230   case ISD::EXTRACT_SUBVECTOR:
231   case ISD::VECTOR_SHUFFLE:
232   case ISD::BUILD_VECTOR:
233   case ISD::SCALAR_TO_VECTOR:
234     return Op->getValueType(0);
235   }
236 
237   // Translate to VVP where possible.
238   if (auto VVPOpc = getVVPOpcode(OC))
239     OC = *VVPOpc;
240 
241   switch (OC) {
242   default:
243   case VEISD::VVP_SETCC:
244     return Op->getOperand(0).getValueType();
245 
246   case VEISD::VVP_SELECT:
247 #define ADD_BINARY_VVP_OP(VVP_NAME, ...) case VEISD::VVP_NAME:
248 #include "VVPNodes.def"
249     return Op->getValueType(0);
250 
251   case VEISD::VVP_LOAD:
252     return Op->getValueType(0);
253 
254   case VEISD::VVP_STORE:
255     return Op->getOperand(1)->getValueType(0);
256 
257   // VEC
258   case VEISD::VEC_BROADCAST:
259     return Op->getValueType(0);
260   }
261 }
262 
263 SDValue getLoadStoreStride(SDValue Op, VECustomDAG &CDAG) {
264   if (Op->getOpcode() == VEISD::VVP_STORE)
265     return Op->getOperand(3);
266   if (Op->getOpcode() == VEISD::VVP_LOAD)
267     return Op->getOperand(2);
268 
269   if (isa<MemSDNode>(Op.getNode())) {
270     // Regular MLOAD/MSTORE/LOAD/STORE
271     // No stride argument -> use the contiguous element size as stride.
272     uint64_t ElemStride = getIdiomaticVectorType(Op.getNode())
273                               ->getVectorElementType()
274                               .getStoreSize();
275     return CDAG.getConstant(ElemStride, MVT::i64);
276   }
277   return SDValue();
278 }
279 
280 SDValue getStoredValue(SDValue Op) {
281   switch (Op->getOpcode()) {
282   case VEISD::VVP_STORE:
283     return Op->getOperand(1);
284   }
285   if (auto *StoreN = dyn_cast<StoreSDNode>(Op.getNode()))
286     return StoreN->getValue();
287   if (auto *StoreN = dyn_cast<MaskedStoreSDNode>(Op.getNode()))
288     return StoreN->getValue();
289   if (auto *StoreN = dyn_cast<VPStoreSDNode>(Op.getNode()))
290     return StoreN->getValue();
291   return SDValue();
292 }
293 
294 SDValue getNodePassthru(SDValue Op) {
295   if (auto *N = dyn_cast<MaskedLoadSDNode>(Op.getNode()))
296     return N->getPassThru();
297   return SDValue();
298 }
299 
300 /// } Node Properties
301 
302 SDValue getNodeAVL(SDValue Op) {
303   auto PosOpt = getAVLPos(Op->getOpcode());
304   return PosOpt ? Op->getOperand(*PosOpt) : SDValue();
305 }
306 
307 SDValue getNodeMask(SDValue Op) {
308   auto PosOpt = getMaskPos(Op->getOpcode());
309   return PosOpt ? Op->getOperand(*PosOpt) : SDValue();
310 }
311 
312 std::pair<SDValue, bool> getAnnotatedNodeAVL(SDValue Op) {
313   SDValue AVL = getNodeAVL(Op);
314   if (!AVL)
315     return {SDValue(), true};
316   if (isLegalAVL(AVL))
317     return {AVL->getOperand(0), true};
318   return {AVL, false};
319 }
320 
321 SDValue VECustomDAG::getConstant(uint64_t Val, EVT VT, bool IsTarget,
322                                  bool IsOpaque) const {
323   return DAG.getConstant(Val, DL, VT, IsTarget, IsOpaque);
324 }
325 
326 SDValue VECustomDAG::getConstantMask(Packing Packing, bool AllTrue) const {
327   auto MaskVT = getLegalVectorType(Packing, MVT::i1);
328 
329   // VEISelDAGtoDAG will replace this pattern with the constant-true VM.
330   auto TrueVal = DAG.getConstant(-1, DL, MVT::i32);
331   auto AVL = getConstant(MaskVT.getVectorNumElements(), MVT::i32);
332   auto Res = getNode(VEISD::VEC_BROADCAST, MaskVT, {TrueVal, AVL});
333   if (AllTrue)
334     return Res;
335 
336   return DAG.getNOT(DL, Res, Res.getValueType());
337 }
338 
339 SDValue VECustomDAG::getMaskBroadcast(EVT ResultVT, SDValue Scalar,
340                                       SDValue AVL) const {
341   // Constant mask splat.
342   if (auto BcConst = dyn_cast<ConstantSDNode>(Scalar))
343     return getConstantMask(getTypePacking(ResultVT),
344                            BcConst->getSExtValue() != 0);
345 
346   // Expand the broadcast to a vector comparison.
347   auto ScalarBoolVT = Scalar.getSimpleValueType();
348   assert(ScalarBoolVT == MVT::i32);
349 
350   // Cast to i32 ty.
351   SDValue CmpElem = DAG.getSExtOrTrunc(Scalar, DL, MVT::i32);
352   unsigned ElemCount = ResultVT.getVectorNumElements();
353   MVT CmpVecTy = MVT::getVectorVT(ScalarBoolVT, ElemCount);
354 
355   // Broadcast to vector.
356   SDValue BCVec =
357       DAG.getNode(VEISD::VEC_BROADCAST, DL, CmpVecTy, {CmpElem, AVL});
358   SDValue ZeroVec =
359       getBroadcast(CmpVecTy, {DAG.getConstant(0, DL, ScalarBoolVT)}, AVL);
360 
361   MVT BoolVecTy = MVT::getVectorVT(MVT::i1, ElemCount);
362 
363   // Broadcast(Data) != Broadcast(0)
364   // TODO: Use a VVP operation for this.
365   return DAG.getSetCC(DL, BoolVecTy, BCVec, ZeroVec, ISD::CondCode::SETNE);
366 }
367 
368 SDValue VECustomDAG::getBroadcast(EVT ResultVT, SDValue Scalar,
369                                   SDValue AVL) const {
370   assert(ResultVT.isVector());
371   auto ScaVT = Scalar.getValueType();
372 
373   if (isMaskType(ResultVT))
374     return getMaskBroadcast(ResultVT, Scalar, AVL);
375 
376   if (isPackedVectorType(ResultVT)) {
377     // v512x packed mode broadcast
378     // Replicate the scalar reg (f32 or i32) onto the opposing half of the full
379     // scalar register. If it's an I64 type, assume that this has already
380     // happened.
381     if (ScaVT == MVT::f32) {
382       Scalar = getNode(VEISD::REPL_F32, MVT::i64, Scalar);
383     } else if (ScaVT == MVT::i32) {
384       Scalar = getNode(VEISD::REPL_I32, MVT::i64, Scalar);
385     }
386   }
387 
388   return getNode(VEISD::VEC_BROADCAST, ResultVT, {Scalar, AVL});
389 }
390 
391 SDValue VECustomDAG::annotateLegalAVL(SDValue AVL) const {
392   if (isLegalAVL(AVL))
393     return AVL;
394   return getNode(VEISD::LEGALAVL, AVL.getValueType(), AVL);
395 }
396 
397 SDValue VECustomDAG::getUnpack(EVT DestVT, SDValue Vec, PackElem Part,
398                                SDValue AVL) const {
399   assert(getAnnotatedNodeAVL(AVL).second && "Expected a pack-legalized AVL");
400 
401   // TODO: Peek through VEC_PACK and VEC_BROADCAST(REPL_<sth> ..) operands.
402   unsigned OC =
403       (Part == PackElem::Lo) ? VEISD::VEC_UNPACK_LO : VEISD::VEC_UNPACK_HI;
404   return DAG.getNode(OC, DL, DestVT, Vec, AVL);
405 }
406 
407 SDValue VECustomDAG::getPack(EVT DestVT, SDValue LoVec, SDValue HiVec,
408                              SDValue AVL) const {
409   assert(getAnnotatedNodeAVL(AVL).second && "Expected a pack-legalized AVL");
410 
411   // TODO: Peek through VEC_UNPACK_LO|HI operands.
412   return DAG.getNode(VEISD::VEC_PACK, DL, DestVT, LoVec, HiVec, AVL);
413 }
414 
415 VETargetMasks VECustomDAG::getTargetSplitMask(SDValue RawMask, SDValue RawAVL,
416                                               PackElem Part) const {
417   // Adjust AVL for this part
418   SDValue NewAVL;
419   SDValue OneV = getConstant(1, MVT::i32);
420   if (Part == PackElem::Hi)
421     NewAVL = getNode(ISD::ADD, MVT::i32, {RawAVL, OneV});
422   else
423     NewAVL = RawAVL;
424   NewAVL = getNode(ISD::SRL, MVT::i32, {NewAVL, OneV});
425 
426   NewAVL = annotateLegalAVL(NewAVL);
427 
428   // Legalize Mask (unpack or all-true)
429   SDValue NewMask;
430   if (!RawMask)
431     NewMask = getConstantMask(Packing::Normal, true);
432   else
433     NewMask = getUnpack(MVT::v256i1, RawMask, Part, NewAVL);
434 
435   return VETargetMasks(NewMask, NewAVL);
436 }
437 
438 SDValue VECustomDAG::getSplitPtrOffset(SDValue Ptr, SDValue ByteStride,
439                                        PackElem Part) const {
440   // High starts at base ptr but has more significant bits in the 64bit vector
441   // element.
442   if (Part == PackElem::Hi)
443     return Ptr;
444   return getNode(ISD::ADD, MVT::i64, {Ptr, ByteStride});
445 }
446 
447 SDValue VECustomDAG::getSplitPtrStride(SDValue PackStride) const {
448   if (auto ConstBytes = dyn_cast<ConstantSDNode>(PackStride))
449     return getConstant(2 * ConstBytes->getSExtValue(), MVT::i64);
450   return getNode(ISD::SHL, MVT::i64, {PackStride, getConstant(1, MVT::i32)});
451 }
452 
453 } // namespace llvm
454