1 //===-- VECustomDAG.h - VE Custom DAG Nodes ------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that VE uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "VECustomDAG.h"
15 
16 #ifndef DEBUG_TYPE
17 #define DEBUG_TYPE "vecustomdag"
18 #endif
19 
20 namespace llvm {
21 
22 bool isPackedVectorType(EVT SomeVT) {
23   if (!SomeVT.isVector())
24     return false;
25   return SomeVT.getVectorNumElements() > StandardVectorWidth;
26 }
27 
28 MVT splitVectorType(MVT VT) {
29   if (!VT.isVector())
30     return VT;
31   return MVT::getVectorVT(VT.getVectorElementType(), StandardVectorWidth);
32 }
33 
34 MVT getLegalVectorType(Packing P, MVT ElemVT) {
35   return MVT::getVectorVT(ElemVT, P == Packing::Normal ? StandardVectorWidth
36                                                        : PackedVectorWidth);
37 }
38 
39 Packing getTypePacking(EVT VT) {
40   assert(VT.isVector());
41   return isPackedVectorType(VT) ? Packing::Dense : Packing::Normal;
42 }
43 
44 bool isMaskType(EVT SomeVT) {
45   if (!SomeVT.isVector())
46     return false;
47   return SomeVT.getVectorElementType() == MVT::i1;
48 }
49 
50 bool isMaskArithmetic(SDValue Op) {
51   switch (Op.getOpcode()) {
52   default:
53     return false;
54   case ISD::AND:
55   case ISD::XOR:
56   case ISD::OR:
57     return isMaskType(Op.getValueType());
58   }
59 }
60 
61 /// \returns the VVP_* SDNode opcode corresponsing to \p OC.
62 Optional<unsigned> getVVPOpcode(unsigned Opcode) {
63   switch (Opcode) {
64 #define HANDLE_VP_TO_VVP(VPOPC, VVPNAME)                                       \
65   case ISD::VPOPC:                                                             \
66     return VEISD::VVPNAME;
67 #define ADD_VVP_OP(VVPNAME, SDNAME)                                            \
68   case VEISD::VVPNAME:                                                         \
69   case ISD::SDNAME:                                                            \
70     return VEISD::VVPNAME;
71 #include "VVPNodes.def"
72   }
73   return None;
74 }
75 
76 bool maySafelyIgnoreMask(SDValue Op) {
77   auto VVPOpc = getVVPOpcode(Op->getOpcode());
78   auto Opc = VVPOpc.getValueOr(Op->getOpcode());
79 
80   switch (Opc) {
81   case VEISD::VVP_SDIV:
82   case VEISD::VVP_UDIV:
83   case VEISD::VVP_FDIV:
84   case VEISD::VVP_SELECT:
85     return false;
86 
87   default:
88     return true;
89   }
90 }
91 
92 bool supportsPackedMode(unsigned Opcode, EVT IdiomVT) {
93   bool IsPackedOp = isPackedVectorType(IdiomVT);
94   bool IsMaskOp = isMaskType(IdiomVT);
95   switch (Opcode) {
96   default:
97     return false;
98 
99   case VEISD::VEC_BROADCAST:
100     return true;
101 #define REGISTER_PACKED(VVP_NAME) case VEISD::VVP_NAME:
102 #include "VVPNodes.def"
103     return IsPackedOp && !IsMaskOp;
104   }
105 }
106 
107 bool isPackingSupportOpcode(unsigned Opc) {
108   switch (Opc) {
109   case VEISD::VEC_PACK:
110   case VEISD::VEC_UNPACK_LO:
111   case VEISD::VEC_UNPACK_HI:
112     return true;
113   }
114   return false;
115 }
116 
117 bool isVVPOrVEC(unsigned Opcode) {
118   switch (Opcode) {
119   case VEISD::VEC_BROADCAST:
120 #define ADD_VVP_OP(VVPNAME, ...) case VEISD::VVPNAME:
121 #include "VVPNodes.def"
122     return true;
123   }
124   return false;
125 }
126 
127 bool isVVPBinaryOp(unsigned VVPOpcode) {
128   switch (VVPOpcode) {
129 #define ADD_BINARY_VVP_OP(VVPNAME, ...)                                        \
130   case VEISD::VVPNAME:                                                         \
131     return true;
132 #include "VVPNodes.def"
133   }
134   return false;
135 }
136 
137 // Return the AVL operand position for this VVP or VEC Op.
138 Optional<int> getAVLPos(unsigned Opc) {
139   // This is only available for VP SDNodes
140   auto PosOpt = ISD::getVPExplicitVectorLengthIdx(Opc);
141   if (PosOpt)
142     return *PosOpt;
143 
144   // VVP Opcodes.
145   if (isVVPBinaryOp(Opc))
146     return 3;
147 
148   // VM Opcodes.
149   switch (Opc) {
150   case VEISD::VEC_BROADCAST:
151     return 1;
152   case VEISD::VVP_SELECT:
153     return 3;
154   }
155 
156   return None;
157 }
158 
159 Optional<int> getMaskPos(unsigned Opc) {
160   // This is only available for VP SDNodes
161   auto PosOpt = ISD::getVPMaskIdx(Opc);
162   if (PosOpt)
163     return *PosOpt;
164 
165   // VVP Opcodes.
166   if (isVVPBinaryOp(Opc))
167     return 2;
168 
169   // VM Opcodes.
170   switch (Opc) {
171   case VEISD::VVP_SELECT:
172     return 2;
173   }
174 
175   return None;
176 }
177 
178 bool isLegalAVL(SDValue AVL) { return AVL->getOpcode() == VEISD::LEGALAVL; }
179 
180 SDValue getNodeAVL(SDValue Op) {
181   auto PosOpt = getAVLPos(Op->getOpcode());
182   return PosOpt ? Op->getOperand(*PosOpt) : SDValue();
183 }
184 
185 SDValue getNodeMask(SDValue Op) {
186   auto PosOpt = getMaskPos(Op->getOpcode());
187   return PosOpt ? Op->getOperand(*PosOpt) : SDValue();
188 }
189 
190 std::pair<SDValue, bool> getAnnotatedNodeAVL(SDValue Op) {
191   SDValue AVL = getNodeAVL(Op);
192   if (!AVL)
193     return {SDValue(), true};
194   if (isLegalAVL(AVL))
195     return {AVL->getOperand(0), true};
196   return {AVL, false};
197 }
198 
199 SDValue VECustomDAG::getConstant(uint64_t Val, EVT VT, bool IsTarget,
200                                  bool IsOpaque) const {
201   return DAG.getConstant(Val, DL, VT, IsTarget, IsOpaque);
202 }
203 
204 SDValue VECustomDAG::getConstantMask(Packing Packing, bool AllTrue) const {
205   auto MaskVT = getLegalVectorType(Packing, MVT::i1);
206 
207   // VEISelDAGtoDAG will replace this pattern with the constant-true VM.
208   auto TrueVal = DAG.getConstant(-1, DL, MVT::i32);
209   auto AVL = getConstant(MaskVT.getVectorNumElements(), MVT::i32);
210   auto Res = getNode(VEISD::VEC_BROADCAST, MaskVT, {TrueVal, AVL});
211   if (AllTrue)
212     return Res;
213 
214   return DAG.getNOT(DL, Res, Res.getValueType());
215 }
216 
217 SDValue VECustomDAG::getMaskBroadcast(EVT ResultVT, SDValue Scalar,
218                                       SDValue AVL) const {
219   // Constant mask splat.
220   if (auto BcConst = dyn_cast<ConstantSDNode>(Scalar))
221     return getConstantMask(getTypePacking(ResultVT),
222                            BcConst->getSExtValue() != 0);
223 
224   // Expand the broadcast to a vector comparison.
225   auto ScalarBoolVT = Scalar.getSimpleValueType();
226   assert(ScalarBoolVT == MVT::i32);
227 
228   // Cast to i32 ty.
229   SDValue CmpElem = DAG.getSExtOrTrunc(Scalar, DL, MVT::i32);
230   unsigned ElemCount = ResultVT.getVectorNumElements();
231   MVT CmpVecTy = MVT::getVectorVT(ScalarBoolVT, ElemCount);
232 
233   // Broadcast to vector.
234   SDValue BCVec =
235       DAG.getNode(VEISD::VEC_BROADCAST, DL, CmpVecTy, {CmpElem, AVL});
236   SDValue ZeroVec =
237       getBroadcast(CmpVecTy, {DAG.getConstant(0, DL, ScalarBoolVT)}, AVL);
238 
239   MVT BoolVecTy = MVT::getVectorVT(MVT::i1, ElemCount);
240 
241   // Broadcast(Data) != Broadcast(0)
242   // TODO: Use a VVP operation for this.
243   return DAG.getSetCC(DL, BoolVecTy, BCVec, ZeroVec, ISD::CondCode::SETNE);
244 }
245 
246 SDValue VECustomDAG::getBroadcast(EVT ResultVT, SDValue Scalar,
247                                   SDValue AVL) const {
248   assert(ResultVT.isVector());
249   auto ScaVT = Scalar.getValueType();
250 
251   if (isMaskType(ResultVT))
252     return getMaskBroadcast(ResultVT, Scalar, AVL);
253 
254   if (isPackedVectorType(ResultVT)) {
255     // v512x packed mode broadcast
256     // Replicate the scalar reg (f32 or i32) onto the opposing half of the full
257     // scalar register. If it's an I64 type, assume that this has already
258     // happened.
259     if (ScaVT == MVT::f32) {
260       Scalar = getNode(VEISD::REPL_F32, MVT::i64, Scalar);
261     } else if (ScaVT == MVT::i32) {
262       Scalar = getNode(VEISD::REPL_I32, MVT::i64, Scalar);
263     }
264   }
265 
266   return getNode(VEISD::VEC_BROADCAST, ResultVT, {Scalar, AVL});
267 }
268 
269 SDValue VECustomDAG::annotateLegalAVL(SDValue AVL) const {
270   if (isLegalAVL(AVL))
271     return AVL;
272   return getNode(VEISD::LEGALAVL, AVL.getValueType(), AVL);
273 }
274 
275 SDValue VECustomDAG::getUnpack(EVT DestVT, SDValue Vec, PackElem Part,
276                                SDValue AVL) const {
277   assert(getAnnotatedNodeAVL(AVL).second && "Expected a pack-legalized AVL");
278 
279   // TODO: Peek through VEC_PACK and VEC_BROADCAST(REPL_<sth> ..) operands.
280   unsigned OC =
281       (Part == PackElem::Lo) ? VEISD::VEC_UNPACK_LO : VEISD::VEC_UNPACK_HI;
282   return DAG.getNode(OC, DL, DestVT, Vec, AVL);
283 }
284 
285 SDValue VECustomDAG::getPack(EVT DestVT, SDValue LoVec, SDValue HiVec,
286                              SDValue AVL) const {
287   assert(getAnnotatedNodeAVL(AVL).second && "Expected a pack-legalized AVL");
288 
289   // TODO: Peek through VEC_UNPACK_LO|HI operands.
290   return DAG.getNode(VEISD::VEC_PACK, DL, DestVT, LoVec, HiVec, AVL);
291 }
292 
293 VETargetMasks VECustomDAG::getTargetSplitMask(SDValue RawMask, SDValue RawAVL,
294                                               PackElem Part) const {
295   // Adjust AVL for this part
296   SDValue NewAVL;
297   SDValue OneV = getConstant(1, MVT::i32);
298   if (Part == PackElem::Hi)
299     NewAVL = getNode(ISD::ADD, MVT::i32, {RawAVL, OneV});
300   else
301     NewAVL = RawAVL;
302   NewAVL = getNode(ISD::SRL, MVT::i32, {NewAVL, OneV});
303 
304   NewAVL = annotateLegalAVL(NewAVL);
305 
306   // Legalize Mask (unpack or all-true)
307   SDValue NewMask;
308   if (!RawMask)
309     NewMask = getConstantMask(Packing::Normal, true);
310   else
311     NewMask = getUnpack(MVT::v256i1, RawMask, Part, NewAVL);
312 
313   return VETargetMasks(NewMask, NewAVL);
314 }
315 
316 } // namespace llvm
317