1 //===-- VECustomDAG.h - VE Custom DAG Nodes ------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that VE uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "VECustomDAG.h"
15 
16 #ifndef DEBUG_TYPE
17 #define DEBUG_TYPE "vecustomdag"
18 #endif
19 
20 namespace llvm {
21 
22 bool isPackedVectorType(EVT SomeVT) {
23   if (!SomeVT.isVector())
24     return false;
25   return SomeVT.getVectorNumElements() > StandardVectorWidth;
26 }
27 
28 MVT splitVectorType(MVT VT) {
29   if (!VT.isVector())
30     return VT;
31   return MVT::getVectorVT(VT.getVectorElementType(), StandardVectorWidth);
32 }
33 
34 MVT getLegalVectorType(Packing P, MVT ElemVT) {
35   return MVT::getVectorVT(ElemVT, P == Packing::Normal ? StandardVectorWidth
36                                                        : PackedVectorWidth);
37 }
38 
39 Packing getTypePacking(EVT VT) {
40   assert(VT.isVector());
41   return isPackedVectorType(VT) ? Packing::Dense : Packing::Normal;
42 }
43 
44 bool isMaskType(EVT SomeVT) {
45   if (!SomeVT.isVector())
46     return false;
47   return SomeVT.getVectorElementType() == MVT::i1;
48 }
49 
50 bool isMaskArithmetic(SDValue Op) {
51   switch (Op.getOpcode()) {
52   default:
53     return false;
54   case ISD::AND:
55   case ISD::XOR:
56   case ISD::OR:
57     return isMaskType(Op.getValueType());
58   }
59 }
60 
61 /// \returns the VVP_* SDNode opcode corresponsing to \p OC.
62 Optional<unsigned> getVVPOpcode(unsigned Opcode) {
63   switch (Opcode) {
64   case ISD::MLOAD:
65     return VEISD::VVP_LOAD;
66   case ISD::MSTORE:
67     return VEISD::VVP_STORE;
68 #define HANDLE_VP_TO_VVP(VPOPC, VVPNAME)                                       \
69   case ISD::VPOPC:                                                             \
70     return VEISD::VVPNAME;
71 #define ADD_VVP_OP(VVPNAME, SDNAME)                                            \
72   case VEISD::VVPNAME:                                                         \
73   case ISD::SDNAME:                                                            \
74     return VEISD::VVPNAME;
75 #include "VVPNodes.def"
76   // TODO: Map those in VVPNodes.def too
77   case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
78     return VEISD::VVP_LOAD;
79   case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
80     return VEISD::VVP_STORE;
81   }
82   return None;
83 }
84 
85 bool maySafelyIgnoreMask(SDValue Op) {
86   auto VVPOpc = getVVPOpcode(Op->getOpcode());
87   auto Opc = VVPOpc.getValueOr(Op->getOpcode());
88 
89   switch (Opc) {
90   case VEISD::VVP_SDIV:
91   case VEISD::VVP_UDIV:
92   case VEISD::VVP_FDIV:
93   case VEISD::VVP_SELECT:
94     return false;
95 
96   default:
97     return true;
98   }
99 }
100 
101 bool supportsPackedMode(unsigned Opcode, EVT IdiomVT) {
102   bool IsPackedOp = isPackedVectorType(IdiomVT);
103   bool IsMaskOp = isMaskType(IdiomVT);
104   switch (Opcode) {
105   default:
106     return false;
107 
108   case VEISD::VEC_BROADCAST:
109     return true;
110 #define REGISTER_PACKED(VVP_NAME) case VEISD::VVP_NAME:
111 #include "VVPNodes.def"
112     return IsPackedOp && !IsMaskOp;
113   }
114 }
115 
116 bool isPackingSupportOpcode(unsigned Opc) {
117   switch (Opc) {
118   case VEISD::VEC_PACK:
119   case VEISD::VEC_UNPACK_LO:
120   case VEISD::VEC_UNPACK_HI:
121     return true;
122   }
123   return false;
124 }
125 
126 bool isVVPOrVEC(unsigned Opcode) {
127   switch (Opcode) {
128   case VEISD::VEC_BROADCAST:
129 #define ADD_VVP_OP(VVPNAME, ...) case VEISD::VVPNAME:
130 #include "VVPNodes.def"
131     return true;
132   }
133   return false;
134 }
135 
136 bool isVVPBinaryOp(unsigned VVPOpcode) {
137   switch (VVPOpcode) {
138 #define ADD_BINARY_VVP_OP(VVPNAME, ...)                                        \
139   case VEISD::VVPNAME:                                                         \
140     return true;
141 #include "VVPNodes.def"
142   }
143   return false;
144 }
145 
146 bool isVVPReductionOp(unsigned Opcode) {
147   switch (Opcode) {
148 #define ADD_REDUCE_VVP_OP(VVP_NAME, SDNAME) case VEISD::VVP_NAME:
149 #include "VVPNodes.def"
150     return true;
151   }
152   return false;
153 }
154 
155 // Return the AVL operand position for this VVP or VEC Op.
156 Optional<int> getAVLPos(unsigned Opc) {
157   // This is only available for VP SDNodes
158   auto PosOpt = ISD::getVPExplicitVectorLengthIdx(Opc);
159   if (PosOpt)
160     return *PosOpt;
161 
162   // VVP Opcodes.
163   if (isVVPBinaryOp(Opc))
164     return 3;
165 
166   // VM Opcodes.
167   switch (Opc) {
168   case VEISD::VEC_BROADCAST:
169     return 1;
170   case VEISD::VVP_SELECT:
171     return 3;
172   case VEISD::VVP_LOAD:
173     return 4;
174   case VEISD::VVP_STORE:
175     return 5;
176   }
177 
178   return None;
179 }
180 
181 Optional<int> getMaskPos(unsigned Opc) {
182   // This is only available for VP SDNodes
183   auto PosOpt = ISD::getVPMaskIdx(Opc);
184   if (PosOpt)
185     return *PosOpt;
186 
187   // VVP Opcodes.
188   if (isVVPBinaryOp(Opc))
189     return 2;
190 
191   // Other opcodes.
192   switch (Opc) {
193   case ISD::MSTORE:
194     return 4;
195   case ISD::MLOAD:
196     return 3;
197   case VEISD::VVP_SELECT:
198     return 2;
199   }
200 
201   return None;
202 }
203 
204 bool isLegalAVL(SDValue AVL) { return AVL->getOpcode() == VEISD::LEGALAVL; }
205 
206 /// Node Properties {
207 
208 SDValue getNodeChain(SDValue Op) {
209   if (MemSDNode *MemN = dyn_cast<MemSDNode>(Op.getNode()))
210     return MemN->getChain();
211 
212   switch (Op->getOpcode()) {
213   case VEISD::VVP_LOAD:
214   case VEISD::VVP_STORE:
215     return Op->getOperand(0);
216   }
217   return SDValue();
218 }
219 
220 SDValue getMemoryPtr(SDValue Op) {
221   if (auto *MemN = dyn_cast<MemSDNode>(Op.getNode()))
222     return MemN->getBasePtr();
223 
224   switch (Op->getOpcode()) {
225   case VEISD::VVP_LOAD:
226     return Op->getOperand(1);
227   case VEISD::VVP_STORE:
228     return Op->getOperand(2);
229   }
230   return SDValue();
231 }
232 
233 Optional<EVT> getIdiomaticVectorType(SDNode *Op) {
234   unsigned OC = Op->getOpcode();
235 
236   // For memory ops -> the transfered data type
237   if (auto MemN = dyn_cast<MemSDNode>(Op))
238     return MemN->getMemoryVT();
239 
240   switch (OC) {
241   // Standard ISD.
242   case ISD::SELECT: // not aliased with VVP_SELECT
243   case ISD::CONCAT_VECTORS:
244   case ISD::EXTRACT_SUBVECTOR:
245   case ISD::VECTOR_SHUFFLE:
246   case ISD::BUILD_VECTOR:
247   case ISD::SCALAR_TO_VECTOR:
248     return Op->getValueType(0);
249   }
250 
251   // Translate to VVP where possible.
252   unsigned OriginalOC = OC;
253   if (auto VVPOpc = getVVPOpcode(OC))
254     OC = *VVPOpc;
255 
256   if (isVVPReductionOp(OC))
257     return Op->getOperand(hasReductionStartParam(OriginalOC) ? 1 : 0)
258         .getValueType();
259 
260   switch (OC) {
261   default:
262   case VEISD::VVP_SETCC:
263     return Op->getOperand(0).getValueType();
264 
265   case VEISD::VVP_SELECT:
266 #define ADD_BINARY_VVP_OP(VVP_NAME, ...) case VEISD::VVP_NAME:
267 #include "VVPNodes.def"
268     return Op->getValueType(0);
269 
270   case VEISD::VVP_LOAD:
271     return Op->getValueType(0);
272 
273   case VEISD::VVP_STORE:
274     return Op->getOperand(1)->getValueType(0);
275 
276   // VEC
277   case VEISD::VEC_BROADCAST:
278     return Op->getValueType(0);
279   }
280 }
281 
282 SDValue getLoadStoreStride(SDValue Op, VECustomDAG &CDAG) {
283   switch (Op->getOpcode()) {
284   case VEISD::VVP_STORE:
285     return Op->getOperand(3);
286   case VEISD::VVP_LOAD:
287     return Op->getOperand(2);
288   }
289 
290   if (auto *StoreN = dyn_cast<VPStridedStoreSDNode>(Op.getNode()))
291     return StoreN->getStride();
292   if (auto *StoreN = dyn_cast<VPStridedLoadSDNode>(Op.getNode()))
293     return StoreN->getStride();
294 
295   if (isa<MemSDNode>(Op.getNode())) {
296     // Regular MLOAD/MSTORE/LOAD/STORE
297     // No stride argument -> use the contiguous element size as stride.
298     uint64_t ElemStride = getIdiomaticVectorType(Op.getNode())
299                               ->getVectorElementType()
300                               .getStoreSize();
301     return CDAG.getConstant(ElemStride, MVT::i64);
302   }
303   return SDValue();
304 }
305 
306 SDValue getGatherScatterIndex(SDValue Op) {
307   if (auto *N = dyn_cast<MaskedGatherScatterSDNode>(Op.getNode()))
308     return N->getIndex();
309   if (auto *N = dyn_cast<VPGatherScatterSDNode>(Op.getNode()))
310     return N->getIndex();
311   return SDValue();
312 }
313 
314 SDValue getGatherScatterScale(SDValue Op) {
315   if (auto *N = dyn_cast<MaskedGatherScatterSDNode>(Op.getNode()))
316     return N->getScale();
317   if (auto *N = dyn_cast<VPGatherScatterSDNode>(Op.getNode()))
318     return N->getScale();
319   return SDValue();
320 }
321 
322 SDValue getStoredValue(SDValue Op) {
323   switch (Op->getOpcode()) {
324   case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
325   case VEISD::VVP_STORE:
326     return Op->getOperand(1);
327   }
328   if (auto *StoreN = dyn_cast<StoreSDNode>(Op.getNode()))
329     return StoreN->getValue();
330   if (auto *StoreN = dyn_cast<MaskedStoreSDNode>(Op.getNode()))
331     return StoreN->getValue();
332   if (auto *StoreN = dyn_cast<VPStridedStoreSDNode>(Op.getNode()))
333     return StoreN->getValue();
334   if (auto *StoreN = dyn_cast<VPStoreSDNode>(Op.getNode()))
335     return StoreN->getValue();
336   if (auto *StoreN = dyn_cast<MaskedScatterSDNode>(Op.getNode()))
337     return StoreN->getValue();
338   if (auto *StoreN = dyn_cast<VPScatterSDNode>(Op.getNode()))
339     return StoreN->getValue();
340   return SDValue();
341 }
342 
343 SDValue getNodePassthru(SDValue Op) {
344   if (auto *N = dyn_cast<MaskedLoadSDNode>(Op.getNode()))
345     return N->getPassThru();
346   if (auto *N = dyn_cast<MaskedGatherSDNode>(Op.getNode()))
347     return N->getPassThru();
348 
349   return SDValue();
350 }
351 
352 bool hasReductionStartParam(unsigned OPC) {
353   // TODO: Ordered reduction opcodes.
354   if (ISD::isVPReduction(OPC))
355     return true;
356   return false;
357 }
358 
359 unsigned getScalarReductionOpcode(unsigned VVPOC, bool IsMask) {
360   assert(!IsMask && "Mask reduction isel");
361 
362   switch (VVPOC) {
363 #define HANDLE_VVP_REDUCE_TO_SCALAR(VVP_RED_ISD, REDUCE_ISD)                   \
364   case VEISD::VVP_RED_ISD:                                                     \
365     return ISD::REDUCE_ISD;
366 #include "VVPNodes.def"
367   default:
368     break;
369   }
370   llvm_unreachable("Cannot not scalarize this reduction Opcode!");
371 }
372 
373 /// } Node Properties
374 
375 SDValue getNodeAVL(SDValue Op) {
376   auto PosOpt = getAVLPos(Op->getOpcode());
377   return PosOpt ? Op->getOperand(*PosOpt) : SDValue();
378 }
379 
380 SDValue getNodeMask(SDValue Op) {
381   auto PosOpt = getMaskPos(Op->getOpcode());
382   return PosOpt ? Op->getOperand(*PosOpt) : SDValue();
383 }
384 
385 std::pair<SDValue, bool> getAnnotatedNodeAVL(SDValue Op) {
386   SDValue AVL = getNodeAVL(Op);
387   if (!AVL)
388     return {SDValue(), true};
389   if (isLegalAVL(AVL))
390     return {AVL->getOperand(0), true};
391   return {AVL, false};
392 }
393 
394 SDValue VECustomDAG::getConstant(uint64_t Val, EVT VT, bool IsTarget,
395                                  bool IsOpaque) const {
396   return DAG.getConstant(Val, DL, VT, IsTarget, IsOpaque);
397 }
398 
399 SDValue VECustomDAG::getConstantMask(Packing Packing, bool AllTrue) const {
400   auto MaskVT = getLegalVectorType(Packing, MVT::i1);
401 
402   // VEISelDAGtoDAG will replace this pattern with the constant-true VM.
403   auto TrueVal = DAG.getConstant(-1, DL, MVT::i32);
404   auto AVL = getConstant(MaskVT.getVectorNumElements(), MVT::i32);
405   auto Res = getNode(VEISD::VEC_BROADCAST, MaskVT, {TrueVal, AVL});
406   if (AllTrue)
407     return Res;
408 
409   return DAG.getNOT(DL, Res, Res.getValueType());
410 }
411 
412 SDValue VECustomDAG::getMaskBroadcast(EVT ResultVT, SDValue Scalar,
413                                       SDValue AVL) const {
414   // Constant mask splat.
415   if (auto BcConst = dyn_cast<ConstantSDNode>(Scalar))
416     return getConstantMask(getTypePacking(ResultVT),
417                            BcConst->getSExtValue() != 0);
418 
419   // Expand the broadcast to a vector comparison.
420   auto ScalarBoolVT = Scalar.getSimpleValueType();
421   assert(ScalarBoolVT == MVT::i32);
422 
423   // Cast to i32 ty.
424   SDValue CmpElem = DAG.getSExtOrTrunc(Scalar, DL, MVT::i32);
425   unsigned ElemCount = ResultVT.getVectorNumElements();
426   MVT CmpVecTy = MVT::getVectorVT(ScalarBoolVT, ElemCount);
427 
428   // Broadcast to vector.
429   SDValue BCVec =
430       DAG.getNode(VEISD::VEC_BROADCAST, DL, CmpVecTy, {CmpElem, AVL});
431   SDValue ZeroVec =
432       getBroadcast(CmpVecTy, {DAG.getConstant(0, DL, ScalarBoolVT)}, AVL);
433 
434   MVT BoolVecTy = MVT::getVectorVT(MVT::i1, ElemCount);
435 
436   // Broadcast(Data) != Broadcast(0)
437   // TODO: Use a VVP operation for this.
438   return DAG.getSetCC(DL, BoolVecTy, BCVec, ZeroVec, ISD::CondCode::SETNE);
439 }
440 
441 SDValue VECustomDAG::getBroadcast(EVT ResultVT, SDValue Scalar,
442                                   SDValue AVL) const {
443   assert(ResultVT.isVector());
444   auto ScaVT = Scalar.getValueType();
445 
446   if (isMaskType(ResultVT))
447     return getMaskBroadcast(ResultVT, Scalar, AVL);
448 
449   if (isPackedVectorType(ResultVT)) {
450     // v512x packed mode broadcast
451     // Replicate the scalar reg (f32 or i32) onto the opposing half of the full
452     // scalar register. If it's an I64 type, assume that this has already
453     // happened.
454     if (ScaVT == MVT::f32) {
455       Scalar = getNode(VEISD::REPL_F32, MVT::i64, Scalar);
456     } else if (ScaVT == MVT::i32) {
457       Scalar = getNode(VEISD::REPL_I32, MVT::i64, Scalar);
458     }
459   }
460 
461   return getNode(VEISD::VEC_BROADCAST, ResultVT, {Scalar, AVL});
462 }
463 
464 SDValue VECustomDAG::annotateLegalAVL(SDValue AVL) const {
465   if (isLegalAVL(AVL))
466     return AVL;
467   return getNode(VEISD::LEGALAVL, AVL.getValueType(), AVL);
468 }
469 
470 SDValue VECustomDAG::getUnpack(EVT DestVT, SDValue Vec, PackElem Part,
471                                SDValue AVL) const {
472   assert(getAnnotatedNodeAVL(AVL).second && "Expected a pack-legalized AVL");
473 
474   // TODO: Peek through VEC_PACK and VEC_BROADCAST(REPL_<sth> ..) operands.
475   unsigned OC =
476       (Part == PackElem::Lo) ? VEISD::VEC_UNPACK_LO : VEISD::VEC_UNPACK_HI;
477   return DAG.getNode(OC, DL, DestVT, Vec, AVL);
478 }
479 
480 SDValue VECustomDAG::getPack(EVT DestVT, SDValue LoVec, SDValue HiVec,
481                              SDValue AVL) const {
482   assert(getAnnotatedNodeAVL(AVL).second && "Expected a pack-legalized AVL");
483 
484   // TODO: Peek through VEC_UNPACK_LO|HI operands.
485   return DAG.getNode(VEISD::VEC_PACK, DL, DestVT, LoVec, HiVec, AVL);
486 }
487 
488 VETargetMasks VECustomDAG::getTargetSplitMask(SDValue RawMask, SDValue RawAVL,
489                                               PackElem Part) const {
490   // Adjust AVL for this part
491   SDValue NewAVL;
492   SDValue OneV = getConstant(1, MVT::i32);
493   if (Part == PackElem::Hi)
494     NewAVL = getNode(ISD::ADD, MVT::i32, {RawAVL, OneV});
495   else
496     NewAVL = RawAVL;
497   NewAVL = getNode(ISD::SRL, MVT::i32, {NewAVL, OneV});
498 
499   NewAVL = annotateLegalAVL(NewAVL);
500 
501   // Legalize Mask (unpack or all-true)
502   SDValue NewMask;
503   if (!RawMask)
504     NewMask = getConstantMask(Packing::Normal, true);
505   else
506     NewMask = getUnpack(MVT::v256i1, RawMask, Part, NewAVL);
507 
508   return VETargetMasks(NewMask, NewAVL);
509 }
510 
511 SDValue VECustomDAG::getSplitPtrOffset(SDValue Ptr, SDValue ByteStride,
512                                        PackElem Part) const {
513   // High starts at base ptr but has more significant bits in the 64bit vector
514   // element.
515   if (Part == PackElem::Hi)
516     return Ptr;
517   return getNode(ISD::ADD, MVT::i64, {Ptr, ByteStride});
518 }
519 
520 SDValue VECustomDAG::getSplitPtrStride(SDValue PackStride) const {
521   if (auto ConstBytes = dyn_cast<ConstantSDNode>(PackStride))
522     return getConstant(2 * ConstBytes->getSExtValue(), MVT::i64);
523   return getNode(ISD::SHL, MVT::i64, {PackStride, getConstant(1, MVT::i32)});
524 }
525 
526 SDValue VECustomDAG::getGatherScatterAddress(SDValue BasePtr, SDValue Scale,
527                                              SDValue Index, SDValue Mask,
528                                              SDValue AVL) const {
529   EVT IndexVT = Index.getValueType();
530 
531   // Apply scale.
532   SDValue ScaledIndex;
533   if (!Scale || isOneConstant(Scale))
534     ScaledIndex = Index;
535   else {
536     SDValue ScaleBroadcast = getBroadcast(IndexVT, Scale, AVL);
537     ScaledIndex =
538         getNode(VEISD::VVP_MUL, IndexVT, {Index, ScaleBroadcast, Mask, AVL});
539   }
540 
541   // Add basePtr.
542   if (isNullConstant(BasePtr))
543     return ScaledIndex;
544 
545   // re-constitute pointer vector (basePtr + index * scale)
546   SDValue BaseBroadcast = getBroadcast(IndexVT, BasePtr, AVL);
547   auto ResPtr =
548       getNode(VEISD::VVP_ADD, IndexVT, {BaseBroadcast, ScaledIndex, Mask, AVL});
549   return ResPtr;
550 }
551 
552 SDValue VECustomDAG::getLegalReductionOpVVP(unsigned VVPOpcode, EVT ResVT,
553                                             SDValue StartV, SDValue VectorV,
554                                             SDValue Mask, SDValue AVL,
555                                             SDNodeFlags Flags) const {
556 
557   // Optionally attach the start param with a scalar op (where it is
558   // unsupported).
559   bool scalarizeStartParam = StartV && !hasReductionStartParam(VVPOpcode);
560   bool IsMaskReduction = isMaskType(VectorV.getValueType());
561   assert(!IsMaskReduction && "TODO Implement");
562   auto AttachStartValue = [&](SDValue ReductionResV) {
563     if (!scalarizeStartParam)
564       return ReductionResV;
565     auto ScalarOC = getScalarReductionOpcode(VVPOpcode, IsMaskReduction);
566     return getNode(ScalarOC, ResVT, {StartV, ReductionResV});
567   };
568 
569   // Fixup: Always Use sequential 'fmul' reduction.
570   if (!scalarizeStartParam && StartV) {
571     assert(hasReductionStartParam(VVPOpcode));
572     return AttachStartValue(
573         getNode(VVPOpcode, ResVT, {StartV, VectorV, Mask, AVL}, Flags));
574   } else
575     return AttachStartValue(
576         getNode(VVPOpcode, ResVT, {VectorV, Mask, AVL}, Flags));
577 }
578 
579 } // namespace llvm
580