Lines Matching refs:CDAG
23 VECustomDAG CDAG(DAG, Op); in splitMaskArithmetic() local
25 CDAG.getConstant(Op.getValueType().getVectorNumElements(), MVT::i32); in splitMaskArithmetic()
28 SDValue LoA = CDAG.getUnpack(MVT::v256i1, A, PackElem::Lo, AVL); in splitMaskArithmetic()
29 SDValue HiA = CDAG.getUnpack(MVT::v256i1, A, PackElem::Hi, AVL); in splitMaskArithmetic()
30 SDValue LoB = CDAG.getUnpack(MVT::v256i1, B, PackElem::Lo, AVL); in splitMaskArithmetic()
31 SDValue HiB = CDAG.getUnpack(MVT::v256i1, B, PackElem::Hi, AVL); in splitMaskArithmetic()
33 auto LoRes = CDAG.getNode(Opc, MVT::v256i1, {LoA, LoB}); in splitMaskArithmetic()
34 auto HiRes = CDAG.getNode(Opc, MVT::v256i1, {HiA, HiB}); in splitMaskArithmetic()
35 return CDAG.getPack(MVT::v512i1, LoRes, HiRes, AVL); in splitMaskArithmetic()
48 VECustomDAG CDAG(DAG, Op); in lowerToVVP() local
53 return lowerVVP_LOAD_STORE(Op, CDAG); in lowerToVVP()
56 return lowerVVP_GATHER_SCATTER(Op, CDAG); in lowerToVVP()
78 AVL = CDAG.getConstant(OpVecVT.getVectorNumElements(), MVT::i32); in lowerToVVP()
80 Mask = CDAG.getConstantMask(Packing, true); in lowerToVVP()
84 return CDAG.getNode(VVPOpcode, LegalVecVT, {Op->getOperand(0), Mask, AVL}); in lowerToVVP()
86 return CDAG.getNode(VVPOpcode, LegalVecVT, in lowerToVVP()
92 return CDAG.getLegalReductionOpVVP(VVPOpcode, Op.getValueType(), StartV, in lowerToVVP()
105 return CDAG.getNode(VVPOpcode, LegalVecVT, {X, Y, Z, Mask, AVL}); in lowerToVVP()
111 return CDAG.getNode(VVPOpcode, LegalVecVT, {OnTrue, OnFalse, Mask, AVL}); in lowerToVVP()
118 return CDAG.getNode(VVPOpcode, LegalResVT, {LHS, RHS, Pred, Mask, AVL}); in lowerToVVP()
124 VECustomDAG &CDAG) const { in lowerVVP_LOAD_STORE()
138 SDValue StrideV = getLoadStoreStride(Op, CDAG); in lowerVVP_LOAD_STORE()
145 AVL = CDAG.getConstant(DataVT.getVectorNumElements(), MVT::i32); in lowerVVP_LOAD_STORE()
149 Mask = CDAG.getConstantMask(Packing, true); in lowerVVP_LOAD_STORE()
155 auto NewLoadV = CDAG.getNode(VEISD::VVP_LOAD, {LegalDataVT, MVT::Other}, in lowerVVP_LOAD_STORE()
162 SDValue DataV = CDAG.getNode(VEISD::VVP_SELECT, DataVT, in lowerVVP_LOAD_STORE()
167 return CDAG.getMergeValues({DataV, NewLoadChainV}); in lowerVVP_LOAD_STORE()
172 return CDAG.getNode(VEISD::VVP_STORE, Op.getNode()->getVTList(), in lowerVVP_LOAD_STORE()
177 VECustomDAG &CDAG) const { in splitPackedLoadStore()
194 SDValue PackStride = getLoadStoreStride(Op, CDAG); in splitPackedLoadStore()
204 auto SplitTM = CDAG.getTargetSplitMask(PackedMask, PackedAVL, Part); in splitPackedLoadStore()
219 CDAG.getUnpack(SplitDataVT, PackData, Part, SplitTM.AVL); in splitPackedLoadStore()
227 OpVec.push_back(CDAG.getSplitPtrOffset(PackPtr, PackStride, Part)); in splitPackedLoadStore()
228 OpVec.push_back(CDAG.getSplitPtrStride(PackStride)); in splitPackedLoadStore()
236 PartOps[(int)Part] = CDAG.getNode(VVPOC, MVT::Other, OpVec); in splitPackedLoadStore()
240 CDAG.getNode(VVPOC, {SplitDataVT, MVT::Other}, OpVec); in splitPackedLoadStore()
248 CDAG.getNode(ISD::TokenFactor, MVT::Other, {LowChain, HiChain}); in splitPackedLoadStore()
257 SDValue PackedVals = CDAG.getPack(PackedVT, PartOps[(int)PackElem::Lo], in splitPackedLoadStore()
260 return CDAG.getMergeValues({PackedVals, FusedChains}); in splitPackedLoadStore()
264 VECustomDAG &CDAG) const { in lowerVVP_GATHER_SCATTER()
285 AVL = CDAG.getConstant(DataVT.getVectorNumElements(), MVT::i32); in lowerVVP_GATHER_SCATTER()
289 Mask = CDAG.getConstantMask(Packing, true); in lowerVVP_GATHER_SCATTER()
292 CDAG.getGatherScatterAddress(BasePtr, Scale, Index, Mask, AVL); in lowerVVP_GATHER_SCATTER()
294 return CDAG.getNode(VEISD::VVP_SCATTER, MVT::Other, in lowerVVP_GATHER_SCATTER()
298 SDValue NewLoadV = CDAG.getNode(VEISD::VVP_GATHER, {LegalDataVT, MVT::Other}, in lowerVVP_GATHER_SCATTER()
305 SDValue DataV = CDAG.getNode(VEISD::VVP_SELECT, LegalDataVT, in lowerVVP_GATHER_SCATTER()
308 return CDAG.getMergeValues({DataV, NewLoadChainV}); in lowerVVP_GATHER_SCATTER()
312 VECustomDAG &CDAG) const { in legalizeInternalLoadStoreOp()
318 return splitPackedLoadStore(Op, CDAG); in legalizeInternalLoadStoreOp()
320 return legalizePackedAVL(Op, CDAG); in legalizeInternalLoadStoreOp()
326 VECustomDAG CDAG(DAG, Op); in legalizeInternalVectorOp() local
332 return legalizeInternalLoadStoreOp(Op, CDAG); in legalizeInternalVectorOp()
338 return splitVectorOp(Op, CDAG); in legalizeInternalVectorOp()
341 return legalizePackedAVL(Op, CDAG); in legalizeInternalVectorOp()
344 SDValue VETargetLowering::splitVectorOp(SDValue Op, VECustomDAG &CDAG) const { in splitVectorOp()
362 auto SplitTM = CDAG.getTargetSplitMask(PackedMask, PackedAVL, Part); in splitVectorOp()
379 CDAG.getUnpack(UnpackedOpVT, PackedOperand, Part, SplitTM.AVL); in splitVectorOp()
388 CDAG.getNode(Op.getOpcode(), ResVT, OpVec, Op->getFlags()); in splitVectorOp()
392 return CDAG.getPack(Op.getValueType(), PartOps[(int)PackElem::Lo], in splitVectorOp()
397 VECustomDAG &CDAG) const { in legalizePackedAVL()
416 LegalAVL = CDAG.getConstant((ConstAVL->getZExtValue() + 1) / 2, MVT::i32); in legalizePackedAVL()
418 auto ConstOne = CDAG.getConstant(1, MVT::i32); in legalizePackedAVL()
419 auto PlusOne = CDAG.getNode(ISD::ADD, MVT::i32, {AVL, ConstOne}); in legalizePackedAVL()
420 LegalAVL = CDAG.getNode(ISD::SRL, MVT::i32, {PlusOne, ConstOne}); in legalizePackedAVL()
424 SDValue AnnotatedLegalAVL = CDAG.annotateLegalAVL(LegalAVL); in legalizePackedAVL()
441 CDAG.getNode(Op->getOpcode(), Op->getVTList(), FixedOperands, Flags); in legalizePackedAVL()