Lines Matching refs:Node
88 SDValue UnrollVSETCC(SDNode *Node);
94 void Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results);
98 void ExpandFP_TO_UINT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
102 void ExpandUINT_TO_FLOAT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
105 SDValue ExpandSEXTINREG(SDNode *Node);
112 SDValue ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node);
119 SDValue ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node);
125 SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node);
128 SDValue ExpandBSWAP(SDNode *Node);
132 SDValue ExpandVSELECT(SDNode *Node);
133 SDValue ExpandVP_SELECT(SDNode *Node);
134 SDValue ExpandVP_MERGE(SDNode *Node);
135 SDValue ExpandVP_REM(SDNode *Node);
136 SDValue ExpandSELECT(SDNode *Node);
139 SDValue ExpandFNEG(SDNode *Node);
140 void ExpandFSUB(SDNode *Node, SmallVectorImpl<SDValue> &Results);
141 void ExpandSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
142 void ExpandBITREVERSE(SDNode *Node, SmallVectorImpl<SDValue> &Results);
143 void ExpandUADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
144 void ExpandSADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
145 void ExpandMULO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
146 void ExpandFixedPointDiv(SDNode *Node, SmallVectorImpl<SDValue> &Results);
147 void ExpandStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
148 void ExpandREM(SDNode *Node, SmallVectorImpl<SDValue> &Results);
150 void UnrollStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
156 void Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results);
161 void PromoteINT_TO_FP(SDNode *Node, SmallVectorImpl<SDValue> &Results);
167 void PromoteFP_TO_INT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
174 void PromoteReduction(SDNode *Node, SmallVectorImpl<SDValue> &Results);
180 void PromoteSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
182 void PromoteSTRICT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
270 SDNode *Node = DAG.UpdateNodeOperands(Op.getNode(), Ops); in LegalizeOp() local
273 llvm::any_of(Node->values(), [](EVT T) { return T.isVector(); }) || in LegalizeOp()
274 llvm::any_of(Node->op_values(), in LegalizeOp()
277 return TranslateLegalizeResults(Op, Node); in LegalizeOp()
283 return TranslateLegalizeResults(Op, Node); in LegalizeOp()
285 LoadSDNode *LD = cast<LoadSDNode>(Node); in LegalizeOp()
293 StoreSDNode *ST = cast<StoreSDNode>(Node); in LegalizeOp()
301 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
310 ValVT = Node->getValueType(0); in LegalizeOp()
313 ValVT = Node->getOperand(1).getValueType(); in LegalizeOp()
316 MVT OpVT = Node->getOperand(1).getSimpleValueType(); in LegalizeOp()
317 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(3))->get(); in LegalizeOp()
320 Action = TLI.getOperationAction(Node->getOpcode(), OpVT); in LegalizeOp()
322 Action = TLI.getOperationAction(Node->getOpcode(), ValVT); in LegalizeOp()
330 TLI.getStrictFPOperationAction(Node->getOpcode(), ValVT) == in LegalizeOp()
333 if (TLI.getOperationAction(Node->getOpcode(), EltVT) in LegalizeOp()
335 TLI.getStrictFPOperationAction(Node->getOpcode(), EltVT) in LegalizeOp()
443 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
453 unsigned Scale = Node->getConstantOperandVal(2); in LegalizeOp()
454 Action = TLI.getFixedPointOperationAction(Node->getOpcode(), in LegalizeOp()
455 Node->getValueType(0), Scale); in LegalizeOp()
475 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
476 Node->getOperand(0).getValueType()); in LegalizeOp()
480 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
481 Node->getOperand(1).getValueType()); in LegalizeOp()
484 MVT OpVT = Node->getOperand(0).getSimpleValueType(); in LegalizeOp()
485 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); in LegalizeOp()
488 Action = TLI.getOperationAction(Node->getOpcode(), OpVT); in LegalizeOp()
494 EVT LegalizeVT = LEGALPOS < 0 ? Node->getValueType(-(1 + LEGALPOS)) \ in LegalizeOp()
495 : Node->getOperand(LEGALPOS).getValueType(); \ in LegalizeOp()
497 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); \ in LegalizeOp()
502 Action = TLI.getOperationAction(Node->getOpcode(), LegalizeVT); \ in LegalizeOp()
507 LLVM_DEBUG(dbgs() << "\nLegalizing vector op: "; Node->dump(&DAG)); in LegalizeOp()
516 Promote(Node, ResultVals); in LegalizeOp()
524 if (LowerOperationWrapper(Node, ResultVals)) in LegalizeOp()
530 Expand(Node, ResultVals); in LegalizeOp()
535 return TranslateLegalizeResults(Op, Node); in LegalizeOp()
543 bool VectorLegalizer::LowerOperationWrapper(SDNode *Node, in LowerOperationWrapper() argument
545 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); in LowerOperationWrapper()
550 if (Res == SDValue(Node, 0)) in LowerOperationWrapper()
555 if (Node->getNumValues() == 1) { in LowerOperationWrapper()
562 assert((Node->getNumValues() == Res->getNumValues()) && in LowerOperationWrapper()
566 for (unsigned I = 0, E = Node->getNumValues(); I != E; ++I) in LowerOperationWrapper()
572 void VectorLegalizer::PromoteReduction(SDNode *Node, in PromoteReduction() argument
574 MVT VecVT = Node->getOperand(1).getSimpleValueType(); in PromoteReduction()
575 MVT NewVecVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VecVT); in PromoteReduction()
576 MVT ScalarVT = Node->getSimpleValueType(0); in PromoteReduction()
579 SDLoc DL(Node); in PromoteReduction()
580 SmallVector<SDValue, 4> Operands(Node->getNumOperands()); in PromoteReduction()
583 if (Node->getOperand(0).getValueType().isFloatingPoint()) in PromoteReduction()
585 DAG.getNode(ISD::FP_EXTEND, DL, NewScalarVT, Node->getOperand(0)); in PromoteReduction()
588 DAG.getNode(ISD::ANY_EXTEND, DL, NewScalarVT, Node->getOperand(0)); in PromoteReduction()
590 for (unsigned j = 1; j != Node->getNumOperands(); ++j) in PromoteReduction()
591 if (Node->getOperand(j).getValueType().isVector() && in PromoteReduction()
592 !(ISD::isVPOpcode(Node->getOpcode()) && in PromoteReduction()
593 ISD::getVPMaskIdx(Node->getOpcode()) == j)) // Skip mask operand. in PromoteReduction()
595 if (Node->getOperand(j).getValueType().isFloatingPoint()) in PromoteReduction()
597 DAG.getNode(ISD::FP_EXTEND, DL, NewVecVT, Node->getOperand(j)); in PromoteReduction()
600 DAG.getNode(ISD::ANY_EXTEND, DL, NewVecVT, Node->getOperand(j)); in PromoteReduction()
602 Operands[j] = Node->getOperand(j); // Skip VL operand. in PromoteReduction()
604 SDValue Res = DAG.getNode(Node->getOpcode(), DL, NewScalarVT, Operands, in PromoteReduction()
605 Node->getFlags()); in PromoteReduction()
616 void VectorLegalizer::PromoteSETCC(SDNode *Node, in PromoteSETCC() argument
618 MVT VecVT = Node->getOperand(0).getSimpleValueType(); in PromoteSETCC()
619 MVT NewVecVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VecVT); in PromoteSETCC()
623 SDLoc DL(Node); in PromoteSETCC()
624 SmallVector<SDValue, 5> Operands(Node->getNumOperands()); in PromoteSETCC()
626 Operands[0] = DAG.getNode(ExtOp, DL, NewVecVT, Node->getOperand(0)); in PromoteSETCC()
627 Operands[1] = DAG.getNode(ExtOp, DL, NewVecVT, Node->getOperand(1)); in PromoteSETCC()
628 Operands[2] = Node->getOperand(2); in PromoteSETCC()
630 if (Node->getOpcode() == ISD::VP_SETCC) { in PromoteSETCC()
631 Operands[3] = Node->getOperand(3); // mask in PromoteSETCC()
632 Operands[4] = Node->getOperand(4); // evl in PromoteSETCC()
635 SDValue Res = DAG.getNode(Node->getOpcode(), DL, Node->getSimpleValueType(0), in PromoteSETCC()
636 Operands, Node->getFlags()); in PromoteSETCC()
641 void VectorLegalizer::PromoteSTRICT(SDNode *Node, in PromoteSTRICT() argument
643 MVT VecVT = Node->getOperand(1).getSimpleValueType(); in PromoteSTRICT()
644 MVT NewVecVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VecVT); in PromoteSTRICT()
648 SDLoc DL(Node); in PromoteSTRICT()
649 SmallVector<SDValue, 5> Operands(Node->getNumOperands()); in PromoteSTRICT()
652 for (unsigned j = 1; j != Node->getNumOperands(); ++j) in PromoteSTRICT()
653 if (Node->getOperand(j).getValueType().isVector() && in PromoteSTRICT()
654 !(ISD::isVPOpcode(Node->getOpcode()) && in PromoteSTRICT()
655 ISD::getVPMaskIdx(Node->getOpcode()) == j)) // Skip mask operand. in PromoteSTRICT()
660 {Node->getOperand(0), Node->getOperand(j)}); in PromoteSTRICT()
664 Operands[j] = Node->getOperand(j); // Skip no vector operand. in PromoteSTRICT()
666 SDVTList VTs = DAG.getVTList(NewVecVT, Node->getValueType(1)); in PromoteSTRICT()
671 DAG.getNode(Node->getOpcode(), DL, VTs, Operands, Node->getFlags()); in PromoteSTRICT()
682 void VectorLegalizer::Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results) { in Promote() argument
685 switch (Node->getOpcode()) { in Promote()
691 PromoteINT_TO_FP(Node, Results); in Promote()
698 PromoteFP_TO_INT(Node, Results); in Promote()
715 PromoteReduction(Node, Results); in Promote()
720 PromoteSETCC(Node, Results); in Promote()
728 PromoteSTRICT(Node, Results); in Promote()
742 assert(Node->getNumValues() == 1 && in Promote()
744 MVT VT = Node->getSimpleValueType(0); in Promote()
745 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); in Promote()
746 SDLoc dl(Node); in Promote()
747 SmallVector<SDValue, 4> Operands(Node->getNumOperands()); in Promote()
749 for (unsigned j = 0; j != Node->getNumOperands(); ++j) { in Promote()
751 bool SkipPromote = ISD::isVPOpcode(Node->getOpcode()) && in Promote()
752 ISD::getVPMaskIdx(Node->getOpcode()) == j; in Promote()
753 if (Node->getOperand(j).getValueType().isVector() && !SkipPromote) in Promote()
754 if (Node->getOperand(j) in Promote()
759 Operands[j] = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(j)); in Promote()
761 Operands[j] = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(j)); in Promote()
763 Operands[j] = Node->getOperand(j); in Promote()
767 DAG.getNode(Node->getOpcode(), dl, NVT, Operands, Node->getFlags()); in Promote()
780 void VectorLegalizer::PromoteINT_TO_FP(SDNode *Node, in PromoteINT_TO_FP() argument
784 bool IsStrict = Node->isStrictFPOpcode(); in PromoteINT_TO_FP()
785 MVT VT = Node->getOperand(IsStrict ? 1 : 0).getSimpleValueType(); in PromoteINT_TO_FP()
786 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); in PromoteINT_TO_FP()
790 SDLoc dl(Node); in PromoteINT_TO_FP()
791 SmallVector<SDValue, 4> Operands(Node->getNumOperands()); in PromoteINT_TO_FP()
793 unsigned Opc = (Node->getOpcode() == ISD::UINT_TO_FP || in PromoteINT_TO_FP()
794 Node->getOpcode() == ISD::STRICT_UINT_TO_FP) in PromoteINT_TO_FP()
797 for (unsigned j = 0; j != Node->getNumOperands(); ++j) { in PromoteINT_TO_FP()
798 if (Node->getOperand(j).getValueType().isVector()) in PromoteINT_TO_FP()
799 Operands[j] = DAG.getNode(Opc, dl, NVT, Node->getOperand(j)); in PromoteINT_TO_FP()
801 Operands[j] = Node->getOperand(j); in PromoteINT_TO_FP()
805 SDValue Res = DAG.getNode(Node->getOpcode(), dl, in PromoteINT_TO_FP()
806 {Node->getValueType(0), MVT::Other}, Operands); in PromoteINT_TO_FP()
813 DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Operands); in PromoteINT_TO_FP()
821 void VectorLegalizer::PromoteFP_TO_INT(SDNode *Node, in PromoteFP_TO_INT() argument
823 MVT VT = Node->getSimpleValueType(0); in PromoteFP_TO_INT()
824 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); in PromoteFP_TO_INT()
825 bool IsStrict = Node->isStrictFPOpcode(); in PromoteFP_TO_INT()
829 unsigned NewOpc = Node->getOpcode(); in PromoteFP_TO_INT()
840 SDLoc dl(Node); in PromoteFP_TO_INT()
844 {Node->getOperand(0), Node->getOperand(1)}); in PromoteFP_TO_INT()
847 Promoted = DAG.getNode(NewOpc, dl, NVT, Node->getOperand(0)); in PromoteFP_TO_INT()
852 if (Node->getOpcode() == ISD::FP_TO_UINT || in PromoteFP_TO_INT()
853 Node->getOpcode() == ISD::STRICT_FP_TO_UINT) in PromoteFP_TO_INT()
877 void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) { in Expand() argument
878 switch (Node->getOpcode()) { in Expand()
880 std::pair<SDValue, SDValue> Tmp = ExpandLoad(Node); in Expand()
886 Results.push_back(ExpandStore(Node)); in Expand()
889 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) in Expand()
890 Results.push_back(Node->getOperand(i)); in Expand()
893 Results.push_back(ExpandSEXTINREG(Node)); in Expand()
896 Results.push_back(ExpandANY_EXTEND_VECTOR_INREG(Node)); in Expand()
899 Results.push_back(ExpandSIGN_EXTEND_VECTOR_INREG(Node)); in Expand()
902 Results.push_back(ExpandZERO_EXTEND_VECTOR_INREG(Node)); in Expand()
905 Results.push_back(ExpandBSWAP(Node)); in Expand()
908 Results.push_back(TLI.expandVPBSWAP(Node, DAG)); in Expand()
911 Results.push_back(ExpandVSELECT(Node)); in Expand()
914 Results.push_back(ExpandVP_SELECT(Node)); in Expand()
918 if (SDValue Expanded = ExpandVP_REM(Node)) { in Expand()
924 Results.push_back(ExpandSELECT(Node)); in Expand()
927 if (Node->getValueType(0).isScalableVector()) { in Expand()
929 DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0)); in Expand()
931 DAG.getNode(ISD::SETCC, SDLoc(Node), CondVT, Node->getOperand(0), in Expand()
932 Node->getOperand(1), Node->getOperand(4)); in Expand()
933 Results.push_back(DAG.getSelect(SDLoc(Node), Node->getValueType(0), SetCC, in Expand()
934 Node->getOperand(2), in Expand()
935 Node->getOperand(3))); in Expand()
941 ExpandFP_TO_UINT(Node, Results); in Expand()
944 ExpandUINT_TO_FLOAT(Node, Results); in Expand()
947 Results.push_back(ExpandFNEG(Node)); in Expand()
950 ExpandFSUB(Node, Results); in Expand()
954 ExpandSETCC(Node, Results); in Expand()
957 if (SDValue Expanded = TLI.expandABS(Node, DAG)) { in Expand()
964 if (SDValue Expanded = TLI.expandABD(Node, DAG)) { in Expand()
970 ExpandBITREVERSE(Node, Results); in Expand()
973 if (SDValue Expanded = TLI.expandVPBITREVERSE(Node, DAG)) { in Expand()
979 if (SDValue Expanded = TLI.expandCTPOP(Node, DAG)) { in Expand()
985 if (SDValue Expanded = TLI.expandVPCTPOP(Node, DAG)) { in Expand()
992 if (SDValue Expanded = TLI.expandCTLZ(Node, DAG)) { in Expand()
999 if (SDValue Expanded = TLI.expandVPCTLZ(Node, DAG)) { in Expand()
1006 if (SDValue Expanded = TLI.expandCTTZ(Node, DAG)) { in Expand()
1013 if (SDValue Expanded = TLI.expandVPCTTZ(Node, DAG)) { in Expand()
1022 if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG)) { in Expand()
1029 if (SDValue Expanded = TLI.expandROT(Node, false /*AllowVectorOps*/, DAG)) { in Expand()
1036 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG)) { in Expand()
1045 if (SDValue Expanded = TLI.expandIntMINMAX(Node, DAG)) { in Expand()
1052 ExpandUADDSUBO(Node, Results); in Expand()
1056 ExpandSADDSUBO(Node, Results); in Expand()
1060 ExpandMULO(Node, Results); in Expand()
1066 if (SDValue Expanded = TLI.expandAddSubSat(Node, DAG)) { in Expand()
1073 if (SDValue Expanded = TLI.expandShlSat(Node, DAG)) { in Expand()
1081 if (Node->getValueType(0).isScalableVector()) { in Expand()
1082 if (SDValue Expanded = TLI.expandFP_TO_INT_SAT(Node, DAG)) { in Expand()
1090 if (SDValue Expanded = TLI.expandFixedPointMul(Node, DAG)) { in Expand()
1104 ExpandFixedPointDiv(Node, Results); in Expand()
1112 ExpandStrictFPOp(Node, Results); in Expand()
1129 Results.push_back(TLI.expandVecReduce(Node, DAG)); in Expand()
1133 Results.push_back(TLI.expandVecReduceSeq(Node, DAG)); in Expand()
1137 ExpandREM(Node, Results); in Expand()
1140 Results.push_back(ExpandVP_MERGE(Node)); in Expand()
1144 SDValue Unrolled = DAG.UnrollVectorOp(Node); in Expand()
1149 SDValue VectorLegalizer::ExpandSELECT(SDNode *Node) { in ExpandSELECT() argument
1153 EVT VT = Node->getValueType(0); in ExpandSELECT()
1154 SDLoc DL(Node); in ExpandSELECT()
1156 SDValue Mask = Node->getOperand(0); in ExpandSELECT()
1157 SDValue Op1 = Node->getOperand(1); in ExpandSELECT()
1158 SDValue Op2 = Node->getOperand(2); in ExpandSELECT()
1177 return DAG.UnrollVectorOp(Node); in ExpandSELECT()
1202 return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val); in ExpandSELECT()
1205 SDValue VectorLegalizer::ExpandSEXTINREG(SDNode *Node) { in ExpandSEXTINREG() argument
1206 EVT VT = Node->getValueType(0); in ExpandSEXTINREG()
1211 return DAG.UnrollVectorOp(Node); in ExpandSEXTINREG()
1213 SDLoc DL(Node); in ExpandSEXTINREG()
1214 EVT OrigTy = cast<VTSDNode>(Node->getOperand(1))->getVT(); in ExpandSEXTINREG()
1220 SDValue Op = DAG.getNode(ISD::SHL, DL, VT, Node->getOperand(0), ShiftSz); in ExpandSEXTINREG()
1226 SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node) { in ExpandANY_EXTEND_VECTOR_INREG() argument
1227 SDLoc DL(Node); in ExpandANY_EXTEND_VECTOR_INREG()
1228 EVT VT = Node->getValueType(0); in ExpandANY_EXTEND_VECTOR_INREG()
1230 SDValue Src = Node->getOperand(0); in ExpandANY_EXTEND_VECTOR_INREG()
1261 SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node) { in ExpandSIGN_EXTEND_VECTOR_INREG() argument
1262 SDLoc DL(Node); in ExpandSIGN_EXTEND_VECTOR_INREG()
1263 EVT VT = Node->getValueType(0); in ExpandSIGN_EXTEND_VECTOR_INREG()
1264 SDValue Src = Node->getOperand(0); in ExpandSIGN_EXTEND_VECTOR_INREG()
1285 SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node) { in ExpandZERO_EXTEND_VECTOR_INREG() argument
1286 SDLoc DL(Node); in ExpandZERO_EXTEND_VECTOR_INREG()
1287 EVT VT = Node->getValueType(0); in ExpandZERO_EXTEND_VECTOR_INREG()
1289 SDValue Src = Node->getOperand(0); in ExpandZERO_EXTEND_VECTOR_INREG()
1328 SDValue VectorLegalizer::ExpandBSWAP(SDNode *Node) { in ExpandBSWAP() argument
1329 EVT VT = Node->getValueType(0); in ExpandBSWAP()
1333 return TLI.expandBSWAP(Node, DAG); in ExpandBSWAP()
1342 SDLoc DL(Node); in ExpandBSWAP()
1343 SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0)); in ExpandBSWAP()
1354 return TLI.expandBSWAP(Node, DAG); in ExpandBSWAP()
1357 return DAG.UnrollVectorOp(Node); in ExpandBSWAP()
1360 void VectorLegalizer::ExpandBITREVERSE(SDNode *Node, in ExpandBITREVERSE() argument
1362 EVT VT = Node->getValueType(0); in ExpandBITREVERSE()
1366 Results.push_back(TLI.expandBITREVERSE(Node, DAG)); in ExpandBITREVERSE()
1372 SDValue Tmp = DAG.UnrollVectorOp(Node); in ExpandBITREVERSE()
1392 SDLoc DL(Node); in ExpandBITREVERSE()
1393 SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0)); in ExpandBITREVERSE()
1409 Results.push_back(TLI.expandBITREVERSE(Node, DAG)); in ExpandBITREVERSE()
1414 SDValue Tmp = DAG.UnrollVectorOp(Node); in ExpandBITREVERSE()
1418 SDValue VectorLegalizer::ExpandVSELECT(SDNode *Node) { in ExpandVSELECT() argument
1421 SDLoc DL(Node); in ExpandVSELECT()
1423 SDValue Mask = Node->getOperand(0); in ExpandVSELECT()
1424 SDValue Op1 = Node->getOperand(1); in ExpandVSELECT()
1425 SDValue Op2 = Node->getOperand(2); in ExpandVSELECT()
1436 return DAG.UnrollVectorOp(Node); in ExpandVSELECT()
1446 return DAG.UnrollVectorOp(Node); in ExpandVSELECT()
1452 return DAG.UnrollVectorOp(Node); in ExpandVSELECT()
1465 return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val); in ExpandVSELECT()
1468 SDValue VectorLegalizer::ExpandVP_SELECT(SDNode *Node) { in ExpandVP_SELECT() argument
1471 SDLoc DL(Node); in ExpandVP_SELECT()
1473 SDValue Mask = Node->getOperand(0); in ExpandVP_SELECT()
1474 SDValue Op1 = Node->getOperand(1); in ExpandVP_SELECT()
1475 SDValue Op2 = Node->getOperand(2); in ExpandVP_SELECT()
1476 SDValue EVL = Node->getOperand(3); in ExpandVP_SELECT()
1485 return DAG.UnrollVectorOp(Node); in ExpandVP_SELECT()
1489 return DAG.UnrollVectorOp(Node); in ExpandVP_SELECT()
1499 SDValue VectorLegalizer::ExpandVP_MERGE(SDNode *Node) { in ExpandVP_MERGE() argument
1504 SDLoc DL(Node); in ExpandVP_MERGE()
1506 SDValue Mask = Node->getOperand(0); in ExpandVP_MERGE()
1507 SDValue Op1 = Node->getOperand(1); in ExpandVP_MERGE()
1508 SDValue Op2 = Node->getOperand(2); in ExpandVP_MERGE()
1509 SDValue EVL = Node->getOperand(3); in ExpandVP_MERGE()
1523 return DAG.UnrollVectorOp(Node); in ExpandVP_MERGE()
1529 return DAG.UnrollVectorOp(Node); in ExpandVP_MERGE()
1537 return DAG.getSelect(DL, Node->getValueType(0), FullMask, Op1, Op2); in ExpandVP_MERGE()
1540 SDValue VectorLegalizer::ExpandVP_REM(SDNode *Node) { in ExpandVP_REM() argument
1542 EVT VT = Node->getValueType(0); in ExpandVP_REM()
1544 unsigned DivOpc = Node->getOpcode() == ISD::VP_SREM ? ISD::VP_SDIV : ISD::VP_UDIV; in ExpandVP_REM()
1551 SDLoc DL(Node); in ExpandVP_REM()
1553 SDValue Dividend = Node->getOperand(0); in ExpandVP_REM()
1554 SDValue Divisor = Node->getOperand(1); in ExpandVP_REM()
1555 SDValue Mask = Node->getOperand(2); in ExpandVP_REM()
1556 SDValue EVL = Node->getOperand(3); in ExpandVP_REM()
1564 void VectorLegalizer::ExpandFP_TO_UINT(SDNode *Node, in ExpandFP_TO_UINT() argument
1568 if (TLI.expandFP_TO_UINT(Node, Result, Chain, DAG)) { in ExpandFP_TO_UINT()
1570 if (Node->isStrictFPOpcode()) in ExpandFP_TO_UINT()
1576 if (Node->isStrictFPOpcode()) { in ExpandFP_TO_UINT()
1577 UnrollStrictFPOp(Node, Results); in ExpandFP_TO_UINT()
1581 Results.push_back(DAG.UnrollVectorOp(Node)); in ExpandFP_TO_UINT()
1584 void VectorLegalizer::ExpandUINT_TO_FLOAT(SDNode *Node, in ExpandUINT_TO_FLOAT() argument
1586 bool IsStrict = Node->isStrictFPOpcode(); in ExpandUINT_TO_FLOAT()
1588 SDValue Src = Node->getOperand(OpNo); in ExpandUINT_TO_FLOAT()
1590 SDLoc DL(Node); in ExpandUINT_TO_FLOAT()
1595 if (TLI.expandUINT_TO_FP(Node, Result, Chain, DAG)) { in ExpandUINT_TO_FLOAT()
1609 UnrollStrictFPOp(Node, Results); in ExpandUINT_TO_FLOAT()
1613 Results.push_back(DAG.UnrollVectorOp(Node)); in ExpandUINT_TO_FLOAT()
1631 DAG.getConstantFP(1ULL << (BW / 2), DL, Node->getValueType(0)); in ExpandUINT_TO_FLOAT()
1642 {Node->getValueType(0), MVT::Other}, in ExpandUINT_TO_FLOAT()
1643 {Node->getOperand(0), HI}); in ExpandUINT_TO_FLOAT()
1644 fHI = DAG.getNode(ISD::STRICT_FMUL, DL, {Node->getValueType(0), MVT::Other}, in ExpandUINT_TO_FLOAT()
1647 {Node->getValueType(0), MVT::Other}, in ExpandUINT_TO_FLOAT()
1648 {Node->getOperand(0), LO}); in ExpandUINT_TO_FLOAT()
1655 DAG.getNode(ISD::STRICT_FADD, DL, {Node->getValueType(0), MVT::Other}, in ExpandUINT_TO_FLOAT()
1666 SDValue fHI = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), HI); in ExpandUINT_TO_FLOAT()
1667 fHI = DAG.getNode(ISD::FMUL, DL, Node->getValueType(0), fHI, TWOHW); in ExpandUINT_TO_FLOAT()
1668 SDValue fLO = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), LO); in ExpandUINT_TO_FLOAT()
1672 DAG.getNode(ISD::FADD, DL, Node->getValueType(0), fHI, fLO)); in ExpandUINT_TO_FLOAT()
1675 SDValue VectorLegalizer::ExpandFNEG(SDNode *Node) { in ExpandFNEG() argument
1676 if (TLI.isOperationLegalOrCustom(ISD::FSUB, Node->getValueType(0))) { in ExpandFNEG()
1677 SDLoc DL(Node); in ExpandFNEG()
1678 SDValue Zero = DAG.getConstantFP(-0.0, DL, Node->getValueType(0)); in ExpandFNEG()
1680 return DAG.getNode(ISD::FSUB, DL, Node->getValueType(0), Zero, in ExpandFNEG()
1681 Node->getOperand(0)); in ExpandFNEG()
1683 return DAG.UnrollVectorOp(Node); in ExpandFNEG()
1686 void VectorLegalizer::ExpandFSUB(SDNode *Node, in ExpandFSUB() argument
1691 EVT VT = Node->getValueType(0); in ExpandFSUB()
1696 SDValue Tmp = DAG.UnrollVectorOp(Node); in ExpandFSUB()
1700 void VectorLegalizer::ExpandSETCC(SDNode *Node, in ExpandSETCC() argument
1703 bool IsVP = Node->getOpcode() == ISD::VP_SETCC; in ExpandSETCC()
1704 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC || in ExpandSETCC()
1705 Node->getOpcode() == ISD::STRICT_FSETCCS; in ExpandSETCC()
1706 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS; in ExpandSETCC()
1709 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); in ExpandSETCC()
1710 SDValue LHS = Node->getOperand(0 + Offset); in ExpandSETCC()
1711 SDValue RHS = Node->getOperand(1 + Offset); in ExpandSETCC()
1712 SDValue CC = Node->getOperand(2 + Offset); in ExpandSETCC()
1719 UnrollStrictFPOp(Node, Results); in ExpandSETCC()
1722 Results.push_back(UnrollVSETCC(Node)); in ExpandSETCC()
1728 Mask = Node->getOperand(3 + Offset); in ExpandSETCC()
1729 EVL = Node->getOperand(4 + Offset); in ExpandSETCC()
1732 SDLoc dl(Node); in ExpandSETCC()
1734 TLI.LegalizeSetCCCondCode(DAG, Node->getValueType(0), LHS, RHS, CC, Mask, in ExpandSETCC()
1742 LHS = DAG.getNode(Node->getOpcode(), dl, Node->getVTList(), in ExpandSETCC()
1743 {Chain, LHS, RHS, CC}, Node->getFlags()); in ExpandSETCC()
1746 LHS = DAG.getNode(ISD::VP_SETCC, dl, Node->getValueType(0), in ExpandSETCC()
1747 {LHS, RHS, CC, Mask, EVL}, Node->getFlags()); in ExpandSETCC()
1749 LHS = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), LHS, RHS, CC, in ExpandSETCC()
1750 Node->getFlags()); in ExpandSETCC()
1767 EVT VT = Node->getValueType(0); in ExpandSETCC()
1772 LHS->setFlags(Node->getFlags()); in ExpandSETCC()
1780 void VectorLegalizer::ExpandUADDSUBO(SDNode *Node, in ExpandUADDSUBO() argument
1783 TLI.expandUADDSUBO(Node, Result, Overflow, DAG); in ExpandUADDSUBO()
1788 void VectorLegalizer::ExpandSADDSUBO(SDNode *Node, in ExpandSADDSUBO() argument
1791 TLI.expandSADDSUBO(Node, Result, Overflow, DAG); in ExpandSADDSUBO()
1796 void VectorLegalizer::ExpandMULO(SDNode *Node, in ExpandMULO() argument
1799 if (!TLI.expandMULO(Node, Result, Overflow, DAG)) in ExpandMULO()
1800 std::tie(Result, Overflow) = DAG.UnrollVectorOverflowOp(Node); in ExpandMULO()
1806 void VectorLegalizer::ExpandFixedPointDiv(SDNode *Node, in ExpandFixedPointDiv() argument
1808 SDNode *N = Node; in ExpandFixedPointDiv()
1814 void VectorLegalizer::ExpandStrictFPOp(SDNode *Node, in ExpandStrictFPOp() argument
1816 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP) { in ExpandStrictFPOp()
1817 ExpandUINT_TO_FLOAT(Node, Results); in ExpandStrictFPOp()
1820 if (Node->getOpcode() == ISD::STRICT_FP_TO_UINT) { in ExpandStrictFPOp()
1821 ExpandFP_TO_UINT(Node, Results); in ExpandStrictFPOp()
1825 if (Node->getOpcode() == ISD::STRICT_FSETCC || in ExpandStrictFPOp()
1826 Node->getOpcode() == ISD::STRICT_FSETCCS) { in ExpandStrictFPOp()
1827 ExpandSETCC(Node, Results); in ExpandStrictFPOp()
1831 UnrollStrictFPOp(Node, Results); in ExpandStrictFPOp()
1834 void VectorLegalizer::ExpandREM(SDNode *Node, in ExpandREM() argument
1836 assert((Node->getOpcode() == ISD::SREM || Node->getOpcode() == ISD::UREM) && in ExpandREM()
1840 if (!TLI.expandREM(Node, Result, DAG)) in ExpandREM()
1841 Result = DAG.UnrollVectorOp(Node); in ExpandREM()
1845 void VectorLegalizer::UnrollStrictFPOp(SDNode *Node, in UnrollStrictFPOp() argument
1847 EVT VT = Node->getValueType(0); in UnrollStrictFPOp()
1850 unsigned NumOpers = Node->getNumOperands(); in UnrollStrictFPOp()
1854 if (Node->getOpcode() == ISD::STRICT_FSETCC || in UnrollStrictFPOp()
1855 Node->getOpcode() == ISD::STRICT_FSETCCS) in UnrollStrictFPOp()
1860 SDValue Chain = Node->getOperand(0); in UnrollStrictFPOp()
1861 SDLoc dl(Node); in UnrollStrictFPOp()
1874 SDValue Oper = Node->getOperand(j); in UnrollStrictFPOp()
1884 SDValue ScalarOp = DAG.getNode(Node->getOpcode(), dl, ValueVTs, Opers); in UnrollStrictFPOp()
1888 if (Node->getOpcode() == ISD::STRICT_FSETCC || in UnrollStrictFPOp()
1889 Node->getOpcode() == ISD::STRICT_FSETCCS) in UnrollStrictFPOp()
1905 SDValue VectorLegalizer::UnrollVSETCC(SDNode *Node) { in UnrollVSETCC() argument
1906 EVT VT = Node->getValueType(0); in UnrollVSETCC()
1909 SDValue LHS = Node->getOperand(0); in UnrollVSETCC()
1910 SDValue RHS = Node->getOperand(1); in UnrollVSETCC()
1911 SDValue CC = Node->getOperand(2); in UnrollVSETCC()
1913 SDLoc dl(Node); in UnrollVSETCC()