Lines Matching refs:VT
38 EVT AMDGPUTargetLowering::getEquivalentMemType(LLVMContext &Ctx, EVT VT) { in getEquivalentMemType() argument
39 unsigned StoreSize = VT.getStoreSizeInBits(); in getEquivalentMemType()
48 EVT VT = Op.getValueType(); in numBitsUnsigned() local
50 return VT.getSizeInBits() - Known.countMinLeadingZeros(); in numBitsUnsigned()
54 EVT VT = Op.getValueType(); in numBitsSigned() local
58 return VT.getSizeInBits() - DAG.ComputeNumSignBits(Op); in numBitsSigned()
134 for (MVT VT : MVT::integer_valuetypes()) { in AMDGPUTargetLowering() local
135 setLoadExtAction(ISD::EXTLOAD, MVT::i64, VT, Expand); in AMDGPUTargetLowering()
136 setLoadExtAction(ISD::SEXTLOAD, MVT::i64, VT, Expand); in AMDGPUTargetLowering()
137 setLoadExtAction(ISD::ZEXTLOAD, MVT::i64, VT, Expand); in AMDGPUTargetLowering()
140 for (MVT VT : MVT::integer_valuetypes()) { in AMDGPUTargetLowering() local
141 if (VT == MVT::i64) in AMDGPUTargetLowering()
144 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); in AMDGPUTargetLowering()
145 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i8, Legal); in AMDGPUTargetLowering()
146 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i16, Legal); in AMDGPUTargetLowering()
147 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i32, Expand); in AMDGPUTargetLowering()
149 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote); in AMDGPUTargetLowering()
150 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i8, Legal); in AMDGPUTargetLowering()
151 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i16, Legal); in AMDGPUTargetLowering()
152 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i32, Expand); in AMDGPUTargetLowering()
154 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i1, Promote); in AMDGPUTargetLowering()
155 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i8, Legal); in AMDGPUTargetLowering()
156 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i16, Legal); in AMDGPUTargetLowering()
157 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i32, Expand); in AMDGPUTargetLowering()
160 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) { in AMDGPUTargetLowering() local
161 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i8, Expand); in AMDGPUTargetLowering()
162 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i8, Expand); in AMDGPUTargetLowering()
163 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v2i8, Expand); in AMDGPUTargetLowering()
164 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i8, Expand); in AMDGPUTargetLowering()
165 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i8, Expand); in AMDGPUTargetLowering()
166 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v4i8, Expand); in AMDGPUTargetLowering()
167 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i16, Expand); in AMDGPUTargetLowering()
168 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i16, Expand); in AMDGPUTargetLowering()
169 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v2i16, Expand); in AMDGPUTargetLowering()
170 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v3i16, Expand); in AMDGPUTargetLowering()
171 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v3i16, Expand); in AMDGPUTargetLowering()
172 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v3i16, Expand); in AMDGPUTargetLowering()
173 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i16, Expand); in AMDGPUTargetLowering()
174 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i16, Expand); in AMDGPUTargetLowering()
175 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v4i16, Expand); in AMDGPUTargetLowering()
401 for (MVT VT : ScalarIntVTs) { in AMDGPUTargetLowering() local
403 setOperationAction(ISD::SDIV, VT, Expand); in AMDGPUTargetLowering()
404 setOperationAction(ISD::UDIV, VT, Expand); in AMDGPUTargetLowering()
405 setOperationAction(ISD::SREM, VT, Expand); in AMDGPUTargetLowering()
406 setOperationAction(ISD::UREM, VT, Expand); in AMDGPUTargetLowering()
409 setOperationAction(ISD::SDIVREM, VT, Custom); in AMDGPUTargetLowering()
410 setOperationAction(ISD::UDIVREM, VT, Custom); in AMDGPUTargetLowering()
413 setOperationAction(ISD::SMUL_LOHI, VT, Expand); in AMDGPUTargetLowering()
414 setOperationAction(ISD::UMUL_LOHI, VT, Expand); in AMDGPUTargetLowering()
416 setOperationAction(ISD::BSWAP, VT, Expand); in AMDGPUTargetLowering()
417 setOperationAction(ISD::CTTZ, VT, Expand); in AMDGPUTargetLowering()
418 setOperationAction(ISD::CTLZ, VT, Expand); in AMDGPUTargetLowering()
421 setOperationAction(ISD::ADDC, VT, Legal); in AMDGPUTargetLowering()
422 setOperationAction(ISD::SUBC, VT, Legal); in AMDGPUTargetLowering()
423 setOperationAction(ISD::ADDE, VT, Legal); in AMDGPUTargetLowering()
424 setOperationAction(ISD::SUBE, VT, Legal); in AMDGPUTargetLowering()
460 for (MVT VT : VectorIntTypes) { in AMDGPUTargetLowering() local
462 setOperationAction(ISD::ADD, VT, Expand); in AMDGPUTargetLowering()
463 setOperationAction(ISD::AND, VT, Expand); in AMDGPUTargetLowering()
464 setOperationAction(ISD::FP_TO_SINT, VT, Expand); in AMDGPUTargetLowering()
465 setOperationAction(ISD::FP_TO_UINT, VT, Expand); in AMDGPUTargetLowering()
466 setOperationAction(ISD::MUL, VT, Expand); in AMDGPUTargetLowering()
467 setOperationAction(ISD::MULHU, VT, Expand); in AMDGPUTargetLowering()
468 setOperationAction(ISD::MULHS, VT, Expand); in AMDGPUTargetLowering()
469 setOperationAction(ISD::OR, VT, Expand); in AMDGPUTargetLowering()
470 setOperationAction(ISD::SHL, VT, Expand); in AMDGPUTargetLowering()
471 setOperationAction(ISD::SRA, VT, Expand); in AMDGPUTargetLowering()
472 setOperationAction(ISD::SRL, VT, Expand); in AMDGPUTargetLowering()
473 setOperationAction(ISD::ROTL, VT, Expand); in AMDGPUTargetLowering()
474 setOperationAction(ISD::ROTR, VT, Expand); in AMDGPUTargetLowering()
475 setOperationAction(ISD::SUB, VT, Expand); in AMDGPUTargetLowering()
476 setOperationAction(ISD::SINT_TO_FP, VT, Expand); in AMDGPUTargetLowering()
477 setOperationAction(ISD::UINT_TO_FP, VT, Expand); in AMDGPUTargetLowering()
478 setOperationAction(ISD::SDIV, VT, Expand); in AMDGPUTargetLowering()
479 setOperationAction(ISD::UDIV, VT, Expand); in AMDGPUTargetLowering()
480 setOperationAction(ISD::SREM, VT, Expand); in AMDGPUTargetLowering()
481 setOperationAction(ISD::UREM, VT, Expand); in AMDGPUTargetLowering()
482 setOperationAction(ISD::SMUL_LOHI, VT, Expand); in AMDGPUTargetLowering()
483 setOperationAction(ISD::UMUL_LOHI, VT, Expand); in AMDGPUTargetLowering()
484 setOperationAction(ISD::SDIVREM, VT, Expand); in AMDGPUTargetLowering()
485 setOperationAction(ISD::UDIVREM, VT, Expand); in AMDGPUTargetLowering()
486 setOperationAction(ISD::SELECT, VT, Expand); in AMDGPUTargetLowering()
487 setOperationAction(ISD::VSELECT, VT, Expand); in AMDGPUTargetLowering()
488 setOperationAction(ISD::SELECT_CC, VT, Expand); in AMDGPUTargetLowering()
489 setOperationAction(ISD::XOR, VT, Expand); in AMDGPUTargetLowering()
490 setOperationAction(ISD::BSWAP, VT, Expand); in AMDGPUTargetLowering()
491 setOperationAction(ISD::CTPOP, VT, Expand); in AMDGPUTargetLowering()
492 setOperationAction(ISD::CTTZ, VT, Expand); in AMDGPUTargetLowering()
493 setOperationAction(ISD::CTLZ, VT, Expand); in AMDGPUTargetLowering()
494 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Expand); in AMDGPUTargetLowering()
495 setOperationAction(ISD::SETCC, VT, Expand); in AMDGPUTargetLowering()
501 for (MVT VT : FloatVectorTypes) { in AMDGPUTargetLowering() local
502 setOperationAction(ISD::FABS, VT, Expand); in AMDGPUTargetLowering()
503 setOperationAction(ISD::FMINNUM, VT, Expand); in AMDGPUTargetLowering()
504 setOperationAction(ISD::FMAXNUM, VT, Expand); in AMDGPUTargetLowering()
505 setOperationAction(ISD::FADD, VT, Expand); in AMDGPUTargetLowering()
506 setOperationAction(ISD::FCEIL, VT, Expand); in AMDGPUTargetLowering()
507 setOperationAction(ISD::FCOS, VT, Expand); in AMDGPUTargetLowering()
508 setOperationAction(ISD::FDIV, VT, Expand); in AMDGPUTargetLowering()
509 setOperationAction(ISD::FEXP2, VT, Expand); in AMDGPUTargetLowering()
510 setOperationAction(ISD::FEXP, VT, Expand); in AMDGPUTargetLowering()
511 setOperationAction(ISD::FLOG2, VT, Expand); in AMDGPUTargetLowering()
512 setOperationAction(ISD::FREM, VT, Expand); in AMDGPUTargetLowering()
513 setOperationAction(ISD::FLOG, VT, Expand); in AMDGPUTargetLowering()
514 setOperationAction(ISD::FLOG10, VT, Expand); in AMDGPUTargetLowering()
515 setOperationAction(ISD::FPOW, VT, Expand); in AMDGPUTargetLowering()
516 setOperationAction(ISD::FFLOOR, VT, Expand); in AMDGPUTargetLowering()
517 setOperationAction(ISD::FTRUNC, VT, Expand); in AMDGPUTargetLowering()
518 setOperationAction(ISD::FMUL, VT, Expand); in AMDGPUTargetLowering()
519 setOperationAction(ISD::FMA, VT, Expand); in AMDGPUTargetLowering()
520 setOperationAction(ISD::FRINT, VT, Expand); in AMDGPUTargetLowering()
521 setOperationAction(ISD::FNEARBYINT, VT, Expand); in AMDGPUTargetLowering()
522 setOperationAction(ISD::FSQRT, VT, Expand); in AMDGPUTargetLowering()
523 setOperationAction(ISD::FSIN, VT, Expand); in AMDGPUTargetLowering()
524 setOperationAction(ISD::FSUB, VT, Expand); in AMDGPUTargetLowering()
525 setOperationAction(ISD::FNEG, VT, Expand); in AMDGPUTargetLowering()
526 setOperationAction(ISD::VSELECT, VT, Expand); in AMDGPUTargetLowering()
527 setOperationAction(ISD::SELECT_CC, VT, Expand); in AMDGPUTargetLowering()
528 setOperationAction(ISD::FCOPYSIGN, VT, Expand); in AMDGPUTargetLowering()
529 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Expand); in AMDGPUTargetLowering()
530 setOperationAction(ISD::SETCC, VT, Expand); in AMDGPUTargetLowering()
531 setOperationAction(ISD::FCANONICALIZE, VT, Expand); in AMDGPUTargetLowering()
666 static bool opMustUseVOP3Encoding(const SDNode *N, MVT VT) { in opMustUseVOP3Encoding() argument
667 return N->getNumOperands() > 2 || VT == MVT::f64; in opMustUseVOP3Encoding()
717 MVT VT = N->getValueType(0).getScalarType().getSimpleVT(); in allUsesHaveSourceMods() local
724 if (!opMustUseVOP3Encoding(U, VT)) { in allUsesHaveSourceMods()
733 EVT AMDGPUTargetLowering::getTypeForExtReturn(LLVMContext &Context, EVT VT, in getTypeForExtReturn() argument
735 assert(!VT.isVector() && "only scalar expected"); in getTypeForExtReturn()
738 unsigned Size = VT.getSizeInBits(); in getTypeForExtReturn()
754 bool AMDGPUTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT, in isFPImmLegal() argument
756 EVT ScalarVT = VT.getScalarType(); in isFPImmLegal()
762 bool AMDGPUTargetLowering::ShouldShrinkFPConstant(EVT VT) const { in ShouldShrinkFPConstant()
763 EVT ScalarVT = VT.getScalarType(); in ShouldShrinkFPConstant()
886 bool AMDGPUTargetLowering::isFAbsFree(EVT VT) const { in isFAbsFree()
887 assert(VT.isFloatingPoint()); in isFAbsFree()
890 return VT == MVT::f32 || VT == MVT::f64 || in isFAbsFree()
891 (Subtarget->has16BitInsts() && VT == MVT::f16); in isFAbsFree()
894 bool AMDGPUTargetLowering::isFNegFree(EVT VT) const { in isFNegFree()
895 assert(VT.isFloatingPoint()); in isFNegFree()
897 VT = VT.getScalarType(); in isFNegFree()
898 return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16; in isFNegFree()
1256 InVals.push_back(DAG.getUNDEF(CLI.Ins[I].VT)); in lowerUnhandledCall()
1398 EVT VT = Op.getValueType(); in LowerCONCAT_VECTORS() local
1399 if (VT == MVT::v4i16 || VT == MVT::v4f16) { in LowerCONCAT_VECTORS()
1405 return DAG.getNode(ISD::BITCAST, SL, VT, BV); in LowerCONCAT_VECTORS()
1419 EVT VT = Op.getValueType(); in LowerEXTRACT_SUBVECTOR() local
1423 if (((SrcVT == MVT::v4f16 && VT == MVT::v2f16) || in LowerEXTRACT_SUBVECTOR()
1424 (SrcVT == MVT::v4i16 && VT == MVT::v2i16)) && in LowerEXTRACT_SUBVECTOR()
1429 VT.getVectorNumElements()); in LowerEXTRACT_SUBVECTOR()
1435 SDValue AMDGPUTargetLowering::combineFMinMaxLegacy(const SDLoc &DL, EVT VT, in combineFMinMaxLegacy() argument
1462 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, RHS, LHS); in combineFMinMaxLegacy()
1463 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, LHS, RHS); in combineFMinMaxLegacy()
1481 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, LHS, RHS); in combineFMinMaxLegacy()
1482 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, RHS, LHS); in combineFMinMaxLegacy()
1487 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, RHS, LHS); in combineFMinMaxLegacy()
1488 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, LHS, RHS); in combineFMinMaxLegacy()
1499 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, LHS, RHS); in combineFMinMaxLegacy()
1500 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, RHS, LHS); in combineFMinMaxLegacy()
1543 AMDGPUTargetLowering::getSplitDestVTs(const EVT &VT, SelectionDAG &DAG) const { in getSplitDestVTs() argument
1545 EVT EltVT = VT.getVectorElementType(); in getSplitDestVTs()
1546 unsigned NumElts = VT.getVectorNumElements(); in getSplitDestVTs()
1576 EVT VT = Op.getValueType(); in SplitVectorLoad() local
1582 if (VT.getVectorNumElements() == 2) { in SplitVectorLoad()
1597 std::tie(LoVT, HiVT) = getSplitDestVTs(VT, DAG); in SplitVectorLoad()
1617 Join = DAG.getNode(ISD::CONCAT_VECTORS, SL, VT, LoLoad, HiLoad); in SplitVectorLoad()
1619 Join = DAG.getNode(ISD::INSERT_SUBVECTOR, SL, VT, DAG.getUNDEF(VT), LoLoad, in SplitVectorLoad()
1623 VT, Join, HiLoad, in SplitVectorLoad()
1636 EVT VT = Op.getValueType(); in WidenOrSplitVectorLoad() local
1654 EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), 4); in WidenOrSplitVectorLoad()
1661 {DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, VT, WideLoad, in WidenOrSplitVectorLoad()
1671 EVT VT = Val.getValueType(); in SplitVectorStore() local
1675 if (VT.getVectorNumElements() == 2) in SplitVectorStore()
1687 std::tie(LoVT, HiVT) = getSplitDestVTs(VT, DAG); in SplitVectorStore()
1714 EVT VT = Op.getValueType(); in LowerDIVREM24() local
1728 unsigned BitSize = VT.getSizeInBits(); in LowerDIVREM24()
1741 jq = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS); in LowerDIVREM24()
1744 jq = DAG.getNode(ISD::SRA, DL, VT, jq, in LowerDIVREM24()
1745 DAG.getConstant(BitSize - 2, DL, VT)); in LowerDIVREM24()
1748 jq = DAG.getNode(ISD::OR, DL, VT, jq, DAG.getConstant(1, DL, VT)); in LowerDIVREM24()
1792 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in LowerDIVREM24()
1798 jq = DAG.getNode(ISD::SELECT, DL, VT, cv, jq, DAG.getConstant(0, DL, VT)); in LowerDIVREM24()
1801 SDValue Div = DAG.getNode(ISD::ADD, DL, VT, iq, jq); in LowerDIVREM24()
1804 SDValue Rem = DAG.getNode(ISD::MUL, DL, VT, Div, RHS); in LowerDIVREM24()
1805 Rem = DAG.getNode(ISD::SUB, DL, VT, LHS, Rem); in LowerDIVREM24()
1811 Div = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Div, InRegSize); in LowerDIVREM24()
1812 Rem = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Rem, InRegSize); in LowerDIVREM24()
1814 SDValue TruncMask = DAG.getConstant((UINT64_C(1) << DivBits) - 1, DL, VT); in LowerDIVREM24()
1815 Div = DAG.getNode(ISD::AND, DL, VT, Div, TruncMask); in LowerDIVREM24()
1816 Rem = DAG.getNode(ISD::AND, DL, VT, Rem, TruncMask); in LowerDIVREM24()
1826 EVT VT = Op.getValueType(); in LowerUDIVREM64() local
1828 assert(VT == MVT::i64 && "LowerUDIVREM64 expects an i64"); in LowerUDIVREM64()
1830 EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext()); in LowerUDIVREM64()
1885 SDValue Rcp64 = DAG.getBitcast(VT, in LowerUDIVREM64()
1888 SDValue Zero64 = DAG.getConstant(0, DL, VT); in LowerUDIVREM64()
1889 SDValue One64 = DAG.getConstant(1, DL, VT); in LowerUDIVREM64()
1893 SDValue Neg_RHS = DAG.getNode(ISD::SUB, DL, VT, Zero64, RHS); in LowerUDIVREM64()
1894 SDValue Mullo1 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Rcp64); in LowerUDIVREM64()
1895 SDValue Mulhi1 = DAG.getNode(ISD::MULHU, DL, VT, Rcp64, Mullo1); in LowerUDIVREM64()
1906 SDValue Add1 = DAG.getBitcast(VT, in LowerUDIVREM64()
1909 SDValue Mullo2 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Add1); in LowerUDIVREM64()
1910 SDValue Mulhi2 = DAG.getNode(ISD::MULHU, DL, VT, Add1, Mullo2); in LowerUDIVREM64()
1922 SDValue Add2 = DAG.getBitcast(VT, in LowerUDIVREM64()
1924 SDValue Mulhi3 = DAG.getNode(ISD::MULHU, DL, VT, LHS, Add2); in LowerUDIVREM64()
1926 SDValue Mul3 = DAG.getNode(ISD::MUL, DL, VT, RHS, Mulhi3); in LowerUDIVREM64()
1935 SDValue Sub1 = DAG.getBitcast(VT, in LowerUDIVREM64()
1956 SDValue Sub2 = DAG.getBitcast(VT, in LowerUDIVREM64()
1959 SDValue Add3 = DAG.getNode(ISD::ADD, DL, VT, Mulhi3, One64); in LowerUDIVREM64()
1968 SDValue Add4 = DAG.getNode(ISD::ADD, DL, VT, Add3, One64); in LowerUDIVREM64()
1976 SDValue Sub3 = DAG.getBitcast(VT, in LowerUDIVREM64()
2014 HBit = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, HBit); in LowerUDIVREM64()
2017 REM = DAG.getNode(ISD::SHL, DL, VT, REM, DAG.getConstant(1, DL, VT)); in LowerUDIVREM64()
2019 REM = DAG.getNode(ISD::OR, DL, VT, REM, HBit); in LowerUDIVREM64()
2027 SDValue REM_sub = DAG.getNode(ISD::SUB, DL, VT, REM, RHS); in LowerUDIVREM64()
2040 EVT VT = Op.getValueType(); in LowerUDIVREM() local
2042 if (VT == MVT::i64) { in LowerUDIVREM()
2048 if (VT == MVT::i32) { in LowerUDIVREM()
2060 SDValue Z = DAG.getNode(AMDGPUISD::URECIP, DL, VT, Y); in LowerUDIVREM()
2063 SDValue NegY = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Y); in LowerUDIVREM()
2064 SDValue NegYZ = DAG.getNode(ISD::MUL, DL, VT, NegY, Z); in LowerUDIVREM()
2065 Z = DAG.getNode(ISD::ADD, DL, VT, Z, in LowerUDIVREM()
2066 DAG.getNode(ISD::MULHU, DL, VT, Z, NegYZ)); in LowerUDIVREM()
2069 SDValue Q = DAG.getNode(ISD::MULHU, DL, VT, X, Z); in LowerUDIVREM()
2071 DAG.getNode(ISD::SUB, DL, VT, X, DAG.getNode(ISD::MUL, DL, VT, Q, Y)); in LowerUDIVREM()
2074 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in LowerUDIVREM()
2075 SDValue One = DAG.getConstant(1, DL, VT); in LowerUDIVREM()
2077 Q = DAG.getNode(ISD::SELECT, DL, VT, Cond, in LowerUDIVREM()
2078 DAG.getNode(ISD::ADD, DL, VT, Q, One), Q); in LowerUDIVREM()
2079 R = DAG.getNode(ISD::SELECT, DL, VT, Cond, in LowerUDIVREM()
2080 DAG.getNode(ISD::SUB, DL, VT, R, Y), R); in LowerUDIVREM()
2084 Q = DAG.getNode(ISD::SELECT, DL, VT, Cond, in LowerUDIVREM()
2085 DAG.getNode(ISD::ADD, DL, VT, Q, One), Q); in LowerUDIVREM()
2086 R = DAG.getNode(ISD::SELECT, DL, VT, Cond, in LowerUDIVREM()
2087 DAG.getNode(ISD::SUB, DL, VT, R, Y), R); in LowerUDIVREM()
2095 EVT VT = Op.getValueType(); in LowerSDIVREM() local
2100 SDValue Zero = DAG.getConstant(0, DL, VT); in LowerSDIVREM()
2101 SDValue NegOne = DAG.getConstant(-1, DL, VT); in LowerSDIVREM()
2103 if (VT == MVT::i32) { in LowerSDIVREM()
2108 if (VT == MVT::i64 && in LowerSDIVREM()
2111 EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext()); in LowerSDIVREM()
2119 DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(0)), in LowerSDIVREM()
2120 DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(1)) in LowerSDIVREM()
2127 SDValue DSign = DAG.getNode(ISD::XOR, DL, VT, LHSign, RHSign); in LowerSDIVREM()
2130 LHS = DAG.getNode(ISD::ADD, DL, VT, LHS, LHSign); in LowerSDIVREM()
2131 RHS = DAG.getNode(ISD::ADD, DL, VT, RHS, RHSign); in LowerSDIVREM()
2133 LHS = DAG.getNode(ISD::XOR, DL, VT, LHS, LHSign); in LowerSDIVREM()
2134 RHS = DAG.getNode(ISD::XOR, DL, VT, RHS, RHSign); in LowerSDIVREM()
2136 SDValue Div = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT), LHS, RHS); in LowerSDIVREM()
2139 Div = DAG.getNode(ISD::XOR, DL, VT, Div, DSign); in LowerSDIVREM()
2140 Rem = DAG.getNode(ISD::XOR, DL, VT, Rem, RSign); in LowerSDIVREM()
2142 Div = DAG.getNode(ISD::SUB, DL, VT, Div, DSign); in LowerSDIVREM()
2143 Rem = DAG.getNode(ISD::SUB, DL, VT, Rem, RSign); in LowerSDIVREM()
2155 EVT VT = Op.getValueType(); in LowerFREM() local
2160 SDValue Div = DAG.getNode(ISD::FDIV, SL, VT, X, Y, Flags); in LowerFREM()
2161 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, VT, Div, Flags); in LowerFREM()
2162 SDValue Neg = DAG.getNode(ISD::FNEG, SL, VT, Trunc, Flags); in LowerFREM()
2164 return DAG.getNode(ISD::FMA, SL, VT, Neg, Y, X, Flags); in LowerFREM()
2298 EVT VT = Op.getValueType(); in LowerFROUND() local
2300 SDValue T = DAG.getNode(ISD::FTRUNC, SL, VT, X); in LowerFROUND()
2304 SDValue Diff = DAG.getNode(ISD::FSUB, SL, VT, X, T); in LowerFROUND()
2306 SDValue AbsDiff = DAG.getNode(ISD::FABS, SL, VT, Diff); in LowerFROUND()
2308 const SDValue Zero = DAG.getConstantFP(0.0, SL, VT); in LowerFROUND()
2309 const SDValue One = DAG.getConstantFP(1.0, SL, VT); in LowerFROUND()
2310 const SDValue Half = DAG.getConstantFP(0.5, SL, VT); in LowerFROUND()
2312 SDValue SignOne = DAG.getNode(ISD::FCOPYSIGN, SL, VT, One, X); in LowerFROUND()
2315 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in LowerFROUND()
2319 SDValue Sel = DAG.getNode(ISD::SELECT, SL, VT, Cmp, SignOne, Zero); in LowerFROUND()
2321 return DAG.getNode(ISD::FADD, SL, VT, T, Sel); in LowerFROUND()
2351 EVT VT = Op.getValueType(); in LowerFLOG() local
2355 SDValue Log2Operand = DAG.getNode(ISD::FLOG2, SL, VT, Operand); in LowerFLOG()
2356 SDValue Log2BaseInvertedOperand = DAG.getConstantFP(Log2BaseInverted, SL, VT); in LowerFLOG()
2358 return DAG.getNode(ISD::FMUL, SL, VT, Log2Operand, Log2BaseInvertedOperand); in LowerFLOG()
2363 EVT VT = Op.getValueType(); in lowerFEXP() local
2367 const SDValue K = DAG.getConstantFP(numbers::log2e, SL, VT); in lowerFEXP()
2368 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, Src, K, Op->getFlags()); in lowerFEXP()
2369 return DAG.getNode(ISD::FEXP2, SL, VT, Mul, Op->getFlags()); in lowerFEXP()
2849 MVT VT = Op.getSimpleValueType(); in LowerSIGN_EXTEND_INREG() local
2850 MVT ScalarVT = VT.getScalarType(); in LowerSIGN_EXTEND_INREG()
2852 assert(VT.isVector()); in LowerSIGN_EXTEND_INREG()
2858 unsigned NElts = VT.getVectorNumElements(); in LowerSIGN_EXTEND_INREG()
2866 return DAG.getBuildVector(VT, DL, Args); in LowerSIGN_EXTEND_INREG()
2878 EVT VT = Op.getValueType(); in isI24() local
2879 return VT.getSizeInBits() >= 24 && // Types less than 24-bit should be treated in isI24()
2944 bool AMDGPUTargetLowering::shouldCombineMemoryType(EVT VT) const { in shouldCombineMemoryType()
2946 if (VT.getScalarType() == MVT::i32 || isTypeLegal(VT)) in shouldCombineMemoryType()
2949 if (!VT.isByteSized()) in shouldCombineMemoryType()
2952 unsigned Size = VT.getStoreSize(); in shouldCombineMemoryType()
2954 if ((Size == 1 || Size == 2 || Size == 4) && !VT.isVector()) in shouldCombineMemoryType()
2976 EVT VT = LN->getMemoryVT(); in performLoadCombine() local
2978 unsigned Size = VT.getStoreSize(); in performLoadCombine()
2980 if (Alignment < Size && isTypeLegal(VT)) { in performLoadCombine()
2988 VT, AS, Alignment, LN->getMemOperand()->getFlags(), &IsFast)) { in performLoadCombine()
2991 if (VT.isVector()) in performLoadCombine()
3003 if (!shouldCombineMemoryType(VT)) in performLoadCombine()
3006 EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT); in performLoadCombine()
3012 SDValue BC = DAG.getNode(ISD::BITCAST, SL, VT, NewLoad); in performLoadCombine()
3028 EVT VT = SN->getMemoryVT(); in performStoreCombine() local
3029 unsigned Size = VT.getStoreSize(); in performStoreCombine()
3034 if (Alignment < Size && isTypeLegal(VT)) { in performStoreCombine()
3043 VT, AS, Alignment, SN->getMemOperand()->getFlags(), &IsFast)) { in performStoreCombine()
3044 if (VT.isVector()) in performStoreCombine()
3054 if (!shouldCombineMemoryType(VT)) in performStoreCombine()
3057 EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT); in performStoreCombine()
3065 SDValue CastBack = DAG.getNode(ISD::BITCAST, SL, VT, CastVal); in performStoreCombine()
3148 EVT VT = N->getValueType(0); in performShlCombine() local
3170 if (VT == MVT::i32 && RHSVal == 16 && X.getValueType() == MVT::i16 && in performShlCombine()
3180 if (VT != MVT::i64) in performShlCombine()
3188 return DAG.getZExtOrTrunc(Shl, SL, VT); in performShlCombine()
3192 if (VT != MVT::i64) in performShlCombine()
3255 EVT VT = N->getValueType(0); in performSrlCombine() local
3268 ISD::AND, SL, VT, in performSrlCombine()
3269 DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(0), N->getOperand(1)), in performSrlCombine()
3270 DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(1), N->getOperand(1))); in performSrlCombine()
3275 if (VT != MVT::i64) in performSrlCombine()
3302 EVT VT = N->getValueType(0); in performTruncateCombine() local
3306 if (Src.getOpcode() == ISD::BITCAST && !VT.isVector()) { in performTruncateCombine()
3311 if (VT.getFixedSizeInBits() <= EltVT.getFixedSizeInBits()) { in performTruncateCombine()
3317 return DAG.getNode(ISD::TRUNCATE, SL, VT, Elt0); in performTruncateCombine()
3325 if (Src.getOpcode() == ISD::SRL && !VT.isVector()) { in performTruncateCombine()
3338 return DAG.getNode(ISD::TRUNCATE, SL, VT, SrcElt); in performTruncateCombine()
3348 if (VT.getScalarSizeInBits() < 32) { in performTruncateCombine()
3356 unsigned Size = VT.getScalarSizeInBits(); in performTruncateCombine()
3359 EVT MidVT = VT.isVector() ? in performTruncateCombine()
3361 VT.getVectorNumElements()) : MVT::i32; in performTruncateCombine()
3375 return DAG.getNode(ISD::TRUNCATE, SL, VT, ShrunkShift); in performTruncateCombine()
3405 EVT VT = N->getValueType(0); in performMulCombine() local
3414 unsigned Size = VT.getSizeInBits(); in performMulCombine()
3415 if (VT.isVector() || Size > 64) in performMulCombine()
3419 if (Subtarget->has16BitInsts() && VT.getScalarType().bitsLE(MVT::i16)) in performMulCombine()
3454 return DAG.getSExtOrTrunc(Mul, DL, VT); in performMulCombine()
3459 EVT VT = N->getValueType(0); in performMulhsCombine() local
3461 if (!Subtarget->hasMulI24() || VT.isVector()) in performMulhsCombine()
3487 return DAG.getSExtOrTrunc(Mulhi, DL, VT); in performMulhsCombine()
3492 EVT VT = N->getValueType(0); in performMulhuCombine() local
3494 if (!Subtarget->hasMulU24() || VT.isVector() || VT.getSizeInBits() > 32) in performMulhuCombine()
3520 return DAG.getZExtOrTrunc(Mulhi, DL, VT); in performMulhuCombine()
3533 EVT VT = Op.getValueType(); in getFFBX_U32() local
3534 EVT LegalVT = getTypeToTransformTo(*DAG.getContext(), VT); in getFFBX_U32()
3539 if (VT != MVT::i32) in getFFBX_U32()
3543 if (VT != MVT::i32) in getFFBX_U32()
3544 FFBX = DAG.getNode(ISD::TRUNCATE, DL, VT, FFBX); in getFFBX_U32()
3598 EVT VT = N1.getValueType(); in distributeOpThroughSelect() local
3600 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, Cond, in distributeOpThroughSelect()
3603 return DAG.getNode(Op, SL, VT, NewSelect); in distributeOpThroughSelect()
3620 EVT VT = N.getValueType(); in foldFreeOpFromSelect() local
3655 NewRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in foldFreeOpFromSelect()
3662 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, in foldFreeOpFromSelect()
3665 return DAG.getNode(LHS.getOpcode(), SL, VT, NewSelect); in foldFreeOpFromSelect()
3682 EVT VT = N->getValueType(0); in performSelectCombine() local
3703 return DAG.getNode(ISD::SELECT, SL, VT, NewCond, False, True); in performSelectCombine()
3706 if (VT == MVT::f32 && Subtarget->hasFminFmaxLegacy()) { in performSelectCombine()
3708 = combineFMinMaxLegacy(SDLoc(N), VT, LHS, RHS, True, False, CC, DCI); in performSelectCombine()
3766 EVT VT = N->getValueType(0); in performFNegCombine() local
3796 LHS = DAG.getNode(ISD::FNEG, SL, VT, LHS); in performFNegCombine()
3801 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFNegCombine()
3805 SDValue Res = DAG.getNode(ISD::FADD, SL, VT, LHS, RHS, N0->getFlags()); in performFNegCombine()
3809 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); in performFNegCombine()
3824 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFNegCombine()
3826 SDValue Res = DAG.getNode(Opc, SL, VT, LHS, RHS, N0->getFlags()); in performFNegCombine()
3830 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); in performFNegCombine()
3849 MHS = DAG.getNode(ISD::FNEG, SL, VT, MHS); in performFNegCombine()
3852 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFNegCombine()
3856 SDValue Res = DAG.getNode(Opc, SL, VT, LHS, MHS, RHS); in performFNegCombine()
3860 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); in performFNegCombine()
3882 SDValue NegLHS = DAG.getNode(ISD::FNEG, SL, VT, LHS); in performFNegCombine()
3883 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFNegCombine()
3886 SDValue Res = DAG.getNode(Opposite, SL, VT, NegLHS, NegRHS, N0->getFlags()); in performFNegCombine()
3890 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); in performFNegCombine()
3896 Ops[I] = DAG.getNode(ISD::FNEG, SL, VT, N0->getOperand(I), N0->getFlags()); in performFNegCombine()
3898 SDValue Res = DAG.getNode(AMDGPUISD::FMED3, SL, VT, Ops, N0->getFlags()); in performFNegCombine()
3903 SDValue Neg = DAG.getNode(ISD::FNEG, SL, VT, Res); in performFNegCombine()
3926 return DAG.getNode(Opc, SL, VT, CvtSrc.getOperand(0)); in performFNegCombine()
3935 return DAG.getNode(Opc, SL, VT, Neg, N0->getFlags()); in performFNegCombine()
3942 return DAG.getNode(ISD::FP_ROUND, SL, VT, in performFNegCombine()
3951 return DAG.getNode(ISD::FP_ROUND, SL, VT, Neg, N0.getOperand(1)); in performFNegCombine()
4219 Register Reg, EVT VT, in CreateLiveInRegister() argument
4234 return DAG.getRegister(VReg, VT); in CreateLiveInRegister()
4236 return DAG.getCopyFromReg(DAG.getEntryNode(), SL, VReg, VT); in CreateLiveInRegister()
4254 EVT VT, in loadStackInputValue() argument
4259 int FI = getOrCreateFixedStackObject(MFI, VT.getStoreSize(), Offset); in loadStackInputValue()
4264 return DAG.getLoad(VT, SL, DAG.getEntryNode(), Ptr, SrcPtrInfo, Align(4), in loadStackInputValue()
4290 EVT VT, const SDLoc &SL, in loadInputValue() argument
4295 CreateLiveInRegister(DAG, RC, Arg.getRegister(), VT, SL) : in loadInputValue()
4296 loadStackInputValue(DAG, VT, SL, Arg.getStackOffset()); in loadInputValue()
4303 V = DAG.getNode(ISD::SRL, SL, VT, V, in loadInputValue()
4304 DAG.getShiftAmountConstant(Shift, VT, SL)); in loadInputValue()
4305 return DAG.getNode(ISD::AND, SL, VT, V, in loadInputValue()
4306 DAG.getConstant(Mask >> Shift, SL, VT)); in loadInputValue()
4484 EVT VT = Operand.getValueType(); in getSqrtEstimate() local
4486 if (VT == MVT::f32) { in getSqrtEstimate()
4488 return DAG.getNode(AMDGPUISD::RSQ, SDLoc(Operand), VT, Operand); in getSqrtEstimate()
4500 EVT VT = Operand.getValueType(); in getRecipEstimate() local
4502 if (VT == MVT::f32) { in getRecipEstimate()
4509 return DAG.getNode(AMDGPUISD::RCP, SDLoc(Operand), VT, Operand); in getRecipEstimate()