Lines Matching refs:VT
121 EVT AMDGPUTargetLowering::getEquivalentMemType(LLVMContext &Ctx, EVT VT) { in getEquivalentMemType() argument
122 unsigned StoreSize = VT.getStoreSizeInBits(); in getEquivalentMemType()
131 EVT VT = Op.getValueType(); in numBitsUnsigned() local
133 return VT.getSizeInBits() - Known.countMinLeadingZeros(); in numBitsUnsigned()
137 EVT VT = Op.getValueType(); in numBitsSigned() local
141 return VT.getSizeInBits() - DAG.ComputeNumSignBits(Op); in numBitsSigned()
178 for (MVT VT : MVT::integer_valuetypes()) { in AMDGPUTargetLowering() local
179 setLoadExtAction(ISD::EXTLOAD, MVT::i64, VT, Expand); in AMDGPUTargetLowering()
180 setLoadExtAction(ISD::SEXTLOAD, MVT::i64, VT, Expand); in AMDGPUTargetLowering()
181 setLoadExtAction(ISD::ZEXTLOAD, MVT::i64, VT, Expand); in AMDGPUTargetLowering()
184 for (MVT VT : MVT::integer_valuetypes()) { in AMDGPUTargetLowering() local
185 if (VT == MVT::i64) in AMDGPUTargetLowering()
188 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); in AMDGPUTargetLowering()
189 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i8, Legal); in AMDGPUTargetLowering()
190 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i16, Legal); in AMDGPUTargetLowering()
191 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i32, Expand); in AMDGPUTargetLowering()
193 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote); in AMDGPUTargetLowering()
194 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i8, Legal); in AMDGPUTargetLowering()
195 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i16, Legal); in AMDGPUTargetLowering()
196 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i32, Expand); in AMDGPUTargetLowering()
198 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i1, Promote); in AMDGPUTargetLowering()
199 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i8, Legal); in AMDGPUTargetLowering()
200 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i16, Legal); in AMDGPUTargetLowering()
201 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i32, Expand); in AMDGPUTargetLowering()
204 for (MVT VT : MVT::integer_vector_valuetypes()) { in AMDGPUTargetLowering() local
205 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i8, Expand); in AMDGPUTargetLowering()
206 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i8, Expand); in AMDGPUTargetLowering()
207 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v2i8, Expand); in AMDGPUTargetLowering()
208 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i8, Expand); in AMDGPUTargetLowering()
209 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i8, Expand); in AMDGPUTargetLowering()
210 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v4i8, Expand); in AMDGPUTargetLowering()
211 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i16, Expand); in AMDGPUTargetLowering()
212 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i16, Expand); in AMDGPUTargetLowering()
213 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v2i16, Expand); in AMDGPUTargetLowering()
214 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i16, Expand); in AMDGPUTargetLowering()
215 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i16, Expand); in AMDGPUTargetLowering()
216 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v4i16, Expand); in AMDGPUTargetLowering()
346 for (MVT VT : ScalarIntVTs) { in AMDGPUTargetLowering() local
348 setOperationAction(ISD::SDIV, VT, Expand); in AMDGPUTargetLowering()
349 setOperationAction(ISD::UDIV, VT, Expand); in AMDGPUTargetLowering()
350 setOperationAction(ISD::SREM, VT, Expand); in AMDGPUTargetLowering()
351 setOperationAction(ISD::UREM, VT, Expand); in AMDGPUTargetLowering()
354 setOperationAction(ISD::SDIVREM, VT, Custom); in AMDGPUTargetLowering()
355 setOperationAction(ISD::UDIVREM, VT, Custom); in AMDGPUTargetLowering()
358 setOperationAction(ISD::SMUL_LOHI, VT, Expand); in AMDGPUTargetLowering()
359 setOperationAction(ISD::UMUL_LOHI, VT, Expand); in AMDGPUTargetLowering()
361 setOperationAction(ISD::BSWAP, VT, Expand); in AMDGPUTargetLowering()
362 setOperationAction(ISD::CTTZ, VT, Expand); in AMDGPUTargetLowering()
363 setOperationAction(ISD::CTLZ, VT, Expand); in AMDGPUTargetLowering()
366 setOperationAction(ISD::ADDC, VT, Legal); in AMDGPUTargetLowering()
367 setOperationAction(ISD::SUBC, VT, Legal); in AMDGPUTargetLowering()
368 setOperationAction(ISD::ADDE, VT, Legal); in AMDGPUTargetLowering()
369 setOperationAction(ISD::SUBE, VT, Legal); in AMDGPUTargetLowering()
400 for (MVT VT : VectorIntTypes) { in AMDGPUTargetLowering() local
402 setOperationAction(ISD::ADD, VT, Expand); in AMDGPUTargetLowering()
403 setOperationAction(ISD::AND, VT, Expand); in AMDGPUTargetLowering()
404 setOperationAction(ISD::FP_TO_SINT, VT, Expand); in AMDGPUTargetLowering()
405 setOperationAction(ISD::FP_TO_UINT, VT, Expand); in AMDGPUTargetLowering()
406 setOperationAction(ISD::MUL, VT, Expand); in AMDGPUTargetLowering()
407 setOperationAction(ISD::MULHU, VT, Expand); in AMDGPUTargetLowering()
408 setOperationAction(ISD::MULHS, VT, Expand); in AMDGPUTargetLowering()
409 setOperationAction(ISD::OR, VT, Expand); in AMDGPUTargetLowering()
410 setOperationAction(ISD::SHL, VT, Expand); in AMDGPUTargetLowering()
411 setOperationAction(ISD::SRA, VT, Expand); in AMDGPUTargetLowering()
412 setOperationAction(ISD::SRL, VT, Expand); in AMDGPUTargetLowering()
413 setOperationAction(ISD::ROTL, VT, Expand); in AMDGPUTargetLowering()
414 setOperationAction(ISD::ROTR, VT, Expand); in AMDGPUTargetLowering()
415 setOperationAction(ISD::SUB, VT, Expand); in AMDGPUTargetLowering()
416 setOperationAction(ISD::SINT_TO_FP, VT, Expand); in AMDGPUTargetLowering()
417 setOperationAction(ISD::UINT_TO_FP, VT, Expand); in AMDGPUTargetLowering()
418 setOperationAction(ISD::SDIV, VT, Expand); in AMDGPUTargetLowering()
419 setOperationAction(ISD::UDIV, VT, Expand); in AMDGPUTargetLowering()
420 setOperationAction(ISD::SREM, VT, Expand); in AMDGPUTargetLowering()
421 setOperationAction(ISD::UREM, VT, Expand); in AMDGPUTargetLowering()
422 setOperationAction(ISD::SMUL_LOHI, VT, Expand); in AMDGPUTargetLowering()
423 setOperationAction(ISD::UMUL_LOHI, VT, Expand); in AMDGPUTargetLowering()
424 setOperationAction(ISD::SDIVREM, VT, Custom); in AMDGPUTargetLowering()
425 setOperationAction(ISD::UDIVREM, VT, Expand); in AMDGPUTargetLowering()
426 setOperationAction(ISD::SELECT, VT, Expand); in AMDGPUTargetLowering()
427 setOperationAction(ISD::VSELECT, VT, Expand); in AMDGPUTargetLowering()
428 setOperationAction(ISD::SELECT_CC, VT, Expand); in AMDGPUTargetLowering()
429 setOperationAction(ISD::XOR, VT, Expand); in AMDGPUTargetLowering()
430 setOperationAction(ISD::BSWAP, VT, Expand); in AMDGPUTargetLowering()
431 setOperationAction(ISD::CTPOP, VT, Expand); in AMDGPUTargetLowering()
432 setOperationAction(ISD::CTTZ, VT, Expand); in AMDGPUTargetLowering()
433 setOperationAction(ISD::CTLZ, VT, Expand); in AMDGPUTargetLowering()
434 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Expand); in AMDGPUTargetLowering()
435 setOperationAction(ISD::SETCC, VT, Expand); in AMDGPUTargetLowering()
442 for (MVT VT : FloatVectorTypes) { in AMDGPUTargetLowering() local
443 setOperationAction(ISD::FABS, VT, Expand); in AMDGPUTargetLowering()
444 setOperationAction(ISD::FMINNUM, VT, Expand); in AMDGPUTargetLowering()
445 setOperationAction(ISD::FMAXNUM, VT, Expand); in AMDGPUTargetLowering()
446 setOperationAction(ISD::FADD, VT, Expand); in AMDGPUTargetLowering()
447 setOperationAction(ISD::FCEIL, VT, Expand); in AMDGPUTargetLowering()
448 setOperationAction(ISD::FCOS, VT, Expand); in AMDGPUTargetLowering()
449 setOperationAction(ISD::FDIV, VT, Expand); in AMDGPUTargetLowering()
450 setOperationAction(ISD::FEXP2, VT, Expand); in AMDGPUTargetLowering()
451 setOperationAction(ISD::FEXP, VT, Expand); in AMDGPUTargetLowering()
452 setOperationAction(ISD::FLOG2, VT, Expand); in AMDGPUTargetLowering()
453 setOperationAction(ISD::FREM, VT, Expand); in AMDGPUTargetLowering()
454 setOperationAction(ISD::FLOG, VT, Expand); in AMDGPUTargetLowering()
455 setOperationAction(ISD::FLOG10, VT, Expand); in AMDGPUTargetLowering()
456 setOperationAction(ISD::FPOW, VT, Expand); in AMDGPUTargetLowering()
457 setOperationAction(ISD::FFLOOR, VT, Expand); in AMDGPUTargetLowering()
458 setOperationAction(ISD::FTRUNC, VT, Expand); in AMDGPUTargetLowering()
459 setOperationAction(ISD::FMUL, VT, Expand); in AMDGPUTargetLowering()
460 setOperationAction(ISD::FMA, VT, Expand); in AMDGPUTargetLowering()
461 setOperationAction(ISD::FRINT, VT, Expand); in AMDGPUTargetLowering()
462 setOperationAction(ISD::FNEARBYINT, VT, Expand); in AMDGPUTargetLowering()
463 setOperationAction(ISD::FSQRT, VT, Expand); in AMDGPUTargetLowering()
464 setOperationAction(ISD::FSIN, VT, Expand); in AMDGPUTargetLowering()
465 setOperationAction(ISD::FSUB, VT, Expand); in AMDGPUTargetLowering()
466 setOperationAction(ISD::FNEG, VT, Expand); in AMDGPUTargetLowering()
467 setOperationAction(ISD::VSELECT, VT, Expand); in AMDGPUTargetLowering()
468 setOperationAction(ISD::SELECT_CC, VT, Expand); in AMDGPUTargetLowering()
469 setOperationAction(ISD::FCOPYSIGN, VT, Expand); in AMDGPUTargetLowering()
470 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Expand); in AMDGPUTargetLowering()
471 setOperationAction(ISD::SETCC, VT, Expand); in AMDGPUTargetLowering()
472 setOperationAction(ISD::FCANONICALIZE, VT, Expand); in AMDGPUTargetLowering()
578 static bool opMustUseVOP3Encoding(const SDNode *N, MVT VT) { in opMustUseVOP3Encoding() argument
579 return N->getNumOperands() > 2 || VT == MVT::f64; in opMustUseVOP3Encoding()
617 MVT VT = N->getValueType(0).getScalarType().getSimpleVT(); in allUsesHaveSourceMods() local
624 if (!opMustUseVOP3Encoding(U, VT)) { in allUsesHaveSourceMods()
643 bool AMDGPUTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { in isFPImmLegal()
644 EVT ScalarVT = VT.getScalarType(); in isFPImmLegal()
650 bool AMDGPUTargetLowering::ShouldShrinkFPConstant(EVT VT) const { in ShouldShrinkFPConstant()
651 EVT ScalarVT = VT.getScalarType(); in ShouldShrinkFPConstant()
755 bool AMDGPUTargetLowering::isFAbsFree(EVT VT) const { in isFAbsFree()
756 assert(VT.isFloatingPoint()); in isFAbsFree()
759 return VT == MVT::f32 || VT == MVT::f64 || in isFAbsFree()
760 (Subtarget->has16BitInsts() && VT == MVT::f16); in isFAbsFree()
763 bool AMDGPUTargetLowering::isFNegFree(EVT VT) const { in isFNegFree()
764 assert(VT.isFloatingPoint()); in isFNegFree()
765 return VT == MVT::f32 || VT == MVT::f64 || in isFNegFree()
766 (Subtarget->has16BitInsts() && VT == MVT::f16) || in isFNegFree()
767 (Subtarget->hasVOP3PInsts() && VT == MVT::v2f16); in isFNegFree()
1115 InVals.push_back(DAG.getUNDEF(CLI.Ins[I].VT)); in lowerUnhandledCall()
1242 EVT VT = Op.getValueType(); in LowerCONCAT_VECTORS() local
1243 if (VT == MVT::v4i16 || VT == MVT::v4f16) { in LowerCONCAT_VECTORS()
1249 return DAG.getNode(ISD::BITCAST, SL, VT, BV); in LowerCONCAT_VECTORS()
1263 EVT VT = Op.getValueType(); in LowerEXTRACT_SUBVECTOR() local
1265 VT.getVectorNumElements()); in LowerEXTRACT_SUBVECTOR()
1271 SDValue AMDGPUTargetLowering::combineFMinMaxLegacy(const SDLoc &DL, EVT VT, in combineFMinMaxLegacy() argument
1298 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, RHS, LHS); in combineFMinMaxLegacy()
1299 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, LHS, RHS); in combineFMinMaxLegacy()
1317 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, LHS, RHS); in combineFMinMaxLegacy()
1318 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, RHS, LHS); in combineFMinMaxLegacy()
1323 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, RHS, LHS); in combineFMinMaxLegacy()
1324 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, LHS, RHS); in combineFMinMaxLegacy()
1335 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, LHS, RHS); in combineFMinMaxLegacy()
1336 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, RHS, LHS); in combineFMinMaxLegacy()
1378 EVT VT = Op.getValueType(); in SplitVectorLoad() local
1383 if (VT.getVectorNumElements() == 2) in SplitVectorLoad()
1396 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); in SplitVectorLoad()
1414 DAG.getNode(ISD::CONCAT_VECTORS, SL, VT, LoLoad, HiLoad), in SplitVectorLoad()
1426 EVT VT = Val.getValueType(); in SplitVectorStore() local
1430 if (VT.getVectorNumElements() == 2) in SplitVectorStore()
1442 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); in SplitVectorStore()
1469 EVT VT = Op.getValueType(); in LowerDIVREM24() local
1483 unsigned BitSize = VT.getSizeInBits(); in LowerDIVREM24()
1496 jq = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS); in LowerDIVREM24()
1499 jq = DAG.getNode(ISD::SRA, DL, VT, jq, in LowerDIVREM24()
1500 DAG.getConstant(BitSize - 2, DL, VT)); in LowerDIVREM24()
1503 jq = DAG.getNode(ISD::OR, DL, VT, jq, DAG.getConstant(1, DL, VT)); in LowerDIVREM24()
1542 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in LowerDIVREM24()
1548 jq = DAG.getNode(ISD::SELECT, DL, VT, cv, jq, DAG.getConstant(0, DL, VT)); in LowerDIVREM24()
1551 SDValue Div = DAG.getNode(ISD::ADD, DL, VT, iq, jq); in LowerDIVREM24()
1554 SDValue Rem = DAG.getNode(ISD::MUL, DL, VT, Div, RHS); in LowerDIVREM24()
1555 Rem = DAG.getNode(ISD::SUB, DL, VT, LHS, Rem); in LowerDIVREM24()
1561 Div = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Div, InRegSize); in LowerDIVREM24()
1562 Rem = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Rem, InRegSize); in LowerDIVREM24()
1564 SDValue TruncMask = DAG.getConstant((UINT64_C(1) << DivBits) - 1, DL, VT); in LowerDIVREM24()
1565 Div = DAG.getNode(ISD::AND, DL, VT, Div, TruncMask); in LowerDIVREM24()
1566 Rem = DAG.getNode(ISD::AND, DL, VT, Rem, TruncMask); in LowerDIVREM24()
1576 EVT VT = Op.getValueType(); in LowerUDIVREM64() local
1578 assert(VT == MVT::i64 && "LowerUDIVREM64 expects an i64"); in LowerUDIVREM64()
1580 EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext()); in LowerUDIVREM64()
1630 SDValue Rcp64 = DAG.getBitcast(VT, in LowerUDIVREM64()
1633 SDValue Zero64 = DAG.getConstant(0, DL, VT); in LowerUDIVREM64()
1634 SDValue One64 = DAG.getConstant(1, DL, VT); in LowerUDIVREM64()
1638 SDValue Neg_RHS = DAG.getNode(ISD::SUB, DL, VT, Zero64, RHS); in LowerUDIVREM64()
1639 SDValue Mullo1 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Rcp64); in LowerUDIVREM64()
1640 SDValue Mulhi1 = DAG.getNode(ISD::MULHU, DL, VT, Rcp64, Mullo1); in LowerUDIVREM64()
1651 SDValue Add1 = DAG.getBitcast(VT, in LowerUDIVREM64()
1654 SDValue Mullo2 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Add1); in LowerUDIVREM64()
1655 SDValue Mulhi2 = DAG.getNode(ISD::MULHU, DL, VT, Add1, Mullo2); in LowerUDIVREM64()
1667 SDValue Add2 = DAG.getBitcast(VT, in LowerUDIVREM64()
1669 SDValue Mulhi3 = DAG.getNode(ISD::MULHU, DL, VT, LHS, Add2); in LowerUDIVREM64()
1671 SDValue Mul3 = DAG.getNode(ISD::MUL, DL, VT, RHS, Mulhi3); in LowerUDIVREM64()
1680 SDValue Sub1 = DAG.getBitcast(VT, in LowerUDIVREM64()
1701 SDValue Sub2 = DAG.getBitcast(VT, in LowerUDIVREM64()
1704 SDValue Add3 = DAG.getNode(ISD::ADD, DL, VT, Mulhi3, One64); in LowerUDIVREM64()
1713 SDValue Add4 = DAG.getNode(ISD::ADD, DL, VT, Add3, One64); in LowerUDIVREM64()
1721 SDValue Sub3 = DAG.getBitcast(VT, in LowerUDIVREM64()
1759 HBit = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, HBit); in LowerUDIVREM64()
1762 REM = DAG.getNode(ISD::SHL, DL, VT, REM, DAG.getConstant(1, DL, VT)); in LowerUDIVREM64()
1764 REM = DAG.getNode(ISD::OR, DL, VT, REM, HBit); in LowerUDIVREM64()
1772 SDValue REM_sub = DAG.getNode(ISD::SUB, DL, VT, REM, RHS); in LowerUDIVREM64()
1785 EVT VT = Op.getValueType(); in LowerUDIVREM() local
1787 if (VT == MVT::i64) { in LowerUDIVREM()
1793 if (VT == MVT::i32) { in LowerUDIVREM()
1803 SDValue RCP = DAG.getNode(AMDGPUISD::URECIP, DL, VT, Den); in LowerUDIVREM()
1806 SDValue RCP_LO = DAG.getNode(ISD::MUL, DL, VT, RCP, Den); in LowerUDIVREM()
1809 SDValue RCP_HI = DAG.getNode(ISD::MULHU, DL, VT, RCP, Den); in LowerUDIVREM()
1812 SDValue NEG_RCP_LO = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in LowerUDIVREM()
1816 SDValue ABS_RCP_LO = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, DL, VT), in LowerUDIVREM()
1821 SDValue E = DAG.getNode(ISD::MULHU, DL, VT, ABS_RCP_LO, RCP); in LowerUDIVREM()
1824 SDValue RCP_A_E = DAG.getNode(ISD::ADD, DL, VT, RCP, E); in LowerUDIVREM()
1827 SDValue RCP_S_E = DAG.getNode(ISD::SUB, DL, VT, RCP, E); in LowerUDIVREM()
1830 SDValue Tmp0 = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, DL, VT), in LowerUDIVREM()
1834 SDValue Quotient = DAG.getNode(ISD::MULHU, DL, VT, Tmp0, Num); in LowerUDIVREM()
1837 SDValue Num_S_Remainder = DAG.getNode(ISD::MUL, DL, VT, Quotient, Den); in LowerUDIVREM()
1840 SDValue Remainder = DAG.getNode(ISD::SUB, DL, VT, Num, Num_S_Remainder); in LowerUDIVREM()
1844 DAG.getConstant(-1, DL, VT), in LowerUDIVREM()
1845 DAG.getConstant(0, DL, VT), in LowerUDIVREM()
1850 DAG.getConstant(-1, DL, VT), in LowerUDIVREM()
1851 DAG.getConstant(0, DL, VT), in LowerUDIVREM()
1854 SDValue Tmp1 = DAG.getNode(ISD::AND, DL, VT, Remainder_GE_Den, in LowerUDIVREM()
1860 SDValue Quotient_A_One = DAG.getNode(ISD::ADD, DL, VT, Quotient, in LowerUDIVREM()
1861 DAG.getConstant(1, DL, VT)); in LowerUDIVREM()
1864 SDValue Quotient_S_One = DAG.getNode(ISD::SUB, DL, VT, Quotient, in LowerUDIVREM()
1865 DAG.getConstant(1, DL, VT)); in LowerUDIVREM()
1868 SDValue Div = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, DL, VT), in LowerUDIVREM()
1872 Div = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, DL, VT), in LowerUDIVREM()
1878 SDValue Remainder_S_Den = DAG.getNode(ISD::SUB, DL, VT, Remainder, Den); in LowerUDIVREM()
1881 SDValue Remainder_A_Den = DAG.getNode(ISD::ADD, DL, VT, Remainder, Den); in LowerUDIVREM()
1884 SDValue Rem = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, DL, VT), in LowerUDIVREM()
1888 Rem = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, DL, VT), in LowerUDIVREM()
1900 EVT VT = Op.getValueType(); in LowerSDIVREM() local
1905 SDValue Zero = DAG.getConstant(0, DL, VT); in LowerSDIVREM()
1906 SDValue NegOne = DAG.getConstant(-1, DL, VT); in LowerSDIVREM()
1908 if (VT == MVT::i32) { in LowerSDIVREM()
1913 if (VT == MVT::i64 && in LowerSDIVREM()
1916 EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext()); in LowerSDIVREM()
1924 DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(0)), in LowerSDIVREM()
1925 DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(1)) in LowerSDIVREM()
1932 SDValue DSign = DAG.getNode(ISD::XOR, DL, VT, LHSign, RHSign); in LowerSDIVREM()
1935 LHS = DAG.getNode(ISD::ADD, DL, VT, LHS, LHSign); in LowerSDIVREM()
1936 RHS = DAG.getNode(ISD::ADD, DL, VT, RHS, RHSign); in LowerSDIVREM()
1938 LHS = DAG.getNode(ISD::XOR, DL, VT, LHS, LHSign); in LowerSDIVREM()
1939 RHS = DAG.getNode(ISD::XOR, DL, VT, RHS, RHSign); in LowerSDIVREM()
1941 SDValue Div = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT), LHS, RHS); in LowerSDIVREM()
1944 Div = DAG.getNode(ISD::XOR, DL, VT, Div, DSign); in LowerSDIVREM()
1945 Rem = DAG.getNode(ISD::XOR, DL, VT, Rem, RSign); in LowerSDIVREM()
1947 Div = DAG.getNode(ISD::SUB, DL, VT, Div, DSign); in LowerSDIVREM()
1948 Rem = DAG.getNode(ISD::SUB, DL, VT, Rem, RSign); in LowerSDIVREM()
1960 EVT VT = Op.getValueType(); in LowerFREM() local
1966 SDValue Div = DAG.getNode(ISD::FDIV, SL, VT, X, Y); in LowerFREM()
1967 SDValue Floor = DAG.getNode(ISD::FTRUNC, SL, VT, Div); in LowerFREM()
1968 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, Floor, Y); in LowerFREM()
1970 return DAG.getNode(ISD::FSUB, SL, VT, X, Mul); in LowerFREM()
2104 EVT VT = Op.getValueType(); in LowerFROUND32_16() local
2106 SDValue T = DAG.getNode(ISD::FTRUNC, SL, VT, X); in LowerFROUND32_16()
2110 SDValue Diff = DAG.getNode(ISD::FSUB, SL, VT, X, T); in LowerFROUND32_16()
2112 SDValue AbsDiff = DAG.getNode(ISD::FABS, SL, VT, Diff); in LowerFROUND32_16()
2114 const SDValue Zero = DAG.getConstantFP(0.0, SL, VT); in LowerFROUND32_16()
2115 const SDValue One = DAG.getConstantFP(1.0, SL, VT); in LowerFROUND32_16()
2116 const SDValue Half = DAG.getConstantFP(0.5, SL, VT); in LowerFROUND32_16()
2118 SDValue SignOne = DAG.getNode(ISD::FCOPYSIGN, SL, VT, One, X); in LowerFROUND32_16()
2121 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in LowerFROUND32_16()
2125 SDValue Sel = DAG.getNode(ISD::SELECT, SL, VT, Cmp, SignOne, Zero); in LowerFROUND32_16()
2127 return DAG.getNode(ISD::FADD, SL, VT, T, Sel); in LowerFROUND32_16()
2188 EVT VT = Op.getValueType(); in LowerFROUND() local
2190 if (VT == MVT::f32 || VT == MVT::f16) in LowerFROUND()
2193 if (VT == MVT::f64) in LowerFROUND()
2226 EVT VT = Op.getValueType(); in LowerFLOG() local
2230 SDValue Log2Operand = DAG.getNode(ISD::FLOG2, SL, VT, Operand); in LowerFLOG()
2231 SDValue Log2BaseInvertedOperand = DAG.getConstantFP(Log2BaseInverted, SL, VT); in LowerFLOG()
2233 return DAG.getNode(ISD::FMUL, SL, VT, Log2Operand, Log2BaseInvertedOperand); in LowerFLOG()
2237 static SDValue getLog2EVal(SelectionDAG &DAG, const SDLoc &SL, EVT VT) { in getLog2EVal() argument
2238 switch (VT.getScalarType().getSimpleVT().SimpleTy) { in getLog2EVal()
2240 return DAG.getConstantFP(1.44269504088896340735992468100189214f, SL, VT); in getLog2EVal()
2244 SL, VT); in getLog2EVal()
2247 APFloat(APFloat::IEEEdouble(), "0x1.71547652b82fep+0"), SL, VT); in getLog2EVal()
2255 EVT VT = Op.getValueType(); in lowerFEXP() local
2259 const SDValue K = getLog2EVal(DAG, SL, VT); in lowerFEXP()
2260 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, Src, K, Op->getFlags()); in lowerFEXP()
2261 return DAG.getNode(ISD::FEXP2, SL, VT, Mul, Op->getFlags()); in lowerFEXP()
2685 MVT VT = Op.getSimpleValueType(); in LowerSIGN_EXTEND_INREG() local
2686 MVT ScalarVT = VT.getScalarType(); in LowerSIGN_EXTEND_INREG()
2688 assert(VT.isVector()); in LowerSIGN_EXTEND_INREG()
2694 unsigned NElts = VT.getVectorNumElements(); in LowerSIGN_EXTEND_INREG()
2702 return DAG.getBuildVector(VT, DL, Args); in LowerSIGN_EXTEND_INREG()
2714 EVT VT = Op.getValueType(); in isI24() local
2715 return VT.getSizeInBits() >= 24 && // Types less than 24-bit should be treated in isI24()
2772 bool AMDGPUTargetLowering::shouldCombineMemoryType(EVT VT) const { in shouldCombineMemoryType()
2774 if (VT.getScalarType() == MVT::i32 || isTypeLegal(VT)) in shouldCombineMemoryType()
2777 if (!VT.isByteSized()) in shouldCombineMemoryType()
2780 unsigned Size = VT.getStoreSize(); in shouldCombineMemoryType()
2782 if ((Size == 1 || Size == 2 || Size == 4) && !VT.isVector()) in shouldCombineMemoryType()
2804 EVT VT = LN->getMemoryVT(); in performLoadCombine() local
2806 unsigned Size = VT.getStoreSize(); in performLoadCombine()
2808 if (Align < Size && isTypeLegal(VT)) { in performLoadCombine()
2815 if (!allowsMisalignedMemoryAccesses(VT, AS, Align, &IsFast)) { in performLoadCombine()
2816 if (VT.isVector()) in performLoadCombine()
2828 if (!shouldCombineMemoryType(VT)) in performLoadCombine()
2831 EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT); in performLoadCombine()
2837 SDValue BC = DAG.getNode(ISD::BITCAST, SL, VT, NewLoad); in performLoadCombine()
2853 EVT VT = SN->getMemoryVT(); in performStoreCombine() local
2854 unsigned Size = VT.getStoreSize(); in performStoreCombine()
2859 if (Align < Size && isTypeLegal(VT)) { in performStoreCombine()
2867 if (!allowsMisalignedMemoryAccesses(VT, AS, Align, &IsFast)) { in performStoreCombine()
2868 if (VT.isVector()) in performStoreCombine()
2878 if (!shouldCombineMemoryType(VT)) in performStoreCombine()
2881 EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT); in performStoreCombine()
2889 SDValue CastBack = DAG.getNode(ISD::BITCAST, SL, VT, CastVal); in performStoreCombine()
2949 EVT VT = N->getValueType(0); in performShlCombine() local
2971 if (VT == MVT::i32 && RHSVal == 16 && X.getValueType() == MVT::i16 && in performShlCombine()
2981 if (VT != MVT::i64) in performShlCombine()
2989 return DAG.getZExtOrTrunc(Shl, SL, VT); in performShlCombine()
2993 if (VT != MVT::i64) in performShlCombine()
3089 EVT VT = N->getValueType(0); in performTruncateCombine() local
3098 if (VT.getSizeInBits() <= EltVT.getSizeInBits()) { in performTruncateCombine()
3104 return DAG.getNode(ISD::TRUNCATE, SL, VT, Elt0); in performTruncateCombine()
3112 if (Src.getOpcode() == ISD::SRL && !VT.isVector()) { in performTruncateCombine()
3125 return DAG.getNode(ISD::TRUNCATE, SL, VT, SrcElt); in performTruncateCombine()
3135 if (VT.getScalarSizeInBits() < 32) { in performTruncateCombine()
3143 unsigned Size = VT.getScalarSizeInBits(); in performTruncateCombine()
3146 EVT MidVT = VT.isVector() ? in performTruncateCombine()
3148 VT.getVectorNumElements()) : MVT::i32; in performTruncateCombine()
3162 return DAG.getNode(ISD::TRUNCATE, SL, VT, ShrunkShift); in performTruncateCombine()
3196 EVT VT = N->getValueType(0); in performMulCombine() local
3198 unsigned Size = VT.getSizeInBits(); in performMulCombine()
3199 if (VT.isVector() || Size > 64) in performMulCombine()
3203 if (Subtarget->has16BitInsts() && VT.getScalarType().bitsLE(MVT::i16)) in performMulCombine()
3238 return DAG.getSExtOrTrunc(Mul, DL, VT); in performMulCombine()
3243 EVT VT = N->getValueType(0); in performMulhsCombine() local
3245 if (!Subtarget->hasMulI24() || VT.isVector()) in performMulhsCombine()
3262 return DAG.getSExtOrTrunc(Mulhi, DL, VT); in performMulhsCombine()
3267 EVT VT = N->getValueType(0); in performMulhuCombine() local
3269 if (!Subtarget->hasMulU24() || VT.isVector() || VT.getSizeInBits() > 32) in performMulhuCombine()
3286 return DAG.getZExtOrTrunc(Mulhi, DL, VT); in performMulhuCombine()
3322 EVT VT = Op.getValueType(); in getFFBX_U32() local
3323 EVT LegalVT = getTypeToTransformTo(*DAG.getContext(), VT); in getFFBX_U32()
3328 if (VT != MVT::i32) in getFFBX_U32()
3332 if (VT != MVT::i32) in getFFBX_U32()
3333 FFBX = DAG.getNode(ISD::TRUNCATE, DL, VT, FFBX); in getFFBX_U32()
3387 EVT VT = N1.getValueType(); in distributeOpThroughSelect() local
3389 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, Cond, in distributeOpThroughSelect()
3392 return DAG.getNode(Op, SL, VT, NewSelect); in distributeOpThroughSelect()
3409 EVT VT = N.getValueType(); in foldFreeOpFromSelect() local
3444 NewRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in foldFreeOpFromSelect()
3451 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, in foldFreeOpFromSelect()
3454 return DAG.getNode(LHS.getOpcode(), SL, VT, NewSelect); in foldFreeOpFromSelect()
3471 EVT VT = N->getValueType(0); in performSelectCombine() local
3494 return DAG.getNode(ISD::SELECT, SL, VT, NewCond, False, True); in performSelectCombine()
3497 if (VT == MVT::f32 && Subtarget->hasFminFmaxLegacy()) { in performSelectCombine()
3499 = combineFMinMaxLegacy(SDLoc(N), VT, LHS, RHS, True, False, CC, DCI); in performSelectCombine()
3557 EVT VT = N->getValueType(0); in performFNegCombine() local
3587 LHS = DAG.getNode(ISD::FNEG, SL, VT, LHS); in performFNegCombine()
3592 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFNegCombine()
3596 SDValue Res = DAG.getNode(ISD::FADD, SL, VT, LHS, RHS, N0->getFlags()); in performFNegCombine()
3598 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); in performFNegCombine()
3613 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFNegCombine()
3615 SDValue Res = DAG.getNode(Opc, SL, VT, LHS, RHS, N0->getFlags()); in performFNegCombine()
3617 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); in performFNegCombine()
3635 MHS = DAG.getNode(ISD::FNEG, SL, VT, MHS); in performFNegCombine()
3638 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFNegCombine()
3642 SDValue Res = DAG.getNode(Opc, SL, VT, LHS, MHS, RHS); in performFNegCombine()
3644 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); in performFNegCombine()
3666 SDValue NegLHS = DAG.getNode(ISD::FNEG, SL, VT, LHS); in performFNegCombine()
3667 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFNegCombine()
3670 SDValue Res = DAG.getNode(Opposite, SL, VT, NegLHS, NegRHS, N0->getFlags()); in performFNegCombine()
3672 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); in performFNegCombine()
3678 Ops[I] = DAG.getNode(ISD::FNEG, SL, VT, N0->getOperand(I), N0->getFlags()); in performFNegCombine()
3680 SDValue Res = DAG.getNode(AMDGPUISD::FMED3, SL, VT, Ops, N0->getFlags()); in performFNegCombine()
3682 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); in performFNegCombine()
3699 return DAG.getNode(Opc, SL, VT, CvtSrc.getOperand(0)); in performFNegCombine()
3708 return DAG.getNode(Opc, SL, VT, Neg, N0->getFlags()); in performFNegCombine()
3715 return DAG.getNode(ISD::FP_ROUND, SL, VT, in performFNegCombine()
3724 return DAG.getNode(ISD::FP_ROUND, SL, VT, Neg, N0.getOperand(1)); in performFNegCombine()
3998 unsigned Reg, EVT VT, in CreateLiveInRegister() argument
4013 return DAG.getRegister(VReg, VT); in CreateLiveInRegister()
4015 return DAG.getCopyFromReg(DAG.getEntryNode(), SL, VReg, VT); in CreateLiveInRegister()
4019 EVT VT, in loadStackInputValue() argument
4025 int FI = MFI.CreateFixedObject(VT.getStoreSize(), Offset, true); in loadStackInputValue()
4029 return DAG.getLoad(VT, SL, DAG.getEntryNode(), Ptr, SrcPtrInfo, 4, in loadStackInputValue()
4050 EVT VT, const SDLoc &SL, in loadInputValue() argument
4055 return CreateLiveInRegister(DAG, RC, Arg.getRegister(), VT, SL); in loadInputValue()
4056 return loadStackInputValue(DAG, VT, SL, Arg.getStackOffset()); in loadInputValue()
4231 EVT VT = Operand.getValueType(); in getSqrtEstimate() local
4233 if (VT == MVT::f32) { in getSqrtEstimate()
4235 return DAG.getNode(AMDGPUISD::RSQ, SDLoc(Operand), VT, Operand); in getSqrtEstimate()
4247 EVT VT = Operand.getValueType(); in getRecipEstimate() local
4249 if (VT == MVT::f32) { in getRecipEstimate()
4256 return DAG.getNode(AMDGPUISD::RCP, SDLoc(Operand), VT, Operand); in getRecipEstimate()