Lines Matching refs:S32
516 const LLT S32 = LLT::scalar(32); in castBufferRsrcFromV4I32() local
523 B.buildExtractVectorElementConstant(S32, VectorReg, I).getReg(0); in castBufferRsrcFromV4I32()
584 const LLT S32 = LLT::scalar(32); in AMDGPULegalizerInfo() local
652 S32, S64 in AMDGPULegalizerInfo()
656 S32, S64, S16 in AMDGPULegalizerInfo()
660 S32, S64, S16, V2S16 in AMDGPULegalizerInfo()
663 const LLT MinScalarFPTy = ST.has16BitInsts() ? S16 : S32; in AMDGPULegalizerInfo()
666 getActionDefinitionsBuilder(G_BRCOND).legalFor({S1, S32}); in AMDGPULegalizerInfo()
671 .legalFor({S32, S64, V2S16, S16, V4S16, S1, S128, S256}) in AMDGPULegalizerInfo()
680 .clampMaxNumElements(0, S32, 16) in AMDGPULegalizerInfo()
688 .legalFor({S64, S32, S16, V2S16}) in AMDGPULegalizerInfo()
693 .maxScalar(0, S32); in AMDGPULegalizerInfo()
696 .legalFor({S32, S16, V2S16}) in AMDGPULegalizerInfo()
701 .maxScalar(0, S32); in AMDGPULegalizerInfo()
706 .legalFor({S64, S32, S16, V2S16}) in AMDGPULegalizerInfo()
714 .legalFor({S32, S16, V2S16}) in AMDGPULegalizerInfo()
724 .legalFor({S32, S16, V2S16}) // Clamp modifier in AMDGPULegalizerInfo()
732 .legalFor({S32, S16}) in AMDGPULegalizerInfo()
735 .maxScalar(0, S32) in AMDGPULegalizerInfo()
739 .legalFor({S32, S16}) in AMDGPULegalizerInfo()
749 .legalFor({S32, S16}) // Clamp modifier in AMDGPULegalizerInfo()
763 .legalFor({S32}) in AMDGPULegalizerInfo()
765 .clampScalar(0, S32, S32) in AMDGPULegalizerInfo()
769 .legalFor({S32}) in AMDGPULegalizerInfo()
771 .minScalar(0, S32) in AMDGPULegalizerInfo()
777 Mul.maxScalar(0, S32); in AMDGPULegalizerInfo()
781 .legalFor({S32}) // Clamp modifier. in AMDGPULegalizerInfo()
783 .minScalarOrElt(0, S32) in AMDGPULegalizerInfo()
788 .minScalar(0, S32) in AMDGPULegalizerInfo()
796 .minScalar(0, S32) in AMDGPULegalizerInfo()
803 .customFor({S32, S64}) in AMDGPULegalizerInfo()
804 .clampScalar(0, S32, S64) in AMDGPULegalizerInfo()
809 .legalFor({S32}) in AMDGPULegalizerInfo()
810 .maxScalar(0, S32); in AMDGPULegalizerInfo()
825 .legalFor({S32, S1, S64, V2S32, S16, V2S16, V4S16}) in AMDGPULegalizerInfo()
826 .clampScalar(0, S32, S64) in AMDGPULegalizerInfo()
834 .legalFor({{S32, S1}, {S32, S32}}) in AMDGPULegalizerInfo()
835 .clampScalar(0, S32, S32) in AMDGPULegalizerInfo()
845 .legalFor({S1, S32, S64, S16, GlobalPtr, in AMDGPULegalizerInfo()
848 .clampScalar(0, S32, S64) in AMDGPULegalizerInfo()
852 .legalFor({S32, S64, S16}) in AMDGPULegalizerInfo()
861 .clampScalarOrElt(0, S32, MaxScalar) in AMDGPULegalizerInfo()
863 .clampMaxNumElements(0, S32, 16); in AMDGPULegalizerInfo()
870 .legalFor({{PrivatePtr, S32}}); in AMDGPULegalizerInfo()
885 .legalFor({S32, S64}); in AMDGPULegalizerInfo()
887 .customFor({S32, S64}); in AMDGPULegalizerInfo()
889 .customFor({S32, S64}); in AMDGPULegalizerInfo()
903 FPOpActions.clampMaxNumElementsStrict(0, S32, 2); in AMDGPULegalizerInfo()
921 .clampScalar(0, S32, S64) in AMDGPULegalizerInfo()
930 .clampScalar(0, ST.has16BitInsts() ? S16 : S32, S64); in AMDGPULegalizerInfo()
934 .clampScalar(0, ST.has16BitInsts() ? S16 : S32, S64); in AMDGPULegalizerInfo()
938 .clampScalar(0, ST.has16BitInsts() ? S16 : S32, S64); in AMDGPULegalizerInfo()
949 .customFor({S32, S64}) in AMDGPULegalizerInfo()
953 .legalFor({S32, S64, S16}) in AMDGPULegalizerInfo()
958 .legalFor({{S32, S32}, {S64, S32}, {S16, S16}}) in AMDGPULegalizerInfo()
961 .clampScalar(1, S32, S32) in AMDGPULegalizerInfo()
965 .customFor({{S32, S32}, {S64, S32}, {S16, S16}, {S16, S32}}) in AMDGPULegalizerInfo()
970 .customFor({S32, S64, S16}) in AMDGPULegalizerInfo()
978 .legalFor({S32, S64}) in AMDGPULegalizerInfo()
980 .clampScalar(0, S32, S64); in AMDGPULegalizerInfo()
983 .legalFor({S32, S64}) in AMDGPULegalizerInfo()
985 .clampScalar(0, S32, S64); in AMDGPULegalizerInfo()
989 .legalFor({{S32, S32}, {S64, S32}}) in AMDGPULegalizerInfo()
991 .clampScalar(0, S32, S64) in AMDGPULegalizerInfo()
992 .clampScalar(1, S32, S32) in AMDGPULegalizerInfo()
996 .customFor({{S32, S32}, {S64, S32}}) in AMDGPULegalizerInfo()
998 .minScalar(0, S32) in AMDGPULegalizerInfo()
999 .clampScalar(1, S32, S32) in AMDGPULegalizerInfo()
1004 .legalFor({{S32, S64}, {S16, S32}}) in AMDGPULegalizerInfo()
1009 .legalFor({{S64, S32}, {S32, S16}}) in AMDGPULegalizerInfo()
1010 .narrowScalarFor({{S64, S16}}, changeTo(0, S32)) in AMDGPULegalizerInfo()
1017 .legalFor({S32, S16}) in AMDGPULegalizerInfo()
1023 .legalFor({S32}) in AMDGPULegalizerInfo()
1030 .clampScalar(0, S32, S64); in AMDGPULegalizerInfo()
1035 FMad.customFor({S32, S16}); in AMDGPULegalizerInfo()
1037 FMad.customFor({S32}); in AMDGPULegalizerInfo()
1045 FRem.customFor({S16, S32, S64}); in AMDGPULegalizerInfo()
1047 FRem.minScalar(0, S32) in AMDGPULegalizerInfo()
1048 .customFor({S32, S64}); in AMDGPULegalizerInfo()
1064 .legalFor({{S64, S32}, {S32, S16}, {S64, S16}, in AMDGPULegalizerInfo()
1065 {S32, S1}, {S64, S1}, {S16, S1}}) in AMDGPULegalizerInfo()
1067 .clampScalar(0, S32, S64) in AMDGPULegalizerInfo()
1072 .legalFor({{S32, S32}, {S64, S32}, {S16, S32}}) in AMDGPULegalizerInfo()
1074 .customFor({{S32, S64}, {S64, S64}}); in AMDGPULegalizerInfo()
1077 IToFP.clampScalar(1, S32, S64) in AMDGPULegalizerInfo()
1078 .minScalar(0, S32) in AMDGPULegalizerInfo()
1083 .legalFor({{S32, S32}, {S32, S64}, {S32, S16}}) in AMDGPULegalizerInfo()
1084 .customFor({{S64, S32}, {S64, S64}}) in AMDGPULegalizerInfo()
1085 .narrowScalarFor({{S64, S16}}, changeTo(0, S32)); in AMDGPULegalizerInfo()
1089 FPToI.minScalar(1, S32); in AMDGPULegalizerInfo()
1091 FPToI.minScalar(0, S32) in AMDGPULegalizerInfo()
1097 .customFor({S16, S32}) in AMDGPULegalizerInfo()
1109 .legalFor({S16, S32, S64}) in AMDGPULegalizerInfo()
1115 .legalFor({S32, S64}) in AMDGPULegalizerInfo()
1116 .clampScalar(0, S32, S64) in AMDGPULegalizerInfo()
1121 .legalFor({S32}) in AMDGPULegalizerInfo()
1123 .clampScalar(0, S32, S64) in AMDGPULegalizerInfo()
1134 .legalIf(all(sameSize(0, 1), typeInSet(1, {S64, S32}))) in AMDGPULegalizerInfo()
1151 {S1}, {S32, S64, GlobalPtr, LocalPtr, ConstantPtr, PrivatePtr, FlatPtr}) in AMDGPULegalizerInfo()
1153 {S32}, {S32, S64, GlobalPtr, LocalPtr, ConstantPtr, PrivatePtr, FlatPtr}); in AMDGPULegalizerInfo()
1160 .clampScalar(1, S32, S64) in AMDGPULegalizerInfo()
1162 .legalIf(all(typeInSet(0, {S1, S32}), isPointer(1))); in AMDGPULegalizerInfo()
1169 FCmpBuilder.legalForCartesianProduct({S32}, {S16, S32}); in AMDGPULegalizerInfo()
1173 .clampScalar(1, S32, S64) in AMDGPULegalizerInfo()
1179 ExpOps.customFor({{S32}, {S16}}); in AMDGPULegalizerInfo()
1181 ExpOps.customFor({S32}); in AMDGPULegalizerInfo()
1182 ExpOps.clampScalar(0, MinScalarFPTy, S32) in AMDGPULegalizerInfo()
1186 .clampScalar(0, MinScalarFPTy, S32) in AMDGPULegalizerInfo()
1190 Log2Ops.customFor({S32}); in AMDGPULegalizerInfo()
1200 LogOps.customFor({S32, S16}); in AMDGPULegalizerInfo()
1201 LogOps.clampScalar(0, MinScalarFPTy, S32) in AMDGPULegalizerInfo()
1206 .legalFor({{S32, S32}, {S32, S64}}) in AMDGPULegalizerInfo()
1207 .clampScalar(0, S32, S32) in AMDGPULegalizerInfo()
1209 .clampScalar(1, S32, S64) in AMDGPULegalizerInfo()
1233 .clampScalar(0, S32, S32) in AMDGPULegalizerInfo()
1234 .clampScalar(1, S32, S64) in AMDGPULegalizerInfo()
1241 .legalFor({{S32, S32}, {S32, S64}}) in AMDGPULegalizerInfo()
1242 .clampScalar(0, S32, S32) in AMDGPULegalizerInfo()
1243 .clampScalar(1, S32, S64) in AMDGPULegalizerInfo()
1251 .legalFor({S32, S64}) in AMDGPULegalizerInfo()
1252 .clampScalar(0, S32, S64) in AMDGPULegalizerInfo()
1258 .legalFor({S16, S32, V2S16}) in AMDGPULegalizerInfo()
1263 .clampScalar(0, S16, S32) in AMDGPULegalizerInfo()
1268 .legalFor({S32, S16, V2S16}) in AMDGPULegalizerInfo()
1276 .legalFor({S32, S16}) in AMDGPULegalizerInfo()
1285 .legalFor({S32}) in AMDGPULegalizerInfo()
1290 .maxScalar(0, S32) in AMDGPULegalizerInfo()
1295 .legalFor({S32}) in AMDGPULegalizerInfo()
1296 .minScalar(0, S32) in AMDGPULegalizerInfo()
1305 .legalForCartesianProduct(AddrSpaces32, {S32}) in AMDGPULegalizerInfo()
1321 .legalForCartesianProduct(AddrSpaces32, {S32}) in AMDGPULegalizerInfo()
1384 Actions.legalForTypesWithMemDesc({{S32, GlobalPtr, S32, GlobalAlign32}, in AMDGPULegalizerInfo()
1390 {S32, GlobalPtr, S8, GlobalAlign8}, in AMDGPULegalizerInfo()
1391 {S32, GlobalPtr, S16, GlobalAlign16}, in AMDGPULegalizerInfo()
1393 {S32, LocalPtr, S32, 32}, in AMDGPULegalizerInfo()
1396 {S32, LocalPtr, S8, 8}, in AMDGPULegalizerInfo()
1397 {S32, LocalPtr, S16, 16}, in AMDGPULegalizerInfo()
1398 {V2S16, LocalPtr, S32, 32}, in AMDGPULegalizerInfo()
1400 {S32, PrivatePtr, S32, 32}, in AMDGPULegalizerInfo()
1401 {S32, PrivatePtr, S8, 8}, in AMDGPULegalizerInfo()
1402 {S32, PrivatePtr, S16, 16}, in AMDGPULegalizerInfo()
1403 {V2S16, PrivatePtr, S32, 32}, in AMDGPULegalizerInfo()
1405 {S32, ConstantPtr, S32, GlobalAlign32}, in AMDGPULegalizerInfo()
1543 .minScalar(0, S32) in AMDGPULegalizerInfo()
1544 .narrowScalarIf(isWideScalarExtLoadTruncStore(0), changeTo(0, S32)) in AMDGPULegalizerInfo()
1552 .legalForTypesWithMemDesc({{S32, GlobalPtr, S8, 8}, in AMDGPULegalizerInfo()
1553 {S32, GlobalPtr, S16, 2 * 8}, in AMDGPULegalizerInfo()
1554 {S32, LocalPtr, S8, 8}, in AMDGPULegalizerInfo()
1555 {S32, LocalPtr, S16, 16}, in AMDGPULegalizerInfo()
1556 {S32, PrivatePtr, S8, 8}, in AMDGPULegalizerInfo()
1557 {S32, PrivatePtr, S16, 16}, in AMDGPULegalizerInfo()
1558 {S32, ConstantPtr, S8, 8}, in AMDGPULegalizerInfo()
1559 {S32, ConstantPtr, S16, 2 * 8}}) in AMDGPULegalizerInfo()
1567 {{S32, FlatPtr, S8, 8}, {S32, FlatPtr, S16, 16}}); in AMDGPULegalizerInfo()
1577 ExtLoads.clampScalar(0, S32, S32) in AMDGPULegalizerInfo()
1586 .legalFor({{S32, GlobalPtr}, {S32, LocalPtr}, in AMDGPULegalizerInfo()
1588 {S32, RegionPtr}, {S64, RegionPtr}}); in AMDGPULegalizerInfo()
1590 Atomics.legalFor({{S32, FlatPtr}, {S64, FlatPtr}}); in AMDGPULegalizerInfo()
1595 Atomic.legalFor({{S32, LocalPtr}, {S32, RegionPtr}}); in AMDGPULegalizerInfo()
1602 Atomic.legalFor({{S32, GlobalPtr}}); in AMDGPULegalizerInfo()
1604 Atomic.legalFor({{S32, FlatPtr}}); in AMDGPULegalizerInfo()
1611 {S32, GlobalPtr}, in AMDGPULegalizerInfo()
1620 .customFor({{S32, GlobalPtr}, {S64, GlobalPtr}, in AMDGPULegalizerInfo()
1621 {S32, FlatPtr}, {S64, FlatPtr}}) in AMDGPULegalizerInfo()
1622 .legalFor({{S32, LocalPtr}, {S64, LocalPtr}, in AMDGPULegalizerInfo()
1623 {S32, RegionPtr}, {S64, RegionPtr}}); in AMDGPULegalizerInfo()
1628 .legalForCartesianProduct({S32, S64, S16, V2S32, V2S16, V4S16, GlobalPtr, in AMDGPULegalizerInfo()
1632 {S1, S32}) in AMDGPULegalizerInfo()
1637 .clampMaxNumElements(0, S32, 2) in AMDGPULegalizerInfo()
1642 .legalIf(all(isPointer(0), typeInSet(1, {S1, S32}))); in AMDGPULegalizerInfo()
1647 .legalFor({{S32, S32}, {S64, S32}}); in AMDGPULegalizerInfo()
1666 Shifts.clampScalar(1, S32, S32); in AMDGPULegalizerInfo()
1678 Shifts.clampScalar(1, S32, S32); in AMDGPULegalizerInfo()
1680 Shifts.clampScalar(0, S32, S64); in AMDGPULegalizerInfo()
1683 .minScalar(0, S32) in AMDGPULegalizerInfo()
1735 .clampScalar(EltTypeIdx, S32, S64) in AMDGPULegalizerInfo()
1736 .clampScalar(VecTypeIdx, S32, S64) in AMDGPULegalizerInfo()
1737 .clampScalar(IdxTypeIdx, S32, S32) in AMDGPULegalizerInfo()
1738 .clampMaxNumElements(VecTypeIdx, S32, 32) in AMDGPULegalizerInfo()
1793 .legalForCartesianProduct(AllS32Vectors, {S32}) in AMDGPULegalizerInfo()
1809 .legalFor({V2S16, S32}) in AMDGPULegalizerInfo()
1813 BuildVector.minScalarOrElt(0, S32); in AMDGPULegalizerInfo()
1816 .customFor({V2S16, S32}) in AMDGPULegalizerInfo()
1825 .clampMaxNumElements(0, S32, 32) in AMDGPULegalizerInfo()
1866 .clampScalar(LitTyIdx, S32, S512) in AMDGPULegalizerInfo()
1875 .clampScalar(BigTyIdx, S32, MaxScalar); in AMDGPULegalizerInfo()
1884 changeTo(LitTyIdx, S32)); in AMDGPULegalizerInfo()
1912 .legalFor({{S32}, {S64}}); in AMDGPULegalizerInfo()
1921 SextInReg.lowerFor({{S32}, {S64}, {S16}}); in AMDGPULegalizerInfo()
1925 SextInReg.lowerFor({{S32}, {S64}}); in AMDGPULegalizerInfo()
1930 .clampScalar(0, S32, S64) in AMDGPULegalizerInfo()
1939 .legalFor({{S32, S32}}) in AMDGPULegalizerInfo()
1965 .minScalar(0, S32) in AMDGPULegalizerInfo()
1969 .legalFor({{S32, S32}, {S64, S32}}) in AMDGPULegalizerInfo()
1970 .clampScalar(1, S32, S32) in AMDGPULegalizerInfo()
1971 .clampScalar(0, S32, S64) in AMDGPULegalizerInfo()
2111 const LLT S32 = LLT::scalar(32); in getSegmentAperture() local
2133 return B.buildUnmerge(S32, Dst).getReg(1); in getSegmentAperture()
2167 return B.buildLoad(S32, LoadAddr, *MMO).getReg(0); in getSegmentAperture()
2188 return B.buildLoad(S32, LoadAddr, *MMO).getReg(0); in getSegmentAperture()
2217 const LLT S32 = LLT::scalar(32); in legalizeAddrSpaceCast() local
2272 Register SrcAsInt = B.buildPtrToInt(S32, Src).getReg(0); in legalizeAddrSpaceCast()
2308 auto PtrLo = B.buildPtrToInt(S32, Src); in legalizeAddrSpaceCast()
2309 auto HighAddr = B.buildConstant(S32, AddrHiVal); in legalizeAddrSpaceCast()
2401 LLT S32 = LLT::scalar(32); in extractF64Exponent() local
2403 auto Const0 = B.buildConstant(S32, FractBits - 32); in extractF64Exponent()
2404 auto Const1 = B.buildConstant(S32, ExpBits); in extractF64Exponent()
2406 auto ExpPart = B.buildIntrinsic(Intrinsic::amdgcn_ubfe, {S32}) in extractF64Exponent()
2411 return B.buildSub(S32, ExpPart, B.buildConstant(S32, 1023)); in extractF64Exponent()
2418 const LLT S32 = LLT::scalar(32); in legalizeIntrinsicTrunc() local
2425 auto Unmerge = B.buildUnmerge({S32, S32}, Src); in legalizeIntrinsicTrunc()
2435 const auto SignBitMask = B.buildConstant(S32, UINT32_C(1) << 31); in legalizeIntrinsicTrunc()
2436 auto SignBit = B.buildAnd(S32, Hi, SignBitMask); in legalizeIntrinsicTrunc()
2440 const auto Zero32 = B.buildConstant(S32, 0); in legalizeIntrinsicTrunc()
2448 auto FiftyOne = B.buildConstant(S32, FractBits - 1); in legalizeIntrinsicTrunc()
2467 const LLT S32 = LLT::scalar(32); in legalizeITOFP() local
2471 auto Unmerge = B.buildUnmerge({S32, S32}, Src); in legalizeITOFP()
2472 auto ThirtyTwo = B.buildConstant(S32, 32); in legalizeITOFP()
2487 assert(MRI.getType(Dst) == S32); in legalizeITOFP()
2489 auto One = B.buildConstant(S32, 1); in legalizeITOFP()
2493 auto ThirtyOne = B.buildConstant(S32, 31); in legalizeITOFP()
2494 auto X = B.buildXor(S32, Unmerge.getReg(0), Unmerge.getReg(1)); in legalizeITOFP()
2495 auto OppositeSign = B.buildAShr(S32, X, ThirtyOne); in legalizeITOFP()
2496 auto MaxShAmt = B.buildAdd(S32, ThirtyTwo, OppositeSign); in legalizeITOFP()
2497 auto LS = B.buildIntrinsic(Intrinsic::amdgcn_sffbh, {S32}) in legalizeITOFP()
2499 auto LS2 = B.buildSub(S32, LS, One); in legalizeITOFP()
2500 ShAmt = B.buildUMin(S32, LS2, MaxShAmt); in legalizeITOFP()
2502 ShAmt = B.buildCTLZ(S32, Unmerge.getReg(1)); in legalizeITOFP()
2504 auto Unmerge2 = B.buildUnmerge({S32, S32}, Norm); in legalizeITOFP()
2505 auto Adjust = B.buildUMin(S32, One, Unmerge2.getReg(0)); in legalizeITOFP()
2506 auto Norm2 = B.buildOr(S32, Unmerge2.getReg(1), Adjust); in legalizeITOFP()
2507 auto FVal = Signed ? B.buildSITOFP(S32, Norm2) : B.buildUITOFP(S32, Norm2); in legalizeITOFP()
2508 auto Scale = B.buildSub(S32, ThirtyTwo, ShAmt); in legalizeITOFP()
2525 const LLT S32 = LLT::scalar(32); in legalizeFPTOI() local
2528 assert((SrcLT == S32 || SrcLT == S64) && MRI.getType(Dst) == S64); in legalizeFPTOI()
2543 if (Signed && SrcLT == S32) { in legalizeFPTOI()
2549 Sign = B.buildAShr(S32, Src, B.buildConstant(S32, 31)); in legalizeFPTOI()
2550 Trunc = B.buildFAbs(S32, Trunc, Flags); in legalizeFPTOI()
2560 S32, llvm::bit_cast<float>(UINT32_C(/*2^-32*/ 0x2f800000))); in legalizeFPTOI()
2562 S32, llvm::bit_cast<float>(UINT32_C(/*-2^32*/ 0xcf800000))); in legalizeFPTOI()
2569 auto Hi = (Signed && SrcLT == S64) ? B.buildFPTOSI(S32, FloorMul) in legalizeFPTOI()
2570 : B.buildFPTOUI(S32, FloorMul); in legalizeFPTOI()
2571 auto Lo = B.buildFPTOUI(S32, Fma); in legalizeFPTOI()
2573 if (Signed && SrcLT == S32) { in legalizeFPTOI()
2804 LLT S32 = LLT::scalar(32); in buildAbsGlobalAddress() local
2810 : MRI.createGenericVirtualRegister(S32); in buildAbsGlobalAddress()
2825 Register AddrHi = MRI.createGenericVirtualRegister(S32); in buildAbsGlobalAddress()
2904 LLT S32 = LLT::scalar(32); in legalizeGlobalValue() local
2905 auto Sz = B.buildIntrinsic(Intrinsic::amdgcn_groupstaticsize, {S32}); in legalizeGlobalValue()
3742 const LLT S32 = LLT::scalar(32); in legalizeBuildVector() local
3750 assert(MRI.getType(Src0) == S32); in legalizeBuildVector()
3755 auto Merge = B.buildMergeLikeInstr(S32, {Src0, Src1}); in legalizeBuildVector()
3785 const LLT S32 = LLT::scalar(32); in buildMultiply() local
3793 Zero32 = B.buildConstant(S32, 0).getReg(0); in buildMultiply()
3821 LocalAccum = B.buildZExt(S32, CarryIn[0]).getReg(0); in buildMultiply()
3827 CarryAccum = B.buildZExt(S32, CarryIn[0]).getReg(0); in buildMultiply()
3830 B.buildUAdde(S32, S1, CarryAccum, getZero32(), CarryIn[i]) in buildMultiply()
3841 B.buildUAdde(S32, S1, CarryAccum, LocalAccum, CarryIn.back()); in buildMultiply()
3876 auto Mul = B.buildMul(S32, Src0[j0], Src1[j1]); in buildMultiply()
3881 LocalAccum[0] = B.buildAdd(S32, LocalAccum[0], Mul).getReg(0); in buildMultiply()
3884 B.buildUAdde(S32, S1, LocalAccum[0], Mul, CarryIn.back()) in buildMultiply()
3931 auto Unmerge = B.buildUnmerge(S32, Tmp); in buildMultiply()
3989 Lo = B.buildUAddo(S32, S1, Accum[2 * i - 1], SeparateOddOut[0]); in buildMultiply()
3991 Lo = B.buildAdd(S32, Accum[2 * i - 1], SeparateOddOut[0]); in buildMultiply()
3993 Lo = B.buildUAdde(S32, S1, Accum[2 * i - 1], SeparateOddOut[0], in buildMultiply()
3999 auto Hi = B.buildUAdde(S32, S1, Accum[2 * i], SeparateOddOut[1], in buildMultiply()
4054 LLT S32 = LLT::scalar(32); in legalizeMul() local
4057 Src0Parts.push_back(MRI.createGenericVirtualRegister(S32)); in legalizeMul()
4058 Src1Parts.push_back(MRI.createGenericVirtualRegister(S32)); in legalizeMul()
4156 const LLT S32 = LLT::scalar(32); in loadInputValue() local
4165 auto ShiftAmt = B.buildConstant(S32, Shift); in loadInputValue()
4166 AndMaskSrc = B.buildLShr(S32, LiveIn, ShiftAmt).getReg(0); in loadInputValue()
4169 B.buildAnd(DstReg, AndMaskSrc, B.buildConstant(S32, Mask >> Shift)); in loadInputValue()
4339 LLT S32 = LLT::scalar(32); in legalizeFDIV() local
4344 if (DstTy == S32) in legalizeFDIV()
4358 const LLT S32 = LLT::scalar(32); in legalizeUnsignedDIV_REM32Impl() local
4364 auto FloatY = B.buildUITOFP(S32, Y); in legalizeUnsignedDIV_REM32Impl()
4365 auto RcpIFlag = B.buildInstr(AMDGPU::G_AMDGPU_RCP_IFLAG, {S32}, {FloatY}); in legalizeUnsignedDIV_REM32Impl()
4366 auto Scale = B.buildFConstant(S32, llvm::bit_cast<float>(0x4f7ffffe)); in legalizeUnsignedDIV_REM32Impl()
4367 auto ScaledY = B.buildFMul(S32, RcpIFlag, Scale); in legalizeUnsignedDIV_REM32Impl()
4368 auto Z = B.buildFPTOUI(S32, ScaledY); in legalizeUnsignedDIV_REM32Impl()
4371 auto NegY = B.buildSub(S32, B.buildConstant(S32, 0), Y); in legalizeUnsignedDIV_REM32Impl()
4372 auto NegYZ = B.buildMul(S32, NegY, Z); in legalizeUnsignedDIV_REM32Impl()
4373 Z = B.buildAdd(S32, Z, B.buildUMulH(S32, Z, NegYZ)); in legalizeUnsignedDIV_REM32Impl()
4376 auto Q = B.buildUMulH(S32, X, Z); in legalizeUnsignedDIV_REM32Impl()
4377 auto R = B.buildSub(S32, X, B.buildMul(S32, Q, Y)); in legalizeUnsignedDIV_REM32Impl()
4380 auto One = B.buildConstant(S32, 1); in legalizeUnsignedDIV_REM32Impl()
4383 Q = B.buildSelect(S32, Cond, B.buildAdd(S32, Q, One), Q); in legalizeUnsignedDIV_REM32Impl()
4384 R = B.buildSelect(S32, Cond, B.buildSub(S32, R, Y), R); in legalizeUnsignedDIV_REM32Impl()
4389 B.buildSelect(DstDivReg, Cond, B.buildAdd(S32, Q, One), Q); in legalizeUnsignedDIV_REM32Impl()
4392 B.buildSelect(DstRemReg, Cond, B.buildSub(S32, R, Y), R); in legalizeUnsignedDIV_REM32Impl()
4410 const LLT S32 = LLT::scalar(32); in emitReciprocalU64() local
4411 auto Unmerge = B.buildUnmerge(S32, Val); in emitReciprocalU64()
4413 auto CvtLo = B.buildUITOFP(S32, Unmerge.getReg(0)); in emitReciprocalU64()
4414 auto CvtHi = B.buildUITOFP(S32, Unmerge.getReg(1)); in emitReciprocalU64()
4417 S32, CvtHi, // 2**32 in emitReciprocalU64()
4418 B.buildFConstant(S32, llvm::bit_cast<float>(0x4f800000)), CvtLo); in emitReciprocalU64()
4420 auto Rcp = B.buildInstr(AMDGPU::G_AMDGPU_RCP_IFLAG, {S32}, {Mad}); in emitReciprocalU64()
4422 S32, Rcp, B.buildFConstant(S32, llvm::bit_cast<float>(0x5f7ffffc))); in emitReciprocalU64()
4426 S32, Mul1, B.buildFConstant(S32, llvm::bit_cast<float>(0x2f800000))); in emitReciprocalU64()
4427 auto Trunc = B.buildIntrinsicTrunc(S32, Mul2); in emitReciprocalU64()
4431 S32, Trunc, B.buildFConstant(S32, llvm::bit_cast<float>(0xcf800000)), in emitReciprocalU64()
4434 auto ResultLo = B.buildFPTOUI(S32, Mad2); in emitReciprocalU64()
4435 auto ResultHi = B.buildFPTOUI(S32, Trunc); in emitReciprocalU64()
4445 const LLT S32 = LLT::scalar(32); in legalizeUnsignedDIV_REM64Impl() local
4460 auto UnmergeMulHi1 = B.buildUnmerge(S32, MulHi1); in legalizeUnsignedDIV_REM64Impl()
4464 auto Add1_Lo = B.buildUAddo(S32, S1, RcpLo, MulHi1_Lo); in legalizeUnsignedDIV_REM64Impl()
4465 auto Add1_Hi = B.buildUAdde(S32, S1, RcpHi, MulHi1_Hi, Add1_Lo.getReg(1)); in legalizeUnsignedDIV_REM64Impl()
4470 auto UnmergeMulHi2 = B.buildUnmerge(S32, MulHi2); in legalizeUnsignedDIV_REM64Impl()
4474 auto Zero32 = B.buildConstant(S32, 0); in legalizeUnsignedDIV_REM64Impl()
4475 auto Add2_Lo = B.buildUAddo(S32, S1, Add1_Lo, MulHi2_Lo); in legalizeUnsignedDIV_REM64Impl()
4476 auto Add2_Hi = B.buildUAdde(S32, S1, Add1_Hi, MulHi2_Hi, Add2_Lo.getReg(1)); in legalizeUnsignedDIV_REM64Impl()
4479 auto UnmergeNumer = B.buildUnmerge(S32, Numer); in legalizeUnsignedDIV_REM64Impl()
4485 auto UnmergeMul3 = B.buildUnmerge(S32, Mul3); in legalizeUnsignedDIV_REM64Impl()
4488 auto Sub1_Lo = B.buildUSubo(S32, S1, NumerLo, Mul3_Lo); in legalizeUnsignedDIV_REM64Impl()
4489 auto Sub1_Hi = B.buildUSube(S32, S1, NumerHi, Mul3_Hi, Sub1_Lo.getReg(1)); in legalizeUnsignedDIV_REM64Impl()
4490 auto Sub1_Mi = B.buildSub(S32, NumerHi, Mul3_Hi); in legalizeUnsignedDIV_REM64Impl()
4493 auto UnmergeDenom = B.buildUnmerge(S32, Denom); in legalizeUnsignedDIV_REM64Impl()
4498 auto C1 = B.buildSExt(S32, CmpHi); in legalizeUnsignedDIV_REM64Impl()
4501 auto C2 = B.buildSExt(S32, CmpLo); in legalizeUnsignedDIV_REM64Impl()
4504 auto C3 = B.buildSelect(S32, CmpEq, C2, C1); in legalizeUnsignedDIV_REM64Impl()
4511 auto Sub2_Lo = B.buildUSubo(S32, S1, Sub1_Lo, DenomLo); in legalizeUnsignedDIV_REM64Impl()
4512 auto Sub2_Mi = B.buildUSube(S32, S1, Sub1_Mi, DenomHi, Sub1_Lo.getReg(1)); in legalizeUnsignedDIV_REM64Impl()
4513 auto Sub2_Hi = B.buildUSube(S32, S1, Sub2_Mi, Zero32, Sub2_Lo.getReg(1)); in legalizeUnsignedDIV_REM64Impl()
4520 B.buildSExt(S32, B.buildICmp(CmpInst::ICMP_UGE, S1, Sub2_Hi, DenomHi)); in legalizeUnsignedDIV_REM64Impl()
4522 B.buildSExt(S32, B.buildICmp(CmpInst::ICMP_UGE, S1, Sub2_Lo, DenomLo)); in legalizeUnsignedDIV_REM64Impl()
4524 S32, B.buildICmp(CmpInst::ICMP_EQ, S1, Sub2_Hi, DenomHi), C5, C4); in legalizeUnsignedDIV_REM64Impl()
4528 auto Sub3_Lo = B.buildUSubo(S32, S1, Sub2_Lo, DenomLo); in legalizeUnsignedDIV_REM64Impl()
4530 auto Sub3_Mi = B.buildUSube(S32, S1, Sub2_Mi, DenomHi, Sub2_Lo.getReg(1)); in legalizeUnsignedDIV_REM64Impl()
4531 auto Sub3_Hi = B.buildUSube(S32, S1, Sub3_Mi, Zero32, Sub3_Lo.getReg(1)); in legalizeUnsignedDIV_REM64Impl()
4575 const LLT S32 = LLT::scalar(32); in legalizeUnsignedDIV_REM() local
4581 if (Ty == S32) in legalizeUnsignedDIV_REM()
4596 const LLT S32 = LLT::scalar(32); in legalizeSignedDIV_REM() local
4599 if (Ty != S32 && Ty != S64) in legalizeSignedDIV_REM()
4606 auto SignBitOffset = B.buildConstant(S32, Ty.getSizeInBits() - 1); in legalizeSignedDIV_REM()
4639 if (Ty == S32) in legalizeSignedDIV_REM()
4772 LLT S32 = LLT::scalar(32); in legalizeFDIV16() local
4774 auto LHSExt = B.buildFPExt(S32, LHS, Flags); in legalizeFDIV16()
4775 auto RHSExt = B.buildFPExt(S32, RHS, Flags); in legalizeFDIV16()
4777 auto RCP = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {S32}) in legalizeFDIV16()
4781 auto QUOT = B.buildFMul(S32, LHSExt, RCP, Flags); in legalizeFDIV16()
4836 LLT S32 = LLT::scalar(32); in legalizeFDIV32() local
4839 auto One = B.buildFConstant(S32, 1.0f); in legalizeFDIV32()
4842 B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {S32, S1}) in legalizeFDIV32()
4848 B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {S32, S1}) in legalizeFDIV32()
4854 auto ApproxRcp = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {S32}) in legalizeFDIV32()
4857 auto NegDivScale0 = B.buildFNeg(S32, DenominatorScaled, Flags); in legalizeFDIV32()
4875 auto Fma0 = B.buildFMA(S32, NegDivScale0, ApproxRcp, One, Flags); in legalizeFDIV32()
4876 auto Fma1 = B.buildFMA(S32, Fma0, ApproxRcp, ApproxRcp, Flags); in legalizeFDIV32()
4877 auto Mul = B.buildFMul(S32, NumeratorScaled, Fma1, Flags); in legalizeFDIV32()
4878 auto Fma2 = B.buildFMA(S32, NegDivScale0, Mul, NumeratorScaled, Flags); in legalizeFDIV32()
4879 auto Fma3 = B.buildFMA(S32, Fma2, Fma1, Mul, Flags); in legalizeFDIV32()
4880 auto Fma4 = B.buildFMA(S32, NegDivScale0, Fma3, NumeratorScaled, Flags); in legalizeFDIV32()
4892 auto Fmas = B.buildIntrinsic(Intrinsic::amdgcn_div_fmas, {S32}) in legalizeFDIV32()
4957 LLT S32 = LLT::scalar(32); in legalizeFDIV64() local
4959 auto NumUnmerge = B.buildUnmerge(S32, LHS); in legalizeFDIV64()
4960 auto DenUnmerge = B.buildUnmerge(S32, RHS); in legalizeFDIV64()
4961 auto Scale0Unmerge = B.buildUnmerge(S32, DivScale0); in legalizeFDIV64()
4962 auto Scale1Unmerge = B.buildUnmerge(S32, DivScale1); in legalizeFDIV64()
5033 LLT S32 = LLT::scalar(32); in legalizeFDIVFastIntrin() local
5036 auto Abs = B.buildFAbs(S32, RHS, Flags); in legalizeFDIVFastIntrin()
5039 auto C0 = B.buildFConstant(S32, 0x1p+96f); in legalizeFDIVFastIntrin()
5040 auto C1 = B.buildFConstant(S32, 0x1p-32f); in legalizeFDIVFastIntrin()
5041 auto C2 = B.buildFConstant(S32, 1.0f); in legalizeFDIVFastIntrin()
5044 auto Sel = B.buildSelect(S32, CmpRes, C1, C2, Flags); in legalizeFDIVFastIntrin()
5046 auto Mul0 = B.buildFMul(S32, RHS, Sel, Flags); in legalizeFDIVFastIntrin()
5048 auto RCP = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {S32}) in legalizeFDIVFastIntrin()
5052 auto Mul1 = B.buildFMul(S32, LHS, RCP, Flags); in legalizeFDIVFastIntrin()
5182 const LLT S32 = LLT::scalar(32); in legalizeFSQRTF64() local
5193 auto ZeroInt = B.buildConstant(S32, 0); in legalizeFSQRTF64()
5197 auto ScaleUpFactor = B.buildConstant(S32, 256); in legalizeFSQRTF64()
5198 auto ScaleUp = B.buildSelect(S32, Scaling, ScaleUpFactor, ZeroInt); in legalizeFSQRTF64()
5225 auto ScaleDownFactor = B.buildConstant(S32, -128); in legalizeFSQRTF64()
5226 auto ScaleDown = B.buildSelect(S32, Scaling, ScaleDownFactor, ZeroInt); in legalizeFSQRTF64()
5366 LLT S32 = LLT::scalar(32); in legalizePointerAsRsrcIntrin() local
5369 auto Unmerge = B.buildUnmerge(S32, Pointer); in legalizePointerAsRsrcIntrin()
5373 auto AndMask = B.buildConstant(S32, 0x0000ffff); in legalizePointerAsRsrcIntrin()
5374 auto Masked = B.buildAnd(S32, HighHalf, AndMask); in legalizePointerAsRsrcIntrin()
5384 ShiftedStride = B.buildConstant(S32, ShiftedStrideVal); in legalizePointerAsRsrcIntrin()
5386 auto ExtStride = B.buildAnyExt(S32, Stride); in legalizePointerAsRsrcIntrin()
5387 auto ShiftConst = B.buildConstant(S32, 16); in legalizePointerAsRsrcIntrin()
5388 ShiftedStride = B.buildShl(S32, ExtStride, ShiftConst); in legalizePointerAsRsrcIntrin()
5390 NewHighHalf = B.buildOr(S32, Masked, ShiftedStride); in legalizePointerAsRsrcIntrin()
5469 const LLT S32 = LLT::scalar(32); in splitBufferOffsets() local
5496 BaseReg = B.buildConstant(S32, Overflow).getReg(0); in splitBufferOffsets()
5498 auto OverflowVal = B.buildConstant(S32, Overflow); in splitBufferOffsets()
5499 BaseReg = B.buildAdd(S32, BaseReg, OverflowVal).getReg(0); in splitBufferOffsets()
5504 BaseReg = B.buildConstant(S32, 0).getReg(0); in splitBufferOffsets()
5515 const LLT S32 = LLT::scalar(32); in handleD16VData() local
5524 WideRegs.push_back(B.buildAnyExt(S32, Unmerge.getReg(I)).getReg(0)); in handleD16VData()
5528 return B.buildBuildVector(LLT::fixed_vector(NumElts, S32), WideRegs) in handleD16VData()
5535 Reg = B.buildBitcast(S32, Reg).getReg(0); in handleD16VData()
5537 PackedRegs.resize(2, B.buildUndef(S32).getReg(0)); in handleD16VData()
5538 return B.buildBuildVector(LLT::fixed_vector(2, S32), PackedRegs) in handleD16VData()
5549 return B.buildBitcast(LLT::fixed_vector(3, S32), Reg).getReg(0); in handleD16VData()
5554 Reg = B.buildBitcast(LLT::fixed_vector(2, S32), Reg).getReg(0); in handleD16VData()
5555 auto Unmerge = B.buildUnmerge(S32, Reg); in handleD16VData()
5558 PackedRegs.resize(4, B.buildUndef(S32).getReg(0)); in handleD16VData()
5559 return B.buildBuildVector(LLT::fixed_vector(4, S32), PackedRegs) in handleD16VData()
5609 const LLT S32 = LLT::scalar(32); in legalizeBufferStore() local
5631 VIndex = B.buildConstant(S32, 0).getReg(0); in legalizeBufferStore()
5716 const LLT S32 = LLT::scalar(32); in legalizeBufferLoad() local
5742 VIndex = B.buildConstant(S32, 0).getReg(0); in legalizeBufferLoad()
5806 LLT LoadTy = LLT::fixed_vector(NumLoadDWords, S32); in legalizeBufferLoad()
5815 LoadElts.push_back(B.getMRI()->createGenericVirtualRegister(S32)); in legalizeBufferLoad()
5823 Register LoadDstReg = B.getMRI()->createGenericVirtualRegister(S32); in legalizeBufferLoad()
5835 auto Unmerge = B.buildUnmerge(S32, LoadDstReg); in legalizeBufferLoad()
6072 const LLT S32 = LLT::scalar(32); in convertImageAddrToPacked() local
6073 (void)S32; in convertImageAddrToPacked()
6079 assert(B.getMRI()->getType(SrcOp.getReg()) == S32); in convertImageAddrToPacked()
6126 const LLT S32 = LLT::scalar(32); in legalizeImageIntrinsic() local
6352 RegTy = S32; in legalizeImageIntrinsic()
6359 TFETy = LLT::fixed_vector(RoundedSize / 32 + 1, S32); in legalizeImageIntrinsic()
6360 RegTy = !IsTFE && EltSize == 16 ? V2S16 : S32; in legalizeImageIntrinsic()
6389 if (MRI->getType(Dst1Reg) != S32) in legalizeImageIntrinsic()
6396 if (Ty == S32) { in legalizeImageIntrinsic()
6706 const LLT S32 = LLT::scalar(32); in legalizeBVHIntrinsic() local
6759 auto packLanes = [&Ops, &S32, &V3S32, &B](Register Src) { in legalizeBVHIntrinsic()
6760 auto Unmerge = B.buildUnmerge({S32, S32, S32}, Src); in legalizeBVHIntrinsic()
6776 S32, B.buildMergeLikeInstr(V2S16, {UnmergeRayInvDir.getReg(0), in legalizeBVHIntrinsic()
6780 S32, B.buildMergeLikeInstr(V2S16, {UnmergeRayInvDir.getReg(1), in legalizeBVHIntrinsic()
6784 S32, B.buildMergeLikeInstr(V2S16, {UnmergeRayInvDir.getReg(2), in legalizeBVHIntrinsic()
6794 auto Unmerge = B.buildUnmerge({S32, S32}, NodePtr); in legalizeBVHIntrinsic()
6802 auto packLanes = [&Ops, &S32, &B](Register Src) { in legalizeBVHIntrinsic()
6803 auto Unmerge = B.buildUnmerge({S32, S32, S32}, Src); in legalizeBVHIntrinsic()
6813 Register R1 = MRI.createGenericVirtualRegister(S32); in legalizeBVHIntrinsic()
6814 Register R2 = MRI.createGenericVirtualRegister(S32); in legalizeBVHIntrinsic()
6815 Register R3 = MRI.createGenericVirtualRegister(S32); in legalizeBVHIntrinsic()
6891 LLT S32 = LLT::scalar(32); in legalizeWaveID() local
6893 auto TTMP8 = B.buildCopy(S32, Register(AMDGPU::TTMP8)); in legalizeWaveID()
6894 auto LSB = B.buildConstant(S32, 25); in legalizeWaveID()
6895 auto Width = B.buildConstant(S32, 5); in legalizeWaveID()
7198 LLT S32 = LLT::scalar(32); in legalizeIntrinsic() local
7199 if (MRI.getType(Index) != S32) in legalizeIntrinsic()
7200 MI.getOperand(5).setReg(B.buildAnyExt(S32, Index).getReg(0)); in legalizeIntrinsic()
7207 LLT S32 = LLT::scalar(32); in legalizeIntrinsic() local
7208 if (MRI.getType(Index) != S32) in legalizeIntrinsic()
7209 MI.getOperand(7).setReg(B.buildAnyExt(S32, Index).getReg(0)); in legalizeIntrinsic()