Lines Matching refs:In
53 static bool isExtractHiElt(SDValue In, SDValue &Out) { in isExtractHiElt() argument
54 In = stripBitcast(In); in isExtractHiElt()
56 if (In.getOpcode() == ISD::EXTRACT_VECTOR_ELT) { in isExtractHiElt()
57 if (ConstantSDNode *Idx = dyn_cast<ConstantSDNode>(In.getOperand(1))) { in isExtractHiElt()
60 Out = In.getOperand(0); in isExtractHiElt()
65 if (In.getOpcode() != ISD::TRUNCATE) in isExtractHiElt()
68 SDValue Srl = In.getOperand(0); in isExtractHiElt()
83 static SDValue stripExtractLoElt(SDValue In) { in stripExtractLoElt() argument
84 if (In.getOpcode() == ISD::EXTRACT_VECTOR_ELT) { in stripExtractLoElt()
85 SDValue Idx = In.getOperand(1); in stripExtractLoElt()
86 if (isNullConstant(Idx) && In.getValueSizeInBits() <= 32) in stripExtractLoElt()
87 return In.getOperand(0); in stripExtractLoElt()
90 if (In.getOpcode() == ISD::TRUNCATE) { in stripExtractLoElt()
91 SDValue Src = In.getOperand(0); in stripExtractLoElt()
96 return In; in stripExtractLoElt()
2787 bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(SDValue In, SDValue &Src, in SelectVOP3ModsImpl() argument
2792 Src = In; in SelectVOP3ModsImpl()
2815 bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src, in SelectVOP3Mods() argument
2818 if (SelectVOP3ModsImpl(In, Src, Mods, /*IsCanonicalizing=*/true, in SelectVOP3Mods()
2820 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); in SelectVOP3Mods()
2828 SDValue In, SDValue &Src, SDValue &SrcMods) const { in SelectVOP3ModsNonCanonicalizing() argument
2830 if (SelectVOP3ModsImpl(In, Src, Mods, /*IsCanonicalizing=*/false, in SelectVOP3ModsNonCanonicalizing()
2832 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); in SelectVOP3ModsNonCanonicalizing()
2839 bool AMDGPUDAGToDAGISel::SelectVOP3BMods(SDValue In, SDValue &Src, in SelectVOP3BMods() argument
2842 if (SelectVOP3ModsImpl(In, Src, Mods, in SelectVOP3BMods()
2845 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); in SelectVOP3BMods()
2852 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const { in SelectVOP3NoMods() argument
2853 if (In.getOpcode() == ISD::FABS || In.getOpcode() == ISD::FNEG) in SelectVOP3NoMods()
2856 Src = In; in SelectVOP3NoMods()
2860 bool AMDGPUDAGToDAGISel::SelectVINTERPModsImpl(SDValue In, SDValue &Src, in SelectVINTERPModsImpl() argument
2864 if (SelectVOP3ModsImpl(In, Src, Mods, in SelectVINTERPModsImpl()
2869 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); in SelectVINTERPModsImpl()
2876 bool AMDGPUDAGToDAGISel::SelectVINTERPMods(SDValue In, SDValue &Src, in SelectVINTERPMods() argument
2878 return SelectVINTERPModsImpl(In, Src, SrcMods, /* OpSel */ false); in SelectVINTERPMods()
2881 bool AMDGPUDAGToDAGISel::SelectVINTERPModsHi(SDValue In, SDValue &Src, in SelectVINTERPModsHi() argument
2883 return SelectVINTERPModsImpl(In, Src, SrcMods, /* OpSel */ true); in SelectVINTERPModsHi()
2886 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src, in SelectVOP3Mods0() argument
2889 SDLoc DL(In); in SelectVOP3Mods0()
2893 return SelectVOP3Mods(In, Src, SrcMods); in SelectVOP3Mods0()
2896 bool AMDGPUDAGToDAGISel::SelectVOP3BMods0(SDValue In, SDValue &Src, in SelectVOP3BMods0() argument
2899 SDLoc DL(In); in SelectVOP3BMods0()
2903 return SelectVOP3BMods(In, Src, SrcMods); in SelectVOP3BMods0()
2906 bool AMDGPUDAGToDAGISel::SelectVOP3OMods(SDValue In, SDValue &Src, in SelectVOP3OMods() argument
2908 Src = In; in SelectVOP3OMods()
2910 SDLoc DL(In); in SelectVOP3OMods()
2917 bool AMDGPUDAGToDAGISel::SelectVOP3PMods(SDValue In, SDValue &Src, in SelectVOP3PMods() argument
2920 Src = In; in SelectVOP3PMods()
2957 (VecSize > 32) ? AMDGPU::sub0_sub1 : AMDGPU::sub0, SDLoc(In), in SelectVOP3PMods()
2963 (VecSize > 32) ? AMDGPU::sub0_sub1 : AMDGPU::sub0, SDLoc(In), in SelectVOP3PMods()
2979 SDLoc SL(In); in SelectVOP3PMods()
2993 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); in SelectVOP3PMods()
3001 Src = CurDAG->getTargetConstant(Lit, SDLoc(In), MVT::i64); in SelectVOP3PMods()
3002 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); in SelectVOP3PMods()
3013 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); in SelectVOP3PMods()
3017 bool AMDGPUDAGToDAGISel::SelectVOP3PModsDOT(SDValue In, SDValue &Src, in SelectVOP3PModsDOT() argument
3019 return SelectVOP3PMods(In, Src, SrcMods, true); in SelectVOP3PModsDOT()
3022 bool AMDGPUDAGToDAGISel::SelectVOP3PModsNeg(SDValue In, SDValue &Src) const { in SelectVOP3PModsNeg() argument
3023 const ConstantSDNode *C = cast<ConstantSDNode>(In); in SelectVOP3PModsNeg()
3033 Src = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); in SelectVOP3PModsNeg()
3037 bool AMDGPUDAGToDAGISel::SelectWMMAOpSelVOP3PMods(SDValue In, in SelectWMMAOpSelVOP3PMods() argument
3039 const ConstantSDNode *C = cast<ConstantSDNode>(In); in SelectWMMAOpSelVOP3PMods()
3047 Src = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); in SelectWMMAOpSelVOP3PMods()
3165 bool AMDGPUDAGToDAGISel::SelectWMMAModsF16Neg(SDValue In, SDValue &Src, in SelectWMMAModsF16Neg() argument
3167 Src = In; in SelectWMMAModsF16Neg()
3171 if (auto *BV = dyn_cast<BuildVectorSDNode>(stripBitcast(In))) { in SelectWMMAModsF16Neg()
3183 Src = SDValue(buildRegSequence16(EltsF16, CurDAG, SDLoc(In)), 0); in SelectWMMAModsF16Neg()
3190 if (auto *BV = dyn_cast<BuildVectorSDNode>(stripBitcast(In))) { in SelectWMMAModsF16Neg()
3202 Src = SDValue(buildRegSequence32(EltsV2F16, CurDAG, SDLoc(In)), 0); in SelectWMMAModsF16Neg()
3208 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); in SelectWMMAModsF16Neg()
3212 bool AMDGPUDAGToDAGISel::SelectWMMAModsF16NegAbs(SDValue In, SDValue &Src, in SelectWMMAModsF16NegAbs() argument
3214 Src = In; in SelectWMMAModsF16NegAbs()
3219 if (auto *BV = dyn_cast<BuildVectorSDNode>(stripBitcast(In))) { in SelectWMMAModsF16NegAbs()
3233 selectWMMAModsNegAbs(ModOpcode, Mods, EltsF16, Src, CurDAG, SDLoc(In), in SelectWMMAModsF16NegAbs()
3238 if (auto *BV = dyn_cast<BuildVectorSDNode>(stripBitcast(In))) { in SelectWMMAModsF16NegAbs()
3253 selectWMMAModsNegAbs(ModOpcode, Mods, EltsV2F16, Src, CurDAG, SDLoc(In), in SelectWMMAModsF16NegAbs()
3257 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); in SelectWMMAModsF16NegAbs()
3261 bool AMDGPUDAGToDAGISel::SelectWMMAModsF32NegAbs(SDValue In, SDValue &Src, in SelectWMMAModsF32NegAbs() argument
3263 Src = In; in SelectWMMAModsF32NegAbs()
3268 if (auto *BV = dyn_cast<BuildVectorSDNode>(stripBitcast(In))) { in SelectWMMAModsF32NegAbs()
3281 selectWMMAModsNegAbs(ModOpcode, Mods, EltsF32, Src, CurDAG, SDLoc(In), in SelectWMMAModsF32NegAbs()
3285 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); in SelectWMMAModsF32NegAbs()
3289 bool AMDGPUDAGToDAGISel::SelectWMMAVISrc(SDValue In, SDValue &Src) const { in SelectWMMAVISrc() argument
3290 if (auto *BV = dyn_cast<BuildVectorSDNode>(In)) { in SelectWMMAVISrc()
3296 Src = CurDAG->getTargetConstant(Imm, SDLoc(In), MVT::i32); in SelectWMMAVISrc()
3301 Src = CurDAG->getTargetConstant(Imm, SDLoc(In), MVT::i32); in SelectWMMAVISrc()
3309 SDValue SplatSrc32 = stripBitcast(In); in SelectWMMAVISrc()
3320 Src = CurDAG->getTargetConstant(Imm, SDLoc(In), MVT::i16); in SelectWMMAVISrc()
3331 Src = CurDAG->getTargetConstant(Imm, SDLoc(In), MVT::i32); in SelectWMMAVISrc()
3343 bool AMDGPUDAGToDAGISel::SelectSWMMACIndex8(SDValue In, SDValue &Src, in SelectSWMMACIndex8() argument
3346 Src = In; in SelectSWMMACIndex8()
3348 if (In.getOpcode() == ISD::SRL) { in SelectSWMMACIndex8()
3349 const llvm::SDValue &ShiftSrc = In.getOperand(0); in SelectSWMMACIndex8()
3350 ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(In.getOperand(1)); in SelectSWMMACIndex8()
3358 IndexKey = CurDAG->getTargetConstant(Key, SDLoc(In), MVT::i32); in SelectSWMMACIndex8()
3362 bool AMDGPUDAGToDAGISel::SelectSWMMACIndex16(SDValue In, SDValue &Src, in SelectSWMMACIndex16() argument
3365 Src = In; in SelectSWMMACIndex16()
3367 if (In.getOpcode() == ISD::SRL) { in SelectSWMMACIndex16()
3368 const llvm::SDValue &ShiftSrc = In.getOperand(0); in SelectSWMMACIndex16()
3369 ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(In.getOperand(1)); in SelectSWMMACIndex16()
3377 IndexKey = CurDAG->getTargetConstant(Key, SDLoc(In), MVT::i32); in SelectSWMMACIndex16()
3381 bool AMDGPUDAGToDAGISel::SelectVOP3OpSel(SDValue In, SDValue &Src, in SelectVOP3OpSel() argument
3383 Src = In; in SelectVOP3OpSel()
3385 SrcMods = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32); in SelectVOP3OpSel()
3389 bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods(SDValue In, SDValue &Src, in SelectVOP3OpSelMods() argument
3392 return SelectVOP3Mods(In, Src, SrcMods); in SelectVOP3OpSelMods()
3397 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, in SelectVOP3PMadMixModsImpl() argument
3400 SelectVOP3ModsImpl(In, Src, Mods); in SelectVOP3PMadMixModsImpl()
3438 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsExt(SDValue In, SDValue &Src, in SelectVOP3PMadMixModsExt() argument
3441 if (!SelectVOP3PMadMixModsImpl(In, Src, Mods)) in SelectVOP3PMadMixModsExt()
3443 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); in SelectVOP3PMadMixModsExt()
3447 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixMods(SDValue In, SDValue &Src, in SelectVOP3PMadMixMods() argument
3450 SelectVOP3PMadMixModsImpl(In, Src, Mods); in SelectVOP3PMadMixMods()
3451 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); in SelectVOP3PMadMixMods()
3455 SDValue AMDGPUDAGToDAGISel::getHi16Elt(SDValue In) const { in getHi16Elt()
3456 if (In.isUndef()) in getHi16Elt()
3459 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(In)) { in getHi16Elt()
3460 SDLoc SL(In); in getHi16Elt()
3464 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(In)) { in getHi16Elt()
3465 SDLoc SL(In); in getHi16Elt()
3471 if (isExtractHiElt(In, Src)) in getHi16Elt()