Lines Matching refs:AMDGPU
133 case AMDGPU::V_MAC_F32_e64: in macToMad()
134 return AMDGPU::V_MAD_F32_e64; in macToMad()
135 case AMDGPU::V_MAC_F16_e64: in macToMad()
136 return AMDGPU::V_MAD_F16_e64; in macToMad()
137 case AMDGPU::V_FMAC_F32_e64: in macToMad()
138 return AMDGPU::V_FMA_F32_e64; in macToMad()
139 case AMDGPU::V_FMAC_F16_e64: in macToMad()
140 return AMDGPU::V_FMA_F16_gfx9_e64; in macToMad()
141 case AMDGPU::V_FMAC_LEGACY_F32_e64: in macToMad()
142 return AMDGPU::V_FMA_LEGACY_F32_e64; in macToMad()
143 case AMDGPU::V_FMAC_F64_e64: in macToMad()
144 return AMDGPU::V_FMA_F64_e64; in macToMad()
146 return AMDGPU::INSTRUCTION_LIST_END; in macToMad()
160 if (NewOpc != AMDGPU::INSTRUCTION_LIST_END) { in isInlineConstantIfFolded()
163 int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); in isInlineConstantIfFolded()
183 return OpNo == AMDGPU::getNamedOperandIdx(UseMI.getOpcode(), in frameIndexMayFold()
184 AMDGPU::OpName::vaddr); in frameIndexMayFold()
188 int SIdx = AMDGPU::getNamedOperandIdx(UseMI.getOpcode(), in frameIndexMayFold()
189 AMDGPU::OpName::saddr); in frameIndexMayFold()
193 int VIdx = AMDGPU::getNamedOperandIdx(UseMI.getOpcode(), in frameIndexMayFold()
194 AMDGPU::OpName::vaddr); in frameIndexMayFold()
213 AMDGPU::isFoldableLiteralV216(Fold.ImmToFold, in updateOperand()
220 if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0)) in updateOperand()
221 ModIdx = AMDGPU::OpName::src0_modifiers; in updateOperand()
222 else if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1)) in updateOperand()
223 ModIdx = AMDGPU::OpName::src1_modifiers; in updateOperand()
224 else if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2)) in updateOperand()
225 ModIdx = AMDGPU::OpName::src2_modifiers; in updateOperand()
227 ModIdx = AMDGPU::getNamedOperandIdx(Opcode, ModIdx); in updateOperand()
234 case AMDGPU::OPERAND_REG_IMM_V2FP16: in updateOperand()
235 case AMDGPU::OPERAND_REG_IMM_V2INT16: in updateOperand()
236 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: in updateOperand()
237 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: in updateOperand()
260 auto Liveness = MBB->computeRegisterLiveness(&TRI, AMDGPU::VCC, MI, 16); in updateOperand()
280 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::COPY), Dst1.getReg()) in updateOperand()
281 .addReg(AMDGPU::VCC, RegState::Kill); in updateOperand()
293 MI->setDesc(TII.get(AMDGPU::IMPLICIT_DEF)); in updateOperand()
354 if (NewOpc != AMDGPU::INSTRUCTION_LIST_END) { in tryAddToFoldList()
369 if (Opc == AMDGPU::S_SETREG_B32) in tryAddToFoldList()
370 ImmOpc = AMDGPU::S_SETREG_IMM32_B32; in tryAddToFoldList()
371 else if (Opc == AMDGPU::S_SETREG_B32_mode) in tryAddToFoldList()
372 ImmOpc = AMDGPU::S_SETREG_IMM32_B32_mode; in tryAddToFoldList()
415 if ((Opc == AMDGPU::V_ADD_CO_U32_e64 || in tryAddToFoldList()
416 Opc == AMDGPU::V_SUB_CO_U32_e64 || in tryAddToFoldList()
417 Opc == AMDGPU::V_SUBREV_CO_U32_e64) && // FIXME in tryAddToFoldList()
433 int Op32 = AMDGPU::getVOPe32(MaybeCommutedOpc); in tryAddToFoldList()
483 case AMDGPU::V_MOV_B32_e32: in isUseSafeToFold()
484 case AMDGPU::V_MOV_B32_e64: in isUseSafeToFold()
485 case AMDGPU::V_MOV_B64_PSEUDO: in isUseSafeToFold()
487 return !MI.hasRegisterImplicitUseOperand(AMDGPU::M0); in isUseSafeToFold()
541 if ((OpTy < AMDGPU::OPERAND_REG_INLINE_AC_FIRST || in tryToFoldACImm()
542 OpTy > AMDGPU::OPERAND_REG_INLINE_AC_LAST) && in tryToFoldACImm()
543 (OpTy < AMDGPU::OPERAND_REG_INLINE_C_FIRST || in tryToFoldACImm()
544 OpTy > AMDGPU::OPERAND_REG_INLINE_C_LAST)) in tryToFoldACImm()
617 if (UseOp.isImplicit() || UseOp.getSubReg() != AMDGPU::NoSubRegister) in foldOperand()
653 if (TII->getNamedOperand(*UseMI, AMDGPU::OpName::srsrc)->getReg() != in foldOperand()
660 *TII->getNamedOperand(*UseMI, AMDGPU::OpName::soffset); in foldOperand()
670 AMDGPU::getNamedOperandIdx(UseMI->getOpcode(), in foldOperand()
671 AMDGPU::OpName::vaddr) != -1) { in foldOperand()
672 unsigned NewOpc = AMDGPU::getFlatScratchInstSSfromSV(UseMI->getOpcode()); in foldOperand()
713 if (DestRC == &AMDGPU::AGPR_32RegClass && in foldOperand()
714 TII->isInlineConstant(OpToFold, AMDGPU::OPERAND_REG_INLINE_C_INT32)) { in foldOperand()
715 UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64)); in foldOperand()
726 if (MovOp == AMDGPU::COPY) in foldOperand()
757 getRegSeqInit(Defs, UseReg, AMDGPU::OPERAND_REG_INLINE_C_INT32, TII, in foldOperand()
762 UseMI->setDesc(TII->get(AMDGPU::REG_SEQUENCE)); in foldOperand()
773 TII->isInlineConstant(*Def, AMDGPU::OPERAND_REG_INLINE_C_INT32)) { in foldOperand()
776 auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass); in foldOperand()
778 TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), Tmp).addImm(Imm); in foldOperand()
803 auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass); in foldOperand()
804 BuildMI(MBB, UseMI, DL, TII->get(AMDGPU::COPY), Tmp).add(*Def); in foldOperand()
814 Vgpr = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in foldOperand()
815 BuildMI(MBB, UseMI, DL, TII->get(AMDGPU::COPY), Vgpr).add(*Def); in foldOperand()
818 auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass); in foldOperand()
820 TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), Tmp).addReg(Vgpr); in foldOperand()
834 UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64)); in foldOperand()
837 UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_READ_B32_e64)); in foldOperand()
841 UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_MOV_B32)); in foldOperand()
846 if (UseOpc == AMDGPU::V_READFIRSTLANE_B32 || in foldOperand()
847 (UseOpc == AMDGPU::V_READLANE_B32 && in foldOperand()
849 AMDGPU::getNamedOperandIdx(UseOpc, AMDGPU::OpName::src0))) { in foldOperand()
861 UseMI->setDesc(TII->get(AMDGPU::S_MOV_B32)); in foldOperand()
882 UseMI->setDesc(TII->get(AMDGPU::COPY)); in foldOperand()
916 if (UseOp.getSubReg() && AMDGPU::getRegBitWidth(FoldRC->getID()) == 64) { in foldOperand()
920 if (AMDGPU::getRegBitWidth(UseRC->getID()) != 64) in foldOperand()
924 if (UseOp.getSubReg() == AMDGPU::sub0) { in foldOperand()
927 assert(UseOp.getSubReg() == AMDGPU::sub1); in foldOperand()
944 case AMDGPU::V_AND_B32_e64: in evalBinaryInstruction()
945 case AMDGPU::V_AND_B32_e32: in evalBinaryInstruction()
946 case AMDGPU::S_AND_B32: in evalBinaryInstruction()
949 case AMDGPU::V_OR_B32_e64: in evalBinaryInstruction()
950 case AMDGPU::V_OR_B32_e32: in evalBinaryInstruction()
951 case AMDGPU::S_OR_B32: in evalBinaryInstruction()
954 case AMDGPU::V_XOR_B32_e64: in evalBinaryInstruction()
955 case AMDGPU::V_XOR_B32_e32: in evalBinaryInstruction()
956 case AMDGPU::S_XOR_B32: in evalBinaryInstruction()
959 case AMDGPU::S_XNOR_B32: in evalBinaryInstruction()
962 case AMDGPU::S_NAND_B32: in evalBinaryInstruction()
965 case AMDGPU::S_NOR_B32: in evalBinaryInstruction()
968 case AMDGPU::S_ANDN2_B32: in evalBinaryInstruction()
971 case AMDGPU::S_ORN2_B32: in evalBinaryInstruction()
974 case AMDGPU::V_LSHL_B32_e64: in evalBinaryInstruction()
975 case AMDGPU::V_LSHL_B32_e32: in evalBinaryInstruction()
976 case AMDGPU::S_LSHL_B32: in evalBinaryInstruction()
980 case AMDGPU::V_LSHLREV_B32_e64: in evalBinaryInstruction()
981 case AMDGPU::V_LSHLREV_B32_e32: in evalBinaryInstruction()
984 case AMDGPU::V_LSHR_B32_e64: in evalBinaryInstruction()
985 case AMDGPU::V_LSHR_B32_e32: in evalBinaryInstruction()
986 case AMDGPU::S_LSHR_B32: in evalBinaryInstruction()
989 case AMDGPU::V_LSHRREV_B32_e64: in evalBinaryInstruction()
990 case AMDGPU::V_LSHRREV_B32_e32: in evalBinaryInstruction()
993 case AMDGPU::V_ASHR_I32_e64: in evalBinaryInstruction()
994 case AMDGPU::V_ASHR_I32_e32: in evalBinaryInstruction()
995 case AMDGPU::S_ASHR_I32: in evalBinaryInstruction()
998 case AMDGPU::V_ASHRREV_I32_e64: in evalBinaryInstruction()
999 case AMDGPU::V_ASHRREV_I32_e32: in evalBinaryInstruction()
1008 return IsScalar ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; in getMovOpc()
1033 if (Op.getSubReg() != AMDGPU::NoSubRegister || !Op.getReg().isVirtual()) in getImmOrMaterializedImm()
1054 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); in tryConstantFoldOp()
1059 if ((Opc == AMDGPU::V_NOT_B32_e64 || Opc == AMDGPU::V_NOT_B32_e32 || in tryConstantFoldOp()
1060 Opc == AMDGPU::S_NOT_B32) && in tryConstantFoldOp()
1063 mutateCopyOp(*MI, TII->get(getMovOpc(Opc == AMDGPU::S_NOT_B32))); in tryConstantFoldOp()
1067 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); in tryConstantFoldOp()
1103 if (Opc == AMDGPU::V_OR_B32_e64 || in tryConstantFoldOp()
1104 Opc == AMDGPU::V_OR_B32_e32 || in tryConstantFoldOp()
1105 Opc == AMDGPU::S_OR_B32) { in tryConstantFoldOp()
1109 mutateCopyOp(*MI, TII->get(AMDGPU::COPY)); in tryConstantFoldOp()
1113 mutateCopyOp(*MI, TII->get(getMovOpc(Opc == AMDGPU::S_OR_B32))); in tryConstantFoldOp()
1120 if (MI->getOpcode() == AMDGPU::V_AND_B32_e64 || in tryConstantFoldOp()
1121 MI->getOpcode() == AMDGPU::V_AND_B32_e32 || in tryConstantFoldOp()
1122 MI->getOpcode() == AMDGPU::S_AND_B32) { in tryConstantFoldOp()
1126 mutateCopyOp(*MI, TII->get(getMovOpc(Opc == AMDGPU::S_AND_B32))); in tryConstantFoldOp()
1130 mutateCopyOp(*MI, TII->get(AMDGPU::COPY)); in tryConstantFoldOp()
1138 if (MI->getOpcode() == AMDGPU::V_XOR_B32_e64 || in tryConstantFoldOp()
1139 MI->getOpcode() == AMDGPU::V_XOR_B32_e32 || in tryConstantFoldOp()
1140 MI->getOpcode() == AMDGPU::S_XOR_B32) { in tryConstantFoldOp()
1144 mutateCopyOp(*MI, TII->get(AMDGPU::COPY)); in tryConstantFoldOp()
1155 if (Opc != AMDGPU::V_CNDMASK_B32_e32 && Opc != AMDGPU::V_CNDMASK_B32_e64 && in tryFoldCndMask()
1156 Opc != AMDGPU::V_CNDMASK_B64_PSEUDO) in tryFoldCndMask()
1159 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in tryFoldCndMask()
1160 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in tryFoldCndMask()
1169 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1_modifiers); in tryFoldCndMask()
1171 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers); in tryFoldCndMask()
1178 TII->get(Src0->isReg() ? (unsigned)AMDGPU::COPY : getMovOpc(false)); in tryFoldCndMask()
1179 int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); in tryFoldCndMask()
1182 MI.RemoveOperand(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1)); in tryFoldCndMask()
1193 if (MI.getOpcode() != AMDGPU::V_AND_B32_e64 && in tryFoldZeroHighBits()
1194 MI.getOpcode() != AMDGPU::V_AND_B32_e32) in tryFoldZeroHighBits()
1303 if (DefMI->readsRegister(AMDGPU::EXEC, TRI) && in foldInstOperand()
1331 case AMDGPU::V_MAX_F32_e64: in isClamp()
1332 case AMDGPU::V_MAX_F16_e64: in isClamp()
1333 case AMDGPU::V_MAX_F64_e64: in isClamp()
1334 case AMDGPU::V_PK_MAX_F16: { in isClamp()
1335 if (!TII->getNamedOperand(MI, AMDGPU::OpName::clamp)->getImm()) in isClamp()
1339 const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in isClamp()
1340 const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in isClamp()
1344 Src0->getSubReg() != AMDGPU::NoSubRegister) in isClamp()
1348 if (TII->hasModifiersSet(MI, AMDGPU::OpName::omod)) in isClamp()
1352 = TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm(); in isClamp()
1354 = TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm(); in isClamp()
1358 unsigned UnsetMods = (Op == AMDGPU::V_PK_MAX_F16) ? SISrcMods::OP_SEL_1 in isClamp()
1381 MachineOperand *DefClamp = TII->getNamedOperand(*Def, AMDGPU::OpName::clamp); in tryFoldClamp()
1396 case AMDGPU::V_MUL_F64_e64: { in getOModValue()
1408 case AMDGPU::V_MUL_F32_e64: { in getOModValue()
1420 case AMDGPU::V_MUL_F16_e64: { in getOModValue()
1444 case AMDGPU::V_MUL_F64_e64: in isOMod()
1445 case AMDGPU::V_MUL_F32_e64: in isOMod()
1446 case AMDGPU::V_MUL_F16_e64: { in isOMod()
1448 if ((Op == AMDGPU::V_MUL_F32_e64 && MFI->getMode().FP32OutputDenormals) || in isOMod()
1449 ((Op == AMDGPU::V_MUL_F64_e64 || Op == AMDGPU::V_MUL_F16_e64) && in isOMod()
1455 const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in isOMod()
1456 const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in isOMod()
1468 TII->hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) || in isOMod()
1469 TII->hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) || in isOMod()
1470 TII->hasModifiersSet(MI, AMDGPU::OpName::omod) || in isOMod()
1471 TII->hasModifiersSet(MI, AMDGPU::OpName::clamp)) in isOMod()
1476 case AMDGPU::V_ADD_F64_e64: in isOMod()
1477 case AMDGPU::V_ADD_F32_e64: in isOMod()
1478 case AMDGPU::V_ADD_F16_e64: { in isOMod()
1480 if ((Op == AMDGPU::V_ADD_F32_e64 && MFI->getMode().FP32OutputDenormals) || in isOMod()
1481 ((Op == AMDGPU::V_ADD_F64_e64 || Op == AMDGPU::V_ADD_F16_e64) && in isOMod()
1486 const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in isOMod()
1487 const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in isOMod()
1491 !TII->hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) && in isOMod()
1492 !TII->hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) && in isOMod()
1493 !TII->hasModifiersSet(MI, AMDGPU::OpName::clamp) && in isOMod()
1494 !TII->hasModifiersSet(MI, AMDGPU::OpName::omod)) in isOMod()
1510 RegOp->getSubReg() != AMDGPU::NoSubRegister || in tryFoldOMod()
1515 MachineOperand *DefOMod = TII->getNamedOperand(*Def, AMDGPU::OpName::omod); in tryFoldOMod()
1521 if (TII->hasModifiersSet(*Def, AMDGPU::OpName::clamp)) in tryFoldOMod()
1577 case AMDGPU::AV_32RegClassID: LLVM_FALLTHROUGH; in tryFoldRegSequence()
1578 case AMDGPU::AV_64RegClassID: LLVM_FALLTHROUGH; in tryFoldRegSequence()
1579 case AMDGPU::AV_96RegClassID: LLVM_FALLTHROUGH; in tryFoldRegSequence()
1580 case AMDGPU::AV_128RegClassID: LLVM_FALLTHROUGH; in tryFoldRegSequence()
1581 case AMDGPU::AV_160RegClassID: in tryFoldRegSequence()
1590 TII->get(AMDGPU::REG_SEQUENCE), Dst); in tryFoldRegSequence()
1661 TII->get(AMDGPU::COPY), PhiOut) in tryFoldLCSSAPhi()
1760 if (CurrentKnownM0Val && MI.modifiesRegister(AMDGPU::M0, TRI)) in runOnMachineFunction()
1774 if (MI.getOperand(0).getReg() == AMDGPU::M0) { in runOnMachineFunction()