Lines Matching refs:AMDGPU
160 case AMDGPU::V_MAC_F32_e64: in macToMad()
161 return AMDGPU::V_MAD_F32_e64; in macToMad()
162 case AMDGPU::V_MAC_F16_e64: in macToMad()
163 return AMDGPU::V_MAD_F16_e64; in macToMad()
164 case AMDGPU::V_FMAC_F32_e64: in macToMad()
165 return AMDGPU::V_FMA_F32_e64; in macToMad()
166 case AMDGPU::V_FMAC_F16_e64: in macToMad()
167 return AMDGPU::V_FMA_F16_gfx9_e64; in macToMad()
168 case AMDGPU::V_FMAC_F16_t16_e64: in macToMad()
169 return AMDGPU::V_FMA_F16_gfx9_e64; in macToMad()
170 case AMDGPU::V_FMAC_LEGACY_F32_e64: in macToMad()
171 return AMDGPU::V_FMA_LEGACY_F32_e64; in macToMad()
172 case AMDGPU::V_FMAC_F64_e64: in macToMad()
173 return AMDGPU::V_FMA_F64_e64; in macToMad()
175 return AMDGPU::INSTRUCTION_LIST_END; in macToMad()
187 return OpNo == AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr); in frameIndexMayFold()
191 int SIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr); in frameIndexMayFold()
195 int VIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr); in frameIndexMayFold()
221 case AMDGPU::OPERAND_REG_IMM_V2FP16: in canUseImmWithOpSel()
222 case AMDGPU::OPERAND_REG_IMM_V2INT16: in canUseImmWithOpSel()
223 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: in canUseImmWithOpSel()
224 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: in canUseImmWithOpSel()
241 if (AMDGPU::isInlinableLiteralV216(Fold.ImmToFold, OpType)) { in tryFoldImmWithOpSel()
250 if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0)) { in tryFoldImmWithOpSel()
251 ModIdx = AMDGPU::OpName::src0_modifiers; in tryFoldImmWithOpSel()
253 } else if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1)) { in tryFoldImmWithOpSel()
254 ModIdx = AMDGPU::OpName::src1_modifiers; in tryFoldImmWithOpSel()
256 } else if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2)) { in tryFoldImmWithOpSel()
257 ModIdx = AMDGPU::OpName::src2_modifiers; in tryFoldImmWithOpSel()
261 ModIdx = AMDGPU::getNamedOperandIdx(Opcode, ModIdx); in tryFoldImmWithOpSel()
275 if (AMDGPU::isInlinableLiteralV216(Imm, OpType)) { in tryFoldImmWithOpSel()
286 if (AMDGPU::isInlinableLiteralV216(Lo, OpType)) { in tryFoldImmWithOpSel()
294 if (AMDGPU::isInlinableLiteralV216(SExt, OpType)) { in tryFoldImmWithOpSel()
302 if (OpType == AMDGPU::OPERAND_REG_IMM_V2INT16 || in tryFoldImmWithOpSel()
303 OpType == AMDGPU::OPERAND_REG_INLINE_AC_V2INT16) { in tryFoldImmWithOpSel()
304 if (AMDGPU::isInlinableLiteralV216(Lo << 16, OpType)) { in tryFoldImmWithOpSel()
312 if (AMDGPU::isInlinableLiteralV216(Swapped, OpType)) { in tryFoldImmWithOpSel()
331 bool IsUAdd = Opcode == AMDGPU::V_PK_ADD_U16; in tryFoldImmWithOpSel()
332 bool IsUSub = Opcode == AMDGPU::V_PK_SUB_U16; in tryFoldImmWithOpSel()
335 AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::clamp); in tryFoldImmWithOpSel()
345 IsUAdd ? AMDGPU::V_PK_SUB_U16 : AMDGPU::V_PK_ADD_U16; in tryFoldImmWithOpSel()
376 auto Liveness = MBB->computeRegisterLiveness(TRI, AMDGPU::VCC, MI, 16); in updateOperand()
395 BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(AMDGPU::COPY), in updateOperand()
397 .addReg(AMDGPU::VCC, RegState::Kill); in updateOperand()
409 MI->setDesc(TII->get(AMDGPU::IMPLICIT_DEF)); in updateOperand()
420 int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(MI->getOpcode()); in updateOperand()
475 const unsigned NewOpc = TryAK ? AMDGPU::S_FMAAK_F32 : AMDGPU::S_FMAMK_F32; in tryAddToFoldList()
513 if (NewOpc != AMDGPU::INSTRUCTION_LIST_END) { in tryAddToFoldList()
517 bool AddOpSel = !AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::op_sel) && in tryAddToFoldList()
518 AMDGPU::hasNamedOperand(NewOpc, AMDGPU::OpName::op_sel); in tryAddToFoldList()
533 if (Opc == AMDGPU::S_FMAC_F32 && OpNo == 3) { in tryAddToFoldList()
541 if (Opc == AMDGPU::S_SETREG_B32) in tryAddToFoldList()
542 ImmOpc = AMDGPU::S_SETREG_IMM32_B32; in tryAddToFoldList()
543 else if (Opc == AMDGPU::S_SETREG_B32_mode) in tryAddToFoldList()
544 ImmOpc = AMDGPU::S_SETREG_IMM32_B32_mode; in tryAddToFoldList()
577 if ((Opc != AMDGPU::V_ADD_CO_U32_e64 && Opc != AMDGPU::V_SUB_CO_U32_e64 && in tryAddToFoldList()
578 Opc != AMDGPU::V_SUBREV_CO_U32_e64) || // FIXME in tryAddToFoldList()
595 Op32 = AMDGPU::getVOPe32(MaybeCommutedOpc); in tryAddToFoldList()
604 if ((Opc == AMDGPU::S_FMAAK_F32 || Opc == AMDGPU::S_FMAMK_F32) && in tryAddToFoldList()
606 unsigned ImmIdx = Opc == AMDGPU::S_FMAAK_F32 ? 3 : 2; in tryAddToFoldList()
618 if (Opc == AMDGPU::S_FMAC_F32 && in tryAddToFoldList()
694 if (!AMDGPU::isSISrcInlinableOperand(Desc, UseOpIdx)) in tryToFoldACImm()
766 (UseOp.isImplicit() || UseOp.getSubReg() != AMDGPU::NoSubRegister)) in foldOperand()
801 if (TII->getNamedOperand(*UseMI, AMDGPU::OpName::srsrc)->getReg() != in foldOperand()
808 *TII->getNamedOperand(*UseMI, AMDGPU::OpName::soffset); in foldOperand()
819 AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::vaddr) && in foldOperand()
820 !AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::saddr)) { in foldOperand()
821 unsigned NewOpc = AMDGPU::getFlatScratchInstSSfromSV(Opc); in foldOperand()
846 if (DestRC == &AMDGPU::AGPR_32RegClass && in foldOperand()
847 TII->isInlineConstant(OpToFold, AMDGPU::OPERAND_REG_INLINE_C_INT32)) { in foldOperand()
848 UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64)); in foldOperand()
859 if (MovOp == AMDGPU::COPY) in foldOperand()
893 getRegSeqInit(Defs, UseReg, AMDGPU::OPERAND_REG_INLINE_C_INT32)) { in foldOperand()
897 UseMI->setDesc(TII->get(AMDGPU::REG_SEQUENCE)); in foldOperand()
908 TII->isInlineConstant(*Def, AMDGPU::OPERAND_REG_INLINE_C_INT32)) { in foldOperand()
911 auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass); in foldOperand()
913 TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), Tmp).addImm(Imm); in foldOperand()
938 auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass); in foldOperand()
939 BuildMI(MBB, UseMI, DL, TII->get(AMDGPU::COPY), Tmp).add(*Def); in foldOperand()
949 Vgpr = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in foldOperand()
950 BuildMI(MBB, UseMI, DL, TII->get(AMDGPU::COPY), Vgpr).add(*Def); in foldOperand()
953 auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass); in foldOperand()
955 TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), Tmp).addReg(Vgpr); in foldOperand()
971 UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64)); in foldOperand()
973 UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_READ_B32_e64)); in foldOperand()
976 UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_MOV_B32)); in foldOperand()
981 if (UseOpc == AMDGPU::V_READFIRSTLANE_B32 || in foldOperand()
982 (UseOpc == AMDGPU::V_READLANE_B32 && in foldOperand()
984 AMDGPU::getNamedOperandIdx(UseOpc, AMDGPU::OpName::src0))) { in foldOperand()
996 UseMI->setDesc(TII->get(AMDGPU::S_MOV_B32)); in foldOperand()
1017 UseMI->setDesc(TII->get(AMDGPU::COPY)); in foldOperand()
1065 if (UseOp.getSubReg() && AMDGPU::getRegBitWidth(*FoldRC) == 64) { in foldOperand()
1068 if (AMDGPU::getRegBitWidth(*UseRC) != 64) in foldOperand()
1072 if (UseOp.getSubReg() == AMDGPU::sub0) { in foldOperand()
1075 assert(UseOp.getSubReg() == AMDGPU::sub1); in foldOperand()
1090 case AMDGPU::V_AND_B32_e64: in evalBinaryInstruction()
1091 case AMDGPU::V_AND_B32_e32: in evalBinaryInstruction()
1092 case AMDGPU::S_AND_B32: in evalBinaryInstruction()
1095 case AMDGPU::V_OR_B32_e64: in evalBinaryInstruction()
1096 case AMDGPU::V_OR_B32_e32: in evalBinaryInstruction()
1097 case AMDGPU::S_OR_B32: in evalBinaryInstruction()
1100 case AMDGPU::V_XOR_B32_e64: in evalBinaryInstruction()
1101 case AMDGPU::V_XOR_B32_e32: in evalBinaryInstruction()
1102 case AMDGPU::S_XOR_B32: in evalBinaryInstruction()
1105 case AMDGPU::S_XNOR_B32: in evalBinaryInstruction()
1108 case AMDGPU::S_NAND_B32: in evalBinaryInstruction()
1111 case AMDGPU::S_NOR_B32: in evalBinaryInstruction()
1114 case AMDGPU::S_ANDN2_B32: in evalBinaryInstruction()
1117 case AMDGPU::S_ORN2_B32: in evalBinaryInstruction()
1120 case AMDGPU::V_LSHL_B32_e64: in evalBinaryInstruction()
1121 case AMDGPU::V_LSHL_B32_e32: in evalBinaryInstruction()
1122 case AMDGPU::S_LSHL_B32: in evalBinaryInstruction()
1126 case AMDGPU::V_LSHLREV_B32_e64: in evalBinaryInstruction()
1127 case AMDGPU::V_LSHLREV_B32_e32: in evalBinaryInstruction()
1130 case AMDGPU::V_LSHR_B32_e64: in evalBinaryInstruction()
1131 case AMDGPU::V_LSHR_B32_e32: in evalBinaryInstruction()
1132 case AMDGPU::S_LSHR_B32: in evalBinaryInstruction()
1135 case AMDGPU::V_LSHRREV_B32_e64: in evalBinaryInstruction()
1136 case AMDGPU::V_LSHRREV_B32_e32: in evalBinaryInstruction()
1139 case AMDGPU::V_ASHR_I32_e64: in evalBinaryInstruction()
1140 case AMDGPU::V_ASHR_I32_e32: in evalBinaryInstruction()
1141 case AMDGPU::S_ASHR_I32: in evalBinaryInstruction()
1144 case AMDGPU::V_ASHRREV_I32_e64: in evalBinaryInstruction()
1145 case AMDGPU::V_ASHRREV_I32_e32: in evalBinaryInstruction()
1154 return IsScalar ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; in getMovOpc()
1174 if (!Op.isReg() || Op.getSubReg() != AMDGPU::NoSubRegister || in getImmOrMaterializedImm()
1197 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); in tryConstantFoldOp()
1202 if ((Opc == AMDGPU::V_NOT_B32_e64 || Opc == AMDGPU::V_NOT_B32_e32 || in tryConstantFoldOp()
1203 Opc == AMDGPU::S_NOT_B32) && in tryConstantFoldOp()
1206 mutateCopyOp(*MI, TII->get(getMovOpc(Opc == AMDGPU::S_NOT_B32))); in tryConstantFoldOp()
1210 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); in tryConstantFoldOp()
1245 if (Opc == AMDGPU::V_OR_B32_e64 || in tryConstantFoldOp()
1246 Opc == AMDGPU::V_OR_B32_e32 || in tryConstantFoldOp()
1247 Opc == AMDGPU::S_OR_B32) { in tryConstantFoldOp()
1251 mutateCopyOp(*MI, TII->get(AMDGPU::COPY)); in tryConstantFoldOp()
1255 mutateCopyOp(*MI, TII->get(getMovOpc(Opc == AMDGPU::S_OR_B32))); in tryConstantFoldOp()
1262 if (Opc == AMDGPU::V_AND_B32_e64 || Opc == AMDGPU::V_AND_B32_e32 || in tryConstantFoldOp()
1263 Opc == AMDGPU::S_AND_B32) { in tryConstantFoldOp()
1267 mutateCopyOp(*MI, TII->get(getMovOpc(Opc == AMDGPU::S_AND_B32))); in tryConstantFoldOp()
1271 mutateCopyOp(*MI, TII->get(AMDGPU::COPY)); in tryConstantFoldOp()
1278 if (Opc == AMDGPU::V_XOR_B32_e64 || Opc == AMDGPU::V_XOR_B32_e32 || in tryConstantFoldOp()
1279 Opc == AMDGPU::S_XOR_B32) { in tryConstantFoldOp()
1283 mutateCopyOp(*MI, TII->get(AMDGPU::COPY)); in tryConstantFoldOp()
1294 if (Opc != AMDGPU::V_CNDMASK_B32_e32 && Opc != AMDGPU::V_CNDMASK_B32_e64 && in tryFoldCndMask()
1295 Opc != AMDGPU::V_CNDMASK_B64_PSEUDO) in tryFoldCndMask()
1298 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in tryFoldCndMask()
1299 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in tryFoldCndMask()
1308 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1_modifiers); in tryFoldCndMask()
1310 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers); in tryFoldCndMask()
1317 TII->get(Src0->isReg() ? (unsigned)AMDGPU::COPY : getMovOpc(false)); in tryFoldCndMask()
1318 int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); in tryFoldCndMask()
1321 MI.removeOperand(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1)); in tryFoldCndMask()
1332 if (MI.getOpcode() != AMDGPU::V_AND_B32_e64 && in tryFoldZeroHighBits()
1333 MI.getOpcode() != AMDGPU::V_AND_B32_e32) in tryFoldZeroHighBits()
1401 if (DefMI->readsRegister(AMDGPU::EXEC, TRI) && in foldInstOperand()
1429 if (MI.getOperand(0).getReg() == AMDGPU::M0) { in tryFoldFoldableCopy()
1497 case AMDGPU::V_MAX_F32_e64: in isClamp()
1498 case AMDGPU::V_MAX_F16_e64: in isClamp()
1499 case AMDGPU::V_MAX_F16_t16_e64: in isClamp()
1500 case AMDGPU::V_MAX_F16_fake16_e64: in isClamp()
1501 case AMDGPU::V_MAX_F64_e64: in isClamp()
1502 case AMDGPU::V_MAX_NUM_F64_e64: in isClamp()
1503 case AMDGPU::V_PK_MAX_F16: { in isClamp()
1504 if (!TII->getNamedOperand(MI, AMDGPU::OpName::clamp)->getImm()) in isClamp()
1508 const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in isClamp()
1509 const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in isClamp()
1513 Src0->getSubReg() != AMDGPU::NoSubRegister) in isClamp()
1517 if (TII->hasModifiersSet(MI, AMDGPU::OpName::omod)) in isClamp()
1521 = TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm(); in isClamp()
1523 = TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm(); in isClamp()
1527 unsigned UnsetMods = (Op == AMDGPU::V_PK_MAX_F16) ? SISrcMods::OP_SEL_1 in isClamp()
1550 MachineOperand *DefClamp = TII->getNamedOperand(*Def, AMDGPU::OpName::clamp); in tryFoldClamp()
1572 case AMDGPU::V_MUL_F64_e64: in getOModValue()
1573 case AMDGPU::V_MUL_F64_pseudo_e64: { in getOModValue()
1585 case AMDGPU::V_MUL_F32_e64: { in getOModValue()
1597 case AMDGPU::V_MUL_F16_e64: in getOModValue()
1598 case AMDGPU::V_MUL_F16_t16_e64: in getOModValue()
1599 case AMDGPU::V_MUL_F16_fake16_e64: { in getOModValue()
1623 case AMDGPU::V_MUL_F64_e64: in isOMod()
1624 case AMDGPU::V_MUL_F64_pseudo_e64: in isOMod()
1625 case AMDGPU::V_MUL_F32_e64: in isOMod()
1626 case AMDGPU::V_MUL_F16_t16_e64: in isOMod()
1627 case AMDGPU::V_MUL_F16_fake16_e64: in isOMod()
1628 case AMDGPU::V_MUL_F16_e64: { in isOMod()
1630 if ((Op == AMDGPU::V_MUL_F32_e64 && in isOMod()
1632 ((Op == AMDGPU::V_MUL_F64_e64 || Op == AMDGPU::V_MUL_F64_pseudo_e64 || in isOMod()
1633 Op == AMDGPU::V_MUL_F16_e64 || Op == AMDGPU::V_MUL_F16_t16_e64 || in isOMod()
1634 Op == AMDGPU::V_MUL_F16_fake16_e64) && in isOMod()
1640 const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in isOMod()
1641 const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in isOMod()
1653 TII->hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) || in isOMod()
1654 TII->hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) || in isOMod()
1655 TII->hasModifiersSet(MI, AMDGPU::OpName::omod) || in isOMod()
1656 TII->hasModifiersSet(MI, AMDGPU::OpName::clamp)) in isOMod()
1661 case AMDGPU::V_ADD_F64_e64: in isOMod()
1662 case AMDGPU::V_ADD_F64_pseudo_e64: in isOMod()
1663 case AMDGPU::V_ADD_F32_e64: in isOMod()
1664 case AMDGPU::V_ADD_F16_e64: in isOMod()
1665 case AMDGPU::V_ADD_F16_t16_e64: in isOMod()
1666 case AMDGPU::V_ADD_F16_fake16_e64: { in isOMod()
1668 if ((Op == AMDGPU::V_ADD_F32_e64 && in isOMod()
1670 ((Op == AMDGPU::V_ADD_F64_e64 || Op == AMDGPU::V_ADD_F64_pseudo_e64 || in isOMod()
1671 Op == AMDGPU::V_ADD_F16_e64 || Op == AMDGPU::V_ADD_F16_t16_e64 || in isOMod()
1672 Op == AMDGPU::V_ADD_F16_fake16_e64) && in isOMod()
1677 const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in isOMod()
1678 const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in isOMod()
1682 !TII->hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) && in isOMod()
1683 !TII->hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) && in isOMod()
1684 !TII->hasModifiersSet(MI, AMDGPU::OpName::clamp) && in isOMod()
1685 !TII->hasModifiersSet(MI, AMDGPU::OpName::omod)) in isOMod()
1701 RegOp->getSubReg() != AMDGPU::NoSubRegister || in tryFoldOMod()
1706 MachineOperand *DefOMod = TII->getNamedOperand(*Def, AMDGPU::OpName::omod); in tryFoldOMod()
1712 if (TII->hasModifiersSet(*Def, AMDGPU::OpName::clamp)) in tryFoldOMod()
1781 TII->get(AMDGPU::REG_SEQUENCE), Dst); in tryFoldRegSequence()
1842 CopySrcDef->getOperand(0).getSubReg() != AMDGPU::NoSubRegister || in isAGPRCopy()
1843 OtherCopySrc.getSubReg() != AMDGPU::NoSubRegister || in isAGPRCopy()
1898 unsigned AGPRRegMask = AMDGPU::NoSubRegister; in tryFoldPhiAGPR()
1914 bool IsAGPR32 = (ARC == &AMDGPU::AGPR_32RegClass); in tryFoldPhiAGPR()
1926 unsigned CopyOpc = AMDGPU::COPY; in tryFoldPhiAGPR()
1933 unsigned AGPRSubReg = AMDGPU::NoSubRegister; in tryFoldPhiAGPR()
1949 CopyOpc = AMDGPU::V_ACCVGPR_WRITE_B32_e64; in tryFoldPhiAGPR()
1977 TII->get(AMDGPU::COPY), PhiOut) in tryFoldPhiAGPR()
2113 TII->get(AMDGPU::V_ACCVGPR_READ_B32_e64), TempVGPR) in tryOptimizeAGPRPhis()
2119 TII->get(AMDGPU::COPY), TempAGPR) in tryOptimizeAGPRPhis()
2125 MO->setSubReg(AMDGPU::NoSubRegister); in tryOptimizeAGPRPhis()
2184 if (CurrentKnownM0Val && MI.modifiesRegister(AMDGPU::M0, TRI)) in runOnMachineFunction()