Lines Matching refs:AMDGPU
133 case AMDGPU::V_MAC_F32_e64: in macToMad()
134 return AMDGPU::V_MAD_F32_e64; in macToMad()
135 case AMDGPU::V_MAC_F16_e64: in macToMad()
136 return AMDGPU::V_MAD_F16_e64; in macToMad()
137 case AMDGPU::V_FMAC_F32_e64: in macToMad()
138 return AMDGPU::V_FMA_F32_e64; in macToMad()
139 case AMDGPU::V_FMAC_F16_e64: in macToMad()
140 return AMDGPU::V_FMA_F16_gfx9_e64; in macToMad()
141 case AMDGPU::V_FMAC_LEGACY_F32_e64: in macToMad()
142 return AMDGPU::V_FMA_LEGACY_F32_e64; in macToMad()
143 case AMDGPU::V_FMAC_F64_e64: in macToMad()
144 return AMDGPU::V_FMA_F64_e64; in macToMad()
146 return AMDGPU::INSTRUCTION_LIST_END; in macToMad()
159 return OpNo == AMDGPU::getNamedOperandIdx(UseMI.getOpcode(), in frameIndexMayFold()
160 AMDGPU::OpName::vaddr); in frameIndexMayFold()
164 int SIdx = AMDGPU::getNamedOperandIdx(UseMI.getOpcode(), in frameIndexMayFold()
165 AMDGPU::OpName::saddr); in frameIndexMayFold()
169 int VIdx = AMDGPU::getNamedOperandIdx(UseMI.getOpcode(), in frameIndexMayFold()
170 AMDGPU::OpName::vaddr); in frameIndexMayFold()
191 AMDGPU::isFoldableLiteralV216(Fold.ImmToFold, in updateOperand()
198 if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0)) in updateOperand()
199 ModIdx = AMDGPU::OpName::src0_modifiers; in updateOperand()
200 else if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1)) in updateOperand()
201 ModIdx = AMDGPU::OpName::src1_modifiers; in updateOperand()
202 else if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2)) in updateOperand()
203 ModIdx = AMDGPU::OpName::src2_modifiers; in updateOperand()
205 ModIdx = AMDGPU::getNamedOperandIdx(Opcode, ModIdx); in updateOperand()
212 case AMDGPU::OPERAND_REG_IMM_V2FP16: in updateOperand()
213 case AMDGPU::OPERAND_REG_IMM_V2INT16: in updateOperand()
214 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: in updateOperand()
215 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: in updateOperand()
238 auto Liveness = MBB->computeRegisterLiveness(&TRI, AMDGPU::VCC, MI, 16); in updateOperand()
258 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::COPY), Dst1.getReg()) in updateOperand()
259 .addReg(AMDGPU::VCC, RegState::Kill); in updateOperand()
271 MI->setDesc(TII.get(AMDGPU::IMPLICIT_DEF)); in updateOperand()
282 int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(MI->getOpcode()); in updateOperand()
339 if (NewOpc != AMDGPU::INSTRUCTION_LIST_END) { in tryAddToFoldList()
354 if (Opc == AMDGPU::S_SETREG_B32) in tryAddToFoldList()
355 ImmOpc = AMDGPU::S_SETREG_IMM32_B32; in tryAddToFoldList()
356 else if (Opc == AMDGPU::S_SETREG_B32_mode) in tryAddToFoldList()
357 ImmOpc = AMDGPU::S_SETREG_IMM32_B32_mode; in tryAddToFoldList()
400 if ((Opc == AMDGPU::V_ADD_CO_U32_e64 || in tryAddToFoldList()
401 Opc == AMDGPU::V_SUB_CO_U32_e64 || in tryAddToFoldList()
402 Opc == AMDGPU::V_SUBREV_CO_U32_e64) && // FIXME in tryAddToFoldList()
418 int Op32 = AMDGPU::getVOPe32(MaybeCommutedOpc); in tryAddToFoldList()
468 case AMDGPU::V_MOV_B32_e32: in isUseSafeToFold()
469 case AMDGPU::V_MOV_B32_e64: in isUseSafeToFold()
470 case AMDGPU::V_MOV_B64_PSEUDO: in isUseSafeToFold()
471 case AMDGPU::V_MOV_B64_e32: in isUseSafeToFold()
472 case AMDGPU::V_MOV_B64_e64: in isUseSafeToFold()
474 return !MI.hasRegisterImplicitUseOperand(AMDGPU::M0); in isUseSafeToFold()
528 if ((OpTy < AMDGPU::OPERAND_REG_INLINE_AC_FIRST || in tryToFoldACImm()
529 OpTy > AMDGPU::OPERAND_REG_INLINE_AC_LAST) && in tryToFoldACImm()
530 (OpTy < AMDGPU::OPERAND_REG_INLINE_C_FIRST || in tryToFoldACImm()
531 OpTy > AMDGPU::OPERAND_REG_INLINE_C_LAST)) in tryToFoldACImm()
604 if (UseOp.isImplicit() || UseOp.getSubReg() != AMDGPU::NoSubRegister) in foldOperand()
640 if (TII->getNamedOperand(*UseMI, AMDGPU::OpName::srsrc)->getReg() != in foldOperand()
647 *TII->getNamedOperand(*UseMI, AMDGPU::OpName::soffset); in foldOperand()
657 AMDGPU::getNamedOperandIdx(UseMI->getOpcode(), in foldOperand()
658 AMDGPU::OpName::vaddr) != -1 && in foldOperand()
659 AMDGPU::getNamedOperandIdx(UseMI->getOpcode(), in foldOperand()
660 AMDGPU::OpName::saddr) == -1) { in foldOperand()
661 unsigned NewOpc = AMDGPU::getFlatScratchInstSSfromSV(UseMI->getOpcode()); in foldOperand()
702 if (DestRC == &AMDGPU::AGPR_32RegClass && in foldOperand()
703 TII->isInlineConstant(OpToFold, AMDGPU::OPERAND_REG_INLINE_C_INT32)) { in foldOperand()
704 UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64)); in foldOperand()
715 if (MovOp == AMDGPU::COPY) in foldOperand()
746 getRegSeqInit(Defs, UseReg, AMDGPU::OPERAND_REG_INLINE_C_INT32, TII, in foldOperand()
751 UseMI->setDesc(TII->get(AMDGPU::REG_SEQUENCE)); in foldOperand()
762 TII->isInlineConstant(*Def, AMDGPU::OPERAND_REG_INLINE_C_INT32)) { in foldOperand()
765 auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass); in foldOperand()
767 TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), Tmp).addImm(Imm); in foldOperand()
792 auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass); in foldOperand()
793 BuildMI(MBB, UseMI, DL, TII->get(AMDGPU::COPY), Tmp).add(*Def); in foldOperand()
803 Vgpr = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in foldOperand()
804 BuildMI(MBB, UseMI, DL, TII->get(AMDGPU::COPY), Vgpr).add(*Def); in foldOperand()
807 auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass); in foldOperand()
809 TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), Tmp).addReg(Vgpr); in foldOperand()
823 UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64)); in foldOperand()
826 UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_READ_B32_e64)); in foldOperand()
830 UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_MOV_B32)); in foldOperand()
835 if (UseOpc == AMDGPU::V_READFIRSTLANE_B32 || in foldOperand()
836 (UseOpc == AMDGPU::V_READLANE_B32 && in foldOperand()
838 AMDGPU::getNamedOperandIdx(UseOpc, AMDGPU::OpName::src0))) { in foldOperand()
850 UseMI->setDesc(TII->get(AMDGPU::S_MOV_B32)); in foldOperand()
871 UseMI->setDesc(TII->get(AMDGPU::COPY)); in foldOperand()
921 if (UseOp.getSubReg() && AMDGPU::getRegBitWidth(FoldRC->getID()) == 64) { in foldOperand()
925 if (AMDGPU::getRegBitWidth(UseRC->getID()) != 64) in foldOperand()
929 if (UseOp.getSubReg() == AMDGPU::sub0) { in foldOperand()
932 assert(UseOp.getSubReg() == AMDGPU::sub1); in foldOperand()
949 case AMDGPU::V_AND_B32_e64: in evalBinaryInstruction()
950 case AMDGPU::V_AND_B32_e32: in evalBinaryInstruction()
951 case AMDGPU::S_AND_B32: in evalBinaryInstruction()
954 case AMDGPU::V_OR_B32_e64: in evalBinaryInstruction()
955 case AMDGPU::V_OR_B32_e32: in evalBinaryInstruction()
956 case AMDGPU::S_OR_B32: in evalBinaryInstruction()
959 case AMDGPU::V_XOR_B32_e64: in evalBinaryInstruction()
960 case AMDGPU::V_XOR_B32_e32: in evalBinaryInstruction()
961 case AMDGPU::S_XOR_B32: in evalBinaryInstruction()
964 case AMDGPU::S_XNOR_B32: in evalBinaryInstruction()
967 case AMDGPU::S_NAND_B32: in evalBinaryInstruction()
970 case AMDGPU::S_NOR_B32: in evalBinaryInstruction()
973 case AMDGPU::S_ANDN2_B32: in evalBinaryInstruction()
976 case AMDGPU::S_ORN2_B32: in evalBinaryInstruction()
979 case AMDGPU::V_LSHL_B32_e64: in evalBinaryInstruction()
980 case AMDGPU::V_LSHL_B32_e32: in evalBinaryInstruction()
981 case AMDGPU::S_LSHL_B32: in evalBinaryInstruction()
985 case AMDGPU::V_LSHLREV_B32_e64: in evalBinaryInstruction()
986 case AMDGPU::V_LSHLREV_B32_e32: in evalBinaryInstruction()
989 case AMDGPU::V_LSHR_B32_e64: in evalBinaryInstruction()
990 case AMDGPU::V_LSHR_B32_e32: in evalBinaryInstruction()
991 case AMDGPU::S_LSHR_B32: in evalBinaryInstruction()
994 case AMDGPU::V_LSHRREV_B32_e64: in evalBinaryInstruction()
995 case AMDGPU::V_LSHRREV_B32_e32: in evalBinaryInstruction()
998 case AMDGPU::V_ASHR_I32_e64: in evalBinaryInstruction()
999 case AMDGPU::V_ASHR_I32_e32: in evalBinaryInstruction()
1000 case AMDGPU::S_ASHR_I32: in evalBinaryInstruction()
1003 case AMDGPU::V_ASHRREV_I32_e64: in evalBinaryInstruction()
1004 case AMDGPU::V_ASHRREV_I32_e32: in evalBinaryInstruction()
1013 return IsScalar ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; in getMovOpc()
1038 if (Op.getSubReg() != AMDGPU::NoSubRegister || !Op.getReg().isVirtual()) in getImmOrMaterializedImm()
1059 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); in tryConstantFoldOp()
1064 if ((Opc == AMDGPU::V_NOT_B32_e64 || Opc == AMDGPU::V_NOT_B32_e32 || in tryConstantFoldOp()
1065 Opc == AMDGPU::S_NOT_B32) && in tryConstantFoldOp()
1068 mutateCopyOp(*MI, TII->get(getMovOpc(Opc == AMDGPU::S_NOT_B32))); in tryConstantFoldOp()
1072 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); in tryConstantFoldOp()
1108 if (Opc == AMDGPU::V_OR_B32_e64 || in tryConstantFoldOp()
1109 Opc == AMDGPU::V_OR_B32_e32 || in tryConstantFoldOp()
1110 Opc == AMDGPU::S_OR_B32) { in tryConstantFoldOp()
1114 mutateCopyOp(*MI, TII->get(AMDGPU::COPY)); in tryConstantFoldOp()
1118 mutateCopyOp(*MI, TII->get(getMovOpc(Opc == AMDGPU::S_OR_B32))); in tryConstantFoldOp()
1125 if (MI->getOpcode() == AMDGPU::V_AND_B32_e64 || in tryConstantFoldOp()
1126 MI->getOpcode() == AMDGPU::V_AND_B32_e32 || in tryConstantFoldOp()
1127 MI->getOpcode() == AMDGPU::S_AND_B32) { in tryConstantFoldOp()
1131 mutateCopyOp(*MI, TII->get(getMovOpc(Opc == AMDGPU::S_AND_B32))); in tryConstantFoldOp()
1135 mutateCopyOp(*MI, TII->get(AMDGPU::COPY)); in tryConstantFoldOp()
1143 if (MI->getOpcode() == AMDGPU::V_XOR_B32_e64 || in tryConstantFoldOp()
1144 MI->getOpcode() == AMDGPU::V_XOR_B32_e32 || in tryConstantFoldOp()
1145 MI->getOpcode() == AMDGPU::S_XOR_B32) { in tryConstantFoldOp()
1149 mutateCopyOp(*MI, TII->get(AMDGPU::COPY)); in tryConstantFoldOp()
1160 if (Opc != AMDGPU::V_CNDMASK_B32_e32 && Opc != AMDGPU::V_CNDMASK_B32_e64 && in tryFoldCndMask()
1161 Opc != AMDGPU::V_CNDMASK_B64_PSEUDO) in tryFoldCndMask()
1164 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in tryFoldCndMask()
1165 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in tryFoldCndMask()
1174 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1_modifiers); in tryFoldCndMask()
1176 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers); in tryFoldCndMask()
1183 TII->get(Src0->isReg() ? (unsigned)AMDGPU::COPY : getMovOpc(false)); in tryFoldCndMask()
1184 int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); in tryFoldCndMask()
1187 MI.removeOperand(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1)); in tryFoldCndMask()
1198 if (MI.getOpcode() != AMDGPU::V_AND_B32_e64 && in tryFoldZeroHighBits()
1199 MI.getOpcode() != AMDGPU::V_AND_B32_e32) in tryFoldZeroHighBits()
1268 if (DefMI->readsRegister(AMDGPU::EXEC, TRI) && in foldInstOperand()
1297 case AMDGPU::V_MAX_F32_e64: in isClamp()
1298 case AMDGPU::V_MAX_F16_e64: in isClamp()
1299 case AMDGPU::V_MAX_F64_e64: in isClamp()
1300 case AMDGPU::V_PK_MAX_F16: { in isClamp()
1301 if (!TII->getNamedOperand(MI, AMDGPU::OpName::clamp)->getImm()) in isClamp()
1305 const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in isClamp()
1306 const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in isClamp()
1310 Src0->getSubReg() != AMDGPU::NoSubRegister) in isClamp()
1314 if (TII->hasModifiersSet(MI, AMDGPU::OpName::omod)) in isClamp()
1318 = TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm(); in isClamp()
1320 = TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm(); in isClamp()
1324 unsigned UnsetMods = (Op == AMDGPU::V_PK_MAX_F16) ? SISrcMods::OP_SEL_1 in isClamp()
1347 MachineOperand *DefClamp = TII->getNamedOperand(*Def, AMDGPU::OpName::clamp); in tryFoldClamp()
1369 case AMDGPU::V_MUL_F64_e64: { in getOModValue()
1381 case AMDGPU::V_MUL_F32_e64: { in getOModValue()
1393 case AMDGPU::V_MUL_F16_e64: { in getOModValue()
1417 case AMDGPU::V_MUL_F64_e64: in isOMod()
1418 case AMDGPU::V_MUL_F32_e64: in isOMod()
1419 case AMDGPU::V_MUL_F16_e64: { in isOMod()
1421 if ((Op == AMDGPU::V_MUL_F32_e64 && MFI->getMode().FP32OutputDenormals) || in isOMod()
1422 ((Op == AMDGPU::V_MUL_F64_e64 || Op == AMDGPU::V_MUL_F16_e64) && in isOMod()
1428 const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in isOMod()
1429 const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in isOMod()
1441 TII->hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) || in isOMod()
1442 TII->hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) || in isOMod()
1443 TII->hasModifiersSet(MI, AMDGPU::OpName::omod) || in isOMod()
1444 TII->hasModifiersSet(MI, AMDGPU::OpName::clamp)) in isOMod()
1449 case AMDGPU::V_ADD_F64_e64: in isOMod()
1450 case AMDGPU::V_ADD_F32_e64: in isOMod()
1451 case AMDGPU::V_ADD_F16_e64: { in isOMod()
1453 if ((Op == AMDGPU::V_ADD_F32_e64 && MFI->getMode().FP32OutputDenormals) || in isOMod()
1454 ((Op == AMDGPU::V_ADD_F64_e64 || Op == AMDGPU::V_ADD_F16_e64) && in isOMod()
1459 const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in isOMod()
1460 const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in isOMod()
1464 !TII->hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) && in isOMod()
1465 !TII->hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) && in isOMod()
1466 !TII->hasModifiersSet(MI, AMDGPU::OpName::clamp) && in isOMod()
1467 !TII->hasModifiersSet(MI, AMDGPU::OpName::omod)) in isOMod()
1483 RegOp->getSubReg() != AMDGPU::NoSubRegister || in tryFoldOMod()
1488 MachineOperand *DefOMod = TII->getNamedOperand(*Def, AMDGPU::OpName::omod); in tryFoldOMod()
1494 if (TII->hasModifiersSet(*Def, AMDGPU::OpName::clamp)) in tryFoldOMod()
1563 TII->get(AMDGPU::REG_SEQUENCE), Dst); in tryFoldRegSequence()
1634 TII->get(AMDGPU::COPY), PhiOut) in tryFoldLCSSAPhi()
1742 if (CurrentKnownM0Val && MI.modifiesRegister(AMDGPU::M0, TRI)) in runOnMachineFunction()
1756 if (MI.getOperand(0).getReg() == AMDGPU::M0) { in runOnMachineFunction()