Lines Matching refs:AMDGPU
113 using namespace AMDGPU::SDWA;
309 if (TII->getNamedOperand(*MI, AMDGPU::OpName::src0) == SrcOp) { in getSrcMods()
310 if (auto *Mod = TII->getNamedOperand(*MI, AMDGPU::OpName::src0_modifiers)) { in getSrcMods()
313 } else if (TII->getNamedOperand(*MI, AMDGPU::OpName::src1) == SrcOp) { in getSrcMods()
314 if (auto *Mod = TII->getNamedOperand(*MI, AMDGPU::OpName::src1_modifiers)) { in getSrcMods()
344 MachineOperand *Src = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in convertToSDWA()
345 MachineOperand *SrcSel = TII->getNamedOperand(MI, AMDGPU::OpName::src0_sel); in convertToSDWA()
347 TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers); in convertToSDWA()
351 Src = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in convertToSDWA()
352 SrcSel = TII->getNamedOperand(MI, AMDGPU::OpName::src1_sel); in convertToSDWA()
353 SrcMods = TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers); in convertToSDWA()
363 MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); in convertToSDWA()
365 TII->getNamedOperand(MI, AMDGPU::OpName::dst_unused); in convertToSDWA()
368 DstUnused->getImm() == AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE) { in convertToSDWA()
374 TII->getNamedImmOperand(MI, AMDGPU::OpName::dst_sel)); in convertToSDWA()
375 if (DstSel == AMDGPU::SDWA::SdwaSel::WORD_1 && in convertToSDWA()
376 getSrcSel() == AMDGPU::SDWA::SdwaSel::WORD_0) { in convertToSDWA()
378 auto DstIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), in convertToSDWA()
379 AMDGPU::OpName::vdst); in convertToSDWA()
392 if ((MI.getOpcode() == AMDGPU::V_FMAC_F16_sdwa || in convertToSDWA()
393 MI.getOpcode() == AMDGPU::V_FMAC_F32_sdwa || in convertToSDWA()
394 MI.getOpcode() == AMDGPU::V_MAC_F16_sdwa || in convertToSDWA()
395 MI.getOpcode() == AMDGPU::V_MAC_F32_sdwa) && in convertToSDWA()
436 if ((MI.getOpcode() == AMDGPU::V_FMAC_F16_sdwa || in convertToSDWA()
437 MI.getOpcode() == AMDGPU::V_FMAC_F32_sdwa || in convertToSDWA()
438 MI.getOpcode() == AMDGPU::V_MAC_F16_sdwa || in convertToSDWA()
439 MI.getOpcode() == AMDGPU::V_MAC_F32_sdwa) && in convertToSDWA()
440 getDstSel() != AMDGPU::SDWA::DWORD) { in convertToSDWA()
445 MachineOperand *Operand = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); in convertToSDWA()
450 MachineOperand *DstSel= TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel); in convertToSDWA()
453 MachineOperand *DstUnused= TII->getNamedOperand(MI, AMDGPU::OpName::dst_unused); in convertToSDWA()
486 MI.tieOperands(AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdst), in convertToSDWA()
525 case AMDGPU::V_LSHRREV_B32_e32: in matchSDWAOperand()
526 case AMDGPU::V_ASHRREV_I32_e32: in matchSDWAOperand()
527 case AMDGPU::V_LSHLREV_B32_e32: in matchSDWAOperand()
528 case AMDGPU::V_LSHRREV_B32_e64: in matchSDWAOperand()
529 case AMDGPU::V_ASHRREV_I32_e64: in matchSDWAOperand()
530 case AMDGPU::V_LSHLREV_B32_e64: { in matchSDWAOperand()
539 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in matchSDWAOperand()
547 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in matchSDWAOperand()
548 MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); in matchSDWAOperand()
553 if (Opcode == AMDGPU::V_LSHLREV_B32_e32 || in matchSDWAOperand()
554 Opcode == AMDGPU::V_LSHLREV_B32_e64) { in matchSDWAOperand()
560 Opcode != AMDGPU::V_LSHRREV_B32_e32 && in matchSDWAOperand()
561 Opcode != AMDGPU::V_LSHRREV_B32_e64); in matchSDWAOperand()
566 case AMDGPU::V_LSHRREV_B16_e32: in matchSDWAOperand()
567 case AMDGPU::V_ASHRREV_I16_e32: in matchSDWAOperand()
568 case AMDGPU::V_LSHLREV_B16_e32: in matchSDWAOperand()
569 case AMDGPU::V_LSHRREV_B16_e64: in matchSDWAOperand()
570 case AMDGPU::V_ASHRREV_I16_e64: in matchSDWAOperand()
571 case AMDGPU::V_LSHLREV_B16_e64: { in matchSDWAOperand()
580 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in matchSDWAOperand()
585 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in matchSDWAOperand()
586 MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); in matchSDWAOperand()
592 if (Opcode == AMDGPU::V_LSHLREV_B16_e32 || in matchSDWAOperand()
593 Opcode == AMDGPU::V_LSHLREV_B16_e64) { in matchSDWAOperand()
598 Opcode != AMDGPU::V_LSHRREV_B16_e32 && in matchSDWAOperand()
599 Opcode != AMDGPU::V_LSHRREV_B16_e64); in matchSDWAOperand()
604 case AMDGPU::V_BFE_I32_e64: in matchSDWAOperand()
605 case AMDGPU::V_BFE_U32_e64: { in matchSDWAOperand()
620 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in matchSDWAOperand()
625 MachineOperand *Src2 = TII->getNamedOperand(MI, AMDGPU::OpName::src2); in matchSDWAOperand()
649 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in matchSDWAOperand()
650 MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); in matchSDWAOperand()
657 Src0, Dst, SrcSel, false, false, Opcode != AMDGPU::V_BFE_U32_e64); in matchSDWAOperand()
660 case AMDGPU::V_AND_B32_e32: in matchSDWAOperand()
661 case AMDGPU::V_AND_B32_e64: { in matchSDWAOperand()
666 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in matchSDWAOperand()
667 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in matchSDWAOperand()
679 MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); in matchSDWAOperand()
689 case AMDGPU::V_OR_B32_e32: in matchSDWAOperand()
690 case AMDGPU::V_OR_B32_e64: { in matchSDWAOperand()
722 MachineOperand *OrSDWA = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in matchSDWAOperand()
723 MachineOperand *OrOther = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in matchSDWAOperand()
727 OrSDWA = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in matchSDWAOperand()
728 OrOther = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in matchSDWAOperand()
766 TII->getNamedImmOperand(*SDWAInst, AMDGPU::OpName::dst_sel)); in matchSDWAOperand()
768 TII->getNamedImmOperand(*OtherInst, AMDGPU::OpName::dst_sel)); in matchSDWAOperand()
808 TII->getNamedImmOperand(*OtherInst, AMDGPU::OpName::dst_unused)); in matchSDWAOperand()
813 MachineOperand *OrDst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); in matchSDWAOperand()
863 assert((Opc == AMDGPU::V_ADD_CO_U32_e64 || Opc == AMDGPU::V_SUB_CO_U32_e64) && in pseudoOpConvertToVOP2()
869 Opc = AMDGPU::getVOPe32(Opc); in pseudoOpConvertToVOP2()
871 const MachineOperand *Sdst = TII->getNamedOperand(MI, AMDGPU::OpName::sdst); in pseudoOpConvertToVOP2()
880 MachineOperand *CarryIn = TII->getNamedOperand(MISucc, AMDGPU::OpName::src2); in pseudoOpConvertToVOP2()
883 MachineOperand *CarryOut = TII->getNamedOperand(MISucc, AMDGPU::OpName::sdst); in pseudoOpConvertToVOP2()
890 auto Liveness = MBB.computeRegisterLiveness(TRI, AMDGPU::VCC, MI, 25); in pseudoOpConvertToVOP2()
896 if (I->modifiesRegister(AMDGPU::VCC, TRI)) in pseudoOpConvertToVOP2()
902 .add(*TII->getNamedOperand(MI, AMDGPU::OpName::vdst)) in pseudoOpConvertToVOP2()
903 .add(*TII->getNamedOperand(MI, AMDGPU::OpName::src0)) in pseudoOpConvertToVOP2()
904 .add(*TII->getNamedOperand(MI, AMDGPU::OpName::src1)) in pseudoOpConvertToVOP2()
922 if (AMDGPU::getSDWAOp(Opc) == -1) in isConvertibleToSDWA()
923 Opc = AMDGPU::getVOPe32(Opc); in isConvertibleToSDWA()
925 if (AMDGPU::getSDWAOp(Opc) == -1) in isConvertibleToSDWA()
928 if (!ST.hasSDWAOmod() && TII->hasModifiersSet(MI, AMDGPU::OpName::omod)) in isConvertibleToSDWA()
933 const MachineOperand *SDst = TII->getNamedOperand(MI, AMDGPU::OpName::sdst); in isConvertibleToSDWA()
934 if (SDst && (SDst->getReg() != AMDGPU::VCC && in isConvertibleToSDWA()
935 SDst->getReg() != AMDGPU::VCC_LO)) in isConvertibleToSDWA()
940 (TII->hasModifiersSet(MI, AMDGPU::OpName::clamp) || in isConvertibleToSDWA()
941 TII->hasModifiersSet(MI, AMDGPU::OpName::omod))) in isConvertibleToSDWA()
944 } else if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst) || in isConvertibleToSDWA()
945 !TII->getNamedOperand(MI, AMDGPU::OpName::vdst)) { in isConvertibleToSDWA()
949 if (!ST.hasSDWAMac() && (Opc == AMDGPU::V_FMAC_F16_e32 || in isConvertibleToSDWA()
950 Opc == AMDGPU::V_FMAC_F32_e32 || in isConvertibleToSDWA()
951 Opc == AMDGPU::V_MAC_F16_e32 || in isConvertibleToSDWA()
952 Opc == AMDGPU::V_MAC_F32_e32)) in isConvertibleToSDWA()
960 if (Opc == AMDGPU::V_CNDMASK_B32_e32) in isConvertibleToSDWA()
963 if (MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0)) { in isConvertibleToSDWA()
968 if (MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1)) { in isConvertibleToSDWA()
987 SDWAOpcode = AMDGPU::getSDWAOp(Opcode); in convertToSDWA()
989 SDWAOpcode = AMDGPU::getSDWAOp(AMDGPU::getVOPe32(Opcode)); in convertToSDWA()
1001 MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); in convertToSDWA()
1003 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::vdst)); in convertToSDWA()
1005 } else if ((Dst = TII->getNamedOperand(MI, AMDGPU::OpName::sdst))) { in convertToSDWA()
1006 assert(Dst && AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::sdst)); in convertToSDWA()
1009 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::sdst)); in convertToSDWA()
1015 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in convertToSDWA()
1016 assert(Src0 && AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src0) && in convertToSDWA()
1017 AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src0_modifiers)); in convertToSDWA()
1018 if (auto *Mod = TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)) in convertToSDWA()
1025 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in convertToSDWA()
1027 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src1) && in convertToSDWA()
1028 AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src1_modifiers)); in convertToSDWA()
1029 if (auto *Mod = TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)) in convertToSDWA()
1036 if (SDWAOpcode == AMDGPU::V_FMAC_F16_sdwa || in convertToSDWA()
1037 SDWAOpcode == AMDGPU::V_FMAC_F32_sdwa || in convertToSDWA()
1038 SDWAOpcode == AMDGPU::V_MAC_F16_sdwa || in convertToSDWA()
1039 SDWAOpcode == AMDGPU::V_MAC_F32_sdwa) { in convertToSDWA()
1041 MachineOperand *Src2 = TII->getNamedOperand(MI, AMDGPU::OpName::src2); in convertToSDWA()
1047 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::clamp)); in convertToSDWA()
1048 MachineOperand *Clamp = TII->getNamedOperand(MI, AMDGPU::OpName::clamp); in convertToSDWA()
1056 if (AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::omod)) { in convertToSDWA()
1057 MachineOperand *OMod = TII->getNamedOperand(MI, AMDGPU::OpName::omod); in convertToSDWA()
1066 if (AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::dst_sel)) { in convertToSDWA()
1067 MachineOperand *DstSel = TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel); in convertToSDWA()
1071 SDWAInst.addImm(AMDGPU::SDWA::SdwaSel::DWORD); in convertToSDWA()
1076 if (AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::dst_unused)) { in convertToSDWA()
1077 MachineOperand *DstUnused = TII->getNamedOperand(MI, AMDGPU::OpName::dst_unused); in convertToSDWA()
1081 SDWAInst.addImm(AMDGPU::SDWA::DstUnused::UNUSED_PAD); in convertToSDWA()
1086 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src0_sel)); in convertToSDWA()
1087 MachineOperand *Src0Sel = TII->getNamedOperand(MI, AMDGPU::OpName::src0_sel); in convertToSDWA()
1091 SDWAInst.addImm(AMDGPU::SDWA::SdwaSel::DWORD); in convertToSDWA()
1096 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src1_sel)); in convertToSDWA()
1097 MachineOperand *Src1Sel = TII->getNamedOperand(MI, AMDGPU::OpName::src1_sel); in convertToSDWA()
1101 SDWAInst.addImm(AMDGPU::SDWA::SdwaSel::DWORD); in convertToSDWA()
1106 auto DstUnused = TII->getNamedOperand(MI, AMDGPU::OpName::dst_unused); in convertToSDWA()
1108 DstUnused->getImm() == AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE) { in convertToSDWA()
1114 auto PreserveDstIdx = AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::vdst); in convertToSDWA()
1176 Register VGPR = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in legalizeScalarOperands()
1178 TII->get(AMDGPU::V_MOV_B32_e32), VGPR); in legalizeScalarOperands()
1212 (PotentialMI->getOpcode() == AMDGPU::V_ADD_CO_U32_e64 || in runOnMachineFunction()
1213 PotentialMI->getOpcode() == AMDGPU::V_SUB_CO_U32_e64)) in runOnMachineFunction()