112 using namespace AMDGPU::SDWA;
308   if (TII->getNamedOperand(*MI, AMDGPU::OpName::src0) == SrcOp) {  in getSrcMods()
309     if (auto *Mod = TII->getNamedOperand(*MI, AMDGPU::OpName::src0_modifiers)) {  in getSrcMods()
312   } else if (TII->getNamedOperand(*MI, AMDGPU::OpName::src1) == SrcOp) {  in getSrcMods()
313     if (auto *Mod = TII->getNamedOperand(*MI, AMDGPU::OpName::src1_modifiers)) {  in getSrcMods()
343   MachineOperand *Src = TII->getNamedOperand(MI, AMDGPU::OpName::src0);  in convertToSDWA()
344   MachineOperand *SrcSel = TII->getNamedOperand(MI, AMDGPU::OpName::src0_sel);  in convertToSDWA()
346       TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);  in convertToSDWA()
350     Src = TII->getNamedOperand(MI, AMDGPU::OpName::src1);  in convertToSDWA()
351     SrcSel = TII->getNamedOperand(MI, AMDGPU::OpName::src1_sel);  in convertToSDWA()
352     SrcMods = TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);  in convertToSDWA()
362       MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);  in convertToSDWA()
364         TII->getNamedOperand(MI, AMDGPU::OpName::dst_unused);  in convertToSDWA()
367           DstUnused->getImm() == AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE) {  in convertToSDWA()
373             TII->getNamedImmOperand(MI, AMDGPU::OpName::dst_sel));  in convertToSDWA()
374         if (DstSel == AMDGPU::SDWA::SdwaSel::WORD_1 &&  in convertToSDWA()
375             getSrcSel() == AMDGPU::SDWA::SdwaSel::WORD_0) {  in convertToSDWA()
377           auto DstIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),  in convertToSDWA()
378                                                    AMDGPU::OpName::vdst);  in convertToSDWA()
391     if ((MI.getOpcode() == AMDGPU::V_FMAC_F16_sdwa ||  in convertToSDWA()
392          MI.getOpcode() == AMDGPU::V_FMAC_F32_sdwa ||  in convertToSDWA()
393          MI.getOpcode() == AMDGPU::V_MAC_F16_sdwa ||  in convertToSDWA()
394          MI.getOpcode() == AMDGPU::V_MAC_F32_sdwa) &&  in convertToSDWA()
435   if ((MI.getOpcode() == AMDGPU::V_FMAC_F16_sdwa ||  in convertToSDWA()
436        MI.getOpcode() == AMDGPU::V_FMAC_F32_sdwa ||  in convertToSDWA()
437        MI.getOpcode() == AMDGPU::V_MAC_F16_sdwa ||  in convertToSDWA()
438        MI.getOpcode() == AMDGPU::V_MAC_F32_sdwa) &&  in convertToSDWA()
439       getDstSel() != AMDGPU::SDWA::DWORD) {  in convertToSDWA()
444   MachineOperand *Operand = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);  in convertToSDWA()
449   MachineOperand *DstSel= TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel);  in convertToSDWA()
452   MachineOperand *DstUnused= TII->getNamedOperand(MI, AMDGPU::OpName::dst_unused);  in convertToSDWA()
485   MI.tieOperands(AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdst),  in convertToSDWA()
523   case AMDGPU::V_LSHRREV_B32_e32:  in matchSDWAOperand()
524   case AMDGPU::V_ASHRREV_I32_e32:  in matchSDWAOperand()
525   case AMDGPU::V_LSHLREV_B32_e32:  in matchSDWAOperand()
526   case AMDGPU::V_LSHRREV_B32_e64:  in matchSDWAOperand()
527   case AMDGPU::V_ASHRREV_I32_e64:  in matchSDWAOperand()
528   case AMDGPU::V_LSHLREV_B32_e64: {  in matchSDWAOperand()
537     MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);  in matchSDWAOperand()
545     MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);  in matchSDWAOperand()
546     MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);  in matchSDWAOperand()
550     if (Opcode == AMDGPU::V_LSHLREV_B32_e32 ||  in matchSDWAOperand()
551         Opcode == AMDGPU::V_LSHLREV_B32_e64) {  in matchSDWAOperand()
557           Opcode != AMDGPU::V_LSHRREV_B32_e32 &&  in matchSDWAOperand()
558           Opcode != AMDGPU::V_LSHRREV_B32_e64);  in matchSDWAOperand()
563   case AMDGPU::V_LSHRREV_B16_e32:  in matchSDWAOperand()
564   case AMDGPU::V_ASHRREV_I16_e32:  in matchSDWAOperand()
565   case AMDGPU::V_LSHLREV_B16_e32:  in matchSDWAOperand()
566   case AMDGPU::V_LSHRREV_B16_e64:  in matchSDWAOperand()
567   case AMDGPU::V_ASHRREV_I16_e64:  in matchSDWAOperand()
568   case AMDGPU::V_LSHLREV_B16_e64: {  in matchSDWAOperand()
577     MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);  in matchSDWAOperand()
582     MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);  in matchSDWAOperand()
583     MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);  in matchSDWAOperand()
588     if (Opcode == AMDGPU::V_LSHLREV_B16_e32 ||  in matchSDWAOperand()
589         Opcode == AMDGPU::V_LSHLREV_B16_e64) {  in matchSDWAOperand()
594             Opcode != AMDGPU::V_LSHRREV_B16_e32 &&  in matchSDWAOperand()
595             Opcode != AMDGPU::V_LSHRREV_B16_e64);  in matchSDWAOperand()
600   case AMDGPU::V_BFE_I32_e64:  in matchSDWAOperand()
601   case AMDGPU::V_BFE_U32_e64: {  in matchSDWAOperand()
616     MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);  in matchSDWAOperand()
621     MachineOperand *Src2 = TII->getNamedOperand(MI, AMDGPU::OpName::src2);  in matchSDWAOperand()
645     MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);  in matchSDWAOperand()
646     MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);  in matchSDWAOperand()
652           Src0, Dst, SrcSel, false, false, Opcode != AMDGPU::V_BFE_U32_e64);  in matchSDWAOperand()
655   case AMDGPU::V_AND_B32_e32:  in matchSDWAOperand()
656   case AMDGPU::V_AND_B32_e64: {  in matchSDWAOperand()
661     MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);  in matchSDWAOperand()
662     MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);  in matchSDWAOperand()
674     MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);  in matchSDWAOperand()
683   case AMDGPU::V_OR_B32_e32:  in matchSDWAOperand()
684   case AMDGPU::V_OR_B32_e64: {  in matchSDWAOperand()
715     MachineOperand *OrSDWA = TII->getNamedOperand(MI, AMDGPU::OpName::src0);  in matchSDWAOperand()
716     MachineOperand *OrOther = TII->getNamedOperand(MI, AMDGPU::OpName::src1);  in matchSDWAOperand()
720       OrSDWA = TII->getNamedOperand(MI, AMDGPU::OpName::src1);  in matchSDWAOperand()
721       OrOther = TII->getNamedOperand(MI, AMDGPU::OpName::src0);  in matchSDWAOperand()
759       TII->getNamedImmOperand(*SDWAInst, AMDGPU::OpName::dst_sel));;  in matchSDWAOperand()
761       TII->getNamedImmOperand(*OtherInst, AMDGPU::OpName::dst_sel));  in matchSDWAOperand()
801       TII->getNamedImmOperand(*OtherInst, AMDGPU::OpName::dst_unused));  in matchSDWAOperand()
806     MachineOperand *OrDst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);  in matchSDWAOperand()
856   assert((Opc == AMDGPU::V_ADD_CO_U32_e64 || Opc == AMDGPU::V_SUB_CO_U32_e64) &&  in pseudoOpConvertToVOP2()
862   Opc = AMDGPU::getVOPe32(Opc);  in pseudoOpConvertToVOP2()
864   const MachineOperand *Sdst = TII->getNamedOperand(MI, AMDGPU::OpName::sdst);  in pseudoOpConvertToVOP2()
874   int SuccOpc = AMDGPU::getVOPe32(MISucc.getOpcode());  in pseudoOpConvertToVOP2()
876   MachineOperand *CarryIn = TII->getNamedOperand(MISucc, AMDGPU::OpName::src2);  in pseudoOpConvertToVOP2()
879   MachineOperand *CarryOut = TII->getNamedOperand(MISucc, AMDGPU::OpName::sdst);  in pseudoOpConvertToVOP2()
886   auto Liveness = MBB.computeRegisterLiveness(TRI, AMDGPU::VCC, MI, 25);  in pseudoOpConvertToVOP2()
892     if (I->modifiesRegister(AMDGPU::VCC, TRI))  in pseudoOpConvertToVOP2()
899     .add(*TII->getNamedOperand(MI, AMDGPU::OpName::vdst))  in pseudoOpConvertToVOP2()
900     .add(*TII->getNamedOperand(MI, AMDGPU::OpName::src0))  in pseudoOpConvertToVOP2()
901     .add(*TII->getNamedOperand(MI, AMDGPU::OpName::src1))  in pseudoOpConvertToVOP2()
908     .add(*TII->getNamedOperand(MISucc, AMDGPU::OpName::vdst))  in pseudoOpConvertToVOP2()
909     .add(*TII->getNamedOperand(MISucc, AMDGPU::OpName::src0))  in pseudoOpConvertToVOP2()
910     .add(*TII->getNamedOperand(MISucc, AMDGPU::OpName::src1))  in pseudoOpConvertToVOP2()
924   if (AMDGPU::getSDWAOp(Opc) == -1)  in isConvertibleToSDWA()
925     Opc = AMDGPU::getVOPe32(Opc);  in isConvertibleToSDWA()
927   if (AMDGPU::getSDWAOp(Opc) == -1)  in isConvertibleToSDWA()
930   if (!ST.hasSDWAOmod() && TII->hasModifiersSet(MI, AMDGPU::OpName::omod))  in isConvertibleToSDWA()
935       const MachineOperand *SDst = TII->getNamedOperand(MI, AMDGPU::OpName::sdst);  in isConvertibleToSDWA()
936       if (SDst && (SDst->getReg() != AMDGPU::VCC &&  in isConvertibleToSDWA()
937                    SDst->getReg() != AMDGPU::VCC_LO))  in isConvertibleToSDWA()
942         (TII->hasModifiersSet(MI, AMDGPU::OpName::clamp) ||  in isConvertibleToSDWA()
943          TII->hasModifiersSet(MI, AMDGPU::OpName::omod)))  in isConvertibleToSDWA()
946   } else if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst) ||  in isConvertibleToSDWA()
947              !TII->getNamedOperand(MI, AMDGPU::OpName::vdst)) {  in isConvertibleToSDWA()
951   if (!ST.hasSDWAMac() && (Opc == AMDGPU::V_FMAC_F16_e32 ||  in isConvertibleToSDWA()
952                            Opc == AMDGPU::V_FMAC_F32_e32 ||  in isConvertibleToSDWA()
953                            Opc == AMDGPU::V_MAC_F16_e32 ||  in isConvertibleToSDWA()
954                            Opc == AMDGPU::V_MAC_F32_e32))  in isConvertibleToSDWA()
962   if (Opc == AMDGPU::V_CNDMASK_B32_e32)  in isConvertibleToSDWA()
965   if (MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0)) {  in isConvertibleToSDWA()
970   if (MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1)) {  in isConvertibleToSDWA()
989     SDWAOpcode = AMDGPU::getSDWAOp(Opcode);  in convertToSDWA()
991       SDWAOpcode = AMDGPU::getSDWAOp(AMDGPU::getVOPe32(Opcode));  in convertToSDWA()
1003   MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);  in convertToSDWA()
1005     assert(AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::vdst) != -1);  in convertToSDWA()
1007   } else if ((Dst = TII->getNamedOperand(MI, AMDGPU::OpName::sdst))) {  in convertToSDWA()
1009            AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::sdst) != -1);  in convertToSDWA()
1012     assert(AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::sdst) != -1);  in convertToSDWA()
1018   MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);  in convertToSDWA()
1021     AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src0) != -1 &&  in convertToSDWA()
1022     AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src0_modifiers) != -1);  in convertToSDWA()
1023   if (auto *Mod = TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers))  in convertToSDWA()
1030   MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);  in convertToSDWA()
1033       AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src1) != -1 &&  in convertToSDWA()
1034       AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src1_modifiers) != -1);  in convertToSDWA()
1035     if (auto *Mod = TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers))  in convertToSDWA()
1042   if (SDWAOpcode == AMDGPU::V_FMAC_F16_sdwa ||  in convertToSDWA()
1043       SDWAOpcode == AMDGPU::V_FMAC_F32_sdwa ||  in convertToSDWA()
1044       SDWAOpcode == AMDGPU::V_MAC_F16_sdwa ||  in convertToSDWA()
1045       SDWAOpcode == AMDGPU::V_MAC_F32_sdwa) {  in convertToSDWA()
1047     MachineOperand *Src2 = TII->getNamedOperand(MI, AMDGPU::OpName::src2);  in convertToSDWA()
1053   assert(AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::clamp) != -1);  in convertToSDWA()
1054   MachineOperand *Clamp = TII->getNamedOperand(MI, AMDGPU::OpName::clamp);  in convertToSDWA()
1062   if (AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::omod) != -1) {  in convertToSDWA()
1063     MachineOperand *OMod = TII->getNamedOperand(MI, AMDGPU::OpName::omod);  in convertToSDWA()
1072   if (AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::dst_sel) != -1) {  in convertToSDWA()
1073     MachineOperand *DstSel = TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel);  in convertToSDWA()
1077       SDWAInst.addImm(AMDGPU::SDWA::SdwaSel::DWORD);  in convertToSDWA()
1082   if (AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::dst_unused) != -1) {  in convertToSDWA()
1083     MachineOperand *DstUnused = TII->getNamedOperand(MI, AMDGPU::OpName::dst_unused);  in convertToSDWA()
1087       SDWAInst.addImm(AMDGPU::SDWA::DstUnused::UNUSED_PAD);  in convertToSDWA()
1092   assert(AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src0_sel) != -1);  in convertToSDWA()
1093   MachineOperand *Src0Sel = TII->getNamedOperand(MI, AMDGPU::OpName::src0_sel);  in convertToSDWA()
1097     SDWAInst.addImm(AMDGPU::SDWA::SdwaSel::DWORD);  in convertToSDWA()
1102     assert(AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src1_sel) != -1);  in convertToSDWA()
1103     MachineOperand *Src1Sel = TII->getNamedOperand(MI, AMDGPU::OpName::src1_sel);  in convertToSDWA()
1107       SDWAInst.addImm(AMDGPU::SDWA::SdwaSel::DWORD);  in convertToSDWA()
1112   auto DstUnused = TII->getNamedOperand(MI, AMDGPU::OpName::dst_unused);  in convertToSDWA()
1114       DstUnused->getImm() == AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE) {  in convertToSDWA()
1120     auto PreserveDstIdx = AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::vdst);  in convertToSDWA()
1182     Register VGPR = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);  in legalizeScalarOperands()
1184                         TII->get(AMDGPU::V_MOV_B32_e32), VGPR);  in legalizeScalarOperands()
1218            (PotentialMI->getOpcode() == AMDGPU::V_ADD_CO_U32_e64 ||  in runOnMachineFunction()
1219             PotentialMI->getOpcode() == AMDGPU::V_SUB_CO_U32_e64))  in runOnMachineFunction()