Lines Matching refs:AMDGPU
90 Register TmpVGPR = AMDGPU::NoRegister;
96 Register SavedExecReg = AMDGPU::NoRegister;
130 ExecReg = AMDGPU::EXEC_LO; in SGPRSpillBuilder()
131 MovOpc = AMDGPU::S_MOV_B32; in SGPRSpillBuilder()
132 NotOpc = AMDGPU::S_NOT_B32; in SGPRSpillBuilder()
134 ExecReg = AMDGPU::EXEC; in SGPRSpillBuilder()
135 MovOpc = AMDGPU::S_MOV_B64; in SGPRSpillBuilder()
136 NotOpc = AMDGPU::S_NOT_B64; in SGPRSpillBuilder()
139 assert(SuperReg != AMDGPU::M0 && "m0 should never spill"); in SGPRSpillBuilder()
140 assert(SuperReg != AMDGPU::EXEC_LO && SuperReg != AMDGPU::EXEC_HI && in SGPRSpillBuilder()
141 SuperReg != AMDGPU::EXEC && "exec should never spill"); in SGPRSpillBuilder()
173 TmpVGPR = RS->scavengeRegisterBackwards(AMDGPU::VGPR_32RegClass, MI, false, in prepare()
184 TmpVGPR = AMDGPU::VGPR0; in prepare()
201 IsWave32 ? AMDGPU::SGPR_32RegClass : AMDGPU::SGPR_64RegClass; in prepare()
221 if (RS->isRegUsed(AMDGPU::SCC)) in prepare()
296 if (RS->isRegUsed(AMDGPU::SCC)) in readWriteTmpVGPR()
321 : AMDGPUGenRegisterInfo(AMDGPU::PC_REG, ST.getAMDGPUDwarfFlavour()), ST(ST), in SIRegisterInfo()
324 assert(getSubRegIndexLaneMask(AMDGPU::sub0).getAsInteger() == 3 && in SIRegisterInfo()
325 getSubRegIndexLaneMask(AMDGPU::sub31).getAsInteger() == (3ULL << 62) && in SIRegisterInfo()
326 (getSubRegIndexLaneMask(AMDGPU::lo16) | in SIRegisterInfo()
327 getSubRegIndexLaneMask(AMDGPU::hi16)).getAsInteger() == in SIRegisterInfo()
328 getSubRegIndexLaneMask(AMDGPU::sub0).getAsInteger() && in SIRegisterInfo()
332 RegPressureIgnoredUnits.set(*regunits(MCRegister::from(AMDGPU::M0)).begin()); in SIRegisterInfo()
333 for (auto Reg : AMDGPU::VGPR_16RegClass) { in SIRegisterInfo()
334 if (AMDGPU::isHi(Reg, *this)) in SIRegisterInfo()
363 Row.fill(AMDGPU::NoSubRegister); in SIRegisterInfo()
406 static const MCPhysReg NoCalleeSavedReg = AMDGPU::NoRegister; in getCalleeSavedRegs()
443 return VGPR >= AMDGPU::VGPR0 && VGPR < AMDGPU::VGPR8; in isChainScratchRegister()
454 if (RC == &AMDGPU::VGPR_32RegClass || RC == &AMDGPU::AGPR_32RegClass) in getLargestLegalSuperClass()
455 return &AMDGPU::AV_32RegClass; in getLargestLegalSuperClass()
456 if (RC == &AMDGPU::VReg_64RegClass || RC == &AMDGPU::AReg_64RegClass) in getLargestLegalSuperClass()
457 return &AMDGPU::AV_64RegClass; in getLargestLegalSuperClass()
458 if (RC == &AMDGPU::VReg_64_Align2RegClass || in getLargestLegalSuperClass()
459 RC == &AMDGPU::AReg_64_Align2RegClass) in getLargestLegalSuperClass()
460 return &AMDGPU::AV_64_Align2RegClass; in getLargestLegalSuperClass()
461 if (RC == &AMDGPU::VReg_96RegClass || RC == &AMDGPU::AReg_96RegClass) in getLargestLegalSuperClass()
462 return &AMDGPU::AV_96RegClass; in getLargestLegalSuperClass()
463 if (RC == &AMDGPU::VReg_96_Align2RegClass || in getLargestLegalSuperClass()
464 RC == &AMDGPU::AReg_96_Align2RegClass) in getLargestLegalSuperClass()
465 return &AMDGPU::AV_96_Align2RegClass; in getLargestLegalSuperClass()
466 if (RC == &AMDGPU::VReg_128RegClass || RC == &AMDGPU::AReg_128RegClass) in getLargestLegalSuperClass()
467 return &AMDGPU::AV_128RegClass; in getLargestLegalSuperClass()
468 if (RC == &AMDGPU::VReg_128_Align2RegClass || in getLargestLegalSuperClass()
469 RC == &AMDGPU::AReg_128_Align2RegClass) in getLargestLegalSuperClass()
470 return &AMDGPU::AV_128_Align2RegClass; in getLargestLegalSuperClass()
471 if (RC == &AMDGPU::VReg_160RegClass || RC == &AMDGPU::AReg_160RegClass) in getLargestLegalSuperClass()
472 return &AMDGPU::AV_160RegClass; in getLargestLegalSuperClass()
473 if (RC == &AMDGPU::VReg_160_Align2RegClass || in getLargestLegalSuperClass()
474 RC == &AMDGPU::AReg_160_Align2RegClass) in getLargestLegalSuperClass()
475 return &AMDGPU::AV_160_Align2RegClass; in getLargestLegalSuperClass()
476 if (RC == &AMDGPU::VReg_192RegClass || RC == &AMDGPU::AReg_192RegClass) in getLargestLegalSuperClass()
477 return &AMDGPU::AV_192RegClass; in getLargestLegalSuperClass()
478 if (RC == &AMDGPU::VReg_192_Align2RegClass || in getLargestLegalSuperClass()
479 RC == &AMDGPU::AReg_192_Align2RegClass) in getLargestLegalSuperClass()
480 return &AMDGPU::AV_192_Align2RegClass; in getLargestLegalSuperClass()
481 if (RC == &AMDGPU::VReg_256RegClass || RC == &AMDGPU::AReg_256RegClass) in getLargestLegalSuperClass()
482 return &AMDGPU::AV_256RegClass; in getLargestLegalSuperClass()
483 if (RC == &AMDGPU::VReg_256_Align2RegClass || in getLargestLegalSuperClass()
484 RC == &AMDGPU::AReg_256_Align2RegClass) in getLargestLegalSuperClass()
485 return &AMDGPU::AV_256_Align2RegClass; in getLargestLegalSuperClass()
486 if (RC == &AMDGPU::VReg_512RegClass || RC == &AMDGPU::AReg_512RegClass) in getLargestLegalSuperClass()
487 return &AMDGPU::AV_512RegClass; in getLargestLegalSuperClass()
488 if (RC == &AMDGPU::VReg_512_Align2RegClass || in getLargestLegalSuperClass()
489 RC == &AMDGPU::AReg_512_Align2RegClass) in getLargestLegalSuperClass()
490 return &AMDGPU::AV_512_Align2RegClass; in getLargestLegalSuperClass()
491 if (RC == &AMDGPU::VReg_1024RegClass || RC == &AMDGPU::AReg_1024RegClass) in getLargestLegalSuperClass()
492 return &AMDGPU::AV_1024RegClass; in getLargestLegalSuperClass()
493 if (RC == &AMDGPU::VReg_1024_Align2RegClass || in getLargestLegalSuperClass()
494 RC == &AMDGPU::AReg_1024_Align2RegClass) in getLargestLegalSuperClass()
495 return &AMDGPU::AV_1024_Align2RegClass; in getLargestLegalSuperClass()
522 Register SIRegisterInfo::getBaseRegister() const { return AMDGPU::SGPR34; } in getBaseRegister()
554 MCRegister BaseReg(AMDGPU::SGPR_32RegClass.getRegister(BaseIdx)); in getAlignedHighSGPRForRC()
555 return getMatchingSuperReg(BaseReg, AMDGPU::sub0, RC); in getAlignedHighSGPRForRC()
560 return getAlignedHighSGPRForRC(MF, /*Align=*/4, &AMDGPU::SGPR_128RegClass); in reservedPrivateSegmentBufferReg()
565 Reserved.set(AMDGPU::MODE); in getReservedRegs()
573 reserveRegisterTuples(Reserved, AMDGPU::EXEC); in getReservedRegs()
574 reserveRegisterTuples(Reserved, AMDGPU::FLAT_SCR); in getReservedRegs()
577 reserveRegisterTuples(Reserved, AMDGPU::M0); in getReservedRegs()
580 reserveRegisterTuples(Reserved, AMDGPU::SRC_VCCZ); in getReservedRegs()
581 reserveRegisterTuples(Reserved, AMDGPU::SRC_EXECZ); in getReservedRegs()
582 reserveRegisterTuples(Reserved, AMDGPU::SRC_SCC); in getReservedRegs()
585 reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_BASE); in getReservedRegs()
586 reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_LIMIT); in getReservedRegs()
587 reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_BASE); in getReservedRegs()
588 reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_LIMIT); in getReservedRegs()
591 reserveRegisterTuples(Reserved, AMDGPU::SRC_POPS_EXITING_WAVE_ID); in getReservedRegs()
594 reserveRegisterTuples(Reserved, AMDGPU::XNACK_MASK); in getReservedRegs()
597 reserveRegisterTuples(Reserved, AMDGPU::LDS_DIRECT); in getReservedRegs()
600 reserveRegisterTuples(Reserved, AMDGPU::TBA); in getReservedRegs()
601 reserveRegisterTuples(Reserved, AMDGPU::TMA); in getReservedRegs()
602 reserveRegisterTuples(Reserved, AMDGPU::TTMP0_TTMP1); in getReservedRegs()
603 reserveRegisterTuples(Reserved, AMDGPU::TTMP2_TTMP3); in getReservedRegs()
604 reserveRegisterTuples(Reserved, AMDGPU::TTMP4_TTMP5); in getReservedRegs()
605 reserveRegisterTuples(Reserved, AMDGPU::TTMP6_TTMP7); in getReservedRegs()
606 reserveRegisterTuples(Reserved, AMDGPU::TTMP8_TTMP9); in getReservedRegs()
607 reserveRegisterTuples(Reserved, AMDGPU::TTMP10_TTMP11); in getReservedRegs()
608 reserveRegisterTuples(Reserved, AMDGPU::TTMP12_TTMP13); in getReservedRegs()
609 reserveRegisterTuples(Reserved, AMDGPU::TTMP14_TTMP15); in getReservedRegs()
612 reserveRegisterTuples(Reserved, AMDGPU::SGPR_NULL64); in getReservedRegs()
617 Reserved.set(AMDGPU::VCC); in getReservedRegs()
618 Reserved.set(AMDGPU::VCC_HI); in getReservedRegs()
624 unsigned TotalNumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs(); in getReservedRegs()
626 unsigned Reg = AMDGPU::SGPR_32RegClass.getRegister(i); in getReservedRegs()
631 if (ScratchRSrcReg != AMDGPU::NoRegister) { in getReservedRegs()
673 unsigned TotalNumVGPRs = AMDGPU::VGPR_32RegClass.getNumRegs(); in getReservedRegs()
697 unsigned Reg = AMDGPU::VGPR_32RegClass.getRegister(i); in getReservedRegs()
703 unsigned Reg = AMDGPU::AGPR_32RegClass.getRegister(i); in getReservedRegs()
708 for (MCRegister Reg : AMDGPU::AGPR_32RegClass) in getReservedRegs()
784 int OffIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), in getScratchInstrOffset()
785 AMDGPU::OpName::offset); in getScratchInstrOffset()
794 assert((Idx == AMDGPU::getNamedOperandIdx(MI->getOpcode(), in getFrameIndexInstrOffset()
795 AMDGPU::OpName::vaddr) || in getFrameIndexInstrOffset()
796 (Idx == AMDGPU::getNamedOperandIdx(MI->getOpcode(), in getFrameIndexInstrOffset()
797 AMDGPU::OpName::saddr))) && in getFrameIndexInstrOffset()
829 unsigned MovOpc = ST.enableFlatScratch() ? AMDGPU::S_MOV_B32 in materializeFrameBaseRegister()
830 : AMDGPU::V_MOV_B32_e32; in materializeFrameBaseRegister()
833 ST.enableFlatScratch() ? &AMDGPU::SReg_32_XEXEC_HIRegClass in materializeFrameBaseRegister()
834 : &AMDGPU::VGPR_32RegClass); in materializeFrameBaseRegister()
842 Register OffsetReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); in materializeFrameBaseRegister()
845 ST.enableFlatScratch() ? &AMDGPU::SReg_32_XM0RegClass in materializeFrameBaseRegister()
846 : &AMDGPU::VGPR_32RegClass); in materializeFrameBaseRegister()
848 BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::S_MOV_B32), OffsetReg) in materializeFrameBaseRegister()
854 BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::S_ADD_I32), BaseReg) in materializeFrameBaseRegister()
887 TII->getNamedOperand(MI, IsFlat ? AMDGPU::OpName::saddr in resolveFrameIndex()
888 : AMDGPU::OpName::vaddr); in resolveFrameIndex()
890 MachineOperand *OffsetOp = TII->getNamedOperand(MI, AMDGPU::OpName::offset); in resolveFrameIndex()
906 MachineOperand *SOffset = TII->getNamedOperand(MI, AMDGPU::OpName::soffset); in resolveFrameIndex()
937 return &AMDGPU::VGPR_32RegClass; in getPointerRegClass()
944 if (RC == &AMDGPU::SCC_CLASSRegClass) in getCrossCopyRegClass()
953 case AMDGPU::SI_SPILL_S1024_SAVE: in getNumSubRegsForSpillOp()
954 case AMDGPU::SI_SPILL_S1024_RESTORE: in getNumSubRegsForSpillOp()
955 case AMDGPU::SI_SPILL_V1024_SAVE: in getNumSubRegsForSpillOp()
956 case AMDGPU::SI_SPILL_V1024_RESTORE: in getNumSubRegsForSpillOp()
957 case AMDGPU::SI_SPILL_A1024_SAVE: in getNumSubRegsForSpillOp()
958 case AMDGPU::SI_SPILL_A1024_RESTORE: in getNumSubRegsForSpillOp()
959 case AMDGPU::SI_SPILL_AV1024_SAVE: in getNumSubRegsForSpillOp()
960 case AMDGPU::SI_SPILL_AV1024_RESTORE: in getNumSubRegsForSpillOp()
962 case AMDGPU::SI_SPILL_S512_SAVE: in getNumSubRegsForSpillOp()
963 case AMDGPU::SI_SPILL_S512_RESTORE: in getNumSubRegsForSpillOp()
964 case AMDGPU::SI_SPILL_V512_SAVE: in getNumSubRegsForSpillOp()
965 case AMDGPU::SI_SPILL_V512_RESTORE: in getNumSubRegsForSpillOp()
966 case AMDGPU::SI_SPILL_A512_SAVE: in getNumSubRegsForSpillOp()
967 case AMDGPU::SI_SPILL_A512_RESTORE: in getNumSubRegsForSpillOp()
968 case AMDGPU::SI_SPILL_AV512_SAVE: in getNumSubRegsForSpillOp()
969 case AMDGPU::SI_SPILL_AV512_RESTORE: in getNumSubRegsForSpillOp()
971 case AMDGPU::SI_SPILL_S384_SAVE: in getNumSubRegsForSpillOp()
972 case AMDGPU::SI_SPILL_S384_RESTORE: in getNumSubRegsForSpillOp()
973 case AMDGPU::SI_SPILL_V384_SAVE: in getNumSubRegsForSpillOp()
974 case AMDGPU::SI_SPILL_V384_RESTORE: in getNumSubRegsForSpillOp()
975 case AMDGPU::SI_SPILL_A384_SAVE: in getNumSubRegsForSpillOp()
976 case AMDGPU::SI_SPILL_A384_RESTORE: in getNumSubRegsForSpillOp()
977 case AMDGPU::SI_SPILL_AV384_SAVE: in getNumSubRegsForSpillOp()
978 case AMDGPU::SI_SPILL_AV384_RESTORE: in getNumSubRegsForSpillOp()
980 case AMDGPU::SI_SPILL_S352_SAVE: in getNumSubRegsForSpillOp()
981 case AMDGPU::SI_SPILL_S352_RESTORE: in getNumSubRegsForSpillOp()
982 case AMDGPU::SI_SPILL_V352_SAVE: in getNumSubRegsForSpillOp()
983 case AMDGPU::SI_SPILL_V352_RESTORE: in getNumSubRegsForSpillOp()
984 case AMDGPU::SI_SPILL_A352_SAVE: in getNumSubRegsForSpillOp()
985 case AMDGPU::SI_SPILL_A352_RESTORE: in getNumSubRegsForSpillOp()
986 case AMDGPU::SI_SPILL_AV352_SAVE: in getNumSubRegsForSpillOp()
987 case AMDGPU::SI_SPILL_AV352_RESTORE: in getNumSubRegsForSpillOp()
989 case AMDGPU::SI_SPILL_S320_SAVE: in getNumSubRegsForSpillOp()
990 case AMDGPU::SI_SPILL_S320_RESTORE: in getNumSubRegsForSpillOp()
991 case AMDGPU::SI_SPILL_V320_SAVE: in getNumSubRegsForSpillOp()
992 case AMDGPU::SI_SPILL_V320_RESTORE: in getNumSubRegsForSpillOp()
993 case AMDGPU::SI_SPILL_A320_SAVE: in getNumSubRegsForSpillOp()
994 case AMDGPU::SI_SPILL_A320_RESTORE: in getNumSubRegsForSpillOp()
995 case AMDGPU::SI_SPILL_AV320_SAVE: in getNumSubRegsForSpillOp()
996 case AMDGPU::SI_SPILL_AV320_RESTORE: in getNumSubRegsForSpillOp()
998 case AMDGPU::SI_SPILL_S288_SAVE: in getNumSubRegsForSpillOp()
999 case AMDGPU::SI_SPILL_S288_RESTORE: in getNumSubRegsForSpillOp()
1000 case AMDGPU::SI_SPILL_V288_SAVE: in getNumSubRegsForSpillOp()
1001 case AMDGPU::SI_SPILL_V288_RESTORE: in getNumSubRegsForSpillOp()
1002 case AMDGPU::SI_SPILL_A288_SAVE: in getNumSubRegsForSpillOp()
1003 case AMDGPU::SI_SPILL_A288_RESTORE: in getNumSubRegsForSpillOp()
1004 case AMDGPU::SI_SPILL_AV288_SAVE: in getNumSubRegsForSpillOp()
1005 case AMDGPU::SI_SPILL_AV288_RESTORE: in getNumSubRegsForSpillOp()
1007 case AMDGPU::SI_SPILL_S256_SAVE: in getNumSubRegsForSpillOp()
1008 case AMDGPU::SI_SPILL_S256_RESTORE: in getNumSubRegsForSpillOp()
1009 case AMDGPU::SI_SPILL_V256_SAVE: in getNumSubRegsForSpillOp()
1010 case AMDGPU::SI_SPILL_V256_RESTORE: in getNumSubRegsForSpillOp()
1011 case AMDGPU::SI_SPILL_A256_SAVE: in getNumSubRegsForSpillOp()
1012 case AMDGPU::SI_SPILL_A256_RESTORE: in getNumSubRegsForSpillOp()
1013 case AMDGPU::SI_SPILL_AV256_SAVE: in getNumSubRegsForSpillOp()
1014 case AMDGPU::SI_SPILL_AV256_RESTORE: in getNumSubRegsForSpillOp()
1016 case AMDGPU::SI_SPILL_S224_SAVE: in getNumSubRegsForSpillOp()
1017 case AMDGPU::SI_SPILL_S224_RESTORE: in getNumSubRegsForSpillOp()
1018 case AMDGPU::SI_SPILL_V224_SAVE: in getNumSubRegsForSpillOp()
1019 case AMDGPU::SI_SPILL_V224_RESTORE: in getNumSubRegsForSpillOp()
1020 case AMDGPU::SI_SPILL_A224_SAVE: in getNumSubRegsForSpillOp()
1021 case AMDGPU::SI_SPILL_A224_RESTORE: in getNumSubRegsForSpillOp()
1022 case AMDGPU::SI_SPILL_AV224_SAVE: in getNumSubRegsForSpillOp()
1023 case AMDGPU::SI_SPILL_AV224_RESTORE: in getNumSubRegsForSpillOp()
1025 case AMDGPU::SI_SPILL_S192_SAVE: in getNumSubRegsForSpillOp()
1026 case AMDGPU::SI_SPILL_S192_RESTORE: in getNumSubRegsForSpillOp()
1027 case AMDGPU::SI_SPILL_V192_SAVE: in getNumSubRegsForSpillOp()
1028 case AMDGPU::SI_SPILL_V192_RESTORE: in getNumSubRegsForSpillOp()
1029 case AMDGPU::SI_SPILL_A192_SAVE: in getNumSubRegsForSpillOp()
1030 case AMDGPU::SI_SPILL_A192_RESTORE: in getNumSubRegsForSpillOp()
1031 case AMDGPU::SI_SPILL_AV192_SAVE: in getNumSubRegsForSpillOp()
1032 case AMDGPU::SI_SPILL_AV192_RESTORE: in getNumSubRegsForSpillOp()
1034 case AMDGPU::SI_SPILL_S160_SAVE: in getNumSubRegsForSpillOp()
1035 case AMDGPU::SI_SPILL_S160_RESTORE: in getNumSubRegsForSpillOp()
1036 case AMDGPU::SI_SPILL_V160_SAVE: in getNumSubRegsForSpillOp()
1037 case AMDGPU::SI_SPILL_V160_RESTORE: in getNumSubRegsForSpillOp()
1038 case AMDGPU::SI_SPILL_A160_SAVE: in getNumSubRegsForSpillOp()
1039 case AMDGPU::SI_SPILL_A160_RESTORE: in getNumSubRegsForSpillOp()
1040 case AMDGPU::SI_SPILL_AV160_SAVE: in getNumSubRegsForSpillOp()
1041 case AMDGPU::SI_SPILL_AV160_RESTORE: in getNumSubRegsForSpillOp()
1043 case AMDGPU::SI_SPILL_S128_SAVE: in getNumSubRegsForSpillOp()
1044 case AMDGPU::SI_SPILL_S128_RESTORE: in getNumSubRegsForSpillOp()
1045 case AMDGPU::SI_SPILL_V128_SAVE: in getNumSubRegsForSpillOp()
1046 case AMDGPU::SI_SPILL_V128_RESTORE: in getNumSubRegsForSpillOp()
1047 case AMDGPU::SI_SPILL_A128_SAVE: in getNumSubRegsForSpillOp()
1048 case AMDGPU::SI_SPILL_A128_RESTORE: in getNumSubRegsForSpillOp()
1049 case AMDGPU::SI_SPILL_AV128_SAVE: in getNumSubRegsForSpillOp()
1050 case AMDGPU::SI_SPILL_AV128_RESTORE: in getNumSubRegsForSpillOp()
1052 case AMDGPU::SI_SPILL_S96_SAVE: in getNumSubRegsForSpillOp()
1053 case AMDGPU::SI_SPILL_S96_RESTORE: in getNumSubRegsForSpillOp()
1054 case AMDGPU::SI_SPILL_V96_SAVE: in getNumSubRegsForSpillOp()
1055 case AMDGPU::SI_SPILL_V96_RESTORE: in getNumSubRegsForSpillOp()
1056 case AMDGPU::SI_SPILL_A96_SAVE: in getNumSubRegsForSpillOp()
1057 case AMDGPU::SI_SPILL_A96_RESTORE: in getNumSubRegsForSpillOp()
1058 case AMDGPU::SI_SPILL_AV96_SAVE: in getNumSubRegsForSpillOp()
1059 case AMDGPU::SI_SPILL_AV96_RESTORE: in getNumSubRegsForSpillOp()
1061 case AMDGPU::SI_SPILL_S64_SAVE: in getNumSubRegsForSpillOp()
1062 case AMDGPU::SI_SPILL_S64_RESTORE: in getNumSubRegsForSpillOp()
1063 case AMDGPU::SI_SPILL_V64_SAVE: in getNumSubRegsForSpillOp()
1064 case AMDGPU::SI_SPILL_V64_RESTORE: in getNumSubRegsForSpillOp()
1065 case AMDGPU::SI_SPILL_A64_SAVE: in getNumSubRegsForSpillOp()
1066 case AMDGPU::SI_SPILL_A64_RESTORE: in getNumSubRegsForSpillOp()
1067 case AMDGPU::SI_SPILL_AV64_SAVE: in getNumSubRegsForSpillOp()
1068 case AMDGPU::SI_SPILL_AV64_RESTORE: in getNumSubRegsForSpillOp()
1070 case AMDGPU::SI_SPILL_S32_SAVE: in getNumSubRegsForSpillOp()
1071 case AMDGPU::SI_SPILL_S32_RESTORE: in getNumSubRegsForSpillOp()
1072 case AMDGPU::SI_SPILL_V32_SAVE: in getNumSubRegsForSpillOp()
1073 case AMDGPU::SI_SPILL_V32_RESTORE: in getNumSubRegsForSpillOp()
1074 case AMDGPU::SI_SPILL_A32_SAVE: in getNumSubRegsForSpillOp()
1075 case AMDGPU::SI_SPILL_A32_RESTORE: in getNumSubRegsForSpillOp()
1076 case AMDGPU::SI_SPILL_AV32_SAVE: in getNumSubRegsForSpillOp()
1077 case AMDGPU::SI_SPILL_AV32_RESTORE: in getNumSubRegsForSpillOp()
1078 case AMDGPU::SI_SPILL_WWM_V32_SAVE: in getNumSubRegsForSpillOp()
1079 case AMDGPU::SI_SPILL_WWM_V32_RESTORE: in getNumSubRegsForSpillOp()
1080 case AMDGPU::SI_SPILL_WWM_AV32_SAVE: in getNumSubRegsForSpillOp()
1081 case AMDGPU::SI_SPILL_WWM_AV32_RESTORE: in getNumSubRegsForSpillOp()
1089 case AMDGPU::BUFFER_STORE_DWORD_OFFEN: in getOffsetMUBUFStore()
1090 return AMDGPU::BUFFER_STORE_DWORD_OFFSET; in getOffsetMUBUFStore()
1091 case AMDGPU::BUFFER_STORE_BYTE_OFFEN: in getOffsetMUBUFStore()
1092 return AMDGPU::BUFFER_STORE_BYTE_OFFSET; in getOffsetMUBUFStore()
1093 case AMDGPU::BUFFER_STORE_SHORT_OFFEN: in getOffsetMUBUFStore()
1094 return AMDGPU::BUFFER_STORE_SHORT_OFFSET; in getOffsetMUBUFStore()
1095 case AMDGPU::BUFFER_STORE_DWORDX2_OFFEN: in getOffsetMUBUFStore()
1096 return AMDGPU::BUFFER_STORE_DWORDX2_OFFSET; in getOffsetMUBUFStore()
1097 case AMDGPU::BUFFER_STORE_DWORDX3_OFFEN: in getOffsetMUBUFStore()
1098 return AMDGPU::BUFFER_STORE_DWORDX3_OFFSET; in getOffsetMUBUFStore()
1099 case AMDGPU::BUFFER_STORE_DWORDX4_OFFEN: in getOffsetMUBUFStore()
1100 return AMDGPU::BUFFER_STORE_DWORDX4_OFFSET; in getOffsetMUBUFStore()
1101 case AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFEN: in getOffsetMUBUFStore()
1102 return AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFSET; in getOffsetMUBUFStore()
1103 case AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFEN: in getOffsetMUBUFStore()
1104 return AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFSET; in getOffsetMUBUFStore()
1112 case AMDGPU::BUFFER_LOAD_DWORD_OFFEN: in getOffsetMUBUFLoad()
1113 return AMDGPU::BUFFER_LOAD_DWORD_OFFSET; in getOffsetMUBUFLoad()
1114 case AMDGPU::BUFFER_LOAD_UBYTE_OFFEN: in getOffsetMUBUFLoad()
1115 return AMDGPU::BUFFER_LOAD_UBYTE_OFFSET; in getOffsetMUBUFLoad()
1116 case AMDGPU::BUFFER_LOAD_SBYTE_OFFEN: in getOffsetMUBUFLoad()
1117 return AMDGPU::BUFFER_LOAD_SBYTE_OFFSET; in getOffsetMUBUFLoad()
1118 case AMDGPU::BUFFER_LOAD_USHORT_OFFEN: in getOffsetMUBUFLoad()
1119 return AMDGPU::BUFFER_LOAD_USHORT_OFFSET; in getOffsetMUBUFLoad()
1120 case AMDGPU::BUFFER_LOAD_SSHORT_OFFEN: in getOffsetMUBUFLoad()
1121 return AMDGPU::BUFFER_LOAD_SSHORT_OFFSET; in getOffsetMUBUFLoad()
1122 case AMDGPU::BUFFER_LOAD_DWORDX2_OFFEN: in getOffsetMUBUFLoad()
1123 return AMDGPU::BUFFER_LOAD_DWORDX2_OFFSET; in getOffsetMUBUFLoad()
1124 case AMDGPU::BUFFER_LOAD_DWORDX3_OFFEN: in getOffsetMUBUFLoad()
1125 return AMDGPU::BUFFER_LOAD_DWORDX3_OFFSET; in getOffsetMUBUFLoad()
1126 case AMDGPU::BUFFER_LOAD_DWORDX4_OFFEN: in getOffsetMUBUFLoad()
1127 return AMDGPU::BUFFER_LOAD_DWORDX4_OFFSET; in getOffsetMUBUFLoad()
1128 case AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFEN: in getOffsetMUBUFLoad()
1129 return AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFSET; in getOffsetMUBUFLoad()
1130 case AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFEN: in getOffsetMUBUFLoad()
1131 return AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFSET; in getOffsetMUBUFLoad()
1132 case AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFEN: in getOffsetMUBUFLoad()
1133 return AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFSET; in getOffsetMUBUFLoad()
1134 case AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFEN: in getOffsetMUBUFLoad()
1135 return AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFSET; in getOffsetMUBUFLoad()
1136 case AMDGPU::BUFFER_LOAD_SHORT_D16_OFFEN: in getOffsetMUBUFLoad()
1137 return AMDGPU::BUFFER_LOAD_SHORT_D16_OFFSET; in getOffsetMUBUFLoad()
1138 case AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFEN: in getOffsetMUBUFLoad()
1139 return AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFSET; in getOffsetMUBUFLoad()
1147 case AMDGPU::BUFFER_STORE_DWORD_OFFSET: in getOffenMUBUFStore()
1148 return AMDGPU::BUFFER_STORE_DWORD_OFFEN; in getOffenMUBUFStore()
1149 case AMDGPU::BUFFER_STORE_BYTE_OFFSET: in getOffenMUBUFStore()
1150 return AMDGPU::BUFFER_STORE_BYTE_OFFEN; in getOffenMUBUFStore()
1151 case AMDGPU::BUFFER_STORE_SHORT_OFFSET: in getOffenMUBUFStore()
1152 return AMDGPU::BUFFER_STORE_SHORT_OFFEN; in getOffenMUBUFStore()
1153 case AMDGPU::BUFFER_STORE_DWORDX2_OFFSET: in getOffenMUBUFStore()
1154 return AMDGPU::BUFFER_STORE_DWORDX2_OFFEN; in getOffenMUBUFStore()
1155 case AMDGPU::BUFFER_STORE_DWORDX3_OFFSET: in getOffenMUBUFStore()
1156 return AMDGPU::BUFFER_STORE_DWORDX3_OFFEN; in getOffenMUBUFStore()
1157 case AMDGPU::BUFFER_STORE_DWORDX4_OFFSET: in getOffenMUBUFStore()
1158 return AMDGPU::BUFFER_STORE_DWORDX4_OFFEN; in getOffenMUBUFStore()
1159 case AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFSET: in getOffenMUBUFStore()
1160 return AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFEN; in getOffenMUBUFStore()
1161 case AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFSET: in getOffenMUBUFStore()
1162 return AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFEN; in getOffenMUBUFStore()
1170 case AMDGPU::BUFFER_LOAD_DWORD_OFFSET: in getOffenMUBUFLoad()
1171 return AMDGPU::BUFFER_LOAD_DWORD_OFFEN; in getOffenMUBUFLoad()
1172 case AMDGPU::BUFFER_LOAD_UBYTE_OFFSET: in getOffenMUBUFLoad()
1173 return AMDGPU::BUFFER_LOAD_UBYTE_OFFEN; in getOffenMUBUFLoad()
1174 case AMDGPU::BUFFER_LOAD_SBYTE_OFFSET: in getOffenMUBUFLoad()
1175 return AMDGPU::BUFFER_LOAD_SBYTE_OFFEN; in getOffenMUBUFLoad()
1176 case AMDGPU::BUFFER_LOAD_USHORT_OFFSET: in getOffenMUBUFLoad()
1177 return AMDGPU::BUFFER_LOAD_USHORT_OFFEN; in getOffenMUBUFLoad()
1178 case AMDGPU::BUFFER_LOAD_SSHORT_OFFSET: in getOffenMUBUFLoad()
1179 return AMDGPU::BUFFER_LOAD_SSHORT_OFFEN; in getOffenMUBUFLoad()
1180 case AMDGPU::BUFFER_LOAD_DWORDX2_OFFSET: in getOffenMUBUFLoad()
1181 return AMDGPU::BUFFER_LOAD_DWORDX2_OFFEN; in getOffenMUBUFLoad()
1182 case AMDGPU::BUFFER_LOAD_DWORDX3_OFFSET: in getOffenMUBUFLoad()
1183 return AMDGPU::BUFFER_LOAD_DWORDX3_OFFEN; in getOffenMUBUFLoad()
1184 case AMDGPU::BUFFER_LOAD_DWORDX4_OFFSET: in getOffenMUBUFLoad()
1185 return AMDGPU::BUFFER_LOAD_DWORDX4_OFFEN; in getOffenMUBUFLoad()
1186 case AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFSET: in getOffenMUBUFLoad()
1187 return AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFEN; in getOffenMUBUFLoad()
1188 case AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFSET: in getOffenMUBUFLoad()
1189 return AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFEN; in getOffenMUBUFLoad()
1190 case AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFSET: in getOffenMUBUFLoad()
1191 return AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFEN; in getOffenMUBUFLoad()
1192 case AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFSET: in getOffenMUBUFLoad()
1193 return AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFEN; in getOffenMUBUFLoad()
1194 case AMDGPU::BUFFER_LOAD_SHORT_D16_OFFSET: in getOffenMUBUFLoad()
1195 return AMDGPU::BUFFER_LOAD_SHORT_D16_OFFEN; in getOffenMUBUFLoad()
1196 case AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFSET: in getOffenMUBUFLoad()
1197 return AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFEN; in getOffenMUBUFLoad()
1214 if (Reg == AMDGPU::NoRegister) in spillVGPRtoAGPR()
1230 auto CopyMIB = BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), Dst) in spillVGPRtoAGPR()
1235 unsigned Opc = (IsStore ^ IsVGPR) ? AMDGPU::V_ACCVGPR_WRITE_B32_e64 in spillVGPRtoAGPR()
1236 : AMDGPU::V_ACCVGPR_READ_B32_e64; in spillVGPRtoAGPR()
1262 const MachineOperand *Reg = TII->getNamedOperand(*MI, AMDGPU::OpName::vdata); in buildMUBUFOffsetLoadStore()
1269 .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::srsrc)) in buildMUBUFOffsetLoadStore()
1270 .add(*TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)) in buildMUBUFOffsetLoadStore()
1277 AMDGPU::OpName::vdata_in); in buildMUBUFOffsetLoadStore()
1287 bool HasVAddr = AMDGPU::hasNamedOperand(LoadStoreOp, AMDGPU::OpName::vaddr); in getFlatScratchSpillOpcode()
1289 !HasVAddr && !AMDGPU::hasNamedOperand(LoadStoreOp, AMDGPU::OpName::saddr); in getFlatScratchSpillOpcode()
1293 LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORD_SADDR in getFlatScratchSpillOpcode()
1294 : AMDGPU::SCRATCH_LOAD_DWORD_SADDR; in getFlatScratchSpillOpcode()
1297 LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORDX2_SADDR in getFlatScratchSpillOpcode()
1298 : AMDGPU::SCRATCH_LOAD_DWORDX2_SADDR; in getFlatScratchSpillOpcode()
1301 LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORDX3_SADDR in getFlatScratchSpillOpcode()
1302 : AMDGPU::SCRATCH_LOAD_DWORDX3_SADDR; in getFlatScratchSpillOpcode()
1305 LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORDX4_SADDR in getFlatScratchSpillOpcode()
1306 : AMDGPU::SCRATCH_LOAD_DWORDX4_SADDR; in getFlatScratchSpillOpcode()
1313 LoadStoreOp = AMDGPU::getFlatScratchInstSVfromSS(LoadStoreOp); in getFlatScratchSpillOpcode()
1315 LoadStoreOp = AMDGPU::getFlatScratchInstSTfromSS(LoadStoreOp); in getFlatScratchSpillOpcode()
1343 const unsigned RegWidth = AMDGPU::getRegBitWidth(*RC) / 8; in buildSpillLoadStore()
1386 if (ST.getConstantBusLimit(AMDGPU::V_ADD_U32_e64) >= 2) { in buildSpillLoadStore()
1387 BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ADD_U32_e64), TmpVGPR) in buildSpillLoadStore()
1392 BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpVGPR) in buildSpillLoadStore()
1394 BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ADD_U32_e32), TmpVGPR) in buildSpillLoadStore()
1400 BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpVGPR) in buildSpillLoadStore()
1417 SOffset = RS->scavengeRegisterBackwards(AMDGPU::SGPR_32RegClass, MI, false, 0, false); in buildSpillLoadStore()
1420 CanClobberSCC = !RS->isRegUsed(AMDGPU::SCC); in buildSpillLoadStore()
1422 CanClobberSCC = LiveUnits->available(AMDGPU::SCC); in buildSpillLoadStore()
1423 for (MCRegister Reg : AMDGPU::SGPR_32RegClass) { in buildSpillLoadStore()
1431 if (ScratchOffsetReg != AMDGPU::NoRegister && !CanClobberSCC) in buildSpillLoadStore()
1438 TmpOffsetVGPR = RS->scavengeRegisterBackwards(AMDGPU::VGPR_32RegClass, MI, false, 0); in buildSpillLoadStore()
1441 for (MCRegister Reg : AMDGPU::VGPR_32RegClass) { in buildSpillLoadStore()
1481 } else if (ScratchOffsetReg == AMDGPU::NoRegister) { in buildSpillLoadStore()
1482 BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_MOV_B32), SOffset).addImm(Offset); in buildSpillLoadStore()
1485 auto Add = BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), SOffset) in buildSpillLoadStore()
1494 if (IsFlat && SOffset == AMDGPU::NoRegister) { in buildSpillLoadStore()
1495 assert(AMDGPU::getNamedOperandIdx(LoadStoreOp, AMDGPU::OpName::vaddr) < 0 in buildSpillLoadStore()
1499 LoadStoreOp = AMDGPU::getFlatScratchInstSVfromSS(LoadStoreOp); in buildSpillLoadStore()
1502 LoadStoreOp = AMDGPU::getFlatScratchInstSTfromSS(LoadStoreOp); in buildSpillLoadStore()
1613 TII->get(AMDGPU::V_ACCVGPR_READ_B32_e64), in buildSpillLoadStore()
1624 TmpOffsetVGPR = RS->scavengeRegisterBackwards(AMDGPU::VGPR_32RegClass, in buildSpillLoadStore()
1648 if (SOffset == AMDGPU::NoRegister) { in buildSpillLoadStore()
1664 MIB.addImm(LastUse ? AMDGPU::CPol::TH_LU : 0); // cpol in buildSpillLoadStore()
1673 if (!IsStore && IsAGPR && TmpIntermediateVGPR != AMDGPU::NoRegister) { in buildSpillLoadStore()
1674 MIB = BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), in buildSpillLoadStore()
1713 BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), SOffset) in buildSpillLoadStore()
1738 unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_LOAD_DWORD_SADDR in buildVGPRSpillLoadStore()
1739 : AMDGPU::BUFFER_LOAD_DWORD_OFFSET; in buildVGPRSpillLoadStore()
1743 unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_STORE_DWORD_SADDR in buildVGPRSpillLoadStore()
1744 : AMDGPU::BUFFER_STORE_DWORD_OFFSET; in buildVGPRSpillLoadStore()
1788 SB.TII.get(AMDGPU::SI_SPILL_S32_TO_VGPR), Spill.VGPR) in spillSGPR()
1835 SB.TII.get(AMDGPU::SI_SPILL_S32_TO_VGPR), SB.TmpVGPR) in spillSGPR()
1897 SB.TII.get(AMDGPU::SI_RESTORE_S32_FROM_VGPR), SubReg) in restoreSGPR()
1930 SB.TII.get(AMDGPU::SI_RESTORE_S32_FROM_VGPR), SubReg) in restoreSGPR()
1976 BuildMI(*SB.MBB, MI, SB.DL, SB.TII.get(AMDGPU::V_WRITELANE_B32), in spillEmergencySGPR()
2010 auto MIB = BuildMI(*SB.MBB, MI, SB.DL, SB.TII.get(AMDGPU::V_READLANE_B32), in spillEmergencySGPR()
2031 case AMDGPU::SI_SPILL_S1024_SAVE: in eliminateSGPRToVGPRSpillFrameIndex()
2032 case AMDGPU::SI_SPILL_S512_SAVE: in eliminateSGPRToVGPRSpillFrameIndex()
2033 case AMDGPU::SI_SPILL_S384_SAVE: in eliminateSGPRToVGPRSpillFrameIndex()
2034 case AMDGPU::SI_SPILL_S352_SAVE: in eliminateSGPRToVGPRSpillFrameIndex()
2035 case AMDGPU::SI_SPILL_S320_SAVE: in eliminateSGPRToVGPRSpillFrameIndex()
2036 case AMDGPU::SI_SPILL_S288_SAVE: in eliminateSGPRToVGPRSpillFrameIndex()
2037 case AMDGPU::SI_SPILL_S256_SAVE: in eliminateSGPRToVGPRSpillFrameIndex()
2038 case AMDGPU::SI_SPILL_S224_SAVE: in eliminateSGPRToVGPRSpillFrameIndex()
2039 case AMDGPU::SI_SPILL_S192_SAVE: in eliminateSGPRToVGPRSpillFrameIndex()
2040 case AMDGPU::SI_SPILL_S160_SAVE: in eliminateSGPRToVGPRSpillFrameIndex()
2041 case AMDGPU::SI_SPILL_S128_SAVE: in eliminateSGPRToVGPRSpillFrameIndex()
2042 case AMDGPU::SI_SPILL_S96_SAVE: in eliminateSGPRToVGPRSpillFrameIndex()
2043 case AMDGPU::SI_SPILL_S64_SAVE: in eliminateSGPRToVGPRSpillFrameIndex()
2044 case AMDGPU::SI_SPILL_S32_SAVE: in eliminateSGPRToVGPRSpillFrameIndex()
2046 case AMDGPU::SI_SPILL_S1024_RESTORE: in eliminateSGPRToVGPRSpillFrameIndex()
2047 case AMDGPU::SI_SPILL_S512_RESTORE: in eliminateSGPRToVGPRSpillFrameIndex()
2048 case AMDGPU::SI_SPILL_S384_RESTORE: in eliminateSGPRToVGPRSpillFrameIndex()
2049 case AMDGPU::SI_SPILL_S352_RESTORE: in eliminateSGPRToVGPRSpillFrameIndex()
2050 case AMDGPU::SI_SPILL_S320_RESTORE: in eliminateSGPRToVGPRSpillFrameIndex()
2051 case AMDGPU::SI_SPILL_S288_RESTORE: in eliminateSGPRToVGPRSpillFrameIndex()
2052 case AMDGPU::SI_SPILL_S256_RESTORE: in eliminateSGPRToVGPRSpillFrameIndex()
2053 case AMDGPU::SI_SPILL_S224_RESTORE: in eliminateSGPRToVGPRSpillFrameIndex()
2054 case AMDGPU::SI_SPILL_S192_RESTORE: in eliminateSGPRToVGPRSpillFrameIndex()
2055 case AMDGPU::SI_SPILL_S160_RESTORE: in eliminateSGPRToVGPRSpillFrameIndex()
2056 case AMDGPU::SI_SPILL_S128_RESTORE: in eliminateSGPRToVGPRSpillFrameIndex()
2057 case AMDGPU::SI_SPILL_S96_RESTORE: in eliminateSGPRToVGPRSpillFrameIndex()
2058 case AMDGPU::SI_SPILL_S64_RESTORE: in eliminateSGPRToVGPRSpillFrameIndex()
2059 case AMDGPU::SI_SPILL_S32_RESTORE: in eliminateSGPRToVGPRSpillFrameIndex()
2087 case AMDGPU::SI_SPILL_S1024_SAVE: in eliminateFrameIndex()
2088 case AMDGPU::SI_SPILL_S512_SAVE: in eliminateFrameIndex()
2089 case AMDGPU::SI_SPILL_S384_SAVE: in eliminateFrameIndex()
2090 case AMDGPU::SI_SPILL_S352_SAVE: in eliminateFrameIndex()
2091 case AMDGPU::SI_SPILL_S320_SAVE: in eliminateFrameIndex()
2092 case AMDGPU::SI_SPILL_S288_SAVE: in eliminateFrameIndex()
2093 case AMDGPU::SI_SPILL_S256_SAVE: in eliminateFrameIndex()
2094 case AMDGPU::SI_SPILL_S224_SAVE: in eliminateFrameIndex()
2095 case AMDGPU::SI_SPILL_S192_SAVE: in eliminateFrameIndex()
2096 case AMDGPU::SI_SPILL_S160_SAVE: in eliminateFrameIndex()
2097 case AMDGPU::SI_SPILL_S128_SAVE: in eliminateFrameIndex()
2098 case AMDGPU::SI_SPILL_S96_SAVE: in eliminateFrameIndex()
2099 case AMDGPU::SI_SPILL_S64_SAVE: in eliminateFrameIndex()
2100 case AMDGPU::SI_SPILL_S32_SAVE: { in eliminateFrameIndex()
2105 case AMDGPU::SI_SPILL_S1024_RESTORE: in eliminateFrameIndex()
2106 case AMDGPU::SI_SPILL_S512_RESTORE: in eliminateFrameIndex()
2107 case AMDGPU::SI_SPILL_S384_RESTORE: in eliminateFrameIndex()
2108 case AMDGPU::SI_SPILL_S352_RESTORE: in eliminateFrameIndex()
2109 case AMDGPU::SI_SPILL_S320_RESTORE: in eliminateFrameIndex()
2110 case AMDGPU::SI_SPILL_S288_RESTORE: in eliminateFrameIndex()
2111 case AMDGPU::SI_SPILL_S256_RESTORE: in eliminateFrameIndex()
2112 case AMDGPU::SI_SPILL_S224_RESTORE: in eliminateFrameIndex()
2113 case AMDGPU::SI_SPILL_S192_RESTORE: in eliminateFrameIndex()
2114 case AMDGPU::SI_SPILL_S160_RESTORE: in eliminateFrameIndex()
2115 case AMDGPU::SI_SPILL_S128_RESTORE: in eliminateFrameIndex()
2116 case AMDGPU::SI_SPILL_S96_RESTORE: in eliminateFrameIndex()
2117 case AMDGPU::SI_SPILL_S64_RESTORE: in eliminateFrameIndex()
2118 case AMDGPU::SI_SPILL_S32_RESTORE: { in eliminateFrameIndex()
2123 case AMDGPU::SI_SPILL_V1024_SAVE: in eliminateFrameIndex()
2124 case AMDGPU::SI_SPILL_V512_SAVE: in eliminateFrameIndex()
2125 case AMDGPU::SI_SPILL_V384_SAVE: in eliminateFrameIndex()
2126 case AMDGPU::SI_SPILL_V352_SAVE: in eliminateFrameIndex()
2127 case AMDGPU::SI_SPILL_V320_SAVE: in eliminateFrameIndex()
2128 case AMDGPU::SI_SPILL_V288_SAVE: in eliminateFrameIndex()
2129 case AMDGPU::SI_SPILL_V256_SAVE: in eliminateFrameIndex()
2130 case AMDGPU::SI_SPILL_V224_SAVE: in eliminateFrameIndex()
2131 case AMDGPU::SI_SPILL_V192_SAVE: in eliminateFrameIndex()
2132 case AMDGPU::SI_SPILL_V160_SAVE: in eliminateFrameIndex()
2133 case AMDGPU::SI_SPILL_V128_SAVE: in eliminateFrameIndex()
2134 case AMDGPU::SI_SPILL_V96_SAVE: in eliminateFrameIndex()
2135 case AMDGPU::SI_SPILL_V64_SAVE: in eliminateFrameIndex()
2136 case AMDGPU::SI_SPILL_V32_SAVE: in eliminateFrameIndex()
2137 case AMDGPU::SI_SPILL_A1024_SAVE: in eliminateFrameIndex()
2138 case AMDGPU::SI_SPILL_A512_SAVE: in eliminateFrameIndex()
2139 case AMDGPU::SI_SPILL_A384_SAVE: in eliminateFrameIndex()
2140 case AMDGPU::SI_SPILL_A352_SAVE: in eliminateFrameIndex()
2141 case AMDGPU::SI_SPILL_A320_SAVE: in eliminateFrameIndex()
2142 case AMDGPU::SI_SPILL_A288_SAVE: in eliminateFrameIndex()
2143 case AMDGPU::SI_SPILL_A256_SAVE: in eliminateFrameIndex()
2144 case AMDGPU::SI_SPILL_A224_SAVE: in eliminateFrameIndex()
2145 case AMDGPU::SI_SPILL_A192_SAVE: in eliminateFrameIndex()
2146 case AMDGPU::SI_SPILL_A160_SAVE: in eliminateFrameIndex()
2147 case AMDGPU::SI_SPILL_A128_SAVE: in eliminateFrameIndex()
2148 case AMDGPU::SI_SPILL_A96_SAVE: in eliminateFrameIndex()
2149 case AMDGPU::SI_SPILL_A64_SAVE: in eliminateFrameIndex()
2150 case AMDGPU::SI_SPILL_A32_SAVE: in eliminateFrameIndex()
2151 case AMDGPU::SI_SPILL_AV1024_SAVE: in eliminateFrameIndex()
2152 case AMDGPU::SI_SPILL_AV512_SAVE: in eliminateFrameIndex()
2153 case AMDGPU::SI_SPILL_AV384_SAVE: in eliminateFrameIndex()
2154 case AMDGPU::SI_SPILL_AV352_SAVE: in eliminateFrameIndex()
2155 case AMDGPU::SI_SPILL_AV320_SAVE: in eliminateFrameIndex()
2156 case AMDGPU::SI_SPILL_AV288_SAVE: in eliminateFrameIndex()
2157 case AMDGPU::SI_SPILL_AV256_SAVE: in eliminateFrameIndex()
2158 case AMDGPU::SI_SPILL_AV224_SAVE: in eliminateFrameIndex()
2159 case AMDGPU::SI_SPILL_AV192_SAVE: in eliminateFrameIndex()
2160 case AMDGPU::SI_SPILL_AV160_SAVE: in eliminateFrameIndex()
2161 case AMDGPU::SI_SPILL_AV128_SAVE: in eliminateFrameIndex()
2162 case AMDGPU::SI_SPILL_AV96_SAVE: in eliminateFrameIndex()
2163 case AMDGPU::SI_SPILL_AV64_SAVE: in eliminateFrameIndex()
2164 case AMDGPU::SI_SPILL_AV32_SAVE: in eliminateFrameIndex()
2165 case AMDGPU::SI_SPILL_WWM_V32_SAVE: in eliminateFrameIndex()
2166 case AMDGPU::SI_SPILL_WWM_AV32_SAVE: { in eliminateFrameIndex()
2168 AMDGPU::OpName::vdata); in eliminateFrameIndex()
2169 assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() == in eliminateFrameIndex()
2172 unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_STORE_DWORD_SADDR in eliminateFrameIndex()
2173 : AMDGPU::BUFFER_STORE_DWORD_OFFSET; in eliminateFrameIndex()
2178 RS->isRegUsed(AMDGPU::SCC)); in eliminateFrameIndex()
2182 TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(), in eliminateFrameIndex()
2191 case AMDGPU::SI_SPILL_V32_RESTORE: in eliminateFrameIndex()
2192 case AMDGPU::SI_SPILL_V64_RESTORE: in eliminateFrameIndex()
2193 case AMDGPU::SI_SPILL_V96_RESTORE: in eliminateFrameIndex()
2194 case AMDGPU::SI_SPILL_V128_RESTORE: in eliminateFrameIndex()
2195 case AMDGPU::SI_SPILL_V160_RESTORE: in eliminateFrameIndex()
2196 case AMDGPU::SI_SPILL_V192_RESTORE: in eliminateFrameIndex()
2197 case AMDGPU::SI_SPILL_V224_RESTORE: in eliminateFrameIndex()
2198 case AMDGPU::SI_SPILL_V256_RESTORE: in eliminateFrameIndex()
2199 case AMDGPU::SI_SPILL_V288_RESTORE: in eliminateFrameIndex()
2200 case AMDGPU::SI_SPILL_V320_RESTORE: in eliminateFrameIndex()
2201 case AMDGPU::SI_SPILL_V352_RESTORE: in eliminateFrameIndex()
2202 case AMDGPU::SI_SPILL_V384_RESTORE: in eliminateFrameIndex()
2203 case AMDGPU::SI_SPILL_V512_RESTORE: in eliminateFrameIndex()
2204 case AMDGPU::SI_SPILL_V1024_RESTORE: in eliminateFrameIndex()
2205 case AMDGPU::SI_SPILL_A32_RESTORE: in eliminateFrameIndex()
2206 case AMDGPU::SI_SPILL_A64_RESTORE: in eliminateFrameIndex()
2207 case AMDGPU::SI_SPILL_A96_RESTORE: in eliminateFrameIndex()
2208 case AMDGPU::SI_SPILL_A128_RESTORE: in eliminateFrameIndex()
2209 case AMDGPU::SI_SPILL_A160_RESTORE: in eliminateFrameIndex()
2210 case AMDGPU::SI_SPILL_A192_RESTORE: in eliminateFrameIndex()
2211 case AMDGPU::SI_SPILL_A224_RESTORE: in eliminateFrameIndex()
2212 case AMDGPU::SI_SPILL_A256_RESTORE: in eliminateFrameIndex()
2213 case AMDGPU::SI_SPILL_A288_RESTORE: in eliminateFrameIndex()
2214 case AMDGPU::SI_SPILL_A320_RESTORE: in eliminateFrameIndex()
2215 case AMDGPU::SI_SPILL_A352_RESTORE: in eliminateFrameIndex()
2216 case AMDGPU::SI_SPILL_A384_RESTORE: in eliminateFrameIndex()
2217 case AMDGPU::SI_SPILL_A512_RESTORE: in eliminateFrameIndex()
2218 case AMDGPU::SI_SPILL_A1024_RESTORE: in eliminateFrameIndex()
2219 case AMDGPU::SI_SPILL_AV32_RESTORE: in eliminateFrameIndex()
2220 case AMDGPU::SI_SPILL_AV64_RESTORE: in eliminateFrameIndex()
2221 case AMDGPU::SI_SPILL_AV96_RESTORE: in eliminateFrameIndex()
2222 case AMDGPU::SI_SPILL_AV128_RESTORE: in eliminateFrameIndex()
2223 case AMDGPU::SI_SPILL_AV160_RESTORE: in eliminateFrameIndex()
2224 case AMDGPU::SI_SPILL_AV192_RESTORE: in eliminateFrameIndex()
2225 case AMDGPU::SI_SPILL_AV224_RESTORE: in eliminateFrameIndex()
2226 case AMDGPU::SI_SPILL_AV256_RESTORE: in eliminateFrameIndex()
2227 case AMDGPU::SI_SPILL_AV288_RESTORE: in eliminateFrameIndex()
2228 case AMDGPU::SI_SPILL_AV320_RESTORE: in eliminateFrameIndex()
2229 case AMDGPU::SI_SPILL_AV352_RESTORE: in eliminateFrameIndex()
2230 case AMDGPU::SI_SPILL_AV384_RESTORE: in eliminateFrameIndex()
2231 case AMDGPU::SI_SPILL_AV512_RESTORE: in eliminateFrameIndex()
2232 case AMDGPU::SI_SPILL_AV1024_RESTORE: in eliminateFrameIndex()
2233 case AMDGPU::SI_SPILL_WWM_V32_RESTORE: in eliminateFrameIndex()
2234 case AMDGPU::SI_SPILL_WWM_AV32_RESTORE: { in eliminateFrameIndex()
2236 AMDGPU::OpName::vdata); in eliminateFrameIndex()
2237 assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() == in eliminateFrameIndex()
2240 unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_LOAD_DWORD_SADDR in eliminateFrameIndex()
2241 : AMDGPU::BUFFER_LOAD_DWORD_OFFSET; in eliminateFrameIndex()
2246 RS->isRegUsed(AMDGPU::SCC)); in eliminateFrameIndex()
2251 TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(), in eliminateFrameIndex()
2269 AMDGPU::getNamedOperandIdx(MI->getOpcode(), in eliminateFrameIndex()
2270 AMDGPU::OpName::saddr)); in eliminateFrameIndex()
2280 TII->getNamedOperand(*MI, AMDGPU::OpName::offset); in eliminateFrameIndex()
2293 if (AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::vaddr)) { in eliminateFrameIndex()
2294 NewOpc = AMDGPU::getFlatScratchInstSVfromSVS(Opc); in eliminateFrameIndex()
2298 NewOpc = AMDGPU::getFlatScratchInstSTfromSS(Opc); in eliminateFrameIndex()
2304 int VDstIn = AMDGPU::getNamedOperandIdx(Opc, in eliminateFrameIndex()
2305 AMDGPU::OpName::vdst_in); in eliminateFrameIndex()
2313 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr)); in eliminateFrameIndex()
2317 AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vdst); in eliminateFrameIndex()
2319 AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vdst_in); in eliminateFrameIndex()
2337 FIOp.ChangeToRegister(AMDGPU::M0, false); in eliminateFrameIndex()
2345 const TargetRegisterClass *RC = UseSGPR ? &AMDGPU::SReg_32_XM0RegClass in eliminateFrameIndex()
2346 : &AMDGPU::VGPR_32RegClass; in eliminateFrameIndex()
2354 unsigned Opc = UseSGPR ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; in eliminateFrameIndex()
2365 RS->isRegUsed(AMDGPU::SCC) && !MI->definesRegister(AMDGPU::SCC); in eliminateFrameIndex()
2369 : RS->scavengeRegisterBackwards(AMDGPU::SReg_32_XM0RegClass, in eliminateFrameIndex()
2386 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADDC_U32), TmpSReg) in eliminateFrameIndex()
2389 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_BITCMP1_B32)) in eliminateFrameIndex()
2392 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_BITSET0_B32), TmpSReg) in eliminateFrameIndex()
2396 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), TmpSReg) in eliminateFrameIndex()
2402 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpReg) in eliminateFrameIndex()
2407 if (NeedSaveSCC && !MI->registerDefIsDead(AMDGPU::SCC)) { in eliminateFrameIndex()
2409 BuildMI(*MBB, std::next(MI), DL, TII->get(AMDGPU::S_ADDC_U32), in eliminateFrameIndex()
2413 I = BuildMI(*MBB, std::next(I), DL, TII->get(AMDGPU::S_BITCMP1_B32)) in eliminateFrameIndex()
2416 BuildMI(*MBB, std::next(I), DL, TII->get(AMDGPU::S_BITSET0_B32), in eliminateFrameIndex()
2421 BuildMI(*MBB, std::next(MI), DL, TII->get(AMDGPU::S_ADD_I32), in eliminateFrameIndex()
2438 RS->isRegUsed(AMDGPU::SCC) && !MI->definesRegister(AMDGPU::SCC); in eliminateFrameIndex()
2440 ? &AMDGPU::SReg_32RegClass in eliminateFrameIndex()
2441 : &AMDGPU::VGPR_32RegClass; in eliminateFrameIndex()
2442 bool IsCopy = MI->getOpcode() == AMDGPU::V_MOV_B32_e32 || in eliminateFrameIndex()
2443 MI->getOpcode() == AMDGPU::V_MOV_B32_e64; in eliminateFrameIndex()
2450 unsigned OpCode = IsSALU && !LiveSCC ? AMDGPU::S_LSHR_B32 in eliminateFrameIndex()
2451 : AMDGPU::V_LSHRREV_B32_e64; in eliminateFrameIndex()
2453 if (OpCode == AMDGPU::V_LSHRREV_B32_e64) in eliminateFrameIndex()
2463 AMDGPU::SReg_32RegClass, Shift, false, 0); in eliminateFrameIndex()
2464 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), in eliminateFrameIndex()
2477 BuildMI(*MBB, *MIB, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64), in eliminateFrameIndex()
2482 const bool IsVOP2 = MIB->getOpcode() == AMDGPU::V_ADD_U32_e32; in eliminateFrameIndex()
2485 if (IsVOP2 || AMDGPU::isInlinableLiteral32(Offset, ST.hasInv2PiInlineImm())) { in eliminateFrameIndex()
2492 assert(MIB->getOpcode() == AMDGPU::V_ADD_CO_U32_e64 && in eliminateFrameIndex()
2498 ConstOffsetReg = getSubReg(MIB.getReg(1), AMDGPU::sub0); in eliminateFrameIndex()
2502 BuildMI(*MBB, *MIB, DL, TII->get(AMDGPU::S_MOV_B32), ConstOffsetReg) in eliminateFrameIndex()
2518 AMDGPU::SReg_32_XM0RegClass, MI, false, 0, false); in eliminateFrameIndex()
2521 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_LSHR_B32), ScaledReg) in eliminateFrameIndex()
2524 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), ScaledReg) in eliminateFrameIndex()
2528 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::COPY), ResultReg) in eliminateFrameIndex()
2535 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), ScaledReg) in eliminateFrameIndex()
2538 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_LSHL_B32), ScaledReg) in eliminateFrameIndex()
2557 AMDGPU::getNamedOperandIdx(MI->getOpcode(), in eliminateFrameIndex()
2558 AMDGPU::OpName::vaddr)); in eliminateFrameIndex()
2560 auto &SOffset = *TII->getNamedOperand(*MI, AMDGPU::OpName::soffset); in eliminateFrameIndex()
2563 if (FrameReg != AMDGPU::NoRegister) in eliminateFrameIndex()
2568 = TII->getNamedOperand(*MI, AMDGPU::OpName::offset)->getImm(); in eliminateFrameIndex()
2583 Register TmpReg = RS->scavengeRegisterBackwards(AMDGPU::VGPR_32RegClass, in eliminateFrameIndex()
2585 BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpReg) in eliminateFrameIndex()
2598 unsigned AMDGPU::getRegBitWidth(const TargetRegisterClass &RC) { in getRegBitWidth()
2605 return &AMDGPU::VReg_64RegClass; in getAnyVGPRClassForBitWidth()
2607 return &AMDGPU::VReg_96RegClass; in getAnyVGPRClassForBitWidth()
2609 return &AMDGPU::VReg_128RegClass; in getAnyVGPRClassForBitWidth()
2611 return &AMDGPU::VReg_160RegClass; in getAnyVGPRClassForBitWidth()
2613 return &AMDGPU::VReg_192RegClass; in getAnyVGPRClassForBitWidth()
2615 return &AMDGPU::VReg_224RegClass; in getAnyVGPRClassForBitWidth()
2617 return &AMDGPU::VReg_256RegClass; in getAnyVGPRClassForBitWidth()
2619 return &AMDGPU::VReg_288RegClass; in getAnyVGPRClassForBitWidth()
2621 return &AMDGPU::VReg_320RegClass; in getAnyVGPRClassForBitWidth()
2623 return &AMDGPU::VReg_352RegClass; in getAnyVGPRClassForBitWidth()
2625 return &AMDGPU::VReg_384RegClass; in getAnyVGPRClassForBitWidth()
2627 return &AMDGPU::VReg_512RegClass; in getAnyVGPRClassForBitWidth()
2629 return &AMDGPU::VReg_1024RegClass; in getAnyVGPRClassForBitWidth()
2637 return &AMDGPU::VReg_64_Align2RegClass; in getAlignedVGPRClassForBitWidth()
2639 return &AMDGPU::VReg_96_Align2RegClass; in getAlignedVGPRClassForBitWidth()
2641 return &AMDGPU::VReg_128_Align2RegClass; in getAlignedVGPRClassForBitWidth()
2643 return &AMDGPU::VReg_160_Align2RegClass; in getAlignedVGPRClassForBitWidth()
2645 return &AMDGPU::VReg_192_Align2RegClass; in getAlignedVGPRClassForBitWidth()
2647 return &AMDGPU::VReg_224_Align2RegClass; in getAlignedVGPRClassForBitWidth()
2649 return &AMDGPU::VReg_256_Align2RegClass; in getAlignedVGPRClassForBitWidth()
2651 return &AMDGPU::VReg_288_Align2RegClass; in getAlignedVGPRClassForBitWidth()
2653 return &AMDGPU::VReg_320_Align2RegClass; in getAlignedVGPRClassForBitWidth()
2655 return &AMDGPU::VReg_352_Align2RegClass; in getAlignedVGPRClassForBitWidth()
2657 return &AMDGPU::VReg_384_Align2RegClass; in getAlignedVGPRClassForBitWidth()
2659 return &AMDGPU::VReg_512_Align2RegClass; in getAlignedVGPRClassForBitWidth()
2661 return &AMDGPU::VReg_1024_Align2RegClass; in getAlignedVGPRClassForBitWidth()
2669 return &AMDGPU::VReg_1RegClass; in getVGPRClassForBitWidth()
2671 return &AMDGPU::VGPR_16RegClass; in getVGPRClassForBitWidth()
2673 return &AMDGPU::VGPR_32RegClass; in getVGPRClassForBitWidth()
2681 return &AMDGPU::AReg_64RegClass; in getAnyAGPRClassForBitWidth()
2683 return &AMDGPU::AReg_96RegClass; in getAnyAGPRClassForBitWidth()
2685 return &AMDGPU::AReg_128RegClass; in getAnyAGPRClassForBitWidth()
2687 return &AMDGPU::AReg_160RegClass; in getAnyAGPRClassForBitWidth()
2689 return &AMDGPU::AReg_192RegClass; in getAnyAGPRClassForBitWidth()
2691 return &AMDGPU::AReg_224RegClass; in getAnyAGPRClassForBitWidth()
2693 return &AMDGPU::AReg_256RegClass; in getAnyAGPRClassForBitWidth()
2695 return &AMDGPU::AReg_288RegClass; in getAnyAGPRClassForBitWidth()
2697 return &AMDGPU::AReg_320RegClass; in getAnyAGPRClassForBitWidth()
2699 return &AMDGPU::AReg_352RegClass; in getAnyAGPRClassForBitWidth()
2701 return &AMDGPU::AReg_384RegClass; in getAnyAGPRClassForBitWidth()
2703 return &AMDGPU::AReg_512RegClass; in getAnyAGPRClassForBitWidth()
2705 return &AMDGPU::AReg_1024RegClass; in getAnyAGPRClassForBitWidth()
2713 return &AMDGPU::AReg_64_Align2RegClass; in getAlignedAGPRClassForBitWidth()
2715 return &AMDGPU::AReg_96_Align2RegClass; in getAlignedAGPRClassForBitWidth()
2717 return &AMDGPU::AReg_128_Align2RegClass; in getAlignedAGPRClassForBitWidth()
2719 return &AMDGPU::AReg_160_Align2RegClass; in getAlignedAGPRClassForBitWidth()
2721 return &AMDGPU::AReg_192_Align2RegClass; in getAlignedAGPRClassForBitWidth()
2723 return &AMDGPU::AReg_224_Align2RegClass; in getAlignedAGPRClassForBitWidth()
2725 return &AMDGPU::AReg_256_Align2RegClass; in getAlignedAGPRClassForBitWidth()
2727 return &AMDGPU::AReg_288_Align2RegClass; in getAlignedAGPRClassForBitWidth()
2729 return &AMDGPU::AReg_320_Align2RegClass; in getAlignedAGPRClassForBitWidth()
2731 return &AMDGPU::AReg_352_Align2RegClass; in getAlignedAGPRClassForBitWidth()
2733 return &AMDGPU::AReg_384_Align2RegClass; in getAlignedAGPRClassForBitWidth()
2735 return &AMDGPU::AReg_512_Align2RegClass; in getAlignedAGPRClassForBitWidth()
2737 return &AMDGPU::AReg_1024_Align2RegClass; in getAlignedAGPRClassForBitWidth()
2745 return &AMDGPU::AGPR_LO16RegClass; in getAGPRClassForBitWidth()
2747 return &AMDGPU::AGPR_32RegClass; in getAGPRClassForBitWidth()
2755 return &AMDGPU::AV_64RegClass; in getAnyVectorSuperClassForBitWidth()
2757 return &AMDGPU::AV_96RegClass; in getAnyVectorSuperClassForBitWidth()
2759 return &AMDGPU::AV_128RegClass; in getAnyVectorSuperClassForBitWidth()
2761 return &AMDGPU::AV_160RegClass; in getAnyVectorSuperClassForBitWidth()
2763 return &AMDGPU::AV_192RegClass; in getAnyVectorSuperClassForBitWidth()
2765 return &AMDGPU::AV_224RegClass; in getAnyVectorSuperClassForBitWidth()
2767 return &AMDGPU::AV_256RegClass; in getAnyVectorSuperClassForBitWidth()
2769 return &AMDGPU::AV_288RegClass; in getAnyVectorSuperClassForBitWidth()
2771 return &AMDGPU::AV_320RegClass; in getAnyVectorSuperClassForBitWidth()
2773 return &AMDGPU::AV_352RegClass; in getAnyVectorSuperClassForBitWidth()
2775 return &AMDGPU::AV_384RegClass; in getAnyVectorSuperClassForBitWidth()
2777 return &AMDGPU::AV_512RegClass; in getAnyVectorSuperClassForBitWidth()
2779 return &AMDGPU::AV_1024RegClass; in getAnyVectorSuperClassForBitWidth()
2787 return &AMDGPU::AV_64_Align2RegClass; in getAlignedVectorSuperClassForBitWidth()
2789 return &AMDGPU::AV_96_Align2RegClass; in getAlignedVectorSuperClassForBitWidth()
2791 return &AMDGPU::AV_128_Align2RegClass; in getAlignedVectorSuperClassForBitWidth()
2793 return &AMDGPU::AV_160_Align2RegClass; in getAlignedVectorSuperClassForBitWidth()
2795 return &AMDGPU::AV_192_Align2RegClass; in getAlignedVectorSuperClassForBitWidth()
2797 return &AMDGPU::AV_224_Align2RegClass; in getAlignedVectorSuperClassForBitWidth()
2799 return &AMDGPU::AV_256_Align2RegClass; in getAlignedVectorSuperClassForBitWidth()
2801 return &AMDGPU::AV_288_Align2RegClass; in getAlignedVectorSuperClassForBitWidth()
2803 return &AMDGPU::AV_320_Align2RegClass; in getAlignedVectorSuperClassForBitWidth()
2805 return &AMDGPU::AV_352_Align2RegClass; in getAlignedVectorSuperClassForBitWidth()
2807 return &AMDGPU::AV_384_Align2RegClass; in getAlignedVectorSuperClassForBitWidth()
2809 return &AMDGPU::AV_512_Align2RegClass; in getAlignedVectorSuperClassForBitWidth()
2811 return &AMDGPU::AV_1024_Align2RegClass; in getAlignedVectorSuperClassForBitWidth()
2819 return &AMDGPU::AV_32RegClass; in getVectorSuperClassForBitWidth()
2828 return &AMDGPU::SGPR_LO16RegClass; in getSGPRClassForBitWidth()
2830 return &AMDGPU::SReg_32RegClass; in getSGPRClassForBitWidth()
2832 return &AMDGPU::SReg_64RegClass; in getSGPRClassForBitWidth()
2834 return &AMDGPU::SGPR_96RegClass; in getSGPRClassForBitWidth()
2836 return &AMDGPU::SGPR_128RegClass; in getSGPRClassForBitWidth()
2838 return &AMDGPU::SGPR_160RegClass; in getSGPRClassForBitWidth()
2840 return &AMDGPU::SGPR_192RegClass; in getSGPRClassForBitWidth()
2842 return &AMDGPU::SGPR_224RegClass; in getSGPRClassForBitWidth()
2844 return &AMDGPU::SGPR_256RegClass; in getSGPRClassForBitWidth()
2846 return &AMDGPU::SGPR_288RegClass; in getSGPRClassForBitWidth()
2848 return &AMDGPU::SGPR_320RegClass; in getSGPRClassForBitWidth()
2850 return &AMDGPU::SGPR_352RegClass; in getSGPRClassForBitWidth()
2852 return &AMDGPU::SGPR_384RegClass; in getSGPRClassForBitWidth()
2854 return &AMDGPU::SGPR_512RegClass; in getSGPRClassForBitWidth()
2856 return &AMDGPU::SGPR_1024RegClass; in getSGPRClassForBitWidth()
2891 return &AMDGPU::SGPR_32RegClass; in getEquivalentSGPRClass()
2908 if (OpType >= AMDGPU::OPERAND_REG_INLINE_AC_FIRST && in opCanUseInlineConstant()
2909 OpType <= AMDGPU::OPERAND_REG_INLINE_AC_LAST) in opCanUseInlineConstant()
2912 return OpType >= AMDGPU::OPERAND_SRC_FIRST && in opCanUseInlineConstant()
2913 OpType <= AMDGPU::OPERAND_SRC_LAST; in opCanUseInlineConstant()
2942 return OpType >= AMDGPU::OPERAND_REG_IMM_FIRST && in opCanUseLiteralConstant()
2943 OpType <= AMDGPU::OPERAND_REG_IMM_LAST; in opCanUseLiteralConstant()
2977 const unsigned RegBitWidth = AMDGPU::getRegBitWidth(*RC); in getRegSplitParts()
3048 case AMDGPU::VGPR_32RegClassID: in getRegPressureLimit()
3050 case AMDGPU::SGPR_32RegClassID: in getRegPressureLimit()
3051 case AMDGPU::SGPR_LO16RegClassID: in getRegPressureLimit()
3058 if (Idx == AMDGPU::RegisterPressureSets::VGPR_32 || in getRegPressureSetLimit()
3059 Idx == AMDGPU::RegisterPressureSets::AGPR_32) in getRegPressureSetLimit()
3060 return getRegPressureLimit(&AMDGPU::VGPR_32RegClass, in getRegPressureSetLimit()
3063 if (Idx == AMDGPU::RegisterPressureSets::SReg_32) in getRegPressureSetLimit()
3064 return getRegPressureLimit(&AMDGPU::SGPR_32RegClass, in getRegPressureSetLimit()
3081 return AMDGPU::SGPR30_SGPR31; in getReturnAddressReg()
3088 case AMDGPU::VGPRRegBankID: in getRegClassForSizeOnBank()
3091 case AMDGPU::VCCRegBankID: in getRegClassForSizeOnBank()
3093 return isWave32 ? &AMDGPU::SReg_32_XM0_XEXECRegClass in getRegClassForSizeOnBank()
3094 : &AMDGPU::SReg_64_XEXECRegClass; in getRegClassForSizeOnBank()
3095 case AMDGPU::SGPRRegBankID: in getRegClassForSizeOnBank()
3097 case AMDGPU::AGPRRegBankID: in getRegClassForSizeOnBank()
3118 return isWave32 ? AMDGPU::VCC_LO : AMDGPU::VCC; in getVCC()
3122 return isWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC; in getExec()
3127 return ST.needsAlignedVGPRs() ? &AMDGPU::VReg_64_Align2RegClass in getVGPR64Class()
3128 : &AMDGPU::VReg_64RegClass; in getVGPR64Class()
3134 case AMDGPU::SReg_1RegClassID: in getRegClass()
3136 case AMDGPU::SReg_1_XEXECRegClassID: in getRegClass()
3137 return isWave32 ? &AMDGPU::SReg_32_XM0_XEXECRegClass in getRegClass()
3138 : &AMDGPU::SReg_64_XEXECRegClass; in getRegClass()
3203 for (const TargetRegisterClass &RC : { AMDGPU::VGPR_32RegClass, in get32BitRegister()
3204 AMDGPU::SReg_32RegClass, in get32BitRegister()
3205 AMDGPU::AGPR_32RegClass } ) { in get32BitRegister()
3206 if (MCPhysReg Super = getMatchingSuperReg(Reg, AMDGPU::lo16, &RC)) in get32BitRegister()
3209 if (MCPhysReg Super = getMatchingSuperReg(Reg, AMDGPU::hi16, in get32BitRegister()
3210 &AMDGPU::VGPR_32RegClass)) { in get32BitRegister()
3214 return AMDGPU::NoRegister; in get32BitRegister()
3253 return ArrayRef(AMDGPU::SGPR_128RegClass.begin(), ST.getMaxNumSGPRs(MF) / 4); in getAllSGPR128()
3258 return ArrayRef(AMDGPU::SGPR_64RegClass.begin(), ST.getMaxNumSGPRs(MF) / 2); in getAllSGPR64()
3263 return ArrayRef(AMDGPU::SGPR_32RegClass.begin(), ST.getMaxNumSGPRs(MF)); in getAllSGPR32()