Lines Matching refs:AMDGPU
167 AMDGPU::S_WAIT_LOADCNT, AMDGPU::S_WAIT_DSCNT, AMDGPU::S_WAIT_EXPCNT,
168 AMDGPU::S_WAIT_STORECNT, AMDGPU::S_WAIT_SAMPLECNT, AMDGPU::S_WAIT_BVHCNT,
169 AMDGPU::S_WAIT_KMCNT};
187 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Inst.getOpcode()); in getVmemType()
188 const AMDGPU::MIMGBaseOpcodeInfo *BaseInfo = in getVmemType()
189 AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode); in getVmemType()
194 unsigned &getCounterRef(AMDGPU::Waitcnt &Wait, InstCounterType T) { in getCounterRef()
215 void addWait(AMDGPU::Waitcnt &Wait, InstCounterType T, unsigned Count) { in addWait()
220 void setNoWait(AMDGPU::Waitcnt &Wait, InstCounterType T) { in setNoWait()
224 unsigned getWait(AMDGPU::Waitcnt &Wait, InstCounterType T) { in getWait()
306 void simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const;
308 void determineWait(InstCounterType T, int RegNo, AMDGPU::Waitcnt &Wait) const;
309 void applyWaitcnt(const AMDGPU::Waitcnt &Wait);
445 AMDGPU::IsaVersion IV;
452 IV(AMDGPU::getIsaVersion(ST->getCPU())), MaxCounter(MaxCounter) {} in WaitcntGenerator()
467 MachineInstr &OldWaitcntInstr, AMDGPU::Waitcnt &Wait,
477 AMDGPU::Waitcnt Wait) = 0;
494 MachineInstr &OldWaitcntInstr, AMDGPU::Waitcnt &Wait,
499 AMDGPU::Waitcnt Wait) override;
529 MachineInstr &OldWaitcntInstr, AMDGPU::Waitcnt &Wait,
534 AMDGPU::Waitcnt Wait) override;
699 bool generateWaitcnt(AMDGPU::Waitcnt Wait,
725 unsigned Reg = TRI->getEncodingValue(AMDGPU::getMCReg(Op.getReg(), *ST)) & in getRegInterval()
726 AMDGPU::HWEncoding::REG_IDX_MASK; in getRegInterval()
785 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::addr); in updateByEvent()
793 if (AMDGPU::hasNamedOperand(Inst.getOpcode(), AMDGPU::OpName::data0)) { in updateByEvent()
796 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data0), in updateByEvent()
799 if (AMDGPU::hasNamedOperand(Inst.getOpcode(), AMDGPU::OpName::data1)) { in updateByEvent()
801 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), in updateByEvent()
802 AMDGPU::OpName::data1), in updateByEvent()
806 Inst.getOpcode() != AMDGPU::DS_APPEND && in updateByEvent()
807 Inst.getOpcode() != AMDGPU::DS_CONSUME && in updateByEvent()
808 Inst.getOpcode() != AMDGPU::DS_ORDERED_COUNT) { in updateByEvent()
821 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data), in updateByEvent()
826 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data), in updateByEvent()
835 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data), in updateByEvent()
848 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data), in updateByEvent()
855 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::vdst), in updateByEvent()
868 TRI->getEncodingValue(AMDGPU::getMCReg(DefMO.getReg(), *ST)), in updateByEvent()
882 } else if (Inst.getOpcode() == AMDGPU::BUFFER_STORE_DWORD || in updateByEvent()
883 Inst.getOpcode() == AMDGPU::BUFFER_STORE_DWORDX2 || in updateByEvent()
884 Inst.getOpcode() == AMDGPU::BUFFER_STORE_DWORDX4) { in updateByEvent()
885 MachineOperand *MO = TII->getNamedOperand(Inst, AMDGPU::OpName::data); in updateByEvent()
1027 void WaitcntBrackets::simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const { in simplifyWaitcnt()
1047 AMDGPU::Waitcnt &Wait) const { in determineWait()
1075 void WaitcntBrackets::applyWaitcnt(const AMDGPU::Waitcnt &Wait) { in applyWaitcnt()
1125 int OpIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpName); in updateOperandIfDifferent()
1141 case AMDGPU::S_WAIT_LOADCNT: in counterTypeForInstr()
1143 case AMDGPU::S_WAIT_EXPCNT: in counterTypeForInstr()
1145 case AMDGPU::S_WAIT_STORECNT: in counterTypeForInstr()
1147 case AMDGPU::S_WAIT_SAMPLECNT: in counterTypeForInstr()
1149 case AMDGPU::S_WAIT_BVHCNT: in counterTypeForInstr()
1151 case AMDGPU::S_WAIT_DSCNT: in counterTypeForInstr()
1153 case AMDGPU::S_WAIT_KMCNT: in counterTypeForInstr()
1176 AMDGPU::Waitcnt &Wait, MachineBasicBlock::instr_iterator It) const { in applyPreexistingWaitcnt()
1194 if (Opcode == AMDGPU::S_WAITCNT) { in applyPreexistingWaitcnt()
1196 AMDGPU::Waitcnt OldWait = AMDGPU::decodeWaitcnt(IV, IEnc); in applyPreexistingWaitcnt()
1208 assert(Opcode == AMDGPU::S_WAITCNT_VSCNT); in applyPreexistingWaitcnt()
1209 assert(II.getOperand(0).getReg() == AMDGPU::SGPR_NULL); in applyPreexistingWaitcnt()
1212 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm(); in applyPreexistingWaitcnt()
1226 Modified |= updateOperandIfDifferent(*WaitcntInstr, AMDGPU::OpName::simm16, in applyPreexistingWaitcnt()
1227 AMDGPU::encodeWaitcnt(IV, Wait)); in applyPreexistingWaitcnt()
1248 AMDGPU::OpName::simm16, Wait.StoreCnt); in applyPreexistingWaitcnt()
1270 AMDGPU::Waitcnt Wait) { in createNewWaitcnt()
1280 unsigned Enc = AMDGPU::encodeWaitcnt(IV, Wait); in createNewWaitcnt()
1282 BuildMI(Block, It, DL, TII->get(AMDGPU::S_WAITCNT)).addImm(Enc); in createNewWaitcnt()
1294 BuildMI(Block, It, DL, TII->get(AMDGPU::S_WAITCNT_VSCNT)) in createNewWaitcnt()
1295 .addReg(AMDGPU::SGPR_NULL, RegState::Undef) in createNewWaitcnt()
1313 AMDGPU::Waitcnt &Wait, MachineBasicBlock::instr_iterator It) const { in applyPreexistingWaitcnt()
1335 if (Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT) { in applyPreexistingWaitcnt()
1337 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm(); in applyPreexistingWaitcnt()
1338 AMDGPU::Waitcnt OldWait = AMDGPU::decodeLoadcntDscnt(IV, OldEnc); in applyPreexistingWaitcnt()
1343 } else if (Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT) { in applyPreexistingWaitcnt()
1345 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm(); in applyPreexistingWaitcnt()
1346 AMDGPU::Waitcnt OldWait = AMDGPU::decodeStorecntDscnt(IV, OldEnc); in applyPreexistingWaitcnt()
1355 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm(); in applyPreexistingWaitcnt()
1380 unsigned NewEnc = AMDGPU::encodeLoadcntDscnt(IV, Wait); in applyPreexistingWaitcnt()
1382 AMDGPU::OpName::simm16, NewEnc); in applyPreexistingWaitcnt()
1405 unsigned NewEnc = AMDGPU::encodeStorecntDscnt(IV, Wait); in applyPreexistingWaitcnt()
1407 AMDGPU::OpName::simm16, NewEnc); in applyPreexistingWaitcnt()
1467 AMDGPU::OpName::simm16, NewCnt); in applyPreexistingWaitcnt()
1492 AMDGPU::Waitcnt Wait) { in createNewWaitcnt()
1504 unsigned Enc = AMDGPU::encodeLoadcntDscnt(IV, Wait); in createNewWaitcnt()
1506 SWaitInst = BuildMI(Block, It, DL, TII->get(AMDGPU::S_WAIT_LOADCNT_DSCNT)) in createNewWaitcnt()
1512 unsigned Enc = AMDGPU::encodeStorecntDscnt(IV, Wait); in createNewWaitcnt()
1515 BuildMI(Block, It, DL, TII->get(AMDGPU::S_WAIT_STORECNT_DSCNT)) in createNewWaitcnt()
1555 return (Opc == AMDGPU::S_CBRANCH_VCCNZ || Opc == AMDGPU::S_CBRANCH_VCCZ) && in readsVCCZ()
1595 AMDGPU::Waitcnt Wait; in generateWaitcntInstBefore()
1601 if (MI.getOpcode() == AMDGPU::BUFFER_WBINVL1 || in generateWaitcntInstBefore()
1602 MI.getOpcode() == AMDGPU::BUFFER_WBINVL1_SC || in generateWaitcntInstBefore()
1603 MI.getOpcode() == AMDGPU::BUFFER_WBINVL1_VOL || in generateWaitcntInstBefore()
1604 MI.getOpcode() == AMDGPU::BUFFER_GL0_INV || in generateWaitcntInstBefore()
1605 MI.getOpcode() == AMDGPU::BUFFER_GL1_INV) { in generateWaitcntInstBefore()
1612 if (MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG || in generateWaitcntInstBefore()
1613 MI.getOpcode() == AMDGPU::SI_RETURN || in generateWaitcntInstBefore()
1614 MI.getOpcode() == AMDGPU::S_SETPC_B64_return || in generateWaitcntInstBefore()
1617 AMDGPU::Waitcnt::allZeroExceptVsCnt(ST->hasExtendedWaitCounts())); in generateWaitcntInstBefore()
1625 else if (MI.getOpcode() == AMDGPU::S_ENDPGM || in generateWaitcntInstBefore()
1626 MI.getOpcode() == AMDGPU::S_ENDPGM_SAVED) { in generateWaitcntInstBefore()
1633 else if ((MI.getOpcode() == AMDGPU::S_SENDMSG || in generateWaitcntInstBefore()
1634 MI.getOpcode() == AMDGPU::S_SENDMSGHALT) && in generateWaitcntInstBefore()
1636 ((MI.getOperand(0).getImm() & AMDGPU::SendMsg::ID_MASK_PreGFX11_) == in generateWaitcntInstBefore()
1637 AMDGPU::SendMsg::ID_GS_DONE_PreGFX11)) { in generateWaitcntInstBefore()
1699 if (MI.modifiesRegister(AMDGPU::EXEC, TRI)) { in generateWaitcntInstBefore()
1714 Wait = AMDGPU::Waitcnt(); in generateWaitcntInstBefore()
1717 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0); in generateWaitcntInstBefore()
1728 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::dst); in generateWaitcntInstBefore()
1838 AMDGPU::Waitcnt::allZero(ST->hasExtendedWaitCounts(), ST->hasVscnt())); in generateWaitcntInstBefore()
1854 Wait = AMDGPU::Waitcnt::allZeroExceptVsCnt(ST->hasExtendedWaitCounts()); in generateWaitcntInstBefore()
1887 AMDGPU::Waitcnt Wait; in generateWaitcntBlockEnd()
1907 bool SIInsertWaitcnts::generateWaitcnt(AMDGPU::Waitcnt Wait, in generateWaitcnt()
1928 TII->getNamedOperand(*It, AMDGPU::OpName::waitexp); in generateWaitcnt()
2030 return Opc == AMDGPU::GLOBAL_INV || Opc == AMDGPU::GLOBAL_WB || in isCacheInvOrWBInst()
2031 Opc == AMDGPU::GLOBAL_WBINV; in isCacheInvOrWBInst()
2043 TII->hasModifiersSet(Inst, AMDGPU::OpName::gds)) { in updateEventWaitcntAfter()
2078 !llvm::AMDGPU::getMUBUFIsBufferInv(Inst.getOpcode())) { in updateEventWaitcntAfter()
2092 AMDGPU::Waitcnt::allZeroExceptVsCnt(ST->hasExtendedWaitCounts())); in updateEventWaitcntAfter()
2096 ScoreBrackets->applyWaitcnt(AMDGPU::Waitcnt()); in updateEventWaitcntAfter()
2101 int64_t Imm = TII->getNamedOperand(Inst, AMDGPU::OpName::waitexp)->getImm(); in updateEventWaitcntAfter()
2104 unsigned Imm = TII->getNamedOperand(Inst, AMDGPU::OpName::tgt)->getImm(); in updateEventWaitcntAfter()
2105 if (Imm >= AMDGPU::Exp::ET_PARAM0 && Imm <= AMDGPU::Exp::ET_PARAM31) in updateEventWaitcntAfter()
2107 else if (Imm >= AMDGPU::Exp::ET_POS0 && Imm <= AMDGPU::Exp::ET_POS_LAST) in updateEventWaitcntAfter()
2113 case AMDGPU::S_SENDMSG: in updateEventWaitcntAfter()
2114 case AMDGPU::S_SENDMSG_RTN_B32: in updateEventWaitcntAfter()
2115 case AMDGPU::S_SENDMSG_RTN_B64: in updateEventWaitcntAfter()
2116 case AMDGPU::S_SENDMSGHALT: in updateEventWaitcntAfter()
2119 case AMDGPU::S_MEMTIME: in updateEventWaitcntAfter()
2120 case AMDGPU::S_MEMREALTIME: in updateEventWaitcntAfter()
2121 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0: in updateEventWaitcntAfter()
2122 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM: in updateEventWaitcntAfter()
2123 case AMDGPU::S_BARRIER_LEAVE: in updateEventWaitcntAfter()
2124 case AMDGPU::S_GET_BARRIER_STATE_M0: in updateEventWaitcntAfter()
2125 case AMDGPU::S_GET_BARRIER_STATE_IMM: in updateEventWaitcntAfter()
2197 return Opcode == AMDGPU::S_WAITCNT || in isWaitInstr()
2198 (Opcode == AMDGPU::S_WAITCNT_VSCNT && Inst.getOperand(0).isReg() && in isWaitInstr()
2199 Inst.getOperand(0).getReg() == AMDGPU::SGPR_NULL) || in isWaitInstr()
2200 Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT || in isWaitInstr()
2201 Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT || in isWaitInstr()
2260 if (Inst.definesRegister(AMDGPU::VCC_LO) || in insertWaitcntInBlock()
2261 Inst.definesRegister(AMDGPU::VCC_HI)) { in insertWaitcntInBlock()
2265 } else if (Inst.definesRegister(AMDGPU::VCC)) { in insertWaitcntInBlock()
2325 TII->get(ST->isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64), in insertWaitcntInBlock()
2450 AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(ST->getCPU()); in runOnMachineFunction()
2475 Limits.LoadcntMax = AMDGPU::getLoadcntBitMask(IV); in runOnMachineFunction()
2476 Limits.DscntMax = AMDGPU::getDscntBitMask(IV); in runOnMachineFunction()
2478 Limits.LoadcntMax = AMDGPU::getVmcntBitMask(IV); in runOnMachineFunction()
2479 Limits.DscntMax = AMDGPU::getLgkmcntBitMask(IV); in runOnMachineFunction()
2481 Limits.ExpcntMax = AMDGPU::getExpcntBitMask(IV); in runOnMachineFunction()
2482 Limits.StorecntMax = AMDGPU::getStorecntBitMask(IV); in runOnMachineFunction()
2483 Limits.SamplecntMax = AMDGPU::getSamplecntBitMask(IV); in runOnMachineFunction()
2484 Limits.BvhcntMax = AMDGPU::getBvhcntBitMask(IV); in runOnMachineFunction()
2485 Limits.KmcntMax = AMDGPU::getKmcntBitMask(IV); in runOnMachineFunction()
2494 TRI->getEncodingValue(AMDGPU::VGPR0) & AMDGPU::HWEncoding::REG_IDX_MASK; in runOnMachineFunction()
2497 TRI->getEncodingValue(AMDGPU::SGPR0) & AMDGPU::HWEncoding::REG_IDX_MASK; in runOnMachineFunction()
2518 BuildMI(EntryBB, I, DebugLoc(), TII->get(AMDGPU::S_WAIT_LOADCNT_DSCNT)) in runOnMachineFunction()
2529 BuildMI(EntryBB, I, DebugLoc(), TII->get(AMDGPU::S_WAITCNT)).addImm(0); in runOnMachineFunction()
2611 if (MI.getOpcode() == AMDGPU::S_ENDPGM || in runOnMachineFunction()
2612 MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) in runOnMachineFunction()
2631 if (I->getOpcode() == AMDGPU::S_DCACHE_WB) in runOnMachineFunction()
2637 if ((I->getOpcode() == AMDGPU::S_ENDPGM || in runOnMachineFunction()
2638 I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) && in runOnMachineFunction()
2641 BuildMI(*MBB, I, I->getDebugLoc(), TII->get(AMDGPU::S_DCACHE_WB)); in runOnMachineFunction()
2652 BuildMI(*MI->getParent(), MI, DebugLoc(), TII->get(AMDGPU::S_NOP)) in runOnMachineFunction()
2655 BuildMI(*MI->getParent(), MI, DebugLoc(), TII->get(AMDGPU::S_SENDMSG)) in runOnMachineFunction()
2656 .addImm(AMDGPU::SendMsg::ID_DEALLOC_VGPRS_GFX11Plus); in runOnMachineFunction()