Lines Matching refs:ScoreBrackets
466 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
493 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
528 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
608 WaitcntBrackets &ScoreBrackets);
693 WaitcntBrackets &ScoreBrackets,
697 WaitcntBrackets &ScoreBrackets,
701 MachineBasicBlock &Block, WaitcntBrackets &ScoreBrackets,
704 WaitcntBrackets *ScoreBrackets);
706 WaitcntBrackets &ScoreBrackets);
1175 WaitcntBrackets &ScoreBrackets, MachineInstr &OldWaitcntInstr, in applyPreexistingWaitcnt() argument
1198 ScoreBrackets.simplifyWaitcnt(OldWait); in applyPreexistingWaitcnt()
1214 ScoreBrackets.simplifyWaitcnt(InstCounterType::STORE_CNT, OldVSCnt); in applyPreexistingWaitcnt()
1230 ScoreBrackets.applyWaitcnt(LOAD_CNT, Wait.LoadCnt); in applyPreexistingWaitcnt()
1231 ScoreBrackets.applyWaitcnt(EXP_CNT, Wait.ExpCnt); in applyPreexistingWaitcnt()
1232 ScoreBrackets.applyWaitcnt(DS_CNT, Wait.DsCnt); in applyPreexistingWaitcnt()
1251 ScoreBrackets.applyWaitcnt(STORE_CNT, Wait.StoreCnt); in applyPreexistingWaitcnt()
1312 WaitcntBrackets &ScoreBrackets, MachineInstr &OldWaitcntInstr, in applyPreexistingWaitcnt() argument
1340 ScoreBrackets.simplifyWaitcnt(OldWait); in applyPreexistingWaitcnt()
1348 ScoreBrackets.simplifyWaitcnt(OldWait); in applyPreexistingWaitcnt()
1357 ScoreBrackets.simplifyWaitcnt(CT.value(), OldCnt); in applyPreexistingWaitcnt()
1384 ScoreBrackets.applyWaitcnt(LOAD_CNT, Wait.LoadCnt); in applyPreexistingWaitcnt()
1385 ScoreBrackets.applyWaitcnt(DS_CNT, Wait.DsCnt); in applyPreexistingWaitcnt()
1409 ScoreBrackets.applyWaitcnt(STORE_CNT, Wait.StoreCnt); in applyPreexistingWaitcnt()
1410 ScoreBrackets.applyWaitcnt(DS_CNT, Wait.DsCnt); in applyPreexistingWaitcnt()
1470 ScoreBrackets.applyWaitcnt(CT, NewCnt); in applyPreexistingWaitcnt()
1587 WaitcntBrackets &ScoreBrackets, in generateWaitcntInstBefore() argument
1628 ScoreBrackets.getScoreRange(STORE_CNT) != 0 && in generateWaitcntInstBefore()
1629 !ScoreBrackets.hasPendingEvent(SCRATCH_WRITE_ACCESS)) in generateWaitcntInstBefore()
1655 EmitWaitcnt |= ScoreBrackets->updateByWait(DS_CNT, in generateWaitcntInstBefore()
1656 ScoreBrackets->getScoreUB(DS_CNT)); in generateWaitcntInstBefore()
1659 EmitWaitcnt |= ScoreBrackets->updateByWait(LOAD_CNT, in generateWaitcntInstBefore()
1660 ScoreBrackets->getScoreUB(LOAD_CNT)); in generateWaitcntInstBefore()
1667 EmitWaitcnt |= ScoreBrackets->updateByWait(EXP_CNT, in generateWaitcntInstBefore()
1668 ScoreBrackets->getScoreUB(EXP_CNT)); in generateWaitcntInstBefore()
1669 EmitWaitcnt |= ScoreBrackets->updateByWait(DS_CNT, in generateWaitcntInstBefore()
1670 ScoreBrackets->getScoreUB(DS_CNT)); in generateWaitcntInstBefore()
1679 EmitWaitcnt |= ScoreBrackets->updateByWait(EXP_CNT, in generateWaitcntInstBefore()
1680 ScoreBrackets->getScoreUB(EXP_CNT)); in generateWaitcntInstBefore()
1681 EmitWaitcnt |= ScoreBrackets->updateByWait(LOAD_CNT, in generateWaitcntInstBefore()
1682 ScoreBrackets->getScoreUB(LOAD_CNT)); in generateWaitcntInstBefore()
1702 if (ScoreBrackets.hasPendingEvent(EXP_GPR_LOCK) || in generateWaitcntInstBefore()
1703 ScoreBrackets.hasPendingEvent(EXP_PARAM_ACCESS) || in generateWaitcntInstBefore()
1704 ScoreBrackets.hasPendingEvent(EXP_POS_ACCESS) || in generateWaitcntInstBefore()
1705 ScoreBrackets.hasPendingEvent(GDS_GPR_LOCK)) { in generateWaitcntInstBefore()
1721 ScoreBrackets.getRegInterval(&MI, MRI, TRI, CallAddrOpIdx); in generateWaitcntInstBefore()
1725 ScoreBrackets.determineWait(SmemAccessCounter, RegNo, Wait); in generateWaitcntInstBefore()
1731 ScoreBrackets.getRegInterval(&MI, MRI, TRI, RtnAddrOpIdx); in generateWaitcntInstBefore()
1735 ScoreBrackets.determineWait(SmemAccessCounter, RegNo, Wait); in generateWaitcntInstBefore()
1777 const auto &LDSDMAStores = ScoreBrackets.getLDSDMAStores(); in generateWaitcntInstBefore()
1781 ScoreBrackets.determineWait(LOAD_CNT, RegNo + I + 1, Wait); in generateWaitcntInstBefore()
1786 ScoreBrackets.determineWait(LOAD_CNT, RegNo, Wait); in generateWaitcntInstBefore()
1788 ScoreBrackets.determineWait(EXP_CNT, RegNo, Wait); in generateWaitcntInstBefore()
1802 RegInterval Interval = ScoreBrackets.getRegInterval(&MI, MRI, TRI, I); in generateWaitcntInstBefore()
1812 ScoreBrackets.hasOtherPendingVmemTypes(RegNo, in generateWaitcntInstBefore()
1814 ScoreBrackets.determineWait(LOAD_CNT, RegNo, Wait); in generateWaitcntInstBefore()
1815 ScoreBrackets.determineWait(SAMPLE_CNT, RegNo, Wait); in generateWaitcntInstBefore()
1816 ScoreBrackets.determineWait(BVH_CNT, RegNo, Wait); in generateWaitcntInstBefore()
1817 ScoreBrackets.clearVgprVmemTypes(RegNo); in generateWaitcntInstBefore()
1819 if (Op.isDef() || ScoreBrackets.hasPendingEvent(EXP_LDS_ACCESS)) { in generateWaitcntInstBefore()
1820 ScoreBrackets.determineWait(EXP_CNT, RegNo, Wait); in generateWaitcntInstBefore()
1822 ScoreBrackets.determineWait(DS_CNT, RegNo, Wait); in generateWaitcntInstBefore()
1824 ScoreBrackets.determineWait(SmemAccessCounter, RegNo, Wait); in generateWaitcntInstBefore()
1845 if (ScoreBrackets.hasPendingEvent(SMEM_ACCESS)) { in generateWaitcntInstBefore()
1851 ScoreBrackets.simplifyWaitcnt(Wait); in generateWaitcntInstBefore()
1870 if (ScoreBrackets.hasPendingEvent(LOAD_CNT)) in generateWaitcntInstBefore()
1872 if (ScoreBrackets.hasPendingEvent(SAMPLE_CNT)) in generateWaitcntInstBefore()
1874 if (ScoreBrackets.hasPendingEvent(BVH_CNT)) in generateWaitcntInstBefore()
1878 return generateWaitcnt(Wait, MI.getIterator(), *MI.getParent(), ScoreBrackets, in generateWaitcntInstBefore()
1885 WaitcntBrackets &ScoreBrackets, in generateWaitcntBlockEnd() argument
1889 unsigned LoadCntPending = ScoreBrackets.hasPendingEvent(LOAD_CNT); in generateWaitcntBlockEnd()
1890 unsigned SampleCntPending = ScoreBrackets.hasPendingEvent(SAMPLE_CNT); in generateWaitcntBlockEnd()
1891 unsigned BvhCntPending = ScoreBrackets.hasPendingEvent(BVH_CNT); in generateWaitcntBlockEnd()
1903 return generateWaitcnt(Wait, Block.instr_end(), Block, ScoreBrackets, in generateWaitcntBlockEnd()
1910 WaitcntBrackets &ScoreBrackets, in generateWaitcnt() argument
1918 WCG->applyPreexistingWaitcnt(ScoreBrackets, *OldWaitcntInstr, Wait, It); in generateWaitcnt()
1922 ScoreBrackets.applyWaitcnt(Wait); in generateWaitcnt()
2035 WaitcntBrackets *ScoreBrackets) { in updateEventWaitcntAfter() argument
2044 ScoreBrackets->updateByEvent(TII, TRI, MRI, GDS_ACCESS, Inst); in updateEventWaitcntAfter()
2045 ScoreBrackets->updateByEvent(TII, TRI, MRI, GDS_GPR_LOCK, Inst); in updateEventWaitcntAfter()
2047 ScoreBrackets->updateByEvent(TII, TRI, MRI, LDS_ACCESS, Inst); in updateEventWaitcntAfter()
2060 ScoreBrackets->updateByEvent(TII, TRI, MRI, getVmemWaitEventType(Inst), in updateEventWaitcntAfter()
2066 ScoreBrackets->updateByEvent(TII, TRI, MRI, LDS_ACCESS, Inst); in updateEventWaitcntAfter()
2076 ScoreBrackets->setPendingFlat(); in updateEventWaitcntAfter()
2079 ScoreBrackets->updateByEvent(TII, TRI, MRI, getVmemWaitEventType(Inst), in updateEventWaitcntAfter()
2084 ScoreBrackets->updateByEvent(TII, TRI, MRI, VMW_GPR_LOCK, Inst); in updateEventWaitcntAfter()
2087 ScoreBrackets->updateByEvent(TII, TRI, MRI, SMEM_ACCESS, Inst); in updateEventWaitcntAfter()
2091 ScoreBrackets->applyWaitcnt( in updateEventWaitcntAfter()
2093 ScoreBrackets->setStateOnFunctionEntryOrReturn(); in updateEventWaitcntAfter()
2096 ScoreBrackets->applyWaitcnt(AMDGPU::Waitcnt()); in updateEventWaitcntAfter()
2099 ScoreBrackets->updateByEvent(TII, TRI, MRI, EXP_LDS_ACCESS, Inst); in updateEventWaitcntAfter()
2102 ScoreBrackets->applyWaitcnt(EXP_CNT, Imm); in updateEventWaitcntAfter()
2106 ScoreBrackets->updateByEvent(TII, TRI, MRI, EXP_PARAM_ACCESS, Inst); in updateEventWaitcntAfter()
2108 ScoreBrackets->updateByEvent(TII, TRI, MRI, EXP_POS_ACCESS, Inst); in updateEventWaitcntAfter()
2110 ScoreBrackets->updateByEvent(TII, TRI, MRI, EXP_GPR_LOCK, Inst); in updateEventWaitcntAfter()
2117 ScoreBrackets->updateByEvent(TII, TRI, MRI, SQ_MESSAGE, Inst); in updateEventWaitcntAfter()
2126 ScoreBrackets->updateByEvent(TII, TRI, MRI, SMEM_ACCESS, Inst); in updateEventWaitcntAfter()
2208 WaitcntBrackets &ScoreBrackets) { in insertWaitcntInBlock() argument
2213 ScoreBrackets.dump(); in insertWaitcntInBlock()
2248 isPreheaderToFlush(Block, ScoreBrackets); in insertWaitcntInBlock()
2251 Modified |= generateWaitcntInstBefore(Inst, ScoreBrackets, OldWaitcntInstr, in insertWaitcntInBlock()
2274 ScoreBrackets.hasPendingEvent(SMEM_ACCESS)) { in insertWaitcntInBlock()
2300 updateEventWaitcntAfter(Inst, &ScoreBrackets); in insertWaitcntInBlock()
2308 ScoreBrackets->setScoreLB(LOAD_CNT, in insertWaitcntInBlock()
2309 ScoreBrackets->getScoreUB(LOAD_CNT)); in insertWaitcntInBlock()
2315 ScoreBrackets.dump(); in insertWaitcntInBlock()
2336 isPreheaderToFlush(Block, ScoreBrackets)) in insertWaitcntInBlock()
2337 Modified |= generateWaitcntBlockEnd(Block, ScoreBrackets, OldWaitcntInstr); in insertWaitcntInBlock()
2345 WaitcntBrackets &ScoreBrackets) { in isPreheaderToFlush() argument
2359 shouldFlushVmCnt(Loop, ScoreBrackets)) { in isPreheaderToFlush()