Lines Matching refs:Wait
194 unsigned &getCounterRef(AMDGPU::Waitcnt &Wait, InstCounterType T) { in getCounterRef() argument
197 return Wait.LoadCnt; in getCounterRef()
199 return Wait.ExpCnt; in getCounterRef()
201 return Wait.DsCnt; in getCounterRef()
203 return Wait.StoreCnt; in getCounterRef()
205 return Wait.SampleCnt; in getCounterRef()
207 return Wait.BvhCnt; in getCounterRef()
209 return Wait.KmCnt; in getCounterRef()
215 void addWait(AMDGPU::Waitcnt &Wait, InstCounterType T, unsigned Count) { in addWait() argument
216 unsigned &WC = getCounterRef(Wait, T); in addWait()
220 void setNoWait(AMDGPU::Waitcnt &Wait, InstCounterType T) { in setNoWait() argument
221 getCounterRef(Wait, T) = ~0u; in setNoWait()
224 unsigned getWait(AMDGPU::Waitcnt &Wait, InstCounterType T) { in getWait() argument
225 return getCounterRef(Wait, T); in getWait()
306 void simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const;
308 void determineWait(InstCounterType T, int RegNo, AMDGPU::Waitcnt &Wait) const;
309 void applyWaitcnt(const AMDGPU::Waitcnt &Wait);
467 MachineInstr &OldWaitcntInstr, AMDGPU::Waitcnt &Wait,
477 AMDGPU::Waitcnt Wait) = 0;
494 MachineInstr &OldWaitcntInstr, AMDGPU::Waitcnt &Wait,
499 AMDGPU::Waitcnt Wait) override;
529 MachineInstr &OldWaitcntInstr, AMDGPU::Waitcnt &Wait,
534 AMDGPU::Waitcnt Wait) override;
699 bool generateWaitcnt(AMDGPU::Waitcnt Wait,
1027 void WaitcntBrackets::simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const { in simplifyWaitcnt()
1028 simplifyWaitcnt(LOAD_CNT, Wait.LoadCnt); in simplifyWaitcnt()
1029 simplifyWaitcnt(EXP_CNT, Wait.ExpCnt); in simplifyWaitcnt()
1030 simplifyWaitcnt(DS_CNT, Wait.DsCnt); in simplifyWaitcnt()
1031 simplifyWaitcnt(STORE_CNT, Wait.StoreCnt); in simplifyWaitcnt()
1032 simplifyWaitcnt(SAMPLE_CNT, Wait.SampleCnt); in simplifyWaitcnt()
1033 simplifyWaitcnt(BVH_CNT, Wait.BvhCnt); in simplifyWaitcnt()
1034 simplifyWaitcnt(KM_CNT, Wait.KmCnt); in simplifyWaitcnt()
1047 AMDGPU::Waitcnt &Wait) const { in determineWait()
1060 addWait(Wait, T, 0); in determineWait()
1065 addWait(Wait, T, 0); in determineWait()
1070 addWait(Wait, T, NeededWait); in determineWait()
1075 void WaitcntBrackets::applyWaitcnt(const AMDGPU::Waitcnt &Wait) { in applyWaitcnt() argument
1076 applyWaitcnt(LOAD_CNT, Wait.LoadCnt); in applyWaitcnt()
1077 applyWaitcnt(EXP_CNT, Wait.ExpCnt); in applyWaitcnt()
1078 applyWaitcnt(DS_CNT, Wait.DsCnt); in applyWaitcnt()
1079 applyWaitcnt(STORE_CNT, Wait.StoreCnt); in applyWaitcnt()
1080 applyWaitcnt(SAMPLE_CNT, Wait.SampleCnt); in applyWaitcnt()
1081 applyWaitcnt(BVH_CNT, Wait.BvhCnt); in applyWaitcnt()
1082 applyWaitcnt(KM_CNT, Wait.KmCnt); in applyWaitcnt()
1176 AMDGPU::Waitcnt &Wait, MachineBasicBlock::instr_iterator It) const { in applyPreexistingWaitcnt() argument
1199 Wait = Wait.combined(OldWait); in applyPreexistingWaitcnt()
1202 if (WaitcntInstr || (!Wait.hasWaitExceptStoreCnt() && IsSoft)) { in applyPreexistingWaitcnt()
1215 Wait.StoreCnt = std::min(Wait.StoreCnt, OldVSCnt); in applyPreexistingWaitcnt()
1217 if (WaitcntVsCntInstr || (!Wait.hasWaitStoreCnt() && IsSoft)) { in applyPreexistingWaitcnt()
1227 AMDGPU::encodeWaitcnt(IV, Wait)); in applyPreexistingWaitcnt()
1230 ScoreBrackets.applyWaitcnt(LOAD_CNT, Wait.LoadCnt); in applyPreexistingWaitcnt()
1231 ScoreBrackets.applyWaitcnt(EXP_CNT, Wait.ExpCnt); in applyPreexistingWaitcnt()
1232 ScoreBrackets.applyWaitcnt(DS_CNT, Wait.DsCnt); in applyPreexistingWaitcnt()
1233 Wait.LoadCnt = ~0u; in applyPreexistingWaitcnt()
1234 Wait.ExpCnt = ~0u; in applyPreexistingWaitcnt()
1235 Wait.DsCnt = ~0u; in applyPreexistingWaitcnt()
1248 AMDGPU::OpName::simm16, Wait.StoreCnt); in applyPreexistingWaitcnt()
1251 ScoreBrackets.applyWaitcnt(STORE_CNT, Wait.StoreCnt); in applyPreexistingWaitcnt()
1252 Wait.StoreCnt = ~0u; in applyPreexistingWaitcnt()
1270 AMDGPU::Waitcnt Wait) { in createNewWaitcnt() argument
1279 if (Wait.hasWaitExceptStoreCnt()) { in createNewWaitcnt()
1280 unsigned Enc = AMDGPU::encodeWaitcnt(IV, Wait); in createNewWaitcnt()
1290 if (Wait.hasWaitStoreCnt()) { in createNewWaitcnt()
1296 .addImm(Wait.StoreCnt); in createNewWaitcnt()
1313 AMDGPU::Waitcnt &Wait, MachineBasicBlock::instr_iterator It) const { in applyPreexistingWaitcnt() argument
1341 Wait = Wait.combined(OldWait); in applyPreexistingWaitcnt()
1349 Wait = Wait.combined(OldWait); in applyPreexistingWaitcnt()
1358 addWait(Wait, CT.value(), OldCnt); in applyPreexistingWaitcnt()
1379 if (Wait.LoadCnt != ~0u && Wait.DsCnt != ~0u) { in applyPreexistingWaitcnt()
1380 unsigned NewEnc = AMDGPU::encodeLoadcntDscnt(IV, Wait); in applyPreexistingWaitcnt()
1384 ScoreBrackets.applyWaitcnt(LOAD_CNT, Wait.LoadCnt); in applyPreexistingWaitcnt()
1385 ScoreBrackets.applyWaitcnt(DS_CNT, Wait.DsCnt); in applyPreexistingWaitcnt()
1386 Wait.LoadCnt = ~0u; in applyPreexistingWaitcnt()
1387 Wait.DsCnt = ~0u; in applyPreexistingWaitcnt()
1404 if (Wait.StoreCnt != ~0u && Wait.DsCnt != ~0u) { in applyPreexistingWaitcnt()
1405 unsigned NewEnc = AMDGPU::encodeStorecntDscnt(IV, Wait); in applyPreexistingWaitcnt()
1409 ScoreBrackets.applyWaitcnt(STORE_CNT, Wait.StoreCnt); in applyPreexistingWaitcnt()
1410 ScoreBrackets.applyWaitcnt(DS_CNT, Wait.DsCnt); in applyPreexistingWaitcnt()
1411 Wait.StoreCnt = ~0u; in applyPreexistingWaitcnt()
1412 Wait.DsCnt = ~0u; in applyPreexistingWaitcnt()
1433 if (Wait.DsCnt != ~0u) { in applyPreexistingWaitcnt()
1442 if (Wait.LoadCnt != ~0u) { in applyPreexistingWaitcnt()
1445 } else if (Wait.StoreCnt != ~0u) { in applyPreexistingWaitcnt()
1464 unsigned NewCnt = getWait(Wait, CT); in applyPreexistingWaitcnt()
1471 setNoWait(Wait, CT); in applyPreexistingWaitcnt()
1492 AMDGPU::Waitcnt Wait) { in createNewWaitcnt() argument
1500 if (Wait.DsCnt != ~0u) { in createNewWaitcnt()
1503 if (Wait.LoadCnt != ~0u) { in createNewWaitcnt()
1504 unsigned Enc = AMDGPU::encodeLoadcntDscnt(IV, Wait); in createNewWaitcnt()
1509 Wait.LoadCnt = ~0u; in createNewWaitcnt()
1510 Wait.DsCnt = ~0u; in createNewWaitcnt()
1511 } else if (Wait.StoreCnt != ~0u) { in createNewWaitcnt()
1512 unsigned Enc = AMDGPU::encodeStorecntDscnt(IV, Wait); in createNewWaitcnt()
1518 Wait.StoreCnt = ~0u; in createNewWaitcnt()
1519 Wait.DsCnt = ~0u; in createNewWaitcnt()
1535 unsigned Count = getWait(Wait, CT); in createNewWaitcnt()
1595 AMDGPU::Waitcnt Wait; in generateWaitcntInstBefore() local
1606 Wait.LoadCnt = 0; in generateWaitcntInstBefore()
1616 Wait = Wait.combined( in generateWaitcntInstBefore()
1638 Wait.LoadCnt = 0; in generateWaitcntInstBefore()
1706 Wait.ExpCnt = 0; in generateWaitcntInstBefore()
1714 Wait = AMDGPU::Waitcnt(); in generateWaitcntInstBefore()
1725 ScoreBrackets.determineWait(SmemAccessCounter, RegNo, Wait); in generateWaitcntInstBefore()
1735 ScoreBrackets.determineWait(SmemAccessCounter, RegNo, Wait); in generateWaitcntInstBefore()
1756 addWait(Wait, SmemAccessCounter, 0); in generateWaitcntInstBefore()
1781 ScoreBrackets.determineWait(LOAD_CNT, RegNo + I + 1, Wait); in generateWaitcntInstBefore()
1786 ScoreBrackets.determineWait(LOAD_CNT, RegNo, Wait); in generateWaitcntInstBefore()
1788 ScoreBrackets.determineWait(EXP_CNT, RegNo, Wait); in generateWaitcntInstBefore()
1814 ScoreBrackets.determineWait(LOAD_CNT, RegNo, Wait); in generateWaitcntInstBefore()
1815 ScoreBrackets.determineWait(SAMPLE_CNT, RegNo, Wait); in generateWaitcntInstBefore()
1816 ScoreBrackets.determineWait(BVH_CNT, RegNo, Wait); in generateWaitcntInstBefore()
1820 ScoreBrackets.determineWait(EXP_CNT, RegNo, Wait); in generateWaitcntInstBefore()
1822 ScoreBrackets.determineWait(DS_CNT, RegNo, Wait); in generateWaitcntInstBefore()
1824 ScoreBrackets.determineWait(SmemAccessCounter, RegNo, Wait); in generateWaitcntInstBefore()
1837 Wait = Wait.combined( in generateWaitcntInstBefore()
1846 Wait.DsCnt = 0; in generateWaitcntInstBefore()
1851 ScoreBrackets.simplifyWaitcnt(Wait); in generateWaitcntInstBefore()
1854 Wait = AMDGPU::Waitcnt::allZeroExceptVsCnt(ST->hasExtendedWaitCounts()); in generateWaitcntInstBefore()
1857 Wait.LoadCnt = 0; in generateWaitcntInstBefore()
1859 Wait.ExpCnt = 0; in generateWaitcntInstBefore()
1861 Wait.DsCnt = 0; in generateWaitcntInstBefore()
1863 Wait.SampleCnt = 0; in generateWaitcntInstBefore()
1865 Wait.BvhCnt = 0; in generateWaitcntInstBefore()
1867 Wait.KmCnt = 0; in generateWaitcntInstBefore()
1871 Wait.LoadCnt = 0; in generateWaitcntInstBefore()
1873 Wait.SampleCnt = 0; in generateWaitcntInstBefore()
1875 Wait.BvhCnt = 0; in generateWaitcntInstBefore()
1878 return generateWaitcnt(Wait, MI.getIterator(), *MI.getParent(), ScoreBrackets, in generateWaitcntInstBefore()
1887 AMDGPU::Waitcnt Wait; in generateWaitcntBlockEnd() local
1897 Wait.LoadCnt = 0; in generateWaitcntBlockEnd()
1899 Wait.SampleCnt = 0; in generateWaitcntBlockEnd()
1901 Wait.BvhCnt = 0; in generateWaitcntBlockEnd()
1903 return generateWaitcnt(Wait, Block.instr_end(), Block, ScoreBrackets, in generateWaitcntBlockEnd()
1907 bool SIInsertWaitcnts::generateWaitcnt(AMDGPU::Waitcnt Wait, in generateWaitcnt() argument
1918 WCG->applyPreexistingWaitcnt(ScoreBrackets, *OldWaitcntInstr, Wait, It); in generateWaitcnt()
1922 ScoreBrackets.applyWaitcnt(Wait); in generateWaitcnt()
1925 if (Wait.ExpCnt != ~0u && It != Block.instr_end() && in generateWaitcnt()
1929 if (Wait.ExpCnt < WaitExp->getImm()) { in generateWaitcnt()
1930 WaitExp->setImm(Wait.ExpCnt); in generateWaitcnt()
1933 Wait.ExpCnt = ~0u; in generateWaitcnt()
1939 if (WCG->createNewWaitcnt(Block, It, Wait)) in generateWaitcnt()