Lines Matching refs:ScoreBrackets
389 WaitcntBrackets &ScoreBrackets);
440 WaitcntBrackets &ScoreBrackets,
444 WaitcntBrackets &ScoreBrackets,
448 MachineBasicBlock &Block, WaitcntBrackets &ScoreBrackets,
451 WaitcntBrackets *ScoreBrackets);
453 WaitcntBrackets &ScoreBrackets);
454 bool applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
835 WaitcntBrackets &ScoreBrackets, MachineInstr &OldWaitcntInstr, in applyPreexistingWaitcnt() argument
891 ScoreBrackets.applyWaitcnt(Wait); in applyPreexistingWaitcnt()
921 ScoreBrackets.applyWaitcnt(Wait); in applyPreexistingWaitcnt()
974 WaitcntBrackets &ScoreBrackets, in generateWaitcntInstBefore() argument
1028 EmitWaitcnt |= ScoreBrackets->updateByWait(LGKM_CNT, in generateWaitcntInstBefore()
1029 ScoreBrackets->getScoreUB(LGKM_CNT)); in generateWaitcntInstBefore()
1032 EmitWaitcnt |= ScoreBrackets->updateByWait(VM_CNT, in generateWaitcntInstBefore()
1033 ScoreBrackets->getScoreUB(VM_CNT)); in generateWaitcntInstBefore()
1040 EmitWaitcnt |= ScoreBrackets->updateByWait(EXP_CNT, in generateWaitcntInstBefore()
1041 ScoreBrackets->getScoreUB(EXP_CNT)); in generateWaitcntInstBefore()
1042 EmitWaitcnt |= ScoreBrackets->updateByWait(LGKM_CNT, in generateWaitcntInstBefore()
1043 ScoreBrackets->getScoreUB(LGKM_CNT)); in generateWaitcntInstBefore()
1052 EmitWaitcnt |= ScoreBrackets->updateByWait(EXP_CNT, in generateWaitcntInstBefore()
1053 ScoreBrackets->getScoreUB(EXP_CNT)); in generateWaitcntInstBefore()
1054 EmitWaitcnt |= ScoreBrackets->updateByWait(VM_CNT, in generateWaitcntInstBefore()
1055 ScoreBrackets->getScoreUB(VM_CNT)); in generateWaitcntInstBefore()
1075 if (ScoreBrackets.hasPendingEvent(EXP_GPR_LOCK) || in generateWaitcntInstBefore()
1076 ScoreBrackets.hasPendingEvent(EXP_PARAM_ACCESS) || in generateWaitcntInstBefore()
1077 ScoreBrackets.hasPendingEvent(EXP_POS_ACCESS) || in generateWaitcntInstBefore()
1078 ScoreBrackets.hasPendingEvent(GDS_GPR_LOCK)) { in generateWaitcntInstBefore()
1094 ScoreBrackets.getRegInterval(&MI, TII, MRI, TRI, CallAddrOpIdx); in generateWaitcntInstBefore()
1098 ScoreBrackets.determineWait( in generateWaitcntInstBefore()
1099 LGKM_CNT, ScoreBrackets.getRegScore(RegNo, LGKM_CNT), Wait); in generateWaitcntInstBefore()
1105 ScoreBrackets.getRegInterval(&MI, TII, MRI, TRI, RtnAddrOpIdx); in generateWaitcntInstBefore()
1109 ScoreBrackets.determineWait( in generateWaitcntInstBefore()
1110 LGKM_CNT, ScoreBrackets.getRegScore(RegNo, LGKM_CNT), Wait); in generateWaitcntInstBefore()
1142 ScoreBrackets.determineWait( in generateWaitcntInstBefore()
1143 VM_CNT, ScoreBrackets.getRegScore(RegNo, VM_CNT), Wait); in generateWaitcntInstBefore()
1145 ScoreBrackets.determineWait( in generateWaitcntInstBefore()
1146 EXP_CNT, ScoreBrackets.getRegScore(RegNo, EXP_CNT), Wait); in generateWaitcntInstBefore()
1156 ScoreBrackets.getRegInterval(&MI, TII, MRI, TRI, I); in generateWaitcntInstBefore()
1166 ScoreBrackets.hasOtherPendingVmemTypes(RegNo, in generateWaitcntInstBefore()
1168 ScoreBrackets.determineWait( in generateWaitcntInstBefore()
1169 VM_CNT, ScoreBrackets.getRegScore(RegNo, VM_CNT), Wait); in generateWaitcntInstBefore()
1170 ScoreBrackets.clearVgprVmemTypes(RegNo); in generateWaitcntInstBefore()
1172 if (Op.isDef() || ScoreBrackets.hasPendingEvent(EXP_LDS_ACCESS)) { in generateWaitcntInstBefore()
1173 ScoreBrackets.determineWait( in generateWaitcntInstBefore()
1174 EXP_CNT, ScoreBrackets.getRegScore(RegNo, EXP_CNT), Wait); in generateWaitcntInstBefore()
1177 ScoreBrackets.determineWait( in generateWaitcntInstBefore()
1178 LGKM_CNT, ScoreBrackets.getRegScore(RegNo, LGKM_CNT), Wait); in generateWaitcntInstBefore()
1197 if (ScoreBrackets.getScoreLB(LGKM_CNT) < in generateWaitcntInstBefore()
1198 ScoreBrackets.getScoreUB(LGKM_CNT) && in generateWaitcntInstBefore()
1199 ScoreBrackets.hasPendingEvent(SMEM_ACCESS)) { in generateWaitcntInstBefore()
1205 ScoreBrackets.simplifyWaitcnt(Wait); in generateWaitcntInstBefore()
1220 unsigned UB = ScoreBrackets.getScoreUB(VM_CNT); in generateWaitcntInstBefore()
1221 unsigned LB = ScoreBrackets.getScoreLB(VM_CNT); in generateWaitcntInstBefore()
1226 return generateWaitcnt(Wait, MI.getIterator(), *MI.getParent(), ScoreBrackets, in generateWaitcntInstBefore()
1233 WaitcntBrackets &ScoreBrackets, in generateWaitcntBlockEnd() argument
1237 unsigned UB = ScoreBrackets.getScoreUB(VM_CNT); in generateWaitcntBlockEnd()
1238 unsigned LB = ScoreBrackets.getScoreLB(VM_CNT); in generateWaitcntBlockEnd()
1244 return generateWaitcnt(Wait, Block.instr_end(), Block, ScoreBrackets, in generateWaitcntBlockEnd()
1251 WaitcntBrackets &ScoreBrackets, in generateWaitcnt() argument
1260 applyPreexistingWaitcnt(ScoreBrackets, *OldWaitcntInstr, Wait, It); in generateWaitcnt()
1262 ScoreBrackets.applyWaitcnt(Wait); in generateWaitcnt()
1367 WaitcntBrackets *ScoreBrackets) { in updateEventWaitcntAfter() argument
1375 ScoreBrackets->updateByEvent(TII, TRI, MRI, GDS_ACCESS, Inst); in updateEventWaitcntAfter()
1376 ScoreBrackets->updateByEvent(TII, TRI, MRI, GDS_GPR_LOCK, Inst); in updateEventWaitcntAfter()
1378 ScoreBrackets->updateByEvent(TII, TRI, MRI, LDS_ACCESS, Inst); in updateEventWaitcntAfter()
1388 ScoreBrackets->updateByEvent(TII, TRI, MRI, VMEM_ACCESS, Inst); in updateEventWaitcntAfter()
1390 ScoreBrackets->updateByEvent(TII, TRI, MRI, VMEM_READ_ACCESS, Inst); in updateEventWaitcntAfter()
1392 ScoreBrackets->updateByEvent(TII, TRI, MRI, VMEM_WRITE_ACCESS, Inst); in updateEventWaitcntAfter()
1397 ScoreBrackets->updateByEvent(TII, TRI, MRI, LDS_ACCESS, Inst); in updateEventWaitcntAfter()
1407 ScoreBrackets->setPendingFlat(); in updateEventWaitcntAfter()
1411 ScoreBrackets->updateByEvent(TII, TRI, MRI, VMEM_ACCESS, Inst); in updateEventWaitcntAfter()
1415 ScoreBrackets->updateByEvent(TII, TRI, MRI, VMEM_READ_ACCESS, Inst); in updateEventWaitcntAfter()
1417 ScoreBrackets->updateByEvent(TII, TRI, MRI, VMEM_WRITE_ACCESS, Inst); in updateEventWaitcntAfter()
1421 ScoreBrackets->updateByEvent(TII, TRI, MRI, VMW_GPR_LOCK, Inst); in updateEventWaitcntAfter()
1424 ScoreBrackets->updateByEvent(TII, TRI, MRI, SMEM_ACCESS, Inst); in updateEventWaitcntAfter()
1428 ScoreBrackets->applyWaitcnt(AMDGPU::Waitcnt::allZero(ST->hasVscnt())); in updateEventWaitcntAfter()
1431 ScoreBrackets->applyWaitcnt(AMDGPU::Waitcnt()); in updateEventWaitcntAfter()
1434 ScoreBrackets->updateByEvent(TII, TRI, MRI, EXP_LDS_ACCESS, Inst); in updateEventWaitcntAfter()
1437 ScoreBrackets->applyWaitcnt(EXP_CNT, Imm); in updateEventWaitcntAfter()
1441 ScoreBrackets->updateByEvent(TII, TRI, MRI, EXP_PARAM_ACCESS, Inst); in updateEventWaitcntAfter()
1443 ScoreBrackets->updateByEvent(TII, TRI, MRI, EXP_POS_ACCESS, Inst); in updateEventWaitcntAfter()
1445 ScoreBrackets->updateByEvent(TII, TRI, MRI, EXP_GPR_LOCK, Inst); in updateEventWaitcntAfter()
1452 ScoreBrackets->updateByEvent(TII, TRI, MRI, SQ_MESSAGE, Inst); in updateEventWaitcntAfter()
1456 ScoreBrackets->updateByEvent(TII, TRI, MRI, SMEM_ACCESS, Inst); in updateEventWaitcntAfter()
1536 WaitcntBrackets &ScoreBrackets) { in insertWaitcntInBlock() argument
1541 ScoreBrackets.dump(); in insertWaitcntInBlock()
1579 isPreheaderToFlush(Block, ScoreBrackets); in insertWaitcntInBlock()
1582 Modified |= generateWaitcntInstBefore(Inst, ScoreBrackets, OldWaitcntInstr, in insertWaitcntInBlock()
1605 ScoreBrackets.getScoreLB(LGKM_CNT) < in insertWaitcntInBlock()
1606 ScoreBrackets.getScoreUB(LGKM_CNT) && in insertWaitcntInBlock()
1607 ScoreBrackets.hasPendingEvent(SMEM_ACCESS)) { in insertWaitcntInBlock()
1633 updateEventWaitcntAfter(Inst, &ScoreBrackets); in insertWaitcntInBlock()
1641 ScoreBrackets->setScoreLB(VM_CNT, in insertWaitcntInBlock()
1642 ScoreBrackets->getScoreUB(VM_CNT)); in insertWaitcntInBlock()
1648 ScoreBrackets.dump(); in insertWaitcntInBlock()
1669 isPreheaderToFlush(Block, ScoreBrackets)) in insertWaitcntInBlock()
1670 Modified |= generateWaitcntBlockEnd(Block, ScoreBrackets, OldWaitcntInstr); in insertWaitcntInBlock()
1678 WaitcntBrackets &ScoreBrackets) { in isPreheaderToFlush() argument
1695 if (Loop->getLoopPreheader() == &MBB && shouldFlushVmCnt(Loop, ScoreBrackets)) in isPreheaderToFlush()