Lines Matching refs:AMDGPU
164 void addWait(AMDGPU::Waitcnt &Wait, InstCounterType T, unsigned Count) { in addWait()
262 bool simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const;
265 AMDGPU::Waitcnt &Wait) const;
266 void applyWaitcnt(const AMDGPU::Waitcnt &Wait);
362 AMDGPU::IsaVersion IV;
531 if (Inst.getOpcode() != AMDGPU::DS_APPEND && in updateByEvent()
532 Inst.getOpcode() != AMDGPU::DS_CONSUME) { in updateByEvent()
535 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::addr), in updateByEvent()
539 if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), in updateByEvent()
540 AMDGPU::OpName::data0) != -1) { in updateByEvent()
543 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data0), in updateByEvent()
546 if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), in updateByEvent()
547 AMDGPU::OpName::data1) != -1) { in updateByEvent()
549 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), in updateByEvent()
550 AMDGPU::OpName::data1), in updateByEvent()
553 } else if (AMDGPU::getAtomicNoRetOp(Inst.getOpcode()) != -1 && in updateByEvent()
554 Inst.getOpcode() != AMDGPU::DS_GWS_INIT && in updateByEvent()
555 Inst.getOpcode() != AMDGPU::DS_GWS_SEMA_V && in updateByEvent()
556 Inst.getOpcode() != AMDGPU::DS_GWS_SEMA_BR && in updateByEvent()
557 Inst.getOpcode() != AMDGPU::DS_GWS_SEMA_P && in updateByEvent()
558 Inst.getOpcode() != AMDGPU::DS_GWS_BARRIER && in updateByEvent()
559 Inst.getOpcode() != AMDGPU::DS_APPEND && in updateByEvent()
560 Inst.getOpcode() != AMDGPU::DS_CONSUME && in updateByEvent()
561 Inst.getOpcode() != AMDGPU::DS_ORDERED_COUNT) { in updateByEvent()
573 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data), in updateByEvent()
575 } else if (AMDGPU::getAtomicNoRetOp(Inst.getOpcode()) != -1) { in updateByEvent()
578 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data), in updateByEvent()
584 } else if (AMDGPU::getAtomicNoRetOp(Inst.getOpcode()) != -1) { in updateByEvent()
587 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data), in updateByEvent()
597 } else if (AMDGPU::getAtomicNoRetOp(Inst.getOpcode()) != -1) { in updateByEvent()
600 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data), in updateByEvent()
626 } else if (Inst.getOpcode() == AMDGPU::BUFFER_STORE_DWORD || in updateByEvent()
627 Inst.getOpcode() == AMDGPU::BUFFER_STORE_DWORDX2 || in updateByEvent()
628 Inst.getOpcode() == AMDGPU::BUFFER_STORE_DWORDX4) { in updateByEvent()
629 MachineOperand *MO = TII->getNamedOperand(Inst, AMDGPU::OpName::data); in updateByEvent()
705 bool WaitcntBrackets::simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const { in simplifyWaitcnt()
723 AMDGPU::Waitcnt &Wait) const { in determineWait()
747 void WaitcntBrackets::applyWaitcnt(const AMDGPU::Waitcnt &Wait) { in applyWaitcnt()
792 return (Opc == AMDGPU::S_CBRANCH_VCCNZ || Opc == AMDGPU::S_CBRANCH_VCCZ) && in readsVCCZ()
815 AMDGPU::Waitcnt Wait; in generateWaitcntInstBefore()
819 if (MI.getOpcode() == AMDGPU::BUFFER_WBINVL1 || in generateWaitcntInstBefore()
820 MI.getOpcode() == AMDGPU::BUFFER_WBINVL1_SC || in generateWaitcntInstBefore()
821 MI.getOpcode() == AMDGPU::BUFFER_WBINVL1_VOL) { in generateWaitcntInstBefore()
828 if (MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG || in generateWaitcntInstBefore()
829 MI.getOpcode() == AMDGPU::S_SETPC_B64_return) { in generateWaitcntInstBefore()
830 Wait = AMDGPU::Waitcnt::allZero(); in generateWaitcntInstBefore()
833 else if ((MI.getOpcode() == AMDGPU::S_SENDMSG || in generateWaitcntInstBefore()
834 MI.getOpcode() == AMDGPU::S_SENDMSGHALT) && in generateWaitcntInstBefore()
835 ((MI.getOperand(0).getImm() & AMDGPU::SendMsg::ID_MASK_) == in generateWaitcntInstBefore()
836 AMDGPU::SendMsg::ID_GS_DONE)) { in generateWaitcntInstBefore()
898 if (MI.modifiesRegister(AMDGPU::EXEC, TRI)) { in generateWaitcntInstBefore()
1000 if (MI.getOpcode() == AMDGPU::S_BARRIER && in generateWaitcntInstBefore()
1002 Wait = AMDGPU::Waitcnt::allZero(); in generateWaitcntInstBefore()
1026 ScoreBrackets.applyWaitcnt(AMDGPU::decodeWaitcnt(IV, Imm)); in generateWaitcntInstBefore()
1034 Wait = AMDGPU::Waitcnt::allZero(); in generateWaitcntInstBefore()
1045 AMDGPU::Waitcnt OldWait; in generateWaitcntInstBefore()
1048 AMDGPU::decodeWaitcnt(IV, OldWaitcntInstr->getOperand(0).getImm()); in generateWaitcntInstBefore()
1056 unsigned Enc = AMDGPU::encodeWaitcnt(IV, Wait); in generateWaitcntInstBefore()
1065 MI.getDebugLoc(), TII->get(AMDGPU::S_WAITCNT)) in generateWaitcntInstBefore()
1100 TII->hasModifiersSet(Inst, AMDGPU::OpName::gds)) { in updateEventWaitcntAfter()
1123 Inst.getOpcode() != AMDGPU::BUFFER_WBINVL1 && in updateEventWaitcntAfter()
1124 Inst.getOpcode() != AMDGPU::BUFFER_WBINVL1_SC && in updateEventWaitcntAfter()
1125 Inst.getOpcode() != AMDGPU::BUFFER_WBINVL1_VOL) { in updateEventWaitcntAfter()
1128 (Inst.mayStore() || AMDGPU::getAtomicNoRetOp(Inst.getOpcode()) != -1)) { in updateEventWaitcntAfter()
1135 case AMDGPU::S_SENDMSG: in updateEventWaitcntAfter()
1136 case AMDGPU::S_SENDMSGHALT: in updateEventWaitcntAfter()
1139 case AMDGPU::EXP: in updateEventWaitcntAfter()
1140 case AMDGPU::EXP_DONE: { in updateEventWaitcntAfter()
1141 int Imm = TII->getNamedOperand(Inst, AMDGPU::OpName::tgt)->getImm(); in updateEventWaitcntAfter()
1150 case AMDGPU::S_MEMTIME: in updateEventWaitcntAfter()
1151 case AMDGPU::S_MEMREALTIME: in updateEventWaitcntAfter()
1248 if (Inst.getOpcode() == AMDGPU::S_WAITCNT) { in insertWaitcntInBlock()
1258 ScoreBrackets.applyWaitcnt(AMDGPU::decodeWaitcnt(IV, Imm)); in insertWaitcntInBlock()
1309 if (Inst.getOpcode() == AMDGPU::DS_GWS_INIT || in insertWaitcntInBlock()
1310 Inst.getOpcode() == AMDGPU::DS_GWS_SEMA_V || in insertWaitcntInBlock()
1311 Inst.getOpcode() == AMDGPU::DS_GWS_SEMA_BR || in insertWaitcntInBlock()
1312 Inst.getOpcode() == AMDGPU::DS_GWS_SEMA_P || in insertWaitcntInBlock()
1313 Inst.getOpcode() == AMDGPU::DS_GWS_BARRIER) { in insertWaitcntInBlock()
1315 ScoreBrackets.applyWaitcnt(AMDGPU::Waitcnt::allZero()); in insertWaitcntInBlock()
1324 BuildMI(Block, Inst, Inst.getDebugLoc(), TII->get(AMDGPU::S_MOV_B64), in insertWaitcntInBlock()
1325 AMDGPU::VCC) in insertWaitcntInBlock()
1326 .addReg(AMDGPU::VCC); in insertWaitcntInBlock()
1342 IV = AMDGPU::getIsaVersion(ST->getCPU()); in runOnMachineFunction()
1349 HardwareLimits.VmcntMax = AMDGPU::getVmcntBitMask(IV); in runOnMachineFunction()
1350 HardwareLimits.ExpcntMax = AMDGPU::getExpcntBitMask(IV); in runOnMachineFunction()
1351 HardwareLimits.LgkmcntMax = AMDGPU::getLgkmcntBitMask(IV); in runOnMachineFunction()
1358 RegisterEncoding.VGPR0 = TRI->getEncodingValue(AMDGPU::VGPR0); in runOnMachineFunction()
1361 RegisterEncoding.SGPR0 = TRI->getEncodingValue(AMDGPU::SGPR0); in runOnMachineFunction()
1444 if (I->getOpcode() == AMDGPU::S_ENDPGM || in runOnMachineFunction()
1445 I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) in runOnMachineFunction()
1463 if (I->getOpcode() == AMDGPU::S_DCACHE_WB) in runOnMachineFunction()
1469 if ((I->getOpcode() == AMDGPU::S_ENDPGM || in runOnMachineFunction()
1470 I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) && in runOnMachineFunction()
1473 BuildMI(*MBB, I, I->getDebugLoc(), TII->get(AMDGPU::S_DCACHE_WB)); in runOnMachineFunction()
1487 BuildMI(EntryBB, EntryBB.getFirstNonPHI(), DebugLoc(), TII->get(AMDGPU::S_WAITCNT)) in runOnMachineFunction()