Lines Matching refs:AMDGPU
159 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Inst.getOpcode()); in getVmemType()
160 return AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode)->Sampler in getVmemType()
165 void addWait(AMDGPU::Waitcnt &Wait, InstCounterType T, unsigned Count) { in addWait()
249 void simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const;
252 AMDGPU::Waitcnt &Wait) const;
253 void applyWaitcnt(const AMDGPU::Waitcnt &Wait);
361 AMDGPU::IsaVersion IV;
448 AMDGPU::Waitcnt &Wait, const MachineInstr *MI);
468 unsigned Reg = TRI->getEncodingValue(AMDGPU::getMCReg(Op.getReg(), *ST)); in getRegInterval()
525 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::addr); in updateByEvent()
533 if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), in updateByEvent()
534 AMDGPU::OpName::data0) != -1) { in updateByEvent()
537 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data0), in updateByEvent()
540 if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), in updateByEvent()
541 AMDGPU::OpName::data1) != -1) { in updateByEvent()
543 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), in updateByEvent()
544 AMDGPU::OpName::data1), in updateByEvent()
548 Inst.getOpcode() != AMDGPU::DS_GWS_INIT && in updateByEvent()
549 Inst.getOpcode() != AMDGPU::DS_GWS_SEMA_V && in updateByEvent()
550 Inst.getOpcode() != AMDGPU::DS_GWS_SEMA_BR && in updateByEvent()
551 Inst.getOpcode() != AMDGPU::DS_GWS_SEMA_P && in updateByEvent()
552 Inst.getOpcode() != AMDGPU::DS_GWS_BARRIER && in updateByEvent()
553 Inst.getOpcode() != AMDGPU::DS_APPEND && in updateByEvent()
554 Inst.getOpcode() != AMDGPU::DS_CONSUME && in updateByEvent()
555 Inst.getOpcode() != AMDGPU::DS_ORDERED_COUNT) { in updateByEvent()
568 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data), in updateByEvent()
573 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data), in updateByEvent()
582 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data), in updateByEvent()
595 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data), in updateByEvent()
609 TRI->getEncodingValue(AMDGPU::getMCReg(DefMO.getReg(), *ST)), in updateByEvent()
623 } else if (Inst.getOpcode() == AMDGPU::BUFFER_STORE_DWORD || in updateByEvent()
624 Inst.getOpcode() == AMDGPU::BUFFER_STORE_DWORDX2 || in updateByEvent()
625 Inst.getOpcode() == AMDGPU::BUFFER_STORE_DWORDX4) { in updateByEvent()
626 MachineOperand *MO = TII->getNamedOperand(Inst, AMDGPU::OpName::data); in updateByEvent()
715 void WaitcntBrackets::simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const { in simplifyWaitcnt()
735 AMDGPU::Waitcnt &Wait) const { in determineWait()
762 void WaitcntBrackets::applyWaitcnt(const AMDGPU::Waitcnt &Wait) { in applyWaitcnt()
812 AMDGPU::Waitcnt &Wait, in applyPreexistingWaitcnt()
822 if (II->getOpcode() == AMDGPU::S_WAITCNT) { in applyPreexistingWaitcnt()
828 AMDGPU::Waitcnt OldWait = AMDGPU::decodeWaitcnt(IV, IEnc); in applyPreexistingWaitcnt()
841 assert(II->getOpcode() == AMDGPU::S_WAITCNT_VSCNT); in applyPreexistingWaitcnt()
842 assert(II->getOperand(0).getReg() == AMDGPU::SGPR_NULL); in applyPreexistingWaitcnt()
845 TII->getNamedOperand(*II, AMDGPU::OpName::simm16)->getImm(); in applyPreexistingWaitcnt()
861 unsigned NewEnc = AMDGPU::encodeWaitcnt(IV, Wait); in applyPreexistingWaitcnt()
885 TII->getNamedOperand(*WaitcntVsCntInstr, AMDGPU::OpName::simm16) in applyPreexistingWaitcnt()
888 TII->getNamedOperand(*WaitcntVsCntInstr, AMDGPU::OpName::simm16) in applyPreexistingWaitcnt()
909 return (Opc == AMDGPU::S_CBRANCH_VCCNZ || Opc == AMDGPU::S_CBRANCH_VCCZ) && in readsVCCZ()
946 AMDGPU::Waitcnt Wait; in generateWaitcntInstBefore()
953 if (MI.getOpcode() == AMDGPU::BUFFER_WBINVL1 || in generateWaitcntInstBefore()
954 MI.getOpcode() == AMDGPU::BUFFER_WBINVL1_SC || in generateWaitcntInstBefore()
955 MI.getOpcode() == AMDGPU::BUFFER_WBINVL1_VOL || in generateWaitcntInstBefore()
956 MI.getOpcode() == AMDGPU::BUFFER_GL0_INV || in generateWaitcntInstBefore()
957 MI.getOpcode() == AMDGPU::BUFFER_GL1_INV) { in generateWaitcntInstBefore()
964 if (MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG || in generateWaitcntInstBefore()
965 MI.getOpcode() == AMDGPU::S_SETPC_B64_return || in generateWaitcntInstBefore()
967 Wait = Wait.combined(AMDGPU::Waitcnt::allZero(ST->hasVscnt())); in generateWaitcntInstBefore()
970 else if ((MI.getOpcode() == AMDGPU::S_SENDMSG || in generateWaitcntInstBefore()
971 MI.getOpcode() == AMDGPU::S_SENDMSGHALT) && in generateWaitcntInstBefore()
972 ((MI.getOperand(0).getImm() & AMDGPU::SendMsg::ID_MASK_) == in generateWaitcntInstBefore()
973 AMDGPU::SendMsg::ID_GS_DONE)) { in generateWaitcntInstBefore()
1035 if (MI.modifiesRegister(AMDGPU::EXEC, TRI)) { in generateWaitcntInstBefore()
1050 Wait = AMDGPU::Waitcnt(); in generateWaitcntInstBefore()
1053 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0); in generateWaitcntInstBefore()
1065 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::dst); in generateWaitcntInstBefore()
1148 if (MI.getOpcode() == AMDGPU::S_BARRIER && in generateWaitcntInstBefore()
1150 Wait = Wait.combined(AMDGPU::Waitcnt::allZero(ST->hasVscnt())); in generateWaitcntInstBefore()
1168 Wait = AMDGPU::Waitcnt::allZero(ST->hasVscnt()); in generateWaitcntInstBefore()
1192 unsigned Enc = AMDGPU::encodeWaitcnt(IV, Wait); in generateWaitcntInstBefore()
1194 MI.getDebugLoc(), TII->get(AMDGPU::S_WAITCNT)) in generateWaitcntInstBefore()
1209 TII->get(AMDGPU::S_WAITCNT_VSCNT)) in generateWaitcntInstBefore()
1210 .addReg(AMDGPU::SGPR_NULL, RegState::Undef) in generateWaitcntInstBefore()
1288 TII->hasModifiersSet(Inst, AMDGPU::OpName::gds)) { in updateEventWaitcntAfter()
1323 !llvm::AMDGPU::getMUBUFIsBufferInv(Inst.getOpcode())) { in updateEventWaitcntAfter()
1342 ScoreBrackets->applyWaitcnt(AMDGPU::Waitcnt::allZero(ST->hasVscnt())); in updateEventWaitcntAfter()
1345 ScoreBrackets->applyWaitcnt(AMDGPU::Waitcnt()); in updateEventWaitcntAfter()
1348 unsigned Imm = TII->getNamedOperand(Inst, AMDGPU::OpName::tgt)->getImm(); in updateEventWaitcntAfter()
1349 if (Imm >= AMDGPU::Exp::ET_PARAM0 && Imm <= AMDGPU::Exp::ET_PARAM31) in updateEventWaitcntAfter()
1351 else if (Imm >= AMDGPU::Exp::ET_POS0 && Imm <= AMDGPU::Exp::ET_POS_LAST) in updateEventWaitcntAfter()
1357 case AMDGPU::S_SENDMSG: in updateEventWaitcntAfter()
1358 case AMDGPU::S_SENDMSGHALT: in updateEventWaitcntAfter()
1361 case AMDGPU::S_MEMTIME: in updateEventWaitcntAfter()
1362 case AMDGPU::S_MEMREALTIME: in updateEventWaitcntAfter()
1476 if (Inst.getOpcode() == AMDGPU::S_WAITCNT || in insertWaitcntInBlock()
1477 (Inst.getOpcode() == AMDGPU::S_WAITCNT_VSCNT && in insertWaitcntInBlock()
1479 Inst.getOperand(0).getReg() == AMDGPU::SGPR_NULL)) { in insertWaitcntInBlock()
1495 if (Inst.definesRegister(AMDGPU::VCC_LO) || in insertWaitcntInBlock()
1496 Inst.definesRegister(AMDGPU::VCC_HI)) { in insertWaitcntInBlock()
1500 } else if (Inst.definesRegister(AMDGPU::VCC)) { in insertWaitcntInBlock()
1562 TII->get(ST->isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64), in insertWaitcntInBlock()
1580 IV = AMDGPU::getIsaVersion(ST->getCPU()); in runOnMachineFunction()
1588 HardwareLimits.VmcntMax = AMDGPU::getVmcntBitMask(IV); in runOnMachineFunction()
1589 HardwareLimits.ExpcntMax = AMDGPU::getExpcntBitMask(IV); in runOnMachineFunction()
1590 HardwareLimits.LgkmcntMax = AMDGPU::getLgkmcntBitMask(IV); in runOnMachineFunction()
1598 RegisterEncoding.VGPR0 = TRI->getEncodingValue(AMDGPU::VGPR0); in runOnMachineFunction()
1600 RegisterEncoding.SGPR0 = TRI->getEncodingValue(AMDGPU::SGPR0); in runOnMachineFunction()
1619 BuildMI(EntryBB, I, DebugLoc(), TII->get(AMDGPU::S_WAITCNT)).addImm(0); in runOnMachineFunction()
1621 BuildMI(EntryBB, I, DebugLoc(), TII->get(AMDGPU::S_WAITCNT_VSCNT)) in runOnMachineFunction()
1622 .addReg(AMDGPU::SGPR_NULL, RegState::Undef) in runOnMachineFunction()
1698 if (I->getOpcode() == AMDGPU::S_ENDPGM || in runOnMachineFunction()
1699 I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) in runOnMachineFunction()
1717 if (I->getOpcode() == AMDGPU::S_DCACHE_WB) in runOnMachineFunction()
1723 if ((I->getOpcode() == AMDGPU::S_ENDPGM || in runOnMachineFunction()
1724 I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) && in runOnMachineFunction()
1727 BuildMI(*MBB, I, I->getDebugLoc(), TII->get(AMDGPU::S_DCACHE_WB)); in runOnMachineFunction()