Lines Matching refs:AMDGPU
148 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Inst.getOpcode()); in getVmemType()
149 const AMDGPU::MIMGBaseOpcodeInfo *BaseInfo = in getVmemType()
150 AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode); in getVmemType()
155 void addWait(AMDGPU::Waitcnt &Wait, InstCounterType T, unsigned Count) { in addWait()
241 void simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const;
244 AMDGPU::Waitcnt &Wait) const;
245 void applyWaitcnt(const AMDGPU::Waitcnt &Wait);
355 AMDGPU::IsaVersion IV;
446 bool generateWaitcnt(AMDGPU::Waitcnt Wait,
456 AMDGPU::Waitcnt &Wait,
477 unsigned Reg = TRI->getEncodingValue(AMDGPU::getMCReg(Op.getReg(), *ST)); in getRegInterval()
520 MI.getOpcode() != AMDGPU::BUFFER_STORE_LDS_DWORD; in mayWriteLDSThroughDMA()
542 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::addr); in updateByEvent()
550 if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), in updateByEvent()
551 AMDGPU::OpName::data0) != -1) { in updateByEvent()
554 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data0), in updateByEvent()
557 if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), in updateByEvent()
558 AMDGPU::OpName::data1) != -1) { in updateByEvent()
560 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), in updateByEvent()
561 AMDGPU::OpName::data1), in updateByEvent()
565 Inst.getOpcode() != AMDGPU::DS_GWS_INIT && in updateByEvent()
566 Inst.getOpcode() != AMDGPU::DS_GWS_SEMA_V && in updateByEvent()
567 Inst.getOpcode() != AMDGPU::DS_GWS_SEMA_BR && in updateByEvent()
568 Inst.getOpcode() != AMDGPU::DS_GWS_SEMA_P && in updateByEvent()
569 Inst.getOpcode() != AMDGPU::DS_GWS_BARRIER && in updateByEvent()
570 Inst.getOpcode() != AMDGPU::DS_APPEND && in updateByEvent()
571 Inst.getOpcode() != AMDGPU::DS_CONSUME && in updateByEvent()
572 Inst.getOpcode() != AMDGPU::DS_ORDERED_COUNT) { in updateByEvent()
585 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data), in updateByEvent()
590 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data), in updateByEvent()
599 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data), in updateByEvent()
612 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data), in updateByEvent()
619 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::vdst), in updateByEvent()
632 TRI->getEncodingValue(AMDGPU::getMCReg(DefMO.getReg(), *ST)), in updateByEvent()
646 } else if (Inst.getOpcode() == AMDGPU::BUFFER_STORE_DWORD || in updateByEvent()
647 Inst.getOpcode() == AMDGPU::BUFFER_STORE_DWORDX2 || in updateByEvent()
648 Inst.getOpcode() == AMDGPU::BUFFER_STORE_DWORDX4) { in updateByEvent()
649 MachineOperand *MO = TII->getNamedOperand(Inst, AMDGPU::OpName::data); in updateByEvent()
738 void WaitcntBrackets::simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const { in simplifyWaitcnt()
758 AMDGPU::Waitcnt &Wait) const { in determineWait()
785 void WaitcntBrackets::applyWaitcnt(const AMDGPU::Waitcnt &Wait) { in applyWaitcnt()
836 AMDGPU::Waitcnt &Wait, MachineBasicBlock::instr_iterator It) { in applyPreexistingWaitcnt()
846 if (II.getOpcode() == AMDGPU::S_WAITCNT) { in applyPreexistingWaitcnt()
852 AMDGPU::Waitcnt OldWait = AMDGPU::decodeWaitcnt(IV, IEnc); in applyPreexistingWaitcnt()
865 assert(II.getOpcode() == AMDGPU::S_WAITCNT_VSCNT); in applyPreexistingWaitcnt()
866 assert(II.getOperand(0).getReg() == AMDGPU::SGPR_NULL); in applyPreexistingWaitcnt()
869 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm(); in applyPreexistingWaitcnt()
885 unsigned NewEnc = AMDGPU::encodeWaitcnt(IV, Wait); in applyPreexistingWaitcnt()
914 TII->getNamedOperand(*WaitcntVsCntInstr, AMDGPU::OpName::simm16) in applyPreexistingWaitcnt()
917 TII->getNamedOperand(*WaitcntVsCntInstr, AMDGPU::OpName::simm16) in applyPreexistingWaitcnt()
942 return (Opc == AMDGPU::S_CBRANCH_VCCNZ || Opc == AMDGPU::S_CBRANCH_VCCZ) && in readsVCCZ()
982 AMDGPU::Waitcnt Wait; in generateWaitcntInstBefore()
988 if (MI.getOpcode() == AMDGPU::BUFFER_WBINVL1 || in generateWaitcntInstBefore()
989 MI.getOpcode() == AMDGPU::BUFFER_WBINVL1_SC || in generateWaitcntInstBefore()
990 MI.getOpcode() == AMDGPU::BUFFER_WBINVL1_VOL || in generateWaitcntInstBefore()
991 MI.getOpcode() == AMDGPU::BUFFER_GL0_INV || in generateWaitcntInstBefore()
992 MI.getOpcode() == AMDGPU::BUFFER_GL1_INV) { in generateWaitcntInstBefore()
999 if (MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG || in generateWaitcntInstBefore()
1000 MI.getOpcode() == AMDGPU::SI_RETURN || in generateWaitcntInstBefore()
1001 MI.getOpcode() == AMDGPU::S_SETPC_B64_return || in generateWaitcntInstBefore()
1003 Wait = Wait.combined(AMDGPU::Waitcnt::allZero(ST->hasVscnt())); in generateWaitcntInstBefore()
1006 else if ((MI.getOpcode() == AMDGPU::S_SENDMSG || in generateWaitcntInstBefore()
1007 MI.getOpcode() == AMDGPU::S_SENDMSGHALT) && in generateWaitcntInstBefore()
1009 ((MI.getOperand(0).getImm() & AMDGPU::SendMsg::ID_MASK_PreGFX11_) == in generateWaitcntInstBefore()
1010 AMDGPU::SendMsg::ID_GS_DONE_PreGFX11)) { in generateWaitcntInstBefore()
1072 if (MI.modifiesRegister(AMDGPU::EXEC, TRI)) { in generateWaitcntInstBefore()
1087 Wait = AMDGPU::Waitcnt(); in generateWaitcntInstBefore()
1090 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0); in generateWaitcntInstBefore()
1102 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::dst); in generateWaitcntInstBefore()
1188 if (MI.getOpcode() == AMDGPU::S_BARRIER && in generateWaitcntInstBefore()
1190 Wait = Wait.combined(AMDGPU::Waitcnt::allZero(ST->hasVscnt())); in generateWaitcntInstBefore()
1208 Wait = AMDGPU::Waitcnt::allZero(ST->hasVscnt()); in generateWaitcntInstBefore()
1235 AMDGPU::Waitcnt Wait; in generateWaitcntBlockEnd()
1248 bool SIInsertWaitcnts::generateWaitcnt(AMDGPU::Waitcnt Wait, in generateWaitcnt()
1268 TII->getNamedOperand(*It, AMDGPU::OpName::waitexp); in generateWaitcnt()
1282 unsigned Enc = AMDGPU::encodeWaitcnt(IV, Wait); in generateWaitcnt()
1284 BuildMI(Block, It, DL, TII->get(AMDGPU::S_WAITCNT)).addImm(Enc); in generateWaitcnt()
1296 auto SWaitInst = BuildMI(Block, It, DL, TII->get(AMDGPU::S_WAITCNT_VSCNT)) in generateWaitcnt()
1297 .addReg(AMDGPU::SGPR_NULL, RegState::Undef) in generateWaitcnt()
1374 TII->hasModifiersSet(Inst, AMDGPU::OpName::gds)) { in updateEventWaitcntAfter()
1409 !llvm::AMDGPU::getMUBUFIsBufferInv(Inst.getOpcode())) { in updateEventWaitcntAfter()
1428 ScoreBrackets->applyWaitcnt(AMDGPU::Waitcnt::allZero(ST->hasVscnt())); in updateEventWaitcntAfter()
1431 ScoreBrackets->applyWaitcnt(AMDGPU::Waitcnt()); in updateEventWaitcntAfter()
1436 int64_t Imm = TII->getNamedOperand(Inst, AMDGPU::OpName::waitexp)->getImm(); in updateEventWaitcntAfter()
1439 unsigned Imm = TII->getNamedOperand(Inst, AMDGPU::OpName::tgt)->getImm(); in updateEventWaitcntAfter()
1440 if (Imm >= AMDGPU::Exp::ET_PARAM0 && Imm <= AMDGPU::Exp::ET_PARAM31) in updateEventWaitcntAfter()
1442 else if (Imm >= AMDGPU::Exp::ET_POS0 && Imm <= AMDGPU::Exp::ET_POS_LAST) in updateEventWaitcntAfter()
1448 case AMDGPU::S_SENDMSG: in updateEventWaitcntAfter()
1449 case AMDGPU::S_SENDMSG_RTN_B32: in updateEventWaitcntAfter()
1450 case AMDGPU::S_SENDMSG_RTN_B64: in updateEventWaitcntAfter()
1451 case AMDGPU::S_SENDMSGHALT: in updateEventWaitcntAfter()
1454 case AMDGPU::S_MEMTIME: in updateEventWaitcntAfter()
1455 case AMDGPU::S_MEMREALTIME: in updateEventWaitcntAfter()
1568 if (Inst.getOpcode() == AMDGPU::S_WAITCNT || in insertWaitcntInBlock()
1569 (Inst.getOpcode() == AMDGPU::S_WAITCNT_VSCNT && in insertWaitcntInBlock()
1571 Inst.getOperand(0).getReg() == AMDGPU::SGPR_NULL)) { in insertWaitcntInBlock()
1591 if (Inst.definesRegister(AMDGPU::VCC_LO) || in insertWaitcntInBlock()
1592 Inst.definesRegister(AMDGPU::VCC_HI)) { in insertWaitcntInBlock()
1596 } else if (Inst.definesRegister(AMDGPU::VCC)) { in insertWaitcntInBlock()
1658 TII->get(ST->isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64), in insertWaitcntInBlock()
1768 IV = AMDGPU::getIsaVersion(ST->getCPU()); in runOnMachineFunction()
1778 Limits.VmcntMax = AMDGPU::getVmcntBitMask(IV); in runOnMachineFunction()
1779 Limits.ExpcntMax = AMDGPU::getExpcntBitMask(IV); in runOnMachineFunction()
1780 Limits.LgkmcntMax = AMDGPU::getLgkmcntBitMask(IV); in runOnMachineFunction()
1789 Encoding.VGPR0 = TRI->getEncodingValue(AMDGPU::VGPR0); in runOnMachineFunction()
1791 Encoding.SGPR0 = TRI->getEncodingValue(AMDGPU::SGPR0); in runOnMachineFunction()
1810 BuildMI(EntryBB, I, DebugLoc(), TII->get(AMDGPU::S_WAITCNT)).addImm(0); in runOnMachineFunction()
1812 BuildMI(EntryBB, I, DebugLoc(), TII->get(AMDGPU::S_WAITCNT_VSCNT)) in runOnMachineFunction()
1813 .addReg(AMDGPU::SGPR_NULL, RegState::Undef) in runOnMachineFunction()
1885 if (MI.getOpcode() == AMDGPU::S_ENDPGM || in runOnMachineFunction()
1886 MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) in runOnMachineFunction()
1905 if (I->getOpcode() == AMDGPU::S_DCACHE_WB) in runOnMachineFunction()
1911 if ((I->getOpcode() == AMDGPU::S_ENDPGM || in runOnMachineFunction()
1912 I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) && in runOnMachineFunction()
1915 BuildMI(*MBB, I, I->getDebugLoc(), TII->get(AMDGPU::S_DCACHE_WB)); in runOnMachineFunction()