Lines Matching refs:MI
217 void reportUnsupported(const MachineBasicBlock::iterator &MI,
233 constructFromMIWithMMO(const MachineBasicBlock::iterator &MI) const;
242 getLoadInfo(const MachineBasicBlock::iterator &MI) const;
247 getStoreInfo(const MachineBasicBlock::iterator &MI) const;
252 getAtomicFenceInfo(const MachineBasicBlock::iterator &MI) const;
257 getAtomicCmpxchgOrRmwInfo(const MachineBasicBlock::iterator &MI) const;
278 bool enableNamedBit(const MachineBasicBlock::iterator MI,
289 virtual bool enableLoadCacheBypass(const MachineBasicBlock::iterator &MI,
296 virtual bool enableStoreCacheBypass(const MachineBasicBlock::iterator &MI,
303 virtual bool enableRMWCacheBypass(const MachineBasicBlock::iterator &MI,
310 virtual bool enableVolatileAndOrNonTemporal(MachineBasicBlock::iterator &MI,
322 virtual bool insertWait(MachineBasicBlock::iterator &MI,
334 virtual bool insertAcquire(MachineBasicBlock::iterator &MI,
345 virtual bool insertRelease(MachineBasicBlock::iterator &MI,
355 MachineBasicBlock::iterator &MI) const { in tryForceStoreSC0SC1()
365 bool enableGLCBit(const MachineBasicBlock::iterator &MI) const { in enableGLCBit()
366 return enableNamedBit(MI, AMDGPU::CPol::GLC); in enableGLCBit()
371 bool enableSLCBit(const MachineBasicBlock::iterator &MI) const { in enableSLCBit()
372 return enableNamedBit(MI, AMDGPU::CPol::SLC); in enableSLCBit()
379 bool enableLoadCacheBypass(const MachineBasicBlock::iterator &MI,
383 bool enableStoreCacheBypass(const MachineBasicBlock::iterator &MI,
387 bool enableRMWCacheBypass(const MachineBasicBlock::iterator &MI,
391 bool enableVolatileAndOrNonTemporal(MachineBasicBlock::iterator &MI,
396 bool insertWait(MachineBasicBlock::iterator &MI,
403 bool insertAcquire(MachineBasicBlock::iterator &MI,
408 bool insertRelease(MachineBasicBlock::iterator &MI,
420 bool insertAcquire(MachineBasicBlock::iterator &MI,
432 bool enableLoadCacheBypass(const MachineBasicBlock::iterator &MI,
436 bool enableStoreCacheBypass(const MachineBasicBlock::iterator &MI,
440 bool enableRMWCacheBypass(const MachineBasicBlock::iterator &MI,
444 bool enableVolatileAndOrNonTemporal(MachineBasicBlock::iterator &MI,
449 bool insertWait(MachineBasicBlock::iterator &MI,
456 bool insertAcquire(MachineBasicBlock::iterator &MI,
461 bool insertRelease(MachineBasicBlock::iterator &MI,
473 bool enableSC0Bit(const MachineBasicBlock::iterator &MI) const { in enableSC0Bit()
474 return enableNamedBit(MI, AMDGPU::CPol::SC0); in enableSC0Bit()
479 bool enableSC1Bit(const MachineBasicBlock::iterator &MI) const { in enableSC1Bit()
480 return enableNamedBit(MI, AMDGPU::CPol::SC1); in enableSC1Bit()
485 bool enableNTBit(const MachineBasicBlock::iterator &MI) const { in enableNTBit()
486 return enableNamedBit(MI, AMDGPU::CPol::NT); in enableNTBit()
493 bool enableLoadCacheBypass(const MachineBasicBlock::iterator &MI,
497 bool enableStoreCacheBypass(const MachineBasicBlock::iterator &MI,
501 bool enableRMWCacheBypass(const MachineBasicBlock::iterator &MI,
505 bool enableVolatileAndOrNonTemporal(MachineBasicBlock::iterator &MI,
510 bool insertAcquire(MachineBasicBlock::iterator &MI, SIAtomicScope Scope,
513 bool insertRelease(MachineBasicBlock::iterator &MI, SIAtomicScope Scope,
518 MachineBasicBlock::iterator &MI) const override { in tryForceStoreSC0SC1()
525 Changed |= enableSC0Bit(MI); in tryForceStoreSC0SC1()
526 Changed |= enableSC1Bit(MI); in tryForceStoreSC0SC1()
537 bool enableDLCBit(const MachineBasicBlock::iterator &MI) const { in enableDLCBit()
538 return enableNamedBit(MI, AMDGPU::CPol::DLC); in enableDLCBit()
545 bool enableLoadCacheBypass(const MachineBasicBlock::iterator &MI,
549 bool enableVolatileAndOrNonTemporal(MachineBasicBlock::iterator &MI,
554 bool insertWait(MachineBasicBlock::iterator &MI,
561 bool insertAcquire(MachineBasicBlock::iterator &MI,
571 bool enableLoadCacheBypass(const MachineBasicBlock::iterator &MI,
575 bool enableVolatileAndOrNonTemporal(MachineBasicBlock::iterator &MI,
585 bool setTH(const MachineBasicBlock::iterator MI,
589 bool setScope(const MachineBasicBlock::iterator MI,
595 bool insertWait(MachineBasicBlock::iterator &MI, SIAtomicScope Scope,
599 bool insertAcquire(MachineBasicBlock::iterator &MI, SIAtomicScope Scope,
602 bool enableVolatileAndOrNonTemporal(MachineBasicBlock::iterator &MI,
619 bool isAtomicRet(const MachineInstr &MI) const { in isAtomicRet()
620 return SIInstrInfo::isAtomicRet(MI); in isAtomicRet()
630 MachineBasicBlock::iterator &MI);
634 MachineBasicBlock::iterator &MI);
638 MachineBasicBlock::iterator &MI);
642 MachineBasicBlock::iterator &MI);
663 void SIMemOpAccess::reportUnsupported(const MachineBasicBlock::iterator &MI, in reportUnsupported() argument
665 const Function &Func = MI->getParent()->getParent()->getFunction(); in reportUnsupported()
666 DiagnosticInfoUnsupported Diag(Func, Msg, MI->getDebugLoc()); in reportUnsupported()
724 const MachineBasicBlock::iterator &MI) const { in constructFromMIWithMMO()
725 assert(MI->getNumMemOperands() > 0); in constructFromMIWithMMO()
736 for (const auto &MMO : MI->memoperands()) { in constructFromMIWithMMO()
746 reportUnsupported(MI, in constructFromMIWithMMO()
766 reportUnsupported(MI, "Unsupported atomic synchronization scope"); in constructFromMIWithMMO()
774 reportUnsupported(MI, "Unsupported atomic address space"); in constructFromMIWithMMO()
784 SIMemOpAccess::getLoadInfo(const MachineBasicBlock::iterator &MI) const { in getLoadInfo()
785 assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic); in getLoadInfo()
787 if (!(MI->mayLoad() && !MI->mayStore())) in getLoadInfo()
791 if (MI->getNumMemOperands() == 0) in getLoadInfo()
794 return constructFromMIWithMMO(MI); in getLoadInfo()
798 SIMemOpAccess::getStoreInfo(const MachineBasicBlock::iterator &MI) const { in getStoreInfo()
799 assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic); in getStoreInfo()
801 if (!(!MI->mayLoad() && MI->mayStore())) in getStoreInfo()
805 if (MI->getNumMemOperands() == 0) in getStoreInfo()
808 return constructFromMIWithMMO(MI); in getStoreInfo()
812 SIMemOpAccess::getAtomicFenceInfo(const MachineBasicBlock::iterator &MI) const { in getAtomicFenceInfo()
813 assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic); in getAtomicFenceInfo()
815 if (MI->getOpcode() != AMDGPU::ATOMIC_FENCE) in getAtomicFenceInfo()
819 static_cast<AtomicOrdering>(MI->getOperand(0).getImm()); in getAtomicFenceInfo()
821 SyncScope::ID SSID = static_cast<SyncScope::ID>(MI->getOperand(1).getImm()); in getAtomicFenceInfo()
824 reportUnsupported(MI, "Unsupported atomic synchronization scope"); in getAtomicFenceInfo()
836 reportUnsupported(MI, "Unsupported atomic address space"); in getAtomicFenceInfo()
845 const MachineBasicBlock::iterator &MI) const { in getAtomicCmpxchgOrRmwInfo()
846 assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic); in getAtomicCmpxchgOrRmwInfo()
848 if (!(MI->mayLoad() && MI->mayStore())) in getAtomicCmpxchgOrRmwInfo()
852 if (MI->getNumMemOperands() == 0) in getAtomicCmpxchgOrRmwInfo()
855 return constructFromMIWithMMO(MI); in getAtomicCmpxchgOrRmwInfo()
864 bool SICacheControl::enableNamedBit(const MachineBasicBlock::iterator MI, in enableNamedBit() argument
866 MachineOperand *CPol = TII->getNamedOperand(*MI, AMDGPU::OpName::cpol); in enableNamedBit()
893 const MachineBasicBlock::iterator &MI, in enableLoadCacheBypass() argument
896 assert(MI->mayLoad() && !MI->mayStore()); in enableLoadCacheBypass()
905 Changed |= enableGLCBit(MI); in enableLoadCacheBypass()
928 const MachineBasicBlock::iterator &MI, in enableStoreCacheBypass() argument
931 assert(!MI->mayLoad() && MI->mayStore()); in enableStoreCacheBypass()
941 const MachineBasicBlock::iterator &MI, in enableRMWCacheBypass() argument
944 assert(MI->mayLoad() && MI->mayStore()); in enableRMWCacheBypass()
956 MachineBasicBlock::iterator &MI, SIAtomicAddrSpace AddrSpace, SIMemOp Op, in enableVolatileAndOrNonTemporal() argument
961 assert(MI->mayLoad() ^ MI->mayStore()); in enableVolatileAndOrNonTemporal()
976 Changed |= enableGLCBit(MI); in enableVolatileAndOrNonTemporal()
983 Changed |= insertWait(MI, SIAtomicScope::SYSTEM, AddrSpace, Op, false, in enableVolatileAndOrNonTemporal()
992 Changed |= enableGLCBit(MI); in enableVolatileAndOrNonTemporal()
993 Changed |= enableSLCBit(MI); in enableVolatileAndOrNonTemporal()
1000 bool SIGfx6CacheControl::insertWait(MachineBasicBlock::iterator &MI, in insertWait() argument
1008 MachineBasicBlock &MBB = *MI->getParent(); in insertWait()
1009 DebugLoc DL = MI->getDebugLoc(); in insertWait()
1012 ++MI; in insertWait()
1087 BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_WAITCNT_soft)) in insertWait()
1093 --MI; in insertWait()
1098 bool SIGfx6CacheControl::insertAcquire(MachineBasicBlock::iterator &MI, in insertAcquire() argument
1107 MachineBasicBlock &MBB = *MI->getParent(); in insertAcquire()
1108 DebugLoc DL = MI->getDebugLoc(); in insertAcquire()
1111 ++MI; in insertAcquire()
1117 BuildMI(MBB, MI, DL, TII->get(AMDGPU::BUFFER_WBINVL1)); in insertAcquire()
1138 --MI; in insertAcquire()
1143 bool SIGfx6CacheControl::insertRelease(MachineBasicBlock::iterator &MI, in insertRelease() argument
1148 return insertWait(MI, Scope, AddrSpace, SIMemOp::LOAD | SIMemOp::STORE, in insertRelease()
1152 bool SIGfx7CacheControl::insertAcquire(MachineBasicBlock::iterator &MI, in insertAcquire() argument
1161 MachineBasicBlock &MBB = *MI->getParent(); in insertAcquire()
1162 DebugLoc DL = MI->getDebugLoc(); in insertAcquire()
1171 ++MI; in insertAcquire()
1177 BuildMI(MBB, MI, DL, TII->get(InvalidateL1)); in insertAcquire()
1198 --MI; in insertAcquire()
1204 const MachineBasicBlock::iterator &MI, in enableLoadCacheBypass() argument
1207 assert(MI->mayLoad() && !MI->mayStore()); in enableLoadCacheBypass()
1216 Changed |= enableGLCBit(MI); in enableLoadCacheBypass()
1224 Changed |= enableGLCBit(MI); in enableLoadCacheBypass()
1246 const MachineBasicBlock::iterator &MI, in enableStoreCacheBypass() argument
1249 assert(!MI->mayLoad() && MI->mayStore()); in enableStoreCacheBypass()
1281 const MachineBasicBlock::iterator &MI, in enableRMWCacheBypass() argument
1284 assert(MI->mayLoad() && MI->mayStore()); in enableRMWCacheBypass()
1309 MachineBasicBlock::iterator &MI, SIAtomicAddrSpace AddrSpace, SIMemOp Op, in enableVolatileAndOrNonTemporal() argument
1314 assert(MI->mayLoad() ^ MI->mayStore()); in enableVolatileAndOrNonTemporal()
1329 Changed |= enableGLCBit(MI); in enableVolatileAndOrNonTemporal()
1336 Changed |= insertWait(MI, SIAtomicScope::SYSTEM, AddrSpace, Op, false, in enableVolatileAndOrNonTemporal()
1345 Changed |= enableGLCBit(MI); in enableVolatileAndOrNonTemporal()
1346 Changed |= enableSLCBit(MI); in enableVolatileAndOrNonTemporal()
1353 bool SIGfx90ACacheControl::insertWait(MachineBasicBlock::iterator &MI, in insertWait() argument
1377 return SIGfx7CacheControl::insertWait(MI, Scope, AddrSpace, Op, in insertWait()
1381 bool SIGfx90ACacheControl::insertAcquire(MachineBasicBlock::iterator &MI, in insertAcquire() argument
1390 MachineBasicBlock &MBB = *MI->getParent(); in insertAcquire()
1391 DebugLoc DL = MI->getDebugLoc(); in insertAcquire()
1394 ++MI; in insertAcquire()
1402 BuildMI(MBB, MI, DL, TII->get(AMDGPU::BUFFER_INVL2)); in insertAcquire()
1440 --MI; in insertAcquire()
1442 Changed |= SIGfx7CacheControl::insertAcquire(MI, Scope, AddrSpace, Pos); in insertAcquire()
1447 bool SIGfx90ACacheControl::insertRelease(MachineBasicBlock::iterator &MI, in insertRelease() argument
1454 MachineBasicBlock &MBB = *MI->getParent(); in insertRelease()
1455 const DebugLoc &DL = MI->getDebugLoc(); in insertRelease()
1458 ++MI; in insertRelease()
1469 BuildMI(MBB, MI, DL, TII->get(AMDGPU::BUFFER_WBL2)) in insertRelease()
1488 --MI; in insertRelease()
1491 SIGfx7CacheControl::insertRelease(MI, Scope, AddrSpace, in insertRelease()
1498 const MachineBasicBlock::iterator &MI, SIAtomicScope Scope, in enableLoadCacheBypass() argument
1500 assert(MI->mayLoad() && !MI->mayStore()); in enableLoadCacheBypass()
1507 Changed |= enableSC0Bit(MI); in enableLoadCacheBypass()
1508 Changed |= enableSC1Bit(MI); in enableLoadCacheBypass()
1512 Changed |= enableSC1Bit(MI); in enableLoadCacheBypass()
1520 Changed |= enableSC0Bit(MI); in enableLoadCacheBypass()
1542 const MachineBasicBlock::iterator &MI, in enableStoreCacheBypass() argument
1544 assert(!MI->mayLoad() && MI->mayStore()); in enableStoreCacheBypass()
1551 Changed |= enableSC0Bit(MI); in enableStoreCacheBypass()
1552 Changed |= enableSC1Bit(MI); in enableStoreCacheBypass()
1556 Changed |= enableSC1Bit(MI); in enableStoreCacheBypass()
1560 Changed |= enableSC0Bit(MI); in enableStoreCacheBypass()
1582 const MachineBasicBlock::iterator &MI, SIAtomicScope Scope, in enableRMWCacheBypass() argument
1584 assert(MI->mayLoad() && MI->mayStore()); in enableRMWCacheBypass()
1591 Changed |= enableSC1Bit(MI); in enableRMWCacheBypass()
1611 MachineBasicBlock::iterator &MI, SIAtomicAddrSpace AddrSpace, SIMemOp Op, in enableVolatileAndOrNonTemporal() argument
1616 assert(MI->mayLoad() ^ MI->mayStore()); in enableVolatileAndOrNonTemporal()
1628 Changed |= enableSC0Bit(MI); in enableVolatileAndOrNonTemporal()
1629 Changed |= enableSC1Bit(MI); in enableVolatileAndOrNonTemporal()
1636 Changed |= insertWait(MI, SIAtomicScope::SYSTEM, AddrSpace, Op, false, in enableVolatileAndOrNonTemporal()
1643 Changed |= enableNTBit(MI); in enableVolatileAndOrNonTemporal()
1650 bool SIGfx940CacheControl::insertAcquire(MachineBasicBlock::iterator &MI, in insertAcquire() argument
1659 MachineBasicBlock &MBB = *MI->getParent(); in insertAcquire()
1660 DebugLoc DL = MI->getDebugLoc(); in insertAcquire()
1663 ++MI; in insertAcquire()
1671 BuildMI(MBB, MI, DL, TII->get(AMDGPU::BUFFER_INV)) in insertAcquire()
1685 BuildMI(MBB, MI, DL, TII->get(AMDGPU::BUFFER_INV)) in insertAcquire()
1703 BuildMI(MBB, MI, DL, TII->get(AMDGPU::BUFFER_INV)) in insertAcquire()
1731 --MI; in insertAcquire()
1736 bool SIGfx940CacheControl::insertRelease(MachineBasicBlock::iterator &MI, in insertRelease() argument
1743 MachineBasicBlock &MBB = *MI->getParent(); in insertRelease()
1744 DebugLoc DL = MI->getDebugLoc(); in insertRelease()
1747 ++MI; in insertRelease()
1758 BuildMI(MBB, MI, DL, TII->get(AMDGPU::BUFFER_WBL2)) in insertRelease()
1767 BuildMI(MBB, MI, DL, TII->get(AMDGPU::BUFFER_WBL2)) in insertRelease()
1789 --MI; in insertRelease()
1793 Changed |= insertWait(MI, Scope, AddrSpace, SIMemOp::LOAD | SIMemOp::STORE, in insertRelease()
1800 const MachineBasicBlock::iterator &MI, in enableLoadCacheBypass() argument
1803 assert(MI->mayLoad() && !MI->mayStore()); in enableLoadCacheBypass()
1812 Changed |= enableGLCBit(MI); in enableLoadCacheBypass()
1813 Changed |= enableDLCBit(MI); in enableLoadCacheBypass()
1821 Changed |= enableGLCBit(MI); in enableLoadCacheBypass()
1843 MachineBasicBlock::iterator &MI, SIAtomicAddrSpace AddrSpace, SIMemOp Op, in enableVolatileAndOrNonTemporal() argument
1849 assert(MI->mayLoad() ^ MI->mayStore()); in enableVolatileAndOrNonTemporal()
1864 Changed |= enableGLCBit(MI); in enableVolatileAndOrNonTemporal()
1865 Changed |= enableDLCBit(MI); in enableVolatileAndOrNonTemporal()
1873 Changed |= insertWait(MI, SIAtomicScope::SYSTEM, AddrSpace, Op, false, in enableVolatileAndOrNonTemporal()
1884 Changed |= enableGLCBit(MI); in enableVolatileAndOrNonTemporal()
1885 Changed |= enableSLCBit(MI); in enableVolatileAndOrNonTemporal()
1893 bool SIGfx10CacheControl::insertWait(MachineBasicBlock::iterator &MI, in insertWait() argument
1901 MachineBasicBlock &MBB = *MI->getParent(); in insertWait()
1902 DebugLoc DL = MI->getDebugLoc(); in insertWait()
1905 ++MI; in insertWait()
1996 BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_WAITCNT_soft)) in insertWait()
2002 BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_WAITCNT_VSCNT_soft)) in insertWait()
2009 --MI; in insertWait()
2014 bool SIGfx10CacheControl::insertAcquire(MachineBasicBlock::iterator &MI, in insertAcquire() argument
2023 MachineBasicBlock &MBB = *MI->getParent(); in insertAcquire()
2024 DebugLoc DL = MI->getDebugLoc(); in insertAcquire()
2027 ++MI; in insertAcquire()
2033 BuildMI(MBB, MI, DL, TII->get(AMDGPU::BUFFER_GL0_INV)); in insertAcquire()
2034 BuildMI(MBB, MI, DL, TII->get(AMDGPU::BUFFER_GL1_INV)); in insertAcquire()
2043 BuildMI(MBB, MI, DL, TII->get(AMDGPU::BUFFER_GL0_INV)); in insertAcquire()
2064 --MI; in insertAcquire()
2070 const MachineBasicBlock::iterator &MI, SIAtomicScope Scope, in enableLoadCacheBypass() argument
2072 assert(MI->mayLoad() && !MI->mayStore()); in enableLoadCacheBypass()
2081 Changed |= enableGLCBit(MI); in enableLoadCacheBypass()
2089 Changed |= enableGLCBit(MI); in enableLoadCacheBypass()
2111 MachineBasicBlock::iterator &MI, SIAtomicAddrSpace AddrSpace, SIMemOp Op, in enableVolatileAndOrNonTemporal() argument
2117 assert(MI->mayLoad() ^ MI->mayStore()); in enableVolatileAndOrNonTemporal()
2132 Changed |= enableGLCBit(MI); in enableVolatileAndOrNonTemporal()
2135 Changed |= enableDLCBit(MI); in enableVolatileAndOrNonTemporal()
2142 Changed |= insertWait(MI, SIAtomicScope::SYSTEM, AddrSpace, Op, false, in enableVolatileAndOrNonTemporal()
2153 Changed |= enableGLCBit(MI); in enableVolatileAndOrNonTemporal()
2154 Changed |= enableSLCBit(MI); in enableVolatileAndOrNonTemporal()
2157 Changed |= enableDLCBit(MI); in enableVolatileAndOrNonTemporal()
2164 bool SIGfx12CacheControl::setTH(const MachineBasicBlock::iterator MI, in setTH() argument
2166 MachineOperand *CPol = TII->getNamedOperand(*MI, OpName::cpol); in setTH()
2179 bool SIGfx12CacheControl::setScope(const MachineBasicBlock::iterator MI, in setScope() argument
2181 MachineOperand *CPol = TII->getNamedOperand(*MI, OpName::cpol); in setScope()
2194 bool SIGfx12CacheControl::insertWait(MachineBasicBlock::iterator &MI, in insertWait() argument
2201 MachineBasicBlock &MBB = *MI->getParent(); in insertWait()
2202 DebugLoc DL = MI->getDebugLoc(); in insertWait()
2209 ++MI; in insertWait()
2268 BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_WAIT_BVHCNT_soft)).addImm(0); in insertWait()
2269 BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_WAIT_SAMPLECNT_soft)).addImm(0); in insertWait()
2270 BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_WAIT_LOADCNT_soft)).addImm(0); in insertWait()
2275 BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_WAIT_STORECNT_soft)).addImm(0); in insertWait()
2280 BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_WAIT_DSCNT_soft)).addImm(0); in insertWait()
2285 --MI; in insertWait()
2290 bool SIGfx12CacheControl::insertAcquire(MachineBasicBlock::iterator &MI, in insertAcquire() argument
2297 MachineBasicBlock &MBB = *MI->getParent(); in insertAcquire()
2298 DebugLoc DL = MI->getDebugLoc(); in insertAcquire()
2336 ++MI; in insertAcquire()
2338 BuildMI(MBB, MI, DL, TII->get(AMDGPU::GLOBAL_INV)).addImm(ScopeImm); in insertAcquire()
2341 --MI; in insertAcquire()
2347 MachineBasicBlock::iterator &MI, SIAtomicAddrSpace AddrSpace, SIMemOp Op, in enableVolatileAndOrNonTemporal() argument
2351 assert(MI->mayLoad() ^ MI->mayStore()); in enableVolatileAndOrNonTemporal()
2363 Changed |= setTH(MI, AMDGPU::CPol::TH_NT); in enableVolatileAndOrNonTemporal()
2367 Changed |= setScope(MI, AMDGPU::CPol::SCOPE_SYS); in enableVolatileAndOrNonTemporal()
2374 Changed |= insertWait(MI, SIAtomicScope::SYSTEM, AddrSpace, Op, false, in enableVolatileAndOrNonTemporal()
2385 for (auto &MI : AtomicPseudoMIs) in removeAtomicPseudoMIs() local
2386 MI->eraseFromParent(); in removeAtomicPseudoMIs()
2393 MachineBasicBlock::iterator &MI) { in expandLoad() argument
2394 assert(MI->mayLoad() && !MI->mayStore()); in expandLoad()
2402 Changed |= CC->enableLoadCacheBypass(MI, MOI.getScope(), in expandLoad()
2407 Changed |= CC->insertWait(MI, MOI.getScope(), in expandLoad()
2415 Changed |= CC->insertWait(MI, MOI.getScope(), in expandLoad()
2420 Changed |= CC->insertAcquire(MI, MOI.getScope(), in expandLoad()
2431 Changed |= CC->enableVolatileAndOrNonTemporal(MI, MOI.getInstrAddrSpace(), in expandLoad()
2438 MachineBasicBlock::iterator &MI) { in expandStore() argument
2439 assert(!MI->mayLoad() && MI->mayStore()); in expandStore()
2447 Changed |= CC->enableStoreCacheBypass(MI, MOI.getScope(), in expandStore()
2453 Changed |= CC->insertRelease(MI, MOI.getScope(), in expandStore()
2465 MI, MOI.getInstrAddrSpace(), SIMemOp::STORE, MOI.isVolatile(), in expandStore()
2471 MachineBasicBlock::iterator &MI) { in expandAtomicFence() argument
2472 assert(MI->getOpcode() == AMDGPU::ATOMIC_FENCE); in expandAtomicFence()
2474 AtomicPseudoMIs.push_back(MI); in expandAtomicFence()
2479 Changed |= CC->insertWait(MI, MOI.getScope(), MOI.getOrderingAddrSpace(), in expandAtomicFence()
2494 Changed |= CC->insertRelease(MI, MOI.getScope(), in expandAtomicFence()
2507 Changed |= CC->insertAcquire(MI, MOI.getScope(), in expandAtomicFence()
2518 MachineBasicBlock::iterator &MI) { in expandAtomicCmpxchgOrRmw() argument
2519 assert(MI->mayLoad() && MI->mayStore()); in expandAtomicCmpxchgOrRmw()
2529 Changed |= CC->enableRMWCacheBypass(MI, MOI.getScope(), in expandAtomicCmpxchgOrRmw()
2537 Changed |= CC->insertRelease(MI, MOI.getScope(), in expandAtomicCmpxchgOrRmw()
2547 Changed |= CC->insertWait(MI, MOI.getScope(), in expandAtomicCmpxchgOrRmw()
2549 isAtomicRet(*MI) ? SIMemOp::LOAD : in expandAtomicCmpxchgOrRmw()
2553 Changed |= CC->insertAcquire(MI, MOI.getScope(), in expandAtomicCmpxchgOrRmw()
2571 for (auto MI = MBB.begin(); MI != MBB.end(); ++MI) { in runOnMachineFunction() local
2574 if (MI->isBundle() && MI->mayLoadOrStore()) { in runOnMachineFunction()
2575 MachineBasicBlock::instr_iterator II(MI->getIterator()); in runOnMachineFunction()
2584 MI->eraseFromParent(); in runOnMachineFunction()
2585 MI = II->getIterator(); in runOnMachineFunction()
2588 if (!(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic)) in runOnMachineFunction()
2591 if (const auto &MOI = MOA.getLoadInfo(MI)) in runOnMachineFunction()
2592 Changed |= expandLoad(*MOI, MI); in runOnMachineFunction()
2593 else if (const auto &MOI = MOA.getStoreInfo(MI)) { in runOnMachineFunction()
2594 Changed |= expandStore(*MOI, MI); in runOnMachineFunction()
2595 Changed |= CC->tryForceStoreSC0SC1(*MOI, MI); in runOnMachineFunction()
2596 } else if (const auto &MOI = MOA.getAtomicFenceInfo(MI)) in runOnMachineFunction()
2597 Changed |= expandAtomicFence(*MOI, MI); in runOnMachineFunction()
2598 else if (const auto &MOI = MOA.getAtomicCmpxchgOrRmwInfo(MI)) in runOnMachineFunction()
2599 Changed |= expandAtomicCmpxchgOrRmw(*MOI, MI); in runOnMachineFunction()