Lines Matching refs:AI
85 bool tryExpandAtomicRMW(AtomicRMWInst *AI);
97 AtomicRMWInst *widenPartwordAtomicRMW(AtomicRMWInst *AI);
99 void expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI);
125 llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
530 AtomicRMWInst *AI = Builder.CreateAtomicRMW( in expandAtomicStore() local
536 tryExpandAtomicRMW(AI); in expandAtomicStore()
564 bool AtomicExpand::tryExpandAtomicRMW(AtomicRMWInst *AI) { in tryExpandAtomicRMW() argument
565 LLVMContext &Ctx = AI->getModule()->getContext(); in tryExpandAtomicRMW()
566 TargetLowering::AtomicExpansionKind Kind = TLI->shouldExpandAtomicRMWInIR(AI); in tryExpandAtomicRMW()
572 unsigned ValueSize = getAtomicOpSize(AI); in tryExpandAtomicRMW()
574 expandPartwordAtomicRMW(AI, in tryExpandAtomicRMW()
578 return buildAtomicRMWValue(AI->getOperation(), Builder, Loaded, in tryExpandAtomicRMW()
579 AI->getValOperand()); in tryExpandAtomicRMW()
581 expandAtomicOpToLLSC(AI, AI->getType(), AI->getPointerOperand(), in tryExpandAtomicRMW()
582 AI->getAlign(), AI->getOrdering(), PerformOp); in tryExpandAtomicRMW()
588 unsigned ValueSize = getAtomicOpSize(AI); in tryExpandAtomicRMW()
590 expandPartwordAtomicRMW(AI, in tryExpandAtomicRMW()
595 auto MemScope = SSNs[AI->getSyncScopeID()].empty() in tryExpandAtomicRMW()
597 : SSNs[AI->getSyncScopeID()]; in tryExpandAtomicRMW()
598 OptimizationRemarkEmitter ORE(AI->getFunction()); in tryExpandAtomicRMW()
600 return OptimizationRemark(DEBUG_TYPE, "Passed", AI) in tryExpandAtomicRMW()
602 << AI->getOperationName(AI->getOperation()) << " operation at " in tryExpandAtomicRMW()
605 expandAtomicRMWToCmpXchg(AI, createCmpXchgInstFun); in tryExpandAtomicRMW()
610 expandAtomicRMWToMaskedIntrinsic(AI); in tryExpandAtomicRMW()
614 TLI->emitBitTestAtomicRMWIntrinsic(AI); in tryExpandAtomicRMW()
618 TLI->emitCmpArithAtomicRMWIntrinsic(AI); in tryExpandAtomicRMW()
622 return lowerAtomicRMWInst(AI); in tryExpandAtomicRMW()
624 TLI->emitExpandAtomicRMW(AI); in tryExpandAtomicRMW()
847 AtomicRMWInst *AI, TargetLoweringBase::AtomicExpansionKind ExpansionKind) { in expandPartwordAtomicRMW() argument
848 AtomicOrdering MemOpOrder = AI->getOrdering(); in expandPartwordAtomicRMW()
849 SyncScope::ID SSID = AI->getSyncScopeID(); in expandPartwordAtomicRMW()
851 ReplacementIRBuilder Builder(AI, *DL); in expandPartwordAtomicRMW()
854 createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(), in expandPartwordAtomicRMW()
855 AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8); in expandPartwordAtomicRMW()
858 if (AI->getOperation() == AtomicRMWInst::Xchg || in expandPartwordAtomicRMW()
859 AI->getOperation() == AtomicRMWInst::Add || in expandPartwordAtomicRMW()
860 AI->getOperation() == AtomicRMWInst::Sub || in expandPartwordAtomicRMW()
861 AI->getOperation() == AtomicRMWInst::Nand) { in expandPartwordAtomicRMW()
863 Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType), in expandPartwordAtomicRMW()
868 return performMaskedAtomicOp(AI->getOperation(), Builder, Loaded, in expandPartwordAtomicRMW()
869 ValOperand_Shifted, AI->getValOperand(), PMV); in expandPartwordAtomicRMW()
885 AI->replaceAllUsesWith(FinalOldResult); in expandPartwordAtomicRMW()
886 AI->eraseFromParent(); in expandPartwordAtomicRMW()
890 AtomicRMWInst *AtomicExpand::widenPartwordAtomicRMW(AtomicRMWInst *AI) { in widenPartwordAtomicRMW() argument
891 ReplacementIRBuilder Builder(AI, *DL); in widenPartwordAtomicRMW()
892 AtomicRMWInst::BinOp Op = AI->getOperation(); in widenPartwordAtomicRMW()
899 createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(), in widenPartwordAtomicRMW()
900 AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8); in widenPartwordAtomicRMW()
903 Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType), in widenPartwordAtomicRMW()
916 AI->getOrdering(), AI->getSyncScopeID()); in widenPartwordAtomicRMW()
920 AI->replaceAllUsesWith(FinalOldResult); in widenPartwordAtomicRMW()
921 AI->eraseFromParent(); in widenPartwordAtomicRMW()
1062 void AtomicExpand::expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI) { in expandAtomicRMWToMaskedIntrinsic() argument
1063 ReplacementIRBuilder Builder(AI, *DL); in expandAtomicRMWToMaskedIntrinsic()
1066 createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(), in expandAtomicRMWToMaskedIntrinsic()
1067 AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8); in expandAtomicRMWToMaskedIntrinsic()
1073 AtomicRMWInst::BinOp RMWOp = AI->getOperation(); in expandAtomicRMWToMaskedIntrinsic()
1078 Builder.CreateCast(CastOp, AI->getValOperand(), PMV.WordType), in expandAtomicRMWToMaskedIntrinsic()
1081 Builder, AI, PMV.AlignedAddr, ValOperand_Shifted, PMV.Mask, PMV.ShiftAmt, in expandAtomicRMWToMaskedIntrinsic()
1082 AI->getOrdering()); in expandAtomicRMWToMaskedIntrinsic()
1084 AI->replaceAllUsesWith(FinalOldResult); in expandAtomicRMWToMaskedIntrinsic()
1085 AI->eraseFromParent(); in expandAtomicRMWToMaskedIntrinsic()
1562 bool llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI, in expandAtomicRMWToCmpXchg() argument
1564 ReplacementIRBuilder Builder(AI, AI->getModule()->getDataLayout()); in expandAtomicRMWToCmpXchg()
1566 AI->getFunction()->hasFnAttribute(Attribute::StrictFP)); in expandAtomicRMWToCmpXchg()
1571 Builder, AI->getType(), AI->getPointerOperand(), AI->getAlign(), in expandAtomicRMWToCmpXchg()
1572 AI->getOrdering(), AI->getSyncScopeID(), in expandAtomicRMWToCmpXchg()
1574 return buildAtomicRMWValue(AI->getOperation(), Builder, Loaded, in expandAtomicRMWToCmpXchg()
1575 AI->getValOperand()); in expandAtomicRMWToCmpXchg()
1579 AI->replaceAllUsesWith(Loaded); in expandAtomicRMWToCmpXchg()
1580 AI->eraseFromParent(); in expandAtomicRMWToCmpXchg()