83   bool tryExpandAtomicRMW(AtomicRMWInst *AI);
95   AtomicRMWInst *widenPartwordAtomicRMW(AtomicRMWInst *AI);
97   void expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI);
124   llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
523   AtomicRMWInst *AI = Builder.CreateAtomicRMW(  in expandAtomicStore()  local
529   tryExpandAtomicRMW(AI);  in expandAtomicStore()
559 bool AtomicExpand::tryExpandAtomicRMW(AtomicRMWInst *AI) {  in tryExpandAtomicRMW()  argument
560   LLVMContext &Ctx = AI->getModule()->getContext();  in tryExpandAtomicRMW()
561   TargetLowering::AtomicExpansionKind Kind = TLI->shouldExpandAtomicRMWInIR(AI);  in tryExpandAtomicRMW()
567     unsigned ValueSize = getAtomicOpSize(AI);  in tryExpandAtomicRMW()
569       expandPartwordAtomicRMW(AI,  in tryExpandAtomicRMW()
573         return buildAtomicRMWValue(AI->getOperation(), Builder, Loaded,  in tryExpandAtomicRMW()
574                                    AI->getValOperand());  in tryExpandAtomicRMW()
576       expandAtomicOpToLLSC(AI, AI->getType(), AI->getPointerOperand(),  in tryExpandAtomicRMW()
577                            AI->getAlign(), AI->getOrdering(), PerformOp);  in tryExpandAtomicRMW()
583     unsigned ValueSize = getAtomicOpSize(AI);  in tryExpandAtomicRMW()
586       if (AI->getType()->isFloatingPointTy())  in tryExpandAtomicRMW()
589       expandPartwordAtomicRMW(AI,  in tryExpandAtomicRMW()
594       auto MemScope = SSNs[AI->getSyncScopeID()].empty()  in tryExpandAtomicRMW()
596                           : SSNs[AI->getSyncScopeID()];  in tryExpandAtomicRMW()
597       OptimizationRemarkEmitter ORE(AI->getFunction());  in tryExpandAtomicRMW()
599         return OptimizationRemark(DEBUG_TYPE, "Passed", AI)  in tryExpandAtomicRMW()
601                << AI->getOperationName(AI->getOperation()) << " operation at "  in tryExpandAtomicRMW()
604       expandAtomicRMWToCmpXchg(AI, createCmpXchgInstFun);  in tryExpandAtomicRMW()
609     expandAtomicRMWToMaskedIntrinsic(AI);  in tryExpandAtomicRMW()
613     TLI->emitBitTestAtomicRMWIntrinsic(AI);  in tryExpandAtomicRMW()
617     return lowerAtomicRMWInst(AI);  in tryExpandAtomicRMW()
815     AtomicRMWInst *AI, TargetLoweringBase::AtomicExpansionKind ExpansionKind) {  in expandPartwordAtomicRMW()  argument
816   AtomicOrdering MemOpOrder = AI->getOrdering();  in expandPartwordAtomicRMW()
817   SyncScope::ID SSID = AI->getSyncScopeID();  in expandPartwordAtomicRMW()
819   IRBuilder<> Builder(AI);  in expandPartwordAtomicRMW()
822       createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),  in expandPartwordAtomicRMW()
823                        AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);  in expandPartwordAtomicRMW()
826       Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType),  in expandPartwordAtomicRMW()
830     return performMaskedAtomicOp(AI->getOperation(), Builder, Loaded,  in expandPartwordAtomicRMW()
831                                  ValOperand_Shifted, AI->getValOperand(), PMV);  in expandPartwordAtomicRMW()
847   AI->replaceAllUsesWith(FinalOldResult);  in expandPartwordAtomicRMW()
848   AI->eraseFromParent();  in expandPartwordAtomicRMW()
852 AtomicRMWInst *AtomicExpand::widenPartwordAtomicRMW(AtomicRMWInst *AI) {  in widenPartwordAtomicRMW()  argument
853   IRBuilder<> Builder(AI);  in widenPartwordAtomicRMW()
854   AtomicRMWInst::BinOp Op = AI->getOperation();  in widenPartwordAtomicRMW()
861       createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),  in widenPartwordAtomicRMW()
862                        AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);  in widenPartwordAtomicRMW()
865       Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType),  in widenPartwordAtomicRMW()
878                               PMV.AlignedAddrAlignment, AI->getOrdering());  in widenPartwordAtomicRMW()
881   AI->replaceAllUsesWith(FinalOldResult);  in widenPartwordAtomicRMW()
882   AI->eraseFromParent();  in widenPartwordAtomicRMW()
1023 void AtomicExpand::expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI) {  in expandAtomicRMWToMaskedIntrinsic()  argument
1024   IRBuilder<> Builder(AI);  in expandAtomicRMWToMaskedIntrinsic()
1027       createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),  in expandAtomicRMWToMaskedIntrinsic()
1028                        AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);  in expandAtomicRMWToMaskedIntrinsic()
1034   AtomicRMWInst::BinOp RMWOp = AI->getOperation();  in expandAtomicRMWToMaskedIntrinsic()
1039       Builder.CreateCast(CastOp, AI->getValOperand(), PMV.WordType),  in expandAtomicRMWToMaskedIntrinsic()
1042       Builder, AI, PMV.AlignedAddr, ValOperand_Shifted, PMV.Mask, PMV.ShiftAmt,  in expandAtomicRMWToMaskedIntrinsic()
1043       AI->getOrdering());  in expandAtomicRMWToMaskedIntrinsic()
1045   AI->replaceAllUsesWith(FinalOldResult);  in expandAtomicRMWToMaskedIntrinsic()
1046   AI->eraseFromParent();  in expandAtomicRMWToMaskedIntrinsic()
1525 bool llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,  in expandAtomicRMWToCmpXchg()  argument
1527   IRBuilder<> Builder(AI);  in expandAtomicRMWToCmpXchg()
1529       Builder, AI->getType(), AI->getPointerOperand(), AI->getAlign(),  in expandAtomicRMWToCmpXchg()
1530       AI->getOrdering(), AI->getSyncScopeID(),  in expandAtomicRMWToCmpXchg()
1532         return buildAtomicRMWValue(AI->getOperation(), Builder, Loaded,  in expandAtomicRMWToCmpXchg()
1533                                    AI->getValOperand());  in expandAtomicRMWToCmpXchg()
1537   AI->replaceAllUsesWith(Loaded);  in expandAtomicRMWToCmpXchg()
1538   AI->eraseFromParent();  in expandAtomicRMWToCmpXchg()