80     bool tryExpandAtomicRMW(AtomicRMWInst *AI);
93     AtomicRMWInst *widenPartwordAtomicRMW(AtomicRMWInst *AI);
95     void expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI);
121     llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
496   AtomicRMWInst *AI = Builder.CreateAtomicRMW(  in expandAtomicStore()  local
502   return tryExpandAtomicRMW(AI);  in expandAtomicStore()
572 bool AtomicExpand::tryExpandAtomicRMW(AtomicRMWInst *AI) {  in tryExpandAtomicRMW()  argument
573   switch (TLI->shouldExpandAtomicRMWInIR(AI)) {  in tryExpandAtomicRMW()
578     unsigned ValueSize = getAtomicOpSize(AI);  in tryExpandAtomicRMW()
580       expandPartwordAtomicRMW(AI,  in tryExpandAtomicRMW()
584         return performAtomicOp(AI->getOperation(), Builder, Loaded,  in tryExpandAtomicRMW()
585                                AI->getValOperand());  in tryExpandAtomicRMW()
587       expandAtomicOpToLLSC(AI, AI->getType(), AI->getPointerOperand(),  in tryExpandAtomicRMW()
588                            AI->getAlign(), AI->getOrdering(), PerformOp);  in tryExpandAtomicRMW()
594     unsigned ValueSize = getAtomicOpSize(AI);  in tryExpandAtomicRMW()
597       if (AI->getType()->isFloatingPointTy())  in tryExpandAtomicRMW()
600       expandPartwordAtomicRMW(AI,  in tryExpandAtomicRMW()
603       expandAtomicRMWToCmpXchg(AI, createCmpXchgInstFun);  in tryExpandAtomicRMW()
608     expandAtomicRMWToMaskedIntrinsic(AI);  in tryExpandAtomicRMW()
808     AtomicRMWInst *AI, TargetLoweringBase::AtomicExpansionKind ExpansionKind) {  in expandPartwordAtomicRMW()  argument
809   AtomicOrdering MemOpOrder = AI->getOrdering();  in expandPartwordAtomicRMW()
810   SyncScope::ID SSID = AI->getSyncScopeID();  in expandPartwordAtomicRMW()
812   IRBuilder<> Builder(AI);  in expandPartwordAtomicRMW()
815       createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),  in expandPartwordAtomicRMW()
816                        AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);  in expandPartwordAtomicRMW()
819       Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType),  in expandPartwordAtomicRMW()
823     return performMaskedAtomicOp(AI->getOperation(), Builder, Loaded,  in expandPartwordAtomicRMW()
824                                  ValOperand_Shifted, AI->getValOperand(), PMV);  in expandPartwordAtomicRMW()
841   AI->replaceAllUsesWith(FinalOldResult);  in expandPartwordAtomicRMW()
842   AI->eraseFromParent();  in expandPartwordAtomicRMW()
846 AtomicRMWInst *AtomicExpand::widenPartwordAtomicRMW(AtomicRMWInst *AI) {  in widenPartwordAtomicRMW()  argument
847   IRBuilder<> Builder(AI);  in widenPartwordAtomicRMW()
848   AtomicRMWInst::BinOp Op = AI->getOperation();  in widenPartwordAtomicRMW()
855       createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),  in widenPartwordAtomicRMW()
856                        AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);  in widenPartwordAtomicRMW()
859       Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType),  in widenPartwordAtomicRMW()
872                               PMV.AlignedAddrAlignment, AI->getOrdering());  in widenPartwordAtomicRMW()
875   AI->replaceAllUsesWith(FinalOldResult);  in widenPartwordAtomicRMW()
876   AI->eraseFromParent();  in widenPartwordAtomicRMW()
1017 void AtomicExpand::expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI) {  in expandAtomicRMWToMaskedIntrinsic()  argument
1018   IRBuilder<> Builder(AI);  in expandAtomicRMWToMaskedIntrinsic()
1021       createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),  in expandAtomicRMWToMaskedIntrinsic()
1022                        AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);  in expandAtomicRMWToMaskedIntrinsic()
1028   AtomicRMWInst::BinOp RMWOp = AI->getOperation();  in expandAtomicRMWToMaskedIntrinsic()
1033       Builder.CreateCast(CastOp, AI->getValOperand(), PMV.WordType),  in expandAtomicRMWToMaskedIntrinsic()
1036       Builder, AI, PMV.AlignedAddr, ValOperand_Shifted, PMV.Mask, PMV.ShiftAmt,  in expandAtomicRMWToMaskedIntrinsic()
1037       AI->getOrdering());  in expandAtomicRMWToMaskedIntrinsic()
1039   AI->replaceAllUsesWith(FinalOldResult);  in expandAtomicRMWToMaskedIntrinsic()
1040   AI->eraseFromParent();  in expandAtomicRMWToMaskedIntrinsic()
1518 bool llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,  in expandAtomicRMWToCmpXchg()  argument
1520   IRBuilder<> Builder(AI);  in expandAtomicRMWToCmpXchg()
1522       Builder, AI->getType(), AI->getPointerOperand(), AI->getAlign(),  in expandAtomicRMWToCmpXchg()
1523       AI->getOrdering(), AI->getSyncScopeID(),  in expandAtomicRMWToCmpXchg()
1525         return performAtomicOp(AI->getOperation(), Builder, Loaded,  in expandAtomicRMWToCmpXchg()
1526                                AI->getValOperand());  in expandAtomicRMWToCmpXchg()
1530   AI->replaceAllUsesWith(Loaded);  in expandAtomicRMWToCmpXchg()
1531   AI->eraseFromParent();  in expandAtomicRMWToCmpXchg()