Lines Matching refs:PMV
638 raw_ostream &operator<<(raw_ostream &O, const PartwordMaskValues &PMV) { in operator <<() argument
648 PrintObj(PMV.WordType); in operator <<()
650 PrintObj(PMV.ValueType); in operator <<()
652 PrintObj(PMV.AlignedAddr); in operator <<()
653 O << " AlignedAddrAlignment: " << PMV.AlignedAddrAlignment.value() << '\n'; in operator <<()
655 PrintObj(PMV.ShiftAmt); in operator <<()
657 PrintObj(PMV.Mask); in operator <<()
659 PrintObj(PMV.Inv_Mask); in operator <<()
686 PartwordMaskValues PMV; in createMaskInstrs() local
693 PMV.ValueType = ValueType; in createMaskInstrs()
694 PMV.WordType = MinWordSize > ValueSize ? Type::getIntNTy(Ctx, MinWordSize * 8) in createMaskInstrs()
696 if (PMV.ValueType == PMV.WordType) { in createMaskInstrs()
697 PMV.AlignedAddr = Addr; in createMaskInstrs()
698 PMV.AlignedAddrAlignment = AddrAlign; in createMaskInstrs()
699 PMV.ShiftAmt = ConstantInt::get(PMV.ValueType, 0); in createMaskInstrs()
700 PMV.Mask = ConstantInt::get(PMV.ValueType, ~0, /*isSigned*/ true); in createMaskInstrs()
701 return PMV; in createMaskInstrs()
707 PMV.WordType->getPointerTo(Addr->getType()->getPointerAddressSpace()); in createMaskInstrs()
711 PMV.AlignedAddr = Builder.CreateIntToPtr( in createMaskInstrs()
714 PMV.AlignedAddrAlignment = Align(MinWordSize); in createMaskInstrs()
719 PMV.ShiftAmt = Builder.CreateShl(PtrLSB, 3); in createMaskInstrs()
722 PMV.ShiftAmt = Builder.CreateShl( in createMaskInstrs()
726 PMV.ShiftAmt = Builder.CreateTrunc(PMV.ShiftAmt, PMV.WordType, "ShiftAmt"); in createMaskInstrs()
727 PMV.Mask = Builder.CreateShl( in createMaskInstrs()
728 ConstantInt::get(PMV.WordType, (1 << (ValueSize * 8)) - 1), PMV.ShiftAmt, in createMaskInstrs()
730 PMV.Inv_Mask = Builder.CreateNot(PMV.Mask, "Inv_Mask"); in createMaskInstrs()
731 return PMV; in createMaskInstrs()
735 const PartwordMaskValues &PMV) { in extractMaskedValue() argument
736 assert(WideWord->getType() == PMV.WordType && "Widened type mismatch"); in extractMaskedValue()
737 if (PMV.WordType == PMV.ValueType) in extractMaskedValue()
740 Value *Shift = Builder.CreateLShr(WideWord, PMV.ShiftAmt, "shifted"); in extractMaskedValue()
741 Value *Trunc = Builder.CreateTrunc(Shift, PMV.ValueType, "extracted"); in extractMaskedValue()
746 Value *Updated, const PartwordMaskValues &PMV) { in insertMaskedValue() argument
747 assert(WideWord->getType() == PMV.WordType && "Widened type mismatch"); in insertMaskedValue()
748 assert(Updated->getType() == PMV.ValueType && "Value type mismatch"); in insertMaskedValue()
749 if (PMV.WordType == PMV.ValueType) in insertMaskedValue()
752 Value *ZExt = Builder.CreateZExt(Updated, PMV.WordType, "extended"); in insertMaskedValue()
754 Builder.CreateShl(ZExt, PMV.ShiftAmt, "shifted", /*HasNUW*/ true); in insertMaskedValue()
755 Value *And = Builder.CreateAnd(WideWord, PMV.Inv_Mask, "unmasked"); in insertMaskedValue()
766 const PartwordMaskValues &PMV) { in performMaskedAtomicOp() argument
772 Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask); in performMaskedAtomicOp()
785 Value *NewVal_Masked = Builder.CreateAnd(NewVal, PMV.Mask); in performMaskedAtomicOp()
786 Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask); in performMaskedAtomicOp()
797 Value *Loaded_Extract = extractMaskedValue(Builder, Loaded, PMV); in performMaskedAtomicOp()
799 Value *FinalVal = insertMaskedValue(Builder, Loaded, NewVal, PMV); in performMaskedAtomicOp()
821 PartwordMaskValues PMV = in expandPartwordAtomicRMW() local
826 Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType), in expandPartwordAtomicRMW()
827 PMV.ShiftAmt, "ValOperand_Shifted"); in expandPartwordAtomicRMW()
831 ValOperand_Shifted, AI->getValOperand(), PMV); in expandPartwordAtomicRMW()
836 OldResult = insertRMWCmpXchgLoop(Builder, PMV.WordType, PMV.AlignedAddr, in expandPartwordAtomicRMW()
837 PMV.AlignedAddrAlignment, MemOpOrder, SSID, in expandPartwordAtomicRMW()
841 OldResult = insertRMWLLSCLoop(Builder, PMV.WordType, PMV.AlignedAddr, in expandPartwordAtomicRMW()
842 PMV.AlignedAddrAlignment, MemOpOrder, in expandPartwordAtomicRMW()
846 Value *FinalOldResult = extractMaskedValue(Builder, OldResult, PMV); in expandPartwordAtomicRMW()
860 PartwordMaskValues PMV = in widenPartwordAtomicRMW() local
865 Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType), in widenPartwordAtomicRMW()
866 PMV.ShiftAmt, "ValOperand_Shifted"); in widenPartwordAtomicRMW()
872 Builder.CreateOr(PMV.Inv_Mask, ValOperand_Shifted, "AndOperand"); in widenPartwordAtomicRMW()
877 Builder.CreateAtomicRMW(Op, PMV.AlignedAddr, NewOperand, in widenPartwordAtomicRMW()
878 PMV.AlignedAddrAlignment, AI->getOrdering()); in widenPartwordAtomicRMW()
880 Value *FinalOldResult = extractMaskedValue(Builder, NewAI, PMV); in widenPartwordAtomicRMW()
942 PartwordMaskValues PMV = in expandPartwordCmpXchg() local
948 Builder.CreateShl(Builder.CreateZExt(NewVal, PMV.WordType), PMV.ShiftAmt); in expandPartwordCmpXchg()
950 Builder.CreateShl(Builder.CreateZExt(Cmp, PMV.WordType), PMV.ShiftAmt); in expandPartwordCmpXchg()
954 LoadInst *InitLoaded = Builder.CreateLoad(PMV.WordType, PMV.AlignedAddr); in expandPartwordCmpXchg()
956 Value *InitLoaded_MaskOut = Builder.CreateAnd(InitLoaded, PMV.Inv_Mask); in expandPartwordCmpXchg()
961 PHINode *Loaded_MaskOut = Builder.CreatePHI(PMV.WordType, 2); in expandPartwordCmpXchg()
968 PMV.AlignedAddr, FullWord_Cmp, FullWord_NewVal, PMV.AlignedAddrAlignment, in expandPartwordCmpXchg()
991 Value *OldVal_MaskOut = Builder.CreateAnd(OldVal, PMV.Inv_Mask); in expandPartwordCmpXchg()
1001 Value *FinalOldVal = extractMaskedValue(Builder, OldVal, PMV); in expandPartwordCmpXchg()
1026 PartwordMaskValues PMV = in expandAtomicRMWToMaskedIntrinsic() local
1039 Builder.CreateCast(CastOp, AI->getValOperand(), PMV.WordType), in expandAtomicRMWToMaskedIntrinsic()
1040 PMV.ShiftAmt, "ValOperand_Shifted"); in expandAtomicRMWToMaskedIntrinsic()
1042 Builder, AI, PMV.AlignedAddr, ValOperand_Shifted, PMV.Mask, PMV.ShiftAmt, in expandAtomicRMWToMaskedIntrinsic()
1044 Value *FinalOldResult = extractMaskedValue(Builder, OldResult, PMV); in expandAtomicRMWToMaskedIntrinsic()
1052 PartwordMaskValues PMV = createMaskInstrs( in expandAtomicCmpXchgToMaskedIntrinsic() local
1057 Builder.CreateZExt(CI->getCompareOperand(), PMV.WordType), PMV.ShiftAmt, in expandAtomicCmpXchgToMaskedIntrinsic()
1060 Builder.CreateZExt(CI->getNewValOperand(), PMV.WordType), PMV.ShiftAmt, in expandAtomicCmpXchgToMaskedIntrinsic()
1063 Builder, CI, PMV.AlignedAddr, CmpVal_Shifted, NewVal_Shifted, PMV.Mask, in expandAtomicCmpXchgToMaskedIntrinsic()
1065 Value *FinalOldVal = extractMaskedValue(Builder, OldVal, PMV); in expandAtomicCmpXchgToMaskedIntrinsic()
1069 CmpVal_Shifted, Builder.CreateAnd(OldVal, PMV.Mask), "Success"); in expandAtomicCmpXchgToMaskedIntrinsic()
1272 PartwordMaskValues PMV = in expandAtomicCmpXchg() local
1280 TLI->emitLoadLinked(Builder, PMV.WordType, PMV.AlignedAddr, MemOpOrder); in expandAtomicCmpXchg()
1282 extractMaskedValue(Builder, UnreleasedLoad, PMV); in expandAtomicCmpXchg()
1297 Builder.CreatePHI(PMV.WordType, 2, "loaded.trystore"); in expandAtomicCmpXchg()
1300 insertMaskedValue(Builder, LoadedTryStore, CI->getNewValOperand(), PMV); in expandAtomicCmpXchg()
1302 PMV.AlignedAddr, MemOpOrder); in expandAtomicCmpXchg()
1313 TLI->emitLoadLinked(Builder, PMV.WordType, PMV.AlignedAddr, MemOpOrder); in expandAtomicCmpXchg()
1314 Value *SecondLoadExtract = extractMaskedValue(Builder, SecondLoad, PMV); in expandAtomicCmpXchg()
1374 Value *Loaded = extractMaskedValue(Builder, LoadedFull, PMV); in expandAtomicCmpXchg()