Lines Matching refs:PMV
647 raw_ostream &operator<<(raw_ostream &O, const PartwordMaskValues &PMV) { in operator <<() argument
657 PrintObj(PMV.WordType); in operator <<()
659 PrintObj(PMV.ValueType); in operator <<()
661 PrintObj(PMV.AlignedAddr); in operator <<()
662 O << " AlignedAddrAlignment: " << PMV.AlignedAddrAlignment.value() << '\n'; in operator <<()
664 PrintObj(PMV.ShiftAmt); in operator <<()
666 PrintObj(PMV.Mask); in operator <<()
668 PrintObj(PMV.Inv_Mask); in operator <<()
695 PartwordMaskValues PMV; in createMaskInstrs() local
702 PMV.ValueType = PMV.IntValueType = ValueType; in createMaskInstrs()
703 if (PMV.ValueType->isFloatingPointTy()) in createMaskInstrs()
704 PMV.IntValueType = in createMaskInstrs()
707 PMV.WordType = MinWordSize > ValueSize ? Type::getIntNTy(Ctx, MinWordSize * 8) in createMaskInstrs()
709 if (PMV.ValueType == PMV.WordType) { in createMaskInstrs()
710 PMV.AlignedAddr = Addr; in createMaskInstrs()
711 PMV.AlignedAddrAlignment = AddrAlign; in createMaskInstrs()
712 PMV.ShiftAmt = ConstantInt::get(PMV.ValueType, 0); in createMaskInstrs()
713 PMV.Mask = ConstantInt::get(PMV.ValueType, ~0, /*isSigned*/ true); in createMaskInstrs()
714 return PMV; in createMaskInstrs()
717 PMV.AlignedAddrAlignment = Align(MinWordSize); in createMaskInstrs()
726 PMV.AlignedAddr = Builder.CreateIntrinsic( in createMaskInstrs()
735 PMV.AlignedAddr = Addr; in createMaskInstrs()
741 PMV.ShiftAmt = Builder.CreateShl(PtrLSB, 3); in createMaskInstrs()
744 PMV.ShiftAmt = Builder.CreateShl( in createMaskInstrs()
748 PMV.ShiftAmt = Builder.CreateTrunc(PMV.ShiftAmt, PMV.WordType, "ShiftAmt"); in createMaskInstrs()
749 PMV.Mask = Builder.CreateShl( in createMaskInstrs()
750 ConstantInt::get(PMV.WordType, (1 << (ValueSize * 8)) - 1), PMV.ShiftAmt, in createMaskInstrs()
753 PMV.Inv_Mask = Builder.CreateNot(PMV.Mask, "Inv_Mask"); in createMaskInstrs()
755 return PMV; in createMaskInstrs()
759 const PartwordMaskValues &PMV) { in extractMaskedValue() argument
760 assert(WideWord->getType() == PMV.WordType && "Widened type mismatch"); in extractMaskedValue()
761 if (PMV.WordType == PMV.ValueType) in extractMaskedValue()
764 Value *Shift = Builder.CreateLShr(WideWord, PMV.ShiftAmt, "shifted"); in extractMaskedValue()
765 Value *Trunc = Builder.CreateTrunc(Shift, PMV.IntValueType, "extracted"); in extractMaskedValue()
766 return Builder.CreateBitCast(Trunc, PMV.ValueType); in extractMaskedValue()
770 Value *Updated, const PartwordMaskValues &PMV) { in insertMaskedValue() argument
771 assert(WideWord->getType() == PMV.WordType && "Widened type mismatch"); in insertMaskedValue()
772 assert(Updated->getType() == PMV.ValueType && "Value type mismatch"); in insertMaskedValue()
773 if (PMV.WordType == PMV.ValueType) in insertMaskedValue()
776 Updated = Builder.CreateBitCast(Updated, PMV.IntValueType); in insertMaskedValue()
778 Value *ZExt = Builder.CreateZExt(Updated, PMV.WordType, "extended"); in insertMaskedValue()
780 Builder.CreateShl(ZExt, PMV.ShiftAmt, "shifted", /*HasNUW*/ true); in insertMaskedValue()
781 Value *And = Builder.CreateAnd(WideWord, PMV.Inv_Mask, "unmasked"); in insertMaskedValue()
792 const PartwordMaskValues &PMV) { in performMaskedAtomicOp() argument
798 Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask); in performMaskedAtomicOp()
811 Value *NewVal_Masked = Builder.CreateAnd(NewVal, PMV.Mask); in performMaskedAtomicOp()
812 Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask); in performMaskedAtomicOp()
829 Value *Loaded_Extract = extractMaskedValue(Builder, Loaded, PMV); in performMaskedAtomicOp()
831 Value *FinalVal = insertMaskedValue(Builder, Loaded, NewVal, PMV); in performMaskedAtomicOp()
853 PartwordMaskValues PMV = in expandPartwordAtomicRMW() local
863 Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType), in expandPartwordAtomicRMW()
864 PMV.ShiftAmt, "ValOperand_Shifted"); in expandPartwordAtomicRMW()
869 ValOperand_Shifted, AI->getValOperand(), PMV); in expandPartwordAtomicRMW()
874 OldResult = insertRMWCmpXchgLoop(Builder, PMV.WordType, PMV.AlignedAddr, in expandPartwordAtomicRMW()
875 PMV.AlignedAddrAlignment, MemOpOrder, SSID, in expandPartwordAtomicRMW()
879 OldResult = insertRMWLLSCLoop(Builder, PMV.WordType, PMV.AlignedAddr, in expandPartwordAtomicRMW()
880 PMV.AlignedAddrAlignment, MemOpOrder, in expandPartwordAtomicRMW()
884 Value *FinalOldResult = extractMaskedValue(Builder, OldResult, PMV); in expandPartwordAtomicRMW()
898 PartwordMaskValues PMV = in widenPartwordAtomicRMW() local
903 Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType), in widenPartwordAtomicRMW()
904 PMV.ShiftAmt, "ValOperand_Shifted"); in widenPartwordAtomicRMW()
910 Builder.CreateOr(PMV.Inv_Mask, ValOperand_Shifted, "AndOperand"); in widenPartwordAtomicRMW()
915 Op, PMV.AlignedAddr, NewOperand, PMV.AlignedAddrAlignment, in widenPartwordAtomicRMW()
919 Value *FinalOldResult = extractMaskedValue(Builder, NewAI, PMV); in widenPartwordAtomicRMW()
981 PartwordMaskValues PMV = in expandPartwordCmpXchg() local
987 Builder.CreateShl(Builder.CreateZExt(NewVal, PMV.WordType), PMV.ShiftAmt); in expandPartwordCmpXchg()
989 Builder.CreateShl(Builder.CreateZExt(Cmp, PMV.WordType), PMV.ShiftAmt); in expandPartwordCmpXchg()
993 LoadInst *InitLoaded = Builder.CreateLoad(PMV.WordType, PMV.AlignedAddr); in expandPartwordCmpXchg()
995 Value *InitLoaded_MaskOut = Builder.CreateAnd(InitLoaded, PMV.Inv_Mask); in expandPartwordCmpXchg()
1000 PHINode *Loaded_MaskOut = Builder.CreatePHI(PMV.WordType, 2); in expandPartwordCmpXchg()
1007 PMV.AlignedAddr, FullWord_Cmp, FullWord_NewVal, PMV.AlignedAddrAlignment, in expandPartwordCmpXchg()
1030 Value *OldVal_MaskOut = Builder.CreateAnd(OldVal, PMV.Inv_Mask); in expandPartwordCmpXchg()
1040 Value *FinalOldVal = extractMaskedValue(Builder, OldVal, PMV); in expandPartwordCmpXchg()
1065 PartwordMaskValues PMV = in expandAtomicRMWToMaskedIntrinsic() local
1078 Builder.CreateCast(CastOp, AI->getValOperand(), PMV.WordType), in expandAtomicRMWToMaskedIntrinsic()
1079 PMV.ShiftAmt, "ValOperand_Shifted"); in expandAtomicRMWToMaskedIntrinsic()
1081 Builder, AI, PMV.AlignedAddr, ValOperand_Shifted, PMV.Mask, PMV.ShiftAmt, in expandAtomicRMWToMaskedIntrinsic()
1083 Value *FinalOldResult = extractMaskedValue(Builder, OldResult, PMV); in expandAtomicRMWToMaskedIntrinsic()
1091 PartwordMaskValues PMV = createMaskInstrs( in expandAtomicCmpXchgToMaskedIntrinsic() local
1096 Builder.CreateZExt(CI->getCompareOperand(), PMV.WordType), PMV.ShiftAmt, in expandAtomicCmpXchgToMaskedIntrinsic()
1099 Builder.CreateZExt(CI->getNewValOperand(), PMV.WordType), PMV.ShiftAmt, in expandAtomicCmpXchgToMaskedIntrinsic()
1102 Builder, CI, PMV.AlignedAddr, CmpVal_Shifted, NewVal_Shifted, PMV.Mask, in expandAtomicCmpXchgToMaskedIntrinsic()
1104 Value *FinalOldVal = extractMaskedValue(Builder, OldVal, PMV); in expandAtomicCmpXchgToMaskedIntrinsic()
1108 CmpVal_Shifted, Builder.CreateAnd(OldVal, PMV.Mask), "Success"); in expandAtomicCmpXchgToMaskedIntrinsic()
1308 PartwordMaskValues PMV = in expandAtomicCmpXchg() local
1316 TLI->emitLoadLinked(Builder, PMV.WordType, PMV.AlignedAddr, MemOpOrder); in expandAtomicCmpXchg()
1318 extractMaskedValue(Builder, UnreleasedLoad, PMV); in expandAtomicCmpXchg()
1333 Builder.CreatePHI(PMV.WordType, 2, "loaded.trystore"); in expandAtomicCmpXchg()
1336 insertMaskedValue(Builder, LoadedTryStore, CI->getNewValOperand(), PMV); in expandAtomicCmpXchg()
1338 PMV.AlignedAddr, MemOpOrder); in expandAtomicCmpXchg()
1349 TLI->emitLoadLinked(Builder, PMV.WordType, PMV.AlignedAddr, MemOpOrder); in expandAtomicCmpXchg()
1350 Value *SecondLoadExtract = extractMaskedValue(Builder, SecondLoad, PMV); in expandAtomicCmpXchg()
1411 Value *Loaded = extractMaskedValue(Builder, LoadedFull, PMV); in expandAtomicCmpXchg()