Lines Matching refs:B
77 Value *buildReduction(IRBuilder<> &B, AtomicRMWInst::BinOp Op, Value *V,
79 Value *buildScan(IRBuilder<> &B, AtomicRMWInst::BinOp Op, Value *V,
81 Value *buildShiftRight(IRBuilder<> &B, Value *V, Value *const Identity) const;
84 buildScanIteratively(IRBuilder<> &B, AtomicRMWInst::BinOp Op,
347 static Value *buildNonAtomicBinOp(IRBuilder<> &B, AtomicRMWInst::BinOp Op, in buildNonAtomicBinOp() argument
355 return B.CreateBinOp(Instruction::Add, LHS, RHS); in buildNonAtomicBinOp()
357 return B.CreateFAdd(LHS, RHS); in buildNonAtomicBinOp()
359 return B.CreateBinOp(Instruction::Sub, LHS, RHS); in buildNonAtomicBinOp()
361 return B.CreateFSub(LHS, RHS); in buildNonAtomicBinOp()
363 return B.CreateBinOp(Instruction::And, LHS, RHS); in buildNonAtomicBinOp()
365 return B.CreateBinOp(Instruction::Or, LHS, RHS); in buildNonAtomicBinOp()
367 return B.CreateBinOp(Instruction::Xor, LHS, RHS); in buildNonAtomicBinOp()
382 return B.CreateMaxNum(LHS, RHS); in buildNonAtomicBinOp()
384 return B.CreateMinNum(LHS, RHS); in buildNonAtomicBinOp()
386 Value *Cond = B.CreateICmp(Pred, LHS, RHS); in buildNonAtomicBinOp()
387 return B.CreateSelect(Cond, LHS, RHS); in buildNonAtomicBinOp()
392 Value *AMDGPUAtomicOptimizerImpl::buildReduction(IRBuilder<> &B, in buildReduction() argument
397 Type *IntNTy = B.getIntNTy(AtomicTy->getPrimitiveSizeInBits()); in buildReduction()
398 Module *M = B.GetInsertBlock()->getModule(); in buildReduction()
405 B, Op, V, in buildReduction()
406 B.CreateCall(UpdateDPP, in buildReduction()
407 {Identity, V, B.getInt32(DPP::ROW_XMASK0 | 1 << Idx), in buildReduction()
408 B.getInt32(0xf), B.getInt32(0xf), B.getFalse()})); in buildReduction()
413 V = B.CreateBitCast(V, IntNTy); in buildReduction()
414 Value *Permlanex16Call = B.CreateIntrinsic( in buildReduction()
416 {V, V, B.getInt32(-1), B.getInt32(-1), B.getFalse(), B.getFalse()}); in buildReduction()
417 V = buildNonAtomicBinOp(B, Op, B.CreateBitCast(V, AtomicTy), in buildReduction()
418 B.CreateBitCast(Permlanex16Call, AtomicTy)); in buildReduction()
425 V = B.CreateBitCast(V, IntNTy); in buildReduction()
427 B.CreateIntrinsic(Intrinsic::amdgcn_permlane64, {}, V); in buildReduction()
428 return buildNonAtomicBinOp(B, Op, B.CreateBitCast(V, AtomicTy), in buildReduction()
429 B.CreateBitCast(Permlane64Call, AtomicTy)); in buildReduction()
436 V = B.CreateBitCast(V, IntNTy); in buildReduction()
437 Value *Lane0 = B.CreateCall(ReadLane, {V, B.getInt32(0)}); in buildReduction()
438 Value *Lane32 = B.CreateCall(ReadLane, {V, B.getInt32(32)}); in buildReduction()
439 return buildNonAtomicBinOp(B, Op, B.CreateBitCast(Lane0, AtomicTy), in buildReduction()
440 B.CreateBitCast(Lane32, AtomicTy)); in buildReduction()
445 Value *AMDGPUAtomicOptimizerImpl::buildScan(IRBuilder<> &B, in buildScan() argument
449 Type *IntNTy = B.getIntNTy(AtomicTy->getPrimitiveSizeInBits()); in buildScan()
451 Module *M = B.GetInsertBlock()->getModule(); in buildScan()
457 B, Op, V, in buildScan()
458 B.CreateCall(UpdateDPP, in buildScan()
459 {Identity, V, B.getInt32(DPP::ROW_SHR0 | 1 << Idx), in buildScan()
460 B.getInt32(0xf), B.getInt32(0xf), B.getFalse()})); in buildScan()
465 B, Op, V, in buildScan()
466 B.CreateCall(UpdateDPP, in buildScan()
467 {Identity, V, B.getInt32(DPP::BCAST15), B.getInt32(0xa), in buildScan()
468 B.getInt32(0xf), B.getFalse()})); in buildScan()
470 B, Op, V, in buildScan()
471 B.CreateCall(UpdateDPP, in buildScan()
472 {Identity, V, B.getInt32(DPP::BCAST31), B.getInt32(0xc), in buildScan()
473 B.getInt32(0xf), B.getFalse()})); in buildScan()
481 V = B.CreateBitCast(V, IntNTy); in buildScan()
482 Value *PermX = B.CreateIntrinsic( in buildScan()
484 {V, V, B.getInt32(-1), B.getInt32(-1), B.getFalse(), B.getFalse()}); in buildScan()
487 B.CreateCall(UpdateDPP, {Identity, B.CreateBitCast(PermX, AtomicTy), in buildScan()
488 B.getInt32(DPP::QUAD_PERM_ID), B.getInt32(0xa), in buildScan()
489 B.getInt32(0xf), B.getFalse()}); in buildScan()
490 V = buildNonAtomicBinOp(B, Op, B.CreateBitCast(V, AtomicTy), UpdateDPPCall); in buildScan()
494 V = B.CreateBitCast(V, IntNTy); in buildScan()
495 Value *const Lane31 = B.CreateIntrinsic(Intrinsic::amdgcn_readlane, {}, in buildScan()
496 {V, B.getInt32(31)}); in buildScan()
498 Value *UpdateDPPCall = B.CreateCall( in buildScan()
499 UpdateDPP, {Identity, Lane31, B.getInt32(DPP::QUAD_PERM_ID), in buildScan()
500 B.getInt32(0xc), B.getInt32(0xf), B.getFalse()}); in buildScan()
502 V = buildNonAtomicBinOp(B, Op, B.CreateBitCast(V, AtomicTy), in buildScan()
511 Value *AMDGPUAtomicOptimizerImpl::buildShiftRight(IRBuilder<> &B, Value *V, in buildShiftRight() argument
514 Type *IntNTy = B.getIntNTy(AtomicTy->getPrimitiveSizeInBits()); in buildShiftRight()
516 Module *M = B.GetInsertBlock()->getModule(); in buildShiftRight()
521 V = B.CreateCall(UpdateDPP, in buildShiftRight()
522 {Identity, V, B.getInt32(DPP::WAVE_SHR1), B.getInt32(0xf), in buildShiftRight()
523 B.getInt32(0xf), B.getFalse()}); in buildShiftRight()
533 V = B.CreateCall(UpdateDPP, in buildShiftRight()
534 {Identity, V, B.getInt32(DPP::ROW_SHR0 + 1), in buildShiftRight()
535 B.getInt32(0xf), B.getInt32(0xf), B.getFalse()}); in buildShiftRight()
538 V = B.CreateCall( in buildShiftRight()
540 {B.CreateCall(ReadLane, {B.CreateBitCast(Old, IntNTy), B.getInt32(15)}), in buildShiftRight()
541 B.getInt32(16), B.CreateBitCast(V, IntNTy)}); in buildShiftRight()
542 V = B.CreateBitCast(V, AtomicTy); in buildShiftRight()
545 V = B.CreateBitCast(V, IntNTy); in buildShiftRight()
546 V = B.CreateCall(WriteLane, in buildShiftRight()
547 {B.CreateCall(ReadLane, {B.CreateBitCast(Old, IntNTy), in buildShiftRight()
548 B.getInt32(31)}), in buildShiftRight()
549 B.getInt32(32), V}); in buildShiftRight()
552 V = B.CreateCall( in buildShiftRight()
554 {B.CreateCall(ReadLane, {Old, B.getInt32(47)}), B.getInt32(48), V}); in buildShiftRight()
555 V = B.CreateBitCast(V, AtomicTy); in buildShiftRight()
568 IRBuilder<> &B, AtomicRMWInst::BinOp Op, Value *const Identity, Value *V, in buildScanIteratively() argument
571 auto *WaveTy = B.getIntNTy(ST->getWavefrontSize()); in buildScanIteratively()
576 B.CreateIntrinsic(Intrinsic::amdgcn_ballot, WaveTy, B.getTrue()); in buildScanIteratively()
579 B.SetInsertPoint(ComputeLoop); in buildScanIteratively()
581 auto *Accumulator = B.CreatePHI(Ty, 2, "Accumulator"); in buildScanIteratively()
585 OldValuePhi = B.CreatePHI(Ty, 2, "OldValuePhi"); in buildScanIteratively()
588 auto *ActiveBits = B.CreatePHI(WaveTy, 2, "ActiveBits"); in buildScanIteratively()
593 B.CreateIntrinsic(Intrinsic::cttz, WaveTy, {ActiveBits, B.getTrue()}); in buildScanIteratively()
595 Type *IntNTy = B.getIntNTy(Ty->getPrimitiveSizeInBits()); in buildScanIteratively()
596 auto *LaneIdxInt = B.CreateTrunc(FF1, IntNTy); in buildScanIteratively()
599 V = B.CreateBitCast(V, IntNTy); in buildScanIteratively()
601 B.CreateIntrinsic(Intrinsic::amdgcn_readlane, {}, {V, LaneIdxInt}); in buildScanIteratively()
602 LaneValue = B.CreateBitCast(LaneValue, Ty); in buildScanIteratively()
609 B.CreateIntrinsic(Intrinsic::amdgcn_writelane, {}, in buildScanIteratively()
610 {B.CreateBitCast(Accumulator, IntNTy), LaneIdxInt, in buildScanIteratively()
611 B.CreateBitCast(OldValuePhi, IntNTy)}); in buildScanIteratively()
612 OldValue = B.CreateBitCast(OldValue, Ty); in buildScanIteratively()
617 auto *NewAccumulator = buildNonAtomicBinOp(B, Op, Accumulator, LaneValue); in buildScanIteratively()
622 auto *Mask = B.CreateShl(ConstantInt::get(WaveTy, 1), FF1); in buildScanIteratively()
624 auto *InverseMask = B.CreateXor(Mask, ConstantInt::get(WaveTy, -1)); in buildScanIteratively()
625 auto *NewActiveBits = B.CreateAnd(ActiveBits, InverseMask); in buildScanIteratively()
629 auto *IsEnd = B.CreateICmpEQ(NewActiveBits, ConstantInt::get(WaveTy, 0)); in buildScanIteratively()
630 B.CreateCondBr(IsEnd, ComputeEnd, ComputeLoop); in buildScanIteratively()
632 B.SetInsertPoint(ComputeEnd); in buildScanIteratively()
668 static Value *buildMul(IRBuilder<> &B, Value *LHS, Value *RHS) { in buildMul() argument
670 return (CI && CI->isOne()) ? RHS : B.CreateMul(LHS, RHS); in buildMul()
678 IRBuilder<> B(&I); in optimizeAtomic() local
681 B.setIsFPConstrained(I.getFunction()->hasFnAttribute(Attribute::StrictFP)); in optimizeAtomic()
698 Value *const Cond = B.CreateIntrinsic(Intrinsic::amdgcn_ps_live, {}, {}); in optimizeAtomic()
706 B.SetInsertPoint(&I); in optimizeAtomic()
710 Type *Int32Ty = B.getInt32Ty(); in optimizeAtomic()
711 Type *IntNTy = B.getIntNTy(Ty->getPrimitiveSizeInBits()); in optimizeAtomic()
722 Type *const WaveTy = B.getIntNTy(ST->getWavefrontSize()); in optimizeAtomic()
724 B.CreateIntrinsic(Intrinsic::amdgcn_ballot, WaveTy, B.getTrue()); in optimizeAtomic()
732 Mbcnt = B.CreateIntrinsic(Intrinsic::amdgcn_mbcnt_lo, {}, in optimizeAtomic()
733 {Ballot, B.getInt32(0)}); in optimizeAtomic()
735 Value *const ExtractLo = B.CreateTrunc(Ballot, Int32Ty); in optimizeAtomic()
736 Value *const ExtractHi = B.CreateTrunc(B.CreateLShr(Ballot, 32), Int32Ty); in optimizeAtomic()
737 Mbcnt = B.CreateIntrinsic(Intrinsic::amdgcn_mbcnt_lo, {}, in optimizeAtomic()
738 {ExtractLo, B.getInt32(0)}); in optimizeAtomic()
740 B.CreateIntrinsic(Intrinsic::amdgcn_mbcnt_hi, {}, {ExtractHi, Mbcnt}); in optimizeAtomic()
769 V = B.CreateBitCast(V, IntNTy); in optimizeAtomic()
770 Identity = B.CreateBitCast(Identity, IntNTy); in optimizeAtomic()
771 NewV = B.CreateIntrinsic(Intrinsic::amdgcn_set_inactive, IntNTy, in optimizeAtomic()
773 NewV = B.CreateBitCast(NewV, Ty); in optimizeAtomic()
774 V = B.CreateBitCast(V, Ty); in optimizeAtomic()
775 Identity = B.CreateBitCast(Identity, Ty); in optimizeAtomic()
780 NewV = buildReduction(B, ScanOp, NewV, Identity); in optimizeAtomic()
782 NewV = buildScan(B, ScanOp, NewV, Identity); in optimizeAtomic()
784 ExclScan = buildShiftRight(B, NewV, Identity); in optimizeAtomic()
788 Value *const LastLaneIdx = B.getInt32(ST->getWavefrontSize() - 1); in optimizeAtomic()
790 NewV = B.CreateBitCast(NewV, IntNTy); in optimizeAtomic()
791 NewV = B.CreateIntrinsic(Intrinsic::amdgcn_readlane, {}, in optimizeAtomic()
793 NewV = B.CreateBitCast(NewV, Ty); in optimizeAtomic()
796 NewV = B.CreateIntrinsic(Intrinsic::amdgcn_strict_wwm, Ty, NewV); in optimizeAtomic()
801 std::tie(ExclScan, NewV) = buildScanIteratively(B, ScanOp, Identity, V, I, in optimizeAtomic()
815 Value *const Ctpop = B.CreateIntCast( in optimizeAtomic()
816 B.CreateUnaryIntrinsic(Intrinsic::ctpop, Ballot), Ty, false); in optimizeAtomic()
817 NewV = buildMul(B, V, Ctpop); in optimizeAtomic()
822 Value *const Ctpop = B.CreateIntCast( in optimizeAtomic()
823 B.CreateUnaryIntrinsic(Intrinsic::ctpop, Ballot), Int32Ty, false); in optimizeAtomic()
824 Value *const CtpopFP = B.CreateUIToFP(Ctpop, Ty); in optimizeAtomic()
825 NewV = B.CreateFMul(V, CtpopFP); in optimizeAtomic()
844 Value *const Ctpop = B.CreateIntCast( in optimizeAtomic()
845 B.CreateUnaryIntrinsic(Intrinsic::ctpop, Ballot), Ty, false); in optimizeAtomic()
846 NewV = buildMul(B, V, B.CreateAnd(Ctpop, 1)); in optimizeAtomic()
854 Value *const Cond = B.CreateICmpEQ(Mbcnt, B.getInt32(0)); in optimizeAtomic()
879 B.SetInsertPoint(ComputeEnd); in optimizeAtomic()
881 B.Insert(Terminator); in optimizeAtomic()
885 B.SetInsertPoint(EntryBB); in optimizeAtomic()
886 B.CreateBr(ComputeLoop); in optimizeAtomic()
899 B.SetInsertPoint(SingleLaneTerminator); in optimizeAtomic()
904 B.Insert(NewI); in optimizeAtomic()
909 B.SetInsertPoint(&I); in optimizeAtomic()
913 PHINode *const PHI = B.CreatePHI(Ty, 2); in optimizeAtomic()
923 Value *const ExtractLo = B.CreateTrunc(PHI, Int32Ty); in optimizeAtomic()
924 Value *const ExtractHi = B.CreateTrunc(B.CreateLShr(PHI, 32), Int32Ty); in optimizeAtomic()
926 B.CreateIntrinsic(Intrinsic::amdgcn_readfirstlane, {}, ExtractLo); in optimizeAtomic()
928 B.CreateIntrinsic(Intrinsic::amdgcn_readfirstlane, {}, ExtractHi); in optimizeAtomic()
929 Value *const PartialInsert = B.CreateInsertElement( in optimizeAtomic()
930 PoisonValue::get(VecTy), ReadFirstLaneLo, B.getInt32(0)); in optimizeAtomic()
932 B.CreateInsertElement(PartialInsert, ReadFirstLaneHi, B.getInt32(1)); in optimizeAtomic()
933 BroadcastI = B.CreateBitCast(Insert, Ty); in optimizeAtomic()
935 Value *CastedPhi = B.CreateBitCast(PHI, IntNTy); in optimizeAtomic()
937 B.CreateIntrinsic(Intrinsic::amdgcn_readfirstlane, {}, CastedPhi); in optimizeAtomic()
938 BroadcastI = B.CreateBitCast(BroadcastI, Ty); in optimizeAtomic()
952 B.CreateIntrinsic(Intrinsic::amdgcn_strict_wwm, Ty, ExclScan); in optimizeAtomic()
959 Mbcnt = isAtomicFloatingPointTy ? B.CreateUIToFP(Mbcnt, Ty) in optimizeAtomic()
960 : B.CreateIntCast(Mbcnt, Ty, false); in optimizeAtomic()
966 LaneOffset = buildMul(B, V, Mbcnt); in optimizeAtomic()
976 LaneOffset = B.CreateSelect(Cond, Identity, V); in optimizeAtomic()
979 LaneOffset = buildMul(B, V, B.CreateAnd(Mbcnt, 1)); in optimizeAtomic()
983 LaneOffset = B.CreateFMul(V, Mbcnt); in optimizeAtomic()
988 Value *const Result = buildNonAtomicBinOp(B, Op, BroadcastI, LaneOffset); in optimizeAtomic()
992 B.SetInsertPoint(PixelExitBB, PixelExitBB->getFirstNonPHIIt()); in optimizeAtomic()
994 PHINode *const PHI = B.CreatePHI(Ty, 2); in optimizeAtomic()