Lines Matching refs:B
51 Value *buildReduction(IRBuilder<> &B, AtomicRMWInst::BinOp Op, Value *V,
53 Value *buildScan(IRBuilder<> &B, AtomicRMWInst::BinOp Op, Value *V,
55 Value *buildShiftRight(IRBuilder<> &B, Value *V, Value *const Identity) const;
249 static Value *buildNonAtomicBinOp(IRBuilder<> &B, AtomicRMWInst::BinOp Op, in buildNonAtomicBinOp() argument
257 return B.CreateBinOp(Instruction::Add, LHS, RHS); in buildNonAtomicBinOp()
259 return B.CreateBinOp(Instruction::Sub, LHS, RHS); in buildNonAtomicBinOp()
261 return B.CreateBinOp(Instruction::And, LHS, RHS); in buildNonAtomicBinOp()
263 return B.CreateBinOp(Instruction::Or, LHS, RHS); in buildNonAtomicBinOp()
265 return B.CreateBinOp(Instruction::Xor, LHS, RHS); in buildNonAtomicBinOp()
280 Value *Cond = B.CreateICmp(Pred, LHS, RHS); in buildNonAtomicBinOp()
281 return B.CreateSelect(Cond, LHS, RHS); in buildNonAtomicBinOp()
286 Value *AMDGPUAtomicOptimizer::buildReduction(IRBuilder<> &B, in buildReduction() argument
290 Module *M = B.GetInsertBlock()->getModule(); in buildReduction()
297 B, Op, V, in buildReduction()
298 B.CreateCall(UpdateDPP, in buildReduction()
299 {Identity, V, B.getInt32(DPP::ROW_XMASK0 | 1 << Idx), in buildReduction()
300 B.getInt32(0xf), B.getInt32(0xf), B.getFalse()})); in buildReduction()
306 B, Op, V, in buildReduction()
307 B.CreateIntrinsic( in buildReduction()
309 {V, V, B.getInt32(-1), B.getInt32(-1), B.getFalse(), B.getFalse()})); in buildReduction()
317 B, Op, V, B.CreateIntrinsic(Intrinsic::amdgcn_permlane64, {}, V)); in buildReduction()
324 Value *const Lane0 = B.CreateCall(ReadLane, {V, B.getInt32(0)}); in buildReduction()
325 Value *const Lane32 = B.CreateCall(ReadLane, {V, B.getInt32(32)}); in buildReduction()
326 return buildNonAtomicBinOp(B, Op, Lane0, Lane32); in buildReduction()
331 Value *AMDGPUAtomicOptimizer::buildScan(IRBuilder<> &B, AtomicRMWInst::BinOp Op, in buildScan() argument
334 Module *M = B.GetInsertBlock()->getModule(); in buildScan()
340 B, Op, V, in buildScan()
341 B.CreateCall(UpdateDPP, in buildScan()
342 {Identity, V, B.getInt32(DPP::ROW_SHR0 | 1 << Idx), in buildScan()
343 B.getInt32(0xf), B.getInt32(0xf), B.getFalse()})); in buildScan()
348 B, Op, V, in buildScan()
349 B.CreateCall(UpdateDPP, in buildScan()
350 {Identity, V, B.getInt32(DPP::BCAST15), B.getInt32(0xa), in buildScan()
351 B.getInt32(0xf), B.getFalse()})); in buildScan()
353 B, Op, V, in buildScan()
354 B.CreateCall(UpdateDPP, in buildScan()
355 {Identity, V, B.getInt32(DPP::BCAST31), B.getInt32(0xc), in buildScan()
356 B.getInt32(0xf), B.getFalse()})); in buildScan()
364 Value *const PermX = B.CreateIntrinsic( in buildScan()
366 {V, V, B.getInt32(-1), B.getInt32(-1), B.getFalse(), B.getFalse()}); in buildScan()
368 B, Op, V, in buildScan()
369 B.CreateCall(UpdateDPP, in buildScan()
370 {Identity, PermX, B.getInt32(DPP::QUAD_PERM_ID), in buildScan()
371 B.getInt32(0xa), B.getInt32(0xf), B.getFalse()})); in buildScan()
374 Value *const Lane31 = B.CreateIntrinsic(Intrinsic::amdgcn_readlane, {}, in buildScan()
375 {V, B.getInt32(31)}); in buildScan()
377 B, Op, V, in buildScan()
378 B.CreateCall(UpdateDPP, in buildScan()
379 {Identity, Lane31, B.getInt32(DPP::QUAD_PERM_ID), in buildScan()
380 B.getInt32(0xc), B.getInt32(0xf), B.getFalse()})); in buildScan()
388 Value *AMDGPUAtomicOptimizer::buildShiftRight(IRBuilder<> &B, Value *V, in buildShiftRight() argument
391 Module *M = B.GetInsertBlock()->getModule(); in buildShiftRight()
397 V = B.CreateCall(UpdateDPP, in buildShiftRight()
398 {Identity, V, B.getInt32(DPP::WAVE_SHR1), B.getInt32(0xf), in buildShiftRight()
399 B.getInt32(0xf), B.getFalse()}); in buildShiftRight()
409 V = B.CreateCall(UpdateDPP, in buildShiftRight()
410 {Identity, V, B.getInt32(DPP::ROW_SHR0 + 1), in buildShiftRight()
411 B.getInt32(0xf), B.getInt32(0xf), B.getFalse()}); in buildShiftRight()
414 V = B.CreateCall(WriteLane, {B.CreateCall(ReadLane, {Old, B.getInt32(15)}), in buildShiftRight()
415 B.getInt32(16), V}); in buildShiftRight()
419 V = B.CreateCall( in buildShiftRight()
421 {B.CreateCall(ReadLane, {Old, B.getInt32(31)}), B.getInt32(32), V}); in buildShiftRight()
424 V = B.CreateCall( in buildShiftRight()
426 {B.CreateCall(ReadLane, {Old, B.getInt32(47)}), B.getInt32(48), V}); in buildShiftRight()
454 static Value *buildMul(IRBuilder<> &B, Value *LHS, Value *RHS) { in buildMul() argument
456 return (CI && CI->isOne()) ? RHS : B.CreateMul(LHS, RHS); in buildMul()
464 IRBuilder<> B(&I); in optimizeAtomic() local
480 Value *const Cond = B.CreateIntrinsic(Intrinsic::amdgcn_ps_live, {}, {}); in optimizeAtomic()
488 B.SetInsertPoint(&I); in optimizeAtomic()
493 auto *const VecTy = FixedVectorType::get(B.getInt32Ty(), 2); in optimizeAtomic()
501 Type *const WaveTy = B.getIntNTy(ST->getWavefrontSize()); in optimizeAtomic()
503 B.CreateIntrinsic(Intrinsic::amdgcn_ballot, WaveTy, B.getTrue()); in optimizeAtomic()
511 Mbcnt = B.CreateIntrinsic(Intrinsic::amdgcn_mbcnt_lo, {}, in optimizeAtomic()
512 {Ballot, B.getInt32(0)}); in optimizeAtomic()
514 Value *const BitCast = B.CreateBitCast(Ballot, VecTy); in optimizeAtomic()
515 Value *const ExtractLo = B.CreateExtractElement(BitCast, B.getInt32(0)); in optimizeAtomic()
516 Value *const ExtractHi = B.CreateExtractElement(BitCast, B.getInt32(1)); in optimizeAtomic()
517 Mbcnt = B.CreateIntrinsic(Intrinsic::amdgcn_mbcnt_lo, {}, in optimizeAtomic()
518 {ExtractLo, B.getInt32(0)}); in optimizeAtomic()
520 B.CreateIntrinsic(Intrinsic::amdgcn_mbcnt_hi, {}, {ExtractHi, Mbcnt}); in optimizeAtomic()
522 Mbcnt = B.CreateIntCast(Mbcnt, Ty, false); in optimizeAtomic()
524 Value *const Identity = B.getInt(getIdentityValueForAtomicOp(Op, TyBitWidth)); in optimizeAtomic()
536 NewV = B.CreateIntrinsic(Intrinsic::amdgcn_set_inactive, Ty, {V, Identity}); in optimizeAtomic()
544 NewV = buildReduction(B, ScanOp, NewV, Identity); in optimizeAtomic()
546 NewV = buildScan(B, ScanOp, NewV, Identity); in optimizeAtomic()
548 ExclScan = buildShiftRight(B, NewV, Identity); in optimizeAtomic()
553 Value *const LastLaneIdx = B.getInt32(ST->getWavefrontSize() - 1); in optimizeAtomic()
555 NewV = B.CreateIntrinsic(Intrinsic::amdgcn_readlane, {}, in optimizeAtomic()
560 NewV = B.CreateIntrinsic(Intrinsic::amdgcn_strict_wwm, Ty, NewV); in optimizeAtomic()
570 Value *const Ctpop = B.CreateIntCast( in optimizeAtomic()
571 B.CreateUnaryIntrinsic(Intrinsic::ctpop, Ballot), Ty, false); in optimizeAtomic()
572 NewV = buildMul(B, V, Ctpop); in optimizeAtomic()
590 Value *const Ctpop = B.CreateIntCast( in optimizeAtomic()
591 B.CreateUnaryIntrinsic(Intrinsic::ctpop, Ballot), Ty, false); in optimizeAtomic()
592 NewV = buildMul(B, V, B.CreateAnd(Ctpop, 1)); in optimizeAtomic()
600 Value *const Cond = B.CreateICmpEQ(Mbcnt, B.getIntN(TyBitWidth, 0)); in optimizeAtomic()
614 B.SetInsertPoint(SingleLaneTerminator); in optimizeAtomic()
619 B.Insert(NewI); in optimizeAtomic()
624 B.SetInsertPoint(&I); in optimizeAtomic()
628 PHINode *const PHI = B.CreatePHI(Ty, 2); in optimizeAtomic()
638 Value *const ExtractLo = B.CreateTrunc(PHI, B.getInt32Ty()); in optimizeAtomic()
640 B.CreateTrunc(B.CreateLShr(PHI, 32), B.getInt32Ty()); in optimizeAtomic()
642 B.CreateIntrinsic(Intrinsic::amdgcn_readfirstlane, {}, ExtractLo); in optimizeAtomic()
644 B.CreateIntrinsic(Intrinsic::amdgcn_readfirstlane, {}, ExtractHi); in optimizeAtomic()
645 Value *const PartialInsert = B.CreateInsertElement( in optimizeAtomic()
646 UndefValue::get(VecTy), ReadFirstLaneLo, B.getInt32(0)); in optimizeAtomic()
648 B.CreateInsertElement(PartialInsert, ReadFirstLaneHi, B.getInt32(1)); in optimizeAtomic()
649 BroadcastI = B.CreateBitCast(Insert, Ty); in optimizeAtomic()
652 BroadcastI = B.CreateIntrinsic(Intrinsic::amdgcn_readfirstlane, {}, PHI); in optimizeAtomic()
664 B.CreateIntrinsic(Intrinsic::amdgcn_strict_wwm, Ty, ExclScan); in optimizeAtomic()
671 LaneOffset = buildMul(B, V, Mbcnt); in optimizeAtomic()
679 LaneOffset = B.CreateSelect(Cond, Identity, V); in optimizeAtomic()
682 LaneOffset = buildMul(B, V, B.CreateAnd(Mbcnt, 1)); in optimizeAtomic()
686 Value *const Result = buildNonAtomicBinOp(B, Op, BroadcastI, LaneOffset); in optimizeAtomic()
690 B.SetInsertPoint(PixelExitBB->getFirstNonPHI()); in optimizeAtomic()
692 PHINode *const PHI = B.CreatePHI(Ty, 2); in optimizeAtomic()