Lines Matching refs:Bld
535 CGBuilderTy &Bld = CGF.Builder; in getNVPTXWarpID() local
539 return Bld.CreateAShr(RT.getGPUThreadID(CGF), LaneIDBits, "nvptx_warp_id"); in getNVPTXWarpID()
546 CGBuilderTy &Bld = CGF.Builder; in getNVPTXLaneID() local
551 return Bld.CreateAnd(RT.getGPUThreadID(CGF), Bld.getInt32(LaneIDMask), in getNVPTXLaneID()
1050 CGBuilderTy &Bld = CGF.Builder; in emitKernelInit() local
1051 Bld.restoreIP(OMPBuilder.createTargetInit(Bld, IsSPMD, requiresFullRuntime())); in emitKernelInit()
1063 CGBuilderTy &Bld = CGF.Builder; in emitKernelDeinit() local
1064 OMPBuilder.createTargetDeinit(Bld, IsSPMD, requiresFullRuntime()); in emitKernelDeinit()
1392 CGBuilderTy &Bld = CGF.Builder; in emitGenericVarsProlog() local
1424 llvm::Value *CastedVoidPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitGenericVarsProlog()
1443 Size = Bld.CreateNUWAdd( in emitGenericVarsProlog()
1448 Size = Bld.CreateUDiv(Size, AlignVal); in emitGenericVarsProlog()
1449 Size = Bld.CreateNUWMul(Size, AlignVal); in emitGenericVarsProlog()
1532 CGBuilderTy &Bld = CGF.Builder; in emitParallelCall() local
1537 ID = Bld.CreateBitOrPointerCast(WFn, CGM.Int8PtrTy); in emitParallelCall()
1538 llvm::Value *FnPtr = Bld.CreateBitOrPointerCast(OutlinedFn, CGM.Int8PtrTy); in emitParallelCall()
1554 Address Dst = Bld.CreateConstArrayGEP(CapturedVarsAddrs, Idx); in emitParallelCall()
1557 PtrV = Bld.CreateIntToPtr(V, CGF.VoidPtrTy); in emitParallelCall()
1559 PtrV = Bld.CreatePointerBitCastOrAddrSpaceCast(V, CGF.VoidPtrTy); in emitParallelCall()
1568 IfCondVal = Bld.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.Int32Ty, in emitParallelCall()
1576 NumThreadsVal = Bld.CreateZExtOrTrunc(NumThreadsVal, CGF.Int32Ty), in emitParallelCall()
1588 Bld.CreateBitOrPointerCast(CapturedVarsAddrs.getPointer(), in emitParallelCall()
1738 CGBuilderTy &Bld = CGF.Builder; in createRuntimeShuffleFunction() local
1756 Bld.CreateIntCast(RT.getGPUWarpSize(CGF), CGM.Int16Ty, /*isSigned=*/true); in createRuntimeShuffleFunction()
1768 CGBuilderTy &Bld = CGF.Builder; in shuffleAndStore() local
1783 Address PtrEnd = Bld.CreatePointerBitCastOrAddrSpaceCast( in shuffleAndStore()
1784 Bld.CreateConstGEP(SrcAddr, 1), CGF.VoidPtrTy, CGF.Int8Ty); in shuffleAndStore()
1792 Ptr = Bld.CreatePointerBitCastOrAddrSpaceCast(Ptr, IntTy->getPointerTo(), in shuffleAndStore()
1794 ElemPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in shuffleAndStore()
1800 llvm::BasicBlock *CurrentBB = Bld.GetInsertBlock(); in shuffleAndStore()
1803 Bld.CreatePHI(Ptr.getType(), /*NumReservedValues=*/2); in shuffleAndStore()
1806 Bld.CreatePHI(ElemPtr.getType(), /*NumReservedValues=*/2); in shuffleAndStore()
1811 llvm::Value *PtrDiff = Bld.CreatePtrDiff( in shuffleAndStore()
1813 Bld.CreatePointerBitCastOrAddrSpaceCast(Ptr.getPointer(), in shuffleAndStore()
1815 Bld.CreateCondBr(Bld.CreateICmpSGT(PtrDiff, Bld.getInt64(IntSize - 1)), in shuffleAndStore()
1827 Address LocalPtr = Bld.CreateConstGEP(Ptr, 1); in shuffleAndStore()
1828 Address LocalElemPtr = Bld.CreateConstGEP(ElemPtr, 1); in shuffleAndStore()
1843 Ptr = Bld.CreateConstGEP(Ptr, 1); in shuffleAndStore()
1844 ElemPtr = Bld.CreateConstGEP(ElemPtr, 1); in shuffleAndStore()
1880 CGBuilderTy &Bld = CGF.Builder; local
1909 Address SrcElementPtrAddr = Bld.CreateConstArrayGEP(SrcBase, Idx);
1917 DestElementPtrAddr = Bld.CreateConstArrayGEP(DestBase, Idx);
1926 Address SrcElementPtrAddr = Bld.CreateConstArrayGEP(SrcBase, Idx);
1934 DestElementPtrAddr = Bld.CreateConstArrayGEP(DestBase, Idx);
1943 Address SrcElementPtrAddr = Bld.CreateConstArrayGEP(SrcBase, Idx);
1953 Bld.CreateNUWMul(ElementSizeInChars, ScratchpadIndex);
1955 Bld.CreateNUWAdd(DestBase.getPointer(), CurrentOffset);
1957 Bld.CreateIntToPtr(ScratchPadElemAbsolutePtrVal, CGF.VoidPtrTy);
1968 Bld.CreateNUWMul(ElementSizeInChars, ScratchpadIndex);
1970 Bld.CreateNUWAdd(SrcBase.getPointer(), CurrentOffset);
1972 Bld.CreateIntToPtr(ScratchPadElemAbsolutePtrVal, CGF.VoidPtrTy);
1979 DestElementPtrAddr = Bld.CreateConstArrayGEP(DestBase, Idx);
1989 SrcElementAddr = Bld.CreateElementBitCast(
1991 DestElementAddr = Bld.CreateElementBitCast(DestElementAddr,
2036 CGF.EmitStoreOfScalar(Bld.CreatePointerBitCastOrAddrSpaceCast(
2051 ScratchpadBasePtr = Bld.CreateNUWAdd(
2053 Bld.CreateNUWMul(ScratchpadWidth, ElementSizeInChars));
2056 ScratchpadBasePtr = Bld.CreateNUWSub(
2058 ScratchpadBasePtr = Bld.CreateUDiv(
2061 ScratchpadBasePtr = Bld.CreateNUWAdd(
2063 ScratchpadBasePtr = Bld.CreateNUWMul(
2122 CGBuilderTy &Bld = CGF.Builder; in emitInterWarpCopyFunction() local
2158 Bld.CreatePointerBitCastOrAddrSpaceCast( in emitInterWarpCopyFunction()
2199 Bld.CreateICmpULT(Cnt, llvm::ConstantInt::get(CGM.IntTy, NumIters)); in emitInterWarpCopyFunction()
2200 Bld.CreateCondBr(Cmp, BodyBB, ExitBB); in emitInterWarpCopyFunction()
2212 llvm::Value *IsWarpMaster = Bld.CreateIsNull(LaneID, "warp_master"); in emitInterWarpCopyFunction()
2213 Bld.CreateCondBr(IsWarpMaster, ThenBB, ElseBB); in emitInterWarpCopyFunction()
2217 Address ElemPtrPtrAddr = Bld.CreateConstArrayGEP(LocalReduceList, Idx); in emitInterWarpCopyFunction()
2222 ElemPtr = Bld.CreateElementBitCast(ElemPtr, CopyType); in emitInterWarpCopyFunction()
2224 ElemPtr = Bld.CreateGEP(ElemPtr, Cnt); in emitInterWarpCopyFunction()
2228 llvm::Value *MediumPtrVal = Bld.CreateInBoundsGEP( in emitInterWarpCopyFunction()
2234 Bld.CreateBitCast( in emitInterWarpCopyFunction()
2250 Bld.CreateBr(MergeBB); in emitInterWarpCopyFunction()
2253 Bld.CreateBr(MergeBB); in emitInterWarpCopyFunction()
2275 Bld.CreateICmpULT(ThreadID, NumWarpsVal, "is_active_thread"); in emitInterWarpCopyFunction()
2276 Bld.CreateCondBr(IsActiveThread, W0ThenBB, W0ElseBB); in emitInterWarpCopyFunction()
2281 llvm::Value *SrcMediumPtrVal = Bld.CreateInBoundsGEP( in emitInterWarpCopyFunction()
2286 Bld.CreateBitCast( in emitInterWarpCopyFunction()
2293 Address TargetElemPtrPtr = Bld.CreateConstArrayGEP(LocalReduceList, Idx); in emitInterWarpCopyFunction()
2297 TargetElemPtr = Bld.CreateElementBitCast(TargetElemPtr, CopyType); in emitInterWarpCopyFunction()
2299 TargetElemPtr = Bld.CreateGEP(TargetElemPtr, Cnt); in emitInterWarpCopyFunction()
2306 Bld.CreateBr(W0MergeBB); in emitInterWarpCopyFunction()
2309 Bld.CreateBr(W0MergeBB); in emitInterWarpCopyFunction()
2314 Cnt = Bld.CreateNSWAdd(Cnt, llvm::ConstantInt::get(CGM.IntTy, /*V=*/1)); in emitInterWarpCopyFunction()
2429 CGBuilderTy &Bld = CGF.Builder; in emitShuffleAndReduceFunction() local
2434 Bld.CreatePointerBitCastOrAddrSpaceCast( in emitShuffleAndReduceFunction()
2487 llvm::Value *CondAlgo0 = Bld.CreateIsNull(AlgoVerArgVal); in emitShuffleAndReduceFunction()
2489 llvm::Value *Algo1 = Bld.CreateICmpEQ(AlgoVerArgVal, Bld.getInt16(1)); in emitShuffleAndReduceFunction()
2490 llvm::Value *CondAlgo1 = Bld.CreateAnd( in emitShuffleAndReduceFunction()
2491 Algo1, Bld.CreateICmpULT(LaneIDArgVal, RemoteLaneOffsetArgVal)); in emitShuffleAndReduceFunction()
2493 llvm::Value *Algo2 = Bld.CreateICmpEQ(AlgoVerArgVal, Bld.getInt16(2)); in emitShuffleAndReduceFunction()
2494 llvm::Value *CondAlgo2 = Bld.CreateAnd( in emitShuffleAndReduceFunction()
2495 Algo2, Bld.CreateIsNull(Bld.CreateAnd(LaneIDArgVal, Bld.getInt16(1)))); in emitShuffleAndReduceFunction()
2496 CondAlgo2 = Bld.CreateAnd( in emitShuffleAndReduceFunction()
2497 CondAlgo2, Bld.CreateICmpSGT(RemoteLaneOffsetArgVal, Bld.getInt16(0))); in emitShuffleAndReduceFunction()
2499 llvm::Value *CondReduce = Bld.CreateOr(CondAlgo0, CondAlgo1); in emitShuffleAndReduceFunction()
2500 CondReduce = Bld.CreateOr(CondReduce, CondAlgo2); in emitShuffleAndReduceFunction()
2505 Bld.CreateCondBr(CondReduce, ThenBB, ElseBB); in emitShuffleAndReduceFunction()
2509 llvm::Value *LocalReduceListPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitShuffleAndReduceFunction()
2511 llvm::Value *RemoteReduceListPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitShuffleAndReduceFunction()
2515 Bld.CreateBr(MergeBB); in emitShuffleAndReduceFunction()
2518 Bld.CreateBr(MergeBB); in emitShuffleAndReduceFunction()
2524 Algo1 = Bld.CreateICmpEQ(AlgoVerArgVal, Bld.getInt16(1)); in emitShuffleAndReduceFunction()
2525 llvm::Value *CondCopy = Bld.CreateAnd( in emitShuffleAndReduceFunction()
2526 Algo1, Bld.CreateICmpUGE(LaneIDArgVal, RemoteLaneOffsetArgVal)); in emitShuffleAndReduceFunction()
2531 Bld.CreateCondBr(CondCopy, CpyThenBB, CpyElseBB); in emitShuffleAndReduceFunction()
2536 Bld.CreateBr(CpyMergeBB); in emitShuffleAndReduceFunction()
2539 Bld.CreateBr(CpyMergeBB); in emitShuffleAndReduceFunction()
2585 CGBuilderTy &Bld = CGF.Builder; in emitListToGlobalCopyFunction() local
2591 Bld.CreatePointerBitCastOrAddrSpaceCast( in emitListToGlobalCopyFunction()
2599 llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitListToGlobalCopyFunction()
2609 Address ElemPtrPtrAddr = Bld.CreateConstArrayGEP(LocalReduceList, Idx); in emitListToGlobalCopyFunction()
2614 ElemPtrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitListToGlobalCopyFunction()
2624 llvm::Value *BufferPtr = Bld.CreateInBoundsGEP(GlobAddr.getElementType(), in emitListToGlobalCopyFunction()
2698 CGBuilderTy &Bld = CGF.Builder; in emitListToGlobalReduceFunction() local
2704 llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitListToGlobalReduceFunction()
2726 llvm::Value *BufferPtr = Bld.CreateInBoundsGEP( in emitListToGlobalReduceFunction()
2794 CGBuilderTy &Bld = CGF.Builder; in emitGlobalToListCopyFunction() local
2800 Bld.CreatePointerBitCastOrAddrSpaceCast( in emitGlobalToListCopyFunction()
2808 llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitGlobalToListCopyFunction()
2819 Address ElemPtrPtrAddr = Bld.CreateConstArrayGEP(LocalReduceList, Idx); in emitGlobalToListCopyFunction()
2824 ElemPtrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitGlobalToListCopyFunction()
2834 llvm::Value *BufferPtr = Bld.CreateInBoundsGEP(GlobAddr.getElementType(), in emitGlobalToListCopyFunction()
2908 CGBuilderTy &Bld = CGF.Builder; in emitGlobalToListReduceFunction() local
2914 llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitGlobalToListReduceFunction()
2936 llvm::Value *BufferPtr = Bld.CreateInBoundsGEP( in emitGlobalToListReduceFunction()
3542 CGBuilderTy &Bld = CGF.Builder; in createParallelDataSharingWrapper() local
3567 Address Src = Bld.CreateConstInBoundsGEP(SharedArgListAddress, Idx); in createParallelDataSharingWrapper()
3568 Address TypedAddress = Bld.CreatePointerBitCastOrAddrSpaceCast( in createParallelDataSharingWrapper()
3577 Src = Bld.CreateConstInBoundsGEP(SharedArgListAddress, Idx); in createParallelDataSharingWrapper()
3578 TypedAddress = Bld.CreatePointerBitCastOrAddrSpaceCast( in createParallelDataSharingWrapper()
3592 Address Src = Bld.CreateConstInBoundsGEP(SharedArgListAddress, I + Idx); in createParallelDataSharingWrapper()
3593 Address TypedAddress = Bld.CreatePointerBitCastOrAddrSpaceCast( in createParallelDataSharingWrapper()
4003 CGBuilderTy &Bld = CGF.Builder; in getGPUNumThreads() local
4012 return Bld.CreateCall(F, llvm::None, "nvptx_num_threads"); in getGPUNumThreads()