Lines Matching refs:Bld
508 CGBuilderTy &Bld = CGF.Builder; in getNVPTXWarpID() local
512 return Bld.CreateAShr(RT.getGPUThreadID(CGF), LaneIDBits, "nvptx_warp_id"); in getNVPTXWarpID()
519 CGBuilderTy &Bld = CGF.Builder; in getNVPTXLaneID() local
525 return Bld.CreateAnd(RT.getGPUThreadID(CGF), Bld.getInt32(LaneIDMask), in getNVPTXLaneID()
773 CGBuilderTy &Bld = CGF.Builder; in emitKernelInit() local
774 Bld.restoreIP(OMPBuilder.createTargetInit( in emitKernelInit()
775 Bld, IsSPMD, MinThreadsVal, MaxThreadsVal, MinTeamsVal, MaxTeamsVal)); in emitKernelInit()
810 CGBuilderTy &Bld = CGF.Builder; in emitKernelDeinit() local
811 OMPBuilder.createTargetDeinit(Bld, ReductionDataSize, in emitKernelDeinit()
1065 CGBuilderTy &Bld = CGF.Builder; in emitGenericVarsProlog() local
1097 llvm::Value *CastedVoidPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitGenericVarsProlog()
1138 CGBuilderTy &Bld = CGF.Builder; in getKmpcAllocShared() local
1143 Size = Bld.CreateNUWAdd( in getKmpcAllocShared()
1147 Size = Bld.CreateUDiv(Size, AlignVal); in getKmpcAllocShared()
1148 Size = Bld.CreateNUWMul(Size, AlignVal); in getKmpcAllocShared()
1236 CGBuilderTy &Bld = CGF.Builder; in emitParallelCall() local
1241 ID = Bld.CreateBitOrPointerCast(WFn, CGM.Int8PtrTy); in emitParallelCall()
1242 llvm::Value *FnPtr = Bld.CreateBitOrPointerCast(OutlinedFn, CGM.Int8PtrTy); in emitParallelCall()
1258 Address Dst = Bld.CreateConstArrayGEP(CapturedVarsAddrs, Idx); in emitParallelCall()
1261 PtrV = Bld.CreateIntToPtr(V, CGF.VoidPtrTy); in emitParallelCall()
1263 PtrV = Bld.CreatePointerBitCastOrAddrSpaceCast(V, CGF.VoidPtrTy); in emitParallelCall()
1272 IfCondVal = Bld.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.Int32Ty, in emitParallelCall()
1280 NumThreadsVal = Bld.CreateZExtOrTrunc(NumThreadsVal, CGF.Int32Ty), in emitParallelCall()
1292 Bld.CreateBitOrPointerCast(CapturedVarsAddrs.getPointer(), in emitParallelCall()
1440 CGBuilderTy &Bld = CGF.Builder; in createRuntimeShuffleFunction() local
1458 Bld.CreateIntCast(RT.getGPUWarpSize(CGF), CGM.Int16Ty, /*isSigned=*/true); in createRuntimeShuffleFunction()
1470 CGBuilderTy &Bld = CGF.Builder; in shuffleAndStore() local
1485 Address PtrEnd = Bld.CreatePointerBitCastOrAddrSpaceCast( in shuffleAndStore()
1486 Bld.CreateConstGEP(SrcAddr, 1), CGF.VoidPtrTy, CGF.Int8Ty); in shuffleAndStore()
1494 Ptr = Bld.CreatePointerBitCastOrAddrSpaceCast(Ptr, IntTy->getPointerTo(), in shuffleAndStore()
1496 ElemPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in shuffleAndStore()
1502 llvm::BasicBlock *CurrentBB = Bld.GetInsertBlock(); in shuffleAndStore()
1505 Bld.CreatePHI(Ptr.getType(), /*NumReservedValues=*/2); in shuffleAndStore()
1508 Bld.CreatePHI(ElemPtr.getType(), /*NumReservedValues=*/2); in shuffleAndStore()
1513 llvm::Value *PtrDiff = Bld.CreatePtrDiff( in shuffleAndStore()
1515 Bld.CreatePointerBitCastOrAddrSpaceCast(Ptr.getPointer(), in shuffleAndStore()
1517 Bld.CreateCondBr(Bld.CreateICmpSGT(PtrDiff, Bld.getInt64(IntSize - 1)), in shuffleAndStore()
1529 Address LocalPtr = Bld.CreateConstGEP(Ptr, 1); in shuffleAndStore()
1530 Address LocalElemPtr = Bld.CreateConstGEP(ElemPtr, 1); in shuffleAndStore()
1545 Ptr = Bld.CreateConstGEP(Ptr, 1); in shuffleAndStore()
1546 ElemPtr = Bld.CreateConstGEP(ElemPtr, 1); in shuffleAndStore()
1577 CGBuilderTy &Bld = CGF.Builder; local
1599 Address SrcElementPtrAddr = Bld.CreateConstArrayGEP(SrcBase, Idx);
1606 DestElementPtrAddr = Bld.CreateConstArrayGEP(DestBase, Idx);
1615 Address SrcElementPtrAddr = Bld.CreateConstArrayGEP(SrcBase, Idx);
1622 DestElementPtrAddr = Bld.CreateConstArrayGEP(DestBase, Idx);
1679 CGF.EmitStoreOfScalar(Bld.CreatePointerBitCastOrAddrSpaceCast(
1732 CGBuilderTy &Bld = CGF.Builder; in emitInterWarpCopyFunction() local
1768 Bld.CreatePointerBitCastOrAddrSpaceCast( in emitInterWarpCopyFunction()
1809 Bld.CreateICmpULT(Cnt, llvm::ConstantInt::get(CGM.IntTy, NumIters)); in emitInterWarpCopyFunction()
1810 Bld.CreateCondBr(Cmp, BodyBB, ExitBB); in emitInterWarpCopyFunction()
1822 llvm::Value *IsWarpMaster = Bld.CreateIsNull(LaneID, "warp_master"); in emitInterWarpCopyFunction()
1823 Bld.CreateCondBr(IsWarpMaster, ThenBB, ElseBB); in emitInterWarpCopyFunction()
1827 Address ElemPtrPtrAddr = Bld.CreateConstArrayGEP(LocalReduceList, Idx); in emitInterWarpCopyFunction()
1833 ElemPtr = Bld.CreateGEP(ElemPtr, Cnt); in emitInterWarpCopyFunction()
1837 llvm::Value *MediumPtrVal = Bld.CreateInBoundsGEP( in emitInterWarpCopyFunction()
1854 Bld.CreateBr(MergeBB); in emitInterWarpCopyFunction()
1857 Bld.CreateBr(MergeBB); in emitInterWarpCopyFunction()
1879 Bld.CreateICmpULT(ThreadID, NumWarpsVal, "is_active_thread"); in emitInterWarpCopyFunction()
1880 Bld.CreateCondBr(IsActiveThread, W0ThenBB, W0ElseBB); in emitInterWarpCopyFunction()
1885 llvm::Value *SrcMediumPtrVal = Bld.CreateInBoundsGEP( in emitInterWarpCopyFunction()
1892 Address TargetElemPtrPtr = Bld.CreateConstArrayGEP(LocalReduceList, Idx); in emitInterWarpCopyFunction()
1897 TargetElemPtr = Bld.CreateGEP(TargetElemPtr, Cnt); in emitInterWarpCopyFunction()
1904 Bld.CreateBr(W0MergeBB); in emitInterWarpCopyFunction()
1907 Bld.CreateBr(W0MergeBB); in emitInterWarpCopyFunction()
1912 Cnt = Bld.CreateNSWAdd(Cnt, llvm::ConstantInt::get(CGM.IntTy, /*V=*/1)); in emitInterWarpCopyFunction()
2027 CGBuilderTy &Bld = CGF.Builder; in emitShuffleAndReduceFunction() local
2032 Bld.CreatePointerBitCastOrAddrSpaceCast( in emitShuffleAndReduceFunction()
2085 llvm::Value *CondAlgo0 = Bld.CreateIsNull(AlgoVerArgVal); in emitShuffleAndReduceFunction()
2087 llvm::Value *Algo1 = Bld.CreateICmpEQ(AlgoVerArgVal, Bld.getInt16(1)); in emitShuffleAndReduceFunction()
2088 llvm::Value *CondAlgo1 = Bld.CreateAnd( in emitShuffleAndReduceFunction()
2089 Algo1, Bld.CreateICmpULT(LaneIDArgVal, RemoteLaneOffsetArgVal)); in emitShuffleAndReduceFunction()
2091 llvm::Value *Algo2 = Bld.CreateICmpEQ(AlgoVerArgVal, Bld.getInt16(2)); in emitShuffleAndReduceFunction()
2092 llvm::Value *CondAlgo2 = Bld.CreateAnd( in emitShuffleAndReduceFunction()
2093 Algo2, Bld.CreateIsNull(Bld.CreateAnd(LaneIDArgVal, Bld.getInt16(1)))); in emitShuffleAndReduceFunction()
2094 CondAlgo2 = Bld.CreateAnd( in emitShuffleAndReduceFunction()
2095 CondAlgo2, Bld.CreateICmpSGT(RemoteLaneOffsetArgVal, Bld.getInt16(0))); in emitShuffleAndReduceFunction()
2097 llvm::Value *CondReduce = Bld.CreateOr(CondAlgo0, CondAlgo1); in emitShuffleAndReduceFunction()
2098 CondReduce = Bld.CreateOr(CondReduce, CondAlgo2); in emitShuffleAndReduceFunction()
2103 Bld.CreateCondBr(CondReduce, ThenBB, ElseBB); in emitShuffleAndReduceFunction()
2107 llvm::Value *LocalReduceListPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitShuffleAndReduceFunction()
2109 llvm::Value *RemoteReduceListPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitShuffleAndReduceFunction()
2113 Bld.CreateBr(MergeBB); in emitShuffleAndReduceFunction()
2116 Bld.CreateBr(MergeBB); in emitShuffleAndReduceFunction()
2122 Algo1 = Bld.CreateICmpEQ(AlgoVerArgVal, Bld.getInt16(1)); in emitShuffleAndReduceFunction()
2123 llvm::Value *CondCopy = Bld.CreateAnd( in emitShuffleAndReduceFunction()
2124 Algo1, Bld.CreateICmpUGE(LaneIDArgVal, RemoteLaneOffsetArgVal)); in emitShuffleAndReduceFunction()
2129 Bld.CreateCondBr(CondCopy, CpyThenBB, CpyElseBB); in emitShuffleAndReduceFunction()
2134 Bld.CreateBr(CpyMergeBB); in emitShuffleAndReduceFunction()
2137 Bld.CreateBr(CpyMergeBB); in emitShuffleAndReduceFunction()
2183 CGBuilderTy &Bld = CGF.Builder; in emitListToGlobalCopyFunction() local
2189 Bld.CreatePointerBitCastOrAddrSpaceCast( in emitListToGlobalCopyFunction()
2197 llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitListToGlobalCopyFunction()
2206 Address ElemPtrPtrAddr = Bld.CreateConstArrayGEP(LocalReduceList, Idx); in emitListToGlobalCopyFunction()
2211 ElemPtrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitListToGlobalCopyFunction()
2219 Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs); in emitListToGlobalCopyFunction()
2295 CGBuilderTy &Bld = CGF.Builder; in emitListToGlobalReduceFunction() local
2301 llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitListToGlobalReduceFunction()
2320 Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs); in emitListToGlobalReduceFunction()
2389 CGBuilderTy &Bld = CGF.Builder; in emitGlobalToListCopyFunction() local
2395 Bld.CreatePointerBitCastOrAddrSpaceCast( in emitGlobalToListCopyFunction()
2403 llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitGlobalToListCopyFunction()
2413 Address ElemPtrPtrAddr = Bld.CreateConstArrayGEP(LocalReduceList, Idx); in emitGlobalToListCopyFunction()
2418 ElemPtrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitGlobalToListCopyFunction()
2426 Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs); in emitGlobalToListCopyFunction()
2502 CGBuilderTy &Bld = CGF.Builder; in emitGlobalToListReduceFunction() local
2508 llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( in emitGlobalToListReduceFunction()
2527 Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs); in emitGlobalToListReduceFunction()
3112 CGBuilderTy &Bld = CGF.Builder; in createParallelDataSharingWrapper() local
3137 Address Src = Bld.CreateConstInBoundsGEP(SharedArgListAddress, Idx); in createParallelDataSharingWrapper()
3138 Address TypedAddress = Bld.CreatePointerBitCastOrAddrSpaceCast( in createParallelDataSharingWrapper()
3147 Src = Bld.CreateConstInBoundsGEP(SharedArgListAddress, Idx); in createParallelDataSharingWrapper()
3148 TypedAddress = Bld.CreatePointerBitCastOrAddrSpaceCast( in createParallelDataSharingWrapper()
3162 Address Src = Bld.CreateConstInBoundsGEP(SharedArgListAddress, I + Idx); in createParallelDataSharingWrapper()
3163 Address TypedAddress = Bld.CreatePointerBitCastOrAddrSpaceCast( in createParallelDataSharingWrapper()
3544 CGBuilderTy &Bld = CGF.Builder; in getGPUNumThreads() local
3553 return Bld.CreateCall(F, std::nullopt, "nvptx_num_threads"); in getGPUNumThreads()