Lines Matching refs:CGF
47 void Enter(CodeGenFunction &CGF) override { in Enter() argument
48 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs); in Enter()
50 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes); in Enter()
51 auto *ThenBlock = CGF.createBasicBlock("omp_if.then"); in Enter()
52 ContBlock = CGF.createBasicBlock("omp_if.end"); in Enter()
54 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock); in Enter()
55 CGF.EmitBlock(ThenBlock); in Enter()
58 void Done(CodeGenFunction &CGF) { in Done() argument
60 CGF.EmitBranch(ContBlock); in Done()
61 CGF.EmitBlock(ContBlock, true); in Done()
63 void Exit(CodeGenFunction &CGF) override { in Exit() argument
64 CGF.EmitRuntimeCall(ExitCallee, ExitArgs); in Exit()
185 CodeGenFunction &CGF; member in __anone34f085d0111::CheckVarsEscapingDeclContext
206 if (auto *CSI = CGF.CapturedStmtInfo) { in markAsEscaped()
232 if ((!CGF.CapturedStmtInfo || in markAsEscaped()
233 (IsForCombinedParallelRegion && CGF.CapturedStmtInfo)) && in markAsEscaped()
311 unsigned WarpSize = CGF.getTarget().getGridValue().GV_Warp_Size; in buildRecordForGlobalizedVars()
317 CGF.getContext(), EscapedDeclsForParallel, EscapedDeclsForTeams, in buildRecordForGlobalizedVars()
322 CheckVarsEscapingDeclContext(CodeGenFunction &CGF, in CheckVarsEscapingDeclContext() argument
324 : CGF(CGF), EscapedDecls(TeamsReductions.begin(), TeamsReductions.end()) { in CheckVarsEscapingDeclContext()
507 static llvm::Value *getNVPTXWarpID(CodeGenFunction &CGF) { in getNVPTXWarpID() argument
508 CGBuilderTy &Bld = CGF.Builder; in getNVPTXWarpID()
510 llvm::Log2_32(CGF.getTarget().getGridValue().GV_Warp_Size); in getNVPTXWarpID()
511 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in getNVPTXWarpID()
512 return Bld.CreateAShr(RT.getGPUThreadID(CGF), LaneIDBits, "nvptx_warp_id"); in getNVPTXWarpID()
518 static llvm::Value *getNVPTXLaneID(CodeGenFunction &CGF) { in getNVPTXLaneID() argument
519 CGBuilderTy &Bld = CGF.Builder; in getNVPTXLaneID()
521 llvm::Log2_32(CGF.getTarget().getGridValue().GV_Warp_Size); in getNVPTXLaneID()
524 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in getNVPTXLaneID()
525 return Bld.CreateAnd(RT.getGPUThreadID(CGF), Bld.getInt32(LaneIDMask), in getNVPTXLaneID()
746 void Enter(CodeGenFunction &CGF) override { in emitNonSPMDKernel() argument
747 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitNonSPMDKernel()
748 RT.emitKernelInit(D, CGF, EST, /* IsSPMD */ false); in emitNonSPMDKernel()
750 RT.setLocThreadIdInsertPt(CGF, /*AtCurrentPoint=*/true); in emitNonSPMDKernel()
752 void Exit(CodeGenFunction &CGF) override { in emitNonSPMDKernel() argument
753 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitNonSPMDKernel()
754 RT.clearLocThreadIdInsertPt(CGF); in emitNonSPMDKernel()
755 RT.emitKernelDeinit(CGF, EST, /* IsSPMD */ false); in emitNonSPMDKernel()
766 CodeGenFunction &CGF, in emitKernelInit() argument
770 computeMinAndMaxThreadsAndTeams(D, CGF, MinThreadsVal, MaxThreadsVal, in emitKernelInit()
773 CGBuilderTy &Bld = CGF.Builder; in emitKernelInit()
777 emitGenericVarsProlog(CGF, EST.Loc); in emitKernelInit()
780 void CGOpenMPRuntimeGPU::emitKernelDeinit(CodeGenFunction &CGF, in emitKernelDeinit() argument
784 emitGenericVarsEpilog(CGF); in emitKernelDeinit()
810 CGBuilderTy &Bld = CGF.Builder; in emitKernelDeinit()
841 void Enter(CodeGenFunction &CGF) override { in emitSPMDKernel() argument
846 RT.emitKernelInit(D, CGF, EST, /* IsSPMD */ true); in emitSPMDKernel()
848 RT.setLocThreadIdInsertPt(CGF, /*AtCurrentPoint=*/true); in emitSPMDKernel()
850 void Exit(CodeGenFunction &CGF) override { in emitSPMDKernel() argument
855 RT.clearLocThreadIdInsertPt(CGF); in emitSPMDKernel()
856 RT.emitKernelDeinit(CGF, EST, /* IsSPMD */ true); in emitSPMDKernel()
916 void CGOpenMPRuntimeGPU::emitProcBindClause(CodeGenFunction &CGF, in emitProcBindClause() argument
922 void CGOpenMPRuntimeGPU::emitNumThreadsClause(CodeGenFunction &CGF, in emitNumThreadsClause() argument
928 void CGOpenMPRuntimeGPU::emitNumTeamsClause(CodeGenFunction &CGF, in emitNumTeamsClause() argument
934 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitParallelOutlinedFunction() argument
942 CGF, D, ThreadIDVar, InnermostKind, CodeGen)); in emitParallelOutlinedFunction()
992 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitTeamsOutlinedFunction() argument
1032 void Enter(CodeGenFunction &CGF) override { in emitTeamsOutlinedFunction() argument
1034 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitTeamsOutlinedFunction()
1036 auto I = Rt.FunctionGlobalizedDecls.try_emplace(CGF.CurFn).first; in emitTeamsOutlinedFunction()
1046 Rt.emitGenericVarsProlog(CGF, Loc); in emitTeamsOutlinedFunction()
1048 void Exit(CodeGenFunction &CGF) override { in emitTeamsOutlinedFunction() argument
1049 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()) in emitTeamsOutlinedFunction()
1050 .emitGenericVarsEpilog(CGF); in emitTeamsOutlinedFunction()
1055 CGF, D, ThreadIDVar, InnermostKind, CodeGen); in emitTeamsOutlinedFunction()
1060 void CGOpenMPRuntimeGPU::emitGenericVarsProlog(CodeGenFunction &CGF, in emitGenericVarsProlog() argument
1065 CGBuilderTy &Bld = CGF.Builder; in emitGenericVarsProlog()
1067 const auto I = FunctionGlobalizedDecls.find(CGF.CurFn); in emitGenericVarsProlog()
1080 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType()); in emitGenericVarsProlog()
1081 ParValue = CGF.EmitLoadOfScalar(ParLVal, Loc); in emitGenericVarsProlog()
1085 llvm::Value *AllocArgs[] = {CGF.getTypeSize(VD->getType())}; in emitGenericVarsProlog()
1087 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitGenericVarsProlog()
1096 llvm::PointerType *VarPtrTy = CGF.ConvertTypeForMem(VarTy)->getPointerTo(); in emitGenericVarsProlog()
1099 LValue VarAddr = CGF.MakeNaturalAlignAddrLValue(CastedVoidPtr, VarTy); in emitGenericVarsProlog()
1100 Rec.second.PrivateAddr = VarAddr.getAddress(CGF); in emitGenericVarsProlog()
1105 CGF.EmitStoreOfScalar(ParValue, VarAddr); in emitGenericVarsProlog()
1106 I->getSecond().MappedParams->setVarAddr(CGF, VD, VarAddr.getAddress(CGF)); in emitGenericVarsProlog()
1108 if (auto *DI = CGF.getDebugInfo()) in emitGenericVarsProlog()
1115 getKmpcAllocShared(CGF, VD); in emitGenericVarsProlog()
1117 LValue Base = CGF.MakeAddrLValue(AddrSizePair.first, VD->getType(), in emitGenericVarsProlog()
1120 I->getSecond().MappedParams->setVarAddr(CGF, VD, Base.getAddress(CGF)); in emitGenericVarsProlog()
1122 I->getSecond().MappedParams->apply(CGF); in emitGenericVarsProlog()
1125 bool CGOpenMPRuntimeGPU::isDelayedVariableLengthDecl(CodeGenFunction &CGF, in isDelayedVariableLengthDecl() argument
1127 const auto I = FunctionGlobalizedDecls.find(CGF.CurFn); in isDelayedVariableLengthDecl()
1136 CGOpenMPRuntimeGPU::getKmpcAllocShared(CodeGenFunction &CGF, in getKmpcAllocShared() argument
1138 CGBuilderTy &Bld = CGF.Builder; in getKmpcAllocShared()
1141 llvm::Value *Size = CGF.getTypeSize(VD->getType()); in getKmpcAllocShared()
1144 Size, llvm::ConstantInt::get(CGF.SizeTy, Align.getQuantity() - 1)); in getKmpcAllocShared()
1146 llvm::ConstantInt::get(CGF.SizeTy, Align.getQuantity()); in getKmpcAllocShared()
1153 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in getKmpcAllocShared()
1163 CodeGenFunction &CGF, in getKmpcFreeShared() argument
1166 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in getKmpcFreeShared()
1171 void CGOpenMPRuntimeGPU::emitGenericVarsEpilog(CodeGenFunction &CGF) { in emitGenericVarsEpilog() argument
1175 const auto I = FunctionGlobalizedDecls.find(CGF.CurFn); in emitGenericVarsEpilog()
1181 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitGenericVarsEpilog()
1188 I->getSecond().MappedParams->restore(CGF); in emitGenericVarsEpilog()
1191 CGF.getTypeSize(VD->getType())}; in emitGenericVarsEpilog()
1192 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitGenericVarsEpilog()
1199 void CGOpenMPRuntimeGPU::emitTeamsCall(CodeGenFunction &CGF, in emitTeamsCall() argument
1204 if (!CGF.HaveInsertPoint()) in emitTeamsCall()
1209 Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, in emitTeamsCall()
1211 CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddr); in emitTeamsCall()
1218 OutlinedFnArgs.push_back(emitThreadIDAddress(CGF, Loc).getPointer()); in emitTeamsCall()
1221 emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs); in emitTeamsCall()
1224 void CGOpenMPRuntimeGPU::emitParallelCall(CodeGenFunction &CGF, in emitParallelCall() argument
1230 if (!CGF.HaveInsertPoint()) in emitParallelCall()
1234 NumThreads](CodeGenFunction &CGF, in emitParallelCall()
1236 CGBuilderTy &Bld = CGF.Builder; in emitParallelCall()
1247 CodeGenFunction::OMPPrivateScope PrivateArgScope(CGF); in emitParallelCall()
1249 Address CapturedVarsAddrs = CGF.CreateDefaultAlignTempAlloca( in emitParallelCall()
1255 ASTContext &Ctx = CGF.getContext(); in emitParallelCall()
1261 PtrV = Bld.CreateIntToPtr(V, CGF.VoidPtrTy); in emitParallelCall()
1263 PtrV = Bld.CreatePointerBitCastOrAddrSpaceCast(V, CGF.VoidPtrTy); in emitParallelCall()
1264 CGF.EmitStoreOfScalar(PtrV, Dst, /*Volatile=*/false, in emitParallelCall()
1272 IfCondVal = Bld.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.Int32Ty, in emitParallelCall()
1275 IfCondVal = llvm::ConstantInt::get(CGF.Int32Ty, 1); in emitParallelCall()
1278 NumThreadsVal = llvm::ConstantInt::get(CGF.Int32Ty, -1); in emitParallelCall()
1280 NumThreadsVal = Bld.CreateZExtOrTrunc(NumThreadsVal, CGF.Int32Ty), in emitParallelCall()
1283 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitParallelCall()
1286 getThreadID(CGF, Loc), in emitParallelCall()
1289 llvm::ConstantInt::get(CGF.Int32Ty, -1), in emitParallelCall()
1293 CGF.VoidPtrPtrTy), in emitParallelCall()
1295 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitParallelCall()
1301 RCG(CGF); in emitParallelCall()
1304 void CGOpenMPRuntimeGPU::syncCTAThreads(CodeGenFunction &CGF) { in syncCTAThreads() argument
1306 if (!CGF.HaveInsertPoint()) in syncCTAThreads()
1313 llvm::ConstantInt::get(CGF.Int32Ty, /*V=*/0, /*isSigned=*/true)}; in syncCTAThreads()
1314 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in syncCTAThreads()
1319 void CGOpenMPRuntimeGPU::emitBarrierCall(CodeGenFunction &CGF, in emitBarrierCall() argument
1324 if (!CGF.HaveInsertPoint()) in emitBarrierCall()
1328 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags), in emitBarrierCall()
1329 getThreadID(CGF, Loc)}; in emitBarrierCall()
1331 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitBarrierCall()
1337 CodeGenFunction &CGF, StringRef CriticalName, in emitCriticalRegion() argument
1340 llvm::BasicBlock *LoopBB = CGF.createBasicBlock("omp.critical.loop"); in emitCriticalRegion()
1341 llvm::BasicBlock *TestBB = CGF.createBasicBlock("omp.critical.test"); in emitCriticalRegion()
1342 llvm::BasicBlock *SyncBB = CGF.createBasicBlock("omp.critical.sync"); in emitCriticalRegion()
1343 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.critical.body"); in emitCriticalRegion()
1344 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.critical.exit"); in emitCriticalRegion()
1346 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitCriticalRegion()
1349 llvm::Value *Mask = CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitCriticalRegion()
1352 llvm::Value *ThreadID = RT.getGPUThreadID(CGF); in emitCriticalRegion()
1355 llvm::Value *TeamWidth = RT.getGPUNumThreads(CGF); in emitCriticalRegion()
1359 CGF.getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/0); in emitCriticalRegion()
1360 Address Counter = CGF.CreateMemTemp(Int32Ty, "critical_counter"); in emitCriticalRegion()
1361 LValue CounterLVal = CGF.MakeAddrLValue(Counter, Int32Ty); in emitCriticalRegion()
1362 CGF.EmitStoreOfScalar(llvm::Constant::getNullValue(CGM.Int32Ty), CounterLVal, in emitCriticalRegion()
1366 CGF.EmitBlock(LoopBB); in emitCriticalRegion()
1367 llvm::Value *CounterVal = CGF.EmitLoadOfScalar(CounterLVal, Loc); in emitCriticalRegion()
1368 llvm::Value *CmpLoopBound = CGF.Builder.CreateICmpSLT(CounterVal, TeamWidth); in emitCriticalRegion()
1369 CGF.Builder.CreateCondBr(CmpLoopBound, TestBB, ExitBB); in emitCriticalRegion()
1373 CGF.EmitBlock(TestBB); in emitCriticalRegion()
1374 CounterVal = CGF.EmitLoadOfScalar(CounterLVal, Loc); in emitCriticalRegion()
1376 CGF.Builder.CreateICmpEQ(ThreadID, CounterVal); in emitCriticalRegion()
1377 CGF.Builder.CreateCondBr(CmpThreadToCounter, BodyBB, SyncBB); in emitCriticalRegion()
1380 CGF.EmitBlock(BodyBB); in emitCriticalRegion()
1383 CGOpenMPRuntime::emitCriticalRegion(CGF, CriticalName, CriticalOpGen, Loc, in emitCriticalRegion()
1390 CGF.EmitBlock(SyncBB); in emitCriticalRegion()
1392 (void)CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitCriticalRegion()
1397 CGF.Builder.CreateNSWAdd(CounterVal, CGF.Builder.getInt32(1)); in emitCriticalRegion()
1398 CGF.EmitStoreOfScalar(IncCounterVal, CounterLVal); in emitCriticalRegion()
1399 CGF.EmitBranch(LoopBB); in emitCriticalRegion()
1402 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in emitCriticalRegion()
1406 static llvm::Value *castValueToType(CodeGenFunction &CGF, llvm::Value *Val, in castValueToType() argument
1409 assert(!CGF.getContext().getTypeSizeInChars(CastTy).isZero() && in castValueToType()
1411 assert(!CGF.getContext().getTypeSizeInChars(ValTy).isZero() && in castValueToType()
1413 llvm::Type *LLVMCastTy = CGF.ConvertTypeForMem(CastTy); in castValueToType()
1416 if (CGF.getContext().getTypeSizeInChars(ValTy) == in castValueToType()
1417 CGF.getContext().getTypeSizeInChars(CastTy)) in castValueToType()
1418 return CGF.Builder.CreateBitCast(Val, LLVMCastTy); in castValueToType()
1420 return CGF.Builder.CreateIntCast(Val, LLVMCastTy, in castValueToType()
1422 Address CastItem = CGF.CreateMemTemp(CastTy); in castValueToType()
1424 CGF.EmitStoreOfScalar(Val, ValCastItem, /*Volatile=*/false, ValTy, in castValueToType()
1427 return CGF.EmitLoadOfScalar(CastItem, /*Volatile=*/false, CastTy, Loc, in castValueToType()
1434 static llvm::Value *createRuntimeShuffleFunction(CodeGenFunction &CGF, in createRuntimeShuffleFunction() argument
1439 CodeGenModule &CGM = CGF.CGM; in createRuntimeShuffleFunction()
1440 CGBuilderTy &Bld = CGF.Builder; in createRuntimeShuffleFunction()
1445 CharUnits Size = CGF.getContext().getTypeSizeInChars(ElemType); in createRuntimeShuffleFunction()
1454 QualType CastTy = CGF.getContext().getIntTypeForBitwidth( in createRuntimeShuffleFunction()
1456 llvm::Value *ElemCast = castValueToType(CGF, Elem, ElemType, CastTy, Loc); in createRuntimeShuffleFunction()
1458 Bld.CreateIntCast(RT.getGPUWarpSize(CGF), CGM.Int16Ty, /*isSigned=*/true); in createRuntimeShuffleFunction()
1460 llvm::Value *ShuffledVal = CGF.EmitRuntimeCall( in createRuntimeShuffleFunction()
1464 return castValueToType(CGF, ShuffledVal, CastTy, ElemType, Loc); in createRuntimeShuffleFunction()
1467 static void shuffleAndStore(CodeGenFunction &CGF, Address SrcAddr, in shuffleAndStore() argument
1470 CGBuilderTy &Bld = CGF.Builder; in shuffleAndStore()
1472 CharUnits Size = CGF.getContext().getTypeSizeInChars(ElemType); in shuffleAndStore()
1486 Bld.CreateConstGEP(SrcAddr, 1), CGF.VoidPtrTy, CGF.Int8Ty); in shuffleAndStore()
1490 QualType IntType = CGF.getContext().getIntTypeForBitwidth( in shuffleAndStore()
1491 CGF.getContext().toBits(CharUnits::fromQuantity(IntSize)), in shuffleAndStore()
1493 llvm::Type *IntTy = CGF.ConvertTypeForMem(IntType); in shuffleAndStore()
1499 llvm::BasicBlock *PreCondBB = CGF.createBasicBlock(".shuffle.pre_cond"); in shuffleAndStore()
1500 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".shuffle.then"); in shuffleAndStore()
1501 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".shuffle.exit"); in shuffleAndStore()
1503 CGF.EmitBlock(PreCondBB); in shuffleAndStore()
1514 CGF.Int8Ty, PtrEnd.getPointer(), in shuffleAndStore()
1516 CGF.VoidPtrTy)); in shuffleAndStore()
1519 CGF.EmitBlock(ThenBB); in shuffleAndStore()
1521 CGF, in shuffleAndStore()
1522 CGF.EmitLoadOfScalar(Ptr, /*Volatile=*/false, IntType, Loc, in shuffleAndStore()
1526 CGF.EmitStoreOfScalar(Res, ElemPtr, /*Volatile=*/false, IntType, in shuffleAndStore()
1533 CGF.EmitBranch(PreCondBB); in shuffleAndStore()
1534 CGF.EmitBlock(ExitBB); in shuffleAndStore()
1537 CGF, in shuffleAndStore()
1538 CGF.EmitLoadOfScalar(Ptr, /*Volatile=*/false, IntType, Loc, in shuffleAndStore()
1542 CGF.EmitStoreOfScalar(Res, ElemPtr, /*Volatile=*/false, IntType, in shuffleAndStore()
1571 CopyAction Action, CodeGenFunction &CGF, QualType ReductionArrayTy, in emitReductionListCopy() argument
1575 CodeGenModule &CGM = CGF.CGM;
1577 CGBuilderTy &Bld = CGF.Builder;
1594 llvm::Type *PrivateLlvmPtrType = CGF.ConvertType(PrivatePtrType);
1600 SrcElementAddr = CGF.EmitLoadOfPointer(
1608 CGF.CreateMemTemp(Private->getType(), ".omp.reduction.element");
1616 SrcElementAddr = CGF.EmitLoadOfPointer(
1623 DestElementAddr = CGF.EmitLoadOfPointer(
1633 CGF.ConvertTypeForMem(Private->getType()));
1640 shuffleAndStore(CGF, SrcElementAddr, DestElementAddr, Private->getType(),
1643 switch (CGF.getEvaluationKind(Private->getType())) {
1645 llvm::Value *Elem = CGF.EmitLoadOfScalar(
1650 CGF.EmitStoreOfScalar(
1656 CodeGenFunction::ComplexPairTy Elem = CGF.EmitLoadOfComplex(
1657 CGF.MakeAddrLValue(SrcElementAddr, Private->getType()),
1659 CGF.EmitStoreOfComplex(
1660 Elem, CGF.MakeAddrLValue(DestElementAddr, Private->getType()),
1665 CGF.EmitAggregateCopy(
1666 CGF.MakeAddrLValue(DestElementAddr, Private->getType()),
1667 CGF.MakeAddrLValue(SrcElementAddr, Private->getType()),
1679 CGF.EmitStoreOfScalar(Bld.CreatePointerBitCastOrAddrSpaceCast(
1680 DestElementAddr.getPointer(), CGF.VoidPtrTy),
1729 CodeGenFunction CGF(CGM); in emitInterWarpCopyFunction() local
1730 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitInterWarpCopyFunction()
1732 CGBuilderTy &Bld = CGF.Builder; in emitInterWarpCopyFunction()
1745 unsigned WarpSize = CGF.getTarget().getGridValue().GV_Warp_Size; in emitInterWarpCopyFunction()
1757 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitInterWarpCopyFunction()
1759 llvm::Value *ThreadID = RT.getGPUThreadID(CGF); in emitInterWarpCopyFunction()
1761 llvm::Value *LaneID = getNVPTXLaneID(CGF); in emitInterWarpCopyFunction()
1763 llvm::Value *WarpID = getNVPTXWarpID(CGF); in emitInterWarpCopyFunction()
1765 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitInterWarpCopyFunction()
1766 llvm::Type *ElemTy = CGF.ConvertTypeForMem(ReductionArrayTy); in emitInterWarpCopyFunction()
1769 CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
1773 ElemTy, CGF.getPointerAlign()); in emitInterWarpCopyFunction()
1791 llvm::Type *CopyType = CGF.ConvertTypeForMem(CType); in emitInterWarpCopyFunction()
1798 CntAddr = CGF.CreateMemTemp(C.IntTy, ".cnt.addr"); in emitInterWarpCopyFunction()
1799 CGF.EmitStoreOfScalar(llvm::Constant::getNullValue(CGM.IntTy), CntAddr, in emitInterWarpCopyFunction()
1801 PrecondBB = CGF.createBasicBlock("precond"); in emitInterWarpCopyFunction()
1802 ExitBB = CGF.createBasicBlock("exit"); in emitInterWarpCopyFunction()
1803 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("body"); in emitInterWarpCopyFunction()
1805 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitInterWarpCopyFunction()
1806 CGF.EmitBlock(PrecondBB); in emitInterWarpCopyFunction()
1807 Cnt = CGF.EmitLoadOfScalar(CntAddr, /*Volatile=*/false, C.IntTy, Loc); in emitInterWarpCopyFunction()
1811 CGF.EmitBlock(BodyBB); in emitInterWarpCopyFunction()
1814 CGM.getOpenMPRuntime().emitBarrierCall(CGF, Loc, OMPD_unknown, in emitInterWarpCopyFunction()
1817 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("then"); in emitInterWarpCopyFunction()
1818 llvm::BasicBlock *ElseBB = CGF.createBasicBlock("else"); in emitInterWarpCopyFunction()
1819 llvm::BasicBlock *MergeBB = CGF.createBasicBlock("ifcont"); in emitInterWarpCopyFunction()
1824 CGF.EmitBlock(ThenBB); in emitInterWarpCopyFunction()
1828 llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
1846 llvm::Value *Elem = CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
1850 CGF.EmitStoreOfScalar(Elem, MediumPtr, /*Volatile=*/true, CType, in emitInterWarpCopyFunction()
1856 CGF.EmitBlock(ElseBB); in emitInterWarpCopyFunction()
1859 CGF.EmitBlock(MergeBB); in emitInterWarpCopyFunction()
1862 CGM.getOpenMPRuntime().emitBarrierCall(CGF, Loc, OMPD_unknown, in emitInterWarpCopyFunction()
1869 llvm::BasicBlock *W0ThenBB = CGF.createBasicBlock("then"); in emitInterWarpCopyFunction()
1870 llvm::BasicBlock *W0ElseBB = CGF.createBasicBlock("else"); in emitInterWarpCopyFunction()
1871 llvm::BasicBlock *W0MergeBB = CGF.createBasicBlock("ifcont"); in emitInterWarpCopyFunction()
1873 Address AddrNumWarpsArg = CGF.GetAddrOfLocalVar(&NumWarpsArg); in emitInterWarpCopyFunction()
1874 llvm::Value *NumWarpsVal = CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
1882 CGF.EmitBlock(W0ThenBB); in emitInterWarpCopyFunction()
1893 llvm::Value *TargetElemPtrVal = CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
1901 CGF.EmitLoadOfScalar(SrcMediumPtr, /*Volatile=*/true, CType, Loc); in emitInterWarpCopyFunction()
1902 CGF.EmitStoreOfScalar(SrcMediumValue, TargetElemPtr, /*Volatile=*/false, in emitInterWarpCopyFunction()
1906 CGF.EmitBlock(W0ElseBB); in emitInterWarpCopyFunction()
1909 CGF.EmitBlock(W0MergeBB); in emitInterWarpCopyFunction()
1913 CGF.EmitStoreOfScalar(Cnt, CntAddr, /*Volatile=*/false, C.IntTy); in emitInterWarpCopyFunction()
1914 CGF.EmitBranch(PrecondBB); in emitInterWarpCopyFunction()
1915 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitInterWarpCopyFunction()
1916 CGF.EmitBlock(ExitBB); in emitInterWarpCopyFunction()
1923 CGF.FinishFunction(); in emitInterWarpCopyFunction()
2024 CodeGenFunction CGF(CGM); in emitShuffleAndReduceFunction() local
2025 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitShuffleAndReduceFunction()
2027 CGBuilderTy &Bld = CGF.Builder; in emitShuffleAndReduceFunction()
2029 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitShuffleAndReduceFunction()
2030 llvm::Type *ElemTy = CGF.ConvertTypeForMem(ReductionArrayTy); in emitShuffleAndReduceFunction()
2033 CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false, in emitShuffleAndReduceFunction()
2036 ElemTy, CGF.getPointerAlign()); in emitShuffleAndReduceFunction()
2038 Address AddrLaneIDArg = CGF.GetAddrOfLocalVar(&LaneIDArg); in emitShuffleAndReduceFunction()
2039 llvm::Value *LaneIDArgVal = CGF.EmitLoadOfScalar( in emitShuffleAndReduceFunction()
2042 Address AddrRemoteLaneOffsetArg = CGF.GetAddrOfLocalVar(&RemoteLaneOffsetArg); in emitShuffleAndReduceFunction()
2043 llvm::Value *RemoteLaneOffsetArgVal = CGF.EmitLoadOfScalar( in emitShuffleAndReduceFunction()
2046 Address AddrAlgoVerArg = CGF.GetAddrOfLocalVar(&AlgoVerArg); in emitShuffleAndReduceFunction()
2047 llvm::Value *AlgoVerArgVal = CGF.EmitLoadOfScalar( in emitShuffleAndReduceFunction()
2053 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.remote_reduce_list"); in emitShuffleAndReduceFunction()
2058 emitReductionListCopy(RemoteLaneToThread, CGF, ReductionArrayTy, Privates, in emitShuffleAndReduceFunction()
2100 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("then"); in emitShuffleAndReduceFunction()
2101 llvm::BasicBlock *ElseBB = CGF.createBasicBlock("else"); in emitShuffleAndReduceFunction()
2102 llvm::BasicBlock *MergeBB = CGF.createBasicBlock("ifcont"); in emitShuffleAndReduceFunction()
2105 CGF.EmitBlock(ThenBB); in emitShuffleAndReduceFunction()
2108 LocalReduceList.getPointer(), CGF.VoidPtrTy); in emitShuffleAndReduceFunction()
2110 RemoteReduceList.getPointer(), CGF.VoidPtrTy); in emitShuffleAndReduceFunction()
2112 CGF, Loc, ReduceFn, {LocalReduceListPtr, RemoteReduceListPtr}); in emitShuffleAndReduceFunction()
2115 CGF.EmitBlock(ElseBB); in emitShuffleAndReduceFunction()
2118 CGF.EmitBlock(MergeBB); in emitShuffleAndReduceFunction()
2126 llvm::BasicBlock *CpyThenBB = CGF.createBasicBlock("then"); in emitShuffleAndReduceFunction()
2127 llvm::BasicBlock *CpyElseBB = CGF.createBasicBlock("else"); in emitShuffleAndReduceFunction()
2128 llvm::BasicBlock *CpyMergeBB = CGF.createBasicBlock("ifcont"); in emitShuffleAndReduceFunction()
2131 CGF.EmitBlock(CpyThenBB); in emitShuffleAndReduceFunction()
2132 emitReductionListCopy(ThreadCopy, CGF, ReductionArrayTy, Privates, in emitShuffleAndReduceFunction()
2136 CGF.EmitBlock(CpyElseBB); in emitShuffleAndReduceFunction()
2139 CGF.EmitBlock(CpyMergeBB); in emitShuffleAndReduceFunction()
2141 CGF.FinishFunction(); in emitShuffleAndReduceFunction()
2180 CodeGenFunction CGF(CGM); in emitListToGlobalCopyFunction() local
2181 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitListToGlobalCopyFunction()
2183 CGBuilderTy &Bld = CGF.Builder; in emitListToGlobalCopyFunction()
2185 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitListToGlobalCopyFunction()
2186 Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg); in emitListToGlobalCopyFunction()
2187 llvm::Type *ElemTy = CGF.ConvertTypeForMem(ReductionArrayTy); in emitListToGlobalCopyFunction()
2190 CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false, in emitListToGlobalCopyFunction()
2193 ElemTy, CGF.getPointerAlign()); in emitListToGlobalCopyFunction()
2198 CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc), in emitListToGlobalCopyFunction()
2200 llvm::Value *Idxs[] = {CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), in emitListToGlobalCopyFunction()
2207 llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar( in emitListToGlobalCopyFunction()
2210 ElemTy = CGF.ConvertTypeForMem(Private->getType()); in emitListToGlobalCopyFunction()
2220 LValue GlobLVal = CGF.EmitLValueForField( in emitListToGlobalCopyFunction()
2221 CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD); in emitListToGlobalCopyFunction()
2222 Address GlobAddr = GlobLVal.getAddress(CGF); in emitListToGlobalCopyFunction()
2224 CGF.ConvertTypeForMem(Private->getType()), in emitListToGlobalCopyFunction()
2226 switch (CGF.getEvaluationKind(Private->getType())) { in emitListToGlobalCopyFunction()
2228 llvm::Value *V = CGF.EmitLoadOfScalar( in emitListToGlobalCopyFunction()
2231 CGF.EmitStoreOfScalar(V, GlobLVal); in emitListToGlobalCopyFunction()
2235 CodeGenFunction::ComplexPairTy V = CGF.EmitLoadOfComplex( in emitListToGlobalCopyFunction()
2236 CGF.MakeAddrLValue(ElemPtr, Private->getType()), Loc); in emitListToGlobalCopyFunction()
2237 CGF.EmitStoreOfComplex(V, GlobLVal, /*isInit=*/false); in emitListToGlobalCopyFunction()
2241 CGF.EmitAggregateCopy(GlobLVal, in emitListToGlobalCopyFunction()
2242 CGF.MakeAddrLValue(ElemPtr, Private->getType()), in emitListToGlobalCopyFunction()
2249 CGF.FinishFunction(); in emitListToGlobalCopyFunction()
2292 CodeGenFunction CGF(CGM); in emitListToGlobalReduceFunction() local
2293 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitListToGlobalReduceFunction()
2295 CGBuilderTy &Bld = CGF.Builder; in emitListToGlobalReduceFunction()
2297 Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg); in emitListToGlobalReduceFunction()
2302 CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc), in emitListToGlobalReduceFunction()
2308 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); in emitListToGlobalReduceFunction()
2310 llvm::Value *Idxs[] = {CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), in emitListToGlobalReduceFunction()
2315 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitListToGlobalReduceFunction()
2321 LValue GlobLVal = CGF.EmitLValueForField( in emitListToGlobalReduceFunction()
2322 CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD); in emitListToGlobalReduceFunction()
2323 Address GlobAddr = GlobLVal.getAddress(CGF); in emitListToGlobalReduceFunction()
2324 CGF.EmitStoreOfScalar(GlobAddr.getPointer(), Elem, /*Volatile=*/false, in emitListToGlobalReduceFunction()
2329 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitListToGlobalReduceFunction()
2330 llvm::Value *Size = CGF.Builder.CreateIntCast( in emitListToGlobalReduceFunction()
2331 CGF.getVLASize( in emitListToGlobalReduceFunction()
2332 CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) in emitListToGlobalReduceFunction()
2334 CGF.SizeTy, /*isSigned=*/false); in emitListToGlobalReduceFunction()
2335 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), in emitListToGlobalReduceFunction()
2342 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitListToGlobalReduceFunction()
2343 llvm::Value *ReducedPtr = CGF.EmitLoadOfScalar( in emitListToGlobalReduceFunction()
2346 CGF, Loc, ReduceFn, {GlobalReduceList, ReducedPtr}); in emitListToGlobalReduceFunction()
2347 CGF.FinishFunction(); in emitListToGlobalReduceFunction()
2386 CodeGenFunction CGF(CGM); in emitGlobalToListCopyFunction() local
2387 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitGlobalToListCopyFunction()
2389 CGBuilderTy &Bld = CGF.Builder; in emitGlobalToListCopyFunction()
2391 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitGlobalToListCopyFunction()
2392 Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg); in emitGlobalToListCopyFunction()
2393 llvm::Type *ElemTy = CGF.ConvertTypeForMem(ReductionArrayTy); in emitGlobalToListCopyFunction()
2396 CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false, in emitGlobalToListCopyFunction()
2399 ElemTy, CGF.getPointerAlign()); in emitGlobalToListCopyFunction()
2404 CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc), in emitGlobalToListCopyFunction()
2407 llvm::Value *Idxs[] = {CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), in emitGlobalToListCopyFunction()
2414 llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar( in emitGlobalToListCopyFunction()
2417 ElemTy = CGF.ConvertTypeForMem(Private->getType()); in emitGlobalToListCopyFunction()
2427 LValue GlobLVal = CGF.EmitLValueForField( in emitGlobalToListCopyFunction()
2428 CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD); in emitGlobalToListCopyFunction()
2429 Address GlobAddr = GlobLVal.getAddress(CGF); in emitGlobalToListCopyFunction()
2431 CGF.ConvertTypeForMem(Private->getType()), in emitGlobalToListCopyFunction()
2433 switch (CGF.getEvaluationKind(Private->getType())) { in emitGlobalToListCopyFunction()
2435 llvm::Value *V = CGF.EmitLoadOfScalar(GlobLVal, Loc); in emitGlobalToListCopyFunction()
2436 CGF.EmitStoreOfScalar(V, ElemPtr, /*Volatile=*/false, Private->getType(), in emitGlobalToListCopyFunction()
2442 CodeGenFunction::ComplexPairTy V = CGF.EmitLoadOfComplex(GlobLVal, Loc); in emitGlobalToListCopyFunction()
2443 CGF.EmitStoreOfComplex(V, CGF.MakeAddrLValue(ElemPtr, Private->getType()), in emitGlobalToListCopyFunction()
2448 CGF.EmitAggregateCopy(CGF.MakeAddrLValue(ElemPtr, Private->getType()), in emitGlobalToListCopyFunction()
2456 CGF.FinishFunction(); in emitGlobalToListCopyFunction()
2499 CodeGenFunction CGF(CGM); in emitGlobalToListReduceFunction() local
2500 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitGlobalToListReduceFunction()
2502 CGBuilderTy &Bld = CGF.Builder; in emitGlobalToListReduceFunction()
2504 Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg); in emitGlobalToListReduceFunction()
2509 CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc), in emitGlobalToListReduceFunction()
2515 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); in emitGlobalToListReduceFunction()
2517 llvm::Value *Idxs[] = {CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), in emitGlobalToListReduceFunction()
2522 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitGlobalToListReduceFunction()
2528 LValue GlobLVal = CGF.EmitLValueForField( in emitGlobalToListReduceFunction()
2529 CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD); in emitGlobalToListReduceFunction()
2530 Address GlobAddr = GlobLVal.getAddress(CGF); in emitGlobalToListReduceFunction()
2531 CGF.EmitStoreOfScalar(GlobAddr.getPointer(), Elem, /*Volatile=*/false, in emitGlobalToListReduceFunction()
2536 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitGlobalToListReduceFunction()
2537 llvm::Value *Size = CGF.Builder.CreateIntCast( in emitGlobalToListReduceFunction()
2538 CGF.getVLASize( in emitGlobalToListReduceFunction()
2539 CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) in emitGlobalToListReduceFunction()
2541 CGF.SizeTy, /*isSigned=*/false); in emitGlobalToListReduceFunction()
2542 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), in emitGlobalToListReduceFunction()
2549 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitGlobalToListReduceFunction()
2550 llvm::Value *ReducedPtr = CGF.EmitLoadOfScalar( in emitGlobalToListReduceFunction()
2553 CGF, Loc, ReduceFn, {ReducedPtr, GlobalReduceList}); in emitGlobalToListReduceFunction()
2554 CGF.FinishFunction(); in emitGlobalToListReduceFunction()
2801 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates, in emitReduction() argument
2804 if (!CGF.HaveInsertPoint()) in emitReduction()
2815 CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs, in emitReduction()
2839 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitReduction()
2855 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); in emitReduction()
2859 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitReduction()
2860 CGF.Builder.CreateStore( in emitReduction()
2861 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReduction()
2862 CGF.EmitLValue(RHSExprs[I]).getPointer(CGF), CGF.VoidPtrTy), in emitReduction()
2867 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitReduction()
2868 llvm::Value *Size = CGF.Builder.CreateIntCast( in emitReduction()
2869 CGF.getVLASize( in emitReduction()
2870 CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) in emitReduction()
2872 CGF.SizeTy, /*isSigned=*/false); in emitReduction()
2873 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), in emitReduction()
2878 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReduction()
2879 ReductionList.getPointer(), CGF.VoidPtrTy); in emitReduction()
2881 CGF.CurFn->getName(), Loc, CGF.ConvertTypeForMem(ReductionArrayTy), in emitReduction()
2884 CGF.getTypeSize(C.getRecordType(ReductionRec)); in emitReduction()
2886 CGF.Builder.CreateSExtOrTrunc(ReductionDataSize, CGF.Int64Ty); in emitReduction()
2896 Res = CGF.EmitRuntimeCall( in emitReduction()
2903 auto *KernelTeamsReductionPtr = CGF.EmitRuntimeCall( in emitReduction()
2921 CGF.Builder.getInt32(C.getLangOpts().OpenMPCUDAReductionBufNum), in emitReduction()
2931 Res = CGF.EmitRuntimeCall( in emitReduction()
2938 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.reduction.done"); in emitReduction()
2939 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.then"); in emitReduction()
2940 llvm::Value *Cond = CGF.Builder.CreateICmpEQ( in emitReduction()
2942 CGF.Builder.CreateCondBr(Cond, ThenBB, ExitBB); in emitReduction()
2948 CGF.EmitBlock(ThenBB); in emitReduction()
2952 this](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
2957 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), in emitReduction()
2965 RCG(CGF); in emitReduction()
2967 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitReduction()
2968 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in emitReduction()
3004 CGOpenMPRuntimeGPU::getParameterAddress(CodeGenFunction &CGF, in getParameterAddress() argument
3010 Address LocalAddr = CGF.GetAddrOfLocalVar(TargetParam); in getParameterAddress()
3016 CGF.getTypes().getTargetAddressSpace(NativePointeeTy); in getParameterAddress()
3018 llvm::Value *TargetAddr = CGF.EmitLoadOfScalar(LocalAddr, /*Volatile=*/false, in getParameterAddress()
3021 TargetAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getParameterAddress()
3023 llvm::PointerType::get(CGF.getLLVMContext(), NativePointeeAddrSpace)); in getParameterAddress()
3024 Address NativeParamAddr = CGF.CreateMemTemp(NativeParamType); in getParameterAddress()
3025 CGF.EmitStoreOfScalar(TargetAddr, NativeParamAddr, /*Volatile=*/false, in getParameterAddress()
3031 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, in emitOutlinedFunctionCall() argument
3048 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(NativeArg, TargetType)); in emitOutlinedFunctionCall()
3050 CGOpenMPRuntime::emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, TargetArgs); in emitOutlinedFunctionCall()
3096 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true); in createParallelDataSharingWrapper() local
3097 CGF.StartFunction(GlobalDecl(), Ctx.VoidTy, Fn, CGFI, WrapperArgs, in createParallelDataSharingWrapper()
3103 Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, in createParallelDataSharingWrapper()
3105 CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddr); in createParallelDataSharingWrapper()
3109 Args.emplace_back(CGF.GetAddrOfLocalVar(&WrapperArg).getPointer()); in createParallelDataSharingWrapper()
3112 CGBuilderTy &Bld = CGF.Builder; in createParallelDataSharingWrapper()
3118 CGF.CreateDefaultAlignTempAlloca(CGF.VoidPtrPtrTy, "global_args"); in createParallelDataSharingWrapper()
3121 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in createParallelDataSharingWrapper()
3130 SharedArgListAddress = CGF.EmitLoadOfPointer( in createParallelDataSharingWrapper()
3131 GlobalArgs, CGF.getContext() in createParallelDataSharingWrapper()
3132 .getPointerType(CGF.getContext().VoidPtrTy) in createParallelDataSharingWrapper()
3139 Src, CGF.SizeTy->getPointerTo(), CGF.SizeTy); in createParallelDataSharingWrapper()
3140 llvm::Value *LB = CGF.EmitLoadOfScalar( in createParallelDataSharingWrapper()
3143 CGF.getContext().getPointerType(CGF.getContext().getSizeType()), in createParallelDataSharingWrapper()
3149 Src, CGF.SizeTy->getPointerTo(), CGF.SizeTy); in createParallelDataSharingWrapper()
3150 llvm::Value *UB = CGF.EmitLoadOfScalar( in createParallelDataSharingWrapper()
3153 CGF.getContext().getPointerType(CGF.getContext().getSizeType()), in createParallelDataSharingWrapper()
3159 ASTContext &CGFContext = CGF.getContext(); in createParallelDataSharingWrapper()
3164 Src, CGF.ConvertTypeForMem(CGFContext.getPointerType(ElemTy)), in createParallelDataSharingWrapper()
3165 CGF.ConvertTypeForMem(ElemTy)); in createParallelDataSharingWrapper()
3166 llvm::Value *Arg = CGF.EmitLoadOfScalar(TypedAddress, in createParallelDataSharingWrapper()
3172 Arg = castValueToType(CGF, Arg, ElemTy, CGFContext.getUIntPtrType(), in createParallelDataSharingWrapper()
3179 emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedParallelFn, Args); in createParallelDataSharingWrapper()
3180 CGF.FinishFunction(); in createParallelDataSharingWrapper()
3184 void CGOpenMPRuntimeGPU::emitFunctionProlog(CodeGenFunction &CGF, in emitFunctionProlog() argument
3190 assert(FunctionGlobalizedDecls.count(CGF.CurFn) == 0 && in emitFunctionProlog()
3202 NeedToDelayGlobalization = CGF.CapturedStmtInfo->getKind() == CR_OpenMP; in emitFunctionProlog()
3209 CheckVarsEscapingDeclContext VarChecker(CGF, TeamAndReductions.second); in emitFunctionProlog()
3222 auto I = FunctionGlobalizedDecls.try_emplace(CGF.CurFn).first; in emitFunctionProlog()
3238 emitGenericVarsProlog(CGF, D->getBeginLoc()); in emitFunctionProlog()
3242 void Emit(CodeGenFunction &CGF, Flags flags) override { in emitFunctionProlog()
3243 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()) in emitFunctionProlog()
3244 .emitGenericVarsEpilog(CGF); in emitFunctionProlog()
3247 CGF.EHStack.pushCleanup<GlobalizationScope>(NormalAndEHCleanup); in emitFunctionProlog()
3251 Address CGOpenMPRuntimeGPU::getAddressOfLocalVariable(CodeGenFunction &CGF, in getAddressOfLocalVariable() argument
3279 llvm::Type *VarTy = CGF.ConvertTypeForMem(VD->getType()); in getAddressOfLocalVariable()
3289 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddressOfLocalVariable()
3299 auto I = FunctionGlobalizedDecls.find(CGF.CurFn); in getAddressOfLocalVariable()
3320 void CGOpenMPRuntimeGPU::functionFinished(CodeGenFunction &CGF) { in functionFinished() argument
3321 FunctionGlobalizedDecls.erase(CGF.CurFn); in functionFinished()
3322 CGOpenMPRuntime::functionFinished(CGF); in functionFinished()
3326 CodeGenFunction &CGF, const OMPLoopDirective &S, in getDefaultDistScheduleAndChunk() argument
3329 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in getDefaultDistScheduleAndChunk()
3332 Chunk = CGF.EmitScalarConversion( in getDefaultDistScheduleAndChunk()
3333 RT.getGPUNumThreads(CGF), in getDefaultDistScheduleAndChunk()
3334 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0), in getDefaultDistScheduleAndChunk()
3339 CGF, S, ScheduleKind, Chunk); in getDefaultDistScheduleAndChunk()
3343 CodeGenFunction &CGF, const OMPLoopDirective &S, in getDefaultScheduleAndChunk() argument
3349 ChunkExpr = IntegerLiteral::Create(CGF.getContext(), ChunkSize, in getDefaultScheduleAndChunk()
3350 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0), in getDefaultScheduleAndChunk()
3355 CodeGenFunction &CGF, const OMPExecutableDirective &D) const { in adjustTargetSpecificDataForLambdas() argument
3371 Address VDAddr = CGF.GetAddrOfLocalVar(VD); in adjustTargetSpecificDataForLambdas()
3374 VDLVal = CGF.EmitLoadOfReferenceLValue(VDAddr, VD->getType()); in adjustTargetSpecificDataForLambdas()
3376 VDLVal = CGF.MakeAddrLValue( in adjustTargetSpecificDataForLambdas()
3381 if (ThisCapture && CGF.CapturedStmtInfo->isCXXThisExprCaptured()) { in adjustTargetSpecificDataForLambdas()
3383 CGF.EmitLValueForFieldInitialization(VDLVal, ThisCapture); in adjustTargetSpecificDataForLambdas()
3384 llvm::Value *CXXThis = CGF.LoadCXXThis(); in adjustTargetSpecificDataForLambdas()
3385 CGF.EmitStoreOfScalar(CXXThis, ThisLVal); in adjustTargetSpecificDataForLambdas()
3397 LValue VarLVal = CGF.EmitLValueForFieldInitialization(VDLVal, It->second); in adjustTargetSpecificDataForLambdas()
3398 Address VDAddr = CGF.GetAddrOfLocalVar(cast<VarDecl>(VD)); in adjustTargetSpecificDataForLambdas()
3400 VDAddr = CGF.EmitLoadOfReferenceLValue(VDAddr, in adjustTargetSpecificDataForLambdas()
3402 .getAddress(CGF); in adjustTargetSpecificDataForLambdas()
3403 CGF.EmitStoreOfScalar(VDAddr.getPointer(), VarLVal); in adjustTargetSpecificDataForLambdas()
3543 llvm::Value *CGOpenMPRuntimeGPU::getGPUNumThreads(CodeGenFunction &CGF) { in getGPUNumThreads() argument
3544 CGBuilderTy &Bld = CGF.Builder; in getGPUNumThreads()
3545 llvm::Module *M = &CGF.CGM.getModule(); in getGPUNumThreads()
3550 llvm::FunctionType::get(CGF.Int32Ty, std::nullopt, false), in getGPUNumThreads()
3551 llvm::GlobalVariable::ExternalLinkage, LocSize, &CGF.CGM.getModule()); in getGPUNumThreads()
3556 llvm::Value *CGOpenMPRuntimeGPU::getGPUThreadID(CodeGenFunction &CGF) { in getGPUThreadID() argument
3558 return CGF.EmitRuntimeCall( in getGPUThreadID()
3564 llvm::Value *CGOpenMPRuntimeGPU::getGPUWarpSize(CodeGenFunction &CGF) { in getGPUWarpSize() argument
3566 return CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in getGPUWarpSize()