Lines Matching refs:CGF
47 void Enter(CodeGenFunction &CGF) override { in Enter() argument
48 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs); in Enter()
50 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes); in Enter()
51 auto *ThenBlock = CGF.createBasicBlock("omp_if.then"); in Enter()
52 ContBlock = CGF.createBasicBlock("omp_if.end"); in Enter()
54 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock); in Enter()
55 CGF.EmitBlock(ThenBlock); in Enter()
58 void Done(CodeGenFunction &CGF) { in Done() argument
60 CGF.EmitBranch(ContBlock); in Done()
61 CGF.EmitBlock(ContBlock, true); in Done()
63 void Exit(CodeGenFunction &CGF) override { in Exit() argument
64 CGF.EmitRuntimeCall(ExitCallee, ExitArgs); in Exit()
224 CodeGenFunction &CGF; member in __anon0383ec9b0111::CheckVarsEscapingDeclContext
243 if (auto *CSI = CGF.CapturedStmtInfo) { in markAsEscaped()
268 if ((!CGF.CapturedStmtInfo || in markAsEscaped()
269 (IsForCombinedParallelRegion && CGF.CapturedStmtInfo)) && in markAsEscaped()
342 unsigned WarpSize = CGF.getTarget().getGridValue(llvm::omp::GV_Warp_Size); in buildRecordForGlobalizedVars()
348 CGF.getContext(), EscapedDeclsForParallel, EscapedDeclsForTeams, in buildRecordForGlobalizedVars()
353 CheckVarsEscapingDeclContext(CodeGenFunction &CGF, in CheckVarsEscapingDeclContext() argument
355 : CGF(CGF), EscapedDecls(TeamsReductions.begin(), TeamsReductions.end()) { in CheckVarsEscapingDeclContext()
536 static llvm::Value *getNVPTXWarpID(CodeGenFunction &CGF) { in getNVPTXWarpID() argument
537 CGBuilderTy &Bld = CGF.Builder; in getNVPTXWarpID()
539 CGF.getTarget().getGridValue(llvm::omp::GV_Warp_Size_Log2); in getNVPTXWarpID()
540 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in getNVPTXWarpID()
541 return Bld.CreateAShr(RT.getGPUThreadID(CGF), LaneIDBits, "nvptx_warp_id"); in getNVPTXWarpID()
547 static llvm::Value *getNVPTXLaneID(CodeGenFunction &CGF) { in getNVPTXLaneID() argument
548 CGBuilderTy &Bld = CGF.Builder; in getNVPTXLaneID()
549 unsigned LaneIDMask = CGF.getContext().getTargetInfo().getGridValue( in getNVPTXLaneID()
551 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in getNVPTXLaneID()
552 return Bld.CreateAnd(RT.getGPUThreadID(CGF), Bld.getInt32(LaneIDMask), in getNVPTXLaneID()
1028 void Enter(CodeGenFunction &CGF) override { in emitNonSPMDKernel() argument
1030 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitNonSPMDKernel()
1031 RT.emitKernelInit(CGF, EST, /* IsSPMD */ false); in emitNonSPMDKernel()
1033 RT.setLocThreadIdInsertPt(CGF, /*AtCurrentPoint=*/true); in emitNonSPMDKernel()
1035 void Exit(CodeGenFunction &CGF) override { in emitNonSPMDKernel() argument
1037 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitNonSPMDKernel()
1038 RT.clearLocThreadIdInsertPt(CGF); in emitNonSPMDKernel()
1039 RT.emitKernelDeinit(CGF, EST, /* IsSPMD */ false); in emitNonSPMDKernel()
1049 void CGOpenMPRuntimeGPU::emitKernelInit(CodeGenFunction &CGF, in emitKernelInit() argument
1051 CGBuilderTy &Bld = CGF.Builder; in emitKernelInit()
1055 emitGenericVarsProlog(CGF, EST.Loc); in emitKernelInit()
1058 void CGOpenMPRuntimeGPU::emitKernelDeinit(CodeGenFunction &CGF, in emitKernelDeinit() argument
1062 emitGenericVarsEpilog(CGF); in emitKernelDeinit()
1064 CGBuilderTy &Bld = CGF.Builder; in emitKernelDeinit()
1089 void Enter(CodeGenFunction &CGF) override { in emitSPMDKernel() argument
1090 RT.emitKernelInit(CGF, EST, /* IsSPMD */ true); in emitSPMDKernel()
1092 RT.setLocThreadIdInsertPt(CGF, /*AtCurrentPoint=*/true); in emitSPMDKernel()
1094 void Exit(CodeGenFunction &CGF) override { in emitSPMDKernel() argument
1095 RT.clearLocThreadIdInsertPt(CGF); in emitSPMDKernel()
1096 RT.emitKernelDeinit(CGF, EST, /* IsSPMD */ true); in emitSPMDKernel()
1201 void CGOpenMPRuntimeGPU::emitProcBindClause(CodeGenFunction &CGF, in emitProcBindClause() argument
1208 CGOpenMPRuntime::emitProcBindClause(CGF, ProcBind, Loc); in emitProcBindClause()
1211 void CGOpenMPRuntimeGPU::emitNumThreadsClause(CodeGenFunction &CGF, in emitNumThreadsClause() argument
1218 CGOpenMPRuntime::emitNumThreadsClause(CGF, NumThreads, Loc); in emitNumThreadsClause()
1221 void CGOpenMPRuntimeGPU::emitNumTeamsClause(CodeGenFunction &CGF, in emitNumTeamsClause() argument
1237 void Enter(CodeGenFunction &CGF) override { in emitParallelOutlinedFunction() argument
1241 void Exit(CodeGenFunction &CGF) override { in emitParallelOutlinedFunction() argument
1343 void Enter(CodeGenFunction &CGF) override { in emitTeamsOutlinedFunction() argument
1345 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitTeamsOutlinedFunction()
1347 auto I = Rt.FunctionGlobalizedDecls.try_emplace(CGF.CurFn).first; in emitTeamsOutlinedFunction()
1357 Rt.emitGenericVarsProlog(CGF, Loc); in emitTeamsOutlinedFunction()
1359 void Exit(CodeGenFunction &CGF) override { in emitTeamsOutlinedFunction() argument
1360 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()) in emitTeamsOutlinedFunction()
1361 .emitGenericVarsEpilog(CGF); in emitTeamsOutlinedFunction()
1371 void CGOpenMPRuntimeGPU::emitGenericVarsProlog(CodeGenFunction &CGF, in emitGenericVarsProlog() argument
1378 CGBuilderTy &Bld = CGF.Builder; in emitGenericVarsProlog()
1380 const auto I = FunctionGlobalizedDecls.find(CGF.CurFn); in emitGenericVarsProlog()
1393 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType()); in emitGenericVarsProlog()
1394 ParValue = CGF.EmitLoadOfScalar(ParLVal, Loc); in emitGenericVarsProlog()
1398 llvm::Value *AllocArgs[] = {CGF.getTypeSize(VD->getType())}; in emitGenericVarsProlog()
1400 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitGenericVarsProlog()
1405 llvm::PointerType *VarPtrTy = CGF.ConvertTypeForMem(VarTy)->getPointerTo(); in emitGenericVarsProlog()
1408 LValue VarAddr = CGF.MakeNaturalAlignAddrLValue(CastedVoidPtr, VarTy); in emitGenericVarsProlog()
1409 Rec.second.PrivateAddr = VarAddr.getAddress(CGF); in emitGenericVarsProlog()
1414 CGF.EmitStoreOfScalar(ParValue, VarAddr); in emitGenericVarsProlog()
1415 I->getSecond().MappedParams->setVarAddr(CGF, VD, VarAddr.getAddress(CGF)); in emitGenericVarsProlog()
1417 if (auto *DI = CGF.getDebugInfo()) in emitGenericVarsProlog()
1423 llvm::Value *Size = CGF.getTypeSize(VD->getType()); in emitGenericVarsProlog()
1426 Size, llvm::ConstantInt::get(CGF.SizeTy, Align.getQuantity() - 1)); in emitGenericVarsProlog()
1428 llvm::ConstantInt::get(CGF.SizeTy, Align.getQuantity()); in emitGenericVarsProlog()
1434 llvm::Value *AllocArgs[] = {CGF.getTypeSize(VD->getType())}; in emitGenericVarsProlog()
1436 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitGenericVarsProlog()
1442 {VoidPtr, CGF.getTypeSize(VD->getType())})); in emitGenericVarsProlog()
1443 LValue Base = CGF.MakeAddrLValue(VoidPtr, VD->getType(), in emitGenericVarsProlog()
1446 I->getSecond().MappedParams->setVarAddr(CGF, cast<VarDecl>(VD), in emitGenericVarsProlog()
1447 Base.getAddress(CGF)); in emitGenericVarsProlog()
1449 I->getSecond().MappedParams->apply(CGF); in emitGenericVarsProlog()
1452 void CGOpenMPRuntimeGPU::emitGenericVarsEpilog(CodeGenFunction &CGF, in emitGenericVarsEpilog() argument
1458 const auto I = FunctionGlobalizedDecls.find(CGF.CurFn); in emitGenericVarsEpilog()
1463 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitGenericVarsEpilog()
1470 I->getSecond().MappedParams->restore(CGF); in emitGenericVarsEpilog()
1473 CGF.getTypeSize(VD->getType())}; in emitGenericVarsEpilog()
1474 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitGenericVarsEpilog()
1481 void CGOpenMPRuntimeGPU::emitTeamsCall(CodeGenFunction &CGF, in emitTeamsCall() argument
1486 if (!CGF.HaveInsertPoint()) in emitTeamsCall()
1489 Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, in emitTeamsCall()
1491 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0)); in emitTeamsCall()
1493 OutlinedFnArgs.push_back(emitThreadIDAddress(CGF, Loc).getPointer()); in emitTeamsCall()
1496 emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs); in emitTeamsCall()
1499 void CGOpenMPRuntimeGPU::emitParallelCall(CodeGenFunction &CGF, in emitParallelCall() argument
1504 if (!CGF.HaveInsertPoint()) in emitParallelCall()
1508 IfCond](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitParallelCall() argument
1509 CGBuilderTy &Bld = CGF.Builder; in emitParallelCall()
1519 CodeGenFunction::OMPPrivateScope PrivateArgScope(CGF); in emitParallelCall()
1521 Address CapturedVarsAddrs = CGF.CreateDefaultAlignTempAlloca( in emitParallelCall()
1527 ASTContext &Ctx = CGF.getContext(); in emitParallelCall()
1533 PtrV = Bld.CreateIntToPtr(V, CGF.VoidPtrTy); in emitParallelCall()
1535 PtrV = Bld.CreatePointerBitCastOrAddrSpaceCast(V, CGF.VoidPtrTy); in emitParallelCall()
1536 CGF.EmitStoreOfScalar(PtrV, Dst, /*Volatile=*/false, in emitParallelCall()
1544 IfCondVal = Bld.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.Int32Ty, in emitParallelCall()
1547 IfCondVal = llvm::ConstantInt::get(CGF.Int32Ty, 1); in emitParallelCall()
1550 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitParallelCall()
1553 getThreadID(CGF, Loc), in emitParallelCall()
1555 llvm::ConstantInt::get(CGF.Int32Ty, -1), in emitParallelCall()
1556 llvm::ConstantInt::get(CGF.Int32Ty, -1), in emitParallelCall()
1560 CGF.VoidPtrPtrTy), in emitParallelCall()
1562 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitParallelCall()
1568 RCG(CGF); in emitParallelCall()
1571 void CGOpenMPRuntimeGPU::syncCTAThreads(CodeGenFunction &CGF) { in syncCTAThreads() argument
1573 if (!CGF.HaveInsertPoint()) in syncCTAThreads()
1580 llvm::ConstantInt::get(CGF.Int32Ty, /*V=*/0, /*isSigned=*/true)}; in syncCTAThreads()
1581 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in syncCTAThreads()
1586 void CGOpenMPRuntimeGPU::emitBarrierCall(CodeGenFunction &CGF, in emitBarrierCall() argument
1591 if (!CGF.HaveInsertPoint()) in emitBarrierCall()
1595 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags), in emitBarrierCall()
1596 getThreadID(CGF, Loc)}; in emitBarrierCall()
1598 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitBarrierCall()
1604 CodeGenFunction &CGF, StringRef CriticalName, in emitCriticalRegion() argument
1607 llvm::BasicBlock *LoopBB = CGF.createBasicBlock("omp.critical.loop"); in emitCriticalRegion()
1608 llvm::BasicBlock *TestBB = CGF.createBasicBlock("omp.critical.test"); in emitCriticalRegion()
1609 llvm::BasicBlock *SyncBB = CGF.createBasicBlock("omp.critical.sync"); in emitCriticalRegion()
1610 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.critical.body"); in emitCriticalRegion()
1611 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.critical.exit"); in emitCriticalRegion()
1613 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitCriticalRegion()
1616 llvm::Value *Mask = CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitCriticalRegion()
1619 llvm::Value *ThreadID = RT.getGPUThreadID(CGF); in emitCriticalRegion()
1622 llvm::Value *TeamWidth = RT.getGPUNumThreads(CGF); in emitCriticalRegion()
1626 CGF.getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/0); in emitCriticalRegion()
1627 Address Counter = CGF.CreateMemTemp(Int32Ty, "critical_counter"); in emitCriticalRegion()
1628 LValue CounterLVal = CGF.MakeAddrLValue(Counter, Int32Ty); in emitCriticalRegion()
1629 CGF.EmitStoreOfScalar(llvm::Constant::getNullValue(CGM.Int32Ty), CounterLVal, in emitCriticalRegion()
1633 CGF.EmitBlock(LoopBB); in emitCriticalRegion()
1634 llvm::Value *CounterVal = CGF.EmitLoadOfScalar(CounterLVal, Loc); in emitCriticalRegion()
1635 llvm::Value *CmpLoopBound = CGF.Builder.CreateICmpSLT(CounterVal, TeamWidth); in emitCriticalRegion()
1636 CGF.Builder.CreateCondBr(CmpLoopBound, TestBB, ExitBB); in emitCriticalRegion()
1640 CGF.EmitBlock(TestBB); in emitCriticalRegion()
1641 CounterVal = CGF.EmitLoadOfScalar(CounterLVal, Loc); in emitCriticalRegion()
1643 CGF.Builder.CreateICmpEQ(ThreadID, CounterVal); in emitCriticalRegion()
1644 CGF.Builder.CreateCondBr(CmpThreadToCounter, BodyBB, SyncBB); in emitCriticalRegion()
1647 CGF.EmitBlock(BodyBB); in emitCriticalRegion()
1650 CGOpenMPRuntime::emitCriticalRegion(CGF, CriticalName, CriticalOpGen, Loc, in emitCriticalRegion()
1657 CGF.EmitBlock(SyncBB); in emitCriticalRegion()
1659 (void)CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitCriticalRegion()
1664 CGF.Builder.CreateNSWAdd(CounterVal, CGF.Builder.getInt32(1)); in emitCriticalRegion()
1665 CGF.EmitStoreOfScalar(IncCounterVal, CounterLVal); in emitCriticalRegion()
1666 CGF.EmitBranch(LoopBB); in emitCriticalRegion()
1669 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in emitCriticalRegion()
1673 static llvm::Value *castValueToType(CodeGenFunction &CGF, llvm::Value *Val, in castValueToType() argument
1676 assert(!CGF.getContext().getTypeSizeInChars(CastTy).isZero() && in castValueToType()
1678 assert(!CGF.getContext().getTypeSizeInChars(ValTy).isZero() && in castValueToType()
1680 llvm::Type *LLVMCastTy = CGF.ConvertTypeForMem(CastTy); in castValueToType()
1683 if (CGF.getContext().getTypeSizeInChars(ValTy) == in castValueToType()
1684 CGF.getContext().getTypeSizeInChars(CastTy)) in castValueToType()
1685 return CGF.Builder.CreateBitCast(Val, LLVMCastTy); in castValueToType()
1687 return CGF.Builder.CreateIntCast(Val, LLVMCastTy, in castValueToType()
1689 Address CastItem = CGF.CreateMemTemp(CastTy); in castValueToType()
1690 Address ValCastItem = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in castValueToType()
1692 CGF.EmitStoreOfScalar(Val, ValCastItem, /*Volatile=*/false, ValTy, in castValueToType()
1695 return CGF.EmitLoadOfScalar(CastItem, /*Volatile=*/false, CastTy, Loc, in castValueToType()
1702 static llvm::Value *createRuntimeShuffleFunction(CodeGenFunction &CGF, in createRuntimeShuffleFunction() argument
1707 CodeGenModule &CGM = CGF.CGM; in createRuntimeShuffleFunction()
1708 CGBuilderTy &Bld = CGF.Builder; in createRuntimeShuffleFunction()
1713 CharUnits Size = CGF.getContext().getTypeSizeInChars(ElemType); in createRuntimeShuffleFunction()
1722 QualType CastTy = CGF.getContext().getIntTypeForBitwidth( in createRuntimeShuffleFunction()
1724 llvm::Value *ElemCast = castValueToType(CGF, Elem, ElemType, CastTy, Loc); in createRuntimeShuffleFunction()
1726 Bld.CreateIntCast(RT.getGPUWarpSize(CGF), CGM.Int16Ty, /*isSigned=*/true); in createRuntimeShuffleFunction()
1728 llvm::Value *ShuffledVal = CGF.EmitRuntimeCall( in createRuntimeShuffleFunction()
1732 return castValueToType(CGF, ShuffledVal, CastTy, ElemType, Loc); in createRuntimeShuffleFunction()
1735 static void shuffleAndStore(CodeGenFunction &CGF, Address SrcAddr, in shuffleAndStore() argument
1738 CGBuilderTy &Bld = CGF.Builder; in shuffleAndStore()
1740 CharUnits Size = CGF.getContext().getTypeSizeInChars(ElemType); in shuffleAndStore()
1754 Bld.CreateConstGEP(SrcAddr, 1), CGF.VoidPtrTy); in shuffleAndStore()
1758 QualType IntType = CGF.getContext().getIntTypeForBitwidth( in shuffleAndStore()
1759 CGF.getContext().toBits(CharUnits::fromQuantity(IntSize)), in shuffleAndStore()
1761 llvm::Type *IntTy = CGF.ConvertTypeForMem(IntType); in shuffleAndStore()
1766 llvm::BasicBlock *PreCondBB = CGF.createBasicBlock(".shuffle.pre_cond"); in shuffleAndStore()
1767 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".shuffle.then"); in shuffleAndStore()
1768 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".shuffle.exit"); in shuffleAndStore()
1770 CGF.EmitBlock(PreCondBB); in shuffleAndStore()
1781 Ptr.getPointer(), CGF.VoidPtrTy)); in shuffleAndStore()
1784 CGF.EmitBlock(ThenBB); in shuffleAndStore()
1786 CGF, in shuffleAndStore()
1787 CGF.EmitLoadOfScalar(Ptr, /*Volatile=*/false, IntType, Loc, in shuffleAndStore()
1791 CGF.EmitStoreOfScalar(Res, ElemPtr, /*Volatile=*/false, IntType, in shuffleAndStore()
1798 CGF.EmitBranch(PreCondBB); in shuffleAndStore()
1799 CGF.EmitBlock(ExitBB); in shuffleAndStore()
1802 CGF, in shuffleAndStore()
1803 CGF.EmitLoadOfScalar(Ptr, /*Volatile=*/false, IntType, Loc, in shuffleAndStore()
1807 CGF.EmitStoreOfScalar(Res, ElemPtr, /*Volatile=*/false, IntType, in shuffleAndStore()
1841 CopyAction Action, CodeGenFunction &CGF, QualType ReductionArrayTy, in emitReductionListCopy() argument
1845 CodeGenModule &CGM = CGF.CGM;
1847 CGBuilderTy &Bld = CGF.Builder;
1875 SrcElementAddr = CGF.EmitLoadOfPointer(
1883 CGF.CreateMemTemp(Private->getType(), ".omp.reduction.element");
1891 SrcElementAddr = CGF.EmitLoadOfPointer(
1898 DestElementAddr = CGF.EmitLoadOfPointer(
1906 SrcElementAddr = CGF.EmitLoadOfPointer(
1912 llvm::Value *ElementSizeInChars = CGF.getTypeSize(Private->getType());
1918 Bld.CreateIntToPtr(ScratchPadElemAbsolutePtrVal, CGF.VoidPtrTy);
1927 llvm::Value *ElementSizeInChars = CGF.getTypeSize(Private->getType());
1933 Bld.CreateIntToPtr(ScratchPadElemAbsolutePtrVal, CGF.VoidPtrTy);
1942 CGF.CreateMemTemp(Private->getType(), ".omp.reduction.element");
1951 SrcElementAddr, CGF.ConvertTypeForMem(Private->getType()));
1958 shuffleAndStore(CGF, SrcElementAddr, DestElementAddr, Private->getType(),
1961 switch (CGF.getEvaluationKind(Private->getType())) {
1963 llvm::Value *Elem = CGF.EmitLoadOfScalar(
1968 CGF.EmitStoreOfScalar(
1974 CodeGenFunction::ComplexPairTy Elem = CGF.EmitLoadOfComplex(
1975 CGF.MakeAddrLValue(SrcElementAddr, Private->getType()),
1977 CGF.EmitStoreOfComplex(
1978 Elem, CGF.MakeAddrLValue(DestElementAddr, Private->getType()),
1983 CGF.EmitAggregateCopy(
1984 CGF.MakeAddrLValue(DestElementAddr, Private->getType()),
1985 CGF.MakeAddrLValue(SrcElementAddr, Private->getType()),
1997 CGF.EmitStoreOfScalar(Bld.CreatePointerBitCastOrAddrSpaceCast(
1998 DestElementAddr.getPointer(), CGF.VoidPtrTy),
2009 llvm::Value *ElementSizeInChars = CGF.getTypeSize(Private->getType());
2027 DestBase = Address(ScratchpadBasePtr, CGF.getPointerAlign());
2029 SrcBase = Address(ScratchpadBasePtr, CGF.getPointerAlign());
2076 CodeGenFunction CGF(CGM); in emitInterWarpCopyFunction() local
2077 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitInterWarpCopyFunction()
2079 CGBuilderTy &Bld = CGF.Builder; in emitInterWarpCopyFunction()
2092 unsigned WarpSize = CGF.getTarget().getGridValue(llvm::omp::GV_Warp_Size); in emitInterWarpCopyFunction()
2104 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitInterWarpCopyFunction()
2106 llvm::Value *ThreadID = RT.getGPUThreadID(CGF); in emitInterWarpCopyFunction()
2108 llvm::Value *LaneID = getNVPTXLaneID(CGF); in emitInterWarpCopyFunction()
2110 llvm::Value *WarpID = getNVPTXWarpID(CGF); in emitInterWarpCopyFunction()
2112 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitInterWarpCopyFunction()
2115 CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
2118 CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo()), in emitInterWarpCopyFunction()
2119 CGF.getPointerAlign()); in emitInterWarpCopyFunction()
2137 llvm::Type *CopyType = CGF.ConvertTypeForMem(CType); in emitInterWarpCopyFunction()
2144 CntAddr = CGF.CreateMemTemp(C.IntTy, ".cnt.addr"); in emitInterWarpCopyFunction()
2145 CGF.EmitStoreOfScalar(llvm::Constant::getNullValue(CGM.IntTy), CntAddr, in emitInterWarpCopyFunction()
2147 PrecondBB = CGF.createBasicBlock("precond"); in emitInterWarpCopyFunction()
2148 ExitBB = CGF.createBasicBlock("exit"); in emitInterWarpCopyFunction()
2149 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("body"); in emitInterWarpCopyFunction()
2151 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitInterWarpCopyFunction()
2152 CGF.EmitBlock(PrecondBB); in emitInterWarpCopyFunction()
2153 Cnt = CGF.EmitLoadOfScalar(CntAddr, /*Volatile=*/false, C.IntTy, Loc); in emitInterWarpCopyFunction()
2157 CGF.EmitBlock(BodyBB); in emitInterWarpCopyFunction()
2160 CGM.getOpenMPRuntime().emitBarrierCall(CGF, Loc, OMPD_unknown, in emitInterWarpCopyFunction()
2163 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("then"); in emitInterWarpCopyFunction()
2164 llvm::BasicBlock *ElseBB = CGF.createBasicBlock("else"); in emitInterWarpCopyFunction()
2165 llvm::BasicBlock *MergeBB = CGF.createBasicBlock("ifcont"); in emitInterWarpCopyFunction()
2170 CGF.EmitBlock(ThenBB); in emitInterWarpCopyFunction()
2174 llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
2197 llvm::Value *Elem = CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
2201 CGF.EmitStoreOfScalar(Elem, MediumPtr, /*Volatile=*/true, CType, in emitInterWarpCopyFunction()
2207 CGF.EmitBlock(ElseBB); in emitInterWarpCopyFunction()
2210 CGF.EmitBlock(MergeBB); in emitInterWarpCopyFunction()
2213 CGM.getOpenMPRuntime().emitBarrierCall(CGF, Loc, OMPD_unknown, in emitInterWarpCopyFunction()
2220 llvm::BasicBlock *W0ThenBB = CGF.createBasicBlock("then"); in emitInterWarpCopyFunction()
2221 llvm::BasicBlock *W0ElseBB = CGF.createBasicBlock("else"); in emitInterWarpCopyFunction()
2222 llvm::BasicBlock *W0MergeBB = CGF.createBasicBlock("ifcont"); in emitInterWarpCopyFunction()
2224 Address AddrNumWarpsArg = CGF.GetAddrOfLocalVar(&NumWarpsArg); in emitInterWarpCopyFunction()
2225 llvm::Value *NumWarpsVal = CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
2233 CGF.EmitBlock(W0ThenBB); in emitInterWarpCopyFunction()
2245 llvm::Value *TargetElemPtrVal = CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
2257 CGF.EmitLoadOfScalar(SrcMediumPtr, /*Volatile=*/true, CType, Loc); in emitInterWarpCopyFunction()
2258 CGF.EmitStoreOfScalar(SrcMediumValue, TargetElemPtr, /*Volatile=*/false, in emitInterWarpCopyFunction()
2262 CGF.EmitBlock(W0ElseBB); in emitInterWarpCopyFunction()
2265 CGF.EmitBlock(W0MergeBB); in emitInterWarpCopyFunction()
2269 CGF.EmitStoreOfScalar(Cnt, CntAddr, /*Volatile=*/false, C.IntTy); in emitInterWarpCopyFunction()
2270 CGF.EmitBranch(PrecondBB); in emitInterWarpCopyFunction()
2271 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitInterWarpCopyFunction()
2272 CGF.EmitBlock(ExitBB); in emitInterWarpCopyFunction()
2279 CGF.FinishFunction(); in emitInterWarpCopyFunction()
2380 CodeGenFunction CGF(CGM); in emitShuffleAndReduceFunction() local
2381 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitShuffleAndReduceFunction()
2383 CGBuilderTy &Bld = CGF.Builder; in emitShuffleAndReduceFunction()
2385 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitShuffleAndReduceFunction()
2388 CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false, in emitShuffleAndReduceFunction()
2390 CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo()), in emitShuffleAndReduceFunction()
2391 CGF.getPointerAlign()); in emitShuffleAndReduceFunction()
2393 Address AddrLaneIDArg = CGF.GetAddrOfLocalVar(&LaneIDArg); in emitShuffleAndReduceFunction()
2394 llvm::Value *LaneIDArgVal = CGF.EmitLoadOfScalar( in emitShuffleAndReduceFunction()
2397 Address AddrRemoteLaneOffsetArg = CGF.GetAddrOfLocalVar(&RemoteLaneOffsetArg); in emitShuffleAndReduceFunction()
2398 llvm::Value *RemoteLaneOffsetArgVal = CGF.EmitLoadOfScalar( in emitShuffleAndReduceFunction()
2401 Address AddrAlgoVerArg = CGF.GetAddrOfLocalVar(&AlgoVerArg); in emitShuffleAndReduceFunction()
2402 llvm::Value *AlgoVerArgVal = CGF.EmitLoadOfScalar( in emitShuffleAndReduceFunction()
2408 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.remote_reduce_list"); in emitShuffleAndReduceFunction()
2413 emitReductionListCopy(RemoteLaneToThread, CGF, ReductionArrayTy, Privates, in emitShuffleAndReduceFunction()
2455 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("then"); in emitShuffleAndReduceFunction()
2456 llvm::BasicBlock *ElseBB = CGF.createBasicBlock("else"); in emitShuffleAndReduceFunction()
2457 llvm::BasicBlock *MergeBB = CGF.createBasicBlock("ifcont"); in emitShuffleAndReduceFunction()
2460 CGF.EmitBlock(ThenBB); in emitShuffleAndReduceFunction()
2463 LocalReduceList.getPointer(), CGF.VoidPtrTy); in emitShuffleAndReduceFunction()
2465 RemoteReduceList.getPointer(), CGF.VoidPtrTy); in emitShuffleAndReduceFunction()
2467 CGF, Loc, ReduceFn, {LocalReduceListPtr, RemoteReduceListPtr}); in emitShuffleAndReduceFunction()
2470 CGF.EmitBlock(ElseBB); in emitShuffleAndReduceFunction()
2473 CGF.EmitBlock(MergeBB); in emitShuffleAndReduceFunction()
2481 llvm::BasicBlock *CpyThenBB = CGF.createBasicBlock("then"); in emitShuffleAndReduceFunction()
2482 llvm::BasicBlock *CpyElseBB = CGF.createBasicBlock("else"); in emitShuffleAndReduceFunction()
2483 llvm::BasicBlock *CpyMergeBB = CGF.createBasicBlock("ifcont"); in emitShuffleAndReduceFunction()
2486 CGF.EmitBlock(CpyThenBB); in emitShuffleAndReduceFunction()
2487 emitReductionListCopy(ThreadCopy, CGF, ReductionArrayTy, Privates, in emitShuffleAndReduceFunction()
2491 CGF.EmitBlock(CpyElseBB); in emitShuffleAndReduceFunction()
2494 CGF.EmitBlock(CpyMergeBB); in emitShuffleAndReduceFunction()
2496 CGF.FinishFunction(); in emitShuffleAndReduceFunction()
2535 CodeGenFunction CGF(CGM); in emitListToGlobalCopyFunction() local
2536 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitListToGlobalCopyFunction()
2538 CGBuilderTy &Bld = CGF.Builder; in emitListToGlobalCopyFunction()
2540 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitListToGlobalCopyFunction()
2541 Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg); in emitListToGlobalCopyFunction()
2544 CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false, in emitListToGlobalCopyFunction()
2546 CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo()), in emitListToGlobalCopyFunction()
2547 CGF.getPointerAlign()); in emitListToGlobalCopyFunction()
2552 CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc), in emitListToGlobalCopyFunction()
2554 llvm::Value *Idxs[] = {llvm::ConstantInt::getNullValue(CGF.Int32Ty), in emitListToGlobalCopyFunction()
2555 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), in emitListToGlobalCopyFunction()
2562 llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar( in emitListToGlobalCopyFunction()
2566 ElemPtrPtr, CGF.ConvertTypeForMem(Private->getType())->getPointerTo()); in emitListToGlobalCopyFunction()
2572 LValue GlobLVal = CGF.EmitLValueForField( in emitListToGlobalCopyFunction()
2573 CGF.MakeNaturalAlignAddrLValue(BufferArrPtr, StaticTy), FD); in emitListToGlobalCopyFunction()
2574 Address GlobAddr = GlobLVal.getAddress(CGF); in emitListToGlobalCopyFunction()
2578 switch (CGF.getEvaluationKind(Private->getType())) { in emitListToGlobalCopyFunction()
2580 llvm::Value *V = CGF.EmitLoadOfScalar( in emitListToGlobalCopyFunction()
2583 CGF.EmitStoreOfScalar(V, GlobLVal); in emitListToGlobalCopyFunction()
2587 CodeGenFunction::ComplexPairTy V = CGF.EmitLoadOfComplex( in emitListToGlobalCopyFunction()
2588 CGF.MakeAddrLValue(ElemPtr, Private->getType()), Loc); in emitListToGlobalCopyFunction()
2589 CGF.EmitStoreOfComplex(V, GlobLVal, /*isInit=*/false); in emitListToGlobalCopyFunction()
2593 CGF.EmitAggregateCopy(GlobLVal, in emitListToGlobalCopyFunction()
2594 CGF.MakeAddrLValue(ElemPtr, Private->getType()), in emitListToGlobalCopyFunction()
2601 CGF.FinishFunction(); in emitListToGlobalCopyFunction()
2644 CodeGenFunction CGF(CGM); in emitListToGlobalReduceFunction() local
2645 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitListToGlobalReduceFunction()
2647 CGBuilderTy &Bld = CGF.Builder; in emitListToGlobalReduceFunction()
2649 Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg); in emitListToGlobalReduceFunction()
2654 CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc), in emitListToGlobalReduceFunction()
2660 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); in emitListToGlobalReduceFunction()
2662 llvm::Value *Idxs[] = {llvm::ConstantInt::getNullValue(CGF.Int32Ty), in emitListToGlobalReduceFunction()
2663 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), in emitListToGlobalReduceFunction()
2668 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitListToGlobalReduceFunction()
2672 LValue GlobLVal = CGF.EmitLValueForField( in emitListToGlobalReduceFunction()
2673 CGF.MakeNaturalAlignAddrLValue(BufferArrPtr, StaticTy), FD); in emitListToGlobalReduceFunction()
2674 Address GlobAddr = GlobLVal.getAddress(CGF); in emitListToGlobalReduceFunction()
2677 llvm::Value *Ptr = CGF.EmitCastToVoidPtr(BufferPtr); in emitListToGlobalReduceFunction()
2678 CGF.EmitStoreOfScalar(Ptr, Elem, /*Volatile=*/false, C.VoidPtrTy); in emitListToGlobalReduceFunction()
2682 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitListToGlobalReduceFunction()
2683 llvm::Value *Size = CGF.Builder.CreateIntCast( in emitListToGlobalReduceFunction()
2684 CGF.getVLASize( in emitListToGlobalReduceFunction()
2685 CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) in emitListToGlobalReduceFunction()
2687 CGF.SizeTy, /*isSigned=*/false); in emitListToGlobalReduceFunction()
2688 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), in emitListToGlobalReduceFunction()
2695 CGF.EmitCastToVoidPtr(ReductionList.getPointer()); in emitListToGlobalReduceFunction()
2696 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitListToGlobalReduceFunction()
2697 llvm::Value *ReducedPtr = CGF.EmitLoadOfScalar( in emitListToGlobalReduceFunction()
2700 CGF, Loc, ReduceFn, {GlobalReduceList, ReducedPtr}); in emitListToGlobalReduceFunction()
2701 CGF.FinishFunction(); in emitListToGlobalReduceFunction()
2740 CodeGenFunction CGF(CGM); in emitGlobalToListCopyFunction() local
2741 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitGlobalToListCopyFunction()
2743 CGBuilderTy &Bld = CGF.Builder; in emitGlobalToListCopyFunction()
2745 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitGlobalToListCopyFunction()
2746 Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg); in emitGlobalToListCopyFunction()
2749 CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false, in emitGlobalToListCopyFunction()
2751 CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo()), in emitGlobalToListCopyFunction()
2752 CGF.getPointerAlign()); in emitGlobalToListCopyFunction()
2757 CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc), in emitGlobalToListCopyFunction()
2760 llvm::Value *Idxs[] = {llvm::ConstantInt::getNullValue(CGF.Int32Ty), in emitGlobalToListCopyFunction()
2761 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), in emitGlobalToListCopyFunction()
2768 llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar( in emitGlobalToListCopyFunction()
2772 ElemPtrPtr, CGF.ConvertTypeForMem(Private->getType())->getPointerTo()); in emitGlobalToListCopyFunction()
2778 LValue GlobLVal = CGF.EmitLValueForField( in emitGlobalToListCopyFunction()
2779 CGF.MakeNaturalAlignAddrLValue(BufferArrPtr, StaticTy), FD); in emitGlobalToListCopyFunction()
2780 Address GlobAddr = GlobLVal.getAddress(CGF); in emitGlobalToListCopyFunction()
2784 switch (CGF.getEvaluationKind(Private->getType())) { in emitGlobalToListCopyFunction()
2786 llvm::Value *V = CGF.EmitLoadOfScalar(GlobLVal, Loc); in emitGlobalToListCopyFunction()
2787 CGF.EmitStoreOfScalar(V, ElemPtr, /*Volatile=*/false, Private->getType(), in emitGlobalToListCopyFunction()
2793 CodeGenFunction::ComplexPairTy V = CGF.EmitLoadOfComplex(GlobLVal, Loc); in emitGlobalToListCopyFunction()
2794 CGF.EmitStoreOfComplex(V, CGF.MakeAddrLValue(ElemPtr, Private->getType()), in emitGlobalToListCopyFunction()
2799 CGF.EmitAggregateCopy(CGF.MakeAddrLValue(ElemPtr, Private->getType()), in emitGlobalToListCopyFunction()
2807 CGF.FinishFunction(); in emitGlobalToListCopyFunction()
2850 CodeGenFunction CGF(CGM); in emitGlobalToListReduceFunction() local
2851 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitGlobalToListReduceFunction()
2853 CGBuilderTy &Bld = CGF.Builder; in emitGlobalToListReduceFunction()
2855 Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg); in emitGlobalToListReduceFunction()
2860 CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc), in emitGlobalToListReduceFunction()
2866 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); in emitGlobalToListReduceFunction()
2868 llvm::Value *Idxs[] = {llvm::ConstantInt::getNullValue(CGF.Int32Ty), in emitGlobalToListReduceFunction()
2869 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), in emitGlobalToListReduceFunction()
2874 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitGlobalToListReduceFunction()
2878 LValue GlobLVal = CGF.EmitLValueForField( in emitGlobalToListReduceFunction()
2879 CGF.MakeNaturalAlignAddrLValue(BufferArrPtr, StaticTy), FD); in emitGlobalToListReduceFunction()
2880 Address GlobAddr = GlobLVal.getAddress(CGF); in emitGlobalToListReduceFunction()
2883 llvm::Value *Ptr = CGF.EmitCastToVoidPtr(BufferPtr); in emitGlobalToListReduceFunction()
2884 CGF.EmitStoreOfScalar(Ptr, Elem, /*Volatile=*/false, C.VoidPtrTy); in emitGlobalToListReduceFunction()
2888 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitGlobalToListReduceFunction()
2889 llvm::Value *Size = CGF.Builder.CreateIntCast( in emitGlobalToListReduceFunction()
2890 CGF.getVLASize( in emitGlobalToListReduceFunction()
2891 CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) in emitGlobalToListReduceFunction()
2893 CGF.SizeTy, /*isSigned=*/false); in emitGlobalToListReduceFunction()
2894 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), in emitGlobalToListReduceFunction()
2901 CGF.EmitCastToVoidPtr(ReductionList.getPointer()); in emitGlobalToListReduceFunction()
2902 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitGlobalToListReduceFunction()
2903 llvm::Value *ReducedPtr = CGF.EmitLoadOfScalar( in emitGlobalToListReduceFunction()
2906 CGF, Loc, ReduceFn, {ReducedPtr, GlobalReduceList}); in emitGlobalToListReduceFunction()
2907 CGF.FinishFunction(); in emitGlobalToListReduceFunction()
3154 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates, in emitReduction() argument
3157 if (!CGF.HaveInsertPoint()) in emitReduction()
3168 CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs, in emitReduction()
3180 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitReduction()
3181 llvm::Value *ThreadId = getThreadID(CGF, Loc); in emitReduction()
3198 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); in emitReduction()
3202 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitReduction()
3203 CGF.Builder.CreateStore( in emitReduction()
3204 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReduction()
3205 CGF.EmitLValue(RHSExprs[I]).getPointer(CGF), CGF.VoidPtrTy), in emitReduction()
3210 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitReduction()
3211 llvm::Value *Size = CGF.Builder.CreateIntCast( in emitReduction()
3212 CGF.getVLASize( in emitReduction()
3213 CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) in emitReduction()
3215 CGF.SizeTy, /*isSigned=*/false); in emitReduction()
3216 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), in emitReduction()
3221 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReduction()
3222 ReductionList.getPointer(), CGF.VoidPtrTy); in emitReduction()
3224 Loc, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), Privates, in emitReduction()
3226 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy); in emitReduction()
3235 CGF.Builder.getInt32(RHSExprs.size()), in emitReduction()
3241 Res = CGF.EmitRuntimeCall( in emitReduction()
3264 llvm::Value *GlobalBufferPtr = CGF.EmitLoadOfScalar( in emitReduction()
3282 CGF.Builder.getInt32(C.getLangOpts().OpenMPCUDAReductionBufNum), in emitReduction()
3291 Res = CGF.EmitRuntimeCall( in emitReduction()
3298 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.reduction.done"); in emitReduction()
3299 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.then"); in emitReduction()
3300 llvm::Value *Cond = CGF.Builder.CreateICmpEQ( in emitReduction()
3302 CGF.Builder.CreateCondBr(Cond, ThenBB, ExitBB); in emitReduction()
3308 CGF.EmitBlock(ThenBB); in emitReduction()
3312 this](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
3317 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), in emitReduction()
3332 RCG(CGF); in emitReduction()
3334 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitReduction()
3335 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in emitReduction()
3371 CGOpenMPRuntimeGPU::getParameterAddress(CodeGenFunction &CGF, in getParameterAddress() argument
3377 Address LocalAddr = CGF.GetAddrOfLocalVar(TargetParam); in getParameterAddress()
3383 CGF.getContext().getTargetAddressSpace(NativePointeeTy); in getParameterAddress()
3385 llvm::Value *TargetAddr = CGF.EmitLoadOfScalar( in getParameterAddress()
3388 TargetAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getParameterAddress()
3392 TargetAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getParameterAddress()
3395 Address NativeParamAddr = CGF.CreateMemTemp(NativeParamType); in getParameterAddress()
3396 CGF.EmitStoreOfScalar(TargetAddr, NativeParamAddr, /*Volatile=*/false, in getParameterAddress()
3402 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, in emitOutlinedFunctionCall() argument
3418 llvm::Value *TargetArg = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitOutlinedFunctionCall()
3422 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TargetArg, TargetType)); in emitOutlinedFunctionCall()
3424 CGOpenMPRuntime::emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, TargetArgs); in emitOutlinedFunctionCall()
3470 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true); in createParallelDataSharingWrapper() local
3471 CGF.StartFunction(GlobalDecl(), Ctx.VoidTy, Fn, CGFI, WrapperArgs, in createParallelDataSharingWrapper()
3477 Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, in createParallelDataSharingWrapper()
3479 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0)); in createParallelDataSharingWrapper()
3483 Args.emplace_back(CGF.GetAddrOfLocalVar(&WrapperArg).getPointer()); in createParallelDataSharingWrapper()
3486 CGBuilderTy &Bld = CGF.Builder; in createParallelDataSharingWrapper()
3492 CGF.CreateDefaultAlignTempAlloca(CGF.VoidPtrPtrTy, "global_args"); in createParallelDataSharingWrapper()
3495 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in createParallelDataSharingWrapper()
3504 SharedArgListAddress = CGF.EmitLoadOfPointer( in createParallelDataSharingWrapper()
3505 GlobalArgs, CGF.getContext() in createParallelDataSharingWrapper()
3506 .getPointerType(CGF.getContext().getPointerType( in createParallelDataSharingWrapper()
3507 CGF.getContext().VoidPtrTy)) in createParallelDataSharingWrapper()
3514 Src, CGF.SizeTy->getPointerTo()); in createParallelDataSharingWrapper()
3515 llvm::Value *LB = CGF.EmitLoadOfScalar( in createParallelDataSharingWrapper()
3518 CGF.getContext().getPointerType(CGF.getContext().getSizeType()), in createParallelDataSharingWrapper()
3524 Src, CGF.SizeTy->getPointerTo()); in createParallelDataSharingWrapper()
3525 llvm::Value *UB = CGF.EmitLoadOfScalar( in createParallelDataSharingWrapper()
3528 CGF.getContext().getPointerType(CGF.getContext().getSizeType()), in createParallelDataSharingWrapper()
3534 ASTContext &CGFContext = CGF.getContext(); in createParallelDataSharingWrapper()
3539 Src, CGF.ConvertTypeForMem(CGFContext.getPointerType(ElemTy))); in createParallelDataSharingWrapper()
3540 llvm::Value *Arg = CGF.EmitLoadOfScalar(TypedAddress, in createParallelDataSharingWrapper()
3546 Arg = castValueToType(CGF, Arg, ElemTy, CGFContext.getUIntPtrType(), in createParallelDataSharingWrapper()
3553 emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedParallelFn, Args); in createParallelDataSharingWrapper()
3554 CGF.FinishFunction(); in createParallelDataSharingWrapper()
3558 void CGOpenMPRuntimeGPU::emitFunctionProlog(CodeGenFunction &CGF, in emitFunctionProlog() argument
3564 assert(FunctionGlobalizedDecls.count(CGF.CurFn) == 0 && in emitFunctionProlog()
3576 NeedToDelayGlobalization = CGF.CapturedStmtInfo->getKind() == CR_OpenMP; in emitFunctionProlog()
3583 CheckVarsEscapingDeclContext VarChecker(CGF, TeamAndReductions.second); in emitFunctionProlog()
3593 auto I = FunctionGlobalizedDecls.try_emplace(CGF.CurFn).first; in emitFunctionProlog()
3607 CheckVarsEscapingDeclContext VarChecker(CGF, llvm::None); in emitFunctionProlog()
3617 emitGenericVarsProlog(CGF, D->getBeginLoc(), /*WithSPMDCheck=*/true); in emitFunctionProlog()
3621 void Emit(CodeGenFunction &CGF, Flags flags) override { in emitFunctionProlog()
3622 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()) in emitFunctionProlog()
3623 .emitGenericVarsEpilog(CGF, /*WithSPMDCheck=*/true); in emitFunctionProlog()
3626 CGF.EHStack.pushCleanup<GlobalizationScope>(NormalAndEHCleanup); in emitFunctionProlog()
3630 Address CGOpenMPRuntimeGPU::getAddressOfLocalVariable(CodeGenFunction &CGF, in getAddressOfLocalVariable() argument
3658 llvm::Type *VarTy = CGF.ConvertTypeForMem(VD->getType()); in getAddressOfLocalVariable()
3668 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddressOfLocalVariable()
3678 auto I = FunctionGlobalizedDecls.find(CGF.CurFn); in getAddressOfLocalVariable()
3699 void CGOpenMPRuntimeGPU::functionFinished(CodeGenFunction &CGF) { in functionFinished() argument
3700 FunctionGlobalizedDecls.erase(CGF.CurFn); in functionFinished()
3701 CGOpenMPRuntime::functionFinished(CGF); in functionFinished()
3705 CodeGenFunction &CGF, const OMPLoopDirective &S, in getDefaultDistScheduleAndChunk() argument
3708 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in getDefaultDistScheduleAndChunk()
3711 Chunk = CGF.EmitScalarConversion( in getDefaultDistScheduleAndChunk()
3712 RT.getGPUNumThreads(CGF), in getDefaultDistScheduleAndChunk()
3713 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0), in getDefaultDistScheduleAndChunk()
3718 CGF, S, ScheduleKind, Chunk); in getDefaultDistScheduleAndChunk()
3722 CodeGenFunction &CGF, const OMPLoopDirective &S, in getDefaultScheduleAndChunk() argument
3728 ChunkExpr = IntegerLiteral::Create(CGF.getContext(), ChunkSize, in getDefaultScheduleAndChunk()
3729 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0), in getDefaultScheduleAndChunk()
3734 CodeGenFunction &CGF, const OMPExecutableDirective &D) const { in adjustTargetSpecificDataForLambdas() argument
3750 Address VDAddr = CGF.GetAddrOfLocalVar(VD); in adjustTargetSpecificDataForLambdas()
3753 VDLVal = CGF.EmitLoadOfReferenceLValue(VDAddr, VD->getType()); in adjustTargetSpecificDataForLambdas()
3755 VDLVal = CGF.MakeAddrLValue( in adjustTargetSpecificDataForLambdas()
3760 if (ThisCapture && CGF.CapturedStmtInfo->isCXXThisExprCaptured()) { in adjustTargetSpecificDataForLambdas()
3762 CGF.EmitLValueForFieldInitialization(VDLVal, ThisCapture); in adjustTargetSpecificDataForLambdas()
3763 llvm::Value *CXXThis = CGF.LoadCXXThis(); in adjustTargetSpecificDataForLambdas()
3764 CGF.EmitStoreOfScalar(CXXThis, ThisLVal); in adjustTargetSpecificDataForLambdas()
3774 LValue VarLVal = CGF.EmitLValueForFieldInitialization(VDLVal, It->second); in adjustTargetSpecificDataForLambdas()
3775 Address VDAddr = CGF.GetAddrOfLocalVar(VD); in adjustTargetSpecificDataForLambdas()
3777 VDAddr = CGF.EmitLoadOfReferenceLValue(VDAddr, in adjustTargetSpecificDataForLambdas()
3779 .getAddress(CGF); in adjustTargetSpecificDataForLambdas()
3780 CGF.EmitStoreOfScalar(VDAddr.getPointer(), VarLVal); in adjustTargetSpecificDataForLambdas()