Lines Matching refs:CGF
30 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) { in emitPreInitStmt() argument
37 CGF.EmitVarDecl(cast<VarDecl>(*I)); in emitPreInitStmt()
40 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in emitPreInitStmt()
41 CGF.EmitAutoVarCleanups(Emission); in emitPreInitStmt()
50 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) { in isCapturedVar() argument
51 return CGF.LambdaCaptureFields.lookup(VD) || in isCapturedVar()
52 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) || in isCapturedVar()
53 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl)); in isCapturedVar()
58 CodeGenFunction &CGF, const OMPExecutableDirective &S, in OMPLexicalScope() argument
61 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()), in OMPLexicalScope()
62 InlinedShareds(CGF) { in OMPLexicalScope()
64 emitPreInitStmt(CGF, S); in OMPLexicalScope()
76 CGF.getContext(), const_cast<VarDecl *>(VD), in OMPLexicalScope()
77 isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo && in OMPLexicalScope()
80 InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address { in OMPLexicalScope()
81 return CGF.EmitLValue(&DRE).getAddress(); in OMPLexicalScope()
100 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) in OMPParallelScope() argument
101 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None, in OMPParallelScope()
115 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) in OMPTeamsScope() argument
116 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None, in OMPTeamsScope()
123 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) { in emitPreInitStmt() argument
128 CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType())); in emitPreInitStmt()
130 (void)PreCondVars.apply(CGF); in emitPreInitStmt()
133 CGF.EmitVarDecl(cast<VarDecl>(*I)); in emitPreInitStmt()
135 PreCondVars.restore(CGF); in emitPreInitStmt()
139 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S) in OMPLoopScope() argument
140 : CodeGenFunction::RunCleanupsScope(CGF) { in OMPLoopScope()
141 emitPreInitStmt(CGF, S); in OMPLoopScope()
148 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) { in isCapturedVar() argument
149 return CGF.LambdaCaptureFields.lookup(VD) || in isCapturedVar()
150 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) || in isCapturedVar()
151 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) && in isCapturedVar()
152 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD)); in isCapturedVar()
156 OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) in OMPSimdLexicalScope() argument
157 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()), in OMPSimdLexicalScope()
158 InlinedShareds(CGF) { in OMPSimdLexicalScope()
165 CGF.EmitVarDecl(cast<VarDecl>(*I)); in OMPSimdLexicalScope()
168 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in OMPSimdLexicalScope()
169 CGF.EmitAutoVarCleanups(Emission); in OMPSimdLexicalScope()
177 CGF.EmitVarDecl(*OED); in OMPSimdLexicalScope()
182 CGF.EmitOMPPrivateClause(S, InlinedShareds); in OMPSimdLexicalScope()
185 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl())); in OMPSimdLexicalScope()
194 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD), in OMPSimdLexicalScope()
195 isCapturedVar(CGF, VD) || in OMPSimdLexicalScope()
196 (CGF.CapturedStmtInfo && in OMPSimdLexicalScope()
200 InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address { in OMPSimdLexicalScope()
201 return CGF.EmitLValue(&DRE).getAddress(); in OMPSimdLexicalScope()
213 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
299 static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, in castValueFromUintptr() argument
303 ASTContext &Ctx = CGF.getContext(); in castValueFromUintptr()
305 llvm::Value *CastedPtr = CGF.EmitScalarConversion( in castValueFromUintptr()
309 CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType)) in castValueFromUintptr()
317 TmpAddr = CGF.CreateMemTemp(RefType, Twine(Name, ".ref")); in castValueFromUintptr()
318 LValue TmpLVal = CGF.MakeAddrLValue(TmpAddr, RefType); in castValueFromUintptr()
319 CGF.EmitStoreThroughLValue(RValue::get(RefVal), TmpLVal, /*isInit=*/true); in castValueFromUintptr()
364 CodeGenFunction &CGF, FunctionArgList &Args, in emitOutlinedFunctionPrologue() argument
376 CodeGenModule &CGM = CGF.CGM; in emitOutlinedFunctionPrologue()
461 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs, in emitOutlinedFunctionPrologue()
469 LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt], in emitOutlinedFunctionPrologue()
472 LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]); in emitOutlinedFunctionPrologue()
480 Address RefAddr = CGF.CreateMemTemp( in emitOutlinedFunctionPrologue()
482 CGF.EmitStoreOfScalar(LocalAddr.getPointer(), RefAddr, in emitOutlinedFunctionPrologue()
493 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(), in emitOutlinedFunctionPrologue()
497 ArgLVal = CGF.MakeAddrLValue( in emitOutlinedFunctionPrologue()
498 castValueFromUintptr(CGF, I->getLocation(), FD->getType(), in emitOutlinedFunctionPrologue()
502 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation()); in emitOutlinedFunctionPrologue()
511 ArgAddr = CGF.EmitLoadOfReference(ArgLVal); in emitOutlinedFunctionPrologue()
515 ArgAddr = CGF.EmitLoadOfPointer( in emitOutlinedFunctionPrologue()
532 ? castValueFromUintptr(CGF, I->getLocation(), in emitOutlinedFunctionPrologue()
539 CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation()); in emitOutlinedFunctionPrologue()
1097 [&Emission](CodeGenFunction &CGF) { in EmitOMPReductionClauseInit() argument
1098 CGF.EmitAutoVarInit(Emission); in EmitOMPReductionClauseInit()
1191 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitPostUpdateForReductionClause() argument
1193 if (!CGF.HaveInsertPoint()) in emitPostUpdateForReductionClause()
1199 if (llvm::Value *Cond = CondGen(CGF)) { in emitPostUpdateForReductionClause()
1202 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu"); in emitPostUpdateForReductionClause()
1203 DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done"); in emitPostUpdateForReductionClause()
1204 CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB); in emitPostUpdateForReductionClause()
1205 CGF.EmitBlock(ThenBB); in emitPostUpdateForReductionClause()
1208 CGF.EmitIgnoredExpr(PostUpdate); in emitPostUpdateForReductionClause()
1212 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); in emitPostUpdateForReductionClause()
1226 CodeGenFunction &CGF, const OMPExecutableDirective &S, in emitCommonOMPParallelDirective() argument
1231 CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction( in emitCommonOMPParallelDirective()
1234 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); in emitCommonOMPParallelDirective()
1236 CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(), in emitCommonOMPParallelDirective()
1238 CGF.CGM.getOpenMPRuntime().emitNumThreadsClause( in emitCommonOMPParallelDirective()
1239 CGF, NumThreads, NumThreadsClause->getBeginLoc()); in emitCommonOMPParallelDirective()
1242 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF); in emitCommonOMPParallelDirective()
1243 CGF.CGM.getOpenMPRuntime().emitProcBindClause( in emitCommonOMPParallelDirective()
1244 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc()); in emitCommonOMPParallelDirective()
1255 OMPParallelScope Scope(CGF, S); in emitCommonOMPParallelDirective()
1261 CodeGenBoundParameters(CGF, S, CapturedVars); in emitCommonOMPParallelDirective()
1262 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); in emitCommonOMPParallelDirective()
1263 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn, in emitCommonOMPParallelDirective()
1273 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelDirective() argument
1274 Action.Enter(CGF); in EmitOMPParallelDirective()
1275 OMPPrivateScope PrivateScope(CGF); in EmitOMPParallelDirective()
1276 bool Copyins = CGF.EmitOMPCopyinClause(S); in EmitOMPParallelDirective()
1277 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in EmitOMPParallelDirective()
1282 CGF.CGM.getOpenMPRuntime().emitBarrierCall( in EmitOMPParallelDirective()
1283 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, in EmitOMPParallelDirective()
1286 CGF.EmitOMPPrivateClause(S, PrivateScope); in EmitOMPParallelDirective()
1287 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPParallelDirective()
1289 CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt()); in EmitOMPParallelDirective()
1290 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in EmitOMPParallelDirective()
1448 static void emitAlignedClause(CodeGenFunction &CGF, in emitAlignedClause() argument
1450 if (!CGF.HaveInsertPoint()) in emitAlignedClause()
1456 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr)); in emitAlignedClause()
1466 CGF.getContext() in emitAlignedClause()
1467 .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign( in emitAlignedClause()
1474 llvm::Value *PtrValue = CGF.EmitScalarExpr(E); in emitAlignedClause()
1475 CGF.EmitAlignmentAssumption( in emitAlignedClause()
1532 static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, in emitPreCond() argument
1535 if (!CGF.HaveInsertPoint()) in emitPreCond()
1538 CodeGenFunction::OMPPrivateScope PreCondScope(CGF); in emitPreCond()
1539 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope); in emitPreCond()
1543 CGF.EmitIgnoredExpr(I); in emitPreCond()
1547 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount); in emitPreCond()
1585 static void emitSimdlenSafelenClause(CodeGenFunction &CGF, in emitSimdlenSafelenClause() argument
1588 if (!CGF.HaveInsertPoint()) in emitSimdlenSafelenClause()
1591 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(), in emitSimdlenSafelenClause()
1594 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue()); in emitSimdlenSafelenClause()
1599 CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>()); in emitSimdlenSafelenClause()
1601 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(), in emitSimdlenSafelenClause()
1604 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue()); in emitSimdlenSafelenClause()
1608 CGF.LoopStack.setParallel(/*Enable=*/false); in emitSimdlenSafelenClause()
1665 static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, in emitOMPLoopBodyWithStopPoint() argument
1668 CGF.EmitOMPLoopBody(S, LoopExit); in emitOMPLoopBodyWithStopPoint()
1669 CGF.EmitStopPoint(&S); in emitOMPLoopBodyWithStopPoint()
1673 static LValue EmitOMPHelperVar(CodeGenFunction &CGF, in EmitOMPHelperVar() argument
1676 CGF.EmitVarDecl(*VDecl); in EmitOMPHelperVar()
1677 return CGF.EmitLValue(Helper); in EmitOMPHelperVar()
1680 static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, in emitOMPSimdRegion() argument
1682 Action.Enter(CGF); in emitOMPSimdRegion()
1685 OMPLoopScope PreInitScope(CGF, S); in emitOMPSimdRegion()
1694 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable())); in emitOMPSimdRegion()
1695 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable())); in emitOMPSimdRegion()
1703 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { in emitOMPSimdRegion()
1707 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then"); in emitOMPSimdRegion()
1708 ContBlock = CGF.createBasicBlock("simd.if.end"); in emitOMPSimdRegion()
1709 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, in emitOMPSimdRegion()
1710 CGF.getProfileCount(&S)); in emitOMPSimdRegion()
1711 CGF.EmitBlock(ThenBlock); in emitOMPSimdRegion()
1712 CGF.incrementProfileCounter(&S); in emitOMPSimdRegion()
1718 CGF.EmitVarDecl(*IVDecl); in emitOMPSimdRegion()
1719 CGF.EmitIgnoredExpr(S.getInit()); in emitOMPSimdRegion()
1725 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); in emitOMPSimdRegion()
1727 CGF.EmitIgnoredExpr(S.getCalcLastIteration()); in emitOMPSimdRegion()
1730 CGF.EmitOMPSimdInit(S); in emitOMPSimdRegion()
1732 emitAlignedClause(CGF, S); in emitOMPSimdRegion()
1733 (void)CGF.EmitOMPLinearClauseInit(S); in emitOMPSimdRegion()
1735 CodeGenFunction::OMPPrivateScope LoopScope(CGF); in emitOMPSimdRegion()
1736 CGF.EmitOMPPrivateLoopCounters(S, LoopScope); in emitOMPSimdRegion()
1737 CGF.EmitOMPLinearClause(S, LoopScope); in emitOMPSimdRegion()
1738 CGF.EmitOMPPrivateClause(S, LoopScope); in emitOMPSimdRegion()
1739 CGF.EmitOMPReductionClauseInit(S, LoopScope); in emitOMPSimdRegion()
1740 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); in emitOMPSimdRegion()
1743 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitOMPSimdRegion()
1744 CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(), in emitOMPSimdRegion()
1746 [&S](CodeGenFunction &CGF) { in emitOMPSimdRegion() argument
1747 CGF.EmitOMPLoopBody(S, CodeGenFunction::JumpDest()); in emitOMPSimdRegion()
1748 CGF.EmitStopPoint(&S); in emitOMPSimdRegion()
1751 CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; }); in emitOMPSimdRegion()
1754 CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true); in emitOMPSimdRegion()
1755 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd); in emitOMPSimdRegion()
1756 emitPostUpdateForReductionClause(CGF, S, in emitOMPSimdRegion()
1759 CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; }); in emitOMPSimdRegion()
1762 CGF.EmitBranch(ContBlock); in emitOMPSimdRegion()
1763 CGF.EmitBlock(ContBlock, true); in emitOMPSimdRegion()
1768 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPSimdDirective() argument
1769 emitOMPSimdRegion(CGF, S, Action); in EmitOMPSimdDirective()
1851 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) { in EmitOMPOuterLoop() argument
1852 CodeGenLoop(CGF, S, LoopExit); in EmitOMPOuterLoop()
1854 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) { in EmitOMPOuterLoop() argument
1855 CodeGenOrdered(CGF, Loc, IVSize, IVSigned); in EmitOMPOuterLoop()
1872 auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) { in EmitOMPOuterLoop() argument
1874 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), in EmitOMPOuterLoop()
1967 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc, in EmitOMPForOuterLoop()
1971 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize, in EmitOMPForOuterLoop()
2051 emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, in emitDistributeParallelForInnerBounds() argument
2055 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable())); in emitDistributeParallelForInnerBounds()
2057 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable())); in emitDistributeParallelForInnerBounds()
2065 LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable()); in emitDistributeParallelForInnerBounds()
2066 LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable()); in emitDistributeParallelForInnerBounds()
2067 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar( in emitDistributeParallelForInnerBounds()
2069 PrevLBVal = CGF.EmitScalarConversion( in emitDistributeParallelForInnerBounds()
2073 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar( in emitDistributeParallelForInnerBounds()
2075 PrevUBVal = CGF.EmitScalarConversion( in emitDistributeParallelForInnerBounds()
2080 CGF.EmitStoreOfScalar(PrevLBVal, LB); in emitDistributeParallelForInnerBounds()
2081 CGF.EmitStoreOfScalar(PrevUBVal, UB); in emitDistributeParallelForInnerBounds()
2094 emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, in emitDistributeParallelForDispatchBounds() argument
2105 CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc()); in emitDistributeParallelForDispatchBounds()
2107 CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc()); in emitDistributeParallelForDispatchBounds()
2112 CodeGenFunction &CGF, const OMPExecutableDirective &S, in emitDistributeParallelForDistributeInnerBoundParams() argument
2116 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable())); in emitDistributeParallelForDistributeInnerBoundParams()
2117 llvm::Value *LBCast = CGF.Builder.CreateIntCast( in emitDistributeParallelForDistributeInnerBoundParams()
2118 CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false); in emitDistributeParallelForDistributeInnerBoundParams()
2121 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable())); in emitDistributeParallelForDistributeInnerBoundParams()
2123 llvm::Value *UBCast = CGF.Builder.CreateIntCast( in emitDistributeParallelForDistributeInnerBoundParams()
2124 CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false); in emitDistributeParallelForDistributeInnerBoundParams()
2129 emitInnerParallelForWhenCombined(CodeGenFunction &CGF, in emitInnerParallelForWhenCombined() argument
2132 auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF, in emitInnerParallelForWhenCombined()
2134 Action.Enter(CGF); in emitInnerParallelForWhenCombined()
2145 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(), in emitInnerParallelForWhenCombined()
2147 CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(), in emitInnerParallelForWhenCombined()
2153 CGF, S, in emitInnerParallelForWhenCombined()
2161 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeParallelForDirective() argument
2162 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPDistributeParallelForDirective()
2171 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeParallelForSimdDirective() argument
2172 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPDistributeParallelForSimdDirective()
2181 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeSimdDirective() argument
2182 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPDistributeSimdDirective()
2191 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetSimdDeviceFunction() argument
2192 emitOMPSimdRegion(CGF, S, Action); in EmitOMPTargetSimdDeviceFunction()
2204 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetSimdDirective() argument
2205 emitOMPSimdRegion(CGF, S, Action); in EmitOMPTargetSimdDirective()
2380 [&S, LoopExit](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
2381 CGF.EmitOMPLoopBody(S, LoopExit); in EmitOMPWorksharingLoop()
2382 CGF.EmitStopPoint(&S); in EmitOMPWorksharingLoop()
2387 auto &&CodeGen = [&S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
2388 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), in EmitOMPWorksharingLoop()
2407 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
2408 return CGF.Builder.CreateIsNotNull( in EmitOMPWorksharingLoop()
2409 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPWorksharingLoop()
2418 *this, S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
2419 return CGF.Builder.CreateIsNotNull( in EmitOMPWorksharingLoop()
2420 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPWorksharingLoop()
2428 EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
2429 return CGF.Builder.CreateIsNotNull( in EmitOMPWorksharingLoop()
2430 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPWorksharingLoop()
2446 emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) { in emitForLoopBounds() argument
2449 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable())); in emitForLoopBounds()
2451 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable())); in emitForLoopBounds()
2460 emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, in emitDispatchForLoopBounds() argument
2464 const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType()); in emitDispatchForLoopBounds()
2465 llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0); in emitDispatchForLoopBounds()
2466 llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration()); in emitDispatchForLoopBounds()
2472 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF, in EmitOMPForDirective()
2474 OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel()); in EmitOMPForDirective()
2475 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), in EmitOMPForDirective()
2492 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF, in EmitOMPForSimdDirective() argument
2494 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), in EmitOMPForSimdDirective()
2508 static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, in createSectionLVal() argument
2511 LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty); in createSectionLVal()
2513 CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true); in createSectionLVal()
2522 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) { in EmitSections() argument
2523 ASTContext &C = CGF.getContext(); in EmitSections()
2527 LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.", in EmitSections()
2528 CGF.Builder.getInt32(0)); in EmitSections()
2530 ? CGF.Builder.getInt32(CS->size() - 1) in EmitSections()
2531 : CGF.Builder.getInt32(0); in EmitSections()
2533 createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal); in EmitSections()
2534 LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.", in EmitSections()
2535 CGF.Builder.getInt32(1)); in EmitSections()
2536 LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.", in EmitSections()
2537 CGF.Builder.getInt32(0)); in EmitSections()
2539 LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv."); in EmitSections()
2541 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV); in EmitSections()
2543 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB); in EmitSections()
2550 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) { in EmitSections() argument
2562 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit"); in EmitSections()
2564 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()), in EmitSections()
2569 auto CaseBB = CGF.createBasicBlock(".omp.sections.case"); in EmitSections()
2570 CGF.EmitBlock(CaseBB); in EmitSections()
2571 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB); in EmitSections()
2572 CGF.EmitStmt(SubStmt); in EmitSections()
2573 CGF.EmitBranch(ExitBB); in EmitSections()
2577 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case"); in EmitSections()
2578 CGF.EmitBlock(CaseBB); in EmitSections()
2579 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB); in EmitSections()
2580 CGF.EmitStmt(CapturedStmt); in EmitSections()
2581 CGF.EmitBranch(ExitBB); in EmitSections()
2583 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in EmitSections()
2586 CodeGenFunction::OMPPrivateScope LoopScope(CGF); in EmitSections()
2587 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) { in EmitSections()
2591 CGF.CGM.getOpenMPRuntime().emitBarrierCall( in EmitSections()
2592 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, in EmitSections()
2595 CGF.EmitOMPPrivateClause(S, LoopScope); in EmitSections()
2596 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); in EmitSections()
2597 CGF.EmitOMPReductionClauseInit(S, LoopScope); in EmitSections()
2600 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in EmitSections()
2608 CGF.CGM.getOpenMPRuntime().emitForStaticInit( in EmitSections()
2609 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit); in EmitSections()
2611 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc()); in EmitSections()
2612 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect( in EmitSections()
2613 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal); in EmitSections()
2614 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB); in EmitSections()
2616 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV); in EmitSections()
2618 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen, in EmitSections()
2621 auto &&CodeGen = [&S](CodeGenFunction &CGF) { in EmitSections() argument
2622 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), in EmitSections()
2625 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen); in EmitSections()
2626 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in EmitSections()
2628 emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) { in EmitSections() argument
2629 return CGF.Builder.CreateIsNotNull( in EmitSections()
2630 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitSections()
2635 CGF.EmitOMPLastprivateClauseFinal( in EmitSections()
2637 CGF.Builder.CreateIsNotNull( in EmitSections()
2638 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()))); in EmitSections()
2673 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPSectionDirective() argument
2674 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in EmitOMPSectionDirective()
2699 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPSingleDirective() argument
2700 Action.Enter(CGF); in EmitOMPSingleDirective()
2701 OMPPrivateScope SingleScope(CGF); in EmitOMPSingleDirective()
2702 (void)CGF.EmitOMPFirstprivateClause(S, SingleScope); in EmitOMPSingleDirective()
2703 CGF.EmitOMPPrivateClause(S, SingleScope); in EmitOMPSingleDirective()
2705 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in EmitOMPSingleDirective()
2723 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPMasterDirective() argument
2724 Action.Enter(CGF); in EmitOMPMasterDirective()
2725 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in EmitOMPMasterDirective()
2732 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPCriticalDirective() argument
2733 Action.Enter(CGF); in EmitOMPCriticalDirective()
2734 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in EmitOMPCriticalDirective()
2749 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelForDirective() argument
2750 Action.Enter(CGF); in EmitOMPParallelForDirective()
2751 OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel()); in EmitOMPParallelForDirective()
2752 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, in EmitOMPParallelForDirective()
2763 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelForSimdDirective() argument
2764 Action.Enter(CGF); in EmitOMPParallelForSimdDirective()
2765 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, in EmitOMPParallelForSimdDirective()
2776 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelSectionsDirective() argument
2777 Action.Enter(CGF); in EmitOMPParallelSectionsDirective()
2778 CGF.EmitSections(S); in EmitOMPParallelSectionsDirective()
2891 CapturedRegion](CodeGenFunction &CGF, in EmitOMPTaskBasedDirective()
2894 OMPPrivateScope Scope(CGF); in EmitOMPTaskBasedDirective()
2898 llvm::Value *CopyFn = CGF.Builder.CreateLoad( in EmitOMPTaskBasedDirective()
2899 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam))); in EmitOMPTaskBasedDirective()
2900 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar( in EmitOMPTaskBasedDirective()
2908 Address PrivatePtr = CGF.CreateMemTemp( in EmitOMPTaskBasedDirective()
2909 CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr"); in EmitOMPTaskBasedDirective()
2916 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), in EmitOMPTaskBasedDirective()
2924 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), in EmitOMPTaskBasedDirective()
2929 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(), in EmitOMPTaskBasedDirective()
2933 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD), in EmitOMPTaskBasedDirective()
2935 CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr, in EmitOMPTaskBasedDirective()
2938 Scope.addPrivate(Pair.first, [&CGF, &DRE]() { in EmitOMPTaskBasedDirective()
2939 return CGF.EmitLValue(&DRE).getAddress(); in EmitOMPTaskBasedDirective()
2943 Address Replacement(CGF.Builder.CreateLoad(Pair.second), in EmitOMPTaskBasedDirective()
2944 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTaskBasedDirective()
2949 OMPLexicalScope LexScope(CGF, S, CapturedRegion); in EmitOMPTaskBasedDirective()
2952 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad( in EmitOMPTaskBasedDirective()
2953 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9))); in EmitOMPTaskBasedDirective()
2955 RedCG.emitSharedLValue(CGF, Cnt); in EmitOMPTaskBasedDirective()
2956 RedCG.emitAggregateType(CGF, Cnt); in EmitOMPTaskBasedDirective()
2960 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(), in EmitOMPTaskBasedDirective()
2962 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( in EmitOMPTaskBasedDirective()
2963 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); in EmitOMPTaskBasedDirective()
2965 Address(CGF.EmitScalarConversion( in EmitOMPTaskBasedDirective()
2966 Replacement.getPointer(), CGF.getContext().VoidPtrTy, in EmitOMPTaskBasedDirective()
2967 CGF.getContext().getPointerType( in EmitOMPTaskBasedDirective()
2971 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); in EmitOMPTaskBasedDirective()
2998 OMPPrivateScope InRedScope(CGF); in EmitOMPTaskBasedDirective()
3002 RedCG.emitSharedLValue(CGF, Cnt); in EmitOMPTaskBasedDirective()
3003 RedCG.emitAggregateType(CGF, Cnt); in EmitOMPTaskBasedDirective()
3009 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(), in EmitOMPTaskBasedDirective()
3012 CGF.EmitLoadOfScalar(CGF.EmitLValue(TaskgroupDescriptors[Cnt]), in EmitOMPTaskBasedDirective()
3014 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( in EmitOMPTaskBasedDirective()
3015 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); in EmitOMPTaskBasedDirective()
3017 CGF.EmitScalarConversion( in EmitOMPTaskBasedDirective()
3018 Replacement.getPointer(), CGF.getContext().VoidPtrTy, in EmitOMPTaskBasedDirective()
3019 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()), in EmitOMPTaskBasedDirective()
3022 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); in EmitOMPTaskBasedDirective()
3029 Action.Enter(CGF); in EmitOMPTaskBasedDirective()
3030 BodyGen(CGF); in EmitOMPTaskBasedDirective()
3127 &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTaskBasedDirective() argument
3129 OMPPrivateScope Scope(CGF); in EmitOMPTargetTaskBasedDirective()
3132 llvm::Value *CopyFn = CGF.Builder.CreateLoad( in EmitOMPTargetTaskBasedDirective()
3133 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam))); in EmitOMPTargetTaskBasedDirective()
3134 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar( in EmitOMPTargetTaskBasedDirective()
3143 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), in EmitOMPTargetTaskBasedDirective()
3148 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(), in EmitOMPTargetTaskBasedDirective()
3151 Address Replacement(CGF.Builder.CreateLoad(Pair.second), in EmitOMPTargetTaskBasedDirective()
3152 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTargetTaskBasedDirective()
3159 InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
3160 CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0, CGF.getPointerSize()); in EmitOMPTargetTaskBasedDirective()
3161 InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
3162 CGF.GetAddrOfLocalVar(PVD), /*Index=*/0, CGF.getPointerSize()); in EmitOMPTargetTaskBasedDirective()
3163 InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
3164 CGF.GetAddrOfLocalVar(SVD), /*Index=*/0, CGF.getSizeSize()); in EmitOMPTargetTaskBasedDirective()
3167 Action.Enter(CGF); in EmitOMPTargetTaskBasedDirective()
3168 OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false); in EmitOMPTargetTaskBasedDirective()
3169 BodyGen(CGF); in EmitOMPTargetTaskBasedDirective()
3200 auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskDirective() argument
3201 CGF.EmitStmt(CS->getCapturedStmt()); in EmitOMPTaskDirective()
3204 IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn, in EmitOMPTaskDirective()
3206 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn, in EmitOMPTaskDirective()
3228 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTaskgroupDirective() argument
3229 Action.Enter(CGF); in EmitOMPTaskgroupDirective()
3252 CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(), in EmitOMPTaskgroupDirective()
3255 CGF.EmitVarDecl(*VD); in EmitOMPTaskgroupDirective()
3256 CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD), in EmitOMPTaskgroupDirective()
3259 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in EmitOMPTaskgroupDirective()
3442 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
3443 CodeGenLoop(CGF, S, LoopExit); in EmitOMPDistributeLoop()
3445 [&S, StaticChunked](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
3447 CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound()); in EmitOMPDistributeLoop()
3448 CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound()); in EmitOMPDistributeLoop()
3449 CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound()); in EmitOMPDistributeLoop()
3450 CGF.EmitIgnoredExpr(S.getCombinedInit()); in EmitOMPDistributeLoop()
3466 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
3467 return CGF.Builder.CreateIsNotNull( in EmitOMPDistributeLoop()
3468 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPDistributeLoop()
3477 *this, S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
3478 return CGF.Builder.CreateIsNotNull( in EmitOMPDistributeLoop()
3479 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPDistributeLoop()
3500 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeDirective() argument
3501 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPDistributeDirective()
3509 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true); in emitOutlinedOrderedFunction() local
3511 CGF.CapturedStmtInfo = &CapStmtInfo; in emitOutlinedOrderedFunction()
3512 llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S); in emitOutlinedOrderedFunction()
3526 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF, in EmitOMPOrderedDirective()
3531 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); in EmitOMPOrderedDirective()
3533 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(), in EmitOMPOrderedDirective()
3536 Action.Enter(CGF); in EmitOMPOrderedDirective()
3537 CGF.EmitStmt(CS->getCapturedStmt()); in EmitOMPOrderedDirective()
3544 static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val, in convertToScalarValue() argument
3547 assert(CGF.hasScalarEvaluationKind(DestType) && in convertToScalarValue()
3550 return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, in convertToScalarValue()
3552 : CGF.EmitComplexToScalarConversion( in convertToScalarValue()
3557 convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, in convertToComplexValue() argument
3559 assert(CGF.getEvaluationKind(DestType) == TEK_Complex && in convertToComplexValue()
3566 llvm::Value *ScalarVal = CGF.EmitScalarConversion( in convertToComplexValue()
3575 ComplexVal.first = CGF.EmitScalarConversion( in convertToComplexValue()
3577 ComplexVal.second = CGF.EmitScalarConversion( in convertToComplexValue()
3583 static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst, in emitSimpleAtomicStore() argument
3586 CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal); in emitSimpleAtomicStore()
3588 CGF.EmitAtomicStore(RVal, LVal, in emitSimpleAtomicStore()
3613 static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, in emitOMPAtomicReadExpr() argument
3619 LValue XLValue = CGF.EmitLValue(X); in emitOMPAtomicReadExpr()
3620 LValue VLValue = CGF.EmitLValue(V); in emitOMPAtomicReadExpr()
3622 ? CGF.EmitLoadOfLValue(XLValue, Loc) in emitOMPAtomicReadExpr()
3623 : CGF.EmitAtomicLoad( in emitOMPAtomicReadExpr()
3633 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); in emitOMPAtomicReadExpr()
3634 CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc); in emitOMPAtomicReadExpr()
3637 static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, in emitOMPAtomicWriteExpr() argument
3642 emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E)); in emitOMPAtomicWriteExpr()
3648 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); in emitOMPAtomicWriteExpr()
3651 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, in emitOMPAtomicRMW() argument
3656 ASTContext &Context = CGF.getContext(); in emitOMPAtomicRMW()
3736 UpdateVal = CGF.Builder.CreateIntCast( in emitOMPAtomicRMW()
3741 CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO); in emitOMPAtomicRMW()
3769 static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, in emitOMPAtomicUpdateExpr() argument
3783 LValue XLValue = CGF.EmitLValue(X); in emitOMPAtomicUpdateExpr()
3784 RValue ExprRValue = CGF.EmitAnyExpr(E); in emitOMPAtomicUpdateExpr()
3792 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) { in emitOMPAtomicUpdateExpr()
3793 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); in emitOMPAtomicUpdateExpr()
3794 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); in emitOMPAtomicUpdateExpr()
3795 return CGF.EmitAnyExpr(UE); in emitOMPAtomicUpdateExpr()
3797 (void)CGF.EmitOMPAtomicSimpleUpdateExpr( in emitOMPAtomicUpdateExpr()
3804 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); in emitOMPAtomicUpdateExpr()
3807 static RValue convertToType(CodeGenFunction &CGF, RValue Value, in convertToType() argument
3810 switch (CGF.getEvaluationKind(ResType)) { in convertToType()
3813 convertToScalarValue(CGF, Value, SourceType, ResType, Loc)); in convertToType()
3815 auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc); in convertToType()
3824 static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst, in emitOMPAtomicCaptureExpr() argument
3832 LValue VLValue = CGF.EmitLValue(V); in emitOMPAtomicCaptureExpr()
3833 LValue XLValue = CGF.EmitLValue(X); in emitOMPAtomicCaptureExpr()
3834 RValue ExprRValue = CGF.EmitAnyExpr(E); in emitOMPAtomicCaptureExpr()
3855 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr, in emitOMPAtomicCaptureExpr()
3857 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); in emitOMPAtomicCaptureExpr()
3858 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); in emitOMPAtomicCaptureExpr()
3859 RValue Res = CGF.EmitAnyExpr(UE); in emitOMPAtomicCaptureExpr()
3863 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr( in emitOMPAtomicCaptureExpr()
3873 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); in emitOMPAtomicCaptureExpr()
3874 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second); in emitOMPAtomicCaptureExpr()
3875 NewVVal = CGF.EmitAnyExpr(UE); in emitOMPAtomicCaptureExpr()
3881 ExprRValue = convertToType(CGF, ExprRValue, E->getType(), in emitOMPAtomicCaptureExpr()
3888 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr( in emitOMPAtomicCaptureExpr()
3897 CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc); in emitOMPAtomicCaptureExpr()
3903 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); in emitOMPAtomicCaptureExpr()
3906 static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, in emitOMPAtomicExpr() argument
3913 emitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc); in emitOMPAtomicExpr()
3916 emitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc); in emitOMPAtomicExpr()
3920 emitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc); in emitOMPAtomicExpr()
3923 emitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE, in emitOMPAtomicExpr()
4003 auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF, in EmitOMPAtomicDirective()
4005 CGF.EmitStopPoint(CS); in EmitOMPAtomicDirective()
4006 emitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(), in EmitOMPAtomicDirective()
4014 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, in emitCommonOMPTargetDirective() argument
4018 CodeGenModule &CGM = CGF.CGM; in emitCommonOMPTargetDirective()
4022 OMPLexicalScope Scope(CGF, S, OMPD_target); in emitCommonOMPTargetDirective()
4024 CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitCommonOMPTargetDirective() argument
4025 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in emitCommonOMPTargetDirective()
4054 if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val) in emitCommonOMPTargetDirective()
4060 assert(CGF.CurFuncDecl && "No parent declaration for target region!"); in emitCommonOMPTargetDirective()
4064 if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl)) in emitCommonOMPTargetDirective()
4066 else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl)) in emitCommonOMPTargetDirective()
4070 CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl))); in emitCommonOMPTargetDirective()
4075 OMPLexicalScope Scope(CGF, S, OMPD_task); in emitCommonOMPTargetDirective()
4076 auto &&SizeEmitter = [](CodeGenFunction &CGF, const OMPLoopDirective &D) { in emitCommonOMPTargetDirective() argument
4077 OMPLoopScope(CGF, D); in emitCommonOMPTargetDirective()
4079 llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations()); in emitCommonOMPTargetDirective()
4080 NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty, in emitCommonOMPTargetDirective()
4084 CGM.getOpenMPRuntime().emitTargetNumIterationsCall(CGF, S, Device, in emitCommonOMPTargetDirective()
4086 CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device); in emitCommonOMPTargetDirective()
4089 static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, in emitTargetRegion() argument
4091 Action.Enter(CGF); in emitTargetRegion()
4092 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetRegion()
4093 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in emitTargetRegion()
4094 CGF.EmitOMPPrivateClause(S, PrivateScope); in emitTargetRegion()
4097 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitTargetRegion()
4099 CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt()); in emitTargetRegion()
4105 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetDeviceFunction() argument
4106 emitTargetRegion(CGF, S, Action); in EmitOMPTargetDeviceFunction()
4117 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetDirective() argument
4118 emitTargetRegion(CGF, S, Action); in EmitOMPTargetDirective()
4123 static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, in emitCommonOMPTeamsDirective() argument
4129 CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction( in emitCommonOMPTeamsDirective()
4138 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit, in emitCommonOMPTeamsDirective()
4142 OMPTeamsScope Scope(CGF, S); in emitCommonOMPTeamsDirective()
4144 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); in emitCommonOMPTeamsDirective()
4145 CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn, in emitCommonOMPTeamsDirective()
4151 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTeamsDirective() argument
4152 Action.Enter(CGF); in EmitOMPTeamsDirective()
4153 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDirective()
4154 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in EmitOMPTeamsDirective()
4155 CGF.EmitOMPPrivateClause(S, PrivateScope); in EmitOMPTeamsDirective()
4156 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDirective()
4158 CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt()); in EmitOMPTeamsDirective()
4159 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDirective()
4166 static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, in emitTargetTeamsRegion() argument
4169 Action.Enter(CGF); in emitTargetTeamsRegion()
4171 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetTeamsRegion() argument
4172 Action.Enter(CGF); in emitTargetTeamsRegion()
4173 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsRegion()
4174 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in emitTargetTeamsRegion()
4175 CGF.EmitOMPPrivateClause(S, PrivateScope); in emitTargetTeamsRegion()
4176 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsRegion()
4179 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitTargetTeamsRegion()
4180 CGF.EmitStmt(CS->getCapturedStmt()); in emitTargetTeamsRegion()
4181 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsRegion()
4183 emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen); in emitTargetTeamsRegion()
4184 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsRegion()
4191 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDeviceFunction() argument
4192 emitTargetTeamsRegion(CGF, Action, S); in EmitOMPTargetTeamsDeviceFunction()
4204 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDirective() argument
4205 emitTargetTeamsRegion(CGF, Action, S); in EmitOMPTargetTeamsDirective()
4211 emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, in emitTargetTeamsDistributeRegion() argument
4213 Action.Enter(CGF); in emitTargetTeamsDistributeRegion()
4214 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeRegion() argument
4215 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in emitTargetTeamsDistributeRegion()
4219 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeRegion()
4221 Action.Enter(CGF); in emitTargetTeamsDistributeRegion()
4222 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeRegion()
4223 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeRegion()
4225 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in emitTargetTeamsDistributeRegion()
4227 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeRegion()
4229 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen); in emitTargetTeamsDistributeRegion()
4230 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeRegion()
4237 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeDeviceFunction() argument
4238 emitTargetTeamsDistributeRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeDeviceFunction()
4250 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeDirective() argument
4251 emitTargetTeamsDistributeRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeDirective()
4257 CodeGenFunction &CGF, PrePostActionTy &Action, in emitTargetTeamsDistributeSimdRegion() argument
4259 Action.Enter(CGF); in emitTargetTeamsDistributeSimdRegion()
4260 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeSimdRegion() argument
4261 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in emitTargetTeamsDistributeSimdRegion()
4265 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeSimdRegion()
4267 Action.Enter(CGF); in emitTargetTeamsDistributeSimdRegion()
4268 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeSimdRegion()
4269 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeSimdRegion()
4271 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in emitTargetTeamsDistributeSimdRegion()
4273 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeSimdRegion()
4275 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen); in emitTargetTeamsDistributeSimdRegion()
4276 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeSimdRegion()
4283 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeSimdDeviceFunction() argument
4284 emitTargetTeamsDistributeSimdRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeSimdDeviceFunction()
4296 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeSimdDirective() argument
4297 emitTargetTeamsDistributeSimdRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeSimdDirective()
4305 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeDirective() argument
4306 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPTeamsDistributeDirective()
4310 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeDirective()
4312 Action.Enter(CGF); in EmitOMPTeamsDistributeDirective()
4313 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeDirective()
4314 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeDirective()
4316 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in EmitOMPTeamsDistributeDirective()
4318 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeDirective()
4327 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeSimdDirective() argument
4328 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPTeamsDistributeSimdDirective()
4332 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeSimdDirective()
4334 Action.Enter(CGF); in EmitOMPTeamsDistributeSimdDirective()
4335 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeSimdDirective()
4336 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeSimdDirective()
4338 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd, in EmitOMPTeamsDistributeSimdDirective()
4340 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeSimdDirective()
4349 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeParallelForDirective() argument
4350 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPTeamsDistributeParallelForDirective()
4355 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeParallelForDirective()
4357 Action.Enter(CGF); in EmitOMPTeamsDistributeParallelForDirective()
4358 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeParallelForDirective()
4359 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeParallelForDirective()
4361 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in EmitOMPTeamsDistributeParallelForDirective()
4363 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeParallelForDirective()
4372 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeParallelForSimdDirective() argument
4373 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPTeamsDistributeParallelForSimdDirective()
4378 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeParallelForSimdDirective()
4380 Action.Enter(CGF); in EmitOMPTeamsDistributeParallelForSimdDirective()
4381 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeParallelForSimdDirective()
4382 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeParallelForSimdDirective()
4384 CGF.CGM.getOpenMPRuntime().emitInlinedDirective( in EmitOMPTeamsDistributeParallelForSimdDirective()
4385 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); in EmitOMPTeamsDistributeParallelForSimdDirective()
4386 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeParallelForSimdDirective()
4394 CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, in emitTargetTeamsDistributeParallelForRegion() argument
4396 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForRegion()
4397 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeParallelForRegion() argument
4398 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in emitTargetTeamsDistributeParallelForRegion()
4403 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeParallelForRegion()
4405 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForRegion()
4406 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeParallelForRegion()
4407 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeParallelForRegion()
4409 CGF.CGM.getOpenMPRuntime().emitInlinedDirective( in emitTargetTeamsDistributeParallelForRegion()
4410 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); in emitTargetTeamsDistributeParallelForRegion()
4411 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeParallelForRegion()
4414 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for, in emitTargetTeamsDistributeParallelForRegion()
4416 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeParallelForRegion()
4425 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForDeviceFunction() argument
4426 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForDeviceFunction()
4438 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForDirective() argument
4439 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForDirective()
4445 CodeGenFunction &CGF, in emitTargetTeamsDistributeParallelForSimdRegion() argument
4448 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForSimdRegion()
4449 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeParallelForSimdRegion() argument
4450 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in emitTargetTeamsDistributeParallelForSimdRegion()
4455 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeParallelForSimdRegion()
4457 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForSimdRegion()
4458 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeParallelForSimdRegion()
4459 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeParallelForSimdRegion()
4461 CGF.CGM.getOpenMPRuntime().emitInlinedDirective( in emitTargetTeamsDistributeParallelForSimdRegion()
4462 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); in emitTargetTeamsDistributeParallelForSimdRegion()
4463 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeParallelForSimdRegion()
4466 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd, in emitTargetTeamsDistributeParallelForSimdRegion()
4468 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeParallelForSimdRegion()
4477 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction() argument
4478 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction()
4490 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForSimdDirective() argument
4491 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForSimdDirective()
4608 void Enter(CodeGenFunction &CGF) override { in EmitOMPTargetDataDirective() argument
4615 CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetDataDirective() argument
4616 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTargetDataDirective() argument
4617 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in EmitOMPTargetDataDirective()
4622 &InnermostCodeGen](CodeGenFunction &CGF, in EmitOMPTargetDataDirective()
4629 Action.Enter(CGF); in EmitOMPTargetDataDirective()
4632 OMPPrivateScope PrivateScope(CGF); in EmitOMPTargetDataDirective()
4635 CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope, in EmitOMPTargetDataDirective()
4638 RCG(CGF); in EmitOMPTargetDataDirective()
4640 RCG(CGF); in EmitOMPTargetDataDirective()
4651 OMPLexicalScope Scope(CGF, S); in EmitOMPTargetDataDirective()
4652 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data, in EmitOMPTargetDataDirective()
4725 static void emitTargetParallelRegion(CodeGenFunction &CGF, in emitTargetParallelRegion() argument
4730 Action.Enter(CGF); in emitTargetParallelRegion()
4731 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetParallelRegion() argument
4732 Action.Enter(CGF); in emitTargetParallelRegion()
4733 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetParallelRegion()
4734 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in emitTargetParallelRegion()
4735 CGF.EmitOMPPrivateClause(S, PrivateScope); in emitTargetParallelRegion()
4736 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetParallelRegion()
4739 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitTargetParallelRegion()
4741 CGF.EmitStmt(CS->getCapturedStmt()); in emitTargetParallelRegion()
4742 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in emitTargetParallelRegion()
4744 emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen, in emitTargetParallelRegion()
4746 emitPostUpdateForReductionClause(CGF, S, in emitTargetParallelRegion()
4753 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelDeviceFunction() argument
4754 emitTargetParallelRegion(CGF, S, Action); in EmitOMPTargetParallelDeviceFunction()
4766 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelDirective() argument
4767 emitTargetParallelRegion(CGF, S, Action); in EmitOMPTargetParallelDirective()
4772 static void emitTargetParallelForRegion(CodeGenFunction &CGF, in emitTargetParallelForRegion() argument
4775 Action.Enter(CGF); in emitTargetParallelForRegion()
4778 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetParallelForRegion() argument
4779 Action.Enter(CGF); in emitTargetParallelForRegion()
4781 CGF, OMPD_target_parallel_for, S.hasCancel()); in emitTargetParallelForRegion()
4782 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, in emitTargetParallelForRegion()
4785 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen, in emitTargetParallelForRegion()
4793 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForDeviceFunction() argument
4794 emitTargetParallelForRegion(CGF, S, Action); in EmitOMPTargetParallelForDeviceFunction()
4806 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForDirective() argument
4807 emitTargetParallelForRegion(CGF, S, Action); in EmitOMPTargetParallelForDirective()
4813 emitTargetParallelForSimdRegion(CodeGenFunction &CGF, in emitTargetParallelForSimdRegion() argument
4816 Action.Enter(CGF); in emitTargetParallelForSimdRegion()
4819 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetParallelForSimdRegion() argument
4820 Action.Enter(CGF); in emitTargetParallelForSimdRegion()
4821 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, in emitTargetParallelForSimdRegion()
4824 emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen, in emitTargetParallelForSimdRegion()
4832 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForSimdDeviceFunction() argument
4833 emitTargetParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetParallelForSimdDeviceFunction()
4845 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForSimdDirective() argument
4846 emitTargetParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetParallelForSimdDirective()
4852 static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, in mapParam() argument
4857 [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); }); in mapParam()
4891 auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
4903 OMPLoopScope PreInitScope(CGF, S); in EmitOMPTaskLoopBasedDirective()
4904 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { in EmitOMPTaskLoopBasedDirective()
4908 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then"); in EmitOMPTaskLoopBasedDirective()
4909 ContBlock = CGF.createBasicBlock("taskloop.if.end"); in EmitOMPTaskLoopBasedDirective()
4910 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, in EmitOMPTaskLoopBasedDirective()
4911 CGF.getProfileCount(&S)); in EmitOMPTaskLoopBasedDirective()
4912 CGF.EmitBlock(ThenBlock); in EmitOMPTaskLoopBasedDirective()
4913 CGF.incrementProfileCounter(&S); in EmitOMPTaskLoopBasedDirective()
4917 CGF.EmitOMPSimdInit(S); in EmitOMPTaskLoopBasedDirective()
4919 OMPPrivateScope LoopScope(CGF); in EmitOMPTaskLoopBasedDirective()
4927 mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP, in EmitOMPTaskLoopBasedDirective()
4929 mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP, in EmitOMPTaskLoopBasedDirective()
4931 mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope); in EmitOMPTaskLoopBasedDirective()
4932 mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP, in EmitOMPTaskLoopBasedDirective()
4934 CGF.EmitOMPPrivateLoopCounters(S, LoopScope); in EmitOMPTaskLoopBasedDirective()
4935 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); in EmitOMPTaskLoopBasedDirective()
4940 CGF.EmitVarDecl(*IVDecl); in EmitOMPTaskLoopBasedDirective()
4941 CGF.EmitIgnoredExpr(S.getInit()); in EmitOMPTaskLoopBasedDirective()
4947 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); in EmitOMPTaskLoopBasedDirective()
4949 CGF.EmitIgnoredExpr(S.getCalcLastIteration()); in EmitOMPTaskLoopBasedDirective()
4952 CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(), in EmitOMPTaskLoopBasedDirective()
4954 [&S](CodeGenFunction &CGF) { in EmitOMPTaskLoopBasedDirective() argument
4955 CGF.EmitOMPLoopBody(S, JumpDest()); in EmitOMPTaskLoopBasedDirective()
4956 CGF.EmitStopPoint(&S); in EmitOMPTaskLoopBasedDirective()
4961 CGF.EmitBranch(ContBlock); in EmitOMPTaskLoopBasedDirective()
4962 CGF.EmitBlock(ContBlock, true); in EmitOMPTaskLoopBasedDirective()
4966 CGF.EmitOMPLastprivateClauseFinal( in EmitOMPTaskLoopBasedDirective()
4968 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar( in EmitOMPTaskLoopBasedDirective()
4969 CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false, in EmitOMPTaskLoopBasedDirective()
4974 IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn, in EmitOMPTaskLoopBasedDirective()
4977 &Data](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
4978 OMPLoopScope PreInitScope(CGF, S); in EmitOMPTaskLoopBasedDirective()
4979 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S, in EmitOMPTaskLoopBasedDirective()
4983 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop, in EmitOMPTaskLoopBasedDirective()
4991 [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF, in EmitOMPTaskLoopBasedDirective()
4993 Action.Enter(CGF); in EmitOMPTaskLoopBasedDirective()
4994 CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, in EmitOMPTaskLoopBasedDirective()
5036 auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitSimpleOMPExecutableDirective() argument
5038 emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action); in EmitSimpleOMPExecutableDirective()
5040 OMPPrivateScope LoopGlobals(CGF); in EmitSimpleOMPExecutableDirective()
5044 if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) { in EmitSimpleOMPExecutableDirective()
5045 LValue GlobLVal = CGF.EmitLValue(E); in EmitSimpleOMPExecutableDirective()
5051 if (!CGF.LocalDeclMap.count(VD)) in EmitSimpleOMPExecutableDirective()
5052 CGF.EmitVarDecl(*VD); in EmitSimpleOMPExecutableDirective()
5064 if (!CGF.LocalDeclMap.count(VD)) in EmitSimpleOMPExecutableDirective()
5065 CGF.EmitVarDecl(*VD); in EmitSimpleOMPExecutableDirective()
5071 CGF.EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt()); in EmitSimpleOMPExecutableDirective()