Lines Matching refs:CGF
47 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) { in emitPreInitStmt() argument
54 CGF.EmitVarDecl(cast<VarDecl>(*I)); in emitPreInitStmt()
57 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in emitPreInitStmt()
58 CGF.EmitAutoVarCleanups(Emission); in emitPreInitStmt()
67 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) { in isCapturedVar() argument
68 return CGF.LambdaCaptureFields.lookup(VD) || in isCapturedVar()
69 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) || in isCapturedVar()
70 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) && in isCapturedVar()
71 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD)); in isCapturedVar()
76 CodeGenFunction &CGF, const OMPExecutableDirective &S, in OMPLexicalScope() argument
79 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()), in OMPLexicalScope()
80 InlinedShareds(CGF) { in OMPLexicalScope()
82 emitPreInitStmt(CGF, S); in OMPLexicalScope()
94 CGF.getContext(), const_cast<VarDecl *>(VD), in OMPLexicalScope()
95 isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo && in OMPLexicalScope()
98 InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress(CGF)); in OMPLexicalScope()
116 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) in OMPParallelScope() argument
117 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None, in OMPParallelScope()
131 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) in OMPTeamsScope() argument
132 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None, in OMPTeamsScope()
139 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopBasedDirective &S) { in emitPreInitStmt() argument
148 CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType())); in emitPreInitStmt()
158 CGF, OrigVD, in emitPreInitStmt()
159 Address(llvm::UndefValue::get(CGF.ConvertTypeForMem( in emitPreInitStmt()
160 CGF.getContext().getPointerType(OrigVDTy))), in emitPreInitStmt()
161 CGF.ConvertTypeForMem(OrigVDTy), in emitPreInitStmt()
162 CGF.getContext().getDeclAlign(OrigVD))); in emitPreInitStmt()
166 (void)PreCondVars.apply(CGF); in emitPreInitStmt()
171 [&CGF](unsigned Cnt, const Stmt *CurStmt) { in emitPreInitStmt()
174 CGF.EmitStmt(Init); in emitPreInitStmt()
175 CGF.EmitStmt(CXXFor->getRangeStmt()); in emitPreInitStmt()
176 CGF.EmitStmt(CXXFor->getEndStmt()); in emitPreInitStmt()
190 CGF.EmitVarDecl(cast<VarDecl>(*I)); in emitPreInitStmt()
192 PreCondVars.restore(CGF); in emitPreInitStmt()
196 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopBasedDirective &S) in OMPLoopScope() argument
197 : CodeGenFunction::RunCleanupsScope(CGF) { in OMPLoopScope()
198 emitPreInitStmt(CGF, S); in OMPLoopScope()
205 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) { in isCapturedVar() argument
206 return CGF.LambdaCaptureFields.lookup(VD) || in isCapturedVar()
207 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) || in isCapturedVar()
208 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) && in isCapturedVar()
209 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD)); in isCapturedVar()
213 OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) in OMPSimdLexicalScope() argument
214 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()), in OMPSimdLexicalScope()
215 InlinedShareds(CGF) { in OMPSimdLexicalScope()
222 CGF.EmitVarDecl(cast<VarDecl>(*I)); in OMPSimdLexicalScope()
225 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in OMPSimdLexicalScope()
226 CGF.EmitAutoVarCleanups(Emission); in OMPSimdLexicalScope()
234 CGF.EmitVarDecl(*OED); in OMPSimdLexicalScope()
240 CGF.EmitVarDecl(*OED); in OMPSimdLexicalScope()
245 CGF.EmitOMPPrivateClause(S, InlinedShareds); in OMPSimdLexicalScope()
248 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl())); in OMPSimdLexicalScope()
268 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD), in OMPSimdLexicalScope()
269 isCapturedVar(CGF, VD) || in OMPSimdLexicalScope()
270 (CGF.CapturedStmtInfo && in OMPSimdLexicalScope()
274 InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress(CGF)); in OMPSimdLexicalScope()
285 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
371 static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, in castValueFromUintptr() argument
374 ASTContext &Ctx = CGF.getContext(); in castValueFromUintptr()
376 llvm::Value *CastedPtr = CGF.EmitScalarConversion( in castValueFromUintptr()
377 AddrLV.getAddress(CGF).getPointer(), Ctx.getUIntPtrType(), in castValueFromUintptr()
380 CGF.MakeNaturalAlignAddrLValue(CastedPtr, DstType).getAddress(CGF); in castValueFromUintptr()
425 CodeGenFunction &CGF, FunctionArgList &Args, in emitOutlinedFunctionPrologue() argument
437 CodeGenModule &CGM = CGF.CGM; in emitOutlinedFunctionPrologue()
532 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs, in emitOutlinedFunctionPrologue()
542 LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt], in emitOutlinedFunctionPrologue()
545 LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]); in emitOutlinedFunctionPrologue()
558 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(), in emitOutlinedFunctionPrologue()
562 ArgLVal = CGF.MakeAddrLValue( in emitOutlinedFunctionPrologue()
563 castValueFromUintptr(CGF, I->getLocation(), FD->getType(), in emitOutlinedFunctionPrologue()
567 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation()); in emitOutlinedFunctionPrologue()
573 Address ArgAddr = ArgLVal.getAddress(CGF); in emitOutlinedFunctionPrologue()
575 ArgAddr = CGF.EmitLoadOfReference(ArgLVal); in emitOutlinedFunctionPrologue()
578 ArgAddr = CGF.EmitLoadOfPointer( in emitOutlinedFunctionPrologue()
592 CGF, I->getLocation(), FD->getType(), in emitOutlinedFunctionPrologue()
594 : ArgLVal.getAddress(CGF)}}); in emitOutlinedFunctionPrologue()
598 CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation()); in emitOutlinedFunctionPrologue()
599 LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress(CGF)}}); in emitOutlinedFunctionPrologue()
1238 [&Emission](CodeGenFunction &CGF) { in EmitOMPReductionClauseInit() argument
1239 CGF.EmitAutoVarInit(Emission); in EmitOMPReductionClauseInit()
1448 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitPostUpdateForReductionClause() argument
1450 if (!CGF.HaveInsertPoint()) in emitPostUpdateForReductionClause()
1456 if (llvm::Value *Cond = CondGen(CGF)) { in emitPostUpdateForReductionClause()
1459 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu"); in emitPostUpdateForReductionClause()
1460 DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done"); in emitPostUpdateForReductionClause()
1461 CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB); in emitPostUpdateForReductionClause()
1462 CGF.EmitBlock(ThenBB); in emitPostUpdateForReductionClause()
1465 CGF.EmitIgnoredExpr(PostUpdate); in emitPostUpdateForReductionClause()
1469 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); in emitPostUpdateForReductionClause()
1483 checkForLastprivateConditionalUpdate(CodeGenFunction &CGF, in checkForLastprivateConditionalUpdate() argument
1485 if (CGF.getLangOpts().OpenMP < 50) in checkForLastprivateConditionalUpdate()
1496 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref); in checkForLastprivateConditionalUpdate()
1507 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref); in checkForLastprivateConditionalUpdate()
1518 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref); in checkForLastprivateConditionalUpdate()
1535 CGF.CGM.getOpenMPRuntime().checkAndEmitSharedLastprivateConditional( in checkForLastprivateConditionalUpdate()
1536 CGF, S, PrivateDecls); in checkForLastprivateConditionalUpdate()
1540 CodeGenFunction &CGF, const OMPExecutableDirective &S, in emitCommonOMPParallelDirective() argument
1546 CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction( in emitCommonOMPParallelDirective()
1549 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); in emitCommonOMPParallelDirective()
1550 NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(), in emitCommonOMPParallelDirective()
1552 CGF.CGM.getOpenMPRuntime().emitNumThreadsClause( in emitCommonOMPParallelDirective()
1553 CGF, NumThreads, NumThreadsClause->getBeginLoc()); in emitCommonOMPParallelDirective()
1556 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF); in emitCommonOMPParallelDirective()
1557 CGF.CGM.getOpenMPRuntime().emitProcBindClause( in emitCommonOMPParallelDirective()
1558 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc()); in emitCommonOMPParallelDirective()
1569 OMPParallelScope Scope(CGF, S); in emitCommonOMPParallelDirective()
1575 CodeGenBoundParameters(CGF, S, CapturedVars); in emitCommonOMPParallelDirective()
1576 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); in emitCommonOMPParallelDirective()
1577 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn, in emitCommonOMPParallelDirective()
1597 CodeGenFunction &CGF, const VarDecl *VD) { in getAddressOfLocalVariable() argument
1598 CodeGenModule &CGM = CGF.CGM; in getAddressOfLocalVariable()
1609 Size = CGF.getTypeSize(CVD->getType()); in getAddressOfLocalVariable()
1611 Size = CGF.Builder.CreateNUWAdd( in getAddressOfLocalVariable()
1613 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align)); in getAddressOfLocalVariable()
1614 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align)); in getAddressOfLocalVariable()
1623 llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator()); in getAddressOfLocalVariable()
1627 Allocator = CGF.Builder.CreateIntToPtr(Allocator, CGM.VoidPtrTy); in getAddressOfLocalVariable()
1629 Allocator = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Allocator, in getAddressOfLocalVariable()
1633 CGF.Builder, Size, Allocator, in getAddressOfLocalVariable()
1636 OMPBuilder.createOMPFree(CGF.Builder, Addr, Allocator); in getAddressOfLocalVariable()
1638 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(NormalAndEHCleanup, FreeCI); in getAddressOfLocalVariable()
1639 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddressOfLocalVariable()
1641 CGF.ConvertTypeForMem(CGM.getContext().getPointerType(CVD->getType())), in getAddressOfLocalVariable()
1643 return Address(Addr, CGF.ConvertTypeForMem(CVD->getType()), Align); in getAddressOfLocalVariable()
1647 CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, in getAddrOfThreadPrivate() argument
1649 CodeGenModule &CGM = CGF.CGM; in getAddrOfThreadPrivate()
1658 CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy); in getAddrOfThreadPrivate()
1664 OMPBuilder.createCachedThreadPrivate(CGF.Builder, Data, Size, CacheName); in getAddrOfThreadPrivate()
1682 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP, in EmitOMPInlinedRegionBody() argument
1684 CGBuilderTy &Builder = CGF.Builder; in EmitOMPInlinedRegionBody()
1690 OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB); in EmitOMPInlinedRegionBody()
1691 CGF.EmitStmt(RegionBodyStmt); in EmitOMPInlinedRegionBody()
1699 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP, in EmitOMPOutlinedRegionBody() argument
1701 CGBuilderTy &Builder = CGF.Builder; in EmitOMPOutlinedRegionBody()
1707 OMPBuilderCBHelpers::OutlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB); in EmitOMPOutlinedRegionBody()
1708 CGF.EmitStmt(RegionBodyStmt); in EmitOMPOutlinedRegionBody()
1774 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelDirective() argument
1775 Action.Enter(CGF); in EmitOMPParallelDirective()
1776 OMPPrivateScope PrivateScope(CGF); in EmitOMPParallelDirective()
1777 bool Copyins = CGF.EmitOMPCopyinClause(S); in EmitOMPParallelDirective()
1778 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in EmitOMPParallelDirective()
1783 CGF.CGM.getOpenMPRuntime().emitBarrierCall( in EmitOMPParallelDirective()
1784 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, in EmitOMPParallelDirective()
1787 CGF.EmitOMPPrivateClause(S, PrivateScope); in EmitOMPParallelDirective()
1788 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPParallelDirective()
1790 CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt()); in EmitOMPParallelDirective()
1791 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in EmitOMPParallelDirective()
1817 OMPTransformDirectiveScopeRAII(CodeGenFunction &CGF, const Stmt *S) { in OMPTransformDirectiveScopeRAII() argument
1819 Scope = new OMPLoopScope(CGF, *Dir); in OMPTransformDirectiveScopeRAII()
1821 CapInfoRAII = new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI); in OMPTransformDirectiveScopeRAII()
1834 static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop, in emitBody() argument
1840 CGF.getContext().getSourceManager(), CS->getLBracLoc(), in emitBody()
1844 CodeGenFunction::LexicalScope Scope(CGF, S->getSourceRange()); in emitBody()
1846 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level); in emitBody()
1860 CGF.EmitStmt(CXXFor->getLoopVarStmt()); in emitBody()
1866 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1); in emitBody()
1870 CGF.EmitStmt(S); in emitBody()
1948 CodeGenFunction CGF(ParentCGF.CGM, /*suppressNewContext=*/true); in emitCapturedStmtFunc() local
1951 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, CSI.get()); in emitCapturedStmtFunc()
1952 llvm::Function *F = CGF.GenerateCapturedStmtFunction(*S); in emitCapturedStmtFunc()
2209 static void emitAlignedClause(CodeGenFunction &CGF, in emitAlignedClause() argument
2211 if (!CGF.HaveInsertPoint()) in emitAlignedClause()
2217 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr)); in emitAlignedClause()
2227 CGF.getContext() in emitAlignedClause()
2228 .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign( in emitAlignedClause()
2235 llvm::Value *PtrValue = CGF.EmitScalarExpr(E); in emitAlignedClause()
2236 CGF.emitAlignmentAssumption( in emitAlignedClause()
2238 llvm::ConstantInt::get(CGF.getLLVMContext(), Alignment)); in emitAlignedClause()
2286 static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, in emitPreCond() argument
2289 if (!CGF.HaveInsertPoint()) in emitPreCond()
2292 CodeGenFunction::OMPPrivateScope PreCondScope(CGF); in emitPreCond()
2293 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope); in emitPreCond()
2297 CGF.EmitIgnoredExpr(I); in emitPreCond()
2310 CGF.CreateMemTemp(VD->getType().getNonReferenceType()); in emitPreCond()
2311 (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr); in emitPreCond()
2313 (void)PreCondVars.apply(CGF); in emitPreCond()
2317 CGF.EmitIgnoredExpr(E); in emitPreCond()
2320 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount); in emitPreCond()
2321 PreCondVars.restore(CGF); in emitPreCond()
2358 static void emitSimdlenSafelenClause(CodeGenFunction &CGF, in emitSimdlenSafelenClause() argument
2360 if (!CGF.HaveInsertPoint()) in emitSimdlenSafelenClause()
2363 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(), in emitSimdlenSafelenClause()
2366 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue()); in emitSimdlenSafelenClause()
2370 CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>()); in emitSimdlenSafelenClause()
2372 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(), in emitSimdlenSafelenClause()
2375 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue()); in emitSimdlenSafelenClause()
2379 CGF.LoopStack.setParallel(/*Enable=*/false); in emitSimdlenSafelenClause()
2448 static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, in emitOMPLoopBodyWithStopPoint() argument
2451 CGF.EmitOMPLoopBody(S, LoopExit); in emitOMPLoopBodyWithStopPoint()
2452 CGF.EmitStopPoint(&S); in emitOMPLoopBodyWithStopPoint()
2456 static LValue EmitOMPHelperVar(CodeGenFunction &CGF, in EmitOMPHelperVar() argument
2459 CGF.EmitVarDecl(*VDecl); in EmitOMPHelperVar()
2460 return CGF.EmitLValue(Helper); in EmitOMPHelperVar()
2463 static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S, in emitCommonSimdLoop() argument
2466 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](CodeGenFunction &CGF, in emitCommonSimdLoop()
2468 CGOpenMPRuntime::NontemporalDeclsRAII NontemporalsRegion(CGF.CGM, S); in emitCommonSimdLoop()
2469 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in emitCommonSimdLoop()
2470 SimdInitGen(CGF); in emitCommonSimdLoop()
2472 BodyCodeGen(CGF); in emitCommonSimdLoop()
2474 auto &&ElseGen = [&BodyCodeGen](CodeGenFunction &CGF, PrePostActionTy &) { in emitCommonSimdLoop() argument
2475 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in emitCommonSimdLoop()
2476 CGF.LoopStack.setVectorizeEnable(/*Enable=*/false); in emitCommonSimdLoop()
2478 BodyCodeGen(CGF); in emitCommonSimdLoop()
2483 if (CGF.getLangOpts().OpenMP >= 50 && in emitCommonSimdLoop()
2492 CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, IfCond, ThenGen, ElseGen); in emitCommonSimdLoop()
2495 ThenRCG(CGF); in emitCommonSimdLoop()
2499 static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, in emitOMPSimdRegion() argument
2501 Action.Enter(CGF); in emitOMPSimdRegion()
2504 OMPLoopScope PreInitScope(CGF, S); in emitOMPSimdRegion()
2513 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable())); in emitOMPSimdRegion()
2514 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable())); in emitOMPSimdRegion()
2522 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { in emitOMPSimdRegion()
2526 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then"); in emitOMPSimdRegion()
2527 ContBlock = CGF.createBasicBlock("simd.if.end"); in emitOMPSimdRegion()
2528 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, in emitOMPSimdRegion()
2529 CGF.getProfileCount(&S)); in emitOMPSimdRegion()
2530 CGF.EmitBlock(ThenBlock); in emitOMPSimdRegion()
2531 CGF.incrementProfileCounter(&S); in emitOMPSimdRegion()
2537 CGF.EmitVarDecl(*IVDecl); in emitOMPSimdRegion()
2538 CGF.EmitIgnoredExpr(S.getInit()); in emitOMPSimdRegion()
2544 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); in emitOMPSimdRegion()
2546 CGF.EmitIgnoredExpr(S.getCalcLastIteration()); in emitOMPSimdRegion()
2549 emitAlignedClause(CGF, S); in emitOMPSimdRegion()
2550 (void)CGF.EmitOMPLinearClauseInit(S); in emitOMPSimdRegion()
2552 CodeGenFunction::OMPPrivateScope LoopScope(CGF); in emitOMPSimdRegion()
2553 CGF.EmitOMPPrivateLoopCounters(S, LoopScope); in emitOMPSimdRegion()
2554 CGF.EmitOMPLinearClause(S, LoopScope); in emitOMPSimdRegion()
2555 CGF.EmitOMPPrivateClause(S, LoopScope); in emitOMPSimdRegion()
2556 CGF.EmitOMPReductionClauseInit(S, LoopScope); in emitOMPSimdRegion()
2558 CGF, S, CGF.EmitLValue(S.getIterationVariable())); in emitOMPSimdRegion()
2559 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); in emitOMPSimdRegion()
2562 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitOMPSimdRegion()
2565 CGF, S, in emitOMPSimdRegion()
2566 [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitOMPSimdRegion() argument
2567 CGF.EmitOMPSimdInit(S); in emitOMPSimdRegion()
2569 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { in emitOMPSimdRegion() argument
2570 CGF.EmitOMPInnerLoop( in emitOMPSimdRegion()
2572 [&S](CodeGenFunction &CGF) { in emitOMPSimdRegion() argument
2573 emitOMPLoopBodyWithStopPoint(CGF, S, in emitOMPSimdRegion()
2578 CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; }); in emitOMPSimdRegion()
2581 CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true); in emitOMPSimdRegion()
2582 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd); in emitOMPSimdRegion()
2583 emitPostUpdateForReductionClause(CGF, S, in emitOMPSimdRegion()
2586 CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; }); in emitOMPSimdRegion()
2590 CGF.EmitBranch(ContBlock); in emitOMPSimdRegion()
2591 CGF.EmitBlock(ContBlock, true); in emitOMPSimdRegion()
2630 auto &&CodeGenIRBuilder = [this, &S, UseOMPIRBuilder](CodeGenFunction &CGF, in EmitOMPSimdDirective()
2666 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPSimdDirective() argument
2667 emitOMPSimdRegion(CGF, S, Action); in EmitOMPSimdDirective()
2810 [&S, IsMonotonic](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPOuterLoop() argument
2814 CGF.LoopStack.setParallel(!IsMonotonic); in EmitOMPOuterLoop()
2817 CGF.LoopStack.setParallel(/*Enable=*/true); in EmitOMPOuterLoop()
2819 CGF.EmitOMPSimdInit(S); in EmitOMPOuterLoop()
2823 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPOuterLoop() argument
2830 CGF.EmitOMPInnerLoop( in EmitOMPOuterLoop()
2832 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) { in EmitOMPOuterLoop() argument
2833 CodeGenLoop(CGF, S, LoopExit); in EmitOMPOuterLoop()
2835 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) { in EmitOMPOuterLoop() argument
2836 CodeGenOrdered(CGF, Loc, IVSize, IVSigned); in EmitOMPOuterLoop()
2855 auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) { in EmitOMPOuterLoop() argument
2857 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), in EmitOMPOuterLoop()
2948 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc, in EmitOMPForOuterLoop()
2952 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize, in EmitOMPForOuterLoop()
3032 emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, in emitDistributeParallelForInnerBounds() argument
3036 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable())); in emitDistributeParallelForInnerBounds()
3038 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable())); in emitDistributeParallelForInnerBounds()
3046 LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable()); in emitDistributeParallelForInnerBounds()
3047 LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable()); in emitDistributeParallelForInnerBounds()
3048 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar( in emitDistributeParallelForInnerBounds()
3050 PrevLBVal = CGF.EmitScalarConversion( in emitDistributeParallelForInnerBounds()
3054 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar( in emitDistributeParallelForInnerBounds()
3056 PrevUBVal = CGF.EmitScalarConversion( in emitDistributeParallelForInnerBounds()
3061 CGF.EmitStoreOfScalar(PrevLBVal, LB); in emitDistributeParallelForInnerBounds()
3062 CGF.EmitStoreOfScalar(PrevUBVal, UB); in emitDistributeParallelForInnerBounds()
3075 emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, in emitDistributeParallelForDispatchBounds() argument
3086 CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc()); in emitDistributeParallelForDispatchBounds()
3088 CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc()); in emitDistributeParallelForDispatchBounds()
3093 CodeGenFunction &CGF, const OMPExecutableDirective &S, in emitDistributeParallelForDistributeInnerBoundParams() argument
3097 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable())); in emitDistributeParallelForDistributeInnerBoundParams()
3099 CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(LB.getAddress(CGF)), in emitDistributeParallelForDistributeInnerBoundParams()
3100 CGF.SizeTy, /*isSigned=*/false); in emitDistributeParallelForDistributeInnerBoundParams()
3103 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable())); in emitDistributeParallelForDistributeInnerBoundParams()
3106 CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(UB.getAddress(CGF)), in emitDistributeParallelForDistributeInnerBoundParams()
3107 CGF.SizeTy, /*isSigned=*/false); in emitDistributeParallelForDistributeInnerBoundParams()
3112 emitInnerParallelForWhenCombined(CodeGenFunction &CGF, in emitInnerParallelForWhenCombined() argument
3115 auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF, in emitInnerParallelForWhenCombined()
3117 Action.Enter(CGF); in emitInnerParallelForWhenCombined()
3128 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(), in emitInnerParallelForWhenCombined()
3130 CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(), in emitInnerParallelForWhenCombined()
3136 CGF, S, in emitInnerParallelForWhenCombined()
3144 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeParallelForDirective() argument
3145 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPDistributeParallelForDirective()
3154 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeParallelForSimdDirective() argument
3155 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPDistributeParallelForSimdDirective()
3164 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeSimdDirective() argument
3165 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPDistributeSimdDirective()
3174 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetSimdDeviceFunction() argument
3175 emitOMPSimdRegion(CGF, S, Action); in EmitOMPTargetSimdDeviceFunction()
3187 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetSimdDirective() argument
3188 emitOMPSimdRegion(CGF, S, Action); in EmitOMPTargetSimdDirective()
3342 [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPWorksharingLoop() argument
3344 CGF.EmitOMPSimdInit(S); in EmitOMPWorksharingLoop()
3347 CGF.LoopStack.setParallel(/*Enable=*/true); in EmitOMPWorksharingLoop()
3352 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPWorksharingLoop() argument
3359 IVSize, IVSigned, Ordered, IL.getAddress(CGF), in EmitOMPWorksharingLoop()
3360 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF), in EmitOMPWorksharingLoop()
3362 CGF.CGM.getOpenMPRuntime().emitForStaticInit( in EmitOMPWorksharingLoop()
3363 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, in EmitOMPWorksharingLoop()
3367 CGF.EmitIgnoredExpr(S.getEnsureUpperBound()); in EmitOMPWorksharingLoop()
3369 CGF.EmitIgnoredExpr(S.getInit()); in EmitOMPWorksharingLoop()
3383 CGF.EmitOMPInnerLoop( in EmitOMPWorksharingLoop()
3388 [&S, LoopExit](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3389 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit); in EmitOMPWorksharingLoop()
3395 auto &&CodeGen = [&S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3396 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), in EmitOMPWorksharingLoop()
3410 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3411 return CGF.Builder.CreateIsNotNull( in EmitOMPWorksharingLoop()
3412 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPWorksharingLoop()
3421 *this, S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3422 return CGF.Builder.CreateIsNotNull( in EmitOMPWorksharingLoop()
3423 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPWorksharingLoop()
3431 EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3432 return CGF.Builder.CreateIsNotNull( in EmitOMPWorksharingLoop()
3433 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPWorksharingLoop()
3450 emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) { in emitForLoopBounds() argument
3453 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable())); in emitForLoopBounds()
3455 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable())); in emitForLoopBounds()
3464 emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, in emitDispatchForLoopBounds() argument
3468 const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType()); in emitDispatchForLoopBounds()
3469 llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0); in emitDispatchForLoopBounds()
3470 llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration()); in emitDispatchForLoopBounds()
3482 CodeGenFunction &CGF, const OMPLoopDirective &S, in emitScanBasedDirectiveDecls() argument
3484 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast( in emitScanBasedDirectiveDecls()
3485 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false); in emitScanBasedDirectiveDecls()
3511 RedCG.emitSharedOrigLValue(CGF, Count); in emitScanBasedDirectiveDecls()
3512 RedCG.emitAggregateType(CGF, Count); in emitScanBasedDirectiveDecls()
3515 CGF, in emitScanBasedDirectiveDecls()
3521 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(*ITA)->getDecl())); in emitScanBasedDirectiveDecls()
3534 CodeGenFunction &CGF, const OMPLoopDirective &S, in emitScanBasedDirectiveFinals() argument
3536 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast( in emitScanBasedDirectiveFinals()
3537 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false); in emitScanBasedDirectiveFinals()
3557 llvm::Value *OMPLast = CGF.Builder.CreateNSWSub( in emitScanBasedDirectiveFinals()
3559 llvm::ConstantInt::get(CGF.SizeTy, 1, /*isSigned=*/false)); in emitScanBasedDirectiveFinals()
3565 CGF, in emitScanBasedDirectiveFinals()
3569 LValue DestLVal = CGF.EmitLValue(OrigExpr); in emitScanBasedDirectiveFinals()
3570 LValue SrcLVal = CGF.EmitLValue(CopyArrayElem); in emitScanBasedDirectiveFinals()
3571 CGF.EmitOMPCopy(PrivateExpr->getType(), DestLVal.getAddress(CGF), in emitScanBasedDirectiveFinals()
3572 SrcLVal.getAddress(CGF), in emitScanBasedDirectiveFinals()
3599 CodeGenFunction &CGF, const OMPLoopDirective &S, in emitScanBasedDirective() argument
3603 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast( in emitScanBasedDirective()
3604 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false); in emitScanBasedDirective()
3620 CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S); in emitScanBasedDirective()
3628 CGF.OMPFirstScanLoop = true; in emitScanBasedDirective()
3629 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in emitScanBasedDirective()
3630 FirstGen(CGF); in emitScanBasedDirective()
3635 &Privates](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitScanBasedDirective() argument
3636 Action.Enter(CGF); in emitScanBasedDirective()
3640 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock(); in emitScanBasedDirective()
3641 llvm::BasicBlock *LoopBB = CGF.createBasicBlock("omp.outer.log.scan.body"); in emitScanBasedDirective()
3642 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.outer.log.scan.exit"); in emitScanBasedDirective()
3644 CGF.CGM.getIntrinsic(llvm::Intrinsic::log2, CGF.DoubleTy); in emitScanBasedDirective()
3646 CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy); in emitScanBasedDirective()
3647 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg); in emitScanBasedDirective()
3648 F = CGF.CGM.getIntrinsic(llvm::Intrinsic::ceil, CGF.DoubleTy); in emitScanBasedDirective()
3649 LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal); in emitScanBasedDirective()
3650 LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy); in emitScanBasedDirective()
3651 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub( in emitScanBasedDirective()
3652 OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1)); in emitScanBasedDirective()
3653 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getBeginLoc()); in emitScanBasedDirective()
3654 CGF.EmitBlock(LoopBB); in emitScanBasedDirective()
3655 auto *Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2); in emitScanBasedDirective()
3657 auto *Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2); in emitScanBasedDirective()
3658 Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB); in emitScanBasedDirective()
3659 Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB); in emitScanBasedDirective()
3663 CGF.createBasicBlock("omp.inner.log.scan.body"); in emitScanBasedDirective()
3665 CGF.createBasicBlock("omp.inner.log.scan.exit"); in emitScanBasedDirective()
3666 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K); in emitScanBasedDirective()
3667 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB); in emitScanBasedDirective()
3668 CGF.EmitBlock(InnerLoopBB); in emitScanBasedDirective()
3669 auto *IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2); in emitScanBasedDirective()
3672 CodeGenFunction::OMPPrivateScope PrivScope(CGF); in emitScanBasedDirective()
3681 CGF, in emitScanBasedDirective()
3685 LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF); in emitScanBasedDirective()
3690 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K); in emitScanBasedDirective()
3692 CGF, in emitScanBasedDirective()
3696 RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF); in emitScanBasedDirective()
3703 CGF.CGM.getOpenMPRuntime().emitReduction( in emitScanBasedDirective()
3704 CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps, in emitScanBasedDirective()
3708 CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1)); in emitScanBasedDirective()
3709 IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock()); in emitScanBasedDirective()
3710 CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K); in emitScanBasedDirective()
3711 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB); in emitScanBasedDirective()
3712 CGF.EmitBlock(InnerExitBB); in emitScanBasedDirective()
3714 CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1)); in emitScanBasedDirective()
3715 Counter->addIncoming(Next, CGF.Builder.GetInsertBlock()); in emitScanBasedDirective()
3718 CGF.Builder.CreateShl(Pow2K, 1, "", /*HasNUW=*/true); in emitScanBasedDirective()
3719 Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock()); in emitScanBasedDirective()
3720 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(Next, LogVal); in emitScanBasedDirective()
3721 CGF.Builder.CreateCondBr(Cmp, LoopBB, ExitBB); in emitScanBasedDirective()
3722 auto DL1 = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getEndLoc()); in emitScanBasedDirective()
3723 CGF.EmitBlock(ExitBB); in emitScanBasedDirective()
3726 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc()); in emitScanBasedDirective()
3727 CGF.CGM.getOpenMPRuntime().emitBarrierCall( in emitScanBasedDirective()
3728 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, in emitScanBasedDirective()
3732 RCG(CGF); in emitScanBasedDirective()
3735 CGF.OMPFirstScanLoop = false; in emitScanBasedDirective()
3736 SecondGen(CGF); in emitScanBasedDirective()
3739 static bool emitWorksharingDirective(CodeGenFunction &CGF, in emitWorksharingDirective() argument
3747 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) { in emitWorksharingDirective() argument
3748 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in emitWorksharingDirective()
3749 OMPLoopScope LoopScope(CGF, S); in emitWorksharingDirective()
3750 return CGF.EmitScalarExpr(S.getNumIterations()); in emitWorksharingDirective()
3752 const auto &&FirstGen = [&S, HasCancel](CodeGenFunction &CGF) { in emitWorksharingDirective() argument
3754 CGF, S.getDirectiveKind(), HasCancel); in emitWorksharingDirective()
3755 (void)CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), in emitWorksharingDirective()
3759 CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getBeginLoc(), in emitWorksharingDirective()
3763 &HasLastprivates](CodeGenFunction &CGF) { in emitWorksharingDirective() argument
3765 CGF, S.getDirectiveKind(), HasCancel); in emitWorksharingDirective()
3766 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), in emitWorksharingDirective()
3771 emitScanBasedDirectiveDecls(CGF, S, NumIteratorsGen); in emitWorksharingDirective()
3772 emitScanBasedDirective(CGF, S, NumIteratorsGen, FirstGen, SecondGen); in emitWorksharingDirective()
3774 emitScanBasedDirectiveFinals(CGF, S, NumIteratorsGen); in emitWorksharingDirective()
3776 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(), in emitWorksharingDirective()
3778 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), in emitWorksharingDirective()
3839 UseOMPIRBuilder](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPForDirective() argument
3870 HasLastprivates = emitWorksharingDirective(CGF, S, S.hasCancel()); in EmitOMPForDirective()
3891 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF, in EmitOMPForSimdDirective()
3893 HasLastprivates = emitWorksharingDirective(CGF, S, /*HasCancel=*/false); in EmitOMPForSimdDirective()
3909 static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, in createSectionLVal() argument
3912 LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty); in createSectionLVal()
3914 CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true); in createSectionLVal()
3923 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) { in EmitSections() argument
3924 const ASTContext &C = CGF.getContext(); in EmitSections()
3928 LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.", in EmitSections()
3929 CGF.Builder.getInt32(0)); in EmitSections()
3931 ? CGF.Builder.getInt32(CS->size() - 1) in EmitSections()
3932 : CGF.Builder.getInt32(0); in EmitSections()
3934 createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal); in EmitSections()
3935 LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.", in EmitSections()
3936 CGF.Builder.getInt32(1)); in EmitSections()
3937 LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.", in EmitSections()
3938 CGF.Builder.getInt32(0)); in EmitSections()
3940 LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv."); in EmitSections()
3942 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV); in EmitSections()
3944 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB); in EmitSections()
3953 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) { in EmitSections() argument
3965 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit"); in EmitSections()
3967 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()), in EmitSections()
3972 auto CaseBB = CGF.createBasicBlock(".omp.sections.case"); in EmitSections()
3973 CGF.EmitBlock(CaseBB); in EmitSections()
3974 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB); in EmitSections()
3975 CGF.EmitStmt(SubStmt); in EmitSections()
3976 CGF.EmitBranch(ExitBB); in EmitSections()
3980 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case"); in EmitSections()
3981 CGF.EmitBlock(CaseBB); in EmitSections()
3982 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB); in EmitSections()
3983 CGF.EmitStmt(CapturedStmt); in EmitSections()
3984 CGF.EmitBranch(ExitBB); in EmitSections()
3986 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in EmitSections()
3989 CodeGenFunction::OMPPrivateScope LoopScope(CGF); in EmitSections()
3990 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) { in EmitSections()
3994 CGF.CGM.getOpenMPRuntime().emitBarrierCall( in EmitSections()
3995 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, in EmitSections()
3998 CGF.EmitOMPPrivateClause(S, LoopScope); in EmitSections()
3999 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(CGF, S, IV); in EmitSections()
4000 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); in EmitSections()
4001 CGF.EmitOMPReductionClauseInit(S, LoopScope); in EmitSections()
4004 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in EmitSections()
4010 /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(CGF), in EmitSections()
4011 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF)); in EmitSections()
4012 CGF.CGM.getOpenMPRuntime().emitForStaticInit( in EmitSections()
4013 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit); in EmitSections()
4015 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc()); in EmitSections()
4016 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect( in EmitSections()
4017 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal); in EmitSections()
4018 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB); in EmitSections()
4020 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV); in EmitSections()
4022 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, Cond, Inc, BodyGen, in EmitSections()
4025 auto &&CodeGen = [&S](CodeGenFunction &CGF) { in EmitSections() argument
4026 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), in EmitSections()
4029 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen); in EmitSections()
4030 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in EmitSections()
4032 emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) { in EmitSections() argument
4033 return CGF.Builder.CreateIsNotNull( in EmitSections()
4034 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitSections()
4039 CGF.EmitOMPLastprivateClauseFinal( in EmitSections()
4041 CGF.Builder.CreateIsNotNull( in EmitSections()
4042 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()))); in EmitSections()
4178 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPSingleDirective() argument
4179 Action.Enter(CGF); in EmitOMPSingleDirective()
4180 OMPPrivateScope SingleScope(CGF); in EmitOMPSingleDirective()
4181 (void)CGF.EmitOMPFirstprivateClause(S, SingleScope); in EmitOMPSingleDirective()
4182 CGF.EmitOMPPrivateClause(S, SingleScope); in EmitOMPSingleDirective()
4184 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in EmitOMPSingleDirective()
4205 static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S) { in emitMaster() argument
4206 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitMaster() argument
4207 Action.Enter(CGF); in emitMaster()
4208 CGF.EmitStmt(S.getRawStmt()); in emitMaster()
4210 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc()); in emitMaster()
4241 static void emitMasked(CodeGenFunction &CGF, const OMPExecutableDirective &S) { in emitMasked() argument
4242 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitMasked() argument
4243 Action.Enter(CGF); in emitMasked()
4244 CGF.EmitStmt(S.getRawStmt()); in emitMasked()
4249 CGF.CGM.getOpenMPRuntime().emitMaskedRegion(CGF, CodeGen, S.getBeginLoc(), in emitMasked()
4325 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPCriticalDirective() argument
4326 Action.Enter(CGF); in EmitOMPCriticalDirective()
4327 CGF.EmitStmt(S.getAssociatedStmt()); in EmitOMPCriticalDirective()
4343 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelForDirective() argument
4344 Action.Enter(CGF); in EmitOMPParallelForDirective()
4345 (void)emitWorksharingDirective(CGF, S, S.hasCancel()); in EmitOMPParallelForDirective()
4348 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) { in EmitOMPParallelForDirective() argument
4349 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in EmitOMPParallelForDirective()
4351 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI); in EmitOMPParallelForDirective()
4352 OMPLoopScope LoopScope(CGF, S); in EmitOMPParallelForDirective()
4353 return CGF.EmitScalarExpr(S.getNumIterations()); in EmitOMPParallelForDirective()
4376 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelForSimdDirective() argument
4377 Action.Enter(CGF); in EmitOMPParallelForSimdDirective()
4378 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false); in EmitOMPParallelForSimdDirective()
4381 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) { in EmitOMPParallelForSimdDirective() argument
4382 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in EmitOMPParallelForSimdDirective()
4384 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI); in EmitOMPParallelForSimdDirective()
4385 OMPLoopScope LoopScope(CGF, S); in EmitOMPParallelForSimdDirective()
4386 return CGF.EmitScalarExpr(S.getNumIterations()); in EmitOMPParallelForSimdDirective()
4409 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelMasterDirective() argument
4410 Action.Enter(CGF); in EmitOMPParallelMasterDirective()
4411 OMPPrivateScope PrivateScope(CGF); in EmitOMPParallelMasterDirective()
4412 bool Copyins = CGF.EmitOMPCopyinClause(S); in EmitOMPParallelMasterDirective()
4413 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in EmitOMPParallelMasterDirective()
4418 CGF.CGM.getOpenMPRuntime().emitBarrierCall( in EmitOMPParallelMasterDirective()
4419 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, in EmitOMPParallelMasterDirective()
4422 CGF.EmitOMPPrivateClause(S, PrivateScope); in EmitOMPParallelMasterDirective()
4423 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPParallelMasterDirective()
4425 emitMaster(CGF, S); in EmitOMPParallelMasterDirective()
4426 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in EmitOMPParallelMasterDirective()
4444 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelSectionsDirective() argument
4445 Action.Enter(CGF); in EmitOMPParallelSectionsDirective()
4446 CGF.EmitSections(S); in EmitOMPParallelSectionsDirective()
4632 CapturedRegion](CodeGenFunction &CGF, in EmitOMPTaskBasedDirective()
4638 OMPPrivateScope Scope(CGF); in EmitOMPTaskBasedDirective()
4640 if (auto *DI = CGF.getDebugInfo()) { in EmitOMPTaskBasedDirective()
4642 CGF.CapturedStmtInfo->getCaptureFields(); in EmitOMPTaskBasedDirective()
4643 llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue(); in EmitOMPTaskBasedDirective()
4645 unsigned CharWidth = CGF.getContext().getCharWidth(); in EmitOMPTaskBasedDirective()
4663 CGF.getContext().getASTRecordLayout(CaptureRecord); in EmitOMPTaskBasedDirective()
4666 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo()) in EmitOMPTaskBasedDirective()
4668 CGF.Builder, false); in EmitOMPTaskBasedDirective()
4669 llvm::Instruction &Last = CGF.Builder.GetInsertBlock()->back(); in EmitOMPTaskBasedDirective()
4691 llvm::Value *CopyFn = CGF.Builder.CreateLoad( in EmitOMPTaskBasedDirective()
4692 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam))); in EmitOMPTaskBasedDirective()
4693 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar( in EmitOMPTaskBasedDirective()
4703 Address PrivatePtr = CGF.CreateMemTemp( in EmitOMPTaskBasedDirective()
4704 CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr"); in EmitOMPTaskBasedDirective()
4712 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), in EmitOMPTaskBasedDirective()
4722 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), in EmitOMPTaskBasedDirective()
4731 Ty = CGF.getContext().getPointerType(Ty); in EmitOMPTaskBasedDirective()
4733 Ty = CGF.getContext().getPointerType(Ty); in EmitOMPTaskBasedDirective()
4734 Address PrivatePtr = CGF.CreateMemTemp( in EmitOMPTaskBasedDirective()
4735 CGF.getContext().getPointerType(Ty), ".local.ptr.addr"); in EmitOMPTaskBasedDirective()
4745 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(), in EmitOMPTaskBasedDirective()
4747 CopyFn = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in EmitOMPTaskBasedDirective()
4749 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall( in EmitOMPTaskBasedDirective()
4750 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs); in EmitOMPTaskBasedDirective()
4753 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD), in EmitOMPTaskBasedDirective()
4755 CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr, in EmitOMPTaskBasedDirective()
4758 Scope.addPrivate(Pair.first, CGF.EmitLValue(&DRE).getAddress(CGF)); in EmitOMPTaskBasedDirective()
4762 CGF.Builder.CreateLoad(Pair.second), in EmitOMPTaskBasedDirective()
4763 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()), in EmitOMPTaskBasedDirective()
4764 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTaskBasedDirective()
4766 if (auto *DI = CGF.getDebugInfo()) in EmitOMPTaskBasedDirective()
4767 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo()) in EmitOMPTaskBasedDirective()
4769 Pair.first, Pair.second.getPointer(), CGF.Builder, in EmitOMPTaskBasedDirective()
4777 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first); in EmitOMPTaskBasedDirective()
4780 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(VDType)), in EmitOMPTaskBasedDirective()
4781 CGF.getPointerAlign()); in EmitOMPTaskBasedDirective()
4783 Ptr = CGF.Builder.CreateLoad(Replacement); in EmitOMPTaskBasedDirective()
4784 Replacement = Address(Ptr, CGF.ConvertTypeForMem(VDType), in EmitOMPTaskBasedDirective()
4785 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTaskBasedDirective()
4788 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first); in EmitOMPTaskBasedDirective()
4789 Address Replacement(Ptr, CGF.ConvertTypeForMem(VDType), in EmitOMPTaskBasedDirective()
4790 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTaskBasedDirective()
4796 OMPPrivateScope FirstprivateScope(CGF); in EmitOMPTaskBasedDirective()
4799 CGF.Builder.CreateLoad(Pair.second), in EmitOMPTaskBasedDirective()
4800 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()), in EmitOMPTaskBasedDirective()
4801 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTaskBasedDirective()
4805 OMPLexicalScope LexScope(CGF, S, CapturedRegion); in EmitOMPTaskBasedDirective()
4808 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad( in EmitOMPTaskBasedDirective()
4809 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9))); in EmitOMPTaskBasedDirective()
4811 RedCG.emitSharedOrigLValue(CGF, Cnt); in EmitOMPTaskBasedDirective()
4812 RedCG.emitAggregateType(CGF, Cnt); in EmitOMPTaskBasedDirective()
4816 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(), in EmitOMPTaskBasedDirective()
4818 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( in EmitOMPTaskBasedDirective()
4819 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); in EmitOMPTaskBasedDirective()
4821 Address(CGF.EmitScalarConversion( in EmitOMPTaskBasedDirective()
4822 Replacement.getPointer(), CGF.getContext().VoidPtrTy, in EmitOMPTaskBasedDirective()
4823 CGF.getContext().getPointerType( in EmitOMPTaskBasedDirective()
4826 CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()), in EmitOMPTaskBasedDirective()
4828 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); in EmitOMPTaskBasedDirective()
4854 OMPPrivateScope InRedScope(CGF); in EmitOMPTaskBasedDirective()
4858 RedCG.emitSharedOrigLValue(CGF, Cnt); in EmitOMPTaskBasedDirective()
4859 RedCG.emitAggregateType(CGF, Cnt); in EmitOMPTaskBasedDirective()
4865 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(), in EmitOMPTaskBasedDirective()
4869 ReductionsPtr = CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr), in EmitOMPTaskBasedDirective()
4872 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in EmitOMPTaskBasedDirective()
4874 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( in EmitOMPTaskBasedDirective()
4875 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); in EmitOMPTaskBasedDirective()
4877 CGF.EmitScalarConversion( in EmitOMPTaskBasedDirective()
4878 Replacement.getPointer(), CGF.getContext().VoidPtrTy, in EmitOMPTaskBasedDirective()
4879 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()), in EmitOMPTaskBasedDirective()
4881 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()), in EmitOMPTaskBasedDirective()
4883 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); in EmitOMPTaskBasedDirective()
4889 CGOpenMPRuntime::UntiedTaskLocalDeclsRAII LocalVarsScope(CGF, in EmitOMPTaskBasedDirective()
4891 Action.Enter(CGF); in EmitOMPTaskBasedDirective()
4892 BodyGen(CGF); in EmitOMPTaskBasedDirective()
5006 &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTaskBasedDirective() argument
5008 OMPPrivateScope Scope(CGF); in EmitOMPTargetTaskBasedDirective()
5011 llvm::Value *CopyFn = CGF.Builder.CreateLoad( in EmitOMPTargetTaskBasedDirective()
5012 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam))); in EmitOMPTargetTaskBasedDirective()
5013 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar( in EmitOMPTargetTaskBasedDirective()
5024 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), in EmitOMPTargetTaskBasedDirective()
5030 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(), in EmitOMPTargetTaskBasedDirective()
5032 CopyFn = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in EmitOMPTargetTaskBasedDirective()
5034 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall( in EmitOMPTargetTaskBasedDirective()
5035 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs); in EmitOMPTargetTaskBasedDirective()
5038 CGF.Builder.CreateLoad(Pair.second), in EmitOMPTargetTaskBasedDirective()
5039 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()), in EmitOMPTargetTaskBasedDirective()
5040 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTargetTaskBasedDirective()
5044 CGF.processInReduction(S, Data, CGF, CS, Scope); in EmitOMPTargetTaskBasedDirective()
5046 InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
5047 CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0); in EmitOMPTargetTaskBasedDirective()
5048 InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
5049 CGF.GetAddrOfLocalVar(PVD), /*Index=*/0); in EmitOMPTargetTaskBasedDirective()
5050 InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
5051 CGF.GetAddrOfLocalVar(SVD), /*Index=*/0); in EmitOMPTargetTaskBasedDirective()
5054 InputInfo.MappersArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
5055 CGF.GetAddrOfLocalVar(MVD), /*Index=*/0); in EmitOMPTargetTaskBasedDirective()
5058 Action.Enter(CGF); in EmitOMPTargetTaskBasedDirective()
5059 OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false); in EmitOMPTargetTaskBasedDirective()
5060 BodyGen(CGF); in EmitOMPTargetTaskBasedDirective()
5075 CodeGenFunction &CGF, in processInReduction() argument
5080 OMPLexicalScope LexScope(CGF, S, CapturedRegion); in processInReduction()
5083 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad( in processInReduction()
5084 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(4))); in processInReduction()
5086 RedCG.emitSharedOrigLValue(CGF, Cnt); in processInReduction()
5087 RedCG.emitAggregateType(CGF, Cnt); in processInReduction()
5091 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(), in processInReduction()
5093 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( in processInReduction()
5094 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); in processInReduction()
5096 Address(CGF.EmitScalarConversion( in processInReduction()
5097 Replacement.getPointer(), CGF.getContext().VoidPtrTy, in processInReduction()
5098 CGF.getContext().getPointerType( in processInReduction()
5101 CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()), in processInReduction()
5103 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); in processInReduction()
5126 OMPPrivateScope InRedScope(CGF); in processInReduction()
5130 RedCG.emitSharedOrigLValue(CGF, Cnt); in processInReduction()
5131 RedCG.emitAggregateType(CGF, Cnt); in processInReduction()
5135 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(), in processInReduction()
5140 CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr), TRExpr->getExprLoc()); in processInReduction()
5142 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in processInReduction()
5144 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( in processInReduction()
5145 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); in processInReduction()
5147 CGF.EmitScalarConversion( in processInReduction()
5148 Replacement.getPointer(), CGF.getContext().VoidPtrTy, in processInReduction()
5149 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()), in processInReduction()
5151 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()), in processInReduction()
5153 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); in processInReduction()
5177 auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskDirective() argument
5178 CGF.EmitStmt(CS->getCapturedStmt()); in EmitOMPTaskDirective()
5181 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn, in EmitOMPTaskDirective()
5183 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn, in EmitOMPTaskDirective()
5232 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTaskgroupDirective() argument
5233 Action.Enter(CGF); in EmitOMPTaskgroupDirective()
5248 CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(), in EmitOMPTaskgroupDirective()
5251 CGF.EmitVarDecl(*VD); in EmitOMPTaskgroupDirective()
5252 CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD), in EmitOMPTaskgroupDirective()
5255 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in EmitOMPTaskgroupDirective()
5665 [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeLoop() argument
5667 CGF.EmitOMPSimdInit(S); in EmitOMPDistributeLoop()
5670 StaticChunked](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeLoop() argument
5671 CGF.EmitOMPInnerLoop( in EmitOMPDistributeLoop()
5673 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
5674 CodeGenLoop(CGF, S, LoopExit); in EmitOMPDistributeLoop()
5676 [&S, StaticChunked](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
5678 CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound()); in EmitOMPDistributeLoop()
5679 CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound()); in EmitOMPDistributeLoop()
5680 CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound()); in EmitOMPDistributeLoop()
5681 CGF.EmitIgnoredExpr(S.getCombinedInit()); in EmitOMPDistributeLoop()
5698 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
5699 return CGF.Builder.CreateIsNotNull( in EmitOMPDistributeLoop()
5700 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPDistributeLoop()
5709 *this, S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
5710 return CGF.Builder.CreateIsNotNull( in EmitOMPDistributeLoop()
5711 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPDistributeLoop()
5732 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeDirective() argument
5733 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPDistributeDirective()
5742 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true); in emitOutlinedOrderedFunction() local
5744 CGF.CapturedStmtInfo = &CapStmtInfo; in emitOutlinedOrderedFunction()
5745 llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S, Loc); in emitOutlinedOrderedFunction()
5828 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF, in EmitOMPOrderedDirective()
5833 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); in EmitOMPOrderedDirective()
5836 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(), in EmitOMPOrderedDirective()
5839 Action.Enter(CGF); in EmitOMPOrderedDirective()
5840 CGF.EmitStmt(CS->getCapturedStmt()); in EmitOMPOrderedDirective()
5847 static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val, in convertToScalarValue() argument
5850 assert(CGF.hasScalarEvaluationKind(DestType) && in convertToScalarValue()
5853 return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, in convertToScalarValue()
5855 : CGF.EmitComplexToScalarConversion( in convertToScalarValue()
5860 convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, in convertToComplexValue() argument
5862 assert(CGF.getEvaluationKind(DestType) == TEK_Complex && in convertToComplexValue()
5869 llvm::Value *ScalarVal = CGF.EmitScalarConversion( in convertToComplexValue()
5878 ComplexVal.first = CGF.EmitScalarConversion( in convertToComplexValue()
5880 ComplexVal.second = CGF.EmitScalarConversion( in convertToComplexValue()
5886 static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO, in emitSimpleAtomicStore() argument
5889 CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal); in emitSimpleAtomicStore()
5891 CGF.EmitAtomicStore(RVal, LVal, AO, LVal.isVolatile(), /*isInit=*/false); in emitSimpleAtomicStore()
5894 static RValue emitSimpleAtomicLoad(CodeGenFunction &CGF, in emitSimpleAtomicLoad() argument
5898 return CGF.EmitLoadOfLValue(LVal, Loc); in emitSimpleAtomicLoad()
5899 return CGF.EmitAtomicLoad( in emitSimpleAtomicLoad()
5922 static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, in emitOMPAtomicReadExpr() argument
5928 LValue XLValue = CGF.EmitLValue(X); in emitOMPAtomicReadExpr()
5929 LValue VLValue = CGF.EmitLValue(V); in emitOMPAtomicReadExpr()
5930 RValue Res = emitSimpleAtomicLoad(CGF, AO, XLValue, Loc); in emitOMPAtomicReadExpr()
5939 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc, in emitOMPAtomicReadExpr()
5949 CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc); in emitOMPAtomicReadExpr()
5950 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, V); in emitOMPAtomicReadExpr()
5953 static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, in emitOMPAtomicWriteExpr() argument
5958 emitSimpleAtomicStore(CGF, AO, CGF.EmitLValue(X), CGF.EmitAnyExpr(E)); in emitOMPAtomicWriteExpr()
5959 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X); in emitOMPAtomicWriteExpr()
5968 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc, in emitOMPAtomicWriteExpr()
5980 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, in emitOMPAtomicRMW() argument
5985 ASTContext &Context = CGF.getContext(); in emitOMPAtomicRMW()
5992 X.getAddress(CGF).getElementType())) || in emitOMPAtomicRMW()
5997 auto &&CheckAtomicSupport = [&CGF](llvm::Type *T, BinaryOperatorKind BO) { in emitOMPAtomicRMW()
6002 return llvm::isPowerOf2_64(CGF.CGM.getDataLayout().getTypeStoreSize(T)); in emitOMPAtomicRMW()
6008 !CheckAtomicSupport(X.getAddress(CGF).getElementType(), BO)) in emitOMPAtomicRMW()
6011 bool IsInteger = X.getAddress(CGF).getElementType()->isIntegerTy(); in emitOMPAtomicRMW()
6087 UpdateVal = CGF.Builder.CreateIntCast( in emitOMPAtomicRMW()
6088 IC, X.getAddress(CGF).getElementType(), in emitOMPAtomicRMW()
6091 UpdateVal = CGF.Builder.CreateCast(llvm::Instruction::CastOps::UIToFP, IC, in emitOMPAtomicRMW()
6092 X.getAddress(CGF).getElementType()); in emitOMPAtomicRMW()
6095 CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(CGF), UpdateVal, AO); in emitOMPAtomicRMW()
6123 static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, in emitOMPAtomicUpdateExpr() argument
6137 LValue XLValue = CGF.EmitLValue(X); in emitOMPAtomicUpdateExpr()
6138 RValue ExprRValue = CGF.EmitAnyExpr(E); in emitOMPAtomicUpdateExpr()
6143 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) { in emitOMPAtomicUpdateExpr()
6144 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); in emitOMPAtomicUpdateExpr()
6145 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); in emitOMPAtomicUpdateExpr()
6146 return CGF.EmitAnyExpr(UE); in emitOMPAtomicUpdateExpr()
6148 (void)CGF.EmitOMPAtomicSimpleUpdateExpr( in emitOMPAtomicUpdateExpr()
6150 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X); in emitOMPAtomicUpdateExpr()
6159 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc, in emitOMPAtomicUpdateExpr()
6171 static RValue convertToType(CodeGenFunction &CGF, RValue Value, in convertToType() argument
6174 switch (CGF.getEvaluationKind(ResType)) { in convertToType()
6177 convertToScalarValue(CGF, Value, SourceType, ResType, Loc)); in convertToType()
6179 auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc); in convertToType()
6188 static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, in emitOMPAtomicCaptureExpr() argument
6197 LValue VLValue = CGF.EmitLValue(V); in emitOMPAtomicCaptureExpr()
6198 LValue XLValue = CGF.EmitLValue(X); in emitOMPAtomicCaptureExpr()
6199 RValue ExprRValue = CGF.EmitAnyExpr(E); in emitOMPAtomicCaptureExpr()
6217 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr, in emitOMPAtomicCaptureExpr()
6219 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); in emitOMPAtomicCaptureExpr()
6220 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); in emitOMPAtomicCaptureExpr()
6221 RValue Res = CGF.EmitAnyExpr(UE); in emitOMPAtomicCaptureExpr()
6225 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr( in emitOMPAtomicCaptureExpr()
6227 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X); in emitOMPAtomicCaptureExpr()
6236 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); in emitOMPAtomicCaptureExpr()
6237 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second); in emitOMPAtomicCaptureExpr()
6238 NewVVal = CGF.EmitAnyExpr(UE); in emitOMPAtomicCaptureExpr()
6244 ExprRValue = convertToType(CGF, ExprRValue, E->getType(), in emitOMPAtomicCaptureExpr()
6251 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr( in emitOMPAtomicCaptureExpr()
6254 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X); in emitOMPAtomicCaptureExpr()
6261 CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc); in emitOMPAtomicCaptureExpr()
6262 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, V); in emitOMPAtomicCaptureExpr()
6264 if (CGF.CGM.getLangOpts().OpenMP < 51) { in emitOMPAtomicCaptureExpr()
6274 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc, in emitOMPAtomicCaptureExpr()
6278 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc, in emitOMPAtomicCaptureExpr()
6283 CGF.CGM.getOpenMPRuntime().emitFlush( in emitOMPAtomicCaptureExpr()
6284 CGF, llvm::None, Loc, llvm::AtomicOrdering::AcquireRelease); in emitOMPAtomicCaptureExpr()
6295 static void emitOMPAtomicCompareExpr(CodeGenFunction &CGF, in emitOMPAtomicCompareExpr() argument
6303 CGF.CGM.getOpenMPRuntime().getOMPBuilder(); in emitOMPAtomicCompareExpr()
6321 LValue XLVal = CGF.EmitLValue(X); in emitOMPAtomicCompareExpr()
6322 Address XAddr = XLVal.getAddress(CGF); in emitOMPAtomicCompareExpr()
6324 auto EmitRValueWithCastIfNeeded = [&CGF, Loc](const Expr *X, const Expr *E) { in emitOMPAtomicCompareExpr()
6326 return CGF.EmitScalarExpr(E); in emitOMPAtomicCompareExpr()
6328 llvm::Value *V = CGF.EmitScalarExpr(NewE); in emitOMPAtomicCompareExpr()
6331 return CGF.EmitScalarConversion(V, NewE->getType(), X->getType(), Loc); in emitOMPAtomicCompareExpr()
6337 EVal = CGF.Builder.CreateIntCast( in emitOMPAtomicCompareExpr()
6338 CI, XLVal.getAddress(CGF).getElementType(), in emitOMPAtomicCompareExpr()
6342 DVal = CGF.Builder.CreateIntCast( in emitOMPAtomicCompareExpr()
6343 CI, XLVal.getAddress(CGF).getElementType(), in emitOMPAtomicCompareExpr()
6352 LValue LV = CGF.EmitLValue(V); in emitOMPAtomicCompareExpr()
6353 Address Addr = LV.getAddress(CGF); in emitOMPAtomicCompareExpr()
6359 LValue LV = CGF.EmitLValue(R); in emitOMPAtomicCompareExpr()
6360 Address Addr = LV.getAddress(CGF); in emitOMPAtomicCompareExpr()
6366 CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare( in emitOMPAtomicCompareExpr()
6367 CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr, in emitOMPAtomicCompareExpr()
6371 static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, in emitOMPAtomicExpr() argument
6379 emitOMPAtomicReadExpr(CGF, AO, X, V, Loc); in emitOMPAtomicExpr()
6382 emitOMPAtomicWriteExpr(CGF, AO, X, E, Loc); in emitOMPAtomicExpr()
6386 emitOMPAtomicUpdateExpr(CGF, AO, X, E, UE, IsXLHSInRHSPart, Loc); in emitOMPAtomicExpr()
6389 emitOMPAtomicCaptureExpr(CGF, AO, IsPostfixUpdate, V, X, E, UE, in emitOMPAtomicExpr()
6393 emitOMPAtomicCompareExpr(CGF, AO, X, V, R, E, D, CE, IsXLHSInRHSPart, in emitOMPAtomicExpr()
6553 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, in emitCommonOMPTargetDirective() argument
6557 CodeGenModule &CGM = CGF.CGM; in emitCommonOMPTargetDirective()
6561 OMPLexicalScope Scope(CGF, S, OMPD_target); in emitCommonOMPTargetDirective()
6563 CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitCommonOMPTargetDirective() argument
6564 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in emitCommonOMPTargetDirective()
6569 auto LPCRegion = CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S); in emitCommonOMPTargetDirective()
6595 if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val) in emitCommonOMPTargetDirective()
6608 assert(CGF.CurFuncDecl && "No parent declaration for target region!"); in emitCommonOMPTargetDirective()
6612 if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl)) in emitCommonOMPTargetDirective()
6614 else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl)) in emitCommonOMPTargetDirective()
6618 CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl))); in emitCommonOMPTargetDirective()
6623 OMPLexicalScope Scope(CGF, S, OMPD_task); in emitCommonOMPTargetDirective()
6625 [IsOffloadEntry](CodeGenFunction &CGF, in emitCommonOMPTargetDirective()
6628 OMPLoopScope(CGF, D); in emitCommonOMPTargetDirective()
6630 llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations()); in emitCommonOMPTargetDirective()
6631 NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty, in emitCommonOMPTargetDirective()
6637 CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device, in emitCommonOMPTargetDirective()
6641 static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, in emitTargetRegion() argument
6643 Action.Enter(CGF); in emitTargetRegion()
6644 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetRegion()
6645 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in emitTargetRegion()
6646 CGF.EmitOMPPrivateClause(S, PrivateScope); in emitTargetRegion()
6649 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitTargetRegion()
6651 CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt()); in emitTargetRegion()
6652 CGF.EnsureInsertPoint(); in emitTargetRegion()
6658 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetDeviceFunction() argument
6659 emitTargetRegion(CGF, S, Action); in EmitOMPTargetDeviceFunction()
6670 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetDirective() argument
6671 emitTargetRegion(CGF, S, Action); in EmitOMPTargetDirective()
6676 static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, in emitCommonOMPTeamsDirective() argument
6682 CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction( in emitCommonOMPTeamsDirective()
6691 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit, in emitCommonOMPTeamsDirective()
6695 OMPTeamsScope Scope(CGF, S); in emitCommonOMPTeamsDirective()
6697 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); in emitCommonOMPTeamsDirective()
6698 CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn, in emitCommonOMPTeamsDirective()
6704 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTeamsDirective() argument
6705 Action.Enter(CGF); in EmitOMPTeamsDirective()
6706 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDirective()
6707 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in EmitOMPTeamsDirective()
6708 CGF.EmitOMPPrivateClause(S, PrivateScope); in EmitOMPTeamsDirective()
6709 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDirective()
6711 CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt()); in EmitOMPTeamsDirective()
6712 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDirective()
6719 static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, in emitTargetTeamsRegion() argument
6722 Action.Enter(CGF); in emitTargetTeamsRegion()
6724 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetTeamsRegion() argument
6725 Action.Enter(CGF); in emitTargetTeamsRegion()
6726 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsRegion()
6727 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in emitTargetTeamsRegion()
6728 CGF.EmitOMPPrivateClause(S, PrivateScope); in emitTargetTeamsRegion()
6729 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsRegion()
6732 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitTargetTeamsRegion()
6733 CGF.EmitStmt(CS->getCapturedStmt()); in emitTargetTeamsRegion()
6734 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsRegion()
6736 emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen); in emitTargetTeamsRegion()
6737 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsRegion()
6744 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDeviceFunction() argument
6745 emitTargetTeamsRegion(CGF, Action, S); in EmitOMPTargetTeamsDeviceFunction()
6757 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDirective() argument
6758 emitTargetTeamsRegion(CGF, Action, S); in EmitOMPTargetTeamsDirective()
6764 emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, in emitTargetTeamsDistributeRegion() argument
6766 Action.Enter(CGF); in emitTargetTeamsDistributeRegion()
6767 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeRegion() argument
6768 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in emitTargetTeamsDistributeRegion()
6772 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeRegion()
6774 Action.Enter(CGF); in emitTargetTeamsDistributeRegion()
6775 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeRegion()
6776 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeRegion()
6778 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in emitTargetTeamsDistributeRegion()
6780 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeRegion()
6782 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen); in emitTargetTeamsDistributeRegion()
6783 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeRegion()
6790 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeDeviceFunction() argument
6791 emitTargetTeamsDistributeRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeDeviceFunction()
6803 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeDirective() argument
6804 emitTargetTeamsDistributeRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeDirective()
6810 CodeGenFunction &CGF, PrePostActionTy &Action, in emitTargetTeamsDistributeSimdRegion() argument
6812 Action.Enter(CGF); in emitTargetTeamsDistributeSimdRegion()
6813 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeSimdRegion() argument
6814 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in emitTargetTeamsDistributeSimdRegion()
6818 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeSimdRegion()
6820 Action.Enter(CGF); in emitTargetTeamsDistributeSimdRegion()
6821 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeSimdRegion()
6822 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeSimdRegion()
6824 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in emitTargetTeamsDistributeSimdRegion()
6826 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeSimdRegion()
6828 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen); in emitTargetTeamsDistributeSimdRegion()
6829 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeSimdRegion()
6836 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeSimdDeviceFunction() argument
6837 emitTargetTeamsDistributeSimdRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeSimdDeviceFunction()
6849 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeSimdDirective() argument
6850 emitTargetTeamsDistributeSimdRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeSimdDirective()
6858 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeDirective() argument
6859 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPTeamsDistributeDirective()
6863 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeDirective()
6865 Action.Enter(CGF); in EmitOMPTeamsDistributeDirective()
6866 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeDirective()
6867 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeDirective()
6869 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in EmitOMPTeamsDistributeDirective()
6871 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeDirective()
6880 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeSimdDirective() argument
6881 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPTeamsDistributeSimdDirective()
6885 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeSimdDirective()
6887 Action.Enter(CGF); in EmitOMPTeamsDistributeSimdDirective()
6888 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeSimdDirective()
6889 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeSimdDirective()
6891 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd, in EmitOMPTeamsDistributeSimdDirective()
6893 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeSimdDirective()
6902 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeParallelForDirective() argument
6903 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPTeamsDistributeParallelForDirective()
6908 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeParallelForDirective()
6910 Action.Enter(CGF); in EmitOMPTeamsDistributeParallelForDirective()
6911 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeParallelForDirective()
6912 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeParallelForDirective()
6914 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in EmitOMPTeamsDistributeParallelForDirective()
6916 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeParallelForDirective()
6925 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeParallelForSimdDirective() argument
6926 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPTeamsDistributeParallelForSimdDirective()
6931 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeParallelForSimdDirective()
6933 Action.Enter(CGF); in EmitOMPTeamsDistributeParallelForSimdDirective()
6934 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeParallelForSimdDirective()
6935 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeParallelForSimdDirective()
6937 CGF.CGM.getOpenMPRuntime().emitInlinedDirective( in EmitOMPTeamsDistributeParallelForSimdDirective()
6938 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); in EmitOMPTeamsDistributeParallelForSimdDirective()
6939 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeParallelForSimdDirective()
7002 CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, in emitTargetTeamsDistributeParallelForRegion() argument
7004 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForRegion()
7005 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeParallelForRegion() argument
7006 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in emitTargetTeamsDistributeParallelForRegion()
7011 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeParallelForRegion()
7013 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForRegion()
7014 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeParallelForRegion()
7015 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeParallelForRegion()
7017 CGF.CGM.getOpenMPRuntime().emitInlinedDirective( in emitTargetTeamsDistributeParallelForRegion()
7018 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); in emitTargetTeamsDistributeParallelForRegion()
7019 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeParallelForRegion()
7022 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for, in emitTargetTeamsDistributeParallelForRegion()
7024 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeParallelForRegion()
7033 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForDeviceFunction() argument
7034 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForDeviceFunction()
7046 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForDirective() argument
7047 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForDirective()
7053 CodeGenFunction &CGF, in emitTargetTeamsDistributeParallelForSimdRegion() argument
7056 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForSimdRegion()
7057 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeParallelForSimdRegion() argument
7058 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in emitTargetTeamsDistributeParallelForSimdRegion()
7063 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeParallelForSimdRegion()
7065 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForSimdRegion()
7066 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeParallelForSimdRegion()
7067 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeParallelForSimdRegion()
7069 CGF.CGM.getOpenMPRuntime().emitInlinedDirective( in emitTargetTeamsDistributeParallelForSimdRegion()
7070 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); in emitTargetTeamsDistributeParallelForSimdRegion()
7071 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeParallelForSimdRegion()
7074 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd, in emitTargetTeamsDistributeParallelForSimdRegion()
7076 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeParallelForSimdRegion()
7085 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction() argument
7086 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction()
7098 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForSimdDirective() argument
7099 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForSimdDirective()
7281 void Enter(CodeGenFunction &CGF) override { in EmitOMPTargetDataDirective() argument
7288 CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetDataDirective() argument
7289 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTargetDataDirective() argument
7290 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in EmitOMPTargetDataDirective()
7295 &InnermostCodeGen](CodeGenFunction &CGF, in EmitOMPTargetDataDirective()
7302 Action.Enter(CGF); in EmitOMPTargetDataDirective()
7305 OMPPrivateScope PrivateScope(CGF); in EmitOMPTargetDataDirective()
7308 CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope, in EmitOMPTargetDataDirective()
7311 CGF.EmitOMPUseDeviceAddrClause(*C, PrivateScope, in EmitOMPTargetDataDirective()
7314 RCG(CGF); in EmitOMPTargetDataDirective()
7316 OMPLexicalScope Scope(CGF, S, OMPD_unknown); in EmitOMPTargetDataDirective()
7317 RCG(CGF); in EmitOMPTargetDataDirective()
7328 OMPLexicalScope Scope(CGF, S); in EmitOMPTargetDataDirective()
7329 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data, in EmitOMPTargetDataDirective()
7402 static void emitTargetParallelRegion(CodeGenFunction &CGF, in emitTargetParallelRegion() argument
7407 Action.Enter(CGF); in emitTargetParallelRegion()
7408 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetParallelRegion() argument
7409 Action.Enter(CGF); in emitTargetParallelRegion()
7410 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetParallelRegion()
7411 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in emitTargetParallelRegion()
7412 CGF.EmitOMPPrivateClause(S, PrivateScope); in emitTargetParallelRegion()
7413 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetParallelRegion()
7416 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitTargetParallelRegion()
7418 CGF.EmitStmt(CS->getCapturedStmt()); in emitTargetParallelRegion()
7419 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in emitTargetParallelRegion()
7421 emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen, in emitTargetParallelRegion()
7423 emitPostUpdateForReductionClause(CGF, S, in emitTargetParallelRegion()
7430 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelDeviceFunction() argument
7431 emitTargetParallelRegion(CGF, S, Action); in EmitOMPTargetParallelDeviceFunction()
7443 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelDirective() argument
7444 emitTargetParallelRegion(CGF, S, Action); in EmitOMPTargetParallelDirective()
7449 static void emitTargetParallelForRegion(CodeGenFunction &CGF, in emitTargetParallelForRegion() argument
7452 Action.Enter(CGF); in emitTargetParallelForRegion()
7455 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetParallelForRegion() argument
7456 Action.Enter(CGF); in emitTargetParallelForRegion()
7458 CGF, OMPD_target_parallel_for, S.hasCancel()); in emitTargetParallelForRegion()
7459 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, in emitTargetParallelForRegion()
7462 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen, in emitTargetParallelForRegion()
7470 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForDeviceFunction() argument
7471 emitTargetParallelForRegion(CGF, S, Action); in EmitOMPTargetParallelForDeviceFunction()
7483 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForDirective() argument
7484 emitTargetParallelForRegion(CGF, S, Action); in EmitOMPTargetParallelForDirective()
7490 emitTargetParallelForSimdRegion(CodeGenFunction &CGF, in emitTargetParallelForSimdRegion() argument
7493 Action.Enter(CGF); in emitTargetParallelForSimdRegion()
7496 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetParallelForSimdRegion() argument
7497 Action.Enter(CGF); in emitTargetParallelForSimdRegion()
7498 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, in emitTargetParallelForSimdRegion()
7501 emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen, in emitTargetParallelForSimdRegion()
7509 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForSimdDeviceFunction() argument
7510 emitTargetParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetParallelForSimdDeviceFunction()
7522 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForSimdDirective() argument
7523 emitTargetParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetParallelForSimdDirective()
7529 static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, in mapParam() argument
7533 Privates.addPrivate(VDecl, CGF.GetAddrOfLocalVar(PVD)); in mapParam()
7571 auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
7583 OMPLoopScope PreInitScope(CGF, S); in EmitOMPTaskLoopBasedDirective()
7584 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { in EmitOMPTaskLoopBasedDirective()
7588 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then"); in EmitOMPTaskLoopBasedDirective()
7589 ContBlock = CGF.createBasicBlock("taskloop.if.end"); in EmitOMPTaskLoopBasedDirective()
7590 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, in EmitOMPTaskLoopBasedDirective()
7591 CGF.getProfileCount(&S)); in EmitOMPTaskLoopBasedDirective()
7592 CGF.EmitBlock(ThenBlock); in EmitOMPTaskLoopBasedDirective()
7593 CGF.incrementProfileCounter(&S); in EmitOMPTaskLoopBasedDirective()
7596 (void)CGF.EmitOMPLinearClauseInit(S); in EmitOMPTaskLoopBasedDirective()
7598 OMPPrivateScope LoopScope(CGF); in EmitOMPTaskLoopBasedDirective()
7606 mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP, in EmitOMPTaskLoopBasedDirective()
7608 mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP, in EmitOMPTaskLoopBasedDirective()
7610 mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope); in EmitOMPTaskLoopBasedDirective()
7611 mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP, in EmitOMPTaskLoopBasedDirective()
7613 CGF.EmitOMPPrivateLoopCounters(S, LoopScope); in EmitOMPTaskLoopBasedDirective()
7614 CGF.EmitOMPLinearClause(S, LoopScope); in EmitOMPTaskLoopBasedDirective()
7615 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); in EmitOMPTaskLoopBasedDirective()
7620 CGF.EmitVarDecl(*IVDecl); in EmitOMPTaskLoopBasedDirective()
7621 CGF.EmitIgnoredExpr(S.getInit()); in EmitOMPTaskLoopBasedDirective()
7627 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); in EmitOMPTaskLoopBasedDirective()
7629 CGF.EmitIgnoredExpr(S.getCalcLastIteration()); in EmitOMPTaskLoopBasedDirective()
7633 OMPLexicalScope Scope(CGF, S, OMPD_taskloop, /*EmitPreInitStmt=*/false); in EmitOMPTaskLoopBasedDirective()
7635 CGF, S, in EmitOMPTaskLoopBasedDirective()
7636 [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
7638 CGF.EmitOMPSimdInit(S); in EmitOMPTaskLoopBasedDirective()
7640 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
7641 CGF.EmitOMPInnerLoop( in EmitOMPTaskLoopBasedDirective()
7643 [&S](CodeGenFunction &CGF) { in EmitOMPTaskLoopBasedDirective() argument
7644 emitOMPLoopBodyWithStopPoint(CGF, S, in EmitOMPTaskLoopBasedDirective()
7652 CGF.EmitBranch(ContBlock); in EmitOMPTaskLoopBasedDirective()
7653 CGF.EmitBlock(ContBlock, true); in EmitOMPTaskLoopBasedDirective()
7657 CGF.EmitOMPLastprivateClauseFinal( in EmitOMPTaskLoopBasedDirective()
7659 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar( in EmitOMPTaskLoopBasedDirective()
7660 CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false, in EmitOMPTaskLoopBasedDirective()
7664 CGF.EmitOMPLinearClauseFinal(S, [LIP, &S](CodeGenFunction &CGF) { in EmitOMPTaskLoopBasedDirective() argument
7665 return CGF.Builder.CreateIsNotNull( in EmitOMPTaskLoopBasedDirective()
7666 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false, in EmitOMPTaskLoopBasedDirective()
7671 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn, in EmitOMPTaskLoopBasedDirective()
7674 &Data](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
7675 OMPLoopScope PreInitScope(CGF, S); in EmitOMPTaskLoopBasedDirective()
7676 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S, in EmitOMPTaskLoopBasedDirective()
7680 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop, in EmitOMPTaskLoopBasedDirective()
7688 [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF, in EmitOMPTaskLoopBasedDirective()
7690 Action.Enter(CGF); in EmitOMPTaskLoopBasedDirective()
7691 CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, in EmitOMPTaskLoopBasedDirective()
7714 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPMasterTaskLoopDirective() argument
7715 Action.Enter(CGF); in EmitOMPMasterTaskLoopDirective()
7726 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPMasterTaskLoopSimdDirective() argument
7727 Action.Enter(CGF); in EmitOMPMasterTaskLoopSimdDirective()
7738 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelMasterTaskLoopDirective() argument
7739 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF, in EmitOMPParallelMasterTaskLoopDirective()
7741 Action.Enter(CGF); in EmitOMPParallelMasterTaskLoopDirective()
7742 CGF.EmitOMPTaskLoopBasedDirective(S); in EmitOMPParallelMasterTaskLoopDirective()
7744 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false); in EmitOMPParallelMasterTaskLoopDirective()
7745 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen, in EmitOMPParallelMasterTaskLoopDirective()
7756 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelMasterTaskLoopSimdDirective() argument
7757 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF, in EmitOMPParallelMasterTaskLoopSimdDirective()
7759 Action.Enter(CGF); in EmitOMPParallelMasterTaskLoopSimdDirective()
7760 CGF.EmitOMPTaskLoopBasedDirective(S); in EmitOMPParallelMasterTaskLoopSimdDirective()
7762 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false); in EmitOMPParallelMasterTaskLoopSimdDirective()
7763 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen, in EmitOMPParallelMasterTaskLoopSimdDirective()
7797 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPGenericLoopDirective() argument
7798 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); in EmitOMPGenericLoopDirective()
7812 auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitSimpleOMPExecutableDirective() argument
7813 OMPPrivateScope GlobalsScope(CGF); in EmitSimpleOMPExecutableDirective()
7824 if (!CGF.LocalDeclMap.count(VD)) { in EmitSimpleOMPExecutableDirective()
7825 LValue GlobLVal = CGF.EmitLValue(Ref); in EmitSimpleOMPExecutableDirective()
7826 GlobalsScope.addPrivate(VD, GlobLVal.getAddress(CGF)); in EmitSimpleOMPExecutableDirective()
7833 ParentLoopDirectiveForScanRegion ScanRegion(CGF, D); in EmitSimpleOMPExecutableDirective()
7834 emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action); in EmitSimpleOMPExecutableDirective()
7839 if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) { in EmitSimpleOMPExecutableDirective()
7840 LValue GlobLVal = CGF.EmitLValue(E); in EmitSimpleOMPExecutableDirective()
7841 GlobalsScope.addPrivate(VD, GlobLVal.getAddress(CGF)); in EmitSimpleOMPExecutableDirective()
7845 if (!CGF.LocalDeclMap.count(VD)) in EmitSimpleOMPExecutableDirective()
7846 CGF.EmitVarDecl(*VD); in EmitSimpleOMPExecutableDirective()
7858 if (!CGF.LocalDeclMap.count(VD)) in EmitSimpleOMPExecutableDirective()
7859 CGF.EmitVarDecl(*VD); in EmitSimpleOMPExecutableDirective()
7865 CGF.EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt()); in EmitSimpleOMPExecutableDirective()