Lines Matching refs:CGF
42 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) { in emitPreInitStmt() argument
49 CGF.EmitVarDecl(cast<VarDecl>(*I)); in emitPreInitStmt()
52 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in emitPreInitStmt()
53 CGF.EmitAutoVarCleanups(Emission); in emitPreInitStmt()
62 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) { in isCapturedVar() argument
63 return CGF.LambdaCaptureFields.lookup(VD) || in isCapturedVar()
64 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) || in isCapturedVar()
65 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) && in isCapturedVar()
66 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD)); in isCapturedVar()
71 CodeGenFunction &CGF, const OMPExecutableDirective &S, in OMPLexicalScope() argument
74 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()), in OMPLexicalScope()
75 InlinedShareds(CGF) { in OMPLexicalScope()
77 emitPreInitStmt(CGF, S); in OMPLexicalScope()
89 CGF.getContext(), const_cast<VarDecl *>(VD), in OMPLexicalScope()
90 isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo && in OMPLexicalScope()
93 InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address { in OMPLexicalScope()
94 return CGF.EmitLValue(&DRE).getAddress(CGF); in OMPLexicalScope()
113 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) in OMPParallelScope() argument
114 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None, in OMPParallelScope()
128 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) in OMPTeamsScope() argument
129 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None, in OMPTeamsScope()
136 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopBasedDirective &S) { in emitPreInitStmt() argument
145 CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType())); in emitPreInitStmt()
154 CGF, OrigVD, in emitPreInitStmt()
155 Address(llvm::UndefValue::get(CGF.ConvertTypeForMem( in emitPreInitStmt()
156 CGF.getContext().getPointerType( in emitPreInitStmt()
158 CGF.getContext().getDeclAlign(OrigVD))); in emitPreInitStmt()
162 (void)PreCondVars.apply(CGF); in emitPreInitStmt()
167 [&CGF](unsigned Cnt, const Stmt *CurStmt) { in emitPreInitStmt()
170 CGF.EmitStmt(Init); in emitPreInitStmt()
171 CGF.EmitStmt(CXXFor->getRangeStmt()); in emitPreInitStmt()
172 CGF.EmitStmt(CXXFor->getEndStmt()); in emitPreInitStmt()
186 CGF.EmitVarDecl(cast<VarDecl>(*I)); in emitPreInitStmt()
188 PreCondVars.restore(CGF); in emitPreInitStmt()
192 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopBasedDirective &S) in OMPLoopScope() argument
193 : CodeGenFunction::RunCleanupsScope(CGF) { in OMPLoopScope()
194 emitPreInitStmt(CGF, S); in OMPLoopScope()
201 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) { in isCapturedVar() argument
202 return CGF.LambdaCaptureFields.lookup(VD) || in isCapturedVar()
203 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) || in isCapturedVar()
204 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) && in isCapturedVar()
205 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD)); in isCapturedVar()
209 OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) in OMPSimdLexicalScope() argument
210 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()), in OMPSimdLexicalScope()
211 InlinedShareds(CGF) { in OMPSimdLexicalScope()
218 CGF.EmitVarDecl(cast<VarDecl>(*I)); in OMPSimdLexicalScope()
221 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in OMPSimdLexicalScope()
222 CGF.EmitAutoVarCleanups(Emission); in OMPSimdLexicalScope()
230 CGF.EmitVarDecl(*OED); in OMPSimdLexicalScope()
236 CGF.EmitVarDecl(*OED); in OMPSimdLexicalScope()
241 CGF.EmitOMPPrivateClause(S, InlinedShareds); in OMPSimdLexicalScope()
244 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl())); in OMPSimdLexicalScope()
264 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD), in OMPSimdLexicalScope()
265 isCapturedVar(CGF, VD) || in OMPSimdLexicalScope()
266 (CGF.CapturedStmtInfo && in OMPSimdLexicalScope()
270 InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address { in OMPSimdLexicalScope()
271 return CGF.EmitLValue(&DRE).getAddress(CGF); in OMPSimdLexicalScope()
283 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
369 static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, in castValueFromUintptr() argument
372 ASTContext &Ctx = CGF.getContext(); in castValueFromUintptr()
374 llvm::Value *CastedPtr = CGF.EmitScalarConversion( in castValueFromUintptr()
375 AddrLV.getAddress(CGF).getPointer(), Ctx.getUIntPtrType(), in castValueFromUintptr()
378 CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType)) in castValueFromUintptr()
379 .getAddress(CGF); in castValueFromUintptr()
424 CodeGenFunction &CGF, FunctionArgList &Args, in emitOutlinedFunctionPrologue() argument
436 CodeGenModule &CGM = CGF.CGM; in emitOutlinedFunctionPrologue()
525 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs, in emitOutlinedFunctionPrologue()
535 LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt], in emitOutlinedFunctionPrologue()
538 LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]); in emitOutlinedFunctionPrologue()
551 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(), in emitOutlinedFunctionPrologue()
555 ArgLVal = CGF.MakeAddrLValue( in emitOutlinedFunctionPrologue()
556 castValueFromUintptr(CGF, I->getLocation(), FD->getType(), in emitOutlinedFunctionPrologue()
560 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation()); in emitOutlinedFunctionPrologue()
566 Address ArgAddr = ArgLVal.getAddress(CGF); in emitOutlinedFunctionPrologue()
568 ArgAddr = CGF.EmitLoadOfReference(ArgLVal); in emitOutlinedFunctionPrologue()
571 ArgAddr = CGF.EmitLoadOfPointer( in emitOutlinedFunctionPrologue()
586 CGF, I->getLocation(), FD->getType(), in emitOutlinedFunctionPrologue()
588 : ArgLVal.getAddress(CGF)}}); in emitOutlinedFunctionPrologue()
592 CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation()); in emitOutlinedFunctionPrologue()
593 LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress(CGF)}}); in emitOutlinedFunctionPrologue()
1247 [&Emission](CodeGenFunction &CGF) { in EmitOMPReductionClauseInit() argument
1248 CGF.EmitAutoVarInit(Emission); in EmitOMPReductionClauseInit()
1464 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitPostUpdateForReductionClause() argument
1466 if (!CGF.HaveInsertPoint()) in emitPostUpdateForReductionClause()
1472 if (llvm::Value *Cond = CondGen(CGF)) { in emitPostUpdateForReductionClause()
1475 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu"); in emitPostUpdateForReductionClause()
1476 DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done"); in emitPostUpdateForReductionClause()
1477 CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB); in emitPostUpdateForReductionClause()
1478 CGF.EmitBlock(ThenBB); in emitPostUpdateForReductionClause()
1481 CGF.EmitIgnoredExpr(PostUpdate); in emitPostUpdateForReductionClause()
1485 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); in emitPostUpdateForReductionClause()
1499 checkForLastprivateConditionalUpdate(CodeGenFunction &CGF, in checkForLastprivateConditionalUpdate() argument
1501 if (CGF.getLangOpts().OpenMP < 50) in checkForLastprivateConditionalUpdate()
1512 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref); in checkForLastprivateConditionalUpdate()
1523 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref); in checkForLastprivateConditionalUpdate()
1534 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref); in checkForLastprivateConditionalUpdate()
1551 CGF.CGM.getOpenMPRuntime().checkAndEmitSharedLastprivateConditional( in checkForLastprivateConditionalUpdate()
1552 CGF, S, PrivateDecls); in checkForLastprivateConditionalUpdate()
1556 CodeGenFunction &CGF, const OMPExecutableDirective &S, in emitCommonOMPParallelDirective() argument
1561 CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction( in emitCommonOMPParallelDirective()
1564 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); in emitCommonOMPParallelDirective()
1566 CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(), in emitCommonOMPParallelDirective()
1568 CGF.CGM.getOpenMPRuntime().emitNumThreadsClause( in emitCommonOMPParallelDirective()
1569 CGF, NumThreads, NumThreadsClause->getBeginLoc()); in emitCommonOMPParallelDirective()
1572 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF); in emitCommonOMPParallelDirective()
1573 CGF.CGM.getOpenMPRuntime().emitProcBindClause( in emitCommonOMPParallelDirective()
1574 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc()); in emitCommonOMPParallelDirective()
1585 OMPParallelScope Scope(CGF, S); in emitCommonOMPParallelDirective()
1591 CodeGenBoundParameters(CGF, S, CapturedVars); in emitCommonOMPParallelDirective()
1592 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); in emitCommonOMPParallelDirective()
1593 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn, in emitCommonOMPParallelDirective()
1613 CodeGenFunction &CGF, const VarDecl *VD) { in getAddressOfLocalVariable() argument
1614 CodeGenModule &CGM = CGF.CGM; in getAddressOfLocalVariable()
1625 Size = CGF.getTypeSize(CVD->getType()); in getAddressOfLocalVariable()
1627 Size = CGF.Builder.CreateNUWAdd( in getAddressOfLocalVariable()
1629 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align)); in getAddressOfLocalVariable()
1630 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align)); in getAddressOfLocalVariable()
1639 llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator()); in getAddressOfLocalVariable()
1643 Allocator = CGF.Builder.CreateIntToPtr(Allocator, CGM.VoidPtrTy); in getAddressOfLocalVariable()
1645 Allocator = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Allocator, in getAddressOfLocalVariable()
1649 CGF.Builder, Size, Allocator, in getAddressOfLocalVariable()
1652 OMPBuilder.createOMPFree(CGF.Builder, Addr, Allocator); in getAddressOfLocalVariable()
1654 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(NormalAndEHCleanup, FreeCI); in getAddressOfLocalVariable()
1655 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddressOfLocalVariable()
1657 CGF.ConvertTypeForMem(CGM.getContext().getPointerType(CVD->getType())), in getAddressOfLocalVariable()
1663 CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, in getAddrOfThreadPrivate() argument
1665 CodeGenModule &CGM = CGF.CGM; in getAddrOfThreadPrivate()
1674 CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy); in getAddrOfThreadPrivate()
1680 OMPBuilder.createCachedThreadPrivate(CGF.Builder, Data, Size, CacheName); in getAddrOfThreadPrivate()
1758 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelDirective() argument
1759 Action.Enter(CGF); in EmitOMPParallelDirective()
1760 OMPPrivateScope PrivateScope(CGF); in EmitOMPParallelDirective()
1761 bool Copyins = CGF.EmitOMPCopyinClause(S); in EmitOMPParallelDirective()
1762 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in EmitOMPParallelDirective()
1767 CGF.CGM.getOpenMPRuntime().emitBarrierCall( in EmitOMPParallelDirective()
1768 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, in EmitOMPParallelDirective()
1771 CGF.EmitOMPPrivateClause(S, PrivateScope); in EmitOMPParallelDirective()
1772 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPParallelDirective()
1774 CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt()); in EmitOMPParallelDirective()
1775 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in EmitOMPParallelDirective()
1797 OMPTransformDirectiveScopeRAII(CodeGenFunction &CGF, const Stmt *S) { in OMPTransformDirectiveScopeRAII() argument
1799 Scope = new OMPLoopScope(CGF, *Dir); in OMPTransformDirectiveScopeRAII()
1801 CapInfoRAII = new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI); in OMPTransformDirectiveScopeRAII()
1814 static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop, in emitBody() argument
1820 CGF.getContext().getSourceManager(), CS->getLBracLoc(), in emitBody()
1824 CodeGenFunction::LexicalScope Scope(CGF, S->getSourceRange()); in emitBody()
1826 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level); in emitBody()
1842 CGF.EmitStmt(CXXFor->getLoopVarStmt()); in emitBody()
1848 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1); in emitBody()
1852 CGF.EmitStmt(S); in emitBody()
1930 CodeGenFunction CGF(ParentCGF.CGM, /*suppressNewContext=*/true); in emitCapturedStmtFunc() local
1933 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, CSI.get()); in emitCapturedStmtFunc()
1934 llvm::Function *F = CGF.GenerateCapturedStmtFunction(*S); in emitCapturedStmtFunc()
2174 static void emitAlignedClause(CodeGenFunction &CGF, in emitAlignedClause() argument
2176 if (!CGF.HaveInsertPoint()) in emitAlignedClause()
2182 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr)); in emitAlignedClause()
2192 CGF.getContext() in emitAlignedClause()
2193 .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign( in emitAlignedClause()
2200 llvm::Value *PtrValue = CGF.EmitScalarExpr(E); in emitAlignedClause()
2201 CGF.emitAlignmentAssumption( in emitAlignedClause()
2203 llvm::ConstantInt::get(CGF.getLLVMContext(), Alignment)); in emitAlignedClause()
2258 static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, in emitPreCond() argument
2261 if (!CGF.HaveInsertPoint()) in emitPreCond()
2264 CodeGenFunction::OMPPrivateScope PreCondScope(CGF); in emitPreCond()
2265 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope); in emitPreCond()
2269 CGF.EmitIgnoredExpr(I); in emitPreCond()
2282 CGF.CreateMemTemp(VD->getType().getNonReferenceType()); in emitPreCond()
2283 (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr); in emitPreCond()
2285 (void)PreCondVars.apply(CGF); in emitPreCond()
2289 CGF.EmitIgnoredExpr(E); in emitPreCond()
2292 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount); in emitPreCond()
2293 PreCondVars.restore(CGF); in emitPreCond()
2331 static void emitSimdlenSafelenClause(CodeGenFunction &CGF, in emitSimdlenSafelenClause() argument
2333 if (!CGF.HaveInsertPoint()) in emitSimdlenSafelenClause()
2336 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(), in emitSimdlenSafelenClause()
2339 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue()); in emitSimdlenSafelenClause()
2343 CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>()); in emitSimdlenSafelenClause()
2345 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(), in emitSimdlenSafelenClause()
2348 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue()); in emitSimdlenSafelenClause()
2352 CGF.LoopStack.setParallel(/*Enable=*/false); in emitSimdlenSafelenClause()
2421 static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, in emitOMPLoopBodyWithStopPoint() argument
2424 CGF.EmitOMPLoopBody(S, LoopExit); in emitOMPLoopBodyWithStopPoint()
2425 CGF.EmitStopPoint(&S); in emitOMPLoopBodyWithStopPoint()
2429 static LValue EmitOMPHelperVar(CodeGenFunction &CGF, in EmitOMPHelperVar() argument
2432 CGF.EmitVarDecl(*VDecl); in EmitOMPHelperVar()
2433 return CGF.EmitLValue(Helper); in EmitOMPHelperVar()
2436 static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S, in emitCommonSimdLoop() argument
2439 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](CodeGenFunction &CGF, in emitCommonSimdLoop()
2441 CGOpenMPRuntime::NontemporalDeclsRAII NontemporalsRegion(CGF.CGM, S); in emitCommonSimdLoop()
2442 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in emitCommonSimdLoop()
2443 SimdInitGen(CGF); in emitCommonSimdLoop()
2445 BodyCodeGen(CGF); in emitCommonSimdLoop()
2447 auto &&ElseGen = [&BodyCodeGen](CodeGenFunction &CGF, PrePostActionTy &) { in emitCommonSimdLoop() argument
2448 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in emitCommonSimdLoop()
2449 CGF.LoopStack.setVectorizeEnable(/*Enable=*/false); in emitCommonSimdLoop()
2451 BodyCodeGen(CGF); in emitCommonSimdLoop()
2456 if (CGF.getLangOpts().OpenMP >= 50 && in emitCommonSimdLoop()
2465 CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, IfCond, ThenGen, ElseGen); in emitCommonSimdLoop()
2468 ThenRCG(CGF); in emitCommonSimdLoop()
2472 static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, in emitOMPSimdRegion() argument
2474 Action.Enter(CGF); in emitOMPSimdRegion()
2477 OMPLoopScope PreInitScope(CGF, S); in emitOMPSimdRegion()
2486 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable())); in emitOMPSimdRegion()
2487 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable())); in emitOMPSimdRegion()
2495 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { in emitOMPSimdRegion()
2499 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then"); in emitOMPSimdRegion()
2500 ContBlock = CGF.createBasicBlock("simd.if.end"); in emitOMPSimdRegion()
2501 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, in emitOMPSimdRegion()
2502 CGF.getProfileCount(&S)); in emitOMPSimdRegion()
2503 CGF.EmitBlock(ThenBlock); in emitOMPSimdRegion()
2504 CGF.incrementProfileCounter(&S); in emitOMPSimdRegion()
2510 CGF.EmitVarDecl(*IVDecl); in emitOMPSimdRegion()
2511 CGF.EmitIgnoredExpr(S.getInit()); in emitOMPSimdRegion()
2517 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); in emitOMPSimdRegion()
2519 CGF.EmitIgnoredExpr(S.getCalcLastIteration()); in emitOMPSimdRegion()
2522 emitAlignedClause(CGF, S); in emitOMPSimdRegion()
2523 (void)CGF.EmitOMPLinearClauseInit(S); in emitOMPSimdRegion()
2525 CodeGenFunction::OMPPrivateScope LoopScope(CGF); in emitOMPSimdRegion()
2526 CGF.EmitOMPPrivateLoopCounters(S, LoopScope); in emitOMPSimdRegion()
2527 CGF.EmitOMPLinearClause(S, LoopScope); in emitOMPSimdRegion()
2528 CGF.EmitOMPPrivateClause(S, LoopScope); in emitOMPSimdRegion()
2529 CGF.EmitOMPReductionClauseInit(S, LoopScope); in emitOMPSimdRegion()
2531 CGF, S, CGF.EmitLValue(S.getIterationVariable())); in emitOMPSimdRegion()
2532 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); in emitOMPSimdRegion()
2535 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitOMPSimdRegion()
2538 CGF, S, in emitOMPSimdRegion()
2539 [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitOMPSimdRegion() argument
2540 CGF.EmitOMPSimdInit(S); in emitOMPSimdRegion()
2542 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { in emitOMPSimdRegion() argument
2543 CGF.EmitOMPInnerLoop( in emitOMPSimdRegion()
2545 [&S](CodeGenFunction &CGF) { in emitOMPSimdRegion() argument
2546 emitOMPLoopBodyWithStopPoint(CGF, S, in emitOMPSimdRegion()
2551 CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; }); in emitOMPSimdRegion()
2554 CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true); in emitOMPSimdRegion()
2555 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd); in emitOMPSimdRegion()
2556 emitPostUpdateForReductionClause(CGF, S, in emitOMPSimdRegion()
2559 CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; }); in emitOMPSimdRegion()
2562 CGF.EmitBranch(ContBlock); in emitOMPSimdRegion()
2563 CGF.EmitBlock(ContBlock, true); in emitOMPSimdRegion()
2570 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPSimdDirective() argument
2571 emitOMPSimdRegion(CGF, S, Action); in EmitOMPSimdDirective()
2674 [&S, IsMonotonic](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPOuterLoop() argument
2678 CGF.LoopStack.setParallel(!IsMonotonic); in EmitOMPOuterLoop()
2681 CGF.LoopStack.setParallel(/*Enable=*/true); in EmitOMPOuterLoop()
2683 CGF.EmitOMPSimdInit(S); in EmitOMPOuterLoop()
2687 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPOuterLoop() argument
2694 CGF.EmitOMPInnerLoop( in EmitOMPOuterLoop()
2696 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) { in EmitOMPOuterLoop() argument
2697 CodeGenLoop(CGF, S, LoopExit); in EmitOMPOuterLoop()
2699 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) { in EmitOMPOuterLoop() argument
2700 CodeGenOrdered(CGF, Loc, IVSize, IVSigned); in EmitOMPOuterLoop()
2719 auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) { in EmitOMPOuterLoop() argument
2721 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), in EmitOMPOuterLoop()
2814 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc, in EmitOMPForOuterLoop()
2818 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize, in EmitOMPForOuterLoop()
2898 emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, in emitDistributeParallelForInnerBounds() argument
2902 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable())); in emitDistributeParallelForInnerBounds()
2904 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable())); in emitDistributeParallelForInnerBounds()
2912 LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable()); in emitDistributeParallelForInnerBounds()
2913 LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable()); in emitDistributeParallelForInnerBounds()
2914 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar( in emitDistributeParallelForInnerBounds()
2916 PrevLBVal = CGF.EmitScalarConversion( in emitDistributeParallelForInnerBounds()
2920 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar( in emitDistributeParallelForInnerBounds()
2922 PrevUBVal = CGF.EmitScalarConversion( in emitDistributeParallelForInnerBounds()
2927 CGF.EmitStoreOfScalar(PrevLBVal, LB); in emitDistributeParallelForInnerBounds()
2928 CGF.EmitStoreOfScalar(PrevUBVal, UB); in emitDistributeParallelForInnerBounds()
2941 emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, in emitDistributeParallelForDispatchBounds() argument
2952 CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc()); in emitDistributeParallelForDispatchBounds()
2954 CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc()); in emitDistributeParallelForDispatchBounds()
2959 CodeGenFunction &CGF, const OMPExecutableDirective &S, in emitDistributeParallelForDistributeInnerBoundParams() argument
2963 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable())); in emitDistributeParallelForDistributeInnerBoundParams()
2965 CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(LB.getAddress(CGF)), in emitDistributeParallelForDistributeInnerBoundParams()
2966 CGF.SizeTy, /*isSigned=*/false); in emitDistributeParallelForDistributeInnerBoundParams()
2969 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable())); in emitDistributeParallelForDistributeInnerBoundParams()
2972 CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(UB.getAddress(CGF)), in emitDistributeParallelForDistributeInnerBoundParams()
2973 CGF.SizeTy, /*isSigned=*/false); in emitDistributeParallelForDistributeInnerBoundParams()
2978 emitInnerParallelForWhenCombined(CodeGenFunction &CGF, in emitInnerParallelForWhenCombined() argument
2981 auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF, in emitInnerParallelForWhenCombined()
2983 Action.Enter(CGF); in emitInnerParallelForWhenCombined()
2994 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(), in emitInnerParallelForWhenCombined()
2996 CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(), in emitInnerParallelForWhenCombined()
3002 CGF, S, in emitInnerParallelForWhenCombined()
3010 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeParallelForDirective() argument
3011 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPDistributeParallelForDirective()
3020 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeParallelForSimdDirective() argument
3021 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPDistributeParallelForSimdDirective()
3030 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeSimdDirective() argument
3031 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPDistributeSimdDirective()
3040 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetSimdDeviceFunction() argument
3041 emitOMPSimdRegion(CGF, S, Action); in EmitOMPTargetSimdDeviceFunction()
3053 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetSimdDirective() argument
3054 emitOMPSimdRegion(CGF, S, Action); in EmitOMPTargetSimdDirective()
3206 [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPWorksharingLoop() argument
3208 CGF.EmitOMPSimdInit(S); in EmitOMPWorksharingLoop()
3211 CGF.LoopStack.setParallel(/*Enable=*/true); in EmitOMPWorksharingLoop()
3216 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPWorksharingLoop() argument
3223 IVSize, IVSigned, Ordered, IL.getAddress(CGF), in EmitOMPWorksharingLoop()
3224 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF), in EmitOMPWorksharingLoop()
3226 CGF.CGM.getOpenMPRuntime().emitForStaticInit( in EmitOMPWorksharingLoop()
3227 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, in EmitOMPWorksharingLoop()
3231 CGF.EmitIgnoredExpr(S.getEnsureUpperBound()); in EmitOMPWorksharingLoop()
3233 CGF.EmitIgnoredExpr(S.getInit()); in EmitOMPWorksharingLoop()
3247 CGF.EmitOMPInnerLoop( in EmitOMPWorksharingLoop()
3252 [&S, LoopExit](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3253 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit); in EmitOMPWorksharingLoop()
3259 auto &&CodeGen = [&S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3260 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), in EmitOMPWorksharingLoop()
3274 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3275 return CGF.Builder.CreateIsNotNull( in EmitOMPWorksharingLoop()
3276 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPWorksharingLoop()
3285 *this, S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3286 return CGF.Builder.CreateIsNotNull( in EmitOMPWorksharingLoop()
3287 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPWorksharingLoop()
3295 EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3296 return CGF.Builder.CreateIsNotNull( in EmitOMPWorksharingLoop()
3297 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPWorksharingLoop()
3313 emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) { in emitForLoopBounds() argument
3316 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable())); in emitForLoopBounds()
3318 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable())); in emitForLoopBounds()
3327 emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, in emitDispatchForLoopBounds() argument
3331 const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType()); in emitDispatchForLoopBounds()
3332 llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0); in emitDispatchForLoopBounds()
3333 llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration()); in emitDispatchForLoopBounds()
3345 CodeGenFunction &CGF, const OMPLoopDirective &S, in emitScanBasedDirectiveDecls() argument
3347 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast( in emitScanBasedDirectiveDecls()
3348 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false); in emitScanBasedDirectiveDecls()
3374 RedCG.emitSharedOrigLValue(CGF, Count); in emitScanBasedDirectiveDecls()
3375 RedCG.emitAggregateType(CGF, Count); in emitScanBasedDirectiveDecls()
3378 CGF, in emitScanBasedDirectiveDecls()
3384 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(*ITA)->getDecl())); in emitScanBasedDirectiveDecls()
3411 CodeGenFunction &CGF, const OMPLoopDirective &S, in emitScanBasedDirective() argument
3415 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast( in emitScanBasedDirective()
3416 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false); in emitScanBasedDirective()
3432 CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S); in emitScanBasedDirective()
3440 CGF.OMPFirstScanLoop = true; in emitScanBasedDirective()
3441 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in emitScanBasedDirective()
3442 FirstGen(CGF); in emitScanBasedDirective()
3447 &Privates](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitScanBasedDirective() argument
3448 Action.Enter(CGF); in emitScanBasedDirective()
3452 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock(); in emitScanBasedDirective()
3453 llvm::BasicBlock *LoopBB = CGF.createBasicBlock("omp.outer.log.scan.body"); in emitScanBasedDirective()
3454 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.outer.log.scan.exit"); in emitScanBasedDirective()
3456 CGF.CGM.getIntrinsic(llvm::Intrinsic::log2, CGF.DoubleTy); in emitScanBasedDirective()
3458 CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy); in emitScanBasedDirective()
3459 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg); in emitScanBasedDirective()
3460 F = CGF.CGM.getIntrinsic(llvm::Intrinsic::ceil, CGF.DoubleTy); in emitScanBasedDirective()
3461 LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal); in emitScanBasedDirective()
3462 LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy); in emitScanBasedDirective()
3463 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub( in emitScanBasedDirective()
3464 OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1)); in emitScanBasedDirective()
3465 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getBeginLoc()); in emitScanBasedDirective()
3466 CGF.EmitBlock(LoopBB); in emitScanBasedDirective()
3467 auto *Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2); in emitScanBasedDirective()
3469 auto *Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2); in emitScanBasedDirective()
3470 Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB); in emitScanBasedDirective()
3471 Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB); in emitScanBasedDirective()
3475 CGF.createBasicBlock("omp.inner.log.scan.body"); in emitScanBasedDirective()
3477 CGF.createBasicBlock("omp.inner.log.scan.exit"); in emitScanBasedDirective()
3478 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K); in emitScanBasedDirective()
3479 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB); in emitScanBasedDirective()
3480 CGF.EmitBlock(InnerLoopBB); in emitScanBasedDirective()
3481 auto *IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2); in emitScanBasedDirective()
3484 CodeGenFunction::OMPPrivateScope PrivScope(CGF); in emitScanBasedDirective()
3493 CGF, in emitScanBasedDirective()
3497 LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF); in emitScanBasedDirective()
3502 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K); in emitScanBasedDirective()
3504 CGF, in emitScanBasedDirective()
3508 RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF); in emitScanBasedDirective()
3515 CGF.CGM.getOpenMPRuntime().emitReduction( in emitScanBasedDirective()
3516 CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps, in emitScanBasedDirective()
3520 CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1)); in emitScanBasedDirective()
3521 IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock()); in emitScanBasedDirective()
3522 CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K); in emitScanBasedDirective()
3523 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB); in emitScanBasedDirective()
3524 CGF.EmitBlock(InnerExitBB); in emitScanBasedDirective()
3526 CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1)); in emitScanBasedDirective()
3527 Counter->addIncoming(Next, CGF.Builder.GetInsertBlock()); in emitScanBasedDirective()
3530 CGF.Builder.CreateShl(Pow2K, 1, "", /*HasNUW=*/true); in emitScanBasedDirective()
3531 Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock()); in emitScanBasedDirective()
3532 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(Next, LogVal); in emitScanBasedDirective()
3533 CGF.Builder.CreateCondBr(Cmp, LoopBB, ExitBB); in emitScanBasedDirective()
3534 auto DL1 = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getEndLoc()); in emitScanBasedDirective()
3535 CGF.EmitBlock(ExitBB); in emitScanBasedDirective()
3538 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc()); in emitScanBasedDirective()
3539 CGF.CGM.getOpenMPRuntime().emitBarrierCall( in emitScanBasedDirective()
3540 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, in emitScanBasedDirective()
3544 RCG(CGF); in emitScanBasedDirective()
3547 CGF.OMPFirstScanLoop = false; in emitScanBasedDirective()
3548 SecondGen(CGF); in emitScanBasedDirective()
3551 static bool emitWorksharingDirective(CodeGenFunction &CGF, in emitWorksharingDirective() argument
3559 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) { in emitWorksharingDirective() argument
3560 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in emitWorksharingDirective()
3561 OMPLoopScope LoopScope(CGF, S); in emitWorksharingDirective()
3562 return CGF.EmitScalarExpr(S.getNumIterations()); in emitWorksharingDirective()
3564 const auto &&FirstGen = [&S, HasCancel](CodeGenFunction &CGF) { in emitWorksharingDirective() argument
3566 CGF, S.getDirectiveKind(), HasCancel); in emitWorksharingDirective()
3567 (void)CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), in emitWorksharingDirective()
3571 CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getBeginLoc(), in emitWorksharingDirective()
3575 &HasLastprivates](CodeGenFunction &CGF) { in emitWorksharingDirective() argument
3577 CGF, S.getDirectiveKind(), HasCancel); in emitWorksharingDirective()
3578 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), in emitWorksharingDirective()
3583 emitScanBasedDirectiveDecls(CGF, S, NumIteratorsGen); in emitWorksharingDirective()
3584 emitScanBasedDirective(CGF, S, NumIteratorsGen, FirstGen, SecondGen); in emitWorksharingDirective()
3586 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(), in emitWorksharingDirective()
3588 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), in emitWorksharingDirective()
3610 UseOMPIRBuilder](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPForDirective() argument
3627 HasLastprivates = emitWorksharingDirective(CGF, S, S.hasCancel()); in EmitOMPForDirective()
3648 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF, in EmitOMPForSimdDirective()
3650 HasLastprivates = emitWorksharingDirective(CGF, S, /*HasCancel=*/false); in EmitOMPForSimdDirective()
3666 static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, in createSectionLVal() argument
3669 LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty); in createSectionLVal()
3671 CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true); in createSectionLVal()
3680 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) { in EmitSections() argument
3681 const ASTContext &C = CGF.getContext(); in EmitSections()
3685 LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.", in EmitSections()
3686 CGF.Builder.getInt32(0)); in EmitSections()
3688 ? CGF.Builder.getInt32(CS->size() - 1) in EmitSections()
3689 : CGF.Builder.getInt32(0); in EmitSections()
3691 createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal); in EmitSections()
3692 LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.", in EmitSections()
3693 CGF.Builder.getInt32(1)); in EmitSections()
3694 LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.", in EmitSections()
3695 CGF.Builder.getInt32(0)); in EmitSections()
3697 LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv."); in EmitSections()
3699 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV); in EmitSections()
3701 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB); in EmitSections()
3710 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) { in EmitSections() argument
3722 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit"); in EmitSections()
3724 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()), in EmitSections()
3729 auto CaseBB = CGF.createBasicBlock(".omp.sections.case"); in EmitSections()
3730 CGF.EmitBlock(CaseBB); in EmitSections()
3731 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB); in EmitSections()
3732 CGF.EmitStmt(SubStmt); in EmitSections()
3733 CGF.EmitBranch(ExitBB); in EmitSections()
3737 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case"); in EmitSections()
3738 CGF.EmitBlock(CaseBB); in EmitSections()
3739 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB); in EmitSections()
3740 CGF.EmitStmt(CapturedStmt); in EmitSections()
3741 CGF.EmitBranch(ExitBB); in EmitSections()
3743 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in EmitSections()
3746 CodeGenFunction::OMPPrivateScope LoopScope(CGF); in EmitSections()
3747 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) { in EmitSections()
3751 CGF.CGM.getOpenMPRuntime().emitBarrierCall( in EmitSections()
3752 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, in EmitSections()
3755 CGF.EmitOMPPrivateClause(S, LoopScope); in EmitSections()
3756 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(CGF, S, IV); in EmitSections()
3757 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); in EmitSections()
3758 CGF.EmitOMPReductionClauseInit(S, LoopScope); in EmitSections()
3761 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in EmitSections()
3767 /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(CGF), in EmitSections()
3768 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF)); in EmitSections()
3769 CGF.CGM.getOpenMPRuntime().emitForStaticInit( in EmitSections()
3770 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit); in EmitSections()
3772 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc()); in EmitSections()
3773 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect( in EmitSections()
3774 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal); in EmitSections()
3775 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB); in EmitSections()
3777 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV); in EmitSections()
3779 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, Cond, Inc, BodyGen, in EmitSections()
3782 auto &&CodeGen = [&S](CodeGenFunction &CGF) { in EmitSections() argument
3783 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), in EmitSections()
3786 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen); in EmitSections()
3787 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in EmitSections()
3789 emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) { in EmitSections() argument
3790 return CGF.Builder.CreateIsNotNull( in EmitSections()
3791 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitSections()
3796 CGF.EmitOMPLastprivateClauseFinal( in EmitSections()
3798 CGF.Builder.CreateIsNotNull( in EmitSections()
3799 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()))); in EmitSections()
3942 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPSingleDirective() argument
3943 Action.Enter(CGF); in EmitOMPSingleDirective()
3944 OMPPrivateScope SingleScope(CGF); in EmitOMPSingleDirective()
3945 (void)CGF.EmitOMPFirstprivateClause(S, SingleScope); in EmitOMPSingleDirective()
3946 CGF.EmitOMPPrivateClause(S, SingleScope); in EmitOMPSingleDirective()
3948 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in EmitOMPSingleDirective()
3969 static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S) { in emitMaster() argument
3970 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitMaster() argument
3971 Action.Enter(CGF); in emitMaster()
3972 CGF.EmitStmt(S.getRawStmt()); in emitMaster()
3974 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc()); in emitMaster()
4007 static void emitMasked(CodeGenFunction &CGF, const OMPExecutableDirective &S) { in emitMasked() argument
4008 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitMasked() argument
4009 Action.Enter(CGF); in emitMasked()
4010 CGF.EmitStmt(S.getRawStmt()); in emitMasked()
4015 CGF.CGM.getOpenMPRuntime().emitMaskedRegion(CGF, CodeGen, S.getBeginLoc(), in emitMasked()
4095 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPCriticalDirective() argument
4096 Action.Enter(CGF); in EmitOMPCriticalDirective()
4097 CGF.EmitStmt(S.getAssociatedStmt()); in EmitOMPCriticalDirective()
4113 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelForDirective() argument
4114 Action.Enter(CGF); in EmitOMPParallelForDirective()
4115 (void)emitWorksharingDirective(CGF, S, S.hasCancel()); in EmitOMPParallelForDirective()
4122 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) { in EmitOMPParallelForDirective() argument
4123 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in EmitOMPParallelForDirective()
4125 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI); in EmitOMPParallelForDirective()
4126 OMPLoopScope LoopScope(CGF, S); in EmitOMPParallelForDirective()
4127 return CGF.EmitScalarExpr(S.getNumIterations()); in EmitOMPParallelForDirective()
4144 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelForSimdDirective() argument
4145 Action.Enter(CGF); in EmitOMPParallelForSimdDirective()
4146 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false); in EmitOMPParallelForSimdDirective()
4153 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) { in EmitOMPParallelForSimdDirective() argument
4154 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in EmitOMPParallelForSimdDirective()
4156 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI); in EmitOMPParallelForSimdDirective()
4157 OMPLoopScope LoopScope(CGF, S); in EmitOMPParallelForSimdDirective()
4158 return CGF.EmitScalarExpr(S.getNumIterations()); in EmitOMPParallelForSimdDirective()
4175 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelMasterDirective() argument
4176 Action.Enter(CGF); in EmitOMPParallelMasterDirective()
4177 OMPPrivateScope PrivateScope(CGF); in EmitOMPParallelMasterDirective()
4178 bool Copyins = CGF.EmitOMPCopyinClause(S); in EmitOMPParallelMasterDirective()
4179 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in EmitOMPParallelMasterDirective()
4184 CGF.CGM.getOpenMPRuntime().emitBarrierCall( in EmitOMPParallelMasterDirective()
4185 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, in EmitOMPParallelMasterDirective()
4188 CGF.EmitOMPPrivateClause(S, PrivateScope); in EmitOMPParallelMasterDirective()
4189 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPParallelMasterDirective()
4191 emitMaster(CGF, S); in EmitOMPParallelMasterDirective()
4192 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in EmitOMPParallelMasterDirective()
4210 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelSectionsDirective() argument
4211 Action.Enter(CGF); in EmitOMPParallelSectionsDirective()
4212 CGF.EmitSections(S); in EmitOMPParallelSectionsDirective()
4368 CapturedRegion](CodeGenFunction &CGF, in EmitOMPTaskBasedDirective()
4374 OMPPrivateScope Scope(CGF); in EmitOMPTaskBasedDirective()
4379 llvm::Value *CopyFn = CGF.Builder.CreateLoad( in EmitOMPTaskBasedDirective()
4380 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam))); in EmitOMPTaskBasedDirective()
4381 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar( in EmitOMPTaskBasedDirective()
4391 Address PrivatePtr = CGF.CreateMemTemp( in EmitOMPTaskBasedDirective()
4392 CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr"); in EmitOMPTaskBasedDirective()
4400 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), in EmitOMPTaskBasedDirective()
4410 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), in EmitOMPTaskBasedDirective()
4419 Ty = CGF.getContext().getPointerType(Ty); in EmitOMPTaskBasedDirective()
4421 Ty = CGF.getContext().getPointerType(Ty); in EmitOMPTaskBasedDirective()
4422 Address PrivatePtr = CGF.CreateMemTemp( in EmitOMPTaskBasedDirective()
4423 CGF.getContext().getPointerType(Ty), ".local.ptr.addr"); in EmitOMPTaskBasedDirective()
4433 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(), in EmitOMPTaskBasedDirective()
4435 CopyFn = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in EmitOMPTaskBasedDirective()
4437 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall( in EmitOMPTaskBasedDirective()
4438 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs); in EmitOMPTaskBasedDirective()
4441 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD), in EmitOMPTaskBasedDirective()
4443 CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr, in EmitOMPTaskBasedDirective()
4446 Scope.addPrivate(Pair.first, [&CGF, &DRE]() { in EmitOMPTaskBasedDirective()
4447 return CGF.EmitLValue(&DRE).getAddress(CGF); in EmitOMPTaskBasedDirective()
4451 Address Replacement(CGF.Builder.CreateLoad(Pair.second), in EmitOMPTaskBasedDirective()
4452 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTaskBasedDirective()
4459 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first); in EmitOMPTaskBasedDirective()
4460 Address Replacement(Ptr, CGF.getPointerAlign()); in EmitOMPTaskBasedDirective()
4462 Ptr = CGF.Builder.CreateLoad(Replacement); in EmitOMPTaskBasedDirective()
4463 Replacement = Address(Ptr, CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTaskBasedDirective()
4466 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first); in EmitOMPTaskBasedDirective()
4467 Address Replacement(Ptr, CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTaskBasedDirective()
4473 OMPPrivateScope FirstprivateScope(CGF); in EmitOMPTaskBasedDirective()
4475 Address Replacement(CGF.Builder.CreateLoad(Pair.second), in EmitOMPTaskBasedDirective()
4476 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTaskBasedDirective()
4481 OMPLexicalScope LexScope(CGF, S, CapturedRegion); in EmitOMPTaskBasedDirective()
4484 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad( in EmitOMPTaskBasedDirective()
4485 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9))); in EmitOMPTaskBasedDirective()
4487 RedCG.emitSharedOrigLValue(CGF, Cnt); in EmitOMPTaskBasedDirective()
4488 RedCG.emitAggregateType(CGF, Cnt); in EmitOMPTaskBasedDirective()
4492 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(), in EmitOMPTaskBasedDirective()
4494 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( in EmitOMPTaskBasedDirective()
4495 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); in EmitOMPTaskBasedDirective()
4497 Address(CGF.EmitScalarConversion( in EmitOMPTaskBasedDirective()
4498 Replacement.getPointer(), CGF.getContext().VoidPtrTy, in EmitOMPTaskBasedDirective()
4499 CGF.getContext().getPointerType( in EmitOMPTaskBasedDirective()
4503 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); in EmitOMPTaskBasedDirective()
4530 OMPPrivateScope InRedScope(CGF); in EmitOMPTaskBasedDirective()
4534 RedCG.emitSharedOrigLValue(CGF, Cnt); in EmitOMPTaskBasedDirective()
4535 RedCG.emitAggregateType(CGF, Cnt); in EmitOMPTaskBasedDirective()
4541 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(), in EmitOMPTaskBasedDirective()
4545 ReductionsPtr = CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr), in EmitOMPTaskBasedDirective()
4548 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in EmitOMPTaskBasedDirective()
4550 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( in EmitOMPTaskBasedDirective()
4551 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); in EmitOMPTaskBasedDirective()
4553 CGF.EmitScalarConversion( in EmitOMPTaskBasedDirective()
4554 Replacement.getPointer(), CGF.getContext().VoidPtrTy, in EmitOMPTaskBasedDirective()
4555 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()), in EmitOMPTaskBasedDirective()
4558 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); in EmitOMPTaskBasedDirective()
4565 CGOpenMPRuntime::UntiedTaskLocalDeclsRAII LocalVarsScope(CGF, in EmitOMPTaskBasedDirective()
4567 Action.Enter(CGF); in EmitOMPTaskBasedDirective()
4568 BodyGen(CGF); in EmitOMPTaskBasedDirective()
4680 &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTaskBasedDirective() argument
4682 OMPPrivateScope Scope(CGF); in EmitOMPTargetTaskBasedDirective()
4685 llvm::Value *CopyFn = CGF.Builder.CreateLoad( in EmitOMPTargetTaskBasedDirective()
4686 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam))); in EmitOMPTargetTaskBasedDirective()
4687 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar( in EmitOMPTargetTaskBasedDirective()
4698 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), in EmitOMPTargetTaskBasedDirective()
4704 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(), in EmitOMPTargetTaskBasedDirective()
4706 CopyFn = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in EmitOMPTargetTaskBasedDirective()
4708 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall( in EmitOMPTargetTaskBasedDirective()
4709 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs); in EmitOMPTargetTaskBasedDirective()
4711 Address Replacement(CGF.Builder.CreateLoad(Pair.second), in EmitOMPTargetTaskBasedDirective()
4712 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTargetTaskBasedDirective()
4719 InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
4720 CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0); in EmitOMPTargetTaskBasedDirective()
4721 InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
4722 CGF.GetAddrOfLocalVar(PVD), /*Index=*/0); in EmitOMPTargetTaskBasedDirective()
4723 InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
4724 CGF.GetAddrOfLocalVar(SVD), /*Index=*/0); in EmitOMPTargetTaskBasedDirective()
4727 InputInfo.MappersArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
4728 CGF.GetAddrOfLocalVar(MVD), /*Index=*/0); in EmitOMPTargetTaskBasedDirective()
4731 Action.Enter(CGF); in EmitOMPTargetTaskBasedDirective()
4732 OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false); in EmitOMPTargetTaskBasedDirective()
4733 BodyGen(CGF); in EmitOMPTargetTaskBasedDirective()
4764 auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskDirective() argument
4765 CGF.EmitStmt(CS->getCapturedStmt()); in EmitOMPTaskDirective()
4768 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn, in EmitOMPTaskDirective()
4770 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn, in EmitOMPTaskDirective()
4794 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTaskgroupDirective() argument
4795 Action.Enter(CGF); in EmitOMPTaskgroupDirective()
4810 CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(), in EmitOMPTaskgroupDirective()
4813 CGF.EmitVarDecl(*VD); in EmitOMPTaskgroupDirective()
4814 CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD), in EmitOMPTaskgroupDirective()
4817 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in EmitOMPTaskgroupDirective()
5228 [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeLoop() argument
5230 CGF.EmitOMPSimdInit(S); in EmitOMPDistributeLoop()
5233 StaticChunked](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeLoop() argument
5234 CGF.EmitOMPInnerLoop( in EmitOMPDistributeLoop()
5236 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
5237 CodeGenLoop(CGF, S, LoopExit); in EmitOMPDistributeLoop()
5239 [&S, StaticChunked](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
5241 CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound()); in EmitOMPDistributeLoop()
5242 CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound()); in EmitOMPDistributeLoop()
5243 CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound()); in EmitOMPDistributeLoop()
5244 CGF.EmitIgnoredExpr(S.getCombinedInit()); in EmitOMPDistributeLoop()
5261 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
5262 return CGF.Builder.CreateIsNotNull( in EmitOMPDistributeLoop()
5263 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPDistributeLoop()
5272 *this, S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
5273 return CGF.Builder.CreateIsNotNull( in EmitOMPDistributeLoop()
5274 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPDistributeLoop()
5295 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeDirective() argument
5296 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPDistributeDirective()
5305 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true); in emitOutlinedOrderedFunction() local
5307 CGF.CapturedStmtInfo = &CapStmtInfo; in emitOutlinedOrderedFunction()
5308 llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S, Loc); in emitOutlinedOrderedFunction()
5324 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF, in EmitOMPOrderedDirective()
5329 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); in EmitOMPOrderedDirective()
5332 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(), in EmitOMPOrderedDirective()
5335 Action.Enter(CGF); in EmitOMPOrderedDirective()
5336 CGF.EmitStmt(CS->getCapturedStmt()); in EmitOMPOrderedDirective()
5343 static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val, in convertToScalarValue() argument
5346 assert(CGF.hasScalarEvaluationKind(DestType) && in convertToScalarValue()
5349 return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, in convertToScalarValue()
5351 : CGF.EmitComplexToScalarConversion( in convertToScalarValue()
5356 convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, in convertToComplexValue() argument
5358 assert(CGF.getEvaluationKind(DestType) == TEK_Complex && in convertToComplexValue()
5365 llvm::Value *ScalarVal = CGF.EmitScalarConversion( in convertToComplexValue()
5374 ComplexVal.first = CGF.EmitScalarConversion( in convertToComplexValue()
5376 ComplexVal.second = CGF.EmitScalarConversion( in convertToComplexValue()
5382 static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO, in emitSimpleAtomicStore() argument
5385 CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal); in emitSimpleAtomicStore()
5387 CGF.EmitAtomicStore(RVal, LVal, AO, LVal.isVolatile(), /*isInit=*/false); in emitSimpleAtomicStore()
5390 static RValue emitSimpleAtomicLoad(CodeGenFunction &CGF, in emitSimpleAtomicLoad() argument
5394 return CGF.EmitLoadOfLValue(LVal, Loc); in emitSimpleAtomicLoad()
5395 return CGF.EmitAtomicLoad( in emitSimpleAtomicLoad()
5418 static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, in emitOMPAtomicReadExpr() argument
5424 LValue XLValue = CGF.EmitLValue(X); in emitOMPAtomicReadExpr()
5425 LValue VLValue = CGF.EmitLValue(V); in emitOMPAtomicReadExpr()
5426 RValue Res = emitSimpleAtomicLoad(CGF, AO, XLValue, Loc); in emitOMPAtomicReadExpr()
5435 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc, in emitOMPAtomicReadExpr()
5445 CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc); in emitOMPAtomicReadExpr()
5446 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, V); in emitOMPAtomicReadExpr()
5449 static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, in emitOMPAtomicWriteExpr() argument
5454 emitSimpleAtomicStore(CGF, AO, CGF.EmitLValue(X), CGF.EmitAnyExpr(E)); in emitOMPAtomicWriteExpr()
5455 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X); in emitOMPAtomicWriteExpr()
5464 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc, in emitOMPAtomicWriteExpr()
5476 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, in emitOMPAtomicRMW() argument
5481 ASTContext &Context = CGF.getContext(); in emitOMPAtomicRMW()
5489 X.getAddress(CGF).getElementType())) || in emitOMPAtomicRMW()
5490 !X.getAddress(CGF).getElementType()->isIntegerTy() || in emitOMPAtomicRMW()
5561 UpdateVal = CGF.Builder.CreateIntCast( in emitOMPAtomicRMW()
5562 IC, X.getAddress(CGF).getElementType(), in emitOMPAtomicRMW()
5566 CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(CGF), UpdateVal, AO); in emitOMPAtomicRMW()
5594 static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, in emitOMPAtomicUpdateExpr() argument
5608 LValue XLValue = CGF.EmitLValue(X); in emitOMPAtomicUpdateExpr()
5609 RValue ExprRValue = CGF.EmitAnyExpr(E); in emitOMPAtomicUpdateExpr()
5614 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) { in emitOMPAtomicUpdateExpr()
5615 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); in emitOMPAtomicUpdateExpr()
5616 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); in emitOMPAtomicUpdateExpr()
5617 return CGF.EmitAnyExpr(UE); in emitOMPAtomicUpdateExpr()
5619 (void)CGF.EmitOMPAtomicSimpleUpdateExpr( in emitOMPAtomicUpdateExpr()
5621 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X); in emitOMPAtomicUpdateExpr()
5630 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc, in emitOMPAtomicUpdateExpr()
5642 static RValue convertToType(CodeGenFunction &CGF, RValue Value, in convertToType() argument
5645 switch (CGF.getEvaluationKind(ResType)) { in convertToType()
5648 convertToScalarValue(CGF, Value, SourceType, ResType, Loc)); in convertToType()
5650 auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc); in convertToType()
5659 static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, in emitOMPAtomicCaptureExpr() argument
5668 LValue VLValue = CGF.EmitLValue(V); in emitOMPAtomicCaptureExpr()
5669 LValue XLValue = CGF.EmitLValue(X); in emitOMPAtomicCaptureExpr()
5670 RValue ExprRValue = CGF.EmitAnyExpr(E); in emitOMPAtomicCaptureExpr()
5688 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr, in emitOMPAtomicCaptureExpr()
5690 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); in emitOMPAtomicCaptureExpr()
5691 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); in emitOMPAtomicCaptureExpr()
5692 RValue Res = CGF.EmitAnyExpr(UE); in emitOMPAtomicCaptureExpr()
5696 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr( in emitOMPAtomicCaptureExpr()
5698 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X); in emitOMPAtomicCaptureExpr()
5707 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); in emitOMPAtomicCaptureExpr()
5708 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second); in emitOMPAtomicCaptureExpr()
5709 NewVVal = CGF.EmitAnyExpr(UE); in emitOMPAtomicCaptureExpr()
5715 ExprRValue = convertToType(CGF, ExprRValue, E->getType(), in emitOMPAtomicCaptureExpr()
5722 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr( in emitOMPAtomicCaptureExpr()
5725 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X); in emitOMPAtomicCaptureExpr()
5732 CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc); in emitOMPAtomicCaptureExpr()
5733 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, V); in emitOMPAtomicCaptureExpr()
5735 if (CGF.CGM.getLangOpts().OpenMP < 51) { in emitOMPAtomicCaptureExpr()
5745 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc, in emitOMPAtomicCaptureExpr()
5749 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc, in emitOMPAtomicCaptureExpr()
5754 CGF.CGM.getOpenMPRuntime().emitFlush( in emitOMPAtomicCaptureExpr()
5755 CGF, llvm::None, Loc, llvm::AtomicOrdering::AcquireRelease); in emitOMPAtomicCaptureExpr()
5766 static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, in emitOMPAtomicExpr() argument
5773 emitOMPAtomicReadExpr(CGF, AO, X, V, Loc); in emitOMPAtomicExpr()
5776 emitOMPAtomicWriteExpr(CGF, AO, X, E, Loc); in emitOMPAtomicExpr()
5780 emitOMPAtomicUpdateExpr(CGF, AO, X, E, UE, IsXLHSInRHSPart, Loc); in emitOMPAtomicExpr()
5783 emitOMPAtomicCaptureExpr(CGF, AO, IsPostfixUpdate, V, X, E, UE, in emitOMPAtomicExpr()
5927 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, in emitCommonOMPTargetDirective() argument
5931 CodeGenModule &CGM = CGF.CGM; in emitCommonOMPTargetDirective()
5935 OMPLexicalScope Scope(CGF, S, OMPD_target); in emitCommonOMPTargetDirective()
5937 CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitCommonOMPTargetDirective() argument
5938 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in emitCommonOMPTargetDirective()
5944 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S); in emitCommonOMPTargetDirective()
5970 if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val) in emitCommonOMPTargetDirective()
5976 assert(CGF.CurFuncDecl && "No parent declaration for target region!"); in emitCommonOMPTargetDirective()
5980 if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl)) in emitCommonOMPTargetDirective()
5982 else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl)) in emitCommonOMPTargetDirective()
5986 CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl))); in emitCommonOMPTargetDirective()
5991 OMPLexicalScope Scope(CGF, S, OMPD_task); in emitCommonOMPTargetDirective()
5993 [IsOffloadEntry](CodeGenFunction &CGF, in emitCommonOMPTargetDirective()
5996 OMPLoopScope(CGF, D); in emitCommonOMPTargetDirective()
5998 llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations()); in emitCommonOMPTargetDirective()
5999 NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty, in emitCommonOMPTargetDirective()
6005 CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device, in emitCommonOMPTargetDirective()
6009 static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, in emitTargetRegion() argument
6011 Action.Enter(CGF); in emitTargetRegion()
6012 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetRegion()
6013 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in emitTargetRegion()
6014 CGF.EmitOMPPrivateClause(S, PrivateScope); in emitTargetRegion()
6017 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitTargetRegion()
6019 CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt()); in emitTargetRegion()
6020 CGF.EnsureInsertPoint(); in emitTargetRegion()
6026 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetDeviceFunction() argument
6027 emitTargetRegion(CGF, S, Action); in EmitOMPTargetDeviceFunction()
6038 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetDirective() argument
6039 emitTargetRegion(CGF, S, Action); in EmitOMPTargetDirective()
6044 static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, in emitCommonOMPTeamsDirective() argument
6050 CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction( in emitCommonOMPTeamsDirective()
6059 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit, in emitCommonOMPTeamsDirective()
6063 OMPTeamsScope Scope(CGF, S); in emitCommonOMPTeamsDirective()
6065 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); in emitCommonOMPTeamsDirective()
6066 CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn, in emitCommonOMPTeamsDirective()
6072 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTeamsDirective() argument
6073 Action.Enter(CGF); in EmitOMPTeamsDirective()
6074 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDirective()
6075 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in EmitOMPTeamsDirective()
6076 CGF.EmitOMPPrivateClause(S, PrivateScope); in EmitOMPTeamsDirective()
6077 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDirective()
6079 CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt()); in EmitOMPTeamsDirective()
6080 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDirective()
6087 static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, in emitTargetTeamsRegion() argument
6090 Action.Enter(CGF); in emitTargetTeamsRegion()
6092 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetTeamsRegion() argument
6093 Action.Enter(CGF); in emitTargetTeamsRegion()
6094 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsRegion()
6095 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in emitTargetTeamsRegion()
6096 CGF.EmitOMPPrivateClause(S, PrivateScope); in emitTargetTeamsRegion()
6097 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsRegion()
6100 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitTargetTeamsRegion()
6101 CGF.EmitStmt(CS->getCapturedStmt()); in emitTargetTeamsRegion()
6102 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsRegion()
6104 emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen); in emitTargetTeamsRegion()
6105 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsRegion()
6112 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDeviceFunction() argument
6113 emitTargetTeamsRegion(CGF, Action, S); in EmitOMPTargetTeamsDeviceFunction()
6125 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDirective() argument
6126 emitTargetTeamsRegion(CGF, Action, S); in EmitOMPTargetTeamsDirective()
6132 emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, in emitTargetTeamsDistributeRegion() argument
6134 Action.Enter(CGF); in emitTargetTeamsDistributeRegion()
6135 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeRegion() argument
6136 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in emitTargetTeamsDistributeRegion()
6140 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeRegion()
6142 Action.Enter(CGF); in emitTargetTeamsDistributeRegion()
6143 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeRegion()
6144 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeRegion()
6146 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in emitTargetTeamsDistributeRegion()
6148 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeRegion()
6150 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen); in emitTargetTeamsDistributeRegion()
6151 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeRegion()
6158 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeDeviceFunction() argument
6159 emitTargetTeamsDistributeRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeDeviceFunction()
6171 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeDirective() argument
6172 emitTargetTeamsDistributeRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeDirective()
6178 CodeGenFunction &CGF, PrePostActionTy &Action, in emitTargetTeamsDistributeSimdRegion() argument
6180 Action.Enter(CGF); in emitTargetTeamsDistributeSimdRegion()
6181 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeSimdRegion() argument
6182 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in emitTargetTeamsDistributeSimdRegion()
6186 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeSimdRegion()
6188 Action.Enter(CGF); in emitTargetTeamsDistributeSimdRegion()
6189 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeSimdRegion()
6190 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeSimdRegion()
6192 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in emitTargetTeamsDistributeSimdRegion()
6194 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeSimdRegion()
6196 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen); in emitTargetTeamsDistributeSimdRegion()
6197 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeSimdRegion()
6204 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeSimdDeviceFunction() argument
6205 emitTargetTeamsDistributeSimdRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeSimdDeviceFunction()
6217 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeSimdDirective() argument
6218 emitTargetTeamsDistributeSimdRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeSimdDirective()
6226 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeDirective() argument
6227 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPTeamsDistributeDirective()
6231 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeDirective()
6233 Action.Enter(CGF); in EmitOMPTeamsDistributeDirective()
6234 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeDirective()
6235 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeDirective()
6237 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in EmitOMPTeamsDistributeDirective()
6239 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeDirective()
6248 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeSimdDirective() argument
6249 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPTeamsDistributeSimdDirective()
6253 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeSimdDirective()
6255 Action.Enter(CGF); in EmitOMPTeamsDistributeSimdDirective()
6256 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeSimdDirective()
6257 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeSimdDirective()
6259 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd, in EmitOMPTeamsDistributeSimdDirective()
6261 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeSimdDirective()
6270 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeParallelForDirective() argument
6271 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPTeamsDistributeParallelForDirective()
6276 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeParallelForDirective()
6278 Action.Enter(CGF); in EmitOMPTeamsDistributeParallelForDirective()
6279 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeParallelForDirective()
6280 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeParallelForDirective()
6282 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in EmitOMPTeamsDistributeParallelForDirective()
6284 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeParallelForDirective()
6293 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeParallelForSimdDirective() argument
6294 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPTeamsDistributeParallelForSimdDirective()
6299 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeParallelForSimdDirective()
6301 Action.Enter(CGF); in EmitOMPTeamsDistributeParallelForSimdDirective()
6302 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeParallelForSimdDirective()
6303 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeParallelForSimdDirective()
6305 CGF.CGM.getOpenMPRuntime().emitInlinedDirective( in EmitOMPTeamsDistributeParallelForSimdDirective()
6306 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); in EmitOMPTeamsDistributeParallelForSimdDirective()
6307 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeParallelForSimdDirective()
6316 CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, in emitTargetTeamsDistributeParallelForRegion() argument
6318 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForRegion()
6319 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeParallelForRegion() argument
6320 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in emitTargetTeamsDistributeParallelForRegion()
6325 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeParallelForRegion()
6327 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForRegion()
6328 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeParallelForRegion()
6329 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeParallelForRegion()
6331 CGF.CGM.getOpenMPRuntime().emitInlinedDirective( in emitTargetTeamsDistributeParallelForRegion()
6332 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); in emitTargetTeamsDistributeParallelForRegion()
6333 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeParallelForRegion()
6336 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for, in emitTargetTeamsDistributeParallelForRegion()
6338 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeParallelForRegion()
6347 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForDeviceFunction() argument
6348 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForDeviceFunction()
6360 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForDirective() argument
6361 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForDirective()
6367 CodeGenFunction &CGF, in emitTargetTeamsDistributeParallelForSimdRegion() argument
6370 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForSimdRegion()
6371 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeParallelForSimdRegion() argument
6372 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in emitTargetTeamsDistributeParallelForSimdRegion()
6377 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeParallelForSimdRegion()
6379 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForSimdRegion()
6380 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeParallelForSimdRegion()
6381 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeParallelForSimdRegion()
6383 CGF.CGM.getOpenMPRuntime().emitInlinedDirective( in emitTargetTeamsDistributeParallelForSimdRegion()
6384 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); in emitTargetTeamsDistributeParallelForSimdRegion()
6385 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeParallelForSimdRegion()
6388 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd, in emitTargetTeamsDistributeParallelForSimdRegion()
6390 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeParallelForSimdRegion()
6399 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction() argument
6400 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction()
6412 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForSimdDirective() argument
6413 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForSimdDirective()
6601 void Enter(CodeGenFunction &CGF) override { in EmitOMPTargetDataDirective() argument
6608 CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetDataDirective() argument
6609 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTargetDataDirective() argument
6610 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in EmitOMPTargetDataDirective()
6615 &InnermostCodeGen](CodeGenFunction &CGF, in EmitOMPTargetDataDirective()
6622 Action.Enter(CGF); in EmitOMPTargetDataDirective()
6625 OMPPrivateScope PrivateScope(CGF); in EmitOMPTargetDataDirective()
6628 CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope, in EmitOMPTargetDataDirective()
6631 CGF.EmitOMPUseDeviceAddrClause(*C, PrivateScope, in EmitOMPTargetDataDirective()
6634 RCG(CGF); in EmitOMPTargetDataDirective()
6636 OMPLexicalScope Scope(CGF, S, OMPD_unknown); in EmitOMPTargetDataDirective()
6637 RCG(CGF); in EmitOMPTargetDataDirective()
6648 OMPLexicalScope Scope(CGF, S); in EmitOMPTargetDataDirective()
6649 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data, in EmitOMPTargetDataDirective()
6722 static void emitTargetParallelRegion(CodeGenFunction &CGF, in emitTargetParallelRegion() argument
6727 Action.Enter(CGF); in emitTargetParallelRegion()
6728 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetParallelRegion() argument
6729 Action.Enter(CGF); in emitTargetParallelRegion()
6730 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetParallelRegion()
6731 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in emitTargetParallelRegion()
6732 CGF.EmitOMPPrivateClause(S, PrivateScope); in emitTargetParallelRegion()
6733 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetParallelRegion()
6736 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitTargetParallelRegion()
6738 CGF.EmitStmt(CS->getCapturedStmt()); in emitTargetParallelRegion()
6739 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in emitTargetParallelRegion()
6741 emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen, in emitTargetParallelRegion()
6743 emitPostUpdateForReductionClause(CGF, S, in emitTargetParallelRegion()
6750 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelDeviceFunction() argument
6751 emitTargetParallelRegion(CGF, S, Action); in EmitOMPTargetParallelDeviceFunction()
6763 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelDirective() argument
6764 emitTargetParallelRegion(CGF, S, Action); in EmitOMPTargetParallelDirective()
6769 static void emitTargetParallelForRegion(CodeGenFunction &CGF, in emitTargetParallelForRegion() argument
6772 Action.Enter(CGF); in emitTargetParallelForRegion()
6775 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetParallelForRegion() argument
6776 Action.Enter(CGF); in emitTargetParallelForRegion()
6778 CGF, OMPD_target_parallel_for, S.hasCancel()); in emitTargetParallelForRegion()
6779 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, in emitTargetParallelForRegion()
6782 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen, in emitTargetParallelForRegion()
6790 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForDeviceFunction() argument
6791 emitTargetParallelForRegion(CGF, S, Action); in EmitOMPTargetParallelForDeviceFunction()
6803 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForDirective() argument
6804 emitTargetParallelForRegion(CGF, S, Action); in EmitOMPTargetParallelForDirective()
6810 emitTargetParallelForSimdRegion(CodeGenFunction &CGF, in emitTargetParallelForSimdRegion() argument
6813 Action.Enter(CGF); in emitTargetParallelForSimdRegion()
6816 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetParallelForSimdRegion() argument
6817 Action.Enter(CGF); in emitTargetParallelForSimdRegion()
6818 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, in emitTargetParallelForSimdRegion()
6821 emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen, in emitTargetParallelForSimdRegion()
6829 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForSimdDeviceFunction() argument
6830 emitTargetParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetParallelForSimdDeviceFunction()
6842 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForSimdDirective() argument
6843 emitTargetParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetParallelForSimdDirective()
6849 static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, in mapParam() argument
6854 [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); }); in mapParam()
6892 auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
6904 OMPLoopScope PreInitScope(CGF, S); in EmitOMPTaskLoopBasedDirective()
6905 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { in EmitOMPTaskLoopBasedDirective()
6909 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then"); in EmitOMPTaskLoopBasedDirective()
6910 ContBlock = CGF.createBasicBlock("taskloop.if.end"); in EmitOMPTaskLoopBasedDirective()
6911 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, in EmitOMPTaskLoopBasedDirective()
6912 CGF.getProfileCount(&S)); in EmitOMPTaskLoopBasedDirective()
6913 CGF.EmitBlock(ThenBlock); in EmitOMPTaskLoopBasedDirective()
6914 CGF.incrementProfileCounter(&S); in EmitOMPTaskLoopBasedDirective()
6917 (void)CGF.EmitOMPLinearClauseInit(S); in EmitOMPTaskLoopBasedDirective()
6919 OMPPrivateScope LoopScope(CGF); in EmitOMPTaskLoopBasedDirective()
6927 mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP, in EmitOMPTaskLoopBasedDirective()
6929 mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP, in EmitOMPTaskLoopBasedDirective()
6931 mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope); in EmitOMPTaskLoopBasedDirective()
6932 mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP, in EmitOMPTaskLoopBasedDirective()
6934 CGF.EmitOMPPrivateLoopCounters(S, LoopScope); in EmitOMPTaskLoopBasedDirective()
6935 CGF.EmitOMPLinearClause(S, LoopScope); in EmitOMPTaskLoopBasedDirective()
6936 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); in EmitOMPTaskLoopBasedDirective()
6941 CGF.EmitVarDecl(*IVDecl); in EmitOMPTaskLoopBasedDirective()
6942 CGF.EmitIgnoredExpr(S.getInit()); in EmitOMPTaskLoopBasedDirective()
6948 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); in EmitOMPTaskLoopBasedDirective()
6950 CGF.EmitIgnoredExpr(S.getCalcLastIteration()); in EmitOMPTaskLoopBasedDirective()
6954 OMPLexicalScope Scope(CGF, S, OMPD_taskloop, /*EmitPreInitStmt=*/false); in EmitOMPTaskLoopBasedDirective()
6956 CGF, S, in EmitOMPTaskLoopBasedDirective()
6957 [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
6959 CGF.EmitOMPSimdInit(S); in EmitOMPTaskLoopBasedDirective()
6961 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
6962 CGF.EmitOMPInnerLoop( in EmitOMPTaskLoopBasedDirective()
6964 [&S](CodeGenFunction &CGF) { in EmitOMPTaskLoopBasedDirective() argument
6965 emitOMPLoopBodyWithStopPoint(CGF, S, in EmitOMPTaskLoopBasedDirective()
6973 CGF.EmitBranch(ContBlock); in EmitOMPTaskLoopBasedDirective()
6974 CGF.EmitBlock(ContBlock, true); in EmitOMPTaskLoopBasedDirective()
6978 CGF.EmitOMPLastprivateClauseFinal( in EmitOMPTaskLoopBasedDirective()
6980 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar( in EmitOMPTaskLoopBasedDirective()
6981 CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false, in EmitOMPTaskLoopBasedDirective()
6984 CGF.EmitOMPLinearClauseFinal(S, [LIP, &S](CodeGenFunction &CGF) { in EmitOMPTaskLoopBasedDirective() argument
6985 return CGF.Builder.CreateIsNotNull( in EmitOMPTaskLoopBasedDirective()
6986 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false, in EmitOMPTaskLoopBasedDirective()
6991 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn, in EmitOMPTaskLoopBasedDirective()
6994 &Data](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
6995 OMPLoopScope PreInitScope(CGF, S); in EmitOMPTaskLoopBasedDirective()
6996 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S, in EmitOMPTaskLoopBasedDirective()
7000 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop, in EmitOMPTaskLoopBasedDirective()
7008 [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF, in EmitOMPTaskLoopBasedDirective()
7010 Action.Enter(CGF); in EmitOMPTaskLoopBasedDirective()
7011 CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, in EmitOMPTaskLoopBasedDirective()
7034 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPMasterTaskLoopDirective() argument
7035 Action.Enter(CGF); in EmitOMPMasterTaskLoopDirective()
7046 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPMasterTaskLoopSimdDirective() argument
7047 Action.Enter(CGF); in EmitOMPMasterTaskLoopSimdDirective()
7058 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelMasterTaskLoopDirective() argument
7059 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF, in EmitOMPParallelMasterTaskLoopDirective()
7061 Action.Enter(CGF); in EmitOMPParallelMasterTaskLoopDirective()
7062 CGF.EmitOMPTaskLoopBasedDirective(S); in EmitOMPParallelMasterTaskLoopDirective()
7064 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false); in EmitOMPParallelMasterTaskLoopDirective()
7065 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen, in EmitOMPParallelMasterTaskLoopDirective()
7076 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelMasterTaskLoopSimdDirective() argument
7077 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF, in EmitOMPParallelMasterTaskLoopSimdDirective()
7079 Action.Enter(CGF); in EmitOMPParallelMasterTaskLoopSimdDirective()
7080 CGF.EmitOMPTaskLoopBasedDirective(S); in EmitOMPParallelMasterTaskLoopSimdDirective()
7082 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false); in EmitOMPParallelMasterTaskLoopSimdDirective()
7083 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen, in EmitOMPParallelMasterTaskLoopSimdDirective()
7122 auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitSimpleOMPExecutableDirective() argument
7123 OMPPrivateScope GlobalsScope(CGF); in EmitSimpleOMPExecutableDirective()
7134 if (!CGF.LocalDeclMap.count(VD)) { in EmitSimpleOMPExecutableDirective()
7135 LValue GlobLVal = CGF.EmitLValue(Ref); in EmitSimpleOMPExecutableDirective()
7137 VD, [&GlobLVal, &CGF]() { return GlobLVal.getAddress(CGF); }); in EmitSimpleOMPExecutableDirective()
7144 ParentLoopDirectiveForScanRegion ScanRegion(CGF, D); in EmitSimpleOMPExecutableDirective()
7145 emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action); in EmitSimpleOMPExecutableDirective()
7150 if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) { in EmitSimpleOMPExecutableDirective()
7151 LValue GlobLVal = CGF.EmitLValue(E); in EmitSimpleOMPExecutableDirective()
7153 VD, [&GlobLVal, &CGF]() { return GlobLVal.getAddress(CGF); }); in EmitSimpleOMPExecutableDirective()
7157 if (!CGF.LocalDeclMap.count(VD)) in EmitSimpleOMPExecutableDirective()
7158 CGF.EmitVarDecl(*VD); in EmitSimpleOMPExecutableDirective()
7170 if (!CGF.LocalDeclMap.count(VD)) in EmitSimpleOMPExecutableDirective()
7171 CGF.EmitVarDecl(*VD); in EmitSimpleOMPExecutableDirective()
7177 CGF.EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt()); in EmitSimpleOMPExecutableDirective()