Lines Matching refs:CGF
48 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) { in emitPreInitStmt() argument
55 CGF.EmitVarDecl(cast<VarDecl>(*I)); in emitPreInitStmt()
58 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in emitPreInitStmt()
59 CGF.EmitAutoVarCleanups(Emission); in emitPreInitStmt()
68 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) { in isCapturedVar() argument
69 return CGF.LambdaCaptureFields.lookup(VD) || in isCapturedVar()
70 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) || in isCapturedVar()
71 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) && in isCapturedVar()
72 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD)); in isCapturedVar()
77 CodeGenFunction &CGF, const OMPExecutableDirective &S, in OMPLexicalScope() argument
80 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()), in OMPLexicalScope()
81 InlinedShareds(CGF) { in OMPLexicalScope()
83 emitPreInitStmt(CGF, S); in OMPLexicalScope()
95 CGF.getContext(), const_cast<VarDecl *>(VD), in OMPLexicalScope()
96 isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo && in OMPLexicalScope()
99 InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress(CGF)); in OMPLexicalScope()
117 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) in OMPParallelScope() argument
118 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt, in OMPParallelScope()
132 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) in OMPTeamsScope() argument
133 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt, in OMPTeamsScope()
140 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopBasedDirective &S) { in emitPreInitStmt() argument
149 CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType())); in emitPreInitStmt()
159 CGF, OrigVD, in emitPreInitStmt()
160 Address(llvm::UndefValue::get(CGF.ConvertTypeForMem( in emitPreInitStmt()
161 CGF.getContext().getPointerType(OrigVDTy))), in emitPreInitStmt()
162 CGF.ConvertTypeForMem(OrigVDTy), in emitPreInitStmt()
163 CGF.getContext().getDeclAlign(OrigVD))); in emitPreInitStmt()
167 (void)PreCondVars.apply(CGF); in emitPreInitStmt()
172 [&CGF](unsigned Cnt, const Stmt *CurStmt) { in emitPreInitStmt()
175 CGF.EmitStmt(Init); in emitPreInitStmt()
176 CGF.EmitStmt(CXXFor->getRangeStmt()); in emitPreInitStmt()
177 CGF.EmitStmt(CXXFor->getEndStmt()); in emitPreInitStmt()
191 CGF.EmitVarDecl(cast<VarDecl>(*I)); in emitPreInitStmt()
193 PreCondVars.restore(CGF); in emitPreInitStmt()
197 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopBasedDirective &S) in OMPLoopScope() argument
198 : CodeGenFunction::RunCleanupsScope(CGF) { in OMPLoopScope()
199 emitPreInitStmt(CGF, S); in OMPLoopScope()
206 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) { in isCapturedVar() argument
207 return CGF.LambdaCaptureFields.lookup(VD) || in isCapturedVar()
208 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) || in isCapturedVar()
209 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) && in isCapturedVar()
210 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD)); in isCapturedVar()
214 OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) in OMPSimdLexicalScope() argument
215 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()), in OMPSimdLexicalScope()
216 InlinedShareds(CGF) { in OMPSimdLexicalScope()
223 CGF.EmitVarDecl(cast<VarDecl>(*I)); in OMPSimdLexicalScope()
226 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in OMPSimdLexicalScope()
227 CGF.EmitAutoVarCleanups(Emission); in OMPSimdLexicalScope()
235 CGF.EmitVarDecl(*OED); in OMPSimdLexicalScope()
241 CGF.EmitVarDecl(*OED); in OMPSimdLexicalScope()
246 CGF.EmitOMPPrivateClause(S, InlinedShareds); in OMPSimdLexicalScope()
249 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl())); in OMPSimdLexicalScope()
269 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD), in OMPSimdLexicalScope()
270 isCapturedVar(CGF, VD) || in OMPSimdLexicalScope()
271 (CGF.CapturedStmtInfo && in OMPSimdLexicalScope()
275 InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress(CGF)); in OMPSimdLexicalScope()
286 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
372 static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, in castValueFromUintptr() argument
375 ASTContext &Ctx = CGF.getContext(); in castValueFromUintptr()
377 llvm::Value *CastedPtr = CGF.EmitScalarConversion( in castValueFromUintptr()
378 AddrLV.getAddress(CGF).getPointer(), Ctx.getUIntPtrType(), in castValueFromUintptr()
381 CGF.MakeNaturalAlignAddrLValue(CastedPtr, DstType).getAddress(CGF); in castValueFromUintptr()
426 CodeGenFunction &CGF, FunctionArgList &Args, in emitOutlinedFunctionPrologue() argument
438 CodeGenModule &CGM = CGF.CGM; in emitOutlinedFunctionPrologue()
533 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs, in emitOutlinedFunctionPrologue()
543 LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt], in emitOutlinedFunctionPrologue()
546 LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]); in emitOutlinedFunctionPrologue()
559 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(), in emitOutlinedFunctionPrologue()
563 ArgLVal = CGF.MakeAddrLValue( in emitOutlinedFunctionPrologue()
564 castValueFromUintptr(CGF, I->getLocation(), FD->getType(), in emitOutlinedFunctionPrologue()
568 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation()); in emitOutlinedFunctionPrologue()
574 Address ArgAddr = ArgLVal.getAddress(CGF); in emitOutlinedFunctionPrologue()
576 ArgAddr = CGF.EmitLoadOfReference(ArgLVal); in emitOutlinedFunctionPrologue()
579 ArgAddr = CGF.EmitLoadOfPointer( in emitOutlinedFunctionPrologue()
593 CGF, I->getLocation(), FD->getType(), in emitOutlinedFunctionPrologue()
595 : ArgLVal.getAddress(CGF)}}); in emitOutlinedFunctionPrologue()
599 CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation()); in emitOutlinedFunctionPrologue()
600 LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress(CGF)}}); in emitOutlinedFunctionPrologue()
1236 [&Emission](CodeGenFunction &CGF) { in EmitOMPReductionClauseInit() argument
1237 CGF.EmitAutoVarInit(Emission); in EmitOMPReductionClauseInit()
1445 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitPostUpdateForReductionClause() argument
1447 if (!CGF.HaveInsertPoint()) in emitPostUpdateForReductionClause()
1453 if (llvm::Value *Cond = CondGen(CGF)) { in emitPostUpdateForReductionClause()
1456 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu"); in emitPostUpdateForReductionClause()
1457 DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done"); in emitPostUpdateForReductionClause()
1458 CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB); in emitPostUpdateForReductionClause()
1459 CGF.EmitBlock(ThenBB); in emitPostUpdateForReductionClause()
1462 CGF.EmitIgnoredExpr(PostUpdate); in emitPostUpdateForReductionClause()
1466 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); in emitPostUpdateForReductionClause()
1480 checkForLastprivateConditionalUpdate(CodeGenFunction &CGF, in checkForLastprivateConditionalUpdate() argument
1482 if (CGF.getLangOpts().OpenMP < 50) in checkForLastprivateConditionalUpdate()
1493 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref); in checkForLastprivateConditionalUpdate()
1504 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref); in checkForLastprivateConditionalUpdate()
1515 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref); in checkForLastprivateConditionalUpdate()
1532 CGF.CGM.getOpenMPRuntime().checkAndEmitSharedLastprivateConditional( in checkForLastprivateConditionalUpdate()
1533 CGF, S, PrivateDecls); in checkForLastprivateConditionalUpdate()
1537 CodeGenFunction &CGF, const OMPExecutableDirective &S, in emitCommonOMPParallelDirective() argument
1543 CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction( in emitCommonOMPParallelDirective()
1544 CGF, S, *CS->getCapturedDecl()->param_begin(), InnermostKind, in emitCommonOMPParallelDirective()
1547 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); in emitCommonOMPParallelDirective()
1548 NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(), in emitCommonOMPParallelDirective()
1550 CGF.CGM.getOpenMPRuntime().emitNumThreadsClause( in emitCommonOMPParallelDirective()
1551 CGF, NumThreads, NumThreadsClause->getBeginLoc()); in emitCommonOMPParallelDirective()
1554 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF); in emitCommonOMPParallelDirective()
1555 CGF.CGM.getOpenMPRuntime().emitProcBindClause( in emitCommonOMPParallelDirective()
1556 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc()); in emitCommonOMPParallelDirective()
1567 OMPParallelScope Scope(CGF, S); in emitCommonOMPParallelDirective()
1573 CodeGenBoundParameters(CGF, S, CapturedVars); in emitCommonOMPParallelDirective()
1574 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); in emitCommonOMPParallelDirective()
1575 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn, in emitCommonOMPParallelDirective()
1594 static void emitOMPCopyinClause(CodeGenFunction &CGF, in emitOMPCopyinClause() argument
1596 bool Copyins = CGF.EmitOMPCopyinClause(S); in emitOMPCopyinClause()
1601 CGF.CGM.getOpenMPRuntime().emitBarrierCall( in emitOMPCopyinClause()
1602 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, in emitOMPCopyinClause()
1608 CodeGenFunction &CGF, const VarDecl *VD) { in getAddressOfLocalVariable() argument
1609 CodeGenModule &CGM = CGF.CGM; in getAddressOfLocalVariable()
1620 Size = CGF.getTypeSize(CVD->getType()); in getAddressOfLocalVariable()
1622 Size = CGF.Builder.CreateNUWAdd( in getAddressOfLocalVariable()
1624 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align)); in getAddressOfLocalVariable()
1625 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align)); in getAddressOfLocalVariable()
1634 llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator()); in getAddressOfLocalVariable()
1638 Allocator = CGF.Builder.CreateIntToPtr(Allocator, CGM.VoidPtrTy); in getAddressOfLocalVariable()
1640 Allocator = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Allocator, in getAddressOfLocalVariable()
1644 CGF.Builder, Size, Allocator, in getAddressOfLocalVariable()
1647 OMPBuilder.createOMPFree(CGF.Builder, Addr, Allocator); in getAddressOfLocalVariable()
1649 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(NormalAndEHCleanup, FreeCI); in getAddressOfLocalVariable()
1650 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddressOfLocalVariable()
1652 CGF.ConvertTypeForMem(CGM.getContext().getPointerType(CVD->getType())), in getAddressOfLocalVariable()
1654 return Address(Addr, CGF.ConvertTypeForMem(CVD->getType()), Align); in getAddressOfLocalVariable()
1658 CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, in getAddrOfThreadPrivate() argument
1660 CodeGenModule &CGM = CGF.CGM; in getAddrOfThreadPrivate()
1669 CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy); in getAddrOfThreadPrivate()
1675 OMPBuilder.createCachedThreadPrivate(CGF.Builder, Data, Size, CacheName); in getAddrOfThreadPrivate()
1693 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP, in EmitOMPInlinedRegionBody() argument
1695 CGBuilderTy &Builder = CGF.Builder; in EmitOMPInlinedRegionBody()
1701 OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB); in EmitOMPInlinedRegionBody()
1702 CGF.EmitStmt(RegionBodyStmt); in EmitOMPInlinedRegionBody()
1710 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP, in EmitOMPOutlinedRegionBody() argument
1712 CGBuilderTy &Builder = CGF.Builder; in EmitOMPOutlinedRegionBody()
1718 OMPBuilderCBHelpers::OutlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB); in EmitOMPOutlinedRegionBody()
1719 CGF.EmitStmt(RegionBodyStmt); in EmitOMPOutlinedRegionBody()
1785 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelDirective() argument
1786 Action.Enter(CGF); in EmitOMPParallelDirective()
1787 OMPPrivateScope PrivateScope(CGF); in EmitOMPParallelDirective()
1788 emitOMPCopyinClause(CGF, S); in EmitOMPParallelDirective()
1789 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in EmitOMPParallelDirective()
1790 CGF.EmitOMPPrivateClause(S, PrivateScope); in EmitOMPParallelDirective()
1791 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPParallelDirective()
1793 CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt()); in EmitOMPParallelDirective()
1794 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in EmitOMPParallelDirective()
1825 OMPTransformDirectiveScopeRAII(CodeGenFunction &CGF, const Stmt *S) { in OMPTransformDirectiveScopeRAII() argument
1827 Scope = new OMPLoopScope(CGF, *Dir); in OMPTransformDirectiveScopeRAII()
1829 CapInfoRAII = new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI); in OMPTransformDirectiveScopeRAII()
1842 static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop, in emitBody() argument
1848 CGF.getContext().getSourceManager(), CS->getLBracLoc(), in emitBody()
1852 CodeGenFunction::LexicalScope Scope(CGF, S->getSourceRange()); in emitBody()
1854 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level); in emitBody()
1868 CGF.EmitStmt(CXXFor->getLoopVarStmt()); in emitBody()
1874 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1); in emitBody()
1878 CGF.EmitStmt(S); in emitBody()
1956 CodeGenFunction CGF(ParentCGF.CGM, /*suppressNewContext=*/true); in emitCapturedStmtFunc() local
1959 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, CSI.get()); in emitCapturedStmtFunc()
1960 llvm::Function *F = CGF.GenerateCapturedStmtFunction(*S); in emitCapturedStmtFunc()
2217 static void emitAlignedClause(CodeGenFunction &CGF, in emitAlignedClause() argument
2219 if (!CGF.HaveInsertPoint()) in emitAlignedClause()
2225 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr)); in emitAlignedClause()
2235 CGF.getContext() in emitAlignedClause()
2236 .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign( in emitAlignedClause()
2243 llvm::Value *PtrValue = CGF.EmitScalarExpr(E); in emitAlignedClause()
2244 CGF.emitAlignmentAssumption( in emitAlignedClause()
2246 llvm::ConstantInt::get(CGF.getLLVMContext(), Alignment)); in emitAlignedClause()
2294 static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, in emitPreCond() argument
2297 if (!CGF.HaveInsertPoint()) in emitPreCond()
2300 CodeGenFunction::OMPPrivateScope PreCondScope(CGF); in emitPreCond()
2301 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope); in emitPreCond()
2305 CGF.EmitIgnoredExpr(I); in emitPreCond()
2318 CGF.CreateMemTemp(VD->getType().getNonReferenceType()); in emitPreCond()
2319 (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr); in emitPreCond()
2321 (void)PreCondVars.apply(CGF); in emitPreCond()
2325 CGF.EmitIgnoredExpr(E); in emitPreCond()
2328 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount); in emitPreCond()
2329 PreCondVars.restore(CGF); in emitPreCond()
2366 static void emitSimdlenSafelenClause(CodeGenFunction &CGF, in emitSimdlenSafelenClause() argument
2368 if (!CGF.HaveInsertPoint()) in emitSimdlenSafelenClause()
2371 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(), in emitSimdlenSafelenClause()
2374 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue()); in emitSimdlenSafelenClause()
2378 CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>()); in emitSimdlenSafelenClause()
2380 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(), in emitSimdlenSafelenClause()
2383 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue()); in emitSimdlenSafelenClause()
2387 CGF.LoopStack.setParallel(/*Enable=*/false); in emitSimdlenSafelenClause()
2456 static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, in emitOMPLoopBodyWithStopPoint() argument
2459 CGF.EmitOMPLoopBody(S, LoopExit); in emitOMPLoopBodyWithStopPoint()
2460 CGF.EmitStopPoint(&S); in emitOMPLoopBodyWithStopPoint()
2464 static LValue EmitOMPHelperVar(CodeGenFunction &CGF, in EmitOMPHelperVar() argument
2467 CGF.EmitVarDecl(*VDecl); in EmitOMPHelperVar()
2468 return CGF.EmitLValue(Helper); in EmitOMPHelperVar()
2471 static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S, in emitCommonSimdLoop() argument
2474 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](CodeGenFunction &CGF, in emitCommonSimdLoop()
2476 CGOpenMPRuntime::NontemporalDeclsRAII NontemporalsRegion(CGF.CGM, S); in emitCommonSimdLoop()
2477 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in emitCommonSimdLoop()
2478 SimdInitGen(CGF); in emitCommonSimdLoop()
2480 BodyCodeGen(CGF); in emitCommonSimdLoop()
2482 auto &&ElseGen = [&BodyCodeGen](CodeGenFunction &CGF, PrePostActionTy &) { in emitCommonSimdLoop() argument
2483 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in emitCommonSimdLoop()
2484 CGF.LoopStack.setVectorizeEnable(/*Enable=*/false); in emitCommonSimdLoop()
2486 BodyCodeGen(CGF); in emitCommonSimdLoop()
2491 if (CGF.getLangOpts().OpenMP >= 50 && in emitCommonSimdLoop()
2500 CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, IfCond, ThenGen, ElseGen); in emitCommonSimdLoop()
2503 ThenRCG(CGF); in emitCommonSimdLoop()
2507 static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, in emitOMPSimdRegion() argument
2509 Action.Enter(CGF); in emitOMPSimdRegion()
2512 OMPLoopScope PreInitScope(CGF, S); in emitOMPSimdRegion()
2521 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable())); in emitOMPSimdRegion()
2522 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable())); in emitOMPSimdRegion()
2530 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { in emitOMPSimdRegion()
2534 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then"); in emitOMPSimdRegion()
2535 ContBlock = CGF.createBasicBlock("simd.if.end"); in emitOMPSimdRegion()
2536 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, in emitOMPSimdRegion()
2537 CGF.getProfileCount(&S)); in emitOMPSimdRegion()
2538 CGF.EmitBlock(ThenBlock); in emitOMPSimdRegion()
2539 CGF.incrementProfileCounter(&S); in emitOMPSimdRegion()
2545 CGF.EmitVarDecl(*IVDecl); in emitOMPSimdRegion()
2546 CGF.EmitIgnoredExpr(S.getInit()); in emitOMPSimdRegion()
2552 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); in emitOMPSimdRegion()
2554 CGF.EmitIgnoredExpr(S.getCalcLastIteration()); in emitOMPSimdRegion()
2557 emitAlignedClause(CGF, S); in emitOMPSimdRegion()
2558 (void)CGF.EmitOMPLinearClauseInit(S); in emitOMPSimdRegion()
2560 CodeGenFunction::OMPPrivateScope LoopScope(CGF); in emitOMPSimdRegion()
2561 CGF.EmitOMPPrivateClause(S, LoopScope); in emitOMPSimdRegion()
2562 CGF.EmitOMPPrivateLoopCounters(S, LoopScope); in emitOMPSimdRegion()
2563 CGF.EmitOMPLinearClause(S, LoopScope); in emitOMPSimdRegion()
2564 CGF.EmitOMPReductionClauseInit(S, LoopScope); in emitOMPSimdRegion()
2566 CGF, S, CGF.EmitLValue(S.getIterationVariable())); in emitOMPSimdRegion()
2567 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); in emitOMPSimdRegion()
2570 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitOMPSimdRegion()
2573 CGF, S, in emitOMPSimdRegion()
2574 [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitOMPSimdRegion() argument
2575 CGF.EmitOMPSimdInit(S); in emitOMPSimdRegion()
2577 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { in emitOMPSimdRegion() argument
2578 CGF.EmitOMPInnerLoop( in emitOMPSimdRegion()
2580 [&S](CodeGenFunction &CGF) { in emitOMPSimdRegion() argument
2581 emitOMPLoopBodyWithStopPoint(CGF, S, in emitOMPSimdRegion()
2586 CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; }); in emitOMPSimdRegion()
2589 CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true); in emitOMPSimdRegion()
2590 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd); in emitOMPSimdRegion()
2591 emitPostUpdateForReductionClause(CGF, S, in emitOMPSimdRegion()
2594 CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; }); in emitOMPSimdRegion()
2598 CGF.EmitBranch(ContBlock); in emitOMPSimdRegion()
2599 CGF.EmitBlock(ContBlock, true); in emitOMPSimdRegion()
2635 GetAlignedMapping(const OMPSimdDirective &S, CodeGenFunction &CGF) { in GetAlignedMapping() argument
2641 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr)); in GetAlignedMapping()
2651 CGF.getContext() in GetAlignedMapping()
2652 .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign( in GetAlignedMapping()
2658 llvm::Value *PtrValue = CGF.EmitScalarExpr(E); in GetAlignedMapping()
2659 AlignedVars[PtrValue] = CGF.Builder.getInt64(Alignment.getSExtValue()); in GetAlignedMapping()
2669 auto &&CodeGenIRBuilder = [this, &S, UseOMPIRBuilder](CodeGenFunction &CGF, in EmitOMPSimdDirective()
2674 GetAlignedMapping(S, CGF); in EmitOMPSimdDirective()
2724 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPSimdDirective() argument
2725 emitOMPSimdRegion(CGF, S, Action); in EmitOMPSimdDirective()
2868 [&S, IsMonotonic](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPOuterLoop() argument
2872 CGF.LoopStack.setParallel(!IsMonotonic); in EmitOMPOuterLoop()
2875 CGF.LoopStack.setParallel(/*Enable=*/true); in EmitOMPOuterLoop()
2877 CGF.EmitOMPSimdInit(S); in EmitOMPOuterLoop()
2881 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPOuterLoop() argument
2888 CGF.EmitOMPInnerLoop( in EmitOMPOuterLoop()
2890 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) { in EmitOMPOuterLoop() argument
2891 CodeGenLoop(CGF, S, LoopExit); in EmitOMPOuterLoop()
2893 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) { in EmitOMPOuterLoop() argument
2894 CodeGenOrdered(CGF, Loc, IVSize, IVSigned); in EmitOMPOuterLoop()
2913 auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) { in EmitOMPOuterLoop() argument
2915 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), in EmitOMPOuterLoop()
3006 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc, in EmitOMPForOuterLoop()
3010 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize, in EmitOMPForOuterLoop()
3090 emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, in emitDistributeParallelForInnerBounds() argument
3094 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable())); in emitDistributeParallelForInnerBounds()
3096 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable())); in emitDistributeParallelForInnerBounds()
3104 LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable()); in emitDistributeParallelForInnerBounds()
3105 LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable()); in emitDistributeParallelForInnerBounds()
3106 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar( in emitDistributeParallelForInnerBounds()
3108 PrevLBVal = CGF.EmitScalarConversion( in emitDistributeParallelForInnerBounds()
3112 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar( in emitDistributeParallelForInnerBounds()
3114 PrevUBVal = CGF.EmitScalarConversion( in emitDistributeParallelForInnerBounds()
3119 CGF.EmitStoreOfScalar(PrevLBVal, LB); in emitDistributeParallelForInnerBounds()
3120 CGF.EmitStoreOfScalar(PrevUBVal, UB); in emitDistributeParallelForInnerBounds()
3133 emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, in emitDistributeParallelForDispatchBounds() argument
3144 CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc()); in emitDistributeParallelForDispatchBounds()
3146 CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc()); in emitDistributeParallelForDispatchBounds()
3151 CodeGenFunction &CGF, const OMPExecutableDirective &S, in emitDistributeParallelForDistributeInnerBoundParams() argument
3155 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable())); in emitDistributeParallelForDistributeInnerBoundParams()
3157 CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(LB.getAddress(CGF)), in emitDistributeParallelForDistributeInnerBoundParams()
3158 CGF.SizeTy, /*isSigned=*/false); in emitDistributeParallelForDistributeInnerBoundParams()
3161 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable())); in emitDistributeParallelForDistributeInnerBoundParams()
3164 CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(UB.getAddress(CGF)), in emitDistributeParallelForDistributeInnerBoundParams()
3165 CGF.SizeTy, /*isSigned=*/false); in emitDistributeParallelForDistributeInnerBoundParams()
3170 emitInnerParallelForWhenCombined(CodeGenFunction &CGF, in emitInnerParallelForWhenCombined() argument
3173 auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF, in emitInnerParallelForWhenCombined()
3175 Action.Enter(CGF); in emitInnerParallelForWhenCombined()
3186 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(), in emitInnerParallelForWhenCombined()
3188 CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(), in emitInnerParallelForWhenCombined()
3194 CGF, S, in emitInnerParallelForWhenCombined()
3202 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeParallelForDirective() argument
3203 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPDistributeParallelForDirective()
3212 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeParallelForSimdDirective() argument
3213 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPDistributeParallelForSimdDirective()
3222 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeSimdDirective() argument
3223 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPDistributeSimdDirective()
3232 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetSimdDeviceFunction() argument
3233 emitOMPSimdRegion(CGF, S, Action); in EmitOMPTargetSimdDeviceFunction()
3245 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetSimdDirective() argument
3246 emitOMPSimdRegion(CGF, S, Action); in EmitOMPTargetSimdDirective()
3400 [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPWorksharingLoop() argument
3402 CGF.EmitOMPSimdInit(S); in EmitOMPWorksharingLoop()
3405 CGF.LoopStack.setParallel(/*Enable=*/true); in EmitOMPWorksharingLoop()
3410 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPWorksharingLoop() argument
3417 IVSize, IVSigned, Ordered, IL.getAddress(CGF), in EmitOMPWorksharingLoop()
3418 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF), in EmitOMPWorksharingLoop()
3420 CGF.CGM.getOpenMPRuntime().emitForStaticInit( in EmitOMPWorksharingLoop()
3421 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, in EmitOMPWorksharingLoop()
3425 CGF.EmitIgnoredExpr(S.getEnsureUpperBound()); in EmitOMPWorksharingLoop()
3427 CGF.EmitIgnoredExpr(S.getInit()); in EmitOMPWorksharingLoop()
3441 CGF.EmitOMPInnerLoop( in EmitOMPWorksharingLoop()
3446 [&S, LoopExit](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3447 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit); in EmitOMPWorksharingLoop()
3453 auto &&CodeGen = [&S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3454 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), in EmitOMPWorksharingLoop()
3468 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3469 return CGF.Builder.CreateIsNotNull( in EmitOMPWorksharingLoop()
3470 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPWorksharingLoop()
3479 *this, S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3480 return CGF.Builder.CreateIsNotNull( in EmitOMPWorksharingLoop()
3481 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPWorksharingLoop()
3489 EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3490 return CGF.Builder.CreateIsNotNull( in EmitOMPWorksharingLoop()
3491 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPWorksharingLoop()
3508 emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) { in emitForLoopBounds() argument
3511 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable())); in emitForLoopBounds()
3513 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable())); in emitForLoopBounds()
3522 emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, in emitDispatchForLoopBounds() argument
3526 const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType()); in emitDispatchForLoopBounds()
3527 llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0); in emitDispatchForLoopBounds()
3528 llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration()); in emitDispatchForLoopBounds()
3540 CodeGenFunction &CGF, const OMPLoopDirective &S, in emitScanBasedDirectiveDecls() argument
3542 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast( in emitScanBasedDirectiveDecls()
3543 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false); in emitScanBasedDirectiveDecls()
3569 RedCG.emitSharedOrigLValue(CGF, Count); in emitScanBasedDirectiveDecls()
3570 RedCG.emitAggregateType(CGF, Count); in emitScanBasedDirectiveDecls()
3573 CGF, in emitScanBasedDirectiveDecls()
3579 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(*ITA)->getDecl())); in emitScanBasedDirectiveDecls()
3592 CodeGenFunction &CGF, const OMPLoopDirective &S, in emitScanBasedDirectiveFinals() argument
3594 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast( in emitScanBasedDirectiveFinals()
3595 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false); in emitScanBasedDirectiveFinals()
3615 llvm::Value *OMPLast = CGF.Builder.CreateNSWSub( in emitScanBasedDirectiveFinals()
3617 llvm::ConstantInt::get(CGF.SizeTy, 1, /*isSigned=*/false)); in emitScanBasedDirectiveFinals()
3623 CGF, in emitScanBasedDirectiveFinals()
3627 LValue DestLVal = CGF.EmitLValue(OrigExpr); in emitScanBasedDirectiveFinals()
3628 LValue SrcLVal = CGF.EmitLValue(CopyArrayElem); in emitScanBasedDirectiveFinals()
3629 CGF.EmitOMPCopy(PrivateExpr->getType(), DestLVal.getAddress(CGF), in emitScanBasedDirectiveFinals()
3630 SrcLVal.getAddress(CGF), in emitScanBasedDirectiveFinals()
3657 CodeGenFunction &CGF, const OMPLoopDirective &S, in emitScanBasedDirective() argument
3661 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast( in emitScanBasedDirective()
3662 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false); in emitScanBasedDirective()
3678 CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S); in emitScanBasedDirective()
3686 CGF.OMPFirstScanLoop = true; in emitScanBasedDirective()
3687 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in emitScanBasedDirective()
3688 FirstGen(CGF); in emitScanBasedDirective()
3693 &Privates](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitScanBasedDirective() argument
3694 Action.Enter(CGF); in emitScanBasedDirective()
3698 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock(); in emitScanBasedDirective()
3699 llvm::BasicBlock *LoopBB = CGF.createBasicBlock("omp.outer.log.scan.body"); in emitScanBasedDirective()
3700 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.outer.log.scan.exit"); in emitScanBasedDirective()
3702 CGF.CGM.getIntrinsic(llvm::Intrinsic::log2, CGF.DoubleTy); in emitScanBasedDirective()
3704 CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy); in emitScanBasedDirective()
3705 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg); in emitScanBasedDirective()
3706 F = CGF.CGM.getIntrinsic(llvm::Intrinsic::ceil, CGF.DoubleTy); in emitScanBasedDirective()
3707 LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal); in emitScanBasedDirective()
3708 LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy); in emitScanBasedDirective()
3709 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub( in emitScanBasedDirective()
3710 OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1)); in emitScanBasedDirective()
3711 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getBeginLoc()); in emitScanBasedDirective()
3712 CGF.EmitBlock(LoopBB); in emitScanBasedDirective()
3713 auto *Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2); in emitScanBasedDirective()
3715 auto *Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2); in emitScanBasedDirective()
3716 Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB); in emitScanBasedDirective()
3717 Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB); in emitScanBasedDirective()
3721 CGF.createBasicBlock("omp.inner.log.scan.body"); in emitScanBasedDirective()
3723 CGF.createBasicBlock("omp.inner.log.scan.exit"); in emitScanBasedDirective()
3724 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K); in emitScanBasedDirective()
3725 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB); in emitScanBasedDirective()
3726 CGF.EmitBlock(InnerLoopBB); in emitScanBasedDirective()
3727 auto *IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2); in emitScanBasedDirective()
3730 CodeGenFunction::OMPPrivateScope PrivScope(CGF); in emitScanBasedDirective()
3739 CGF, in emitScanBasedDirective()
3743 LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF); in emitScanBasedDirective()
3748 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K); in emitScanBasedDirective()
3750 CGF, in emitScanBasedDirective()
3754 RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF); in emitScanBasedDirective()
3761 CGF.CGM.getOpenMPRuntime().emitReduction( in emitScanBasedDirective()
3762 CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps, in emitScanBasedDirective()
3766 CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1)); in emitScanBasedDirective()
3767 IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock()); in emitScanBasedDirective()
3768 CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K); in emitScanBasedDirective()
3769 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB); in emitScanBasedDirective()
3770 CGF.EmitBlock(InnerExitBB); in emitScanBasedDirective()
3772 CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1)); in emitScanBasedDirective()
3773 Counter->addIncoming(Next, CGF.Builder.GetInsertBlock()); in emitScanBasedDirective()
3776 CGF.Builder.CreateShl(Pow2K, 1, "", /*HasNUW=*/true); in emitScanBasedDirective()
3777 Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock()); in emitScanBasedDirective()
3778 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(Next, LogVal); in emitScanBasedDirective()
3779 CGF.Builder.CreateCondBr(Cmp, LoopBB, ExitBB); in emitScanBasedDirective()
3780 auto DL1 = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getEndLoc()); in emitScanBasedDirective()
3781 CGF.EmitBlock(ExitBB); in emitScanBasedDirective()
3784 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc()); in emitScanBasedDirective()
3785 CGF.CGM.getOpenMPRuntime().emitBarrierCall( in emitScanBasedDirective()
3786 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, in emitScanBasedDirective()
3790 RCG(CGF); in emitScanBasedDirective()
3793 CGF.OMPFirstScanLoop = false; in emitScanBasedDirective()
3794 SecondGen(CGF); in emitScanBasedDirective()
3797 static bool emitWorksharingDirective(CodeGenFunction &CGF, in emitWorksharingDirective() argument
3805 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) { in emitWorksharingDirective() argument
3806 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in emitWorksharingDirective()
3807 OMPLoopScope LoopScope(CGF, S); in emitWorksharingDirective()
3808 return CGF.EmitScalarExpr(S.getNumIterations()); in emitWorksharingDirective()
3810 const auto &&FirstGen = [&S, HasCancel](CodeGenFunction &CGF) { in emitWorksharingDirective() argument
3812 CGF, S.getDirectiveKind(), HasCancel); in emitWorksharingDirective()
3813 (void)CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), in emitWorksharingDirective()
3817 CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getBeginLoc(), in emitWorksharingDirective()
3821 &HasLastprivates](CodeGenFunction &CGF) { in emitWorksharingDirective() argument
3823 CGF, S.getDirectiveKind(), HasCancel); in emitWorksharingDirective()
3824 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), in emitWorksharingDirective()
3829 emitScanBasedDirectiveDecls(CGF, S, NumIteratorsGen); in emitWorksharingDirective()
3830 emitScanBasedDirective(CGF, S, NumIteratorsGen, FirstGen, SecondGen); in emitWorksharingDirective()
3832 emitScanBasedDirectiveFinals(CGF, S, NumIteratorsGen); in emitWorksharingDirective()
3834 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(), in emitWorksharingDirective()
3836 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), in emitWorksharingDirective()
3897 UseOMPIRBuilder](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPForDirective() argument
3928 HasLastprivates = emitWorksharingDirective(CGF, S, S.hasCancel()); in EmitOMPForDirective()
3949 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF, in EmitOMPForSimdDirective()
3951 HasLastprivates = emitWorksharingDirective(CGF, S, /*HasCancel=*/false); in EmitOMPForSimdDirective()
3967 static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, in createSectionLVal() argument
3970 LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty); in createSectionLVal()
3972 CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true); in createSectionLVal()
3981 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) { in EmitSections() argument
3982 const ASTContext &C = CGF.getContext(); in EmitSections()
3986 LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.", in EmitSections()
3987 CGF.Builder.getInt32(0)); in EmitSections()
3989 ? CGF.Builder.getInt32(CS->size() - 1) in EmitSections()
3990 : CGF.Builder.getInt32(0); in EmitSections()
3992 createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal); in EmitSections()
3993 LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.", in EmitSections()
3994 CGF.Builder.getInt32(1)); in EmitSections()
3995 LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.", in EmitSections()
3996 CGF.Builder.getInt32(0)); in EmitSections()
3998 LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv."); in EmitSections()
4000 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV); in EmitSections()
4002 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB); in EmitSections()
4011 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) { in EmitSections() argument
4023 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit"); in EmitSections()
4025 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()), in EmitSections()
4030 auto CaseBB = CGF.createBasicBlock(".omp.sections.case"); in EmitSections()
4031 CGF.EmitBlock(CaseBB); in EmitSections()
4032 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB); in EmitSections()
4033 CGF.EmitStmt(SubStmt); in EmitSections()
4034 CGF.EmitBranch(ExitBB); in EmitSections()
4038 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case"); in EmitSections()
4039 CGF.EmitBlock(CaseBB); in EmitSections()
4040 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB); in EmitSections()
4041 CGF.EmitStmt(CapturedStmt); in EmitSections()
4042 CGF.EmitBranch(ExitBB); in EmitSections()
4044 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in EmitSections()
4047 CodeGenFunction::OMPPrivateScope LoopScope(CGF); in EmitSections()
4048 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) { in EmitSections()
4052 CGF.CGM.getOpenMPRuntime().emitBarrierCall( in EmitSections()
4053 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, in EmitSections()
4056 CGF.EmitOMPPrivateClause(S, LoopScope); in EmitSections()
4057 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(CGF, S, IV); in EmitSections()
4058 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); in EmitSections()
4059 CGF.EmitOMPReductionClauseInit(S, LoopScope); in EmitSections()
4062 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in EmitSections()
4068 /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(CGF), in EmitSections()
4069 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF)); in EmitSections()
4070 CGF.CGM.getOpenMPRuntime().emitForStaticInit( in EmitSections()
4071 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit); in EmitSections()
4073 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc()); in EmitSections()
4074 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect( in EmitSections()
4075 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal); in EmitSections()
4076 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB); in EmitSections()
4078 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV); in EmitSections()
4080 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, Cond, Inc, BodyGen, in EmitSections()
4083 auto &&CodeGen = [&S](CodeGenFunction &CGF) { in EmitSections() argument
4084 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), in EmitSections()
4087 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen); in EmitSections()
4088 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in EmitSections()
4090 emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) { in EmitSections() argument
4091 return CGF.Builder.CreateIsNotNull( in EmitSections()
4092 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitSections()
4097 CGF.EmitOMPLastprivateClauseFinal( in EmitSections()
4099 CGF.Builder.CreateIsNotNull( in EmitSections()
4100 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()))); in EmitSections()
4236 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPSingleDirective() argument
4237 Action.Enter(CGF); in EmitOMPSingleDirective()
4238 OMPPrivateScope SingleScope(CGF); in EmitOMPSingleDirective()
4239 (void)CGF.EmitOMPFirstprivateClause(S, SingleScope); in EmitOMPSingleDirective()
4240 CGF.EmitOMPPrivateClause(S, SingleScope); in EmitOMPSingleDirective()
4242 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in EmitOMPSingleDirective()
4263 static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S) { in emitMaster() argument
4264 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitMaster() argument
4265 Action.Enter(CGF); in emitMaster()
4266 CGF.EmitStmt(S.getRawStmt()); in emitMaster()
4268 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc()); in emitMaster()
4299 static void emitMasked(CodeGenFunction &CGF, const OMPExecutableDirective &S) { in emitMasked() argument
4300 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitMasked() argument
4301 Action.Enter(CGF); in emitMasked()
4302 CGF.EmitStmt(S.getRawStmt()); in emitMasked()
4307 CGF.CGM.getOpenMPRuntime().emitMaskedRegion(CGF, CodeGen, S.getBeginLoc(), in emitMasked()
4383 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPCriticalDirective() argument
4384 Action.Enter(CGF); in EmitOMPCriticalDirective()
4385 CGF.EmitStmt(S.getAssociatedStmt()); in EmitOMPCriticalDirective()
4401 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelForDirective() argument
4402 Action.Enter(CGF); in EmitOMPParallelForDirective()
4403 emitOMPCopyinClause(CGF, S); in EmitOMPParallelForDirective()
4404 (void)emitWorksharingDirective(CGF, S, S.hasCancel()); in EmitOMPParallelForDirective()
4407 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) { in EmitOMPParallelForDirective() argument
4408 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in EmitOMPParallelForDirective()
4410 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI); in EmitOMPParallelForDirective()
4411 OMPLoopScope LoopScope(CGF, S); in EmitOMPParallelForDirective()
4412 return CGF.EmitScalarExpr(S.getNumIterations()); in EmitOMPParallelForDirective()
4435 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelForSimdDirective() argument
4436 Action.Enter(CGF); in EmitOMPParallelForSimdDirective()
4437 emitOMPCopyinClause(CGF, S); in EmitOMPParallelForSimdDirective()
4438 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false); in EmitOMPParallelForSimdDirective()
4441 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) { in EmitOMPParallelForSimdDirective() argument
4442 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in EmitOMPParallelForSimdDirective()
4444 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI); in EmitOMPParallelForSimdDirective()
4445 OMPLoopScope LoopScope(CGF, S); in EmitOMPParallelForSimdDirective()
4446 return CGF.EmitScalarExpr(S.getNumIterations()); in EmitOMPParallelForSimdDirective()
4469 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelMasterDirective() argument
4470 Action.Enter(CGF); in EmitOMPParallelMasterDirective()
4471 OMPPrivateScope PrivateScope(CGF); in EmitOMPParallelMasterDirective()
4472 emitOMPCopyinClause(CGF, S); in EmitOMPParallelMasterDirective()
4473 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in EmitOMPParallelMasterDirective()
4474 CGF.EmitOMPPrivateClause(S, PrivateScope); in EmitOMPParallelMasterDirective()
4475 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPParallelMasterDirective()
4477 emitMaster(CGF, S); in EmitOMPParallelMasterDirective()
4478 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in EmitOMPParallelMasterDirective()
4496 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelMaskedDirective() argument
4497 Action.Enter(CGF); in EmitOMPParallelMaskedDirective()
4498 OMPPrivateScope PrivateScope(CGF); in EmitOMPParallelMaskedDirective()
4499 emitOMPCopyinClause(CGF, S); in EmitOMPParallelMaskedDirective()
4500 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in EmitOMPParallelMaskedDirective()
4501 CGF.EmitOMPPrivateClause(S, PrivateScope); in EmitOMPParallelMaskedDirective()
4502 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPParallelMaskedDirective()
4504 emitMasked(CGF, S); in EmitOMPParallelMaskedDirective()
4505 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in EmitOMPParallelMaskedDirective()
4523 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelSectionsDirective() argument
4524 Action.Enter(CGF); in EmitOMPParallelSectionsDirective()
4525 emitOMPCopyinClause(CGF, S); in EmitOMPParallelSectionsDirective()
4526 CGF.EmitSections(S); in EmitOMPParallelSectionsDirective()
4712 CapturedRegion](CodeGenFunction &CGF, in EmitOMPTaskBasedDirective()
4718 OMPPrivateScope Scope(CGF); in EmitOMPTaskBasedDirective()
4720 if (auto *DI = CGF.getDebugInfo()) { in EmitOMPTaskBasedDirective()
4722 CGF.CapturedStmtInfo->getCaptureFields(); in EmitOMPTaskBasedDirective()
4723 llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue(); in EmitOMPTaskBasedDirective()
4725 unsigned CharWidth = CGF.getContext().getCharWidth(); in EmitOMPTaskBasedDirective()
4743 CGF.getContext().getASTRecordLayout(CaptureRecord); in EmitOMPTaskBasedDirective()
4746 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo()) in EmitOMPTaskBasedDirective()
4748 CGF.Builder, false); in EmitOMPTaskBasedDirective()
4749 llvm::Instruction &Last = CGF.Builder.GetInsertBlock()->back(); in EmitOMPTaskBasedDirective()
4771 llvm::Value *CopyFn = CGF.Builder.CreateLoad( in EmitOMPTaskBasedDirective()
4772 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam))); in EmitOMPTaskBasedDirective()
4773 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar( in EmitOMPTaskBasedDirective()
4783 Address PrivatePtr = CGF.CreateMemTemp( in EmitOMPTaskBasedDirective()
4784 CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr"); in EmitOMPTaskBasedDirective()
4792 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), in EmitOMPTaskBasedDirective()
4802 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), in EmitOMPTaskBasedDirective()
4811 Ty = CGF.getContext().getPointerType(Ty); in EmitOMPTaskBasedDirective()
4813 Ty = CGF.getContext().getPointerType(Ty); in EmitOMPTaskBasedDirective()
4814 Address PrivatePtr = CGF.CreateMemTemp( in EmitOMPTaskBasedDirective()
4815 CGF.getContext().getPointerType(Ty), ".local.ptr.addr"); in EmitOMPTaskBasedDirective()
4825 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(), in EmitOMPTaskBasedDirective()
4827 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall( in EmitOMPTaskBasedDirective()
4828 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs); in EmitOMPTaskBasedDirective()
4831 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD), in EmitOMPTaskBasedDirective()
4833 CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr, in EmitOMPTaskBasedDirective()
4836 Scope.addPrivate(Pair.first, CGF.EmitLValue(&DRE).getAddress(CGF)); in EmitOMPTaskBasedDirective()
4840 CGF.Builder.CreateLoad(Pair.second), in EmitOMPTaskBasedDirective()
4841 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()), in EmitOMPTaskBasedDirective()
4842 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTaskBasedDirective()
4844 if (auto *DI = CGF.getDebugInfo()) in EmitOMPTaskBasedDirective()
4845 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo()) in EmitOMPTaskBasedDirective()
4847 Pair.first, Pair.second.getPointer(), CGF.Builder, in EmitOMPTaskBasedDirective()
4855 VDType = CGF.getContext().getPointerType(VDType); in EmitOMPTaskBasedDirective()
4857 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first); in EmitOMPTaskBasedDirective()
4860 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(VDType)), in EmitOMPTaskBasedDirective()
4861 CGF.getPointerAlign()); in EmitOMPTaskBasedDirective()
4863 Ptr = CGF.Builder.CreateLoad(Replacement); in EmitOMPTaskBasedDirective()
4864 Replacement = Address(Ptr, CGF.ConvertTypeForMem(VDType), in EmitOMPTaskBasedDirective()
4865 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTaskBasedDirective()
4868 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first); in EmitOMPTaskBasedDirective()
4869 Address Replacement(Ptr, CGF.ConvertTypeForMem(VDType), in EmitOMPTaskBasedDirective()
4870 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTaskBasedDirective()
4876 OMPPrivateScope FirstprivateScope(CGF); in EmitOMPTaskBasedDirective()
4879 CGF.Builder.CreateLoad(Pair.second), in EmitOMPTaskBasedDirective()
4880 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()), in EmitOMPTaskBasedDirective()
4881 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTaskBasedDirective()
4885 OMPLexicalScope LexScope(CGF, S, CapturedRegion); in EmitOMPTaskBasedDirective()
4888 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad( in EmitOMPTaskBasedDirective()
4889 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9))); in EmitOMPTaskBasedDirective()
4891 RedCG.emitSharedOrigLValue(CGF, Cnt); in EmitOMPTaskBasedDirective()
4892 RedCG.emitAggregateType(CGF, Cnt); in EmitOMPTaskBasedDirective()
4896 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(), in EmitOMPTaskBasedDirective()
4898 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( in EmitOMPTaskBasedDirective()
4899 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); in EmitOMPTaskBasedDirective()
4901 Address(CGF.EmitScalarConversion( in EmitOMPTaskBasedDirective()
4902 Replacement.getPointer(), CGF.getContext().VoidPtrTy, in EmitOMPTaskBasedDirective()
4903 CGF.getContext().getPointerType( in EmitOMPTaskBasedDirective()
4906 CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()), in EmitOMPTaskBasedDirective()
4908 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); in EmitOMPTaskBasedDirective()
4934 OMPPrivateScope InRedScope(CGF); in EmitOMPTaskBasedDirective()
4938 RedCG.emitSharedOrigLValue(CGF, Cnt); in EmitOMPTaskBasedDirective()
4939 RedCG.emitAggregateType(CGF, Cnt); in EmitOMPTaskBasedDirective()
4945 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(), in EmitOMPTaskBasedDirective()
4949 ReductionsPtr = CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr), in EmitOMPTaskBasedDirective()
4952 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in EmitOMPTaskBasedDirective()
4954 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( in EmitOMPTaskBasedDirective()
4955 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); in EmitOMPTaskBasedDirective()
4957 CGF.EmitScalarConversion( in EmitOMPTaskBasedDirective()
4958 Replacement.getPointer(), CGF.getContext().VoidPtrTy, in EmitOMPTaskBasedDirective()
4959 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()), in EmitOMPTaskBasedDirective()
4961 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()), in EmitOMPTaskBasedDirective()
4963 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); in EmitOMPTaskBasedDirective()
4969 CGOpenMPRuntime::UntiedTaskLocalDeclsRAII LocalVarsScope(CGF, in EmitOMPTaskBasedDirective()
4971 Action.Enter(CGF); in EmitOMPTaskBasedDirective()
4972 BodyGen(CGF); in EmitOMPTaskBasedDirective()
5086 &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTaskBasedDirective() argument
5088 OMPPrivateScope Scope(CGF); in EmitOMPTargetTaskBasedDirective()
5091 llvm::Value *CopyFn = CGF.Builder.CreateLoad( in EmitOMPTargetTaskBasedDirective()
5092 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam))); in EmitOMPTargetTaskBasedDirective()
5093 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar( in EmitOMPTargetTaskBasedDirective()
5104 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), in EmitOMPTargetTaskBasedDirective()
5110 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(), in EmitOMPTargetTaskBasedDirective()
5112 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall( in EmitOMPTargetTaskBasedDirective()
5113 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs); in EmitOMPTargetTaskBasedDirective()
5116 CGF.Builder.CreateLoad(Pair.second), in EmitOMPTargetTaskBasedDirective()
5117 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()), in EmitOMPTargetTaskBasedDirective()
5118 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTargetTaskBasedDirective()
5122 CGF.processInReduction(S, Data, CGF, CS, Scope); in EmitOMPTargetTaskBasedDirective()
5124 InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
5125 CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0); in EmitOMPTargetTaskBasedDirective()
5126 InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
5127 CGF.GetAddrOfLocalVar(PVD), /*Index=*/0); in EmitOMPTargetTaskBasedDirective()
5128 InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
5129 CGF.GetAddrOfLocalVar(SVD), /*Index=*/0); in EmitOMPTargetTaskBasedDirective()
5132 InputInfo.MappersArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
5133 CGF.GetAddrOfLocalVar(MVD), /*Index=*/0); in EmitOMPTargetTaskBasedDirective()
5136 Action.Enter(CGF); in EmitOMPTargetTaskBasedDirective()
5137 OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false); in EmitOMPTargetTaskBasedDirective()
5139 if (CGF.CGM.getLangOpts().OpenMP >= 51 && in EmitOMPTargetTaskBasedDirective()
5144 CGF.CGM.getOpenMPRuntime().emitThreadLimitClause( in EmitOMPTargetTaskBasedDirective()
5145 CGF, TL->getThreadLimit(), S.getBeginLoc()); in EmitOMPTargetTaskBasedDirective()
5147 BodyGen(CGF); in EmitOMPTargetTaskBasedDirective()
5162 CodeGenFunction &CGF, in processInReduction() argument
5167 OMPLexicalScope LexScope(CGF, S, CapturedRegion); in processInReduction()
5170 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad( in processInReduction()
5171 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(4))); in processInReduction()
5173 RedCG.emitSharedOrigLValue(CGF, Cnt); in processInReduction()
5174 RedCG.emitAggregateType(CGF, Cnt); in processInReduction()
5178 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(), in processInReduction()
5180 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( in processInReduction()
5181 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); in processInReduction()
5183 Address(CGF.EmitScalarConversion( in processInReduction()
5184 Replacement.getPointer(), CGF.getContext().VoidPtrTy, in processInReduction()
5185 CGF.getContext().getPointerType( in processInReduction()
5188 CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()), in processInReduction()
5190 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); in processInReduction()
5213 OMPPrivateScope InRedScope(CGF); in processInReduction()
5217 RedCG.emitSharedOrigLValue(CGF, Cnt); in processInReduction()
5218 RedCG.emitAggregateType(CGF, Cnt); in processInReduction()
5222 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(), in processInReduction()
5227 CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr), TRExpr->getExprLoc()); in processInReduction()
5229 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in processInReduction()
5231 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( in processInReduction()
5232 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); in processInReduction()
5234 CGF.EmitScalarConversion( in processInReduction()
5235 Replacement.getPointer(), CGF.getContext().VoidPtrTy, in processInReduction()
5236 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()), in processInReduction()
5238 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()), in processInReduction()
5240 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); in processInReduction()
5264 auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskDirective() argument
5265 CGF.EmitStmt(CS->getCapturedStmt()); in EmitOMPTaskDirective()
5268 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn, in EmitOMPTaskDirective()
5270 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn, in EmitOMPTaskDirective()
5330 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTaskgroupDirective() argument
5331 Action.Enter(CGF); in EmitOMPTaskgroupDirective()
5346 CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(), in EmitOMPTaskgroupDirective()
5349 CGF.EmitVarDecl(*VD); in EmitOMPTaskgroupDirective()
5350 CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD), in EmitOMPTaskgroupDirective()
5353 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in EmitOMPTaskgroupDirective()
5763 [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeLoop() argument
5765 CGF.EmitOMPSimdInit(S); in EmitOMPDistributeLoop()
5768 StaticChunked](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeLoop() argument
5769 CGF.EmitOMPInnerLoop( in EmitOMPDistributeLoop()
5771 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
5772 CodeGenLoop(CGF, S, LoopExit); in EmitOMPDistributeLoop()
5774 [&S, StaticChunked](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
5776 CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound()); in EmitOMPDistributeLoop()
5777 CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound()); in EmitOMPDistributeLoop()
5778 CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound()); in EmitOMPDistributeLoop()
5779 CGF.EmitIgnoredExpr(S.getCombinedInit()); in EmitOMPDistributeLoop()
5796 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
5797 return CGF.Builder.CreateIsNotNull( in EmitOMPDistributeLoop()
5798 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPDistributeLoop()
5807 *this, S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
5808 return CGF.Builder.CreateIsNotNull( in EmitOMPDistributeLoop()
5809 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPDistributeLoop()
5830 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeDirective() argument
5831 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPDistributeDirective()
5840 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true); in emitOutlinedOrderedFunction() local
5842 CGF.CapturedStmtInfo = &CapStmtInfo; in emitOutlinedOrderedFunction()
5843 llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S, Loc); in emitOutlinedOrderedFunction()
5849 static void emitRestoreIP(CodeGenFunction &CGF, const T *C, in emitRestoreIP() argument
5854 QualType Int64Ty = CGF.CGM.getContext().getIntTypeForBitwidth( in emitRestoreIP()
5860 llvm::Value *StoreValue = CGF.EmitScalarConversion( in emitRestoreIP()
5861 CGF.EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty, in emitRestoreIP()
5867 CGF.Builder.restoreIP( in emitRestoreIP()
5868 OMPBuilder.createOrderedDepend(CGF.Builder, AllocaIP, NumLoops, in emitRestoreIP()
5942 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF, in EmitOMPOrderedDirective()
5947 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); in EmitOMPOrderedDirective()
5950 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(), in EmitOMPOrderedDirective()
5953 Action.Enter(CGF); in EmitOMPOrderedDirective()
5954 CGF.EmitStmt(CS->getCapturedStmt()); in EmitOMPOrderedDirective()
5961 static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val, in convertToScalarValue() argument
5964 assert(CGF.hasScalarEvaluationKind(DestType) && in convertToScalarValue()
5967 return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, in convertToScalarValue()
5969 : CGF.EmitComplexToScalarConversion( in convertToScalarValue()
5974 convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, in convertToComplexValue() argument
5976 assert(CGF.getEvaluationKind(DestType) == TEK_Complex && in convertToComplexValue()
5983 llvm::Value *ScalarVal = CGF.EmitScalarConversion( in convertToComplexValue()
5992 ComplexVal.first = CGF.EmitScalarConversion( in convertToComplexValue()
5994 ComplexVal.second = CGF.EmitScalarConversion( in convertToComplexValue()
6000 static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO, in emitSimpleAtomicStore() argument
6003 CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal); in emitSimpleAtomicStore()
6005 CGF.EmitAtomicStore(RVal, LVal, AO, LVal.isVolatile(), /*isInit=*/false); in emitSimpleAtomicStore()
6008 static RValue emitSimpleAtomicLoad(CodeGenFunction &CGF, in emitSimpleAtomicLoad() argument
6012 return CGF.EmitLoadOfLValue(LVal, Loc); in emitSimpleAtomicLoad()
6013 return CGF.EmitAtomicLoad( in emitSimpleAtomicLoad()
6036 static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, in emitOMPAtomicReadExpr() argument
6042 LValue XLValue = CGF.EmitLValue(X); in emitOMPAtomicReadExpr()
6043 LValue VLValue = CGF.EmitLValue(V); in emitOMPAtomicReadExpr()
6044 RValue Res = emitSimpleAtomicLoad(CGF, AO, XLValue, Loc); in emitOMPAtomicReadExpr()
6053 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, std::nullopt, Loc, in emitOMPAtomicReadExpr()
6063 CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc); in emitOMPAtomicReadExpr()
6064 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, V); in emitOMPAtomicReadExpr()
6067 static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, in emitOMPAtomicWriteExpr() argument
6072 emitSimpleAtomicStore(CGF, AO, CGF.EmitLValue(X), CGF.EmitAnyExpr(E)); in emitOMPAtomicWriteExpr()
6073 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X); in emitOMPAtomicWriteExpr()
6082 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, std::nullopt, Loc, in emitOMPAtomicWriteExpr()
6094 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, in emitOMPAtomicRMW() argument
6099 ASTContext &Context = CGF.getContext(); in emitOMPAtomicRMW()
6106 X.getAddress(CGF).getElementType())) || in emitOMPAtomicRMW()
6111 auto &&CheckAtomicSupport = [&CGF](llvm::Type *T, BinaryOperatorKind BO) { in emitOMPAtomicRMW()
6116 return llvm::isPowerOf2_64(CGF.CGM.getDataLayout().getTypeStoreSize(T)); in emitOMPAtomicRMW()
6122 !CheckAtomicSupport(X.getAddress(CGF).getElementType(), BO)) in emitOMPAtomicRMW()
6125 bool IsInteger = X.getAddress(CGF).getElementType()->isIntegerTy(); in emitOMPAtomicRMW()
6201 UpdateVal = CGF.Builder.CreateIntCast( in emitOMPAtomicRMW()
6202 IC, X.getAddress(CGF).getElementType(), in emitOMPAtomicRMW()
6205 UpdateVal = CGF.Builder.CreateCast(llvm::Instruction::CastOps::UIToFP, IC, in emitOMPAtomicRMW()
6206 X.getAddress(CGF).getElementType()); in emitOMPAtomicRMW()
6209 CGF.Builder.CreateAtomicRMW(RMWOp, X.getAddress(CGF), UpdateVal, AO); in emitOMPAtomicRMW()
6237 static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, in emitOMPAtomicUpdateExpr() argument
6251 LValue XLValue = CGF.EmitLValue(X); in emitOMPAtomicUpdateExpr()
6252 RValue ExprRValue = CGF.EmitAnyExpr(E); in emitOMPAtomicUpdateExpr()
6257 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) { in emitOMPAtomicUpdateExpr()
6258 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); in emitOMPAtomicUpdateExpr()
6259 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); in emitOMPAtomicUpdateExpr()
6260 return CGF.EmitAnyExpr(UE); in emitOMPAtomicUpdateExpr()
6262 (void)CGF.EmitOMPAtomicSimpleUpdateExpr( in emitOMPAtomicUpdateExpr()
6264 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X); in emitOMPAtomicUpdateExpr()
6273 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, std::nullopt, Loc, in emitOMPAtomicUpdateExpr()
6285 static RValue convertToType(CodeGenFunction &CGF, RValue Value, in convertToType() argument
6288 switch (CGF.getEvaluationKind(ResType)) { in convertToType()
6291 convertToScalarValue(CGF, Value, SourceType, ResType, Loc)); in convertToType()
6293 auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc); in convertToType()
6302 static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, in emitOMPAtomicCaptureExpr() argument
6311 LValue VLValue = CGF.EmitLValue(V); in emitOMPAtomicCaptureExpr()
6312 LValue XLValue = CGF.EmitLValue(X); in emitOMPAtomicCaptureExpr()
6313 RValue ExprRValue = CGF.EmitAnyExpr(E); in emitOMPAtomicCaptureExpr()
6331 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr, in emitOMPAtomicCaptureExpr()
6333 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); in emitOMPAtomicCaptureExpr()
6334 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); in emitOMPAtomicCaptureExpr()
6335 RValue Res = CGF.EmitAnyExpr(UE); in emitOMPAtomicCaptureExpr()
6339 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr( in emitOMPAtomicCaptureExpr()
6341 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X); in emitOMPAtomicCaptureExpr()
6350 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); in emitOMPAtomicCaptureExpr()
6351 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second); in emitOMPAtomicCaptureExpr()
6352 NewVVal = CGF.EmitAnyExpr(UE); in emitOMPAtomicCaptureExpr()
6358 ExprRValue = convertToType(CGF, ExprRValue, E->getType(), in emitOMPAtomicCaptureExpr()
6365 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr( in emitOMPAtomicCaptureExpr()
6368 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X); in emitOMPAtomicCaptureExpr()
6375 CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc); in emitOMPAtomicCaptureExpr()
6376 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, V); in emitOMPAtomicCaptureExpr()
6378 if (CGF.CGM.getLangOpts().OpenMP < 51) { in emitOMPAtomicCaptureExpr()
6388 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, std::nullopt, Loc, in emitOMPAtomicCaptureExpr()
6392 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, std::nullopt, Loc, in emitOMPAtomicCaptureExpr()
6397 CGF.CGM.getOpenMPRuntime().emitFlush( in emitOMPAtomicCaptureExpr()
6398 CGF, std::nullopt, Loc, llvm::AtomicOrdering::AcquireRelease); in emitOMPAtomicCaptureExpr()
6410 CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO, in emitOMPAtomicCompareExpr() argument
6415 CGF.CGM.getOpenMPRuntime().getOMPBuilder(); in emitOMPAtomicCompareExpr()
6433 LValue XLVal = CGF.EmitLValue(X); in emitOMPAtomicCompareExpr()
6434 Address XAddr = XLVal.getAddress(CGF); in emitOMPAtomicCompareExpr()
6436 auto EmitRValueWithCastIfNeeded = [&CGF, Loc](const Expr *X, const Expr *E) { in emitOMPAtomicCompareExpr()
6438 return CGF.EmitScalarExpr(E); in emitOMPAtomicCompareExpr()
6440 llvm::Value *V = CGF.EmitScalarExpr(NewE); in emitOMPAtomicCompareExpr()
6443 return CGF.EmitScalarConversion(V, NewE->getType(), X->getType(), Loc); in emitOMPAtomicCompareExpr()
6449 EVal = CGF.Builder.CreateIntCast( in emitOMPAtomicCompareExpr()
6450 CI, XLVal.getAddress(CGF).getElementType(), in emitOMPAtomicCompareExpr()
6454 DVal = CGF.Builder.CreateIntCast( in emitOMPAtomicCompareExpr()
6455 CI, XLVal.getAddress(CGF).getElementType(), in emitOMPAtomicCompareExpr()
6464 LValue LV = CGF.EmitLValue(V); in emitOMPAtomicCompareExpr()
6465 Address Addr = LV.getAddress(CGF); in emitOMPAtomicCompareExpr()
6471 LValue LV = CGF.EmitLValue(R); in emitOMPAtomicCompareExpr()
6472 Address Addr = LV.getAddress(CGF); in emitOMPAtomicCompareExpr()
6481 CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare( in emitOMPAtomicCompareExpr()
6482 CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr, in emitOMPAtomicCompareExpr()
6485 CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare( in emitOMPAtomicCompareExpr()
6486 CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr, in emitOMPAtomicCompareExpr()
6490 static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, in emitOMPAtomicExpr() argument
6499 emitOMPAtomicReadExpr(CGF, AO, X, V, Loc); in emitOMPAtomicExpr()
6502 emitOMPAtomicWriteExpr(CGF, AO, X, E, Loc); in emitOMPAtomicExpr()
6506 emitOMPAtomicUpdateExpr(CGF, AO, X, E, UE, IsXLHSInRHSPart, Loc); in emitOMPAtomicExpr()
6509 emitOMPAtomicCaptureExpr(CGF, AO, IsPostfixUpdate, V, X, E, UE, in emitOMPAtomicExpr()
6513 emitOMPAtomicCompareExpr(CGF, AO, FailAO, X, V, R, E, D, CE, in emitOMPAtomicExpr()
6602 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, in emitCommonOMPTargetDirective() argument
6606 CodeGenModule &CGM = CGF.CGM; in emitCommonOMPTargetDirective()
6610 OMPLexicalScope Scope(CGF, S, OMPD_target); in emitCommonOMPTargetDirective()
6612 CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitCommonOMPTargetDirective() argument
6613 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in emitCommonOMPTargetDirective()
6618 auto LPCRegion = CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S); in emitCommonOMPTargetDirective()
6644 if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val) in emitCommonOMPTargetDirective()
6657 assert(CGF.CurFuncDecl && "No parent declaration for target region!"); in emitCommonOMPTargetDirective()
6661 if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl)) in emitCommonOMPTargetDirective()
6663 else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl)) in emitCommonOMPTargetDirective()
6667 CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl))); in emitCommonOMPTargetDirective()
6672 OMPLexicalScope Scope(CGF, S, OMPD_task); in emitCommonOMPTargetDirective()
6674 [IsOffloadEntry](CodeGenFunction &CGF, in emitCommonOMPTargetDirective()
6677 OMPLoopScope(CGF, D); in emitCommonOMPTargetDirective()
6679 llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations()); in emitCommonOMPTargetDirective()
6680 NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty, in emitCommonOMPTargetDirective()
6686 CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device, in emitCommonOMPTargetDirective()
6690 static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, in emitTargetRegion() argument
6692 Action.Enter(CGF); in emitTargetRegion()
6693 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetRegion()
6694 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in emitTargetRegion()
6695 CGF.EmitOMPPrivateClause(S, PrivateScope); in emitTargetRegion()
6698 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitTargetRegion()
6700 CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt()); in emitTargetRegion()
6701 CGF.EnsureInsertPoint(); in emitTargetRegion()
6707 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetDeviceFunction() argument
6708 emitTargetRegion(CGF, S, Action); in EmitOMPTargetDeviceFunction()
6719 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetDirective() argument
6720 emitTargetRegion(CGF, S, Action); in EmitOMPTargetDirective()
6725 static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, in emitCommonOMPTeamsDirective() argument
6731 CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction( in emitCommonOMPTeamsDirective()
6732 CGF, S, *CS->getCapturedDecl()->param_begin(), InnermostKind, in emitCommonOMPTeamsDirective()
6741 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit, in emitCommonOMPTeamsDirective()
6745 OMPTeamsScope Scope(CGF, S); in emitCommonOMPTeamsDirective()
6747 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); in emitCommonOMPTeamsDirective()
6748 CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn, in emitCommonOMPTeamsDirective()
6754 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTeamsDirective() argument
6755 Action.Enter(CGF); in EmitOMPTeamsDirective()
6756 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDirective()
6757 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in EmitOMPTeamsDirective()
6758 CGF.EmitOMPPrivateClause(S, PrivateScope); in EmitOMPTeamsDirective()
6759 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDirective()
6761 CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt()); in EmitOMPTeamsDirective()
6762 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDirective()
6769 static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, in emitTargetTeamsRegion() argument
6772 Action.Enter(CGF); in emitTargetTeamsRegion()
6774 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetTeamsRegion() argument
6775 Action.Enter(CGF); in emitTargetTeamsRegion()
6776 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsRegion()
6777 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in emitTargetTeamsRegion()
6778 CGF.EmitOMPPrivateClause(S, PrivateScope); in emitTargetTeamsRegion()
6779 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsRegion()
6782 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitTargetTeamsRegion()
6783 CGF.EmitStmt(CS->getCapturedStmt()); in emitTargetTeamsRegion()
6784 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsRegion()
6786 emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen); in emitTargetTeamsRegion()
6787 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsRegion()
6794 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDeviceFunction() argument
6795 emitTargetTeamsRegion(CGF, Action, S); in EmitOMPTargetTeamsDeviceFunction()
6807 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDirective() argument
6808 emitTargetTeamsRegion(CGF, Action, S); in EmitOMPTargetTeamsDirective()
6814 emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, in emitTargetTeamsDistributeRegion() argument
6816 Action.Enter(CGF); in emitTargetTeamsDistributeRegion()
6817 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeRegion() argument
6818 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in emitTargetTeamsDistributeRegion()
6822 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeRegion()
6824 Action.Enter(CGF); in emitTargetTeamsDistributeRegion()
6825 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeRegion()
6826 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeRegion()
6828 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in emitTargetTeamsDistributeRegion()
6830 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeRegion()
6832 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen); in emitTargetTeamsDistributeRegion()
6833 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeRegion()
6840 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeDeviceFunction() argument
6841 emitTargetTeamsDistributeRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeDeviceFunction()
6853 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeDirective() argument
6854 emitTargetTeamsDistributeRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeDirective()
6860 CodeGenFunction &CGF, PrePostActionTy &Action, in emitTargetTeamsDistributeSimdRegion() argument
6862 Action.Enter(CGF); in emitTargetTeamsDistributeSimdRegion()
6863 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeSimdRegion() argument
6864 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in emitTargetTeamsDistributeSimdRegion()
6868 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeSimdRegion()
6870 Action.Enter(CGF); in emitTargetTeamsDistributeSimdRegion()
6871 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeSimdRegion()
6872 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeSimdRegion()
6874 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in emitTargetTeamsDistributeSimdRegion()
6876 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeSimdRegion()
6878 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen); in emitTargetTeamsDistributeSimdRegion()
6879 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeSimdRegion()
6886 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeSimdDeviceFunction() argument
6887 emitTargetTeamsDistributeSimdRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeSimdDeviceFunction()
6899 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeSimdDirective() argument
6900 emitTargetTeamsDistributeSimdRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeSimdDirective()
6908 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeDirective() argument
6909 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPTeamsDistributeDirective()
6913 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeDirective()
6915 Action.Enter(CGF); in EmitOMPTeamsDistributeDirective()
6916 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeDirective()
6917 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeDirective()
6919 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in EmitOMPTeamsDistributeDirective()
6921 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeDirective()
6930 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeSimdDirective() argument
6931 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPTeamsDistributeSimdDirective()
6935 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeSimdDirective()
6937 Action.Enter(CGF); in EmitOMPTeamsDistributeSimdDirective()
6938 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeSimdDirective()
6939 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeSimdDirective()
6941 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd, in EmitOMPTeamsDistributeSimdDirective()
6943 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeSimdDirective()
6952 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeParallelForDirective() argument
6953 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPTeamsDistributeParallelForDirective()
6958 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeParallelForDirective()
6960 Action.Enter(CGF); in EmitOMPTeamsDistributeParallelForDirective()
6961 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeParallelForDirective()
6962 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeParallelForDirective()
6964 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in EmitOMPTeamsDistributeParallelForDirective()
6966 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeParallelForDirective()
6975 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeParallelForSimdDirective() argument
6976 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPTeamsDistributeParallelForSimdDirective()
6981 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeParallelForSimdDirective()
6983 Action.Enter(CGF); in EmitOMPTeamsDistributeParallelForSimdDirective()
6984 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeParallelForSimdDirective()
6985 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeParallelForSimdDirective()
6987 CGF.CGM.getOpenMPRuntime().emitInlinedDirective( in EmitOMPTeamsDistributeParallelForSimdDirective()
6988 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); in EmitOMPTeamsDistributeParallelForSimdDirective()
6989 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeParallelForSimdDirective()
7052 CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, in emitTargetTeamsDistributeParallelForRegion() argument
7054 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForRegion()
7055 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeParallelForRegion() argument
7056 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in emitTargetTeamsDistributeParallelForRegion()
7061 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeParallelForRegion()
7063 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForRegion()
7064 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeParallelForRegion()
7065 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeParallelForRegion()
7067 CGF.CGM.getOpenMPRuntime().emitInlinedDirective( in emitTargetTeamsDistributeParallelForRegion()
7068 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); in emitTargetTeamsDistributeParallelForRegion()
7069 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeParallelForRegion()
7072 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for, in emitTargetTeamsDistributeParallelForRegion()
7074 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeParallelForRegion()
7083 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForDeviceFunction() argument
7084 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForDeviceFunction()
7096 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForDirective() argument
7097 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForDirective()
7103 CodeGenFunction &CGF, in emitTargetTeamsDistributeParallelForSimdRegion() argument
7106 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForSimdRegion()
7107 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeParallelForSimdRegion() argument
7108 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in emitTargetTeamsDistributeParallelForSimdRegion()
7113 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeParallelForSimdRegion()
7115 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForSimdRegion()
7116 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeParallelForSimdRegion()
7117 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeParallelForSimdRegion()
7119 CGF.CGM.getOpenMPRuntime().emitInlinedDirective( in emitTargetTeamsDistributeParallelForSimdRegion()
7120 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); in emitTargetTeamsDistributeParallelForSimdRegion()
7121 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeParallelForSimdRegion()
7124 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd, in emitTargetTeamsDistributeParallelForSimdRegion()
7126 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeParallelForSimdRegion()
7135 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction() argument
7136 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction()
7148 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForSimdDirective() argument
7149 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForSimdDirective()
7319 void Enter(CodeGenFunction &CGF) override { in EmitOMPTargetDataDirective() argument
7325 auto &&CodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetDataDirective() argument
7326 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTargetDataDirective() argument
7327 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in EmitOMPTargetDataDirective()
7331 auto &&PrivCodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetDataDirective() argument
7337 Action.Enter(CGF); in EmitOMPTargetDataDirective()
7340 OMPPrivateScope PrivateScope(CGF); in EmitOMPTargetDataDirective()
7343 CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope, in EmitOMPTargetDataDirective()
7346 CGF.EmitOMPUseDeviceAddrClause(*C, PrivateScope, in EmitOMPTargetDataDirective()
7349 RCG(CGF); in EmitOMPTargetDataDirective()
7360 CGF.EmitVarDecl(*OED); in EmitOMPTargetDataDirective()
7366 CGF.EmitVarDecl(*OED); in EmitOMPTargetDataDirective()
7372 OMPLexicalScope Scope(CGF, S, CaptureRegion); in EmitOMPTargetDataDirective()
7373 RCG(CGF); in EmitOMPTargetDataDirective()
7384 OMPLexicalScope Scope(CGF, S); in EmitOMPTargetDataDirective()
7385 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data, in EmitOMPTargetDataDirective()
7458 static void emitTargetParallelRegion(CodeGenFunction &CGF, in emitTargetParallelRegion() argument
7463 Action.Enter(CGF); in emitTargetParallelRegion()
7464 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetParallelRegion() argument
7465 Action.Enter(CGF); in emitTargetParallelRegion()
7466 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetParallelRegion()
7467 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in emitTargetParallelRegion()
7468 CGF.EmitOMPPrivateClause(S, PrivateScope); in emitTargetParallelRegion()
7469 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetParallelRegion()
7472 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitTargetParallelRegion()
7474 CGF.EmitStmt(CS->getCapturedStmt()); in emitTargetParallelRegion()
7475 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in emitTargetParallelRegion()
7477 emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen, in emitTargetParallelRegion()
7479 emitPostUpdateForReductionClause(CGF, S, in emitTargetParallelRegion()
7486 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelDeviceFunction() argument
7487 emitTargetParallelRegion(CGF, S, Action); in EmitOMPTargetParallelDeviceFunction()
7499 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelDirective() argument
7500 emitTargetParallelRegion(CGF, S, Action); in EmitOMPTargetParallelDirective()
7505 static void emitTargetParallelForRegion(CodeGenFunction &CGF, in emitTargetParallelForRegion() argument
7508 Action.Enter(CGF); in emitTargetParallelForRegion()
7511 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetParallelForRegion() argument
7512 Action.Enter(CGF); in emitTargetParallelForRegion()
7514 CGF, OMPD_target_parallel_for, S.hasCancel()); in emitTargetParallelForRegion()
7515 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, in emitTargetParallelForRegion()
7518 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen, in emitTargetParallelForRegion()
7526 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForDeviceFunction() argument
7527 emitTargetParallelForRegion(CGF, S, Action); in EmitOMPTargetParallelForDeviceFunction()
7539 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForDirective() argument
7540 emitTargetParallelForRegion(CGF, S, Action); in EmitOMPTargetParallelForDirective()
7546 emitTargetParallelForSimdRegion(CodeGenFunction &CGF, in emitTargetParallelForSimdRegion() argument
7549 Action.Enter(CGF); in emitTargetParallelForSimdRegion()
7552 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetParallelForSimdRegion() argument
7553 Action.Enter(CGF); in emitTargetParallelForSimdRegion()
7554 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, in emitTargetParallelForSimdRegion()
7557 emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen, in emitTargetParallelForSimdRegion()
7565 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForSimdDeviceFunction() argument
7566 emitTargetParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetParallelForSimdDeviceFunction()
7578 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForSimdDirective() argument
7579 emitTargetParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetParallelForSimdDirective()
7585 static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, in mapParam() argument
7589 Privates.addPrivate(VDecl, CGF.GetAddrOfLocalVar(PVD)); in mapParam()
7627 auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
7639 OMPLoopScope PreInitScope(CGF, S); in EmitOMPTaskLoopBasedDirective()
7640 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { in EmitOMPTaskLoopBasedDirective()
7644 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then"); in EmitOMPTaskLoopBasedDirective()
7645 ContBlock = CGF.createBasicBlock("taskloop.if.end"); in EmitOMPTaskLoopBasedDirective()
7646 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, in EmitOMPTaskLoopBasedDirective()
7647 CGF.getProfileCount(&S)); in EmitOMPTaskLoopBasedDirective()
7648 CGF.EmitBlock(ThenBlock); in EmitOMPTaskLoopBasedDirective()
7649 CGF.incrementProfileCounter(&S); in EmitOMPTaskLoopBasedDirective()
7652 (void)CGF.EmitOMPLinearClauseInit(S); in EmitOMPTaskLoopBasedDirective()
7654 OMPPrivateScope LoopScope(CGF); in EmitOMPTaskLoopBasedDirective()
7662 mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP, in EmitOMPTaskLoopBasedDirective()
7664 mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP, in EmitOMPTaskLoopBasedDirective()
7666 mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope); in EmitOMPTaskLoopBasedDirective()
7667 mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP, in EmitOMPTaskLoopBasedDirective()
7669 CGF.EmitOMPPrivateLoopCounters(S, LoopScope); in EmitOMPTaskLoopBasedDirective()
7670 CGF.EmitOMPLinearClause(S, LoopScope); in EmitOMPTaskLoopBasedDirective()
7671 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); in EmitOMPTaskLoopBasedDirective()
7676 CGF.EmitVarDecl(*IVDecl); in EmitOMPTaskLoopBasedDirective()
7677 CGF.EmitIgnoredExpr(S.getInit()); in EmitOMPTaskLoopBasedDirective()
7683 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); in EmitOMPTaskLoopBasedDirective()
7685 CGF.EmitIgnoredExpr(S.getCalcLastIteration()); in EmitOMPTaskLoopBasedDirective()
7689 OMPLexicalScope Scope(CGF, S, OMPD_taskloop, /*EmitPreInitStmt=*/false); in EmitOMPTaskLoopBasedDirective()
7691 CGF, S, in EmitOMPTaskLoopBasedDirective()
7692 [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
7694 CGF.EmitOMPSimdInit(S); in EmitOMPTaskLoopBasedDirective()
7696 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
7697 CGF.EmitOMPInnerLoop( in EmitOMPTaskLoopBasedDirective()
7699 [&S](CodeGenFunction &CGF) { in EmitOMPTaskLoopBasedDirective() argument
7700 emitOMPLoopBodyWithStopPoint(CGF, S, in EmitOMPTaskLoopBasedDirective()
7708 CGF.EmitBranch(ContBlock); in EmitOMPTaskLoopBasedDirective()
7709 CGF.EmitBlock(ContBlock, true); in EmitOMPTaskLoopBasedDirective()
7713 CGF.EmitOMPLastprivateClauseFinal( in EmitOMPTaskLoopBasedDirective()
7715 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar( in EmitOMPTaskLoopBasedDirective()
7716 CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false, in EmitOMPTaskLoopBasedDirective()
7720 CGF.EmitOMPLinearClauseFinal(S, [LIP, &S](CodeGenFunction &CGF) { in EmitOMPTaskLoopBasedDirective() argument
7721 return CGF.Builder.CreateIsNotNull( in EmitOMPTaskLoopBasedDirective()
7722 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false, in EmitOMPTaskLoopBasedDirective()
7727 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn, in EmitOMPTaskLoopBasedDirective()
7730 &Data](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
7731 OMPLoopScope PreInitScope(CGF, S); in EmitOMPTaskLoopBasedDirective()
7732 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S, in EmitOMPTaskLoopBasedDirective()
7736 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop, in EmitOMPTaskLoopBasedDirective()
7744 [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF, in EmitOMPTaskLoopBasedDirective()
7746 Action.Enter(CGF); in EmitOMPTaskLoopBasedDirective()
7747 CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, in EmitOMPTaskLoopBasedDirective()
7770 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPMasterTaskLoopDirective() argument
7771 Action.Enter(CGF); in EmitOMPMasterTaskLoopDirective()
7782 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPMasterTaskLoopSimdDirective() argument
7783 Action.Enter(CGF); in EmitOMPMasterTaskLoopSimdDirective()
7794 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelMasterTaskLoopDirective() argument
7795 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF, in EmitOMPParallelMasterTaskLoopDirective()
7797 Action.Enter(CGF); in EmitOMPParallelMasterTaskLoopDirective()
7798 CGF.EmitOMPTaskLoopBasedDirective(S); in EmitOMPParallelMasterTaskLoopDirective()
7800 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false); in EmitOMPParallelMasterTaskLoopDirective()
7801 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen, in EmitOMPParallelMasterTaskLoopDirective()
7812 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelMasterTaskLoopSimdDirective() argument
7813 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF, in EmitOMPParallelMasterTaskLoopSimdDirective()
7815 Action.Enter(CGF); in EmitOMPParallelMasterTaskLoopSimdDirective()
7816 CGF.EmitOMPTaskLoopBasedDirective(S); in EmitOMPParallelMasterTaskLoopSimdDirective()
7818 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false); in EmitOMPParallelMasterTaskLoopSimdDirective()
7819 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen, in EmitOMPParallelMasterTaskLoopSimdDirective()
7853 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPGenericLoopDirective() argument
7859 OMPPrivateScope LoopScope(CGF); in EmitOMPGenericLoopDirective()
7860 CGF.EmitOMPPrivateLoopCounters(S, LoopScope); in EmitOMPGenericLoopDirective()
7862 CGF.EmitStmt(CS); in EmitOMPGenericLoopDirective()
7865 CGF.EmitStmt(CS); in EmitOMPGenericLoopDirective()
7876 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelGenericLoopDirective() argument
7877 Action.Enter(CGF); in EmitOMPParallelGenericLoopDirective()
7878 emitOMPCopyinClause(CGF, S); in EmitOMPParallelGenericLoopDirective()
7879 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false); in EmitOMPParallelGenericLoopDirective()
7896 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsGenericLoopDirective() argument
7897 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPTeamsGenericLoopDirective()
7902 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsGenericLoopDirective()
7904 Action.Enter(CGF); in EmitOMPTeamsGenericLoopDirective()
7905 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsGenericLoopDirective()
7906 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsGenericLoopDirective()
7908 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in EmitOMPTeamsGenericLoopDirective()
7910 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsGenericLoopDirective()
7918 emitTargetTeamsGenericLoopRegion(CodeGenFunction &CGF, in emitTargetTeamsGenericLoopRegion() argument
7921 Action.Enter(CGF); in emitTargetTeamsGenericLoopRegion()
7924 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsGenericLoopRegion() argument
7925 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in emitTargetTeamsGenericLoopRegion()
7930 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsGenericLoopRegion()
7932 Action.Enter(CGF); in emitTargetTeamsGenericLoopRegion()
7933 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsGenericLoopRegion()
7934 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsGenericLoopRegion()
7936 CGF.CGM.getOpenMPRuntime().emitInlinedDirective( in emitTargetTeamsGenericLoopRegion()
7937 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); in emitTargetTeamsGenericLoopRegion()
7938 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsGenericLoopRegion()
7941 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for, in emitTargetTeamsGenericLoopRegion()
7943 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsGenericLoopRegion()
7951 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsGenericLoopDirective() argument
7952 emitTargetTeamsGenericLoopRegion(CGF, S, Action); in EmitOMPTargetTeamsGenericLoopDirective()
7961 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsGenericLoopDeviceFunction() argument
7962 emitTargetTeamsGenericLoopRegion(CGF, S, Action); in EmitOMPTargetTeamsGenericLoopDeviceFunction()
7974 CodeGenFunction &CGF, const OMPTargetParallelGenericLoopDirective &S, in emitTargetParallelGenericLoopRegion() argument
7976 Action.Enter(CGF); in emitTargetParallelGenericLoopRegion()
7978 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetParallelGenericLoopRegion() argument
7979 Action.Enter(CGF); in emitTargetParallelGenericLoopRegion()
7981 CGF, OMPD_target_parallel_loop, /*hasCancel=*/false); in emitTargetParallelGenericLoopRegion()
7982 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, in emitTargetParallelGenericLoopRegion()
7985 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen, in emitTargetParallelGenericLoopRegion()
7993 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelGenericLoopDeviceFunction() argument
7994 emitTargetParallelGenericLoopRegion(CGF, S, Action); in EmitOMPTargetParallelGenericLoopDeviceFunction()
8008 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelGenericLoopDirective() argument
8009 emitTargetParallelGenericLoopRegion(CGF, S, Action); in EmitOMPTargetParallelGenericLoopDirective()
8022 auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitSimpleOMPExecutableDirective() argument
8023 OMPPrivateScope GlobalsScope(CGF); in EmitSimpleOMPExecutableDirective()
8034 if (!CGF.LocalDeclMap.count(VD)) { in EmitSimpleOMPExecutableDirective()
8035 LValue GlobLVal = CGF.EmitLValue(Ref); in EmitSimpleOMPExecutableDirective()
8036 GlobalsScope.addPrivate(VD, GlobLVal.getAddress(CGF)); in EmitSimpleOMPExecutableDirective()
8043 ParentLoopDirectiveForScanRegion ScanRegion(CGF, D); in EmitSimpleOMPExecutableDirective()
8044 emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action); in EmitSimpleOMPExecutableDirective()
8049 if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) { in EmitSimpleOMPExecutableDirective()
8050 LValue GlobLVal = CGF.EmitLValue(E); in EmitSimpleOMPExecutableDirective()
8051 GlobalsScope.addPrivate(VD, GlobLVal.getAddress(CGF)); in EmitSimpleOMPExecutableDirective()
8055 if (!CGF.LocalDeclMap.count(VD)) in EmitSimpleOMPExecutableDirective()
8056 CGF.EmitVarDecl(*VD); in EmitSimpleOMPExecutableDirective()
8068 if (!CGF.LocalDeclMap.count(VD)) in EmitSimpleOMPExecutableDirective()
8069 CGF.EmitVarDecl(*VD); in EmitSimpleOMPExecutableDirective()
8075 CGF.EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt()); in EmitSimpleOMPExecutableDirective()