Lines Matching refs:CGF
122 void Enter(CodeGenFunction &CGF) override { in Enter() argument
123 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs); in Enter()
125 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes); in Enter()
126 auto *ThenBlock = CGF.createBasicBlock("omp_if.then"); in Enter()
127 ContBlock = CGF.createBasicBlock("omp_if.end"); in Enter()
129 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock); in Enter()
130 CGF.EmitBlock(ThenBlock); in Enter()
133 void Done(CodeGenFunction &CGF) { in Done() argument
135 CGF.EmitBranch(ContBlock); in Done()
136 CGF.EmitBlock(ContBlock, true); in Done()
138 void Exit(CodeGenFunction &CGF) override { in Exit() argument
139 CGF.EmitRuntimeCall(ExitCallee, ExitArgs); in Exit()
300 CodeGenFunction &CGF; member in __anona929754a0111::CheckVarsEscapingDeclContext
316 if (auto *CSI = CGF.CapturedStmtInfo) { in markAsEscaped()
342 if ((!CGF.CapturedStmtInfo || in markAsEscaped()
343 (IsForCombinedParallelRegion && CGF.CapturedStmtInfo)) && in markAsEscaped()
421 CGF.getContext(), EscapedDeclsForParallel, EscapedDeclsForTeams, in buildRecordForGlobalizedVars()
426 CheckVarsEscapingDeclContext(CodeGenFunction &CGF, in CheckVarsEscapingDeclContext() argument
428 : CGF(CGF), EscapedDecls(TeamsReductions.begin(), TeamsReductions.end()) { in CheckVarsEscapingDeclContext()
607 static llvm::Value *getNVPTXWarpSize(CodeGenFunction &CGF) { in getNVPTXWarpSize() argument
608 return CGF.EmitRuntimeCall( in getNVPTXWarpSize()
610 &CGF.CGM.getModule(), llvm::Intrinsic::nvvm_read_ptx_sreg_warpsize), in getNVPTXWarpSize()
615 static llvm::Value *getNVPTXThreadID(CodeGenFunction &CGF) { in getNVPTXThreadID() argument
616 return CGF.EmitRuntimeCall( in getNVPTXThreadID()
618 &CGF.CGM.getModule(), llvm::Intrinsic::nvvm_read_ptx_sreg_tid_x), in getNVPTXThreadID()
625 static llvm::Value *getNVPTXWarpID(CodeGenFunction &CGF) { in getNVPTXWarpID() argument
626 CGBuilderTy &Bld = CGF.Builder; in getNVPTXWarpID()
627 return Bld.CreateAShr(getNVPTXThreadID(CGF), LaneIDBits, "nvptx_warp_id"); in getNVPTXWarpID()
633 static llvm::Value *getNVPTXLaneID(CodeGenFunction &CGF) { in getNVPTXLaneID() argument
634 CGBuilderTy &Bld = CGF.Builder; in getNVPTXLaneID()
635 return Bld.CreateAnd(getNVPTXThreadID(CGF), Bld.getInt32(LaneIDMask), in getNVPTXLaneID()
640 static llvm::Value *getNVPTXNumThreads(CodeGenFunction &CGF) { in getNVPTXNumThreads() argument
641 return CGF.EmitRuntimeCall( in getNVPTXNumThreads()
643 &CGF.CGM.getModule(), llvm::Intrinsic::nvvm_read_ptx_sreg_ntid_x), in getNVPTXNumThreads()
652 static llvm::Value *getThreadLimit(CodeGenFunction &CGF, in getThreadLimit() argument
654 CGBuilderTy &Bld = CGF.Builder; in getThreadLimit()
656 ? getNVPTXNumThreads(CGF) in getThreadLimit()
657 : Bld.CreateNUWSub(getNVPTXNumThreads(CGF), getNVPTXWarpSize(CGF), in getThreadLimit()
668 static llvm::Value *getMasterThreadID(CodeGenFunction &CGF) { in getMasterThreadID() argument
669 CGBuilderTy &Bld = CGF.Builder; in getMasterThreadID()
670 llvm::Value *NumThreads = getNVPTXNumThreads(CGF); in getMasterThreadID()
673 llvm::Value *Mask = Bld.CreateNUWSub(getNVPTXWarpSize(CGF), Bld.getInt32(1)); in getMasterThreadID()
1184 void Enter(CodeGenFunction &CGF) override { in emitNonSPMDKernel() argument
1186 static_cast<CGOpenMPRuntimeNVPTX &>(CGF.CGM.getOpenMPRuntime()); in emitNonSPMDKernel()
1187 RT.emitNonSPMDEntryHeader(CGF, EST, WST); in emitNonSPMDKernel()
1189 RT.setLocThreadIdInsertPt(CGF, /*AtCurrentPoint=*/true); in emitNonSPMDKernel()
1191 void Exit(CodeGenFunction &CGF) override { in emitNonSPMDKernel() argument
1193 static_cast<CGOpenMPRuntimeNVPTX &>(CGF.CGM.getOpenMPRuntime()); in emitNonSPMDKernel()
1194 RT.clearLocThreadIdInsertPt(CGF); in emitNonSPMDKernel()
1195 RT.emitNonSPMDEntryFooter(CGF, EST); in emitNonSPMDKernel()
1224 void CGOpenMPRuntimeNVPTX::emitNonSPMDEntryHeader(CodeGenFunction &CGF, in emitNonSPMDEntryHeader() argument
1227 CGBuilderTy &Bld = CGF.Builder; in emitNonSPMDEntryHeader()
1229 llvm::BasicBlock *WorkerBB = CGF.createBasicBlock(".worker"); in emitNonSPMDEntryHeader()
1230 llvm::BasicBlock *MasterCheckBB = CGF.createBasicBlock(".mastercheck"); in emitNonSPMDEntryHeader()
1231 llvm::BasicBlock *MasterBB = CGF.createBasicBlock(".master"); in emitNonSPMDEntryHeader()
1232 EST.ExitBB = CGF.createBasicBlock(".exit"); in emitNonSPMDEntryHeader()
1235 Bld.CreateICmpULT(getNVPTXThreadID(CGF), getThreadLimit(CGF)); in emitNonSPMDEntryHeader()
1238 CGF.EmitBlock(WorkerBB); in emitNonSPMDEntryHeader()
1239 emitCall(CGF, WST.Loc, WST.WorkerFn); in emitNonSPMDEntryHeader()
1240 CGF.EmitBranch(EST.ExitBB); in emitNonSPMDEntryHeader()
1242 CGF.EmitBlock(MasterCheckBB); in emitNonSPMDEntryHeader()
1244 Bld.CreateICmpEQ(getNVPTXThreadID(CGF), getMasterThreadID(CGF)); in emitNonSPMDEntryHeader()
1247 CGF.EmitBlock(MasterBB); in emitNonSPMDEntryHeader()
1253 llvm::Value *Args[] = {getThreadLimit(CGF), in emitNonSPMDEntryHeader()
1255 CGF.EmitRuntimeCall( in emitNonSPMDEntryHeader()
1259 CGF.EmitRuntimeCall( in emitNonSPMDEntryHeader()
1263 emitGenericVarsProlog(CGF, WST.Loc); in emitNonSPMDEntryHeader()
1266 void CGOpenMPRuntimeNVPTX::emitNonSPMDEntryFooter(CodeGenFunction &CGF, in emitNonSPMDEntryFooter() argument
1269 if (!CGF.HaveInsertPoint()) in emitNonSPMDEntryFooter()
1272 emitGenericVarsEpilog(CGF); in emitNonSPMDEntryFooter()
1275 EST.ExitBB = CGF.createBasicBlock(".exit"); in emitNonSPMDEntryFooter()
1277 llvm::BasicBlock *TerminateBB = CGF.createBasicBlock(".termination.notifier"); in emitNonSPMDEntryFooter()
1278 CGF.EmitBranch(TerminateBB); in emitNonSPMDEntryFooter()
1280 CGF.EmitBlock(TerminateBB); in emitNonSPMDEntryFooter()
1283 llvm::Value *Args[] = {CGF.Builder.getInt16(/*IsOMPRuntimeInitialized=*/1)}; in emitNonSPMDEntryFooter()
1284 CGF.EmitRuntimeCall( in emitNonSPMDEntryFooter()
1287 syncCTAThreads(CGF); in emitNonSPMDEntryFooter()
1289 CGF.EmitBranch(EST.ExitBB); in emitNonSPMDEntryFooter()
1291 CGF.EmitBlock(EST.ExitBB); in emitNonSPMDEntryFooter()
1318 void Enter(CodeGenFunction &CGF) override { in emitSPMDKernel() argument
1319 RT.emitSPMDEntryHeader(CGF, EST, D); in emitSPMDKernel()
1321 RT.setLocThreadIdInsertPt(CGF, /*AtCurrentPoint=*/true); in emitSPMDKernel()
1323 void Exit(CodeGenFunction &CGF) override { in emitSPMDKernel() argument
1324 RT.clearLocThreadIdInsertPt(CGF); in emitSPMDKernel()
1325 RT.emitSPMDEntryFooter(CGF, EST); in emitSPMDKernel()
1347 CodeGenFunction &CGF, EntryFunctionState &EST, in emitSPMDEntryHeader() argument
1349 CGBuilderTy &Bld = CGF.Builder; in emitSPMDEntryHeader()
1352 llvm::BasicBlock *ExecuteBB = CGF.createBasicBlock(".execute"); in emitSPMDEntryHeader()
1353 EST.ExitBB = CGF.createBasicBlock(".exit"); in emitSPMDEntryHeader()
1355 llvm::Value *Args[] = {getThreadLimit(CGF, /*IsInSPMDExecutionMode=*/true), in emitSPMDEntryHeader()
1359 CGF.EmitRuntimeCall( in emitSPMDEntryHeader()
1364 CGF.EmitRuntimeCall(createNVPTXRuntimeFunction( in emitSPMDEntryHeader()
1368 CGF.EmitBranch(ExecuteBB); in emitSPMDEntryHeader()
1370 CGF.EmitBlock(ExecuteBB); in emitSPMDEntryHeader()
1375 void CGOpenMPRuntimeNVPTX::emitSPMDEntryFooter(CodeGenFunction &CGF, in emitSPMDEntryFooter() argument
1378 if (!CGF.HaveInsertPoint()) in emitSPMDEntryFooter()
1382 EST.ExitBB = CGF.createBasicBlock(".exit"); in emitSPMDEntryFooter()
1384 llvm::BasicBlock *OMPDeInitBB = CGF.createBasicBlock(".omp.deinit"); in emitSPMDEntryFooter()
1385 CGF.EmitBranch(OMPDeInitBB); in emitSPMDEntryFooter()
1387 CGF.EmitBlock(OMPDeInitBB); in emitSPMDEntryFooter()
1390 CGF.Builder.getInt16(RequiresFullRuntime ? 1 : 0)}; in emitSPMDEntryFooter()
1391 CGF.EmitRuntimeCall( in emitSPMDEntryFooter()
1394 CGF.EmitBranch(EST.ExitBB); in emitSPMDEntryFooter()
1396 CGF.EmitBlock(EST.ExitBB); in emitSPMDEntryFooter()
1419 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true); in emitWorkerFunction() local
1420 CGF.StartFunction(GlobalDecl(), Ctx.VoidTy, WST.WorkerFn, WST.CGFI, {}, in emitWorkerFunction()
1422 emitWorkerLoop(CGF, WST); in emitWorkerFunction()
1423 CGF.FinishFunction(); in emitWorkerFunction()
1426 void CGOpenMPRuntimeNVPTX::emitWorkerLoop(CodeGenFunction &CGF, in emitWorkerLoop() argument
1437 CGBuilderTy &Bld = CGF.Builder; in emitWorkerLoop()
1439 llvm::BasicBlock *AwaitBB = CGF.createBasicBlock(".await.work"); in emitWorkerLoop()
1440 llvm::BasicBlock *SelectWorkersBB = CGF.createBasicBlock(".select.workers"); in emitWorkerLoop()
1441 llvm::BasicBlock *ExecuteBB = CGF.createBasicBlock(".execute.parallel"); in emitWorkerLoop()
1442 llvm::BasicBlock *TerminateBB = CGF.createBasicBlock(".terminate.parallel"); in emitWorkerLoop()
1443 llvm::BasicBlock *BarrierBB = CGF.createBasicBlock(".barrier.parallel"); in emitWorkerLoop()
1444 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".exit"); in emitWorkerLoop()
1446 CGF.EmitBranch(AwaitBB); in emitWorkerLoop()
1449 CGF.EmitBlock(AwaitBB); in emitWorkerLoop()
1451 syncCTAThreads(CGF); in emitWorkerLoop()
1454 CGF.CreateDefaultAlignTempAlloca(CGF.Int8PtrTy, /*Name=*/"work_fn"); in emitWorkerLoop()
1456 CGF.CreateDefaultAlignTempAlloca(CGF.Int8Ty, /*Name=*/"exec_status"); in emitWorkerLoop()
1457 CGF.InitTempAlloca(ExecStatus, Bld.getInt8(/*C=*/0)); in emitWorkerLoop()
1458 CGF.InitTempAlloca(WorkFn, llvm::Constant::getNullValue(CGF.Int8PtrTy)); in emitWorkerLoop()
1463 llvm::Value *Ret = CGF.EmitRuntimeCall( in emitWorkerLoop()
1465 Bld.CreateStore(Bld.CreateZExt(Ret, CGF.Int8Ty), ExecStatus); in emitWorkerLoop()
1473 CGF.EmitBlock(SelectWorkersBB); in emitWorkerLoop()
1479 CGF.EmitBlock(ExecuteBB); in emitWorkerLoop()
1481 setLocThreadIdInsertPt(CGF, /*AtCurrentPoint=*/true); in emitWorkerLoop()
1491 llvm::BasicBlock *ExecuteFNBB = CGF.createBasicBlock(".execute.fn"); in emitWorkerLoop()
1492 llvm::BasicBlock *CheckNextBB = CGF.createBasicBlock(".check.next"); in emitWorkerLoop()
1496 CGF.EmitBlock(ExecuteFNBB); in emitWorkerLoop()
1502 emitCall(CGF, WST.Loc, W, in emitWorkerLoop()
1503 {Bld.getInt16(/*ParallelLevel=*/0), getThreadID(CGF, WST.Loc)}); in emitWorkerLoop()
1506 CGF.EmitBranch(TerminateBB); in emitWorkerLoop()
1508 CGF.EmitBlock(CheckNextBB); in emitWorkerLoop()
1522 emitCall(CGF, WST.Loc, WorkFnCast, in emitWorkerLoop()
1523 {Bld.getInt16(/*ParallelLevel=*/0), getThreadID(CGF, WST.Loc)}); in emitWorkerLoop()
1525 CGF.EmitBranch(TerminateBB); in emitWorkerLoop()
1528 CGF.EmitBlock(TerminateBB); in emitWorkerLoop()
1529 CGF.EmitRuntimeCall( in emitWorkerLoop()
1532 CGF.EmitBranch(BarrierBB); in emitWorkerLoop()
1535 CGF.EmitBlock(BarrierBB); in emitWorkerLoop()
1537 syncCTAThreads(CGF); in emitWorkerLoop()
1538 CGF.EmitBranch(AwaitBB); in emitWorkerLoop()
1541 CGF.EmitBlock(ExitBB); in emitWorkerLoop()
1543 clearLocThreadIdInsertPt(CGF); in emitWorkerLoop()
1906 void CGOpenMPRuntimeNVPTX::emitProcBindClause(CodeGenFunction &CGF, in emitProcBindClause() argument
1913 CGOpenMPRuntime::emitProcBindClause(CGF, ProcBind, Loc); in emitProcBindClause()
1916 void CGOpenMPRuntimeNVPTX::emitNumThreadsClause(CodeGenFunction &CGF, in emitNumThreadsClause() argument
1923 CGOpenMPRuntime::emitNumThreadsClause(CGF, NumThreads, Loc); in emitNumThreadsClause()
1926 void CGOpenMPRuntimeNVPTX::emitNumTeamsClause(CodeGenFunction &CGF, in emitNumTeamsClause() argument
1942 void Enter(CodeGenFunction &CGF) override { in emitParallelOutlinedFunction() argument
1946 void Exit(CodeGenFunction &CGF) override { in emitParallelOutlinedFunction() argument
2046 void Enter(CodeGenFunction &CGF) override { in emitTeamsOutlinedFunction() argument
2048 static_cast<CGOpenMPRuntimeNVPTX &>(CGF.CGM.getOpenMPRuntime()); in emitTeamsOutlinedFunction()
2050 auto I = Rt.FunctionGlobalizedDecls.try_emplace(CGF.CurFn).first; in emitTeamsOutlinedFunction()
2063 Rt.emitGenericVarsProlog(CGF, Loc); in emitTeamsOutlinedFunction()
2065 void Exit(CodeGenFunction &CGF) override { in emitTeamsOutlinedFunction() argument
2066 static_cast<CGOpenMPRuntimeNVPTX &>(CGF.CGM.getOpenMPRuntime()) in emitTeamsOutlinedFunction()
2067 .emitGenericVarsEpilog(CGF); in emitTeamsOutlinedFunction()
2081 void CGOpenMPRuntimeNVPTX::emitGenericVarsProlog(CodeGenFunction &CGF, in emitGenericVarsProlog() argument
2088 CGBuilderTy &Bld = CGF.Builder; in emitGenericVarsProlog()
2090 const auto I = FunctionGlobalizedDecls.find(CGF.CurFn); in emitGenericVarsProlog()
2108 CGF.ConvertTypeForMem(GlobalRecTy)->getPointerTo(); in emitGenericVarsProlog()
2114 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".exit"); in emitGenericVarsProlog()
2115 llvm::BasicBlock *SPMDBB = CGF.createBasicBlock(".spmd"); in emitGenericVarsProlog()
2116 llvm::BasicBlock *NonSPMDBB = CGF.createBasicBlock(".non-spmd"); in emitGenericVarsProlog()
2118 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitGenericVarsProlog()
2119 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitGenericVarsProlog()
2120 llvm::Value *PL = CGF.EmitRuntimeCall( in emitGenericVarsProlog()
2125 llvm::Value *IsSPMD = Bld.CreateIsNotNull(CGF.EmitNounwindRuntimeCall( in emitGenericVarsProlog()
2129 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitGenericVarsProlog()
2130 CGF.EmitBlock(SPMDBB); in emitGenericVarsProlog()
2133 CGF.EmitBranch(ExitBB); in emitGenericVarsProlog()
2135 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitGenericVarsProlog()
2136 CGF.EmitBlock(NonSPMDBB); in emitGenericVarsProlog()
2158 Size, CGF.Builder.getInt16(/*UseSharedMemory=*/0)}; in emitGenericVarsProlog()
2159 llvm::Value *GlobalRecValue = CGF.EmitRuntimeCall( in emitGenericVarsProlog()
2165 CGF.EmitBlock(ExitBB); in emitGenericVarsProlog()
2201 llvm::Value *IsInSharedMemory = CGF.EmitLoadOfScalar( in emitGenericVarsProlog()
2213 llvm::Value *Ld = CGF.EmitLoadOfScalar( in emitGenericVarsProlog()
2223 CGF.EmitRuntimeCall(createNVPTXRuntimeFunction( in emitGenericVarsProlog()
2232 Address FrameAddr = CGF.EmitLoadOfPointer( in emitGenericVarsProlog()
2243 GlobalRecValue, CGF.ConvertTypeForMem(GlobalRecTy)->getPointerTo()); in emitGenericVarsProlog()
2249 CGF.Builder.getInt16(/*UseSharedMemory=*/0)}; in emitGenericVarsProlog()
2250 llvm::Value *GlobalRecValue = CGF.EmitRuntimeCall( in emitGenericVarsProlog()
2260 CGF.MakeNaturalAlignPointeeAddrLValue(GlobalRecCastAddr, GlobalRecTy); in emitGenericVarsProlog()
2269 CGF.ConvertTypeForMem(SecGlobalRecTy)->getPointerTo(); in emitGenericVarsProlog()
2270 SecBase = CGF.MakeNaturalAlignPointeeAddrLValue( in emitGenericVarsProlog()
2281 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType()); in emitGenericVarsProlog()
2282 ParValue = CGF.EmitLoadOfScalar(ParLVal, Loc); in emitGenericVarsProlog()
2284 LValue VarAddr = CGF.EmitLValueForField(Base, Rec.second.FD); in emitGenericVarsProlog()
2290 llvm::Value *Ptr = CGF.Builder.CreateInBoundsGEP( in emitGenericVarsProlog()
2292 {Bld.getInt32(0), getNVPTXLaneID(CGF)}); in emitGenericVarsProlog()
2295 VarAddr = CGF.MakeAddrLValue( in emitGenericVarsProlog()
2308 LValue SecVarAddr = CGF.EmitLValueForField(SecBase, SecIt->second.FD); in emitGenericVarsProlog()
2316 Address LocalAddr = CGF.CreateMemTemp(VarTy, Rec.second.FD->getName()); in emitGenericVarsProlog()
2324 CGF.EmitStoreOfScalar(ParValue, VarAddr); in emitGenericVarsProlog()
2325 I->getSecond().MappedParams->setVarAddr(CGF, VD, VarAddr.getAddress()); in emitGenericVarsProlog()
2336 CGBuilderTy &Bld = CGF.Builder; in emitGenericVarsProlog()
2337 llvm::Value *Size = CGF.getTypeSize(VD->getType()); in emitGenericVarsProlog()
2340 Size, llvm::ConstantInt::get(CGF.SizeTy, Align.getQuantity() - 1)); in emitGenericVarsProlog()
2342 llvm::ConstantInt::get(CGF.SizeTy, Align.getQuantity()); in emitGenericVarsProlog()
2348 Size, CGF.Builder.getInt16(/*UseSharedMemory=*/0)}; in emitGenericVarsProlog()
2349 llvm::Value *GlobalRecValue = CGF.EmitRuntimeCall( in emitGenericVarsProlog()
2354 GlobalRecValue, CGF.ConvertTypeForMem(VD->getType())->getPointerTo()); in emitGenericVarsProlog()
2355 LValue Base = CGF.MakeAddrLValue(GlobalRecCastAddr, VD->getType(), in emitGenericVarsProlog()
2358 I->getSecond().MappedParams->setVarAddr(CGF, cast<VarDecl>(VD), in emitGenericVarsProlog()
2362 I->getSecond().MappedParams->apply(CGF); in emitGenericVarsProlog()
2365 void CGOpenMPRuntimeNVPTX::emitGenericVarsEpilog(CodeGenFunction &CGF, in emitGenericVarsEpilog() argument
2371 const auto I = FunctionGlobalizedDecls.find(CGF.CurFn); in emitGenericVarsEpilog()
2373 I->getSecond().MappedParams->restore(CGF); in emitGenericVarsEpilog()
2374 if (!CGF.HaveInsertPoint()) in emitGenericVarsEpilog()
2378 CGF.EmitRuntimeCall( in emitGenericVarsEpilog()
2386 CGBuilderTy &Bld = CGF.Builder; in emitGenericVarsEpilog()
2387 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".exit"); in emitGenericVarsEpilog()
2388 llvm::BasicBlock *NonSPMDBB = CGF.createBasicBlock(".non-spmd"); in emitGenericVarsEpilog()
2391 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitGenericVarsEpilog()
2392 CGF.EmitBlock(NonSPMDBB); in emitGenericVarsEpilog()
2393 CGF.EmitRuntimeCall( in emitGenericVarsEpilog()
2396 CGF.EmitCastToVoidPtr(I->getSecond().GlobalRecordAddr)); in emitGenericVarsEpilog()
2397 CGF.EmitBlock(ExitBB); in emitGenericVarsEpilog()
2406 llvm::Value *IsInSharedMemory = CGF.EmitLoadOfScalar( in emitGenericVarsEpilog()
2415 CGF.EmitRuntimeCall( in emitGenericVarsEpilog()
2421 CGF.EmitRuntimeCall(createNVPTXRuntimeFunction( in emitGenericVarsEpilog()
2429 void CGOpenMPRuntimeNVPTX::emitTeamsCall(CodeGenFunction &CGF, in emitTeamsCall() argument
2434 if (!CGF.HaveInsertPoint()) in emitTeamsCall()
2437 Address ZeroAddr = CGF.CreateMemTemp( in emitTeamsCall()
2438 CGF.getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1), in emitTeamsCall()
2440 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0)); in emitTeamsCall()
2442 OutlinedFnArgs.push_back(emitThreadIDAddress(CGF, Loc).getPointer()); in emitTeamsCall()
2445 emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs); in emitTeamsCall()
2449 CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *OutlinedFn, in emitParallelCall() argument
2451 if (!CGF.HaveInsertPoint()) in emitParallelCall()
2455 emitSPMDParallelCall(CGF, Loc, OutlinedFn, CapturedVars, IfCond); in emitParallelCall()
2457 emitNonSPMDParallelCall(CGF, Loc, OutlinedFn, CapturedVars, IfCond); in emitParallelCall()
2461 CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *OutlinedFn, in emitNonSPMDParallelCall() argument
2468 Address ZeroAddr = CGF.CreateMemTemp(CGF.getContext().getIntTypeForBitwidth( in emitNonSPMDParallelCall()
2471 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0)); in emitNonSPMDParallelCall()
2475 CodeGenFunction &CGF, PrePostActionTy &Action) { in emitNonSPMDParallelCall() argument
2476 Action.Enter(CGF); in emitNonSPMDParallelCall()
2482 emitOutlinedFunctionCall(CGF, Loc, Fn, OutlinedFnArgs); in emitNonSPMDParallelCall()
2484 auto &&SeqGen = [this, &CodeGen, Loc](CodeGenFunction &CGF, in emitNonSPMDParallelCall()
2488 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitNonSPMDParallelCall()
2489 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitNonSPMDParallelCall()
2498 RCG(CGF); in emitNonSPMDParallelCall()
2501 auto &&L0ParallelGen = [this, CapturedVars, Fn](CodeGenFunction &CGF, in emitNonSPMDParallelCall() argument
2503 CGBuilderTy &Bld = CGF.Builder; in emitNonSPMDParallelCall()
2510 CGF.EmitRuntimeCall( in emitNonSPMDParallelCall()
2516 CodeGenFunction::OMPPrivateScope PrivateArgScope(CGF); in emitNonSPMDParallelCall()
2522 CGF.CreateDefaultAlignTempAlloca(CGF.VoidPtrPtrTy, "shared_arg_refs"); in emitNonSPMDParallelCall()
2528 CGF.EmitRuntimeCall(createNVPTXRuntimeFunction( in emitNonSPMDParallelCall()
2534 ASTContext &Ctx = CGF.getContext(); in emitNonSPMDParallelCall()
2535 Address SharedArgListAddress = CGF.EmitLoadOfPointer( in emitNonSPMDParallelCall()
2540 CGF.getPointerSize()); in emitNonSPMDParallelCall()
2543 PtrV = Bld.CreateIntToPtr(V, CGF.VoidPtrTy); in emitNonSPMDParallelCall()
2545 PtrV = Bld.CreatePointerBitCastOrAddrSpaceCast(V, CGF.VoidPtrTy); in emitNonSPMDParallelCall()
2546 CGF.EmitStoreOfScalar(PtrV, Dst, /*Volatile=*/false, in emitNonSPMDParallelCall()
2554 syncCTAThreads(CGF); in emitNonSPMDParallelCall()
2562 syncCTAThreads(CGF); in emitNonSPMDParallelCall()
2565 CGF.EmitRuntimeCall( in emitNonSPMDParallelCall()
2573 CodeGenFunction &CGF, PrePostActionTy &Action) { in emitNonSPMDParallelCall() argument
2575 SeqGen(CGF, Action); in emitNonSPMDParallelCall()
2577 L0ParallelGen(CGF, Action); in emitNonSPMDParallelCall()
2585 CGBuilderTy &Bld = CGF.Builder; in emitNonSPMDParallelCall()
2586 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".exit"); in emitNonSPMDParallelCall()
2587 llvm::BasicBlock *SeqBB = CGF.createBasicBlock(".sequential"); in emitNonSPMDParallelCall()
2588 llvm::BasicBlock *ParallelCheckBB = CGF.createBasicBlock(".parcheck"); in emitNonSPMDParallelCall()
2589 llvm::BasicBlock *MasterBB = CGF.createBasicBlock(".master"); in emitNonSPMDParallelCall()
2590 llvm::Value *IsSPMD = Bld.CreateIsNotNull(CGF.EmitNounwindRuntimeCall( in emitNonSPMDParallelCall()
2594 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitNonSPMDParallelCall()
2595 CGF.EmitBlock(ParallelCheckBB); in emitNonSPMDParallelCall()
2596 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitNonSPMDParallelCall()
2597 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitNonSPMDParallelCall()
2598 llvm::Value *PL = CGF.EmitRuntimeCall( in emitNonSPMDParallelCall()
2603 CGF.EmitBlock(SeqBB); in emitNonSPMDParallelCall()
2604 SeqGen(CGF, Action); in emitNonSPMDParallelCall()
2605 CGF.EmitBranch(ExitBB); in emitNonSPMDParallelCall()
2607 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitNonSPMDParallelCall()
2608 CGF.EmitBlock(MasterBB); in emitNonSPMDParallelCall()
2609 L0ParallelGen(CGF, Action); in emitNonSPMDParallelCall()
2610 CGF.EmitBranch(ExitBB); in emitNonSPMDParallelCall()
2612 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitNonSPMDParallelCall()
2614 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in emitNonSPMDParallelCall()
2619 emitOMPIfClause(CGF, IfCond, LNParallelGen, SeqGen); in emitNonSPMDParallelCall()
2621 CodeGenFunction::RunCleanupsScope Scope(CGF); in emitNonSPMDParallelCall()
2623 ThenRCG(CGF); in emitNonSPMDParallelCall()
2628 CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *OutlinedFn, in emitSPMDParallelCall() argument
2635 Address ZeroAddr = CGF.CreateMemTemp(CGF.getContext().getIntTypeForBitwidth( in emitSPMDParallelCall()
2638 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0)); in emitSPMDParallelCall()
2642 &ThreadIDAddr](CodeGenFunction &CGF, in emitSPMDParallelCall()
2644 Action.Enter(CGF); in emitSPMDParallelCall()
2650 emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs); in emitSPMDParallelCall()
2652 auto &&SeqGen = [this, &CodeGen, Loc](CodeGenFunction &CGF, in emitSPMDParallelCall() argument
2656 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitSPMDParallelCall()
2657 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitSPMDParallelCall()
2666 RCG(CGF); in emitSPMDParallelCall()
2671 ThreadIDAddr = emitThreadIDAddress(CGF, Loc); in emitSPMDParallelCall()
2673 RCG(CGF); in emitSPMDParallelCall()
2679 RCG(CGF); in emitSPMDParallelCall()
2683 void CGOpenMPRuntimeNVPTX::syncCTAThreads(CodeGenFunction &CGF) { in syncCTAThreads() argument
2685 if (!CGF.HaveInsertPoint()) in syncCTAThreads()
2692 llvm::ConstantInt::get(CGF.Int32Ty, /*V=*/0, /*isSigned=*/true)}; in syncCTAThreads()
2693 CGF.EmitRuntimeCall( in syncCTAThreads()
2697 void CGOpenMPRuntimeNVPTX::emitBarrierCall(CodeGenFunction &CGF, in emitBarrierCall() argument
2702 if (!CGF.HaveInsertPoint()) in emitBarrierCall()
2706 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags), in emitBarrierCall()
2707 getThreadID(CGF, Loc)}; in emitBarrierCall()
2708 CGF.EmitRuntimeCall(createNVPTXRuntimeFunction(OMPRTL__kmpc_barrier), Args); in emitBarrierCall()
2712 CodeGenFunction &CGF, StringRef CriticalName, in emitCriticalRegion() argument
2715 llvm::BasicBlock *LoopBB = CGF.createBasicBlock("omp.critical.loop"); in emitCriticalRegion()
2716 llvm::BasicBlock *TestBB = CGF.createBasicBlock("omp.critical.test"); in emitCriticalRegion()
2717 llvm::BasicBlock *SyncBB = CGF.createBasicBlock("omp.critical.sync"); in emitCriticalRegion()
2718 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.critical.body"); in emitCriticalRegion()
2719 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.critical.exit"); in emitCriticalRegion()
2722 llvm::Value *ThreadID = getNVPTXThreadID(CGF); in emitCriticalRegion()
2725 llvm::Value *TeamWidth = getNVPTXNumThreads(CGF); in emitCriticalRegion()
2729 CGF.getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/0); in emitCriticalRegion()
2730 Address Counter = CGF.CreateMemTemp(Int32Ty, "critical_counter"); in emitCriticalRegion()
2731 LValue CounterLVal = CGF.MakeAddrLValue(Counter, Int32Ty); in emitCriticalRegion()
2732 CGF.EmitStoreOfScalar(llvm::Constant::getNullValue(CGM.Int32Ty), CounterLVal, in emitCriticalRegion()
2736 CGF.EmitBlock(LoopBB); in emitCriticalRegion()
2737 llvm::Value *CounterVal = CGF.EmitLoadOfScalar(CounterLVal, Loc); in emitCriticalRegion()
2738 llvm::Value *CmpLoopBound = CGF.Builder.CreateICmpSLT(CounterVal, TeamWidth); in emitCriticalRegion()
2739 CGF.Builder.CreateCondBr(CmpLoopBound, TestBB, ExitBB); in emitCriticalRegion()
2743 CGF.EmitBlock(TestBB); in emitCriticalRegion()
2744 CounterVal = CGF.EmitLoadOfScalar(CounterLVal, Loc); in emitCriticalRegion()
2746 CGF.Builder.CreateICmpEQ(ThreadID, CounterVal); in emitCriticalRegion()
2747 CGF.Builder.CreateCondBr(CmpThreadToCounter, BodyBB, SyncBB); in emitCriticalRegion()
2750 CGF.EmitBlock(BodyBB); in emitCriticalRegion()
2753 CGOpenMPRuntime::emitCriticalRegion(CGF, CriticalName, CriticalOpGen, Loc, in emitCriticalRegion()
2760 CGF.EmitBlock(SyncBB); in emitCriticalRegion()
2761 emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false, in emitCriticalRegion()
2765 CGF.Builder.CreateNSWAdd(CounterVal, CGF.Builder.getInt32(1)); in emitCriticalRegion()
2766 CGF.EmitStoreOfScalar(IncCounterVal, CounterLVal); in emitCriticalRegion()
2767 CGF.EmitBranch(LoopBB); in emitCriticalRegion()
2770 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in emitCriticalRegion()
2774 static llvm::Value *castValueToType(CodeGenFunction &CGF, llvm::Value *Val, in castValueToType() argument
2777 assert(!CGF.getContext().getTypeSizeInChars(CastTy).isZero() && in castValueToType()
2779 assert(!CGF.getContext().getTypeSizeInChars(ValTy).isZero() && in castValueToType()
2781 llvm::Type *LLVMCastTy = CGF.ConvertTypeForMem(CastTy); in castValueToType()
2784 if (CGF.getContext().getTypeSizeInChars(ValTy) == in castValueToType()
2785 CGF.getContext().getTypeSizeInChars(CastTy)) in castValueToType()
2786 return CGF.Builder.CreateBitCast(Val, LLVMCastTy); in castValueToType()
2788 return CGF.Builder.CreateIntCast(Val, LLVMCastTy, in castValueToType()
2790 Address CastItem = CGF.CreateMemTemp(CastTy); in castValueToType()
2791 Address ValCastItem = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in castValueToType()
2793 CGF.EmitStoreOfScalar(Val, ValCastItem, /*Volatile=*/false, ValTy); in castValueToType()
2794 return CGF.EmitLoadOfScalar(CastItem, /*Volatile=*/false, CastTy, Loc); in castValueToType()
2799 static llvm::Value *createRuntimeShuffleFunction(CodeGenFunction &CGF, in createRuntimeShuffleFunction() argument
2804 CodeGenModule &CGM = CGF.CGM; in createRuntimeShuffleFunction()
2805 CGBuilderTy &Bld = CGF.Builder; in createRuntimeShuffleFunction()
2809 CharUnits Size = CGF.getContext().getTypeSizeInChars(ElemType); in createRuntimeShuffleFunction()
2818 QualType CastTy = CGF.getContext().getIntTypeForBitwidth( in createRuntimeShuffleFunction()
2820 llvm::Value *ElemCast = castValueToType(CGF, Elem, ElemType, CastTy, Loc); in createRuntimeShuffleFunction()
2822 Bld.CreateIntCast(getNVPTXWarpSize(CGF), CGM.Int16Ty, /*isSigned=*/true); in createRuntimeShuffleFunction()
2824 llvm::Value *ShuffledVal = CGF.EmitRuntimeCall( in createRuntimeShuffleFunction()
2827 return castValueToType(CGF, ShuffledVal, CastTy, ElemType, Loc); in createRuntimeShuffleFunction()
2830 static void shuffleAndStore(CodeGenFunction &CGF, Address SrcAddr, in shuffleAndStore() argument
2833 CGBuilderTy &Bld = CGF.Builder; in shuffleAndStore()
2835 CharUnits Size = CGF.getContext().getTypeSizeInChars(ElemType); in shuffleAndStore()
2849 Bld.CreateConstGEP(SrcAddr, 1, Size), CGF.VoidPtrTy); in shuffleAndStore()
2853 QualType IntType = CGF.getContext().getIntTypeForBitwidth( in shuffleAndStore()
2854 CGF.getContext().toBits(CharUnits::fromQuantity(IntSize)), in shuffleAndStore()
2856 llvm::Type *IntTy = CGF.ConvertTypeForMem(IntType); in shuffleAndStore()
2861 llvm::BasicBlock *PreCondBB = CGF.createBasicBlock(".shuffle.pre_cond"); in shuffleAndStore()
2862 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".shuffle.then"); in shuffleAndStore()
2863 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".shuffle.exit"); in shuffleAndStore()
2865 CGF.EmitBlock(PreCondBB); in shuffleAndStore()
2876 Ptr.getPointer(), CGF.VoidPtrTy)); in shuffleAndStore()
2879 CGF.EmitBlock(ThenBB); in shuffleAndStore()
2881 CGF, CGF.EmitLoadOfScalar(Ptr, /*Volatile=*/false, IntType, Loc), in shuffleAndStore()
2883 CGF.EmitStoreOfScalar(Res, ElemPtr, /*Volatile=*/false, IntType); in shuffleAndStore()
2890 CGF.EmitBranch(PreCondBB); in shuffleAndStore()
2891 CGF.EmitBlock(ExitBB); in shuffleAndStore()
2894 CGF, CGF.EmitLoadOfScalar(Ptr, /*Volatile=*/false, IntType, Loc), in shuffleAndStore()
2896 CGF.EmitStoreOfScalar(Res, ElemPtr, /*Volatile=*/false, IntType); in shuffleAndStore()
2929 CopyAction Action, CodeGenFunction &CGF, QualType ReductionArrayTy, in emitReductionListCopy() argument
2933 CodeGenModule &CGM = CGF.CGM;
2935 CGBuilderTy &Bld = CGF.Builder;
2963 Bld.CreateConstArrayGEP(SrcBase, Idx, CGF.getPointerSize());
2964 SrcElementAddr = CGF.EmitLoadOfPointer(
2971 Bld.CreateConstArrayGEP(DestBase, Idx, CGF.getPointerSize());
2973 CGF.CreateMemTemp(Private->getType(), ".omp.reduction.element");
2981 Bld.CreateConstArrayGEP(SrcBase, Idx, CGF.getPointerSize());
2982 SrcElementAddr = CGF.EmitLoadOfPointer(
2989 Bld.CreateConstArrayGEP(DestBase, Idx, CGF.getPointerSize());
2990 DestElementAddr = CGF.EmitLoadOfPointer(
2998 Bld.CreateConstArrayGEP(SrcBase, Idx, CGF.getPointerSize());
2999 SrcElementAddr = CGF.EmitLoadOfPointer(
3005 llvm::Value *ElementSizeInChars = CGF.getTypeSize(Private->getType());
3011 Bld.CreateIntToPtr(ScratchPadElemAbsolutePtrVal, CGF.VoidPtrTy);
3020 llvm::Value *ElementSizeInChars = CGF.getTypeSize(Private->getType());
3026 Bld.CreateIntToPtr(ScratchPadElemAbsolutePtrVal, CGF.VoidPtrTy);
3034 Bld.CreateConstArrayGEP(DestBase, Idx, CGF.getPointerSize());
3036 CGF.CreateMemTemp(Private->getType(), ".omp.reduction.element");
3045 SrcElementAddr, CGF.ConvertTypeForMem(Private->getType()));
3052 shuffleAndStore(CGF, SrcElementAddr, DestElementAddr, Private->getType(),
3057 CGF.EmitLoadOfScalar(SrcElementAddr, /*Volatile=*/false,
3060 CGF.EmitStoreOfScalar(Elem, DestElementAddr, /*Volatile=*/false,
3063 CGF.EmitAggregateCopy(
3064 CGF.MakeAddrLValue(DestElementAddr, Private->getType()),
3065 CGF.MakeAddrLValue(SrcElementAddr, Private->getType()),
3076 CGF.EmitStoreOfScalar(Bld.CreatePointerBitCastOrAddrSpaceCast(
3077 DestElementAddr.getPointer(), CGF.VoidPtrTy),
3088 llvm::Value *ElementSizeInChars = CGF.getTypeSize(Private->getType());
3106 DestBase = Address(ScratchpadBasePtr, CGF.getPointerAlign());
3108 SrcBase = Address(ScratchpadBasePtr, CGF.getPointerAlign());
3155 CodeGenFunction CGF(CGM); in emitInterWarpCopyFunction() local
3156 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitInterWarpCopyFunction()
3158 CGBuilderTy &Bld = CGF.Builder; in emitInterWarpCopyFunction()
3183 llvm::Value *ThreadID = getNVPTXThreadID(CGF); in emitInterWarpCopyFunction()
3185 llvm::Value *LaneID = getNVPTXLaneID(CGF); in emitInterWarpCopyFunction()
3187 llvm::Value *WarpID = getNVPTXWarpID(CGF); in emitInterWarpCopyFunction()
3189 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitInterWarpCopyFunction()
3192 CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false, in emitInterWarpCopyFunction()
3194 CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo()), in emitInterWarpCopyFunction()
3195 CGF.getPointerAlign()); in emitInterWarpCopyFunction()
3213 llvm::Type *CopyType = CGF.ConvertTypeForMem(CType); in emitInterWarpCopyFunction()
3220 CntAddr = CGF.CreateMemTemp(C.IntTy, ".cnt.addr"); in emitInterWarpCopyFunction()
3221 CGF.EmitStoreOfScalar(llvm::Constant::getNullValue(CGM.IntTy), CntAddr, in emitInterWarpCopyFunction()
3223 PrecondBB = CGF.createBasicBlock("precond"); in emitInterWarpCopyFunction()
3224 ExitBB = CGF.createBasicBlock("exit"); in emitInterWarpCopyFunction()
3225 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("body"); in emitInterWarpCopyFunction()
3227 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitInterWarpCopyFunction()
3228 CGF.EmitBlock(PrecondBB); in emitInterWarpCopyFunction()
3229 Cnt = CGF.EmitLoadOfScalar(CntAddr, /*Volatile=*/false, C.IntTy, Loc); in emitInterWarpCopyFunction()
3233 CGF.EmitBlock(BodyBB); in emitInterWarpCopyFunction()
3236 CGM.getOpenMPRuntime().emitBarrierCall(CGF, Loc, OMPD_unknown, in emitInterWarpCopyFunction()
3239 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("then"); in emitInterWarpCopyFunction()
3240 llvm::BasicBlock *ElseBB = CGF.createBasicBlock("else"); in emitInterWarpCopyFunction()
3241 llvm::BasicBlock *MergeBB = CGF.createBasicBlock("ifcont"); in emitInterWarpCopyFunction()
3246 CGF.EmitBlock(ThenBB); in emitInterWarpCopyFunction()
3250 Bld.CreateConstArrayGEP(LocalReduceList, Idx, CGF.getPointerSize()); in emitInterWarpCopyFunction()
3251 llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
3273 CGF.EmitLoadOfScalar(ElemPtr, /*Volatile=*/false, CType, Loc); in emitInterWarpCopyFunction()
3275 CGF.EmitStoreOfScalar(Elem, MediumPtr, /*Volatile=*/true, CType); in emitInterWarpCopyFunction()
3279 CGF.EmitBlock(ElseBB); in emitInterWarpCopyFunction()
3282 CGF.EmitBlock(MergeBB); in emitInterWarpCopyFunction()
3285 CGM.getOpenMPRuntime().emitBarrierCall(CGF, Loc, OMPD_unknown, in emitInterWarpCopyFunction()
3292 llvm::BasicBlock *W0ThenBB = CGF.createBasicBlock("then"); in emitInterWarpCopyFunction()
3293 llvm::BasicBlock *W0ElseBB = CGF.createBasicBlock("else"); in emitInterWarpCopyFunction()
3294 llvm::BasicBlock *W0MergeBB = CGF.createBasicBlock("ifcont"); in emitInterWarpCopyFunction()
3296 Address AddrNumWarpsArg = CGF.GetAddrOfLocalVar(&NumWarpsArg); in emitInterWarpCopyFunction()
3297 llvm::Value *NumWarpsVal = CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
3305 CGF.EmitBlock(W0ThenBB); in emitInterWarpCopyFunction()
3317 Bld.CreateConstArrayGEP(LocalReduceList, Idx, CGF.getPointerSize()); in emitInterWarpCopyFunction()
3318 llvm::Value *TargetElemPtrVal = CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
3329 CGF.EmitLoadOfScalar(SrcMediumPtr, /*Volatile=*/true, CType, Loc); in emitInterWarpCopyFunction()
3330 CGF.EmitStoreOfScalar(SrcMediumValue, TargetElemPtr, /*Volatile=*/false, in emitInterWarpCopyFunction()
3334 CGF.EmitBlock(W0ElseBB); in emitInterWarpCopyFunction()
3337 CGF.EmitBlock(W0MergeBB); in emitInterWarpCopyFunction()
3341 CGF.EmitStoreOfScalar(Cnt, CntAddr, /*Volatile=*/false, C.IntTy); in emitInterWarpCopyFunction()
3342 CGF.EmitBranch(PrecondBB); in emitInterWarpCopyFunction()
3343 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitInterWarpCopyFunction()
3344 CGF.EmitBlock(ExitBB); in emitInterWarpCopyFunction()
3351 CGF.FinishFunction(); in emitInterWarpCopyFunction()
3451 CodeGenFunction CGF(CGM); in emitShuffleAndReduceFunction() local
3452 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitShuffleAndReduceFunction()
3454 CGBuilderTy &Bld = CGF.Builder; in emitShuffleAndReduceFunction()
3456 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitShuffleAndReduceFunction()
3459 CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false, in emitShuffleAndReduceFunction()
3461 CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo()), in emitShuffleAndReduceFunction()
3462 CGF.getPointerAlign()); in emitShuffleAndReduceFunction()
3464 Address AddrLaneIDArg = CGF.GetAddrOfLocalVar(&LaneIDArg); in emitShuffleAndReduceFunction()
3465 llvm::Value *LaneIDArgVal = CGF.EmitLoadOfScalar( in emitShuffleAndReduceFunction()
3468 Address AddrRemoteLaneOffsetArg = CGF.GetAddrOfLocalVar(&RemoteLaneOffsetArg); in emitShuffleAndReduceFunction()
3469 llvm::Value *RemoteLaneOffsetArgVal = CGF.EmitLoadOfScalar( in emitShuffleAndReduceFunction()
3472 Address AddrAlgoVerArg = CGF.GetAddrOfLocalVar(&AlgoVerArg); in emitShuffleAndReduceFunction()
3473 llvm::Value *AlgoVerArgVal = CGF.EmitLoadOfScalar( in emitShuffleAndReduceFunction()
3479 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.remote_reduce_list"); in emitShuffleAndReduceFunction()
3484 emitReductionListCopy(RemoteLaneToThread, CGF, ReductionArrayTy, Privates, in emitShuffleAndReduceFunction()
3526 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("then"); in emitShuffleAndReduceFunction()
3527 llvm::BasicBlock *ElseBB = CGF.createBasicBlock("else"); in emitShuffleAndReduceFunction()
3528 llvm::BasicBlock *MergeBB = CGF.createBasicBlock("ifcont"); in emitShuffleAndReduceFunction()
3531 CGF.EmitBlock(ThenBB); in emitShuffleAndReduceFunction()
3534 LocalReduceList.getPointer(), CGF.VoidPtrTy); in emitShuffleAndReduceFunction()
3536 RemoteReduceList.getPointer(), CGF.VoidPtrTy); in emitShuffleAndReduceFunction()
3538 CGF, Loc, ReduceFn, {LocalReduceListPtr, RemoteReduceListPtr}); in emitShuffleAndReduceFunction()
3541 CGF.EmitBlock(ElseBB); in emitShuffleAndReduceFunction()
3544 CGF.EmitBlock(MergeBB); in emitShuffleAndReduceFunction()
3552 llvm::BasicBlock *CpyThenBB = CGF.createBasicBlock("then"); in emitShuffleAndReduceFunction()
3553 llvm::BasicBlock *CpyElseBB = CGF.createBasicBlock("else"); in emitShuffleAndReduceFunction()
3554 llvm::BasicBlock *CpyMergeBB = CGF.createBasicBlock("ifcont"); in emitShuffleAndReduceFunction()
3557 CGF.EmitBlock(CpyThenBB); in emitShuffleAndReduceFunction()
3558 emitReductionListCopy(ThreadCopy, CGF, ReductionArrayTy, Privates, in emitShuffleAndReduceFunction()
3562 CGF.EmitBlock(CpyElseBB); in emitShuffleAndReduceFunction()
3565 CGF.EmitBlock(CpyMergeBB); in emitShuffleAndReduceFunction()
3567 CGF.FinishFunction(); in emitShuffleAndReduceFunction()
3814 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates, in emitReduction() argument
3817 if (!CGF.HaveInsertPoint()) in emitReduction()
3828 CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs, in emitReduction()
3840 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitReduction()
3841 llvm::Value *ThreadId = getThreadID(CGF, Loc); in emitReduction()
3859 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); in emitReduction()
3863 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx, in emitReduction()
3864 CGF.getPointerSize()); in emitReduction()
3865 CGF.Builder.CreateStore( in emitReduction()
3866 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReduction()
3867 CGF.EmitLValue(RHSExprs[I]).getPointer(), CGF.VoidPtrTy), in emitReduction()
3872 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx, in emitReduction()
3873 CGF.getPointerSize()); in emitReduction()
3874 llvm::Value *Size = CGF.Builder.CreateIntCast( in emitReduction()
3875 CGF.getVLASize( in emitReduction()
3876 CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) in emitReduction()
3878 CGF.SizeTy, /*isSigned=*/false); in emitReduction()
3879 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), in emitReduction()
3884 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy); in emitReduction()
3885 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReduction()
3886 ReductionList.getPointer(), CGF.VoidPtrTy); in emitReduction()
3888 CGM, Loc, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), in emitReduction()
3897 CGF.Builder.getInt32(RHSExprs.size()), in emitReduction()
3903 Res = CGF.EmitRuntimeCall(createNVPTXRuntimeFunction( in emitReduction()
3911 Res = CGF.EmitRuntimeCall( in emitReduction()
3918 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.reduction.done"); in emitReduction()
3919 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.then"); in emitReduction()
3920 llvm::Value *Cond = CGF.Builder.CreateICmpEQ( in emitReduction()
3922 CGF.Builder.CreateCondBr(Cond, ThenBB, ExitBB); in emitReduction()
3928 CGF.EmitBlock(ThenBB); in emitReduction()
3932 this](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
3937 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), in emitReduction()
3952 RCG(CGF); in emitReduction()
3955 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitReduction()
3966 RCG(CGF); in emitReduction()
3969 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitReduction()
3970 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in emitReduction()
4006 CGOpenMPRuntimeNVPTX::getParameterAddress(CodeGenFunction &CGF, in getParameterAddress() argument
4012 Address LocalAddr = CGF.GetAddrOfLocalVar(TargetParam); in getParameterAddress()
4018 CGF.getContext().getTargetAddressSpace(NativePointeeTy); in getParameterAddress()
4020 llvm::Value *TargetAddr = CGF.EmitLoadOfScalar( in getParameterAddress()
4023 TargetAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getParameterAddress()
4027 TargetAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getParameterAddress()
4030 Address NativeParamAddr = CGF.CreateMemTemp(NativeParamType); in getParameterAddress()
4031 CGF.EmitStoreOfScalar(TargetAddr, NativeParamAddr, /*Volatile=*/false, in getParameterAddress()
4037 CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *OutlinedFn, in emitOutlinedFunctionCall() argument
4054 llvm::Value *TargetArg = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitOutlinedFunctionCall()
4058 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TargetArg, TargetType)); in emitOutlinedFunctionCall()
4060 CGOpenMPRuntime::emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, TargetArgs); in emitOutlinedFunctionCall()
4097 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true); in createParallelDataSharingWrapper() local
4098 CGF.StartFunction(GlobalDecl(), Ctx.VoidTy, Fn, CGFI, WrapperArgs, in createParallelDataSharingWrapper()
4104 Address ZeroAddr = CGF.CreateMemTemp( in createParallelDataSharingWrapper()
4105 CGF.getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1), in createParallelDataSharingWrapper()
4107 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0)); in createParallelDataSharingWrapper()
4111 Args.emplace_back(CGF.GetAddrOfLocalVar(&WrapperArg).getPointer()); in createParallelDataSharingWrapper()
4114 CGBuilderTy &Bld = CGF.Builder; in createParallelDataSharingWrapper()
4120 CGF.CreateDefaultAlignTempAlloca(CGF.VoidPtrPtrTy, "global_args"); in createParallelDataSharingWrapper()
4123 CGF.EmitRuntimeCall( in createParallelDataSharingWrapper()
4132 SharedArgListAddress = CGF.EmitLoadOfPointer( in createParallelDataSharingWrapper()
4133 GlobalArgs, CGF.getContext() in createParallelDataSharingWrapper()
4134 .getPointerType(CGF.getContext().getPointerType( in createParallelDataSharingWrapper()
4135 CGF.getContext().VoidPtrTy)) in createParallelDataSharingWrapper()
4141 CGF.getPointerSize()); in createParallelDataSharingWrapper()
4143 Src, CGF.SizeTy->getPointerTo()); in createParallelDataSharingWrapper()
4144 llvm::Value *LB = CGF.EmitLoadOfScalar( in createParallelDataSharingWrapper()
4147 CGF.getContext().getPointerType(CGF.getContext().getSizeType()), in createParallelDataSharingWrapper()
4152 CGF.getPointerSize()); in createParallelDataSharingWrapper()
4154 Src, CGF.SizeTy->getPointerTo()); in createParallelDataSharingWrapper()
4155 llvm::Value *UB = CGF.EmitLoadOfScalar( in createParallelDataSharingWrapper()
4158 CGF.getContext().getPointerType(CGF.getContext().getSizeType()), in createParallelDataSharingWrapper()
4164 ASTContext &CGFContext = CGF.getContext(); in createParallelDataSharingWrapper()
4168 CGF.getPointerSize()); in createParallelDataSharingWrapper()
4170 Src, CGF.ConvertTypeForMem(CGFContext.getPointerType(ElemTy))); in createParallelDataSharingWrapper()
4171 llvm::Value *Arg = CGF.EmitLoadOfScalar(TypedAddress, in createParallelDataSharingWrapper()
4177 Arg = castValueToType(CGF, Arg, ElemTy, CGFContext.getUIntPtrType(), in createParallelDataSharingWrapper()
4184 emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedParallelFn, Args); in createParallelDataSharingWrapper()
4185 CGF.FinishFunction(); in createParallelDataSharingWrapper()
4189 void CGOpenMPRuntimeNVPTX::emitFunctionProlog(CodeGenFunction &CGF, in emitFunctionProlog() argument
4195 assert(FunctionGlobalizedDecls.count(CGF.CurFn) == 0 && in emitFunctionProlog()
4207 NeedToDelayGlobalization = CGF.CapturedStmtInfo->getKind() == CR_OpenMP; in emitFunctionProlog()
4214 CheckVarsEscapingDeclContext VarChecker(CGF, TeamAndReductions.second); in emitFunctionProlog()
4224 auto I = FunctionGlobalizedDecls.try_emplace(CGF.CurFn).first; in emitFunctionProlog()
4240 CheckVarsEscapingDeclContext VarChecker(CGF, llvm::None); in emitFunctionProlog()
4254 emitGenericVarsProlog(CGF, D->getBeginLoc(), /*WithSPMDCheck=*/true); in emitFunctionProlog()
4258 void Emit(CodeGenFunction &CGF, Flags flags) override { in emitFunctionProlog()
4259 static_cast<CGOpenMPRuntimeNVPTX &>(CGF.CGM.getOpenMPRuntime()) in emitFunctionProlog()
4260 .emitGenericVarsEpilog(CGF, /*WithSPMDCheck=*/true); in emitFunctionProlog()
4263 CGF.EHStack.pushCleanup<GlobalizationScope>(NormalAndEHCleanup); in emitFunctionProlog()
4267 Address CGOpenMPRuntimeNVPTX::getAddressOfLocalVariable(CodeGenFunction &CGF, in getAddressOfLocalVariable() argument
4273 auto I = FunctionGlobalizedDecls.find(CGF.CurFn); in getAddressOfLocalVariable()
4293 void CGOpenMPRuntimeNVPTX::functionFinished(CodeGenFunction &CGF) { in functionFinished() argument
4294 FunctionGlobalizedDecls.erase(CGF.CurFn); in functionFinished()
4295 CGOpenMPRuntime::functionFinished(CGF); in functionFinished()
4299 CodeGenFunction &CGF, const OMPLoopDirective &S, in getDefaultDistScheduleAndChunk() argument
4304 Chunk = CGF.EmitScalarConversion(getNVPTXNumThreads(CGF), in getDefaultDistScheduleAndChunk()
4305 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0), in getDefaultDistScheduleAndChunk()
4310 CGF, S, ScheduleKind, Chunk); in getDefaultDistScheduleAndChunk()
4314 CodeGenFunction &CGF, const OMPLoopDirective &S, in getDefaultScheduleAndChunk() argument
4320 ChunkExpr = IntegerLiteral::Create(CGF.getContext(), ChunkSize, in getDefaultScheduleAndChunk()
4321 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0), in getDefaultScheduleAndChunk()
4326 CodeGenFunction &CGF, const OMPExecutableDirective &D) const { in adjustTargetSpecificDataForLambdas() argument
4342 Address VDAddr = CGF.GetAddrOfLocalVar(VD); in adjustTargetSpecificDataForLambdas()
4345 VDLVal = CGF.EmitLoadOfReferenceLValue(VDAddr, VD->getType()); in adjustTargetSpecificDataForLambdas()
4347 VDLVal = CGF.MakeAddrLValue( in adjustTargetSpecificDataForLambdas()
4352 if (ThisCapture && CGF.CapturedStmtInfo->isCXXThisExprCaptured()) { in adjustTargetSpecificDataForLambdas()
4354 CGF.EmitLValueForFieldInitialization(VDLVal, ThisCapture); in adjustTargetSpecificDataForLambdas()
4355 llvm::Value *CXXThis = CGF.LoadCXXThis(); in adjustTargetSpecificDataForLambdas()
4356 CGF.EmitStoreOfScalar(CXXThis, ThisLVal); in adjustTargetSpecificDataForLambdas()
4366 LValue VarLVal = CGF.EmitLValueForFieldInitialization(VDLVal, It->second); in adjustTargetSpecificDataForLambdas()
4367 Address VDAddr = CGF.GetAddrOfLocalVar(VD); in adjustTargetSpecificDataForLambdas()
4369 VDAddr = CGF.EmitLoadOfReferenceLValue(VDAddr, in adjustTargetSpecificDataForLambdas()
4372 CGF.EmitStoreOfScalar(VDAddr.getPointer(), VarLVal); in adjustTargetSpecificDataForLambdas()