Lines Matching refs:CGF
46 void Enter(CodeGenFunction &CGF) override { in Enter() argument
47 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs); in Enter()
49 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes); in Enter()
50 auto *ThenBlock = CGF.createBasicBlock("omp_if.then"); in Enter()
51 ContBlock = CGF.createBasicBlock("omp_if.end"); in Enter()
53 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock); in Enter()
54 CGF.EmitBlock(ThenBlock); in Enter()
57 void Done(CodeGenFunction &CGF) { in Done() argument
59 CGF.EmitBranch(ContBlock); in Done()
60 CGF.EmitBlock(ContBlock, true); in Done()
62 void Exit(CodeGenFunction &CGF) override { in Exit() argument
63 CGF.EmitRuntimeCall(ExitCallee, ExitArgs); in Exit()
222 CodeGenFunction &CGF; member in __anona9bff4970111::CheckVarsEscapingDeclContext
241 if (auto *CSI = CGF.CapturedStmtInfo) { in markAsEscaped()
266 if ((!CGF.CapturedStmtInfo || in markAsEscaped()
267 (IsForCombinedParallelRegion && CGF.CapturedStmtInfo)) && in markAsEscaped()
340 unsigned WarpSize = CGF.getTarget().getGridValue().GV_Warp_Size; in buildRecordForGlobalizedVars()
346 CGF.getContext(), EscapedDeclsForParallel, EscapedDeclsForTeams, in buildRecordForGlobalizedVars()
351 CheckVarsEscapingDeclContext(CodeGenFunction &CGF, in CheckVarsEscapingDeclContext() argument
353 : CGF(CGF), EscapedDecls(TeamsReductions.begin(), TeamsReductions.end()) { in CheckVarsEscapingDeclContext()
534 static llvm::Value *getNVPTXWarpID(CodeGenFunction &CGF) { in getNVPTXWarpID() argument
535 CGBuilderTy &Bld = CGF.Builder; in getNVPTXWarpID()
537 llvm::Log2_32(CGF.getTarget().getGridValue().GV_Warp_Size); in getNVPTXWarpID()
538 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in getNVPTXWarpID()
539 return Bld.CreateAShr(RT.getGPUThreadID(CGF), LaneIDBits, "nvptx_warp_id"); in getNVPTXWarpID()
545 static llvm::Value *getNVPTXLaneID(CodeGenFunction &CGF) { in getNVPTXLaneID() argument
546 CGBuilderTy &Bld = CGF.Builder; in getNVPTXLaneID()
548 llvm::Log2_32(CGF.getTarget().getGridValue().GV_Warp_Size); in getNVPTXLaneID()
550 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in getNVPTXLaneID()
551 return Bld.CreateAnd(RT.getGPUThreadID(CGF), Bld.getInt32(LaneIDMask), in getNVPTXLaneID()
1027 void Enter(CodeGenFunction &CGF) override { in emitNonSPMDKernel() argument
1029 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitNonSPMDKernel()
1030 RT.emitKernelInit(CGF, EST, /* IsSPMD */ false); in emitNonSPMDKernel()
1032 RT.setLocThreadIdInsertPt(CGF, /*AtCurrentPoint=*/true); in emitNonSPMDKernel()
1034 void Exit(CodeGenFunction &CGF) override { in emitNonSPMDKernel() argument
1036 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitNonSPMDKernel()
1037 RT.clearLocThreadIdInsertPt(CGF); in emitNonSPMDKernel()
1038 RT.emitKernelDeinit(CGF, EST, /* IsSPMD */ false); in emitNonSPMDKernel()
1048 void CGOpenMPRuntimeGPU::emitKernelInit(CodeGenFunction &CGF, in emitKernelInit() argument
1050 CGBuilderTy &Bld = CGF.Builder; in emitKernelInit()
1054 emitGenericVarsProlog(CGF, EST.Loc); in emitKernelInit()
1057 void CGOpenMPRuntimeGPU::emitKernelDeinit(CodeGenFunction &CGF, in emitKernelDeinit() argument
1061 emitGenericVarsEpilog(CGF); in emitKernelDeinit()
1063 CGBuilderTy &Bld = CGF.Builder; in emitKernelDeinit()
1088 void Enter(CodeGenFunction &CGF) override { in emitSPMDKernel() argument
1089 RT.emitKernelInit(CGF, EST, /* IsSPMD */ true); in emitSPMDKernel()
1091 RT.setLocThreadIdInsertPt(CGF, /*AtCurrentPoint=*/true); in emitSPMDKernel()
1093 void Exit(CodeGenFunction &CGF) override { in emitSPMDKernel() argument
1094 RT.clearLocThreadIdInsertPt(CGF); in emitSPMDKernel()
1095 RT.emitKernelDeinit(CGF, EST, /* IsSPMD */ true); in emitSPMDKernel()
1219 void CGOpenMPRuntimeGPU::emitProcBindClause(CodeGenFunction &CGF, in emitProcBindClause() argument
1226 CGOpenMPRuntime::emitProcBindClause(CGF, ProcBind, Loc); in emitProcBindClause()
1229 void CGOpenMPRuntimeGPU::emitNumThreadsClause(CodeGenFunction &CGF, in emitNumThreadsClause() argument
1235 void CGOpenMPRuntimeGPU::emitNumTeamsClause(CodeGenFunction &CGF, in emitNumTeamsClause() argument
1251 void Enter(CodeGenFunction &CGF) override { in emitParallelOutlinedFunction() argument
1255 void Exit(CodeGenFunction &CGF) override { in emitParallelOutlinedFunction() argument
1357 void Enter(CodeGenFunction &CGF) override { in emitTeamsOutlinedFunction() argument
1359 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitTeamsOutlinedFunction()
1361 auto I = Rt.FunctionGlobalizedDecls.try_emplace(CGF.CurFn).first; in emitTeamsOutlinedFunction()
1371 Rt.emitGenericVarsProlog(CGF, Loc); in emitTeamsOutlinedFunction()
1373 void Exit(CodeGenFunction &CGF) override { in emitTeamsOutlinedFunction() argument
1374 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()) in emitTeamsOutlinedFunction()
1375 .emitGenericVarsEpilog(CGF); in emitTeamsOutlinedFunction()
1385 void CGOpenMPRuntimeGPU::emitGenericVarsProlog(CodeGenFunction &CGF, in emitGenericVarsProlog() argument
1392 CGBuilderTy &Bld = CGF.Builder; in emitGenericVarsProlog()
1394 const auto I = FunctionGlobalizedDecls.find(CGF.CurFn); in emitGenericVarsProlog()
1407 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType()); in emitGenericVarsProlog()
1408 ParValue = CGF.EmitLoadOfScalar(ParLVal, Loc); in emitGenericVarsProlog()
1412 llvm::Value *AllocArgs[] = {CGF.getTypeSize(VD->getType())}; in emitGenericVarsProlog()
1414 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitGenericVarsProlog()
1423 llvm::PointerType *VarPtrTy = CGF.ConvertTypeForMem(VarTy)->getPointerTo(); in emitGenericVarsProlog()
1426 LValue VarAddr = CGF.MakeNaturalAlignAddrLValue(CastedVoidPtr, VarTy); in emitGenericVarsProlog()
1427 Rec.second.PrivateAddr = VarAddr.getAddress(CGF); in emitGenericVarsProlog()
1432 CGF.EmitStoreOfScalar(ParValue, VarAddr); in emitGenericVarsProlog()
1433 I->getSecond().MappedParams->setVarAddr(CGF, VD, VarAddr.getAddress(CGF)); in emitGenericVarsProlog()
1435 if (auto *DI = CGF.getDebugInfo()) in emitGenericVarsProlog()
1441 llvm::Value *Size = CGF.getTypeSize(VD->getType()); in emitGenericVarsProlog()
1444 Size, llvm::ConstantInt::get(CGF.SizeTy, Align.getQuantity() - 1)); in emitGenericVarsProlog()
1446 llvm::ConstantInt::get(CGF.SizeTy, Align.getQuantity()); in emitGenericVarsProlog()
1452 llvm::Value *AllocArgs[] = {CGF.getTypeSize(VD->getType())}; in emitGenericVarsProlog()
1454 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitGenericVarsProlog()
1463 {VoidPtr, CGF.getTypeSize(VD->getType())})); in emitGenericVarsProlog()
1464 LValue Base = CGF.MakeAddrLValue(VoidPtr, VD->getType(), in emitGenericVarsProlog()
1467 I->getSecond().MappedParams->setVarAddr(CGF, cast<VarDecl>(VD), in emitGenericVarsProlog()
1468 Base.getAddress(CGF)); in emitGenericVarsProlog()
1470 I->getSecond().MappedParams->apply(CGF); in emitGenericVarsProlog()
1473 void CGOpenMPRuntimeGPU::emitGenericVarsEpilog(CodeGenFunction &CGF, in emitGenericVarsEpilog() argument
1479 const auto I = FunctionGlobalizedDecls.find(CGF.CurFn); in emitGenericVarsEpilog()
1484 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitGenericVarsEpilog()
1491 I->getSecond().MappedParams->restore(CGF); in emitGenericVarsEpilog()
1494 CGF.getTypeSize(VD->getType())}; in emitGenericVarsEpilog()
1495 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitGenericVarsEpilog()
1502 void CGOpenMPRuntimeGPU::emitTeamsCall(CodeGenFunction &CGF, in emitTeamsCall() argument
1507 if (!CGF.HaveInsertPoint()) in emitTeamsCall()
1510 Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, in emitTeamsCall()
1512 CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddr); in emitTeamsCall()
1514 OutlinedFnArgs.push_back(emitThreadIDAddress(CGF, Loc).getPointer()); in emitTeamsCall()
1517 emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs); in emitTeamsCall()
1520 void CGOpenMPRuntimeGPU::emitParallelCall(CodeGenFunction &CGF, in emitParallelCall() argument
1526 if (!CGF.HaveInsertPoint()) in emitParallelCall()
1530 NumThreads](CodeGenFunction &CGF, in emitParallelCall()
1532 CGBuilderTy &Bld = CGF.Builder; in emitParallelCall()
1543 CodeGenFunction::OMPPrivateScope PrivateArgScope(CGF); in emitParallelCall()
1545 Address CapturedVarsAddrs = CGF.CreateDefaultAlignTempAlloca( in emitParallelCall()
1551 ASTContext &Ctx = CGF.getContext(); in emitParallelCall()
1557 PtrV = Bld.CreateIntToPtr(V, CGF.VoidPtrTy); in emitParallelCall()
1559 PtrV = Bld.CreatePointerBitCastOrAddrSpaceCast(V, CGF.VoidPtrTy); in emitParallelCall()
1560 CGF.EmitStoreOfScalar(PtrV, Dst, /*Volatile=*/false, in emitParallelCall()
1568 IfCondVal = Bld.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.Int32Ty, in emitParallelCall()
1571 IfCondVal = llvm::ConstantInt::get(CGF.Int32Ty, 1); in emitParallelCall()
1574 NumThreadsVal = llvm::ConstantInt::get(CGF.Int32Ty, -1); in emitParallelCall()
1576 NumThreadsVal = Bld.CreateZExtOrTrunc(NumThreadsVal, CGF.Int32Ty), in emitParallelCall()
1579 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitParallelCall()
1582 getThreadID(CGF, Loc), in emitParallelCall()
1585 llvm::ConstantInt::get(CGF.Int32Ty, -1), in emitParallelCall()
1589 CGF.VoidPtrPtrTy), in emitParallelCall()
1591 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitParallelCall()
1597 RCG(CGF); in emitParallelCall()
1600 void CGOpenMPRuntimeGPU::syncCTAThreads(CodeGenFunction &CGF) { in syncCTAThreads() argument
1602 if (!CGF.HaveInsertPoint()) in syncCTAThreads()
1609 llvm::ConstantInt::get(CGF.Int32Ty, /*V=*/0, /*isSigned=*/true)}; in syncCTAThreads()
1610 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in syncCTAThreads()
1615 void CGOpenMPRuntimeGPU::emitBarrierCall(CodeGenFunction &CGF, in emitBarrierCall() argument
1620 if (!CGF.HaveInsertPoint()) in emitBarrierCall()
1624 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags), in emitBarrierCall()
1625 getThreadID(CGF, Loc)}; in emitBarrierCall()
1627 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitBarrierCall()
1633 CodeGenFunction &CGF, StringRef CriticalName, in emitCriticalRegion() argument
1636 llvm::BasicBlock *LoopBB = CGF.createBasicBlock("omp.critical.loop"); in emitCriticalRegion()
1637 llvm::BasicBlock *TestBB = CGF.createBasicBlock("omp.critical.test"); in emitCriticalRegion()
1638 llvm::BasicBlock *SyncBB = CGF.createBasicBlock("omp.critical.sync"); in emitCriticalRegion()
1639 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.critical.body"); in emitCriticalRegion()
1640 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.critical.exit"); in emitCriticalRegion()
1642 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitCriticalRegion()
1645 llvm::Value *Mask = CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitCriticalRegion()
1648 llvm::Value *ThreadID = RT.getGPUThreadID(CGF); in emitCriticalRegion()
1651 llvm::Value *TeamWidth = RT.getGPUNumThreads(CGF); in emitCriticalRegion()
1655 CGF.getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/0); in emitCriticalRegion()
1656 Address Counter = CGF.CreateMemTemp(Int32Ty, "critical_counter"); in emitCriticalRegion()
1657 LValue CounterLVal = CGF.MakeAddrLValue(Counter, Int32Ty); in emitCriticalRegion()
1658 CGF.EmitStoreOfScalar(llvm::Constant::getNullValue(CGM.Int32Ty), CounterLVal, in emitCriticalRegion()
1662 CGF.EmitBlock(LoopBB); in emitCriticalRegion()
1663 llvm::Value *CounterVal = CGF.EmitLoadOfScalar(CounterLVal, Loc); in emitCriticalRegion()
1664 llvm::Value *CmpLoopBound = CGF.Builder.CreateICmpSLT(CounterVal, TeamWidth); in emitCriticalRegion()
1665 CGF.Builder.CreateCondBr(CmpLoopBound, TestBB, ExitBB); in emitCriticalRegion()
1669 CGF.EmitBlock(TestBB); in emitCriticalRegion()
1670 CounterVal = CGF.EmitLoadOfScalar(CounterLVal, Loc); in emitCriticalRegion()
1672 CGF.Builder.CreateICmpEQ(ThreadID, CounterVal); in emitCriticalRegion()
1673 CGF.Builder.CreateCondBr(CmpThreadToCounter, BodyBB, SyncBB); in emitCriticalRegion()
1676 CGF.EmitBlock(BodyBB); in emitCriticalRegion()
1679 CGOpenMPRuntime::emitCriticalRegion(CGF, CriticalName, CriticalOpGen, Loc, in emitCriticalRegion()
1686 CGF.EmitBlock(SyncBB); in emitCriticalRegion()
1688 (void)CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitCriticalRegion()
1693 CGF.Builder.CreateNSWAdd(CounterVal, CGF.Builder.getInt32(1)); in emitCriticalRegion()
1694 CGF.EmitStoreOfScalar(IncCounterVal, CounterLVal); in emitCriticalRegion()
1695 CGF.EmitBranch(LoopBB); in emitCriticalRegion()
1698 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in emitCriticalRegion()
1702 static llvm::Value *castValueToType(CodeGenFunction &CGF, llvm::Value *Val, in castValueToType() argument
1705 assert(!CGF.getContext().getTypeSizeInChars(CastTy).isZero() && in castValueToType()
1707 assert(!CGF.getContext().getTypeSizeInChars(ValTy).isZero() && in castValueToType()
1709 llvm::Type *LLVMCastTy = CGF.ConvertTypeForMem(CastTy); in castValueToType()
1712 if (CGF.getContext().getTypeSizeInChars(ValTy) == in castValueToType()
1713 CGF.getContext().getTypeSizeInChars(CastTy)) in castValueToType()
1714 return CGF.Builder.CreateBitCast(Val, LLVMCastTy); in castValueToType()
1716 return CGF.Builder.CreateIntCast(Val, LLVMCastTy, in castValueToType()
1718 Address CastItem = CGF.CreateMemTemp(CastTy); in castValueToType()
1719 Address ValCastItem = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in castValueToType()
1722 CGF.EmitStoreOfScalar(Val, ValCastItem, /*Volatile=*/false, ValTy, in castValueToType()
1725 return CGF.EmitLoadOfScalar(CastItem, /*Volatile=*/false, CastTy, Loc, in castValueToType()
1732 static llvm::Value *createRuntimeShuffleFunction(CodeGenFunction &CGF, in createRuntimeShuffleFunction() argument
1737 CodeGenModule &CGM = CGF.CGM; in createRuntimeShuffleFunction()
1738 CGBuilderTy &Bld = CGF.Builder; in createRuntimeShuffleFunction()
1743 CharUnits Size = CGF.getContext().getTypeSizeInChars(ElemType); in createRuntimeShuffleFunction()
1752 QualType CastTy = CGF.getContext().getIntTypeForBitwidth( in createRuntimeShuffleFunction()
1754 llvm::Value *ElemCast = castValueToType(CGF, Elem, ElemType, CastTy, Loc); in createRuntimeShuffleFunction()
1756 Bld.CreateIntCast(RT.getGPUWarpSize(CGF), CGM.Int16Ty, /*isSigned=*/true); in createRuntimeShuffleFunction()
1758 llvm::Value *ShuffledVal = CGF.EmitRuntimeCall( in createRuntimeShuffleFunction()
1762 return castValueToType(CGF, ShuffledVal, CastTy, ElemType, Loc); in createRuntimeShuffleFunction()
1765 static void shuffleAndStore(CodeGenFunction &CGF, Address SrcAddr, in shuffleAndStore() argument
1768 CGBuilderTy &Bld = CGF.Builder; in shuffleAndStore()
1770 CharUnits Size = CGF.getContext().getTypeSizeInChars(ElemType); in shuffleAndStore()
1784 Bld.CreateConstGEP(SrcAddr, 1), CGF.VoidPtrTy, CGF.Int8Ty); in shuffleAndStore()
1788 QualType IntType = CGF.getContext().getIntTypeForBitwidth( in shuffleAndStore()
1789 CGF.getContext().toBits(CharUnits::fromQuantity(IntSize)), in shuffleAndStore()
1791 llvm::Type *IntTy = CGF.ConvertTypeForMem(IntType); in shuffleAndStore()
1797 llvm::BasicBlock *PreCondBB = CGF.createBasicBlock(".shuffle.pre_cond"); in shuffleAndStore()
1798 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".shuffle.then"); in shuffleAndStore()
1799 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".shuffle.exit"); in shuffleAndStore()
1801 CGF.EmitBlock(PreCondBB); in shuffleAndStore()
1812 CGF.Int8Ty, PtrEnd.getPointer(), in shuffleAndStore()
1814 CGF.VoidPtrTy)); in shuffleAndStore()
1817 CGF.EmitBlock(ThenBB); in shuffleAndStore()
1819 CGF, in shuffleAndStore()
1820 CGF.EmitLoadOfScalar(Ptr, /*Volatile=*/false, IntType, Loc, in shuffleAndStore()
1824 CGF.EmitStoreOfScalar(Res, ElemPtr, /*Volatile=*/false, IntType, in shuffleAndStore()
1831 CGF.EmitBranch(PreCondBB); in shuffleAndStore()
1832 CGF.EmitBlock(ExitBB); in shuffleAndStore()
1835 CGF, in shuffleAndStore()
1836 CGF.EmitLoadOfScalar(Ptr, /*Volatile=*/false, IntType, Loc, in shuffleAndStore()
1840 CGF.EmitStoreOfScalar(Res, ElemPtr, /*Volatile=*/false, IntType, in shuffleAndStore()
1874 CopyAction Action, CodeGenFunction &CGF, QualType ReductionArrayTy, in emitReductionListCopy() argument
1878 CodeGenModule &CGM = CGF.CGM;
1880 CGBuilderTy &Bld = CGF.Builder;
1904 llvm::Type *PrivateLlvmPtrType = CGF.ConvertType(PrivatePtrType);
1911 CGF.EmitLoadOfPointer(CGF.Builder.CreateElementBitCast(
1919 CGF.CreateMemTemp(Private->getType(), ".omp.reduction.element");
1928 CGF.EmitLoadOfPointer(CGF.Builder.CreateElementBitCast(
1936 CGF.EmitLoadOfPointer(CGF.Builder.CreateElementBitCast(
1945 CGF.EmitLoadOfPointer(CGF.Builder.CreateElementBitCast(
1951 llvm::Value *ElementSizeInChars = CGF.getTypeSize(Private->getType());
1957 Bld.CreateIntToPtr(ScratchPadElemAbsolutePtrVal, CGF.VoidPtrTy);
1958 DestElementAddr = Address(ScratchPadElemAbsolutePtrVal, CGF.Int8Ty,
1966 llvm::Value *ElementSizeInChars = CGF.getTypeSize(Private->getType());
1972 Bld.CreateIntToPtr(ScratchPadElemAbsolutePtrVal, CGF.VoidPtrTy);
1973 SrcElementAddr = Address(ScratchPadElemAbsolutePtrVal, CGF.Int8Ty,
1981 CGF.CreateMemTemp(Private->getType(), ".omp.reduction.element");
1990 SrcElementAddr, CGF.ConvertTypeForMem(Private->getType()));
1997 shuffleAndStore(CGF, SrcElementAddr, DestElementAddr, Private->getType(),
2000 switch (CGF.getEvaluationKind(Private->getType())) {
2002 llvm::Value *Elem = CGF.EmitLoadOfScalar(
2007 CGF.EmitStoreOfScalar(
2013 CodeGenFunction::ComplexPairTy Elem = CGF.EmitLoadOfComplex(
2014 CGF.MakeAddrLValue(SrcElementAddr, Private->getType()),
2016 CGF.EmitStoreOfComplex(
2017 Elem, CGF.MakeAddrLValue(DestElementAddr, Private->getType()),
2022 CGF.EmitAggregateCopy(
2023 CGF.MakeAddrLValue(DestElementAddr, Private->getType()),
2024 CGF.MakeAddrLValue(SrcElementAddr, Private->getType()),
2036 CGF.EmitStoreOfScalar(Bld.CreatePointerBitCastOrAddrSpaceCast(
2037 DestElementAddr.getPointer(), CGF.VoidPtrTy),
2050 llvm::Value *ElementSizeInChars = CGF.getTypeSize(Private->getType());
2069 Address(ScratchpadBasePtr, CGF.VoidPtrTy, CGF.getPointerAlign());
2072 Address(ScratchpadBasePtr, CGF.VoidPtrTy, CGF.getPointerAlign());
2119 CodeGenFunction CGF(CGM); in emitInterWarpCopyFunction() local
2120 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitInterWarpCopyFunction()
2122 CGBuilderTy &Bld = CGF.Builder; in emitInterWarpCopyFunction()
2135 unsigned WarpSize = CGF.getTarget().getGridValue().GV_Warp_Size; in emitInterWarpCopyFunction()
2147 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitInterWarpCopyFunction()
2149 llvm::Value *ThreadID = RT.getGPUThreadID(CGF); in emitInterWarpCopyFunction()
2151 llvm::Value *LaneID = getNVPTXLaneID(CGF); in emitInterWarpCopyFunction()
2153 llvm::Value *WarpID = getNVPTXWarpID(CGF); in emitInterWarpCopyFunction()
2155 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitInterWarpCopyFunction()
2156 llvm::Type *ElemTy = CGF.ConvertTypeForMem(ReductionArrayTy); in emitInterWarpCopyFunction()
2159 CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
2163 ElemTy, CGF.getPointerAlign()); in emitInterWarpCopyFunction()
2181 llvm::Type *CopyType = CGF.ConvertTypeForMem(CType); in emitInterWarpCopyFunction()
2188 CntAddr = CGF.CreateMemTemp(C.IntTy, ".cnt.addr"); in emitInterWarpCopyFunction()
2189 CGF.EmitStoreOfScalar(llvm::Constant::getNullValue(CGM.IntTy), CntAddr, in emitInterWarpCopyFunction()
2191 PrecondBB = CGF.createBasicBlock("precond"); in emitInterWarpCopyFunction()
2192 ExitBB = CGF.createBasicBlock("exit"); in emitInterWarpCopyFunction()
2193 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("body"); in emitInterWarpCopyFunction()
2195 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitInterWarpCopyFunction()
2196 CGF.EmitBlock(PrecondBB); in emitInterWarpCopyFunction()
2197 Cnt = CGF.EmitLoadOfScalar(CntAddr, /*Volatile=*/false, C.IntTy, Loc); in emitInterWarpCopyFunction()
2201 CGF.EmitBlock(BodyBB); in emitInterWarpCopyFunction()
2204 CGM.getOpenMPRuntime().emitBarrierCall(CGF, Loc, OMPD_unknown, in emitInterWarpCopyFunction()
2207 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("then"); in emitInterWarpCopyFunction()
2208 llvm::BasicBlock *ElseBB = CGF.createBasicBlock("else"); in emitInterWarpCopyFunction()
2209 llvm::BasicBlock *MergeBB = CGF.createBasicBlock("ifcont"); in emitInterWarpCopyFunction()
2214 CGF.EmitBlock(ThenBB); in emitInterWarpCopyFunction()
2218 llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
2221 Address ElemPtr(ElemPtrPtr, CGF.Int8Ty, Align); in emitInterWarpCopyFunction()
2242 llvm::Value *Elem = CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
2246 CGF.EmitStoreOfScalar(Elem, MediumPtr, /*Volatile=*/true, CType, in emitInterWarpCopyFunction()
2252 CGF.EmitBlock(ElseBB); in emitInterWarpCopyFunction()
2255 CGF.EmitBlock(MergeBB); in emitInterWarpCopyFunction()
2258 CGM.getOpenMPRuntime().emitBarrierCall(CGF, Loc, OMPD_unknown, in emitInterWarpCopyFunction()
2265 llvm::BasicBlock *W0ThenBB = CGF.createBasicBlock("then"); in emitInterWarpCopyFunction()
2266 llvm::BasicBlock *W0ElseBB = CGF.createBasicBlock("else"); in emitInterWarpCopyFunction()
2267 llvm::BasicBlock *W0MergeBB = CGF.createBasicBlock("ifcont"); in emitInterWarpCopyFunction()
2269 Address AddrNumWarpsArg = CGF.GetAddrOfLocalVar(&NumWarpsArg); in emitInterWarpCopyFunction()
2270 llvm::Value *NumWarpsVal = CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
2278 CGF.EmitBlock(W0ThenBB); in emitInterWarpCopyFunction()
2294 llvm::Value *TargetElemPtrVal = CGF.EmitLoadOfScalar( in emitInterWarpCopyFunction()
2296 Address TargetElemPtr(TargetElemPtrVal, CGF.Int8Ty, Align); in emitInterWarpCopyFunction()
2303 CGF.EmitLoadOfScalar(SrcMediumPtr, /*Volatile=*/true, CType, Loc); in emitInterWarpCopyFunction()
2304 CGF.EmitStoreOfScalar(SrcMediumValue, TargetElemPtr, /*Volatile=*/false, in emitInterWarpCopyFunction()
2308 CGF.EmitBlock(W0ElseBB); in emitInterWarpCopyFunction()
2311 CGF.EmitBlock(W0MergeBB); in emitInterWarpCopyFunction()
2315 CGF.EmitStoreOfScalar(Cnt, CntAddr, /*Volatile=*/false, C.IntTy); in emitInterWarpCopyFunction()
2316 CGF.EmitBranch(PrecondBB); in emitInterWarpCopyFunction()
2317 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitInterWarpCopyFunction()
2318 CGF.EmitBlock(ExitBB); in emitInterWarpCopyFunction()
2325 CGF.FinishFunction(); in emitInterWarpCopyFunction()
2426 CodeGenFunction CGF(CGM); in emitShuffleAndReduceFunction() local
2427 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitShuffleAndReduceFunction()
2429 CGBuilderTy &Bld = CGF.Builder; in emitShuffleAndReduceFunction()
2431 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitShuffleAndReduceFunction()
2432 llvm::Type *ElemTy = CGF.ConvertTypeForMem(ReductionArrayTy); in emitShuffleAndReduceFunction()
2435 CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false, in emitShuffleAndReduceFunction()
2438 ElemTy, CGF.getPointerAlign()); in emitShuffleAndReduceFunction()
2440 Address AddrLaneIDArg = CGF.GetAddrOfLocalVar(&LaneIDArg); in emitShuffleAndReduceFunction()
2441 llvm::Value *LaneIDArgVal = CGF.EmitLoadOfScalar( in emitShuffleAndReduceFunction()
2444 Address AddrRemoteLaneOffsetArg = CGF.GetAddrOfLocalVar(&RemoteLaneOffsetArg); in emitShuffleAndReduceFunction()
2445 llvm::Value *RemoteLaneOffsetArgVal = CGF.EmitLoadOfScalar( in emitShuffleAndReduceFunction()
2448 Address AddrAlgoVerArg = CGF.GetAddrOfLocalVar(&AlgoVerArg); in emitShuffleAndReduceFunction()
2449 llvm::Value *AlgoVerArgVal = CGF.EmitLoadOfScalar( in emitShuffleAndReduceFunction()
2455 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.remote_reduce_list"); in emitShuffleAndReduceFunction()
2460 emitReductionListCopy(RemoteLaneToThread, CGF, ReductionArrayTy, Privates, in emitShuffleAndReduceFunction()
2502 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("then"); in emitShuffleAndReduceFunction()
2503 llvm::BasicBlock *ElseBB = CGF.createBasicBlock("else"); in emitShuffleAndReduceFunction()
2504 llvm::BasicBlock *MergeBB = CGF.createBasicBlock("ifcont"); in emitShuffleAndReduceFunction()
2507 CGF.EmitBlock(ThenBB); in emitShuffleAndReduceFunction()
2510 LocalReduceList.getPointer(), CGF.VoidPtrTy); in emitShuffleAndReduceFunction()
2512 RemoteReduceList.getPointer(), CGF.VoidPtrTy); in emitShuffleAndReduceFunction()
2514 CGF, Loc, ReduceFn, {LocalReduceListPtr, RemoteReduceListPtr}); in emitShuffleAndReduceFunction()
2517 CGF.EmitBlock(ElseBB); in emitShuffleAndReduceFunction()
2520 CGF.EmitBlock(MergeBB); in emitShuffleAndReduceFunction()
2528 llvm::BasicBlock *CpyThenBB = CGF.createBasicBlock("then"); in emitShuffleAndReduceFunction()
2529 llvm::BasicBlock *CpyElseBB = CGF.createBasicBlock("else"); in emitShuffleAndReduceFunction()
2530 llvm::BasicBlock *CpyMergeBB = CGF.createBasicBlock("ifcont"); in emitShuffleAndReduceFunction()
2533 CGF.EmitBlock(CpyThenBB); in emitShuffleAndReduceFunction()
2534 emitReductionListCopy(ThreadCopy, CGF, ReductionArrayTy, Privates, in emitShuffleAndReduceFunction()
2538 CGF.EmitBlock(CpyElseBB); in emitShuffleAndReduceFunction()
2541 CGF.EmitBlock(CpyMergeBB); in emitShuffleAndReduceFunction()
2543 CGF.FinishFunction(); in emitShuffleAndReduceFunction()
2582 CodeGenFunction CGF(CGM); in emitListToGlobalCopyFunction() local
2583 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitListToGlobalCopyFunction()
2585 CGBuilderTy &Bld = CGF.Builder; in emitListToGlobalCopyFunction()
2587 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitListToGlobalCopyFunction()
2588 Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg); in emitListToGlobalCopyFunction()
2589 llvm::Type *ElemTy = CGF.ConvertTypeForMem(ReductionArrayTy); in emitListToGlobalCopyFunction()
2592 CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false, in emitListToGlobalCopyFunction()
2595 ElemTy, CGF.getPointerAlign()); in emitListToGlobalCopyFunction()
2600 CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc), in emitListToGlobalCopyFunction()
2602 llvm::Value *Idxs[] = {llvm::ConstantInt::getNullValue(CGF.Int32Ty), in emitListToGlobalCopyFunction()
2603 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), in emitListToGlobalCopyFunction()
2610 llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar( in emitListToGlobalCopyFunction()
2613 ElemTy = CGF.ConvertTypeForMem(Private->getType()); in emitListToGlobalCopyFunction()
2621 LValue GlobLVal = CGF.EmitLValueForField( in emitListToGlobalCopyFunction()
2622 CGF.MakeNaturalAlignAddrLValue(BufferArrPtr, StaticTy), FD); in emitListToGlobalCopyFunction()
2623 Address GlobAddr = GlobLVal.getAddress(CGF); in emitListToGlobalCopyFunction()
2627 CGF.ConvertTypeForMem(Private->getType()), in emitListToGlobalCopyFunction()
2629 switch (CGF.getEvaluationKind(Private->getType())) { in emitListToGlobalCopyFunction()
2631 llvm::Value *V = CGF.EmitLoadOfScalar( in emitListToGlobalCopyFunction()
2634 CGF.EmitStoreOfScalar(V, GlobLVal); in emitListToGlobalCopyFunction()
2638 CodeGenFunction::ComplexPairTy V = CGF.EmitLoadOfComplex( in emitListToGlobalCopyFunction()
2639 CGF.MakeAddrLValue(ElemPtr, Private->getType()), Loc); in emitListToGlobalCopyFunction()
2640 CGF.EmitStoreOfComplex(V, GlobLVal, /*isInit=*/false); in emitListToGlobalCopyFunction()
2644 CGF.EmitAggregateCopy(GlobLVal, in emitListToGlobalCopyFunction()
2645 CGF.MakeAddrLValue(ElemPtr, Private->getType()), in emitListToGlobalCopyFunction()
2652 CGF.FinishFunction(); in emitListToGlobalCopyFunction()
2695 CodeGenFunction CGF(CGM); in emitListToGlobalReduceFunction() local
2696 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitListToGlobalReduceFunction()
2698 CGBuilderTy &Bld = CGF.Builder; in emitListToGlobalReduceFunction()
2700 Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg); in emitListToGlobalReduceFunction()
2705 CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc), in emitListToGlobalReduceFunction()
2711 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); in emitListToGlobalReduceFunction()
2713 llvm::Value *Idxs[] = {llvm::ConstantInt::getNullValue(CGF.Int32Ty), in emitListToGlobalReduceFunction()
2714 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), in emitListToGlobalReduceFunction()
2719 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitListToGlobalReduceFunction()
2723 LValue GlobLVal = CGF.EmitLValueForField( in emitListToGlobalReduceFunction()
2724 CGF.MakeNaturalAlignAddrLValue(BufferArrPtr, StaticTy), FD); in emitListToGlobalReduceFunction()
2725 Address GlobAddr = GlobLVal.getAddress(CGF); in emitListToGlobalReduceFunction()
2728 llvm::Value *Ptr = CGF.EmitCastToVoidPtr(BufferPtr); in emitListToGlobalReduceFunction()
2729 CGF.EmitStoreOfScalar(Ptr, Elem, /*Volatile=*/false, C.VoidPtrTy); in emitListToGlobalReduceFunction()
2733 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitListToGlobalReduceFunction()
2734 llvm::Value *Size = CGF.Builder.CreateIntCast( in emitListToGlobalReduceFunction()
2735 CGF.getVLASize( in emitListToGlobalReduceFunction()
2736 CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) in emitListToGlobalReduceFunction()
2738 CGF.SizeTy, /*isSigned=*/false); in emitListToGlobalReduceFunction()
2739 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), in emitListToGlobalReduceFunction()
2746 CGF.EmitCastToVoidPtr(ReductionList.getPointer()); in emitListToGlobalReduceFunction()
2747 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitListToGlobalReduceFunction()
2748 llvm::Value *ReducedPtr = CGF.EmitLoadOfScalar( in emitListToGlobalReduceFunction()
2751 CGF, Loc, ReduceFn, {GlobalReduceList, ReducedPtr}); in emitListToGlobalReduceFunction()
2752 CGF.FinishFunction(); in emitListToGlobalReduceFunction()
2791 CodeGenFunction CGF(CGM); in emitGlobalToListCopyFunction() local
2792 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitGlobalToListCopyFunction()
2794 CGBuilderTy &Bld = CGF.Builder; in emitGlobalToListCopyFunction()
2796 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitGlobalToListCopyFunction()
2797 Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg); in emitGlobalToListCopyFunction()
2798 llvm::Type *ElemTy = CGF.ConvertTypeForMem(ReductionArrayTy); in emitGlobalToListCopyFunction()
2801 CGF.EmitLoadOfScalar(AddrReduceListArg, /*Volatile=*/false, in emitGlobalToListCopyFunction()
2804 ElemTy, CGF.getPointerAlign()); in emitGlobalToListCopyFunction()
2809 CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc), in emitGlobalToListCopyFunction()
2812 llvm::Value *Idxs[] = {llvm::ConstantInt::getNullValue(CGF.Int32Ty), in emitGlobalToListCopyFunction()
2813 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), in emitGlobalToListCopyFunction()
2820 llvm::Value *ElemPtrPtr = CGF.EmitLoadOfScalar( in emitGlobalToListCopyFunction()
2823 ElemTy = CGF.ConvertTypeForMem(Private->getType()); in emitGlobalToListCopyFunction()
2831 LValue GlobLVal = CGF.EmitLValueForField( in emitGlobalToListCopyFunction()
2832 CGF.MakeNaturalAlignAddrLValue(BufferArrPtr, StaticTy), FD); in emitGlobalToListCopyFunction()
2833 Address GlobAddr = GlobLVal.getAddress(CGF); in emitGlobalToListCopyFunction()
2837 CGF.ConvertTypeForMem(Private->getType()), in emitGlobalToListCopyFunction()
2839 switch (CGF.getEvaluationKind(Private->getType())) { in emitGlobalToListCopyFunction()
2841 llvm::Value *V = CGF.EmitLoadOfScalar(GlobLVal, Loc); in emitGlobalToListCopyFunction()
2842 CGF.EmitStoreOfScalar(V, ElemPtr, /*Volatile=*/false, Private->getType(), in emitGlobalToListCopyFunction()
2848 CodeGenFunction::ComplexPairTy V = CGF.EmitLoadOfComplex(GlobLVal, Loc); in emitGlobalToListCopyFunction()
2849 CGF.EmitStoreOfComplex(V, CGF.MakeAddrLValue(ElemPtr, Private->getType()), in emitGlobalToListCopyFunction()
2854 CGF.EmitAggregateCopy(CGF.MakeAddrLValue(ElemPtr, Private->getType()), in emitGlobalToListCopyFunction()
2862 CGF.FinishFunction(); in emitGlobalToListCopyFunction()
2905 CodeGenFunction CGF(CGM); in emitGlobalToListReduceFunction() local
2906 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitGlobalToListReduceFunction()
2908 CGBuilderTy &Bld = CGF.Builder; in emitGlobalToListReduceFunction()
2910 Address AddrBufferArg = CGF.GetAddrOfLocalVar(&BufferArg); in emitGlobalToListReduceFunction()
2915 CGF.EmitLoadOfScalar(AddrBufferArg, /*Volatile=*/false, C.VoidPtrTy, Loc), in emitGlobalToListReduceFunction()
2921 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); in emitGlobalToListReduceFunction()
2923 llvm::Value *Idxs[] = {llvm::ConstantInt::getNullValue(CGF.Int32Ty), in emitGlobalToListReduceFunction()
2924 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), in emitGlobalToListReduceFunction()
2929 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitGlobalToListReduceFunction()
2933 LValue GlobLVal = CGF.EmitLValueForField( in emitGlobalToListReduceFunction()
2934 CGF.MakeNaturalAlignAddrLValue(BufferArrPtr, StaticTy), FD); in emitGlobalToListReduceFunction()
2935 Address GlobAddr = GlobLVal.getAddress(CGF); in emitGlobalToListReduceFunction()
2938 llvm::Value *Ptr = CGF.EmitCastToVoidPtr(BufferPtr); in emitGlobalToListReduceFunction()
2939 CGF.EmitStoreOfScalar(Ptr, Elem, /*Volatile=*/false, C.VoidPtrTy); in emitGlobalToListReduceFunction()
2943 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitGlobalToListReduceFunction()
2944 llvm::Value *Size = CGF.Builder.CreateIntCast( in emitGlobalToListReduceFunction()
2945 CGF.getVLASize( in emitGlobalToListReduceFunction()
2946 CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) in emitGlobalToListReduceFunction()
2948 CGF.SizeTy, /*isSigned=*/false); in emitGlobalToListReduceFunction()
2949 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), in emitGlobalToListReduceFunction()
2956 CGF.EmitCastToVoidPtr(ReductionList.getPointer()); in emitGlobalToListReduceFunction()
2957 Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); in emitGlobalToListReduceFunction()
2958 llvm::Value *ReducedPtr = CGF.EmitLoadOfScalar( in emitGlobalToListReduceFunction()
2961 CGF, Loc, ReduceFn, {ReducedPtr, GlobalReduceList}); in emitGlobalToListReduceFunction()
2962 CGF.FinishFunction(); in emitGlobalToListReduceFunction()
3209 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates, in emitReduction() argument
3212 if (!CGF.HaveInsertPoint()) in emitReduction()
3223 CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs, in emitReduction()
3235 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitReduction()
3236 llvm::Value *ThreadId = getThreadID(CGF, Loc); in emitReduction()
3253 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); in emitReduction()
3257 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitReduction()
3258 CGF.Builder.CreateStore( in emitReduction()
3259 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReduction()
3260 CGF.EmitLValue(RHSExprs[I]).getPointer(CGF), CGF.VoidPtrTy), in emitReduction()
3265 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitReduction()
3266 llvm::Value *Size = CGF.Builder.CreateIntCast( in emitReduction()
3267 CGF.getVLASize( in emitReduction()
3268 CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) in emitReduction()
3270 CGF.SizeTy, /*isSigned=*/false); in emitReduction()
3271 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), in emitReduction()
3276 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReduction()
3277 ReductionList.getPointer(), CGF.VoidPtrTy); in emitReduction()
3279 emitReductionFunction(Loc, CGF.ConvertTypeForMem(ReductionArrayTy), in emitReduction()
3281 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy); in emitReduction()
3290 CGF.Builder.getInt32(RHSExprs.size()), in emitReduction()
3296 Res = CGF.EmitRuntimeCall( in emitReduction()
3319 llvm::Value *GlobalBufferPtr = CGF.EmitLoadOfScalar( in emitReduction()
3320 Address(KernelTeamsReductionPtr, CGF.VoidPtrTy, CGM.getPointerAlign()), in emitReduction()
3337 CGF.Builder.getInt32(C.getLangOpts().OpenMPCUDAReductionBufNum), in emitReduction()
3346 Res = CGF.EmitRuntimeCall( in emitReduction()
3353 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.reduction.done"); in emitReduction()
3354 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.then"); in emitReduction()
3355 llvm::Value *Cond = CGF.Builder.CreateICmpEQ( in emitReduction()
3357 CGF.Builder.CreateCondBr(Cond, ThenBB, ExitBB); in emitReduction()
3363 CGF.EmitBlock(ThenBB); in emitReduction()
3367 this](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
3372 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), in emitReduction()
3387 RCG(CGF); in emitReduction()
3389 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitReduction()
3390 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in emitReduction()
3426 CGOpenMPRuntimeGPU::getParameterAddress(CodeGenFunction &CGF, in getParameterAddress() argument
3432 Address LocalAddr = CGF.GetAddrOfLocalVar(TargetParam); in getParameterAddress()
3438 CGF.getContext().getTargetAddressSpace(NativePointeeTy); in getParameterAddress()
3440 llvm::Value *TargetAddr = CGF.EmitLoadOfScalar( in getParameterAddress()
3443 TargetAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getParameterAddress()
3447 TargetAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getParameterAddress()
3451 Address NativeParamAddr = CGF.CreateMemTemp(NativeParamType); in getParameterAddress()
3452 CGF.EmitStoreOfScalar(TargetAddr, NativeParamAddr, /*Volatile=*/false, in getParameterAddress()
3458 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, in emitOutlinedFunctionCall() argument
3474 llvm::Value *TargetArg = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitOutlinedFunctionCall()
3478 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TargetArg, TargetType)); in emitOutlinedFunctionCall()
3480 CGOpenMPRuntime::emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, TargetArgs); in emitOutlinedFunctionCall()
3526 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true); in createParallelDataSharingWrapper() local
3527 CGF.StartFunction(GlobalDecl(), Ctx.VoidTy, Fn, CGFI, WrapperArgs, in createParallelDataSharingWrapper()
3533 Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, in createParallelDataSharingWrapper()
3535 CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddr); in createParallelDataSharingWrapper()
3539 Args.emplace_back(CGF.GetAddrOfLocalVar(&WrapperArg).getPointer()); in createParallelDataSharingWrapper()
3542 CGBuilderTy &Bld = CGF.Builder; in createParallelDataSharingWrapper()
3548 CGF.CreateDefaultAlignTempAlloca(CGF.VoidPtrPtrTy, "global_args"); in createParallelDataSharingWrapper()
3551 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in createParallelDataSharingWrapper()
3560 SharedArgListAddress = CGF.EmitLoadOfPointer( in createParallelDataSharingWrapper()
3561 GlobalArgs, CGF.getContext() in createParallelDataSharingWrapper()
3562 .getPointerType(CGF.getContext().VoidPtrTy) in createParallelDataSharingWrapper()
3569 Src, CGF.SizeTy->getPointerTo(), CGF.SizeTy); in createParallelDataSharingWrapper()
3570 llvm::Value *LB = CGF.EmitLoadOfScalar( in createParallelDataSharingWrapper()
3573 CGF.getContext().getPointerType(CGF.getContext().getSizeType()), in createParallelDataSharingWrapper()
3579 Src, CGF.SizeTy->getPointerTo(), CGF.SizeTy); in createParallelDataSharingWrapper()
3580 llvm::Value *UB = CGF.EmitLoadOfScalar( in createParallelDataSharingWrapper()
3583 CGF.getContext().getPointerType(CGF.getContext().getSizeType()), in createParallelDataSharingWrapper()
3589 ASTContext &CGFContext = CGF.getContext(); in createParallelDataSharingWrapper()
3594 Src, CGF.ConvertTypeForMem(CGFContext.getPointerType(ElemTy)), in createParallelDataSharingWrapper()
3595 CGF.ConvertTypeForMem(ElemTy)); in createParallelDataSharingWrapper()
3596 llvm::Value *Arg = CGF.EmitLoadOfScalar(TypedAddress, in createParallelDataSharingWrapper()
3602 Arg = castValueToType(CGF, Arg, ElemTy, CGFContext.getUIntPtrType(), in createParallelDataSharingWrapper()
3609 emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedParallelFn, Args); in createParallelDataSharingWrapper()
3610 CGF.FinishFunction(); in createParallelDataSharingWrapper()
3614 void CGOpenMPRuntimeGPU::emitFunctionProlog(CodeGenFunction &CGF, in emitFunctionProlog() argument
3620 assert(FunctionGlobalizedDecls.count(CGF.CurFn) == 0 && in emitFunctionProlog()
3632 NeedToDelayGlobalization = CGF.CapturedStmtInfo->getKind() == CR_OpenMP; in emitFunctionProlog()
3639 CheckVarsEscapingDeclContext VarChecker(CGF, TeamAndReductions.second); in emitFunctionProlog()
3649 auto I = FunctionGlobalizedDecls.try_emplace(CGF.CurFn).first; in emitFunctionProlog()
3663 CheckVarsEscapingDeclContext VarChecker(CGF, llvm::None); in emitFunctionProlog()
3673 emitGenericVarsProlog(CGF, D->getBeginLoc(), /*WithSPMDCheck=*/true); in emitFunctionProlog()
3677 void Emit(CodeGenFunction &CGF, Flags flags) override { in emitFunctionProlog()
3678 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()) in emitFunctionProlog()
3679 .emitGenericVarsEpilog(CGF, /*WithSPMDCheck=*/true); in emitFunctionProlog()
3682 CGF.EHStack.pushCleanup<GlobalizationScope>(NormalAndEHCleanup); in emitFunctionProlog()
3686 Address CGOpenMPRuntimeGPU::getAddressOfLocalVariable(CodeGenFunction &CGF, in getAddressOfLocalVariable() argument
3714 llvm::Type *VarTy = CGF.ConvertTypeForMem(VD->getType()); in getAddressOfLocalVariable()
3724 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddressOfLocalVariable()
3734 auto I = FunctionGlobalizedDecls.find(CGF.CurFn); in getAddressOfLocalVariable()
3755 void CGOpenMPRuntimeGPU::functionFinished(CodeGenFunction &CGF) { in functionFinished() argument
3756 FunctionGlobalizedDecls.erase(CGF.CurFn); in functionFinished()
3757 CGOpenMPRuntime::functionFinished(CGF); in functionFinished()
3761 CodeGenFunction &CGF, const OMPLoopDirective &S, in getDefaultDistScheduleAndChunk() argument
3764 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in getDefaultDistScheduleAndChunk()
3767 Chunk = CGF.EmitScalarConversion( in getDefaultDistScheduleAndChunk()
3768 RT.getGPUNumThreads(CGF), in getDefaultDistScheduleAndChunk()
3769 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0), in getDefaultDistScheduleAndChunk()
3774 CGF, S, ScheduleKind, Chunk); in getDefaultDistScheduleAndChunk()
3778 CodeGenFunction &CGF, const OMPLoopDirective &S, in getDefaultScheduleAndChunk() argument
3784 ChunkExpr = IntegerLiteral::Create(CGF.getContext(), ChunkSize, in getDefaultScheduleAndChunk()
3785 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0), in getDefaultScheduleAndChunk()
3790 CodeGenFunction &CGF, const OMPExecutableDirective &D) const { in adjustTargetSpecificDataForLambdas() argument
3806 Address VDAddr = CGF.GetAddrOfLocalVar(VD); in adjustTargetSpecificDataForLambdas()
3809 VDLVal = CGF.EmitLoadOfReferenceLValue(VDAddr, VD->getType()); in adjustTargetSpecificDataForLambdas()
3811 VDLVal = CGF.MakeAddrLValue( in adjustTargetSpecificDataForLambdas()
3816 if (ThisCapture && CGF.CapturedStmtInfo->isCXXThisExprCaptured()) { in adjustTargetSpecificDataForLambdas()
3818 CGF.EmitLValueForFieldInitialization(VDLVal, ThisCapture); in adjustTargetSpecificDataForLambdas()
3819 llvm::Value *CXXThis = CGF.LoadCXXThis(); in adjustTargetSpecificDataForLambdas()
3820 CGF.EmitStoreOfScalar(CXXThis, ThisLVal); in adjustTargetSpecificDataForLambdas()
3830 LValue VarLVal = CGF.EmitLValueForFieldInitialization(VDLVal, It->second); in adjustTargetSpecificDataForLambdas()
3831 Address VDAddr = CGF.GetAddrOfLocalVar(VD); in adjustTargetSpecificDataForLambdas()
3833 VDAddr = CGF.EmitLoadOfReferenceLValue(VDAddr, in adjustTargetSpecificDataForLambdas()
3835 .getAddress(CGF); in adjustTargetSpecificDataForLambdas()
3836 CGF.EmitStoreOfScalar(VDAddr.getPointer(), VarLVal); in adjustTargetSpecificDataForLambdas()
4002 llvm::Value *CGOpenMPRuntimeGPU::getGPUNumThreads(CodeGenFunction &CGF) { in getGPUNumThreads() argument
4003 CGBuilderTy &Bld = CGF.Builder; in getGPUNumThreads()
4004 llvm::Module *M = &CGF.CGM.getModule(); in getGPUNumThreads()
4009 llvm::FunctionType::get(CGF.Int32Ty, llvm::None, false), in getGPUNumThreads()
4010 llvm::GlobalVariable::ExternalLinkage, LocSize, &CGF.CGM.getModule()); in getGPUNumThreads()
4015 llvm::Value *CGOpenMPRuntimeGPU::getGPUThreadID(CodeGenFunction &CGF) { in getGPUThreadID() argument
4017 return CGF.EmitRuntimeCall( in getGPUThreadID()
4023 llvm::Value *CGOpenMPRuntimeGPU::getGPUWarpSize(CodeGenFunction &CGF) { in getGPUWarpSize() argument
4025 return CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in getGPUWarpSize()