Lines Matching refs:CGF
72 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
76 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
145 void Enter(CodeGenFunction &CGF) override { in Enter() argument
148 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue( in Enter()
149 CGF.GetAddrOfLocalVar(PartIDVar), in Enter()
152 CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation()); in Enter()
153 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done."); in Enter()
154 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB); in Enter()
155 CGF.EmitBlock(DoneBB); in Enter()
156 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock); in Enter()
157 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp.")); in Enter()
158 UntiedSwitch->addCase(CGF.Builder.getInt32(0), in Enter()
159 CGF.Builder.GetInsertBlock()); in Enter()
160 emitUntiedSwitch(CGF); in Enter()
163 void emitUntiedSwitch(CodeGenFunction &CGF) const { in emitUntiedSwitch()
165 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue( in emitUntiedSwitch()
166 CGF.GetAddrOfLocalVar(PartIDVar), in emitUntiedSwitch()
168 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()), in emitUntiedSwitch()
170 UntiedCodeGen(CGF); in emitUntiedSwitch()
172 CGF.getJumpDestInCurrentScope(".untied.next."); in emitUntiedSwitch()
173 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock); in emitUntiedSwitch()
174 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp.")); in emitUntiedSwitch()
175 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()), in emitUntiedSwitch()
176 CGF.Builder.GetInsertBlock()); in emitUntiedSwitch()
177 CGF.EmitBranchThroughCleanup(CurPoint); in emitUntiedSwitch()
178 CGF.EmitBlock(CurPoint.getBlock()); in emitUntiedSwitch()
198 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
203 void emitUntiedSwitch(CodeGenFunction &CGF) override { in emitUntiedSwitch() argument
204 Action.emitUntiedSwitch(CGF); in emitUntiedSwitch()
271 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override { in getThreadIDVariableLValue() argument
273 return OuterRegionInfo->getThreadIDVariableLValue(CGF); in getThreadIDVariableLValue()
284 void emitUntiedSwitch(CodeGenFunction &CGF) override { in emitUntiedSwitch() argument
286 OuterRegionInfo->emitUntiedSwitch(CGF); in emitUntiedSwitch()
340 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS) in CGOpenMPInnerExprInfo() argument
341 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen, in CGOpenMPInnerExprInfo()
344 PrivScope(CGF) { in CGOpenMPInnerExprInfo()
356 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD), in CGOpenMPInnerExprInfo()
361 VD, [&CGF, &DRE]() { return CGF.EmitLValue(&DRE).getAddress(); }); in CGOpenMPInnerExprInfo()
374 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override { in EmitBody() argument
398 CodeGenFunction &CGF; member in __anon0274564a0111::InlinedOpenMPRegionRAII
408 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen, in InlinedOpenMPRegionRAII() argument
410 : CGF(CGF) { in InlinedOpenMPRegionRAII()
412 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo( in InlinedOpenMPRegionRAII()
413 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel); in InlinedOpenMPRegionRAII()
414 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields); in InlinedOpenMPRegionRAII()
415 LambdaThisCaptureField = CGF.LambdaThisCaptureField; in InlinedOpenMPRegionRAII()
416 CGF.LambdaThisCaptureField = nullptr; in InlinedOpenMPRegionRAII()
417 BlockInfo = CGF.BlockInfo; in InlinedOpenMPRegionRAII()
418 CGF.BlockInfo = nullptr; in InlinedOpenMPRegionRAII()
424 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI(); in ~InlinedOpenMPRegionRAII()
425 delete CGF.CapturedStmtInfo; in ~InlinedOpenMPRegionRAII()
426 CGF.CapturedStmtInfo = OldCSI; in ~InlinedOpenMPRegionRAII()
427 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields); in ~InlinedOpenMPRegionRAII()
428 CGF.LambdaThisCaptureField = LambdaThisCaptureField; in ~InlinedOpenMPRegionRAII()
429 CGF.BlockInfo = BlockInfo; in ~InlinedOpenMPRegionRAII()
729 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { in Emit() argument
730 if (!CGF.HaveInsertPoint()) in Emit()
732 Action->Exit(CGF); in Emit()
738 void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const { in operator ()()
739 CodeGenFunction::RunCleanupsScope Scope(CGF); in operator ()()
741 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction); in operator ()()
742 Callback(CodeGen, CGF, *PrePostAction); in operator ()()
745 Callback(CodeGen, CGF, Action); in operator ()()
762 static void emitInitWithReductionInitializer(CodeGenFunction &CGF, in emitInitWithReductionInitializer() argument
769 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD); in emitInitWithReductionInitializer()
778 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitInitWithReductionInitializer()
785 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func); in emitInitWithReductionInitializer()
786 CGF.EmitIgnoredExpr(InitOp); in emitInitWithReductionInitializer()
788 llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty); in emitInitWithReductionInitializer()
789 std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"}); in emitInitWithReductionInitializer()
791 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true, in emitInitWithReductionInitializer()
793 LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty); in emitInitWithReductionInitializer()
795 switch (CGF.getEvaluationKind(Ty)) { in emitInitWithReductionInitializer()
797 InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation()); in emitInitWithReductionInitializer()
801 RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation())); in emitInitWithReductionInitializer()
808 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal); in emitInitWithReductionInitializer()
809 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(), in emitInitWithReductionInitializer()
819 static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, in EmitOMPAggregateInit() argument
829 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr); in EmitOMPAggregateInit()
831 CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType()); in EmitOMPAggregateInit()
834 CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType()); in EmitOMPAggregateInit()
841 llvm::Value *DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements); in EmitOMPAggregateInit()
843 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body"); in EmitOMPAggregateInit()
844 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done"); in EmitOMPAggregateInit()
846 CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty"); in EmitOMPAggregateInit()
847 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); in EmitOMPAggregateInit()
850 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock(); in EmitOMPAggregateInit()
851 CGF.EmitBlock(BodyBB); in EmitOMPAggregateInit()
853 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy); in EmitOMPAggregateInit()
858 SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2, in EmitOMPAggregateInit()
865 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI( in EmitOMPAggregateInit()
874 CodeGenFunction::RunCleanupsScope InitScope(CGF); in EmitOMPAggregateInit()
876 emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent, in EmitOMPAggregateInit()
879 CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(), in EmitOMPAggregateInit()
885 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateInit()
887 SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateInit()
891 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateInit()
895 CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done"); in EmitOMPAggregateInit()
896 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB); in EmitOMPAggregateInit()
897 DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateInit()
900 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); in EmitOMPAggregateInit()
903 LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) { in emitSharedLValue() argument
904 return CGF.EmitOMPSharedLValue(E); in emitSharedLValue()
907 LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF, in emitSharedLValueUB() argument
910 return CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false); in emitSharedLValueUB()
915 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal, in emitAggregateInitialization() argument
924 EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(), in emitAggregateInitialization()
947 void ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, unsigned N) { in emitSharedLValue() argument
950 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref); in emitSharedLValue()
951 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref); in emitSharedLValue()
955 void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) { in emitAggregateType() argument
962 CGF.getTypeSize( in emitAggregateType()
974 Size = CGF.Builder.CreatePtrDiff(SharedAddresses[N].second.getPointer(), in emitAggregateType()
976 Size = CGF.Builder.CreateNUWAdd( in emitAggregateType()
978 SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf); in emitAggregateType()
980 SizeInChars = CGF.getTypeSize( in emitAggregateType()
982 Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf); in emitAggregateType()
986 CGF, in emitAggregateType()
988 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()), in emitAggregateType()
990 CGF.EmitVariablyModifiedType(PrivateType); in emitAggregateType()
993 void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N, in emitAggregateType() argument
1005 CGF, in emitAggregateType()
1007 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()), in emitAggregateType()
1009 CGF.EmitVariablyModifiedType(PrivateType); in emitAggregateType()
1013 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal, in emitInitialization() argument
1021 PrivateAddr = CGF.Builder.CreateElementBitCast( in emitInitialization()
1022 PrivateAddr, CGF.ConvertTypeForMem(PrivateType)); in emitInitialization()
1024 SharedLVal = CGF.MakeAddrLValue( in emitInitialization()
1025 CGF.Builder.CreateElementBitCast(SharedLVal.getAddress(), in emitInitialization()
1026 CGF.ConvertTypeForMem(SharedType)), in emitInitialization()
1028 CGF.CGM.getTBAAInfoForSubobject(SharedAddresses[N].first, SharedType)); in emitInitialization()
1029 if (CGF.getContext().getAsArrayType(PrivateVD->getType())) { in emitInitialization()
1030 emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD); in emitInitialization()
1032 emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp, in emitInitialization()
1035 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() && in emitInitialization()
1036 !CGF.isTrivialInitializer(PrivateVD->getInit())) { in emitInitialization()
1037 CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr, in emitInitialization()
1051 void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N, in emitCleanups() argument
1058 PrivateAddr = CGF.Builder.CreateElementBitCast( in emitCleanups()
1059 PrivateAddr, CGF.ConvertTypeForMem(PrivateType)); in emitCleanups()
1060 CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType); in emitCleanups()
1064 static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, in loadToBegin() argument
1068 !CGF.getContext().hasSameType(BaseTy, ElTy)) { in loadToBegin()
1070 BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(), PtrTy); in loadToBegin()
1072 LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(), BaseTy); in loadToBegin()
1073 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal); in loadToBegin()
1077 return CGF.MakeAddrLValue( in loadToBegin()
1078 CGF.Builder.CreateElementBitCast(BaseLV.getAddress(), in loadToBegin()
1079 CGF.ConvertTypeForMem(ElTy)), in loadToBegin()
1081 CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType())); in loadToBegin()
1084 static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, in castToBase() argument
1092 !CGF.getContext().hasSameType(BaseTy, ElTy)) { in castToBase()
1093 Tmp = CGF.CreateMemTemp(BaseTy); in castToBase()
1095 CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp); in castToBase()
1104 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty); in castToBase()
1106 CGF.Builder.CreateStore(Addr, Tmp); in castToBase()
1132 Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, in adjustPrivateAddress() argument
1137 LValue OriginalBaseLValue = CGF.EmitLValue(DE); in adjustPrivateAddress()
1139 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(), in adjustPrivateAddress()
1141 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff( in adjustPrivateAddress()
1144 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in adjustPrivateAddress()
1147 llvm::Value *Ptr = CGF.Builder.CreateGEP(PrivatePointer, Adjustment); in adjustPrivateAddress()
1148 return castToBase(CGF, OrigVD->getType(), in adjustPrivateAddress()
1164 LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) { in getThreadIDVariableLValue() argument
1165 return CGF.EmitLoadOfPointerLValue( in getThreadIDVariableLValue()
1166 CGF.GetAddrOfLocalVar(getThreadIDVariable()), in getThreadIDVariableLValue()
1170 void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) { in EmitBody() argument
1171 if (!CGF.HaveInsertPoint()) in EmitBody()
1178 CGF.EHStack.pushTerminate(); in EmitBody()
1179 CodeGen(CGF); in EmitBody()
1180 CGF.EHStack.popTerminate(); in EmitBody()
1184 CodeGenFunction &CGF) { in getThreadIDVariableLValue() argument
1185 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()), in getThreadIDVariableLValue()
1278 CodeGenFunction CGF(CGM); in emitCombinerOrInitializer() local
1281 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(), in emitCombinerOrInitializer()
1283 CodeGenFunction::OMPPrivateScope Scope(CGF); in emitCombinerOrInitializer()
1284 Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm); in emitCombinerOrInitializer()
1285 Scope.addPrivate(In, [&CGF, AddrIn, PtrTy]() { in emitCombinerOrInitializer()
1286 return CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>()) in emitCombinerOrInitializer()
1289 Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm); in emitCombinerOrInitializer()
1290 Scope.addPrivate(Out, [&CGF, AddrOut, PtrTy]() { in emitCombinerOrInitializer()
1291 return CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>()) in emitCombinerOrInitializer()
1296 !CGF.isTrivialInitializer(Out->getInit())) { in emitCombinerOrInitializer()
1297 CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out), in emitCombinerOrInitializer()
1302 CGF.EmitIgnoredExpr(CombinerInitializer); in emitCombinerOrInitializer()
1304 CGF.FinishFunction(); in emitCombinerOrInitializer()
1309 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) { in emitUserDefinedReduction() argument
1328 if (CGF) { in emitUserDefinedReduction()
1329 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn); in emitUserDefinedReduction()
1349 CodeGenFunction CGF(CGM, true); in emitParallelOrTeamsOutlinedFunction() local
1369 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitParallelOrTeamsOutlinedFunction()
1370 return CGF.GenerateOpenMPCapturedStmtFunction(*CS); in emitParallelOrTeamsOutlinedFunction()
1394 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF, in emitTaskOutlinedFunction()
1396 llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc()); in emitTaskOutlinedFunction()
1397 llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTaskOutlinedFunction()
1400 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar), in emitTaskOutlinedFunction()
1403 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), TaskArgs); in emitTaskOutlinedFunction()
1415 CodeGenFunction CGF(CGM, true); in emitTaskOutlinedFunction() local
1419 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitTaskOutlinedFunction()
1420 llvm::Value *Res = CGF.GenerateCapturedStmtFunction(*CS); in emitTaskOutlinedFunction()
1504 void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF, in setLocThreadIdInsertPt() argument
1506 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in setLocThreadIdInsertPt()
1509 llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty); in setLocThreadIdInsertPt()
1512 Undef, CGF.Int32Ty, "svcpt", CGF.Builder.GetInsertBlock()); in setLocThreadIdInsertPt()
1515 new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt"); in setLocThreadIdInsertPt()
1516 Elem.second.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt); in setLocThreadIdInsertPt()
1520 void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) { in clearLocThreadIdInsertPt() argument
1521 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in clearLocThreadIdInsertPt()
1529 llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF, in emitUpdateLocation() argument
1538 assert(CGF.CurFn && "No function in current CodeGenFunction."); in emitUpdateLocation()
1542 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn); in emitUpdateLocation()
1550 Address AI = CGF.CreateMemTemp(IdentQTy, ".kmpc_loc.addr"); in emitUpdateLocation()
1551 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in emitUpdateLocation()
1556 setLocThreadIdInsertPt(CGF); in emitUpdateLocation()
1557 CGBuilderTy::InsertPointGuard IPG(CGF.Builder); in emitUpdateLocation()
1558 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt); in emitUpdateLocation()
1559 CGF.Builder.CreateMemCpy(LocValue, getOrCreateDefaultLocation(Flags), in emitUpdateLocation()
1560 CGF.getTypeSize(IdentQTy)); in emitUpdateLocation()
1564 LValue Base = CGF.MakeAddrLValue(LocValue, IdentQTy); in emitUpdateLocation()
1567 CGF.EmitLValueForField(Base, *std::next(Fields, IdentField_PSource)); in emitUpdateLocation()
1574 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc); in emitUpdateLocation()
1576 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl)) in emitUpdateLocation()
1579 OMPDebugLoc = CGF.Builder.CreateGlobalStringPtr(OS2.str()); in emitUpdateLocation()
1583 CGF.EmitStoreOfScalar(OMPDebugLoc, PSource); in emitUpdateLocation()
1590 llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF, in getThreadID() argument
1592 assert(CGF.CurFn && "No function in current CodeGenFunction."); in getThreadID()
1597 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn); in getThreadID()
1604 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions || in getThreadID()
1605 !CGF.getLangOpts().CXXExceptions || in getThreadID()
1606 CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) { in getThreadID()
1608 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { in getThreadID()
1611 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF); in getThreadID()
1612 ThreadID = CGF.EmitLoadOfScalar(LVal, Loc); in getThreadID()
1615 if (CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) { in getThreadID()
1616 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in getThreadID()
1628 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in getThreadID()
1630 setLocThreadIdInsertPt(CGF); in getThreadID()
1631 CGBuilderTy::InsertPointGuard IPG(CGF.Builder); in getThreadID()
1632 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt); in getThreadID()
1633 llvm::CallInst *Call = CGF.Builder.CreateCall( in getThreadID()
1635 emitUpdateLocation(CGF, Loc)); in getThreadID()
1636 Call->setCallingConv(CGF.getRuntimeCC()); in getThreadID()
1641 void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) { in functionFinished() argument
1642 assert(CGF.CurFn && "No function in current CodeGenFunction."); in functionFinished()
1643 if (OpenMPLocThreadIDMap.count(CGF.CurFn)) { in functionFinished()
1644 clearLocThreadIdInsertPt(CGF); in functionFinished()
1645 OpenMPLocThreadIDMap.erase(CGF.CurFn); in functionFinished()
1647 if (FunctionUDRMap.count(CGF.CurFn) > 0) { in functionFinished()
1648 for(auto *D : FunctionUDRMap[CGF.CurFn]) in functionFinished()
1650 FunctionUDRMap.erase(CGF.CurFn); in functionFinished()
2485 Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, in getAddrOfThreadPrivate() argument
2494 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in getAddrOfThreadPrivate()
2495 CGF.Builder.CreatePointerCast(VDAddr.getPointer(), in getAddrOfThreadPrivate()
2499 return Address(CGF.EmitRuntimeCall( in getAddrOfThreadPrivate()
2505 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor, in emitThreadPrivateVarInit() argument
2509 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc); in emitThreadPrivateVarInit()
2510 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num), in emitThreadPrivateVarInit()
2515 OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy), in emitThreadPrivateVarInit()
2517 CGF.EmitRuntimeCall( in emitThreadPrivateVarInit()
2523 bool PerformInit, CodeGenFunction *CGF) { in emitThreadPrivateVarDefinition() argument
2621 if (!CGF) { in emitThreadPrivateVarDefinition()
2636 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc); in emitThreadPrivateVarDefinition()
2775 Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, in getAddrOfArtificialThreadPrivate() argument
2780 llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType); in getAddrOfArtificialThreadPrivate()
2784 emitUpdateLocation(CGF, SourceLocation()), in getAddrOfArtificialThreadPrivate()
2785 getThreadID(CGF, SourceLocation()), in getAddrOfArtificialThreadPrivate()
2786 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy), in getAddrOfArtificialThreadPrivate()
2787 CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy, in getAddrOfArtificialThreadPrivate()
2792 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddrOfArtificialThreadPrivate()
2793 CGF.EmitRuntimeCall( in getAddrOfArtificialThreadPrivate()
2799 void CGOpenMPRuntime::emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond, in emitOMPIfClause() argument
2802 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange()); in emitOMPIfClause()
2807 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) { in emitOMPIfClause()
2809 ThenGen(CGF); in emitOMPIfClause()
2811 ElseGen(CGF); in emitOMPIfClause()
2817 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then"); in emitOMPIfClause()
2818 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else"); in emitOMPIfClause()
2819 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end"); in emitOMPIfClause()
2820 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0); in emitOMPIfClause()
2823 CGF.EmitBlock(ThenBlock); in emitOMPIfClause()
2824 ThenGen(CGF); in emitOMPIfClause()
2825 CGF.EmitBranch(ContBlock); in emitOMPIfClause()
2828 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitOMPIfClause()
2829 CGF.EmitBlock(ElseBlock); in emitOMPIfClause()
2830 ElseGen(CGF); in emitOMPIfClause()
2832 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitOMPIfClause()
2833 CGF.EmitBranch(ContBlock); in emitOMPIfClause()
2835 CGF.EmitBlock(ContBlock, /*IsFinished=*/true); in emitOMPIfClause()
2838 void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, in emitParallelCall() argument
2842 if (!CGF.HaveInsertPoint()) in emitParallelCall()
2844 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitParallelCall()
2845 auto &&ThenGen = [OutlinedFn, CapturedVars, RTLoc](CodeGenFunction &CGF, in emitParallelCall()
2848 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitParallelCall()
2851 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars in emitParallelCall()
2852 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())}; in emitParallelCall()
2858 CGF.EmitRuntimeCall(RTLFn, RealArgs); in emitParallelCall()
2860 auto &&ElseGen = [OutlinedFn, CapturedVars, RTLoc, Loc](CodeGenFunction &CGF, in emitParallelCall() argument
2862 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitParallelCall()
2863 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc); in emitParallelCall()
2867 CGF.EmitRuntimeCall( in emitParallelCall()
2871 Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, in emitParallelCall()
2873 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0)); in emitParallelCall()
2879 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs); in emitParallelCall()
2882 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID}; in emitParallelCall()
2883 CGF.EmitRuntimeCall( in emitParallelCall()
2888 emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen); in emitParallelCall()
2891 ThenRCG(CGF); in emitParallelCall()
2901 Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF, in emitThreadIDAddress() argument
2904 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitThreadIDAddress()
2906 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(); in emitThreadIDAddress()
2908 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitThreadIDAddress()
2910 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true); in emitThreadIDAddress()
2911 Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp."); in emitThreadIDAddress()
2912 CGF.EmitStoreOfScalar(ThreadID, in emitThreadIDAddress()
2913 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty)); in emitThreadIDAddress()
2960 void Enter(CodeGenFunction &CGF) override { in Enter() argument
2961 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs); in Enter()
2963 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes); in Enter()
2964 auto *ThenBlock = CGF.createBasicBlock("omp_if.then"); in Enter()
2965 ContBlock = CGF.createBasicBlock("omp_if.end"); in Enter()
2967 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock); in Enter()
2968 CGF.EmitBlock(ThenBlock); in Enter()
2971 void Done(CodeGenFunction &CGF) { in Done() argument
2973 CGF.EmitBranch(ContBlock); in Done()
2974 CGF.EmitBlock(ContBlock, true); in Done()
2976 void Exit(CodeGenFunction &CGF) override { in Exit() argument
2977 CGF.EmitRuntimeCall(ExitCallee, ExitArgs); in Exit()
2982 void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF, in emitCriticalRegion() argument
2990 if (!CGF.HaveInsertPoint()) in emitCriticalRegion()
2992 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitCriticalRegion()
2997 EnterArgs.push_back(CGF.Builder.CreateIntCast( in emitCriticalRegion()
2998 CGF.EmitScalarExpr(Hint), CGM.IntPtrTy, /*isSigned=*/false)); in emitCriticalRegion()
3005 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen); in emitCriticalRegion()
3008 void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF, in emitMasterRegion() argument
3011 if (!CGF.HaveInsertPoint()) in emitMasterRegion()
3018 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitMasterRegion()
3023 emitInlinedDirective(CGF, OMPD_master, MasterOpGen); in emitMasterRegion()
3024 Action.Done(CGF); in emitMasterRegion()
3027 void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF, in emitTaskyieldCall() argument
3029 if (!CGF.HaveInsertPoint()) in emitTaskyieldCall()
3033 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitTaskyieldCall()
3035 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskyield), Args); in emitTaskyieldCall()
3036 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitTaskyieldCall()
3037 Region->emitUntiedSwitch(CGF); in emitTaskyieldCall()
3040 void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF, in emitTaskgroupRegion() argument
3043 if (!CGF.HaveInsertPoint()) in emitTaskgroupRegion()
3049 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitTaskgroupRegion()
3054 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen); in emitTaskgroupRegion()
3059 static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array, in emitAddrOfVarFromArray() argument
3063 CGF.Builder.CreateConstArrayGEP(Array, Index, CGF.getPointerSize()); in emitAddrOfVarFromArray()
3064 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr); in emitAddrOfVarFromArray()
3066 Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var)); in emitAddrOfVarFromArray()
3067 Addr = CGF.Builder.CreateElementBitCast( in emitAddrOfVarFromArray()
3068 Addr, CGF.ConvertTypeForMem(Var->getType())); in emitAddrOfVarFromArray()
3095 CodeGenFunction CGF(CGM); in emitCopyprivateCopyFunction() local
3096 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitCopyprivateCopyFunction()
3099 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitCopyprivateCopyFunction()
3100 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)), in emitCopyprivateCopyFunction()
3101 ArgsType), CGF.getPointerAlign()); in emitCopyprivateCopyFunction()
3102 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitCopyprivateCopyFunction()
3103 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)), in emitCopyprivateCopyFunction()
3104 ArgsType), CGF.getPointerAlign()); in emitCopyprivateCopyFunction()
3112 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar); in emitCopyprivateCopyFunction()
3116 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar); in emitCopyprivateCopyFunction()
3120 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]); in emitCopyprivateCopyFunction()
3122 CGF.FinishFunction(); in emitCopyprivateCopyFunction()
3126 void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF, in emitSingleRegion() argument
3133 if (!CGF.HaveInsertPoint()) in emitSingleRegion()
3153 DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it"); in emitSingleRegion()
3154 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt); in emitSingleRegion()
3157 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitSingleRegion()
3162 emitInlinedDirective(CGF, OMPD_single, SingleOpGen); in emitSingleRegion()
3165 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt); in emitSingleRegion()
3167 Action.Done(CGF); in emitSingleRegion()
3177 CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list"); in emitSingleRegion()
3179 Address Elem = CGF.Builder.CreateConstArrayGEP( in emitSingleRegion()
3180 CopyprivateList, I, CGF.getPointerSize()); in emitSingleRegion()
3181 CGF.Builder.CreateStore( in emitSingleRegion()
3182 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitSingleRegion()
3183 CGF.EmitLValue(CopyprivateVars[I]).getPointer(), CGF.VoidPtrTy), in emitSingleRegion()
3189 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(), in emitSingleRegion()
3191 llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy); in emitSingleRegion()
3193 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList, in emitSingleRegion()
3194 CGF.VoidPtrTy); in emitSingleRegion()
3195 llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt); in emitSingleRegion()
3197 emitUpdateLocation(CGF, Loc), // ident_t *<loc> in emitSingleRegion()
3198 getThreadID(CGF, Loc), // i32 <gtid> in emitSingleRegion()
3204 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_copyprivate), Args); in emitSingleRegion()
3208 void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF, in emitOrderedRegion() argument
3211 if (!CGF.HaveInsertPoint()) in emitOrderedRegion()
3218 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitOrderedRegion()
3223 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen); in emitOrderedRegion()
3226 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen); in emitOrderedRegion()
3244 void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, in emitBarrierCall() argument
3247 if (!CGF.HaveInsertPoint()) in emitBarrierCall()
3254 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags), in emitBarrierCall()
3255 getThreadID(CGF, Loc)}; in emitBarrierCall()
3257 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { in emitBarrierCall()
3259 llvm::Value *Result = CGF.EmitRuntimeCall( in emitBarrierCall()
3265 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit"); in emitBarrierCall()
3266 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue"); in emitBarrierCall()
3267 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result); in emitBarrierCall()
3268 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); in emitBarrierCall()
3269 CGF.EmitBlock(ExitBB); in emitBarrierCall()
3272 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); in emitBarrierCall()
3273 CGF.EmitBranchThroughCleanup(CancelDestination); in emitBarrierCall()
3274 CGF.EmitBlock(ContBB, /*IsFinished=*/true); in emitBarrierCall()
3279 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_barrier), Args); in emitBarrierCall()
3382 CodeGenFunction &CGF, SourceLocation Loc, in emitForDispatchInit() argument
3385 if (!CGF.HaveInsertPoint()) in emitForDispatchInit()
3400 : CGF.Builder.getIntN(IVSize, 1); in emitForDispatchInit()
3402 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitForDispatchInit()
3403 CGF.Builder.getInt32(addMonoNonMonoModifier( in emitForDispatchInit()
3407 CGF.Builder.getIntN(IVSize, 1), // Stride in emitForDispatchInit()
3410 CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args); in emitForDispatchInit()
3414 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId, in emitForStaticInitCall() argument
3418 if (!CGF.HaveInsertPoint()) in emitForStaticInitCall()
3439 Chunk = CGF.Builder.getIntN(Values.IVSize, 1); in emitForStaticInitCall()
3450 CGF.Builder.getInt32(addMonoNonMonoModifier(Schedule, M1, in emitForStaticInitCall()
3456 CGF.Builder.getIntN(Values.IVSize, 1), // Incr in emitForStaticInitCall()
3459 CGF.EmitRuntimeCall(ForStaticInitFunction, Args); in emitForStaticInitCall()
3462 void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF, in emitForStaticInit() argument
3471 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc, in emitForStaticInit()
3475 llvm::Value *ThreadId = getThreadID(CGF, Loc); in emitForStaticInit()
3478 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction, in emitForStaticInit()
3483 CodeGenFunction &CGF, SourceLocation Loc, in emitDistributeStaticInit() argument
3489 emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE); in emitDistributeStaticInit()
3490 llvm::Value *ThreadId = getThreadID(CGF, Loc); in emitDistributeStaticInit()
3493 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction, in emitDistributeStaticInit()
3498 void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF, in emitForStaticFinish() argument
3501 if (!CGF.HaveInsertPoint()) in emitForStaticFinish()
3505 emitUpdateLocation(CGF, Loc, in emitForStaticFinish()
3511 getThreadID(CGF, Loc)}; in emitForStaticFinish()
3512 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_for_static_fini), in emitForStaticFinish()
3516 void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF, in emitForOrderedIterationEnd() argument
3520 if (!CGF.HaveInsertPoint()) in emitForOrderedIterationEnd()
3523 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitForOrderedIterationEnd()
3524 CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args); in emitForOrderedIterationEnd()
3527 llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF, in emitForNext() argument
3537 emitUpdateLocation(CGF, Loc), in emitForNext()
3538 getThreadID(CGF, Loc), in emitForNext()
3545 CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args); in emitForNext()
3546 return CGF.EmitScalarConversion( in emitForNext()
3547 Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1), in emitForNext()
3548 CGF.getContext().BoolTy, Loc); in emitForNext()
3551 void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF, in emitNumThreadsClause() argument
3554 if (!CGF.HaveInsertPoint()) in emitNumThreadsClause()
3558 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitNumThreadsClause()
3559 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)}; in emitNumThreadsClause()
3560 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_threads), in emitNumThreadsClause()
3564 void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF, in emitProcBindClause() argument
3567 if (!CGF.HaveInsertPoint()) in emitProcBindClause()
3594 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitProcBindClause()
3596 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_proc_bind), Args); in emitProcBindClause()
3599 void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>, in emitFlush() argument
3601 if (!CGF.HaveInsertPoint()) in emitFlush()
3604 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_flush), in emitFlush()
3605 emitUpdateLocation(CGF, Loc)); in emitFlush()
3845 CodeGenFunction CGF(CGM); in createOffloadingBinaryDescriptorRegistration() local
3848 CGF.disableDebugInfo(); in createOffloadingBinaryDescriptorRegistration()
3854 CGF.StartFunction(GlobalDecl(), C.VoidTy, UnRegFn, FI, Args); in createOffloadingBinaryDescriptorRegistration()
3855 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_unregister_lib), in createOffloadingBinaryDescriptorRegistration()
3857 CGF.FinishFunction(); in createOffloadingBinaryDescriptorRegistration()
3861 CodeGenFunction CGF(CGM); in createOffloadingBinaryDescriptorRegistration() local
3864 CGF.disableDebugInfo(); in createOffloadingBinaryDescriptorRegistration()
3881 CGF.StartFunction(GlobalDecl(), C.VoidTy, RegFn, FI, FunctionArgList()); in createOffloadingBinaryDescriptorRegistration()
3882 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_register_lib), Desc); in createOffloadingBinaryDescriptorRegistration()
3888 CGM.getCXXABI().registerGlobalDtor(CGF, RegUnregVar, UnRegFn, Desc); in createOffloadingBinaryDescriptorRegistration()
3889 CGF.FinishFunction(); in createOffloadingBinaryDescriptorRegistration()
4392 CodeGenFunction CGF(CGM); in emitProxyTaskFunction() local
4393 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args, in emitProxyTaskFunction()
4401 llvm::Value *GtidParam = CGF.EmitLoadOfScalar( in emitProxyTaskFunction()
4402 CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc); in emitProxyTaskFunction()
4403 LValue TDBase = CGF.EmitLoadOfPointerLValue( in emitProxyTaskFunction()
4404 CGF.GetAddrOfLocalVar(&TaskTypeArg), in emitProxyTaskFunction()
4409 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin()); in emitProxyTaskFunction()
4412 LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI); in emitProxyTaskFunction()
4416 LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI); in emitProxyTaskFunction()
4417 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitProxyTaskFunction()
4418 CGF.EmitLoadOfScalar(SharedsLVal, Loc), in emitProxyTaskFunction()
4419 CGF.ConvertTypeForMem(SharedsPtrTy)); in emitProxyTaskFunction()
4424 LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI); in emitProxyTaskFunction()
4425 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitProxyTaskFunction()
4426 PrivatesLVal.getPointer(), CGF.VoidPtrTy); in emitProxyTaskFunction()
4428 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitProxyTaskFunction()
4433 CGF.Builder in emitProxyTaskFunction()
4435 TDBase.getAddress(), CGF.VoidPtrTy) in emitProxyTaskFunction()
4441 LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI); in emitProxyTaskFunction()
4442 llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc); in emitProxyTaskFunction()
4444 LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI); in emitProxyTaskFunction()
4445 llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc); in emitProxyTaskFunction()
4447 LValue StLVal = CGF.EmitLValueForField(Base, *StFI); in emitProxyTaskFunction()
4448 llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc); in emitProxyTaskFunction()
4450 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI); in emitProxyTaskFunction()
4451 llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc); in emitProxyTaskFunction()
4453 LValue RLVal = CGF.EmitLValueForField(Base, *RFI); in emitProxyTaskFunction()
4454 llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc); in emitProxyTaskFunction()
4463 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction, in emitProxyTaskFunction()
4465 CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)), in emitProxyTaskFunction()
4466 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty)); in emitProxyTaskFunction()
4467 CGF.FinishFunction(); in emitProxyTaskFunction()
4497 CodeGenFunction CGF(CGM); in emitDestructorsFunction() local
4498 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo, in emitDestructorsFunction()
4501 LValue Base = CGF.EmitLoadOfPointerLValue( in emitDestructorsFunction()
4502 CGF.GetAddrOfLocalVar(&TaskTypeArg), in emitDestructorsFunction()
4507 Base = CGF.EmitLValueForField(Base, *FI); in emitDestructorsFunction()
4512 LValue FieldLValue = CGF.EmitLValueForField(Base, Field); in emitDestructorsFunction()
4513 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType()); in emitDestructorsFunction()
4516 CGF.FinishFunction(); in emitDestructorsFunction()
4593 CodeGenFunction CGF(CGM); in emitTaskPrivateMappingFunction() local
4594 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap, in emitTaskPrivateMappingFunction()
4598 LValue Base = CGF.EmitLoadOfPointerLValue( in emitTaskPrivateMappingFunction()
4599 CGF.GetAddrOfLocalVar(&TaskPrivatesArg), in emitTaskPrivateMappingFunction()
4604 LValue FieldLVal = CGF.EmitLValueForField(Base, Field); in emitTaskPrivateMappingFunction()
4607 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType()); in emitTaskPrivateMappingFunction()
4608 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue( in emitTaskPrivateMappingFunction()
4610 CGF.EmitStoreOfScalar(FieldLVal.getPointer(), RefLoadLVal); in emitTaskPrivateMappingFunction()
4613 CGF.FinishFunction(); in emitTaskPrivateMappingFunction()
4623 static void emitPrivatesInit(CodeGenFunction &CGF, in emitPrivatesInit() argument
4630 ASTContext &C = CGF.getContext(); in emitPrivatesInit()
4632 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI); in emitPrivatesInit()
4647 SrcBase = CGF.MakeAddrLValue( in emitPrivatesInit()
4648 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitPrivatesInit()
4649 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)), in emitPrivatesInit()
4657 !CGF.isTrivialInitializer(Init)))) { in emitPrivatesInit()
4658 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI); in emitPrivatesInit()
4676 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type); in emitPrivatesInit()
4678 SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField); in emitPrivatesInit()
4679 SharedRefLValue = CGF.MakeAddrLValue( in emitPrivatesInit()
4686 if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) { in emitPrivatesInit()
4688 CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type); in emitPrivatesInit()
4692 CGF.EmitOMPAggregateAssign( in emitPrivatesInit()
4694 [&CGF, Elem, Init, &CapturesInfo](Address DestElement, in emitPrivatesInit()
4697 CodeGenFunction::OMPPrivateScope InitScope(CGF); in emitPrivatesInit()
4703 CGF, &CapturesInfo); in emitPrivatesInit()
4704 CGF.EmitAnyExprToMem(Init, DestElement, in emitPrivatesInit()
4710 CodeGenFunction::OMPPrivateScope InitScope(CGF); in emitPrivatesInit()
4715 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo); in emitPrivatesInit()
4716 CGF.EmitExprAsInit(Init, VD, PrivateLValue, in emitPrivatesInit()
4720 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false); in emitPrivatesInit()
4728 static bool checkInitIsRequired(CodeGenFunction &CGF, in checkInitIsRequired() argument
4735 !CGF.isTrivialInitializer(Init)); in checkInitIsRequired()
4782 CodeGenFunction CGF(CGM); in emitTaskDupFunction() local
4783 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc, in emitTaskDupFunction()
4786 LValue TDBase = CGF.EmitLoadOfPointerLValue( in emitTaskDupFunction()
4787 CGF.GetAddrOfLocalVar(&DstArg), in emitTaskDupFunction()
4792 LValue Base = CGF.EmitLValueForField( in emitTaskDupFunction()
4794 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI); in emitTaskDupFunction()
4795 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar( in emitTaskDupFunction()
4796 CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc); in emitTaskDupFunction()
4797 CGF.EmitStoreOfScalar(Lastpriv, LILVal); in emitTaskDupFunction()
4804 LValue TDBase = CGF.EmitLoadOfPointerLValue( in emitTaskDupFunction()
4805 CGF.GetAddrOfLocalVar(&SrcArg), in emitTaskDupFunction()
4807 LValue Base = CGF.EmitLValueForField( in emitTaskDupFunction()
4810 CGF.EmitLoadOfScalar(CGF.EmitLValueForField( in emitTaskDupFunction()
4814 CGF.getNaturalTypeAlignment(SharedsTy)); in emitTaskDupFunction()
4816 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD, in emitTaskDupFunction()
4818 CGF.FinishFunction(); in emitTaskDupFunction()
4838 CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskInit() argument
4904 llvm::Type *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy); in emitTaskInit()
4908 CGF.getTypeSize(KmpTaskTWithPrivatesQTy); in emitTaskInit()
4920 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
4956 ? CGF.Builder.CreateSelect(Data.Final.getPointer(), in emitTaskInit()
4957 CGF.Builder.getInt32(FinalFlag), in emitTaskInit()
4958 CGF.Builder.getInt32(/*C=*/0)) in emitTaskInit()
4959 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0); in emitTaskInit()
4960 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags)); in emitTaskInit()
4962 llvm::Value *AllocArgs[] = {emitUpdateLocation(CGF, Loc), in emitTaskInit()
4963 getThreadID(CGF, Loc), TaskFlags, in emitTaskInit()
4965 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
4967 llvm::Value *NewTask = CGF.EmitRuntimeCall( in emitTaskInit()
4970 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
4972 LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy, in emitTaskInit()
4975 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin()); in emitTaskInit()
4981 Address(CGF.EmitLoadOfScalar( in emitTaskInit()
4982 CGF.EmitLValueForField( in emitTaskInit()
4986 CGF.getNaturalTypeAlignment(SharedsTy)); in emitTaskInit()
4987 LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy); in emitTaskInit()
4988 LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy); in emitTaskInit()
4989 CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap); in emitTaskInit()
4994 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD, in emitTaskInit()
4998 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) { in emitTaskInit()
5015 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI); in emitTaskInit()
5016 LValue DestructorsLV = CGF.EmitLValueForField( in emitTaskInit()
5018 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
5024 LValue Data2LV = CGF.EmitLValueForField( in emitTaskInit()
5026 LValue PriorityLV = CGF.EmitLValueForField( in emitTaskInit()
5028 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV); in emitTaskInit()
5038 void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskCall() argument
5044 if (!CGF.HaveInsertPoint()) in emitTaskCall()
5048 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data); in emitTaskCall()
5065 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy); in emitTaskCall()
5084 CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr"); in emitTaskCall()
5087 LValue Addr = CGF.EmitLValue(E); in emitTaskCall()
5093 CGF.EmitOMPArraySectionExpr(ASE, /*LowerBound=*/false); in emitTaskCall()
5095 CGF.Builder.CreateConstGEP1_32(UpAddrLVal.getPointer(), /*Idx0=*/1); in emitTaskCall()
5097 CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGM.SizeTy); in emitTaskCall()
5098 llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGM.SizeTy); in emitTaskCall()
5099 Size = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr); in emitTaskCall()
5101 Size = CGF.getTypeSize(Ty); in emitTaskCall()
5103 LValue Base = CGF.MakeAddrLValue( in emitTaskCall()
5104 CGF.Builder.CreateConstArrayGEP(DependenciesArray, I, DependencySize), in emitTaskCall()
5107 LValue BaseAddrLVal = CGF.EmitLValueForField( in emitTaskCall()
5109 CGF.EmitStoreOfScalar( in emitTaskCall()
5110 CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGF.IntPtrTy), in emitTaskCall()
5113 LValue LenLVal = CGF.EmitLValueForField( in emitTaskCall()
5115 CGF.EmitStoreOfScalar(Size, LenLVal); in emitTaskCall()
5132 LValue FlagsLVal = CGF.EmitLValueForField( in emitTaskCall()
5134 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind), in emitTaskCall()
5137 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskCall()
5138 CGF.Builder.CreateStructGEP(DependenciesArray, 0, CharUnits::Zero()), in emitTaskCall()
5139 CGF.VoidPtrTy); in emitTaskCall()
5148 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitTaskCall()
5149 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc); in emitTaskCall()
5156 DepTaskArgs[3] = CGF.Builder.getInt32(NumDependencies); in emitTaskCall()
5158 DepTaskArgs[5] = CGF.Builder.getInt32(0); in emitTaskCall()
5159 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitTaskCall()
5163 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) { in emitTaskCall() argument
5166 LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI); in emitTaskCall()
5167 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal); in emitTaskCall()
5170 CGF.EmitRuntimeCall( in emitTaskCall()
5173 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), in emitTaskCall()
5178 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitTaskCall()
5179 Region->emitUntiedSwitch(CGF); in emitTaskCall()
5186 DepWaitTaskArgs[2] = CGF.Builder.getInt32(NumDependencies); in emitTaskCall()
5188 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0); in emitTaskCall()
5189 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitTaskCall()
5193 Loc](CodeGenFunction &CGF, PrePostActionTy &) { in emitTaskCall() argument
5194 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitTaskCall()
5195 CodeGenFunction::RunCleanupsScope LocalScope(CGF); in emitTaskCall()
5201 CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__kmpc_omp_wait_deps), in emitTaskCall()
5205 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTaskCall() argument
5206 Action.Enter(CGF); in emitTaskCall()
5208 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry, in emitTaskCall()
5221 RCG(CGF); in emitTaskCall()
5225 emitOMPIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen); in emitTaskCall()
5228 ThenRCG(CGF); in emitTaskCall()
5232 void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskLoopCall() argument
5238 if (!CGF.HaveInsertPoint()) in emitTaskLoopCall()
5241 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data); in emitTaskLoopCall()
5247 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitTaskLoopCall()
5248 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc); in emitTaskLoopCall()
5251 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy, in emitTaskLoopCall()
5254 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1); in emitTaskLoopCall()
5257 LValue LBLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
5262 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(), LBLVal.getQuals(), in emitTaskLoopCall()
5264 LValue UBLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
5269 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(), UBLVal.getQuals(), in emitTaskLoopCall()
5271 LValue StLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
5276 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(), StLVal.getQuals(), in emitTaskLoopCall()
5279 LValue RedLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
5283 CGF.EmitStoreOfScalar(Data.Reductions, RedLVal); in emitTaskLoopCall()
5285 CGF.EmitNullInitialization(RedLVal.getAddress(), in emitTaskLoopCall()
5286 CGF.getContext().VoidPtrTy); in emitTaskLoopCall()
5296 CGF.EmitLoadOfScalar(StLVal, Loc), in emitTaskLoopCall()
5298 CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler in emitTaskLoopCall()
5300 CGF.IntTy, Data.Schedule.getPointer() in emitTaskLoopCall()
5304 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty, in emitTaskLoopCall()
5306 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0), in emitTaskLoopCall()
5307 Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskLoopCall()
5308 Result.TaskDupFn, CGF.VoidPtrTy) in emitTaskLoopCall()
5309 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)}; in emitTaskLoopCall()
5310 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_taskloop), TaskArgs); in emitTaskLoopCall()
5323 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar, in EmitOMPAggregateReduction() argument
5325 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *, in EmitOMPAggregateReduction() argument
5331 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar); in EmitOMPAggregateReduction()
5332 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar); in EmitOMPAggregateReduction()
5336 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr); in EmitOMPAggregateReduction()
5341 llvm::Value *LHSEnd = CGF.Builder.CreateGEP(LHSBegin, NumElements); in EmitOMPAggregateReduction()
5343 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body"); in EmitOMPAggregateReduction()
5344 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done"); in EmitOMPAggregateReduction()
5346 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty"); in EmitOMPAggregateReduction()
5347 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); in EmitOMPAggregateReduction()
5350 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock(); in EmitOMPAggregateReduction()
5351 CGF.EmitBlock(BodyBB); in EmitOMPAggregateReduction()
5353 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy); in EmitOMPAggregateReduction()
5355 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI( in EmitOMPAggregateReduction()
5362 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI( in EmitOMPAggregateReduction()
5370 CodeGenFunction::OMPPrivateScope Scope(CGF); in EmitOMPAggregateReduction()
5374 RedOpGen(CGF, XExpr, EExpr, UpExpr); in EmitOMPAggregateReduction()
5378 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateReduction()
5380 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateReduction()
5384 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done"); in EmitOMPAggregateReduction()
5385 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB); in EmitOMPAggregateReduction()
5386 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateReduction()
5387 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateReduction()
5390 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); in EmitOMPAggregateReduction()
5396 static void emitReductionCombiner(CodeGenFunction &CGF, in emitReductionCombiner() argument
5405 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD); in emitReductionCombiner()
5407 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func); in emitReductionCombiner()
5408 CGF.EmitIgnoredExpr(ReductionOp); in emitReductionCombiner()
5411 CGF.EmitIgnoredExpr(ReductionOp); in emitReductionCombiner()
5436 CodeGenFunction CGF(CGM); in emitReductionFunction() local
5437 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitReductionFunction()
5441 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReductionFunction()
5442 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)), in emitReductionFunction()
5443 ArgsType), CGF.getPointerAlign()); in emitReductionFunction()
5444 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReductionFunction()
5445 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)), in emitReductionFunction()
5446 ArgsType), CGF.getPointerAlign()); in emitReductionFunction()
5451 CodeGenFunction::OMPPrivateScope Scope(CGF); in emitReductionFunction()
5457 Scope.addPrivate(RHSVar, [&CGF, RHS, Idx, RHSVar]() { in emitReductionFunction()
5458 return emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar); in emitReductionFunction()
5462 Scope.addPrivate(LHSVar, [&CGF, LHS, Idx, LHSVar]() { in emitReductionFunction()
5463 return emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar); in emitReductionFunction()
5470 CGF.Builder.CreateConstArrayGEP(LHS, Idx, CGF.getPointerSize()); in emitReductionFunction()
5471 llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem); in emitReductionFunction()
5473 CGF.getContext().getAsVariableArrayType(PrivTy); in emitReductionFunction()
5476 CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy))); in emitReductionFunction()
5477 CGF.EmitVariablyModifiedType(PrivTy); in emitReductionFunction()
5490 CGF, (*IPriv)->getType(), LHSVar, RHSVar, in emitReductionFunction()
5491 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) { in emitReductionFunction() argument
5492 emitReductionCombiner(CGF, E); in emitReductionFunction()
5496 emitReductionCombiner(CGF, E); in emitReductionFunction()
5503 CGF.FinishFunction(); in emitReductionFunction()
5507 void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF, in emitSingleReductionCombiner() argument
5517 CGF, PrivateRef->getType(), LHSVar, RHSVar, in emitSingleReductionCombiner()
5518 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) { in emitSingleReductionCombiner() argument
5519 emitReductionCombiner(CGF, ReductionOp); in emitSingleReductionCombiner()
5523 emitReductionCombiner(CGF, ReductionOp); in emitSingleReductionCombiner()
5527 void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, in emitReduction() argument
5533 if (!CGF.HaveInsertPoint()) in emitReduction()
5577 CodeGenFunction::RunCleanupsScope Scope(CGF); in emitReduction()
5582 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), in emitReduction()
5604 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); in emitReduction()
5609 CGF.Builder.CreateConstArrayGEP(ReductionList, Idx, CGF.getPointerSize()); in emitReduction()
5610 CGF.Builder.CreateStore( in emitReduction()
5611 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReduction()
5612 CGF.EmitLValue(RHSExprs[I]).getPointer(), CGF.VoidPtrTy), in emitReduction()
5617 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx, in emitReduction()
5618 CGF.getPointerSize()); in emitReduction()
5619 llvm::Value *Size = CGF.Builder.CreateIntCast( in emitReduction()
5620 CGF.getVLASize( in emitReduction()
5621 CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) in emitReduction()
5623 CGF.SizeTy, /*isSigned=*/false); in emitReduction()
5624 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), in emitReduction()
5631 CGM, Loc, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), in emitReduction()
5640 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE); in emitReduction()
5641 llvm::Value *ThreadId = getThreadID(CGF, Loc); in emitReduction()
5642 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy); in emitReduction()
5643 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReduction()
5644 ReductionList.getPointer(), CGF.VoidPtrTy); in emitReduction()
5648 CGF.Builder.getInt32(RHSExprs.size()), // i32 <n> in emitReduction()
5654 llvm::Value *Res = CGF.EmitRuntimeCall( in emitReduction()
5660 llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(".omp.reduction.default"); in emitReduction()
5662 CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2); in emitReduction()
5670 llvm::BasicBlock *Case1BB = CGF.createBasicBlock(".omp.reduction.case1"); in emitReduction()
5671 SwInst->addCase(CGF.Builder.getInt32(1), Case1BB); in emitReduction()
5672 CGF.EmitBlock(Case1BB); in emitReduction()
5681 CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
5682 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitReduction()
5687 RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), in emitReduction()
5701 RCG(CGF); in emitReduction()
5703 CGF.EmitBranch(DefaultBB); in emitReduction()
5710 llvm::BasicBlock *Case2BB = CGF.createBasicBlock(".omp.reduction.case2"); in emitReduction()
5711 SwInst->addCase(CGF.Builder.getInt32(2), Case2BB); in emitReduction()
5712 CGF.EmitBlock(Case2BB); in emitReduction()
5715 CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
5749 Loc](CodeGenFunction &CGF, const Expr *XExpr, in emitReduction()
5751 LValue X = CGF.EmitLValue(XExpr); in emitReduction()
5754 E = CGF.EmitAnyExpr(EExpr); in emitReduction()
5755 CGF.EmitOMPAtomicSimpleUpdateExpr( in emitReduction()
5758 [&CGF, UpExpr, VD, Loc](RValue XRValue) { in emitReduction()
5759 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitReduction()
5761 VD, [&CGF, VD, XRValue, Loc]() { in emitReduction()
5762 Address LHSTemp = CGF.CreateMemTemp(VD->getType()); in emitReduction()
5763 CGF.emitOMPSimpleStore( in emitReduction()
5764 CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue, in emitReduction()
5769 return CGF.EmitAnyExpr(UpExpr); in emitReduction()
5776 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar, in emitReduction()
5780 AtomicRedGen(CGF, XExpr, EExpr, UpExpr); in emitReduction()
5784 auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *, in emitReduction()
5786 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitReduction()
5789 CGF, Name, in emitReduction()
5790 [=](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
5791 Action.Enter(CGF); in emitReduction()
5792 emitReductionCombiner(CGF, E); in emitReduction()
5801 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar, in emitReduction()
5804 CritRedGen(CGF, nullptr, nullptr, nullptr); in emitReduction()
5824 AtomicRCG(CGF); in emitReduction()
5826 AtomicRCG(CGF); in emitReduction()
5829 CGF.EmitBranch(DefaultBB); in emitReduction()
5830 CGF.EmitBlock(DefaultBB, /*IsFinished=*/true); in emitReduction()
5875 CodeGenFunction CGF(CGM); in emitReduceInitFunction() local
5876 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc); in emitReduceInitFunction()
5877 Address PrivateAddr = CGF.EmitLoadOfPointer( in emitReduceInitFunction()
5878 CGF.GetAddrOfLocalVar(&Param), in emitReduceInitFunction()
5885 CGF, CGM.getContext().getSizeType(), in emitReduceInitFunction()
5887 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false, in emitReduceInitFunction()
5890 RCG.emitAggregateType(CGF, N, Size); in emitReduceInitFunction()
5898 CGF, CGM.getContext().VoidPtrTy, in emitReduceInitFunction()
5900 SharedAddr = CGF.EmitLoadOfPointer( in emitReduceInitFunction()
5903 SharedLVal = CGF.MakeAddrLValue(SharedAddr, CGM.getContext().VoidPtrTy); in emitReduceInitFunction()
5905 SharedLVal = CGF.MakeNaturalAlignAddrLValue( in emitReduceInitFunction()
5912 RCG.emitInitialization(CGF, N, PrivateAddr, SharedLVal, in emitReduceInitFunction()
5914 CGF.FinishFunction(); in emitReduceInitFunction()
5952 CodeGenFunction CGF(CGM); in emitReduceCombFunction() local
5953 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc); in emitReduceCombFunction()
5959 CGF, CGM.getContext().getSizeType(), in emitReduceCombFunction()
5961 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false, in emitReduceCombFunction()
5964 RCG.emitAggregateType(CGF, N, Size); in emitReduceCombFunction()
5968 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitReduceCombFunction()
5969 PrivateScope.addPrivate(LHSVD, [&C, &CGF, &ParamInOut, LHSVD]() { in emitReduceCombFunction()
5971 Address PtrAddr = CGF.EmitLoadOfPointer( in emitReduceCombFunction()
5972 CGF.GetAddrOfLocalVar(&ParamInOut), in emitReduceCombFunction()
5974 return CGF.Builder.CreateElementBitCast( in emitReduceCombFunction()
5975 PtrAddr, CGF.ConvertTypeForMem(LHSVD->getType())); in emitReduceCombFunction()
5977 PrivateScope.addPrivate(RHSVD, [&C, &CGF, &ParamIn, RHSVD]() { in emitReduceCombFunction()
5979 Address PtrAddr = CGF.EmitLoadOfPointer( in emitReduceCombFunction()
5980 CGF.GetAddrOfLocalVar(&ParamIn), in emitReduceCombFunction()
5982 return CGF.Builder.CreateElementBitCast( in emitReduceCombFunction()
5983 PtrAddr, CGF.ConvertTypeForMem(RHSVD->getType())); in emitReduceCombFunction()
5990 CGF, ReductionOp, PrivateRef, cast<DeclRefExpr>(LHS), in emitReduceCombFunction()
5992 CGF.FinishFunction(); in emitReduceCombFunction()
6022 CodeGenFunction CGF(CGM); in emitReduceFiniFunction() local
6023 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc); in emitReduceFiniFunction()
6024 Address PrivateAddr = CGF.EmitLoadOfPointer( in emitReduceFiniFunction()
6025 CGF.GetAddrOfLocalVar(&Param), in emitReduceFiniFunction()
6032 CGF, CGM.getContext().getSizeType(), in emitReduceFiniFunction()
6034 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false, in emitReduceFiniFunction()
6037 RCG.emitAggregateType(CGF, N, Size); in emitReduceFiniFunction()
6040 RCG.emitCleanups(CGF, N, PrivateAddr); in emitReduceFiniFunction()
6041 CGF.FinishFunction(); in emitReduceFiniFunction()
6046 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs, in emitTaskReductionInit() argument
6048 if (!CGF.HaveInsertPoint() || Data.ReductionVars.empty()) in emitTaskReductionInit()
6077 Address TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input."); in emitTaskReductionInit()
6084 llvm::Value *GEP = CGF.EmitCheckedInBoundsGEP( in emitTaskReductionInit()
6088 LValue ElemLVal = CGF.MakeNaturalAlignAddrLValue(GEP, RDType); in emitTaskReductionInit()
6090 LValue SharedLVal = CGF.EmitLValueForField(ElemLVal, SharedFD); in emitTaskReductionInit()
6091 RCG.emitSharedLValue(CGF, Cnt); in emitTaskReductionInit()
6093 CGF.EmitCastToVoidPtr(RCG.getSharedLValue(Cnt).getPointer()); in emitTaskReductionInit()
6094 CGF.EmitStoreOfScalar(CastedShared, SharedLVal); in emitTaskReductionInit()
6095 RCG.emitAggregateType(CGF, Cnt); in emitTaskReductionInit()
6106 SizeValInChars = CGF.Builder.CreateIntCast(SizeValInChars, CGM.SizeTy, in emitTaskReductionInit()
6108 LValue SizeLVal = CGF.EmitLValueForField(ElemLVal, SizeFD); in emitTaskReductionInit()
6109 CGF.EmitStoreOfScalar(SizeValInChars, SizeLVal); in emitTaskReductionInit()
6111 LValue InitLVal = CGF.EmitLValueForField(ElemLVal, InitFD); in emitTaskReductionInit()
6113 CGF.EmitCastToVoidPtr(emitReduceInitFunction(CGM, Loc, RCG, Cnt)); in emitTaskReductionInit()
6114 CGF.EmitStoreOfScalar(InitAddr, InitLVal); in emitTaskReductionInit()
6117 LValue FiniLVal = CGF.EmitLValueForField(ElemLVal, FiniFD); in emitTaskReductionInit()
6120 ? CGF.EmitCastToVoidPtr(Fini) in emitTaskReductionInit()
6122 CGF.EmitStoreOfScalar(FiniAddr, FiniLVal); in emitTaskReductionInit()
6124 LValue CombLVal = CGF.EmitLValueForField(ElemLVal, CombFD); in emitTaskReductionInit()
6125 llvm::Value *CombAddr = CGF.EmitCastToVoidPtr(emitReduceCombFunction( in emitTaskReductionInit()
6128 CGF.EmitStoreOfScalar(CombAddr, CombLVal); in emitTaskReductionInit()
6130 LValue FlagsLVal = CGF.EmitLValueForField(ElemLVal, FlagsFD); in emitTaskReductionInit()
6132 CGF.EmitStoreOfScalar( in emitTaskReductionInit()
6136 CGF.EmitNullInitialization(FlagsLVal.getAddress(), FlagsLVal.getType()); in emitTaskReductionInit()
6141 CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), CGM.IntTy, in emitTaskReductionInit()
6144 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskRedInput.getPointer(), in emitTaskReductionInit()
6146 return CGF.EmitRuntimeCall( in emitTaskReductionInit()
6150 void CGOpenMPRuntime::emitTaskReductionFixups(CodeGenFunction &CGF, in emitTaskReductionFixups() argument
6158 llvm::Value *SizeVal = CGF.Builder.CreateIntCast(Sizes.second, CGM.SizeTy, in emitTaskReductionFixups()
6161 CGF, CGM.getContext().getSizeType(), in emitTaskReductionFixups()
6163 CGF.Builder.CreateStore(SizeVal, SizeAddr, /*IsVolatile=*/false); in emitTaskReductionFixups()
6168 CGF, CGM.getContext().VoidPtrTy, in emitTaskReductionFixups()
6170 CGF.Builder.CreateStore( in emitTaskReductionFixups()
6171 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskReductionFixups()
6177 Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF, in getTaskReductionItem() argument
6184 CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), CGM.IntTy, in getTaskReductionItem()
6187 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(SharedLVal.getPointer(), in getTaskReductionItem()
6190 CGF.EmitRuntimeCall( in getTaskReductionItem()
6195 void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, in emitTaskwaitCall() argument
6197 if (!CGF.HaveInsertPoint()) in emitTaskwaitCall()
6201 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitTaskwaitCall()
6203 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskwait), Args); in emitTaskwaitCall()
6204 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitTaskwaitCall()
6205 Region->emitUntiedSwitch(CGF); in emitTaskwaitCall()
6208 void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF, in emitInlinedDirective() argument
6212 if (!CGF.HaveInsertPoint()) in emitInlinedDirective()
6214 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel); in emitInlinedDirective()
6215 CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr); in emitInlinedDirective()
6244 CodeGenFunction &CGF, SourceLocation Loc, in emitCancellationPointCall() argument
6246 if (!CGF.HaveInsertPoint()) in emitCancellationPointCall()
6251 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { in emitCancellationPointCall()
6256 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitCancellationPointCall()
6257 CGF.Builder.getInt32(getCancellationKind(CancelRegion))}; in emitCancellationPointCall()
6259 llvm::Value *Result = CGF.EmitRuntimeCall( in emitCancellationPointCall()
6264 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit"); in emitCancellationPointCall()
6265 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue"); in emitCancellationPointCall()
6266 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result); in emitCancellationPointCall()
6267 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); in emitCancellationPointCall()
6268 CGF.EmitBlock(ExitBB); in emitCancellationPointCall()
6271 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); in emitCancellationPointCall()
6272 CGF.EmitBranchThroughCleanup(CancelDest); in emitCancellationPointCall()
6273 CGF.EmitBlock(ContBB, /*IsFinished=*/true); in emitCancellationPointCall()
6278 void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, in emitCancelCall() argument
6281 if (!CGF.HaveInsertPoint()) in emitCancelCall()
6286 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { in emitCancelCall()
6287 auto &&ThenGen = [Loc, CancelRegion, OMPRegionInfo](CodeGenFunction &CGF, in emitCancelCall()
6289 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitCancelCall()
6291 RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc), in emitCancelCall()
6292 CGF.Builder.getInt32(getCancellationKind(CancelRegion))}; in emitCancelCall()
6294 llvm::Value *Result = CGF.EmitRuntimeCall( in emitCancelCall()
6299 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit"); in emitCancelCall()
6300 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue"); in emitCancelCall()
6301 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result); in emitCancelCall()
6302 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); in emitCancelCall()
6303 CGF.EmitBlock(ExitBB); in emitCancelCall()
6306 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); in emitCancelCall()
6307 CGF.EmitBranchThroughCleanup(CancelDest); in emitCancelCall()
6308 CGF.EmitBlock(ContBB, /*IsFinished=*/true); in emitCancelCall()
6311 emitOMPIfClause(CGF, IfCond, ThenGen, in emitCancelCall()
6315 ThenRCG(CGF); in emitCancelCall()
6356 CodeGenFunction CGF(CGM, true); in emitTargetOutlinedFunctionHelper() local
6358 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitTargetOutlinedFunctionHelper()
6360 OutlinedFn = CGF.GenerateOpenMPCapturedStmtFunction(CS); in emitTargetOutlinedFunctionHelper()
6414 CodeGenFunction &CGF, in emitNumTeamsForTargetDirective() argument
6416 assert(!CGF.getLangOpts().OpenMPIsDevice && "Clauses associated with the " in emitNumTeamsForTargetDirective()
6420 CGBuilderTy &Bld = CGF.Builder; in emitNumTeamsForTargetDirective()
6427 CodeGenFunction::RunCleanupsScope NumTeamsScope(CGF); in emitNumTeamsForTargetDirective()
6428 llvm::Value *NumTeams = CGF.EmitScalarExpr(NumTeamsClause->getNumTeams(), in emitNumTeamsForTargetDirective()
6430 return Bld.CreateIntCast(NumTeams, CGF.Int32Ty, in emitNumTeamsForTargetDirective()
6455 CGOpenMPInnerExprInfo CGInfo(CGF, CS); in emitNumTeamsForTargetDirective()
6456 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitNumTeamsForTargetDirective()
6457 llvm::Value *NumTeams = CGF.EmitScalarExpr(NTE->getNumTeams()); in emitNumTeamsForTargetDirective()
6458 return Bld.CreateIntCast(NumTeams, CGF.Int32Ty, in emitNumTeamsForTargetDirective()
6482 CodeGenFunction &CGF, in emitNumThreadsForTargetDirective() argument
6484 assert(!CGF.getLangOpts().OpenMPIsDevice && "Clauses associated with the " in emitNumThreadsForTargetDirective()
6488 CGBuilderTy &Bld = CGF.Builder; in emitNumThreadsForTargetDirective()
6512 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF); in emitNumThreadsForTargetDirective()
6514 CGF.EmitScalarExpr(ThreadLimitClause->getThreadLimit(), in emitNumThreadsForTargetDirective()
6516 ThreadLimitVal = Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, in emitNumThreadsForTargetDirective()
6522 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); in emitNumThreadsForTargetDirective()
6524 CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(), in emitNumThreadsForTargetDirective()
6527 Bld.CreateIntCast(NumThreads, CGF.Int32Ty, /*IsSigned=*/true); in emitNumThreadsForTargetDirective()
6558 CGOpenMPInnerExprInfo CGInfo(CGF, CS); in emitNumThreadsForTargetDirective()
6559 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitNumThreadsForTargetDirective()
6560 llvm::Value *ThreadLimit = CGF.EmitScalarExpr(TLE->getThreadLimit()); in emitNumThreadsForTargetDirective()
6561 return CGF.Builder.CreateIntCast(ThreadLimit, CGF.Int32Ty, in emitNumThreadsForTargetDirective()
6567 return CGF.Builder.getInt32(0); in emitNumThreadsForTargetDirective()
6689 CodeGenFunction &CGF; member in __anon0274564a3011::MappableExprsHandler
6719 return CGF.getTypeSize(BaseTy); in getExprTypeSize()
6723 ElemSize = CGF.getTypeSize(PTy->getPointeeType().getCanonicalType()); in getExprTypeSize()
6727 ElemSize = CGF.getTypeSize(ATy->getElementType().getCanonicalType()); in getExprTypeSize()
6735 llvm::Value *LengthVal = CGF.EmitScalarExpr(OAE->getLength()); in getExprTypeSize()
6737 CGF.Builder.CreateIntCast(LengthVal, CGF.SizeTy, /*isSigned=*/false); in getExprTypeSize()
6738 return CGF.Builder.CreateNUWMul(LengthVal, ElemSize); in getExprTypeSize()
6740 return CGF.getTypeSize(ExprTy); in getExprTypeSize()
6817 if (!Length->EvaluateAsInt(Result, CGF.getContext())) in isFinalArraySectionExpression()
7014 BP = CGF.LoadCXXThisAddress(); in generateInfoForComponentList()
7018 BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress(); in generateInfoForComponentList()
7022 BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress(); in generateInfoForComponentList()
7029 BP = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetLink(VD); in generateInfoForComponentList()
7039 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>()); in generateInfoForComponentList()
7114 CGF.EmitOMPSharedLValue(I->getAssociatedExpression()).getAddress(); in generateInfoForComponentList()
7133 CharUnits TypeSize = CGF.getContext().getTypeSizeInChars( in generateInfoForComponentList()
7135 Address HB = CGF.Builder.CreateConstGEP( in generateInfoForComponentList()
7136 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(LB, in generateInfoForComponentList()
7137 CGF.VoidPtrTy), in generateInfoForComponentList()
7160 CGF.EmitOMPSharedLValue(MC.getAssociatedExpression()) in generateInfoForComponentList()
7162 Size = CGF.Builder.CreatePtrDiff( in generateInfoForComponentList()
7163 CGF.EmitCastToVoidPtr(ComponentLB.getPointer()), in generateInfoForComponentList()
7164 CGF.EmitCastToVoidPtr(LB.getPointer())); in generateInfoForComponentList()
7172 LB = CGF.Builder.CreateConstGEP(ComponentLB, 1, in generateInfoForComponentList()
7173 CGF.getPointerSize()); in generateInfoForComponentList()
7177 Size = CGF.Builder.CreatePtrDiff( in generateInfoForComponentList()
7178 CGF.EmitCastToVoidPtr( in generateInfoForComponentList()
7179 CGF.Builder.CreateConstGEP(HB, 1, CharUnits::One()) in generateInfoForComponentList()
7181 CGF.EmitCastToVoidPtr(LB.getPointer())); in generateInfoForComponentList()
7294 const CGRecordLayout &RL = CGF.getTypes().getCGRecordLayout(RD); in getPlainLayout()
7310 if (Base->isEmpty() || CGF.getContext() in getPlainLayout()
7352 MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF) in MappableExprsHandler() argument
7353 : CurDir(Dir), CGF(CGF) { in MappableExprsHandler()
7379 llvm::Value *HAddr = CGF.Builder.CreateConstGEP1_32(HB, /*Idx0=*/1); in emitCombinedEntry()
7380 llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(LB, CGF.VoidPtrTy); in emitCombinedEntry()
7381 llvm::Value *CHAddr = CGF.Builder.CreatePointerCast(HAddr, CGF.VoidPtrTy); in emitCombinedEntry()
7382 llvm::Value *Diff = CGF.Builder.CreatePtrDiff(CHAddr, CLAddr); in emitCombinedEntry()
7383 llvm::Value *Size = CGF.Builder.CreateIntCast(Diff, CGF.SizeTy, in emitCombinedEntry()
7496 llvm::Value *Ptr = this->CGF.EmitLoadOfScalar( in generateAllInfo()
7497 this->CGF.EmitLValue(IE), IE->getExprLoc()); in generateAllInfo()
7500 Sizes.push_back(llvm::Constant::getNullValue(this->CGF.SizeTy)); in generateAllInfo()
7552 llvm::Value *BasePtr = this->CGF.EmitLValue(L.IE).getPointer(); in generateAllInfo()
7553 llvm::Value *Ptr = this->CGF.EmitLoadOfScalar( in generateAllInfo()
7554 this->CGF.EmitLValue(L.IE), L.IE->getExprLoc()); in generateAllInfo()
7557 CurSizes.push_back(llvm::Constant::getNullValue(this->CGF.SizeTy)); in generateAllInfo()
7592 Address VDAddr = Address(Arg, CGF.getContext().getDeclAlign(VD)); in generateInfoForLambdaCaptures()
7593 LValue VDLVal = CGF.MakeAddrLValue( in generateInfoForLambdaCaptures()
7600 CGF.EmitLValueForFieldInitialization(VDLVal, ThisCapture); in generateInfoForLambdaCaptures()
7601 LValue ThisLValVal = CGF.EmitLValueForField(VDLVal, ThisCapture); in generateInfoForLambdaCaptures()
7605 Sizes.push_back(CGF.getTypeSize(CGF.getContext().VoidPtrTy)); in generateInfoForLambdaCaptures()
7615 LValue VarLVal = CGF.EmitLValueForFieldInitialization(VDLVal, It->second); in generateInfoForLambdaCaptures()
7616 LValue VarLValVal = CGF.EmitLValueForField(VDLVal, It->second); in generateInfoForLambdaCaptures()
7620 Sizes.push_back(CGF.getTypeSize( in generateInfoForLambdaCaptures()
7678 Sizes.push_back(CGF.getTypeSize(CGF.getContext().VoidPtrTy)); in generateInfoForCapture()
7873 CurSizes.push_back(CGF.getTypeSize(PtrTy->getPointeeType())); in generateDefaultMapInfo()
7883 CurSizes.push_back(CGF.getTypeSize(RI.getType())); in generateDefaultMapInfo()
7888 CurSizes.push_back(llvm::Constant::getNullValue(CGF.SizeTy)); in generateDefaultMapInfo()
7897 CurSizes.push_back(CGF.getTypeSize(ElementType)); in generateDefaultMapInfo()
7922 emitOffloadingArrays(CodeGenFunction &CGF, in emitOffloadingArrays() argument
7928 CodeGenModule &CGM = CGF.CGM; in emitOffloadingArrays()
7929 ASTContext &Ctx = CGF.getContext(); in emitOffloadingArrays()
7951 CGF.CreateMemTemp(PointerArrayType, ".offload_baseptrs").getPointer(); in emitOffloadingArrays()
7953 CGF.CreateMemTemp(PointerArrayType, ".offload_ptrs").getPointer(); in emitOffloadingArrays()
7963 CGF.CreateMemTemp(SizeArrayType, ".offload_sizes").getPointer(); in emitOffloadingArrays()
7987 llvm::ConstantDataArray::get(CGF.Builder.getContext(), Mapping); in emitOffloadingArrays()
7999 llvm::Value *BP = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArrays()
8002 BP = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitOffloadingArrays()
8005 CGF.Builder.CreateStore(BPVal, BPAddr); in emitOffloadingArrays()
8012 llvm::Value *P = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArrays()
8015 P = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitOffloadingArrays()
8018 CGF.Builder.CreateStore(PVal, PAddr); in emitOffloadingArrays()
8021 llvm::Value *S = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArrays()
8027 CGF.Builder.CreateStore( in emitOffloadingArrays()
8028 CGF.Builder.CreateIntCast(Sizes[I], CGM.SizeTy, /*isSigned=*/true), in emitOffloadingArrays()
8037 CodeGenFunction &CGF, llvm::Value *&BasePointersArrayArg, in emitOffloadingArraysArgument() argument
8040 CodeGenModule &CGM = CGF.CGM; in emitOffloadingArraysArgument()
8042 BasePointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
8046 PointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
8051 SizesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
8054 MapTypesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
8215 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *Device, in emitTargetNumIterationsCall() argument
8217 CodeGenFunction &CGF, const OMPLoopDirective &D)> &SizeEmitter) { in emitTargetNumIterationsCall()
8226 auto &&CodeGen = [LD, &Device, &SizeEmitter, this](CodeGenFunction &CGF, in emitTargetNumIterationsCall()
8228 llvm::Value *NumIterations = SizeEmitter(CGF, *LD); in emitTargetNumIterationsCall()
8233 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTargetNumIterationsCall()
8234 CGF.Int64Ty, /*isSigned=*/true); in emitTargetNumIterationsCall()
8236 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTargetNumIterationsCall()
8239 CGF.EmitRuntimeCall( in emitTargetNumIterationsCall()
8242 emitInlinedDirective(CGF, OMPD_unknown, CodeGen); in emitTargetNumIterationsCall()
8245 void CGOpenMPRuntime::emitTargetCall(CodeGenFunction &CGF, in emitTargetCall() argument
8250 if (!CGF.HaveInsertPoint()) in emitTargetCall()
8258 auto &&ArgsCodegen = [&CS, &CapturedVars](CodeGenFunction &CGF, in emitTargetCall()
8260 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars); in emitTargetCall()
8262 emitInlinedDirective(CGF, OMPD_unknown, ArgsCodegen); in emitTargetCall()
8269 &CapturedVars](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetCall() argument
8285 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTargetCall()
8286 CGF.Int64Ty, /*isSigned=*/true); in emitTargetCall()
8288 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTargetCall()
8293 CGF.Builder.getInt32(InputInfo.NumberOfTargetItems); in emitTargetCall()
8298 llvm::Value *NumTeams = emitNumTeamsForTargetDirective(*this, CGF, D); in emitTargetCall()
8299 llvm::Value *NumThreads = emitNumThreadsForTargetDirective(*this, CGF, D); in emitTargetCall()
8346 Return = CGF.EmitRuntimeCall( in emitTargetCall()
8358 Return = CGF.EmitRuntimeCall( in emitTargetCall()
8366 CGF.createBasicBlock("omp_offload.failed"); in emitTargetCall()
8368 CGF.createBasicBlock("omp_offload.cont"); in emitTargetCall()
8369 llvm::Value *Failed = CGF.Builder.CreateIsNotNull(Return); in emitTargetCall()
8370 CGF.Builder.CreateCondBr(Failed, OffloadFailedBlock, OffloadContBlock); in emitTargetCall()
8372 CGF.EmitBlock(OffloadFailedBlock); in emitTargetCall()
8375 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars); in emitTargetCall()
8377 emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn, CapturedVars); in emitTargetCall()
8378 CGF.EmitBranch(OffloadContBlock); in emitTargetCall()
8380 CGF.EmitBlock(OffloadContBlock, /*IsFinished=*/true); in emitTargetCall()
8385 RequiresOuterTask](CodeGenFunction &CGF, in emitTargetCall()
8389 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars); in emitTargetCall()
8391 emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn, CapturedVars); in emitTargetCall()
8396 &CS](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetCall() argument
8404 MappableExprsHandler MEHandler(D, CGF); in emitTargetCall()
8423 CurSizes.push_back(CGF.getTypeSize(RI->getType())); in emitTargetCall()
8472 emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info); in emitTargetCall()
8473 emitOffloadingArraysArgument(CGF, Info.BasePointersArray, in emitTargetCall()
8484 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo); in emitTargetCall()
8486 emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen); in emitTargetCall()
8490 CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetCall() argument
8493 CGF.EmitOMPTargetTaskBasedDirective(D, ElseGen, InputInfo); in emitTargetCall()
8495 emitInlinedDirective(CGF, D.getDirectiveKind(), ElseGen); in emitTargetCall()
8505 emitOMPIfClause(CGF, IfCond, TargetThenGen, TargetElseGen); in emitTargetCall()
8508 ThenRCG(CGF); in emitTargetCall()
8512 ElseRCG(CGF); in emitTargetCall()
8786 CodeGenFunction &CGF, const OMPExecutableDirective &D) const { in adjustTargetSpecificDataForLambdas() argument
8836 void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF, in emitTeamsCall() argument
8841 if (!CGF.HaveInsertPoint()) in emitTeamsCall()
8844 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitTeamsCall()
8845 CodeGenFunction::RunCleanupsScope Scope(CGF); in emitTeamsCall()
8850 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars in emitTeamsCall()
8851 CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy())}; in emitTeamsCall()
8857 CGF.EmitRuntimeCall(RTLFn, RealArgs); in emitTeamsCall()
8860 void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF, in emitNumTeamsClause() argument
8864 if (!CGF.HaveInsertPoint()) in emitNumTeamsClause()
8867 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitNumTeamsClause()
8871 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(NumTeams), in emitNumTeamsClause()
8872 CGF.CGM.Int32Ty, /* isSigned = */ true) in emitNumTeamsClause()
8873 : CGF.Builder.getInt32(0); in emitNumTeamsClause()
8877 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(ThreadLimit), in emitNumTeamsClause()
8878 CGF.CGM.Int32Ty, /* isSigned = */ true) in emitNumTeamsClause()
8879 : CGF.Builder.getInt32(0); in emitNumTeamsClause()
8882 llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal, in emitNumTeamsClause()
8884 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_teams), in emitNumTeamsClause()
8889 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, in emitTargetDataCalls() argument
8891 if (!CGF.HaveInsertPoint()) in emitTargetDataCalls()
8902 &CodeGen](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetDataCalls() argument
8910 MappableExprsHandler MCHandler(D, CGF); in emitTargetDataCalls()
8914 emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info); in emitTargetDataCalls()
8920 emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg, in emitTargetDataCalls()
8926 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTargetDataCalls()
8927 CGF.Int64Ty, /*isSigned=*/true); in emitTargetDataCalls()
8929 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTargetDataCalls()
8933 llvm::Value *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs); in emitTargetDataCalls()
8938 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_target_data_begin), in emitTargetDataCalls()
8944 CodeGen(CGF); in emitTargetDataCalls()
8948 auto &&EndThenGen = [this, Device, &Info](CodeGenFunction &CGF, in emitTargetDataCalls()
8956 emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg, in emitTargetDataCalls()
8962 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTargetDataCalls()
8963 CGF.Int64Ty, /*isSigned=*/true); in emitTargetDataCalls()
8965 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTargetDataCalls()
8969 llvm::Value *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs); in emitTargetDataCalls()
8974 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_target_data_end), in emitTargetDataCalls()
8981 auto &&BeginElseGen = [&Info, &CodeGen, &NoPrivAction](CodeGenFunction &CGF, in emitTargetDataCalls()
8985 CodeGen(CGF); in emitTargetDataCalls()
8991 auto &&EndElseGen = [](CodeGenFunction &CGF, PrePostActionTy &) {}; in emitTargetDataCalls() argument
8994 emitOMPIfClause(CGF, IfCond, BeginThenGen, BeginElseGen); in emitTargetDataCalls()
8997 RCG(CGF); in emitTargetDataCalls()
9004 CodeGen(CGF); in emitTargetDataCalls()
9008 emitOMPIfClause(CGF, IfCond, EndThenGen, EndElseGen); in emitTargetDataCalls()
9011 RCG(CGF); in emitTargetDataCalls()
9016 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, in emitTargetDataStandAloneCall() argument
9018 if (!CGF.HaveInsertPoint()) in emitTargetDataStandAloneCall()
9030 &MapTypesArray](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetDataStandAloneCall() argument
9034 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTargetDataStandAloneCall()
9035 CGF.Int64Ty, /*isSigned=*/true); in emitTargetDataStandAloneCall()
9037 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTargetDataStandAloneCall()
9042 CGF.Builder.getInt32(InputInfo.NumberOfTargetItems); in emitTargetDataStandAloneCall()
9122 CGF.EmitRuntimeCall(createRuntimeFunction(RTLFn), OffloadingArgs); in emitTargetDataStandAloneCall()
9126 CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetDataStandAloneCall() argument
9134 MappableExprsHandler MEHandler(D, CGF); in emitTargetDataStandAloneCall()
9139 emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info); in emitTargetDataStandAloneCall()
9140 emitOffloadingArraysArgument(CGF, Info.BasePointersArray, in emitTargetDataStandAloneCall()
9152 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo); in emitTargetDataStandAloneCall()
9154 emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen); in emitTargetDataStandAloneCall()
9158 emitOMPIfClause(CGF, IfCond, TargetThenGen, in emitTargetDataStandAloneCall()
9159 [](CodeGenFunction &CGF, PrePostActionTy &) {}); in emitTargetDataStandAloneCall() argument
9162 ThenRCG(CGF); in emitTargetDataStandAloneCall()
9415 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { in Emit() argument
9416 if (!CGF.HaveInsertPoint()) in Emit()
9418 CGF.EmitRuntimeCall(RTLFn, Args); in Emit()
9423 void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF, in emitDoacrossInit() argument
9426 if (!CGF.HaveInsertPoint()) in emitDoacrossInit()
9452 Address DimsAddr = CGF.CreateMemTemp(ArrayTy, "dims"); in emitDoacrossInit()
9453 CGF.EmitNullInitialization(DimsAddr, ArrayTy); in emitDoacrossInit()
9458 CGF.MakeAddrLValue(CGF.Builder.CreateConstArrayGEP( in emitDoacrossInit()
9462 LValue UpperLVal = CGF.EmitLValueForField( in emitDoacrossInit()
9465 CGF.EmitScalarConversion(CGF.EmitScalarExpr(NumIterations[I]), in emitDoacrossInit()
9468 CGF.EmitStoreOfScalar(NumIterVal, UpperLVal); in emitDoacrossInit()
9470 LValue StrideLVal = CGF.EmitLValueForField( in emitDoacrossInit()
9472 CGF.EmitStoreOfScalar(llvm::ConstantInt::getSigned(CGM.Int64Ty, /*V=*/1), in emitDoacrossInit()
9479 emitUpdateLocation(CGF, D.getBeginLoc()), in emitDoacrossInit()
9480 getThreadID(CGF, D.getBeginLoc()), in emitDoacrossInit()
9482 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDoacrossInit()
9483 CGF.Builder in emitDoacrossInit()
9489 CGF.EmitRuntimeCall(RTLFn, Args); in emitDoacrossInit()
9491 emitUpdateLocation(CGF, D.getEndLoc()), getThreadID(CGF, D.getEndLoc())}; in emitDoacrossInit()
9493 CGF.EHStack.pushCleanup<DoacrossCleanupTy>(NormalAndEHCleanup, FiniRTLFn, in emitDoacrossInit()
9497 void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, in emitDoacrossOrdered() argument
9504 Address CntAddr = CGF.CreateMemTemp(ArrayTy, ".cnt.addr"); in emitDoacrossOrdered()
9508 llvm::Value *CntVal = CGF.EmitScalarConversion( in emitDoacrossOrdered()
9509 CGF.EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty, in emitDoacrossOrdered()
9511 CGF.EmitStoreOfScalar( in emitDoacrossOrdered()
9513 CGF.Builder.CreateConstArrayGEP( in emitDoacrossOrdered()
9518 emitUpdateLocation(CGF, C->getBeginLoc()), in emitDoacrossOrdered()
9519 getThreadID(CGF, C->getBeginLoc()), in emitDoacrossOrdered()
9520 CGF.Builder in emitDoacrossOrdered()
9531 CGF.EmitRuntimeCall(RTLFn, Args); in emitDoacrossOrdered()
9534 void CGOpenMPRuntime::emitCall(CodeGenFunction &CGF, SourceLocation Loc, in emitCall() argument
9538 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc); in emitCall()
9542 CGF.EmitNounwindRuntimeCall(Fn, Args); in emitCall()
9546 CGF.EmitRuntimeCall(Callee, Args); in emitCall()
9550 CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *OutlinedFn, in emitOutlinedFunctionCall() argument
9552 emitCall(CGF, Loc, OutlinedFn, Args); in emitOutlinedFunctionCall()
9555 Address CGOpenMPRuntime::getParameterAddress(CodeGenFunction &CGF, in getParameterAddress() argument
9558 return CGF.GetAddrOfLocalVar(NativeParam); in getParameterAddress()
9561 Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF, in getAddressOfLocalVariable() argument
9586 void CGOpenMPSIMDRuntime::emitParallelCall(CodeGenFunction &CGF, in emitParallelCall() argument
9595 CodeGenFunction &CGF, StringRef CriticalName, in emitCriticalRegion() argument
9601 void CGOpenMPSIMDRuntime::emitMasterRegion(CodeGenFunction &CGF, in emitMasterRegion() argument
9607 void CGOpenMPSIMDRuntime::emitTaskyieldCall(CodeGenFunction &CGF, in emitTaskyieldCall() argument
9613 CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, in emitTaskgroupRegion() argument
9619 CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, in emitSingleRegion() argument
9626 void CGOpenMPSIMDRuntime::emitOrderedRegion(CodeGenFunction &CGF, in emitOrderedRegion() argument
9633 void CGOpenMPSIMDRuntime::emitBarrierCall(CodeGenFunction &CGF, in emitBarrierCall() argument
9642 CodeGenFunction &CGF, SourceLocation Loc, in emitForDispatchInit() argument
9649 CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, in emitForStaticInit() argument
9655 CodeGenFunction &CGF, SourceLocation Loc, in emitDistributeStaticInit() argument
9660 void CGOpenMPSIMDRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF, in emitForOrderedIterationEnd() argument
9667 void CGOpenMPSIMDRuntime::emitForStaticFinish(CodeGenFunction &CGF, in emitForStaticFinish() argument
9673 llvm::Value *CGOpenMPSIMDRuntime::emitForNext(CodeGenFunction &CGF, in emitForNext() argument
9681 void CGOpenMPSIMDRuntime::emitNumThreadsClause(CodeGenFunction &CGF, in emitNumThreadsClause() argument
9687 void CGOpenMPSIMDRuntime::emitProcBindClause(CodeGenFunction &CGF, in emitProcBindClause() argument
9693 Address CGOpenMPSIMDRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, in getAddrOfThreadPrivate() argument
9702 CodeGenFunction *CGF) { in emitThreadPrivateVarDefinition() argument
9707 CodeGenFunction &CGF, QualType VarType, StringRef Name) { in getAddrOfArtificialThreadPrivate() argument
9711 void CGOpenMPSIMDRuntime::emitFlush(CodeGenFunction &CGF, in emitFlush() argument
9717 void CGOpenMPSIMDRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskCall() argument
9727 CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, in emitTaskLoopCall() argument
9734 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates, in emitReduction() argument
9738 CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs, in emitReduction()
9743 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs, in emitTaskReductionInit() argument
9748 void CGOpenMPSIMDRuntime::emitTaskReductionFixups(CodeGenFunction &CGF, in emitTaskReductionFixups() argument
9755 Address CGOpenMPSIMDRuntime::getTaskReductionItem(CodeGenFunction &CGF, in getTaskReductionItem() argument
9762 void CGOpenMPSIMDRuntime::emitTaskwaitCall(CodeGenFunction &CGF, in emitTaskwaitCall() argument
9768 CodeGenFunction &CGF, SourceLocation Loc, in emitCancellationPointCall() argument
9773 void CGOpenMPSIMDRuntime::emitCancelCall(CodeGenFunction &CGF, in emitCancelCall() argument
9786 void CGOpenMPSIMDRuntime::emitTargetCall(CodeGenFunction &CGF, in emitTargetCall() argument
9810 void CGOpenMPSIMDRuntime::emitTeamsCall(CodeGenFunction &CGF, in emitTeamsCall() argument
9818 void CGOpenMPSIMDRuntime::emitNumTeamsClause(CodeGenFunction &CGF, in emitNumTeamsClause() argument
9826 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, in emitTargetDataCalls() argument
9832 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, in emitTargetDataStandAloneCall() argument
9837 void CGOpenMPSIMDRuntime::emitDoacrossInit(CodeGenFunction &CGF, in emitDoacrossInit() argument
9843 void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, in emitDoacrossOrdered() argument
9855 CGOpenMPSIMDRuntime::getParameterAddress(CodeGenFunction &CGF, in getParameterAddress() argument