Lines Matching refs:CGF
88 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
92 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
161 void Enter(CodeGenFunction &CGF) override { in Enter() argument
164 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue( in Enter()
165 CGF.GetAddrOfLocalVar(PartIDVar), in Enter()
168 CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation()); in Enter()
169 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done."); in Enter()
170 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB); in Enter()
171 CGF.EmitBlock(DoneBB); in Enter()
172 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock); in Enter()
173 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp.")); in Enter()
174 UntiedSwitch->addCase(CGF.Builder.getInt32(0), in Enter()
175 CGF.Builder.GetInsertBlock()); in Enter()
176 emitUntiedSwitch(CGF); in Enter()
179 void emitUntiedSwitch(CodeGenFunction &CGF) const { in emitUntiedSwitch()
181 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue( in emitUntiedSwitch()
182 CGF.GetAddrOfLocalVar(PartIDVar), in emitUntiedSwitch()
184 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()), in emitUntiedSwitch()
186 UntiedCodeGen(CGF); in emitUntiedSwitch()
188 CGF.getJumpDestInCurrentScope(".untied.next."); in emitUntiedSwitch()
189 CGF.EmitBranch(CGF.ReturnBlock.getBlock()); in emitUntiedSwitch()
190 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp.")); in emitUntiedSwitch()
191 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()), in emitUntiedSwitch()
192 CGF.Builder.GetInsertBlock()); in emitUntiedSwitch()
193 CGF.EmitBranchThroughCleanup(CurPoint); in emitUntiedSwitch()
194 CGF.EmitBlock(CurPoint.getBlock()); in emitUntiedSwitch()
214 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
219 void emitUntiedSwitch(CodeGenFunction &CGF) override { in emitUntiedSwitch() argument
220 Action.emitUntiedSwitch(CGF); in emitUntiedSwitch()
287 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override { in getThreadIDVariableLValue() argument
289 return OuterRegionInfo->getThreadIDVariableLValue(CGF); in getThreadIDVariableLValue()
300 void emitUntiedSwitch(CodeGenFunction &CGF) override { in emitUntiedSwitch() argument
302 OuterRegionInfo->emitUntiedSwitch(CGF); in emitUntiedSwitch()
356 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS) in CGOpenMPInnerExprInfo() argument
357 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen, in CGOpenMPInnerExprInfo()
360 PrivScope(CGF) { in CGOpenMPInnerExprInfo()
372 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD), in CGOpenMPInnerExprInfo()
376 PrivScope.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress(CGF)); in CGOpenMPInnerExprInfo()
389 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override { in EmitBody() argument
413 CodeGenFunction &CGF; member in __anon90c6c1b10111::InlinedOpenMPRegionRAII
424 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen, in InlinedOpenMPRegionRAII() argument
427 : CGF(CGF), NoInheritance(NoInheritance) { in InlinedOpenMPRegionRAII()
429 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo( in InlinedOpenMPRegionRAII()
430 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel); in InlinedOpenMPRegionRAII()
432 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields); in InlinedOpenMPRegionRAII()
433 LambdaThisCaptureField = CGF.LambdaThisCaptureField; in InlinedOpenMPRegionRAII()
434 CGF.LambdaThisCaptureField = nullptr; in InlinedOpenMPRegionRAII()
435 BlockInfo = CGF.BlockInfo; in InlinedOpenMPRegionRAII()
436 CGF.BlockInfo = nullptr; in InlinedOpenMPRegionRAII()
443 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI(); in ~InlinedOpenMPRegionRAII()
444 delete CGF.CapturedStmtInfo; in ~InlinedOpenMPRegionRAII()
445 CGF.CapturedStmtInfo = OldCSI; in ~InlinedOpenMPRegionRAII()
447 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields); in ~InlinedOpenMPRegionRAII()
448 CGF.LambdaThisCaptureField = LambdaThisCaptureField; in ~InlinedOpenMPRegionRAII()
449 CGF.BlockInfo = BlockInfo; in ~InlinedOpenMPRegionRAII()
563 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { in Emit() argument
564 if (!CGF.HaveInsertPoint()) in Emit()
566 Action->Exit(CGF); in Emit()
572 void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const { in operator ()()
573 CodeGenFunction::RunCleanupsScope Scope(CGF); in operator ()()
575 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction); in operator ()()
576 Callback(CodeGen, CGF, *PrePostAction); in operator ()()
579 Callback(CodeGen, CGF, Action); in operator ()()
596 static void emitInitWithReductionInitializer(CodeGenFunction &CGF, in emitInitWithReductionInitializer() argument
603 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD); in emitInitWithReductionInitializer()
612 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitInitWithReductionInitializer()
617 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func); in emitInitWithReductionInitializer()
618 CGF.EmitIgnoredExpr(InitOp); in emitInitWithReductionInitializer()
620 llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty); in emitInitWithReductionInitializer()
621 std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"}); in emitInitWithReductionInitializer()
623 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true, in emitInitWithReductionInitializer()
625 LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty); in emitInitWithReductionInitializer()
627 switch (CGF.getEvaluationKind(Ty)) { in emitInitWithReductionInitializer()
629 InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation()); in emitInitWithReductionInitializer()
633 RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation())); in emitInitWithReductionInitializer()
637 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, LV); in emitInitWithReductionInitializer()
638 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(), in emitInitWithReductionInitializer()
644 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal); in emitInitWithReductionInitializer()
645 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(), in emitInitWithReductionInitializer()
655 static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, in EmitOMPAggregateInit() argument
665 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr); in EmitOMPAggregateInit()
675 CGF.Builder.CreateGEP(DestAddr.getElementType(), DestBegin, NumElements); in EmitOMPAggregateInit()
677 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body"); in EmitOMPAggregateInit()
678 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done"); in EmitOMPAggregateInit()
680 CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty"); in EmitOMPAggregateInit()
681 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); in EmitOMPAggregateInit()
684 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock(); in EmitOMPAggregateInit()
685 CGF.EmitBlock(BodyBB); in EmitOMPAggregateInit()
687 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy); in EmitOMPAggregateInit()
692 SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2, in EmitOMPAggregateInit()
699 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI( in EmitOMPAggregateInit()
708 CodeGenFunction::RunCleanupsScope InitScope(CGF); in EmitOMPAggregateInit()
710 emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent, in EmitOMPAggregateInit()
713 CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(), in EmitOMPAggregateInit()
719 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateInit()
722 SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateInit()
726 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateInit()
731 CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done"); in EmitOMPAggregateInit()
732 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB); in EmitOMPAggregateInit()
733 DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateInit()
736 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); in EmitOMPAggregateInit()
739 LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) { in emitSharedLValue() argument
740 return CGF.EmitOMPSharedLValue(E); in emitSharedLValue()
743 LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF, in emitSharedLValueUB() argument
746 return CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false); in emitSharedLValueUB()
751 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, in emitAggregateInitialization() argument
760 EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(), in emitAggregateInitialization()
786 void ReductionCodeGen::emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N) { in emitSharedOrigLValue() argument
789 LValue First = emitSharedLValue(CGF, ClausesData[N].Shared); in emitSharedOrigLValue()
790 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Shared); in emitSharedOrigLValue()
795 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref); in emitSharedOrigLValue()
796 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref); in emitSharedOrigLValue()
801 void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) { in emitAggregateType() argument
806 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()), in emitAggregateType()
812 auto *ElemType = OrigAddresses[N].first.getAddress(CGF).getElementType(); in emitAggregateType()
815 Size = CGF.Builder.CreatePtrDiff(ElemType, in emitAggregateType()
816 OrigAddresses[N].second.getPointer(CGF), in emitAggregateType()
817 OrigAddresses[N].first.getPointer(CGF)); in emitAggregateType()
818 Size = CGF.Builder.CreateNUWAdd( in emitAggregateType()
820 SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf); in emitAggregateType()
823 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()); in emitAggregateType()
824 Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf); in emitAggregateType()
828 CGF, in emitAggregateType()
830 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()), in emitAggregateType()
832 CGF.EmitVariablyModifiedType(PrivateType); in emitAggregateType()
835 void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N, in emitAggregateType() argument
845 CGF, in emitAggregateType()
847 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()), in emitAggregateType()
849 CGF.EmitVariablyModifiedType(PrivateType); in emitAggregateType()
853 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, in emitInitialization() argument
860 if (CGF.getContext().getAsArrayType(PrivateVD->getType())) { in emitInitialization()
862 (void)DefaultInit(CGF); in emitInitialization()
863 emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD); in emitInitialization()
865 (void)DefaultInit(CGF); in emitInitialization()
867 emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp, in emitInitialization()
869 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() && in emitInitialization()
870 !CGF.isTrivialInitializer(PrivateVD->getInit())) { in emitInitialization()
871 CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr, in emitInitialization()
883 void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N, in emitCleanups() argument
889 PrivateAddr.withElementType(CGF.ConvertTypeForMem(PrivateType)); in emitCleanups()
890 CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType); in emitCleanups()
894 static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, in loadToBegin() argument
898 !CGF.getContext().hasSameType(BaseTy, ElTy)) { in loadToBegin()
900 BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(CGF), PtrTy); in loadToBegin()
902 LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(CGF), BaseTy); in loadToBegin()
903 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal); in loadToBegin()
907 return CGF.MakeAddrLValue( in loadToBegin()
908 BaseLV.getAddress(CGF).withElementType(CGF.ConvertTypeForMem(ElTy)), in loadToBegin()
910 CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType())); in loadToBegin()
913 static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, in castToBase() argument
920 !CGF.getContext().hasSameType(BaseTy, ElTy)) { in castToBase()
921 Tmp = CGF.CreateMemTemp(BaseTy); in castToBase()
923 CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp); in castToBase()
931 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in castToBase()
933 CGF.Builder.CreateStore(Addr, Tmp); in castToBase()
937 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in castToBase()
962 Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, in adjustPrivateAddress() argument
967 LValue OriginalBaseLValue = CGF.EmitLValue(DE); in adjustPrivateAddress()
969 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(), in adjustPrivateAddress()
971 Address SharedAddr = SharedAddresses[N].first.getAddress(CGF); in adjustPrivateAddress()
972 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff( in adjustPrivateAddress()
973 SharedAddr.getElementType(), BaseLValue.getPointer(CGF), in adjustPrivateAddress()
976 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in adjustPrivateAddress()
978 llvm::Value *Ptr = CGF.Builder.CreateGEP( in adjustPrivateAddress()
980 return castToBase(CGF, OrigVD->getType(), in adjustPrivateAddress()
982 OriginalBaseLValue.getAddress(CGF), Ptr); in adjustPrivateAddress()
995 LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) { in getThreadIDVariableLValue() argument
996 return CGF.EmitLoadOfPointerLValue( in getThreadIDVariableLValue()
997 CGF.GetAddrOfLocalVar(getThreadIDVariable()), in getThreadIDVariableLValue()
1001 void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt *S) { in EmitBody() argument
1002 if (!CGF.HaveInsertPoint()) in EmitBody()
1009 CGF.EHStack.pushTerminate(); in EmitBody()
1011 CGF.incrementProfileCounter(S); in EmitBody()
1012 CodeGen(CGF); in EmitBody()
1013 CGF.EHStack.popTerminate(); in EmitBody()
1017 CodeGenFunction &CGF) { in getThreadIDVariableLValue() argument
1018 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()), in getThreadIDVariableLValue()
1102 CodeGenFunction CGF(CGM); in emitCombinerOrInitializer() local
1105 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(), in emitCombinerOrInitializer()
1107 CodeGenFunction::OMPPrivateScope Scope(CGF); in emitCombinerOrInitializer()
1108 Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm); in emitCombinerOrInitializer()
1110 In, CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>()) in emitCombinerOrInitializer()
1111 .getAddress(CGF)); in emitCombinerOrInitializer()
1112 Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm); in emitCombinerOrInitializer()
1114 Out, CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>()) in emitCombinerOrInitializer()
1115 .getAddress(CGF)); in emitCombinerOrInitializer()
1118 !CGF.isTrivialInitializer(Out->getInit())) { in emitCombinerOrInitializer()
1119 CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out), in emitCombinerOrInitializer()
1124 CGF.EmitIgnoredExpr(CombinerInitializer); in emitCombinerOrInitializer()
1126 CGF.FinishFunction(); in emitCombinerOrInitializer()
1131 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) { in emitUserDefinedReduction() argument
1150 if (CGF) { in emitUserDefinedReduction()
1151 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn); in emitUserDefinedReduction()
1169 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder, CodeGenFunction &CGF, in PushAndPopStackRAII()
1187 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) { in PushAndPopStackRAII()
1190 CGBuilderTy::InsertPointGuard IPG(CGF.Builder); in PushAndPopStackRAII()
1191 CGF.Builder.restoreIP(IP); in PushAndPopStackRAII()
1193 CGF.getOMPCancelDestination(OMPD_parallel); in PushAndPopStackRAII()
1194 CGF.EmitBranchThroughCleanup(Dest); in PushAndPopStackRAII()
1216 CodeGenFunction CGF(CGM, true); in emitParallelOrTeamsOutlinedFunction() local
1240 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind); in emitParallelOrTeamsOutlinedFunction()
1243 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitParallelOrTeamsOutlinedFunction()
1244 return CGF.GenerateOpenMPCapturedStmtFunction(*CS, D.getBeginLoc()); in emitParallelOrTeamsOutlinedFunction()
1252 std::string CGOpenMPRuntime::getOutlinedHelperName(CodeGenFunction &CGF) const { in getOutlinedHelperName()
1253 return getOutlinedHelperName(CGF.CurFn->getName()); in getOutlinedHelperName()
1262 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitParallelOutlinedFunction() argument
1267 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(CGF), in emitParallelOutlinedFunction()
1272 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitTeamsOutlinedFunction() argument
1277 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(CGF), in emitTeamsOutlinedFunction()
1286 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF, in emitTaskOutlinedFunction()
1288 llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc()); in emitTaskOutlinedFunction()
1289 llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTaskOutlinedFunction()
1292 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar), in emitTaskOutlinedFunction()
1294 .getPointer(CGF)}; in emitTaskOutlinedFunction()
1295 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskOutlinedFunction()
1318 CodeGenFunction CGF(CGM, true); in emitTaskOutlinedFunction() local
1321 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitTaskOutlinedFunction()
1322 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS); in emitTaskOutlinedFunction()
1328 void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF, in setLocThreadIdInsertPt() argument
1330 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in setLocThreadIdInsertPt()
1333 llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty); in setLocThreadIdInsertPt()
1336 Undef, CGF.Int32Ty, "svcpt", CGF.Builder.GetInsertBlock()); in setLocThreadIdInsertPt()
1339 new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt"); in setLocThreadIdInsertPt()
1340 Elem.second.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt); in setLocThreadIdInsertPt()
1344 void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) { in clearLocThreadIdInsertPt() argument
1345 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in clearLocThreadIdInsertPt()
1353 static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF, in getIdentStringFromSourceLocation() argument
1358 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc); in getIdentStringFromSourceLocation()
1360 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl)) in getIdentStringFromSourceLocation()
1366 llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF, in emitUpdateLocation() argument
1377 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl)) in emitUpdateLocation()
1379 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc); in emitUpdateLocation()
1391 llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF, in getThreadID() argument
1393 assert(CGF.CurFn && "No function in current CodeGenFunction."); in getThreadID()
1398 OMPBuilder.updateToLocation(CGF.Builder.saveIP()); in getThreadID()
1401 getIdentStringFromSourceLocation(CGF, Loc, Buffer), SrcLocStrSize); in getThreadID()
1409 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn); in getThreadID()
1417 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { in getThreadID()
1420 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF); in getThreadID()
1421 llvm::BasicBlock *TopBlock = CGF.AllocaInsertPt->getParent(); in getThreadID()
1422 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions || in getThreadID()
1423 !CGF.getLangOpts().CXXExceptions || in getThreadID()
1424 CGF.Builder.GetInsertBlock() == TopBlock || in getThreadID()
1425 !isa<llvm::Instruction>(LVal.getPointer(CGF)) || in getThreadID()
1426 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() == in getThreadID()
1428 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() == in getThreadID()
1429 CGF.Builder.GetInsertBlock()) { in getThreadID()
1430 ThreadID = CGF.EmitLoadOfScalar(LVal, Loc); in getThreadID()
1433 if (CGF.Builder.GetInsertBlock() == TopBlock) { in getThreadID()
1434 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in getThreadID()
1446 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in getThreadID()
1448 setLocThreadIdInsertPt(CGF); in getThreadID()
1449 CGBuilderTy::InsertPointGuard IPG(CGF.Builder); in getThreadID()
1450 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt); in getThreadID()
1451 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc); in getThreadID()
1452 llvm::CallInst *Call = CGF.Builder.CreateCall( in getThreadID()
1455 emitUpdateLocation(CGF, Loc)); in getThreadID()
1456 Call->setCallingConv(CGF.getRuntimeCC()); in getThreadID()
1461 void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) { in functionFinished() argument
1462 assert(CGF.CurFn && "No function in current CodeGenFunction."); in functionFinished()
1463 if (OpenMPLocThreadIDMap.count(CGF.CurFn)) { in functionFinished()
1464 clearLocThreadIdInsertPt(CGF); in functionFinished()
1465 OpenMPLocThreadIDMap.erase(CGF.CurFn); in functionFinished()
1467 if (FunctionUDRMap.count(CGF.CurFn) > 0) { in functionFinished()
1468 for(const auto *D : FunctionUDRMap[CGF.CurFn]) in functionFinished()
1470 FunctionUDRMap.erase(CGF.CurFn); in functionFinished()
1472 auto I = FunctionUDMMap.find(CGF.CurFn); in functionFinished()
1478 LastprivateConditionalToTypes.erase(CGF.CurFn); in functionFinished()
1479 FunctionToUntiedTaskStackMap.erase(CGF.CurFn); in functionFinished()
1596 Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, in getAddrOfThreadPrivate() argument
1606 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in getAddrOfThreadPrivate()
1607 CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy), in getAddrOfThreadPrivate()
1611 CGF.EmitRuntimeCall( in getAddrOfThreadPrivate()
1615 CGF.Int8Ty, VDAddr.getAlignment()); in getAddrOfThreadPrivate()
1619 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor, in emitThreadPrivateVarInit() argument
1623 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc); in emitThreadPrivateVarInit()
1624 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitThreadPrivateVarInit()
1630 OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy), in emitThreadPrivateVarInit()
1632 CGF.EmitRuntimeCall( in emitThreadPrivateVarInit()
1640 bool PerformInit, CodeGenFunction *CGF) { in emitThreadPrivateVarDefinition() argument
1738 if (!CGF) { in emitThreadPrivateVarDefinition()
1753 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc); in emitThreadPrivateVarDefinition()
1793 Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, in getAddrOfArtificialThreadPrivate() argument
1797 llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType); in getAddrOfArtificialThreadPrivate()
1808 emitUpdateLocation(CGF, SourceLocation()), in getAddrOfArtificialThreadPrivate()
1809 getThreadID(CGF, SourceLocation()), in getAddrOfArtificialThreadPrivate()
1810 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy), in getAddrOfArtificialThreadPrivate()
1811 CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy, in getAddrOfArtificialThreadPrivate()
1817 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddrOfArtificialThreadPrivate()
1818 CGF.EmitRuntimeCall( in getAddrOfArtificialThreadPrivate()
1826 void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond, in emitIfClause() argument
1829 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange()); in emitIfClause()
1834 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) { in emitIfClause()
1836 ThenGen(CGF); in emitIfClause()
1838 ElseGen(CGF); in emitIfClause()
1844 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then"); in emitIfClause()
1845 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else"); in emitIfClause()
1846 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end"); in emitIfClause()
1847 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0); in emitIfClause()
1850 CGF.EmitBlock(ThenBlock); in emitIfClause()
1851 ThenGen(CGF); in emitIfClause()
1852 CGF.EmitBranch(ContBlock); in emitIfClause()
1855 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitIfClause()
1856 CGF.EmitBlock(ElseBlock); in emitIfClause()
1857 ElseGen(CGF); in emitIfClause()
1859 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitIfClause()
1860 CGF.EmitBranch(ContBlock); in emitIfClause()
1862 CGF.EmitBlock(ContBlock, /*IsFinished=*/true); in emitIfClause()
1865 void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, in emitParallelCall() argument
1870 if (!CGF.HaveInsertPoint()) in emitParallelCall()
1872 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitParallelCall()
1875 this](CodeGenFunction &CGF, PrePostActionTy &) { in emitParallelCall() argument
1877 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitParallelCall()
1880 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars in emitParallelCall()
1881 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())}; in emitParallelCall()
1888 CGF.EmitRuntimeCall(RTLFn, RealArgs); in emitParallelCall()
1891 this](CodeGenFunction &CGF, PrePostActionTy &) { in emitParallelCall() argument
1892 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitParallelCall()
1893 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc); in emitParallelCall()
1897 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitParallelCall()
1902 Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc); in emitParallelCall()
1904 CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, in emitParallelCall()
1906 CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddrBound); in emitParallelCall()
1921 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs); in emitParallelCall()
1924 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID}; in emitParallelCall()
1925 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitParallelCall()
1930 emitIfClause(CGF, IfCond, ThenGen, ElseGen); in emitParallelCall()
1933 ThenRCG(CGF); in emitParallelCall()
1943 Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF, in emitThreadIDAddress() argument
1946 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitThreadIDAddress()
1948 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(CGF); in emitThreadIDAddress()
1950 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitThreadIDAddress()
1952 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true); in emitThreadIDAddress()
1953 Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp."); in emitThreadIDAddress()
1954 CGF.EmitStoreOfScalar(ThreadID, in emitThreadIDAddress()
1955 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty)); in emitThreadIDAddress()
1983 void Enter(CodeGenFunction &CGF) override { in Enter() argument
1984 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs); in Enter()
1986 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes); in Enter()
1987 auto *ThenBlock = CGF.createBasicBlock("omp_if.then"); in Enter()
1988 ContBlock = CGF.createBasicBlock("omp_if.end"); in Enter()
1990 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock); in Enter()
1991 CGF.EmitBlock(ThenBlock); in Enter()
1994 void Done(CodeGenFunction &CGF) { in Done() argument
1996 CGF.EmitBranch(ContBlock); in Done()
1997 CGF.EmitBlock(ContBlock, true); in Done()
1999 void Exit(CodeGenFunction &CGF) override { in Exit() argument
2000 CGF.EmitRuntimeCall(ExitCallee, ExitArgs); in Exit()
2005 void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF, in emitCriticalRegion() argument
2013 if (!CGF.HaveInsertPoint()) in emitCriticalRegion()
2015 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitCriticalRegion()
2020 EnterArgs.push_back(CGF.Builder.CreateIntCast( in emitCriticalRegion()
2021 CGF.EmitScalarExpr(Hint), CGM.Int32Ty, /*isSigned=*/false)); in emitCriticalRegion()
2032 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen); in emitCriticalRegion()
2035 void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF, in emitMasterRegion() argument
2038 if (!CGF.HaveInsertPoint()) in emitMasterRegion()
2045 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitMasterRegion()
2054 emitInlinedDirective(CGF, OMPD_master, MasterOpGen); in emitMasterRegion()
2055 Action.Done(CGF); in emitMasterRegion()
2058 void CGOpenMPRuntime::emitMaskedRegion(CodeGenFunction &CGF, in emitMaskedRegion() argument
2061 if (!CGF.HaveInsertPoint()) in emitMaskedRegion()
2069 ? CGF.EmitScalarExpr(Filter, CGF.Int32Ty) in emitMaskedRegion()
2071 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitMaskedRegion()
2073 llvm::Value *ArgsEnd[] = {emitUpdateLocation(CGF, Loc), in emitMaskedRegion()
2074 getThreadID(CGF, Loc)}; in emitMaskedRegion()
2083 emitInlinedDirective(CGF, OMPD_masked, MaskedOpGen); in emitMaskedRegion()
2084 Action.Done(CGF); in emitMaskedRegion()
2087 void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF, in emitTaskyieldCall() argument
2089 if (!CGF.HaveInsertPoint()) in emitTaskyieldCall()
2091 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) { in emitTaskyieldCall()
2092 OMPBuilder.createTaskyield(CGF.Builder); in emitTaskyieldCall()
2096 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitTaskyieldCall()
2098 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskyieldCall()
2103 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitTaskyieldCall()
2104 Region->emitUntiedSwitch(CGF); in emitTaskyieldCall()
2107 void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF, in emitTaskgroupRegion() argument
2110 if (!CGF.HaveInsertPoint()) in emitTaskgroupRegion()
2116 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitTaskgroupRegion()
2124 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen); in emitTaskgroupRegion()
2129 static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array, in emitAddrOfVarFromArray() argument
2132 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index); in emitAddrOfVarFromArray()
2133 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr); in emitAddrOfVarFromArray()
2135 llvm::Type *ElemTy = CGF.ConvertTypeForMem(Var->getType()); in emitAddrOfVarFromArray()
2137 CGF.Builder.CreateBitCast( in emitAddrOfVarFromArray()
2139 ElemTy, CGF.getContext().getDeclAlign(Var)); in emitAddrOfVarFromArray()
2165 CodeGenFunction CGF(CGM); in emitCopyprivateCopyFunction() local
2166 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitCopyprivateCopyFunction()
2169 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitCopyprivateCopyFunction()
2170 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)), in emitCopyprivateCopyFunction()
2172 ArgsElemType, CGF.getPointerAlign()); in emitCopyprivateCopyFunction()
2173 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitCopyprivateCopyFunction()
2174 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)), in emitCopyprivateCopyFunction()
2176 ArgsElemType, CGF.getPointerAlign()); in emitCopyprivateCopyFunction()
2184 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar); in emitCopyprivateCopyFunction()
2188 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar); in emitCopyprivateCopyFunction()
2192 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]); in emitCopyprivateCopyFunction()
2194 CGF.FinishFunction(); in emitCopyprivateCopyFunction()
2198 void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF, in emitSingleRegion() argument
2205 if (!CGF.HaveInsertPoint()) in emitSingleRegion()
2225 DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it"); in emitSingleRegion()
2226 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt); in emitSingleRegion()
2229 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitSingleRegion()
2238 emitInlinedDirective(CGF, OMPD_single, SingleOpGen); in emitSingleRegion()
2241 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt); in emitSingleRegion()
2243 Action.Done(CGF); in emitSingleRegion()
2253 CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list"); in emitSingleRegion()
2255 Address Elem = CGF.Builder.CreateConstArrayGEP(CopyprivateList, I); in emitSingleRegion()
2256 CGF.Builder.CreateStore( in emitSingleRegion()
2257 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitSingleRegion()
2258 CGF.EmitLValue(CopyprivateVars[I]).getPointer(CGF), in emitSingleRegion()
2259 CGF.VoidPtrTy), in emitSingleRegion()
2265 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy), CopyprivateVars, in emitSingleRegion()
2267 llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy); in emitSingleRegion()
2268 Address CL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitSingleRegion()
2269 CopyprivateList, CGF.VoidPtrTy, CGF.Int8Ty); in emitSingleRegion()
2270 llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt); in emitSingleRegion()
2272 emitUpdateLocation(CGF, Loc), // ident_t *<loc> in emitSingleRegion()
2273 getThreadID(CGF, Loc), // i32 <gtid> in emitSingleRegion()
2279 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitSingleRegion()
2285 void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF, in emitOrderedRegion() argument
2288 if (!CGF.HaveInsertPoint()) in emitOrderedRegion()
2295 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitOrderedRegion()
2303 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen); in emitOrderedRegion()
2306 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen); in emitOrderedRegion()
2325 CodeGenFunction &CGF, const OMPLoopDirective &S, in getDefaultScheduleAndChunk() argument
2336 CGF.getContext(), ChunkSize, in getDefaultScheduleAndChunk()
2337 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0), in getDefaultScheduleAndChunk()
2342 void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, in emitBarrierCall() argument
2347 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo); in emitBarrierCall()
2348 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) { in emitBarrierCall()
2349 CGF.Builder.restoreIP(OMPBuilder.createBarrier( in emitBarrierCall()
2350 CGF.Builder, Kind, ForceSimpleCall, EmitChecks)); in emitBarrierCall()
2354 if (!CGF.HaveInsertPoint()) in emitBarrierCall()
2361 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags), in emitBarrierCall()
2362 getThreadID(CGF, Loc)}; in emitBarrierCall()
2365 llvm::Value *Result = CGF.EmitRuntimeCall( in emitBarrierCall()
2373 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit"); in emitBarrierCall()
2374 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue"); in emitBarrierCall()
2375 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result); in emitBarrierCall()
2376 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); in emitBarrierCall()
2377 CGF.EmitBlock(ExitBB); in emitBarrierCall()
2380 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); in emitBarrierCall()
2381 CGF.EmitBranchThroughCleanup(CancelDestination); in emitBarrierCall()
2382 CGF.EmitBlock(ContBB, /*IsFinished=*/true); in emitBarrierCall()
2387 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitBarrierCall()
2392 void CGOpenMPRuntime::emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, in emitErrorCall() argument
2395 ME ? CGF.EmitStringLiteralLValue(cast<StringLiteral>(ME)).getPointer(CGF) in emitErrorCall()
2396 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitErrorCall()
2400 emitUpdateLocation(CGF, Loc, /*Flags=*/0, /*GenLoc=*/true), in emitErrorCall()
2402 CGF.Builder.CreatePointerCast(MVL, CGM.Int8PtrTy)}; in emitErrorCall()
2403 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitErrorCall()
2522 CodeGenFunction &CGF, SourceLocation Loc, in emitForDispatchInit() argument
2525 if (!CGF.HaveInsertPoint()) in emitForDispatchInit()
2540 : CGF.Builder.getIntN(IVSize, 1); in emitForDispatchInit()
2542 emitUpdateLocation(CGF, Loc), in emitForDispatchInit()
2543 getThreadID(CGF, Loc), in emitForDispatchInit()
2544 CGF.Builder.getInt32(addMonoNonMonoModifier( in emitForDispatchInit()
2548 CGF.Builder.getIntN(IVSize, 1), // Stride in emitForDispatchInit()
2551 CGF.EmitRuntimeCall(OMPBuilder.createDispatchInitFunction(IVSize, IVSigned), in emitForDispatchInit()
2556 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId, in emitForStaticInitCall() argument
2560 if (!CGF.HaveInsertPoint()) in emitForStaticInitCall()
2581 Chunk = CGF.Builder.getIntN(Values.IVSize, 1); in emitForStaticInitCall()
2592 CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1, in emitForStaticInitCall()
2598 CGF.Builder.getIntN(Values.IVSize, 1), // Incr in emitForStaticInitCall()
2601 CGF.EmitRuntimeCall(ForStaticInitFunction, Args); in emitForStaticInitCall()
2604 void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF, in emitForStaticInit() argument
2613 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc, in emitForStaticInit()
2617 llvm::Value *ThreadId = getThreadID(CGF, Loc); in emitForStaticInit()
2621 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc); in emitForStaticInit()
2622 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction, in emitForStaticInit()
2627 CodeGenFunction &CGF, SourceLocation Loc, in emitDistributeStaticInit() argument
2633 emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE); in emitDistributeStaticInit()
2634 llvm::Value *ThreadId = getThreadID(CGF, Loc); in emitDistributeStaticInit()
2642 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction, in emitDistributeStaticInit()
2647 void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF, in emitForStaticFinish() argument
2650 if (!CGF.HaveInsertPoint()) in emitForStaticFinish()
2654 emitUpdateLocation(CGF, Loc, in emitForStaticFinish()
2660 getThreadID(CGF, Loc)}; in emitForStaticFinish()
2661 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc); in emitForStaticFinish()
2665 CGF.EmitRuntimeCall( in emitForStaticFinish()
2670 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitForStaticFinish()
2675 void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF, in emitForOrderedIterationEnd() argument
2679 if (!CGF.HaveInsertPoint()) in emitForOrderedIterationEnd()
2682 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitForOrderedIterationEnd()
2683 CGF.EmitRuntimeCall(OMPBuilder.createDispatchFiniFunction(IVSize, IVSigned), in emitForOrderedIterationEnd()
2687 llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF, in emitForNext() argument
2697 emitUpdateLocation(CGF, Loc), in emitForNext()
2698 getThreadID(CGF, Loc), in emitForNext()
2704 llvm::Value *Call = CGF.EmitRuntimeCall( in emitForNext()
2706 return CGF.EmitScalarConversion( in emitForNext()
2707 Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1), in emitForNext()
2708 CGF.getContext().BoolTy, Loc); in emitForNext()
2711 void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF, in emitNumThreadsClause() argument
2714 if (!CGF.HaveInsertPoint()) in emitNumThreadsClause()
2718 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitNumThreadsClause()
2719 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)}; in emitNumThreadsClause()
2720 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitNumThreadsClause()
2725 void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF, in emitProcBindClause() argument
2728 if (!CGF.HaveInsertPoint()) in emitProcBindClause()
2733 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitProcBindClause()
2735 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitProcBindClause()
2740 void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>, in emitFlush() argument
2742 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) { in emitFlush()
2743 OMPBuilder.createFlush(CGF.Builder); in emitFlush()
2745 if (!CGF.HaveInsertPoint()) in emitFlush()
2748 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitFlush()
2750 emitUpdateLocation(CGF, Loc)); in emitFlush()
3003 CodeGenFunction CGF(CGM); in emitProxyTaskFunction() local
3004 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args, in emitProxyTaskFunction()
3012 llvm::Value *GtidParam = CGF.EmitLoadOfScalar( in emitProxyTaskFunction()
3013 CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc); in emitProxyTaskFunction()
3014 LValue TDBase = CGF.EmitLoadOfPointerLValue( in emitProxyTaskFunction()
3015 CGF.GetAddrOfLocalVar(&TaskTypeArg), in emitProxyTaskFunction()
3020 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin()); in emitProxyTaskFunction()
3023 LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI); in emitProxyTaskFunction()
3024 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF); in emitProxyTaskFunction()
3027 LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI); in emitProxyTaskFunction()
3028 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitProxyTaskFunction()
3029 CGF.EmitLoadOfScalar(SharedsLVal, Loc), in emitProxyTaskFunction()
3030 CGF.ConvertTypeForMem(SharedsPtrTy)); in emitProxyTaskFunction()
3035 LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI); in emitProxyTaskFunction()
3036 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitProxyTaskFunction()
3037 PrivatesLVal.getPointer(CGF), CGF.VoidPtrTy); in emitProxyTaskFunction()
3039 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitProxyTaskFunction()
3044 CGF.Builder in emitProxyTaskFunction()
3045 .CreatePointerBitCastOrAddrSpaceCast(TDBase.getAddress(CGF), in emitProxyTaskFunction()
3046 CGF.VoidPtrTy, CGF.Int8Ty) in emitProxyTaskFunction()
3052 LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI); in emitProxyTaskFunction()
3053 llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc); in emitProxyTaskFunction()
3055 LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI); in emitProxyTaskFunction()
3056 llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc); in emitProxyTaskFunction()
3058 LValue StLVal = CGF.EmitLValueForField(Base, *StFI); in emitProxyTaskFunction()
3059 llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc); in emitProxyTaskFunction()
3061 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI); in emitProxyTaskFunction()
3062 llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc); in emitProxyTaskFunction()
3064 LValue RLVal = CGF.EmitLValueForField(Base, *RFI); in emitProxyTaskFunction()
3065 llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc); in emitProxyTaskFunction()
3074 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction, in emitProxyTaskFunction()
3076 CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)), in emitProxyTaskFunction()
3077 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty)); in emitProxyTaskFunction()
3078 CGF.FinishFunction(); in emitProxyTaskFunction()
3108 CodeGenFunction CGF(CGM); in emitDestructorsFunction() local
3109 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo, in emitDestructorsFunction()
3112 LValue Base = CGF.EmitLoadOfPointerLValue( in emitDestructorsFunction()
3113 CGF.GetAddrOfLocalVar(&TaskTypeArg), in emitDestructorsFunction()
3118 Base = CGF.EmitLValueForField(Base, *FI); in emitDestructorsFunction()
3123 LValue FieldLValue = CGF.EmitLValueForField(Base, Field); in emitDestructorsFunction()
3124 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(CGF), Field->getType()); in emitDestructorsFunction()
3127 CGF.FinishFunction(); in emitDestructorsFunction()
3216 CodeGenFunction CGF(CGM); in emitTaskPrivateMappingFunction() local
3217 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap, in emitTaskPrivateMappingFunction()
3221 LValue Base = CGF.EmitLoadOfPointerLValue( in emitTaskPrivateMappingFunction()
3222 CGF.GetAddrOfLocalVar(&TaskPrivatesArg), in emitTaskPrivateMappingFunction()
3227 LValue FieldLVal = CGF.EmitLValueForField(Base, Field); in emitTaskPrivateMappingFunction()
3230 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType()); in emitTaskPrivateMappingFunction()
3231 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue( in emitTaskPrivateMappingFunction()
3232 RefLVal.getAddress(CGF), RefLVal.getType()->castAs<PointerType>()); in emitTaskPrivateMappingFunction()
3233 CGF.EmitStoreOfScalar(FieldLVal.getPointer(CGF), RefLoadLVal); in emitTaskPrivateMappingFunction()
3236 CGF.FinishFunction(); in emitTaskPrivateMappingFunction()
3241 static void emitPrivatesInit(CodeGenFunction &CGF, in emitPrivatesInit() argument
3248 ASTContext &C = CGF.getContext(); in emitPrivatesInit()
3250 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI); in emitPrivatesInit()
3265 SrcBase = CGF.MakeAddrLValue( in emitPrivatesInit()
3266 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitPrivatesInit()
3267 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy), in emitPrivatesInit()
3268 CGF.ConvertTypeForMem(SharedsTy)), in emitPrivatesInit()
3281 !CGF.isTrivialInitializer(Init)))) { in emitPrivatesInit()
3282 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI); in emitPrivatesInit()
3300 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type); in emitPrivatesInit()
3302 SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField); in emitPrivatesInit()
3303 SharedRefLValue = CGF.MakeAddrLValue( in emitPrivatesInit()
3304 SharedRefLValue.getAddress(CGF).withAlignment( in emitPrivatesInit()
3308 } else if (CGF.LambdaCaptureFields.count( in emitPrivatesInit()
3310 isa_and_nonnull<BlockDecl>(CGF.CurCodeDecl)) { in emitPrivatesInit()
3311 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef); in emitPrivatesInit()
3315 CGF, [](CodeGenFunction &, PrePostActionTy &) {}, OMPD_unknown, in emitPrivatesInit()
3317 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef); in emitPrivatesInit()
3321 if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) { in emitPrivatesInit()
3323 CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type); in emitPrivatesInit()
3327 CGF.EmitOMPAggregateAssign( in emitPrivatesInit()
3328 PrivateLValue.getAddress(CGF), SharedRefLValue.getAddress(CGF), in emitPrivatesInit()
3330 [&CGF, Elem, Init, &CapturesInfo](Address DestElement, in emitPrivatesInit()
3333 CodeGenFunction::OMPPrivateScope InitScope(CGF); in emitPrivatesInit()
3338 CGF, &CapturesInfo); in emitPrivatesInit()
3339 CGF.EmitAnyExprToMem(Init, DestElement, in emitPrivatesInit()
3345 CodeGenFunction::OMPPrivateScope InitScope(CGF); in emitPrivatesInit()
3346 InitScope.addPrivate(Elem, SharedRefLValue.getAddress(CGF)); in emitPrivatesInit()
3348 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo); in emitPrivatesInit()
3349 CGF.EmitExprAsInit(Init, VD, PrivateLValue, in emitPrivatesInit()
3353 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false); in emitPrivatesInit()
3361 static bool checkInitIsRequired(CodeGenFunction &CGF, in checkInitIsRequired() argument
3370 !CGF.isTrivialInitializer(Init)); in checkInitIsRequired()
3417 CodeGenFunction CGF(CGM); in emitTaskDupFunction() local
3418 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc, in emitTaskDupFunction()
3421 LValue TDBase = CGF.EmitLoadOfPointerLValue( in emitTaskDupFunction()
3422 CGF.GetAddrOfLocalVar(&DstArg), in emitTaskDupFunction()
3427 LValue Base = CGF.EmitLValueForField( in emitTaskDupFunction()
3429 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI); in emitTaskDupFunction()
3430 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar( in emitTaskDupFunction()
3431 CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc); in emitTaskDupFunction()
3432 CGF.EmitStoreOfScalar(Lastpriv, LILVal); in emitTaskDupFunction()
3439 LValue TDBase = CGF.EmitLoadOfPointerLValue( in emitTaskDupFunction()
3440 CGF.GetAddrOfLocalVar(&SrcArg), in emitTaskDupFunction()
3442 LValue Base = CGF.EmitLValueForField( in emitTaskDupFunction()
3445 CGF.EmitLoadOfScalar(CGF.EmitLValueForField( in emitTaskDupFunction()
3449 CGF.Int8Ty, CGM.getNaturalTypeAlignment(SharedsTy)); in emitTaskDupFunction()
3451 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD, in emitTaskDupFunction()
3453 CGF.FinishFunction(); in emitTaskDupFunction()
3476 CodeGenFunction &CGF; member in __anon90c6c1b11111::OMPIteratorGeneratorScope
3484 OMPIteratorGeneratorScope(CodeGenFunction &CGF, const OMPIteratorExpr *E) in OMPIteratorGeneratorScope() argument
3485 : CodeGenFunction::OMPPrivateScope(CGF), CGF(CGF), E(E) { in OMPIteratorGeneratorScope()
3490 Uppers.push_back(CGF.EmitScalarExpr(E->getHelper(I).Upper)); in OMPIteratorGeneratorScope()
3492 addPrivate(VD, CGF.CreateMemTemp(VD->getType(), VD->getName())); in OMPIteratorGeneratorScope()
3496 CGF.CreateMemTemp(HelperData.CounterVD->getType(), "counter.addr")); in OMPIteratorGeneratorScope()
3503 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(HelperData.CounterVD), in OMPIteratorGeneratorScope()
3506 CGF.EmitStoreOfScalar( in OMPIteratorGeneratorScope()
3507 llvm::ConstantInt::get(CLVal.getAddress(CGF).getElementType(), 0), in OMPIteratorGeneratorScope()
3510 ContDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.cont")); in OMPIteratorGeneratorScope()
3512 ExitDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.exit")); in OMPIteratorGeneratorScope()
3517 CGF.EmitBlock(ContDest.getBlock()); in OMPIteratorGeneratorScope()
3519 CGF.EmitLoadOfScalar(CLVal, HelperData.CounterVD->getLocation()); in OMPIteratorGeneratorScope()
3522 ? CGF.Builder.CreateICmpSLT(CVal, N) in OMPIteratorGeneratorScope()
3523 : CGF.Builder.CreateICmpULT(CVal, N); in OMPIteratorGeneratorScope()
3524 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("iter.body"); in OMPIteratorGeneratorScope()
3525 CGF.Builder.CreateCondBr(Cmp, BodyBB, ExitDest.getBlock()); in OMPIteratorGeneratorScope()
3527 CGF.EmitBlock(BodyBB); in OMPIteratorGeneratorScope()
3529 CGF.EmitIgnoredExpr(HelperData.Update); in OMPIteratorGeneratorScope()
3538 CGF.EmitIgnoredExpr(HelperData.CounterUpdate); in ~OMPIteratorGeneratorScope()
3540 CGF.EmitBranchThroughCleanup(ContDests[I - 1]); in ~OMPIteratorGeneratorScope()
3542 CGF.EmitBlock(ExitDests[I - 1].getBlock(), /*IsFinished=*/I == 1); in ~OMPIteratorGeneratorScope()
3549 getPointerAndSize(CodeGenFunction &CGF, const Expr *E) { in getPointerAndSize() argument
3554 Addr = CGF.EmitScalarExpr(Base); in getPointerAndSize()
3556 Addr = CGF.EmitLValue(E).getPointer(CGF); in getPointerAndSize()
3561 SizeVal = CGF.getTypeSize(OASE->getBase()->getType()->getPointeeType()); in getPointerAndSize()
3563 llvm::Value *Sz = CGF.EmitScalarExpr(SE); in getPointerAndSize()
3564 Sz = CGF.EmitScalarConversion( in getPointerAndSize()
3565 Sz, SE->getType(), CGF.getContext().getSizeType(), SE->getExprLoc()); in getPointerAndSize()
3566 SizeVal = CGF.Builder.CreateNUWMul(SizeVal, Sz); in getPointerAndSize()
3571 CGF.EmitOMPArraySectionExpr(ASE, /*IsLowerBound=*/false); in getPointerAndSize()
3572 Address UpAddrAddress = UpAddrLVal.getAddress(CGF); in getPointerAndSize()
3573 llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32( in getPointerAndSize()
3575 llvm::Value *LowIntPtr = CGF.Builder.CreatePtrToInt(Addr, CGF.SizeTy); in getPointerAndSize()
3576 llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGF.SizeTy); in getPointerAndSize()
3577 SizeVal = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr); in getPointerAndSize()
3579 SizeVal = CGF.getTypeSize(Ty); in getPointerAndSize()
3600 CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskInit() argument
3675 llvm::Type *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy); in emitTaskInit()
3679 CGF.getTypeSize(KmpTaskTWithPrivatesQTy); in emitTaskInit()
3690 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
3730 ? CGF.Builder.CreateSelect(Data.Final.getPointer(), in emitTaskInit()
3731 CGF.Builder.getInt32(FinalFlag), in emitTaskInit()
3732 CGF.Builder.getInt32(/*C=*/0)) in emitTaskInit()
3733 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0); in emitTaskInit()
3734 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags)); in emitTaskInit()
3736 SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc), in emitTaskInit()
3737 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize, in emitTaskInit()
3738 SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
3749 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTaskInit()
3750 CGF.Int64Ty, /*isSigned=*/true); in emitTaskInit()
3752 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTaskInit()
3754 NewTask = CGF.EmitRuntimeCall( in emitTaskInit()
3760 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskInit()
3769 LValue EvtLVal = CGF.EmitLValue(Evt); in emitTaskInit()
3773 llvm::Value *Loc = emitUpdateLocation(CGF, DC->getBeginLoc()); in emitTaskInit()
3774 llvm::Value *Tid = getThreadID(CGF, DC->getBeginLoc()); in emitTaskInit()
3775 Tid = CGF.Builder.CreateIntCast(Tid, CGF.IntTy, /*isSigned=*/false); in emitTaskInit()
3776 llvm::Value *EvtVal = CGF.EmitRuntimeCall( in emitTaskInit()
3780 EvtVal = CGF.EmitScalarConversion(EvtVal, C.VoidPtrTy, Evt->getType(), in emitTaskInit()
3782 CGF.EmitStoreOfScalar(EvtVal, EvtLVal); in emitTaskInit()
3796 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper); in emitTaskInit()
3797 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false); in emitTaskInit()
3799 NumOfElements ? CGF.Builder.CreateNUWMul(NumOfElements, Sz) : Sz; in emitTaskInit()
3811 NumOfElements = CGF.Builder.CreateNUWAdd( in emitTaskInit()
3812 llvm::ConstantInt::get(CGF.SizeTy, NumAffinities), NumOfElements); in emitTaskInit()
3817 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE, in emitTaskInit()
3825 CGF.EmitVarDecl(*PD); in emitTaskInit()
3826 AffinitiesArray = CGF.GetAddrOfLocalVar(PD); in emitTaskInit()
3827 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty, in emitTaskInit()
3835 CGF.CreateMemTemp(KmpTaskAffinityInfoArrayTy, ".affs.arr.addr"); in emitTaskInit()
3836 AffinitiesArray = CGF.Builder.CreateConstArrayGEP(AffinitiesArray, 0); in emitTaskInit()
3853 std::tie(Addr, Size) = getPointerAndSize(CGF, E); in emitTaskInit()
3855 CGF.MakeAddrLValue(CGF.Builder.CreateConstGEP(AffinitiesArray, Pos), in emitTaskInit()
3858 LValue BaseAddrLVal = CGF.EmitLValueForField( in emitTaskInit()
3860 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy), in emitTaskInit()
3863 LValue LenLVal = CGF.EmitLValueForField( in emitTaskInit()
3865 CGF.EmitStoreOfScalar(Size, LenLVal); in emitTaskInit()
3871 PosLVal = CGF.MakeAddrLValue( in emitTaskInit()
3872 CGF.CreateMemTemp(C.getSizeType(), "affs.counter.addr"), in emitTaskInit()
3874 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal); in emitTaskInit()
3882 CGF, cast_or_null<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts())); in emitTaskInit()
3886 std::tie(Addr, Size) = getPointerAndSize(CGF, E); in emitTaskInit()
3887 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); in emitTaskInit()
3888 LValue Base = CGF.MakeAddrLValue( in emitTaskInit()
3889 CGF.Builder.CreateGEP(AffinitiesArray, Idx), KmpTaskAffinityInfoTy); in emitTaskInit()
3891 LValue BaseAddrLVal = CGF.EmitLValueForField( in emitTaskInit()
3893 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy), in emitTaskInit()
3896 LValue LenLVal = CGF.EmitLValueForField( in emitTaskInit()
3898 CGF.EmitStoreOfScalar(Size, LenLVal); in emitTaskInit()
3899 Idx = CGF.Builder.CreateNUWAdd( in emitTaskInit()
3901 CGF.EmitStoreOfScalar(Idx, PosLVal); in emitTaskInit()
3907 llvm::Value *LocRef = emitUpdateLocation(CGF, Loc); in emitTaskInit()
3908 llvm::Value *GTid = getThreadID(CGF, Loc); in emitTaskInit()
3909 llvm::Value *AffinListPtr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
3913 (void)CGF.EmitRuntimeCall( in emitTaskInit()
3919 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
3921 LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy, in emitTaskInit()
3924 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin()); in emitTaskInit()
3930 CGF.EmitLoadOfScalar( in emitTaskInit()
3931 CGF.EmitLValueForField( in emitTaskInit()
3935 CGF.Int8Ty, CGM.getNaturalTypeAlignment(SharedsTy)); in emitTaskInit()
3936 LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy); in emitTaskInit()
3937 LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy); in emitTaskInit()
3938 CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap); in emitTaskInit()
3943 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD, in emitTaskInit()
3947 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) { in emitTaskInit()
3964 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI); in emitTaskInit()
3965 LValue DestructorsLV = CGF.EmitLValueForField( in emitTaskInit()
3967 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
3973 LValue Data2LV = CGF.EmitLValueForField( in emitTaskInit()
3975 LValue PriorityLV = CGF.EmitLValueForField( in emitTaskInit()
3977 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV); in emitTaskInit()
4034 CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal, in getDepobjElements() argument
4042 LValue Base = CGF.EmitLoadOfPointerLValue( in getDepobjElements()
4043 DepobjLVal.getAddress(CGF).withElementType( in getDepobjElements()
4044 CGF.ConvertTypeForMem(KmpDependInfoPtrTy)), in getDepobjElements()
4046 Address DepObjAddr = CGF.Builder.CreateGEP( in getDepobjElements()
4047 Base.getAddress(CGF), in getDepobjElements()
4048 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); in getDepobjElements()
4049 LValue NumDepsBase = CGF.MakeAddrLValue( in getDepobjElements()
4052 LValue BaseAddrLVal = CGF.EmitLValueForField( in getDepobjElements()
4056 llvm::Value *NumDeps = CGF.EmitLoadOfScalar(BaseAddrLVal, Loc); in getDepobjElements()
4060 static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy, in emitDependData() argument
4064 CodeGenModule &CGM = CGF.CGM; in emitDependData()
4070 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy); in emitDependData()
4073 CGF, cast_or_null<OMPIteratorExpr>( in emitDependData()
4082 std::tie(Addr, Size) = getPointerAndSize(CGF, E); in emitDependData()
4083 Addr = CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy); in emitDependData()
4085 Addr = llvm::ConstantInt::get(CGF.IntPtrTy, 0); in emitDependData()
4086 Size = llvm::ConstantInt::get(CGF.SizeTy, 0); in emitDependData()
4090 Base = CGF.MakeAddrLValue( in emitDependData()
4091 CGF.Builder.CreateConstGEP(DependenciesArray, *P), KmpDependInfoTy); in emitDependData()
4095 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); in emitDependData()
4096 Base = CGF.MakeAddrLValue( in emitDependData()
4097 CGF.Builder.CreateGEP(DependenciesArray, Idx), KmpDependInfoTy); in emitDependData()
4100 LValue BaseAddrLVal = CGF.EmitLValueForField( in emitDependData()
4104 CGF.EmitStoreOfScalar(Addr, BaseAddrLVal); in emitDependData()
4106 LValue LenLVal = CGF.EmitLValueForField( in emitDependData()
4109 CGF.EmitStoreOfScalar(Size, LenLVal); in emitDependData()
4112 LValue FlagsLVal = CGF.EmitLValueForField( in emitDependData()
4116 CGF.EmitStoreOfScalar( in emitDependData()
4123 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); in emitDependData()
4124 Idx = CGF.Builder.CreateNUWAdd(Idx, in emitDependData()
4126 CGF.EmitStoreOfScalar(Idx, PosLVal); in emitDependData()
4132 CodeGenFunction &CGF, QualType &KmpDependInfoTy, in emitDepobjElementsSizes() argument
4138 ASTContext &C = CGF.getContext(); in emitDepobjElementsSizes()
4141 CGF, cast_or_null<OMPIteratorExpr>( in emitDepobjElementsSizes()
4147 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts()); in emitDepobjElementsSizes()
4149 getDepobjElements(CGF, DepobjLVal, E->getExprLoc()); in emitDepobjElementsSizes()
4150 LValue NumLVal = CGF.MakeAddrLValue( in emitDepobjElementsSizes()
4151 CGF.CreateMemTemp(C.getUIntPtrType(), "depobj.size.addr"), in emitDepobjElementsSizes()
4153 CGF.Builder.CreateStore(llvm::ConstantInt::get(CGF.IntPtrTy, 0), in emitDepobjElementsSizes()
4154 NumLVal.getAddress(CGF)); in emitDepobjElementsSizes()
4155 llvm::Value *PrevVal = CGF.EmitLoadOfScalar(NumLVal, E->getExprLoc()); in emitDepobjElementsSizes()
4156 llvm::Value *Add = CGF.Builder.CreateNUWAdd(PrevVal, NumDeps); in emitDepobjElementsSizes()
4157 CGF.EmitStoreOfScalar(Add, NumLVal); in emitDepobjElementsSizes()
4163 CGF.EmitLoadOfScalar(SizeLVals[I], Data.DepExprs[I]->getExprLoc()); in emitDepobjElementsSizes()
4169 void CGOpenMPRuntime::emitDepobjElements(CodeGenFunction &CGF, in emitDepobjElements() argument
4176 llvm::Value *ElSize = CGF.getTypeSize(KmpDependInfoTy); in emitDepobjElements()
4179 CGF, cast_or_null<OMPIteratorExpr>( in emitDepobjElements()
4186 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts()); in emitDepobjElements()
4188 getDepobjElements(CGF, DepobjLVal, E->getExprLoc()); in emitDepobjElements()
4191 llvm::Value *Size = CGF.Builder.CreateNUWMul( in emitDepobjElements()
4193 CGF.Builder.CreateIntCast(NumDeps, CGF.SizeTy, /*isSigned=*/false)); in emitDepobjElements()
4194 llvm::Value *Pos = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); in emitDepobjElements()
4195 Address DepAddr = CGF.Builder.CreateGEP(DependenciesArray, Pos); in emitDepobjElements()
4196 CGF.Builder.CreateMemCpy(DepAddr, Base.getAddress(CGF), Size); in emitDepobjElements()
4200 llvm::Value *Add = CGF.Builder.CreateNUWAdd(Pos, NumDeps); in emitDepobjElements()
4201 CGF.EmitStoreOfScalar(Add, PosLVal); in emitDepobjElements()
4207 CodeGenFunction &CGF, ArrayRef<OMPTaskDataTy::DependData> Dependencies, in emitDependClause() argument
4228 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.IntPtrTy, 0); in emitDependClause()
4230 llvm::ConstantInt::get(CGF.IntPtrTy, 0); in emitDependClause()
4236 emitDepobjElementsSizes(CGF, KmpDependInfoTy, D); in emitDependClause()
4239 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, Size); in emitDependClause()
4248 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper); in emitDependClause()
4249 Sz = CGF.Builder.CreateIntCast(Sz, CGF.IntPtrTy, /*isSigned=*/false); in emitDependClause()
4250 llvm::Value *NumClauseDeps = CGF.Builder.CreateNUWMul( in emitDependClause()
4251 Sz, llvm::ConstantInt::get(CGF.IntPtrTy, D.DepExprs.size())); in emitDependClause()
4253 CGF.Builder.CreateNUWAdd(NumOfRegularWithIterators, NumClauseDeps); in emitDependClause()
4266 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, NumOfElements); in emitDependClause()
4270 CGF.Builder.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements); in emitDependClause()
4275 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE, in emitDependClause()
4284 CGF.EmitVarDecl(*PD); in emitDependClause()
4285 DependenciesArray = CGF.GetAddrOfLocalVar(PD); in emitDependClause()
4286 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty, in emitDependClause()
4293 CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr"); in emitDependClause()
4294 DependenciesArray = CGF.Builder.CreateConstArrayGEP(DependenciesArray, 0); in emitDependClause()
4303 emitDependData(CGF, KmpDependInfoTy, &Pos, Dependencies[I], in emitDependClause()
4307 LValue PosLVal = CGF.MakeAddrLValue( in emitDependClause()
4308 CGF.CreateMemTemp(C.getSizeType(), "dep.counter.addr"), C.getSizeType()); in emitDependClause()
4309 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal); in emitDependClause()
4314 emitDependData(CGF, KmpDependInfoTy, &PosLVal, Dependencies[I], in emitDependClause()
4322 emitDepobjElements(CGF, KmpDependInfoTy, PosLVal, Dependencies[I], in emitDependClause()
4326 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDependClause()
4327 DependenciesArray, CGF.VoidPtrTy, CGF.Int8Ty); in emitDependClause()
4332 CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies, in emitDepobjDependClause() argument
4354 NumDepsVal = llvm::ConstantInt::get(CGF.SizeTy, 1); in emitDepobjDependClause()
4356 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper); in emitDepobjDependClause()
4357 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false); in emitDepobjDependClause()
4358 NumDepsVal = CGF.Builder.CreateNUWMul(NumDepsVal, Sz); in emitDepobjDependClause()
4360 Size = CGF.Builder.CreateNUWAdd(llvm::ConstantInt::get(CGF.SizeTy, 1), in emitDepobjDependClause()
4365 Size = CGF.Builder.CreateNUWMul(Size, RecSize); in emitDepobjDependClause()
4367 CGF.Builder.CreateIntCast(NumDepsVal, CGF.IntPtrTy, /*isSigned=*/false); in emitDepobjDependClause()
4374 NumDepsVal = llvm::ConstantInt::get(CGF.IntPtrTy, NumDependencies); in emitDepobjDependClause()
4377 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitDepobjDependClause()
4379 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitDepobjDependClause()
4383 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitDepobjDependClause()
4386 llvm::Type *KmpDependInfoLlvmTy = CGF.ConvertTypeForMem(KmpDependInfoTy); in emitDepobjDependClause()
4387 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDepobjDependClause()
4391 LValue Base = CGF.MakeAddrLValue(DependenciesArray, KmpDependInfoTy); in emitDepobjDependClause()
4393 LValue BaseAddrLVal = CGF.EmitLValueForField( in emitDepobjDependClause()
4397 CGF.EmitStoreOfScalar(NumDepsVal, BaseAddrLVal); in emitDepobjDependClause()
4402 PosLVal = CGF.MakeAddrLValue( in emitDepobjDependClause()
4403 CGF.CreateMemTemp(C.getSizeType(), "iterator.counter.addr"), in emitDepobjDependClause()
4405 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Idx), PosLVal, in emitDepobjDependClause()
4411 emitDependData(CGF, KmpDependInfoTy, Pos, Dependencies, DependenciesArray); in emitDepobjDependClause()
4412 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDepobjDependClause()
4413 CGF.Builder.CreateConstGEP(DependenciesArray, 1), CGF.VoidPtrTy, in emitDepobjDependClause()
4414 CGF.Int8Ty); in emitDepobjDependClause()
4418 void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal, in emitDestroyClause() argument
4423 LValue Base = CGF.EmitLoadOfPointerLValue( in emitDestroyClause()
4424 DepobjLVal.getAddress(CGF), C.VoidPtrTy.castAs<PointerType>()); in emitDestroyClause()
4426 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDestroyClause()
4427 Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy), in emitDestroyClause()
4428 CGF.ConvertTypeForMem(KmpDependInfoTy)); in emitDestroyClause()
4429 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP( in emitDestroyClause()
4431 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); in emitDestroyClause()
4432 DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(DepObjAddr, in emitDestroyClause()
4433 CGF.VoidPtrTy); in emitDestroyClause()
4434 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitDestroyClause()
4436 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitDestroyClause()
4440 (void)CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitDestroyClause()
4445 void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, in emitUpdateClause() argument
4453 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy); in emitUpdateClause()
4456 std::tie(NumDeps, Base) = getDepobjElements(CGF, DepobjLVal, Loc); in emitUpdateClause()
4458 Address Begin = Base.getAddress(CGF); in emitUpdateClause()
4460 llvm::Value *End = CGF.Builder.CreateGEP( in emitUpdateClause()
4463 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.body"); in emitUpdateClause()
4464 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.done"); in emitUpdateClause()
4465 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock(); in emitUpdateClause()
4466 CGF.EmitBlock(BodyBB); in emitUpdateClause()
4468 CGF.Builder.CreatePHI(Begin.getType(), 2, "omp.elementPast"); in emitUpdateClause()
4471 Base = CGF.MakeAddrLValue(Begin, KmpDependInfoTy, Base.getBaseInfo(), in emitUpdateClause()
4475 LValue FlagsLVal = CGF.EmitLValueForField( in emitUpdateClause()
4478 CGF.EmitStoreOfScalar( in emitUpdateClause()
4484 CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext"); in emitUpdateClause()
4486 CGF.Builder.GetInsertBlock()); in emitUpdateClause()
4488 CGF.Builder.CreateICmpEQ(ElementNext.getPointer(), End, "omp.isempty"); in emitUpdateClause()
4489 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); in emitUpdateClause()
4491 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); in emitUpdateClause()
4494 void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskCall() argument
4500 if (!CGF.HaveInsertPoint()) in emitTaskCall()
4504 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data); in emitTaskCall()
4514 emitDependClause(CGF, Data.Dependences, Loc); in emitTaskCall()
4522 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitTaskCall()
4523 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc); in emitTaskCall()
4532 DepTaskArgs[5] = CGF.Builder.getInt32(0); in emitTaskCall()
4533 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitTaskCall()
4536 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) { in emitTaskCall() argument
4539 LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI); in emitTaskCall()
4540 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal); in emitTaskCall()
4543 CGF.EmitRuntimeCall( in emitTaskCall()
4548 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskCall()
4554 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitTaskCall()
4555 Region->emitUntiedSwitch(CGF); in emitTaskCall()
4564 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0); in emitTaskCall()
4565 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitTaskCall()
4567 llvm::ConstantInt::get(CGF.Int32Ty, Data.HasNowaitClause); in emitTaskCall()
4572 Loc](CodeGenFunction &CGF, PrePostActionTy &) { in emitTaskCall() argument
4573 CodeGenFunction::RunCleanupsScope LocalScope(CGF); in emitTaskCall()
4579 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskCall()
4584 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTaskCall() argument
4585 Action.Enter(CGF); in emitTaskCall()
4587 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry, in emitTaskCall()
4603 RCG(CGF); in emitTaskCall()
4607 emitIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen); in emitTaskCall()
4610 ThenRCG(CGF); in emitTaskCall()
4614 void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskLoopCall() argument
4620 if (!CGF.HaveInsertPoint()) in emitTaskLoopCall()
4623 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data); in emitTaskLoopCall()
4629 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitTaskLoopCall()
4630 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc); in emitTaskLoopCall()
4633 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy, in emitTaskLoopCall()
4636 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1); in emitTaskLoopCall()
4639 LValue LBLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
4644 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(CGF), in emitTaskLoopCall()
4647 LValue UBLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
4652 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(CGF), in emitTaskLoopCall()
4655 LValue StLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
4660 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(CGF), in emitTaskLoopCall()
4664 LValue RedLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
4668 CGF.EmitStoreOfScalar(Data.Reductions, RedLVal); in emitTaskLoopCall()
4670 CGF.EmitNullInitialization(RedLVal.getAddress(CGF), in emitTaskLoopCall()
4671 CGF.getContext().VoidPtrTy); in emitTaskLoopCall()
4679 LBLVal.getPointer(CGF), in emitTaskLoopCall()
4680 UBLVal.getPointer(CGF), in emitTaskLoopCall()
4681 CGF.EmitLoadOfScalar(StLVal, Loc), in emitTaskLoopCall()
4683 CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler in emitTaskLoopCall()
4685 CGF.IntTy, Data.Schedule.getPointer() in emitTaskLoopCall()
4689 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty, in emitTaskLoopCall()
4691 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0), in emitTaskLoopCall()
4692 Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskLoopCall()
4693 Result.TaskDupFn, CGF.VoidPtrTy) in emitTaskLoopCall()
4694 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)}; in emitTaskLoopCall()
4695 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskLoopCall()
4710 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar, in EmitOMPAggregateReduction() argument
4712 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *, in EmitOMPAggregateReduction() argument
4718 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar); in EmitOMPAggregateReduction()
4719 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar); in EmitOMPAggregateReduction()
4723 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr); in EmitOMPAggregateReduction()
4729 CGF.Builder.CreateGEP(LHSAddr.getElementType(), LHSBegin, NumElements); in EmitOMPAggregateReduction()
4731 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body"); in EmitOMPAggregateReduction()
4732 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done"); in EmitOMPAggregateReduction()
4734 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty"); in EmitOMPAggregateReduction()
4735 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); in EmitOMPAggregateReduction()
4738 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock(); in EmitOMPAggregateReduction()
4739 CGF.EmitBlock(BodyBB); in EmitOMPAggregateReduction()
4741 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy); in EmitOMPAggregateReduction()
4743 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI( in EmitOMPAggregateReduction()
4750 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI( in EmitOMPAggregateReduction()
4758 CodeGenFunction::OMPPrivateScope Scope(CGF); in EmitOMPAggregateReduction()
4762 RedOpGen(CGF, XExpr, EExpr, UpExpr); in EmitOMPAggregateReduction()
4766 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateReduction()
4769 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateReduction()
4774 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done"); in EmitOMPAggregateReduction()
4775 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB); in EmitOMPAggregateReduction()
4776 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateReduction()
4777 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateReduction()
4780 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); in EmitOMPAggregateReduction()
4786 static void emitReductionCombiner(CodeGenFunction &CGF, in emitReductionCombiner() argument
4795 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD); in emitReductionCombiner()
4797 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func); in emitReductionCombiner()
4798 CGF.EmitIgnoredExpr(ReductionOp); in emitReductionCombiner()
4801 CGF.EmitIgnoredExpr(ReductionOp); in emitReductionCombiner()
4826 CodeGenFunction CGF(CGM); in emitReductionFunction() local
4827 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitReductionFunction()
4831 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReductionFunction()
4832 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)), in emitReductionFunction()
4834 ArgsElemType, CGF.getPointerAlign()); in emitReductionFunction()
4835 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReductionFunction()
4836 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)), in emitReductionFunction()
4838 ArgsElemType, CGF.getPointerAlign()); in emitReductionFunction()
4843 CodeGenFunction::OMPPrivateScope Scope(CGF); in emitReductionFunction()
4849 Scope.addPrivate(RHSVar, emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar)); in emitReductionFunction()
4852 Scope.addPrivate(LHSVar, emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar)); in emitReductionFunction()
4857 Address Elem = CGF.Builder.CreateConstArrayGEP(LHS, Idx); in emitReductionFunction()
4858 llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem); in emitReductionFunction()
4860 CGF.getContext().getAsVariableArrayType(PrivTy); in emitReductionFunction()
4863 CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy))); in emitReductionFunction()
4864 CGF.EmitVariablyModifiedType(PrivTy); in emitReductionFunction()
4877 CGF, (*IPriv)->getType(), LHSVar, RHSVar, in emitReductionFunction()
4878 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) { in emitReductionFunction() argument
4879 emitReductionCombiner(CGF, E); in emitReductionFunction()
4883 emitReductionCombiner(CGF, E); in emitReductionFunction()
4890 CGF.FinishFunction(); in emitReductionFunction()
4894 void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF, in emitSingleReductionCombiner() argument
4904 CGF, PrivateRef->getType(), LHSVar, RHSVar, in emitSingleReductionCombiner()
4905 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) { in emitSingleReductionCombiner() argument
4906 emitReductionCombiner(CGF, ReductionOp); in emitSingleReductionCombiner()
4910 emitReductionCombiner(CGF, ReductionOp); in emitSingleReductionCombiner()
4914 void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, in emitReduction() argument
4920 if (!CGF.HaveInsertPoint()) in emitReduction()
4964 CodeGenFunction::RunCleanupsScope Scope(CGF); in emitReduction()
4969 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), in emitReduction()
4991 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); in emitReduction()
4995 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitReduction()
4996 CGF.Builder.CreateStore( in emitReduction()
4997 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReduction()
4998 CGF.EmitLValue(RHSExprs[I]).getPointer(CGF), CGF.VoidPtrTy), in emitReduction()
5003 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitReduction()
5004 llvm::Value *Size = CGF.Builder.CreateIntCast( in emitReduction()
5005 CGF.getVLASize( in emitReduction()
5006 CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) in emitReduction()
5008 CGF.SizeTy, /*isSigned=*/false); in emitReduction()
5009 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), in emitReduction()
5016 CGF.CurFn->getName(), Loc, CGF.ConvertTypeForMem(ReductionArrayTy), in emitReduction()
5025 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE); in emitReduction()
5026 llvm::Value *ThreadId = getThreadID(CGF, Loc); in emitReduction()
5027 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy); in emitReduction()
5028 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReduction()
5029 ReductionList.getPointer(), CGF.VoidPtrTy); in emitReduction()
5033 CGF.Builder.getInt32(RHSExprs.size()), // i32 <n> in emitReduction()
5039 llvm::Value *Res = CGF.EmitRuntimeCall( in emitReduction()
5046 llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(".omp.reduction.default"); in emitReduction()
5048 CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2); in emitReduction()
5056 llvm::BasicBlock *Case1BB = CGF.createBasicBlock(".omp.reduction.case1"); in emitReduction()
5057 SwInst->addCase(CGF.Builder.getInt32(1), Case1BB); in emitReduction()
5058 CGF.EmitBlock(Case1BB); in emitReduction()
5067 CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
5068 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitReduction()
5073 RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), in emitReduction()
5088 RCG(CGF); in emitReduction()
5090 CGF.EmitBranch(DefaultBB); in emitReduction()
5097 llvm::BasicBlock *Case2BB = CGF.createBasicBlock(".omp.reduction.case2"); in emitReduction()
5098 SwInst->addCase(CGF.Builder.getInt32(2), Case2BB); in emitReduction()
5099 CGF.EmitBlock(Case2BB); in emitReduction()
5102 CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
5136 Loc](CodeGenFunction &CGF, const Expr *XExpr, in emitReduction()
5138 LValue X = CGF.EmitLValue(XExpr); in emitReduction()
5141 E = CGF.EmitAnyExpr(EExpr); in emitReduction()
5142 CGF.EmitOMPAtomicSimpleUpdateExpr( in emitReduction()
5145 [&CGF, UpExpr, VD, Loc](RValue XRValue) { in emitReduction()
5146 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitReduction()
5147 Address LHSTemp = CGF.CreateMemTemp(VD->getType()); in emitReduction()
5148 CGF.emitOMPSimpleStore( in emitReduction()
5149 CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue, in emitReduction()
5153 return CGF.EmitAnyExpr(UpExpr); in emitReduction()
5160 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar, in emitReduction()
5164 AtomicRedGen(CGF, XExpr, EExpr, UpExpr); in emitReduction()
5168 auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *, in emitReduction()
5170 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitReduction()
5173 CGF, Name, in emitReduction()
5174 [=](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
5175 Action.Enter(CGF); in emitReduction()
5176 emitReductionCombiner(CGF, E); in emitReduction()
5185 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar, in emitReduction()
5188 CritRedGen(CGF, nullptr, nullptr, nullptr); in emitReduction()
5209 AtomicRCG(CGF); in emitReduction()
5211 AtomicRCG(CGF); in emitReduction()
5214 CGF.EmitBranch(DefaultBB); in emitReduction()
5215 CGF.EmitBlock(DefaultBB, /*IsFinished=*/true); in emitReduction()
5265 CodeGenFunction CGF(CGM); in emitReduceInitFunction() local
5266 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc); in emitReduceInitFunction()
5268 Address PrivateAddr = CGF.EmitLoadOfPointer( in emitReduceInitFunction()
5269 CGF.GetAddrOfLocalVar(&Param).withElementType( in emitReduceInitFunction()
5270 CGF.ConvertTypeForMem(PrivateType)->getPointerTo()), in emitReduceInitFunction()
5277 CGF, CGM.getContext().getSizeType(), in emitReduceInitFunction()
5279 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false, in emitReduceInitFunction()
5282 RCG.emitAggregateType(CGF, N, Size); in emitReduceInitFunction()
5288 Address SharedAddr = CGF.GetAddrOfLocalVar(&ParamOrig); in emitReduceInitFunction()
5289 OrigAddr = CGF.EmitLoadOfPointer( in emitReduceInitFunction()
5296 RCG.emitInitialization(CGF, N, PrivateAddr, OrigAddr, in emitReduceInitFunction()
5298 CGF.FinishFunction(); in emitReduceInitFunction()
5336 CodeGenFunction CGF(CGM); in emitReduceCombFunction() local
5337 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc); in emitReduceCombFunction()
5343 CGF, CGM.getContext().getSizeType(), in emitReduceCombFunction()
5345 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false, in emitReduceCombFunction()
5348 RCG.emitAggregateType(CGF, N, Size); in emitReduceCombFunction()
5352 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitReduceCombFunction()
5356 CGF.EmitLoadOfPointer( in emitReduceCombFunction()
5357 CGF.GetAddrOfLocalVar(&ParamInOut) in emitReduceCombFunction()
5359 CGF.ConvertTypeForMem(LHSVD->getType())->getPointerTo()), in emitReduceCombFunction()
5364 CGF.EmitLoadOfPointer( in emitReduceCombFunction()
5365 CGF.GetAddrOfLocalVar(&ParamIn).withElementType( in emitReduceCombFunction()
5366 CGF.ConvertTypeForMem(RHSVD->getType())->getPointerTo()), in emitReduceCombFunction()
5373 CGF, ReductionOp, PrivateRef, cast<DeclRefExpr>(LHS), in emitReduceCombFunction()
5375 CGF.FinishFunction(); in emitReduceCombFunction()
5405 CodeGenFunction CGF(CGM); in emitReduceFiniFunction() local
5406 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc); in emitReduceFiniFunction()
5407 Address PrivateAddr = CGF.EmitLoadOfPointer( in emitReduceFiniFunction()
5408 CGF.GetAddrOfLocalVar(&Param), C.VoidPtrTy.castAs<PointerType>()); in emitReduceFiniFunction()
5414 CGF, CGM.getContext().getSizeType(), in emitReduceFiniFunction()
5416 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false, in emitReduceFiniFunction()
5419 RCG.emitAggregateType(CGF, N, Size); in emitReduceFiniFunction()
5422 RCG.emitCleanups(CGF, N, PrivateAddr); in emitReduceFiniFunction()
5423 CGF.FinishFunction(Loc); in emitReduceFiniFunction()
5428 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs, in emitTaskReductionInit() argument
5430 if (!CGF.HaveInsertPoint() || Data.ReductionVars.empty()) in emitTaskReductionInit()
5462 Address TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input."); in emitTaskReductionInit()
5469 llvm::Value *GEP = CGF.EmitCheckedInBoundsGEP( in emitTaskReductionInit()
5473 LValue ElemLVal = CGF.MakeNaturalAlignAddrLValue(GEP, RDType); in emitTaskReductionInit()
5475 LValue SharedLVal = CGF.EmitLValueForField(ElemLVal, SharedFD); in emitTaskReductionInit()
5476 RCG.emitSharedOrigLValue(CGF, Cnt); in emitTaskReductionInit()
5477 llvm::Value *Shared = RCG.getSharedLValue(Cnt).getPointer(CGF); in emitTaskReductionInit()
5478 CGF.EmitStoreOfScalar(Shared, SharedLVal); in emitTaskReductionInit()
5480 LValue OrigLVal = CGF.EmitLValueForField(ElemLVal, OrigFD); in emitTaskReductionInit()
5481 llvm::Value *Orig = RCG.getOrigLValue(Cnt).getPointer(CGF); in emitTaskReductionInit()
5482 CGF.EmitStoreOfScalar(Orig, OrigLVal); in emitTaskReductionInit()
5483 RCG.emitAggregateType(CGF, Cnt); in emitTaskReductionInit()
5493 SizeValInChars = CGF.Builder.CreateIntCast(SizeValInChars, CGM.SizeTy, in emitTaskReductionInit()
5495 LValue SizeLVal = CGF.EmitLValueForField(ElemLVal, SizeFD); in emitTaskReductionInit()
5496 CGF.EmitStoreOfScalar(SizeValInChars, SizeLVal); in emitTaskReductionInit()
5498 LValue InitLVal = CGF.EmitLValueForField(ElemLVal, InitFD); in emitTaskReductionInit()
5500 CGF.EmitStoreOfScalar(InitAddr, InitLVal); in emitTaskReductionInit()
5502 LValue FiniLVal = CGF.EmitLValueForField(ElemLVal, FiniFD); in emitTaskReductionInit()
5506 CGF.EmitStoreOfScalar(FiniAddr, FiniLVal); in emitTaskReductionInit()
5508 LValue CombLVal = CGF.EmitLValueForField(ElemLVal, CombFD); in emitTaskReductionInit()
5512 CGF.EmitStoreOfScalar(CombAddr, CombLVal); in emitTaskReductionInit()
5514 LValue FlagsLVal = CGF.EmitLValueForField(ElemLVal, FlagsFD); in emitTaskReductionInit()
5516 CGF.EmitStoreOfScalar( in emitTaskReductionInit()
5520 CGF.EmitNullInitialization(FlagsLVal.getAddress(CGF), in emitTaskReductionInit()
5526 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc); in emitTaskReductionInit()
5527 llvm::Value *GTid = CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), in emitTaskReductionInit()
5534 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskReductionInit()
5536 return CGF.EmitRuntimeCall( in emitTaskReductionInit()
5543 CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), CGM.IntTy, in emitTaskReductionInit()
5546 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskRedInput.getPointer(), in emitTaskReductionInit()
5548 return CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskReductionInit()
5553 void CGOpenMPRuntime::emitTaskReductionFini(CodeGenFunction &CGF, in emitTaskReductionFini() argument
5558 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc); in emitTaskReductionFini()
5559 llvm::Value *GTid = CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), in emitTaskReductionFini()
5565 (void)CGF.EmitRuntimeCall( in emitTaskReductionFini()
5571 void CGOpenMPRuntime::emitTaskReductionFixups(CodeGenFunction &CGF, in emitTaskReductionFixups() argument
5579 llvm::Value *SizeVal = CGF.Builder.CreateIntCast(Sizes.second, CGM.SizeTy, in emitTaskReductionFixups()
5582 CGF, CGM.getContext().getSizeType(), in emitTaskReductionFixups()
5584 CGF.Builder.CreateStore(SizeVal, SizeAddr, /*IsVolatile=*/false); in emitTaskReductionFixups()
5588 Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF, in getTaskReductionItem() argument
5594 llvm::Value *Args[] = {CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), in getTaskReductionItem()
5598 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getTaskReductionItem()
5599 SharedLVal.getPointer(CGF), CGM.VoidPtrTy)}; in getTaskReductionItem()
5601 CGF.EmitRuntimeCall( in getTaskReductionItem()
5605 CGF.Int8Ty, SharedLVal.getAlignment()); in getTaskReductionItem()
5608 void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskwaitCall() argument
5610 if (!CGF.HaveInsertPoint()) in emitTaskwaitCall()
5613 if (CGF.CGM.getLangOpts().OpenMPIRBuilder && Data.Dependences.empty()) { in emitTaskwaitCall()
5615 OMPBuilder.createTaskwait(CGF.Builder); in emitTaskwaitCall()
5617 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitTaskwaitCall()
5618 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc); in emitTaskwaitCall()
5623 emitDependClause(CGF, Data.Dependences, Loc); in emitTaskwaitCall()
5630 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0); in emitTaskwaitCall()
5631 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitTaskwaitCall()
5633 llvm::ConstantInt::get(CGF.Int32Ty, Data.HasNowaitClause); in emitTaskwaitCall()
5635 CodeGenFunction::RunCleanupsScope LocalScope(CGF); in emitTaskwaitCall()
5641 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskwaitCall()
5651 CGF.EmitRuntimeCall( in emitTaskwaitCall()
5657 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitTaskwaitCall()
5658 Region->emitUntiedSwitch(CGF); in emitTaskwaitCall()
5661 void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF, in emitInlinedDirective() argument
5665 if (!CGF.HaveInsertPoint()) in emitInlinedDirective()
5667 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel, in emitInlinedDirective()
5671 CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr); in emitInlinedDirective()
5700 CodeGenFunction &CGF, SourceLocation Loc, in emitCancellationPointCall() argument
5702 if (!CGF.HaveInsertPoint()) in emitCancellationPointCall()
5707 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { in emitCancellationPointCall()
5712 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitCancellationPointCall()
5713 CGF.Builder.getInt32(getCancellationKind(CancelRegion))}; in emitCancellationPointCall()
5715 llvm::Value *Result = CGF.EmitRuntimeCall( in emitCancellationPointCall()
5723 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit"); in emitCancellationPointCall()
5724 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue"); in emitCancellationPointCall()
5725 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result); in emitCancellationPointCall()
5726 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); in emitCancellationPointCall()
5727 CGF.EmitBlock(ExitBB); in emitCancellationPointCall()
5729 emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false); in emitCancellationPointCall()
5732 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); in emitCancellationPointCall()
5733 CGF.EmitBranchThroughCleanup(CancelDest); in emitCancellationPointCall()
5734 CGF.EmitBlock(ContBB, /*IsFinished=*/true); in emitCancellationPointCall()
5739 void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, in emitCancelCall() argument
5742 if (!CGF.HaveInsertPoint()) in emitCancelCall()
5748 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { in emitCancelCall()
5750 OMPRegionInfo](CodeGenFunction &CGF, PrePostActionTy &) { in emitCancelCall() argument
5751 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitCancelCall()
5753 RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc), in emitCancelCall()
5754 CGF.Builder.getInt32(getCancellationKind(CancelRegion))}; in emitCancelCall()
5756 llvm::Value *Result = CGF.EmitRuntimeCall( in emitCancelCall()
5762 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit"); in emitCancelCall()
5763 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue"); in emitCancelCall()
5764 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result); in emitCancelCall()
5765 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); in emitCancelCall()
5766 CGF.EmitBlock(ExitBB); in emitCancelCall()
5768 RT.emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false); in emitCancelCall()
5771 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); in emitCancelCall()
5772 CGF.EmitBranchThroughCleanup(CancelDest); in emitCancelCall()
5773 CGF.EmitBlock(ContBB, /*IsFinished=*/true); in emitCancelCall()
5776 emitIfClause(CGF, IfCond, ThenGen, in emitCancelCall()
5780 ThenRCG(CGF); in emitCancelCall()
5794 void Enter(CodeGenFunction &CGF) override { in Enter() argument
5795 if (!CGF.HaveInsertPoint()) in Enter()
5798 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsInit( in Enter()
5799 CGF, AllocatorData.first, AllocatorData.second); in Enter()
5802 void Exit(CodeGenFunction &CGF) override { in Exit() argument
5803 if (!CGF.HaveInsertPoint()) in Exit()
5806 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsFini(CGF, in Exit()
5834 void CGOpenMPRuntime::emitUsesAllocatorsInit(CodeGenFunction &CGF, in emitUsesAllocatorsInit() argument
5837 llvm::Value *ThreadId = getThreadID(CGF, Allocator->getExprLoc()); in emitUsesAllocatorsInit()
5838 ThreadId = CGF.Builder.CreateIntCast(ThreadId, CGF.IntTy, /*isSigned=*/true); in emitUsesAllocatorsInit()
5840 llvm::Value *MemSpaceHandle = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitUsesAllocatorsInit()
5842 CGF.IntTy, cast<ConstantArrayType>( in emitUsesAllocatorsInit()
5846 LValue AllocatorTraitsLVal = CGF.EmitLValue(AllocatorTraits); in emitUsesAllocatorsInit()
5847 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitUsesAllocatorsInit()
5848 AllocatorTraitsLVal.getAddress(CGF), CGF.VoidPtrPtrTy, CGF.VoidPtrTy); in emitUsesAllocatorsInit()
5849 AllocatorTraitsLVal = CGF.MakeAddrLValue(Addr, CGF.getContext().VoidPtrTy, in emitUsesAllocatorsInit()
5855 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitUsesAllocatorsInit()
5859 CGF.EmitAutoVarAlloca(*cast<VarDecl>( in emitUsesAllocatorsInit()
5861 LValue AllocatorLVal = CGF.EmitLValue(Allocator->IgnoreParenImpCasts()); in emitUsesAllocatorsInit()
5863 CGF.EmitScalarConversion(AllocatorVal, CGF.getContext().VoidPtrTy, in emitUsesAllocatorsInit()
5865 CGF.EmitStoreOfScalar(AllocatorVal, AllocatorLVal); in emitUsesAllocatorsInit()
5868 void CGOpenMPRuntime::emitUsesAllocatorsFini(CodeGenFunction &CGF, in emitUsesAllocatorsFini() argument
5870 llvm::Value *ThreadId = getThreadID(CGF, Allocator->getExprLoc()); in emitUsesAllocatorsFini()
5871 ThreadId = CGF.Builder.CreateIntCast(ThreadId, CGF.IntTy, /*isSigned=*/true); in emitUsesAllocatorsFini()
5872 LValue AllocatorLVal = CGF.EmitLValue(Allocator->IgnoreParenImpCasts()); in emitUsesAllocatorsFini()
5874 CGF.EmitLoadOfScalar(AllocatorLVal, Allocator->getExprLoc()); in emitUsesAllocatorsFini()
5875 AllocatorVal = CGF.EmitScalarConversion(AllocatorVal, Allocator->getType(), in emitUsesAllocatorsFini()
5876 CGF.getContext().VoidPtrTy, in emitUsesAllocatorsFini()
5878 (void)CGF.EmitRuntimeCall( in emitUsesAllocatorsFini()
5885 const OMPExecutableDirective &D, CodeGenFunction &CGF, in computeMinAndMaxThreadsAndTeams() argument
5889 getNumTeamsExprForTargetDirective(CGF, D, MinTeamsVal, MaxTeamsVal); in computeMinAndMaxThreadsAndTeams()
5890 getNumThreadsExprForTargetDirective(CGF, D, MaxThreadsVal, in computeMinAndMaxThreadsAndTeams()
5928 CodeGenFunction CGF(CGM, true); in emitTargetOutlinedFunctionHelper() local
5930 [&CGF, &D, &CodeGen](StringRef EntryFnName) { in emitTargetOutlinedFunctionHelper()
5934 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitTargetOutlinedFunctionHelper()
5935 return CGF.GenerateOpenMPCapturedStmtFunction(CS, D.getBeginLoc()); in emitTargetOutlinedFunctionHelper()
6007 CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &MinTeamsVal, in getNumTeamsExprForTargetDirective() argument
6019 CGOpenMPRuntime::getSingleCompoundChild(CGF.getContext(), Body); in getNumTeamsExprForTargetDirective()
6026 if (NumTeams->isIntegerConstantExpr(CGF.getContext())) in getNumTeamsExprForTargetDirective()
6028 NumTeams->getIntegerConstantExpr(CGF.getContext())) in getNumTeamsExprForTargetDirective()
6056 if (NumTeams->isIntegerConstantExpr(CGF.getContext())) in getNumTeamsExprForTargetDirective()
6057 if (auto Constant = NumTeams->getIntegerConstantExpr(CGF.getContext())) in getNumTeamsExprForTargetDirective()
6139 CodeGenFunction &CGF, const OMPExecutableDirective &D) { in emitNumTeamsForTargetDirective() argument
6140 assert(!CGF.getLangOpts().OpenMPIsTargetDevice && in emitNumTeamsForTargetDirective()
6143 CGBuilderTy &Bld = CGF.Builder; in emitNumTeamsForTargetDirective()
6146 getNumTeamsExprForTargetDirective(CGF, D, MinNT, MaxNT); in emitNumTeamsForTargetDirective()
6153 CGOpenMPInnerExprInfo CGInfo(CGF, *CS); in emitNumTeamsForTargetDirective()
6154 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitNumTeamsForTargetDirective()
6155 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(NumTeams, in emitNumTeamsForTargetDirective()
6157 return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty, in emitNumTeamsForTargetDirective()
6165 CodeGenFunction::RunCleanupsScope NumTeamsScope(CGF); in emitNumTeamsForTargetDirective()
6166 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(NumTeams, in emitNumTeamsForTargetDirective()
6168 return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty, in emitNumTeamsForTargetDirective()
6177 return llvm::ConstantInt::get(CGF.Int32Ty, MinNT); in emitNumTeamsForTargetDirective()
6184 static void getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS, in getNumThreads() argument
6188 CGF.getContext(), CS->getCapturedStmt()); in getNumThreads()
6197 CGOpenMPInnerExprInfo CGInfo(CGF, *CS); in getNumThreads()
6198 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in getNumThreads()
6210 if (CondExpr->EvaluateAsBooleanCondition(Result, CGF.getContext())) { in getNumThreads()
6216 CodeGenFunction::LexicalScope Scope(CGF, CondExpr->getSourceRange()); in getNumThreads()
6221 CGF.EmitVarDecl(cast<VarDecl>(*I)); in getNumThreads()
6224 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in getNumThreads()
6225 CGF.EmitAutoVarCleanups(Emission); in getNumThreads()
6228 *CondVal = CGF.EvaluateExprAsBool(CondExpr); in getNumThreads()
6236 CGOpenMPInnerExprInfo CGInfo(CGF, *CS); in getNumThreads()
6237 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in getNumThreads()
6241 if (NTExpr->isIntegerConstantExpr(CGF.getContext())) in getNumThreads()
6242 if (auto Constant = NTExpr->getIntegerConstantExpr(CGF.getContext())) in getNumThreads()
6254 CodeGenFunction::LexicalScope Scope(CGF, NTExpr->getSourceRange()); in getNumThreads()
6259 CGF.EmitVarDecl(cast<VarDecl>(*I)); in getNumThreads()
6262 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in getNumThreads()
6263 CGF.EmitAutoVarCleanups(Emission); in getNumThreads()
6276 CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &UpperBound, in getNumThreadsExprForTargetDirective() argument
6278 assert((!CGF.getLangOpts().OpenMPIsTargetDevice || UpperBoundOnly) && in getNumThreadsExprForTargetDirective()
6289 if (E->isIntegerConstantExpr(CGF.getContext())) { in getNumThreadsExprForTargetDirective()
6290 if (auto Constant = E->getIntegerConstantExpr(CGF.getContext())) in getNumThreadsExprForTargetDirective()
6311 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal); in getNumThreadsExprForTargetDirective()
6313 CGF.getContext(), CS->getCapturedStmt()); in getNumThreadsExprForTargetDirective()
6321 CGOpenMPInnerExprInfo CGInfo(CGF, *CS); in getNumThreadsExprForTargetDirective()
6322 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in getNumThreadsExprForTargetDirective()
6324 CGF, ThreadLimitClause->getThreadLimit()->getSourceRange()); in getNumThreadsExprForTargetDirective()
6329 CGF.EmitVarDecl(cast<VarDecl>(*I)); in getNumThreadsExprForTargetDirective()
6332 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in getNumThreadsExprForTargetDirective()
6333 CGF.EmitAutoVarCleanups(Emission); in getNumThreadsExprForTargetDirective()
6347 CGF.getContext(), CS->getCapturedStmt()); in getNumThreadsExprForTargetDirective()
6352 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal); in getNumThreadsExprForTargetDirective()
6360 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF); in getNumThreadsExprForTargetDirective()
6365 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal); in getNumThreadsExprForTargetDirective()
6367 CGF.getContext(), CS->getCapturedStmt()); in getNumThreadsExprForTargetDirective()
6371 getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal); in getNumThreadsExprForTargetDirective()
6378 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF); in getNumThreadsExprForTargetDirective()
6382 getNumThreads(CGF, D.getInnermostCapturedStmt(), NTPtr, UpperBound, in getNumThreadsExprForTargetDirective()
6404 if (Cond->EvaluateAsBooleanCondition(Result, CGF.getContext())) { in getNumThreadsExprForTargetDirective()
6408 CodeGenFunction::RunCleanupsScope Scope(CGF); in getNumThreadsExprForTargetDirective()
6409 *CondVal = CGF.EvaluateExprAsBool(Cond); in getNumThreadsExprForTargetDirective()
6414 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF); in getNumThreadsExprForTargetDirective()
6419 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); in getNumThreadsExprForTargetDirective()
6436 CodeGenFunction &CGF, const OMPExecutableDirective &D) { in emitNumThreadsForTargetDirective() argument
6444 CGF, D, UpperBound, /* UpperBoundOnly */ false, &CondVal, in emitNumThreadsForTargetDirective()
6450 CGF.EmitScalarExpr(ThreadLimitExpr, /*IgnoreResultAssign=*/true); in emitNumThreadsForTargetDirective()
6451 ThreadLimitVal = CGF.Builder.CreateIntCast(ThreadLimitVal, CGF.Int32Ty, in emitNumThreadsForTargetDirective()
6457 NumThreadsVal = CGF.Builder.getInt32(UpperBound); in emitNumThreadsForTargetDirective()
6459 NumThreadsVal = CGF.EmitScalarExpr(NT, /*IgnoreResultAssign=*/true); in emitNumThreadsForTargetDirective()
6460 NumThreadsVal = CGF.Builder.CreateIntCast(NumThreadsVal, CGF.Int32Ty, in emitNumThreadsForTargetDirective()
6470 NumThreadsVal = CGF.Builder.getInt32(0); in emitNumThreadsForTargetDirective()
6476 CodeGenFunction::RunCleanupsScope Scope(CGF); in emitNumThreadsForTargetDirective()
6477 NumThreadsVal = CGF.Builder.CreateSelect(CondVal, NumThreadsVal, in emitNumThreadsForTargetDirective()
6478 CGF.Builder.getInt32(1)); in emitNumThreadsForTargetDirective()
6484 NumThreadsVal = CGF.Builder.CreateSelect( in emitNumThreadsForTargetDirective()
6485 CGF.Builder.CreateICmpULT(ThreadLimitVal, NumThreadsVal), in emitNumThreadsForTargetDirective()
6621 CodeGenFunction &CGF; member in __anon90c6c1b12c11::MappableExprsHandler
6651 CGF.getTypeSize(OAE->getBase()->getType()->getPointeeType()); in getExprTypeSize()
6653 llvm::Value *Sz = CGF.EmitScalarExpr(SE); in getExprTypeSize()
6654 Sz = CGF.EmitScalarConversion(Sz, SE->getType(), in getExprTypeSize()
6655 CGF.getContext().getSizeType(), in getExprTypeSize()
6657 Size = CGF.Builder.CreateNUWMul(Size, Sz); in getExprTypeSize()
6679 return CGF.getTypeSize(BaseTy); in getExprTypeSize()
6683 ElemSize = CGF.getTypeSize(PTy->getPointeeType().getCanonicalType()); in getExprTypeSize()
6687 ElemSize = CGF.getTypeSize(ATy->getElementType().getCanonicalType()); in getExprTypeSize()
6696 llvm::Value *LengthVal = CGF.EmitScalarExpr(LenExpr); in getExprTypeSize()
6697 LengthVal = CGF.EmitScalarConversion(LengthVal, LenExpr->getType(), in getExprTypeSize()
6698 CGF.getContext().getSizeType(), in getExprTypeSize()
6700 return CGF.Builder.CreateNUWMul(LengthVal, ElemSize); in getExprTypeSize()
6705 llvm::Value *LengthVal = CGF.getTypeSize(BaseTy); in getExprTypeSize()
6706 llvm::Value *LBVal = CGF.EmitScalarExpr(OAE->getLowerBound()); in getExprTypeSize()
6707 LBVal = CGF.EmitScalarConversion(LBVal, OAE->getLowerBound()->getType(), in getExprTypeSize()
6708 CGF.getContext().getSizeType(), in getExprTypeSize()
6710 LBVal = CGF.Builder.CreateNUWMul(LBVal, ElemSize); in getExprTypeSize()
6711 llvm::Value *Cmp = CGF.Builder.CreateICmpUGT(LengthVal, LBVal); in getExprTypeSize()
6712 llvm::Value *TrueVal = CGF.Builder.CreateNUWSub(LengthVal, LBVal); in getExprTypeSize()
6713 LengthVal = CGF.Builder.CreateSelect( in getExprTypeSize()
6714 Cmp, TrueVal, llvm::ConstantInt::get(CGF.SizeTy, 0)); in getExprTypeSize()
6717 return CGF.getTypeSize(ExprTy); in getExprTypeSize()
6805 if (!Length->EvaluateAsInt(Result, CGF.getContext())) in isFinalArraySectionExpression()
7030 BP = CGF.LoadCXXThisAddress(); in generateInfoForComponentList()
7034 BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress(CGF); in generateInfoForComponentList()
7038 CGF.EmitScalarExpr(OAShE->getBase()), in generateInfoForComponentList()
7039 CGF.ConvertTypeForMem(OAShE->getBase()->getType()->getPointeeType()), in generateInfoForComponentList()
7040 CGF.getContext().getTypeAlignInChars(OAShE->getBase()->getType())); in generateInfoForComponentList()
7044 BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress(CGF); in generateInfoForComponentList()
7052 CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) { in generateInfoForComponentList()
7054 BP = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD); in generateInfoForComponentList()
7069 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() || in generateInfoForComponentList()
7071 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>()); in generateInfoForComponentList()
7144 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>()); in generateInfoForComponentList()
7212 auto &&EmitMemberExprBase = [](CodeGenFunction &CGF, in generateInfoForComponentList()
7222 CGF.EmitPointerWithAlignment(BaseExpr, &BaseInfo, &TBAAInfo); in generateInfoForComponentList()
7224 BaseLV = CGF.MakeAddrLValue(Addr, PtrTy, BaseInfo, TBAAInfo); in generateInfoForComponentList()
7226 BaseLV = CGF.EmitOMPSharedLValue(BaseExpr); in generateInfoForComponentList()
7232 Address(CGF.EmitScalarExpr(OAShE->getBase()), in generateInfoForComponentList()
7233 CGF.ConvertTypeForMem( in generateInfoForComponentList()
7235 CGF.getContext().getTypeAlignInChars( in generateInfoForComponentList()
7239 LValue BaseLVal = EmitMemberExprBase(CGF, ME); in generateInfoForComponentList()
7240 LowestElem = CGF.EmitLValueForFieldInitialization( in generateInfoForComponentList()
7242 .getAddress(CGF); in generateInfoForComponentList()
7243 LB = CGF.EmitLoadOfReferenceLValue(LowestElem, MapDecl->getType()) in generateInfoForComponentList()
7244 .getAddress(CGF); in generateInfoForComponentList()
7247 CGF.EmitOMPSharedLValue(I->getAssociatedExpression()) in generateInfoForComponentList()
7248 .getAddress(CGF); in generateInfoForComponentList()
7269 CharUnits TypeSize = CGF.getContext().getTypeSizeInChars( in generateInfoForComponentList()
7271 Address HB = CGF.Builder.CreateConstGEP( in generateInfoForComponentList()
7272 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in generateInfoForComponentList()
7273 LowestElem, CGF.VoidPtrTy, CGF.Int8Ty), in generateInfoForComponentList()
7303 LValue BaseLVal = EmitMemberExprBase(CGF, ME); in generateInfoForComponentList()
7305 CGF.EmitLValueForFieldInitialization(BaseLVal, FD) in generateInfoForComponentList()
7306 .getAddress(CGF); in generateInfoForComponentList()
7309 CGF.EmitOMPSharedLValue(MC.getAssociatedExpression()) in generateInfoForComponentList()
7310 .getAddress(CGF); in generateInfoForComponentList()
7312 Size = CGF.Builder.CreatePtrDiff( in generateInfoForComponentList()
7313 CGF.Int8Ty, ComponentLB.getPointer(), LB.getPointer()); in generateInfoForComponentList()
7323 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateInfoForComponentList()
7324 Size, CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForComponentList()
7329 LB = CGF.Builder.CreateConstGEP(ComponentLB, 1); in generateInfoForComponentList()
7336 Size = CGF.Builder.CreatePtrDiff( in generateInfoForComponentList()
7337 CGF.Int8Ty, CGF.Builder.CreateConstGEP(HB, 1).getPointer(), in generateInfoForComponentList()
7340 CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForComponentList()
7360 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateInfoForComponentList()
7361 Size, CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForComponentList()
7370 StructBaseCombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateInfoForComponentList()
7371 Size, CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForComponentList()
7432 CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false) in generateInfoForComponentList()
7433 .getAddress(CGF); in generateInfoForComponentList()
7445 CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false) in generateInfoForComponentList()
7446 .getAddress(CGF); in generateInfoForComponentList()
7475 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>()); in generateInfoForComponentList()
7487 const ASTContext &Context = CGF.getContext(); in generateInfoForComponentList()
7491 MapValuesArrayTy CurOffsets = {llvm::ConstantInt::get(CGF.CGM.Int64Ty, 0)}; in generateInfoForComponentList()
7492 MapValuesArrayTy CurCounts = {llvm::ConstantInt::get(CGF.CGM.Int64Ty, 1)}; in generateInfoForComponentList()
7494 MapValuesArrayTy DimSizes{llvm::ConstantInt::get(CGF.CGM.Int64Ty, 1)}; in generateInfoForComponentList()
7539 llvm::ConstantInt::get(CGF.Int64Ty, ElementTypeSize)); in generateInfoForComponentList()
7547 CGF.Int64Ty, CAT->getSize().getZExtValue())); in generateInfoForComponentList()
7549 DimSizes.push_back(CGF.Builder.CreateIntCast( in generateInfoForComponentList()
7550 CGF.EmitScalarExpr(VAT->getSizeExpr()), CGF.Int64Ty, in generateInfoForComponentList()
7559 llvm::ConstantInt::get(CGF.CGM.Int64Ty, ElementTypeSize); in generateInfoForComponentList()
7573 llvm::Value *Offset = CGF.Builder.CreateIntCast( in generateInfoForComponentList()
7574 CGF.EmitScalarExpr(AE->getIdx()), CGF.Int64Ty, in generateInfoForComponentList()
7577 CurCounts.push_back(llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/1)); in generateInfoForComponentList()
7592 Offset = llvm::ConstantInt::get(CGF.Int64Ty, 0); in generateInfoForComponentList()
7594 Offset = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(OffsetExpr), in generateInfoForComponentList()
7595 CGF.Int64Ty, in generateInfoForComponentList()
7610 Count = llvm::ConstantInt::get(CGF.Int64Ty, 1); in generateInfoForComponentList()
7619 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(StrideExpr), in generateInfoForComponentList()
7620 CGF.Int64Ty, /*isSigned=*/false) in generateInfoForComponentList()
7623 Count = CGF.Builder.CreateUDiv( in generateInfoForComponentList()
7624 CGF.Builder.CreateNUWSub(*DI, Offset), Stride); in generateInfoForComponentList()
7626 Count = CGF.Builder.CreateNUWSub(*DI, Offset); in generateInfoForComponentList()
7629 Count = CGF.EmitScalarExpr(CountExpr); in generateInfoForComponentList()
7631 Count = CGF.Builder.CreateIntCast(Count, CGF.Int64Ty, /*isSigned=*/false); in generateInfoForComponentList()
7644 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(StrideExpr), in generateInfoForComponentList()
7645 CGF.Int64Ty, /*isSigned=*/false) in generateInfoForComponentList()
7647 DimProd = CGF.Builder.CreateNUWMul(DimProd, *(DI - 1)); in generateInfoForComponentList()
7649 CurStrides.push_back(CGF.Builder.CreateNUWMul(DimProd, Stride)); in generateInfoForComponentList()
7694 const CGRecordLayout &RL = CGF.getTypes().getCGRecordLayout(RD); in getPlainLayout()
7710 if (Base->isEmpty() || CGF.getContext() in getPlainLayout()
7735 if (!Field->isBitField() && !Field->isZeroSize(CGF.getContext())) { in getPlainLayout()
7861 CodeGenFunction &CGF, bool IsDevAddr) { in generateAllInfoForClauses() argument
7869 llvm::Constant::getNullValue(CGF.Int64Ty)); in generateAllInfoForClauses()
7877 &InfoGen](CodeGenFunction &CGF, const Expr *IE, const ValueDecl *VD, in generateAllInfoForClauses()
7900 Ptr = CGF.EmitLValue(IE).getPointer(CGF); in generateAllInfoForClauses()
7902 Ptr = CGF.EmitScalarExpr(IE); in generateAllInfoForClauses()
7904 Ptr = CGF.EmitLoadOfScalar(CGF.EmitLValue(IE), IE->getExprLoc()); in generateAllInfoForClauses()
7906 UseDeviceDataCombinedInfoGen(VD, Ptr, CGF, IsDevAddr); in generateAllInfoForClauses()
7910 auto &&IsMapInfoExist = [&Info](CodeGenFunction &CGF, const ValueDecl *VD, in generateAllInfoForClauses()
7940 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() || in generateAllInfoForClauses()
7977 if (IsMapInfoExist(CGF, VD, IE, /*IsDevAddr=*/false)) in generateAllInfoForClauses()
7979 MapInfoGen(CGF, IE, VD, Components, C->isImplicit(), in generateAllInfoForClauses()
7999 if (IsMapInfoExist(CGF, VD, IE, /*IsDevAddr=*/true)) in generateAllInfoForClauses()
8001 MapInfoGen(CGF, IE, VD, Components, C->isImplicit(), in generateAllInfoForClauses()
8083 Ptr = this->CGF.EmitLValue(L.IE).getPointer(CGF); in generateAllInfoForClauses()
8085 Ptr = this->CGF.EmitScalarExpr(L.IE); in generateAllInfoForClauses()
8094 BasePtr = this->CGF.EmitLValue(L.IE).getPointer(CGF); in generateAllInfoForClauses()
8095 Ptr = this->CGF.EmitLoadOfScalar(this->CGF.EmitLValue(L.IE), in generateAllInfoForClauses()
8112 llvm::Constant::getNullValue(this->CGF.Int64Ty)); in generateAllInfoForClauses()
8140 MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF) in MappableExprsHandler() argument
8141 : CurDir(&Dir), CGF(CGF) { in MappableExprsHandler()
8186 MappableExprsHandler(const OMPDeclareMapperDecl &Dir, CodeGenFunction &CGF) in MappableExprsHandler() argument
8187 : CurDir(&Dir), CGF(CGF) {} in MappableExprsHandler()
8217 CGF.CurFuncDecl ? dyn_cast<CXXMethodDecl>(CGF.CurFuncDecl) : nullptr; in emitCombinedEntry()
8232 CGF.Builder.CreateIntCast(CGF.getTypeSize(Ty), CGF.Int64Ty, in emitCombinedEntry()
8239 llvm::Value *HAddr = CGF.Builder.CreateConstGEP1_32( in emitCombinedEntry()
8241 llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(LB, CGF.VoidPtrTy); in emitCombinedEntry()
8242 llvm::Value *CHAddr = CGF.Builder.CreatePointerCast(HAddr, CGF.VoidPtrTy); in emitCombinedEntry()
8243 llvm::Value *Diff = CGF.Builder.CreatePtrDiff(CGF.Int8Ty, CHAddr, CLAddr); in emitCombinedEntry()
8244 llvm::Value *Size = CGF.Builder.CreateIntCast(Diff, CGF.Int64Ty, in emitCombinedEntry()
8324 Address VDAddr(Arg, CGF.ConvertTypeForMem(VDType), in generateInfoForLambdaCaptures()
8325 CGF.getContext().getDeclAlign(VD)); in generateInfoForLambdaCaptures()
8326 LValue VDLVal = CGF.MakeAddrLValue(VDAddr, VDType); in generateInfoForLambdaCaptures()
8332 CGF.EmitLValueForFieldInitialization(VDLVal, ThisCapture); in generateInfoForLambdaCaptures()
8333 LValue ThisLValVal = CGF.EmitLValueForField(VDLVal, ThisCapture); in generateInfoForLambdaCaptures()
8334 LambdaPointers.try_emplace(ThisLVal.getPointer(CGF), in generateInfoForLambdaCaptures()
8335 VDLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8337 CombinedInfo.BasePointers.push_back(ThisLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8340 CombinedInfo.Pointers.push_back(ThisLValVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8342 CGF.Builder.CreateIntCast(CGF.getTypeSize(CGF.getContext().VoidPtrTy), in generateInfoForLambdaCaptures()
8343 CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForLambdaCaptures()
8359 LValue VarLVal = CGF.EmitLValueForFieldInitialization(VDLVal, It->second); in generateInfoForLambdaCaptures()
8361 LValue VarLValVal = CGF.EmitLValueForField(VDLVal, It->second); in generateInfoForLambdaCaptures()
8362 LambdaPointers.try_emplace(VarLVal.getPointer(CGF), in generateInfoForLambdaCaptures()
8363 VDLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8365 CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8368 CombinedInfo.Pointers.push_back(VarLValVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8369 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateInfoForLambdaCaptures()
8370 CGF.getTypeSize( in generateInfoForLambdaCaptures()
8372 CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForLambdaCaptures()
8374 RValue VarRVal = CGF.EmitLoadOfLValue(VarLVal, RD->getLocation()); in generateInfoForLambdaCaptures()
8375 LambdaPointers.try_emplace(VarLVal.getPointer(CGF), in generateInfoForLambdaCaptures()
8376 VDLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8378 CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8382 CombinedInfo.Sizes.push_back(llvm::ConstantInt::get(CGF.Int64Ty, 0)); in generateInfoForLambdaCaptures()
8453 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateInfoForCapture()
8454 CGF.getTypeSize(CGF.getContext().VoidPtrTy), CGF.Int64Ty, in generateInfoForCapture()
8699 CGF.Builder.CreateIntCast(CGF.getTypeSize(PtrTy->getPointeeType()), in generateDefaultMapInfo()
8700 CGF.Int64Ty, /*isSigned=*/true)); in generateDefaultMapInfo()
8716 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateDefaultMapInfo()
8717 CGF.getTypeSize(RI.getType()), CGF.Int64Ty, /*isSigned=*/true)); in generateDefaultMapInfo()
8722 CombinedInfo.Sizes.push_back(llvm::Constant::getNullValue(CGF.Int64Ty)); in generateDefaultMapInfo()
8731 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateDefaultMapInfo()
8732 CGF.getTypeSize(ElementType), CGF.Int64Ty, /*isSigned=*/true)); in generateDefaultMapInfo()
8744 Address PtrAddr = CGF.EmitLoadOfReference(CGF.MakeAddrLValue( in generateDefaultMapInfo()
8745 CV, ElementType, CGF.getContext().getDeclAlign(VD), in generateDefaultMapInfo()
8783 emitMappingInformation(CodeGenFunction &CGF, llvm::OpenMPIRBuilder &OMPBuilder, in emitMappingInformation() argument
8802 PrintingPolicy P(CGF.getContext().getLangOpts()); in emitMappingInformation()
8810 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc); in emitMappingInformation()
8820 CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo, in emitOffloadingArrays() argument
8823 CodeGenModule &CGM = CGF.CGM; in emitOffloadingArrays()
8830 InsertPointTy AllocaIP(CGF.AllocaInsertPt->getParent(), in emitOffloadingArrays()
8831 CGF.AllocaInsertPt->getIterator()); in emitOffloadingArrays()
8832 InsertPointTy CodeGenIP(CGF.Builder.GetInsertBlock(), in emitOffloadingArrays()
8833 CGF.Builder.GetInsertPoint()); in emitOffloadingArrays()
8836 return emitMappingInformation(CGF, OMPBuilder, MapExpr); in emitOffloadingArrays()
8855 MFunc = CGF.CGM.getOpenMPRuntime().getOrCreateUserDefinedMapperFunc( in emitOffloadingArrays()
9009 CodeGenFunction *CGF) { in emitUserDefinedMapper() argument
9265 if (CGF) { in emitUserDefinedMapper()
9266 auto &Decls = FunctionUDMMap.FindAndConstruct(CGF->CurFn); in emitUserDefinedMapper()
9357 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitTargetNumIterationsCall() argument
9358 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, in emitTargetNumIterationsCall() argument
9368 return llvm::ConstantInt::get(CGF.Int64Ty, 0); in emitTargetNumIterationsCall()
9371 if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD)) in emitTargetNumIterationsCall()
9373 return llvm::ConstantInt::get(CGF.Int64Ty, 0); in emitTargetNumIterationsCall()
9381 bool OffloadingMandatory, CodeGenFunction &CGF) { in emitTargetCallFallback() argument
9383 CGF.Builder.CreateUnreachable(); in emitTargetCallFallback()
9387 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars); in emitTargetCallFallback()
9389 OMPRuntime->emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn, in emitTargetCallFallback()
9396 CodeGenFunction &CGF) { in emitDeviceID() argument
9403 llvm::Value *DevVal = CGF.EmitScalarExpr(Device.getPointer()); in emitDeviceID()
9405 CGF.Builder.CreateIntCast(DevVal, CGF.Int64Ty, /*isSigned=*/true); in emitDeviceID()
9407 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitDeviceID()
9413 CodeGenFunction &CGF) { in emitDynCGGroupMem() argument
9414 llvm::Value *DynCGroupMem = CGF.Builder.getInt32(0); in emitDynCGGroupMem()
9417 CodeGenFunction::RunCleanupsScope DynCGroupMemScope(CGF); in emitDynCGGroupMem()
9418 llvm::Value *DynCGroupMemVal = CGF.EmitScalarExpr( in emitDynCGGroupMem()
9420 DynCGroupMem = CGF.Builder.CreateIntCast(DynCGroupMemVal, CGF.Int32Ty, in emitDynCGGroupMem()
9434 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, in emitTargetCallKernelLaunch() argument
9437 CodeGenFunction &CGF, CodeGenModule &CGM) { in emitTargetCallKernelLaunch()
9444 MappableExprsHandler MEHandler(D, CGF); in emitTargetCallKernelLaunch()
9465 CurInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in emitTargetCallKernelLaunch()
9466 CGF.getTypeSize(RI->getType()), CGF.Int64Ty, /*isSigned=*/true)); in emitTargetCallKernelLaunch()
9520 emitOffloadingArrays(CGF, CombinedInfo, Info, OMPBuilder); in emitTargetCallKernelLaunch()
9521 bool EmitDebug = CGF.CGM.getCodeGenOpts().getDebugInfo() != in emitTargetCallKernelLaunch()
9523 OMPBuilder.emitOffloadingArraysArgument(CGF.Builder, Info.RTArgs, Info, in emitTargetCallKernelLaunch()
9529 CGF.VoidPtrTy, CGM.getPointerAlign()); in emitTargetCallKernelLaunch()
9531 Address(Info.RTArgs.PointersArray, CGF.VoidPtrTy, CGM.getPointerAlign()); in emitTargetCallKernelLaunch()
9533 Address(Info.RTArgs.SizesArray, CGF.Int64Ty, CGM.getPointerAlign()); in emitTargetCallKernelLaunch()
9535 Address(Info.RTArgs.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign()); in emitTargetCallKernelLaunch()
9542 SizeEmitter](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetCallKernelLaunch() argument
9551 RequiresOuterTask, CS, OffloadingMandatory, CGF); in emitTargetCallKernelLaunch()
9565 OffloadingMandatory, &CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) in emitTargetCallKernelLaunch()
9567 CGF.Builder.restoreIP(IP); in emitTargetCallKernelLaunch()
9569 RequiresOuterTask, CS, OffloadingMandatory, CGF); in emitTargetCallKernelLaunch()
9570 return CGF.Builder.saveIP(); in emitTargetCallKernelLaunch()
9573 llvm::Value *DeviceID = emitDeviceID(Device, CGF); in emitTargetCallKernelLaunch()
9574 llvm::Value *NumTeams = OMPRuntime->emitNumTeamsForTargetDirective(CGF, D); in emitTargetCallKernelLaunch()
9576 OMPRuntime->emitNumThreadsForTargetDirective(CGF, D); in emitTargetCallKernelLaunch()
9577 llvm::Value *RTLoc = OMPRuntime->emitUpdateLocation(CGF, D.getBeginLoc()); in emitTargetCallKernelLaunch()
9579 OMPRuntime->emitTargetNumIterationsCall(CGF, D, SizeEmitter); in emitTargetCallKernelLaunch()
9580 llvm::Value *DynCGGroupMem = emitDynCGGroupMem(D, CGF); in emitTargetCallKernelLaunch()
9582 CGF.AllocaInsertPt->getParent(), CGF.AllocaInsertPt->getIterator()); in emitTargetCallKernelLaunch()
9592 CGF.Builder.restoreIP(OMPRuntime->getOMPBuilder().emitKernelLaunch( in emitTargetCallKernelLaunch()
9593 CGF.Builder, OutlinedFn, OutlinedFnID, EmitTargetCallFallbackCB, Args, in emitTargetCallKernelLaunch()
9598 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo); in emitTargetCallKernelLaunch()
9600 OMPRuntime->emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen); in emitTargetCallKernelLaunch()
9608 bool OffloadingMandatory, CodeGenFunction &CGF) { in emitTargetCallElse() argument
9613 OffloadingMandatory](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetCallElse() argument
9615 RequiresOuterTask, CS, OffloadingMandatory, CGF); in emitTargetCallElse()
9620 CGF.EmitOMPTargetTaskBasedDirective(D, ElseGen, InputInfo); in emitTargetCallElse()
9622 OMPRuntime->emitInlinedDirective(CGF, D.getDirectiveKind(), ElseGen); in emitTargetCallElse()
9627 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitTargetCall() argument
9630 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, in emitTargetCall() argument
9633 if (!CGF.HaveInsertPoint()) in emitTargetCall()
9650 auto &&ArgsCodegen = [&CS, &CapturedVars](CodeGenFunction &CGF, in emitTargetCall()
9652 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars); in emitTargetCall()
9654 emitInlinedDirective(CGF, OMPD_unknown, ArgsCodegen); in emitTargetCall()
9663 &MapNamesArray, SizeEmitter](CodeGenFunction &CGF, in emitTargetCall() argument
9668 MapNamesArray, SizeEmitter, CGF, CGM); in emitTargetCall()
9673 OffloadingMandatory](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetCall() argument
9675 CS, OffloadingMandatory, CGF); in emitTargetCall()
9684 emitIfClause(CGF, IfCond, TargetThenGen, TargetElseGen); in emitTargetCall()
9687 ThenRCG(CGF); in emitTargetCall()
9691 ElseRCG(CGF); in emitTargetCall()
10007 CodeGenFunction &CGF, const OMPExecutableDirective &D) const { in adjustTargetSpecificDataForLambdas() argument
10117 CodeGenFunction CGF(CGM); in emitRequiresDirectiveRegFun() local
10122 CGF.StartFunction(GlobalDecl(), C.VoidTy, RequiresRegFn, FI, {}); in emitRequiresDirectiveRegFun()
10132 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitRequiresDirectiveRegFun()
10136 CGF.FinishFunction(); in emitRequiresDirectiveRegFun()
10141 void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF, in emitTeamsCall() argument
10146 if (!CGF.HaveInsertPoint()) in emitTeamsCall()
10149 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitTeamsCall()
10150 CodeGenFunction::RunCleanupsScope Scope(CGF); in emitTeamsCall()
10155 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars in emitTeamsCall()
10156 CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy())}; in emitTeamsCall()
10163 CGF.EmitRuntimeCall(RTLFn, RealArgs); in emitTeamsCall()
10166 void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF, in emitNumTeamsClause() argument
10170 if (!CGF.HaveInsertPoint()) in emitNumTeamsClause()
10173 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitNumTeamsClause()
10177 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(NumTeams), in emitNumTeamsClause()
10178 CGF.CGM.Int32Ty, /* isSigned = */ true) in emitNumTeamsClause()
10179 : CGF.Builder.getInt32(0); in emitNumTeamsClause()
10183 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(ThreadLimit), in emitNumTeamsClause()
10184 CGF.CGM.Int32Ty, /* isSigned = */ true) in emitNumTeamsClause()
10185 : CGF.Builder.getInt32(0); in emitNumTeamsClause()
10188 llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal, in emitNumTeamsClause()
10190 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitNumTeamsClause()
10195 void CGOpenMPRuntime::emitThreadLimitClause(CodeGenFunction &CGF, in emitThreadLimitClause() argument
10198 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitThreadLimitClause()
10201 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(ThreadLimit), in emitThreadLimitClause()
10202 CGF.CGM.Int32Ty, /* isSigned = */ true) in emitThreadLimitClause()
10203 : CGF.Builder.getInt32(0); in emitThreadLimitClause()
10206 llvm::Value *ThreadLimitArgs[] = {RTLoc, getThreadID(CGF, Loc), in emitThreadLimitClause()
10208 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitThreadLimitClause()
10214 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, in emitTargetDataCalls() argument
10217 if (!CGF.HaveInsertPoint()) in emitTargetDataCalls()
10228 IfCondVal = CGF.EvaluateExprAsBool(IfCond); in emitTargetDataCalls()
10233 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTargetDataCalls()
10234 CGF.Int64Ty, /*isSigned=*/true); in emitTargetDataCalls()
10236 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTargetDataCalls()
10243 CGF.Builder.restoreIP(CodeGenIP); in emitTargetDataCalls()
10245 MappableExprsHandler MEHandler(D, CGF); in emitTargetDataCalls()
10249 return emitMappingInformation(CGF, OMPBuilder, MapExpr); in emitTargetDataCalls()
10262 CGF.Builder.restoreIP(CodeGenIP); in emitTargetDataCalls()
10266 CodeGen(CGF); in emitTargetDataCalls()
10271 CodeGen(CGF); in emitTargetDataCalls()
10277 CodeGen(CGF); in emitTargetDataCalls()
10281 return InsertPointTy(CGF.Builder.GetInsertBlock(), in emitTargetDataCalls()
10282 CGF.Builder.GetInsertPoint()); in emitTargetDataCalls()
10295 MFunc = CGF.CGM.getOpenMPRuntime().getOrCreateUserDefinedMapperFunc( in emitTargetDataCalls()
10302 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTargetDataCalls()
10304 InsertPointTy AllocaIP(CGF.AllocaInsertPt->getParent(), in emitTargetDataCalls()
10305 CGF.AllocaInsertPt->getIterator()); in emitTargetDataCalls()
10306 InsertPointTy CodeGenIP(CGF.Builder.GetInsertBlock(), in emitTargetDataCalls()
10307 CGF.Builder.GetInsertPoint()); in emitTargetDataCalls()
10309 CGF.Builder.restoreIP(OMPBuilder.createTargetData( in emitTargetDataCalls()
10315 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, in emitTargetDataStandAloneCall() argument
10317 if (!CGF.HaveInsertPoint()) in emitTargetDataStandAloneCall()
10330 &MapNamesArray](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetDataStandAloneCall() argument
10334 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTargetDataStandAloneCall()
10335 CGF.Int64Ty, /*isSigned=*/true); in emitTargetDataStandAloneCall()
10337 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTargetDataStandAloneCall()
10342 CGF.Builder.getInt32(InputInfo.NumberOfTargetItems); in emitTargetDataStandAloneCall()
10345 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTargetDataStandAloneCall()
10444 CGF.EmitRuntimeCall( in emitTargetDataStandAloneCall()
10450 &MapNamesArray](CodeGenFunction &CGF, in emitTargetDataStandAloneCall()
10456 MappableExprsHandler MEHandler(D, CGF); in emitTargetDataStandAloneCall()
10461 emitOffloadingArrays(CGF, CombinedInfo, Info, OMPBuilder, in emitTargetDataStandAloneCall()
10465 bool EmitDebug = CGF.CGM.getCodeGenOpts().getDebugInfo() != in emitTargetDataStandAloneCall()
10467 OMPBuilder.emitOffloadingArraysArgument(CGF.Builder, Info.RTArgs, Info, in emitTargetDataStandAloneCall()
10472 CGF.VoidPtrTy, CGM.getPointerAlign()); in emitTargetDataStandAloneCall()
10473 InputInfo.PointersArray = Address(Info.RTArgs.PointersArray, CGF.VoidPtrTy, in emitTargetDataStandAloneCall()
10476 Address(Info.RTArgs.SizesArray, CGF.Int64Ty, CGM.getPointerAlign()); in emitTargetDataStandAloneCall()
10478 Address(Info.RTArgs.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign()); in emitTargetDataStandAloneCall()
10482 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo); in emitTargetDataStandAloneCall()
10484 emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen); in emitTargetDataStandAloneCall()
10488 emitIfClause(CGF, IfCond, TargetThenGen, in emitTargetDataStandAloneCall()
10489 [](CodeGenFunction &CGF, PrePostActionTy &) {}); in emitTargetDataStandAloneCall() argument
10492 ThenRCG(CGF); in emitTargetDataStandAloneCall()
11101 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { in Emit() argument
11102 if (!CGF.HaveInsertPoint()) in Emit()
11104 CGF.EmitRuntimeCall(RTLFn, Args); in Emit()
11109 void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF, in emitDoacrossInit() argument
11112 if (!CGF.HaveInsertPoint()) in emitDoacrossInit()
11138 Address DimsAddr = CGF.CreateMemTemp(ArrayTy, "dims"); in emitDoacrossInit()
11139 CGF.EmitNullInitialization(DimsAddr, ArrayTy); in emitDoacrossInit()
11143 LValue DimsLVal = CGF.MakeAddrLValue( in emitDoacrossInit()
11144 CGF.Builder.CreateConstArrayGEP(DimsAddr, I), KmpDimTy); in emitDoacrossInit()
11146 LValue UpperLVal = CGF.EmitLValueForField( in emitDoacrossInit()
11148 llvm::Value *NumIterVal = CGF.EmitScalarConversion( in emitDoacrossInit()
11149 CGF.EmitScalarExpr(NumIterations[I]), NumIterations[I]->getType(), in emitDoacrossInit()
11151 CGF.EmitStoreOfScalar(NumIterVal, UpperLVal); in emitDoacrossInit()
11153 LValue StrideLVal = CGF.EmitLValueForField( in emitDoacrossInit()
11155 CGF.EmitStoreOfScalar(llvm::ConstantInt::getSigned(CGM.Int64Ty, /*V=*/1), in emitDoacrossInit()
11162 emitUpdateLocation(CGF, D.getBeginLoc()), in emitDoacrossInit()
11163 getThreadID(CGF, D.getBeginLoc()), in emitDoacrossInit()
11165 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDoacrossInit()
11166 CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).getPointer(), in emitDoacrossInit()
11171 CGF.EmitRuntimeCall(RTLFn, Args); in emitDoacrossInit()
11173 emitUpdateLocation(CGF, D.getEndLoc()), getThreadID(CGF, D.getEndLoc())}; in emitDoacrossInit()
11176 CGF.EHStack.pushCleanup<DoacrossCleanupTy>(NormalAndEHCleanup, FiniRTLFn, in emitDoacrossInit()
11181 static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM, in EmitDoacrossOrdered() argument
11189 Address CntAddr = CGF.CreateMemTemp(ArrayTy, ".cnt.addr"); in EmitDoacrossOrdered()
11193 llvm::Value *CntVal = CGF.EmitScalarConversion( in EmitDoacrossOrdered()
11194 CGF.EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty, in EmitDoacrossOrdered()
11196 CGF.EmitStoreOfScalar(CntVal, CGF.Builder.CreateConstArrayGEP(CntAddr, I), in EmitDoacrossOrdered()
11200 ULoc, ThreadID, CGF.Builder.CreateConstArrayGEP(CntAddr, 0).getPointer()}; in EmitDoacrossOrdered()
11212 CGF.EmitRuntimeCall(RTLFn, Args); in EmitDoacrossOrdered()
11215 void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, in emitDoacrossOrdered() argument
11218 CGF, CGM, C, emitUpdateLocation(CGF, C->getBeginLoc()), in emitDoacrossOrdered()
11219 getThreadID(CGF, C->getBeginLoc())); in emitDoacrossOrdered()
11222 void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, in emitDoacrossOrdered() argument
11225 CGF, CGM, C, emitUpdateLocation(CGF, C->getBeginLoc()), in emitDoacrossOrdered()
11226 getThreadID(CGF, C->getBeginLoc())); in emitDoacrossOrdered()
11229 void CGOpenMPRuntime::emitCall(CodeGenFunction &CGF, SourceLocation Loc, in emitCall() argument
11233 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc); in emitCall()
11237 CGF.EmitNounwindRuntimeCall(Fn, Args); in emitCall()
11241 CGF.EmitRuntimeCall(Callee, Args); in emitCall()
11245 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, in emitOutlinedFunctionCall() argument
11247 emitCall(CGF, Loc, OutlinedFn, Args); in emitOutlinedFunctionCall()
11250 void CGOpenMPRuntime::emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) { in emitFunctionProlog() argument
11256 Address CGOpenMPRuntime::getParameterAddress(CodeGenFunction &CGF, in getParameterAddress() argument
11259 return CGF.GetAddrOfLocalVar(NativeParam); in getParameterAddress()
11264 static llvm::Value *getAllocatorVal(CodeGenFunction &CGF, in getAllocatorVal() argument
11268 AllocVal = CGF.EmitScalarExpr(Allocator); in getAllocatorVal()
11271 AllocVal = CGF.EmitScalarConversion(AllocVal, Allocator->getType(), in getAllocatorVal()
11272 CGF.getContext().VoidPtrTy, in getAllocatorVal()
11277 CGF.CGM.getTypes().ConvertType(CGF.getContext().VoidPtrTy)); in getAllocatorVal()
11292 Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF, in getAddressOfLocalVariable() argument
11298 auto It = FunctionToUntiedTaskStackMap.find(CGF.CurFn); in getAddressOfLocalVariable()
11316 Size = CGF.getTypeSize(CVD->getType()); in getAddressOfLocalVariable()
11318 Size = CGF.Builder.CreateNUWAdd( in getAddressOfLocalVariable()
11320 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align)); in getAddressOfLocalVariable()
11321 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align)); in getAddressOfLocalVariable()
11326 llvm::Value *ThreadID = getThreadID(CGF, CVD->getBeginLoc()); in getAddressOfLocalVariable()
11329 llvm::Value *AllocVal = getAllocatorVal(CGF, Allocator); in getAddressOfLocalVariable()
11339 llvm::Value *Addr = CGF.EmitRuntimeCall( in getAddressOfLocalVariable()
11345 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddressOfLocalVariable()
11346 Addr, CGF.ConvertTypeForMem(Ty), getName({CVD->getName(), ".addr"})); in getAddressOfLocalVariable()
11348 CGF.EmitStoreOfScalar(Addr, UntiedAddr, /*Volatile=*/false, Ty); in getAddressOfLocalVariable()
11363 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { in getAddressOfLocalVariable() argument
11364 if (!CGF.HaveInsertPoint()) in getAddressOfLocalVariable()
11367 Args[0] = CGF.CGM.getOpenMPRuntime().getThreadID( in getAddressOfLocalVariable()
11368 CGF, SourceLocation::getFromRawEncoding(LocEncoding)); in getAddressOfLocalVariable()
11369 Args[1] = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddressOfLocalVariable()
11370 Addr.getPointer(), CGF.VoidPtrTy); in getAddressOfLocalVariable()
11371 llvm::Value *AllocVal = getAllocatorVal(CGF, AllocExpr); in getAddressOfLocalVariable()
11373 CGF.EmitRuntimeCall(RTLFn, Args); in getAddressOfLocalVariable()
11379 : Address(Addr, CGF.ConvertTypeForMem(CVD->getType()), Align); in getAddressOfLocalVariable()
11380 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>( in getAddressOfLocalVariable()
11385 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in getAddressOfLocalVariable()
11386 Region->emitUntiedSwitch(CGF); in getAddressOfLocalVariable()
11392 bool CGOpenMPRuntime::isLocalVarInUntiedTask(CodeGenFunction &CGF, in isLocalVarInUntiedTask() argument
11394 auto It = FunctionToUntiedTaskStackMap.find(CGF.CurFn); in isLocalVarInUntiedTask()
11433 CodeGenFunction &CGF, in UntiedTaskLocalDeclsRAII() argument
11436 : CGM(CGF.CGM), NeedToPush(!LocalVars.empty()) { in UntiedTaskLocalDeclsRAII()
11440 CGF.CurFn, CGM.getOpenMPRuntime().UntiedLocalVarsStack.size()); in UntiedTaskLocalDeclsRAII()
11538 CodeGenFunction &CGF, const OMPExecutableDirective &S, LValue IVLVal) in LastprivateConditionalRAII() argument
11539 : CGM(CGF.CGM), in LastprivateConditionalRAII()
11566 Data.Fn = CGF.CurFn;
11570 CodeGenFunction &CGF, const OMPExecutableDirective &S) in LastprivateConditionalRAII() argument
11571 : CGM(CGF.CGM), Action(ActionToDo::DoNotPush) { in LastprivateConditionalRAII()
11583 Data.Fn = CGF.CurFn; in LastprivateConditionalRAII()
11590 CodeGenFunction &CGF, const OMPExecutableDirective &S) { in disable() argument
11591 return LastprivateConditionalRAII(CGF, S); in disable()
11610 Address CGOpenMPRuntime::emitLastprivateConditionalInit(CodeGenFunction &CGF, in emitLastprivateConditionalInit() argument
11613 auto I = LastprivateConditionalToTypes.find(CGF.CurFn); in emitLastprivateConditionalInit()
11615 I = LastprivateConditionalToTypes.try_emplace(CGF.CurFn).first; in emitLastprivateConditionalInit()
11628 Address Addr = CGF.CreateMemTemp(NewType, C.getDeclAlign(VD), VD->getName()); in emitLastprivateConditionalInit()
11629 BaseLVal = CGF.MakeAddrLValue(Addr, NewType, AlignmentSource::Decl); in emitLastprivateConditionalInit()
11638 CGF.EmitLValueForField(BaseLVal, FiredField); in emitLastprivateConditionalInit()
11639 CGF.EmitStoreOfScalar( in emitLastprivateConditionalInit()
11640 llvm::ConstantInt::getNullValue(CGF.ConvertTypeForMem(C.CharTy)), in emitLastprivateConditionalInit()
11642 return CGF.EmitLValueForField(BaseLVal, VDField).getAddress(CGF); in emitLastprivateConditionalInit()
11716 void CGOpenMPRuntime::emitLastprivateConditionalUpdate(CodeGenFunction &CGF, in emitLastprivateConditionalUpdate() argument
11723 llvm::Type *LLIVTy = CGF.ConvertTypeForMem(IVLVal.getType()); in emitLastprivateConditionalUpdate()
11728 LValue LastIVLVal = CGF.MakeNaturalAlignAddrLValue(LastIV, IVLVal.getType()); in emitLastprivateConditionalUpdate()
11733 CGF.ConvertTypeForMem(LVal.getType()), UniqueDeclName); in emitLastprivateConditionalUpdate()
11735 LValue LastLVal = CGF.MakeAddrLValue( in emitLastprivateConditionalUpdate()
11740 llvm::Value *IVVal = CGF.EmitLoadOfScalar(IVLVal, Loc); in emitLastprivateConditionalUpdate()
11748 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitLastprivateConditionalUpdate() argument
11749 Action.Enter(CGF); in emitLastprivateConditionalUpdate()
11750 llvm::Value *LastIVVal = CGF.EmitLoadOfScalar(LastIVLVal, Loc); in emitLastprivateConditionalUpdate()
11755 CmpRes = CGF.Builder.CreateICmpSLE(LastIVVal, IVVal); in emitLastprivateConditionalUpdate()
11759 CmpRes = CGF.Builder.CreateICmpULE(LastIVVal, IVVal); in emitLastprivateConditionalUpdate()
11761 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("lp_cond_then"); in emitLastprivateConditionalUpdate()
11762 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("lp_cond_exit"); in emitLastprivateConditionalUpdate()
11763 CGF.Builder.CreateCondBr(CmpRes, ThenBB, ExitBB); in emitLastprivateConditionalUpdate()
11765 CGF.EmitBlock(ThenBB); in emitLastprivateConditionalUpdate()
11768 CGF.EmitStoreOfScalar(IVVal, LastIVLVal); in emitLastprivateConditionalUpdate()
11771 switch (CGF.getEvaluationKind(LVal.getType())) { in emitLastprivateConditionalUpdate()
11773 llvm::Value *PrivVal = CGF.EmitLoadOfScalar(LVal, Loc); in emitLastprivateConditionalUpdate()
11774 CGF.EmitStoreOfScalar(PrivVal, LastLVal); in emitLastprivateConditionalUpdate()
11778 CodeGenFunction::ComplexPairTy PrivVal = CGF.EmitLoadOfComplex(LVal, Loc); in emitLastprivateConditionalUpdate()
11779 CGF.EmitStoreOfComplex(PrivVal, LastLVal, /*isInit=*/false); in emitLastprivateConditionalUpdate()
11787 CGF.EmitBranch(ExitBB); in emitLastprivateConditionalUpdate()
11789 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitLastprivateConditionalUpdate()
11790 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in emitLastprivateConditionalUpdate()
11796 ThenRCG(CGF); in emitLastprivateConditionalUpdate()
11798 emitCriticalRegion(CGF, UniqueDeclName, CodeGen, Loc); in emitLastprivateConditionalUpdate()
11802 void CGOpenMPRuntime::checkAndEmitLastprivateConditional(CodeGenFunction &CGF, in checkAndEmitLastprivateConditional() argument
11804 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty()) in checkAndEmitLastprivateConditional()
11816 if (FoundFn != CGF.CurFn) { in checkAndEmitLastprivateConditional()
11824 LValue PrivLVal = CGF.EmitLValue(FoundE); in checkAndEmitLastprivateConditional()
11825 Address StructAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in checkAndEmitLastprivateConditional()
11826 PrivLVal.getAddress(CGF), in checkAndEmitLastprivateConditional()
11827 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(StructTy)), in checkAndEmitLastprivateConditional()
11828 CGF.ConvertTypeForMem(StructTy)); in checkAndEmitLastprivateConditional()
11830 CGF.MakeAddrLValue(StructAddr, StructTy, AlignmentSource::Decl); in checkAndEmitLastprivateConditional()
11831 LValue FiredLVal = CGF.EmitLValueForField(BaseLVal, FiredDecl); in checkAndEmitLastprivateConditional()
11832 CGF.EmitAtomicStore(RValue::get(llvm::ConstantInt::get( in checkAndEmitLastprivateConditional()
11833 CGF.ConvertTypeForMem(FiredDecl->getType()), 1)), in checkAndEmitLastprivateConditional()
11841 LValue LVal = CGF.EmitLValue(FoundE); in checkAndEmitLastprivateConditional()
11842 emitLastprivateConditionalUpdate(CGF, IVLVal, UniqueDeclName, LVal, in checkAndEmitLastprivateConditional()
11847 CodeGenFunction &CGF, const OMPExecutableDirective &D, in checkAndEmitSharedLastprivateConditional() argument
11849 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty()) in checkAndEmitSharedLastprivateConditional()
11854 if (It == Range.end() || It->Fn != CGF.CurFn) in checkAndEmitSharedLastprivateConditional()
11872 CGF.EmitLValueForField(BaseLVal, std::get<2>(I->getSecond())); in checkAndEmitSharedLastprivateConditional()
11873 llvm::Value *Res = CGF.EmitLoadOfScalar(FiredLVal, D.getBeginLoc()); in checkAndEmitSharedLastprivateConditional()
11874 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Res); in checkAndEmitSharedLastprivateConditional()
11875 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("lpc.then"); in checkAndEmitSharedLastprivateConditional()
11876 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("lpc.done"); in checkAndEmitSharedLastprivateConditional()
11878 CGF.Builder.CreateCondBr(Cmp, ThenBB, DoneBB); in checkAndEmitSharedLastprivateConditional()
11879 CGF.EmitBlock(ThenBB); in checkAndEmitSharedLastprivateConditional()
11880 Address Addr = CGF.GetAddrOfLocalVar(VD); in checkAndEmitSharedLastprivateConditional()
11883 LVal = CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(), in checkAndEmitSharedLastprivateConditional()
11886 LVal = CGF.MakeAddrLValue(Addr, VD->getType().getNonReferenceType(), in checkAndEmitSharedLastprivateConditional()
11888 emitLastprivateConditionalUpdate(CGF, It->IVLVal, Pair.second, LVal, in checkAndEmitSharedLastprivateConditional()
11890 auto AL = ApplyDebugLocation::CreateArtificial(CGF); in checkAndEmitSharedLastprivateConditional()
11891 CGF.EmitBlock(DoneBB, /*IsFinal=*/true); in checkAndEmitSharedLastprivateConditional()
11897 CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD, in emitLastprivateConditionalFinalUpdate() argument
11899 if (CGF.getLangOpts().OpenMP < 50) in emitLastprivateConditionalFinalUpdate()
11909 LValue LPLVal = CGF.MakeAddrLValue( in emitLastprivateConditionalFinalUpdate()
11912 llvm::Value *Res = CGF.EmitLoadOfScalar(LPLVal, Loc); in emitLastprivateConditionalFinalUpdate()
11913 CGF.EmitStoreOfScalar(Res, PrivLVal); in emitLastprivateConditionalFinalUpdate()
11917 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitParallelOutlinedFunction() argument
11924 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitTeamsOutlinedFunction() argument
11938 void CGOpenMPSIMDRuntime::emitParallelCall(CodeGenFunction &CGF, in emitParallelCall() argument
11948 CodeGenFunction &CGF, StringRef CriticalName, in emitCriticalRegion() argument
11954 void CGOpenMPSIMDRuntime::emitMasterRegion(CodeGenFunction &CGF, in emitMasterRegion() argument
11960 void CGOpenMPSIMDRuntime::emitMaskedRegion(CodeGenFunction &CGF, in emitMaskedRegion() argument
11967 void CGOpenMPSIMDRuntime::emitTaskyieldCall(CodeGenFunction &CGF, in emitTaskyieldCall() argument
11973 CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, in emitTaskgroupRegion() argument
11979 CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, in emitSingleRegion() argument
11986 void CGOpenMPSIMDRuntime::emitOrderedRegion(CodeGenFunction &CGF, in emitOrderedRegion() argument
11993 void CGOpenMPSIMDRuntime::emitBarrierCall(CodeGenFunction &CGF, in emitBarrierCall() argument
12002 CodeGenFunction &CGF, SourceLocation Loc, in emitForDispatchInit() argument
12009 CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, in emitForStaticInit() argument
12015 CodeGenFunction &CGF, SourceLocation Loc, in emitDistributeStaticInit() argument
12020 void CGOpenMPSIMDRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF, in emitForOrderedIterationEnd() argument
12027 void CGOpenMPSIMDRuntime::emitForStaticFinish(CodeGenFunction &CGF, in emitForStaticFinish() argument
12033 llvm::Value *CGOpenMPSIMDRuntime::emitForNext(CodeGenFunction &CGF, in emitForNext() argument
12041 void CGOpenMPSIMDRuntime::emitNumThreadsClause(CodeGenFunction &CGF, in emitNumThreadsClause() argument
12047 void CGOpenMPSIMDRuntime::emitProcBindClause(CodeGenFunction &CGF, in emitProcBindClause() argument
12053 Address CGOpenMPSIMDRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, in getAddrOfThreadPrivate() argument
12062 CodeGenFunction *CGF) { in emitThreadPrivateVarDefinition() argument
12067 CodeGenFunction &CGF, QualType VarType, StringRef Name) { in getAddrOfArtificialThreadPrivate() argument
12071 void CGOpenMPSIMDRuntime::emitFlush(CodeGenFunction &CGF, in emitFlush() argument
12078 void CGOpenMPSIMDRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskCall() argument
12088 CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, in emitTaskLoopCall() argument
12095 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates, in emitReduction() argument
12099 CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs, in emitReduction()
12104 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs, in emitTaskReductionInit() argument
12109 void CGOpenMPSIMDRuntime::emitTaskReductionFini(CodeGenFunction &CGF, in emitTaskReductionFini() argument
12115 void CGOpenMPSIMDRuntime::emitTaskReductionFixups(CodeGenFunction &CGF, in emitTaskReductionFixups() argument
12122 Address CGOpenMPSIMDRuntime::getTaskReductionItem(CodeGenFunction &CGF, in getTaskReductionItem() argument
12129 void CGOpenMPSIMDRuntime::emitTaskwaitCall(CodeGenFunction &CGF, in emitTaskwaitCall() argument
12136 CodeGenFunction &CGF, SourceLocation Loc, in emitCancellationPointCall() argument
12141 void CGOpenMPSIMDRuntime::emitCancelCall(CodeGenFunction &CGF, in emitCancelCall() argument
12155 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitTargetCall() argument
12158 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, in emitTargetCall() argument
12176 void CGOpenMPSIMDRuntime::emitTeamsCall(CodeGenFunction &CGF, in emitTeamsCall() argument
12184 void CGOpenMPSIMDRuntime::emitNumTeamsClause(CodeGenFunction &CGF, in emitNumTeamsClause() argument
12192 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, in emitTargetDataCalls() argument
12199 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, in emitTargetDataStandAloneCall() argument
12204 void CGOpenMPSIMDRuntime::emitDoacrossInit(CodeGenFunction &CGF, in emitDoacrossInit() argument
12210 void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, in emitDoacrossOrdered() argument
12215 void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, in emitDoacrossOrdered() argument
12227 CGOpenMPSIMDRuntime::getParameterAddress(CodeGenFunction &CGF, in getParameterAddress() argument