Lines Matching refs:CGF
86 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
90 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
159 void Enter(CodeGenFunction &CGF) override { in Enter() argument
162 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue( in Enter()
163 CGF.GetAddrOfLocalVar(PartIDVar), in Enter()
166 CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation()); in Enter()
167 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done."); in Enter()
168 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB); in Enter()
169 CGF.EmitBlock(DoneBB); in Enter()
170 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock); in Enter()
171 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp.")); in Enter()
172 UntiedSwitch->addCase(CGF.Builder.getInt32(0), in Enter()
173 CGF.Builder.GetInsertBlock()); in Enter()
174 emitUntiedSwitch(CGF); in Enter()
177 void emitUntiedSwitch(CodeGenFunction &CGF) const { in emitUntiedSwitch()
179 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue( in emitUntiedSwitch()
180 CGF.GetAddrOfLocalVar(PartIDVar), in emitUntiedSwitch()
182 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()), in emitUntiedSwitch()
184 UntiedCodeGen(CGF); in emitUntiedSwitch()
186 CGF.getJumpDestInCurrentScope(".untied.next."); in emitUntiedSwitch()
187 CGF.EmitBranch(CGF.ReturnBlock.getBlock()); in emitUntiedSwitch()
188 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp.")); in emitUntiedSwitch()
189 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()), in emitUntiedSwitch()
190 CGF.Builder.GetInsertBlock()); in emitUntiedSwitch()
191 CGF.EmitBranchThroughCleanup(CurPoint); in emitUntiedSwitch()
192 CGF.EmitBlock(CurPoint.getBlock()); in emitUntiedSwitch()
212 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
217 void emitUntiedSwitch(CodeGenFunction &CGF) override { in emitUntiedSwitch() argument
218 Action.emitUntiedSwitch(CGF); in emitUntiedSwitch()
285 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override { in getThreadIDVariableLValue() argument
287 return OuterRegionInfo->getThreadIDVariableLValue(CGF); in getThreadIDVariableLValue()
298 void emitUntiedSwitch(CodeGenFunction &CGF) override { in emitUntiedSwitch() argument
300 OuterRegionInfo->emitUntiedSwitch(CGF); in emitUntiedSwitch()
354 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS) in CGOpenMPInnerExprInfo() argument
355 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen, in CGOpenMPInnerExprInfo()
358 PrivScope(CGF) { in CGOpenMPInnerExprInfo()
370 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD), in CGOpenMPInnerExprInfo()
374 PrivScope.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress(CGF)); in CGOpenMPInnerExprInfo()
387 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override { in EmitBody() argument
411 CodeGenFunction &CGF; member in __anond842882b0111::InlinedOpenMPRegionRAII
422 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen, in InlinedOpenMPRegionRAII() argument
425 : CGF(CGF), NoInheritance(NoInheritance) { in InlinedOpenMPRegionRAII()
427 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo( in InlinedOpenMPRegionRAII()
428 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel); in InlinedOpenMPRegionRAII()
430 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields); in InlinedOpenMPRegionRAII()
431 LambdaThisCaptureField = CGF.LambdaThisCaptureField; in InlinedOpenMPRegionRAII()
432 CGF.LambdaThisCaptureField = nullptr; in InlinedOpenMPRegionRAII()
433 BlockInfo = CGF.BlockInfo; in InlinedOpenMPRegionRAII()
434 CGF.BlockInfo = nullptr; in InlinedOpenMPRegionRAII()
441 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI(); in ~InlinedOpenMPRegionRAII()
442 delete CGF.CapturedStmtInfo; in ~InlinedOpenMPRegionRAII()
443 CGF.CapturedStmtInfo = OldCSI; in ~InlinedOpenMPRegionRAII()
445 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields); in ~InlinedOpenMPRegionRAII()
446 CGF.LambdaThisCaptureField = LambdaThisCaptureField; in ~InlinedOpenMPRegionRAII()
447 CGF.BlockInfo = BlockInfo; in ~InlinedOpenMPRegionRAII()
587 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { in Emit() argument
588 if (!CGF.HaveInsertPoint()) in Emit()
590 Action->Exit(CGF); in Emit()
596 void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const { in operator ()()
597 CodeGenFunction::RunCleanupsScope Scope(CGF); in operator ()()
599 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction); in operator ()()
600 Callback(CodeGen, CGF, *PrePostAction); in operator ()()
603 Callback(CodeGen, CGF, Action); in operator ()()
620 static void emitInitWithReductionInitializer(CodeGenFunction &CGF, in emitInitWithReductionInitializer() argument
627 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD); in emitInitWithReductionInitializer()
636 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitInitWithReductionInitializer()
641 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func); in emitInitWithReductionInitializer()
642 CGF.EmitIgnoredExpr(InitOp); in emitInitWithReductionInitializer()
644 llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty); in emitInitWithReductionInitializer()
645 std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"}); in emitInitWithReductionInitializer()
647 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true, in emitInitWithReductionInitializer()
649 LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty); in emitInitWithReductionInitializer()
651 switch (CGF.getEvaluationKind(Ty)) { in emitInitWithReductionInitializer()
653 InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation()); in emitInitWithReductionInitializer()
657 RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation())); in emitInitWithReductionInitializer()
661 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, LV); in emitInitWithReductionInitializer()
662 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(), in emitInitWithReductionInitializer()
668 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal); in emitInitWithReductionInitializer()
669 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(), in emitInitWithReductionInitializer()
679 static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, in EmitOMPAggregateInit() argument
689 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr); in EmitOMPAggregateInit()
692 CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType()); in EmitOMPAggregateInit()
700 CGF.Builder.CreateGEP(DestAddr.getElementType(), DestBegin, NumElements); in EmitOMPAggregateInit()
702 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body"); in EmitOMPAggregateInit()
703 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done"); in EmitOMPAggregateInit()
705 CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty"); in EmitOMPAggregateInit()
706 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); in EmitOMPAggregateInit()
709 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock(); in EmitOMPAggregateInit()
710 CGF.EmitBlock(BodyBB); in EmitOMPAggregateInit()
712 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy); in EmitOMPAggregateInit()
717 SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2, in EmitOMPAggregateInit()
724 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI( in EmitOMPAggregateInit()
733 CodeGenFunction::RunCleanupsScope InitScope(CGF); in EmitOMPAggregateInit()
735 emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent, in EmitOMPAggregateInit()
738 CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(), in EmitOMPAggregateInit()
744 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateInit()
747 SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateInit()
751 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateInit()
756 CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done"); in EmitOMPAggregateInit()
757 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB); in EmitOMPAggregateInit()
758 DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateInit()
761 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); in EmitOMPAggregateInit()
764 LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) { in emitSharedLValue() argument
765 return CGF.EmitOMPSharedLValue(E); in emitSharedLValue()
768 LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF, in emitSharedLValueUB() argument
771 return CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false); in emitSharedLValueUB()
776 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, in emitAggregateInitialization() argument
785 EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(), in emitAggregateInitialization()
811 void ReductionCodeGen::emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N) { in emitSharedOrigLValue() argument
814 LValue First = emitSharedLValue(CGF, ClausesData[N].Shared); in emitSharedOrigLValue()
815 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Shared); in emitSharedOrigLValue()
820 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref); in emitSharedOrigLValue()
821 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref); in emitSharedOrigLValue()
826 void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) { in emitAggregateType() argument
831 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()), in emitAggregateType()
837 auto *ElemType = OrigAddresses[N].first.getAddress(CGF).getElementType(); in emitAggregateType()
840 Size = CGF.Builder.CreatePtrDiff(ElemType, in emitAggregateType()
841 OrigAddresses[N].second.getPointer(CGF), in emitAggregateType()
842 OrigAddresses[N].first.getPointer(CGF)); in emitAggregateType()
843 Size = CGF.Builder.CreateNUWAdd( in emitAggregateType()
845 SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf); in emitAggregateType()
848 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()); in emitAggregateType()
849 Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf); in emitAggregateType()
853 CGF, in emitAggregateType()
855 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()), in emitAggregateType()
857 CGF.EmitVariablyModifiedType(PrivateType); in emitAggregateType()
860 void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N, in emitAggregateType() argument
870 CGF, in emitAggregateType()
872 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()), in emitAggregateType()
874 CGF.EmitVariablyModifiedType(PrivateType); in emitAggregateType()
878 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, in emitInitialization() argument
885 if (CGF.getContext().getAsArrayType(PrivateVD->getType())) { in emitInitialization()
887 (void)DefaultInit(CGF); in emitInitialization()
888 emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD); in emitInitialization()
890 (void)DefaultInit(CGF); in emitInitialization()
892 emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp, in emitInitialization()
894 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() && in emitInitialization()
895 !CGF.isTrivialInitializer(PrivateVD->getInit())) { in emitInitialization()
896 CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr, in emitInitialization()
908 void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N, in emitCleanups() argument
913 PrivateAddr = CGF.Builder.CreateElementBitCast( in emitCleanups()
914 PrivateAddr, CGF.ConvertTypeForMem(PrivateType)); in emitCleanups()
915 CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType); in emitCleanups()
919 static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, in loadToBegin() argument
923 !CGF.getContext().hasSameType(BaseTy, ElTy)) { in loadToBegin()
925 BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(CGF), PtrTy); in loadToBegin()
927 LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(CGF), BaseTy); in loadToBegin()
928 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal); in loadToBegin()
932 return CGF.MakeAddrLValue( in loadToBegin()
933 CGF.Builder.CreateElementBitCast(BaseLV.getAddress(CGF), in loadToBegin()
934 CGF.ConvertTypeForMem(ElTy)), in loadToBegin()
936 CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType())); in loadToBegin()
939 static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, in castToBase() argument
946 !CGF.getContext().hasSameType(BaseTy, ElTy)) { in castToBase()
947 Tmp = CGF.CreateMemTemp(BaseTy); in castToBase()
949 CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp); in castToBase()
957 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in castToBase()
959 CGF.Builder.CreateStore(Addr, Tmp); in castToBase()
963 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in castToBase()
988 Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, in adjustPrivateAddress() argument
993 LValue OriginalBaseLValue = CGF.EmitLValue(DE); in adjustPrivateAddress()
995 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(), in adjustPrivateAddress()
997 Address SharedAddr = SharedAddresses[N].first.getAddress(CGF); in adjustPrivateAddress()
998 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff( in adjustPrivateAddress()
999 SharedAddr.getElementType(), BaseLValue.getPointer(CGF), in adjustPrivateAddress()
1002 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in adjustPrivateAddress()
1004 llvm::Value *Ptr = CGF.Builder.CreateGEP( in adjustPrivateAddress()
1006 return castToBase(CGF, OrigVD->getType(), in adjustPrivateAddress()
1008 OriginalBaseLValue.getAddress(CGF), Ptr); in adjustPrivateAddress()
1021 LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) { in getThreadIDVariableLValue() argument
1022 return CGF.EmitLoadOfPointerLValue( in getThreadIDVariableLValue()
1023 CGF.GetAddrOfLocalVar(getThreadIDVariable()), in getThreadIDVariableLValue()
1027 void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt *S) { in EmitBody() argument
1028 if (!CGF.HaveInsertPoint()) in EmitBody()
1035 CGF.EHStack.pushTerminate(); in EmitBody()
1037 CGF.incrementProfileCounter(S); in EmitBody()
1038 CodeGen(CGF); in EmitBody()
1039 CGF.EHStack.popTerminate(); in EmitBody()
1043 CodeGenFunction &CGF) { in getThreadIDVariableLValue() argument
1044 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()), in getThreadIDVariableLValue()
1124 CodeGenFunction CGF(CGM); in emitCombinerOrInitializer() local
1127 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(), in emitCombinerOrInitializer()
1129 CodeGenFunction::OMPPrivateScope Scope(CGF); in emitCombinerOrInitializer()
1130 Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm); in emitCombinerOrInitializer()
1132 In, CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>()) in emitCombinerOrInitializer()
1133 .getAddress(CGF)); in emitCombinerOrInitializer()
1134 Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm); in emitCombinerOrInitializer()
1136 Out, CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>()) in emitCombinerOrInitializer()
1137 .getAddress(CGF)); in emitCombinerOrInitializer()
1140 !CGF.isTrivialInitializer(Out->getInit())) { in emitCombinerOrInitializer()
1141 CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out), in emitCombinerOrInitializer()
1146 CGF.EmitIgnoredExpr(CombinerInitializer); in emitCombinerOrInitializer()
1148 CGF.FinishFunction(); in emitCombinerOrInitializer()
1153 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) { in emitUserDefinedReduction() argument
1172 if (CGF) { in emitUserDefinedReduction()
1173 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn); in emitUserDefinedReduction()
1191 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder, CodeGenFunction &CGF, in PushAndPopStackRAII()
1209 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) { in PushAndPopStackRAII()
1212 CGBuilderTy::InsertPointGuard IPG(CGF.Builder); in PushAndPopStackRAII()
1213 CGF.Builder.restoreIP(IP); in PushAndPopStackRAII()
1215 CGF.getOMPCancelDestination(OMPD_parallel); in PushAndPopStackRAII()
1216 CGF.EmitBranchThroughCleanup(Dest); in PushAndPopStackRAII()
1238 CodeGenFunction CGF(CGM, true); in emitParallelOrTeamsOutlinedFunction() local
1262 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind); in emitParallelOrTeamsOutlinedFunction()
1265 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitParallelOrTeamsOutlinedFunction()
1266 return CGF.GenerateOpenMPCapturedStmtFunction(*CS, D.getBeginLoc()); in emitParallelOrTeamsOutlinedFunction()
1290 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF, in emitTaskOutlinedFunction()
1292 llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc()); in emitTaskOutlinedFunction()
1293 llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTaskOutlinedFunction()
1296 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar), in emitTaskOutlinedFunction()
1298 .getPointer(CGF)}; in emitTaskOutlinedFunction()
1299 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskOutlinedFunction()
1322 CodeGenFunction CGF(CGM, true); in emitTaskOutlinedFunction() local
1325 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitTaskOutlinedFunction()
1326 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS); in emitTaskOutlinedFunction()
1332 void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF, in setLocThreadIdInsertPt() argument
1334 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in setLocThreadIdInsertPt()
1337 llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty); in setLocThreadIdInsertPt()
1340 Undef, CGF.Int32Ty, "svcpt", CGF.Builder.GetInsertBlock()); in setLocThreadIdInsertPt()
1343 new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt"); in setLocThreadIdInsertPt()
1344 Elem.second.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt); in setLocThreadIdInsertPt()
1348 void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) { in clearLocThreadIdInsertPt() argument
1349 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in clearLocThreadIdInsertPt()
1357 static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF, in getIdentStringFromSourceLocation() argument
1362 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc); in getIdentStringFromSourceLocation()
1364 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl)) in getIdentStringFromSourceLocation()
1370 llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF, in emitUpdateLocation() argument
1380 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl)) in emitUpdateLocation()
1382 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc); in emitUpdateLocation()
1394 llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF, in getThreadID() argument
1396 assert(CGF.CurFn && "No function in current CodeGenFunction."); in getThreadID()
1401 OMPBuilder.updateToLocation(CGF.Builder.saveIP()); in getThreadID()
1404 getIdentStringFromSourceLocation(CGF, Loc, Buffer), SrcLocStrSize); in getThreadID()
1412 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn); in getThreadID()
1420 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { in getThreadID()
1423 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF); in getThreadID()
1424 llvm::BasicBlock *TopBlock = CGF.AllocaInsertPt->getParent(); in getThreadID()
1425 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions || in getThreadID()
1426 !CGF.getLangOpts().CXXExceptions || in getThreadID()
1427 CGF.Builder.GetInsertBlock() == TopBlock || in getThreadID()
1428 !isa<llvm::Instruction>(LVal.getPointer(CGF)) || in getThreadID()
1429 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() == in getThreadID()
1431 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() == in getThreadID()
1432 CGF.Builder.GetInsertBlock()) { in getThreadID()
1433 ThreadID = CGF.EmitLoadOfScalar(LVal, Loc); in getThreadID()
1436 if (CGF.Builder.GetInsertBlock() == TopBlock) { in getThreadID()
1437 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in getThreadID()
1449 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in getThreadID()
1451 setLocThreadIdInsertPt(CGF); in getThreadID()
1452 CGBuilderTy::InsertPointGuard IPG(CGF.Builder); in getThreadID()
1453 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt); in getThreadID()
1454 llvm::CallInst *Call = CGF.Builder.CreateCall( in getThreadID()
1457 emitUpdateLocation(CGF, Loc)); in getThreadID()
1458 Call->setCallingConv(CGF.getRuntimeCC()); in getThreadID()
1463 void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) { in functionFinished() argument
1464 assert(CGF.CurFn && "No function in current CodeGenFunction."); in functionFinished()
1465 if (OpenMPLocThreadIDMap.count(CGF.CurFn)) { in functionFinished()
1466 clearLocThreadIdInsertPt(CGF); in functionFinished()
1467 OpenMPLocThreadIDMap.erase(CGF.CurFn); in functionFinished()
1469 if (FunctionUDRMap.count(CGF.CurFn) > 0) { in functionFinished()
1470 for(const auto *D : FunctionUDRMap[CGF.CurFn]) in functionFinished()
1472 FunctionUDRMap.erase(CGF.CurFn); in functionFinished()
1474 auto I = FunctionUDMMap.find(CGF.CurFn); in functionFinished()
1480 LastprivateConditionalToTypes.erase(CGF.CurFn); in functionFinished()
1481 FunctionToUntiedTaskStackMap.erase(CGF.CurFn); in functionFinished()
1674 Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, in getAddrOfThreadPrivate() argument
1684 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in getAddrOfThreadPrivate()
1685 CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy), in getAddrOfThreadPrivate()
1689 CGF.EmitRuntimeCall( in getAddrOfThreadPrivate()
1693 CGF.Int8Ty, VDAddr.getAlignment()); in getAddrOfThreadPrivate()
1697 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor, in emitThreadPrivateVarInit() argument
1701 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc); in emitThreadPrivateVarInit()
1702 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitThreadPrivateVarInit()
1708 OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy), in emitThreadPrivateVarInit()
1710 CGF.EmitRuntimeCall( in emitThreadPrivateVarInit()
1718 bool PerformInit, CodeGenFunction *CGF) { in emitThreadPrivateVarDefinition() argument
1817 if (!CGF) { in emitThreadPrivateVarDefinition()
1832 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc); in emitThreadPrivateVarDefinition()
1968 Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, in getAddrOfArtificialThreadPrivate() argument
1972 llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType); in getAddrOfArtificialThreadPrivate()
1983 emitUpdateLocation(CGF, SourceLocation()), in getAddrOfArtificialThreadPrivate()
1984 getThreadID(CGF, SourceLocation()), in getAddrOfArtificialThreadPrivate()
1985 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy), in getAddrOfArtificialThreadPrivate()
1986 CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy, in getAddrOfArtificialThreadPrivate()
1991 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddrOfArtificialThreadPrivate()
1992 CGF.EmitRuntimeCall( in getAddrOfArtificialThreadPrivate()
2000 void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond, in emitIfClause() argument
2003 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange()); in emitIfClause()
2008 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) { in emitIfClause()
2010 ThenGen(CGF); in emitIfClause()
2012 ElseGen(CGF); in emitIfClause()
2018 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then"); in emitIfClause()
2019 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else"); in emitIfClause()
2020 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end"); in emitIfClause()
2021 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0); in emitIfClause()
2024 CGF.EmitBlock(ThenBlock); in emitIfClause()
2025 ThenGen(CGF); in emitIfClause()
2026 CGF.EmitBranch(ContBlock); in emitIfClause()
2029 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitIfClause()
2030 CGF.EmitBlock(ElseBlock); in emitIfClause()
2031 ElseGen(CGF); in emitIfClause()
2033 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitIfClause()
2034 CGF.EmitBranch(ContBlock); in emitIfClause()
2036 CGF.EmitBlock(ContBlock, /*IsFinished=*/true); in emitIfClause()
2039 void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, in emitParallelCall() argument
2044 if (!CGF.HaveInsertPoint()) in emitParallelCall()
2046 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitParallelCall()
2049 this](CodeGenFunction &CGF, PrePostActionTy &) { in emitParallelCall() argument
2051 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitParallelCall()
2054 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars in emitParallelCall()
2055 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())}; in emitParallelCall()
2062 CGF.EmitRuntimeCall(RTLFn, RealArgs); in emitParallelCall()
2065 this](CodeGenFunction &CGF, PrePostActionTy &) { in emitParallelCall() argument
2066 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitParallelCall()
2067 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc); in emitParallelCall()
2071 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitParallelCall()
2076 Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc); in emitParallelCall()
2078 CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, in emitParallelCall()
2080 CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddrBound); in emitParallelCall()
2095 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs); in emitParallelCall()
2098 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID}; in emitParallelCall()
2099 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitParallelCall()
2104 emitIfClause(CGF, IfCond, ThenGen, ElseGen); in emitParallelCall()
2107 ThenRCG(CGF); in emitParallelCall()
2117 Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF, in emitThreadIDAddress() argument
2120 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitThreadIDAddress()
2122 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(CGF); in emitThreadIDAddress()
2124 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitThreadIDAddress()
2126 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true); in emitThreadIDAddress()
2127 Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp."); in emitThreadIDAddress()
2128 CGF.EmitStoreOfScalar(ThreadID, in emitThreadIDAddress()
2129 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty)); in emitThreadIDAddress()
2177 void Enter(CodeGenFunction &CGF) override { in Enter() argument
2178 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs); in Enter()
2180 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes); in Enter()
2181 auto *ThenBlock = CGF.createBasicBlock("omp_if.then"); in Enter()
2182 ContBlock = CGF.createBasicBlock("omp_if.end"); in Enter()
2184 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock); in Enter()
2185 CGF.EmitBlock(ThenBlock); in Enter()
2188 void Done(CodeGenFunction &CGF) { in Done() argument
2190 CGF.EmitBranch(ContBlock); in Done()
2191 CGF.EmitBlock(ContBlock, true); in Done()
2193 void Exit(CodeGenFunction &CGF) override { in Exit() argument
2194 CGF.EmitRuntimeCall(ExitCallee, ExitArgs); in Exit()
2199 void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF, in emitCriticalRegion() argument
2207 if (!CGF.HaveInsertPoint()) in emitCriticalRegion()
2209 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitCriticalRegion()
2214 EnterArgs.push_back(CGF.Builder.CreateIntCast( in emitCriticalRegion()
2215 CGF.EmitScalarExpr(Hint), CGM.Int32Ty, /*isSigned=*/false)); in emitCriticalRegion()
2226 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen); in emitCriticalRegion()
2229 void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF, in emitMasterRegion() argument
2232 if (!CGF.HaveInsertPoint()) in emitMasterRegion()
2239 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitMasterRegion()
2248 emitInlinedDirective(CGF, OMPD_master, MasterOpGen); in emitMasterRegion()
2249 Action.Done(CGF); in emitMasterRegion()
2252 void CGOpenMPRuntime::emitMaskedRegion(CodeGenFunction &CGF, in emitMaskedRegion() argument
2255 if (!CGF.HaveInsertPoint()) in emitMaskedRegion()
2263 ? CGF.EmitScalarExpr(Filter, CGF.Int32Ty) in emitMaskedRegion()
2265 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitMaskedRegion()
2267 llvm::Value *ArgsEnd[] = {emitUpdateLocation(CGF, Loc), in emitMaskedRegion()
2268 getThreadID(CGF, Loc)}; in emitMaskedRegion()
2277 emitInlinedDirective(CGF, OMPD_masked, MaskedOpGen); in emitMaskedRegion()
2278 Action.Done(CGF); in emitMaskedRegion()
2281 void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF, in emitTaskyieldCall() argument
2283 if (!CGF.HaveInsertPoint()) in emitTaskyieldCall()
2285 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) { in emitTaskyieldCall()
2286 OMPBuilder.createTaskyield(CGF.Builder); in emitTaskyieldCall()
2290 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitTaskyieldCall()
2292 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskyieldCall()
2297 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitTaskyieldCall()
2298 Region->emitUntiedSwitch(CGF); in emitTaskyieldCall()
2301 void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF, in emitTaskgroupRegion() argument
2304 if (!CGF.HaveInsertPoint()) in emitTaskgroupRegion()
2310 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitTaskgroupRegion()
2318 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen); in emitTaskgroupRegion()
2323 static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array, in emitAddrOfVarFromArray() argument
2326 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index); in emitAddrOfVarFromArray()
2327 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr); in emitAddrOfVarFromArray()
2329 llvm::Type *ElemTy = CGF.ConvertTypeForMem(Var->getType()); in emitAddrOfVarFromArray()
2331 CGF.Builder.CreateBitCast( in emitAddrOfVarFromArray()
2333 ElemTy, CGF.getContext().getDeclAlign(Var)); in emitAddrOfVarFromArray()
2359 CodeGenFunction CGF(CGM); in emitCopyprivateCopyFunction() local
2360 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitCopyprivateCopyFunction()
2363 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitCopyprivateCopyFunction()
2364 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)), in emitCopyprivateCopyFunction()
2366 ArgsElemType, CGF.getPointerAlign()); in emitCopyprivateCopyFunction()
2367 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitCopyprivateCopyFunction()
2368 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)), in emitCopyprivateCopyFunction()
2370 ArgsElemType, CGF.getPointerAlign()); in emitCopyprivateCopyFunction()
2378 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar); in emitCopyprivateCopyFunction()
2382 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar); in emitCopyprivateCopyFunction()
2386 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]); in emitCopyprivateCopyFunction()
2388 CGF.FinishFunction(); in emitCopyprivateCopyFunction()
2392 void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF, in emitSingleRegion() argument
2399 if (!CGF.HaveInsertPoint()) in emitSingleRegion()
2419 DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it"); in emitSingleRegion()
2420 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt); in emitSingleRegion()
2423 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitSingleRegion()
2432 emitInlinedDirective(CGF, OMPD_single, SingleOpGen); in emitSingleRegion()
2435 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt); in emitSingleRegion()
2437 Action.Done(CGF); in emitSingleRegion()
2447 CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list"); in emitSingleRegion()
2449 Address Elem = CGF.Builder.CreateConstArrayGEP(CopyprivateList, I); in emitSingleRegion()
2450 CGF.Builder.CreateStore( in emitSingleRegion()
2451 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitSingleRegion()
2452 CGF.EmitLValue(CopyprivateVars[I]).getPointer(CGF), in emitSingleRegion()
2453 CGF.VoidPtrTy), in emitSingleRegion()
2459 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy), CopyprivateVars, in emitSingleRegion()
2461 llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy); in emitSingleRegion()
2462 Address CL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitSingleRegion()
2463 CopyprivateList, CGF.VoidPtrTy, CGF.Int8Ty); in emitSingleRegion()
2464 llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt); in emitSingleRegion()
2466 emitUpdateLocation(CGF, Loc), // ident_t *<loc> in emitSingleRegion()
2467 getThreadID(CGF, Loc), // i32 <gtid> in emitSingleRegion()
2473 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitSingleRegion()
2479 void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF, in emitOrderedRegion() argument
2482 if (!CGF.HaveInsertPoint()) in emitOrderedRegion()
2489 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitOrderedRegion()
2497 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen); in emitOrderedRegion()
2500 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen); in emitOrderedRegion()
2519 CodeGenFunction &CGF, const OMPLoopDirective &S, in getDefaultScheduleAndChunk() argument
2530 CGF.getContext(), ChunkSize, in getDefaultScheduleAndChunk()
2531 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0), in getDefaultScheduleAndChunk()
2536 void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, in emitBarrierCall() argument
2541 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo); in emitBarrierCall()
2542 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) { in emitBarrierCall()
2543 CGF.Builder.restoreIP(OMPBuilder.createBarrier( in emitBarrierCall()
2544 CGF.Builder, Kind, ForceSimpleCall, EmitChecks)); in emitBarrierCall()
2548 if (!CGF.HaveInsertPoint()) in emitBarrierCall()
2555 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags), in emitBarrierCall()
2556 getThreadID(CGF, Loc)}; in emitBarrierCall()
2559 llvm::Value *Result = CGF.EmitRuntimeCall( in emitBarrierCall()
2567 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit"); in emitBarrierCall()
2568 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue"); in emitBarrierCall()
2569 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result); in emitBarrierCall()
2570 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); in emitBarrierCall()
2571 CGF.EmitBlock(ExitBB); in emitBarrierCall()
2574 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); in emitBarrierCall()
2575 CGF.EmitBranchThroughCleanup(CancelDestination); in emitBarrierCall()
2576 CGF.EmitBlock(ContBB, /*IsFinished=*/true); in emitBarrierCall()
2581 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitBarrierCall()
2700 CodeGenFunction &CGF, SourceLocation Loc, in emitForDispatchInit() argument
2703 if (!CGF.HaveInsertPoint()) in emitForDispatchInit()
2718 : CGF.Builder.getIntN(IVSize, 1); in emitForDispatchInit()
2720 emitUpdateLocation(CGF, Loc), in emitForDispatchInit()
2721 getThreadID(CGF, Loc), in emitForDispatchInit()
2722 CGF.Builder.getInt32(addMonoNonMonoModifier( in emitForDispatchInit()
2726 CGF.Builder.getIntN(IVSize, 1), // Stride in emitForDispatchInit()
2729 CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args); in emitForDispatchInit()
2733 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId, in emitForStaticInitCall() argument
2737 if (!CGF.HaveInsertPoint()) in emitForStaticInitCall()
2758 Chunk = CGF.Builder.getIntN(Values.IVSize, 1); in emitForStaticInitCall()
2769 CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1, in emitForStaticInitCall()
2775 CGF.Builder.getIntN(Values.IVSize, 1), // Incr in emitForStaticInitCall()
2778 CGF.EmitRuntimeCall(ForStaticInitFunction, Args); in emitForStaticInitCall()
2781 void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF, in emitForStaticInit() argument
2790 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc, in emitForStaticInit()
2794 llvm::Value *ThreadId = getThreadID(CGF, Loc); in emitForStaticInit()
2797 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc); in emitForStaticInit()
2798 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction, in emitForStaticInit()
2803 CodeGenFunction &CGF, SourceLocation Loc, in emitDistributeStaticInit() argument
2809 emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE); in emitDistributeStaticInit()
2810 llvm::Value *ThreadId = getThreadID(CGF, Loc); in emitDistributeStaticInit()
2818 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction, in emitDistributeStaticInit()
2823 void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF, in emitForStaticFinish() argument
2826 if (!CGF.HaveInsertPoint()) in emitForStaticFinish()
2830 emitUpdateLocation(CGF, Loc, in emitForStaticFinish()
2836 getThreadID(CGF, Loc)}; in emitForStaticFinish()
2837 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc); in emitForStaticFinish()
2840 CGF.EmitRuntimeCall( in emitForStaticFinish()
2845 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitForStaticFinish()
2850 void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF, in emitForOrderedIterationEnd() argument
2854 if (!CGF.HaveInsertPoint()) in emitForOrderedIterationEnd()
2857 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitForOrderedIterationEnd()
2858 CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args); in emitForOrderedIterationEnd()
2861 llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF, in emitForNext() argument
2871 emitUpdateLocation(CGF, Loc), in emitForNext()
2872 getThreadID(CGF, Loc), in emitForNext()
2879 CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args); in emitForNext()
2880 return CGF.EmitScalarConversion( in emitForNext()
2881 Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1), in emitForNext()
2882 CGF.getContext().BoolTy, Loc); in emitForNext()
2885 void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF, in emitNumThreadsClause() argument
2888 if (!CGF.HaveInsertPoint()) in emitNumThreadsClause()
2892 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitNumThreadsClause()
2893 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)}; in emitNumThreadsClause()
2894 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitNumThreadsClause()
2899 void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF, in emitProcBindClause() argument
2902 if (!CGF.HaveInsertPoint()) in emitProcBindClause()
2907 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitProcBindClause()
2909 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitProcBindClause()
2914 void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>, in emitFlush() argument
2916 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) { in emitFlush()
2917 OMPBuilder.createFlush(CGF.Builder); in emitFlush()
2919 if (!CGF.HaveInsertPoint()) in emitFlush()
2922 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitFlush()
2924 emitUpdateLocation(CGF, Loc)); in emitFlush()
3520 CodeGenFunction CGF(CGM); in emitProxyTaskFunction() local
3521 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args, in emitProxyTaskFunction()
3529 llvm::Value *GtidParam = CGF.EmitLoadOfScalar( in emitProxyTaskFunction()
3530 CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc); in emitProxyTaskFunction()
3531 LValue TDBase = CGF.EmitLoadOfPointerLValue( in emitProxyTaskFunction()
3532 CGF.GetAddrOfLocalVar(&TaskTypeArg), in emitProxyTaskFunction()
3537 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin()); in emitProxyTaskFunction()
3540 LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI); in emitProxyTaskFunction()
3541 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF); in emitProxyTaskFunction()
3544 LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI); in emitProxyTaskFunction()
3545 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitProxyTaskFunction()
3546 CGF.EmitLoadOfScalar(SharedsLVal, Loc), in emitProxyTaskFunction()
3547 CGF.ConvertTypeForMem(SharedsPtrTy)); in emitProxyTaskFunction()
3552 LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI); in emitProxyTaskFunction()
3553 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitProxyTaskFunction()
3554 PrivatesLVal.getPointer(CGF), CGF.VoidPtrTy); in emitProxyTaskFunction()
3556 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitProxyTaskFunction()
3561 CGF.Builder in emitProxyTaskFunction()
3562 .CreatePointerBitCastOrAddrSpaceCast(TDBase.getAddress(CGF), in emitProxyTaskFunction()
3563 CGF.VoidPtrTy, CGF.Int8Ty) in emitProxyTaskFunction()
3569 LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI); in emitProxyTaskFunction()
3570 llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc); in emitProxyTaskFunction()
3572 LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI); in emitProxyTaskFunction()
3573 llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc); in emitProxyTaskFunction()
3575 LValue StLVal = CGF.EmitLValueForField(Base, *StFI); in emitProxyTaskFunction()
3576 llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc); in emitProxyTaskFunction()
3578 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI); in emitProxyTaskFunction()
3579 llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc); in emitProxyTaskFunction()
3581 LValue RLVal = CGF.EmitLValueForField(Base, *RFI); in emitProxyTaskFunction()
3582 llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc); in emitProxyTaskFunction()
3591 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction, in emitProxyTaskFunction()
3593 CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)), in emitProxyTaskFunction()
3594 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty)); in emitProxyTaskFunction()
3595 CGF.FinishFunction(); in emitProxyTaskFunction()
3625 CodeGenFunction CGF(CGM); in emitDestructorsFunction() local
3626 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo, in emitDestructorsFunction()
3629 LValue Base = CGF.EmitLoadOfPointerLValue( in emitDestructorsFunction()
3630 CGF.GetAddrOfLocalVar(&TaskTypeArg), in emitDestructorsFunction()
3635 Base = CGF.EmitLValueForField(Base, *FI); in emitDestructorsFunction()
3640 LValue FieldLValue = CGF.EmitLValueForField(Base, Field); in emitDestructorsFunction()
3641 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(CGF), Field->getType()); in emitDestructorsFunction()
3644 CGF.FinishFunction(); in emitDestructorsFunction()
3733 CodeGenFunction CGF(CGM); in emitTaskPrivateMappingFunction() local
3734 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap, in emitTaskPrivateMappingFunction()
3738 LValue Base = CGF.EmitLoadOfPointerLValue( in emitTaskPrivateMappingFunction()
3739 CGF.GetAddrOfLocalVar(&TaskPrivatesArg), in emitTaskPrivateMappingFunction()
3744 LValue FieldLVal = CGF.EmitLValueForField(Base, Field); in emitTaskPrivateMappingFunction()
3747 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType()); in emitTaskPrivateMappingFunction()
3748 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue( in emitTaskPrivateMappingFunction()
3749 RefLVal.getAddress(CGF), RefLVal.getType()->castAs<PointerType>()); in emitTaskPrivateMappingFunction()
3750 CGF.EmitStoreOfScalar(FieldLVal.getPointer(CGF), RefLoadLVal); in emitTaskPrivateMappingFunction()
3753 CGF.FinishFunction(); in emitTaskPrivateMappingFunction()
3758 static void emitPrivatesInit(CodeGenFunction &CGF, in emitPrivatesInit() argument
3765 ASTContext &C = CGF.getContext(); in emitPrivatesInit()
3767 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI); in emitPrivatesInit()
3782 SrcBase = CGF.MakeAddrLValue( in emitPrivatesInit()
3783 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitPrivatesInit()
3784 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy), in emitPrivatesInit()
3785 CGF.ConvertTypeForMem(SharedsTy)), in emitPrivatesInit()
3798 !CGF.isTrivialInitializer(Init)))) { in emitPrivatesInit()
3799 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI); in emitPrivatesInit()
3817 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type); in emitPrivatesInit()
3819 SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField); in emitPrivatesInit()
3820 SharedRefLValue = CGF.MakeAddrLValue( in emitPrivatesInit()
3821 SharedRefLValue.getAddress(CGF).withAlignment( in emitPrivatesInit()
3825 } else if (CGF.LambdaCaptureFields.count( in emitPrivatesInit()
3827 isa_and_nonnull<BlockDecl>(CGF.CurCodeDecl)) { in emitPrivatesInit()
3828 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef); in emitPrivatesInit()
3832 CGF, [](CodeGenFunction &, PrePostActionTy &) {}, OMPD_unknown, in emitPrivatesInit()
3834 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef); in emitPrivatesInit()
3838 if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) { in emitPrivatesInit()
3840 CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type); in emitPrivatesInit()
3844 CGF.EmitOMPAggregateAssign( in emitPrivatesInit()
3845 PrivateLValue.getAddress(CGF), SharedRefLValue.getAddress(CGF), in emitPrivatesInit()
3847 [&CGF, Elem, Init, &CapturesInfo](Address DestElement, in emitPrivatesInit()
3850 CodeGenFunction::OMPPrivateScope InitScope(CGF); in emitPrivatesInit()
3855 CGF, &CapturesInfo); in emitPrivatesInit()
3856 CGF.EmitAnyExprToMem(Init, DestElement, in emitPrivatesInit()
3862 CodeGenFunction::OMPPrivateScope InitScope(CGF); in emitPrivatesInit()
3863 InitScope.addPrivate(Elem, SharedRefLValue.getAddress(CGF)); in emitPrivatesInit()
3865 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo); in emitPrivatesInit()
3866 CGF.EmitExprAsInit(Init, VD, PrivateLValue, in emitPrivatesInit()
3870 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false); in emitPrivatesInit()
3878 static bool checkInitIsRequired(CodeGenFunction &CGF, in checkInitIsRequired() argument
3887 !CGF.isTrivialInitializer(Init)); in checkInitIsRequired()
3934 CodeGenFunction CGF(CGM); in emitTaskDupFunction() local
3935 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc, in emitTaskDupFunction()
3938 LValue TDBase = CGF.EmitLoadOfPointerLValue( in emitTaskDupFunction()
3939 CGF.GetAddrOfLocalVar(&DstArg), in emitTaskDupFunction()
3944 LValue Base = CGF.EmitLValueForField( in emitTaskDupFunction()
3946 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI); in emitTaskDupFunction()
3947 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar( in emitTaskDupFunction()
3948 CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc); in emitTaskDupFunction()
3949 CGF.EmitStoreOfScalar(Lastpriv, LILVal); in emitTaskDupFunction()
3956 LValue TDBase = CGF.EmitLoadOfPointerLValue( in emitTaskDupFunction()
3957 CGF.GetAddrOfLocalVar(&SrcArg), in emitTaskDupFunction()
3959 LValue Base = CGF.EmitLValueForField( in emitTaskDupFunction()
3962 CGF.EmitLoadOfScalar(CGF.EmitLValueForField( in emitTaskDupFunction()
3966 CGF.Int8Ty, CGM.getNaturalTypeAlignment(SharedsTy)); in emitTaskDupFunction()
3968 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD, in emitTaskDupFunction()
3970 CGF.FinishFunction(); in emitTaskDupFunction()
3993 CodeGenFunction &CGF; member in __anond842882b1411::OMPIteratorGeneratorScope
4001 OMPIteratorGeneratorScope(CodeGenFunction &CGF, const OMPIteratorExpr *E) in OMPIteratorGeneratorScope() argument
4002 : CodeGenFunction::OMPPrivateScope(CGF), CGF(CGF), E(E) { in OMPIteratorGeneratorScope()
4007 Uppers.push_back(CGF.EmitScalarExpr(E->getHelper(I).Upper)); in OMPIteratorGeneratorScope()
4009 addPrivate(VD, CGF.CreateMemTemp(VD->getType(), VD->getName())); in OMPIteratorGeneratorScope()
4013 CGF.CreateMemTemp(HelperData.CounterVD->getType(), "counter.addr")); in OMPIteratorGeneratorScope()
4020 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(HelperData.CounterVD), in OMPIteratorGeneratorScope()
4023 CGF.EmitStoreOfScalar( in OMPIteratorGeneratorScope()
4024 llvm::ConstantInt::get(CLVal.getAddress(CGF).getElementType(), 0), in OMPIteratorGeneratorScope()
4027 ContDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.cont")); in OMPIteratorGeneratorScope()
4029 ExitDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.exit")); in OMPIteratorGeneratorScope()
4034 CGF.EmitBlock(ContDest.getBlock()); in OMPIteratorGeneratorScope()
4036 CGF.EmitLoadOfScalar(CLVal, HelperData.CounterVD->getLocation()); in OMPIteratorGeneratorScope()
4039 ? CGF.Builder.CreateICmpSLT(CVal, N) in OMPIteratorGeneratorScope()
4040 : CGF.Builder.CreateICmpULT(CVal, N); in OMPIteratorGeneratorScope()
4041 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("iter.body"); in OMPIteratorGeneratorScope()
4042 CGF.Builder.CreateCondBr(Cmp, BodyBB, ExitDest.getBlock()); in OMPIteratorGeneratorScope()
4044 CGF.EmitBlock(BodyBB); in OMPIteratorGeneratorScope()
4046 CGF.EmitIgnoredExpr(HelperData.Update); in OMPIteratorGeneratorScope()
4055 CGF.EmitIgnoredExpr(HelperData.CounterUpdate); in ~OMPIteratorGeneratorScope()
4057 CGF.EmitBranchThroughCleanup(ContDests[I - 1]); in ~OMPIteratorGeneratorScope()
4059 CGF.EmitBlock(ExitDests[I - 1].getBlock(), /*IsFinished=*/I == 1); in ~OMPIteratorGeneratorScope()
4066 getPointerAndSize(CodeGenFunction &CGF, const Expr *E) { in getPointerAndSize() argument
4071 Addr = CGF.EmitScalarExpr(Base); in getPointerAndSize()
4073 Addr = CGF.EmitLValue(E).getPointer(CGF); in getPointerAndSize()
4078 SizeVal = CGF.getTypeSize(OASE->getBase()->getType()->getPointeeType()); in getPointerAndSize()
4080 llvm::Value *Sz = CGF.EmitScalarExpr(SE); in getPointerAndSize()
4081 Sz = CGF.EmitScalarConversion( in getPointerAndSize()
4082 Sz, SE->getType(), CGF.getContext().getSizeType(), SE->getExprLoc()); in getPointerAndSize()
4083 SizeVal = CGF.Builder.CreateNUWMul(SizeVal, Sz); in getPointerAndSize()
4088 CGF.EmitOMPArraySectionExpr(ASE, /*IsLowerBound=*/false); in getPointerAndSize()
4089 Address UpAddrAddress = UpAddrLVal.getAddress(CGF); in getPointerAndSize()
4090 llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32( in getPointerAndSize()
4092 llvm::Value *LowIntPtr = CGF.Builder.CreatePtrToInt(Addr, CGF.SizeTy); in getPointerAndSize()
4093 llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGF.SizeTy); in getPointerAndSize()
4094 SizeVal = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr); in getPointerAndSize()
4096 SizeVal = CGF.getTypeSize(Ty); in getPointerAndSize()
4117 CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskInit() argument
4192 llvm::Type *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy); in emitTaskInit()
4196 CGF.getTypeSize(KmpTaskTWithPrivatesQTy); in emitTaskInit()
4207 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
4247 ? CGF.Builder.CreateSelect(Data.Final.getPointer(), in emitTaskInit()
4248 CGF.Builder.getInt32(FinalFlag), in emitTaskInit()
4249 CGF.Builder.getInt32(/*C=*/0)) in emitTaskInit()
4250 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0); in emitTaskInit()
4251 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags)); in emitTaskInit()
4253 SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc), in emitTaskInit()
4254 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize, in emitTaskInit()
4255 SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
4266 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTaskInit()
4267 CGF.Int64Ty, /*isSigned=*/true); in emitTaskInit()
4269 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTaskInit()
4271 NewTask = CGF.EmitRuntimeCall( in emitTaskInit()
4277 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskInit()
4286 LValue EvtLVal = CGF.EmitLValue(Evt); in emitTaskInit()
4290 llvm::Value *Loc = emitUpdateLocation(CGF, DC->getBeginLoc()); in emitTaskInit()
4291 llvm::Value *Tid = getThreadID(CGF, DC->getBeginLoc()); in emitTaskInit()
4292 Tid = CGF.Builder.CreateIntCast(Tid, CGF.IntTy, /*isSigned=*/false); in emitTaskInit()
4293 llvm::Value *EvtVal = CGF.EmitRuntimeCall( in emitTaskInit()
4297 EvtVal = CGF.EmitScalarConversion(EvtVal, C.VoidPtrTy, Evt->getType(), in emitTaskInit()
4299 CGF.EmitStoreOfScalar(EvtVal, EvtLVal); in emitTaskInit()
4313 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper); in emitTaskInit()
4314 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false); in emitTaskInit()
4316 NumOfElements ? CGF.Builder.CreateNUWMul(NumOfElements, Sz) : Sz; in emitTaskInit()
4328 NumOfElements = CGF.Builder.CreateNUWAdd( in emitTaskInit()
4329 llvm::ConstantInt::get(CGF.SizeTy, NumAffinities), NumOfElements); in emitTaskInit()
4334 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE, in emitTaskInit()
4342 CGF.EmitVarDecl(*PD); in emitTaskInit()
4343 AffinitiesArray = CGF.GetAddrOfLocalVar(PD); in emitTaskInit()
4344 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty, in emitTaskInit()
4352 CGF.CreateMemTemp(KmpTaskAffinityInfoArrayTy, ".affs.arr.addr"); in emitTaskInit()
4353 AffinitiesArray = CGF.Builder.CreateConstArrayGEP(AffinitiesArray, 0); in emitTaskInit()
4370 std::tie(Addr, Size) = getPointerAndSize(CGF, E); in emitTaskInit()
4372 CGF.MakeAddrLValue(CGF.Builder.CreateConstGEP(AffinitiesArray, Pos), in emitTaskInit()
4375 LValue BaseAddrLVal = CGF.EmitLValueForField( in emitTaskInit()
4377 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy), in emitTaskInit()
4380 LValue LenLVal = CGF.EmitLValueForField( in emitTaskInit()
4382 CGF.EmitStoreOfScalar(Size, LenLVal); in emitTaskInit()
4388 PosLVal = CGF.MakeAddrLValue( in emitTaskInit()
4389 CGF.CreateMemTemp(C.getSizeType(), "affs.counter.addr"), in emitTaskInit()
4391 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal); in emitTaskInit()
4399 CGF, cast_or_null<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts())); in emitTaskInit()
4403 std::tie(Addr, Size) = getPointerAndSize(CGF, E); in emitTaskInit()
4404 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); in emitTaskInit()
4405 LValue Base = CGF.MakeAddrLValue( in emitTaskInit()
4406 CGF.Builder.CreateGEP(AffinitiesArray, Idx), KmpTaskAffinityInfoTy); in emitTaskInit()
4408 LValue BaseAddrLVal = CGF.EmitLValueForField( in emitTaskInit()
4410 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy), in emitTaskInit()
4413 LValue LenLVal = CGF.EmitLValueForField( in emitTaskInit()
4415 CGF.EmitStoreOfScalar(Size, LenLVal); in emitTaskInit()
4416 Idx = CGF.Builder.CreateNUWAdd( in emitTaskInit()
4418 CGF.EmitStoreOfScalar(Idx, PosLVal); in emitTaskInit()
4424 llvm::Value *LocRef = emitUpdateLocation(CGF, Loc); in emitTaskInit()
4425 llvm::Value *GTid = getThreadID(CGF, Loc); in emitTaskInit()
4426 llvm::Value *AffinListPtr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
4430 (void)CGF.EmitRuntimeCall( in emitTaskInit()
4436 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
4438 LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy, in emitTaskInit()
4441 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin()); in emitTaskInit()
4447 CGF.EmitLoadOfScalar( in emitTaskInit()
4448 CGF.EmitLValueForField( in emitTaskInit()
4452 CGF.Int8Ty, CGM.getNaturalTypeAlignment(SharedsTy)); in emitTaskInit()
4453 LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy); in emitTaskInit()
4454 LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy); in emitTaskInit()
4455 CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap); in emitTaskInit()
4460 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD, in emitTaskInit()
4464 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) { in emitTaskInit()
4481 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI); in emitTaskInit()
4482 LValue DestructorsLV = CGF.EmitLValueForField( in emitTaskInit()
4484 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
4490 LValue Data2LV = CGF.EmitLValueForField( in emitTaskInit()
4492 LValue PriorityLV = CGF.EmitLValueForField( in emitTaskInit()
4494 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV); in emitTaskInit()
4564 CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal, in getDepobjElements() argument
4572 LValue Base = CGF.EmitLoadOfPointerLValue( in getDepobjElements()
4573 CGF.Builder.CreateElementBitCast( in getDepobjElements()
4574 DepobjLVal.getAddress(CGF), in getDepobjElements()
4575 CGF.ConvertTypeForMem(KmpDependInfoPtrTy)), in getDepobjElements()
4577 Address DepObjAddr = CGF.Builder.CreateGEP( in getDepobjElements()
4578 Base.getAddress(CGF), in getDepobjElements()
4579 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); in getDepobjElements()
4580 LValue NumDepsBase = CGF.MakeAddrLValue( in getDepobjElements()
4583 LValue BaseAddrLVal = CGF.EmitLValueForField( in getDepobjElements()
4585 llvm::Value *NumDeps = CGF.EmitLoadOfScalar(BaseAddrLVal, Loc); in getDepobjElements()
4589 static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy, in emitDependData() argument
4593 CodeGenModule &CGM = CGF.CGM; in emitDependData()
4599 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy); in emitDependData()
4602 CGF, cast_or_null<OMPIteratorExpr>( in emitDependData()
4611 std::tie(Addr, Size) = getPointerAndSize(CGF, E); in emitDependData()
4612 Addr = CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy); in emitDependData()
4614 Addr = llvm::ConstantInt::get(CGF.IntPtrTy, 0); in emitDependData()
4615 Size = llvm::ConstantInt::get(CGF.SizeTy, 0); in emitDependData()
4619 Base = CGF.MakeAddrLValue( in emitDependData()
4620 CGF.Builder.CreateConstGEP(DependenciesArray, *P), KmpDependInfoTy); in emitDependData()
4624 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); in emitDependData()
4625 Base = CGF.MakeAddrLValue( in emitDependData()
4626 CGF.Builder.CreateGEP(DependenciesArray, Idx), KmpDependInfoTy); in emitDependData()
4629 LValue BaseAddrLVal = CGF.EmitLValueForField( in emitDependData()
4631 CGF.EmitStoreOfScalar(Addr, BaseAddrLVal); in emitDependData()
4633 LValue LenLVal = CGF.EmitLValueForField( in emitDependData()
4635 CGF.EmitStoreOfScalar(Size, LenLVal); in emitDependData()
4638 LValue FlagsLVal = CGF.EmitLValueForField( in emitDependData()
4640 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind), in emitDependData()
4646 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); in emitDependData()
4647 Idx = CGF.Builder.CreateNUWAdd(Idx, in emitDependData()
4649 CGF.EmitStoreOfScalar(Idx, PosLVal); in emitDependData()
4655 CodeGenFunction &CGF, QualType &KmpDependInfoTy, in emitDepobjElementsSizes() argument
4661 ASTContext &C = CGF.getContext(); in emitDepobjElementsSizes()
4664 CGF, cast_or_null<OMPIteratorExpr>( in emitDepobjElementsSizes()
4670 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts()); in emitDepobjElementsSizes()
4672 getDepobjElements(CGF, DepobjLVal, E->getExprLoc()); in emitDepobjElementsSizes()
4673 LValue NumLVal = CGF.MakeAddrLValue( in emitDepobjElementsSizes()
4674 CGF.CreateMemTemp(C.getUIntPtrType(), "depobj.size.addr"), in emitDepobjElementsSizes()
4676 CGF.Builder.CreateStore(llvm::ConstantInt::get(CGF.IntPtrTy, 0), in emitDepobjElementsSizes()
4677 NumLVal.getAddress(CGF)); in emitDepobjElementsSizes()
4678 llvm::Value *PrevVal = CGF.EmitLoadOfScalar(NumLVal, E->getExprLoc()); in emitDepobjElementsSizes()
4679 llvm::Value *Add = CGF.Builder.CreateNUWAdd(PrevVal, NumDeps); in emitDepobjElementsSizes()
4680 CGF.EmitStoreOfScalar(Add, NumLVal); in emitDepobjElementsSizes()
4686 CGF.EmitLoadOfScalar(SizeLVals[I], Data.DepExprs[I]->getExprLoc()); in emitDepobjElementsSizes()
4692 void CGOpenMPRuntime::emitDepobjElements(CodeGenFunction &CGF, in emitDepobjElements() argument
4699 llvm::Value *ElSize = CGF.getTypeSize(KmpDependInfoTy); in emitDepobjElements()
4702 CGF, cast_or_null<OMPIteratorExpr>( in emitDepobjElements()
4709 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts()); in emitDepobjElements()
4711 getDepobjElements(CGF, DepobjLVal, E->getExprLoc()); in emitDepobjElements()
4714 llvm::Value *Size = CGF.Builder.CreateNUWMul( in emitDepobjElements()
4716 CGF.Builder.CreateIntCast(NumDeps, CGF.SizeTy, /*isSigned=*/false)); in emitDepobjElements()
4717 llvm::Value *Pos = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); in emitDepobjElements()
4718 Address DepAddr = CGF.Builder.CreateGEP(DependenciesArray, Pos); in emitDepobjElements()
4719 CGF.Builder.CreateMemCpy(DepAddr, Base.getAddress(CGF), Size); in emitDepobjElements()
4723 llvm::Value *Add = CGF.Builder.CreateNUWAdd(Pos, NumDeps); in emitDepobjElements()
4724 CGF.EmitStoreOfScalar(Add, PosLVal); in emitDepobjElements()
4730 CodeGenFunction &CGF, ArrayRef<OMPTaskDataTy::DependData> Dependencies, in emitDependClause() argument
4751 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.IntPtrTy, 0); in emitDependClause()
4753 llvm::ConstantInt::get(CGF.IntPtrTy, 0); in emitDependClause()
4758 emitDepobjElementsSizes(CGF, KmpDependInfoTy, D); in emitDependClause()
4761 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, Size); in emitDependClause()
4770 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper); in emitDependClause()
4771 Sz = CGF.Builder.CreateIntCast(Sz, CGF.IntPtrTy, /*isSigned=*/false); in emitDependClause()
4772 llvm::Value *NumClauseDeps = CGF.Builder.CreateNUWMul( in emitDependClause()
4773 Sz, llvm::ConstantInt::get(CGF.IntPtrTy, D.DepExprs.size())); in emitDependClause()
4775 CGF.Builder.CreateNUWAdd(NumOfRegularWithIterators, NumClauseDeps); in emitDependClause()
4788 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, NumOfElements); in emitDependClause()
4792 CGF.Builder.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements); in emitDependClause()
4797 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE, in emitDependClause()
4806 CGF.EmitVarDecl(*PD); in emitDependClause()
4807 DependenciesArray = CGF.GetAddrOfLocalVar(PD); in emitDependClause()
4808 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty, in emitDependClause()
4815 CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr"); in emitDependClause()
4816 DependenciesArray = CGF.Builder.CreateConstArrayGEP(DependenciesArray, 0); in emitDependClause()
4825 emitDependData(CGF, KmpDependInfoTy, &Pos, Dependencies[I], in emitDependClause()
4829 LValue PosLVal = CGF.MakeAddrLValue( in emitDependClause()
4830 CGF.CreateMemTemp(C.getSizeType(), "dep.counter.addr"), C.getSizeType()); in emitDependClause()
4831 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal); in emitDependClause()
4836 emitDependData(CGF, KmpDependInfoTy, &PosLVal, Dependencies[I], in emitDependClause()
4844 emitDepobjElements(CGF, KmpDependInfoTy, PosLVal, Dependencies[I], in emitDependClause()
4848 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDependClause()
4849 DependenciesArray, CGF.VoidPtrTy, CGF.Int8Ty); in emitDependClause()
4854 CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies, in emitDepobjDependClause() argument
4876 NumDepsVal = llvm::ConstantInt::get(CGF.SizeTy, 1); in emitDepobjDependClause()
4878 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper); in emitDepobjDependClause()
4879 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false); in emitDepobjDependClause()
4880 NumDepsVal = CGF.Builder.CreateNUWMul(NumDepsVal, Sz); in emitDepobjDependClause()
4882 Size = CGF.Builder.CreateNUWAdd(llvm::ConstantInt::get(CGF.SizeTy, 1), in emitDepobjDependClause()
4887 Size = CGF.Builder.CreateNUWMul(Size, RecSize); in emitDepobjDependClause()
4889 CGF.Builder.CreateIntCast(NumDepsVal, CGF.IntPtrTy, /*isSigned=*/false); in emitDepobjDependClause()
4896 NumDepsVal = llvm::ConstantInt::get(CGF.IntPtrTy, NumDependencies); in emitDepobjDependClause()
4899 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitDepobjDependClause()
4901 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitDepobjDependClause()
4905 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitDepobjDependClause()
4908 llvm::Type *KmpDependInfoLlvmTy = CGF.ConvertTypeForMem(KmpDependInfoTy); in emitDepobjDependClause()
4909 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDepobjDependClause()
4913 LValue Base = CGF.MakeAddrLValue(DependenciesArray, KmpDependInfoTy); in emitDepobjDependClause()
4915 LValue BaseAddrLVal = CGF.EmitLValueForField( in emitDepobjDependClause()
4917 CGF.EmitStoreOfScalar(NumDepsVal, BaseAddrLVal); in emitDepobjDependClause()
4922 PosLVal = CGF.MakeAddrLValue( in emitDepobjDependClause()
4923 CGF.CreateMemTemp(C.getSizeType(), "iterator.counter.addr"), in emitDepobjDependClause()
4925 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Idx), PosLVal, in emitDepobjDependClause()
4931 emitDependData(CGF, KmpDependInfoTy, Pos, Dependencies, DependenciesArray); in emitDepobjDependClause()
4932 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDepobjDependClause()
4933 CGF.Builder.CreateConstGEP(DependenciesArray, 1), CGF.VoidPtrTy, in emitDepobjDependClause()
4934 CGF.Int8Ty); in emitDepobjDependClause()
4938 void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal, in emitDestroyClause() argument
4943 LValue Base = CGF.EmitLoadOfPointerLValue( in emitDestroyClause()
4944 DepobjLVal.getAddress(CGF), C.VoidPtrTy.castAs<PointerType>()); in emitDestroyClause()
4946 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDestroyClause()
4947 Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy), in emitDestroyClause()
4948 CGF.ConvertTypeForMem(KmpDependInfoTy)); in emitDestroyClause()
4949 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP( in emitDestroyClause()
4951 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); in emitDestroyClause()
4952 DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(DepObjAddr, in emitDestroyClause()
4953 CGF.VoidPtrTy); in emitDestroyClause()
4954 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitDestroyClause()
4956 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitDestroyClause()
4960 (void)CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitDestroyClause()
4965 void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, in emitUpdateClause() argument
4973 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy); in emitUpdateClause()
4976 std::tie(NumDeps, Base) = getDepobjElements(CGF, DepobjLVal, Loc); in emitUpdateClause()
4978 Address Begin = Base.getAddress(CGF); in emitUpdateClause()
4980 llvm::Value *End = CGF.Builder.CreateGEP( in emitUpdateClause()
4983 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.body"); in emitUpdateClause()
4984 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.done"); in emitUpdateClause()
4985 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock(); in emitUpdateClause()
4986 CGF.EmitBlock(BodyBB); in emitUpdateClause()
4988 CGF.Builder.CreatePHI(Begin.getType(), 2, "omp.elementPast"); in emitUpdateClause()
4991 Base = CGF.MakeAddrLValue(Begin, KmpDependInfoTy, Base.getBaseInfo(), in emitUpdateClause()
4995 LValue FlagsLVal = CGF.EmitLValueForField( in emitUpdateClause()
4997 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind), in emitUpdateClause()
5002 CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext"); in emitUpdateClause()
5004 CGF.Builder.GetInsertBlock()); in emitUpdateClause()
5006 CGF.Builder.CreateICmpEQ(ElementNext.getPointer(), End, "omp.isempty"); in emitUpdateClause()
5007 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); in emitUpdateClause()
5009 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); in emitUpdateClause()
5012 void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskCall() argument
5018 if (!CGF.HaveInsertPoint()) in emitTaskCall()
5022 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data); in emitTaskCall()
5032 emitDependClause(CGF, Data.Dependences, Loc); in emitTaskCall()
5040 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitTaskCall()
5041 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc); in emitTaskCall()
5050 DepTaskArgs[5] = CGF.Builder.getInt32(0); in emitTaskCall()
5051 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitTaskCall()
5054 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) { in emitTaskCall() argument
5057 LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI); in emitTaskCall()
5058 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal); in emitTaskCall()
5061 CGF.EmitRuntimeCall( in emitTaskCall()
5066 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskCall()
5072 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitTaskCall()
5073 Region->emitUntiedSwitch(CGF); in emitTaskCall()
5082 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0); in emitTaskCall()
5083 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitTaskCall()
5088 Loc](CodeGenFunction &CGF, PrePostActionTy &) { in emitTaskCall() argument
5089 CodeGenFunction::RunCleanupsScope LocalScope(CGF); in emitTaskCall()
5095 CGF.EmitRuntimeCall( in emitTaskCall()
5100 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTaskCall() argument
5101 Action.Enter(CGF); in emitTaskCall()
5103 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry, in emitTaskCall()
5119 RCG(CGF); in emitTaskCall()
5123 emitIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen); in emitTaskCall()
5126 ThenRCG(CGF); in emitTaskCall()
5130 void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskLoopCall() argument
5136 if (!CGF.HaveInsertPoint()) in emitTaskLoopCall()
5139 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data); in emitTaskLoopCall()
5145 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitTaskLoopCall()
5146 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc); in emitTaskLoopCall()
5149 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy, in emitTaskLoopCall()
5152 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1); in emitTaskLoopCall()
5155 LValue LBLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
5160 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(CGF), in emitTaskLoopCall()
5163 LValue UBLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
5168 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(CGF), in emitTaskLoopCall()
5171 LValue StLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
5176 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(CGF), in emitTaskLoopCall()
5180 LValue RedLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
5184 CGF.EmitStoreOfScalar(Data.Reductions, RedLVal); in emitTaskLoopCall()
5186 CGF.EmitNullInitialization(RedLVal.getAddress(CGF), in emitTaskLoopCall()
5187 CGF.getContext().VoidPtrTy); in emitTaskLoopCall()
5195 LBLVal.getPointer(CGF), in emitTaskLoopCall()
5196 UBLVal.getPointer(CGF), in emitTaskLoopCall()
5197 CGF.EmitLoadOfScalar(StLVal, Loc), in emitTaskLoopCall()
5199 CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler in emitTaskLoopCall()
5201 CGF.IntTy, Data.Schedule.getPointer() in emitTaskLoopCall()
5205 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty, in emitTaskLoopCall()
5207 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0), in emitTaskLoopCall()
5208 Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskLoopCall()
5209 Result.TaskDupFn, CGF.VoidPtrTy) in emitTaskLoopCall()
5210 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)}; in emitTaskLoopCall()
5211 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskLoopCall()
5226 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar, in EmitOMPAggregateReduction() argument
5228 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *, in EmitOMPAggregateReduction() argument
5234 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar); in EmitOMPAggregateReduction()
5235 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar); in EmitOMPAggregateReduction()
5239 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr); in EmitOMPAggregateReduction()
5245 CGF.Builder.CreateGEP(LHSAddr.getElementType(), LHSBegin, NumElements); in EmitOMPAggregateReduction()
5247 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body"); in EmitOMPAggregateReduction()
5248 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done"); in EmitOMPAggregateReduction()
5250 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty"); in EmitOMPAggregateReduction()
5251 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); in EmitOMPAggregateReduction()
5254 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock(); in EmitOMPAggregateReduction()
5255 CGF.EmitBlock(BodyBB); in EmitOMPAggregateReduction()
5257 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy); in EmitOMPAggregateReduction()
5259 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI( in EmitOMPAggregateReduction()
5266 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI( in EmitOMPAggregateReduction()
5274 CodeGenFunction::OMPPrivateScope Scope(CGF); in EmitOMPAggregateReduction()
5278 RedOpGen(CGF, XExpr, EExpr, UpExpr); in EmitOMPAggregateReduction()
5282 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateReduction()
5285 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateReduction()
5290 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done"); in EmitOMPAggregateReduction()
5291 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB); in EmitOMPAggregateReduction()
5292 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateReduction()
5293 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateReduction()
5296 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); in EmitOMPAggregateReduction()
5302 static void emitReductionCombiner(CodeGenFunction &CGF, in emitReductionCombiner() argument
5311 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD); in emitReductionCombiner()
5313 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func); in emitReductionCombiner()
5314 CGF.EmitIgnoredExpr(ReductionOp); in emitReductionCombiner()
5317 CGF.EmitIgnoredExpr(ReductionOp); in emitReductionCombiner()
5342 CodeGenFunction CGF(CGM); in emitReductionFunction() local
5343 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitReductionFunction()
5347 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReductionFunction()
5348 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)), in emitReductionFunction()
5350 ArgsElemType, CGF.getPointerAlign()); in emitReductionFunction()
5351 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReductionFunction()
5352 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)), in emitReductionFunction()
5354 ArgsElemType, CGF.getPointerAlign()); in emitReductionFunction()
5359 CodeGenFunction::OMPPrivateScope Scope(CGF); in emitReductionFunction()
5365 Scope.addPrivate(RHSVar, emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar)); in emitReductionFunction()
5368 Scope.addPrivate(LHSVar, emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar)); in emitReductionFunction()
5373 Address Elem = CGF.Builder.CreateConstArrayGEP(LHS, Idx); in emitReductionFunction()
5374 llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem); in emitReductionFunction()
5376 CGF.getContext().getAsVariableArrayType(PrivTy); in emitReductionFunction()
5379 CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy))); in emitReductionFunction()
5380 CGF.EmitVariablyModifiedType(PrivTy); in emitReductionFunction()
5393 CGF, (*IPriv)->getType(), LHSVar, RHSVar, in emitReductionFunction()
5394 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) { in emitReductionFunction() argument
5395 emitReductionCombiner(CGF, E); in emitReductionFunction()
5399 emitReductionCombiner(CGF, E); in emitReductionFunction()
5406 CGF.FinishFunction(); in emitReductionFunction()
5410 void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF, in emitSingleReductionCombiner() argument
5420 CGF, PrivateRef->getType(), LHSVar, RHSVar, in emitSingleReductionCombiner()
5421 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) { in emitSingleReductionCombiner() argument
5422 emitReductionCombiner(CGF, ReductionOp); in emitSingleReductionCombiner()
5426 emitReductionCombiner(CGF, ReductionOp); in emitSingleReductionCombiner()
5430 void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, in emitReduction() argument
5436 if (!CGF.HaveInsertPoint()) in emitReduction()
5480 CodeGenFunction::RunCleanupsScope Scope(CGF); in emitReduction()
5485 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), in emitReduction()
5507 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); in emitReduction()
5511 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitReduction()
5512 CGF.Builder.CreateStore( in emitReduction()
5513 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReduction()
5514 CGF.EmitLValue(RHSExprs[I]).getPointer(CGF), CGF.VoidPtrTy), in emitReduction()
5519 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitReduction()
5520 llvm::Value *Size = CGF.Builder.CreateIntCast( in emitReduction()
5521 CGF.getVLASize( in emitReduction()
5522 CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) in emitReduction()
5524 CGF.SizeTy, /*isSigned=*/false); in emitReduction()
5525 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), in emitReduction()
5532 emitReductionFunction(Loc, CGF.ConvertTypeForMem(ReductionArrayTy), in emitReduction()
5541 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE); in emitReduction()
5542 llvm::Value *ThreadId = getThreadID(CGF, Loc); in emitReduction()
5543 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy); in emitReduction()
5544 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReduction()
5545 ReductionList.getPointer(), CGF.VoidPtrTy); in emitReduction()
5549 CGF.Builder.getInt32(RHSExprs.size()), // i32 <n> in emitReduction()
5555 llvm::Value *Res = CGF.EmitRuntimeCall( in emitReduction()
5562 llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(".omp.reduction.default"); in emitReduction()
5564 CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2); in emitReduction()
5572 llvm::BasicBlock *Case1BB = CGF.createBasicBlock(".omp.reduction.case1"); in emitReduction()
5573 SwInst->addCase(CGF.Builder.getInt32(1), Case1BB); in emitReduction()
5574 CGF.EmitBlock(Case1BB); in emitReduction()
5583 CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
5584 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitReduction()
5589 RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), in emitReduction()
5604 RCG(CGF); in emitReduction()
5606 CGF.EmitBranch(DefaultBB); in emitReduction()
5613 llvm::BasicBlock *Case2BB = CGF.createBasicBlock(".omp.reduction.case2"); in emitReduction()
5614 SwInst->addCase(CGF.Builder.getInt32(2), Case2BB); in emitReduction()
5615 CGF.EmitBlock(Case2BB); in emitReduction()
5618 CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
5652 Loc](CodeGenFunction &CGF, const Expr *XExpr, in emitReduction()
5654 LValue X = CGF.EmitLValue(XExpr); in emitReduction()
5657 E = CGF.EmitAnyExpr(EExpr); in emitReduction()
5658 CGF.EmitOMPAtomicSimpleUpdateExpr( in emitReduction()
5661 [&CGF, UpExpr, VD, Loc](RValue XRValue) { in emitReduction()
5662 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitReduction()
5663 Address LHSTemp = CGF.CreateMemTemp(VD->getType()); in emitReduction()
5664 CGF.emitOMPSimpleStore( in emitReduction()
5665 CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue, in emitReduction()
5669 return CGF.EmitAnyExpr(UpExpr); in emitReduction()
5676 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar, in emitReduction()
5680 AtomicRedGen(CGF, XExpr, EExpr, UpExpr); in emitReduction()
5684 auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *, in emitReduction()
5686 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitReduction()
5689 CGF, Name, in emitReduction()
5690 [=](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
5691 Action.Enter(CGF); in emitReduction()
5692 emitReductionCombiner(CGF, E); in emitReduction()
5701 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar, in emitReduction()
5704 CritRedGen(CGF, nullptr, nullptr, nullptr); in emitReduction()
5725 AtomicRCG(CGF); in emitReduction()
5727 AtomicRCG(CGF); in emitReduction()
5730 CGF.EmitBranch(DefaultBB); in emitReduction()
5731 CGF.EmitBlock(DefaultBB, /*IsFinished=*/true); in emitReduction()
5781 CodeGenFunction CGF(CGM); in emitReduceInitFunction() local
5782 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc); in emitReduceInitFunction()
5784 Address PrivateAddr = CGF.EmitLoadOfPointer( in emitReduceInitFunction()
5785 CGF.Builder.CreateElementBitCast( in emitReduceInitFunction()
5786 CGF.GetAddrOfLocalVar(&Param), in emitReduceInitFunction()
5787 CGF.ConvertTypeForMem(PrivateType)->getPointerTo()), in emitReduceInitFunction()
5794 CGF, CGM.getContext().getSizeType(), in emitReduceInitFunction()
5796 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false, in emitReduceInitFunction()
5799 RCG.emitAggregateType(CGF, N, Size); in emitReduceInitFunction()
5805 Address SharedAddr = CGF.GetAddrOfLocalVar(&ParamOrig); in emitReduceInitFunction()
5806 OrigAddr = CGF.EmitLoadOfPointer( in emitReduceInitFunction()
5813 RCG.emitInitialization(CGF, N, PrivateAddr, OrigAddr, in emitReduceInitFunction()
5815 CGF.FinishFunction(); in emitReduceInitFunction()
5853 CodeGenFunction CGF(CGM); in emitReduceCombFunction() local
5854 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc); in emitReduceCombFunction()
5860 CGF, CGM.getContext().getSizeType(), in emitReduceCombFunction()
5862 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false, in emitReduceCombFunction()
5865 RCG.emitAggregateType(CGF, N, Size); in emitReduceCombFunction()
5869 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitReduceCombFunction()
5873 CGF.EmitLoadOfPointer( in emitReduceCombFunction()
5874 CGF.Builder.CreateElementBitCast( in emitReduceCombFunction()
5875 CGF.GetAddrOfLocalVar(&ParamInOut), in emitReduceCombFunction()
5876 CGF.ConvertTypeForMem(LHSVD->getType())->getPointerTo()), in emitReduceCombFunction()
5881 CGF.EmitLoadOfPointer( in emitReduceCombFunction()
5882 CGF.Builder.CreateElementBitCast( in emitReduceCombFunction()
5883 CGF.GetAddrOfLocalVar(&ParamIn), in emitReduceCombFunction()
5884 CGF.ConvertTypeForMem(RHSVD->getType())->getPointerTo()), in emitReduceCombFunction()
5891 CGF, ReductionOp, PrivateRef, cast<DeclRefExpr>(LHS), in emitReduceCombFunction()
5893 CGF.FinishFunction(); in emitReduceCombFunction()
5923 CodeGenFunction CGF(CGM); in emitReduceFiniFunction() local
5924 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc); in emitReduceFiniFunction()
5925 Address PrivateAddr = CGF.EmitLoadOfPointer( in emitReduceFiniFunction()
5926 CGF.GetAddrOfLocalVar(&Param), C.VoidPtrTy.castAs<PointerType>()); in emitReduceFiniFunction()
5932 CGF, CGM.getContext().getSizeType(), in emitReduceFiniFunction()
5934 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false, in emitReduceFiniFunction()
5937 RCG.emitAggregateType(CGF, N, Size); in emitReduceFiniFunction()
5940 RCG.emitCleanups(CGF, N, PrivateAddr); in emitReduceFiniFunction()
5941 CGF.FinishFunction(Loc); in emitReduceFiniFunction()
5946 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs, in emitTaskReductionInit() argument
5948 if (!CGF.HaveInsertPoint() || Data.ReductionVars.empty()) in emitTaskReductionInit()
5979 Address TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input."); in emitTaskReductionInit()
5986 llvm::Value *GEP = CGF.EmitCheckedInBoundsGEP( in emitTaskReductionInit()
5990 LValue ElemLVal = CGF.MakeNaturalAlignAddrLValue(GEP, RDType); in emitTaskReductionInit()
5992 LValue SharedLVal = CGF.EmitLValueForField(ElemLVal, SharedFD); in emitTaskReductionInit()
5993 RCG.emitSharedOrigLValue(CGF, Cnt); in emitTaskReductionInit()
5995 CGF.EmitCastToVoidPtr(RCG.getSharedLValue(Cnt).getPointer(CGF)); in emitTaskReductionInit()
5996 CGF.EmitStoreOfScalar(CastedShared, SharedLVal); in emitTaskReductionInit()
5998 LValue OrigLVal = CGF.EmitLValueForField(ElemLVal, OrigFD); in emitTaskReductionInit()
6000 CGF.EmitCastToVoidPtr(RCG.getOrigLValue(Cnt).getPointer(CGF)); in emitTaskReductionInit()
6001 CGF.EmitStoreOfScalar(CastedOrig, OrigLVal); in emitTaskReductionInit()
6002 RCG.emitAggregateType(CGF, Cnt); in emitTaskReductionInit()
6012 SizeValInChars = CGF.Builder.CreateIntCast(SizeValInChars, CGM.SizeTy, in emitTaskReductionInit()
6014 LValue SizeLVal = CGF.EmitLValueForField(ElemLVal, SizeFD); in emitTaskReductionInit()
6015 CGF.EmitStoreOfScalar(SizeValInChars, SizeLVal); in emitTaskReductionInit()
6017 LValue InitLVal = CGF.EmitLValueForField(ElemLVal, InitFD); in emitTaskReductionInit()
6019 CGF.EmitCastToVoidPtr(emitReduceInitFunction(CGM, Loc, RCG, Cnt)); in emitTaskReductionInit()
6020 CGF.EmitStoreOfScalar(InitAddr, InitLVal); in emitTaskReductionInit()
6022 LValue FiniLVal = CGF.EmitLValueForField(ElemLVal, FiniFD); in emitTaskReductionInit()
6025 ? CGF.EmitCastToVoidPtr(Fini) in emitTaskReductionInit()
6027 CGF.EmitStoreOfScalar(FiniAddr, FiniLVal); in emitTaskReductionInit()
6029 LValue CombLVal = CGF.EmitLValueForField(ElemLVal, CombFD); in emitTaskReductionInit()
6030 llvm::Value *CombAddr = CGF.EmitCastToVoidPtr(emitReduceCombFunction( in emitTaskReductionInit()
6033 CGF.EmitStoreOfScalar(CombAddr, CombLVal); in emitTaskReductionInit()
6035 LValue FlagsLVal = CGF.EmitLValueForField(ElemLVal, FlagsFD); in emitTaskReductionInit()
6037 CGF.EmitStoreOfScalar( in emitTaskReductionInit()
6041 CGF.EmitNullInitialization(FlagsLVal.getAddress(CGF), in emitTaskReductionInit()
6047 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc); in emitTaskReductionInit()
6048 llvm::Value *GTid = CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), in emitTaskReductionInit()
6055 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskReductionInit()
6057 return CGF.EmitRuntimeCall( in emitTaskReductionInit()
6064 CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), CGM.IntTy, in emitTaskReductionInit()
6067 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskRedInput.getPointer(), in emitTaskReductionInit()
6069 return CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskReductionInit()
6074 void CGOpenMPRuntime::emitTaskReductionFini(CodeGenFunction &CGF, in emitTaskReductionFini() argument
6079 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc); in emitTaskReductionFini()
6080 llvm::Value *GTid = CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), in emitTaskReductionFini()
6086 (void)CGF.EmitRuntimeCall( in emitTaskReductionFini()
6092 void CGOpenMPRuntime::emitTaskReductionFixups(CodeGenFunction &CGF, in emitTaskReductionFixups() argument
6100 llvm::Value *SizeVal = CGF.Builder.CreateIntCast(Sizes.second, CGM.SizeTy, in emitTaskReductionFixups()
6103 CGF, CGM.getContext().getSizeType(), in emitTaskReductionFixups()
6105 CGF.Builder.CreateStore(SizeVal, SizeAddr, /*IsVolatile=*/false); in emitTaskReductionFixups()
6109 Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF, in getTaskReductionItem() argument
6115 llvm::Value *Args[] = {CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), in getTaskReductionItem()
6119 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getTaskReductionItem()
6120 SharedLVal.getPointer(CGF), CGM.VoidPtrTy)}; in getTaskReductionItem()
6122 CGF.EmitRuntimeCall( in getTaskReductionItem()
6126 CGF.Int8Ty, SharedLVal.getAlignment()); in getTaskReductionItem()
6129 void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskwaitCall() argument
6131 if (!CGF.HaveInsertPoint()) in emitTaskwaitCall()
6134 if (CGF.CGM.getLangOpts().OpenMPIRBuilder && Data.Dependences.empty()) { in emitTaskwaitCall()
6136 OMPBuilder.createTaskwait(CGF.Builder); in emitTaskwaitCall()
6138 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitTaskwaitCall()
6139 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc); in emitTaskwaitCall()
6144 emitDependClause(CGF, Data.Dependences, Loc); in emitTaskwaitCall()
6151 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0); in emitTaskwaitCall()
6152 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitTaskwaitCall()
6154 CodeGenFunction::RunCleanupsScope LocalScope(CGF); in emitTaskwaitCall()
6160 CGF.EmitRuntimeCall( in emitTaskwaitCall()
6170 CGF.EmitRuntimeCall( in emitTaskwaitCall()
6176 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitTaskwaitCall()
6177 Region->emitUntiedSwitch(CGF); in emitTaskwaitCall()
6180 void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF, in emitInlinedDirective() argument
6184 if (!CGF.HaveInsertPoint()) in emitInlinedDirective()
6186 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel, in emitInlinedDirective()
6190 CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr); in emitInlinedDirective()
6219 CodeGenFunction &CGF, SourceLocation Loc, in emitCancellationPointCall() argument
6221 if (!CGF.HaveInsertPoint()) in emitCancellationPointCall()
6226 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { in emitCancellationPointCall()
6231 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitCancellationPointCall()
6232 CGF.Builder.getInt32(getCancellationKind(CancelRegion))}; in emitCancellationPointCall()
6234 llvm::Value *Result = CGF.EmitRuntimeCall( in emitCancellationPointCall()
6242 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit"); in emitCancellationPointCall()
6243 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue"); in emitCancellationPointCall()
6244 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result); in emitCancellationPointCall()
6245 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); in emitCancellationPointCall()
6246 CGF.EmitBlock(ExitBB); in emitCancellationPointCall()
6248 emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false); in emitCancellationPointCall()
6251 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); in emitCancellationPointCall()
6252 CGF.EmitBranchThroughCleanup(CancelDest); in emitCancellationPointCall()
6253 CGF.EmitBlock(ContBB, /*IsFinished=*/true); in emitCancellationPointCall()
6258 void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, in emitCancelCall() argument
6261 if (!CGF.HaveInsertPoint()) in emitCancelCall()
6267 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { in emitCancelCall()
6269 OMPRegionInfo](CodeGenFunction &CGF, PrePostActionTy &) { in emitCancelCall() argument
6270 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitCancelCall()
6272 RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc), in emitCancelCall()
6273 CGF.Builder.getInt32(getCancellationKind(CancelRegion))}; in emitCancelCall()
6275 llvm::Value *Result = CGF.EmitRuntimeCall( in emitCancelCall()
6281 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit"); in emitCancelCall()
6282 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue"); in emitCancelCall()
6283 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result); in emitCancelCall()
6284 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); in emitCancelCall()
6285 CGF.EmitBlock(ExitBB); in emitCancelCall()
6287 RT.emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false); in emitCancelCall()
6290 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); in emitCancelCall()
6291 CGF.EmitBranchThroughCleanup(CancelDest); in emitCancelCall()
6292 CGF.EmitBlock(ContBB, /*IsFinished=*/true); in emitCancelCall()
6295 emitIfClause(CGF, IfCond, ThenGen, in emitCancelCall()
6299 ThenRCG(CGF); in emitCancelCall()
6313 void Enter(CodeGenFunction &CGF) override { in Enter() argument
6314 if (!CGF.HaveInsertPoint()) in Enter()
6317 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsInit( in Enter()
6318 CGF, AllocatorData.first, AllocatorData.second); in Enter()
6321 void Exit(CodeGenFunction &CGF) override { in Exit() argument
6322 if (!CGF.HaveInsertPoint()) in Exit()
6325 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsFini(CGF, in Exit()
6353 void CGOpenMPRuntime::emitUsesAllocatorsInit(CodeGenFunction &CGF, in emitUsesAllocatorsInit() argument
6356 llvm::Value *ThreadId = getThreadID(CGF, Allocator->getExprLoc()); in emitUsesAllocatorsInit()
6357 ThreadId = CGF.Builder.CreateIntCast(ThreadId, CGF.IntTy, /*isSigned=*/true); in emitUsesAllocatorsInit()
6359 llvm::Value *MemSpaceHandle = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitUsesAllocatorsInit()
6361 CGF.IntTy, cast<ConstantArrayType>( in emitUsesAllocatorsInit()
6365 LValue AllocatorTraitsLVal = CGF.EmitLValue(AllocatorTraits); in emitUsesAllocatorsInit()
6366 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitUsesAllocatorsInit()
6367 AllocatorTraitsLVal.getAddress(CGF), CGF.VoidPtrPtrTy, CGF.VoidPtrTy); in emitUsesAllocatorsInit()
6368 AllocatorTraitsLVal = CGF.MakeAddrLValue(Addr, CGF.getContext().VoidPtrTy, in emitUsesAllocatorsInit()
6372 CGF.EmitLoadOfScalar(AllocatorTraitsLVal, AllocatorTraits->getExprLoc()); in emitUsesAllocatorsInit()
6375 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitUsesAllocatorsInit()
6379 CGF.EmitVarDecl(*cast<VarDecl>( in emitUsesAllocatorsInit()
6381 LValue AllocatorLVal = CGF.EmitLValue(Allocator->IgnoreParenImpCasts()); in emitUsesAllocatorsInit()
6383 CGF.EmitScalarConversion(AllocatorVal, CGF.getContext().VoidPtrTy, in emitUsesAllocatorsInit()
6385 CGF.EmitStoreOfScalar(AllocatorVal, AllocatorLVal); in emitUsesAllocatorsInit()
6388 void CGOpenMPRuntime::emitUsesAllocatorsFini(CodeGenFunction &CGF, in emitUsesAllocatorsFini() argument
6390 llvm::Value *ThreadId = getThreadID(CGF, Allocator->getExprLoc()); in emitUsesAllocatorsFini()
6391 ThreadId = CGF.Builder.CreateIntCast(ThreadId, CGF.IntTy, /*isSigned=*/true); in emitUsesAllocatorsFini()
6392 LValue AllocatorLVal = CGF.EmitLValue(Allocator->IgnoreParenImpCasts()); in emitUsesAllocatorsFini()
6394 CGF.EmitLoadOfScalar(AllocatorLVal, Allocator->getExprLoc()); in emitUsesAllocatorsFini()
6395 AllocatorVal = CGF.EmitScalarConversion(AllocatorVal, Allocator->getType(), in emitUsesAllocatorsFini()
6396 CGF.getContext().VoidPtrTy, in emitUsesAllocatorsFini()
6398 (void)CGF.EmitRuntimeCall( in emitUsesAllocatorsFini()
6433 CodeGenFunction CGF(CGM, true); in emitTargetOutlinedFunctionHelper() local
6435 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitTargetOutlinedFunctionHelper()
6438 OutlinedFn = CGF.GenerateOpenMPCapturedStmtFunction(CS, D.getBeginLoc()); in emitTargetOutlinedFunctionHelper()
6488 getNumTeamsExprForTargetDirective(CGF, D, DefaultValTeams); in emitTargetOutlinedFunctionHelper()
6494 getNumThreadsExprForTargetDirective(CGF, D, DefaultValThreads); in emitTargetOutlinedFunctionHelper()
6557 CodeGenFunction &CGF, const OMPExecutableDirective &D, in getNumTeamsExprForTargetDirective() argument
6569 CGOpenMPRuntime::getSingleCompoundChild(CGF.getContext(), Body); in getNumTeamsExprForTargetDirective()
6576 if (NumTeams->isIntegerConstantExpr(CGF.getContext())) in getNumTeamsExprForTargetDirective()
6578 NumTeams->getIntegerConstantExpr(CGF.getContext())) in getNumTeamsExprForTargetDirective()
6605 if (NumTeams->isIntegerConstantExpr(CGF.getContext())) in getNumTeamsExprForTargetDirective()
6606 if (auto Constant = NumTeams->getIntegerConstantExpr(CGF.getContext())) in getNumTeamsExprForTargetDirective()
6686 CodeGenFunction &CGF, const OMPExecutableDirective &D) { in emitNumTeamsForTargetDirective() argument
6687 assert(!CGF.getLangOpts().OpenMPIsDevice && in emitNumTeamsForTargetDirective()
6690 CGBuilderTy &Bld = CGF.Builder; in emitNumTeamsForTargetDirective()
6692 const Expr *NumTeams = getNumTeamsExprForTargetDirective(CGF, D, DefaultNT); in emitNumTeamsForTargetDirective()
6699 CGOpenMPInnerExprInfo CGInfo(CGF, *CS); in emitNumTeamsForTargetDirective()
6700 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitNumTeamsForTargetDirective()
6701 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(NumTeams, in emitNumTeamsForTargetDirective()
6703 return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty, in emitNumTeamsForTargetDirective()
6711 CodeGenFunction::RunCleanupsScope NumTeamsScope(CGF); in emitNumTeamsForTargetDirective()
6712 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(NumTeams, in emitNumTeamsForTargetDirective()
6714 return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty, in emitNumTeamsForTargetDirective()
6722 return llvm::ConstantInt::get(CGF.Int32Ty, DefaultNT); in emitNumTeamsForTargetDirective()
6725 static llvm::Value *getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS, in getNumThreads() argument
6728 CGF.getContext(), CS->getCapturedStmt()); in getNumThreads()
6736 CGOpenMPInnerExprInfo CGInfo(CGF, *CS); in getNumThreads()
6737 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in getNumThreads()
6749 if (Cond->EvaluateAsBooleanCondition(Result, CGF.getContext())) { in getNumThreads()
6751 return CGF.Builder.getInt32(1); in getNumThreads()
6753 CodeGenFunction::LexicalScope Scope(CGF, Cond->getSourceRange()); in getNumThreads()
6758 CGF.EmitVarDecl(cast<VarDecl>(*I)); in getNumThreads()
6761 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in getNumThreads()
6762 CGF.EmitAutoVarCleanups(Emission); in getNumThreads()
6766 CondVal = CGF.EvaluateExprAsBool(Cond); in getNumThreads()
6773 CGOpenMPInnerExprInfo CGInfo(CGF, *CS); in getNumThreads()
6774 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in getNumThreads()
6778 CGF, NumThreadsClause->getNumThreads()->getSourceRange()); in getNumThreads()
6783 CGF.EmitVarDecl(cast<VarDecl>(*I)); in getNumThreads()
6786 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in getNumThreads()
6787 CGF.EmitAutoVarCleanups(Emission); in getNumThreads()
6791 NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads()); in getNumThreads()
6792 NumThreads = CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, in getNumThreads()
6795 NumThreads = CGF.Builder.CreateSelect( in getNumThreads()
6796 CGF.Builder.CreateICmpULT(DefaultThreadLimitVal, NumThreads), in getNumThreads()
6800 : CGF.Builder.getInt32(0); in getNumThreads()
6804 NumThreads = CGF.Builder.CreateSelect(CondVal, NumThreads, in getNumThreads()
6805 CGF.Builder.getInt32(1)); in getNumThreads()
6810 return CGF.Builder.getInt32(1); in getNumThreads()
6814 : CGF.Builder.getInt32(0); in getNumThreads()
6818 CodeGenFunction &CGF, const OMPExecutableDirective &D, in getNumThreadsExprForTargetDirective() argument
6833 if (ThreadLimit->isIntegerConstantExpr(CGF.getContext())) in getNumThreadsExprForTargetDirective()
6835 ThreadLimit->getIntegerConstantExpr(CGF.getContext())) in getNumThreadsExprForTargetDirective()
6850 if (ThreadLimit->isIntegerConstantExpr(CGF.getContext())) in getNumThreadsExprForTargetDirective()
6852 ThreadLimit->getIntegerConstantExpr(CGF.getContext())) in getNumThreadsExprForTargetDirective()
6858 if (NumThreads->isIntegerConstantExpr(CGF.getContext())) { in getNumThreadsExprForTargetDirective()
6860 NumThreads->getIntegerConstantExpr(CGF.getContext())) { in getNumThreadsExprForTargetDirective()
6940 CodeGenFunction &CGF, const OMPExecutableDirective &D) { in emitNumThreadsForTargetDirective() argument
6941 assert(!CGF.getLangOpts().OpenMPIsDevice && in emitNumThreadsForTargetDirective()
6947 CGBuilderTy &Bld = CGF.Builder; in emitNumThreadsForTargetDirective()
6953 if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal)) in emitNumThreadsForTargetDirective()
6956 CGF.getContext(), CS->getCapturedStmt()); in emitNumThreadsForTargetDirective()
6959 CGOpenMPInnerExprInfo CGInfo(CGF, *CS); in emitNumThreadsForTargetDirective()
6960 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitNumThreadsForTargetDirective()
6964 CGF, ThreadLimitClause->getThreadLimit()->getSourceRange()); in emitNumThreadsForTargetDirective()
6969 CGF.EmitVarDecl(cast<VarDecl>(*I)); in emitNumThreadsForTargetDirective()
6972 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in emitNumThreadsForTargetDirective()
6973 CGF.EmitAutoVarCleanups(Emission); in emitNumThreadsForTargetDirective()
6977 llvm::Value *ThreadLimit = CGF.EmitScalarExpr( in emitNumThreadsForTargetDirective()
6980 Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false); in emitNumThreadsForTargetDirective()
6986 CGF.getContext(), CS->getCapturedStmt()); in emitNumThreadsForTargetDirective()
6992 if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal)) in emitNumThreadsForTargetDirective()
7002 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF); in emitNumThreadsForTargetDirective()
7004 llvm::Value *ThreadLimit = CGF.EmitScalarExpr( in emitNumThreadsForTargetDirective()
7007 Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false); in emitNumThreadsForTargetDirective()
7010 if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal)) in emitNumThreadsForTargetDirective()
7013 CGF.getContext(), CS->getCapturedStmt()); in emitNumThreadsForTargetDirective()
7017 if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal)) in emitNumThreadsForTargetDirective()
7025 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF); in emitNumThreadsForTargetDirective()
7027 llvm::Value *ThreadLimit = CGF.EmitScalarExpr( in emitNumThreadsForTargetDirective()
7030 Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false); in emitNumThreadsForTargetDirective()
7032 return getNumThreads(CGF, D.getInnermostCapturedStmt(), ThreadLimitVal); in emitNumThreadsForTargetDirective()
7053 if (Cond->EvaluateAsBooleanCondition(Result, CGF.getContext())) { in emitNumThreadsForTargetDirective()
7057 CodeGenFunction::RunCleanupsScope Scope(CGF); in emitNumThreadsForTargetDirective()
7058 CondVal = CGF.EvaluateExprAsBool(Cond); in emitNumThreadsForTargetDirective()
7063 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF); in emitNumThreadsForTargetDirective()
7065 llvm::Value *ThreadLimit = CGF.EmitScalarExpr( in emitNumThreadsForTargetDirective()
7068 Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false); in emitNumThreadsForTargetDirective()
7071 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); in emitNumThreadsForTargetDirective()
7073 llvm::Value *NumThreads = CGF.EmitScalarExpr( in emitNumThreadsForTargetDirective()
7076 Bld.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned=*/false); in emitNumThreadsForTargetDirective()
7377 CodeGenFunction &CGF; member in __anond842882b2d11::MappableExprsHandler
7400 CGF.getTypeSize(OAE->getBase()->getType()->getPointeeType()); in getExprTypeSize()
7402 llvm::Value *Sz = CGF.EmitScalarExpr(SE); in getExprTypeSize()
7403 Sz = CGF.EmitScalarConversion(Sz, SE->getType(), in getExprTypeSize()
7404 CGF.getContext().getSizeType(), in getExprTypeSize()
7406 Size = CGF.Builder.CreateNUWMul(Size, Sz); in getExprTypeSize()
7428 return CGF.getTypeSize(BaseTy); in getExprTypeSize()
7432 ElemSize = CGF.getTypeSize(PTy->getPointeeType().getCanonicalType()); in getExprTypeSize()
7436 ElemSize = CGF.getTypeSize(ATy->getElementType().getCanonicalType()); in getExprTypeSize()
7445 llvm::Value *LengthVal = CGF.EmitScalarExpr(LenExpr); in getExprTypeSize()
7446 LengthVal = CGF.EmitScalarConversion(LengthVal, LenExpr->getType(), in getExprTypeSize()
7447 CGF.getContext().getSizeType(), in getExprTypeSize()
7449 return CGF.Builder.CreateNUWMul(LengthVal, ElemSize); in getExprTypeSize()
7454 llvm::Value *LengthVal = CGF.getTypeSize(BaseTy); in getExprTypeSize()
7455 llvm::Value *LBVal = CGF.EmitScalarExpr(OAE->getLowerBound()); in getExprTypeSize()
7456 LBVal = CGF.EmitScalarConversion(LBVal, OAE->getLowerBound()->getType(), in getExprTypeSize()
7457 CGF.getContext().getSizeType(), in getExprTypeSize()
7459 LBVal = CGF.Builder.CreateNUWMul(LBVal, ElemSize); in getExprTypeSize()
7460 llvm::Value *Cmp = CGF.Builder.CreateICmpUGT(LengthVal, LBVal); in getExprTypeSize()
7461 llvm::Value *TrueVal = CGF.Builder.CreateNUWSub(LengthVal, LBVal); in getExprTypeSize()
7462 LengthVal = CGF.Builder.CreateSelect( in getExprTypeSize()
7463 Cmp, TrueVal, llvm::ConstantInt::get(CGF.SizeTy, 0)); in getExprTypeSize()
7466 return CGF.getTypeSize(ExprTy); in getExprTypeSize()
7552 if (!Length->EvaluateAsInt(Result, CGF.getContext())) in isFinalArraySectionExpression()
7766 BP = CGF.LoadCXXThisAddress(); in generateInfoForComponentList()
7770 BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress(CGF); in generateInfoForComponentList()
7774 CGF.EmitScalarExpr(OAShE->getBase()), in generateInfoForComponentList()
7775 CGF.ConvertTypeForMem(OAShE->getBase()->getType()->getPointeeType()), in generateInfoForComponentList()
7776 CGF.getContext().getTypeAlignInChars(OAShE->getBase()->getType())); in generateInfoForComponentList()
7780 BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress(CGF); in generateInfoForComponentList()
7787 CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) { in generateInfoForComponentList()
7789 BP = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD); in generateInfoForComponentList()
7804 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() || in generateInfoForComponentList()
7806 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>()); in generateInfoForComponentList()
7860 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>()); in generateInfoForComponentList()
7926 auto &&EmitMemberExprBase = [](CodeGenFunction &CGF, in generateInfoForComponentList()
7936 CGF.EmitPointerWithAlignment(BaseExpr, &BaseInfo, &TBAAInfo); in generateInfoForComponentList()
7938 BaseLV = CGF.MakeAddrLValue(Addr, PtrTy, BaseInfo, TBAAInfo); in generateInfoForComponentList()
7940 BaseLV = CGF.EmitOMPSharedLValue(BaseExpr); in generateInfoForComponentList()
7946 Address(CGF.EmitScalarExpr(OAShE->getBase()), in generateInfoForComponentList()
7947 CGF.ConvertTypeForMem( in generateInfoForComponentList()
7949 CGF.getContext().getTypeAlignInChars( in generateInfoForComponentList()
7953 LValue BaseLVal = EmitMemberExprBase(CGF, ME); in generateInfoForComponentList()
7954 LowestElem = CGF.EmitLValueForFieldInitialization( in generateInfoForComponentList()
7956 .getAddress(CGF); in generateInfoForComponentList()
7957 LB = CGF.EmitLoadOfReferenceLValue(LowestElem, MapDecl->getType()) in generateInfoForComponentList()
7958 .getAddress(CGF); in generateInfoForComponentList()
7961 CGF.EmitOMPSharedLValue(I->getAssociatedExpression()) in generateInfoForComponentList()
7962 .getAddress(CGF); in generateInfoForComponentList()
7983 CharUnits TypeSize = CGF.getContext().getTypeSizeInChars( in generateInfoForComponentList()
7985 Address HB = CGF.Builder.CreateConstGEP( in generateInfoForComponentList()
7986 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in generateInfoForComponentList()
7987 LowestElem, CGF.VoidPtrTy, CGF.Int8Ty), in generateInfoForComponentList()
8017 LValue BaseLVal = EmitMemberExprBase(CGF, ME); in generateInfoForComponentList()
8019 CGF.EmitLValueForFieldInitialization(BaseLVal, FD) in generateInfoForComponentList()
8020 .getAddress(CGF); in generateInfoForComponentList()
8023 CGF.EmitOMPSharedLValue(MC.getAssociatedExpression()) in generateInfoForComponentList()
8024 .getAddress(CGF); in generateInfoForComponentList()
8026 Size = CGF.Builder.CreatePtrDiff( in generateInfoForComponentList()
8027 CGF.Int8Ty, CGF.EmitCastToVoidPtr(ComponentLB.getPointer()), in generateInfoForComponentList()
8028 CGF.EmitCastToVoidPtr(LB.getPointer())); in generateInfoForComponentList()
8036 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateInfoForComponentList()
8037 Size, CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForComponentList()
8042 LB = CGF.Builder.CreateConstGEP(ComponentLB, 1); in generateInfoForComponentList()
8047 Size = CGF.Builder.CreatePtrDiff( in generateInfoForComponentList()
8048 CGF.Int8Ty, CGF.Builder.CreateConstGEP(HB, 1).getPointer(), in generateInfoForComponentList()
8049 CGF.EmitCastToVoidPtr(LB.getPointer())); in generateInfoForComponentList()
8051 CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForComponentList()
8065 CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForComponentList()
8115 CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false) in generateInfoForComponentList()
8116 .getAddress(CGF); in generateInfoForComponentList()
8151 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>()); in generateInfoForComponentList()
8163 const ASTContext &Context = CGF.getContext(); in generateInfoForComponentList()
8167 MapValuesArrayTy CurOffsets = {llvm::ConstantInt::get(CGF.CGM.Int64Ty, 0)}; in generateInfoForComponentList()
8168 MapValuesArrayTy CurCounts = {llvm::ConstantInt::get(CGF.CGM.Int64Ty, 1)}; in generateInfoForComponentList()
8170 MapValuesArrayTy DimSizes{llvm::ConstantInt::get(CGF.CGM.Int64Ty, 1)}; in generateInfoForComponentList()
8215 llvm::ConstantInt::get(CGF.Int64Ty, ElementTypeSize)); in generateInfoForComponentList()
8223 CGF.Int64Ty, CAT->getSize().getZExtValue())); in generateInfoForComponentList()
8225 DimSizes.push_back(CGF.Builder.CreateIntCast( in generateInfoForComponentList()
8226 CGF.EmitScalarExpr(VAT->getSizeExpr()), CGF.Int64Ty, in generateInfoForComponentList()
8235 llvm::ConstantInt::get(CGF.CGM.Int64Ty, ElementTypeSize); in generateInfoForComponentList()
8249 llvm::Value *Offset = CGF.Builder.CreateIntCast( in generateInfoForComponentList()
8250 CGF.EmitScalarExpr(AE->getIdx()), CGF.Int64Ty, in generateInfoForComponentList()
8253 CurCounts.push_back(llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/1)); in generateInfoForComponentList()
8268 Offset = llvm::ConstantInt::get(CGF.Int64Ty, 0); in generateInfoForComponentList()
8270 Offset = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(OffsetExpr), in generateInfoForComponentList()
8271 CGF.Int64Ty, in generateInfoForComponentList()
8286 Count = llvm::ConstantInt::get(CGF.Int64Ty, 1); in generateInfoForComponentList()
8295 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(StrideExpr), in generateInfoForComponentList()
8296 CGF.Int64Ty, /*isSigned=*/false) in generateInfoForComponentList()
8299 Count = CGF.Builder.CreateUDiv( in generateInfoForComponentList()
8300 CGF.Builder.CreateNUWSub(*DI, Offset), Stride); in generateInfoForComponentList()
8302 Count = CGF.Builder.CreateNUWSub(*DI, Offset); in generateInfoForComponentList()
8305 Count = CGF.EmitScalarExpr(CountExpr); in generateInfoForComponentList()
8307 Count = CGF.Builder.CreateIntCast(Count, CGF.Int64Ty, /*isSigned=*/false); in generateInfoForComponentList()
8320 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(StrideExpr), in generateInfoForComponentList()
8321 CGF.Int64Ty, /*isSigned=*/false) in generateInfoForComponentList()
8323 DimProd = CGF.Builder.CreateNUWMul(DimProd, *(DI - 1)); in generateInfoForComponentList()
8325 CurStrides.push_back(CGF.Builder.CreateNUWMul(DimProd, Stride)); in generateInfoForComponentList()
8391 const CGRecordLayout &RL = CGF.getTypes().getCGRecordLayout(RD); in getPlainLayout()
8407 if (Base->isEmpty() || CGF.getContext() in getPlainLayout()
8432 if (!Field->isBitField() && !Field->isZeroSize(CGF.getContext())) { in getPlainLayout()
8587 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() || in generateAllInfoForClauses()
8620 CGF.EmitLoadOfScalar(CGF.EmitLValue(IE), IE->getExprLoc()); in generateAllInfoForClauses()
8625 llvm::Constant::getNullValue(CGF.Int64Ty)); in generateAllInfoForClauses()
8694 Ptr = CGF.EmitLValue(IE).getPointer(CGF); in generateAllInfoForClauses()
8696 Ptr = CGF.EmitScalarExpr(IE); in generateAllInfoForClauses()
8701 llvm::Constant::getNullValue(CGF.Int64Ty)); in generateAllInfoForClauses()
8755 Ptr = this->CGF.EmitLValue(L.IE).getPointer(CGF); in generateAllInfoForClauses()
8757 Ptr = this->CGF.EmitScalarExpr(L.IE); in generateAllInfoForClauses()
8764 BasePtr = this->CGF.EmitLValue(L.IE).getPointer(CGF); in generateAllInfoForClauses()
8765 Ptr = this->CGF.EmitLoadOfScalar(this->CGF.EmitLValue(L.IE), in generateAllInfoForClauses()
8777 llvm::Constant::getNullValue(this->CGF.Int64Ty)); in generateAllInfoForClauses()
8797 MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF) in MappableExprsHandler() argument
8798 : CurDir(&Dir), CGF(CGF) { in MappableExprsHandler()
8839 MappableExprsHandler(const OMPDeclareMapperDecl &Dir, CodeGenFunction &CGF) in MappableExprsHandler() argument
8840 : CurDir(&Dir), CGF(CGF) {} in MappableExprsHandler()
8871 CGF.Builder.CreateConstGEP1_32(HBAddr.getElementType(), HB, /*Idx0=*/1); in emitCombinedEntry()
8872 llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(LB, CGF.VoidPtrTy); in emitCombinedEntry()
8873 llvm::Value *CHAddr = CGF.Builder.CreatePointerCast(HAddr, CGF.VoidPtrTy); in emitCombinedEntry()
8874 llvm::Value *Diff = CGF.Builder.CreatePtrDiff(CGF.Int8Ty, CHAddr, CLAddr); in emitCombinedEntry()
8875 llvm::Value *Size = CGF.Builder.CreateIntCast(Diff, CGF.Int64Ty, in emitCombinedEntry()
8947 Address VDAddr(Arg, CGF.ConvertTypeForMem(VDType), in generateInfoForLambdaCaptures()
8948 CGF.getContext().getDeclAlign(VD)); in generateInfoForLambdaCaptures()
8949 LValue VDLVal = CGF.MakeAddrLValue(VDAddr, VDType); in generateInfoForLambdaCaptures()
8955 CGF.EmitLValueForFieldInitialization(VDLVal, ThisCapture); in generateInfoForLambdaCaptures()
8956 LValue ThisLValVal = CGF.EmitLValueForField(VDLVal, ThisCapture); in generateInfoForLambdaCaptures()
8957 LambdaPointers.try_emplace(ThisLVal.getPointer(CGF), in generateInfoForLambdaCaptures()
8958 VDLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8960 CombinedInfo.BasePointers.push_back(ThisLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8961 CombinedInfo.Pointers.push_back(ThisLValVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8963 CGF.Builder.CreateIntCast(CGF.getTypeSize(CGF.getContext().VoidPtrTy), in generateInfoForLambdaCaptures()
8964 CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForLambdaCaptures()
8977 LValue VarLVal = CGF.EmitLValueForFieldInitialization(VDLVal, It->second); in generateInfoForLambdaCaptures()
8979 LValue VarLValVal = CGF.EmitLValueForField(VDLVal, It->second); in generateInfoForLambdaCaptures()
8980 LambdaPointers.try_emplace(VarLVal.getPointer(CGF), in generateInfoForLambdaCaptures()
8981 VDLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8983 CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8984 CombinedInfo.Pointers.push_back(VarLValVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8985 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateInfoForLambdaCaptures()
8986 CGF.getTypeSize( in generateInfoForLambdaCaptures()
8988 CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForLambdaCaptures()
8990 RValue VarRVal = CGF.EmitLoadOfLValue(VarLVal, RD->getLocation()); in generateInfoForLambdaCaptures()
8991 LambdaPointers.try_emplace(VarLVal.getPointer(CGF), in generateInfoForLambdaCaptures()
8992 VDLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8994 CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8996 CombinedInfo.Sizes.push_back(llvm::ConstantInt::get(CGF.Int64Ty, 0)); in generateInfoForLambdaCaptures()
9058 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateInfoForCapture()
9059 CGF.getTypeSize(CGF.getContext().VoidPtrTy), CGF.Int64Ty, in generateInfoForCapture()
9284 CGF.Builder.CreateIntCast(CGF.getTypeSize(PtrTy->getPointeeType()), in generateDefaultMapInfo()
9285 CGF.Int64Ty, /*isSigned=*/true)); in generateDefaultMapInfo()
9297 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateDefaultMapInfo()
9298 CGF.getTypeSize(RI.getType()), CGF.Int64Ty, /*isSigned=*/true)); in generateDefaultMapInfo()
9303 CombinedInfo.Sizes.push_back(llvm::Constant::getNullValue(CGF.Int64Ty)); in generateDefaultMapInfo()
9312 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateDefaultMapInfo()
9313 CGF.getTypeSize(ElementType), CGF.Int64Ty, /*isSigned=*/true)); in generateDefaultMapInfo()
9323 Address PtrAddr = CGF.EmitLoadOfReference(CGF.MakeAddrLValue( in generateDefaultMapInfo()
9324 CV, ElementType, CGF.getContext().getDeclAlign(VD), in generateDefaultMapInfo()
9347 CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo, in emitNonContiguousDescriptor() argument
9349 CodeGenModule &CGM = CGF.CGM; in emitNonContiguousDescriptor()
9361 ASTContext &C = CGF.getContext(); in emitNonContiguousDescriptor()
9384 Address DimsAddr = CGF.CreateMemTemp(ArrayTy, "dims"); in emitNonContiguousDescriptor()
9387 LValue DimsLVal = CGF.MakeAddrLValue( in emitNonContiguousDescriptor()
9388 CGF.Builder.CreateConstArrayGEP(DimsAddr, II), DimTy); in emitNonContiguousDescriptor()
9390 LValue OffsetLVal = CGF.EmitLValueForField( in emitNonContiguousDescriptor()
9392 CGF.EmitStoreOfScalar(NonContigInfo.Offsets[L][RevIdx], OffsetLVal); in emitNonContiguousDescriptor()
9394 LValue CountLVal = CGF.EmitLValueForField( in emitNonContiguousDescriptor()
9396 CGF.EmitStoreOfScalar(NonContigInfo.Counts[L][RevIdx], CountLVal); in emitNonContiguousDescriptor()
9398 LValue StrideLVal = CGF.EmitLValueForField( in emitNonContiguousDescriptor()
9400 CGF.EmitStoreOfScalar(NonContigInfo.Strides[L][RevIdx], StrideLVal); in emitNonContiguousDescriptor()
9403 Address DAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitNonContiguousDescriptor()
9405 llvm::Value *P = CGF.Builder.CreateConstInBoundsGEP2_32( in emitNonContiguousDescriptor()
9408 Address PAddr(P, CGM.VoidPtrTy, CGF.getPointerAlign()); in emitNonContiguousDescriptor()
9409 CGF.Builder.CreateStore(DAddr.getPointer(), PAddr); in emitNonContiguousDescriptor()
9429 emitMappingInformation(CodeGenFunction &CGF, llvm::OpenMPIRBuilder &OMPBuilder, in emitMappingInformation() argument
9448 PrintingPolicy P(CGF.getContext().getLangOpts()); in emitMappingInformation()
9456 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc); in emitMappingInformation()
9466 CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo, in emitOffloadingArrays() argument
9469 CodeGenModule &CGM = CGF.CGM; in emitOffloadingArrays()
9470 ASTContext &Ctx = CGF.getContext(); in emitOffloadingArrays()
9486 CGF.CreateMemTemp(PointerArrayType, ".offload_baseptrs").getPointer(); in emitOffloadingArrays()
9488 CGF.CreateMemTemp(PointerArrayType, ".offload_ptrs").getPointer(); in emitOffloadingArrays()
9490 CGF.CreateMemTemp(PointerArrayType, ".offload_mappers"); in emitOffloadingArrays()
9499 CombinedInfo.Sizes.size(), llvm::ConstantInt::get(CGF.Int64Ty, 0)); in emitOffloadingArrays()
9507 CGF.Int64Ty, CombinedInfo.NonContigInfo.Dims[I]); in emitOffloadingArrays()
9521 CGF.CreateMemTemp(SizeArrayType, ".offload_sizes").getPointer(); in emitOffloadingArrays()
9534 Address Buffer = CGF.CreateMemTemp(SizeArrayType, ".offload_sizes"); in emitOffloadingArrays()
9536 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitOffloadingArrays()
9538 CGF.Builder.CreateMemCpy( in emitOffloadingArrays()
9543 CGF.getTypeSize(SizeArrayType)); in emitOffloadingArrays()
9564 llvm::Type::getInt8Ty(CGF.Builder.getContext())->getPointerTo()); in emitOffloadingArrays()
9567 return emitMappingInformation(CGF, OMPBuilder, MapExpr); in emitOffloadingArrays()
9597 llvm::Value *BP = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArrays()
9600 BP = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitOffloadingArrays()
9604 CGF.Builder.CreateStore(BPVal, BPAddr); in emitOffloadingArrays()
9612 llvm::Value *P = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArrays()
9615 P = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitOffloadingArrays()
9618 CGF.Builder.CreateStore(PVal, PAddr); in emitOffloadingArrays()
9621 llvm::Value *S = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArrays()
9627 CGF.Builder.CreateStore(CGF.Builder.CreateIntCast(CombinedInfo.Sizes[I], in emitOffloadingArrays()
9638 MFunc = CGF.Builder.CreatePointerCast(MFunc, CGM.VoidPtrTy); in emitOffloadingArrays()
9641 Address MAddr = CGF.Builder.CreateConstArrayGEP(MappersArray, I); in emitOffloadingArrays()
9642 CGF.Builder.CreateStore(MFunc, MAddr); in emitOffloadingArrays()
9650 emitNonContiguousDescriptor(CGF, CombinedInfo, Info); in emitOffloadingArrays()
9667 CodeGenFunction &CGF, llvm::Value *&BasePointersArrayArg, in emitOffloadingArraysArgument() argument
9674 CodeGenModule &CGM = CGF.CGM; in emitOffloadingArraysArgument()
9676 BasePointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
9680 PointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
9685 SizesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
9688 MapTypesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
9697 if (CGF.CGM.getCodeGenOpts().getDebugInfo() == codegenoptions::NoDebugInfo) in emitOffloadingArraysArgument()
9700 MapNamesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
9711 CGF.Builder.CreatePointerCast(Info.MappersArray, CGM.VoidPtrPtrTy); in emitOffloadingArraysArgument()
9866 CodeGenFunction *CGF) { in emitUserDefinedMapper() argument
10107 if (CGF) { in emitUserDefinedMapper()
10108 auto &Decls = FunctionUDMMap.FindAndConstruct(CGF->CurFn); in emitUserDefinedMapper()
10191 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitTargetNumIterationsCall() argument
10192 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, in emitTargetNumIterationsCall() argument
10201 return llvm::ConstantInt::get(CGF.Int64Ty, 0); in emitTargetNumIterationsCall()
10204 if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD)) in emitTargetNumIterationsCall()
10206 return llvm::ConstantInt::get(CGF.Int64Ty, 0); in emitTargetNumIterationsCall()
10210 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitTargetCall() argument
10213 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, in emitTargetCall() argument
10216 if (!CGF.HaveInsertPoint()) in emitTargetCall()
10229 auto &&ArgsCodegen = [&CS, &CapturedVars](CodeGenFunction &CGF, in emitTargetCall()
10231 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars); in emitTargetCall()
10233 emitInlinedDirective(CGF, OMPD_unknown, ArgsCodegen); in emitTargetCall()
10240 &CS, OffloadingMandatory](CodeGenFunction &CGF) { in emitTargetCall() argument
10242 CGF.Builder.CreateUnreachable(); in emitTargetCall()
10246 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars); in emitTargetCall()
10248 emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn, CapturedVars); in emitTargetCall()
10254 FallbackGen](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetCall() argument
10257 FallbackGen(CGF); in emitTargetCall()
10280 llvm::Value *DevVal = CGF.EmitScalarExpr(Device.getPointer()); in emitTargetCall()
10282 CGF.Builder.CreateIntCast(DevVal, CGF.Int64Ty, /*isSigned=*/true); in emitTargetCall()
10284 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTargetCall()
10289 CGF.Builder.getInt32(InputInfo.NumberOfTargetItems); in emitTargetCall()
10294 llvm::Value *NumTeams = emitNumTeamsForTargetDirective(CGF, D); in emitTargetCall()
10295 llvm::Value *NumThreads = emitNumThreadsForTargetDirective(CGF, D); in emitTargetCall()
10298 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTargetCall()
10302 emitTargetNumIterationsCall(CGF, D, SizeEmitter); in emitTargetCall()
10306 CGF.Builder.getInt32(/* Version */ 1), in emitTargetCall()
10318 CGF.Builder.getInt32(0), in emitTargetCall()
10320 CGF.Builder.getInt32(0), in emitTargetCall()
10339 CGF.Builder.restoreIP( in emitTargetCall()
10341 CGF.Builder, Return, RTLoc, DeviceID, NumTeams, in emitTargetCall()
10343 : OMPBuilder.emitTargetKernel(CGF.Builder, Return, RTLoc, in emitTargetCall()
10348 CGF.createBasicBlock("omp_offload.failed"); in emitTargetCall()
10350 CGF.createBasicBlock("omp_offload.cont"); in emitTargetCall()
10351 llvm::Value *Failed = CGF.Builder.CreateIsNotNull(Return); in emitTargetCall()
10352 CGF.Builder.CreateCondBr(Failed, OffloadFailedBlock, OffloadContBlock); in emitTargetCall()
10354 CGF.EmitBlock(OffloadFailedBlock); in emitTargetCall()
10355 FallbackGen(CGF); in emitTargetCall()
10357 CGF.EmitBranch(OffloadContBlock); in emitTargetCall()
10359 CGF.EmitBlock(OffloadContBlock, /*IsFinished=*/true); in emitTargetCall()
10363 auto &&ElseGen = [FallbackGen](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetCall() argument
10364 FallbackGen(CGF); in emitTargetCall()
10369 &CS](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetCall() argument
10374 MappableExprsHandler MEHandler(D, CGF); in emitTargetCall()
10392 CurInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in emitTargetCall()
10393 CGF.getTypeSize(RI->getType()), CGF.Int64Ty, /*isSigned=*/true)); in emitTargetCall()
10446 emitOffloadingArrays(CGF, CombinedInfo, Info, OMPBuilder); in emitTargetCall()
10448 CGF, Info.BasePointersArray, Info.PointersArray, Info.SizesArray, in emitTargetCall()
10454 Address(Info.BasePointersArray, CGF.VoidPtrTy, CGM.getPointerAlign()); in emitTargetCall()
10456 Address(Info.PointersArray, CGF.VoidPtrTy, CGM.getPointerAlign()); in emitTargetCall()
10458 Address(Info.SizesArray, CGF.Int64Ty, CGM.getPointerAlign()); in emitTargetCall()
10460 Address(Info.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign()); in emitTargetCall()
10464 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo); in emitTargetCall()
10466 emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen); in emitTargetCall()
10470 CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetCall() argument
10473 CGF.EmitOMPTargetTaskBasedDirective(D, ElseGen, InputInfo); in emitTargetCall()
10475 emitInlinedDirective(CGF, D.getDirectiveKind(), ElseGen); in emitTargetCall()
10485 emitIfClause(CGF, IfCond, TargetThenGen, TargetElseGen); in emitTargetCall()
10488 ThenRCG(CGF); in emitTargetCall()
10492 ElseRCG(CGF); in emitTargetCall()
10835 CodeGenFunction &CGF, const OMPExecutableDirective &D) const { in adjustTargetSpecificDataForLambdas() argument
10945 CodeGenFunction CGF(CGM); in emitRequiresDirectiveRegFun() local
10950 CGF.StartFunction(GlobalDecl(), C.VoidTy, RequiresRegFn, FI, {}); in emitRequiresDirectiveRegFun()
10964 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitRequiresDirectiveRegFun()
10967 CGF.FinishFunction(); in emitRequiresDirectiveRegFun()
10972 void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF, in emitTeamsCall() argument
10977 if (!CGF.HaveInsertPoint()) in emitTeamsCall()
10980 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitTeamsCall()
10981 CodeGenFunction::RunCleanupsScope Scope(CGF); in emitTeamsCall()
10986 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars in emitTeamsCall()
10987 CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy())}; in emitTeamsCall()
10994 CGF.EmitRuntimeCall(RTLFn, RealArgs); in emitTeamsCall()
10997 void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF, in emitNumTeamsClause() argument
11001 if (!CGF.HaveInsertPoint()) in emitNumTeamsClause()
11004 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitNumTeamsClause()
11008 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(NumTeams), in emitNumTeamsClause()
11009 CGF.CGM.Int32Ty, /* isSigned = */ true) in emitNumTeamsClause()
11010 : CGF.Builder.getInt32(0); in emitNumTeamsClause()
11014 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(ThreadLimit), in emitNumTeamsClause()
11015 CGF.CGM.Int32Ty, /* isSigned = */ true) in emitNumTeamsClause()
11016 : CGF.Builder.getInt32(0); in emitNumTeamsClause()
11019 llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal, in emitNumTeamsClause()
11021 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitNumTeamsClause()
11027 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, in emitTargetDataCalls() argument
11029 if (!CGF.HaveInsertPoint()) in emitTargetDataCalls()
11040 &CodeGen](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetDataCalls() argument
11045 MappableExprsHandler MEHandler(D, CGF); in emitTargetDataCalls()
11049 emitOffloadingArrays(CGF, CombinedInfo, Info, OMPBuilder, in emitTargetDataCalls()
11058 emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg, in emitTargetDataCalls()
11065 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTargetDataCalls()
11066 CGF.Int64Ty, /*isSigned=*/true); in emitTargetDataCalls()
11068 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTargetDataCalls()
11072 llvm::Value *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs); in emitTargetDataCalls()
11075 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTargetDataCalls()
11086 CGF.EmitRuntimeCall( in emitTargetDataCalls()
11094 CodeGen(CGF); in emitTargetDataCalls()
11098 auto &&EndThenGen = [this, Device, &Info, &D](CodeGenFunction &CGF, in emitTargetDataCalls()
11108 emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg, in emitTargetDataCalls()
11116 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTargetDataCalls()
11117 CGF.Int64Ty, /*isSigned=*/true); in emitTargetDataCalls()
11119 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTargetDataCalls()
11123 llvm::Value *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs); in emitTargetDataCalls()
11126 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTargetDataCalls()
11137 CGF.EmitRuntimeCall( in emitTargetDataCalls()
11146 auto &&BeginElseGen = [&Info, &CodeGen, &NoPrivAction](CodeGenFunction &CGF, in emitTargetDataCalls()
11150 CodeGen(CGF); in emitTargetDataCalls()
11156 auto &&EndElseGen = [](CodeGenFunction &CGF, PrePostActionTy &) {}; in emitTargetDataCalls() argument
11159 emitIfClause(CGF, IfCond, BeginThenGen, BeginElseGen); in emitTargetDataCalls()
11162 RCG(CGF); in emitTargetDataCalls()
11169 CodeGen(CGF); in emitTargetDataCalls()
11173 emitIfClause(CGF, IfCond, EndThenGen, EndElseGen); in emitTargetDataCalls()
11176 RCG(CGF); in emitTargetDataCalls()
11181 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, in emitTargetDataStandAloneCall() argument
11183 if (!CGF.HaveInsertPoint()) in emitTargetDataStandAloneCall()
11196 &MapNamesArray](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetDataStandAloneCall() argument
11200 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTargetDataStandAloneCall()
11201 CGF.Int64Ty, /*isSigned=*/true); in emitTargetDataStandAloneCall()
11203 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTargetDataStandAloneCall()
11208 CGF.Builder.getInt32(InputInfo.NumberOfTargetItems); in emitTargetDataStandAloneCall()
11211 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTargetDataStandAloneCall()
11310 CGF.EmitRuntimeCall( in emitTargetDataStandAloneCall()
11316 &MapNamesArray](CodeGenFunction &CGF, in emitTargetDataStandAloneCall()
11322 MappableExprsHandler MEHandler(D, CGF); in emitTargetDataStandAloneCall()
11327 emitOffloadingArrays(CGF, CombinedInfo, Info, OMPBuilder, in emitTargetDataStandAloneCall()
11332 CGF, Info.BasePointersArray, Info.PointersArray, Info.SizesArray, in emitTargetDataStandAloneCall()
11337 Address(Info.BasePointersArray, CGF.VoidPtrTy, CGM.getPointerAlign()); in emitTargetDataStandAloneCall()
11339 Address(Info.PointersArray, CGF.VoidPtrTy, CGM.getPointerAlign()); in emitTargetDataStandAloneCall()
11341 Address(Info.SizesArray, CGF.Int64Ty, CGM.getPointerAlign()); in emitTargetDataStandAloneCall()
11343 Address(Info.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign()); in emitTargetDataStandAloneCall()
11347 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo); in emitTargetDataStandAloneCall()
11349 emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen); in emitTargetDataStandAloneCall()
11353 emitIfClause(CGF, IfCond, TargetThenGen, in emitTargetDataStandAloneCall()
11354 [](CodeGenFunction &CGF, PrePostActionTy &) {}); in emitTargetDataStandAloneCall() argument
11357 ThenRCG(CGF); in emitTargetDataStandAloneCall()
11966 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { in Emit() argument
11967 if (!CGF.HaveInsertPoint()) in Emit()
11969 CGF.EmitRuntimeCall(RTLFn, Args); in Emit()
11974 void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF, in emitDoacrossInit() argument
11977 if (!CGF.HaveInsertPoint()) in emitDoacrossInit()
12003 Address DimsAddr = CGF.CreateMemTemp(ArrayTy, "dims"); in emitDoacrossInit()
12004 CGF.EmitNullInitialization(DimsAddr, ArrayTy); in emitDoacrossInit()
12008 LValue DimsLVal = CGF.MakeAddrLValue( in emitDoacrossInit()
12009 CGF.Builder.CreateConstArrayGEP(DimsAddr, I), KmpDimTy); in emitDoacrossInit()
12011 LValue UpperLVal = CGF.EmitLValueForField( in emitDoacrossInit()
12013 llvm::Value *NumIterVal = CGF.EmitScalarConversion( in emitDoacrossInit()
12014 CGF.EmitScalarExpr(NumIterations[I]), NumIterations[I]->getType(), in emitDoacrossInit()
12016 CGF.EmitStoreOfScalar(NumIterVal, UpperLVal); in emitDoacrossInit()
12018 LValue StrideLVal = CGF.EmitLValueForField( in emitDoacrossInit()
12020 CGF.EmitStoreOfScalar(llvm::ConstantInt::getSigned(CGM.Int64Ty, /*V=*/1), in emitDoacrossInit()
12027 emitUpdateLocation(CGF, D.getBeginLoc()), in emitDoacrossInit()
12028 getThreadID(CGF, D.getBeginLoc()), in emitDoacrossInit()
12030 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDoacrossInit()
12031 CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).getPointer(), in emitDoacrossInit()
12036 CGF.EmitRuntimeCall(RTLFn, Args); in emitDoacrossInit()
12038 emitUpdateLocation(CGF, D.getEndLoc()), getThreadID(CGF, D.getEndLoc())}; in emitDoacrossInit()
12041 CGF.EHStack.pushCleanup<DoacrossCleanupTy>(NormalAndEHCleanup, FiniRTLFn, in emitDoacrossInit()
12045 void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, in emitDoacrossOrdered() argument
12052 Address CntAddr = CGF.CreateMemTemp(ArrayTy, ".cnt.addr"); in emitDoacrossOrdered()
12056 llvm::Value *CntVal = CGF.EmitScalarConversion( in emitDoacrossOrdered()
12057 CGF.EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty, in emitDoacrossOrdered()
12059 CGF.EmitStoreOfScalar(CntVal, CGF.Builder.CreateConstArrayGEP(CntAddr, I), in emitDoacrossOrdered()
12063 emitUpdateLocation(CGF, C->getBeginLoc()), in emitDoacrossOrdered()
12064 getThreadID(CGF, C->getBeginLoc()), in emitDoacrossOrdered()
12065 CGF.Builder.CreateConstArrayGEP(CntAddr, 0).getPointer()}; in emitDoacrossOrdered()
12075 CGF.EmitRuntimeCall(RTLFn, Args); in emitDoacrossOrdered()
12078 void CGOpenMPRuntime::emitCall(CodeGenFunction &CGF, SourceLocation Loc, in emitCall() argument
12082 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc); in emitCall()
12086 CGF.EmitNounwindRuntimeCall(Fn, Args); in emitCall()
12090 CGF.EmitRuntimeCall(Callee, Args); in emitCall()
12094 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, in emitOutlinedFunctionCall() argument
12096 emitCall(CGF, Loc, OutlinedFn, Args); in emitOutlinedFunctionCall()
12099 void CGOpenMPRuntime::emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) { in emitFunctionProlog() argument
12105 Address CGOpenMPRuntime::getParameterAddress(CodeGenFunction &CGF, in getParameterAddress() argument
12108 return CGF.GetAddrOfLocalVar(NativeParam); in getParameterAddress()
12113 static llvm::Value *getAllocatorVal(CodeGenFunction &CGF, in getAllocatorVal() argument
12117 AllocVal = CGF.EmitScalarExpr(Allocator); in getAllocatorVal()
12120 AllocVal = CGF.EmitScalarConversion(AllocVal, Allocator->getType(), in getAllocatorVal()
12121 CGF.getContext().VoidPtrTy, in getAllocatorVal()
12126 CGF.CGM.getTypes().ConvertType(CGF.getContext().VoidPtrTy)); in getAllocatorVal()
12141 Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF, in getAddressOfLocalVariable() argument
12147 auto It = FunctionToUntiedTaskStackMap.find(CGF.CurFn); in getAddressOfLocalVariable()
12165 Size = CGF.getTypeSize(CVD->getType()); in getAddressOfLocalVariable()
12167 Size = CGF.Builder.CreateNUWAdd( in getAddressOfLocalVariable()
12169 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align)); in getAddressOfLocalVariable()
12170 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align)); in getAddressOfLocalVariable()
12175 llvm::Value *ThreadID = getThreadID(CGF, CVD->getBeginLoc()); in getAddressOfLocalVariable()
12178 llvm::Value *AllocVal = getAllocatorVal(CGF, Allocator); in getAddressOfLocalVariable()
12188 llvm::Value *Addr = CGF.EmitRuntimeCall( in getAddressOfLocalVariable()
12194 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddressOfLocalVariable()
12195 Addr, CGF.ConvertTypeForMem(Ty), getName({CVD->getName(), ".addr"})); in getAddressOfLocalVariable()
12197 CGF.EmitStoreOfScalar(Addr, UntiedAddr, /*Volatile=*/false, Ty); in getAddressOfLocalVariable()
12212 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { in getAddressOfLocalVariable() argument
12213 if (!CGF.HaveInsertPoint()) in getAddressOfLocalVariable()
12216 Args[0] = CGF.CGM.getOpenMPRuntime().getThreadID( in getAddressOfLocalVariable()
12217 CGF, SourceLocation::getFromRawEncoding(LocEncoding)); in getAddressOfLocalVariable()
12218 Args[1] = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddressOfLocalVariable()
12219 Addr.getPointer(), CGF.VoidPtrTy); in getAddressOfLocalVariable()
12220 llvm::Value *AllocVal = getAllocatorVal(CGF, AllocExpr); in getAddressOfLocalVariable()
12222 CGF.EmitRuntimeCall(RTLFn, Args); in getAddressOfLocalVariable()
12228 : Address(Addr, CGF.ConvertTypeForMem(CVD->getType()), Align); in getAddressOfLocalVariable()
12229 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>( in getAddressOfLocalVariable()
12234 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in getAddressOfLocalVariable()
12235 Region->emitUntiedSwitch(CGF); in getAddressOfLocalVariable()
12241 bool CGOpenMPRuntime::isLocalVarInUntiedTask(CodeGenFunction &CGF, in isLocalVarInUntiedTask() argument
12243 auto It = FunctionToUntiedTaskStackMap.find(CGF.CurFn); in isLocalVarInUntiedTask()
12282 CodeGenFunction &CGF, in UntiedTaskLocalDeclsRAII() argument
12285 : CGM(CGF.CGM), NeedToPush(!LocalVars.empty()) { in UntiedTaskLocalDeclsRAII()
12289 CGF.CurFn, CGM.getOpenMPRuntime().UntiedLocalVarsStack.size()); in UntiedTaskLocalDeclsRAII()
12387 CodeGenFunction &CGF, const OMPExecutableDirective &S, LValue IVLVal) in LastprivateConditionalRAII() argument
12388 : CGM(CGF.CGM), in LastprivateConditionalRAII()
12415 Data.Fn = CGF.CurFn;
12419 CodeGenFunction &CGF, const OMPExecutableDirective &S) in LastprivateConditionalRAII() argument
12420 : CGM(CGF.CGM), Action(ActionToDo::DoNotPush) { in LastprivateConditionalRAII()
12432 Data.Fn = CGF.CurFn; in LastprivateConditionalRAII()
12439 CodeGenFunction &CGF, const OMPExecutableDirective &S) { in disable() argument
12440 return LastprivateConditionalRAII(CGF, S); in disable()
12459 Address CGOpenMPRuntime::emitLastprivateConditionalInit(CodeGenFunction &CGF, in emitLastprivateConditionalInit() argument
12462 auto I = LastprivateConditionalToTypes.find(CGF.CurFn); in emitLastprivateConditionalInit()
12464 I = LastprivateConditionalToTypes.try_emplace(CGF.CurFn).first; in emitLastprivateConditionalInit()
12477 Address Addr = CGF.CreateMemTemp(NewType, C.getDeclAlign(VD), VD->getName()); in emitLastprivateConditionalInit()
12478 BaseLVal = CGF.MakeAddrLValue(Addr, NewType, AlignmentSource::Decl); in emitLastprivateConditionalInit()
12487 CGF.EmitLValueForField(BaseLVal, FiredField); in emitLastprivateConditionalInit()
12488 CGF.EmitStoreOfScalar( in emitLastprivateConditionalInit()
12489 llvm::ConstantInt::getNullValue(CGF.ConvertTypeForMem(C.CharTy)), in emitLastprivateConditionalInit()
12491 return CGF.EmitLValueForField(BaseLVal, VDField).getAddress(CGF); in emitLastprivateConditionalInit()
12565 void CGOpenMPRuntime::emitLastprivateConditionalUpdate(CodeGenFunction &CGF, in emitLastprivateConditionalUpdate() argument
12572 llvm::Type *LLIVTy = CGF.ConvertTypeForMem(IVLVal.getType()); in emitLastprivateConditionalUpdate()
12577 LValue LastIVLVal = CGF.MakeNaturalAlignAddrLValue(LastIV, IVLVal.getType()); in emitLastprivateConditionalUpdate()
12582 CGF.ConvertTypeForMem(LVal.getType()), UniqueDeclName); in emitLastprivateConditionalUpdate()
12584 LValue LastLVal = CGF.MakeAddrLValue( in emitLastprivateConditionalUpdate()
12589 llvm::Value *IVVal = CGF.EmitLoadOfScalar(IVLVal, Loc); in emitLastprivateConditionalUpdate()
12597 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitLastprivateConditionalUpdate() argument
12598 Action.Enter(CGF); in emitLastprivateConditionalUpdate()
12599 llvm::Value *LastIVVal = CGF.EmitLoadOfScalar(LastIVLVal, Loc); in emitLastprivateConditionalUpdate()
12604 CmpRes = CGF.Builder.CreateICmpSLE(LastIVVal, IVVal); in emitLastprivateConditionalUpdate()
12608 CmpRes = CGF.Builder.CreateICmpULE(LastIVVal, IVVal); in emitLastprivateConditionalUpdate()
12610 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("lp_cond_then"); in emitLastprivateConditionalUpdate()
12611 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("lp_cond_exit"); in emitLastprivateConditionalUpdate()
12612 CGF.Builder.CreateCondBr(CmpRes, ThenBB, ExitBB); in emitLastprivateConditionalUpdate()
12614 CGF.EmitBlock(ThenBB); in emitLastprivateConditionalUpdate()
12617 CGF.EmitStoreOfScalar(IVVal, LastIVLVal); in emitLastprivateConditionalUpdate()
12620 switch (CGF.getEvaluationKind(LVal.getType())) { in emitLastprivateConditionalUpdate()
12622 llvm::Value *PrivVal = CGF.EmitLoadOfScalar(LVal, Loc); in emitLastprivateConditionalUpdate()
12623 CGF.EmitStoreOfScalar(PrivVal, LastLVal); in emitLastprivateConditionalUpdate()
12627 CodeGenFunction::ComplexPairTy PrivVal = CGF.EmitLoadOfComplex(LVal, Loc); in emitLastprivateConditionalUpdate()
12628 CGF.EmitStoreOfComplex(PrivVal, LastLVal, /*isInit=*/false); in emitLastprivateConditionalUpdate()
12636 CGF.EmitBranch(ExitBB); in emitLastprivateConditionalUpdate()
12638 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitLastprivateConditionalUpdate()
12639 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in emitLastprivateConditionalUpdate()
12645 ThenRCG(CGF); in emitLastprivateConditionalUpdate()
12647 emitCriticalRegion(CGF, UniqueDeclName, CodeGen, Loc); in emitLastprivateConditionalUpdate()
12651 void CGOpenMPRuntime::checkAndEmitLastprivateConditional(CodeGenFunction &CGF, in checkAndEmitLastprivateConditional() argument
12653 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty()) in checkAndEmitLastprivateConditional()
12665 if (FoundFn != CGF.CurFn) { in checkAndEmitLastprivateConditional()
12673 LValue PrivLVal = CGF.EmitLValue(FoundE); in checkAndEmitLastprivateConditional()
12674 Address StructAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in checkAndEmitLastprivateConditional()
12675 PrivLVal.getAddress(CGF), in checkAndEmitLastprivateConditional()
12676 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(StructTy)), in checkAndEmitLastprivateConditional()
12677 CGF.ConvertTypeForMem(StructTy)); in checkAndEmitLastprivateConditional()
12679 CGF.MakeAddrLValue(StructAddr, StructTy, AlignmentSource::Decl); in checkAndEmitLastprivateConditional()
12680 LValue FiredLVal = CGF.EmitLValueForField(BaseLVal, FiredDecl); in checkAndEmitLastprivateConditional()
12681 CGF.EmitAtomicStore(RValue::get(llvm::ConstantInt::get( in checkAndEmitLastprivateConditional()
12682 CGF.ConvertTypeForMem(FiredDecl->getType()), 1)), in checkAndEmitLastprivateConditional()
12690 LValue LVal = CGF.EmitLValue(FoundE); in checkAndEmitLastprivateConditional()
12691 emitLastprivateConditionalUpdate(CGF, IVLVal, UniqueDeclName, LVal, in checkAndEmitLastprivateConditional()
12696 CodeGenFunction &CGF, const OMPExecutableDirective &D, in checkAndEmitSharedLastprivateConditional() argument
12698 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty()) in checkAndEmitSharedLastprivateConditional()
12703 if (It == Range.end() || It->Fn != CGF.CurFn) in checkAndEmitSharedLastprivateConditional()
12721 CGF.EmitLValueForField(BaseLVal, std::get<2>(I->getSecond())); in checkAndEmitSharedLastprivateConditional()
12722 llvm::Value *Res = CGF.EmitLoadOfScalar(FiredLVal, D.getBeginLoc()); in checkAndEmitSharedLastprivateConditional()
12723 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Res); in checkAndEmitSharedLastprivateConditional()
12724 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("lpc.then"); in checkAndEmitSharedLastprivateConditional()
12725 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("lpc.done"); in checkAndEmitSharedLastprivateConditional()
12727 CGF.Builder.CreateCondBr(Cmp, ThenBB, DoneBB); in checkAndEmitSharedLastprivateConditional()
12728 CGF.EmitBlock(ThenBB); in checkAndEmitSharedLastprivateConditional()
12729 Address Addr = CGF.GetAddrOfLocalVar(VD); in checkAndEmitSharedLastprivateConditional()
12732 LVal = CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(), in checkAndEmitSharedLastprivateConditional()
12735 LVal = CGF.MakeAddrLValue(Addr, VD->getType().getNonReferenceType(), in checkAndEmitSharedLastprivateConditional()
12737 emitLastprivateConditionalUpdate(CGF, It->IVLVal, Pair.second, LVal, in checkAndEmitSharedLastprivateConditional()
12739 auto AL = ApplyDebugLocation::CreateArtificial(CGF); in checkAndEmitSharedLastprivateConditional()
12740 CGF.EmitBlock(DoneBB, /*IsFinal=*/true); in checkAndEmitSharedLastprivateConditional()
12746 CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD, in emitLastprivateConditionalFinalUpdate() argument
12748 if (CGF.getLangOpts().OpenMP < 50) in emitLastprivateConditionalFinalUpdate()
12758 LValue LPLVal = CGF.MakeAddrLValue( in emitLastprivateConditionalFinalUpdate()
12761 llvm::Value *Res = CGF.EmitLoadOfScalar(LPLVal, Loc); in emitLastprivateConditionalFinalUpdate()
12762 CGF.EmitStoreOfScalar(Res, PrivLVal); in emitLastprivateConditionalFinalUpdate()
12785 void CGOpenMPSIMDRuntime::emitParallelCall(CodeGenFunction &CGF, in emitParallelCall() argument
12795 CodeGenFunction &CGF, StringRef CriticalName, in emitCriticalRegion() argument
12801 void CGOpenMPSIMDRuntime::emitMasterRegion(CodeGenFunction &CGF, in emitMasterRegion() argument
12807 void CGOpenMPSIMDRuntime::emitMaskedRegion(CodeGenFunction &CGF, in emitMaskedRegion() argument
12814 void CGOpenMPSIMDRuntime::emitTaskyieldCall(CodeGenFunction &CGF, in emitTaskyieldCall() argument
12820 CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, in emitTaskgroupRegion() argument
12826 CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, in emitSingleRegion() argument
12833 void CGOpenMPSIMDRuntime::emitOrderedRegion(CodeGenFunction &CGF, in emitOrderedRegion() argument
12840 void CGOpenMPSIMDRuntime::emitBarrierCall(CodeGenFunction &CGF, in emitBarrierCall() argument
12849 CodeGenFunction &CGF, SourceLocation Loc, in emitForDispatchInit() argument
12856 CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, in emitForStaticInit() argument
12862 CodeGenFunction &CGF, SourceLocation Loc, in emitDistributeStaticInit() argument
12867 void CGOpenMPSIMDRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF, in emitForOrderedIterationEnd() argument
12874 void CGOpenMPSIMDRuntime::emitForStaticFinish(CodeGenFunction &CGF, in emitForStaticFinish() argument
12880 llvm::Value *CGOpenMPSIMDRuntime::emitForNext(CodeGenFunction &CGF, in emitForNext() argument
12888 void CGOpenMPSIMDRuntime::emitNumThreadsClause(CodeGenFunction &CGF, in emitNumThreadsClause() argument
12894 void CGOpenMPSIMDRuntime::emitProcBindClause(CodeGenFunction &CGF, in emitProcBindClause() argument
12900 Address CGOpenMPSIMDRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, in getAddrOfThreadPrivate() argument
12909 CodeGenFunction *CGF) { in emitThreadPrivateVarDefinition() argument
12914 CodeGenFunction &CGF, QualType VarType, StringRef Name) { in getAddrOfArtificialThreadPrivate() argument
12918 void CGOpenMPSIMDRuntime::emitFlush(CodeGenFunction &CGF, in emitFlush() argument
12925 void CGOpenMPSIMDRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskCall() argument
12935 CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, in emitTaskLoopCall() argument
12942 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates, in emitReduction() argument
12946 CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs, in emitReduction()
12951 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs, in emitTaskReductionInit() argument
12956 void CGOpenMPSIMDRuntime::emitTaskReductionFini(CodeGenFunction &CGF, in emitTaskReductionFini() argument
12962 void CGOpenMPSIMDRuntime::emitTaskReductionFixups(CodeGenFunction &CGF, in emitTaskReductionFixups() argument
12969 Address CGOpenMPSIMDRuntime::getTaskReductionItem(CodeGenFunction &CGF, in getTaskReductionItem() argument
12976 void CGOpenMPSIMDRuntime::emitTaskwaitCall(CodeGenFunction &CGF, in emitTaskwaitCall() argument
12983 CodeGenFunction &CGF, SourceLocation Loc, in emitCancellationPointCall() argument
12988 void CGOpenMPSIMDRuntime::emitCancelCall(CodeGenFunction &CGF, in emitCancelCall() argument
13002 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitTargetCall() argument
13005 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, in emitTargetCall() argument
13023 void CGOpenMPSIMDRuntime::emitTeamsCall(CodeGenFunction &CGF, in emitTeamsCall() argument
13031 void CGOpenMPSIMDRuntime::emitNumTeamsClause(CodeGenFunction &CGF, in emitNumTeamsClause() argument
13039 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, in emitTargetDataCalls() argument
13045 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, in emitTargetDataStandAloneCall() argument
13050 void CGOpenMPSIMDRuntime::emitDoacrossInit(CodeGenFunction &CGF, in emitDoacrossInit() argument
13056 void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, in emitDoacrossOrdered() argument
13068 CGOpenMPSIMDRuntime::getParameterAddress(CodeGenFunction &CGF, in getParameterAddress() argument