Lines Matching refs:CGF
83 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
87 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
156 void Enter(CodeGenFunction &CGF) override { in Enter() argument
159 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue( in Enter()
160 CGF.GetAddrOfLocalVar(PartIDVar), in Enter()
163 CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation()); in Enter()
164 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done."); in Enter()
165 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB); in Enter()
166 CGF.EmitBlock(DoneBB); in Enter()
167 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock); in Enter()
168 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp.")); in Enter()
169 UntiedSwitch->addCase(CGF.Builder.getInt32(0), in Enter()
170 CGF.Builder.GetInsertBlock()); in Enter()
171 emitUntiedSwitch(CGF); in Enter()
174 void emitUntiedSwitch(CodeGenFunction &CGF) const { in emitUntiedSwitch()
176 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue( in emitUntiedSwitch()
177 CGF.GetAddrOfLocalVar(PartIDVar), in emitUntiedSwitch()
179 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()), in emitUntiedSwitch()
181 UntiedCodeGen(CGF); in emitUntiedSwitch()
183 CGF.getJumpDestInCurrentScope(".untied.next."); in emitUntiedSwitch()
184 CGF.EmitBranch(CGF.ReturnBlock.getBlock()); in emitUntiedSwitch()
185 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp.")); in emitUntiedSwitch()
186 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()), in emitUntiedSwitch()
187 CGF.Builder.GetInsertBlock()); in emitUntiedSwitch()
188 CGF.EmitBranchThroughCleanup(CurPoint); in emitUntiedSwitch()
189 CGF.EmitBlock(CurPoint.getBlock()); in emitUntiedSwitch()
209 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
214 void emitUntiedSwitch(CodeGenFunction &CGF) override { in emitUntiedSwitch() argument
215 Action.emitUntiedSwitch(CGF); in emitUntiedSwitch()
282 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override { in getThreadIDVariableLValue() argument
284 return OuterRegionInfo->getThreadIDVariableLValue(CGF); in getThreadIDVariableLValue()
295 void emitUntiedSwitch(CodeGenFunction &CGF) override { in emitUntiedSwitch() argument
297 OuterRegionInfo->emitUntiedSwitch(CGF); in emitUntiedSwitch()
351 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS) in CGOpenMPInnerExprInfo() argument
352 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen, in CGOpenMPInnerExprInfo()
355 PrivScope(CGF) { in CGOpenMPInnerExprInfo()
367 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD), in CGOpenMPInnerExprInfo()
372 VD, [&CGF, &DRE]() { return CGF.EmitLValue(&DRE).getAddress(CGF); }); in CGOpenMPInnerExprInfo()
385 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override { in EmitBody() argument
409 CodeGenFunction &CGF; member in __anoneb75deaf0111::InlinedOpenMPRegionRAII
420 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen, in InlinedOpenMPRegionRAII() argument
423 : CGF(CGF), NoInheritance(NoInheritance) { in InlinedOpenMPRegionRAII()
425 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo( in InlinedOpenMPRegionRAII()
426 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel); in InlinedOpenMPRegionRAII()
428 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields); in InlinedOpenMPRegionRAII()
429 LambdaThisCaptureField = CGF.LambdaThisCaptureField; in InlinedOpenMPRegionRAII()
430 CGF.LambdaThisCaptureField = nullptr; in InlinedOpenMPRegionRAII()
431 BlockInfo = CGF.BlockInfo; in InlinedOpenMPRegionRAII()
432 CGF.BlockInfo = nullptr; in InlinedOpenMPRegionRAII()
439 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI(); in ~InlinedOpenMPRegionRAII()
440 delete CGF.CapturedStmtInfo; in ~InlinedOpenMPRegionRAII()
441 CGF.CapturedStmtInfo = OldCSI; in ~InlinedOpenMPRegionRAII()
443 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields); in ~InlinedOpenMPRegionRAII()
444 CGF.LambdaThisCaptureField = LambdaThisCaptureField; in ~InlinedOpenMPRegionRAII()
445 CGF.BlockInfo = BlockInfo; in ~InlinedOpenMPRegionRAII()
585 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { in Emit() argument
586 if (!CGF.HaveInsertPoint()) in Emit()
588 Action->Exit(CGF); in Emit()
594 void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const { in operator ()()
595 CodeGenFunction::RunCleanupsScope Scope(CGF); in operator ()()
597 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction); in operator ()()
598 Callback(CodeGen, CGF, *PrePostAction); in operator ()()
601 Callback(CodeGen, CGF, Action); in operator ()()
618 static void emitInitWithReductionInitializer(CodeGenFunction &CGF, in emitInitWithReductionInitializer() argument
625 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD); in emitInitWithReductionInitializer()
634 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitInitWithReductionInitializer()
641 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func); in emitInitWithReductionInitializer()
642 CGF.EmitIgnoredExpr(InitOp); in emitInitWithReductionInitializer()
644 llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty); in emitInitWithReductionInitializer()
645 std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"}); in emitInitWithReductionInitializer()
647 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true, in emitInitWithReductionInitializer()
649 LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty); in emitInitWithReductionInitializer()
651 switch (CGF.getEvaluationKind(Ty)) { in emitInitWithReductionInitializer()
653 InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation()); in emitInitWithReductionInitializer()
657 RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation())); in emitInitWithReductionInitializer()
661 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, LV); in emitInitWithReductionInitializer()
662 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(), in emitInitWithReductionInitializer()
668 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal); in emitInitWithReductionInitializer()
669 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(), in emitInitWithReductionInitializer()
679 static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, in EmitOMPAggregateInit() argument
689 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr); in EmitOMPAggregateInit()
691 CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType()); in EmitOMPAggregateInit()
694 CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType()); in EmitOMPAggregateInit()
702 CGF.Builder.CreateGEP(DestAddr.getElementType(), DestBegin, NumElements); in EmitOMPAggregateInit()
704 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body"); in EmitOMPAggregateInit()
705 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done"); in EmitOMPAggregateInit()
707 CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty"); in EmitOMPAggregateInit()
708 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); in EmitOMPAggregateInit()
711 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock(); in EmitOMPAggregateInit()
712 CGF.EmitBlock(BodyBB); in EmitOMPAggregateInit()
714 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy); in EmitOMPAggregateInit()
719 SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2, in EmitOMPAggregateInit()
726 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI( in EmitOMPAggregateInit()
735 CodeGenFunction::RunCleanupsScope InitScope(CGF); in EmitOMPAggregateInit()
737 emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent, in EmitOMPAggregateInit()
740 CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(), in EmitOMPAggregateInit()
746 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateInit()
749 SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateInit()
753 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateInit()
758 CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done"); in EmitOMPAggregateInit()
759 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB); in EmitOMPAggregateInit()
760 DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateInit()
763 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); in EmitOMPAggregateInit()
766 LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) { in emitSharedLValue() argument
767 return CGF.EmitOMPSharedLValue(E); in emitSharedLValue()
770 LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF, in emitSharedLValueUB() argument
773 return CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false); in emitSharedLValueUB()
778 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal, in emitAggregateInitialization() argument
787 EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(), in emitAggregateInitialization()
791 DRD, SharedLVal.getAddress(CGF)); in emitAggregateInitialization()
813 void ReductionCodeGen::emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N) { in emitSharedOrigLValue() argument
816 LValue First = emitSharedLValue(CGF, ClausesData[N].Shared); in emitSharedOrigLValue()
817 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Shared); in emitSharedOrigLValue()
822 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref); in emitSharedOrigLValue()
823 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref); in emitSharedOrigLValue()
828 void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) { in emitAggregateType() argument
835 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()), in emitAggregateType()
842 cast<llvm::PointerType>(OrigAddresses[N].first.getPointer(CGF)->getType()) in emitAggregateType()
846 Size = CGF.Builder.CreatePtrDiff(OrigAddresses[N].second.getPointer(CGF), in emitAggregateType()
847 OrigAddresses[N].first.getPointer(CGF)); in emitAggregateType()
848 Size = CGF.Builder.CreateNUWAdd( in emitAggregateType()
850 SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf); in emitAggregateType()
853 CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()); in emitAggregateType()
854 Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf); in emitAggregateType()
858 CGF, in emitAggregateType()
860 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()), in emitAggregateType()
862 CGF.EmitVariablyModifiedType(PrivateType); in emitAggregateType()
865 void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N, in emitAggregateType() argument
877 CGF, in emitAggregateType()
879 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()), in emitAggregateType()
881 CGF.EmitVariablyModifiedType(PrivateType); in emitAggregateType()
885 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal, in emitInitialization() argument
893 PrivateAddr = CGF.Builder.CreateElementBitCast( in emitInitialization()
894 PrivateAddr, CGF.ConvertTypeForMem(PrivateType)); in emitInitialization()
896 SharedLVal = CGF.MakeAddrLValue( in emitInitialization()
897 CGF.Builder.CreateElementBitCast(SharedLVal.getAddress(CGF), in emitInitialization()
898 CGF.ConvertTypeForMem(SharedType)), in emitInitialization()
900 CGF.CGM.getTBAAInfoForSubobject(SharedAddresses[N].first, SharedType)); in emitInitialization()
901 if (CGF.getContext().getAsArrayType(PrivateVD->getType())) { in emitInitialization()
903 (void)DefaultInit(CGF); in emitInitialization()
904 emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD); in emitInitialization()
906 (void)DefaultInit(CGF); in emitInitialization()
907 emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp, in emitInitialization()
908 PrivateAddr, SharedLVal.getAddress(CGF), in emitInitialization()
910 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() && in emitInitialization()
911 !CGF.isTrivialInitializer(PrivateVD->getInit())) { in emitInitialization()
912 CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr, in emitInitialization()
926 void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N, in emitCleanups() argument
933 PrivateAddr = CGF.Builder.CreateElementBitCast( in emitCleanups()
934 PrivateAddr, CGF.ConvertTypeForMem(PrivateType)); in emitCleanups()
935 CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType); in emitCleanups()
939 static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, in loadToBegin() argument
943 !CGF.getContext().hasSameType(BaseTy, ElTy)) { in loadToBegin()
945 BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(CGF), PtrTy); in loadToBegin()
947 LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(CGF), BaseTy); in loadToBegin()
948 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal); in loadToBegin()
952 return CGF.MakeAddrLValue( in loadToBegin()
953 CGF.Builder.CreateElementBitCast(BaseLV.getAddress(CGF), in loadToBegin()
954 CGF.ConvertTypeForMem(ElTy)), in loadToBegin()
956 CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType())); in loadToBegin()
959 static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, in castToBase() argument
967 !CGF.getContext().hasSameType(BaseTy, ElTy)) { in castToBase()
968 Tmp = CGF.CreateMemTemp(BaseTy); in castToBase()
970 CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp); in castToBase()
979 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty); in castToBase()
981 CGF.Builder.CreateStore(Addr, Tmp); in castToBase()
1007 Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, in adjustPrivateAddress() argument
1012 LValue OriginalBaseLValue = CGF.EmitLValue(DE); in adjustPrivateAddress()
1014 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(), in adjustPrivateAddress()
1016 Address SharedAddr = SharedAddresses[N].first.getAddress(CGF); in adjustPrivateAddress()
1017 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff( in adjustPrivateAddress()
1018 BaseLValue.getPointer(CGF), SharedAddr.getPointer()); in adjustPrivateAddress()
1020 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in adjustPrivateAddress()
1022 llvm::Value *Ptr = CGF.Builder.CreateGEP( in adjustPrivateAddress()
1024 return castToBase(CGF, OrigVD->getType(), in adjustPrivateAddress()
1026 OriginalBaseLValue.getAddress(CGF).getType(), in adjustPrivateAddress()
1040 LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) { in getThreadIDVariableLValue() argument
1041 return CGF.EmitLoadOfPointerLValue( in getThreadIDVariableLValue()
1042 CGF.GetAddrOfLocalVar(getThreadIDVariable()), in getThreadIDVariableLValue()
1046 void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt *S) { in EmitBody() argument
1047 if (!CGF.HaveInsertPoint()) in EmitBody()
1054 CGF.EHStack.pushTerminate(); in EmitBody()
1056 CGF.incrementProfileCounter(S); in EmitBody()
1057 CodeGen(CGF); in EmitBody()
1058 CGF.EHStack.popTerminate(); in EmitBody()
1062 CodeGenFunction &CGF) { in getThreadIDVariableLValue() argument
1063 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()), in getThreadIDVariableLValue()
1143 CodeGenFunction CGF(CGM); in emitCombinerOrInitializer() local
1146 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(), in emitCombinerOrInitializer()
1148 CodeGenFunction::OMPPrivateScope Scope(CGF); in emitCombinerOrInitializer()
1149 Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm); in emitCombinerOrInitializer()
1150 Scope.addPrivate(In, [&CGF, AddrIn, PtrTy]() { in emitCombinerOrInitializer()
1151 return CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>()) in emitCombinerOrInitializer()
1152 .getAddress(CGF); in emitCombinerOrInitializer()
1154 Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm); in emitCombinerOrInitializer()
1155 Scope.addPrivate(Out, [&CGF, AddrOut, PtrTy]() { in emitCombinerOrInitializer()
1156 return CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>()) in emitCombinerOrInitializer()
1157 .getAddress(CGF); in emitCombinerOrInitializer()
1161 !CGF.isTrivialInitializer(Out->getInit())) { in emitCombinerOrInitializer()
1162 CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out), in emitCombinerOrInitializer()
1167 CGF.EmitIgnoredExpr(CombinerInitializer); in emitCombinerOrInitializer()
1169 CGF.FinishFunction(); in emitCombinerOrInitializer()
1174 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) { in emitUserDefinedReduction() argument
1193 if (CGF) { in emitUserDefinedReduction()
1194 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn); in emitUserDefinedReduction()
1212 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder, CodeGenFunction &CGF, in PushAndPopStackRAII()
1230 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) { in PushAndPopStackRAII()
1233 CGBuilderTy::InsertPointGuard IPG(CGF.Builder); in PushAndPopStackRAII()
1234 CGF.Builder.restoreIP(IP); in PushAndPopStackRAII()
1236 CGF.getOMPCancelDestination(OMPD_parallel); in PushAndPopStackRAII()
1237 CGF.EmitBranchThroughCleanup(Dest); in PushAndPopStackRAII()
1259 CodeGenFunction CGF(CGM, true); in emitParallelOrTeamsOutlinedFunction() local
1283 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind); in emitParallelOrTeamsOutlinedFunction()
1286 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitParallelOrTeamsOutlinedFunction()
1287 return CGF.GenerateOpenMPCapturedStmtFunction(*CS, D.getBeginLoc()); in emitParallelOrTeamsOutlinedFunction()
1311 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF, in emitTaskOutlinedFunction()
1313 llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc()); in emitTaskOutlinedFunction()
1314 llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTaskOutlinedFunction()
1317 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar), in emitTaskOutlinedFunction()
1319 .getPointer(CGF)}; in emitTaskOutlinedFunction()
1320 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskOutlinedFunction()
1343 CodeGenFunction CGF(CGM, true); in emitTaskOutlinedFunction() local
1346 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitTaskOutlinedFunction()
1347 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS); in emitTaskOutlinedFunction()
1398 void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF, in setLocThreadIdInsertPt() argument
1400 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in setLocThreadIdInsertPt()
1403 llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty); in setLocThreadIdInsertPt()
1406 Undef, CGF.Int32Ty, "svcpt", CGF.Builder.GetInsertBlock()); in setLocThreadIdInsertPt()
1409 new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt"); in setLocThreadIdInsertPt()
1410 Elem.second.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt); in setLocThreadIdInsertPt()
1414 void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) { in clearLocThreadIdInsertPt() argument
1415 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in clearLocThreadIdInsertPt()
1423 static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF, in getIdentStringFromSourceLocation() argument
1428 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc); in getIdentStringFromSourceLocation()
1430 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl)) in getIdentStringFromSourceLocation()
1436 llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF, in emitUpdateLocation() argument
1445 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl)) in emitUpdateLocation()
1447 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc); in emitUpdateLocation()
1459 llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF, in getThreadID() argument
1461 assert(CGF.CurFn && "No function in current CodeGenFunction."); in getThreadID()
1466 OMPBuilder.updateToLocation(CGF.Builder.saveIP()); in getThreadID()
1468 getIdentStringFromSourceLocation(CGF, Loc, Buffer)); in getThreadID()
1476 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn); in getThreadID()
1484 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { in getThreadID()
1487 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF); in getThreadID()
1488 llvm::BasicBlock *TopBlock = CGF.AllocaInsertPt->getParent(); in getThreadID()
1489 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions || in getThreadID()
1490 !CGF.getLangOpts().CXXExceptions || in getThreadID()
1491 CGF.Builder.GetInsertBlock() == TopBlock || in getThreadID()
1492 !isa<llvm::Instruction>(LVal.getPointer(CGF)) || in getThreadID()
1493 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() == in getThreadID()
1495 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() == in getThreadID()
1496 CGF.Builder.GetInsertBlock()) { in getThreadID()
1497 ThreadID = CGF.EmitLoadOfScalar(LVal, Loc); in getThreadID()
1500 if (CGF.Builder.GetInsertBlock() == TopBlock) { in getThreadID()
1501 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in getThreadID()
1513 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in getThreadID()
1515 setLocThreadIdInsertPt(CGF); in getThreadID()
1516 CGBuilderTy::InsertPointGuard IPG(CGF.Builder); in getThreadID()
1517 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt); in getThreadID()
1518 llvm::CallInst *Call = CGF.Builder.CreateCall( in getThreadID()
1521 emitUpdateLocation(CGF, Loc)); in getThreadID()
1522 Call->setCallingConv(CGF.getRuntimeCC()); in getThreadID()
1527 void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) { in functionFinished() argument
1528 assert(CGF.CurFn && "No function in current CodeGenFunction."); in functionFinished()
1529 if (OpenMPLocThreadIDMap.count(CGF.CurFn)) { in functionFinished()
1530 clearLocThreadIdInsertPt(CGF); in functionFinished()
1531 OpenMPLocThreadIDMap.erase(CGF.CurFn); in functionFinished()
1533 if (FunctionUDRMap.count(CGF.CurFn) > 0) { in functionFinished()
1534 for(const auto *D : FunctionUDRMap[CGF.CurFn]) in functionFinished()
1536 FunctionUDRMap.erase(CGF.CurFn); in functionFinished()
1538 auto I = FunctionUDMMap.find(CGF.CurFn); in functionFinished()
1544 LastprivateConditionalToTypes.erase(CGF.CurFn); in functionFinished()
1545 FunctionToUntiedTaskStackMap.erase(CGF.CurFn); in functionFinished()
1729 Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, in getAddrOfThreadPrivate() argument
1738 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in getAddrOfThreadPrivate()
1739 CGF.Builder.CreatePointerCast(VDAddr.getPointer(), in getAddrOfThreadPrivate()
1743 return Address(CGF.EmitRuntimeCall( in getAddrOfThreadPrivate()
1751 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor, in emitThreadPrivateVarInit() argument
1755 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc); in emitThreadPrivateVarInit()
1756 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitThreadPrivateVarInit()
1762 OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy), in emitThreadPrivateVarInit()
1764 CGF.EmitRuntimeCall( in emitThreadPrivateVarInit()
1772 bool PerformInit, CodeGenFunction *CGF) { in emitThreadPrivateVarDefinition() argument
1870 if (!CGF) { in emitThreadPrivateVarDefinition()
1885 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc); in emitThreadPrivateVarDefinition()
2005 Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, in getAddrOfArtificialThreadPrivate() argument
2009 llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType); in getAddrOfArtificialThreadPrivate()
2019 emitUpdateLocation(CGF, SourceLocation()), in getAddrOfArtificialThreadPrivate()
2020 getThreadID(CGF, SourceLocation()), in getAddrOfArtificialThreadPrivate()
2021 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy), in getAddrOfArtificialThreadPrivate()
2022 CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy, in getAddrOfArtificialThreadPrivate()
2027 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddrOfArtificialThreadPrivate()
2028 CGF.EmitRuntimeCall( in getAddrOfArtificialThreadPrivate()
2036 void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond, in emitIfClause() argument
2039 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange()); in emitIfClause()
2044 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) { in emitIfClause()
2046 ThenGen(CGF); in emitIfClause()
2048 ElseGen(CGF); in emitIfClause()
2054 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then"); in emitIfClause()
2055 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else"); in emitIfClause()
2056 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end"); in emitIfClause()
2057 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0); in emitIfClause()
2060 CGF.EmitBlock(ThenBlock); in emitIfClause()
2061 ThenGen(CGF); in emitIfClause()
2062 CGF.EmitBranch(ContBlock); in emitIfClause()
2065 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitIfClause()
2066 CGF.EmitBlock(ElseBlock); in emitIfClause()
2067 ElseGen(CGF); in emitIfClause()
2069 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitIfClause()
2070 CGF.EmitBranch(ContBlock); in emitIfClause()
2072 CGF.EmitBlock(ContBlock, /*IsFinished=*/true); in emitIfClause()
2075 void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, in emitParallelCall() argument
2079 if (!CGF.HaveInsertPoint()) in emitParallelCall()
2081 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitParallelCall()
2084 this](CodeGenFunction &CGF, PrePostActionTy &) { in emitParallelCall() argument
2086 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitParallelCall()
2089 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars in emitParallelCall()
2090 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())}; in emitParallelCall()
2097 CGF.EmitRuntimeCall(RTLFn, RealArgs); in emitParallelCall()
2100 this](CodeGenFunction &CGF, PrePostActionTy &) { in emitParallelCall() argument
2101 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitParallelCall()
2102 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc); in emitParallelCall()
2106 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitParallelCall()
2111 Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc); in emitParallelCall()
2113 CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, in emitParallelCall()
2115 CGF.InitTempAlloca(ZeroAddrBound, CGF.Builder.getInt32(/*C*/ 0)); in emitParallelCall()
2130 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs); in emitParallelCall()
2133 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID}; in emitParallelCall()
2134 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitParallelCall()
2139 emitIfClause(CGF, IfCond, ThenGen, ElseGen); in emitParallelCall()
2142 ThenRCG(CGF); in emitParallelCall()
2152 Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF, in emitThreadIDAddress() argument
2155 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitThreadIDAddress()
2157 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(CGF); in emitThreadIDAddress()
2159 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitThreadIDAddress()
2161 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true); in emitThreadIDAddress()
2162 Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp."); in emitThreadIDAddress()
2163 CGF.EmitStoreOfScalar(ThreadID, in emitThreadIDAddress()
2164 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty)); in emitThreadIDAddress()
2212 void Enter(CodeGenFunction &CGF) override { in Enter() argument
2213 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs); in Enter()
2215 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes); in Enter()
2216 auto *ThenBlock = CGF.createBasicBlock("omp_if.then"); in Enter()
2217 ContBlock = CGF.createBasicBlock("omp_if.end"); in Enter()
2219 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock); in Enter()
2220 CGF.EmitBlock(ThenBlock); in Enter()
2223 void Done(CodeGenFunction &CGF) { in Done() argument
2225 CGF.EmitBranch(ContBlock); in Done()
2226 CGF.EmitBlock(ContBlock, true); in Done()
2228 void Exit(CodeGenFunction &CGF) override { in Exit() argument
2229 CGF.EmitRuntimeCall(ExitCallee, ExitArgs); in Exit()
2234 void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF, in emitCriticalRegion() argument
2242 if (!CGF.HaveInsertPoint()) in emitCriticalRegion()
2244 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitCriticalRegion()
2249 EnterArgs.push_back(CGF.Builder.CreateIntCast( in emitCriticalRegion()
2250 CGF.EmitScalarExpr(Hint), CGM.Int32Ty, /*isSigned=*/false)); in emitCriticalRegion()
2261 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen); in emitCriticalRegion()
2264 void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF, in emitMasterRegion() argument
2267 if (!CGF.HaveInsertPoint()) in emitMasterRegion()
2274 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitMasterRegion()
2283 emitInlinedDirective(CGF, OMPD_master, MasterOpGen); in emitMasterRegion()
2284 Action.Done(CGF); in emitMasterRegion()
2287 void CGOpenMPRuntime::emitMaskedRegion(CodeGenFunction &CGF, in emitMaskedRegion() argument
2290 if (!CGF.HaveInsertPoint()) in emitMaskedRegion()
2298 ? CGF.EmitScalarExpr(Filter, CGF.Int32Ty) in emitMaskedRegion()
2300 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitMaskedRegion()
2302 llvm::Value *ArgsEnd[] = {emitUpdateLocation(CGF, Loc), in emitMaskedRegion()
2303 getThreadID(CGF, Loc)}; in emitMaskedRegion()
2312 emitInlinedDirective(CGF, OMPD_masked, MaskedOpGen); in emitMaskedRegion()
2313 Action.Done(CGF); in emitMaskedRegion()
2316 void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF, in emitTaskyieldCall() argument
2318 if (!CGF.HaveInsertPoint()) in emitTaskyieldCall()
2320 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) { in emitTaskyieldCall()
2321 OMPBuilder.createTaskyield(CGF.Builder); in emitTaskyieldCall()
2325 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitTaskyieldCall()
2327 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskyieldCall()
2332 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitTaskyieldCall()
2333 Region->emitUntiedSwitch(CGF); in emitTaskyieldCall()
2336 void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF, in emitTaskgroupRegion() argument
2339 if (!CGF.HaveInsertPoint()) in emitTaskgroupRegion()
2345 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitTaskgroupRegion()
2353 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen); in emitTaskgroupRegion()
2358 static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array, in emitAddrOfVarFromArray() argument
2361 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index); in emitAddrOfVarFromArray()
2362 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr); in emitAddrOfVarFromArray()
2364 Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var)); in emitAddrOfVarFromArray()
2365 Addr = CGF.Builder.CreateElementBitCast( in emitAddrOfVarFromArray()
2366 Addr, CGF.ConvertTypeForMem(Var->getType())); in emitAddrOfVarFromArray()
2393 CodeGenFunction CGF(CGM); in emitCopyprivateCopyFunction() local
2394 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitCopyprivateCopyFunction()
2397 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitCopyprivateCopyFunction()
2398 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)), in emitCopyprivateCopyFunction()
2399 ArgsType), CGF.getPointerAlign()); in emitCopyprivateCopyFunction()
2400 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitCopyprivateCopyFunction()
2401 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)), in emitCopyprivateCopyFunction()
2402 ArgsType), CGF.getPointerAlign()); in emitCopyprivateCopyFunction()
2410 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar); in emitCopyprivateCopyFunction()
2414 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar); in emitCopyprivateCopyFunction()
2418 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]); in emitCopyprivateCopyFunction()
2420 CGF.FinishFunction(); in emitCopyprivateCopyFunction()
2424 void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF, in emitSingleRegion() argument
2431 if (!CGF.HaveInsertPoint()) in emitSingleRegion()
2451 DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it"); in emitSingleRegion()
2452 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt); in emitSingleRegion()
2455 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitSingleRegion()
2464 emitInlinedDirective(CGF, OMPD_single, SingleOpGen); in emitSingleRegion()
2467 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt); in emitSingleRegion()
2469 Action.Done(CGF); in emitSingleRegion()
2479 CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list"); in emitSingleRegion()
2481 Address Elem = CGF.Builder.CreateConstArrayGEP(CopyprivateList, I); in emitSingleRegion()
2482 CGF.Builder.CreateStore( in emitSingleRegion()
2483 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitSingleRegion()
2484 CGF.EmitLValue(CopyprivateVars[I]).getPointer(CGF), in emitSingleRegion()
2485 CGF.VoidPtrTy), in emitSingleRegion()
2491 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(), in emitSingleRegion()
2493 llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy); in emitSingleRegion()
2495 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList, in emitSingleRegion()
2496 CGF.VoidPtrTy); in emitSingleRegion()
2497 llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt); in emitSingleRegion()
2499 emitUpdateLocation(CGF, Loc), // ident_t *<loc> in emitSingleRegion()
2500 getThreadID(CGF, Loc), // i32 <gtid> in emitSingleRegion()
2506 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitSingleRegion()
2512 void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF, in emitOrderedRegion() argument
2515 if (!CGF.HaveInsertPoint()) in emitOrderedRegion()
2522 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitOrderedRegion()
2530 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen); in emitOrderedRegion()
2533 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen); in emitOrderedRegion()
2552 CodeGenFunction &CGF, const OMPLoopDirective &S, in getDefaultScheduleAndChunk() argument
2563 CGF.getContext(), ChunkSize, in getDefaultScheduleAndChunk()
2564 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0), in getDefaultScheduleAndChunk()
2569 void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, in emitBarrierCall() argument
2574 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo); in emitBarrierCall()
2575 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) { in emitBarrierCall()
2576 CGF.Builder.restoreIP(OMPBuilder.createBarrier( in emitBarrierCall()
2577 CGF.Builder, Kind, ForceSimpleCall, EmitChecks)); in emitBarrierCall()
2581 if (!CGF.HaveInsertPoint()) in emitBarrierCall()
2588 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags), in emitBarrierCall()
2589 getThreadID(CGF, Loc)}; in emitBarrierCall()
2592 llvm::Value *Result = CGF.EmitRuntimeCall( in emitBarrierCall()
2600 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit"); in emitBarrierCall()
2601 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue"); in emitBarrierCall()
2602 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result); in emitBarrierCall()
2603 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); in emitBarrierCall()
2604 CGF.EmitBlock(ExitBB); in emitBarrierCall()
2607 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); in emitBarrierCall()
2608 CGF.EmitBranchThroughCleanup(CancelDestination); in emitBarrierCall()
2609 CGF.EmitBlock(ContBB, /*IsFinished=*/true); in emitBarrierCall()
2614 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitBarrierCall()
2733 CodeGenFunction &CGF, SourceLocation Loc, in emitForDispatchInit() argument
2736 if (!CGF.HaveInsertPoint()) in emitForDispatchInit()
2751 : CGF.Builder.getIntN(IVSize, 1); in emitForDispatchInit()
2753 emitUpdateLocation(CGF, Loc), in emitForDispatchInit()
2754 getThreadID(CGF, Loc), in emitForDispatchInit()
2755 CGF.Builder.getInt32(addMonoNonMonoModifier( in emitForDispatchInit()
2759 CGF.Builder.getIntN(IVSize, 1), // Stride in emitForDispatchInit()
2762 CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args); in emitForDispatchInit()
2766 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId, in emitForStaticInitCall() argument
2770 if (!CGF.HaveInsertPoint()) in emitForStaticInitCall()
2791 Chunk = CGF.Builder.getIntN(Values.IVSize, 1); in emitForStaticInitCall()
2802 CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1, in emitForStaticInitCall()
2808 CGF.Builder.getIntN(Values.IVSize, 1), // Incr in emitForStaticInitCall()
2811 CGF.EmitRuntimeCall(ForStaticInitFunction, Args); in emitForStaticInitCall()
2814 void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF, in emitForStaticInit() argument
2823 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc, in emitForStaticInit()
2827 llvm::Value *ThreadId = getThreadID(CGF, Loc); in emitForStaticInit()
2830 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc); in emitForStaticInit()
2831 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction, in emitForStaticInit()
2836 CodeGenFunction &CGF, SourceLocation Loc, in emitDistributeStaticInit() argument
2842 emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE); in emitDistributeStaticInit()
2843 llvm::Value *ThreadId = getThreadID(CGF, Loc); in emitDistributeStaticInit()
2846 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction, in emitDistributeStaticInit()
2851 void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF, in emitForStaticFinish() argument
2854 if (!CGF.HaveInsertPoint()) in emitForStaticFinish()
2858 emitUpdateLocation(CGF, Loc, in emitForStaticFinish()
2864 getThreadID(CGF, Loc)}; in emitForStaticFinish()
2865 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc); in emitForStaticFinish()
2866 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitForStaticFinish()
2871 void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF, in emitForOrderedIterationEnd() argument
2875 if (!CGF.HaveInsertPoint()) in emitForOrderedIterationEnd()
2878 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitForOrderedIterationEnd()
2879 CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args); in emitForOrderedIterationEnd()
2882 llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF, in emitForNext() argument
2892 emitUpdateLocation(CGF, Loc), in emitForNext()
2893 getThreadID(CGF, Loc), in emitForNext()
2900 CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args); in emitForNext()
2901 return CGF.EmitScalarConversion( in emitForNext()
2902 Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1), in emitForNext()
2903 CGF.getContext().BoolTy, Loc); in emitForNext()
2906 void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF, in emitNumThreadsClause() argument
2909 if (!CGF.HaveInsertPoint()) in emitNumThreadsClause()
2913 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitNumThreadsClause()
2914 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)}; in emitNumThreadsClause()
2915 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitNumThreadsClause()
2920 void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF, in emitProcBindClause() argument
2923 if (!CGF.HaveInsertPoint()) in emitProcBindClause()
2928 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitProcBindClause()
2930 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitProcBindClause()
2935 void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>, in emitFlush() argument
2937 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) { in emitFlush()
2938 OMPBuilder.createFlush(CGF.Builder); in emitFlush()
2940 if (!CGF.HaveInsertPoint()) in emitFlush()
2943 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitFlush()
2945 emitUpdateLocation(CGF, Loc)); in emitFlush()
3589 CodeGenFunction CGF(CGM); in emitProxyTaskFunction() local
3590 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args, in emitProxyTaskFunction()
3598 llvm::Value *GtidParam = CGF.EmitLoadOfScalar( in emitProxyTaskFunction()
3599 CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc); in emitProxyTaskFunction()
3600 LValue TDBase = CGF.EmitLoadOfPointerLValue( in emitProxyTaskFunction()
3601 CGF.GetAddrOfLocalVar(&TaskTypeArg), in emitProxyTaskFunction()
3606 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin()); in emitProxyTaskFunction()
3609 LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI); in emitProxyTaskFunction()
3610 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF); in emitProxyTaskFunction()
3613 LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI); in emitProxyTaskFunction()
3614 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitProxyTaskFunction()
3615 CGF.EmitLoadOfScalar(SharedsLVal, Loc), in emitProxyTaskFunction()
3616 CGF.ConvertTypeForMem(SharedsPtrTy)); in emitProxyTaskFunction()
3621 LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI); in emitProxyTaskFunction()
3622 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitProxyTaskFunction()
3623 PrivatesLVal.getPointer(CGF), CGF.VoidPtrTy); in emitProxyTaskFunction()
3625 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitProxyTaskFunction()
3630 CGF.Builder in emitProxyTaskFunction()
3632 TDBase.getAddress(CGF), CGF.VoidPtrTy) in emitProxyTaskFunction()
3638 LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI); in emitProxyTaskFunction()
3639 llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc); in emitProxyTaskFunction()
3641 LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI); in emitProxyTaskFunction()
3642 llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc); in emitProxyTaskFunction()
3644 LValue StLVal = CGF.EmitLValueForField(Base, *StFI); in emitProxyTaskFunction()
3645 llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc); in emitProxyTaskFunction()
3647 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI); in emitProxyTaskFunction()
3648 llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc); in emitProxyTaskFunction()
3650 LValue RLVal = CGF.EmitLValueForField(Base, *RFI); in emitProxyTaskFunction()
3651 llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc); in emitProxyTaskFunction()
3660 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction, in emitProxyTaskFunction()
3662 CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)), in emitProxyTaskFunction()
3663 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty)); in emitProxyTaskFunction()
3664 CGF.FinishFunction(); in emitProxyTaskFunction()
3694 CodeGenFunction CGF(CGM); in emitDestructorsFunction() local
3695 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo, in emitDestructorsFunction()
3698 LValue Base = CGF.EmitLoadOfPointerLValue( in emitDestructorsFunction()
3699 CGF.GetAddrOfLocalVar(&TaskTypeArg), in emitDestructorsFunction()
3704 Base = CGF.EmitLValueForField(Base, *FI); in emitDestructorsFunction()
3709 LValue FieldLValue = CGF.EmitLValueForField(Base, Field); in emitDestructorsFunction()
3710 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(CGF), Field->getType()); in emitDestructorsFunction()
3713 CGF.FinishFunction(); in emitDestructorsFunction()
3802 CodeGenFunction CGF(CGM); in emitTaskPrivateMappingFunction() local
3803 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap, in emitTaskPrivateMappingFunction()
3807 LValue Base = CGF.EmitLoadOfPointerLValue( in emitTaskPrivateMappingFunction()
3808 CGF.GetAddrOfLocalVar(&TaskPrivatesArg), in emitTaskPrivateMappingFunction()
3813 LValue FieldLVal = CGF.EmitLValueForField(Base, Field); in emitTaskPrivateMappingFunction()
3816 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType()); in emitTaskPrivateMappingFunction()
3817 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue( in emitTaskPrivateMappingFunction()
3818 RefLVal.getAddress(CGF), RefLVal.getType()->castAs<PointerType>()); in emitTaskPrivateMappingFunction()
3819 CGF.EmitStoreOfScalar(FieldLVal.getPointer(CGF), RefLoadLVal); in emitTaskPrivateMappingFunction()
3822 CGF.FinishFunction(); in emitTaskPrivateMappingFunction()
3827 static void emitPrivatesInit(CodeGenFunction &CGF, in emitPrivatesInit() argument
3834 ASTContext &C = CGF.getContext(); in emitPrivatesInit()
3836 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI); in emitPrivatesInit()
3851 SrcBase = CGF.MakeAddrLValue( in emitPrivatesInit()
3852 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitPrivatesInit()
3853 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)), in emitPrivatesInit()
3866 !CGF.isTrivialInitializer(Init)))) { in emitPrivatesInit()
3867 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI); in emitPrivatesInit()
3885 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type); in emitPrivatesInit()
3887 SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField); in emitPrivatesInit()
3888 SharedRefLValue = CGF.MakeAddrLValue( in emitPrivatesInit()
3889 Address(SharedRefLValue.getPointer(CGF), in emitPrivatesInit()
3893 } else if (CGF.LambdaCaptureFields.count( in emitPrivatesInit()
3895 dyn_cast_or_null<BlockDecl>(CGF.CurCodeDecl)) { in emitPrivatesInit()
3896 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef); in emitPrivatesInit()
3900 CGF, [](CodeGenFunction &, PrePostActionTy &) {}, OMPD_unknown, in emitPrivatesInit()
3902 SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef); in emitPrivatesInit()
3906 if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) { in emitPrivatesInit()
3908 CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type); in emitPrivatesInit()
3912 CGF.EmitOMPAggregateAssign( in emitPrivatesInit()
3913 PrivateLValue.getAddress(CGF), SharedRefLValue.getAddress(CGF), in emitPrivatesInit()
3915 [&CGF, Elem, Init, &CapturesInfo](Address DestElement, in emitPrivatesInit()
3918 CodeGenFunction::OMPPrivateScope InitScope(CGF); in emitPrivatesInit()
3924 CGF, &CapturesInfo); in emitPrivatesInit()
3925 CGF.EmitAnyExprToMem(Init, DestElement, in emitPrivatesInit()
3931 CodeGenFunction::OMPPrivateScope InitScope(CGF); in emitPrivatesInit()
3932 InitScope.addPrivate(Elem, [SharedRefLValue, &CGF]() -> Address { in emitPrivatesInit()
3933 return SharedRefLValue.getAddress(CGF); in emitPrivatesInit()
3936 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo); in emitPrivatesInit()
3937 CGF.EmitExprAsInit(Init, VD, PrivateLValue, in emitPrivatesInit()
3941 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false); in emitPrivatesInit()
3949 static bool checkInitIsRequired(CodeGenFunction &CGF, in checkInitIsRequired() argument
3958 !CGF.isTrivialInitializer(Init)); in checkInitIsRequired()
4005 CodeGenFunction CGF(CGM); in emitTaskDupFunction() local
4006 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc, in emitTaskDupFunction()
4009 LValue TDBase = CGF.EmitLoadOfPointerLValue( in emitTaskDupFunction()
4010 CGF.GetAddrOfLocalVar(&DstArg), in emitTaskDupFunction()
4015 LValue Base = CGF.EmitLValueForField( in emitTaskDupFunction()
4017 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI); in emitTaskDupFunction()
4018 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar( in emitTaskDupFunction()
4019 CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc); in emitTaskDupFunction()
4020 CGF.EmitStoreOfScalar(Lastpriv, LILVal); in emitTaskDupFunction()
4027 LValue TDBase = CGF.EmitLoadOfPointerLValue( in emitTaskDupFunction()
4028 CGF.GetAddrOfLocalVar(&SrcArg), in emitTaskDupFunction()
4030 LValue Base = CGF.EmitLValueForField( in emitTaskDupFunction()
4033 CGF.EmitLoadOfScalar(CGF.EmitLValueForField( in emitTaskDupFunction()
4039 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD, in emitTaskDupFunction()
4041 CGF.FinishFunction(); in emitTaskDupFunction()
4064 CodeGenFunction &CGF; member in __anoneb75deaf1b11::OMPIteratorGeneratorScope
4072 OMPIteratorGeneratorScope(CodeGenFunction &CGF, const OMPIteratorExpr *E) in OMPIteratorGeneratorScope() argument
4073 : CodeGenFunction::OMPPrivateScope(CGF), CGF(CGF), E(E) { in OMPIteratorGeneratorScope()
4078 Uppers.push_back(CGF.EmitScalarExpr(E->getHelper(I).Upper)); in OMPIteratorGeneratorScope()
4080 addPrivate(VD, [&CGF, VD]() { in OMPIteratorGeneratorScope()
4081 return CGF.CreateMemTemp(VD->getType(), VD->getName()); in OMPIteratorGeneratorScope()
4084 addPrivate(HelperData.CounterVD, [&CGF, &HelperData]() { in OMPIteratorGeneratorScope()
4085 return CGF.CreateMemTemp(HelperData.CounterVD->getType(), in OMPIteratorGeneratorScope()
4094 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(HelperData.CounterVD), in OMPIteratorGeneratorScope()
4097 CGF.EmitStoreOfScalar( in OMPIteratorGeneratorScope()
4098 llvm::ConstantInt::get(CLVal.getAddress(CGF).getElementType(), 0), in OMPIteratorGeneratorScope()
4101 ContDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.cont")); in OMPIteratorGeneratorScope()
4103 ExitDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.exit")); in OMPIteratorGeneratorScope()
4108 CGF.EmitBlock(ContDest.getBlock()); in OMPIteratorGeneratorScope()
4110 CGF.EmitLoadOfScalar(CLVal, HelperData.CounterVD->getLocation()); in OMPIteratorGeneratorScope()
4113 ? CGF.Builder.CreateICmpSLT(CVal, N) in OMPIteratorGeneratorScope()
4114 : CGF.Builder.CreateICmpULT(CVal, N); in OMPIteratorGeneratorScope()
4115 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("iter.body"); in OMPIteratorGeneratorScope()
4116 CGF.Builder.CreateCondBr(Cmp, BodyBB, ExitDest.getBlock()); in OMPIteratorGeneratorScope()
4118 CGF.EmitBlock(BodyBB); in OMPIteratorGeneratorScope()
4120 CGF.EmitIgnoredExpr(HelperData.Update); in OMPIteratorGeneratorScope()
4129 CGF.EmitIgnoredExpr(HelperData.CounterUpdate); in ~OMPIteratorGeneratorScope()
4131 CGF.EmitBranchThroughCleanup(ContDests[I - 1]); in ~OMPIteratorGeneratorScope()
4133 CGF.EmitBlock(ExitDests[I - 1].getBlock(), /*IsFinished=*/I == 1); in ~OMPIteratorGeneratorScope()
4140 getPointerAndSize(CodeGenFunction &CGF, const Expr *E) { in getPointerAndSize() argument
4145 Addr = CGF.EmitScalarExpr(Base); in getPointerAndSize()
4147 Addr = CGF.EmitLValue(E).getPointer(CGF); in getPointerAndSize()
4152 SizeVal = CGF.getTypeSize(OASE->getBase()->getType()->getPointeeType()); in getPointerAndSize()
4154 llvm::Value *Sz = CGF.EmitScalarExpr(SE); in getPointerAndSize()
4155 Sz = CGF.EmitScalarConversion( in getPointerAndSize()
4156 Sz, SE->getType(), CGF.getContext().getSizeType(), SE->getExprLoc()); in getPointerAndSize()
4157 SizeVal = CGF.Builder.CreateNUWMul(SizeVal, Sz); in getPointerAndSize()
4162 CGF.EmitOMPArraySectionExpr(ASE, /*IsLowerBound=*/false); in getPointerAndSize()
4163 Address UpAddrAddress = UpAddrLVal.getAddress(CGF); in getPointerAndSize()
4164 llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32( in getPointerAndSize()
4166 llvm::Value *LowIntPtr = CGF.Builder.CreatePtrToInt(Addr, CGF.SizeTy); in getPointerAndSize()
4167 llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGF.SizeTy); in getPointerAndSize()
4168 SizeVal = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr); in getPointerAndSize()
4170 SizeVal = CGF.getTypeSize(Ty); in getPointerAndSize()
4191 CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskInit() argument
4266 llvm::Type *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy); in emitTaskInit()
4270 CGF.getTypeSize(KmpTaskTWithPrivatesQTy); in emitTaskInit()
4281 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
4321 ? CGF.Builder.CreateSelect(Data.Final.getPointer(), in emitTaskInit()
4322 CGF.Builder.getInt32(FinalFlag), in emitTaskInit()
4323 CGF.Builder.getInt32(/*C=*/0)) in emitTaskInit()
4324 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0); in emitTaskInit()
4325 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags)); in emitTaskInit()
4327 SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc), in emitTaskInit()
4328 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize, in emitTaskInit()
4329 SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
4340 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTaskInit()
4341 CGF.Int64Ty, /*isSigned=*/true); in emitTaskInit()
4343 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTaskInit()
4345 NewTask = CGF.EmitRuntimeCall( in emitTaskInit()
4351 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskInit()
4360 LValue EvtLVal = CGF.EmitLValue(Evt); in emitTaskInit()
4364 llvm::Value *Loc = emitUpdateLocation(CGF, DC->getBeginLoc()); in emitTaskInit()
4365 llvm::Value *Tid = getThreadID(CGF, DC->getBeginLoc()); in emitTaskInit()
4366 Tid = CGF.Builder.CreateIntCast(Tid, CGF.IntTy, /*isSigned=*/false); in emitTaskInit()
4367 llvm::Value *EvtVal = CGF.EmitRuntimeCall( in emitTaskInit()
4371 EvtVal = CGF.EmitScalarConversion(EvtVal, C.VoidPtrTy, Evt->getType(), in emitTaskInit()
4373 CGF.EmitStoreOfScalar(EvtVal, EvtLVal); in emitTaskInit()
4387 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper); in emitTaskInit()
4388 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false); in emitTaskInit()
4390 NumOfElements ? CGF.Builder.CreateNUWMul(NumOfElements, Sz) : Sz; in emitTaskInit()
4402 NumOfElements = CGF.Builder.CreateNUWAdd( in emitTaskInit()
4403 llvm::ConstantInt::get(CGF.SizeTy, NumAffinities), NumOfElements); in emitTaskInit()
4408 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, in emitTaskInit()
4416 CGF.EmitVarDecl(*PD); in emitTaskInit()
4417 AffinitiesArray = CGF.GetAddrOfLocalVar(PD); in emitTaskInit()
4418 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty, in emitTaskInit()
4426 CGF.CreateMemTemp(KmpTaskAffinityInfoArrayTy, ".affs.arr.addr"); in emitTaskInit()
4427 AffinitiesArray = CGF.Builder.CreateConstArrayGEP(AffinitiesArray, 0); in emitTaskInit()
4444 std::tie(Addr, Size) = getPointerAndSize(CGF, E); in emitTaskInit()
4446 CGF.MakeAddrLValue(CGF.Builder.CreateConstGEP(AffinitiesArray, Pos), in emitTaskInit()
4449 LValue BaseAddrLVal = CGF.EmitLValueForField( in emitTaskInit()
4451 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy), in emitTaskInit()
4454 LValue LenLVal = CGF.EmitLValueForField( in emitTaskInit()
4456 CGF.EmitStoreOfScalar(Size, LenLVal); in emitTaskInit()
4462 PosLVal = CGF.MakeAddrLValue( in emitTaskInit()
4463 CGF.CreateMemTemp(C.getSizeType(), "affs.counter.addr"), in emitTaskInit()
4465 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal); in emitTaskInit()
4473 CGF, cast_or_null<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts())); in emitTaskInit()
4477 std::tie(Addr, Size) = getPointerAndSize(CGF, E); in emitTaskInit()
4478 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); in emitTaskInit()
4479 LValue Base = CGF.MakeAddrLValue( in emitTaskInit()
4480 Address(CGF.Builder.CreateGEP(AffinitiesArray.getElementType(), in emitTaskInit()
4485 LValue BaseAddrLVal = CGF.EmitLValueForField( in emitTaskInit()
4487 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy), in emitTaskInit()
4490 LValue LenLVal = CGF.EmitLValueForField( in emitTaskInit()
4492 CGF.EmitStoreOfScalar(Size, LenLVal); in emitTaskInit()
4493 Idx = CGF.Builder.CreateNUWAdd( in emitTaskInit()
4495 CGF.EmitStoreOfScalar(Idx, PosLVal); in emitTaskInit()
4501 llvm::Value *LocRef = emitUpdateLocation(CGF, Loc); in emitTaskInit()
4502 llvm::Value *GTid = getThreadID(CGF, Loc); in emitTaskInit()
4503 llvm::Value *AffinListPtr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
4507 (void)CGF.EmitRuntimeCall( in emitTaskInit()
4513 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
4515 LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy, in emitTaskInit()
4518 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin()); in emitTaskInit()
4524 Address(CGF.EmitLoadOfScalar( in emitTaskInit()
4525 CGF.EmitLValueForField( in emitTaskInit()
4530 LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy); in emitTaskInit()
4531 LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy); in emitTaskInit()
4532 CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap); in emitTaskInit()
4537 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD, in emitTaskInit()
4541 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) { in emitTaskInit()
4558 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI); in emitTaskInit()
4559 LValue DestructorsLV = CGF.EmitLValueForField( in emitTaskInit()
4561 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
4567 LValue Data2LV = CGF.EmitLValueForField( in emitTaskInit()
4569 LValue PriorityLV = CGF.EmitLValueForField( in emitTaskInit()
4571 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV); in emitTaskInit()
4632 CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal, in getDepobjElements() argument
4639 LValue Base = CGF.EmitLoadOfPointerLValue( in getDepobjElements()
4640 DepobjLVal.getAddress(CGF), in getDepobjElements()
4643 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getDepobjElements()
4644 Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy)); in getDepobjElements()
4645 Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(), in getDepobjElements()
4647 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP( in getDepobjElements()
4649 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); in getDepobjElements()
4650 LValue NumDepsBase = CGF.MakeAddrLValue( in getDepobjElements()
4654 LValue BaseAddrLVal = CGF.EmitLValueForField( in getDepobjElements()
4656 llvm::Value *NumDeps = CGF.EmitLoadOfScalar(BaseAddrLVal, Loc); in getDepobjElements()
4660 static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy, in emitDependData() argument
4664 CodeGenModule &CGM = CGF.CGM; in emitDependData()
4670 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy); in emitDependData()
4673 CGF, cast_or_null<OMPIteratorExpr>( in emitDependData()
4679 std::tie(Addr, Size) = getPointerAndSize(CGF, E); in emitDependData()
4682 Base = CGF.MakeAddrLValue( in emitDependData()
4683 CGF.Builder.CreateConstGEP(DependenciesArray, *P), KmpDependInfoTy); in emitDependData()
4686 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); in emitDependData()
4687 Base = CGF.MakeAddrLValue( in emitDependData()
4688 Address(CGF.Builder.CreateGEP(DependenciesArray.getElementType(), in emitDependData()
4694 LValue BaseAddrLVal = CGF.EmitLValueForField( in emitDependData()
4696 CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy), in emitDependData()
4699 LValue LenLVal = CGF.EmitLValueForField( in emitDependData()
4701 CGF.EmitStoreOfScalar(Size, LenLVal); in emitDependData()
4704 LValue FlagsLVal = CGF.EmitLValueForField( in emitDependData()
4706 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind), in emitDependData()
4712 llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); in emitDependData()
4713 Idx = CGF.Builder.CreateNUWAdd(Idx, in emitDependData()
4715 CGF.EmitStoreOfScalar(Idx, PosLVal); in emitDependData()
4721 emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy, in emitDepobjElementsSizes() argument
4727 ASTContext &C = CGF.getContext(); in emitDepobjElementsSizes()
4733 llvm::Type *KmpDependInfoPtrT = CGF.ConvertTypeForMem(KmpDependInfoPtrTy); in emitDepobjElementsSizes()
4736 CGF, cast_or_null<OMPIteratorExpr>( in emitDepobjElementsSizes()
4740 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts()); in emitDepobjElementsSizes()
4741 LValue Base = CGF.EmitLoadOfPointerLValue( in emitDepobjElementsSizes()
4742 DepobjLVal.getAddress(CGF), in emitDepobjElementsSizes()
4744 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDepobjElementsSizes()
4745 Base.getAddress(CGF), KmpDependInfoPtrT); in emitDepobjElementsSizes()
4746 Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(), in emitDepobjElementsSizes()
4748 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP( in emitDepobjElementsSizes()
4750 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); in emitDepobjElementsSizes()
4751 LValue NumDepsBase = CGF.MakeAddrLValue( in emitDepobjElementsSizes()
4755 LValue BaseAddrLVal = CGF.EmitLValueForField( in emitDepobjElementsSizes()
4758 CGF.EmitLoadOfScalar(BaseAddrLVal, E->getExprLoc()); in emitDepobjElementsSizes()
4759 LValue NumLVal = CGF.MakeAddrLValue( in emitDepobjElementsSizes()
4760 CGF.CreateMemTemp(C.getUIntPtrType(), "depobj.size.addr"), in emitDepobjElementsSizes()
4762 CGF.InitTempAlloca(NumLVal.getAddress(CGF), in emitDepobjElementsSizes()
4763 llvm::ConstantInt::get(CGF.IntPtrTy, 0)); in emitDepobjElementsSizes()
4764 llvm::Value *PrevVal = CGF.EmitLoadOfScalar(NumLVal, E->getExprLoc()); in emitDepobjElementsSizes()
4765 llvm::Value *Add = CGF.Builder.CreateNUWAdd(PrevVal, NumDeps); in emitDepobjElementsSizes()
4766 CGF.EmitStoreOfScalar(Add, NumLVal); in emitDepobjElementsSizes()
4772 CGF.EmitLoadOfScalar(SizeLVals[I], Data.DepExprs[I]->getExprLoc()); in emitDepobjElementsSizes()
4778 static void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy, in emitDepobjElements() argument
4784 ASTContext &C = CGF.getContext(); in emitDepobjElements()
4790 llvm::Type *KmpDependInfoPtrT = CGF.ConvertTypeForMem(KmpDependInfoPtrTy); in emitDepobjElements()
4791 llvm::Value *ElSize = CGF.getTypeSize(KmpDependInfoTy); in emitDepobjElements()
4794 CGF, cast_or_null<OMPIteratorExpr>( in emitDepobjElements()
4799 LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts()); in emitDepobjElements()
4800 LValue Base = CGF.EmitLoadOfPointerLValue( in emitDepobjElements()
4801 DepobjLVal.getAddress(CGF), in emitDepobjElements()
4803 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDepobjElements()
4804 Base.getAddress(CGF), KmpDependInfoPtrT); in emitDepobjElements()
4805 Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(), in emitDepobjElements()
4809 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP( in emitDepobjElements()
4811 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); in emitDepobjElements()
4812 LValue NumDepsBase = CGF.MakeAddrLValue( in emitDepobjElements()
4816 LValue BaseAddrLVal = CGF.EmitLValueForField( in emitDepobjElements()
4819 CGF.EmitLoadOfScalar(BaseAddrLVal, E->getExprLoc()); in emitDepobjElements()
4822 llvm::Value *Size = CGF.Builder.CreateNUWMul( in emitDepobjElements()
4824 CGF.Builder.CreateIntCast(NumDeps, CGF.SizeTy, /*isSigned=*/false)); in emitDepobjElements()
4825 llvm::Value *Pos = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); in emitDepobjElements()
4827 Address(CGF.Builder.CreateGEP(DependenciesArray.getElementType(), in emitDepobjElements()
4830 CGF.Builder.CreateMemCpy(DepAddr, Base.getAddress(CGF), Size); in emitDepobjElements()
4834 llvm::Value *Add = CGF.Builder.CreateNUWAdd(Pos, NumDeps); in emitDepobjElements()
4835 CGF.EmitStoreOfScalar(Add, PosLVal); in emitDepobjElements()
4841 CodeGenFunction &CGF, ArrayRef<OMPTaskDataTy::DependData> Dependencies, in emitDependClause() argument
4862 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.IntPtrTy, 0); in emitDependClause()
4864 llvm::ConstantInt::get(CGF.IntPtrTy, 1); in emitDependClause()
4869 emitDepobjElementsSizes(CGF, KmpDependInfoTy, D); in emitDependClause()
4872 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, Size); in emitDependClause()
4880 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper); in emitDependClause()
4881 Sz = CGF.Builder.CreateIntCast(Sz, CGF.IntPtrTy, /*isSigned=*/false); in emitDependClause()
4883 CGF.Builder.CreateNUWMul(NumOfRegularWithIterators, Sz); in emitDependClause()
4896 CGF.Builder.CreateNUWAdd(NumOfDepobjElements, NumOfElements); in emitDependClause()
4900 CGF.Builder.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements); in emitDependClause()
4905 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, in emitDependClause()
4914 CGF.EmitVarDecl(*PD); in emitDependClause()
4915 DependenciesArray = CGF.GetAddrOfLocalVar(PD); in emitDependClause()
4916 NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty, in emitDependClause()
4923 CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr"); in emitDependClause()
4924 DependenciesArray = CGF.Builder.CreateConstArrayGEP(DependenciesArray, 0); in emitDependClause()
4933 emitDependData(CGF, KmpDependInfoTy, &Pos, Dependencies[I], in emitDependClause()
4937 LValue PosLVal = CGF.MakeAddrLValue( in emitDependClause()
4938 CGF.CreateMemTemp(C.getSizeType(), "dep.counter.addr"), C.getSizeType()); in emitDependClause()
4939 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal); in emitDependClause()
4944 emitDependData(CGF, KmpDependInfoTy, &PosLVal, Dependencies[I], in emitDependClause()
4952 emitDepobjElements(CGF, KmpDependInfoTy, PosLVal, Dependencies[I], in emitDependClause()
4956 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDependClause()
4957 DependenciesArray, CGF.VoidPtrTy); in emitDependClause()
4962 CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies, in emitDepobjDependClause() argument
4984 NumDepsVal = llvm::ConstantInt::get(CGF.SizeTy, 1); in emitDepobjDependClause()
4986 llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper); in emitDepobjDependClause()
4987 Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false); in emitDepobjDependClause()
4988 NumDepsVal = CGF.Builder.CreateNUWMul(NumDepsVal, Sz); in emitDepobjDependClause()
4990 Size = CGF.Builder.CreateNUWAdd(llvm::ConstantInt::get(CGF.SizeTy, 1), in emitDepobjDependClause()
4995 Size = CGF.Builder.CreateNUWMul(Size, RecSize); in emitDepobjDependClause()
4997 CGF.Builder.CreateIntCast(NumDepsVal, CGF.IntPtrTy, /*isSigned=*/false); in emitDepobjDependClause()
5004 NumDepsVal = llvm::ConstantInt::get(CGF.IntPtrTy, NumDependencies); in emitDepobjDependClause()
5007 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitDepobjDependClause()
5009 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitDepobjDependClause()
5013 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitDepobjDependClause()
5016 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDepobjDependClause()
5017 Addr, CGF.ConvertTypeForMem(KmpDependInfoTy)->getPointerTo()); in emitDepobjDependClause()
5020 LValue Base = CGF.MakeAddrLValue(DependenciesArray, KmpDependInfoTy); in emitDepobjDependClause()
5022 LValue BaseAddrLVal = CGF.EmitLValueForField( in emitDepobjDependClause()
5024 CGF.EmitStoreOfScalar(NumDepsVal, BaseAddrLVal); in emitDepobjDependClause()
5029 PosLVal = CGF.MakeAddrLValue( in emitDepobjDependClause()
5030 CGF.CreateMemTemp(C.getSizeType(), "iterator.counter.addr"), in emitDepobjDependClause()
5032 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Idx), PosLVal, in emitDepobjDependClause()
5038 emitDependData(CGF, KmpDependInfoTy, Pos, Dependencies, DependenciesArray); in emitDepobjDependClause()
5039 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDepobjDependClause()
5040 CGF.Builder.CreateConstGEP(DependenciesArray, 1), CGF.VoidPtrTy); in emitDepobjDependClause()
5044 void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal, in emitDestroyClause() argument
5049 LValue Base = CGF.EmitLoadOfPointerLValue( in emitDestroyClause()
5050 DepobjLVal.getAddress(CGF), in emitDestroyClause()
5053 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDestroyClause()
5054 Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy)); in emitDestroyClause()
5055 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP( in emitDestroyClause()
5057 llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); in emitDestroyClause()
5058 DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(DepObjAddr, in emitDestroyClause()
5059 CGF.VoidPtrTy); in emitDestroyClause()
5060 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitDestroyClause()
5062 llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitDestroyClause()
5066 (void)CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitDestroyClause()
5071 void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, in emitUpdateClause() argument
5079 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy); in emitUpdateClause()
5082 std::tie(NumDeps, Base) = getDepobjElements(CGF, DepobjLVal, Loc); in emitUpdateClause()
5084 Address Begin = Base.getAddress(CGF); in emitUpdateClause()
5086 llvm::Value *End = CGF.Builder.CreateGEP( in emitUpdateClause()
5089 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.body"); in emitUpdateClause()
5090 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.done"); in emitUpdateClause()
5091 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock(); in emitUpdateClause()
5092 CGF.EmitBlock(BodyBB); in emitUpdateClause()
5094 CGF.Builder.CreatePHI(Begin.getType(), 2, "omp.elementPast"); in emitUpdateClause()
5097 Base = CGF.MakeAddrLValue(Begin, KmpDependInfoTy, Base.getBaseInfo(), in emitUpdateClause()
5101 LValue FlagsLVal = CGF.EmitLValueForField( in emitUpdateClause()
5103 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind), in emitUpdateClause()
5108 CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext"); in emitUpdateClause()
5110 CGF.Builder.GetInsertBlock()); in emitUpdateClause()
5112 CGF.Builder.CreateICmpEQ(ElementNext.getPointer(), End, "omp.isempty"); in emitUpdateClause()
5113 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); in emitUpdateClause()
5115 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); in emitUpdateClause()
5118 void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskCall() argument
5124 if (!CGF.HaveInsertPoint()) in emitTaskCall()
5128 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data); in emitTaskCall()
5138 emitDependClause(CGF, Data.Dependences, Loc); in emitTaskCall()
5146 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitTaskCall()
5147 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc); in emitTaskCall()
5156 DepTaskArgs[5] = CGF.Builder.getInt32(0); in emitTaskCall()
5157 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitTaskCall()
5160 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) { in emitTaskCall() argument
5163 LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI); in emitTaskCall()
5164 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal); in emitTaskCall()
5167 CGF.EmitRuntimeCall( in emitTaskCall()
5172 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskCall()
5178 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitTaskCall()
5179 Region->emitUntiedSwitch(CGF); in emitTaskCall()
5188 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0); in emitTaskCall()
5189 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitTaskCall()
5194 Loc](CodeGenFunction &CGF, PrePostActionTy &) { in emitTaskCall() argument
5195 CodeGenFunction::RunCleanupsScope LocalScope(CGF); in emitTaskCall()
5201 CGF.EmitRuntimeCall( in emitTaskCall()
5206 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTaskCall() argument
5207 Action.Enter(CGF); in emitTaskCall()
5209 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry, in emitTaskCall()
5225 RCG(CGF); in emitTaskCall()
5229 emitIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen); in emitTaskCall()
5232 ThenRCG(CGF); in emitTaskCall()
5236 void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskLoopCall() argument
5242 if (!CGF.HaveInsertPoint()) in emitTaskLoopCall()
5245 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data); in emitTaskLoopCall()
5251 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitTaskLoopCall()
5252 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc); in emitTaskLoopCall()
5255 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy, in emitTaskLoopCall()
5258 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1); in emitTaskLoopCall()
5261 LValue LBLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
5266 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(CGF), in emitTaskLoopCall()
5269 LValue UBLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
5274 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(CGF), in emitTaskLoopCall()
5277 LValue StLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
5282 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(CGF), in emitTaskLoopCall()
5286 LValue RedLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
5290 CGF.EmitStoreOfScalar(Data.Reductions, RedLVal); in emitTaskLoopCall()
5292 CGF.EmitNullInitialization(RedLVal.getAddress(CGF), in emitTaskLoopCall()
5293 CGF.getContext().VoidPtrTy); in emitTaskLoopCall()
5301 LBLVal.getPointer(CGF), in emitTaskLoopCall()
5302 UBLVal.getPointer(CGF), in emitTaskLoopCall()
5303 CGF.EmitLoadOfScalar(StLVal, Loc), in emitTaskLoopCall()
5305 CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler in emitTaskLoopCall()
5307 CGF.IntTy, Data.Schedule.getPointer() in emitTaskLoopCall()
5311 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty, in emitTaskLoopCall()
5313 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0), in emitTaskLoopCall()
5314 Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskLoopCall()
5315 Result.TaskDupFn, CGF.VoidPtrTy) in emitTaskLoopCall()
5316 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)}; in emitTaskLoopCall()
5317 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskLoopCall()
5332 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar, in EmitOMPAggregateReduction() argument
5334 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *, in EmitOMPAggregateReduction() argument
5340 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar); in EmitOMPAggregateReduction()
5341 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar); in EmitOMPAggregateReduction()
5345 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr); in EmitOMPAggregateReduction()
5351 CGF.Builder.CreateGEP(LHSAddr.getElementType(), LHSBegin, NumElements); in EmitOMPAggregateReduction()
5353 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body"); in EmitOMPAggregateReduction()
5354 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done"); in EmitOMPAggregateReduction()
5356 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty"); in EmitOMPAggregateReduction()
5357 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); in EmitOMPAggregateReduction()
5360 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock(); in EmitOMPAggregateReduction()
5361 CGF.EmitBlock(BodyBB); in EmitOMPAggregateReduction()
5363 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy); in EmitOMPAggregateReduction()
5365 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI( in EmitOMPAggregateReduction()
5372 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI( in EmitOMPAggregateReduction()
5380 CodeGenFunction::OMPPrivateScope Scope(CGF); in EmitOMPAggregateReduction()
5384 RedOpGen(CGF, XExpr, EExpr, UpExpr); in EmitOMPAggregateReduction()
5388 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateReduction()
5391 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateReduction()
5396 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done"); in EmitOMPAggregateReduction()
5397 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB); in EmitOMPAggregateReduction()
5398 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateReduction()
5399 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateReduction()
5402 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); in EmitOMPAggregateReduction()
5408 static void emitReductionCombiner(CodeGenFunction &CGF, in emitReductionCombiner() argument
5417 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD); in emitReductionCombiner()
5419 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func); in emitReductionCombiner()
5420 CGF.EmitIgnoredExpr(ReductionOp); in emitReductionCombiner()
5423 CGF.EmitIgnoredExpr(ReductionOp); in emitReductionCombiner()
5448 CodeGenFunction CGF(CGM); in emitReductionFunction() local
5449 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); in emitReductionFunction()
5453 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReductionFunction()
5454 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)), in emitReductionFunction()
5455 ArgsType), CGF.getPointerAlign()); in emitReductionFunction()
5456 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReductionFunction()
5457 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)), in emitReductionFunction()
5458 ArgsType), CGF.getPointerAlign()); in emitReductionFunction()
5463 CodeGenFunction::OMPPrivateScope Scope(CGF); in emitReductionFunction()
5469 Scope.addPrivate(RHSVar, [&CGF, RHS, Idx, RHSVar]() { in emitReductionFunction()
5470 return emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar); in emitReductionFunction()
5474 Scope.addPrivate(LHSVar, [&CGF, LHS, Idx, LHSVar]() { in emitReductionFunction()
5475 return emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar); in emitReductionFunction()
5481 Address Elem = CGF.Builder.CreateConstArrayGEP(LHS, Idx); in emitReductionFunction()
5482 llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem); in emitReductionFunction()
5484 CGF.getContext().getAsVariableArrayType(PrivTy); in emitReductionFunction()
5487 CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy))); in emitReductionFunction()
5488 CGF.EmitVariablyModifiedType(PrivTy); in emitReductionFunction()
5501 CGF, (*IPriv)->getType(), LHSVar, RHSVar, in emitReductionFunction()
5502 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) { in emitReductionFunction() argument
5503 emitReductionCombiner(CGF, E); in emitReductionFunction()
5507 emitReductionCombiner(CGF, E); in emitReductionFunction()
5514 CGF.FinishFunction(); in emitReductionFunction()
5518 void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF, in emitSingleReductionCombiner() argument
5528 CGF, PrivateRef->getType(), LHSVar, RHSVar, in emitSingleReductionCombiner()
5529 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) { in emitSingleReductionCombiner() argument
5530 emitReductionCombiner(CGF, ReductionOp); in emitSingleReductionCombiner()
5534 emitReductionCombiner(CGF, ReductionOp); in emitSingleReductionCombiner()
5538 void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, in emitReduction() argument
5544 if (!CGF.HaveInsertPoint()) in emitReduction()
5588 CodeGenFunction::RunCleanupsScope Scope(CGF); in emitReduction()
5593 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), in emitReduction()
5615 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); in emitReduction()
5619 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitReduction()
5620 CGF.Builder.CreateStore( in emitReduction()
5621 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReduction()
5622 CGF.EmitLValue(RHSExprs[I]).getPointer(CGF), CGF.VoidPtrTy), in emitReduction()
5627 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); in emitReduction()
5628 llvm::Value *Size = CGF.Builder.CreateIntCast( in emitReduction()
5629 CGF.getVLASize( in emitReduction()
5630 CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) in emitReduction()
5632 CGF.SizeTy, /*isSigned=*/false); in emitReduction()
5633 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), in emitReduction()
5640 Loc, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), Privates, in emitReduction()
5649 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE); in emitReduction()
5650 llvm::Value *ThreadId = getThreadID(CGF, Loc); in emitReduction()
5651 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy); in emitReduction()
5652 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReduction()
5653 ReductionList.getPointer(), CGF.VoidPtrTy); in emitReduction()
5657 CGF.Builder.getInt32(RHSExprs.size()), // i32 <n> in emitReduction()
5663 llvm::Value *Res = CGF.EmitRuntimeCall( in emitReduction()
5670 llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(".omp.reduction.default"); in emitReduction()
5672 CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2); in emitReduction()
5680 llvm::BasicBlock *Case1BB = CGF.createBasicBlock(".omp.reduction.case1"); in emitReduction()
5681 SwInst->addCase(CGF.Builder.getInt32(1), Case1BB); in emitReduction()
5682 CGF.EmitBlock(Case1BB); in emitReduction()
5691 CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
5692 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitReduction()
5697 RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), in emitReduction()
5712 RCG(CGF); in emitReduction()
5714 CGF.EmitBranch(DefaultBB); in emitReduction()
5721 llvm::BasicBlock *Case2BB = CGF.createBasicBlock(".omp.reduction.case2"); in emitReduction()
5722 SwInst->addCase(CGF.Builder.getInt32(2), Case2BB); in emitReduction()
5723 CGF.EmitBlock(Case2BB); in emitReduction()
5726 CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
5760 Loc](CodeGenFunction &CGF, const Expr *XExpr, in emitReduction()
5762 LValue X = CGF.EmitLValue(XExpr); in emitReduction()
5765 E = CGF.EmitAnyExpr(EExpr); in emitReduction()
5766 CGF.EmitOMPAtomicSimpleUpdateExpr( in emitReduction()
5769 [&CGF, UpExpr, VD, Loc](RValue XRValue) { in emitReduction()
5770 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitReduction()
5772 VD, [&CGF, VD, XRValue, Loc]() { in emitReduction()
5773 Address LHSTemp = CGF.CreateMemTemp(VD->getType()); in emitReduction()
5774 CGF.emitOMPSimpleStore( in emitReduction()
5775 CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue, in emitReduction()
5780 return CGF.EmitAnyExpr(UpExpr); in emitReduction()
5787 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar, in emitReduction()
5791 AtomicRedGen(CGF, XExpr, EExpr, UpExpr); in emitReduction()
5795 auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *, in emitReduction()
5797 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitReduction()
5800 CGF, Name, in emitReduction()
5801 [=](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
5802 Action.Enter(CGF); in emitReduction()
5803 emitReductionCombiner(CGF, E); in emitReduction()
5812 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar, in emitReduction()
5815 CritRedGen(CGF, nullptr, nullptr, nullptr); in emitReduction()
5836 AtomicRCG(CGF); in emitReduction()
5838 AtomicRCG(CGF); in emitReduction()
5841 CGF.EmitBranch(DefaultBB); in emitReduction()
5842 CGF.EmitBlock(DefaultBB, /*IsFinished=*/true); in emitReduction()
5892 CodeGenFunction CGF(CGM); in emitReduceInitFunction() local
5893 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc); in emitReduceInitFunction()
5894 Address PrivateAddr = CGF.EmitLoadOfPointer( in emitReduceInitFunction()
5895 CGF.GetAddrOfLocalVar(&Param), in emitReduceInitFunction()
5902 CGF, CGM.getContext().getSizeType(), in emitReduceInitFunction()
5904 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false, in emitReduceInitFunction()
5907 RCG.emitAggregateType(CGF, N, Size); in emitReduceInitFunction()
5913 Address SharedAddr = CGF.GetAddrOfLocalVar(&ParamOrig); in emitReduceInitFunction()
5914 SharedAddr = CGF.EmitLoadOfPointer( in emitReduceInitFunction()
5917 OrigLVal = CGF.MakeAddrLValue(SharedAddr, CGM.getContext().VoidPtrTy); in emitReduceInitFunction()
5919 OrigLVal = CGF.MakeNaturalAlignAddrLValue( in emitReduceInitFunction()
5926 RCG.emitInitialization(CGF, N, PrivateAddr, OrigLVal, in emitReduceInitFunction()
5928 CGF.FinishFunction(); in emitReduceInitFunction()
5966 CodeGenFunction CGF(CGM); in emitReduceCombFunction() local
5967 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc); in emitReduceCombFunction()
5973 CGF, CGM.getContext().getSizeType(), in emitReduceCombFunction()
5975 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false, in emitReduceCombFunction()
5978 RCG.emitAggregateType(CGF, N, Size); in emitReduceCombFunction()
5982 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitReduceCombFunction()
5983 PrivateScope.addPrivate(LHSVD, [&C, &CGF, &ParamInOut, LHSVD]() { in emitReduceCombFunction()
5985 Address PtrAddr = CGF.EmitLoadOfPointer( in emitReduceCombFunction()
5986 CGF.GetAddrOfLocalVar(&ParamInOut), in emitReduceCombFunction()
5988 return CGF.Builder.CreateElementBitCast( in emitReduceCombFunction()
5989 PtrAddr, CGF.ConvertTypeForMem(LHSVD->getType())); in emitReduceCombFunction()
5991 PrivateScope.addPrivate(RHSVD, [&C, &CGF, &ParamIn, RHSVD]() { in emitReduceCombFunction()
5993 Address PtrAddr = CGF.EmitLoadOfPointer( in emitReduceCombFunction()
5994 CGF.GetAddrOfLocalVar(&ParamIn), in emitReduceCombFunction()
5996 return CGF.Builder.CreateElementBitCast( in emitReduceCombFunction()
5997 PtrAddr, CGF.ConvertTypeForMem(RHSVD->getType())); in emitReduceCombFunction()
6004 CGF, ReductionOp, PrivateRef, cast<DeclRefExpr>(LHS), in emitReduceCombFunction()
6006 CGF.FinishFunction(); in emitReduceCombFunction()
6036 CodeGenFunction CGF(CGM); in emitReduceFiniFunction() local
6037 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc); in emitReduceFiniFunction()
6038 Address PrivateAddr = CGF.EmitLoadOfPointer( in emitReduceFiniFunction()
6039 CGF.GetAddrOfLocalVar(&Param), in emitReduceFiniFunction()
6046 CGF, CGM.getContext().getSizeType(), in emitReduceFiniFunction()
6048 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false, in emitReduceFiniFunction()
6051 RCG.emitAggregateType(CGF, N, Size); in emitReduceFiniFunction()
6054 RCG.emitCleanups(CGF, N, PrivateAddr); in emitReduceFiniFunction()
6055 CGF.FinishFunction(Loc); in emitReduceFiniFunction()
6060 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs, in emitTaskReductionInit() argument
6062 if (!CGF.HaveInsertPoint() || Data.ReductionVars.empty()) in emitTaskReductionInit()
6093 Address TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input."); in emitTaskReductionInit()
6100 llvm::Value *GEP = CGF.EmitCheckedInBoundsGEP( in emitTaskReductionInit()
6104 LValue ElemLVal = CGF.MakeNaturalAlignAddrLValue(GEP, RDType); in emitTaskReductionInit()
6106 LValue SharedLVal = CGF.EmitLValueForField(ElemLVal, SharedFD); in emitTaskReductionInit()
6107 RCG.emitSharedOrigLValue(CGF, Cnt); in emitTaskReductionInit()
6109 CGF.EmitCastToVoidPtr(RCG.getSharedLValue(Cnt).getPointer(CGF)); in emitTaskReductionInit()
6110 CGF.EmitStoreOfScalar(CastedShared, SharedLVal); in emitTaskReductionInit()
6112 LValue OrigLVal = CGF.EmitLValueForField(ElemLVal, OrigFD); in emitTaskReductionInit()
6114 CGF.EmitCastToVoidPtr(RCG.getOrigLValue(Cnt).getPointer(CGF)); in emitTaskReductionInit()
6115 CGF.EmitStoreOfScalar(CastedOrig, OrigLVal); in emitTaskReductionInit()
6116 RCG.emitAggregateType(CGF, Cnt); in emitTaskReductionInit()
6126 SizeValInChars = CGF.Builder.CreateIntCast(SizeValInChars, CGM.SizeTy, in emitTaskReductionInit()
6128 LValue SizeLVal = CGF.EmitLValueForField(ElemLVal, SizeFD); in emitTaskReductionInit()
6129 CGF.EmitStoreOfScalar(SizeValInChars, SizeLVal); in emitTaskReductionInit()
6131 LValue InitLVal = CGF.EmitLValueForField(ElemLVal, InitFD); in emitTaskReductionInit()
6133 CGF.EmitCastToVoidPtr(emitReduceInitFunction(CGM, Loc, RCG, Cnt)); in emitTaskReductionInit()
6134 CGF.EmitStoreOfScalar(InitAddr, InitLVal); in emitTaskReductionInit()
6136 LValue FiniLVal = CGF.EmitLValueForField(ElemLVal, FiniFD); in emitTaskReductionInit()
6139 ? CGF.EmitCastToVoidPtr(Fini) in emitTaskReductionInit()
6141 CGF.EmitStoreOfScalar(FiniAddr, FiniLVal); in emitTaskReductionInit()
6143 LValue CombLVal = CGF.EmitLValueForField(ElemLVal, CombFD); in emitTaskReductionInit()
6144 llvm::Value *CombAddr = CGF.EmitCastToVoidPtr(emitReduceCombFunction( in emitTaskReductionInit()
6147 CGF.EmitStoreOfScalar(CombAddr, CombLVal); in emitTaskReductionInit()
6149 LValue FlagsLVal = CGF.EmitLValueForField(ElemLVal, FlagsFD); in emitTaskReductionInit()
6151 CGF.EmitStoreOfScalar( in emitTaskReductionInit()
6155 CGF.EmitNullInitialization(FlagsLVal.getAddress(CGF), in emitTaskReductionInit()
6161 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc); in emitTaskReductionInit()
6162 llvm::Value *GTid = CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), in emitTaskReductionInit()
6169 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskReductionInit()
6171 return CGF.EmitRuntimeCall( in emitTaskReductionInit()
6178 CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), CGM.IntTy, in emitTaskReductionInit()
6181 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskRedInput.getPointer(), in emitTaskReductionInit()
6183 return CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskReductionInit()
6188 void CGOpenMPRuntime::emitTaskReductionFini(CodeGenFunction &CGF, in emitTaskReductionFini() argument
6193 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc); in emitTaskReductionFini()
6194 llvm::Value *GTid = CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), in emitTaskReductionFini()
6200 (void)CGF.EmitRuntimeCall( in emitTaskReductionFini()
6206 void CGOpenMPRuntime::emitTaskReductionFixups(CodeGenFunction &CGF, in emitTaskReductionFixups() argument
6214 llvm::Value *SizeVal = CGF.Builder.CreateIntCast(Sizes.second, CGM.SizeTy, in emitTaskReductionFixups()
6217 CGF, CGM.getContext().getSizeType(), in emitTaskReductionFixups()
6219 CGF.Builder.CreateStore(SizeVal, SizeAddr, /*IsVolatile=*/false); in emitTaskReductionFixups()
6223 Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF, in getTaskReductionItem() argument
6229 llvm::Value *Args[] = {CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), in getTaskReductionItem()
6233 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getTaskReductionItem()
6234 SharedLVal.getPointer(CGF), CGM.VoidPtrTy)}; in getTaskReductionItem()
6236 CGF.EmitRuntimeCall( in getTaskReductionItem()
6243 void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, in emitTaskwaitCall() argument
6245 if (!CGF.HaveInsertPoint()) in emitTaskwaitCall()
6248 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) { in emitTaskwaitCall()
6249 OMPBuilder.createTaskwait(CGF.Builder); in emitTaskwaitCall()
6253 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitTaskwaitCall()
6255 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitTaskwaitCall()
6260 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitTaskwaitCall()
6261 Region->emitUntiedSwitch(CGF); in emitTaskwaitCall()
6264 void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF, in emitInlinedDirective() argument
6268 if (!CGF.HaveInsertPoint()) in emitInlinedDirective()
6270 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel, in emitInlinedDirective()
6274 CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr); in emitInlinedDirective()
6303 CodeGenFunction &CGF, SourceLocation Loc, in emitCancellationPointCall() argument
6305 if (!CGF.HaveInsertPoint()) in emitCancellationPointCall()
6310 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { in emitCancellationPointCall()
6315 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitCancellationPointCall()
6316 CGF.Builder.getInt32(getCancellationKind(CancelRegion))}; in emitCancellationPointCall()
6318 llvm::Value *Result = CGF.EmitRuntimeCall( in emitCancellationPointCall()
6326 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit"); in emitCancellationPointCall()
6327 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue"); in emitCancellationPointCall()
6328 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result); in emitCancellationPointCall()
6329 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); in emitCancellationPointCall()
6330 CGF.EmitBlock(ExitBB); in emitCancellationPointCall()
6332 emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false); in emitCancellationPointCall()
6335 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); in emitCancellationPointCall()
6336 CGF.EmitBranchThroughCleanup(CancelDest); in emitCancellationPointCall()
6337 CGF.EmitBlock(ContBB, /*IsFinished=*/true); in emitCancellationPointCall()
6342 void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, in emitCancelCall() argument
6345 if (!CGF.HaveInsertPoint()) in emitCancelCall()
6351 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { in emitCancelCall()
6353 OMPRegionInfo](CodeGenFunction &CGF, PrePostActionTy &) { in emitCancelCall() argument
6354 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); in emitCancelCall()
6356 RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc), in emitCancelCall()
6357 CGF.Builder.getInt32(getCancellationKind(CancelRegion))}; in emitCancelCall()
6359 llvm::Value *Result = CGF.EmitRuntimeCall( in emitCancelCall()
6365 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit"); in emitCancelCall()
6366 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue"); in emitCancelCall()
6367 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result); in emitCancelCall()
6368 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); in emitCancelCall()
6369 CGF.EmitBlock(ExitBB); in emitCancelCall()
6371 RT.emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false); in emitCancelCall()
6374 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); in emitCancelCall()
6375 CGF.EmitBranchThroughCleanup(CancelDest); in emitCancelCall()
6376 CGF.EmitBlock(ContBB, /*IsFinished=*/true); in emitCancelCall()
6379 emitIfClause(CGF, IfCond, ThenGen, in emitCancelCall()
6383 ThenRCG(CGF); in emitCancelCall()
6397 void Enter(CodeGenFunction &CGF) override { in Enter() argument
6398 if (!CGF.HaveInsertPoint()) in Enter()
6401 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsInit( in Enter()
6402 CGF, AllocatorData.first, AllocatorData.second); in Enter()
6405 void Exit(CodeGenFunction &CGF) override { in Exit() argument
6406 if (!CGF.HaveInsertPoint()) in Exit()
6409 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsFini(CGF, in Exit()
6437 void CGOpenMPRuntime::emitUsesAllocatorsInit(CodeGenFunction &CGF, in emitUsesAllocatorsInit() argument
6440 llvm::Value *ThreadId = getThreadID(CGF, Allocator->getExprLoc()); in emitUsesAllocatorsInit()
6441 ThreadId = CGF.Builder.CreateIntCast(ThreadId, CGF.IntTy, /*isSigned=*/true); in emitUsesAllocatorsInit()
6443 llvm::Value *MemSpaceHandle = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitUsesAllocatorsInit()
6445 CGF.IntTy, cast<ConstantArrayType>( in emitUsesAllocatorsInit()
6449 LValue AllocatorTraitsLVal = CGF.EmitLValue(AllocatorTraits); in emitUsesAllocatorsInit()
6450 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitUsesAllocatorsInit()
6451 AllocatorTraitsLVal.getAddress(CGF), CGF.VoidPtrPtrTy); in emitUsesAllocatorsInit()
6452 AllocatorTraitsLVal = CGF.MakeAddrLValue(Addr, CGF.getContext().VoidPtrTy, in emitUsesAllocatorsInit()
6456 CGF.EmitLoadOfScalar(AllocatorTraitsLVal, AllocatorTraits->getExprLoc()); in emitUsesAllocatorsInit()
6459 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitUsesAllocatorsInit()
6463 CGF.EmitVarDecl(*cast<VarDecl>( in emitUsesAllocatorsInit()
6465 LValue AllocatorLVal = CGF.EmitLValue(Allocator->IgnoreParenImpCasts()); in emitUsesAllocatorsInit()
6467 CGF.EmitScalarConversion(AllocatorVal, CGF.getContext().VoidPtrTy, in emitUsesAllocatorsInit()
6469 CGF.EmitStoreOfScalar(AllocatorVal, AllocatorLVal); in emitUsesAllocatorsInit()
6472 void CGOpenMPRuntime::emitUsesAllocatorsFini(CodeGenFunction &CGF, in emitUsesAllocatorsFini() argument
6474 llvm::Value *ThreadId = getThreadID(CGF, Allocator->getExprLoc()); in emitUsesAllocatorsFini()
6475 ThreadId = CGF.Builder.CreateIntCast(ThreadId, CGF.IntTy, /*isSigned=*/true); in emitUsesAllocatorsFini()
6476 LValue AllocatorLVal = CGF.EmitLValue(Allocator->IgnoreParenImpCasts()); in emitUsesAllocatorsFini()
6478 CGF.EmitLoadOfScalar(AllocatorLVal, Allocator->getExprLoc()); in emitUsesAllocatorsFini()
6479 AllocatorVal = CGF.EmitScalarConversion(AllocatorVal, Allocator->getType(), in emitUsesAllocatorsFini()
6480 CGF.getContext().VoidPtrTy, in emitUsesAllocatorsFini()
6482 (void)CGF.EmitRuntimeCall( in emitUsesAllocatorsFini()
6515 CodeGenFunction CGF(CGM, true); in emitTargetOutlinedFunctionHelper() local
6517 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitTargetOutlinedFunctionHelper()
6519 OutlinedFn = CGF.GenerateOpenMPCapturedStmtFunction(CS, D.getBeginLoc()); in emitTargetOutlinedFunctionHelper()
6558 getNumTeamsExprForTargetDirective(CGF, D, DefaultValTeams); in emitTargetOutlinedFunctionHelper()
6564 getNumThreadsExprForTargetDirective(CGF, D, DefaultValThreads); in emitTargetOutlinedFunctionHelper()
6624 CodeGenFunction &CGF, const OMPExecutableDirective &D, in getNumTeamsExprForTargetDirective() argument
6636 CGOpenMPRuntime::getSingleCompoundChild(CGF.getContext(), Body); in getNumTeamsExprForTargetDirective()
6643 if (NumTeams->isIntegerConstantExpr(CGF.getContext())) in getNumTeamsExprForTargetDirective()
6645 NumTeams->getIntegerConstantExpr(CGF.getContext())) in getNumTeamsExprForTargetDirective()
6672 if (NumTeams->isIntegerConstantExpr(CGF.getContext())) in getNumTeamsExprForTargetDirective()
6673 if (auto Constant = NumTeams->getIntegerConstantExpr(CGF.getContext())) in getNumTeamsExprForTargetDirective()
6752 CodeGenFunction &CGF, const OMPExecutableDirective &D) { in emitNumTeamsForTargetDirective() argument
6753 assert(!CGF.getLangOpts().OpenMPIsDevice && in emitNumTeamsForTargetDirective()
6756 CGBuilderTy &Bld = CGF.Builder; in emitNumTeamsForTargetDirective()
6758 const Expr *NumTeams = getNumTeamsExprForTargetDirective(CGF, D, DefaultNT); in emitNumTeamsForTargetDirective()
6765 CGOpenMPInnerExprInfo CGInfo(CGF, *CS); in emitNumTeamsForTargetDirective()
6766 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitNumTeamsForTargetDirective()
6767 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(NumTeams, in emitNumTeamsForTargetDirective()
6769 return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty, in emitNumTeamsForTargetDirective()
6777 CodeGenFunction::RunCleanupsScope NumTeamsScope(CGF); in emitNumTeamsForTargetDirective()
6778 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(NumTeams, in emitNumTeamsForTargetDirective()
6780 return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty, in emitNumTeamsForTargetDirective()
6793 static llvm::Value *getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS, in getNumThreads() argument
6796 CGF.getContext(), CS->getCapturedStmt()); in getNumThreads()
6804 CGOpenMPInnerExprInfo CGInfo(CGF, *CS); in getNumThreads()
6805 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in getNumThreads()
6817 if (Cond->EvaluateAsBooleanCondition(Result, CGF.getContext())) { in getNumThreads()
6819 return CGF.Builder.getInt32(1); in getNumThreads()
6821 CodeGenFunction::LexicalScope Scope(CGF, Cond->getSourceRange()); in getNumThreads()
6826 CGF.EmitVarDecl(cast<VarDecl>(*I)); in getNumThreads()
6829 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in getNumThreads()
6830 CGF.EmitAutoVarCleanups(Emission); in getNumThreads()
6834 CondVal = CGF.EvaluateExprAsBool(Cond); in getNumThreads()
6841 CGOpenMPInnerExprInfo CGInfo(CGF, *CS); in getNumThreads()
6842 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in getNumThreads()
6846 CGF, NumThreadsClause->getNumThreads()->getSourceRange()); in getNumThreads()
6851 CGF.EmitVarDecl(cast<VarDecl>(*I)); in getNumThreads()
6854 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in getNumThreads()
6855 CGF.EmitAutoVarCleanups(Emission); in getNumThreads()
6859 NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads()); in getNumThreads()
6860 NumThreads = CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, in getNumThreads()
6863 NumThreads = CGF.Builder.CreateSelect( in getNumThreads()
6864 CGF.Builder.CreateICmpULT(DefaultThreadLimitVal, NumThreads), in getNumThreads()
6868 : CGF.Builder.getInt32(0); in getNumThreads()
6872 NumThreads = CGF.Builder.CreateSelect(CondVal, NumThreads, in getNumThreads()
6873 CGF.Builder.getInt32(1)); in getNumThreads()
6878 return CGF.Builder.getInt32(1); in getNumThreads()
6882 : CGF.Builder.getInt32(0); in getNumThreads()
6886 CodeGenFunction &CGF, const OMPExecutableDirective &D, in getNumThreadsExprForTargetDirective() argument
6901 if (ThreadLimit->isIntegerConstantExpr(CGF.getContext())) in getNumThreadsExprForTargetDirective()
6903 ThreadLimit->getIntegerConstantExpr(CGF.getContext())) in getNumThreadsExprForTargetDirective()
6918 if (ThreadLimit->isIntegerConstantExpr(CGF.getContext())) in getNumThreadsExprForTargetDirective()
6920 ThreadLimit->getIntegerConstantExpr(CGF.getContext())) in getNumThreadsExprForTargetDirective()
6926 if (NumThreads->isIntegerConstantExpr(CGF.getContext())) { in getNumThreadsExprForTargetDirective()
6928 NumThreads->getIntegerConstantExpr(CGF.getContext())) { in getNumThreadsExprForTargetDirective()
7008 CodeGenFunction &CGF, const OMPExecutableDirective &D) { in emitNumThreadsForTargetDirective() argument
7009 assert(!CGF.getLangOpts().OpenMPIsDevice && in emitNumThreadsForTargetDirective()
7015 CGBuilderTy &Bld = CGF.Builder; in emitNumThreadsForTargetDirective()
7021 if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal)) in emitNumThreadsForTargetDirective()
7024 CGF.getContext(), CS->getCapturedStmt()); in emitNumThreadsForTargetDirective()
7027 CGOpenMPInnerExprInfo CGInfo(CGF, *CS); in emitNumThreadsForTargetDirective()
7028 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitNumThreadsForTargetDirective()
7032 CGF, ThreadLimitClause->getThreadLimit()->getSourceRange()); in emitNumThreadsForTargetDirective()
7037 CGF.EmitVarDecl(cast<VarDecl>(*I)); in emitNumThreadsForTargetDirective()
7040 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in emitNumThreadsForTargetDirective()
7041 CGF.EmitAutoVarCleanups(Emission); in emitNumThreadsForTargetDirective()
7045 llvm::Value *ThreadLimit = CGF.EmitScalarExpr( in emitNumThreadsForTargetDirective()
7048 Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false); in emitNumThreadsForTargetDirective()
7054 CGF.getContext(), CS->getCapturedStmt()); in emitNumThreadsForTargetDirective()
7060 if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal)) in emitNumThreadsForTargetDirective()
7070 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF); in emitNumThreadsForTargetDirective()
7072 llvm::Value *ThreadLimit = CGF.EmitScalarExpr( in emitNumThreadsForTargetDirective()
7075 Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false); in emitNumThreadsForTargetDirective()
7078 if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal)) in emitNumThreadsForTargetDirective()
7081 CGF.getContext(), CS->getCapturedStmt()); in emitNumThreadsForTargetDirective()
7085 if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal)) in emitNumThreadsForTargetDirective()
7093 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF); in emitNumThreadsForTargetDirective()
7095 llvm::Value *ThreadLimit = CGF.EmitScalarExpr( in emitNumThreadsForTargetDirective()
7098 Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false); in emitNumThreadsForTargetDirective()
7100 return getNumThreads(CGF, D.getInnermostCapturedStmt(), ThreadLimitVal); in emitNumThreadsForTargetDirective()
7121 if (Cond->EvaluateAsBooleanCondition(Result, CGF.getContext())) { in emitNumThreadsForTargetDirective()
7125 CodeGenFunction::RunCleanupsScope Scope(CGF); in emitNumThreadsForTargetDirective()
7126 CondVal = CGF.EvaluateExprAsBool(Cond); in emitNumThreadsForTargetDirective()
7131 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF); in emitNumThreadsForTargetDirective()
7133 llvm::Value *ThreadLimit = CGF.EmitScalarExpr( in emitNumThreadsForTargetDirective()
7136 Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false); in emitNumThreadsForTargetDirective()
7139 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); in emitNumThreadsForTargetDirective()
7141 llvm::Value *NumThreads = CGF.EmitScalarExpr( in emitNumThreadsForTargetDirective()
7144 Bld.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned=*/false); in emitNumThreadsForTargetDirective()
7436 CodeGenFunction &CGF; member in __anoneb75deaf3d11::MappableExprsHandler
7456 CGF.getTypeSize(OAE->getBase()->getType()->getPointeeType()); in getExprTypeSize()
7458 llvm::Value *Sz = CGF.EmitScalarExpr(SE); in getExprTypeSize()
7459 Sz = CGF.EmitScalarConversion(Sz, SE->getType(), in getExprTypeSize()
7460 CGF.getContext().getSizeType(), in getExprTypeSize()
7462 Size = CGF.Builder.CreateNUWMul(Size, Sz); in getExprTypeSize()
7484 return CGF.getTypeSize(BaseTy); in getExprTypeSize()
7488 ElemSize = CGF.getTypeSize(PTy->getPointeeType().getCanonicalType()); in getExprTypeSize()
7492 ElemSize = CGF.getTypeSize(ATy->getElementType().getCanonicalType()); in getExprTypeSize()
7501 llvm::Value *LengthVal = CGF.EmitScalarExpr(LenExpr); in getExprTypeSize()
7502 LengthVal = CGF.EmitScalarConversion(LengthVal, LenExpr->getType(), in getExprTypeSize()
7503 CGF.getContext().getSizeType(), in getExprTypeSize()
7505 return CGF.Builder.CreateNUWMul(LengthVal, ElemSize); in getExprTypeSize()
7510 llvm::Value *LengthVal = CGF.getTypeSize(BaseTy); in getExprTypeSize()
7511 llvm::Value *LBVal = CGF.EmitScalarExpr(OAE->getLowerBound()); in getExprTypeSize()
7512 LBVal = CGF.EmitScalarConversion(LBVal, OAE->getLowerBound()->getType(), in getExprTypeSize()
7513 CGF.getContext().getSizeType(), in getExprTypeSize()
7515 LBVal = CGF.Builder.CreateNUWMul(LBVal, ElemSize); in getExprTypeSize()
7516 llvm::Value *Cmp = CGF.Builder.CreateICmpUGT(LengthVal, LBVal); in getExprTypeSize()
7517 llvm::Value *TrueVal = CGF.Builder.CreateNUWSub(LengthVal, LBVal); in getExprTypeSize()
7518 LengthVal = CGF.Builder.CreateSelect( in getExprTypeSize()
7519 Cmp, TrueVal, llvm::ConstantInt::get(CGF.SizeTy, 0)); in getExprTypeSize()
7522 return CGF.getTypeSize(ExprTy); in getExprTypeSize()
7610 if (!Length->EvaluateAsInt(Result, CGF.getContext())) in isFinalArraySectionExpression()
7824 BP = CGF.LoadCXXThisAddress(); in generateInfoForComponentList()
7828 BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress(CGF); in generateInfoForComponentList()
7832 CGF.EmitScalarExpr(OAShE->getBase()), in generateInfoForComponentList()
7833 CGF.getContext().getTypeAlignInChars(OAShE->getBase()->getType())); in generateInfoForComponentList()
7837 BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress(CGF); in generateInfoForComponentList()
7844 CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) { in generateInfoForComponentList()
7846 BP = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD); in generateInfoForComponentList()
7861 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() || in generateInfoForComponentList()
7863 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>()); in generateInfoForComponentList()
7917 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>()); in generateInfoForComponentList()
7983 auto &&EmitMemberExprBase = [](CodeGenFunction &CGF, in generateInfoForComponentList()
7993 CGF.EmitPointerWithAlignment(BaseExpr, &BaseInfo, &TBAAInfo); in generateInfoForComponentList()
7995 BaseLV = CGF.MakeAddrLValue(Addr, PtrTy, BaseInfo, TBAAInfo); in generateInfoForComponentList()
7997 BaseLV = CGF.EmitOMPSharedLValue(BaseExpr); in generateInfoForComponentList()
8002 LowestElem = LB = Address(CGF.EmitScalarExpr(OAShE->getBase()), in generateInfoForComponentList()
8003 CGF.getContext().getTypeAlignInChars( in generateInfoForComponentList()
8007 LValue BaseLVal = EmitMemberExprBase(CGF, ME); in generateInfoForComponentList()
8008 LowestElem = CGF.EmitLValueForFieldInitialization( in generateInfoForComponentList()
8010 .getAddress(CGF); in generateInfoForComponentList()
8011 LB = CGF.EmitLoadOfReferenceLValue(LowestElem, MapDecl->getType()) in generateInfoForComponentList()
8012 .getAddress(CGF); in generateInfoForComponentList()
8015 CGF.EmitOMPSharedLValue(I->getAssociatedExpression()) in generateInfoForComponentList()
8016 .getAddress(CGF); in generateInfoForComponentList()
8037 CharUnits TypeSize = CGF.getContext().getTypeSizeInChars( in generateInfoForComponentList()
8039 Address HB = CGF.Builder.CreateConstGEP( in generateInfoForComponentList()
8040 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(LowestElem, in generateInfoForComponentList()
8041 CGF.VoidPtrTy), in generateInfoForComponentList()
8071 LValue BaseLVal = EmitMemberExprBase(CGF, ME); in generateInfoForComponentList()
8073 CGF.EmitLValueForFieldInitialization(BaseLVal, FD) in generateInfoForComponentList()
8074 .getAddress(CGF); in generateInfoForComponentList()
8077 CGF.EmitOMPSharedLValue(MC.getAssociatedExpression()) in generateInfoForComponentList()
8078 .getAddress(CGF); in generateInfoForComponentList()
8080 Size = CGF.Builder.CreatePtrDiff( in generateInfoForComponentList()
8081 CGF.EmitCastToVoidPtr(ComponentLB.getPointer()), in generateInfoForComponentList()
8082 CGF.EmitCastToVoidPtr(LB.getPointer())); in generateInfoForComponentList()
8090 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateInfoForComponentList()
8091 Size, CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForComponentList()
8096 LB = CGF.Builder.CreateConstGEP(ComponentLB, 1); in generateInfoForComponentList()
8101 Size = CGF.Builder.CreatePtrDiff( in generateInfoForComponentList()
8102 CGF.Builder.CreateConstGEP(HB, 1).getPointer(), in generateInfoForComponentList()
8103 CGF.EmitCastToVoidPtr(LB.getPointer())); in generateInfoForComponentList()
8105 CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForComponentList()
8119 CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForComponentList()
8169 CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false) in generateInfoForComponentList()
8170 .getAddress(CGF); in generateInfoForComponentList()
8205 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>()); in generateInfoForComponentList()
8217 const ASTContext &Context = CGF.getContext(); in generateInfoForComponentList()
8221 MapValuesArrayTy CurOffsets = {llvm::ConstantInt::get(CGF.CGM.Int64Ty, 0)}; in generateInfoForComponentList()
8222 MapValuesArrayTy CurCounts = {llvm::ConstantInt::get(CGF.CGM.Int64Ty, 1)}; in generateInfoForComponentList()
8224 MapValuesArrayTy DimSizes{llvm::ConstantInt::get(CGF.CGM.Int64Ty, 1)}; in generateInfoForComponentList()
8269 llvm::ConstantInt::get(CGF.Int64Ty, ElementTypeSize)); in generateInfoForComponentList()
8277 CGF.Int64Ty, CAT->getSize().getZExtValue())); in generateInfoForComponentList()
8279 DimSizes.push_back(CGF.Builder.CreateIntCast( in generateInfoForComponentList()
8280 CGF.EmitScalarExpr(VAT->getSizeExpr()), CGF.Int64Ty, in generateInfoForComponentList()
8289 llvm::ConstantInt::get(CGF.CGM.Int64Ty, ElementTypeSize); in generateInfoForComponentList()
8303 llvm::Value *Offset = CGF.Builder.CreateIntCast( in generateInfoForComponentList()
8304 CGF.EmitScalarExpr(AE->getIdx()), CGF.Int64Ty, in generateInfoForComponentList()
8307 CurCounts.push_back(llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/1)); in generateInfoForComponentList()
8322 Offset = llvm::ConstantInt::get(CGF.Int64Ty, 0); in generateInfoForComponentList()
8324 Offset = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(OffsetExpr), in generateInfoForComponentList()
8325 CGF.Int64Ty, in generateInfoForComponentList()
8340 Count = llvm::ConstantInt::get(CGF.Int64Ty, 1); in generateInfoForComponentList()
8349 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(StrideExpr), in generateInfoForComponentList()
8350 CGF.Int64Ty, /*isSigned=*/false) in generateInfoForComponentList()
8353 Count = CGF.Builder.CreateUDiv( in generateInfoForComponentList()
8354 CGF.Builder.CreateNUWSub(*DI, Offset), Stride); in generateInfoForComponentList()
8356 Count = CGF.Builder.CreateNUWSub(*DI, Offset); in generateInfoForComponentList()
8359 Count = CGF.EmitScalarExpr(CountExpr); in generateInfoForComponentList()
8361 Count = CGF.Builder.CreateIntCast(Count, CGF.Int64Ty, /*isSigned=*/false); in generateInfoForComponentList()
8374 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(StrideExpr), in generateInfoForComponentList()
8375 CGF.Int64Ty, /*isSigned=*/false) in generateInfoForComponentList()
8377 DimProd = CGF.Builder.CreateNUWMul(DimProd, *(DI - 1)); in generateInfoForComponentList()
8379 CurStrides.push_back(CGF.Builder.CreateNUWMul(DimProd, Stride)); in generateInfoForComponentList()
8436 const CGRecordLayout &RL = CGF.getTypes().getCGRecordLayout(RD); in getPlainLayout()
8452 if (Base->isEmpty() || CGF.getContext() in getPlainLayout()
8477 if (!Field->isBitField() && !Field->isZeroSize(CGF.getContext())) { in getPlainLayout()
8638 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() || in generateAllInfoForClauses()
8671 CGF.EmitLoadOfScalar(CGF.EmitLValue(IE), IE->getExprLoc()); in generateAllInfoForClauses()
8676 llvm::Constant::getNullValue(CGF.Int64Ty)); in generateAllInfoForClauses()
8745 Ptr = CGF.EmitLValue(IE).getPointer(CGF); in generateAllInfoForClauses()
8747 Ptr = CGF.EmitScalarExpr(IE); in generateAllInfoForClauses()
8752 llvm::Constant::getNullValue(CGF.Int64Ty)); in generateAllInfoForClauses()
8806 Ptr = this->CGF.EmitLValue(L.IE).getPointer(CGF); in generateAllInfoForClauses()
8808 Ptr = this->CGF.EmitScalarExpr(L.IE); in generateAllInfoForClauses()
8815 BasePtr = this->CGF.EmitLValue(L.IE).getPointer(CGF); in generateAllInfoForClauses()
8816 Ptr = this->CGF.EmitLoadOfScalar(this->CGF.EmitLValue(L.IE), in generateAllInfoForClauses()
8828 llvm::Constant::getNullValue(this->CGF.Int64Ty)); in generateAllInfoForClauses()
8848 MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF) in MappableExprsHandler() argument
8849 : CurDir(&Dir), CGF(CGF) { in MappableExprsHandler()
8875 MappableExprsHandler(const OMPDeclareMapperDecl &Dir, CodeGenFunction &CGF) in MappableExprsHandler() argument
8876 : CurDir(&Dir), CGF(CGF) {} in MappableExprsHandler()
8907 CGF.Builder.CreateConstGEP1_32(HBAddr.getElementType(), HB, /*Idx0=*/1); in emitCombinedEntry()
8908 llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(LB, CGF.VoidPtrTy); in emitCombinedEntry()
8909 llvm::Value *CHAddr = CGF.Builder.CreatePointerCast(HAddr, CGF.VoidPtrTy); in emitCombinedEntry()
8910 llvm::Value *Diff = CGF.Builder.CreatePtrDiff(CHAddr, CLAddr); in emitCombinedEntry()
8911 llvm::Value *Size = CGF.Builder.CreateIntCast(Diff, CGF.Int64Ty, in emitCombinedEntry()
8971 Address VDAddr = Address(Arg, CGF.getContext().getDeclAlign(VD)); in generateInfoForLambdaCaptures()
8972 LValue VDLVal = CGF.MakeAddrLValue( in generateInfoForLambdaCaptures()
8979 CGF.EmitLValueForFieldInitialization(VDLVal, ThisCapture); in generateInfoForLambdaCaptures()
8980 LValue ThisLValVal = CGF.EmitLValueForField(VDLVal, ThisCapture); in generateInfoForLambdaCaptures()
8981 LambdaPointers.try_emplace(ThisLVal.getPointer(CGF), in generateInfoForLambdaCaptures()
8982 VDLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8984 CombinedInfo.BasePointers.push_back(ThisLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8985 CombinedInfo.Pointers.push_back(ThisLValVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
8987 CGF.Builder.CreateIntCast(CGF.getTypeSize(CGF.getContext().VoidPtrTy), in generateInfoForLambdaCaptures()
8988 CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForLambdaCaptures()
9001 LValue VarLVal = CGF.EmitLValueForFieldInitialization(VDLVal, It->second); in generateInfoForLambdaCaptures()
9003 LValue VarLValVal = CGF.EmitLValueForField(VDLVal, It->second); in generateInfoForLambdaCaptures()
9004 LambdaPointers.try_emplace(VarLVal.getPointer(CGF), in generateInfoForLambdaCaptures()
9005 VDLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
9007 CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
9008 CombinedInfo.Pointers.push_back(VarLValVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
9009 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateInfoForLambdaCaptures()
9010 CGF.getTypeSize( in generateInfoForLambdaCaptures()
9012 CGF.Int64Ty, /*isSigned=*/true)); in generateInfoForLambdaCaptures()
9014 RValue VarRVal = CGF.EmitLoadOfLValue(VarLVal, RD->getLocation()); in generateInfoForLambdaCaptures()
9015 LambdaPointers.try_emplace(VarLVal.getPointer(CGF), in generateInfoForLambdaCaptures()
9016 VDLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
9018 CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF)); in generateInfoForLambdaCaptures()
9020 CombinedInfo.Sizes.push_back(llvm::ConstantInt::get(CGF.Int64Ty, 0)); in generateInfoForLambdaCaptures()
9077 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateInfoForCapture()
9078 CGF.getTypeSize(CGF.getContext().VoidPtrTy), CGF.Int64Ty, in generateInfoForCapture()
9308 CGF.Builder.CreateIntCast(CGF.getTypeSize(PtrTy->getPointeeType()), in generateDefaultMapInfo()
9309 CGF.Int64Ty, /*isSigned=*/true)); in generateDefaultMapInfo()
9321 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateDefaultMapInfo()
9322 CGF.getTypeSize(RI.getType()), CGF.Int64Ty, /*isSigned=*/true)); in generateDefaultMapInfo()
9327 CombinedInfo.Sizes.push_back(llvm::Constant::getNullValue(CGF.Int64Ty)); in generateDefaultMapInfo()
9336 CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in generateDefaultMapInfo()
9337 CGF.getTypeSize(ElementType), CGF.Int64Ty, /*isSigned=*/true)); in generateDefaultMapInfo()
9347 Address PtrAddr = CGF.EmitLoadOfReference(CGF.MakeAddrLValue( in generateDefaultMapInfo()
9348 CV, ElementType, CGF.getContext().getDeclAlign(VD), in generateDefaultMapInfo()
9371 CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo, in emitNonContiguousDescriptor() argument
9373 CodeGenModule &CGM = CGF.CGM; in emitNonContiguousDescriptor()
9385 ASTContext &C = CGF.getContext(); in emitNonContiguousDescriptor()
9408 Address DimsAddr = CGF.CreateMemTemp(ArrayTy, "dims"); in emitNonContiguousDescriptor()
9411 LValue DimsLVal = CGF.MakeAddrLValue( in emitNonContiguousDescriptor()
9412 CGF.Builder.CreateConstArrayGEP(DimsAddr, II), DimTy); in emitNonContiguousDescriptor()
9414 LValue OffsetLVal = CGF.EmitLValueForField( in emitNonContiguousDescriptor()
9416 CGF.EmitStoreOfScalar(NonContigInfo.Offsets[L][RevIdx], OffsetLVal); in emitNonContiguousDescriptor()
9418 LValue CountLVal = CGF.EmitLValueForField( in emitNonContiguousDescriptor()
9420 CGF.EmitStoreOfScalar(NonContigInfo.Counts[L][RevIdx], CountLVal); in emitNonContiguousDescriptor()
9422 LValue StrideLVal = CGF.EmitLValueForField( in emitNonContiguousDescriptor()
9424 CGF.EmitStoreOfScalar(NonContigInfo.Strides[L][RevIdx], StrideLVal); in emitNonContiguousDescriptor()
9427 Address DAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitNonContiguousDescriptor()
9429 llvm::Value *P = CGF.Builder.CreateConstInBoundsGEP2_32( in emitNonContiguousDescriptor()
9432 Address PAddr(P, CGF.getPointerAlign()); in emitNonContiguousDescriptor()
9433 CGF.Builder.CreateStore(DAddr.getPointer(), PAddr); in emitNonContiguousDescriptor()
9441 emitMappingInformation(CodeGenFunction &CGF, llvm::OpenMPIRBuilder &OMPBuilder, in emitMappingInformation() argument
9449 PrintingPolicy P(CGF.getContext().getLangOpts()); in emitMappingInformation()
9458 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc); in emitMappingInformation()
9472 CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo, in emitOffloadingArrays() argument
9475 CodeGenModule &CGM = CGF.CGM; in emitOffloadingArrays()
9476 ASTContext &Ctx = CGF.getContext(); in emitOffloadingArrays()
9498 CGF.CreateMemTemp(PointerArrayType, ".offload_baseptrs").getPointer(); in emitOffloadingArrays()
9500 CGF.CreateMemTemp(PointerArrayType, ".offload_ptrs").getPointer(); in emitOffloadingArrays()
9502 CGF.CreateMemTemp(PointerArrayType, ".offload_mappers"); in emitOffloadingArrays()
9515 CGF.CreateMemTemp(SizeArrayType, ".offload_sizes").getPointer(); in emitOffloadingArrays()
9524 CGF.Int64Ty, CombinedInfo.NonContigInfo.Dims[I])); in emitOffloadingArrays()
9555 llvm::Type::getInt8Ty(CGF.Builder.getContext())->getPointerTo()); in emitOffloadingArrays()
9558 return emitMappingInformation(CGF, OMPBuilder, MapExpr); in emitOffloadingArrays()
9588 llvm::Value *BP = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArrays()
9591 BP = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitOffloadingArrays()
9594 CGF.Builder.CreateStore(BPVal, BPAddr); in emitOffloadingArrays()
9602 llvm::Value *P = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArrays()
9605 P = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitOffloadingArrays()
9608 CGF.Builder.CreateStore(PVal, PAddr); in emitOffloadingArrays()
9611 llvm::Value *S = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArrays()
9617 CGF.Builder.CreateStore(CGF.Builder.CreateIntCast(CombinedInfo.Sizes[I], in emitOffloadingArrays()
9628 MFunc = CGF.Builder.CreatePointerCast(MFunc, CGM.VoidPtrTy); in emitOffloadingArrays()
9631 Address MAddr = CGF.Builder.CreateConstArrayGEP(MappersArray, I); in emitOffloadingArrays()
9632 CGF.Builder.CreateStore(MFunc, MAddr); in emitOffloadingArrays()
9640 emitNonContiguousDescriptor(CGF, CombinedInfo, Info); in emitOffloadingArrays()
9657 CodeGenFunction &CGF, llvm::Value *&BasePointersArrayArg, in emitOffloadingArraysArgument() argument
9664 CodeGenModule &CGM = CGF.CGM; in emitOffloadingArraysArgument()
9666 BasePointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
9670 PointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
9675 SizesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
9678 MapTypesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
9687 if (CGF.CGM.getCodeGenOpts().getDebugInfo() == codegenoptions::NoDebugInfo) in emitOffloadingArraysArgument()
9690 MapNamesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
9701 CGF.Builder.CreatePointerCast(Info.MappersArray, CGM.VoidPtrPtrTy); in emitOffloadingArraysArgument()
9855 CodeGenFunction *CGF) { in emitUserDefinedMapper() argument
10097 if (CGF) { in emitUserDefinedMapper()
10098 auto &Decls = FunctionUDMMap.FindAndConstruct(CGF->CurFn); in emitUserDefinedMapper()
10182 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitTargetNumIterationsCall() argument
10184 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, in emitTargetNumIterationsCall() argument
10195 auto &&CodeGen = [LD, DeviceID, SizeEmitter, &D, this](CodeGenFunction &CGF, in emitTargetNumIterationsCall()
10197 if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD)) { in emitTargetNumIterationsCall()
10198 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTargetNumIterationsCall()
10200 CGF.EmitRuntimeCall( in emitTargetNumIterationsCall()
10206 emitInlinedDirective(CGF, OMPD_unknown, CodeGen); in emitTargetNumIterationsCall()
10210 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitTargetCall() argument
10213 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, in emitTargetCall() argument
10216 if (!CGF.HaveInsertPoint()) in emitTargetCall()
10225 auto &&ArgsCodegen = [&CS, &CapturedVars](CodeGenFunction &CGF, in emitTargetCall()
10227 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars); in emitTargetCall()
10229 emitInlinedDirective(CGF, OMPD_unknown, ArgsCodegen); in emitTargetCall()
10238 SizeEmitter](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetCall() argument
10243 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars); in emitTargetCall()
10245 emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn, CapturedVars); in emitTargetCall()
10267 llvm::Value *DevVal = CGF.EmitScalarExpr(Device.getPointer()); in emitTargetCall()
10269 CGF.Builder.CreateIntCast(DevVal, CGF.Int64Ty, /*isSigned=*/true); in emitTargetCall()
10271 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTargetCall()
10276 CGF.Builder.getInt32(InputInfo.NumberOfTargetItems); in emitTargetCall()
10281 llvm::Value *NumTeams = emitNumTeamsForTargetDirective(CGF, D); in emitTargetCall()
10282 llvm::Value *NumThreads = emitNumThreadsForTargetDirective(CGF, D); in emitTargetCall()
10285 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTargetCall()
10288 emitTargetNumIterationsCall(CGF, D, DeviceID, SizeEmitter); in emitTargetCall()
10342 OffloadingArgs.push_back(CGF.Builder.getInt32(0)); in emitTargetCall()
10344 OffloadingArgs.push_back(CGF.Builder.getInt32(0)); in emitTargetCall()
10347 Return = CGF.EmitRuntimeCall( in emitTargetCall()
10368 OffloadingArgs.push_back(CGF.Builder.getInt32(0)); in emitTargetCall()
10370 OffloadingArgs.push_back(CGF.Builder.getInt32(0)); in emitTargetCall()
10373 Return = CGF.EmitRuntimeCall( in emitTargetCall()
10382 CGF.createBasicBlock("omp_offload.failed"); in emitTargetCall()
10384 CGF.createBasicBlock("omp_offload.cont"); in emitTargetCall()
10385 llvm::Value *Failed = CGF.Builder.CreateIsNotNull(Return); in emitTargetCall()
10386 CGF.Builder.CreateCondBr(Failed, OffloadFailedBlock, OffloadContBlock); in emitTargetCall()
10388 CGF.EmitBlock(OffloadFailedBlock); in emitTargetCall()
10391 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars); in emitTargetCall()
10393 emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn, CapturedVars); in emitTargetCall()
10394 CGF.EmitBranch(OffloadContBlock); in emitTargetCall()
10396 CGF.EmitBlock(OffloadContBlock, /*IsFinished=*/true); in emitTargetCall()
10401 RequiresOuterTask](CodeGenFunction &CGF, in emitTargetCall()
10405 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars); in emitTargetCall()
10407 emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn, CapturedVars); in emitTargetCall()
10412 &CS](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetCall() argument
10417 MappableExprsHandler MEHandler(D, CGF); in emitTargetCall()
10435 CurInfo.Sizes.push_back(CGF.Builder.CreateIntCast( in emitTargetCall()
10436 CGF.getTypeSize(RI->getType()), CGF.Int64Ty, /*isSigned=*/true)); in emitTargetCall()
10489 emitOffloadingArrays(CGF, CombinedInfo, Info, OMPBuilder); in emitTargetCall()
10491 CGF, Info.BasePointersArray, Info.PointersArray, Info.SizesArray, in emitTargetCall()
10505 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo); in emitTargetCall()
10507 emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen); in emitTargetCall()
10511 CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetCall() argument
10514 CGF.EmitOMPTargetTaskBasedDirective(D, ElseGen, InputInfo); in emitTargetCall()
10516 emitInlinedDirective(CGF, D.getDirectiveKind(), ElseGen); in emitTargetCall()
10526 emitIfClause(CGF, IfCond, TargetThenGen, TargetElseGen); in emitTargetCall()
10529 ThenRCG(CGF); in emitTargetCall()
10533 ElseRCG(CGF); in emitTargetCall()
10875 CodeGenFunction &CGF, const OMPExecutableDirective &D) const { in adjustTargetSpecificDataForLambdas() argument
10985 CodeGenFunction CGF(CGM); in emitRequiresDirectiveRegFun() local
10990 CGF.StartFunction(GlobalDecl(), C.VoidTy, RequiresRegFn, FI, {}); in emitRequiresDirectiveRegFun()
11004 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitRequiresDirectiveRegFun()
11007 CGF.FinishFunction(); in emitRequiresDirectiveRegFun()
11012 void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF, in emitTeamsCall() argument
11017 if (!CGF.HaveInsertPoint()) in emitTeamsCall()
11020 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitTeamsCall()
11021 CodeGenFunction::RunCleanupsScope Scope(CGF); in emitTeamsCall()
11026 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars in emitTeamsCall()
11027 CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy())}; in emitTeamsCall()
11034 CGF.EmitRuntimeCall(RTLFn, RealArgs); in emitTeamsCall()
11037 void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF, in emitNumTeamsClause() argument
11041 if (!CGF.HaveInsertPoint()) in emitNumTeamsClause()
11044 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitNumTeamsClause()
11048 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(NumTeams), in emitNumTeamsClause()
11049 CGF.CGM.Int32Ty, /* isSigned = */ true) in emitNumTeamsClause()
11050 : CGF.Builder.getInt32(0); in emitNumTeamsClause()
11054 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(ThreadLimit), in emitNumTeamsClause()
11055 CGF.CGM.Int32Ty, /* isSigned = */ true) in emitNumTeamsClause()
11056 : CGF.Builder.getInt32(0); in emitNumTeamsClause()
11059 llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal, in emitNumTeamsClause()
11061 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitNumTeamsClause()
11067 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, in emitTargetDataCalls() argument
11069 if (!CGF.HaveInsertPoint()) in emitTargetDataCalls()
11080 &CodeGen](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetDataCalls() argument
11085 MappableExprsHandler MEHandler(D, CGF); in emitTargetDataCalls()
11089 emitOffloadingArrays(CGF, CombinedInfo, Info, OMPBuilder, in emitTargetDataCalls()
11098 emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg, in emitTargetDataCalls()
11105 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTargetDataCalls()
11106 CGF.Int64Ty, /*isSigned=*/true); in emitTargetDataCalls()
11108 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTargetDataCalls()
11112 llvm::Value *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs); in emitTargetDataCalls()
11115 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTargetDataCalls()
11126 CGF.EmitRuntimeCall( in emitTargetDataCalls()
11134 CodeGen(CGF); in emitTargetDataCalls()
11138 auto &&EndThenGen = [this, Device, &Info, &D](CodeGenFunction &CGF, in emitTargetDataCalls()
11148 emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg, in emitTargetDataCalls()
11156 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTargetDataCalls()
11157 CGF.Int64Ty, /*isSigned=*/true); in emitTargetDataCalls()
11159 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTargetDataCalls()
11163 llvm::Value *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs); in emitTargetDataCalls()
11166 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTargetDataCalls()
11177 CGF.EmitRuntimeCall( in emitTargetDataCalls()
11186 auto &&BeginElseGen = [&Info, &CodeGen, &NoPrivAction](CodeGenFunction &CGF, in emitTargetDataCalls()
11190 CodeGen(CGF); in emitTargetDataCalls()
11196 auto &&EndElseGen = [](CodeGenFunction &CGF, PrePostActionTy &) {}; in emitTargetDataCalls() argument
11199 emitIfClause(CGF, IfCond, BeginThenGen, BeginElseGen); in emitTargetDataCalls()
11202 RCG(CGF); in emitTargetDataCalls()
11209 CodeGen(CGF); in emitTargetDataCalls()
11213 emitIfClause(CGF, IfCond, EndThenGen, EndElseGen); in emitTargetDataCalls()
11216 RCG(CGF); in emitTargetDataCalls()
11221 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, in emitTargetDataStandAloneCall() argument
11223 if (!CGF.HaveInsertPoint()) in emitTargetDataStandAloneCall()
11236 &MapNamesArray](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetDataStandAloneCall() argument
11240 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTargetDataStandAloneCall()
11241 CGF.Int64Ty, /*isSigned=*/true); in emitTargetDataStandAloneCall()
11243 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); in emitTargetDataStandAloneCall()
11248 CGF.Builder.getInt32(InputInfo.NumberOfTargetItems); in emitTargetDataStandAloneCall()
11251 llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); in emitTargetDataStandAloneCall()
11349 CGF.EmitRuntimeCall( in emitTargetDataStandAloneCall()
11355 &MapNamesArray](CodeGenFunction &CGF, in emitTargetDataStandAloneCall()
11361 MappableExprsHandler MEHandler(D, CGF); in emitTargetDataStandAloneCall()
11366 emitOffloadingArrays(CGF, CombinedInfo, Info, OMPBuilder, in emitTargetDataStandAloneCall()
11371 CGF, Info.BasePointersArray, Info.PointersArray, Info.SizesArray, in emitTargetDataStandAloneCall()
11385 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo); in emitTargetDataStandAloneCall()
11387 emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen); in emitTargetDataStandAloneCall()
11391 emitIfClause(CGF, IfCond, TargetThenGen, in emitTargetDataStandAloneCall()
11392 [](CodeGenFunction &CGF, PrePostActionTy &) {}); in emitTargetDataStandAloneCall() argument
11395 ThenRCG(CGF); in emitTargetDataStandAloneCall()
11978 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { in Emit() argument
11979 if (!CGF.HaveInsertPoint()) in Emit()
11981 CGF.EmitRuntimeCall(RTLFn, Args); in Emit()
11986 void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF, in emitDoacrossInit() argument
11989 if (!CGF.HaveInsertPoint()) in emitDoacrossInit()
12015 Address DimsAddr = CGF.CreateMemTemp(ArrayTy, "dims"); in emitDoacrossInit()
12016 CGF.EmitNullInitialization(DimsAddr, ArrayTy); in emitDoacrossInit()
12020 LValue DimsLVal = CGF.MakeAddrLValue( in emitDoacrossInit()
12021 CGF.Builder.CreateConstArrayGEP(DimsAddr, I), KmpDimTy); in emitDoacrossInit()
12023 LValue UpperLVal = CGF.EmitLValueForField( in emitDoacrossInit()
12025 llvm::Value *NumIterVal = CGF.EmitScalarConversion( in emitDoacrossInit()
12026 CGF.EmitScalarExpr(NumIterations[I]), NumIterations[I]->getType(), in emitDoacrossInit()
12028 CGF.EmitStoreOfScalar(NumIterVal, UpperLVal); in emitDoacrossInit()
12030 LValue StrideLVal = CGF.EmitLValueForField( in emitDoacrossInit()
12032 CGF.EmitStoreOfScalar(llvm::ConstantInt::getSigned(CGM.Int64Ty, /*V=*/1), in emitDoacrossInit()
12039 emitUpdateLocation(CGF, D.getBeginLoc()), in emitDoacrossInit()
12040 getThreadID(CGF, D.getBeginLoc()), in emitDoacrossInit()
12042 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDoacrossInit()
12043 CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).getPointer(), in emitDoacrossInit()
12048 CGF.EmitRuntimeCall(RTLFn, Args); in emitDoacrossInit()
12050 emitUpdateLocation(CGF, D.getEndLoc()), getThreadID(CGF, D.getEndLoc())}; in emitDoacrossInit()
12053 CGF.EHStack.pushCleanup<DoacrossCleanupTy>(NormalAndEHCleanup, FiniRTLFn, in emitDoacrossInit()
12057 void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, in emitDoacrossOrdered() argument
12064 Address CntAddr = CGF.CreateMemTemp(ArrayTy, ".cnt.addr"); in emitDoacrossOrdered()
12068 llvm::Value *CntVal = CGF.EmitScalarConversion( in emitDoacrossOrdered()
12069 CGF.EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty, in emitDoacrossOrdered()
12071 CGF.EmitStoreOfScalar(CntVal, CGF.Builder.CreateConstArrayGEP(CntAddr, I), in emitDoacrossOrdered()
12075 emitUpdateLocation(CGF, C->getBeginLoc()), in emitDoacrossOrdered()
12076 getThreadID(CGF, C->getBeginLoc()), in emitDoacrossOrdered()
12077 CGF.Builder.CreateConstArrayGEP(CntAddr, 0).getPointer()}; in emitDoacrossOrdered()
12087 CGF.EmitRuntimeCall(RTLFn, Args); in emitDoacrossOrdered()
12090 void CGOpenMPRuntime::emitCall(CodeGenFunction &CGF, SourceLocation Loc, in emitCall() argument
12094 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc); in emitCall()
12098 CGF.EmitNounwindRuntimeCall(Fn, Args); in emitCall()
12102 CGF.EmitRuntimeCall(Callee, Args); in emitCall()
12106 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, in emitOutlinedFunctionCall() argument
12108 emitCall(CGF, Loc, OutlinedFn, Args); in emitOutlinedFunctionCall()
12111 void CGOpenMPRuntime::emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) { in emitFunctionProlog() argument
12117 Address CGOpenMPRuntime::getParameterAddress(CodeGenFunction &CGF, in getParameterAddress() argument
12120 return CGF.GetAddrOfLocalVar(NativeParam); in getParameterAddress()
12123 Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF, in getAddressOfLocalVariable() argument
12129 auto It = FunctionToUntiedTaskStackMap.find(CGF.CurFn); in getAddressOfLocalVariable()
12147 Size = CGF.getTypeSize(CVD->getType()); in getAddressOfLocalVariable()
12149 Size = CGF.Builder.CreateNUWAdd( in getAddressOfLocalVariable()
12151 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align)); in getAddressOfLocalVariable()
12152 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align)); in getAddressOfLocalVariable()
12157 llvm::Value *ThreadID = getThreadID(CGF, CVD->getBeginLoc()); in getAddressOfLocalVariable()
12161 llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator()); in getAddressOfLocalVariable()
12164 Allocator = CGF.EmitScalarConversion( in getAddressOfLocalVariable()
12165 Allocator, AA->getAllocator()->getType(), CGF.getContext().VoidPtrTy, in getAddressOfLocalVariable()
12170 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in getAddressOfLocalVariable()
12176 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddressOfLocalVariable()
12177 Addr, CGF.ConvertTypeForMem(Ty), getName({CVD->getName(), ".addr"})); in getAddressOfLocalVariable()
12179 CGF.EmitStoreOfScalar(Addr, UntiedAddr, /*Volatile=*/false, Ty); in getAddressOfLocalVariable()
12194 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { in getAddressOfLocalVariable() argument
12195 if (!CGF.HaveInsertPoint()) in getAddressOfLocalVariable()
12198 Args[0] = CGF.CGM.getOpenMPRuntime().getThreadID( in getAddressOfLocalVariable()
12199 CGF, SourceLocation::getFromRawEncoding(LocEncoding)); in getAddressOfLocalVariable()
12200 Args[1] = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddressOfLocalVariable()
12201 Addr.getPointer(), CGF.VoidPtrTy); in getAddressOfLocalVariable()
12202 llvm::Value *AllocVal = CGF.EmitScalarExpr(Allocator); in getAddressOfLocalVariable()
12205 AllocVal = CGF.EmitScalarConversion(AllocVal, Allocator->getType(), in getAddressOfLocalVariable()
12206 CGF.getContext().VoidPtrTy, in getAddressOfLocalVariable()
12210 CGF.EmitRuntimeCall(RTLFn, Args); in getAddressOfLocalVariable()
12215 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>( in getAddressOfLocalVariable()
12220 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in getAddressOfLocalVariable()
12221 Region->emitUntiedSwitch(CGF); in getAddressOfLocalVariable()
12227 bool CGOpenMPRuntime::isLocalVarInUntiedTask(CodeGenFunction &CGF, in isLocalVarInUntiedTask() argument
12229 auto It = FunctionToUntiedTaskStackMap.find(CGF.CurFn); in isLocalVarInUntiedTask()
12268 CodeGenFunction &CGF, in UntiedTaskLocalDeclsRAII() argument
12271 : CGM(CGF.CGM), NeedToPush(!LocalVars.empty()) { in UntiedTaskLocalDeclsRAII()
12275 CGF.CurFn, CGM.getOpenMPRuntime().UntiedLocalVarsStack.size()); in UntiedTaskLocalDeclsRAII()
12373 CodeGenFunction &CGF, const OMPExecutableDirective &S, LValue IVLVal) in LastprivateConditionalRAII() argument
12374 : CGM(CGF.CGM), in LastprivateConditionalRAII()
12401 Data.Fn = CGF.CurFn;
12405 CodeGenFunction &CGF, const OMPExecutableDirective &S) in LastprivateConditionalRAII() argument
12406 : CGM(CGF.CGM), Action(ActionToDo::DoNotPush) { in LastprivateConditionalRAII()
12418 Data.Fn = CGF.CurFn; in LastprivateConditionalRAII()
12425 CodeGenFunction &CGF, const OMPExecutableDirective &S) { in disable() argument
12426 return LastprivateConditionalRAII(CGF, S); in disable()
12445 Address CGOpenMPRuntime::emitLastprivateConditionalInit(CodeGenFunction &CGF, in emitLastprivateConditionalInit() argument
12448 auto I = LastprivateConditionalToTypes.find(CGF.CurFn); in emitLastprivateConditionalInit()
12450 I = LastprivateConditionalToTypes.try_emplace(CGF.CurFn).first; in emitLastprivateConditionalInit()
12463 Address Addr = CGF.CreateMemTemp(NewType, C.getDeclAlign(VD), VD->getName()); in emitLastprivateConditionalInit()
12464 BaseLVal = CGF.MakeAddrLValue(Addr, NewType, AlignmentSource::Decl); in emitLastprivateConditionalInit()
12473 CGF.EmitLValueForField(BaseLVal, FiredField); in emitLastprivateConditionalInit()
12474 CGF.EmitStoreOfScalar( in emitLastprivateConditionalInit()
12475 llvm::ConstantInt::getNullValue(CGF.ConvertTypeForMem(C.CharTy)), in emitLastprivateConditionalInit()
12477 return CGF.EmitLValueForField(BaseLVal, VDField).getAddress(CGF); in emitLastprivateConditionalInit()
12551 void CGOpenMPRuntime::emitLastprivateConditionalUpdate(CodeGenFunction &CGF, in emitLastprivateConditionalUpdate() argument
12558 llvm::Type *LLIVTy = CGF.ConvertTypeForMem(IVLVal.getType()); in emitLastprivateConditionalUpdate()
12563 LValue LastIVLVal = CGF.MakeNaturalAlignAddrLValue(LastIV, IVLVal.getType()); in emitLastprivateConditionalUpdate()
12568 CGF.ConvertTypeForMem(LVal.getType()), UniqueDeclName); in emitLastprivateConditionalUpdate()
12572 CGF.MakeAddrLValue(Last, LVal.getType(), LVal.getAlignment()); in emitLastprivateConditionalUpdate()
12576 llvm::Value *IVVal = CGF.EmitLoadOfScalar(IVLVal, Loc); in emitLastprivateConditionalUpdate()
12584 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitLastprivateConditionalUpdate() argument
12585 Action.Enter(CGF); in emitLastprivateConditionalUpdate()
12586 llvm::Value *LastIVVal = CGF.EmitLoadOfScalar(LastIVLVal, Loc); in emitLastprivateConditionalUpdate()
12591 CmpRes = CGF.Builder.CreateICmpSLE(LastIVVal, IVVal); in emitLastprivateConditionalUpdate()
12595 CmpRes = CGF.Builder.CreateICmpULE(LastIVVal, IVVal); in emitLastprivateConditionalUpdate()
12597 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("lp_cond_then"); in emitLastprivateConditionalUpdate()
12598 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("lp_cond_exit"); in emitLastprivateConditionalUpdate()
12599 CGF.Builder.CreateCondBr(CmpRes, ThenBB, ExitBB); in emitLastprivateConditionalUpdate()
12601 CGF.EmitBlock(ThenBB); in emitLastprivateConditionalUpdate()
12604 CGF.EmitStoreOfScalar(IVVal, LastIVLVal); in emitLastprivateConditionalUpdate()
12607 switch (CGF.getEvaluationKind(LVal.getType())) { in emitLastprivateConditionalUpdate()
12609 llvm::Value *PrivVal = CGF.EmitLoadOfScalar(LVal, Loc); in emitLastprivateConditionalUpdate()
12610 CGF.EmitStoreOfScalar(PrivVal, LastLVal); in emitLastprivateConditionalUpdate()
12614 CodeGenFunction::ComplexPairTy PrivVal = CGF.EmitLoadOfComplex(LVal, Loc); in emitLastprivateConditionalUpdate()
12615 CGF.EmitStoreOfComplex(PrivVal, LastLVal, /*isInit=*/false); in emitLastprivateConditionalUpdate()
12623 CGF.EmitBranch(ExitBB); in emitLastprivateConditionalUpdate()
12625 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitLastprivateConditionalUpdate()
12626 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in emitLastprivateConditionalUpdate()
12632 ThenRCG(CGF); in emitLastprivateConditionalUpdate()
12634 emitCriticalRegion(CGF, UniqueDeclName, CodeGen, Loc); in emitLastprivateConditionalUpdate()
12638 void CGOpenMPRuntime::checkAndEmitLastprivateConditional(CodeGenFunction &CGF, in checkAndEmitLastprivateConditional() argument
12640 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty()) in checkAndEmitLastprivateConditional()
12652 if (FoundFn != CGF.CurFn) { in checkAndEmitLastprivateConditional()
12660 LValue PrivLVal = CGF.EmitLValue(FoundE); in checkAndEmitLastprivateConditional()
12661 Address StructAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in checkAndEmitLastprivateConditional()
12662 PrivLVal.getAddress(CGF), in checkAndEmitLastprivateConditional()
12663 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(StructTy))); in checkAndEmitLastprivateConditional()
12665 CGF.MakeAddrLValue(StructAddr, StructTy, AlignmentSource::Decl); in checkAndEmitLastprivateConditional()
12666 LValue FiredLVal = CGF.EmitLValueForField(BaseLVal, FiredDecl); in checkAndEmitLastprivateConditional()
12667 CGF.EmitAtomicStore(RValue::get(llvm::ConstantInt::get( in checkAndEmitLastprivateConditional()
12668 CGF.ConvertTypeForMem(FiredDecl->getType()), 1)), in checkAndEmitLastprivateConditional()
12676 LValue LVal = CGF.EmitLValue(FoundE); in checkAndEmitLastprivateConditional()
12677 emitLastprivateConditionalUpdate(CGF, IVLVal, UniqueDeclName, LVal, in checkAndEmitLastprivateConditional()
12682 CodeGenFunction &CGF, const OMPExecutableDirective &D, in checkAndEmitSharedLastprivateConditional() argument
12684 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty()) in checkAndEmitSharedLastprivateConditional()
12689 if (It == Range.end() || It->Fn != CGF.CurFn) in checkAndEmitSharedLastprivateConditional()
12707 CGF.EmitLValueForField(BaseLVal, std::get<2>(I->getSecond())); in checkAndEmitSharedLastprivateConditional()
12708 llvm::Value *Res = CGF.EmitLoadOfScalar(FiredLVal, D.getBeginLoc()); in checkAndEmitSharedLastprivateConditional()
12709 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Res); in checkAndEmitSharedLastprivateConditional()
12710 llvm::BasicBlock *ThenBB = CGF.createBasicBlock("lpc.then"); in checkAndEmitSharedLastprivateConditional()
12711 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("lpc.done"); in checkAndEmitSharedLastprivateConditional()
12713 CGF.Builder.CreateCondBr(Cmp, ThenBB, DoneBB); in checkAndEmitSharedLastprivateConditional()
12714 CGF.EmitBlock(ThenBB); in checkAndEmitSharedLastprivateConditional()
12715 Address Addr = CGF.GetAddrOfLocalVar(VD); in checkAndEmitSharedLastprivateConditional()
12718 LVal = CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(), in checkAndEmitSharedLastprivateConditional()
12721 LVal = CGF.MakeAddrLValue(Addr, VD->getType().getNonReferenceType(), in checkAndEmitSharedLastprivateConditional()
12723 emitLastprivateConditionalUpdate(CGF, It->IVLVal, Pair.second, LVal, in checkAndEmitSharedLastprivateConditional()
12725 auto AL = ApplyDebugLocation::CreateArtificial(CGF); in checkAndEmitSharedLastprivateConditional()
12726 CGF.EmitBlock(DoneBB, /*IsFinal=*/true); in checkAndEmitSharedLastprivateConditional()
12732 CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD, in emitLastprivateConditionalFinalUpdate() argument
12734 if (CGF.getLangOpts().OpenMP < 50) in emitLastprivateConditionalFinalUpdate()
12744 LValue LPLVal = CGF.MakeAddrLValue( in emitLastprivateConditionalFinalUpdate()
12746 llvm::Value *Res = CGF.EmitLoadOfScalar(LPLVal, Loc); in emitLastprivateConditionalFinalUpdate()
12747 CGF.EmitStoreOfScalar(Res, PrivLVal); in emitLastprivateConditionalFinalUpdate()
12770 void CGOpenMPSIMDRuntime::emitParallelCall(CodeGenFunction &CGF, in emitParallelCall() argument
12779 CodeGenFunction &CGF, StringRef CriticalName, in emitCriticalRegion() argument
12785 void CGOpenMPSIMDRuntime::emitMasterRegion(CodeGenFunction &CGF, in emitMasterRegion() argument
12791 void CGOpenMPSIMDRuntime::emitMaskedRegion(CodeGenFunction &CGF, in emitMaskedRegion() argument
12798 void CGOpenMPSIMDRuntime::emitTaskyieldCall(CodeGenFunction &CGF, in emitTaskyieldCall() argument
12804 CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, in emitTaskgroupRegion() argument
12810 CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, in emitSingleRegion() argument
12817 void CGOpenMPSIMDRuntime::emitOrderedRegion(CodeGenFunction &CGF, in emitOrderedRegion() argument
12824 void CGOpenMPSIMDRuntime::emitBarrierCall(CodeGenFunction &CGF, in emitBarrierCall() argument
12833 CodeGenFunction &CGF, SourceLocation Loc, in emitForDispatchInit() argument
12840 CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, in emitForStaticInit() argument
12846 CodeGenFunction &CGF, SourceLocation Loc, in emitDistributeStaticInit() argument
12851 void CGOpenMPSIMDRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF, in emitForOrderedIterationEnd() argument
12858 void CGOpenMPSIMDRuntime::emitForStaticFinish(CodeGenFunction &CGF, in emitForStaticFinish() argument
12864 llvm::Value *CGOpenMPSIMDRuntime::emitForNext(CodeGenFunction &CGF, in emitForNext() argument
12872 void CGOpenMPSIMDRuntime::emitNumThreadsClause(CodeGenFunction &CGF, in emitNumThreadsClause() argument
12878 void CGOpenMPSIMDRuntime::emitProcBindClause(CodeGenFunction &CGF, in emitProcBindClause() argument
12884 Address CGOpenMPSIMDRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, in getAddrOfThreadPrivate() argument
12893 CodeGenFunction *CGF) { in emitThreadPrivateVarDefinition() argument
12898 CodeGenFunction &CGF, QualType VarType, StringRef Name) { in getAddrOfArtificialThreadPrivate() argument
12902 void CGOpenMPSIMDRuntime::emitFlush(CodeGenFunction &CGF, in emitFlush() argument
12909 void CGOpenMPSIMDRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskCall() argument
12919 CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, in emitTaskLoopCall() argument
12926 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates, in emitReduction() argument
12930 CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs, in emitReduction()
12935 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs, in emitTaskReductionInit() argument
12940 void CGOpenMPSIMDRuntime::emitTaskReductionFini(CodeGenFunction &CGF, in emitTaskReductionFini() argument
12946 void CGOpenMPSIMDRuntime::emitTaskReductionFixups(CodeGenFunction &CGF, in emitTaskReductionFixups() argument
12953 Address CGOpenMPSIMDRuntime::getTaskReductionItem(CodeGenFunction &CGF, in getTaskReductionItem() argument
12960 void CGOpenMPSIMDRuntime::emitTaskwaitCall(CodeGenFunction &CGF, in emitTaskwaitCall() argument
12966 CodeGenFunction &CGF, SourceLocation Loc, in emitCancellationPointCall() argument
12971 void CGOpenMPSIMDRuntime::emitCancelCall(CodeGenFunction &CGF, in emitCancelCall() argument
12985 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitTargetCall() argument
12988 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, in emitTargetCall() argument
13006 void CGOpenMPSIMDRuntime::emitTeamsCall(CodeGenFunction &CGF, in emitTeamsCall() argument
13014 void CGOpenMPSIMDRuntime::emitNumTeamsClause(CodeGenFunction &CGF, in emitNumTeamsClause() argument
13022 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, in emitTargetDataCalls() argument
13028 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, in emitTargetDataStandAloneCall() argument
13033 void CGOpenMPSIMDRuntime::emitDoacrossInit(CodeGenFunction &CGF, in emitDoacrossInit() argument
13039 void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, in emitDoacrossOrdered() argument
13051 CGOpenMPSIMDRuntime::getParameterAddress(CodeGenFunction &CGF, in getParameterAddress() argument