Lines Matching refs:llvm
40 llvm::IntegerType *IntTy, *SizeTy;
41 llvm::Type *VoidTy;
42 llvm::PointerType *PtrTy;
45 llvm::LLVMContext &Context;
47 llvm::Module &TheModule;
50 llvm::Function *Kernel; // stub function to help launch kernel
53 llvm::SmallVector<KernelInfo, 16> EmittedKernels;
57 llvm::DenseMap<StringRef, llvm::GlobalValue *> KernelHandles;
59 llvm::DenseMap<llvm::GlobalValue *, llvm::Function *> KernelStubs;
61 llvm::GlobalVariable *Var;
65 llvm::SmallVector<VarInfo, 16> DeviceVars;
69 llvm::GlobalVariable *GpuBinaryHandle = nullptr;
75 llvm::Constant *Zeros[2];
77 llvm::FunctionCallee getSetupArgumentFn() const;
78 llvm::FunctionCallee getLaunchFn() const;
80 llvm::FunctionType *getRegisterGlobalsFnTy() const;
81 llvm::FunctionType *getCallbackFnTy() const;
82 llvm::FunctionType *getRegisterLinkedBinaryFnTy() const;
87 llvm::Function *makeRegisterGlobalsFn();
92 llvm::Constant *makeConstantString(const std::string &Str, in makeConstantString()
95 return llvm::ConstantExpr::getGetElementPtr(ConstStr.getElementType(), in makeConstantString()
102 llvm::Constant *makeConstantArray(StringRef Str, in makeConstantArray()
107 llvm::Constant *Value = in makeConstantArray()
108 llvm::ConstantDataArray::getString(Context, Str, AddNull); in makeConstantArray()
109 auto *GV = new llvm::GlobalVariable( in makeConstantArray()
111 llvm::GlobalValue::PrivateLinkage, Value, Name); in makeConstantArray()
116 GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::None); in makeConstantArray()
119 GV->setAlignment(llvm::Align(Alignment)); in makeConstantArray()
120 return llvm::ConstantExpr::getGetElementPtr(GV->getValueType(), GV, Zeros); in makeConstantArray()
124 llvm::Function *makeDummyFunction(llvm::FunctionType *FnTy) { in makeDummyFunction()
127 llvm::Function *DummyFunc = llvm::Function::Create( in makeDummyFunction()
128 FnTy, llvm::GlobalValue::InternalLinkage, "dummy", &TheModule); in makeDummyFunction()
130 llvm::BasicBlock *DummyBlock = in makeDummyFunction()
131 llvm::BasicBlock::Create(Context, "", DummyFunc); in makeDummyFunction()
143 void registerDeviceVar(const VarDecl *VD, llvm::GlobalVariable &Var, in registerDeviceVar()
151 void registerDeviceSurf(const VarDecl *VD, llvm::GlobalVariable &Var, in registerDeviceSurf()
159 void registerDeviceTex(const VarDecl *VD, llvm::GlobalVariable &Var, in registerDeviceTex()
168 llvm::Function *makeModuleCtorFunction();
170 llvm::Function *makeModuleDtorFunction();
179 llvm::GlobalValue *getKernelHandle(llvm::Function *F, GlobalDecl GD) override;
180 llvm::Function *getKernelStub(llvm::GlobalValue *Handle) override { in getKernelStub()
187 llvm::GlobalVariable &Var) override;
190 llvm::GlobalValue::LinkageTypes &Linkage) override;
192 llvm::Function *finalizeModule() override;
233 Zeros[0] = llvm::ConstantInt::get(SizeTy, 0); in CGNVCUDARuntime()
238 llvm::FunctionCallee CGNVCUDARuntime::getSetupArgumentFn() const { in getSetupArgumentFn()
240 llvm::Type *Params[] = {PtrTy, SizeTy, SizeTy}; in getSetupArgumentFn()
242 llvm::FunctionType::get(IntTy, Params, false), in getSetupArgumentFn()
246 llvm::FunctionCallee CGNVCUDARuntime::getLaunchFn() const { in getLaunchFn()
250 llvm::FunctionType::get(IntTy, PtrTy, false), "hipLaunchByPtr"); in getLaunchFn()
253 return CGM.CreateRuntimeFunction(llvm::FunctionType::get(IntTy, PtrTy, false), in getLaunchFn()
257 llvm::FunctionType *CGNVCUDARuntime::getRegisterGlobalsFnTy() const { in getRegisterGlobalsFnTy()
258 return llvm::FunctionType::get(VoidTy, PtrTy, false); in getRegisterGlobalsFnTy()
261 llvm::FunctionType *CGNVCUDARuntime::getCallbackFnTy() const { in getCallbackFnTy()
262 return llvm::FunctionType::get(VoidTy, PtrTy, false); in getCallbackFnTy()
265 llvm::FunctionType *CGNVCUDARuntime::getRegisterLinkedBinaryFnTy() const { in getRegisterLinkedBinaryFnTy()
266 llvm::Type *Params[] = {llvm::PointerType::getUnqual(Context), PtrTy, PtrTy, in getRegisterLinkedBinaryFnTy()
267 llvm::PointerType::getUnqual(Context)}; in getRegisterLinkedBinaryFnTy()
268 return llvm::FunctionType::get(VoidTy, Params, false); in getRegisterLinkedBinaryFnTy()
286 llvm::raw_svector_ostream Out(Buffer); in getDeviceSideName()
296 llvm::raw_svector_ostream Out(Buffer); in getDeviceSideName()
308 dyn_cast<llvm::GlobalVariable>(KernelHandles[CGF.CurFn->getName()])) { in emitDeviceStub()
331 llvm::ConstantInt::get(SizeTy, std::max<size_t>(1, Args.size()))); in emitDeviceStubBodyNew()
334 llvm::Value* VarPtr = CGF.GetAddrOfLocalVar(Args[i]).getPointer(); in emitDeviceStubBodyNew()
335 llvm::Value *VoidVarPtr = CGF.Builder.CreatePointerCast(VarPtr, PtrTy); in emitDeviceStubBodyNew()
341 llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end"); in emitDeviceStubBodyNew()
387 llvm::FunctionCallee cudaPopConfigFn = CGM.CreateRuntimeFunction( in emitDeviceStubBodyNew()
388 llvm::FunctionType::get(IntTy, in emitDeviceStubBodyNew()
401 llvm::Value *Kernel = in emitDeviceStubBodyNew()
417 llvm::Type *Ty = CGM.getTypes().ConvertType(CQT); in emitDeviceStubBodyNew()
418 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty); in emitDeviceStubBodyNew()
422 llvm::FunctionCallee cudaLaunchKernelFn = in emitDeviceStubBodyNew()
434 llvm::FunctionCallee cudaSetupArgFn = getSetupArgumentFn(); in emitDeviceStubBodyLegacy()
435 llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end"); in emitDeviceStubBodyLegacy()
440 llvm::Value *Args[] = { in emitDeviceStubBodyLegacy()
443 llvm::ConstantInt::get(SizeTy, TInfo.Width.getQuantity()), in emitDeviceStubBodyLegacy()
444 llvm::ConstantInt::get(SizeTy, Offset.getQuantity()), in emitDeviceStubBodyLegacy()
446 llvm::CallBase *CB = CGF.EmitRuntimeCallOrInvoke(cudaSetupArgFn, Args); in emitDeviceStubBodyLegacy()
447 llvm::Constant *Zero = llvm::ConstantInt::get(IntTy, 0); in emitDeviceStubBodyLegacy()
448 llvm::Value *CBZero = CGF.Builder.CreateICmpEQ(CB, Zero); in emitDeviceStubBodyLegacy()
449 llvm::BasicBlock *NextBlock = CGF.createBasicBlock("setup.next"); in emitDeviceStubBodyLegacy()
456 llvm::FunctionCallee cudaLaunchFn = getLaunchFn(); in emitDeviceStubBodyLegacy()
457 llvm::Value *Arg = in emitDeviceStubBodyLegacy()
467 static void replaceManagedVar(llvm::GlobalVariable *Var, in replaceManagedVar()
468 llvm::GlobalVariable *ManagedVar) { in replaceManagedVar()
469 SmallVector<SmallVector<llvm::User *, 8>, 8> WorkList; in replaceManagedVar()
476 if (isa<llvm::ConstantExpr>(U)) { in replaceManagedVar()
484 if (auto *I = dyn_cast<llvm::Instruction>(U)) { in replaceManagedVar()
485 llvm::Value *OldV = Var; in replaceManagedVar()
486 llvm::Instruction *NewV = in replaceManagedVar()
487 new llvm::LoadInst(Var->getType(), ManagedVar, "ld.managed", false, in replaceManagedVar()
488 llvm::Align(Var->getAlignment()), I); in replaceManagedVar()
493 auto *CE = cast<llvm::ConstantExpr>(Op); in replaceManagedVar()
520 llvm::Function *CGNVCUDARuntime::makeRegisterGlobalsFn() { in makeRegisterGlobalsFn()
525 llvm::Function *RegisterKernelsFunc = llvm::Function::Create( in makeRegisterGlobalsFn()
526 getRegisterGlobalsFnTy(), llvm::GlobalValue::InternalLinkage, in makeRegisterGlobalsFn()
528 llvm::BasicBlock *EntryBB = in makeRegisterGlobalsFn()
529 llvm::BasicBlock::Create(Context, "entry", RegisterKernelsFunc); in makeRegisterGlobalsFn()
535 llvm::Type *RegisterFuncParams[] = { in makeRegisterGlobalsFn()
537 PtrTy, PtrTy, PtrTy, PtrTy, llvm::PointerType::getUnqual(Context)}; in makeRegisterGlobalsFn()
538 llvm::FunctionCallee RegisterFunc = CGM.CreateRuntimeFunction( in makeRegisterGlobalsFn()
539 llvm::FunctionType::get(IntTy, RegisterFuncParams, false), in makeRegisterGlobalsFn()
545 llvm::Argument &GpuBinaryHandlePtr = *RegisterKernelsFunc->arg_begin(); in makeRegisterGlobalsFn()
547 llvm::Constant *KernelName = in makeRegisterGlobalsFn()
549 llvm::Constant *NullPtr = llvm::ConstantPointerNull::get(PtrTy); in makeRegisterGlobalsFn()
550 llvm::Value *Args[] = { in makeRegisterGlobalsFn()
555 llvm::ConstantInt::get(IntTy, -1), in makeRegisterGlobalsFn()
560 llvm::ConstantPointerNull::get(llvm::PointerType::getUnqual(Context))}; in makeRegisterGlobalsFn()
564 llvm::Type *VarSizeTy = IntTy; in makeRegisterGlobalsFn()
572 llvm::Type *RegisterVarParams[] = {PtrTy, PtrTy, PtrTy, PtrTy, in makeRegisterGlobalsFn()
574 llvm::FunctionCallee RegisterVar = CGM.CreateRuntimeFunction( in makeRegisterGlobalsFn()
575 llvm::FunctionType::get(VoidTy, RegisterVarParams, false), in makeRegisterGlobalsFn()
579 llvm::Type *RegisterManagedVarParams[] = {PtrTy, PtrTy, PtrTy, in makeRegisterGlobalsFn()
581 llvm::FunctionCallee RegisterManagedVar = CGM.CreateRuntimeFunction( in makeRegisterGlobalsFn()
582 llvm::FunctionType::get(VoidTy, RegisterManagedVarParams, false), in makeRegisterGlobalsFn()
586 llvm::FunctionCallee RegisterSurf = CGM.CreateRuntimeFunction( in makeRegisterGlobalsFn()
587 llvm::FunctionType::get( in makeRegisterGlobalsFn()
592 llvm::FunctionCallee RegisterTex = CGM.CreateRuntimeFunction( in makeRegisterGlobalsFn()
593 llvm::FunctionType::get( in makeRegisterGlobalsFn()
597 llvm::GlobalVariable *Var = Info.Var; in makeRegisterGlobalsFn()
601 llvm::Constant *VarName = makeConstantString(getDeviceSideName(Info.D)); in makeRegisterGlobalsFn()
607 auto *ManagedVar = new llvm::GlobalVariable( in makeRegisterGlobalsFn()
612 : llvm::ConstantPointerNull::get(Var->getType()), in makeRegisterGlobalsFn()
614 llvm::GlobalVariable::NotThreadLocal); in makeRegisterGlobalsFn()
621 llvm::Value *Args[] = { in makeRegisterGlobalsFn()
626 llvm::ConstantInt::get(VarSizeTy, VarSize), in makeRegisterGlobalsFn()
627 llvm::ConstantInt::get(IntTy, Var->getAlignment())}; in makeRegisterGlobalsFn()
631 llvm::Value *Args[] = { in makeRegisterGlobalsFn()
636 llvm::ConstantInt::get(IntTy, Info.Flags.isExtern()), in makeRegisterGlobalsFn()
637 llvm::ConstantInt::get(VarSizeTy, VarSize), in makeRegisterGlobalsFn()
638 llvm::ConstantInt::get(IntTy, Info.Flags.isConstant()), in makeRegisterGlobalsFn()
639 llvm::ConstantInt::get(IntTy, 0)}; in makeRegisterGlobalsFn()
648 llvm::ConstantInt::get(IntTy, Info.Flags.getSurfTexType()), in makeRegisterGlobalsFn()
649 llvm::ConstantInt::get(IntTy, Info.Flags.isExtern())}); in makeRegisterGlobalsFn()
655 llvm::ConstantInt::get(IntTy, Info.Flags.getSurfTexType()), in makeRegisterGlobalsFn()
656 llvm::ConstantInt::get(IntTy, Info.Flags.isNormalized()), in makeRegisterGlobalsFn()
657 llvm::ConstantInt::get(IntTy, Info.Flags.isExtern())}); in makeRegisterGlobalsFn()
685 llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() { in makeModuleCtorFunction()
697 llvm::Function *RegisterGlobalsFunc = makeRegisterGlobalsFn(); in makeModuleCtorFunction()
704 llvm::FunctionCallee RegisterFatbinFunc = CGM.CreateRuntimeFunction( in makeModuleCtorFunction()
705 llvm::FunctionType::get(PtrTy, PtrTy, false), in makeModuleCtorFunction()
708 llvm::StructType *FatbinWrapperTy = in makeModuleCtorFunction()
709 llvm::StructType::get(IntTy, IntTy, PtrTy, PtrTy); in makeModuleCtorFunction()
715 std::unique_ptr<llvm::MemoryBuffer> CudaGpuBinary = nullptr; in makeModuleCtorFunction()
728 llvm::Function *ModuleCtorFunc = llvm::Function::Create( in makeModuleCtorFunction()
729 llvm::FunctionType::get(VoidTy, false), in makeModuleCtorFunction()
730 llvm::GlobalValue::InternalLinkage, in makeModuleCtorFunction()
732 llvm::BasicBlock *CtorEntryBB = in makeModuleCtorFunction()
733 llvm::BasicBlock::Create(Context, "entry", ModuleCtorFunc); in makeModuleCtorFunction()
742 llvm::Constant *FatBinStr; in makeModuleCtorFunction()
762 FatBinStr = new llvm::GlobalVariable( in makeModuleCtorFunction()
764 /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, nullptr, in makeModuleCtorFunction()
766 llvm::GlobalVariable::NotThreadLocal); in makeModuleCtorFunction()
767 cast<llvm::GlobalVariable>(FatBinStr)->setSection(FatbinConstantName); in makeModuleCtorFunction()
805 Values.add(llvm::ConstantPointerNull::get(PtrTy)); in makeModuleCtorFunction()
806 llvm::GlobalVariable *FatbinWrapper = Values.finishAndCreateGlobal( in makeModuleCtorFunction()
819 auto Linkage = CudaGpuBinary ? llvm::GlobalValue::InternalLinkage : in makeModuleCtorFunction()
820 llvm::GlobalValue::LinkOnceAnyLinkage; in makeModuleCtorFunction()
821 llvm::BasicBlock *IfBlock = in makeModuleCtorFunction()
822 llvm::BasicBlock::Create(Context, "if", ModuleCtorFunc); in makeModuleCtorFunction()
823 llvm::BasicBlock *ExitBlock = in makeModuleCtorFunction()
824 llvm::BasicBlock::Create(Context, "exit", ModuleCtorFunc); in makeModuleCtorFunction()
827 GpuBinaryHandle = new llvm::GlobalVariable( in makeModuleCtorFunction()
829 /*Initializer=*/llvm::ConstantPointerNull::get(PtrTy), in makeModuleCtorFunction()
831 if (Linkage == llvm::GlobalValue::LinkOnceAnyLinkage) in makeModuleCtorFunction()
836 if (Linkage != llvm::GlobalValue::InternalLinkage) in makeModuleCtorFunction()
837 GpuBinaryHandle->setVisibility(llvm::GlobalValue::HiddenVisibility); in makeModuleCtorFunction()
843 llvm::Constant *Zero = in makeModuleCtorFunction()
844 llvm::Constant::getNullValue(HandleValue->getType()); in makeModuleCtorFunction()
845 llvm::Value *EQZero = CtorBuilder.CreateICmpEQ(HandleValue, Zero); in makeModuleCtorFunction()
851 llvm::CallInst *RegisterFatbinCall = in makeModuleCtorFunction()
868 llvm::CallInst *RegisterFatbinCall = in makeModuleCtorFunction()
870 GpuBinaryHandle = new llvm::GlobalVariable( in makeModuleCtorFunction()
871 TheModule, PtrTy, false, llvm::GlobalValue::InternalLinkage, in makeModuleCtorFunction()
872 llvm::ConstantPointerNull::get(PtrTy), "__cuda_gpubin_handle"); in makeModuleCtorFunction()
885 llvm::FunctionCallee RegisterFatbinEndFunc = CGM.CreateRuntimeFunction( in makeModuleCtorFunction()
886 llvm::FunctionType::get(VoidTy, PtrTy, false), in makeModuleCtorFunction()
893 llvm::raw_svector_ostream OS(ModuleID); in makeModuleCtorFunction()
894 OS << ModuleIDPrefix << llvm::format("%" PRIx64, FatbinWrapper->getGUID()); in makeModuleCtorFunction()
895 llvm::Constant *ModuleIDConstant = makeConstantArray( in makeModuleCtorFunction()
899 llvm::GlobalAlias::create(llvm::GlobalValue::ExternalLinkage, in makeModuleCtorFunction()
906 llvm::FunctionCallee RegisterLinkedBinaryFunc = CGM.CreateRuntimeFunction( in makeModuleCtorFunction()
910 llvm::Value *Args[] = {RegisterGlobalsFunc, FatbinWrapper, ModuleIDConstant, in makeModuleCtorFunction()
918 if (llvm::Function *CleanupFn = makeModuleDtorFunction()) { in makeModuleCtorFunction()
920 llvm::FunctionType *AtExitTy = in makeModuleCtorFunction()
921 llvm::FunctionType::get(IntTy, CleanupFn->getType(), false); in makeModuleCtorFunction()
922 llvm::FunctionCallee AtExitFunc = in makeModuleCtorFunction()
923 CGM.CreateRuntimeFunction(AtExitTy, "atexit", llvm::AttributeList(), in makeModuleCtorFunction()
951 llvm::Function *CGNVCUDARuntime::makeModuleDtorFunction() { in makeModuleDtorFunction()
957 llvm::FunctionCallee UnregisterFatbinFunc = CGM.CreateRuntimeFunction( in makeModuleDtorFunction()
958 llvm::FunctionType::get(VoidTy, PtrTy, false), in makeModuleDtorFunction()
961 llvm::Function *ModuleDtorFunc = llvm::Function::Create( in makeModuleDtorFunction()
962 llvm::FunctionType::get(VoidTy, false), in makeModuleDtorFunction()
963 llvm::GlobalValue::InternalLinkage, in makeModuleDtorFunction()
966 llvm::BasicBlock *DtorEntryBB = in makeModuleDtorFunction()
967 llvm::BasicBlock::Create(Context, "entry", ModuleDtorFunc); in makeModuleDtorFunction()
979 llvm::BasicBlock *IfBlock = in makeModuleDtorFunction()
980 llvm::BasicBlock::Create(Context, "if", ModuleDtorFunc); in makeModuleDtorFunction()
981 llvm::BasicBlock *ExitBlock = in makeModuleDtorFunction()
982 llvm::BasicBlock::Create(Context, "exit", ModuleDtorFunc); in makeModuleDtorFunction()
983 llvm::Constant *Zero = llvm::Constant::getNullValue(HandleValue->getType()); in makeModuleDtorFunction()
984 llvm::Value *NEZero = DtorBuilder.CreateICmpNE(HandleValue, Zero); in makeModuleDtorFunction()
1005 const VarDecl *D, llvm::GlobalValue::LinkageTypes &Linkage) { in internalizeDeviceSideVar()
1025 Linkage = llvm::GlobalValue::InternalLinkage; in internalizeDeviceSideVar()
1030 llvm::GlobalVariable &GV) { in handleVarRegistration()
1084 llvm::GlobalVariable *Var = Info.Var; in transformManagedVars()
1087 auto *ManagedVar = new llvm::GlobalVariable( in transformManagedVars()
1092 : llvm::ConstantPointerNull::get(Var->getType()), in transformManagedVars()
1094 llvm::GlobalVariable::NotThreadLocal, in transformManagedVars()
1119 llvm::Module &M = CGM.getModule(); in createOffloadingEntries()
1121 llvm::offloading::emitOffloadingEntry( in createOffloadingEntries()
1124 llvm::offloading::OffloadGlobalEntry, Section); in createOffloadingEntries()
1131 ? static_cast<int32_t>(llvm::offloading::OffloadGlobalExtern) in createOffloadingEntries()
1134 ? static_cast<int32_t>(llvm::offloading::OffloadGlobalConstant) in createOffloadingEntries()
1137 ? static_cast<int32_t>(llvm::offloading::OffloadGlobalNormalized) in createOffloadingEntries()
1140 llvm::offloading::emitOffloadingEntry( in createOffloadingEntries()
1142 (I.Flags.isManaged() ? llvm::offloading::OffloadGlobalManagedEntry in createOffloadingEntries()
1143 : llvm::offloading::OffloadGlobalEntry) | in createOffloadingEntries()
1147 llvm::offloading::emitOffloadingEntry( in createOffloadingEntries()
1149 llvm::offloading::OffloadGlobalSurfaceEntry | Flags, in createOffloadingEntries()
1152 llvm::offloading::emitOffloadingEntry( in createOffloadingEntries()
1154 llvm::offloading::OffloadGlobalTextureEntry | Flags, in createOffloadingEntries()
1161 llvm::Function *CGNVCUDARuntime::finalizeModule() { in finalizeModule()
1178 !llvm::GlobalValue::isLocalLinkage(Info.Var->getLinkage()) && in finalizeModule()
1196 llvm::GlobalValue *CGNVCUDARuntime::getKernelHandle(llvm::Function *F, in getKernelHandle()
1223 auto *Var = new llvm::GlobalVariable( in getKernelHandle()