Lines Matching refs:llvm
38 llvm::IntegerType *IntTy, *SizeTy;
39 llvm::Type *VoidTy;
40 llvm::PointerType *CharPtrTy, *VoidPtrTy, *VoidPtrPtrTy;
43 llvm::LLVMContext &Context;
45 llvm::Module &TheModule;
48 llvm::Function *Kernel; // stub function to help launch kernel
51 llvm::SmallVector<KernelInfo, 16> EmittedKernels;
55 llvm::DenseMap<llvm::Function *, llvm::GlobalValue *> KernelHandles;
57 llvm::DenseMap<llvm::GlobalValue *, llvm::Function *> KernelStubs;
59 llvm::GlobalVariable *Var;
63 llvm::SmallVector<VarInfo, 16> DeviceVars;
67 llvm::GlobalVariable *GpuBinaryHandle = nullptr;
73 llvm::FunctionCallee getSetupArgumentFn() const;
74 llvm::FunctionCallee getLaunchFn() const;
76 llvm::FunctionType *getRegisterGlobalsFnTy() const;
77 llvm::FunctionType *getCallbackFnTy() const;
78 llvm::FunctionType *getRegisterLinkedBinaryFnTy() const;
83 llvm::Function *makeRegisterGlobalsFn();
88 llvm::Constant *makeConstantString(const std::string &Str, in makeConstantString()
92 llvm::Constant *Zeros[] = {llvm::ConstantInt::get(SizeTy, 0), in makeConstantString()
93 llvm::ConstantInt::get(SizeTy, 0)}; in makeConstantString()
95 llvm::GlobalVariable *GV = in makeConstantString()
96 cast<llvm::GlobalVariable>(ConstStr.getPointer()); in makeConstantString()
101 GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::None); in makeConstantString()
104 GV->setAlignment(llvm::Align(Alignment)); in makeConstantString()
106 return llvm::ConstantExpr::getGetElementPtr(ConstStr.getElementType(), in makeConstantString()
111 llvm::Function *makeDummyFunction(llvm::FunctionType *FnTy) { in makeDummyFunction()
114 llvm::Function *DummyFunc = llvm::Function::Create( in makeDummyFunction()
115 FnTy, llvm::GlobalValue::InternalLinkage, "dummy", &TheModule); in makeDummyFunction()
117 llvm::BasicBlock *DummyBlock = in makeDummyFunction()
118 llvm::BasicBlock::Create(Context, "", DummyFunc); in makeDummyFunction()
130 void registerDeviceVar(const VarDecl *VD, llvm::GlobalVariable &Var, in registerDeviceVar()
138 void registerDeviceSurf(const VarDecl *VD, llvm::GlobalVariable &Var, in registerDeviceSurf()
146 void registerDeviceTex(const VarDecl *VD, llvm::GlobalVariable &Var, in registerDeviceTex()
155 llvm::Function *makeModuleCtorFunction();
157 llvm::Function *makeModuleDtorFunction();
164 llvm::GlobalValue *getKernelHandle(llvm::Function *F, GlobalDecl GD) override;
165 llvm::Function *getKernelStub(llvm::GlobalValue *Handle) override { in getKernelStub()
172 llvm::GlobalVariable &Var) override;
175 llvm::GlobalValue::LinkageTypes &Linkage) override;
177 llvm::Function *finalizeModule() override;
222 CharPtrTy = llvm::PointerType::getUnqual(Types.ConvertType(Ctx.CharTy)); in CGNVCUDARuntime()
223 VoidPtrTy = cast<llvm::PointerType>(Types.ConvertType(Ctx.VoidPtrTy)); in CGNVCUDARuntime()
227 llvm::FunctionCallee CGNVCUDARuntime::getSetupArgumentFn() const { in getSetupArgumentFn()
229 llvm::Type *Params[] = {VoidPtrTy, SizeTy, SizeTy}; in getSetupArgumentFn()
231 llvm::FunctionType::get(IntTy, Params, false), in getSetupArgumentFn()
235 llvm::FunctionCallee CGNVCUDARuntime::getLaunchFn() const { in getLaunchFn()
239 llvm::FunctionType::get(IntTy, CharPtrTy, false), "hipLaunchByPtr"); in getLaunchFn()
243 llvm::FunctionType::get(IntTy, CharPtrTy, false), "cudaLaunch"); in getLaunchFn()
247 llvm::FunctionType *CGNVCUDARuntime::getRegisterGlobalsFnTy() const { in getRegisterGlobalsFnTy()
248 return llvm::FunctionType::get(VoidTy, VoidPtrPtrTy, false); in getRegisterGlobalsFnTy()
251 llvm::FunctionType *CGNVCUDARuntime::getCallbackFnTy() const { in getCallbackFnTy()
252 return llvm::FunctionType::get(VoidTy, VoidPtrTy, false); in getCallbackFnTy()
255 llvm::FunctionType *CGNVCUDARuntime::getRegisterLinkedBinaryFnTy() const { in getRegisterLinkedBinaryFnTy()
258 llvm::Type *Params[] = {RegisterGlobalsFnTy->getPointerTo(), VoidPtrTy, in getRegisterLinkedBinaryFnTy()
260 return llvm::FunctionType::get(VoidTy, Params, false); in getRegisterLinkedBinaryFnTy()
278 llvm::raw_svector_ostream Out(Buffer); in getDeviceSideName()
289 llvm::raw_svector_ostream Out(Buffer); in getDeviceSideName()
300 if (auto *GV = dyn_cast<llvm::GlobalVariable>(KernelHandles[CGF.CurFn])) { in emitDeviceStub()
323 llvm::ConstantInt::get(SizeTy, std::max<size_t>(1, Args.size()))); in emitDeviceStubBodyNew()
326 llvm::Value* VarPtr = CGF.GetAddrOfLocalVar(Args[i]).getPointer(); in emitDeviceStubBodyNew()
327 llvm::Value *VoidVarPtr = CGF.Builder.CreatePointerCast(VarPtr, VoidPtrTy); in emitDeviceStubBodyNew()
333 llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end"); in emitDeviceStubBodyNew()
369 llvm::FunctionCallee cudaPopConfigFn = CGM.CreateRuntimeFunction( in emitDeviceStubBodyNew()
370 llvm::FunctionType::get(IntTy, in emitDeviceStubBodyNew()
383 llvm::Value *Kernel = in emitDeviceStubBodyNew()
399 llvm::Type *Ty = CGM.getTypes().ConvertType(CQT); in emitDeviceStubBodyNew()
400 llvm::FunctionType *FTy = dyn_cast<llvm::FunctionType>(Ty); in emitDeviceStubBodyNew()
404 llvm::FunctionCallee cudaLaunchKernelFn = in emitDeviceStubBodyNew()
416 llvm::FunctionCallee cudaSetupArgFn = getSetupArgumentFn(); in emitDeviceStubBodyLegacy()
417 llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end"); in emitDeviceStubBodyLegacy()
422 llvm::Value *Args[] = { in emitDeviceStubBodyLegacy()
425 llvm::ConstantInt::get(SizeTy, TInfo.Width.getQuantity()), in emitDeviceStubBodyLegacy()
426 llvm::ConstantInt::get(SizeTy, Offset.getQuantity()), in emitDeviceStubBodyLegacy()
428 llvm::CallBase *CB = CGF.EmitRuntimeCallOrInvoke(cudaSetupArgFn, Args); in emitDeviceStubBodyLegacy()
429 llvm::Constant *Zero = llvm::ConstantInt::get(IntTy, 0); in emitDeviceStubBodyLegacy()
430 llvm::Value *CBZero = CGF.Builder.CreateICmpEQ(CB, Zero); in emitDeviceStubBodyLegacy()
431 llvm::BasicBlock *NextBlock = CGF.createBasicBlock("setup.next"); in emitDeviceStubBodyLegacy()
438 llvm::FunctionCallee cudaLaunchFn = getLaunchFn(); in emitDeviceStubBodyLegacy()
439 llvm::Value *Arg = in emitDeviceStubBodyLegacy()
449 static void replaceManagedVar(llvm::GlobalVariable *Var, in replaceManagedVar()
450 llvm::GlobalVariable *ManagedVar) { in replaceManagedVar()
451 SmallVector<SmallVector<llvm::User *, 8>, 8> WorkList; in replaceManagedVar()
458 if (isa<llvm::ConstantExpr>(U)) { in replaceManagedVar()
466 if (auto *I = dyn_cast<llvm::Instruction>(U)) { in replaceManagedVar()
467 llvm::Value *OldV = Var; in replaceManagedVar()
468 llvm::Instruction *NewV = in replaceManagedVar()
469 new llvm::LoadInst(Var->getType(), ManagedVar, "ld.managed", false, in replaceManagedVar()
470 llvm::Align(Var->getAlignment()), I); in replaceManagedVar()
475 auto *CE = cast<llvm::ConstantExpr>(Op); in replaceManagedVar()
476 auto *NewInst = llvm::createReplacementInstr(CE, I); in replaceManagedVar()
502 llvm::Function *CGNVCUDARuntime::makeRegisterGlobalsFn() { in makeRegisterGlobalsFn()
507 llvm::Function *RegisterKernelsFunc = llvm::Function::Create( in makeRegisterGlobalsFn()
508 getRegisterGlobalsFnTy(), llvm::GlobalValue::InternalLinkage, in makeRegisterGlobalsFn()
510 llvm::BasicBlock *EntryBB = in makeRegisterGlobalsFn()
511 llvm::BasicBlock::Create(Context, "entry", RegisterKernelsFunc); in makeRegisterGlobalsFn()
517 llvm::Type *RegisterFuncParams[] = { in makeRegisterGlobalsFn()
520 llvm::FunctionCallee RegisterFunc = CGM.CreateRuntimeFunction( in makeRegisterGlobalsFn()
521 llvm::FunctionType::get(IntTy, RegisterFuncParams, false), in makeRegisterGlobalsFn()
527 llvm::Argument &GpuBinaryHandlePtr = *RegisterKernelsFunc->arg_begin(); in makeRegisterGlobalsFn()
529 llvm::Constant *KernelName = in makeRegisterGlobalsFn()
531 llvm::Constant *NullPtr = llvm::ConstantPointerNull::get(VoidPtrTy); in makeRegisterGlobalsFn()
532 llvm::Value *Args[] = { in makeRegisterGlobalsFn()
537 llvm::ConstantInt::get(IntTy, -1), in makeRegisterGlobalsFn()
542 llvm::ConstantPointerNull::get(IntTy->getPointerTo())}; in makeRegisterGlobalsFn()
546 llvm::Type *VarSizeTy = IntTy; in makeRegisterGlobalsFn()
554 llvm::Type *RegisterVarParams[] = {VoidPtrPtrTy, CharPtrTy, CharPtrTy, in makeRegisterGlobalsFn()
557 llvm::FunctionCallee RegisterVar = CGM.CreateRuntimeFunction( in makeRegisterGlobalsFn()
558 llvm::FunctionType::get(VoidTy, RegisterVarParams, false), in makeRegisterGlobalsFn()
562 llvm::Type *RegisterManagedVarParams[] = {VoidPtrPtrTy, CharPtrTy, CharPtrTy, in makeRegisterGlobalsFn()
564 llvm::FunctionCallee RegisterManagedVar = CGM.CreateRuntimeFunction( in makeRegisterGlobalsFn()
565 llvm::FunctionType::get(VoidTy, RegisterManagedVarParams, false), in makeRegisterGlobalsFn()
569 llvm::FunctionCallee RegisterSurf = CGM.CreateRuntimeFunction( in makeRegisterGlobalsFn()
570 llvm::FunctionType::get( in makeRegisterGlobalsFn()
576 llvm::FunctionCallee RegisterTex = CGM.CreateRuntimeFunction( in makeRegisterGlobalsFn()
577 llvm::FunctionType::get( in makeRegisterGlobalsFn()
583 llvm::GlobalVariable *Var = Info.Var; in makeRegisterGlobalsFn()
587 llvm::Constant *VarName = makeConstantString(getDeviceSideName(Info.D)); in makeRegisterGlobalsFn()
593 auto ManagedVar = new llvm::GlobalVariable( in makeRegisterGlobalsFn()
598 : llvm::ConstantPointerNull::get(Var->getType()), in makeRegisterGlobalsFn()
600 llvm::GlobalVariable::NotThreadLocal); in makeRegisterGlobalsFn()
607 llvm::Value *Args[] = { in makeRegisterGlobalsFn()
612 llvm::ConstantInt::get(VarSizeTy, VarSize), in makeRegisterGlobalsFn()
613 llvm::ConstantInt::get(IntTy, Var->getAlignment())}; in makeRegisterGlobalsFn()
617 llvm::Value *Args[] = { in makeRegisterGlobalsFn()
622 llvm::ConstantInt::get(IntTy, Info.Flags.isExtern()), in makeRegisterGlobalsFn()
623 llvm::ConstantInt::get(VarSizeTy, VarSize), in makeRegisterGlobalsFn()
624 llvm::ConstantInt::get(IntTy, Info.Flags.isConstant()), in makeRegisterGlobalsFn()
625 llvm::ConstantInt::get(IntTy, 0)}; in makeRegisterGlobalsFn()
634 VarName, llvm::ConstantInt::get(IntTy, Info.Flags.getSurfTexType()), in makeRegisterGlobalsFn()
635 llvm::ConstantInt::get(IntTy, Info.Flags.isExtern())}); in makeRegisterGlobalsFn()
641 VarName, llvm::ConstantInt::get(IntTy, Info.Flags.getSurfTexType()), in makeRegisterGlobalsFn()
642 llvm::ConstantInt::get(IntTy, Info.Flags.isNormalized()), in makeRegisterGlobalsFn()
643 llvm::ConstantInt::get(IntTy, Info.Flags.isExtern())}); in makeRegisterGlobalsFn()
671 llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() { in makeModuleCtorFunction()
683 llvm::Function *RegisterGlobalsFunc = makeRegisterGlobalsFn(); in makeModuleCtorFunction()
690 llvm::FunctionCallee RegisterFatbinFunc = CGM.CreateRuntimeFunction( in makeModuleCtorFunction()
691 llvm::FunctionType::get(VoidPtrPtrTy, VoidPtrTy, false), in makeModuleCtorFunction()
694 llvm::StructType *FatbinWrapperTy = in makeModuleCtorFunction()
695 llvm::StructType::get(IntTy, IntTy, VoidPtrTy, VoidPtrTy); in makeModuleCtorFunction()
701 std::unique_ptr<llvm::MemoryBuffer> CudaGpuBinary = nullptr; in makeModuleCtorFunction()
703 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> CudaGpuBinaryOrErr = in makeModuleCtorFunction()
704 llvm::MemoryBuffer::getFileOrSTDIN(CudaGpuBinaryFileName); in makeModuleCtorFunction()
713 llvm::Function *ModuleCtorFunc = llvm::Function::Create( in makeModuleCtorFunction()
714 llvm::FunctionType::get(VoidTy, VoidPtrTy, false), in makeModuleCtorFunction()
715 llvm::GlobalValue::InternalLinkage, in makeModuleCtorFunction()
717 llvm::BasicBlock *CtorEntryBB = in makeModuleCtorFunction()
718 llvm::BasicBlock::Create(Context, "entry", ModuleCtorFunc); in makeModuleCtorFunction()
727 llvm::Constant *FatBinStr; in makeModuleCtorFunction()
748 FatBinStr = new llvm::GlobalVariable( in makeModuleCtorFunction()
750 /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, nullptr, in makeModuleCtorFunction()
752 llvm::GlobalVariable::NotThreadLocal); in makeModuleCtorFunction()
753 cast<llvm::GlobalVariable>(FatBinStr)->setSection(FatbinConstantName); in makeModuleCtorFunction()
791 Values.add(llvm::ConstantPointerNull::get(VoidPtrTy)); in makeModuleCtorFunction()
792 llvm::GlobalVariable *FatbinWrapper = Values.finishAndCreateGlobal( in makeModuleCtorFunction()
805 auto Linkage = CudaGpuBinary ? llvm::GlobalValue::InternalLinkage : in makeModuleCtorFunction()
806 llvm::GlobalValue::LinkOnceAnyLinkage; in makeModuleCtorFunction()
807 llvm::BasicBlock *IfBlock = in makeModuleCtorFunction()
808 llvm::BasicBlock::Create(Context, "if", ModuleCtorFunc); in makeModuleCtorFunction()
809 llvm::BasicBlock *ExitBlock = in makeModuleCtorFunction()
810 llvm::BasicBlock::Create(Context, "exit", ModuleCtorFunc); in makeModuleCtorFunction()
813 GpuBinaryHandle = new llvm::GlobalVariable( in makeModuleCtorFunction()
816 /*Initializer=*/llvm::ConstantPointerNull::get(VoidPtrPtrTy), in makeModuleCtorFunction()
820 if (Linkage != llvm::GlobalValue::InternalLinkage) in makeModuleCtorFunction()
821 GpuBinaryHandle->setVisibility(llvm::GlobalValue::HiddenVisibility); in makeModuleCtorFunction()
827 llvm::Constant *Zero = in makeModuleCtorFunction()
828 llvm::Constant::getNullValue(HandleValue->getType()); in makeModuleCtorFunction()
829 llvm::Value *EQZero = CtorBuilder.CreateICmpEQ(HandleValue, Zero); in makeModuleCtorFunction()
835 llvm::CallInst *RegisterFatbinCall = CtorBuilder.CreateCall( in makeModuleCtorFunction()
853 llvm::CallInst *RegisterFatbinCall = CtorBuilder.CreateCall( in makeModuleCtorFunction()
856 GpuBinaryHandle = new llvm::GlobalVariable( in makeModuleCtorFunction()
857 TheModule, VoidPtrPtrTy, false, llvm::GlobalValue::InternalLinkage, in makeModuleCtorFunction()
858 llvm::ConstantPointerNull::get(VoidPtrPtrTy), "__cuda_gpubin_handle"); in makeModuleCtorFunction()
871 llvm::FunctionCallee RegisterFatbinEndFunc = CGM.CreateRuntimeFunction( in makeModuleCtorFunction()
872 llvm::FunctionType::get(VoidTy, VoidPtrPtrTy, false), in makeModuleCtorFunction()
879 llvm::raw_svector_ostream OS(ModuleID); in makeModuleCtorFunction()
880 OS << ModuleIDPrefix << llvm::format("%" PRIx64, FatbinWrapper->getGUID()); in makeModuleCtorFunction()
881 llvm::Constant *ModuleIDConstant = makeConstantString( in makeModuleCtorFunction()
885 llvm::GlobalAlias::create(llvm::GlobalValue::ExternalLinkage, in makeModuleCtorFunction()
892 llvm::FunctionCallee RegisterLinkedBinaryFunc = CGM.CreateRuntimeFunction( in makeModuleCtorFunction()
896 llvm::Value *Args[] = {RegisterGlobalsFunc, in makeModuleCtorFunction()
906 if (llvm::Function *CleanupFn = makeModuleDtorFunction()) { in makeModuleCtorFunction()
908 llvm::FunctionType *AtExitTy = in makeModuleCtorFunction()
909 llvm::FunctionType::get(IntTy, CleanupFn->getType(), false); in makeModuleCtorFunction()
910 llvm::FunctionCallee AtExitFunc = in makeModuleCtorFunction()
911 CGM.CreateRuntimeFunction(AtExitTy, "atexit", llvm::AttributeList(), in makeModuleCtorFunction()
939 llvm::Function *CGNVCUDARuntime::makeModuleDtorFunction() { in makeModuleDtorFunction()
945 llvm::FunctionCallee UnregisterFatbinFunc = CGM.CreateRuntimeFunction( in makeModuleDtorFunction()
946 llvm::FunctionType::get(VoidTy, VoidPtrPtrTy, false), in makeModuleDtorFunction()
949 llvm::Function *ModuleDtorFunc = llvm::Function::Create( in makeModuleDtorFunction()
950 llvm::FunctionType::get(VoidTy, VoidPtrTy, false), in makeModuleDtorFunction()
951 llvm::GlobalValue::InternalLinkage, in makeModuleDtorFunction()
954 llvm::BasicBlock *DtorEntryBB = in makeModuleDtorFunction()
955 llvm::BasicBlock::Create(Context, "entry", ModuleDtorFunc); in makeModuleDtorFunction()
966 llvm::BasicBlock *IfBlock = in makeModuleDtorFunction()
967 llvm::BasicBlock::Create(Context, "if", ModuleDtorFunc); in makeModuleDtorFunction()
968 llvm::BasicBlock *ExitBlock = in makeModuleDtorFunction()
969 llvm::BasicBlock::Create(Context, "exit", ModuleDtorFunc); in makeModuleDtorFunction()
970 llvm::Constant *Zero = llvm::Constant::getNullValue(HandleValue->getType()); in makeModuleDtorFunction()
971 llvm::Value *NEZero = DtorBuilder.CreateICmpNE(HandleValue, Zero); in makeModuleDtorFunction()
992 const VarDecl *D, llvm::GlobalValue::LinkageTypes &Linkage) { in internalizeDeviceSideVar()
1012 Linkage = llvm::GlobalValue::InternalLinkage; in internalizeDeviceSideVar()
1017 llvm::GlobalVariable &GV) { in handleVarRegistration()
1071 llvm::GlobalVariable *Var = Info.Var; in transformManagedVars()
1074 auto ManagedVar = new llvm::GlobalVariable( in transformManagedVars()
1079 : llvm::ConstantPointerNull::get(Var->getType()), in transformManagedVars()
1081 llvm::GlobalVariable::NotThreadLocal, in transformManagedVars()
1101 llvm::Function *CGNVCUDARuntime::finalizeModule() { in finalizeModule()
1118 !llvm::GlobalValue::isLocalLinkage(Info.Var->getLinkage()) && in finalizeModule()
1131 llvm::GlobalValue *CGNVCUDARuntime::getKernelHandle(llvm::Function *F, in getKernelHandle()
1143 auto *Var = new llvm::GlobalVariable( in getKernelHandle()