1 //===----- CGCUDANV.cpp - Interface to NVIDIA CUDA Runtime ----------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This provides a class for CUDA code generation targeting the NVIDIA CUDA
11 // runtime library.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "CGCUDARuntime.h"
16 #include "CodeGenFunction.h"
17 #include "CodeGenModule.h"
18 #include "clang/AST/Decl.h"
19 #include "clang/CodeGen/ConstantInitBuilder.h"
20 #include "llvm/IR/BasicBlock.h"
21 #include "llvm/IR/CallSite.h"
22 #include "llvm/IR/Constants.h"
23 #include "llvm/IR/DerivedTypes.h"
24 #include "llvm/Support/Format.h"
25 
26 using namespace clang;
27 using namespace CodeGen;
28 
29 namespace {
30 
31 class CGNVCUDARuntime : public CGCUDARuntime {
32 
33 private:
34   llvm::IntegerType *IntTy, *SizeTy;
35   llvm::Type *VoidTy;
36   llvm::PointerType *CharPtrTy, *VoidPtrTy, *VoidPtrPtrTy;
37 
38   /// Convenience reference to LLVM Context
39   llvm::LLVMContext &Context;
40   /// Convenience reference to the current module
41   llvm::Module &TheModule;
42   /// Keeps track of kernel launch stubs emitted in this module
43   llvm::SmallVector<llvm::Function *, 16> EmittedKernels;
44   llvm::SmallVector<std::pair<llvm::GlobalVariable *, unsigned>, 16> DeviceVars;
45   /// Keeps track of variable containing handle of GPU binary. Populated by
46   /// ModuleCtorFunction() and used to create corresponding cleanup calls in
47   /// ModuleDtorFunction()
48   llvm::GlobalVariable *GpuBinaryHandle = nullptr;
49   /// Whether we generate relocatable device code.
50   bool RelocatableDeviceCode;
51 
52   llvm::Constant *getSetupArgumentFn() const;
53   llvm::Constant *getLaunchFn() const;
54 
55   llvm::FunctionType *getRegisterGlobalsFnTy() const;
56   llvm::FunctionType *getCallbackFnTy() const;
57   llvm::FunctionType *getRegisterLinkedBinaryFnTy() const;
58 
59   /// Creates a function to register all kernel stubs generated in this module.
60   llvm::Function *makeRegisterGlobalsFn();
61 
62   /// Helper function that generates a constant string and returns a pointer to
63   /// the start of the string.  The result of this function can be used anywhere
64   /// where the C code specifies const char*.
65   llvm::Constant *makeConstantString(const std::string &Str,
66                                      const std::string &Name = "",
67                                      const std::string &SectionName = "",
68                                      unsigned Alignment = 0) {
69     llvm::Constant *Zeros[] = {llvm::ConstantInt::get(SizeTy, 0),
70                                llvm::ConstantInt::get(SizeTy, 0)};
71     auto ConstStr = CGM.GetAddrOfConstantCString(Str, Name.c_str());
72     llvm::GlobalVariable *GV =
73         cast<llvm::GlobalVariable>(ConstStr.getPointer());
74     if (!SectionName.empty())
75       GV->setSection(SectionName);
76     if (Alignment)
77       GV->setAlignment(Alignment);
78 
79     return llvm::ConstantExpr::getGetElementPtr(ConstStr.getElementType(),
80                                                 ConstStr.getPointer(), Zeros);
81   }
82 
83   /// Helper function that generates an empty dummy function returning void.
84   llvm::Function *makeDummyFunction(llvm::FunctionType *FnTy) {
85     assert(FnTy->getReturnType()->isVoidTy() &&
86            "Can only generate dummy functions returning void!");
87     llvm::Function *DummyFunc = llvm::Function::Create(
88         FnTy, llvm::GlobalValue::InternalLinkage, "dummy", &TheModule);
89 
90     llvm::BasicBlock *DummyBlock =
91         llvm::BasicBlock::Create(Context, "", DummyFunc);
92     CGBuilderTy FuncBuilder(CGM, Context);
93     FuncBuilder.SetInsertPoint(DummyBlock);
94     FuncBuilder.CreateRetVoid();
95 
96     return DummyFunc;
97   }
98 
99   void emitDeviceStubBody(CodeGenFunction &CGF, FunctionArgList &Args);
100 
101 public:
102   CGNVCUDARuntime(CodeGenModule &CGM);
103 
104   void emitDeviceStub(CodeGenFunction &CGF, FunctionArgList &Args) override;
105   void registerDeviceVar(llvm::GlobalVariable &Var, unsigned Flags) override {
106     DeviceVars.push_back(std::make_pair(&Var, Flags));
107   }
108 
109   /// Creates module constructor function
110   llvm::Function *makeModuleCtorFunction() override;
111   /// Creates module destructor function
112   llvm::Function *makeModuleDtorFunction() override;
113 };
114 
115 }
116 
117 CGNVCUDARuntime::CGNVCUDARuntime(CodeGenModule &CGM)
118     : CGCUDARuntime(CGM), Context(CGM.getLLVMContext()),
119       TheModule(CGM.getModule()),
120       RelocatableDeviceCode(CGM.getLangOpts().CUDARelocatableDeviceCode) {
121   CodeGen::CodeGenTypes &Types = CGM.getTypes();
122   ASTContext &Ctx = CGM.getContext();
123 
124   IntTy = CGM.IntTy;
125   SizeTy = CGM.SizeTy;
126   VoidTy = CGM.VoidTy;
127 
128   CharPtrTy = llvm::PointerType::getUnqual(Types.ConvertType(Ctx.CharTy));
129   VoidPtrTy = cast<llvm::PointerType>(Types.ConvertType(Ctx.VoidPtrTy));
130   VoidPtrPtrTy = VoidPtrTy->getPointerTo();
131 }
132 
133 llvm::Constant *CGNVCUDARuntime::getSetupArgumentFn() const {
134   // cudaError_t cudaSetupArgument(void *, size_t, size_t)
135   llvm::Type *Params[] = {VoidPtrTy, SizeTy, SizeTy};
136   return CGM.CreateRuntimeFunction(llvm::FunctionType::get(IntTy,
137                                                            Params, false),
138                                    "cudaSetupArgument");
139 }
140 
141 llvm::Constant *CGNVCUDARuntime::getLaunchFn() const {
142   // cudaError_t cudaLaunch(char *)
143   return CGM.CreateRuntimeFunction(
144       llvm::FunctionType::get(IntTy, CharPtrTy, false), "cudaLaunch");
145 }
146 
147 llvm::FunctionType *CGNVCUDARuntime::getRegisterGlobalsFnTy() const {
148   return llvm::FunctionType::get(VoidTy, VoidPtrPtrTy, false);
149 }
150 
151 llvm::FunctionType *CGNVCUDARuntime::getCallbackFnTy() const {
152   return llvm::FunctionType::get(VoidTy, VoidPtrTy, false);
153 }
154 
155 llvm::FunctionType *CGNVCUDARuntime::getRegisterLinkedBinaryFnTy() const {
156   auto CallbackFnTy = getCallbackFnTy();
157   auto RegisterGlobalsFnTy = getRegisterGlobalsFnTy();
158   llvm::Type *Params[] = {RegisterGlobalsFnTy->getPointerTo(), VoidPtrTy,
159                           VoidPtrTy, CallbackFnTy->getPointerTo()};
160   return llvm::FunctionType::get(VoidTy, Params, false);
161 }
162 
163 void CGNVCUDARuntime::emitDeviceStub(CodeGenFunction &CGF,
164                                      FunctionArgList &Args) {
165   EmittedKernels.push_back(CGF.CurFn);
166   emitDeviceStubBody(CGF, Args);
167 }
168 
169 void CGNVCUDARuntime::emitDeviceStubBody(CodeGenFunction &CGF,
170                                          FunctionArgList &Args) {
171   // Emit a call to cudaSetupArgument for each arg in Args.
172   llvm::Constant *cudaSetupArgFn = getSetupArgumentFn();
173   llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end");
174   CharUnits Offset = CharUnits::Zero();
175   for (const VarDecl *A : Args) {
176     CharUnits TyWidth, TyAlign;
177     std::tie(TyWidth, TyAlign) =
178         CGM.getContext().getTypeInfoInChars(A->getType());
179     Offset = Offset.alignTo(TyAlign);
180     llvm::Value *Args[] = {
181         CGF.Builder.CreatePointerCast(CGF.GetAddrOfLocalVar(A).getPointer(),
182                                       VoidPtrTy),
183         llvm::ConstantInt::get(SizeTy, TyWidth.getQuantity()),
184         llvm::ConstantInt::get(SizeTy, Offset.getQuantity()),
185     };
186     llvm::CallSite CS = CGF.EmitRuntimeCallOrInvoke(cudaSetupArgFn, Args);
187     llvm::Constant *Zero = llvm::ConstantInt::get(IntTy, 0);
188     llvm::Value *CSZero = CGF.Builder.CreateICmpEQ(CS.getInstruction(), Zero);
189     llvm::BasicBlock *NextBlock = CGF.createBasicBlock("setup.next");
190     CGF.Builder.CreateCondBr(CSZero, NextBlock, EndBlock);
191     CGF.EmitBlock(NextBlock);
192     Offset += TyWidth;
193   }
194 
195   // Emit the call to cudaLaunch
196   llvm::Constant *cudaLaunchFn = getLaunchFn();
197   llvm::Value *Arg = CGF.Builder.CreatePointerCast(CGF.CurFn, CharPtrTy);
198   CGF.EmitRuntimeCallOrInvoke(cudaLaunchFn, Arg);
199   CGF.EmitBranch(EndBlock);
200 
201   CGF.EmitBlock(EndBlock);
202 }
203 
204 /// Creates a function that sets up state on the host side for CUDA objects that
205 /// have a presence on both the host and device sides. Specifically, registers
206 /// the host side of kernel functions and device global variables with the CUDA
207 /// runtime.
208 /// \code
209 /// void __cuda_register_globals(void** GpuBinaryHandle) {
210 ///    __cudaRegisterFunction(GpuBinaryHandle,Kernel0,...);
211 ///    ...
212 ///    __cudaRegisterFunction(GpuBinaryHandle,KernelM,...);
213 ///    __cudaRegisterVar(GpuBinaryHandle, GlobalVar0, ...);
214 ///    ...
215 ///    __cudaRegisterVar(GpuBinaryHandle, GlobalVarN, ...);
216 /// }
217 /// \endcode
218 llvm::Function *CGNVCUDARuntime::makeRegisterGlobalsFn() {
219   // No need to register anything
220   if (EmittedKernels.empty() && DeviceVars.empty())
221     return nullptr;
222 
223   llvm::Function *RegisterKernelsFunc = llvm::Function::Create(
224       getRegisterGlobalsFnTy(), llvm::GlobalValue::InternalLinkage,
225       "__cuda_register_globals", &TheModule);
226   llvm::BasicBlock *EntryBB =
227       llvm::BasicBlock::Create(Context, "entry", RegisterKernelsFunc);
228   CGBuilderTy Builder(CGM, Context);
229   Builder.SetInsertPoint(EntryBB);
230 
231   // void __cudaRegisterFunction(void **, const char *, char *, const char *,
232   //                             int, uint3*, uint3*, dim3*, dim3*, int*)
233   llvm::Type *RegisterFuncParams[] = {
234       VoidPtrPtrTy, CharPtrTy, CharPtrTy, CharPtrTy, IntTy,
235       VoidPtrTy,    VoidPtrTy, VoidPtrTy, VoidPtrTy, IntTy->getPointerTo()};
236   llvm::Constant *RegisterFunc = CGM.CreateRuntimeFunction(
237       llvm::FunctionType::get(IntTy, RegisterFuncParams, false),
238       "__cudaRegisterFunction");
239 
240   // Extract GpuBinaryHandle passed as the first argument passed to
241   // __cuda_register_globals() and generate __cudaRegisterFunction() call for
242   // each emitted kernel.
243   llvm::Argument &GpuBinaryHandlePtr = *RegisterKernelsFunc->arg_begin();
244   for (llvm::Function *Kernel : EmittedKernels) {
245     llvm::Constant *KernelName = makeConstantString(Kernel->getName());
246     llvm::Constant *NullPtr = llvm::ConstantPointerNull::get(VoidPtrTy);
247     llvm::Value *Args[] = {
248         &GpuBinaryHandlePtr, Builder.CreateBitCast(Kernel, VoidPtrTy),
249         KernelName, KernelName, llvm::ConstantInt::get(IntTy, -1), NullPtr,
250         NullPtr, NullPtr, NullPtr,
251         llvm::ConstantPointerNull::get(IntTy->getPointerTo())};
252     Builder.CreateCall(RegisterFunc, Args);
253   }
254 
255   // void __cudaRegisterVar(void **, char *, char *, const char *,
256   //                        int, int, int, int)
257   llvm::Type *RegisterVarParams[] = {VoidPtrPtrTy, CharPtrTy, CharPtrTy,
258                                      CharPtrTy,    IntTy,     IntTy,
259                                      IntTy,        IntTy};
260   llvm::Constant *RegisterVar = CGM.CreateRuntimeFunction(
261       llvm::FunctionType::get(IntTy, RegisterVarParams, false),
262       "__cudaRegisterVar");
263   for (auto &Pair : DeviceVars) {
264     llvm::GlobalVariable *Var = Pair.first;
265     unsigned Flags = Pair.second;
266     llvm::Constant *VarName = makeConstantString(Var->getName());
267     uint64_t VarSize =
268         CGM.getDataLayout().getTypeAllocSize(Var->getValueType());
269     llvm::Value *Args[] = {
270         &GpuBinaryHandlePtr,
271         Builder.CreateBitCast(Var, VoidPtrTy),
272         VarName,
273         VarName,
274         llvm::ConstantInt::get(IntTy, (Flags & ExternDeviceVar) ? 1 : 0),
275         llvm::ConstantInt::get(IntTy, VarSize),
276         llvm::ConstantInt::get(IntTy, (Flags & ConstantDeviceVar) ? 1 : 0),
277         llvm::ConstantInt::get(IntTy, 0)};
278     Builder.CreateCall(RegisterVar, Args);
279   }
280 
281   Builder.CreateRetVoid();
282   return RegisterKernelsFunc;
283 }
284 
285 /// Creates a global constructor function for the module:
286 /// \code
287 /// void __cuda_module_ctor(void*) {
288 ///     Handle = __cudaRegisterFatBinary(GpuBinaryBlob);
289 ///     __cuda_register_globals(Handle);
290 /// }
291 /// \endcode
292 llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() {
293   // No need to generate ctors/dtors if there is no GPU binary.
294   std::string GpuBinaryFileName = CGM.getCodeGenOpts().CudaGpuBinaryFileName;
295   if (GpuBinaryFileName.empty())
296     return nullptr;
297 
298   // void __cuda_register_globals(void* handle);
299   llvm::Function *RegisterGlobalsFunc = makeRegisterGlobalsFn();
300   // We always need a function to pass in as callback. Create a dummy
301   // implementation if we don't need to register anything.
302   if (RelocatableDeviceCode && !RegisterGlobalsFunc)
303     RegisterGlobalsFunc = makeDummyFunction(getRegisterGlobalsFnTy());
304 
305   // void ** __cudaRegisterFatBinary(void *);
306   llvm::Constant *RegisterFatbinFunc = CGM.CreateRuntimeFunction(
307       llvm::FunctionType::get(VoidPtrPtrTy, VoidPtrTy, false),
308       "__cudaRegisterFatBinary");
309   // struct { int magic, int version, void * gpu_binary, void * dont_care };
310   llvm::StructType *FatbinWrapperTy =
311       llvm::StructType::get(IntTy, IntTy, VoidPtrTy, VoidPtrTy);
312 
313   // Register GPU binary with the CUDA runtime, store returned handle in a
314   // global variable and save a reference in GpuBinaryHandle to be cleaned up
315   // in destructor on exit. Then associate all known kernels with the GPU binary
316   // handle so CUDA runtime can figure out what to call on the GPU side.
317   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> GpuBinaryOrErr =
318       llvm::MemoryBuffer::getFileOrSTDIN(GpuBinaryFileName);
319   if (std::error_code EC = GpuBinaryOrErr.getError()) {
320     CGM.getDiags().Report(diag::err_cannot_open_file)
321         << GpuBinaryFileName << EC.message();
322     return nullptr;
323   }
324 
325   llvm::Function *ModuleCtorFunc = llvm::Function::Create(
326       llvm::FunctionType::get(VoidTy, VoidPtrTy, false),
327       llvm::GlobalValue::InternalLinkage, "__cuda_module_ctor", &TheModule);
328   llvm::BasicBlock *CtorEntryBB =
329       llvm::BasicBlock::Create(Context, "entry", ModuleCtorFunc);
330   CGBuilderTy CtorBuilder(CGM, Context);
331 
332   CtorBuilder.SetInsertPoint(CtorEntryBB);
333 
334   const char *FatbinConstantName;
335   if (RelocatableDeviceCode)
336     // TODO: Figure out how this is called on mac OS!
337     FatbinConstantName = "__nv_relfatbin";
338   else
339     FatbinConstantName =
340         CGM.getTriple().isMacOSX() ? "__NV_CUDA,__nv_fatbin" : ".nv_fatbin";
341   // NVIDIA's cuobjdump looks for fatbins in this section.
342   const char *FatbinSectionName =
343       CGM.getTriple().isMacOSX() ? "__NV_CUDA,__fatbin" : ".nvFatBinSegment";
344   // TODO: Figure out how this is called on mac OS!
345   const char *NVModuleIDSectionName = "__nv_module_id";
346 
347   // Create initialized wrapper structure that points to the loaded GPU binary
348   ConstantInitBuilder Builder(CGM);
349   auto Values = Builder.beginStruct(FatbinWrapperTy);
350   // Fatbin wrapper magic.
351   Values.addInt(IntTy, 0x466243b1);
352   // Fatbin version.
353   Values.addInt(IntTy, 1);
354   // Data.
355   Values.add(makeConstantString(GpuBinaryOrErr.get()->getBuffer(), "",
356                                 FatbinConstantName, 8));
357   // Unused in fatbin v1.
358   Values.add(llvm::ConstantPointerNull::get(VoidPtrTy));
359   llvm::GlobalVariable *FatbinWrapper = Values.finishAndCreateGlobal(
360       "__cuda_fatbin_wrapper", CGM.getPointerAlign(),
361       /*constant*/ true);
362   FatbinWrapper->setSection(FatbinSectionName);
363 
364   // Register binary with CUDA runtime. This is substantially different in
365   // default mode vs. separate compilation!
366   if (!RelocatableDeviceCode) {
367     // GpuBinaryHandle = __cudaRegisterFatBinary(&FatbinWrapper);
368     llvm::CallInst *RegisterFatbinCall = CtorBuilder.CreateCall(
369         RegisterFatbinFunc,
370         CtorBuilder.CreateBitCast(FatbinWrapper, VoidPtrTy));
371     GpuBinaryHandle = new llvm::GlobalVariable(
372         TheModule, VoidPtrPtrTy, false, llvm::GlobalValue::InternalLinkage,
373         llvm::ConstantPointerNull::get(VoidPtrPtrTy), "__cuda_gpubin_handle");
374     CtorBuilder.CreateAlignedStore(RegisterFatbinCall, GpuBinaryHandle,
375                                    CGM.getPointerAlign());
376 
377     // Call __cuda_register_globals(GpuBinaryHandle);
378     if (RegisterGlobalsFunc)
379       CtorBuilder.CreateCall(RegisterGlobalsFunc, RegisterFatbinCall);
380   } else {
381     // Generate a unique module ID.
382     SmallString<64> NVModuleID;
383     llvm::raw_svector_ostream OS(NVModuleID);
384     OS << "__nv_" << llvm::format("%x", FatbinWrapper->getGUID());
385     llvm::Constant *NVModuleIDConstant =
386         makeConstantString(NVModuleID.str(), "", NVModuleIDSectionName, 32);
387 
388     // Create an alias for the FatbinWrapper that nvcc will look for.
389     llvm::GlobalAlias::create(llvm::GlobalValue::ExternalLinkage,
390                               Twine("__fatbinwrap") + NVModuleID,
391                               FatbinWrapper);
392 
393     // void __cudaRegisterLinkedBinary%NVModuleID%(void (*)(void *), void *,
394     // void *, void (*)(void **))
395     SmallString<128> RegisterLinkedBinaryName("__cudaRegisterLinkedBinary");
396     RegisterLinkedBinaryName += NVModuleID;
397     llvm::Constant *RegisterLinkedBinaryFunc = CGM.CreateRuntimeFunction(
398         getRegisterLinkedBinaryFnTy(), RegisterLinkedBinaryName);
399 
400     assert(RegisterGlobalsFunc && "Expecting at least dummy function!");
401     llvm::Value *Args[] = {RegisterGlobalsFunc,
402                            CtorBuilder.CreateBitCast(FatbinWrapper, VoidPtrTy),
403                            NVModuleIDConstant,
404                            makeDummyFunction(getCallbackFnTy())};
405     CtorBuilder.CreateCall(RegisterLinkedBinaryFunc, Args);
406   }
407 
408   CtorBuilder.CreateRetVoid();
409   return ModuleCtorFunc;
410 }
411 
412 /// Creates a global destructor function that unregisters the GPU code blob
413 /// registered by constructor.
414 /// \code
415 /// void __cuda_module_dtor(void*) {
416 ///     __cudaUnregisterFatBinary(Handle);
417 /// }
418 /// \endcode
419 llvm::Function *CGNVCUDARuntime::makeModuleDtorFunction() {
420   // No need for destructor if we don't have a handle to unregister.
421   if (!GpuBinaryHandle)
422     return nullptr;
423 
424   // void __cudaUnregisterFatBinary(void ** handle);
425   llvm::Constant *UnregisterFatbinFunc = CGM.CreateRuntimeFunction(
426       llvm::FunctionType::get(VoidTy, VoidPtrPtrTy, false),
427       "__cudaUnregisterFatBinary");
428 
429   llvm::Function *ModuleDtorFunc = llvm::Function::Create(
430       llvm::FunctionType::get(VoidTy, VoidPtrTy, false),
431       llvm::GlobalValue::InternalLinkage, "__cuda_module_dtor", &TheModule);
432   llvm::BasicBlock *DtorEntryBB =
433       llvm::BasicBlock::Create(Context, "entry", ModuleDtorFunc);
434   CGBuilderTy DtorBuilder(CGM, Context);
435   DtorBuilder.SetInsertPoint(DtorEntryBB);
436 
437   auto HandleValue =
438       DtorBuilder.CreateAlignedLoad(GpuBinaryHandle, CGM.getPointerAlign());
439   DtorBuilder.CreateCall(UnregisterFatbinFunc, HandleValue);
440 
441   DtorBuilder.CreateRetVoid();
442   return ModuleDtorFunc;
443 }
444 
445 CGCUDARuntime *CodeGen::CreateNVCUDARuntime(CodeGenModule &CGM) {
446   return new CGNVCUDARuntime(CGM);
447 }
448