1 //===- AMDGPUOpenMP.cpp - AMDGPUOpenMP ToolChain Implementation -*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "AMDGPUOpenMP.h"
10 #include "AMDGPU.h"
11 #include "CommonArgs.h"
12 #include "ToolChains/ROCm.h"
13 #include "clang/Basic/DiagnosticDriver.h"
14 #include "clang/Driver/Compilation.h"
15 #include "clang/Driver/Driver.h"
16 #include "clang/Driver/DriverDiagnostic.h"
17 #include "clang/Driver/InputInfo.h"
18 #include "clang/Driver/Options.h"
19 #include "llvm/ADT/STLExtras.h"
20 #include "llvm/Support/FileSystem.h"
21 #include "llvm/Support/FormatAdapters.h"
22 #include "llvm/Support/FormatVariadic.h"
23 #include "llvm/Support/Path.h"
24 
25 using namespace clang::driver;
26 using namespace clang::driver::toolchains;
27 using namespace clang::driver::tools;
28 using namespace clang;
29 using namespace llvm::opt;
30 
31 namespace {
32 
33 static const char *getOutputFileName(Compilation &C, StringRef Base,
34                                      const char *Postfix,
35                                      const char *Extension) {
36   const char *OutputFileName;
37   if (C.getDriver().isSaveTempsEnabled()) {
38     OutputFileName =
39         C.getArgs().MakeArgString(Base.str() + Postfix + "." + Extension);
40   } else {
41     std::string TmpName =
42         C.getDriver().GetTemporaryPath(Base.str() + Postfix, Extension);
43     OutputFileName = C.addTempFile(C.getArgs().MakeArgString(TmpName));
44   }
45   return OutputFileName;
46 }
47 
48 static void addLLCOptArg(const llvm::opt::ArgList &Args,
49                          llvm::opt::ArgStringList &CmdArgs) {
50   if (Arg *A = Args.getLastArg(options::OPT_O_Group)) {
51     StringRef OOpt = "0";
52     if (A->getOption().matches(options::OPT_O4) ||
53         A->getOption().matches(options::OPT_Ofast))
54       OOpt = "3";
55     else if (A->getOption().matches(options::OPT_O0))
56       OOpt = "0";
57     else if (A->getOption().matches(options::OPT_O)) {
58       // Clang and opt support -Os/-Oz; llc only supports -O0, -O1, -O2 and -O3
59       // so we map -Os/-Oz to -O2.
60       // Only clang supports -Og, and maps it to -O1.
61       // We map anything else to -O2.
62       OOpt = llvm::StringSwitch<const char *>(A->getValue())
63                  .Case("1", "1")
64                  .Case("2", "2")
65                  .Case("3", "3")
66                  .Case("s", "2")
67                  .Case("z", "2")
68                  .Case("g", "1")
69                  .Default("0");
70     }
71     CmdArgs.push_back(Args.MakeArgString("-O" + OOpt));
72   }
73 }
74 
75 static bool checkSystemForAMDGPU(const ArgList &Args, const AMDGPUToolChain &TC,
76                                  std::string &GPUArch) {
77   if (auto Err = TC.getSystemGPUArch(Args, GPUArch)) {
78     std::string ErrMsg =
79         llvm::formatv("{0}", llvm::fmt_consume(std::move(Err)));
80     TC.getDriver().Diag(diag::err_drv_undetermined_amdgpu_arch) << ErrMsg;
81     return false;
82   }
83 
84   return true;
85 }
86 } // namespace
87 
88 const char *AMDGCN::OpenMPLinker::constructLLVMLinkCommand(
89     const toolchains::AMDGPUOpenMPToolChain &AMDGPUOpenMPTC, Compilation &C,
90     const JobAction &JA, const InputInfoList &Inputs, const ArgList &Args,
91     StringRef SubArchName, StringRef OutputFilePrefix) const {
92   ArgStringList CmdArgs;
93 
94   for (const auto &II : Inputs)
95     if (II.isFilename())
96       CmdArgs.push_back(II.getFilename());
97 
98   if (Args.hasArg(options::OPT_l)) {
99     auto Lm = Args.getAllArgValues(options::OPT_l);
100     bool HasLibm = false;
101     for (auto &Lib : Lm) {
102       if (Lib == "m") {
103         HasLibm = true;
104         break;
105       }
106     }
107 
108     if (HasLibm) {
109       // This is not certain to work. The device libs added here, and passed to
110       // llvm-link, are missing attributes that they expect to be inserted when
111       // passed to mlink-builtin-bitcode. The amdgpu backend does not generate
112       // conservatively correct code when attributes are missing, so this may
113       // be the root cause of miscompilations. Passing via mlink-builtin-bitcode
114       // ultimately hits CodeGenModule::addDefaultFunctionDefinitionAttributes
115       // on each function, see D28538 for context.
116       // Potential workarounds:
117       //  - unconditionally link all of the device libs to every translation
118       //    unit in clang via mlink-builtin-bitcode
119       //  - build a libm bitcode file as part of the DeviceRTL and explictly
120       //    mlink-builtin-bitcode the rocm device libs components at build time
121       //  - drop this llvm-link fork in favour or some calls into LLVM, chosen
122       //    to do basically the same work as llvm-link but with that call first
123       //  - write an opt pass that sets that on every function it sees and pipe
124       //    the device-libs bitcode through that on the way to this llvm-link
125       SmallVector<std::string, 12> BCLibs =
126           AMDGPUOpenMPTC.getCommonDeviceLibNames(Args, SubArchName.str());
127       llvm::for_each(BCLibs, [&](StringRef BCFile) {
128         CmdArgs.push_back(Args.MakeArgString(BCFile));
129       });
130     }
131   }
132 
133   AddStaticDeviceLibsLinking(C, *this, JA, Inputs, Args, CmdArgs, "amdgcn",
134                              SubArchName, /*isBitCodeSDL=*/true,
135                              /*postClangLink=*/false);
136   // Add an intermediate output file.
137   CmdArgs.push_back("-o");
138   const char *OutputFileName =
139       getOutputFileName(C, OutputFilePrefix, "-linked", "bc");
140   CmdArgs.push_back(OutputFileName);
141   const char *Exec =
142       Args.MakeArgString(getToolChain().GetProgramPath("llvm-link"));
143   C.addCommand(std::make_unique<Command>(
144       JA, *this, ResponseFileSupport::AtFileCurCP(), Exec, CmdArgs, Inputs,
145       InputInfo(&JA, Args.MakeArgString(OutputFileName))));
146   return OutputFileName;
147 }
148 
149 const char *AMDGCN::OpenMPLinker::constructLlcCommand(
150     Compilation &C, const JobAction &JA, const InputInfoList &Inputs,
151     const llvm::opt::ArgList &Args, llvm::StringRef SubArchName,
152     llvm::StringRef OutputFilePrefix, const char *InputFileName,
153     bool OutputIsAsm) const {
154   // Construct llc command.
155   ArgStringList LlcArgs;
156   // The input to llc is the output from opt.
157   LlcArgs.push_back(InputFileName);
158   // Pass optimization arg to llc.
159   addLLCOptArg(Args, LlcArgs);
160   LlcArgs.push_back("-mtriple=amdgcn-amd-amdhsa");
161   LlcArgs.push_back(Args.MakeArgString("-mcpu=" + SubArchName));
162   LlcArgs.push_back(
163       Args.MakeArgString(Twine("-filetype=") + (OutputIsAsm ? "asm" : "obj")));
164 
165   for (const Arg *A : Args.filtered(options::OPT_mllvm)) {
166     LlcArgs.push_back(A->getValue(0));
167   }
168 
169   // Add output filename
170   LlcArgs.push_back("-o");
171   const char *LlcOutputFile =
172       getOutputFileName(C, OutputFilePrefix, "", OutputIsAsm ? "s" : "o");
173   LlcArgs.push_back(LlcOutputFile);
174   const char *Llc = Args.MakeArgString(getToolChain().GetProgramPath("llc"));
175   C.addCommand(std::make_unique<Command>(
176       JA, *this, ResponseFileSupport::AtFileCurCP(), Llc, LlcArgs, Inputs,
177       InputInfo(&JA, Args.MakeArgString(LlcOutputFile))));
178   return LlcOutputFile;
179 }
180 
181 void AMDGCN::OpenMPLinker::constructLldCommand(
182     Compilation &C, const JobAction &JA, const InputInfoList &Inputs,
183     const InputInfo &Output, const llvm::opt::ArgList &Args,
184     const char *InputFileName) const {
185   // Construct lld command.
186   // The output from ld.lld is an HSA code object file.
187   ArgStringList LldArgs{"-flavor",    "gnu", "--no-undefined",
188                         "-shared",    "-o",  Output.getFilename(),
189                         InputFileName};
190 
191   const char *Lld = Args.MakeArgString(getToolChain().GetProgramPath("lld"));
192   C.addCommand(std::make_unique<Command>(
193       JA, *this, ResponseFileSupport::AtFileCurCP(), Lld, LldArgs, Inputs,
194       InputInfo(&JA, Args.MakeArgString(Output.getFilename()))));
195 }
196 
197 // For amdgcn the inputs of the linker job are device bitcode and output is
198 // object file. It calls llvm-link, opt, llc, then lld steps.
199 void AMDGCN::OpenMPLinker::ConstructJob(Compilation &C, const JobAction &JA,
200                                         const InputInfo &Output,
201                                         const InputInfoList &Inputs,
202                                         const ArgList &Args,
203                                         const char *LinkingOutput) const {
204   const ToolChain &TC = getToolChain();
205   assert(getToolChain().getTriple().isAMDGCN() && "Unsupported target");
206 
207   const toolchains::AMDGPUOpenMPToolChain &AMDGPUOpenMPTC =
208       static_cast<const toolchains::AMDGPUOpenMPToolChain &>(TC);
209 
210   std::string GPUArch = Args.getLastArgValue(options::OPT_march_EQ).str();
211   if (GPUArch.empty()) {
212     if (!checkSystemForAMDGPU(Args, AMDGPUOpenMPTC, GPUArch))
213       return;
214   }
215 
216   // Prefix for temporary file name.
217   std::string Prefix;
218   for (const auto &II : Inputs)
219     if (II.isFilename())
220       Prefix = llvm::sys::path::stem(II.getFilename()).str() + "-" + GPUArch;
221   assert(Prefix.length() && "no linker inputs are files ");
222 
223   // Each command outputs different files.
224   const char *LLVMLinkCommand = constructLLVMLinkCommand(
225       AMDGPUOpenMPTC, C, JA, Inputs, Args, GPUArch, Prefix);
226 
227   // Produce readable assembly if save-temps is enabled.
228   if (C.getDriver().isSaveTempsEnabled())
229     constructLlcCommand(C, JA, Inputs, Args, GPUArch, Prefix, LLVMLinkCommand,
230                         /*OutputIsAsm=*/true);
231   const char *LlcCommand = constructLlcCommand(C, JA, Inputs, Args, GPUArch,
232                                                Prefix, LLVMLinkCommand);
233   constructLldCommand(C, JA, Inputs, Output, Args, LlcCommand);
234 }
235 
236 AMDGPUOpenMPToolChain::AMDGPUOpenMPToolChain(const Driver &D,
237                                              const llvm::Triple &Triple,
238                                              const ToolChain &HostTC,
239                                              const ArgList &Args)
240     : ROCMToolChain(D, Triple, Args), HostTC(HostTC) {
241   // Lookup binaries into the driver directory, this is used to
242   // discover the clang-offload-bundler executable.
243   getProgramPaths().push_back(getDriver().Dir);
244 }
245 
246 void AMDGPUOpenMPToolChain::addClangTargetOptions(
247     const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args,
248     Action::OffloadKind DeviceOffloadingKind) const {
249   HostTC.addClangTargetOptions(DriverArgs, CC1Args, DeviceOffloadingKind);
250 
251   std::string GPUArch = DriverArgs.getLastArgValue(options::OPT_march_EQ).str();
252   if (GPUArch.empty()) {
253     if (!checkSystemForAMDGPU(DriverArgs, *this, GPUArch))
254       return;
255   }
256 
257   assert(DeviceOffloadingKind == Action::OFK_OpenMP &&
258          "Only OpenMP offloading kinds are supported.");
259 
260   CC1Args.push_back("-target-cpu");
261   CC1Args.push_back(DriverArgs.MakeArgStringRef(GPUArch));
262   CC1Args.push_back("-fcuda-is-device");
263 
264   if (DriverArgs.hasArg(options::OPT_nogpulib))
265     return;
266 
267   std::string BitcodeSuffix;
268   if (DriverArgs.hasFlag(options::OPT_fopenmp_target_new_runtime,
269                          options::OPT_fno_openmp_target_new_runtime, true))
270     BitcodeSuffix = "new-amdgpu-" + GPUArch;
271   else
272     BitcodeSuffix = "amdgcn-" + GPUArch;
273 
274   addOpenMPDeviceRTL(getDriver(), DriverArgs, CC1Args, BitcodeSuffix,
275                      getTriple());
276 }
277 
278 llvm::opt::DerivedArgList *AMDGPUOpenMPToolChain::TranslateArgs(
279     const llvm::opt::DerivedArgList &Args, StringRef BoundArch,
280     Action::OffloadKind DeviceOffloadKind) const {
281   DerivedArgList *DAL =
282       HostTC.TranslateArgs(Args, BoundArch, DeviceOffloadKind);
283   if (!DAL)
284     DAL = new DerivedArgList(Args.getBaseArgs());
285 
286   const OptTable &Opts = getDriver().getOpts();
287 
288   if (DeviceOffloadKind != Action::OFK_OpenMP) {
289     for (Arg *A : Args) {
290       DAL->append(A);
291     }
292   }
293 
294   if (!BoundArch.empty()) {
295     DAL->eraseArg(options::OPT_march_EQ);
296     DAL->AddJoinedArg(nullptr, Opts.getOption(options::OPT_march_EQ),
297                       BoundArch);
298   }
299 
300   return DAL;
301 }
302 
303 Tool *AMDGPUOpenMPToolChain::buildLinker() const {
304   assert(getTriple().isAMDGCN());
305   return new tools::AMDGCN::OpenMPLinker(*this);
306 }
307 
308 void AMDGPUOpenMPToolChain::addClangWarningOptions(
309     ArgStringList &CC1Args) const {
310   HostTC.addClangWarningOptions(CC1Args);
311 }
312 
313 ToolChain::CXXStdlibType
314 AMDGPUOpenMPToolChain::GetCXXStdlibType(const ArgList &Args) const {
315   return HostTC.GetCXXStdlibType(Args);
316 }
317 
318 void AMDGPUOpenMPToolChain::AddClangSystemIncludeArgs(
319     const ArgList &DriverArgs, ArgStringList &CC1Args) const {
320   HostTC.AddClangSystemIncludeArgs(DriverArgs, CC1Args);
321 }
322 
323 void AMDGPUOpenMPToolChain::AddIAMCUIncludeArgs(const ArgList &Args,
324                                                 ArgStringList &CC1Args) const {
325   HostTC.AddIAMCUIncludeArgs(Args, CC1Args);
326 }
327 
328 SanitizerMask AMDGPUOpenMPToolChain::getSupportedSanitizers() const {
329   // The AMDGPUOpenMPToolChain only supports sanitizers in the sense that it
330   // allows sanitizer arguments on the command line if they are supported by the
331   // host toolchain. The AMDGPUOpenMPToolChain will actually ignore any command
332   // line arguments for any of these "supported" sanitizers. That means that no
333   // sanitization of device code is actually supported at this time.
334   //
335   // This behavior is necessary because the host and device toolchains
336   // invocations often share the command line, so the device toolchain must
337   // tolerate flags meant only for the host toolchain.
338   return HostTC.getSupportedSanitizers();
339 }
340 
341 VersionTuple
342 AMDGPUOpenMPToolChain::computeMSVCVersion(const Driver *D,
343                                           const ArgList &Args) const {
344   return HostTC.computeMSVCVersion(D, Args);
345 }
346