1 //===-- AMDGPUTargetMachine.cpp - TargetMachine for hw codegen targets-----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// The AMDGPU target machine contains all of the hardware specific
11 /// information  needed to emit code for R600 and SI GPUs.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "AMDGPUTargetMachine.h"
16 #include "AMDGPU.h"
17 #include "AMDGPUAliasAnalysis.h"
18 #include "AMDGPUCallLowering.h"
19 #include "AMDGPUInstructionSelector.h"
20 #include "AMDGPULegalizerInfo.h"
21 #include "AMDGPUMacroFusion.h"
22 #include "AMDGPUTargetObjectFile.h"
23 #include "AMDGPUTargetTransformInfo.h"
24 #include "GCNIterativeScheduler.h"
25 #include "GCNSchedStrategy.h"
26 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
27 #include "R600MachineScheduler.h"
28 #include "SIMachineFunctionInfo.h"
29 #include "SIMachineScheduler.h"
30 #include "TargetInfo/AMDGPUTargetInfo.h"
31 #include "llvm/CodeGen/GlobalISel/IRTranslator.h"
32 #include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
33 #include "llvm/CodeGen/GlobalISel/Legalizer.h"
34 #include "llvm/CodeGen/GlobalISel/Localizer.h"
35 #include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
36 #include "llvm/CodeGen/MIRParser/MIParser.h"
37 #include "llvm/CodeGen/Passes.h"
38 #include "llvm/CodeGen/TargetPassConfig.h"
39 #include "llvm/IR/Attributes.h"
40 #include "llvm/IR/Function.h"
41 #include "llvm/IR/LegacyPassManager.h"
42 #include "llvm/InitializePasses.h"
43 #include "llvm/Pass.h"
44 #include "llvm/Support/CommandLine.h"
45 #include "llvm/Support/Compiler.h"
46 #include "llvm/Support/TargetRegistry.h"
47 #include "llvm/Target/TargetLoweringObjectFile.h"
48 #include "llvm/Transforms/IPO.h"
49 #include "llvm/Transforms/IPO/AlwaysInliner.h"
50 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
51 #include "llvm/Transforms/Scalar.h"
52 #include "llvm/Transforms/Scalar/GVN.h"
53 #include "llvm/Transforms/Utils.h"
54 #include "llvm/Transforms/Vectorize.h"
55 #include <memory>
56 
57 using namespace llvm;
58 
59 static cl::opt<bool> EnableR600StructurizeCFG(
60   "r600-ir-structurize",
61   cl::desc("Use StructurizeCFG IR pass"),
62   cl::init(true));
63 
64 static cl::opt<bool> EnableSROA(
65   "amdgpu-sroa",
66   cl::desc("Run SROA after promote alloca pass"),
67   cl::ReallyHidden,
68   cl::init(true));
69 
70 static cl::opt<bool>
71 EnableEarlyIfConversion("amdgpu-early-ifcvt", cl::Hidden,
72                         cl::desc("Run early if-conversion"),
73                         cl::init(false));
74 
75 static cl::opt<bool>
76 OptExecMaskPreRA("amdgpu-opt-exec-mask-pre-ra", cl::Hidden,
77             cl::desc("Run pre-RA exec mask optimizations"),
78             cl::init(true));
79 
80 static cl::opt<bool> EnableR600IfConvert(
81   "r600-if-convert",
82   cl::desc("Use if conversion pass"),
83   cl::ReallyHidden,
84   cl::init(true));
85 
86 // Option to disable vectorizer for tests.
87 static cl::opt<bool> EnableLoadStoreVectorizer(
88   "amdgpu-load-store-vectorizer",
89   cl::desc("Enable load store vectorizer"),
90   cl::init(true),
91   cl::Hidden);
92 
93 // Option to control global loads scalarization
94 static cl::opt<bool> ScalarizeGlobal(
95   "amdgpu-scalarize-global-loads",
96   cl::desc("Enable global load scalarization"),
97   cl::init(true),
98   cl::Hidden);
99 
100 // Option to run internalize pass.
101 static cl::opt<bool> InternalizeSymbols(
102   "amdgpu-internalize-symbols",
103   cl::desc("Enable elimination of non-kernel functions and unused globals"),
104   cl::init(false),
105   cl::Hidden);
106 
107 // Option to inline all early.
108 static cl::opt<bool> EarlyInlineAll(
109   "amdgpu-early-inline-all",
110   cl::desc("Inline all functions early"),
111   cl::init(false),
112   cl::Hidden);
113 
114 static cl::opt<bool> EnableSDWAPeephole(
115   "amdgpu-sdwa-peephole",
116   cl::desc("Enable SDWA peepholer"),
117   cl::init(true));
118 
119 static cl::opt<bool> EnableDPPCombine(
120   "amdgpu-dpp-combine",
121   cl::desc("Enable DPP combiner"),
122   cl::init(true));
123 
124 // Enable address space based alias analysis
125 static cl::opt<bool> EnableAMDGPUAliasAnalysis("enable-amdgpu-aa", cl::Hidden,
126   cl::desc("Enable AMDGPU Alias Analysis"),
127   cl::init(true));
128 
129 // Option to run late CFG structurizer
130 static cl::opt<bool, true> LateCFGStructurize(
131   "amdgpu-late-structurize",
132   cl::desc("Enable late CFG structurization"),
133   cl::location(AMDGPUTargetMachine::EnableLateStructurizeCFG),
134   cl::Hidden);
135 
136 static cl::opt<bool, true> EnableAMDGPUFunctionCallsOpt(
137   "amdgpu-function-calls",
138   cl::desc("Enable AMDGPU function call support"),
139   cl::location(AMDGPUTargetMachine::EnableFunctionCalls),
140   cl::init(true),
141   cl::Hidden);
142 
143 static cl::opt<bool, true> EnableAMDGPUFixedFunctionABIOpt(
144   "amdgpu-fixed-function-abi",
145   cl::desc("Enable all implicit function arguments"),
146   cl::location(AMDGPUTargetMachine::EnableFixedFunctionABI),
147   cl::init(false),
148   cl::Hidden);
149 
150 // Enable lib calls simplifications
151 static cl::opt<bool> EnableLibCallSimplify(
152   "amdgpu-simplify-libcall",
153   cl::desc("Enable amdgpu library simplifications"),
154   cl::init(true),
155   cl::Hidden);
156 
157 static cl::opt<bool> EnableLowerKernelArguments(
158   "amdgpu-ir-lower-kernel-arguments",
159   cl::desc("Lower kernel argument loads in IR pass"),
160   cl::init(true),
161   cl::Hidden);
162 
163 static cl::opt<bool> EnableRegReassign(
164   "amdgpu-reassign-regs",
165   cl::desc("Enable register reassign optimizations on gfx10+"),
166   cl::init(true),
167   cl::Hidden);
168 
169 // Enable atomic optimization
170 static cl::opt<bool> EnableAtomicOptimizations(
171   "amdgpu-atomic-optimizations",
172   cl::desc("Enable atomic optimizations"),
173   cl::init(false),
174   cl::Hidden);
175 
176 // Enable Mode register optimization
177 static cl::opt<bool> EnableSIModeRegisterPass(
178   "amdgpu-mode-register",
179   cl::desc("Enable mode register pass"),
180   cl::init(true),
181   cl::Hidden);
182 
183 // Option is used in lit tests to prevent deadcoding of patterns inspected.
184 static cl::opt<bool>
185 EnableDCEInRA("amdgpu-dce-in-ra",
186     cl::init(true), cl::Hidden,
187     cl::desc("Enable machine DCE inside regalloc"));
188 
189 static cl::opt<bool> EnableScalarIRPasses(
190   "amdgpu-scalar-ir-passes",
191   cl::desc("Enable scalar IR passes"),
192   cl::init(true),
193   cl::Hidden);
194 
195 static cl::opt<bool> EnableStructurizerWorkarounds(
196     "amdgpu-enable-structurizer-workarounds",
197     cl::desc("Enable workarounds for the StructurizeCFG pass"), cl::init(false),
198     cl::Hidden);
199 
200 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
201   // Register the target
202   RegisterTargetMachine<R600TargetMachine> X(getTheAMDGPUTarget());
203   RegisterTargetMachine<GCNTargetMachine> Y(getTheGCNTarget());
204 
205   PassRegistry *PR = PassRegistry::getPassRegistry();
206   initializeR600ClauseMergePassPass(*PR);
207   initializeR600ControlFlowFinalizerPass(*PR);
208   initializeR600PacketizerPass(*PR);
209   initializeR600ExpandSpecialInstrsPassPass(*PR);
210   initializeR600VectorRegMergerPass(*PR);
211   initializeGlobalISel(*PR);
212   initializeAMDGPUDAGToDAGISelPass(*PR);
213   initializeGCNDPPCombinePass(*PR);
214   initializeSILowerI1CopiesPass(*PR);
215   initializeSILowerSGPRSpillsPass(*PR);
216   initializeSIFixSGPRCopiesPass(*PR);
217   initializeSIFixVGPRCopiesPass(*PR);
218   initializeSIFixupVectorISelPass(*PR);
219   initializeSIFoldOperandsPass(*PR);
220   initializeSIPeepholeSDWAPass(*PR);
221   initializeSIShrinkInstructionsPass(*PR);
222   initializeSIOptimizeExecMaskingPreRAPass(*PR);
223   initializeSILoadStoreOptimizerPass(*PR);
224   initializeAMDGPUFixFunctionBitcastsPass(*PR);
225   initializeAMDGPUAlwaysInlinePass(*PR);
226   initializeAMDGPUAnnotateKernelFeaturesPass(*PR);
227   initializeAMDGPUAnnotateUniformValuesPass(*PR);
228   initializeAMDGPUArgumentUsageInfoPass(*PR);
229   initializeAMDGPUAtomicOptimizerPass(*PR);
230   initializeAMDGPULowerKernelArgumentsPass(*PR);
231   initializeAMDGPULowerKernelAttributesPass(*PR);
232   initializeAMDGPULowerIntrinsicsPass(*PR);
233   initializeAMDGPUOpenCLEnqueuedBlockLoweringPass(*PR);
234   initializeAMDGPUPostLegalizerCombinerPass(*PR);
235   initializeAMDGPUPreLegalizerCombinerPass(*PR);
236   initializeAMDGPUPromoteAllocaPass(*PR);
237   initializeAMDGPUCodeGenPreparePass(*PR);
238   initializeAMDGPUPropagateAttributesEarlyPass(*PR);
239   initializeAMDGPUPropagateAttributesLatePass(*PR);
240   initializeAMDGPURewriteOutArgumentsPass(*PR);
241   initializeAMDGPUUnifyMetadataPass(*PR);
242   initializeSIAnnotateControlFlowPass(*PR);
243   initializeSIInsertWaitcntsPass(*PR);
244   initializeSIModeRegisterPass(*PR);
245   initializeSIWholeQuadModePass(*PR);
246   initializeSILowerControlFlowPass(*PR);
247   initializeSIRemoveShortExecBranchesPass(*PR);
248   initializeSIPreEmitPeepholePass(*PR);
249   initializeSIInsertSkipsPass(*PR);
250   initializeSIMemoryLegalizerPass(*PR);
251   initializeSIOptimizeExecMaskingPass(*PR);
252   initializeSIPreAllocateWWMRegsPass(*PR);
253   initializeSIFormMemoryClausesPass(*PR);
254   initializeSIPostRABundlerPass(*PR);
255   initializeAMDGPUUnifyDivergentExitNodesPass(*PR);
256   initializeAMDGPUAAWrapperPassPass(*PR);
257   initializeAMDGPUExternalAAWrapperPass(*PR);
258   initializeAMDGPUUseNativeCallsPass(*PR);
259   initializeAMDGPUSimplifyLibCallsPass(*PR);
260   initializeAMDGPUInlinerPass(*PR);
261   initializeAMDGPUPrintfRuntimeBindingPass(*PR);
262   initializeGCNRegBankReassignPass(*PR);
263   initializeGCNNSAReassignPass(*PR);
264   initializeSIAddIMGInitPass(*PR);
265 }
266 
267 static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
268   return std::make_unique<AMDGPUTargetObjectFile>();
269 }
270 
271 static ScheduleDAGInstrs *createR600MachineScheduler(MachineSchedContext *C) {
272   return new ScheduleDAGMILive(C, std::make_unique<R600SchedStrategy>());
273 }
274 
275 static ScheduleDAGInstrs *createSIMachineScheduler(MachineSchedContext *C) {
276   return new SIScheduleDAGMI(C);
277 }
278 
279 static ScheduleDAGInstrs *
280 createGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
281   ScheduleDAGMILive *DAG =
282     new GCNScheduleDAGMILive(C, std::make_unique<GCNMaxOccupancySchedStrategy>(C));
283   DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
284   DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
285   DAG->addMutation(createAMDGPUMacroFusionDAGMutation());
286   return DAG;
287 }
288 
289 static ScheduleDAGInstrs *
290 createIterativeGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
291   auto DAG = new GCNIterativeScheduler(C,
292     GCNIterativeScheduler::SCHEDULE_LEGACYMAXOCCUPANCY);
293   DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
294   DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
295   return DAG;
296 }
297 
298 static ScheduleDAGInstrs *createMinRegScheduler(MachineSchedContext *C) {
299   return new GCNIterativeScheduler(C,
300     GCNIterativeScheduler::SCHEDULE_MINREGFORCED);
301 }
302 
303 static ScheduleDAGInstrs *
304 createIterativeILPMachineScheduler(MachineSchedContext *C) {
305   auto DAG = new GCNIterativeScheduler(C,
306     GCNIterativeScheduler::SCHEDULE_ILP);
307   DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
308   DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
309   DAG->addMutation(createAMDGPUMacroFusionDAGMutation());
310   return DAG;
311 }
312 
313 static MachineSchedRegistry
314 R600SchedRegistry("r600", "Run R600's custom scheduler",
315                    createR600MachineScheduler);
316 
317 static MachineSchedRegistry
318 SISchedRegistry("si", "Run SI's custom scheduler",
319                 createSIMachineScheduler);
320 
321 static MachineSchedRegistry
322 GCNMaxOccupancySchedRegistry("gcn-max-occupancy",
323                              "Run GCN scheduler to maximize occupancy",
324                              createGCNMaxOccupancyMachineScheduler);
325 
326 static MachineSchedRegistry
327 IterativeGCNMaxOccupancySchedRegistry("gcn-max-occupancy-experimental",
328   "Run GCN scheduler to maximize occupancy (experimental)",
329   createIterativeGCNMaxOccupancyMachineScheduler);
330 
331 static MachineSchedRegistry
332 GCNMinRegSchedRegistry("gcn-minreg",
333   "Run GCN iterative scheduler for minimal register usage (experimental)",
334   createMinRegScheduler);
335 
336 static MachineSchedRegistry
337 GCNILPSchedRegistry("gcn-ilp",
338   "Run GCN iterative scheduler for ILP scheduling (experimental)",
339   createIterativeILPMachineScheduler);
340 
341 static StringRef computeDataLayout(const Triple &TT) {
342   if (TT.getArch() == Triple::r600) {
343     // 32-bit pointers.
344       return "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128"
345              "-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5";
346   }
347 
348   // 32-bit private, local, and region pointers. 64-bit global, constant and
349   // flat, non-integral buffer fat pointers.
350     return "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32"
351          "-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128"
352          "-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
353          "-ni:7";
354 }
355 
356 LLVM_READNONE
357 static StringRef getGPUOrDefault(const Triple &TT, StringRef GPU) {
358   if (!GPU.empty())
359     return GPU;
360 
361   // Need to default to a target with flat support for HSA.
362   if (TT.getArch() == Triple::amdgcn)
363     return TT.getOS() == Triple::AMDHSA ? "generic-hsa" : "generic";
364 
365   return "r600";
366 }
367 
368 static Reloc::Model getEffectiveRelocModel(Optional<Reloc::Model> RM) {
369   // The AMDGPU toolchain only supports generating shared objects, so we
370   // must always use PIC.
371   return Reloc::PIC_;
372 }
373 
374 AMDGPUTargetMachine::AMDGPUTargetMachine(const Target &T, const Triple &TT,
375                                          StringRef CPU, StringRef FS,
376                                          TargetOptions Options,
377                                          Optional<Reloc::Model> RM,
378                                          Optional<CodeModel::Model> CM,
379                                          CodeGenOpt::Level OptLevel)
380     : LLVMTargetMachine(T, computeDataLayout(TT), TT, getGPUOrDefault(TT, CPU),
381                         FS, Options, getEffectiveRelocModel(RM),
382                         getEffectiveCodeModel(CM, CodeModel::Small), OptLevel),
383       TLOF(createTLOF(getTargetTriple())) {
384   initAsmInfo();
385   if (TT.getArch() == Triple::amdgcn) {
386     if (getMCSubtargetInfo()->checkFeatures("+wavefrontsize64"))
387       MRI.reset(llvm::createGCNMCRegisterInfo(AMDGPUDwarfFlavour::Wave64));
388     else if (getMCSubtargetInfo()->checkFeatures("+wavefrontsize32"))
389       MRI.reset(llvm::createGCNMCRegisterInfo(AMDGPUDwarfFlavour::Wave32));
390   }
391 }
392 
393 bool AMDGPUTargetMachine::EnableLateStructurizeCFG = false;
394 bool AMDGPUTargetMachine::EnableFunctionCalls = false;
395 bool AMDGPUTargetMachine::EnableFixedFunctionABI = false;
396 
397 AMDGPUTargetMachine::~AMDGPUTargetMachine() = default;
398 
399 StringRef AMDGPUTargetMachine::getGPUName(const Function &F) const {
400   Attribute GPUAttr = F.getFnAttribute("target-cpu");
401   return GPUAttr.hasAttribute(Attribute::None) ?
402     getTargetCPU() : GPUAttr.getValueAsString();
403 }
404 
405 StringRef AMDGPUTargetMachine::getFeatureString(const Function &F) const {
406   Attribute FSAttr = F.getFnAttribute("target-features");
407 
408   return FSAttr.hasAttribute(Attribute::None) ?
409     getTargetFeatureString() :
410     FSAttr.getValueAsString();
411 }
412 
413 /// Predicate for Internalize pass.
414 static bool mustPreserveGV(const GlobalValue &GV) {
415   if (const Function *F = dyn_cast<Function>(&GV))
416     return F->isDeclaration() || AMDGPU::isEntryFunctionCC(F->getCallingConv());
417 
418   return !GV.use_empty();
419 }
420 
421 void AMDGPUTargetMachine::adjustPassManager(PassManagerBuilder &Builder) {
422   Builder.DivergentTarget = true;
423 
424   bool EnableOpt = getOptLevel() > CodeGenOpt::None;
425   bool Internalize = InternalizeSymbols;
426   bool EarlyInline = EarlyInlineAll && EnableOpt && !EnableFunctionCalls;
427   bool AMDGPUAA = EnableAMDGPUAliasAnalysis && EnableOpt;
428   bool LibCallSimplify = EnableLibCallSimplify && EnableOpt;
429 
430   if (EnableFunctionCalls) {
431     delete Builder.Inliner;
432     Builder.Inliner = createAMDGPUFunctionInliningPass();
433   }
434 
435   Builder.addExtension(
436     PassManagerBuilder::EP_ModuleOptimizerEarly,
437     [Internalize, EarlyInline, AMDGPUAA, this](const PassManagerBuilder &,
438                                                legacy::PassManagerBase &PM) {
439       if (AMDGPUAA) {
440         PM.add(createAMDGPUAAWrapperPass());
441         PM.add(createAMDGPUExternalAAWrapperPass());
442       }
443       PM.add(createAMDGPUUnifyMetadataPass());
444       PM.add(createAMDGPUPrintfRuntimeBinding());
445       if (Internalize)
446         PM.add(createInternalizePass(mustPreserveGV));
447       PM.add(createAMDGPUPropagateAttributesLatePass(this));
448       if (Internalize)
449         PM.add(createGlobalDCEPass());
450       if (EarlyInline)
451         PM.add(createAMDGPUAlwaysInlinePass(false));
452   });
453 
454   Builder.addExtension(
455     PassManagerBuilder::EP_EarlyAsPossible,
456     [AMDGPUAA, LibCallSimplify, this](const PassManagerBuilder &,
457                                       legacy::PassManagerBase &PM) {
458       if (AMDGPUAA) {
459         PM.add(createAMDGPUAAWrapperPass());
460         PM.add(createAMDGPUExternalAAWrapperPass());
461       }
462       PM.add(llvm::createAMDGPUPropagateAttributesEarlyPass(this));
463       PM.add(llvm::createAMDGPUUseNativeCallsPass());
464       if (LibCallSimplify)
465         PM.add(llvm::createAMDGPUSimplifyLibCallsPass(this));
466   });
467 
468   Builder.addExtension(
469     PassManagerBuilder::EP_CGSCCOptimizerLate,
470     [](const PassManagerBuilder &, legacy::PassManagerBase &PM) {
471       // Add infer address spaces pass to the opt pipeline after inlining
472       // but before SROA to increase SROA opportunities.
473       PM.add(createInferAddressSpacesPass());
474 
475       // This should run after inlining to have any chance of doing anything,
476       // and before other cleanup optimizations.
477       PM.add(createAMDGPULowerKernelAttributesPass());
478   });
479 }
480 
481 //===----------------------------------------------------------------------===//
482 // R600 Target Machine (R600 -> Cayman)
483 //===----------------------------------------------------------------------===//
484 
485 R600TargetMachine::R600TargetMachine(const Target &T, const Triple &TT,
486                                      StringRef CPU, StringRef FS,
487                                      TargetOptions Options,
488                                      Optional<Reloc::Model> RM,
489                                      Optional<CodeModel::Model> CM,
490                                      CodeGenOpt::Level OL, bool JIT)
491     : AMDGPUTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL) {
492   setRequiresStructuredCFG(true);
493 
494   // Override the default since calls aren't supported for r600.
495   if (EnableFunctionCalls &&
496       EnableAMDGPUFunctionCallsOpt.getNumOccurrences() == 0)
497     EnableFunctionCalls = false;
498 }
499 
500 const R600Subtarget *R600TargetMachine::getSubtargetImpl(
501   const Function &F) const {
502   StringRef GPU = getGPUName(F);
503   StringRef FS = getFeatureString(F);
504 
505   SmallString<128> SubtargetKey(GPU);
506   SubtargetKey.append(FS);
507 
508   auto &I = SubtargetMap[SubtargetKey];
509   if (!I) {
510     // This needs to be done before we create a new subtarget since any
511     // creation will depend on the TM and the code generation flags on the
512     // function that reside in TargetOptions.
513     resetTargetOptions(F);
514     I = std::make_unique<R600Subtarget>(TargetTriple, GPU, FS, *this);
515   }
516 
517   return I.get();
518 }
519 
520 TargetTransformInfo
521 R600TargetMachine::getTargetTransformInfo(const Function &F) {
522   return TargetTransformInfo(R600TTIImpl(this, F));
523 }
524 
525 //===----------------------------------------------------------------------===//
526 // GCN Target Machine (SI+)
527 //===----------------------------------------------------------------------===//
528 
529 GCNTargetMachine::GCNTargetMachine(const Target &T, const Triple &TT,
530                                    StringRef CPU, StringRef FS,
531                                    TargetOptions Options,
532                                    Optional<Reloc::Model> RM,
533                                    Optional<CodeModel::Model> CM,
534                                    CodeGenOpt::Level OL, bool JIT)
535     : AMDGPUTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL) {}
536 
537 const GCNSubtarget *GCNTargetMachine::getSubtargetImpl(const Function &F) const {
538   StringRef GPU = getGPUName(F);
539   StringRef FS = getFeatureString(F);
540 
541   SmallString<128> SubtargetKey(GPU);
542   SubtargetKey.append(FS);
543 
544   auto &I = SubtargetMap[SubtargetKey];
545   if (!I) {
546     // This needs to be done before we create a new subtarget since any
547     // creation will depend on the TM and the code generation flags on the
548     // function that reside in TargetOptions.
549     resetTargetOptions(F);
550     I = std::make_unique<GCNSubtarget>(TargetTriple, GPU, FS, *this);
551   }
552 
553   I->setScalarizeGlobalBehavior(ScalarizeGlobal);
554 
555   return I.get();
556 }
557 
558 TargetTransformInfo
559 GCNTargetMachine::getTargetTransformInfo(const Function &F) {
560   return TargetTransformInfo(GCNTTIImpl(this, F));
561 }
562 
563 //===----------------------------------------------------------------------===//
564 // AMDGPU Pass Setup
565 //===----------------------------------------------------------------------===//
566 
567 namespace {
568 
569 class AMDGPUPassConfig : public TargetPassConfig {
570 public:
571   AMDGPUPassConfig(LLVMTargetMachine &TM, PassManagerBase &PM)
572     : TargetPassConfig(TM, PM) {
573     // Exceptions and StackMaps are not supported, so these passes will never do
574     // anything.
575     disablePass(&StackMapLivenessID);
576     disablePass(&FuncletLayoutID);
577   }
578 
579   AMDGPUTargetMachine &getAMDGPUTargetMachine() const {
580     return getTM<AMDGPUTargetMachine>();
581   }
582 
583   ScheduleDAGInstrs *
584   createMachineScheduler(MachineSchedContext *C) const override {
585     ScheduleDAGMILive *DAG = createGenericSchedLive(C);
586     DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
587     DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
588     return DAG;
589   }
590 
591   void addEarlyCSEOrGVNPass();
592   void addStraightLineScalarOptimizationPasses();
593   void addIRPasses() override;
594   void addCodeGenPrepare() override;
595   bool addPreISel() override;
596   bool addInstSelector() override;
597   bool addGCPasses() override;
598 
599   std::unique_ptr<CSEConfigBase> getCSEConfig() const override;
600 };
601 
602 std::unique_ptr<CSEConfigBase> AMDGPUPassConfig::getCSEConfig() const {
603   return getStandardCSEConfigForOpt(TM->getOptLevel());
604 }
605 
606 class R600PassConfig final : public AMDGPUPassConfig {
607 public:
608   R600PassConfig(LLVMTargetMachine &TM, PassManagerBase &PM)
609     : AMDGPUPassConfig(TM, PM) {}
610 
611   ScheduleDAGInstrs *createMachineScheduler(
612     MachineSchedContext *C) const override {
613     return createR600MachineScheduler(C);
614   }
615 
616   bool addPreISel() override;
617   bool addInstSelector() override;
618   void addPreRegAlloc() override;
619   void addPreSched2() override;
620   void addPreEmitPass() override;
621 };
622 
623 class GCNPassConfig final : public AMDGPUPassConfig {
624 public:
625   GCNPassConfig(LLVMTargetMachine &TM, PassManagerBase &PM)
626     : AMDGPUPassConfig(TM, PM) {
627     // It is necessary to know the register usage of the entire call graph.  We
628     // allow calls without EnableAMDGPUFunctionCalls if they are marked
629     // noinline, so this is always required.
630     setRequiresCodeGenSCCOrder(true);
631   }
632 
633   GCNTargetMachine &getGCNTargetMachine() const {
634     return getTM<GCNTargetMachine>();
635   }
636 
637   ScheduleDAGInstrs *
638   createMachineScheduler(MachineSchedContext *C) const override;
639 
640   bool addPreISel() override;
641   void addMachineSSAOptimization() override;
642   bool addILPOpts() override;
643   bool addInstSelector() override;
644   bool addIRTranslator() override;
645   void addPreLegalizeMachineIR() override;
646   bool addLegalizeMachineIR() override;
647   void addPreRegBankSelect() override;
648   bool addRegBankSelect() override;
649   bool addGlobalInstructionSelect() override;
650   void addFastRegAlloc() override;
651   void addOptimizedRegAlloc() override;
652   void addPreRegAlloc() override;
653   bool addPreRewrite() override;
654   void addPostRegAlloc() override;
655   void addPreSched2() override;
656   void addPreEmitPass() override;
657 };
658 
659 } // end anonymous namespace
660 
661 void AMDGPUPassConfig::addEarlyCSEOrGVNPass() {
662   if (getOptLevel() == CodeGenOpt::Aggressive)
663     addPass(createGVNPass());
664   else
665     addPass(createEarlyCSEPass());
666 }
667 
668 void AMDGPUPassConfig::addStraightLineScalarOptimizationPasses() {
669   addPass(createLICMPass());
670   addPass(createSeparateConstOffsetFromGEPPass());
671   addPass(createSpeculativeExecutionPass());
672   // ReassociateGEPs exposes more opportunites for SLSR. See
673   // the example in reassociate-geps-and-slsr.ll.
674   addPass(createStraightLineStrengthReducePass());
675   // SeparateConstOffsetFromGEP and SLSR creates common expressions which GVN or
676   // EarlyCSE can reuse.
677   addEarlyCSEOrGVNPass();
678   // Run NaryReassociate after EarlyCSE/GVN to be more effective.
679   addPass(createNaryReassociatePass());
680   // NaryReassociate on GEPs creates redundant common expressions, so run
681   // EarlyCSE after it.
682   addPass(createEarlyCSEPass());
683 }
684 
685 void AMDGPUPassConfig::addIRPasses() {
686   const AMDGPUTargetMachine &TM = getAMDGPUTargetMachine();
687 
688   // There is no reason to run these.
689   disablePass(&StackMapLivenessID);
690   disablePass(&FuncletLayoutID);
691   disablePass(&PatchableFunctionID);
692 
693   addPass(createAMDGPUPrintfRuntimeBinding());
694 
695   // This must occur before inlining, as the inliner will not look through
696   // bitcast calls.
697   addPass(createAMDGPUFixFunctionBitcastsPass());
698 
699   // A call to propagate attributes pass in the backend in case opt was not run.
700   addPass(createAMDGPUPropagateAttributesEarlyPass(&TM));
701 
702   addPass(createAtomicExpandPass());
703 
704 
705   addPass(createAMDGPULowerIntrinsicsPass());
706 
707   // Function calls are not supported, so make sure we inline everything.
708   addPass(createAMDGPUAlwaysInlinePass());
709   addPass(createAlwaysInlinerLegacyPass());
710   // We need to add the barrier noop pass, otherwise adding the function
711   // inlining pass will cause all of the PassConfigs passes to be run
712   // one function at a time, which means if we have a nodule with two
713   // functions, then we will generate code for the first function
714   // without ever running any passes on the second.
715   addPass(createBarrierNoopPass());
716 
717   // Handle uses of OpenCL image2d_t, image3d_t and sampler_t arguments.
718   if (TM.getTargetTriple().getArch() == Triple::r600)
719     addPass(createR600OpenCLImageTypeLoweringPass());
720 
721   // Replace OpenCL enqueued block function pointers with global variables.
722   addPass(createAMDGPUOpenCLEnqueuedBlockLoweringPass());
723 
724   if (TM.getOptLevel() > CodeGenOpt::None) {
725     addPass(createInferAddressSpacesPass());
726     addPass(createAMDGPUPromoteAlloca());
727 
728     if (EnableSROA)
729       addPass(createSROAPass());
730 
731     if (EnableScalarIRPasses)
732       addStraightLineScalarOptimizationPasses();
733 
734     if (EnableAMDGPUAliasAnalysis) {
735       addPass(createAMDGPUAAWrapperPass());
736       addPass(createExternalAAWrapperPass([](Pass &P, Function &,
737                                              AAResults &AAR) {
738         if (auto *WrapperPass = P.getAnalysisIfAvailable<AMDGPUAAWrapperPass>())
739           AAR.addAAResult(WrapperPass->getResult());
740         }));
741     }
742   }
743 
744   if (TM.getTargetTriple().getArch() == Triple::amdgcn) {
745     // TODO: May want to move later or split into an early and late one.
746     addPass(createAMDGPUCodeGenPreparePass());
747   }
748 
749   TargetPassConfig::addIRPasses();
750 
751   // EarlyCSE is not always strong enough to clean up what LSR produces. For
752   // example, GVN can combine
753   //
754   //   %0 = add %a, %b
755   //   %1 = add %b, %a
756   //
757   // and
758   //
759   //   %0 = shl nsw %a, 2
760   //   %1 = shl %a, 2
761   //
762   // but EarlyCSE can do neither of them.
763   if (getOptLevel() != CodeGenOpt::None && EnableScalarIRPasses)
764     addEarlyCSEOrGVNPass();
765 }
766 
767 void AMDGPUPassConfig::addCodeGenPrepare() {
768   if (TM->getTargetTriple().getArch() == Triple::amdgcn)
769     addPass(createAMDGPUAnnotateKernelFeaturesPass());
770 
771   if (TM->getTargetTriple().getArch() == Triple::amdgcn &&
772       EnableLowerKernelArguments)
773     addPass(createAMDGPULowerKernelArgumentsPass());
774 
775   addPass(&AMDGPUPerfHintAnalysisID);
776 
777   TargetPassConfig::addCodeGenPrepare();
778 
779   if (EnableLoadStoreVectorizer)
780     addPass(createLoadStoreVectorizerPass());
781 }
782 
783 bool AMDGPUPassConfig::addPreISel() {
784   addPass(createLowerSwitchPass());
785   addPass(createFlattenCFGPass());
786   return false;
787 }
788 
789 bool AMDGPUPassConfig::addInstSelector() {
790   // Defer the verifier until FinalizeISel.
791   addPass(createAMDGPUISelDag(&getAMDGPUTargetMachine(), getOptLevel()), false);
792   return false;
793 }
794 
795 bool AMDGPUPassConfig::addGCPasses() {
796   // Do nothing. GC is not supported.
797   return false;
798 }
799 
800 //===----------------------------------------------------------------------===//
801 // R600 Pass Setup
802 //===----------------------------------------------------------------------===//
803 
804 bool R600PassConfig::addPreISel() {
805   AMDGPUPassConfig::addPreISel();
806 
807   if (EnableR600StructurizeCFG)
808     addPass(createStructurizeCFGPass());
809   return false;
810 }
811 
812 bool R600PassConfig::addInstSelector() {
813   addPass(createR600ISelDag(&getAMDGPUTargetMachine(), getOptLevel()));
814   return false;
815 }
816 
817 void R600PassConfig::addPreRegAlloc() {
818   addPass(createR600VectorRegMerger());
819 }
820 
821 void R600PassConfig::addPreSched2() {
822   addPass(createR600EmitClauseMarkers(), false);
823   if (EnableR600IfConvert)
824     addPass(&IfConverterID, false);
825   addPass(createR600ClauseMergePass(), false);
826 }
827 
828 void R600PassConfig::addPreEmitPass() {
829   addPass(createAMDGPUCFGStructurizerPass(), false);
830   addPass(createR600ExpandSpecialInstrsPass(), false);
831   addPass(&FinalizeMachineBundlesID, false);
832   addPass(createR600Packetizer(), false);
833   addPass(createR600ControlFlowFinalizer(), false);
834 }
835 
836 TargetPassConfig *R600TargetMachine::createPassConfig(PassManagerBase &PM) {
837   return new R600PassConfig(*this, PM);
838 }
839 
840 //===----------------------------------------------------------------------===//
841 // GCN Pass Setup
842 //===----------------------------------------------------------------------===//
843 
844 ScheduleDAGInstrs *GCNPassConfig::createMachineScheduler(
845   MachineSchedContext *C) const {
846   const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
847   if (ST.enableSIScheduler())
848     return createSIMachineScheduler(C);
849   return createGCNMaxOccupancyMachineScheduler(C);
850 }
851 
852 bool GCNPassConfig::addPreISel() {
853   AMDGPUPassConfig::addPreISel();
854 
855   if (EnableAtomicOptimizations) {
856     addPass(createAMDGPUAtomicOptimizerPass());
857   }
858 
859   // FIXME: We need to run a pass to propagate the attributes when calls are
860   // supported.
861 
862   // Merge divergent exit nodes. StructurizeCFG won't recognize the multi-exit
863   // regions formed by them.
864   addPass(&AMDGPUUnifyDivergentExitNodesID);
865   if (!LateCFGStructurize) {
866     if (EnableStructurizerWorkarounds) {
867       addPass(createFixIrreduciblePass());
868       addPass(createUnifyLoopExitsPass());
869     }
870     addPass(createStructurizeCFGPass(false)); // true -> SkipUniformRegions
871   }
872   addPass(createSinkingPass());
873   addPass(createAMDGPUAnnotateUniformValues());
874   if (!LateCFGStructurize) {
875     addPass(createSIAnnotateControlFlowPass());
876   }
877   addPass(createLCSSAPass());
878 
879   return false;
880 }
881 
882 void GCNPassConfig::addMachineSSAOptimization() {
883   TargetPassConfig::addMachineSSAOptimization();
884 
885   // We want to fold operands after PeepholeOptimizer has run (or as part of
886   // it), because it will eliminate extra copies making it easier to fold the
887   // real source operand. We want to eliminate dead instructions after, so that
888   // we see fewer uses of the copies. We then need to clean up the dead
889   // instructions leftover after the operands are folded as well.
890   //
891   // XXX - Can we get away without running DeadMachineInstructionElim again?
892   addPass(&SIFoldOperandsID);
893   if (EnableDPPCombine)
894     addPass(&GCNDPPCombineID);
895   addPass(&DeadMachineInstructionElimID);
896   addPass(&SILoadStoreOptimizerID);
897   if (EnableSDWAPeephole) {
898     addPass(&SIPeepholeSDWAID);
899     addPass(&EarlyMachineLICMID);
900     addPass(&MachineCSEID);
901     addPass(&SIFoldOperandsID);
902     addPass(&DeadMachineInstructionElimID);
903   }
904   addPass(createSIShrinkInstructionsPass());
905 }
906 
907 bool GCNPassConfig::addILPOpts() {
908   if (EnableEarlyIfConversion)
909     addPass(&EarlyIfConverterID);
910 
911   TargetPassConfig::addILPOpts();
912   return false;
913 }
914 
915 bool GCNPassConfig::addInstSelector() {
916   AMDGPUPassConfig::addInstSelector();
917   addPass(&SIFixSGPRCopiesID);
918   addPass(createSILowerI1CopiesPass());
919   // TODO: We have to add FinalizeISel
920   // to expand V_ADD/SUB_U64_PSEUDO before SIFixupVectorISel
921   // that expects V_ADD/SUB -> A_ADDC/SUBB pairs expanded.
922   // Will be removed as soon as SIFixupVectorISel is changed
923   // to work with V_ADD/SUB_U64_PSEUDO instead.
924   addPass(&FinalizeISelID);
925   addPass(createSIFixupVectorISelPass());
926   addPass(createSIAddIMGInitPass());
927   return false;
928 }
929 
930 bool GCNPassConfig::addIRTranslator() {
931   addPass(new IRTranslator());
932   return false;
933 }
934 
935 void GCNPassConfig::addPreLegalizeMachineIR() {
936   bool IsOptNone = getOptLevel() == CodeGenOpt::None;
937   addPass(createAMDGPUPreLegalizeCombiner(IsOptNone));
938   addPass(new Localizer());
939 }
940 
941 bool GCNPassConfig::addLegalizeMachineIR() {
942   addPass(new Legalizer());
943   return false;
944 }
945 
946 void GCNPassConfig::addPreRegBankSelect() {
947   bool IsOptNone = getOptLevel() == CodeGenOpt::None;
948   addPass(createAMDGPUPostLegalizeCombiner(IsOptNone));
949 }
950 
951 bool GCNPassConfig::addRegBankSelect() {
952   addPass(new RegBankSelect());
953   return false;
954 }
955 
956 bool GCNPassConfig::addGlobalInstructionSelect() {
957   addPass(new InstructionSelect());
958   return false;
959 }
960 
961 void GCNPassConfig::addPreRegAlloc() {
962   if (LateCFGStructurize) {
963     addPass(createAMDGPUMachineCFGStructurizerPass());
964   }
965   addPass(createSIWholeQuadModePass());
966 }
967 
968 void GCNPassConfig::addFastRegAlloc() {
969   // FIXME: We have to disable the verifier here because of PHIElimination +
970   // TwoAddressInstructions disabling it.
971 
972   // This must be run immediately after phi elimination and before
973   // TwoAddressInstructions, otherwise the processing of the tied operand of
974   // SI_ELSE will introduce a copy of the tied operand source after the else.
975   insertPass(&PHIEliminationID, &SILowerControlFlowID, false);
976 
977   // This must be run just after RegisterCoalescing.
978   insertPass(&RegisterCoalescerID, &SIPreAllocateWWMRegsID, false);
979 
980   TargetPassConfig::addFastRegAlloc();
981 }
982 
983 void GCNPassConfig::addOptimizedRegAlloc() {
984   if (OptExecMaskPreRA) {
985     insertPass(&MachineSchedulerID, &SIOptimizeExecMaskingPreRAID);
986     insertPass(&SIOptimizeExecMaskingPreRAID, &SIFormMemoryClausesID);
987   } else {
988     insertPass(&MachineSchedulerID, &SIFormMemoryClausesID);
989   }
990 
991   // This must be run immediately after phi elimination and before
992   // TwoAddressInstructions, otherwise the processing of the tied operand of
993   // SI_ELSE will introduce a copy of the tied operand source after the else.
994   insertPass(&PHIEliminationID, &SILowerControlFlowID, false);
995 
996   // This must be run just after RegisterCoalescing.
997   insertPass(&RegisterCoalescerID, &SIPreAllocateWWMRegsID, false);
998 
999   if (EnableDCEInRA)
1000     insertPass(&DetectDeadLanesID, &DeadMachineInstructionElimID);
1001 
1002   TargetPassConfig::addOptimizedRegAlloc();
1003 }
1004 
1005 bool GCNPassConfig::addPreRewrite() {
1006   if (EnableRegReassign) {
1007     addPass(&GCNNSAReassignID);
1008     addPass(&GCNRegBankReassignID);
1009   }
1010   return true;
1011 }
1012 
1013 void GCNPassConfig::addPostRegAlloc() {
1014   addPass(&SIFixVGPRCopiesID);
1015   if (getOptLevel() > CodeGenOpt::None)
1016     addPass(&SIOptimizeExecMaskingID);
1017   TargetPassConfig::addPostRegAlloc();
1018 
1019   // Equivalent of PEI for SGPRs.
1020   addPass(&SILowerSGPRSpillsID);
1021 }
1022 
1023 void GCNPassConfig::addPreSched2() {
1024   addPass(&SIPostRABundlerID);
1025 }
1026 
1027 void GCNPassConfig::addPreEmitPass() {
1028   addPass(createSIMemoryLegalizerPass());
1029   addPass(createSIInsertWaitcntsPass());
1030   addPass(createSIShrinkInstructionsPass());
1031   addPass(createSIModeRegisterPass());
1032 
1033   // The hazard recognizer that runs as part of the post-ra scheduler does not
1034   // guarantee to be able handle all hazards correctly. This is because if there
1035   // are multiple scheduling regions in a basic block, the regions are scheduled
1036   // bottom up, so when we begin to schedule a region we don't know what
1037   // instructions were emitted directly before it.
1038   //
1039   // Here we add a stand-alone hazard recognizer pass which can handle all
1040   // cases.
1041   //
1042   // FIXME: This stand-alone pass will emit indiv. S_NOP 0, as needed. It would
1043   // be better for it to emit S_NOP <N> when possible.
1044   addPass(&PostRAHazardRecognizerID);
1045 
1046   addPass(&SIRemoveShortExecBranchesID);
1047   addPass(&SIPreEmitPeepholeID);
1048   addPass(&SIInsertSkipsPassID);
1049   addPass(&BranchRelaxationPassID);
1050 }
1051 
1052 TargetPassConfig *GCNTargetMachine::createPassConfig(PassManagerBase &PM) {
1053   return new GCNPassConfig(*this, PM);
1054 }
1055 
1056 yaml::MachineFunctionInfo *GCNTargetMachine::createDefaultFuncInfoYAML() const {
1057   return new yaml::SIMachineFunctionInfo();
1058 }
1059 
1060 yaml::MachineFunctionInfo *
1061 GCNTargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {
1062   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1063   return new yaml::SIMachineFunctionInfo(*MFI,
1064                                          *MF.getSubtarget().getRegisterInfo());
1065 }
1066 
1067 bool GCNTargetMachine::parseMachineFunctionInfo(
1068     const yaml::MachineFunctionInfo &MFI_, PerFunctionMIParsingState &PFS,
1069     SMDiagnostic &Error, SMRange &SourceRange) const {
1070   const yaml::SIMachineFunctionInfo &YamlMFI =
1071       reinterpret_cast<const yaml::SIMachineFunctionInfo &>(MFI_);
1072   MachineFunction &MF = PFS.MF;
1073   SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1074 
1075   MFI->initializeBaseYamlFields(YamlMFI);
1076 
1077   auto parseRegister = [&](const yaml::StringValue &RegName, Register &RegVal) {
1078     Register TempReg;
1079     if (parseNamedRegisterReference(PFS, TempReg, RegName.Value, Error)) {
1080       SourceRange = RegName.SourceRange;
1081       return true;
1082     }
1083     RegVal = TempReg;
1084 
1085     return false;
1086   };
1087 
1088   auto diagnoseRegisterClass = [&](const yaml::StringValue &RegName) {
1089     // Create a diagnostic for a the register string literal.
1090     const MemoryBuffer &Buffer =
1091         *PFS.SM->getMemoryBuffer(PFS.SM->getMainFileID());
1092     Error = SMDiagnostic(*PFS.SM, SMLoc(), Buffer.getBufferIdentifier(), 1,
1093                          RegName.Value.size(), SourceMgr::DK_Error,
1094                          "incorrect register class for field", RegName.Value,
1095                          None, None);
1096     SourceRange = RegName.SourceRange;
1097     return true;
1098   };
1099 
1100   if (parseRegister(YamlMFI.ScratchRSrcReg, MFI->ScratchRSrcReg) ||
1101       parseRegister(YamlMFI.FrameOffsetReg, MFI->FrameOffsetReg) ||
1102       parseRegister(YamlMFI.StackPtrOffsetReg, MFI->StackPtrOffsetReg))
1103     return true;
1104 
1105   if (MFI->ScratchRSrcReg != AMDGPU::PRIVATE_RSRC_REG &&
1106       !AMDGPU::SGPR_128RegClass.contains(MFI->ScratchRSrcReg)) {
1107     return diagnoseRegisterClass(YamlMFI.ScratchRSrcReg);
1108   }
1109 
1110   if (MFI->FrameOffsetReg != AMDGPU::FP_REG &&
1111       !AMDGPU::SGPR_32RegClass.contains(MFI->FrameOffsetReg)) {
1112     return diagnoseRegisterClass(YamlMFI.FrameOffsetReg);
1113   }
1114 
1115   if (MFI->StackPtrOffsetReg != AMDGPU::SP_REG &&
1116       !AMDGPU::SGPR_32RegClass.contains(MFI->StackPtrOffsetReg)) {
1117     return diagnoseRegisterClass(YamlMFI.StackPtrOffsetReg);
1118   }
1119 
1120   auto parseAndCheckArgument = [&](const Optional<yaml::SIArgument> &A,
1121                                    const TargetRegisterClass &RC,
1122                                    ArgDescriptor &Arg, unsigned UserSGPRs,
1123                                    unsigned SystemSGPRs) {
1124     // Skip parsing if it's not present.
1125     if (!A)
1126       return false;
1127 
1128     if (A->IsRegister) {
1129       Register Reg;
1130       if (parseNamedRegisterReference(PFS, Reg, A->RegisterName.Value, Error)) {
1131         SourceRange = A->RegisterName.SourceRange;
1132         return true;
1133       }
1134       if (!RC.contains(Reg))
1135         return diagnoseRegisterClass(A->RegisterName);
1136       Arg = ArgDescriptor::createRegister(Reg);
1137     } else
1138       Arg = ArgDescriptor::createStack(A->StackOffset);
1139     // Check and apply the optional mask.
1140     if (A->Mask)
1141       Arg = ArgDescriptor::createArg(Arg, A->Mask.getValue());
1142 
1143     MFI->NumUserSGPRs += UserSGPRs;
1144     MFI->NumSystemSGPRs += SystemSGPRs;
1145     return false;
1146   };
1147 
1148   if (YamlMFI.ArgInfo &&
1149       (parseAndCheckArgument(YamlMFI.ArgInfo->PrivateSegmentBuffer,
1150                              AMDGPU::SGPR_128RegClass,
1151                              MFI->ArgInfo.PrivateSegmentBuffer, 4, 0) ||
1152        parseAndCheckArgument(YamlMFI.ArgInfo->DispatchPtr,
1153                              AMDGPU::SReg_64RegClass, MFI->ArgInfo.DispatchPtr,
1154                              2, 0) ||
1155        parseAndCheckArgument(YamlMFI.ArgInfo->QueuePtr, AMDGPU::SReg_64RegClass,
1156                              MFI->ArgInfo.QueuePtr, 2, 0) ||
1157        parseAndCheckArgument(YamlMFI.ArgInfo->KernargSegmentPtr,
1158                              AMDGPU::SReg_64RegClass,
1159                              MFI->ArgInfo.KernargSegmentPtr, 2, 0) ||
1160        parseAndCheckArgument(YamlMFI.ArgInfo->DispatchID,
1161                              AMDGPU::SReg_64RegClass, MFI->ArgInfo.DispatchID,
1162                              2, 0) ||
1163        parseAndCheckArgument(YamlMFI.ArgInfo->FlatScratchInit,
1164                              AMDGPU::SReg_64RegClass,
1165                              MFI->ArgInfo.FlatScratchInit, 2, 0) ||
1166        parseAndCheckArgument(YamlMFI.ArgInfo->PrivateSegmentSize,
1167                              AMDGPU::SGPR_32RegClass,
1168                              MFI->ArgInfo.PrivateSegmentSize, 0, 0) ||
1169        parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupIDX,
1170                              AMDGPU::SGPR_32RegClass, MFI->ArgInfo.WorkGroupIDX,
1171                              0, 1) ||
1172        parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupIDY,
1173                              AMDGPU::SGPR_32RegClass, MFI->ArgInfo.WorkGroupIDY,
1174                              0, 1) ||
1175        parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupIDZ,
1176                              AMDGPU::SGPR_32RegClass, MFI->ArgInfo.WorkGroupIDZ,
1177                              0, 1) ||
1178        parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupInfo,
1179                              AMDGPU::SGPR_32RegClass,
1180                              MFI->ArgInfo.WorkGroupInfo, 0, 1) ||
1181        parseAndCheckArgument(YamlMFI.ArgInfo->PrivateSegmentWaveByteOffset,
1182                              AMDGPU::SGPR_32RegClass,
1183                              MFI->ArgInfo.PrivateSegmentWaveByteOffset, 0, 1) ||
1184        parseAndCheckArgument(YamlMFI.ArgInfo->ImplicitArgPtr,
1185                              AMDGPU::SReg_64RegClass,
1186                              MFI->ArgInfo.ImplicitArgPtr, 0, 0) ||
1187        parseAndCheckArgument(YamlMFI.ArgInfo->ImplicitBufferPtr,
1188                              AMDGPU::SReg_64RegClass,
1189                              MFI->ArgInfo.ImplicitBufferPtr, 2, 0) ||
1190        parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDX,
1191                              AMDGPU::VGPR_32RegClass,
1192                              MFI->ArgInfo.WorkItemIDX, 0, 0) ||
1193        parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDY,
1194                              AMDGPU::VGPR_32RegClass,
1195                              MFI->ArgInfo.WorkItemIDY, 0, 0) ||
1196        parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDZ,
1197                              AMDGPU::VGPR_32RegClass,
1198                              MFI->ArgInfo.WorkItemIDZ, 0, 0)))
1199     return true;
1200 
1201   MFI->Mode.IEEE = YamlMFI.Mode.IEEE;
1202   MFI->Mode.DX10Clamp = YamlMFI.Mode.DX10Clamp;
1203   MFI->Mode.FP32InputDenormals = YamlMFI.Mode.FP32InputDenormals;
1204   MFI->Mode.FP32OutputDenormals = YamlMFI.Mode.FP32OutputDenormals;
1205   MFI->Mode.FP64FP16InputDenormals = YamlMFI.Mode.FP64FP16InputDenormals;
1206   MFI->Mode.FP64FP16OutputDenormals = YamlMFI.Mode.FP64FP16OutputDenormals;
1207 
1208   return false;
1209 }
1210