1 //===-- AMDGPUSubtarget.cpp - AMDGPU Subtarget Information ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Implements the AMDGPU specific subclass of TargetSubtarget.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "AMDGPUSubtarget.h"
15 #include "AMDGPU.h"
16 #include "AMDGPUTargetMachine.h"
17 #include "AMDGPUCallLowering.h"
18 #include "AMDGPUInstructionSelector.h"
19 #include "AMDGPULegalizerInfo.h"
20 #include "AMDGPURegisterBankInfo.h"
21 #include "SIMachineFunctionInfo.h"
22 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
23 #include "llvm/ADT/SmallString.h"
24 #include "llvm/CodeGen/MachineScheduler.h"
25 #include "llvm/MC/MCSubtargetInfo.h"
26 #include "llvm/IR/MDBuilder.h"
27 #include "llvm/CodeGen/TargetFrameLowering.h"
28 #include <algorithm>
29 
30 using namespace llvm;
31 
32 #define DEBUG_TYPE "amdgpu-subtarget"
33 
34 #define GET_SUBTARGETINFO_TARGET_DESC
35 #define GET_SUBTARGETINFO_CTOR
36 #define AMDGPUSubtarget GCNSubtarget
37 #include "AMDGPUGenSubtargetInfo.inc"
38 #define GET_SUBTARGETINFO_TARGET_DESC
39 #define GET_SUBTARGETINFO_CTOR
40 #undef AMDGPUSubtarget
41 #include "R600GenSubtargetInfo.inc"
42 
43 GCNSubtarget::~GCNSubtarget() = default;
44 
45 R600Subtarget &
46 R600Subtarget::initializeSubtargetDependencies(const Triple &TT,
47                                                StringRef GPU, StringRef FS) {
48   SmallString<256> FullFS("+promote-alloca,");
49   FullFS += FS;
50   ParseSubtargetFeatures(GPU, FullFS);
51 
52   // FIXME: I don't think think Evergreen has any useful support for
53   // denormals, but should be checked. Should we issue a warning somewhere
54   // if someone tries to enable these?
55   if (getGeneration() <= AMDGPUSubtarget::NORTHERN_ISLANDS) {
56     FP32Denormals = false;
57   }
58 
59   HasMulU24 = getGeneration() >= EVERGREEN;
60   HasMulI24 = hasCaymanISA();
61 
62   return *this;
63 }
64 
65 GCNSubtarget &
66 GCNSubtarget::initializeSubtargetDependencies(const Triple &TT,
67                                               StringRef GPU, StringRef FS) {
68   // Determine default and user-specified characteristics
69   // On SI+, we want FP64 denormals to be on by default. FP32 denormals can be
70   // enabled, but some instructions do not respect them and they run at the
71   // double precision rate, so don't enable by default.
72   //
73   // We want to be able to turn these off, but making this a subtarget feature
74   // for SI has the unhelpful behavior that it unsets everything else if you
75   // disable it.
76   //
77   // Similarly we want enable-prt-strict-null to be on by default and not to
78   // unset everything else if it is disabled
79 
80   // Assuming ECC is enabled is the conservative default.
81   SmallString<256> FullFS("+promote-alloca,+load-store-opt,+sram-ecc,");
82 
83   if (isAmdHsaOS()) // Turn on FlatForGlobal for HSA.
84     FullFS += "+flat-for-global,+unaligned-buffer-access,+trap-handler,";
85 
86   // FIXME: I don't think think Evergreen has any useful support for
87   // denormals, but should be checked. Should we issue a warning somewhere
88   // if someone tries to enable these?
89   if (getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS) {
90     FullFS += "+fp64-fp16-denormals,";
91   } else {
92     FullFS += "-fp32-denormals,";
93   }
94 
95   FullFS += "+enable-prt-strict-null,"; // This is overridden by a disable in FS
96 
97   FullFS += FS;
98 
99   ParseSubtargetFeatures(GPU, FullFS);
100 
101   // We don't support FP64 for EG/NI atm.
102   assert(!hasFP64() || (getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS));
103 
104   // Unless +-flat-for-global is specified, turn on FlatForGlobal for all OS-es
105   // on VI and newer hardware to avoid assertion failures due to missing ADDR64
106   // variants of MUBUF instructions.
107   if (!hasAddr64() && !FS.contains("flat-for-global")) {
108     FlatForGlobal = true;
109   }
110 
111   // Set defaults if needed.
112   if (MaxPrivateElementSize == 0)
113     MaxPrivateElementSize = 4;
114 
115   if (LDSBankCount == 0)
116     LDSBankCount = 32;
117 
118   if (TT.getArch() == Triple::amdgcn) {
119     if (LocalMemorySize == 0)
120       LocalMemorySize = 32768;
121 
122     // Do something sensible for unspecified target.
123     if (!HasMovrel && !HasVGPRIndexMode)
124       HasMovrel = true;
125   }
126 
127   // Don't crash on invalid devices.
128   if (WavefrontSize == 0)
129     WavefrontSize = 64;
130 
131   HasFminFmaxLegacy = getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS;
132 
133   // ECC is on by default, but turn it off if the hardware doesn't support it
134   // anyway. This matters for the gfx9 targets with d16 loads, but don't support
135   // ECC.
136   if (DoesNotSupportSRAMECC && EnableSRAMECC) {
137     ToggleFeature(AMDGPU::FeatureSRAMECC);
138     EnableSRAMECC = false;
139   }
140 
141   return *this;
142 }
143 
144 AMDGPUSubtarget::AMDGPUSubtarget(const Triple &TT) :
145   TargetTriple(TT),
146   Has16BitInsts(false),
147   HasMadMixInsts(false),
148   FP32Denormals(false),
149   FPExceptions(false),
150   HasSDWA(false),
151   HasVOP3PInsts(false),
152   HasMulI24(true),
153   HasMulU24(true),
154   HasInv2PiInlineImm(false),
155   HasFminFmaxLegacy(true),
156   EnablePromoteAlloca(false),
157   HasTrigReducedRange(false),
158   LocalMemorySize(0),
159   WavefrontSize(0)
160   { }
161 
162 GCNSubtarget::GCNSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
163                            const GCNTargetMachine &TM) :
164     AMDGPUGenSubtargetInfo(TT, GPU, FS),
165     AMDGPUSubtarget(TT),
166     TargetTriple(TT),
167     Gen(TT.getOS() == Triple::AMDHSA ? SEA_ISLANDS : SOUTHERN_ISLANDS),
168     InstrItins(getInstrItineraryForCPU(GPU)),
169     LDSBankCount(0),
170     MaxPrivateElementSize(0),
171 
172     FastFMAF32(false),
173     HalfRate64Ops(false),
174 
175     FP64FP16Denormals(false),
176     FlatForGlobal(false),
177     AutoWaitcntBeforeBarrier(false),
178     CodeObjectV3(false),
179     UnalignedScratchAccess(false),
180     UnalignedBufferAccess(false),
181 
182     HasApertureRegs(false),
183     EnableXNACK(false),
184     TrapHandler(false),
185 
186     EnableHugePrivateBuffer(false),
187     EnableLoadStoreOpt(false),
188     EnableUnsafeDSOffsetFolding(false),
189     EnableSIScheduler(false),
190     EnableDS128(false),
191     EnablePRTStrictNull(false),
192     DumpCode(false),
193 
194     FP64(false),
195     GCN3Encoding(false),
196     CIInsts(false),
197     VIInsts(false),
198     GFX9Insts(false),
199     SGPRInitBug(false),
200     HasSMemRealTime(false),
201     HasIntClamp(false),
202     HasFmaMixInsts(false),
203     HasMovrel(false),
204     HasVGPRIndexMode(false),
205     HasScalarStores(false),
206     HasScalarAtomics(false),
207     HasSDWAOmod(false),
208     HasSDWAScalar(false),
209     HasSDWASdst(false),
210     HasSDWAMac(false),
211     HasSDWAOutModsVOPC(false),
212     HasDPP(false),
213     HasR128A16(false),
214     HasDLInsts(false),
215     HasDot1Insts(false),
216     HasDot2Insts(false),
217     EnableSRAMECC(false),
218     DoesNotSupportSRAMECC(false),
219     FlatAddressSpace(false),
220     FlatInstOffsets(false),
221     FlatGlobalInsts(false),
222     FlatScratchInsts(false),
223     AddNoCarryInsts(false),
224     HasUnpackedD16VMem(false),
225 
226     ScalarizeGlobal(false),
227 
228     FeatureDisable(false),
229     InstrInfo(initializeSubtargetDependencies(TT, GPU, FS)),
230     TLInfo(TM, *this),
231     FrameLowering(TargetFrameLowering::StackGrowsUp, getStackAlignment(), 0) {
232   CallLoweringInfo.reset(new AMDGPUCallLowering(*getTargetLowering()));
233   Legalizer.reset(new AMDGPULegalizerInfo(*this, TM));
234   RegBankInfo.reset(new AMDGPURegisterBankInfo(*getRegisterInfo()));
235   InstSelector.reset(new AMDGPUInstructionSelector(
236   *this, *static_cast<AMDGPURegisterBankInfo *>(RegBankInfo.get()), TM));
237 }
238 
239 unsigned AMDGPUSubtarget::getMaxLocalMemSizeWithWaveCount(unsigned NWaves,
240   const Function &F) const {
241   if (NWaves == 1)
242     return getLocalMemorySize();
243   unsigned WorkGroupSize = getFlatWorkGroupSizes(F).second;
244   unsigned WorkGroupsPerCu = getMaxWorkGroupsPerCU(WorkGroupSize);
245   unsigned MaxWaves = getMaxWavesPerEU();
246   return getLocalMemorySize() * MaxWaves / WorkGroupsPerCu / NWaves;
247 }
248 
249 unsigned AMDGPUSubtarget::getOccupancyWithLocalMemSize(uint32_t Bytes,
250   const Function &F) const {
251   unsigned WorkGroupSize = getFlatWorkGroupSizes(F).second;
252   unsigned WorkGroupsPerCu = getMaxWorkGroupsPerCU(WorkGroupSize);
253   unsigned MaxWaves = getMaxWavesPerEU();
254   unsigned Limit = getLocalMemorySize() * MaxWaves / WorkGroupsPerCu;
255   unsigned NumWaves = Limit / (Bytes ? Bytes : 1u);
256   NumWaves = std::min(NumWaves, MaxWaves);
257   NumWaves = std::max(NumWaves, 1u);
258   return NumWaves;
259 }
260 
261 unsigned
262 AMDGPUSubtarget::getOccupancyWithLocalMemSize(const MachineFunction &MF) const {
263   const auto *MFI = MF.getInfo<SIMachineFunctionInfo>();
264   return getOccupancyWithLocalMemSize(MFI->getLDSSize(), MF.getFunction());
265 }
266 
267 std::pair<unsigned, unsigned>
268 AMDGPUSubtarget::getDefaultFlatWorkGroupSize(CallingConv::ID CC) const {
269   switch (CC) {
270   case CallingConv::AMDGPU_CS:
271   case CallingConv::AMDGPU_KERNEL:
272   case CallingConv::SPIR_KERNEL:
273     return std::make_pair(getWavefrontSize() * 2, getWavefrontSize() * 4);
274   case CallingConv::AMDGPU_VS:
275   case CallingConv::AMDGPU_LS:
276   case CallingConv::AMDGPU_HS:
277   case CallingConv::AMDGPU_ES:
278   case CallingConv::AMDGPU_GS:
279   case CallingConv::AMDGPU_PS:
280     return std::make_pair(1, getWavefrontSize());
281   default:
282     return std::make_pair(1, 16 * getWavefrontSize());
283   }
284 }
285 
286 std::pair<unsigned, unsigned> AMDGPUSubtarget::getFlatWorkGroupSizes(
287   const Function &F) const {
288   // FIXME: 1024 if function.
289   // Default minimum/maximum flat work group sizes.
290   std::pair<unsigned, unsigned> Default =
291     getDefaultFlatWorkGroupSize(F.getCallingConv());
292 
293   // TODO: Do not process "amdgpu-max-work-group-size" attribute once mesa
294   // starts using "amdgpu-flat-work-group-size" attribute.
295   Default.second = AMDGPU::getIntegerAttribute(
296     F, "amdgpu-max-work-group-size", Default.second);
297   Default.first = std::min(Default.first, Default.second);
298 
299   // Requested minimum/maximum flat work group sizes.
300   std::pair<unsigned, unsigned> Requested = AMDGPU::getIntegerPairAttribute(
301     F, "amdgpu-flat-work-group-size", Default);
302 
303   // Make sure requested minimum is less than requested maximum.
304   if (Requested.first > Requested.second)
305     return Default;
306 
307   // Make sure requested values do not violate subtarget's specifications.
308   if (Requested.first < getMinFlatWorkGroupSize())
309     return Default;
310   if (Requested.second > getMaxFlatWorkGroupSize())
311     return Default;
312 
313   return Requested;
314 }
315 
316 std::pair<unsigned, unsigned> AMDGPUSubtarget::getWavesPerEU(
317   const Function &F) const {
318   // Default minimum/maximum number of waves per execution unit.
319   std::pair<unsigned, unsigned> Default(1, getMaxWavesPerEU());
320 
321   // Default/requested minimum/maximum flat work group sizes.
322   std::pair<unsigned, unsigned> FlatWorkGroupSizes = getFlatWorkGroupSizes(F);
323 
324   // If minimum/maximum flat work group sizes were explicitly requested using
325   // "amdgpu-flat-work-group-size" attribute, then set default minimum/maximum
326   // number of waves per execution unit to values implied by requested
327   // minimum/maximum flat work group sizes.
328   unsigned MinImpliedByFlatWorkGroupSize =
329     getMaxWavesPerEU(FlatWorkGroupSizes.second);
330   bool RequestedFlatWorkGroupSize = false;
331 
332   // TODO: Do not process "amdgpu-max-work-group-size" attribute once mesa
333   // starts using "amdgpu-flat-work-group-size" attribute.
334   if (F.hasFnAttribute("amdgpu-max-work-group-size") ||
335       F.hasFnAttribute("amdgpu-flat-work-group-size")) {
336     Default.first = MinImpliedByFlatWorkGroupSize;
337     RequestedFlatWorkGroupSize = true;
338   }
339 
340   // Requested minimum/maximum number of waves per execution unit.
341   std::pair<unsigned, unsigned> Requested = AMDGPU::getIntegerPairAttribute(
342     F, "amdgpu-waves-per-eu", Default, true);
343 
344   // Make sure requested minimum is less than requested maximum.
345   if (Requested.second && Requested.first > Requested.second)
346     return Default;
347 
348   // Make sure requested values do not violate subtarget's specifications.
349   if (Requested.first < getMinWavesPerEU() ||
350       Requested.first > getMaxWavesPerEU())
351     return Default;
352   if (Requested.second > getMaxWavesPerEU())
353     return Default;
354 
355   // Make sure requested values are compatible with values implied by requested
356   // minimum/maximum flat work group sizes.
357   if (RequestedFlatWorkGroupSize &&
358       Requested.first < MinImpliedByFlatWorkGroupSize)
359     return Default;
360 
361   return Requested;
362 }
363 
364 bool AMDGPUSubtarget::makeLIDRangeMetadata(Instruction *I) const {
365   Function *Kernel = I->getParent()->getParent();
366   unsigned MinSize = 0;
367   unsigned MaxSize = getFlatWorkGroupSizes(*Kernel).second;
368   bool IdQuery = false;
369 
370   // If reqd_work_group_size is present it narrows value down.
371   if (auto *CI = dyn_cast<CallInst>(I)) {
372     const Function *F = CI->getCalledFunction();
373     if (F) {
374       unsigned Dim = UINT_MAX;
375       switch (F->getIntrinsicID()) {
376       case Intrinsic::amdgcn_workitem_id_x:
377       case Intrinsic::r600_read_tidig_x:
378         IdQuery = true;
379         LLVM_FALLTHROUGH;
380       case Intrinsic::r600_read_local_size_x:
381         Dim = 0;
382         break;
383       case Intrinsic::amdgcn_workitem_id_y:
384       case Intrinsic::r600_read_tidig_y:
385         IdQuery = true;
386         LLVM_FALLTHROUGH;
387       case Intrinsic::r600_read_local_size_y:
388         Dim = 1;
389         break;
390       case Intrinsic::amdgcn_workitem_id_z:
391       case Intrinsic::r600_read_tidig_z:
392         IdQuery = true;
393         LLVM_FALLTHROUGH;
394       case Intrinsic::r600_read_local_size_z:
395         Dim = 2;
396         break;
397       default:
398         break;
399       }
400       if (Dim <= 3) {
401         if (auto Node = Kernel->getMetadata("reqd_work_group_size"))
402           if (Node->getNumOperands() == 3)
403             MinSize = MaxSize = mdconst::extract<ConstantInt>(
404                                   Node->getOperand(Dim))->getZExtValue();
405       }
406     }
407   }
408 
409   if (!MaxSize)
410     return false;
411 
412   // Range metadata is [Lo, Hi). For ID query we need to pass max size
413   // as Hi. For size query we need to pass Hi + 1.
414   if (IdQuery)
415     MinSize = 0;
416   else
417     ++MaxSize;
418 
419   MDBuilder MDB(I->getContext());
420   MDNode *MaxWorkGroupSizeRange = MDB.createRange(APInt(32, MinSize),
421                                                   APInt(32, MaxSize));
422   I->setMetadata(LLVMContext::MD_range, MaxWorkGroupSizeRange);
423   return true;
424 }
425 
426 uint64_t AMDGPUSubtarget::getExplicitKernArgSize(const Function &F,
427                                                  unsigned &MaxAlign) const {
428   assert(F.getCallingConv() == CallingConv::AMDGPU_KERNEL ||
429          F.getCallingConv() == CallingConv::SPIR_KERNEL);
430 
431   const DataLayout &DL = F.getParent()->getDataLayout();
432   uint64_t ExplicitArgBytes = 0;
433   MaxAlign = 1;
434 
435   for (const Argument &Arg : F.args()) {
436     Type *ArgTy = Arg.getType();
437 
438     unsigned Align = DL.getABITypeAlignment(ArgTy);
439     uint64_t AllocSize = DL.getTypeAllocSize(ArgTy);
440     ExplicitArgBytes = alignTo(ExplicitArgBytes, Align) + AllocSize;
441     MaxAlign = std::max(MaxAlign, Align);
442   }
443 
444   return ExplicitArgBytes;
445 }
446 
447 unsigned AMDGPUSubtarget::getKernArgSegmentSize(const Function &F,
448                                                 unsigned &MaxAlign) const {
449   uint64_t ExplicitArgBytes = getExplicitKernArgSize(F, MaxAlign);
450 
451   unsigned ExplicitOffset = getExplicitKernelArgOffset(F);
452 
453   uint64_t TotalSize = ExplicitOffset + ExplicitArgBytes;
454   unsigned ImplicitBytes = getImplicitArgNumBytes(F);
455   if (ImplicitBytes != 0) {
456     unsigned Alignment = getAlignmentForImplicitArgPtr();
457     TotalSize = alignTo(ExplicitArgBytes, Alignment) + ImplicitBytes;
458   }
459 
460   // Being able to dereference past the end is useful for emitting scalar loads.
461   return alignTo(TotalSize, 4);
462 }
463 
464 R600Subtarget::R600Subtarget(const Triple &TT, StringRef GPU, StringRef FS,
465                              const TargetMachine &TM) :
466   R600GenSubtargetInfo(TT, GPU, FS),
467   AMDGPUSubtarget(TT),
468   InstrInfo(*this),
469   FrameLowering(TargetFrameLowering::StackGrowsUp, getStackAlignment(), 0),
470   FMA(false),
471   CaymanISA(false),
472   CFALUBug(false),
473   HasVertexCache(false),
474   R600ALUInst(false),
475   FP64(false),
476   TexVTXClauseSize(0),
477   Gen(R600),
478   TLInfo(TM, initializeSubtargetDependencies(TT, GPU, FS)),
479   InstrItins(getInstrItineraryForCPU(GPU)) { }
480 
481 void GCNSubtarget::overrideSchedPolicy(MachineSchedPolicy &Policy,
482                                       unsigned NumRegionInstrs) const {
483   // Track register pressure so the scheduler can try to decrease
484   // pressure once register usage is above the threshold defined by
485   // SIRegisterInfo::getRegPressureSetLimit()
486   Policy.ShouldTrackPressure = true;
487 
488   // Enabling both top down and bottom up scheduling seems to give us less
489   // register spills than just using one of these approaches on its own.
490   Policy.OnlyTopDown = false;
491   Policy.OnlyBottomUp = false;
492 
493   // Enabling ShouldTrackLaneMasks crashes the SI Machine Scheduler.
494   if (!enableSIScheduler())
495     Policy.ShouldTrackLaneMasks = true;
496 }
497 
498 unsigned GCNSubtarget::getOccupancyWithNumSGPRs(unsigned SGPRs) const {
499   if (getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
500     if (SGPRs <= 80)
501       return 10;
502     if (SGPRs <= 88)
503       return 9;
504     if (SGPRs <= 100)
505       return 8;
506     return 7;
507   }
508   if (SGPRs <= 48)
509     return 10;
510   if (SGPRs <= 56)
511     return 9;
512   if (SGPRs <= 64)
513     return 8;
514   if (SGPRs <= 72)
515     return 7;
516   if (SGPRs <= 80)
517     return 6;
518   return 5;
519 }
520 
521 unsigned GCNSubtarget::getOccupancyWithNumVGPRs(unsigned VGPRs) const {
522   if (VGPRs <= 24)
523     return 10;
524   if (VGPRs <= 28)
525     return 9;
526   if (VGPRs <= 32)
527     return 8;
528   if (VGPRs <= 36)
529     return 7;
530   if (VGPRs <= 40)
531     return 6;
532   if (VGPRs <= 48)
533     return 5;
534   if (VGPRs <= 64)
535     return 4;
536   if (VGPRs <= 84)
537     return 3;
538   if (VGPRs <= 128)
539     return 2;
540   return 1;
541 }
542 
543 unsigned GCNSubtarget::getReservedNumSGPRs(const MachineFunction &MF) const {
544   const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
545   if (MFI.hasFlatScratchInit()) {
546     if (getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
547       return 6; // FLAT_SCRATCH, XNACK, VCC (in that order).
548     if (getGeneration() == AMDGPUSubtarget::SEA_ISLANDS)
549       return 4; // FLAT_SCRATCH, VCC (in that order).
550   }
551 
552   if (isXNACKEnabled())
553     return 4; // XNACK, VCC (in that order).
554   return 2; // VCC.
555 }
556 
557 unsigned GCNSubtarget::getMaxNumSGPRs(const MachineFunction &MF) const {
558   const Function &F = MF.getFunction();
559   const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
560 
561   // Compute maximum number of SGPRs function can use using default/requested
562   // minimum number of waves per execution unit.
563   std::pair<unsigned, unsigned> WavesPerEU = MFI.getWavesPerEU();
564   unsigned MaxNumSGPRs = getMaxNumSGPRs(WavesPerEU.first, false);
565   unsigned MaxAddressableNumSGPRs = getMaxNumSGPRs(WavesPerEU.first, true);
566 
567   // Check if maximum number of SGPRs was explicitly requested using
568   // "amdgpu-num-sgpr" attribute.
569   if (F.hasFnAttribute("amdgpu-num-sgpr")) {
570     unsigned Requested = AMDGPU::getIntegerAttribute(
571       F, "amdgpu-num-sgpr", MaxNumSGPRs);
572 
573     // Make sure requested value does not violate subtarget's specifications.
574     if (Requested && (Requested <= getReservedNumSGPRs(MF)))
575       Requested = 0;
576 
577     // If more SGPRs are required to support the input user/system SGPRs,
578     // increase to accommodate them.
579     //
580     // FIXME: This really ends up using the requested number of SGPRs + number
581     // of reserved special registers in total. Theoretically you could re-use
582     // the last input registers for these special registers, but this would
583     // require a lot of complexity to deal with the weird aliasing.
584     unsigned InputNumSGPRs = MFI.getNumPreloadedSGPRs();
585     if (Requested && Requested < InputNumSGPRs)
586       Requested = InputNumSGPRs;
587 
588     // Make sure requested value is compatible with values implied by
589     // default/requested minimum/maximum number of waves per execution unit.
590     if (Requested && Requested > getMaxNumSGPRs(WavesPerEU.first, false))
591       Requested = 0;
592     if (WavesPerEU.second &&
593         Requested && Requested < getMinNumSGPRs(WavesPerEU.second))
594       Requested = 0;
595 
596     if (Requested)
597       MaxNumSGPRs = Requested;
598   }
599 
600   if (hasSGPRInitBug())
601     MaxNumSGPRs = AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG;
602 
603   return std::min(MaxNumSGPRs - getReservedNumSGPRs(MF),
604                   MaxAddressableNumSGPRs);
605 }
606 
607 unsigned GCNSubtarget::getMaxNumVGPRs(const MachineFunction &MF) const {
608   const Function &F = MF.getFunction();
609   const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
610 
611   // Compute maximum number of VGPRs function can use using default/requested
612   // minimum number of waves per execution unit.
613   std::pair<unsigned, unsigned> WavesPerEU = MFI.getWavesPerEU();
614   unsigned MaxNumVGPRs = getMaxNumVGPRs(WavesPerEU.first);
615 
616   // Check if maximum number of VGPRs was explicitly requested using
617   // "amdgpu-num-vgpr" attribute.
618   if (F.hasFnAttribute("amdgpu-num-vgpr")) {
619     unsigned Requested = AMDGPU::getIntegerAttribute(
620       F, "amdgpu-num-vgpr", MaxNumVGPRs);
621 
622     // Make sure requested value is compatible with values implied by
623     // default/requested minimum/maximum number of waves per execution unit.
624     if (Requested && Requested > getMaxNumVGPRs(WavesPerEU.first))
625       Requested = 0;
626     if (WavesPerEU.second &&
627         Requested && Requested < getMinNumVGPRs(WavesPerEU.second))
628       Requested = 0;
629 
630     if (Requested)
631       MaxNumVGPRs = Requested;
632   }
633 
634   return MaxNumVGPRs;
635 }
636 
637 namespace {
638 struct MemOpClusterMutation : ScheduleDAGMutation {
639   const SIInstrInfo *TII;
640 
641   MemOpClusterMutation(const SIInstrInfo *tii) : TII(tii) {}
642 
643   void apply(ScheduleDAGInstrs *DAG) override {
644     SUnit *SUa = nullptr;
645     // Search for two consequent memory operations and link them
646     // to prevent scheduler from moving them apart.
647     // In DAG pre-process SUnits are in the original order of
648     // the instructions before scheduling.
649     for (SUnit &SU : DAG->SUnits) {
650       MachineInstr &MI2 = *SU.getInstr();
651       if (!MI2.mayLoad() && !MI2.mayStore()) {
652         SUa = nullptr;
653         continue;
654       }
655       if (!SUa) {
656         SUa = &SU;
657         continue;
658       }
659 
660       MachineInstr &MI1 = *SUa->getInstr();
661       if ((TII->isVMEM(MI1) && TII->isVMEM(MI2)) ||
662           (TII->isFLAT(MI1) && TII->isFLAT(MI2)) ||
663           (TII->isSMRD(MI1) && TII->isSMRD(MI2)) ||
664           (TII->isDS(MI1)   && TII->isDS(MI2))) {
665         SU.addPredBarrier(SUa);
666 
667         for (const SDep &SI : SU.Preds) {
668           if (SI.getSUnit() != SUa)
669             SUa->addPred(SDep(SI.getSUnit(), SDep::Artificial));
670         }
671 
672         if (&SU != &DAG->ExitSU) {
673           for (const SDep &SI : SUa->Succs) {
674             if (SI.getSUnit() != &SU)
675               SI.getSUnit()->addPred(SDep(&SU, SDep::Artificial));
676           }
677         }
678       }
679 
680       SUa = &SU;
681     }
682   }
683 };
684 } // namespace
685 
686 void GCNSubtarget::getPostRAMutations(
687     std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations) const {
688   Mutations.push_back(llvm::make_unique<MemOpClusterMutation>(&InstrInfo));
689 }
690 
691 const AMDGPUSubtarget &AMDGPUSubtarget::get(const MachineFunction &MF) {
692   if (MF.getTarget().getTargetTriple().getArch() == Triple::amdgcn)
693     return static_cast<const AMDGPUSubtarget&>(MF.getSubtarget<GCNSubtarget>());
694   else
695     return static_cast<const AMDGPUSubtarget&>(MF.getSubtarget<R600Subtarget>());
696 }
697 
698 const AMDGPUSubtarget &AMDGPUSubtarget::get(const TargetMachine &TM, const Function &F) {
699   if (TM.getTargetTriple().getArch() == Triple::amdgcn)
700     return static_cast<const AMDGPUSubtarget&>(TM.getSubtarget<GCNSubtarget>(F));
701   else
702     return static_cast<const AMDGPUSubtarget&>(TM.getSubtarget<R600Subtarget>(F));
703 }
704