1 //===-- AMDGPUSubtarget.cpp - AMDGPU Subtarget Information ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Implements the AMDGPU specific subclass of TargetSubtarget.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "AMDGPUSubtarget.h"
15 #include "AMDGPU.h"
16 #include "AMDGPUCallLowering.h"
17 #include "AMDGPUInstructionSelector.h"
18 #include "AMDGPULegalizerInfo.h"
19 #include "AMDGPURegisterBankInfo.h"
20 #include "AMDGPUTargetMachine.h"
21 #include "SIMachineFunctionInfo.h"
22 #include "Utils/AMDGPUBaseInfo.h"
23 #include "llvm/ADT/SmallString.h"
24 #include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
25 #include "llvm/CodeGen/MachineScheduler.h"
26 #include "llvm/CodeGen/TargetFrameLowering.h"
27 #include "llvm/IR/IntrinsicsAMDGPU.h"
28 #include "llvm/IR/IntrinsicsR600.h"
29 #include "llvm/IR/MDBuilder.h"
30 #include "llvm/MC/MCSubtargetInfo.h"
31 #include <algorithm>
32 
33 using namespace llvm;
34 
35 #define DEBUG_TYPE "amdgpu-subtarget"
36 
37 #define GET_SUBTARGETINFO_TARGET_DESC
38 #define GET_SUBTARGETINFO_CTOR
39 #define AMDGPUSubtarget GCNSubtarget
40 #include "AMDGPUGenSubtargetInfo.inc"
41 #define GET_SUBTARGETINFO_TARGET_DESC
42 #define GET_SUBTARGETINFO_CTOR
43 #undef AMDGPUSubtarget
44 #include "R600GenSubtargetInfo.inc"
45 
46 static cl::opt<bool> DisablePowerSched(
47   "amdgpu-disable-power-sched",
48   cl::desc("Disable scheduling to minimize mAI power bursts"),
49   cl::init(false));
50 
51 static cl::opt<bool> EnableVGPRIndexMode(
52   "amdgpu-vgpr-index-mode",
53   cl::desc("Use GPR indexing mode instead of movrel for vector indexing"),
54   cl::init(false));
55 
56 static cl::opt<bool> EnableFlatScratch(
57   "amdgpu-enable-flat-scratch",
58   cl::desc("Use flat scratch instructions"),
59   cl::init(false));
60 
61 static cl::opt<bool> UseAA("amdgpu-use-aa-in-codegen",
62                            cl::desc("Enable the use of AA during codegen."),
63                            cl::init(true));
64 
65 GCNSubtarget::~GCNSubtarget() = default;
66 
67 R600Subtarget &
68 R600Subtarget::initializeSubtargetDependencies(const Triple &TT,
69                                                StringRef GPU, StringRef FS) {
70   SmallString<256> FullFS("+promote-alloca,");
71   FullFS += FS;
72   ParseSubtargetFeatures(GPU, /*TuneCPU*/ GPU, FullFS);
73 
74   HasMulU24 = getGeneration() >= EVERGREEN;
75   HasMulI24 = hasCaymanISA();
76 
77   return *this;
78 }
79 
80 GCNSubtarget &
81 GCNSubtarget::initializeSubtargetDependencies(const Triple &TT,
82                                               StringRef GPU, StringRef FS) {
83   // Determine default and user-specified characteristics
84   //
85   // We want to be able to turn these off, but making this a subtarget feature
86   // for SI has the unhelpful behavior that it unsets everything else if you
87   // disable it.
88   //
89   // Similarly we want enable-prt-strict-null to be on by default and not to
90   // unset everything else if it is disabled
91 
92   SmallString<256> FullFS("+promote-alloca,+load-store-opt,+enable-ds128,");
93 
94   // Turn on features that HSA ABI requires. Also turn on FlatForGlobal by default
95   if (isAmdHsaOS())
96     FullFS += "+flat-for-global,+unaligned-access-mode,+trap-handler,";
97 
98   FullFS += "+enable-prt-strict-null,"; // This is overridden by a disable in FS
99 
100   // Disable mutually exclusive bits.
101   if (FS.find_lower("+wavefrontsize") != StringRef::npos) {
102     if (FS.find_lower("wavefrontsize16") == StringRef::npos)
103       FullFS += "-wavefrontsize16,";
104     if (FS.find_lower("wavefrontsize32") == StringRef::npos)
105       FullFS += "-wavefrontsize32,";
106     if (FS.find_lower("wavefrontsize64") == StringRef::npos)
107       FullFS += "-wavefrontsize64,";
108   }
109 
110   FullFS += FS;
111 
112   ParseSubtargetFeatures(GPU, /*TuneCPU*/ GPU, FullFS);
113 
114   // Implement the "generic" processors, which acts as the default when no
115   // generation features are enabled (e.g for -mcpu=''). HSA OS defaults to
116   // the first amdgcn target that supports flat addressing. Other OSes defaults
117   // to the first amdgcn target.
118   if (Gen == AMDGPUSubtarget::INVALID) {
119      Gen = TT.getOS() == Triple::AMDHSA ? AMDGPUSubtarget::SEA_ISLANDS
120                                         : AMDGPUSubtarget::SOUTHERN_ISLANDS;
121   }
122 
123   // We don't support FP64 for EG/NI atm.
124   assert(!hasFP64() || (getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS));
125 
126   // Targets must either support 64-bit offsets for MUBUF instructions, and/or
127   // support flat operations, otherwise they cannot access a 64-bit global
128   // address space
129   assert(hasAddr64() || hasFlat());
130   // Unless +-flat-for-global is specified, turn on FlatForGlobal for targets
131   // that do not support ADDR64 variants of MUBUF instructions. Such targets
132   // cannot use a 64 bit offset with a MUBUF instruction to access the global
133   // address space
134   if (!hasAddr64() && !FS.contains("flat-for-global") && !FlatForGlobal) {
135     ToggleFeature(AMDGPU::FeatureFlatForGlobal);
136     FlatForGlobal = true;
137   }
138   // Unless +-flat-for-global is specified, use MUBUF instructions for global
139   // address space access if flat operations are not available.
140   if (!hasFlat() && !FS.contains("flat-for-global") && FlatForGlobal) {
141     ToggleFeature(AMDGPU::FeatureFlatForGlobal);
142     FlatForGlobal = false;
143   }
144 
145   // Set defaults if needed.
146   if (MaxPrivateElementSize == 0)
147     MaxPrivateElementSize = 4;
148 
149   if (LDSBankCount == 0)
150     LDSBankCount = 32;
151 
152   if (TT.getArch() == Triple::amdgcn) {
153     if (LocalMemorySize == 0)
154       LocalMemorySize = 32768;
155 
156     // Do something sensible for unspecified target.
157     if (!HasMovrel && !HasVGPRIndexMode)
158       HasMovrel = true;
159   }
160 
161   // Don't crash on invalid devices.
162   if (WavefrontSizeLog2 == 0)
163     WavefrontSizeLog2 = 5;
164 
165   HasFminFmaxLegacy = getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS;
166 
167   TargetID.setTargetIDFromFeaturesString(FS);
168 
169   LLVM_DEBUG(dbgs() << "xnack setting for subtarget: "
170                     << TargetID.getXnackSetting() << '\n');
171   LLVM_DEBUG(dbgs() << "sramecc setting for subtarget: "
172                     << TargetID.getSramEccSetting() << '\n');
173 
174   return *this;
175 }
176 
177 AMDGPUSubtarget::AMDGPUSubtarget(const Triple &TT) :
178   TargetTriple(TT),
179   GCN3Encoding(false),
180   Has16BitInsts(false),
181   HasMadMixInsts(false),
182   HasMadMacF32Insts(false),
183   HasDsSrc2Insts(false),
184   HasSDWA(false),
185   HasVOP3PInsts(false),
186   HasMulI24(true),
187   HasMulU24(true),
188   HasInv2PiInlineImm(false),
189   HasFminFmaxLegacy(true),
190   EnablePromoteAlloca(false),
191   HasTrigReducedRange(false),
192   MaxWavesPerEU(10),
193   LocalMemorySize(0),
194   WavefrontSizeLog2(0)
195   { }
196 
197 GCNSubtarget::GCNSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
198                            const GCNTargetMachine &TM)
199     : // clang-format off
200     AMDGPUGenSubtargetInfo(TT, GPU, /*TuneCPU*/ GPU, FS),
201     AMDGPUSubtarget(TT),
202     TargetTriple(TT),
203     TargetID(*this),
204     Gen(INVALID),
205     InstrItins(getInstrItineraryForCPU(GPU)),
206     LDSBankCount(0),
207     MaxPrivateElementSize(0),
208 
209     FastFMAF32(false),
210     FastDenormalF32(false),
211     HalfRate64Ops(false),
212     FullRate64Ops(false),
213 
214     FlatForGlobal(false),
215     AutoWaitcntBeforeBarrier(false),
216     UnalignedScratchAccess(false),
217     UnalignedAccessMode(false),
218 
219     HasApertureRegs(false),
220     SupportsXNACK(false),
221     EnableXNACK(false),
222     EnableTgSplit(false),
223     EnableCuMode(false),
224     TrapHandler(false),
225 
226     EnableLoadStoreOpt(false),
227     EnableUnsafeDSOffsetFolding(false),
228     EnableSIScheduler(false),
229     EnableDS128(false),
230     EnablePRTStrictNull(false),
231     DumpCode(false),
232 
233     FP64(false),
234     CIInsts(false),
235     GFX8Insts(false),
236     GFX9Insts(false),
237     GFX90AInsts(false),
238     GFX10Insts(false),
239     GFX10_3Insts(false),
240     GFX7GFX8GFX9Insts(false),
241     SGPRInitBug(false),
242     NegativeScratchOffsetBug(false),
243     NegativeUnalignedScratchOffsetBug(false),
244     HasSMemRealTime(false),
245     HasIntClamp(false),
246     HasFmaMixInsts(false),
247     HasMovrel(false),
248     HasVGPRIndexMode(false),
249     HasScalarStores(false),
250     HasScalarAtomics(false),
251     HasSDWAOmod(false),
252     HasSDWAScalar(false),
253     HasSDWASdst(false),
254     HasSDWAMac(false),
255     HasSDWAOutModsVOPC(false),
256     HasDPP(false),
257     HasDPP8(false),
258     Has64BitDPP(false),
259     HasPackedFP32Ops(false),
260     HasExtendedImageInsts(false),
261     HasR128A16(false),
262     HasGFX10A16(false),
263     HasG16(false),
264     HasNSAEncoding(false),
265     GFX10_BEncoding(false),
266     HasDLInsts(false),
267     HasDot1Insts(false),
268     HasDot2Insts(false),
269     HasDot3Insts(false),
270     HasDot4Insts(false),
271     HasDot5Insts(false),
272     HasDot6Insts(false),
273     HasDot7Insts(false),
274     HasMAIInsts(false),
275     HasPkFmacF16Inst(false),
276     HasAtomicFaddInsts(false),
277     SupportsSRAMECC(false),
278     EnableSRAMECC(false),
279     HasNoSdstCMPX(false),
280     HasVscnt(false),
281     HasGetWaveIdInst(false),
282     HasSMemTimeInst(false),
283     HasShaderCyclesRegister(false),
284     HasRegisterBanking(false),
285     HasVOP3Literal(false),
286     HasNoDataDepHazard(false),
287     FlatAddressSpace(false),
288     FlatInstOffsets(false),
289     FlatGlobalInsts(false),
290     FlatScratchInsts(false),
291     ScalarFlatScratchInsts(false),
292     AddNoCarryInsts(false),
293     HasUnpackedD16VMem(false),
294     LDSMisalignedBug(false),
295     HasMFMAInlineLiteralBug(false),
296     UnalignedBufferAccess(false),
297     UnalignedDSAccess(false),
298     HasPackedTID(false),
299 
300     ScalarizeGlobal(false),
301 
302     HasVcmpxPermlaneHazard(false),
303     HasVMEMtoScalarWriteHazard(false),
304     HasSMEMtoVectorWriteHazard(false),
305     HasInstFwdPrefetchBug(false),
306     HasVcmpxExecWARHazard(false),
307     HasLdsBranchVmemWARHazard(false),
308     HasNSAtoVMEMBug(false),
309     HasOffset3fBug(false),
310     HasFlatSegmentOffsetBug(false),
311     HasImageStoreD16Bug(false),
312     HasImageGather4D16Bug(false),
313 
314     FeatureDisable(false),
315     InstrInfo(initializeSubtargetDependencies(TT, GPU, FS)),
316     TLInfo(TM, *this),
317     FrameLowering(TargetFrameLowering::StackGrowsUp, getStackAlignment(), 0) {
318   // clang-format on
319   MaxWavesPerEU = AMDGPU::IsaInfo::getMaxWavesPerEU(this);
320   CallLoweringInfo.reset(new AMDGPUCallLowering(*getTargetLowering()));
321   InlineAsmLoweringInfo.reset(new InlineAsmLowering(getTargetLowering()));
322   Legalizer.reset(new AMDGPULegalizerInfo(*this, TM));
323   RegBankInfo.reset(new AMDGPURegisterBankInfo(*this));
324   InstSelector.reset(new AMDGPUInstructionSelector(
325   *this, *static_cast<AMDGPURegisterBankInfo *>(RegBankInfo.get()), TM));
326 }
327 
328 bool GCNSubtarget::enableFlatScratch() const {
329   return EnableFlatScratch && hasFlatScratchInsts();
330 }
331 
332 unsigned GCNSubtarget::getConstantBusLimit(unsigned Opcode) const {
333   if (getGeneration() < GFX10)
334     return 1;
335 
336   switch (Opcode) {
337   case AMDGPU::V_LSHLREV_B64_e64:
338   case AMDGPU::V_LSHLREV_B64_gfx10:
339   case AMDGPU::V_LSHL_B64_e64:
340   case AMDGPU::V_LSHRREV_B64_e64:
341   case AMDGPU::V_LSHRREV_B64_gfx10:
342   case AMDGPU::V_LSHR_B64_e64:
343   case AMDGPU::V_ASHRREV_I64_e64:
344   case AMDGPU::V_ASHRREV_I64_gfx10:
345   case AMDGPU::V_ASHR_I64_e64:
346     return 1;
347   }
348 
349   return 2;
350 }
351 
352 unsigned AMDGPUSubtarget::getMaxLocalMemSizeWithWaveCount(unsigned NWaves,
353   const Function &F) const {
354   if (NWaves == 1)
355     return getLocalMemorySize();
356   unsigned WorkGroupSize = getFlatWorkGroupSizes(F).second;
357   unsigned WorkGroupsPerCu = getMaxWorkGroupsPerCU(WorkGroupSize);
358   if (!WorkGroupsPerCu)
359     return 0;
360   unsigned MaxWaves = getMaxWavesPerEU();
361   return getLocalMemorySize() * MaxWaves / WorkGroupsPerCu / NWaves;
362 }
363 
364 // FIXME: Should return min,max range.
365 unsigned AMDGPUSubtarget::getOccupancyWithLocalMemSize(uint32_t Bytes,
366   const Function &F) const {
367   const unsigned MaxWorkGroupSize = getFlatWorkGroupSizes(F).second;
368   const unsigned MaxWorkGroupsPerCu = getMaxWorkGroupsPerCU(MaxWorkGroupSize);
369   if (!MaxWorkGroupsPerCu)
370     return 0;
371 
372   const unsigned WaveSize = getWavefrontSize();
373 
374   // FIXME: Do we need to account for alignment requirement of LDS rounding the
375   // size up?
376   // Compute restriction based on LDS usage
377   unsigned NumGroups = getLocalMemorySize() / (Bytes ? Bytes : 1u);
378 
379   // This can be queried with more LDS than is possible, so just assume the
380   // worst.
381   if (NumGroups == 0)
382     return 1;
383 
384   NumGroups = std::min(MaxWorkGroupsPerCu, NumGroups);
385 
386   // Round to the number of waves.
387   const unsigned MaxGroupNumWaves = (MaxWorkGroupSize + WaveSize - 1) / WaveSize;
388   unsigned MaxWaves = NumGroups * MaxGroupNumWaves;
389 
390   // Clamp to the maximum possible number of waves.
391   MaxWaves = std::min(MaxWaves, getMaxWavesPerEU());
392 
393   // FIXME: Needs to be a multiple of the group size?
394   //MaxWaves = MaxGroupNumWaves * (MaxWaves / MaxGroupNumWaves);
395 
396   assert(MaxWaves > 0 && MaxWaves <= getMaxWavesPerEU() &&
397          "computed invalid occupancy");
398   return MaxWaves;
399 }
400 
401 unsigned
402 AMDGPUSubtarget::getOccupancyWithLocalMemSize(const MachineFunction &MF) const {
403   const auto *MFI = MF.getInfo<SIMachineFunctionInfo>();
404   return getOccupancyWithLocalMemSize(MFI->getLDSSize(), MF.getFunction());
405 }
406 
407 std::pair<unsigned, unsigned>
408 AMDGPUSubtarget::getDefaultFlatWorkGroupSize(CallingConv::ID CC) const {
409   switch (CC) {
410   case CallingConv::AMDGPU_VS:
411   case CallingConv::AMDGPU_LS:
412   case CallingConv::AMDGPU_HS:
413   case CallingConv::AMDGPU_ES:
414   case CallingConv::AMDGPU_GS:
415   case CallingConv::AMDGPU_PS:
416     return std::make_pair(1, getWavefrontSize());
417   default:
418     return std::make_pair(1u, getMaxFlatWorkGroupSize());
419   }
420 }
421 
422 std::pair<unsigned, unsigned> AMDGPUSubtarget::getFlatWorkGroupSizes(
423   const Function &F) const {
424   // Default minimum/maximum flat work group sizes.
425   std::pair<unsigned, unsigned> Default =
426     getDefaultFlatWorkGroupSize(F.getCallingConv());
427 
428   // Requested minimum/maximum flat work group sizes.
429   std::pair<unsigned, unsigned> Requested = AMDGPU::getIntegerPairAttribute(
430     F, "amdgpu-flat-work-group-size", Default);
431 
432   // Make sure requested minimum is less than requested maximum.
433   if (Requested.first > Requested.second)
434     return Default;
435 
436   // Make sure requested values do not violate subtarget's specifications.
437   if (Requested.first < getMinFlatWorkGroupSize())
438     return Default;
439   if (Requested.second > getMaxFlatWorkGroupSize())
440     return Default;
441 
442   return Requested;
443 }
444 
445 std::pair<unsigned, unsigned> AMDGPUSubtarget::getWavesPerEU(
446   const Function &F) const {
447   // Default minimum/maximum number of waves per execution unit.
448   std::pair<unsigned, unsigned> Default(1, getMaxWavesPerEU());
449 
450   // Default/requested minimum/maximum flat work group sizes.
451   std::pair<unsigned, unsigned> FlatWorkGroupSizes = getFlatWorkGroupSizes(F);
452 
453   // If minimum/maximum flat work group sizes were explicitly requested using
454   // "amdgpu-flat-work-group-size" attribute, then set default minimum/maximum
455   // number of waves per execution unit to values implied by requested
456   // minimum/maximum flat work group sizes.
457   unsigned MinImpliedByFlatWorkGroupSize =
458     getWavesPerEUForWorkGroup(FlatWorkGroupSizes.second);
459   Default.first = MinImpliedByFlatWorkGroupSize;
460   bool RequestedFlatWorkGroupSize =
461       F.hasFnAttribute("amdgpu-flat-work-group-size");
462 
463   // Requested minimum/maximum number of waves per execution unit.
464   std::pair<unsigned, unsigned> Requested = AMDGPU::getIntegerPairAttribute(
465     F, "amdgpu-waves-per-eu", Default, true);
466 
467   // Make sure requested minimum is less than requested maximum.
468   if (Requested.second && Requested.first > Requested.second)
469     return Default;
470 
471   // Make sure requested values do not violate subtarget's specifications.
472   if (Requested.first < getMinWavesPerEU() ||
473       Requested.second > getMaxWavesPerEU())
474     return Default;
475 
476   // Make sure requested values are compatible with values implied by requested
477   // minimum/maximum flat work group sizes.
478   if (RequestedFlatWorkGroupSize &&
479       Requested.first < MinImpliedByFlatWorkGroupSize)
480     return Default;
481 
482   return Requested;
483 }
484 
485 static unsigned getReqdWorkGroupSize(const Function &Kernel, unsigned Dim) {
486   auto Node = Kernel.getMetadata("reqd_work_group_size");
487   if (Node && Node->getNumOperands() == 3)
488     return mdconst::extract<ConstantInt>(Node->getOperand(Dim))->getZExtValue();
489   return std::numeric_limits<unsigned>::max();
490 }
491 
492 bool AMDGPUSubtarget::isMesaKernel(const Function &F) const {
493   return isMesa3DOS() && !AMDGPU::isShader(F.getCallingConv());
494 }
495 
496 unsigned AMDGPUSubtarget::getMaxWorkitemID(const Function &Kernel,
497                                            unsigned Dimension) const {
498   unsigned ReqdSize = getReqdWorkGroupSize(Kernel, Dimension);
499   if (ReqdSize != std::numeric_limits<unsigned>::max())
500     return ReqdSize - 1;
501   return getFlatWorkGroupSizes(Kernel).second - 1;
502 }
503 
504 bool AMDGPUSubtarget::makeLIDRangeMetadata(Instruction *I) const {
505   Function *Kernel = I->getParent()->getParent();
506   unsigned MinSize = 0;
507   unsigned MaxSize = getFlatWorkGroupSizes(*Kernel).second;
508   bool IdQuery = false;
509 
510   // If reqd_work_group_size is present it narrows value down.
511   if (auto *CI = dyn_cast<CallInst>(I)) {
512     const Function *F = CI->getCalledFunction();
513     if (F) {
514       unsigned Dim = UINT_MAX;
515       switch (F->getIntrinsicID()) {
516       case Intrinsic::amdgcn_workitem_id_x:
517       case Intrinsic::r600_read_tidig_x:
518         IdQuery = true;
519         LLVM_FALLTHROUGH;
520       case Intrinsic::r600_read_local_size_x:
521         Dim = 0;
522         break;
523       case Intrinsic::amdgcn_workitem_id_y:
524       case Intrinsic::r600_read_tidig_y:
525         IdQuery = true;
526         LLVM_FALLTHROUGH;
527       case Intrinsic::r600_read_local_size_y:
528         Dim = 1;
529         break;
530       case Intrinsic::amdgcn_workitem_id_z:
531       case Intrinsic::r600_read_tidig_z:
532         IdQuery = true;
533         LLVM_FALLTHROUGH;
534       case Intrinsic::r600_read_local_size_z:
535         Dim = 2;
536         break;
537       default:
538         break;
539       }
540 
541       if (Dim <= 3) {
542         unsigned ReqdSize = getReqdWorkGroupSize(*Kernel, Dim);
543         if (ReqdSize != std::numeric_limits<unsigned>::max())
544           MinSize = MaxSize = ReqdSize;
545       }
546     }
547   }
548 
549   if (!MaxSize)
550     return false;
551 
552   // Range metadata is [Lo, Hi). For ID query we need to pass max size
553   // as Hi. For size query we need to pass Hi + 1.
554   if (IdQuery)
555     MinSize = 0;
556   else
557     ++MaxSize;
558 
559   MDBuilder MDB(I->getContext());
560   MDNode *MaxWorkGroupSizeRange = MDB.createRange(APInt(32, MinSize),
561                                                   APInt(32, MaxSize));
562   I->setMetadata(LLVMContext::MD_range, MaxWorkGroupSizeRange);
563   return true;
564 }
565 
566 unsigned AMDGPUSubtarget::getImplicitArgNumBytes(const Function &F) const {
567   if (isMesaKernel(F))
568     return 16;
569   return AMDGPU::getIntegerAttribute(F, "amdgpu-implicitarg-num-bytes", 0);
570 }
571 
572 uint64_t AMDGPUSubtarget::getExplicitKernArgSize(const Function &F,
573                                                  Align &MaxAlign) const {
574   assert(F.getCallingConv() == CallingConv::AMDGPU_KERNEL ||
575          F.getCallingConv() == CallingConv::SPIR_KERNEL);
576 
577   const DataLayout &DL = F.getParent()->getDataLayout();
578   uint64_t ExplicitArgBytes = 0;
579   MaxAlign = Align(1);
580 
581   for (const Argument &Arg : F.args()) {
582     const bool IsByRef = Arg.hasByRefAttr();
583     Type *ArgTy = IsByRef ? Arg.getParamByRefType() : Arg.getType();
584     MaybeAlign Alignment = IsByRef ? Arg.getParamAlign() : None;
585     if (!Alignment)
586       Alignment = DL.getABITypeAlign(ArgTy);
587 
588     uint64_t AllocSize = DL.getTypeAllocSize(ArgTy);
589     ExplicitArgBytes = alignTo(ExplicitArgBytes, Alignment) + AllocSize;
590     MaxAlign = max(MaxAlign, Alignment);
591   }
592 
593   return ExplicitArgBytes;
594 }
595 
596 unsigned AMDGPUSubtarget::getKernArgSegmentSize(const Function &F,
597                                                 Align &MaxAlign) const {
598   uint64_t ExplicitArgBytes = getExplicitKernArgSize(F, MaxAlign);
599 
600   unsigned ExplicitOffset = getExplicitKernelArgOffset(F);
601 
602   uint64_t TotalSize = ExplicitOffset + ExplicitArgBytes;
603   unsigned ImplicitBytes = getImplicitArgNumBytes(F);
604   if (ImplicitBytes != 0) {
605     const Align Alignment = getAlignmentForImplicitArgPtr();
606     TotalSize = alignTo(ExplicitArgBytes, Alignment) + ImplicitBytes;
607   }
608 
609   // Being able to dereference past the end is useful for emitting scalar loads.
610   return alignTo(TotalSize, 4);
611 }
612 
613 AMDGPUDwarfFlavour AMDGPUSubtarget::getAMDGPUDwarfFlavour() const {
614   return getWavefrontSize() == 32 ? AMDGPUDwarfFlavour::Wave32
615                                   : AMDGPUDwarfFlavour::Wave64;
616 }
617 
618 R600Subtarget::R600Subtarget(const Triple &TT, StringRef GPU, StringRef FS,
619                              const TargetMachine &TM) :
620   R600GenSubtargetInfo(TT, GPU, /*TuneCPU*/GPU, FS),
621   AMDGPUSubtarget(TT),
622   InstrInfo(*this),
623   FrameLowering(TargetFrameLowering::StackGrowsUp, getStackAlignment(), 0),
624   FMA(false),
625   CaymanISA(false),
626   CFALUBug(false),
627   HasVertexCache(false),
628   R600ALUInst(false),
629   FP64(false),
630   TexVTXClauseSize(0),
631   Gen(R600),
632   TLInfo(TM, initializeSubtargetDependencies(TT, GPU, FS)),
633   InstrItins(getInstrItineraryForCPU(GPU)) { }
634 
635 void GCNSubtarget::overrideSchedPolicy(MachineSchedPolicy &Policy,
636                                       unsigned NumRegionInstrs) const {
637   // Track register pressure so the scheduler can try to decrease
638   // pressure once register usage is above the threshold defined by
639   // SIRegisterInfo::getRegPressureSetLimit()
640   Policy.ShouldTrackPressure = true;
641 
642   // Enabling both top down and bottom up scheduling seems to give us less
643   // register spills than just using one of these approaches on its own.
644   Policy.OnlyTopDown = false;
645   Policy.OnlyBottomUp = false;
646 
647   // Enabling ShouldTrackLaneMasks crashes the SI Machine Scheduler.
648   if (!enableSIScheduler())
649     Policy.ShouldTrackLaneMasks = true;
650 }
651 
652 bool GCNSubtarget::hasMadF16() const {
653   return InstrInfo.pseudoToMCOpcode(AMDGPU::V_MAD_F16_e64) != -1;
654 }
655 
656 bool GCNSubtarget::useVGPRIndexMode() const {
657   return !hasMovrel() || (EnableVGPRIndexMode && hasVGPRIndexMode());
658 }
659 
660 bool GCNSubtarget::useAA() const { return UseAA; }
661 
662 unsigned GCNSubtarget::getOccupancyWithNumSGPRs(unsigned SGPRs) const {
663   if (getGeneration() >= AMDGPUSubtarget::GFX10)
664     return getMaxWavesPerEU();
665 
666   if (getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
667     if (SGPRs <= 80)
668       return 10;
669     if (SGPRs <= 88)
670       return 9;
671     if (SGPRs <= 100)
672       return 8;
673     return 7;
674   }
675   if (SGPRs <= 48)
676     return 10;
677   if (SGPRs <= 56)
678     return 9;
679   if (SGPRs <= 64)
680     return 8;
681   if (SGPRs <= 72)
682     return 7;
683   if (SGPRs <= 80)
684     return 6;
685   return 5;
686 }
687 
688 unsigned GCNSubtarget::getOccupancyWithNumVGPRs(unsigned VGPRs) const {
689   unsigned MaxWaves = getMaxWavesPerEU();
690   unsigned Granule = getVGPRAllocGranule();
691   if (VGPRs < Granule)
692     return MaxWaves;
693   unsigned RoundedRegs = ((VGPRs + Granule - 1) / Granule) * Granule;
694   return std::min(std::max(getTotalNumVGPRs() / RoundedRegs, 1u), MaxWaves);
695 }
696 
697 unsigned GCNSubtarget::getReservedNumSGPRs(const MachineFunction &MF) const {
698   const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
699   if (getGeneration() >= AMDGPUSubtarget::GFX10)
700     return 2; // VCC. FLAT_SCRATCH and XNACK are no longer in SGPRs.
701 
702   if (MFI.hasFlatScratchInit()) {
703     if (getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
704       return 6; // FLAT_SCRATCH, XNACK, VCC (in that order).
705     if (getGeneration() == AMDGPUSubtarget::SEA_ISLANDS)
706       return 4; // FLAT_SCRATCH, VCC (in that order).
707   }
708 
709   if (isXNACKEnabled())
710     return 4; // XNACK, VCC (in that order).
711   return 2; // VCC.
712 }
713 
714 unsigned GCNSubtarget::computeOccupancy(const Function &F, unsigned LDSSize,
715                                         unsigned NumSGPRs,
716                                         unsigned NumVGPRs) const {
717   unsigned Occupancy =
718     std::min(getMaxWavesPerEU(),
719              getOccupancyWithLocalMemSize(LDSSize, F));
720   if (NumSGPRs)
721     Occupancy = std::min(Occupancy, getOccupancyWithNumSGPRs(NumSGPRs));
722   if (NumVGPRs)
723     Occupancy = std::min(Occupancy, getOccupancyWithNumVGPRs(NumVGPRs));
724   return Occupancy;
725 }
726 
727 unsigned GCNSubtarget::getMaxNumSGPRs(const MachineFunction &MF) const {
728   const Function &F = MF.getFunction();
729   const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
730 
731   // Compute maximum number of SGPRs function can use using default/requested
732   // minimum number of waves per execution unit.
733   std::pair<unsigned, unsigned> WavesPerEU = MFI.getWavesPerEU();
734   unsigned MaxNumSGPRs = getMaxNumSGPRs(WavesPerEU.first, false);
735   unsigned MaxAddressableNumSGPRs = getMaxNumSGPRs(WavesPerEU.first, true);
736 
737   // Check if maximum number of SGPRs was explicitly requested using
738   // "amdgpu-num-sgpr" attribute.
739   if (F.hasFnAttribute("amdgpu-num-sgpr")) {
740     unsigned Requested = AMDGPU::getIntegerAttribute(
741       F, "amdgpu-num-sgpr", MaxNumSGPRs);
742 
743     // Make sure requested value does not violate subtarget's specifications.
744     if (Requested && (Requested <= getReservedNumSGPRs(MF)))
745       Requested = 0;
746 
747     // If more SGPRs are required to support the input user/system SGPRs,
748     // increase to accommodate them.
749     //
750     // FIXME: This really ends up using the requested number of SGPRs + number
751     // of reserved special registers in total. Theoretically you could re-use
752     // the last input registers for these special registers, but this would
753     // require a lot of complexity to deal with the weird aliasing.
754     unsigned InputNumSGPRs = MFI.getNumPreloadedSGPRs();
755     if (Requested && Requested < InputNumSGPRs)
756       Requested = InputNumSGPRs;
757 
758     // Make sure requested value is compatible with values implied by
759     // default/requested minimum/maximum number of waves per execution unit.
760     if (Requested && Requested > getMaxNumSGPRs(WavesPerEU.first, false))
761       Requested = 0;
762     if (WavesPerEU.second &&
763         Requested && Requested < getMinNumSGPRs(WavesPerEU.second))
764       Requested = 0;
765 
766     if (Requested)
767       MaxNumSGPRs = Requested;
768   }
769 
770   if (hasSGPRInitBug())
771     MaxNumSGPRs = AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG;
772 
773   return std::min(MaxNumSGPRs - getReservedNumSGPRs(MF),
774                   MaxAddressableNumSGPRs);
775 }
776 
777 unsigned GCNSubtarget::getMaxNumVGPRs(const MachineFunction &MF) const {
778   const Function &F = MF.getFunction();
779   const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
780 
781   // Compute maximum number of VGPRs function can use using default/requested
782   // minimum number of waves per execution unit.
783   std::pair<unsigned, unsigned> WavesPerEU = MFI.getWavesPerEU();
784   unsigned MaxNumVGPRs = getMaxNumVGPRs(WavesPerEU.first);
785 
786   // Check if maximum number of VGPRs was explicitly requested using
787   // "amdgpu-num-vgpr" attribute.
788   if (F.hasFnAttribute("amdgpu-num-vgpr")) {
789     unsigned Requested = AMDGPU::getIntegerAttribute(
790       F, "amdgpu-num-vgpr", MaxNumVGPRs);
791 
792     if (hasGFX90AInsts())
793       Requested *= 2;
794 
795     // Make sure requested value is compatible with values implied by
796     // default/requested minimum/maximum number of waves per execution unit.
797     if (Requested && Requested > getMaxNumVGPRs(WavesPerEU.first))
798       Requested = 0;
799     if (WavesPerEU.second &&
800         Requested && Requested < getMinNumVGPRs(WavesPerEU.second))
801       Requested = 0;
802 
803     if (Requested)
804       MaxNumVGPRs = Requested;
805   }
806 
807   return MaxNumVGPRs;
808 }
809 
810 void GCNSubtarget::adjustSchedDependency(SUnit *Def, int DefOpIdx, SUnit *Use,
811                                          int UseOpIdx, SDep &Dep) const {
812   if (Dep.getKind() != SDep::Kind::Data || !Dep.getReg() ||
813       !Def->isInstr() || !Use->isInstr())
814     return;
815 
816   MachineInstr *DefI = Def->getInstr();
817   MachineInstr *UseI = Use->getInstr();
818 
819   if (DefI->isBundle()) {
820     const SIRegisterInfo *TRI = getRegisterInfo();
821     auto Reg = Dep.getReg();
822     MachineBasicBlock::const_instr_iterator I(DefI->getIterator());
823     MachineBasicBlock::const_instr_iterator E(DefI->getParent()->instr_end());
824     unsigned Lat = 0;
825     for (++I; I != E && I->isBundledWithPred(); ++I) {
826       if (I->modifiesRegister(Reg, TRI))
827         Lat = InstrInfo.getInstrLatency(getInstrItineraryData(), *I);
828       else if (Lat)
829         --Lat;
830     }
831     Dep.setLatency(Lat);
832   } else if (UseI->isBundle()) {
833     const SIRegisterInfo *TRI = getRegisterInfo();
834     auto Reg = Dep.getReg();
835     MachineBasicBlock::const_instr_iterator I(UseI->getIterator());
836     MachineBasicBlock::const_instr_iterator E(UseI->getParent()->instr_end());
837     unsigned Lat = InstrInfo.getInstrLatency(getInstrItineraryData(), *DefI);
838     for (++I; I != E && I->isBundledWithPred() && Lat; ++I) {
839       if (I->readsRegister(Reg, TRI))
840         break;
841       --Lat;
842     }
843     Dep.setLatency(Lat);
844   }
845 }
846 
847 namespace {
848 struct FillMFMAShadowMutation : ScheduleDAGMutation {
849   const SIInstrInfo *TII;
850 
851   ScheduleDAGMI *DAG;
852 
853   FillMFMAShadowMutation(const SIInstrInfo *tii) : TII(tii) {}
854 
855   bool isSALU(const SUnit *SU) const {
856     const MachineInstr *MI = SU->getInstr();
857     return MI && TII->isSALU(*MI) && !MI->isTerminator();
858   }
859 
860   bool isVALU(const SUnit *SU) const {
861     const MachineInstr *MI = SU->getInstr();
862     return MI && TII->isVALU(*MI);
863   }
864 
865   bool canAddEdge(const SUnit *Succ, const SUnit *Pred) const {
866     if (Pred->NodeNum < Succ->NodeNum)
867       return true;
868 
869     SmallVector<const SUnit*, 64> Succs({Succ}), Preds({Pred});
870 
871     for (unsigned I = 0; I < Succs.size(); ++I) {
872       for (const SDep &SI : Succs[I]->Succs) {
873         const SUnit *SU = SI.getSUnit();
874         if (SU != Succs[I] && !llvm::is_contained(Succs, SU))
875           Succs.push_back(SU);
876       }
877     }
878 
879     SmallPtrSet<const SUnit*, 32> Visited;
880     while (!Preds.empty()) {
881       const SUnit *SU = Preds.pop_back_val();
882       if (llvm::is_contained(Succs, SU))
883         return false;
884       Visited.insert(SU);
885       for (const SDep &SI : SU->Preds)
886         if (SI.getSUnit() != SU && !Visited.count(SI.getSUnit()))
887           Preds.push_back(SI.getSUnit());
888     }
889 
890     return true;
891   }
892 
893   // Link as much SALU intructions in chain as possible. Return the size
894   // of the chain. Links up to MaxChain instructions.
895   unsigned linkSALUChain(SUnit *From, SUnit *To, unsigned MaxChain,
896                          SmallPtrSetImpl<SUnit *> &Visited) const {
897     SmallVector<SUnit *, 8> Worklist({To});
898     unsigned Linked = 0;
899 
900     while (!Worklist.empty() && MaxChain-- > 0) {
901       SUnit *SU = Worklist.pop_back_val();
902       if (!Visited.insert(SU).second)
903         continue;
904 
905       LLVM_DEBUG(dbgs() << "Inserting edge from\n" ; DAG->dumpNode(*From);
906                  dbgs() << "to\n"; DAG->dumpNode(*SU); dbgs() << '\n');
907 
908       if (SU->addPred(SDep(From, SDep::Artificial), false))
909         ++Linked;
910 
911       for (SDep &SI : From->Succs) {
912         SUnit *SUv = SI.getSUnit();
913         if (SUv != From && isVALU(SUv) && canAddEdge(SUv, SU))
914           SUv->addPred(SDep(SU, SDep::Artificial), false);
915       }
916 
917       for (SDep &SI : SU->Succs) {
918         SUnit *Succ = SI.getSUnit();
919         if (Succ != SU && isSALU(Succ) && canAddEdge(From, Succ))
920           Worklist.push_back(Succ);
921       }
922     }
923 
924     return Linked;
925   }
926 
927   void apply(ScheduleDAGInstrs *DAGInstrs) override {
928     const GCNSubtarget &ST = DAGInstrs->MF.getSubtarget<GCNSubtarget>();
929     if (!ST.hasMAIInsts() || DisablePowerSched)
930       return;
931     DAG = static_cast<ScheduleDAGMI*>(DAGInstrs);
932     const TargetSchedModel *TSchedModel = DAGInstrs->getSchedModel();
933     if (!TSchedModel || DAG->SUnits.empty())
934       return;
935 
936     // Scan for MFMA long latency instructions and try to add a dependency
937     // of available SALU instructions to give them a chance to fill MFMA
938     // shadow. That is desirable to fill MFMA shadow with SALU instructions
939     // rather than VALU to prevent power consumption bursts and throttle.
940     auto LastSALU = DAG->SUnits.begin();
941     auto E = DAG->SUnits.end();
942     SmallPtrSet<SUnit*, 32> Visited;
943     for (SUnit &SU : DAG->SUnits) {
944       MachineInstr &MAI = *SU.getInstr();
945       if (!TII->isMAI(MAI) ||
946            MAI.getOpcode() == AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
947            MAI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64)
948         continue;
949 
950       unsigned Lat = TSchedModel->computeInstrLatency(&MAI) - 1;
951 
952       LLVM_DEBUG(dbgs() << "Found MFMA: "; DAG->dumpNode(SU);
953                  dbgs() << "Need " << Lat
954                         << " instructions to cover latency.\n");
955 
956       // Find up to Lat independent scalar instructions as early as
957       // possible such that they can be scheduled after this MFMA.
958       for ( ; Lat && LastSALU != E; ++LastSALU) {
959         if (Visited.count(&*LastSALU))
960           continue;
961 
962         if (!isSALU(&*LastSALU) || !canAddEdge(&*LastSALU, &SU))
963           continue;
964 
965         Lat -= linkSALUChain(&SU, &*LastSALU, Lat, Visited);
966       }
967     }
968   }
969 };
970 } // namespace
971 
972 void GCNSubtarget::getPostRAMutations(
973     std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations) const {
974   Mutations.push_back(std::make_unique<FillMFMAShadowMutation>(&InstrInfo));
975 }
976 
977 const AMDGPUSubtarget &AMDGPUSubtarget::get(const MachineFunction &MF) {
978   if (MF.getTarget().getTargetTriple().getArch() == Triple::amdgcn)
979     return static_cast<const AMDGPUSubtarget&>(MF.getSubtarget<GCNSubtarget>());
980   else
981     return static_cast<const AMDGPUSubtarget&>(MF.getSubtarget<R600Subtarget>());
982 }
983 
984 const AMDGPUSubtarget &AMDGPUSubtarget::get(const TargetMachine &TM, const Function &F) {
985   if (TM.getTargetTriple().getArch() == Triple::amdgcn)
986     return static_cast<const AMDGPUSubtarget&>(TM.getSubtarget<GCNSubtarget>(F));
987   else
988     return static_cast<const AMDGPUSubtarget&>(TM.getSubtarget<R600Subtarget>(F));
989 }
990