1 //===-- AMDGPUSubtarget.cpp - AMDGPU Subtarget Information ----------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// \brief Implements the AMDGPU specific subclass of TargetSubtarget.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "AMDGPUSubtarget.h"
16 #include "AMDGPU.h"
17 #include "AMDGPUTargetMachine.h"
18 #include "AMDGPUCallLowering.h"
19 #include "AMDGPUInstructionSelector.h"
20 #include "AMDGPULegalizerInfo.h"
21 #include "AMDGPURegisterBankInfo.h"
22 #include "SIMachineFunctionInfo.h"
23 #include "llvm/ADT/SmallString.h"
24 #include "llvm/CodeGen/MachineScheduler.h"
25 #include "llvm/IR/MDBuilder.h"
26 #include "llvm/Target/TargetFrameLowering.h"
27 #include <algorithm>
28 
29 using namespace llvm;
30 
31 #define DEBUG_TYPE "amdgpu-subtarget"
32 
33 #define GET_SUBTARGETINFO_TARGET_DESC
34 #define GET_SUBTARGETINFO_CTOR
35 #include "AMDGPUGenSubtargetInfo.inc"
36 
37 AMDGPUSubtarget::~AMDGPUSubtarget() = default;
38 
39 AMDGPUSubtarget &
40 AMDGPUSubtarget::initializeSubtargetDependencies(const Triple &TT,
41                                                  StringRef GPU, StringRef FS) {
42   // Determine default and user-specified characteristics
43   // On SI+, we want FP64 denormals to be on by default. FP32 denormals can be
44   // enabled, but some instructions do not respect them and they run at the
45   // double precision rate, so don't enable by default.
46   //
47   // We want to be able to turn these off, but making this a subtarget feature
48   // for SI has the unhelpful behavior that it unsets everything else if you
49   // disable it.
50 
51   SmallString<256> FullFS("+promote-alloca,+fp64-fp16-denormals,+dx10-clamp,+load-store-opt,");
52   if (isAmdHsaOS()) // Turn on FlatForGlobal for HSA.
53     FullFS += "+flat-address-space,+flat-for-global,+unaligned-buffer-access,+trap-handler,";
54 
55   FullFS += FS;
56 
57   ParseSubtargetFeatures(GPU, FullFS);
58 
59   // Unless +-flat-for-global is specified, turn on FlatForGlobal for all OS-es
60   // on VI and newer hardware to avoid assertion failures due to missing ADDR64
61   // variants of MUBUF instructions.
62   if (!hasAddr64() && !FS.contains("flat-for-global")) {
63     FlatForGlobal = true;
64   }
65 
66   // FIXME: I don't think think Evergreen has any useful support for
67   // denormals, but should be checked. Should we issue a warning somewhere
68   // if someone tries to enable these?
69   if (getGeneration() <= AMDGPUSubtarget::NORTHERN_ISLANDS) {
70     FP64FP16Denormals = false;
71     FP32Denormals = false;
72   }
73 
74   // Set defaults if needed.
75   if (MaxPrivateElementSize == 0)
76     MaxPrivateElementSize = 4;
77 
78   if (LDSBankCount == 0)
79     LDSBankCount = 32;
80 
81   if (TT.getArch() == Triple::amdgcn) {
82     if (LocalMemorySize == 0)
83       LocalMemorySize = 32768;
84 
85     // Do something sensible for unspecified target.
86     if (!HasMovrel && !HasVGPRIndexMode)
87       HasMovrel = true;
88   }
89 
90   return *this;
91 }
92 
93 AMDGPUSubtarget::AMDGPUSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
94                                  const TargetMachine &TM)
95   : AMDGPUGenSubtargetInfo(TT, GPU, FS),
96     TargetTriple(TT),
97     Gen(TT.getArch() == Triple::amdgcn ? SOUTHERN_ISLANDS : R600),
98     IsaVersion(ISAVersion0_0_0),
99     WavefrontSize(64),
100     LocalMemorySize(0),
101     LDSBankCount(0),
102     MaxPrivateElementSize(0),
103 
104     FastFMAF32(false),
105     HalfRate64Ops(false),
106 
107     FP32Denormals(false),
108     FP64FP16Denormals(false),
109     FPExceptions(false),
110     DX10Clamp(false),
111     FlatForGlobal(false),
112     AutoWaitcntBeforeBarrier(false),
113     CodeObjectV3(false),
114     UnalignedScratchAccess(false),
115     UnalignedBufferAccess(false),
116 
117     HasApertureRegs(false),
118     EnableXNACK(false),
119     TrapHandler(false),
120     DebuggerInsertNops(false),
121     DebuggerReserveRegs(false),
122     DebuggerEmitPrologue(false),
123 
124     EnableVGPRSpilling(false),
125     EnablePromoteAlloca(false),
126     EnableLoadStoreOpt(false),
127     EnableUnsafeDSOffsetFolding(false),
128     EnableSIScheduler(false),
129     DumpCode(false),
130 
131     FP64(false),
132     IsGCN(false),
133     GCN3Encoding(false),
134     CIInsts(false),
135     GFX9Insts(false),
136     SGPRInitBug(false),
137     HasSMemRealTime(false),
138     Has16BitInsts(false),
139     HasIntClamp(false),
140     HasVOP3PInsts(false),
141     HasMovrel(false),
142     HasVGPRIndexMode(false),
143     HasScalarStores(false),
144     HasInv2PiInlineImm(false),
145     HasSDWA(false),
146     HasSDWAOmod(false),
147     HasSDWAScalar(false),
148     HasSDWASdst(false),
149     HasSDWAMac(false),
150     HasSDWAOutModsVOPC(false),
151     HasDPP(false),
152     FlatAddressSpace(false),
153     FlatInstOffsets(false),
154     FlatGlobalInsts(false),
155     FlatScratchInsts(false),
156     AddNoCarryInsts(false),
157 
158     R600ALUInst(false),
159     CaymanISA(false),
160     CFALUBug(false),
161     HasVertexCache(false),
162     TexVTXClauseSize(0),
163     ScalarizeGlobal(false),
164 
165     FeatureDisable(false),
166     InstrItins(getInstrItineraryForCPU(GPU)) {
167   AS = AMDGPU::getAMDGPUAS(TT);
168   initializeSubtargetDependencies(TT, GPU, FS);
169 }
170 
171 unsigned AMDGPUSubtarget::getMaxLocalMemSizeWithWaveCount(unsigned NWaves,
172   const Function &F) const {
173   if (NWaves == 1)
174     return getLocalMemorySize();
175   unsigned WorkGroupSize = getFlatWorkGroupSizes(F).second;
176   unsigned WorkGroupsPerCu = getMaxWorkGroupsPerCU(WorkGroupSize);
177   unsigned MaxWaves = getMaxWavesPerEU();
178   return getLocalMemorySize() * MaxWaves / WorkGroupsPerCu / NWaves;
179 }
180 
181 unsigned AMDGPUSubtarget::getOccupancyWithLocalMemSize(uint32_t Bytes,
182   const Function &F) const {
183   unsigned WorkGroupSize = getFlatWorkGroupSizes(F).second;
184   unsigned WorkGroupsPerCu = getMaxWorkGroupsPerCU(WorkGroupSize);
185   unsigned MaxWaves = getMaxWavesPerEU();
186   unsigned Limit = getLocalMemorySize() * MaxWaves / WorkGroupsPerCu;
187   unsigned NumWaves = Limit / (Bytes ? Bytes : 1u);
188   NumWaves = std::min(NumWaves, MaxWaves);
189   NumWaves = std::max(NumWaves, 1u);
190   return NumWaves;
191 }
192 
193 std::pair<unsigned, unsigned> AMDGPUSubtarget::getFlatWorkGroupSizes(
194   const Function &F) const {
195   // Default minimum/maximum flat work group sizes.
196   std::pair<unsigned, unsigned> Default =
197     AMDGPU::isCompute(F.getCallingConv()) ?
198       std::pair<unsigned, unsigned>(getWavefrontSize() * 2,
199                                     getWavefrontSize() * 4) :
200       std::pair<unsigned, unsigned>(1, getWavefrontSize());
201 
202   // TODO: Do not process "amdgpu-max-work-group-size" attribute once mesa
203   // starts using "amdgpu-flat-work-group-size" attribute.
204   Default.second = AMDGPU::getIntegerAttribute(
205     F, "amdgpu-max-work-group-size", Default.second);
206   Default.first = std::min(Default.first, Default.second);
207 
208   // Requested minimum/maximum flat work group sizes.
209   std::pair<unsigned, unsigned> Requested = AMDGPU::getIntegerPairAttribute(
210     F, "amdgpu-flat-work-group-size", Default);
211 
212   // Make sure requested minimum is less than requested maximum.
213   if (Requested.first > Requested.second)
214     return Default;
215 
216   // Make sure requested values do not violate subtarget's specifications.
217   if (Requested.first < getMinFlatWorkGroupSize())
218     return Default;
219   if (Requested.second > getMaxFlatWorkGroupSize())
220     return Default;
221 
222   return Requested;
223 }
224 
225 std::pair<unsigned, unsigned> AMDGPUSubtarget::getWavesPerEU(
226   const Function &F) const {
227   // Default minimum/maximum number of waves per execution unit.
228   std::pair<unsigned, unsigned> Default(1, getMaxWavesPerEU());
229 
230   // Default/requested minimum/maximum flat work group sizes.
231   std::pair<unsigned, unsigned> FlatWorkGroupSizes = getFlatWorkGroupSizes(F);
232 
233   // If minimum/maximum flat work group sizes were explicitly requested using
234   // "amdgpu-flat-work-group-size" attribute, then set default minimum/maximum
235   // number of waves per execution unit to values implied by requested
236   // minimum/maximum flat work group sizes.
237   unsigned MinImpliedByFlatWorkGroupSize =
238     getMaxWavesPerEU(FlatWorkGroupSizes.second);
239   bool RequestedFlatWorkGroupSize = false;
240 
241   // TODO: Do not process "amdgpu-max-work-group-size" attribute once mesa
242   // starts using "amdgpu-flat-work-group-size" attribute.
243   if (F.hasFnAttribute("amdgpu-max-work-group-size") ||
244       F.hasFnAttribute("amdgpu-flat-work-group-size")) {
245     Default.first = MinImpliedByFlatWorkGroupSize;
246     RequestedFlatWorkGroupSize = true;
247   }
248 
249   // Requested minimum/maximum number of waves per execution unit.
250   std::pair<unsigned, unsigned> Requested = AMDGPU::getIntegerPairAttribute(
251     F, "amdgpu-waves-per-eu", Default, true);
252 
253   // Make sure requested minimum is less than requested maximum.
254   if (Requested.second && Requested.first > Requested.second)
255     return Default;
256 
257   // Make sure requested values do not violate subtarget's specifications.
258   if (Requested.first < getMinWavesPerEU() ||
259       Requested.first > getMaxWavesPerEU())
260     return Default;
261   if (Requested.second > getMaxWavesPerEU())
262     return Default;
263 
264   // Make sure requested values are compatible with values implied by requested
265   // minimum/maximum flat work group sizes.
266   if (RequestedFlatWorkGroupSize &&
267       Requested.first < MinImpliedByFlatWorkGroupSize)
268     return Default;
269 
270   return Requested;
271 }
272 
273 bool AMDGPUSubtarget::makeLIDRangeMetadata(Instruction *I) const {
274   Function *Kernel = I->getParent()->getParent();
275   unsigned MinSize = 0;
276   unsigned MaxSize = getFlatWorkGroupSizes(*Kernel).second;
277   bool IdQuery = false;
278 
279   // If reqd_work_group_size is present it narrows value down.
280   if (auto *CI = dyn_cast<CallInst>(I)) {
281     const Function *F = CI->getCalledFunction();
282     if (F) {
283       unsigned Dim = UINT_MAX;
284       switch (F->getIntrinsicID()) {
285       case Intrinsic::amdgcn_workitem_id_x:
286       case Intrinsic::r600_read_tidig_x:
287         IdQuery = true;
288         LLVM_FALLTHROUGH;
289       case Intrinsic::r600_read_local_size_x:
290         Dim = 0;
291         break;
292       case Intrinsic::amdgcn_workitem_id_y:
293       case Intrinsic::r600_read_tidig_y:
294         IdQuery = true;
295         LLVM_FALLTHROUGH;
296       case Intrinsic::r600_read_local_size_y:
297         Dim = 1;
298         break;
299       case Intrinsic::amdgcn_workitem_id_z:
300       case Intrinsic::r600_read_tidig_z:
301         IdQuery = true;
302         LLVM_FALLTHROUGH;
303       case Intrinsic::r600_read_local_size_z:
304         Dim = 2;
305         break;
306       default:
307         break;
308       }
309       if (Dim <= 3) {
310         if (auto Node = Kernel->getMetadata("reqd_work_group_size"))
311           if (Node->getNumOperands() == 3)
312             MinSize = MaxSize = mdconst::extract<ConstantInt>(
313                                   Node->getOperand(Dim))->getZExtValue();
314       }
315     }
316   }
317 
318   if (!MaxSize)
319     return false;
320 
321   // Range metadata is [Lo, Hi). For ID query we need to pass max size
322   // as Hi. For size query we need to pass Hi + 1.
323   if (IdQuery)
324     MinSize = 0;
325   else
326     ++MaxSize;
327 
328   MDBuilder MDB(I->getContext());
329   MDNode *MaxWorkGroupSizeRange = MDB.createRange(APInt(32, MinSize),
330                                                   APInt(32, MaxSize));
331   I->setMetadata(LLVMContext::MD_range, MaxWorkGroupSizeRange);
332   return true;
333 }
334 
335 R600Subtarget::R600Subtarget(const Triple &TT, StringRef GPU, StringRef FS,
336                              const TargetMachine &TM) :
337   AMDGPUSubtarget(TT, GPU, FS, TM),
338   InstrInfo(*this),
339   FrameLowering(TargetFrameLowering::StackGrowsUp, getStackAlignment(), 0),
340   TLInfo(TM, *this) {}
341 
342 SISubtarget::SISubtarget(const Triple &TT, StringRef GPU, StringRef FS,
343                          const TargetMachine &TM)
344     : AMDGPUSubtarget(TT, GPU, FS, TM), InstrInfo(*this),
345       FrameLowering(TargetFrameLowering::StackGrowsUp, getStackAlignment(), 0),
346       TLInfo(TM, *this) {
347   CallLoweringInfo.reset(new AMDGPUCallLowering(*getTargetLowering()));
348   Legalizer.reset(new AMDGPULegalizerInfo());
349 
350   RegBankInfo.reset(new AMDGPURegisterBankInfo(*getRegisterInfo()));
351   InstSelector.reset(new AMDGPUInstructionSelector(
352       *this, *static_cast<AMDGPURegisterBankInfo *>(RegBankInfo.get())));
353 }
354 
355 void SISubtarget::overrideSchedPolicy(MachineSchedPolicy &Policy,
356                                       unsigned NumRegionInstrs) const {
357   // Track register pressure so the scheduler can try to decrease
358   // pressure once register usage is above the threshold defined by
359   // SIRegisterInfo::getRegPressureSetLimit()
360   Policy.ShouldTrackPressure = true;
361 
362   // Enabling both top down and bottom up scheduling seems to give us less
363   // register spills than just using one of these approaches on its own.
364   Policy.OnlyTopDown = false;
365   Policy.OnlyBottomUp = false;
366 
367   // Enabling ShouldTrackLaneMasks crashes the SI Machine Scheduler.
368   if (!enableSIScheduler())
369     Policy.ShouldTrackLaneMasks = true;
370 }
371 
372 bool SISubtarget::isVGPRSpillingEnabled(const Function& F) const {
373   return EnableVGPRSpilling || !AMDGPU::isShader(F.getCallingConv());
374 }
375 
376 unsigned SISubtarget::getKernArgSegmentSize(const MachineFunction &MF,
377                                             unsigned ExplicitArgBytes) const {
378   unsigned ImplicitBytes = getImplicitArgNumBytes(MF);
379   if (ImplicitBytes == 0)
380     return ExplicitArgBytes;
381 
382   unsigned Alignment = getAlignmentForImplicitArgPtr();
383   return alignTo(ExplicitArgBytes, Alignment) + ImplicitBytes;
384 }
385 
386 unsigned SISubtarget::getOccupancyWithNumSGPRs(unsigned SGPRs) const {
387   if (getGeneration() >= SISubtarget::VOLCANIC_ISLANDS) {
388     if (SGPRs <= 80)
389       return 10;
390     if (SGPRs <= 88)
391       return 9;
392     if (SGPRs <= 100)
393       return 8;
394     return 7;
395   }
396   if (SGPRs <= 48)
397     return 10;
398   if (SGPRs <= 56)
399     return 9;
400   if (SGPRs <= 64)
401     return 8;
402   if (SGPRs <= 72)
403     return 7;
404   if (SGPRs <= 80)
405     return 6;
406   return 5;
407 }
408 
409 unsigned SISubtarget::getOccupancyWithNumVGPRs(unsigned VGPRs) const {
410   if (VGPRs <= 24)
411     return 10;
412   if (VGPRs <= 28)
413     return 9;
414   if (VGPRs <= 32)
415     return 8;
416   if (VGPRs <= 36)
417     return 7;
418   if (VGPRs <= 40)
419     return 6;
420   if (VGPRs <= 48)
421     return 5;
422   if (VGPRs <= 64)
423     return 4;
424   if (VGPRs <= 84)
425     return 3;
426   if (VGPRs <= 128)
427     return 2;
428   return 1;
429 }
430 
431 unsigned SISubtarget::getReservedNumSGPRs(const MachineFunction &MF) const {
432   const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
433   if (MFI.hasFlatScratchInit()) {
434     if (getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
435       return 6; // FLAT_SCRATCH, XNACK, VCC (in that order).
436     if (getGeneration() == AMDGPUSubtarget::SEA_ISLANDS)
437       return 4; // FLAT_SCRATCH, VCC (in that order).
438   }
439 
440   if (isXNACKEnabled())
441     return 4; // XNACK, VCC (in that order).
442   return 2; // VCC.
443 }
444 
445 unsigned SISubtarget::getMaxNumSGPRs(const MachineFunction &MF) const {
446   const Function &F = *MF.getFunction();
447   const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
448 
449   // Compute maximum number of SGPRs function can use using default/requested
450   // minimum number of waves per execution unit.
451   std::pair<unsigned, unsigned> WavesPerEU = MFI.getWavesPerEU();
452   unsigned MaxNumSGPRs = getMaxNumSGPRs(WavesPerEU.first, false);
453   unsigned MaxAddressableNumSGPRs = getMaxNumSGPRs(WavesPerEU.first, true);
454 
455   // Check if maximum number of SGPRs was explicitly requested using
456   // "amdgpu-num-sgpr" attribute.
457   if (F.hasFnAttribute("amdgpu-num-sgpr")) {
458     unsigned Requested = AMDGPU::getIntegerAttribute(
459       F, "amdgpu-num-sgpr", MaxNumSGPRs);
460 
461     // Make sure requested value does not violate subtarget's specifications.
462     if (Requested && (Requested <= getReservedNumSGPRs(MF)))
463       Requested = 0;
464 
465     // If more SGPRs are required to support the input user/system SGPRs,
466     // increase to accommodate them.
467     //
468     // FIXME: This really ends up using the requested number of SGPRs + number
469     // of reserved special registers in total. Theoretically you could re-use
470     // the last input registers for these special registers, but this would
471     // require a lot of complexity to deal with the weird aliasing.
472     unsigned InputNumSGPRs = MFI.getNumPreloadedSGPRs();
473     if (Requested && Requested < InputNumSGPRs)
474       Requested = InputNumSGPRs;
475 
476     // Make sure requested value is compatible with values implied by
477     // default/requested minimum/maximum number of waves per execution unit.
478     if (Requested && Requested > getMaxNumSGPRs(WavesPerEU.first, false))
479       Requested = 0;
480     if (WavesPerEU.second &&
481         Requested && Requested < getMinNumSGPRs(WavesPerEU.second))
482       Requested = 0;
483 
484     if (Requested)
485       MaxNumSGPRs = Requested;
486   }
487 
488   if (hasSGPRInitBug())
489     MaxNumSGPRs = AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG;
490 
491   return std::min(MaxNumSGPRs - getReservedNumSGPRs(MF),
492                   MaxAddressableNumSGPRs);
493 }
494 
495 unsigned SISubtarget::getMaxNumVGPRs(const MachineFunction &MF) const {
496   const Function &F = *MF.getFunction();
497   const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
498 
499   // Compute maximum number of VGPRs function can use using default/requested
500   // minimum number of waves per execution unit.
501   std::pair<unsigned, unsigned> WavesPerEU = MFI.getWavesPerEU();
502   unsigned MaxNumVGPRs = getMaxNumVGPRs(WavesPerEU.first);
503 
504   // Check if maximum number of VGPRs was explicitly requested using
505   // "amdgpu-num-vgpr" attribute.
506   if (F.hasFnAttribute("amdgpu-num-vgpr")) {
507     unsigned Requested = AMDGPU::getIntegerAttribute(
508       F, "amdgpu-num-vgpr", MaxNumVGPRs);
509 
510     // Make sure requested value does not violate subtarget's specifications.
511     if (Requested && Requested <= getReservedNumVGPRs(MF))
512       Requested = 0;
513 
514     // Make sure requested value is compatible with values implied by
515     // default/requested minimum/maximum number of waves per execution unit.
516     if (Requested && Requested > getMaxNumVGPRs(WavesPerEU.first))
517       Requested = 0;
518     if (WavesPerEU.second &&
519         Requested && Requested < getMinNumVGPRs(WavesPerEU.second))
520       Requested = 0;
521 
522     if (Requested)
523       MaxNumVGPRs = Requested;
524   }
525 
526   return MaxNumVGPRs - getReservedNumVGPRs(MF);
527 }
528 
529 struct MemOpClusterMutation : ScheduleDAGMutation {
530   const SIInstrInfo *TII;
531 
532   MemOpClusterMutation(const SIInstrInfo *tii) : TII(tii) {}
533 
534   void apply(ScheduleDAGInstrs *DAGInstrs) override {
535     ScheduleDAGMI *DAG = static_cast<ScheduleDAGMI*>(DAGInstrs);
536 
537     SUnit *SUa = nullptr;
538     // Search for two consequent memory operations and link them
539     // to prevent scheduler from moving them apart.
540     // In DAG pre-process SUnits are in the original order of
541     // the instructions before scheduling.
542     for (SUnit &SU : DAG->SUnits) {
543       MachineInstr &MI2 = *SU.getInstr();
544       if (!MI2.mayLoad() && !MI2.mayStore()) {
545         SUa = nullptr;
546         continue;
547       }
548       if (!SUa) {
549         SUa = &SU;
550         continue;
551       }
552 
553       MachineInstr &MI1 = *SUa->getInstr();
554       if ((TII->isVMEM(MI1) && TII->isVMEM(MI2)) ||
555           (TII->isFLAT(MI1) && TII->isFLAT(MI2)) ||
556           (TII->isSMRD(MI1) && TII->isSMRD(MI2)) ||
557           (TII->isDS(MI1)   && TII->isDS(MI2))) {
558         SU.addPredBarrier(SUa);
559 
560         for (const SDep &SI : SU.Preds) {
561           if (SI.getSUnit() != SUa)
562             SUa->addPred(SDep(SI.getSUnit(), SDep::Artificial));
563         }
564 
565         if (&SU != &DAG->ExitSU) {
566           for (const SDep &SI : SUa->Succs) {
567             if (SI.getSUnit() != &SU)
568               SI.getSUnit()->addPred(SDep(&SU, SDep::Artificial));
569           }
570         }
571       }
572 
573       SUa = &SU;
574     }
575   }
576 };
577 
578 void SISubtarget::getPostRAMutations(
579     std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations) const {
580   Mutations.push_back(llvm::make_unique<MemOpClusterMutation>(&InstrInfo));
581 }
582