1 //==- SIMachineFunctionInfo.h - SIMachineFunctionInfo interface --*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #ifndef LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H
14 #define LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H
15 
16 #include "AMDGPUArgumentUsageInfo.h"
17 #include "AMDGPUMachineFunction.h"
18 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
19 #include "SIInstrInfo.h"
20 #include "llvm/ADT/SetVector.h"
21 #include "llvm/CodeGen/MIRYamlMapping.h"
22 #include "llvm/CodeGen/PseudoSourceValue.h"
23 #include "llvm/Support/raw_ostream.h"
24 
25 namespace llvm {
26 
27 class MachineFrameInfo;
28 class MachineFunction;
29 class SIMachineFunctionInfo;
30 class SIRegisterInfo;
31 class TargetRegisterClass;
32 
33 class AMDGPUPseudoSourceValue : public PseudoSourceValue {
34 public:
35   enum AMDGPUPSVKind : unsigned {
36     PSVBuffer = PseudoSourceValue::TargetCustom,
37     PSVImage,
38     GWSResource
39   };
40 
41 protected:
42   AMDGPUPseudoSourceValue(unsigned Kind, const TargetInstrInfo &TII)
43       : PseudoSourceValue(Kind, TII) {}
44 
45 public:
46   bool isConstant(const MachineFrameInfo *) const override {
47     // This should probably be true for most images, but we will start by being
48     // conservative.
49     return false;
50   }
51 
52   bool isAliased(const MachineFrameInfo *) const override {
53     return true;
54   }
55 
56   bool mayAlias(const MachineFrameInfo *) const override {
57     return true;
58   }
59 };
60 
61 class AMDGPUBufferPseudoSourceValue final : public AMDGPUPseudoSourceValue {
62 public:
63   explicit AMDGPUBufferPseudoSourceValue(const TargetInstrInfo &TII)
64       : AMDGPUPseudoSourceValue(PSVBuffer, TII) {}
65 
66   static bool classof(const PseudoSourceValue *V) {
67     return V->kind() == PSVBuffer;
68   }
69 
70   void printCustom(raw_ostream &OS) const override { OS << "BufferResource"; }
71 };
72 
73 class AMDGPUImagePseudoSourceValue final : public AMDGPUPseudoSourceValue {
74 public:
75   // TODO: Is the img rsrc useful?
76   explicit AMDGPUImagePseudoSourceValue(const TargetInstrInfo &TII)
77       : AMDGPUPseudoSourceValue(PSVImage, TII) {}
78 
79   static bool classof(const PseudoSourceValue *V) {
80     return V->kind() == PSVImage;
81   }
82 
83   void printCustom(raw_ostream &OS) const override { OS << "ImageResource"; }
84 };
85 
86 class AMDGPUGWSResourcePseudoSourceValue final : public AMDGPUPseudoSourceValue {
87 public:
88   explicit AMDGPUGWSResourcePseudoSourceValue(const TargetInstrInfo &TII)
89       : AMDGPUPseudoSourceValue(GWSResource, TII) {}
90 
91   static bool classof(const PseudoSourceValue *V) {
92     return V->kind() == GWSResource;
93   }
94 
95   // These are inaccessible memory from IR.
96   bool isAliased(const MachineFrameInfo *) const override {
97     return false;
98   }
99 
100   // These are inaccessible memory from IR.
101   bool mayAlias(const MachineFrameInfo *) const override {
102     return false;
103   }
104 
105   void printCustom(raw_ostream &OS) const override {
106     OS << "GWSResource";
107   }
108 };
109 
110 namespace yaml {
111 
112 struct SIArgument {
113   bool IsRegister;
114   union {
115     StringValue RegisterName;
116     unsigned StackOffset;
117   };
118   Optional<unsigned> Mask;
119 
120   // Default constructor, which creates a stack argument.
121   SIArgument() : IsRegister(false), StackOffset(0) {}
122   SIArgument(const SIArgument &Other) {
123     IsRegister = Other.IsRegister;
124     if (IsRegister) {
125       ::new ((void *)std::addressof(RegisterName))
126           StringValue(Other.RegisterName);
127     } else
128       StackOffset = Other.StackOffset;
129     Mask = Other.Mask;
130   }
131   SIArgument &operator=(const SIArgument &Other) {
132     IsRegister = Other.IsRegister;
133     if (IsRegister) {
134       ::new ((void *)std::addressof(RegisterName))
135           StringValue(Other.RegisterName);
136     } else
137       StackOffset = Other.StackOffset;
138     Mask = Other.Mask;
139     return *this;
140   }
141   ~SIArgument() {
142     if (IsRegister)
143       RegisterName.~StringValue();
144   }
145 
146   // Helper to create a register or stack argument.
147   static inline SIArgument createArgument(bool IsReg) {
148     if (IsReg)
149       return SIArgument(IsReg);
150     return SIArgument();
151   }
152 
153 private:
154   // Construct a register argument.
155   SIArgument(bool) : IsRegister(true), RegisterName() {}
156 };
157 
158 template <> struct MappingTraits<SIArgument> {
159   static void mapping(IO &YamlIO, SIArgument &A) {
160     if (YamlIO.outputting()) {
161       if (A.IsRegister)
162         YamlIO.mapRequired("reg", A.RegisterName);
163       else
164         YamlIO.mapRequired("offset", A.StackOffset);
165     } else {
166       auto Keys = YamlIO.keys();
167       if (is_contained(Keys, "reg")) {
168         A = SIArgument::createArgument(true);
169         YamlIO.mapRequired("reg", A.RegisterName);
170       } else if (is_contained(Keys, "offset"))
171         YamlIO.mapRequired("offset", A.StackOffset);
172       else
173         YamlIO.setError("missing required key 'reg' or 'offset'");
174     }
175     YamlIO.mapOptional("mask", A.Mask);
176   }
177   static const bool flow = true;
178 };
179 
180 struct SIArgumentInfo {
181   Optional<SIArgument> PrivateSegmentBuffer;
182   Optional<SIArgument> DispatchPtr;
183   Optional<SIArgument> QueuePtr;
184   Optional<SIArgument> KernargSegmentPtr;
185   Optional<SIArgument> DispatchID;
186   Optional<SIArgument> FlatScratchInit;
187   Optional<SIArgument> PrivateSegmentSize;
188 
189   Optional<SIArgument> WorkGroupIDX;
190   Optional<SIArgument> WorkGroupIDY;
191   Optional<SIArgument> WorkGroupIDZ;
192   Optional<SIArgument> WorkGroupInfo;
193   Optional<SIArgument> PrivateSegmentWaveByteOffset;
194 
195   Optional<SIArgument> ImplicitArgPtr;
196   Optional<SIArgument> ImplicitBufferPtr;
197 
198   Optional<SIArgument> WorkItemIDX;
199   Optional<SIArgument> WorkItemIDY;
200   Optional<SIArgument> WorkItemIDZ;
201 };
202 
203 template <> struct MappingTraits<SIArgumentInfo> {
204   static void mapping(IO &YamlIO, SIArgumentInfo &AI) {
205     YamlIO.mapOptional("privateSegmentBuffer", AI.PrivateSegmentBuffer);
206     YamlIO.mapOptional("dispatchPtr", AI.DispatchPtr);
207     YamlIO.mapOptional("queuePtr", AI.QueuePtr);
208     YamlIO.mapOptional("kernargSegmentPtr", AI.KernargSegmentPtr);
209     YamlIO.mapOptional("dispatchID", AI.DispatchID);
210     YamlIO.mapOptional("flatScratchInit", AI.FlatScratchInit);
211     YamlIO.mapOptional("privateSegmentSize", AI.PrivateSegmentSize);
212 
213     YamlIO.mapOptional("workGroupIDX", AI.WorkGroupIDX);
214     YamlIO.mapOptional("workGroupIDY", AI.WorkGroupIDY);
215     YamlIO.mapOptional("workGroupIDZ", AI.WorkGroupIDZ);
216     YamlIO.mapOptional("workGroupInfo", AI.WorkGroupInfo);
217     YamlIO.mapOptional("privateSegmentWaveByteOffset",
218                        AI.PrivateSegmentWaveByteOffset);
219 
220     YamlIO.mapOptional("implicitArgPtr", AI.ImplicitArgPtr);
221     YamlIO.mapOptional("implicitBufferPtr", AI.ImplicitBufferPtr);
222 
223     YamlIO.mapOptional("workItemIDX", AI.WorkItemIDX);
224     YamlIO.mapOptional("workItemIDY", AI.WorkItemIDY);
225     YamlIO.mapOptional("workItemIDZ", AI.WorkItemIDZ);
226   }
227 };
228 
229 // Default to default mode for default calling convention.
230 struct SIMode {
231   bool IEEE = true;
232   bool DX10Clamp = true;
233   bool FP32InputDenormals = true;
234   bool FP32OutputDenormals = true;
235   bool FP64FP16InputDenormals = true;
236   bool FP64FP16OutputDenormals = true;
237 
238   SIMode() = default;
239 
240   SIMode(const AMDGPU::SIModeRegisterDefaults &Mode) {
241     IEEE = Mode.IEEE;
242     DX10Clamp = Mode.DX10Clamp;
243     FP32InputDenormals = Mode.FP32InputDenormals;
244     FP32OutputDenormals = Mode.FP32OutputDenormals;
245     FP64FP16InputDenormals = Mode.FP64FP16InputDenormals;
246     FP64FP16OutputDenormals = Mode.FP64FP16OutputDenormals;
247   }
248 
249   bool operator ==(const SIMode Other) const {
250     return IEEE == Other.IEEE &&
251            DX10Clamp == Other.DX10Clamp &&
252            FP32InputDenormals == Other.FP32InputDenormals &&
253            FP32OutputDenormals == Other.FP32OutputDenormals &&
254            FP64FP16InputDenormals == Other.FP64FP16InputDenormals &&
255            FP64FP16OutputDenormals == Other.FP64FP16OutputDenormals;
256   }
257 };
258 
259 template <> struct MappingTraits<SIMode> {
260   static void mapping(IO &YamlIO, SIMode &Mode) {
261     YamlIO.mapOptional("ieee", Mode.IEEE, true);
262     YamlIO.mapOptional("dx10-clamp", Mode.DX10Clamp, true);
263     YamlIO.mapOptional("fp32-input-denormals", Mode.FP32InputDenormals, true);
264     YamlIO.mapOptional("fp32-output-denormals", Mode.FP32OutputDenormals, true);
265     YamlIO.mapOptional("fp64-fp16-input-denormals", Mode.FP64FP16InputDenormals, true);
266     YamlIO.mapOptional("fp64-fp16-output-denormals", Mode.FP64FP16OutputDenormals, true);
267   }
268 };
269 
270 struct SIMachineFunctionInfo final : public yaml::MachineFunctionInfo {
271   uint64_t ExplicitKernArgSize = 0;
272   unsigned MaxKernArgAlign = 0;
273   uint32_t LDSSize = 0;
274   uint32_t GDSSize = 0;
275   Align DynLDSAlign;
276   bool IsEntryFunction = false;
277   bool NoSignedZerosFPMath = false;
278   bool MemoryBound = false;
279   bool WaveLimiter = false;
280   bool HasSpilledSGPRs = false;
281   bool HasSpilledVGPRs = false;
282   uint32_t HighBitsOf32BitAddress = 0;
283 
284   // TODO: 10 may be a better default since it's the maximum.
285   unsigned Occupancy = 0;
286 
287   SmallVector<StringValue> WWMReservedRegs;
288 
289   StringValue ScratchRSrcReg = "$private_rsrc_reg";
290   StringValue FrameOffsetReg = "$fp_reg";
291   StringValue StackPtrOffsetReg = "$sp_reg";
292 
293   unsigned BytesInStackArgArea = 0;
294   bool ReturnsVoid = true;
295 
296   Optional<SIArgumentInfo> ArgInfo;
297   SIMode Mode;
298   Optional<FrameIndex> ScavengeFI;
299   StringValue VGPRForAGPRCopy;
300 
301   SIMachineFunctionInfo() = default;
302   SIMachineFunctionInfo(const llvm::SIMachineFunctionInfo &,
303                         const TargetRegisterInfo &TRI,
304                         const llvm::MachineFunction &MF);
305 
306   void mappingImpl(yaml::IO &YamlIO) override;
307   ~SIMachineFunctionInfo() = default;
308 };
309 
310 template <> struct MappingTraits<SIMachineFunctionInfo> {
311   static void mapping(IO &YamlIO, SIMachineFunctionInfo &MFI) {
312     YamlIO.mapOptional("explicitKernArgSize", MFI.ExplicitKernArgSize,
313                        UINT64_C(0));
314     YamlIO.mapOptional("maxKernArgAlign", MFI.MaxKernArgAlign, 0u);
315     YamlIO.mapOptional("ldsSize", MFI.LDSSize, 0u);
316     YamlIO.mapOptional("gdsSize", MFI.GDSSize, 0u);
317     YamlIO.mapOptional("dynLDSAlign", MFI.DynLDSAlign, Align());
318     YamlIO.mapOptional("isEntryFunction", MFI.IsEntryFunction, false);
319     YamlIO.mapOptional("noSignedZerosFPMath", MFI.NoSignedZerosFPMath, false);
320     YamlIO.mapOptional("memoryBound", MFI.MemoryBound, false);
321     YamlIO.mapOptional("waveLimiter", MFI.WaveLimiter, false);
322     YamlIO.mapOptional("hasSpilledSGPRs", MFI.HasSpilledSGPRs, false);
323     YamlIO.mapOptional("hasSpilledVGPRs", MFI.HasSpilledVGPRs, false);
324     YamlIO.mapOptional("scratchRSrcReg", MFI.ScratchRSrcReg,
325                        StringValue("$private_rsrc_reg"));
326     YamlIO.mapOptional("frameOffsetReg", MFI.FrameOffsetReg,
327                        StringValue("$fp_reg"));
328     YamlIO.mapOptional("stackPtrOffsetReg", MFI.StackPtrOffsetReg,
329                        StringValue("$sp_reg"));
330     YamlIO.mapOptional("bytesInStackArgArea", MFI.BytesInStackArgArea, 0u);
331     YamlIO.mapOptional("returnsVoid", MFI.ReturnsVoid, true);
332     YamlIO.mapOptional("argumentInfo", MFI.ArgInfo);
333     YamlIO.mapOptional("mode", MFI.Mode, SIMode());
334     YamlIO.mapOptional("highBitsOf32BitAddress",
335                        MFI.HighBitsOf32BitAddress, 0u);
336     YamlIO.mapOptional("occupancy", MFI.Occupancy, 0);
337     YamlIO.mapOptional("wwmReservedRegs", MFI.WWMReservedRegs);
338     YamlIO.mapOptional("scavengeFI", MFI.ScavengeFI);
339     YamlIO.mapOptional("vgprForAGPRCopy", MFI.VGPRForAGPRCopy,
340                        StringValue()); // Don't print out when it's empty.
341   }
342 };
343 
344 } // end namespace yaml
345 
346 /// This class keeps track of the SPI_SP_INPUT_ADDR config register, which
347 /// tells the hardware which interpolation parameters to load.
348 class SIMachineFunctionInfo final : public AMDGPUMachineFunction {
349   friend class GCNTargetMachine;
350 
351   // Registers that may be reserved for spilling purposes. These may be the same
352   // as the input registers.
353   Register ScratchRSrcReg = AMDGPU::PRIVATE_RSRC_REG;
354 
355   // This is the the unswizzled offset from the current dispatch's scratch wave
356   // base to the beginning of the current function's frame.
357   Register FrameOffsetReg = AMDGPU::FP_REG;
358 
359   // This is an ABI register used in the non-entry calling convention to
360   // communicate the unswizzled offset from the current dispatch's scratch wave
361   // base to the beginning of the new function's frame.
362   Register StackPtrOffsetReg = AMDGPU::SP_REG;
363 
364   AMDGPUFunctionArgInfo ArgInfo;
365 
366   // Graphics info.
367   unsigned PSInputAddr = 0;
368   unsigned PSInputEnable = 0;
369 
370   /// Number of bytes of arguments this function has on the stack. If the callee
371   /// is expected to restore the argument stack this should be a multiple of 16,
372   /// all usable during a tail call.
373   ///
374   /// The alternative would forbid tail call optimisation in some cases: if we
375   /// want to transfer control from a function with 8-bytes of stack-argument
376   /// space to a function with 16-bytes then misalignment of this value would
377   /// make a stack adjustment necessary, which could not be undone by the
378   /// callee.
379   unsigned BytesInStackArgArea = 0;
380 
381   bool ReturnsVoid = true;
382 
383   // A pair of default/requested minimum/maximum flat work group sizes.
384   // Minimum - first, maximum - second.
385   std::pair<unsigned, unsigned> FlatWorkGroupSizes = {0, 0};
386 
387   // A pair of default/requested minimum/maximum number of waves per execution
388   // unit. Minimum - first, maximum - second.
389   std::pair<unsigned, unsigned> WavesPerEU = {0, 0};
390 
391   std::unique_ptr<const AMDGPUBufferPseudoSourceValue> BufferPSV;
392   std::unique_ptr<const AMDGPUImagePseudoSourceValue> ImagePSV;
393   std::unique_ptr<const AMDGPUGWSResourcePseudoSourceValue> GWSResourcePSV;
394 
395 private:
396   unsigned NumUserSGPRs = 0;
397   unsigned NumSystemSGPRs = 0;
398 
399   bool HasSpilledSGPRs = false;
400   bool HasSpilledVGPRs = false;
401   bool HasNonSpillStackObjects = false;
402   bool IsStackRealigned = false;
403 
404   unsigned NumSpilledSGPRs = 0;
405   unsigned NumSpilledVGPRs = 0;
406 
407   // Feature bits required for inputs passed in user SGPRs.
408   bool PrivateSegmentBuffer : 1;
409   bool DispatchPtr : 1;
410   bool QueuePtr : 1;
411   bool KernargSegmentPtr : 1;
412   bool DispatchID : 1;
413   bool FlatScratchInit : 1;
414 
415   // Feature bits required for inputs passed in system SGPRs.
416   bool WorkGroupIDX : 1; // Always initialized.
417   bool WorkGroupIDY : 1;
418   bool WorkGroupIDZ : 1;
419   bool WorkGroupInfo : 1;
420   bool PrivateSegmentWaveByteOffset : 1;
421 
422   bool WorkItemIDX : 1; // Always initialized.
423   bool WorkItemIDY : 1;
424   bool WorkItemIDZ : 1;
425 
426   // Private memory buffer
427   // Compute directly in sgpr[0:1]
428   // Other shaders indirect 64-bits at sgpr[0:1]
429   bool ImplicitBufferPtr : 1;
430 
431   // Pointer to where the ABI inserts special kernel arguments separate from the
432   // user arguments. This is an offset from the KernargSegmentPtr.
433   bool ImplicitArgPtr : 1;
434 
435   bool MayNeedAGPRs : 1;
436 
437   // The hard-wired high half of the address of the global information table
438   // for AMDPAL OS type. 0xffffffff represents no hard-wired high half, since
439   // current hardware only allows a 16 bit value.
440   unsigned GITPtrHigh;
441 
442   unsigned HighBitsOf32BitAddress;
443 
444   // Current recorded maximum possible occupancy.
445   unsigned Occupancy;
446 
447   mutable Optional<bool> UsesAGPRs;
448 
449   MCPhysReg getNextUserSGPR() const;
450 
451   MCPhysReg getNextSystemSGPR() const;
452 
453 public:
454   struct SpilledReg {
455     Register VGPR;
456     int Lane = -1;
457 
458     SpilledReg() = default;
459     SpilledReg(Register R, int L) : VGPR (R), Lane (L) {}
460 
461     bool hasLane() { return Lane != -1;}
462     bool hasReg() { return VGPR != 0;}
463   };
464 
465   struct SGPRSpillVGPR {
466     // VGPR used for SGPR spills
467     Register VGPR;
468 
469     // If the VGPR is is used for SGPR spills in a non-entrypoint function, the
470     // stack slot used to save/restore it in the prolog/epilog.
471     Optional<int> FI;
472 
473     SGPRSpillVGPR(Register V, Optional<int> F) : VGPR(V), FI(F) {}
474   };
475 
476   struct VGPRSpillToAGPR {
477     SmallVector<MCPhysReg, 32> Lanes;
478     bool FullyAllocated = false;
479     bool IsDead = false;
480   };
481 
482   // Track VGPRs reserved for WWM.
483   SmallSetVector<Register, 8> WWMReservedRegs;
484 
485   /// Track stack slots used for save/restore of reserved WWM VGPRs in the
486   /// prolog/epilog.
487 
488   /// FIXME: This is temporary state only needed in PrologEpilogInserter, and
489   /// doesn't really belong here. It does not require serialization
490   SmallVector<int, 8> WWMReservedFrameIndexes;
491 
492   void allocateWWMReservedSpillSlots(MachineFrameInfo &MFI,
493                                      const SIRegisterInfo &TRI);
494 
495   auto wwmAllocation() const {
496     assert(WWMReservedRegs.size() == WWMReservedFrameIndexes.size());
497     return zip(WWMReservedRegs, WWMReservedFrameIndexes);
498   }
499 
500 private:
501   // Track VGPR + wave index for each subregister of the SGPR spilled to
502   // frameindex key.
503   DenseMap<int, std::vector<SpilledReg>> SGPRToVGPRSpills;
504   unsigned NumVGPRSpillLanes = 0;
505   SmallVector<SGPRSpillVGPR, 2> SpillVGPRs;
506 
507   DenseMap<int, VGPRSpillToAGPR> VGPRToAGPRSpills;
508 
509   // AGPRs used for VGPR spills.
510   SmallVector<MCPhysReg, 32> SpillAGPR;
511 
512   // VGPRs used for AGPR spills.
513   SmallVector<MCPhysReg, 32> SpillVGPR;
514 
515   // Emergency stack slot. Sometimes, we create this before finalizing the stack
516   // frame, so save it here and add it to the RegScavenger later.
517   Optional<int> ScavengeFI;
518 
519 private:
520   Register VGPRForAGPRCopy;
521 
522 public:
523   Register getVGPRForAGPRCopy() const {
524     return VGPRForAGPRCopy;
525   }
526 
527   void setVGPRForAGPRCopy(Register NewVGPRForAGPRCopy) {
528     VGPRForAGPRCopy = NewVGPRForAGPRCopy;
529   }
530 
531 public: // FIXME
532   /// If this is set, an SGPR used for save/restore of the register used for the
533   /// frame pointer.
534   Register SGPRForFPSaveRestoreCopy;
535   Optional<int> FramePointerSaveIndex;
536 
537   /// If this is set, an SGPR used for save/restore of the register used for the
538   /// base pointer.
539   Register SGPRForBPSaveRestoreCopy;
540   Optional<int> BasePointerSaveIndex;
541 
542   bool isCalleeSavedReg(const MCPhysReg *CSRegs, MCPhysReg Reg);
543 
544 public:
545   SIMachineFunctionInfo(const MachineFunction &MF);
546 
547   bool initializeBaseYamlFields(const yaml::SIMachineFunctionInfo &YamlMFI,
548                                 const MachineFunction &MF,
549                                 PerFunctionMIParsingState &PFS,
550                                 SMDiagnostic &Error, SMRange &SourceRange);
551 
552   void reserveWWMRegister(Register Reg) {
553     WWMReservedRegs.insert(Reg);
554   }
555 
556   ArrayRef<SpilledReg> getSGPRToVGPRSpills(int FrameIndex) const {
557     auto I = SGPRToVGPRSpills.find(FrameIndex);
558     return (I == SGPRToVGPRSpills.end()) ?
559       ArrayRef<SpilledReg>() : makeArrayRef(I->second);
560   }
561 
562   ArrayRef<SGPRSpillVGPR> getSGPRSpillVGPRs() const { return SpillVGPRs; }
563 
564   ArrayRef<MCPhysReg> getAGPRSpillVGPRs() const {
565     return SpillAGPR;
566   }
567 
568   ArrayRef<MCPhysReg> getVGPRSpillAGPRs() const {
569     return SpillVGPR;
570   }
571 
572   MCPhysReg getVGPRToAGPRSpill(int FrameIndex, unsigned Lane) const {
573     auto I = VGPRToAGPRSpills.find(FrameIndex);
574     return (I == VGPRToAGPRSpills.end()) ? (MCPhysReg)AMDGPU::NoRegister
575                                          : I->second.Lanes[Lane];
576   }
577 
578   void setVGPRToAGPRSpillDead(int FrameIndex) {
579     auto I = VGPRToAGPRSpills.find(FrameIndex);
580     if (I != VGPRToAGPRSpills.end())
581       I->second.IsDead = true;
582   }
583 
584   bool haveFreeLanesForSGPRSpill(const MachineFunction &MF,
585                                  unsigned NumLane) const;
586   bool allocateSGPRSpillToVGPR(MachineFunction &MF, int FI);
587   bool allocateVGPRSpillToAGPR(MachineFunction &MF, int FI, bool isAGPRtoVGPR);
588 
589   /// If \p ResetSGPRSpillStackIDs is true, reset the stack ID from sgpr-spill
590   /// to the default stack.
591   bool removeDeadFrameIndices(MachineFrameInfo &MFI,
592                               bool ResetSGPRSpillStackIDs);
593 
594   int getScavengeFI(MachineFrameInfo &MFI, const SIRegisterInfo &TRI);
595   Optional<int> getOptionalScavengeFI() const { return ScavengeFI; }
596 
597   unsigned getBytesInStackArgArea() const {
598     return BytesInStackArgArea;
599   }
600 
601   void setBytesInStackArgArea(unsigned Bytes) {
602     BytesInStackArgArea = Bytes;
603   }
604 
605   // Add user SGPRs.
606   Register addPrivateSegmentBuffer(const SIRegisterInfo &TRI);
607   Register addDispatchPtr(const SIRegisterInfo &TRI);
608   Register addQueuePtr(const SIRegisterInfo &TRI);
609   Register addKernargSegmentPtr(const SIRegisterInfo &TRI);
610   Register addDispatchID(const SIRegisterInfo &TRI);
611   Register addFlatScratchInit(const SIRegisterInfo &TRI);
612   Register addImplicitBufferPtr(const SIRegisterInfo &TRI);
613 
614   // Add system SGPRs.
615   Register addWorkGroupIDX() {
616     ArgInfo.WorkGroupIDX = ArgDescriptor::createRegister(getNextSystemSGPR());
617     NumSystemSGPRs += 1;
618     return ArgInfo.WorkGroupIDX.getRegister();
619   }
620 
621   Register addWorkGroupIDY() {
622     ArgInfo.WorkGroupIDY = ArgDescriptor::createRegister(getNextSystemSGPR());
623     NumSystemSGPRs += 1;
624     return ArgInfo.WorkGroupIDY.getRegister();
625   }
626 
627   Register addWorkGroupIDZ() {
628     ArgInfo.WorkGroupIDZ = ArgDescriptor::createRegister(getNextSystemSGPR());
629     NumSystemSGPRs += 1;
630     return ArgInfo.WorkGroupIDZ.getRegister();
631   }
632 
633   Register addWorkGroupInfo() {
634     ArgInfo.WorkGroupInfo = ArgDescriptor::createRegister(getNextSystemSGPR());
635     NumSystemSGPRs += 1;
636     return ArgInfo.WorkGroupInfo.getRegister();
637   }
638 
639   // Add special VGPR inputs
640   void setWorkItemIDX(ArgDescriptor Arg) {
641     ArgInfo.WorkItemIDX = Arg;
642   }
643 
644   void setWorkItemIDY(ArgDescriptor Arg) {
645     ArgInfo.WorkItemIDY = Arg;
646   }
647 
648   void setWorkItemIDZ(ArgDescriptor Arg) {
649     ArgInfo.WorkItemIDZ = Arg;
650   }
651 
652   Register addPrivateSegmentWaveByteOffset() {
653     ArgInfo.PrivateSegmentWaveByteOffset
654       = ArgDescriptor::createRegister(getNextSystemSGPR());
655     NumSystemSGPRs += 1;
656     return ArgInfo.PrivateSegmentWaveByteOffset.getRegister();
657   }
658 
659   void setPrivateSegmentWaveByteOffset(Register Reg) {
660     ArgInfo.PrivateSegmentWaveByteOffset = ArgDescriptor::createRegister(Reg);
661   }
662 
663   bool hasPrivateSegmentBuffer() const {
664     return PrivateSegmentBuffer;
665   }
666 
667   bool hasDispatchPtr() const {
668     return DispatchPtr;
669   }
670 
671   bool hasQueuePtr() const {
672     return QueuePtr;
673   }
674 
675   bool hasKernargSegmentPtr() const {
676     return KernargSegmentPtr;
677   }
678 
679   bool hasDispatchID() const {
680     return DispatchID;
681   }
682 
683   bool hasFlatScratchInit() const {
684     return FlatScratchInit;
685   }
686 
687   bool hasWorkGroupIDX() const {
688     return WorkGroupIDX;
689   }
690 
691   bool hasWorkGroupIDY() const {
692     return WorkGroupIDY;
693   }
694 
695   bool hasWorkGroupIDZ() const {
696     return WorkGroupIDZ;
697   }
698 
699   bool hasWorkGroupInfo() const {
700     return WorkGroupInfo;
701   }
702 
703   bool hasPrivateSegmentWaveByteOffset() const {
704     return PrivateSegmentWaveByteOffset;
705   }
706 
707   bool hasWorkItemIDX() const {
708     return WorkItemIDX;
709   }
710 
711   bool hasWorkItemIDY() const {
712     return WorkItemIDY;
713   }
714 
715   bool hasWorkItemIDZ() const {
716     return WorkItemIDZ;
717   }
718 
719   bool hasImplicitArgPtr() const {
720     return ImplicitArgPtr;
721   }
722 
723   bool hasImplicitBufferPtr() const {
724     return ImplicitBufferPtr;
725   }
726 
727   AMDGPUFunctionArgInfo &getArgInfo() {
728     return ArgInfo;
729   }
730 
731   const AMDGPUFunctionArgInfo &getArgInfo() const {
732     return ArgInfo;
733   }
734 
735   std::tuple<const ArgDescriptor *, const TargetRegisterClass *, LLT>
736   getPreloadedValue(AMDGPUFunctionArgInfo::PreloadedValue Value) const {
737     return ArgInfo.getPreloadedValue(Value);
738   }
739 
740   MCRegister getPreloadedReg(AMDGPUFunctionArgInfo::PreloadedValue Value) const {
741     auto Arg = std::get<0>(ArgInfo.getPreloadedValue(Value));
742     return Arg ? Arg->getRegister() : MCRegister();
743   }
744 
745   unsigned getGITPtrHigh() const {
746     return GITPtrHigh;
747   }
748 
749   Register getGITPtrLoReg(const MachineFunction &MF) const;
750 
751   uint32_t get32BitAddressHighBits() const {
752     return HighBitsOf32BitAddress;
753   }
754 
755   unsigned getNumUserSGPRs() const {
756     return NumUserSGPRs;
757   }
758 
759   unsigned getNumPreloadedSGPRs() const {
760     return NumUserSGPRs + NumSystemSGPRs;
761   }
762 
763   Register getPrivateSegmentWaveByteOffsetSystemSGPR() const {
764     return ArgInfo.PrivateSegmentWaveByteOffset.getRegister();
765   }
766 
767   /// Returns the physical register reserved for use as the resource
768   /// descriptor for scratch accesses.
769   Register getScratchRSrcReg() const {
770     return ScratchRSrcReg;
771   }
772 
773   void setScratchRSrcReg(Register Reg) {
774     assert(Reg != 0 && "Should never be unset");
775     ScratchRSrcReg = Reg;
776   }
777 
778   Register getFrameOffsetReg() const {
779     return FrameOffsetReg;
780   }
781 
782   void setFrameOffsetReg(Register Reg) {
783     assert(Reg != 0 && "Should never be unset");
784     FrameOffsetReg = Reg;
785   }
786 
787   void setStackPtrOffsetReg(Register Reg) {
788     assert(Reg != 0 && "Should never be unset");
789     StackPtrOffsetReg = Reg;
790   }
791 
792   // Note the unset value for this is AMDGPU::SP_REG rather than
793   // NoRegister. This is mostly a workaround for MIR tests where state that
794   // can't be directly computed from the function is not preserved in serialized
795   // MIR.
796   Register getStackPtrOffsetReg() const {
797     return StackPtrOffsetReg;
798   }
799 
800   Register getQueuePtrUserSGPR() const {
801     return ArgInfo.QueuePtr.getRegister();
802   }
803 
804   Register getImplicitBufferPtrUserSGPR() const {
805     return ArgInfo.ImplicitBufferPtr.getRegister();
806   }
807 
808   bool hasSpilledSGPRs() const {
809     return HasSpilledSGPRs;
810   }
811 
812   void setHasSpilledSGPRs(bool Spill = true) {
813     HasSpilledSGPRs = Spill;
814   }
815 
816   bool hasSpilledVGPRs() const {
817     return HasSpilledVGPRs;
818   }
819 
820   void setHasSpilledVGPRs(bool Spill = true) {
821     HasSpilledVGPRs = Spill;
822   }
823 
824   bool hasNonSpillStackObjects() const {
825     return HasNonSpillStackObjects;
826   }
827 
828   void setHasNonSpillStackObjects(bool StackObject = true) {
829     HasNonSpillStackObjects = StackObject;
830   }
831 
832   bool isStackRealigned() const {
833     return IsStackRealigned;
834   }
835 
836   void setIsStackRealigned(bool Realigned = true) {
837     IsStackRealigned = Realigned;
838   }
839 
840   unsigned getNumSpilledSGPRs() const {
841     return NumSpilledSGPRs;
842   }
843 
844   unsigned getNumSpilledVGPRs() const {
845     return NumSpilledVGPRs;
846   }
847 
848   void addToSpilledSGPRs(unsigned num) {
849     NumSpilledSGPRs += num;
850   }
851 
852   void addToSpilledVGPRs(unsigned num) {
853     NumSpilledVGPRs += num;
854   }
855 
856   unsigned getPSInputAddr() const {
857     return PSInputAddr;
858   }
859 
860   unsigned getPSInputEnable() const {
861     return PSInputEnable;
862   }
863 
864   bool isPSInputAllocated(unsigned Index) const {
865     return PSInputAddr & (1 << Index);
866   }
867 
868   void markPSInputAllocated(unsigned Index) {
869     PSInputAddr |= 1 << Index;
870   }
871 
872   void markPSInputEnabled(unsigned Index) {
873     PSInputEnable |= 1 << Index;
874   }
875 
876   bool returnsVoid() const {
877     return ReturnsVoid;
878   }
879 
880   void setIfReturnsVoid(bool Value) {
881     ReturnsVoid = Value;
882   }
883 
884   /// \returns A pair of default/requested minimum/maximum flat work group sizes
885   /// for this function.
886   std::pair<unsigned, unsigned> getFlatWorkGroupSizes() const {
887     return FlatWorkGroupSizes;
888   }
889 
890   /// \returns Default/requested minimum flat work group size for this function.
891   unsigned getMinFlatWorkGroupSize() const {
892     return FlatWorkGroupSizes.first;
893   }
894 
895   /// \returns Default/requested maximum flat work group size for this function.
896   unsigned getMaxFlatWorkGroupSize() const {
897     return FlatWorkGroupSizes.second;
898   }
899 
900   /// \returns A pair of default/requested minimum/maximum number of waves per
901   /// execution unit.
902   std::pair<unsigned, unsigned> getWavesPerEU() const {
903     return WavesPerEU;
904   }
905 
906   /// \returns Default/requested minimum number of waves per execution unit.
907   unsigned getMinWavesPerEU() const {
908     return WavesPerEU.first;
909   }
910 
911   /// \returns Default/requested maximum number of waves per execution unit.
912   unsigned getMaxWavesPerEU() const {
913     return WavesPerEU.second;
914   }
915 
916   /// \returns SGPR used for \p Dim's work group ID.
917   Register getWorkGroupIDSGPR(unsigned Dim) const {
918     switch (Dim) {
919     case 0:
920       assert(hasWorkGroupIDX());
921       return ArgInfo.WorkGroupIDX.getRegister();
922     case 1:
923       assert(hasWorkGroupIDY());
924       return ArgInfo.WorkGroupIDY.getRegister();
925     case 2:
926       assert(hasWorkGroupIDZ());
927       return ArgInfo.WorkGroupIDZ.getRegister();
928     }
929     llvm_unreachable("unexpected dimension");
930   }
931 
932   const AMDGPUBufferPseudoSourceValue *getBufferPSV(const SIInstrInfo &TII) {
933     if (!BufferPSV)
934       BufferPSV = std::make_unique<AMDGPUBufferPseudoSourceValue>(TII);
935 
936     return BufferPSV.get();
937   }
938 
939   const AMDGPUImagePseudoSourceValue *getImagePSV(const SIInstrInfo &TII) {
940     if (!ImagePSV)
941       ImagePSV = std::make_unique<AMDGPUImagePseudoSourceValue>(TII);
942 
943     return ImagePSV.get();
944   }
945 
946   const AMDGPUGWSResourcePseudoSourceValue *getGWSPSV(const SIInstrInfo &TII) {
947     if (!GWSResourcePSV) {
948       GWSResourcePSV =
949           std::make_unique<AMDGPUGWSResourcePseudoSourceValue>(TII);
950     }
951 
952     return GWSResourcePSV.get();
953   }
954 
955   unsigned getOccupancy() const {
956     return Occupancy;
957   }
958 
959   unsigned getMinAllowedOccupancy() const {
960     if (!isMemoryBound() && !needsWaveLimiter())
961       return Occupancy;
962     return (Occupancy < 4) ? Occupancy : 4;
963   }
964 
965   void limitOccupancy(const MachineFunction &MF);
966 
967   void limitOccupancy(unsigned Limit) {
968     if (Occupancy > Limit)
969       Occupancy = Limit;
970   }
971 
972   void increaseOccupancy(const MachineFunction &MF, unsigned Limit) {
973     if (Occupancy < Limit)
974       Occupancy = Limit;
975     limitOccupancy(MF);
976   }
977 
978   bool mayNeedAGPRs() const {
979     return MayNeedAGPRs;
980   }
981 
982   // \returns true if a function has a use of AGPRs via inline asm or
983   // has a call which may use it.
984   bool mayUseAGPRs(const MachineFunction &MF) const;
985 
986   // \returns true if a function needs or may need AGPRs.
987   bool usesAGPRs(const MachineFunction &MF) const;
988 };
989 
990 } // end namespace llvm
991 
992 #endif // LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H
993