1 //===- SIMachineFunctionInfo.cpp - SI Machine Function Info ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "SIMachineFunctionInfo.h"
10 #include "AMDGPUArgumentUsageInfo.h"
11 #include "AMDGPUSubtarget.h"
12 #include "SIRegisterInfo.h"
13 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "llvm/ADT/Optional.h"
16 #include "llvm/CodeGen/MachineBasicBlock.h"
17 #include "llvm/CodeGen/MachineFrameInfo.h"
18 #include "llvm/CodeGen/MachineFunction.h"
19 #include "llvm/CodeGen/MachineRegisterInfo.h"
20 #include "llvm/IR/CallingConv.h"
21 #include "llvm/IR/Function.h"
22 #include <cassert>
23 #include <vector>
24 
25 #define MAX_LANES 64
26 
27 using namespace llvm;
28 
29 SIMachineFunctionInfo::SIMachineFunctionInfo(const MachineFunction &MF)
30   : AMDGPUMachineFunction(MF),
31     Mode(MF.getFunction()),
32     PrivateSegmentBuffer(false),
33     DispatchPtr(false),
34     QueuePtr(false),
35     KernargSegmentPtr(false),
36     DispatchID(false),
37     FlatScratchInit(false),
38     WorkGroupIDX(false),
39     WorkGroupIDY(false),
40     WorkGroupIDZ(false),
41     WorkGroupInfo(false),
42     PrivateSegmentWaveByteOffset(false),
43     WorkItemIDX(false),
44     WorkItemIDY(false),
45     WorkItemIDZ(false),
46     ImplicitBufferPtr(false),
47     ImplicitArgPtr(false),
48     GITPtrHigh(0xffffffff),
49     HighBitsOf32BitAddress(0),
50     GDSSize(0) {
51   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
52   const Function &F = MF.getFunction();
53   FlatWorkGroupSizes = ST.getFlatWorkGroupSizes(F);
54   WavesPerEU = ST.getWavesPerEU(F);
55 
56   Occupancy = getMaxWavesPerEU();
57   limitOccupancy(MF);
58   CallingConv::ID CC = F.getCallingConv();
59 
60   if (CC == CallingConv::AMDGPU_KERNEL || CC == CallingConv::SPIR_KERNEL) {
61     if (!F.arg_empty())
62       KernargSegmentPtr = true;
63     WorkGroupIDX = true;
64     WorkItemIDX = true;
65   } else if (CC == CallingConv::AMDGPU_PS) {
66     PSInputAddr = AMDGPU::getInitialPSInputAddr(F);
67   }
68 
69   if (!isEntryFunction()) {
70     // Non-entry functions have no special inputs for now, other registers
71     // required for scratch access.
72     ScratchRSrcReg = AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3;
73     ScratchWaveOffsetReg = AMDGPU::SGPR33;
74 
75     // TODO: Pick a high register, and shift down, similar to a kernel.
76     FrameOffsetReg = AMDGPU::SGPR34;
77     StackPtrOffsetReg = AMDGPU::SGPR32;
78 
79     ArgInfo.PrivateSegmentBuffer =
80       ArgDescriptor::createRegister(ScratchRSrcReg);
81     ArgInfo.PrivateSegmentWaveByteOffset =
82       ArgDescriptor::createRegister(ScratchWaveOffsetReg);
83 
84     if (F.hasFnAttribute("amdgpu-implicitarg-ptr"))
85       ImplicitArgPtr = true;
86   } else {
87     if (F.hasFnAttribute("amdgpu-implicitarg-ptr")) {
88       KernargSegmentPtr = true;
89       MaxKernArgAlign = std::max(ST.getAlignmentForImplicitArgPtr(),
90                                  MaxKernArgAlign);
91     }
92   }
93 
94   if (F.hasFnAttribute("amdgpu-work-group-id-x"))
95     WorkGroupIDX = true;
96 
97   if (F.hasFnAttribute("amdgpu-work-group-id-y"))
98     WorkGroupIDY = true;
99 
100   if (F.hasFnAttribute("amdgpu-work-group-id-z"))
101     WorkGroupIDZ = true;
102 
103   if (F.hasFnAttribute("amdgpu-work-item-id-x"))
104     WorkItemIDX = true;
105 
106   if (F.hasFnAttribute("amdgpu-work-item-id-y"))
107     WorkItemIDY = true;
108 
109   if (F.hasFnAttribute("amdgpu-work-item-id-z"))
110     WorkItemIDZ = true;
111 
112   const MachineFrameInfo &FrameInfo = MF.getFrameInfo();
113   bool HasStackObjects = FrameInfo.hasStackObjects();
114 
115   if (isEntryFunction()) {
116     // X, XY, and XYZ are the only supported combinations, so make sure Y is
117     // enabled if Z is.
118     if (WorkItemIDZ)
119       WorkItemIDY = true;
120 
121     PrivateSegmentWaveByteOffset = true;
122 
123     // HS and GS always have the scratch wave offset in SGPR5 on GFX9.
124     if (ST.getGeneration() >= AMDGPUSubtarget::GFX9 &&
125         (CC == CallingConv::AMDGPU_HS || CC == CallingConv::AMDGPU_GS))
126       ArgInfo.PrivateSegmentWaveByteOffset =
127           ArgDescriptor::createRegister(AMDGPU::SGPR5);
128   }
129 
130   bool isAmdHsaOrMesa = ST.isAmdHsaOrMesa(F);
131   if (isAmdHsaOrMesa) {
132     PrivateSegmentBuffer = true;
133 
134     if (F.hasFnAttribute("amdgpu-dispatch-ptr"))
135       DispatchPtr = true;
136 
137     if (F.hasFnAttribute("amdgpu-queue-ptr"))
138       QueuePtr = true;
139 
140     if (F.hasFnAttribute("amdgpu-dispatch-id"))
141       DispatchID = true;
142   } else if (ST.isMesaGfxShader(F)) {
143     ImplicitBufferPtr = true;
144   }
145 
146   if (F.hasFnAttribute("amdgpu-kernarg-segment-ptr"))
147     KernargSegmentPtr = true;
148 
149   if (ST.hasFlatAddressSpace() && isEntryFunction() && isAmdHsaOrMesa) {
150     auto hasNonSpillStackObjects = [&]() {
151       // Avoid expensive checking if there's no stack objects.
152       if (!HasStackObjects)
153         return false;
154       for (auto OI = FrameInfo.getObjectIndexBegin(),
155                 OE = FrameInfo.getObjectIndexEnd(); OI != OE; ++OI)
156         if (!FrameInfo.isSpillSlotObjectIndex(OI))
157           return true;
158       // All stack objects are spill slots.
159       return false;
160     };
161     // TODO: This could be refined a lot. The attribute is a poor way of
162     // detecting calls that may require it before argument lowering.
163     if (hasNonSpillStackObjects() || F.hasFnAttribute("amdgpu-flat-scratch"))
164       FlatScratchInit = true;
165   }
166 
167   Attribute A = F.getFnAttribute("amdgpu-git-ptr-high");
168   StringRef S = A.getValueAsString();
169   if (!S.empty())
170     S.consumeInteger(0, GITPtrHigh);
171 
172   A = F.getFnAttribute("amdgpu-32bit-address-high-bits");
173   S = A.getValueAsString();
174   if (!S.empty())
175     S.consumeInteger(0, HighBitsOf32BitAddress);
176 
177   S = F.getFnAttribute("amdgpu-gds-size").getValueAsString();
178   if (!S.empty())
179     S.consumeInteger(0, GDSSize);
180 }
181 
182 void SIMachineFunctionInfo::limitOccupancy(const MachineFunction &MF) {
183   limitOccupancy(getMaxWavesPerEU());
184   const GCNSubtarget& ST = MF.getSubtarget<GCNSubtarget>();
185   limitOccupancy(ST.getOccupancyWithLocalMemSize(getLDSSize(),
186                  MF.getFunction()));
187 }
188 
189 unsigned SIMachineFunctionInfo::addPrivateSegmentBuffer(
190   const SIRegisterInfo &TRI) {
191   ArgInfo.PrivateSegmentBuffer =
192     ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
193     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_128RegClass));
194   NumUserSGPRs += 4;
195   return ArgInfo.PrivateSegmentBuffer.getRegister();
196 }
197 
198 unsigned SIMachineFunctionInfo::addDispatchPtr(const SIRegisterInfo &TRI) {
199   ArgInfo.DispatchPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
200     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
201   NumUserSGPRs += 2;
202   return ArgInfo.DispatchPtr.getRegister();
203 }
204 
205 unsigned SIMachineFunctionInfo::addQueuePtr(const SIRegisterInfo &TRI) {
206   ArgInfo.QueuePtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
207     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
208   NumUserSGPRs += 2;
209   return ArgInfo.QueuePtr.getRegister();
210 }
211 
212 unsigned SIMachineFunctionInfo::addKernargSegmentPtr(const SIRegisterInfo &TRI) {
213   ArgInfo.KernargSegmentPtr
214     = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
215     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
216   NumUserSGPRs += 2;
217   return ArgInfo.KernargSegmentPtr.getRegister();
218 }
219 
220 unsigned SIMachineFunctionInfo::addDispatchID(const SIRegisterInfo &TRI) {
221   ArgInfo.DispatchID = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
222     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
223   NumUserSGPRs += 2;
224   return ArgInfo.DispatchID.getRegister();
225 }
226 
227 unsigned SIMachineFunctionInfo::addFlatScratchInit(const SIRegisterInfo &TRI) {
228   ArgInfo.FlatScratchInit = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
229     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
230   NumUserSGPRs += 2;
231   return ArgInfo.FlatScratchInit.getRegister();
232 }
233 
234 unsigned SIMachineFunctionInfo::addImplicitBufferPtr(const SIRegisterInfo &TRI) {
235   ArgInfo.ImplicitBufferPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
236     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
237   NumUserSGPRs += 2;
238   return ArgInfo.ImplicitBufferPtr.getRegister();
239 }
240 
241 static bool isCalleeSavedReg(const MCPhysReg *CSRegs, MCPhysReg Reg) {
242   for (unsigned I = 0; CSRegs[I]; ++I) {
243     if (CSRegs[I] == Reg)
244       return true;
245   }
246 
247   return false;
248 }
249 
250 /// \p returns true if \p NumLanes slots are available in VGPRs already used for
251 /// SGPR spilling.
252 //
253 // FIXME: This only works after processFunctionBeforeFrameFinalized
254 bool SIMachineFunctionInfo::haveFreeLanesForSGPRSpill(const MachineFunction &MF,
255                                                       unsigned NumNeed) const {
256   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
257   unsigned WaveSize = ST.getWavefrontSize();
258   return NumVGPRSpillLanes + NumNeed <= WaveSize * SpillVGPRs.size();
259 }
260 
261 /// Reserve a slice of a VGPR to support spilling for FrameIndex \p FI.
262 bool SIMachineFunctionInfo::allocateSGPRSpillToVGPR(MachineFunction &MF,
263                                                     int FI) {
264   std::vector<SpilledReg> &SpillLanes = SGPRToVGPRSpills[FI];
265 
266   // This has already been allocated.
267   if (!SpillLanes.empty())
268     return true;
269 
270   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
271   const SIRegisterInfo *TRI = ST.getRegisterInfo();
272   MachineFrameInfo &FrameInfo = MF.getFrameInfo();
273   MachineRegisterInfo &MRI = MF.getRegInfo();
274   unsigned WaveSize = ST.getWavefrontSize();
275 
276   unsigned Size = FrameInfo.getObjectSize(FI);
277   assert(Size >= 4 && Size <= 64 && "invalid sgpr spill size");
278   assert(TRI->spillSGPRToVGPR() && "not spilling SGPRs to VGPRs");
279 
280   int NumLanes = Size / 4;
281 
282   const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs();
283 
284   // Make sure to handle the case where a wide SGPR spill may span between two
285   // VGPRs.
286   for (int I = 0; I < NumLanes; ++I, ++NumVGPRSpillLanes) {
287     unsigned LaneVGPR;
288     unsigned VGPRIndex = (NumVGPRSpillLanes % WaveSize);
289 
290     if (VGPRIndex == 0) {
291       LaneVGPR = TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass, MF);
292       if (LaneVGPR == AMDGPU::NoRegister) {
293         // We have no VGPRs left for spilling SGPRs. Reset because we will not
294         // partially spill the SGPR to VGPRs.
295         SGPRToVGPRSpills.erase(FI);
296         NumVGPRSpillLanes -= I;
297         return false;
298       }
299 
300       Optional<int> CSRSpillFI;
301       if ((FrameInfo.hasCalls() || !isEntryFunction()) && CSRegs &&
302           isCalleeSavedReg(CSRegs, LaneVGPR)) {
303         CSRSpillFI = FrameInfo.CreateSpillStackObject(4, 4);
304       }
305 
306       SpillVGPRs.push_back(SGPRSpillVGPRCSR(LaneVGPR, CSRSpillFI));
307 
308       // Add this register as live-in to all blocks to avoid machine verifer
309       // complaining about use of an undefined physical register.
310       for (MachineBasicBlock &BB : MF)
311         BB.addLiveIn(LaneVGPR);
312     } else {
313       LaneVGPR = SpillVGPRs.back().VGPR;
314     }
315 
316     SpillLanes.push_back(SpilledReg(LaneVGPR, VGPRIndex));
317   }
318 
319   return true;
320 }
321 
322 void SIMachineFunctionInfo::removeSGPRToVGPRFrameIndices(MachineFrameInfo &MFI) {
323   // The FP spill hasn't been inserted yet, so keep it around.
324   for (auto &R : SGPRToVGPRSpills) {
325     if (R.first != FramePointerSaveIndex)
326       MFI.RemoveStackObject(R.first);
327   }
328 
329   // All other SPGRs must be allocated on the default stack, so reset the stack
330   // ID.
331   for (int i = MFI.getObjectIndexBegin(), e = MFI.getObjectIndexEnd(); i != e;
332        ++i)
333     if (i != FramePointerSaveIndex)
334       MFI.setStackID(i, TargetStackID::Default);
335 }
336 
337 MCPhysReg SIMachineFunctionInfo::getNextUserSGPR() const {
338   assert(NumSystemSGPRs == 0 && "System SGPRs must be added after user SGPRs");
339   return AMDGPU::SGPR0 + NumUserSGPRs;
340 }
341 
342 MCPhysReg SIMachineFunctionInfo::getNextSystemSGPR() const {
343   return AMDGPU::SGPR0 + NumUserSGPRs + NumSystemSGPRs;
344 }
345 
346 static yaml::StringValue regToString(unsigned Reg,
347                                      const TargetRegisterInfo &TRI) {
348   yaml::StringValue Dest;
349   {
350     raw_string_ostream OS(Dest.Value);
351     OS << printReg(Reg, &TRI);
352   }
353   return Dest;
354 }
355 
356 static Optional<yaml::SIArgumentInfo>
357 convertArgumentInfo(const AMDGPUFunctionArgInfo &ArgInfo,
358                     const TargetRegisterInfo &TRI) {
359   yaml::SIArgumentInfo AI;
360 
361   auto convertArg = [&](Optional<yaml::SIArgument> &A,
362                         const ArgDescriptor &Arg) {
363     if (!Arg)
364       return false;
365 
366     // Create a register or stack argument.
367     yaml::SIArgument SA = yaml::SIArgument::createArgument(Arg.isRegister());
368     if (Arg.isRegister()) {
369       raw_string_ostream OS(SA.RegisterName.Value);
370       OS << printReg(Arg.getRegister(), &TRI);
371     } else
372       SA.StackOffset = Arg.getStackOffset();
373     // Check and update the optional mask.
374     if (Arg.isMasked())
375       SA.Mask = Arg.getMask();
376 
377     A = SA;
378     return true;
379   };
380 
381   bool Any = false;
382   Any |= convertArg(AI.PrivateSegmentBuffer, ArgInfo.PrivateSegmentBuffer);
383   Any |= convertArg(AI.DispatchPtr, ArgInfo.DispatchPtr);
384   Any |= convertArg(AI.QueuePtr, ArgInfo.QueuePtr);
385   Any |= convertArg(AI.KernargSegmentPtr, ArgInfo.KernargSegmentPtr);
386   Any |= convertArg(AI.DispatchID, ArgInfo.DispatchID);
387   Any |= convertArg(AI.FlatScratchInit, ArgInfo.FlatScratchInit);
388   Any |= convertArg(AI.PrivateSegmentSize, ArgInfo.PrivateSegmentSize);
389   Any |= convertArg(AI.WorkGroupIDX, ArgInfo.WorkGroupIDX);
390   Any |= convertArg(AI.WorkGroupIDY, ArgInfo.WorkGroupIDY);
391   Any |= convertArg(AI.WorkGroupIDZ, ArgInfo.WorkGroupIDZ);
392   Any |= convertArg(AI.WorkGroupInfo, ArgInfo.WorkGroupInfo);
393   Any |= convertArg(AI.PrivateSegmentWaveByteOffset,
394                     ArgInfo.PrivateSegmentWaveByteOffset);
395   Any |= convertArg(AI.ImplicitArgPtr, ArgInfo.ImplicitArgPtr);
396   Any |= convertArg(AI.ImplicitBufferPtr, ArgInfo.ImplicitBufferPtr);
397   Any |= convertArg(AI.WorkItemIDX, ArgInfo.WorkItemIDX);
398   Any |= convertArg(AI.WorkItemIDY, ArgInfo.WorkItemIDY);
399   Any |= convertArg(AI.WorkItemIDZ, ArgInfo.WorkItemIDZ);
400 
401   if (Any)
402     return AI;
403 
404   return None;
405 }
406 
407 yaml::SIMachineFunctionInfo::SIMachineFunctionInfo(
408   const llvm::SIMachineFunctionInfo& MFI,
409   const TargetRegisterInfo &TRI)
410   : ExplicitKernArgSize(MFI.getExplicitKernArgSize()),
411     MaxKernArgAlign(MFI.getMaxKernArgAlign()),
412     LDSSize(MFI.getLDSSize()),
413     IsEntryFunction(MFI.isEntryFunction()),
414     NoSignedZerosFPMath(MFI.hasNoSignedZerosFPMath()),
415     MemoryBound(MFI.isMemoryBound()),
416     WaveLimiter(MFI.needsWaveLimiter()),
417     ScratchRSrcReg(regToString(MFI.getScratchRSrcReg(), TRI)),
418     ScratchWaveOffsetReg(regToString(MFI.getScratchWaveOffsetReg(), TRI)),
419     FrameOffsetReg(regToString(MFI.getFrameOffsetReg(), TRI)),
420     StackPtrOffsetReg(regToString(MFI.getStackPtrOffsetReg(), TRI)),
421     ArgInfo(convertArgumentInfo(MFI.getArgInfo(), TRI)),
422     Mode(MFI.getMode()) {}
423 
424 void yaml::SIMachineFunctionInfo::mappingImpl(yaml::IO &YamlIO) {
425   MappingTraits<SIMachineFunctionInfo>::mapping(YamlIO, *this);
426 }
427 
428 bool SIMachineFunctionInfo::initializeBaseYamlFields(
429   const yaml::SIMachineFunctionInfo &YamlMFI) {
430   ExplicitKernArgSize = YamlMFI.ExplicitKernArgSize;
431   MaxKernArgAlign = YamlMFI.MaxKernArgAlign;
432   LDSSize = YamlMFI.LDSSize;
433   IsEntryFunction = YamlMFI.IsEntryFunction;
434   NoSignedZerosFPMath = YamlMFI.NoSignedZerosFPMath;
435   MemoryBound = YamlMFI.MemoryBound;
436   WaveLimiter = YamlMFI.WaveLimiter;
437   return false;
438 }
439