1 //===- SIMachineFunctionInfo.cpp - SI Machine Function Info ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "SIMachineFunctionInfo.h"
10 #include "AMDGPUArgumentUsageInfo.h"
11 #include "AMDGPUSubtarget.h"
12 #include "SIRegisterInfo.h"
13 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "llvm/ADT/Optional.h"
16 #include "llvm/CodeGen/MachineBasicBlock.h"
17 #include "llvm/CodeGen/MachineFrameInfo.h"
18 #include "llvm/CodeGen/MachineFunction.h"
19 #include "llvm/CodeGen/MachineRegisterInfo.h"
20 #include "llvm/IR/CallingConv.h"
21 #include "llvm/IR/Function.h"
22 #include <cassert>
23 #include <vector>
24 
25 #define MAX_LANES 64
26 
27 using namespace llvm;
28 
29 SIMachineFunctionInfo::SIMachineFunctionInfo(const MachineFunction &MF)
30   : AMDGPUMachineFunction(MF),
31     Mode(MF.getFunction()),
32     PrivateSegmentBuffer(false),
33     DispatchPtr(false),
34     QueuePtr(false),
35     KernargSegmentPtr(false),
36     DispatchID(false),
37     FlatScratchInit(false),
38     WorkGroupIDX(false),
39     WorkGroupIDY(false),
40     WorkGroupIDZ(false),
41     WorkGroupInfo(false),
42     PrivateSegmentWaveByteOffset(false),
43     WorkItemIDX(false),
44     WorkItemIDY(false),
45     WorkItemIDZ(false),
46     ImplicitBufferPtr(false),
47     ImplicitArgPtr(false),
48     GITPtrHigh(0xffffffff),
49     HighBitsOf32BitAddress(0),
50     GDSSize(0) {
51   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
52   const Function &F = MF.getFunction();
53   FlatWorkGroupSizes = ST.getFlatWorkGroupSizes(F);
54   WavesPerEU = ST.getWavesPerEU(F);
55 
56   Occupancy = getMaxWavesPerEU();
57   limitOccupancy(MF);
58   CallingConv::ID CC = F.getCallingConv();
59 
60   if (CC == CallingConv::AMDGPU_KERNEL || CC == CallingConv::SPIR_KERNEL) {
61     if (!F.arg_empty())
62       KernargSegmentPtr = true;
63     WorkGroupIDX = true;
64     WorkItemIDX = true;
65   } else if (CC == CallingConv::AMDGPU_PS) {
66     PSInputAddr = AMDGPU::getInitialPSInputAddr(F);
67   }
68 
69   if (!isEntryFunction()) {
70     // Non-entry functions have no special inputs for now, other registers
71     // required for scratch access.
72     ScratchRSrcReg = AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3;
73     ScratchWaveOffsetReg = AMDGPU::SGPR33;
74     FrameOffsetReg = AMDGPU::SGPR5;
75     StackPtrOffsetReg = AMDGPU::SGPR32;
76 
77     ArgInfo.PrivateSegmentBuffer =
78       ArgDescriptor::createRegister(ScratchRSrcReg);
79     ArgInfo.PrivateSegmentWaveByteOffset =
80       ArgDescriptor::createRegister(ScratchWaveOffsetReg);
81 
82     if (F.hasFnAttribute("amdgpu-implicitarg-ptr"))
83       ImplicitArgPtr = true;
84   } else {
85     if (F.hasFnAttribute("amdgpu-implicitarg-ptr")) {
86       KernargSegmentPtr = true;
87       MaxKernArgAlign = std::max(ST.getAlignmentForImplicitArgPtr(),
88                                  MaxKernArgAlign);
89     }
90   }
91 
92   if (F.hasFnAttribute("amdgpu-work-group-id-x"))
93     WorkGroupIDX = true;
94 
95   if (F.hasFnAttribute("amdgpu-work-group-id-y"))
96     WorkGroupIDY = true;
97 
98   if (F.hasFnAttribute("amdgpu-work-group-id-z"))
99     WorkGroupIDZ = true;
100 
101   if (F.hasFnAttribute("amdgpu-work-item-id-x"))
102     WorkItemIDX = true;
103 
104   if (F.hasFnAttribute("amdgpu-work-item-id-y"))
105     WorkItemIDY = true;
106 
107   if (F.hasFnAttribute("amdgpu-work-item-id-z"))
108     WorkItemIDZ = true;
109 
110   const MachineFrameInfo &FrameInfo = MF.getFrameInfo();
111   bool HasStackObjects = FrameInfo.hasStackObjects();
112 
113   if (isEntryFunction()) {
114     // X, XY, and XYZ are the only supported combinations, so make sure Y is
115     // enabled if Z is.
116     if (WorkItemIDZ)
117       WorkItemIDY = true;
118 
119     PrivateSegmentWaveByteOffset = true;
120 
121     // HS and GS always have the scratch wave offset in SGPR5 on GFX9.
122     if (ST.getGeneration() >= AMDGPUSubtarget::GFX9 &&
123         (CC == CallingConv::AMDGPU_HS || CC == CallingConv::AMDGPU_GS))
124       ArgInfo.PrivateSegmentWaveByteOffset =
125           ArgDescriptor::createRegister(AMDGPU::SGPR5);
126   }
127 
128   bool isAmdHsaOrMesa = ST.isAmdHsaOrMesa(F);
129   if (isAmdHsaOrMesa) {
130     PrivateSegmentBuffer = true;
131 
132     if (F.hasFnAttribute("amdgpu-dispatch-ptr"))
133       DispatchPtr = true;
134 
135     if (F.hasFnAttribute("amdgpu-queue-ptr"))
136       QueuePtr = true;
137 
138     if (F.hasFnAttribute("amdgpu-dispatch-id"))
139       DispatchID = true;
140   } else if (ST.isMesaGfxShader(F)) {
141     ImplicitBufferPtr = true;
142   }
143 
144   if (F.hasFnAttribute("amdgpu-kernarg-segment-ptr"))
145     KernargSegmentPtr = true;
146 
147   if (ST.hasFlatAddressSpace() && isEntryFunction() && isAmdHsaOrMesa) {
148     auto hasNonSpillStackObjects = [&]() {
149       // Avoid expensive checking if there's no stack objects.
150       if (!HasStackObjects)
151         return false;
152       for (auto OI = FrameInfo.getObjectIndexBegin(),
153                 OE = FrameInfo.getObjectIndexEnd(); OI != OE; ++OI)
154         if (!FrameInfo.isSpillSlotObjectIndex(OI))
155           return true;
156       // All stack objects are spill slots.
157       return false;
158     };
159     // TODO: This could be refined a lot. The attribute is a poor way of
160     // detecting calls that may require it before argument lowering.
161     if (hasNonSpillStackObjects() || F.hasFnAttribute("amdgpu-flat-scratch"))
162       FlatScratchInit = true;
163   }
164 
165   Attribute A = F.getFnAttribute("amdgpu-git-ptr-high");
166   StringRef S = A.getValueAsString();
167   if (!S.empty())
168     S.consumeInteger(0, GITPtrHigh);
169 
170   A = F.getFnAttribute("amdgpu-32bit-address-high-bits");
171   S = A.getValueAsString();
172   if (!S.empty())
173     S.consumeInteger(0, HighBitsOf32BitAddress);
174 
175   S = F.getFnAttribute("amdgpu-gds-size").getValueAsString();
176   if (!S.empty())
177     S.consumeInteger(0, GDSSize);
178 }
179 
180 void SIMachineFunctionInfo::limitOccupancy(const MachineFunction &MF) {
181   limitOccupancy(getMaxWavesPerEU());
182   const GCNSubtarget& ST = MF.getSubtarget<GCNSubtarget>();
183   limitOccupancy(ST.getOccupancyWithLocalMemSize(getLDSSize(),
184                  MF.getFunction()));
185 }
186 
187 unsigned SIMachineFunctionInfo::addPrivateSegmentBuffer(
188   const SIRegisterInfo &TRI) {
189   ArgInfo.PrivateSegmentBuffer =
190     ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
191     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_128RegClass));
192   NumUserSGPRs += 4;
193   return ArgInfo.PrivateSegmentBuffer.getRegister();
194 }
195 
196 unsigned SIMachineFunctionInfo::addDispatchPtr(const SIRegisterInfo &TRI) {
197   ArgInfo.DispatchPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
198     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
199   NumUserSGPRs += 2;
200   return ArgInfo.DispatchPtr.getRegister();
201 }
202 
203 unsigned SIMachineFunctionInfo::addQueuePtr(const SIRegisterInfo &TRI) {
204   ArgInfo.QueuePtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
205     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
206   NumUserSGPRs += 2;
207   return ArgInfo.QueuePtr.getRegister();
208 }
209 
210 unsigned SIMachineFunctionInfo::addKernargSegmentPtr(const SIRegisterInfo &TRI) {
211   ArgInfo.KernargSegmentPtr
212     = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
213     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
214   NumUserSGPRs += 2;
215   return ArgInfo.KernargSegmentPtr.getRegister();
216 }
217 
218 unsigned SIMachineFunctionInfo::addDispatchID(const SIRegisterInfo &TRI) {
219   ArgInfo.DispatchID = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
220     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
221   NumUserSGPRs += 2;
222   return ArgInfo.DispatchID.getRegister();
223 }
224 
225 unsigned SIMachineFunctionInfo::addFlatScratchInit(const SIRegisterInfo &TRI) {
226   ArgInfo.FlatScratchInit = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
227     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
228   NumUserSGPRs += 2;
229   return ArgInfo.FlatScratchInit.getRegister();
230 }
231 
232 unsigned SIMachineFunctionInfo::addImplicitBufferPtr(const SIRegisterInfo &TRI) {
233   ArgInfo.ImplicitBufferPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
234     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
235   NumUserSGPRs += 2;
236   return ArgInfo.ImplicitBufferPtr.getRegister();
237 }
238 
239 static bool isCalleeSavedReg(const MCPhysReg *CSRegs, MCPhysReg Reg) {
240   for (unsigned I = 0; CSRegs[I]; ++I) {
241     if (CSRegs[I] == Reg)
242       return true;
243   }
244 
245   return false;
246 }
247 
248 /// Reserve a slice of a VGPR to support spilling for FrameIndex \p FI.
249 bool SIMachineFunctionInfo::allocateSGPRSpillToVGPR(MachineFunction &MF,
250                                                     int FI) {
251   std::vector<SpilledReg> &SpillLanes = SGPRToVGPRSpills[FI];
252 
253   // This has already been allocated.
254   if (!SpillLanes.empty())
255     return true;
256 
257   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
258   const SIRegisterInfo *TRI = ST.getRegisterInfo();
259   MachineFrameInfo &FrameInfo = MF.getFrameInfo();
260   MachineRegisterInfo &MRI = MF.getRegInfo();
261   unsigned WaveSize = ST.getWavefrontSize();
262 
263   unsigned Size = FrameInfo.getObjectSize(FI);
264   assert(Size >= 4 && Size <= 64 && "invalid sgpr spill size");
265   assert(TRI->spillSGPRToVGPR() && "not spilling SGPRs to VGPRs");
266 
267   int NumLanes = Size / 4;
268 
269   const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs();
270 
271   // Make sure to handle the case where a wide SGPR spill may span between two
272   // VGPRs.
273   for (int I = 0; I < NumLanes; ++I, ++NumVGPRSpillLanes) {
274     unsigned LaneVGPR;
275     unsigned VGPRIndex = (NumVGPRSpillLanes % WaveSize);
276 
277     if (VGPRIndex == 0) {
278       LaneVGPR = TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass, MF);
279       if (LaneVGPR == AMDGPU::NoRegister) {
280         // We have no VGPRs left for spilling SGPRs. Reset because we will not
281         // partially spill the SGPR to VGPRs.
282         SGPRToVGPRSpills.erase(FI);
283         NumVGPRSpillLanes -= I;
284         return false;
285       }
286 
287       Optional<int> CSRSpillFI;
288       if ((FrameInfo.hasCalls() || !isEntryFunction()) && CSRegs &&
289           isCalleeSavedReg(CSRegs, LaneVGPR)) {
290         CSRSpillFI = FrameInfo.CreateSpillStackObject(4, 4);
291       }
292 
293       SpillVGPRs.push_back(SGPRSpillVGPRCSR(LaneVGPR, CSRSpillFI));
294 
295       // Add this register as live-in to all blocks to avoid machine verifer
296       // complaining about use of an undefined physical register.
297       for (MachineBasicBlock &BB : MF)
298         BB.addLiveIn(LaneVGPR);
299     } else {
300       LaneVGPR = SpillVGPRs.back().VGPR;
301     }
302 
303     SpillLanes.push_back(SpilledReg(LaneVGPR, VGPRIndex));
304   }
305 
306   return true;
307 }
308 
309 void SIMachineFunctionInfo::removeSGPRToVGPRFrameIndices(MachineFrameInfo &MFI) {
310   for (auto &R : SGPRToVGPRSpills)
311     MFI.RemoveStackObject(R.first);
312   // All other SPGRs must be allocated on the default stack, so reset
313   // the stack ID.
314   for (unsigned i = MFI.getObjectIndexBegin(), e = MFI.getObjectIndexEnd();
315        i != e; ++i)
316     MFI.setStackID(i, 0);
317 }
318 
319 MCPhysReg SIMachineFunctionInfo::getNextUserSGPR() const {
320   assert(NumSystemSGPRs == 0 && "System SGPRs must be added after user SGPRs");
321   return AMDGPU::SGPR0 + NumUserSGPRs;
322 }
323 
324 MCPhysReg SIMachineFunctionInfo::getNextSystemSGPR() const {
325   return AMDGPU::SGPR0 + NumUserSGPRs + NumSystemSGPRs;
326 }
327 
328 static yaml::StringValue regToString(unsigned Reg,
329                                      const TargetRegisterInfo &TRI) {
330   yaml::StringValue Dest;
331   {
332     raw_string_ostream OS(Dest.Value);
333     OS << printReg(Reg, &TRI);
334   }
335   return Dest;
336 }
337 
338 static Optional<yaml::SIArgumentInfo>
339 convertArgumentInfo(const AMDGPUFunctionArgInfo &ArgInfo,
340                     const TargetRegisterInfo &TRI) {
341   yaml::SIArgumentInfo AI;
342 
343   auto convertArg = [&](Optional<yaml::SIArgument> &A,
344                         const ArgDescriptor &Arg) {
345     if (!Arg)
346       return false;
347 
348     // Create a register or stack argument.
349     yaml::SIArgument SA = yaml::SIArgument::createArgument(Arg.isRegister());
350     if (Arg.isRegister()) {
351       raw_string_ostream OS(SA.RegisterName.Value);
352       OS << printReg(Arg.getRegister(), &TRI);
353     } else
354       SA.StackOffset = Arg.getStackOffset();
355     // Check and update the optional mask.
356     if (Arg.isMasked())
357       SA.Mask = Arg.getMask();
358 
359     A = SA;
360     return true;
361   };
362 
363   bool Any = false;
364   Any |= convertArg(AI.PrivateSegmentBuffer, ArgInfo.PrivateSegmentBuffer);
365   Any |= convertArg(AI.DispatchPtr, ArgInfo.DispatchPtr);
366   Any |= convertArg(AI.QueuePtr, ArgInfo.QueuePtr);
367   Any |= convertArg(AI.KernargSegmentPtr, ArgInfo.KernargSegmentPtr);
368   Any |= convertArg(AI.DispatchID, ArgInfo.DispatchID);
369   Any |= convertArg(AI.FlatScratchInit, ArgInfo.FlatScratchInit);
370   Any |= convertArg(AI.PrivateSegmentSize, ArgInfo.PrivateSegmentSize);
371   Any |= convertArg(AI.WorkGroupIDX, ArgInfo.WorkGroupIDX);
372   Any |= convertArg(AI.WorkGroupIDY, ArgInfo.WorkGroupIDY);
373   Any |= convertArg(AI.WorkGroupIDZ, ArgInfo.WorkGroupIDZ);
374   Any |= convertArg(AI.WorkGroupInfo, ArgInfo.WorkGroupInfo);
375   Any |= convertArg(AI.PrivateSegmentWaveByteOffset,
376                     ArgInfo.PrivateSegmentWaveByteOffset);
377   Any |= convertArg(AI.ImplicitArgPtr, ArgInfo.ImplicitArgPtr);
378   Any |= convertArg(AI.ImplicitBufferPtr, ArgInfo.ImplicitBufferPtr);
379   Any |= convertArg(AI.WorkItemIDX, ArgInfo.WorkItemIDX);
380   Any |= convertArg(AI.WorkItemIDY, ArgInfo.WorkItemIDY);
381   Any |= convertArg(AI.WorkItemIDZ, ArgInfo.WorkItemIDZ);
382 
383   if (Any)
384     return AI;
385 
386   return None;
387 }
388 
389 yaml::SIMachineFunctionInfo::SIMachineFunctionInfo(
390   const llvm::SIMachineFunctionInfo& MFI,
391   const TargetRegisterInfo &TRI)
392   : ExplicitKernArgSize(MFI.getExplicitKernArgSize()),
393     MaxKernArgAlign(MFI.getMaxKernArgAlign()),
394     LDSSize(MFI.getLDSSize()),
395     IsEntryFunction(MFI.isEntryFunction()),
396     NoSignedZerosFPMath(MFI.hasNoSignedZerosFPMath()),
397     MemoryBound(MFI.isMemoryBound()),
398     WaveLimiter(MFI.needsWaveLimiter()),
399     ScratchRSrcReg(regToString(MFI.getScratchRSrcReg(), TRI)),
400     ScratchWaveOffsetReg(regToString(MFI.getScratchWaveOffsetReg(), TRI)),
401     FrameOffsetReg(regToString(MFI.getFrameOffsetReg(), TRI)),
402     StackPtrOffsetReg(regToString(MFI.getStackPtrOffsetReg(), TRI)),
403     ArgInfo(convertArgumentInfo(MFI.getArgInfo(), TRI)) {}
404 
405 void yaml::SIMachineFunctionInfo::mappingImpl(yaml::IO &YamlIO) {
406   MappingTraits<SIMachineFunctionInfo>::mapping(YamlIO, *this);
407 }
408 
409 bool SIMachineFunctionInfo::initializeBaseYamlFields(
410   const yaml::SIMachineFunctionInfo &YamlMFI) {
411   ExplicitKernArgSize = YamlMFI.ExplicitKernArgSize;
412   MaxKernArgAlign = YamlMFI.MaxKernArgAlign;
413   LDSSize = YamlMFI.LDSSize;
414   IsEntryFunction = YamlMFI.IsEntryFunction;
415   NoSignedZerosFPMath = YamlMFI.NoSignedZerosFPMath;
416   MemoryBound = YamlMFI.MemoryBound;
417   WaveLimiter = YamlMFI.WaveLimiter;
418   return false;
419 }
420