1 //===- SIMachineFunctionInfo.cpp - SI Machine Function Info ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "SIMachineFunctionInfo.h"
10 #include "AMDGPUArgumentUsageInfo.h"
11 #include "AMDGPUSubtarget.h"
12 #include "SIRegisterInfo.h"
13 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "llvm/ADT/Optional.h"
16 #include "llvm/CodeGen/MachineBasicBlock.h"
17 #include "llvm/CodeGen/MachineFrameInfo.h"
18 #include "llvm/CodeGen/MachineFunction.h"
19 #include "llvm/CodeGen/MachineRegisterInfo.h"
20 #include "llvm/IR/CallingConv.h"
21 #include "llvm/IR/Function.h"
22 #include <cassert>
23 #include <vector>
24 
25 #define MAX_LANES 64
26 
27 using namespace llvm;
28 
29 SIMachineFunctionInfo::SIMachineFunctionInfo(const MachineFunction &MF)
30   : AMDGPUMachineFunction(MF),
31     Mode(MF.getFunction()),
32     PrivateSegmentBuffer(false),
33     DispatchPtr(false),
34     QueuePtr(false),
35     KernargSegmentPtr(false),
36     DispatchID(false),
37     FlatScratchInit(false),
38     WorkGroupIDX(false),
39     WorkGroupIDY(false),
40     WorkGroupIDZ(false),
41     WorkGroupInfo(false),
42     PrivateSegmentWaveByteOffset(false),
43     WorkItemIDX(false),
44     WorkItemIDY(false),
45     WorkItemIDZ(false),
46     ImplicitBufferPtr(false),
47     ImplicitArgPtr(false),
48     GITPtrHigh(0xffffffff),
49     HighBitsOf32BitAddress(0),
50     GDSSize(0) {
51   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
52   const Function &F = MF.getFunction();
53   FlatWorkGroupSizes = ST.getFlatWorkGroupSizes(F);
54   WavesPerEU = ST.getWavesPerEU(F);
55 
56   Occupancy = getMaxWavesPerEU();
57   limitOccupancy(MF);
58   CallingConv::ID CC = F.getCallingConv();
59 
60   if (CC == CallingConv::AMDGPU_KERNEL || CC == CallingConv::SPIR_KERNEL) {
61     if (!F.arg_empty())
62       KernargSegmentPtr = true;
63     WorkGroupIDX = true;
64     WorkItemIDX = true;
65   } else if (CC == CallingConv::AMDGPU_PS) {
66     PSInputAddr = AMDGPU::getInitialPSInputAddr(F);
67   }
68 
69   if (!isEntryFunction()) {
70     // Non-entry functions have no special inputs for now, other registers
71     // required for scratch access.
72     ScratchRSrcReg = AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3;
73     ScratchWaveOffsetReg = AMDGPU::SGPR33;
74     FrameOffsetReg = AMDGPU::SGPR5;
75     StackPtrOffsetReg = AMDGPU::SGPR32;
76 
77     ArgInfo.PrivateSegmentBuffer =
78       ArgDescriptor::createRegister(ScratchRSrcReg);
79     ArgInfo.PrivateSegmentWaveByteOffset =
80       ArgDescriptor::createRegister(ScratchWaveOffsetReg);
81 
82     if (F.hasFnAttribute("amdgpu-implicitarg-ptr"))
83       ImplicitArgPtr = true;
84   } else {
85     if (F.hasFnAttribute("amdgpu-implicitarg-ptr")) {
86       KernargSegmentPtr = true;
87       MaxKernArgAlign = std::max(ST.getAlignmentForImplicitArgPtr(),
88                                  MaxKernArgAlign);
89     }
90   }
91 
92   if (F.hasFnAttribute("amdgpu-work-group-id-x"))
93     WorkGroupIDX = true;
94 
95   if (F.hasFnAttribute("amdgpu-work-group-id-y"))
96     WorkGroupIDY = true;
97 
98   if (F.hasFnAttribute("amdgpu-work-group-id-z"))
99     WorkGroupIDZ = true;
100 
101   if (F.hasFnAttribute("amdgpu-work-item-id-x"))
102     WorkItemIDX = true;
103 
104   if (F.hasFnAttribute("amdgpu-work-item-id-y"))
105     WorkItemIDY = true;
106 
107   if (F.hasFnAttribute("amdgpu-work-item-id-z"))
108     WorkItemIDZ = true;
109 
110   const MachineFrameInfo &FrameInfo = MF.getFrameInfo();
111   bool HasStackObjects = FrameInfo.hasStackObjects();
112 
113   if (isEntryFunction()) {
114     // X, XY, and XYZ are the only supported combinations, so make sure Y is
115     // enabled if Z is.
116     if (WorkItemIDZ)
117       WorkItemIDY = true;
118 
119     PrivateSegmentWaveByteOffset = true;
120 
121     // HS and GS always have the scratch wave offset in SGPR5 on GFX9.
122     if (ST.getGeneration() >= AMDGPUSubtarget::GFX9 &&
123         (CC == CallingConv::AMDGPU_HS || CC == CallingConv::AMDGPU_GS))
124       ArgInfo.PrivateSegmentWaveByteOffset =
125           ArgDescriptor::createRegister(AMDGPU::SGPR5);
126   }
127 
128   bool isAmdHsaOrMesa = ST.isAmdHsaOrMesa(F);
129   if (isAmdHsaOrMesa) {
130     PrivateSegmentBuffer = true;
131 
132     if (F.hasFnAttribute("amdgpu-dispatch-ptr"))
133       DispatchPtr = true;
134 
135     if (F.hasFnAttribute("amdgpu-queue-ptr"))
136       QueuePtr = true;
137 
138     if (F.hasFnAttribute("amdgpu-dispatch-id"))
139       DispatchID = true;
140   } else if (ST.isMesaGfxShader(F)) {
141     ImplicitBufferPtr = true;
142   }
143 
144   if (F.hasFnAttribute("amdgpu-kernarg-segment-ptr"))
145     KernargSegmentPtr = true;
146 
147   if (ST.hasFlatAddressSpace() && isEntryFunction() && isAmdHsaOrMesa) {
148     // TODO: This could be refined a lot. The attribute is a poor way of
149     // detecting calls that may require it before argument lowering.
150     if (HasStackObjects || F.hasFnAttribute("amdgpu-flat-scratch"))
151       FlatScratchInit = true;
152   }
153 
154   Attribute A = F.getFnAttribute("amdgpu-git-ptr-high");
155   StringRef S = A.getValueAsString();
156   if (!S.empty())
157     S.consumeInteger(0, GITPtrHigh);
158 
159   A = F.getFnAttribute("amdgpu-32bit-address-high-bits");
160   S = A.getValueAsString();
161   if (!S.empty())
162     S.consumeInteger(0, HighBitsOf32BitAddress);
163 
164   S = F.getFnAttribute("amdgpu-gds-size").getValueAsString();
165   if (!S.empty())
166     S.consumeInteger(0, GDSSize);
167 }
168 
169 void SIMachineFunctionInfo::limitOccupancy(const MachineFunction &MF) {
170   limitOccupancy(getMaxWavesPerEU());
171   const GCNSubtarget& ST = MF.getSubtarget<GCNSubtarget>();
172   limitOccupancy(ST.getOccupancyWithLocalMemSize(getLDSSize(),
173                  MF.getFunction()));
174 }
175 
176 unsigned SIMachineFunctionInfo::addPrivateSegmentBuffer(
177   const SIRegisterInfo &TRI) {
178   ArgInfo.PrivateSegmentBuffer =
179     ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
180     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_128RegClass));
181   NumUserSGPRs += 4;
182   return ArgInfo.PrivateSegmentBuffer.getRegister();
183 }
184 
185 unsigned SIMachineFunctionInfo::addDispatchPtr(const SIRegisterInfo &TRI) {
186   ArgInfo.DispatchPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
187     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
188   NumUserSGPRs += 2;
189   return ArgInfo.DispatchPtr.getRegister();
190 }
191 
192 unsigned SIMachineFunctionInfo::addQueuePtr(const SIRegisterInfo &TRI) {
193   ArgInfo.QueuePtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
194     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
195   NumUserSGPRs += 2;
196   return ArgInfo.QueuePtr.getRegister();
197 }
198 
199 unsigned SIMachineFunctionInfo::addKernargSegmentPtr(const SIRegisterInfo &TRI) {
200   ArgInfo.KernargSegmentPtr
201     = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
202     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
203   NumUserSGPRs += 2;
204   return ArgInfo.KernargSegmentPtr.getRegister();
205 }
206 
207 unsigned SIMachineFunctionInfo::addDispatchID(const SIRegisterInfo &TRI) {
208   ArgInfo.DispatchID = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
209     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
210   NumUserSGPRs += 2;
211   return ArgInfo.DispatchID.getRegister();
212 }
213 
214 unsigned SIMachineFunctionInfo::addFlatScratchInit(const SIRegisterInfo &TRI) {
215   ArgInfo.FlatScratchInit = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
216     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
217   NumUserSGPRs += 2;
218   return ArgInfo.FlatScratchInit.getRegister();
219 }
220 
221 unsigned SIMachineFunctionInfo::addImplicitBufferPtr(const SIRegisterInfo &TRI) {
222   ArgInfo.ImplicitBufferPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
223     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
224   NumUserSGPRs += 2;
225   return ArgInfo.ImplicitBufferPtr.getRegister();
226 }
227 
228 static bool isCalleeSavedReg(const MCPhysReg *CSRegs, MCPhysReg Reg) {
229   for (unsigned I = 0; CSRegs[I]; ++I) {
230     if (CSRegs[I] == Reg)
231       return true;
232   }
233 
234   return false;
235 }
236 
237 /// Reserve a slice of a VGPR to support spilling for FrameIndex \p FI.
238 bool SIMachineFunctionInfo::allocateSGPRSpillToVGPR(MachineFunction &MF,
239                                                     int FI) {
240   std::vector<SpilledReg> &SpillLanes = SGPRToVGPRSpills[FI];
241 
242   // This has already been allocated.
243   if (!SpillLanes.empty())
244     return true;
245 
246   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
247   const SIRegisterInfo *TRI = ST.getRegisterInfo();
248   MachineFrameInfo &FrameInfo = MF.getFrameInfo();
249   MachineRegisterInfo &MRI = MF.getRegInfo();
250   unsigned WaveSize = ST.getWavefrontSize();
251 
252   unsigned Size = FrameInfo.getObjectSize(FI);
253   assert(Size >= 4 && Size <= 64 && "invalid sgpr spill size");
254   assert(TRI->spillSGPRToVGPR() && "not spilling SGPRs to VGPRs");
255 
256   int NumLanes = Size / 4;
257 
258   const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs();
259 
260   // Make sure to handle the case where a wide SGPR spill may span between two
261   // VGPRs.
262   for (int I = 0; I < NumLanes; ++I, ++NumVGPRSpillLanes) {
263     unsigned LaneVGPR;
264     unsigned VGPRIndex = (NumVGPRSpillLanes % WaveSize);
265 
266     if (VGPRIndex == 0) {
267       LaneVGPR = TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass, MF);
268       if (LaneVGPR == AMDGPU::NoRegister) {
269         // We have no VGPRs left for spilling SGPRs. Reset because we will not
270         // partially spill the SGPR to VGPRs.
271         SGPRToVGPRSpills.erase(FI);
272         NumVGPRSpillLanes -= I;
273         return false;
274       }
275 
276       Optional<int> CSRSpillFI;
277       if ((FrameInfo.hasCalls() || !isEntryFunction()) && CSRegs &&
278           isCalleeSavedReg(CSRegs, LaneVGPR)) {
279         CSRSpillFI = FrameInfo.CreateSpillStackObject(4, 4);
280       }
281 
282       SpillVGPRs.push_back(SGPRSpillVGPRCSR(LaneVGPR, CSRSpillFI));
283 
284       // Add this register as live-in to all blocks to avoid machine verifer
285       // complaining about use of an undefined physical register.
286       for (MachineBasicBlock &BB : MF)
287         BB.addLiveIn(LaneVGPR);
288     } else {
289       LaneVGPR = SpillVGPRs.back().VGPR;
290     }
291 
292     SpillLanes.push_back(SpilledReg(LaneVGPR, VGPRIndex));
293   }
294 
295   return true;
296 }
297 
298 void SIMachineFunctionInfo::removeSGPRToVGPRFrameIndices(MachineFrameInfo &MFI) {
299   for (auto &R : SGPRToVGPRSpills)
300     MFI.RemoveStackObject(R.first);
301   // All other SPGRs must be allocated on the default stack, so reset
302   // the stack ID.
303   for (unsigned i = MFI.getObjectIndexBegin(), e = MFI.getObjectIndexEnd();
304        i != e; ++i)
305     MFI.setStackID(i, 0);
306 }
307 
308 MCPhysReg SIMachineFunctionInfo::getNextUserSGPR() const {
309   assert(NumSystemSGPRs == 0 && "System SGPRs must be added after user SGPRs");
310   return AMDGPU::SGPR0 + NumUserSGPRs;
311 }
312 
313 MCPhysReg SIMachineFunctionInfo::getNextSystemSGPR() const {
314   return AMDGPU::SGPR0 + NumUserSGPRs + NumSystemSGPRs;
315 }
316 
317 static yaml::StringValue regToString(unsigned Reg,
318                                      const TargetRegisterInfo &TRI) {
319   yaml::StringValue Dest;
320   {
321     raw_string_ostream OS(Dest.Value);
322     OS << printReg(Reg, &TRI);
323   }
324   return Dest;
325 }
326 
327 static Optional<yaml::SIArgumentInfo>
328 convertArgumentInfo(const AMDGPUFunctionArgInfo &ArgInfo,
329                     const TargetRegisterInfo &TRI) {
330   yaml::SIArgumentInfo AI;
331 
332   auto convertArg = [&](Optional<yaml::SIArgument> &A,
333                         const ArgDescriptor &Arg) {
334     if (!Arg)
335       return false;
336 
337     // Create a register or stack argument.
338     yaml::SIArgument SA = yaml::SIArgument::createArgument(Arg.isRegister());
339     if (Arg.isRegister()) {
340       raw_string_ostream OS(SA.RegisterName.Value);
341       OS << printReg(Arg.getRegister(), &TRI);
342     } else
343       SA.StackOffset = Arg.getStackOffset();
344     // Check and update the optional mask.
345     if (Arg.isMasked())
346       SA.Mask = Arg.getMask();
347 
348     A = SA;
349     return true;
350   };
351 
352   bool Any = false;
353   Any |= convertArg(AI.PrivateSegmentBuffer, ArgInfo.PrivateSegmentBuffer);
354   Any |= convertArg(AI.DispatchPtr, ArgInfo.DispatchPtr);
355   Any |= convertArg(AI.QueuePtr, ArgInfo.QueuePtr);
356   Any |= convertArg(AI.KernargSegmentPtr, ArgInfo.KernargSegmentPtr);
357   Any |= convertArg(AI.DispatchID, ArgInfo.DispatchID);
358   Any |= convertArg(AI.FlatScratchInit, ArgInfo.FlatScratchInit);
359   Any |= convertArg(AI.PrivateSegmentSize, ArgInfo.PrivateSegmentSize);
360   Any |= convertArg(AI.WorkGroupIDX, ArgInfo.WorkGroupIDX);
361   Any |= convertArg(AI.WorkGroupIDY, ArgInfo.WorkGroupIDY);
362   Any |= convertArg(AI.WorkGroupIDZ, ArgInfo.WorkGroupIDZ);
363   Any |= convertArg(AI.WorkGroupInfo, ArgInfo.WorkGroupInfo);
364   Any |= convertArg(AI.PrivateSegmentWaveByteOffset,
365                     ArgInfo.PrivateSegmentWaveByteOffset);
366   Any |= convertArg(AI.ImplicitArgPtr, ArgInfo.ImplicitArgPtr);
367   Any |= convertArg(AI.ImplicitBufferPtr, ArgInfo.ImplicitBufferPtr);
368   Any |= convertArg(AI.WorkItemIDX, ArgInfo.WorkItemIDX);
369   Any |= convertArg(AI.WorkItemIDY, ArgInfo.WorkItemIDY);
370   Any |= convertArg(AI.WorkItemIDZ, ArgInfo.WorkItemIDZ);
371 
372   if (Any)
373     return AI;
374 
375   return None;
376 }
377 
378 yaml::SIMachineFunctionInfo::SIMachineFunctionInfo(
379   const llvm::SIMachineFunctionInfo& MFI,
380   const TargetRegisterInfo &TRI)
381   : ExplicitKernArgSize(MFI.getExplicitKernArgSize()),
382     MaxKernArgAlign(MFI.getMaxKernArgAlign()),
383     LDSSize(MFI.getLDSSize()),
384     IsEntryFunction(MFI.isEntryFunction()),
385     NoSignedZerosFPMath(MFI.hasNoSignedZerosFPMath()),
386     MemoryBound(MFI.isMemoryBound()),
387     WaveLimiter(MFI.needsWaveLimiter()),
388     ScratchRSrcReg(regToString(MFI.getScratchRSrcReg(), TRI)),
389     ScratchWaveOffsetReg(regToString(MFI.getScratchWaveOffsetReg(), TRI)),
390     FrameOffsetReg(regToString(MFI.getFrameOffsetReg(), TRI)),
391     StackPtrOffsetReg(regToString(MFI.getStackPtrOffsetReg(), TRI)),
392     ArgInfo(convertArgumentInfo(MFI.getArgInfo(), TRI)) {}
393 
394 void yaml::SIMachineFunctionInfo::mappingImpl(yaml::IO &YamlIO) {
395   MappingTraits<SIMachineFunctionInfo>::mapping(YamlIO, *this);
396 }
397 
398 bool SIMachineFunctionInfo::initializeBaseYamlFields(
399   const yaml::SIMachineFunctionInfo &YamlMFI) {
400   ExplicitKernArgSize = YamlMFI.ExplicitKernArgSize;
401   MaxKernArgAlign = YamlMFI.MaxKernArgAlign;
402   LDSSize = YamlMFI.LDSSize;
403   IsEntryFunction = YamlMFI.IsEntryFunction;
404   NoSignedZerosFPMath = YamlMFI.NoSignedZerosFPMath;
405   MemoryBound = YamlMFI.MemoryBound;
406   WaveLimiter = YamlMFI.WaveLimiter;
407   return false;
408 }
409