1 //===-- SILowerControlFlow.cpp - Use predicates for control flow ----------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// \brief This pass lowers the pseudo control flow instructions to real
12 /// machine instructions.
13 ///
14 /// All control flow is handled using predicated instructions and
15 /// a predicate stack.  Each Scalar ALU controls the operations of 64 Vector
16 /// ALUs.  The Scalar ALU can update the predicate for any of the Vector ALUs
17 /// by writting to the 64-bit EXEC register (each bit corresponds to a
18 /// single vector ALU).  Typically, for predicates, a vector ALU will write
19 /// to its bit of the VCC register (like EXEC VCC is 64-bits, one for each
20 /// Vector ALU) and then the ScalarALU will AND the VCC register with the
21 /// EXEC to update the predicates.
22 ///
23 /// For example:
24 /// %VCC = V_CMP_GT_F32 %VGPR1, %VGPR2
25 /// %SGPR0 = SI_IF %VCC
26 ///   %VGPR0 = V_ADD_F32 %VGPR0, %VGPR0
27 /// %SGPR0 = SI_ELSE %SGPR0
28 ///   %VGPR0 = V_SUB_F32 %VGPR0, %VGPR0
29 /// SI_END_CF %SGPR0
30 ///
31 /// becomes:
32 ///
33 /// %SGPR0 = S_AND_SAVEEXEC_B64 %VCC  // Save and update the exec mask
34 /// %SGPR0 = S_XOR_B64 %SGPR0, %EXEC  // Clear live bits from saved exec mask
35 /// S_CBRANCH_EXECZ label0            // This instruction is an optional
36 ///                                   // optimization which allows us to
37 ///                                   // branch if all the bits of
38 ///                                   // EXEC are zero.
39 /// %VGPR0 = V_ADD_F32 %VGPR0, %VGPR0 // Do the IF block of the branch
40 ///
41 /// label0:
42 /// %SGPR0 = S_OR_SAVEEXEC_B64 %EXEC   // Restore the exec mask for the Then block
43 /// %EXEC = S_XOR_B64 %SGPR0, %EXEC    // Clear live bits from saved exec mask
44 /// S_BRANCH_EXECZ label1              // Use our branch optimization
45 ///                                    // instruction again.
46 /// %VGPR0 = V_SUB_F32 %VGPR0, %VGPR   // Do the THEN block
47 /// label1:
48 /// %EXEC = S_OR_B64 %EXEC, %SGPR0     // Re-enable saved exec mask bits
49 //===----------------------------------------------------------------------===//
50 
51 #include "AMDGPU.h"
52 #include "AMDGPUSubtarget.h"
53 #include "SIInstrInfo.h"
54 #include "SIMachineFunctionInfo.h"
55 #include "llvm/CodeGen/MachineFrameInfo.h"
56 #include "llvm/CodeGen/MachineFunction.h"
57 #include "llvm/CodeGen/MachineFunctionPass.h"
58 #include "llvm/CodeGen/MachineInstrBuilder.h"
59 #include "llvm/CodeGen/MachineRegisterInfo.h"
60 #include "llvm/IR/Constants.h"
61 
62 using namespace llvm;
63 
64 namespace {
65 
66 class SILowerControlFlowPass : public MachineFunctionPass {
67 
68 private:
69   static const unsigned SkipThreshold = 12;
70 
71   static char ID;
72   const SIRegisterInfo *TRI;
73   const SIInstrInfo *TII;
74 
75   bool shouldSkip(MachineBasicBlock *From, MachineBasicBlock *To);
76 
77   void Skip(MachineInstr &From, MachineOperand &To);
78   void SkipIfDead(MachineInstr &MI);
79 
80   void If(MachineInstr &MI);
81   void Else(MachineInstr &MI);
82   void Break(MachineInstr &MI);
83   void IfBreak(MachineInstr &MI);
84   void ElseBreak(MachineInstr &MI);
85   void Loop(MachineInstr &MI);
86   void EndCf(MachineInstr &MI);
87 
88   void Kill(MachineInstr &MI);
89   void Branch(MachineInstr &MI);
90 
91   void LoadM0(MachineInstr &MI, MachineInstr *MovRel, int Offset = 0);
92   void computeIndirectRegAndOffset(unsigned VecReg, unsigned &Reg, int &Offset);
93   void IndirectSrc(MachineInstr &MI);
94   void IndirectDst(MachineInstr &MI);
95 
96 public:
97   SILowerControlFlowPass(TargetMachine &tm) :
98     MachineFunctionPass(ID), TRI(nullptr), TII(nullptr) { }
99 
100   bool runOnMachineFunction(MachineFunction &MF) override;
101 
102   const char *getPassName() const override {
103     return "SI Lower control flow instructions";
104   }
105 
106 };
107 
108 } // End anonymous namespace
109 
110 char SILowerControlFlowPass::ID = 0;
111 
112 FunctionPass *llvm::createSILowerControlFlowPass(TargetMachine &tm) {
113   return new SILowerControlFlowPass(tm);
114 }
115 
116 bool SILowerControlFlowPass::shouldSkip(MachineBasicBlock *From,
117                                         MachineBasicBlock *To) {
118 
119   unsigned NumInstr = 0;
120 
121   for (MachineBasicBlock *MBB = From; MBB != To && !MBB->succ_empty();
122        MBB = *MBB->succ_begin()) {
123 
124     for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
125          NumInstr < SkipThreshold && I != E; ++I) {
126 
127       if (I->isBundle() || !I->isBundled())
128         if (++NumInstr >= SkipThreshold)
129           return true;
130     }
131   }
132 
133   return false;
134 }
135 
136 void SILowerControlFlowPass::Skip(MachineInstr &From, MachineOperand &To) {
137 
138   if (!shouldSkip(*From.getParent()->succ_begin(), To.getMBB()))
139     return;
140 
141   DebugLoc DL = From.getDebugLoc();
142   BuildMI(*From.getParent(), &From, DL, TII->get(AMDGPU::S_CBRANCH_EXECZ))
143     .addOperand(To);
144 }
145 
146 void SILowerControlFlowPass::SkipIfDead(MachineInstr &MI) {
147 
148   MachineBasicBlock &MBB = *MI.getParent();
149   DebugLoc DL = MI.getDebugLoc();
150 
151   if (MBB.getParent()->getInfo<SIMachineFunctionInfo>()->getShaderType() !=
152       ShaderType::PIXEL ||
153       !shouldSkip(&MBB, &MBB.getParent()->back()))
154     return;
155 
156   MachineBasicBlock::iterator Insert = &MI;
157   ++Insert;
158 
159   // If the exec mask is non-zero, skip the next two instructions
160   BuildMI(MBB, Insert, DL, TII->get(AMDGPU::S_CBRANCH_EXECNZ))
161     .addImm(3);
162 
163   // Exec mask is zero: Export to NULL target...
164   BuildMI(MBB, Insert, DL, TII->get(AMDGPU::EXP))
165           .addImm(0)
166           .addImm(0x09) // V_008DFC_SQ_EXP_NULL
167           .addImm(0)
168           .addImm(1)
169           .addImm(1)
170           .addReg(AMDGPU::VGPR0)
171           .addReg(AMDGPU::VGPR0)
172           .addReg(AMDGPU::VGPR0)
173           .addReg(AMDGPU::VGPR0);
174 
175   // ... and terminate wavefront
176   BuildMI(MBB, Insert, DL, TII->get(AMDGPU::S_ENDPGM));
177 }
178 
179 void SILowerControlFlowPass::If(MachineInstr &MI) {
180   MachineBasicBlock &MBB = *MI.getParent();
181   DebugLoc DL = MI.getDebugLoc();
182   unsigned Reg = MI.getOperand(0).getReg();
183   unsigned Vcc = MI.getOperand(1).getReg();
184 
185   BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_AND_SAVEEXEC_B64), Reg)
186           .addReg(Vcc);
187 
188   BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_XOR_B64), Reg)
189           .addReg(AMDGPU::EXEC)
190           .addReg(Reg);
191 
192   Skip(MI, MI.getOperand(2));
193 
194   MI.eraseFromParent();
195 }
196 
197 void SILowerControlFlowPass::Else(MachineInstr &MI) {
198   MachineBasicBlock &MBB = *MI.getParent();
199   DebugLoc DL = MI.getDebugLoc();
200   unsigned Dst = MI.getOperand(0).getReg();
201   unsigned Src = MI.getOperand(1).getReg();
202 
203   BuildMI(MBB, MBB.getFirstNonPHI(), DL,
204           TII->get(AMDGPU::S_OR_SAVEEXEC_B64), Dst)
205           .addReg(Src); // Saved EXEC
206 
207   BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_XOR_B64), AMDGPU::EXEC)
208           .addReg(AMDGPU::EXEC)
209           .addReg(Dst);
210 
211   Skip(MI, MI.getOperand(2));
212 
213   MI.eraseFromParent();
214 }
215 
216 void SILowerControlFlowPass::Break(MachineInstr &MI) {
217   MachineBasicBlock &MBB = *MI.getParent();
218   DebugLoc DL = MI.getDebugLoc();
219 
220   unsigned Dst = MI.getOperand(0).getReg();
221   unsigned Src = MI.getOperand(1).getReg();
222 
223   BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_OR_B64), Dst)
224           .addReg(AMDGPU::EXEC)
225           .addReg(Src);
226 
227   MI.eraseFromParent();
228 }
229 
230 void SILowerControlFlowPass::IfBreak(MachineInstr &MI) {
231   MachineBasicBlock &MBB = *MI.getParent();
232   DebugLoc DL = MI.getDebugLoc();
233 
234   unsigned Dst = MI.getOperand(0).getReg();
235   unsigned Vcc = MI.getOperand(1).getReg();
236   unsigned Src = MI.getOperand(2).getReg();
237 
238   BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_OR_B64), Dst)
239           .addReg(Vcc)
240           .addReg(Src);
241 
242   MI.eraseFromParent();
243 }
244 
245 void SILowerControlFlowPass::ElseBreak(MachineInstr &MI) {
246   MachineBasicBlock &MBB = *MI.getParent();
247   DebugLoc DL = MI.getDebugLoc();
248 
249   unsigned Dst = MI.getOperand(0).getReg();
250   unsigned Saved = MI.getOperand(1).getReg();
251   unsigned Src = MI.getOperand(2).getReg();
252 
253   BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_OR_B64), Dst)
254           .addReg(Saved)
255           .addReg(Src);
256 
257   MI.eraseFromParent();
258 }
259 
260 void SILowerControlFlowPass::Loop(MachineInstr &MI) {
261   MachineBasicBlock &MBB = *MI.getParent();
262   DebugLoc DL = MI.getDebugLoc();
263   unsigned Src = MI.getOperand(0).getReg();
264 
265   BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_ANDN2_B64), AMDGPU::EXEC)
266           .addReg(AMDGPU::EXEC)
267           .addReg(Src);
268 
269   BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_CBRANCH_EXECNZ))
270     .addOperand(MI.getOperand(1));
271 
272   MI.eraseFromParent();
273 }
274 
275 void SILowerControlFlowPass::EndCf(MachineInstr &MI) {
276   MachineBasicBlock &MBB = *MI.getParent();
277   DebugLoc DL = MI.getDebugLoc();
278   unsigned Reg = MI.getOperand(0).getReg();
279 
280   BuildMI(MBB, MBB.getFirstNonPHI(), DL,
281           TII->get(AMDGPU::S_OR_B64), AMDGPU::EXEC)
282           .addReg(AMDGPU::EXEC)
283           .addReg(Reg);
284 
285   MI.eraseFromParent();
286 }
287 
288 void SILowerControlFlowPass::Branch(MachineInstr &MI) {
289   if (MI.getOperand(0).getMBB() == MI.getParent()->getNextNode())
290     MI.eraseFromParent();
291 
292   // If these aren't equal, this is probably an infinite loop.
293 }
294 
295 void SILowerControlFlowPass::Kill(MachineInstr &MI) {
296   MachineBasicBlock &MBB = *MI.getParent();
297   DebugLoc DL = MI.getDebugLoc();
298   const MachineOperand &Op = MI.getOperand(0);
299 
300 #ifndef NDEBUG
301   const SIMachineFunctionInfo *MFI
302     = MBB.getParent()->getInfo<SIMachineFunctionInfo>();
303   // Kill is only allowed in pixel / geometry shaders.
304   assert(MFI->getShaderType() == ShaderType::PIXEL ||
305          MFI->getShaderType() == ShaderType::GEOMETRY);
306 #endif
307 
308   // Clear this thread from the exec mask if the operand is negative
309   if ((Op.isImm())) {
310     // Constant operand: Set exec mask to 0 or do nothing
311     if (Op.getImm() & 0x80000000) {
312       BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_MOV_B64), AMDGPU::EXEC)
313               .addImm(0);
314     }
315   } else {
316     BuildMI(MBB, &MI, DL, TII->get(AMDGPU::V_CMPX_LE_F32_e32))
317            .addImm(0)
318            .addOperand(Op);
319   }
320 
321   MI.eraseFromParent();
322 }
323 
324 void SILowerControlFlowPass::LoadM0(MachineInstr &MI, MachineInstr *MovRel, int Offset) {
325 
326   MachineBasicBlock &MBB = *MI.getParent();
327   DebugLoc DL = MI.getDebugLoc();
328   MachineBasicBlock::iterator I = MI;
329 
330   unsigned Save = MI.getOperand(1).getReg();
331   unsigned Idx = MI.getOperand(3).getReg();
332 
333   if (AMDGPU::SReg_32RegClass.contains(Idx)) {
334     if (Offset) {
335       BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0)
336               .addReg(Idx)
337               .addImm(Offset);
338     } else {
339       BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0)
340               .addReg(Idx);
341     }
342     MBB.insert(I, MovRel);
343   } else {
344 
345     assert(AMDGPU::SReg_64RegClass.contains(Save));
346     assert(AMDGPU::VGPR_32RegClass.contains(Idx));
347 
348     // Save the EXEC mask
349     BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_MOV_B64), Save)
350             .addReg(AMDGPU::EXEC);
351 
352     // Read the next variant into VCC (lower 32 bits) <- also loop target
353     BuildMI(MBB, &MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32),
354             AMDGPU::VCC_LO)
355             .addReg(Idx);
356 
357     // Move index from VCC into M0
358     BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0)
359             .addReg(AMDGPU::VCC_LO);
360 
361     // Compare the just read M0 value to all possible Idx values
362     BuildMI(MBB, &MI, DL, TII->get(AMDGPU::V_CMP_EQ_U32_e32))
363       .addReg(AMDGPU::M0)
364       .addReg(Idx);
365 
366     // Update EXEC, save the original EXEC value to VCC
367     BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_AND_SAVEEXEC_B64), AMDGPU::VCC)
368             .addReg(AMDGPU::VCC);
369 
370     if (Offset) {
371       BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0)
372               .addReg(AMDGPU::M0)
373               .addImm(Offset);
374     }
375     // Do the actual move
376     MBB.insert(I, MovRel);
377 
378     // Update EXEC, switch all done bits to 0 and all todo bits to 1
379     BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_XOR_B64), AMDGPU::EXEC)
380             .addReg(AMDGPU::EXEC)
381             .addReg(AMDGPU::VCC);
382 
383     // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover
384     BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_CBRANCH_EXECNZ))
385       .addImm(-7);
386 
387     // Restore EXEC
388     BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_MOV_B64), AMDGPU::EXEC)
389             .addReg(Save);
390 
391   }
392   MI.eraseFromParent();
393 }
394 
395 /// \param @VecReg The register which holds element zero of the vector
396 ///                 being addressed into.
397 /// \param[out] @Reg The base register to use in the indirect addressing instruction.
398 /// \param[in,out] @Offset As an input, this is the constant offset part of the
399 //                         indirect Index. e.g. v0 = v[VecReg + Offset]
400 //                         As an output, this is a constant value that needs
401 //                         to be added to the value stored in M0.
402 void SILowerControlFlowPass::computeIndirectRegAndOffset(unsigned VecReg,
403                                                          unsigned &Reg,
404                                                          int &Offset) {
405   unsigned SubReg = TRI->getSubReg(VecReg, AMDGPU::sub0);
406   if (!SubReg)
407     SubReg = VecReg;
408 
409   const TargetRegisterClass *RC = TRI->getPhysRegClass(SubReg);
410   int RegIdx = TRI->getHWRegIndex(SubReg) + Offset;
411 
412   if (RegIdx < 0) {
413     Offset = RegIdx;
414     RegIdx = 0;
415   } else {
416     Offset = 0;
417   }
418 
419   Reg = RC->getRegister(RegIdx);
420 }
421 
422 void SILowerControlFlowPass::IndirectSrc(MachineInstr &MI) {
423 
424   MachineBasicBlock &MBB = *MI.getParent();
425   DebugLoc DL = MI.getDebugLoc();
426 
427   unsigned Dst = MI.getOperand(0).getReg();
428   unsigned Vec = MI.getOperand(2).getReg();
429   int Off = MI.getOperand(4).getImm();
430   unsigned Reg;
431 
432   computeIndirectRegAndOffset(Vec, Reg, Off);
433 
434   MachineInstr *MovRel =
435     BuildMI(*MBB.getParent(), DL, TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst)
436             .addReg(Reg)
437             .addReg(AMDGPU::M0, RegState::Implicit)
438             .addReg(Vec, RegState::Implicit);
439 
440   LoadM0(MI, MovRel, Off);
441 }
442 
443 void SILowerControlFlowPass::IndirectDst(MachineInstr &MI) {
444 
445   MachineBasicBlock &MBB = *MI.getParent();
446   DebugLoc DL = MI.getDebugLoc();
447 
448   unsigned Dst = MI.getOperand(0).getReg();
449   int Off = MI.getOperand(4).getImm();
450   unsigned Val = MI.getOperand(5).getReg();
451   unsigned Reg;
452 
453   computeIndirectRegAndOffset(Dst, Reg, Off);
454 
455   MachineInstr *MovRel =
456     BuildMI(*MBB.getParent(), DL, TII->get(AMDGPU::V_MOVRELD_B32_e32))
457             .addReg(Reg, RegState::Define)
458             .addReg(Val)
459             .addReg(AMDGPU::M0, RegState::Implicit)
460             .addReg(Dst, RegState::Implicit);
461 
462   LoadM0(MI, MovRel, Off);
463 }
464 
465 bool SILowerControlFlowPass::runOnMachineFunction(MachineFunction &MF) {
466   TII = static_cast<const SIInstrInfo *>(MF.getSubtarget().getInstrInfo());
467   TRI =
468       static_cast<const SIRegisterInfo *>(MF.getSubtarget().getRegisterInfo());
469   SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
470 
471   bool HaveKill = false;
472   bool NeedWQM = false;
473   bool NeedFlat = false;
474   unsigned Depth = 0;
475 
476   for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
477        BI != BE; ++BI) {
478 
479     MachineBasicBlock &MBB = *BI;
480     MachineBasicBlock::iterator I, Next;
481     for (I = MBB.begin(); I != MBB.end(); I = Next) {
482       Next = std::next(I);
483 
484       MachineInstr &MI = *I;
485       if (TII->isWQM(MI.getOpcode()) || TII->isDS(MI.getOpcode()))
486         NeedWQM = true;
487 
488       // Flat uses m0 in case it needs to access LDS.
489       if (TII->isFLAT(MI.getOpcode()))
490         NeedFlat = true;
491 
492       switch (MI.getOpcode()) {
493         default: break;
494         case AMDGPU::SI_IF:
495           ++Depth;
496           If(MI);
497           break;
498 
499         case AMDGPU::SI_ELSE:
500           Else(MI);
501           break;
502 
503         case AMDGPU::SI_BREAK:
504           Break(MI);
505           break;
506 
507         case AMDGPU::SI_IF_BREAK:
508           IfBreak(MI);
509           break;
510 
511         case AMDGPU::SI_ELSE_BREAK:
512           ElseBreak(MI);
513           break;
514 
515         case AMDGPU::SI_LOOP:
516           ++Depth;
517           Loop(MI);
518           break;
519 
520         case AMDGPU::SI_END_CF:
521           if (--Depth == 0 && HaveKill) {
522             SkipIfDead(MI);
523             HaveKill = false;
524           }
525           EndCf(MI);
526           break;
527 
528         case AMDGPU::SI_KILL:
529           if (Depth == 0)
530             SkipIfDead(MI);
531           else
532             HaveKill = true;
533           Kill(MI);
534           break;
535 
536         case AMDGPU::S_BRANCH:
537           Branch(MI);
538           break;
539 
540         case AMDGPU::SI_INDIRECT_SRC:
541           IndirectSrc(MI);
542           break;
543 
544         case AMDGPU::SI_INDIRECT_DST_V1:
545         case AMDGPU::SI_INDIRECT_DST_V2:
546         case AMDGPU::SI_INDIRECT_DST_V4:
547         case AMDGPU::SI_INDIRECT_DST_V8:
548         case AMDGPU::SI_INDIRECT_DST_V16:
549           IndirectDst(MI);
550           break;
551       }
552     }
553   }
554 
555   if (NeedWQM && MFI->getShaderType() == ShaderType::PIXEL) {
556     MachineBasicBlock &MBB = MF.front();
557     BuildMI(MBB, MBB.getFirstNonPHI(), DebugLoc(), TII->get(AMDGPU::S_WQM_B64),
558             AMDGPU::EXEC).addReg(AMDGPU::EXEC);
559   }
560 
561   // FIXME: This seems inappropriate to do here.
562   if (NeedFlat && MFI->IsKernel) {
563     // Insert the prologue initializing the SGPRs pointing to the scratch space
564     // for flat accesses.
565     const MachineFrameInfo *FrameInfo = MF.getFrameInfo();
566 
567     // TODO: What to use with function calls?
568 
569     // FIXME: This is reporting stack size that is used in a scratch buffer
570     // rather than registers as well.
571     uint64_t StackSizeBytes = FrameInfo->getStackSize();
572 
573     int IndirectBegin
574       = static_cast<const AMDGPUInstrInfo*>(TII)->getIndirectIndexBegin(MF);
575     // Convert register index to 256-byte unit.
576     uint64_t StackOffset = IndirectBegin < 0 ? 0 : (4 * IndirectBegin / 256);
577 
578     assert((StackSizeBytes < 0xffff) && StackOffset < 0xffff &&
579            "Stack limits should be smaller than 16-bits");
580 
581     // Initialize the flat scratch register pair.
582     // TODO: Can we use one s_mov_b64 here?
583 
584     // Offset is in units of 256-bytes.
585     MachineBasicBlock &MBB = MF.front();
586     DebugLoc NoDL;
587     MachineBasicBlock::iterator Start = MBB.getFirstNonPHI();
588     const MCInstrDesc &SMovK = TII->get(AMDGPU::S_MOVK_I32);
589 
590     assert(isInt<16>(StackOffset) && isInt<16>(StackSizeBytes));
591 
592     BuildMI(MBB, Start, NoDL, SMovK, AMDGPU::FLAT_SCR_LO)
593       .addImm(StackOffset);
594 
595     // Documentation says size is "per-thread scratch size in bytes"
596     BuildMI(MBB, Start, NoDL, SMovK, AMDGPU::FLAT_SCR_HI)
597       .addImm(StackSizeBytes);
598   }
599 
600   return true;
601 }
602