1 //===-- SIWholeQuadMode.cpp - enter and suspend whole quad mode -----------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// \brief This pass adds instructions to enable whole quad mode for pixel
12 /// shaders.
13 ///
14 /// Whole quad mode is required for derivative computations, but it interferes
15 /// with shader side effects (stores and atomics). This pass is run on the
16 /// scheduled machine IR but before register coalescing, so that machine SSA is
17 /// available for analysis. It ensures that WQM is enabled when necessary, but
18 /// disabled around stores and atomics.
19 ///
20 /// When necessary, this pass creates a function prolog
21 ///
22 ///   S_MOV_B64 LiveMask, EXEC
23 ///   S_WQM_B64 EXEC, EXEC
24 ///
25 /// to enter WQM at the top of the function and surrounds blocks of Exact
26 /// instructions by
27 ///
28 ///   S_AND_SAVEEXEC_B64 Tmp, LiveMask
29 ///   ...
30 ///   S_MOV_B64 EXEC, Tmp
31 ///
32 /// In order to avoid excessive switching during sequences of Exact
33 /// instructions, the pass first analyzes which instructions must be run in WQM
34 /// (aka which instructions produce values that lead to derivative
35 /// computations).
36 ///
37 /// Basic blocks are always exited in WQM as long as some successor needs WQM.
38 ///
39 /// There is room for improvement given better control flow analysis:
40 ///
41 ///  (1) at the top level (outside of control flow statements, and as long as
42 ///      kill hasn't been used), one SGPR can be saved by recovering WQM from
43 ///      the LiveMask (this is implemented for the entry block).
44 ///
45 ///  (2) when entire regions (e.g. if-else blocks or entire loops) only
46 ///      consist of exact and don't-care instructions, the switch only has to
47 ///      be done at the entry and exit points rather than potentially in each
48 ///      block of the region.
49 ///
50 //===----------------------------------------------------------------------===//
51 
52 #include "AMDGPU.h"
53 #include "AMDGPUSubtarget.h"
54 #include "SIInstrInfo.h"
55 #include "SIMachineFunctionInfo.h"
56 #include "llvm/CodeGen/MachineFunction.h"
57 #include "llvm/CodeGen/MachineFunctionPass.h"
58 #include "llvm/CodeGen/MachineInstrBuilder.h"
59 #include "llvm/CodeGen/MachineRegisterInfo.h"
60 
61 using namespace llvm;
62 
63 #define DEBUG_TYPE "si-wqm"
64 
65 namespace {
66 
67 enum {
68   StateWQM = 0x1,
69   StateExact = 0x2,
70 };
71 
72 struct PrintState {
73 public:
74   explicit PrintState(int State) : State(State) {}
75 
76   int State;
77 };
78 
79 static raw_ostream &operator<<(raw_ostream &OS, const PrintState &PS) {
80   if (PS.State & StateWQM)
81     OS << "WQM";
82   if (PS.State & StateExact) {
83     if (PS.State & StateWQM)
84       OS << '|';
85     OS << "Exact";
86   }
87 
88   return OS;
89 }
90 
91 struct InstrInfo {
92   char Needs = 0;
93   char OutNeeds = 0;
94 };
95 
96 struct BlockInfo {
97   char Needs = 0;
98   char InNeeds = 0;
99   char OutNeeds = 0;
100 };
101 
102 struct WorkItem {
103   MachineBasicBlock *MBB = nullptr;
104   MachineInstr *MI = nullptr;
105 
106   WorkItem() {}
107   WorkItem(MachineBasicBlock *MBB) : MBB(MBB) {}
108   WorkItem(MachineInstr *MI) : MI(MI) {}
109 };
110 
111 class SIWholeQuadMode : public MachineFunctionPass {
112 private:
113   const SIInstrInfo *TII;
114   const SIRegisterInfo *TRI;
115   MachineRegisterInfo *MRI;
116   LiveIntervals *LIS;
117 
118   DenseMap<const MachineInstr *, InstrInfo> Instructions;
119   DenseMap<MachineBasicBlock *, BlockInfo> Blocks;
120   SmallVector<MachineInstr *, 1> LiveMaskQueries;
121 
122   void printInfo();
123 
124   void markInstruction(MachineInstr &MI, char Flag,
125                        std::vector<WorkItem> &Worklist);
126   void markUsesWQM(const MachineInstr &MI, std::vector<WorkItem> &Worklist);
127   char scanInstructions(MachineFunction &MF, std::vector<WorkItem> &Worklist);
128   void propagateInstruction(MachineInstr &MI, std::vector<WorkItem> &Worklist);
129   void propagateBlock(MachineBasicBlock &MBB, std::vector<WorkItem> &Worklist);
130   char analyzeFunction(MachineFunction &MF);
131 
132   bool requiresCorrectState(const MachineInstr &MI) const;
133 
134   MachineBasicBlock::iterator saveSCC(MachineBasicBlock &MBB,
135                                       MachineBasicBlock::iterator Before);
136   MachineBasicBlock::iterator
137   prepareInsertion(MachineBasicBlock &MBB, MachineBasicBlock::iterator First,
138                    MachineBasicBlock::iterator Last, bool PreferLast,
139                    bool SaveSCC);
140   void toExact(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
141                unsigned SaveWQM, unsigned LiveMaskReg);
142   void toWQM(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
143              unsigned SavedWQM);
144   void processBlock(MachineBasicBlock &MBB, unsigned LiveMaskReg, bool isEntry);
145 
146   void lowerLiveMaskQueries(unsigned LiveMaskReg);
147 
148 public:
149   static char ID;
150 
151   SIWholeQuadMode() :
152     MachineFunctionPass(ID) { }
153 
154   bool runOnMachineFunction(MachineFunction &MF) override;
155 
156   StringRef getPassName() const override { return "SI Whole Quad Mode"; }
157 
158   void getAnalysisUsage(AnalysisUsage &AU) const override {
159     AU.addRequired<LiveIntervals>();
160     AU.setPreservesCFG();
161     MachineFunctionPass::getAnalysisUsage(AU);
162   }
163 };
164 
165 } // End anonymous namespace
166 
167 char SIWholeQuadMode::ID = 0;
168 
169 INITIALIZE_PASS_BEGIN(SIWholeQuadMode, DEBUG_TYPE, "SI Whole Quad Mode", false,
170                       false)
171 INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
172 INITIALIZE_PASS_END(SIWholeQuadMode, DEBUG_TYPE, "SI Whole Quad Mode", false,
173                     false)
174 
175 char &llvm::SIWholeQuadModeID = SIWholeQuadMode::ID;
176 
177 FunctionPass *llvm::createSIWholeQuadModePass() {
178   return new SIWholeQuadMode;
179 }
180 
181 void SIWholeQuadMode::printInfo() {
182   for (const auto &BII : Blocks) {
183     dbgs() << "\nBB#" << BII.first->getNumber() << ":\n"
184            << "  InNeeds = " << PrintState(BII.second.InNeeds)
185            << ", Needs = " << PrintState(BII.second.Needs)
186            << ", OutNeeds = " << PrintState(BII.second.OutNeeds) << "\n\n";
187 
188     for (const MachineInstr &MI : *BII.first) {
189       auto III = Instructions.find(&MI);
190       if (III == Instructions.end())
191         continue;
192 
193       dbgs() << "  " << MI << "    Needs = " << PrintState(III->second.Needs)
194              << ", OutNeeds = " << PrintState(III->second.OutNeeds) << '\n';
195     }
196   }
197 }
198 
199 void SIWholeQuadMode::markInstruction(MachineInstr &MI, char Flag,
200                                       std::vector<WorkItem> &Worklist) {
201   InstrInfo &II = Instructions[&MI];
202 
203   assert(Flag == StateWQM || Flag == StateExact);
204 
205   // Ignore if the instruction is already marked. The typical case is that we
206   // mark an instruction WQM multiple times, but for atomics it can happen that
207   // Flag is StateWQM, but Needs is already set to StateExact. In this case,
208   // letting the atomic run in StateExact is correct as per the relevant specs.
209   if (II.Needs)
210     return;
211 
212   II.Needs = Flag;
213   Worklist.push_back(&MI);
214 }
215 
216 /// Mark all instructions defining the uses in \p MI as WQM.
217 void SIWholeQuadMode::markUsesWQM(const MachineInstr &MI,
218                                   std::vector<WorkItem> &Worklist) {
219   for (const MachineOperand &Use : MI.uses()) {
220     if (!Use.isReg() || !Use.isUse())
221       continue;
222 
223     unsigned Reg = Use.getReg();
224 
225     // Handle physical registers that we need to track; this is mostly relevant
226     // for VCC, which can appear as the (implicit) input of a uniform branch,
227     // e.g. when a loop counter is stored in a VGPR.
228     if (!TargetRegisterInfo::isVirtualRegister(Reg)) {
229       if (Reg == AMDGPU::EXEC)
230         continue;
231 
232       for (MCRegUnitIterator RegUnit(Reg, TRI); RegUnit.isValid(); ++RegUnit) {
233         LiveRange &LR = LIS->getRegUnit(*RegUnit);
234         const VNInfo *Value = LR.Query(LIS->getInstructionIndex(MI)).valueIn();
235         if (!Value)
236           continue;
237 
238         // Since we're in machine SSA, we do not need to track physical
239         // registers across basic blocks.
240         if (Value->isPHIDef())
241           continue;
242 
243         markInstruction(*LIS->getInstructionFromIndex(Value->def), StateWQM,
244                         Worklist);
245       }
246 
247       continue;
248     }
249 
250     for (MachineInstr &DefMI : MRI->def_instructions(Use.getReg()))
251       markInstruction(DefMI, StateWQM, Worklist);
252   }
253 }
254 
255 // Scan instructions to determine which ones require an Exact execmask and
256 // which ones seed WQM requirements.
257 char SIWholeQuadMode::scanInstructions(MachineFunction &MF,
258                                        std::vector<WorkItem> &Worklist) {
259   char GlobalFlags = 0;
260   bool WQMOutputs = MF.getFunction()->hasFnAttribute("amdgpu-ps-wqm-outputs");
261 
262   for (auto BI = MF.begin(), BE = MF.end(); BI != BE; ++BI) {
263     MachineBasicBlock &MBB = *BI;
264 
265     for (auto II = MBB.begin(), IE = MBB.end(); II != IE; ++II) {
266       MachineInstr &MI = *II;
267       unsigned Opcode = MI.getOpcode();
268       char Flags = 0;
269 
270       if (TII->isDS(Opcode)) {
271         Flags = StateWQM;
272       } else if (TII->isWQM(Opcode)) {
273         // Sampling instructions don't need to produce results for all pixels
274         // in a quad, they just require all inputs of a quad to have been
275         // computed for derivatives.
276         markUsesWQM(MI, Worklist);
277         GlobalFlags |= StateWQM;
278         continue;
279       } else if (TII->isDisableWQM(MI)) {
280         Flags = StateExact;
281       } else {
282         if (Opcode == AMDGPU::SI_PS_LIVE) {
283           LiveMaskQueries.push_back(&MI);
284         } else if (WQMOutputs) {
285           // The function is in machine SSA form, which means that physical
286           // VGPRs correspond to shader inputs and outputs. Inputs are
287           // only used, outputs are only defined.
288           for (const MachineOperand &MO : MI.defs()) {
289             if (!MO.isReg())
290               continue;
291 
292             unsigned Reg = MO.getReg();
293 
294             if (!TRI->isVirtualRegister(Reg) &&
295                 TRI->hasVGPRs(TRI->getPhysRegClass(Reg))) {
296               Flags = StateWQM;
297               break;
298             }
299           }
300         }
301 
302         if (!Flags)
303           continue;
304       }
305 
306       markInstruction(MI, Flags, Worklist);
307       GlobalFlags |= Flags;
308     }
309   }
310 
311   return GlobalFlags;
312 }
313 
314 void SIWholeQuadMode::propagateInstruction(MachineInstr &MI,
315                                            std::vector<WorkItem>& Worklist) {
316   MachineBasicBlock *MBB = MI.getParent();
317   InstrInfo II = Instructions[&MI]; // take a copy to prevent dangling references
318   BlockInfo &BI = Blocks[MBB];
319 
320   // Control flow-type instructions and stores to temporary memory that are
321   // followed by WQM computations must themselves be in WQM.
322   if ((II.OutNeeds & StateWQM) && !II.Needs &&
323       (MI.isTerminator() || (TII->usesVM_CNT(MI) && MI.mayStore()))) {
324     Instructions[&MI].Needs = StateWQM;
325     II.Needs = StateWQM;
326   }
327 
328   // Propagate to block level
329   BI.Needs |= II.Needs;
330   if ((BI.InNeeds | II.Needs) != BI.InNeeds) {
331     BI.InNeeds |= II.Needs;
332     Worklist.push_back(MBB);
333   }
334 
335   // Propagate backwards within block
336   if (MachineInstr *PrevMI = MI.getPrevNode()) {
337     char InNeeds = II.Needs | II.OutNeeds;
338     if (!PrevMI->isPHI()) {
339       InstrInfo &PrevII = Instructions[PrevMI];
340       if ((PrevII.OutNeeds | InNeeds) != PrevII.OutNeeds) {
341         PrevII.OutNeeds |= InNeeds;
342         Worklist.push_back(PrevMI);
343       }
344     }
345   }
346 
347   // Propagate WQM flag to instruction inputs
348   assert(II.Needs != (StateWQM | StateExact));
349 
350   if (II.Needs == StateWQM)
351     markUsesWQM(MI, Worklist);
352 }
353 
354 void SIWholeQuadMode::propagateBlock(MachineBasicBlock &MBB,
355                                      std::vector<WorkItem>& Worklist) {
356   BlockInfo BI = Blocks[&MBB]; // Make a copy to prevent dangling references.
357 
358   // Propagate through instructions
359   if (!MBB.empty()) {
360     MachineInstr *LastMI = &*MBB.rbegin();
361     InstrInfo &LastII = Instructions[LastMI];
362     if ((LastII.OutNeeds | BI.OutNeeds) != LastII.OutNeeds) {
363       LastII.OutNeeds |= BI.OutNeeds;
364       Worklist.push_back(LastMI);
365     }
366   }
367 
368   // Predecessor blocks must provide for our WQM/Exact needs.
369   for (MachineBasicBlock *Pred : MBB.predecessors()) {
370     BlockInfo &PredBI = Blocks[Pred];
371     if ((PredBI.OutNeeds | BI.InNeeds) == PredBI.OutNeeds)
372       continue;
373 
374     PredBI.OutNeeds |= BI.InNeeds;
375     PredBI.InNeeds |= BI.InNeeds;
376     Worklist.push_back(Pred);
377   }
378 
379   // All successors must be prepared to accept the same set of WQM/Exact data.
380   for (MachineBasicBlock *Succ : MBB.successors()) {
381     BlockInfo &SuccBI = Blocks[Succ];
382     if ((SuccBI.InNeeds | BI.OutNeeds) == SuccBI.InNeeds)
383       continue;
384 
385     SuccBI.InNeeds |= BI.OutNeeds;
386     Worklist.push_back(Succ);
387   }
388 }
389 
390 char SIWholeQuadMode::analyzeFunction(MachineFunction &MF) {
391   std::vector<WorkItem> Worklist;
392   char GlobalFlags = scanInstructions(MF, Worklist);
393 
394   while (!Worklist.empty()) {
395     WorkItem WI = Worklist.back();
396     Worklist.pop_back();
397 
398     if (WI.MI)
399       propagateInstruction(*WI.MI, Worklist);
400     else
401       propagateBlock(*WI.MBB, Worklist);
402   }
403 
404   return GlobalFlags;
405 }
406 
407 /// Whether \p MI really requires the exec state computed during analysis.
408 ///
409 /// Scalar instructions must occasionally be marked WQM for correct propagation
410 /// (e.g. thread masks leading up to branches), but when it comes to actual
411 /// execution, they don't care about EXEC.
412 bool SIWholeQuadMode::requiresCorrectState(const MachineInstr &MI) const {
413   if (MI.isTerminator())
414     return true;
415 
416   // Skip instructions that are not affected by EXEC
417   if (TII->isScalarUnit(MI))
418     return false;
419 
420   // Generic instructions such as COPY will either disappear by register
421   // coalescing or be lowered to SALU or VALU instructions.
422   if (MI.isTransient()) {
423     if (MI.getNumExplicitOperands() >= 1) {
424       const MachineOperand &Op = MI.getOperand(0);
425       if (Op.isReg()) {
426         if (TRI->isSGPRReg(*MRI, Op.getReg())) {
427           // SGPR instructions are not affected by EXEC
428           return false;
429         }
430       }
431     }
432   }
433 
434   return true;
435 }
436 
437 MachineBasicBlock::iterator
438 SIWholeQuadMode::saveSCC(MachineBasicBlock &MBB,
439                          MachineBasicBlock::iterator Before) {
440   unsigned SaveReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
441 
442   MachineInstr *Save =
443       BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::COPY), SaveReg)
444           .addReg(AMDGPU::SCC);
445   MachineInstr *Restore =
446       BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::COPY), AMDGPU::SCC)
447           .addReg(SaveReg);
448 
449   LIS->InsertMachineInstrInMaps(*Save);
450   LIS->InsertMachineInstrInMaps(*Restore);
451   LIS->createAndComputeVirtRegInterval(SaveReg);
452 
453   return Restore;
454 }
455 
456 // Return an iterator in the (inclusive) range [First, Last] at which
457 // instructions can be safely inserted, keeping in mind that some of the
458 // instructions we want to add necessarily clobber SCC.
459 MachineBasicBlock::iterator SIWholeQuadMode::prepareInsertion(
460     MachineBasicBlock &MBB, MachineBasicBlock::iterator First,
461     MachineBasicBlock::iterator Last, bool PreferLast, bool SaveSCC) {
462   if (!SaveSCC)
463     return PreferLast ? Last : First;
464 
465   LiveRange &LR = LIS->getRegUnit(*MCRegUnitIterator(AMDGPU::SCC, TRI));
466   auto MBBE = MBB.end();
467   SlotIndex FirstIdx = First != MBBE ? LIS->getInstructionIndex(*First)
468                                      : LIS->getMBBEndIdx(&MBB);
469   SlotIndex LastIdx =
470       Last != MBBE ? LIS->getInstructionIndex(*Last) : LIS->getMBBEndIdx(&MBB);
471   SlotIndex Idx = PreferLast ? LastIdx : FirstIdx;
472   const LiveRange::Segment *S;
473 
474   for (;;) {
475     S = LR.getSegmentContaining(Idx);
476     if (!S)
477       break;
478 
479     if (PreferLast) {
480       SlotIndex Next = S->start.getBaseIndex();
481       if (Next < FirstIdx)
482         break;
483       Idx = Next;
484     } else {
485       SlotIndex Next = S->end.getNextIndex().getBaseIndex();
486       if (Next > LastIdx)
487         break;
488       Idx = Next;
489     }
490   }
491 
492   MachineBasicBlock::iterator MBBI;
493 
494   if (MachineInstr *MI = LIS->getInstructionFromIndex(Idx))
495     MBBI = MI;
496   else {
497     assert(Idx == LIS->getMBBEndIdx(&MBB));
498     MBBI = MBB.end();
499   }
500 
501   if (S)
502     MBBI = saveSCC(MBB, MBBI);
503 
504   return MBBI;
505 }
506 
507 void SIWholeQuadMode::toExact(MachineBasicBlock &MBB,
508                               MachineBasicBlock::iterator Before,
509                               unsigned SaveWQM, unsigned LiveMaskReg) {
510   MachineInstr *MI;
511 
512   if (SaveWQM) {
513     MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::S_AND_SAVEEXEC_B64),
514                  SaveWQM)
515              .addReg(LiveMaskReg);
516   } else {
517     MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::S_AND_B64),
518                  AMDGPU::EXEC)
519              .addReg(AMDGPU::EXEC)
520              .addReg(LiveMaskReg);
521   }
522 
523   LIS->InsertMachineInstrInMaps(*MI);
524 }
525 
526 void SIWholeQuadMode::toWQM(MachineBasicBlock &MBB,
527                             MachineBasicBlock::iterator Before,
528                             unsigned SavedWQM) {
529   MachineInstr *MI;
530 
531   if (SavedWQM) {
532     MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::COPY), AMDGPU::EXEC)
533              .addReg(SavedWQM);
534   } else {
535     MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::S_WQM_B64),
536                  AMDGPU::EXEC)
537              .addReg(AMDGPU::EXEC);
538   }
539 
540   LIS->InsertMachineInstrInMaps(*MI);
541 }
542 
543 void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, unsigned LiveMaskReg,
544                                    bool isEntry) {
545   auto BII = Blocks.find(&MBB);
546   if (BII == Blocks.end())
547     return;
548 
549   const BlockInfo &BI = BII->second;
550 
551   if (!(BI.InNeeds & StateWQM))
552     return;
553 
554   // This is a non-entry block that is WQM throughout, so no need to do
555   // anything.
556   if (!isEntry && !(BI.Needs & StateExact) && BI.OutNeeds != StateExact)
557     return;
558 
559   DEBUG(dbgs() << "\nProcessing block BB#" << MBB.getNumber() << ":\n");
560 
561   unsigned SavedWQMReg = 0;
562   bool WQMFromExec = isEntry;
563   char State = isEntry ? StateExact : StateWQM;
564 
565   auto II = MBB.getFirstNonPHI(), IE = MBB.end();
566   if (isEntry)
567     ++II; // Skip the instruction that saves LiveMask
568 
569   MachineBasicBlock::iterator First = IE;
570   for (;;) {
571     MachineBasicBlock::iterator Next = II;
572     char Needs = 0;
573     char OutNeeds = 0;
574 
575     if (First == IE)
576       First = II;
577 
578     if (II != IE) {
579       MachineInstr &MI = *II;
580 
581       if (requiresCorrectState(MI)) {
582         auto III = Instructions.find(&MI);
583         if (III != Instructions.end()) {
584           Needs = III->second.Needs;
585           OutNeeds = III->second.OutNeeds;
586         }
587       }
588 
589       if (MI.isTerminator() && !Needs && OutNeeds == StateExact)
590         Needs = StateExact;
591 
592       if (MI.getOpcode() == AMDGPU::SI_ELSE && BI.OutNeeds == StateExact)
593         MI.getOperand(3).setImm(1);
594 
595       ++Next;
596     } else {
597       // End of basic block
598       if (BI.OutNeeds & StateWQM)
599         Needs = StateWQM;
600       else if (BI.OutNeeds == StateExact)
601         Needs = StateExact;
602     }
603 
604     if (Needs) {
605       if (Needs != State) {
606         MachineBasicBlock::iterator Before =
607             prepareInsertion(MBB, First, II, Needs == StateWQM,
608                              Needs == StateExact || WQMFromExec);
609 
610         if (Needs == StateExact) {
611           if (!WQMFromExec && (OutNeeds & StateWQM))
612             SavedWQMReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass);
613 
614           toExact(MBB, Before, SavedWQMReg, LiveMaskReg);
615         } else {
616           assert(WQMFromExec == (SavedWQMReg == 0));
617 
618           toWQM(MBB, Before, SavedWQMReg);
619 
620           if (SavedWQMReg) {
621             LIS->createAndComputeVirtRegInterval(SavedWQMReg);
622             SavedWQMReg = 0;
623           }
624         }
625 
626         State = Needs;
627       }
628 
629       First = IE;
630     }
631 
632     if (II == IE)
633       break;
634     II = Next;
635   }
636 }
637 
638 void SIWholeQuadMode::lowerLiveMaskQueries(unsigned LiveMaskReg) {
639   for (MachineInstr *MI : LiveMaskQueries) {
640     const DebugLoc &DL = MI->getDebugLoc();
641     unsigned Dest = MI->getOperand(0).getReg();
642     MachineInstr *Copy =
643         BuildMI(*MI->getParent(), MI, DL, TII->get(AMDGPU::COPY), Dest)
644             .addReg(LiveMaskReg);
645 
646     LIS->ReplaceMachineInstrInMaps(*MI, *Copy);
647     MI->eraseFromParent();
648   }
649 }
650 
651 bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) {
652   if (MF.getFunction()->getCallingConv() != CallingConv::AMDGPU_PS)
653     return false;
654 
655   Instructions.clear();
656   Blocks.clear();
657   LiveMaskQueries.clear();
658 
659   const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
660 
661   TII = ST.getInstrInfo();
662   TRI = &TII->getRegisterInfo();
663   MRI = &MF.getRegInfo();
664   LIS = &getAnalysis<LiveIntervals>();
665 
666   char GlobalFlags = analyzeFunction(MF);
667   if (!(GlobalFlags & StateWQM)) {
668     lowerLiveMaskQueries(AMDGPU::EXEC);
669     return !LiveMaskQueries.empty();
670   }
671 
672   // Store a copy of the original live mask when required
673   unsigned LiveMaskReg = 0;
674   {
675     MachineBasicBlock &Entry = MF.front();
676     MachineBasicBlock::iterator EntryMI = Entry.getFirstNonPHI();
677 
678     if (GlobalFlags & StateExact || !LiveMaskQueries.empty()) {
679       LiveMaskReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass);
680       MachineInstr *MI = BuildMI(Entry, EntryMI, DebugLoc(),
681                                  TII->get(AMDGPU::COPY), LiveMaskReg)
682                              .addReg(AMDGPU::EXEC);
683       LIS->InsertMachineInstrInMaps(*MI);
684     }
685 
686     if (GlobalFlags == StateWQM) {
687       // For a shader that needs only WQM, we can just set it once.
688       BuildMI(Entry, EntryMI, DebugLoc(), TII->get(AMDGPU::S_WQM_B64),
689               AMDGPU::EXEC)
690           .addReg(AMDGPU::EXEC);
691 
692       lowerLiveMaskQueries(LiveMaskReg);
693       // EntryMI may become invalid here
694       return true;
695     }
696   }
697 
698   DEBUG(printInfo());
699 
700   lowerLiveMaskQueries(LiveMaskReg);
701 
702   // Handle the general case
703   for (auto BII : Blocks)
704     processBlock(*BII.first, LiveMaskReg, BII.first == &*MF.begin());
705 
706   // Physical registers like SCC aren't tracked by default anyway, so just
707   // removing the ranges we computed is the simplest option for maintaining
708   // the analysis results.
709   LIS->removeRegUnit(*MCRegUnitIterator(AMDGPU::SCC, TRI));
710 
711   return true;
712 }
713