1 //===-- SIWholeQuadMode.cpp - enter and suspend whole quad mode -----------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 /// \file 11 /// \brief This pass adds instructions to enable whole quad mode for pixel 12 /// shaders. 13 /// 14 /// Whole quad mode is required for derivative computations, but it interferes 15 /// with shader side effects (stores and atomics). This pass is run on the 16 /// scheduled machine IR but before register coalescing, so that machine SSA is 17 /// available for analysis. It ensures that WQM is enabled when necessary, but 18 /// disabled around stores and atomics. 19 /// 20 /// When necessary, this pass creates a function prolog 21 /// 22 /// S_MOV_B64 LiveMask, EXEC 23 /// S_WQM_B64 EXEC, EXEC 24 /// 25 /// to enter WQM at the top of the function and surrounds blocks of Exact 26 /// instructions by 27 /// 28 /// S_AND_SAVEEXEC_B64 Tmp, LiveMask 29 /// ... 30 /// S_MOV_B64 EXEC, Tmp 31 /// 32 /// In order to avoid excessive switching during sequences of Exact 33 /// instructions, the pass first analyzes which instructions must be run in WQM 34 /// (aka which instructions produce values that lead to derivative 35 /// computations). 36 /// 37 /// Basic blocks are always exited in WQM as long as some successor needs WQM. 38 /// 39 /// There is room for improvement given better control flow analysis: 40 /// 41 /// (1) at the top level (outside of control flow statements, and as long as 42 /// kill hasn't been used), one SGPR can be saved by recovering WQM from 43 /// the LiveMask (this is implemented for the entry block). 44 /// 45 /// (2) when entire regions (e.g. if-else blocks or entire loops) only 46 /// consist of exact and don't-care instructions, the switch only has to 47 /// be done at the entry and exit points rather than potentially in each 48 /// block of the region. 49 /// 50 //===----------------------------------------------------------------------===// 51 52 #include "AMDGPU.h" 53 #include "AMDGPUSubtarget.h" 54 #include "SIInstrInfo.h" 55 #include "SIMachineFunctionInfo.h" 56 #include "llvm/CodeGen/MachineFunction.h" 57 #include "llvm/CodeGen/MachineFunctionPass.h" 58 #include "llvm/CodeGen/MachineInstrBuilder.h" 59 #include "llvm/CodeGen/MachineRegisterInfo.h" 60 61 using namespace llvm; 62 63 #define DEBUG_TYPE "si-wqm" 64 65 namespace { 66 67 enum { 68 StateWQM = 0x1, 69 StateExact = 0x2, 70 }; 71 72 struct PrintState { 73 public: 74 explicit PrintState(int State) : State(State) {} 75 76 int State; 77 }; 78 79 static raw_ostream &operator<<(raw_ostream &OS, const PrintState &PS) { 80 if (PS.State & StateWQM) 81 OS << "WQM"; 82 if (PS.State & StateExact) { 83 if (PS.State & StateWQM) 84 OS << '|'; 85 OS << "Exact"; 86 } 87 88 return OS; 89 } 90 91 struct InstrInfo { 92 char Needs = 0; 93 char OutNeeds = 0; 94 }; 95 96 struct BlockInfo { 97 char Needs = 0; 98 char InNeeds = 0; 99 char OutNeeds = 0; 100 }; 101 102 struct WorkItem { 103 MachineBasicBlock *MBB = nullptr; 104 MachineInstr *MI = nullptr; 105 106 WorkItem() {} 107 WorkItem(MachineBasicBlock *MBB) : MBB(MBB) {} 108 WorkItem(MachineInstr *MI) : MI(MI) {} 109 }; 110 111 class SIWholeQuadMode : public MachineFunctionPass { 112 private: 113 const SIInstrInfo *TII; 114 const SIRegisterInfo *TRI; 115 MachineRegisterInfo *MRI; 116 LiveIntervals *LIS; 117 118 DenseMap<const MachineInstr *, InstrInfo> Instructions; 119 DenseMap<MachineBasicBlock *, BlockInfo> Blocks; 120 SmallVector<MachineInstr *, 1> LiveMaskQueries; 121 122 void printInfo(); 123 124 void markInstruction(MachineInstr &MI, char Flag, 125 std::vector<WorkItem> &Worklist); 126 void markUsesWQM(const MachineInstr &MI, std::vector<WorkItem> &Worklist); 127 char scanInstructions(MachineFunction &MF, std::vector<WorkItem> &Worklist); 128 void propagateInstruction(MachineInstr &MI, std::vector<WorkItem> &Worklist); 129 void propagateBlock(MachineBasicBlock &MBB, std::vector<WorkItem> &Worklist); 130 char analyzeFunction(MachineFunction &MF); 131 132 bool requiresCorrectState(const MachineInstr &MI) const; 133 134 MachineBasicBlock::iterator saveSCC(MachineBasicBlock &MBB, 135 MachineBasicBlock::iterator Before); 136 MachineBasicBlock::iterator 137 prepareInsertion(MachineBasicBlock &MBB, MachineBasicBlock::iterator First, 138 MachineBasicBlock::iterator Last, bool PreferLast, 139 bool SaveSCC); 140 void toExact(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before, 141 unsigned SaveWQM, unsigned LiveMaskReg); 142 void toWQM(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before, 143 unsigned SavedWQM); 144 void processBlock(MachineBasicBlock &MBB, unsigned LiveMaskReg, bool isEntry); 145 146 void lowerLiveMaskQueries(unsigned LiveMaskReg); 147 148 public: 149 static char ID; 150 151 SIWholeQuadMode() : 152 MachineFunctionPass(ID) { } 153 154 bool runOnMachineFunction(MachineFunction &MF) override; 155 156 StringRef getPassName() const override { return "SI Whole Quad Mode"; } 157 158 void getAnalysisUsage(AnalysisUsage &AU) const override { 159 AU.addRequired<LiveIntervals>(); 160 AU.setPreservesCFG(); 161 MachineFunctionPass::getAnalysisUsage(AU); 162 } 163 }; 164 165 } // End anonymous namespace 166 167 char SIWholeQuadMode::ID = 0; 168 169 INITIALIZE_PASS_BEGIN(SIWholeQuadMode, DEBUG_TYPE, "SI Whole Quad Mode", false, 170 false) 171 INITIALIZE_PASS_DEPENDENCY(LiveIntervals) 172 INITIALIZE_PASS_END(SIWholeQuadMode, DEBUG_TYPE, "SI Whole Quad Mode", false, 173 false) 174 175 char &llvm::SIWholeQuadModeID = SIWholeQuadMode::ID; 176 177 FunctionPass *llvm::createSIWholeQuadModePass() { 178 return new SIWholeQuadMode; 179 } 180 181 void SIWholeQuadMode::printInfo() { 182 for (const auto &BII : Blocks) { 183 dbgs() << "\nBB#" << BII.first->getNumber() << ":\n" 184 << " InNeeds = " << PrintState(BII.second.InNeeds) 185 << ", Needs = " << PrintState(BII.second.Needs) 186 << ", OutNeeds = " << PrintState(BII.second.OutNeeds) << "\n\n"; 187 188 for (const MachineInstr &MI : *BII.first) { 189 auto III = Instructions.find(&MI); 190 if (III == Instructions.end()) 191 continue; 192 193 dbgs() << " " << MI << " Needs = " << PrintState(III->second.Needs) 194 << ", OutNeeds = " << PrintState(III->second.OutNeeds) << '\n'; 195 } 196 } 197 } 198 199 void SIWholeQuadMode::markInstruction(MachineInstr &MI, char Flag, 200 std::vector<WorkItem> &Worklist) { 201 InstrInfo &II = Instructions[&MI]; 202 203 assert(Flag == StateWQM || Flag == StateExact); 204 205 // Ignore if the instruction is already marked. The typical case is that we 206 // mark an instruction WQM multiple times, but for atomics it can happen that 207 // Flag is StateWQM, but Needs is already set to StateExact. In this case, 208 // letting the atomic run in StateExact is correct as per the relevant specs. 209 if (II.Needs) 210 return; 211 212 II.Needs = Flag; 213 Worklist.push_back(&MI); 214 } 215 216 /// Mark all instructions defining the uses in \p MI as WQM. 217 void SIWholeQuadMode::markUsesWQM(const MachineInstr &MI, 218 std::vector<WorkItem> &Worklist) { 219 for (const MachineOperand &Use : MI.uses()) { 220 if (!Use.isReg() || !Use.isUse()) 221 continue; 222 223 unsigned Reg = Use.getReg(); 224 225 // Handle physical registers that we need to track; this is mostly relevant 226 // for VCC, which can appear as the (implicit) input of a uniform branch, 227 // e.g. when a loop counter is stored in a VGPR. 228 if (!TargetRegisterInfo::isVirtualRegister(Reg)) { 229 if (Reg == AMDGPU::EXEC) 230 continue; 231 232 for (MCRegUnitIterator RegUnit(Reg, TRI); RegUnit.isValid(); ++RegUnit) { 233 LiveRange &LR = LIS->getRegUnit(*RegUnit); 234 const VNInfo *Value = LR.Query(LIS->getInstructionIndex(MI)).valueIn(); 235 if (!Value) 236 continue; 237 238 // Since we're in machine SSA, we do not need to track physical 239 // registers across basic blocks. 240 if (Value->isPHIDef()) 241 continue; 242 243 markInstruction(*LIS->getInstructionFromIndex(Value->def), StateWQM, 244 Worklist); 245 } 246 247 continue; 248 } 249 250 for (MachineInstr &DefMI : MRI->def_instructions(Use.getReg())) 251 markInstruction(DefMI, StateWQM, Worklist); 252 } 253 } 254 255 // Scan instructions to determine which ones require an Exact execmask and 256 // which ones seed WQM requirements. 257 char SIWholeQuadMode::scanInstructions(MachineFunction &MF, 258 std::vector<WorkItem> &Worklist) { 259 char GlobalFlags = 0; 260 bool WQMOutputs = MF.getFunction()->hasFnAttribute("amdgpu-ps-wqm-outputs"); 261 262 for (auto BI = MF.begin(), BE = MF.end(); BI != BE; ++BI) { 263 MachineBasicBlock &MBB = *BI; 264 265 for (auto II = MBB.begin(), IE = MBB.end(); II != IE; ++II) { 266 MachineInstr &MI = *II; 267 unsigned Opcode = MI.getOpcode(); 268 char Flags = 0; 269 270 if (TII->isDS(Opcode)) { 271 Flags = StateWQM; 272 } else if (TII->isWQM(Opcode)) { 273 // Sampling instructions don't need to produce results for all pixels 274 // in a quad, they just require all inputs of a quad to have been 275 // computed for derivatives. 276 markUsesWQM(MI, Worklist); 277 GlobalFlags |= StateWQM; 278 continue; 279 } else if (TII->isDisableWQM(MI)) { 280 Flags = StateExact; 281 } else { 282 if (Opcode == AMDGPU::SI_PS_LIVE) { 283 LiveMaskQueries.push_back(&MI); 284 } else if (WQMOutputs) { 285 // The function is in machine SSA form, which means that physical 286 // VGPRs correspond to shader inputs and outputs. Inputs are 287 // only used, outputs are only defined. 288 for (const MachineOperand &MO : MI.defs()) { 289 if (!MO.isReg()) 290 continue; 291 292 unsigned Reg = MO.getReg(); 293 294 if (!TRI->isVirtualRegister(Reg) && 295 TRI->hasVGPRs(TRI->getPhysRegClass(Reg))) { 296 Flags = StateWQM; 297 break; 298 } 299 } 300 } 301 302 if (!Flags) 303 continue; 304 } 305 306 markInstruction(MI, Flags, Worklist); 307 GlobalFlags |= Flags; 308 } 309 } 310 311 return GlobalFlags; 312 } 313 314 void SIWholeQuadMode::propagateInstruction(MachineInstr &MI, 315 std::vector<WorkItem>& Worklist) { 316 MachineBasicBlock *MBB = MI.getParent(); 317 InstrInfo II = Instructions[&MI]; // take a copy to prevent dangling references 318 BlockInfo &BI = Blocks[MBB]; 319 320 // Control flow-type instructions and stores to temporary memory that are 321 // followed by WQM computations must themselves be in WQM. 322 if ((II.OutNeeds & StateWQM) && !II.Needs && 323 (MI.isTerminator() || (TII->usesVM_CNT(MI) && MI.mayStore()))) { 324 Instructions[&MI].Needs = StateWQM; 325 II.Needs = StateWQM; 326 } 327 328 // Propagate to block level 329 BI.Needs |= II.Needs; 330 if ((BI.InNeeds | II.Needs) != BI.InNeeds) { 331 BI.InNeeds |= II.Needs; 332 Worklist.push_back(MBB); 333 } 334 335 // Propagate backwards within block 336 if (MachineInstr *PrevMI = MI.getPrevNode()) { 337 char InNeeds = II.Needs | II.OutNeeds; 338 if (!PrevMI->isPHI()) { 339 InstrInfo &PrevII = Instructions[PrevMI]; 340 if ((PrevII.OutNeeds | InNeeds) != PrevII.OutNeeds) { 341 PrevII.OutNeeds |= InNeeds; 342 Worklist.push_back(PrevMI); 343 } 344 } 345 } 346 347 // Propagate WQM flag to instruction inputs 348 assert(II.Needs != (StateWQM | StateExact)); 349 350 if (II.Needs == StateWQM) 351 markUsesWQM(MI, Worklist); 352 } 353 354 void SIWholeQuadMode::propagateBlock(MachineBasicBlock &MBB, 355 std::vector<WorkItem>& Worklist) { 356 BlockInfo BI = Blocks[&MBB]; // Make a copy to prevent dangling references. 357 358 // Propagate through instructions 359 if (!MBB.empty()) { 360 MachineInstr *LastMI = &*MBB.rbegin(); 361 InstrInfo &LastII = Instructions[LastMI]; 362 if ((LastII.OutNeeds | BI.OutNeeds) != LastII.OutNeeds) { 363 LastII.OutNeeds |= BI.OutNeeds; 364 Worklist.push_back(LastMI); 365 } 366 } 367 368 // Predecessor blocks must provide for our WQM/Exact needs. 369 for (MachineBasicBlock *Pred : MBB.predecessors()) { 370 BlockInfo &PredBI = Blocks[Pred]; 371 if ((PredBI.OutNeeds | BI.InNeeds) == PredBI.OutNeeds) 372 continue; 373 374 PredBI.OutNeeds |= BI.InNeeds; 375 PredBI.InNeeds |= BI.InNeeds; 376 Worklist.push_back(Pred); 377 } 378 379 // All successors must be prepared to accept the same set of WQM/Exact data. 380 for (MachineBasicBlock *Succ : MBB.successors()) { 381 BlockInfo &SuccBI = Blocks[Succ]; 382 if ((SuccBI.InNeeds | BI.OutNeeds) == SuccBI.InNeeds) 383 continue; 384 385 SuccBI.InNeeds |= BI.OutNeeds; 386 Worklist.push_back(Succ); 387 } 388 } 389 390 char SIWholeQuadMode::analyzeFunction(MachineFunction &MF) { 391 std::vector<WorkItem> Worklist; 392 char GlobalFlags = scanInstructions(MF, Worklist); 393 394 while (!Worklist.empty()) { 395 WorkItem WI = Worklist.back(); 396 Worklist.pop_back(); 397 398 if (WI.MI) 399 propagateInstruction(*WI.MI, Worklist); 400 else 401 propagateBlock(*WI.MBB, Worklist); 402 } 403 404 return GlobalFlags; 405 } 406 407 /// Whether \p MI really requires the exec state computed during analysis. 408 /// 409 /// Scalar instructions must occasionally be marked WQM for correct propagation 410 /// (e.g. thread masks leading up to branches), but when it comes to actual 411 /// execution, they don't care about EXEC. 412 bool SIWholeQuadMode::requiresCorrectState(const MachineInstr &MI) const { 413 if (MI.isTerminator()) 414 return true; 415 416 // Skip instructions that are not affected by EXEC 417 if (TII->isScalarUnit(MI)) 418 return false; 419 420 // Generic instructions such as COPY will either disappear by register 421 // coalescing or be lowered to SALU or VALU instructions. 422 if (MI.isTransient()) { 423 if (MI.getNumExplicitOperands() >= 1) { 424 const MachineOperand &Op = MI.getOperand(0); 425 if (Op.isReg()) { 426 if (TRI->isSGPRReg(*MRI, Op.getReg())) { 427 // SGPR instructions are not affected by EXEC 428 return false; 429 } 430 } 431 } 432 } 433 434 return true; 435 } 436 437 MachineBasicBlock::iterator 438 SIWholeQuadMode::saveSCC(MachineBasicBlock &MBB, 439 MachineBasicBlock::iterator Before) { 440 unsigned SaveReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); 441 442 MachineInstr *Save = 443 BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::COPY), SaveReg) 444 .addReg(AMDGPU::SCC); 445 MachineInstr *Restore = 446 BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::COPY), AMDGPU::SCC) 447 .addReg(SaveReg); 448 449 LIS->InsertMachineInstrInMaps(*Save); 450 LIS->InsertMachineInstrInMaps(*Restore); 451 LIS->createAndComputeVirtRegInterval(SaveReg); 452 453 return Restore; 454 } 455 456 // Return an iterator in the (inclusive) range [First, Last] at which 457 // instructions can be safely inserted, keeping in mind that some of the 458 // instructions we want to add necessarily clobber SCC. 459 MachineBasicBlock::iterator SIWholeQuadMode::prepareInsertion( 460 MachineBasicBlock &MBB, MachineBasicBlock::iterator First, 461 MachineBasicBlock::iterator Last, bool PreferLast, bool SaveSCC) { 462 if (!SaveSCC) 463 return PreferLast ? Last : First; 464 465 LiveRange &LR = LIS->getRegUnit(*MCRegUnitIterator(AMDGPU::SCC, TRI)); 466 auto MBBE = MBB.end(); 467 SlotIndex FirstIdx = First != MBBE ? LIS->getInstructionIndex(*First) 468 : LIS->getMBBEndIdx(&MBB); 469 SlotIndex LastIdx = 470 Last != MBBE ? LIS->getInstructionIndex(*Last) : LIS->getMBBEndIdx(&MBB); 471 SlotIndex Idx = PreferLast ? LastIdx : FirstIdx; 472 const LiveRange::Segment *S; 473 474 for (;;) { 475 S = LR.getSegmentContaining(Idx); 476 if (!S) 477 break; 478 479 if (PreferLast) { 480 SlotIndex Next = S->start.getBaseIndex(); 481 if (Next < FirstIdx) 482 break; 483 Idx = Next; 484 } else { 485 SlotIndex Next = S->end.getNextIndex().getBaseIndex(); 486 if (Next > LastIdx) 487 break; 488 Idx = Next; 489 } 490 } 491 492 MachineBasicBlock::iterator MBBI; 493 494 if (MachineInstr *MI = LIS->getInstructionFromIndex(Idx)) 495 MBBI = MI; 496 else { 497 assert(Idx == LIS->getMBBEndIdx(&MBB)); 498 MBBI = MBB.end(); 499 } 500 501 if (S) 502 MBBI = saveSCC(MBB, MBBI); 503 504 return MBBI; 505 } 506 507 void SIWholeQuadMode::toExact(MachineBasicBlock &MBB, 508 MachineBasicBlock::iterator Before, 509 unsigned SaveWQM, unsigned LiveMaskReg) { 510 MachineInstr *MI; 511 512 if (SaveWQM) { 513 MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::S_AND_SAVEEXEC_B64), 514 SaveWQM) 515 .addReg(LiveMaskReg); 516 } else { 517 MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::S_AND_B64), 518 AMDGPU::EXEC) 519 .addReg(AMDGPU::EXEC) 520 .addReg(LiveMaskReg); 521 } 522 523 LIS->InsertMachineInstrInMaps(*MI); 524 } 525 526 void SIWholeQuadMode::toWQM(MachineBasicBlock &MBB, 527 MachineBasicBlock::iterator Before, 528 unsigned SavedWQM) { 529 MachineInstr *MI; 530 531 if (SavedWQM) { 532 MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::COPY), AMDGPU::EXEC) 533 .addReg(SavedWQM); 534 } else { 535 MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::S_WQM_B64), 536 AMDGPU::EXEC) 537 .addReg(AMDGPU::EXEC); 538 } 539 540 LIS->InsertMachineInstrInMaps(*MI); 541 } 542 543 void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, unsigned LiveMaskReg, 544 bool isEntry) { 545 auto BII = Blocks.find(&MBB); 546 if (BII == Blocks.end()) 547 return; 548 549 const BlockInfo &BI = BII->second; 550 551 if (!(BI.InNeeds & StateWQM)) 552 return; 553 554 // This is a non-entry block that is WQM throughout, so no need to do 555 // anything. 556 if (!isEntry && !(BI.Needs & StateExact) && BI.OutNeeds != StateExact) 557 return; 558 559 DEBUG(dbgs() << "\nProcessing block BB#" << MBB.getNumber() << ":\n"); 560 561 unsigned SavedWQMReg = 0; 562 bool WQMFromExec = isEntry; 563 char State = isEntry ? StateExact : StateWQM; 564 565 auto II = MBB.getFirstNonPHI(), IE = MBB.end(); 566 if (isEntry) 567 ++II; // Skip the instruction that saves LiveMask 568 569 MachineBasicBlock::iterator First = IE; 570 for (;;) { 571 MachineBasicBlock::iterator Next = II; 572 char Needs = 0; 573 char OutNeeds = 0; 574 575 if (First == IE) 576 First = II; 577 578 if (II != IE) { 579 MachineInstr &MI = *II; 580 581 if (requiresCorrectState(MI)) { 582 auto III = Instructions.find(&MI); 583 if (III != Instructions.end()) { 584 Needs = III->second.Needs; 585 OutNeeds = III->second.OutNeeds; 586 } 587 } 588 589 if (MI.isTerminator() && !Needs && OutNeeds == StateExact) 590 Needs = StateExact; 591 592 if (MI.getOpcode() == AMDGPU::SI_ELSE && BI.OutNeeds == StateExact) 593 MI.getOperand(3).setImm(1); 594 595 ++Next; 596 } else { 597 // End of basic block 598 if (BI.OutNeeds & StateWQM) 599 Needs = StateWQM; 600 else if (BI.OutNeeds == StateExact) 601 Needs = StateExact; 602 } 603 604 if (Needs) { 605 if (Needs != State) { 606 MachineBasicBlock::iterator Before = 607 prepareInsertion(MBB, First, II, Needs == StateWQM, 608 Needs == StateExact || WQMFromExec); 609 610 if (Needs == StateExact) { 611 if (!WQMFromExec && (OutNeeds & StateWQM)) 612 SavedWQMReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass); 613 614 toExact(MBB, Before, SavedWQMReg, LiveMaskReg); 615 } else { 616 assert(WQMFromExec == (SavedWQMReg == 0)); 617 618 toWQM(MBB, Before, SavedWQMReg); 619 620 if (SavedWQMReg) { 621 LIS->createAndComputeVirtRegInterval(SavedWQMReg); 622 SavedWQMReg = 0; 623 } 624 } 625 626 State = Needs; 627 } 628 629 First = IE; 630 } 631 632 if (II == IE) 633 break; 634 II = Next; 635 } 636 } 637 638 void SIWholeQuadMode::lowerLiveMaskQueries(unsigned LiveMaskReg) { 639 for (MachineInstr *MI : LiveMaskQueries) { 640 const DebugLoc &DL = MI->getDebugLoc(); 641 unsigned Dest = MI->getOperand(0).getReg(); 642 MachineInstr *Copy = 643 BuildMI(*MI->getParent(), MI, DL, TII->get(AMDGPU::COPY), Dest) 644 .addReg(LiveMaskReg); 645 646 LIS->ReplaceMachineInstrInMaps(*MI, *Copy); 647 MI->eraseFromParent(); 648 } 649 } 650 651 bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) { 652 if (MF.getFunction()->getCallingConv() != CallingConv::AMDGPU_PS) 653 return false; 654 655 Instructions.clear(); 656 Blocks.clear(); 657 LiveMaskQueries.clear(); 658 659 const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); 660 661 TII = ST.getInstrInfo(); 662 TRI = &TII->getRegisterInfo(); 663 MRI = &MF.getRegInfo(); 664 LIS = &getAnalysis<LiveIntervals>(); 665 666 char GlobalFlags = analyzeFunction(MF); 667 if (!(GlobalFlags & StateWQM)) { 668 lowerLiveMaskQueries(AMDGPU::EXEC); 669 return !LiveMaskQueries.empty(); 670 } 671 672 // Store a copy of the original live mask when required 673 unsigned LiveMaskReg = 0; 674 { 675 MachineBasicBlock &Entry = MF.front(); 676 MachineBasicBlock::iterator EntryMI = Entry.getFirstNonPHI(); 677 678 if (GlobalFlags & StateExact || !LiveMaskQueries.empty()) { 679 LiveMaskReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass); 680 MachineInstr *MI = BuildMI(Entry, EntryMI, DebugLoc(), 681 TII->get(AMDGPU::COPY), LiveMaskReg) 682 .addReg(AMDGPU::EXEC); 683 LIS->InsertMachineInstrInMaps(*MI); 684 } 685 686 if (GlobalFlags == StateWQM) { 687 // For a shader that needs only WQM, we can just set it once. 688 BuildMI(Entry, EntryMI, DebugLoc(), TII->get(AMDGPU::S_WQM_B64), 689 AMDGPU::EXEC) 690 .addReg(AMDGPU::EXEC); 691 692 lowerLiveMaskQueries(LiveMaskReg); 693 // EntryMI may become invalid here 694 return true; 695 } 696 } 697 698 DEBUG(printInfo()); 699 700 lowerLiveMaskQueries(LiveMaskReg); 701 702 // Handle the general case 703 for (auto BII : Blocks) 704 processBlock(*BII.first, LiveMaskReg, BII.first == &*MF.begin()); 705 706 // Physical registers like SCC aren't tracked by default anyway, so just 707 // removing the ranges we computed is the simplest option for maintaining 708 // the analysis results. 709 LIS->removeRegUnit(*MCRegUnitIterator(AMDGPU::SCC, TRI)); 710 711 return true; 712 } 713