1 //===----------------------- SIFrameLowering.cpp --------------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //==-----------------------------------------------------------------------===// 9 10 #include "SIFrameLowering.h" 11 #include "SIInstrInfo.h" 12 #include "SIMachineFunctionInfo.h" 13 #include "SIRegisterInfo.h" 14 #include "llvm/CodeGen/MachineFrameInfo.h" 15 #include "llvm/CodeGen/MachineFunction.h" 16 #include "llvm/CodeGen/MachineInstrBuilder.h" 17 #include "llvm/CodeGen/RegisterScavenging.h" 18 19 using namespace llvm; 20 21 22 static bool hasOnlySGPRSpills(const SIMachineFunctionInfo *FuncInfo, 23 const MachineFrameInfo *FrameInfo) { 24 return FuncInfo->hasSpilledSGPRs() && 25 (!FuncInfo->hasSpilledVGPRs() && !FuncInfo->hasNonSpillStackObjects()); 26 } 27 28 static ArrayRef<MCPhysReg> getAllSGPR128() { 29 return makeArrayRef(AMDGPU::SReg_128RegClass.begin(), 30 AMDGPU::SReg_128RegClass.getNumRegs()); 31 } 32 33 static ArrayRef<MCPhysReg> getAllSGPRs() { 34 return makeArrayRef(AMDGPU::SGPR_32RegClass.begin(), 35 AMDGPU::SGPR_32RegClass.getNumRegs()); 36 } 37 38 void SIFrameLowering::emitPrologue(MachineFunction &MF, 39 MachineBasicBlock &MBB) const { 40 if (!MF.getFrameInfo()->hasStackObjects()) 41 return; 42 43 assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported"); 44 45 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 46 47 // If we only have SGPR spills, we won't actually be using scratch memory 48 // since these spill to VGPRs. 49 // 50 // FIXME: We should be cleaning up these unused SGPR spill frame indices 51 // somewhere. 52 if (hasOnlySGPRSpills(MFI, MF.getFrameInfo())) 53 return; 54 55 const SIInstrInfo *TII = 56 static_cast<const SIInstrInfo *>(MF.getSubtarget().getInstrInfo()); 57 const SIRegisterInfo *TRI = &TII->getRegisterInfo(); 58 const AMDGPUSubtarget &ST = MF.getSubtarget<AMDGPUSubtarget>(); 59 MachineRegisterInfo &MRI = MF.getRegInfo(); 60 MachineBasicBlock::iterator I = MBB.begin(); 61 62 // We need to insert initialization of the scratch resource descriptor. 63 unsigned ScratchRsrcReg = MFI->getScratchRSrcReg(); 64 assert(ScratchRsrcReg != AMDGPU::NoRegister); 65 66 unsigned ScratchWaveOffsetReg = MFI->getScratchWaveOffsetReg(); 67 assert(ScratchWaveOffsetReg != AMDGPU::NoRegister); 68 69 unsigned PreloadedScratchWaveOffsetReg = TRI->getPreloadedValue( 70 MF, SIRegisterInfo::PRIVATE_SEGMENT_WAVE_BYTE_OFFSET); 71 72 unsigned PreloadedPrivateBufferReg = AMDGPU::NoRegister; 73 if (ST.isAmdHsaOS()) { 74 PreloadedPrivateBufferReg = TRI->getPreloadedValue( 75 MF, SIRegisterInfo::PRIVATE_SEGMENT_BUFFER); 76 } 77 78 if (MFI->hasFlatScratchInit()) { 79 // We don't need this if we only have spills since there is no user facing 80 // scratch. 81 82 // TODO: If we know we don't have flat instructions earlier, we can omit 83 // this from the input registers. 84 // 85 // TODO: We only need to know if we access scratch space through a flat 86 // pointer. Because we only detect if flat instructions are used at all, 87 // this will be used more often than necessary on VI. 88 89 DebugLoc DL; 90 91 unsigned FlatScratchInitReg 92 = TRI->getPreloadedValue(MF, SIRegisterInfo::FLAT_SCRATCH_INIT); 93 94 MRI.addLiveIn(FlatScratchInitReg); 95 MBB.addLiveIn(FlatScratchInitReg); 96 97 // Copy the size in bytes. 98 unsigned FlatScrInitHi = TRI->getSubReg(FlatScratchInitReg, AMDGPU::sub1); 99 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), AMDGPU::FLAT_SCR_LO) 100 .addReg(FlatScrInitHi, RegState::Kill); 101 102 unsigned FlatScrInitLo = TRI->getSubReg(FlatScratchInitReg, AMDGPU::sub0); 103 104 // Add wave offset in bytes to private base offset. 105 // See comment in AMDKernelCodeT.h for enable_sgpr_flat_scratch_init. 106 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADD_U32), FlatScrInitLo) 107 .addReg(FlatScrInitLo) 108 .addReg(ScratchWaveOffsetReg); 109 110 // Convert offset to 256-byte units. 111 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_LSHR_B32), AMDGPU::FLAT_SCR_HI) 112 .addReg(FlatScrInitLo, RegState::Kill) 113 .addImm(8); 114 } 115 116 // If we reserved the original input registers, we don't need to copy to the 117 // reserved registers. 118 if (ScratchRsrcReg == PreloadedPrivateBufferReg) { 119 // We should always reserve these 5 registers at the same time. 120 assert(ScratchWaveOffsetReg == PreloadedScratchWaveOffsetReg && 121 "scratch wave offset and private segment buffer inconsistent"); 122 return; 123 } 124 125 126 // We added live-ins during argument lowering, but since they were not used 127 // they were deleted. We're adding the uses now, so add them back. 128 MRI.addLiveIn(PreloadedScratchWaveOffsetReg); 129 MBB.addLiveIn(PreloadedScratchWaveOffsetReg); 130 131 if (ST.isAmdHsaOS()) { 132 MRI.addLiveIn(PreloadedPrivateBufferReg); 133 MBB.addLiveIn(PreloadedPrivateBufferReg); 134 } 135 136 if (!ST.hasSGPRInitBug()) { 137 // We reserved the last registers for this. Shift it down to the end of those 138 // which were actually used. 139 // 140 // FIXME: It might be safer to use a pseudoregister before replacement. 141 142 // FIXME: We should be able to eliminate unused input registers. We only 143 // cannot do this for the resources required for scratch access. For now we 144 // skip over user SGPRs and may leave unused holes. 145 146 // We find the resource first because it has an alignment requirement. 147 if (ScratchRsrcReg == TRI->reservedPrivateSegmentBufferReg(MF)) { 148 MachineRegisterInfo &MRI = MF.getRegInfo(); 149 150 unsigned NumPreloaded = MFI->getNumPreloadedSGPRs() / 4; 151 // Skip the last 2 elements because the last one is reserved for VCC, and 152 // this is the 2nd to last element already. 153 for (MCPhysReg Reg : getAllSGPR128().drop_back(2).slice(NumPreloaded)) { 154 // Pick the first unallocated one. Make sure we don't clobber the other 155 // reserved input we needed. 156 if (!MRI.isPhysRegUsed(Reg)) { 157 assert(MRI.isAllocatable(Reg)); 158 MRI.replaceRegWith(ScratchRsrcReg, Reg); 159 ScratchRsrcReg = Reg; 160 MFI->setScratchRSrcReg(ScratchRsrcReg); 161 break; 162 } 163 } 164 } 165 166 if (ScratchWaveOffsetReg == TRI->reservedPrivateSegmentWaveByteOffsetReg(MF)) { 167 MachineRegisterInfo &MRI = MF.getRegInfo(); 168 unsigned NumPreloaded = MFI->getNumPreloadedSGPRs(); 169 170 // We need to drop register from the end of the list that we cannot use 171 // for the scratch wave offset. 172 // + 2 s102 and s103 do not exist on VI. 173 // + 2 for vcc 174 // + 2 for xnack_mask 175 // + 2 for flat_scratch 176 // + 4 for registers reserved for scratch resource register 177 // + 1 for register reserved for scratch wave offset. (By exluding this 178 // register from the list to consider, it means that when this 179 // register is being used for the scratch wave offset and there 180 // are no other free SGPRs, then the value will stay in this register. 181 // ---- 182 // 13 183 for (MCPhysReg Reg : getAllSGPRs().drop_back(13).slice(NumPreloaded)) { 184 // Pick the first unallocated SGPR. Be careful not to pick an alias of the 185 // scratch descriptor, since we haven’t added its uses yet. 186 if (!MRI.isPhysRegUsed(Reg)) { 187 assert(MRI.isAllocatable(Reg) && 188 !TRI->isSubRegisterEq(ScratchRsrcReg, Reg)); 189 190 MRI.replaceRegWith(ScratchWaveOffsetReg, Reg); 191 ScratchWaveOffsetReg = Reg; 192 MFI->setScratchWaveOffsetReg(ScratchWaveOffsetReg); 193 break; 194 } 195 } 196 } 197 } 198 199 200 assert(!TRI->isSubRegister(ScratchRsrcReg, ScratchWaveOffsetReg)); 201 202 const MCInstrDesc &SMovB32 = TII->get(AMDGPU::S_MOV_B32); 203 DebugLoc DL; 204 205 if (PreloadedScratchWaveOffsetReg != ScratchWaveOffsetReg) { 206 // Make sure we emit the copy for the offset first. We may have chosen to copy 207 // the buffer resource into a register that aliases the input offset register. 208 BuildMI(MBB, I, DL, SMovB32, ScratchWaveOffsetReg) 209 .addReg(PreloadedScratchWaveOffsetReg, RegState::Kill); 210 } 211 212 if (ST.isAmdHsaOS()) { 213 // Insert copies from argument register. 214 assert( 215 !TRI->isSubRegisterEq(PreloadedPrivateBufferReg, ScratchRsrcReg) && 216 !TRI->isSubRegisterEq(PreloadedPrivateBufferReg, ScratchWaveOffsetReg)); 217 218 unsigned Rsrc01 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0_sub1); 219 unsigned Rsrc23 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub2_sub3); 220 221 unsigned Lo = TRI->getSubReg(PreloadedPrivateBufferReg, AMDGPU::sub0_sub1); 222 unsigned Hi = TRI->getSubReg(PreloadedPrivateBufferReg, AMDGPU::sub2_sub3); 223 224 const MCInstrDesc &SMovB64 = TII->get(AMDGPU::S_MOV_B64); 225 226 BuildMI(MBB, I, DL, SMovB64, Rsrc01) 227 .addReg(Lo, RegState::Kill); 228 BuildMI(MBB, I, DL, SMovB64, Rsrc23) 229 .addReg(Hi, RegState::Kill); 230 } else { 231 unsigned Rsrc0 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0); 232 unsigned Rsrc1 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub1); 233 unsigned Rsrc2 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub2); 234 unsigned Rsrc3 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub3); 235 236 // Use relocations to get the pointer, and setup the other bits manually. 237 uint64_t Rsrc23 = TII->getScratchRsrcWords23(); 238 BuildMI(MBB, I, DL, SMovB32, Rsrc0) 239 .addExternalSymbol("SCRATCH_RSRC_DWORD0") 240 .addReg(ScratchRsrcReg, RegState::ImplicitDefine); 241 242 BuildMI(MBB, I, DL, SMovB32, Rsrc1) 243 .addExternalSymbol("SCRATCH_RSRC_DWORD1") 244 .addReg(ScratchRsrcReg, RegState::ImplicitDefine); 245 246 BuildMI(MBB, I, DL, SMovB32, Rsrc2) 247 .addImm(Rsrc23 & 0xffffffff) 248 .addReg(ScratchRsrcReg, RegState::ImplicitDefine); 249 250 BuildMI(MBB, I, DL, SMovB32, Rsrc3) 251 .addImm(Rsrc23 >> 32) 252 .addReg(ScratchRsrcReg, RegState::ImplicitDefine); 253 } 254 255 // Make the register selected live throughout the function. 256 for (MachineBasicBlock &OtherBB : MF) { 257 if (&OtherBB == &MBB) 258 continue; 259 260 OtherBB.addLiveIn(ScratchRsrcReg); 261 OtherBB.addLiveIn(ScratchWaveOffsetReg); 262 } 263 } 264 265 void SIFrameLowering::processFunctionBeforeFrameFinalized( 266 MachineFunction &MF, 267 RegScavenger *RS) const { 268 MachineFrameInfo *MFI = MF.getFrameInfo(); 269 270 if (!MFI->hasStackObjects()) 271 return; 272 273 bool MayNeedScavengingEmergencySlot = MFI->hasStackObjects(); 274 275 assert((RS || !MayNeedScavengingEmergencySlot) && 276 "RegScavenger required if spilling"); 277 278 if (MayNeedScavengingEmergencySlot) { 279 int ScavengeFI = MFI->CreateSpillStackObject( 280 AMDGPU::SGPR_32RegClass.getSize(), 281 AMDGPU::SGPR_32RegClass.getAlignment()); 282 RS->addScavengingFrameIndex(ScavengeFI); 283 } 284 } 285