1 //===-- AMDGPUAsmPrinter.cpp - AMDGPU Assebly printer --------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 /// \file 11 /// 12 /// The AMDGPUAsmPrinter is used to print both assembly string and also binary 13 /// code. When passed an MCAsmStreamer it prints assembly and when passed 14 /// an MCObjectStreamer it outputs binary code. 15 // 16 //===----------------------------------------------------------------------===// 17 // 18 19 #include "AMDGPUAsmPrinter.h" 20 #include "MCTargetDesc/AMDGPUTargetStreamer.h" 21 #include "InstPrinter/AMDGPUInstPrinter.h" 22 #include "Utils/AMDGPUBaseInfo.h" 23 #include "AMDGPU.h" 24 #include "AMDKernelCodeT.h" 25 #include "AMDGPUSubtarget.h" 26 #include "R600Defines.h" 27 #include "R600MachineFunctionInfo.h" 28 #include "R600RegisterInfo.h" 29 #include "SIDefines.h" 30 #include "SIMachineFunctionInfo.h" 31 #include "SIInstrInfo.h" 32 #include "SIRegisterInfo.h" 33 #include "llvm/CodeGen/MachineFrameInfo.h" 34 #include "llvm/IR/DiagnosticInfo.h" 35 #include "llvm/MC/MCContext.h" 36 #include "llvm/MC/MCSectionELF.h" 37 #include "llvm/MC/MCStreamer.h" 38 #include "llvm/Support/ELF.h" 39 #include "llvm/Support/MathExtras.h" 40 #include "llvm/Support/TargetRegistry.h" 41 #include "llvm/Target/TargetLoweringObjectFile.h" 42 #include "AMDGPURuntimeMetadata.h" 43 44 using namespace ::AMDGPU; 45 using namespace llvm; 46 47 // TODO: This should get the default rounding mode from the kernel. We just set 48 // the default here, but this could change if the OpenCL rounding mode pragmas 49 // are used. 50 // 51 // The denormal mode here should match what is reported by the OpenCL runtime 52 // for the CL_FP_DENORM bit from CL_DEVICE_{HALF|SINGLE|DOUBLE}_FP_CONFIG, but 53 // can also be override to flush with the -cl-denorms-are-zero compiler flag. 54 // 55 // AMD OpenCL only sets flush none and reports CL_FP_DENORM for double 56 // precision, and leaves single precision to flush all and does not report 57 // CL_FP_DENORM for CL_DEVICE_SINGLE_FP_CONFIG. Mesa's OpenCL currently reports 58 // CL_FP_DENORM for both. 59 // 60 // FIXME: It seems some instructions do not support single precision denormals 61 // regardless of the mode (exp_*_f32, rcp_*_f32, rsq_*_f32, rsq_*f32, sqrt_f32, 62 // and sin_f32, cos_f32 on most parts). 63 64 // We want to use these instructions, and using fp32 denormals also causes 65 // instructions to run at the double precision rate for the device so it's 66 // probably best to just report no single precision denormals. 67 static uint32_t getFPMode(const MachineFunction &F) { 68 const SISubtarget& ST = F.getSubtarget<SISubtarget>(); 69 // TODO: Is there any real use for the flush in only / flush out only modes? 70 71 uint32_t FP32Denormals = 72 ST.hasFP32Denormals() ? FP_DENORM_FLUSH_NONE : FP_DENORM_FLUSH_IN_FLUSH_OUT; 73 74 uint32_t FP64Denormals = 75 ST.hasFP64Denormals() ? FP_DENORM_FLUSH_NONE : FP_DENORM_FLUSH_IN_FLUSH_OUT; 76 77 return FP_ROUND_MODE_SP(FP_ROUND_ROUND_TO_NEAREST) | 78 FP_ROUND_MODE_DP(FP_ROUND_ROUND_TO_NEAREST) | 79 FP_DENORM_MODE_SP(FP32Denormals) | 80 FP_DENORM_MODE_DP(FP64Denormals); 81 } 82 83 static AsmPrinter * 84 createAMDGPUAsmPrinterPass(TargetMachine &tm, 85 std::unique_ptr<MCStreamer> &&Streamer) { 86 return new AMDGPUAsmPrinter(tm, std::move(Streamer)); 87 } 88 89 extern "C" void LLVMInitializeAMDGPUAsmPrinter() { 90 TargetRegistry::RegisterAsmPrinter(TheAMDGPUTarget, createAMDGPUAsmPrinterPass); 91 TargetRegistry::RegisterAsmPrinter(TheGCNTarget, createAMDGPUAsmPrinterPass); 92 } 93 94 AMDGPUAsmPrinter::AMDGPUAsmPrinter(TargetMachine &TM, 95 std::unique_ptr<MCStreamer> Streamer) 96 : AsmPrinter(TM, std::move(Streamer)) {} 97 98 const char *AMDGPUAsmPrinter::getPassName() const { 99 return "AMDGPU Assembly Printer"; 100 } 101 102 void AMDGPUAsmPrinter::EmitStartOfAsmFile(Module &M) { 103 if (TM.getTargetTriple().getOS() != Triple::AMDHSA) 104 return; 105 106 // Need to construct an MCSubtargetInfo here in case we have no functions 107 // in the module. 108 std::unique_ptr<MCSubtargetInfo> STI(TM.getTarget().createMCSubtargetInfo( 109 TM.getTargetTriple().str(), TM.getTargetCPU(), 110 TM.getTargetFeatureString())); 111 112 AMDGPUTargetStreamer *TS = 113 static_cast<AMDGPUTargetStreamer *>(OutStreamer->getTargetStreamer()); 114 115 TS->EmitDirectiveHSACodeObjectVersion(2, 1); 116 117 AMDGPU::IsaVersion ISA = AMDGPU::getIsaVersion(STI->getFeatureBits()); 118 TS->EmitDirectiveHSACodeObjectISA(ISA.Major, ISA.Minor, ISA.Stepping, 119 "AMD", "AMDGPU"); 120 emitStartOfRuntimeMetadata(M); 121 } 122 123 void AMDGPUAsmPrinter::EmitFunctionBodyStart() { 124 const AMDGPUSubtarget &STM = MF->getSubtarget<AMDGPUSubtarget>(); 125 SIProgramInfo KernelInfo; 126 if (STM.isAmdHsaOS()) { 127 getSIProgramInfo(KernelInfo, *MF); 128 EmitAmdKernelCodeT(*MF, KernelInfo); 129 } 130 } 131 132 void AMDGPUAsmPrinter::EmitFunctionEntryLabel() { 133 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 134 const AMDGPUSubtarget &STM = MF->getSubtarget<AMDGPUSubtarget>(); 135 if (MFI->isKernel() && STM.isAmdHsaOS()) { 136 AMDGPUTargetStreamer *TS = 137 static_cast<AMDGPUTargetStreamer *>(OutStreamer->getTargetStreamer()); 138 TS->EmitAMDGPUSymbolType(CurrentFnSym->getName(), 139 ELF::STT_AMDGPU_HSA_KERNEL); 140 } 141 142 AsmPrinter::EmitFunctionEntryLabel(); 143 } 144 145 void AMDGPUAsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) { 146 147 // Group segment variables aren't emitted in HSA. 148 if (AMDGPU::isGroupSegment(GV)) 149 return; 150 151 AsmPrinter::EmitGlobalVariable(GV); 152 } 153 154 bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) { 155 156 // The starting address of all shader programs must be 256 bytes aligned. 157 MF.setAlignment(8); 158 159 SetupMachineFunction(MF); 160 161 MCContext &Context = getObjFileLowering().getContext(); 162 MCSectionELF *ConfigSection = 163 Context.getELFSection(".AMDGPU.config", ELF::SHT_PROGBITS, 0); 164 OutStreamer->SwitchSection(ConfigSection); 165 166 const AMDGPUSubtarget &STM = MF.getSubtarget<AMDGPUSubtarget>(); 167 SIProgramInfo KernelInfo; 168 if (STM.getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS) { 169 getSIProgramInfo(KernelInfo, MF); 170 if (!STM.isAmdHsaOS()) { 171 EmitProgramInfoSI(MF, KernelInfo); 172 } 173 } else { 174 EmitProgramInfoR600(MF); 175 } 176 177 DisasmLines.clear(); 178 HexLines.clear(); 179 DisasmLineMaxLen = 0; 180 181 EmitFunctionBody(); 182 183 if (isVerbose()) { 184 MCSectionELF *CommentSection = 185 Context.getELFSection(".AMDGPU.csdata", ELF::SHT_PROGBITS, 0); 186 OutStreamer->SwitchSection(CommentSection); 187 188 if (STM.getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS) { 189 OutStreamer->emitRawComment(" Kernel info:", false); 190 OutStreamer->emitRawComment(" codeLenInByte = " + Twine(KernelInfo.CodeLen), 191 false); 192 OutStreamer->emitRawComment(" NumSgprs: " + Twine(KernelInfo.NumSGPR), 193 false); 194 OutStreamer->emitRawComment(" NumVgprs: " + Twine(KernelInfo.NumVGPR), 195 false); 196 OutStreamer->emitRawComment(" FloatMode: " + Twine(KernelInfo.FloatMode), 197 false); 198 OutStreamer->emitRawComment(" IeeeMode: " + Twine(KernelInfo.IEEEMode), 199 false); 200 OutStreamer->emitRawComment(" ScratchSize: " + Twine(KernelInfo.ScratchSize), 201 false); 202 OutStreamer->emitRawComment(" LDSByteSize: " + Twine(KernelInfo.LDSSize) + 203 " bytes/workgroup (compile time only)", false); 204 205 OutStreamer->emitRawComment(" ReservedVGPRFirst: " + Twine(KernelInfo.ReservedVGPRFirst), 206 false); 207 OutStreamer->emitRawComment(" ReservedVGPRCount: " + Twine(KernelInfo.ReservedVGPRCount), 208 false); 209 210 if (MF.getSubtarget<SISubtarget>().debuggerEmitPrologue()) { 211 OutStreamer->emitRawComment(" DebuggerWavefrontPrivateSegmentOffsetSGPR: s" + 212 Twine(KernelInfo.DebuggerWavefrontPrivateSegmentOffsetSGPR), false); 213 OutStreamer->emitRawComment(" DebuggerPrivateSegmentBufferSGPR: s" + 214 Twine(KernelInfo.DebuggerPrivateSegmentBufferSGPR), false); 215 } 216 217 OutStreamer->emitRawComment(" COMPUTE_PGM_RSRC2:USER_SGPR: " + 218 Twine(G_00B84C_USER_SGPR(KernelInfo.ComputePGMRSrc2)), 219 false); 220 OutStreamer->emitRawComment(" COMPUTE_PGM_RSRC2:TGID_X_EN: " + 221 Twine(G_00B84C_TGID_X_EN(KernelInfo.ComputePGMRSrc2)), 222 false); 223 OutStreamer->emitRawComment(" COMPUTE_PGM_RSRC2:TGID_Y_EN: " + 224 Twine(G_00B84C_TGID_Y_EN(KernelInfo.ComputePGMRSrc2)), 225 false); 226 OutStreamer->emitRawComment(" COMPUTE_PGM_RSRC2:TGID_Z_EN: " + 227 Twine(G_00B84C_TGID_Z_EN(KernelInfo.ComputePGMRSrc2)), 228 false); 229 OutStreamer->emitRawComment(" COMPUTE_PGM_RSRC2:TIDIG_COMP_CNT: " + 230 Twine(G_00B84C_TIDIG_COMP_CNT(KernelInfo.ComputePGMRSrc2)), 231 false); 232 233 } else { 234 R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>(); 235 OutStreamer->emitRawComment( 236 Twine("SQ_PGM_RESOURCES:STACK_SIZE = " + Twine(MFI->CFStackSize))); 237 } 238 } 239 240 if (STM.dumpCode()) { 241 242 OutStreamer->SwitchSection( 243 Context.getELFSection(".AMDGPU.disasm", ELF::SHT_NOTE, 0)); 244 245 for (size_t i = 0; i < DisasmLines.size(); ++i) { 246 std::string Comment(DisasmLineMaxLen - DisasmLines[i].size(), ' '); 247 Comment += " ; " + HexLines[i] + "\n"; 248 249 OutStreamer->EmitBytes(StringRef(DisasmLines[i])); 250 OutStreamer->EmitBytes(StringRef(Comment)); 251 } 252 } 253 254 emitRuntimeMetadata(*MF.getFunction()); 255 256 return false; 257 } 258 259 void AMDGPUAsmPrinter::EmitProgramInfoR600(const MachineFunction &MF) { 260 unsigned MaxGPR = 0; 261 bool killPixel = false; 262 const R600Subtarget &STM = MF.getSubtarget<R600Subtarget>(); 263 const R600RegisterInfo *RI = STM.getRegisterInfo(); 264 const R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>(); 265 266 for (const MachineBasicBlock &MBB : MF) { 267 for (const MachineInstr &MI : MBB) { 268 if (MI.getOpcode() == AMDGPU::KILLGT) 269 killPixel = true; 270 unsigned numOperands = MI.getNumOperands(); 271 for (unsigned op_idx = 0; op_idx < numOperands; op_idx++) { 272 const MachineOperand &MO = MI.getOperand(op_idx); 273 if (!MO.isReg()) 274 continue; 275 unsigned HWReg = RI->getEncodingValue(MO.getReg()) & 0xff; 276 277 // Register with value > 127 aren't GPR 278 if (HWReg > 127) 279 continue; 280 MaxGPR = std::max(MaxGPR, HWReg); 281 } 282 } 283 } 284 285 unsigned RsrcReg; 286 if (STM.getGeneration() >= R600Subtarget::EVERGREEN) { 287 // Evergreen / Northern Islands 288 switch (MF.getFunction()->getCallingConv()) { 289 default: // Fall through 290 case CallingConv::AMDGPU_CS: RsrcReg = R_0288D4_SQ_PGM_RESOURCES_LS; break; 291 case CallingConv::AMDGPU_GS: RsrcReg = R_028878_SQ_PGM_RESOURCES_GS; break; 292 case CallingConv::AMDGPU_PS: RsrcReg = R_028844_SQ_PGM_RESOURCES_PS; break; 293 case CallingConv::AMDGPU_VS: RsrcReg = R_028860_SQ_PGM_RESOURCES_VS; break; 294 } 295 } else { 296 // R600 / R700 297 switch (MF.getFunction()->getCallingConv()) { 298 default: // Fall through 299 case CallingConv::AMDGPU_GS: // Fall through 300 case CallingConv::AMDGPU_CS: // Fall through 301 case CallingConv::AMDGPU_VS: RsrcReg = R_028868_SQ_PGM_RESOURCES_VS; break; 302 case CallingConv::AMDGPU_PS: RsrcReg = R_028850_SQ_PGM_RESOURCES_PS; break; 303 } 304 } 305 306 OutStreamer->EmitIntValue(RsrcReg, 4); 307 OutStreamer->EmitIntValue(S_NUM_GPRS(MaxGPR + 1) | 308 S_STACK_SIZE(MFI->CFStackSize), 4); 309 OutStreamer->EmitIntValue(R_02880C_DB_SHADER_CONTROL, 4); 310 OutStreamer->EmitIntValue(S_02880C_KILL_ENABLE(killPixel), 4); 311 312 if (AMDGPU::isCompute(MF.getFunction()->getCallingConv())) { 313 OutStreamer->EmitIntValue(R_0288E8_SQ_LDS_ALLOC, 4); 314 OutStreamer->EmitIntValue(alignTo(MFI->LDSSize, 4) >> 2, 4); 315 } 316 } 317 318 void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo, 319 const MachineFunction &MF) const { 320 const SISubtarget &STM = MF.getSubtarget<SISubtarget>(); 321 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 322 uint64_t CodeSize = 0; 323 unsigned MaxSGPR = 0; 324 unsigned MaxVGPR = 0; 325 bool VCCUsed = false; 326 bool FlatUsed = false; 327 const SIRegisterInfo *RI = STM.getRegisterInfo(); 328 const SIInstrInfo *TII = STM.getInstrInfo(); 329 330 for (const MachineBasicBlock &MBB : MF) { 331 for (const MachineInstr &MI : MBB) { 332 // TODO: CodeSize should account for multiple functions. 333 334 // TODO: Should we count size of debug info? 335 if (MI.isDebugValue()) 336 continue; 337 338 CodeSize += TII->getInstSizeInBytes(MI); 339 340 unsigned numOperands = MI.getNumOperands(); 341 for (unsigned op_idx = 0; op_idx < numOperands; op_idx++) { 342 const MachineOperand &MO = MI.getOperand(op_idx); 343 unsigned width = 0; 344 bool isSGPR = false; 345 346 if (!MO.isReg()) 347 continue; 348 349 unsigned reg = MO.getReg(); 350 switch (reg) { 351 case AMDGPU::EXEC: 352 case AMDGPU::EXEC_LO: 353 case AMDGPU::EXEC_HI: 354 case AMDGPU::SCC: 355 case AMDGPU::M0: 356 continue; 357 358 case AMDGPU::VCC: 359 case AMDGPU::VCC_LO: 360 case AMDGPU::VCC_HI: 361 VCCUsed = true; 362 continue; 363 364 case AMDGPU::FLAT_SCR: 365 case AMDGPU::FLAT_SCR_LO: 366 case AMDGPU::FLAT_SCR_HI: 367 FlatUsed = true; 368 continue; 369 370 case AMDGPU::TBA: 371 case AMDGPU::TBA_LO: 372 case AMDGPU::TBA_HI: 373 case AMDGPU::TMA: 374 case AMDGPU::TMA_LO: 375 case AMDGPU::TMA_HI: 376 llvm_unreachable("Trap Handler registers should not be used"); 377 continue; 378 379 default: 380 break; 381 } 382 383 if (AMDGPU::SReg_32RegClass.contains(reg)) { 384 if (AMDGPU::TTMP_32RegClass.contains(reg)) { 385 llvm_unreachable("Trap Handler registers should not be used"); 386 } 387 isSGPR = true; 388 width = 1; 389 } else if (AMDGPU::VGPR_32RegClass.contains(reg)) { 390 isSGPR = false; 391 width = 1; 392 } else if (AMDGPU::SReg_64RegClass.contains(reg)) { 393 if (AMDGPU::TTMP_64RegClass.contains(reg)) { 394 llvm_unreachable("Trap Handler registers should not be used"); 395 } 396 isSGPR = true; 397 width = 2; 398 } else if (AMDGPU::VReg_64RegClass.contains(reg)) { 399 isSGPR = false; 400 width = 2; 401 } else if (AMDGPU::VReg_96RegClass.contains(reg)) { 402 isSGPR = false; 403 width = 3; 404 } else if (AMDGPU::SReg_128RegClass.contains(reg)) { 405 isSGPR = true; 406 width = 4; 407 } else if (AMDGPU::VReg_128RegClass.contains(reg)) { 408 isSGPR = false; 409 width = 4; 410 } else if (AMDGPU::SReg_256RegClass.contains(reg)) { 411 isSGPR = true; 412 width = 8; 413 } else if (AMDGPU::VReg_256RegClass.contains(reg)) { 414 isSGPR = false; 415 width = 8; 416 } else if (AMDGPU::SReg_512RegClass.contains(reg)) { 417 isSGPR = true; 418 width = 16; 419 } else if (AMDGPU::VReg_512RegClass.contains(reg)) { 420 isSGPR = false; 421 width = 16; 422 } else { 423 llvm_unreachable("Unknown register class"); 424 } 425 unsigned hwReg = RI->getEncodingValue(reg) & 0xff; 426 unsigned maxUsed = hwReg + width - 1; 427 if (isSGPR) { 428 MaxSGPR = maxUsed > MaxSGPR ? maxUsed : MaxSGPR; 429 } else { 430 MaxVGPR = maxUsed > MaxVGPR ? maxUsed : MaxVGPR; 431 } 432 } 433 } 434 } 435 436 unsigned ExtraSGPRs = 0; 437 438 if (VCCUsed) 439 ExtraSGPRs = 2; 440 441 if (STM.getGeneration() < SISubtarget::VOLCANIC_ISLANDS) { 442 if (FlatUsed) 443 ExtraSGPRs = 4; 444 } else { 445 if (STM.isXNACKEnabled()) 446 ExtraSGPRs = 4; 447 448 if (FlatUsed) 449 ExtraSGPRs = 6; 450 } 451 452 MaxSGPR += ExtraSGPRs; 453 454 // Record first reserved register and reserved register count fields, and 455 // update max register counts if "amdgpu-debugger-reserve-regs" attribute was 456 // specified. 457 if (STM.debuggerReserveRegs()) { 458 ProgInfo.ReservedVGPRFirst = MaxVGPR + 1; 459 ProgInfo.ReservedVGPRCount = MFI->getDebuggerReservedVGPRCount(); 460 MaxVGPR += MFI->getDebuggerReservedVGPRCount(); 461 } 462 463 // Update DebuggerWavefrontPrivateSegmentOffsetSGPR and 464 // DebuggerPrivateSegmentBufferSGPR fields if "amdgpu-debugger-emit-prologue" 465 // attribute was specified. 466 if (STM.debuggerEmitPrologue()) { 467 ProgInfo.DebuggerWavefrontPrivateSegmentOffsetSGPR = 468 RI->getHWRegIndex(MFI->getScratchWaveOffsetReg()); 469 ProgInfo.DebuggerPrivateSegmentBufferSGPR = 470 RI->getHWRegIndex(MFI->getScratchRSrcReg()); 471 } 472 473 // We found the maximum register index. They start at 0, so add one to get the 474 // number of registers. 475 ProgInfo.NumVGPR = MaxVGPR + 1; 476 ProgInfo.NumSGPR = MaxSGPR + 1; 477 478 if (STM.hasSGPRInitBug()) { 479 if (ProgInfo.NumSGPR > SISubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG) { 480 LLVMContext &Ctx = MF.getFunction()->getContext(); 481 DiagnosticInfoResourceLimit Diag(*MF.getFunction(), 482 "SGPRs with SGPR init bug", 483 ProgInfo.NumSGPR, DS_Error); 484 Ctx.diagnose(Diag); 485 } 486 487 ProgInfo.NumSGPR = SISubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG; 488 } 489 490 if (MFI->NumUserSGPRs > STM.getMaxNumUserSGPRs()) { 491 LLVMContext &Ctx = MF.getFunction()->getContext(); 492 DiagnosticInfoResourceLimit Diag(*MF.getFunction(), "user SGPRs", 493 MFI->NumUserSGPRs, DS_Error); 494 Ctx.diagnose(Diag); 495 } 496 497 if (MFI->LDSSize > static_cast<unsigned>(STM.getLocalMemorySize())) { 498 LLVMContext &Ctx = MF.getFunction()->getContext(); 499 DiagnosticInfoResourceLimit Diag(*MF.getFunction(), "local memory", 500 MFI->LDSSize, DS_Error); 501 Ctx.diagnose(Diag); 502 } 503 504 ProgInfo.VGPRBlocks = (ProgInfo.NumVGPR - 1) / 4; 505 ProgInfo.SGPRBlocks = (ProgInfo.NumSGPR - 1) / 8; 506 // Set the value to initialize FP_ROUND and FP_DENORM parts of the mode 507 // register. 508 ProgInfo.FloatMode = getFPMode(MF); 509 510 ProgInfo.IEEEMode = 0; 511 512 // Make clamp modifier on NaN input returns 0. 513 ProgInfo.DX10Clamp = 1; 514 515 const MachineFrameInfo *FrameInfo = MF.getFrameInfo(); 516 ProgInfo.ScratchSize = FrameInfo->getStackSize(); 517 518 ProgInfo.FlatUsed = FlatUsed; 519 ProgInfo.VCCUsed = VCCUsed; 520 ProgInfo.CodeLen = CodeSize; 521 522 unsigned LDSAlignShift; 523 if (STM.getGeneration() < SISubtarget::SEA_ISLANDS) { 524 // LDS is allocated in 64 dword blocks. 525 LDSAlignShift = 8; 526 } else { 527 // LDS is allocated in 128 dword blocks. 528 LDSAlignShift = 9; 529 } 530 531 unsigned LDSSpillSize = MFI->LDSWaveSpillSize * 532 MFI->getMaximumWorkGroupSize(MF); 533 534 ProgInfo.LDSSize = MFI->LDSSize + LDSSpillSize; 535 ProgInfo.LDSBlocks = 536 alignTo(ProgInfo.LDSSize, 1ULL << LDSAlignShift) >> LDSAlignShift; 537 538 // Scratch is allocated in 256 dword blocks. 539 unsigned ScratchAlignShift = 10; 540 // We need to program the hardware with the amount of scratch memory that 541 // is used by the entire wave. ProgInfo.ScratchSize is the amount of 542 // scratch memory used per thread. 543 ProgInfo.ScratchBlocks = 544 alignTo(ProgInfo.ScratchSize * STM.getWavefrontSize(), 545 1ULL << ScratchAlignShift) >> 546 ScratchAlignShift; 547 548 ProgInfo.ComputePGMRSrc1 = 549 S_00B848_VGPRS(ProgInfo.VGPRBlocks) | 550 S_00B848_SGPRS(ProgInfo.SGPRBlocks) | 551 S_00B848_PRIORITY(ProgInfo.Priority) | 552 S_00B848_FLOAT_MODE(ProgInfo.FloatMode) | 553 S_00B848_PRIV(ProgInfo.Priv) | 554 S_00B848_DX10_CLAMP(ProgInfo.DX10Clamp) | 555 S_00B848_DEBUG_MODE(ProgInfo.DebugMode) | 556 S_00B848_IEEE_MODE(ProgInfo.IEEEMode); 557 558 // 0 = X, 1 = XY, 2 = XYZ 559 unsigned TIDIGCompCnt = 0; 560 if (MFI->hasWorkItemIDZ()) 561 TIDIGCompCnt = 2; 562 else if (MFI->hasWorkItemIDY()) 563 TIDIGCompCnt = 1; 564 565 ProgInfo.ComputePGMRSrc2 = 566 S_00B84C_SCRATCH_EN(ProgInfo.ScratchBlocks > 0) | 567 S_00B84C_USER_SGPR(MFI->getNumUserSGPRs()) | 568 S_00B84C_TGID_X_EN(MFI->hasWorkGroupIDX()) | 569 S_00B84C_TGID_Y_EN(MFI->hasWorkGroupIDY()) | 570 S_00B84C_TGID_Z_EN(MFI->hasWorkGroupIDZ()) | 571 S_00B84C_TG_SIZE_EN(MFI->hasWorkGroupInfo()) | 572 S_00B84C_TIDIG_COMP_CNT(TIDIGCompCnt) | 573 S_00B84C_EXCP_EN_MSB(0) | 574 S_00B84C_LDS_SIZE(ProgInfo.LDSBlocks) | 575 S_00B84C_EXCP_EN(0); 576 } 577 578 static unsigned getRsrcReg(CallingConv::ID CallConv) { 579 switch (CallConv) { 580 default: // Fall through 581 case CallingConv::AMDGPU_CS: return R_00B848_COMPUTE_PGM_RSRC1; 582 case CallingConv::AMDGPU_GS: return R_00B228_SPI_SHADER_PGM_RSRC1_GS; 583 case CallingConv::AMDGPU_PS: return R_00B028_SPI_SHADER_PGM_RSRC1_PS; 584 case CallingConv::AMDGPU_VS: return R_00B128_SPI_SHADER_PGM_RSRC1_VS; 585 } 586 } 587 588 void AMDGPUAsmPrinter::EmitProgramInfoSI(const MachineFunction &MF, 589 const SIProgramInfo &KernelInfo) { 590 const SISubtarget &STM = MF.getSubtarget<SISubtarget>(); 591 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 592 unsigned RsrcReg = getRsrcReg(MF.getFunction()->getCallingConv()); 593 594 if (AMDGPU::isCompute(MF.getFunction()->getCallingConv())) { 595 OutStreamer->EmitIntValue(R_00B848_COMPUTE_PGM_RSRC1, 4); 596 597 OutStreamer->EmitIntValue(KernelInfo.ComputePGMRSrc1, 4); 598 599 OutStreamer->EmitIntValue(R_00B84C_COMPUTE_PGM_RSRC2, 4); 600 OutStreamer->EmitIntValue(KernelInfo.ComputePGMRSrc2, 4); 601 602 OutStreamer->EmitIntValue(R_00B860_COMPUTE_TMPRING_SIZE, 4); 603 OutStreamer->EmitIntValue(S_00B860_WAVESIZE(KernelInfo.ScratchBlocks), 4); 604 605 // TODO: Should probably note flat usage somewhere. SC emits a "FlatPtr32 = 606 // 0" comment but I don't see a corresponding field in the register spec. 607 } else { 608 OutStreamer->EmitIntValue(RsrcReg, 4); 609 OutStreamer->EmitIntValue(S_00B028_VGPRS(KernelInfo.VGPRBlocks) | 610 S_00B028_SGPRS(KernelInfo.SGPRBlocks), 4); 611 if (STM.isVGPRSpillingEnabled(*MF.getFunction())) { 612 OutStreamer->EmitIntValue(R_0286E8_SPI_TMPRING_SIZE, 4); 613 OutStreamer->EmitIntValue(S_0286E8_WAVESIZE(KernelInfo.ScratchBlocks), 4); 614 } 615 } 616 617 if (MF.getFunction()->getCallingConv() == CallingConv::AMDGPU_PS) { 618 OutStreamer->EmitIntValue(R_00B02C_SPI_SHADER_PGM_RSRC2_PS, 4); 619 OutStreamer->EmitIntValue(S_00B02C_EXTRA_LDS_SIZE(KernelInfo.LDSBlocks), 4); 620 OutStreamer->EmitIntValue(R_0286CC_SPI_PS_INPUT_ENA, 4); 621 OutStreamer->EmitIntValue(MFI->PSInputEna, 4); 622 OutStreamer->EmitIntValue(R_0286D0_SPI_PS_INPUT_ADDR, 4); 623 OutStreamer->EmitIntValue(MFI->getPSInputAddr(), 4); 624 } 625 626 OutStreamer->EmitIntValue(R_SPILLED_SGPRS, 4); 627 OutStreamer->EmitIntValue(MFI->getNumSpilledSGPRs(), 4); 628 OutStreamer->EmitIntValue(R_SPILLED_VGPRS, 4); 629 OutStreamer->EmitIntValue(MFI->getNumSpilledVGPRs(), 4); 630 } 631 632 // This is supposed to be log2(Size) 633 static amd_element_byte_size_t getElementByteSizeValue(unsigned Size) { 634 switch (Size) { 635 case 4: 636 return AMD_ELEMENT_4_BYTES; 637 case 8: 638 return AMD_ELEMENT_8_BYTES; 639 case 16: 640 return AMD_ELEMENT_16_BYTES; 641 default: 642 llvm_unreachable("invalid private_element_size"); 643 } 644 } 645 646 void AMDGPUAsmPrinter::EmitAmdKernelCodeT(const MachineFunction &MF, 647 const SIProgramInfo &KernelInfo) const { 648 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 649 const SISubtarget &STM = MF.getSubtarget<SISubtarget>(); 650 amd_kernel_code_t header; 651 652 AMDGPU::initDefaultAMDKernelCodeT(header, STM.getFeatureBits()); 653 654 header.compute_pgm_resource_registers = 655 KernelInfo.ComputePGMRSrc1 | 656 (KernelInfo.ComputePGMRSrc2 << 32); 657 header.code_properties = AMD_CODE_PROPERTY_IS_PTR64; 658 659 660 AMD_HSA_BITS_SET(header.code_properties, 661 AMD_CODE_PROPERTY_PRIVATE_ELEMENT_SIZE, 662 getElementByteSizeValue(STM.getMaxPrivateElementSize())); 663 664 if (MFI->hasPrivateSegmentBuffer()) { 665 header.code_properties |= 666 AMD_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER; 667 } 668 669 if (MFI->hasDispatchPtr()) 670 header.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR; 671 672 if (MFI->hasQueuePtr()) 673 header.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR; 674 675 if (MFI->hasKernargSegmentPtr()) 676 header.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR; 677 678 if (MFI->hasDispatchID()) 679 header.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID; 680 681 if (MFI->hasFlatScratchInit()) 682 header.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT; 683 684 // TODO: Private segment size 685 686 if (MFI->hasGridWorkgroupCountX()) { 687 header.code_properties |= 688 AMD_CODE_PROPERTY_ENABLE_SGPR_GRID_WORKGROUP_COUNT_X; 689 } 690 691 if (MFI->hasGridWorkgroupCountY()) { 692 header.code_properties |= 693 AMD_CODE_PROPERTY_ENABLE_SGPR_GRID_WORKGROUP_COUNT_Y; 694 } 695 696 if (MFI->hasGridWorkgroupCountZ()) { 697 header.code_properties |= 698 AMD_CODE_PROPERTY_ENABLE_SGPR_GRID_WORKGROUP_COUNT_Z; 699 } 700 701 if (MFI->hasDispatchPtr()) 702 header.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR; 703 704 if (STM.debuggerSupported()) 705 header.code_properties |= AMD_CODE_PROPERTY_IS_DEBUG_SUPPORTED; 706 707 if (STM.isXNACKEnabled()) 708 header.code_properties |= AMD_CODE_PROPERTY_IS_XNACK_SUPPORTED; 709 710 header.kernarg_segment_byte_size = MFI->ABIArgOffset; 711 header.wavefront_sgpr_count = KernelInfo.NumSGPR; 712 header.workitem_vgpr_count = KernelInfo.NumVGPR; 713 header.workitem_private_segment_byte_size = KernelInfo.ScratchSize; 714 header.workgroup_group_segment_byte_size = KernelInfo.LDSSize; 715 header.reserved_vgpr_first = KernelInfo.ReservedVGPRFirst; 716 header.reserved_vgpr_count = KernelInfo.ReservedVGPRCount; 717 718 if (STM.debuggerEmitPrologue()) { 719 header.debug_wavefront_private_segment_offset_sgpr = 720 KernelInfo.DebuggerWavefrontPrivateSegmentOffsetSGPR; 721 header.debug_private_segment_buffer_sgpr = 722 KernelInfo.DebuggerPrivateSegmentBufferSGPR; 723 } 724 725 AMDGPUTargetStreamer *TS = 726 static_cast<AMDGPUTargetStreamer *>(OutStreamer->getTargetStreamer()); 727 728 OutStreamer->SwitchSection(getObjFileLowering().getTextSection()); 729 TS->EmitAMDKernelCodeT(header); 730 } 731 732 bool AMDGPUAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, 733 unsigned AsmVariant, 734 const char *ExtraCode, raw_ostream &O) { 735 if (ExtraCode && ExtraCode[0]) { 736 if (ExtraCode[1] != 0) 737 return true; // Unknown modifier. 738 739 switch (ExtraCode[0]) { 740 default: 741 // See if this is a generic print operand 742 return AsmPrinter::PrintAsmOperand(MI, OpNo, AsmVariant, ExtraCode, O); 743 case 'r': 744 break; 745 } 746 } 747 748 AMDGPUInstPrinter::printRegOperand(MI->getOperand(OpNo).getReg(), O, 749 *TM.getSubtargetImpl(*MF->getFunction())->getRegisterInfo()); 750 return false; 751 } 752 753 // Emit a key and an integer value for runtime metadata. 754 static void emitRuntimeMDIntValue(std::unique_ptr<MCStreamer> &Streamer, 755 RuntimeMD::Key K, uint64_t V, 756 unsigned Size) { 757 Streamer->EmitIntValue(K, 1); 758 Streamer->EmitIntValue(V, Size); 759 } 760 761 // Emit a key and a string value for runtime metadata. 762 static void emitRuntimeMDStringValue(std::unique_ptr<MCStreamer> &Streamer, 763 RuntimeMD::Key K, StringRef S) { 764 Streamer->EmitIntValue(K, 1); 765 Streamer->EmitIntValue(S.size(), 4); 766 Streamer->EmitBytes(S); 767 } 768 769 // Emit a key and three integer values for runtime metadata. 770 // The three integer values are obtained from MDNode \p Node; 771 static void emitRuntimeMDThreeIntValues(std::unique_ptr<MCStreamer> &Streamer, 772 RuntimeMD::Key K, MDNode *Node, 773 unsigned Size) { 774 Streamer->EmitIntValue(K, 1); 775 Streamer->EmitIntValue(mdconst::extract<ConstantInt>( 776 Node->getOperand(0))->getZExtValue(), Size); 777 Streamer->EmitIntValue(mdconst::extract<ConstantInt>( 778 Node->getOperand(1))->getZExtValue(), Size); 779 Streamer->EmitIntValue(mdconst::extract<ConstantInt>( 780 Node->getOperand(2))->getZExtValue(), Size); 781 } 782 783 void AMDGPUAsmPrinter::emitStartOfRuntimeMetadata(const Module &M) { 784 OutStreamer->SwitchSection(getObjFileLowering().getContext() 785 .getELFSection(RuntimeMD::SectionName, ELF::SHT_PROGBITS, 0)); 786 787 emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyMDVersion, 788 RuntimeMD::MDVersion << 8 | RuntimeMD::MDRevision, 2); 789 if (auto MD = M.getNamedMetadata("opencl.ocl.version")) { 790 if (MD->getNumOperands()) { 791 auto Node = MD->getOperand(0); 792 if (Node->getNumOperands() > 1) { 793 emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyLanguage, 794 RuntimeMD::OpenCL_C, 1); 795 uint16_t Major = mdconst::extract<ConstantInt>(Node->getOperand(0)) 796 ->getZExtValue(); 797 uint16_t Minor = mdconst::extract<ConstantInt>(Node->getOperand(1)) 798 ->getZExtValue(); 799 emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyLanguageVersion, 800 Major * 100 + Minor * 10, 2); 801 } 802 } 803 } 804 } 805 806 static std::string getOCLTypeName(Type *Ty, bool isSigned) { 807 if (VectorType* VecTy = dyn_cast<VectorType>(Ty)) { 808 Type* EleTy = VecTy->getElementType(); 809 unsigned Size = VecTy->getVectorNumElements(); 810 return (Twine(getOCLTypeName(EleTy, isSigned)) + Twine(Size)).str(); 811 } 812 switch (Ty->getTypeID()) { 813 case Type::HalfTyID: return "half"; 814 case Type::FloatTyID: return "float"; 815 case Type::DoubleTyID: return "double"; 816 case Type::IntegerTyID: { 817 if (!isSigned) 818 return (Twine('u') + Twine(getOCLTypeName(Ty, true))).str(); 819 auto IntTy = cast<IntegerType>(Ty); 820 auto BW = IntTy->getIntegerBitWidth(); 821 switch (BW) { 822 case 8: 823 return "char"; 824 case 16: 825 return "short"; 826 case 32: 827 return "int"; 828 case 64: 829 return "long"; 830 default: 831 return (Twine('i') + Twine(BW)).str(); 832 } 833 } 834 default: 835 llvm_unreachable("invalid type"); 836 } 837 } 838 839 static RuntimeMD::KernelArg::ValueType getRuntimeMDValueType( 840 Type *Ty, StringRef TypeName) { 841 if (auto VT = dyn_cast<VectorType>(Ty)) 842 return getRuntimeMDValueType(VT->getElementType(), TypeName); 843 else if (auto PT = dyn_cast<PointerType>(Ty)) 844 return getRuntimeMDValueType(PT->getElementType(), TypeName); 845 else if (Ty->isHalfTy()) 846 return RuntimeMD::KernelArg::F16; 847 else if (Ty->isFloatTy()) 848 return RuntimeMD::KernelArg::F32; 849 else if (Ty->isDoubleTy()) 850 return RuntimeMD::KernelArg::F64; 851 else if (IntegerType* intTy = dyn_cast<IntegerType>(Ty)) { 852 bool Signed = !TypeName.startswith("u"); 853 switch (intTy->getIntegerBitWidth()) { 854 case 8: 855 return Signed ? RuntimeMD::KernelArg::I8 : RuntimeMD::KernelArg::U8; 856 case 16: 857 return Signed ? RuntimeMD::KernelArg::I16 : RuntimeMD::KernelArg::U16; 858 case 32: 859 return Signed ? RuntimeMD::KernelArg::I32 : RuntimeMD::KernelArg::U32; 860 case 64: 861 return Signed ? RuntimeMD::KernelArg::I64 : RuntimeMD::KernelArg::U64; 862 default: 863 // Runtime does not recognize other integer types. Report as 864 // struct type. 865 return RuntimeMD::KernelArg::Struct; 866 } 867 } else 868 return RuntimeMD::KernelArg::Struct; 869 } 870 871 void AMDGPUAsmPrinter::emitRuntimeMetadata(const Function &F) { 872 if (!F.getMetadata("kernel_arg_type")) 873 return; 874 875 MCContext &Context = getObjFileLowering().getContext(); 876 OutStreamer->SwitchSection( 877 Context.getELFSection(RuntimeMD::SectionName, ELF::SHT_PROGBITS, 0)); 878 OutStreamer->EmitIntValue(RuntimeMD::KeyKernelBegin, 1); 879 emitRuntimeMDStringValue(OutStreamer, RuntimeMD::KeyKernelName, F.getName()); 880 881 for (auto &Arg:F.args()) { 882 // Emit KeyArgBegin. 883 unsigned I = Arg.getArgNo(); 884 OutStreamer->EmitIntValue(RuntimeMD::KeyArgBegin, 1); 885 886 // Emit KeyArgSize and KeyArgAlign. 887 auto T = Arg.getType(); 888 auto DL = F.getParent()->getDataLayout(); 889 emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyArgSize, 890 DL.getTypeAllocSize(T), 4); 891 emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyArgAlign, 892 DL.getABITypeAlignment(T), 4); 893 894 // Emit KeyArgTypeName. 895 auto TypeName = dyn_cast<MDString>(F.getMetadata( 896 "kernel_arg_type")->getOperand(I))->getString(); 897 emitRuntimeMDStringValue(OutStreamer, RuntimeMD::KeyArgTypeName, TypeName); 898 899 // Emit KeyArgName. 900 if (auto ArgNameMD = F.getMetadata("kernel_arg_name")) { 901 auto ArgName = cast<MDString>(ArgNameMD->getOperand( 902 I))->getString(); 903 emitRuntimeMDStringValue(OutStreamer, RuntimeMD::KeyArgName, ArgName); 904 } 905 906 // Emit KeyArgIsVolatile, KeyArgIsRestrict, KeyArgIsConst and KeyArgIsPipe. 907 auto TypeQual = cast<MDString>(F.getMetadata( 908 "kernel_arg_type_qual")->getOperand(I))->getString(); 909 SmallVector<StringRef, 1> SplitQ; 910 TypeQual.split(SplitQ, " ", -1, false/* drop empty entry*/); 911 for (auto &I:SplitQ) { 912 auto Key = StringSwitch<RuntimeMD::Key>(I) 913 .Case("volatile", RuntimeMD::KeyArgIsVolatile) 914 .Case("restrict", RuntimeMD::KeyArgIsRestrict) 915 .Case("const", RuntimeMD::KeyArgIsConst) 916 .Case("pipe", RuntimeMD::KeyArgIsPipe) 917 .Default(RuntimeMD::KeyNull); 918 OutStreamer->EmitIntValue(Key, 1); 919 } 920 921 // Emit KeyArgTypeKind. 922 auto BaseTypeName = cast<MDString>( 923 F.getMetadata("kernel_arg_base_type")->getOperand(I))->getString(); 924 auto TypeKind = StringSwitch<RuntimeMD::KernelArg::TypeKind>(BaseTypeName) 925 .Case("sampler_t", RuntimeMD::KernelArg::Sampler) 926 .Case("queue_t", RuntimeMD::KernelArg::Queue) 927 .Cases("image1d_t", "image1d_array_t", "image1d_buffer_t", 928 "image2d_t" , "image2d_array_t", RuntimeMD::KernelArg::Image) 929 .Cases("image2d_depth_t", "image2d_array_depth_t", 930 "image2d_msaa_t", "image2d_array_msaa_t", 931 "image2d_msaa_depth_t", RuntimeMD::KernelArg::Image) 932 .Cases("image2d_array_msaa_depth_t", "image3d_t", 933 RuntimeMD::KernelArg::Image) 934 .Default(isa<PointerType>(T) ? RuntimeMD::KernelArg::Pointer : 935 RuntimeMD::KernelArg::Value); 936 emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyArgTypeKind, TypeKind, 1); 937 938 // Emit KeyArgValueType. 939 emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyArgValueType, 940 getRuntimeMDValueType(T, BaseTypeName), 2); 941 942 // Emit KeyArgAccQual. 943 auto AccQual = cast<MDString>(F.getMetadata( 944 "kernel_arg_access_qual")->getOperand(I))->getString(); 945 auto AQ = StringSwitch<RuntimeMD::KernelArg::AccessQualifer>(AccQual) 946 .Case("read_only", RuntimeMD::KernelArg::ReadOnly) 947 .Case("write_only", RuntimeMD::KernelArg::WriteOnly) 948 .Case("read_write", RuntimeMD::KernelArg::ReadWrite) 949 .Default(RuntimeMD::KernelArg::None); 950 emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyArgAccQual, 951 AQ, 1); 952 953 // Emit KeyArgAddrQual. 954 if (isa<PointerType>(T)) 955 emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyArgAddrQual, 956 T->getPointerAddressSpace(), 1); 957 958 // Emit KeyArgEnd 959 OutStreamer->EmitIntValue(RuntimeMD::KeyArgEnd, 1); 960 } 961 962 // Emit KeyReqdWorkGroupSize, KeyWorkGroupSizeHint, and KeyVecTypeHint. 963 if (auto RWGS = F.getMetadata("reqd_work_group_size")) 964 emitRuntimeMDThreeIntValues(OutStreamer, RuntimeMD::KeyReqdWorkGroupSize, 965 RWGS, 4); 966 if (auto WGSH = F.getMetadata("work_group_size_hint")) 967 emitRuntimeMDThreeIntValues(OutStreamer, RuntimeMD::KeyWorkGroupSizeHint, 968 WGSH, 4); 969 if (auto VTH = F.getMetadata("vec_type_hint")) { 970 auto TypeName = getOCLTypeName(cast<ValueAsMetadata>( 971 VTH->getOperand(0))->getType(), mdconst::extract<ConstantInt>( 972 VTH->getOperand(1))->getZExtValue()); 973 emitRuntimeMDStringValue(OutStreamer, RuntimeMD::KeyVecTypeHint, 974 TypeName); 975 } 976 977 // Emit KeyKernelEnd 978 OutStreamer->EmitIntValue(RuntimeMD::KeyKernelEnd, 1); 979 } 980