1 //===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "AMDGPUBaseInfo.h" 10 #include "AMDGPU.h" 11 #include "AMDGPUAsmUtils.h" 12 #include "AMDKernelCodeT.h" 13 #include "GCNSubtarget.h" 14 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 15 #include "llvm/BinaryFormat/ELF.h" 16 #include "llvm/IR/Attributes.h" 17 #include "llvm/IR/Function.h" 18 #include "llvm/IR/GlobalValue.h" 19 #include "llvm/IR/IntrinsicsAMDGPU.h" 20 #include "llvm/IR/IntrinsicsR600.h" 21 #include "llvm/IR/LLVMContext.h" 22 #include "llvm/MC/MCSubtargetInfo.h" 23 #include "llvm/Support/AMDHSAKernelDescriptor.h" 24 #include "llvm/Support/CommandLine.h" 25 #include "llvm/Support/TargetParser.h" 26 27 #define GET_INSTRINFO_NAMED_OPS 28 #define GET_INSTRMAP_INFO 29 #include "AMDGPUGenInstrInfo.inc" 30 31 static llvm::cl::opt<unsigned> AmdhsaCodeObjectVersion( 32 "amdhsa-code-object-version", llvm::cl::Hidden, 33 llvm::cl::desc("AMDHSA Code Object Version"), llvm::cl::init(4), 34 llvm::cl::ZeroOrMore); 35 36 namespace { 37 38 /// \returns Bit mask for given bit \p Shift and bit \p Width. 39 unsigned getBitMask(unsigned Shift, unsigned Width) { 40 return ((1 << Width) - 1) << Shift; 41 } 42 43 /// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width. 44 /// 45 /// \returns Packed \p Dst. 46 unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) { 47 Dst &= ~(1 << Shift) & ~getBitMask(Shift, Width); 48 Dst |= (Src << Shift) & getBitMask(Shift, Width); 49 return Dst; 50 } 51 52 /// Unpacks bits from \p Src for given bit \p Shift and bit \p Width. 53 /// 54 /// \returns Unpacked bits. 55 unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) { 56 return (Src & getBitMask(Shift, Width)) >> Shift; 57 } 58 59 /// \returns Vmcnt bit shift (lower bits). 60 unsigned getVmcntBitShiftLo() { return 0; } 61 62 /// \returns Vmcnt bit width (lower bits). 63 unsigned getVmcntBitWidthLo() { return 4; } 64 65 /// \returns Expcnt bit shift. 66 unsigned getExpcntBitShift() { return 4; } 67 68 /// \returns Expcnt bit width. 69 unsigned getExpcntBitWidth() { return 3; } 70 71 /// \returns Lgkmcnt bit shift. 72 unsigned getLgkmcntBitShift() { return 8; } 73 74 /// \returns Lgkmcnt bit width. 75 unsigned getLgkmcntBitWidth(unsigned VersionMajor) { 76 return (VersionMajor >= 10) ? 6 : 4; 77 } 78 79 /// \returns Vmcnt bit shift (higher bits). 80 unsigned getVmcntBitShiftHi() { return 14; } 81 82 /// \returns Vmcnt bit width (higher bits). 83 unsigned getVmcntBitWidthHi() { return 2; } 84 85 } // end namespace anonymous 86 87 namespace llvm { 88 89 namespace AMDGPU { 90 91 Optional<uint8_t> getHsaAbiVersion(const MCSubtargetInfo *STI) { 92 if (STI && STI->getTargetTriple().getOS() != Triple::AMDHSA) 93 return None; 94 95 switch (AmdhsaCodeObjectVersion) { 96 case 2: 97 return ELF::ELFABIVERSION_AMDGPU_HSA_V2; 98 case 3: 99 return ELF::ELFABIVERSION_AMDGPU_HSA_V3; 100 case 4: 101 return ELF::ELFABIVERSION_AMDGPU_HSA_V4; 102 default: 103 report_fatal_error(Twine("Unsupported AMDHSA Code Object Version ") + 104 Twine(AmdhsaCodeObjectVersion)); 105 } 106 } 107 108 bool isHsaAbiVersion2(const MCSubtargetInfo *STI) { 109 if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI)) 110 return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V2; 111 return false; 112 } 113 114 bool isHsaAbiVersion3(const MCSubtargetInfo *STI) { 115 if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI)) 116 return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V3; 117 return false; 118 } 119 120 bool isHsaAbiVersion4(const MCSubtargetInfo *STI) { 121 if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI)) 122 return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V4; 123 return false; 124 } 125 126 bool isHsaAbiVersion3Or4(const MCSubtargetInfo *STI) { 127 return isHsaAbiVersion3(STI) || isHsaAbiVersion4(STI); 128 } 129 130 #define GET_MIMGBaseOpcodesTable_IMPL 131 #define GET_MIMGDimInfoTable_IMPL 132 #define GET_MIMGInfoTable_IMPL 133 #define GET_MIMGLZMappingTable_IMPL 134 #define GET_MIMGMIPMappingTable_IMPL 135 #define GET_MIMGG16MappingTable_IMPL 136 #include "AMDGPUGenSearchableTables.inc" 137 138 int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding, 139 unsigned VDataDwords, unsigned VAddrDwords) { 140 const MIMGInfo *Info = getMIMGOpcodeHelper(BaseOpcode, MIMGEncoding, 141 VDataDwords, VAddrDwords); 142 return Info ? Info->Opcode : -1; 143 } 144 145 const MIMGBaseOpcodeInfo *getMIMGBaseOpcode(unsigned Opc) { 146 const MIMGInfo *Info = getMIMGInfo(Opc); 147 return Info ? getMIMGBaseOpcodeInfo(Info->BaseOpcode) : nullptr; 148 } 149 150 int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) { 151 const MIMGInfo *OrigInfo = getMIMGInfo(Opc); 152 const MIMGInfo *NewInfo = 153 getMIMGOpcodeHelper(OrigInfo->BaseOpcode, OrigInfo->MIMGEncoding, 154 NewChannels, OrigInfo->VAddrDwords); 155 return NewInfo ? NewInfo->Opcode : -1; 156 } 157 158 struct MUBUFInfo { 159 uint16_t Opcode; 160 uint16_t BaseOpcode; 161 uint8_t elements; 162 bool has_vaddr; 163 bool has_srsrc; 164 bool has_soffset; 165 }; 166 167 struct MTBUFInfo { 168 uint16_t Opcode; 169 uint16_t BaseOpcode; 170 uint8_t elements; 171 bool has_vaddr; 172 bool has_srsrc; 173 bool has_soffset; 174 }; 175 176 struct SMInfo { 177 uint16_t Opcode; 178 bool IsBuffer; 179 }; 180 181 struct VOPInfo { 182 uint16_t Opcode; 183 bool IsSingle; 184 }; 185 186 #define GET_MTBUFInfoTable_DECL 187 #define GET_MTBUFInfoTable_IMPL 188 #define GET_MUBUFInfoTable_DECL 189 #define GET_MUBUFInfoTable_IMPL 190 #define GET_SMInfoTable_DECL 191 #define GET_SMInfoTable_IMPL 192 #define GET_VOP1InfoTable_DECL 193 #define GET_VOP1InfoTable_IMPL 194 #define GET_VOP2InfoTable_DECL 195 #define GET_VOP2InfoTable_IMPL 196 #define GET_VOP3InfoTable_DECL 197 #define GET_VOP3InfoTable_IMPL 198 #include "AMDGPUGenSearchableTables.inc" 199 200 int getMTBUFBaseOpcode(unsigned Opc) { 201 const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opc); 202 return Info ? Info->BaseOpcode : -1; 203 } 204 205 int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements) { 206 const MTBUFInfo *Info = getMTBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements); 207 return Info ? Info->Opcode : -1; 208 } 209 210 int getMTBUFElements(unsigned Opc) { 211 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 212 return Info ? Info->elements : 0; 213 } 214 215 bool getMTBUFHasVAddr(unsigned Opc) { 216 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 217 return Info ? Info->has_vaddr : false; 218 } 219 220 bool getMTBUFHasSrsrc(unsigned Opc) { 221 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 222 return Info ? Info->has_srsrc : false; 223 } 224 225 bool getMTBUFHasSoffset(unsigned Opc) { 226 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 227 return Info ? Info->has_soffset : false; 228 } 229 230 int getMUBUFBaseOpcode(unsigned Opc) { 231 const MUBUFInfo *Info = getMUBUFInfoFromOpcode(Opc); 232 return Info ? Info->BaseOpcode : -1; 233 } 234 235 int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements) { 236 const MUBUFInfo *Info = getMUBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements); 237 return Info ? Info->Opcode : -1; 238 } 239 240 int getMUBUFElements(unsigned Opc) { 241 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 242 return Info ? Info->elements : 0; 243 } 244 245 bool getMUBUFHasVAddr(unsigned Opc) { 246 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 247 return Info ? Info->has_vaddr : false; 248 } 249 250 bool getMUBUFHasSrsrc(unsigned Opc) { 251 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 252 return Info ? Info->has_srsrc : false; 253 } 254 255 bool getMUBUFHasSoffset(unsigned Opc) { 256 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 257 return Info ? Info->has_soffset : false; 258 } 259 260 bool getSMEMIsBuffer(unsigned Opc) { 261 const SMInfo *Info = getSMEMOpcodeHelper(Opc); 262 return Info ? Info->IsBuffer : false; 263 } 264 265 bool getVOP1IsSingle(unsigned Opc) { 266 const VOPInfo *Info = getVOP1OpcodeHelper(Opc); 267 return Info ? Info->IsSingle : false; 268 } 269 270 bool getVOP2IsSingle(unsigned Opc) { 271 const VOPInfo *Info = getVOP2OpcodeHelper(Opc); 272 return Info ? Info->IsSingle : false; 273 } 274 275 bool getVOP3IsSingle(unsigned Opc) { 276 const VOPInfo *Info = getVOP3OpcodeHelper(Opc); 277 return Info ? Info->IsSingle : false; 278 } 279 280 // Wrapper for Tablegen'd function. enum Subtarget is not defined in any 281 // header files, so we need to wrap it in a function that takes unsigned 282 // instead. 283 int getMCOpcode(uint16_t Opcode, unsigned Gen) { 284 return getMCOpcodeGen(Opcode, static_cast<Subtarget>(Gen)); 285 } 286 287 namespace IsaInfo { 288 289 AMDGPUTargetID::AMDGPUTargetID(const MCSubtargetInfo &STI) 290 : STI(STI), XnackSetting(TargetIDSetting::Any), 291 SramEccSetting(TargetIDSetting::Any) { 292 if (!STI.getFeatureBits().test(FeatureSupportsXNACK)) 293 XnackSetting = TargetIDSetting::Unsupported; 294 if (!STI.getFeatureBits().test(FeatureSupportsSRAMECC)) 295 SramEccSetting = TargetIDSetting::Unsupported; 296 } 297 298 void AMDGPUTargetID::setTargetIDFromFeaturesString(StringRef FS) { 299 // Check if xnack or sramecc is explicitly enabled or disabled. In the 300 // absence of the target features we assume we must generate code that can run 301 // in any environment. 302 SubtargetFeatures Features(FS); 303 Optional<bool> XnackRequested; 304 Optional<bool> SramEccRequested; 305 306 for (const std::string &Feature : Features.getFeatures()) { 307 if (Feature == "+xnack") 308 XnackRequested = true; 309 else if (Feature == "-xnack") 310 XnackRequested = false; 311 else if (Feature == "+sramecc") 312 SramEccRequested = true; 313 else if (Feature == "-sramecc") 314 SramEccRequested = false; 315 } 316 317 bool XnackSupported = isXnackSupported(); 318 bool SramEccSupported = isSramEccSupported(); 319 320 if (XnackRequested) { 321 if (XnackSupported) { 322 XnackSetting = 323 *XnackRequested ? TargetIDSetting::On : TargetIDSetting::Off; 324 } else { 325 // If a specific xnack setting was requested and this GPU does not support 326 // xnack emit a warning. Setting will remain set to "Unsupported". 327 if (*XnackRequested) { 328 errs() << "warning: xnack 'On' was requested for a processor that does " 329 "not support it!\n"; 330 } else { 331 errs() << "warning: xnack 'Off' was requested for a processor that " 332 "does not support it!\n"; 333 } 334 } 335 } 336 337 if (SramEccRequested) { 338 if (SramEccSupported) { 339 SramEccSetting = 340 *SramEccRequested ? TargetIDSetting::On : TargetIDSetting::Off; 341 } else { 342 // If a specific sramecc setting was requested and this GPU does not 343 // support sramecc emit a warning. Setting will remain set to 344 // "Unsupported". 345 if (*SramEccRequested) { 346 errs() << "warning: sramecc 'On' was requested for a processor that " 347 "does not support it!\n"; 348 } else { 349 errs() << "warning: sramecc 'Off' was requested for a processor that " 350 "does not support it!\n"; 351 } 352 } 353 } 354 } 355 356 static TargetIDSetting 357 getTargetIDSettingFromFeatureString(StringRef FeatureString) { 358 if (FeatureString.endswith("-")) 359 return TargetIDSetting::Off; 360 if (FeatureString.endswith("+")) 361 return TargetIDSetting::On; 362 363 llvm_unreachable("Malformed feature string"); 364 } 365 366 void AMDGPUTargetID::setTargetIDFromTargetIDStream(StringRef TargetID) { 367 SmallVector<StringRef, 3> TargetIDSplit; 368 TargetID.split(TargetIDSplit, ':'); 369 370 for (const auto &FeatureString : TargetIDSplit) { 371 if (FeatureString.startswith("xnack")) 372 XnackSetting = getTargetIDSettingFromFeatureString(FeatureString); 373 if (FeatureString.startswith("sramecc")) 374 SramEccSetting = getTargetIDSettingFromFeatureString(FeatureString); 375 } 376 } 377 378 std::string AMDGPUTargetID::toString() const { 379 std::string StringRep = ""; 380 raw_string_ostream StreamRep(StringRep); 381 382 auto TargetTriple = STI.getTargetTriple(); 383 auto Version = getIsaVersion(STI.getCPU()); 384 385 StreamRep << TargetTriple.getArchName() << '-' 386 << TargetTriple.getVendorName() << '-' 387 << TargetTriple.getOSName() << '-' 388 << TargetTriple.getEnvironmentName() << '-'; 389 390 std::string Processor = ""; 391 // TODO: Following else statement is present here because we used various 392 // alias names for GPUs up until GFX9 (e.g. 'fiji' is same as 'gfx803'). 393 // Remove once all aliases are removed from GCNProcessors.td. 394 if (Version.Major >= 9) 395 Processor = STI.getCPU().str(); 396 else 397 Processor = (Twine("gfx") + Twine(Version.Major) + Twine(Version.Minor) + 398 Twine(Version.Stepping)) 399 .str(); 400 401 std::string Features = ""; 402 if (Optional<uint8_t> HsaAbiVersion = getHsaAbiVersion(&STI)) { 403 switch (*HsaAbiVersion) { 404 case ELF::ELFABIVERSION_AMDGPU_HSA_V2: 405 // Code object V2 only supported specific processors and had fixed 406 // settings for the XNACK. 407 if (Processor == "gfx600") { 408 } else if (Processor == "gfx601") { 409 } else if (Processor == "gfx602") { 410 } else if (Processor == "gfx700") { 411 } else if (Processor == "gfx701") { 412 } else if (Processor == "gfx702") { 413 } else if (Processor == "gfx703") { 414 } else if (Processor == "gfx704") { 415 } else if (Processor == "gfx705") { 416 } else if (Processor == "gfx801") { 417 if (!isXnackOnOrAny()) 418 report_fatal_error( 419 "AMD GPU code object V2 does not support processor " + Processor + 420 " without XNACK"); 421 } else if (Processor == "gfx802") { 422 } else if (Processor == "gfx803") { 423 } else if (Processor == "gfx805") { 424 } else if (Processor == "gfx810") { 425 if (!isXnackOnOrAny()) 426 report_fatal_error( 427 "AMD GPU code object V2 does not support processor " + Processor + 428 " without XNACK"); 429 } else if (Processor == "gfx900") { 430 if (isXnackOnOrAny()) 431 Processor = "gfx901"; 432 } else if (Processor == "gfx902") { 433 if (isXnackOnOrAny()) 434 Processor = "gfx903"; 435 } else if (Processor == "gfx904") { 436 if (isXnackOnOrAny()) 437 Processor = "gfx905"; 438 } else if (Processor == "gfx906") { 439 if (isXnackOnOrAny()) 440 Processor = "gfx907"; 441 } else { 442 report_fatal_error( 443 "AMD GPU code object V2 does not support processor " + Processor); 444 } 445 break; 446 case ELF::ELFABIVERSION_AMDGPU_HSA_V3: 447 // xnack. 448 if (isXnackOnOrAny()) 449 Features += "+xnack"; 450 // In code object v2 and v3, "sramecc" feature was spelled with a 451 // hyphen ("sram-ecc"). 452 if (isSramEccOnOrAny()) 453 Features += "+sram-ecc"; 454 break; 455 case ELF::ELFABIVERSION_AMDGPU_HSA_V4: 456 // sramecc. 457 if (getSramEccSetting() == TargetIDSetting::Off) 458 Features += ":sramecc-"; 459 else if (getSramEccSetting() == TargetIDSetting::On) 460 Features += ":sramecc+"; 461 // xnack. 462 if (getXnackSetting() == TargetIDSetting::Off) 463 Features += ":xnack-"; 464 else if (getXnackSetting() == TargetIDSetting::On) 465 Features += ":xnack+"; 466 break; 467 default: 468 break; 469 } 470 } 471 472 StreamRep << Processor << Features; 473 474 StreamRep.flush(); 475 return StringRep; 476 } 477 478 unsigned getWavefrontSize(const MCSubtargetInfo *STI) { 479 if (STI->getFeatureBits().test(FeatureWavefrontSize16)) 480 return 16; 481 if (STI->getFeatureBits().test(FeatureWavefrontSize32)) 482 return 32; 483 484 return 64; 485 } 486 487 unsigned getLocalMemorySize(const MCSubtargetInfo *STI) { 488 if (STI->getFeatureBits().test(FeatureLocalMemorySize32768)) 489 return 32768; 490 if (STI->getFeatureBits().test(FeatureLocalMemorySize65536)) 491 return 65536; 492 493 return 0; 494 } 495 496 unsigned getEUsPerCU(const MCSubtargetInfo *STI) { 497 // "Per CU" really means "per whatever functional block the waves of a 498 // workgroup must share". For gfx10 in CU mode this is the CU, which contains 499 // two SIMDs. 500 if (isGFX10Plus(*STI) && STI->getFeatureBits().test(FeatureCuMode)) 501 return 2; 502 // Pre-gfx10 a CU contains four SIMDs. For gfx10 in WGP mode the WGP contains 503 // two CUs, so a total of four SIMDs. 504 return 4; 505 } 506 507 unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo *STI, 508 unsigned FlatWorkGroupSize) { 509 assert(FlatWorkGroupSize != 0); 510 if (STI->getTargetTriple().getArch() != Triple::amdgcn) 511 return 8; 512 unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize); 513 if (N == 1) 514 return 40; 515 N = 40 / N; 516 return std::min(N, 16u); 517 } 518 519 unsigned getMinWavesPerEU(const MCSubtargetInfo *STI) { 520 return 1; 521 } 522 523 unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI) { 524 // FIXME: Need to take scratch memory into account. 525 if (isGFX90A(*STI)) 526 return 8; 527 if (!isGFX10Plus(*STI)) 528 return 10; 529 return hasGFX10_3Insts(*STI) ? 16 : 20; 530 } 531 532 unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo *STI, 533 unsigned FlatWorkGroupSize) { 534 return divideCeil(getWavesPerWorkGroup(STI, FlatWorkGroupSize), 535 getEUsPerCU(STI)); 536 } 537 538 unsigned getMinFlatWorkGroupSize(const MCSubtargetInfo *STI) { 539 return 1; 540 } 541 542 unsigned getMaxFlatWorkGroupSize(const MCSubtargetInfo *STI) { 543 // Some subtargets allow encoding 2048, but this isn't tested or supported. 544 return 1024; 545 } 546 547 unsigned getWavesPerWorkGroup(const MCSubtargetInfo *STI, 548 unsigned FlatWorkGroupSize) { 549 return divideCeil(FlatWorkGroupSize, getWavefrontSize(STI)); 550 } 551 552 unsigned getSGPRAllocGranule(const MCSubtargetInfo *STI) { 553 IsaVersion Version = getIsaVersion(STI->getCPU()); 554 if (Version.Major >= 10) 555 return getAddressableNumSGPRs(STI); 556 if (Version.Major >= 8) 557 return 16; 558 return 8; 559 } 560 561 unsigned getSGPREncodingGranule(const MCSubtargetInfo *STI) { 562 return 8; 563 } 564 565 unsigned getTotalNumSGPRs(const MCSubtargetInfo *STI) { 566 IsaVersion Version = getIsaVersion(STI->getCPU()); 567 if (Version.Major >= 8) 568 return 800; 569 return 512; 570 } 571 572 unsigned getAddressableNumSGPRs(const MCSubtargetInfo *STI) { 573 if (STI->getFeatureBits().test(FeatureSGPRInitBug)) 574 return FIXED_NUM_SGPRS_FOR_INIT_BUG; 575 576 IsaVersion Version = getIsaVersion(STI->getCPU()); 577 if (Version.Major >= 10) 578 return 106; 579 if (Version.Major >= 8) 580 return 102; 581 return 104; 582 } 583 584 unsigned getMinNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 585 assert(WavesPerEU != 0); 586 587 IsaVersion Version = getIsaVersion(STI->getCPU()); 588 if (Version.Major >= 10) 589 return 0; 590 591 if (WavesPerEU >= getMaxWavesPerEU(STI)) 592 return 0; 593 594 unsigned MinNumSGPRs = getTotalNumSGPRs(STI) / (WavesPerEU + 1); 595 if (STI->getFeatureBits().test(FeatureTrapHandler)) 596 MinNumSGPRs -= std::min(MinNumSGPRs, (unsigned)TRAP_NUM_SGPRS); 597 MinNumSGPRs = alignDown(MinNumSGPRs, getSGPRAllocGranule(STI)) + 1; 598 return std::min(MinNumSGPRs, getAddressableNumSGPRs(STI)); 599 } 600 601 unsigned getMaxNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU, 602 bool Addressable) { 603 assert(WavesPerEU != 0); 604 605 unsigned AddressableNumSGPRs = getAddressableNumSGPRs(STI); 606 IsaVersion Version = getIsaVersion(STI->getCPU()); 607 if (Version.Major >= 10) 608 return Addressable ? AddressableNumSGPRs : 108; 609 if (Version.Major >= 8 && !Addressable) 610 AddressableNumSGPRs = 112; 611 unsigned MaxNumSGPRs = getTotalNumSGPRs(STI) / WavesPerEU; 612 if (STI->getFeatureBits().test(FeatureTrapHandler)) 613 MaxNumSGPRs -= std::min(MaxNumSGPRs, (unsigned)TRAP_NUM_SGPRS); 614 MaxNumSGPRs = alignDown(MaxNumSGPRs, getSGPRAllocGranule(STI)); 615 return std::min(MaxNumSGPRs, AddressableNumSGPRs); 616 } 617 618 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed, 619 bool FlatScrUsed, bool XNACKUsed) { 620 unsigned ExtraSGPRs = 0; 621 if (VCCUsed) 622 ExtraSGPRs = 2; 623 624 IsaVersion Version = getIsaVersion(STI->getCPU()); 625 if (Version.Major >= 10) 626 return ExtraSGPRs; 627 628 if (Version.Major < 8) { 629 if (FlatScrUsed) 630 ExtraSGPRs = 4; 631 } else { 632 if (XNACKUsed) 633 ExtraSGPRs = 4; 634 635 if (FlatScrUsed) 636 ExtraSGPRs = 6; 637 } 638 639 return ExtraSGPRs; 640 } 641 642 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed, 643 bool FlatScrUsed) { 644 return getNumExtraSGPRs(STI, VCCUsed, FlatScrUsed, 645 STI->getFeatureBits().test(AMDGPU::FeatureXNACK)); 646 } 647 648 unsigned getNumSGPRBlocks(const MCSubtargetInfo *STI, unsigned NumSGPRs) { 649 NumSGPRs = alignTo(std::max(1u, NumSGPRs), getSGPREncodingGranule(STI)); 650 // SGPRBlocks is actual number of SGPR blocks minus 1. 651 return NumSGPRs / getSGPREncodingGranule(STI) - 1; 652 } 653 654 unsigned getVGPRAllocGranule(const MCSubtargetInfo *STI, 655 Optional<bool> EnableWavefrontSize32) { 656 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 657 return 8; 658 659 bool IsWave32 = EnableWavefrontSize32 ? 660 *EnableWavefrontSize32 : 661 STI->getFeatureBits().test(FeatureWavefrontSize32); 662 663 if (hasGFX10_3Insts(*STI)) 664 return IsWave32 ? 16 : 8; 665 666 return IsWave32 ? 8 : 4; 667 } 668 669 unsigned getVGPREncodingGranule(const MCSubtargetInfo *STI, 670 Optional<bool> EnableWavefrontSize32) { 671 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 672 return 8; 673 674 bool IsWave32 = EnableWavefrontSize32 ? 675 *EnableWavefrontSize32 : 676 STI->getFeatureBits().test(FeatureWavefrontSize32); 677 678 return IsWave32 ? 8 : 4; 679 } 680 681 unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) { 682 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 683 return 512; 684 if (!isGFX10Plus(*STI)) 685 return 256; 686 return STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1024 : 512; 687 } 688 689 unsigned getAddressableNumVGPRs(const MCSubtargetInfo *STI) { 690 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 691 return 512; 692 return 256; 693 } 694 695 unsigned getMinNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 696 assert(WavesPerEU != 0); 697 698 if (WavesPerEU >= getMaxWavesPerEU(STI)) 699 return 0; 700 unsigned MinNumVGPRs = 701 alignDown(getTotalNumVGPRs(STI) / (WavesPerEU + 1), 702 getVGPRAllocGranule(STI)) + 1; 703 return std::min(MinNumVGPRs, getAddressableNumVGPRs(STI)); 704 } 705 706 unsigned getMaxNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 707 assert(WavesPerEU != 0); 708 709 unsigned MaxNumVGPRs = alignDown(getTotalNumVGPRs(STI) / WavesPerEU, 710 getVGPRAllocGranule(STI)); 711 unsigned AddressableNumVGPRs = getAddressableNumVGPRs(STI); 712 return std::min(MaxNumVGPRs, AddressableNumVGPRs); 713 } 714 715 unsigned getNumVGPRBlocks(const MCSubtargetInfo *STI, unsigned NumVGPRs, 716 Optional<bool> EnableWavefrontSize32) { 717 NumVGPRs = alignTo(std::max(1u, NumVGPRs), 718 getVGPREncodingGranule(STI, EnableWavefrontSize32)); 719 // VGPRBlocks is actual number of VGPR blocks minus 1. 720 return NumVGPRs / getVGPREncodingGranule(STI, EnableWavefrontSize32) - 1; 721 } 722 723 } // end namespace IsaInfo 724 725 void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header, 726 const MCSubtargetInfo *STI) { 727 IsaVersion Version = getIsaVersion(STI->getCPU()); 728 729 memset(&Header, 0, sizeof(Header)); 730 731 Header.amd_kernel_code_version_major = 1; 732 Header.amd_kernel_code_version_minor = 2; 733 Header.amd_machine_kind = 1; // AMD_MACHINE_KIND_AMDGPU 734 Header.amd_machine_version_major = Version.Major; 735 Header.amd_machine_version_minor = Version.Minor; 736 Header.amd_machine_version_stepping = Version.Stepping; 737 Header.kernel_code_entry_byte_offset = sizeof(Header); 738 Header.wavefront_size = 6; 739 740 // If the code object does not support indirect functions, then the value must 741 // be 0xffffffff. 742 Header.call_convention = -1; 743 744 // These alignment values are specified in powers of two, so alignment = 745 // 2^n. The minimum alignment is 2^4 = 16. 746 Header.kernarg_segment_alignment = 4; 747 Header.group_segment_alignment = 4; 748 Header.private_segment_alignment = 4; 749 750 if (Version.Major >= 10) { 751 if (STI->getFeatureBits().test(FeatureWavefrontSize32)) { 752 Header.wavefront_size = 5; 753 Header.code_properties |= AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32; 754 } 755 Header.compute_pgm_resource_registers |= 756 S_00B848_WGP_MODE(STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1) | 757 S_00B848_MEM_ORDERED(1); 758 } 759 } 760 761 amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor( 762 const MCSubtargetInfo *STI) { 763 IsaVersion Version = getIsaVersion(STI->getCPU()); 764 765 amdhsa::kernel_descriptor_t KD; 766 memset(&KD, 0, sizeof(KD)); 767 768 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 769 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, 770 amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE); 771 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 772 amdhsa::COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP, 1); 773 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 774 amdhsa::COMPUTE_PGM_RSRC1_ENABLE_IEEE_MODE, 1); 775 AMDHSA_BITS_SET(KD.compute_pgm_rsrc2, 776 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, 1); 777 if (Version.Major >= 10) { 778 AMDHSA_BITS_SET(KD.kernel_code_properties, 779 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, 780 STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1 : 0); 781 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 782 amdhsa::COMPUTE_PGM_RSRC1_WGP_MODE, 783 STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1); 784 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 785 amdhsa::COMPUTE_PGM_RSRC1_MEM_ORDERED, 1); 786 } 787 if (AMDGPU::isGFX90A(*STI)) { 788 AMDHSA_BITS_SET(KD.compute_pgm_rsrc3, 789 amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, 790 STI->getFeatureBits().test(FeatureTgSplit) ? 1 : 0); 791 } 792 return KD; 793 } 794 795 bool isGroupSegment(const GlobalValue *GV) { 796 return GV->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS; 797 } 798 799 bool isGlobalSegment(const GlobalValue *GV) { 800 return GV->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS; 801 } 802 803 bool isReadOnlySegment(const GlobalValue *GV) { 804 unsigned AS = GV->getAddressSpace(); 805 return AS == AMDGPUAS::CONSTANT_ADDRESS || 806 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT; 807 } 808 809 bool shouldEmitConstantsToTextSection(const Triple &TT) { 810 return TT.getArch() == Triple::r600; 811 } 812 813 int getIntegerAttribute(const Function &F, StringRef Name, int Default) { 814 Attribute A = F.getFnAttribute(Name); 815 int Result = Default; 816 817 if (A.isStringAttribute()) { 818 StringRef Str = A.getValueAsString(); 819 if (Str.getAsInteger(0, Result)) { 820 LLVMContext &Ctx = F.getContext(); 821 Ctx.emitError("can't parse integer attribute " + Name); 822 } 823 } 824 825 return Result; 826 } 827 828 std::pair<int, int> getIntegerPairAttribute(const Function &F, 829 StringRef Name, 830 std::pair<int, int> Default, 831 bool OnlyFirstRequired) { 832 Attribute A = F.getFnAttribute(Name); 833 if (!A.isStringAttribute()) 834 return Default; 835 836 LLVMContext &Ctx = F.getContext(); 837 std::pair<int, int> Ints = Default; 838 std::pair<StringRef, StringRef> Strs = A.getValueAsString().split(','); 839 if (Strs.first.trim().getAsInteger(0, Ints.first)) { 840 Ctx.emitError("can't parse first integer attribute " + Name); 841 return Default; 842 } 843 if (Strs.second.trim().getAsInteger(0, Ints.second)) { 844 if (!OnlyFirstRequired || !Strs.second.trim().empty()) { 845 Ctx.emitError("can't parse second integer attribute " + Name); 846 return Default; 847 } 848 } 849 850 return Ints; 851 } 852 853 unsigned getVmcntBitMask(const IsaVersion &Version) { 854 unsigned VmcntLo = (1 << getVmcntBitWidthLo()) - 1; 855 if (Version.Major < 9) 856 return VmcntLo; 857 858 unsigned VmcntHi = ((1 << getVmcntBitWidthHi()) - 1) << getVmcntBitWidthLo(); 859 return VmcntLo | VmcntHi; 860 } 861 862 unsigned getExpcntBitMask(const IsaVersion &Version) { 863 return (1 << getExpcntBitWidth()) - 1; 864 } 865 866 unsigned getLgkmcntBitMask(const IsaVersion &Version) { 867 return (1 << getLgkmcntBitWidth(Version.Major)) - 1; 868 } 869 870 unsigned getWaitcntBitMask(const IsaVersion &Version) { 871 unsigned VmcntLo = getBitMask(getVmcntBitShiftLo(), getVmcntBitWidthLo()); 872 unsigned Expcnt = getBitMask(getExpcntBitShift(), getExpcntBitWidth()); 873 unsigned Lgkmcnt = getBitMask(getLgkmcntBitShift(), 874 getLgkmcntBitWidth(Version.Major)); 875 unsigned Waitcnt = VmcntLo | Expcnt | Lgkmcnt; 876 if (Version.Major < 9) 877 return Waitcnt; 878 879 unsigned VmcntHi = getBitMask(getVmcntBitShiftHi(), getVmcntBitWidthHi()); 880 return Waitcnt | VmcntHi; 881 } 882 883 unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) { 884 unsigned VmcntLo = 885 unpackBits(Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo()); 886 if (Version.Major < 9) 887 return VmcntLo; 888 889 unsigned VmcntHi = 890 unpackBits(Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi()); 891 VmcntHi <<= getVmcntBitWidthLo(); 892 return VmcntLo | VmcntHi; 893 } 894 895 unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) { 896 return unpackBits(Waitcnt, getExpcntBitShift(), getExpcntBitWidth()); 897 } 898 899 unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) { 900 return unpackBits(Waitcnt, getLgkmcntBitShift(), 901 getLgkmcntBitWidth(Version.Major)); 902 } 903 904 void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, 905 unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt) { 906 Vmcnt = decodeVmcnt(Version, Waitcnt); 907 Expcnt = decodeExpcnt(Version, Waitcnt); 908 Lgkmcnt = decodeLgkmcnt(Version, Waitcnt); 909 } 910 911 Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded) { 912 Waitcnt Decoded; 913 Decoded.VmCnt = decodeVmcnt(Version, Encoded); 914 Decoded.ExpCnt = decodeExpcnt(Version, Encoded); 915 Decoded.LgkmCnt = decodeLgkmcnt(Version, Encoded); 916 return Decoded; 917 } 918 919 unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt, 920 unsigned Vmcnt) { 921 Waitcnt = 922 packBits(Vmcnt, Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo()); 923 if (Version.Major < 9) 924 return Waitcnt; 925 926 Vmcnt >>= getVmcntBitWidthLo(); 927 return packBits(Vmcnt, Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi()); 928 } 929 930 unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt, 931 unsigned Expcnt) { 932 return packBits(Expcnt, Waitcnt, getExpcntBitShift(), getExpcntBitWidth()); 933 } 934 935 unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt, 936 unsigned Lgkmcnt) { 937 return packBits(Lgkmcnt, Waitcnt, getLgkmcntBitShift(), 938 getLgkmcntBitWidth(Version.Major)); 939 } 940 941 unsigned encodeWaitcnt(const IsaVersion &Version, 942 unsigned Vmcnt, unsigned Expcnt, unsigned Lgkmcnt) { 943 unsigned Waitcnt = getWaitcntBitMask(Version); 944 Waitcnt = encodeVmcnt(Version, Waitcnt, Vmcnt); 945 Waitcnt = encodeExpcnt(Version, Waitcnt, Expcnt); 946 Waitcnt = encodeLgkmcnt(Version, Waitcnt, Lgkmcnt); 947 return Waitcnt; 948 } 949 950 unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded) { 951 return encodeWaitcnt(Version, Decoded.VmCnt, Decoded.ExpCnt, Decoded.LgkmCnt); 952 } 953 954 //===----------------------------------------------------------------------===// 955 // hwreg 956 //===----------------------------------------------------------------------===// 957 958 namespace Hwreg { 959 960 int64_t getHwregId(const StringRef Name) { 961 for (int Id = ID_SYMBOLIC_FIRST_; Id < ID_SYMBOLIC_LAST_; ++Id) { 962 if (IdSymbolic[Id] && Name == IdSymbolic[Id]) 963 return Id; 964 } 965 return ID_UNKNOWN_; 966 } 967 968 static unsigned getLastSymbolicHwreg(const MCSubtargetInfo &STI) { 969 if (isSI(STI) || isCI(STI) || isVI(STI)) 970 return ID_SYMBOLIC_FIRST_GFX9_; 971 else if (isGFX9(STI)) 972 return ID_SYMBOLIC_FIRST_GFX10_; 973 else if (isGFX10(STI) && !isGFX10_BEncoding(STI)) 974 return ID_SYMBOLIC_FIRST_GFX1030_; 975 else 976 return ID_SYMBOLIC_LAST_; 977 } 978 979 bool isValidHwreg(int64_t Id, const MCSubtargetInfo &STI) { 980 return 981 ID_SYMBOLIC_FIRST_ <= Id && Id < getLastSymbolicHwreg(STI) && 982 IdSymbolic[Id] && (Id != ID_XNACK_MASK || !AMDGPU::isGFX10_BEncoding(STI)); 983 } 984 985 bool isValidHwreg(int64_t Id) { 986 return 0 <= Id && isUInt<ID_WIDTH_>(Id); 987 } 988 989 bool isValidHwregOffset(int64_t Offset) { 990 return 0 <= Offset && isUInt<OFFSET_WIDTH_>(Offset); 991 } 992 993 bool isValidHwregWidth(int64_t Width) { 994 return 0 <= (Width - 1) && isUInt<WIDTH_M1_WIDTH_>(Width - 1); 995 } 996 997 uint64_t encodeHwreg(uint64_t Id, uint64_t Offset, uint64_t Width) { 998 return (Id << ID_SHIFT_) | 999 (Offset << OFFSET_SHIFT_) | 1000 ((Width - 1) << WIDTH_M1_SHIFT_); 1001 } 1002 1003 StringRef getHwreg(unsigned Id, const MCSubtargetInfo &STI) { 1004 return isValidHwreg(Id, STI) ? IdSymbolic[Id] : ""; 1005 } 1006 1007 void decodeHwreg(unsigned Val, unsigned &Id, unsigned &Offset, unsigned &Width) { 1008 Id = (Val & ID_MASK_) >> ID_SHIFT_; 1009 Offset = (Val & OFFSET_MASK_) >> OFFSET_SHIFT_; 1010 Width = ((Val & WIDTH_M1_MASK_) >> WIDTH_M1_SHIFT_) + 1; 1011 } 1012 1013 } // namespace Hwreg 1014 1015 //===----------------------------------------------------------------------===// 1016 // exp tgt 1017 //===----------------------------------------------------------------------===// 1018 1019 namespace Exp { 1020 1021 struct ExpTgt { 1022 StringLiteral Name; 1023 unsigned Tgt; 1024 unsigned MaxIndex; 1025 }; 1026 1027 static constexpr ExpTgt ExpTgtInfo[] = { 1028 {{"null"}, ET_NULL, ET_NULL_MAX_IDX}, 1029 {{"mrtz"}, ET_MRTZ, ET_MRTZ_MAX_IDX}, 1030 {{"prim"}, ET_PRIM, ET_PRIM_MAX_IDX}, 1031 {{"mrt"}, ET_MRT0, ET_MRT_MAX_IDX}, 1032 {{"pos"}, ET_POS0, ET_POS_MAX_IDX}, 1033 {{"param"}, ET_PARAM0, ET_PARAM_MAX_IDX}, 1034 }; 1035 1036 bool getTgtName(unsigned Id, StringRef &Name, int &Index) { 1037 for (const ExpTgt &Val : ExpTgtInfo) { 1038 if (Val.Tgt <= Id && Id <= Val.Tgt + Val.MaxIndex) { 1039 Index = (Val.MaxIndex == 0) ? -1 : (Id - Val.Tgt); 1040 Name = Val.Name; 1041 return true; 1042 } 1043 } 1044 return false; 1045 } 1046 1047 unsigned getTgtId(const StringRef Name) { 1048 1049 for (const ExpTgt &Val : ExpTgtInfo) { 1050 if (Val.MaxIndex == 0 && Name == Val.Name) 1051 return Val.Tgt; 1052 1053 if (Val.MaxIndex > 0 && Name.startswith(Val.Name)) { 1054 StringRef Suffix = Name.drop_front(Val.Name.size()); 1055 1056 unsigned Id; 1057 if (Suffix.getAsInteger(10, Id) || Id > Val.MaxIndex) 1058 return ET_INVALID; 1059 1060 // Disable leading zeroes 1061 if (Suffix.size() > 1 && Suffix[0] == '0') 1062 return ET_INVALID; 1063 1064 return Val.Tgt + Id; 1065 } 1066 } 1067 return ET_INVALID; 1068 } 1069 1070 bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI) { 1071 return (Id != ET_POS4 && Id != ET_PRIM) || isGFX10Plus(STI); 1072 } 1073 1074 } // namespace Exp 1075 1076 //===----------------------------------------------------------------------===// 1077 // MTBUF Format 1078 //===----------------------------------------------------------------------===// 1079 1080 namespace MTBUFFormat { 1081 1082 int64_t getDfmt(const StringRef Name) { 1083 for (int Id = DFMT_MIN; Id <= DFMT_MAX; ++Id) { 1084 if (Name == DfmtSymbolic[Id]) 1085 return Id; 1086 } 1087 return DFMT_UNDEF; 1088 } 1089 1090 StringRef getDfmtName(unsigned Id) { 1091 assert(Id <= DFMT_MAX); 1092 return DfmtSymbolic[Id]; 1093 } 1094 1095 static StringLiteral const *getNfmtLookupTable(const MCSubtargetInfo &STI) { 1096 if (isSI(STI) || isCI(STI)) 1097 return NfmtSymbolicSICI; 1098 if (isVI(STI) || isGFX9(STI)) 1099 return NfmtSymbolicVI; 1100 return NfmtSymbolicGFX10; 1101 } 1102 1103 int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI) { 1104 auto lookupTable = getNfmtLookupTable(STI); 1105 for (int Id = NFMT_MIN; Id <= NFMT_MAX; ++Id) { 1106 if (Name == lookupTable[Id]) 1107 return Id; 1108 } 1109 return NFMT_UNDEF; 1110 } 1111 1112 StringRef getNfmtName(unsigned Id, const MCSubtargetInfo &STI) { 1113 assert(Id <= NFMT_MAX); 1114 return getNfmtLookupTable(STI)[Id]; 1115 } 1116 1117 bool isValidDfmtNfmt(unsigned Id, const MCSubtargetInfo &STI) { 1118 unsigned Dfmt; 1119 unsigned Nfmt; 1120 decodeDfmtNfmt(Id, Dfmt, Nfmt); 1121 return isValidNfmt(Nfmt, STI); 1122 } 1123 1124 bool isValidNfmt(unsigned Id, const MCSubtargetInfo &STI) { 1125 return !getNfmtName(Id, STI).empty(); 1126 } 1127 1128 int64_t encodeDfmtNfmt(unsigned Dfmt, unsigned Nfmt) { 1129 return (Dfmt << DFMT_SHIFT) | (Nfmt << NFMT_SHIFT); 1130 } 1131 1132 void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt) { 1133 Dfmt = (Format >> DFMT_SHIFT) & DFMT_MASK; 1134 Nfmt = (Format >> NFMT_SHIFT) & NFMT_MASK; 1135 } 1136 1137 int64_t getUnifiedFormat(const StringRef Name) { 1138 for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) { 1139 if (Name == UfmtSymbolic[Id]) 1140 return Id; 1141 } 1142 return UFMT_UNDEF; 1143 } 1144 1145 StringRef getUnifiedFormatName(unsigned Id) { 1146 return isValidUnifiedFormat(Id) ? UfmtSymbolic[Id] : ""; 1147 } 1148 1149 bool isValidUnifiedFormat(unsigned Id) { 1150 return Id <= UFMT_LAST; 1151 } 1152 1153 int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt) { 1154 int64_t Fmt = encodeDfmtNfmt(Dfmt, Nfmt); 1155 for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) { 1156 if (Fmt == DfmtNfmt2UFmt[Id]) 1157 return Id; 1158 } 1159 return UFMT_UNDEF; 1160 } 1161 1162 bool isValidFormatEncoding(unsigned Val, const MCSubtargetInfo &STI) { 1163 return isGFX10Plus(STI) ? (Val <= UFMT_MAX) : (Val <= DFMT_NFMT_MAX); 1164 } 1165 1166 unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI) { 1167 if (isGFX10Plus(STI)) 1168 return UFMT_DEFAULT; 1169 return DFMT_NFMT_DEFAULT; 1170 } 1171 1172 } // namespace MTBUFFormat 1173 1174 //===----------------------------------------------------------------------===// 1175 // SendMsg 1176 //===----------------------------------------------------------------------===// 1177 1178 namespace SendMsg { 1179 1180 int64_t getMsgId(const StringRef Name) { 1181 for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) { 1182 if (IdSymbolic[i] && Name == IdSymbolic[i]) 1183 return i; 1184 } 1185 return ID_UNKNOWN_; 1186 } 1187 1188 bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI, bool Strict) { 1189 if (Strict) { 1190 switch (MsgId) { 1191 case ID_SAVEWAVE: 1192 return isVI(STI) || isGFX9Plus(STI); 1193 case ID_STALL_WAVE_GEN: 1194 case ID_HALT_WAVES: 1195 case ID_ORDERED_PS_DONE: 1196 case ID_GS_ALLOC_REQ: 1197 case ID_GET_DOORBELL: 1198 return isGFX9Plus(STI); 1199 case ID_EARLY_PRIM_DEALLOC: 1200 return isGFX9(STI); 1201 case ID_GET_DDID: 1202 return isGFX10Plus(STI); 1203 default: 1204 return 0 <= MsgId && MsgId < ID_GAPS_LAST_ && IdSymbolic[MsgId]; 1205 } 1206 } else { 1207 return 0 <= MsgId && isUInt<ID_WIDTH_>(MsgId); 1208 } 1209 } 1210 1211 StringRef getMsgName(int64_t MsgId) { 1212 assert(0 <= MsgId && MsgId < ID_GAPS_LAST_); 1213 return IdSymbolic[MsgId]; 1214 } 1215 1216 int64_t getMsgOpId(int64_t MsgId, const StringRef Name) { 1217 const char* const *S = (MsgId == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic; 1218 const int F = (MsgId == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_; 1219 const int L = (MsgId == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_; 1220 for (int i = F; i < L; ++i) { 1221 if (Name == S[i]) { 1222 return i; 1223 } 1224 } 1225 return OP_UNKNOWN_; 1226 } 1227 1228 bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI, 1229 bool Strict) { 1230 assert(isValidMsgId(MsgId, STI, Strict)); 1231 1232 if (!Strict) 1233 return 0 <= OpId && isUInt<OP_WIDTH_>(OpId); 1234 1235 switch(MsgId) 1236 { 1237 case ID_GS: 1238 return (OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_) && OpId != OP_GS_NOP; 1239 case ID_GS_DONE: 1240 return OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_; 1241 case ID_SYSMSG: 1242 return OP_SYS_FIRST_ <= OpId && OpId < OP_SYS_LAST_; 1243 default: 1244 return OpId == OP_NONE_; 1245 } 1246 } 1247 1248 StringRef getMsgOpName(int64_t MsgId, int64_t OpId) { 1249 assert(msgRequiresOp(MsgId)); 1250 return (MsgId == ID_SYSMSG)? OpSysSymbolic[OpId] : OpGsSymbolic[OpId]; 1251 } 1252 1253 bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId, 1254 const MCSubtargetInfo &STI, bool Strict) { 1255 assert(isValidMsgOp(MsgId, OpId, STI, Strict)); 1256 1257 if (!Strict) 1258 return 0 <= StreamId && isUInt<STREAM_ID_WIDTH_>(StreamId); 1259 1260 switch(MsgId) 1261 { 1262 case ID_GS: 1263 return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_; 1264 case ID_GS_DONE: 1265 return (OpId == OP_GS_NOP)? 1266 (StreamId == STREAM_ID_NONE_) : 1267 (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_); 1268 default: 1269 return StreamId == STREAM_ID_NONE_; 1270 } 1271 } 1272 1273 bool msgRequiresOp(int64_t MsgId) { 1274 return MsgId == ID_GS || MsgId == ID_GS_DONE || MsgId == ID_SYSMSG; 1275 } 1276 1277 bool msgSupportsStream(int64_t MsgId, int64_t OpId) { 1278 return (MsgId == ID_GS || MsgId == ID_GS_DONE) && OpId != OP_GS_NOP; 1279 } 1280 1281 void decodeMsg(unsigned Val, 1282 uint16_t &MsgId, 1283 uint16_t &OpId, 1284 uint16_t &StreamId) { 1285 MsgId = Val & ID_MASK_; 1286 OpId = (Val & OP_MASK_) >> OP_SHIFT_; 1287 StreamId = (Val & STREAM_ID_MASK_) >> STREAM_ID_SHIFT_; 1288 } 1289 1290 uint64_t encodeMsg(uint64_t MsgId, 1291 uint64_t OpId, 1292 uint64_t StreamId) { 1293 return (MsgId << ID_SHIFT_) | 1294 (OpId << OP_SHIFT_) | 1295 (StreamId << STREAM_ID_SHIFT_); 1296 } 1297 1298 } // namespace SendMsg 1299 1300 //===----------------------------------------------------------------------===// 1301 // 1302 //===----------------------------------------------------------------------===// 1303 1304 unsigned getInitialPSInputAddr(const Function &F) { 1305 return getIntegerAttribute(F, "InitialPSInputAddr", 0); 1306 } 1307 1308 bool isShader(CallingConv::ID cc) { 1309 switch(cc) { 1310 case CallingConv::AMDGPU_VS: 1311 case CallingConv::AMDGPU_LS: 1312 case CallingConv::AMDGPU_HS: 1313 case CallingConv::AMDGPU_ES: 1314 case CallingConv::AMDGPU_GS: 1315 case CallingConv::AMDGPU_PS: 1316 case CallingConv::AMDGPU_CS: 1317 return true; 1318 default: 1319 return false; 1320 } 1321 } 1322 1323 bool isGraphics(CallingConv::ID cc) { 1324 return isShader(cc) || cc == CallingConv::AMDGPU_Gfx; 1325 } 1326 1327 bool isCompute(CallingConv::ID cc) { 1328 return !isGraphics(cc) || cc == CallingConv::AMDGPU_CS; 1329 } 1330 1331 bool isEntryFunctionCC(CallingConv::ID CC) { 1332 switch (CC) { 1333 case CallingConv::AMDGPU_KERNEL: 1334 case CallingConv::SPIR_KERNEL: 1335 case CallingConv::AMDGPU_VS: 1336 case CallingConv::AMDGPU_GS: 1337 case CallingConv::AMDGPU_PS: 1338 case CallingConv::AMDGPU_CS: 1339 case CallingConv::AMDGPU_ES: 1340 case CallingConv::AMDGPU_HS: 1341 case CallingConv::AMDGPU_LS: 1342 return true; 1343 default: 1344 return false; 1345 } 1346 } 1347 1348 bool isModuleEntryFunctionCC(CallingConv::ID CC) { 1349 switch (CC) { 1350 case CallingConv::AMDGPU_Gfx: 1351 return true; 1352 default: 1353 return isEntryFunctionCC(CC); 1354 } 1355 } 1356 1357 bool hasXNACK(const MCSubtargetInfo &STI) { 1358 return STI.getFeatureBits()[AMDGPU::FeatureXNACK]; 1359 } 1360 1361 bool hasSRAMECC(const MCSubtargetInfo &STI) { 1362 return STI.getFeatureBits()[AMDGPU::FeatureSRAMECC]; 1363 } 1364 1365 bool hasMIMG_R128(const MCSubtargetInfo &STI) { 1366 return STI.getFeatureBits()[AMDGPU::FeatureMIMG_R128] && !STI.getFeatureBits()[AMDGPU::FeatureR128A16]; 1367 } 1368 1369 bool hasGFX10A16(const MCSubtargetInfo &STI) { 1370 return STI.getFeatureBits()[AMDGPU::FeatureGFX10A16]; 1371 } 1372 1373 bool hasG16(const MCSubtargetInfo &STI) { 1374 return STI.getFeatureBits()[AMDGPU::FeatureG16]; 1375 } 1376 1377 bool hasPackedD16(const MCSubtargetInfo &STI) { 1378 return !STI.getFeatureBits()[AMDGPU::FeatureUnpackedD16VMem]; 1379 } 1380 1381 bool isSI(const MCSubtargetInfo &STI) { 1382 return STI.getFeatureBits()[AMDGPU::FeatureSouthernIslands]; 1383 } 1384 1385 bool isCI(const MCSubtargetInfo &STI) { 1386 return STI.getFeatureBits()[AMDGPU::FeatureSeaIslands]; 1387 } 1388 1389 bool isVI(const MCSubtargetInfo &STI) { 1390 return STI.getFeatureBits()[AMDGPU::FeatureVolcanicIslands]; 1391 } 1392 1393 bool isGFX9(const MCSubtargetInfo &STI) { 1394 return STI.getFeatureBits()[AMDGPU::FeatureGFX9]; 1395 } 1396 1397 bool isGFX9Plus(const MCSubtargetInfo &STI) { 1398 return isGFX9(STI) || isGFX10Plus(STI); 1399 } 1400 1401 bool isGFX10(const MCSubtargetInfo &STI) { 1402 return STI.getFeatureBits()[AMDGPU::FeatureGFX10]; 1403 } 1404 1405 bool isGFX10Plus(const MCSubtargetInfo &STI) { return isGFX10(STI); } 1406 1407 bool isGCN3Encoding(const MCSubtargetInfo &STI) { 1408 return STI.getFeatureBits()[AMDGPU::FeatureGCN3Encoding]; 1409 } 1410 1411 bool isGFX10_BEncoding(const MCSubtargetInfo &STI) { 1412 return STI.getFeatureBits()[AMDGPU::FeatureGFX10_BEncoding]; 1413 } 1414 1415 bool hasGFX10_3Insts(const MCSubtargetInfo &STI) { 1416 return STI.getFeatureBits()[AMDGPU::FeatureGFX10_3Insts]; 1417 } 1418 1419 bool isGFX90A(const MCSubtargetInfo &STI) { 1420 return STI.getFeatureBits()[AMDGPU::FeatureGFX90AInsts]; 1421 } 1422 1423 bool isSGPR(unsigned Reg, const MCRegisterInfo* TRI) { 1424 const MCRegisterClass SGPRClass = TRI->getRegClass(AMDGPU::SReg_32RegClassID); 1425 const unsigned FirstSubReg = TRI->getSubReg(Reg, AMDGPU::sub0); 1426 return SGPRClass.contains(FirstSubReg != 0 ? FirstSubReg : Reg) || 1427 Reg == AMDGPU::SCC; 1428 } 1429 1430 bool isRegIntersect(unsigned Reg0, unsigned Reg1, const MCRegisterInfo* TRI) { 1431 for (MCRegAliasIterator R(Reg0, TRI, true); R.isValid(); ++R) { 1432 if (*R == Reg1) return true; 1433 } 1434 return false; 1435 } 1436 1437 #define MAP_REG2REG \ 1438 using namespace AMDGPU; \ 1439 switch(Reg) { \ 1440 default: return Reg; \ 1441 CASE_CI_VI(FLAT_SCR) \ 1442 CASE_CI_VI(FLAT_SCR_LO) \ 1443 CASE_CI_VI(FLAT_SCR_HI) \ 1444 CASE_VI_GFX9PLUS(TTMP0) \ 1445 CASE_VI_GFX9PLUS(TTMP1) \ 1446 CASE_VI_GFX9PLUS(TTMP2) \ 1447 CASE_VI_GFX9PLUS(TTMP3) \ 1448 CASE_VI_GFX9PLUS(TTMP4) \ 1449 CASE_VI_GFX9PLUS(TTMP5) \ 1450 CASE_VI_GFX9PLUS(TTMP6) \ 1451 CASE_VI_GFX9PLUS(TTMP7) \ 1452 CASE_VI_GFX9PLUS(TTMP8) \ 1453 CASE_VI_GFX9PLUS(TTMP9) \ 1454 CASE_VI_GFX9PLUS(TTMP10) \ 1455 CASE_VI_GFX9PLUS(TTMP11) \ 1456 CASE_VI_GFX9PLUS(TTMP12) \ 1457 CASE_VI_GFX9PLUS(TTMP13) \ 1458 CASE_VI_GFX9PLUS(TTMP14) \ 1459 CASE_VI_GFX9PLUS(TTMP15) \ 1460 CASE_VI_GFX9PLUS(TTMP0_TTMP1) \ 1461 CASE_VI_GFX9PLUS(TTMP2_TTMP3) \ 1462 CASE_VI_GFX9PLUS(TTMP4_TTMP5) \ 1463 CASE_VI_GFX9PLUS(TTMP6_TTMP7) \ 1464 CASE_VI_GFX9PLUS(TTMP8_TTMP9) \ 1465 CASE_VI_GFX9PLUS(TTMP10_TTMP11) \ 1466 CASE_VI_GFX9PLUS(TTMP12_TTMP13) \ 1467 CASE_VI_GFX9PLUS(TTMP14_TTMP15) \ 1468 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3) \ 1469 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7) \ 1470 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11) \ 1471 CASE_VI_GFX9PLUS(TTMP12_TTMP13_TTMP14_TTMP15) \ 1472 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7) \ 1473 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11) \ 1474 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \ 1475 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \ 1476 } 1477 1478 #define CASE_CI_VI(node) \ 1479 assert(!isSI(STI)); \ 1480 case node: return isCI(STI) ? node##_ci : node##_vi; 1481 1482 #define CASE_VI_GFX9PLUS(node) \ 1483 case node: return isGFX9Plus(STI) ? node##_gfx9plus : node##_vi; 1484 1485 unsigned getMCReg(unsigned Reg, const MCSubtargetInfo &STI) { 1486 if (STI.getTargetTriple().getArch() == Triple::r600) 1487 return Reg; 1488 MAP_REG2REG 1489 } 1490 1491 #undef CASE_CI_VI 1492 #undef CASE_VI_GFX9PLUS 1493 1494 #define CASE_CI_VI(node) case node##_ci: case node##_vi: return node; 1495 #define CASE_VI_GFX9PLUS(node) case node##_vi: case node##_gfx9plus: return node; 1496 1497 unsigned mc2PseudoReg(unsigned Reg) { 1498 MAP_REG2REG 1499 } 1500 1501 #undef CASE_CI_VI 1502 #undef CASE_VI_GFX9PLUS 1503 #undef MAP_REG2REG 1504 1505 bool isSISrcOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1506 assert(OpNo < Desc.NumOperands); 1507 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1508 return OpType >= AMDGPU::OPERAND_SRC_FIRST && 1509 OpType <= AMDGPU::OPERAND_SRC_LAST; 1510 } 1511 1512 bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1513 assert(OpNo < Desc.NumOperands); 1514 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1515 switch (OpType) { 1516 case AMDGPU::OPERAND_REG_IMM_FP32: 1517 case AMDGPU::OPERAND_REG_IMM_FP64: 1518 case AMDGPU::OPERAND_REG_IMM_FP16: 1519 case AMDGPU::OPERAND_REG_IMM_V2FP16: 1520 case AMDGPU::OPERAND_REG_IMM_V2INT16: 1521 case AMDGPU::OPERAND_REG_INLINE_C_FP32: 1522 case AMDGPU::OPERAND_REG_INLINE_C_FP64: 1523 case AMDGPU::OPERAND_REG_INLINE_C_FP16: 1524 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: 1525 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: 1526 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: 1527 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: 1528 case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: 1529 case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16: 1530 case AMDGPU::OPERAND_REG_IMM_V2FP32: 1531 case AMDGPU::OPERAND_REG_INLINE_C_V2FP32: 1532 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: 1533 return true; 1534 default: 1535 return false; 1536 } 1537 } 1538 1539 bool isSISrcInlinableOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1540 assert(OpNo < Desc.NumOperands); 1541 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1542 return OpType >= AMDGPU::OPERAND_REG_INLINE_C_FIRST && 1543 OpType <= AMDGPU::OPERAND_REG_INLINE_C_LAST; 1544 } 1545 1546 // Avoid using MCRegisterClass::getSize, since that function will go away 1547 // (move from MC* level to Target* level). Return size in bits. 1548 unsigned getRegBitWidth(unsigned RCID) { 1549 switch (RCID) { 1550 case AMDGPU::VGPR_LO16RegClassID: 1551 case AMDGPU::VGPR_HI16RegClassID: 1552 case AMDGPU::SGPR_LO16RegClassID: 1553 case AMDGPU::AGPR_LO16RegClassID: 1554 return 16; 1555 case AMDGPU::SGPR_32RegClassID: 1556 case AMDGPU::VGPR_32RegClassID: 1557 case AMDGPU::VRegOrLds_32RegClassID: 1558 case AMDGPU::AGPR_32RegClassID: 1559 case AMDGPU::VS_32RegClassID: 1560 case AMDGPU::AV_32RegClassID: 1561 case AMDGPU::SReg_32RegClassID: 1562 case AMDGPU::SReg_32_XM0RegClassID: 1563 case AMDGPU::SRegOrLds_32RegClassID: 1564 return 32; 1565 case AMDGPU::SGPR_64RegClassID: 1566 case AMDGPU::VS_64RegClassID: 1567 case AMDGPU::AV_64RegClassID: 1568 case AMDGPU::SReg_64RegClassID: 1569 case AMDGPU::VReg_64RegClassID: 1570 case AMDGPU::AReg_64RegClassID: 1571 case AMDGPU::SReg_64_XEXECRegClassID: 1572 case AMDGPU::VReg_64_Align2RegClassID: 1573 case AMDGPU::AReg_64_Align2RegClassID: 1574 return 64; 1575 case AMDGPU::SGPR_96RegClassID: 1576 case AMDGPU::SReg_96RegClassID: 1577 case AMDGPU::VReg_96RegClassID: 1578 case AMDGPU::AReg_96RegClassID: 1579 case AMDGPU::VReg_96_Align2RegClassID: 1580 case AMDGPU::AReg_96_Align2RegClassID: 1581 case AMDGPU::AV_96RegClassID: 1582 return 96; 1583 case AMDGPU::SGPR_128RegClassID: 1584 case AMDGPU::SReg_128RegClassID: 1585 case AMDGPU::VReg_128RegClassID: 1586 case AMDGPU::AReg_128RegClassID: 1587 case AMDGPU::VReg_128_Align2RegClassID: 1588 case AMDGPU::AReg_128_Align2RegClassID: 1589 case AMDGPU::AV_128RegClassID: 1590 return 128; 1591 case AMDGPU::SGPR_160RegClassID: 1592 case AMDGPU::SReg_160RegClassID: 1593 case AMDGPU::VReg_160RegClassID: 1594 case AMDGPU::AReg_160RegClassID: 1595 case AMDGPU::VReg_160_Align2RegClassID: 1596 case AMDGPU::AReg_160_Align2RegClassID: 1597 case AMDGPU::AV_160RegClassID: 1598 return 160; 1599 case AMDGPU::SGPR_192RegClassID: 1600 case AMDGPU::SReg_192RegClassID: 1601 case AMDGPU::VReg_192RegClassID: 1602 case AMDGPU::AReg_192RegClassID: 1603 case AMDGPU::VReg_192_Align2RegClassID: 1604 case AMDGPU::AReg_192_Align2RegClassID: 1605 return 192; 1606 case AMDGPU::SGPR_256RegClassID: 1607 case AMDGPU::SReg_256RegClassID: 1608 case AMDGPU::VReg_256RegClassID: 1609 case AMDGPU::AReg_256RegClassID: 1610 case AMDGPU::VReg_256_Align2RegClassID: 1611 case AMDGPU::AReg_256_Align2RegClassID: 1612 return 256; 1613 case AMDGPU::SGPR_512RegClassID: 1614 case AMDGPU::SReg_512RegClassID: 1615 case AMDGPU::VReg_512RegClassID: 1616 case AMDGPU::AReg_512RegClassID: 1617 case AMDGPU::VReg_512_Align2RegClassID: 1618 case AMDGPU::AReg_512_Align2RegClassID: 1619 return 512; 1620 case AMDGPU::SGPR_1024RegClassID: 1621 case AMDGPU::SReg_1024RegClassID: 1622 case AMDGPU::VReg_1024RegClassID: 1623 case AMDGPU::AReg_1024RegClassID: 1624 case AMDGPU::VReg_1024_Align2RegClassID: 1625 case AMDGPU::AReg_1024_Align2RegClassID: 1626 return 1024; 1627 default: 1628 llvm_unreachable("Unexpected register class"); 1629 } 1630 } 1631 1632 unsigned getRegBitWidth(const MCRegisterClass &RC) { 1633 return getRegBitWidth(RC.getID()); 1634 } 1635 1636 unsigned getRegOperandSize(const MCRegisterInfo *MRI, const MCInstrDesc &Desc, 1637 unsigned OpNo) { 1638 assert(OpNo < Desc.NumOperands); 1639 unsigned RCID = Desc.OpInfo[OpNo].RegClass; 1640 return getRegBitWidth(MRI->getRegClass(RCID)) / 8; 1641 } 1642 1643 bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi) { 1644 if (isInlinableIntLiteral(Literal)) 1645 return true; 1646 1647 uint64_t Val = static_cast<uint64_t>(Literal); 1648 return (Val == DoubleToBits(0.0)) || 1649 (Val == DoubleToBits(1.0)) || 1650 (Val == DoubleToBits(-1.0)) || 1651 (Val == DoubleToBits(0.5)) || 1652 (Val == DoubleToBits(-0.5)) || 1653 (Val == DoubleToBits(2.0)) || 1654 (Val == DoubleToBits(-2.0)) || 1655 (Val == DoubleToBits(4.0)) || 1656 (Val == DoubleToBits(-4.0)) || 1657 (Val == 0x3fc45f306dc9c882 && HasInv2Pi); 1658 } 1659 1660 bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi) { 1661 if (isInlinableIntLiteral(Literal)) 1662 return true; 1663 1664 // The actual type of the operand does not seem to matter as long 1665 // as the bits match one of the inline immediate values. For example: 1666 // 1667 // -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal, 1668 // so it is a legal inline immediate. 1669 // 1670 // 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in 1671 // floating-point, so it is a legal inline immediate. 1672 1673 uint32_t Val = static_cast<uint32_t>(Literal); 1674 return (Val == FloatToBits(0.0f)) || 1675 (Val == FloatToBits(1.0f)) || 1676 (Val == FloatToBits(-1.0f)) || 1677 (Val == FloatToBits(0.5f)) || 1678 (Val == FloatToBits(-0.5f)) || 1679 (Val == FloatToBits(2.0f)) || 1680 (Val == FloatToBits(-2.0f)) || 1681 (Val == FloatToBits(4.0f)) || 1682 (Val == FloatToBits(-4.0f)) || 1683 (Val == 0x3e22f983 && HasInv2Pi); 1684 } 1685 1686 bool isInlinableLiteral16(int16_t Literal, bool HasInv2Pi) { 1687 if (!HasInv2Pi) 1688 return false; 1689 1690 if (isInlinableIntLiteral(Literal)) 1691 return true; 1692 1693 uint16_t Val = static_cast<uint16_t>(Literal); 1694 return Val == 0x3C00 || // 1.0 1695 Val == 0xBC00 || // -1.0 1696 Val == 0x3800 || // 0.5 1697 Val == 0xB800 || // -0.5 1698 Val == 0x4000 || // 2.0 1699 Val == 0xC000 || // -2.0 1700 Val == 0x4400 || // 4.0 1701 Val == 0xC400 || // -4.0 1702 Val == 0x3118; // 1/2pi 1703 } 1704 1705 bool isInlinableLiteralV216(int32_t Literal, bool HasInv2Pi) { 1706 assert(HasInv2Pi); 1707 1708 if (isInt<16>(Literal) || isUInt<16>(Literal)) { 1709 int16_t Trunc = static_cast<int16_t>(Literal); 1710 return AMDGPU::isInlinableLiteral16(Trunc, HasInv2Pi); 1711 } 1712 if (!(Literal & 0xffff)) 1713 return AMDGPU::isInlinableLiteral16(Literal >> 16, HasInv2Pi); 1714 1715 int16_t Lo16 = static_cast<int16_t>(Literal); 1716 int16_t Hi16 = static_cast<int16_t>(Literal >> 16); 1717 return Lo16 == Hi16 && isInlinableLiteral16(Lo16, HasInv2Pi); 1718 } 1719 1720 bool isInlinableIntLiteralV216(int32_t Literal) { 1721 int16_t Lo16 = static_cast<int16_t>(Literal); 1722 if (isInt<16>(Literal) || isUInt<16>(Literal)) 1723 return isInlinableIntLiteral(Lo16); 1724 1725 int16_t Hi16 = static_cast<int16_t>(Literal >> 16); 1726 if (!(Literal & 0xffff)) 1727 return isInlinableIntLiteral(Hi16); 1728 return Lo16 == Hi16 && isInlinableIntLiteral(Lo16); 1729 } 1730 1731 bool isFoldableLiteralV216(int32_t Literal, bool HasInv2Pi) { 1732 assert(HasInv2Pi); 1733 1734 int16_t Lo16 = static_cast<int16_t>(Literal); 1735 if (isInt<16>(Literal) || isUInt<16>(Literal)) 1736 return true; 1737 1738 int16_t Hi16 = static_cast<int16_t>(Literal >> 16); 1739 if (!(Literal & 0xffff)) 1740 return true; 1741 return Lo16 == Hi16; 1742 } 1743 1744 bool isArgPassedInSGPR(const Argument *A) { 1745 const Function *F = A->getParent(); 1746 1747 // Arguments to compute shaders are never a source of divergence. 1748 CallingConv::ID CC = F->getCallingConv(); 1749 switch (CC) { 1750 case CallingConv::AMDGPU_KERNEL: 1751 case CallingConv::SPIR_KERNEL: 1752 return true; 1753 case CallingConv::AMDGPU_VS: 1754 case CallingConv::AMDGPU_LS: 1755 case CallingConv::AMDGPU_HS: 1756 case CallingConv::AMDGPU_ES: 1757 case CallingConv::AMDGPU_GS: 1758 case CallingConv::AMDGPU_PS: 1759 case CallingConv::AMDGPU_CS: 1760 case CallingConv::AMDGPU_Gfx: 1761 // For non-compute shaders, SGPR inputs are marked with either inreg or byval. 1762 // Everything else is in VGPRs. 1763 return F->getAttributes().hasParamAttribute(A->getArgNo(), Attribute::InReg) || 1764 F->getAttributes().hasParamAttribute(A->getArgNo(), Attribute::ByVal); 1765 default: 1766 // TODO: Should calls support inreg for SGPR inputs? 1767 return false; 1768 } 1769 } 1770 1771 static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) { 1772 return isGCN3Encoding(ST) || isGFX10Plus(ST); 1773 } 1774 1775 static bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST) { 1776 return isGFX9Plus(ST); 1777 } 1778 1779 bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST, 1780 int64_t EncodedOffset) { 1781 return hasSMEMByteOffset(ST) ? isUInt<20>(EncodedOffset) 1782 : isUInt<8>(EncodedOffset); 1783 } 1784 1785 bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST, 1786 int64_t EncodedOffset, 1787 bool IsBuffer) { 1788 return !IsBuffer && 1789 hasSMRDSignedImmOffset(ST) && 1790 isInt<21>(EncodedOffset); 1791 } 1792 1793 static bool isDwordAligned(uint64_t ByteOffset) { 1794 return (ByteOffset & 3) == 0; 1795 } 1796 1797 uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST, 1798 uint64_t ByteOffset) { 1799 if (hasSMEMByteOffset(ST)) 1800 return ByteOffset; 1801 1802 assert(isDwordAligned(ByteOffset)); 1803 return ByteOffset >> 2; 1804 } 1805 1806 Optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST, 1807 int64_t ByteOffset, bool IsBuffer) { 1808 // The signed version is always a byte offset. 1809 if (!IsBuffer && hasSMRDSignedImmOffset(ST)) { 1810 assert(hasSMEMByteOffset(ST)); 1811 return isInt<20>(ByteOffset) ? Optional<int64_t>(ByteOffset) : None; 1812 } 1813 1814 if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST)) 1815 return None; 1816 1817 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset); 1818 return isLegalSMRDEncodedUnsignedOffset(ST, EncodedOffset) 1819 ? Optional<int64_t>(EncodedOffset) 1820 : None; 1821 } 1822 1823 Optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST, 1824 int64_t ByteOffset) { 1825 if (!isCI(ST) || !isDwordAligned(ByteOffset)) 1826 return None; 1827 1828 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset); 1829 return isUInt<32>(EncodedOffset) ? Optional<int64_t>(EncodedOffset) : None; 1830 } 1831 1832 unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST, bool Signed) { 1833 // Address offset is 12-bit signed for GFX10, 13-bit for GFX9. 1834 if (AMDGPU::isGFX10(ST)) 1835 return Signed ? 12 : 11; 1836 1837 return Signed ? 13 : 12; 1838 } 1839 1840 // Given Imm, split it into the values to put into the SOffset and ImmOffset 1841 // fields in an MUBUF instruction. Return false if it is not possible (due to a 1842 // hardware bug needing a workaround). 1843 // 1844 // The required alignment ensures that individual address components remain 1845 // aligned if they are aligned to begin with. It also ensures that additional 1846 // offsets within the given alignment can be added to the resulting ImmOffset. 1847 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset, 1848 const GCNSubtarget *Subtarget, Align Alignment) { 1849 const uint32_t MaxImm = alignDown(4095, Alignment.value()); 1850 uint32_t Overflow = 0; 1851 1852 if (Imm > MaxImm) { 1853 if (Imm <= MaxImm + 64) { 1854 // Use an SOffset inline constant for 4..64 1855 Overflow = Imm - MaxImm; 1856 Imm = MaxImm; 1857 } else { 1858 // Try to keep the same value in SOffset for adjacent loads, so that 1859 // the corresponding register contents can be re-used. 1860 // 1861 // Load values with all low-bits (except for alignment bits) set into 1862 // SOffset, so that a larger range of values can be covered using 1863 // s_movk_i32. 1864 // 1865 // Atomic operations fail to work correctly when individual address 1866 // components are unaligned, even if their sum is aligned. 1867 uint32_t High = (Imm + Alignment.value()) & ~4095; 1868 uint32_t Low = (Imm + Alignment.value()) & 4095; 1869 Imm = Low; 1870 Overflow = High - Alignment.value(); 1871 } 1872 } 1873 1874 // There is a hardware bug in SI and CI which prevents address clamping in 1875 // MUBUF instructions from working correctly with SOffsets. The immediate 1876 // offset is unaffected. 1877 if (Overflow > 0 && 1878 Subtarget->getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS) 1879 return false; 1880 1881 ImmOffset = Imm; 1882 SOffset = Overflow; 1883 return true; 1884 } 1885 1886 SIModeRegisterDefaults::SIModeRegisterDefaults(const Function &F) { 1887 *this = getDefaultForCallingConv(F.getCallingConv()); 1888 1889 StringRef IEEEAttr = F.getFnAttribute("amdgpu-ieee").getValueAsString(); 1890 if (!IEEEAttr.empty()) 1891 IEEE = IEEEAttr == "true"; 1892 1893 StringRef DX10ClampAttr 1894 = F.getFnAttribute("amdgpu-dx10-clamp").getValueAsString(); 1895 if (!DX10ClampAttr.empty()) 1896 DX10Clamp = DX10ClampAttr == "true"; 1897 1898 StringRef DenormF32Attr = F.getFnAttribute("denormal-fp-math-f32").getValueAsString(); 1899 if (!DenormF32Attr.empty()) { 1900 DenormalMode DenormMode = parseDenormalFPAttribute(DenormF32Attr); 1901 FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE; 1902 FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE; 1903 } 1904 1905 StringRef DenormAttr = F.getFnAttribute("denormal-fp-math").getValueAsString(); 1906 if (!DenormAttr.empty()) { 1907 DenormalMode DenormMode = parseDenormalFPAttribute(DenormAttr); 1908 1909 if (DenormF32Attr.empty()) { 1910 FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE; 1911 FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE; 1912 } 1913 1914 FP64FP16InputDenormals = DenormMode.Input == DenormalMode::IEEE; 1915 FP64FP16OutputDenormals = DenormMode.Output == DenormalMode::IEEE; 1916 } 1917 } 1918 1919 namespace { 1920 1921 struct SourceOfDivergence { 1922 unsigned Intr; 1923 }; 1924 const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr); 1925 1926 #define GET_SourcesOfDivergence_IMPL 1927 #define GET_Gfx9BufferFormat_IMPL 1928 #define GET_Gfx10PlusBufferFormat_IMPL 1929 #include "AMDGPUGenSearchableTables.inc" 1930 1931 } // end anonymous namespace 1932 1933 bool isIntrinsicSourceOfDivergence(unsigned IntrID) { 1934 return lookupSourceOfDivergence(IntrID); 1935 } 1936 1937 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp, 1938 uint8_t NumComponents, 1939 uint8_t NumFormat, 1940 const MCSubtargetInfo &STI) { 1941 return isGFX10Plus(STI) 1942 ? getGfx10PlusBufferFormatInfo(BitsPerComp, NumComponents, 1943 NumFormat) 1944 : getGfx9BufferFormatInfo(BitsPerComp, NumComponents, NumFormat); 1945 } 1946 1947 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format, 1948 const MCSubtargetInfo &STI) { 1949 return isGFX10Plus(STI) ? getGfx10PlusBufferFormatInfo(Format) 1950 : getGfx9BufferFormatInfo(Format); 1951 } 1952 1953 } // namespace AMDGPU 1954 1955 raw_ostream &operator<<(raw_ostream &OS, 1956 const AMDGPU::IsaInfo::TargetIDSetting S) { 1957 switch (S) { 1958 case (AMDGPU::IsaInfo::TargetIDSetting::Unsupported): 1959 OS << "Unsupported"; 1960 break; 1961 case (AMDGPU::IsaInfo::TargetIDSetting::Any): 1962 OS << "Any"; 1963 break; 1964 case (AMDGPU::IsaInfo::TargetIDSetting::Off): 1965 OS << "Off"; 1966 break; 1967 case (AMDGPU::IsaInfo::TargetIDSetting::On): 1968 OS << "On"; 1969 break; 1970 } 1971 return OS; 1972 } 1973 1974 } // namespace llvm 1975