1 //===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "AMDGPUBaseInfo.h" 10 #include "AMDGPU.h" 11 #include "AMDGPUAsmUtils.h" 12 #include "AMDKernelCodeT.h" 13 #include "GCNSubtarget.h" 14 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 15 #include "llvm/BinaryFormat/ELF.h" 16 #include "llvm/IR/Attributes.h" 17 #include "llvm/IR/Function.h" 18 #include "llvm/IR/GlobalValue.h" 19 #include "llvm/IR/IntrinsicsAMDGPU.h" 20 #include "llvm/IR/IntrinsicsR600.h" 21 #include "llvm/IR/LLVMContext.h" 22 #include "llvm/MC/MCSubtargetInfo.h" 23 #include "llvm/Support/AMDHSAKernelDescriptor.h" 24 #include "llvm/Support/CommandLine.h" 25 #include "llvm/Support/TargetParser.h" 26 27 #define GET_INSTRINFO_NAMED_OPS 28 #define GET_INSTRMAP_INFO 29 #include "AMDGPUGenInstrInfo.inc" 30 31 static llvm::cl::opt<unsigned> AmdhsaCodeObjectVersion( 32 "amdhsa-code-object-version", llvm::cl::Hidden, 33 llvm::cl::desc("AMDHSA Code Object Version"), llvm::cl::init(4), 34 llvm::cl::ZeroOrMore); 35 36 namespace { 37 38 /// \returns Bit mask for given bit \p Shift and bit \p Width. 39 unsigned getBitMask(unsigned Shift, unsigned Width) { 40 return ((1 << Width) - 1) << Shift; 41 } 42 43 /// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width. 44 /// 45 /// \returns Packed \p Dst. 46 unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) { 47 Dst &= ~(1 << Shift) & ~getBitMask(Shift, Width); 48 Dst |= (Src << Shift) & getBitMask(Shift, Width); 49 return Dst; 50 } 51 52 /// Unpacks bits from \p Src for given bit \p Shift and bit \p Width. 53 /// 54 /// \returns Unpacked bits. 55 unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) { 56 return (Src & getBitMask(Shift, Width)) >> Shift; 57 } 58 59 /// \returns Vmcnt bit shift (lower bits). 60 unsigned getVmcntBitShiftLo() { return 0; } 61 62 /// \returns Vmcnt bit width (lower bits). 63 unsigned getVmcntBitWidthLo() { return 4; } 64 65 /// \returns Expcnt bit shift. 66 unsigned getExpcntBitShift() { return 4; } 67 68 /// \returns Expcnt bit width. 69 unsigned getExpcntBitWidth() { return 3; } 70 71 /// \returns Lgkmcnt bit shift. 72 unsigned getLgkmcntBitShift() { return 8; } 73 74 /// \returns Lgkmcnt bit width. 75 unsigned getLgkmcntBitWidth(unsigned VersionMajor) { 76 return (VersionMajor >= 10) ? 6 : 4; 77 } 78 79 /// \returns Vmcnt bit shift (higher bits). 80 unsigned getVmcntBitShiftHi() { return 14; } 81 82 /// \returns Vmcnt bit width (higher bits). 83 unsigned getVmcntBitWidthHi() { return 2; } 84 85 } // end namespace anonymous 86 87 namespace llvm { 88 89 namespace AMDGPU { 90 91 Optional<uint8_t> getHsaAbiVersion(const MCSubtargetInfo *STI) { 92 if (STI && STI->getTargetTriple().getOS() != Triple::AMDHSA) 93 return None; 94 95 switch (AmdhsaCodeObjectVersion) { 96 case 2: 97 return ELF::ELFABIVERSION_AMDGPU_HSA_V2; 98 case 3: 99 return ELF::ELFABIVERSION_AMDGPU_HSA_V3; 100 case 4: 101 return ELF::ELFABIVERSION_AMDGPU_HSA_V4; 102 default: 103 report_fatal_error(Twine("Unsupported AMDHSA Code Object Version ") + 104 Twine(AmdhsaCodeObjectVersion)); 105 } 106 } 107 108 bool isHsaAbiVersion2(const MCSubtargetInfo *STI) { 109 if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI)) 110 return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V2; 111 return false; 112 } 113 114 bool isHsaAbiVersion3(const MCSubtargetInfo *STI) { 115 if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI)) 116 return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V3; 117 return false; 118 } 119 120 bool isHsaAbiVersion4(const MCSubtargetInfo *STI) { 121 if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI)) 122 return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V4; 123 return false; 124 } 125 126 bool isHsaAbiVersion3Or4(const MCSubtargetInfo *STI) { 127 return isHsaAbiVersion3(STI) || isHsaAbiVersion4(STI); 128 } 129 130 #define GET_MIMGBaseOpcodesTable_IMPL 131 #define GET_MIMGDimInfoTable_IMPL 132 #define GET_MIMGInfoTable_IMPL 133 #define GET_MIMGLZMappingTable_IMPL 134 #define GET_MIMGMIPMappingTable_IMPL 135 #define GET_MIMGG16MappingTable_IMPL 136 #include "AMDGPUGenSearchableTables.inc" 137 138 int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding, 139 unsigned VDataDwords, unsigned VAddrDwords) { 140 const MIMGInfo *Info = getMIMGOpcodeHelper(BaseOpcode, MIMGEncoding, 141 VDataDwords, VAddrDwords); 142 return Info ? Info->Opcode : -1; 143 } 144 145 const MIMGBaseOpcodeInfo *getMIMGBaseOpcode(unsigned Opc) { 146 const MIMGInfo *Info = getMIMGInfo(Opc); 147 return Info ? getMIMGBaseOpcodeInfo(Info->BaseOpcode) : nullptr; 148 } 149 150 int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) { 151 const MIMGInfo *OrigInfo = getMIMGInfo(Opc); 152 const MIMGInfo *NewInfo = 153 getMIMGOpcodeHelper(OrigInfo->BaseOpcode, OrigInfo->MIMGEncoding, 154 NewChannels, OrigInfo->VAddrDwords); 155 return NewInfo ? NewInfo->Opcode : -1; 156 } 157 158 struct MUBUFInfo { 159 uint16_t Opcode; 160 uint16_t BaseOpcode; 161 uint8_t elements; 162 bool has_vaddr; 163 bool has_srsrc; 164 bool has_soffset; 165 }; 166 167 struct MTBUFInfo { 168 uint16_t Opcode; 169 uint16_t BaseOpcode; 170 uint8_t elements; 171 bool has_vaddr; 172 bool has_srsrc; 173 bool has_soffset; 174 }; 175 176 struct SMInfo { 177 uint16_t Opcode; 178 bool IsBuffer; 179 }; 180 181 struct VOPInfo { 182 uint16_t Opcode; 183 bool IsSingle; 184 }; 185 186 #define GET_MTBUFInfoTable_DECL 187 #define GET_MTBUFInfoTable_IMPL 188 #define GET_MUBUFInfoTable_DECL 189 #define GET_MUBUFInfoTable_IMPL 190 #define GET_SMInfoTable_DECL 191 #define GET_SMInfoTable_IMPL 192 #define GET_VOP1InfoTable_DECL 193 #define GET_VOP1InfoTable_IMPL 194 #define GET_VOP2InfoTable_DECL 195 #define GET_VOP2InfoTable_IMPL 196 #define GET_VOP3InfoTable_DECL 197 #define GET_VOP3InfoTable_IMPL 198 #include "AMDGPUGenSearchableTables.inc" 199 200 int getMTBUFBaseOpcode(unsigned Opc) { 201 const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opc); 202 return Info ? Info->BaseOpcode : -1; 203 } 204 205 int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements) { 206 const MTBUFInfo *Info = getMTBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements); 207 return Info ? Info->Opcode : -1; 208 } 209 210 int getMTBUFElements(unsigned Opc) { 211 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 212 return Info ? Info->elements : 0; 213 } 214 215 bool getMTBUFHasVAddr(unsigned Opc) { 216 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 217 return Info ? Info->has_vaddr : false; 218 } 219 220 bool getMTBUFHasSrsrc(unsigned Opc) { 221 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 222 return Info ? Info->has_srsrc : false; 223 } 224 225 bool getMTBUFHasSoffset(unsigned Opc) { 226 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 227 return Info ? Info->has_soffset : false; 228 } 229 230 int getMUBUFBaseOpcode(unsigned Opc) { 231 const MUBUFInfo *Info = getMUBUFInfoFromOpcode(Opc); 232 return Info ? Info->BaseOpcode : -1; 233 } 234 235 int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements) { 236 const MUBUFInfo *Info = getMUBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements); 237 return Info ? Info->Opcode : -1; 238 } 239 240 int getMUBUFElements(unsigned Opc) { 241 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 242 return Info ? Info->elements : 0; 243 } 244 245 bool getMUBUFHasVAddr(unsigned Opc) { 246 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 247 return Info ? Info->has_vaddr : false; 248 } 249 250 bool getMUBUFHasSrsrc(unsigned Opc) { 251 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 252 return Info ? Info->has_srsrc : false; 253 } 254 255 bool getMUBUFHasSoffset(unsigned Opc) { 256 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 257 return Info ? Info->has_soffset : false; 258 } 259 260 bool getSMEMIsBuffer(unsigned Opc) { 261 const SMInfo *Info = getSMEMOpcodeHelper(Opc); 262 return Info ? Info->IsBuffer : false; 263 } 264 265 bool getVOP1IsSingle(unsigned Opc) { 266 const VOPInfo *Info = getVOP1OpcodeHelper(Opc); 267 return Info ? Info->IsSingle : false; 268 } 269 270 bool getVOP2IsSingle(unsigned Opc) { 271 const VOPInfo *Info = getVOP2OpcodeHelper(Opc); 272 return Info ? Info->IsSingle : false; 273 } 274 275 bool getVOP3IsSingle(unsigned Opc) { 276 const VOPInfo *Info = getVOP3OpcodeHelper(Opc); 277 return Info ? Info->IsSingle : false; 278 } 279 280 // Wrapper for Tablegen'd function. enum Subtarget is not defined in any 281 // header files, so we need to wrap it in a function that takes unsigned 282 // instead. 283 int getMCOpcode(uint16_t Opcode, unsigned Gen) { 284 return getMCOpcodeGen(Opcode, static_cast<Subtarget>(Gen)); 285 } 286 287 namespace IsaInfo { 288 289 AMDGPUTargetID::AMDGPUTargetID(const MCSubtargetInfo &STI) 290 : STI(STI), XnackSetting(TargetIDSetting::Any), 291 SramEccSetting(TargetIDSetting::Any) { 292 if (!STI.getFeatureBits().test(FeatureSupportsXNACK)) 293 XnackSetting = TargetIDSetting::Unsupported; 294 if (!STI.getFeatureBits().test(FeatureSupportsSRAMECC)) 295 SramEccSetting = TargetIDSetting::Unsupported; 296 } 297 298 void AMDGPUTargetID::setTargetIDFromFeaturesString(StringRef FS) { 299 // Check if xnack or sramecc is explicitly enabled or disabled. In the 300 // absence of the target features we assume we must generate code that can run 301 // in any environment. 302 SubtargetFeatures Features(FS); 303 Optional<bool> XnackRequested; 304 Optional<bool> SramEccRequested; 305 306 for (const std::string &Feature : Features.getFeatures()) { 307 if (Feature == "+xnack") 308 XnackRequested = true; 309 else if (Feature == "-xnack") 310 XnackRequested = false; 311 else if (Feature == "+sramecc") 312 SramEccRequested = true; 313 else if (Feature == "-sramecc") 314 SramEccRequested = false; 315 } 316 317 bool XnackSupported = isXnackSupported(); 318 bool SramEccSupported = isSramEccSupported(); 319 320 if (XnackRequested) { 321 if (XnackSupported) { 322 XnackSetting = 323 *XnackRequested ? TargetIDSetting::On : TargetIDSetting::Off; 324 } else { 325 // If a specific xnack setting was requested and this GPU does not support 326 // xnack emit a warning. Setting will remain set to "Unsupported". 327 if (*XnackRequested) { 328 errs() << "warning: xnack 'On' was requested for a processor that does " 329 "not support it!\n"; 330 } else { 331 errs() << "warning: xnack 'Off' was requested for a processor that " 332 "does not support it!\n"; 333 } 334 } 335 } 336 337 if (SramEccRequested) { 338 if (SramEccSupported) { 339 SramEccSetting = 340 *SramEccRequested ? TargetIDSetting::On : TargetIDSetting::Off; 341 } else { 342 // If a specific sramecc setting was requested and this GPU does not 343 // support sramecc emit a warning. Setting will remain set to 344 // "Unsupported". 345 if (*SramEccRequested) { 346 errs() << "warning: sramecc 'On' was requested for a processor that " 347 "does not support it!\n"; 348 } else { 349 errs() << "warning: sramecc 'Off' was requested for a processor that " 350 "does not support it!\n"; 351 } 352 } 353 } 354 } 355 356 static TargetIDSetting 357 getTargetIDSettingFromFeatureString(StringRef FeatureString) { 358 if (FeatureString.endswith("-")) 359 return TargetIDSetting::Off; 360 if (FeatureString.endswith("+")) 361 return TargetIDSetting::On; 362 363 llvm_unreachable("Malformed feature string"); 364 } 365 366 void AMDGPUTargetID::setTargetIDFromTargetIDStream(StringRef TargetID) { 367 SmallVector<StringRef, 3> TargetIDSplit; 368 TargetID.split(TargetIDSplit, ':'); 369 370 for (const auto &FeatureString : TargetIDSplit) { 371 if (FeatureString.startswith("xnack")) 372 XnackSetting = getTargetIDSettingFromFeatureString(FeatureString); 373 if (FeatureString.startswith("sramecc")) 374 SramEccSetting = getTargetIDSettingFromFeatureString(FeatureString); 375 } 376 } 377 378 std::string AMDGPUTargetID::toString() const { 379 std::string StringRep = ""; 380 raw_string_ostream StreamRep(StringRep); 381 382 auto TargetTriple = STI.getTargetTriple(); 383 auto Version = getIsaVersion(STI.getCPU()); 384 385 StreamRep << TargetTriple.getArchName() << '-' 386 << TargetTriple.getVendorName() << '-' 387 << TargetTriple.getOSName() << '-' 388 << TargetTriple.getEnvironmentName() << '-'; 389 390 std::string Processor = ""; 391 // TODO: Following else statement is present here because we used various 392 // alias names for GPUs up until GFX9 (e.g. 'fiji' is same as 'gfx803'). 393 // Remove once all aliases are removed from GCNProcessors.td. 394 if (Version.Major >= 9) 395 Processor = STI.getCPU().str(); 396 else 397 Processor = (Twine("gfx") + Twine(Version.Major) + Twine(Version.Minor) + 398 Twine(Version.Stepping)) 399 .str(); 400 401 std::string Features = ""; 402 if (Optional<uint8_t> HsaAbiVersion = getHsaAbiVersion(&STI)) { 403 switch (*HsaAbiVersion) { 404 case ELF::ELFABIVERSION_AMDGPU_HSA_V2: 405 // Code object V2 only supported specific processors and had fixed 406 // settings for the XNACK. 407 if (Processor == "gfx600") { 408 } else if (Processor == "gfx601") { 409 } else if (Processor == "gfx602") { 410 } else if (Processor == "gfx700") { 411 } else if (Processor == "gfx701") { 412 } else if (Processor == "gfx702") { 413 } else if (Processor == "gfx703") { 414 } else if (Processor == "gfx704") { 415 } else if (Processor == "gfx705") { 416 } else if (Processor == "gfx801") { 417 if (!isXnackOnOrAny()) 418 report_fatal_error( 419 "AMD GPU code object V2 does not support processor " + Processor + 420 " without XNACK"); 421 } else if (Processor == "gfx802") { 422 } else if (Processor == "gfx803") { 423 } else if (Processor == "gfx805") { 424 } else if (Processor == "gfx810") { 425 if (!isXnackOnOrAny()) 426 report_fatal_error( 427 "AMD GPU code object V2 does not support processor " + Processor + 428 " without XNACK"); 429 } else if (Processor == "gfx900") { 430 if (isXnackOnOrAny()) 431 Processor = "gfx901"; 432 } else if (Processor == "gfx902") { 433 if (isXnackOnOrAny()) 434 Processor = "gfx903"; 435 } else if (Processor == "gfx904") { 436 if (isXnackOnOrAny()) 437 Processor = "gfx905"; 438 } else if (Processor == "gfx906") { 439 if (isXnackOnOrAny()) 440 Processor = "gfx907"; 441 } else if (Processor == "gfx90c") { 442 if (isXnackOnOrAny()) 443 report_fatal_error( 444 "AMD GPU code object V2 does not support processor " + Processor + 445 " with XNACK being ON or ANY"); 446 } else { 447 report_fatal_error( 448 "AMD GPU code object V2 does not support processor " + Processor); 449 } 450 break; 451 case ELF::ELFABIVERSION_AMDGPU_HSA_V3: 452 // xnack. 453 if (isXnackOnOrAny()) 454 Features += "+xnack"; 455 // In code object v2 and v3, "sramecc" feature was spelled with a 456 // hyphen ("sram-ecc"). 457 if (isSramEccOnOrAny()) 458 Features += "+sram-ecc"; 459 break; 460 case ELF::ELFABIVERSION_AMDGPU_HSA_V4: 461 // sramecc. 462 if (getSramEccSetting() == TargetIDSetting::Off) 463 Features += ":sramecc-"; 464 else if (getSramEccSetting() == TargetIDSetting::On) 465 Features += ":sramecc+"; 466 // xnack. 467 if (getXnackSetting() == TargetIDSetting::Off) 468 Features += ":xnack-"; 469 else if (getXnackSetting() == TargetIDSetting::On) 470 Features += ":xnack+"; 471 break; 472 default: 473 break; 474 } 475 } 476 477 StreamRep << Processor << Features; 478 479 StreamRep.flush(); 480 return StringRep; 481 } 482 483 unsigned getWavefrontSize(const MCSubtargetInfo *STI) { 484 if (STI->getFeatureBits().test(FeatureWavefrontSize16)) 485 return 16; 486 if (STI->getFeatureBits().test(FeatureWavefrontSize32)) 487 return 32; 488 489 return 64; 490 } 491 492 unsigned getLocalMemorySize(const MCSubtargetInfo *STI) { 493 if (STI->getFeatureBits().test(FeatureLocalMemorySize32768)) 494 return 32768; 495 if (STI->getFeatureBits().test(FeatureLocalMemorySize65536)) 496 return 65536; 497 498 return 0; 499 } 500 501 unsigned getEUsPerCU(const MCSubtargetInfo *STI) { 502 // "Per CU" really means "per whatever functional block the waves of a 503 // workgroup must share". For gfx10 in CU mode this is the CU, which contains 504 // two SIMDs. 505 if (isGFX10Plus(*STI) && STI->getFeatureBits().test(FeatureCuMode)) 506 return 2; 507 // Pre-gfx10 a CU contains four SIMDs. For gfx10 in WGP mode the WGP contains 508 // two CUs, so a total of four SIMDs. 509 return 4; 510 } 511 512 unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo *STI, 513 unsigned FlatWorkGroupSize) { 514 assert(FlatWorkGroupSize != 0); 515 if (STI->getTargetTriple().getArch() != Triple::amdgcn) 516 return 8; 517 unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize); 518 if (N == 1) 519 return 40; 520 N = 40 / N; 521 return std::min(N, 16u); 522 } 523 524 unsigned getMinWavesPerEU(const MCSubtargetInfo *STI) { 525 return 1; 526 } 527 528 unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI) { 529 // FIXME: Need to take scratch memory into account. 530 if (isGFX90A(*STI)) 531 return 8; 532 if (!isGFX10Plus(*STI)) 533 return 10; 534 return hasGFX10_3Insts(*STI) ? 16 : 20; 535 } 536 537 unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo *STI, 538 unsigned FlatWorkGroupSize) { 539 return divideCeil(getWavesPerWorkGroup(STI, FlatWorkGroupSize), 540 getEUsPerCU(STI)); 541 } 542 543 unsigned getMinFlatWorkGroupSize(const MCSubtargetInfo *STI) { 544 return 1; 545 } 546 547 unsigned getMaxFlatWorkGroupSize(const MCSubtargetInfo *STI) { 548 // Some subtargets allow encoding 2048, but this isn't tested or supported. 549 return 1024; 550 } 551 552 unsigned getWavesPerWorkGroup(const MCSubtargetInfo *STI, 553 unsigned FlatWorkGroupSize) { 554 return divideCeil(FlatWorkGroupSize, getWavefrontSize(STI)); 555 } 556 557 unsigned getSGPRAllocGranule(const MCSubtargetInfo *STI) { 558 IsaVersion Version = getIsaVersion(STI->getCPU()); 559 if (Version.Major >= 10) 560 return getAddressableNumSGPRs(STI); 561 if (Version.Major >= 8) 562 return 16; 563 return 8; 564 } 565 566 unsigned getSGPREncodingGranule(const MCSubtargetInfo *STI) { 567 return 8; 568 } 569 570 unsigned getTotalNumSGPRs(const MCSubtargetInfo *STI) { 571 IsaVersion Version = getIsaVersion(STI->getCPU()); 572 if (Version.Major >= 8) 573 return 800; 574 return 512; 575 } 576 577 unsigned getAddressableNumSGPRs(const MCSubtargetInfo *STI) { 578 if (STI->getFeatureBits().test(FeatureSGPRInitBug)) 579 return FIXED_NUM_SGPRS_FOR_INIT_BUG; 580 581 IsaVersion Version = getIsaVersion(STI->getCPU()); 582 if (Version.Major >= 10) 583 return 106; 584 if (Version.Major >= 8) 585 return 102; 586 return 104; 587 } 588 589 unsigned getMinNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 590 assert(WavesPerEU != 0); 591 592 IsaVersion Version = getIsaVersion(STI->getCPU()); 593 if (Version.Major >= 10) 594 return 0; 595 596 if (WavesPerEU >= getMaxWavesPerEU(STI)) 597 return 0; 598 599 unsigned MinNumSGPRs = getTotalNumSGPRs(STI) / (WavesPerEU + 1); 600 if (STI->getFeatureBits().test(FeatureTrapHandler)) 601 MinNumSGPRs -= std::min(MinNumSGPRs, (unsigned)TRAP_NUM_SGPRS); 602 MinNumSGPRs = alignDown(MinNumSGPRs, getSGPRAllocGranule(STI)) + 1; 603 return std::min(MinNumSGPRs, getAddressableNumSGPRs(STI)); 604 } 605 606 unsigned getMaxNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU, 607 bool Addressable) { 608 assert(WavesPerEU != 0); 609 610 unsigned AddressableNumSGPRs = getAddressableNumSGPRs(STI); 611 IsaVersion Version = getIsaVersion(STI->getCPU()); 612 if (Version.Major >= 10) 613 return Addressable ? AddressableNumSGPRs : 108; 614 if (Version.Major >= 8 && !Addressable) 615 AddressableNumSGPRs = 112; 616 unsigned MaxNumSGPRs = getTotalNumSGPRs(STI) / WavesPerEU; 617 if (STI->getFeatureBits().test(FeatureTrapHandler)) 618 MaxNumSGPRs -= std::min(MaxNumSGPRs, (unsigned)TRAP_NUM_SGPRS); 619 MaxNumSGPRs = alignDown(MaxNumSGPRs, getSGPRAllocGranule(STI)); 620 return std::min(MaxNumSGPRs, AddressableNumSGPRs); 621 } 622 623 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed, 624 bool FlatScrUsed, bool XNACKUsed) { 625 unsigned ExtraSGPRs = 0; 626 if (VCCUsed) 627 ExtraSGPRs = 2; 628 629 IsaVersion Version = getIsaVersion(STI->getCPU()); 630 if (Version.Major >= 10) 631 return ExtraSGPRs; 632 633 if (Version.Major < 8) { 634 if (FlatScrUsed) 635 ExtraSGPRs = 4; 636 } else { 637 if (XNACKUsed) 638 ExtraSGPRs = 4; 639 640 if (FlatScrUsed) 641 ExtraSGPRs = 6; 642 } 643 644 return ExtraSGPRs; 645 } 646 647 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed, 648 bool FlatScrUsed) { 649 return getNumExtraSGPRs(STI, VCCUsed, FlatScrUsed, 650 STI->getFeatureBits().test(AMDGPU::FeatureXNACK)); 651 } 652 653 unsigned getNumSGPRBlocks(const MCSubtargetInfo *STI, unsigned NumSGPRs) { 654 NumSGPRs = alignTo(std::max(1u, NumSGPRs), getSGPREncodingGranule(STI)); 655 // SGPRBlocks is actual number of SGPR blocks minus 1. 656 return NumSGPRs / getSGPREncodingGranule(STI) - 1; 657 } 658 659 unsigned getVGPRAllocGranule(const MCSubtargetInfo *STI, 660 Optional<bool> EnableWavefrontSize32) { 661 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 662 return 8; 663 664 bool IsWave32 = EnableWavefrontSize32 ? 665 *EnableWavefrontSize32 : 666 STI->getFeatureBits().test(FeatureWavefrontSize32); 667 668 if (hasGFX10_3Insts(*STI)) 669 return IsWave32 ? 16 : 8; 670 671 return IsWave32 ? 8 : 4; 672 } 673 674 unsigned getVGPREncodingGranule(const MCSubtargetInfo *STI, 675 Optional<bool> EnableWavefrontSize32) { 676 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 677 return 8; 678 679 bool IsWave32 = EnableWavefrontSize32 ? 680 *EnableWavefrontSize32 : 681 STI->getFeatureBits().test(FeatureWavefrontSize32); 682 683 return IsWave32 ? 8 : 4; 684 } 685 686 unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) { 687 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 688 return 512; 689 if (!isGFX10Plus(*STI)) 690 return 256; 691 return STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1024 : 512; 692 } 693 694 unsigned getAddressableNumVGPRs(const MCSubtargetInfo *STI) { 695 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 696 return 512; 697 return 256; 698 } 699 700 unsigned getMinNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 701 assert(WavesPerEU != 0); 702 703 if (WavesPerEU >= getMaxWavesPerEU(STI)) 704 return 0; 705 unsigned MinNumVGPRs = 706 alignDown(getTotalNumVGPRs(STI) / (WavesPerEU + 1), 707 getVGPRAllocGranule(STI)) + 1; 708 return std::min(MinNumVGPRs, getAddressableNumVGPRs(STI)); 709 } 710 711 unsigned getMaxNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 712 assert(WavesPerEU != 0); 713 714 unsigned MaxNumVGPRs = alignDown(getTotalNumVGPRs(STI) / WavesPerEU, 715 getVGPRAllocGranule(STI)); 716 unsigned AddressableNumVGPRs = getAddressableNumVGPRs(STI); 717 return std::min(MaxNumVGPRs, AddressableNumVGPRs); 718 } 719 720 unsigned getNumVGPRBlocks(const MCSubtargetInfo *STI, unsigned NumVGPRs, 721 Optional<bool> EnableWavefrontSize32) { 722 NumVGPRs = alignTo(std::max(1u, NumVGPRs), 723 getVGPREncodingGranule(STI, EnableWavefrontSize32)); 724 // VGPRBlocks is actual number of VGPR blocks minus 1. 725 return NumVGPRs / getVGPREncodingGranule(STI, EnableWavefrontSize32) - 1; 726 } 727 728 } // end namespace IsaInfo 729 730 void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header, 731 const MCSubtargetInfo *STI) { 732 IsaVersion Version = getIsaVersion(STI->getCPU()); 733 734 memset(&Header, 0, sizeof(Header)); 735 736 Header.amd_kernel_code_version_major = 1; 737 Header.amd_kernel_code_version_minor = 2; 738 Header.amd_machine_kind = 1; // AMD_MACHINE_KIND_AMDGPU 739 Header.amd_machine_version_major = Version.Major; 740 Header.amd_machine_version_minor = Version.Minor; 741 Header.amd_machine_version_stepping = Version.Stepping; 742 Header.kernel_code_entry_byte_offset = sizeof(Header); 743 Header.wavefront_size = 6; 744 745 // If the code object does not support indirect functions, then the value must 746 // be 0xffffffff. 747 Header.call_convention = -1; 748 749 // These alignment values are specified in powers of two, so alignment = 750 // 2^n. The minimum alignment is 2^4 = 16. 751 Header.kernarg_segment_alignment = 4; 752 Header.group_segment_alignment = 4; 753 Header.private_segment_alignment = 4; 754 755 if (Version.Major >= 10) { 756 if (STI->getFeatureBits().test(FeatureWavefrontSize32)) { 757 Header.wavefront_size = 5; 758 Header.code_properties |= AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32; 759 } 760 Header.compute_pgm_resource_registers |= 761 S_00B848_WGP_MODE(STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1) | 762 S_00B848_MEM_ORDERED(1); 763 } 764 } 765 766 amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor( 767 const MCSubtargetInfo *STI) { 768 IsaVersion Version = getIsaVersion(STI->getCPU()); 769 770 amdhsa::kernel_descriptor_t KD; 771 memset(&KD, 0, sizeof(KD)); 772 773 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 774 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, 775 amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE); 776 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 777 amdhsa::COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP, 1); 778 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 779 amdhsa::COMPUTE_PGM_RSRC1_ENABLE_IEEE_MODE, 1); 780 AMDHSA_BITS_SET(KD.compute_pgm_rsrc2, 781 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, 1); 782 if (Version.Major >= 10) { 783 AMDHSA_BITS_SET(KD.kernel_code_properties, 784 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, 785 STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1 : 0); 786 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 787 amdhsa::COMPUTE_PGM_RSRC1_WGP_MODE, 788 STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1); 789 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 790 amdhsa::COMPUTE_PGM_RSRC1_MEM_ORDERED, 1); 791 } 792 if (AMDGPU::isGFX90A(*STI)) { 793 AMDHSA_BITS_SET(KD.compute_pgm_rsrc3, 794 amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, 795 STI->getFeatureBits().test(FeatureTgSplit) ? 1 : 0); 796 } 797 return KD; 798 } 799 800 bool isGroupSegment(const GlobalValue *GV) { 801 return GV->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS; 802 } 803 804 bool isGlobalSegment(const GlobalValue *GV) { 805 return GV->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS; 806 } 807 808 bool isReadOnlySegment(const GlobalValue *GV) { 809 unsigned AS = GV->getAddressSpace(); 810 return AS == AMDGPUAS::CONSTANT_ADDRESS || 811 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT; 812 } 813 814 bool shouldEmitConstantsToTextSection(const Triple &TT) { 815 return TT.getArch() == Triple::r600; 816 } 817 818 int getIntegerAttribute(const Function &F, StringRef Name, int Default) { 819 Attribute A = F.getFnAttribute(Name); 820 int Result = Default; 821 822 if (A.isStringAttribute()) { 823 StringRef Str = A.getValueAsString(); 824 if (Str.getAsInteger(0, Result)) { 825 LLVMContext &Ctx = F.getContext(); 826 Ctx.emitError("can't parse integer attribute " + Name); 827 } 828 } 829 830 return Result; 831 } 832 833 std::pair<int, int> getIntegerPairAttribute(const Function &F, 834 StringRef Name, 835 std::pair<int, int> Default, 836 bool OnlyFirstRequired) { 837 Attribute A = F.getFnAttribute(Name); 838 if (!A.isStringAttribute()) 839 return Default; 840 841 LLVMContext &Ctx = F.getContext(); 842 std::pair<int, int> Ints = Default; 843 std::pair<StringRef, StringRef> Strs = A.getValueAsString().split(','); 844 if (Strs.first.trim().getAsInteger(0, Ints.first)) { 845 Ctx.emitError("can't parse first integer attribute " + Name); 846 return Default; 847 } 848 if (Strs.second.trim().getAsInteger(0, Ints.second)) { 849 if (!OnlyFirstRequired || !Strs.second.trim().empty()) { 850 Ctx.emitError("can't parse second integer attribute " + Name); 851 return Default; 852 } 853 } 854 855 return Ints; 856 } 857 858 unsigned getVmcntBitMask(const IsaVersion &Version) { 859 unsigned VmcntLo = (1 << getVmcntBitWidthLo()) - 1; 860 if (Version.Major < 9) 861 return VmcntLo; 862 863 unsigned VmcntHi = ((1 << getVmcntBitWidthHi()) - 1) << getVmcntBitWidthLo(); 864 return VmcntLo | VmcntHi; 865 } 866 867 unsigned getExpcntBitMask(const IsaVersion &Version) { 868 return (1 << getExpcntBitWidth()) - 1; 869 } 870 871 unsigned getLgkmcntBitMask(const IsaVersion &Version) { 872 return (1 << getLgkmcntBitWidth(Version.Major)) - 1; 873 } 874 875 unsigned getWaitcntBitMask(const IsaVersion &Version) { 876 unsigned VmcntLo = getBitMask(getVmcntBitShiftLo(), getVmcntBitWidthLo()); 877 unsigned Expcnt = getBitMask(getExpcntBitShift(), getExpcntBitWidth()); 878 unsigned Lgkmcnt = getBitMask(getLgkmcntBitShift(), 879 getLgkmcntBitWidth(Version.Major)); 880 unsigned Waitcnt = VmcntLo | Expcnt | Lgkmcnt; 881 if (Version.Major < 9) 882 return Waitcnt; 883 884 unsigned VmcntHi = getBitMask(getVmcntBitShiftHi(), getVmcntBitWidthHi()); 885 return Waitcnt | VmcntHi; 886 } 887 888 unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) { 889 unsigned VmcntLo = 890 unpackBits(Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo()); 891 if (Version.Major < 9) 892 return VmcntLo; 893 894 unsigned VmcntHi = 895 unpackBits(Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi()); 896 VmcntHi <<= getVmcntBitWidthLo(); 897 return VmcntLo | VmcntHi; 898 } 899 900 unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) { 901 return unpackBits(Waitcnt, getExpcntBitShift(), getExpcntBitWidth()); 902 } 903 904 unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) { 905 return unpackBits(Waitcnt, getLgkmcntBitShift(), 906 getLgkmcntBitWidth(Version.Major)); 907 } 908 909 void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, 910 unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt) { 911 Vmcnt = decodeVmcnt(Version, Waitcnt); 912 Expcnt = decodeExpcnt(Version, Waitcnt); 913 Lgkmcnt = decodeLgkmcnt(Version, Waitcnt); 914 } 915 916 Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded) { 917 Waitcnt Decoded; 918 Decoded.VmCnt = decodeVmcnt(Version, Encoded); 919 Decoded.ExpCnt = decodeExpcnt(Version, Encoded); 920 Decoded.LgkmCnt = decodeLgkmcnt(Version, Encoded); 921 return Decoded; 922 } 923 924 unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt, 925 unsigned Vmcnt) { 926 Waitcnt = 927 packBits(Vmcnt, Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo()); 928 if (Version.Major < 9) 929 return Waitcnt; 930 931 Vmcnt >>= getVmcntBitWidthLo(); 932 return packBits(Vmcnt, Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi()); 933 } 934 935 unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt, 936 unsigned Expcnt) { 937 return packBits(Expcnt, Waitcnt, getExpcntBitShift(), getExpcntBitWidth()); 938 } 939 940 unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt, 941 unsigned Lgkmcnt) { 942 return packBits(Lgkmcnt, Waitcnt, getLgkmcntBitShift(), 943 getLgkmcntBitWidth(Version.Major)); 944 } 945 946 unsigned encodeWaitcnt(const IsaVersion &Version, 947 unsigned Vmcnt, unsigned Expcnt, unsigned Lgkmcnt) { 948 unsigned Waitcnt = getWaitcntBitMask(Version); 949 Waitcnt = encodeVmcnt(Version, Waitcnt, Vmcnt); 950 Waitcnt = encodeExpcnt(Version, Waitcnt, Expcnt); 951 Waitcnt = encodeLgkmcnt(Version, Waitcnt, Lgkmcnt); 952 return Waitcnt; 953 } 954 955 unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded) { 956 return encodeWaitcnt(Version, Decoded.VmCnt, Decoded.ExpCnt, Decoded.LgkmCnt); 957 } 958 959 //===----------------------------------------------------------------------===// 960 // hwreg 961 //===----------------------------------------------------------------------===// 962 963 namespace Hwreg { 964 965 int64_t getHwregId(const StringRef Name) { 966 for (int Id = ID_SYMBOLIC_FIRST_; Id < ID_SYMBOLIC_LAST_; ++Id) { 967 if (IdSymbolic[Id] && Name == IdSymbolic[Id]) 968 return Id; 969 } 970 return ID_UNKNOWN_; 971 } 972 973 static unsigned getLastSymbolicHwreg(const MCSubtargetInfo &STI) { 974 if (isSI(STI) || isCI(STI) || isVI(STI)) 975 return ID_SYMBOLIC_FIRST_GFX9_; 976 else if (isGFX9(STI)) 977 return ID_SYMBOLIC_FIRST_GFX10_; 978 else if (isGFX10(STI) && !isGFX10_BEncoding(STI)) 979 return ID_SYMBOLIC_FIRST_GFX1030_; 980 else 981 return ID_SYMBOLIC_LAST_; 982 } 983 984 bool isValidHwreg(int64_t Id, const MCSubtargetInfo &STI) { 985 return 986 ID_SYMBOLIC_FIRST_ <= Id && Id < getLastSymbolicHwreg(STI) && 987 IdSymbolic[Id] && (Id != ID_XNACK_MASK || !AMDGPU::isGFX10_BEncoding(STI)); 988 } 989 990 bool isValidHwreg(int64_t Id) { 991 return 0 <= Id && isUInt<ID_WIDTH_>(Id); 992 } 993 994 bool isValidHwregOffset(int64_t Offset) { 995 return 0 <= Offset && isUInt<OFFSET_WIDTH_>(Offset); 996 } 997 998 bool isValidHwregWidth(int64_t Width) { 999 return 0 <= (Width - 1) && isUInt<WIDTH_M1_WIDTH_>(Width - 1); 1000 } 1001 1002 uint64_t encodeHwreg(uint64_t Id, uint64_t Offset, uint64_t Width) { 1003 return (Id << ID_SHIFT_) | 1004 (Offset << OFFSET_SHIFT_) | 1005 ((Width - 1) << WIDTH_M1_SHIFT_); 1006 } 1007 1008 StringRef getHwreg(unsigned Id, const MCSubtargetInfo &STI) { 1009 return isValidHwreg(Id, STI) ? IdSymbolic[Id] : ""; 1010 } 1011 1012 void decodeHwreg(unsigned Val, unsigned &Id, unsigned &Offset, unsigned &Width) { 1013 Id = (Val & ID_MASK_) >> ID_SHIFT_; 1014 Offset = (Val & OFFSET_MASK_) >> OFFSET_SHIFT_; 1015 Width = ((Val & WIDTH_M1_MASK_) >> WIDTH_M1_SHIFT_) + 1; 1016 } 1017 1018 } // namespace Hwreg 1019 1020 //===----------------------------------------------------------------------===// 1021 // exp tgt 1022 //===----------------------------------------------------------------------===// 1023 1024 namespace Exp { 1025 1026 struct ExpTgt { 1027 StringLiteral Name; 1028 unsigned Tgt; 1029 unsigned MaxIndex; 1030 }; 1031 1032 static constexpr ExpTgt ExpTgtInfo[] = { 1033 {{"null"}, ET_NULL, ET_NULL_MAX_IDX}, 1034 {{"mrtz"}, ET_MRTZ, ET_MRTZ_MAX_IDX}, 1035 {{"prim"}, ET_PRIM, ET_PRIM_MAX_IDX}, 1036 {{"mrt"}, ET_MRT0, ET_MRT_MAX_IDX}, 1037 {{"pos"}, ET_POS0, ET_POS_MAX_IDX}, 1038 {{"param"}, ET_PARAM0, ET_PARAM_MAX_IDX}, 1039 }; 1040 1041 bool getTgtName(unsigned Id, StringRef &Name, int &Index) { 1042 for (const ExpTgt &Val : ExpTgtInfo) { 1043 if (Val.Tgt <= Id && Id <= Val.Tgt + Val.MaxIndex) { 1044 Index = (Val.MaxIndex == 0) ? -1 : (Id - Val.Tgt); 1045 Name = Val.Name; 1046 return true; 1047 } 1048 } 1049 return false; 1050 } 1051 1052 unsigned getTgtId(const StringRef Name) { 1053 1054 for (const ExpTgt &Val : ExpTgtInfo) { 1055 if (Val.MaxIndex == 0 && Name == Val.Name) 1056 return Val.Tgt; 1057 1058 if (Val.MaxIndex > 0 && Name.startswith(Val.Name)) { 1059 StringRef Suffix = Name.drop_front(Val.Name.size()); 1060 1061 unsigned Id; 1062 if (Suffix.getAsInteger(10, Id) || Id > Val.MaxIndex) 1063 return ET_INVALID; 1064 1065 // Disable leading zeroes 1066 if (Suffix.size() > 1 && Suffix[0] == '0') 1067 return ET_INVALID; 1068 1069 return Val.Tgt + Id; 1070 } 1071 } 1072 return ET_INVALID; 1073 } 1074 1075 bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI) { 1076 return (Id != ET_POS4 && Id != ET_PRIM) || isGFX10Plus(STI); 1077 } 1078 1079 } // namespace Exp 1080 1081 //===----------------------------------------------------------------------===// 1082 // MTBUF Format 1083 //===----------------------------------------------------------------------===// 1084 1085 namespace MTBUFFormat { 1086 1087 int64_t getDfmt(const StringRef Name) { 1088 for (int Id = DFMT_MIN; Id <= DFMT_MAX; ++Id) { 1089 if (Name == DfmtSymbolic[Id]) 1090 return Id; 1091 } 1092 return DFMT_UNDEF; 1093 } 1094 1095 StringRef getDfmtName(unsigned Id) { 1096 assert(Id <= DFMT_MAX); 1097 return DfmtSymbolic[Id]; 1098 } 1099 1100 static StringLiteral const *getNfmtLookupTable(const MCSubtargetInfo &STI) { 1101 if (isSI(STI) || isCI(STI)) 1102 return NfmtSymbolicSICI; 1103 if (isVI(STI) || isGFX9(STI)) 1104 return NfmtSymbolicVI; 1105 return NfmtSymbolicGFX10; 1106 } 1107 1108 int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI) { 1109 auto lookupTable = getNfmtLookupTable(STI); 1110 for (int Id = NFMT_MIN; Id <= NFMT_MAX; ++Id) { 1111 if (Name == lookupTable[Id]) 1112 return Id; 1113 } 1114 return NFMT_UNDEF; 1115 } 1116 1117 StringRef getNfmtName(unsigned Id, const MCSubtargetInfo &STI) { 1118 assert(Id <= NFMT_MAX); 1119 return getNfmtLookupTable(STI)[Id]; 1120 } 1121 1122 bool isValidDfmtNfmt(unsigned Id, const MCSubtargetInfo &STI) { 1123 unsigned Dfmt; 1124 unsigned Nfmt; 1125 decodeDfmtNfmt(Id, Dfmt, Nfmt); 1126 return isValidNfmt(Nfmt, STI); 1127 } 1128 1129 bool isValidNfmt(unsigned Id, const MCSubtargetInfo &STI) { 1130 return !getNfmtName(Id, STI).empty(); 1131 } 1132 1133 int64_t encodeDfmtNfmt(unsigned Dfmt, unsigned Nfmt) { 1134 return (Dfmt << DFMT_SHIFT) | (Nfmt << NFMT_SHIFT); 1135 } 1136 1137 void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt) { 1138 Dfmt = (Format >> DFMT_SHIFT) & DFMT_MASK; 1139 Nfmt = (Format >> NFMT_SHIFT) & NFMT_MASK; 1140 } 1141 1142 int64_t getUnifiedFormat(const StringRef Name) { 1143 for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) { 1144 if (Name == UfmtSymbolic[Id]) 1145 return Id; 1146 } 1147 return UFMT_UNDEF; 1148 } 1149 1150 StringRef getUnifiedFormatName(unsigned Id) { 1151 return isValidUnifiedFormat(Id) ? UfmtSymbolic[Id] : ""; 1152 } 1153 1154 bool isValidUnifiedFormat(unsigned Id) { 1155 return Id <= UFMT_LAST; 1156 } 1157 1158 int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt) { 1159 int64_t Fmt = encodeDfmtNfmt(Dfmt, Nfmt); 1160 for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) { 1161 if (Fmt == DfmtNfmt2UFmt[Id]) 1162 return Id; 1163 } 1164 return UFMT_UNDEF; 1165 } 1166 1167 bool isValidFormatEncoding(unsigned Val, const MCSubtargetInfo &STI) { 1168 return isGFX10Plus(STI) ? (Val <= UFMT_MAX) : (Val <= DFMT_NFMT_MAX); 1169 } 1170 1171 unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI) { 1172 if (isGFX10Plus(STI)) 1173 return UFMT_DEFAULT; 1174 return DFMT_NFMT_DEFAULT; 1175 } 1176 1177 } // namespace MTBUFFormat 1178 1179 //===----------------------------------------------------------------------===// 1180 // SendMsg 1181 //===----------------------------------------------------------------------===// 1182 1183 namespace SendMsg { 1184 1185 int64_t getMsgId(const StringRef Name) { 1186 for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) { 1187 if (IdSymbolic[i] && Name == IdSymbolic[i]) 1188 return i; 1189 } 1190 return ID_UNKNOWN_; 1191 } 1192 1193 bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI, bool Strict) { 1194 if (Strict) { 1195 switch (MsgId) { 1196 case ID_SAVEWAVE: 1197 return isVI(STI) || isGFX9Plus(STI); 1198 case ID_STALL_WAVE_GEN: 1199 case ID_HALT_WAVES: 1200 case ID_ORDERED_PS_DONE: 1201 case ID_GS_ALLOC_REQ: 1202 case ID_GET_DOORBELL: 1203 return isGFX9Plus(STI); 1204 case ID_EARLY_PRIM_DEALLOC: 1205 return isGFX9(STI); 1206 case ID_GET_DDID: 1207 return isGFX10Plus(STI); 1208 default: 1209 return 0 <= MsgId && MsgId < ID_GAPS_LAST_ && IdSymbolic[MsgId]; 1210 } 1211 } else { 1212 return 0 <= MsgId && isUInt<ID_WIDTH_>(MsgId); 1213 } 1214 } 1215 1216 StringRef getMsgName(int64_t MsgId) { 1217 assert(0 <= MsgId && MsgId < ID_GAPS_LAST_); 1218 return IdSymbolic[MsgId]; 1219 } 1220 1221 int64_t getMsgOpId(int64_t MsgId, const StringRef Name) { 1222 const char* const *S = (MsgId == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic; 1223 const int F = (MsgId == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_; 1224 const int L = (MsgId == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_; 1225 for (int i = F; i < L; ++i) { 1226 if (Name == S[i]) { 1227 return i; 1228 } 1229 } 1230 return OP_UNKNOWN_; 1231 } 1232 1233 bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI, 1234 bool Strict) { 1235 assert(isValidMsgId(MsgId, STI, Strict)); 1236 1237 if (!Strict) 1238 return 0 <= OpId && isUInt<OP_WIDTH_>(OpId); 1239 1240 switch(MsgId) 1241 { 1242 case ID_GS: 1243 return (OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_) && OpId != OP_GS_NOP; 1244 case ID_GS_DONE: 1245 return OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_; 1246 case ID_SYSMSG: 1247 return OP_SYS_FIRST_ <= OpId && OpId < OP_SYS_LAST_; 1248 default: 1249 return OpId == OP_NONE_; 1250 } 1251 } 1252 1253 StringRef getMsgOpName(int64_t MsgId, int64_t OpId) { 1254 assert(msgRequiresOp(MsgId)); 1255 return (MsgId == ID_SYSMSG)? OpSysSymbolic[OpId] : OpGsSymbolic[OpId]; 1256 } 1257 1258 bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId, 1259 const MCSubtargetInfo &STI, bool Strict) { 1260 assert(isValidMsgOp(MsgId, OpId, STI, Strict)); 1261 1262 if (!Strict) 1263 return 0 <= StreamId && isUInt<STREAM_ID_WIDTH_>(StreamId); 1264 1265 switch(MsgId) 1266 { 1267 case ID_GS: 1268 return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_; 1269 case ID_GS_DONE: 1270 return (OpId == OP_GS_NOP)? 1271 (StreamId == STREAM_ID_NONE_) : 1272 (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_); 1273 default: 1274 return StreamId == STREAM_ID_NONE_; 1275 } 1276 } 1277 1278 bool msgRequiresOp(int64_t MsgId) { 1279 return MsgId == ID_GS || MsgId == ID_GS_DONE || MsgId == ID_SYSMSG; 1280 } 1281 1282 bool msgSupportsStream(int64_t MsgId, int64_t OpId) { 1283 return (MsgId == ID_GS || MsgId == ID_GS_DONE) && OpId != OP_GS_NOP; 1284 } 1285 1286 void decodeMsg(unsigned Val, 1287 uint16_t &MsgId, 1288 uint16_t &OpId, 1289 uint16_t &StreamId) { 1290 MsgId = Val & ID_MASK_; 1291 OpId = (Val & OP_MASK_) >> OP_SHIFT_; 1292 StreamId = (Val & STREAM_ID_MASK_) >> STREAM_ID_SHIFT_; 1293 } 1294 1295 uint64_t encodeMsg(uint64_t MsgId, 1296 uint64_t OpId, 1297 uint64_t StreamId) { 1298 return (MsgId << ID_SHIFT_) | 1299 (OpId << OP_SHIFT_) | 1300 (StreamId << STREAM_ID_SHIFT_); 1301 } 1302 1303 } // namespace SendMsg 1304 1305 //===----------------------------------------------------------------------===// 1306 // 1307 //===----------------------------------------------------------------------===// 1308 1309 unsigned getInitialPSInputAddr(const Function &F) { 1310 return getIntegerAttribute(F, "InitialPSInputAddr", 0); 1311 } 1312 1313 bool isShader(CallingConv::ID cc) { 1314 switch(cc) { 1315 case CallingConv::AMDGPU_VS: 1316 case CallingConv::AMDGPU_LS: 1317 case CallingConv::AMDGPU_HS: 1318 case CallingConv::AMDGPU_ES: 1319 case CallingConv::AMDGPU_GS: 1320 case CallingConv::AMDGPU_PS: 1321 case CallingConv::AMDGPU_CS: 1322 return true; 1323 default: 1324 return false; 1325 } 1326 } 1327 1328 bool isGraphics(CallingConv::ID cc) { 1329 return isShader(cc) || cc == CallingConv::AMDGPU_Gfx; 1330 } 1331 1332 bool isCompute(CallingConv::ID cc) { 1333 return !isGraphics(cc) || cc == CallingConv::AMDGPU_CS; 1334 } 1335 1336 bool isEntryFunctionCC(CallingConv::ID CC) { 1337 switch (CC) { 1338 case CallingConv::AMDGPU_KERNEL: 1339 case CallingConv::SPIR_KERNEL: 1340 case CallingConv::AMDGPU_VS: 1341 case CallingConv::AMDGPU_GS: 1342 case CallingConv::AMDGPU_PS: 1343 case CallingConv::AMDGPU_CS: 1344 case CallingConv::AMDGPU_ES: 1345 case CallingConv::AMDGPU_HS: 1346 case CallingConv::AMDGPU_LS: 1347 return true; 1348 default: 1349 return false; 1350 } 1351 } 1352 1353 bool isModuleEntryFunctionCC(CallingConv::ID CC) { 1354 switch (CC) { 1355 case CallingConv::AMDGPU_Gfx: 1356 return true; 1357 default: 1358 return isEntryFunctionCC(CC); 1359 } 1360 } 1361 1362 bool hasXNACK(const MCSubtargetInfo &STI) { 1363 return STI.getFeatureBits()[AMDGPU::FeatureXNACK]; 1364 } 1365 1366 bool hasSRAMECC(const MCSubtargetInfo &STI) { 1367 return STI.getFeatureBits()[AMDGPU::FeatureSRAMECC]; 1368 } 1369 1370 bool hasMIMG_R128(const MCSubtargetInfo &STI) { 1371 return STI.getFeatureBits()[AMDGPU::FeatureMIMG_R128] && !STI.getFeatureBits()[AMDGPU::FeatureR128A16]; 1372 } 1373 1374 bool hasGFX10A16(const MCSubtargetInfo &STI) { 1375 return STI.getFeatureBits()[AMDGPU::FeatureGFX10A16]; 1376 } 1377 1378 bool hasG16(const MCSubtargetInfo &STI) { 1379 return STI.getFeatureBits()[AMDGPU::FeatureG16]; 1380 } 1381 1382 bool hasPackedD16(const MCSubtargetInfo &STI) { 1383 return !STI.getFeatureBits()[AMDGPU::FeatureUnpackedD16VMem]; 1384 } 1385 1386 bool isSI(const MCSubtargetInfo &STI) { 1387 return STI.getFeatureBits()[AMDGPU::FeatureSouthernIslands]; 1388 } 1389 1390 bool isCI(const MCSubtargetInfo &STI) { 1391 return STI.getFeatureBits()[AMDGPU::FeatureSeaIslands]; 1392 } 1393 1394 bool isVI(const MCSubtargetInfo &STI) { 1395 return STI.getFeatureBits()[AMDGPU::FeatureVolcanicIslands]; 1396 } 1397 1398 bool isGFX9(const MCSubtargetInfo &STI) { 1399 return STI.getFeatureBits()[AMDGPU::FeatureGFX9]; 1400 } 1401 1402 bool isGFX9Plus(const MCSubtargetInfo &STI) { 1403 return isGFX9(STI) || isGFX10Plus(STI); 1404 } 1405 1406 bool isGFX10(const MCSubtargetInfo &STI) { 1407 return STI.getFeatureBits()[AMDGPU::FeatureGFX10]; 1408 } 1409 1410 bool isGFX10Plus(const MCSubtargetInfo &STI) { return isGFX10(STI); } 1411 1412 bool isGCN3Encoding(const MCSubtargetInfo &STI) { 1413 return STI.getFeatureBits()[AMDGPU::FeatureGCN3Encoding]; 1414 } 1415 1416 bool isGFX10_BEncoding(const MCSubtargetInfo &STI) { 1417 return STI.getFeatureBits()[AMDGPU::FeatureGFX10_BEncoding]; 1418 } 1419 1420 bool hasGFX10_3Insts(const MCSubtargetInfo &STI) { 1421 return STI.getFeatureBits()[AMDGPU::FeatureGFX10_3Insts]; 1422 } 1423 1424 bool isGFX90A(const MCSubtargetInfo &STI) { 1425 return STI.getFeatureBits()[AMDGPU::FeatureGFX90AInsts]; 1426 } 1427 1428 bool isSGPR(unsigned Reg, const MCRegisterInfo* TRI) { 1429 const MCRegisterClass SGPRClass = TRI->getRegClass(AMDGPU::SReg_32RegClassID); 1430 const unsigned FirstSubReg = TRI->getSubReg(Reg, AMDGPU::sub0); 1431 return SGPRClass.contains(FirstSubReg != 0 ? FirstSubReg : Reg) || 1432 Reg == AMDGPU::SCC; 1433 } 1434 1435 bool isRegIntersect(unsigned Reg0, unsigned Reg1, const MCRegisterInfo* TRI) { 1436 for (MCRegAliasIterator R(Reg0, TRI, true); R.isValid(); ++R) { 1437 if (*R == Reg1) return true; 1438 } 1439 return false; 1440 } 1441 1442 #define MAP_REG2REG \ 1443 using namespace AMDGPU; \ 1444 switch(Reg) { \ 1445 default: return Reg; \ 1446 CASE_CI_VI(FLAT_SCR) \ 1447 CASE_CI_VI(FLAT_SCR_LO) \ 1448 CASE_CI_VI(FLAT_SCR_HI) \ 1449 CASE_VI_GFX9PLUS(TTMP0) \ 1450 CASE_VI_GFX9PLUS(TTMP1) \ 1451 CASE_VI_GFX9PLUS(TTMP2) \ 1452 CASE_VI_GFX9PLUS(TTMP3) \ 1453 CASE_VI_GFX9PLUS(TTMP4) \ 1454 CASE_VI_GFX9PLUS(TTMP5) \ 1455 CASE_VI_GFX9PLUS(TTMP6) \ 1456 CASE_VI_GFX9PLUS(TTMP7) \ 1457 CASE_VI_GFX9PLUS(TTMP8) \ 1458 CASE_VI_GFX9PLUS(TTMP9) \ 1459 CASE_VI_GFX9PLUS(TTMP10) \ 1460 CASE_VI_GFX9PLUS(TTMP11) \ 1461 CASE_VI_GFX9PLUS(TTMP12) \ 1462 CASE_VI_GFX9PLUS(TTMP13) \ 1463 CASE_VI_GFX9PLUS(TTMP14) \ 1464 CASE_VI_GFX9PLUS(TTMP15) \ 1465 CASE_VI_GFX9PLUS(TTMP0_TTMP1) \ 1466 CASE_VI_GFX9PLUS(TTMP2_TTMP3) \ 1467 CASE_VI_GFX9PLUS(TTMP4_TTMP5) \ 1468 CASE_VI_GFX9PLUS(TTMP6_TTMP7) \ 1469 CASE_VI_GFX9PLUS(TTMP8_TTMP9) \ 1470 CASE_VI_GFX9PLUS(TTMP10_TTMP11) \ 1471 CASE_VI_GFX9PLUS(TTMP12_TTMP13) \ 1472 CASE_VI_GFX9PLUS(TTMP14_TTMP15) \ 1473 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3) \ 1474 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7) \ 1475 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11) \ 1476 CASE_VI_GFX9PLUS(TTMP12_TTMP13_TTMP14_TTMP15) \ 1477 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7) \ 1478 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11) \ 1479 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \ 1480 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \ 1481 } 1482 1483 #define CASE_CI_VI(node) \ 1484 assert(!isSI(STI)); \ 1485 case node: return isCI(STI) ? node##_ci : node##_vi; 1486 1487 #define CASE_VI_GFX9PLUS(node) \ 1488 case node: return isGFX9Plus(STI) ? node##_gfx9plus : node##_vi; 1489 1490 unsigned getMCReg(unsigned Reg, const MCSubtargetInfo &STI) { 1491 if (STI.getTargetTriple().getArch() == Triple::r600) 1492 return Reg; 1493 MAP_REG2REG 1494 } 1495 1496 #undef CASE_CI_VI 1497 #undef CASE_VI_GFX9PLUS 1498 1499 #define CASE_CI_VI(node) case node##_ci: case node##_vi: return node; 1500 #define CASE_VI_GFX9PLUS(node) case node##_vi: case node##_gfx9plus: return node; 1501 1502 unsigned mc2PseudoReg(unsigned Reg) { 1503 MAP_REG2REG 1504 } 1505 1506 #undef CASE_CI_VI 1507 #undef CASE_VI_GFX9PLUS 1508 #undef MAP_REG2REG 1509 1510 bool isSISrcOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1511 assert(OpNo < Desc.NumOperands); 1512 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1513 return OpType >= AMDGPU::OPERAND_SRC_FIRST && 1514 OpType <= AMDGPU::OPERAND_SRC_LAST; 1515 } 1516 1517 bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1518 assert(OpNo < Desc.NumOperands); 1519 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1520 switch (OpType) { 1521 case AMDGPU::OPERAND_REG_IMM_FP32: 1522 case AMDGPU::OPERAND_REG_IMM_FP64: 1523 case AMDGPU::OPERAND_REG_IMM_FP16: 1524 case AMDGPU::OPERAND_REG_IMM_V2FP16: 1525 case AMDGPU::OPERAND_REG_IMM_V2INT16: 1526 case AMDGPU::OPERAND_REG_INLINE_C_FP32: 1527 case AMDGPU::OPERAND_REG_INLINE_C_FP64: 1528 case AMDGPU::OPERAND_REG_INLINE_C_FP16: 1529 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: 1530 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: 1531 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: 1532 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: 1533 case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: 1534 case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16: 1535 case AMDGPU::OPERAND_REG_IMM_V2FP32: 1536 case AMDGPU::OPERAND_REG_INLINE_C_V2FP32: 1537 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: 1538 return true; 1539 default: 1540 return false; 1541 } 1542 } 1543 1544 bool isSISrcInlinableOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1545 assert(OpNo < Desc.NumOperands); 1546 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1547 return OpType >= AMDGPU::OPERAND_REG_INLINE_C_FIRST && 1548 OpType <= AMDGPU::OPERAND_REG_INLINE_C_LAST; 1549 } 1550 1551 // Avoid using MCRegisterClass::getSize, since that function will go away 1552 // (move from MC* level to Target* level). Return size in bits. 1553 unsigned getRegBitWidth(unsigned RCID) { 1554 switch (RCID) { 1555 case AMDGPU::VGPR_LO16RegClassID: 1556 case AMDGPU::VGPR_HI16RegClassID: 1557 case AMDGPU::SGPR_LO16RegClassID: 1558 case AMDGPU::AGPR_LO16RegClassID: 1559 return 16; 1560 case AMDGPU::SGPR_32RegClassID: 1561 case AMDGPU::VGPR_32RegClassID: 1562 case AMDGPU::VRegOrLds_32RegClassID: 1563 case AMDGPU::AGPR_32RegClassID: 1564 case AMDGPU::VS_32RegClassID: 1565 case AMDGPU::AV_32RegClassID: 1566 case AMDGPU::SReg_32RegClassID: 1567 case AMDGPU::SReg_32_XM0RegClassID: 1568 case AMDGPU::SRegOrLds_32RegClassID: 1569 return 32; 1570 case AMDGPU::SGPR_64RegClassID: 1571 case AMDGPU::VS_64RegClassID: 1572 case AMDGPU::AV_64RegClassID: 1573 case AMDGPU::SReg_64RegClassID: 1574 case AMDGPU::VReg_64RegClassID: 1575 case AMDGPU::AReg_64RegClassID: 1576 case AMDGPU::SReg_64_XEXECRegClassID: 1577 case AMDGPU::VReg_64_Align2RegClassID: 1578 case AMDGPU::AReg_64_Align2RegClassID: 1579 return 64; 1580 case AMDGPU::SGPR_96RegClassID: 1581 case AMDGPU::SReg_96RegClassID: 1582 case AMDGPU::VReg_96RegClassID: 1583 case AMDGPU::AReg_96RegClassID: 1584 case AMDGPU::VReg_96_Align2RegClassID: 1585 case AMDGPU::AReg_96_Align2RegClassID: 1586 case AMDGPU::AV_96RegClassID: 1587 return 96; 1588 case AMDGPU::SGPR_128RegClassID: 1589 case AMDGPU::SReg_128RegClassID: 1590 case AMDGPU::VReg_128RegClassID: 1591 case AMDGPU::AReg_128RegClassID: 1592 case AMDGPU::VReg_128_Align2RegClassID: 1593 case AMDGPU::AReg_128_Align2RegClassID: 1594 case AMDGPU::AV_128RegClassID: 1595 return 128; 1596 case AMDGPU::SGPR_160RegClassID: 1597 case AMDGPU::SReg_160RegClassID: 1598 case AMDGPU::VReg_160RegClassID: 1599 case AMDGPU::AReg_160RegClassID: 1600 case AMDGPU::VReg_160_Align2RegClassID: 1601 case AMDGPU::AReg_160_Align2RegClassID: 1602 case AMDGPU::AV_160RegClassID: 1603 return 160; 1604 case AMDGPU::SGPR_192RegClassID: 1605 case AMDGPU::SReg_192RegClassID: 1606 case AMDGPU::VReg_192RegClassID: 1607 case AMDGPU::AReg_192RegClassID: 1608 case AMDGPU::VReg_192_Align2RegClassID: 1609 case AMDGPU::AReg_192_Align2RegClassID: 1610 return 192; 1611 case AMDGPU::SGPR_256RegClassID: 1612 case AMDGPU::SReg_256RegClassID: 1613 case AMDGPU::VReg_256RegClassID: 1614 case AMDGPU::AReg_256RegClassID: 1615 case AMDGPU::VReg_256_Align2RegClassID: 1616 case AMDGPU::AReg_256_Align2RegClassID: 1617 return 256; 1618 case AMDGPU::SGPR_512RegClassID: 1619 case AMDGPU::SReg_512RegClassID: 1620 case AMDGPU::VReg_512RegClassID: 1621 case AMDGPU::AReg_512RegClassID: 1622 case AMDGPU::VReg_512_Align2RegClassID: 1623 case AMDGPU::AReg_512_Align2RegClassID: 1624 return 512; 1625 case AMDGPU::SGPR_1024RegClassID: 1626 case AMDGPU::SReg_1024RegClassID: 1627 case AMDGPU::VReg_1024RegClassID: 1628 case AMDGPU::AReg_1024RegClassID: 1629 case AMDGPU::VReg_1024_Align2RegClassID: 1630 case AMDGPU::AReg_1024_Align2RegClassID: 1631 return 1024; 1632 default: 1633 llvm_unreachable("Unexpected register class"); 1634 } 1635 } 1636 1637 unsigned getRegBitWidth(const MCRegisterClass &RC) { 1638 return getRegBitWidth(RC.getID()); 1639 } 1640 1641 unsigned getRegOperandSize(const MCRegisterInfo *MRI, const MCInstrDesc &Desc, 1642 unsigned OpNo) { 1643 assert(OpNo < Desc.NumOperands); 1644 unsigned RCID = Desc.OpInfo[OpNo].RegClass; 1645 return getRegBitWidth(MRI->getRegClass(RCID)) / 8; 1646 } 1647 1648 bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi) { 1649 if (isInlinableIntLiteral(Literal)) 1650 return true; 1651 1652 uint64_t Val = static_cast<uint64_t>(Literal); 1653 return (Val == DoubleToBits(0.0)) || 1654 (Val == DoubleToBits(1.0)) || 1655 (Val == DoubleToBits(-1.0)) || 1656 (Val == DoubleToBits(0.5)) || 1657 (Val == DoubleToBits(-0.5)) || 1658 (Val == DoubleToBits(2.0)) || 1659 (Val == DoubleToBits(-2.0)) || 1660 (Val == DoubleToBits(4.0)) || 1661 (Val == DoubleToBits(-4.0)) || 1662 (Val == 0x3fc45f306dc9c882 && HasInv2Pi); 1663 } 1664 1665 bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi) { 1666 if (isInlinableIntLiteral(Literal)) 1667 return true; 1668 1669 // The actual type of the operand does not seem to matter as long 1670 // as the bits match one of the inline immediate values. For example: 1671 // 1672 // -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal, 1673 // so it is a legal inline immediate. 1674 // 1675 // 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in 1676 // floating-point, so it is a legal inline immediate. 1677 1678 uint32_t Val = static_cast<uint32_t>(Literal); 1679 return (Val == FloatToBits(0.0f)) || 1680 (Val == FloatToBits(1.0f)) || 1681 (Val == FloatToBits(-1.0f)) || 1682 (Val == FloatToBits(0.5f)) || 1683 (Val == FloatToBits(-0.5f)) || 1684 (Val == FloatToBits(2.0f)) || 1685 (Val == FloatToBits(-2.0f)) || 1686 (Val == FloatToBits(4.0f)) || 1687 (Val == FloatToBits(-4.0f)) || 1688 (Val == 0x3e22f983 && HasInv2Pi); 1689 } 1690 1691 bool isInlinableLiteral16(int16_t Literal, bool HasInv2Pi) { 1692 if (!HasInv2Pi) 1693 return false; 1694 1695 if (isInlinableIntLiteral(Literal)) 1696 return true; 1697 1698 uint16_t Val = static_cast<uint16_t>(Literal); 1699 return Val == 0x3C00 || // 1.0 1700 Val == 0xBC00 || // -1.0 1701 Val == 0x3800 || // 0.5 1702 Val == 0xB800 || // -0.5 1703 Val == 0x4000 || // 2.0 1704 Val == 0xC000 || // -2.0 1705 Val == 0x4400 || // 4.0 1706 Val == 0xC400 || // -4.0 1707 Val == 0x3118; // 1/2pi 1708 } 1709 1710 bool isInlinableLiteralV216(int32_t Literal, bool HasInv2Pi) { 1711 assert(HasInv2Pi); 1712 1713 if (isInt<16>(Literal) || isUInt<16>(Literal)) { 1714 int16_t Trunc = static_cast<int16_t>(Literal); 1715 return AMDGPU::isInlinableLiteral16(Trunc, HasInv2Pi); 1716 } 1717 if (!(Literal & 0xffff)) 1718 return AMDGPU::isInlinableLiteral16(Literal >> 16, HasInv2Pi); 1719 1720 int16_t Lo16 = static_cast<int16_t>(Literal); 1721 int16_t Hi16 = static_cast<int16_t>(Literal >> 16); 1722 return Lo16 == Hi16 && isInlinableLiteral16(Lo16, HasInv2Pi); 1723 } 1724 1725 bool isInlinableIntLiteralV216(int32_t Literal) { 1726 int16_t Lo16 = static_cast<int16_t>(Literal); 1727 if (isInt<16>(Literal) || isUInt<16>(Literal)) 1728 return isInlinableIntLiteral(Lo16); 1729 1730 int16_t Hi16 = static_cast<int16_t>(Literal >> 16); 1731 if (!(Literal & 0xffff)) 1732 return isInlinableIntLiteral(Hi16); 1733 return Lo16 == Hi16 && isInlinableIntLiteral(Lo16); 1734 } 1735 1736 bool isFoldableLiteralV216(int32_t Literal, bool HasInv2Pi) { 1737 assert(HasInv2Pi); 1738 1739 int16_t Lo16 = static_cast<int16_t>(Literal); 1740 if (isInt<16>(Literal) || isUInt<16>(Literal)) 1741 return true; 1742 1743 int16_t Hi16 = static_cast<int16_t>(Literal >> 16); 1744 if (!(Literal & 0xffff)) 1745 return true; 1746 return Lo16 == Hi16; 1747 } 1748 1749 bool isArgPassedInSGPR(const Argument *A) { 1750 const Function *F = A->getParent(); 1751 1752 // Arguments to compute shaders are never a source of divergence. 1753 CallingConv::ID CC = F->getCallingConv(); 1754 switch (CC) { 1755 case CallingConv::AMDGPU_KERNEL: 1756 case CallingConv::SPIR_KERNEL: 1757 return true; 1758 case CallingConv::AMDGPU_VS: 1759 case CallingConv::AMDGPU_LS: 1760 case CallingConv::AMDGPU_HS: 1761 case CallingConv::AMDGPU_ES: 1762 case CallingConv::AMDGPU_GS: 1763 case CallingConv::AMDGPU_PS: 1764 case CallingConv::AMDGPU_CS: 1765 case CallingConv::AMDGPU_Gfx: 1766 // For non-compute shaders, SGPR inputs are marked with either inreg or byval. 1767 // Everything else is in VGPRs. 1768 return F->getAttributes().hasParamAttribute(A->getArgNo(), Attribute::InReg) || 1769 F->getAttributes().hasParamAttribute(A->getArgNo(), Attribute::ByVal); 1770 default: 1771 // TODO: Should calls support inreg for SGPR inputs? 1772 return false; 1773 } 1774 } 1775 1776 static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) { 1777 return isGCN3Encoding(ST) || isGFX10Plus(ST); 1778 } 1779 1780 static bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST) { 1781 return isGFX9Plus(ST); 1782 } 1783 1784 bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST, 1785 int64_t EncodedOffset) { 1786 return hasSMEMByteOffset(ST) ? isUInt<20>(EncodedOffset) 1787 : isUInt<8>(EncodedOffset); 1788 } 1789 1790 bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST, 1791 int64_t EncodedOffset, 1792 bool IsBuffer) { 1793 return !IsBuffer && 1794 hasSMRDSignedImmOffset(ST) && 1795 isInt<21>(EncodedOffset); 1796 } 1797 1798 static bool isDwordAligned(uint64_t ByteOffset) { 1799 return (ByteOffset & 3) == 0; 1800 } 1801 1802 uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST, 1803 uint64_t ByteOffset) { 1804 if (hasSMEMByteOffset(ST)) 1805 return ByteOffset; 1806 1807 assert(isDwordAligned(ByteOffset)); 1808 return ByteOffset >> 2; 1809 } 1810 1811 Optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST, 1812 int64_t ByteOffset, bool IsBuffer) { 1813 // The signed version is always a byte offset. 1814 if (!IsBuffer && hasSMRDSignedImmOffset(ST)) { 1815 assert(hasSMEMByteOffset(ST)); 1816 return isInt<20>(ByteOffset) ? Optional<int64_t>(ByteOffset) : None; 1817 } 1818 1819 if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST)) 1820 return None; 1821 1822 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset); 1823 return isLegalSMRDEncodedUnsignedOffset(ST, EncodedOffset) 1824 ? Optional<int64_t>(EncodedOffset) 1825 : None; 1826 } 1827 1828 Optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST, 1829 int64_t ByteOffset) { 1830 if (!isCI(ST) || !isDwordAligned(ByteOffset)) 1831 return None; 1832 1833 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset); 1834 return isUInt<32>(EncodedOffset) ? Optional<int64_t>(EncodedOffset) : None; 1835 } 1836 1837 unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST, bool Signed) { 1838 // Address offset is 12-bit signed for GFX10, 13-bit for GFX9. 1839 if (AMDGPU::isGFX10(ST)) 1840 return Signed ? 12 : 11; 1841 1842 return Signed ? 13 : 12; 1843 } 1844 1845 // Given Imm, split it into the values to put into the SOffset and ImmOffset 1846 // fields in an MUBUF instruction. Return false if it is not possible (due to a 1847 // hardware bug needing a workaround). 1848 // 1849 // The required alignment ensures that individual address components remain 1850 // aligned if they are aligned to begin with. It also ensures that additional 1851 // offsets within the given alignment can be added to the resulting ImmOffset. 1852 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset, 1853 const GCNSubtarget *Subtarget, Align Alignment) { 1854 const uint32_t MaxImm = alignDown(4095, Alignment.value()); 1855 uint32_t Overflow = 0; 1856 1857 if (Imm > MaxImm) { 1858 if (Imm <= MaxImm + 64) { 1859 // Use an SOffset inline constant for 4..64 1860 Overflow = Imm - MaxImm; 1861 Imm = MaxImm; 1862 } else { 1863 // Try to keep the same value in SOffset for adjacent loads, so that 1864 // the corresponding register contents can be re-used. 1865 // 1866 // Load values with all low-bits (except for alignment bits) set into 1867 // SOffset, so that a larger range of values can be covered using 1868 // s_movk_i32. 1869 // 1870 // Atomic operations fail to work correctly when individual address 1871 // components are unaligned, even if their sum is aligned. 1872 uint32_t High = (Imm + Alignment.value()) & ~4095; 1873 uint32_t Low = (Imm + Alignment.value()) & 4095; 1874 Imm = Low; 1875 Overflow = High - Alignment.value(); 1876 } 1877 } 1878 1879 // There is a hardware bug in SI and CI which prevents address clamping in 1880 // MUBUF instructions from working correctly with SOffsets. The immediate 1881 // offset is unaffected. 1882 if (Overflow > 0 && 1883 Subtarget->getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS) 1884 return false; 1885 1886 ImmOffset = Imm; 1887 SOffset = Overflow; 1888 return true; 1889 } 1890 1891 SIModeRegisterDefaults::SIModeRegisterDefaults(const Function &F) { 1892 *this = getDefaultForCallingConv(F.getCallingConv()); 1893 1894 StringRef IEEEAttr = F.getFnAttribute("amdgpu-ieee").getValueAsString(); 1895 if (!IEEEAttr.empty()) 1896 IEEE = IEEEAttr == "true"; 1897 1898 StringRef DX10ClampAttr 1899 = F.getFnAttribute("amdgpu-dx10-clamp").getValueAsString(); 1900 if (!DX10ClampAttr.empty()) 1901 DX10Clamp = DX10ClampAttr == "true"; 1902 1903 StringRef DenormF32Attr = F.getFnAttribute("denormal-fp-math-f32").getValueAsString(); 1904 if (!DenormF32Attr.empty()) { 1905 DenormalMode DenormMode = parseDenormalFPAttribute(DenormF32Attr); 1906 FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE; 1907 FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE; 1908 } 1909 1910 StringRef DenormAttr = F.getFnAttribute("denormal-fp-math").getValueAsString(); 1911 if (!DenormAttr.empty()) { 1912 DenormalMode DenormMode = parseDenormalFPAttribute(DenormAttr); 1913 1914 if (DenormF32Attr.empty()) { 1915 FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE; 1916 FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE; 1917 } 1918 1919 FP64FP16InputDenormals = DenormMode.Input == DenormalMode::IEEE; 1920 FP64FP16OutputDenormals = DenormMode.Output == DenormalMode::IEEE; 1921 } 1922 } 1923 1924 namespace { 1925 1926 struct SourceOfDivergence { 1927 unsigned Intr; 1928 }; 1929 const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr); 1930 1931 #define GET_SourcesOfDivergence_IMPL 1932 #define GET_Gfx9BufferFormat_IMPL 1933 #define GET_Gfx10PlusBufferFormat_IMPL 1934 #include "AMDGPUGenSearchableTables.inc" 1935 1936 } // end anonymous namespace 1937 1938 bool isIntrinsicSourceOfDivergence(unsigned IntrID) { 1939 return lookupSourceOfDivergence(IntrID); 1940 } 1941 1942 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp, 1943 uint8_t NumComponents, 1944 uint8_t NumFormat, 1945 const MCSubtargetInfo &STI) { 1946 return isGFX10Plus(STI) 1947 ? getGfx10PlusBufferFormatInfo(BitsPerComp, NumComponents, 1948 NumFormat) 1949 : getGfx9BufferFormatInfo(BitsPerComp, NumComponents, NumFormat); 1950 } 1951 1952 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format, 1953 const MCSubtargetInfo &STI) { 1954 return isGFX10Plus(STI) ? getGfx10PlusBufferFormatInfo(Format) 1955 : getGfx9BufferFormatInfo(Format); 1956 } 1957 1958 } // namespace AMDGPU 1959 1960 raw_ostream &operator<<(raw_ostream &OS, 1961 const AMDGPU::IsaInfo::TargetIDSetting S) { 1962 switch (S) { 1963 case (AMDGPU::IsaInfo::TargetIDSetting::Unsupported): 1964 OS << "Unsupported"; 1965 break; 1966 case (AMDGPU::IsaInfo::TargetIDSetting::Any): 1967 OS << "Any"; 1968 break; 1969 case (AMDGPU::IsaInfo::TargetIDSetting::Off): 1970 OS << "Off"; 1971 break; 1972 case (AMDGPU::IsaInfo::TargetIDSetting::On): 1973 OS << "On"; 1974 break; 1975 } 1976 return OS; 1977 } 1978 1979 } // namespace llvm 1980