1 //===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "AMDGPUBaseInfo.h" 10 #include "AMDGPU.h" 11 #include "AMDGPUAsmUtils.h" 12 #include "AMDKernelCodeT.h" 13 #include "GCNSubtarget.h" 14 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 15 #include "llvm/BinaryFormat/ELF.h" 16 #include "llvm/IR/Attributes.h" 17 #include "llvm/IR/Function.h" 18 #include "llvm/IR/GlobalValue.h" 19 #include "llvm/IR/IntrinsicsAMDGPU.h" 20 #include "llvm/IR/IntrinsicsR600.h" 21 #include "llvm/IR/LLVMContext.h" 22 #include "llvm/MC/MCSubtargetInfo.h" 23 #include "llvm/Support/AMDHSAKernelDescriptor.h" 24 #include "llvm/Support/CommandLine.h" 25 #include "llvm/Support/TargetParser.h" 26 27 #define GET_INSTRINFO_NAMED_OPS 28 #define GET_INSTRMAP_INFO 29 #include "AMDGPUGenInstrInfo.inc" 30 31 static llvm::cl::opt<unsigned> AmdhsaCodeObjectVersion( 32 "amdhsa-code-object-version", llvm::cl::Hidden, 33 llvm::cl::desc("AMDHSA Code Object Version"), llvm::cl::init(4), 34 llvm::cl::ZeroOrMore); 35 36 namespace { 37 38 /// \returns Bit mask for given bit \p Shift and bit \p Width. 39 unsigned getBitMask(unsigned Shift, unsigned Width) { 40 return ((1 << Width) - 1) << Shift; 41 } 42 43 /// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width. 44 /// 45 /// \returns Packed \p Dst. 46 unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) { 47 Dst &= ~(1 << Shift) & ~getBitMask(Shift, Width); 48 Dst |= (Src << Shift) & getBitMask(Shift, Width); 49 return Dst; 50 } 51 52 /// Unpacks bits from \p Src for given bit \p Shift and bit \p Width. 53 /// 54 /// \returns Unpacked bits. 55 unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) { 56 return (Src & getBitMask(Shift, Width)) >> Shift; 57 } 58 59 /// \returns Vmcnt bit shift (lower bits). 60 unsigned getVmcntBitShiftLo() { return 0; } 61 62 /// \returns Vmcnt bit width (lower bits). 63 unsigned getVmcntBitWidthLo() { return 4; } 64 65 /// \returns Expcnt bit shift. 66 unsigned getExpcntBitShift() { return 4; } 67 68 /// \returns Expcnt bit width. 69 unsigned getExpcntBitWidth() { return 3; } 70 71 /// \returns Lgkmcnt bit shift. 72 unsigned getLgkmcntBitShift() { return 8; } 73 74 /// \returns Lgkmcnt bit width. 75 unsigned getLgkmcntBitWidth(unsigned VersionMajor) { 76 return (VersionMajor >= 10) ? 6 : 4; 77 } 78 79 /// \returns Vmcnt bit shift (higher bits). 80 unsigned getVmcntBitShiftHi() { return 14; } 81 82 /// \returns Vmcnt bit width (higher bits). 83 unsigned getVmcntBitWidthHi() { return 2; } 84 85 } // end namespace anonymous 86 87 namespace llvm { 88 89 namespace AMDGPU { 90 91 Optional<uint8_t> getHsaAbiVersion(const MCSubtargetInfo *STI) { 92 if (STI && STI->getTargetTriple().getOS() != Triple::AMDHSA) 93 return None; 94 95 switch (AmdhsaCodeObjectVersion) { 96 case 2: 97 return ELF::ELFABIVERSION_AMDGPU_HSA_V2; 98 case 3: 99 return ELF::ELFABIVERSION_AMDGPU_HSA_V3; 100 case 4: 101 return ELF::ELFABIVERSION_AMDGPU_HSA_V4; 102 default: 103 report_fatal_error(Twine("Unsupported AMDHSA Code Object Version ") + 104 Twine(AmdhsaCodeObjectVersion)); 105 } 106 } 107 108 bool isHsaAbiVersion2(const MCSubtargetInfo *STI) { 109 if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI)) 110 return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V2; 111 return false; 112 } 113 114 bool isHsaAbiVersion3(const MCSubtargetInfo *STI) { 115 if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI)) 116 return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V3; 117 return false; 118 } 119 120 bool isHsaAbiVersion4(const MCSubtargetInfo *STI) { 121 if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI)) 122 return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V4; 123 return false; 124 } 125 126 bool isHsaAbiVersion3Or4(const MCSubtargetInfo *STI) { 127 return isHsaAbiVersion3(STI) || isHsaAbiVersion4(STI); 128 } 129 130 #define GET_MIMGBaseOpcodesTable_IMPL 131 #define GET_MIMGDimInfoTable_IMPL 132 #define GET_MIMGInfoTable_IMPL 133 #define GET_MIMGLZMappingTable_IMPL 134 #define GET_MIMGMIPMappingTable_IMPL 135 #define GET_MIMGBiasMappingTable_IMPL 136 #define GET_MIMGG16MappingTable_IMPL 137 #include "AMDGPUGenSearchableTables.inc" 138 139 int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding, 140 unsigned VDataDwords, unsigned VAddrDwords) { 141 const MIMGInfo *Info = getMIMGOpcodeHelper(BaseOpcode, MIMGEncoding, 142 VDataDwords, VAddrDwords); 143 return Info ? Info->Opcode : -1; 144 } 145 146 const MIMGBaseOpcodeInfo *getMIMGBaseOpcode(unsigned Opc) { 147 const MIMGInfo *Info = getMIMGInfo(Opc); 148 return Info ? getMIMGBaseOpcodeInfo(Info->BaseOpcode) : nullptr; 149 } 150 151 int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) { 152 const MIMGInfo *OrigInfo = getMIMGInfo(Opc); 153 const MIMGInfo *NewInfo = 154 getMIMGOpcodeHelper(OrigInfo->BaseOpcode, OrigInfo->MIMGEncoding, 155 NewChannels, OrigInfo->VAddrDwords); 156 return NewInfo ? NewInfo->Opcode : -1; 157 } 158 159 unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode, 160 const MIMGDimInfo *Dim, bool IsA16, 161 bool IsG16Supported) { 162 unsigned AddrWords = BaseOpcode->NumExtraArgs; 163 unsigned AddrComponents = (BaseOpcode->Coordinates ? Dim->NumCoords : 0) + 164 (BaseOpcode->LodOrClampOrMip ? 1 : 0); 165 if (IsA16) 166 AddrWords += divideCeil(AddrComponents, 2); 167 else 168 AddrWords += AddrComponents; 169 170 // Note: For subtargets that support A16 but not G16, enabling A16 also 171 // enables 16 bit gradients. 172 // For subtargets that support A16 (operand) and G16 (done with a different 173 // instruction encoding), they are independent. 174 175 if (BaseOpcode->Gradients) { 176 if ((IsA16 && !IsG16Supported) || BaseOpcode->G16) 177 // There are two gradients per coordinate, we pack them separately. 178 // For the 3d case, 179 // we get (dy/du, dx/du) (-, dz/du) (dy/dv, dx/dv) (-, dz/dv) 180 AddrWords += alignTo<2>(Dim->NumGradients / 2); 181 else 182 AddrWords += Dim->NumGradients; 183 } 184 return AddrWords; 185 } 186 187 struct MUBUFInfo { 188 uint16_t Opcode; 189 uint16_t BaseOpcode; 190 uint8_t elements; 191 bool has_vaddr; 192 bool has_srsrc; 193 bool has_soffset; 194 bool IsBufferInv; 195 }; 196 197 struct MTBUFInfo { 198 uint16_t Opcode; 199 uint16_t BaseOpcode; 200 uint8_t elements; 201 bool has_vaddr; 202 bool has_srsrc; 203 bool has_soffset; 204 }; 205 206 struct SMInfo { 207 uint16_t Opcode; 208 bool IsBuffer; 209 }; 210 211 struct VOPInfo { 212 uint16_t Opcode; 213 bool IsSingle; 214 }; 215 216 #define GET_MTBUFInfoTable_DECL 217 #define GET_MTBUFInfoTable_IMPL 218 #define GET_MUBUFInfoTable_DECL 219 #define GET_MUBUFInfoTable_IMPL 220 #define GET_SMInfoTable_DECL 221 #define GET_SMInfoTable_IMPL 222 #define GET_VOP1InfoTable_DECL 223 #define GET_VOP1InfoTable_IMPL 224 #define GET_VOP2InfoTable_DECL 225 #define GET_VOP2InfoTable_IMPL 226 #define GET_VOP3InfoTable_DECL 227 #define GET_VOP3InfoTable_IMPL 228 #include "AMDGPUGenSearchableTables.inc" 229 230 int getMTBUFBaseOpcode(unsigned Opc) { 231 const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opc); 232 return Info ? Info->BaseOpcode : -1; 233 } 234 235 int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements) { 236 const MTBUFInfo *Info = getMTBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements); 237 return Info ? Info->Opcode : -1; 238 } 239 240 int getMTBUFElements(unsigned Opc) { 241 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 242 return Info ? Info->elements : 0; 243 } 244 245 bool getMTBUFHasVAddr(unsigned Opc) { 246 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 247 return Info ? Info->has_vaddr : false; 248 } 249 250 bool getMTBUFHasSrsrc(unsigned Opc) { 251 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 252 return Info ? Info->has_srsrc : false; 253 } 254 255 bool getMTBUFHasSoffset(unsigned Opc) { 256 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 257 return Info ? Info->has_soffset : false; 258 } 259 260 int getMUBUFBaseOpcode(unsigned Opc) { 261 const MUBUFInfo *Info = getMUBUFInfoFromOpcode(Opc); 262 return Info ? Info->BaseOpcode : -1; 263 } 264 265 int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements) { 266 const MUBUFInfo *Info = getMUBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements); 267 return Info ? Info->Opcode : -1; 268 } 269 270 int getMUBUFElements(unsigned Opc) { 271 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 272 return Info ? Info->elements : 0; 273 } 274 275 bool getMUBUFHasVAddr(unsigned Opc) { 276 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 277 return Info ? Info->has_vaddr : false; 278 } 279 280 bool getMUBUFHasSrsrc(unsigned Opc) { 281 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 282 return Info ? Info->has_srsrc : false; 283 } 284 285 bool getMUBUFHasSoffset(unsigned Opc) { 286 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 287 return Info ? Info->has_soffset : false; 288 } 289 290 bool getMUBUFIsBufferInv(unsigned Opc) { 291 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 292 return Info ? Info->IsBufferInv : false; 293 } 294 295 bool getSMEMIsBuffer(unsigned Opc) { 296 const SMInfo *Info = getSMEMOpcodeHelper(Opc); 297 return Info ? Info->IsBuffer : false; 298 } 299 300 bool getVOP1IsSingle(unsigned Opc) { 301 const VOPInfo *Info = getVOP1OpcodeHelper(Opc); 302 return Info ? Info->IsSingle : false; 303 } 304 305 bool getVOP2IsSingle(unsigned Opc) { 306 const VOPInfo *Info = getVOP2OpcodeHelper(Opc); 307 return Info ? Info->IsSingle : false; 308 } 309 310 bool getVOP3IsSingle(unsigned Opc) { 311 const VOPInfo *Info = getVOP3OpcodeHelper(Opc); 312 return Info ? Info->IsSingle : false; 313 } 314 315 // Wrapper for Tablegen'd function. enum Subtarget is not defined in any 316 // header files, so we need to wrap it in a function that takes unsigned 317 // instead. 318 int getMCOpcode(uint16_t Opcode, unsigned Gen) { 319 return getMCOpcodeGen(Opcode, static_cast<Subtarget>(Gen)); 320 } 321 322 namespace IsaInfo { 323 324 AMDGPUTargetID::AMDGPUTargetID(const MCSubtargetInfo &STI) 325 : STI(STI), XnackSetting(TargetIDSetting::Any), 326 SramEccSetting(TargetIDSetting::Any) { 327 if (!STI.getFeatureBits().test(FeatureSupportsXNACK)) 328 XnackSetting = TargetIDSetting::Unsupported; 329 if (!STI.getFeatureBits().test(FeatureSupportsSRAMECC)) 330 SramEccSetting = TargetIDSetting::Unsupported; 331 } 332 333 void AMDGPUTargetID::setTargetIDFromFeaturesString(StringRef FS) { 334 // Check if xnack or sramecc is explicitly enabled or disabled. In the 335 // absence of the target features we assume we must generate code that can run 336 // in any environment. 337 SubtargetFeatures Features(FS); 338 Optional<bool> XnackRequested; 339 Optional<bool> SramEccRequested; 340 341 for (const std::string &Feature : Features.getFeatures()) { 342 if (Feature == "+xnack") 343 XnackRequested = true; 344 else if (Feature == "-xnack") 345 XnackRequested = false; 346 else if (Feature == "+sramecc") 347 SramEccRequested = true; 348 else if (Feature == "-sramecc") 349 SramEccRequested = false; 350 } 351 352 bool XnackSupported = isXnackSupported(); 353 bool SramEccSupported = isSramEccSupported(); 354 355 if (XnackRequested) { 356 if (XnackSupported) { 357 XnackSetting = 358 *XnackRequested ? TargetIDSetting::On : TargetIDSetting::Off; 359 } else { 360 // If a specific xnack setting was requested and this GPU does not support 361 // xnack emit a warning. Setting will remain set to "Unsupported". 362 if (*XnackRequested) { 363 errs() << "warning: xnack 'On' was requested for a processor that does " 364 "not support it!\n"; 365 } else { 366 errs() << "warning: xnack 'Off' was requested for a processor that " 367 "does not support it!\n"; 368 } 369 } 370 } 371 372 if (SramEccRequested) { 373 if (SramEccSupported) { 374 SramEccSetting = 375 *SramEccRequested ? TargetIDSetting::On : TargetIDSetting::Off; 376 } else { 377 // If a specific sramecc setting was requested and this GPU does not 378 // support sramecc emit a warning. Setting will remain set to 379 // "Unsupported". 380 if (*SramEccRequested) { 381 errs() << "warning: sramecc 'On' was requested for a processor that " 382 "does not support it!\n"; 383 } else { 384 errs() << "warning: sramecc 'Off' was requested for a processor that " 385 "does not support it!\n"; 386 } 387 } 388 } 389 } 390 391 static TargetIDSetting 392 getTargetIDSettingFromFeatureString(StringRef FeatureString) { 393 if (FeatureString.endswith("-")) 394 return TargetIDSetting::Off; 395 if (FeatureString.endswith("+")) 396 return TargetIDSetting::On; 397 398 llvm_unreachable("Malformed feature string"); 399 } 400 401 void AMDGPUTargetID::setTargetIDFromTargetIDStream(StringRef TargetID) { 402 SmallVector<StringRef, 3> TargetIDSplit; 403 TargetID.split(TargetIDSplit, ':'); 404 405 for (const auto &FeatureString : TargetIDSplit) { 406 if (FeatureString.startswith("xnack")) 407 XnackSetting = getTargetIDSettingFromFeatureString(FeatureString); 408 if (FeatureString.startswith("sramecc")) 409 SramEccSetting = getTargetIDSettingFromFeatureString(FeatureString); 410 } 411 } 412 413 std::string AMDGPUTargetID::toString() const { 414 std::string StringRep; 415 raw_string_ostream StreamRep(StringRep); 416 417 auto TargetTriple = STI.getTargetTriple(); 418 auto Version = getIsaVersion(STI.getCPU()); 419 420 StreamRep << TargetTriple.getArchName() << '-' 421 << TargetTriple.getVendorName() << '-' 422 << TargetTriple.getOSName() << '-' 423 << TargetTriple.getEnvironmentName() << '-'; 424 425 std::string Processor; 426 // TODO: Following else statement is present here because we used various 427 // alias names for GPUs up until GFX9 (e.g. 'fiji' is same as 'gfx803'). 428 // Remove once all aliases are removed from GCNProcessors.td. 429 if (Version.Major >= 9) 430 Processor = STI.getCPU().str(); 431 else 432 Processor = (Twine("gfx") + Twine(Version.Major) + Twine(Version.Minor) + 433 Twine(Version.Stepping)) 434 .str(); 435 436 std::string Features; 437 if (Optional<uint8_t> HsaAbiVersion = getHsaAbiVersion(&STI)) { 438 switch (*HsaAbiVersion) { 439 case ELF::ELFABIVERSION_AMDGPU_HSA_V2: 440 // Code object V2 only supported specific processors and had fixed 441 // settings for the XNACK. 442 if (Processor == "gfx600") { 443 } else if (Processor == "gfx601") { 444 } else if (Processor == "gfx602") { 445 } else if (Processor == "gfx700") { 446 } else if (Processor == "gfx701") { 447 } else if (Processor == "gfx702") { 448 } else if (Processor == "gfx703") { 449 } else if (Processor == "gfx704") { 450 } else if (Processor == "gfx705") { 451 } else if (Processor == "gfx801") { 452 if (!isXnackOnOrAny()) 453 report_fatal_error( 454 "AMD GPU code object V2 does not support processor " + 455 Twine(Processor) + " without XNACK"); 456 } else if (Processor == "gfx802") { 457 } else if (Processor == "gfx803") { 458 } else if (Processor == "gfx805") { 459 } else if (Processor == "gfx810") { 460 if (!isXnackOnOrAny()) 461 report_fatal_error( 462 "AMD GPU code object V2 does not support processor " + 463 Twine(Processor) + " without XNACK"); 464 } else if (Processor == "gfx900") { 465 if (isXnackOnOrAny()) 466 Processor = "gfx901"; 467 } else if (Processor == "gfx902") { 468 if (isXnackOnOrAny()) 469 Processor = "gfx903"; 470 } else if (Processor == "gfx904") { 471 if (isXnackOnOrAny()) 472 Processor = "gfx905"; 473 } else if (Processor == "gfx906") { 474 if (isXnackOnOrAny()) 475 Processor = "gfx907"; 476 } else if (Processor == "gfx90c") { 477 if (isXnackOnOrAny()) 478 report_fatal_error( 479 "AMD GPU code object V2 does not support processor " + 480 Twine(Processor) + " with XNACK being ON or ANY"); 481 } else { 482 report_fatal_error( 483 "AMD GPU code object V2 does not support processor " + 484 Twine(Processor)); 485 } 486 break; 487 case ELF::ELFABIVERSION_AMDGPU_HSA_V3: 488 // xnack. 489 if (isXnackOnOrAny()) 490 Features += "+xnack"; 491 // In code object v2 and v3, "sramecc" feature was spelled with a 492 // hyphen ("sram-ecc"). 493 if (isSramEccOnOrAny()) 494 Features += "+sram-ecc"; 495 break; 496 case ELF::ELFABIVERSION_AMDGPU_HSA_V4: 497 // sramecc. 498 if (getSramEccSetting() == TargetIDSetting::Off) 499 Features += ":sramecc-"; 500 else if (getSramEccSetting() == TargetIDSetting::On) 501 Features += ":sramecc+"; 502 // xnack. 503 if (getXnackSetting() == TargetIDSetting::Off) 504 Features += ":xnack-"; 505 else if (getXnackSetting() == TargetIDSetting::On) 506 Features += ":xnack+"; 507 break; 508 default: 509 break; 510 } 511 } 512 513 StreamRep << Processor << Features; 514 515 StreamRep.flush(); 516 return StringRep; 517 } 518 519 unsigned getWavefrontSize(const MCSubtargetInfo *STI) { 520 if (STI->getFeatureBits().test(FeatureWavefrontSize16)) 521 return 16; 522 if (STI->getFeatureBits().test(FeatureWavefrontSize32)) 523 return 32; 524 525 return 64; 526 } 527 528 unsigned getLocalMemorySize(const MCSubtargetInfo *STI) { 529 if (STI->getFeatureBits().test(FeatureLocalMemorySize32768)) 530 return 32768; 531 if (STI->getFeatureBits().test(FeatureLocalMemorySize65536)) 532 return 65536; 533 534 return 0; 535 } 536 537 unsigned getEUsPerCU(const MCSubtargetInfo *STI) { 538 // "Per CU" really means "per whatever functional block the waves of a 539 // workgroup must share". For gfx10 in CU mode this is the CU, which contains 540 // two SIMDs. 541 if (isGFX10Plus(*STI) && STI->getFeatureBits().test(FeatureCuMode)) 542 return 2; 543 // Pre-gfx10 a CU contains four SIMDs. For gfx10 in WGP mode the WGP contains 544 // two CUs, so a total of four SIMDs. 545 return 4; 546 } 547 548 unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo *STI, 549 unsigned FlatWorkGroupSize) { 550 assert(FlatWorkGroupSize != 0); 551 if (STI->getTargetTriple().getArch() != Triple::amdgcn) 552 return 8; 553 unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize); 554 if (N == 1) 555 return 40; 556 N = 40 / N; 557 return std::min(N, 16u); 558 } 559 560 unsigned getMinWavesPerEU(const MCSubtargetInfo *STI) { 561 return 1; 562 } 563 564 unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI) { 565 // FIXME: Need to take scratch memory into account. 566 if (isGFX90A(*STI)) 567 return 8; 568 if (!isGFX10Plus(*STI)) 569 return 10; 570 return hasGFX10_3Insts(*STI) ? 16 : 20; 571 } 572 573 unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo *STI, 574 unsigned FlatWorkGroupSize) { 575 return divideCeil(getWavesPerWorkGroup(STI, FlatWorkGroupSize), 576 getEUsPerCU(STI)); 577 } 578 579 unsigned getMinFlatWorkGroupSize(const MCSubtargetInfo *STI) { 580 return 1; 581 } 582 583 unsigned getMaxFlatWorkGroupSize(const MCSubtargetInfo *STI) { 584 // Some subtargets allow encoding 2048, but this isn't tested or supported. 585 return 1024; 586 } 587 588 unsigned getWavesPerWorkGroup(const MCSubtargetInfo *STI, 589 unsigned FlatWorkGroupSize) { 590 return divideCeil(FlatWorkGroupSize, getWavefrontSize(STI)); 591 } 592 593 unsigned getSGPRAllocGranule(const MCSubtargetInfo *STI) { 594 IsaVersion Version = getIsaVersion(STI->getCPU()); 595 if (Version.Major >= 10) 596 return getAddressableNumSGPRs(STI); 597 if (Version.Major >= 8) 598 return 16; 599 return 8; 600 } 601 602 unsigned getSGPREncodingGranule(const MCSubtargetInfo *STI) { 603 return 8; 604 } 605 606 unsigned getTotalNumSGPRs(const MCSubtargetInfo *STI) { 607 IsaVersion Version = getIsaVersion(STI->getCPU()); 608 if (Version.Major >= 8) 609 return 800; 610 return 512; 611 } 612 613 unsigned getAddressableNumSGPRs(const MCSubtargetInfo *STI) { 614 if (STI->getFeatureBits().test(FeatureSGPRInitBug)) 615 return FIXED_NUM_SGPRS_FOR_INIT_BUG; 616 617 IsaVersion Version = getIsaVersion(STI->getCPU()); 618 if (Version.Major >= 10) 619 return 106; 620 if (Version.Major >= 8) 621 return 102; 622 return 104; 623 } 624 625 unsigned getMinNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 626 assert(WavesPerEU != 0); 627 628 IsaVersion Version = getIsaVersion(STI->getCPU()); 629 if (Version.Major >= 10) 630 return 0; 631 632 if (WavesPerEU >= getMaxWavesPerEU(STI)) 633 return 0; 634 635 unsigned MinNumSGPRs = getTotalNumSGPRs(STI) / (WavesPerEU + 1); 636 if (STI->getFeatureBits().test(FeatureTrapHandler)) 637 MinNumSGPRs -= std::min(MinNumSGPRs, (unsigned)TRAP_NUM_SGPRS); 638 MinNumSGPRs = alignDown(MinNumSGPRs, getSGPRAllocGranule(STI)) + 1; 639 return std::min(MinNumSGPRs, getAddressableNumSGPRs(STI)); 640 } 641 642 unsigned getMaxNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU, 643 bool Addressable) { 644 assert(WavesPerEU != 0); 645 646 unsigned AddressableNumSGPRs = getAddressableNumSGPRs(STI); 647 IsaVersion Version = getIsaVersion(STI->getCPU()); 648 if (Version.Major >= 10) 649 return Addressable ? AddressableNumSGPRs : 108; 650 if (Version.Major >= 8 && !Addressable) 651 AddressableNumSGPRs = 112; 652 unsigned MaxNumSGPRs = getTotalNumSGPRs(STI) / WavesPerEU; 653 if (STI->getFeatureBits().test(FeatureTrapHandler)) 654 MaxNumSGPRs -= std::min(MaxNumSGPRs, (unsigned)TRAP_NUM_SGPRS); 655 MaxNumSGPRs = alignDown(MaxNumSGPRs, getSGPRAllocGranule(STI)); 656 return std::min(MaxNumSGPRs, AddressableNumSGPRs); 657 } 658 659 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed, 660 bool FlatScrUsed, bool XNACKUsed) { 661 unsigned ExtraSGPRs = 0; 662 if (VCCUsed) 663 ExtraSGPRs = 2; 664 665 IsaVersion Version = getIsaVersion(STI->getCPU()); 666 if (Version.Major >= 10) 667 return ExtraSGPRs; 668 669 if (Version.Major < 8) { 670 if (FlatScrUsed) 671 ExtraSGPRs = 4; 672 } else { 673 if (XNACKUsed) 674 ExtraSGPRs = 4; 675 676 if (FlatScrUsed || 677 STI->getFeatureBits().test(AMDGPU::FeatureArchitectedFlatScratch)) 678 ExtraSGPRs = 6; 679 } 680 681 return ExtraSGPRs; 682 } 683 684 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed, 685 bool FlatScrUsed) { 686 return getNumExtraSGPRs(STI, VCCUsed, FlatScrUsed, 687 STI->getFeatureBits().test(AMDGPU::FeatureXNACK)); 688 } 689 690 unsigned getNumSGPRBlocks(const MCSubtargetInfo *STI, unsigned NumSGPRs) { 691 NumSGPRs = alignTo(std::max(1u, NumSGPRs), getSGPREncodingGranule(STI)); 692 // SGPRBlocks is actual number of SGPR blocks minus 1. 693 return NumSGPRs / getSGPREncodingGranule(STI) - 1; 694 } 695 696 unsigned getVGPRAllocGranule(const MCSubtargetInfo *STI, 697 Optional<bool> EnableWavefrontSize32) { 698 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 699 return 8; 700 701 bool IsWave32 = EnableWavefrontSize32 ? 702 *EnableWavefrontSize32 : 703 STI->getFeatureBits().test(FeatureWavefrontSize32); 704 705 if (hasGFX10_3Insts(*STI)) 706 return IsWave32 ? 16 : 8; 707 708 return IsWave32 ? 8 : 4; 709 } 710 711 unsigned getVGPREncodingGranule(const MCSubtargetInfo *STI, 712 Optional<bool> EnableWavefrontSize32) { 713 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 714 return 8; 715 716 bool IsWave32 = EnableWavefrontSize32 ? 717 *EnableWavefrontSize32 : 718 STI->getFeatureBits().test(FeatureWavefrontSize32); 719 720 return IsWave32 ? 8 : 4; 721 } 722 723 unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) { 724 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 725 return 512; 726 if (!isGFX10Plus(*STI)) 727 return 256; 728 return STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1024 : 512; 729 } 730 731 unsigned getAddressableNumVGPRs(const MCSubtargetInfo *STI) { 732 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 733 return 512; 734 return 256; 735 } 736 737 unsigned getMinNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 738 assert(WavesPerEU != 0); 739 740 if (WavesPerEU >= getMaxWavesPerEU(STI)) 741 return 0; 742 unsigned MinNumVGPRs = 743 alignDown(getTotalNumVGPRs(STI) / (WavesPerEU + 1), 744 getVGPRAllocGranule(STI)) + 1; 745 return std::min(MinNumVGPRs, getAddressableNumVGPRs(STI)); 746 } 747 748 unsigned getMaxNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 749 assert(WavesPerEU != 0); 750 751 unsigned MaxNumVGPRs = alignDown(getTotalNumVGPRs(STI) / WavesPerEU, 752 getVGPRAllocGranule(STI)); 753 unsigned AddressableNumVGPRs = getAddressableNumVGPRs(STI); 754 return std::min(MaxNumVGPRs, AddressableNumVGPRs); 755 } 756 757 unsigned getNumVGPRBlocks(const MCSubtargetInfo *STI, unsigned NumVGPRs, 758 Optional<bool> EnableWavefrontSize32) { 759 NumVGPRs = alignTo(std::max(1u, NumVGPRs), 760 getVGPREncodingGranule(STI, EnableWavefrontSize32)); 761 // VGPRBlocks is actual number of VGPR blocks minus 1. 762 return NumVGPRs / getVGPREncodingGranule(STI, EnableWavefrontSize32) - 1; 763 } 764 765 } // end namespace IsaInfo 766 767 void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header, 768 const MCSubtargetInfo *STI) { 769 IsaVersion Version = getIsaVersion(STI->getCPU()); 770 771 memset(&Header, 0, sizeof(Header)); 772 773 Header.amd_kernel_code_version_major = 1; 774 Header.amd_kernel_code_version_minor = 2; 775 Header.amd_machine_kind = 1; // AMD_MACHINE_KIND_AMDGPU 776 Header.amd_machine_version_major = Version.Major; 777 Header.amd_machine_version_minor = Version.Minor; 778 Header.amd_machine_version_stepping = Version.Stepping; 779 Header.kernel_code_entry_byte_offset = sizeof(Header); 780 Header.wavefront_size = 6; 781 782 // If the code object does not support indirect functions, then the value must 783 // be 0xffffffff. 784 Header.call_convention = -1; 785 786 // These alignment values are specified in powers of two, so alignment = 787 // 2^n. The minimum alignment is 2^4 = 16. 788 Header.kernarg_segment_alignment = 4; 789 Header.group_segment_alignment = 4; 790 Header.private_segment_alignment = 4; 791 792 if (Version.Major >= 10) { 793 if (STI->getFeatureBits().test(FeatureWavefrontSize32)) { 794 Header.wavefront_size = 5; 795 Header.code_properties |= AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32; 796 } 797 Header.compute_pgm_resource_registers |= 798 S_00B848_WGP_MODE(STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1) | 799 S_00B848_MEM_ORDERED(1); 800 } 801 } 802 803 amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor( 804 const MCSubtargetInfo *STI) { 805 IsaVersion Version = getIsaVersion(STI->getCPU()); 806 807 amdhsa::kernel_descriptor_t KD; 808 memset(&KD, 0, sizeof(KD)); 809 810 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 811 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, 812 amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE); 813 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 814 amdhsa::COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP, 1); 815 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 816 amdhsa::COMPUTE_PGM_RSRC1_ENABLE_IEEE_MODE, 1); 817 AMDHSA_BITS_SET(KD.compute_pgm_rsrc2, 818 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, 1); 819 if (Version.Major >= 10) { 820 AMDHSA_BITS_SET(KD.kernel_code_properties, 821 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, 822 STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1 : 0); 823 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 824 amdhsa::COMPUTE_PGM_RSRC1_WGP_MODE, 825 STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1); 826 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 827 amdhsa::COMPUTE_PGM_RSRC1_MEM_ORDERED, 1); 828 } 829 if (AMDGPU::isGFX90A(*STI)) { 830 AMDHSA_BITS_SET(KD.compute_pgm_rsrc3, 831 amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, 832 STI->getFeatureBits().test(FeatureTgSplit) ? 1 : 0); 833 } 834 return KD; 835 } 836 837 bool isGroupSegment(const GlobalValue *GV) { 838 return GV->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS; 839 } 840 841 bool isGlobalSegment(const GlobalValue *GV) { 842 return GV->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS; 843 } 844 845 bool isReadOnlySegment(const GlobalValue *GV) { 846 unsigned AS = GV->getAddressSpace(); 847 return AS == AMDGPUAS::CONSTANT_ADDRESS || 848 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT; 849 } 850 851 bool shouldEmitConstantsToTextSection(const Triple &TT) { 852 return TT.getArch() == Triple::r600; 853 } 854 855 int getIntegerAttribute(const Function &F, StringRef Name, int Default) { 856 Attribute A = F.getFnAttribute(Name); 857 int Result = Default; 858 859 if (A.isStringAttribute()) { 860 StringRef Str = A.getValueAsString(); 861 if (Str.getAsInteger(0, Result)) { 862 LLVMContext &Ctx = F.getContext(); 863 Ctx.emitError("can't parse integer attribute " + Name); 864 } 865 } 866 867 return Result; 868 } 869 870 std::pair<int, int> getIntegerPairAttribute(const Function &F, 871 StringRef Name, 872 std::pair<int, int> Default, 873 bool OnlyFirstRequired) { 874 Attribute A = F.getFnAttribute(Name); 875 if (!A.isStringAttribute()) 876 return Default; 877 878 LLVMContext &Ctx = F.getContext(); 879 std::pair<int, int> Ints = Default; 880 std::pair<StringRef, StringRef> Strs = A.getValueAsString().split(','); 881 if (Strs.first.trim().getAsInteger(0, Ints.first)) { 882 Ctx.emitError("can't parse first integer attribute " + Name); 883 return Default; 884 } 885 if (Strs.second.trim().getAsInteger(0, Ints.second)) { 886 if (!OnlyFirstRequired || !Strs.second.trim().empty()) { 887 Ctx.emitError("can't parse second integer attribute " + Name); 888 return Default; 889 } 890 } 891 892 return Ints; 893 } 894 895 unsigned getVmcntBitMask(const IsaVersion &Version) { 896 unsigned VmcntLo = (1 << getVmcntBitWidthLo()) - 1; 897 if (Version.Major < 9) 898 return VmcntLo; 899 900 unsigned VmcntHi = ((1 << getVmcntBitWidthHi()) - 1) << getVmcntBitWidthLo(); 901 return VmcntLo | VmcntHi; 902 } 903 904 unsigned getExpcntBitMask(const IsaVersion &Version) { 905 return (1 << getExpcntBitWidth()) - 1; 906 } 907 908 unsigned getLgkmcntBitMask(const IsaVersion &Version) { 909 return (1 << getLgkmcntBitWidth(Version.Major)) - 1; 910 } 911 912 unsigned getWaitcntBitMask(const IsaVersion &Version) { 913 unsigned VmcntLo = getBitMask(getVmcntBitShiftLo(), getVmcntBitWidthLo()); 914 unsigned Expcnt = getBitMask(getExpcntBitShift(), getExpcntBitWidth()); 915 unsigned Lgkmcnt = getBitMask(getLgkmcntBitShift(), 916 getLgkmcntBitWidth(Version.Major)); 917 unsigned Waitcnt = VmcntLo | Expcnt | Lgkmcnt; 918 if (Version.Major < 9) 919 return Waitcnt; 920 921 unsigned VmcntHi = getBitMask(getVmcntBitShiftHi(), getVmcntBitWidthHi()); 922 return Waitcnt | VmcntHi; 923 } 924 925 unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) { 926 unsigned VmcntLo = 927 unpackBits(Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo()); 928 if (Version.Major < 9) 929 return VmcntLo; 930 931 unsigned VmcntHi = 932 unpackBits(Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi()); 933 VmcntHi <<= getVmcntBitWidthLo(); 934 return VmcntLo | VmcntHi; 935 } 936 937 unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) { 938 return unpackBits(Waitcnt, getExpcntBitShift(), getExpcntBitWidth()); 939 } 940 941 unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) { 942 return unpackBits(Waitcnt, getLgkmcntBitShift(), 943 getLgkmcntBitWidth(Version.Major)); 944 } 945 946 void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, 947 unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt) { 948 Vmcnt = decodeVmcnt(Version, Waitcnt); 949 Expcnt = decodeExpcnt(Version, Waitcnt); 950 Lgkmcnt = decodeLgkmcnt(Version, Waitcnt); 951 } 952 953 Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded) { 954 Waitcnt Decoded; 955 Decoded.VmCnt = decodeVmcnt(Version, Encoded); 956 Decoded.ExpCnt = decodeExpcnt(Version, Encoded); 957 Decoded.LgkmCnt = decodeLgkmcnt(Version, Encoded); 958 return Decoded; 959 } 960 961 unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt, 962 unsigned Vmcnt) { 963 Waitcnt = 964 packBits(Vmcnt, Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo()); 965 if (Version.Major < 9) 966 return Waitcnt; 967 968 Vmcnt >>= getVmcntBitWidthLo(); 969 return packBits(Vmcnt, Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi()); 970 } 971 972 unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt, 973 unsigned Expcnt) { 974 return packBits(Expcnt, Waitcnt, getExpcntBitShift(), getExpcntBitWidth()); 975 } 976 977 unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt, 978 unsigned Lgkmcnt) { 979 return packBits(Lgkmcnt, Waitcnt, getLgkmcntBitShift(), 980 getLgkmcntBitWidth(Version.Major)); 981 } 982 983 unsigned encodeWaitcnt(const IsaVersion &Version, 984 unsigned Vmcnt, unsigned Expcnt, unsigned Lgkmcnt) { 985 unsigned Waitcnt = getWaitcntBitMask(Version); 986 Waitcnt = encodeVmcnt(Version, Waitcnt, Vmcnt); 987 Waitcnt = encodeExpcnt(Version, Waitcnt, Expcnt); 988 Waitcnt = encodeLgkmcnt(Version, Waitcnt, Lgkmcnt); 989 return Waitcnt; 990 } 991 992 unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded) { 993 return encodeWaitcnt(Version, Decoded.VmCnt, Decoded.ExpCnt, Decoded.LgkmCnt); 994 } 995 996 //===----------------------------------------------------------------------===// 997 // hwreg 998 //===----------------------------------------------------------------------===// 999 1000 namespace Hwreg { 1001 1002 int64_t getHwregId(const StringRef Name) { 1003 for (int Id = ID_SYMBOLIC_FIRST_; Id < ID_SYMBOLIC_LAST_; ++Id) { 1004 if (IdSymbolic[Id] && Name == IdSymbolic[Id]) 1005 return Id; 1006 } 1007 return ID_UNKNOWN_; 1008 } 1009 1010 static unsigned getLastSymbolicHwreg(const MCSubtargetInfo &STI) { 1011 if (isSI(STI) || isCI(STI) || isVI(STI)) 1012 return ID_SYMBOLIC_FIRST_GFX9_; 1013 else if (isGFX9(STI)) 1014 return ID_SYMBOLIC_FIRST_GFX10_; 1015 else if (isGFX10(STI) && !isGFX10_BEncoding(STI)) 1016 return ID_SYMBOLIC_FIRST_GFX1030_; 1017 else 1018 return ID_SYMBOLIC_LAST_; 1019 } 1020 1021 bool isValidHwreg(int64_t Id, const MCSubtargetInfo &STI) { 1022 switch (Id) { 1023 case ID_HW_ID: 1024 return isSI(STI) || isCI(STI) || isVI(STI) || isGFX9(STI); 1025 case ID_HW_ID1: 1026 case ID_HW_ID2: 1027 return isGFX10Plus(STI); 1028 case ID_XNACK_MASK: 1029 return isGFX10(STI) && !AMDGPU::isGFX10_BEncoding(STI); 1030 default: 1031 return ID_SYMBOLIC_FIRST_ <= Id && Id < getLastSymbolicHwreg(STI) && 1032 IdSymbolic[Id]; 1033 } 1034 } 1035 1036 bool isValidHwreg(int64_t Id) { 1037 return 0 <= Id && isUInt<ID_WIDTH_>(Id); 1038 } 1039 1040 bool isValidHwregOffset(int64_t Offset) { 1041 return 0 <= Offset && isUInt<OFFSET_WIDTH_>(Offset); 1042 } 1043 1044 bool isValidHwregWidth(int64_t Width) { 1045 return 0 <= (Width - 1) && isUInt<WIDTH_M1_WIDTH_>(Width - 1); 1046 } 1047 1048 uint64_t encodeHwreg(uint64_t Id, uint64_t Offset, uint64_t Width) { 1049 return (Id << ID_SHIFT_) | 1050 (Offset << OFFSET_SHIFT_) | 1051 ((Width - 1) << WIDTH_M1_SHIFT_); 1052 } 1053 1054 StringRef getHwreg(unsigned Id, const MCSubtargetInfo &STI) { 1055 return isValidHwreg(Id, STI) ? IdSymbolic[Id] : ""; 1056 } 1057 1058 void decodeHwreg(unsigned Val, unsigned &Id, unsigned &Offset, unsigned &Width) { 1059 Id = (Val & ID_MASK_) >> ID_SHIFT_; 1060 Offset = (Val & OFFSET_MASK_) >> OFFSET_SHIFT_; 1061 Width = ((Val & WIDTH_M1_MASK_) >> WIDTH_M1_SHIFT_) + 1; 1062 } 1063 1064 } // namespace Hwreg 1065 1066 //===----------------------------------------------------------------------===// 1067 // exp tgt 1068 //===----------------------------------------------------------------------===// 1069 1070 namespace Exp { 1071 1072 struct ExpTgt { 1073 StringLiteral Name; 1074 unsigned Tgt; 1075 unsigned MaxIndex; 1076 }; 1077 1078 static constexpr ExpTgt ExpTgtInfo[] = { 1079 {{"null"}, ET_NULL, ET_NULL_MAX_IDX}, 1080 {{"mrtz"}, ET_MRTZ, ET_MRTZ_MAX_IDX}, 1081 {{"prim"}, ET_PRIM, ET_PRIM_MAX_IDX}, 1082 {{"mrt"}, ET_MRT0, ET_MRT_MAX_IDX}, 1083 {{"pos"}, ET_POS0, ET_POS_MAX_IDX}, 1084 {{"param"}, ET_PARAM0, ET_PARAM_MAX_IDX}, 1085 }; 1086 1087 bool getTgtName(unsigned Id, StringRef &Name, int &Index) { 1088 for (const ExpTgt &Val : ExpTgtInfo) { 1089 if (Val.Tgt <= Id && Id <= Val.Tgt + Val.MaxIndex) { 1090 Index = (Val.MaxIndex == 0) ? -1 : (Id - Val.Tgt); 1091 Name = Val.Name; 1092 return true; 1093 } 1094 } 1095 return false; 1096 } 1097 1098 unsigned getTgtId(const StringRef Name) { 1099 1100 for (const ExpTgt &Val : ExpTgtInfo) { 1101 if (Val.MaxIndex == 0 && Name == Val.Name) 1102 return Val.Tgt; 1103 1104 if (Val.MaxIndex > 0 && Name.startswith(Val.Name)) { 1105 StringRef Suffix = Name.drop_front(Val.Name.size()); 1106 1107 unsigned Id; 1108 if (Suffix.getAsInteger(10, Id) || Id > Val.MaxIndex) 1109 return ET_INVALID; 1110 1111 // Disable leading zeroes 1112 if (Suffix.size() > 1 && Suffix[0] == '0') 1113 return ET_INVALID; 1114 1115 return Val.Tgt + Id; 1116 } 1117 } 1118 return ET_INVALID; 1119 } 1120 1121 bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI) { 1122 return (Id != ET_POS4 && Id != ET_PRIM) || isGFX10Plus(STI); 1123 } 1124 1125 } // namespace Exp 1126 1127 //===----------------------------------------------------------------------===// 1128 // MTBUF Format 1129 //===----------------------------------------------------------------------===// 1130 1131 namespace MTBUFFormat { 1132 1133 int64_t getDfmt(const StringRef Name) { 1134 for (int Id = DFMT_MIN; Id <= DFMT_MAX; ++Id) { 1135 if (Name == DfmtSymbolic[Id]) 1136 return Id; 1137 } 1138 return DFMT_UNDEF; 1139 } 1140 1141 StringRef getDfmtName(unsigned Id) { 1142 assert(Id <= DFMT_MAX); 1143 return DfmtSymbolic[Id]; 1144 } 1145 1146 static StringLiteral const *getNfmtLookupTable(const MCSubtargetInfo &STI) { 1147 if (isSI(STI) || isCI(STI)) 1148 return NfmtSymbolicSICI; 1149 if (isVI(STI) || isGFX9(STI)) 1150 return NfmtSymbolicVI; 1151 return NfmtSymbolicGFX10; 1152 } 1153 1154 int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI) { 1155 auto lookupTable = getNfmtLookupTable(STI); 1156 for (int Id = NFMT_MIN; Id <= NFMT_MAX; ++Id) { 1157 if (Name == lookupTable[Id]) 1158 return Id; 1159 } 1160 return NFMT_UNDEF; 1161 } 1162 1163 StringRef getNfmtName(unsigned Id, const MCSubtargetInfo &STI) { 1164 assert(Id <= NFMT_MAX); 1165 return getNfmtLookupTable(STI)[Id]; 1166 } 1167 1168 bool isValidDfmtNfmt(unsigned Id, const MCSubtargetInfo &STI) { 1169 unsigned Dfmt; 1170 unsigned Nfmt; 1171 decodeDfmtNfmt(Id, Dfmt, Nfmt); 1172 return isValidNfmt(Nfmt, STI); 1173 } 1174 1175 bool isValidNfmt(unsigned Id, const MCSubtargetInfo &STI) { 1176 return !getNfmtName(Id, STI).empty(); 1177 } 1178 1179 int64_t encodeDfmtNfmt(unsigned Dfmt, unsigned Nfmt) { 1180 return (Dfmt << DFMT_SHIFT) | (Nfmt << NFMT_SHIFT); 1181 } 1182 1183 void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt) { 1184 Dfmt = (Format >> DFMT_SHIFT) & DFMT_MASK; 1185 Nfmt = (Format >> NFMT_SHIFT) & NFMT_MASK; 1186 } 1187 1188 int64_t getUnifiedFormat(const StringRef Name) { 1189 for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) { 1190 if (Name == UfmtSymbolic[Id]) 1191 return Id; 1192 } 1193 return UFMT_UNDEF; 1194 } 1195 1196 StringRef getUnifiedFormatName(unsigned Id) { 1197 return isValidUnifiedFormat(Id) ? UfmtSymbolic[Id] : ""; 1198 } 1199 1200 bool isValidUnifiedFormat(unsigned Id) { 1201 return Id <= UFMT_LAST; 1202 } 1203 1204 int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt) { 1205 int64_t Fmt = encodeDfmtNfmt(Dfmt, Nfmt); 1206 for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) { 1207 if (Fmt == DfmtNfmt2UFmt[Id]) 1208 return Id; 1209 } 1210 return UFMT_UNDEF; 1211 } 1212 1213 bool isValidFormatEncoding(unsigned Val, const MCSubtargetInfo &STI) { 1214 return isGFX10Plus(STI) ? (Val <= UFMT_MAX) : (Val <= DFMT_NFMT_MAX); 1215 } 1216 1217 unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI) { 1218 if (isGFX10Plus(STI)) 1219 return UFMT_DEFAULT; 1220 return DFMT_NFMT_DEFAULT; 1221 } 1222 1223 } // namespace MTBUFFormat 1224 1225 //===----------------------------------------------------------------------===// 1226 // SendMsg 1227 //===----------------------------------------------------------------------===// 1228 1229 namespace SendMsg { 1230 1231 int64_t getMsgId(const StringRef Name) { 1232 for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) { 1233 if (IdSymbolic[i] && Name == IdSymbolic[i]) 1234 return i; 1235 } 1236 return ID_UNKNOWN_; 1237 } 1238 1239 bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI, bool Strict) { 1240 if (Strict) { 1241 switch (MsgId) { 1242 case ID_SAVEWAVE: 1243 return isVI(STI) || isGFX9Plus(STI); 1244 case ID_STALL_WAVE_GEN: 1245 case ID_HALT_WAVES: 1246 case ID_ORDERED_PS_DONE: 1247 case ID_GS_ALLOC_REQ: 1248 case ID_GET_DOORBELL: 1249 return isGFX9Plus(STI); 1250 case ID_EARLY_PRIM_DEALLOC: 1251 return isGFX9(STI); 1252 case ID_GET_DDID: 1253 return isGFX10Plus(STI); 1254 default: 1255 return 0 <= MsgId && MsgId < ID_GAPS_LAST_ && IdSymbolic[MsgId]; 1256 } 1257 } else { 1258 return 0 <= MsgId && isUInt<ID_WIDTH_>(MsgId); 1259 } 1260 } 1261 1262 StringRef getMsgName(int64_t MsgId) { 1263 assert(0 <= MsgId && MsgId < ID_GAPS_LAST_); 1264 return IdSymbolic[MsgId]; 1265 } 1266 1267 int64_t getMsgOpId(int64_t MsgId, const StringRef Name) { 1268 const char* const *S = (MsgId == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic; 1269 const int F = (MsgId == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_; 1270 const int L = (MsgId == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_; 1271 for (int i = F; i < L; ++i) { 1272 if (Name == S[i]) { 1273 return i; 1274 } 1275 } 1276 return OP_UNKNOWN_; 1277 } 1278 1279 bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI, 1280 bool Strict) { 1281 assert(isValidMsgId(MsgId, STI, Strict)); 1282 1283 if (!Strict) 1284 return 0 <= OpId && isUInt<OP_WIDTH_>(OpId); 1285 1286 switch(MsgId) 1287 { 1288 case ID_GS: 1289 return (OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_) && OpId != OP_GS_NOP; 1290 case ID_GS_DONE: 1291 return OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_; 1292 case ID_SYSMSG: 1293 return OP_SYS_FIRST_ <= OpId && OpId < OP_SYS_LAST_; 1294 default: 1295 return OpId == OP_NONE_; 1296 } 1297 } 1298 1299 StringRef getMsgOpName(int64_t MsgId, int64_t OpId) { 1300 assert(msgRequiresOp(MsgId)); 1301 return (MsgId == ID_SYSMSG)? OpSysSymbolic[OpId] : OpGsSymbolic[OpId]; 1302 } 1303 1304 bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId, 1305 const MCSubtargetInfo &STI, bool Strict) { 1306 assert(isValidMsgOp(MsgId, OpId, STI, Strict)); 1307 1308 if (!Strict) 1309 return 0 <= StreamId && isUInt<STREAM_ID_WIDTH_>(StreamId); 1310 1311 switch(MsgId) 1312 { 1313 case ID_GS: 1314 return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_; 1315 case ID_GS_DONE: 1316 return (OpId == OP_GS_NOP)? 1317 (StreamId == STREAM_ID_NONE_) : 1318 (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_); 1319 default: 1320 return StreamId == STREAM_ID_NONE_; 1321 } 1322 } 1323 1324 bool msgRequiresOp(int64_t MsgId) { 1325 return MsgId == ID_GS || MsgId == ID_GS_DONE || MsgId == ID_SYSMSG; 1326 } 1327 1328 bool msgSupportsStream(int64_t MsgId, int64_t OpId) { 1329 return (MsgId == ID_GS || MsgId == ID_GS_DONE) && OpId != OP_GS_NOP; 1330 } 1331 1332 void decodeMsg(unsigned Val, 1333 uint16_t &MsgId, 1334 uint16_t &OpId, 1335 uint16_t &StreamId) { 1336 MsgId = Val & ID_MASK_; 1337 OpId = (Val & OP_MASK_) >> OP_SHIFT_; 1338 StreamId = (Val & STREAM_ID_MASK_) >> STREAM_ID_SHIFT_; 1339 } 1340 1341 uint64_t encodeMsg(uint64_t MsgId, 1342 uint64_t OpId, 1343 uint64_t StreamId) { 1344 return (MsgId << ID_SHIFT_) | 1345 (OpId << OP_SHIFT_) | 1346 (StreamId << STREAM_ID_SHIFT_); 1347 } 1348 1349 } // namespace SendMsg 1350 1351 //===----------------------------------------------------------------------===// 1352 // 1353 //===----------------------------------------------------------------------===// 1354 1355 unsigned getInitialPSInputAddr(const Function &F) { 1356 return getIntegerAttribute(F, "InitialPSInputAddr", 0); 1357 } 1358 1359 bool getHasColorExport(const Function &F) { 1360 // As a safe default always respond as if PS has color exports. 1361 return getIntegerAttribute( 1362 F, "amdgpu-color-export", 1363 F.getCallingConv() == CallingConv::AMDGPU_PS ? 1 : 0) != 0; 1364 } 1365 1366 bool getHasDepthExport(const Function &F) { 1367 return getIntegerAttribute(F, "amdgpu-depth-export", 0) != 0; 1368 } 1369 1370 bool isShader(CallingConv::ID cc) { 1371 switch(cc) { 1372 case CallingConv::AMDGPU_VS: 1373 case CallingConv::AMDGPU_LS: 1374 case CallingConv::AMDGPU_HS: 1375 case CallingConv::AMDGPU_ES: 1376 case CallingConv::AMDGPU_GS: 1377 case CallingConv::AMDGPU_PS: 1378 case CallingConv::AMDGPU_CS: 1379 return true; 1380 default: 1381 return false; 1382 } 1383 } 1384 1385 bool isGraphics(CallingConv::ID cc) { 1386 return isShader(cc) || cc == CallingConv::AMDGPU_Gfx; 1387 } 1388 1389 bool isCompute(CallingConv::ID cc) { 1390 return !isGraphics(cc) || cc == CallingConv::AMDGPU_CS; 1391 } 1392 1393 bool isEntryFunctionCC(CallingConv::ID CC) { 1394 switch (CC) { 1395 case CallingConv::AMDGPU_KERNEL: 1396 case CallingConv::SPIR_KERNEL: 1397 case CallingConv::AMDGPU_VS: 1398 case CallingConv::AMDGPU_GS: 1399 case CallingConv::AMDGPU_PS: 1400 case CallingConv::AMDGPU_CS: 1401 case CallingConv::AMDGPU_ES: 1402 case CallingConv::AMDGPU_HS: 1403 case CallingConv::AMDGPU_LS: 1404 return true; 1405 default: 1406 return false; 1407 } 1408 } 1409 1410 bool isModuleEntryFunctionCC(CallingConv::ID CC) { 1411 switch (CC) { 1412 case CallingConv::AMDGPU_Gfx: 1413 return true; 1414 default: 1415 return isEntryFunctionCC(CC); 1416 } 1417 } 1418 1419 bool hasXNACK(const MCSubtargetInfo &STI) { 1420 return STI.getFeatureBits()[AMDGPU::FeatureXNACK]; 1421 } 1422 1423 bool hasSRAMECC(const MCSubtargetInfo &STI) { 1424 return STI.getFeatureBits()[AMDGPU::FeatureSRAMECC]; 1425 } 1426 1427 bool hasMIMG_R128(const MCSubtargetInfo &STI) { 1428 return STI.getFeatureBits()[AMDGPU::FeatureMIMG_R128] && !STI.getFeatureBits()[AMDGPU::FeatureR128A16]; 1429 } 1430 1431 bool hasGFX10A16(const MCSubtargetInfo &STI) { 1432 return STI.getFeatureBits()[AMDGPU::FeatureGFX10A16]; 1433 } 1434 1435 bool hasG16(const MCSubtargetInfo &STI) { 1436 return STI.getFeatureBits()[AMDGPU::FeatureG16]; 1437 } 1438 1439 bool hasPackedD16(const MCSubtargetInfo &STI) { 1440 return !STI.getFeatureBits()[AMDGPU::FeatureUnpackedD16VMem]; 1441 } 1442 1443 bool isSI(const MCSubtargetInfo &STI) { 1444 return STI.getFeatureBits()[AMDGPU::FeatureSouthernIslands]; 1445 } 1446 1447 bool isCI(const MCSubtargetInfo &STI) { 1448 return STI.getFeatureBits()[AMDGPU::FeatureSeaIslands]; 1449 } 1450 1451 bool isVI(const MCSubtargetInfo &STI) { 1452 return STI.getFeatureBits()[AMDGPU::FeatureVolcanicIslands]; 1453 } 1454 1455 bool isGFX9(const MCSubtargetInfo &STI) { 1456 return STI.getFeatureBits()[AMDGPU::FeatureGFX9]; 1457 } 1458 1459 bool isGFX9Plus(const MCSubtargetInfo &STI) { 1460 return isGFX9(STI) || isGFX10Plus(STI); 1461 } 1462 1463 bool isGFX10(const MCSubtargetInfo &STI) { 1464 return STI.getFeatureBits()[AMDGPU::FeatureGFX10]; 1465 } 1466 1467 bool isGFX10Plus(const MCSubtargetInfo &STI) { return isGFX10(STI); } 1468 1469 bool isGCN3Encoding(const MCSubtargetInfo &STI) { 1470 return STI.getFeatureBits()[AMDGPU::FeatureGCN3Encoding]; 1471 } 1472 1473 bool isGFX10_AEncoding(const MCSubtargetInfo &STI) { 1474 return STI.getFeatureBits()[AMDGPU::FeatureGFX10_AEncoding]; 1475 } 1476 1477 bool isGFX10_BEncoding(const MCSubtargetInfo &STI) { 1478 return STI.getFeatureBits()[AMDGPU::FeatureGFX10_BEncoding]; 1479 } 1480 1481 bool hasGFX10_3Insts(const MCSubtargetInfo &STI) { 1482 return STI.getFeatureBits()[AMDGPU::FeatureGFX10_3Insts]; 1483 } 1484 1485 bool isGFX90A(const MCSubtargetInfo &STI) { 1486 return STI.getFeatureBits()[AMDGPU::FeatureGFX90AInsts]; 1487 } 1488 1489 bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI) { 1490 return STI.getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch]; 1491 } 1492 1493 bool isSGPR(unsigned Reg, const MCRegisterInfo* TRI) { 1494 const MCRegisterClass SGPRClass = TRI->getRegClass(AMDGPU::SReg_32RegClassID); 1495 const unsigned FirstSubReg = TRI->getSubReg(Reg, AMDGPU::sub0); 1496 return SGPRClass.contains(FirstSubReg != 0 ? FirstSubReg : Reg) || 1497 Reg == AMDGPU::SCC; 1498 } 1499 1500 bool isRegIntersect(unsigned Reg0, unsigned Reg1, const MCRegisterInfo* TRI) { 1501 for (MCRegAliasIterator R(Reg0, TRI, true); R.isValid(); ++R) { 1502 if (*R == Reg1) return true; 1503 } 1504 return false; 1505 } 1506 1507 #define MAP_REG2REG \ 1508 using namespace AMDGPU; \ 1509 switch(Reg) { \ 1510 default: return Reg; \ 1511 CASE_CI_VI(FLAT_SCR) \ 1512 CASE_CI_VI(FLAT_SCR_LO) \ 1513 CASE_CI_VI(FLAT_SCR_HI) \ 1514 CASE_VI_GFX9PLUS(TTMP0) \ 1515 CASE_VI_GFX9PLUS(TTMP1) \ 1516 CASE_VI_GFX9PLUS(TTMP2) \ 1517 CASE_VI_GFX9PLUS(TTMP3) \ 1518 CASE_VI_GFX9PLUS(TTMP4) \ 1519 CASE_VI_GFX9PLUS(TTMP5) \ 1520 CASE_VI_GFX9PLUS(TTMP6) \ 1521 CASE_VI_GFX9PLUS(TTMP7) \ 1522 CASE_VI_GFX9PLUS(TTMP8) \ 1523 CASE_VI_GFX9PLUS(TTMP9) \ 1524 CASE_VI_GFX9PLUS(TTMP10) \ 1525 CASE_VI_GFX9PLUS(TTMP11) \ 1526 CASE_VI_GFX9PLUS(TTMP12) \ 1527 CASE_VI_GFX9PLUS(TTMP13) \ 1528 CASE_VI_GFX9PLUS(TTMP14) \ 1529 CASE_VI_GFX9PLUS(TTMP15) \ 1530 CASE_VI_GFX9PLUS(TTMP0_TTMP1) \ 1531 CASE_VI_GFX9PLUS(TTMP2_TTMP3) \ 1532 CASE_VI_GFX9PLUS(TTMP4_TTMP5) \ 1533 CASE_VI_GFX9PLUS(TTMP6_TTMP7) \ 1534 CASE_VI_GFX9PLUS(TTMP8_TTMP9) \ 1535 CASE_VI_GFX9PLUS(TTMP10_TTMP11) \ 1536 CASE_VI_GFX9PLUS(TTMP12_TTMP13) \ 1537 CASE_VI_GFX9PLUS(TTMP14_TTMP15) \ 1538 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3) \ 1539 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7) \ 1540 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11) \ 1541 CASE_VI_GFX9PLUS(TTMP12_TTMP13_TTMP14_TTMP15) \ 1542 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7) \ 1543 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11) \ 1544 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \ 1545 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \ 1546 } 1547 1548 #define CASE_CI_VI(node) \ 1549 assert(!isSI(STI)); \ 1550 case node: return isCI(STI) ? node##_ci : node##_vi; 1551 1552 #define CASE_VI_GFX9PLUS(node) \ 1553 case node: return isGFX9Plus(STI) ? node##_gfx9plus : node##_vi; 1554 1555 unsigned getMCReg(unsigned Reg, const MCSubtargetInfo &STI) { 1556 if (STI.getTargetTriple().getArch() == Triple::r600) 1557 return Reg; 1558 MAP_REG2REG 1559 } 1560 1561 #undef CASE_CI_VI 1562 #undef CASE_VI_GFX9PLUS 1563 1564 #define CASE_CI_VI(node) case node##_ci: case node##_vi: return node; 1565 #define CASE_VI_GFX9PLUS(node) case node##_vi: case node##_gfx9plus: return node; 1566 1567 unsigned mc2PseudoReg(unsigned Reg) { 1568 MAP_REG2REG 1569 } 1570 1571 #undef CASE_CI_VI 1572 #undef CASE_VI_GFX9PLUS 1573 #undef MAP_REG2REG 1574 1575 bool isSISrcOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1576 assert(OpNo < Desc.NumOperands); 1577 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1578 return OpType >= AMDGPU::OPERAND_SRC_FIRST && 1579 OpType <= AMDGPU::OPERAND_SRC_LAST; 1580 } 1581 1582 bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1583 assert(OpNo < Desc.NumOperands); 1584 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1585 switch (OpType) { 1586 case AMDGPU::OPERAND_REG_IMM_FP32: 1587 case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED: 1588 case AMDGPU::OPERAND_REG_IMM_FP64: 1589 case AMDGPU::OPERAND_REG_IMM_FP16: 1590 case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED: 1591 case AMDGPU::OPERAND_REG_IMM_V2FP16: 1592 case AMDGPU::OPERAND_REG_IMM_V2INT16: 1593 case AMDGPU::OPERAND_REG_INLINE_C_FP32: 1594 case AMDGPU::OPERAND_REG_INLINE_C_FP64: 1595 case AMDGPU::OPERAND_REG_INLINE_C_FP16: 1596 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: 1597 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: 1598 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: 1599 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: 1600 case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: 1601 case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16: 1602 case AMDGPU::OPERAND_REG_IMM_V2FP32: 1603 case AMDGPU::OPERAND_REG_INLINE_C_V2FP32: 1604 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: 1605 return true; 1606 default: 1607 return false; 1608 } 1609 } 1610 1611 bool isSISrcInlinableOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1612 assert(OpNo < Desc.NumOperands); 1613 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1614 return OpType >= AMDGPU::OPERAND_REG_INLINE_C_FIRST && 1615 OpType <= AMDGPU::OPERAND_REG_INLINE_C_LAST; 1616 } 1617 1618 // Avoid using MCRegisterClass::getSize, since that function will go away 1619 // (move from MC* level to Target* level). Return size in bits. 1620 unsigned getRegBitWidth(unsigned RCID) { 1621 switch (RCID) { 1622 case AMDGPU::VGPR_LO16RegClassID: 1623 case AMDGPU::VGPR_HI16RegClassID: 1624 case AMDGPU::SGPR_LO16RegClassID: 1625 case AMDGPU::AGPR_LO16RegClassID: 1626 return 16; 1627 case AMDGPU::SGPR_32RegClassID: 1628 case AMDGPU::VGPR_32RegClassID: 1629 case AMDGPU::VRegOrLds_32RegClassID: 1630 case AMDGPU::AGPR_32RegClassID: 1631 case AMDGPU::VS_32RegClassID: 1632 case AMDGPU::AV_32RegClassID: 1633 case AMDGPU::SReg_32RegClassID: 1634 case AMDGPU::SReg_32_XM0RegClassID: 1635 case AMDGPU::SRegOrLds_32RegClassID: 1636 return 32; 1637 case AMDGPU::SGPR_64RegClassID: 1638 case AMDGPU::VS_64RegClassID: 1639 case AMDGPU::SReg_64RegClassID: 1640 case AMDGPU::VReg_64RegClassID: 1641 case AMDGPU::AReg_64RegClassID: 1642 case AMDGPU::SReg_64_XEXECRegClassID: 1643 case AMDGPU::VReg_64_Align2RegClassID: 1644 case AMDGPU::AReg_64_Align2RegClassID: 1645 case AMDGPU::AV_64RegClassID: 1646 case AMDGPU::AV_64_Align2RegClassID: 1647 return 64; 1648 case AMDGPU::SGPR_96RegClassID: 1649 case AMDGPU::SReg_96RegClassID: 1650 case AMDGPU::VReg_96RegClassID: 1651 case AMDGPU::AReg_96RegClassID: 1652 case AMDGPU::VReg_96_Align2RegClassID: 1653 case AMDGPU::AReg_96_Align2RegClassID: 1654 case AMDGPU::AV_96RegClassID: 1655 case AMDGPU::AV_96_Align2RegClassID: 1656 return 96; 1657 case AMDGPU::SGPR_128RegClassID: 1658 case AMDGPU::SReg_128RegClassID: 1659 case AMDGPU::VReg_128RegClassID: 1660 case AMDGPU::AReg_128RegClassID: 1661 case AMDGPU::VReg_128_Align2RegClassID: 1662 case AMDGPU::AReg_128_Align2RegClassID: 1663 case AMDGPU::AV_128RegClassID: 1664 case AMDGPU::AV_128_Align2RegClassID: 1665 return 128; 1666 case AMDGPU::SGPR_160RegClassID: 1667 case AMDGPU::SReg_160RegClassID: 1668 case AMDGPU::VReg_160RegClassID: 1669 case AMDGPU::AReg_160RegClassID: 1670 case AMDGPU::VReg_160_Align2RegClassID: 1671 case AMDGPU::AReg_160_Align2RegClassID: 1672 case AMDGPU::AV_160RegClassID: 1673 case AMDGPU::AV_160_Align2RegClassID: 1674 return 160; 1675 case AMDGPU::SGPR_192RegClassID: 1676 case AMDGPU::SReg_192RegClassID: 1677 case AMDGPU::VReg_192RegClassID: 1678 case AMDGPU::AReg_192RegClassID: 1679 case AMDGPU::VReg_192_Align2RegClassID: 1680 case AMDGPU::AReg_192_Align2RegClassID: 1681 case AMDGPU::AV_192RegClassID: 1682 case AMDGPU::AV_192_Align2RegClassID: 1683 return 192; 1684 case AMDGPU::SGPR_224RegClassID: 1685 case AMDGPU::SReg_224RegClassID: 1686 case AMDGPU::VReg_224RegClassID: 1687 case AMDGPU::AReg_224RegClassID: 1688 case AMDGPU::VReg_224_Align2RegClassID: 1689 case AMDGPU::AReg_224_Align2RegClassID: 1690 case AMDGPU::AV_224RegClassID: 1691 case AMDGPU::AV_224_Align2RegClassID: 1692 return 224; 1693 case AMDGPU::SGPR_256RegClassID: 1694 case AMDGPU::SReg_256RegClassID: 1695 case AMDGPU::VReg_256RegClassID: 1696 case AMDGPU::AReg_256RegClassID: 1697 case AMDGPU::VReg_256_Align2RegClassID: 1698 case AMDGPU::AReg_256_Align2RegClassID: 1699 case AMDGPU::AV_256RegClassID: 1700 case AMDGPU::AV_256_Align2RegClassID: 1701 return 256; 1702 case AMDGPU::SGPR_512RegClassID: 1703 case AMDGPU::SReg_512RegClassID: 1704 case AMDGPU::VReg_512RegClassID: 1705 case AMDGPU::AReg_512RegClassID: 1706 case AMDGPU::VReg_512_Align2RegClassID: 1707 case AMDGPU::AReg_512_Align2RegClassID: 1708 case AMDGPU::AV_512RegClassID: 1709 case AMDGPU::AV_512_Align2RegClassID: 1710 return 512; 1711 case AMDGPU::SGPR_1024RegClassID: 1712 case AMDGPU::SReg_1024RegClassID: 1713 case AMDGPU::VReg_1024RegClassID: 1714 case AMDGPU::AReg_1024RegClassID: 1715 case AMDGPU::VReg_1024_Align2RegClassID: 1716 case AMDGPU::AReg_1024_Align2RegClassID: 1717 case AMDGPU::AV_1024RegClassID: 1718 case AMDGPU::AV_1024_Align2RegClassID: 1719 return 1024; 1720 default: 1721 llvm_unreachable("Unexpected register class"); 1722 } 1723 } 1724 1725 unsigned getRegBitWidth(const MCRegisterClass &RC) { 1726 return getRegBitWidth(RC.getID()); 1727 } 1728 1729 unsigned getRegOperandSize(const MCRegisterInfo *MRI, const MCInstrDesc &Desc, 1730 unsigned OpNo) { 1731 assert(OpNo < Desc.NumOperands); 1732 unsigned RCID = Desc.OpInfo[OpNo].RegClass; 1733 return getRegBitWidth(MRI->getRegClass(RCID)) / 8; 1734 } 1735 1736 bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi) { 1737 if (isInlinableIntLiteral(Literal)) 1738 return true; 1739 1740 uint64_t Val = static_cast<uint64_t>(Literal); 1741 return (Val == DoubleToBits(0.0)) || 1742 (Val == DoubleToBits(1.0)) || 1743 (Val == DoubleToBits(-1.0)) || 1744 (Val == DoubleToBits(0.5)) || 1745 (Val == DoubleToBits(-0.5)) || 1746 (Val == DoubleToBits(2.0)) || 1747 (Val == DoubleToBits(-2.0)) || 1748 (Val == DoubleToBits(4.0)) || 1749 (Val == DoubleToBits(-4.0)) || 1750 (Val == 0x3fc45f306dc9c882 && HasInv2Pi); 1751 } 1752 1753 bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi) { 1754 if (isInlinableIntLiteral(Literal)) 1755 return true; 1756 1757 // The actual type of the operand does not seem to matter as long 1758 // as the bits match one of the inline immediate values. For example: 1759 // 1760 // -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal, 1761 // so it is a legal inline immediate. 1762 // 1763 // 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in 1764 // floating-point, so it is a legal inline immediate. 1765 1766 uint32_t Val = static_cast<uint32_t>(Literal); 1767 return (Val == FloatToBits(0.0f)) || 1768 (Val == FloatToBits(1.0f)) || 1769 (Val == FloatToBits(-1.0f)) || 1770 (Val == FloatToBits(0.5f)) || 1771 (Val == FloatToBits(-0.5f)) || 1772 (Val == FloatToBits(2.0f)) || 1773 (Val == FloatToBits(-2.0f)) || 1774 (Val == FloatToBits(4.0f)) || 1775 (Val == FloatToBits(-4.0f)) || 1776 (Val == 0x3e22f983 && HasInv2Pi); 1777 } 1778 1779 bool isInlinableLiteral16(int16_t Literal, bool HasInv2Pi) { 1780 if (!HasInv2Pi) 1781 return false; 1782 1783 if (isInlinableIntLiteral(Literal)) 1784 return true; 1785 1786 uint16_t Val = static_cast<uint16_t>(Literal); 1787 return Val == 0x3C00 || // 1.0 1788 Val == 0xBC00 || // -1.0 1789 Val == 0x3800 || // 0.5 1790 Val == 0xB800 || // -0.5 1791 Val == 0x4000 || // 2.0 1792 Val == 0xC000 || // -2.0 1793 Val == 0x4400 || // 4.0 1794 Val == 0xC400 || // -4.0 1795 Val == 0x3118; // 1/2pi 1796 } 1797 1798 bool isInlinableLiteralV216(int32_t Literal, bool HasInv2Pi) { 1799 assert(HasInv2Pi); 1800 1801 if (isInt<16>(Literal) || isUInt<16>(Literal)) { 1802 int16_t Trunc = static_cast<int16_t>(Literal); 1803 return AMDGPU::isInlinableLiteral16(Trunc, HasInv2Pi); 1804 } 1805 if (!(Literal & 0xffff)) 1806 return AMDGPU::isInlinableLiteral16(Literal >> 16, HasInv2Pi); 1807 1808 int16_t Lo16 = static_cast<int16_t>(Literal); 1809 int16_t Hi16 = static_cast<int16_t>(Literal >> 16); 1810 return Lo16 == Hi16 && isInlinableLiteral16(Lo16, HasInv2Pi); 1811 } 1812 1813 bool isInlinableIntLiteralV216(int32_t Literal) { 1814 int16_t Lo16 = static_cast<int16_t>(Literal); 1815 if (isInt<16>(Literal) || isUInt<16>(Literal)) 1816 return isInlinableIntLiteral(Lo16); 1817 1818 int16_t Hi16 = static_cast<int16_t>(Literal >> 16); 1819 if (!(Literal & 0xffff)) 1820 return isInlinableIntLiteral(Hi16); 1821 return Lo16 == Hi16 && isInlinableIntLiteral(Lo16); 1822 } 1823 1824 bool isFoldableLiteralV216(int32_t Literal, bool HasInv2Pi) { 1825 assert(HasInv2Pi); 1826 1827 int16_t Lo16 = static_cast<int16_t>(Literal); 1828 if (isInt<16>(Literal) || isUInt<16>(Literal)) 1829 return true; 1830 1831 int16_t Hi16 = static_cast<int16_t>(Literal >> 16); 1832 if (!(Literal & 0xffff)) 1833 return true; 1834 return Lo16 == Hi16; 1835 } 1836 1837 bool isArgPassedInSGPR(const Argument *A) { 1838 const Function *F = A->getParent(); 1839 1840 // Arguments to compute shaders are never a source of divergence. 1841 CallingConv::ID CC = F->getCallingConv(); 1842 switch (CC) { 1843 case CallingConv::AMDGPU_KERNEL: 1844 case CallingConv::SPIR_KERNEL: 1845 return true; 1846 case CallingConv::AMDGPU_VS: 1847 case CallingConv::AMDGPU_LS: 1848 case CallingConv::AMDGPU_HS: 1849 case CallingConv::AMDGPU_ES: 1850 case CallingConv::AMDGPU_GS: 1851 case CallingConv::AMDGPU_PS: 1852 case CallingConv::AMDGPU_CS: 1853 case CallingConv::AMDGPU_Gfx: 1854 // For non-compute shaders, SGPR inputs are marked with either inreg or byval. 1855 // Everything else is in VGPRs. 1856 return F->getAttributes().hasParamAttr(A->getArgNo(), Attribute::InReg) || 1857 F->getAttributes().hasParamAttr(A->getArgNo(), Attribute::ByVal); 1858 default: 1859 // TODO: Should calls support inreg for SGPR inputs? 1860 return false; 1861 } 1862 } 1863 1864 static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) { 1865 return isGCN3Encoding(ST) || isGFX10Plus(ST); 1866 } 1867 1868 static bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST) { 1869 return isGFX9Plus(ST); 1870 } 1871 1872 bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST, 1873 int64_t EncodedOffset) { 1874 return hasSMEMByteOffset(ST) ? isUInt<20>(EncodedOffset) 1875 : isUInt<8>(EncodedOffset); 1876 } 1877 1878 bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST, 1879 int64_t EncodedOffset, 1880 bool IsBuffer) { 1881 return !IsBuffer && 1882 hasSMRDSignedImmOffset(ST) && 1883 isInt<21>(EncodedOffset); 1884 } 1885 1886 static bool isDwordAligned(uint64_t ByteOffset) { 1887 return (ByteOffset & 3) == 0; 1888 } 1889 1890 uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST, 1891 uint64_t ByteOffset) { 1892 if (hasSMEMByteOffset(ST)) 1893 return ByteOffset; 1894 1895 assert(isDwordAligned(ByteOffset)); 1896 return ByteOffset >> 2; 1897 } 1898 1899 Optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST, 1900 int64_t ByteOffset, bool IsBuffer) { 1901 // The signed version is always a byte offset. 1902 if (!IsBuffer && hasSMRDSignedImmOffset(ST)) { 1903 assert(hasSMEMByteOffset(ST)); 1904 return isInt<20>(ByteOffset) ? Optional<int64_t>(ByteOffset) : None; 1905 } 1906 1907 if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST)) 1908 return None; 1909 1910 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset); 1911 return isLegalSMRDEncodedUnsignedOffset(ST, EncodedOffset) 1912 ? Optional<int64_t>(EncodedOffset) 1913 : None; 1914 } 1915 1916 Optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST, 1917 int64_t ByteOffset) { 1918 if (!isCI(ST) || !isDwordAligned(ByteOffset)) 1919 return None; 1920 1921 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset); 1922 return isUInt<32>(EncodedOffset) ? Optional<int64_t>(EncodedOffset) : None; 1923 } 1924 1925 unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST, bool Signed) { 1926 // Address offset is 12-bit signed for GFX10, 13-bit for GFX9. 1927 if (AMDGPU::isGFX10(ST)) 1928 return Signed ? 12 : 11; 1929 1930 return Signed ? 13 : 12; 1931 } 1932 1933 // Given Imm, split it into the values to put into the SOffset and ImmOffset 1934 // fields in an MUBUF instruction. Return false if it is not possible (due to a 1935 // hardware bug needing a workaround). 1936 // 1937 // The required alignment ensures that individual address components remain 1938 // aligned if they are aligned to begin with. It also ensures that additional 1939 // offsets within the given alignment can be added to the resulting ImmOffset. 1940 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset, 1941 const GCNSubtarget *Subtarget, Align Alignment) { 1942 const uint32_t MaxImm = alignDown(4095, Alignment.value()); 1943 uint32_t Overflow = 0; 1944 1945 if (Imm > MaxImm) { 1946 if (Imm <= MaxImm + 64) { 1947 // Use an SOffset inline constant for 4..64 1948 Overflow = Imm - MaxImm; 1949 Imm = MaxImm; 1950 } else { 1951 // Try to keep the same value in SOffset for adjacent loads, so that 1952 // the corresponding register contents can be re-used. 1953 // 1954 // Load values with all low-bits (except for alignment bits) set into 1955 // SOffset, so that a larger range of values can be covered using 1956 // s_movk_i32. 1957 // 1958 // Atomic operations fail to work correctly when individual address 1959 // components are unaligned, even if their sum is aligned. 1960 uint32_t High = (Imm + Alignment.value()) & ~4095; 1961 uint32_t Low = (Imm + Alignment.value()) & 4095; 1962 Imm = Low; 1963 Overflow = High - Alignment.value(); 1964 } 1965 } 1966 1967 // There is a hardware bug in SI and CI which prevents address clamping in 1968 // MUBUF instructions from working correctly with SOffsets. The immediate 1969 // offset is unaffected. 1970 if (Overflow > 0 && 1971 Subtarget->getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS) 1972 return false; 1973 1974 ImmOffset = Imm; 1975 SOffset = Overflow; 1976 return true; 1977 } 1978 1979 SIModeRegisterDefaults::SIModeRegisterDefaults(const Function &F) { 1980 *this = getDefaultForCallingConv(F.getCallingConv()); 1981 1982 StringRef IEEEAttr = F.getFnAttribute("amdgpu-ieee").getValueAsString(); 1983 if (!IEEEAttr.empty()) 1984 IEEE = IEEEAttr == "true"; 1985 1986 StringRef DX10ClampAttr 1987 = F.getFnAttribute("amdgpu-dx10-clamp").getValueAsString(); 1988 if (!DX10ClampAttr.empty()) 1989 DX10Clamp = DX10ClampAttr == "true"; 1990 1991 StringRef DenormF32Attr = F.getFnAttribute("denormal-fp-math-f32").getValueAsString(); 1992 if (!DenormF32Attr.empty()) { 1993 DenormalMode DenormMode = parseDenormalFPAttribute(DenormF32Attr); 1994 FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE; 1995 FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE; 1996 } 1997 1998 StringRef DenormAttr = F.getFnAttribute("denormal-fp-math").getValueAsString(); 1999 if (!DenormAttr.empty()) { 2000 DenormalMode DenormMode = parseDenormalFPAttribute(DenormAttr); 2001 2002 if (DenormF32Attr.empty()) { 2003 FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE; 2004 FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE; 2005 } 2006 2007 FP64FP16InputDenormals = DenormMode.Input == DenormalMode::IEEE; 2008 FP64FP16OutputDenormals = DenormMode.Output == DenormalMode::IEEE; 2009 } 2010 } 2011 2012 namespace { 2013 2014 struct SourceOfDivergence { 2015 unsigned Intr; 2016 }; 2017 const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr); 2018 2019 #define GET_SourcesOfDivergence_IMPL 2020 #define GET_Gfx9BufferFormat_IMPL 2021 #define GET_Gfx10PlusBufferFormat_IMPL 2022 #include "AMDGPUGenSearchableTables.inc" 2023 2024 } // end anonymous namespace 2025 2026 bool isIntrinsicSourceOfDivergence(unsigned IntrID) { 2027 return lookupSourceOfDivergence(IntrID); 2028 } 2029 2030 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp, 2031 uint8_t NumComponents, 2032 uint8_t NumFormat, 2033 const MCSubtargetInfo &STI) { 2034 return isGFX10Plus(STI) 2035 ? getGfx10PlusBufferFormatInfo(BitsPerComp, NumComponents, 2036 NumFormat) 2037 : getGfx9BufferFormatInfo(BitsPerComp, NumComponents, NumFormat); 2038 } 2039 2040 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format, 2041 const MCSubtargetInfo &STI) { 2042 return isGFX10Plus(STI) ? getGfx10PlusBufferFormatInfo(Format) 2043 : getGfx9BufferFormatInfo(Format); 2044 } 2045 2046 } // namespace AMDGPU 2047 2048 raw_ostream &operator<<(raw_ostream &OS, 2049 const AMDGPU::IsaInfo::TargetIDSetting S) { 2050 switch (S) { 2051 case (AMDGPU::IsaInfo::TargetIDSetting::Unsupported): 2052 OS << "Unsupported"; 2053 break; 2054 case (AMDGPU::IsaInfo::TargetIDSetting::Any): 2055 OS << "Any"; 2056 break; 2057 case (AMDGPU::IsaInfo::TargetIDSetting::Off): 2058 OS << "Off"; 2059 break; 2060 case (AMDGPU::IsaInfo::TargetIDSetting::On): 2061 OS << "On"; 2062 break; 2063 } 2064 return OS; 2065 } 2066 2067 } // namespace llvm 2068