1 //===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "AMDGPUBaseInfo.h" 10 #include "AMDGPU.h" 11 #include "AMDGPUAsmUtils.h" 12 #include "AMDKernelCodeT.h" 13 #include "GCNSubtarget.h" 14 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 15 #include "llvm/BinaryFormat/ELF.h" 16 #include "llvm/IR/Attributes.h" 17 #include "llvm/IR/Function.h" 18 #include "llvm/IR/GlobalValue.h" 19 #include "llvm/IR/IntrinsicsAMDGPU.h" 20 #include "llvm/IR/IntrinsicsR600.h" 21 #include "llvm/IR/LLVMContext.h" 22 #include "llvm/MC/MCSubtargetInfo.h" 23 #include "llvm/Support/AMDHSAKernelDescriptor.h" 24 #include "llvm/Support/CommandLine.h" 25 #include "llvm/Support/TargetParser.h" 26 27 #define GET_INSTRINFO_NAMED_OPS 28 #define GET_INSTRMAP_INFO 29 #include "AMDGPUGenInstrInfo.inc" 30 31 static llvm::cl::opt<unsigned> AmdhsaCodeObjectVersion( 32 "amdhsa-code-object-version", llvm::cl::Hidden, 33 llvm::cl::desc("AMDHSA Code Object Version"), llvm::cl::init(4), 34 llvm::cl::ZeroOrMore); 35 36 namespace { 37 38 /// \returns Bit mask for given bit \p Shift and bit \p Width. 39 unsigned getBitMask(unsigned Shift, unsigned Width) { 40 return ((1 << Width) - 1) << Shift; 41 } 42 43 /// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width. 44 /// 45 /// \returns Packed \p Dst. 46 unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) { 47 Dst &= ~(1 << Shift) & ~getBitMask(Shift, Width); 48 Dst |= (Src << Shift) & getBitMask(Shift, Width); 49 return Dst; 50 } 51 52 /// Unpacks bits from \p Src for given bit \p Shift and bit \p Width. 53 /// 54 /// \returns Unpacked bits. 55 unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) { 56 return (Src & getBitMask(Shift, Width)) >> Shift; 57 } 58 59 /// \returns Vmcnt bit shift (lower bits). 60 unsigned getVmcntBitShiftLo() { return 0; } 61 62 /// \returns Vmcnt bit width (lower bits). 63 unsigned getVmcntBitWidthLo() { return 4; } 64 65 /// \returns Expcnt bit shift. 66 unsigned getExpcntBitShift() { return 4; } 67 68 /// \returns Expcnt bit width. 69 unsigned getExpcntBitWidth() { return 3; } 70 71 /// \returns Lgkmcnt bit shift. 72 unsigned getLgkmcntBitShift() { return 8; } 73 74 /// \returns Lgkmcnt bit width. 75 unsigned getLgkmcntBitWidth(unsigned VersionMajor) { 76 return (VersionMajor >= 10) ? 6 : 4; 77 } 78 79 /// \returns Vmcnt bit shift (higher bits). 80 unsigned getVmcntBitShiftHi() { return 14; } 81 82 /// \returns Vmcnt bit width (higher bits). 83 unsigned getVmcntBitWidthHi() { return 2; } 84 85 } // end namespace anonymous 86 87 namespace llvm { 88 89 namespace AMDGPU { 90 91 Optional<uint8_t> getHsaAbiVersion(const MCSubtargetInfo *STI) { 92 if (STI && STI->getTargetTriple().getOS() != Triple::AMDHSA) 93 return None; 94 95 switch (AmdhsaCodeObjectVersion) { 96 case 2: 97 return ELF::ELFABIVERSION_AMDGPU_HSA_V2; 98 case 3: 99 return ELF::ELFABIVERSION_AMDGPU_HSA_V3; 100 case 4: 101 return ELF::ELFABIVERSION_AMDGPU_HSA_V4; 102 default: 103 report_fatal_error(Twine("Unsupported AMDHSA Code Object Version ") + 104 Twine(AmdhsaCodeObjectVersion)); 105 } 106 } 107 108 bool isHsaAbiVersion2(const MCSubtargetInfo *STI) { 109 if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI)) 110 return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V2; 111 return false; 112 } 113 114 bool isHsaAbiVersion3(const MCSubtargetInfo *STI) { 115 if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI)) 116 return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V3; 117 return false; 118 } 119 120 bool isHsaAbiVersion4(const MCSubtargetInfo *STI) { 121 if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI)) 122 return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V4; 123 return false; 124 } 125 126 bool isHsaAbiVersion3Or4(const MCSubtargetInfo *STI) { 127 return isHsaAbiVersion3(STI) || isHsaAbiVersion4(STI); 128 } 129 130 #define GET_MIMGBaseOpcodesTable_IMPL 131 #define GET_MIMGDimInfoTable_IMPL 132 #define GET_MIMGInfoTable_IMPL 133 #define GET_MIMGLZMappingTable_IMPL 134 #define GET_MIMGMIPMappingTable_IMPL 135 #define GET_MIMGG16MappingTable_IMPL 136 #include "AMDGPUGenSearchableTables.inc" 137 138 int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding, 139 unsigned VDataDwords, unsigned VAddrDwords) { 140 const MIMGInfo *Info = getMIMGOpcodeHelper(BaseOpcode, MIMGEncoding, 141 VDataDwords, VAddrDwords); 142 return Info ? Info->Opcode : -1; 143 } 144 145 const MIMGBaseOpcodeInfo *getMIMGBaseOpcode(unsigned Opc) { 146 const MIMGInfo *Info = getMIMGInfo(Opc); 147 return Info ? getMIMGBaseOpcodeInfo(Info->BaseOpcode) : nullptr; 148 } 149 150 int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) { 151 const MIMGInfo *OrigInfo = getMIMGInfo(Opc); 152 const MIMGInfo *NewInfo = 153 getMIMGOpcodeHelper(OrigInfo->BaseOpcode, OrigInfo->MIMGEncoding, 154 NewChannels, OrigInfo->VAddrDwords); 155 return NewInfo ? NewInfo->Opcode : -1; 156 } 157 158 unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode, 159 const MIMGDimInfo *Dim, bool IsA16, 160 bool IsG16Supported) { 161 unsigned AddrWords = BaseOpcode->NumExtraArgs; 162 unsigned AddrComponents = (BaseOpcode->Coordinates ? Dim->NumCoords : 0) + 163 (BaseOpcode->LodOrClampOrMip ? 1 : 0); 164 if (IsA16) 165 AddrWords += divideCeil(AddrComponents, 2); 166 else 167 AddrWords += AddrComponents; 168 169 // Note: For subtargets that support A16 but not G16, enabling A16 also 170 // enables 16 bit gradients. 171 // For subtargets that support A16 (operand) and G16 (done with a different 172 // instruction encoding), they are independent. 173 174 if (BaseOpcode->Gradients) { 175 if ((IsA16 && !IsG16Supported) || BaseOpcode->G16) 176 // There are two gradients per coordinate, we pack them separately. 177 // For the 3d case, 178 // we get (dy/du, dx/du) (-, dz/du) (dy/dv, dx/dv) (-, dz/dv) 179 AddrWords += alignTo<2>(Dim->NumGradients / 2); 180 else 181 AddrWords += Dim->NumGradients; 182 } 183 return AddrWords; 184 } 185 186 struct MUBUFInfo { 187 uint16_t Opcode; 188 uint16_t BaseOpcode; 189 uint8_t elements; 190 bool has_vaddr; 191 bool has_srsrc; 192 bool has_soffset; 193 bool IsBufferInv; 194 }; 195 196 struct MTBUFInfo { 197 uint16_t Opcode; 198 uint16_t BaseOpcode; 199 uint8_t elements; 200 bool has_vaddr; 201 bool has_srsrc; 202 bool has_soffset; 203 }; 204 205 struct SMInfo { 206 uint16_t Opcode; 207 bool IsBuffer; 208 }; 209 210 struct VOPInfo { 211 uint16_t Opcode; 212 bool IsSingle; 213 }; 214 215 #define GET_MTBUFInfoTable_DECL 216 #define GET_MTBUFInfoTable_IMPL 217 #define GET_MUBUFInfoTable_DECL 218 #define GET_MUBUFInfoTable_IMPL 219 #define GET_SMInfoTable_DECL 220 #define GET_SMInfoTable_IMPL 221 #define GET_VOP1InfoTable_DECL 222 #define GET_VOP1InfoTable_IMPL 223 #define GET_VOP2InfoTable_DECL 224 #define GET_VOP2InfoTable_IMPL 225 #define GET_VOP3InfoTable_DECL 226 #define GET_VOP3InfoTable_IMPL 227 #include "AMDGPUGenSearchableTables.inc" 228 229 int getMTBUFBaseOpcode(unsigned Opc) { 230 const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opc); 231 return Info ? Info->BaseOpcode : -1; 232 } 233 234 int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements) { 235 const MTBUFInfo *Info = getMTBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements); 236 return Info ? Info->Opcode : -1; 237 } 238 239 int getMTBUFElements(unsigned Opc) { 240 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 241 return Info ? Info->elements : 0; 242 } 243 244 bool getMTBUFHasVAddr(unsigned Opc) { 245 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 246 return Info ? Info->has_vaddr : false; 247 } 248 249 bool getMTBUFHasSrsrc(unsigned Opc) { 250 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 251 return Info ? Info->has_srsrc : false; 252 } 253 254 bool getMTBUFHasSoffset(unsigned Opc) { 255 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 256 return Info ? Info->has_soffset : false; 257 } 258 259 int getMUBUFBaseOpcode(unsigned Opc) { 260 const MUBUFInfo *Info = getMUBUFInfoFromOpcode(Opc); 261 return Info ? Info->BaseOpcode : -1; 262 } 263 264 int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements) { 265 const MUBUFInfo *Info = getMUBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements); 266 return Info ? Info->Opcode : -1; 267 } 268 269 int getMUBUFElements(unsigned Opc) { 270 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 271 return Info ? Info->elements : 0; 272 } 273 274 bool getMUBUFHasVAddr(unsigned Opc) { 275 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 276 return Info ? Info->has_vaddr : false; 277 } 278 279 bool getMUBUFHasSrsrc(unsigned Opc) { 280 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 281 return Info ? Info->has_srsrc : false; 282 } 283 284 bool getMUBUFHasSoffset(unsigned Opc) { 285 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 286 return Info ? Info->has_soffset : false; 287 } 288 289 bool getMUBUFIsBufferInv(unsigned Opc) { 290 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 291 return Info ? Info->IsBufferInv : false; 292 } 293 294 bool getSMEMIsBuffer(unsigned Opc) { 295 const SMInfo *Info = getSMEMOpcodeHelper(Opc); 296 return Info ? Info->IsBuffer : false; 297 } 298 299 bool getVOP1IsSingle(unsigned Opc) { 300 const VOPInfo *Info = getVOP1OpcodeHelper(Opc); 301 return Info ? Info->IsSingle : false; 302 } 303 304 bool getVOP2IsSingle(unsigned Opc) { 305 const VOPInfo *Info = getVOP2OpcodeHelper(Opc); 306 return Info ? Info->IsSingle : false; 307 } 308 309 bool getVOP3IsSingle(unsigned Opc) { 310 const VOPInfo *Info = getVOP3OpcodeHelper(Opc); 311 return Info ? Info->IsSingle : false; 312 } 313 314 // Wrapper for Tablegen'd function. enum Subtarget is not defined in any 315 // header files, so we need to wrap it in a function that takes unsigned 316 // instead. 317 int getMCOpcode(uint16_t Opcode, unsigned Gen) { 318 return getMCOpcodeGen(Opcode, static_cast<Subtarget>(Gen)); 319 } 320 321 namespace IsaInfo { 322 323 AMDGPUTargetID::AMDGPUTargetID(const MCSubtargetInfo &STI) 324 : STI(STI), XnackSetting(TargetIDSetting::Any), 325 SramEccSetting(TargetIDSetting::Any) { 326 if (!STI.getFeatureBits().test(FeatureSupportsXNACK)) 327 XnackSetting = TargetIDSetting::Unsupported; 328 if (!STI.getFeatureBits().test(FeatureSupportsSRAMECC)) 329 SramEccSetting = TargetIDSetting::Unsupported; 330 } 331 332 void AMDGPUTargetID::setTargetIDFromFeaturesString(StringRef FS) { 333 // Check if xnack or sramecc is explicitly enabled or disabled. In the 334 // absence of the target features we assume we must generate code that can run 335 // in any environment. 336 SubtargetFeatures Features(FS); 337 Optional<bool> XnackRequested; 338 Optional<bool> SramEccRequested; 339 340 for (const std::string &Feature : Features.getFeatures()) { 341 if (Feature == "+xnack") 342 XnackRequested = true; 343 else if (Feature == "-xnack") 344 XnackRequested = false; 345 else if (Feature == "+sramecc") 346 SramEccRequested = true; 347 else if (Feature == "-sramecc") 348 SramEccRequested = false; 349 } 350 351 bool XnackSupported = isXnackSupported(); 352 bool SramEccSupported = isSramEccSupported(); 353 354 if (XnackRequested) { 355 if (XnackSupported) { 356 XnackSetting = 357 *XnackRequested ? TargetIDSetting::On : TargetIDSetting::Off; 358 } else { 359 // If a specific xnack setting was requested and this GPU does not support 360 // xnack emit a warning. Setting will remain set to "Unsupported". 361 if (*XnackRequested) { 362 errs() << "warning: xnack 'On' was requested for a processor that does " 363 "not support it!\n"; 364 } else { 365 errs() << "warning: xnack 'Off' was requested for a processor that " 366 "does not support it!\n"; 367 } 368 } 369 } 370 371 if (SramEccRequested) { 372 if (SramEccSupported) { 373 SramEccSetting = 374 *SramEccRequested ? TargetIDSetting::On : TargetIDSetting::Off; 375 } else { 376 // If a specific sramecc setting was requested and this GPU does not 377 // support sramecc emit a warning. Setting will remain set to 378 // "Unsupported". 379 if (*SramEccRequested) { 380 errs() << "warning: sramecc 'On' was requested for a processor that " 381 "does not support it!\n"; 382 } else { 383 errs() << "warning: sramecc 'Off' was requested for a processor that " 384 "does not support it!\n"; 385 } 386 } 387 } 388 } 389 390 static TargetIDSetting 391 getTargetIDSettingFromFeatureString(StringRef FeatureString) { 392 if (FeatureString.endswith("-")) 393 return TargetIDSetting::Off; 394 if (FeatureString.endswith("+")) 395 return TargetIDSetting::On; 396 397 llvm_unreachable("Malformed feature string"); 398 } 399 400 void AMDGPUTargetID::setTargetIDFromTargetIDStream(StringRef TargetID) { 401 SmallVector<StringRef, 3> TargetIDSplit; 402 TargetID.split(TargetIDSplit, ':'); 403 404 for (const auto &FeatureString : TargetIDSplit) { 405 if (FeatureString.startswith("xnack")) 406 XnackSetting = getTargetIDSettingFromFeatureString(FeatureString); 407 if (FeatureString.startswith("sramecc")) 408 SramEccSetting = getTargetIDSettingFromFeatureString(FeatureString); 409 } 410 } 411 412 std::string AMDGPUTargetID::toString() const { 413 std::string StringRep; 414 raw_string_ostream StreamRep(StringRep); 415 416 auto TargetTriple = STI.getTargetTriple(); 417 auto Version = getIsaVersion(STI.getCPU()); 418 419 StreamRep << TargetTriple.getArchName() << '-' 420 << TargetTriple.getVendorName() << '-' 421 << TargetTriple.getOSName() << '-' 422 << TargetTriple.getEnvironmentName() << '-'; 423 424 std::string Processor; 425 // TODO: Following else statement is present here because we used various 426 // alias names for GPUs up until GFX9 (e.g. 'fiji' is same as 'gfx803'). 427 // Remove once all aliases are removed from GCNProcessors.td. 428 if (Version.Major >= 9) 429 Processor = STI.getCPU().str(); 430 else 431 Processor = (Twine("gfx") + Twine(Version.Major) + Twine(Version.Minor) + 432 Twine(Version.Stepping)) 433 .str(); 434 435 std::string Features; 436 if (Optional<uint8_t> HsaAbiVersion = getHsaAbiVersion(&STI)) { 437 switch (*HsaAbiVersion) { 438 case ELF::ELFABIVERSION_AMDGPU_HSA_V2: 439 // Code object V2 only supported specific processors and had fixed 440 // settings for the XNACK. 441 if (Processor == "gfx600") { 442 } else if (Processor == "gfx601") { 443 } else if (Processor == "gfx602") { 444 } else if (Processor == "gfx700") { 445 } else if (Processor == "gfx701") { 446 } else if (Processor == "gfx702") { 447 } else if (Processor == "gfx703") { 448 } else if (Processor == "gfx704") { 449 } else if (Processor == "gfx705") { 450 } else if (Processor == "gfx801") { 451 if (!isXnackOnOrAny()) 452 report_fatal_error( 453 "AMD GPU code object V2 does not support processor " + 454 Twine(Processor) + " without XNACK"); 455 } else if (Processor == "gfx802") { 456 } else if (Processor == "gfx803") { 457 } else if (Processor == "gfx805") { 458 } else if (Processor == "gfx810") { 459 if (!isXnackOnOrAny()) 460 report_fatal_error( 461 "AMD GPU code object V2 does not support processor " + 462 Twine(Processor) + " without XNACK"); 463 } else if (Processor == "gfx900") { 464 if (isXnackOnOrAny()) 465 Processor = "gfx901"; 466 } else if (Processor == "gfx902") { 467 if (isXnackOnOrAny()) 468 Processor = "gfx903"; 469 } else if (Processor == "gfx904") { 470 if (isXnackOnOrAny()) 471 Processor = "gfx905"; 472 } else if (Processor == "gfx906") { 473 if (isXnackOnOrAny()) 474 Processor = "gfx907"; 475 } else if (Processor == "gfx90c") { 476 if (isXnackOnOrAny()) 477 report_fatal_error( 478 "AMD GPU code object V2 does not support processor " + 479 Twine(Processor) + " with XNACK being ON or ANY"); 480 } else { 481 report_fatal_error( 482 "AMD GPU code object V2 does not support processor " + 483 Twine(Processor)); 484 } 485 break; 486 case ELF::ELFABIVERSION_AMDGPU_HSA_V3: 487 // xnack. 488 if (isXnackOnOrAny()) 489 Features += "+xnack"; 490 // In code object v2 and v3, "sramecc" feature was spelled with a 491 // hyphen ("sram-ecc"). 492 if (isSramEccOnOrAny()) 493 Features += "+sram-ecc"; 494 break; 495 case ELF::ELFABIVERSION_AMDGPU_HSA_V4: 496 // sramecc. 497 if (getSramEccSetting() == TargetIDSetting::Off) 498 Features += ":sramecc-"; 499 else if (getSramEccSetting() == TargetIDSetting::On) 500 Features += ":sramecc+"; 501 // xnack. 502 if (getXnackSetting() == TargetIDSetting::Off) 503 Features += ":xnack-"; 504 else if (getXnackSetting() == TargetIDSetting::On) 505 Features += ":xnack+"; 506 break; 507 default: 508 break; 509 } 510 } 511 512 StreamRep << Processor << Features; 513 514 StreamRep.flush(); 515 return StringRep; 516 } 517 518 unsigned getWavefrontSize(const MCSubtargetInfo *STI) { 519 if (STI->getFeatureBits().test(FeatureWavefrontSize16)) 520 return 16; 521 if (STI->getFeatureBits().test(FeatureWavefrontSize32)) 522 return 32; 523 524 return 64; 525 } 526 527 unsigned getLocalMemorySize(const MCSubtargetInfo *STI) { 528 if (STI->getFeatureBits().test(FeatureLocalMemorySize32768)) 529 return 32768; 530 if (STI->getFeatureBits().test(FeatureLocalMemorySize65536)) 531 return 65536; 532 533 return 0; 534 } 535 536 unsigned getEUsPerCU(const MCSubtargetInfo *STI) { 537 // "Per CU" really means "per whatever functional block the waves of a 538 // workgroup must share". For gfx10 in CU mode this is the CU, which contains 539 // two SIMDs. 540 if (isGFX10Plus(*STI) && STI->getFeatureBits().test(FeatureCuMode)) 541 return 2; 542 // Pre-gfx10 a CU contains four SIMDs. For gfx10 in WGP mode the WGP contains 543 // two CUs, so a total of four SIMDs. 544 return 4; 545 } 546 547 unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo *STI, 548 unsigned FlatWorkGroupSize) { 549 assert(FlatWorkGroupSize != 0); 550 if (STI->getTargetTriple().getArch() != Triple::amdgcn) 551 return 8; 552 unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize); 553 if (N == 1) 554 return 40; 555 N = 40 / N; 556 return std::min(N, 16u); 557 } 558 559 unsigned getMinWavesPerEU(const MCSubtargetInfo *STI) { 560 return 1; 561 } 562 563 unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI) { 564 // FIXME: Need to take scratch memory into account. 565 if (isGFX90A(*STI)) 566 return 8; 567 if (!isGFX10Plus(*STI)) 568 return 10; 569 return hasGFX10_3Insts(*STI) ? 16 : 20; 570 } 571 572 unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo *STI, 573 unsigned FlatWorkGroupSize) { 574 return divideCeil(getWavesPerWorkGroup(STI, FlatWorkGroupSize), 575 getEUsPerCU(STI)); 576 } 577 578 unsigned getMinFlatWorkGroupSize(const MCSubtargetInfo *STI) { 579 return 1; 580 } 581 582 unsigned getMaxFlatWorkGroupSize(const MCSubtargetInfo *STI) { 583 // Some subtargets allow encoding 2048, but this isn't tested or supported. 584 return 1024; 585 } 586 587 unsigned getWavesPerWorkGroup(const MCSubtargetInfo *STI, 588 unsigned FlatWorkGroupSize) { 589 return divideCeil(FlatWorkGroupSize, getWavefrontSize(STI)); 590 } 591 592 unsigned getSGPRAllocGranule(const MCSubtargetInfo *STI) { 593 IsaVersion Version = getIsaVersion(STI->getCPU()); 594 if (Version.Major >= 10) 595 return getAddressableNumSGPRs(STI); 596 if (Version.Major >= 8) 597 return 16; 598 return 8; 599 } 600 601 unsigned getSGPREncodingGranule(const MCSubtargetInfo *STI) { 602 return 8; 603 } 604 605 unsigned getTotalNumSGPRs(const MCSubtargetInfo *STI) { 606 IsaVersion Version = getIsaVersion(STI->getCPU()); 607 if (Version.Major >= 8) 608 return 800; 609 return 512; 610 } 611 612 unsigned getAddressableNumSGPRs(const MCSubtargetInfo *STI) { 613 if (STI->getFeatureBits().test(FeatureSGPRInitBug)) 614 return FIXED_NUM_SGPRS_FOR_INIT_BUG; 615 616 IsaVersion Version = getIsaVersion(STI->getCPU()); 617 if (Version.Major >= 10) 618 return 106; 619 if (Version.Major >= 8) 620 return 102; 621 return 104; 622 } 623 624 unsigned getMinNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 625 assert(WavesPerEU != 0); 626 627 IsaVersion Version = getIsaVersion(STI->getCPU()); 628 if (Version.Major >= 10) 629 return 0; 630 631 if (WavesPerEU >= getMaxWavesPerEU(STI)) 632 return 0; 633 634 unsigned MinNumSGPRs = getTotalNumSGPRs(STI) / (WavesPerEU + 1); 635 if (STI->getFeatureBits().test(FeatureTrapHandler)) 636 MinNumSGPRs -= std::min(MinNumSGPRs, (unsigned)TRAP_NUM_SGPRS); 637 MinNumSGPRs = alignDown(MinNumSGPRs, getSGPRAllocGranule(STI)) + 1; 638 return std::min(MinNumSGPRs, getAddressableNumSGPRs(STI)); 639 } 640 641 unsigned getMaxNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU, 642 bool Addressable) { 643 assert(WavesPerEU != 0); 644 645 unsigned AddressableNumSGPRs = getAddressableNumSGPRs(STI); 646 IsaVersion Version = getIsaVersion(STI->getCPU()); 647 if (Version.Major >= 10) 648 return Addressable ? AddressableNumSGPRs : 108; 649 if (Version.Major >= 8 && !Addressable) 650 AddressableNumSGPRs = 112; 651 unsigned MaxNumSGPRs = getTotalNumSGPRs(STI) / WavesPerEU; 652 if (STI->getFeatureBits().test(FeatureTrapHandler)) 653 MaxNumSGPRs -= std::min(MaxNumSGPRs, (unsigned)TRAP_NUM_SGPRS); 654 MaxNumSGPRs = alignDown(MaxNumSGPRs, getSGPRAllocGranule(STI)); 655 return std::min(MaxNumSGPRs, AddressableNumSGPRs); 656 } 657 658 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed, 659 bool FlatScrUsed, bool XNACKUsed) { 660 unsigned ExtraSGPRs = 0; 661 if (VCCUsed) 662 ExtraSGPRs = 2; 663 664 IsaVersion Version = getIsaVersion(STI->getCPU()); 665 if (Version.Major >= 10) 666 return ExtraSGPRs; 667 668 if (Version.Major < 8) { 669 if (FlatScrUsed) 670 ExtraSGPRs = 4; 671 } else { 672 if (XNACKUsed) 673 ExtraSGPRs = 4; 674 675 if (FlatScrUsed || 676 STI->getFeatureBits().test(AMDGPU::FeatureArchitectedFlatScratch)) 677 ExtraSGPRs = 6; 678 } 679 680 return ExtraSGPRs; 681 } 682 683 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed, 684 bool FlatScrUsed) { 685 return getNumExtraSGPRs(STI, VCCUsed, FlatScrUsed, 686 STI->getFeatureBits().test(AMDGPU::FeatureXNACK)); 687 } 688 689 unsigned getNumSGPRBlocks(const MCSubtargetInfo *STI, unsigned NumSGPRs) { 690 NumSGPRs = alignTo(std::max(1u, NumSGPRs), getSGPREncodingGranule(STI)); 691 // SGPRBlocks is actual number of SGPR blocks minus 1. 692 return NumSGPRs / getSGPREncodingGranule(STI) - 1; 693 } 694 695 unsigned getVGPRAllocGranule(const MCSubtargetInfo *STI, 696 Optional<bool> EnableWavefrontSize32) { 697 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 698 return 8; 699 700 bool IsWave32 = EnableWavefrontSize32 ? 701 *EnableWavefrontSize32 : 702 STI->getFeatureBits().test(FeatureWavefrontSize32); 703 704 if (hasGFX10_3Insts(*STI)) 705 return IsWave32 ? 16 : 8; 706 707 return IsWave32 ? 8 : 4; 708 } 709 710 unsigned getVGPREncodingGranule(const MCSubtargetInfo *STI, 711 Optional<bool> EnableWavefrontSize32) { 712 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 713 return 8; 714 715 bool IsWave32 = EnableWavefrontSize32 ? 716 *EnableWavefrontSize32 : 717 STI->getFeatureBits().test(FeatureWavefrontSize32); 718 719 return IsWave32 ? 8 : 4; 720 } 721 722 unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) { 723 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 724 return 512; 725 if (!isGFX10Plus(*STI)) 726 return 256; 727 return STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1024 : 512; 728 } 729 730 unsigned getAddressableNumVGPRs(const MCSubtargetInfo *STI) { 731 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 732 return 512; 733 return 256; 734 } 735 736 unsigned getMinNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 737 assert(WavesPerEU != 0); 738 739 if (WavesPerEU >= getMaxWavesPerEU(STI)) 740 return 0; 741 unsigned MinNumVGPRs = 742 alignDown(getTotalNumVGPRs(STI) / (WavesPerEU + 1), 743 getVGPRAllocGranule(STI)) + 1; 744 return std::min(MinNumVGPRs, getAddressableNumVGPRs(STI)); 745 } 746 747 unsigned getMaxNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 748 assert(WavesPerEU != 0); 749 750 unsigned MaxNumVGPRs = alignDown(getTotalNumVGPRs(STI) / WavesPerEU, 751 getVGPRAllocGranule(STI)); 752 unsigned AddressableNumVGPRs = getAddressableNumVGPRs(STI); 753 return std::min(MaxNumVGPRs, AddressableNumVGPRs); 754 } 755 756 unsigned getNumVGPRBlocks(const MCSubtargetInfo *STI, unsigned NumVGPRs, 757 Optional<bool> EnableWavefrontSize32) { 758 NumVGPRs = alignTo(std::max(1u, NumVGPRs), 759 getVGPREncodingGranule(STI, EnableWavefrontSize32)); 760 // VGPRBlocks is actual number of VGPR blocks minus 1. 761 return NumVGPRs / getVGPREncodingGranule(STI, EnableWavefrontSize32) - 1; 762 } 763 764 } // end namespace IsaInfo 765 766 void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header, 767 const MCSubtargetInfo *STI) { 768 IsaVersion Version = getIsaVersion(STI->getCPU()); 769 770 memset(&Header, 0, sizeof(Header)); 771 772 Header.amd_kernel_code_version_major = 1; 773 Header.amd_kernel_code_version_minor = 2; 774 Header.amd_machine_kind = 1; // AMD_MACHINE_KIND_AMDGPU 775 Header.amd_machine_version_major = Version.Major; 776 Header.amd_machine_version_minor = Version.Minor; 777 Header.amd_machine_version_stepping = Version.Stepping; 778 Header.kernel_code_entry_byte_offset = sizeof(Header); 779 Header.wavefront_size = 6; 780 781 // If the code object does not support indirect functions, then the value must 782 // be 0xffffffff. 783 Header.call_convention = -1; 784 785 // These alignment values are specified in powers of two, so alignment = 786 // 2^n. The minimum alignment is 2^4 = 16. 787 Header.kernarg_segment_alignment = 4; 788 Header.group_segment_alignment = 4; 789 Header.private_segment_alignment = 4; 790 791 if (Version.Major >= 10) { 792 if (STI->getFeatureBits().test(FeatureWavefrontSize32)) { 793 Header.wavefront_size = 5; 794 Header.code_properties |= AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32; 795 } 796 Header.compute_pgm_resource_registers |= 797 S_00B848_WGP_MODE(STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1) | 798 S_00B848_MEM_ORDERED(1); 799 } 800 } 801 802 amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor( 803 const MCSubtargetInfo *STI) { 804 IsaVersion Version = getIsaVersion(STI->getCPU()); 805 806 amdhsa::kernel_descriptor_t KD; 807 memset(&KD, 0, sizeof(KD)); 808 809 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 810 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, 811 amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE); 812 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 813 amdhsa::COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP, 1); 814 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 815 amdhsa::COMPUTE_PGM_RSRC1_ENABLE_IEEE_MODE, 1); 816 AMDHSA_BITS_SET(KD.compute_pgm_rsrc2, 817 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, 1); 818 if (Version.Major >= 10) { 819 AMDHSA_BITS_SET(KD.kernel_code_properties, 820 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, 821 STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1 : 0); 822 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 823 amdhsa::COMPUTE_PGM_RSRC1_WGP_MODE, 824 STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1); 825 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 826 amdhsa::COMPUTE_PGM_RSRC1_MEM_ORDERED, 1); 827 } 828 if (AMDGPU::isGFX90A(*STI)) { 829 AMDHSA_BITS_SET(KD.compute_pgm_rsrc3, 830 amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, 831 STI->getFeatureBits().test(FeatureTgSplit) ? 1 : 0); 832 } 833 return KD; 834 } 835 836 bool isGroupSegment(const GlobalValue *GV) { 837 return GV->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS; 838 } 839 840 bool isGlobalSegment(const GlobalValue *GV) { 841 return GV->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS; 842 } 843 844 bool isReadOnlySegment(const GlobalValue *GV) { 845 unsigned AS = GV->getAddressSpace(); 846 return AS == AMDGPUAS::CONSTANT_ADDRESS || 847 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT; 848 } 849 850 bool shouldEmitConstantsToTextSection(const Triple &TT) { 851 return TT.getArch() == Triple::r600; 852 } 853 854 int getIntegerAttribute(const Function &F, StringRef Name, int Default) { 855 Attribute A = F.getFnAttribute(Name); 856 int Result = Default; 857 858 if (A.isStringAttribute()) { 859 StringRef Str = A.getValueAsString(); 860 if (Str.getAsInteger(0, Result)) { 861 LLVMContext &Ctx = F.getContext(); 862 Ctx.emitError("can't parse integer attribute " + Name); 863 } 864 } 865 866 return Result; 867 } 868 869 std::pair<int, int> getIntegerPairAttribute(const Function &F, 870 StringRef Name, 871 std::pair<int, int> Default, 872 bool OnlyFirstRequired) { 873 Attribute A = F.getFnAttribute(Name); 874 if (!A.isStringAttribute()) 875 return Default; 876 877 LLVMContext &Ctx = F.getContext(); 878 std::pair<int, int> Ints = Default; 879 std::pair<StringRef, StringRef> Strs = A.getValueAsString().split(','); 880 if (Strs.first.trim().getAsInteger(0, Ints.first)) { 881 Ctx.emitError("can't parse first integer attribute " + Name); 882 return Default; 883 } 884 if (Strs.second.trim().getAsInteger(0, Ints.second)) { 885 if (!OnlyFirstRequired || !Strs.second.trim().empty()) { 886 Ctx.emitError("can't parse second integer attribute " + Name); 887 return Default; 888 } 889 } 890 891 return Ints; 892 } 893 894 unsigned getVmcntBitMask(const IsaVersion &Version) { 895 unsigned VmcntLo = (1 << getVmcntBitWidthLo()) - 1; 896 if (Version.Major < 9) 897 return VmcntLo; 898 899 unsigned VmcntHi = ((1 << getVmcntBitWidthHi()) - 1) << getVmcntBitWidthLo(); 900 return VmcntLo | VmcntHi; 901 } 902 903 unsigned getExpcntBitMask(const IsaVersion &Version) { 904 return (1 << getExpcntBitWidth()) - 1; 905 } 906 907 unsigned getLgkmcntBitMask(const IsaVersion &Version) { 908 return (1 << getLgkmcntBitWidth(Version.Major)) - 1; 909 } 910 911 unsigned getWaitcntBitMask(const IsaVersion &Version) { 912 unsigned VmcntLo = getBitMask(getVmcntBitShiftLo(), getVmcntBitWidthLo()); 913 unsigned Expcnt = getBitMask(getExpcntBitShift(), getExpcntBitWidth()); 914 unsigned Lgkmcnt = getBitMask(getLgkmcntBitShift(), 915 getLgkmcntBitWidth(Version.Major)); 916 unsigned Waitcnt = VmcntLo | Expcnt | Lgkmcnt; 917 if (Version.Major < 9) 918 return Waitcnt; 919 920 unsigned VmcntHi = getBitMask(getVmcntBitShiftHi(), getVmcntBitWidthHi()); 921 return Waitcnt | VmcntHi; 922 } 923 924 unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) { 925 unsigned VmcntLo = 926 unpackBits(Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo()); 927 if (Version.Major < 9) 928 return VmcntLo; 929 930 unsigned VmcntHi = 931 unpackBits(Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi()); 932 VmcntHi <<= getVmcntBitWidthLo(); 933 return VmcntLo | VmcntHi; 934 } 935 936 unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) { 937 return unpackBits(Waitcnt, getExpcntBitShift(), getExpcntBitWidth()); 938 } 939 940 unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) { 941 return unpackBits(Waitcnt, getLgkmcntBitShift(), 942 getLgkmcntBitWidth(Version.Major)); 943 } 944 945 void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, 946 unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt) { 947 Vmcnt = decodeVmcnt(Version, Waitcnt); 948 Expcnt = decodeExpcnt(Version, Waitcnt); 949 Lgkmcnt = decodeLgkmcnt(Version, Waitcnt); 950 } 951 952 Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded) { 953 Waitcnt Decoded; 954 Decoded.VmCnt = decodeVmcnt(Version, Encoded); 955 Decoded.ExpCnt = decodeExpcnt(Version, Encoded); 956 Decoded.LgkmCnt = decodeLgkmcnt(Version, Encoded); 957 return Decoded; 958 } 959 960 unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt, 961 unsigned Vmcnt) { 962 Waitcnt = 963 packBits(Vmcnt, Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo()); 964 if (Version.Major < 9) 965 return Waitcnt; 966 967 Vmcnt >>= getVmcntBitWidthLo(); 968 return packBits(Vmcnt, Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi()); 969 } 970 971 unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt, 972 unsigned Expcnt) { 973 return packBits(Expcnt, Waitcnt, getExpcntBitShift(), getExpcntBitWidth()); 974 } 975 976 unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt, 977 unsigned Lgkmcnt) { 978 return packBits(Lgkmcnt, Waitcnt, getLgkmcntBitShift(), 979 getLgkmcntBitWidth(Version.Major)); 980 } 981 982 unsigned encodeWaitcnt(const IsaVersion &Version, 983 unsigned Vmcnt, unsigned Expcnt, unsigned Lgkmcnt) { 984 unsigned Waitcnt = getWaitcntBitMask(Version); 985 Waitcnt = encodeVmcnt(Version, Waitcnt, Vmcnt); 986 Waitcnt = encodeExpcnt(Version, Waitcnt, Expcnt); 987 Waitcnt = encodeLgkmcnt(Version, Waitcnt, Lgkmcnt); 988 return Waitcnt; 989 } 990 991 unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded) { 992 return encodeWaitcnt(Version, Decoded.VmCnt, Decoded.ExpCnt, Decoded.LgkmCnt); 993 } 994 995 //===----------------------------------------------------------------------===// 996 // hwreg 997 //===----------------------------------------------------------------------===// 998 999 namespace Hwreg { 1000 1001 int64_t getHwregId(const StringRef Name) { 1002 for (int Id = ID_SYMBOLIC_FIRST_; Id < ID_SYMBOLIC_LAST_; ++Id) { 1003 if (IdSymbolic[Id] && Name == IdSymbolic[Id]) 1004 return Id; 1005 } 1006 return ID_UNKNOWN_; 1007 } 1008 1009 static unsigned getLastSymbolicHwreg(const MCSubtargetInfo &STI) { 1010 if (isSI(STI) || isCI(STI) || isVI(STI)) 1011 return ID_SYMBOLIC_FIRST_GFX9_; 1012 else if (isGFX9(STI)) 1013 return ID_SYMBOLIC_FIRST_GFX10_; 1014 else if (isGFX10(STI) && !isGFX10_BEncoding(STI)) 1015 return ID_SYMBOLIC_FIRST_GFX1030_; 1016 else 1017 return ID_SYMBOLIC_LAST_; 1018 } 1019 1020 bool isValidHwreg(int64_t Id, const MCSubtargetInfo &STI) { 1021 switch (Id) { 1022 case ID_HW_ID: 1023 return isSI(STI) || isCI(STI) || isVI(STI) || isGFX9(STI); 1024 case ID_HW_ID1: 1025 case ID_HW_ID2: 1026 return isGFX10Plus(STI); 1027 case ID_XNACK_MASK: 1028 return isGFX10(STI) && !AMDGPU::isGFX10_BEncoding(STI); 1029 default: 1030 return ID_SYMBOLIC_FIRST_ <= Id && Id < getLastSymbolicHwreg(STI) && 1031 IdSymbolic[Id]; 1032 } 1033 } 1034 1035 bool isValidHwreg(int64_t Id) { 1036 return 0 <= Id && isUInt<ID_WIDTH_>(Id); 1037 } 1038 1039 bool isValidHwregOffset(int64_t Offset) { 1040 return 0 <= Offset && isUInt<OFFSET_WIDTH_>(Offset); 1041 } 1042 1043 bool isValidHwregWidth(int64_t Width) { 1044 return 0 <= (Width - 1) && isUInt<WIDTH_M1_WIDTH_>(Width - 1); 1045 } 1046 1047 uint64_t encodeHwreg(uint64_t Id, uint64_t Offset, uint64_t Width) { 1048 return (Id << ID_SHIFT_) | 1049 (Offset << OFFSET_SHIFT_) | 1050 ((Width - 1) << WIDTH_M1_SHIFT_); 1051 } 1052 1053 StringRef getHwreg(unsigned Id, const MCSubtargetInfo &STI) { 1054 return isValidHwreg(Id, STI) ? IdSymbolic[Id] : ""; 1055 } 1056 1057 void decodeHwreg(unsigned Val, unsigned &Id, unsigned &Offset, unsigned &Width) { 1058 Id = (Val & ID_MASK_) >> ID_SHIFT_; 1059 Offset = (Val & OFFSET_MASK_) >> OFFSET_SHIFT_; 1060 Width = ((Val & WIDTH_M1_MASK_) >> WIDTH_M1_SHIFT_) + 1; 1061 } 1062 1063 } // namespace Hwreg 1064 1065 //===----------------------------------------------------------------------===// 1066 // exp tgt 1067 //===----------------------------------------------------------------------===// 1068 1069 namespace Exp { 1070 1071 struct ExpTgt { 1072 StringLiteral Name; 1073 unsigned Tgt; 1074 unsigned MaxIndex; 1075 }; 1076 1077 static constexpr ExpTgt ExpTgtInfo[] = { 1078 {{"null"}, ET_NULL, ET_NULL_MAX_IDX}, 1079 {{"mrtz"}, ET_MRTZ, ET_MRTZ_MAX_IDX}, 1080 {{"prim"}, ET_PRIM, ET_PRIM_MAX_IDX}, 1081 {{"mrt"}, ET_MRT0, ET_MRT_MAX_IDX}, 1082 {{"pos"}, ET_POS0, ET_POS_MAX_IDX}, 1083 {{"param"}, ET_PARAM0, ET_PARAM_MAX_IDX}, 1084 }; 1085 1086 bool getTgtName(unsigned Id, StringRef &Name, int &Index) { 1087 for (const ExpTgt &Val : ExpTgtInfo) { 1088 if (Val.Tgt <= Id && Id <= Val.Tgt + Val.MaxIndex) { 1089 Index = (Val.MaxIndex == 0) ? -1 : (Id - Val.Tgt); 1090 Name = Val.Name; 1091 return true; 1092 } 1093 } 1094 return false; 1095 } 1096 1097 unsigned getTgtId(const StringRef Name) { 1098 1099 for (const ExpTgt &Val : ExpTgtInfo) { 1100 if (Val.MaxIndex == 0 && Name == Val.Name) 1101 return Val.Tgt; 1102 1103 if (Val.MaxIndex > 0 && Name.startswith(Val.Name)) { 1104 StringRef Suffix = Name.drop_front(Val.Name.size()); 1105 1106 unsigned Id; 1107 if (Suffix.getAsInteger(10, Id) || Id > Val.MaxIndex) 1108 return ET_INVALID; 1109 1110 // Disable leading zeroes 1111 if (Suffix.size() > 1 && Suffix[0] == '0') 1112 return ET_INVALID; 1113 1114 return Val.Tgt + Id; 1115 } 1116 } 1117 return ET_INVALID; 1118 } 1119 1120 bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI) { 1121 return (Id != ET_POS4 && Id != ET_PRIM) || isGFX10Plus(STI); 1122 } 1123 1124 } // namespace Exp 1125 1126 //===----------------------------------------------------------------------===// 1127 // MTBUF Format 1128 //===----------------------------------------------------------------------===// 1129 1130 namespace MTBUFFormat { 1131 1132 int64_t getDfmt(const StringRef Name) { 1133 for (int Id = DFMT_MIN; Id <= DFMT_MAX; ++Id) { 1134 if (Name == DfmtSymbolic[Id]) 1135 return Id; 1136 } 1137 return DFMT_UNDEF; 1138 } 1139 1140 StringRef getDfmtName(unsigned Id) { 1141 assert(Id <= DFMT_MAX); 1142 return DfmtSymbolic[Id]; 1143 } 1144 1145 static StringLiteral const *getNfmtLookupTable(const MCSubtargetInfo &STI) { 1146 if (isSI(STI) || isCI(STI)) 1147 return NfmtSymbolicSICI; 1148 if (isVI(STI) || isGFX9(STI)) 1149 return NfmtSymbolicVI; 1150 return NfmtSymbolicGFX10; 1151 } 1152 1153 int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI) { 1154 auto lookupTable = getNfmtLookupTable(STI); 1155 for (int Id = NFMT_MIN; Id <= NFMT_MAX; ++Id) { 1156 if (Name == lookupTable[Id]) 1157 return Id; 1158 } 1159 return NFMT_UNDEF; 1160 } 1161 1162 StringRef getNfmtName(unsigned Id, const MCSubtargetInfo &STI) { 1163 assert(Id <= NFMT_MAX); 1164 return getNfmtLookupTable(STI)[Id]; 1165 } 1166 1167 bool isValidDfmtNfmt(unsigned Id, const MCSubtargetInfo &STI) { 1168 unsigned Dfmt; 1169 unsigned Nfmt; 1170 decodeDfmtNfmt(Id, Dfmt, Nfmt); 1171 return isValidNfmt(Nfmt, STI); 1172 } 1173 1174 bool isValidNfmt(unsigned Id, const MCSubtargetInfo &STI) { 1175 return !getNfmtName(Id, STI).empty(); 1176 } 1177 1178 int64_t encodeDfmtNfmt(unsigned Dfmt, unsigned Nfmt) { 1179 return (Dfmt << DFMT_SHIFT) | (Nfmt << NFMT_SHIFT); 1180 } 1181 1182 void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt) { 1183 Dfmt = (Format >> DFMT_SHIFT) & DFMT_MASK; 1184 Nfmt = (Format >> NFMT_SHIFT) & NFMT_MASK; 1185 } 1186 1187 int64_t getUnifiedFormat(const StringRef Name) { 1188 for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) { 1189 if (Name == UfmtSymbolic[Id]) 1190 return Id; 1191 } 1192 return UFMT_UNDEF; 1193 } 1194 1195 StringRef getUnifiedFormatName(unsigned Id) { 1196 return isValidUnifiedFormat(Id) ? UfmtSymbolic[Id] : ""; 1197 } 1198 1199 bool isValidUnifiedFormat(unsigned Id) { 1200 return Id <= UFMT_LAST; 1201 } 1202 1203 int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt) { 1204 int64_t Fmt = encodeDfmtNfmt(Dfmt, Nfmt); 1205 for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) { 1206 if (Fmt == DfmtNfmt2UFmt[Id]) 1207 return Id; 1208 } 1209 return UFMT_UNDEF; 1210 } 1211 1212 bool isValidFormatEncoding(unsigned Val, const MCSubtargetInfo &STI) { 1213 return isGFX10Plus(STI) ? (Val <= UFMT_MAX) : (Val <= DFMT_NFMT_MAX); 1214 } 1215 1216 unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI) { 1217 if (isGFX10Plus(STI)) 1218 return UFMT_DEFAULT; 1219 return DFMT_NFMT_DEFAULT; 1220 } 1221 1222 } // namespace MTBUFFormat 1223 1224 //===----------------------------------------------------------------------===// 1225 // SendMsg 1226 //===----------------------------------------------------------------------===// 1227 1228 namespace SendMsg { 1229 1230 int64_t getMsgId(const StringRef Name) { 1231 for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) { 1232 if (IdSymbolic[i] && Name == IdSymbolic[i]) 1233 return i; 1234 } 1235 return ID_UNKNOWN_; 1236 } 1237 1238 bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI, bool Strict) { 1239 if (Strict) { 1240 switch (MsgId) { 1241 case ID_SAVEWAVE: 1242 return isVI(STI) || isGFX9Plus(STI); 1243 case ID_STALL_WAVE_GEN: 1244 case ID_HALT_WAVES: 1245 case ID_ORDERED_PS_DONE: 1246 case ID_GS_ALLOC_REQ: 1247 case ID_GET_DOORBELL: 1248 return isGFX9Plus(STI); 1249 case ID_EARLY_PRIM_DEALLOC: 1250 return isGFX9(STI); 1251 case ID_GET_DDID: 1252 return isGFX10Plus(STI); 1253 default: 1254 return 0 <= MsgId && MsgId < ID_GAPS_LAST_ && IdSymbolic[MsgId]; 1255 } 1256 } else { 1257 return 0 <= MsgId && isUInt<ID_WIDTH_>(MsgId); 1258 } 1259 } 1260 1261 StringRef getMsgName(int64_t MsgId) { 1262 assert(0 <= MsgId && MsgId < ID_GAPS_LAST_); 1263 return IdSymbolic[MsgId]; 1264 } 1265 1266 int64_t getMsgOpId(int64_t MsgId, const StringRef Name) { 1267 const char* const *S = (MsgId == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic; 1268 const int F = (MsgId == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_; 1269 const int L = (MsgId == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_; 1270 for (int i = F; i < L; ++i) { 1271 if (Name == S[i]) { 1272 return i; 1273 } 1274 } 1275 return OP_UNKNOWN_; 1276 } 1277 1278 bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI, 1279 bool Strict) { 1280 assert(isValidMsgId(MsgId, STI, Strict)); 1281 1282 if (!Strict) 1283 return 0 <= OpId && isUInt<OP_WIDTH_>(OpId); 1284 1285 switch(MsgId) 1286 { 1287 case ID_GS: 1288 return (OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_) && OpId != OP_GS_NOP; 1289 case ID_GS_DONE: 1290 return OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_; 1291 case ID_SYSMSG: 1292 return OP_SYS_FIRST_ <= OpId && OpId < OP_SYS_LAST_; 1293 default: 1294 return OpId == OP_NONE_; 1295 } 1296 } 1297 1298 StringRef getMsgOpName(int64_t MsgId, int64_t OpId) { 1299 assert(msgRequiresOp(MsgId)); 1300 return (MsgId == ID_SYSMSG)? OpSysSymbolic[OpId] : OpGsSymbolic[OpId]; 1301 } 1302 1303 bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId, 1304 const MCSubtargetInfo &STI, bool Strict) { 1305 assert(isValidMsgOp(MsgId, OpId, STI, Strict)); 1306 1307 if (!Strict) 1308 return 0 <= StreamId && isUInt<STREAM_ID_WIDTH_>(StreamId); 1309 1310 switch(MsgId) 1311 { 1312 case ID_GS: 1313 return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_; 1314 case ID_GS_DONE: 1315 return (OpId == OP_GS_NOP)? 1316 (StreamId == STREAM_ID_NONE_) : 1317 (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_); 1318 default: 1319 return StreamId == STREAM_ID_NONE_; 1320 } 1321 } 1322 1323 bool msgRequiresOp(int64_t MsgId) { 1324 return MsgId == ID_GS || MsgId == ID_GS_DONE || MsgId == ID_SYSMSG; 1325 } 1326 1327 bool msgSupportsStream(int64_t MsgId, int64_t OpId) { 1328 return (MsgId == ID_GS || MsgId == ID_GS_DONE) && OpId != OP_GS_NOP; 1329 } 1330 1331 void decodeMsg(unsigned Val, 1332 uint16_t &MsgId, 1333 uint16_t &OpId, 1334 uint16_t &StreamId) { 1335 MsgId = Val & ID_MASK_; 1336 OpId = (Val & OP_MASK_) >> OP_SHIFT_; 1337 StreamId = (Val & STREAM_ID_MASK_) >> STREAM_ID_SHIFT_; 1338 } 1339 1340 uint64_t encodeMsg(uint64_t MsgId, 1341 uint64_t OpId, 1342 uint64_t StreamId) { 1343 return (MsgId << ID_SHIFT_) | 1344 (OpId << OP_SHIFT_) | 1345 (StreamId << STREAM_ID_SHIFT_); 1346 } 1347 1348 } // namespace SendMsg 1349 1350 //===----------------------------------------------------------------------===// 1351 // 1352 //===----------------------------------------------------------------------===// 1353 1354 unsigned getInitialPSInputAddr(const Function &F) { 1355 return getIntegerAttribute(F, "InitialPSInputAddr", 0); 1356 } 1357 1358 bool getHasColorExport(const Function &F) { 1359 // As a safe default always respond as if PS has color exports. 1360 return getIntegerAttribute( 1361 F, "amdgpu-color-export", 1362 F.getCallingConv() == CallingConv::AMDGPU_PS ? 1 : 0) != 0; 1363 } 1364 1365 bool getHasDepthExport(const Function &F) { 1366 return getIntegerAttribute(F, "amdgpu-depth-export", 0) != 0; 1367 } 1368 1369 bool isShader(CallingConv::ID cc) { 1370 switch(cc) { 1371 case CallingConv::AMDGPU_VS: 1372 case CallingConv::AMDGPU_LS: 1373 case CallingConv::AMDGPU_HS: 1374 case CallingConv::AMDGPU_ES: 1375 case CallingConv::AMDGPU_GS: 1376 case CallingConv::AMDGPU_PS: 1377 case CallingConv::AMDGPU_CS: 1378 return true; 1379 default: 1380 return false; 1381 } 1382 } 1383 1384 bool isGraphics(CallingConv::ID cc) { 1385 return isShader(cc) || cc == CallingConv::AMDGPU_Gfx; 1386 } 1387 1388 bool isCompute(CallingConv::ID cc) { 1389 return !isGraphics(cc) || cc == CallingConv::AMDGPU_CS; 1390 } 1391 1392 bool isEntryFunctionCC(CallingConv::ID CC) { 1393 switch (CC) { 1394 case CallingConv::AMDGPU_KERNEL: 1395 case CallingConv::SPIR_KERNEL: 1396 case CallingConv::AMDGPU_VS: 1397 case CallingConv::AMDGPU_GS: 1398 case CallingConv::AMDGPU_PS: 1399 case CallingConv::AMDGPU_CS: 1400 case CallingConv::AMDGPU_ES: 1401 case CallingConv::AMDGPU_HS: 1402 case CallingConv::AMDGPU_LS: 1403 return true; 1404 default: 1405 return false; 1406 } 1407 } 1408 1409 bool isModuleEntryFunctionCC(CallingConv::ID CC) { 1410 switch (CC) { 1411 case CallingConv::AMDGPU_Gfx: 1412 return true; 1413 default: 1414 return isEntryFunctionCC(CC); 1415 } 1416 } 1417 1418 bool hasXNACK(const MCSubtargetInfo &STI) { 1419 return STI.getFeatureBits()[AMDGPU::FeatureXNACK]; 1420 } 1421 1422 bool hasSRAMECC(const MCSubtargetInfo &STI) { 1423 return STI.getFeatureBits()[AMDGPU::FeatureSRAMECC]; 1424 } 1425 1426 bool hasMIMG_R128(const MCSubtargetInfo &STI) { 1427 return STI.getFeatureBits()[AMDGPU::FeatureMIMG_R128] && !STI.getFeatureBits()[AMDGPU::FeatureR128A16]; 1428 } 1429 1430 bool hasGFX10A16(const MCSubtargetInfo &STI) { 1431 return STI.getFeatureBits()[AMDGPU::FeatureGFX10A16]; 1432 } 1433 1434 bool hasG16(const MCSubtargetInfo &STI) { 1435 return STI.getFeatureBits()[AMDGPU::FeatureG16]; 1436 } 1437 1438 bool hasPackedD16(const MCSubtargetInfo &STI) { 1439 return !STI.getFeatureBits()[AMDGPU::FeatureUnpackedD16VMem]; 1440 } 1441 1442 bool isSI(const MCSubtargetInfo &STI) { 1443 return STI.getFeatureBits()[AMDGPU::FeatureSouthernIslands]; 1444 } 1445 1446 bool isCI(const MCSubtargetInfo &STI) { 1447 return STI.getFeatureBits()[AMDGPU::FeatureSeaIslands]; 1448 } 1449 1450 bool isVI(const MCSubtargetInfo &STI) { 1451 return STI.getFeatureBits()[AMDGPU::FeatureVolcanicIslands]; 1452 } 1453 1454 bool isGFX9(const MCSubtargetInfo &STI) { 1455 return STI.getFeatureBits()[AMDGPU::FeatureGFX9]; 1456 } 1457 1458 bool isGFX9Plus(const MCSubtargetInfo &STI) { 1459 return isGFX9(STI) || isGFX10Plus(STI); 1460 } 1461 1462 bool isGFX10(const MCSubtargetInfo &STI) { 1463 return STI.getFeatureBits()[AMDGPU::FeatureGFX10]; 1464 } 1465 1466 bool isGFX10Plus(const MCSubtargetInfo &STI) { return isGFX10(STI); } 1467 1468 bool isGCN3Encoding(const MCSubtargetInfo &STI) { 1469 return STI.getFeatureBits()[AMDGPU::FeatureGCN3Encoding]; 1470 } 1471 1472 bool isGFX10_AEncoding(const MCSubtargetInfo &STI) { 1473 return STI.getFeatureBits()[AMDGPU::FeatureGFX10_AEncoding]; 1474 } 1475 1476 bool isGFX10_BEncoding(const MCSubtargetInfo &STI) { 1477 return STI.getFeatureBits()[AMDGPU::FeatureGFX10_BEncoding]; 1478 } 1479 1480 bool hasGFX10_3Insts(const MCSubtargetInfo &STI) { 1481 return STI.getFeatureBits()[AMDGPU::FeatureGFX10_3Insts]; 1482 } 1483 1484 bool isGFX90A(const MCSubtargetInfo &STI) { 1485 return STI.getFeatureBits()[AMDGPU::FeatureGFX90AInsts]; 1486 } 1487 1488 bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI) { 1489 return STI.getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch]; 1490 } 1491 1492 bool isSGPR(unsigned Reg, const MCRegisterInfo* TRI) { 1493 const MCRegisterClass SGPRClass = TRI->getRegClass(AMDGPU::SReg_32RegClassID); 1494 const unsigned FirstSubReg = TRI->getSubReg(Reg, AMDGPU::sub0); 1495 return SGPRClass.contains(FirstSubReg != 0 ? FirstSubReg : Reg) || 1496 Reg == AMDGPU::SCC; 1497 } 1498 1499 bool isRegIntersect(unsigned Reg0, unsigned Reg1, const MCRegisterInfo* TRI) { 1500 for (MCRegAliasIterator R(Reg0, TRI, true); R.isValid(); ++R) { 1501 if (*R == Reg1) return true; 1502 } 1503 return false; 1504 } 1505 1506 #define MAP_REG2REG \ 1507 using namespace AMDGPU; \ 1508 switch(Reg) { \ 1509 default: return Reg; \ 1510 CASE_CI_VI(FLAT_SCR) \ 1511 CASE_CI_VI(FLAT_SCR_LO) \ 1512 CASE_CI_VI(FLAT_SCR_HI) \ 1513 CASE_VI_GFX9PLUS(TTMP0) \ 1514 CASE_VI_GFX9PLUS(TTMP1) \ 1515 CASE_VI_GFX9PLUS(TTMP2) \ 1516 CASE_VI_GFX9PLUS(TTMP3) \ 1517 CASE_VI_GFX9PLUS(TTMP4) \ 1518 CASE_VI_GFX9PLUS(TTMP5) \ 1519 CASE_VI_GFX9PLUS(TTMP6) \ 1520 CASE_VI_GFX9PLUS(TTMP7) \ 1521 CASE_VI_GFX9PLUS(TTMP8) \ 1522 CASE_VI_GFX9PLUS(TTMP9) \ 1523 CASE_VI_GFX9PLUS(TTMP10) \ 1524 CASE_VI_GFX9PLUS(TTMP11) \ 1525 CASE_VI_GFX9PLUS(TTMP12) \ 1526 CASE_VI_GFX9PLUS(TTMP13) \ 1527 CASE_VI_GFX9PLUS(TTMP14) \ 1528 CASE_VI_GFX9PLUS(TTMP15) \ 1529 CASE_VI_GFX9PLUS(TTMP0_TTMP1) \ 1530 CASE_VI_GFX9PLUS(TTMP2_TTMP3) \ 1531 CASE_VI_GFX9PLUS(TTMP4_TTMP5) \ 1532 CASE_VI_GFX9PLUS(TTMP6_TTMP7) \ 1533 CASE_VI_GFX9PLUS(TTMP8_TTMP9) \ 1534 CASE_VI_GFX9PLUS(TTMP10_TTMP11) \ 1535 CASE_VI_GFX9PLUS(TTMP12_TTMP13) \ 1536 CASE_VI_GFX9PLUS(TTMP14_TTMP15) \ 1537 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3) \ 1538 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7) \ 1539 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11) \ 1540 CASE_VI_GFX9PLUS(TTMP12_TTMP13_TTMP14_TTMP15) \ 1541 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7) \ 1542 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11) \ 1543 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \ 1544 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \ 1545 } 1546 1547 #define CASE_CI_VI(node) \ 1548 assert(!isSI(STI)); \ 1549 case node: return isCI(STI) ? node##_ci : node##_vi; 1550 1551 #define CASE_VI_GFX9PLUS(node) \ 1552 case node: return isGFX9Plus(STI) ? node##_gfx9plus : node##_vi; 1553 1554 unsigned getMCReg(unsigned Reg, const MCSubtargetInfo &STI) { 1555 if (STI.getTargetTriple().getArch() == Triple::r600) 1556 return Reg; 1557 MAP_REG2REG 1558 } 1559 1560 #undef CASE_CI_VI 1561 #undef CASE_VI_GFX9PLUS 1562 1563 #define CASE_CI_VI(node) case node##_ci: case node##_vi: return node; 1564 #define CASE_VI_GFX9PLUS(node) case node##_vi: case node##_gfx9plus: return node; 1565 1566 unsigned mc2PseudoReg(unsigned Reg) { 1567 MAP_REG2REG 1568 } 1569 1570 #undef CASE_CI_VI 1571 #undef CASE_VI_GFX9PLUS 1572 #undef MAP_REG2REG 1573 1574 bool isSISrcOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1575 assert(OpNo < Desc.NumOperands); 1576 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1577 return OpType >= AMDGPU::OPERAND_SRC_FIRST && 1578 OpType <= AMDGPU::OPERAND_SRC_LAST; 1579 } 1580 1581 bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1582 assert(OpNo < Desc.NumOperands); 1583 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1584 switch (OpType) { 1585 case AMDGPU::OPERAND_REG_IMM_FP32: 1586 case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED: 1587 case AMDGPU::OPERAND_REG_IMM_FP64: 1588 case AMDGPU::OPERAND_REG_IMM_FP16: 1589 case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED: 1590 case AMDGPU::OPERAND_REG_IMM_V2FP16: 1591 case AMDGPU::OPERAND_REG_IMM_V2INT16: 1592 case AMDGPU::OPERAND_REG_INLINE_C_FP32: 1593 case AMDGPU::OPERAND_REG_INLINE_C_FP64: 1594 case AMDGPU::OPERAND_REG_INLINE_C_FP16: 1595 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: 1596 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: 1597 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: 1598 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: 1599 case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: 1600 case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16: 1601 case AMDGPU::OPERAND_REG_IMM_V2FP32: 1602 case AMDGPU::OPERAND_REG_INLINE_C_V2FP32: 1603 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: 1604 return true; 1605 default: 1606 return false; 1607 } 1608 } 1609 1610 bool isSISrcInlinableOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1611 assert(OpNo < Desc.NumOperands); 1612 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1613 return OpType >= AMDGPU::OPERAND_REG_INLINE_C_FIRST && 1614 OpType <= AMDGPU::OPERAND_REG_INLINE_C_LAST; 1615 } 1616 1617 // Avoid using MCRegisterClass::getSize, since that function will go away 1618 // (move from MC* level to Target* level). Return size in bits. 1619 unsigned getRegBitWidth(unsigned RCID) { 1620 switch (RCID) { 1621 case AMDGPU::VGPR_LO16RegClassID: 1622 case AMDGPU::VGPR_HI16RegClassID: 1623 case AMDGPU::SGPR_LO16RegClassID: 1624 case AMDGPU::AGPR_LO16RegClassID: 1625 return 16; 1626 case AMDGPU::SGPR_32RegClassID: 1627 case AMDGPU::VGPR_32RegClassID: 1628 case AMDGPU::VRegOrLds_32RegClassID: 1629 case AMDGPU::AGPR_32RegClassID: 1630 case AMDGPU::VS_32RegClassID: 1631 case AMDGPU::AV_32RegClassID: 1632 case AMDGPU::SReg_32RegClassID: 1633 case AMDGPU::SReg_32_XM0RegClassID: 1634 case AMDGPU::SRegOrLds_32RegClassID: 1635 return 32; 1636 case AMDGPU::SGPR_64RegClassID: 1637 case AMDGPU::VS_64RegClassID: 1638 case AMDGPU::SReg_64RegClassID: 1639 case AMDGPU::VReg_64RegClassID: 1640 case AMDGPU::AReg_64RegClassID: 1641 case AMDGPU::SReg_64_XEXECRegClassID: 1642 case AMDGPU::VReg_64_Align2RegClassID: 1643 case AMDGPU::AReg_64_Align2RegClassID: 1644 case AMDGPU::AV_64RegClassID: 1645 case AMDGPU::AV_64_Align2RegClassID: 1646 return 64; 1647 case AMDGPU::SGPR_96RegClassID: 1648 case AMDGPU::SReg_96RegClassID: 1649 case AMDGPU::VReg_96RegClassID: 1650 case AMDGPU::AReg_96RegClassID: 1651 case AMDGPU::VReg_96_Align2RegClassID: 1652 case AMDGPU::AReg_96_Align2RegClassID: 1653 case AMDGPU::AV_96RegClassID: 1654 case AMDGPU::AV_96_Align2RegClassID: 1655 return 96; 1656 case AMDGPU::SGPR_128RegClassID: 1657 case AMDGPU::SReg_128RegClassID: 1658 case AMDGPU::VReg_128RegClassID: 1659 case AMDGPU::AReg_128RegClassID: 1660 case AMDGPU::VReg_128_Align2RegClassID: 1661 case AMDGPU::AReg_128_Align2RegClassID: 1662 case AMDGPU::AV_128RegClassID: 1663 case AMDGPU::AV_128_Align2RegClassID: 1664 return 128; 1665 case AMDGPU::SGPR_160RegClassID: 1666 case AMDGPU::SReg_160RegClassID: 1667 case AMDGPU::VReg_160RegClassID: 1668 case AMDGPU::AReg_160RegClassID: 1669 case AMDGPU::VReg_160_Align2RegClassID: 1670 case AMDGPU::AReg_160_Align2RegClassID: 1671 case AMDGPU::AV_160RegClassID: 1672 case AMDGPU::AV_160_Align2RegClassID: 1673 return 160; 1674 case AMDGPU::SGPR_192RegClassID: 1675 case AMDGPU::SReg_192RegClassID: 1676 case AMDGPU::VReg_192RegClassID: 1677 case AMDGPU::AReg_192RegClassID: 1678 case AMDGPU::VReg_192_Align2RegClassID: 1679 case AMDGPU::AReg_192_Align2RegClassID: 1680 case AMDGPU::AV_192RegClassID: 1681 case AMDGPU::AV_192_Align2RegClassID: 1682 return 192; 1683 case AMDGPU::SGPR_224RegClassID: 1684 case AMDGPU::SReg_224RegClassID: 1685 case AMDGPU::VReg_224RegClassID: 1686 case AMDGPU::AReg_224RegClassID: 1687 case AMDGPU::VReg_224_Align2RegClassID: 1688 case AMDGPU::AReg_224_Align2RegClassID: 1689 case AMDGPU::AV_224RegClassID: 1690 case AMDGPU::AV_224_Align2RegClassID: 1691 return 224; 1692 case AMDGPU::SGPR_256RegClassID: 1693 case AMDGPU::SReg_256RegClassID: 1694 case AMDGPU::VReg_256RegClassID: 1695 case AMDGPU::AReg_256RegClassID: 1696 case AMDGPU::VReg_256_Align2RegClassID: 1697 case AMDGPU::AReg_256_Align2RegClassID: 1698 case AMDGPU::AV_256RegClassID: 1699 case AMDGPU::AV_256_Align2RegClassID: 1700 return 256; 1701 case AMDGPU::SGPR_512RegClassID: 1702 case AMDGPU::SReg_512RegClassID: 1703 case AMDGPU::VReg_512RegClassID: 1704 case AMDGPU::AReg_512RegClassID: 1705 case AMDGPU::VReg_512_Align2RegClassID: 1706 case AMDGPU::AReg_512_Align2RegClassID: 1707 case AMDGPU::AV_512RegClassID: 1708 case AMDGPU::AV_512_Align2RegClassID: 1709 return 512; 1710 case AMDGPU::SGPR_1024RegClassID: 1711 case AMDGPU::SReg_1024RegClassID: 1712 case AMDGPU::VReg_1024RegClassID: 1713 case AMDGPU::AReg_1024RegClassID: 1714 case AMDGPU::VReg_1024_Align2RegClassID: 1715 case AMDGPU::AReg_1024_Align2RegClassID: 1716 case AMDGPU::AV_1024RegClassID: 1717 case AMDGPU::AV_1024_Align2RegClassID: 1718 return 1024; 1719 default: 1720 llvm_unreachable("Unexpected register class"); 1721 } 1722 } 1723 1724 unsigned getRegBitWidth(const MCRegisterClass &RC) { 1725 return getRegBitWidth(RC.getID()); 1726 } 1727 1728 unsigned getRegOperandSize(const MCRegisterInfo *MRI, const MCInstrDesc &Desc, 1729 unsigned OpNo) { 1730 assert(OpNo < Desc.NumOperands); 1731 unsigned RCID = Desc.OpInfo[OpNo].RegClass; 1732 return getRegBitWidth(MRI->getRegClass(RCID)) / 8; 1733 } 1734 1735 bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi) { 1736 if (isInlinableIntLiteral(Literal)) 1737 return true; 1738 1739 uint64_t Val = static_cast<uint64_t>(Literal); 1740 return (Val == DoubleToBits(0.0)) || 1741 (Val == DoubleToBits(1.0)) || 1742 (Val == DoubleToBits(-1.0)) || 1743 (Val == DoubleToBits(0.5)) || 1744 (Val == DoubleToBits(-0.5)) || 1745 (Val == DoubleToBits(2.0)) || 1746 (Val == DoubleToBits(-2.0)) || 1747 (Val == DoubleToBits(4.0)) || 1748 (Val == DoubleToBits(-4.0)) || 1749 (Val == 0x3fc45f306dc9c882 && HasInv2Pi); 1750 } 1751 1752 bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi) { 1753 if (isInlinableIntLiteral(Literal)) 1754 return true; 1755 1756 // The actual type of the operand does not seem to matter as long 1757 // as the bits match one of the inline immediate values. For example: 1758 // 1759 // -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal, 1760 // so it is a legal inline immediate. 1761 // 1762 // 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in 1763 // floating-point, so it is a legal inline immediate. 1764 1765 uint32_t Val = static_cast<uint32_t>(Literal); 1766 return (Val == FloatToBits(0.0f)) || 1767 (Val == FloatToBits(1.0f)) || 1768 (Val == FloatToBits(-1.0f)) || 1769 (Val == FloatToBits(0.5f)) || 1770 (Val == FloatToBits(-0.5f)) || 1771 (Val == FloatToBits(2.0f)) || 1772 (Val == FloatToBits(-2.0f)) || 1773 (Val == FloatToBits(4.0f)) || 1774 (Val == FloatToBits(-4.0f)) || 1775 (Val == 0x3e22f983 && HasInv2Pi); 1776 } 1777 1778 bool isInlinableLiteral16(int16_t Literal, bool HasInv2Pi) { 1779 if (!HasInv2Pi) 1780 return false; 1781 1782 if (isInlinableIntLiteral(Literal)) 1783 return true; 1784 1785 uint16_t Val = static_cast<uint16_t>(Literal); 1786 return Val == 0x3C00 || // 1.0 1787 Val == 0xBC00 || // -1.0 1788 Val == 0x3800 || // 0.5 1789 Val == 0xB800 || // -0.5 1790 Val == 0x4000 || // 2.0 1791 Val == 0xC000 || // -2.0 1792 Val == 0x4400 || // 4.0 1793 Val == 0xC400 || // -4.0 1794 Val == 0x3118; // 1/2pi 1795 } 1796 1797 bool isInlinableLiteralV216(int32_t Literal, bool HasInv2Pi) { 1798 assert(HasInv2Pi); 1799 1800 if (isInt<16>(Literal) || isUInt<16>(Literal)) { 1801 int16_t Trunc = static_cast<int16_t>(Literal); 1802 return AMDGPU::isInlinableLiteral16(Trunc, HasInv2Pi); 1803 } 1804 if (!(Literal & 0xffff)) 1805 return AMDGPU::isInlinableLiteral16(Literal >> 16, HasInv2Pi); 1806 1807 int16_t Lo16 = static_cast<int16_t>(Literal); 1808 int16_t Hi16 = static_cast<int16_t>(Literal >> 16); 1809 return Lo16 == Hi16 && isInlinableLiteral16(Lo16, HasInv2Pi); 1810 } 1811 1812 bool isInlinableIntLiteralV216(int32_t Literal) { 1813 int16_t Lo16 = static_cast<int16_t>(Literal); 1814 if (isInt<16>(Literal) || isUInt<16>(Literal)) 1815 return isInlinableIntLiteral(Lo16); 1816 1817 int16_t Hi16 = static_cast<int16_t>(Literal >> 16); 1818 if (!(Literal & 0xffff)) 1819 return isInlinableIntLiteral(Hi16); 1820 return Lo16 == Hi16 && isInlinableIntLiteral(Lo16); 1821 } 1822 1823 bool isFoldableLiteralV216(int32_t Literal, bool HasInv2Pi) { 1824 assert(HasInv2Pi); 1825 1826 int16_t Lo16 = static_cast<int16_t>(Literal); 1827 if (isInt<16>(Literal) || isUInt<16>(Literal)) 1828 return true; 1829 1830 int16_t Hi16 = static_cast<int16_t>(Literal >> 16); 1831 if (!(Literal & 0xffff)) 1832 return true; 1833 return Lo16 == Hi16; 1834 } 1835 1836 bool isArgPassedInSGPR(const Argument *A) { 1837 const Function *F = A->getParent(); 1838 1839 // Arguments to compute shaders are never a source of divergence. 1840 CallingConv::ID CC = F->getCallingConv(); 1841 switch (CC) { 1842 case CallingConv::AMDGPU_KERNEL: 1843 case CallingConv::SPIR_KERNEL: 1844 return true; 1845 case CallingConv::AMDGPU_VS: 1846 case CallingConv::AMDGPU_LS: 1847 case CallingConv::AMDGPU_HS: 1848 case CallingConv::AMDGPU_ES: 1849 case CallingConv::AMDGPU_GS: 1850 case CallingConv::AMDGPU_PS: 1851 case CallingConv::AMDGPU_CS: 1852 case CallingConv::AMDGPU_Gfx: 1853 // For non-compute shaders, SGPR inputs are marked with either inreg or byval. 1854 // Everything else is in VGPRs. 1855 return F->getAttributes().hasParamAttr(A->getArgNo(), Attribute::InReg) || 1856 F->getAttributes().hasParamAttr(A->getArgNo(), Attribute::ByVal); 1857 default: 1858 // TODO: Should calls support inreg for SGPR inputs? 1859 return false; 1860 } 1861 } 1862 1863 static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) { 1864 return isGCN3Encoding(ST) || isGFX10Plus(ST); 1865 } 1866 1867 static bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST) { 1868 return isGFX9Plus(ST); 1869 } 1870 1871 bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST, 1872 int64_t EncodedOffset) { 1873 return hasSMEMByteOffset(ST) ? isUInt<20>(EncodedOffset) 1874 : isUInt<8>(EncodedOffset); 1875 } 1876 1877 bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST, 1878 int64_t EncodedOffset, 1879 bool IsBuffer) { 1880 return !IsBuffer && 1881 hasSMRDSignedImmOffset(ST) && 1882 isInt<21>(EncodedOffset); 1883 } 1884 1885 static bool isDwordAligned(uint64_t ByteOffset) { 1886 return (ByteOffset & 3) == 0; 1887 } 1888 1889 uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST, 1890 uint64_t ByteOffset) { 1891 if (hasSMEMByteOffset(ST)) 1892 return ByteOffset; 1893 1894 assert(isDwordAligned(ByteOffset)); 1895 return ByteOffset >> 2; 1896 } 1897 1898 Optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST, 1899 int64_t ByteOffset, bool IsBuffer) { 1900 // The signed version is always a byte offset. 1901 if (!IsBuffer && hasSMRDSignedImmOffset(ST)) { 1902 assert(hasSMEMByteOffset(ST)); 1903 return isInt<20>(ByteOffset) ? Optional<int64_t>(ByteOffset) : None; 1904 } 1905 1906 if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST)) 1907 return None; 1908 1909 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset); 1910 return isLegalSMRDEncodedUnsignedOffset(ST, EncodedOffset) 1911 ? Optional<int64_t>(EncodedOffset) 1912 : None; 1913 } 1914 1915 Optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST, 1916 int64_t ByteOffset) { 1917 if (!isCI(ST) || !isDwordAligned(ByteOffset)) 1918 return None; 1919 1920 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset); 1921 return isUInt<32>(EncodedOffset) ? Optional<int64_t>(EncodedOffset) : None; 1922 } 1923 1924 unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST, bool Signed) { 1925 // Address offset is 12-bit signed for GFX10, 13-bit for GFX9. 1926 if (AMDGPU::isGFX10(ST)) 1927 return Signed ? 12 : 11; 1928 1929 return Signed ? 13 : 12; 1930 } 1931 1932 // Given Imm, split it into the values to put into the SOffset and ImmOffset 1933 // fields in an MUBUF instruction. Return false if it is not possible (due to a 1934 // hardware bug needing a workaround). 1935 // 1936 // The required alignment ensures that individual address components remain 1937 // aligned if they are aligned to begin with. It also ensures that additional 1938 // offsets within the given alignment can be added to the resulting ImmOffset. 1939 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset, 1940 const GCNSubtarget *Subtarget, Align Alignment) { 1941 const uint32_t MaxImm = alignDown(4095, Alignment.value()); 1942 uint32_t Overflow = 0; 1943 1944 if (Imm > MaxImm) { 1945 if (Imm <= MaxImm + 64) { 1946 // Use an SOffset inline constant for 4..64 1947 Overflow = Imm - MaxImm; 1948 Imm = MaxImm; 1949 } else { 1950 // Try to keep the same value in SOffset for adjacent loads, so that 1951 // the corresponding register contents can be re-used. 1952 // 1953 // Load values with all low-bits (except for alignment bits) set into 1954 // SOffset, so that a larger range of values can be covered using 1955 // s_movk_i32. 1956 // 1957 // Atomic operations fail to work correctly when individual address 1958 // components are unaligned, even if their sum is aligned. 1959 uint32_t High = (Imm + Alignment.value()) & ~4095; 1960 uint32_t Low = (Imm + Alignment.value()) & 4095; 1961 Imm = Low; 1962 Overflow = High - Alignment.value(); 1963 } 1964 } 1965 1966 // There is a hardware bug in SI and CI which prevents address clamping in 1967 // MUBUF instructions from working correctly with SOffsets. The immediate 1968 // offset is unaffected. 1969 if (Overflow > 0 && 1970 Subtarget->getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS) 1971 return false; 1972 1973 ImmOffset = Imm; 1974 SOffset = Overflow; 1975 return true; 1976 } 1977 1978 SIModeRegisterDefaults::SIModeRegisterDefaults(const Function &F) { 1979 *this = getDefaultForCallingConv(F.getCallingConv()); 1980 1981 StringRef IEEEAttr = F.getFnAttribute("amdgpu-ieee").getValueAsString(); 1982 if (!IEEEAttr.empty()) 1983 IEEE = IEEEAttr == "true"; 1984 1985 StringRef DX10ClampAttr 1986 = F.getFnAttribute("amdgpu-dx10-clamp").getValueAsString(); 1987 if (!DX10ClampAttr.empty()) 1988 DX10Clamp = DX10ClampAttr == "true"; 1989 1990 StringRef DenormF32Attr = F.getFnAttribute("denormal-fp-math-f32").getValueAsString(); 1991 if (!DenormF32Attr.empty()) { 1992 DenormalMode DenormMode = parseDenormalFPAttribute(DenormF32Attr); 1993 FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE; 1994 FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE; 1995 } 1996 1997 StringRef DenormAttr = F.getFnAttribute("denormal-fp-math").getValueAsString(); 1998 if (!DenormAttr.empty()) { 1999 DenormalMode DenormMode = parseDenormalFPAttribute(DenormAttr); 2000 2001 if (DenormF32Attr.empty()) { 2002 FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE; 2003 FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE; 2004 } 2005 2006 FP64FP16InputDenormals = DenormMode.Input == DenormalMode::IEEE; 2007 FP64FP16OutputDenormals = DenormMode.Output == DenormalMode::IEEE; 2008 } 2009 } 2010 2011 namespace { 2012 2013 struct SourceOfDivergence { 2014 unsigned Intr; 2015 }; 2016 const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr); 2017 2018 #define GET_SourcesOfDivergence_IMPL 2019 #define GET_Gfx9BufferFormat_IMPL 2020 #define GET_Gfx10PlusBufferFormat_IMPL 2021 #include "AMDGPUGenSearchableTables.inc" 2022 2023 } // end anonymous namespace 2024 2025 bool isIntrinsicSourceOfDivergence(unsigned IntrID) { 2026 return lookupSourceOfDivergence(IntrID); 2027 } 2028 2029 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp, 2030 uint8_t NumComponents, 2031 uint8_t NumFormat, 2032 const MCSubtargetInfo &STI) { 2033 return isGFX10Plus(STI) 2034 ? getGfx10PlusBufferFormatInfo(BitsPerComp, NumComponents, 2035 NumFormat) 2036 : getGfx9BufferFormatInfo(BitsPerComp, NumComponents, NumFormat); 2037 } 2038 2039 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format, 2040 const MCSubtargetInfo &STI) { 2041 return isGFX10Plus(STI) ? getGfx10PlusBufferFormatInfo(Format) 2042 : getGfx9BufferFormatInfo(Format); 2043 } 2044 2045 } // namespace AMDGPU 2046 2047 raw_ostream &operator<<(raw_ostream &OS, 2048 const AMDGPU::IsaInfo::TargetIDSetting S) { 2049 switch (S) { 2050 case (AMDGPU::IsaInfo::TargetIDSetting::Unsupported): 2051 OS << "Unsupported"; 2052 break; 2053 case (AMDGPU::IsaInfo::TargetIDSetting::Any): 2054 OS << "Any"; 2055 break; 2056 case (AMDGPU::IsaInfo::TargetIDSetting::Off): 2057 OS << "Off"; 2058 break; 2059 case (AMDGPU::IsaInfo::TargetIDSetting::On): 2060 OS << "On"; 2061 break; 2062 } 2063 return OS; 2064 } 2065 2066 } // namespace llvm 2067