1 //===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "AMDGPUBaseInfo.h" 10 #include "AMDGPU.h" 11 #include "AMDGPUAsmUtils.h" 12 #include "AMDKernelCodeT.h" 13 #include "GCNSubtarget.h" 14 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 15 #include "llvm/BinaryFormat/ELF.h" 16 #include "llvm/IR/Attributes.h" 17 #include "llvm/IR/Function.h" 18 #include "llvm/IR/GlobalValue.h" 19 #include "llvm/IR/IntrinsicsAMDGPU.h" 20 #include "llvm/IR/IntrinsicsR600.h" 21 #include "llvm/IR/LLVMContext.h" 22 #include "llvm/MC/MCSubtargetInfo.h" 23 #include "llvm/Support/AMDHSAKernelDescriptor.h" 24 #include "llvm/Support/CommandLine.h" 25 #include "llvm/Support/TargetParser.h" 26 27 #define GET_INSTRINFO_NAMED_OPS 28 #define GET_INSTRMAP_INFO 29 #include "AMDGPUGenInstrInfo.inc" 30 31 static llvm::cl::opt<unsigned> AmdhsaCodeObjectVersion( 32 "amdhsa-code-object-version", llvm::cl::Hidden, 33 llvm::cl::desc("AMDHSA Code Object Version"), llvm::cl::init(4), 34 llvm::cl::ZeroOrMore); 35 36 namespace { 37 38 /// \returns Bit mask for given bit \p Shift and bit \p Width. 39 unsigned getBitMask(unsigned Shift, unsigned Width) { 40 return ((1 << Width) - 1) << Shift; 41 } 42 43 /// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width. 44 /// 45 /// \returns Packed \p Dst. 46 unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) { 47 Dst &= ~(1 << Shift) & ~getBitMask(Shift, Width); 48 Dst |= (Src << Shift) & getBitMask(Shift, Width); 49 return Dst; 50 } 51 52 /// Unpacks bits from \p Src for given bit \p Shift and bit \p Width. 53 /// 54 /// \returns Unpacked bits. 55 unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) { 56 return (Src & getBitMask(Shift, Width)) >> Shift; 57 } 58 59 /// \returns Vmcnt bit shift (lower bits). 60 unsigned getVmcntBitShiftLo() { return 0; } 61 62 /// \returns Vmcnt bit width (lower bits). 63 unsigned getVmcntBitWidthLo() { return 4; } 64 65 /// \returns Expcnt bit shift. 66 unsigned getExpcntBitShift() { return 4; } 67 68 /// \returns Expcnt bit width. 69 unsigned getExpcntBitWidth() { return 3; } 70 71 /// \returns Lgkmcnt bit shift. 72 unsigned getLgkmcntBitShift() { return 8; } 73 74 /// \returns Lgkmcnt bit width. 75 unsigned getLgkmcntBitWidth(unsigned VersionMajor) { 76 return (VersionMajor >= 10) ? 6 : 4; 77 } 78 79 /// \returns Vmcnt bit shift (higher bits). 80 unsigned getVmcntBitShiftHi() { return 14; } 81 82 /// \returns Vmcnt bit width (higher bits). 83 unsigned getVmcntBitWidthHi() { return 2; } 84 85 } // end namespace anonymous 86 87 namespace llvm { 88 89 namespace AMDGPU { 90 91 Optional<uint8_t> getHsaAbiVersion(const MCSubtargetInfo *STI) { 92 if (STI && STI->getTargetTriple().getOS() != Triple::AMDHSA) 93 return None; 94 95 switch (AmdhsaCodeObjectVersion) { 96 case 2: 97 return ELF::ELFABIVERSION_AMDGPU_HSA_V2; 98 case 3: 99 return ELF::ELFABIVERSION_AMDGPU_HSA_V3; 100 case 4: 101 return ELF::ELFABIVERSION_AMDGPU_HSA_V4; 102 default: 103 report_fatal_error(Twine("Unsupported AMDHSA Code Object Version ") + 104 Twine(AmdhsaCodeObjectVersion)); 105 } 106 } 107 108 bool isHsaAbiVersion2(const MCSubtargetInfo *STI) { 109 if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI)) 110 return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V2; 111 return false; 112 } 113 114 bool isHsaAbiVersion3(const MCSubtargetInfo *STI) { 115 if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI)) 116 return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V3; 117 return false; 118 } 119 120 bool isHsaAbiVersion4(const MCSubtargetInfo *STI) { 121 if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI)) 122 return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V4; 123 return false; 124 } 125 126 bool isHsaAbiVersion3Or4(const MCSubtargetInfo *STI) { 127 return isHsaAbiVersion3(STI) || isHsaAbiVersion4(STI); 128 } 129 130 #define GET_MIMGBaseOpcodesTable_IMPL 131 #define GET_MIMGDimInfoTable_IMPL 132 #define GET_MIMGInfoTable_IMPL 133 #define GET_MIMGLZMappingTable_IMPL 134 #define GET_MIMGMIPMappingTable_IMPL 135 #define GET_MIMGG16MappingTable_IMPL 136 #include "AMDGPUGenSearchableTables.inc" 137 138 int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding, 139 unsigned VDataDwords, unsigned VAddrDwords) { 140 const MIMGInfo *Info = getMIMGOpcodeHelper(BaseOpcode, MIMGEncoding, 141 VDataDwords, VAddrDwords); 142 return Info ? Info->Opcode : -1; 143 } 144 145 const MIMGBaseOpcodeInfo *getMIMGBaseOpcode(unsigned Opc) { 146 const MIMGInfo *Info = getMIMGInfo(Opc); 147 return Info ? getMIMGBaseOpcodeInfo(Info->BaseOpcode) : nullptr; 148 } 149 150 int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) { 151 const MIMGInfo *OrigInfo = getMIMGInfo(Opc); 152 const MIMGInfo *NewInfo = 153 getMIMGOpcodeHelper(OrigInfo->BaseOpcode, OrigInfo->MIMGEncoding, 154 NewChannels, OrigInfo->VAddrDwords); 155 return NewInfo ? NewInfo->Opcode : -1; 156 } 157 158 unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode, 159 const MIMGDimInfo *Dim, bool IsA16, 160 bool IsG16Supported) { 161 unsigned AddrWords = BaseOpcode->NumExtraArgs; 162 unsigned AddrComponents = (BaseOpcode->Coordinates ? Dim->NumCoords : 0) + 163 (BaseOpcode->LodOrClampOrMip ? 1 : 0); 164 if (IsA16) 165 AddrWords += divideCeil(AddrComponents, 2); 166 else 167 AddrWords += AddrComponents; 168 169 // Note: For subtargets that support A16 but not G16, enabling A16 also 170 // enables 16 bit gradients. 171 // For subtargets that support A16 (operand) and G16 (done with a different 172 // instruction encoding), they are independent. 173 174 if (BaseOpcode->Gradients) { 175 if ((IsA16 && !IsG16Supported) || BaseOpcode->G16) 176 // There are two gradients per coordinate, we pack them separately. 177 // For the 3d case, 178 // we get (dy/du, dx/du) (-, dz/du) (dy/dv, dx/dv) (-, dz/dv) 179 AddrWords += alignTo<2>(Dim->NumGradients / 2); 180 else 181 AddrWords += Dim->NumGradients; 182 } 183 return AddrWords; 184 } 185 186 struct MUBUFInfo { 187 uint16_t Opcode; 188 uint16_t BaseOpcode; 189 uint8_t elements; 190 bool has_vaddr; 191 bool has_srsrc; 192 bool has_soffset; 193 bool IsBufferInv; 194 }; 195 196 struct MTBUFInfo { 197 uint16_t Opcode; 198 uint16_t BaseOpcode; 199 uint8_t elements; 200 bool has_vaddr; 201 bool has_srsrc; 202 bool has_soffset; 203 }; 204 205 struct SMInfo { 206 uint16_t Opcode; 207 bool IsBuffer; 208 }; 209 210 struct VOPInfo { 211 uint16_t Opcode; 212 bool IsSingle; 213 }; 214 215 #define GET_MTBUFInfoTable_DECL 216 #define GET_MTBUFInfoTable_IMPL 217 #define GET_MUBUFInfoTable_DECL 218 #define GET_MUBUFInfoTable_IMPL 219 #define GET_SMInfoTable_DECL 220 #define GET_SMInfoTable_IMPL 221 #define GET_VOP1InfoTable_DECL 222 #define GET_VOP1InfoTable_IMPL 223 #define GET_VOP2InfoTable_DECL 224 #define GET_VOP2InfoTable_IMPL 225 #define GET_VOP3InfoTable_DECL 226 #define GET_VOP3InfoTable_IMPL 227 #include "AMDGPUGenSearchableTables.inc" 228 229 int getMTBUFBaseOpcode(unsigned Opc) { 230 const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opc); 231 return Info ? Info->BaseOpcode : -1; 232 } 233 234 int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements) { 235 const MTBUFInfo *Info = getMTBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements); 236 return Info ? Info->Opcode : -1; 237 } 238 239 int getMTBUFElements(unsigned Opc) { 240 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 241 return Info ? Info->elements : 0; 242 } 243 244 bool getMTBUFHasVAddr(unsigned Opc) { 245 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 246 return Info ? Info->has_vaddr : false; 247 } 248 249 bool getMTBUFHasSrsrc(unsigned Opc) { 250 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 251 return Info ? Info->has_srsrc : false; 252 } 253 254 bool getMTBUFHasSoffset(unsigned Opc) { 255 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 256 return Info ? Info->has_soffset : false; 257 } 258 259 int getMUBUFBaseOpcode(unsigned Opc) { 260 const MUBUFInfo *Info = getMUBUFInfoFromOpcode(Opc); 261 return Info ? Info->BaseOpcode : -1; 262 } 263 264 int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements) { 265 const MUBUFInfo *Info = getMUBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements); 266 return Info ? Info->Opcode : -1; 267 } 268 269 int getMUBUFElements(unsigned Opc) { 270 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 271 return Info ? Info->elements : 0; 272 } 273 274 bool getMUBUFHasVAddr(unsigned Opc) { 275 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 276 return Info ? Info->has_vaddr : false; 277 } 278 279 bool getMUBUFHasSrsrc(unsigned Opc) { 280 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 281 return Info ? Info->has_srsrc : false; 282 } 283 284 bool getMUBUFHasSoffset(unsigned Opc) { 285 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 286 return Info ? Info->has_soffset : false; 287 } 288 289 bool getMUBUFIsBufferInv(unsigned Opc) { 290 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 291 return Info ? Info->IsBufferInv : false; 292 } 293 294 bool getSMEMIsBuffer(unsigned Opc) { 295 const SMInfo *Info = getSMEMOpcodeHelper(Opc); 296 return Info ? Info->IsBuffer : false; 297 } 298 299 bool getVOP1IsSingle(unsigned Opc) { 300 const VOPInfo *Info = getVOP1OpcodeHelper(Opc); 301 return Info ? Info->IsSingle : false; 302 } 303 304 bool getVOP2IsSingle(unsigned Opc) { 305 const VOPInfo *Info = getVOP2OpcodeHelper(Opc); 306 return Info ? Info->IsSingle : false; 307 } 308 309 bool getVOP3IsSingle(unsigned Opc) { 310 const VOPInfo *Info = getVOP3OpcodeHelper(Opc); 311 return Info ? Info->IsSingle : false; 312 } 313 314 // Wrapper for Tablegen'd function. enum Subtarget is not defined in any 315 // header files, so we need to wrap it in a function that takes unsigned 316 // instead. 317 int getMCOpcode(uint16_t Opcode, unsigned Gen) { 318 return getMCOpcodeGen(Opcode, static_cast<Subtarget>(Gen)); 319 } 320 321 namespace IsaInfo { 322 323 AMDGPUTargetID::AMDGPUTargetID(const MCSubtargetInfo &STI) 324 : STI(STI), XnackSetting(TargetIDSetting::Any), 325 SramEccSetting(TargetIDSetting::Any) { 326 if (!STI.getFeatureBits().test(FeatureSupportsXNACK)) 327 XnackSetting = TargetIDSetting::Unsupported; 328 if (!STI.getFeatureBits().test(FeatureSupportsSRAMECC)) 329 SramEccSetting = TargetIDSetting::Unsupported; 330 } 331 332 void AMDGPUTargetID::setTargetIDFromFeaturesString(StringRef FS) { 333 // Check if xnack or sramecc is explicitly enabled or disabled. In the 334 // absence of the target features we assume we must generate code that can run 335 // in any environment. 336 SubtargetFeatures Features(FS); 337 Optional<bool> XnackRequested; 338 Optional<bool> SramEccRequested; 339 340 for (const std::string &Feature : Features.getFeatures()) { 341 if (Feature == "+xnack") 342 XnackRequested = true; 343 else if (Feature == "-xnack") 344 XnackRequested = false; 345 else if (Feature == "+sramecc") 346 SramEccRequested = true; 347 else if (Feature == "-sramecc") 348 SramEccRequested = false; 349 } 350 351 bool XnackSupported = isXnackSupported(); 352 bool SramEccSupported = isSramEccSupported(); 353 354 if (XnackRequested) { 355 if (XnackSupported) { 356 XnackSetting = 357 *XnackRequested ? TargetIDSetting::On : TargetIDSetting::Off; 358 } else { 359 // If a specific xnack setting was requested and this GPU does not support 360 // xnack emit a warning. Setting will remain set to "Unsupported". 361 if (*XnackRequested) { 362 errs() << "warning: xnack 'On' was requested for a processor that does " 363 "not support it!\n"; 364 } else { 365 errs() << "warning: xnack 'Off' was requested for a processor that " 366 "does not support it!\n"; 367 } 368 } 369 } 370 371 if (SramEccRequested) { 372 if (SramEccSupported) { 373 SramEccSetting = 374 *SramEccRequested ? TargetIDSetting::On : TargetIDSetting::Off; 375 } else { 376 // If a specific sramecc setting was requested and this GPU does not 377 // support sramecc emit a warning. Setting will remain set to 378 // "Unsupported". 379 if (*SramEccRequested) { 380 errs() << "warning: sramecc 'On' was requested for a processor that " 381 "does not support it!\n"; 382 } else { 383 errs() << "warning: sramecc 'Off' was requested for a processor that " 384 "does not support it!\n"; 385 } 386 } 387 } 388 } 389 390 static TargetIDSetting 391 getTargetIDSettingFromFeatureString(StringRef FeatureString) { 392 if (FeatureString.endswith("-")) 393 return TargetIDSetting::Off; 394 if (FeatureString.endswith("+")) 395 return TargetIDSetting::On; 396 397 llvm_unreachable("Malformed feature string"); 398 } 399 400 void AMDGPUTargetID::setTargetIDFromTargetIDStream(StringRef TargetID) { 401 SmallVector<StringRef, 3> TargetIDSplit; 402 TargetID.split(TargetIDSplit, ':'); 403 404 for (const auto &FeatureString : TargetIDSplit) { 405 if (FeatureString.startswith("xnack")) 406 XnackSetting = getTargetIDSettingFromFeatureString(FeatureString); 407 if (FeatureString.startswith("sramecc")) 408 SramEccSetting = getTargetIDSettingFromFeatureString(FeatureString); 409 } 410 } 411 412 std::string AMDGPUTargetID::toString() const { 413 std::string StringRep = ""; 414 raw_string_ostream StreamRep(StringRep); 415 416 auto TargetTriple = STI.getTargetTriple(); 417 auto Version = getIsaVersion(STI.getCPU()); 418 419 StreamRep << TargetTriple.getArchName() << '-' 420 << TargetTriple.getVendorName() << '-' 421 << TargetTriple.getOSName() << '-' 422 << TargetTriple.getEnvironmentName() << '-'; 423 424 std::string Processor = ""; 425 // TODO: Following else statement is present here because we used various 426 // alias names for GPUs up until GFX9 (e.g. 'fiji' is same as 'gfx803'). 427 // Remove once all aliases are removed from GCNProcessors.td. 428 if (Version.Major >= 9) 429 Processor = STI.getCPU().str(); 430 else 431 Processor = (Twine("gfx") + Twine(Version.Major) + Twine(Version.Minor) + 432 Twine(Version.Stepping)) 433 .str(); 434 435 std::string Features = ""; 436 if (Optional<uint8_t> HsaAbiVersion = getHsaAbiVersion(&STI)) { 437 switch (*HsaAbiVersion) { 438 case ELF::ELFABIVERSION_AMDGPU_HSA_V2: 439 // Code object V2 only supported specific processors and had fixed 440 // settings for the XNACK. 441 if (Processor == "gfx600") { 442 } else if (Processor == "gfx601") { 443 } else if (Processor == "gfx602") { 444 } else if (Processor == "gfx700") { 445 } else if (Processor == "gfx701") { 446 } else if (Processor == "gfx702") { 447 } else if (Processor == "gfx703") { 448 } else if (Processor == "gfx704") { 449 } else if (Processor == "gfx705") { 450 } else if (Processor == "gfx801") { 451 if (!isXnackOnOrAny()) 452 report_fatal_error( 453 "AMD GPU code object V2 does not support processor " + Processor + 454 " without XNACK"); 455 } else if (Processor == "gfx802") { 456 } else if (Processor == "gfx803") { 457 } else if (Processor == "gfx805") { 458 } else if (Processor == "gfx810") { 459 if (!isXnackOnOrAny()) 460 report_fatal_error( 461 "AMD GPU code object V2 does not support processor " + Processor + 462 " without XNACK"); 463 } else if (Processor == "gfx900") { 464 if (isXnackOnOrAny()) 465 Processor = "gfx901"; 466 } else if (Processor == "gfx902") { 467 if (isXnackOnOrAny()) 468 Processor = "gfx903"; 469 } else if (Processor == "gfx904") { 470 if (isXnackOnOrAny()) 471 Processor = "gfx905"; 472 } else if (Processor == "gfx906") { 473 if (isXnackOnOrAny()) 474 Processor = "gfx907"; 475 } else if (Processor == "gfx90c") { 476 if (isXnackOnOrAny()) 477 report_fatal_error( 478 "AMD GPU code object V2 does not support processor " + Processor + 479 " with XNACK being ON or ANY"); 480 } else { 481 report_fatal_error( 482 "AMD GPU code object V2 does not support processor " + Processor); 483 } 484 break; 485 case ELF::ELFABIVERSION_AMDGPU_HSA_V3: 486 // xnack. 487 if (isXnackOnOrAny()) 488 Features += "+xnack"; 489 // In code object v2 and v3, "sramecc" feature was spelled with a 490 // hyphen ("sram-ecc"). 491 if (isSramEccOnOrAny()) 492 Features += "+sram-ecc"; 493 break; 494 case ELF::ELFABIVERSION_AMDGPU_HSA_V4: 495 // sramecc. 496 if (getSramEccSetting() == TargetIDSetting::Off) 497 Features += ":sramecc-"; 498 else if (getSramEccSetting() == TargetIDSetting::On) 499 Features += ":sramecc+"; 500 // xnack. 501 if (getXnackSetting() == TargetIDSetting::Off) 502 Features += ":xnack-"; 503 else if (getXnackSetting() == TargetIDSetting::On) 504 Features += ":xnack+"; 505 break; 506 default: 507 break; 508 } 509 } 510 511 StreamRep << Processor << Features; 512 513 StreamRep.flush(); 514 return StringRep; 515 } 516 517 unsigned getWavefrontSize(const MCSubtargetInfo *STI) { 518 if (STI->getFeatureBits().test(FeatureWavefrontSize16)) 519 return 16; 520 if (STI->getFeatureBits().test(FeatureWavefrontSize32)) 521 return 32; 522 523 return 64; 524 } 525 526 unsigned getLocalMemorySize(const MCSubtargetInfo *STI) { 527 if (STI->getFeatureBits().test(FeatureLocalMemorySize32768)) 528 return 32768; 529 if (STI->getFeatureBits().test(FeatureLocalMemorySize65536)) 530 return 65536; 531 532 return 0; 533 } 534 535 unsigned getEUsPerCU(const MCSubtargetInfo *STI) { 536 // "Per CU" really means "per whatever functional block the waves of a 537 // workgroup must share". For gfx10 in CU mode this is the CU, which contains 538 // two SIMDs. 539 if (isGFX10Plus(*STI) && STI->getFeatureBits().test(FeatureCuMode)) 540 return 2; 541 // Pre-gfx10 a CU contains four SIMDs. For gfx10 in WGP mode the WGP contains 542 // two CUs, so a total of four SIMDs. 543 return 4; 544 } 545 546 unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo *STI, 547 unsigned FlatWorkGroupSize) { 548 assert(FlatWorkGroupSize != 0); 549 if (STI->getTargetTriple().getArch() != Triple::amdgcn) 550 return 8; 551 unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize); 552 if (N == 1) 553 return 40; 554 N = 40 / N; 555 return std::min(N, 16u); 556 } 557 558 unsigned getMinWavesPerEU(const MCSubtargetInfo *STI) { 559 return 1; 560 } 561 562 unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI) { 563 // FIXME: Need to take scratch memory into account. 564 if (isGFX90A(*STI)) 565 return 8; 566 if (!isGFX10Plus(*STI)) 567 return 10; 568 return hasGFX10_3Insts(*STI) ? 16 : 20; 569 } 570 571 unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo *STI, 572 unsigned FlatWorkGroupSize) { 573 return divideCeil(getWavesPerWorkGroup(STI, FlatWorkGroupSize), 574 getEUsPerCU(STI)); 575 } 576 577 unsigned getMinFlatWorkGroupSize(const MCSubtargetInfo *STI) { 578 return 1; 579 } 580 581 unsigned getMaxFlatWorkGroupSize(const MCSubtargetInfo *STI) { 582 // Some subtargets allow encoding 2048, but this isn't tested or supported. 583 return 1024; 584 } 585 586 unsigned getWavesPerWorkGroup(const MCSubtargetInfo *STI, 587 unsigned FlatWorkGroupSize) { 588 return divideCeil(FlatWorkGroupSize, getWavefrontSize(STI)); 589 } 590 591 unsigned getSGPRAllocGranule(const MCSubtargetInfo *STI) { 592 IsaVersion Version = getIsaVersion(STI->getCPU()); 593 if (Version.Major >= 10) 594 return getAddressableNumSGPRs(STI); 595 if (Version.Major >= 8) 596 return 16; 597 return 8; 598 } 599 600 unsigned getSGPREncodingGranule(const MCSubtargetInfo *STI) { 601 return 8; 602 } 603 604 unsigned getTotalNumSGPRs(const MCSubtargetInfo *STI) { 605 IsaVersion Version = getIsaVersion(STI->getCPU()); 606 if (Version.Major >= 8) 607 return 800; 608 return 512; 609 } 610 611 unsigned getAddressableNumSGPRs(const MCSubtargetInfo *STI) { 612 if (STI->getFeatureBits().test(FeatureSGPRInitBug)) 613 return FIXED_NUM_SGPRS_FOR_INIT_BUG; 614 615 IsaVersion Version = getIsaVersion(STI->getCPU()); 616 if (Version.Major >= 10) 617 return 106; 618 if (Version.Major >= 8) 619 return 102; 620 return 104; 621 } 622 623 unsigned getMinNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 624 assert(WavesPerEU != 0); 625 626 IsaVersion Version = getIsaVersion(STI->getCPU()); 627 if (Version.Major >= 10) 628 return 0; 629 630 if (WavesPerEU >= getMaxWavesPerEU(STI)) 631 return 0; 632 633 unsigned MinNumSGPRs = getTotalNumSGPRs(STI) / (WavesPerEU + 1); 634 if (STI->getFeatureBits().test(FeatureTrapHandler)) 635 MinNumSGPRs -= std::min(MinNumSGPRs, (unsigned)TRAP_NUM_SGPRS); 636 MinNumSGPRs = alignDown(MinNumSGPRs, getSGPRAllocGranule(STI)) + 1; 637 return std::min(MinNumSGPRs, getAddressableNumSGPRs(STI)); 638 } 639 640 unsigned getMaxNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU, 641 bool Addressable) { 642 assert(WavesPerEU != 0); 643 644 unsigned AddressableNumSGPRs = getAddressableNumSGPRs(STI); 645 IsaVersion Version = getIsaVersion(STI->getCPU()); 646 if (Version.Major >= 10) 647 return Addressable ? AddressableNumSGPRs : 108; 648 if (Version.Major >= 8 && !Addressable) 649 AddressableNumSGPRs = 112; 650 unsigned MaxNumSGPRs = getTotalNumSGPRs(STI) / WavesPerEU; 651 if (STI->getFeatureBits().test(FeatureTrapHandler)) 652 MaxNumSGPRs -= std::min(MaxNumSGPRs, (unsigned)TRAP_NUM_SGPRS); 653 MaxNumSGPRs = alignDown(MaxNumSGPRs, getSGPRAllocGranule(STI)); 654 return std::min(MaxNumSGPRs, AddressableNumSGPRs); 655 } 656 657 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed, 658 bool FlatScrUsed, bool XNACKUsed) { 659 unsigned ExtraSGPRs = 0; 660 if (VCCUsed) 661 ExtraSGPRs = 2; 662 663 IsaVersion Version = getIsaVersion(STI->getCPU()); 664 if (Version.Major >= 10) 665 return ExtraSGPRs; 666 667 if (Version.Major < 8) { 668 if (FlatScrUsed) 669 ExtraSGPRs = 4; 670 } else { 671 if (XNACKUsed) 672 ExtraSGPRs = 4; 673 674 if (FlatScrUsed || 675 STI->getFeatureBits().test(AMDGPU::FeatureArchitectedFlatScratch)) 676 ExtraSGPRs = 6; 677 } 678 679 return ExtraSGPRs; 680 } 681 682 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed, 683 bool FlatScrUsed) { 684 return getNumExtraSGPRs(STI, VCCUsed, FlatScrUsed, 685 STI->getFeatureBits().test(AMDGPU::FeatureXNACK)); 686 } 687 688 unsigned getNumSGPRBlocks(const MCSubtargetInfo *STI, unsigned NumSGPRs) { 689 NumSGPRs = alignTo(std::max(1u, NumSGPRs), getSGPREncodingGranule(STI)); 690 // SGPRBlocks is actual number of SGPR blocks minus 1. 691 return NumSGPRs / getSGPREncodingGranule(STI) - 1; 692 } 693 694 unsigned getVGPRAllocGranule(const MCSubtargetInfo *STI, 695 Optional<bool> EnableWavefrontSize32) { 696 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 697 return 8; 698 699 bool IsWave32 = EnableWavefrontSize32 ? 700 *EnableWavefrontSize32 : 701 STI->getFeatureBits().test(FeatureWavefrontSize32); 702 703 if (hasGFX10_3Insts(*STI)) 704 return IsWave32 ? 16 : 8; 705 706 return IsWave32 ? 8 : 4; 707 } 708 709 unsigned getVGPREncodingGranule(const MCSubtargetInfo *STI, 710 Optional<bool> EnableWavefrontSize32) { 711 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 712 return 8; 713 714 bool IsWave32 = EnableWavefrontSize32 ? 715 *EnableWavefrontSize32 : 716 STI->getFeatureBits().test(FeatureWavefrontSize32); 717 718 return IsWave32 ? 8 : 4; 719 } 720 721 unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) { 722 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 723 return 512; 724 if (!isGFX10Plus(*STI)) 725 return 256; 726 return STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1024 : 512; 727 } 728 729 unsigned getAddressableNumVGPRs(const MCSubtargetInfo *STI) { 730 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 731 return 512; 732 return 256; 733 } 734 735 unsigned getMinNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 736 assert(WavesPerEU != 0); 737 738 if (WavesPerEU >= getMaxWavesPerEU(STI)) 739 return 0; 740 unsigned MinNumVGPRs = 741 alignDown(getTotalNumVGPRs(STI) / (WavesPerEU + 1), 742 getVGPRAllocGranule(STI)) + 1; 743 return std::min(MinNumVGPRs, getAddressableNumVGPRs(STI)); 744 } 745 746 unsigned getMaxNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 747 assert(WavesPerEU != 0); 748 749 unsigned MaxNumVGPRs = alignDown(getTotalNumVGPRs(STI) / WavesPerEU, 750 getVGPRAllocGranule(STI)); 751 unsigned AddressableNumVGPRs = getAddressableNumVGPRs(STI); 752 return std::min(MaxNumVGPRs, AddressableNumVGPRs); 753 } 754 755 unsigned getNumVGPRBlocks(const MCSubtargetInfo *STI, unsigned NumVGPRs, 756 Optional<bool> EnableWavefrontSize32) { 757 NumVGPRs = alignTo(std::max(1u, NumVGPRs), 758 getVGPREncodingGranule(STI, EnableWavefrontSize32)); 759 // VGPRBlocks is actual number of VGPR blocks minus 1. 760 return NumVGPRs / getVGPREncodingGranule(STI, EnableWavefrontSize32) - 1; 761 } 762 763 } // end namespace IsaInfo 764 765 void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header, 766 const MCSubtargetInfo *STI) { 767 IsaVersion Version = getIsaVersion(STI->getCPU()); 768 769 memset(&Header, 0, sizeof(Header)); 770 771 Header.amd_kernel_code_version_major = 1; 772 Header.amd_kernel_code_version_minor = 2; 773 Header.amd_machine_kind = 1; // AMD_MACHINE_KIND_AMDGPU 774 Header.amd_machine_version_major = Version.Major; 775 Header.amd_machine_version_minor = Version.Minor; 776 Header.amd_machine_version_stepping = Version.Stepping; 777 Header.kernel_code_entry_byte_offset = sizeof(Header); 778 Header.wavefront_size = 6; 779 780 // If the code object does not support indirect functions, then the value must 781 // be 0xffffffff. 782 Header.call_convention = -1; 783 784 // These alignment values are specified in powers of two, so alignment = 785 // 2^n. The minimum alignment is 2^4 = 16. 786 Header.kernarg_segment_alignment = 4; 787 Header.group_segment_alignment = 4; 788 Header.private_segment_alignment = 4; 789 790 if (Version.Major >= 10) { 791 if (STI->getFeatureBits().test(FeatureWavefrontSize32)) { 792 Header.wavefront_size = 5; 793 Header.code_properties |= AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32; 794 } 795 Header.compute_pgm_resource_registers |= 796 S_00B848_WGP_MODE(STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1) | 797 S_00B848_MEM_ORDERED(1); 798 } 799 } 800 801 amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor( 802 const MCSubtargetInfo *STI) { 803 IsaVersion Version = getIsaVersion(STI->getCPU()); 804 805 amdhsa::kernel_descriptor_t KD; 806 memset(&KD, 0, sizeof(KD)); 807 808 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 809 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, 810 amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE); 811 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 812 amdhsa::COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP, 1); 813 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 814 amdhsa::COMPUTE_PGM_RSRC1_ENABLE_IEEE_MODE, 1); 815 AMDHSA_BITS_SET(KD.compute_pgm_rsrc2, 816 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, 1); 817 if (Version.Major >= 10) { 818 AMDHSA_BITS_SET(KD.kernel_code_properties, 819 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, 820 STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1 : 0); 821 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 822 amdhsa::COMPUTE_PGM_RSRC1_WGP_MODE, 823 STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1); 824 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 825 amdhsa::COMPUTE_PGM_RSRC1_MEM_ORDERED, 1); 826 } 827 if (AMDGPU::isGFX90A(*STI)) { 828 AMDHSA_BITS_SET(KD.compute_pgm_rsrc3, 829 amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, 830 STI->getFeatureBits().test(FeatureTgSplit) ? 1 : 0); 831 } 832 return KD; 833 } 834 835 bool isGroupSegment(const GlobalValue *GV) { 836 return GV->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS; 837 } 838 839 bool isGlobalSegment(const GlobalValue *GV) { 840 return GV->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS; 841 } 842 843 bool isReadOnlySegment(const GlobalValue *GV) { 844 unsigned AS = GV->getAddressSpace(); 845 return AS == AMDGPUAS::CONSTANT_ADDRESS || 846 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT; 847 } 848 849 bool shouldEmitConstantsToTextSection(const Triple &TT) { 850 return TT.getArch() == Triple::r600; 851 } 852 853 int getIntegerAttribute(const Function &F, StringRef Name, int Default) { 854 Attribute A = F.getFnAttribute(Name); 855 int Result = Default; 856 857 if (A.isStringAttribute()) { 858 StringRef Str = A.getValueAsString(); 859 if (Str.getAsInteger(0, Result)) { 860 LLVMContext &Ctx = F.getContext(); 861 Ctx.emitError("can't parse integer attribute " + Name); 862 } 863 } 864 865 return Result; 866 } 867 868 std::pair<int, int> getIntegerPairAttribute(const Function &F, 869 StringRef Name, 870 std::pair<int, int> Default, 871 bool OnlyFirstRequired) { 872 Attribute A = F.getFnAttribute(Name); 873 if (!A.isStringAttribute()) 874 return Default; 875 876 LLVMContext &Ctx = F.getContext(); 877 std::pair<int, int> Ints = Default; 878 std::pair<StringRef, StringRef> Strs = A.getValueAsString().split(','); 879 if (Strs.first.trim().getAsInteger(0, Ints.first)) { 880 Ctx.emitError("can't parse first integer attribute " + Name); 881 return Default; 882 } 883 if (Strs.second.trim().getAsInteger(0, Ints.second)) { 884 if (!OnlyFirstRequired || !Strs.second.trim().empty()) { 885 Ctx.emitError("can't parse second integer attribute " + Name); 886 return Default; 887 } 888 } 889 890 return Ints; 891 } 892 893 unsigned getVmcntBitMask(const IsaVersion &Version) { 894 unsigned VmcntLo = (1 << getVmcntBitWidthLo()) - 1; 895 if (Version.Major < 9) 896 return VmcntLo; 897 898 unsigned VmcntHi = ((1 << getVmcntBitWidthHi()) - 1) << getVmcntBitWidthLo(); 899 return VmcntLo | VmcntHi; 900 } 901 902 unsigned getExpcntBitMask(const IsaVersion &Version) { 903 return (1 << getExpcntBitWidth()) - 1; 904 } 905 906 unsigned getLgkmcntBitMask(const IsaVersion &Version) { 907 return (1 << getLgkmcntBitWidth(Version.Major)) - 1; 908 } 909 910 unsigned getWaitcntBitMask(const IsaVersion &Version) { 911 unsigned VmcntLo = getBitMask(getVmcntBitShiftLo(), getVmcntBitWidthLo()); 912 unsigned Expcnt = getBitMask(getExpcntBitShift(), getExpcntBitWidth()); 913 unsigned Lgkmcnt = getBitMask(getLgkmcntBitShift(), 914 getLgkmcntBitWidth(Version.Major)); 915 unsigned Waitcnt = VmcntLo | Expcnt | Lgkmcnt; 916 if (Version.Major < 9) 917 return Waitcnt; 918 919 unsigned VmcntHi = getBitMask(getVmcntBitShiftHi(), getVmcntBitWidthHi()); 920 return Waitcnt | VmcntHi; 921 } 922 923 unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) { 924 unsigned VmcntLo = 925 unpackBits(Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo()); 926 if (Version.Major < 9) 927 return VmcntLo; 928 929 unsigned VmcntHi = 930 unpackBits(Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi()); 931 VmcntHi <<= getVmcntBitWidthLo(); 932 return VmcntLo | VmcntHi; 933 } 934 935 unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) { 936 return unpackBits(Waitcnt, getExpcntBitShift(), getExpcntBitWidth()); 937 } 938 939 unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) { 940 return unpackBits(Waitcnt, getLgkmcntBitShift(), 941 getLgkmcntBitWidth(Version.Major)); 942 } 943 944 void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, 945 unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt) { 946 Vmcnt = decodeVmcnt(Version, Waitcnt); 947 Expcnt = decodeExpcnt(Version, Waitcnt); 948 Lgkmcnt = decodeLgkmcnt(Version, Waitcnt); 949 } 950 951 Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded) { 952 Waitcnt Decoded; 953 Decoded.VmCnt = decodeVmcnt(Version, Encoded); 954 Decoded.ExpCnt = decodeExpcnt(Version, Encoded); 955 Decoded.LgkmCnt = decodeLgkmcnt(Version, Encoded); 956 return Decoded; 957 } 958 959 unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt, 960 unsigned Vmcnt) { 961 Waitcnt = 962 packBits(Vmcnt, Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo()); 963 if (Version.Major < 9) 964 return Waitcnt; 965 966 Vmcnt >>= getVmcntBitWidthLo(); 967 return packBits(Vmcnt, Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi()); 968 } 969 970 unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt, 971 unsigned Expcnt) { 972 return packBits(Expcnt, Waitcnt, getExpcntBitShift(), getExpcntBitWidth()); 973 } 974 975 unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt, 976 unsigned Lgkmcnt) { 977 return packBits(Lgkmcnt, Waitcnt, getLgkmcntBitShift(), 978 getLgkmcntBitWidth(Version.Major)); 979 } 980 981 unsigned encodeWaitcnt(const IsaVersion &Version, 982 unsigned Vmcnt, unsigned Expcnt, unsigned Lgkmcnt) { 983 unsigned Waitcnt = getWaitcntBitMask(Version); 984 Waitcnt = encodeVmcnt(Version, Waitcnt, Vmcnt); 985 Waitcnt = encodeExpcnt(Version, Waitcnt, Expcnt); 986 Waitcnt = encodeLgkmcnt(Version, Waitcnt, Lgkmcnt); 987 return Waitcnt; 988 } 989 990 unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded) { 991 return encodeWaitcnt(Version, Decoded.VmCnt, Decoded.ExpCnt, Decoded.LgkmCnt); 992 } 993 994 //===----------------------------------------------------------------------===// 995 // hwreg 996 //===----------------------------------------------------------------------===// 997 998 namespace Hwreg { 999 1000 int64_t getHwregId(const StringRef Name) { 1001 for (int Id = ID_SYMBOLIC_FIRST_; Id < ID_SYMBOLIC_LAST_; ++Id) { 1002 if (IdSymbolic[Id] && Name == IdSymbolic[Id]) 1003 return Id; 1004 } 1005 return ID_UNKNOWN_; 1006 } 1007 1008 static unsigned getLastSymbolicHwreg(const MCSubtargetInfo &STI) { 1009 if (isSI(STI) || isCI(STI) || isVI(STI)) 1010 return ID_SYMBOLIC_FIRST_GFX9_; 1011 else if (isGFX9(STI)) 1012 return ID_SYMBOLIC_FIRST_GFX10_; 1013 else if (isGFX10(STI) && !isGFX10_BEncoding(STI)) 1014 return ID_SYMBOLIC_FIRST_GFX1030_; 1015 else 1016 return ID_SYMBOLIC_LAST_; 1017 } 1018 1019 bool isValidHwreg(int64_t Id, const MCSubtargetInfo &STI) { 1020 return 1021 ID_SYMBOLIC_FIRST_ <= Id && Id < getLastSymbolicHwreg(STI) && 1022 IdSymbolic[Id] && (Id != ID_XNACK_MASK || !AMDGPU::isGFX10_BEncoding(STI)); 1023 } 1024 1025 bool isValidHwreg(int64_t Id) { 1026 return 0 <= Id && isUInt<ID_WIDTH_>(Id); 1027 } 1028 1029 bool isValidHwregOffset(int64_t Offset) { 1030 return 0 <= Offset && isUInt<OFFSET_WIDTH_>(Offset); 1031 } 1032 1033 bool isValidHwregWidth(int64_t Width) { 1034 return 0 <= (Width - 1) && isUInt<WIDTH_M1_WIDTH_>(Width - 1); 1035 } 1036 1037 uint64_t encodeHwreg(uint64_t Id, uint64_t Offset, uint64_t Width) { 1038 return (Id << ID_SHIFT_) | 1039 (Offset << OFFSET_SHIFT_) | 1040 ((Width - 1) << WIDTH_M1_SHIFT_); 1041 } 1042 1043 StringRef getHwreg(unsigned Id, const MCSubtargetInfo &STI) { 1044 return isValidHwreg(Id, STI) ? IdSymbolic[Id] : ""; 1045 } 1046 1047 void decodeHwreg(unsigned Val, unsigned &Id, unsigned &Offset, unsigned &Width) { 1048 Id = (Val & ID_MASK_) >> ID_SHIFT_; 1049 Offset = (Val & OFFSET_MASK_) >> OFFSET_SHIFT_; 1050 Width = ((Val & WIDTH_M1_MASK_) >> WIDTH_M1_SHIFT_) + 1; 1051 } 1052 1053 } // namespace Hwreg 1054 1055 //===----------------------------------------------------------------------===// 1056 // exp tgt 1057 //===----------------------------------------------------------------------===// 1058 1059 namespace Exp { 1060 1061 struct ExpTgt { 1062 StringLiteral Name; 1063 unsigned Tgt; 1064 unsigned MaxIndex; 1065 }; 1066 1067 static constexpr ExpTgt ExpTgtInfo[] = { 1068 {{"null"}, ET_NULL, ET_NULL_MAX_IDX}, 1069 {{"mrtz"}, ET_MRTZ, ET_MRTZ_MAX_IDX}, 1070 {{"prim"}, ET_PRIM, ET_PRIM_MAX_IDX}, 1071 {{"mrt"}, ET_MRT0, ET_MRT_MAX_IDX}, 1072 {{"pos"}, ET_POS0, ET_POS_MAX_IDX}, 1073 {{"param"}, ET_PARAM0, ET_PARAM_MAX_IDX}, 1074 }; 1075 1076 bool getTgtName(unsigned Id, StringRef &Name, int &Index) { 1077 for (const ExpTgt &Val : ExpTgtInfo) { 1078 if (Val.Tgt <= Id && Id <= Val.Tgt + Val.MaxIndex) { 1079 Index = (Val.MaxIndex == 0) ? -1 : (Id - Val.Tgt); 1080 Name = Val.Name; 1081 return true; 1082 } 1083 } 1084 return false; 1085 } 1086 1087 unsigned getTgtId(const StringRef Name) { 1088 1089 for (const ExpTgt &Val : ExpTgtInfo) { 1090 if (Val.MaxIndex == 0 && Name == Val.Name) 1091 return Val.Tgt; 1092 1093 if (Val.MaxIndex > 0 && Name.startswith(Val.Name)) { 1094 StringRef Suffix = Name.drop_front(Val.Name.size()); 1095 1096 unsigned Id; 1097 if (Suffix.getAsInteger(10, Id) || Id > Val.MaxIndex) 1098 return ET_INVALID; 1099 1100 // Disable leading zeroes 1101 if (Suffix.size() > 1 && Suffix[0] == '0') 1102 return ET_INVALID; 1103 1104 return Val.Tgt + Id; 1105 } 1106 } 1107 return ET_INVALID; 1108 } 1109 1110 bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI) { 1111 return (Id != ET_POS4 && Id != ET_PRIM) || isGFX10Plus(STI); 1112 } 1113 1114 } // namespace Exp 1115 1116 //===----------------------------------------------------------------------===// 1117 // MTBUF Format 1118 //===----------------------------------------------------------------------===// 1119 1120 namespace MTBUFFormat { 1121 1122 int64_t getDfmt(const StringRef Name) { 1123 for (int Id = DFMT_MIN; Id <= DFMT_MAX; ++Id) { 1124 if (Name == DfmtSymbolic[Id]) 1125 return Id; 1126 } 1127 return DFMT_UNDEF; 1128 } 1129 1130 StringRef getDfmtName(unsigned Id) { 1131 assert(Id <= DFMT_MAX); 1132 return DfmtSymbolic[Id]; 1133 } 1134 1135 static StringLiteral const *getNfmtLookupTable(const MCSubtargetInfo &STI) { 1136 if (isSI(STI) || isCI(STI)) 1137 return NfmtSymbolicSICI; 1138 if (isVI(STI) || isGFX9(STI)) 1139 return NfmtSymbolicVI; 1140 return NfmtSymbolicGFX10; 1141 } 1142 1143 int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI) { 1144 auto lookupTable = getNfmtLookupTable(STI); 1145 for (int Id = NFMT_MIN; Id <= NFMT_MAX; ++Id) { 1146 if (Name == lookupTable[Id]) 1147 return Id; 1148 } 1149 return NFMT_UNDEF; 1150 } 1151 1152 StringRef getNfmtName(unsigned Id, const MCSubtargetInfo &STI) { 1153 assert(Id <= NFMT_MAX); 1154 return getNfmtLookupTable(STI)[Id]; 1155 } 1156 1157 bool isValidDfmtNfmt(unsigned Id, const MCSubtargetInfo &STI) { 1158 unsigned Dfmt; 1159 unsigned Nfmt; 1160 decodeDfmtNfmt(Id, Dfmt, Nfmt); 1161 return isValidNfmt(Nfmt, STI); 1162 } 1163 1164 bool isValidNfmt(unsigned Id, const MCSubtargetInfo &STI) { 1165 return !getNfmtName(Id, STI).empty(); 1166 } 1167 1168 int64_t encodeDfmtNfmt(unsigned Dfmt, unsigned Nfmt) { 1169 return (Dfmt << DFMT_SHIFT) | (Nfmt << NFMT_SHIFT); 1170 } 1171 1172 void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt) { 1173 Dfmt = (Format >> DFMT_SHIFT) & DFMT_MASK; 1174 Nfmt = (Format >> NFMT_SHIFT) & NFMT_MASK; 1175 } 1176 1177 int64_t getUnifiedFormat(const StringRef Name) { 1178 for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) { 1179 if (Name == UfmtSymbolic[Id]) 1180 return Id; 1181 } 1182 return UFMT_UNDEF; 1183 } 1184 1185 StringRef getUnifiedFormatName(unsigned Id) { 1186 return isValidUnifiedFormat(Id) ? UfmtSymbolic[Id] : ""; 1187 } 1188 1189 bool isValidUnifiedFormat(unsigned Id) { 1190 return Id <= UFMT_LAST; 1191 } 1192 1193 int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt) { 1194 int64_t Fmt = encodeDfmtNfmt(Dfmt, Nfmt); 1195 for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) { 1196 if (Fmt == DfmtNfmt2UFmt[Id]) 1197 return Id; 1198 } 1199 return UFMT_UNDEF; 1200 } 1201 1202 bool isValidFormatEncoding(unsigned Val, const MCSubtargetInfo &STI) { 1203 return isGFX10Plus(STI) ? (Val <= UFMT_MAX) : (Val <= DFMT_NFMT_MAX); 1204 } 1205 1206 unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI) { 1207 if (isGFX10Plus(STI)) 1208 return UFMT_DEFAULT; 1209 return DFMT_NFMT_DEFAULT; 1210 } 1211 1212 } // namespace MTBUFFormat 1213 1214 //===----------------------------------------------------------------------===// 1215 // SendMsg 1216 //===----------------------------------------------------------------------===// 1217 1218 namespace SendMsg { 1219 1220 int64_t getMsgId(const StringRef Name) { 1221 for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) { 1222 if (IdSymbolic[i] && Name == IdSymbolic[i]) 1223 return i; 1224 } 1225 return ID_UNKNOWN_; 1226 } 1227 1228 bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI, bool Strict) { 1229 if (Strict) { 1230 switch (MsgId) { 1231 case ID_SAVEWAVE: 1232 return isVI(STI) || isGFX9Plus(STI); 1233 case ID_STALL_WAVE_GEN: 1234 case ID_HALT_WAVES: 1235 case ID_ORDERED_PS_DONE: 1236 case ID_GS_ALLOC_REQ: 1237 case ID_GET_DOORBELL: 1238 return isGFX9Plus(STI); 1239 case ID_EARLY_PRIM_DEALLOC: 1240 return isGFX9(STI); 1241 case ID_GET_DDID: 1242 return isGFX10Plus(STI); 1243 default: 1244 return 0 <= MsgId && MsgId < ID_GAPS_LAST_ && IdSymbolic[MsgId]; 1245 } 1246 } else { 1247 return 0 <= MsgId && isUInt<ID_WIDTH_>(MsgId); 1248 } 1249 } 1250 1251 StringRef getMsgName(int64_t MsgId) { 1252 assert(0 <= MsgId && MsgId < ID_GAPS_LAST_); 1253 return IdSymbolic[MsgId]; 1254 } 1255 1256 int64_t getMsgOpId(int64_t MsgId, const StringRef Name) { 1257 const char* const *S = (MsgId == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic; 1258 const int F = (MsgId == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_; 1259 const int L = (MsgId == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_; 1260 for (int i = F; i < L; ++i) { 1261 if (Name == S[i]) { 1262 return i; 1263 } 1264 } 1265 return OP_UNKNOWN_; 1266 } 1267 1268 bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI, 1269 bool Strict) { 1270 assert(isValidMsgId(MsgId, STI, Strict)); 1271 1272 if (!Strict) 1273 return 0 <= OpId && isUInt<OP_WIDTH_>(OpId); 1274 1275 switch(MsgId) 1276 { 1277 case ID_GS: 1278 return (OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_) && OpId != OP_GS_NOP; 1279 case ID_GS_DONE: 1280 return OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_; 1281 case ID_SYSMSG: 1282 return OP_SYS_FIRST_ <= OpId && OpId < OP_SYS_LAST_; 1283 default: 1284 return OpId == OP_NONE_; 1285 } 1286 } 1287 1288 StringRef getMsgOpName(int64_t MsgId, int64_t OpId) { 1289 assert(msgRequiresOp(MsgId)); 1290 return (MsgId == ID_SYSMSG)? OpSysSymbolic[OpId] : OpGsSymbolic[OpId]; 1291 } 1292 1293 bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId, 1294 const MCSubtargetInfo &STI, bool Strict) { 1295 assert(isValidMsgOp(MsgId, OpId, STI, Strict)); 1296 1297 if (!Strict) 1298 return 0 <= StreamId && isUInt<STREAM_ID_WIDTH_>(StreamId); 1299 1300 switch(MsgId) 1301 { 1302 case ID_GS: 1303 return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_; 1304 case ID_GS_DONE: 1305 return (OpId == OP_GS_NOP)? 1306 (StreamId == STREAM_ID_NONE_) : 1307 (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_); 1308 default: 1309 return StreamId == STREAM_ID_NONE_; 1310 } 1311 } 1312 1313 bool msgRequiresOp(int64_t MsgId) { 1314 return MsgId == ID_GS || MsgId == ID_GS_DONE || MsgId == ID_SYSMSG; 1315 } 1316 1317 bool msgSupportsStream(int64_t MsgId, int64_t OpId) { 1318 return (MsgId == ID_GS || MsgId == ID_GS_DONE) && OpId != OP_GS_NOP; 1319 } 1320 1321 void decodeMsg(unsigned Val, 1322 uint16_t &MsgId, 1323 uint16_t &OpId, 1324 uint16_t &StreamId) { 1325 MsgId = Val & ID_MASK_; 1326 OpId = (Val & OP_MASK_) >> OP_SHIFT_; 1327 StreamId = (Val & STREAM_ID_MASK_) >> STREAM_ID_SHIFT_; 1328 } 1329 1330 uint64_t encodeMsg(uint64_t MsgId, 1331 uint64_t OpId, 1332 uint64_t StreamId) { 1333 return (MsgId << ID_SHIFT_) | 1334 (OpId << OP_SHIFT_) | 1335 (StreamId << STREAM_ID_SHIFT_); 1336 } 1337 1338 } // namespace SendMsg 1339 1340 //===----------------------------------------------------------------------===// 1341 // 1342 //===----------------------------------------------------------------------===// 1343 1344 unsigned getInitialPSInputAddr(const Function &F) { 1345 return getIntegerAttribute(F, "InitialPSInputAddr", 0); 1346 } 1347 1348 bool getHasColorExport(const Function &F) { 1349 // As a safe default always respond as if PS has color exports. 1350 return getIntegerAttribute( 1351 F, "amdgpu-color-export", 1352 F.getCallingConv() == CallingConv::AMDGPU_PS ? 1 : 0) != 0; 1353 } 1354 1355 bool getHasDepthExport(const Function &F) { 1356 return getIntegerAttribute(F, "amdgpu-depth-export", 0) != 0; 1357 } 1358 1359 bool isShader(CallingConv::ID cc) { 1360 switch(cc) { 1361 case CallingConv::AMDGPU_VS: 1362 case CallingConv::AMDGPU_LS: 1363 case CallingConv::AMDGPU_HS: 1364 case CallingConv::AMDGPU_ES: 1365 case CallingConv::AMDGPU_GS: 1366 case CallingConv::AMDGPU_PS: 1367 case CallingConv::AMDGPU_CS: 1368 return true; 1369 default: 1370 return false; 1371 } 1372 } 1373 1374 bool isGraphics(CallingConv::ID cc) { 1375 return isShader(cc) || cc == CallingConv::AMDGPU_Gfx; 1376 } 1377 1378 bool isCompute(CallingConv::ID cc) { 1379 return !isGraphics(cc) || cc == CallingConv::AMDGPU_CS; 1380 } 1381 1382 bool isEntryFunctionCC(CallingConv::ID CC) { 1383 switch (CC) { 1384 case CallingConv::AMDGPU_KERNEL: 1385 case CallingConv::SPIR_KERNEL: 1386 case CallingConv::AMDGPU_VS: 1387 case CallingConv::AMDGPU_GS: 1388 case CallingConv::AMDGPU_PS: 1389 case CallingConv::AMDGPU_CS: 1390 case CallingConv::AMDGPU_ES: 1391 case CallingConv::AMDGPU_HS: 1392 case CallingConv::AMDGPU_LS: 1393 return true; 1394 default: 1395 return false; 1396 } 1397 } 1398 1399 bool isModuleEntryFunctionCC(CallingConv::ID CC) { 1400 switch (CC) { 1401 case CallingConv::AMDGPU_Gfx: 1402 return true; 1403 default: 1404 return isEntryFunctionCC(CC); 1405 } 1406 } 1407 1408 bool hasXNACK(const MCSubtargetInfo &STI) { 1409 return STI.getFeatureBits()[AMDGPU::FeatureXNACK]; 1410 } 1411 1412 bool hasSRAMECC(const MCSubtargetInfo &STI) { 1413 return STI.getFeatureBits()[AMDGPU::FeatureSRAMECC]; 1414 } 1415 1416 bool hasMIMG_R128(const MCSubtargetInfo &STI) { 1417 return STI.getFeatureBits()[AMDGPU::FeatureMIMG_R128] && !STI.getFeatureBits()[AMDGPU::FeatureR128A16]; 1418 } 1419 1420 bool hasGFX10A16(const MCSubtargetInfo &STI) { 1421 return STI.getFeatureBits()[AMDGPU::FeatureGFX10A16]; 1422 } 1423 1424 bool hasG16(const MCSubtargetInfo &STI) { 1425 return STI.getFeatureBits()[AMDGPU::FeatureG16]; 1426 } 1427 1428 bool hasPackedD16(const MCSubtargetInfo &STI) { 1429 return !STI.getFeatureBits()[AMDGPU::FeatureUnpackedD16VMem]; 1430 } 1431 1432 bool isSI(const MCSubtargetInfo &STI) { 1433 return STI.getFeatureBits()[AMDGPU::FeatureSouthernIslands]; 1434 } 1435 1436 bool isCI(const MCSubtargetInfo &STI) { 1437 return STI.getFeatureBits()[AMDGPU::FeatureSeaIslands]; 1438 } 1439 1440 bool isVI(const MCSubtargetInfo &STI) { 1441 return STI.getFeatureBits()[AMDGPU::FeatureVolcanicIslands]; 1442 } 1443 1444 bool isGFX9(const MCSubtargetInfo &STI) { 1445 return STI.getFeatureBits()[AMDGPU::FeatureGFX9]; 1446 } 1447 1448 bool isGFX9Plus(const MCSubtargetInfo &STI) { 1449 return isGFX9(STI) || isGFX10Plus(STI); 1450 } 1451 1452 bool isGFX10(const MCSubtargetInfo &STI) { 1453 return STI.getFeatureBits()[AMDGPU::FeatureGFX10]; 1454 } 1455 1456 bool isGFX10Plus(const MCSubtargetInfo &STI) { return isGFX10(STI); } 1457 1458 bool isGCN3Encoding(const MCSubtargetInfo &STI) { 1459 return STI.getFeatureBits()[AMDGPU::FeatureGCN3Encoding]; 1460 } 1461 1462 bool isGFX10_AEncoding(const MCSubtargetInfo &STI) { 1463 return STI.getFeatureBits()[AMDGPU::FeatureGFX10_AEncoding]; 1464 } 1465 1466 bool isGFX10_BEncoding(const MCSubtargetInfo &STI) { 1467 return STI.getFeatureBits()[AMDGPU::FeatureGFX10_BEncoding]; 1468 } 1469 1470 bool hasGFX10_3Insts(const MCSubtargetInfo &STI) { 1471 return STI.getFeatureBits()[AMDGPU::FeatureGFX10_3Insts]; 1472 } 1473 1474 bool isGFX90A(const MCSubtargetInfo &STI) { 1475 return STI.getFeatureBits()[AMDGPU::FeatureGFX90AInsts]; 1476 } 1477 1478 bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI) { 1479 return STI.getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch]; 1480 } 1481 1482 bool isSGPR(unsigned Reg, const MCRegisterInfo* TRI) { 1483 const MCRegisterClass SGPRClass = TRI->getRegClass(AMDGPU::SReg_32RegClassID); 1484 const unsigned FirstSubReg = TRI->getSubReg(Reg, AMDGPU::sub0); 1485 return SGPRClass.contains(FirstSubReg != 0 ? FirstSubReg : Reg) || 1486 Reg == AMDGPU::SCC; 1487 } 1488 1489 bool isRegIntersect(unsigned Reg0, unsigned Reg1, const MCRegisterInfo* TRI) { 1490 for (MCRegAliasIterator R(Reg0, TRI, true); R.isValid(); ++R) { 1491 if (*R == Reg1) return true; 1492 } 1493 return false; 1494 } 1495 1496 #define MAP_REG2REG \ 1497 using namespace AMDGPU; \ 1498 switch(Reg) { \ 1499 default: return Reg; \ 1500 CASE_CI_VI(FLAT_SCR) \ 1501 CASE_CI_VI(FLAT_SCR_LO) \ 1502 CASE_CI_VI(FLAT_SCR_HI) \ 1503 CASE_VI_GFX9PLUS(TTMP0) \ 1504 CASE_VI_GFX9PLUS(TTMP1) \ 1505 CASE_VI_GFX9PLUS(TTMP2) \ 1506 CASE_VI_GFX9PLUS(TTMP3) \ 1507 CASE_VI_GFX9PLUS(TTMP4) \ 1508 CASE_VI_GFX9PLUS(TTMP5) \ 1509 CASE_VI_GFX9PLUS(TTMP6) \ 1510 CASE_VI_GFX9PLUS(TTMP7) \ 1511 CASE_VI_GFX9PLUS(TTMP8) \ 1512 CASE_VI_GFX9PLUS(TTMP9) \ 1513 CASE_VI_GFX9PLUS(TTMP10) \ 1514 CASE_VI_GFX9PLUS(TTMP11) \ 1515 CASE_VI_GFX9PLUS(TTMP12) \ 1516 CASE_VI_GFX9PLUS(TTMP13) \ 1517 CASE_VI_GFX9PLUS(TTMP14) \ 1518 CASE_VI_GFX9PLUS(TTMP15) \ 1519 CASE_VI_GFX9PLUS(TTMP0_TTMP1) \ 1520 CASE_VI_GFX9PLUS(TTMP2_TTMP3) \ 1521 CASE_VI_GFX9PLUS(TTMP4_TTMP5) \ 1522 CASE_VI_GFX9PLUS(TTMP6_TTMP7) \ 1523 CASE_VI_GFX9PLUS(TTMP8_TTMP9) \ 1524 CASE_VI_GFX9PLUS(TTMP10_TTMP11) \ 1525 CASE_VI_GFX9PLUS(TTMP12_TTMP13) \ 1526 CASE_VI_GFX9PLUS(TTMP14_TTMP15) \ 1527 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3) \ 1528 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7) \ 1529 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11) \ 1530 CASE_VI_GFX9PLUS(TTMP12_TTMP13_TTMP14_TTMP15) \ 1531 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7) \ 1532 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11) \ 1533 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \ 1534 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \ 1535 } 1536 1537 #define CASE_CI_VI(node) \ 1538 assert(!isSI(STI)); \ 1539 case node: return isCI(STI) ? node##_ci : node##_vi; 1540 1541 #define CASE_VI_GFX9PLUS(node) \ 1542 case node: return isGFX9Plus(STI) ? node##_gfx9plus : node##_vi; 1543 1544 unsigned getMCReg(unsigned Reg, const MCSubtargetInfo &STI) { 1545 if (STI.getTargetTriple().getArch() == Triple::r600) 1546 return Reg; 1547 MAP_REG2REG 1548 } 1549 1550 #undef CASE_CI_VI 1551 #undef CASE_VI_GFX9PLUS 1552 1553 #define CASE_CI_VI(node) case node##_ci: case node##_vi: return node; 1554 #define CASE_VI_GFX9PLUS(node) case node##_vi: case node##_gfx9plus: return node; 1555 1556 unsigned mc2PseudoReg(unsigned Reg) { 1557 MAP_REG2REG 1558 } 1559 1560 #undef CASE_CI_VI 1561 #undef CASE_VI_GFX9PLUS 1562 #undef MAP_REG2REG 1563 1564 bool isSISrcOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1565 assert(OpNo < Desc.NumOperands); 1566 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1567 return OpType >= AMDGPU::OPERAND_SRC_FIRST && 1568 OpType <= AMDGPU::OPERAND_SRC_LAST; 1569 } 1570 1571 bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1572 assert(OpNo < Desc.NumOperands); 1573 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1574 switch (OpType) { 1575 case AMDGPU::OPERAND_REG_IMM_FP32: 1576 case AMDGPU::OPERAND_REG_IMM_FP64: 1577 case AMDGPU::OPERAND_REG_IMM_FP16: 1578 case AMDGPU::OPERAND_REG_IMM_V2FP16: 1579 case AMDGPU::OPERAND_REG_IMM_V2INT16: 1580 case AMDGPU::OPERAND_REG_INLINE_C_FP32: 1581 case AMDGPU::OPERAND_REG_INLINE_C_FP64: 1582 case AMDGPU::OPERAND_REG_INLINE_C_FP16: 1583 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: 1584 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: 1585 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: 1586 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: 1587 case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: 1588 case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16: 1589 case AMDGPU::OPERAND_REG_IMM_V2FP32: 1590 case AMDGPU::OPERAND_REG_INLINE_C_V2FP32: 1591 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: 1592 return true; 1593 default: 1594 return false; 1595 } 1596 } 1597 1598 bool isSISrcInlinableOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1599 assert(OpNo < Desc.NumOperands); 1600 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1601 return OpType >= AMDGPU::OPERAND_REG_INLINE_C_FIRST && 1602 OpType <= AMDGPU::OPERAND_REG_INLINE_C_LAST; 1603 } 1604 1605 // Avoid using MCRegisterClass::getSize, since that function will go away 1606 // (move from MC* level to Target* level). Return size in bits. 1607 unsigned getRegBitWidth(unsigned RCID) { 1608 switch (RCID) { 1609 case AMDGPU::VGPR_LO16RegClassID: 1610 case AMDGPU::VGPR_HI16RegClassID: 1611 case AMDGPU::SGPR_LO16RegClassID: 1612 case AMDGPU::AGPR_LO16RegClassID: 1613 return 16; 1614 case AMDGPU::SGPR_32RegClassID: 1615 case AMDGPU::VGPR_32RegClassID: 1616 case AMDGPU::VRegOrLds_32RegClassID: 1617 case AMDGPU::AGPR_32RegClassID: 1618 case AMDGPU::VS_32RegClassID: 1619 case AMDGPU::AV_32RegClassID: 1620 case AMDGPU::SReg_32RegClassID: 1621 case AMDGPU::SReg_32_XM0RegClassID: 1622 case AMDGPU::SRegOrLds_32RegClassID: 1623 return 32; 1624 case AMDGPU::SGPR_64RegClassID: 1625 case AMDGPU::VS_64RegClassID: 1626 case AMDGPU::AV_64RegClassID: 1627 case AMDGPU::SReg_64RegClassID: 1628 case AMDGPU::VReg_64RegClassID: 1629 case AMDGPU::AReg_64RegClassID: 1630 case AMDGPU::SReg_64_XEXECRegClassID: 1631 case AMDGPU::VReg_64_Align2RegClassID: 1632 case AMDGPU::AReg_64_Align2RegClassID: 1633 return 64; 1634 case AMDGPU::SGPR_96RegClassID: 1635 case AMDGPU::SReg_96RegClassID: 1636 case AMDGPU::VReg_96RegClassID: 1637 case AMDGPU::AReg_96RegClassID: 1638 case AMDGPU::VReg_96_Align2RegClassID: 1639 case AMDGPU::AReg_96_Align2RegClassID: 1640 case AMDGPU::AV_96RegClassID: 1641 return 96; 1642 case AMDGPU::SGPR_128RegClassID: 1643 case AMDGPU::SReg_128RegClassID: 1644 case AMDGPU::VReg_128RegClassID: 1645 case AMDGPU::AReg_128RegClassID: 1646 case AMDGPU::VReg_128_Align2RegClassID: 1647 case AMDGPU::AReg_128_Align2RegClassID: 1648 case AMDGPU::AV_128RegClassID: 1649 return 128; 1650 case AMDGPU::SGPR_160RegClassID: 1651 case AMDGPU::SReg_160RegClassID: 1652 case AMDGPU::VReg_160RegClassID: 1653 case AMDGPU::AReg_160RegClassID: 1654 case AMDGPU::VReg_160_Align2RegClassID: 1655 case AMDGPU::AReg_160_Align2RegClassID: 1656 case AMDGPU::AV_160RegClassID: 1657 return 160; 1658 case AMDGPU::SGPR_192RegClassID: 1659 case AMDGPU::SReg_192RegClassID: 1660 case AMDGPU::VReg_192RegClassID: 1661 case AMDGPU::AReg_192RegClassID: 1662 case AMDGPU::VReg_192_Align2RegClassID: 1663 case AMDGPU::AReg_192_Align2RegClassID: 1664 return 192; 1665 case AMDGPU::SGPR_224RegClassID: 1666 case AMDGPU::SReg_224RegClassID: 1667 case AMDGPU::VReg_224RegClassID: 1668 case AMDGPU::AReg_224RegClassID: 1669 case AMDGPU::VReg_224_Align2RegClassID: 1670 case AMDGPU::AReg_224_Align2RegClassID: 1671 return 224; 1672 case AMDGPU::SGPR_256RegClassID: 1673 case AMDGPU::SReg_256RegClassID: 1674 case AMDGPU::VReg_256RegClassID: 1675 case AMDGPU::AReg_256RegClassID: 1676 case AMDGPU::VReg_256_Align2RegClassID: 1677 case AMDGPU::AReg_256_Align2RegClassID: 1678 return 256; 1679 case AMDGPU::SGPR_512RegClassID: 1680 case AMDGPU::SReg_512RegClassID: 1681 case AMDGPU::VReg_512RegClassID: 1682 case AMDGPU::AReg_512RegClassID: 1683 case AMDGPU::VReg_512_Align2RegClassID: 1684 case AMDGPU::AReg_512_Align2RegClassID: 1685 return 512; 1686 case AMDGPU::SGPR_1024RegClassID: 1687 case AMDGPU::SReg_1024RegClassID: 1688 case AMDGPU::VReg_1024RegClassID: 1689 case AMDGPU::AReg_1024RegClassID: 1690 case AMDGPU::VReg_1024_Align2RegClassID: 1691 case AMDGPU::AReg_1024_Align2RegClassID: 1692 return 1024; 1693 default: 1694 llvm_unreachable("Unexpected register class"); 1695 } 1696 } 1697 1698 unsigned getRegBitWidth(const MCRegisterClass &RC) { 1699 return getRegBitWidth(RC.getID()); 1700 } 1701 1702 unsigned getRegOperandSize(const MCRegisterInfo *MRI, const MCInstrDesc &Desc, 1703 unsigned OpNo) { 1704 assert(OpNo < Desc.NumOperands); 1705 unsigned RCID = Desc.OpInfo[OpNo].RegClass; 1706 return getRegBitWidth(MRI->getRegClass(RCID)) / 8; 1707 } 1708 1709 bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi) { 1710 if (isInlinableIntLiteral(Literal)) 1711 return true; 1712 1713 uint64_t Val = static_cast<uint64_t>(Literal); 1714 return (Val == DoubleToBits(0.0)) || 1715 (Val == DoubleToBits(1.0)) || 1716 (Val == DoubleToBits(-1.0)) || 1717 (Val == DoubleToBits(0.5)) || 1718 (Val == DoubleToBits(-0.5)) || 1719 (Val == DoubleToBits(2.0)) || 1720 (Val == DoubleToBits(-2.0)) || 1721 (Val == DoubleToBits(4.0)) || 1722 (Val == DoubleToBits(-4.0)) || 1723 (Val == 0x3fc45f306dc9c882 && HasInv2Pi); 1724 } 1725 1726 bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi) { 1727 if (isInlinableIntLiteral(Literal)) 1728 return true; 1729 1730 // The actual type of the operand does not seem to matter as long 1731 // as the bits match one of the inline immediate values. For example: 1732 // 1733 // -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal, 1734 // so it is a legal inline immediate. 1735 // 1736 // 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in 1737 // floating-point, so it is a legal inline immediate. 1738 1739 uint32_t Val = static_cast<uint32_t>(Literal); 1740 return (Val == FloatToBits(0.0f)) || 1741 (Val == FloatToBits(1.0f)) || 1742 (Val == FloatToBits(-1.0f)) || 1743 (Val == FloatToBits(0.5f)) || 1744 (Val == FloatToBits(-0.5f)) || 1745 (Val == FloatToBits(2.0f)) || 1746 (Val == FloatToBits(-2.0f)) || 1747 (Val == FloatToBits(4.0f)) || 1748 (Val == FloatToBits(-4.0f)) || 1749 (Val == 0x3e22f983 && HasInv2Pi); 1750 } 1751 1752 bool isInlinableLiteral16(int16_t Literal, bool HasInv2Pi) { 1753 if (!HasInv2Pi) 1754 return false; 1755 1756 if (isInlinableIntLiteral(Literal)) 1757 return true; 1758 1759 uint16_t Val = static_cast<uint16_t>(Literal); 1760 return Val == 0x3C00 || // 1.0 1761 Val == 0xBC00 || // -1.0 1762 Val == 0x3800 || // 0.5 1763 Val == 0xB800 || // -0.5 1764 Val == 0x4000 || // 2.0 1765 Val == 0xC000 || // -2.0 1766 Val == 0x4400 || // 4.0 1767 Val == 0xC400 || // -4.0 1768 Val == 0x3118; // 1/2pi 1769 } 1770 1771 bool isInlinableLiteralV216(int32_t Literal, bool HasInv2Pi) { 1772 assert(HasInv2Pi); 1773 1774 if (isInt<16>(Literal) || isUInt<16>(Literal)) { 1775 int16_t Trunc = static_cast<int16_t>(Literal); 1776 return AMDGPU::isInlinableLiteral16(Trunc, HasInv2Pi); 1777 } 1778 if (!(Literal & 0xffff)) 1779 return AMDGPU::isInlinableLiteral16(Literal >> 16, HasInv2Pi); 1780 1781 int16_t Lo16 = static_cast<int16_t>(Literal); 1782 int16_t Hi16 = static_cast<int16_t>(Literal >> 16); 1783 return Lo16 == Hi16 && isInlinableLiteral16(Lo16, HasInv2Pi); 1784 } 1785 1786 bool isInlinableIntLiteralV216(int32_t Literal) { 1787 int16_t Lo16 = static_cast<int16_t>(Literal); 1788 if (isInt<16>(Literal) || isUInt<16>(Literal)) 1789 return isInlinableIntLiteral(Lo16); 1790 1791 int16_t Hi16 = static_cast<int16_t>(Literal >> 16); 1792 if (!(Literal & 0xffff)) 1793 return isInlinableIntLiteral(Hi16); 1794 return Lo16 == Hi16 && isInlinableIntLiteral(Lo16); 1795 } 1796 1797 bool isFoldableLiteralV216(int32_t Literal, bool HasInv2Pi) { 1798 assert(HasInv2Pi); 1799 1800 int16_t Lo16 = static_cast<int16_t>(Literal); 1801 if (isInt<16>(Literal) || isUInt<16>(Literal)) 1802 return true; 1803 1804 int16_t Hi16 = static_cast<int16_t>(Literal >> 16); 1805 if (!(Literal & 0xffff)) 1806 return true; 1807 return Lo16 == Hi16; 1808 } 1809 1810 bool isArgPassedInSGPR(const Argument *A) { 1811 const Function *F = A->getParent(); 1812 1813 // Arguments to compute shaders are never a source of divergence. 1814 CallingConv::ID CC = F->getCallingConv(); 1815 switch (CC) { 1816 case CallingConv::AMDGPU_KERNEL: 1817 case CallingConv::SPIR_KERNEL: 1818 return true; 1819 case CallingConv::AMDGPU_VS: 1820 case CallingConv::AMDGPU_LS: 1821 case CallingConv::AMDGPU_HS: 1822 case CallingConv::AMDGPU_ES: 1823 case CallingConv::AMDGPU_GS: 1824 case CallingConv::AMDGPU_PS: 1825 case CallingConv::AMDGPU_CS: 1826 case CallingConv::AMDGPU_Gfx: 1827 // For non-compute shaders, SGPR inputs are marked with either inreg or byval. 1828 // Everything else is in VGPRs. 1829 return F->getAttributes().hasParamAttr(A->getArgNo(), Attribute::InReg) || 1830 F->getAttributes().hasParamAttr(A->getArgNo(), Attribute::ByVal); 1831 default: 1832 // TODO: Should calls support inreg for SGPR inputs? 1833 return false; 1834 } 1835 } 1836 1837 static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) { 1838 return isGCN3Encoding(ST) || isGFX10Plus(ST); 1839 } 1840 1841 static bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST) { 1842 return isGFX9Plus(ST); 1843 } 1844 1845 bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST, 1846 int64_t EncodedOffset) { 1847 return hasSMEMByteOffset(ST) ? isUInt<20>(EncodedOffset) 1848 : isUInt<8>(EncodedOffset); 1849 } 1850 1851 bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST, 1852 int64_t EncodedOffset, 1853 bool IsBuffer) { 1854 return !IsBuffer && 1855 hasSMRDSignedImmOffset(ST) && 1856 isInt<21>(EncodedOffset); 1857 } 1858 1859 static bool isDwordAligned(uint64_t ByteOffset) { 1860 return (ByteOffset & 3) == 0; 1861 } 1862 1863 uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST, 1864 uint64_t ByteOffset) { 1865 if (hasSMEMByteOffset(ST)) 1866 return ByteOffset; 1867 1868 assert(isDwordAligned(ByteOffset)); 1869 return ByteOffset >> 2; 1870 } 1871 1872 Optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST, 1873 int64_t ByteOffset, bool IsBuffer) { 1874 // The signed version is always a byte offset. 1875 if (!IsBuffer && hasSMRDSignedImmOffset(ST)) { 1876 assert(hasSMEMByteOffset(ST)); 1877 return isInt<20>(ByteOffset) ? Optional<int64_t>(ByteOffset) : None; 1878 } 1879 1880 if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST)) 1881 return None; 1882 1883 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset); 1884 return isLegalSMRDEncodedUnsignedOffset(ST, EncodedOffset) 1885 ? Optional<int64_t>(EncodedOffset) 1886 : None; 1887 } 1888 1889 Optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST, 1890 int64_t ByteOffset) { 1891 if (!isCI(ST) || !isDwordAligned(ByteOffset)) 1892 return None; 1893 1894 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset); 1895 return isUInt<32>(EncodedOffset) ? Optional<int64_t>(EncodedOffset) : None; 1896 } 1897 1898 unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST, bool Signed) { 1899 // Address offset is 12-bit signed for GFX10, 13-bit for GFX9. 1900 if (AMDGPU::isGFX10(ST)) 1901 return Signed ? 12 : 11; 1902 1903 return Signed ? 13 : 12; 1904 } 1905 1906 // Given Imm, split it into the values to put into the SOffset and ImmOffset 1907 // fields in an MUBUF instruction. Return false if it is not possible (due to a 1908 // hardware bug needing a workaround). 1909 // 1910 // The required alignment ensures that individual address components remain 1911 // aligned if they are aligned to begin with. It also ensures that additional 1912 // offsets within the given alignment can be added to the resulting ImmOffset. 1913 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset, 1914 const GCNSubtarget *Subtarget, Align Alignment) { 1915 const uint32_t MaxImm = alignDown(4095, Alignment.value()); 1916 uint32_t Overflow = 0; 1917 1918 if (Imm > MaxImm) { 1919 if (Imm <= MaxImm + 64) { 1920 // Use an SOffset inline constant for 4..64 1921 Overflow = Imm - MaxImm; 1922 Imm = MaxImm; 1923 } else { 1924 // Try to keep the same value in SOffset for adjacent loads, so that 1925 // the corresponding register contents can be re-used. 1926 // 1927 // Load values with all low-bits (except for alignment bits) set into 1928 // SOffset, so that a larger range of values can be covered using 1929 // s_movk_i32. 1930 // 1931 // Atomic operations fail to work correctly when individual address 1932 // components are unaligned, even if their sum is aligned. 1933 uint32_t High = (Imm + Alignment.value()) & ~4095; 1934 uint32_t Low = (Imm + Alignment.value()) & 4095; 1935 Imm = Low; 1936 Overflow = High - Alignment.value(); 1937 } 1938 } 1939 1940 // There is a hardware bug in SI and CI which prevents address clamping in 1941 // MUBUF instructions from working correctly with SOffsets. The immediate 1942 // offset is unaffected. 1943 if (Overflow > 0 && 1944 Subtarget->getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS) 1945 return false; 1946 1947 ImmOffset = Imm; 1948 SOffset = Overflow; 1949 return true; 1950 } 1951 1952 SIModeRegisterDefaults::SIModeRegisterDefaults(const Function &F) { 1953 *this = getDefaultForCallingConv(F.getCallingConv()); 1954 1955 StringRef IEEEAttr = F.getFnAttribute("amdgpu-ieee").getValueAsString(); 1956 if (!IEEEAttr.empty()) 1957 IEEE = IEEEAttr == "true"; 1958 1959 StringRef DX10ClampAttr 1960 = F.getFnAttribute("amdgpu-dx10-clamp").getValueAsString(); 1961 if (!DX10ClampAttr.empty()) 1962 DX10Clamp = DX10ClampAttr == "true"; 1963 1964 StringRef DenormF32Attr = F.getFnAttribute("denormal-fp-math-f32").getValueAsString(); 1965 if (!DenormF32Attr.empty()) { 1966 DenormalMode DenormMode = parseDenormalFPAttribute(DenormF32Attr); 1967 FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE; 1968 FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE; 1969 } 1970 1971 StringRef DenormAttr = F.getFnAttribute("denormal-fp-math").getValueAsString(); 1972 if (!DenormAttr.empty()) { 1973 DenormalMode DenormMode = parseDenormalFPAttribute(DenormAttr); 1974 1975 if (DenormF32Attr.empty()) { 1976 FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE; 1977 FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE; 1978 } 1979 1980 FP64FP16InputDenormals = DenormMode.Input == DenormalMode::IEEE; 1981 FP64FP16OutputDenormals = DenormMode.Output == DenormalMode::IEEE; 1982 } 1983 } 1984 1985 namespace { 1986 1987 struct SourceOfDivergence { 1988 unsigned Intr; 1989 }; 1990 const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr); 1991 1992 #define GET_SourcesOfDivergence_IMPL 1993 #define GET_Gfx9BufferFormat_IMPL 1994 #define GET_Gfx10PlusBufferFormat_IMPL 1995 #include "AMDGPUGenSearchableTables.inc" 1996 1997 } // end anonymous namespace 1998 1999 bool isIntrinsicSourceOfDivergence(unsigned IntrID) { 2000 return lookupSourceOfDivergence(IntrID); 2001 } 2002 2003 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp, 2004 uint8_t NumComponents, 2005 uint8_t NumFormat, 2006 const MCSubtargetInfo &STI) { 2007 return isGFX10Plus(STI) 2008 ? getGfx10PlusBufferFormatInfo(BitsPerComp, NumComponents, 2009 NumFormat) 2010 : getGfx9BufferFormatInfo(BitsPerComp, NumComponents, NumFormat); 2011 } 2012 2013 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format, 2014 const MCSubtargetInfo &STI) { 2015 return isGFX10Plus(STI) ? getGfx10PlusBufferFormatInfo(Format) 2016 : getGfx9BufferFormatInfo(Format); 2017 } 2018 2019 } // namespace AMDGPU 2020 2021 raw_ostream &operator<<(raw_ostream &OS, 2022 const AMDGPU::IsaInfo::TargetIDSetting S) { 2023 switch (S) { 2024 case (AMDGPU::IsaInfo::TargetIDSetting::Unsupported): 2025 OS << "Unsupported"; 2026 break; 2027 case (AMDGPU::IsaInfo::TargetIDSetting::Any): 2028 OS << "Any"; 2029 break; 2030 case (AMDGPU::IsaInfo::TargetIDSetting::Off): 2031 OS << "Off"; 2032 break; 2033 case (AMDGPU::IsaInfo::TargetIDSetting::On): 2034 OS << "On"; 2035 break; 2036 } 2037 return OS; 2038 } 2039 2040 } // namespace llvm 2041