1 //===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "AMDGPUBaseInfo.h" 10 #include "AMDGPU.h" 11 #include "AMDGPUAsmUtils.h" 12 #include "AMDKernelCodeT.h" 13 #include "GCNSubtarget.h" 14 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 15 #include "llvm/BinaryFormat/ELF.h" 16 #include "llvm/IR/Attributes.h" 17 #include "llvm/IR/Function.h" 18 #include "llvm/IR/GlobalValue.h" 19 #include "llvm/IR/IntrinsicsAMDGPU.h" 20 #include "llvm/IR/IntrinsicsR600.h" 21 #include "llvm/IR/LLVMContext.h" 22 #include "llvm/MC/MCSubtargetInfo.h" 23 #include "llvm/Support/AMDHSAKernelDescriptor.h" 24 #include "llvm/Support/CommandLine.h" 25 #include "llvm/Support/TargetParser.h" 26 27 #define GET_INSTRINFO_NAMED_OPS 28 #define GET_INSTRMAP_INFO 29 #include "AMDGPUGenInstrInfo.inc" 30 31 static llvm::cl::opt<unsigned> AmdhsaCodeObjectVersion( 32 "amdhsa-code-object-version", llvm::cl::Hidden, 33 llvm::cl::desc("AMDHSA Code Object Version"), llvm::cl::init(4), 34 llvm::cl::ZeroOrMore); 35 36 namespace { 37 38 /// \returns Bit mask for given bit \p Shift and bit \p Width. 39 unsigned getBitMask(unsigned Shift, unsigned Width) { 40 return ((1 << Width) - 1) << Shift; 41 } 42 43 /// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width. 44 /// 45 /// \returns Packed \p Dst. 46 unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) { 47 Dst &= ~(1 << Shift) & ~getBitMask(Shift, Width); 48 Dst |= (Src << Shift) & getBitMask(Shift, Width); 49 return Dst; 50 } 51 52 /// Unpacks bits from \p Src for given bit \p Shift and bit \p Width. 53 /// 54 /// \returns Unpacked bits. 55 unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) { 56 return (Src & getBitMask(Shift, Width)) >> Shift; 57 } 58 59 /// \returns Vmcnt bit shift (lower bits). 60 unsigned getVmcntBitShiftLo() { return 0; } 61 62 /// \returns Vmcnt bit width (lower bits). 63 unsigned getVmcntBitWidthLo() { return 4; } 64 65 /// \returns Expcnt bit shift. 66 unsigned getExpcntBitShift() { return 4; } 67 68 /// \returns Expcnt bit width. 69 unsigned getExpcntBitWidth() { return 3; } 70 71 /// \returns Lgkmcnt bit shift. 72 unsigned getLgkmcntBitShift() { return 8; } 73 74 /// \returns Lgkmcnt bit width. 75 unsigned getLgkmcntBitWidth(unsigned VersionMajor) { 76 return (VersionMajor >= 10) ? 6 : 4; 77 } 78 79 /// \returns Vmcnt bit shift (higher bits). 80 unsigned getVmcntBitShiftHi() { return 14; } 81 82 /// \returns Vmcnt bit width (higher bits). 83 unsigned getVmcntBitWidthHi() { return 2; } 84 85 } // end namespace anonymous 86 87 namespace llvm { 88 89 namespace AMDGPU { 90 91 Optional<uint8_t> getHsaAbiVersion(const MCSubtargetInfo *STI) { 92 if (STI && STI->getTargetTriple().getOS() != Triple::AMDHSA) 93 return None; 94 95 switch (AmdhsaCodeObjectVersion) { 96 case 2: 97 return ELF::ELFABIVERSION_AMDGPU_HSA_V2; 98 case 3: 99 return ELF::ELFABIVERSION_AMDGPU_HSA_V3; 100 case 4: 101 return ELF::ELFABIVERSION_AMDGPU_HSA_V4; 102 default: 103 report_fatal_error(Twine("Unsupported AMDHSA Code Object Version ") + 104 Twine(AmdhsaCodeObjectVersion)); 105 } 106 } 107 108 bool isHsaAbiVersion2(const MCSubtargetInfo *STI) { 109 if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI)) 110 return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V2; 111 return false; 112 } 113 114 bool isHsaAbiVersion3(const MCSubtargetInfo *STI) { 115 if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI)) 116 return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V3; 117 return false; 118 } 119 120 bool isHsaAbiVersion4(const MCSubtargetInfo *STI) { 121 if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI)) 122 return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V4; 123 return false; 124 } 125 126 bool isHsaAbiVersion3Or4(const MCSubtargetInfo *STI) { 127 return isHsaAbiVersion3(STI) || isHsaAbiVersion4(STI); 128 } 129 130 #define GET_MIMGBaseOpcodesTable_IMPL 131 #define GET_MIMGDimInfoTable_IMPL 132 #define GET_MIMGInfoTable_IMPL 133 #define GET_MIMGLZMappingTable_IMPL 134 #define GET_MIMGMIPMappingTable_IMPL 135 #define GET_MIMGBiasMappingTable_IMPL 136 #define GET_MIMGOffsetMappingTable_IMPL 137 #define GET_MIMGG16MappingTable_IMPL 138 #include "AMDGPUGenSearchableTables.inc" 139 140 int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding, 141 unsigned VDataDwords, unsigned VAddrDwords) { 142 const MIMGInfo *Info = getMIMGOpcodeHelper(BaseOpcode, MIMGEncoding, 143 VDataDwords, VAddrDwords); 144 return Info ? Info->Opcode : -1; 145 } 146 147 const MIMGBaseOpcodeInfo *getMIMGBaseOpcode(unsigned Opc) { 148 const MIMGInfo *Info = getMIMGInfo(Opc); 149 return Info ? getMIMGBaseOpcodeInfo(Info->BaseOpcode) : nullptr; 150 } 151 152 int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) { 153 const MIMGInfo *OrigInfo = getMIMGInfo(Opc); 154 const MIMGInfo *NewInfo = 155 getMIMGOpcodeHelper(OrigInfo->BaseOpcode, OrigInfo->MIMGEncoding, 156 NewChannels, OrigInfo->VAddrDwords); 157 return NewInfo ? NewInfo->Opcode : -1; 158 } 159 160 unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode, 161 const MIMGDimInfo *Dim, bool IsA16, 162 bool IsG16Supported) { 163 unsigned AddrWords = BaseOpcode->NumExtraArgs; 164 unsigned AddrComponents = (BaseOpcode->Coordinates ? Dim->NumCoords : 0) + 165 (BaseOpcode->LodOrClampOrMip ? 1 : 0); 166 if (IsA16) 167 AddrWords += divideCeil(AddrComponents, 2); 168 else 169 AddrWords += AddrComponents; 170 171 // Note: For subtargets that support A16 but not G16, enabling A16 also 172 // enables 16 bit gradients. 173 // For subtargets that support A16 (operand) and G16 (done with a different 174 // instruction encoding), they are independent. 175 176 if (BaseOpcode->Gradients) { 177 if ((IsA16 && !IsG16Supported) || BaseOpcode->G16) 178 // There are two gradients per coordinate, we pack them separately. 179 // For the 3d case, 180 // we get (dy/du, dx/du) (-, dz/du) (dy/dv, dx/dv) (-, dz/dv) 181 AddrWords += alignTo<2>(Dim->NumGradients / 2); 182 else 183 AddrWords += Dim->NumGradients; 184 } 185 return AddrWords; 186 } 187 188 struct MUBUFInfo { 189 uint16_t Opcode; 190 uint16_t BaseOpcode; 191 uint8_t elements; 192 bool has_vaddr; 193 bool has_srsrc; 194 bool has_soffset; 195 bool IsBufferInv; 196 }; 197 198 struct MTBUFInfo { 199 uint16_t Opcode; 200 uint16_t BaseOpcode; 201 uint8_t elements; 202 bool has_vaddr; 203 bool has_srsrc; 204 bool has_soffset; 205 }; 206 207 struct SMInfo { 208 uint16_t Opcode; 209 bool IsBuffer; 210 }; 211 212 struct VOPInfo { 213 uint16_t Opcode; 214 bool IsSingle; 215 }; 216 217 #define GET_MTBUFInfoTable_DECL 218 #define GET_MTBUFInfoTable_IMPL 219 #define GET_MUBUFInfoTable_DECL 220 #define GET_MUBUFInfoTable_IMPL 221 #define GET_SMInfoTable_DECL 222 #define GET_SMInfoTable_IMPL 223 #define GET_VOP1InfoTable_DECL 224 #define GET_VOP1InfoTable_IMPL 225 #define GET_VOP2InfoTable_DECL 226 #define GET_VOP2InfoTable_IMPL 227 #define GET_VOP3InfoTable_DECL 228 #define GET_VOP3InfoTable_IMPL 229 #include "AMDGPUGenSearchableTables.inc" 230 231 int getMTBUFBaseOpcode(unsigned Opc) { 232 const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opc); 233 return Info ? Info->BaseOpcode : -1; 234 } 235 236 int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements) { 237 const MTBUFInfo *Info = getMTBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements); 238 return Info ? Info->Opcode : -1; 239 } 240 241 int getMTBUFElements(unsigned Opc) { 242 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 243 return Info ? Info->elements : 0; 244 } 245 246 bool getMTBUFHasVAddr(unsigned Opc) { 247 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 248 return Info ? Info->has_vaddr : false; 249 } 250 251 bool getMTBUFHasSrsrc(unsigned Opc) { 252 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 253 return Info ? Info->has_srsrc : false; 254 } 255 256 bool getMTBUFHasSoffset(unsigned Opc) { 257 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 258 return Info ? Info->has_soffset : false; 259 } 260 261 int getMUBUFBaseOpcode(unsigned Opc) { 262 const MUBUFInfo *Info = getMUBUFInfoFromOpcode(Opc); 263 return Info ? Info->BaseOpcode : -1; 264 } 265 266 int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements) { 267 const MUBUFInfo *Info = getMUBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements); 268 return Info ? Info->Opcode : -1; 269 } 270 271 int getMUBUFElements(unsigned Opc) { 272 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 273 return Info ? Info->elements : 0; 274 } 275 276 bool getMUBUFHasVAddr(unsigned Opc) { 277 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 278 return Info ? Info->has_vaddr : false; 279 } 280 281 bool getMUBUFHasSrsrc(unsigned Opc) { 282 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 283 return Info ? Info->has_srsrc : false; 284 } 285 286 bool getMUBUFHasSoffset(unsigned Opc) { 287 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 288 return Info ? Info->has_soffset : false; 289 } 290 291 bool getMUBUFIsBufferInv(unsigned Opc) { 292 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 293 return Info ? Info->IsBufferInv : false; 294 } 295 296 bool getSMEMIsBuffer(unsigned Opc) { 297 const SMInfo *Info = getSMEMOpcodeHelper(Opc); 298 return Info ? Info->IsBuffer : false; 299 } 300 301 bool getVOP1IsSingle(unsigned Opc) { 302 const VOPInfo *Info = getVOP1OpcodeHelper(Opc); 303 return Info ? Info->IsSingle : false; 304 } 305 306 bool getVOP2IsSingle(unsigned Opc) { 307 const VOPInfo *Info = getVOP2OpcodeHelper(Opc); 308 return Info ? Info->IsSingle : false; 309 } 310 311 bool getVOP3IsSingle(unsigned Opc) { 312 const VOPInfo *Info = getVOP3OpcodeHelper(Opc); 313 return Info ? Info->IsSingle : false; 314 } 315 316 // Wrapper for Tablegen'd function. enum Subtarget is not defined in any 317 // header files, so we need to wrap it in a function that takes unsigned 318 // instead. 319 int getMCOpcode(uint16_t Opcode, unsigned Gen) { 320 return getMCOpcodeGen(Opcode, static_cast<Subtarget>(Gen)); 321 } 322 323 namespace IsaInfo { 324 325 AMDGPUTargetID::AMDGPUTargetID(const MCSubtargetInfo &STI) 326 : STI(STI), XnackSetting(TargetIDSetting::Any), 327 SramEccSetting(TargetIDSetting::Any) { 328 if (!STI.getFeatureBits().test(FeatureSupportsXNACK)) 329 XnackSetting = TargetIDSetting::Unsupported; 330 if (!STI.getFeatureBits().test(FeatureSupportsSRAMECC)) 331 SramEccSetting = TargetIDSetting::Unsupported; 332 } 333 334 void AMDGPUTargetID::setTargetIDFromFeaturesString(StringRef FS) { 335 // Check if xnack or sramecc is explicitly enabled or disabled. In the 336 // absence of the target features we assume we must generate code that can run 337 // in any environment. 338 SubtargetFeatures Features(FS); 339 Optional<bool> XnackRequested; 340 Optional<bool> SramEccRequested; 341 342 for (const std::string &Feature : Features.getFeatures()) { 343 if (Feature == "+xnack") 344 XnackRequested = true; 345 else if (Feature == "-xnack") 346 XnackRequested = false; 347 else if (Feature == "+sramecc") 348 SramEccRequested = true; 349 else if (Feature == "-sramecc") 350 SramEccRequested = false; 351 } 352 353 bool XnackSupported = isXnackSupported(); 354 bool SramEccSupported = isSramEccSupported(); 355 356 if (XnackRequested) { 357 if (XnackSupported) { 358 XnackSetting = 359 *XnackRequested ? TargetIDSetting::On : TargetIDSetting::Off; 360 } else { 361 // If a specific xnack setting was requested and this GPU does not support 362 // xnack emit a warning. Setting will remain set to "Unsupported". 363 if (*XnackRequested) { 364 errs() << "warning: xnack 'On' was requested for a processor that does " 365 "not support it!\n"; 366 } else { 367 errs() << "warning: xnack 'Off' was requested for a processor that " 368 "does not support it!\n"; 369 } 370 } 371 } 372 373 if (SramEccRequested) { 374 if (SramEccSupported) { 375 SramEccSetting = 376 *SramEccRequested ? TargetIDSetting::On : TargetIDSetting::Off; 377 } else { 378 // If a specific sramecc setting was requested and this GPU does not 379 // support sramecc emit a warning. Setting will remain set to 380 // "Unsupported". 381 if (*SramEccRequested) { 382 errs() << "warning: sramecc 'On' was requested for a processor that " 383 "does not support it!\n"; 384 } else { 385 errs() << "warning: sramecc 'Off' was requested for a processor that " 386 "does not support it!\n"; 387 } 388 } 389 } 390 } 391 392 static TargetIDSetting 393 getTargetIDSettingFromFeatureString(StringRef FeatureString) { 394 if (FeatureString.endswith("-")) 395 return TargetIDSetting::Off; 396 if (FeatureString.endswith("+")) 397 return TargetIDSetting::On; 398 399 llvm_unreachable("Malformed feature string"); 400 } 401 402 void AMDGPUTargetID::setTargetIDFromTargetIDStream(StringRef TargetID) { 403 SmallVector<StringRef, 3> TargetIDSplit; 404 TargetID.split(TargetIDSplit, ':'); 405 406 for (const auto &FeatureString : TargetIDSplit) { 407 if (FeatureString.startswith("xnack")) 408 XnackSetting = getTargetIDSettingFromFeatureString(FeatureString); 409 if (FeatureString.startswith("sramecc")) 410 SramEccSetting = getTargetIDSettingFromFeatureString(FeatureString); 411 } 412 } 413 414 std::string AMDGPUTargetID::toString() const { 415 std::string StringRep; 416 raw_string_ostream StreamRep(StringRep); 417 418 auto TargetTriple = STI.getTargetTriple(); 419 auto Version = getIsaVersion(STI.getCPU()); 420 421 StreamRep << TargetTriple.getArchName() << '-' 422 << TargetTriple.getVendorName() << '-' 423 << TargetTriple.getOSName() << '-' 424 << TargetTriple.getEnvironmentName() << '-'; 425 426 std::string Processor; 427 // TODO: Following else statement is present here because we used various 428 // alias names for GPUs up until GFX9 (e.g. 'fiji' is same as 'gfx803'). 429 // Remove once all aliases are removed from GCNProcessors.td. 430 if (Version.Major >= 9) 431 Processor = STI.getCPU().str(); 432 else 433 Processor = (Twine("gfx") + Twine(Version.Major) + Twine(Version.Minor) + 434 Twine(Version.Stepping)) 435 .str(); 436 437 std::string Features; 438 if (Optional<uint8_t> HsaAbiVersion = getHsaAbiVersion(&STI)) { 439 switch (*HsaAbiVersion) { 440 case ELF::ELFABIVERSION_AMDGPU_HSA_V2: 441 // Code object V2 only supported specific processors and had fixed 442 // settings for the XNACK. 443 if (Processor == "gfx600") { 444 } else if (Processor == "gfx601") { 445 } else if (Processor == "gfx602") { 446 } else if (Processor == "gfx700") { 447 } else if (Processor == "gfx701") { 448 } else if (Processor == "gfx702") { 449 } else if (Processor == "gfx703") { 450 } else if (Processor == "gfx704") { 451 } else if (Processor == "gfx705") { 452 } else if (Processor == "gfx801") { 453 if (!isXnackOnOrAny()) 454 report_fatal_error( 455 "AMD GPU code object V2 does not support processor " + 456 Twine(Processor) + " without XNACK"); 457 } else if (Processor == "gfx802") { 458 } else if (Processor == "gfx803") { 459 } else if (Processor == "gfx805") { 460 } else if (Processor == "gfx810") { 461 if (!isXnackOnOrAny()) 462 report_fatal_error( 463 "AMD GPU code object V2 does not support processor " + 464 Twine(Processor) + " without XNACK"); 465 } else if (Processor == "gfx900") { 466 if (isXnackOnOrAny()) 467 Processor = "gfx901"; 468 } else if (Processor == "gfx902") { 469 if (isXnackOnOrAny()) 470 Processor = "gfx903"; 471 } else if (Processor == "gfx904") { 472 if (isXnackOnOrAny()) 473 Processor = "gfx905"; 474 } else if (Processor == "gfx906") { 475 if (isXnackOnOrAny()) 476 Processor = "gfx907"; 477 } else if (Processor == "gfx90c") { 478 if (isXnackOnOrAny()) 479 report_fatal_error( 480 "AMD GPU code object V2 does not support processor " + 481 Twine(Processor) + " with XNACK being ON or ANY"); 482 } else { 483 report_fatal_error( 484 "AMD GPU code object V2 does not support processor " + 485 Twine(Processor)); 486 } 487 break; 488 case ELF::ELFABIVERSION_AMDGPU_HSA_V3: 489 // xnack. 490 if (isXnackOnOrAny()) 491 Features += "+xnack"; 492 // In code object v2 and v3, "sramecc" feature was spelled with a 493 // hyphen ("sram-ecc"). 494 if (isSramEccOnOrAny()) 495 Features += "+sram-ecc"; 496 break; 497 case ELF::ELFABIVERSION_AMDGPU_HSA_V4: 498 // sramecc. 499 if (getSramEccSetting() == TargetIDSetting::Off) 500 Features += ":sramecc-"; 501 else if (getSramEccSetting() == TargetIDSetting::On) 502 Features += ":sramecc+"; 503 // xnack. 504 if (getXnackSetting() == TargetIDSetting::Off) 505 Features += ":xnack-"; 506 else if (getXnackSetting() == TargetIDSetting::On) 507 Features += ":xnack+"; 508 break; 509 default: 510 break; 511 } 512 } 513 514 StreamRep << Processor << Features; 515 516 StreamRep.flush(); 517 return StringRep; 518 } 519 520 unsigned getWavefrontSize(const MCSubtargetInfo *STI) { 521 if (STI->getFeatureBits().test(FeatureWavefrontSize16)) 522 return 16; 523 if (STI->getFeatureBits().test(FeatureWavefrontSize32)) 524 return 32; 525 526 return 64; 527 } 528 529 unsigned getLocalMemorySize(const MCSubtargetInfo *STI) { 530 if (STI->getFeatureBits().test(FeatureLocalMemorySize32768)) 531 return 32768; 532 if (STI->getFeatureBits().test(FeatureLocalMemorySize65536)) 533 return 65536; 534 535 return 0; 536 } 537 538 unsigned getEUsPerCU(const MCSubtargetInfo *STI) { 539 // "Per CU" really means "per whatever functional block the waves of a 540 // workgroup must share". For gfx10 in CU mode this is the CU, which contains 541 // two SIMDs. 542 if (isGFX10Plus(*STI) && STI->getFeatureBits().test(FeatureCuMode)) 543 return 2; 544 // Pre-gfx10 a CU contains four SIMDs. For gfx10 in WGP mode the WGP contains 545 // two CUs, so a total of four SIMDs. 546 return 4; 547 } 548 549 unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo *STI, 550 unsigned FlatWorkGroupSize) { 551 assert(FlatWorkGroupSize != 0); 552 if (STI->getTargetTriple().getArch() != Triple::amdgcn) 553 return 8; 554 unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize); 555 if (N == 1) 556 return 40; 557 N = 40 / N; 558 return std::min(N, 16u); 559 } 560 561 unsigned getMinWavesPerEU(const MCSubtargetInfo *STI) { 562 return 1; 563 } 564 565 unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI) { 566 // FIXME: Need to take scratch memory into account. 567 if (isGFX90A(*STI)) 568 return 8; 569 if (!isGFX10Plus(*STI)) 570 return 10; 571 return hasGFX10_3Insts(*STI) ? 16 : 20; 572 } 573 574 unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo *STI, 575 unsigned FlatWorkGroupSize) { 576 return divideCeil(getWavesPerWorkGroup(STI, FlatWorkGroupSize), 577 getEUsPerCU(STI)); 578 } 579 580 unsigned getMinFlatWorkGroupSize(const MCSubtargetInfo *STI) { 581 return 1; 582 } 583 584 unsigned getMaxFlatWorkGroupSize(const MCSubtargetInfo *STI) { 585 // Some subtargets allow encoding 2048, but this isn't tested or supported. 586 return 1024; 587 } 588 589 unsigned getWavesPerWorkGroup(const MCSubtargetInfo *STI, 590 unsigned FlatWorkGroupSize) { 591 return divideCeil(FlatWorkGroupSize, getWavefrontSize(STI)); 592 } 593 594 unsigned getSGPRAllocGranule(const MCSubtargetInfo *STI) { 595 IsaVersion Version = getIsaVersion(STI->getCPU()); 596 if (Version.Major >= 10) 597 return getAddressableNumSGPRs(STI); 598 if (Version.Major >= 8) 599 return 16; 600 return 8; 601 } 602 603 unsigned getSGPREncodingGranule(const MCSubtargetInfo *STI) { 604 return 8; 605 } 606 607 unsigned getTotalNumSGPRs(const MCSubtargetInfo *STI) { 608 IsaVersion Version = getIsaVersion(STI->getCPU()); 609 if (Version.Major >= 8) 610 return 800; 611 return 512; 612 } 613 614 unsigned getAddressableNumSGPRs(const MCSubtargetInfo *STI) { 615 if (STI->getFeatureBits().test(FeatureSGPRInitBug)) 616 return FIXED_NUM_SGPRS_FOR_INIT_BUG; 617 618 IsaVersion Version = getIsaVersion(STI->getCPU()); 619 if (Version.Major >= 10) 620 return 106; 621 if (Version.Major >= 8) 622 return 102; 623 return 104; 624 } 625 626 unsigned getMinNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 627 assert(WavesPerEU != 0); 628 629 IsaVersion Version = getIsaVersion(STI->getCPU()); 630 if (Version.Major >= 10) 631 return 0; 632 633 if (WavesPerEU >= getMaxWavesPerEU(STI)) 634 return 0; 635 636 unsigned MinNumSGPRs = getTotalNumSGPRs(STI) / (WavesPerEU + 1); 637 if (STI->getFeatureBits().test(FeatureTrapHandler)) 638 MinNumSGPRs -= std::min(MinNumSGPRs, (unsigned)TRAP_NUM_SGPRS); 639 MinNumSGPRs = alignDown(MinNumSGPRs, getSGPRAllocGranule(STI)) + 1; 640 return std::min(MinNumSGPRs, getAddressableNumSGPRs(STI)); 641 } 642 643 unsigned getMaxNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU, 644 bool Addressable) { 645 assert(WavesPerEU != 0); 646 647 unsigned AddressableNumSGPRs = getAddressableNumSGPRs(STI); 648 IsaVersion Version = getIsaVersion(STI->getCPU()); 649 if (Version.Major >= 10) 650 return Addressable ? AddressableNumSGPRs : 108; 651 if (Version.Major >= 8 && !Addressable) 652 AddressableNumSGPRs = 112; 653 unsigned MaxNumSGPRs = getTotalNumSGPRs(STI) / WavesPerEU; 654 if (STI->getFeatureBits().test(FeatureTrapHandler)) 655 MaxNumSGPRs -= std::min(MaxNumSGPRs, (unsigned)TRAP_NUM_SGPRS); 656 MaxNumSGPRs = alignDown(MaxNumSGPRs, getSGPRAllocGranule(STI)); 657 return std::min(MaxNumSGPRs, AddressableNumSGPRs); 658 } 659 660 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed, 661 bool FlatScrUsed, bool XNACKUsed) { 662 unsigned ExtraSGPRs = 0; 663 if (VCCUsed) 664 ExtraSGPRs = 2; 665 666 IsaVersion Version = getIsaVersion(STI->getCPU()); 667 if (Version.Major >= 10) 668 return ExtraSGPRs; 669 670 if (Version.Major < 8) { 671 if (FlatScrUsed) 672 ExtraSGPRs = 4; 673 } else { 674 if (XNACKUsed) 675 ExtraSGPRs = 4; 676 677 if (FlatScrUsed || 678 STI->getFeatureBits().test(AMDGPU::FeatureArchitectedFlatScratch)) 679 ExtraSGPRs = 6; 680 } 681 682 return ExtraSGPRs; 683 } 684 685 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed, 686 bool FlatScrUsed) { 687 return getNumExtraSGPRs(STI, VCCUsed, FlatScrUsed, 688 STI->getFeatureBits().test(AMDGPU::FeatureXNACK)); 689 } 690 691 unsigned getNumSGPRBlocks(const MCSubtargetInfo *STI, unsigned NumSGPRs) { 692 NumSGPRs = alignTo(std::max(1u, NumSGPRs), getSGPREncodingGranule(STI)); 693 // SGPRBlocks is actual number of SGPR blocks minus 1. 694 return NumSGPRs / getSGPREncodingGranule(STI) - 1; 695 } 696 697 unsigned getVGPRAllocGranule(const MCSubtargetInfo *STI, 698 Optional<bool> EnableWavefrontSize32) { 699 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 700 return 8; 701 702 bool IsWave32 = EnableWavefrontSize32 ? 703 *EnableWavefrontSize32 : 704 STI->getFeatureBits().test(FeatureWavefrontSize32); 705 706 if (hasGFX10_3Insts(*STI)) 707 return IsWave32 ? 16 : 8; 708 709 return IsWave32 ? 8 : 4; 710 } 711 712 unsigned getVGPREncodingGranule(const MCSubtargetInfo *STI, 713 Optional<bool> EnableWavefrontSize32) { 714 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 715 return 8; 716 717 bool IsWave32 = EnableWavefrontSize32 ? 718 *EnableWavefrontSize32 : 719 STI->getFeatureBits().test(FeatureWavefrontSize32); 720 721 return IsWave32 ? 8 : 4; 722 } 723 724 unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) { 725 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 726 return 512; 727 if (!isGFX10Plus(*STI)) 728 return 256; 729 return STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1024 : 512; 730 } 731 732 unsigned getAddressableNumVGPRs(const MCSubtargetInfo *STI) { 733 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 734 return 512; 735 return 256; 736 } 737 738 unsigned getMinNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 739 assert(WavesPerEU != 0); 740 741 if (WavesPerEU >= getMaxWavesPerEU(STI)) 742 return 0; 743 unsigned MinNumVGPRs = 744 alignDown(getTotalNumVGPRs(STI) / (WavesPerEU + 1), 745 getVGPRAllocGranule(STI)) + 1; 746 return std::min(MinNumVGPRs, getAddressableNumVGPRs(STI)); 747 } 748 749 unsigned getMaxNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 750 assert(WavesPerEU != 0); 751 752 unsigned MaxNumVGPRs = alignDown(getTotalNumVGPRs(STI) / WavesPerEU, 753 getVGPRAllocGranule(STI)); 754 unsigned AddressableNumVGPRs = getAddressableNumVGPRs(STI); 755 return std::min(MaxNumVGPRs, AddressableNumVGPRs); 756 } 757 758 unsigned getNumVGPRBlocks(const MCSubtargetInfo *STI, unsigned NumVGPRs, 759 Optional<bool> EnableWavefrontSize32) { 760 NumVGPRs = alignTo(std::max(1u, NumVGPRs), 761 getVGPREncodingGranule(STI, EnableWavefrontSize32)); 762 // VGPRBlocks is actual number of VGPR blocks minus 1. 763 return NumVGPRs / getVGPREncodingGranule(STI, EnableWavefrontSize32) - 1; 764 } 765 766 } // end namespace IsaInfo 767 768 void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header, 769 const MCSubtargetInfo *STI) { 770 IsaVersion Version = getIsaVersion(STI->getCPU()); 771 772 memset(&Header, 0, sizeof(Header)); 773 774 Header.amd_kernel_code_version_major = 1; 775 Header.amd_kernel_code_version_minor = 2; 776 Header.amd_machine_kind = 1; // AMD_MACHINE_KIND_AMDGPU 777 Header.amd_machine_version_major = Version.Major; 778 Header.amd_machine_version_minor = Version.Minor; 779 Header.amd_machine_version_stepping = Version.Stepping; 780 Header.kernel_code_entry_byte_offset = sizeof(Header); 781 Header.wavefront_size = 6; 782 783 // If the code object does not support indirect functions, then the value must 784 // be 0xffffffff. 785 Header.call_convention = -1; 786 787 // These alignment values are specified in powers of two, so alignment = 788 // 2^n. The minimum alignment is 2^4 = 16. 789 Header.kernarg_segment_alignment = 4; 790 Header.group_segment_alignment = 4; 791 Header.private_segment_alignment = 4; 792 793 if (Version.Major >= 10) { 794 if (STI->getFeatureBits().test(FeatureWavefrontSize32)) { 795 Header.wavefront_size = 5; 796 Header.code_properties |= AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32; 797 } 798 Header.compute_pgm_resource_registers |= 799 S_00B848_WGP_MODE(STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1) | 800 S_00B848_MEM_ORDERED(1); 801 } 802 } 803 804 amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor( 805 const MCSubtargetInfo *STI) { 806 IsaVersion Version = getIsaVersion(STI->getCPU()); 807 808 amdhsa::kernel_descriptor_t KD; 809 memset(&KD, 0, sizeof(KD)); 810 811 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 812 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, 813 amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE); 814 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 815 amdhsa::COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP, 1); 816 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 817 amdhsa::COMPUTE_PGM_RSRC1_ENABLE_IEEE_MODE, 1); 818 AMDHSA_BITS_SET(KD.compute_pgm_rsrc2, 819 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, 1); 820 if (Version.Major >= 10) { 821 AMDHSA_BITS_SET(KD.kernel_code_properties, 822 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, 823 STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1 : 0); 824 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 825 amdhsa::COMPUTE_PGM_RSRC1_WGP_MODE, 826 STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1); 827 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 828 amdhsa::COMPUTE_PGM_RSRC1_MEM_ORDERED, 1); 829 } 830 if (AMDGPU::isGFX90A(*STI)) { 831 AMDHSA_BITS_SET(KD.compute_pgm_rsrc3, 832 amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, 833 STI->getFeatureBits().test(FeatureTgSplit) ? 1 : 0); 834 } 835 return KD; 836 } 837 838 bool isGroupSegment(const GlobalValue *GV) { 839 return GV->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS; 840 } 841 842 bool isGlobalSegment(const GlobalValue *GV) { 843 return GV->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS; 844 } 845 846 bool isReadOnlySegment(const GlobalValue *GV) { 847 unsigned AS = GV->getAddressSpace(); 848 return AS == AMDGPUAS::CONSTANT_ADDRESS || 849 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT; 850 } 851 852 bool shouldEmitConstantsToTextSection(const Triple &TT) { 853 return TT.getArch() == Triple::r600; 854 } 855 856 int getIntegerAttribute(const Function &F, StringRef Name, int Default) { 857 Attribute A = F.getFnAttribute(Name); 858 int Result = Default; 859 860 if (A.isStringAttribute()) { 861 StringRef Str = A.getValueAsString(); 862 if (Str.getAsInteger(0, Result)) { 863 LLVMContext &Ctx = F.getContext(); 864 Ctx.emitError("can't parse integer attribute " + Name); 865 } 866 } 867 868 return Result; 869 } 870 871 std::pair<int, int> getIntegerPairAttribute(const Function &F, 872 StringRef Name, 873 std::pair<int, int> Default, 874 bool OnlyFirstRequired) { 875 Attribute A = F.getFnAttribute(Name); 876 if (!A.isStringAttribute()) 877 return Default; 878 879 LLVMContext &Ctx = F.getContext(); 880 std::pair<int, int> Ints = Default; 881 std::pair<StringRef, StringRef> Strs = A.getValueAsString().split(','); 882 if (Strs.first.trim().getAsInteger(0, Ints.first)) { 883 Ctx.emitError("can't parse first integer attribute " + Name); 884 return Default; 885 } 886 if (Strs.second.trim().getAsInteger(0, Ints.second)) { 887 if (!OnlyFirstRequired || !Strs.second.trim().empty()) { 888 Ctx.emitError("can't parse second integer attribute " + Name); 889 return Default; 890 } 891 } 892 893 return Ints; 894 } 895 896 unsigned getVmcntBitMask(const IsaVersion &Version) { 897 unsigned VmcntLo = (1 << getVmcntBitWidthLo()) - 1; 898 if (Version.Major < 9) 899 return VmcntLo; 900 901 unsigned VmcntHi = ((1 << getVmcntBitWidthHi()) - 1) << getVmcntBitWidthLo(); 902 return VmcntLo | VmcntHi; 903 } 904 905 unsigned getExpcntBitMask(const IsaVersion &Version) { 906 return (1 << getExpcntBitWidth()) - 1; 907 } 908 909 unsigned getLgkmcntBitMask(const IsaVersion &Version) { 910 return (1 << getLgkmcntBitWidth(Version.Major)) - 1; 911 } 912 913 unsigned getWaitcntBitMask(const IsaVersion &Version) { 914 unsigned VmcntLo = getBitMask(getVmcntBitShiftLo(), getVmcntBitWidthLo()); 915 unsigned Expcnt = getBitMask(getExpcntBitShift(), getExpcntBitWidth()); 916 unsigned Lgkmcnt = getBitMask(getLgkmcntBitShift(), 917 getLgkmcntBitWidth(Version.Major)); 918 unsigned Waitcnt = VmcntLo | Expcnt | Lgkmcnt; 919 if (Version.Major < 9) 920 return Waitcnt; 921 922 unsigned VmcntHi = getBitMask(getVmcntBitShiftHi(), getVmcntBitWidthHi()); 923 return Waitcnt | VmcntHi; 924 } 925 926 unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) { 927 unsigned VmcntLo = 928 unpackBits(Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo()); 929 if (Version.Major < 9) 930 return VmcntLo; 931 932 unsigned VmcntHi = 933 unpackBits(Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi()); 934 VmcntHi <<= getVmcntBitWidthLo(); 935 return VmcntLo | VmcntHi; 936 } 937 938 unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) { 939 return unpackBits(Waitcnt, getExpcntBitShift(), getExpcntBitWidth()); 940 } 941 942 unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) { 943 return unpackBits(Waitcnt, getLgkmcntBitShift(), 944 getLgkmcntBitWidth(Version.Major)); 945 } 946 947 void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, 948 unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt) { 949 Vmcnt = decodeVmcnt(Version, Waitcnt); 950 Expcnt = decodeExpcnt(Version, Waitcnt); 951 Lgkmcnt = decodeLgkmcnt(Version, Waitcnt); 952 } 953 954 Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded) { 955 Waitcnt Decoded; 956 Decoded.VmCnt = decodeVmcnt(Version, Encoded); 957 Decoded.ExpCnt = decodeExpcnt(Version, Encoded); 958 Decoded.LgkmCnt = decodeLgkmcnt(Version, Encoded); 959 return Decoded; 960 } 961 962 unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt, 963 unsigned Vmcnt) { 964 Waitcnt = 965 packBits(Vmcnt, Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo()); 966 if (Version.Major < 9) 967 return Waitcnt; 968 969 Vmcnt >>= getVmcntBitWidthLo(); 970 return packBits(Vmcnt, Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi()); 971 } 972 973 unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt, 974 unsigned Expcnt) { 975 return packBits(Expcnt, Waitcnt, getExpcntBitShift(), getExpcntBitWidth()); 976 } 977 978 unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt, 979 unsigned Lgkmcnt) { 980 return packBits(Lgkmcnt, Waitcnt, getLgkmcntBitShift(), 981 getLgkmcntBitWidth(Version.Major)); 982 } 983 984 unsigned encodeWaitcnt(const IsaVersion &Version, 985 unsigned Vmcnt, unsigned Expcnt, unsigned Lgkmcnt) { 986 unsigned Waitcnt = getWaitcntBitMask(Version); 987 Waitcnt = encodeVmcnt(Version, Waitcnt, Vmcnt); 988 Waitcnt = encodeExpcnt(Version, Waitcnt, Expcnt); 989 Waitcnt = encodeLgkmcnt(Version, Waitcnt, Lgkmcnt); 990 return Waitcnt; 991 } 992 993 unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded) { 994 return encodeWaitcnt(Version, Decoded.VmCnt, Decoded.ExpCnt, Decoded.LgkmCnt); 995 } 996 997 //===----------------------------------------------------------------------===// 998 // hwreg 999 //===----------------------------------------------------------------------===// 1000 1001 namespace Hwreg { 1002 1003 int64_t getHwregId(const StringRef Name) { 1004 for (int Id = ID_SYMBOLIC_FIRST_; Id < ID_SYMBOLIC_LAST_; ++Id) { 1005 if (IdSymbolic[Id] && Name == IdSymbolic[Id]) 1006 return Id; 1007 } 1008 return ID_UNKNOWN_; 1009 } 1010 1011 static unsigned getLastSymbolicHwreg(const MCSubtargetInfo &STI) { 1012 if (isSI(STI) || isCI(STI) || isVI(STI)) 1013 return ID_SYMBOLIC_FIRST_GFX9_; 1014 else if (isGFX9(STI)) 1015 return ID_SYMBOLIC_FIRST_GFX10_; 1016 else if (isGFX10(STI) && !isGFX10_BEncoding(STI)) 1017 return ID_SYMBOLIC_FIRST_GFX1030_; 1018 else 1019 return ID_SYMBOLIC_LAST_; 1020 } 1021 1022 bool isValidHwreg(int64_t Id, const MCSubtargetInfo &STI) { 1023 switch (Id) { 1024 case ID_HW_ID: 1025 return isSI(STI) || isCI(STI) || isVI(STI) || isGFX9(STI); 1026 case ID_HW_ID1: 1027 case ID_HW_ID2: 1028 return isGFX10Plus(STI); 1029 case ID_XNACK_MASK: 1030 return isGFX10(STI) && !AMDGPU::isGFX10_BEncoding(STI); 1031 default: 1032 return ID_SYMBOLIC_FIRST_ <= Id && Id < getLastSymbolicHwreg(STI) && 1033 IdSymbolic[Id]; 1034 } 1035 } 1036 1037 bool isValidHwreg(int64_t Id) { 1038 return 0 <= Id && isUInt<ID_WIDTH_>(Id); 1039 } 1040 1041 bool isValidHwregOffset(int64_t Offset) { 1042 return 0 <= Offset && isUInt<OFFSET_WIDTH_>(Offset); 1043 } 1044 1045 bool isValidHwregWidth(int64_t Width) { 1046 return 0 <= (Width - 1) && isUInt<WIDTH_M1_WIDTH_>(Width - 1); 1047 } 1048 1049 uint64_t encodeHwreg(uint64_t Id, uint64_t Offset, uint64_t Width) { 1050 return (Id << ID_SHIFT_) | 1051 (Offset << OFFSET_SHIFT_) | 1052 ((Width - 1) << WIDTH_M1_SHIFT_); 1053 } 1054 1055 StringRef getHwreg(unsigned Id, const MCSubtargetInfo &STI) { 1056 return isValidHwreg(Id, STI) ? IdSymbolic[Id] : ""; 1057 } 1058 1059 void decodeHwreg(unsigned Val, unsigned &Id, unsigned &Offset, unsigned &Width) { 1060 Id = (Val & ID_MASK_) >> ID_SHIFT_; 1061 Offset = (Val & OFFSET_MASK_) >> OFFSET_SHIFT_; 1062 Width = ((Val & WIDTH_M1_MASK_) >> WIDTH_M1_SHIFT_) + 1; 1063 } 1064 1065 } // namespace Hwreg 1066 1067 //===----------------------------------------------------------------------===// 1068 // exp tgt 1069 //===----------------------------------------------------------------------===// 1070 1071 namespace Exp { 1072 1073 struct ExpTgt { 1074 StringLiteral Name; 1075 unsigned Tgt; 1076 unsigned MaxIndex; 1077 }; 1078 1079 static constexpr ExpTgt ExpTgtInfo[] = { 1080 {{"null"}, ET_NULL, ET_NULL_MAX_IDX}, 1081 {{"mrtz"}, ET_MRTZ, ET_MRTZ_MAX_IDX}, 1082 {{"prim"}, ET_PRIM, ET_PRIM_MAX_IDX}, 1083 {{"mrt"}, ET_MRT0, ET_MRT_MAX_IDX}, 1084 {{"pos"}, ET_POS0, ET_POS_MAX_IDX}, 1085 {{"param"}, ET_PARAM0, ET_PARAM_MAX_IDX}, 1086 }; 1087 1088 bool getTgtName(unsigned Id, StringRef &Name, int &Index) { 1089 for (const ExpTgt &Val : ExpTgtInfo) { 1090 if (Val.Tgt <= Id && Id <= Val.Tgt + Val.MaxIndex) { 1091 Index = (Val.MaxIndex == 0) ? -1 : (Id - Val.Tgt); 1092 Name = Val.Name; 1093 return true; 1094 } 1095 } 1096 return false; 1097 } 1098 1099 unsigned getTgtId(const StringRef Name) { 1100 1101 for (const ExpTgt &Val : ExpTgtInfo) { 1102 if (Val.MaxIndex == 0 && Name == Val.Name) 1103 return Val.Tgt; 1104 1105 if (Val.MaxIndex > 0 && Name.startswith(Val.Name)) { 1106 StringRef Suffix = Name.drop_front(Val.Name.size()); 1107 1108 unsigned Id; 1109 if (Suffix.getAsInteger(10, Id) || Id > Val.MaxIndex) 1110 return ET_INVALID; 1111 1112 // Disable leading zeroes 1113 if (Suffix.size() > 1 && Suffix[0] == '0') 1114 return ET_INVALID; 1115 1116 return Val.Tgt + Id; 1117 } 1118 } 1119 return ET_INVALID; 1120 } 1121 1122 bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI) { 1123 return (Id != ET_POS4 && Id != ET_PRIM) || isGFX10Plus(STI); 1124 } 1125 1126 } // namespace Exp 1127 1128 //===----------------------------------------------------------------------===// 1129 // MTBUF Format 1130 //===----------------------------------------------------------------------===// 1131 1132 namespace MTBUFFormat { 1133 1134 int64_t getDfmt(const StringRef Name) { 1135 for (int Id = DFMT_MIN; Id <= DFMT_MAX; ++Id) { 1136 if (Name == DfmtSymbolic[Id]) 1137 return Id; 1138 } 1139 return DFMT_UNDEF; 1140 } 1141 1142 StringRef getDfmtName(unsigned Id) { 1143 assert(Id <= DFMT_MAX); 1144 return DfmtSymbolic[Id]; 1145 } 1146 1147 static StringLiteral const *getNfmtLookupTable(const MCSubtargetInfo &STI) { 1148 if (isSI(STI) || isCI(STI)) 1149 return NfmtSymbolicSICI; 1150 if (isVI(STI) || isGFX9(STI)) 1151 return NfmtSymbolicVI; 1152 return NfmtSymbolicGFX10; 1153 } 1154 1155 int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI) { 1156 auto lookupTable = getNfmtLookupTable(STI); 1157 for (int Id = NFMT_MIN; Id <= NFMT_MAX; ++Id) { 1158 if (Name == lookupTable[Id]) 1159 return Id; 1160 } 1161 return NFMT_UNDEF; 1162 } 1163 1164 StringRef getNfmtName(unsigned Id, const MCSubtargetInfo &STI) { 1165 assert(Id <= NFMT_MAX); 1166 return getNfmtLookupTable(STI)[Id]; 1167 } 1168 1169 bool isValidDfmtNfmt(unsigned Id, const MCSubtargetInfo &STI) { 1170 unsigned Dfmt; 1171 unsigned Nfmt; 1172 decodeDfmtNfmt(Id, Dfmt, Nfmt); 1173 return isValidNfmt(Nfmt, STI); 1174 } 1175 1176 bool isValidNfmt(unsigned Id, const MCSubtargetInfo &STI) { 1177 return !getNfmtName(Id, STI).empty(); 1178 } 1179 1180 int64_t encodeDfmtNfmt(unsigned Dfmt, unsigned Nfmt) { 1181 return (Dfmt << DFMT_SHIFT) | (Nfmt << NFMT_SHIFT); 1182 } 1183 1184 void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt) { 1185 Dfmt = (Format >> DFMT_SHIFT) & DFMT_MASK; 1186 Nfmt = (Format >> NFMT_SHIFT) & NFMT_MASK; 1187 } 1188 1189 int64_t getUnifiedFormat(const StringRef Name) { 1190 for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) { 1191 if (Name == UfmtSymbolic[Id]) 1192 return Id; 1193 } 1194 return UFMT_UNDEF; 1195 } 1196 1197 StringRef getUnifiedFormatName(unsigned Id) { 1198 return isValidUnifiedFormat(Id) ? UfmtSymbolic[Id] : ""; 1199 } 1200 1201 bool isValidUnifiedFormat(unsigned Id) { 1202 return Id <= UFMT_LAST; 1203 } 1204 1205 int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt) { 1206 int64_t Fmt = encodeDfmtNfmt(Dfmt, Nfmt); 1207 for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) { 1208 if (Fmt == DfmtNfmt2UFmt[Id]) 1209 return Id; 1210 } 1211 return UFMT_UNDEF; 1212 } 1213 1214 bool isValidFormatEncoding(unsigned Val, const MCSubtargetInfo &STI) { 1215 return isGFX10Plus(STI) ? (Val <= UFMT_MAX) : (Val <= DFMT_NFMT_MAX); 1216 } 1217 1218 unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI) { 1219 if (isGFX10Plus(STI)) 1220 return UFMT_DEFAULT; 1221 return DFMT_NFMT_DEFAULT; 1222 } 1223 1224 } // namespace MTBUFFormat 1225 1226 //===----------------------------------------------------------------------===// 1227 // SendMsg 1228 //===----------------------------------------------------------------------===// 1229 1230 namespace SendMsg { 1231 1232 int64_t getMsgId(const StringRef Name) { 1233 for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) { 1234 if (IdSymbolic[i] && Name == IdSymbolic[i]) 1235 return i; 1236 } 1237 return ID_UNKNOWN_; 1238 } 1239 1240 bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI, bool Strict) { 1241 if (Strict) { 1242 switch (MsgId) { 1243 case ID_SAVEWAVE: 1244 return isVI(STI) || isGFX9Plus(STI); 1245 case ID_STALL_WAVE_GEN: 1246 case ID_HALT_WAVES: 1247 case ID_ORDERED_PS_DONE: 1248 case ID_GS_ALLOC_REQ: 1249 case ID_GET_DOORBELL: 1250 return isGFX9Plus(STI); 1251 case ID_EARLY_PRIM_DEALLOC: 1252 return isGFX9(STI); 1253 case ID_GET_DDID: 1254 return isGFX10Plus(STI); 1255 default: 1256 return 0 <= MsgId && MsgId < ID_GAPS_LAST_ && IdSymbolic[MsgId]; 1257 } 1258 } else { 1259 return 0 <= MsgId && isUInt<ID_WIDTH_>(MsgId); 1260 } 1261 } 1262 1263 StringRef getMsgName(int64_t MsgId) { 1264 assert(0 <= MsgId && MsgId < ID_GAPS_LAST_); 1265 return IdSymbolic[MsgId]; 1266 } 1267 1268 int64_t getMsgOpId(int64_t MsgId, const StringRef Name) { 1269 const char* const *S = (MsgId == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic; 1270 const int F = (MsgId == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_; 1271 const int L = (MsgId == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_; 1272 for (int i = F; i < L; ++i) { 1273 if (Name == S[i]) { 1274 return i; 1275 } 1276 } 1277 return OP_UNKNOWN_; 1278 } 1279 1280 bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI, 1281 bool Strict) { 1282 assert(isValidMsgId(MsgId, STI, Strict)); 1283 1284 if (!Strict) 1285 return 0 <= OpId && isUInt<OP_WIDTH_>(OpId); 1286 1287 switch(MsgId) 1288 { 1289 case ID_GS: 1290 return (OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_) && OpId != OP_GS_NOP; 1291 case ID_GS_DONE: 1292 return OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_; 1293 case ID_SYSMSG: 1294 return OP_SYS_FIRST_ <= OpId && OpId < OP_SYS_LAST_; 1295 default: 1296 return OpId == OP_NONE_; 1297 } 1298 } 1299 1300 StringRef getMsgOpName(int64_t MsgId, int64_t OpId) { 1301 assert(msgRequiresOp(MsgId)); 1302 return (MsgId == ID_SYSMSG)? OpSysSymbolic[OpId] : OpGsSymbolic[OpId]; 1303 } 1304 1305 bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId, 1306 const MCSubtargetInfo &STI, bool Strict) { 1307 assert(isValidMsgOp(MsgId, OpId, STI, Strict)); 1308 1309 if (!Strict) 1310 return 0 <= StreamId && isUInt<STREAM_ID_WIDTH_>(StreamId); 1311 1312 switch(MsgId) 1313 { 1314 case ID_GS: 1315 return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_; 1316 case ID_GS_DONE: 1317 return (OpId == OP_GS_NOP)? 1318 (StreamId == STREAM_ID_NONE_) : 1319 (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_); 1320 default: 1321 return StreamId == STREAM_ID_NONE_; 1322 } 1323 } 1324 1325 bool msgRequiresOp(int64_t MsgId) { 1326 return MsgId == ID_GS || MsgId == ID_GS_DONE || MsgId == ID_SYSMSG; 1327 } 1328 1329 bool msgSupportsStream(int64_t MsgId, int64_t OpId) { 1330 return (MsgId == ID_GS || MsgId == ID_GS_DONE) && OpId != OP_GS_NOP; 1331 } 1332 1333 void decodeMsg(unsigned Val, 1334 uint16_t &MsgId, 1335 uint16_t &OpId, 1336 uint16_t &StreamId) { 1337 MsgId = Val & ID_MASK_; 1338 OpId = (Val & OP_MASK_) >> OP_SHIFT_; 1339 StreamId = (Val & STREAM_ID_MASK_) >> STREAM_ID_SHIFT_; 1340 } 1341 1342 uint64_t encodeMsg(uint64_t MsgId, 1343 uint64_t OpId, 1344 uint64_t StreamId) { 1345 return (MsgId << ID_SHIFT_) | 1346 (OpId << OP_SHIFT_) | 1347 (StreamId << STREAM_ID_SHIFT_); 1348 } 1349 1350 } // namespace SendMsg 1351 1352 //===----------------------------------------------------------------------===// 1353 // 1354 //===----------------------------------------------------------------------===// 1355 1356 unsigned getInitialPSInputAddr(const Function &F) { 1357 return getIntegerAttribute(F, "InitialPSInputAddr", 0); 1358 } 1359 1360 bool getHasColorExport(const Function &F) { 1361 // As a safe default always respond as if PS has color exports. 1362 return getIntegerAttribute( 1363 F, "amdgpu-color-export", 1364 F.getCallingConv() == CallingConv::AMDGPU_PS ? 1 : 0) != 0; 1365 } 1366 1367 bool getHasDepthExport(const Function &F) { 1368 return getIntegerAttribute(F, "amdgpu-depth-export", 0) != 0; 1369 } 1370 1371 bool isShader(CallingConv::ID cc) { 1372 switch(cc) { 1373 case CallingConv::AMDGPU_VS: 1374 case CallingConv::AMDGPU_LS: 1375 case CallingConv::AMDGPU_HS: 1376 case CallingConv::AMDGPU_ES: 1377 case CallingConv::AMDGPU_GS: 1378 case CallingConv::AMDGPU_PS: 1379 case CallingConv::AMDGPU_CS: 1380 return true; 1381 default: 1382 return false; 1383 } 1384 } 1385 1386 bool isGraphics(CallingConv::ID cc) { 1387 return isShader(cc) || cc == CallingConv::AMDGPU_Gfx; 1388 } 1389 1390 bool isCompute(CallingConv::ID cc) { 1391 return !isGraphics(cc) || cc == CallingConv::AMDGPU_CS; 1392 } 1393 1394 bool isEntryFunctionCC(CallingConv::ID CC) { 1395 switch (CC) { 1396 case CallingConv::AMDGPU_KERNEL: 1397 case CallingConv::SPIR_KERNEL: 1398 case CallingConv::AMDGPU_VS: 1399 case CallingConv::AMDGPU_GS: 1400 case CallingConv::AMDGPU_PS: 1401 case CallingConv::AMDGPU_CS: 1402 case CallingConv::AMDGPU_ES: 1403 case CallingConv::AMDGPU_HS: 1404 case CallingConv::AMDGPU_LS: 1405 return true; 1406 default: 1407 return false; 1408 } 1409 } 1410 1411 bool isModuleEntryFunctionCC(CallingConv::ID CC) { 1412 switch (CC) { 1413 case CallingConv::AMDGPU_Gfx: 1414 return true; 1415 default: 1416 return isEntryFunctionCC(CC); 1417 } 1418 } 1419 1420 bool hasXNACK(const MCSubtargetInfo &STI) { 1421 return STI.getFeatureBits()[AMDGPU::FeatureXNACK]; 1422 } 1423 1424 bool hasSRAMECC(const MCSubtargetInfo &STI) { 1425 return STI.getFeatureBits()[AMDGPU::FeatureSRAMECC]; 1426 } 1427 1428 bool hasMIMG_R128(const MCSubtargetInfo &STI) { 1429 return STI.getFeatureBits()[AMDGPU::FeatureMIMG_R128] && !STI.getFeatureBits()[AMDGPU::FeatureR128A16]; 1430 } 1431 1432 bool hasGFX10A16(const MCSubtargetInfo &STI) { 1433 return STI.getFeatureBits()[AMDGPU::FeatureGFX10A16]; 1434 } 1435 1436 bool hasG16(const MCSubtargetInfo &STI) { 1437 return STI.getFeatureBits()[AMDGPU::FeatureG16]; 1438 } 1439 1440 bool hasPackedD16(const MCSubtargetInfo &STI) { 1441 return !STI.getFeatureBits()[AMDGPU::FeatureUnpackedD16VMem]; 1442 } 1443 1444 bool isSI(const MCSubtargetInfo &STI) { 1445 return STI.getFeatureBits()[AMDGPU::FeatureSouthernIslands]; 1446 } 1447 1448 bool isCI(const MCSubtargetInfo &STI) { 1449 return STI.getFeatureBits()[AMDGPU::FeatureSeaIslands]; 1450 } 1451 1452 bool isVI(const MCSubtargetInfo &STI) { 1453 return STI.getFeatureBits()[AMDGPU::FeatureVolcanicIslands]; 1454 } 1455 1456 bool isGFX9(const MCSubtargetInfo &STI) { 1457 return STI.getFeatureBits()[AMDGPU::FeatureGFX9]; 1458 } 1459 1460 bool isGFX9Plus(const MCSubtargetInfo &STI) { 1461 return isGFX9(STI) || isGFX10Plus(STI); 1462 } 1463 1464 bool isGFX10(const MCSubtargetInfo &STI) { 1465 return STI.getFeatureBits()[AMDGPU::FeatureGFX10]; 1466 } 1467 1468 bool isGFX10Plus(const MCSubtargetInfo &STI) { return isGFX10(STI); } 1469 1470 bool isGCN3Encoding(const MCSubtargetInfo &STI) { 1471 return STI.getFeatureBits()[AMDGPU::FeatureGCN3Encoding]; 1472 } 1473 1474 bool isGFX10_AEncoding(const MCSubtargetInfo &STI) { 1475 return STI.getFeatureBits()[AMDGPU::FeatureGFX10_AEncoding]; 1476 } 1477 1478 bool isGFX10_BEncoding(const MCSubtargetInfo &STI) { 1479 return STI.getFeatureBits()[AMDGPU::FeatureGFX10_BEncoding]; 1480 } 1481 1482 bool hasGFX10_3Insts(const MCSubtargetInfo &STI) { 1483 return STI.getFeatureBits()[AMDGPU::FeatureGFX10_3Insts]; 1484 } 1485 1486 bool isGFX90A(const MCSubtargetInfo &STI) { 1487 return STI.getFeatureBits()[AMDGPU::FeatureGFX90AInsts]; 1488 } 1489 1490 bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI) { 1491 return STI.getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch]; 1492 } 1493 1494 bool isSGPR(unsigned Reg, const MCRegisterInfo* TRI) { 1495 const MCRegisterClass SGPRClass = TRI->getRegClass(AMDGPU::SReg_32RegClassID); 1496 const unsigned FirstSubReg = TRI->getSubReg(Reg, AMDGPU::sub0); 1497 return SGPRClass.contains(FirstSubReg != 0 ? FirstSubReg : Reg) || 1498 Reg == AMDGPU::SCC; 1499 } 1500 1501 bool isRegIntersect(unsigned Reg0, unsigned Reg1, const MCRegisterInfo* TRI) { 1502 for (MCRegAliasIterator R(Reg0, TRI, true); R.isValid(); ++R) { 1503 if (*R == Reg1) return true; 1504 } 1505 return false; 1506 } 1507 1508 #define MAP_REG2REG \ 1509 using namespace AMDGPU; \ 1510 switch(Reg) { \ 1511 default: return Reg; \ 1512 CASE_CI_VI(FLAT_SCR) \ 1513 CASE_CI_VI(FLAT_SCR_LO) \ 1514 CASE_CI_VI(FLAT_SCR_HI) \ 1515 CASE_VI_GFX9PLUS(TTMP0) \ 1516 CASE_VI_GFX9PLUS(TTMP1) \ 1517 CASE_VI_GFX9PLUS(TTMP2) \ 1518 CASE_VI_GFX9PLUS(TTMP3) \ 1519 CASE_VI_GFX9PLUS(TTMP4) \ 1520 CASE_VI_GFX9PLUS(TTMP5) \ 1521 CASE_VI_GFX9PLUS(TTMP6) \ 1522 CASE_VI_GFX9PLUS(TTMP7) \ 1523 CASE_VI_GFX9PLUS(TTMP8) \ 1524 CASE_VI_GFX9PLUS(TTMP9) \ 1525 CASE_VI_GFX9PLUS(TTMP10) \ 1526 CASE_VI_GFX9PLUS(TTMP11) \ 1527 CASE_VI_GFX9PLUS(TTMP12) \ 1528 CASE_VI_GFX9PLUS(TTMP13) \ 1529 CASE_VI_GFX9PLUS(TTMP14) \ 1530 CASE_VI_GFX9PLUS(TTMP15) \ 1531 CASE_VI_GFX9PLUS(TTMP0_TTMP1) \ 1532 CASE_VI_GFX9PLUS(TTMP2_TTMP3) \ 1533 CASE_VI_GFX9PLUS(TTMP4_TTMP5) \ 1534 CASE_VI_GFX9PLUS(TTMP6_TTMP7) \ 1535 CASE_VI_GFX9PLUS(TTMP8_TTMP9) \ 1536 CASE_VI_GFX9PLUS(TTMP10_TTMP11) \ 1537 CASE_VI_GFX9PLUS(TTMP12_TTMP13) \ 1538 CASE_VI_GFX9PLUS(TTMP14_TTMP15) \ 1539 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3) \ 1540 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7) \ 1541 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11) \ 1542 CASE_VI_GFX9PLUS(TTMP12_TTMP13_TTMP14_TTMP15) \ 1543 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7) \ 1544 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11) \ 1545 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \ 1546 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \ 1547 } 1548 1549 #define CASE_CI_VI(node) \ 1550 assert(!isSI(STI)); \ 1551 case node: return isCI(STI) ? node##_ci : node##_vi; 1552 1553 #define CASE_VI_GFX9PLUS(node) \ 1554 case node: return isGFX9Plus(STI) ? node##_gfx9plus : node##_vi; 1555 1556 unsigned getMCReg(unsigned Reg, const MCSubtargetInfo &STI) { 1557 if (STI.getTargetTriple().getArch() == Triple::r600) 1558 return Reg; 1559 MAP_REG2REG 1560 } 1561 1562 #undef CASE_CI_VI 1563 #undef CASE_VI_GFX9PLUS 1564 1565 #define CASE_CI_VI(node) case node##_ci: case node##_vi: return node; 1566 #define CASE_VI_GFX9PLUS(node) case node##_vi: case node##_gfx9plus: return node; 1567 1568 unsigned mc2PseudoReg(unsigned Reg) { 1569 MAP_REG2REG 1570 } 1571 1572 #undef CASE_CI_VI 1573 #undef CASE_VI_GFX9PLUS 1574 #undef MAP_REG2REG 1575 1576 bool isSISrcOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1577 assert(OpNo < Desc.NumOperands); 1578 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1579 return OpType >= AMDGPU::OPERAND_SRC_FIRST && 1580 OpType <= AMDGPU::OPERAND_SRC_LAST; 1581 } 1582 1583 bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1584 assert(OpNo < Desc.NumOperands); 1585 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1586 switch (OpType) { 1587 case AMDGPU::OPERAND_REG_IMM_FP32: 1588 case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED: 1589 case AMDGPU::OPERAND_REG_IMM_FP64: 1590 case AMDGPU::OPERAND_REG_IMM_FP16: 1591 case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED: 1592 case AMDGPU::OPERAND_REG_IMM_V2FP16: 1593 case AMDGPU::OPERAND_REG_IMM_V2INT16: 1594 case AMDGPU::OPERAND_REG_INLINE_C_FP32: 1595 case AMDGPU::OPERAND_REG_INLINE_C_FP64: 1596 case AMDGPU::OPERAND_REG_INLINE_C_FP16: 1597 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: 1598 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: 1599 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: 1600 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: 1601 case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: 1602 case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16: 1603 case AMDGPU::OPERAND_REG_IMM_V2FP32: 1604 case AMDGPU::OPERAND_REG_INLINE_C_V2FP32: 1605 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: 1606 return true; 1607 default: 1608 return false; 1609 } 1610 } 1611 1612 bool isSISrcInlinableOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1613 assert(OpNo < Desc.NumOperands); 1614 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1615 return OpType >= AMDGPU::OPERAND_REG_INLINE_C_FIRST && 1616 OpType <= AMDGPU::OPERAND_REG_INLINE_C_LAST; 1617 } 1618 1619 // Avoid using MCRegisterClass::getSize, since that function will go away 1620 // (move from MC* level to Target* level). Return size in bits. 1621 unsigned getRegBitWidth(unsigned RCID) { 1622 switch (RCID) { 1623 case AMDGPU::VGPR_LO16RegClassID: 1624 case AMDGPU::VGPR_HI16RegClassID: 1625 case AMDGPU::SGPR_LO16RegClassID: 1626 case AMDGPU::AGPR_LO16RegClassID: 1627 return 16; 1628 case AMDGPU::SGPR_32RegClassID: 1629 case AMDGPU::VGPR_32RegClassID: 1630 case AMDGPU::VRegOrLds_32RegClassID: 1631 case AMDGPU::AGPR_32RegClassID: 1632 case AMDGPU::VS_32RegClassID: 1633 case AMDGPU::AV_32RegClassID: 1634 case AMDGPU::SReg_32RegClassID: 1635 case AMDGPU::SReg_32_XM0RegClassID: 1636 case AMDGPU::SRegOrLds_32RegClassID: 1637 return 32; 1638 case AMDGPU::SGPR_64RegClassID: 1639 case AMDGPU::VS_64RegClassID: 1640 case AMDGPU::SReg_64RegClassID: 1641 case AMDGPU::VReg_64RegClassID: 1642 case AMDGPU::AReg_64RegClassID: 1643 case AMDGPU::SReg_64_XEXECRegClassID: 1644 case AMDGPU::VReg_64_Align2RegClassID: 1645 case AMDGPU::AReg_64_Align2RegClassID: 1646 case AMDGPU::AV_64RegClassID: 1647 case AMDGPU::AV_64_Align2RegClassID: 1648 return 64; 1649 case AMDGPU::SGPR_96RegClassID: 1650 case AMDGPU::SReg_96RegClassID: 1651 case AMDGPU::VReg_96RegClassID: 1652 case AMDGPU::AReg_96RegClassID: 1653 case AMDGPU::VReg_96_Align2RegClassID: 1654 case AMDGPU::AReg_96_Align2RegClassID: 1655 case AMDGPU::AV_96RegClassID: 1656 case AMDGPU::AV_96_Align2RegClassID: 1657 return 96; 1658 case AMDGPU::SGPR_128RegClassID: 1659 case AMDGPU::SReg_128RegClassID: 1660 case AMDGPU::VReg_128RegClassID: 1661 case AMDGPU::AReg_128RegClassID: 1662 case AMDGPU::VReg_128_Align2RegClassID: 1663 case AMDGPU::AReg_128_Align2RegClassID: 1664 case AMDGPU::AV_128RegClassID: 1665 case AMDGPU::AV_128_Align2RegClassID: 1666 return 128; 1667 case AMDGPU::SGPR_160RegClassID: 1668 case AMDGPU::SReg_160RegClassID: 1669 case AMDGPU::VReg_160RegClassID: 1670 case AMDGPU::AReg_160RegClassID: 1671 case AMDGPU::VReg_160_Align2RegClassID: 1672 case AMDGPU::AReg_160_Align2RegClassID: 1673 case AMDGPU::AV_160RegClassID: 1674 case AMDGPU::AV_160_Align2RegClassID: 1675 return 160; 1676 case AMDGPU::SGPR_192RegClassID: 1677 case AMDGPU::SReg_192RegClassID: 1678 case AMDGPU::VReg_192RegClassID: 1679 case AMDGPU::AReg_192RegClassID: 1680 case AMDGPU::VReg_192_Align2RegClassID: 1681 case AMDGPU::AReg_192_Align2RegClassID: 1682 case AMDGPU::AV_192RegClassID: 1683 case AMDGPU::AV_192_Align2RegClassID: 1684 return 192; 1685 case AMDGPU::SGPR_224RegClassID: 1686 case AMDGPU::SReg_224RegClassID: 1687 case AMDGPU::VReg_224RegClassID: 1688 case AMDGPU::AReg_224RegClassID: 1689 case AMDGPU::VReg_224_Align2RegClassID: 1690 case AMDGPU::AReg_224_Align2RegClassID: 1691 case AMDGPU::AV_224RegClassID: 1692 case AMDGPU::AV_224_Align2RegClassID: 1693 return 224; 1694 case AMDGPU::SGPR_256RegClassID: 1695 case AMDGPU::SReg_256RegClassID: 1696 case AMDGPU::VReg_256RegClassID: 1697 case AMDGPU::AReg_256RegClassID: 1698 case AMDGPU::VReg_256_Align2RegClassID: 1699 case AMDGPU::AReg_256_Align2RegClassID: 1700 case AMDGPU::AV_256RegClassID: 1701 case AMDGPU::AV_256_Align2RegClassID: 1702 return 256; 1703 case AMDGPU::SGPR_512RegClassID: 1704 case AMDGPU::SReg_512RegClassID: 1705 case AMDGPU::VReg_512RegClassID: 1706 case AMDGPU::AReg_512RegClassID: 1707 case AMDGPU::VReg_512_Align2RegClassID: 1708 case AMDGPU::AReg_512_Align2RegClassID: 1709 case AMDGPU::AV_512RegClassID: 1710 case AMDGPU::AV_512_Align2RegClassID: 1711 return 512; 1712 case AMDGPU::SGPR_1024RegClassID: 1713 case AMDGPU::SReg_1024RegClassID: 1714 case AMDGPU::VReg_1024RegClassID: 1715 case AMDGPU::AReg_1024RegClassID: 1716 case AMDGPU::VReg_1024_Align2RegClassID: 1717 case AMDGPU::AReg_1024_Align2RegClassID: 1718 case AMDGPU::AV_1024RegClassID: 1719 case AMDGPU::AV_1024_Align2RegClassID: 1720 return 1024; 1721 default: 1722 llvm_unreachable("Unexpected register class"); 1723 } 1724 } 1725 1726 unsigned getRegBitWidth(const MCRegisterClass &RC) { 1727 return getRegBitWidth(RC.getID()); 1728 } 1729 1730 unsigned getRegOperandSize(const MCRegisterInfo *MRI, const MCInstrDesc &Desc, 1731 unsigned OpNo) { 1732 assert(OpNo < Desc.NumOperands); 1733 unsigned RCID = Desc.OpInfo[OpNo].RegClass; 1734 return getRegBitWidth(MRI->getRegClass(RCID)) / 8; 1735 } 1736 1737 bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi) { 1738 if (isInlinableIntLiteral(Literal)) 1739 return true; 1740 1741 uint64_t Val = static_cast<uint64_t>(Literal); 1742 return (Val == DoubleToBits(0.0)) || 1743 (Val == DoubleToBits(1.0)) || 1744 (Val == DoubleToBits(-1.0)) || 1745 (Val == DoubleToBits(0.5)) || 1746 (Val == DoubleToBits(-0.5)) || 1747 (Val == DoubleToBits(2.0)) || 1748 (Val == DoubleToBits(-2.0)) || 1749 (Val == DoubleToBits(4.0)) || 1750 (Val == DoubleToBits(-4.0)) || 1751 (Val == 0x3fc45f306dc9c882 && HasInv2Pi); 1752 } 1753 1754 bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi) { 1755 if (isInlinableIntLiteral(Literal)) 1756 return true; 1757 1758 // The actual type of the operand does not seem to matter as long 1759 // as the bits match one of the inline immediate values. For example: 1760 // 1761 // -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal, 1762 // so it is a legal inline immediate. 1763 // 1764 // 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in 1765 // floating-point, so it is a legal inline immediate. 1766 1767 uint32_t Val = static_cast<uint32_t>(Literal); 1768 return (Val == FloatToBits(0.0f)) || 1769 (Val == FloatToBits(1.0f)) || 1770 (Val == FloatToBits(-1.0f)) || 1771 (Val == FloatToBits(0.5f)) || 1772 (Val == FloatToBits(-0.5f)) || 1773 (Val == FloatToBits(2.0f)) || 1774 (Val == FloatToBits(-2.0f)) || 1775 (Val == FloatToBits(4.0f)) || 1776 (Val == FloatToBits(-4.0f)) || 1777 (Val == 0x3e22f983 && HasInv2Pi); 1778 } 1779 1780 bool isInlinableLiteral16(int16_t Literal, bool HasInv2Pi) { 1781 if (!HasInv2Pi) 1782 return false; 1783 1784 if (isInlinableIntLiteral(Literal)) 1785 return true; 1786 1787 uint16_t Val = static_cast<uint16_t>(Literal); 1788 return Val == 0x3C00 || // 1.0 1789 Val == 0xBC00 || // -1.0 1790 Val == 0x3800 || // 0.5 1791 Val == 0xB800 || // -0.5 1792 Val == 0x4000 || // 2.0 1793 Val == 0xC000 || // -2.0 1794 Val == 0x4400 || // 4.0 1795 Val == 0xC400 || // -4.0 1796 Val == 0x3118; // 1/2pi 1797 } 1798 1799 bool isInlinableLiteralV216(int32_t Literal, bool HasInv2Pi) { 1800 assert(HasInv2Pi); 1801 1802 if (isInt<16>(Literal) || isUInt<16>(Literal)) { 1803 int16_t Trunc = static_cast<int16_t>(Literal); 1804 return AMDGPU::isInlinableLiteral16(Trunc, HasInv2Pi); 1805 } 1806 if (!(Literal & 0xffff)) 1807 return AMDGPU::isInlinableLiteral16(Literal >> 16, HasInv2Pi); 1808 1809 int16_t Lo16 = static_cast<int16_t>(Literal); 1810 int16_t Hi16 = static_cast<int16_t>(Literal >> 16); 1811 return Lo16 == Hi16 && isInlinableLiteral16(Lo16, HasInv2Pi); 1812 } 1813 1814 bool isInlinableIntLiteralV216(int32_t Literal) { 1815 int16_t Lo16 = static_cast<int16_t>(Literal); 1816 if (isInt<16>(Literal) || isUInt<16>(Literal)) 1817 return isInlinableIntLiteral(Lo16); 1818 1819 int16_t Hi16 = static_cast<int16_t>(Literal >> 16); 1820 if (!(Literal & 0xffff)) 1821 return isInlinableIntLiteral(Hi16); 1822 return Lo16 == Hi16 && isInlinableIntLiteral(Lo16); 1823 } 1824 1825 bool isFoldableLiteralV216(int32_t Literal, bool HasInv2Pi) { 1826 assert(HasInv2Pi); 1827 1828 int16_t Lo16 = static_cast<int16_t>(Literal); 1829 if (isInt<16>(Literal) || isUInt<16>(Literal)) 1830 return true; 1831 1832 int16_t Hi16 = static_cast<int16_t>(Literal >> 16); 1833 if (!(Literal & 0xffff)) 1834 return true; 1835 return Lo16 == Hi16; 1836 } 1837 1838 bool isArgPassedInSGPR(const Argument *A) { 1839 const Function *F = A->getParent(); 1840 1841 // Arguments to compute shaders are never a source of divergence. 1842 CallingConv::ID CC = F->getCallingConv(); 1843 switch (CC) { 1844 case CallingConv::AMDGPU_KERNEL: 1845 case CallingConv::SPIR_KERNEL: 1846 return true; 1847 case CallingConv::AMDGPU_VS: 1848 case CallingConv::AMDGPU_LS: 1849 case CallingConv::AMDGPU_HS: 1850 case CallingConv::AMDGPU_ES: 1851 case CallingConv::AMDGPU_GS: 1852 case CallingConv::AMDGPU_PS: 1853 case CallingConv::AMDGPU_CS: 1854 case CallingConv::AMDGPU_Gfx: 1855 // For non-compute shaders, SGPR inputs are marked with either inreg or byval. 1856 // Everything else is in VGPRs. 1857 return F->getAttributes().hasParamAttr(A->getArgNo(), Attribute::InReg) || 1858 F->getAttributes().hasParamAttr(A->getArgNo(), Attribute::ByVal); 1859 default: 1860 // TODO: Should calls support inreg for SGPR inputs? 1861 return false; 1862 } 1863 } 1864 1865 static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) { 1866 return isGCN3Encoding(ST) || isGFX10Plus(ST); 1867 } 1868 1869 static bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST) { 1870 return isGFX9Plus(ST); 1871 } 1872 1873 bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST, 1874 int64_t EncodedOffset) { 1875 return hasSMEMByteOffset(ST) ? isUInt<20>(EncodedOffset) 1876 : isUInt<8>(EncodedOffset); 1877 } 1878 1879 bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST, 1880 int64_t EncodedOffset, 1881 bool IsBuffer) { 1882 return !IsBuffer && 1883 hasSMRDSignedImmOffset(ST) && 1884 isInt<21>(EncodedOffset); 1885 } 1886 1887 static bool isDwordAligned(uint64_t ByteOffset) { 1888 return (ByteOffset & 3) == 0; 1889 } 1890 1891 uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST, 1892 uint64_t ByteOffset) { 1893 if (hasSMEMByteOffset(ST)) 1894 return ByteOffset; 1895 1896 assert(isDwordAligned(ByteOffset)); 1897 return ByteOffset >> 2; 1898 } 1899 1900 Optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST, 1901 int64_t ByteOffset, bool IsBuffer) { 1902 // The signed version is always a byte offset. 1903 if (!IsBuffer && hasSMRDSignedImmOffset(ST)) { 1904 assert(hasSMEMByteOffset(ST)); 1905 return isInt<20>(ByteOffset) ? Optional<int64_t>(ByteOffset) : None; 1906 } 1907 1908 if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST)) 1909 return None; 1910 1911 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset); 1912 return isLegalSMRDEncodedUnsignedOffset(ST, EncodedOffset) 1913 ? Optional<int64_t>(EncodedOffset) 1914 : None; 1915 } 1916 1917 Optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST, 1918 int64_t ByteOffset) { 1919 if (!isCI(ST) || !isDwordAligned(ByteOffset)) 1920 return None; 1921 1922 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset); 1923 return isUInt<32>(EncodedOffset) ? Optional<int64_t>(EncodedOffset) : None; 1924 } 1925 1926 unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST, bool Signed) { 1927 // Address offset is 12-bit signed for GFX10, 13-bit for GFX9. 1928 if (AMDGPU::isGFX10(ST)) 1929 return Signed ? 12 : 11; 1930 1931 return Signed ? 13 : 12; 1932 } 1933 1934 // Given Imm, split it into the values to put into the SOffset and ImmOffset 1935 // fields in an MUBUF instruction. Return false if it is not possible (due to a 1936 // hardware bug needing a workaround). 1937 // 1938 // The required alignment ensures that individual address components remain 1939 // aligned if they are aligned to begin with. It also ensures that additional 1940 // offsets within the given alignment can be added to the resulting ImmOffset. 1941 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset, 1942 const GCNSubtarget *Subtarget, Align Alignment) { 1943 const uint32_t MaxImm = alignDown(4095, Alignment.value()); 1944 uint32_t Overflow = 0; 1945 1946 if (Imm > MaxImm) { 1947 if (Imm <= MaxImm + 64) { 1948 // Use an SOffset inline constant for 4..64 1949 Overflow = Imm - MaxImm; 1950 Imm = MaxImm; 1951 } else { 1952 // Try to keep the same value in SOffset for adjacent loads, so that 1953 // the corresponding register contents can be re-used. 1954 // 1955 // Load values with all low-bits (except for alignment bits) set into 1956 // SOffset, so that a larger range of values can be covered using 1957 // s_movk_i32. 1958 // 1959 // Atomic operations fail to work correctly when individual address 1960 // components are unaligned, even if their sum is aligned. 1961 uint32_t High = (Imm + Alignment.value()) & ~4095; 1962 uint32_t Low = (Imm + Alignment.value()) & 4095; 1963 Imm = Low; 1964 Overflow = High - Alignment.value(); 1965 } 1966 } 1967 1968 // There is a hardware bug in SI and CI which prevents address clamping in 1969 // MUBUF instructions from working correctly with SOffsets. The immediate 1970 // offset is unaffected. 1971 if (Overflow > 0 && 1972 Subtarget->getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS) 1973 return false; 1974 1975 ImmOffset = Imm; 1976 SOffset = Overflow; 1977 return true; 1978 } 1979 1980 SIModeRegisterDefaults::SIModeRegisterDefaults(const Function &F) { 1981 *this = getDefaultForCallingConv(F.getCallingConv()); 1982 1983 StringRef IEEEAttr = F.getFnAttribute("amdgpu-ieee").getValueAsString(); 1984 if (!IEEEAttr.empty()) 1985 IEEE = IEEEAttr == "true"; 1986 1987 StringRef DX10ClampAttr 1988 = F.getFnAttribute("amdgpu-dx10-clamp").getValueAsString(); 1989 if (!DX10ClampAttr.empty()) 1990 DX10Clamp = DX10ClampAttr == "true"; 1991 1992 StringRef DenormF32Attr = F.getFnAttribute("denormal-fp-math-f32").getValueAsString(); 1993 if (!DenormF32Attr.empty()) { 1994 DenormalMode DenormMode = parseDenormalFPAttribute(DenormF32Attr); 1995 FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE; 1996 FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE; 1997 } 1998 1999 StringRef DenormAttr = F.getFnAttribute("denormal-fp-math").getValueAsString(); 2000 if (!DenormAttr.empty()) { 2001 DenormalMode DenormMode = parseDenormalFPAttribute(DenormAttr); 2002 2003 if (DenormF32Attr.empty()) { 2004 FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE; 2005 FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE; 2006 } 2007 2008 FP64FP16InputDenormals = DenormMode.Input == DenormalMode::IEEE; 2009 FP64FP16OutputDenormals = DenormMode.Output == DenormalMode::IEEE; 2010 } 2011 } 2012 2013 namespace { 2014 2015 struct SourceOfDivergence { 2016 unsigned Intr; 2017 }; 2018 const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr); 2019 2020 #define GET_SourcesOfDivergence_IMPL 2021 #define GET_Gfx9BufferFormat_IMPL 2022 #define GET_Gfx10PlusBufferFormat_IMPL 2023 #include "AMDGPUGenSearchableTables.inc" 2024 2025 } // end anonymous namespace 2026 2027 bool isIntrinsicSourceOfDivergence(unsigned IntrID) { 2028 return lookupSourceOfDivergence(IntrID); 2029 } 2030 2031 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp, 2032 uint8_t NumComponents, 2033 uint8_t NumFormat, 2034 const MCSubtargetInfo &STI) { 2035 return isGFX10Plus(STI) 2036 ? getGfx10PlusBufferFormatInfo(BitsPerComp, NumComponents, 2037 NumFormat) 2038 : getGfx9BufferFormatInfo(BitsPerComp, NumComponents, NumFormat); 2039 } 2040 2041 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format, 2042 const MCSubtargetInfo &STI) { 2043 return isGFX10Plus(STI) ? getGfx10PlusBufferFormatInfo(Format) 2044 : getGfx9BufferFormatInfo(Format); 2045 } 2046 2047 } // namespace AMDGPU 2048 2049 raw_ostream &operator<<(raw_ostream &OS, 2050 const AMDGPU::IsaInfo::TargetIDSetting S) { 2051 switch (S) { 2052 case (AMDGPU::IsaInfo::TargetIDSetting::Unsupported): 2053 OS << "Unsupported"; 2054 break; 2055 case (AMDGPU::IsaInfo::TargetIDSetting::Any): 2056 OS << "Any"; 2057 break; 2058 case (AMDGPU::IsaInfo::TargetIDSetting::Off): 2059 OS << "Off"; 2060 break; 2061 case (AMDGPU::IsaInfo::TargetIDSetting::On): 2062 OS << "On"; 2063 break; 2064 } 2065 return OS; 2066 } 2067 2068 } // namespace llvm 2069