1 //===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "AMDGPUBaseInfo.h" 10 #include "AMDGPU.h" 11 #include "AMDGPUAsmUtils.h" 12 #include "AMDKernelCodeT.h" 13 #include "GCNSubtarget.h" 14 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 15 #include "llvm/BinaryFormat/ELF.h" 16 #include "llvm/IR/Attributes.h" 17 #include "llvm/IR/Function.h" 18 #include "llvm/IR/GlobalValue.h" 19 #include "llvm/IR/IntrinsicsAMDGPU.h" 20 #include "llvm/IR/IntrinsicsR600.h" 21 #include "llvm/IR/LLVMContext.h" 22 #include "llvm/MC/MCSubtargetInfo.h" 23 #include "llvm/Support/AMDHSAKernelDescriptor.h" 24 #include "llvm/Support/CommandLine.h" 25 #include "llvm/Support/TargetParser.h" 26 27 #define GET_INSTRINFO_NAMED_OPS 28 #define GET_INSTRMAP_INFO 29 #include "AMDGPUGenInstrInfo.inc" 30 31 static llvm::cl::opt<unsigned> AmdhsaCodeObjectVersion( 32 "amdhsa-code-object-version", llvm::cl::Hidden, 33 llvm::cl::desc("AMDHSA Code Object Version"), llvm::cl::init(4), 34 llvm::cl::ZeroOrMore); 35 36 namespace { 37 38 /// \returns Bit mask for given bit \p Shift and bit \p Width. 39 unsigned getBitMask(unsigned Shift, unsigned Width) { 40 return ((1 << Width) - 1) << Shift; 41 } 42 43 /// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width. 44 /// 45 /// \returns Packed \p Dst. 46 unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) { 47 Dst &= ~(1 << Shift) & ~getBitMask(Shift, Width); 48 Dst |= (Src << Shift) & getBitMask(Shift, Width); 49 return Dst; 50 } 51 52 /// Unpacks bits from \p Src for given bit \p Shift and bit \p Width. 53 /// 54 /// \returns Unpacked bits. 55 unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) { 56 return (Src & getBitMask(Shift, Width)) >> Shift; 57 } 58 59 /// \returns Vmcnt bit shift (lower bits). 60 unsigned getVmcntBitShiftLo() { return 0; } 61 62 /// \returns Vmcnt bit width (lower bits). 63 unsigned getVmcntBitWidthLo() { return 4; } 64 65 /// \returns Expcnt bit shift. 66 unsigned getExpcntBitShift() { return 4; } 67 68 /// \returns Expcnt bit width. 69 unsigned getExpcntBitWidth() { return 3; } 70 71 /// \returns Lgkmcnt bit shift. 72 unsigned getLgkmcntBitShift() { return 8; } 73 74 /// \returns Lgkmcnt bit width. 75 unsigned getLgkmcntBitWidth(unsigned VersionMajor) { 76 return (VersionMajor >= 10) ? 6 : 4; 77 } 78 79 /// \returns Vmcnt bit shift (higher bits). 80 unsigned getVmcntBitShiftHi() { return 14; } 81 82 /// \returns Vmcnt bit width (higher bits). 83 unsigned getVmcntBitWidthHi() { return 2; } 84 85 } // end namespace anonymous 86 87 namespace llvm { 88 89 namespace AMDGPU { 90 91 Optional<uint8_t> getHsaAbiVersion(const MCSubtargetInfo *STI) { 92 if (STI && STI->getTargetTriple().getOS() != Triple::AMDHSA) 93 return None; 94 95 switch (AmdhsaCodeObjectVersion) { 96 case 2: 97 return ELF::ELFABIVERSION_AMDGPU_HSA_V2; 98 case 3: 99 return ELF::ELFABIVERSION_AMDGPU_HSA_V3; 100 case 4: 101 return ELF::ELFABIVERSION_AMDGPU_HSA_V4; 102 case 5: 103 return ELF::ELFABIVERSION_AMDGPU_HSA_V5; 104 default: 105 report_fatal_error(Twine("Unsupported AMDHSA Code Object Version ") + 106 Twine(AmdhsaCodeObjectVersion)); 107 } 108 } 109 110 bool isHsaAbiVersion2(const MCSubtargetInfo *STI) { 111 if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI)) 112 return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V2; 113 return false; 114 } 115 116 bool isHsaAbiVersion3(const MCSubtargetInfo *STI) { 117 if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI)) 118 return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V3; 119 return false; 120 } 121 122 bool isHsaAbiVersion4(const MCSubtargetInfo *STI) { 123 if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI)) 124 return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V4; 125 return false; 126 } 127 128 bool isHsaAbiVersion5(const MCSubtargetInfo *STI) { 129 if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI)) 130 return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V5; 131 return false; 132 } 133 134 bool isHsaAbiVersion3AndAbove(const MCSubtargetInfo *STI) { 135 return isHsaAbiVersion3(STI) || isHsaAbiVersion4(STI) || 136 isHsaAbiVersion5(STI); 137 } 138 139 unsigned getAmdhsaCodeObjectVersion() { 140 return AmdhsaCodeObjectVersion; 141 } 142 143 // FIXME: All such magic numbers about the ABI should be in a 144 // central TD file. 145 unsigned getHostcallImplicitArgPosition() { 146 switch (AmdhsaCodeObjectVersion) { 147 case 2: 148 case 3: 149 case 4: 150 return 24; 151 case 5: 152 return 80; 153 default: 154 llvm_unreachable("Unexpected code object version"); 155 return 0; 156 } 157 } 158 159 unsigned getHeapPtrImplicitArgPosition() { 160 if (AmdhsaCodeObjectVersion == 5) 161 return 96; 162 llvm_unreachable("hidden_heap is supported only by code object version 5"); 163 return 0; 164 } 165 166 #define GET_MIMGBaseOpcodesTable_IMPL 167 #define GET_MIMGDimInfoTable_IMPL 168 #define GET_MIMGInfoTable_IMPL 169 #define GET_MIMGLZMappingTable_IMPL 170 #define GET_MIMGMIPMappingTable_IMPL 171 #define GET_MIMGBiasMappingTable_IMPL 172 #define GET_MIMGOffsetMappingTable_IMPL 173 #define GET_MIMGG16MappingTable_IMPL 174 #include "AMDGPUGenSearchableTables.inc" 175 176 int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding, 177 unsigned VDataDwords, unsigned VAddrDwords) { 178 const MIMGInfo *Info = getMIMGOpcodeHelper(BaseOpcode, MIMGEncoding, 179 VDataDwords, VAddrDwords); 180 return Info ? Info->Opcode : -1; 181 } 182 183 const MIMGBaseOpcodeInfo *getMIMGBaseOpcode(unsigned Opc) { 184 const MIMGInfo *Info = getMIMGInfo(Opc); 185 return Info ? getMIMGBaseOpcodeInfo(Info->BaseOpcode) : nullptr; 186 } 187 188 int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) { 189 const MIMGInfo *OrigInfo = getMIMGInfo(Opc); 190 const MIMGInfo *NewInfo = 191 getMIMGOpcodeHelper(OrigInfo->BaseOpcode, OrigInfo->MIMGEncoding, 192 NewChannels, OrigInfo->VAddrDwords); 193 return NewInfo ? NewInfo->Opcode : -1; 194 } 195 196 unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode, 197 const MIMGDimInfo *Dim, bool IsA16, 198 bool IsG16Supported) { 199 unsigned AddrWords = BaseOpcode->NumExtraArgs; 200 unsigned AddrComponents = (BaseOpcode->Coordinates ? Dim->NumCoords : 0) + 201 (BaseOpcode->LodOrClampOrMip ? 1 : 0); 202 if (IsA16) 203 AddrWords += divideCeil(AddrComponents, 2); 204 else 205 AddrWords += AddrComponents; 206 207 // Note: For subtargets that support A16 but not G16, enabling A16 also 208 // enables 16 bit gradients. 209 // For subtargets that support A16 (operand) and G16 (done with a different 210 // instruction encoding), they are independent. 211 212 if (BaseOpcode->Gradients) { 213 if ((IsA16 && !IsG16Supported) || BaseOpcode->G16) 214 // There are two gradients per coordinate, we pack them separately. 215 // For the 3d case, 216 // we get (dy/du, dx/du) (-, dz/du) (dy/dv, dx/dv) (-, dz/dv) 217 AddrWords += alignTo<2>(Dim->NumGradients / 2); 218 else 219 AddrWords += Dim->NumGradients; 220 } 221 return AddrWords; 222 } 223 224 struct MUBUFInfo { 225 uint16_t Opcode; 226 uint16_t BaseOpcode; 227 uint8_t elements; 228 bool has_vaddr; 229 bool has_srsrc; 230 bool has_soffset; 231 bool IsBufferInv; 232 }; 233 234 struct MTBUFInfo { 235 uint16_t Opcode; 236 uint16_t BaseOpcode; 237 uint8_t elements; 238 bool has_vaddr; 239 bool has_srsrc; 240 bool has_soffset; 241 }; 242 243 struct SMInfo { 244 uint16_t Opcode; 245 bool IsBuffer; 246 }; 247 248 struct VOPInfo { 249 uint16_t Opcode; 250 bool IsSingle; 251 }; 252 253 #define GET_MTBUFInfoTable_DECL 254 #define GET_MTBUFInfoTable_IMPL 255 #define GET_MUBUFInfoTable_DECL 256 #define GET_MUBUFInfoTable_IMPL 257 #define GET_SMInfoTable_DECL 258 #define GET_SMInfoTable_IMPL 259 #define GET_VOP1InfoTable_DECL 260 #define GET_VOP1InfoTable_IMPL 261 #define GET_VOP2InfoTable_DECL 262 #define GET_VOP2InfoTable_IMPL 263 #define GET_VOP3InfoTable_DECL 264 #define GET_VOP3InfoTable_IMPL 265 #include "AMDGPUGenSearchableTables.inc" 266 267 int getMTBUFBaseOpcode(unsigned Opc) { 268 const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opc); 269 return Info ? Info->BaseOpcode : -1; 270 } 271 272 int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements) { 273 const MTBUFInfo *Info = getMTBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements); 274 return Info ? Info->Opcode : -1; 275 } 276 277 int getMTBUFElements(unsigned Opc) { 278 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 279 return Info ? Info->elements : 0; 280 } 281 282 bool getMTBUFHasVAddr(unsigned Opc) { 283 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 284 return Info ? Info->has_vaddr : false; 285 } 286 287 bool getMTBUFHasSrsrc(unsigned Opc) { 288 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 289 return Info ? Info->has_srsrc : false; 290 } 291 292 bool getMTBUFHasSoffset(unsigned Opc) { 293 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 294 return Info ? Info->has_soffset : false; 295 } 296 297 int getMUBUFBaseOpcode(unsigned Opc) { 298 const MUBUFInfo *Info = getMUBUFInfoFromOpcode(Opc); 299 return Info ? Info->BaseOpcode : -1; 300 } 301 302 int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements) { 303 const MUBUFInfo *Info = getMUBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements); 304 return Info ? Info->Opcode : -1; 305 } 306 307 int getMUBUFElements(unsigned Opc) { 308 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 309 return Info ? Info->elements : 0; 310 } 311 312 bool getMUBUFHasVAddr(unsigned Opc) { 313 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 314 return Info ? Info->has_vaddr : false; 315 } 316 317 bool getMUBUFHasSrsrc(unsigned Opc) { 318 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 319 return Info ? Info->has_srsrc : false; 320 } 321 322 bool getMUBUFHasSoffset(unsigned Opc) { 323 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 324 return Info ? Info->has_soffset : false; 325 } 326 327 bool getMUBUFIsBufferInv(unsigned Opc) { 328 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 329 return Info ? Info->IsBufferInv : false; 330 } 331 332 bool getSMEMIsBuffer(unsigned Opc) { 333 const SMInfo *Info = getSMEMOpcodeHelper(Opc); 334 return Info ? Info->IsBuffer : false; 335 } 336 337 bool getVOP1IsSingle(unsigned Opc) { 338 const VOPInfo *Info = getVOP1OpcodeHelper(Opc); 339 return Info ? Info->IsSingle : false; 340 } 341 342 bool getVOP2IsSingle(unsigned Opc) { 343 const VOPInfo *Info = getVOP2OpcodeHelper(Opc); 344 return Info ? Info->IsSingle : false; 345 } 346 347 bool getVOP3IsSingle(unsigned Opc) { 348 const VOPInfo *Info = getVOP3OpcodeHelper(Opc); 349 return Info ? Info->IsSingle : false; 350 } 351 352 // Wrapper for Tablegen'd function. enum Subtarget is not defined in any 353 // header files, so we need to wrap it in a function that takes unsigned 354 // instead. 355 int getMCOpcode(uint16_t Opcode, unsigned Gen) { 356 return getMCOpcodeGen(Opcode, static_cast<Subtarget>(Gen)); 357 } 358 359 namespace IsaInfo { 360 361 AMDGPUTargetID::AMDGPUTargetID(const MCSubtargetInfo &STI) 362 : STI(STI), XnackSetting(TargetIDSetting::Any), 363 SramEccSetting(TargetIDSetting::Any) { 364 if (!STI.getFeatureBits().test(FeatureSupportsXNACK)) 365 XnackSetting = TargetIDSetting::Unsupported; 366 if (!STI.getFeatureBits().test(FeatureSupportsSRAMECC)) 367 SramEccSetting = TargetIDSetting::Unsupported; 368 } 369 370 void AMDGPUTargetID::setTargetIDFromFeaturesString(StringRef FS) { 371 // Check if xnack or sramecc is explicitly enabled or disabled. In the 372 // absence of the target features we assume we must generate code that can run 373 // in any environment. 374 SubtargetFeatures Features(FS); 375 Optional<bool> XnackRequested; 376 Optional<bool> SramEccRequested; 377 378 for (const std::string &Feature : Features.getFeatures()) { 379 if (Feature == "+xnack") 380 XnackRequested = true; 381 else if (Feature == "-xnack") 382 XnackRequested = false; 383 else if (Feature == "+sramecc") 384 SramEccRequested = true; 385 else if (Feature == "-sramecc") 386 SramEccRequested = false; 387 } 388 389 bool XnackSupported = isXnackSupported(); 390 bool SramEccSupported = isSramEccSupported(); 391 392 if (XnackRequested) { 393 if (XnackSupported) { 394 XnackSetting = 395 *XnackRequested ? TargetIDSetting::On : TargetIDSetting::Off; 396 } else { 397 // If a specific xnack setting was requested and this GPU does not support 398 // xnack emit a warning. Setting will remain set to "Unsupported". 399 if (*XnackRequested) { 400 errs() << "warning: xnack 'On' was requested for a processor that does " 401 "not support it!\n"; 402 } else { 403 errs() << "warning: xnack 'Off' was requested for a processor that " 404 "does not support it!\n"; 405 } 406 } 407 } 408 409 if (SramEccRequested) { 410 if (SramEccSupported) { 411 SramEccSetting = 412 *SramEccRequested ? TargetIDSetting::On : TargetIDSetting::Off; 413 } else { 414 // If a specific sramecc setting was requested and this GPU does not 415 // support sramecc emit a warning. Setting will remain set to 416 // "Unsupported". 417 if (*SramEccRequested) { 418 errs() << "warning: sramecc 'On' was requested for a processor that " 419 "does not support it!\n"; 420 } else { 421 errs() << "warning: sramecc 'Off' was requested for a processor that " 422 "does not support it!\n"; 423 } 424 } 425 } 426 } 427 428 static TargetIDSetting 429 getTargetIDSettingFromFeatureString(StringRef FeatureString) { 430 if (FeatureString.endswith("-")) 431 return TargetIDSetting::Off; 432 if (FeatureString.endswith("+")) 433 return TargetIDSetting::On; 434 435 llvm_unreachable("Malformed feature string"); 436 } 437 438 void AMDGPUTargetID::setTargetIDFromTargetIDStream(StringRef TargetID) { 439 SmallVector<StringRef, 3> TargetIDSplit; 440 TargetID.split(TargetIDSplit, ':'); 441 442 for (const auto &FeatureString : TargetIDSplit) { 443 if (FeatureString.startswith("xnack")) 444 XnackSetting = getTargetIDSettingFromFeatureString(FeatureString); 445 if (FeatureString.startswith("sramecc")) 446 SramEccSetting = getTargetIDSettingFromFeatureString(FeatureString); 447 } 448 } 449 450 std::string AMDGPUTargetID::toString() const { 451 std::string StringRep; 452 raw_string_ostream StreamRep(StringRep); 453 454 auto TargetTriple = STI.getTargetTriple(); 455 auto Version = getIsaVersion(STI.getCPU()); 456 457 StreamRep << TargetTriple.getArchName() << '-' 458 << TargetTriple.getVendorName() << '-' 459 << TargetTriple.getOSName() << '-' 460 << TargetTriple.getEnvironmentName() << '-'; 461 462 std::string Processor; 463 // TODO: Following else statement is present here because we used various 464 // alias names for GPUs up until GFX9 (e.g. 'fiji' is same as 'gfx803'). 465 // Remove once all aliases are removed from GCNProcessors.td. 466 if (Version.Major >= 9) 467 Processor = STI.getCPU().str(); 468 else 469 Processor = (Twine("gfx") + Twine(Version.Major) + Twine(Version.Minor) + 470 Twine(Version.Stepping)) 471 .str(); 472 473 std::string Features; 474 if (Optional<uint8_t> HsaAbiVersion = getHsaAbiVersion(&STI)) { 475 switch (*HsaAbiVersion) { 476 case ELF::ELFABIVERSION_AMDGPU_HSA_V2: 477 // Code object V2 only supported specific processors and had fixed 478 // settings for the XNACK. 479 if (Processor == "gfx600") { 480 } else if (Processor == "gfx601") { 481 } else if (Processor == "gfx602") { 482 } else if (Processor == "gfx700") { 483 } else if (Processor == "gfx701") { 484 } else if (Processor == "gfx702") { 485 } else if (Processor == "gfx703") { 486 } else if (Processor == "gfx704") { 487 } else if (Processor == "gfx705") { 488 } else if (Processor == "gfx801") { 489 if (!isXnackOnOrAny()) 490 report_fatal_error( 491 "AMD GPU code object V2 does not support processor " + 492 Twine(Processor) + " without XNACK"); 493 } else if (Processor == "gfx802") { 494 } else if (Processor == "gfx803") { 495 } else if (Processor == "gfx805") { 496 } else if (Processor == "gfx810") { 497 if (!isXnackOnOrAny()) 498 report_fatal_error( 499 "AMD GPU code object V2 does not support processor " + 500 Twine(Processor) + " without XNACK"); 501 } else if (Processor == "gfx900") { 502 if (isXnackOnOrAny()) 503 Processor = "gfx901"; 504 } else if (Processor == "gfx902") { 505 if (isXnackOnOrAny()) 506 Processor = "gfx903"; 507 } else if (Processor == "gfx904") { 508 if (isXnackOnOrAny()) 509 Processor = "gfx905"; 510 } else if (Processor == "gfx906") { 511 if (isXnackOnOrAny()) 512 Processor = "gfx907"; 513 } else if (Processor == "gfx90c") { 514 if (isXnackOnOrAny()) 515 report_fatal_error( 516 "AMD GPU code object V2 does not support processor " + 517 Twine(Processor) + " with XNACK being ON or ANY"); 518 } else { 519 report_fatal_error( 520 "AMD GPU code object V2 does not support processor " + 521 Twine(Processor)); 522 } 523 break; 524 case ELF::ELFABIVERSION_AMDGPU_HSA_V3: 525 // xnack. 526 if (isXnackOnOrAny()) 527 Features += "+xnack"; 528 // In code object v2 and v3, "sramecc" feature was spelled with a 529 // hyphen ("sram-ecc"). 530 if (isSramEccOnOrAny()) 531 Features += "+sram-ecc"; 532 break; 533 case ELF::ELFABIVERSION_AMDGPU_HSA_V4: 534 case ELF::ELFABIVERSION_AMDGPU_HSA_V5: 535 // sramecc. 536 if (getSramEccSetting() == TargetIDSetting::Off) 537 Features += ":sramecc-"; 538 else if (getSramEccSetting() == TargetIDSetting::On) 539 Features += ":sramecc+"; 540 // xnack. 541 if (getXnackSetting() == TargetIDSetting::Off) 542 Features += ":xnack-"; 543 else if (getXnackSetting() == TargetIDSetting::On) 544 Features += ":xnack+"; 545 break; 546 default: 547 break; 548 } 549 } 550 551 StreamRep << Processor << Features; 552 553 StreamRep.flush(); 554 return StringRep; 555 } 556 557 unsigned getWavefrontSize(const MCSubtargetInfo *STI) { 558 if (STI->getFeatureBits().test(FeatureWavefrontSize16)) 559 return 16; 560 if (STI->getFeatureBits().test(FeatureWavefrontSize32)) 561 return 32; 562 563 return 64; 564 } 565 566 unsigned getLocalMemorySize(const MCSubtargetInfo *STI) { 567 if (STI->getFeatureBits().test(FeatureLocalMemorySize32768)) 568 return 32768; 569 if (STI->getFeatureBits().test(FeatureLocalMemorySize65536)) 570 return 65536; 571 572 return 0; 573 } 574 575 unsigned getEUsPerCU(const MCSubtargetInfo *STI) { 576 // "Per CU" really means "per whatever functional block the waves of a 577 // workgroup must share". For gfx10 in CU mode this is the CU, which contains 578 // two SIMDs. 579 if (isGFX10Plus(*STI) && STI->getFeatureBits().test(FeatureCuMode)) 580 return 2; 581 // Pre-gfx10 a CU contains four SIMDs. For gfx10 in WGP mode the WGP contains 582 // two CUs, so a total of four SIMDs. 583 return 4; 584 } 585 586 unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo *STI, 587 unsigned FlatWorkGroupSize) { 588 assert(FlatWorkGroupSize != 0); 589 if (STI->getTargetTriple().getArch() != Triple::amdgcn) 590 return 8; 591 unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize); 592 if (N == 1) 593 return 40; 594 N = 40 / N; 595 return std::min(N, 16u); 596 } 597 598 unsigned getMinWavesPerEU(const MCSubtargetInfo *STI) { 599 return 1; 600 } 601 602 unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI) { 603 // FIXME: Need to take scratch memory into account. 604 if (isGFX90A(*STI)) 605 return 8; 606 if (!isGFX10Plus(*STI)) 607 return 10; 608 return hasGFX10_3Insts(*STI) ? 16 : 20; 609 } 610 611 unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo *STI, 612 unsigned FlatWorkGroupSize) { 613 return divideCeil(getWavesPerWorkGroup(STI, FlatWorkGroupSize), 614 getEUsPerCU(STI)); 615 } 616 617 unsigned getMinFlatWorkGroupSize(const MCSubtargetInfo *STI) { 618 return 1; 619 } 620 621 unsigned getMaxFlatWorkGroupSize(const MCSubtargetInfo *STI) { 622 // Some subtargets allow encoding 2048, but this isn't tested or supported. 623 return 1024; 624 } 625 626 unsigned getWavesPerWorkGroup(const MCSubtargetInfo *STI, 627 unsigned FlatWorkGroupSize) { 628 return divideCeil(FlatWorkGroupSize, getWavefrontSize(STI)); 629 } 630 631 unsigned getSGPRAllocGranule(const MCSubtargetInfo *STI) { 632 IsaVersion Version = getIsaVersion(STI->getCPU()); 633 if (Version.Major >= 10) 634 return getAddressableNumSGPRs(STI); 635 if (Version.Major >= 8) 636 return 16; 637 return 8; 638 } 639 640 unsigned getSGPREncodingGranule(const MCSubtargetInfo *STI) { 641 return 8; 642 } 643 644 unsigned getTotalNumSGPRs(const MCSubtargetInfo *STI) { 645 IsaVersion Version = getIsaVersion(STI->getCPU()); 646 if (Version.Major >= 8) 647 return 800; 648 return 512; 649 } 650 651 unsigned getAddressableNumSGPRs(const MCSubtargetInfo *STI) { 652 if (STI->getFeatureBits().test(FeatureSGPRInitBug)) 653 return FIXED_NUM_SGPRS_FOR_INIT_BUG; 654 655 IsaVersion Version = getIsaVersion(STI->getCPU()); 656 if (Version.Major >= 10) 657 return 106; 658 if (Version.Major >= 8) 659 return 102; 660 return 104; 661 } 662 663 unsigned getMinNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 664 assert(WavesPerEU != 0); 665 666 IsaVersion Version = getIsaVersion(STI->getCPU()); 667 if (Version.Major >= 10) 668 return 0; 669 670 if (WavesPerEU >= getMaxWavesPerEU(STI)) 671 return 0; 672 673 unsigned MinNumSGPRs = getTotalNumSGPRs(STI) / (WavesPerEU + 1); 674 if (STI->getFeatureBits().test(FeatureTrapHandler)) 675 MinNumSGPRs -= std::min(MinNumSGPRs, (unsigned)TRAP_NUM_SGPRS); 676 MinNumSGPRs = alignDown(MinNumSGPRs, getSGPRAllocGranule(STI)) + 1; 677 return std::min(MinNumSGPRs, getAddressableNumSGPRs(STI)); 678 } 679 680 unsigned getMaxNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU, 681 bool Addressable) { 682 assert(WavesPerEU != 0); 683 684 unsigned AddressableNumSGPRs = getAddressableNumSGPRs(STI); 685 IsaVersion Version = getIsaVersion(STI->getCPU()); 686 if (Version.Major >= 10) 687 return Addressable ? AddressableNumSGPRs : 108; 688 if (Version.Major >= 8 && !Addressable) 689 AddressableNumSGPRs = 112; 690 unsigned MaxNumSGPRs = getTotalNumSGPRs(STI) / WavesPerEU; 691 if (STI->getFeatureBits().test(FeatureTrapHandler)) 692 MaxNumSGPRs -= std::min(MaxNumSGPRs, (unsigned)TRAP_NUM_SGPRS); 693 MaxNumSGPRs = alignDown(MaxNumSGPRs, getSGPRAllocGranule(STI)); 694 return std::min(MaxNumSGPRs, AddressableNumSGPRs); 695 } 696 697 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed, 698 bool FlatScrUsed, bool XNACKUsed) { 699 unsigned ExtraSGPRs = 0; 700 if (VCCUsed) 701 ExtraSGPRs = 2; 702 703 IsaVersion Version = getIsaVersion(STI->getCPU()); 704 if (Version.Major >= 10) 705 return ExtraSGPRs; 706 707 if (Version.Major < 8) { 708 if (FlatScrUsed) 709 ExtraSGPRs = 4; 710 } else { 711 if (XNACKUsed) 712 ExtraSGPRs = 4; 713 714 if (FlatScrUsed || 715 STI->getFeatureBits().test(AMDGPU::FeatureArchitectedFlatScratch)) 716 ExtraSGPRs = 6; 717 } 718 719 return ExtraSGPRs; 720 } 721 722 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed, 723 bool FlatScrUsed) { 724 return getNumExtraSGPRs(STI, VCCUsed, FlatScrUsed, 725 STI->getFeatureBits().test(AMDGPU::FeatureXNACK)); 726 } 727 728 unsigned getNumSGPRBlocks(const MCSubtargetInfo *STI, unsigned NumSGPRs) { 729 NumSGPRs = alignTo(std::max(1u, NumSGPRs), getSGPREncodingGranule(STI)); 730 // SGPRBlocks is actual number of SGPR blocks minus 1. 731 return NumSGPRs / getSGPREncodingGranule(STI) - 1; 732 } 733 734 unsigned getVGPRAllocGranule(const MCSubtargetInfo *STI, 735 Optional<bool> EnableWavefrontSize32) { 736 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 737 return 8; 738 739 bool IsWave32 = EnableWavefrontSize32 ? 740 *EnableWavefrontSize32 : 741 STI->getFeatureBits().test(FeatureWavefrontSize32); 742 743 if (hasGFX10_3Insts(*STI)) 744 return IsWave32 ? 16 : 8; 745 746 return IsWave32 ? 8 : 4; 747 } 748 749 unsigned getVGPREncodingGranule(const MCSubtargetInfo *STI, 750 Optional<bool> EnableWavefrontSize32) { 751 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 752 return 8; 753 754 bool IsWave32 = EnableWavefrontSize32 ? 755 *EnableWavefrontSize32 : 756 STI->getFeatureBits().test(FeatureWavefrontSize32); 757 758 return IsWave32 ? 8 : 4; 759 } 760 761 unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) { 762 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 763 return 512; 764 if (!isGFX10Plus(*STI)) 765 return 256; 766 return STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1024 : 512; 767 } 768 769 unsigned getAddressableNumVGPRs(const MCSubtargetInfo *STI) { 770 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 771 return 512; 772 return 256; 773 } 774 775 unsigned getMinNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 776 assert(WavesPerEU != 0); 777 778 if (WavesPerEU >= getMaxWavesPerEU(STI)) 779 return 0; 780 unsigned MinNumVGPRs = 781 alignDown(getTotalNumVGPRs(STI) / (WavesPerEU + 1), 782 getVGPRAllocGranule(STI)) + 1; 783 return std::min(MinNumVGPRs, getAddressableNumVGPRs(STI)); 784 } 785 786 unsigned getMaxNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 787 assert(WavesPerEU != 0); 788 789 unsigned MaxNumVGPRs = alignDown(getTotalNumVGPRs(STI) / WavesPerEU, 790 getVGPRAllocGranule(STI)); 791 unsigned AddressableNumVGPRs = getAddressableNumVGPRs(STI); 792 return std::min(MaxNumVGPRs, AddressableNumVGPRs); 793 } 794 795 unsigned getNumVGPRBlocks(const MCSubtargetInfo *STI, unsigned NumVGPRs, 796 Optional<bool> EnableWavefrontSize32) { 797 NumVGPRs = alignTo(std::max(1u, NumVGPRs), 798 getVGPREncodingGranule(STI, EnableWavefrontSize32)); 799 // VGPRBlocks is actual number of VGPR blocks minus 1. 800 return NumVGPRs / getVGPREncodingGranule(STI, EnableWavefrontSize32) - 1; 801 } 802 803 } // end namespace IsaInfo 804 805 void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header, 806 const MCSubtargetInfo *STI) { 807 IsaVersion Version = getIsaVersion(STI->getCPU()); 808 809 memset(&Header, 0, sizeof(Header)); 810 811 Header.amd_kernel_code_version_major = 1; 812 Header.amd_kernel_code_version_minor = 2; 813 Header.amd_machine_kind = 1; // AMD_MACHINE_KIND_AMDGPU 814 Header.amd_machine_version_major = Version.Major; 815 Header.amd_machine_version_minor = Version.Minor; 816 Header.amd_machine_version_stepping = Version.Stepping; 817 Header.kernel_code_entry_byte_offset = sizeof(Header); 818 Header.wavefront_size = 6; 819 820 // If the code object does not support indirect functions, then the value must 821 // be 0xffffffff. 822 Header.call_convention = -1; 823 824 // These alignment values are specified in powers of two, so alignment = 825 // 2^n. The minimum alignment is 2^4 = 16. 826 Header.kernarg_segment_alignment = 4; 827 Header.group_segment_alignment = 4; 828 Header.private_segment_alignment = 4; 829 830 if (Version.Major >= 10) { 831 if (STI->getFeatureBits().test(FeatureWavefrontSize32)) { 832 Header.wavefront_size = 5; 833 Header.code_properties |= AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32; 834 } 835 Header.compute_pgm_resource_registers |= 836 S_00B848_WGP_MODE(STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1) | 837 S_00B848_MEM_ORDERED(1); 838 } 839 } 840 841 amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor( 842 const MCSubtargetInfo *STI) { 843 IsaVersion Version = getIsaVersion(STI->getCPU()); 844 845 amdhsa::kernel_descriptor_t KD; 846 memset(&KD, 0, sizeof(KD)); 847 848 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 849 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, 850 amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE); 851 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 852 amdhsa::COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP, 1); 853 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 854 amdhsa::COMPUTE_PGM_RSRC1_ENABLE_IEEE_MODE, 1); 855 AMDHSA_BITS_SET(KD.compute_pgm_rsrc2, 856 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, 1); 857 if (Version.Major >= 10) { 858 AMDHSA_BITS_SET(KD.kernel_code_properties, 859 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, 860 STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1 : 0); 861 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 862 amdhsa::COMPUTE_PGM_RSRC1_WGP_MODE, 863 STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1); 864 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 865 amdhsa::COMPUTE_PGM_RSRC1_MEM_ORDERED, 1); 866 } 867 if (AMDGPU::isGFX90A(*STI)) { 868 AMDHSA_BITS_SET(KD.compute_pgm_rsrc3, 869 amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, 870 STI->getFeatureBits().test(FeatureTgSplit) ? 1 : 0); 871 } 872 return KD; 873 } 874 875 bool isGroupSegment(const GlobalValue *GV) { 876 return GV->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS; 877 } 878 879 bool isGlobalSegment(const GlobalValue *GV) { 880 return GV->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS; 881 } 882 883 bool isReadOnlySegment(const GlobalValue *GV) { 884 unsigned AS = GV->getAddressSpace(); 885 return AS == AMDGPUAS::CONSTANT_ADDRESS || 886 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT; 887 } 888 889 bool shouldEmitConstantsToTextSection(const Triple &TT) { 890 return TT.getArch() == Triple::r600; 891 } 892 893 int getIntegerAttribute(const Function &F, StringRef Name, int Default) { 894 Attribute A = F.getFnAttribute(Name); 895 int Result = Default; 896 897 if (A.isStringAttribute()) { 898 StringRef Str = A.getValueAsString(); 899 if (Str.getAsInteger(0, Result)) { 900 LLVMContext &Ctx = F.getContext(); 901 Ctx.emitError("can't parse integer attribute " + Name); 902 } 903 } 904 905 return Result; 906 } 907 908 std::pair<int, int> getIntegerPairAttribute(const Function &F, 909 StringRef Name, 910 std::pair<int, int> Default, 911 bool OnlyFirstRequired) { 912 Attribute A = F.getFnAttribute(Name); 913 if (!A.isStringAttribute()) 914 return Default; 915 916 LLVMContext &Ctx = F.getContext(); 917 std::pair<int, int> Ints = Default; 918 std::pair<StringRef, StringRef> Strs = A.getValueAsString().split(','); 919 if (Strs.first.trim().getAsInteger(0, Ints.first)) { 920 Ctx.emitError("can't parse first integer attribute " + Name); 921 return Default; 922 } 923 if (Strs.second.trim().getAsInteger(0, Ints.second)) { 924 if (!OnlyFirstRequired || !Strs.second.trim().empty()) { 925 Ctx.emitError("can't parse second integer attribute " + Name); 926 return Default; 927 } 928 } 929 930 return Ints; 931 } 932 933 unsigned getVmcntBitMask(const IsaVersion &Version) { 934 unsigned VmcntLo = (1 << getVmcntBitWidthLo()) - 1; 935 if (Version.Major < 9) 936 return VmcntLo; 937 938 unsigned VmcntHi = ((1 << getVmcntBitWidthHi()) - 1) << getVmcntBitWidthLo(); 939 return VmcntLo | VmcntHi; 940 } 941 942 unsigned getExpcntBitMask(const IsaVersion &Version) { 943 return (1 << getExpcntBitWidth()) - 1; 944 } 945 946 unsigned getLgkmcntBitMask(const IsaVersion &Version) { 947 return (1 << getLgkmcntBitWidth(Version.Major)) - 1; 948 } 949 950 unsigned getWaitcntBitMask(const IsaVersion &Version) { 951 unsigned VmcntLo = getBitMask(getVmcntBitShiftLo(), getVmcntBitWidthLo()); 952 unsigned Expcnt = getBitMask(getExpcntBitShift(), getExpcntBitWidth()); 953 unsigned Lgkmcnt = getBitMask(getLgkmcntBitShift(), 954 getLgkmcntBitWidth(Version.Major)); 955 unsigned Waitcnt = VmcntLo | Expcnt | Lgkmcnt; 956 if (Version.Major < 9) 957 return Waitcnt; 958 959 unsigned VmcntHi = getBitMask(getVmcntBitShiftHi(), getVmcntBitWidthHi()); 960 return Waitcnt | VmcntHi; 961 } 962 963 unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) { 964 unsigned VmcntLo = 965 unpackBits(Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo()); 966 if (Version.Major < 9) 967 return VmcntLo; 968 969 unsigned VmcntHi = 970 unpackBits(Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi()); 971 VmcntHi <<= getVmcntBitWidthLo(); 972 return VmcntLo | VmcntHi; 973 } 974 975 unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) { 976 return unpackBits(Waitcnt, getExpcntBitShift(), getExpcntBitWidth()); 977 } 978 979 unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) { 980 return unpackBits(Waitcnt, getLgkmcntBitShift(), 981 getLgkmcntBitWidth(Version.Major)); 982 } 983 984 void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, 985 unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt) { 986 Vmcnt = decodeVmcnt(Version, Waitcnt); 987 Expcnt = decodeExpcnt(Version, Waitcnt); 988 Lgkmcnt = decodeLgkmcnt(Version, Waitcnt); 989 } 990 991 Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded) { 992 Waitcnt Decoded; 993 Decoded.VmCnt = decodeVmcnt(Version, Encoded); 994 Decoded.ExpCnt = decodeExpcnt(Version, Encoded); 995 Decoded.LgkmCnt = decodeLgkmcnt(Version, Encoded); 996 return Decoded; 997 } 998 999 unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt, 1000 unsigned Vmcnt) { 1001 Waitcnt = 1002 packBits(Vmcnt, Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo()); 1003 if (Version.Major < 9) 1004 return Waitcnt; 1005 1006 Vmcnt >>= getVmcntBitWidthLo(); 1007 return packBits(Vmcnt, Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi()); 1008 } 1009 1010 unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt, 1011 unsigned Expcnt) { 1012 return packBits(Expcnt, Waitcnt, getExpcntBitShift(), getExpcntBitWidth()); 1013 } 1014 1015 unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt, 1016 unsigned Lgkmcnt) { 1017 return packBits(Lgkmcnt, Waitcnt, getLgkmcntBitShift(), 1018 getLgkmcntBitWidth(Version.Major)); 1019 } 1020 1021 unsigned encodeWaitcnt(const IsaVersion &Version, 1022 unsigned Vmcnt, unsigned Expcnt, unsigned Lgkmcnt) { 1023 unsigned Waitcnt = getWaitcntBitMask(Version); 1024 Waitcnt = encodeVmcnt(Version, Waitcnt, Vmcnt); 1025 Waitcnt = encodeExpcnt(Version, Waitcnt, Expcnt); 1026 Waitcnt = encodeLgkmcnt(Version, Waitcnt, Lgkmcnt); 1027 return Waitcnt; 1028 } 1029 1030 unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded) { 1031 return encodeWaitcnt(Version, Decoded.VmCnt, Decoded.ExpCnt, Decoded.LgkmCnt); 1032 } 1033 1034 //===----------------------------------------------------------------------===// 1035 // hwreg 1036 //===----------------------------------------------------------------------===// 1037 1038 namespace Hwreg { 1039 1040 int64_t getHwregId(const StringRef Name, const MCSubtargetInfo &STI) { 1041 if (isGFX10(STI) && Name == "HW_REG_HW_ID") // An alias 1042 return ID_HW_ID1; 1043 for (int Id = ID_SYMBOLIC_FIRST_; Id < ID_SYMBOLIC_LAST_; ++Id) { 1044 if (IdSymbolic[Id] && Name == IdSymbolic[Id]) 1045 return Id; 1046 } 1047 return ID_UNKNOWN_; 1048 } 1049 1050 static unsigned getLastSymbolicHwreg(const MCSubtargetInfo &STI) { 1051 if (isSI(STI) || isCI(STI) || isVI(STI)) 1052 return ID_SYMBOLIC_FIRST_GFX9_; 1053 else if (isGFX9(STI)) 1054 return ID_SYMBOLIC_FIRST_GFX10_; 1055 else if (isGFX10(STI) && !isGFX10_BEncoding(STI)) 1056 return ID_SYMBOLIC_FIRST_GFX1030_; 1057 else 1058 return ID_SYMBOLIC_LAST_; 1059 } 1060 1061 bool isValidHwreg(int64_t Id, const MCSubtargetInfo &STI) { 1062 switch (Id) { 1063 case ID_HW_ID: 1064 return isSI(STI) || isCI(STI) || isVI(STI) || isGFX9(STI); 1065 case ID_HW_ID1: 1066 case ID_HW_ID2: 1067 return isGFX10Plus(STI); 1068 case ID_XNACK_MASK: 1069 return isGFX10(STI) && !AMDGPU::isGFX10_BEncoding(STI); 1070 default: 1071 return ID_SYMBOLIC_FIRST_ <= Id && Id < getLastSymbolicHwreg(STI) && 1072 IdSymbolic[Id]; 1073 } 1074 } 1075 1076 bool isValidHwreg(int64_t Id) { 1077 return 0 <= Id && isUInt<ID_WIDTH_>(Id); 1078 } 1079 1080 bool isValidHwregOffset(int64_t Offset) { 1081 return 0 <= Offset && isUInt<OFFSET_WIDTH_>(Offset); 1082 } 1083 1084 bool isValidHwregWidth(int64_t Width) { 1085 return 0 <= (Width - 1) && isUInt<WIDTH_M1_WIDTH_>(Width - 1); 1086 } 1087 1088 uint64_t encodeHwreg(uint64_t Id, uint64_t Offset, uint64_t Width) { 1089 return (Id << ID_SHIFT_) | 1090 (Offset << OFFSET_SHIFT_) | 1091 ((Width - 1) << WIDTH_M1_SHIFT_); 1092 } 1093 1094 StringRef getHwreg(unsigned Id, const MCSubtargetInfo &STI) { 1095 return isValidHwreg(Id, STI) ? IdSymbolic[Id] : ""; 1096 } 1097 1098 void decodeHwreg(unsigned Val, unsigned &Id, unsigned &Offset, unsigned &Width) { 1099 Id = (Val & ID_MASK_) >> ID_SHIFT_; 1100 Offset = (Val & OFFSET_MASK_) >> OFFSET_SHIFT_; 1101 Width = ((Val & WIDTH_M1_MASK_) >> WIDTH_M1_SHIFT_) + 1; 1102 } 1103 1104 } // namespace Hwreg 1105 1106 //===----------------------------------------------------------------------===// 1107 // exp tgt 1108 //===----------------------------------------------------------------------===// 1109 1110 namespace Exp { 1111 1112 struct ExpTgt { 1113 StringLiteral Name; 1114 unsigned Tgt; 1115 unsigned MaxIndex; 1116 }; 1117 1118 static constexpr ExpTgt ExpTgtInfo[] = { 1119 {{"null"}, ET_NULL, ET_NULL_MAX_IDX}, 1120 {{"mrtz"}, ET_MRTZ, ET_MRTZ_MAX_IDX}, 1121 {{"prim"}, ET_PRIM, ET_PRIM_MAX_IDX}, 1122 {{"mrt"}, ET_MRT0, ET_MRT_MAX_IDX}, 1123 {{"pos"}, ET_POS0, ET_POS_MAX_IDX}, 1124 {{"param"}, ET_PARAM0, ET_PARAM_MAX_IDX}, 1125 }; 1126 1127 bool getTgtName(unsigned Id, StringRef &Name, int &Index) { 1128 for (const ExpTgt &Val : ExpTgtInfo) { 1129 if (Val.Tgt <= Id && Id <= Val.Tgt + Val.MaxIndex) { 1130 Index = (Val.MaxIndex == 0) ? -1 : (Id - Val.Tgt); 1131 Name = Val.Name; 1132 return true; 1133 } 1134 } 1135 return false; 1136 } 1137 1138 unsigned getTgtId(const StringRef Name) { 1139 1140 for (const ExpTgt &Val : ExpTgtInfo) { 1141 if (Val.MaxIndex == 0 && Name == Val.Name) 1142 return Val.Tgt; 1143 1144 if (Val.MaxIndex > 0 && Name.startswith(Val.Name)) { 1145 StringRef Suffix = Name.drop_front(Val.Name.size()); 1146 1147 unsigned Id; 1148 if (Suffix.getAsInteger(10, Id) || Id > Val.MaxIndex) 1149 return ET_INVALID; 1150 1151 // Disable leading zeroes 1152 if (Suffix.size() > 1 && Suffix[0] == '0') 1153 return ET_INVALID; 1154 1155 return Val.Tgt + Id; 1156 } 1157 } 1158 return ET_INVALID; 1159 } 1160 1161 bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI) { 1162 return (Id != ET_POS4 && Id != ET_PRIM) || isGFX10Plus(STI); 1163 } 1164 1165 } // namespace Exp 1166 1167 //===----------------------------------------------------------------------===// 1168 // MTBUF Format 1169 //===----------------------------------------------------------------------===// 1170 1171 namespace MTBUFFormat { 1172 1173 int64_t getDfmt(const StringRef Name) { 1174 for (int Id = DFMT_MIN; Id <= DFMT_MAX; ++Id) { 1175 if (Name == DfmtSymbolic[Id]) 1176 return Id; 1177 } 1178 return DFMT_UNDEF; 1179 } 1180 1181 StringRef getDfmtName(unsigned Id) { 1182 assert(Id <= DFMT_MAX); 1183 return DfmtSymbolic[Id]; 1184 } 1185 1186 static StringLiteral const *getNfmtLookupTable(const MCSubtargetInfo &STI) { 1187 if (isSI(STI) || isCI(STI)) 1188 return NfmtSymbolicSICI; 1189 if (isVI(STI) || isGFX9(STI)) 1190 return NfmtSymbolicVI; 1191 return NfmtSymbolicGFX10; 1192 } 1193 1194 int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI) { 1195 auto lookupTable = getNfmtLookupTable(STI); 1196 for (int Id = NFMT_MIN; Id <= NFMT_MAX; ++Id) { 1197 if (Name == lookupTable[Id]) 1198 return Id; 1199 } 1200 return NFMT_UNDEF; 1201 } 1202 1203 StringRef getNfmtName(unsigned Id, const MCSubtargetInfo &STI) { 1204 assert(Id <= NFMT_MAX); 1205 return getNfmtLookupTable(STI)[Id]; 1206 } 1207 1208 bool isValidDfmtNfmt(unsigned Id, const MCSubtargetInfo &STI) { 1209 unsigned Dfmt; 1210 unsigned Nfmt; 1211 decodeDfmtNfmt(Id, Dfmt, Nfmt); 1212 return isValidNfmt(Nfmt, STI); 1213 } 1214 1215 bool isValidNfmt(unsigned Id, const MCSubtargetInfo &STI) { 1216 return !getNfmtName(Id, STI).empty(); 1217 } 1218 1219 int64_t encodeDfmtNfmt(unsigned Dfmt, unsigned Nfmt) { 1220 return (Dfmt << DFMT_SHIFT) | (Nfmt << NFMT_SHIFT); 1221 } 1222 1223 void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt) { 1224 Dfmt = (Format >> DFMT_SHIFT) & DFMT_MASK; 1225 Nfmt = (Format >> NFMT_SHIFT) & NFMT_MASK; 1226 } 1227 1228 int64_t getUnifiedFormat(const StringRef Name) { 1229 for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) { 1230 if (Name == UfmtSymbolic[Id]) 1231 return Id; 1232 } 1233 return UFMT_UNDEF; 1234 } 1235 1236 StringRef getUnifiedFormatName(unsigned Id) { 1237 return isValidUnifiedFormat(Id) ? UfmtSymbolic[Id] : ""; 1238 } 1239 1240 bool isValidUnifiedFormat(unsigned Id) { 1241 return Id <= UFMT_LAST; 1242 } 1243 1244 int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt) { 1245 int64_t Fmt = encodeDfmtNfmt(Dfmt, Nfmt); 1246 for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) { 1247 if (Fmt == DfmtNfmt2UFmt[Id]) 1248 return Id; 1249 } 1250 return UFMT_UNDEF; 1251 } 1252 1253 bool isValidFormatEncoding(unsigned Val, const MCSubtargetInfo &STI) { 1254 return isGFX10Plus(STI) ? (Val <= UFMT_MAX) : (Val <= DFMT_NFMT_MAX); 1255 } 1256 1257 unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI) { 1258 if (isGFX10Plus(STI)) 1259 return UFMT_DEFAULT; 1260 return DFMT_NFMT_DEFAULT; 1261 } 1262 1263 } // namespace MTBUFFormat 1264 1265 //===----------------------------------------------------------------------===// 1266 // SendMsg 1267 //===----------------------------------------------------------------------===// 1268 1269 namespace SendMsg { 1270 1271 int64_t getMsgId(const StringRef Name) { 1272 for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) { 1273 if (IdSymbolic[i] && Name == IdSymbolic[i]) 1274 return i; 1275 } 1276 return ID_UNKNOWN_; 1277 } 1278 1279 bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI, bool Strict) { 1280 if (Strict) { 1281 switch (MsgId) { 1282 case ID_SAVEWAVE: 1283 return isVI(STI) || isGFX9Plus(STI); 1284 case ID_STALL_WAVE_GEN: 1285 case ID_HALT_WAVES: 1286 case ID_ORDERED_PS_DONE: 1287 case ID_GS_ALLOC_REQ: 1288 case ID_GET_DOORBELL: 1289 return isGFX9Plus(STI); 1290 case ID_EARLY_PRIM_DEALLOC: 1291 return isGFX9(STI); 1292 case ID_GET_DDID: 1293 return isGFX10Plus(STI); 1294 default: 1295 return 0 <= MsgId && MsgId < ID_GAPS_LAST_ && IdSymbolic[MsgId]; 1296 } 1297 } else { 1298 return 0 <= MsgId && isUInt<ID_WIDTH_>(MsgId); 1299 } 1300 } 1301 1302 StringRef getMsgName(int64_t MsgId) { 1303 assert(0 <= MsgId && MsgId < ID_GAPS_LAST_); 1304 return IdSymbolic[MsgId]; 1305 } 1306 1307 int64_t getMsgOpId(int64_t MsgId, const StringRef Name) { 1308 const char* const *S = (MsgId == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic; 1309 const int F = (MsgId == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_; 1310 const int L = (MsgId == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_; 1311 for (int i = F; i < L; ++i) { 1312 if (Name == S[i]) { 1313 return i; 1314 } 1315 } 1316 return OP_UNKNOWN_; 1317 } 1318 1319 bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI, 1320 bool Strict) { 1321 assert(isValidMsgId(MsgId, STI, Strict)); 1322 1323 if (!Strict) 1324 return 0 <= OpId && isUInt<OP_WIDTH_>(OpId); 1325 1326 switch(MsgId) 1327 { 1328 case ID_GS: 1329 return (OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_) && OpId != OP_GS_NOP; 1330 case ID_GS_DONE: 1331 return OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_; 1332 case ID_SYSMSG: 1333 return OP_SYS_FIRST_ <= OpId && OpId < OP_SYS_LAST_; 1334 default: 1335 return OpId == OP_NONE_; 1336 } 1337 } 1338 1339 StringRef getMsgOpName(int64_t MsgId, int64_t OpId) { 1340 assert(msgRequiresOp(MsgId)); 1341 return (MsgId == ID_SYSMSG)? OpSysSymbolic[OpId] : OpGsSymbolic[OpId]; 1342 } 1343 1344 bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId, 1345 const MCSubtargetInfo &STI, bool Strict) { 1346 assert(isValidMsgOp(MsgId, OpId, STI, Strict)); 1347 1348 if (!Strict) 1349 return 0 <= StreamId && isUInt<STREAM_ID_WIDTH_>(StreamId); 1350 1351 switch(MsgId) 1352 { 1353 case ID_GS: 1354 return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_; 1355 case ID_GS_DONE: 1356 return (OpId == OP_GS_NOP)? 1357 (StreamId == STREAM_ID_NONE_) : 1358 (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_); 1359 default: 1360 return StreamId == STREAM_ID_NONE_; 1361 } 1362 } 1363 1364 bool msgRequiresOp(int64_t MsgId) { 1365 return MsgId == ID_GS || MsgId == ID_GS_DONE || MsgId == ID_SYSMSG; 1366 } 1367 1368 bool msgSupportsStream(int64_t MsgId, int64_t OpId) { 1369 return (MsgId == ID_GS || MsgId == ID_GS_DONE) && OpId != OP_GS_NOP; 1370 } 1371 1372 void decodeMsg(unsigned Val, 1373 uint16_t &MsgId, 1374 uint16_t &OpId, 1375 uint16_t &StreamId) { 1376 MsgId = Val & ID_MASK_; 1377 OpId = (Val & OP_MASK_) >> OP_SHIFT_; 1378 StreamId = (Val & STREAM_ID_MASK_) >> STREAM_ID_SHIFT_; 1379 } 1380 1381 uint64_t encodeMsg(uint64_t MsgId, 1382 uint64_t OpId, 1383 uint64_t StreamId) { 1384 return (MsgId << ID_SHIFT_) | 1385 (OpId << OP_SHIFT_) | 1386 (StreamId << STREAM_ID_SHIFT_); 1387 } 1388 1389 } // namespace SendMsg 1390 1391 //===----------------------------------------------------------------------===// 1392 // 1393 //===----------------------------------------------------------------------===// 1394 1395 unsigned getInitialPSInputAddr(const Function &F) { 1396 return getIntegerAttribute(F, "InitialPSInputAddr", 0); 1397 } 1398 1399 bool getHasColorExport(const Function &F) { 1400 // As a safe default always respond as if PS has color exports. 1401 return getIntegerAttribute( 1402 F, "amdgpu-color-export", 1403 F.getCallingConv() == CallingConv::AMDGPU_PS ? 1 : 0) != 0; 1404 } 1405 1406 bool getHasDepthExport(const Function &F) { 1407 return getIntegerAttribute(F, "amdgpu-depth-export", 0) != 0; 1408 } 1409 1410 bool isShader(CallingConv::ID cc) { 1411 switch(cc) { 1412 case CallingConv::AMDGPU_VS: 1413 case CallingConv::AMDGPU_LS: 1414 case CallingConv::AMDGPU_HS: 1415 case CallingConv::AMDGPU_ES: 1416 case CallingConv::AMDGPU_GS: 1417 case CallingConv::AMDGPU_PS: 1418 case CallingConv::AMDGPU_CS: 1419 return true; 1420 default: 1421 return false; 1422 } 1423 } 1424 1425 bool isGraphics(CallingConv::ID cc) { 1426 return isShader(cc) || cc == CallingConv::AMDGPU_Gfx; 1427 } 1428 1429 bool isCompute(CallingConv::ID cc) { 1430 return !isGraphics(cc) || cc == CallingConv::AMDGPU_CS; 1431 } 1432 1433 bool isEntryFunctionCC(CallingConv::ID CC) { 1434 switch (CC) { 1435 case CallingConv::AMDGPU_KERNEL: 1436 case CallingConv::SPIR_KERNEL: 1437 case CallingConv::AMDGPU_VS: 1438 case CallingConv::AMDGPU_GS: 1439 case CallingConv::AMDGPU_PS: 1440 case CallingConv::AMDGPU_CS: 1441 case CallingConv::AMDGPU_ES: 1442 case CallingConv::AMDGPU_HS: 1443 case CallingConv::AMDGPU_LS: 1444 return true; 1445 default: 1446 return false; 1447 } 1448 } 1449 1450 bool isModuleEntryFunctionCC(CallingConv::ID CC) { 1451 switch (CC) { 1452 case CallingConv::AMDGPU_Gfx: 1453 return true; 1454 default: 1455 return isEntryFunctionCC(CC); 1456 } 1457 } 1458 1459 bool isKernelCC(const Function *Func) { 1460 return AMDGPU::isModuleEntryFunctionCC(Func->getCallingConv()); 1461 } 1462 1463 bool hasXNACK(const MCSubtargetInfo &STI) { 1464 return STI.getFeatureBits()[AMDGPU::FeatureXNACK]; 1465 } 1466 1467 bool hasSRAMECC(const MCSubtargetInfo &STI) { 1468 return STI.getFeatureBits()[AMDGPU::FeatureSRAMECC]; 1469 } 1470 1471 bool hasMIMG_R128(const MCSubtargetInfo &STI) { 1472 return STI.getFeatureBits()[AMDGPU::FeatureMIMG_R128] && !STI.getFeatureBits()[AMDGPU::FeatureR128A16]; 1473 } 1474 1475 bool hasGFX10A16(const MCSubtargetInfo &STI) { 1476 return STI.getFeatureBits()[AMDGPU::FeatureGFX10A16]; 1477 } 1478 1479 bool hasG16(const MCSubtargetInfo &STI) { 1480 return STI.getFeatureBits()[AMDGPU::FeatureG16]; 1481 } 1482 1483 bool hasPackedD16(const MCSubtargetInfo &STI) { 1484 return !STI.getFeatureBits()[AMDGPU::FeatureUnpackedD16VMem]; 1485 } 1486 1487 bool isSI(const MCSubtargetInfo &STI) { 1488 return STI.getFeatureBits()[AMDGPU::FeatureSouthernIslands]; 1489 } 1490 1491 bool isCI(const MCSubtargetInfo &STI) { 1492 return STI.getFeatureBits()[AMDGPU::FeatureSeaIslands]; 1493 } 1494 1495 bool isVI(const MCSubtargetInfo &STI) { 1496 return STI.getFeatureBits()[AMDGPU::FeatureVolcanicIslands]; 1497 } 1498 1499 bool isGFX9(const MCSubtargetInfo &STI) { 1500 return STI.getFeatureBits()[AMDGPU::FeatureGFX9]; 1501 } 1502 1503 bool isGFX9Plus(const MCSubtargetInfo &STI) { 1504 return isGFX9(STI) || isGFX10Plus(STI); 1505 } 1506 1507 bool isGFX10(const MCSubtargetInfo &STI) { 1508 return STI.getFeatureBits()[AMDGPU::FeatureGFX10]; 1509 } 1510 1511 bool isGFX10Plus(const MCSubtargetInfo &STI) { return isGFX10(STI); } 1512 1513 bool isGCN3Encoding(const MCSubtargetInfo &STI) { 1514 return STI.getFeatureBits()[AMDGPU::FeatureGCN3Encoding]; 1515 } 1516 1517 bool isGFX10_AEncoding(const MCSubtargetInfo &STI) { 1518 return STI.getFeatureBits()[AMDGPU::FeatureGFX10_AEncoding]; 1519 } 1520 1521 bool isGFX10_BEncoding(const MCSubtargetInfo &STI) { 1522 return STI.getFeatureBits()[AMDGPU::FeatureGFX10_BEncoding]; 1523 } 1524 1525 bool hasGFX10_3Insts(const MCSubtargetInfo &STI) { 1526 return STI.getFeatureBits()[AMDGPU::FeatureGFX10_3Insts]; 1527 } 1528 1529 bool isGFX90A(const MCSubtargetInfo &STI) { 1530 return STI.getFeatureBits()[AMDGPU::FeatureGFX90AInsts]; 1531 } 1532 1533 bool isGFX940(const MCSubtargetInfo &STI) { 1534 return STI.getFeatureBits()[AMDGPU::FeatureGFX940Insts]; 1535 } 1536 1537 bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI) { 1538 return STI.getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch]; 1539 } 1540 1541 bool hasMAIInsts(const MCSubtargetInfo &STI) { 1542 return STI.getFeatureBits()[AMDGPU::FeatureMAIInsts]; 1543 } 1544 1545 int32_t getTotalNumVGPRs(bool has90AInsts, int32_t ArgNumAGPR, 1546 int32_t ArgNumVGPR) { 1547 if (has90AInsts && ArgNumAGPR) 1548 return alignTo(ArgNumVGPR, 4) + ArgNumAGPR; 1549 return std::max(ArgNumVGPR, ArgNumAGPR); 1550 } 1551 1552 bool isSGPR(unsigned Reg, const MCRegisterInfo* TRI) { 1553 const MCRegisterClass SGPRClass = TRI->getRegClass(AMDGPU::SReg_32RegClassID); 1554 const unsigned FirstSubReg = TRI->getSubReg(Reg, AMDGPU::sub0); 1555 return SGPRClass.contains(FirstSubReg != 0 ? FirstSubReg : Reg) || 1556 Reg == AMDGPU::SCC; 1557 } 1558 1559 #define MAP_REG2REG \ 1560 using namespace AMDGPU; \ 1561 switch(Reg) { \ 1562 default: return Reg; \ 1563 CASE_CI_VI(FLAT_SCR) \ 1564 CASE_CI_VI(FLAT_SCR_LO) \ 1565 CASE_CI_VI(FLAT_SCR_HI) \ 1566 CASE_VI_GFX9PLUS(TTMP0) \ 1567 CASE_VI_GFX9PLUS(TTMP1) \ 1568 CASE_VI_GFX9PLUS(TTMP2) \ 1569 CASE_VI_GFX9PLUS(TTMP3) \ 1570 CASE_VI_GFX9PLUS(TTMP4) \ 1571 CASE_VI_GFX9PLUS(TTMP5) \ 1572 CASE_VI_GFX9PLUS(TTMP6) \ 1573 CASE_VI_GFX9PLUS(TTMP7) \ 1574 CASE_VI_GFX9PLUS(TTMP8) \ 1575 CASE_VI_GFX9PLUS(TTMP9) \ 1576 CASE_VI_GFX9PLUS(TTMP10) \ 1577 CASE_VI_GFX9PLUS(TTMP11) \ 1578 CASE_VI_GFX9PLUS(TTMP12) \ 1579 CASE_VI_GFX9PLUS(TTMP13) \ 1580 CASE_VI_GFX9PLUS(TTMP14) \ 1581 CASE_VI_GFX9PLUS(TTMP15) \ 1582 CASE_VI_GFX9PLUS(TTMP0_TTMP1) \ 1583 CASE_VI_GFX9PLUS(TTMP2_TTMP3) \ 1584 CASE_VI_GFX9PLUS(TTMP4_TTMP5) \ 1585 CASE_VI_GFX9PLUS(TTMP6_TTMP7) \ 1586 CASE_VI_GFX9PLUS(TTMP8_TTMP9) \ 1587 CASE_VI_GFX9PLUS(TTMP10_TTMP11) \ 1588 CASE_VI_GFX9PLUS(TTMP12_TTMP13) \ 1589 CASE_VI_GFX9PLUS(TTMP14_TTMP15) \ 1590 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3) \ 1591 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7) \ 1592 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11) \ 1593 CASE_VI_GFX9PLUS(TTMP12_TTMP13_TTMP14_TTMP15) \ 1594 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7) \ 1595 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11) \ 1596 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \ 1597 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \ 1598 } 1599 1600 #define CASE_CI_VI(node) \ 1601 assert(!isSI(STI)); \ 1602 case node: return isCI(STI) ? node##_ci : node##_vi; 1603 1604 #define CASE_VI_GFX9PLUS(node) \ 1605 case node: return isGFX9Plus(STI) ? node##_gfx9plus : node##_vi; 1606 1607 unsigned getMCReg(unsigned Reg, const MCSubtargetInfo &STI) { 1608 if (STI.getTargetTriple().getArch() == Triple::r600) 1609 return Reg; 1610 MAP_REG2REG 1611 } 1612 1613 #undef CASE_CI_VI 1614 #undef CASE_VI_GFX9PLUS 1615 1616 #define CASE_CI_VI(node) case node##_ci: case node##_vi: return node; 1617 #define CASE_VI_GFX9PLUS(node) case node##_vi: case node##_gfx9plus: return node; 1618 1619 unsigned mc2PseudoReg(unsigned Reg) { 1620 MAP_REG2REG 1621 } 1622 1623 #undef CASE_CI_VI 1624 #undef CASE_VI_GFX9PLUS 1625 #undef MAP_REG2REG 1626 1627 bool isSISrcOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1628 assert(OpNo < Desc.NumOperands); 1629 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1630 return OpType >= AMDGPU::OPERAND_SRC_FIRST && 1631 OpType <= AMDGPU::OPERAND_SRC_LAST; 1632 } 1633 1634 bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1635 assert(OpNo < Desc.NumOperands); 1636 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1637 switch (OpType) { 1638 case AMDGPU::OPERAND_REG_IMM_FP32: 1639 case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED: 1640 case AMDGPU::OPERAND_REG_IMM_FP64: 1641 case AMDGPU::OPERAND_REG_IMM_FP16: 1642 case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED: 1643 case AMDGPU::OPERAND_REG_IMM_V2FP16: 1644 case AMDGPU::OPERAND_REG_IMM_V2INT16: 1645 case AMDGPU::OPERAND_REG_INLINE_C_FP32: 1646 case AMDGPU::OPERAND_REG_INLINE_C_FP64: 1647 case AMDGPU::OPERAND_REG_INLINE_C_FP16: 1648 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: 1649 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: 1650 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: 1651 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: 1652 case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: 1653 case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16: 1654 case AMDGPU::OPERAND_REG_IMM_V2FP32: 1655 case AMDGPU::OPERAND_REG_INLINE_C_V2FP32: 1656 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: 1657 return true; 1658 default: 1659 return false; 1660 } 1661 } 1662 1663 bool isSISrcInlinableOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1664 assert(OpNo < Desc.NumOperands); 1665 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1666 return OpType >= AMDGPU::OPERAND_REG_INLINE_C_FIRST && 1667 OpType <= AMDGPU::OPERAND_REG_INLINE_C_LAST; 1668 } 1669 1670 // Avoid using MCRegisterClass::getSize, since that function will go away 1671 // (move from MC* level to Target* level). Return size in bits. 1672 unsigned getRegBitWidth(unsigned RCID) { 1673 switch (RCID) { 1674 case AMDGPU::VGPR_LO16RegClassID: 1675 case AMDGPU::VGPR_HI16RegClassID: 1676 case AMDGPU::SGPR_LO16RegClassID: 1677 case AMDGPU::AGPR_LO16RegClassID: 1678 return 16; 1679 case AMDGPU::SGPR_32RegClassID: 1680 case AMDGPU::VGPR_32RegClassID: 1681 case AMDGPU::VRegOrLds_32RegClassID: 1682 case AMDGPU::AGPR_32RegClassID: 1683 case AMDGPU::VS_32RegClassID: 1684 case AMDGPU::AV_32RegClassID: 1685 case AMDGPU::SReg_32RegClassID: 1686 case AMDGPU::SReg_32_XM0RegClassID: 1687 case AMDGPU::SRegOrLds_32RegClassID: 1688 return 32; 1689 case AMDGPU::SGPR_64RegClassID: 1690 case AMDGPU::VS_64RegClassID: 1691 case AMDGPU::SReg_64RegClassID: 1692 case AMDGPU::VReg_64RegClassID: 1693 case AMDGPU::AReg_64RegClassID: 1694 case AMDGPU::SReg_64_XEXECRegClassID: 1695 case AMDGPU::VReg_64_Align2RegClassID: 1696 case AMDGPU::AReg_64_Align2RegClassID: 1697 case AMDGPU::AV_64RegClassID: 1698 case AMDGPU::AV_64_Align2RegClassID: 1699 return 64; 1700 case AMDGPU::SGPR_96RegClassID: 1701 case AMDGPU::SReg_96RegClassID: 1702 case AMDGPU::VReg_96RegClassID: 1703 case AMDGPU::AReg_96RegClassID: 1704 case AMDGPU::VReg_96_Align2RegClassID: 1705 case AMDGPU::AReg_96_Align2RegClassID: 1706 case AMDGPU::AV_96RegClassID: 1707 case AMDGPU::AV_96_Align2RegClassID: 1708 return 96; 1709 case AMDGPU::SGPR_128RegClassID: 1710 case AMDGPU::SReg_128RegClassID: 1711 case AMDGPU::VReg_128RegClassID: 1712 case AMDGPU::AReg_128RegClassID: 1713 case AMDGPU::VReg_128_Align2RegClassID: 1714 case AMDGPU::AReg_128_Align2RegClassID: 1715 case AMDGPU::AV_128RegClassID: 1716 case AMDGPU::AV_128_Align2RegClassID: 1717 return 128; 1718 case AMDGPU::SGPR_160RegClassID: 1719 case AMDGPU::SReg_160RegClassID: 1720 case AMDGPU::VReg_160RegClassID: 1721 case AMDGPU::AReg_160RegClassID: 1722 case AMDGPU::VReg_160_Align2RegClassID: 1723 case AMDGPU::AReg_160_Align2RegClassID: 1724 case AMDGPU::AV_160RegClassID: 1725 case AMDGPU::AV_160_Align2RegClassID: 1726 return 160; 1727 case AMDGPU::SGPR_192RegClassID: 1728 case AMDGPU::SReg_192RegClassID: 1729 case AMDGPU::VReg_192RegClassID: 1730 case AMDGPU::AReg_192RegClassID: 1731 case AMDGPU::VReg_192_Align2RegClassID: 1732 case AMDGPU::AReg_192_Align2RegClassID: 1733 case AMDGPU::AV_192RegClassID: 1734 case AMDGPU::AV_192_Align2RegClassID: 1735 return 192; 1736 case AMDGPU::SGPR_224RegClassID: 1737 case AMDGPU::SReg_224RegClassID: 1738 case AMDGPU::VReg_224RegClassID: 1739 case AMDGPU::AReg_224RegClassID: 1740 case AMDGPU::VReg_224_Align2RegClassID: 1741 case AMDGPU::AReg_224_Align2RegClassID: 1742 case AMDGPU::AV_224RegClassID: 1743 case AMDGPU::AV_224_Align2RegClassID: 1744 return 224; 1745 case AMDGPU::SGPR_256RegClassID: 1746 case AMDGPU::SReg_256RegClassID: 1747 case AMDGPU::VReg_256RegClassID: 1748 case AMDGPU::AReg_256RegClassID: 1749 case AMDGPU::VReg_256_Align2RegClassID: 1750 case AMDGPU::AReg_256_Align2RegClassID: 1751 case AMDGPU::AV_256RegClassID: 1752 case AMDGPU::AV_256_Align2RegClassID: 1753 return 256; 1754 case AMDGPU::SGPR_512RegClassID: 1755 case AMDGPU::SReg_512RegClassID: 1756 case AMDGPU::VReg_512RegClassID: 1757 case AMDGPU::AReg_512RegClassID: 1758 case AMDGPU::VReg_512_Align2RegClassID: 1759 case AMDGPU::AReg_512_Align2RegClassID: 1760 case AMDGPU::AV_512RegClassID: 1761 case AMDGPU::AV_512_Align2RegClassID: 1762 return 512; 1763 case AMDGPU::SGPR_1024RegClassID: 1764 case AMDGPU::SReg_1024RegClassID: 1765 case AMDGPU::VReg_1024RegClassID: 1766 case AMDGPU::AReg_1024RegClassID: 1767 case AMDGPU::VReg_1024_Align2RegClassID: 1768 case AMDGPU::AReg_1024_Align2RegClassID: 1769 case AMDGPU::AV_1024RegClassID: 1770 case AMDGPU::AV_1024_Align2RegClassID: 1771 return 1024; 1772 default: 1773 llvm_unreachable("Unexpected register class"); 1774 } 1775 } 1776 1777 unsigned getRegBitWidth(const MCRegisterClass &RC) { 1778 return getRegBitWidth(RC.getID()); 1779 } 1780 1781 unsigned getRegOperandSize(const MCRegisterInfo *MRI, const MCInstrDesc &Desc, 1782 unsigned OpNo) { 1783 assert(OpNo < Desc.NumOperands); 1784 unsigned RCID = Desc.OpInfo[OpNo].RegClass; 1785 return getRegBitWidth(MRI->getRegClass(RCID)) / 8; 1786 } 1787 1788 bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi) { 1789 if (isInlinableIntLiteral(Literal)) 1790 return true; 1791 1792 uint64_t Val = static_cast<uint64_t>(Literal); 1793 return (Val == DoubleToBits(0.0)) || 1794 (Val == DoubleToBits(1.0)) || 1795 (Val == DoubleToBits(-1.0)) || 1796 (Val == DoubleToBits(0.5)) || 1797 (Val == DoubleToBits(-0.5)) || 1798 (Val == DoubleToBits(2.0)) || 1799 (Val == DoubleToBits(-2.0)) || 1800 (Val == DoubleToBits(4.0)) || 1801 (Val == DoubleToBits(-4.0)) || 1802 (Val == 0x3fc45f306dc9c882 && HasInv2Pi); 1803 } 1804 1805 bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi) { 1806 if (isInlinableIntLiteral(Literal)) 1807 return true; 1808 1809 // The actual type of the operand does not seem to matter as long 1810 // as the bits match one of the inline immediate values. For example: 1811 // 1812 // -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal, 1813 // so it is a legal inline immediate. 1814 // 1815 // 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in 1816 // floating-point, so it is a legal inline immediate. 1817 1818 uint32_t Val = static_cast<uint32_t>(Literal); 1819 return (Val == FloatToBits(0.0f)) || 1820 (Val == FloatToBits(1.0f)) || 1821 (Val == FloatToBits(-1.0f)) || 1822 (Val == FloatToBits(0.5f)) || 1823 (Val == FloatToBits(-0.5f)) || 1824 (Val == FloatToBits(2.0f)) || 1825 (Val == FloatToBits(-2.0f)) || 1826 (Val == FloatToBits(4.0f)) || 1827 (Val == FloatToBits(-4.0f)) || 1828 (Val == 0x3e22f983 && HasInv2Pi); 1829 } 1830 1831 bool isInlinableLiteral16(int16_t Literal, bool HasInv2Pi) { 1832 if (!HasInv2Pi) 1833 return false; 1834 1835 if (isInlinableIntLiteral(Literal)) 1836 return true; 1837 1838 uint16_t Val = static_cast<uint16_t>(Literal); 1839 return Val == 0x3C00 || // 1.0 1840 Val == 0xBC00 || // -1.0 1841 Val == 0x3800 || // 0.5 1842 Val == 0xB800 || // -0.5 1843 Val == 0x4000 || // 2.0 1844 Val == 0xC000 || // -2.0 1845 Val == 0x4400 || // 4.0 1846 Val == 0xC400 || // -4.0 1847 Val == 0x3118; // 1/2pi 1848 } 1849 1850 bool isInlinableLiteralV216(int32_t Literal, bool HasInv2Pi) { 1851 assert(HasInv2Pi); 1852 1853 if (isInt<16>(Literal) || isUInt<16>(Literal)) { 1854 int16_t Trunc = static_cast<int16_t>(Literal); 1855 return AMDGPU::isInlinableLiteral16(Trunc, HasInv2Pi); 1856 } 1857 if (!(Literal & 0xffff)) 1858 return AMDGPU::isInlinableLiteral16(Literal >> 16, HasInv2Pi); 1859 1860 int16_t Lo16 = static_cast<int16_t>(Literal); 1861 int16_t Hi16 = static_cast<int16_t>(Literal >> 16); 1862 return Lo16 == Hi16 && isInlinableLiteral16(Lo16, HasInv2Pi); 1863 } 1864 1865 bool isInlinableIntLiteralV216(int32_t Literal) { 1866 int16_t Lo16 = static_cast<int16_t>(Literal); 1867 if (isInt<16>(Literal) || isUInt<16>(Literal)) 1868 return isInlinableIntLiteral(Lo16); 1869 1870 int16_t Hi16 = static_cast<int16_t>(Literal >> 16); 1871 if (!(Literal & 0xffff)) 1872 return isInlinableIntLiteral(Hi16); 1873 return Lo16 == Hi16 && isInlinableIntLiteral(Lo16); 1874 } 1875 1876 bool isFoldableLiteralV216(int32_t Literal, bool HasInv2Pi) { 1877 assert(HasInv2Pi); 1878 1879 int16_t Lo16 = static_cast<int16_t>(Literal); 1880 if (isInt<16>(Literal) || isUInt<16>(Literal)) 1881 return true; 1882 1883 int16_t Hi16 = static_cast<int16_t>(Literal >> 16); 1884 if (!(Literal & 0xffff)) 1885 return true; 1886 return Lo16 == Hi16; 1887 } 1888 1889 bool isArgPassedInSGPR(const Argument *A) { 1890 const Function *F = A->getParent(); 1891 1892 // Arguments to compute shaders are never a source of divergence. 1893 CallingConv::ID CC = F->getCallingConv(); 1894 switch (CC) { 1895 case CallingConv::AMDGPU_KERNEL: 1896 case CallingConv::SPIR_KERNEL: 1897 return true; 1898 case CallingConv::AMDGPU_VS: 1899 case CallingConv::AMDGPU_LS: 1900 case CallingConv::AMDGPU_HS: 1901 case CallingConv::AMDGPU_ES: 1902 case CallingConv::AMDGPU_GS: 1903 case CallingConv::AMDGPU_PS: 1904 case CallingConv::AMDGPU_CS: 1905 case CallingConv::AMDGPU_Gfx: 1906 // For non-compute shaders, SGPR inputs are marked with either inreg or byval. 1907 // Everything else is in VGPRs. 1908 return F->getAttributes().hasParamAttr(A->getArgNo(), Attribute::InReg) || 1909 F->getAttributes().hasParamAttr(A->getArgNo(), Attribute::ByVal); 1910 default: 1911 // TODO: Should calls support inreg for SGPR inputs? 1912 return false; 1913 } 1914 } 1915 1916 static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) { 1917 return isGCN3Encoding(ST) || isGFX10Plus(ST); 1918 } 1919 1920 static bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST) { 1921 return isGFX9Plus(ST); 1922 } 1923 1924 bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST, 1925 int64_t EncodedOffset) { 1926 return hasSMEMByteOffset(ST) ? isUInt<20>(EncodedOffset) 1927 : isUInt<8>(EncodedOffset); 1928 } 1929 1930 bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST, 1931 int64_t EncodedOffset, 1932 bool IsBuffer) { 1933 return !IsBuffer && 1934 hasSMRDSignedImmOffset(ST) && 1935 isInt<21>(EncodedOffset); 1936 } 1937 1938 static bool isDwordAligned(uint64_t ByteOffset) { 1939 return (ByteOffset & 3) == 0; 1940 } 1941 1942 uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST, 1943 uint64_t ByteOffset) { 1944 if (hasSMEMByteOffset(ST)) 1945 return ByteOffset; 1946 1947 assert(isDwordAligned(ByteOffset)); 1948 return ByteOffset >> 2; 1949 } 1950 1951 Optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST, 1952 int64_t ByteOffset, bool IsBuffer) { 1953 // The signed version is always a byte offset. 1954 if (!IsBuffer && hasSMRDSignedImmOffset(ST)) { 1955 assert(hasSMEMByteOffset(ST)); 1956 return isInt<20>(ByteOffset) ? Optional<int64_t>(ByteOffset) : None; 1957 } 1958 1959 if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST)) 1960 return None; 1961 1962 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset); 1963 return isLegalSMRDEncodedUnsignedOffset(ST, EncodedOffset) 1964 ? Optional<int64_t>(EncodedOffset) 1965 : None; 1966 } 1967 1968 Optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST, 1969 int64_t ByteOffset) { 1970 if (!isCI(ST) || !isDwordAligned(ByteOffset)) 1971 return None; 1972 1973 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset); 1974 return isUInt<32>(EncodedOffset) ? Optional<int64_t>(EncodedOffset) : None; 1975 } 1976 1977 unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST, bool Signed) { 1978 // Address offset is 12-bit signed for GFX10, 13-bit for GFX9. 1979 if (AMDGPU::isGFX10(ST)) 1980 return Signed ? 12 : 11; 1981 1982 return Signed ? 13 : 12; 1983 } 1984 1985 // Given Imm, split it into the values to put into the SOffset and ImmOffset 1986 // fields in an MUBUF instruction. Return false if it is not possible (due to a 1987 // hardware bug needing a workaround). 1988 // 1989 // The required alignment ensures that individual address components remain 1990 // aligned if they are aligned to begin with. It also ensures that additional 1991 // offsets within the given alignment can be added to the resulting ImmOffset. 1992 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset, 1993 const GCNSubtarget *Subtarget, Align Alignment) { 1994 const uint32_t MaxImm = alignDown(4095, Alignment.value()); 1995 uint32_t Overflow = 0; 1996 1997 if (Imm > MaxImm) { 1998 if (Imm <= MaxImm + 64) { 1999 // Use an SOffset inline constant for 4..64 2000 Overflow = Imm - MaxImm; 2001 Imm = MaxImm; 2002 } else { 2003 // Try to keep the same value in SOffset for adjacent loads, so that 2004 // the corresponding register contents can be re-used. 2005 // 2006 // Load values with all low-bits (except for alignment bits) set into 2007 // SOffset, so that a larger range of values can be covered using 2008 // s_movk_i32. 2009 // 2010 // Atomic operations fail to work correctly when individual address 2011 // components are unaligned, even if their sum is aligned. 2012 uint32_t High = (Imm + Alignment.value()) & ~4095; 2013 uint32_t Low = (Imm + Alignment.value()) & 4095; 2014 Imm = Low; 2015 Overflow = High - Alignment.value(); 2016 } 2017 } 2018 2019 // There is a hardware bug in SI and CI which prevents address clamping in 2020 // MUBUF instructions from working correctly with SOffsets. The immediate 2021 // offset is unaffected. 2022 if (Overflow > 0 && 2023 Subtarget->getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS) 2024 return false; 2025 2026 ImmOffset = Imm; 2027 SOffset = Overflow; 2028 return true; 2029 } 2030 2031 SIModeRegisterDefaults::SIModeRegisterDefaults(const Function &F) { 2032 *this = getDefaultForCallingConv(F.getCallingConv()); 2033 2034 StringRef IEEEAttr = F.getFnAttribute("amdgpu-ieee").getValueAsString(); 2035 if (!IEEEAttr.empty()) 2036 IEEE = IEEEAttr == "true"; 2037 2038 StringRef DX10ClampAttr 2039 = F.getFnAttribute("amdgpu-dx10-clamp").getValueAsString(); 2040 if (!DX10ClampAttr.empty()) 2041 DX10Clamp = DX10ClampAttr == "true"; 2042 2043 StringRef DenormF32Attr = F.getFnAttribute("denormal-fp-math-f32").getValueAsString(); 2044 if (!DenormF32Attr.empty()) { 2045 DenormalMode DenormMode = parseDenormalFPAttribute(DenormF32Attr); 2046 FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE; 2047 FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE; 2048 } 2049 2050 StringRef DenormAttr = F.getFnAttribute("denormal-fp-math").getValueAsString(); 2051 if (!DenormAttr.empty()) { 2052 DenormalMode DenormMode = parseDenormalFPAttribute(DenormAttr); 2053 2054 if (DenormF32Attr.empty()) { 2055 FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE; 2056 FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE; 2057 } 2058 2059 FP64FP16InputDenormals = DenormMode.Input == DenormalMode::IEEE; 2060 FP64FP16OutputDenormals = DenormMode.Output == DenormalMode::IEEE; 2061 } 2062 } 2063 2064 namespace { 2065 2066 struct SourceOfDivergence { 2067 unsigned Intr; 2068 }; 2069 const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr); 2070 2071 #define GET_SourcesOfDivergence_IMPL 2072 #define GET_Gfx9BufferFormat_IMPL 2073 #define GET_Gfx10PlusBufferFormat_IMPL 2074 #include "AMDGPUGenSearchableTables.inc" 2075 2076 } // end anonymous namespace 2077 2078 bool isIntrinsicSourceOfDivergence(unsigned IntrID) { 2079 return lookupSourceOfDivergence(IntrID); 2080 } 2081 2082 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp, 2083 uint8_t NumComponents, 2084 uint8_t NumFormat, 2085 const MCSubtargetInfo &STI) { 2086 return isGFX10Plus(STI) 2087 ? getGfx10PlusBufferFormatInfo(BitsPerComp, NumComponents, 2088 NumFormat) 2089 : getGfx9BufferFormatInfo(BitsPerComp, NumComponents, NumFormat); 2090 } 2091 2092 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format, 2093 const MCSubtargetInfo &STI) { 2094 return isGFX10Plus(STI) ? getGfx10PlusBufferFormatInfo(Format) 2095 : getGfx9BufferFormatInfo(Format); 2096 } 2097 2098 } // namespace AMDGPU 2099 2100 raw_ostream &operator<<(raw_ostream &OS, 2101 const AMDGPU::IsaInfo::TargetIDSetting S) { 2102 switch (S) { 2103 case (AMDGPU::IsaInfo::TargetIDSetting::Unsupported): 2104 OS << "Unsupported"; 2105 break; 2106 case (AMDGPU::IsaInfo::TargetIDSetting::Any): 2107 OS << "Any"; 2108 break; 2109 case (AMDGPU::IsaInfo::TargetIDSetting::Off): 2110 OS << "Off"; 2111 break; 2112 case (AMDGPU::IsaInfo::TargetIDSetting::On): 2113 OS << "On"; 2114 break; 2115 } 2116 return OS; 2117 } 2118 2119 } // namespace llvm 2120