1 //===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "AMDGPUBaseInfo.h" 10 #include "AMDGPU.h" 11 #include "AMDGPUAsmUtils.h" 12 #include "AMDGPUTargetTransformInfo.h" 13 #include "SIDefines.h" 14 #include "llvm/ADT/StringRef.h" 15 #include "llvm/ADT/Triple.h" 16 #include "llvm/BinaryFormat/ELF.h" 17 #include "llvm/CodeGen/MachineMemOperand.h" 18 #include "llvm/IR/Attributes.h" 19 #include "llvm/IR/Constants.h" 20 #include "llvm/IR/Function.h" 21 #include "llvm/IR/GlobalValue.h" 22 #include "llvm/IR/Instruction.h" 23 #include "llvm/IR/IntrinsicsAMDGPU.h" 24 #include "llvm/IR/IntrinsicsR600.h" 25 #include "llvm/IR/LLVMContext.h" 26 #include "llvm/IR/Module.h" 27 #include "llvm/MC/MCContext.h" 28 #include "llvm/MC/MCInstrDesc.h" 29 #include "llvm/MC/MCInstrInfo.h" 30 #include "llvm/MC/MCRegisterInfo.h" 31 #include "llvm/MC/MCSectionELF.h" 32 #include "llvm/MC/MCSubtargetInfo.h" 33 #include "llvm/MC/SubtargetFeature.h" 34 #include "llvm/Support/Casting.h" 35 #include "llvm/Support/ErrorHandling.h" 36 #include "llvm/Support/MathExtras.h" 37 #include <algorithm> 38 #include <cassert> 39 #include <cstdint> 40 #include <cstring> 41 #include <utility> 42 43 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 44 45 #define GET_INSTRINFO_NAMED_OPS 46 #define GET_INSTRMAP_INFO 47 #include "AMDGPUGenInstrInfo.inc" 48 #undef GET_INSTRMAP_INFO 49 #undef GET_INSTRINFO_NAMED_OPS 50 51 namespace { 52 53 /// \returns Bit mask for given bit \p Shift and bit \p Width. 54 unsigned getBitMask(unsigned Shift, unsigned Width) { 55 return ((1 << Width) - 1) << Shift; 56 } 57 58 /// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width. 59 /// 60 /// \returns Packed \p Dst. 61 unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) { 62 Dst &= ~(1 << Shift) & ~getBitMask(Shift, Width); 63 Dst |= (Src << Shift) & getBitMask(Shift, Width); 64 return Dst; 65 } 66 67 /// Unpacks bits from \p Src for given bit \p Shift and bit \p Width. 68 /// 69 /// \returns Unpacked bits. 70 unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) { 71 return (Src & getBitMask(Shift, Width)) >> Shift; 72 } 73 74 /// \returns Vmcnt bit shift (lower bits). 75 unsigned getVmcntBitShiftLo() { return 0; } 76 77 /// \returns Vmcnt bit width (lower bits). 78 unsigned getVmcntBitWidthLo() { return 4; } 79 80 /// \returns Expcnt bit shift. 81 unsigned getExpcntBitShift() { return 4; } 82 83 /// \returns Expcnt bit width. 84 unsigned getExpcntBitWidth() { return 3; } 85 86 /// \returns Lgkmcnt bit shift. 87 unsigned getLgkmcntBitShift() { return 8; } 88 89 /// \returns Lgkmcnt bit width. 90 unsigned getLgkmcntBitWidth(unsigned VersionMajor) { 91 return (VersionMajor >= 10) ? 6 : 4; 92 } 93 94 /// \returns Vmcnt bit shift (higher bits). 95 unsigned getVmcntBitShiftHi() { return 14; } 96 97 /// \returns Vmcnt bit width (higher bits). 98 unsigned getVmcntBitWidthHi() { return 2; } 99 100 } // end namespace anonymous 101 102 namespace llvm { 103 104 namespace AMDGPU { 105 106 #define GET_MIMGBaseOpcodesTable_IMPL 107 #define GET_MIMGDimInfoTable_IMPL 108 #define GET_MIMGInfoTable_IMPL 109 #define GET_MIMGLZMappingTable_IMPL 110 #define GET_MIMGMIPMappingTable_IMPL 111 #include "AMDGPUGenSearchableTables.inc" 112 113 int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding, 114 unsigned VDataDwords, unsigned VAddrDwords) { 115 const MIMGInfo *Info = getMIMGOpcodeHelper(BaseOpcode, MIMGEncoding, 116 VDataDwords, VAddrDwords); 117 return Info ? Info->Opcode : -1; 118 } 119 120 const MIMGBaseOpcodeInfo *getMIMGBaseOpcode(unsigned Opc) { 121 const MIMGInfo *Info = getMIMGInfo(Opc); 122 return Info ? getMIMGBaseOpcodeInfo(Info->BaseOpcode) : nullptr; 123 } 124 125 int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) { 126 const MIMGInfo *OrigInfo = getMIMGInfo(Opc); 127 const MIMGInfo *NewInfo = 128 getMIMGOpcodeHelper(OrigInfo->BaseOpcode, OrigInfo->MIMGEncoding, 129 NewChannels, OrigInfo->VAddrDwords); 130 return NewInfo ? NewInfo->Opcode : -1; 131 } 132 133 struct MUBUFInfo { 134 uint16_t Opcode; 135 uint16_t BaseOpcode; 136 uint8_t elements; 137 bool has_vaddr; 138 bool has_srsrc; 139 bool has_soffset; 140 }; 141 142 struct MTBUFInfo { 143 uint16_t Opcode; 144 uint16_t BaseOpcode; 145 uint8_t elements; 146 bool has_vaddr; 147 bool has_srsrc; 148 bool has_soffset; 149 }; 150 151 #define GET_MTBUFInfoTable_DECL 152 #define GET_MTBUFInfoTable_IMPL 153 #define GET_MUBUFInfoTable_DECL 154 #define GET_MUBUFInfoTable_IMPL 155 #include "AMDGPUGenSearchableTables.inc" 156 157 int getMTBUFBaseOpcode(unsigned Opc) { 158 const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opc); 159 return Info ? Info->BaseOpcode : -1; 160 } 161 162 int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements) { 163 const MTBUFInfo *Info = getMTBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements); 164 return Info ? Info->Opcode : -1; 165 } 166 167 int getMTBUFElements(unsigned Opc) { 168 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 169 return Info ? Info->elements : 0; 170 } 171 172 bool getMTBUFHasVAddr(unsigned Opc) { 173 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 174 return Info ? Info->has_vaddr : false; 175 } 176 177 bool getMTBUFHasSrsrc(unsigned Opc) { 178 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 179 return Info ? Info->has_srsrc : false; 180 } 181 182 bool getMTBUFHasSoffset(unsigned Opc) { 183 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 184 return Info ? Info->has_soffset : false; 185 } 186 187 int getMUBUFBaseOpcode(unsigned Opc) { 188 const MUBUFInfo *Info = getMUBUFInfoFromOpcode(Opc); 189 return Info ? Info->BaseOpcode : -1; 190 } 191 192 int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements) { 193 const MUBUFInfo *Info = getMUBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements); 194 return Info ? Info->Opcode : -1; 195 } 196 197 int getMUBUFElements(unsigned Opc) { 198 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 199 return Info ? Info->elements : 0; 200 } 201 202 bool getMUBUFHasVAddr(unsigned Opc) { 203 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 204 return Info ? Info->has_vaddr : false; 205 } 206 207 bool getMUBUFHasSrsrc(unsigned Opc) { 208 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 209 return Info ? Info->has_srsrc : false; 210 } 211 212 bool getMUBUFHasSoffset(unsigned Opc) { 213 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 214 return Info ? Info->has_soffset : false; 215 } 216 217 // Wrapper for Tablegen'd function. enum Subtarget is not defined in any 218 // header files, so we need to wrap it in a function that takes unsigned 219 // instead. 220 int getMCOpcode(uint16_t Opcode, unsigned Gen) { 221 return getMCOpcodeGen(Opcode, static_cast<Subtarget>(Gen)); 222 } 223 224 namespace IsaInfo { 225 226 void streamIsaVersion(const MCSubtargetInfo *STI, raw_ostream &Stream) { 227 auto TargetTriple = STI->getTargetTriple(); 228 auto Version = getIsaVersion(STI->getCPU()); 229 230 Stream << TargetTriple.getArchName() << '-' 231 << TargetTriple.getVendorName() << '-' 232 << TargetTriple.getOSName() << '-' 233 << TargetTriple.getEnvironmentName() << '-' 234 << "gfx" 235 << Version.Major 236 << Version.Minor 237 << Version.Stepping; 238 239 if (hasXNACK(*STI)) 240 Stream << "+xnack"; 241 if (hasSRAMECC(*STI)) 242 Stream << "+sram-ecc"; 243 244 Stream.flush(); 245 } 246 247 bool hasCodeObjectV3(const MCSubtargetInfo *STI) { 248 return STI->getTargetTriple().getOS() == Triple::AMDHSA && 249 STI->getFeatureBits().test(FeatureCodeObjectV3); 250 } 251 252 unsigned getWavefrontSize(const MCSubtargetInfo *STI) { 253 if (STI->getFeatureBits().test(FeatureWavefrontSize16)) 254 return 16; 255 if (STI->getFeatureBits().test(FeatureWavefrontSize32)) 256 return 32; 257 258 return 64; 259 } 260 261 unsigned getLocalMemorySize(const MCSubtargetInfo *STI) { 262 if (STI->getFeatureBits().test(FeatureLocalMemorySize32768)) 263 return 32768; 264 if (STI->getFeatureBits().test(FeatureLocalMemorySize65536)) 265 return 65536; 266 267 return 0; 268 } 269 270 unsigned getEUsPerCU(const MCSubtargetInfo *STI) { 271 return 4; 272 } 273 274 unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo *STI, 275 unsigned FlatWorkGroupSize) { 276 assert(FlatWorkGroupSize != 0); 277 if (STI->getTargetTriple().getArch() != Triple::amdgcn) 278 return 8; 279 unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize); 280 if (N == 1) 281 return 40; 282 N = 40 / N; 283 return std::min(N, 16u); 284 } 285 286 unsigned getMinWavesPerEU(const MCSubtargetInfo *STI) { 287 return 1; 288 } 289 290 unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI) { 291 // FIXME: Need to take scratch memory into account. 292 if (!isGFX10(*STI)) 293 return 10; 294 return 20; 295 } 296 297 unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo *STI, 298 unsigned FlatWorkGroupSize) { 299 return divideCeil(getWavesPerWorkGroup(STI, FlatWorkGroupSize), 300 getEUsPerCU(STI)); 301 } 302 303 unsigned getMinFlatWorkGroupSize(const MCSubtargetInfo *STI) { 304 return 1; 305 } 306 307 unsigned getMaxFlatWorkGroupSize(const MCSubtargetInfo *STI) { 308 // Some subtargets allow encoding 2048, but this isn't tested or supported. 309 return 1024; 310 } 311 312 unsigned getWavesPerWorkGroup(const MCSubtargetInfo *STI, 313 unsigned FlatWorkGroupSize) { 314 return divideCeil(FlatWorkGroupSize, getWavefrontSize(STI)); 315 } 316 317 unsigned getSGPRAllocGranule(const MCSubtargetInfo *STI) { 318 IsaVersion Version = getIsaVersion(STI->getCPU()); 319 if (Version.Major >= 10) 320 return getAddressableNumSGPRs(STI); 321 if (Version.Major >= 8) 322 return 16; 323 return 8; 324 } 325 326 unsigned getSGPREncodingGranule(const MCSubtargetInfo *STI) { 327 return 8; 328 } 329 330 unsigned getTotalNumSGPRs(const MCSubtargetInfo *STI) { 331 IsaVersion Version = getIsaVersion(STI->getCPU()); 332 if (Version.Major >= 8) 333 return 800; 334 return 512; 335 } 336 337 unsigned getAddressableNumSGPRs(const MCSubtargetInfo *STI) { 338 if (STI->getFeatureBits().test(FeatureSGPRInitBug)) 339 return FIXED_NUM_SGPRS_FOR_INIT_BUG; 340 341 IsaVersion Version = getIsaVersion(STI->getCPU()); 342 if (Version.Major >= 10) 343 return 106; 344 if (Version.Major >= 8) 345 return 102; 346 return 104; 347 } 348 349 unsigned getMinNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 350 assert(WavesPerEU != 0); 351 352 IsaVersion Version = getIsaVersion(STI->getCPU()); 353 if (Version.Major >= 10) 354 return 0; 355 356 if (WavesPerEU >= getMaxWavesPerEU(STI)) 357 return 0; 358 359 unsigned MinNumSGPRs = getTotalNumSGPRs(STI) / (WavesPerEU + 1); 360 if (STI->getFeatureBits().test(FeatureTrapHandler)) 361 MinNumSGPRs -= std::min(MinNumSGPRs, (unsigned)TRAP_NUM_SGPRS); 362 MinNumSGPRs = alignDown(MinNumSGPRs, getSGPRAllocGranule(STI)) + 1; 363 return std::min(MinNumSGPRs, getAddressableNumSGPRs(STI)); 364 } 365 366 unsigned getMaxNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU, 367 bool Addressable) { 368 assert(WavesPerEU != 0); 369 370 unsigned AddressableNumSGPRs = getAddressableNumSGPRs(STI); 371 IsaVersion Version = getIsaVersion(STI->getCPU()); 372 if (Version.Major >= 10) 373 return Addressable ? AddressableNumSGPRs : 108; 374 if (Version.Major >= 8 && !Addressable) 375 AddressableNumSGPRs = 112; 376 unsigned MaxNumSGPRs = getTotalNumSGPRs(STI) / WavesPerEU; 377 if (STI->getFeatureBits().test(FeatureTrapHandler)) 378 MaxNumSGPRs -= std::min(MaxNumSGPRs, (unsigned)TRAP_NUM_SGPRS); 379 MaxNumSGPRs = alignDown(MaxNumSGPRs, getSGPRAllocGranule(STI)); 380 return std::min(MaxNumSGPRs, AddressableNumSGPRs); 381 } 382 383 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed, 384 bool FlatScrUsed, bool XNACKUsed) { 385 unsigned ExtraSGPRs = 0; 386 if (VCCUsed) 387 ExtraSGPRs = 2; 388 389 IsaVersion Version = getIsaVersion(STI->getCPU()); 390 if (Version.Major >= 10) 391 return ExtraSGPRs; 392 393 if (Version.Major < 8) { 394 if (FlatScrUsed) 395 ExtraSGPRs = 4; 396 } else { 397 if (XNACKUsed) 398 ExtraSGPRs = 4; 399 400 if (FlatScrUsed) 401 ExtraSGPRs = 6; 402 } 403 404 return ExtraSGPRs; 405 } 406 407 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed, 408 bool FlatScrUsed) { 409 return getNumExtraSGPRs(STI, VCCUsed, FlatScrUsed, 410 STI->getFeatureBits().test(AMDGPU::FeatureXNACK)); 411 } 412 413 unsigned getNumSGPRBlocks(const MCSubtargetInfo *STI, unsigned NumSGPRs) { 414 NumSGPRs = alignTo(std::max(1u, NumSGPRs), getSGPREncodingGranule(STI)); 415 // SGPRBlocks is actual number of SGPR blocks minus 1. 416 return NumSGPRs / getSGPREncodingGranule(STI) - 1; 417 } 418 419 unsigned getVGPRAllocGranule(const MCSubtargetInfo *STI, 420 Optional<bool> EnableWavefrontSize32) { 421 bool IsWave32 = EnableWavefrontSize32 ? 422 *EnableWavefrontSize32 : 423 STI->getFeatureBits().test(FeatureWavefrontSize32); 424 return IsWave32 ? 8 : 4; 425 } 426 427 unsigned getVGPREncodingGranule(const MCSubtargetInfo *STI, 428 Optional<bool> EnableWavefrontSize32) { 429 return getVGPRAllocGranule(STI, EnableWavefrontSize32); 430 } 431 432 unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) { 433 if (!isGFX10(*STI)) 434 return 256; 435 return STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1024 : 512; 436 } 437 438 unsigned getAddressableNumVGPRs(const MCSubtargetInfo *STI) { 439 return 256; 440 } 441 442 unsigned getMinNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 443 assert(WavesPerEU != 0); 444 445 if (WavesPerEU >= getMaxWavesPerEU(STI)) 446 return 0; 447 unsigned MinNumVGPRs = 448 alignDown(getTotalNumVGPRs(STI) / (WavesPerEU + 1), 449 getVGPRAllocGranule(STI)) + 1; 450 return std::min(MinNumVGPRs, getAddressableNumVGPRs(STI)); 451 } 452 453 unsigned getMaxNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 454 assert(WavesPerEU != 0); 455 456 unsigned MaxNumVGPRs = alignDown(getTotalNumVGPRs(STI) / WavesPerEU, 457 getVGPRAllocGranule(STI)); 458 unsigned AddressableNumVGPRs = getAddressableNumVGPRs(STI); 459 return std::min(MaxNumVGPRs, AddressableNumVGPRs); 460 } 461 462 unsigned getNumVGPRBlocks(const MCSubtargetInfo *STI, unsigned NumVGPRs, 463 Optional<bool> EnableWavefrontSize32) { 464 NumVGPRs = alignTo(std::max(1u, NumVGPRs), 465 getVGPREncodingGranule(STI, EnableWavefrontSize32)); 466 // VGPRBlocks is actual number of VGPR blocks minus 1. 467 return NumVGPRs / getVGPREncodingGranule(STI, EnableWavefrontSize32) - 1; 468 } 469 470 } // end namespace IsaInfo 471 472 void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header, 473 const MCSubtargetInfo *STI) { 474 IsaVersion Version = getIsaVersion(STI->getCPU()); 475 476 memset(&Header, 0, sizeof(Header)); 477 478 Header.amd_kernel_code_version_major = 1; 479 Header.amd_kernel_code_version_minor = 2; 480 Header.amd_machine_kind = 1; // AMD_MACHINE_KIND_AMDGPU 481 Header.amd_machine_version_major = Version.Major; 482 Header.amd_machine_version_minor = Version.Minor; 483 Header.amd_machine_version_stepping = Version.Stepping; 484 Header.kernel_code_entry_byte_offset = sizeof(Header); 485 Header.wavefront_size = 6; 486 487 // If the code object does not support indirect functions, then the value must 488 // be 0xffffffff. 489 Header.call_convention = -1; 490 491 // These alignment values are specified in powers of two, so alignment = 492 // 2^n. The minimum alignment is 2^4 = 16. 493 Header.kernarg_segment_alignment = 4; 494 Header.group_segment_alignment = 4; 495 Header.private_segment_alignment = 4; 496 497 if (Version.Major >= 10) { 498 if (STI->getFeatureBits().test(FeatureWavefrontSize32)) { 499 Header.wavefront_size = 5; 500 Header.code_properties |= AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32; 501 } 502 Header.compute_pgm_resource_registers |= 503 S_00B848_WGP_MODE(STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1) | 504 S_00B848_MEM_ORDERED(1); 505 } 506 } 507 508 amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor( 509 const MCSubtargetInfo *STI) { 510 IsaVersion Version = getIsaVersion(STI->getCPU()); 511 512 amdhsa::kernel_descriptor_t KD; 513 memset(&KD, 0, sizeof(KD)); 514 515 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 516 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, 517 amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE); 518 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 519 amdhsa::COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP, 1); 520 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 521 amdhsa::COMPUTE_PGM_RSRC1_ENABLE_IEEE_MODE, 1); 522 AMDHSA_BITS_SET(KD.compute_pgm_rsrc2, 523 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, 1); 524 if (Version.Major >= 10) { 525 AMDHSA_BITS_SET(KD.kernel_code_properties, 526 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, 527 STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1 : 0); 528 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 529 amdhsa::COMPUTE_PGM_RSRC1_WGP_MODE, 530 STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1); 531 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 532 amdhsa::COMPUTE_PGM_RSRC1_MEM_ORDERED, 1); 533 } 534 return KD; 535 } 536 537 bool isGroupSegment(const GlobalValue *GV) { 538 return GV->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS; 539 } 540 541 bool isGlobalSegment(const GlobalValue *GV) { 542 return GV->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS; 543 } 544 545 bool isReadOnlySegment(const GlobalValue *GV) { 546 unsigned AS = GV->getAddressSpace(); 547 return AS == AMDGPUAS::CONSTANT_ADDRESS || 548 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT; 549 } 550 551 bool shouldEmitConstantsToTextSection(const Triple &TT) { 552 return TT.getOS() == Triple::AMDPAL || TT.getArch() == Triple::r600; 553 } 554 555 int getIntegerAttribute(const Function &F, StringRef Name, int Default) { 556 Attribute A = F.getFnAttribute(Name); 557 int Result = Default; 558 559 if (A.isStringAttribute()) { 560 StringRef Str = A.getValueAsString(); 561 if (Str.getAsInteger(0, Result)) { 562 LLVMContext &Ctx = F.getContext(); 563 Ctx.emitError("can't parse integer attribute " + Name); 564 } 565 } 566 567 return Result; 568 } 569 570 std::pair<int, int> getIntegerPairAttribute(const Function &F, 571 StringRef Name, 572 std::pair<int, int> Default, 573 bool OnlyFirstRequired) { 574 Attribute A = F.getFnAttribute(Name); 575 if (!A.isStringAttribute()) 576 return Default; 577 578 LLVMContext &Ctx = F.getContext(); 579 std::pair<int, int> Ints = Default; 580 std::pair<StringRef, StringRef> Strs = A.getValueAsString().split(','); 581 if (Strs.first.trim().getAsInteger(0, Ints.first)) { 582 Ctx.emitError("can't parse first integer attribute " + Name); 583 return Default; 584 } 585 if (Strs.second.trim().getAsInteger(0, Ints.second)) { 586 if (!OnlyFirstRequired || !Strs.second.trim().empty()) { 587 Ctx.emitError("can't parse second integer attribute " + Name); 588 return Default; 589 } 590 } 591 592 return Ints; 593 } 594 595 unsigned getVmcntBitMask(const IsaVersion &Version) { 596 unsigned VmcntLo = (1 << getVmcntBitWidthLo()) - 1; 597 if (Version.Major < 9) 598 return VmcntLo; 599 600 unsigned VmcntHi = ((1 << getVmcntBitWidthHi()) - 1) << getVmcntBitWidthLo(); 601 return VmcntLo | VmcntHi; 602 } 603 604 unsigned getExpcntBitMask(const IsaVersion &Version) { 605 return (1 << getExpcntBitWidth()) - 1; 606 } 607 608 unsigned getLgkmcntBitMask(const IsaVersion &Version) { 609 return (1 << getLgkmcntBitWidth(Version.Major)) - 1; 610 } 611 612 unsigned getWaitcntBitMask(const IsaVersion &Version) { 613 unsigned VmcntLo = getBitMask(getVmcntBitShiftLo(), getVmcntBitWidthLo()); 614 unsigned Expcnt = getBitMask(getExpcntBitShift(), getExpcntBitWidth()); 615 unsigned Lgkmcnt = getBitMask(getLgkmcntBitShift(), 616 getLgkmcntBitWidth(Version.Major)); 617 unsigned Waitcnt = VmcntLo | Expcnt | Lgkmcnt; 618 if (Version.Major < 9) 619 return Waitcnt; 620 621 unsigned VmcntHi = getBitMask(getVmcntBitShiftHi(), getVmcntBitWidthHi()); 622 return Waitcnt | VmcntHi; 623 } 624 625 unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) { 626 unsigned VmcntLo = 627 unpackBits(Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo()); 628 if (Version.Major < 9) 629 return VmcntLo; 630 631 unsigned VmcntHi = 632 unpackBits(Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi()); 633 VmcntHi <<= getVmcntBitWidthLo(); 634 return VmcntLo | VmcntHi; 635 } 636 637 unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) { 638 return unpackBits(Waitcnt, getExpcntBitShift(), getExpcntBitWidth()); 639 } 640 641 unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) { 642 return unpackBits(Waitcnt, getLgkmcntBitShift(), 643 getLgkmcntBitWidth(Version.Major)); 644 } 645 646 void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, 647 unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt) { 648 Vmcnt = decodeVmcnt(Version, Waitcnt); 649 Expcnt = decodeExpcnt(Version, Waitcnt); 650 Lgkmcnt = decodeLgkmcnt(Version, Waitcnt); 651 } 652 653 Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded) { 654 Waitcnt Decoded; 655 Decoded.VmCnt = decodeVmcnt(Version, Encoded); 656 Decoded.ExpCnt = decodeExpcnt(Version, Encoded); 657 Decoded.LgkmCnt = decodeLgkmcnt(Version, Encoded); 658 return Decoded; 659 } 660 661 unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt, 662 unsigned Vmcnt) { 663 Waitcnt = 664 packBits(Vmcnt, Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo()); 665 if (Version.Major < 9) 666 return Waitcnt; 667 668 Vmcnt >>= getVmcntBitWidthLo(); 669 return packBits(Vmcnt, Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi()); 670 } 671 672 unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt, 673 unsigned Expcnt) { 674 return packBits(Expcnt, Waitcnt, getExpcntBitShift(), getExpcntBitWidth()); 675 } 676 677 unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt, 678 unsigned Lgkmcnt) { 679 return packBits(Lgkmcnt, Waitcnt, getLgkmcntBitShift(), 680 getLgkmcntBitWidth(Version.Major)); 681 } 682 683 unsigned encodeWaitcnt(const IsaVersion &Version, 684 unsigned Vmcnt, unsigned Expcnt, unsigned Lgkmcnt) { 685 unsigned Waitcnt = getWaitcntBitMask(Version); 686 Waitcnt = encodeVmcnt(Version, Waitcnt, Vmcnt); 687 Waitcnt = encodeExpcnt(Version, Waitcnt, Expcnt); 688 Waitcnt = encodeLgkmcnt(Version, Waitcnt, Lgkmcnt); 689 return Waitcnt; 690 } 691 692 unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded) { 693 return encodeWaitcnt(Version, Decoded.VmCnt, Decoded.ExpCnt, Decoded.LgkmCnt); 694 } 695 696 //===----------------------------------------------------------------------===// 697 // hwreg 698 //===----------------------------------------------------------------------===// 699 700 namespace Hwreg { 701 702 int64_t getHwregId(const StringRef Name) { 703 for (int Id = ID_SYMBOLIC_FIRST_; Id < ID_SYMBOLIC_LAST_; ++Id) { 704 if (IdSymbolic[Id] && Name == IdSymbolic[Id]) 705 return Id; 706 } 707 return ID_UNKNOWN_; 708 } 709 710 static unsigned getLastSymbolicHwreg(const MCSubtargetInfo &STI) { 711 if (isSI(STI) || isCI(STI) || isVI(STI)) 712 return ID_SYMBOLIC_FIRST_GFX9_; 713 else if (isGFX9(STI)) 714 return ID_SYMBOLIC_FIRST_GFX10_; 715 else 716 return ID_SYMBOLIC_LAST_; 717 } 718 719 bool isValidHwreg(int64_t Id, const MCSubtargetInfo &STI) { 720 return ID_SYMBOLIC_FIRST_ <= Id && Id < getLastSymbolicHwreg(STI) && 721 IdSymbolic[Id]; 722 } 723 724 bool isValidHwreg(int64_t Id) { 725 return 0 <= Id && isUInt<ID_WIDTH_>(Id); 726 } 727 728 bool isValidHwregOffset(int64_t Offset) { 729 return 0 <= Offset && isUInt<OFFSET_WIDTH_>(Offset); 730 } 731 732 bool isValidHwregWidth(int64_t Width) { 733 return 0 <= (Width - 1) && isUInt<WIDTH_M1_WIDTH_>(Width - 1); 734 } 735 736 uint64_t encodeHwreg(uint64_t Id, uint64_t Offset, uint64_t Width) { 737 return (Id << ID_SHIFT_) | 738 (Offset << OFFSET_SHIFT_) | 739 ((Width - 1) << WIDTH_M1_SHIFT_); 740 } 741 742 StringRef getHwreg(unsigned Id, const MCSubtargetInfo &STI) { 743 return isValidHwreg(Id, STI) ? IdSymbolic[Id] : ""; 744 } 745 746 void decodeHwreg(unsigned Val, unsigned &Id, unsigned &Offset, unsigned &Width) { 747 Id = (Val & ID_MASK_) >> ID_SHIFT_; 748 Offset = (Val & OFFSET_MASK_) >> OFFSET_SHIFT_; 749 Width = ((Val & WIDTH_M1_MASK_) >> WIDTH_M1_SHIFT_) + 1; 750 } 751 752 } // namespace Hwreg 753 754 //===----------------------------------------------------------------------===// 755 // SendMsg 756 //===----------------------------------------------------------------------===// 757 758 namespace SendMsg { 759 760 int64_t getMsgId(const StringRef Name) { 761 for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) { 762 if (IdSymbolic[i] && Name == IdSymbolic[i]) 763 return i; 764 } 765 return ID_UNKNOWN_; 766 } 767 768 static bool isValidMsgId(int64_t MsgId) { 769 return (ID_GAPS_FIRST_ <= MsgId && MsgId < ID_GAPS_LAST_) && IdSymbolic[MsgId]; 770 } 771 772 bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI, bool Strict) { 773 if (Strict) { 774 if (MsgId == ID_GS_ALLOC_REQ || MsgId == ID_GET_DOORBELL) 775 return isGFX9(STI) || isGFX10(STI); 776 else 777 return isValidMsgId(MsgId); 778 } else { 779 return 0 <= MsgId && isUInt<ID_WIDTH_>(MsgId); 780 } 781 } 782 783 StringRef getMsgName(int64_t MsgId) { 784 return isValidMsgId(MsgId)? IdSymbolic[MsgId] : ""; 785 } 786 787 int64_t getMsgOpId(int64_t MsgId, const StringRef Name) { 788 const char* const *S = (MsgId == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic; 789 const int F = (MsgId == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_; 790 const int L = (MsgId == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_; 791 for (int i = F; i < L; ++i) { 792 if (Name == S[i]) { 793 return i; 794 } 795 } 796 return OP_UNKNOWN_; 797 } 798 799 bool isValidMsgOp(int64_t MsgId, int64_t OpId, bool Strict) { 800 801 if (!Strict) 802 return 0 <= OpId && isUInt<OP_WIDTH_>(OpId); 803 804 switch(MsgId) 805 { 806 case ID_GS: 807 return (OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_) && OpId != OP_GS_NOP; 808 case ID_GS_DONE: 809 return OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_; 810 case ID_SYSMSG: 811 return OP_SYS_FIRST_ <= OpId && OpId < OP_SYS_LAST_; 812 default: 813 return OpId == OP_NONE_; 814 } 815 } 816 817 StringRef getMsgOpName(int64_t MsgId, int64_t OpId) { 818 assert(msgRequiresOp(MsgId)); 819 return (MsgId == ID_SYSMSG)? OpSysSymbolic[OpId] : OpGsSymbolic[OpId]; 820 } 821 822 bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId, bool Strict) { 823 824 if (!Strict) 825 return 0 <= StreamId && isUInt<STREAM_ID_WIDTH_>(StreamId); 826 827 switch(MsgId) 828 { 829 case ID_GS: 830 return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_; 831 case ID_GS_DONE: 832 return (OpId == OP_GS_NOP)? 833 (StreamId == STREAM_ID_NONE_) : 834 (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_); 835 default: 836 return StreamId == STREAM_ID_NONE_; 837 } 838 } 839 840 bool msgRequiresOp(int64_t MsgId) { 841 return MsgId == ID_GS || MsgId == ID_GS_DONE || MsgId == ID_SYSMSG; 842 } 843 844 bool msgSupportsStream(int64_t MsgId, int64_t OpId) { 845 return (MsgId == ID_GS || MsgId == ID_GS_DONE) && OpId != OP_GS_NOP; 846 } 847 848 void decodeMsg(unsigned Val, 849 uint16_t &MsgId, 850 uint16_t &OpId, 851 uint16_t &StreamId) { 852 MsgId = Val & ID_MASK_; 853 OpId = (Val & OP_MASK_) >> OP_SHIFT_; 854 StreamId = (Val & STREAM_ID_MASK_) >> STREAM_ID_SHIFT_; 855 } 856 857 uint64_t encodeMsg(uint64_t MsgId, 858 uint64_t OpId, 859 uint64_t StreamId) { 860 return (MsgId << ID_SHIFT_) | 861 (OpId << OP_SHIFT_) | 862 (StreamId << STREAM_ID_SHIFT_); 863 } 864 865 } // namespace SendMsg 866 867 //===----------------------------------------------------------------------===// 868 // 869 //===----------------------------------------------------------------------===// 870 871 unsigned getInitialPSInputAddr(const Function &F) { 872 return getIntegerAttribute(F, "InitialPSInputAddr", 0); 873 } 874 875 bool isShader(CallingConv::ID cc) { 876 switch(cc) { 877 case CallingConv::AMDGPU_VS: 878 case CallingConv::AMDGPU_LS: 879 case CallingConv::AMDGPU_HS: 880 case CallingConv::AMDGPU_ES: 881 case CallingConv::AMDGPU_GS: 882 case CallingConv::AMDGPU_PS: 883 case CallingConv::AMDGPU_CS: 884 return true; 885 default: 886 return false; 887 } 888 } 889 890 bool isCompute(CallingConv::ID cc) { 891 return !isShader(cc) || cc == CallingConv::AMDGPU_CS; 892 } 893 894 bool isEntryFunctionCC(CallingConv::ID CC) { 895 switch (CC) { 896 case CallingConv::AMDGPU_KERNEL: 897 case CallingConv::SPIR_KERNEL: 898 case CallingConv::AMDGPU_VS: 899 case CallingConv::AMDGPU_GS: 900 case CallingConv::AMDGPU_PS: 901 case CallingConv::AMDGPU_CS: 902 case CallingConv::AMDGPU_ES: 903 case CallingConv::AMDGPU_HS: 904 case CallingConv::AMDGPU_LS: 905 return true; 906 default: 907 return false; 908 } 909 } 910 911 bool hasXNACK(const MCSubtargetInfo &STI) { 912 return STI.getFeatureBits()[AMDGPU::FeatureXNACK]; 913 } 914 915 bool hasSRAMECC(const MCSubtargetInfo &STI) { 916 return STI.getFeatureBits()[AMDGPU::FeatureSRAMECC]; 917 } 918 919 bool hasMIMG_R128(const MCSubtargetInfo &STI) { 920 return STI.getFeatureBits()[AMDGPU::FeatureMIMG_R128] && !STI.getFeatureBits()[AMDGPU::FeatureR128A16]; 921 } 922 923 bool hasGFX10A16(const MCSubtargetInfo &STI) { 924 return STI.getFeatureBits()[AMDGPU::FeatureGFX10A16]; 925 } 926 927 bool hasPackedD16(const MCSubtargetInfo &STI) { 928 return !STI.getFeatureBits()[AMDGPU::FeatureUnpackedD16VMem]; 929 } 930 931 bool isSI(const MCSubtargetInfo &STI) { 932 return STI.getFeatureBits()[AMDGPU::FeatureSouthernIslands]; 933 } 934 935 bool isCI(const MCSubtargetInfo &STI) { 936 return STI.getFeatureBits()[AMDGPU::FeatureSeaIslands]; 937 } 938 939 bool isVI(const MCSubtargetInfo &STI) { 940 return STI.getFeatureBits()[AMDGPU::FeatureVolcanicIslands]; 941 } 942 943 bool isGFX9(const MCSubtargetInfo &STI) { 944 return STI.getFeatureBits()[AMDGPU::FeatureGFX9]; 945 } 946 947 bool isGFX10(const MCSubtargetInfo &STI) { 948 return STI.getFeatureBits()[AMDGPU::FeatureGFX10]; 949 } 950 951 bool isGCN3Encoding(const MCSubtargetInfo &STI) { 952 return STI.getFeatureBits()[AMDGPU::FeatureGCN3Encoding]; 953 } 954 955 bool isSGPR(unsigned Reg, const MCRegisterInfo* TRI) { 956 const MCRegisterClass SGPRClass = TRI->getRegClass(AMDGPU::SReg_32RegClassID); 957 const unsigned FirstSubReg = TRI->getSubReg(Reg, AMDGPU::sub0); 958 return SGPRClass.contains(FirstSubReg != 0 ? FirstSubReg : Reg) || 959 Reg == AMDGPU::SCC; 960 } 961 962 bool isRegIntersect(unsigned Reg0, unsigned Reg1, const MCRegisterInfo* TRI) { 963 for (MCRegAliasIterator R(Reg0, TRI, true); R.isValid(); ++R) { 964 if (*R == Reg1) return true; 965 } 966 return false; 967 } 968 969 #define MAP_REG2REG \ 970 using namespace AMDGPU; \ 971 switch(Reg) { \ 972 default: return Reg; \ 973 CASE_CI_VI(FLAT_SCR) \ 974 CASE_CI_VI(FLAT_SCR_LO) \ 975 CASE_CI_VI(FLAT_SCR_HI) \ 976 CASE_VI_GFX9_GFX10(TTMP0) \ 977 CASE_VI_GFX9_GFX10(TTMP1) \ 978 CASE_VI_GFX9_GFX10(TTMP2) \ 979 CASE_VI_GFX9_GFX10(TTMP3) \ 980 CASE_VI_GFX9_GFX10(TTMP4) \ 981 CASE_VI_GFX9_GFX10(TTMP5) \ 982 CASE_VI_GFX9_GFX10(TTMP6) \ 983 CASE_VI_GFX9_GFX10(TTMP7) \ 984 CASE_VI_GFX9_GFX10(TTMP8) \ 985 CASE_VI_GFX9_GFX10(TTMP9) \ 986 CASE_VI_GFX9_GFX10(TTMP10) \ 987 CASE_VI_GFX9_GFX10(TTMP11) \ 988 CASE_VI_GFX9_GFX10(TTMP12) \ 989 CASE_VI_GFX9_GFX10(TTMP13) \ 990 CASE_VI_GFX9_GFX10(TTMP14) \ 991 CASE_VI_GFX9_GFX10(TTMP15) \ 992 CASE_VI_GFX9_GFX10(TTMP0_TTMP1) \ 993 CASE_VI_GFX9_GFX10(TTMP2_TTMP3) \ 994 CASE_VI_GFX9_GFX10(TTMP4_TTMP5) \ 995 CASE_VI_GFX9_GFX10(TTMP6_TTMP7) \ 996 CASE_VI_GFX9_GFX10(TTMP8_TTMP9) \ 997 CASE_VI_GFX9_GFX10(TTMP10_TTMP11) \ 998 CASE_VI_GFX9_GFX10(TTMP12_TTMP13) \ 999 CASE_VI_GFX9_GFX10(TTMP14_TTMP15) \ 1000 CASE_VI_GFX9_GFX10(TTMP0_TTMP1_TTMP2_TTMP3) \ 1001 CASE_VI_GFX9_GFX10(TTMP4_TTMP5_TTMP6_TTMP7) \ 1002 CASE_VI_GFX9_GFX10(TTMP8_TTMP9_TTMP10_TTMP11) \ 1003 CASE_VI_GFX9_GFX10(TTMP12_TTMP13_TTMP14_TTMP15) \ 1004 CASE_VI_GFX9_GFX10(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7) \ 1005 CASE_VI_GFX9_GFX10(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11) \ 1006 CASE_VI_GFX9_GFX10(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \ 1007 CASE_VI_GFX9_GFX10(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \ 1008 } 1009 1010 #define CASE_CI_VI(node) \ 1011 assert(!isSI(STI)); \ 1012 case node: return isCI(STI) ? node##_ci : node##_vi; 1013 1014 #define CASE_VI_GFX9_GFX10(node) \ 1015 case node: return (isGFX9(STI) || isGFX10(STI)) ? node##_gfx9_gfx10 : node##_vi; 1016 1017 unsigned getMCReg(unsigned Reg, const MCSubtargetInfo &STI) { 1018 if (STI.getTargetTriple().getArch() == Triple::r600) 1019 return Reg; 1020 MAP_REG2REG 1021 } 1022 1023 #undef CASE_CI_VI 1024 #undef CASE_VI_GFX9_GFX10 1025 1026 #define CASE_CI_VI(node) case node##_ci: case node##_vi: return node; 1027 #define CASE_VI_GFX9_GFX10(node) case node##_vi: case node##_gfx9_gfx10: return node; 1028 1029 unsigned mc2PseudoReg(unsigned Reg) { 1030 MAP_REG2REG 1031 } 1032 1033 #undef CASE_CI_VI 1034 #undef CASE_VI_GFX9_GFX10 1035 #undef MAP_REG2REG 1036 1037 bool isSISrcOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1038 assert(OpNo < Desc.NumOperands); 1039 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1040 return OpType >= AMDGPU::OPERAND_SRC_FIRST && 1041 OpType <= AMDGPU::OPERAND_SRC_LAST; 1042 } 1043 1044 bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1045 assert(OpNo < Desc.NumOperands); 1046 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1047 switch (OpType) { 1048 case AMDGPU::OPERAND_REG_IMM_FP32: 1049 case AMDGPU::OPERAND_REG_IMM_FP64: 1050 case AMDGPU::OPERAND_REG_IMM_FP16: 1051 case AMDGPU::OPERAND_REG_IMM_V2FP16: 1052 case AMDGPU::OPERAND_REG_IMM_V2INT16: 1053 case AMDGPU::OPERAND_REG_INLINE_C_FP32: 1054 case AMDGPU::OPERAND_REG_INLINE_C_FP64: 1055 case AMDGPU::OPERAND_REG_INLINE_C_FP16: 1056 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: 1057 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: 1058 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: 1059 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: 1060 case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: 1061 case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16: 1062 return true; 1063 default: 1064 return false; 1065 } 1066 } 1067 1068 bool isSISrcInlinableOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1069 assert(OpNo < Desc.NumOperands); 1070 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1071 return OpType >= AMDGPU::OPERAND_REG_INLINE_C_FIRST && 1072 OpType <= AMDGPU::OPERAND_REG_INLINE_C_LAST; 1073 } 1074 1075 // Avoid using MCRegisterClass::getSize, since that function will go away 1076 // (move from MC* level to Target* level). Return size in bits. 1077 unsigned getRegBitWidth(unsigned RCID) { 1078 switch (RCID) { 1079 case AMDGPU::SGPR_32RegClassID: 1080 case AMDGPU::VGPR_32RegClassID: 1081 case AMDGPU::VRegOrLds_32RegClassID: 1082 case AMDGPU::AGPR_32RegClassID: 1083 case AMDGPU::VS_32RegClassID: 1084 case AMDGPU::AV_32RegClassID: 1085 case AMDGPU::SReg_32RegClassID: 1086 case AMDGPU::SReg_32_XM0RegClassID: 1087 case AMDGPU::SRegOrLds_32RegClassID: 1088 return 32; 1089 case AMDGPU::SGPR_64RegClassID: 1090 case AMDGPU::VS_64RegClassID: 1091 case AMDGPU::AV_64RegClassID: 1092 case AMDGPU::SReg_64RegClassID: 1093 case AMDGPU::VReg_64RegClassID: 1094 case AMDGPU::AReg_64RegClassID: 1095 case AMDGPU::SReg_64_XEXECRegClassID: 1096 return 64; 1097 case AMDGPU::SGPR_96RegClassID: 1098 case AMDGPU::SReg_96RegClassID: 1099 case AMDGPU::VReg_96RegClassID: 1100 return 96; 1101 case AMDGPU::SGPR_128RegClassID: 1102 case AMDGPU::SReg_128RegClassID: 1103 case AMDGPU::VReg_128RegClassID: 1104 case AMDGPU::AReg_128RegClassID: 1105 return 128; 1106 case AMDGPU::SGPR_160RegClassID: 1107 case AMDGPU::SReg_160RegClassID: 1108 case AMDGPU::VReg_160RegClassID: 1109 return 160; 1110 case AMDGPU::SReg_256RegClassID: 1111 case AMDGPU::VReg_256RegClassID: 1112 return 256; 1113 case AMDGPU::SReg_512RegClassID: 1114 case AMDGPU::VReg_512RegClassID: 1115 case AMDGPU::AReg_512RegClassID: 1116 return 512; 1117 case AMDGPU::SReg_1024RegClassID: 1118 case AMDGPU::VReg_1024RegClassID: 1119 case AMDGPU::AReg_1024RegClassID: 1120 return 1024; 1121 default: 1122 llvm_unreachable("Unexpected register class"); 1123 } 1124 } 1125 1126 unsigned getRegBitWidth(const MCRegisterClass &RC) { 1127 return getRegBitWidth(RC.getID()); 1128 } 1129 1130 unsigned getRegOperandSize(const MCRegisterInfo *MRI, const MCInstrDesc &Desc, 1131 unsigned OpNo) { 1132 assert(OpNo < Desc.NumOperands); 1133 unsigned RCID = Desc.OpInfo[OpNo].RegClass; 1134 return getRegBitWidth(MRI->getRegClass(RCID)) / 8; 1135 } 1136 1137 bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi) { 1138 if (Literal >= -16 && Literal <= 64) 1139 return true; 1140 1141 uint64_t Val = static_cast<uint64_t>(Literal); 1142 return (Val == DoubleToBits(0.0)) || 1143 (Val == DoubleToBits(1.0)) || 1144 (Val == DoubleToBits(-1.0)) || 1145 (Val == DoubleToBits(0.5)) || 1146 (Val == DoubleToBits(-0.5)) || 1147 (Val == DoubleToBits(2.0)) || 1148 (Val == DoubleToBits(-2.0)) || 1149 (Val == DoubleToBits(4.0)) || 1150 (Val == DoubleToBits(-4.0)) || 1151 (Val == 0x3fc45f306dc9c882 && HasInv2Pi); 1152 } 1153 1154 bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi) { 1155 if (Literal >= -16 && Literal <= 64) 1156 return true; 1157 1158 // The actual type of the operand does not seem to matter as long 1159 // as the bits match one of the inline immediate values. For example: 1160 // 1161 // -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal, 1162 // so it is a legal inline immediate. 1163 // 1164 // 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in 1165 // floating-point, so it is a legal inline immediate. 1166 1167 uint32_t Val = static_cast<uint32_t>(Literal); 1168 return (Val == FloatToBits(0.0f)) || 1169 (Val == FloatToBits(1.0f)) || 1170 (Val == FloatToBits(-1.0f)) || 1171 (Val == FloatToBits(0.5f)) || 1172 (Val == FloatToBits(-0.5f)) || 1173 (Val == FloatToBits(2.0f)) || 1174 (Val == FloatToBits(-2.0f)) || 1175 (Val == FloatToBits(4.0f)) || 1176 (Val == FloatToBits(-4.0f)) || 1177 (Val == 0x3e22f983 && HasInv2Pi); 1178 } 1179 1180 bool isInlinableLiteral16(int16_t Literal, bool HasInv2Pi) { 1181 if (!HasInv2Pi) 1182 return false; 1183 1184 if (Literal >= -16 && Literal <= 64) 1185 return true; 1186 1187 uint16_t Val = static_cast<uint16_t>(Literal); 1188 return Val == 0x3C00 || // 1.0 1189 Val == 0xBC00 || // -1.0 1190 Val == 0x3800 || // 0.5 1191 Val == 0xB800 || // -0.5 1192 Val == 0x4000 || // 2.0 1193 Val == 0xC000 || // -2.0 1194 Val == 0x4400 || // 4.0 1195 Val == 0xC400 || // -4.0 1196 Val == 0x3118; // 1/2pi 1197 } 1198 1199 bool isInlinableLiteralV216(int32_t Literal, bool HasInv2Pi) { 1200 assert(HasInv2Pi); 1201 1202 if (isInt<16>(Literal) || isUInt<16>(Literal)) { 1203 int16_t Trunc = static_cast<int16_t>(Literal); 1204 return AMDGPU::isInlinableLiteral16(Trunc, HasInv2Pi); 1205 } 1206 if (!(Literal & 0xffff)) 1207 return AMDGPU::isInlinableLiteral16(Literal >> 16, HasInv2Pi); 1208 1209 int16_t Lo16 = static_cast<int16_t>(Literal); 1210 int16_t Hi16 = static_cast<int16_t>(Literal >> 16); 1211 return Lo16 == Hi16 && isInlinableLiteral16(Lo16, HasInv2Pi); 1212 } 1213 1214 bool isArgPassedInSGPR(const Argument *A) { 1215 const Function *F = A->getParent(); 1216 1217 // Arguments to compute shaders are never a source of divergence. 1218 CallingConv::ID CC = F->getCallingConv(); 1219 switch (CC) { 1220 case CallingConv::AMDGPU_KERNEL: 1221 case CallingConv::SPIR_KERNEL: 1222 return true; 1223 case CallingConv::AMDGPU_VS: 1224 case CallingConv::AMDGPU_LS: 1225 case CallingConv::AMDGPU_HS: 1226 case CallingConv::AMDGPU_ES: 1227 case CallingConv::AMDGPU_GS: 1228 case CallingConv::AMDGPU_PS: 1229 case CallingConv::AMDGPU_CS: 1230 // For non-compute shaders, SGPR inputs are marked with either inreg or byval. 1231 // Everything else is in VGPRs. 1232 return F->getAttributes().hasParamAttribute(A->getArgNo(), Attribute::InReg) || 1233 F->getAttributes().hasParamAttribute(A->getArgNo(), Attribute::ByVal); 1234 default: 1235 // TODO: Should calls support inreg for SGPR inputs? 1236 return false; 1237 } 1238 } 1239 1240 static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) { 1241 return isGCN3Encoding(ST) || isGFX10(ST); 1242 } 1243 1244 static bool isLegalSMRDEncodedImmOffset(const MCSubtargetInfo &ST, 1245 int64_t EncodedOffset) { 1246 return hasSMEMByteOffset(ST) ? isUInt<20>(EncodedOffset) 1247 : isUInt<8>(EncodedOffset); 1248 } 1249 1250 static bool isDwordAligned(uint64_t ByteOffset) { 1251 return (ByteOffset & 3) == 0; 1252 } 1253 1254 uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST, 1255 uint64_t ByteOffset) { 1256 if (hasSMEMByteOffset(ST)) 1257 return ByteOffset; 1258 1259 assert(isDwordAligned(ByteOffset)); 1260 return ByteOffset >> 2; 1261 } 1262 1263 Optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST, 1264 int64_t ByteOffset) { 1265 if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST)) 1266 return None; 1267 1268 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset); 1269 return isLegalSMRDEncodedImmOffset(ST, EncodedOffset) ? 1270 Optional<int64_t>(EncodedOffset) : None; 1271 } 1272 1273 Optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST, 1274 int64_t ByteOffset) { 1275 if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST)) 1276 return None; 1277 1278 assert(isCI(ST)); 1279 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset); 1280 return isUInt<32>(EncodedOffset) ? Optional<int64_t>(EncodedOffset) : None; 1281 } 1282 1283 // Given Imm, split it into the values to put into the SOffset and ImmOffset 1284 // fields in an MUBUF instruction. Return false if it is not possible (due to a 1285 // hardware bug needing a workaround). 1286 // 1287 // The required alignment ensures that individual address components remain 1288 // aligned if they are aligned to begin with. It also ensures that additional 1289 // offsets within the given alignment can be added to the resulting ImmOffset. 1290 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset, 1291 const GCNSubtarget *Subtarget, uint32_t Align) { 1292 const uint32_t MaxImm = alignDown(4095, Align); 1293 uint32_t Overflow = 0; 1294 1295 if (Imm > MaxImm) { 1296 if (Imm <= MaxImm + 64) { 1297 // Use an SOffset inline constant for 4..64 1298 Overflow = Imm - MaxImm; 1299 Imm = MaxImm; 1300 } else { 1301 // Try to keep the same value in SOffset for adjacent loads, so that 1302 // the corresponding register contents can be re-used. 1303 // 1304 // Load values with all low-bits (except for alignment bits) set into 1305 // SOffset, so that a larger range of values can be covered using 1306 // s_movk_i32. 1307 // 1308 // Atomic operations fail to work correctly when individual address 1309 // components are unaligned, even if their sum is aligned. 1310 uint32_t High = (Imm + Align) & ~4095; 1311 uint32_t Low = (Imm + Align) & 4095; 1312 Imm = Low; 1313 Overflow = High - Align; 1314 } 1315 } 1316 1317 // There is a hardware bug in SI and CI which prevents address clamping in 1318 // MUBUF instructions from working correctly with SOffsets. The immediate 1319 // offset is unaffected. 1320 if (Overflow > 0 && 1321 Subtarget->getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS) 1322 return false; 1323 1324 ImmOffset = Imm; 1325 SOffset = Overflow; 1326 return true; 1327 } 1328 1329 SIModeRegisterDefaults::SIModeRegisterDefaults(const Function &F, 1330 const GCNSubtarget &ST) { 1331 *this = getDefaultForCallingConv(F.getCallingConv()); 1332 1333 StringRef IEEEAttr = F.getFnAttribute("amdgpu-ieee").getValueAsString(); 1334 if (!IEEEAttr.empty()) 1335 IEEE = IEEEAttr == "true"; 1336 1337 StringRef DX10ClampAttr 1338 = F.getFnAttribute("amdgpu-dx10-clamp").getValueAsString(); 1339 if (!DX10ClampAttr.empty()) 1340 DX10Clamp = DX10ClampAttr == "true"; 1341 1342 // FIXME: Split this when denormal-fp-math is used 1343 FP32InputDenormals = ST.hasFP32Denormals(F); 1344 FP32OutputDenormals = FP32InputDenormals; 1345 FP64FP16InputDenormals = ST.hasFP64FP16Denormals(F); 1346 FP64FP16OutputDenormals = FP64FP16InputDenormals; 1347 } 1348 1349 namespace { 1350 1351 struct SourceOfDivergence { 1352 unsigned Intr; 1353 }; 1354 const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr); 1355 1356 #define GET_SourcesOfDivergence_IMPL 1357 #define GET_Gfx9BufferFormat_IMPL 1358 #define GET_Gfx10PlusBufferFormat_IMPL 1359 #include "AMDGPUGenSearchableTables.inc" 1360 1361 } // end anonymous namespace 1362 1363 bool isIntrinsicSourceOfDivergence(unsigned IntrID) { 1364 return lookupSourceOfDivergence(IntrID); 1365 } 1366 1367 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp, 1368 uint8_t NumComponents, 1369 uint8_t NumFormat, 1370 const MCSubtargetInfo &STI) { 1371 return isGFX10(STI) 1372 ? getGfx10PlusBufferFormatInfo(BitsPerComp, NumComponents, 1373 NumFormat) 1374 : getGfx9BufferFormatInfo(BitsPerComp, NumComponents, NumFormat); 1375 } 1376 1377 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format, 1378 const MCSubtargetInfo &STI) { 1379 return isGFX10(STI) ? getGfx10PlusBufferFormatInfo(Format) 1380 : getGfx9BufferFormatInfo(Format); 1381 } 1382 1383 } // namespace AMDGPU 1384 } // namespace llvm 1385