1 //===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "AMDGPUBaseInfo.h"
10 #include "AMDGPU.h"
11 #include "AMDGPUAsmUtils.h"
12 #include "AMDKernelCodeT.h"
13 #include "GCNSubtarget.h"
14 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
15 #include "llvm/BinaryFormat/ELF.h"
16 #include "llvm/IR/Attributes.h"
17 #include "llvm/IR/Function.h"
18 #include "llvm/IR/GlobalValue.h"
19 #include "llvm/IR/IntrinsicsAMDGPU.h"
20 #include "llvm/IR/IntrinsicsR600.h"
21 #include "llvm/IR/LLVMContext.h"
22 #include "llvm/MC/MCSubtargetInfo.h"
23 #include "llvm/Support/AMDHSAKernelDescriptor.h"
24 #include "llvm/Support/CommandLine.h"
25 #include "llvm/Support/TargetParser.h"
26 
27 #define GET_INSTRINFO_NAMED_OPS
28 #define GET_INSTRMAP_INFO
29 #include "AMDGPUGenInstrInfo.inc"
30 
31 static llvm::cl::opt<unsigned> AmdhsaCodeObjectVersion(
32   "amdhsa-code-object-version", llvm::cl::Hidden,
33   llvm::cl::desc("AMDHSA Code Object Version"), llvm::cl::init(4),
34   llvm::cl::ZeroOrMore);
35 
36 namespace {
37 
38 /// \returns Bit mask for given bit \p Shift and bit \p Width.
39 unsigned getBitMask(unsigned Shift, unsigned Width) {
40   return ((1 << Width) - 1) << Shift;
41 }
42 
43 /// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width.
44 ///
45 /// \returns Packed \p Dst.
46 unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) {
47   Dst &= ~(1 << Shift) & ~getBitMask(Shift, Width);
48   Dst |= (Src << Shift) & getBitMask(Shift, Width);
49   return Dst;
50 }
51 
52 /// Unpacks bits from \p Src for given bit \p Shift and bit \p Width.
53 ///
54 /// \returns Unpacked bits.
55 unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) {
56   return (Src & getBitMask(Shift, Width)) >> Shift;
57 }
58 
59 /// \returns Vmcnt bit shift (lower bits).
60 unsigned getVmcntBitShiftLo() { return 0; }
61 
62 /// \returns Vmcnt bit width (lower bits).
63 unsigned getVmcntBitWidthLo() { return 4; }
64 
65 /// \returns Expcnt bit shift.
66 unsigned getExpcntBitShift() { return 4; }
67 
68 /// \returns Expcnt bit width.
69 unsigned getExpcntBitWidth() { return 3; }
70 
71 /// \returns Lgkmcnt bit shift.
72 unsigned getLgkmcntBitShift() { return 8; }
73 
74 /// \returns Lgkmcnt bit width.
75 unsigned getLgkmcntBitWidth(unsigned VersionMajor) {
76   return (VersionMajor >= 10) ? 6 : 4;
77 }
78 
79 /// \returns Vmcnt bit shift (higher bits).
80 unsigned getVmcntBitShiftHi() { return 14; }
81 
82 /// \returns Vmcnt bit width (higher bits).
83 unsigned getVmcntBitWidthHi() { return 2; }
84 
85 } // end namespace anonymous
86 
87 namespace llvm {
88 
89 namespace AMDGPU {
90 
91 Optional<uint8_t> getHsaAbiVersion(const MCSubtargetInfo *STI) {
92   if (STI && STI->getTargetTriple().getOS() != Triple::AMDHSA)
93     return None;
94 
95   switch (AmdhsaCodeObjectVersion) {
96   case 2:
97     return ELF::ELFABIVERSION_AMDGPU_HSA_V2;
98   case 3:
99     return ELF::ELFABIVERSION_AMDGPU_HSA_V3;
100   case 4:
101     return ELF::ELFABIVERSION_AMDGPU_HSA_V4;
102   case 5:
103     return ELF::ELFABIVERSION_AMDGPU_HSA_V5;
104   default:
105     report_fatal_error(Twine("Unsupported AMDHSA Code Object Version ") +
106                        Twine(AmdhsaCodeObjectVersion));
107   }
108 }
109 
110 bool isHsaAbiVersion2(const MCSubtargetInfo *STI) {
111   if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI))
112     return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V2;
113   return false;
114 }
115 
116 bool isHsaAbiVersion3(const MCSubtargetInfo *STI) {
117   if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI))
118     return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V3;
119   return false;
120 }
121 
122 bool isHsaAbiVersion4(const MCSubtargetInfo *STI) {
123   if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI))
124     return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V4;
125   return false;
126 }
127 
128 bool isHsaAbiVersion5(const MCSubtargetInfo *STI) {
129   if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI))
130     return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V5;
131   return false;
132 }
133 
134 bool isHsaAbiVersion3AndAbove(const MCSubtargetInfo *STI) {
135   return isHsaAbiVersion3(STI) || isHsaAbiVersion4(STI) ||
136          isHsaAbiVersion5(STI);
137 }
138 
139 unsigned getAmdhsaCodeObjectVersion() {
140   return AmdhsaCodeObjectVersion;
141 }
142 
143 // FIXME: All such magic numbers about the ABI should be in a
144 // central TD file.
145 unsigned getHostcallImplicitArgPosition() {
146   switch (AmdhsaCodeObjectVersion) {
147   case 2:
148   case 3:
149   case 4:
150     return 24;
151   case 5:
152     return AMDGPU::ImplicitArg::HOSTCALL_PTR_OFFSET;
153   default:
154     llvm_unreachable("Unexpected code object version");
155     return 0;
156   }
157 }
158 
159 #define GET_MIMGBaseOpcodesTable_IMPL
160 #define GET_MIMGDimInfoTable_IMPL
161 #define GET_MIMGInfoTable_IMPL
162 #define GET_MIMGLZMappingTable_IMPL
163 #define GET_MIMGMIPMappingTable_IMPL
164 #define GET_MIMGBiasMappingTable_IMPL
165 #define GET_MIMGOffsetMappingTable_IMPL
166 #define GET_MIMGG16MappingTable_IMPL
167 #define GET_MAIInstInfoTable_IMPL
168 #include "AMDGPUGenSearchableTables.inc"
169 
170 int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding,
171                   unsigned VDataDwords, unsigned VAddrDwords) {
172   const MIMGInfo *Info = getMIMGOpcodeHelper(BaseOpcode, MIMGEncoding,
173                                              VDataDwords, VAddrDwords);
174   return Info ? Info->Opcode : -1;
175 }
176 
177 const MIMGBaseOpcodeInfo *getMIMGBaseOpcode(unsigned Opc) {
178   const MIMGInfo *Info = getMIMGInfo(Opc);
179   return Info ? getMIMGBaseOpcodeInfo(Info->BaseOpcode) : nullptr;
180 }
181 
182 int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) {
183   const MIMGInfo *OrigInfo = getMIMGInfo(Opc);
184   const MIMGInfo *NewInfo =
185       getMIMGOpcodeHelper(OrigInfo->BaseOpcode, OrigInfo->MIMGEncoding,
186                           NewChannels, OrigInfo->VAddrDwords);
187   return NewInfo ? NewInfo->Opcode : -1;
188 }
189 
190 unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode,
191                            const MIMGDimInfo *Dim, bool IsA16,
192                            bool IsG16Supported) {
193   unsigned AddrWords = BaseOpcode->NumExtraArgs;
194   unsigned AddrComponents = (BaseOpcode->Coordinates ? Dim->NumCoords : 0) +
195                             (BaseOpcode->LodOrClampOrMip ? 1 : 0);
196   if (IsA16)
197     AddrWords += divideCeil(AddrComponents, 2);
198   else
199     AddrWords += AddrComponents;
200 
201   // Note: For subtargets that support A16 but not G16, enabling A16 also
202   // enables 16 bit gradients.
203   // For subtargets that support A16 (operand) and G16 (done with a different
204   // instruction encoding), they are independent.
205 
206   if (BaseOpcode->Gradients) {
207     if ((IsA16 && !IsG16Supported) || BaseOpcode->G16)
208       // There are two gradients per coordinate, we pack them separately.
209       // For the 3d case,
210       // we get (dy/du, dx/du) (-, dz/du) (dy/dv, dx/dv) (-, dz/dv)
211       AddrWords += alignTo<2>(Dim->NumGradients / 2);
212     else
213       AddrWords += Dim->NumGradients;
214   }
215   return AddrWords;
216 }
217 
218 struct MUBUFInfo {
219   uint16_t Opcode;
220   uint16_t BaseOpcode;
221   uint8_t elements;
222   bool has_vaddr;
223   bool has_srsrc;
224   bool has_soffset;
225   bool IsBufferInv;
226 };
227 
228 struct MTBUFInfo {
229   uint16_t Opcode;
230   uint16_t BaseOpcode;
231   uint8_t elements;
232   bool has_vaddr;
233   bool has_srsrc;
234   bool has_soffset;
235 };
236 
237 struct SMInfo {
238   uint16_t Opcode;
239   bool IsBuffer;
240 };
241 
242 struct VOPInfo {
243   uint16_t Opcode;
244   bool IsSingle;
245 };
246 
247 #define GET_MTBUFInfoTable_DECL
248 #define GET_MTBUFInfoTable_IMPL
249 #define GET_MUBUFInfoTable_DECL
250 #define GET_MUBUFInfoTable_IMPL
251 #define GET_SMInfoTable_DECL
252 #define GET_SMInfoTable_IMPL
253 #define GET_VOP1InfoTable_DECL
254 #define GET_VOP1InfoTable_IMPL
255 #define GET_VOP2InfoTable_DECL
256 #define GET_VOP2InfoTable_IMPL
257 #define GET_VOP3InfoTable_DECL
258 #define GET_VOP3InfoTable_IMPL
259 #include "AMDGPUGenSearchableTables.inc"
260 
261 int getMTBUFBaseOpcode(unsigned Opc) {
262   const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opc);
263   return Info ? Info->BaseOpcode : -1;
264 }
265 
266 int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements) {
267   const MTBUFInfo *Info = getMTBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements);
268   return Info ? Info->Opcode : -1;
269 }
270 
271 int getMTBUFElements(unsigned Opc) {
272   const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
273   return Info ? Info->elements : 0;
274 }
275 
276 bool getMTBUFHasVAddr(unsigned Opc) {
277   const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
278   return Info ? Info->has_vaddr : false;
279 }
280 
281 bool getMTBUFHasSrsrc(unsigned Opc) {
282   const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
283   return Info ? Info->has_srsrc : false;
284 }
285 
286 bool getMTBUFHasSoffset(unsigned Opc) {
287   const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
288   return Info ? Info->has_soffset : false;
289 }
290 
291 int getMUBUFBaseOpcode(unsigned Opc) {
292   const MUBUFInfo *Info = getMUBUFInfoFromOpcode(Opc);
293   return Info ? Info->BaseOpcode : -1;
294 }
295 
296 int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements) {
297   const MUBUFInfo *Info = getMUBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements);
298   return Info ? Info->Opcode : -1;
299 }
300 
301 int getMUBUFElements(unsigned Opc) {
302   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
303   return Info ? Info->elements : 0;
304 }
305 
306 bool getMUBUFHasVAddr(unsigned Opc) {
307   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
308   return Info ? Info->has_vaddr : false;
309 }
310 
311 bool getMUBUFHasSrsrc(unsigned Opc) {
312   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
313   return Info ? Info->has_srsrc : false;
314 }
315 
316 bool getMUBUFHasSoffset(unsigned Opc) {
317   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
318   return Info ? Info->has_soffset : false;
319 }
320 
321 bool getMUBUFIsBufferInv(unsigned Opc) {
322   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
323   return Info ? Info->IsBufferInv : false;
324 }
325 
326 bool getSMEMIsBuffer(unsigned Opc) {
327   const SMInfo *Info = getSMEMOpcodeHelper(Opc);
328   return Info ? Info->IsBuffer : false;
329 }
330 
331 bool getVOP1IsSingle(unsigned Opc) {
332   const VOPInfo *Info = getVOP1OpcodeHelper(Opc);
333   return Info ? Info->IsSingle : false;
334 }
335 
336 bool getVOP2IsSingle(unsigned Opc) {
337   const VOPInfo *Info = getVOP2OpcodeHelper(Opc);
338   return Info ? Info->IsSingle : false;
339 }
340 
341 bool getVOP3IsSingle(unsigned Opc) {
342   const VOPInfo *Info = getVOP3OpcodeHelper(Opc);
343   return Info ? Info->IsSingle : false;
344 }
345 
346 bool getMAIIsGFX940XDL(unsigned Opc) {
347   const MAIInstInfo *Info = getMAIInstInfoHelper(Opc);
348   return Info ? Info->is_gfx940_xdl : false;
349 }
350 
351 // Wrapper for Tablegen'd function.  enum Subtarget is not defined in any
352 // header files, so we need to wrap it in a function that takes unsigned
353 // instead.
354 int getMCOpcode(uint16_t Opcode, unsigned Gen) {
355   return getMCOpcodeGen(Opcode, static_cast<Subtarget>(Gen));
356 }
357 
358 namespace IsaInfo {
359 
360 AMDGPUTargetID::AMDGPUTargetID(const MCSubtargetInfo &STI)
361     : STI(STI), XnackSetting(TargetIDSetting::Any),
362       SramEccSetting(TargetIDSetting::Any) {
363   if (!STI.getFeatureBits().test(FeatureSupportsXNACK))
364     XnackSetting = TargetIDSetting::Unsupported;
365   if (!STI.getFeatureBits().test(FeatureSupportsSRAMECC))
366     SramEccSetting = TargetIDSetting::Unsupported;
367 }
368 
369 void AMDGPUTargetID::setTargetIDFromFeaturesString(StringRef FS) {
370   // Check if xnack or sramecc is explicitly enabled or disabled.  In the
371   // absence of the target features we assume we must generate code that can run
372   // in any environment.
373   SubtargetFeatures Features(FS);
374   Optional<bool> XnackRequested;
375   Optional<bool> SramEccRequested;
376 
377   for (const std::string &Feature : Features.getFeatures()) {
378     if (Feature == "+xnack")
379       XnackRequested = true;
380     else if (Feature == "-xnack")
381       XnackRequested = false;
382     else if (Feature == "+sramecc")
383       SramEccRequested = true;
384     else if (Feature == "-sramecc")
385       SramEccRequested = false;
386   }
387 
388   bool XnackSupported = isXnackSupported();
389   bool SramEccSupported = isSramEccSupported();
390 
391   if (XnackRequested) {
392     if (XnackSupported) {
393       XnackSetting =
394           *XnackRequested ? TargetIDSetting::On : TargetIDSetting::Off;
395     } else {
396       // If a specific xnack setting was requested and this GPU does not support
397       // xnack emit a warning. Setting will remain set to "Unsupported".
398       if (*XnackRequested) {
399         errs() << "warning: xnack 'On' was requested for a processor that does "
400                   "not support it!\n";
401       } else {
402         errs() << "warning: xnack 'Off' was requested for a processor that "
403                   "does not support it!\n";
404       }
405     }
406   }
407 
408   if (SramEccRequested) {
409     if (SramEccSupported) {
410       SramEccSetting =
411           *SramEccRequested ? TargetIDSetting::On : TargetIDSetting::Off;
412     } else {
413       // If a specific sramecc setting was requested and this GPU does not
414       // support sramecc emit a warning. Setting will remain set to
415       // "Unsupported".
416       if (*SramEccRequested) {
417         errs() << "warning: sramecc 'On' was requested for a processor that "
418                   "does not support it!\n";
419       } else {
420         errs() << "warning: sramecc 'Off' was requested for a processor that "
421                   "does not support it!\n";
422       }
423     }
424   }
425 }
426 
427 static TargetIDSetting
428 getTargetIDSettingFromFeatureString(StringRef FeatureString) {
429   if (FeatureString.endswith("-"))
430     return TargetIDSetting::Off;
431   if (FeatureString.endswith("+"))
432     return TargetIDSetting::On;
433 
434   llvm_unreachable("Malformed feature string");
435 }
436 
437 void AMDGPUTargetID::setTargetIDFromTargetIDStream(StringRef TargetID) {
438   SmallVector<StringRef, 3> TargetIDSplit;
439   TargetID.split(TargetIDSplit, ':');
440 
441   for (const auto &FeatureString : TargetIDSplit) {
442     if (FeatureString.startswith("xnack"))
443       XnackSetting = getTargetIDSettingFromFeatureString(FeatureString);
444     if (FeatureString.startswith("sramecc"))
445       SramEccSetting = getTargetIDSettingFromFeatureString(FeatureString);
446   }
447 }
448 
449 std::string AMDGPUTargetID::toString() const {
450   std::string StringRep;
451   raw_string_ostream StreamRep(StringRep);
452 
453   auto TargetTriple = STI.getTargetTriple();
454   auto Version = getIsaVersion(STI.getCPU());
455 
456   StreamRep << TargetTriple.getArchName() << '-'
457             << TargetTriple.getVendorName() << '-'
458             << TargetTriple.getOSName() << '-'
459             << TargetTriple.getEnvironmentName() << '-';
460 
461   std::string Processor;
462   // TODO: Following else statement is present here because we used various
463   // alias names for GPUs up until GFX9 (e.g. 'fiji' is same as 'gfx803').
464   // Remove once all aliases are removed from GCNProcessors.td.
465   if (Version.Major >= 9)
466     Processor = STI.getCPU().str();
467   else
468     Processor = (Twine("gfx") + Twine(Version.Major) + Twine(Version.Minor) +
469                  Twine(Version.Stepping))
470                     .str();
471 
472   std::string Features;
473   if (Optional<uint8_t> HsaAbiVersion = getHsaAbiVersion(&STI)) {
474     switch (*HsaAbiVersion) {
475     case ELF::ELFABIVERSION_AMDGPU_HSA_V2:
476       // Code object V2 only supported specific processors and had fixed
477       // settings for the XNACK.
478       if (Processor == "gfx600") {
479       } else if (Processor == "gfx601") {
480       } else if (Processor == "gfx602") {
481       } else if (Processor == "gfx700") {
482       } else if (Processor == "gfx701") {
483       } else if (Processor == "gfx702") {
484       } else if (Processor == "gfx703") {
485       } else if (Processor == "gfx704") {
486       } else if (Processor == "gfx705") {
487       } else if (Processor == "gfx801") {
488         if (!isXnackOnOrAny())
489           report_fatal_error(
490               "AMD GPU code object V2 does not support processor " +
491               Twine(Processor) + " without XNACK");
492       } else if (Processor == "gfx802") {
493       } else if (Processor == "gfx803") {
494       } else if (Processor == "gfx805") {
495       } else if (Processor == "gfx810") {
496         if (!isXnackOnOrAny())
497           report_fatal_error(
498               "AMD GPU code object V2 does not support processor " +
499               Twine(Processor) + " without XNACK");
500       } else if (Processor == "gfx900") {
501         if (isXnackOnOrAny())
502           Processor = "gfx901";
503       } else if (Processor == "gfx902") {
504         if (isXnackOnOrAny())
505           Processor = "gfx903";
506       } else if (Processor == "gfx904") {
507         if (isXnackOnOrAny())
508           Processor = "gfx905";
509       } else if (Processor == "gfx906") {
510         if (isXnackOnOrAny())
511           Processor = "gfx907";
512       } else if (Processor == "gfx90c") {
513         if (isXnackOnOrAny())
514           report_fatal_error(
515               "AMD GPU code object V2 does not support processor " +
516               Twine(Processor) + " with XNACK being ON or ANY");
517       } else {
518         report_fatal_error(
519             "AMD GPU code object V2 does not support processor " +
520             Twine(Processor));
521       }
522       break;
523     case ELF::ELFABIVERSION_AMDGPU_HSA_V3:
524       // xnack.
525       if (isXnackOnOrAny())
526         Features += "+xnack";
527       // In code object v2 and v3, "sramecc" feature was spelled with a
528       // hyphen ("sram-ecc").
529       if (isSramEccOnOrAny())
530         Features += "+sram-ecc";
531       break;
532     case ELF::ELFABIVERSION_AMDGPU_HSA_V4:
533     case ELF::ELFABIVERSION_AMDGPU_HSA_V5:
534       // sramecc.
535       if (getSramEccSetting() == TargetIDSetting::Off)
536         Features += ":sramecc-";
537       else if (getSramEccSetting() == TargetIDSetting::On)
538         Features += ":sramecc+";
539       // xnack.
540       if (getXnackSetting() == TargetIDSetting::Off)
541         Features += ":xnack-";
542       else if (getXnackSetting() == TargetIDSetting::On)
543         Features += ":xnack+";
544       break;
545     default:
546       break;
547     }
548   }
549 
550   StreamRep << Processor << Features;
551 
552   StreamRep.flush();
553   return StringRep;
554 }
555 
556 unsigned getWavefrontSize(const MCSubtargetInfo *STI) {
557   if (STI->getFeatureBits().test(FeatureWavefrontSize16))
558     return 16;
559   if (STI->getFeatureBits().test(FeatureWavefrontSize32))
560     return 32;
561 
562   return 64;
563 }
564 
565 unsigned getLocalMemorySize(const MCSubtargetInfo *STI) {
566   if (STI->getFeatureBits().test(FeatureLocalMemorySize32768))
567     return 32768;
568   if (STI->getFeatureBits().test(FeatureLocalMemorySize65536))
569     return 65536;
570 
571   return 0;
572 }
573 
574 unsigned getEUsPerCU(const MCSubtargetInfo *STI) {
575   // "Per CU" really means "per whatever functional block the waves of a
576   // workgroup must share". For gfx10 in CU mode this is the CU, which contains
577   // two SIMDs.
578   if (isGFX10Plus(*STI) && STI->getFeatureBits().test(FeatureCuMode))
579     return 2;
580   // Pre-gfx10 a CU contains four SIMDs. For gfx10 in WGP mode the WGP contains
581   // two CUs, so a total of four SIMDs.
582   return 4;
583 }
584 
585 unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo *STI,
586                                unsigned FlatWorkGroupSize) {
587   assert(FlatWorkGroupSize != 0);
588   if (STI->getTargetTriple().getArch() != Triple::amdgcn)
589     return 8;
590   unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize);
591   if (N == 1)
592     return 40;
593   N = 40 / N;
594   return std::min(N, 16u);
595 }
596 
597 unsigned getMinWavesPerEU(const MCSubtargetInfo *STI) {
598   return 1;
599 }
600 
601 unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI) {
602   // FIXME: Need to take scratch memory into account.
603   if (isGFX90A(*STI))
604     return 8;
605   if (!isGFX10Plus(*STI))
606     return 10;
607   return hasGFX10_3Insts(*STI) ? 16 : 20;
608 }
609 
610 unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo *STI,
611                                    unsigned FlatWorkGroupSize) {
612   return divideCeil(getWavesPerWorkGroup(STI, FlatWorkGroupSize),
613                     getEUsPerCU(STI));
614 }
615 
616 unsigned getMinFlatWorkGroupSize(const MCSubtargetInfo *STI) {
617   return 1;
618 }
619 
620 unsigned getMaxFlatWorkGroupSize(const MCSubtargetInfo *STI) {
621   // Some subtargets allow encoding 2048, but this isn't tested or supported.
622   return 1024;
623 }
624 
625 unsigned getWavesPerWorkGroup(const MCSubtargetInfo *STI,
626                               unsigned FlatWorkGroupSize) {
627   return divideCeil(FlatWorkGroupSize, getWavefrontSize(STI));
628 }
629 
630 unsigned getSGPRAllocGranule(const MCSubtargetInfo *STI) {
631   IsaVersion Version = getIsaVersion(STI->getCPU());
632   if (Version.Major >= 10)
633     return getAddressableNumSGPRs(STI);
634   if (Version.Major >= 8)
635     return 16;
636   return 8;
637 }
638 
639 unsigned getSGPREncodingGranule(const MCSubtargetInfo *STI) {
640   return 8;
641 }
642 
643 unsigned getTotalNumSGPRs(const MCSubtargetInfo *STI) {
644   IsaVersion Version = getIsaVersion(STI->getCPU());
645   if (Version.Major >= 8)
646     return 800;
647   return 512;
648 }
649 
650 unsigned getAddressableNumSGPRs(const MCSubtargetInfo *STI) {
651   if (STI->getFeatureBits().test(FeatureSGPRInitBug))
652     return FIXED_NUM_SGPRS_FOR_INIT_BUG;
653 
654   IsaVersion Version = getIsaVersion(STI->getCPU());
655   if (Version.Major >= 10)
656     return 106;
657   if (Version.Major >= 8)
658     return 102;
659   return 104;
660 }
661 
662 unsigned getMinNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
663   assert(WavesPerEU != 0);
664 
665   IsaVersion Version = getIsaVersion(STI->getCPU());
666   if (Version.Major >= 10)
667     return 0;
668 
669   if (WavesPerEU >= getMaxWavesPerEU(STI))
670     return 0;
671 
672   unsigned MinNumSGPRs = getTotalNumSGPRs(STI) / (WavesPerEU + 1);
673   if (STI->getFeatureBits().test(FeatureTrapHandler))
674     MinNumSGPRs -= std::min(MinNumSGPRs, (unsigned)TRAP_NUM_SGPRS);
675   MinNumSGPRs = alignDown(MinNumSGPRs, getSGPRAllocGranule(STI)) + 1;
676   return std::min(MinNumSGPRs, getAddressableNumSGPRs(STI));
677 }
678 
679 unsigned getMaxNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU,
680                         bool Addressable) {
681   assert(WavesPerEU != 0);
682 
683   unsigned AddressableNumSGPRs = getAddressableNumSGPRs(STI);
684   IsaVersion Version = getIsaVersion(STI->getCPU());
685   if (Version.Major >= 10)
686     return Addressable ? AddressableNumSGPRs : 108;
687   if (Version.Major >= 8 && !Addressable)
688     AddressableNumSGPRs = 112;
689   unsigned MaxNumSGPRs = getTotalNumSGPRs(STI) / WavesPerEU;
690   if (STI->getFeatureBits().test(FeatureTrapHandler))
691     MaxNumSGPRs -= std::min(MaxNumSGPRs, (unsigned)TRAP_NUM_SGPRS);
692   MaxNumSGPRs = alignDown(MaxNumSGPRs, getSGPRAllocGranule(STI));
693   return std::min(MaxNumSGPRs, AddressableNumSGPRs);
694 }
695 
696 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed,
697                           bool FlatScrUsed, bool XNACKUsed) {
698   unsigned ExtraSGPRs = 0;
699   if (VCCUsed)
700     ExtraSGPRs = 2;
701 
702   IsaVersion Version = getIsaVersion(STI->getCPU());
703   if (Version.Major >= 10)
704     return ExtraSGPRs;
705 
706   if (Version.Major < 8) {
707     if (FlatScrUsed)
708       ExtraSGPRs = 4;
709   } else {
710     if (XNACKUsed)
711       ExtraSGPRs = 4;
712 
713     if (FlatScrUsed ||
714         STI->getFeatureBits().test(AMDGPU::FeatureArchitectedFlatScratch))
715       ExtraSGPRs = 6;
716   }
717 
718   return ExtraSGPRs;
719 }
720 
721 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed,
722                           bool FlatScrUsed) {
723   return getNumExtraSGPRs(STI, VCCUsed, FlatScrUsed,
724                           STI->getFeatureBits().test(AMDGPU::FeatureXNACK));
725 }
726 
727 unsigned getNumSGPRBlocks(const MCSubtargetInfo *STI, unsigned NumSGPRs) {
728   NumSGPRs = alignTo(std::max(1u, NumSGPRs), getSGPREncodingGranule(STI));
729   // SGPRBlocks is actual number of SGPR blocks minus 1.
730   return NumSGPRs / getSGPREncodingGranule(STI) - 1;
731 }
732 
733 unsigned getVGPRAllocGranule(const MCSubtargetInfo *STI,
734                              Optional<bool> EnableWavefrontSize32) {
735   if (STI->getFeatureBits().test(FeatureGFX90AInsts))
736     return 8;
737 
738   bool IsWave32 = EnableWavefrontSize32 ?
739       *EnableWavefrontSize32 :
740       STI->getFeatureBits().test(FeatureWavefrontSize32);
741 
742   if (hasGFX10_3Insts(*STI))
743     return IsWave32 ? 16 : 8;
744 
745   return IsWave32 ? 8 : 4;
746 }
747 
748 unsigned getVGPREncodingGranule(const MCSubtargetInfo *STI,
749                                 Optional<bool> EnableWavefrontSize32) {
750   if (STI->getFeatureBits().test(FeatureGFX90AInsts))
751     return 8;
752 
753   bool IsWave32 = EnableWavefrontSize32 ?
754       *EnableWavefrontSize32 :
755       STI->getFeatureBits().test(FeatureWavefrontSize32);
756 
757   return IsWave32 ? 8 : 4;
758 }
759 
760 unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) {
761   if (STI->getFeatureBits().test(FeatureGFX90AInsts))
762     return 512;
763   if (!isGFX10Plus(*STI))
764     return 256;
765   return STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1024 : 512;
766 }
767 
768 unsigned getAddressableNumVGPRs(const MCSubtargetInfo *STI) {
769   if (STI->getFeatureBits().test(FeatureGFX90AInsts))
770     return 512;
771   return 256;
772 }
773 
774 unsigned getMinNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
775   assert(WavesPerEU != 0);
776 
777   if (WavesPerEU >= getMaxWavesPerEU(STI))
778     return 0;
779   unsigned MinNumVGPRs =
780       alignDown(getTotalNumVGPRs(STI) / (WavesPerEU + 1),
781                 getVGPRAllocGranule(STI)) + 1;
782   return std::min(MinNumVGPRs, getAddressableNumVGPRs(STI));
783 }
784 
785 unsigned getMaxNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
786   assert(WavesPerEU != 0);
787 
788   unsigned MaxNumVGPRs = alignDown(getTotalNumVGPRs(STI) / WavesPerEU,
789                                    getVGPRAllocGranule(STI));
790   unsigned AddressableNumVGPRs = getAddressableNumVGPRs(STI);
791   return std::min(MaxNumVGPRs, AddressableNumVGPRs);
792 }
793 
794 unsigned getNumVGPRBlocks(const MCSubtargetInfo *STI, unsigned NumVGPRs,
795                           Optional<bool> EnableWavefrontSize32) {
796   NumVGPRs = alignTo(std::max(1u, NumVGPRs),
797                      getVGPREncodingGranule(STI, EnableWavefrontSize32));
798   // VGPRBlocks is actual number of VGPR blocks minus 1.
799   return NumVGPRs / getVGPREncodingGranule(STI, EnableWavefrontSize32) - 1;
800 }
801 
802 } // end namespace IsaInfo
803 
804 void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header,
805                                const MCSubtargetInfo *STI) {
806   IsaVersion Version = getIsaVersion(STI->getCPU());
807 
808   memset(&Header, 0, sizeof(Header));
809 
810   Header.amd_kernel_code_version_major = 1;
811   Header.amd_kernel_code_version_minor = 2;
812   Header.amd_machine_kind = 1; // AMD_MACHINE_KIND_AMDGPU
813   Header.amd_machine_version_major = Version.Major;
814   Header.amd_machine_version_minor = Version.Minor;
815   Header.amd_machine_version_stepping = Version.Stepping;
816   Header.kernel_code_entry_byte_offset = sizeof(Header);
817   Header.wavefront_size = 6;
818 
819   // If the code object does not support indirect functions, then the value must
820   // be 0xffffffff.
821   Header.call_convention = -1;
822 
823   // These alignment values are specified in powers of two, so alignment =
824   // 2^n.  The minimum alignment is 2^4 = 16.
825   Header.kernarg_segment_alignment = 4;
826   Header.group_segment_alignment = 4;
827   Header.private_segment_alignment = 4;
828 
829   if (Version.Major >= 10) {
830     if (STI->getFeatureBits().test(FeatureWavefrontSize32)) {
831       Header.wavefront_size = 5;
832       Header.code_properties |= AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32;
833     }
834     Header.compute_pgm_resource_registers |=
835       S_00B848_WGP_MODE(STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1) |
836       S_00B848_MEM_ORDERED(1);
837   }
838 }
839 
840 amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor(
841     const MCSubtargetInfo *STI) {
842   IsaVersion Version = getIsaVersion(STI->getCPU());
843 
844   amdhsa::kernel_descriptor_t KD;
845   memset(&KD, 0, sizeof(KD));
846 
847   AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
848                   amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64,
849                   amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE);
850   AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
851                   amdhsa::COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP, 1);
852   AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
853                   amdhsa::COMPUTE_PGM_RSRC1_ENABLE_IEEE_MODE, 1);
854   AMDHSA_BITS_SET(KD.compute_pgm_rsrc2,
855                   amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, 1);
856   if (Version.Major >= 10) {
857     AMDHSA_BITS_SET(KD.kernel_code_properties,
858                     amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32,
859                     STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1 : 0);
860     AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
861                     amdhsa::COMPUTE_PGM_RSRC1_WGP_MODE,
862                     STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1);
863     AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
864                     amdhsa::COMPUTE_PGM_RSRC1_MEM_ORDERED, 1);
865   }
866   if (AMDGPU::isGFX90A(*STI)) {
867     AMDHSA_BITS_SET(KD.compute_pgm_rsrc3,
868                     amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT,
869                     STI->getFeatureBits().test(FeatureTgSplit) ? 1 : 0);
870   }
871   return KD;
872 }
873 
874 bool isGroupSegment(const GlobalValue *GV) {
875   return GV->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS;
876 }
877 
878 bool isGlobalSegment(const GlobalValue *GV) {
879   return GV->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS;
880 }
881 
882 bool isReadOnlySegment(const GlobalValue *GV) {
883   unsigned AS = GV->getAddressSpace();
884   return AS == AMDGPUAS::CONSTANT_ADDRESS ||
885          AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
886 }
887 
888 bool shouldEmitConstantsToTextSection(const Triple &TT) {
889   return TT.getArch() == Triple::r600;
890 }
891 
892 int getIntegerAttribute(const Function &F, StringRef Name, int Default) {
893   Attribute A = F.getFnAttribute(Name);
894   int Result = Default;
895 
896   if (A.isStringAttribute()) {
897     StringRef Str = A.getValueAsString();
898     if (Str.getAsInteger(0, Result)) {
899       LLVMContext &Ctx = F.getContext();
900       Ctx.emitError("can't parse integer attribute " + Name);
901     }
902   }
903 
904   return Result;
905 }
906 
907 std::pair<int, int> getIntegerPairAttribute(const Function &F,
908                                             StringRef Name,
909                                             std::pair<int, int> Default,
910                                             bool OnlyFirstRequired) {
911   Attribute A = F.getFnAttribute(Name);
912   if (!A.isStringAttribute())
913     return Default;
914 
915   LLVMContext &Ctx = F.getContext();
916   std::pair<int, int> Ints = Default;
917   std::pair<StringRef, StringRef> Strs = A.getValueAsString().split(',');
918   if (Strs.first.trim().getAsInteger(0, Ints.first)) {
919     Ctx.emitError("can't parse first integer attribute " + Name);
920     return Default;
921   }
922   if (Strs.second.trim().getAsInteger(0, Ints.second)) {
923     if (!OnlyFirstRequired || !Strs.second.trim().empty()) {
924       Ctx.emitError("can't parse second integer attribute " + Name);
925       return Default;
926     }
927   }
928 
929   return Ints;
930 }
931 
932 unsigned getVmcntBitMask(const IsaVersion &Version) {
933   unsigned VmcntLo = (1 << getVmcntBitWidthLo()) - 1;
934   if (Version.Major < 9)
935     return VmcntLo;
936 
937   unsigned VmcntHi = ((1 << getVmcntBitWidthHi()) - 1) << getVmcntBitWidthLo();
938   return VmcntLo | VmcntHi;
939 }
940 
941 unsigned getExpcntBitMask(const IsaVersion &Version) {
942   return (1 << getExpcntBitWidth()) - 1;
943 }
944 
945 unsigned getLgkmcntBitMask(const IsaVersion &Version) {
946   return (1 << getLgkmcntBitWidth(Version.Major)) - 1;
947 }
948 
949 unsigned getWaitcntBitMask(const IsaVersion &Version) {
950   unsigned VmcntLo = getBitMask(getVmcntBitShiftLo(), getVmcntBitWidthLo());
951   unsigned Expcnt = getBitMask(getExpcntBitShift(), getExpcntBitWidth());
952   unsigned Lgkmcnt = getBitMask(getLgkmcntBitShift(),
953                                 getLgkmcntBitWidth(Version.Major));
954   unsigned Waitcnt = VmcntLo | Expcnt | Lgkmcnt;
955   if (Version.Major < 9)
956     return Waitcnt;
957 
958   unsigned VmcntHi = getBitMask(getVmcntBitShiftHi(), getVmcntBitWidthHi());
959   return Waitcnt | VmcntHi;
960 }
961 
962 unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) {
963   unsigned VmcntLo =
964       unpackBits(Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo());
965   if (Version.Major < 9)
966     return VmcntLo;
967 
968   unsigned VmcntHi =
969       unpackBits(Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi());
970   VmcntHi <<= getVmcntBitWidthLo();
971   return VmcntLo | VmcntHi;
972 }
973 
974 unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) {
975   return unpackBits(Waitcnt, getExpcntBitShift(), getExpcntBitWidth());
976 }
977 
978 unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) {
979   return unpackBits(Waitcnt, getLgkmcntBitShift(),
980                     getLgkmcntBitWidth(Version.Major));
981 }
982 
983 void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt,
984                    unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt) {
985   Vmcnt = decodeVmcnt(Version, Waitcnt);
986   Expcnt = decodeExpcnt(Version, Waitcnt);
987   Lgkmcnt = decodeLgkmcnt(Version, Waitcnt);
988 }
989 
990 Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded) {
991   Waitcnt Decoded;
992   Decoded.VmCnt = decodeVmcnt(Version, Encoded);
993   Decoded.ExpCnt = decodeExpcnt(Version, Encoded);
994   Decoded.LgkmCnt = decodeLgkmcnt(Version, Encoded);
995   return Decoded;
996 }
997 
998 unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt,
999                      unsigned Vmcnt) {
1000   Waitcnt =
1001       packBits(Vmcnt, Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo());
1002   if (Version.Major < 9)
1003     return Waitcnt;
1004 
1005   Vmcnt >>= getVmcntBitWidthLo();
1006   return packBits(Vmcnt, Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi());
1007 }
1008 
1009 unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt,
1010                       unsigned Expcnt) {
1011   return packBits(Expcnt, Waitcnt, getExpcntBitShift(), getExpcntBitWidth());
1012 }
1013 
1014 unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt,
1015                        unsigned Lgkmcnt) {
1016   return packBits(Lgkmcnt, Waitcnt, getLgkmcntBitShift(),
1017                                     getLgkmcntBitWidth(Version.Major));
1018 }
1019 
1020 unsigned encodeWaitcnt(const IsaVersion &Version,
1021                        unsigned Vmcnt, unsigned Expcnt, unsigned Lgkmcnt) {
1022   unsigned Waitcnt = getWaitcntBitMask(Version);
1023   Waitcnt = encodeVmcnt(Version, Waitcnt, Vmcnt);
1024   Waitcnt = encodeExpcnt(Version, Waitcnt, Expcnt);
1025   Waitcnt = encodeLgkmcnt(Version, Waitcnt, Lgkmcnt);
1026   return Waitcnt;
1027 }
1028 
1029 unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded) {
1030   return encodeWaitcnt(Version, Decoded.VmCnt, Decoded.ExpCnt, Decoded.LgkmCnt);
1031 }
1032 
1033 //===----------------------------------------------------------------------===//
1034 // Custom Operands.
1035 //
1036 // A table of custom operands shall describe "primary" operand names
1037 // first followed by aliases if any. It is not required but recommended
1038 // to arrange operands so that operand encoding match operand position
1039 // in the table. This will make disassembly a bit more efficient.
1040 // Unused slots in the table shall have an empty name.
1041 //
1042 //===----------------------------------------------------------------------===//
1043 
1044 template <class T>
1045 static bool isValidOpr(int Idx, const CustomOperand<T> OpInfo[], int OpInfoSize,
1046                        T Context) {
1047   return 0 <= Idx && Idx < OpInfoSize && !OpInfo[Idx].Name.empty() &&
1048          (!OpInfo[Idx].Cond || OpInfo[Idx].Cond(Context));
1049 }
1050 
1051 template <class T>
1052 static int getOprIdx(std::function<bool(const CustomOperand<T> &)> Test,
1053                      const CustomOperand<T> OpInfo[], int OpInfoSize,
1054                      T Context) {
1055   int InvalidIdx = OPR_ID_UNKNOWN;
1056   for (int Idx = 0; Idx < OpInfoSize; ++Idx) {
1057     if (Test(OpInfo[Idx])) {
1058       if (!OpInfo[Idx].Cond || OpInfo[Idx].Cond(Context))
1059         return Idx;
1060       InvalidIdx = OPR_ID_UNSUPPORTED;
1061     }
1062   }
1063   return InvalidIdx;
1064 }
1065 
1066 template <class T>
1067 static int getOprIdx(const StringRef Name, const CustomOperand<T> OpInfo[],
1068                      int OpInfoSize, T Context) {
1069   auto Test = [=](const CustomOperand<T> &Op) { return Op.Name == Name; };
1070   return getOprIdx<T>(Test, OpInfo, OpInfoSize, Context);
1071 }
1072 
1073 template <class T>
1074 static int getOprIdx(int Id, const CustomOperand<T> OpInfo[], int OpInfoSize,
1075                      T Context, bool QuickCheck = true) {
1076   auto Test = [=](const CustomOperand<T> &Op) {
1077     return Op.Encoding == Id && !Op.Name.empty();
1078   };
1079   // This is an optimization that should work in most cases.
1080   // As a side effect, it may cause selection of an alias
1081   // instead of a primary operand name in case of sparse tables.
1082   if (QuickCheck && isValidOpr<T>(Id, OpInfo, OpInfoSize, Context) &&
1083       OpInfo[Id].Encoding == Id) {
1084     return Id;
1085   }
1086   return getOprIdx<T>(Test, OpInfo, OpInfoSize, Context);
1087 }
1088 
1089 //===----------------------------------------------------------------------===//
1090 // hwreg
1091 //===----------------------------------------------------------------------===//
1092 
1093 namespace Hwreg {
1094 
1095 int64_t getHwregId(const StringRef Name, const MCSubtargetInfo &STI) {
1096   int Idx = getOprIdx<const MCSubtargetInfo &>(Name, Opr, OPR_SIZE, STI);
1097   return (Idx < 0) ? Idx : Opr[Idx].Encoding;
1098 }
1099 
1100 bool isValidHwreg(int64_t Id) {
1101   return 0 <= Id && isUInt<ID_WIDTH_>(Id);
1102 }
1103 
1104 bool isValidHwregOffset(int64_t Offset) {
1105   return 0 <= Offset && isUInt<OFFSET_WIDTH_>(Offset);
1106 }
1107 
1108 bool isValidHwregWidth(int64_t Width) {
1109   return 0 <= (Width - 1) && isUInt<WIDTH_M1_WIDTH_>(Width - 1);
1110 }
1111 
1112 uint64_t encodeHwreg(uint64_t Id, uint64_t Offset, uint64_t Width) {
1113   return (Id << ID_SHIFT_) |
1114          (Offset << OFFSET_SHIFT_) |
1115          ((Width - 1) << WIDTH_M1_SHIFT_);
1116 }
1117 
1118 StringRef getHwreg(unsigned Id, const MCSubtargetInfo &STI) {
1119   int Idx = getOprIdx<const MCSubtargetInfo &>(Id, Opr, OPR_SIZE, STI);
1120   return (Idx < 0) ? "" : Opr[Idx].Name;
1121 }
1122 
1123 void decodeHwreg(unsigned Val, unsigned &Id, unsigned &Offset, unsigned &Width) {
1124   Id = (Val & ID_MASK_) >> ID_SHIFT_;
1125   Offset = (Val & OFFSET_MASK_) >> OFFSET_SHIFT_;
1126   Width = ((Val & WIDTH_M1_MASK_) >> WIDTH_M1_SHIFT_) + 1;
1127 }
1128 
1129 } // namespace Hwreg
1130 
1131 //===----------------------------------------------------------------------===//
1132 // exp tgt
1133 //===----------------------------------------------------------------------===//
1134 
1135 namespace Exp {
1136 
1137 struct ExpTgt {
1138   StringLiteral Name;
1139   unsigned Tgt;
1140   unsigned MaxIndex;
1141 };
1142 
1143 static constexpr ExpTgt ExpTgtInfo[] = {
1144   {{"null"},  ET_NULL,   ET_NULL_MAX_IDX},
1145   {{"mrtz"},  ET_MRTZ,   ET_MRTZ_MAX_IDX},
1146   {{"prim"},  ET_PRIM,   ET_PRIM_MAX_IDX},
1147   {{"mrt"},   ET_MRT0,   ET_MRT_MAX_IDX},
1148   {{"pos"},   ET_POS0,   ET_POS_MAX_IDX},
1149   {{"param"}, ET_PARAM0, ET_PARAM_MAX_IDX},
1150 };
1151 
1152 bool getTgtName(unsigned Id, StringRef &Name, int &Index) {
1153   for (const ExpTgt &Val : ExpTgtInfo) {
1154     if (Val.Tgt <= Id && Id <= Val.Tgt + Val.MaxIndex) {
1155       Index = (Val.MaxIndex == 0) ? -1 : (Id - Val.Tgt);
1156       Name = Val.Name;
1157       return true;
1158     }
1159   }
1160   return false;
1161 }
1162 
1163 unsigned getTgtId(const StringRef Name) {
1164 
1165   for (const ExpTgt &Val : ExpTgtInfo) {
1166     if (Val.MaxIndex == 0 && Name == Val.Name)
1167       return Val.Tgt;
1168 
1169     if (Val.MaxIndex > 0 && Name.startswith(Val.Name)) {
1170       StringRef Suffix = Name.drop_front(Val.Name.size());
1171 
1172       unsigned Id;
1173       if (Suffix.getAsInteger(10, Id) || Id > Val.MaxIndex)
1174         return ET_INVALID;
1175 
1176       // Disable leading zeroes
1177       if (Suffix.size() > 1 && Suffix[0] == '0')
1178         return ET_INVALID;
1179 
1180       return Val.Tgt + Id;
1181     }
1182   }
1183   return ET_INVALID;
1184 }
1185 
1186 bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI) {
1187   return (Id != ET_POS4 && Id != ET_PRIM) || isGFX10Plus(STI);
1188 }
1189 
1190 } // namespace Exp
1191 
1192 //===----------------------------------------------------------------------===//
1193 // MTBUF Format
1194 //===----------------------------------------------------------------------===//
1195 
1196 namespace MTBUFFormat {
1197 
1198 int64_t getDfmt(const StringRef Name) {
1199   for (int Id = DFMT_MIN; Id <= DFMT_MAX; ++Id) {
1200     if (Name == DfmtSymbolic[Id])
1201       return Id;
1202   }
1203   return DFMT_UNDEF;
1204 }
1205 
1206 StringRef getDfmtName(unsigned Id) {
1207   assert(Id <= DFMT_MAX);
1208   return DfmtSymbolic[Id];
1209 }
1210 
1211 static StringLiteral const *getNfmtLookupTable(const MCSubtargetInfo &STI) {
1212   if (isSI(STI) || isCI(STI))
1213     return NfmtSymbolicSICI;
1214   if (isVI(STI) || isGFX9(STI))
1215     return NfmtSymbolicVI;
1216   return NfmtSymbolicGFX10;
1217 }
1218 
1219 int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI) {
1220   auto lookupTable = getNfmtLookupTable(STI);
1221   for (int Id = NFMT_MIN; Id <= NFMT_MAX; ++Id) {
1222     if (Name == lookupTable[Id])
1223       return Id;
1224   }
1225   return NFMT_UNDEF;
1226 }
1227 
1228 StringRef getNfmtName(unsigned Id, const MCSubtargetInfo &STI) {
1229   assert(Id <= NFMT_MAX);
1230   return getNfmtLookupTable(STI)[Id];
1231 }
1232 
1233 bool isValidDfmtNfmt(unsigned Id, const MCSubtargetInfo &STI) {
1234   unsigned Dfmt;
1235   unsigned Nfmt;
1236   decodeDfmtNfmt(Id, Dfmt, Nfmt);
1237   return isValidNfmt(Nfmt, STI);
1238 }
1239 
1240 bool isValidNfmt(unsigned Id, const MCSubtargetInfo &STI) {
1241   return !getNfmtName(Id, STI).empty();
1242 }
1243 
1244 int64_t encodeDfmtNfmt(unsigned Dfmt, unsigned Nfmt) {
1245   return (Dfmt << DFMT_SHIFT) | (Nfmt << NFMT_SHIFT);
1246 }
1247 
1248 void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt) {
1249   Dfmt = (Format >> DFMT_SHIFT) & DFMT_MASK;
1250   Nfmt = (Format >> NFMT_SHIFT) & NFMT_MASK;
1251 }
1252 
1253 int64_t getUnifiedFormat(const StringRef Name) {
1254   for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) {
1255     if (Name == UfmtSymbolic[Id])
1256       return Id;
1257   }
1258   return UFMT_UNDEF;
1259 }
1260 
1261 StringRef getUnifiedFormatName(unsigned Id) {
1262   return isValidUnifiedFormat(Id) ? UfmtSymbolic[Id] : "";
1263 }
1264 
1265 bool isValidUnifiedFormat(unsigned Id) {
1266   return Id <= UFMT_LAST;
1267 }
1268 
1269 int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt) {
1270   int64_t Fmt = encodeDfmtNfmt(Dfmt, Nfmt);
1271   for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) {
1272     if (Fmt == DfmtNfmt2UFmt[Id])
1273       return Id;
1274   }
1275   return UFMT_UNDEF;
1276 }
1277 
1278 bool isValidFormatEncoding(unsigned Val, const MCSubtargetInfo &STI) {
1279   return isGFX10Plus(STI) ? (Val <= UFMT_MAX) : (Val <= DFMT_NFMT_MAX);
1280 }
1281 
1282 unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI) {
1283   if (isGFX10Plus(STI))
1284     return UFMT_DEFAULT;
1285   return DFMT_NFMT_DEFAULT;
1286 }
1287 
1288 } // namespace MTBUFFormat
1289 
1290 //===----------------------------------------------------------------------===//
1291 // SendMsg
1292 //===----------------------------------------------------------------------===//
1293 
1294 namespace SendMsg {
1295 
1296 int64_t getMsgId(const StringRef Name, const MCSubtargetInfo &STI) {
1297   int Idx = getOprIdx<const MCSubtargetInfo &>(Name, Msg, MSG_SIZE, STI);
1298   return (Idx < 0) ? Idx : Msg[Idx].Encoding;
1299 }
1300 
1301 bool isValidMsgId(int64_t MsgId) {
1302   return 0 <= MsgId && isUInt<ID_WIDTH_>(MsgId);
1303 }
1304 
1305 StringRef getMsgName(int64_t MsgId, const MCSubtargetInfo &STI) {
1306   int Idx = getOprIdx<const MCSubtargetInfo &>(MsgId, Msg, MSG_SIZE, STI);
1307   return (Idx < 0) ? "" : Msg[Idx].Name;
1308 }
1309 
1310 int64_t getMsgOpId(int64_t MsgId, const StringRef Name) {
1311   const char* const *S = (MsgId == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
1312   const int F = (MsgId == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
1313   const int L = (MsgId == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
1314   for (int i = F; i < L; ++i) {
1315     if (Name == S[i]) {
1316       return i;
1317     }
1318   }
1319   return OP_UNKNOWN_;
1320 }
1321 
1322 bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI,
1323                   bool Strict) {
1324   assert(isValidMsgId(MsgId));
1325 
1326   if (!Strict)
1327     return 0 <= OpId && isUInt<OP_WIDTH_>(OpId);
1328 
1329   switch(MsgId)
1330   {
1331   case ID_GS:
1332     return (OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_) && OpId != OP_GS_NOP;
1333   case ID_GS_DONE:
1334     return OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_;
1335   case ID_SYSMSG:
1336     return OP_SYS_FIRST_ <= OpId && OpId < OP_SYS_LAST_;
1337   default:
1338     return OpId == OP_NONE_;
1339   }
1340 }
1341 
1342 StringRef getMsgOpName(int64_t MsgId, int64_t OpId) {
1343   assert(msgRequiresOp(MsgId));
1344   return (MsgId == ID_SYSMSG)? OpSysSymbolic[OpId] : OpGsSymbolic[OpId];
1345 }
1346 
1347 bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId,
1348                       const MCSubtargetInfo &STI, bool Strict) {
1349   assert(isValidMsgOp(MsgId, OpId, STI, Strict));
1350 
1351   if (!Strict)
1352     return 0 <= StreamId && isUInt<STREAM_ID_WIDTH_>(StreamId);
1353 
1354   switch(MsgId)
1355   {
1356   case ID_GS:
1357     return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_;
1358   case ID_GS_DONE:
1359     return (OpId == OP_GS_NOP)?
1360            (StreamId == STREAM_ID_NONE_) :
1361            (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_);
1362   default:
1363     return StreamId == STREAM_ID_NONE_;
1364   }
1365 }
1366 
1367 bool msgRequiresOp(int64_t MsgId) {
1368   return MsgId == ID_GS || MsgId == ID_GS_DONE || MsgId == ID_SYSMSG;
1369 }
1370 
1371 bool msgSupportsStream(int64_t MsgId, int64_t OpId) {
1372   return (MsgId == ID_GS || MsgId == ID_GS_DONE) && OpId != OP_GS_NOP;
1373 }
1374 
1375 void decodeMsg(unsigned Val,
1376                uint16_t &MsgId,
1377                uint16_t &OpId,
1378                uint16_t &StreamId) {
1379   MsgId = Val & ID_MASK_;
1380   OpId = (Val & OP_MASK_) >> OP_SHIFT_;
1381   StreamId = (Val & STREAM_ID_MASK_) >> STREAM_ID_SHIFT_;
1382 }
1383 
1384 uint64_t encodeMsg(uint64_t MsgId,
1385                    uint64_t OpId,
1386                    uint64_t StreamId) {
1387   return (MsgId << ID_SHIFT_) |
1388          (OpId << OP_SHIFT_) |
1389          (StreamId << STREAM_ID_SHIFT_);
1390 }
1391 
1392 } // namespace SendMsg
1393 
1394 //===----------------------------------------------------------------------===//
1395 //
1396 //===----------------------------------------------------------------------===//
1397 
1398 unsigned getInitialPSInputAddr(const Function &F) {
1399   return getIntegerAttribute(F, "InitialPSInputAddr", 0);
1400 }
1401 
1402 bool getHasColorExport(const Function &F) {
1403   // As a safe default always respond as if PS has color exports.
1404   return getIntegerAttribute(
1405              F, "amdgpu-color-export",
1406              F.getCallingConv() == CallingConv::AMDGPU_PS ? 1 : 0) != 0;
1407 }
1408 
1409 bool getHasDepthExport(const Function &F) {
1410   return getIntegerAttribute(F, "amdgpu-depth-export", 0) != 0;
1411 }
1412 
1413 bool isShader(CallingConv::ID cc) {
1414   switch(cc) {
1415     case CallingConv::AMDGPU_VS:
1416     case CallingConv::AMDGPU_LS:
1417     case CallingConv::AMDGPU_HS:
1418     case CallingConv::AMDGPU_ES:
1419     case CallingConv::AMDGPU_GS:
1420     case CallingConv::AMDGPU_PS:
1421     case CallingConv::AMDGPU_CS:
1422       return true;
1423     default:
1424       return false;
1425   }
1426 }
1427 
1428 bool isGraphics(CallingConv::ID cc) {
1429   return isShader(cc) || cc == CallingConv::AMDGPU_Gfx;
1430 }
1431 
1432 bool isCompute(CallingConv::ID cc) {
1433   return !isGraphics(cc) || cc == CallingConv::AMDGPU_CS;
1434 }
1435 
1436 bool isEntryFunctionCC(CallingConv::ID CC) {
1437   switch (CC) {
1438   case CallingConv::AMDGPU_KERNEL:
1439   case CallingConv::SPIR_KERNEL:
1440   case CallingConv::AMDGPU_VS:
1441   case CallingConv::AMDGPU_GS:
1442   case CallingConv::AMDGPU_PS:
1443   case CallingConv::AMDGPU_CS:
1444   case CallingConv::AMDGPU_ES:
1445   case CallingConv::AMDGPU_HS:
1446   case CallingConv::AMDGPU_LS:
1447     return true;
1448   default:
1449     return false;
1450   }
1451 }
1452 
1453 bool isModuleEntryFunctionCC(CallingConv::ID CC) {
1454   switch (CC) {
1455   case CallingConv::AMDGPU_Gfx:
1456     return true;
1457   default:
1458     return isEntryFunctionCC(CC);
1459   }
1460 }
1461 
1462 bool isKernelCC(const Function *Func) {
1463   return AMDGPU::isModuleEntryFunctionCC(Func->getCallingConv());
1464 }
1465 
1466 bool hasXNACK(const MCSubtargetInfo &STI) {
1467   return STI.getFeatureBits()[AMDGPU::FeatureXNACK];
1468 }
1469 
1470 bool hasSRAMECC(const MCSubtargetInfo &STI) {
1471   return STI.getFeatureBits()[AMDGPU::FeatureSRAMECC];
1472 }
1473 
1474 bool hasMIMG_R128(const MCSubtargetInfo &STI) {
1475   return STI.getFeatureBits()[AMDGPU::FeatureMIMG_R128] && !STI.getFeatureBits()[AMDGPU::FeatureR128A16];
1476 }
1477 
1478 bool hasGFX10A16(const MCSubtargetInfo &STI) {
1479   return STI.getFeatureBits()[AMDGPU::FeatureGFX10A16];
1480 }
1481 
1482 bool hasG16(const MCSubtargetInfo &STI) {
1483   return STI.getFeatureBits()[AMDGPU::FeatureG16];
1484 }
1485 
1486 bool hasPackedD16(const MCSubtargetInfo &STI) {
1487   return !STI.getFeatureBits()[AMDGPU::FeatureUnpackedD16VMem];
1488 }
1489 
1490 bool isSI(const MCSubtargetInfo &STI) {
1491   return STI.getFeatureBits()[AMDGPU::FeatureSouthernIslands];
1492 }
1493 
1494 bool isCI(const MCSubtargetInfo &STI) {
1495   return STI.getFeatureBits()[AMDGPU::FeatureSeaIslands];
1496 }
1497 
1498 bool isVI(const MCSubtargetInfo &STI) {
1499   return STI.getFeatureBits()[AMDGPU::FeatureVolcanicIslands];
1500 }
1501 
1502 bool isGFX9(const MCSubtargetInfo &STI) {
1503   return STI.getFeatureBits()[AMDGPU::FeatureGFX9];
1504 }
1505 
1506 bool isGFX9_GFX10(const MCSubtargetInfo &STI) {
1507   return isGFX9(STI) || isGFX10(STI);
1508 }
1509 
1510 bool isGFX8Plus(const MCSubtargetInfo &STI) {
1511   return isVI(STI) || isGFX9Plus(STI);
1512 }
1513 
1514 bool isGFX9Plus(const MCSubtargetInfo &STI) {
1515   return isGFX9(STI) || isGFX10Plus(STI);
1516 }
1517 
1518 bool isGFX10(const MCSubtargetInfo &STI) {
1519   return STI.getFeatureBits()[AMDGPU::FeatureGFX10];
1520 }
1521 
1522 bool isGFX10Plus(const MCSubtargetInfo &STI) { return isGFX10(STI); }
1523 
1524 bool isNotGFX10Plus(const MCSubtargetInfo &STI) {
1525   return isSI(STI) || isCI(STI) || isVI(STI) || isGFX9(STI);
1526 }
1527 
1528 bool isGFX10Before1030(const MCSubtargetInfo &STI) {
1529   return isGFX10(STI) && !AMDGPU::isGFX10_BEncoding(STI);
1530 }
1531 
1532 bool isGCN3Encoding(const MCSubtargetInfo &STI) {
1533   return STI.getFeatureBits()[AMDGPU::FeatureGCN3Encoding];
1534 }
1535 
1536 bool isGFX10_AEncoding(const MCSubtargetInfo &STI) {
1537   return STI.getFeatureBits()[AMDGPU::FeatureGFX10_AEncoding];
1538 }
1539 
1540 bool isGFX10_BEncoding(const MCSubtargetInfo &STI) {
1541   return STI.getFeatureBits()[AMDGPU::FeatureGFX10_BEncoding];
1542 }
1543 
1544 bool hasGFX10_3Insts(const MCSubtargetInfo &STI) {
1545   return STI.getFeatureBits()[AMDGPU::FeatureGFX10_3Insts];
1546 }
1547 
1548 bool isGFX90A(const MCSubtargetInfo &STI) {
1549   return STI.getFeatureBits()[AMDGPU::FeatureGFX90AInsts];
1550 }
1551 
1552 bool isGFX940(const MCSubtargetInfo &STI) {
1553   return STI.getFeatureBits()[AMDGPU::FeatureGFX940Insts];
1554 }
1555 
1556 bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI) {
1557   return STI.getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch];
1558 }
1559 
1560 bool hasMAIInsts(const MCSubtargetInfo &STI) {
1561   return STI.getFeatureBits()[AMDGPU::FeatureMAIInsts];
1562 }
1563 
1564 int32_t getTotalNumVGPRs(bool has90AInsts, int32_t ArgNumAGPR,
1565                          int32_t ArgNumVGPR) {
1566   if (has90AInsts && ArgNumAGPR)
1567     return alignTo(ArgNumVGPR, 4) + ArgNumAGPR;
1568   return std::max(ArgNumVGPR, ArgNumAGPR);
1569 }
1570 
1571 bool isSGPR(unsigned Reg, const MCRegisterInfo* TRI) {
1572   const MCRegisterClass SGPRClass = TRI->getRegClass(AMDGPU::SReg_32RegClassID);
1573   const unsigned FirstSubReg = TRI->getSubReg(Reg, AMDGPU::sub0);
1574   return SGPRClass.contains(FirstSubReg != 0 ? FirstSubReg : Reg) ||
1575     Reg == AMDGPU::SCC;
1576 }
1577 
1578 #define MAP_REG2REG \
1579   using namespace AMDGPU; \
1580   switch(Reg) { \
1581   default: return Reg; \
1582   CASE_CI_VI(FLAT_SCR) \
1583   CASE_CI_VI(FLAT_SCR_LO) \
1584   CASE_CI_VI(FLAT_SCR_HI) \
1585   CASE_VI_GFX9PLUS(TTMP0) \
1586   CASE_VI_GFX9PLUS(TTMP1) \
1587   CASE_VI_GFX9PLUS(TTMP2) \
1588   CASE_VI_GFX9PLUS(TTMP3) \
1589   CASE_VI_GFX9PLUS(TTMP4) \
1590   CASE_VI_GFX9PLUS(TTMP5) \
1591   CASE_VI_GFX9PLUS(TTMP6) \
1592   CASE_VI_GFX9PLUS(TTMP7) \
1593   CASE_VI_GFX9PLUS(TTMP8) \
1594   CASE_VI_GFX9PLUS(TTMP9) \
1595   CASE_VI_GFX9PLUS(TTMP10) \
1596   CASE_VI_GFX9PLUS(TTMP11) \
1597   CASE_VI_GFX9PLUS(TTMP12) \
1598   CASE_VI_GFX9PLUS(TTMP13) \
1599   CASE_VI_GFX9PLUS(TTMP14) \
1600   CASE_VI_GFX9PLUS(TTMP15) \
1601   CASE_VI_GFX9PLUS(TTMP0_TTMP1) \
1602   CASE_VI_GFX9PLUS(TTMP2_TTMP3) \
1603   CASE_VI_GFX9PLUS(TTMP4_TTMP5) \
1604   CASE_VI_GFX9PLUS(TTMP6_TTMP7) \
1605   CASE_VI_GFX9PLUS(TTMP8_TTMP9) \
1606   CASE_VI_GFX9PLUS(TTMP10_TTMP11) \
1607   CASE_VI_GFX9PLUS(TTMP12_TTMP13) \
1608   CASE_VI_GFX9PLUS(TTMP14_TTMP15) \
1609   CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3) \
1610   CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7) \
1611   CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11) \
1612   CASE_VI_GFX9PLUS(TTMP12_TTMP13_TTMP14_TTMP15) \
1613   CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7) \
1614   CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11) \
1615   CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
1616   CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
1617   }
1618 
1619 #define CASE_CI_VI(node) \
1620   assert(!isSI(STI)); \
1621   case node: return isCI(STI) ? node##_ci : node##_vi;
1622 
1623 #define CASE_VI_GFX9PLUS(node) \
1624   case node: return isGFX9Plus(STI) ? node##_gfx9plus : node##_vi;
1625 
1626 unsigned getMCReg(unsigned Reg, const MCSubtargetInfo &STI) {
1627   if (STI.getTargetTriple().getArch() == Triple::r600)
1628     return Reg;
1629   MAP_REG2REG
1630 }
1631 
1632 #undef CASE_CI_VI
1633 #undef CASE_VI_GFX9PLUS
1634 
1635 #define CASE_CI_VI(node)   case node##_ci: case node##_vi:   return node;
1636 #define CASE_VI_GFX9PLUS(node) case node##_vi: case node##_gfx9plus: return node;
1637 
1638 unsigned mc2PseudoReg(unsigned Reg) {
1639   MAP_REG2REG
1640 }
1641 
1642 #undef CASE_CI_VI
1643 #undef CASE_VI_GFX9PLUS
1644 #undef MAP_REG2REG
1645 
1646 bool isSISrcOperand(const MCInstrDesc &Desc, unsigned OpNo) {
1647   assert(OpNo < Desc.NumOperands);
1648   unsigned OpType = Desc.OpInfo[OpNo].OperandType;
1649   return OpType >= AMDGPU::OPERAND_SRC_FIRST &&
1650          OpType <= AMDGPU::OPERAND_SRC_LAST;
1651 }
1652 
1653 bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo) {
1654   assert(OpNo < Desc.NumOperands);
1655   unsigned OpType = Desc.OpInfo[OpNo].OperandType;
1656   switch (OpType) {
1657   case AMDGPU::OPERAND_REG_IMM_FP32:
1658   case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED:
1659   case AMDGPU::OPERAND_REG_IMM_FP64:
1660   case AMDGPU::OPERAND_REG_IMM_FP16:
1661   case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED:
1662   case AMDGPU::OPERAND_REG_IMM_V2FP16:
1663   case AMDGPU::OPERAND_REG_IMM_V2INT16:
1664   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1665   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1666   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1667   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1668   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1669   case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
1670   case AMDGPU::OPERAND_REG_INLINE_AC_FP16:
1671   case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16:
1672   case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16:
1673   case AMDGPU::OPERAND_REG_IMM_V2FP32:
1674   case AMDGPU::OPERAND_REG_INLINE_C_V2FP32:
1675   case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
1676     return true;
1677   default:
1678     return false;
1679   }
1680 }
1681 
1682 bool isSISrcInlinableOperand(const MCInstrDesc &Desc, unsigned OpNo) {
1683   assert(OpNo < Desc.NumOperands);
1684   unsigned OpType = Desc.OpInfo[OpNo].OperandType;
1685   return OpType >= AMDGPU::OPERAND_REG_INLINE_C_FIRST &&
1686          OpType <= AMDGPU::OPERAND_REG_INLINE_C_LAST;
1687 }
1688 
1689 // Avoid using MCRegisterClass::getSize, since that function will go away
1690 // (move from MC* level to Target* level). Return size in bits.
1691 unsigned getRegBitWidth(unsigned RCID) {
1692   switch (RCID) {
1693   case AMDGPU::VGPR_LO16RegClassID:
1694   case AMDGPU::VGPR_HI16RegClassID:
1695   case AMDGPU::SGPR_LO16RegClassID:
1696   case AMDGPU::AGPR_LO16RegClassID:
1697     return 16;
1698   case AMDGPU::SGPR_32RegClassID:
1699   case AMDGPU::VGPR_32RegClassID:
1700   case AMDGPU::VRegOrLds_32RegClassID:
1701   case AMDGPU::AGPR_32RegClassID:
1702   case AMDGPU::VS_32RegClassID:
1703   case AMDGPU::AV_32RegClassID:
1704   case AMDGPU::SReg_32RegClassID:
1705   case AMDGPU::SReg_32_XM0RegClassID:
1706   case AMDGPU::SRegOrLds_32RegClassID:
1707     return 32;
1708   case AMDGPU::SGPR_64RegClassID:
1709   case AMDGPU::VS_64RegClassID:
1710   case AMDGPU::SReg_64RegClassID:
1711   case AMDGPU::VReg_64RegClassID:
1712   case AMDGPU::AReg_64RegClassID:
1713   case AMDGPU::SReg_64_XEXECRegClassID:
1714   case AMDGPU::VReg_64_Align2RegClassID:
1715   case AMDGPU::AReg_64_Align2RegClassID:
1716   case AMDGPU::AV_64RegClassID:
1717   case AMDGPU::AV_64_Align2RegClassID:
1718     return 64;
1719   case AMDGPU::SGPR_96RegClassID:
1720   case AMDGPU::SReg_96RegClassID:
1721   case AMDGPU::VReg_96RegClassID:
1722   case AMDGPU::AReg_96RegClassID:
1723   case AMDGPU::VReg_96_Align2RegClassID:
1724   case AMDGPU::AReg_96_Align2RegClassID:
1725   case AMDGPU::AV_96RegClassID:
1726   case AMDGPU::AV_96_Align2RegClassID:
1727     return 96;
1728   case AMDGPU::SGPR_128RegClassID:
1729   case AMDGPU::SReg_128RegClassID:
1730   case AMDGPU::VReg_128RegClassID:
1731   case AMDGPU::AReg_128RegClassID:
1732   case AMDGPU::VReg_128_Align2RegClassID:
1733   case AMDGPU::AReg_128_Align2RegClassID:
1734   case AMDGPU::AV_128RegClassID:
1735   case AMDGPU::AV_128_Align2RegClassID:
1736     return 128;
1737   case AMDGPU::SGPR_160RegClassID:
1738   case AMDGPU::SReg_160RegClassID:
1739   case AMDGPU::VReg_160RegClassID:
1740   case AMDGPU::AReg_160RegClassID:
1741   case AMDGPU::VReg_160_Align2RegClassID:
1742   case AMDGPU::AReg_160_Align2RegClassID:
1743   case AMDGPU::AV_160RegClassID:
1744   case AMDGPU::AV_160_Align2RegClassID:
1745     return 160;
1746   case AMDGPU::SGPR_192RegClassID:
1747   case AMDGPU::SReg_192RegClassID:
1748   case AMDGPU::VReg_192RegClassID:
1749   case AMDGPU::AReg_192RegClassID:
1750   case AMDGPU::VReg_192_Align2RegClassID:
1751   case AMDGPU::AReg_192_Align2RegClassID:
1752   case AMDGPU::AV_192RegClassID:
1753   case AMDGPU::AV_192_Align2RegClassID:
1754     return 192;
1755   case AMDGPU::SGPR_224RegClassID:
1756   case AMDGPU::SReg_224RegClassID:
1757   case AMDGPU::VReg_224RegClassID:
1758   case AMDGPU::AReg_224RegClassID:
1759   case AMDGPU::VReg_224_Align2RegClassID:
1760   case AMDGPU::AReg_224_Align2RegClassID:
1761   case AMDGPU::AV_224RegClassID:
1762   case AMDGPU::AV_224_Align2RegClassID:
1763     return 224;
1764   case AMDGPU::SGPR_256RegClassID:
1765   case AMDGPU::SReg_256RegClassID:
1766   case AMDGPU::VReg_256RegClassID:
1767   case AMDGPU::AReg_256RegClassID:
1768   case AMDGPU::VReg_256_Align2RegClassID:
1769   case AMDGPU::AReg_256_Align2RegClassID:
1770   case AMDGPU::AV_256RegClassID:
1771   case AMDGPU::AV_256_Align2RegClassID:
1772     return 256;
1773   case AMDGPU::SGPR_512RegClassID:
1774   case AMDGPU::SReg_512RegClassID:
1775   case AMDGPU::VReg_512RegClassID:
1776   case AMDGPU::AReg_512RegClassID:
1777   case AMDGPU::VReg_512_Align2RegClassID:
1778   case AMDGPU::AReg_512_Align2RegClassID:
1779   case AMDGPU::AV_512RegClassID:
1780   case AMDGPU::AV_512_Align2RegClassID:
1781     return 512;
1782   case AMDGPU::SGPR_1024RegClassID:
1783   case AMDGPU::SReg_1024RegClassID:
1784   case AMDGPU::VReg_1024RegClassID:
1785   case AMDGPU::AReg_1024RegClassID:
1786   case AMDGPU::VReg_1024_Align2RegClassID:
1787   case AMDGPU::AReg_1024_Align2RegClassID:
1788   case AMDGPU::AV_1024RegClassID:
1789   case AMDGPU::AV_1024_Align2RegClassID:
1790     return 1024;
1791   default:
1792     llvm_unreachable("Unexpected register class");
1793   }
1794 }
1795 
1796 unsigned getRegBitWidth(const MCRegisterClass &RC) {
1797   return getRegBitWidth(RC.getID());
1798 }
1799 
1800 unsigned getRegOperandSize(const MCRegisterInfo *MRI, const MCInstrDesc &Desc,
1801                            unsigned OpNo) {
1802   assert(OpNo < Desc.NumOperands);
1803   unsigned RCID = Desc.OpInfo[OpNo].RegClass;
1804   return getRegBitWidth(MRI->getRegClass(RCID)) / 8;
1805 }
1806 
1807 bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi) {
1808   if (isInlinableIntLiteral(Literal))
1809     return true;
1810 
1811   uint64_t Val = static_cast<uint64_t>(Literal);
1812   return (Val == DoubleToBits(0.0)) ||
1813          (Val == DoubleToBits(1.0)) ||
1814          (Val == DoubleToBits(-1.0)) ||
1815          (Val == DoubleToBits(0.5)) ||
1816          (Val == DoubleToBits(-0.5)) ||
1817          (Val == DoubleToBits(2.0)) ||
1818          (Val == DoubleToBits(-2.0)) ||
1819          (Val == DoubleToBits(4.0)) ||
1820          (Val == DoubleToBits(-4.0)) ||
1821          (Val == 0x3fc45f306dc9c882 && HasInv2Pi);
1822 }
1823 
1824 bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi) {
1825   if (isInlinableIntLiteral(Literal))
1826     return true;
1827 
1828   // The actual type of the operand does not seem to matter as long
1829   // as the bits match one of the inline immediate values.  For example:
1830   //
1831   // -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal,
1832   // so it is a legal inline immediate.
1833   //
1834   // 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in
1835   // floating-point, so it is a legal inline immediate.
1836 
1837   uint32_t Val = static_cast<uint32_t>(Literal);
1838   return (Val == FloatToBits(0.0f)) ||
1839          (Val == FloatToBits(1.0f)) ||
1840          (Val == FloatToBits(-1.0f)) ||
1841          (Val == FloatToBits(0.5f)) ||
1842          (Val == FloatToBits(-0.5f)) ||
1843          (Val == FloatToBits(2.0f)) ||
1844          (Val == FloatToBits(-2.0f)) ||
1845          (Val == FloatToBits(4.0f)) ||
1846          (Val == FloatToBits(-4.0f)) ||
1847          (Val == 0x3e22f983 && HasInv2Pi);
1848 }
1849 
1850 bool isInlinableLiteral16(int16_t Literal, bool HasInv2Pi) {
1851   if (!HasInv2Pi)
1852     return false;
1853 
1854   if (isInlinableIntLiteral(Literal))
1855     return true;
1856 
1857   uint16_t Val = static_cast<uint16_t>(Literal);
1858   return Val == 0x3C00 || // 1.0
1859          Val == 0xBC00 || // -1.0
1860          Val == 0x3800 || // 0.5
1861          Val == 0xB800 || // -0.5
1862          Val == 0x4000 || // 2.0
1863          Val == 0xC000 || // -2.0
1864          Val == 0x4400 || // 4.0
1865          Val == 0xC400 || // -4.0
1866          Val == 0x3118;   // 1/2pi
1867 }
1868 
1869 bool isInlinableLiteralV216(int32_t Literal, bool HasInv2Pi) {
1870   assert(HasInv2Pi);
1871 
1872   if (isInt<16>(Literal) || isUInt<16>(Literal)) {
1873     int16_t Trunc = static_cast<int16_t>(Literal);
1874     return AMDGPU::isInlinableLiteral16(Trunc, HasInv2Pi);
1875   }
1876   if (!(Literal & 0xffff))
1877     return AMDGPU::isInlinableLiteral16(Literal >> 16, HasInv2Pi);
1878 
1879   int16_t Lo16 = static_cast<int16_t>(Literal);
1880   int16_t Hi16 = static_cast<int16_t>(Literal >> 16);
1881   return Lo16 == Hi16 && isInlinableLiteral16(Lo16, HasInv2Pi);
1882 }
1883 
1884 bool isInlinableIntLiteralV216(int32_t Literal) {
1885   int16_t Lo16 = static_cast<int16_t>(Literal);
1886   if (isInt<16>(Literal) || isUInt<16>(Literal))
1887     return isInlinableIntLiteral(Lo16);
1888 
1889   int16_t Hi16 = static_cast<int16_t>(Literal >> 16);
1890   if (!(Literal & 0xffff))
1891     return isInlinableIntLiteral(Hi16);
1892   return Lo16 == Hi16 && isInlinableIntLiteral(Lo16);
1893 }
1894 
1895 bool isFoldableLiteralV216(int32_t Literal, bool HasInv2Pi) {
1896   assert(HasInv2Pi);
1897 
1898   int16_t Lo16 = static_cast<int16_t>(Literal);
1899   if (isInt<16>(Literal) || isUInt<16>(Literal))
1900     return true;
1901 
1902   int16_t Hi16 = static_cast<int16_t>(Literal >> 16);
1903   if (!(Literal & 0xffff))
1904     return true;
1905   return Lo16 == Hi16;
1906 }
1907 
1908 bool isArgPassedInSGPR(const Argument *A) {
1909   const Function *F = A->getParent();
1910 
1911   // Arguments to compute shaders are never a source of divergence.
1912   CallingConv::ID CC = F->getCallingConv();
1913   switch (CC) {
1914   case CallingConv::AMDGPU_KERNEL:
1915   case CallingConv::SPIR_KERNEL:
1916     return true;
1917   case CallingConv::AMDGPU_VS:
1918   case CallingConv::AMDGPU_LS:
1919   case CallingConv::AMDGPU_HS:
1920   case CallingConv::AMDGPU_ES:
1921   case CallingConv::AMDGPU_GS:
1922   case CallingConv::AMDGPU_PS:
1923   case CallingConv::AMDGPU_CS:
1924   case CallingConv::AMDGPU_Gfx:
1925     // For non-compute shaders, SGPR inputs are marked with either inreg or byval.
1926     // Everything else is in VGPRs.
1927     return F->getAttributes().hasParamAttr(A->getArgNo(), Attribute::InReg) ||
1928            F->getAttributes().hasParamAttr(A->getArgNo(), Attribute::ByVal);
1929   default:
1930     // TODO: Should calls support inreg for SGPR inputs?
1931     return false;
1932   }
1933 }
1934 
1935 static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) {
1936   return isGCN3Encoding(ST) || isGFX10Plus(ST);
1937 }
1938 
1939 static bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST) {
1940   return isGFX9Plus(ST);
1941 }
1942 
1943 bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST,
1944                                       int64_t EncodedOffset) {
1945   return hasSMEMByteOffset(ST) ? isUInt<20>(EncodedOffset)
1946                                : isUInt<8>(EncodedOffset);
1947 }
1948 
1949 bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST,
1950                                     int64_t EncodedOffset,
1951                                     bool IsBuffer) {
1952   return !IsBuffer &&
1953          hasSMRDSignedImmOffset(ST) &&
1954          isInt<21>(EncodedOffset);
1955 }
1956 
1957 static bool isDwordAligned(uint64_t ByteOffset) {
1958   return (ByteOffset & 3) == 0;
1959 }
1960 
1961 uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST,
1962                                 uint64_t ByteOffset) {
1963   if (hasSMEMByteOffset(ST))
1964     return ByteOffset;
1965 
1966   assert(isDwordAligned(ByteOffset));
1967   return ByteOffset >> 2;
1968 }
1969 
1970 Optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST,
1971                                        int64_t ByteOffset, bool IsBuffer) {
1972   // The signed version is always a byte offset.
1973   if (!IsBuffer && hasSMRDSignedImmOffset(ST)) {
1974     assert(hasSMEMByteOffset(ST));
1975     return isInt<20>(ByteOffset) ? Optional<int64_t>(ByteOffset) : None;
1976   }
1977 
1978   if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST))
1979     return None;
1980 
1981   int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
1982   return isLegalSMRDEncodedUnsignedOffset(ST, EncodedOffset)
1983              ? Optional<int64_t>(EncodedOffset)
1984              : None;
1985 }
1986 
1987 Optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST,
1988                                                 int64_t ByteOffset) {
1989   if (!isCI(ST) || !isDwordAligned(ByteOffset))
1990     return None;
1991 
1992   int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
1993   return isUInt<32>(EncodedOffset) ? Optional<int64_t>(EncodedOffset) : None;
1994 }
1995 
1996 unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST, bool Signed) {
1997   // Address offset is 12-bit signed for GFX10, 13-bit for GFX9.
1998   if (AMDGPU::isGFX10(ST))
1999     return Signed ? 12 : 11;
2000 
2001   return Signed ? 13 : 12;
2002 }
2003 
2004 // Given Imm, split it into the values to put into the SOffset and ImmOffset
2005 // fields in an MUBUF instruction. Return false if it is not possible (due to a
2006 // hardware bug needing a workaround).
2007 //
2008 // The required alignment ensures that individual address components remain
2009 // aligned if they are aligned to begin with. It also ensures that additional
2010 // offsets within the given alignment can be added to the resulting ImmOffset.
2011 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset,
2012                       const GCNSubtarget *Subtarget, Align Alignment) {
2013   const uint32_t MaxImm = alignDown(4095, Alignment.value());
2014   uint32_t Overflow = 0;
2015 
2016   if (Imm > MaxImm) {
2017     if (Imm <= MaxImm + 64) {
2018       // Use an SOffset inline constant for 4..64
2019       Overflow = Imm - MaxImm;
2020       Imm = MaxImm;
2021     } else {
2022       // Try to keep the same value in SOffset for adjacent loads, so that
2023       // the corresponding register contents can be re-used.
2024       //
2025       // Load values with all low-bits (except for alignment bits) set into
2026       // SOffset, so that a larger range of values can be covered using
2027       // s_movk_i32.
2028       //
2029       // Atomic operations fail to work correctly when individual address
2030       // components are unaligned, even if their sum is aligned.
2031       uint32_t High = (Imm + Alignment.value()) & ~4095;
2032       uint32_t Low = (Imm + Alignment.value()) & 4095;
2033       Imm = Low;
2034       Overflow = High - Alignment.value();
2035     }
2036   }
2037 
2038   // There is a hardware bug in SI and CI which prevents address clamping in
2039   // MUBUF instructions from working correctly with SOffsets. The immediate
2040   // offset is unaffected.
2041   if (Overflow > 0 &&
2042       Subtarget->getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS)
2043     return false;
2044 
2045   ImmOffset = Imm;
2046   SOffset = Overflow;
2047   return true;
2048 }
2049 
2050 SIModeRegisterDefaults::SIModeRegisterDefaults(const Function &F) {
2051   *this = getDefaultForCallingConv(F.getCallingConv());
2052 
2053   StringRef IEEEAttr = F.getFnAttribute("amdgpu-ieee").getValueAsString();
2054   if (!IEEEAttr.empty())
2055     IEEE = IEEEAttr == "true";
2056 
2057   StringRef DX10ClampAttr
2058     = F.getFnAttribute("amdgpu-dx10-clamp").getValueAsString();
2059   if (!DX10ClampAttr.empty())
2060     DX10Clamp = DX10ClampAttr == "true";
2061 
2062   StringRef DenormF32Attr = F.getFnAttribute("denormal-fp-math-f32").getValueAsString();
2063   if (!DenormF32Attr.empty()) {
2064     DenormalMode DenormMode = parseDenormalFPAttribute(DenormF32Attr);
2065     FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE;
2066     FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE;
2067   }
2068 
2069   StringRef DenormAttr = F.getFnAttribute("denormal-fp-math").getValueAsString();
2070   if (!DenormAttr.empty()) {
2071     DenormalMode DenormMode = parseDenormalFPAttribute(DenormAttr);
2072 
2073     if (DenormF32Attr.empty()) {
2074       FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE;
2075       FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE;
2076     }
2077 
2078     FP64FP16InputDenormals = DenormMode.Input == DenormalMode::IEEE;
2079     FP64FP16OutputDenormals = DenormMode.Output == DenormalMode::IEEE;
2080   }
2081 }
2082 
2083 namespace {
2084 
2085 struct SourceOfDivergence {
2086   unsigned Intr;
2087 };
2088 const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr);
2089 
2090 #define GET_SourcesOfDivergence_IMPL
2091 #define GET_Gfx9BufferFormat_IMPL
2092 #define GET_Gfx10PlusBufferFormat_IMPL
2093 #include "AMDGPUGenSearchableTables.inc"
2094 
2095 } // end anonymous namespace
2096 
2097 bool isIntrinsicSourceOfDivergence(unsigned IntrID) {
2098   return lookupSourceOfDivergence(IntrID);
2099 }
2100 
2101 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp,
2102                                                   uint8_t NumComponents,
2103                                                   uint8_t NumFormat,
2104                                                   const MCSubtargetInfo &STI) {
2105   return isGFX10Plus(STI)
2106              ? getGfx10PlusBufferFormatInfo(BitsPerComp, NumComponents,
2107                                             NumFormat)
2108              : getGfx9BufferFormatInfo(BitsPerComp, NumComponents, NumFormat);
2109 }
2110 
2111 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format,
2112                                                   const MCSubtargetInfo &STI) {
2113   return isGFX10Plus(STI) ? getGfx10PlusBufferFormatInfo(Format)
2114                           : getGfx9BufferFormatInfo(Format);
2115 }
2116 
2117 } // namespace AMDGPU
2118 
2119 raw_ostream &operator<<(raw_ostream &OS,
2120                         const AMDGPU::IsaInfo::TargetIDSetting S) {
2121   switch (S) {
2122   case (AMDGPU::IsaInfo::TargetIDSetting::Unsupported):
2123     OS << "Unsupported";
2124     break;
2125   case (AMDGPU::IsaInfo::TargetIDSetting::Any):
2126     OS << "Any";
2127     break;
2128   case (AMDGPU::IsaInfo::TargetIDSetting::Off):
2129     OS << "Off";
2130     break;
2131   case (AMDGPU::IsaInfo::TargetIDSetting::On):
2132     OS << "On";
2133     break;
2134   }
2135   return OS;
2136 }
2137 
2138 } // namespace llvm
2139