1 //===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "AMDGPUBaseInfo.h"
10 #include "AMDGPU.h"
11 #include "AMDGPUAsmUtils.h"
12 #include "AMDKernelCodeT.h"
13 #include "GCNSubtarget.h"
14 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
15 #include "llvm/BinaryFormat/ELF.h"
16 #include "llvm/IR/Attributes.h"
17 #include "llvm/IR/Function.h"
18 #include "llvm/IR/GlobalValue.h"
19 #include "llvm/IR/IntrinsicsAMDGPU.h"
20 #include "llvm/IR/IntrinsicsR600.h"
21 #include "llvm/IR/LLVMContext.h"
22 #include "llvm/MC/MCSubtargetInfo.h"
23 #include "llvm/Support/AMDHSAKernelDescriptor.h"
24 #include "llvm/Support/CommandLine.h"
25 #include "llvm/Support/TargetParser.h"
26 
27 #define GET_INSTRINFO_NAMED_OPS
28 #define GET_INSTRMAP_INFO
29 #include "AMDGPUGenInstrInfo.inc"
30 
31 static llvm::cl::opt<unsigned> AmdhsaCodeObjectVersion(
32   "amdhsa-code-object-version", llvm::cl::Hidden,
33   llvm::cl::desc("AMDHSA Code Object Version"), llvm::cl::init(4),
34   llvm::cl::ZeroOrMore);
35 
36 namespace {
37 
38 /// \returns Bit mask for given bit \p Shift and bit \p Width.
39 unsigned getBitMask(unsigned Shift, unsigned Width) {
40   return ((1 << Width) - 1) << Shift;
41 }
42 
43 /// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width.
44 ///
45 /// \returns Packed \p Dst.
46 unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) {
47   Dst &= ~(1 << Shift) & ~getBitMask(Shift, Width);
48   Dst |= (Src << Shift) & getBitMask(Shift, Width);
49   return Dst;
50 }
51 
52 /// Unpacks bits from \p Src for given bit \p Shift and bit \p Width.
53 ///
54 /// \returns Unpacked bits.
55 unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) {
56   return (Src & getBitMask(Shift, Width)) >> Shift;
57 }
58 
59 /// \returns Vmcnt bit shift (lower bits).
60 unsigned getVmcntBitShiftLo() { return 0; }
61 
62 /// \returns Vmcnt bit width (lower bits).
63 unsigned getVmcntBitWidthLo() { return 4; }
64 
65 /// \returns Expcnt bit shift.
66 unsigned getExpcntBitShift() { return 4; }
67 
68 /// \returns Expcnt bit width.
69 unsigned getExpcntBitWidth() { return 3; }
70 
71 /// \returns Lgkmcnt bit shift.
72 unsigned getLgkmcntBitShift() { return 8; }
73 
74 /// \returns Lgkmcnt bit width.
75 unsigned getLgkmcntBitWidth(unsigned VersionMajor) {
76   return (VersionMajor >= 10) ? 6 : 4;
77 }
78 
79 /// \returns Vmcnt bit shift (higher bits).
80 unsigned getVmcntBitShiftHi() { return 14; }
81 
82 /// \returns Vmcnt bit width (higher bits).
83 unsigned getVmcntBitWidthHi() { return 2; }
84 
85 } // end namespace anonymous
86 
87 namespace llvm {
88 
89 namespace AMDGPU {
90 
91 Optional<uint8_t> getHsaAbiVersion(const MCSubtargetInfo *STI) {
92   if (STI && STI->getTargetTriple().getOS() != Triple::AMDHSA)
93     return None;
94 
95   switch (AmdhsaCodeObjectVersion) {
96   case 2:
97     return ELF::ELFABIVERSION_AMDGPU_HSA_V2;
98   case 3:
99     return ELF::ELFABIVERSION_AMDGPU_HSA_V3;
100   case 4:
101     return ELF::ELFABIVERSION_AMDGPU_HSA_V4;
102   case 5:
103     return ELF::ELFABIVERSION_AMDGPU_HSA_V5;
104   default:
105     report_fatal_error(Twine("Unsupported AMDHSA Code Object Version ") +
106                        Twine(AmdhsaCodeObjectVersion));
107   }
108 }
109 
110 bool isHsaAbiVersion2(const MCSubtargetInfo *STI) {
111   if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI))
112     return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V2;
113   return false;
114 }
115 
116 bool isHsaAbiVersion3(const MCSubtargetInfo *STI) {
117   if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI))
118     return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V3;
119   return false;
120 }
121 
122 bool isHsaAbiVersion4(const MCSubtargetInfo *STI) {
123   if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI))
124     return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V4;
125   return false;
126 }
127 
128 bool isHsaAbiVersion5(const MCSubtargetInfo *STI) {
129   if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI))
130     return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V5;
131   return false;
132 }
133 
134 bool isHsaAbiVersion3AndAbove(const MCSubtargetInfo *STI) {
135   return isHsaAbiVersion3(STI) || isHsaAbiVersion4(STI) ||
136          isHsaAbiVersion5(STI);
137 }
138 
139 unsigned getAmdhsaCodeObjectVersion() {
140   return AmdhsaCodeObjectVersion;
141 }
142 
143 // FIXME: All such magic numbers about the ABI should be in a
144 // central TD file.
145 unsigned getHostcallImplicitArgPosition() {
146   switch (AmdhsaCodeObjectVersion) {
147   case 2:
148   case 3:
149   case 4:
150     return 24;
151   case 5:
152     return AMDGPU::ImplicitArg::HOSTCALL_PTR_OFFSET;
153   default:
154     llvm_unreachable("Unexpected code object version");
155     return 0;
156   }
157 }
158 
159 #define GET_MIMGBaseOpcodesTable_IMPL
160 #define GET_MIMGDimInfoTable_IMPL
161 #define GET_MIMGInfoTable_IMPL
162 #define GET_MIMGLZMappingTable_IMPL
163 #define GET_MIMGMIPMappingTable_IMPL
164 #define GET_MIMGBiasMappingTable_IMPL
165 #define GET_MIMGOffsetMappingTable_IMPL
166 #define GET_MIMGG16MappingTable_IMPL
167 #define GET_MAIInstInfoTable_IMPL
168 #include "AMDGPUGenSearchableTables.inc"
169 
170 int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding,
171                   unsigned VDataDwords, unsigned VAddrDwords) {
172   const MIMGInfo *Info = getMIMGOpcodeHelper(BaseOpcode, MIMGEncoding,
173                                              VDataDwords, VAddrDwords);
174   return Info ? Info->Opcode : -1;
175 }
176 
177 const MIMGBaseOpcodeInfo *getMIMGBaseOpcode(unsigned Opc) {
178   const MIMGInfo *Info = getMIMGInfo(Opc);
179   return Info ? getMIMGBaseOpcodeInfo(Info->BaseOpcode) : nullptr;
180 }
181 
182 int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) {
183   const MIMGInfo *OrigInfo = getMIMGInfo(Opc);
184   const MIMGInfo *NewInfo =
185       getMIMGOpcodeHelper(OrigInfo->BaseOpcode, OrigInfo->MIMGEncoding,
186                           NewChannels, OrigInfo->VAddrDwords);
187   return NewInfo ? NewInfo->Opcode : -1;
188 }
189 
190 unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode,
191                            const MIMGDimInfo *Dim, bool IsA16,
192                            bool IsG16Supported) {
193   unsigned AddrWords = BaseOpcode->NumExtraArgs;
194   unsigned AddrComponents = (BaseOpcode->Coordinates ? Dim->NumCoords : 0) +
195                             (BaseOpcode->LodOrClampOrMip ? 1 : 0);
196   if (IsA16)
197     AddrWords += divideCeil(AddrComponents, 2);
198   else
199     AddrWords += AddrComponents;
200 
201   // Note: For subtargets that support A16 but not G16, enabling A16 also
202   // enables 16 bit gradients.
203   // For subtargets that support A16 (operand) and G16 (done with a different
204   // instruction encoding), they are independent.
205 
206   if (BaseOpcode->Gradients) {
207     if ((IsA16 && !IsG16Supported) || BaseOpcode->G16)
208       // There are two gradients per coordinate, we pack them separately.
209       // For the 3d case,
210       // we get (dy/du, dx/du) (-, dz/du) (dy/dv, dx/dv) (-, dz/dv)
211       AddrWords += alignTo<2>(Dim->NumGradients / 2);
212     else
213       AddrWords += Dim->NumGradients;
214   }
215   return AddrWords;
216 }
217 
218 struct MUBUFInfo {
219   uint16_t Opcode;
220   uint16_t BaseOpcode;
221   uint8_t elements;
222   bool has_vaddr;
223   bool has_srsrc;
224   bool has_soffset;
225   bool IsBufferInv;
226 };
227 
228 struct MTBUFInfo {
229   uint16_t Opcode;
230   uint16_t BaseOpcode;
231   uint8_t elements;
232   bool has_vaddr;
233   bool has_srsrc;
234   bool has_soffset;
235 };
236 
237 struct SMInfo {
238   uint16_t Opcode;
239   bool IsBuffer;
240 };
241 
242 struct VOPInfo {
243   uint16_t Opcode;
244   bool IsSingle;
245 };
246 
247 #define GET_MTBUFInfoTable_DECL
248 #define GET_MTBUFInfoTable_IMPL
249 #define GET_MUBUFInfoTable_DECL
250 #define GET_MUBUFInfoTable_IMPL
251 #define GET_SMInfoTable_DECL
252 #define GET_SMInfoTable_IMPL
253 #define GET_VOP1InfoTable_DECL
254 #define GET_VOP1InfoTable_IMPL
255 #define GET_VOP2InfoTable_DECL
256 #define GET_VOP2InfoTable_IMPL
257 #define GET_VOP3InfoTable_DECL
258 #define GET_VOP3InfoTable_IMPL
259 #include "AMDGPUGenSearchableTables.inc"
260 
261 int getMTBUFBaseOpcode(unsigned Opc) {
262   const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opc);
263   return Info ? Info->BaseOpcode : -1;
264 }
265 
266 int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements) {
267   const MTBUFInfo *Info = getMTBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements);
268   return Info ? Info->Opcode : -1;
269 }
270 
271 int getMTBUFElements(unsigned Opc) {
272   const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
273   return Info ? Info->elements : 0;
274 }
275 
276 bool getMTBUFHasVAddr(unsigned Opc) {
277   const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
278   return Info ? Info->has_vaddr : false;
279 }
280 
281 bool getMTBUFHasSrsrc(unsigned Opc) {
282   const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
283   return Info ? Info->has_srsrc : false;
284 }
285 
286 bool getMTBUFHasSoffset(unsigned Opc) {
287   const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
288   return Info ? Info->has_soffset : false;
289 }
290 
291 int getMUBUFBaseOpcode(unsigned Opc) {
292   const MUBUFInfo *Info = getMUBUFInfoFromOpcode(Opc);
293   return Info ? Info->BaseOpcode : -1;
294 }
295 
296 int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements) {
297   const MUBUFInfo *Info = getMUBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements);
298   return Info ? Info->Opcode : -1;
299 }
300 
301 int getMUBUFElements(unsigned Opc) {
302   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
303   return Info ? Info->elements : 0;
304 }
305 
306 bool getMUBUFHasVAddr(unsigned Opc) {
307   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
308   return Info ? Info->has_vaddr : false;
309 }
310 
311 bool getMUBUFHasSrsrc(unsigned Opc) {
312   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
313   return Info ? Info->has_srsrc : false;
314 }
315 
316 bool getMUBUFHasSoffset(unsigned Opc) {
317   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
318   return Info ? Info->has_soffset : false;
319 }
320 
321 bool getMUBUFIsBufferInv(unsigned Opc) {
322   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
323   return Info ? Info->IsBufferInv : false;
324 }
325 
326 bool getSMEMIsBuffer(unsigned Opc) {
327   const SMInfo *Info = getSMEMOpcodeHelper(Opc);
328   return Info ? Info->IsBuffer : false;
329 }
330 
331 bool getVOP1IsSingle(unsigned Opc) {
332   const VOPInfo *Info = getVOP1OpcodeHelper(Opc);
333   return Info ? Info->IsSingle : false;
334 }
335 
336 bool getVOP2IsSingle(unsigned Opc) {
337   const VOPInfo *Info = getVOP2OpcodeHelper(Opc);
338   return Info ? Info->IsSingle : false;
339 }
340 
341 bool getVOP3IsSingle(unsigned Opc) {
342   const VOPInfo *Info = getVOP3OpcodeHelper(Opc);
343   return Info ? Info->IsSingle : false;
344 }
345 
346 bool getMAIIsDGEMM(unsigned Opc) {
347   const MAIInstInfo *Info = getMAIInstInfoHelper(Opc);
348   return Info ? Info->is_dgemm : false;
349 }
350 
351 bool getMAIIsGFX940XDL(unsigned Opc) {
352   const MAIInstInfo *Info = getMAIInstInfoHelper(Opc);
353   return Info ? Info->is_gfx940_xdl : false;
354 }
355 
356 // Wrapper for Tablegen'd function.  enum Subtarget is not defined in any
357 // header files, so we need to wrap it in a function that takes unsigned
358 // instead.
359 int getMCOpcode(uint16_t Opcode, unsigned Gen) {
360   return getMCOpcodeGen(Opcode, static_cast<Subtarget>(Gen));
361 }
362 
363 namespace IsaInfo {
364 
365 AMDGPUTargetID::AMDGPUTargetID(const MCSubtargetInfo &STI)
366     : STI(STI), XnackSetting(TargetIDSetting::Any),
367       SramEccSetting(TargetIDSetting::Any) {
368   if (!STI.getFeatureBits().test(FeatureSupportsXNACK))
369     XnackSetting = TargetIDSetting::Unsupported;
370   if (!STI.getFeatureBits().test(FeatureSupportsSRAMECC))
371     SramEccSetting = TargetIDSetting::Unsupported;
372 }
373 
374 void AMDGPUTargetID::setTargetIDFromFeaturesString(StringRef FS) {
375   // Check if xnack or sramecc is explicitly enabled or disabled.  In the
376   // absence of the target features we assume we must generate code that can run
377   // in any environment.
378   SubtargetFeatures Features(FS);
379   Optional<bool> XnackRequested;
380   Optional<bool> SramEccRequested;
381 
382   for (const std::string &Feature : Features.getFeatures()) {
383     if (Feature == "+xnack")
384       XnackRequested = true;
385     else if (Feature == "-xnack")
386       XnackRequested = false;
387     else if (Feature == "+sramecc")
388       SramEccRequested = true;
389     else if (Feature == "-sramecc")
390       SramEccRequested = false;
391   }
392 
393   bool XnackSupported = isXnackSupported();
394   bool SramEccSupported = isSramEccSupported();
395 
396   if (XnackRequested) {
397     if (XnackSupported) {
398       XnackSetting =
399           *XnackRequested ? TargetIDSetting::On : TargetIDSetting::Off;
400     } else {
401       // If a specific xnack setting was requested and this GPU does not support
402       // xnack emit a warning. Setting will remain set to "Unsupported".
403       if (*XnackRequested) {
404         errs() << "warning: xnack 'On' was requested for a processor that does "
405                   "not support it!\n";
406       } else {
407         errs() << "warning: xnack 'Off' was requested for a processor that "
408                   "does not support it!\n";
409       }
410     }
411   }
412 
413   if (SramEccRequested) {
414     if (SramEccSupported) {
415       SramEccSetting =
416           *SramEccRequested ? TargetIDSetting::On : TargetIDSetting::Off;
417     } else {
418       // If a specific sramecc setting was requested and this GPU does not
419       // support sramecc emit a warning. Setting will remain set to
420       // "Unsupported".
421       if (*SramEccRequested) {
422         errs() << "warning: sramecc 'On' was requested for a processor that "
423                   "does not support it!\n";
424       } else {
425         errs() << "warning: sramecc 'Off' was requested for a processor that "
426                   "does not support it!\n";
427       }
428     }
429   }
430 }
431 
432 static TargetIDSetting
433 getTargetIDSettingFromFeatureString(StringRef FeatureString) {
434   if (FeatureString.endswith("-"))
435     return TargetIDSetting::Off;
436   if (FeatureString.endswith("+"))
437     return TargetIDSetting::On;
438 
439   llvm_unreachable("Malformed feature string");
440 }
441 
442 void AMDGPUTargetID::setTargetIDFromTargetIDStream(StringRef TargetID) {
443   SmallVector<StringRef, 3> TargetIDSplit;
444   TargetID.split(TargetIDSplit, ':');
445 
446   for (const auto &FeatureString : TargetIDSplit) {
447     if (FeatureString.startswith("xnack"))
448       XnackSetting = getTargetIDSettingFromFeatureString(FeatureString);
449     if (FeatureString.startswith("sramecc"))
450       SramEccSetting = getTargetIDSettingFromFeatureString(FeatureString);
451   }
452 }
453 
454 std::string AMDGPUTargetID::toString() const {
455   std::string StringRep;
456   raw_string_ostream StreamRep(StringRep);
457 
458   auto TargetTriple = STI.getTargetTriple();
459   auto Version = getIsaVersion(STI.getCPU());
460 
461   StreamRep << TargetTriple.getArchName() << '-'
462             << TargetTriple.getVendorName() << '-'
463             << TargetTriple.getOSName() << '-'
464             << TargetTriple.getEnvironmentName() << '-';
465 
466   std::string Processor;
467   // TODO: Following else statement is present here because we used various
468   // alias names for GPUs up until GFX9 (e.g. 'fiji' is same as 'gfx803').
469   // Remove once all aliases are removed from GCNProcessors.td.
470   if (Version.Major >= 9)
471     Processor = STI.getCPU().str();
472   else
473     Processor = (Twine("gfx") + Twine(Version.Major) + Twine(Version.Minor) +
474                  Twine(Version.Stepping))
475                     .str();
476 
477   std::string Features;
478   if (Optional<uint8_t> HsaAbiVersion = getHsaAbiVersion(&STI)) {
479     switch (*HsaAbiVersion) {
480     case ELF::ELFABIVERSION_AMDGPU_HSA_V2:
481       // Code object V2 only supported specific processors and had fixed
482       // settings for the XNACK.
483       if (Processor == "gfx600") {
484       } else if (Processor == "gfx601") {
485       } else if (Processor == "gfx602") {
486       } else if (Processor == "gfx700") {
487       } else if (Processor == "gfx701") {
488       } else if (Processor == "gfx702") {
489       } else if (Processor == "gfx703") {
490       } else if (Processor == "gfx704") {
491       } else if (Processor == "gfx705") {
492       } else if (Processor == "gfx801") {
493         if (!isXnackOnOrAny())
494           report_fatal_error(
495               "AMD GPU code object V2 does not support processor " +
496               Twine(Processor) + " without XNACK");
497       } else if (Processor == "gfx802") {
498       } else if (Processor == "gfx803") {
499       } else if (Processor == "gfx805") {
500       } else if (Processor == "gfx810") {
501         if (!isXnackOnOrAny())
502           report_fatal_error(
503               "AMD GPU code object V2 does not support processor " +
504               Twine(Processor) + " without XNACK");
505       } else if (Processor == "gfx900") {
506         if (isXnackOnOrAny())
507           Processor = "gfx901";
508       } else if (Processor == "gfx902") {
509         if (isXnackOnOrAny())
510           Processor = "gfx903";
511       } else if (Processor == "gfx904") {
512         if (isXnackOnOrAny())
513           Processor = "gfx905";
514       } else if (Processor == "gfx906") {
515         if (isXnackOnOrAny())
516           Processor = "gfx907";
517       } else if (Processor == "gfx90c") {
518         if (isXnackOnOrAny())
519           report_fatal_error(
520               "AMD GPU code object V2 does not support processor " +
521               Twine(Processor) + " with XNACK being ON or ANY");
522       } else {
523         report_fatal_error(
524             "AMD GPU code object V2 does not support processor " +
525             Twine(Processor));
526       }
527       break;
528     case ELF::ELFABIVERSION_AMDGPU_HSA_V3:
529       // xnack.
530       if (isXnackOnOrAny())
531         Features += "+xnack";
532       // In code object v2 and v3, "sramecc" feature was spelled with a
533       // hyphen ("sram-ecc").
534       if (isSramEccOnOrAny())
535         Features += "+sram-ecc";
536       break;
537     case ELF::ELFABIVERSION_AMDGPU_HSA_V4:
538     case ELF::ELFABIVERSION_AMDGPU_HSA_V5:
539       // sramecc.
540       if (getSramEccSetting() == TargetIDSetting::Off)
541         Features += ":sramecc-";
542       else if (getSramEccSetting() == TargetIDSetting::On)
543         Features += ":sramecc+";
544       // xnack.
545       if (getXnackSetting() == TargetIDSetting::Off)
546         Features += ":xnack-";
547       else if (getXnackSetting() == TargetIDSetting::On)
548         Features += ":xnack+";
549       break;
550     default:
551       break;
552     }
553   }
554 
555   StreamRep << Processor << Features;
556 
557   StreamRep.flush();
558   return StringRep;
559 }
560 
561 unsigned getWavefrontSize(const MCSubtargetInfo *STI) {
562   if (STI->getFeatureBits().test(FeatureWavefrontSize16))
563     return 16;
564   if (STI->getFeatureBits().test(FeatureWavefrontSize32))
565     return 32;
566 
567   return 64;
568 }
569 
570 unsigned getLocalMemorySize(const MCSubtargetInfo *STI) {
571   if (STI->getFeatureBits().test(FeatureLocalMemorySize32768))
572     return 32768;
573   if (STI->getFeatureBits().test(FeatureLocalMemorySize65536))
574     return 65536;
575 
576   return 0;
577 }
578 
579 unsigned getEUsPerCU(const MCSubtargetInfo *STI) {
580   // "Per CU" really means "per whatever functional block the waves of a
581   // workgroup must share". For gfx10 in CU mode this is the CU, which contains
582   // two SIMDs.
583   if (isGFX10Plus(*STI) && STI->getFeatureBits().test(FeatureCuMode))
584     return 2;
585   // Pre-gfx10 a CU contains four SIMDs. For gfx10 in WGP mode the WGP contains
586   // two CUs, so a total of four SIMDs.
587   return 4;
588 }
589 
590 unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo *STI,
591                                unsigned FlatWorkGroupSize) {
592   assert(FlatWorkGroupSize != 0);
593   if (STI->getTargetTriple().getArch() != Triple::amdgcn)
594     return 8;
595   unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize);
596   if (N == 1)
597     return 40;
598   N = 40 / N;
599   return std::min(N, 16u);
600 }
601 
602 unsigned getMinWavesPerEU(const MCSubtargetInfo *STI) {
603   return 1;
604 }
605 
606 unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI) {
607   // FIXME: Need to take scratch memory into account.
608   if (isGFX90A(*STI))
609     return 8;
610   if (!isGFX10Plus(*STI))
611     return 10;
612   return hasGFX10_3Insts(*STI) ? 16 : 20;
613 }
614 
615 unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo *STI,
616                                    unsigned FlatWorkGroupSize) {
617   return divideCeil(getWavesPerWorkGroup(STI, FlatWorkGroupSize),
618                     getEUsPerCU(STI));
619 }
620 
621 unsigned getMinFlatWorkGroupSize(const MCSubtargetInfo *STI) {
622   return 1;
623 }
624 
625 unsigned getMaxFlatWorkGroupSize(const MCSubtargetInfo *STI) {
626   // Some subtargets allow encoding 2048, but this isn't tested or supported.
627   return 1024;
628 }
629 
630 unsigned getWavesPerWorkGroup(const MCSubtargetInfo *STI,
631                               unsigned FlatWorkGroupSize) {
632   return divideCeil(FlatWorkGroupSize, getWavefrontSize(STI));
633 }
634 
635 unsigned getSGPRAllocGranule(const MCSubtargetInfo *STI) {
636   IsaVersion Version = getIsaVersion(STI->getCPU());
637   if (Version.Major >= 10)
638     return getAddressableNumSGPRs(STI);
639   if (Version.Major >= 8)
640     return 16;
641   return 8;
642 }
643 
644 unsigned getSGPREncodingGranule(const MCSubtargetInfo *STI) {
645   return 8;
646 }
647 
648 unsigned getTotalNumSGPRs(const MCSubtargetInfo *STI) {
649   IsaVersion Version = getIsaVersion(STI->getCPU());
650   if (Version.Major >= 8)
651     return 800;
652   return 512;
653 }
654 
655 unsigned getAddressableNumSGPRs(const MCSubtargetInfo *STI) {
656   if (STI->getFeatureBits().test(FeatureSGPRInitBug))
657     return FIXED_NUM_SGPRS_FOR_INIT_BUG;
658 
659   IsaVersion Version = getIsaVersion(STI->getCPU());
660   if (Version.Major >= 10)
661     return 106;
662   if (Version.Major >= 8)
663     return 102;
664   return 104;
665 }
666 
667 unsigned getMinNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
668   assert(WavesPerEU != 0);
669 
670   IsaVersion Version = getIsaVersion(STI->getCPU());
671   if (Version.Major >= 10)
672     return 0;
673 
674   if (WavesPerEU >= getMaxWavesPerEU(STI))
675     return 0;
676 
677   unsigned MinNumSGPRs = getTotalNumSGPRs(STI) / (WavesPerEU + 1);
678   if (STI->getFeatureBits().test(FeatureTrapHandler))
679     MinNumSGPRs -= std::min(MinNumSGPRs, (unsigned)TRAP_NUM_SGPRS);
680   MinNumSGPRs = alignDown(MinNumSGPRs, getSGPRAllocGranule(STI)) + 1;
681   return std::min(MinNumSGPRs, getAddressableNumSGPRs(STI));
682 }
683 
684 unsigned getMaxNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU,
685                         bool Addressable) {
686   assert(WavesPerEU != 0);
687 
688   unsigned AddressableNumSGPRs = getAddressableNumSGPRs(STI);
689   IsaVersion Version = getIsaVersion(STI->getCPU());
690   if (Version.Major >= 10)
691     return Addressable ? AddressableNumSGPRs : 108;
692   if (Version.Major >= 8 && !Addressable)
693     AddressableNumSGPRs = 112;
694   unsigned MaxNumSGPRs = getTotalNumSGPRs(STI) / WavesPerEU;
695   if (STI->getFeatureBits().test(FeatureTrapHandler))
696     MaxNumSGPRs -= std::min(MaxNumSGPRs, (unsigned)TRAP_NUM_SGPRS);
697   MaxNumSGPRs = alignDown(MaxNumSGPRs, getSGPRAllocGranule(STI));
698   return std::min(MaxNumSGPRs, AddressableNumSGPRs);
699 }
700 
701 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed,
702                           bool FlatScrUsed, bool XNACKUsed) {
703   unsigned ExtraSGPRs = 0;
704   if (VCCUsed)
705     ExtraSGPRs = 2;
706 
707   IsaVersion Version = getIsaVersion(STI->getCPU());
708   if (Version.Major >= 10)
709     return ExtraSGPRs;
710 
711   if (Version.Major < 8) {
712     if (FlatScrUsed)
713       ExtraSGPRs = 4;
714   } else {
715     if (XNACKUsed)
716       ExtraSGPRs = 4;
717 
718     if (FlatScrUsed ||
719         STI->getFeatureBits().test(AMDGPU::FeatureArchitectedFlatScratch))
720       ExtraSGPRs = 6;
721   }
722 
723   return ExtraSGPRs;
724 }
725 
726 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed,
727                           bool FlatScrUsed) {
728   return getNumExtraSGPRs(STI, VCCUsed, FlatScrUsed,
729                           STI->getFeatureBits().test(AMDGPU::FeatureXNACK));
730 }
731 
732 unsigned getNumSGPRBlocks(const MCSubtargetInfo *STI, unsigned NumSGPRs) {
733   NumSGPRs = alignTo(std::max(1u, NumSGPRs), getSGPREncodingGranule(STI));
734   // SGPRBlocks is actual number of SGPR blocks minus 1.
735   return NumSGPRs / getSGPREncodingGranule(STI) - 1;
736 }
737 
738 unsigned getVGPRAllocGranule(const MCSubtargetInfo *STI,
739                              Optional<bool> EnableWavefrontSize32) {
740   if (STI->getFeatureBits().test(FeatureGFX90AInsts))
741     return 8;
742 
743   bool IsWave32 = EnableWavefrontSize32 ?
744       *EnableWavefrontSize32 :
745       STI->getFeatureBits().test(FeatureWavefrontSize32);
746 
747   if (hasGFX10_3Insts(*STI))
748     return IsWave32 ? 16 : 8;
749 
750   return IsWave32 ? 8 : 4;
751 }
752 
753 unsigned getVGPREncodingGranule(const MCSubtargetInfo *STI,
754                                 Optional<bool> EnableWavefrontSize32) {
755   if (STI->getFeatureBits().test(FeatureGFX90AInsts))
756     return 8;
757 
758   bool IsWave32 = EnableWavefrontSize32 ?
759       *EnableWavefrontSize32 :
760       STI->getFeatureBits().test(FeatureWavefrontSize32);
761 
762   return IsWave32 ? 8 : 4;
763 }
764 
765 unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) {
766   if (STI->getFeatureBits().test(FeatureGFX90AInsts))
767     return 512;
768   if (!isGFX10Plus(*STI))
769     return 256;
770   return STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1024 : 512;
771 }
772 
773 unsigned getAddressableNumVGPRs(const MCSubtargetInfo *STI) {
774   if (STI->getFeatureBits().test(FeatureGFX90AInsts))
775     return 512;
776   return 256;
777 }
778 
779 unsigned getMinNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
780   assert(WavesPerEU != 0);
781 
782   if (WavesPerEU >= getMaxWavesPerEU(STI))
783     return 0;
784   unsigned MinNumVGPRs =
785       alignDown(getTotalNumVGPRs(STI) / (WavesPerEU + 1),
786                 getVGPRAllocGranule(STI)) + 1;
787   return std::min(MinNumVGPRs, getAddressableNumVGPRs(STI));
788 }
789 
790 unsigned getMaxNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
791   assert(WavesPerEU != 0);
792 
793   unsigned MaxNumVGPRs = alignDown(getTotalNumVGPRs(STI) / WavesPerEU,
794                                    getVGPRAllocGranule(STI));
795   unsigned AddressableNumVGPRs = getAddressableNumVGPRs(STI);
796   return std::min(MaxNumVGPRs, AddressableNumVGPRs);
797 }
798 
799 unsigned getNumVGPRBlocks(const MCSubtargetInfo *STI, unsigned NumVGPRs,
800                           Optional<bool> EnableWavefrontSize32) {
801   NumVGPRs = alignTo(std::max(1u, NumVGPRs),
802                      getVGPREncodingGranule(STI, EnableWavefrontSize32));
803   // VGPRBlocks is actual number of VGPR blocks minus 1.
804   return NumVGPRs / getVGPREncodingGranule(STI, EnableWavefrontSize32) - 1;
805 }
806 
807 } // end namespace IsaInfo
808 
809 void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header,
810                                const MCSubtargetInfo *STI) {
811   IsaVersion Version = getIsaVersion(STI->getCPU());
812 
813   memset(&Header, 0, sizeof(Header));
814 
815   Header.amd_kernel_code_version_major = 1;
816   Header.amd_kernel_code_version_minor = 2;
817   Header.amd_machine_kind = 1; // AMD_MACHINE_KIND_AMDGPU
818   Header.amd_machine_version_major = Version.Major;
819   Header.amd_machine_version_minor = Version.Minor;
820   Header.amd_machine_version_stepping = Version.Stepping;
821   Header.kernel_code_entry_byte_offset = sizeof(Header);
822   Header.wavefront_size = 6;
823 
824   // If the code object does not support indirect functions, then the value must
825   // be 0xffffffff.
826   Header.call_convention = -1;
827 
828   // These alignment values are specified in powers of two, so alignment =
829   // 2^n.  The minimum alignment is 2^4 = 16.
830   Header.kernarg_segment_alignment = 4;
831   Header.group_segment_alignment = 4;
832   Header.private_segment_alignment = 4;
833 
834   if (Version.Major >= 10) {
835     if (STI->getFeatureBits().test(FeatureWavefrontSize32)) {
836       Header.wavefront_size = 5;
837       Header.code_properties |= AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32;
838     }
839     Header.compute_pgm_resource_registers |=
840       S_00B848_WGP_MODE(STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1) |
841       S_00B848_MEM_ORDERED(1);
842   }
843 }
844 
845 amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor(
846     const MCSubtargetInfo *STI) {
847   IsaVersion Version = getIsaVersion(STI->getCPU());
848 
849   amdhsa::kernel_descriptor_t KD;
850   memset(&KD, 0, sizeof(KD));
851 
852   AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
853                   amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64,
854                   amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE);
855   AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
856                   amdhsa::COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP, 1);
857   AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
858                   amdhsa::COMPUTE_PGM_RSRC1_ENABLE_IEEE_MODE, 1);
859   AMDHSA_BITS_SET(KD.compute_pgm_rsrc2,
860                   amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, 1);
861   if (Version.Major >= 10) {
862     AMDHSA_BITS_SET(KD.kernel_code_properties,
863                     amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32,
864                     STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1 : 0);
865     AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
866                     amdhsa::COMPUTE_PGM_RSRC1_WGP_MODE,
867                     STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1);
868     AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
869                     amdhsa::COMPUTE_PGM_RSRC1_MEM_ORDERED, 1);
870   }
871   if (AMDGPU::isGFX90A(*STI)) {
872     AMDHSA_BITS_SET(KD.compute_pgm_rsrc3,
873                     amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT,
874                     STI->getFeatureBits().test(FeatureTgSplit) ? 1 : 0);
875   }
876   return KD;
877 }
878 
879 bool isGroupSegment(const GlobalValue *GV) {
880   return GV->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS;
881 }
882 
883 bool isGlobalSegment(const GlobalValue *GV) {
884   return GV->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS;
885 }
886 
887 bool isReadOnlySegment(const GlobalValue *GV) {
888   unsigned AS = GV->getAddressSpace();
889   return AS == AMDGPUAS::CONSTANT_ADDRESS ||
890          AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
891 }
892 
893 bool shouldEmitConstantsToTextSection(const Triple &TT) {
894   return TT.getArch() == Triple::r600;
895 }
896 
897 int getIntegerAttribute(const Function &F, StringRef Name, int Default) {
898   Attribute A = F.getFnAttribute(Name);
899   int Result = Default;
900 
901   if (A.isStringAttribute()) {
902     StringRef Str = A.getValueAsString();
903     if (Str.getAsInteger(0, Result)) {
904       LLVMContext &Ctx = F.getContext();
905       Ctx.emitError("can't parse integer attribute " + Name);
906     }
907   }
908 
909   return Result;
910 }
911 
912 std::pair<int, int> getIntegerPairAttribute(const Function &F,
913                                             StringRef Name,
914                                             std::pair<int, int> Default,
915                                             bool OnlyFirstRequired) {
916   Attribute A = F.getFnAttribute(Name);
917   if (!A.isStringAttribute())
918     return Default;
919 
920   LLVMContext &Ctx = F.getContext();
921   std::pair<int, int> Ints = Default;
922   std::pair<StringRef, StringRef> Strs = A.getValueAsString().split(',');
923   if (Strs.first.trim().getAsInteger(0, Ints.first)) {
924     Ctx.emitError("can't parse first integer attribute " + Name);
925     return Default;
926   }
927   if (Strs.second.trim().getAsInteger(0, Ints.second)) {
928     if (!OnlyFirstRequired || !Strs.second.trim().empty()) {
929       Ctx.emitError("can't parse second integer attribute " + Name);
930       return Default;
931     }
932   }
933 
934   return Ints;
935 }
936 
937 unsigned getVmcntBitMask(const IsaVersion &Version) {
938   unsigned VmcntLo = (1 << getVmcntBitWidthLo()) - 1;
939   if (Version.Major < 9)
940     return VmcntLo;
941 
942   unsigned VmcntHi = ((1 << getVmcntBitWidthHi()) - 1) << getVmcntBitWidthLo();
943   return VmcntLo | VmcntHi;
944 }
945 
946 unsigned getExpcntBitMask(const IsaVersion &Version) {
947   return (1 << getExpcntBitWidth()) - 1;
948 }
949 
950 unsigned getLgkmcntBitMask(const IsaVersion &Version) {
951   return (1 << getLgkmcntBitWidth(Version.Major)) - 1;
952 }
953 
954 unsigned getWaitcntBitMask(const IsaVersion &Version) {
955   unsigned VmcntLo = getBitMask(getVmcntBitShiftLo(), getVmcntBitWidthLo());
956   unsigned Expcnt = getBitMask(getExpcntBitShift(), getExpcntBitWidth());
957   unsigned Lgkmcnt = getBitMask(getLgkmcntBitShift(),
958                                 getLgkmcntBitWidth(Version.Major));
959   unsigned Waitcnt = VmcntLo | Expcnt | Lgkmcnt;
960   if (Version.Major < 9)
961     return Waitcnt;
962 
963   unsigned VmcntHi = getBitMask(getVmcntBitShiftHi(), getVmcntBitWidthHi());
964   return Waitcnt | VmcntHi;
965 }
966 
967 unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) {
968   unsigned VmcntLo =
969       unpackBits(Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo());
970   if (Version.Major < 9)
971     return VmcntLo;
972 
973   unsigned VmcntHi =
974       unpackBits(Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi());
975   VmcntHi <<= getVmcntBitWidthLo();
976   return VmcntLo | VmcntHi;
977 }
978 
979 unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) {
980   return unpackBits(Waitcnt, getExpcntBitShift(), getExpcntBitWidth());
981 }
982 
983 unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) {
984   return unpackBits(Waitcnt, getLgkmcntBitShift(),
985                     getLgkmcntBitWidth(Version.Major));
986 }
987 
988 void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt,
989                    unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt) {
990   Vmcnt = decodeVmcnt(Version, Waitcnt);
991   Expcnt = decodeExpcnt(Version, Waitcnt);
992   Lgkmcnt = decodeLgkmcnt(Version, Waitcnt);
993 }
994 
995 Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded) {
996   Waitcnt Decoded;
997   Decoded.VmCnt = decodeVmcnt(Version, Encoded);
998   Decoded.ExpCnt = decodeExpcnt(Version, Encoded);
999   Decoded.LgkmCnt = decodeLgkmcnt(Version, Encoded);
1000   return Decoded;
1001 }
1002 
1003 unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt,
1004                      unsigned Vmcnt) {
1005   Waitcnt =
1006       packBits(Vmcnt, Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo());
1007   if (Version.Major < 9)
1008     return Waitcnt;
1009 
1010   Vmcnt >>= getVmcntBitWidthLo();
1011   return packBits(Vmcnt, Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi());
1012 }
1013 
1014 unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt,
1015                       unsigned Expcnt) {
1016   return packBits(Expcnt, Waitcnt, getExpcntBitShift(), getExpcntBitWidth());
1017 }
1018 
1019 unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt,
1020                        unsigned Lgkmcnt) {
1021   return packBits(Lgkmcnt, Waitcnt, getLgkmcntBitShift(),
1022                                     getLgkmcntBitWidth(Version.Major));
1023 }
1024 
1025 unsigned encodeWaitcnt(const IsaVersion &Version,
1026                        unsigned Vmcnt, unsigned Expcnt, unsigned Lgkmcnt) {
1027   unsigned Waitcnt = getWaitcntBitMask(Version);
1028   Waitcnt = encodeVmcnt(Version, Waitcnt, Vmcnt);
1029   Waitcnt = encodeExpcnt(Version, Waitcnt, Expcnt);
1030   Waitcnt = encodeLgkmcnt(Version, Waitcnt, Lgkmcnt);
1031   return Waitcnt;
1032 }
1033 
1034 unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded) {
1035   return encodeWaitcnt(Version, Decoded.VmCnt, Decoded.ExpCnt, Decoded.LgkmCnt);
1036 }
1037 
1038 //===----------------------------------------------------------------------===//
1039 // Custom Operands.
1040 //
1041 // A table of custom operands shall describe "primary" operand names
1042 // first followed by aliases if any. It is not required but recommended
1043 // to arrange operands so that operand encoding match operand position
1044 // in the table. This will make disassembly a bit more efficient.
1045 // Unused slots in the table shall have an empty name.
1046 //
1047 //===----------------------------------------------------------------------===//
1048 
1049 template <class T>
1050 static bool isValidOpr(int Idx, const CustomOperand<T> OpInfo[], int OpInfoSize,
1051                        T Context) {
1052   return 0 <= Idx && Idx < OpInfoSize && !OpInfo[Idx].Name.empty() &&
1053          (!OpInfo[Idx].Cond || OpInfo[Idx].Cond(Context));
1054 }
1055 
1056 template <class T>
1057 static int getOprIdx(std::function<bool(const CustomOperand<T> &)> Test,
1058                      const CustomOperand<T> OpInfo[], int OpInfoSize,
1059                      T Context) {
1060   int InvalidIdx = OPR_ID_UNKNOWN;
1061   for (int Idx = 0; Idx < OpInfoSize; ++Idx) {
1062     if (Test(OpInfo[Idx])) {
1063       if (!OpInfo[Idx].Cond || OpInfo[Idx].Cond(Context))
1064         return Idx;
1065       InvalidIdx = OPR_ID_UNSUPPORTED;
1066     }
1067   }
1068   return InvalidIdx;
1069 }
1070 
1071 template <class T>
1072 static int getOprIdx(const StringRef Name, const CustomOperand<T> OpInfo[],
1073                      int OpInfoSize, T Context) {
1074   auto Test = [=](const CustomOperand<T> &Op) { return Op.Name == Name; };
1075   return getOprIdx<T>(Test, OpInfo, OpInfoSize, Context);
1076 }
1077 
1078 template <class T>
1079 static int getOprIdx(int Id, const CustomOperand<T> OpInfo[], int OpInfoSize,
1080                      T Context, bool QuickCheck = true) {
1081   auto Test = [=](const CustomOperand<T> &Op) {
1082     return Op.Encoding == Id && !Op.Name.empty();
1083   };
1084   // This is an optimization that should work in most cases.
1085   // As a side effect, it may cause selection of an alias
1086   // instead of a primary operand name in case of sparse tables.
1087   if (QuickCheck && isValidOpr<T>(Id, OpInfo, OpInfoSize, Context) &&
1088       OpInfo[Id].Encoding == Id) {
1089     return Id;
1090   }
1091   return getOprIdx<T>(Test, OpInfo, OpInfoSize, Context);
1092 }
1093 
1094 //===----------------------------------------------------------------------===//
1095 // hwreg
1096 //===----------------------------------------------------------------------===//
1097 
1098 namespace Hwreg {
1099 
1100 int64_t getHwregId(const StringRef Name, const MCSubtargetInfo &STI) {
1101   int Idx = getOprIdx<const MCSubtargetInfo &>(Name, Opr, OPR_SIZE, STI);
1102   return (Idx < 0) ? Idx : Opr[Idx].Encoding;
1103 }
1104 
1105 bool isValidHwreg(int64_t Id) {
1106   return 0 <= Id && isUInt<ID_WIDTH_>(Id);
1107 }
1108 
1109 bool isValidHwregOffset(int64_t Offset) {
1110   return 0 <= Offset && isUInt<OFFSET_WIDTH_>(Offset);
1111 }
1112 
1113 bool isValidHwregWidth(int64_t Width) {
1114   return 0 <= (Width - 1) && isUInt<WIDTH_M1_WIDTH_>(Width - 1);
1115 }
1116 
1117 uint64_t encodeHwreg(uint64_t Id, uint64_t Offset, uint64_t Width) {
1118   return (Id << ID_SHIFT_) |
1119          (Offset << OFFSET_SHIFT_) |
1120          ((Width - 1) << WIDTH_M1_SHIFT_);
1121 }
1122 
1123 StringRef getHwreg(unsigned Id, const MCSubtargetInfo &STI) {
1124   int Idx = getOprIdx<const MCSubtargetInfo &>(Id, Opr, OPR_SIZE, STI);
1125   return (Idx < 0) ? "" : Opr[Idx].Name;
1126 }
1127 
1128 void decodeHwreg(unsigned Val, unsigned &Id, unsigned &Offset, unsigned &Width) {
1129   Id = (Val & ID_MASK_) >> ID_SHIFT_;
1130   Offset = (Val & OFFSET_MASK_) >> OFFSET_SHIFT_;
1131   Width = ((Val & WIDTH_M1_MASK_) >> WIDTH_M1_SHIFT_) + 1;
1132 }
1133 
1134 } // namespace Hwreg
1135 
1136 //===----------------------------------------------------------------------===//
1137 // exp tgt
1138 //===----------------------------------------------------------------------===//
1139 
1140 namespace Exp {
1141 
1142 struct ExpTgt {
1143   StringLiteral Name;
1144   unsigned Tgt;
1145   unsigned MaxIndex;
1146 };
1147 
1148 static constexpr ExpTgt ExpTgtInfo[] = {
1149   {{"null"},  ET_NULL,   ET_NULL_MAX_IDX},
1150   {{"mrtz"},  ET_MRTZ,   ET_MRTZ_MAX_IDX},
1151   {{"prim"},  ET_PRIM,   ET_PRIM_MAX_IDX},
1152   {{"mrt"},   ET_MRT0,   ET_MRT_MAX_IDX},
1153   {{"pos"},   ET_POS0,   ET_POS_MAX_IDX},
1154   {{"param"}, ET_PARAM0, ET_PARAM_MAX_IDX},
1155 };
1156 
1157 bool getTgtName(unsigned Id, StringRef &Name, int &Index) {
1158   for (const ExpTgt &Val : ExpTgtInfo) {
1159     if (Val.Tgt <= Id && Id <= Val.Tgt + Val.MaxIndex) {
1160       Index = (Val.MaxIndex == 0) ? -1 : (Id - Val.Tgt);
1161       Name = Val.Name;
1162       return true;
1163     }
1164   }
1165   return false;
1166 }
1167 
1168 unsigned getTgtId(const StringRef Name) {
1169 
1170   for (const ExpTgt &Val : ExpTgtInfo) {
1171     if (Val.MaxIndex == 0 && Name == Val.Name)
1172       return Val.Tgt;
1173 
1174     if (Val.MaxIndex > 0 && Name.startswith(Val.Name)) {
1175       StringRef Suffix = Name.drop_front(Val.Name.size());
1176 
1177       unsigned Id;
1178       if (Suffix.getAsInteger(10, Id) || Id > Val.MaxIndex)
1179         return ET_INVALID;
1180 
1181       // Disable leading zeroes
1182       if (Suffix.size() > 1 && Suffix[0] == '0')
1183         return ET_INVALID;
1184 
1185       return Val.Tgt + Id;
1186     }
1187   }
1188   return ET_INVALID;
1189 }
1190 
1191 bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI) {
1192   return (Id != ET_POS4 && Id != ET_PRIM) || isGFX10Plus(STI);
1193 }
1194 
1195 } // namespace Exp
1196 
1197 //===----------------------------------------------------------------------===//
1198 // MTBUF Format
1199 //===----------------------------------------------------------------------===//
1200 
1201 namespace MTBUFFormat {
1202 
1203 int64_t getDfmt(const StringRef Name) {
1204   for (int Id = DFMT_MIN; Id <= DFMT_MAX; ++Id) {
1205     if (Name == DfmtSymbolic[Id])
1206       return Id;
1207   }
1208   return DFMT_UNDEF;
1209 }
1210 
1211 StringRef getDfmtName(unsigned Id) {
1212   assert(Id <= DFMT_MAX);
1213   return DfmtSymbolic[Id];
1214 }
1215 
1216 static StringLiteral const *getNfmtLookupTable(const MCSubtargetInfo &STI) {
1217   if (isSI(STI) || isCI(STI))
1218     return NfmtSymbolicSICI;
1219   if (isVI(STI) || isGFX9(STI))
1220     return NfmtSymbolicVI;
1221   return NfmtSymbolicGFX10;
1222 }
1223 
1224 int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI) {
1225   auto lookupTable = getNfmtLookupTable(STI);
1226   for (int Id = NFMT_MIN; Id <= NFMT_MAX; ++Id) {
1227     if (Name == lookupTable[Id])
1228       return Id;
1229   }
1230   return NFMT_UNDEF;
1231 }
1232 
1233 StringRef getNfmtName(unsigned Id, const MCSubtargetInfo &STI) {
1234   assert(Id <= NFMT_MAX);
1235   return getNfmtLookupTable(STI)[Id];
1236 }
1237 
1238 bool isValidDfmtNfmt(unsigned Id, const MCSubtargetInfo &STI) {
1239   unsigned Dfmt;
1240   unsigned Nfmt;
1241   decodeDfmtNfmt(Id, Dfmt, Nfmt);
1242   return isValidNfmt(Nfmt, STI);
1243 }
1244 
1245 bool isValidNfmt(unsigned Id, const MCSubtargetInfo &STI) {
1246   return !getNfmtName(Id, STI).empty();
1247 }
1248 
1249 int64_t encodeDfmtNfmt(unsigned Dfmt, unsigned Nfmt) {
1250   return (Dfmt << DFMT_SHIFT) | (Nfmt << NFMT_SHIFT);
1251 }
1252 
1253 void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt) {
1254   Dfmt = (Format >> DFMT_SHIFT) & DFMT_MASK;
1255   Nfmt = (Format >> NFMT_SHIFT) & NFMT_MASK;
1256 }
1257 
1258 int64_t getUnifiedFormat(const StringRef Name) {
1259   for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) {
1260     if (Name == UfmtSymbolic[Id])
1261       return Id;
1262   }
1263   return UFMT_UNDEF;
1264 }
1265 
1266 StringRef getUnifiedFormatName(unsigned Id) {
1267   return isValidUnifiedFormat(Id) ? UfmtSymbolic[Id] : "";
1268 }
1269 
1270 bool isValidUnifiedFormat(unsigned Id) {
1271   return Id <= UFMT_LAST;
1272 }
1273 
1274 int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt) {
1275   int64_t Fmt = encodeDfmtNfmt(Dfmt, Nfmt);
1276   for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) {
1277     if (Fmt == DfmtNfmt2UFmt[Id])
1278       return Id;
1279   }
1280   return UFMT_UNDEF;
1281 }
1282 
1283 bool isValidFormatEncoding(unsigned Val, const MCSubtargetInfo &STI) {
1284   return isGFX10Plus(STI) ? (Val <= UFMT_MAX) : (Val <= DFMT_NFMT_MAX);
1285 }
1286 
1287 unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI) {
1288   if (isGFX10Plus(STI))
1289     return UFMT_DEFAULT;
1290   return DFMT_NFMT_DEFAULT;
1291 }
1292 
1293 } // namespace MTBUFFormat
1294 
1295 //===----------------------------------------------------------------------===//
1296 // SendMsg
1297 //===----------------------------------------------------------------------===//
1298 
1299 namespace SendMsg {
1300 
1301 int64_t getMsgId(const StringRef Name, const MCSubtargetInfo &STI) {
1302   int Idx = getOprIdx<const MCSubtargetInfo &>(Name, Msg, MSG_SIZE, STI);
1303   return (Idx < 0) ? Idx : Msg[Idx].Encoding;
1304 }
1305 
1306 bool isValidMsgId(int64_t MsgId) {
1307   return 0 <= MsgId && isUInt<ID_WIDTH_>(MsgId);
1308 }
1309 
1310 StringRef getMsgName(int64_t MsgId, const MCSubtargetInfo &STI) {
1311   int Idx = getOprIdx<const MCSubtargetInfo &>(MsgId, Msg, MSG_SIZE, STI);
1312   return (Idx < 0) ? "" : Msg[Idx].Name;
1313 }
1314 
1315 int64_t getMsgOpId(int64_t MsgId, const StringRef Name) {
1316   const char* const *S = (MsgId == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
1317   const int F = (MsgId == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
1318   const int L = (MsgId == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
1319   for (int i = F; i < L; ++i) {
1320     if (Name == S[i]) {
1321       return i;
1322     }
1323   }
1324   return OP_UNKNOWN_;
1325 }
1326 
1327 bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI,
1328                   bool Strict) {
1329   assert(isValidMsgId(MsgId));
1330 
1331   if (!Strict)
1332     return 0 <= OpId && isUInt<OP_WIDTH_>(OpId);
1333 
1334   switch(MsgId)
1335   {
1336   case ID_GS:
1337     return (OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_) && OpId != OP_GS_NOP;
1338   case ID_GS_DONE:
1339     return OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_;
1340   case ID_SYSMSG:
1341     return OP_SYS_FIRST_ <= OpId && OpId < OP_SYS_LAST_;
1342   default:
1343     return OpId == OP_NONE_;
1344   }
1345 }
1346 
1347 StringRef getMsgOpName(int64_t MsgId, int64_t OpId) {
1348   assert(msgRequiresOp(MsgId));
1349   return (MsgId == ID_SYSMSG)? OpSysSymbolic[OpId] : OpGsSymbolic[OpId];
1350 }
1351 
1352 bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId,
1353                       const MCSubtargetInfo &STI, bool Strict) {
1354   assert(isValidMsgOp(MsgId, OpId, STI, Strict));
1355 
1356   if (!Strict)
1357     return 0 <= StreamId && isUInt<STREAM_ID_WIDTH_>(StreamId);
1358 
1359   switch(MsgId)
1360   {
1361   case ID_GS:
1362     return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_;
1363   case ID_GS_DONE:
1364     return (OpId == OP_GS_NOP)?
1365            (StreamId == STREAM_ID_NONE_) :
1366            (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_);
1367   default:
1368     return StreamId == STREAM_ID_NONE_;
1369   }
1370 }
1371 
1372 bool msgRequiresOp(int64_t MsgId) {
1373   return MsgId == ID_GS || MsgId == ID_GS_DONE || MsgId == ID_SYSMSG;
1374 }
1375 
1376 bool msgSupportsStream(int64_t MsgId, int64_t OpId) {
1377   return (MsgId == ID_GS || MsgId == ID_GS_DONE) && OpId != OP_GS_NOP;
1378 }
1379 
1380 void decodeMsg(unsigned Val,
1381                uint16_t &MsgId,
1382                uint16_t &OpId,
1383                uint16_t &StreamId) {
1384   MsgId = Val & ID_MASK_;
1385   OpId = (Val & OP_MASK_) >> OP_SHIFT_;
1386   StreamId = (Val & STREAM_ID_MASK_) >> STREAM_ID_SHIFT_;
1387 }
1388 
1389 uint64_t encodeMsg(uint64_t MsgId,
1390                    uint64_t OpId,
1391                    uint64_t StreamId) {
1392   return (MsgId << ID_SHIFT_) |
1393          (OpId << OP_SHIFT_) |
1394          (StreamId << STREAM_ID_SHIFT_);
1395 }
1396 
1397 } // namespace SendMsg
1398 
1399 //===----------------------------------------------------------------------===//
1400 //
1401 //===----------------------------------------------------------------------===//
1402 
1403 unsigned getInitialPSInputAddr(const Function &F) {
1404   return getIntegerAttribute(F, "InitialPSInputAddr", 0);
1405 }
1406 
1407 bool getHasColorExport(const Function &F) {
1408   // As a safe default always respond as if PS has color exports.
1409   return getIntegerAttribute(
1410              F, "amdgpu-color-export",
1411              F.getCallingConv() == CallingConv::AMDGPU_PS ? 1 : 0) != 0;
1412 }
1413 
1414 bool getHasDepthExport(const Function &F) {
1415   return getIntegerAttribute(F, "amdgpu-depth-export", 0) != 0;
1416 }
1417 
1418 bool isShader(CallingConv::ID cc) {
1419   switch(cc) {
1420     case CallingConv::AMDGPU_VS:
1421     case CallingConv::AMDGPU_LS:
1422     case CallingConv::AMDGPU_HS:
1423     case CallingConv::AMDGPU_ES:
1424     case CallingConv::AMDGPU_GS:
1425     case CallingConv::AMDGPU_PS:
1426     case CallingConv::AMDGPU_CS:
1427       return true;
1428     default:
1429       return false;
1430   }
1431 }
1432 
1433 bool isGraphics(CallingConv::ID cc) {
1434   return isShader(cc) || cc == CallingConv::AMDGPU_Gfx;
1435 }
1436 
1437 bool isCompute(CallingConv::ID cc) {
1438   return !isGraphics(cc) || cc == CallingConv::AMDGPU_CS;
1439 }
1440 
1441 bool isEntryFunctionCC(CallingConv::ID CC) {
1442   switch (CC) {
1443   case CallingConv::AMDGPU_KERNEL:
1444   case CallingConv::SPIR_KERNEL:
1445   case CallingConv::AMDGPU_VS:
1446   case CallingConv::AMDGPU_GS:
1447   case CallingConv::AMDGPU_PS:
1448   case CallingConv::AMDGPU_CS:
1449   case CallingConv::AMDGPU_ES:
1450   case CallingConv::AMDGPU_HS:
1451   case CallingConv::AMDGPU_LS:
1452     return true;
1453   default:
1454     return false;
1455   }
1456 }
1457 
1458 bool isModuleEntryFunctionCC(CallingConv::ID CC) {
1459   switch (CC) {
1460   case CallingConv::AMDGPU_Gfx:
1461     return true;
1462   default:
1463     return isEntryFunctionCC(CC);
1464   }
1465 }
1466 
1467 bool isKernelCC(const Function *Func) {
1468   return AMDGPU::isModuleEntryFunctionCC(Func->getCallingConv());
1469 }
1470 
1471 bool hasXNACK(const MCSubtargetInfo &STI) {
1472   return STI.getFeatureBits()[AMDGPU::FeatureXNACK];
1473 }
1474 
1475 bool hasSRAMECC(const MCSubtargetInfo &STI) {
1476   return STI.getFeatureBits()[AMDGPU::FeatureSRAMECC];
1477 }
1478 
1479 bool hasMIMG_R128(const MCSubtargetInfo &STI) {
1480   return STI.getFeatureBits()[AMDGPU::FeatureMIMG_R128] && !STI.getFeatureBits()[AMDGPU::FeatureR128A16];
1481 }
1482 
1483 bool hasGFX10A16(const MCSubtargetInfo &STI) {
1484   return STI.getFeatureBits()[AMDGPU::FeatureGFX10A16];
1485 }
1486 
1487 bool hasG16(const MCSubtargetInfo &STI) {
1488   return STI.getFeatureBits()[AMDGPU::FeatureG16];
1489 }
1490 
1491 bool hasPackedD16(const MCSubtargetInfo &STI) {
1492   return !STI.getFeatureBits()[AMDGPU::FeatureUnpackedD16VMem];
1493 }
1494 
1495 bool isSI(const MCSubtargetInfo &STI) {
1496   return STI.getFeatureBits()[AMDGPU::FeatureSouthernIslands];
1497 }
1498 
1499 bool isCI(const MCSubtargetInfo &STI) {
1500   return STI.getFeatureBits()[AMDGPU::FeatureSeaIslands];
1501 }
1502 
1503 bool isVI(const MCSubtargetInfo &STI) {
1504   return STI.getFeatureBits()[AMDGPU::FeatureVolcanicIslands];
1505 }
1506 
1507 bool isGFX9(const MCSubtargetInfo &STI) {
1508   return STI.getFeatureBits()[AMDGPU::FeatureGFX9];
1509 }
1510 
1511 bool isGFX9_GFX10(const MCSubtargetInfo &STI) {
1512   return isGFX9(STI) || isGFX10(STI);
1513 }
1514 
1515 bool isGFX8Plus(const MCSubtargetInfo &STI) {
1516   return isVI(STI) || isGFX9Plus(STI);
1517 }
1518 
1519 bool isGFX9Plus(const MCSubtargetInfo &STI) {
1520   return isGFX9(STI) || isGFX10Plus(STI);
1521 }
1522 
1523 bool isGFX10(const MCSubtargetInfo &STI) {
1524   return STI.getFeatureBits()[AMDGPU::FeatureGFX10];
1525 }
1526 
1527 bool isGFX10Plus(const MCSubtargetInfo &STI) { return isGFX10(STI); }
1528 
1529 bool isNotGFX10Plus(const MCSubtargetInfo &STI) {
1530   return isSI(STI) || isCI(STI) || isVI(STI) || isGFX9(STI);
1531 }
1532 
1533 bool isGFX10Before1030(const MCSubtargetInfo &STI) {
1534   return isGFX10(STI) && !AMDGPU::isGFX10_BEncoding(STI);
1535 }
1536 
1537 bool isGCN3Encoding(const MCSubtargetInfo &STI) {
1538   return STI.getFeatureBits()[AMDGPU::FeatureGCN3Encoding];
1539 }
1540 
1541 bool isGFX10_AEncoding(const MCSubtargetInfo &STI) {
1542   return STI.getFeatureBits()[AMDGPU::FeatureGFX10_AEncoding];
1543 }
1544 
1545 bool isGFX10_BEncoding(const MCSubtargetInfo &STI) {
1546   return STI.getFeatureBits()[AMDGPU::FeatureGFX10_BEncoding];
1547 }
1548 
1549 bool hasGFX10_3Insts(const MCSubtargetInfo &STI) {
1550   return STI.getFeatureBits()[AMDGPU::FeatureGFX10_3Insts];
1551 }
1552 
1553 bool isGFX90A(const MCSubtargetInfo &STI) {
1554   return STI.getFeatureBits()[AMDGPU::FeatureGFX90AInsts];
1555 }
1556 
1557 bool isGFX940(const MCSubtargetInfo &STI) {
1558   return STI.getFeatureBits()[AMDGPU::FeatureGFX940Insts];
1559 }
1560 
1561 bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI) {
1562   return STI.getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch];
1563 }
1564 
1565 bool hasMAIInsts(const MCSubtargetInfo &STI) {
1566   return STI.getFeatureBits()[AMDGPU::FeatureMAIInsts];
1567 }
1568 
1569 int32_t getTotalNumVGPRs(bool has90AInsts, int32_t ArgNumAGPR,
1570                          int32_t ArgNumVGPR) {
1571   if (has90AInsts && ArgNumAGPR)
1572     return alignTo(ArgNumVGPR, 4) + ArgNumAGPR;
1573   return std::max(ArgNumVGPR, ArgNumAGPR);
1574 }
1575 
1576 bool isSGPR(unsigned Reg, const MCRegisterInfo* TRI) {
1577   const MCRegisterClass SGPRClass = TRI->getRegClass(AMDGPU::SReg_32RegClassID);
1578   const unsigned FirstSubReg = TRI->getSubReg(Reg, AMDGPU::sub0);
1579   return SGPRClass.contains(FirstSubReg != 0 ? FirstSubReg : Reg) ||
1580     Reg == AMDGPU::SCC;
1581 }
1582 
1583 #define MAP_REG2REG \
1584   using namespace AMDGPU; \
1585   switch(Reg) { \
1586   default: return Reg; \
1587   CASE_CI_VI(FLAT_SCR) \
1588   CASE_CI_VI(FLAT_SCR_LO) \
1589   CASE_CI_VI(FLAT_SCR_HI) \
1590   CASE_VI_GFX9PLUS(TTMP0) \
1591   CASE_VI_GFX9PLUS(TTMP1) \
1592   CASE_VI_GFX9PLUS(TTMP2) \
1593   CASE_VI_GFX9PLUS(TTMP3) \
1594   CASE_VI_GFX9PLUS(TTMP4) \
1595   CASE_VI_GFX9PLUS(TTMP5) \
1596   CASE_VI_GFX9PLUS(TTMP6) \
1597   CASE_VI_GFX9PLUS(TTMP7) \
1598   CASE_VI_GFX9PLUS(TTMP8) \
1599   CASE_VI_GFX9PLUS(TTMP9) \
1600   CASE_VI_GFX9PLUS(TTMP10) \
1601   CASE_VI_GFX9PLUS(TTMP11) \
1602   CASE_VI_GFX9PLUS(TTMP12) \
1603   CASE_VI_GFX9PLUS(TTMP13) \
1604   CASE_VI_GFX9PLUS(TTMP14) \
1605   CASE_VI_GFX9PLUS(TTMP15) \
1606   CASE_VI_GFX9PLUS(TTMP0_TTMP1) \
1607   CASE_VI_GFX9PLUS(TTMP2_TTMP3) \
1608   CASE_VI_GFX9PLUS(TTMP4_TTMP5) \
1609   CASE_VI_GFX9PLUS(TTMP6_TTMP7) \
1610   CASE_VI_GFX9PLUS(TTMP8_TTMP9) \
1611   CASE_VI_GFX9PLUS(TTMP10_TTMP11) \
1612   CASE_VI_GFX9PLUS(TTMP12_TTMP13) \
1613   CASE_VI_GFX9PLUS(TTMP14_TTMP15) \
1614   CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3) \
1615   CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7) \
1616   CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11) \
1617   CASE_VI_GFX9PLUS(TTMP12_TTMP13_TTMP14_TTMP15) \
1618   CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7) \
1619   CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11) \
1620   CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
1621   CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
1622   }
1623 
1624 #define CASE_CI_VI(node) \
1625   assert(!isSI(STI)); \
1626   case node: return isCI(STI) ? node##_ci : node##_vi;
1627 
1628 #define CASE_VI_GFX9PLUS(node) \
1629   case node: return isGFX9Plus(STI) ? node##_gfx9plus : node##_vi;
1630 
1631 unsigned getMCReg(unsigned Reg, const MCSubtargetInfo &STI) {
1632   if (STI.getTargetTriple().getArch() == Triple::r600)
1633     return Reg;
1634   MAP_REG2REG
1635 }
1636 
1637 #undef CASE_CI_VI
1638 #undef CASE_VI_GFX9PLUS
1639 
1640 #define CASE_CI_VI(node)   case node##_ci: case node##_vi:   return node;
1641 #define CASE_VI_GFX9PLUS(node) case node##_vi: case node##_gfx9plus: return node;
1642 
1643 unsigned mc2PseudoReg(unsigned Reg) {
1644   MAP_REG2REG
1645 }
1646 
1647 #undef CASE_CI_VI
1648 #undef CASE_VI_GFX9PLUS
1649 #undef MAP_REG2REG
1650 
1651 bool isSISrcOperand(const MCInstrDesc &Desc, unsigned OpNo) {
1652   assert(OpNo < Desc.NumOperands);
1653   unsigned OpType = Desc.OpInfo[OpNo].OperandType;
1654   return OpType >= AMDGPU::OPERAND_SRC_FIRST &&
1655          OpType <= AMDGPU::OPERAND_SRC_LAST;
1656 }
1657 
1658 bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo) {
1659   assert(OpNo < Desc.NumOperands);
1660   unsigned OpType = Desc.OpInfo[OpNo].OperandType;
1661   switch (OpType) {
1662   case AMDGPU::OPERAND_REG_IMM_FP32:
1663   case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED:
1664   case AMDGPU::OPERAND_REG_IMM_FP64:
1665   case AMDGPU::OPERAND_REG_IMM_FP16:
1666   case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED:
1667   case AMDGPU::OPERAND_REG_IMM_V2FP16:
1668   case AMDGPU::OPERAND_REG_IMM_V2INT16:
1669   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1670   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1671   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1672   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1673   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1674   case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
1675   case AMDGPU::OPERAND_REG_INLINE_AC_FP16:
1676   case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16:
1677   case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16:
1678   case AMDGPU::OPERAND_REG_IMM_V2FP32:
1679   case AMDGPU::OPERAND_REG_INLINE_C_V2FP32:
1680   case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
1681     return true;
1682   default:
1683     return false;
1684   }
1685 }
1686 
1687 bool isSISrcInlinableOperand(const MCInstrDesc &Desc, unsigned OpNo) {
1688   assert(OpNo < Desc.NumOperands);
1689   unsigned OpType = Desc.OpInfo[OpNo].OperandType;
1690   return OpType >= AMDGPU::OPERAND_REG_INLINE_C_FIRST &&
1691          OpType <= AMDGPU::OPERAND_REG_INLINE_C_LAST;
1692 }
1693 
1694 // Avoid using MCRegisterClass::getSize, since that function will go away
1695 // (move from MC* level to Target* level). Return size in bits.
1696 unsigned getRegBitWidth(unsigned RCID) {
1697   switch (RCID) {
1698   case AMDGPU::VGPR_LO16RegClassID:
1699   case AMDGPU::VGPR_HI16RegClassID:
1700   case AMDGPU::SGPR_LO16RegClassID:
1701   case AMDGPU::AGPR_LO16RegClassID:
1702     return 16;
1703   case AMDGPU::SGPR_32RegClassID:
1704   case AMDGPU::VGPR_32RegClassID:
1705   case AMDGPU::VRegOrLds_32RegClassID:
1706   case AMDGPU::AGPR_32RegClassID:
1707   case AMDGPU::VS_32RegClassID:
1708   case AMDGPU::AV_32RegClassID:
1709   case AMDGPU::SReg_32RegClassID:
1710   case AMDGPU::SReg_32_XM0RegClassID:
1711   case AMDGPU::SRegOrLds_32RegClassID:
1712     return 32;
1713   case AMDGPU::SGPR_64RegClassID:
1714   case AMDGPU::VS_64RegClassID:
1715   case AMDGPU::SReg_64RegClassID:
1716   case AMDGPU::VReg_64RegClassID:
1717   case AMDGPU::AReg_64RegClassID:
1718   case AMDGPU::SReg_64_XEXECRegClassID:
1719   case AMDGPU::VReg_64_Align2RegClassID:
1720   case AMDGPU::AReg_64_Align2RegClassID:
1721   case AMDGPU::AV_64RegClassID:
1722   case AMDGPU::AV_64_Align2RegClassID:
1723     return 64;
1724   case AMDGPU::SGPR_96RegClassID:
1725   case AMDGPU::SReg_96RegClassID:
1726   case AMDGPU::VReg_96RegClassID:
1727   case AMDGPU::AReg_96RegClassID:
1728   case AMDGPU::VReg_96_Align2RegClassID:
1729   case AMDGPU::AReg_96_Align2RegClassID:
1730   case AMDGPU::AV_96RegClassID:
1731   case AMDGPU::AV_96_Align2RegClassID:
1732     return 96;
1733   case AMDGPU::SGPR_128RegClassID:
1734   case AMDGPU::SReg_128RegClassID:
1735   case AMDGPU::VReg_128RegClassID:
1736   case AMDGPU::AReg_128RegClassID:
1737   case AMDGPU::VReg_128_Align2RegClassID:
1738   case AMDGPU::AReg_128_Align2RegClassID:
1739   case AMDGPU::AV_128RegClassID:
1740   case AMDGPU::AV_128_Align2RegClassID:
1741     return 128;
1742   case AMDGPU::SGPR_160RegClassID:
1743   case AMDGPU::SReg_160RegClassID:
1744   case AMDGPU::VReg_160RegClassID:
1745   case AMDGPU::AReg_160RegClassID:
1746   case AMDGPU::VReg_160_Align2RegClassID:
1747   case AMDGPU::AReg_160_Align2RegClassID:
1748   case AMDGPU::AV_160RegClassID:
1749   case AMDGPU::AV_160_Align2RegClassID:
1750     return 160;
1751   case AMDGPU::SGPR_192RegClassID:
1752   case AMDGPU::SReg_192RegClassID:
1753   case AMDGPU::VReg_192RegClassID:
1754   case AMDGPU::AReg_192RegClassID:
1755   case AMDGPU::VReg_192_Align2RegClassID:
1756   case AMDGPU::AReg_192_Align2RegClassID:
1757   case AMDGPU::AV_192RegClassID:
1758   case AMDGPU::AV_192_Align2RegClassID:
1759     return 192;
1760   case AMDGPU::SGPR_224RegClassID:
1761   case AMDGPU::SReg_224RegClassID:
1762   case AMDGPU::VReg_224RegClassID:
1763   case AMDGPU::AReg_224RegClassID:
1764   case AMDGPU::VReg_224_Align2RegClassID:
1765   case AMDGPU::AReg_224_Align2RegClassID:
1766   case AMDGPU::AV_224RegClassID:
1767   case AMDGPU::AV_224_Align2RegClassID:
1768     return 224;
1769   case AMDGPU::SGPR_256RegClassID:
1770   case AMDGPU::SReg_256RegClassID:
1771   case AMDGPU::VReg_256RegClassID:
1772   case AMDGPU::AReg_256RegClassID:
1773   case AMDGPU::VReg_256_Align2RegClassID:
1774   case AMDGPU::AReg_256_Align2RegClassID:
1775   case AMDGPU::AV_256RegClassID:
1776   case AMDGPU::AV_256_Align2RegClassID:
1777     return 256;
1778   case AMDGPU::SGPR_512RegClassID:
1779   case AMDGPU::SReg_512RegClassID:
1780   case AMDGPU::VReg_512RegClassID:
1781   case AMDGPU::AReg_512RegClassID:
1782   case AMDGPU::VReg_512_Align2RegClassID:
1783   case AMDGPU::AReg_512_Align2RegClassID:
1784   case AMDGPU::AV_512RegClassID:
1785   case AMDGPU::AV_512_Align2RegClassID:
1786     return 512;
1787   case AMDGPU::SGPR_1024RegClassID:
1788   case AMDGPU::SReg_1024RegClassID:
1789   case AMDGPU::VReg_1024RegClassID:
1790   case AMDGPU::AReg_1024RegClassID:
1791   case AMDGPU::VReg_1024_Align2RegClassID:
1792   case AMDGPU::AReg_1024_Align2RegClassID:
1793   case AMDGPU::AV_1024RegClassID:
1794   case AMDGPU::AV_1024_Align2RegClassID:
1795     return 1024;
1796   default:
1797     llvm_unreachable("Unexpected register class");
1798   }
1799 }
1800 
1801 unsigned getRegBitWidth(const MCRegisterClass &RC) {
1802   return getRegBitWidth(RC.getID());
1803 }
1804 
1805 unsigned getRegOperandSize(const MCRegisterInfo *MRI, const MCInstrDesc &Desc,
1806                            unsigned OpNo) {
1807   assert(OpNo < Desc.NumOperands);
1808   unsigned RCID = Desc.OpInfo[OpNo].RegClass;
1809   return getRegBitWidth(MRI->getRegClass(RCID)) / 8;
1810 }
1811 
1812 bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi) {
1813   if (isInlinableIntLiteral(Literal))
1814     return true;
1815 
1816   uint64_t Val = static_cast<uint64_t>(Literal);
1817   return (Val == DoubleToBits(0.0)) ||
1818          (Val == DoubleToBits(1.0)) ||
1819          (Val == DoubleToBits(-1.0)) ||
1820          (Val == DoubleToBits(0.5)) ||
1821          (Val == DoubleToBits(-0.5)) ||
1822          (Val == DoubleToBits(2.0)) ||
1823          (Val == DoubleToBits(-2.0)) ||
1824          (Val == DoubleToBits(4.0)) ||
1825          (Val == DoubleToBits(-4.0)) ||
1826          (Val == 0x3fc45f306dc9c882 && HasInv2Pi);
1827 }
1828 
1829 bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi) {
1830   if (isInlinableIntLiteral(Literal))
1831     return true;
1832 
1833   // The actual type of the operand does not seem to matter as long
1834   // as the bits match one of the inline immediate values.  For example:
1835   //
1836   // -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal,
1837   // so it is a legal inline immediate.
1838   //
1839   // 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in
1840   // floating-point, so it is a legal inline immediate.
1841 
1842   uint32_t Val = static_cast<uint32_t>(Literal);
1843   return (Val == FloatToBits(0.0f)) ||
1844          (Val == FloatToBits(1.0f)) ||
1845          (Val == FloatToBits(-1.0f)) ||
1846          (Val == FloatToBits(0.5f)) ||
1847          (Val == FloatToBits(-0.5f)) ||
1848          (Val == FloatToBits(2.0f)) ||
1849          (Val == FloatToBits(-2.0f)) ||
1850          (Val == FloatToBits(4.0f)) ||
1851          (Val == FloatToBits(-4.0f)) ||
1852          (Val == 0x3e22f983 && HasInv2Pi);
1853 }
1854 
1855 bool isInlinableLiteral16(int16_t Literal, bool HasInv2Pi) {
1856   if (!HasInv2Pi)
1857     return false;
1858 
1859   if (isInlinableIntLiteral(Literal))
1860     return true;
1861 
1862   uint16_t Val = static_cast<uint16_t>(Literal);
1863   return Val == 0x3C00 || // 1.0
1864          Val == 0xBC00 || // -1.0
1865          Val == 0x3800 || // 0.5
1866          Val == 0xB800 || // -0.5
1867          Val == 0x4000 || // 2.0
1868          Val == 0xC000 || // -2.0
1869          Val == 0x4400 || // 4.0
1870          Val == 0xC400 || // -4.0
1871          Val == 0x3118;   // 1/2pi
1872 }
1873 
1874 bool isInlinableLiteralV216(int32_t Literal, bool HasInv2Pi) {
1875   assert(HasInv2Pi);
1876 
1877   if (isInt<16>(Literal) || isUInt<16>(Literal)) {
1878     int16_t Trunc = static_cast<int16_t>(Literal);
1879     return AMDGPU::isInlinableLiteral16(Trunc, HasInv2Pi);
1880   }
1881   if (!(Literal & 0xffff))
1882     return AMDGPU::isInlinableLiteral16(Literal >> 16, HasInv2Pi);
1883 
1884   int16_t Lo16 = static_cast<int16_t>(Literal);
1885   int16_t Hi16 = static_cast<int16_t>(Literal >> 16);
1886   return Lo16 == Hi16 && isInlinableLiteral16(Lo16, HasInv2Pi);
1887 }
1888 
1889 bool isInlinableIntLiteralV216(int32_t Literal) {
1890   int16_t Lo16 = static_cast<int16_t>(Literal);
1891   if (isInt<16>(Literal) || isUInt<16>(Literal))
1892     return isInlinableIntLiteral(Lo16);
1893 
1894   int16_t Hi16 = static_cast<int16_t>(Literal >> 16);
1895   if (!(Literal & 0xffff))
1896     return isInlinableIntLiteral(Hi16);
1897   return Lo16 == Hi16 && isInlinableIntLiteral(Lo16);
1898 }
1899 
1900 bool isFoldableLiteralV216(int32_t Literal, bool HasInv2Pi) {
1901   assert(HasInv2Pi);
1902 
1903   int16_t Lo16 = static_cast<int16_t>(Literal);
1904   if (isInt<16>(Literal) || isUInt<16>(Literal))
1905     return true;
1906 
1907   int16_t Hi16 = static_cast<int16_t>(Literal >> 16);
1908   if (!(Literal & 0xffff))
1909     return true;
1910   return Lo16 == Hi16;
1911 }
1912 
1913 bool isArgPassedInSGPR(const Argument *A) {
1914   const Function *F = A->getParent();
1915 
1916   // Arguments to compute shaders are never a source of divergence.
1917   CallingConv::ID CC = F->getCallingConv();
1918   switch (CC) {
1919   case CallingConv::AMDGPU_KERNEL:
1920   case CallingConv::SPIR_KERNEL:
1921     return true;
1922   case CallingConv::AMDGPU_VS:
1923   case CallingConv::AMDGPU_LS:
1924   case CallingConv::AMDGPU_HS:
1925   case CallingConv::AMDGPU_ES:
1926   case CallingConv::AMDGPU_GS:
1927   case CallingConv::AMDGPU_PS:
1928   case CallingConv::AMDGPU_CS:
1929   case CallingConv::AMDGPU_Gfx:
1930     // For non-compute shaders, SGPR inputs are marked with either inreg or byval.
1931     // Everything else is in VGPRs.
1932     return F->getAttributes().hasParamAttr(A->getArgNo(), Attribute::InReg) ||
1933            F->getAttributes().hasParamAttr(A->getArgNo(), Attribute::ByVal);
1934   default:
1935     // TODO: Should calls support inreg for SGPR inputs?
1936     return false;
1937   }
1938 }
1939 
1940 static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) {
1941   return isGCN3Encoding(ST) || isGFX10Plus(ST);
1942 }
1943 
1944 static bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST) {
1945   return isGFX9Plus(ST);
1946 }
1947 
1948 bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST,
1949                                       int64_t EncodedOffset) {
1950   return hasSMEMByteOffset(ST) ? isUInt<20>(EncodedOffset)
1951                                : isUInt<8>(EncodedOffset);
1952 }
1953 
1954 bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST,
1955                                     int64_t EncodedOffset,
1956                                     bool IsBuffer) {
1957   return !IsBuffer &&
1958          hasSMRDSignedImmOffset(ST) &&
1959          isInt<21>(EncodedOffset);
1960 }
1961 
1962 static bool isDwordAligned(uint64_t ByteOffset) {
1963   return (ByteOffset & 3) == 0;
1964 }
1965 
1966 uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST,
1967                                 uint64_t ByteOffset) {
1968   if (hasSMEMByteOffset(ST))
1969     return ByteOffset;
1970 
1971   assert(isDwordAligned(ByteOffset));
1972   return ByteOffset >> 2;
1973 }
1974 
1975 Optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST,
1976                                        int64_t ByteOffset, bool IsBuffer) {
1977   // The signed version is always a byte offset.
1978   if (!IsBuffer && hasSMRDSignedImmOffset(ST)) {
1979     assert(hasSMEMByteOffset(ST));
1980     return isInt<20>(ByteOffset) ? Optional<int64_t>(ByteOffset) : None;
1981   }
1982 
1983   if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST))
1984     return None;
1985 
1986   int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
1987   return isLegalSMRDEncodedUnsignedOffset(ST, EncodedOffset)
1988              ? Optional<int64_t>(EncodedOffset)
1989              : None;
1990 }
1991 
1992 Optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST,
1993                                                 int64_t ByteOffset) {
1994   if (!isCI(ST) || !isDwordAligned(ByteOffset))
1995     return None;
1996 
1997   int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
1998   return isUInt<32>(EncodedOffset) ? Optional<int64_t>(EncodedOffset) : None;
1999 }
2000 
2001 unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST, bool Signed) {
2002   // Address offset is 12-bit signed for GFX10, 13-bit for GFX9.
2003   if (AMDGPU::isGFX10(ST))
2004     return Signed ? 12 : 11;
2005 
2006   return Signed ? 13 : 12;
2007 }
2008 
2009 // Given Imm, split it into the values to put into the SOffset and ImmOffset
2010 // fields in an MUBUF instruction. Return false if it is not possible (due to a
2011 // hardware bug needing a workaround).
2012 //
2013 // The required alignment ensures that individual address components remain
2014 // aligned if they are aligned to begin with. It also ensures that additional
2015 // offsets within the given alignment can be added to the resulting ImmOffset.
2016 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset,
2017                       const GCNSubtarget *Subtarget, Align Alignment) {
2018   const uint32_t MaxImm = alignDown(4095, Alignment.value());
2019   uint32_t Overflow = 0;
2020 
2021   if (Imm > MaxImm) {
2022     if (Imm <= MaxImm + 64) {
2023       // Use an SOffset inline constant for 4..64
2024       Overflow = Imm - MaxImm;
2025       Imm = MaxImm;
2026     } else {
2027       // Try to keep the same value in SOffset for adjacent loads, so that
2028       // the corresponding register contents can be re-used.
2029       //
2030       // Load values with all low-bits (except for alignment bits) set into
2031       // SOffset, so that a larger range of values can be covered using
2032       // s_movk_i32.
2033       //
2034       // Atomic operations fail to work correctly when individual address
2035       // components are unaligned, even if their sum is aligned.
2036       uint32_t High = (Imm + Alignment.value()) & ~4095;
2037       uint32_t Low = (Imm + Alignment.value()) & 4095;
2038       Imm = Low;
2039       Overflow = High - Alignment.value();
2040     }
2041   }
2042 
2043   // There is a hardware bug in SI and CI which prevents address clamping in
2044   // MUBUF instructions from working correctly with SOffsets. The immediate
2045   // offset is unaffected.
2046   if (Overflow > 0 &&
2047       Subtarget->getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS)
2048     return false;
2049 
2050   ImmOffset = Imm;
2051   SOffset = Overflow;
2052   return true;
2053 }
2054 
2055 SIModeRegisterDefaults::SIModeRegisterDefaults(const Function &F) {
2056   *this = getDefaultForCallingConv(F.getCallingConv());
2057 
2058   StringRef IEEEAttr = F.getFnAttribute("amdgpu-ieee").getValueAsString();
2059   if (!IEEEAttr.empty())
2060     IEEE = IEEEAttr == "true";
2061 
2062   StringRef DX10ClampAttr
2063     = F.getFnAttribute("amdgpu-dx10-clamp").getValueAsString();
2064   if (!DX10ClampAttr.empty())
2065     DX10Clamp = DX10ClampAttr == "true";
2066 
2067   StringRef DenormF32Attr = F.getFnAttribute("denormal-fp-math-f32").getValueAsString();
2068   if (!DenormF32Attr.empty()) {
2069     DenormalMode DenormMode = parseDenormalFPAttribute(DenormF32Attr);
2070     FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE;
2071     FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE;
2072   }
2073 
2074   StringRef DenormAttr = F.getFnAttribute("denormal-fp-math").getValueAsString();
2075   if (!DenormAttr.empty()) {
2076     DenormalMode DenormMode = parseDenormalFPAttribute(DenormAttr);
2077 
2078     if (DenormF32Attr.empty()) {
2079       FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE;
2080       FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE;
2081     }
2082 
2083     FP64FP16InputDenormals = DenormMode.Input == DenormalMode::IEEE;
2084     FP64FP16OutputDenormals = DenormMode.Output == DenormalMode::IEEE;
2085   }
2086 }
2087 
2088 namespace {
2089 
2090 struct SourceOfDivergence {
2091   unsigned Intr;
2092 };
2093 const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr);
2094 
2095 #define GET_SourcesOfDivergence_IMPL
2096 #define GET_Gfx9BufferFormat_IMPL
2097 #define GET_Gfx10PlusBufferFormat_IMPL
2098 #include "AMDGPUGenSearchableTables.inc"
2099 
2100 } // end anonymous namespace
2101 
2102 bool isIntrinsicSourceOfDivergence(unsigned IntrID) {
2103   return lookupSourceOfDivergence(IntrID);
2104 }
2105 
2106 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp,
2107                                                   uint8_t NumComponents,
2108                                                   uint8_t NumFormat,
2109                                                   const MCSubtargetInfo &STI) {
2110   return isGFX10Plus(STI)
2111              ? getGfx10PlusBufferFormatInfo(BitsPerComp, NumComponents,
2112                                             NumFormat)
2113              : getGfx9BufferFormatInfo(BitsPerComp, NumComponents, NumFormat);
2114 }
2115 
2116 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format,
2117                                                   const MCSubtargetInfo &STI) {
2118   return isGFX10Plus(STI) ? getGfx10PlusBufferFormatInfo(Format)
2119                           : getGfx9BufferFormatInfo(Format);
2120 }
2121 
2122 } // namespace AMDGPU
2123 
2124 raw_ostream &operator<<(raw_ostream &OS,
2125                         const AMDGPU::IsaInfo::TargetIDSetting S) {
2126   switch (S) {
2127   case (AMDGPU::IsaInfo::TargetIDSetting::Unsupported):
2128     OS << "Unsupported";
2129     break;
2130   case (AMDGPU::IsaInfo::TargetIDSetting::Any):
2131     OS << "Any";
2132     break;
2133   case (AMDGPU::IsaInfo::TargetIDSetting::Off):
2134     OS << "Off";
2135     break;
2136   case (AMDGPU::IsaInfo::TargetIDSetting::On):
2137     OS << "On";
2138     break;
2139   }
2140   return OS;
2141 }
2142 
2143 } // namespace llvm
2144