1 //===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "AMDGPUBaseInfo.h"
10 #include "AMDGPU.h"
11 #include "AMDGPUAsmUtils.h"
12 #include "AMDGPUTargetTransformInfo.h"
13 #include "SIDefines.h"
14 #include "llvm/ADT/StringRef.h"
15 #include "llvm/ADT/Triple.h"
16 #include "llvm/BinaryFormat/ELF.h"
17 #include "llvm/CodeGen/MachineMemOperand.h"
18 #include "llvm/IR/Attributes.h"
19 #include "llvm/IR/Constants.h"
20 #include "llvm/IR/Function.h"
21 #include "llvm/IR/GlobalValue.h"
22 #include "llvm/IR/Instruction.h"
23 #include "llvm/IR/IntrinsicsAMDGPU.h"
24 #include "llvm/IR/IntrinsicsR600.h"
25 #include "llvm/IR/LLVMContext.h"
26 #include "llvm/IR/Module.h"
27 #include "llvm/MC/MCContext.h"
28 #include "llvm/MC/MCInstrDesc.h"
29 #include "llvm/MC/MCInstrInfo.h"
30 #include "llvm/MC/MCRegisterInfo.h"
31 #include "llvm/MC/MCSectionELF.h"
32 #include "llvm/MC/MCSubtargetInfo.h"
33 #include "llvm/MC/SubtargetFeature.h"
34 #include "llvm/Support/Casting.h"
35 #include "llvm/Support/ErrorHandling.h"
36 #include "llvm/Support/MathExtras.h"
37 #include <algorithm>
38 #include <cassert>
39 #include <cstdint>
40 #include <cstring>
41 #include <utility>
42 
43 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
44 
45 #define GET_INSTRINFO_NAMED_OPS
46 #define GET_INSTRMAP_INFO
47 #include "AMDGPUGenInstrInfo.inc"
48 #undef GET_INSTRMAP_INFO
49 #undef GET_INSTRINFO_NAMED_OPS
50 
51 namespace {
52 
53 /// \returns Bit mask for given bit \p Shift and bit \p Width.
54 unsigned getBitMask(unsigned Shift, unsigned Width) {
55   return ((1 << Width) - 1) << Shift;
56 }
57 
58 /// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width.
59 ///
60 /// \returns Packed \p Dst.
61 unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) {
62   Dst &= ~(1 << Shift) & ~getBitMask(Shift, Width);
63   Dst |= (Src << Shift) & getBitMask(Shift, Width);
64   return Dst;
65 }
66 
67 /// Unpacks bits from \p Src for given bit \p Shift and bit \p Width.
68 ///
69 /// \returns Unpacked bits.
70 unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) {
71   return (Src & getBitMask(Shift, Width)) >> Shift;
72 }
73 
74 /// \returns Vmcnt bit shift (lower bits).
75 unsigned getVmcntBitShiftLo() { return 0; }
76 
77 /// \returns Vmcnt bit width (lower bits).
78 unsigned getVmcntBitWidthLo() { return 4; }
79 
80 /// \returns Expcnt bit shift.
81 unsigned getExpcntBitShift() { return 4; }
82 
83 /// \returns Expcnt bit width.
84 unsigned getExpcntBitWidth() { return 3; }
85 
86 /// \returns Lgkmcnt bit shift.
87 unsigned getLgkmcntBitShift() { return 8; }
88 
89 /// \returns Lgkmcnt bit width.
90 unsigned getLgkmcntBitWidth(unsigned VersionMajor) {
91   return (VersionMajor >= 10) ? 6 : 4;
92 }
93 
94 /// \returns Vmcnt bit shift (higher bits).
95 unsigned getVmcntBitShiftHi() { return 14; }
96 
97 /// \returns Vmcnt bit width (higher bits).
98 unsigned getVmcntBitWidthHi() { return 2; }
99 
100 } // end namespace anonymous
101 
102 namespace llvm {
103 
104 namespace AMDGPU {
105 
106 #define GET_MIMGBaseOpcodesTable_IMPL
107 #define GET_MIMGDimInfoTable_IMPL
108 #define GET_MIMGInfoTable_IMPL
109 #define GET_MIMGLZMappingTable_IMPL
110 #define GET_MIMGMIPMappingTable_IMPL
111 #define GET_MIMGG16MappingTable_IMPL
112 #include "AMDGPUGenSearchableTables.inc"
113 
114 int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding,
115                   unsigned VDataDwords, unsigned VAddrDwords) {
116   const MIMGInfo *Info = getMIMGOpcodeHelper(BaseOpcode, MIMGEncoding,
117                                              VDataDwords, VAddrDwords);
118   return Info ? Info->Opcode : -1;
119 }
120 
121 const MIMGBaseOpcodeInfo *getMIMGBaseOpcode(unsigned Opc) {
122   const MIMGInfo *Info = getMIMGInfo(Opc);
123   return Info ? getMIMGBaseOpcodeInfo(Info->BaseOpcode) : nullptr;
124 }
125 
126 int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) {
127   const MIMGInfo *OrigInfo = getMIMGInfo(Opc);
128   const MIMGInfo *NewInfo =
129       getMIMGOpcodeHelper(OrigInfo->BaseOpcode, OrigInfo->MIMGEncoding,
130                           NewChannels, OrigInfo->VAddrDwords);
131   return NewInfo ? NewInfo->Opcode : -1;
132 }
133 
134 struct MUBUFInfo {
135   uint16_t Opcode;
136   uint16_t BaseOpcode;
137   uint8_t elements;
138   bool has_vaddr;
139   bool has_srsrc;
140   bool has_soffset;
141 };
142 
143 struct MTBUFInfo {
144   uint16_t Opcode;
145   uint16_t BaseOpcode;
146   uint8_t elements;
147   bool has_vaddr;
148   bool has_srsrc;
149   bool has_soffset;
150 };
151 
152 struct SMInfo {
153   uint16_t Opcode;
154   bool IsBuffer;
155 };
156 
157 #define GET_MTBUFInfoTable_DECL
158 #define GET_MTBUFInfoTable_IMPL
159 #define GET_MUBUFInfoTable_DECL
160 #define GET_MUBUFInfoTable_IMPL
161 #define GET_SMInfoTable_DECL
162 #define GET_SMInfoTable_IMPL
163 #include "AMDGPUGenSearchableTables.inc"
164 
165 int getMTBUFBaseOpcode(unsigned Opc) {
166   const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opc);
167   return Info ? Info->BaseOpcode : -1;
168 }
169 
170 int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements) {
171   const MTBUFInfo *Info = getMTBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements);
172   return Info ? Info->Opcode : -1;
173 }
174 
175 int getMTBUFElements(unsigned Opc) {
176   const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
177   return Info ? Info->elements : 0;
178 }
179 
180 bool getMTBUFHasVAddr(unsigned Opc) {
181   const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
182   return Info ? Info->has_vaddr : false;
183 }
184 
185 bool getMTBUFHasSrsrc(unsigned Opc) {
186   const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
187   return Info ? Info->has_srsrc : false;
188 }
189 
190 bool getMTBUFHasSoffset(unsigned Opc) {
191   const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
192   return Info ? Info->has_soffset : false;
193 }
194 
195 int getMUBUFBaseOpcode(unsigned Opc) {
196   const MUBUFInfo *Info = getMUBUFInfoFromOpcode(Opc);
197   return Info ? Info->BaseOpcode : -1;
198 }
199 
200 int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements) {
201   const MUBUFInfo *Info = getMUBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements);
202   return Info ? Info->Opcode : -1;
203 }
204 
205 int getMUBUFElements(unsigned Opc) {
206   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
207   return Info ? Info->elements : 0;
208 }
209 
210 bool getMUBUFHasVAddr(unsigned Opc) {
211   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
212   return Info ? Info->has_vaddr : false;
213 }
214 
215 bool getMUBUFHasSrsrc(unsigned Opc) {
216   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
217   return Info ? Info->has_srsrc : false;
218 }
219 
220 bool getMUBUFHasSoffset(unsigned Opc) {
221   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
222   return Info ? Info->has_soffset : false;
223 }
224 
225 bool getSMEMIsBuffer(unsigned Opc) {
226   const SMInfo *Info = getSMEMOpcodeHelper(Opc);
227   return Info ? Info->IsBuffer : false;
228 }
229 
230 // Wrapper for Tablegen'd function.  enum Subtarget is not defined in any
231 // header files, so we need to wrap it in a function that takes unsigned
232 // instead.
233 int getMCOpcode(uint16_t Opcode, unsigned Gen) {
234   return getMCOpcodeGen(Opcode, static_cast<Subtarget>(Gen));
235 }
236 
237 namespace IsaInfo {
238 
239 void streamIsaVersion(const MCSubtargetInfo *STI, raw_ostream &Stream) {
240   auto TargetTriple = STI->getTargetTriple();
241   auto Version = getIsaVersion(STI->getCPU());
242 
243   Stream << TargetTriple.getArchName() << '-'
244          << TargetTriple.getVendorName() << '-'
245          << TargetTriple.getOSName() << '-'
246          << TargetTriple.getEnvironmentName() << '-'
247          << "gfx"
248          << Version.Major
249          << Version.Minor
250          << Version.Stepping;
251 
252   if (hasXNACK(*STI))
253     Stream << "+xnack";
254   if (hasSRAMECC(*STI))
255     Stream << "+sram-ecc";
256 
257   Stream.flush();
258 }
259 
260 bool hasCodeObjectV3(const MCSubtargetInfo *STI) {
261   return STI->getTargetTriple().getOS() == Triple::AMDHSA &&
262              STI->getFeatureBits().test(FeatureCodeObjectV3);
263 }
264 
265 unsigned getWavefrontSize(const MCSubtargetInfo *STI) {
266   if (STI->getFeatureBits().test(FeatureWavefrontSize16))
267     return 16;
268   if (STI->getFeatureBits().test(FeatureWavefrontSize32))
269     return 32;
270 
271   return 64;
272 }
273 
274 unsigned getLocalMemorySize(const MCSubtargetInfo *STI) {
275   if (STI->getFeatureBits().test(FeatureLocalMemorySize32768))
276     return 32768;
277   if (STI->getFeatureBits().test(FeatureLocalMemorySize65536))
278     return 65536;
279 
280   return 0;
281 }
282 
283 unsigned getEUsPerCU(const MCSubtargetInfo *STI) {
284   // "Per CU" really means "per whatever functional block the waves of a
285   // workgroup must share". For gfx10 in CU mode this is the CU, which contains
286   // two SIMDs.
287   if (isGFX10(*STI) && STI->getFeatureBits().test(FeatureCuMode))
288     return 2;
289   // Pre-gfx10 a CU contains four SIMDs. For gfx10 in WGP mode the WGP contains
290   // two CUs, so a total of four SIMDs.
291   return 4;
292 }
293 
294 unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo *STI,
295                                unsigned FlatWorkGroupSize) {
296   assert(FlatWorkGroupSize != 0);
297   if (STI->getTargetTriple().getArch() != Triple::amdgcn)
298     return 8;
299   unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize);
300   if (N == 1)
301     return 40;
302   N = 40 / N;
303   return std::min(N, 16u);
304 }
305 
306 unsigned getMinWavesPerEU(const MCSubtargetInfo *STI) {
307   return 1;
308 }
309 
310 unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI) {
311   // FIXME: Need to take scratch memory into account.
312   if (!isGFX10(*STI))
313     return 10;
314   return hasGFX10_3Insts(*STI) ? 16 : 20;
315 }
316 
317 unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo *STI,
318                                    unsigned FlatWorkGroupSize) {
319   return divideCeil(getWavesPerWorkGroup(STI, FlatWorkGroupSize),
320                     getEUsPerCU(STI));
321 }
322 
323 unsigned getMinFlatWorkGroupSize(const MCSubtargetInfo *STI) {
324   return 1;
325 }
326 
327 unsigned getMaxFlatWorkGroupSize(const MCSubtargetInfo *STI) {
328   // Some subtargets allow encoding 2048, but this isn't tested or supported.
329   return 1024;
330 }
331 
332 unsigned getWavesPerWorkGroup(const MCSubtargetInfo *STI,
333                               unsigned FlatWorkGroupSize) {
334   return divideCeil(FlatWorkGroupSize, getWavefrontSize(STI));
335 }
336 
337 unsigned getSGPRAllocGranule(const MCSubtargetInfo *STI) {
338   IsaVersion Version = getIsaVersion(STI->getCPU());
339   if (Version.Major >= 10)
340     return getAddressableNumSGPRs(STI);
341   if (Version.Major >= 8)
342     return 16;
343   return 8;
344 }
345 
346 unsigned getSGPREncodingGranule(const MCSubtargetInfo *STI) {
347   return 8;
348 }
349 
350 unsigned getTotalNumSGPRs(const MCSubtargetInfo *STI) {
351   IsaVersion Version = getIsaVersion(STI->getCPU());
352   if (Version.Major >= 8)
353     return 800;
354   return 512;
355 }
356 
357 unsigned getAddressableNumSGPRs(const MCSubtargetInfo *STI) {
358   if (STI->getFeatureBits().test(FeatureSGPRInitBug))
359     return FIXED_NUM_SGPRS_FOR_INIT_BUG;
360 
361   IsaVersion Version = getIsaVersion(STI->getCPU());
362   if (Version.Major >= 10)
363     return 106;
364   if (Version.Major >= 8)
365     return 102;
366   return 104;
367 }
368 
369 unsigned getMinNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
370   assert(WavesPerEU != 0);
371 
372   IsaVersion Version = getIsaVersion(STI->getCPU());
373   if (Version.Major >= 10)
374     return 0;
375 
376   if (WavesPerEU >= getMaxWavesPerEU(STI))
377     return 0;
378 
379   unsigned MinNumSGPRs = getTotalNumSGPRs(STI) / (WavesPerEU + 1);
380   if (STI->getFeatureBits().test(FeatureTrapHandler))
381     MinNumSGPRs -= std::min(MinNumSGPRs, (unsigned)TRAP_NUM_SGPRS);
382   MinNumSGPRs = alignDown(MinNumSGPRs, getSGPRAllocGranule(STI)) + 1;
383   return std::min(MinNumSGPRs, getAddressableNumSGPRs(STI));
384 }
385 
386 unsigned getMaxNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU,
387                         bool Addressable) {
388   assert(WavesPerEU != 0);
389 
390   unsigned AddressableNumSGPRs = getAddressableNumSGPRs(STI);
391   IsaVersion Version = getIsaVersion(STI->getCPU());
392   if (Version.Major >= 10)
393     return Addressable ? AddressableNumSGPRs : 108;
394   if (Version.Major >= 8 && !Addressable)
395     AddressableNumSGPRs = 112;
396   unsigned MaxNumSGPRs = getTotalNumSGPRs(STI) / WavesPerEU;
397   if (STI->getFeatureBits().test(FeatureTrapHandler))
398     MaxNumSGPRs -= std::min(MaxNumSGPRs, (unsigned)TRAP_NUM_SGPRS);
399   MaxNumSGPRs = alignDown(MaxNumSGPRs, getSGPRAllocGranule(STI));
400   return std::min(MaxNumSGPRs, AddressableNumSGPRs);
401 }
402 
403 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed,
404                           bool FlatScrUsed, bool XNACKUsed) {
405   unsigned ExtraSGPRs = 0;
406   if (VCCUsed)
407     ExtraSGPRs = 2;
408 
409   IsaVersion Version = getIsaVersion(STI->getCPU());
410   if (Version.Major >= 10)
411     return ExtraSGPRs;
412 
413   if (Version.Major < 8) {
414     if (FlatScrUsed)
415       ExtraSGPRs = 4;
416   } else {
417     if (XNACKUsed)
418       ExtraSGPRs = 4;
419 
420     if (FlatScrUsed)
421       ExtraSGPRs = 6;
422   }
423 
424   return ExtraSGPRs;
425 }
426 
427 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed,
428                           bool FlatScrUsed) {
429   return getNumExtraSGPRs(STI, VCCUsed, FlatScrUsed,
430                           STI->getFeatureBits().test(AMDGPU::FeatureXNACK));
431 }
432 
433 unsigned getNumSGPRBlocks(const MCSubtargetInfo *STI, unsigned NumSGPRs) {
434   NumSGPRs = alignTo(std::max(1u, NumSGPRs), getSGPREncodingGranule(STI));
435   // SGPRBlocks is actual number of SGPR blocks minus 1.
436   return NumSGPRs / getSGPREncodingGranule(STI) - 1;
437 }
438 
439 unsigned getVGPRAllocGranule(const MCSubtargetInfo *STI,
440                              Optional<bool> EnableWavefrontSize32) {
441   bool IsWave32 = EnableWavefrontSize32 ?
442       *EnableWavefrontSize32 :
443       STI->getFeatureBits().test(FeatureWavefrontSize32);
444 
445   if (hasGFX10_3Insts(*STI))
446     return IsWave32 ? 16 : 8;
447 
448   return IsWave32 ? 8 : 4;
449 }
450 
451 unsigned getVGPREncodingGranule(const MCSubtargetInfo *STI,
452                                 Optional<bool> EnableWavefrontSize32) {
453 
454   bool IsWave32 = EnableWavefrontSize32 ?
455       *EnableWavefrontSize32 :
456       STI->getFeatureBits().test(FeatureWavefrontSize32);
457 
458   return IsWave32 ? 8 : 4;
459 }
460 
461 unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) {
462   if (!isGFX10(*STI))
463     return 256;
464   return STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1024 : 512;
465 }
466 
467 unsigned getAddressableNumVGPRs(const MCSubtargetInfo *STI) {
468   return 256;
469 }
470 
471 unsigned getMinNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
472   assert(WavesPerEU != 0);
473 
474   if (WavesPerEU >= getMaxWavesPerEU(STI))
475     return 0;
476   unsigned MinNumVGPRs =
477       alignDown(getTotalNumVGPRs(STI) / (WavesPerEU + 1),
478                 getVGPRAllocGranule(STI)) + 1;
479   return std::min(MinNumVGPRs, getAddressableNumVGPRs(STI));
480 }
481 
482 unsigned getMaxNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
483   assert(WavesPerEU != 0);
484 
485   unsigned MaxNumVGPRs = alignDown(getTotalNumVGPRs(STI) / WavesPerEU,
486                                    getVGPRAllocGranule(STI));
487   unsigned AddressableNumVGPRs = getAddressableNumVGPRs(STI);
488   return std::min(MaxNumVGPRs, AddressableNumVGPRs);
489 }
490 
491 unsigned getNumVGPRBlocks(const MCSubtargetInfo *STI, unsigned NumVGPRs,
492                           Optional<bool> EnableWavefrontSize32) {
493   NumVGPRs = alignTo(std::max(1u, NumVGPRs),
494                      getVGPREncodingGranule(STI, EnableWavefrontSize32));
495   // VGPRBlocks is actual number of VGPR blocks minus 1.
496   return NumVGPRs / getVGPREncodingGranule(STI, EnableWavefrontSize32) - 1;
497 }
498 
499 } // end namespace IsaInfo
500 
501 void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header,
502                                const MCSubtargetInfo *STI) {
503   IsaVersion Version = getIsaVersion(STI->getCPU());
504 
505   memset(&Header, 0, sizeof(Header));
506 
507   Header.amd_kernel_code_version_major = 1;
508   Header.amd_kernel_code_version_minor = 2;
509   Header.amd_machine_kind = 1; // AMD_MACHINE_KIND_AMDGPU
510   Header.amd_machine_version_major = Version.Major;
511   Header.amd_machine_version_minor = Version.Minor;
512   Header.amd_machine_version_stepping = Version.Stepping;
513   Header.kernel_code_entry_byte_offset = sizeof(Header);
514   Header.wavefront_size = 6;
515 
516   // If the code object does not support indirect functions, then the value must
517   // be 0xffffffff.
518   Header.call_convention = -1;
519 
520   // These alignment values are specified in powers of two, so alignment =
521   // 2^n.  The minimum alignment is 2^4 = 16.
522   Header.kernarg_segment_alignment = 4;
523   Header.group_segment_alignment = 4;
524   Header.private_segment_alignment = 4;
525 
526   if (Version.Major >= 10) {
527     if (STI->getFeatureBits().test(FeatureWavefrontSize32)) {
528       Header.wavefront_size = 5;
529       Header.code_properties |= AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32;
530     }
531     Header.compute_pgm_resource_registers |=
532       S_00B848_WGP_MODE(STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1) |
533       S_00B848_MEM_ORDERED(1);
534   }
535 }
536 
537 amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor(
538     const MCSubtargetInfo *STI) {
539   IsaVersion Version = getIsaVersion(STI->getCPU());
540 
541   amdhsa::kernel_descriptor_t KD;
542   memset(&KD, 0, sizeof(KD));
543 
544   AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
545                   amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64,
546                   amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE);
547   AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
548                   amdhsa::COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP, 1);
549   AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
550                   amdhsa::COMPUTE_PGM_RSRC1_ENABLE_IEEE_MODE, 1);
551   AMDHSA_BITS_SET(KD.compute_pgm_rsrc2,
552                   amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, 1);
553   if (Version.Major >= 10) {
554     AMDHSA_BITS_SET(KD.kernel_code_properties,
555                     amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32,
556                     STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1 : 0);
557     AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
558                     amdhsa::COMPUTE_PGM_RSRC1_WGP_MODE,
559                     STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1);
560     AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
561                     amdhsa::COMPUTE_PGM_RSRC1_MEM_ORDERED, 1);
562   }
563   return KD;
564 }
565 
566 bool isGroupSegment(const GlobalValue *GV) {
567   return GV->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS;
568 }
569 
570 bool isGlobalSegment(const GlobalValue *GV) {
571   return GV->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS;
572 }
573 
574 bool isReadOnlySegment(const GlobalValue *GV) {
575   unsigned AS = GV->getAddressSpace();
576   return AS == AMDGPUAS::CONSTANT_ADDRESS ||
577          AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
578 }
579 
580 bool shouldEmitConstantsToTextSection(const Triple &TT) {
581   return TT.getArch() == Triple::r600;
582 }
583 
584 int getIntegerAttribute(const Function &F, StringRef Name, int Default) {
585   Attribute A = F.getFnAttribute(Name);
586   int Result = Default;
587 
588   if (A.isStringAttribute()) {
589     StringRef Str = A.getValueAsString();
590     if (Str.getAsInteger(0, Result)) {
591       LLVMContext &Ctx = F.getContext();
592       Ctx.emitError("can't parse integer attribute " + Name);
593     }
594   }
595 
596   return Result;
597 }
598 
599 std::pair<int, int> getIntegerPairAttribute(const Function &F,
600                                             StringRef Name,
601                                             std::pair<int, int> Default,
602                                             bool OnlyFirstRequired) {
603   Attribute A = F.getFnAttribute(Name);
604   if (!A.isStringAttribute())
605     return Default;
606 
607   LLVMContext &Ctx = F.getContext();
608   std::pair<int, int> Ints = Default;
609   std::pair<StringRef, StringRef> Strs = A.getValueAsString().split(',');
610   if (Strs.first.trim().getAsInteger(0, Ints.first)) {
611     Ctx.emitError("can't parse first integer attribute " + Name);
612     return Default;
613   }
614   if (Strs.second.trim().getAsInteger(0, Ints.second)) {
615     if (!OnlyFirstRequired || !Strs.second.trim().empty()) {
616       Ctx.emitError("can't parse second integer attribute " + Name);
617       return Default;
618     }
619   }
620 
621   return Ints;
622 }
623 
624 unsigned getVmcntBitMask(const IsaVersion &Version) {
625   unsigned VmcntLo = (1 << getVmcntBitWidthLo()) - 1;
626   if (Version.Major < 9)
627     return VmcntLo;
628 
629   unsigned VmcntHi = ((1 << getVmcntBitWidthHi()) - 1) << getVmcntBitWidthLo();
630   return VmcntLo | VmcntHi;
631 }
632 
633 unsigned getExpcntBitMask(const IsaVersion &Version) {
634   return (1 << getExpcntBitWidth()) - 1;
635 }
636 
637 unsigned getLgkmcntBitMask(const IsaVersion &Version) {
638   return (1 << getLgkmcntBitWidth(Version.Major)) - 1;
639 }
640 
641 unsigned getWaitcntBitMask(const IsaVersion &Version) {
642   unsigned VmcntLo = getBitMask(getVmcntBitShiftLo(), getVmcntBitWidthLo());
643   unsigned Expcnt = getBitMask(getExpcntBitShift(), getExpcntBitWidth());
644   unsigned Lgkmcnt = getBitMask(getLgkmcntBitShift(),
645                                 getLgkmcntBitWidth(Version.Major));
646   unsigned Waitcnt = VmcntLo | Expcnt | Lgkmcnt;
647   if (Version.Major < 9)
648     return Waitcnt;
649 
650   unsigned VmcntHi = getBitMask(getVmcntBitShiftHi(), getVmcntBitWidthHi());
651   return Waitcnt | VmcntHi;
652 }
653 
654 unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) {
655   unsigned VmcntLo =
656       unpackBits(Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo());
657   if (Version.Major < 9)
658     return VmcntLo;
659 
660   unsigned VmcntHi =
661       unpackBits(Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi());
662   VmcntHi <<= getVmcntBitWidthLo();
663   return VmcntLo | VmcntHi;
664 }
665 
666 unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) {
667   return unpackBits(Waitcnt, getExpcntBitShift(), getExpcntBitWidth());
668 }
669 
670 unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) {
671   return unpackBits(Waitcnt, getLgkmcntBitShift(),
672                     getLgkmcntBitWidth(Version.Major));
673 }
674 
675 void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt,
676                    unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt) {
677   Vmcnt = decodeVmcnt(Version, Waitcnt);
678   Expcnt = decodeExpcnt(Version, Waitcnt);
679   Lgkmcnt = decodeLgkmcnt(Version, Waitcnt);
680 }
681 
682 Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded) {
683   Waitcnt Decoded;
684   Decoded.VmCnt = decodeVmcnt(Version, Encoded);
685   Decoded.ExpCnt = decodeExpcnt(Version, Encoded);
686   Decoded.LgkmCnt = decodeLgkmcnt(Version, Encoded);
687   return Decoded;
688 }
689 
690 unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt,
691                      unsigned Vmcnt) {
692   Waitcnt =
693       packBits(Vmcnt, Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo());
694   if (Version.Major < 9)
695     return Waitcnt;
696 
697   Vmcnt >>= getVmcntBitWidthLo();
698   return packBits(Vmcnt, Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi());
699 }
700 
701 unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt,
702                       unsigned Expcnt) {
703   return packBits(Expcnt, Waitcnt, getExpcntBitShift(), getExpcntBitWidth());
704 }
705 
706 unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt,
707                        unsigned Lgkmcnt) {
708   return packBits(Lgkmcnt, Waitcnt, getLgkmcntBitShift(),
709                                     getLgkmcntBitWidth(Version.Major));
710 }
711 
712 unsigned encodeWaitcnt(const IsaVersion &Version,
713                        unsigned Vmcnt, unsigned Expcnt, unsigned Lgkmcnt) {
714   unsigned Waitcnt = getWaitcntBitMask(Version);
715   Waitcnt = encodeVmcnt(Version, Waitcnt, Vmcnt);
716   Waitcnt = encodeExpcnt(Version, Waitcnt, Expcnt);
717   Waitcnt = encodeLgkmcnt(Version, Waitcnt, Lgkmcnt);
718   return Waitcnt;
719 }
720 
721 unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded) {
722   return encodeWaitcnt(Version, Decoded.VmCnt, Decoded.ExpCnt, Decoded.LgkmCnt);
723 }
724 
725 //===----------------------------------------------------------------------===//
726 // hwreg
727 //===----------------------------------------------------------------------===//
728 
729 namespace Hwreg {
730 
731 int64_t getHwregId(const StringRef Name) {
732   for (int Id = ID_SYMBOLIC_FIRST_; Id < ID_SYMBOLIC_LAST_; ++Id) {
733     if (IdSymbolic[Id] && Name == IdSymbolic[Id])
734       return Id;
735   }
736   return ID_UNKNOWN_;
737 }
738 
739 static unsigned getLastSymbolicHwreg(const MCSubtargetInfo &STI) {
740   if (isSI(STI) || isCI(STI) || isVI(STI))
741     return ID_SYMBOLIC_FIRST_GFX9_;
742   else if (isGFX9(STI))
743     return ID_SYMBOLIC_FIRST_GFX10_;
744   else if (isGFX10(STI) && !isGFX10_BEncoding(STI))
745     return ID_SYMBOLIC_FIRST_GFX1030_;
746   else
747     return ID_SYMBOLIC_LAST_;
748 }
749 
750 bool isValidHwreg(int64_t Id, const MCSubtargetInfo &STI) {
751   return
752     ID_SYMBOLIC_FIRST_ <= Id && Id < getLastSymbolicHwreg(STI) &&
753     IdSymbolic[Id] && (Id != ID_XNACK_MASK || !AMDGPU::isGFX10_BEncoding(STI));
754 }
755 
756 bool isValidHwreg(int64_t Id) {
757   return 0 <= Id && isUInt<ID_WIDTH_>(Id);
758 }
759 
760 bool isValidHwregOffset(int64_t Offset) {
761   return 0 <= Offset && isUInt<OFFSET_WIDTH_>(Offset);
762 }
763 
764 bool isValidHwregWidth(int64_t Width) {
765   return 0 <= (Width - 1) && isUInt<WIDTH_M1_WIDTH_>(Width - 1);
766 }
767 
768 uint64_t encodeHwreg(uint64_t Id, uint64_t Offset, uint64_t Width) {
769   return (Id << ID_SHIFT_) |
770          (Offset << OFFSET_SHIFT_) |
771          ((Width - 1) << WIDTH_M1_SHIFT_);
772 }
773 
774 StringRef getHwreg(unsigned Id, const MCSubtargetInfo &STI) {
775   return isValidHwreg(Id, STI) ? IdSymbolic[Id] : "";
776 }
777 
778 void decodeHwreg(unsigned Val, unsigned &Id, unsigned &Offset, unsigned &Width) {
779   Id = (Val & ID_MASK_) >> ID_SHIFT_;
780   Offset = (Val & OFFSET_MASK_) >> OFFSET_SHIFT_;
781   Width = ((Val & WIDTH_M1_MASK_) >> WIDTH_M1_SHIFT_) + 1;
782 }
783 
784 } // namespace Hwreg
785 
786 //===----------------------------------------------------------------------===//
787 // MTBUF Format
788 //===----------------------------------------------------------------------===//
789 
790 namespace MTBUFFormat {
791 
792 int64_t getDfmt(const StringRef Name) {
793   for (int Id = DFMT_MIN; Id <= DFMT_MAX; ++Id) {
794     if (Name == DfmtSymbolic[Id])
795       return Id;
796   }
797   return DFMT_UNDEF;
798 }
799 
800 StringRef getDfmtName(unsigned Id) {
801   assert(Id <= DFMT_MAX);
802   return DfmtSymbolic[Id];
803 }
804 
805 static StringLiteral const *getNfmtLookupTable(const MCSubtargetInfo &STI) {
806   if (isSI(STI) || isCI(STI))
807     return NfmtSymbolicSICI;
808   if (isVI(STI) || isGFX9(STI))
809     return NfmtSymbolicVI;
810   return NfmtSymbolicGFX10;
811 }
812 
813 int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI) {
814   auto lookupTable = getNfmtLookupTable(STI);
815   for (int Id = NFMT_MIN; Id <= NFMT_MAX; ++Id) {
816     if (Name == lookupTable[Id])
817       return Id;
818   }
819   return NFMT_UNDEF;
820 }
821 
822 StringRef getNfmtName(unsigned Id, const MCSubtargetInfo &STI) {
823   assert(Id <= NFMT_MAX);
824   return getNfmtLookupTable(STI)[Id];
825 }
826 
827 bool isValidDfmtNfmt(unsigned Id, const MCSubtargetInfo &STI) {
828   unsigned Dfmt;
829   unsigned Nfmt;
830   decodeDfmtNfmt(Id, Dfmt, Nfmt);
831   return isValidNfmt(Nfmt, STI);
832 }
833 
834 bool isValidNfmt(unsigned Id, const MCSubtargetInfo &STI) {
835   return !getNfmtName(Id, STI).empty();
836 }
837 
838 int64_t encodeDfmtNfmt(unsigned Dfmt, unsigned Nfmt) {
839   return (Dfmt << DFMT_SHIFT) | (Nfmt << NFMT_SHIFT);
840 }
841 
842 void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt) {
843   Dfmt = (Format >> DFMT_SHIFT) & DFMT_MASK;
844   Nfmt = (Format >> NFMT_SHIFT) & NFMT_MASK;
845 }
846 
847 int64_t getUnifiedFormat(const StringRef Name) {
848   for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) {
849     if (Name == UfmtSymbolic[Id])
850       return Id;
851   }
852   return UFMT_UNDEF;
853 }
854 
855 StringRef getUnifiedFormatName(unsigned Id) {
856   return isValidUnifiedFormat(Id) ? UfmtSymbolic[Id] : "";
857 }
858 
859 bool isValidUnifiedFormat(unsigned Id) {
860   return Id <= UFMT_LAST;
861 }
862 
863 int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt) {
864   int64_t Fmt = encodeDfmtNfmt(Dfmt, Nfmt);
865   for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) {
866     if (Fmt == DfmtNfmt2UFmt[Id])
867       return Id;
868   }
869   return UFMT_UNDEF;
870 }
871 
872 bool isValidFormatEncoding(unsigned Val, const MCSubtargetInfo &STI) {
873   return isGFX10(STI) ? (Val <= UFMT_MAX) : (Val <= DFMT_NFMT_MAX);
874 }
875 
876 unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI) {
877   if (isGFX10(STI))
878     return UFMT_DEFAULT;
879   return DFMT_NFMT_DEFAULT;
880 }
881 
882 } // namespace MTBUFFormat
883 
884 //===----------------------------------------------------------------------===//
885 // SendMsg
886 //===----------------------------------------------------------------------===//
887 
888 namespace SendMsg {
889 
890 int64_t getMsgId(const StringRef Name) {
891   for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
892     if (IdSymbolic[i] && Name == IdSymbolic[i])
893       return i;
894   }
895   return ID_UNKNOWN_;
896 }
897 
898 static bool isValidMsgId(int64_t MsgId) {
899   return (ID_GAPS_FIRST_ <= MsgId && MsgId < ID_GAPS_LAST_) && IdSymbolic[MsgId];
900 }
901 
902 bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI, bool Strict) {
903   if (Strict) {
904     if (MsgId == ID_GS_ALLOC_REQ || MsgId == ID_GET_DOORBELL)
905       return isGFX9(STI) || isGFX10(STI);
906     else
907       return isValidMsgId(MsgId);
908   } else {
909     return 0 <= MsgId && isUInt<ID_WIDTH_>(MsgId);
910   }
911 }
912 
913 StringRef getMsgName(int64_t MsgId) {
914   return isValidMsgId(MsgId)? IdSymbolic[MsgId] : "";
915 }
916 
917 int64_t getMsgOpId(int64_t MsgId, const StringRef Name) {
918   const char* const *S = (MsgId == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
919   const int F = (MsgId == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
920   const int L = (MsgId == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
921   for (int i = F; i < L; ++i) {
922     if (Name == S[i]) {
923       return i;
924     }
925   }
926   return OP_UNKNOWN_;
927 }
928 
929 bool isValidMsgOp(int64_t MsgId, int64_t OpId, bool Strict) {
930 
931   if (!Strict)
932     return 0 <= OpId && isUInt<OP_WIDTH_>(OpId);
933 
934   switch(MsgId)
935   {
936   case ID_GS:
937     return (OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_) && OpId != OP_GS_NOP;
938   case ID_GS_DONE:
939     return OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_;
940   case ID_SYSMSG:
941     return OP_SYS_FIRST_ <= OpId && OpId < OP_SYS_LAST_;
942   default:
943     return OpId == OP_NONE_;
944   }
945 }
946 
947 StringRef getMsgOpName(int64_t MsgId, int64_t OpId) {
948   assert(msgRequiresOp(MsgId));
949   return (MsgId == ID_SYSMSG)? OpSysSymbolic[OpId] : OpGsSymbolic[OpId];
950 }
951 
952 bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId, bool Strict) {
953 
954   if (!Strict)
955     return 0 <= StreamId && isUInt<STREAM_ID_WIDTH_>(StreamId);
956 
957   switch(MsgId)
958   {
959   case ID_GS:
960     return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_;
961   case ID_GS_DONE:
962     return (OpId == OP_GS_NOP)?
963            (StreamId == STREAM_ID_NONE_) :
964            (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_);
965   default:
966     return StreamId == STREAM_ID_NONE_;
967   }
968 }
969 
970 bool msgRequiresOp(int64_t MsgId) {
971   return MsgId == ID_GS || MsgId == ID_GS_DONE || MsgId == ID_SYSMSG;
972 }
973 
974 bool msgSupportsStream(int64_t MsgId, int64_t OpId) {
975   return (MsgId == ID_GS || MsgId == ID_GS_DONE) && OpId != OP_GS_NOP;
976 }
977 
978 void decodeMsg(unsigned Val,
979                uint16_t &MsgId,
980                uint16_t &OpId,
981                uint16_t &StreamId) {
982   MsgId = Val & ID_MASK_;
983   OpId = (Val & OP_MASK_) >> OP_SHIFT_;
984   StreamId = (Val & STREAM_ID_MASK_) >> STREAM_ID_SHIFT_;
985 }
986 
987 uint64_t encodeMsg(uint64_t MsgId,
988                    uint64_t OpId,
989                    uint64_t StreamId) {
990   return (MsgId << ID_SHIFT_) |
991          (OpId << OP_SHIFT_) |
992          (StreamId << STREAM_ID_SHIFT_);
993 }
994 
995 } // namespace SendMsg
996 
997 //===----------------------------------------------------------------------===//
998 //
999 //===----------------------------------------------------------------------===//
1000 
1001 unsigned getInitialPSInputAddr(const Function &F) {
1002   return getIntegerAttribute(F, "InitialPSInputAddr", 0);
1003 }
1004 
1005 bool isShader(CallingConv::ID cc) {
1006   switch(cc) {
1007     case CallingConv::AMDGPU_VS:
1008     case CallingConv::AMDGPU_LS:
1009     case CallingConv::AMDGPU_HS:
1010     case CallingConv::AMDGPU_ES:
1011     case CallingConv::AMDGPU_GS:
1012     case CallingConv::AMDGPU_PS:
1013     case CallingConv::AMDGPU_CS:
1014       return true;
1015     default:
1016       return false;
1017   }
1018 }
1019 
1020 bool isCompute(CallingConv::ID cc) {
1021   return !isShader(cc) || cc == CallingConv::AMDGPU_CS;
1022 }
1023 
1024 bool isEntryFunctionCC(CallingConv::ID CC) {
1025   switch (CC) {
1026   case CallingConv::AMDGPU_KERNEL:
1027   case CallingConv::SPIR_KERNEL:
1028   case CallingConv::AMDGPU_VS:
1029   case CallingConv::AMDGPU_GS:
1030   case CallingConv::AMDGPU_PS:
1031   case CallingConv::AMDGPU_CS:
1032   case CallingConv::AMDGPU_ES:
1033   case CallingConv::AMDGPU_HS:
1034   case CallingConv::AMDGPU_LS:
1035     return true;
1036   default:
1037     return false;
1038   }
1039 }
1040 
1041 bool hasXNACK(const MCSubtargetInfo &STI) {
1042   return STI.getFeatureBits()[AMDGPU::FeatureXNACK];
1043 }
1044 
1045 bool hasSRAMECC(const MCSubtargetInfo &STI) {
1046   return STI.getFeatureBits()[AMDGPU::FeatureSRAMECC];
1047 }
1048 
1049 bool hasMIMG_R128(const MCSubtargetInfo &STI) {
1050   return STI.getFeatureBits()[AMDGPU::FeatureMIMG_R128] && !STI.getFeatureBits()[AMDGPU::FeatureR128A16];
1051 }
1052 
1053 bool hasGFX10A16(const MCSubtargetInfo &STI) {
1054   return STI.getFeatureBits()[AMDGPU::FeatureGFX10A16];
1055 }
1056 
1057 bool hasG16(const MCSubtargetInfo &STI) {
1058   return STI.getFeatureBits()[AMDGPU::FeatureG16];
1059 }
1060 
1061 bool hasPackedD16(const MCSubtargetInfo &STI) {
1062   return !STI.getFeatureBits()[AMDGPU::FeatureUnpackedD16VMem];
1063 }
1064 
1065 bool isSI(const MCSubtargetInfo &STI) {
1066   return STI.getFeatureBits()[AMDGPU::FeatureSouthernIslands];
1067 }
1068 
1069 bool isCI(const MCSubtargetInfo &STI) {
1070   return STI.getFeatureBits()[AMDGPU::FeatureSeaIslands];
1071 }
1072 
1073 bool isVI(const MCSubtargetInfo &STI) {
1074   return STI.getFeatureBits()[AMDGPU::FeatureVolcanicIslands];
1075 }
1076 
1077 bool isGFX9(const MCSubtargetInfo &STI) {
1078   return STI.getFeatureBits()[AMDGPU::FeatureGFX9];
1079 }
1080 
1081 bool isGFX10(const MCSubtargetInfo &STI) {
1082   return STI.getFeatureBits()[AMDGPU::FeatureGFX10];
1083 }
1084 
1085 bool isGCN3Encoding(const MCSubtargetInfo &STI) {
1086   return STI.getFeatureBits()[AMDGPU::FeatureGCN3Encoding];
1087 }
1088 
1089 bool isGFX10_BEncoding(const MCSubtargetInfo &STI) {
1090   return STI.getFeatureBits()[AMDGPU::FeatureGFX10_BEncoding];
1091 }
1092 
1093 bool hasGFX10_3Insts(const MCSubtargetInfo &STI) {
1094   return STI.getFeatureBits()[AMDGPU::FeatureGFX10_3Insts];
1095 }
1096 
1097 bool isSGPR(unsigned Reg, const MCRegisterInfo* TRI) {
1098   const MCRegisterClass SGPRClass = TRI->getRegClass(AMDGPU::SReg_32RegClassID);
1099   const unsigned FirstSubReg = TRI->getSubReg(Reg, AMDGPU::sub0);
1100   return SGPRClass.contains(FirstSubReg != 0 ? FirstSubReg : Reg) ||
1101     Reg == AMDGPU::SCC;
1102 }
1103 
1104 bool isRegIntersect(unsigned Reg0, unsigned Reg1, const MCRegisterInfo* TRI) {
1105   for (MCRegAliasIterator R(Reg0, TRI, true); R.isValid(); ++R) {
1106     if (*R == Reg1) return true;
1107   }
1108   return false;
1109 }
1110 
1111 #define MAP_REG2REG \
1112   using namespace AMDGPU; \
1113   switch(Reg) { \
1114   default: return Reg; \
1115   CASE_CI_VI(FLAT_SCR) \
1116   CASE_CI_VI(FLAT_SCR_LO) \
1117   CASE_CI_VI(FLAT_SCR_HI) \
1118   CASE_VI_GFX9_GFX10(TTMP0) \
1119   CASE_VI_GFX9_GFX10(TTMP1) \
1120   CASE_VI_GFX9_GFX10(TTMP2) \
1121   CASE_VI_GFX9_GFX10(TTMP3) \
1122   CASE_VI_GFX9_GFX10(TTMP4) \
1123   CASE_VI_GFX9_GFX10(TTMP5) \
1124   CASE_VI_GFX9_GFX10(TTMP6) \
1125   CASE_VI_GFX9_GFX10(TTMP7) \
1126   CASE_VI_GFX9_GFX10(TTMP8) \
1127   CASE_VI_GFX9_GFX10(TTMP9) \
1128   CASE_VI_GFX9_GFX10(TTMP10) \
1129   CASE_VI_GFX9_GFX10(TTMP11) \
1130   CASE_VI_GFX9_GFX10(TTMP12) \
1131   CASE_VI_GFX9_GFX10(TTMP13) \
1132   CASE_VI_GFX9_GFX10(TTMP14) \
1133   CASE_VI_GFX9_GFX10(TTMP15) \
1134   CASE_VI_GFX9_GFX10(TTMP0_TTMP1) \
1135   CASE_VI_GFX9_GFX10(TTMP2_TTMP3) \
1136   CASE_VI_GFX9_GFX10(TTMP4_TTMP5) \
1137   CASE_VI_GFX9_GFX10(TTMP6_TTMP7) \
1138   CASE_VI_GFX9_GFX10(TTMP8_TTMP9) \
1139   CASE_VI_GFX9_GFX10(TTMP10_TTMP11) \
1140   CASE_VI_GFX9_GFX10(TTMP12_TTMP13) \
1141   CASE_VI_GFX9_GFX10(TTMP14_TTMP15) \
1142   CASE_VI_GFX9_GFX10(TTMP0_TTMP1_TTMP2_TTMP3) \
1143   CASE_VI_GFX9_GFX10(TTMP4_TTMP5_TTMP6_TTMP7) \
1144   CASE_VI_GFX9_GFX10(TTMP8_TTMP9_TTMP10_TTMP11) \
1145   CASE_VI_GFX9_GFX10(TTMP12_TTMP13_TTMP14_TTMP15) \
1146   CASE_VI_GFX9_GFX10(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7) \
1147   CASE_VI_GFX9_GFX10(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11) \
1148   CASE_VI_GFX9_GFX10(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
1149   CASE_VI_GFX9_GFX10(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
1150   }
1151 
1152 #define CASE_CI_VI(node) \
1153   assert(!isSI(STI)); \
1154   case node: return isCI(STI) ? node##_ci : node##_vi;
1155 
1156 #define CASE_VI_GFX9_GFX10(node) \
1157   case node: return (isGFX9(STI) || isGFX10(STI)) ? node##_gfx9_gfx10 : node##_vi;
1158 
1159 unsigned getMCReg(unsigned Reg, const MCSubtargetInfo &STI) {
1160   if (STI.getTargetTriple().getArch() == Triple::r600)
1161     return Reg;
1162   MAP_REG2REG
1163 }
1164 
1165 #undef CASE_CI_VI
1166 #undef CASE_VI_GFX9_GFX10
1167 
1168 #define CASE_CI_VI(node)   case node##_ci: case node##_vi:   return node;
1169 #define CASE_VI_GFX9_GFX10(node) case node##_vi: case node##_gfx9_gfx10: return node;
1170 
1171 unsigned mc2PseudoReg(unsigned Reg) {
1172   MAP_REG2REG
1173 }
1174 
1175 #undef CASE_CI_VI
1176 #undef CASE_VI_GFX9_GFX10
1177 #undef MAP_REG2REG
1178 
1179 bool isSISrcOperand(const MCInstrDesc &Desc, unsigned OpNo) {
1180   assert(OpNo < Desc.NumOperands);
1181   unsigned OpType = Desc.OpInfo[OpNo].OperandType;
1182   return OpType >= AMDGPU::OPERAND_SRC_FIRST &&
1183          OpType <= AMDGPU::OPERAND_SRC_LAST;
1184 }
1185 
1186 bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo) {
1187   assert(OpNo < Desc.NumOperands);
1188   unsigned OpType = Desc.OpInfo[OpNo].OperandType;
1189   switch (OpType) {
1190   case AMDGPU::OPERAND_REG_IMM_FP32:
1191   case AMDGPU::OPERAND_REG_IMM_FP64:
1192   case AMDGPU::OPERAND_REG_IMM_FP16:
1193   case AMDGPU::OPERAND_REG_IMM_V2FP16:
1194   case AMDGPU::OPERAND_REG_IMM_V2INT16:
1195   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1196   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1197   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1198   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1199   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1200   case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
1201   case AMDGPU::OPERAND_REG_INLINE_AC_FP16:
1202   case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16:
1203   case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16:
1204     return true;
1205   default:
1206     return false;
1207   }
1208 }
1209 
1210 bool isSISrcInlinableOperand(const MCInstrDesc &Desc, unsigned OpNo) {
1211   assert(OpNo < Desc.NumOperands);
1212   unsigned OpType = Desc.OpInfo[OpNo].OperandType;
1213   return OpType >= AMDGPU::OPERAND_REG_INLINE_C_FIRST &&
1214          OpType <= AMDGPU::OPERAND_REG_INLINE_C_LAST;
1215 }
1216 
1217 // Avoid using MCRegisterClass::getSize, since that function will go away
1218 // (move from MC* level to Target* level). Return size in bits.
1219 unsigned getRegBitWidth(unsigned RCID) {
1220   switch (RCID) {
1221   case AMDGPU::VGPR_LO16RegClassID:
1222   case AMDGPU::VGPR_HI16RegClassID:
1223   case AMDGPU::SGPR_LO16RegClassID:
1224   case AMDGPU::AGPR_LO16RegClassID:
1225     return 16;
1226   case AMDGPU::SGPR_32RegClassID:
1227   case AMDGPU::VGPR_32RegClassID:
1228   case AMDGPU::VRegOrLds_32RegClassID:
1229   case AMDGPU::AGPR_32RegClassID:
1230   case AMDGPU::VS_32RegClassID:
1231   case AMDGPU::AV_32RegClassID:
1232   case AMDGPU::SReg_32RegClassID:
1233   case AMDGPU::SReg_32_XM0RegClassID:
1234   case AMDGPU::SRegOrLds_32RegClassID:
1235     return 32;
1236   case AMDGPU::SGPR_64RegClassID:
1237   case AMDGPU::VS_64RegClassID:
1238   case AMDGPU::AV_64RegClassID:
1239   case AMDGPU::SReg_64RegClassID:
1240   case AMDGPU::VReg_64RegClassID:
1241   case AMDGPU::AReg_64RegClassID:
1242   case AMDGPU::SReg_64_XEXECRegClassID:
1243     return 64;
1244   case AMDGPU::SGPR_96RegClassID:
1245   case AMDGPU::SReg_96RegClassID:
1246   case AMDGPU::VReg_96RegClassID:
1247   case AMDGPU::AReg_96RegClassID:
1248     return 96;
1249   case AMDGPU::SGPR_128RegClassID:
1250   case AMDGPU::SReg_128RegClassID:
1251   case AMDGPU::VReg_128RegClassID:
1252   case AMDGPU::AReg_128RegClassID:
1253     return 128;
1254   case AMDGPU::SGPR_160RegClassID:
1255   case AMDGPU::SReg_160RegClassID:
1256   case AMDGPU::VReg_160RegClassID:
1257   case AMDGPU::AReg_160RegClassID:
1258     return 160;
1259   case AMDGPU::SGPR_192RegClassID:
1260   case AMDGPU::SReg_192RegClassID:
1261   case AMDGPU::VReg_192RegClassID:
1262   case AMDGPU::AReg_192RegClassID:
1263     return 192;
1264   case AMDGPU::SGPR_256RegClassID:
1265   case AMDGPU::SReg_256RegClassID:
1266   case AMDGPU::VReg_256RegClassID:
1267   case AMDGPU::AReg_256RegClassID:
1268     return 256;
1269   case AMDGPU::SGPR_512RegClassID:
1270   case AMDGPU::SReg_512RegClassID:
1271   case AMDGPU::VReg_512RegClassID:
1272   case AMDGPU::AReg_512RegClassID:
1273     return 512;
1274   case AMDGPU::SGPR_1024RegClassID:
1275   case AMDGPU::SReg_1024RegClassID:
1276   case AMDGPU::VReg_1024RegClassID:
1277   case AMDGPU::AReg_1024RegClassID:
1278     return 1024;
1279   default:
1280     llvm_unreachable("Unexpected register class");
1281   }
1282 }
1283 
1284 unsigned getRegBitWidth(const MCRegisterClass &RC) {
1285   return getRegBitWidth(RC.getID());
1286 }
1287 
1288 unsigned getRegOperandSize(const MCRegisterInfo *MRI, const MCInstrDesc &Desc,
1289                            unsigned OpNo) {
1290   assert(OpNo < Desc.NumOperands);
1291   unsigned RCID = Desc.OpInfo[OpNo].RegClass;
1292   return getRegBitWidth(MRI->getRegClass(RCID)) / 8;
1293 }
1294 
1295 bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi) {
1296   if (isInlinableIntLiteral(Literal))
1297     return true;
1298 
1299   uint64_t Val = static_cast<uint64_t>(Literal);
1300   return (Val == DoubleToBits(0.0)) ||
1301          (Val == DoubleToBits(1.0)) ||
1302          (Val == DoubleToBits(-1.0)) ||
1303          (Val == DoubleToBits(0.5)) ||
1304          (Val == DoubleToBits(-0.5)) ||
1305          (Val == DoubleToBits(2.0)) ||
1306          (Val == DoubleToBits(-2.0)) ||
1307          (Val == DoubleToBits(4.0)) ||
1308          (Val == DoubleToBits(-4.0)) ||
1309          (Val == 0x3fc45f306dc9c882 && HasInv2Pi);
1310 }
1311 
1312 bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi) {
1313   if (isInlinableIntLiteral(Literal))
1314     return true;
1315 
1316   // The actual type of the operand does not seem to matter as long
1317   // as the bits match one of the inline immediate values.  For example:
1318   //
1319   // -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal,
1320   // so it is a legal inline immediate.
1321   //
1322   // 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in
1323   // floating-point, so it is a legal inline immediate.
1324 
1325   uint32_t Val = static_cast<uint32_t>(Literal);
1326   return (Val == FloatToBits(0.0f)) ||
1327          (Val == FloatToBits(1.0f)) ||
1328          (Val == FloatToBits(-1.0f)) ||
1329          (Val == FloatToBits(0.5f)) ||
1330          (Val == FloatToBits(-0.5f)) ||
1331          (Val == FloatToBits(2.0f)) ||
1332          (Val == FloatToBits(-2.0f)) ||
1333          (Val == FloatToBits(4.0f)) ||
1334          (Val == FloatToBits(-4.0f)) ||
1335          (Val == 0x3e22f983 && HasInv2Pi);
1336 }
1337 
1338 bool isInlinableLiteral16(int16_t Literal, bool HasInv2Pi) {
1339   if (!HasInv2Pi)
1340     return false;
1341 
1342   if (isInlinableIntLiteral(Literal))
1343     return true;
1344 
1345   uint16_t Val = static_cast<uint16_t>(Literal);
1346   return Val == 0x3C00 || // 1.0
1347          Val == 0xBC00 || // -1.0
1348          Val == 0x3800 || // 0.5
1349          Val == 0xB800 || // -0.5
1350          Val == 0x4000 || // 2.0
1351          Val == 0xC000 || // -2.0
1352          Val == 0x4400 || // 4.0
1353          Val == 0xC400 || // -4.0
1354          Val == 0x3118;   // 1/2pi
1355 }
1356 
1357 bool isInlinableLiteralV216(int32_t Literal, bool HasInv2Pi) {
1358   assert(HasInv2Pi);
1359 
1360   if (isInt<16>(Literal) || isUInt<16>(Literal)) {
1361     int16_t Trunc = static_cast<int16_t>(Literal);
1362     return AMDGPU::isInlinableLiteral16(Trunc, HasInv2Pi);
1363   }
1364   if (!(Literal & 0xffff))
1365     return AMDGPU::isInlinableLiteral16(Literal >> 16, HasInv2Pi);
1366 
1367   int16_t Lo16 = static_cast<int16_t>(Literal);
1368   int16_t Hi16 = static_cast<int16_t>(Literal >> 16);
1369   return Lo16 == Hi16 && isInlinableLiteral16(Lo16, HasInv2Pi);
1370 }
1371 
1372 bool isInlinableIntLiteralV216(int32_t Literal) {
1373   int16_t Lo16 = static_cast<int16_t>(Literal);
1374   if (isInt<16>(Literal) || isUInt<16>(Literal))
1375     return isInlinableIntLiteral(Lo16);
1376 
1377   int16_t Hi16 = static_cast<int16_t>(Literal >> 16);
1378   if (!(Literal & 0xffff))
1379     return isInlinableIntLiteral(Hi16);
1380   return Lo16 == Hi16 && isInlinableIntLiteral(Lo16);
1381 }
1382 
1383 bool isFoldableLiteralV216(int32_t Literal, bool HasInv2Pi) {
1384   assert(HasInv2Pi);
1385 
1386   int16_t Lo16 = static_cast<int16_t>(Literal);
1387   if (isInt<16>(Literal) || isUInt<16>(Literal))
1388     return true;
1389 
1390   int16_t Hi16 = static_cast<int16_t>(Literal >> 16);
1391   if (!(Literal & 0xffff))
1392     return true;
1393   return Lo16 == Hi16;
1394 }
1395 
1396 bool isArgPassedInSGPR(const Argument *A) {
1397   const Function *F = A->getParent();
1398 
1399   // Arguments to compute shaders are never a source of divergence.
1400   CallingConv::ID CC = F->getCallingConv();
1401   switch (CC) {
1402   case CallingConv::AMDGPU_KERNEL:
1403   case CallingConv::SPIR_KERNEL:
1404     return true;
1405   case CallingConv::AMDGPU_VS:
1406   case CallingConv::AMDGPU_LS:
1407   case CallingConv::AMDGPU_HS:
1408   case CallingConv::AMDGPU_ES:
1409   case CallingConv::AMDGPU_GS:
1410   case CallingConv::AMDGPU_PS:
1411   case CallingConv::AMDGPU_CS:
1412     // For non-compute shaders, SGPR inputs are marked with either inreg or byval.
1413     // Everything else is in VGPRs.
1414     return F->getAttributes().hasParamAttribute(A->getArgNo(), Attribute::InReg) ||
1415            F->getAttributes().hasParamAttribute(A->getArgNo(), Attribute::ByVal);
1416   default:
1417     // TODO: Should calls support inreg for SGPR inputs?
1418     return false;
1419   }
1420 }
1421 
1422 static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) {
1423   return isGCN3Encoding(ST) || isGFX10(ST);
1424 }
1425 
1426 static bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST) {
1427   return isGFX9(ST) || isGFX10(ST);
1428 }
1429 
1430 bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST,
1431                                       int64_t EncodedOffset) {
1432   return hasSMEMByteOffset(ST) ? isUInt<20>(EncodedOffset)
1433                                : isUInt<8>(EncodedOffset);
1434 }
1435 
1436 bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST,
1437                                     int64_t EncodedOffset,
1438                                     bool IsBuffer) {
1439   return !IsBuffer &&
1440          hasSMRDSignedImmOffset(ST) &&
1441          isInt<21>(EncodedOffset);
1442 }
1443 
1444 static bool isDwordAligned(uint64_t ByteOffset) {
1445   return (ByteOffset & 3) == 0;
1446 }
1447 
1448 uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST,
1449                                 uint64_t ByteOffset) {
1450   if (hasSMEMByteOffset(ST))
1451     return ByteOffset;
1452 
1453   assert(isDwordAligned(ByteOffset));
1454   return ByteOffset >> 2;
1455 }
1456 
1457 Optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST,
1458                                        int64_t ByteOffset, bool IsBuffer) {
1459   // The signed version is always a byte offset.
1460   if (!IsBuffer && hasSMRDSignedImmOffset(ST)) {
1461     assert(hasSMEMByteOffset(ST));
1462     return isInt<20>(ByteOffset) ? Optional<int64_t>(ByteOffset) : None;
1463   }
1464 
1465   if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST))
1466     return None;
1467 
1468   int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
1469   return isLegalSMRDEncodedUnsignedOffset(ST, EncodedOffset)
1470              ? Optional<int64_t>(EncodedOffset)
1471              : None;
1472 }
1473 
1474 Optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST,
1475                                                 int64_t ByteOffset) {
1476   if (!isCI(ST) || !isDwordAligned(ByteOffset))
1477     return None;
1478 
1479   int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
1480   return isUInt<32>(EncodedOffset) ? Optional<int64_t>(EncodedOffset) : None;
1481 }
1482 
1483 // Given Imm, split it into the values to put into the SOffset and ImmOffset
1484 // fields in an MUBUF instruction. Return false if it is not possible (due to a
1485 // hardware bug needing a workaround).
1486 //
1487 // The required alignment ensures that individual address components remain
1488 // aligned if they are aligned to begin with. It also ensures that additional
1489 // offsets within the given alignment can be added to the resulting ImmOffset.
1490 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset,
1491                       const GCNSubtarget *Subtarget, Align Alignment) {
1492   const uint32_t MaxImm = alignDown(4095, Alignment.value());
1493   uint32_t Overflow = 0;
1494 
1495   if (Imm > MaxImm) {
1496     if (Imm <= MaxImm + 64) {
1497       // Use an SOffset inline constant for 4..64
1498       Overflow = Imm - MaxImm;
1499       Imm = MaxImm;
1500     } else {
1501       // Try to keep the same value in SOffset for adjacent loads, so that
1502       // the corresponding register contents can be re-used.
1503       //
1504       // Load values with all low-bits (except for alignment bits) set into
1505       // SOffset, so that a larger range of values can be covered using
1506       // s_movk_i32.
1507       //
1508       // Atomic operations fail to work correctly when individual address
1509       // components are unaligned, even if their sum is aligned.
1510       uint32_t High = (Imm + Alignment.value()) & ~4095;
1511       uint32_t Low = (Imm + Alignment.value()) & 4095;
1512       Imm = Low;
1513       Overflow = High - Alignment.value();
1514     }
1515   }
1516 
1517   // There is a hardware bug in SI and CI which prevents address clamping in
1518   // MUBUF instructions from working correctly with SOffsets. The immediate
1519   // offset is unaffected.
1520   if (Overflow > 0 &&
1521       Subtarget->getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS)
1522     return false;
1523 
1524   ImmOffset = Imm;
1525   SOffset = Overflow;
1526   return true;
1527 }
1528 
1529 SIModeRegisterDefaults::SIModeRegisterDefaults(const Function &F) {
1530   *this = getDefaultForCallingConv(F.getCallingConv());
1531 
1532   StringRef IEEEAttr = F.getFnAttribute("amdgpu-ieee").getValueAsString();
1533   if (!IEEEAttr.empty())
1534     IEEE = IEEEAttr == "true";
1535 
1536   StringRef DX10ClampAttr
1537     = F.getFnAttribute("amdgpu-dx10-clamp").getValueAsString();
1538   if (!DX10ClampAttr.empty())
1539     DX10Clamp = DX10ClampAttr == "true";
1540 
1541   StringRef DenormF32Attr = F.getFnAttribute("denormal-fp-math-f32").getValueAsString();
1542   if (!DenormF32Attr.empty()) {
1543     DenormalMode DenormMode = parseDenormalFPAttribute(DenormF32Attr);
1544     FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE;
1545     FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE;
1546   }
1547 
1548   StringRef DenormAttr = F.getFnAttribute("denormal-fp-math").getValueAsString();
1549   if (!DenormAttr.empty()) {
1550     DenormalMode DenormMode = parseDenormalFPAttribute(DenormAttr);
1551 
1552     if (DenormF32Attr.empty()) {
1553       FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE;
1554       FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE;
1555     }
1556 
1557     FP64FP16InputDenormals = DenormMode.Input == DenormalMode::IEEE;
1558     FP64FP16OutputDenormals = DenormMode.Output == DenormalMode::IEEE;
1559   }
1560 }
1561 
1562 namespace {
1563 
1564 struct SourceOfDivergence {
1565   unsigned Intr;
1566 };
1567 const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr);
1568 
1569 #define GET_SourcesOfDivergence_IMPL
1570 #define GET_Gfx9BufferFormat_IMPL
1571 #define GET_Gfx10PlusBufferFormat_IMPL
1572 #include "AMDGPUGenSearchableTables.inc"
1573 
1574 } // end anonymous namespace
1575 
1576 bool isIntrinsicSourceOfDivergence(unsigned IntrID) {
1577   return lookupSourceOfDivergence(IntrID);
1578 }
1579 
1580 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp,
1581                                                   uint8_t NumComponents,
1582                                                   uint8_t NumFormat,
1583                                                   const MCSubtargetInfo &STI) {
1584   return isGFX10(STI)
1585              ? getGfx10PlusBufferFormatInfo(BitsPerComp, NumComponents,
1586                                             NumFormat)
1587              : getGfx9BufferFormatInfo(BitsPerComp, NumComponents, NumFormat);
1588 }
1589 
1590 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format,
1591                                                   const MCSubtargetInfo &STI) {
1592   return isGFX10(STI) ? getGfx10PlusBufferFormatInfo(Format)
1593                       : getGfx9BufferFormatInfo(Format);
1594 }
1595 
1596 } // namespace AMDGPU
1597 } // namespace llvm
1598