1 //===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDGPUBaseInfo.h"
11 #include "AMDGPUTargetTransformInfo.h"
12 #include "AMDGPU.h"
13 #include "SIDefines.h"
14 #include "llvm/ADT/StringRef.h"
15 #include "llvm/ADT/Triple.h"
16 #include "llvm/BinaryFormat/ELF.h"
17 #include "llvm/CodeGen/MachineMemOperand.h"
18 #include "llvm/IR/Attributes.h"
19 #include "llvm/IR/Constants.h"
20 #include "llvm/IR/Function.h"
21 #include "llvm/IR/GlobalValue.h"
22 #include "llvm/IR/Instruction.h"
23 #include "llvm/IR/LLVMContext.h"
24 #include "llvm/IR/Module.h"
25 #include "llvm/MC/MCContext.h"
26 #include "llvm/MC/MCInstrDesc.h"
27 #include "llvm/MC/MCInstrInfo.h"
28 #include "llvm/MC/MCRegisterInfo.h"
29 #include "llvm/MC/MCSectionELF.h"
30 #include "llvm/MC/MCSubtargetInfo.h"
31 #include "llvm/MC/SubtargetFeature.h"
32 #include "llvm/Support/Casting.h"
33 #include "llvm/Support/ErrorHandling.h"
34 #include "llvm/Support/MathExtras.h"
35 #include <algorithm>
36 #include <cassert>
37 #include <cstdint>
38 #include <cstring>
39 #include <utility>
40 
41 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
42 
43 #define GET_INSTRINFO_NAMED_OPS
44 #define GET_INSTRMAP_INFO
45 #include "AMDGPUGenInstrInfo.inc"
46 #undef GET_INSTRMAP_INFO
47 #undef GET_INSTRINFO_NAMED_OPS
48 
49 namespace {
50 
51 /// \returns Bit mask for given bit \p Shift and bit \p Width.
52 unsigned getBitMask(unsigned Shift, unsigned Width) {
53   return ((1 << Width) - 1) << Shift;
54 }
55 
56 /// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width.
57 ///
58 /// \returns Packed \p Dst.
59 unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) {
60   Dst &= ~(1 << Shift) & ~getBitMask(Shift, Width);
61   Dst |= (Src << Shift) & getBitMask(Shift, Width);
62   return Dst;
63 }
64 
65 /// Unpacks bits from \p Src for given bit \p Shift and bit \p Width.
66 ///
67 /// \returns Unpacked bits.
68 unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) {
69   return (Src & getBitMask(Shift, Width)) >> Shift;
70 }
71 
72 /// \returns Vmcnt bit shift (lower bits).
73 unsigned getVmcntBitShiftLo() { return 0; }
74 
75 /// \returns Vmcnt bit width (lower bits).
76 unsigned getVmcntBitWidthLo() { return 4; }
77 
78 /// \returns Expcnt bit shift.
79 unsigned getExpcntBitShift() { return 4; }
80 
81 /// \returns Expcnt bit width.
82 unsigned getExpcntBitWidth() { return 3; }
83 
84 /// \returns Lgkmcnt bit shift.
85 unsigned getLgkmcntBitShift() { return 8; }
86 
87 /// \returns Lgkmcnt bit width.
88 unsigned getLgkmcntBitWidth() { return 4; }
89 
90 /// \returns Vmcnt bit shift (higher bits).
91 unsigned getVmcntBitShiftHi() { return 14; }
92 
93 /// \returns Vmcnt bit width (higher bits).
94 unsigned getVmcntBitWidthHi() { return 2; }
95 
96 } // end namespace anonymous
97 
98 namespace llvm {
99 
100 namespace AMDGPU {
101 
102 struct MIMGInfo {
103   uint16_t Opcode;
104   uint16_t BaseOpcode;
105   uint8_t MIMGEncoding;
106   uint8_t VDataDwords;
107   uint8_t VAddrDwords;
108 };
109 
110 #define GET_MIMGBaseOpcodesTable_IMPL
111 #define GET_MIMGDimInfoTable_IMPL
112 #define GET_MIMGInfoTable_IMPL
113 #define GET_MIMGLZMappingTable_IMPL
114 #include "AMDGPUGenSearchableTables.inc"
115 
116 int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding,
117                   unsigned VDataDwords, unsigned VAddrDwords) {
118   const MIMGInfo *Info = getMIMGOpcodeHelper(BaseOpcode, MIMGEncoding,
119                                              VDataDwords, VAddrDwords);
120   return Info ? Info->Opcode : -1;
121 }
122 
123 int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) {
124   const MIMGInfo *OrigInfo = getMIMGInfo(Opc);
125   const MIMGInfo *NewInfo =
126       getMIMGOpcodeHelper(OrigInfo->BaseOpcode, OrigInfo->MIMGEncoding,
127                           NewChannels, OrigInfo->VAddrDwords);
128   return NewInfo ? NewInfo->Opcode : -1;
129 }
130 
131 // Wrapper for Tablegen'd function.  enum Subtarget is not defined in any
132 // header files, so we need to wrap it in a function that takes unsigned
133 // instead.
134 int getMCOpcode(uint16_t Opcode, unsigned Gen) {
135   return getMCOpcodeGen(Opcode, static_cast<Subtarget>(Gen));
136 }
137 
138 namespace IsaInfo {
139 
140 void streamIsaVersion(const MCSubtargetInfo *STI, raw_ostream &Stream) {
141   auto TargetTriple = STI->getTargetTriple();
142   auto Version = getIsaVersion(STI->getCPU());
143 
144   Stream << TargetTriple.getArchName() << '-'
145          << TargetTriple.getVendorName() << '-'
146          << TargetTriple.getOSName() << '-'
147          << TargetTriple.getEnvironmentName() << '-'
148          << "gfx"
149          << Version.Major
150          << Version.Minor
151          << Version.Stepping;
152 
153   if (hasXNACK(*STI))
154     Stream << "+xnack";
155   if (hasSRAMECC(*STI))
156     Stream << "+sram-ecc";
157 
158   Stream.flush();
159 }
160 
161 bool hasCodeObjectV3(const MCSubtargetInfo *STI) {
162   return STI->getTargetTriple().getOS() == Triple::AMDHSA &&
163              STI->getFeatureBits().test(FeatureCodeObjectV3);
164 }
165 
166 unsigned getWavefrontSize(const MCSubtargetInfo *STI) {
167   if (STI->getFeatureBits().test(FeatureWavefrontSize16))
168     return 16;
169   if (STI->getFeatureBits().test(FeatureWavefrontSize32))
170     return 32;
171 
172   return 64;
173 }
174 
175 unsigned getLocalMemorySize(const MCSubtargetInfo *STI) {
176   if (STI->getFeatureBits().test(FeatureLocalMemorySize32768))
177     return 32768;
178   if (STI->getFeatureBits().test(FeatureLocalMemorySize65536))
179     return 65536;
180 
181   return 0;
182 }
183 
184 unsigned getEUsPerCU(const MCSubtargetInfo *STI) {
185   return 4;
186 }
187 
188 unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo *STI,
189                                unsigned FlatWorkGroupSize) {
190   if (!STI->getFeatureBits().test(FeatureGCN))
191     return 8;
192   unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize);
193   if (N == 1)
194     return 40;
195   N = 40 / N;
196   return std::min(N, 16u);
197 }
198 
199 unsigned getMaxWavesPerCU(const MCSubtargetInfo *STI) {
200   return getMaxWavesPerEU() * getEUsPerCU(STI);
201 }
202 
203 unsigned getMaxWavesPerCU(const MCSubtargetInfo *STI,
204                           unsigned FlatWorkGroupSize) {
205   return getWavesPerWorkGroup(STI, FlatWorkGroupSize);
206 }
207 
208 unsigned getMinWavesPerEU(const MCSubtargetInfo *STI) {
209   return 1;
210 }
211 
212 unsigned getMaxWavesPerEU() {
213   // FIXME: Need to take scratch memory into account.
214   return 10;
215 }
216 
217 unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI,
218                           unsigned FlatWorkGroupSize) {
219   return alignTo(getMaxWavesPerCU(STI, FlatWorkGroupSize),
220                  getEUsPerCU(STI)) / getEUsPerCU(STI);
221 }
222 
223 unsigned getMinFlatWorkGroupSize(const MCSubtargetInfo *STI) {
224   return 1;
225 }
226 
227 unsigned getMaxFlatWorkGroupSize(const MCSubtargetInfo *STI) {
228   return 2048;
229 }
230 
231 unsigned getWavesPerWorkGroup(const MCSubtargetInfo *STI,
232                               unsigned FlatWorkGroupSize) {
233   return alignTo(FlatWorkGroupSize, getWavefrontSize(STI)) /
234                  getWavefrontSize(STI);
235 }
236 
237 unsigned getSGPRAllocGranule(const MCSubtargetInfo *STI) {
238   IsaVersion Version = getIsaVersion(STI->getCPU());
239   if (Version.Major >= 8)
240     return 16;
241   return 8;
242 }
243 
244 unsigned getSGPREncodingGranule(const MCSubtargetInfo *STI) {
245   return 8;
246 }
247 
248 unsigned getTotalNumSGPRs(const MCSubtargetInfo *STI) {
249   IsaVersion Version = getIsaVersion(STI->getCPU());
250   if (Version.Major >= 8)
251     return 800;
252   return 512;
253 }
254 
255 unsigned getAddressableNumSGPRs(const MCSubtargetInfo *STI) {
256   if (STI->getFeatureBits().test(FeatureSGPRInitBug))
257     return FIXED_NUM_SGPRS_FOR_INIT_BUG;
258 
259   IsaVersion Version = getIsaVersion(STI->getCPU());
260   if (Version.Major >= 8)
261     return 102;
262   return 104;
263 }
264 
265 unsigned getMinNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
266   assert(WavesPerEU != 0);
267 
268   if (WavesPerEU >= getMaxWavesPerEU())
269     return 0;
270 
271   unsigned MinNumSGPRs = getTotalNumSGPRs(STI) / (WavesPerEU + 1);
272   if (STI->getFeatureBits().test(FeatureTrapHandler))
273     MinNumSGPRs -= std::min(MinNumSGPRs, (unsigned)TRAP_NUM_SGPRS);
274   MinNumSGPRs = alignDown(MinNumSGPRs, getSGPRAllocGranule(STI)) + 1;
275   return std::min(MinNumSGPRs, getAddressableNumSGPRs(STI));
276 }
277 
278 unsigned getMaxNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU,
279                         bool Addressable) {
280   assert(WavesPerEU != 0);
281 
282   IsaVersion Version = getIsaVersion(STI->getCPU());
283   unsigned AddressableNumSGPRs = getAddressableNumSGPRs(STI);
284   if (Version.Major >= 8 && !Addressable)
285     AddressableNumSGPRs = 112;
286   unsigned MaxNumSGPRs = getTotalNumSGPRs(STI) / WavesPerEU;
287   if (STI->getFeatureBits().test(FeatureTrapHandler))
288     MaxNumSGPRs -= std::min(MaxNumSGPRs, (unsigned)TRAP_NUM_SGPRS);
289   MaxNumSGPRs = alignDown(MaxNumSGPRs, getSGPRAllocGranule(STI));
290   return std::min(MaxNumSGPRs, AddressableNumSGPRs);
291 }
292 
293 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed,
294                           bool FlatScrUsed, bool XNACKUsed) {
295   unsigned ExtraSGPRs = 0;
296   if (VCCUsed)
297     ExtraSGPRs = 2;
298 
299   IsaVersion Version = getIsaVersion(STI->getCPU());
300   if (Version.Major < 8) {
301     if (FlatScrUsed)
302       ExtraSGPRs = 4;
303   } else {
304     if (XNACKUsed)
305       ExtraSGPRs = 4;
306 
307     if (FlatScrUsed)
308       ExtraSGPRs = 6;
309   }
310 
311   return ExtraSGPRs;
312 }
313 
314 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed,
315                           bool FlatScrUsed) {
316   return getNumExtraSGPRs(STI, VCCUsed, FlatScrUsed,
317                           STI->getFeatureBits().test(AMDGPU::FeatureXNACK));
318 }
319 
320 unsigned getNumSGPRBlocks(const MCSubtargetInfo *STI, unsigned NumSGPRs) {
321   NumSGPRs = alignTo(std::max(1u, NumSGPRs), getSGPREncodingGranule(STI));
322   // SGPRBlocks is actual number of SGPR blocks minus 1.
323   return NumSGPRs / getSGPREncodingGranule(STI) - 1;
324 }
325 
326 unsigned getVGPRAllocGranule(const MCSubtargetInfo *STI) {
327   return 4;
328 }
329 
330 unsigned getVGPREncodingGranule(const MCSubtargetInfo *STI) {
331   return getVGPRAllocGranule(STI);
332 }
333 
334 unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) {
335   return 256;
336 }
337 
338 unsigned getAddressableNumVGPRs(const MCSubtargetInfo *STI) {
339   return getTotalNumVGPRs(STI);
340 }
341 
342 unsigned getMinNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
343   assert(WavesPerEU != 0);
344 
345   if (WavesPerEU >= getMaxWavesPerEU())
346     return 0;
347   unsigned MinNumVGPRs =
348       alignDown(getTotalNumVGPRs(STI) / (WavesPerEU + 1),
349                 getVGPRAllocGranule(STI)) + 1;
350   return std::min(MinNumVGPRs, getAddressableNumVGPRs(STI));
351 }
352 
353 unsigned getMaxNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
354   assert(WavesPerEU != 0);
355 
356   unsigned MaxNumVGPRs = alignDown(getTotalNumVGPRs(STI) / WavesPerEU,
357                                    getVGPRAllocGranule(STI));
358   unsigned AddressableNumVGPRs = getAddressableNumVGPRs(STI);
359   return std::min(MaxNumVGPRs, AddressableNumVGPRs);
360 }
361 
362 unsigned getNumVGPRBlocks(const MCSubtargetInfo *STI, unsigned NumVGPRs) {
363   NumVGPRs = alignTo(std::max(1u, NumVGPRs), getVGPREncodingGranule(STI));
364   // VGPRBlocks is actual number of VGPR blocks minus 1.
365   return NumVGPRs / getVGPREncodingGranule(STI) - 1;
366 }
367 
368 } // end namespace IsaInfo
369 
370 void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header,
371                                const MCSubtargetInfo *STI) {
372   IsaVersion Version = getIsaVersion(STI->getCPU());
373 
374   memset(&Header, 0, sizeof(Header));
375 
376   Header.amd_kernel_code_version_major = 1;
377   Header.amd_kernel_code_version_minor = 2;
378   Header.amd_machine_kind = 1; // AMD_MACHINE_KIND_AMDGPU
379   Header.amd_machine_version_major = Version.Major;
380   Header.amd_machine_version_minor = Version.Minor;
381   Header.amd_machine_version_stepping = Version.Stepping;
382   Header.kernel_code_entry_byte_offset = sizeof(Header);
383   // wavefront_size is specified as a power of 2: 2^6 = 64 threads.
384   Header.wavefront_size = 6;
385 
386   // If the code object does not support indirect functions, then the value must
387   // be 0xffffffff.
388   Header.call_convention = -1;
389 
390   // These alignment values are specified in powers of two, so alignment =
391   // 2^n.  The minimum alignment is 2^4 = 16.
392   Header.kernarg_segment_alignment = 4;
393   Header.group_segment_alignment = 4;
394   Header.private_segment_alignment = 4;
395 }
396 
397 amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor() {
398   amdhsa::kernel_descriptor_t KD;
399   memset(&KD, 0, sizeof(KD));
400   AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
401                   amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64,
402                   amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE);
403   AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
404                   amdhsa::COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP, 1);
405   AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
406                   amdhsa::COMPUTE_PGM_RSRC1_ENABLE_IEEE_MODE, 1);
407   AMDHSA_BITS_SET(KD.compute_pgm_rsrc2,
408                   amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, 1);
409   return KD;
410 }
411 
412 bool isGroupSegment(const GlobalValue *GV) {
413   return GV->getType()->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS;
414 }
415 
416 bool isGlobalSegment(const GlobalValue *GV) {
417   return GV->getType()->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS;
418 }
419 
420 bool isReadOnlySegment(const GlobalValue *GV) {
421   return GV->getType()->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
422          GV->getType()->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
423 }
424 
425 bool shouldEmitConstantsToTextSection(const Triple &TT) {
426   return TT.getOS() != Triple::AMDHSA;
427 }
428 
429 int getIntegerAttribute(const Function &F, StringRef Name, int Default) {
430   Attribute A = F.getFnAttribute(Name);
431   int Result = Default;
432 
433   if (A.isStringAttribute()) {
434     StringRef Str = A.getValueAsString();
435     if (Str.getAsInteger(0, Result)) {
436       LLVMContext &Ctx = F.getContext();
437       Ctx.emitError("can't parse integer attribute " + Name);
438     }
439   }
440 
441   return Result;
442 }
443 
444 std::pair<int, int> getIntegerPairAttribute(const Function &F,
445                                             StringRef Name,
446                                             std::pair<int, int> Default,
447                                             bool OnlyFirstRequired) {
448   Attribute A = F.getFnAttribute(Name);
449   if (!A.isStringAttribute())
450     return Default;
451 
452   LLVMContext &Ctx = F.getContext();
453   std::pair<int, int> Ints = Default;
454   std::pair<StringRef, StringRef> Strs = A.getValueAsString().split(',');
455   if (Strs.first.trim().getAsInteger(0, Ints.first)) {
456     Ctx.emitError("can't parse first integer attribute " + Name);
457     return Default;
458   }
459   if (Strs.second.trim().getAsInteger(0, Ints.second)) {
460     if (!OnlyFirstRequired || !Strs.second.trim().empty()) {
461       Ctx.emitError("can't parse second integer attribute " + Name);
462       return Default;
463     }
464   }
465 
466   return Ints;
467 }
468 
469 unsigned getVmcntBitMask(const IsaVersion &Version) {
470   unsigned VmcntLo = (1 << getVmcntBitWidthLo()) - 1;
471   if (Version.Major < 9)
472     return VmcntLo;
473 
474   unsigned VmcntHi = ((1 << getVmcntBitWidthHi()) - 1) << getVmcntBitWidthLo();
475   return VmcntLo | VmcntHi;
476 }
477 
478 unsigned getExpcntBitMask(const IsaVersion &Version) {
479   return (1 << getExpcntBitWidth()) - 1;
480 }
481 
482 unsigned getLgkmcntBitMask(const IsaVersion &Version) {
483   return (1 << getLgkmcntBitWidth()) - 1;
484 }
485 
486 unsigned getWaitcntBitMask(const IsaVersion &Version) {
487   unsigned VmcntLo = getBitMask(getVmcntBitShiftLo(), getVmcntBitWidthLo());
488   unsigned Expcnt = getBitMask(getExpcntBitShift(), getExpcntBitWidth());
489   unsigned Lgkmcnt = getBitMask(getLgkmcntBitShift(), getLgkmcntBitWidth());
490   unsigned Waitcnt = VmcntLo | Expcnt | Lgkmcnt;
491   if (Version.Major < 9)
492     return Waitcnt;
493 
494   unsigned VmcntHi = getBitMask(getVmcntBitShiftHi(), getVmcntBitWidthHi());
495   return Waitcnt | VmcntHi;
496 }
497 
498 unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) {
499   unsigned VmcntLo =
500       unpackBits(Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo());
501   if (Version.Major < 9)
502     return VmcntLo;
503 
504   unsigned VmcntHi =
505       unpackBits(Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi());
506   VmcntHi <<= getVmcntBitWidthLo();
507   return VmcntLo | VmcntHi;
508 }
509 
510 unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) {
511   return unpackBits(Waitcnt, getExpcntBitShift(), getExpcntBitWidth());
512 }
513 
514 unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) {
515   return unpackBits(Waitcnt, getLgkmcntBitShift(), getLgkmcntBitWidth());
516 }
517 
518 void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt,
519                    unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt) {
520   Vmcnt = decodeVmcnt(Version, Waitcnt);
521   Expcnt = decodeExpcnt(Version, Waitcnt);
522   Lgkmcnt = decodeLgkmcnt(Version, Waitcnt);
523 }
524 
525 Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded) {
526   Waitcnt Decoded;
527   Decoded.VmCnt = decodeVmcnt(Version, Encoded);
528   Decoded.ExpCnt = decodeExpcnt(Version, Encoded);
529   Decoded.LgkmCnt = decodeLgkmcnt(Version, Encoded);
530   return Decoded;
531 }
532 
533 unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt,
534                      unsigned Vmcnt) {
535   Waitcnt =
536       packBits(Vmcnt, Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo());
537   if (Version.Major < 9)
538     return Waitcnt;
539 
540   Vmcnt >>= getVmcntBitWidthLo();
541   return packBits(Vmcnt, Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi());
542 }
543 
544 unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt,
545                       unsigned Expcnt) {
546   return packBits(Expcnt, Waitcnt, getExpcntBitShift(), getExpcntBitWidth());
547 }
548 
549 unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt,
550                        unsigned Lgkmcnt) {
551   return packBits(Lgkmcnt, Waitcnt, getLgkmcntBitShift(), getLgkmcntBitWidth());
552 }
553 
554 unsigned encodeWaitcnt(const IsaVersion &Version,
555                        unsigned Vmcnt, unsigned Expcnt, unsigned Lgkmcnt) {
556   unsigned Waitcnt = getWaitcntBitMask(Version);
557   Waitcnt = encodeVmcnt(Version, Waitcnt, Vmcnt);
558   Waitcnt = encodeExpcnt(Version, Waitcnt, Expcnt);
559   Waitcnt = encodeLgkmcnt(Version, Waitcnt, Lgkmcnt);
560   return Waitcnt;
561 }
562 
563 unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded) {
564   return encodeWaitcnt(Version, Decoded.VmCnt, Decoded.ExpCnt, Decoded.LgkmCnt);
565 }
566 
567 unsigned getInitialPSInputAddr(const Function &F) {
568   return getIntegerAttribute(F, "InitialPSInputAddr", 0);
569 }
570 
571 bool isShader(CallingConv::ID cc) {
572   switch(cc) {
573     case CallingConv::AMDGPU_VS:
574     case CallingConv::AMDGPU_LS:
575     case CallingConv::AMDGPU_HS:
576     case CallingConv::AMDGPU_ES:
577     case CallingConv::AMDGPU_GS:
578     case CallingConv::AMDGPU_PS:
579     case CallingConv::AMDGPU_CS:
580       return true;
581     default:
582       return false;
583   }
584 }
585 
586 bool isCompute(CallingConv::ID cc) {
587   return !isShader(cc) || cc == CallingConv::AMDGPU_CS;
588 }
589 
590 bool isEntryFunctionCC(CallingConv::ID CC) {
591   switch (CC) {
592   case CallingConv::AMDGPU_KERNEL:
593   case CallingConv::SPIR_KERNEL:
594   case CallingConv::AMDGPU_VS:
595   case CallingConv::AMDGPU_GS:
596   case CallingConv::AMDGPU_PS:
597   case CallingConv::AMDGPU_CS:
598   case CallingConv::AMDGPU_ES:
599   case CallingConv::AMDGPU_HS:
600   case CallingConv::AMDGPU_LS:
601     return true;
602   default:
603     return false;
604   }
605 }
606 
607 bool hasXNACK(const MCSubtargetInfo &STI) {
608   return STI.getFeatureBits()[AMDGPU::FeatureXNACK];
609 }
610 
611 bool hasSRAMECC(const MCSubtargetInfo &STI) {
612   return STI.getFeatureBits()[AMDGPU::FeatureSRAMECC];
613 }
614 
615 bool hasMIMG_R128(const MCSubtargetInfo &STI) {
616   return STI.getFeatureBits()[AMDGPU::FeatureMIMG_R128];
617 }
618 
619 bool hasPackedD16(const MCSubtargetInfo &STI) {
620   return !STI.getFeatureBits()[AMDGPU::FeatureUnpackedD16VMem];
621 }
622 
623 bool isSI(const MCSubtargetInfo &STI) {
624   return STI.getFeatureBits()[AMDGPU::FeatureSouthernIslands];
625 }
626 
627 bool isCI(const MCSubtargetInfo &STI) {
628   return STI.getFeatureBits()[AMDGPU::FeatureSeaIslands];
629 }
630 
631 bool isVI(const MCSubtargetInfo &STI) {
632   return STI.getFeatureBits()[AMDGPU::FeatureVolcanicIslands];
633 }
634 
635 bool isGFX9(const MCSubtargetInfo &STI) {
636   return STI.getFeatureBits()[AMDGPU::FeatureGFX9];
637 }
638 
639 bool isGCN3Encoding(const MCSubtargetInfo &STI) {
640   return STI.getFeatureBits()[AMDGPU::FeatureGCN3Encoding];
641 }
642 
643 bool isSGPR(unsigned Reg, const MCRegisterInfo* TRI) {
644   const MCRegisterClass SGPRClass = TRI->getRegClass(AMDGPU::SReg_32RegClassID);
645   const unsigned FirstSubReg = TRI->getSubReg(Reg, 1);
646   return SGPRClass.contains(FirstSubReg != 0 ? FirstSubReg : Reg) ||
647     Reg == AMDGPU::SCC;
648 }
649 
650 bool isRegIntersect(unsigned Reg0, unsigned Reg1, const MCRegisterInfo* TRI) {
651   for (MCRegAliasIterator R(Reg0, TRI, true); R.isValid(); ++R) {
652     if (*R == Reg1) return true;
653   }
654   return false;
655 }
656 
657 #define MAP_REG2REG \
658   using namespace AMDGPU; \
659   switch(Reg) { \
660   default: return Reg; \
661   CASE_CI_VI(FLAT_SCR) \
662   CASE_CI_VI(FLAT_SCR_LO) \
663   CASE_CI_VI(FLAT_SCR_HI) \
664   CASE_VI_GFX9(TTMP0) \
665   CASE_VI_GFX9(TTMP1) \
666   CASE_VI_GFX9(TTMP2) \
667   CASE_VI_GFX9(TTMP3) \
668   CASE_VI_GFX9(TTMP4) \
669   CASE_VI_GFX9(TTMP5) \
670   CASE_VI_GFX9(TTMP6) \
671   CASE_VI_GFX9(TTMP7) \
672   CASE_VI_GFX9(TTMP8) \
673   CASE_VI_GFX9(TTMP9) \
674   CASE_VI_GFX9(TTMP10) \
675   CASE_VI_GFX9(TTMP11) \
676   CASE_VI_GFX9(TTMP12) \
677   CASE_VI_GFX9(TTMP13) \
678   CASE_VI_GFX9(TTMP14) \
679   CASE_VI_GFX9(TTMP15) \
680   CASE_VI_GFX9(TTMP0_TTMP1) \
681   CASE_VI_GFX9(TTMP2_TTMP3) \
682   CASE_VI_GFX9(TTMP4_TTMP5) \
683   CASE_VI_GFX9(TTMP6_TTMP7) \
684   CASE_VI_GFX9(TTMP8_TTMP9) \
685   CASE_VI_GFX9(TTMP10_TTMP11) \
686   CASE_VI_GFX9(TTMP12_TTMP13) \
687   CASE_VI_GFX9(TTMP14_TTMP15) \
688   CASE_VI_GFX9(TTMP0_TTMP1_TTMP2_TTMP3) \
689   CASE_VI_GFX9(TTMP4_TTMP5_TTMP6_TTMP7) \
690   CASE_VI_GFX9(TTMP8_TTMP9_TTMP10_TTMP11) \
691   CASE_VI_GFX9(TTMP12_TTMP13_TTMP14_TTMP15) \
692   CASE_VI_GFX9(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7) \
693   CASE_VI_GFX9(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11) \
694   CASE_VI_GFX9(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
695   CASE_VI_GFX9(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
696   }
697 
698 #define CASE_CI_VI(node) \
699   assert(!isSI(STI)); \
700   case node: return isCI(STI) ? node##_ci : node##_vi;
701 
702 #define CASE_VI_GFX9(node) \
703   case node: return isGFX9(STI) ? node##_gfx9 : node##_vi;
704 
705 unsigned getMCReg(unsigned Reg, const MCSubtargetInfo &STI) {
706   if (STI.getTargetTriple().getArch() == Triple::r600)
707     return Reg;
708   MAP_REG2REG
709 }
710 
711 #undef CASE_CI_VI
712 #undef CASE_VI_GFX9
713 
714 #define CASE_CI_VI(node)   case node##_ci: case node##_vi:   return node;
715 #define CASE_VI_GFX9(node) case node##_vi: case node##_gfx9: return node;
716 
717 unsigned mc2PseudoReg(unsigned Reg) {
718   MAP_REG2REG
719 }
720 
721 #undef CASE_CI_VI
722 #undef CASE_VI_GFX9
723 #undef MAP_REG2REG
724 
725 bool isSISrcOperand(const MCInstrDesc &Desc, unsigned OpNo) {
726   assert(OpNo < Desc.NumOperands);
727   unsigned OpType = Desc.OpInfo[OpNo].OperandType;
728   return OpType >= AMDGPU::OPERAND_SRC_FIRST &&
729          OpType <= AMDGPU::OPERAND_SRC_LAST;
730 }
731 
732 bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo) {
733   assert(OpNo < Desc.NumOperands);
734   unsigned OpType = Desc.OpInfo[OpNo].OperandType;
735   switch (OpType) {
736   case AMDGPU::OPERAND_REG_IMM_FP32:
737   case AMDGPU::OPERAND_REG_IMM_FP64:
738   case AMDGPU::OPERAND_REG_IMM_FP16:
739   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
740   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
741   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
742   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
743     return true;
744   default:
745     return false;
746   }
747 }
748 
749 bool isSISrcInlinableOperand(const MCInstrDesc &Desc, unsigned OpNo) {
750   assert(OpNo < Desc.NumOperands);
751   unsigned OpType = Desc.OpInfo[OpNo].OperandType;
752   return OpType >= AMDGPU::OPERAND_REG_INLINE_C_FIRST &&
753          OpType <= AMDGPU::OPERAND_REG_INLINE_C_LAST;
754 }
755 
756 // Avoid using MCRegisterClass::getSize, since that function will go away
757 // (move from MC* level to Target* level). Return size in bits.
758 unsigned getRegBitWidth(unsigned RCID) {
759   switch (RCID) {
760   case AMDGPU::SGPR_32RegClassID:
761   case AMDGPU::VGPR_32RegClassID:
762   case AMDGPU::VS_32RegClassID:
763   case AMDGPU::SReg_32RegClassID:
764   case AMDGPU::SReg_32_XM0RegClassID:
765     return 32;
766   case AMDGPU::SGPR_64RegClassID:
767   case AMDGPU::VS_64RegClassID:
768   case AMDGPU::SReg_64RegClassID:
769   case AMDGPU::VReg_64RegClassID:
770   case AMDGPU::SReg_64_XEXECRegClassID:
771     return 64;
772   case AMDGPU::VReg_96RegClassID:
773     return 96;
774   case AMDGPU::SGPR_128RegClassID:
775   case AMDGPU::SReg_128RegClassID:
776   case AMDGPU::VReg_128RegClassID:
777     return 128;
778   case AMDGPU::SReg_256RegClassID:
779   case AMDGPU::VReg_256RegClassID:
780     return 256;
781   case AMDGPU::SReg_512RegClassID:
782   case AMDGPU::VReg_512RegClassID:
783     return 512;
784   default:
785     llvm_unreachable("Unexpected register class");
786   }
787 }
788 
789 unsigned getRegBitWidth(const MCRegisterClass &RC) {
790   return getRegBitWidth(RC.getID());
791 }
792 
793 unsigned getRegOperandSize(const MCRegisterInfo *MRI, const MCInstrDesc &Desc,
794                            unsigned OpNo) {
795   assert(OpNo < Desc.NumOperands);
796   unsigned RCID = Desc.OpInfo[OpNo].RegClass;
797   return getRegBitWidth(MRI->getRegClass(RCID)) / 8;
798 }
799 
800 bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi) {
801   if (Literal >= -16 && Literal <= 64)
802     return true;
803 
804   uint64_t Val = static_cast<uint64_t>(Literal);
805   return (Val == DoubleToBits(0.0)) ||
806          (Val == DoubleToBits(1.0)) ||
807          (Val == DoubleToBits(-1.0)) ||
808          (Val == DoubleToBits(0.5)) ||
809          (Val == DoubleToBits(-0.5)) ||
810          (Val == DoubleToBits(2.0)) ||
811          (Val == DoubleToBits(-2.0)) ||
812          (Val == DoubleToBits(4.0)) ||
813          (Val == DoubleToBits(-4.0)) ||
814          (Val == 0x3fc45f306dc9c882 && HasInv2Pi);
815 }
816 
817 bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi) {
818   if (Literal >= -16 && Literal <= 64)
819     return true;
820 
821   // The actual type of the operand does not seem to matter as long
822   // as the bits match one of the inline immediate values.  For example:
823   //
824   // -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal,
825   // so it is a legal inline immediate.
826   //
827   // 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in
828   // floating-point, so it is a legal inline immediate.
829 
830   uint32_t Val = static_cast<uint32_t>(Literal);
831   return (Val == FloatToBits(0.0f)) ||
832          (Val == FloatToBits(1.0f)) ||
833          (Val == FloatToBits(-1.0f)) ||
834          (Val == FloatToBits(0.5f)) ||
835          (Val == FloatToBits(-0.5f)) ||
836          (Val == FloatToBits(2.0f)) ||
837          (Val == FloatToBits(-2.0f)) ||
838          (Val == FloatToBits(4.0f)) ||
839          (Val == FloatToBits(-4.0f)) ||
840          (Val == 0x3e22f983 && HasInv2Pi);
841 }
842 
843 bool isInlinableLiteral16(int16_t Literal, bool HasInv2Pi) {
844   if (!HasInv2Pi)
845     return false;
846 
847   if (Literal >= -16 && Literal <= 64)
848     return true;
849 
850   uint16_t Val = static_cast<uint16_t>(Literal);
851   return Val == 0x3C00 || // 1.0
852          Val == 0xBC00 || // -1.0
853          Val == 0x3800 || // 0.5
854          Val == 0xB800 || // -0.5
855          Val == 0x4000 || // 2.0
856          Val == 0xC000 || // -2.0
857          Val == 0x4400 || // 4.0
858          Val == 0xC400 || // -4.0
859          Val == 0x3118;   // 1/2pi
860 }
861 
862 bool isInlinableLiteralV216(int32_t Literal, bool HasInv2Pi) {
863   assert(HasInv2Pi);
864 
865   int16_t Lo16 = static_cast<int16_t>(Literal);
866   int16_t Hi16 = static_cast<int16_t>(Literal >> 16);
867   return Lo16 == Hi16 && isInlinableLiteral16(Lo16, HasInv2Pi);
868 }
869 
870 bool isArgPassedInSGPR(const Argument *A) {
871   const Function *F = A->getParent();
872 
873   // Arguments to compute shaders are never a source of divergence.
874   CallingConv::ID CC = F->getCallingConv();
875   switch (CC) {
876   case CallingConv::AMDGPU_KERNEL:
877   case CallingConv::SPIR_KERNEL:
878     return true;
879   case CallingConv::AMDGPU_VS:
880   case CallingConv::AMDGPU_LS:
881   case CallingConv::AMDGPU_HS:
882   case CallingConv::AMDGPU_ES:
883   case CallingConv::AMDGPU_GS:
884   case CallingConv::AMDGPU_PS:
885   case CallingConv::AMDGPU_CS:
886     // For non-compute shaders, SGPR inputs are marked with either inreg or byval.
887     // Everything else is in VGPRs.
888     return F->getAttributes().hasParamAttribute(A->getArgNo(), Attribute::InReg) ||
889            F->getAttributes().hasParamAttribute(A->getArgNo(), Attribute::ByVal);
890   default:
891     // TODO: Should calls support inreg for SGPR inputs?
892     return false;
893   }
894 }
895 
896 int64_t getSMRDEncodedOffset(const MCSubtargetInfo &ST, int64_t ByteOffset) {
897   if (isGCN3Encoding(ST))
898     return ByteOffset;
899   return ByteOffset >> 2;
900 }
901 
902 bool isLegalSMRDImmOffset(const MCSubtargetInfo &ST, int64_t ByteOffset) {
903   int64_t EncodedOffset = getSMRDEncodedOffset(ST, ByteOffset);
904   return isGCN3Encoding(ST) ?
905     isUInt<20>(EncodedOffset) : isUInt<8>(EncodedOffset);
906 }
907 
908 // Given Imm, split it into the values to put into the SOffset and ImmOffset
909 // fields in an MUBUF instruction. Return false if it is not possible (due to a
910 // hardware bug needing a workaround).
911 //
912 // The required alignment ensures that individual address components remain
913 // aligned if they are aligned to begin with. It also ensures that additional
914 // offsets within the given alignment can be added to the resulting ImmOffset.
915 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset,
916                       const GCNSubtarget *Subtarget, uint32_t Align) {
917   const uint32_t MaxImm = alignDown(4095, Align);
918   uint32_t Overflow = 0;
919 
920   if (Imm > MaxImm) {
921     if (Imm <= MaxImm + 64) {
922       // Use an SOffset inline constant for 4..64
923       Overflow = Imm - MaxImm;
924       Imm = MaxImm;
925     } else {
926       // Try to keep the same value in SOffset for adjacent loads, so that
927       // the corresponding register contents can be re-used.
928       //
929       // Load values with all low-bits (except for alignment bits) set into
930       // SOffset, so that a larger range of values can be covered using
931       // s_movk_i32.
932       //
933       // Atomic operations fail to work correctly when individual address
934       // components are unaligned, even if their sum is aligned.
935       uint32_t High = (Imm + Align) & ~4095;
936       uint32_t Low = (Imm + Align) & 4095;
937       Imm = Low;
938       Overflow = High - Align;
939     }
940   }
941 
942   // There is a hardware bug in SI and CI which prevents address clamping in
943   // MUBUF instructions from working correctly with SOffsets. The immediate
944   // offset is unaffected.
945   if (Overflow > 0 &&
946       Subtarget->getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS)
947     return false;
948 
949   ImmOffset = Imm;
950   SOffset = Overflow;
951   return true;
952 }
953 
954 namespace {
955 
956 struct SourceOfDivergence {
957   unsigned Intr;
958 };
959 const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr);
960 
961 #define GET_SourcesOfDivergence_IMPL
962 #include "AMDGPUGenSearchableTables.inc"
963 
964 } // end anonymous namespace
965 
966 bool isIntrinsicSourceOfDivergence(unsigned IntrID) {
967   return lookupSourceOfDivergence(IntrID);
968 }
969 } // namespace AMDGPU
970 } // namespace llvm
971