1 //===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "AMDGPUBaseInfo.h"
10 #include "AMDGPUTargetTransformInfo.h"
11 #include "AMDGPU.h"
12 #include "SIDefines.h"
13 #include "AMDGPUAsmUtils.h"
14 #include "llvm/ADT/StringRef.h"
15 #include "llvm/ADT/Triple.h"
16 #include "llvm/BinaryFormat/ELF.h"
17 #include "llvm/CodeGen/MachineMemOperand.h"
18 #include "llvm/IR/Attributes.h"
19 #include "llvm/IR/Constants.h"
20 #include "llvm/IR/Function.h"
21 #include "llvm/IR/GlobalValue.h"
22 #include "llvm/IR/Instruction.h"
23 #include "llvm/IR/LLVMContext.h"
24 #include "llvm/IR/Module.h"
25 #include "llvm/MC/MCContext.h"
26 #include "llvm/MC/MCInstrDesc.h"
27 #include "llvm/MC/MCInstrInfo.h"
28 #include "llvm/MC/MCRegisterInfo.h"
29 #include "llvm/MC/MCSectionELF.h"
30 #include "llvm/MC/MCSubtargetInfo.h"
31 #include "llvm/MC/SubtargetFeature.h"
32 #include "llvm/Support/Casting.h"
33 #include "llvm/Support/ErrorHandling.h"
34 #include "llvm/Support/MathExtras.h"
35 #include <algorithm>
36 #include <cassert>
37 #include <cstdint>
38 #include <cstring>
39 #include <utility>
40 
41 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
42 
43 #define GET_INSTRINFO_NAMED_OPS
44 #define GET_INSTRMAP_INFO
45 #include "AMDGPUGenInstrInfo.inc"
46 #undef GET_INSTRMAP_INFO
47 #undef GET_INSTRINFO_NAMED_OPS
48 
49 namespace {
50 
51 /// \returns Bit mask for given bit \p Shift and bit \p Width.
52 unsigned getBitMask(unsigned Shift, unsigned Width) {
53   return ((1 << Width) - 1) << Shift;
54 }
55 
56 /// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width.
57 ///
58 /// \returns Packed \p Dst.
59 unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) {
60   Dst &= ~(1 << Shift) & ~getBitMask(Shift, Width);
61   Dst |= (Src << Shift) & getBitMask(Shift, Width);
62   return Dst;
63 }
64 
65 /// Unpacks bits from \p Src for given bit \p Shift and bit \p Width.
66 ///
67 /// \returns Unpacked bits.
68 unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) {
69   return (Src & getBitMask(Shift, Width)) >> Shift;
70 }
71 
72 /// \returns Vmcnt bit shift (lower bits).
73 unsigned getVmcntBitShiftLo() { return 0; }
74 
75 /// \returns Vmcnt bit width (lower bits).
76 unsigned getVmcntBitWidthLo() { return 4; }
77 
78 /// \returns Expcnt bit shift.
79 unsigned getExpcntBitShift() { return 4; }
80 
81 /// \returns Expcnt bit width.
82 unsigned getExpcntBitWidth() { return 3; }
83 
84 /// \returns Lgkmcnt bit shift.
85 unsigned getLgkmcntBitShift() { return 8; }
86 
87 /// \returns Lgkmcnt bit width.
88 unsigned getLgkmcntBitWidth(unsigned VersionMajor) {
89   return (VersionMajor >= 10) ? 6 : 4;
90 }
91 
92 /// \returns Vmcnt bit shift (higher bits).
93 unsigned getVmcntBitShiftHi() { return 14; }
94 
95 /// \returns Vmcnt bit width (higher bits).
96 unsigned getVmcntBitWidthHi() { return 2; }
97 
98 } // end namespace anonymous
99 
100 namespace llvm {
101 
102 namespace AMDGPU {
103 
104 #define GET_MIMGBaseOpcodesTable_IMPL
105 #define GET_MIMGDimInfoTable_IMPL
106 #define GET_MIMGInfoTable_IMPL
107 #define GET_MIMGLZMappingTable_IMPL
108 #define GET_MIMGMIPMappingTable_IMPL
109 #include "AMDGPUGenSearchableTables.inc"
110 
111 int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding,
112                   unsigned VDataDwords, unsigned VAddrDwords) {
113   const MIMGInfo *Info = getMIMGOpcodeHelper(BaseOpcode, MIMGEncoding,
114                                              VDataDwords, VAddrDwords);
115   return Info ? Info->Opcode : -1;
116 }
117 
118 const MIMGBaseOpcodeInfo *getMIMGBaseOpcode(unsigned Opc) {
119   const MIMGInfo *Info = getMIMGInfo(Opc);
120   return Info ? getMIMGBaseOpcodeInfo(Info->BaseOpcode) : nullptr;
121 }
122 
123 int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) {
124   const MIMGInfo *OrigInfo = getMIMGInfo(Opc);
125   const MIMGInfo *NewInfo =
126       getMIMGOpcodeHelper(OrigInfo->BaseOpcode, OrigInfo->MIMGEncoding,
127                           NewChannels, OrigInfo->VAddrDwords);
128   return NewInfo ? NewInfo->Opcode : -1;
129 }
130 
131 struct MUBUFInfo {
132   uint16_t Opcode;
133   uint16_t BaseOpcode;
134   uint8_t dwords;
135   bool has_vaddr;
136   bool has_srsrc;
137   bool has_soffset;
138 };
139 
140 #define GET_MUBUFInfoTable_DECL
141 #define GET_MUBUFInfoTable_IMPL
142 #include "AMDGPUGenSearchableTables.inc"
143 
144 int getMUBUFBaseOpcode(unsigned Opc) {
145   const MUBUFInfo *Info = getMUBUFInfoFromOpcode(Opc);
146   return Info ? Info->BaseOpcode : -1;
147 }
148 
149 int getMUBUFOpcode(unsigned BaseOpc, unsigned Dwords) {
150   const MUBUFInfo *Info = getMUBUFInfoFromBaseOpcodeAndDwords(BaseOpc, Dwords);
151   return Info ? Info->Opcode : -1;
152 }
153 
154 int getMUBUFDwords(unsigned Opc) {
155   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
156   return Info ? Info->dwords : 0;
157 }
158 
159 bool getMUBUFHasVAddr(unsigned Opc) {
160   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
161   return Info ? Info->has_vaddr : false;
162 }
163 
164 bool getMUBUFHasSrsrc(unsigned Opc) {
165   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
166   return Info ? Info->has_srsrc : false;
167 }
168 
169 bool getMUBUFHasSoffset(unsigned Opc) {
170   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
171   return Info ? Info->has_soffset : false;
172 }
173 
174 // Wrapper for Tablegen'd function.  enum Subtarget is not defined in any
175 // header files, so we need to wrap it in a function that takes unsigned
176 // instead.
177 int getMCOpcode(uint16_t Opcode, unsigned Gen) {
178   return getMCOpcodeGen(Opcode, static_cast<Subtarget>(Gen));
179 }
180 
181 namespace IsaInfo {
182 
183 void streamIsaVersion(const MCSubtargetInfo *STI, raw_ostream &Stream) {
184   auto TargetTriple = STI->getTargetTriple();
185   auto Version = getIsaVersion(STI->getCPU());
186 
187   Stream << TargetTriple.getArchName() << '-'
188          << TargetTriple.getVendorName() << '-'
189          << TargetTriple.getOSName() << '-'
190          << TargetTriple.getEnvironmentName() << '-'
191          << "gfx"
192          << Version.Major
193          << Version.Minor
194          << Version.Stepping;
195 
196   if (hasXNACK(*STI))
197     Stream << "+xnack";
198   if (hasSRAMECC(*STI))
199     Stream << "+sram-ecc";
200 
201   Stream.flush();
202 }
203 
204 bool hasCodeObjectV3(const MCSubtargetInfo *STI) {
205   return STI->getTargetTriple().getOS() == Triple::AMDHSA &&
206              STI->getFeatureBits().test(FeatureCodeObjectV3);
207 }
208 
209 unsigned getWavefrontSize(const MCSubtargetInfo *STI) {
210   if (STI->getFeatureBits().test(FeatureWavefrontSize16))
211     return 16;
212   if (STI->getFeatureBits().test(FeatureWavefrontSize32))
213     return 32;
214 
215   return 64;
216 }
217 
218 unsigned getLocalMemorySize(const MCSubtargetInfo *STI) {
219   if (STI->getFeatureBits().test(FeatureLocalMemorySize32768))
220     return 32768;
221   if (STI->getFeatureBits().test(FeatureLocalMemorySize65536))
222     return 65536;
223 
224   return 0;
225 }
226 
227 unsigned getEUsPerCU(const MCSubtargetInfo *STI) {
228   return 4;
229 }
230 
231 unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo *STI,
232                                unsigned FlatWorkGroupSize) {
233   assert(FlatWorkGroupSize != 0);
234   if (STI->getTargetTriple().getArch() != Triple::amdgcn)
235     return 8;
236   unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize);
237   if (N == 1)
238     return 40;
239   N = 40 / N;
240   return std::min(N, 16u);
241 }
242 
243 unsigned getMaxWavesPerCU(const MCSubtargetInfo *STI) {
244   return getMaxWavesPerEU() * getEUsPerCU(STI);
245 }
246 
247 unsigned getMaxWavesPerCU(const MCSubtargetInfo *STI,
248                           unsigned FlatWorkGroupSize) {
249   return getWavesPerWorkGroup(STI, FlatWorkGroupSize);
250 }
251 
252 unsigned getMinWavesPerEU(const MCSubtargetInfo *STI) {
253   return 1;
254 }
255 
256 unsigned getMaxWavesPerEU() {
257   // FIXME: Need to take scratch memory into account.
258   return 10;
259 }
260 
261 unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI,
262                           unsigned FlatWorkGroupSize) {
263   return alignTo(getMaxWavesPerCU(STI, FlatWorkGroupSize),
264                  getEUsPerCU(STI)) / getEUsPerCU(STI);
265 }
266 
267 unsigned getMinFlatWorkGroupSize(const MCSubtargetInfo *STI) {
268   return 1;
269 }
270 
271 unsigned getMaxFlatWorkGroupSize(const MCSubtargetInfo *STI) {
272   return 2048;
273 }
274 
275 unsigned getWavesPerWorkGroup(const MCSubtargetInfo *STI,
276                               unsigned FlatWorkGroupSize) {
277   return alignTo(FlatWorkGroupSize, getWavefrontSize(STI)) /
278                  getWavefrontSize(STI);
279 }
280 
281 unsigned getSGPRAllocGranule(const MCSubtargetInfo *STI) {
282   IsaVersion Version = getIsaVersion(STI->getCPU());
283   if (Version.Major >= 10)
284     return getAddressableNumSGPRs(STI);
285   if (Version.Major >= 8)
286     return 16;
287   return 8;
288 }
289 
290 unsigned getSGPREncodingGranule(const MCSubtargetInfo *STI) {
291   return 8;
292 }
293 
294 unsigned getTotalNumSGPRs(const MCSubtargetInfo *STI) {
295   IsaVersion Version = getIsaVersion(STI->getCPU());
296   if (Version.Major >= 8)
297     return 800;
298   return 512;
299 }
300 
301 unsigned getAddressableNumSGPRs(const MCSubtargetInfo *STI) {
302   if (STI->getFeatureBits().test(FeatureSGPRInitBug))
303     return FIXED_NUM_SGPRS_FOR_INIT_BUG;
304 
305   IsaVersion Version = getIsaVersion(STI->getCPU());
306   if (Version.Major >= 10)
307     return 106;
308   if (Version.Major >= 8)
309     return 102;
310   return 104;
311 }
312 
313 unsigned getMinNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
314   assert(WavesPerEU != 0);
315 
316   IsaVersion Version = getIsaVersion(STI->getCPU());
317   if (Version.Major >= 10)
318     return 0;
319 
320   if (WavesPerEU >= getMaxWavesPerEU())
321     return 0;
322 
323   unsigned MinNumSGPRs = getTotalNumSGPRs(STI) / (WavesPerEU + 1);
324   if (STI->getFeatureBits().test(FeatureTrapHandler))
325     MinNumSGPRs -= std::min(MinNumSGPRs, (unsigned)TRAP_NUM_SGPRS);
326   MinNumSGPRs = alignDown(MinNumSGPRs, getSGPRAllocGranule(STI)) + 1;
327   return std::min(MinNumSGPRs, getAddressableNumSGPRs(STI));
328 }
329 
330 unsigned getMaxNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU,
331                         bool Addressable) {
332   assert(WavesPerEU != 0);
333 
334   unsigned AddressableNumSGPRs = getAddressableNumSGPRs(STI);
335   IsaVersion Version = getIsaVersion(STI->getCPU());
336   if (Version.Major >= 10)
337     return Addressable ? AddressableNumSGPRs : 108;
338   if (Version.Major >= 8 && !Addressable)
339     AddressableNumSGPRs = 112;
340   unsigned MaxNumSGPRs = getTotalNumSGPRs(STI) / WavesPerEU;
341   if (STI->getFeatureBits().test(FeatureTrapHandler))
342     MaxNumSGPRs -= std::min(MaxNumSGPRs, (unsigned)TRAP_NUM_SGPRS);
343   MaxNumSGPRs = alignDown(MaxNumSGPRs, getSGPRAllocGranule(STI));
344   return std::min(MaxNumSGPRs, AddressableNumSGPRs);
345 }
346 
347 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed,
348                           bool FlatScrUsed, bool XNACKUsed) {
349   unsigned ExtraSGPRs = 0;
350   if (VCCUsed)
351     ExtraSGPRs = 2;
352 
353   IsaVersion Version = getIsaVersion(STI->getCPU());
354   if (Version.Major >= 10)
355     return ExtraSGPRs;
356 
357   if (Version.Major < 8) {
358     if (FlatScrUsed)
359       ExtraSGPRs = 4;
360   } else {
361     if (XNACKUsed)
362       ExtraSGPRs = 4;
363 
364     if (FlatScrUsed)
365       ExtraSGPRs = 6;
366   }
367 
368   return ExtraSGPRs;
369 }
370 
371 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed,
372                           bool FlatScrUsed) {
373   return getNumExtraSGPRs(STI, VCCUsed, FlatScrUsed,
374                           STI->getFeatureBits().test(AMDGPU::FeatureXNACK));
375 }
376 
377 unsigned getNumSGPRBlocks(const MCSubtargetInfo *STI, unsigned NumSGPRs) {
378   NumSGPRs = alignTo(std::max(1u, NumSGPRs), getSGPREncodingGranule(STI));
379   // SGPRBlocks is actual number of SGPR blocks minus 1.
380   return NumSGPRs / getSGPREncodingGranule(STI) - 1;
381 }
382 
383 unsigned getVGPRAllocGranule(const MCSubtargetInfo *STI,
384                              Optional<bool> EnableWavefrontSize32) {
385   bool IsWave32 = EnableWavefrontSize32 ?
386       *EnableWavefrontSize32 :
387       STI->getFeatureBits().test(FeatureWavefrontSize32);
388   return IsWave32 ? 8 : 4;
389 }
390 
391 unsigned getVGPREncodingGranule(const MCSubtargetInfo *STI,
392                                 Optional<bool> EnableWavefrontSize32) {
393   return getVGPRAllocGranule(STI, EnableWavefrontSize32);
394 }
395 
396 unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) {
397   return 256;
398 }
399 
400 unsigned getAddressableNumVGPRs(const MCSubtargetInfo *STI) {
401   return getTotalNumVGPRs(STI);
402 }
403 
404 unsigned getMinNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
405   assert(WavesPerEU != 0);
406 
407   if (WavesPerEU >= getMaxWavesPerEU())
408     return 0;
409   unsigned MinNumVGPRs =
410       alignDown(getTotalNumVGPRs(STI) / (WavesPerEU + 1),
411                 getVGPRAllocGranule(STI)) + 1;
412   return std::min(MinNumVGPRs, getAddressableNumVGPRs(STI));
413 }
414 
415 unsigned getMaxNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
416   assert(WavesPerEU != 0);
417 
418   unsigned MaxNumVGPRs = alignDown(getTotalNumVGPRs(STI) / WavesPerEU,
419                                    getVGPRAllocGranule(STI));
420   unsigned AddressableNumVGPRs = getAddressableNumVGPRs(STI);
421   return std::min(MaxNumVGPRs, AddressableNumVGPRs);
422 }
423 
424 unsigned getNumVGPRBlocks(const MCSubtargetInfo *STI, unsigned NumVGPRs,
425                           Optional<bool> EnableWavefrontSize32) {
426   NumVGPRs = alignTo(std::max(1u, NumVGPRs),
427                      getVGPREncodingGranule(STI, EnableWavefrontSize32));
428   // VGPRBlocks is actual number of VGPR blocks minus 1.
429   return NumVGPRs / getVGPREncodingGranule(STI, EnableWavefrontSize32) - 1;
430 }
431 
432 } // end namespace IsaInfo
433 
434 void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header,
435                                const MCSubtargetInfo *STI) {
436   IsaVersion Version = getIsaVersion(STI->getCPU());
437 
438   memset(&Header, 0, sizeof(Header));
439 
440   Header.amd_kernel_code_version_major = 1;
441   Header.amd_kernel_code_version_minor = 2;
442   Header.amd_machine_kind = 1; // AMD_MACHINE_KIND_AMDGPU
443   Header.amd_machine_version_major = Version.Major;
444   Header.amd_machine_version_minor = Version.Minor;
445   Header.amd_machine_version_stepping = Version.Stepping;
446   Header.kernel_code_entry_byte_offset = sizeof(Header);
447   Header.wavefront_size = 6;
448 
449   // If the code object does not support indirect functions, then the value must
450   // be 0xffffffff.
451   Header.call_convention = -1;
452 
453   // These alignment values are specified in powers of two, so alignment =
454   // 2^n.  The minimum alignment is 2^4 = 16.
455   Header.kernarg_segment_alignment = 4;
456   Header.group_segment_alignment = 4;
457   Header.private_segment_alignment = 4;
458 
459   if (Version.Major >= 10) {
460     Header.compute_pgm_resource_registers |=
461       S_00B848_WGP_MODE(STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1) |
462       S_00B848_MEM_ORDERED(1);
463   }
464 }
465 
466 amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor(
467     const MCSubtargetInfo *STI) {
468   IsaVersion Version = getIsaVersion(STI->getCPU());
469 
470   amdhsa::kernel_descriptor_t KD;
471   memset(&KD, 0, sizeof(KD));
472 
473   AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
474                   amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64,
475                   amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE);
476   AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
477                   amdhsa::COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP, 1);
478   AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
479                   amdhsa::COMPUTE_PGM_RSRC1_ENABLE_IEEE_MODE, 1);
480   AMDHSA_BITS_SET(KD.compute_pgm_rsrc2,
481                   amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, 1);
482   if (Version.Major >= 10) {
483     AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
484                     amdhsa::COMPUTE_PGM_RSRC1_WGP_MODE,
485                     STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1);
486     AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
487                     amdhsa::COMPUTE_PGM_RSRC1_MEM_ORDERED, 1);
488   }
489   return KD;
490 }
491 
492 bool isGroupSegment(const GlobalValue *GV) {
493   return GV->getType()->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS;
494 }
495 
496 bool isGlobalSegment(const GlobalValue *GV) {
497   return GV->getType()->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS;
498 }
499 
500 bool isReadOnlySegment(const GlobalValue *GV) {
501   return GV->getType()->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
502          GV->getType()->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
503 }
504 
505 bool shouldEmitConstantsToTextSection(const Triple &TT) {
506   return TT.getOS() != Triple::AMDHSA;
507 }
508 
509 int getIntegerAttribute(const Function &F, StringRef Name, int Default) {
510   Attribute A = F.getFnAttribute(Name);
511   int Result = Default;
512 
513   if (A.isStringAttribute()) {
514     StringRef Str = A.getValueAsString();
515     if (Str.getAsInteger(0, Result)) {
516       LLVMContext &Ctx = F.getContext();
517       Ctx.emitError("can't parse integer attribute " + Name);
518     }
519   }
520 
521   return Result;
522 }
523 
524 std::pair<int, int> getIntegerPairAttribute(const Function &F,
525                                             StringRef Name,
526                                             std::pair<int, int> Default,
527                                             bool OnlyFirstRequired) {
528   Attribute A = F.getFnAttribute(Name);
529   if (!A.isStringAttribute())
530     return Default;
531 
532   LLVMContext &Ctx = F.getContext();
533   std::pair<int, int> Ints = Default;
534   std::pair<StringRef, StringRef> Strs = A.getValueAsString().split(',');
535   if (Strs.first.trim().getAsInteger(0, Ints.first)) {
536     Ctx.emitError("can't parse first integer attribute " + Name);
537     return Default;
538   }
539   if (Strs.second.trim().getAsInteger(0, Ints.second)) {
540     if (!OnlyFirstRequired || !Strs.second.trim().empty()) {
541       Ctx.emitError("can't parse second integer attribute " + Name);
542       return Default;
543     }
544   }
545 
546   return Ints;
547 }
548 
549 unsigned getVmcntBitMask(const IsaVersion &Version) {
550   unsigned VmcntLo = (1 << getVmcntBitWidthLo()) - 1;
551   if (Version.Major < 9)
552     return VmcntLo;
553 
554   unsigned VmcntHi = ((1 << getVmcntBitWidthHi()) - 1) << getVmcntBitWidthLo();
555   return VmcntLo | VmcntHi;
556 }
557 
558 unsigned getExpcntBitMask(const IsaVersion &Version) {
559   return (1 << getExpcntBitWidth()) - 1;
560 }
561 
562 unsigned getLgkmcntBitMask(const IsaVersion &Version) {
563   return (1 << getLgkmcntBitWidth(Version.Major)) - 1;
564 }
565 
566 unsigned getWaitcntBitMask(const IsaVersion &Version) {
567   unsigned VmcntLo = getBitMask(getVmcntBitShiftLo(), getVmcntBitWidthLo());
568   unsigned Expcnt = getBitMask(getExpcntBitShift(), getExpcntBitWidth());
569   unsigned Lgkmcnt = getBitMask(getLgkmcntBitShift(),
570                                 getLgkmcntBitWidth(Version.Major));
571   unsigned Waitcnt = VmcntLo | Expcnt | Lgkmcnt;
572   if (Version.Major < 9)
573     return Waitcnt;
574 
575   unsigned VmcntHi = getBitMask(getVmcntBitShiftHi(), getVmcntBitWidthHi());
576   return Waitcnt | VmcntHi;
577 }
578 
579 unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) {
580   unsigned VmcntLo =
581       unpackBits(Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo());
582   if (Version.Major < 9)
583     return VmcntLo;
584 
585   unsigned VmcntHi =
586       unpackBits(Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi());
587   VmcntHi <<= getVmcntBitWidthLo();
588   return VmcntLo | VmcntHi;
589 }
590 
591 unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) {
592   return unpackBits(Waitcnt, getExpcntBitShift(), getExpcntBitWidth());
593 }
594 
595 unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) {
596   return unpackBits(Waitcnt, getLgkmcntBitShift(),
597                     getLgkmcntBitWidth(Version.Major));
598 }
599 
600 void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt,
601                    unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt) {
602   Vmcnt = decodeVmcnt(Version, Waitcnt);
603   Expcnt = decodeExpcnt(Version, Waitcnt);
604   Lgkmcnt = decodeLgkmcnt(Version, Waitcnt);
605 }
606 
607 Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded) {
608   Waitcnt Decoded;
609   Decoded.VmCnt = decodeVmcnt(Version, Encoded);
610   Decoded.ExpCnt = decodeExpcnt(Version, Encoded);
611   Decoded.LgkmCnt = decodeLgkmcnt(Version, Encoded);
612   return Decoded;
613 }
614 
615 unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt,
616                      unsigned Vmcnt) {
617   Waitcnt =
618       packBits(Vmcnt, Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo());
619   if (Version.Major < 9)
620     return Waitcnt;
621 
622   Vmcnt >>= getVmcntBitWidthLo();
623   return packBits(Vmcnt, Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi());
624 }
625 
626 unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt,
627                       unsigned Expcnt) {
628   return packBits(Expcnt, Waitcnt, getExpcntBitShift(), getExpcntBitWidth());
629 }
630 
631 unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt,
632                        unsigned Lgkmcnt) {
633   return packBits(Lgkmcnt, Waitcnt, getLgkmcntBitShift(),
634                                     getLgkmcntBitWidth(Version.Major));
635 }
636 
637 unsigned encodeWaitcnt(const IsaVersion &Version,
638                        unsigned Vmcnt, unsigned Expcnt, unsigned Lgkmcnt) {
639   unsigned Waitcnt = getWaitcntBitMask(Version);
640   Waitcnt = encodeVmcnt(Version, Waitcnt, Vmcnt);
641   Waitcnt = encodeExpcnt(Version, Waitcnt, Expcnt);
642   Waitcnt = encodeLgkmcnt(Version, Waitcnt, Lgkmcnt);
643   return Waitcnt;
644 }
645 
646 unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded) {
647   return encodeWaitcnt(Version, Decoded.VmCnt, Decoded.ExpCnt, Decoded.LgkmCnt);
648 }
649 
650 //===----------------------------------------------------------------------===//
651 // hwreg
652 //===----------------------------------------------------------------------===//
653 
654 namespace Hwreg {
655 
656 int64_t getHwregId(const StringRef Name) {
657   for (int Id = ID_SYMBOLIC_FIRST_; Id < ID_SYMBOLIC_LAST_; ++Id) {
658     if (IdSymbolic[Id] && Name == IdSymbolic[Id])
659       return Id;
660   }
661   return ID_UNKNOWN_;
662 }
663 
664 static unsigned getLastSymbolicHwreg(const MCSubtargetInfo &STI) {
665   if (isSI(STI) || isCI(STI) || isVI(STI))
666     return ID_SYMBOLIC_FIRST_GFX9_;
667   else if (isGFX9(STI))
668     return ID_SYMBOLIC_FIRST_GFX10_;
669   else
670     return ID_SYMBOLIC_LAST_;
671 }
672 
673 bool isValidHwreg(int64_t Id, const MCSubtargetInfo &STI) {
674   return ID_SYMBOLIC_FIRST_ <= Id && Id < getLastSymbolicHwreg(STI) &&
675          IdSymbolic[Id];
676 }
677 
678 bool isValidHwreg(int64_t Id) {
679   return 0 <= Id && isUInt<ID_WIDTH_>(Id);
680 }
681 
682 bool isValidHwregOffset(int64_t Offset) {
683   return 0 <= Offset && isUInt<OFFSET_WIDTH_>(Offset);
684 }
685 
686 bool isValidHwregWidth(int64_t Width) {
687   return 0 <= (Width - 1) && isUInt<WIDTH_M1_WIDTH_>(Width - 1);
688 }
689 
690 int64_t encodeHwreg(int64_t Id, int64_t Offset, int64_t Width) {
691   return (Id << ID_SHIFT_) |
692          (Offset << OFFSET_SHIFT_) |
693          ((Width - 1) << WIDTH_M1_SHIFT_);
694 }
695 
696 StringRef getHwreg(unsigned Id, const MCSubtargetInfo &STI) {
697   return isValidHwreg(Id, STI) ? IdSymbolic[Id] : "";
698 }
699 
700 void decodeHwreg(unsigned Val, unsigned &Id, unsigned &Offset, unsigned &Width) {
701   Id = (Val & ID_MASK_) >> ID_SHIFT_;
702   Offset = (Val & OFFSET_MASK_) >> OFFSET_SHIFT_;
703   Width = ((Val & WIDTH_M1_MASK_) >> WIDTH_M1_SHIFT_) + 1;
704 }
705 
706 } // namespace Hwreg
707 
708 //===----------------------------------------------------------------------===//
709 //
710 //===----------------------------------------------------------------------===//
711 
712 unsigned getInitialPSInputAddr(const Function &F) {
713   return getIntegerAttribute(F, "InitialPSInputAddr", 0);
714 }
715 
716 bool isShader(CallingConv::ID cc) {
717   switch(cc) {
718     case CallingConv::AMDGPU_VS:
719     case CallingConv::AMDGPU_LS:
720     case CallingConv::AMDGPU_HS:
721     case CallingConv::AMDGPU_ES:
722     case CallingConv::AMDGPU_GS:
723     case CallingConv::AMDGPU_PS:
724     case CallingConv::AMDGPU_CS:
725       return true;
726     default:
727       return false;
728   }
729 }
730 
731 bool isCompute(CallingConv::ID cc) {
732   return !isShader(cc) || cc == CallingConv::AMDGPU_CS;
733 }
734 
735 bool isEntryFunctionCC(CallingConv::ID CC) {
736   switch (CC) {
737   case CallingConv::AMDGPU_KERNEL:
738   case CallingConv::SPIR_KERNEL:
739   case CallingConv::AMDGPU_VS:
740   case CallingConv::AMDGPU_GS:
741   case CallingConv::AMDGPU_PS:
742   case CallingConv::AMDGPU_CS:
743   case CallingConv::AMDGPU_ES:
744   case CallingConv::AMDGPU_HS:
745   case CallingConv::AMDGPU_LS:
746     return true;
747   default:
748     return false;
749   }
750 }
751 
752 bool hasXNACK(const MCSubtargetInfo &STI) {
753   return STI.getFeatureBits()[AMDGPU::FeatureXNACK];
754 }
755 
756 bool hasSRAMECC(const MCSubtargetInfo &STI) {
757   return STI.getFeatureBits()[AMDGPU::FeatureSRAMECC];
758 }
759 
760 bool hasMIMG_R128(const MCSubtargetInfo &STI) {
761   return STI.getFeatureBits()[AMDGPU::FeatureMIMG_R128];
762 }
763 
764 bool hasPackedD16(const MCSubtargetInfo &STI) {
765   return !STI.getFeatureBits()[AMDGPU::FeatureUnpackedD16VMem];
766 }
767 
768 bool isSI(const MCSubtargetInfo &STI) {
769   return STI.getFeatureBits()[AMDGPU::FeatureSouthernIslands];
770 }
771 
772 bool isCI(const MCSubtargetInfo &STI) {
773   return STI.getFeatureBits()[AMDGPU::FeatureSeaIslands];
774 }
775 
776 bool isVI(const MCSubtargetInfo &STI) {
777   return STI.getFeatureBits()[AMDGPU::FeatureVolcanicIslands];
778 }
779 
780 bool isGFX9(const MCSubtargetInfo &STI) {
781   return STI.getFeatureBits()[AMDGPU::FeatureGFX9];
782 }
783 
784 bool isGFX10(const MCSubtargetInfo &STI) {
785   return STI.getFeatureBits()[AMDGPU::FeatureGFX10];
786 }
787 
788 bool isGCN3Encoding(const MCSubtargetInfo &STI) {
789   return STI.getFeatureBits()[AMDGPU::FeatureGCN3Encoding];
790 }
791 
792 bool isSGPR(unsigned Reg, const MCRegisterInfo* TRI) {
793   const MCRegisterClass SGPRClass = TRI->getRegClass(AMDGPU::SReg_32RegClassID);
794   const unsigned FirstSubReg = TRI->getSubReg(Reg, 1);
795   return SGPRClass.contains(FirstSubReg != 0 ? FirstSubReg : Reg) ||
796     Reg == AMDGPU::SCC;
797 }
798 
799 bool isRegIntersect(unsigned Reg0, unsigned Reg1, const MCRegisterInfo* TRI) {
800   for (MCRegAliasIterator R(Reg0, TRI, true); R.isValid(); ++R) {
801     if (*R == Reg1) return true;
802   }
803   return false;
804 }
805 
806 #define MAP_REG2REG \
807   using namespace AMDGPU; \
808   switch(Reg) { \
809   default: return Reg; \
810   CASE_CI_VI(FLAT_SCR) \
811   CASE_CI_VI(FLAT_SCR_LO) \
812   CASE_CI_VI(FLAT_SCR_HI) \
813   CASE_VI_GFX9_GFX10(TTMP0) \
814   CASE_VI_GFX9_GFX10(TTMP1) \
815   CASE_VI_GFX9_GFX10(TTMP2) \
816   CASE_VI_GFX9_GFX10(TTMP3) \
817   CASE_VI_GFX9_GFX10(TTMP4) \
818   CASE_VI_GFX9_GFX10(TTMP5) \
819   CASE_VI_GFX9_GFX10(TTMP6) \
820   CASE_VI_GFX9_GFX10(TTMP7) \
821   CASE_VI_GFX9_GFX10(TTMP8) \
822   CASE_VI_GFX9_GFX10(TTMP9) \
823   CASE_VI_GFX9_GFX10(TTMP10) \
824   CASE_VI_GFX9_GFX10(TTMP11) \
825   CASE_VI_GFX9_GFX10(TTMP12) \
826   CASE_VI_GFX9_GFX10(TTMP13) \
827   CASE_VI_GFX9_GFX10(TTMP14) \
828   CASE_VI_GFX9_GFX10(TTMP15) \
829   CASE_VI_GFX9_GFX10(TTMP0_TTMP1) \
830   CASE_VI_GFX9_GFX10(TTMP2_TTMP3) \
831   CASE_VI_GFX9_GFX10(TTMP4_TTMP5) \
832   CASE_VI_GFX9_GFX10(TTMP6_TTMP7) \
833   CASE_VI_GFX9_GFX10(TTMP8_TTMP9) \
834   CASE_VI_GFX9_GFX10(TTMP10_TTMP11) \
835   CASE_VI_GFX9_GFX10(TTMP12_TTMP13) \
836   CASE_VI_GFX9_GFX10(TTMP14_TTMP15) \
837   CASE_VI_GFX9_GFX10(TTMP0_TTMP1_TTMP2_TTMP3) \
838   CASE_VI_GFX9_GFX10(TTMP4_TTMP5_TTMP6_TTMP7) \
839   CASE_VI_GFX9_GFX10(TTMP8_TTMP9_TTMP10_TTMP11) \
840   CASE_VI_GFX9_GFX10(TTMP12_TTMP13_TTMP14_TTMP15) \
841   CASE_VI_GFX9_GFX10(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7) \
842   CASE_VI_GFX9_GFX10(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11) \
843   CASE_VI_GFX9_GFX10(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
844   CASE_VI_GFX9_GFX10(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
845   }
846 
847 #define CASE_CI_VI(node) \
848   assert(!isSI(STI)); \
849   case node: return isCI(STI) ? node##_ci : node##_vi;
850 
851 #define CASE_VI_GFX9_GFX10(node) \
852   case node: return (isGFX9(STI) || isGFX10(STI)) ? node##_gfx9_gfx10 : node##_vi;
853 
854 unsigned getMCReg(unsigned Reg, const MCSubtargetInfo &STI) {
855   if (STI.getTargetTriple().getArch() == Triple::r600)
856     return Reg;
857   MAP_REG2REG
858 }
859 
860 #undef CASE_CI_VI
861 #undef CASE_VI_GFX9_GFX10
862 
863 #define CASE_CI_VI(node)   case node##_ci: case node##_vi:   return node;
864 #define CASE_VI_GFX9_GFX10(node) case node##_vi: case node##_gfx9_gfx10: return node;
865 
866 unsigned mc2PseudoReg(unsigned Reg) {
867   MAP_REG2REG
868 }
869 
870 #undef CASE_CI_VI
871 #undef CASE_VI_GFX9_GFX10
872 #undef MAP_REG2REG
873 
874 bool isSISrcOperand(const MCInstrDesc &Desc, unsigned OpNo) {
875   assert(OpNo < Desc.NumOperands);
876   unsigned OpType = Desc.OpInfo[OpNo].OperandType;
877   return OpType >= AMDGPU::OPERAND_SRC_FIRST &&
878          OpType <= AMDGPU::OPERAND_SRC_LAST;
879 }
880 
881 bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo) {
882   assert(OpNo < Desc.NumOperands);
883   unsigned OpType = Desc.OpInfo[OpNo].OperandType;
884   switch (OpType) {
885   case AMDGPU::OPERAND_REG_IMM_FP32:
886   case AMDGPU::OPERAND_REG_IMM_FP64:
887   case AMDGPU::OPERAND_REG_IMM_FP16:
888   case AMDGPU::OPERAND_REG_IMM_V2FP16:
889   case AMDGPU::OPERAND_REG_IMM_V2INT16:
890   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
891   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
892   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
893   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
894   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
895     return true;
896   default:
897     return false;
898   }
899 }
900 
901 bool isSISrcInlinableOperand(const MCInstrDesc &Desc, unsigned OpNo) {
902   assert(OpNo < Desc.NumOperands);
903   unsigned OpType = Desc.OpInfo[OpNo].OperandType;
904   return OpType >= AMDGPU::OPERAND_REG_INLINE_C_FIRST &&
905          OpType <= AMDGPU::OPERAND_REG_INLINE_C_LAST;
906 }
907 
908 // Avoid using MCRegisterClass::getSize, since that function will go away
909 // (move from MC* level to Target* level). Return size in bits.
910 unsigned getRegBitWidth(unsigned RCID) {
911   switch (RCID) {
912   case AMDGPU::SGPR_32RegClassID:
913   case AMDGPU::VGPR_32RegClassID:
914   case AMDGPU::VRegOrLds_32RegClassID:
915   case AMDGPU::VS_32RegClassID:
916   case AMDGPU::SReg_32RegClassID:
917   case AMDGPU::SReg_32_XM0RegClassID:
918   case AMDGPU::SRegOrLds_32RegClassID:
919     return 32;
920   case AMDGPU::SGPR_64RegClassID:
921   case AMDGPU::VS_64RegClassID:
922   case AMDGPU::SReg_64RegClassID:
923   case AMDGPU::VReg_64RegClassID:
924   case AMDGPU::SReg_64_XEXECRegClassID:
925     return 64;
926   case AMDGPU::SGPR_96RegClassID:
927   case AMDGPU::SReg_96RegClassID:
928   case AMDGPU::VReg_96RegClassID:
929     return 96;
930   case AMDGPU::SGPR_128RegClassID:
931   case AMDGPU::SReg_128RegClassID:
932   case AMDGPU::VReg_128RegClassID:
933     return 128;
934   case AMDGPU::SGPR_160RegClassID:
935   case AMDGPU::SReg_160RegClassID:
936   case AMDGPU::VReg_160RegClassID:
937     return 160;
938   case AMDGPU::SReg_256RegClassID:
939   case AMDGPU::VReg_256RegClassID:
940     return 256;
941   case AMDGPU::SReg_512RegClassID:
942   case AMDGPU::VReg_512RegClassID:
943     return 512;
944   default:
945     llvm_unreachable("Unexpected register class");
946   }
947 }
948 
949 unsigned getRegBitWidth(const MCRegisterClass &RC) {
950   return getRegBitWidth(RC.getID());
951 }
952 
953 unsigned getRegOperandSize(const MCRegisterInfo *MRI, const MCInstrDesc &Desc,
954                            unsigned OpNo) {
955   assert(OpNo < Desc.NumOperands);
956   unsigned RCID = Desc.OpInfo[OpNo].RegClass;
957   return getRegBitWidth(MRI->getRegClass(RCID)) / 8;
958 }
959 
960 bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi) {
961   if (Literal >= -16 && Literal <= 64)
962     return true;
963 
964   uint64_t Val = static_cast<uint64_t>(Literal);
965   return (Val == DoubleToBits(0.0)) ||
966          (Val == DoubleToBits(1.0)) ||
967          (Val == DoubleToBits(-1.0)) ||
968          (Val == DoubleToBits(0.5)) ||
969          (Val == DoubleToBits(-0.5)) ||
970          (Val == DoubleToBits(2.0)) ||
971          (Val == DoubleToBits(-2.0)) ||
972          (Val == DoubleToBits(4.0)) ||
973          (Val == DoubleToBits(-4.0)) ||
974          (Val == 0x3fc45f306dc9c882 && HasInv2Pi);
975 }
976 
977 bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi) {
978   if (Literal >= -16 && Literal <= 64)
979     return true;
980 
981   // The actual type of the operand does not seem to matter as long
982   // as the bits match one of the inline immediate values.  For example:
983   //
984   // -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal,
985   // so it is a legal inline immediate.
986   //
987   // 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in
988   // floating-point, so it is a legal inline immediate.
989 
990   uint32_t Val = static_cast<uint32_t>(Literal);
991   return (Val == FloatToBits(0.0f)) ||
992          (Val == FloatToBits(1.0f)) ||
993          (Val == FloatToBits(-1.0f)) ||
994          (Val == FloatToBits(0.5f)) ||
995          (Val == FloatToBits(-0.5f)) ||
996          (Val == FloatToBits(2.0f)) ||
997          (Val == FloatToBits(-2.0f)) ||
998          (Val == FloatToBits(4.0f)) ||
999          (Val == FloatToBits(-4.0f)) ||
1000          (Val == 0x3e22f983 && HasInv2Pi);
1001 }
1002 
1003 bool isInlinableLiteral16(int16_t Literal, bool HasInv2Pi) {
1004   if (!HasInv2Pi)
1005     return false;
1006 
1007   if (Literal >= -16 && Literal <= 64)
1008     return true;
1009 
1010   uint16_t Val = static_cast<uint16_t>(Literal);
1011   return Val == 0x3C00 || // 1.0
1012          Val == 0xBC00 || // -1.0
1013          Val == 0x3800 || // 0.5
1014          Val == 0xB800 || // -0.5
1015          Val == 0x4000 || // 2.0
1016          Val == 0xC000 || // -2.0
1017          Val == 0x4400 || // 4.0
1018          Val == 0xC400 || // -4.0
1019          Val == 0x3118;   // 1/2pi
1020 }
1021 
1022 bool isInlinableLiteralV216(int32_t Literal, bool HasInv2Pi) {
1023   assert(HasInv2Pi);
1024 
1025   if (isInt<16>(Literal) || isUInt<16>(Literal)) {
1026     int16_t Trunc = static_cast<int16_t>(Literal);
1027     return AMDGPU::isInlinableLiteral16(Trunc, HasInv2Pi);
1028   }
1029   if (!(Literal & 0xffff))
1030     return AMDGPU::isInlinableLiteral16(Literal >> 16, HasInv2Pi);
1031 
1032   int16_t Lo16 = static_cast<int16_t>(Literal);
1033   int16_t Hi16 = static_cast<int16_t>(Literal >> 16);
1034   return Lo16 == Hi16 && isInlinableLiteral16(Lo16, HasInv2Pi);
1035 }
1036 
1037 bool isArgPassedInSGPR(const Argument *A) {
1038   const Function *F = A->getParent();
1039 
1040   // Arguments to compute shaders are never a source of divergence.
1041   CallingConv::ID CC = F->getCallingConv();
1042   switch (CC) {
1043   case CallingConv::AMDGPU_KERNEL:
1044   case CallingConv::SPIR_KERNEL:
1045     return true;
1046   case CallingConv::AMDGPU_VS:
1047   case CallingConv::AMDGPU_LS:
1048   case CallingConv::AMDGPU_HS:
1049   case CallingConv::AMDGPU_ES:
1050   case CallingConv::AMDGPU_GS:
1051   case CallingConv::AMDGPU_PS:
1052   case CallingConv::AMDGPU_CS:
1053     // For non-compute shaders, SGPR inputs are marked with either inreg or byval.
1054     // Everything else is in VGPRs.
1055     return F->getAttributes().hasParamAttribute(A->getArgNo(), Attribute::InReg) ||
1056            F->getAttributes().hasParamAttribute(A->getArgNo(), Attribute::ByVal);
1057   default:
1058     // TODO: Should calls support inreg for SGPR inputs?
1059     return false;
1060   }
1061 }
1062 
1063 static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) {
1064   return isGCN3Encoding(ST) || isGFX10(ST);
1065 }
1066 
1067 int64_t getSMRDEncodedOffset(const MCSubtargetInfo &ST, int64_t ByteOffset) {
1068   if (hasSMEMByteOffset(ST))
1069     return ByteOffset;
1070   return ByteOffset >> 2;
1071 }
1072 
1073 bool isLegalSMRDImmOffset(const MCSubtargetInfo &ST, int64_t ByteOffset) {
1074   int64_t EncodedOffset = getSMRDEncodedOffset(ST, ByteOffset);
1075   return (hasSMEMByteOffset(ST)) ?
1076     isUInt<20>(EncodedOffset) : isUInt<8>(EncodedOffset);
1077 }
1078 
1079 // Given Imm, split it into the values to put into the SOffset and ImmOffset
1080 // fields in an MUBUF instruction. Return false if it is not possible (due to a
1081 // hardware bug needing a workaround).
1082 //
1083 // The required alignment ensures that individual address components remain
1084 // aligned if they are aligned to begin with. It also ensures that additional
1085 // offsets within the given alignment can be added to the resulting ImmOffset.
1086 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset,
1087                       const GCNSubtarget *Subtarget, uint32_t Align) {
1088   const uint32_t MaxImm = alignDown(4095, Align);
1089   uint32_t Overflow = 0;
1090 
1091   if (Imm > MaxImm) {
1092     if (Imm <= MaxImm + 64) {
1093       // Use an SOffset inline constant for 4..64
1094       Overflow = Imm - MaxImm;
1095       Imm = MaxImm;
1096     } else {
1097       // Try to keep the same value in SOffset for adjacent loads, so that
1098       // the corresponding register contents can be re-used.
1099       //
1100       // Load values with all low-bits (except for alignment bits) set into
1101       // SOffset, so that a larger range of values can be covered using
1102       // s_movk_i32.
1103       //
1104       // Atomic operations fail to work correctly when individual address
1105       // components are unaligned, even if their sum is aligned.
1106       uint32_t High = (Imm + Align) & ~4095;
1107       uint32_t Low = (Imm + Align) & 4095;
1108       Imm = Low;
1109       Overflow = High - Align;
1110     }
1111   }
1112 
1113   // There is a hardware bug in SI and CI which prevents address clamping in
1114   // MUBUF instructions from working correctly with SOffsets. The immediate
1115   // offset is unaffected.
1116   if (Overflow > 0 &&
1117       Subtarget->getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS)
1118     return false;
1119 
1120   ImmOffset = Imm;
1121   SOffset = Overflow;
1122   return true;
1123 }
1124 
1125 SIModeRegisterDefaults::SIModeRegisterDefaults(const Function &F) {
1126   *this = getDefaultForCallingConv(F.getCallingConv());
1127 
1128   StringRef IEEEAttr = F.getFnAttribute("amdgpu-ieee").getValueAsString();
1129   if (!IEEEAttr.empty())
1130     IEEE = IEEEAttr == "true";
1131 
1132   StringRef DX10ClampAttr
1133     = F.getFnAttribute("amdgpu-dx10-clamp").getValueAsString();
1134   if (!DX10ClampAttr.empty())
1135     DX10Clamp = DX10ClampAttr == "true";
1136 }
1137 
1138 namespace {
1139 
1140 struct SourceOfDivergence {
1141   unsigned Intr;
1142 };
1143 const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr);
1144 
1145 #define GET_SourcesOfDivergence_IMPL
1146 #include "AMDGPUGenSearchableTables.inc"
1147 
1148 } // end anonymous namespace
1149 
1150 bool isIntrinsicSourceOfDivergence(unsigned IntrID) {
1151   return lookupSourceOfDivergence(IntrID);
1152 }
1153 
1154 } // namespace AMDGPU
1155 } // namespace llvm
1156