1 //===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "AMDGPUBaseInfo.h"
10 #include "AMDGPU.h"
11 #include "AMDGPUAsmUtils.h"
12 #include "AMDGPUTargetTransformInfo.h"
13 #include "SIDefines.h"
14 #include "llvm/ADT/StringRef.h"
15 #include "llvm/ADT/Triple.h"
16 #include "llvm/BinaryFormat/ELF.h"
17 #include "llvm/CodeGen/MachineMemOperand.h"
18 #include "llvm/IR/Attributes.h"
19 #include "llvm/IR/Constants.h"
20 #include "llvm/IR/Function.h"
21 #include "llvm/IR/GlobalValue.h"
22 #include "llvm/IR/Instruction.h"
23 #include "llvm/IR/IntrinsicsAMDGPU.h"
24 #include "llvm/IR/IntrinsicsR600.h"
25 #include "llvm/IR/LLVMContext.h"
26 #include "llvm/IR/Module.h"
27 #include "llvm/MC/MCContext.h"
28 #include "llvm/MC/MCInstrDesc.h"
29 #include "llvm/MC/MCInstrInfo.h"
30 #include "llvm/MC/MCRegisterInfo.h"
31 #include "llvm/MC/MCSectionELF.h"
32 #include "llvm/MC/MCSubtargetInfo.h"
33 #include "llvm/MC/SubtargetFeature.h"
34 #include "llvm/Support/Casting.h"
35 #include "llvm/Support/ErrorHandling.h"
36 #include "llvm/Support/MathExtras.h"
37 #include <algorithm>
38 #include <cassert>
39 #include <cstdint>
40 #include <cstring>
41 #include <utility>
42 
43 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
44 
45 #define GET_INSTRINFO_NAMED_OPS
46 #define GET_INSTRMAP_INFO
47 #include "AMDGPUGenInstrInfo.inc"
48 #undef GET_INSTRMAP_INFO
49 #undef GET_INSTRINFO_NAMED_OPS
50 
51 namespace {
52 
53 /// \returns Bit mask for given bit \p Shift and bit \p Width.
54 unsigned getBitMask(unsigned Shift, unsigned Width) {
55   return ((1 << Width) - 1) << Shift;
56 }
57 
58 /// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width.
59 ///
60 /// \returns Packed \p Dst.
61 unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) {
62   Dst &= ~(1 << Shift) & ~getBitMask(Shift, Width);
63   Dst |= (Src << Shift) & getBitMask(Shift, Width);
64   return Dst;
65 }
66 
67 /// Unpacks bits from \p Src for given bit \p Shift and bit \p Width.
68 ///
69 /// \returns Unpacked bits.
70 unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) {
71   return (Src & getBitMask(Shift, Width)) >> Shift;
72 }
73 
74 /// \returns Vmcnt bit shift (lower bits).
75 unsigned getVmcntBitShiftLo() { return 0; }
76 
77 /// \returns Vmcnt bit width (lower bits).
78 unsigned getVmcntBitWidthLo() { return 4; }
79 
80 /// \returns Expcnt bit shift.
81 unsigned getExpcntBitShift() { return 4; }
82 
83 /// \returns Expcnt bit width.
84 unsigned getExpcntBitWidth() { return 3; }
85 
86 /// \returns Lgkmcnt bit shift.
87 unsigned getLgkmcntBitShift() { return 8; }
88 
89 /// \returns Lgkmcnt bit width.
90 unsigned getLgkmcntBitWidth(unsigned VersionMajor) {
91   return (VersionMajor >= 10) ? 6 : 4;
92 }
93 
94 /// \returns Vmcnt bit shift (higher bits).
95 unsigned getVmcntBitShiftHi() { return 14; }
96 
97 /// \returns Vmcnt bit width (higher bits).
98 unsigned getVmcntBitWidthHi() { return 2; }
99 
100 } // end namespace anonymous
101 
102 namespace llvm {
103 
104 namespace AMDGPU {
105 
106 #define GET_MIMGBaseOpcodesTable_IMPL
107 #define GET_MIMGDimInfoTable_IMPL
108 #define GET_MIMGInfoTable_IMPL
109 #define GET_MIMGLZMappingTable_IMPL
110 #define GET_MIMGMIPMappingTable_IMPL
111 #define GET_MIMGG16MappingTable_IMPL
112 #include "AMDGPUGenSearchableTables.inc"
113 
114 int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding,
115                   unsigned VDataDwords, unsigned VAddrDwords) {
116   const MIMGInfo *Info = getMIMGOpcodeHelper(BaseOpcode, MIMGEncoding,
117                                              VDataDwords, VAddrDwords);
118   return Info ? Info->Opcode : -1;
119 }
120 
121 const MIMGBaseOpcodeInfo *getMIMGBaseOpcode(unsigned Opc) {
122   const MIMGInfo *Info = getMIMGInfo(Opc);
123   return Info ? getMIMGBaseOpcodeInfo(Info->BaseOpcode) : nullptr;
124 }
125 
126 int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) {
127   const MIMGInfo *OrigInfo = getMIMGInfo(Opc);
128   const MIMGInfo *NewInfo =
129       getMIMGOpcodeHelper(OrigInfo->BaseOpcode, OrigInfo->MIMGEncoding,
130                           NewChannels, OrigInfo->VAddrDwords);
131   return NewInfo ? NewInfo->Opcode : -1;
132 }
133 
134 struct MUBUFInfo {
135   uint16_t Opcode;
136   uint16_t BaseOpcode;
137   uint8_t elements;
138   bool has_vaddr;
139   bool has_srsrc;
140   bool has_soffset;
141 };
142 
143 struct MTBUFInfo {
144   uint16_t Opcode;
145   uint16_t BaseOpcode;
146   uint8_t elements;
147   bool has_vaddr;
148   bool has_srsrc;
149   bool has_soffset;
150 };
151 
152 struct SMInfo {
153   uint16_t Opcode;
154   bool IsBuffer;
155 };
156 
157 #define GET_MTBUFInfoTable_DECL
158 #define GET_MTBUFInfoTable_IMPL
159 #define GET_MUBUFInfoTable_DECL
160 #define GET_MUBUFInfoTable_IMPL
161 #define GET_SMInfoTable_DECL
162 #define GET_SMInfoTable_IMPL
163 #include "AMDGPUGenSearchableTables.inc"
164 
165 int getMTBUFBaseOpcode(unsigned Opc) {
166   const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opc);
167   return Info ? Info->BaseOpcode : -1;
168 }
169 
170 int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements) {
171   const MTBUFInfo *Info = getMTBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements);
172   return Info ? Info->Opcode : -1;
173 }
174 
175 int getMTBUFElements(unsigned Opc) {
176   const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
177   return Info ? Info->elements : 0;
178 }
179 
180 bool getMTBUFHasVAddr(unsigned Opc) {
181   const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
182   return Info ? Info->has_vaddr : false;
183 }
184 
185 bool getMTBUFHasSrsrc(unsigned Opc) {
186   const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
187   return Info ? Info->has_srsrc : false;
188 }
189 
190 bool getMTBUFHasSoffset(unsigned Opc) {
191   const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
192   return Info ? Info->has_soffset : false;
193 }
194 
195 int getMUBUFBaseOpcode(unsigned Opc) {
196   const MUBUFInfo *Info = getMUBUFInfoFromOpcode(Opc);
197   return Info ? Info->BaseOpcode : -1;
198 }
199 
200 int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements) {
201   const MUBUFInfo *Info = getMUBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements);
202   return Info ? Info->Opcode : -1;
203 }
204 
205 int getMUBUFElements(unsigned Opc) {
206   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
207   return Info ? Info->elements : 0;
208 }
209 
210 bool getMUBUFHasVAddr(unsigned Opc) {
211   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
212   return Info ? Info->has_vaddr : false;
213 }
214 
215 bool getMUBUFHasSrsrc(unsigned Opc) {
216   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
217   return Info ? Info->has_srsrc : false;
218 }
219 
220 bool getMUBUFHasSoffset(unsigned Opc) {
221   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
222   return Info ? Info->has_soffset : false;
223 }
224 
225 bool getSMEMIsBuffer(unsigned Opc) {
226   const SMInfo *Info = getSMEMOpcodeHelper(Opc);
227   return Info ? Info->IsBuffer : false;
228 }
229 
230 // Wrapper for Tablegen'd function.  enum Subtarget is not defined in any
231 // header files, so we need to wrap it in a function that takes unsigned
232 // instead.
233 int getMCOpcode(uint16_t Opcode, unsigned Gen) {
234   return getMCOpcodeGen(Opcode, static_cast<Subtarget>(Gen));
235 }
236 
237 namespace IsaInfo {
238 
239 void streamIsaVersion(const MCSubtargetInfo *STI, raw_ostream &Stream) {
240   auto TargetTriple = STI->getTargetTriple();
241   auto Version = getIsaVersion(STI->getCPU());
242 
243   Stream << TargetTriple.getArchName() << '-'
244          << TargetTriple.getVendorName() << '-'
245          << TargetTriple.getOSName() << '-'
246          << TargetTriple.getEnvironmentName() << '-'
247          << "gfx"
248          << Version.Major
249          << Version.Minor
250          << Version.Stepping;
251 
252   if (hasXNACK(*STI))
253     Stream << "+xnack";
254   if (hasSRAMECC(*STI))
255     Stream << "+sram-ecc";
256 
257   Stream.flush();
258 }
259 
260 bool hasCodeObjectV3(const MCSubtargetInfo *STI) {
261   return STI->getTargetTriple().getOS() == Triple::AMDHSA &&
262              STI->getFeatureBits().test(FeatureCodeObjectV3);
263 }
264 
265 unsigned getWavefrontSize(const MCSubtargetInfo *STI) {
266   if (STI->getFeatureBits().test(FeatureWavefrontSize16))
267     return 16;
268   if (STI->getFeatureBits().test(FeatureWavefrontSize32))
269     return 32;
270 
271   return 64;
272 }
273 
274 unsigned getLocalMemorySize(const MCSubtargetInfo *STI) {
275   if (STI->getFeatureBits().test(FeatureLocalMemorySize32768))
276     return 32768;
277   if (STI->getFeatureBits().test(FeatureLocalMemorySize65536))
278     return 65536;
279 
280   return 0;
281 }
282 
283 unsigned getEUsPerCU(const MCSubtargetInfo *STI) {
284   // "Per CU" really means "per whatever functional block the waves of a
285   // workgroup must share". For gfx10 in CU mode this is the CU, which contains
286   // two SIMDs.
287   if (isGFX10(*STI) && STI->getFeatureBits().test(FeatureCuMode))
288     return 2;
289   // Pre-gfx10 a CU contains four SIMDs. For gfx10 in WGP mode the WGP contains
290   // two CUs, so a total of four SIMDs.
291   return 4;
292 }
293 
294 unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo *STI,
295                                unsigned FlatWorkGroupSize) {
296   assert(FlatWorkGroupSize != 0);
297   if (STI->getTargetTriple().getArch() != Triple::amdgcn)
298     return 8;
299   unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize);
300   if (N == 1)
301     return 40;
302   N = 40 / N;
303   return std::min(N, 16u);
304 }
305 
306 unsigned getMinWavesPerEU(const MCSubtargetInfo *STI) {
307   return 1;
308 }
309 
310 unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI) {
311   // FIXME: Need to take scratch memory into account.
312   if (!isGFX10(*STI))
313     return 10;
314   return 20;
315 }
316 
317 unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo *STI,
318                                    unsigned FlatWorkGroupSize) {
319   return divideCeil(getWavesPerWorkGroup(STI, FlatWorkGroupSize),
320                     getEUsPerCU(STI));
321 }
322 
323 unsigned getMinFlatWorkGroupSize(const MCSubtargetInfo *STI) {
324   return 1;
325 }
326 
327 unsigned getMaxFlatWorkGroupSize(const MCSubtargetInfo *STI) {
328   // Some subtargets allow encoding 2048, but this isn't tested or supported.
329   return 1024;
330 }
331 
332 unsigned getWavesPerWorkGroup(const MCSubtargetInfo *STI,
333                               unsigned FlatWorkGroupSize) {
334   return divideCeil(FlatWorkGroupSize, getWavefrontSize(STI));
335 }
336 
337 unsigned getSGPRAllocGranule(const MCSubtargetInfo *STI) {
338   IsaVersion Version = getIsaVersion(STI->getCPU());
339   if (Version.Major >= 10)
340     return getAddressableNumSGPRs(STI);
341   if (Version.Major >= 8)
342     return 16;
343   return 8;
344 }
345 
346 unsigned getSGPREncodingGranule(const MCSubtargetInfo *STI) {
347   return 8;
348 }
349 
350 unsigned getTotalNumSGPRs(const MCSubtargetInfo *STI) {
351   IsaVersion Version = getIsaVersion(STI->getCPU());
352   if (Version.Major >= 8)
353     return 800;
354   return 512;
355 }
356 
357 unsigned getAddressableNumSGPRs(const MCSubtargetInfo *STI) {
358   if (STI->getFeatureBits().test(FeatureSGPRInitBug))
359     return FIXED_NUM_SGPRS_FOR_INIT_BUG;
360 
361   IsaVersion Version = getIsaVersion(STI->getCPU());
362   if (Version.Major >= 10)
363     return 106;
364   if (Version.Major >= 8)
365     return 102;
366   return 104;
367 }
368 
369 unsigned getMinNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
370   assert(WavesPerEU != 0);
371 
372   IsaVersion Version = getIsaVersion(STI->getCPU());
373   if (Version.Major >= 10)
374     return 0;
375 
376   if (WavesPerEU >= getMaxWavesPerEU(STI))
377     return 0;
378 
379   unsigned MinNumSGPRs = getTotalNumSGPRs(STI) / (WavesPerEU + 1);
380   if (STI->getFeatureBits().test(FeatureTrapHandler))
381     MinNumSGPRs -= std::min(MinNumSGPRs, (unsigned)TRAP_NUM_SGPRS);
382   MinNumSGPRs = alignDown(MinNumSGPRs, getSGPRAllocGranule(STI)) + 1;
383   return std::min(MinNumSGPRs, getAddressableNumSGPRs(STI));
384 }
385 
386 unsigned getMaxNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU,
387                         bool Addressable) {
388   assert(WavesPerEU != 0);
389 
390   unsigned AddressableNumSGPRs = getAddressableNumSGPRs(STI);
391   IsaVersion Version = getIsaVersion(STI->getCPU());
392   if (Version.Major >= 10)
393     return Addressable ? AddressableNumSGPRs : 108;
394   if (Version.Major >= 8 && !Addressable)
395     AddressableNumSGPRs = 112;
396   unsigned MaxNumSGPRs = getTotalNumSGPRs(STI) / WavesPerEU;
397   if (STI->getFeatureBits().test(FeatureTrapHandler))
398     MaxNumSGPRs -= std::min(MaxNumSGPRs, (unsigned)TRAP_NUM_SGPRS);
399   MaxNumSGPRs = alignDown(MaxNumSGPRs, getSGPRAllocGranule(STI));
400   return std::min(MaxNumSGPRs, AddressableNumSGPRs);
401 }
402 
403 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed,
404                           bool FlatScrUsed, bool XNACKUsed) {
405   unsigned ExtraSGPRs = 0;
406   if (VCCUsed)
407     ExtraSGPRs = 2;
408 
409   IsaVersion Version = getIsaVersion(STI->getCPU());
410   if (Version.Major >= 10)
411     return ExtraSGPRs;
412 
413   if (Version.Major < 8) {
414     if (FlatScrUsed)
415       ExtraSGPRs = 4;
416   } else {
417     if (XNACKUsed)
418       ExtraSGPRs = 4;
419 
420     if (FlatScrUsed)
421       ExtraSGPRs = 6;
422   }
423 
424   return ExtraSGPRs;
425 }
426 
427 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed,
428                           bool FlatScrUsed) {
429   return getNumExtraSGPRs(STI, VCCUsed, FlatScrUsed,
430                           STI->getFeatureBits().test(AMDGPU::FeatureXNACK));
431 }
432 
433 unsigned getNumSGPRBlocks(const MCSubtargetInfo *STI, unsigned NumSGPRs) {
434   NumSGPRs = alignTo(std::max(1u, NumSGPRs), getSGPREncodingGranule(STI));
435   // SGPRBlocks is actual number of SGPR blocks minus 1.
436   return NumSGPRs / getSGPREncodingGranule(STI) - 1;
437 }
438 
439 unsigned getVGPRAllocGranule(const MCSubtargetInfo *STI,
440                              Optional<bool> EnableWavefrontSize32) {
441   bool IsWave32 = EnableWavefrontSize32 ?
442       *EnableWavefrontSize32 :
443       STI->getFeatureBits().test(FeatureWavefrontSize32);
444   return IsWave32 ? 8 : 4;
445 }
446 
447 unsigned getVGPREncodingGranule(const MCSubtargetInfo *STI,
448                                 Optional<bool> EnableWavefrontSize32) {
449   return getVGPRAllocGranule(STI, EnableWavefrontSize32);
450 }
451 
452 unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) {
453   if (!isGFX10(*STI))
454     return 256;
455   return STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1024 : 512;
456 }
457 
458 unsigned getAddressableNumVGPRs(const MCSubtargetInfo *STI) {
459   return 256;
460 }
461 
462 unsigned getMinNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
463   assert(WavesPerEU != 0);
464 
465   if (WavesPerEU >= getMaxWavesPerEU(STI))
466     return 0;
467   unsigned MinNumVGPRs =
468       alignDown(getTotalNumVGPRs(STI) / (WavesPerEU + 1),
469                 getVGPRAllocGranule(STI)) + 1;
470   return std::min(MinNumVGPRs, getAddressableNumVGPRs(STI));
471 }
472 
473 unsigned getMaxNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
474   assert(WavesPerEU != 0);
475 
476   unsigned MaxNumVGPRs = alignDown(getTotalNumVGPRs(STI) / WavesPerEU,
477                                    getVGPRAllocGranule(STI));
478   unsigned AddressableNumVGPRs = getAddressableNumVGPRs(STI);
479   return std::min(MaxNumVGPRs, AddressableNumVGPRs);
480 }
481 
482 unsigned getNumVGPRBlocks(const MCSubtargetInfo *STI, unsigned NumVGPRs,
483                           Optional<bool> EnableWavefrontSize32) {
484   NumVGPRs = alignTo(std::max(1u, NumVGPRs),
485                      getVGPREncodingGranule(STI, EnableWavefrontSize32));
486   // VGPRBlocks is actual number of VGPR blocks minus 1.
487   return NumVGPRs / getVGPREncodingGranule(STI, EnableWavefrontSize32) - 1;
488 }
489 
490 } // end namespace IsaInfo
491 
492 void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header,
493                                const MCSubtargetInfo *STI) {
494   IsaVersion Version = getIsaVersion(STI->getCPU());
495 
496   memset(&Header, 0, sizeof(Header));
497 
498   Header.amd_kernel_code_version_major = 1;
499   Header.amd_kernel_code_version_minor = 2;
500   Header.amd_machine_kind = 1; // AMD_MACHINE_KIND_AMDGPU
501   Header.amd_machine_version_major = Version.Major;
502   Header.amd_machine_version_minor = Version.Minor;
503   Header.amd_machine_version_stepping = Version.Stepping;
504   Header.kernel_code_entry_byte_offset = sizeof(Header);
505   Header.wavefront_size = 6;
506 
507   // If the code object does not support indirect functions, then the value must
508   // be 0xffffffff.
509   Header.call_convention = -1;
510 
511   // These alignment values are specified in powers of two, so alignment =
512   // 2^n.  The minimum alignment is 2^4 = 16.
513   Header.kernarg_segment_alignment = 4;
514   Header.group_segment_alignment = 4;
515   Header.private_segment_alignment = 4;
516 
517   if (Version.Major >= 10) {
518     if (STI->getFeatureBits().test(FeatureWavefrontSize32)) {
519       Header.wavefront_size = 5;
520       Header.code_properties |= AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32;
521     }
522     Header.compute_pgm_resource_registers |=
523       S_00B848_WGP_MODE(STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1) |
524       S_00B848_MEM_ORDERED(1);
525   }
526 }
527 
528 amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor(
529     const MCSubtargetInfo *STI) {
530   IsaVersion Version = getIsaVersion(STI->getCPU());
531 
532   amdhsa::kernel_descriptor_t KD;
533   memset(&KD, 0, sizeof(KD));
534 
535   AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
536                   amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64,
537                   amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE);
538   AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
539                   amdhsa::COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP, 1);
540   AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
541                   amdhsa::COMPUTE_PGM_RSRC1_ENABLE_IEEE_MODE, 1);
542   AMDHSA_BITS_SET(KD.compute_pgm_rsrc2,
543                   amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, 1);
544   if (Version.Major >= 10) {
545     AMDHSA_BITS_SET(KD.kernel_code_properties,
546                     amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32,
547                     STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1 : 0);
548     AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
549                     amdhsa::COMPUTE_PGM_RSRC1_WGP_MODE,
550                     STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1);
551     AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
552                     amdhsa::COMPUTE_PGM_RSRC1_MEM_ORDERED, 1);
553   }
554   return KD;
555 }
556 
557 bool isGroupSegment(const GlobalValue *GV) {
558   return GV->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS;
559 }
560 
561 bool isGlobalSegment(const GlobalValue *GV) {
562   return GV->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS;
563 }
564 
565 bool isReadOnlySegment(const GlobalValue *GV) {
566   unsigned AS = GV->getAddressSpace();
567   return AS == AMDGPUAS::CONSTANT_ADDRESS ||
568          AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
569 }
570 
571 bool shouldEmitConstantsToTextSection(const Triple &TT) {
572   return TT.getOS() == Triple::AMDPAL || TT.getArch() == Triple::r600;
573 }
574 
575 int getIntegerAttribute(const Function &F, StringRef Name, int Default) {
576   Attribute A = F.getFnAttribute(Name);
577   int Result = Default;
578 
579   if (A.isStringAttribute()) {
580     StringRef Str = A.getValueAsString();
581     if (Str.getAsInteger(0, Result)) {
582       LLVMContext &Ctx = F.getContext();
583       Ctx.emitError("can't parse integer attribute " + Name);
584     }
585   }
586 
587   return Result;
588 }
589 
590 std::pair<int, int> getIntegerPairAttribute(const Function &F,
591                                             StringRef Name,
592                                             std::pair<int, int> Default,
593                                             bool OnlyFirstRequired) {
594   Attribute A = F.getFnAttribute(Name);
595   if (!A.isStringAttribute())
596     return Default;
597 
598   LLVMContext &Ctx = F.getContext();
599   std::pair<int, int> Ints = Default;
600   std::pair<StringRef, StringRef> Strs = A.getValueAsString().split(',');
601   if (Strs.first.trim().getAsInteger(0, Ints.first)) {
602     Ctx.emitError("can't parse first integer attribute " + Name);
603     return Default;
604   }
605   if (Strs.second.trim().getAsInteger(0, Ints.second)) {
606     if (!OnlyFirstRequired || !Strs.second.trim().empty()) {
607       Ctx.emitError("can't parse second integer attribute " + Name);
608       return Default;
609     }
610   }
611 
612   return Ints;
613 }
614 
615 unsigned getVmcntBitMask(const IsaVersion &Version) {
616   unsigned VmcntLo = (1 << getVmcntBitWidthLo()) - 1;
617   if (Version.Major < 9)
618     return VmcntLo;
619 
620   unsigned VmcntHi = ((1 << getVmcntBitWidthHi()) - 1) << getVmcntBitWidthLo();
621   return VmcntLo | VmcntHi;
622 }
623 
624 unsigned getExpcntBitMask(const IsaVersion &Version) {
625   return (1 << getExpcntBitWidth()) - 1;
626 }
627 
628 unsigned getLgkmcntBitMask(const IsaVersion &Version) {
629   return (1 << getLgkmcntBitWidth(Version.Major)) - 1;
630 }
631 
632 unsigned getWaitcntBitMask(const IsaVersion &Version) {
633   unsigned VmcntLo = getBitMask(getVmcntBitShiftLo(), getVmcntBitWidthLo());
634   unsigned Expcnt = getBitMask(getExpcntBitShift(), getExpcntBitWidth());
635   unsigned Lgkmcnt = getBitMask(getLgkmcntBitShift(),
636                                 getLgkmcntBitWidth(Version.Major));
637   unsigned Waitcnt = VmcntLo | Expcnt | Lgkmcnt;
638   if (Version.Major < 9)
639     return Waitcnt;
640 
641   unsigned VmcntHi = getBitMask(getVmcntBitShiftHi(), getVmcntBitWidthHi());
642   return Waitcnt | VmcntHi;
643 }
644 
645 unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) {
646   unsigned VmcntLo =
647       unpackBits(Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo());
648   if (Version.Major < 9)
649     return VmcntLo;
650 
651   unsigned VmcntHi =
652       unpackBits(Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi());
653   VmcntHi <<= getVmcntBitWidthLo();
654   return VmcntLo | VmcntHi;
655 }
656 
657 unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) {
658   return unpackBits(Waitcnt, getExpcntBitShift(), getExpcntBitWidth());
659 }
660 
661 unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) {
662   return unpackBits(Waitcnt, getLgkmcntBitShift(),
663                     getLgkmcntBitWidth(Version.Major));
664 }
665 
666 void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt,
667                    unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt) {
668   Vmcnt = decodeVmcnt(Version, Waitcnt);
669   Expcnt = decodeExpcnt(Version, Waitcnt);
670   Lgkmcnt = decodeLgkmcnt(Version, Waitcnt);
671 }
672 
673 Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded) {
674   Waitcnt Decoded;
675   Decoded.VmCnt = decodeVmcnt(Version, Encoded);
676   Decoded.ExpCnt = decodeExpcnt(Version, Encoded);
677   Decoded.LgkmCnt = decodeLgkmcnt(Version, Encoded);
678   return Decoded;
679 }
680 
681 unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt,
682                      unsigned Vmcnt) {
683   Waitcnt =
684       packBits(Vmcnt, Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo());
685   if (Version.Major < 9)
686     return Waitcnt;
687 
688   Vmcnt >>= getVmcntBitWidthLo();
689   return packBits(Vmcnt, Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi());
690 }
691 
692 unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt,
693                       unsigned Expcnt) {
694   return packBits(Expcnt, Waitcnt, getExpcntBitShift(), getExpcntBitWidth());
695 }
696 
697 unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt,
698                        unsigned Lgkmcnt) {
699   return packBits(Lgkmcnt, Waitcnt, getLgkmcntBitShift(),
700                                     getLgkmcntBitWidth(Version.Major));
701 }
702 
703 unsigned encodeWaitcnt(const IsaVersion &Version,
704                        unsigned Vmcnt, unsigned Expcnt, unsigned Lgkmcnt) {
705   unsigned Waitcnt = getWaitcntBitMask(Version);
706   Waitcnt = encodeVmcnt(Version, Waitcnt, Vmcnt);
707   Waitcnt = encodeExpcnt(Version, Waitcnt, Expcnt);
708   Waitcnt = encodeLgkmcnt(Version, Waitcnt, Lgkmcnt);
709   return Waitcnt;
710 }
711 
712 unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded) {
713   return encodeWaitcnt(Version, Decoded.VmCnt, Decoded.ExpCnt, Decoded.LgkmCnt);
714 }
715 
716 //===----------------------------------------------------------------------===//
717 // hwreg
718 //===----------------------------------------------------------------------===//
719 
720 namespace Hwreg {
721 
722 int64_t getHwregId(const StringRef Name) {
723   for (int Id = ID_SYMBOLIC_FIRST_; Id < ID_SYMBOLIC_LAST_; ++Id) {
724     if (IdSymbolic[Id] && Name == IdSymbolic[Id])
725       return Id;
726   }
727   return ID_UNKNOWN_;
728 }
729 
730 static unsigned getLastSymbolicHwreg(const MCSubtargetInfo &STI) {
731   if (isSI(STI) || isCI(STI) || isVI(STI))
732     return ID_SYMBOLIC_FIRST_GFX9_;
733   else if (isGFX9(STI))
734     return ID_SYMBOLIC_FIRST_GFX10_;
735   else
736     return ID_SYMBOLIC_LAST_;
737 }
738 
739 bool isValidHwreg(int64_t Id, const MCSubtargetInfo &STI) {
740   return ID_SYMBOLIC_FIRST_ <= Id && Id < getLastSymbolicHwreg(STI) &&
741          IdSymbolic[Id];
742 }
743 
744 bool isValidHwreg(int64_t Id) {
745   return 0 <= Id && isUInt<ID_WIDTH_>(Id);
746 }
747 
748 bool isValidHwregOffset(int64_t Offset) {
749   return 0 <= Offset && isUInt<OFFSET_WIDTH_>(Offset);
750 }
751 
752 bool isValidHwregWidth(int64_t Width) {
753   return 0 <= (Width - 1) && isUInt<WIDTH_M1_WIDTH_>(Width - 1);
754 }
755 
756 uint64_t encodeHwreg(uint64_t Id, uint64_t Offset, uint64_t Width) {
757   return (Id << ID_SHIFT_) |
758          (Offset << OFFSET_SHIFT_) |
759          ((Width - 1) << WIDTH_M1_SHIFT_);
760 }
761 
762 StringRef getHwreg(unsigned Id, const MCSubtargetInfo &STI) {
763   return isValidHwreg(Id, STI) ? IdSymbolic[Id] : "";
764 }
765 
766 void decodeHwreg(unsigned Val, unsigned &Id, unsigned &Offset, unsigned &Width) {
767   Id = (Val & ID_MASK_) >> ID_SHIFT_;
768   Offset = (Val & OFFSET_MASK_) >> OFFSET_SHIFT_;
769   Width = ((Val & WIDTH_M1_MASK_) >> WIDTH_M1_SHIFT_) + 1;
770 }
771 
772 } // namespace Hwreg
773 
774 //===----------------------------------------------------------------------===//
775 // SendMsg
776 //===----------------------------------------------------------------------===//
777 
778 namespace SendMsg {
779 
780 int64_t getMsgId(const StringRef Name) {
781   for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
782     if (IdSymbolic[i] && Name == IdSymbolic[i])
783       return i;
784   }
785   return ID_UNKNOWN_;
786 }
787 
788 static bool isValidMsgId(int64_t MsgId) {
789   return (ID_GAPS_FIRST_ <= MsgId && MsgId < ID_GAPS_LAST_) && IdSymbolic[MsgId];
790 }
791 
792 bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI, bool Strict) {
793   if (Strict) {
794     if (MsgId == ID_GS_ALLOC_REQ || MsgId == ID_GET_DOORBELL)
795       return isGFX9(STI) || isGFX10(STI);
796     else
797       return isValidMsgId(MsgId);
798   } else {
799     return 0 <= MsgId && isUInt<ID_WIDTH_>(MsgId);
800   }
801 }
802 
803 StringRef getMsgName(int64_t MsgId) {
804   return isValidMsgId(MsgId)? IdSymbolic[MsgId] : "";
805 }
806 
807 int64_t getMsgOpId(int64_t MsgId, const StringRef Name) {
808   const char* const *S = (MsgId == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
809   const int F = (MsgId == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
810   const int L = (MsgId == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
811   for (int i = F; i < L; ++i) {
812     if (Name == S[i]) {
813       return i;
814     }
815   }
816   return OP_UNKNOWN_;
817 }
818 
819 bool isValidMsgOp(int64_t MsgId, int64_t OpId, bool Strict) {
820 
821   if (!Strict)
822     return 0 <= OpId && isUInt<OP_WIDTH_>(OpId);
823 
824   switch(MsgId)
825   {
826   case ID_GS:
827     return (OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_) && OpId != OP_GS_NOP;
828   case ID_GS_DONE:
829     return OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_;
830   case ID_SYSMSG:
831     return OP_SYS_FIRST_ <= OpId && OpId < OP_SYS_LAST_;
832   default:
833     return OpId == OP_NONE_;
834   }
835 }
836 
837 StringRef getMsgOpName(int64_t MsgId, int64_t OpId) {
838   assert(msgRequiresOp(MsgId));
839   return (MsgId == ID_SYSMSG)? OpSysSymbolic[OpId] : OpGsSymbolic[OpId];
840 }
841 
842 bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId, bool Strict) {
843 
844   if (!Strict)
845     return 0 <= StreamId && isUInt<STREAM_ID_WIDTH_>(StreamId);
846 
847   switch(MsgId)
848   {
849   case ID_GS:
850     return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_;
851   case ID_GS_DONE:
852     return (OpId == OP_GS_NOP)?
853            (StreamId == STREAM_ID_NONE_) :
854            (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_);
855   default:
856     return StreamId == STREAM_ID_NONE_;
857   }
858 }
859 
860 bool msgRequiresOp(int64_t MsgId) {
861   return MsgId == ID_GS || MsgId == ID_GS_DONE || MsgId == ID_SYSMSG;
862 }
863 
864 bool msgSupportsStream(int64_t MsgId, int64_t OpId) {
865   return (MsgId == ID_GS || MsgId == ID_GS_DONE) && OpId != OP_GS_NOP;
866 }
867 
868 void decodeMsg(unsigned Val,
869                uint16_t &MsgId,
870                uint16_t &OpId,
871                uint16_t &StreamId) {
872   MsgId = Val & ID_MASK_;
873   OpId = (Val & OP_MASK_) >> OP_SHIFT_;
874   StreamId = (Val & STREAM_ID_MASK_) >> STREAM_ID_SHIFT_;
875 }
876 
877 uint64_t encodeMsg(uint64_t MsgId,
878                    uint64_t OpId,
879                    uint64_t StreamId) {
880   return (MsgId << ID_SHIFT_) |
881          (OpId << OP_SHIFT_) |
882          (StreamId << STREAM_ID_SHIFT_);
883 }
884 
885 } // namespace SendMsg
886 
887 //===----------------------------------------------------------------------===//
888 //
889 //===----------------------------------------------------------------------===//
890 
891 unsigned getInitialPSInputAddr(const Function &F) {
892   return getIntegerAttribute(F, "InitialPSInputAddr", 0);
893 }
894 
895 bool isShader(CallingConv::ID cc) {
896   switch(cc) {
897     case CallingConv::AMDGPU_VS:
898     case CallingConv::AMDGPU_LS:
899     case CallingConv::AMDGPU_HS:
900     case CallingConv::AMDGPU_ES:
901     case CallingConv::AMDGPU_GS:
902     case CallingConv::AMDGPU_PS:
903     case CallingConv::AMDGPU_CS:
904       return true;
905     default:
906       return false;
907   }
908 }
909 
910 bool isCompute(CallingConv::ID cc) {
911   return !isShader(cc) || cc == CallingConv::AMDGPU_CS;
912 }
913 
914 bool isEntryFunctionCC(CallingConv::ID CC) {
915   switch (CC) {
916   case CallingConv::AMDGPU_KERNEL:
917   case CallingConv::SPIR_KERNEL:
918   case CallingConv::AMDGPU_VS:
919   case CallingConv::AMDGPU_GS:
920   case CallingConv::AMDGPU_PS:
921   case CallingConv::AMDGPU_CS:
922   case CallingConv::AMDGPU_ES:
923   case CallingConv::AMDGPU_HS:
924   case CallingConv::AMDGPU_LS:
925     return true;
926   default:
927     return false;
928   }
929 }
930 
931 bool hasXNACK(const MCSubtargetInfo &STI) {
932   return STI.getFeatureBits()[AMDGPU::FeatureXNACK];
933 }
934 
935 bool hasSRAMECC(const MCSubtargetInfo &STI) {
936   return STI.getFeatureBits()[AMDGPU::FeatureSRAMECC];
937 }
938 
939 bool hasMIMG_R128(const MCSubtargetInfo &STI) {
940   return STI.getFeatureBits()[AMDGPU::FeatureMIMG_R128] && !STI.getFeatureBits()[AMDGPU::FeatureR128A16];
941 }
942 
943 bool hasGFX10A16(const MCSubtargetInfo &STI) {
944   return STI.getFeatureBits()[AMDGPU::FeatureGFX10A16];
945 }
946 
947 bool hasG16(const MCSubtargetInfo &STI) {
948   return STI.getFeatureBits()[AMDGPU::FeatureG16];
949 }
950 
951 bool hasPackedD16(const MCSubtargetInfo &STI) {
952   return !STI.getFeatureBits()[AMDGPU::FeatureUnpackedD16VMem];
953 }
954 
955 bool isSI(const MCSubtargetInfo &STI) {
956   return STI.getFeatureBits()[AMDGPU::FeatureSouthernIslands];
957 }
958 
959 bool isCI(const MCSubtargetInfo &STI) {
960   return STI.getFeatureBits()[AMDGPU::FeatureSeaIslands];
961 }
962 
963 bool isVI(const MCSubtargetInfo &STI) {
964   return STI.getFeatureBits()[AMDGPU::FeatureVolcanicIslands];
965 }
966 
967 bool isGFX9(const MCSubtargetInfo &STI) {
968   return STI.getFeatureBits()[AMDGPU::FeatureGFX9];
969 }
970 
971 bool isGFX10(const MCSubtargetInfo &STI) {
972   return STI.getFeatureBits()[AMDGPU::FeatureGFX10];
973 }
974 
975 bool isGCN3Encoding(const MCSubtargetInfo &STI) {
976   return STI.getFeatureBits()[AMDGPU::FeatureGCN3Encoding];
977 }
978 
979 bool isSGPR(unsigned Reg, const MCRegisterInfo* TRI) {
980   const MCRegisterClass SGPRClass = TRI->getRegClass(AMDGPU::SReg_32RegClassID);
981   const unsigned FirstSubReg = TRI->getSubReg(Reg, AMDGPU::sub0);
982   return SGPRClass.contains(FirstSubReg != 0 ? FirstSubReg : Reg) ||
983     Reg == AMDGPU::SCC;
984 }
985 
986 bool isRegIntersect(unsigned Reg0, unsigned Reg1, const MCRegisterInfo* TRI) {
987   for (MCRegAliasIterator R(Reg0, TRI, true); R.isValid(); ++R) {
988     if (*R == Reg1) return true;
989   }
990   return false;
991 }
992 
993 #define MAP_REG2REG \
994   using namespace AMDGPU; \
995   switch(Reg) { \
996   default: return Reg; \
997   CASE_CI_VI(FLAT_SCR) \
998   CASE_CI_VI(FLAT_SCR_LO) \
999   CASE_CI_VI(FLAT_SCR_HI) \
1000   CASE_VI_GFX9_GFX10(TTMP0) \
1001   CASE_VI_GFX9_GFX10(TTMP1) \
1002   CASE_VI_GFX9_GFX10(TTMP2) \
1003   CASE_VI_GFX9_GFX10(TTMP3) \
1004   CASE_VI_GFX9_GFX10(TTMP4) \
1005   CASE_VI_GFX9_GFX10(TTMP5) \
1006   CASE_VI_GFX9_GFX10(TTMP6) \
1007   CASE_VI_GFX9_GFX10(TTMP7) \
1008   CASE_VI_GFX9_GFX10(TTMP8) \
1009   CASE_VI_GFX9_GFX10(TTMP9) \
1010   CASE_VI_GFX9_GFX10(TTMP10) \
1011   CASE_VI_GFX9_GFX10(TTMP11) \
1012   CASE_VI_GFX9_GFX10(TTMP12) \
1013   CASE_VI_GFX9_GFX10(TTMP13) \
1014   CASE_VI_GFX9_GFX10(TTMP14) \
1015   CASE_VI_GFX9_GFX10(TTMP15) \
1016   CASE_VI_GFX9_GFX10(TTMP0_TTMP1) \
1017   CASE_VI_GFX9_GFX10(TTMP2_TTMP3) \
1018   CASE_VI_GFX9_GFX10(TTMP4_TTMP5) \
1019   CASE_VI_GFX9_GFX10(TTMP6_TTMP7) \
1020   CASE_VI_GFX9_GFX10(TTMP8_TTMP9) \
1021   CASE_VI_GFX9_GFX10(TTMP10_TTMP11) \
1022   CASE_VI_GFX9_GFX10(TTMP12_TTMP13) \
1023   CASE_VI_GFX9_GFX10(TTMP14_TTMP15) \
1024   CASE_VI_GFX9_GFX10(TTMP0_TTMP1_TTMP2_TTMP3) \
1025   CASE_VI_GFX9_GFX10(TTMP4_TTMP5_TTMP6_TTMP7) \
1026   CASE_VI_GFX9_GFX10(TTMP8_TTMP9_TTMP10_TTMP11) \
1027   CASE_VI_GFX9_GFX10(TTMP12_TTMP13_TTMP14_TTMP15) \
1028   CASE_VI_GFX9_GFX10(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7) \
1029   CASE_VI_GFX9_GFX10(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11) \
1030   CASE_VI_GFX9_GFX10(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
1031   CASE_VI_GFX9_GFX10(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
1032   }
1033 
1034 #define CASE_CI_VI(node) \
1035   assert(!isSI(STI)); \
1036   case node: return isCI(STI) ? node##_ci : node##_vi;
1037 
1038 #define CASE_VI_GFX9_GFX10(node) \
1039   case node: return (isGFX9(STI) || isGFX10(STI)) ? node##_gfx9_gfx10 : node##_vi;
1040 
1041 unsigned getMCReg(unsigned Reg, const MCSubtargetInfo &STI) {
1042   if (STI.getTargetTriple().getArch() == Triple::r600)
1043     return Reg;
1044   MAP_REG2REG
1045 }
1046 
1047 #undef CASE_CI_VI
1048 #undef CASE_VI_GFX9_GFX10
1049 
1050 #define CASE_CI_VI(node)   case node##_ci: case node##_vi:   return node;
1051 #define CASE_VI_GFX9_GFX10(node) case node##_vi: case node##_gfx9_gfx10: return node;
1052 
1053 unsigned mc2PseudoReg(unsigned Reg) {
1054   MAP_REG2REG
1055 }
1056 
1057 #undef CASE_CI_VI
1058 #undef CASE_VI_GFX9_GFX10
1059 #undef MAP_REG2REG
1060 
1061 bool isSISrcOperand(const MCInstrDesc &Desc, unsigned OpNo) {
1062   assert(OpNo < Desc.NumOperands);
1063   unsigned OpType = Desc.OpInfo[OpNo].OperandType;
1064   return OpType >= AMDGPU::OPERAND_SRC_FIRST &&
1065          OpType <= AMDGPU::OPERAND_SRC_LAST;
1066 }
1067 
1068 bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo) {
1069   assert(OpNo < Desc.NumOperands);
1070   unsigned OpType = Desc.OpInfo[OpNo].OperandType;
1071   switch (OpType) {
1072   case AMDGPU::OPERAND_REG_IMM_FP32:
1073   case AMDGPU::OPERAND_REG_IMM_FP64:
1074   case AMDGPU::OPERAND_REG_IMM_FP16:
1075   case AMDGPU::OPERAND_REG_IMM_V2FP16:
1076   case AMDGPU::OPERAND_REG_IMM_V2INT16:
1077   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1078   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1079   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1080   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1081   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1082   case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
1083   case AMDGPU::OPERAND_REG_INLINE_AC_FP16:
1084   case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16:
1085   case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16:
1086     return true;
1087   default:
1088     return false;
1089   }
1090 }
1091 
1092 bool isSISrcInlinableOperand(const MCInstrDesc &Desc, unsigned OpNo) {
1093   assert(OpNo < Desc.NumOperands);
1094   unsigned OpType = Desc.OpInfo[OpNo].OperandType;
1095   return OpType >= AMDGPU::OPERAND_REG_INLINE_C_FIRST &&
1096          OpType <= AMDGPU::OPERAND_REG_INLINE_C_LAST;
1097 }
1098 
1099 // Avoid using MCRegisterClass::getSize, since that function will go away
1100 // (move from MC* level to Target* level). Return size in bits.
1101 unsigned getRegBitWidth(unsigned RCID) {
1102   switch (RCID) {
1103   case AMDGPU::VGPR_LO16RegClassID:
1104   case AMDGPU::VGPR_HI16RegClassID:
1105   case AMDGPU::SGPR_LO16RegClassID:
1106   case AMDGPU::AGPR_LO16RegClassID:
1107     return 16;
1108   case AMDGPU::SGPR_32RegClassID:
1109   case AMDGPU::VGPR_32RegClassID:
1110   case AMDGPU::VRegOrLds_32RegClassID:
1111   case AMDGPU::AGPR_32RegClassID:
1112   case AMDGPU::VS_32RegClassID:
1113   case AMDGPU::AV_32RegClassID:
1114   case AMDGPU::SReg_32RegClassID:
1115   case AMDGPU::SReg_32_XM0RegClassID:
1116   case AMDGPU::SRegOrLds_32RegClassID:
1117     return 32;
1118   case AMDGPU::SGPR_64RegClassID:
1119   case AMDGPU::VS_64RegClassID:
1120   case AMDGPU::AV_64RegClassID:
1121   case AMDGPU::SReg_64RegClassID:
1122   case AMDGPU::VReg_64RegClassID:
1123   case AMDGPU::AReg_64RegClassID:
1124   case AMDGPU::SReg_64_XEXECRegClassID:
1125     return 64;
1126   case AMDGPU::SGPR_96RegClassID:
1127   case AMDGPU::SReg_96RegClassID:
1128   case AMDGPU::VReg_96RegClassID:
1129   case AMDGPU::AReg_96RegClassID:
1130     return 96;
1131   case AMDGPU::SGPR_128RegClassID:
1132   case AMDGPU::SReg_128RegClassID:
1133   case AMDGPU::VReg_128RegClassID:
1134   case AMDGPU::AReg_128RegClassID:
1135     return 128;
1136   case AMDGPU::SGPR_160RegClassID:
1137   case AMDGPU::SReg_160RegClassID:
1138   case AMDGPU::VReg_160RegClassID:
1139   case AMDGPU::AReg_160RegClassID:
1140     return 160;
1141   case AMDGPU::SGPR_192RegClassID:
1142   case AMDGPU::SReg_192RegClassID:
1143   case AMDGPU::VReg_192RegClassID:
1144   case AMDGPU::AReg_192RegClassID:
1145     return 192;
1146   case AMDGPU::SGPR_256RegClassID:
1147   case AMDGPU::SReg_256RegClassID:
1148   case AMDGPU::VReg_256RegClassID:
1149   case AMDGPU::AReg_256RegClassID:
1150     return 256;
1151   case AMDGPU::SGPR_512RegClassID:
1152   case AMDGPU::SReg_512RegClassID:
1153   case AMDGPU::VReg_512RegClassID:
1154   case AMDGPU::AReg_512RegClassID:
1155     return 512;
1156   case AMDGPU::SGPR_1024RegClassID:
1157   case AMDGPU::SReg_1024RegClassID:
1158   case AMDGPU::VReg_1024RegClassID:
1159   case AMDGPU::AReg_1024RegClassID:
1160     return 1024;
1161   default:
1162     llvm_unreachable("Unexpected register class");
1163   }
1164 }
1165 
1166 unsigned getRegBitWidth(const MCRegisterClass &RC) {
1167   return getRegBitWidth(RC.getID());
1168 }
1169 
1170 unsigned getRegOperandSize(const MCRegisterInfo *MRI, const MCInstrDesc &Desc,
1171                            unsigned OpNo) {
1172   assert(OpNo < Desc.NumOperands);
1173   unsigned RCID = Desc.OpInfo[OpNo].RegClass;
1174   return getRegBitWidth(MRI->getRegClass(RCID)) / 8;
1175 }
1176 
1177 bool isInlinableIntLiteral(int64_t Literal) {
1178   return Literal >= -16 && Literal <= 64;
1179 }
1180 
1181 bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi) {
1182   if (isInlinableIntLiteral(Literal))
1183     return true;
1184 
1185   uint64_t Val = static_cast<uint64_t>(Literal);
1186   return (Val == DoubleToBits(0.0)) ||
1187          (Val == DoubleToBits(1.0)) ||
1188          (Val == DoubleToBits(-1.0)) ||
1189          (Val == DoubleToBits(0.5)) ||
1190          (Val == DoubleToBits(-0.5)) ||
1191          (Val == DoubleToBits(2.0)) ||
1192          (Val == DoubleToBits(-2.0)) ||
1193          (Val == DoubleToBits(4.0)) ||
1194          (Val == DoubleToBits(-4.0)) ||
1195          (Val == 0x3fc45f306dc9c882 && HasInv2Pi);
1196 }
1197 
1198 bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi) {
1199   if (isInlinableIntLiteral(Literal))
1200     return true;
1201 
1202   // The actual type of the operand does not seem to matter as long
1203   // as the bits match one of the inline immediate values.  For example:
1204   //
1205   // -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal,
1206   // so it is a legal inline immediate.
1207   //
1208   // 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in
1209   // floating-point, so it is a legal inline immediate.
1210 
1211   uint32_t Val = static_cast<uint32_t>(Literal);
1212   return (Val == FloatToBits(0.0f)) ||
1213          (Val == FloatToBits(1.0f)) ||
1214          (Val == FloatToBits(-1.0f)) ||
1215          (Val == FloatToBits(0.5f)) ||
1216          (Val == FloatToBits(-0.5f)) ||
1217          (Val == FloatToBits(2.0f)) ||
1218          (Val == FloatToBits(-2.0f)) ||
1219          (Val == FloatToBits(4.0f)) ||
1220          (Val == FloatToBits(-4.0f)) ||
1221          (Val == 0x3e22f983 && HasInv2Pi);
1222 }
1223 
1224 bool isInlinableLiteral16(int16_t Literal, bool HasInv2Pi) {
1225   if (!HasInv2Pi)
1226     return false;
1227 
1228   if (isInlinableIntLiteral(Literal))
1229     return true;
1230 
1231   uint16_t Val = static_cast<uint16_t>(Literal);
1232   return Val == 0x3C00 || // 1.0
1233          Val == 0xBC00 || // -1.0
1234          Val == 0x3800 || // 0.5
1235          Val == 0xB800 || // -0.5
1236          Val == 0x4000 || // 2.0
1237          Val == 0xC000 || // -2.0
1238          Val == 0x4400 || // 4.0
1239          Val == 0xC400 || // -4.0
1240          Val == 0x3118;   // 1/2pi
1241 }
1242 
1243 bool isInlinableLiteralV216(int32_t Literal, bool HasInv2Pi) {
1244   assert(HasInv2Pi);
1245 
1246   if (isInt<16>(Literal) || isUInt<16>(Literal)) {
1247     int16_t Trunc = static_cast<int16_t>(Literal);
1248     return AMDGPU::isInlinableLiteral16(Trunc, HasInv2Pi);
1249   }
1250   if (!(Literal & 0xffff))
1251     return AMDGPU::isInlinableLiteral16(Literal >> 16, HasInv2Pi);
1252 
1253   int16_t Lo16 = static_cast<int16_t>(Literal);
1254   int16_t Hi16 = static_cast<int16_t>(Literal >> 16);
1255   return Lo16 == Hi16 && isInlinableLiteral16(Lo16, HasInv2Pi);
1256 }
1257 
1258 bool isArgPassedInSGPR(const Argument *A) {
1259   const Function *F = A->getParent();
1260 
1261   // Arguments to compute shaders are never a source of divergence.
1262   CallingConv::ID CC = F->getCallingConv();
1263   switch (CC) {
1264   case CallingConv::AMDGPU_KERNEL:
1265   case CallingConv::SPIR_KERNEL:
1266     return true;
1267   case CallingConv::AMDGPU_VS:
1268   case CallingConv::AMDGPU_LS:
1269   case CallingConv::AMDGPU_HS:
1270   case CallingConv::AMDGPU_ES:
1271   case CallingConv::AMDGPU_GS:
1272   case CallingConv::AMDGPU_PS:
1273   case CallingConv::AMDGPU_CS:
1274     // For non-compute shaders, SGPR inputs are marked with either inreg or byval.
1275     // Everything else is in VGPRs.
1276     return F->getAttributes().hasParamAttribute(A->getArgNo(), Attribute::InReg) ||
1277            F->getAttributes().hasParamAttribute(A->getArgNo(), Attribute::ByVal);
1278   default:
1279     // TODO: Should calls support inreg for SGPR inputs?
1280     return false;
1281   }
1282 }
1283 
1284 static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) {
1285   return isGCN3Encoding(ST) || isGFX10(ST);
1286 }
1287 
1288 static bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST) {
1289   return isGFX9(ST) || isGFX10(ST);
1290 }
1291 
1292 bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST,
1293                                       int64_t EncodedOffset) {
1294   return hasSMEMByteOffset(ST) ? isUInt<20>(EncodedOffset)
1295                                : isUInt<8>(EncodedOffset);
1296 }
1297 
1298 bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST,
1299                                     int64_t EncodedOffset,
1300                                     bool IsBuffer) {
1301   return !IsBuffer &&
1302          hasSMRDSignedImmOffset(ST) &&
1303          isInt<21>(EncodedOffset);
1304 }
1305 
1306 static bool isDwordAligned(uint64_t ByteOffset) {
1307   return (ByteOffset & 3) == 0;
1308 }
1309 
1310 uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST,
1311                                 uint64_t ByteOffset) {
1312   if (hasSMEMByteOffset(ST))
1313     return ByteOffset;
1314 
1315   assert(isDwordAligned(ByteOffset));
1316   return ByteOffset >> 2;
1317 }
1318 
1319 Optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST,
1320                                        int64_t ByteOffset, bool IsBuffer) {
1321   // The signed version is always a byte offset.
1322   if (!IsBuffer && hasSMRDSignedImmOffset(ST)) {
1323     assert(hasSMEMByteOffset(ST));
1324     return isInt<20>(ByteOffset) ? Optional<int64_t>(ByteOffset) : None;
1325   }
1326 
1327   if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST))
1328     return None;
1329 
1330   int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
1331   return isLegalSMRDEncodedUnsignedOffset(ST, EncodedOffset)
1332              ? Optional<int64_t>(EncodedOffset)
1333              : None;
1334 }
1335 
1336 Optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST,
1337                                                 int64_t ByteOffset) {
1338   if (!isCI(ST) || !isDwordAligned(ByteOffset))
1339     return None;
1340 
1341   int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
1342   return isUInt<32>(EncodedOffset) ? Optional<int64_t>(EncodedOffset) : None;
1343 }
1344 
1345 // Given Imm, split it into the values to put into the SOffset and ImmOffset
1346 // fields in an MUBUF instruction. Return false if it is not possible (due to a
1347 // hardware bug needing a workaround).
1348 //
1349 // The required alignment ensures that individual address components remain
1350 // aligned if they are aligned to begin with. It also ensures that additional
1351 // offsets within the given alignment can be added to the resulting ImmOffset.
1352 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset,
1353                       const GCNSubtarget *Subtarget, uint32_t Align) {
1354   const uint32_t MaxImm = alignDown(4095, Align);
1355   uint32_t Overflow = 0;
1356 
1357   if (Imm > MaxImm) {
1358     if (Imm <= MaxImm + 64) {
1359       // Use an SOffset inline constant for 4..64
1360       Overflow = Imm - MaxImm;
1361       Imm = MaxImm;
1362     } else {
1363       // Try to keep the same value in SOffset for adjacent loads, so that
1364       // the corresponding register contents can be re-used.
1365       //
1366       // Load values with all low-bits (except for alignment bits) set into
1367       // SOffset, so that a larger range of values can be covered using
1368       // s_movk_i32.
1369       //
1370       // Atomic operations fail to work correctly when individual address
1371       // components are unaligned, even if their sum is aligned.
1372       uint32_t High = (Imm + Align) & ~4095;
1373       uint32_t Low = (Imm + Align) & 4095;
1374       Imm = Low;
1375       Overflow = High - Align;
1376     }
1377   }
1378 
1379   // There is a hardware bug in SI and CI which prevents address clamping in
1380   // MUBUF instructions from working correctly with SOffsets. The immediate
1381   // offset is unaffected.
1382   if (Overflow > 0 &&
1383       Subtarget->getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS)
1384     return false;
1385 
1386   ImmOffset = Imm;
1387   SOffset = Overflow;
1388   return true;
1389 }
1390 
1391 SIModeRegisterDefaults::SIModeRegisterDefaults(const Function &F) {
1392   *this = getDefaultForCallingConv(F.getCallingConv());
1393 
1394   StringRef IEEEAttr = F.getFnAttribute("amdgpu-ieee").getValueAsString();
1395   if (!IEEEAttr.empty())
1396     IEEE = IEEEAttr == "true";
1397 
1398   StringRef DX10ClampAttr
1399     = F.getFnAttribute("amdgpu-dx10-clamp").getValueAsString();
1400   if (!DX10ClampAttr.empty())
1401     DX10Clamp = DX10ClampAttr == "true";
1402 
1403   StringRef DenormF32Attr = F.getFnAttribute("denormal-fp-math-f32").getValueAsString();
1404   if (!DenormF32Attr.empty()) {
1405     DenormalMode DenormMode = parseDenormalFPAttribute(DenormF32Attr);
1406     FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE;
1407     FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE;
1408   }
1409 
1410   StringRef DenormAttr = F.getFnAttribute("denormal-fp-math").getValueAsString();
1411   if (!DenormAttr.empty()) {
1412     DenormalMode DenormMode = parseDenormalFPAttribute(DenormAttr);
1413 
1414     if (DenormF32Attr.empty()) {
1415       FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE;
1416       FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE;
1417     }
1418 
1419     FP64FP16InputDenormals = DenormMode.Input == DenormalMode::IEEE;
1420     FP64FP16OutputDenormals = DenormMode.Output == DenormalMode::IEEE;
1421   }
1422 }
1423 
1424 namespace {
1425 
1426 struct SourceOfDivergence {
1427   unsigned Intr;
1428 };
1429 const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr);
1430 
1431 #define GET_SourcesOfDivergence_IMPL
1432 #define GET_Gfx9BufferFormat_IMPL
1433 #define GET_Gfx10PlusBufferFormat_IMPL
1434 #include "AMDGPUGenSearchableTables.inc"
1435 
1436 } // end anonymous namespace
1437 
1438 bool isIntrinsicSourceOfDivergence(unsigned IntrID) {
1439   return lookupSourceOfDivergence(IntrID);
1440 }
1441 
1442 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp,
1443                                                   uint8_t NumComponents,
1444                                                   uint8_t NumFormat,
1445                                                   const MCSubtargetInfo &STI) {
1446   return isGFX10(STI)
1447              ? getGfx10PlusBufferFormatInfo(BitsPerComp, NumComponents,
1448                                             NumFormat)
1449              : getGfx9BufferFormatInfo(BitsPerComp, NumComponents, NumFormat);
1450 }
1451 
1452 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format,
1453                                                   const MCSubtargetInfo &STI) {
1454   return isGFX10(STI) ? getGfx10PlusBufferFormatInfo(Format)
1455                       : getGfx9BufferFormatInfo(Format);
1456 }
1457 
1458 } // namespace AMDGPU
1459 } // namespace llvm
1460