1 //=====-- AMDGPUSubtarget.h - Define Subtarget for AMDGPU ------*- C++ -*-====//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //==-----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// AMDGPU specific subclass of TargetSubtarget.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUSUBTARGET_H
15 #define LLVM_LIB_TARGET_AMDGPU_AMDGPUSUBTARGET_H
16 
17 #include "AMDGPU.h"
18 #include "AMDGPUCallLowering.h"
19 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
20 #include "R600FrameLowering.h"
21 #include "R600ISelLowering.h"
22 #include "R600InstrInfo.h"
23 #include "SIFrameLowering.h"
24 #include "SIISelLowering.h"
25 #include "SIInstrInfo.h"
26 #include "Utils/AMDGPUBaseInfo.h"
27 #include "llvm/ADT/Triple.h"
28 #include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
29 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
30 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
31 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
32 #include "llvm/CodeGen/MachineFunction.h"
33 #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
34 #include "llvm/MC/MCInstrItineraries.h"
35 #include "llvm/Support/MathExtras.h"
36 #include <cassert>
37 #include <cstdint>
38 #include <memory>
39 #include <utility>
40 
41 #define GET_SUBTARGETINFO_HEADER
42 #include "AMDGPUGenSubtargetInfo.inc"
43 #define GET_SUBTARGETINFO_HEADER
44 #include "R600GenSubtargetInfo.inc"
45 
46 namespace llvm {
47 
48 class StringRef;
49 
50 class AMDGPUSubtarget {
51 public:
52   enum Generation {
53     R600 = 0,
54     R700 = 1,
55     EVERGREEN = 2,
56     NORTHERN_ISLANDS = 3,
57     SOUTHERN_ISLANDS = 4,
58     SEA_ISLANDS = 5,
59     VOLCANIC_ISLANDS = 6,
60     GFX9 = 7,
61     GFX10 = 8
62   };
63 
64 private:
65   Triple TargetTriple;
66 
67 protected:
68   bool Has16BitInsts;
69   bool HasMadMixInsts;
70   bool HasMadMacF32Insts;
71   bool HasDsSrc2Insts;
72   bool HasSDWA;
73   bool HasVOP3PInsts;
74   bool HasMulI24;
75   bool HasMulU24;
76   bool HasInv2PiInlineImm;
77   bool HasFminFmaxLegacy;
78   bool EnablePromoteAlloca;
79   bool HasTrigReducedRange;
80   unsigned MaxWavesPerEU;
81   int LocalMemorySize;
82   char WavefrontSizeLog2;
83 
84 public:
85   AMDGPUSubtarget(const Triple &TT);
86 
87   static const AMDGPUSubtarget &get(const MachineFunction &MF);
88   static const AMDGPUSubtarget &get(const TargetMachine &TM,
89                                     const Function &F);
90 
91   /// \returns Default range flat work group size for a calling convention.
92   std::pair<unsigned, unsigned> getDefaultFlatWorkGroupSize(CallingConv::ID CC) const;
93 
94   /// \returns Subtarget's default pair of minimum/maximum flat work group sizes
95   /// for function \p F, or minimum/maximum flat work group sizes explicitly
96   /// requested using "amdgpu-flat-work-group-size" attribute attached to
97   /// function \p F.
98   ///
99   /// \returns Subtarget's default values if explicitly requested values cannot
100   /// be converted to integer, or violate subtarget's specifications.
101   std::pair<unsigned, unsigned> getFlatWorkGroupSizes(const Function &F) const;
102 
103   /// \returns Subtarget's default pair of minimum/maximum number of waves per
104   /// execution unit for function \p F, or minimum/maximum number of waves per
105   /// execution unit explicitly requested using "amdgpu-waves-per-eu" attribute
106   /// attached to function \p F.
107   ///
108   /// \returns Subtarget's default values if explicitly requested values cannot
109   /// be converted to integer, violate subtarget's specifications, or are not
110   /// compatible with minimum/maximum number of waves limited by flat work group
111   /// size, register usage, and/or lds usage.
112   std::pair<unsigned, unsigned> getWavesPerEU(const Function &F) const;
113 
114   /// Return the amount of LDS that can be used that will not restrict the
115   /// occupancy lower than WaveCount.
116   unsigned getMaxLocalMemSizeWithWaveCount(unsigned WaveCount,
117                                            const Function &) const;
118 
119   /// Inverse of getMaxLocalMemWithWaveCount. Return the maximum wavecount if
120   /// the given LDS memory size is the only constraint.
121   unsigned getOccupancyWithLocalMemSize(uint32_t Bytes, const Function &) const;
122 
123   unsigned getOccupancyWithLocalMemSize(const MachineFunction &MF) const;
124 
125   bool isAmdHsaOS() const {
126     return TargetTriple.getOS() == Triple::AMDHSA;
127   }
128 
129   bool isAmdPalOS() const {
130     return TargetTriple.getOS() == Triple::AMDPAL;
131   }
132 
133   bool isMesa3DOS() const {
134     return TargetTriple.getOS() == Triple::Mesa3D;
135   }
136 
137   bool isMesaKernel(const Function &F) const {
138     return isMesa3DOS() && !AMDGPU::isShader(F.getCallingConv());
139   }
140 
141   bool isAmdHsaOrMesa(const Function &F) const {
142     return isAmdHsaOS() || isMesaKernel(F);
143   }
144 
145   bool isGCN() const {
146     return TargetTriple.getArch() == Triple::amdgcn;
147   }
148 
149   bool has16BitInsts() const {
150     return Has16BitInsts;
151   }
152 
153   bool hasMadMixInsts() const {
154     return HasMadMixInsts;
155   }
156 
157   bool hasMadMacF32Insts() const {
158     return HasMadMacF32Insts || !isGCN();
159   }
160 
161   bool hasDsSrc2Insts() const {
162     return HasDsSrc2Insts;
163   }
164 
165   bool hasSDWA() const {
166     return HasSDWA;
167   }
168 
169   bool hasVOP3PInsts() const {
170     return HasVOP3PInsts;
171   }
172 
173   bool hasMulI24() const {
174     return HasMulI24;
175   }
176 
177   bool hasMulU24() const {
178     return HasMulU24;
179   }
180 
181   bool hasInv2PiInlineImm() const {
182     return HasInv2PiInlineImm;
183   }
184 
185   bool hasFminFmaxLegacy() const {
186     return HasFminFmaxLegacy;
187   }
188 
189   bool hasTrigReducedRange() const {
190     return HasTrigReducedRange;
191   }
192 
193   bool isPromoteAllocaEnabled() const {
194     return EnablePromoteAlloca;
195   }
196 
197   unsigned getWavefrontSize() const {
198     return 1 << WavefrontSizeLog2;
199   }
200 
201   unsigned getWavefrontSizeLog2() const {
202     return WavefrontSizeLog2;
203   }
204 
205   int getLocalMemorySize() const {
206     return LocalMemorySize;
207   }
208 
209   Align getAlignmentForImplicitArgPtr() const {
210     return isAmdHsaOS() ? Align(8) : Align(4);
211   }
212 
213   /// Returns the offset in bytes from the start of the input buffer
214   ///        of the first explicit kernel argument.
215   unsigned getExplicitKernelArgOffset(const Function &F) const {
216     return isAmdHsaOrMesa(F) ? 0 : 36;
217   }
218 
219   /// \returns Maximum number of work groups per compute unit supported by the
220   /// subtarget and limited by given \p FlatWorkGroupSize.
221   virtual unsigned getMaxWorkGroupsPerCU(unsigned FlatWorkGroupSize) const = 0;
222 
223   /// \returns Minimum flat work group size supported by the subtarget.
224   virtual unsigned getMinFlatWorkGroupSize() const = 0;
225 
226   /// \returns Maximum flat work group size supported by the subtarget.
227   virtual unsigned getMaxFlatWorkGroupSize() const = 0;
228 
229   /// \returns Number of waves per execution unit required to support the given
230   /// \p FlatWorkGroupSize.
231   virtual unsigned
232   getWavesPerEUForWorkGroup(unsigned FlatWorkGroupSize) const = 0;
233 
234   /// \returns Minimum number of waves per execution unit supported by the
235   /// subtarget.
236   virtual unsigned getMinWavesPerEU() const = 0;
237 
238   /// \returns Maximum number of waves per execution unit supported by the
239   /// subtarget without any kind of limitation.
240   unsigned getMaxWavesPerEU() const { return MaxWavesPerEU; }
241 
242   /// Creates value range metadata on an workitemid.* inrinsic call or load.
243   bool makeLIDRangeMetadata(Instruction *I) const;
244 
245   /// \returns Number of bytes of arguments that are passed to a shader or
246   /// kernel in addition to the explicit ones declared for the function.
247   unsigned getImplicitArgNumBytes(const Function &F) const {
248     if (isMesaKernel(F))
249       return 16;
250     return AMDGPU::getIntegerAttribute(F, "amdgpu-implicitarg-num-bytes", 0);
251   }
252   uint64_t getExplicitKernArgSize(const Function &F, Align &MaxAlign) const;
253   unsigned getKernArgSegmentSize(const Function &F, Align &MaxAlign) const;
254 
255   /// \returns Corresponsing DWARF register number mapping flavour for the
256   /// \p WavefrontSize.
257   AMDGPUDwarfFlavour getAMDGPUDwarfFlavour() const {
258     return getWavefrontSize() == 32 ? AMDGPUDwarfFlavour::Wave32
259                                     : AMDGPUDwarfFlavour::Wave64;
260   }
261 
262   virtual ~AMDGPUSubtarget() {}
263 };
264 
265 class GCNSubtarget : public AMDGPUGenSubtargetInfo,
266                      public AMDGPUSubtarget {
267 
268   using AMDGPUSubtarget::getMaxWavesPerEU;
269 
270 public:
271   enum TrapHandlerAbi {
272     TrapHandlerAbiNone = 0,
273     TrapHandlerAbiHsa = 1
274   };
275 
276   enum TrapID {
277     TrapIDHardwareReserved = 0,
278     TrapIDHSADebugTrap = 1,
279     TrapIDLLVMTrap = 2,
280     TrapIDLLVMDebugTrap = 3,
281     TrapIDDebugBreakpoint = 7,
282     TrapIDDebugReserved8 = 8,
283     TrapIDDebugReservedFE = 0xfe,
284     TrapIDDebugReservedFF = 0xff
285   };
286 
287   enum TrapRegValues {
288     LLVMTrapHandlerRegValue = 1
289   };
290 
291 private:
292   /// GlobalISel related APIs.
293   std::unique_ptr<AMDGPUCallLowering> CallLoweringInfo;
294   std::unique_ptr<InlineAsmLowering> InlineAsmLoweringInfo;
295   std::unique_ptr<InstructionSelector> InstSelector;
296   std::unique_ptr<LegalizerInfo> Legalizer;
297   std::unique_ptr<RegisterBankInfo> RegBankInfo;
298 
299 protected:
300   // Basic subtarget description.
301   Triple TargetTriple;
302   unsigned Gen;
303   InstrItineraryData InstrItins;
304   int LDSBankCount;
305   unsigned MaxPrivateElementSize;
306 
307   // Possibly statically set by tablegen, but may want to be overridden.
308   bool FastFMAF32;
309   bool FastDenormalF32;
310   bool HalfRate64Ops;
311 
312   // Dynamially set bits that enable features.
313   bool FlatForGlobal;
314   bool AutoWaitcntBeforeBarrier;
315   bool CodeObjectV3;
316   bool UnalignedScratchAccess;
317   bool UnalignedAccessMode;
318   bool HasApertureRegs;
319   bool EnableXNACK;
320   bool DoesNotSupportXNACK;
321   bool EnableCuMode;
322   bool TrapHandler;
323 
324   // Used as options.
325   bool EnableLoadStoreOpt;
326   bool EnableUnsafeDSOffsetFolding;
327   bool EnableSIScheduler;
328   bool EnableDS128;
329   bool EnablePRTStrictNull;
330   bool DumpCode;
331 
332   // Subtarget statically properties set by tablegen
333   bool FP64;
334   bool FMA;
335   bool MIMG_R128;
336   bool IsGCN;
337   bool GCN3Encoding;
338   bool CIInsts;
339   bool GFX8Insts;
340   bool GFX9Insts;
341   bool GFX10Insts;
342   bool GFX10_3Insts;
343   bool GFX7GFX8GFX9Insts;
344   bool SGPRInitBug;
345   bool HasSMemRealTime;
346   bool HasIntClamp;
347   bool HasFmaMixInsts;
348   bool HasMovrel;
349   bool HasVGPRIndexMode;
350   bool HasScalarStores;
351   bool HasScalarAtomics;
352   bool HasSDWAOmod;
353   bool HasSDWAScalar;
354   bool HasSDWASdst;
355   bool HasSDWAMac;
356   bool HasSDWAOutModsVOPC;
357   bool HasDPP;
358   bool HasDPP8;
359   bool HasR128A16;
360   bool HasGFX10A16;
361   bool HasG16;
362   bool HasNSAEncoding;
363   bool GFX10_BEncoding;
364   bool HasDLInsts;
365   bool HasDot1Insts;
366   bool HasDot2Insts;
367   bool HasDot3Insts;
368   bool HasDot4Insts;
369   bool HasDot5Insts;
370   bool HasDot6Insts;
371   bool HasMAIInsts;
372   bool HasPkFmacF16Inst;
373   bool HasAtomicFaddInsts;
374   bool EnableSRAMECC;
375   bool DoesNotSupportSRAMECC;
376   bool HasNoSdstCMPX;
377   bool HasVscnt;
378   bool HasGetWaveIdInst;
379   bool HasSMemTimeInst;
380   bool HasRegisterBanking;
381   bool HasVOP3Literal;
382   bool HasNoDataDepHazard;
383   bool FlatAddressSpace;
384   bool FlatInstOffsets;
385   bool FlatGlobalInsts;
386   bool FlatScratchInsts;
387   bool ScalarFlatScratchInsts;
388   bool AddNoCarryInsts;
389   bool HasUnpackedD16VMem;
390   bool R600ALUInst;
391   bool CaymanISA;
392   bool CFALUBug;
393   bool LDSMisalignedBug;
394   bool HasMFMAInlineLiteralBug;
395   bool HasVertexCache;
396   short TexVTXClauseSize;
397   bool UnalignedBufferAccess;
398   bool UnalignedDSAccess;
399   bool ScalarizeGlobal;
400 
401   bool HasVcmpxPermlaneHazard;
402   bool HasVMEMtoScalarWriteHazard;
403   bool HasSMEMtoVectorWriteHazard;
404   bool HasInstFwdPrefetchBug;
405   bool HasVcmpxExecWARHazard;
406   bool HasLdsBranchVmemWARHazard;
407   bool HasNSAtoVMEMBug;
408   bool HasOffset3fBug;
409   bool HasFlatSegmentOffsetBug;
410 
411   // Dummy feature to use for assembler in tablegen.
412   bool FeatureDisable;
413 
414   SelectionDAGTargetInfo TSInfo;
415 private:
416   SIInstrInfo InstrInfo;
417   SITargetLowering TLInfo;
418   SIFrameLowering FrameLowering;
419 
420   // See COMPUTE_TMPRING_SIZE.WAVESIZE, 13-bit field in units of 256-dword.
421   static const unsigned MaxWaveScratchSize = (256 * 4) * ((1 << 13) - 1);
422 
423 public:
424   GCNSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
425                const GCNTargetMachine &TM);
426   ~GCNSubtarget() override;
427 
428   GCNSubtarget &initializeSubtargetDependencies(const Triple &TT,
429                                                    StringRef GPU, StringRef FS);
430 
431   const SIInstrInfo *getInstrInfo() const override {
432     return &InstrInfo;
433   }
434 
435   const SIFrameLowering *getFrameLowering() const override {
436     return &FrameLowering;
437   }
438 
439   const SITargetLowering *getTargetLowering() const override {
440     return &TLInfo;
441   }
442 
443   const SIRegisterInfo *getRegisterInfo() const override {
444     return &InstrInfo.getRegisterInfo();
445   }
446 
447   const CallLowering *getCallLowering() const override {
448     return CallLoweringInfo.get();
449   }
450 
451   const InlineAsmLowering *getInlineAsmLowering() const override {
452     return InlineAsmLoweringInfo.get();
453   }
454 
455   InstructionSelector *getInstructionSelector() const override {
456     return InstSelector.get();
457   }
458 
459   const LegalizerInfo *getLegalizerInfo() const override {
460     return Legalizer.get();
461   }
462 
463   const RegisterBankInfo *getRegBankInfo() const override {
464     return RegBankInfo.get();
465   }
466 
467   // Nothing implemented, just prevent crashes on use.
468   const SelectionDAGTargetInfo *getSelectionDAGInfo() const override {
469     return &TSInfo;
470   }
471 
472   const InstrItineraryData *getInstrItineraryData() const override {
473     return &InstrItins;
474   }
475 
476   void ParseSubtargetFeatures(StringRef CPU, StringRef TuneCPU, StringRef FS);
477 
478   Generation getGeneration() const {
479     return (Generation)Gen;
480   }
481 
482   /// Return the number of high bits known to be zero fror a frame index.
483   unsigned getKnownHighZeroBitsForFrameIndex() const {
484     return countLeadingZeros(MaxWaveScratchSize) + getWavefrontSizeLog2();
485   }
486 
487   int getLDSBankCount() const {
488     return LDSBankCount;
489   }
490 
491   unsigned getMaxPrivateElementSize() const {
492     return MaxPrivateElementSize;
493   }
494 
495   unsigned getConstantBusLimit(unsigned Opcode) const;
496 
497   bool hasIntClamp() const {
498     return HasIntClamp;
499   }
500 
501   bool hasFP64() const {
502     return FP64;
503   }
504 
505   bool hasMIMG_R128() const {
506     return MIMG_R128;
507   }
508 
509   bool hasHWFP64() const {
510     return FP64;
511   }
512 
513   bool hasFastFMAF32() const {
514     return FastFMAF32;
515   }
516 
517   bool hasHalfRate64Ops() const {
518     return HalfRate64Ops;
519   }
520 
521   bool hasAddr64() const {
522     return (getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS);
523   }
524 
525   // Return true if the target only has the reverse operand versions of VALU
526   // shift instructions (e.g. v_lshrrev_b32, and no v_lshr_b32).
527   bool hasOnlyRevVALUShifts() const {
528     return getGeneration() >= VOLCANIC_ISLANDS;
529   }
530 
531   bool hasFractBug() const {
532     return getGeneration() == SOUTHERN_ISLANDS;
533   }
534 
535   bool hasBFE() const {
536     return true;
537   }
538 
539   bool hasBFI() const {
540     return true;
541   }
542 
543   bool hasBFM() const {
544     return hasBFE();
545   }
546 
547   bool hasBCNT(unsigned Size) const {
548     return true;
549   }
550 
551   bool hasFFBL() const {
552     return true;
553   }
554 
555   bool hasFFBH() const {
556     return true;
557   }
558 
559   bool hasMed3_16() const {
560     return getGeneration() >= AMDGPUSubtarget::GFX9;
561   }
562 
563   bool hasMin3Max3_16() const {
564     return getGeneration() >= AMDGPUSubtarget::GFX9;
565   }
566 
567   bool hasFmaMixInsts() const {
568     return HasFmaMixInsts;
569   }
570 
571   bool hasCARRY() const {
572     return true;
573   }
574 
575   bool hasFMA() const {
576     return FMA;
577   }
578 
579   bool hasSwap() const {
580     return GFX9Insts;
581   }
582 
583   bool hasScalarPackInsts() const {
584     return GFX9Insts;
585   }
586 
587   bool hasScalarMulHiInsts() const {
588     return GFX9Insts;
589   }
590 
591   TrapHandlerAbi getTrapHandlerAbi() const {
592     return isAmdHsaOS() ? TrapHandlerAbiHsa : TrapHandlerAbiNone;
593   }
594 
595   /// True if the offset field of DS instructions works as expected. On SI, the
596   /// offset uses a 16-bit adder and does not always wrap properly.
597   bool hasUsableDSOffset() const {
598     return getGeneration() >= SEA_ISLANDS;
599   }
600 
601   bool unsafeDSOffsetFoldingEnabled() const {
602     return EnableUnsafeDSOffsetFolding;
603   }
604 
605   /// Condition output from div_scale is usable.
606   bool hasUsableDivScaleConditionOutput() const {
607     return getGeneration() != SOUTHERN_ISLANDS;
608   }
609 
610   /// Extra wait hazard is needed in some cases before
611   /// s_cbranch_vccnz/s_cbranch_vccz.
612   bool hasReadVCCZBug() const {
613     return getGeneration() <= SEA_ISLANDS;
614   }
615 
616   /// Writes to VCC_LO/VCC_HI update the VCCZ flag.
617   bool partialVCCWritesUpdateVCCZ() const {
618     return getGeneration() >= GFX10;
619   }
620 
621   /// A read of an SGPR by SMRD instruction requires 4 wait states when the SGPR
622   /// was written by a VALU instruction.
623   bool hasSMRDReadVALUDefHazard() const {
624     return getGeneration() == SOUTHERN_ISLANDS;
625   }
626 
627   /// A read of an SGPR by a VMEM instruction requires 5 wait states when the
628   /// SGPR was written by a VALU Instruction.
629   bool hasVMEMReadSGPRVALUDefHazard() const {
630     return getGeneration() >= VOLCANIC_ISLANDS;
631   }
632 
633   bool hasRFEHazards() const {
634     return getGeneration() >= VOLCANIC_ISLANDS;
635   }
636 
637   /// Number of hazard wait states for s_setreg_b32/s_setreg_imm32_b32.
638   unsigned getSetRegWaitStates() const {
639     return getGeneration() <= SEA_ISLANDS ? 1 : 2;
640   }
641 
642   bool dumpCode() const {
643     return DumpCode;
644   }
645 
646   /// Return the amount of LDS that can be used that will not restrict the
647   /// occupancy lower than WaveCount.
648   unsigned getMaxLocalMemSizeWithWaveCount(unsigned WaveCount,
649                                            const Function &) const;
650 
651   bool supportsMinMaxDenormModes() const {
652     return getGeneration() >= AMDGPUSubtarget::GFX9;
653   }
654 
655   /// \returns If target supports S_DENORM_MODE.
656   bool hasDenormModeInst() const {
657     return getGeneration() >= AMDGPUSubtarget::GFX10;
658   }
659 
660   bool useFlatForGlobal() const {
661     return FlatForGlobal;
662   }
663 
664   /// \returns If target supports ds_read/write_b128 and user enables generation
665   /// of ds_read/write_b128.
666   bool useDS128() const {
667     return CIInsts && EnableDS128;
668   }
669 
670   /// \return If target supports ds_read/write_b96/128.
671   bool hasDS96AndDS128() const {
672     return CIInsts;
673   }
674 
675   /// Have v_trunc_f64, v_ceil_f64, v_rndne_f64
676   bool haveRoundOpsF64() const {
677     return CIInsts;
678   }
679 
680   /// \returns If MUBUF instructions always perform range checking, even for
681   /// buffer resources used for private memory access.
682   bool privateMemoryResourceIsRangeChecked() const {
683     return getGeneration() < AMDGPUSubtarget::GFX9;
684   }
685 
686   /// \returns If target requires PRT Struct NULL support (zero result registers
687   /// for sparse texture support).
688   bool usePRTStrictNull() const {
689     return EnablePRTStrictNull;
690   }
691 
692   bool hasAutoWaitcntBeforeBarrier() const {
693     return AutoWaitcntBeforeBarrier;
694   }
695 
696   bool hasCodeObjectV3() const {
697     // FIXME: Need to add code object v3 support for mesa and pal.
698     return isAmdHsaOS() ? CodeObjectV3 : false;
699   }
700 
701   bool hasUnalignedBufferAccess() const {
702     return UnalignedBufferAccess;
703   }
704 
705   bool hasUnalignedBufferAccessEnabled() const {
706     return UnalignedBufferAccess && UnalignedAccessMode;
707   }
708 
709   bool hasUnalignedDSAccess() const {
710     return UnalignedDSAccess;
711   }
712 
713   bool hasUnalignedDSAccessEnabled() const {
714     return UnalignedDSAccess && UnalignedAccessMode;
715   }
716 
717   bool hasUnalignedScratchAccess() const {
718     return UnalignedScratchAccess;
719   }
720 
721   bool hasUnalignedAccessMode() const {
722     return UnalignedAccessMode;
723   }
724 
725   bool hasApertureRegs() const {
726     return HasApertureRegs;
727   }
728 
729   bool isTrapHandlerEnabled() const {
730     return TrapHandler;
731   }
732 
733   bool isXNACKEnabled() const {
734     return EnableXNACK;
735   }
736 
737   bool isCuModeEnabled() const {
738     return EnableCuMode;
739   }
740 
741   bool hasFlatAddressSpace() const {
742     return FlatAddressSpace;
743   }
744 
745   bool hasFlatScrRegister() const {
746     return hasFlatAddressSpace();
747   }
748 
749   bool hasFlatInstOffsets() const {
750     return FlatInstOffsets;
751   }
752 
753   bool hasFlatGlobalInsts() const {
754     return FlatGlobalInsts;
755   }
756 
757   bool hasFlatScratchInsts() const {
758     return FlatScratchInsts;
759   }
760 
761   bool hasScalarFlatScratchInsts() const {
762     return ScalarFlatScratchInsts;
763   }
764 
765   bool hasGlobalAddTidInsts() const {
766     return GFX10_BEncoding;
767   }
768 
769   bool hasAtomicCSub() const {
770     return GFX10_BEncoding;
771   }
772 
773   bool hasMultiDwordFlatScratchAddressing() const {
774     return getGeneration() >= GFX9;
775   }
776 
777   bool hasFlatSegmentOffsetBug() const {
778     return HasFlatSegmentOffsetBug;
779   }
780 
781   bool hasFlatLgkmVMemCountInOrder() const {
782     return getGeneration() > GFX9;
783   }
784 
785   bool hasD16LoadStore() const {
786     return getGeneration() >= GFX9;
787   }
788 
789   bool d16PreservesUnusedBits() const {
790     return hasD16LoadStore() && !isSRAMECCEnabled();
791   }
792 
793   bool hasD16Images() const {
794     return getGeneration() >= VOLCANIC_ISLANDS;
795   }
796 
797   /// Return if most LDS instructions have an m0 use that require m0 to be
798   /// iniitalized.
799   bool ldsRequiresM0Init() const {
800     return getGeneration() < GFX9;
801   }
802 
803   // True if the hardware rewinds and replays GWS operations if a wave is
804   // preempted.
805   //
806   // If this is false, a GWS operation requires testing if a nack set the
807   // MEM_VIOL bit, and repeating if so.
808   bool hasGWSAutoReplay() const {
809     return getGeneration() >= GFX9;
810   }
811 
812   /// \returns if target has ds_gws_sema_release_all instruction.
813   bool hasGWSSemaReleaseAll() const {
814     return CIInsts;
815   }
816 
817   /// \returns true if the target has integer add/sub instructions that do not
818   /// produce a carry-out. This includes v_add_[iu]32, v_sub_[iu]32,
819   /// v_add_[iu]16, and v_sub_[iu]16, all of which support the clamp modifier
820   /// for saturation.
821   bool hasAddNoCarry() const {
822     return AddNoCarryInsts;
823   }
824 
825   bool hasUnpackedD16VMem() const {
826     return HasUnpackedD16VMem;
827   }
828 
829   // Covers VS/PS/CS graphics shaders
830   bool isMesaGfxShader(const Function &F) const {
831     return isMesa3DOS() && AMDGPU::isShader(F.getCallingConv());
832   }
833 
834   bool hasMad64_32() const {
835     return getGeneration() >= SEA_ISLANDS;
836   }
837 
838   bool hasSDWAOmod() const {
839     return HasSDWAOmod;
840   }
841 
842   bool hasSDWAScalar() const {
843     return HasSDWAScalar;
844   }
845 
846   bool hasSDWASdst() const {
847     return HasSDWASdst;
848   }
849 
850   bool hasSDWAMac() const {
851     return HasSDWAMac;
852   }
853 
854   bool hasSDWAOutModsVOPC() const {
855     return HasSDWAOutModsVOPC;
856   }
857 
858   bool hasDLInsts() const {
859     return HasDLInsts;
860   }
861 
862   bool hasDot1Insts() const {
863     return HasDot1Insts;
864   }
865 
866   bool hasDot2Insts() const {
867     return HasDot2Insts;
868   }
869 
870   bool hasDot3Insts() const {
871     return HasDot3Insts;
872   }
873 
874   bool hasDot4Insts() const {
875     return HasDot4Insts;
876   }
877 
878   bool hasDot5Insts() const {
879     return HasDot5Insts;
880   }
881 
882   bool hasDot6Insts() const {
883     return HasDot6Insts;
884   }
885 
886   bool hasMAIInsts() const {
887     return HasMAIInsts;
888   }
889 
890   bool hasPkFmacF16Inst() const {
891     return HasPkFmacF16Inst;
892   }
893 
894   bool hasAtomicFaddInsts() const {
895     return HasAtomicFaddInsts;
896   }
897 
898   bool isSRAMECCEnabled() const {
899     return EnableSRAMECC;
900   }
901 
902   bool hasNoSdstCMPX() const {
903     return HasNoSdstCMPX;
904   }
905 
906   bool hasVscnt() const {
907     return HasVscnt;
908   }
909 
910   bool hasGetWaveIdInst() const {
911     return HasGetWaveIdInst;
912   }
913 
914   bool hasSMemTimeInst() const {
915     return HasSMemTimeInst;
916   }
917 
918   bool hasRegisterBanking() const {
919     return HasRegisterBanking;
920   }
921 
922   bool hasVOP3Literal() const {
923     return HasVOP3Literal;
924   }
925 
926   bool hasNoDataDepHazard() const {
927     return HasNoDataDepHazard;
928   }
929 
930   bool vmemWriteNeedsExpWaitcnt() const {
931     return getGeneration() < SEA_ISLANDS;
932   }
933 
934   // Scratch is allocated in 256 dword per wave blocks for the entire
935   // wavefront. When viewed from the perspecive of an arbitrary workitem, this
936   // is 4-byte aligned.
937   //
938   // Only 4-byte alignment is really needed to access anything. Transformations
939   // on the pointer value itself may rely on the alignment / known low bits of
940   // the pointer. Set this to something above the minimum to avoid needing
941   // dynamic realignment in common cases.
942   Align getStackAlignment() const { return Align(16); }
943 
944   bool enableMachineScheduler() const override {
945     return true;
946   }
947 
948   bool enableSubRegLiveness() const override {
949     return true;
950   }
951 
952   void setScalarizeGlobalBehavior(bool b) { ScalarizeGlobal = b; }
953   bool getScalarizeGlobalBehavior() const { return ScalarizeGlobal; }
954 
955   // static wrappers
956   static bool hasHalfRate64Ops(const TargetSubtargetInfo &STI);
957 
958   // XXX - Why is this here if it isn't in the default pass set?
959   bool enableEarlyIfConversion() const override {
960     return true;
961   }
962 
963   void overrideSchedPolicy(MachineSchedPolicy &Policy,
964                            unsigned NumRegionInstrs) const override;
965 
966   unsigned getMaxNumUserSGPRs() const {
967     return 16;
968   }
969 
970   bool hasSMemRealTime() const {
971     return HasSMemRealTime;
972   }
973 
974   bool hasMovrel() const {
975     return HasMovrel;
976   }
977 
978   bool hasVGPRIndexMode() const {
979     return HasVGPRIndexMode;
980   }
981 
982   bool useVGPRIndexMode() const;
983 
984   bool hasScalarCompareEq64() const {
985     return getGeneration() >= VOLCANIC_ISLANDS;
986   }
987 
988   bool hasScalarStores() const {
989     return HasScalarStores;
990   }
991 
992   bool hasScalarAtomics() const {
993     return HasScalarAtomics;
994   }
995 
996   bool hasLDSFPAtomics() const {
997     return GFX8Insts;
998   }
999 
1000   bool hasDPP() const {
1001     return HasDPP;
1002   }
1003 
1004   bool hasDPPBroadcasts() const {
1005     return HasDPP && getGeneration() < GFX10;
1006   }
1007 
1008   bool hasDPPWavefrontShifts() const {
1009     return HasDPP && getGeneration() < GFX10;
1010   }
1011 
1012   bool hasDPP8() const {
1013     return HasDPP8;
1014   }
1015 
1016   bool hasR128A16() const {
1017     return HasR128A16;
1018   }
1019 
1020   bool hasGFX10A16() const {
1021     return HasGFX10A16;
1022   }
1023 
1024   bool hasA16() const { return hasR128A16() || hasGFX10A16(); }
1025 
1026   bool hasG16() const { return HasG16; }
1027 
1028   bool hasOffset3fBug() const {
1029     return HasOffset3fBug;
1030   }
1031 
1032   bool hasNSAEncoding() const {
1033     return HasNSAEncoding;
1034   }
1035 
1036   bool hasGFX10_BEncoding() const {
1037     return GFX10_BEncoding;
1038   }
1039 
1040   bool hasGFX10_3Insts() const {
1041     return GFX10_3Insts;
1042   }
1043 
1044   bool hasMadF16() const;
1045 
1046   bool enableSIScheduler() const {
1047     return EnableSIScheduler;
1048   }
1049 
1050   bool loadStoreOptEnabled() const {
1051     return EnableLoadStoreOpt;
1052   }
1053 
1054   bool hasSGPRInitBug() const {
1055     return SGPRInitBug;
1056   }
1057 
1058   bool hasMFMAInlineLiteralBug() const {
1059     return HasMFMAInlineLiteralBug;
1060   }
1061 
1062   bool has12DWordStoreHazard() const {
1063     return getGeneration() != AMDGPUSubtarget::SOUTHERN_ISLANDS;
1064   }
1065 
1066   // \returns true if the subtarget supports DWORDX3 load/store instructions.
1067   bool hasDwordx3LoadStores() const {
1068     return CIInsts;
1069   }
1070 
1071   bool hasReadM0MovRelInterpHazard() const {
1072     return getGeneration() == AMDGPUSubtarget::GFX9;
1073   }
1074 
1075   bool hasReadM0SendMsgHazard() const {
1076     return getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
1077            getGeneration() <= AMDGPUSubtarget::GFX9;
1078   }
1079 
1080   bool hasVcmpxPermlaneHazard() const {
1081     return HasVcmpxPermlaneHazard;
1082   }
1083 
1084   bool hasVMEMtoScalarWriteHazard() const {
1085     return HasVMEMtoScalarWriteHazard;
1086   }
1087 
1088   bool hasSMEMtoVectorWriteHazard() const {
1089     return HasSMEMtoVectorWriteHazard;
1090   }
1091 
1092   bool hasLDSMisalignedBug() const {
1093     return LDSMisalignedBug && !EnableCuMode;
1094   }
1095 
1096   bool hasInstFwdPrefetchBug() const {
1097     return HasInstFwdPrefetchBug;
1098   }
1099 
1100   bool hasVcmpxExecWARHazard() const {
1101     return HasVcmpxExecWARHazard;
1102   }
1103 
1104   bool hasLdsBranchVmemWARHazard() const {
1105     return HasLdsBranchVmemWARHazard;
1106   }
1107 
1108   bool hasNSAtoVMEMBug() const {
1109     return HasNSAtoVMEMBug;
1110   }
1111 
1112   bool hasHardClauses() const { return getGeneration() >= GFX10; }
1113 
1114   /// Return the maximum number of waves per SIMD for kernels using \p SGPRs
1115   /// SGPRs
1116   unsigned getOccupancyWithNumSGPRs(unsigned SGPRs) const;
1117 
1118   /// Return the maximum number of waves per SIMD for kernels using \p VGPRs
1119   /// VGPRs
1120   unsigned getOccupancyWithNumVGPRs(unsigned VGPRs) const;
1121 
1122   /// Return occupancy for the given function. Used LDS and a number of
1123   /// registers if provided.
1124   /// Note, occupancy can be affected by the scratch allocation as well, but
1125   /// we do not have enough information to compute it.
1126   unsigned computeOccupancy(const Function &F, unsigned LDSSize = 0,
1127                             unsigned NumSGPRs = 0, unsigned NumVGPRs = 0) const;
1128 
1129   /// \returns true if the flat_scratch register should be initialized with the
1130   /// pointer to the wave's scratch memory rather than a size and offset.
1131   bool flatScratchIsPointer() const {
1132     return getGeneration() >= AMDGPUSubtarget::GFX9;
1133   }
1134 
1135   /// \returns true if the machine has merged shaders in which s0-s7 are
1136   /// reserved by the hardware and user SGPRs start at s8
1137   bool hasMergedShaders() const {
1138     return getGeneration() >= GFX9;
1139   }
1140 
1141   /// \returns SGPR allocation granularity supported by the subtarget.
1142   unsigned getSGPRAllocGranule() const {
1143     return AMDGPU::IsaInfo::getSGPRAllocGranule(this);
1144   }
1145 
1146   /// \returns SGPR encoding granularity supported by the subtarget.
1147   unsigned getSGPREncodingGranule() const {
1148     return AMDGPU::IsaInfo::getSGPREncodingGranule(this);
1149   }
1150 
1151   /// \returns Total number of SGPRs supported by the subtarget.
1152   unsigned getTotalNumSGPRs() const {
1153     return AMDGPU::IsaInfo::getTotalNumSGPRs(this);
1154   }
1155 
1156   /// \returns Addressable number of SGPRs supported by the subtarget.
1157   unsigned getAddressableNumSGPRs() const {
1158     return AMDGPU::IsaInfo::getAddressableNumSGPRs(this);
1159   }
1160 
1161   /// \returns Minimum number of SGPRs that meets the given number of waves per
1162   /// execution unit requirement supported by the subtarget.
1163   unsigned getMinNumSGPRs(unsigned WavesPerEU) const {
1164     return AMDGPU::IsaInfo::getMinNumSGPRs(this, WavesPerEU);
1165   }
1166 
1167   /// \returns Maximum number of SGPRs that meets the given number of waves per
1168   /// execution unit requirement supported by the subtarget.
1169   unsigned getMaxNumSGPRs(unsigned WavesPerEU, bool Addressable) const {
1170     return AMDGPU::IsaInfo::getMaxNumSGPRs(this, WavesPerEU, Addressable);
1171   }
1172 
1173   /// \returns Reserved number of SGPRs for given function \p MF.
1174   unsigned getReservedNumSGPRs(const MachineFunction &MF) const;
1175 
1176   /// \returns Maximum number of SGPRs that meets number of waves per execution
1177   /// unit requirement for function \p MF, or number of SGPRs explicitly
1178   /// requested using "amdgpu-num-sgpr" attribute attached to function \p MF.
1179   ///
1180   /// \returns Value that meets number of waves per execution unit requirement
1181   /// if explicitly requested value cannot be converted to integer, violates
1182   /// subtarget's specifications, or does not meet number of waves per execution
1183   /// unit requirement.
1184   unsigned getMaxNumSGPRs(const MachineFunction &MF) const;
1185 
1186   /// \returns VGPR allocation granularity supported by the subtarget.
1187   unsigned getVGPRAllocGranule() const {
1188     return AMDGPU::IsaInfo::getVGPRAllocGranule(this);
1189   }
1190 
1191   /// \returns VGPR encoding granularity supported by the subtarget.
1192   unsigned getVGPREncodingGranule() const {
1193     return AMDGPU::IsaInfo::getVGPREncodingGranule(this);
1194   }
1195 
1196   /// \returns Total number of VGPRs supported by the subtarget.
1197   unsigned getTotalNumVGPRs() const {
1198     return AMDGPU::IsaInfo::getTotalNumVGPRs(this);
1199   }
1200 
1201   /// \returns Addressable number of VGPRs supported by the subtarget.
1202   unsigned getAddressableNumVGPRs() const {
1203     return AMDGPU::IsaInfo::getAddressableNumVGPRs(this);
1204   }
1205 
1206   /// \returns Minimum number of VGPRs that meets given number of waves per
1207   /// execution unit requirement supported by the subtarget.
1208   unsigned getMinNumVGPRs(unsigned WavesPerEU) const {
1209     return AMDGPU::IsaInfo::getMinNumVGPRs(this, WavesPerEU);
1210   }
1211 
1212   /// \returns Maximum number of VGPRs that meets given number of waves per
1213   /// execution unit requirement supported by the subtarget.
1214   unsigned getMaxNumVGPRs(unsigned WavesPerEU) const {
1215     return AMDGPU::IsaInfo::getMaxNumVGPRs(this, WavesPerEU);
1216   }
1217 
1218   /// \returns Maximum number of VGPRs that meets number of waves per execution
1219   /// unit requirement for function \p MF, or number of VGPRs explicitly
1220   /// requested using "amdgpu-num-vgpr" attribute attached to function \p MF.
1221   ///
1222   /// \returns Value that meets number of waves per execution unit requirement
1223   /// if explicitly requested value cannot be converted to integer, violates
1224   /// subtarget's specifications, or does not meet number of waves per execution
1225   /// unit requirement.
1226   unsigned getMaxNumVGPRs(const MachineFunction &MF) const;
1227 
1228   void getPostRAMutations(
1229       std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations)
1230       const override;
1231 
1232   bool isWave32() const {
1233     return getWavefrontSize() == 32;
1234   }
1235 
1236   bool isWave64() const {
1237     return getWavefrontSize() == 64;
1238   }
1239 
1240   const TargetRegisterClass *getBoolRC() const {
1241     return getRegisterInfo()->getBoolRC();
1242   }
1243 
1244   /// \returns Maximum number of work groups per compute unit supported by the
1245   /// subtarget and limited by given \p FlatWorkGroupSize.
1246   unsigned getMaxWorkGroupsPerCU(unsigned FlatWorkGroupSize) const override {
1247     return AMDGPU::IsaInfo::getMaxWorkGroupsPerCU(this, FlatWorkGroupSize);
1248   }
1249 
1250   /// \returns Minimum flat work group size supported by the subtarget.
1251   unsigned getMinFlatWorkGroupSize() const override {
1252     return AMDGPU::IsaInfo::getMinFlatWorkGroupSize(this);
1253   }
1254 
1255   /// \returns Maximum flat work group size supported by the subtarget.
1256   unsigned getMaxFlatWorkGroupSize() const override {
1257     return AMDGPU::IsaInfo::getMaxFlatWorkGroupSize(this);
1258   }
1259 
1260   /// \returns Number of waves per execution unit required to support the given
1261   /// \p FlatWorkGroupSize.
1262   unsigned
1263   getWavesPerEUForWorkGroup(unsigned FlatWorkGroupSize) const override {
1264     return AMDGPU::IsaInfo::getWavesPerEUForWorkGroup(this, FlatWorkGroupSize);
1265   }
1266 
1267   /// \returns Minimum number of waves per execution unit supported by the
1268   /// subtarget.
1269   unsigned getMinWavesPerEU() const override {
1270     return AMDGPU::IsaInfo::getMinWavesPerEU(this);
1271   }
1272 
1273   void adjustSchedDependency(SUnit *Def, int DefOpIdx, SUnit *Use, int UseOpIdx,
1274                              SDep &Dep) const override;
1275 };
1276 
1277 class R600Subtarget final : public R600GenSubtargetInfo,
1278                             public AMDGPUSubtarget {
1279 private:
1280   R600InstrInfo InstrInfo;
1281   R600FrameLowering FrameLowering;
1282   bool FMA;
1283   bool CaymanISA;
1284   bool CFALUBug;
1285   bool HasVertexCache;
1286   bool R600ALUInst;
1287   bool FP64;
1288   short TexVTXClauseSize;
1289   Generation Gen;
1290   R600TargetLowering TLInfo;
1291   InstrItineraryData InstrItins;
1292   SelectionDAGTargetInfo TSInfo;
1293 
1294 public:
1295   R600Subtarget(const Triple &TT, StringRef CPU, StringRef FS,
1296                 const TargetMachine &TM);
1297 
1298   const R600InstrInfo *getInstrInfo() const override { return &InstrInfo; }
1299 
1300   const R600FrameLowering *getFrameLowering() const override {
1301     return &FrameLowering;
1302   }
1303 
1304   const R600TargetLowering *getTargetLowering() const override {
1305     return &TLInfo;
1306   }
1307 
1308   const R600RegisterInfo *getRegisterInfo() const override {
1309     return &InstrInfo.getRegisterInfo();
1310   }
1311 
1312   const InstrItineraryData *getInstrItineraryData() const override {
1313     return &InstrItins;
1314   }
1315 
1316   // Nothing implemented, just prevent crashes on use.
1317   const SelectionDAGTargetInfo *getSelectionDAGInfo() const override {
1318     return &TSInfo;
1319   }
1320 
1321   void ParseSubtargetFeatures(StringRef CPU, StringRef TuneCPU, StringRef FS);
1322 
1323   Generation getGeneration() const {
1324     return Gen;
1325   }
1326 
1327   Align getStackAlignment() const { return Align(4); }
1328 
1329   R600Subtarget &initializeSubtargetDependencies(const Triple &TT,
1330                                                  StringRef GPU, StringRef FS);
1331 
1332   bool hasBFE() const {
1333     return (getGeneration() >= EVERGREEN);
1334   }
1335 
1336   bool hasBFI() const {
1337     return (getGeneration() >= EVERGREEN);
1338   }
1339 
1340   bool hasBCNT(unsigned Size) const {
1341     if (Size == 32)
1342       return (getGeneration() >= EVERGREEN);
1343 
1344     return false;
1345   }
1346 
1347   bool hasBORROW() const {
1348     return (getGeneration() >= EVERGREEN);
1349   }
1350 
1351   bool hasCARRY() const {
1352     return (getGeneration() >= EVERGREEN);
1353   }
1354 
1355   bool hasCaymanISA() const {
1356     return CaymanISA;
1357   }
1358 
1359   bool hasFFBL() const {
1360     return (getGeneration() >= EVERGREEN);
1361   }
1362 
1363   bool hasFFBH() const {
1364     return (getGeneration() >= EVERGREEN);
1365   }
1366 
1367   bool hasFMA() const { return FMA; }
1368 
1369   bool hasCFAluBug() const { return CFALUBug; }
1370 
1371   bool hasVertexCache() const { return HasVertexCache; }
1372 
1373   short getTexVTXClauseSize() const { return TexVTXClauseSize; }
1374 
1375   bool enableMachineScheduler() const override {
1376     return true;
1377   }
1378 
1379   bool enableSubRegLiveness() const override {
1380     return true;
1381   }
1382 
1383   /// \returns Maximum number of work groups per compute unit supported by the
1384   /// subtarget and limited by given \p FlatWorkGroupSize.
1385   unsigned getMaxWorkGroupsPerCU(unsigned FlatWorkGroupSize) const override {
1386     return AMDGPU::IsaInfo::getMaxWorkGroupsPerCU(this, FlatWorkGroupSize);
1387   }
1388 
1389   /// \returns Minimum flat work group size supported by the subtarget.
1390   unsigned getMinFlatWorkGroupSize() const override {
1391     return AMDGPU::IsaInfo::getMinFlatWorkGroupSize(this);
1392   }
1393 
1394   /// \returns Maximum flat work group size supported by the subtarget.
1395   unsigned getMaxFlatWorkGroupSize() const override {
1396     return AMDGPU::IsaInfo::getMaxFlatWorkGroupSize(this);
1397   }
1398 
1399   /// \returns Number of waves per execution unit required to support the given
1400   /// \p FlatWorkGroupSize.
1401   unsigned
1402   getWavesPerEUForWorkGroup(unsigned FlatWorkGroupSize) const override {
1403     return AMDGPU::IsaInfo::getWavesPerEUForWorkGroup(this, FlatWorkGroupSize);
1404   }
1405 
1406   /// \returns Minimum number of waves per execution unit supported by the
1407   /// subtarget.
1408   unsigned getMinWavesPerEU() const override {
1409     return AMDGPU::IsaInfo::getMinWavesPerEU(this);
1410   }
1411 };
1412 
1413 } // end namespace llvm
1414 
1415 #endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUSUBTARGET_H
1416