1 //=====-- AMDGPUSubtarget.h - Define Subtarget for AMDGPU ------*- C++ -*-====//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //==-----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// AMDGPU specific subclass of TargetSubtarget.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUSUBTARGET_H
15 #define LLVM_LIB_TARGET_AMDGPU_AMDGPUSUBTARGET_H
16 
17 #include "AMDGPU.h"
18 #include "AMDGPUCallLowering.h"
19 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
20 #include "R600FrameLowering.h"
21 #include "R600ISelLowering.h"
22 #include "R600InstrInfo.h"
23 #include "SIFrameLowering.h"
24 #include "SIISelLowering.h"
25 #include "SIInstrInfo.h"
26 #include "Utils/AMDGPUBaseInfo.h"
27 #include "llvm/ADT/Triple.h"
28 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
29 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
30 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
31 #include "llvm/CodeGen/MachineFunction.h"
32 #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
33 #include "llvm/MC/MCInstrItineraries.h"
34 #include "llvm/Support/MathExtras.h"
35 #include <cassert>
36 #include <cstdint>
37 #include <memory>
38 #include <utility>
39 
40 #define GET_SUBTARGETINFO_HEADER
41 #include "AMDGPUGenSubtargetInfo.inc"
42 #define GET_SUBTARGETINFO_HEADER
43 #include "R600GenSubtargetInfo.inc"
44 
45 namespace llvm {
46 
47 class StringRef;
48 
49 class AMDGPUSubtarget {
50 public:
51   enum Generation {
52     R600 = 0,
53     R700 = 1,
54     EVERGREEN = 2,
55     NORTHERN_ISLANDS = 3,
56     SOUTHERN_ISLANDS = 4,
57     SEA_ISLANDS = 5,
58     VOLCANIC_ISLANDS = 6,
59     GFX9 = 7,
60     GFX10 = 8
61   };
62 
63 private:
64   Triple TargetTriple;
65 
66 protected:
67   bool Has16BitInsts;
68   bool HasMadMixInsts;
69   bool FPExceptions;
70   bool HasSDWA;
71   bool HasVOP3PInsts;
72   bool HasMulI24;
73   bool HasMulU24;
74   bool HasInv2PiInlineImm;
75   bool HasFminFmaxLegacy;
76   bool EnablePromoteAlloca;
77   bool HasTrigReducedRange;
78   unsigned MaxWavesPerEU;
79   int LocalMemorySize;
80   unsigned WavefrontSize;
81 
82 public:
83   AMDGPUSubtarget(const Triple &TT);
84 
85   static const AMDGPUSubtarget &get(const MachineFunction &MF);
86   static const AMDGPUSubtarget &get(const TargetMachine &TM,
87                                     const Function &F);
88 
89   /// \returns Default range flat work group size for a calling convention.
90   std::pair<unsigned, unsigned> getDefaultFlatWorkGroupSize(CallingConv::ID CC) const;
91 
92   /// \returns Subtarget's default pair of minimum/maximum flat work group sizes
93   /// for function \p F, or minimum/maximum flat work group sizes explicitly
94   /// requested using "amdgpu-flat-work-group-size" attribute attached to
95   /// function \p F.
96   ///
97   /// \returns Subtarget's default values if explicitly requested values cannot
98   /// be converted to integer, or violate subtarget's specifications.
99   std::pair<unsigned, unsigned> getFlatWorkGroupSizes(const Function &F) const;
100 
101   /// \returns Subtarget's default pair of minimum/maximum number of waves per
102   /// execution unit for function \p F, or minimum/maximum number of waves per
103   /// execution unit explicitly requested using "amdgpu-waves-per-eu" attribute
104   /// attached to function \p F.
105   ///
106   /// \returns Subtarget's default values if explicitly requested values cannot
107   /// be converted to integer, violate subtarget's specifications, or are not
108   /// compatible with minimum/maximum number of waves limited by flat work group
109   /// size, register usage, and/or lds usage.
110   std::pair<unsigned, unsigned> getWavesPerEU(const Function &F) const;
111 
112   /// Return the amount of LDS that can be used that will not restrict the
113   /// occupancy lower than WaveCount.
114   unsigned getMaxLocalMemSizeWithWaveCount(unsigned WaveCount,
115                                            const Function &) const;
116 
117   /// Inverse of getMaxLocalMemWithWaveCount. Return the maximum wavecount if
118   /// the given LDS memory size is the only constraint.
119   unsigned getOccupancyWithLocalMemSize(uint32_t Bytes, const Function &) const;
120 
121   unsigned getOccupancyWithLocalMemSize(const MachineFunction &MF) const;
122 
123   bool isAmdHsaOS() const {
124     return TargetTriple.getOS() == Triple::AMDHSA;
125   }
126 
127   bool isAmdPalOS() const {
128     return TargetTriple.getOS() == Triple::AMDPAL;
129   }
130 
131   bool isMesa3DOS() const {
132     return TargetTriple.getOS() == Triple::Mesa3D;
133   }
134 
135   bool isMesaKernel(const Function &F) const {
136     return isMesa3DOS() && !AMDGPU::isShader(F.getCallingConv());
137   }
138 
139   bool isAmdHsaOrMesa(const Function &F) const {
140     return isAmdHsaOS() || isMesaKernel(F);
141   }
142 
143   bool has16BitInsts() const {
144     return Has16BitInsts;
145   }
146 
147   bool hasMadMixInsts() const {
148     return HasMadMixInsts;
149   }
150 
151   bool hasFPExceptions() const {
152     return FPExceptions;
153   }
154 
155   bool hasSDWA() const {
156     return HasSDWA;
157   }
158 
159   bool hasVOP3PInsts() const {
160     return HasVOP3PInsts;
161   }
162 
163   bool hasMulI24() const {
164     return HasMulI24;
165   }
166 
167   bool hasMulU24() const {
168     return HasMulU24;
169   }
170 
171   bool hasInv2PiInlineImm() const {
172     return HasInv2PiInlineImm;
173   }
174 
175   bool hasFminFmaxLegacy() const {
176     return HasFminFmaxLegacy;
177   }
178 
179   bool hasTrigReducedRange() const {
180     return HasTrigReducedRange;
181   }
182 
183   bool isPromoteAllocaEnabled() const {
184     return EnablePromoteAlloca;
185   }
186 
187   unsigned getWavefrontSize() const {
188     return WavefrontSize;
189   }
190 
191   int getLocalMemorySize() const {
192     return LocalMemorySize;
193   }
194 
195   Align getAlignmentForImplicitArgPtr() const {
196     return isAmdHsaOS() ? Align(8) : Align(4);
197   }
198 
199   /// Returns the offset in bytes from the start of the input buffer
200   ///        of the first explicit kernel argument.
201   unsigned getExplicitKernelArgOffset(const Function &F) const {
202     return isAmdHsaOrMesa(F) ? 0 : 36;
203   }
204 
205   /// \returns Maximum number of work groups per compute unit supported by the
206   /// subtarget and limited by given \p FlatWorkGroupSize.
207   virtual unsigned getMaxWorkGroupsPerCU(unsigned FlatWorkGroupSize) const = 0;
208 
209   /// \returns Minimum flat work group size supported by the subtarget.
210   virtual unsigned getMinFlatWorkGroupSize() const = 0;
211 
212   /// \returns Maximum flat work group size supported by the subtarget.
213   virtual unsigned getMaxFlatWorkGroupSize() const = 0;
214 
215   /// \returns Number of waves per execution unit required to support the given
216   /// \p FlatWorkGroupSize.
217   virtual unsigned
218   getWavesPerEUForWorkGroup(unsigned FlatWorkGroupSize) const = 0;
219 
220   /// \returns Minimum number of waves per execution unit supported by the
221   /// subtarget.
222   virtual unsigned getMinWavesPerEU() const = 0;
223 
224   /// \returns Maximum number of waves per execution unit supported by the
225   /// subtarget without any kind of limitation.
226   unsigned getMaxWavesPerEU() const { return MaxWavesPerEU; }
227 
228   /// Creates value range metadata on an workitemid.* inrinsic call or load.
229   bool makeLIDRangeMetadata(Instruction *I) const;
230 
231   /// \returns Number of bytes of arguments that are passed to a shader or
232   /// kernel in addition to the explicit ones declared for the function.
233   unsigned getImplicitArgNumBytes(const Function &F) const {
234     if (isMesaKernel(F))
235       return 16;
236     return AMDGPU::getIntegerAttribute(F, "amdgpu-implicitarg-num-bytes", 0);
237   }
238   uint64_t getExplicitKernArgSize(const Function &F, Align &MaxAlign) const;
239   unsigned getKernArgSegmentSize(const Function &F, Align &MaxAlign) const;
240 
241   /// \returns Corresponsing DWARF register number mapping flavour for the
242   /// \p WavefrontSize.
243   AMDGPUDwarfFlavour getAMDGPUDwarfFlavour() const {
244     return WavefrontSize == 32 ? AMDGPUDwarfFlavour::Wave32
245                                : AMDGPUDwarfFlavour::Wave64;
246   }
247 
248   virtual ~AMDGPUSubtarget() {}
249 };
250 
251 class GCNSubtarget : public AMDGPUGenSubtargetInfo,
252                      public AMDGPUSubtarget {
253 
254   using AMDGPUSubtarget::getMaxWavesPerEU;
255 
256 public:
257   enum TrapHandlerAbi {
258     TrapHandlerAbiNone = 0,
259     TrapHandlerAbiHsa = 1
260   };
261 
262   enum TrapID {
263     TrapIDHardwareReserved = 0,
264     TrapIDHSADebugTrap = 1,
265     TrapIDLLVMTrap = 2,
266     TrapIDLLVMDebugTrap = 3,
267     TrapIDDebugBreakpoint = 7,
268     TrapIDDebugReserved8 = 8,
269     TrapIDDebugReservedFE = 0xfe,
270     TrapIDDebugReservedFF = 0xff
271   };
272 
273   enum TrapRegValues {
274     LLVMTrapHandlerRegValue = 1
275   };
276 
277 private:
278   /// GlobalISel related APIs.
279   std::unique_ptr<AMDGPUCallLowering> CallLoweringInfo;
280   std::unique_ptr<InstructionSelector> InstSelector;
281   std::unique_ptr<LegalizerInfo> Legalizer;
282   std::unique_ptr<RegisterBankInfo> RegBankInfo;
283 
284 protected:
285   // Basic subtarget description.
286   Triple TargetTriple;
287   unsigned Gen;
288   InstrItineraryData InstrItins;
289   int LDSBankCount;
290   unsigned MaxPrivateElementSize;
291 
292   // Possibly statically set by tablegen, but may want to be overridden.
293   bool FastFMAF32;
294   bool FastDenormalF32;
295   bool HalfRate64Ops;
296 
297   // Dynamially set bits that enable features.
298   bool FlatForGlobal;
299   bool AutoWaitcntBeforeBarrier;
300   bool CodeObjectV3;
301   bool UnalignedScratchAccess;
302   bool UnalignedBufferAccess;
303   bool HasApertureRegs;
304   bool EnableXNACK;
305   bool DoesNotSupportXNACK;
306   bool EnableCuMode;
307   bool TrapHandler;
308 
309   // Used as options.
310   bool EnableLoadStoreOpt;
311   bool EnableUnsafeDSOffsetFolding;
312   bool EnableSIScheduler;
313   bool EnableDS128;
314   bool EnablePRTStrictNull;
315   bool DumpCode;
316 
317   // Subtarget statically properties set by tablegen
318   bool FP64;
319   bool FMA;
320   bool MIMG_R128;
321   bool IsGCN;
322   bool GCN3Encoding;
323   bool CIInsts;
324   bool GFX8Insts;
325   bool GFX9Insts;
326   bool GFX10Insts;
327   bool GFX7GFX8GFX9Insts;
328   bool SGPRInitBug;
329   bool HasSMemRealTime;
330   bool HasIntClamp;
331   bool HasFmaMixInsts;
332   bool HasMovrel;
333   bool HasVGPRIndexMode;
334   bool HasScalarStores;
335   bool HasScalarAtomics;
336   bool HasSDWAOmod;
337   bool HasSDWAScalar;
338   bool HasSDWASdst;
339   bool HasSDWAMac;
340   bool HasSDWAOutModsVOPC;
341   bool HasDPP;
342   bool HasDPP8;
343   bool HasR128A16;
344   bool HasGFX10A16;
345   bool HasNSAEncoding;
346   bool HasDLInsts;
347   bool HasDot1Insts;
348   bool HasDot2Insts;
349   bool HasDot3Insts;
350   bool HasDot4Insts;
351   bool HasDot5Insts;
352   bool HasDot6Insts;
353   bool HasMAIInsts;
354   bool HasPkFmacF16Inst;
355   bool HasAtomicFaddInsts;
356   bool EnableSRAMECC;
357   bool DoesNotSupportSRAMECC;
358   bool HasNoSdstCMPX;
359   bool HasVscnt;
360   bool HasRegisterBanking;
361   bool HasVOP3Literal;
362   bool HasNoDataDepHazard;
363   bool FlatAddressSpace;
364   bool FlatInstOffsets;
365   bool FlatGlobalInsts;
366   bool FlatScratchInsts;
367   bool ScalarFlatScratchInsts;
368   bool AddNoCarryInsts;
369   bool HasUnpackedD16VMem;
370   bool R600ALUInst;
371   bool CaymanISA;
372   bool CFALUBug;
373   bool LDSMisalignedBug;
374   bool HasMFMAInlineLiteralBug;
375   bool HasVertexCache;
376   short TexVTXClauseSize;
377   bool ScalarizeGlobal;
378 
379   bool HasVcmpxPermlaneHazard;
380   bool HasVMEMtoScalarWriteHazard;
381   bool HasSMEMtoVectorWriteHazard;
382   bool HasInstFwdPrefetchBug;
383   bool HasVcmpxExecWARHazard;
384   bool HasLdsBranchVmemWARHazard;
385   bool HasNSAtoVMEMBug;
386   bool HasOffset3fBug;
387   bool HasFlatSegmentOffsetBug;
388 
389   // Dummy feature to use for assembler in tablegen.
390   bool FeatureDisable;
391 
392   SelectionDAGTargetInfo TSInfo;
393 private:
394   SIInstrInfo InstrInfo;
395   SITargetLowering TLInfo;
396   SIFrameLowering FrameLowering;
397 
398   // See COMPUTE_TMPRING_SIZE.WAVESIZE, 13-bit field in units of 256-dword.
399   static const unsigned MaxWaveScratchSize = (256 * 4) * ((1 << 13) - 1);
400 
401 public:
402   GCNSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
403                const GCNTargetMachine &TM);
404   ~GCNSubtarget() override;
405 
406   GCNSubtarget &initializeSubtargetDependencies(const Triple &TT,
407                                                    StringRef GPU, StringRef FS);
408 
409   const SIInstrInfo *getInstrInfo() const override {
410     return &InstrInfo;
411   }
412 
413   const SIFrameLowering *getFrameLowering() const override {
414     return &FrameLowering;
415   }
416 
417   const SITargetLowering *getTargetLowering() const override {
418     return &TLInfo;
419   }
420 
421   const SIRegisterInfo *getRegisterInfo() const override {
422     return &InstrInfo.getRegisterInfo();
423   }
424 
425   const CallLowering *getCallLowering() const override {
426     return CallLoweringInfo.get();
427   }
428 
429   InstructionSelector *getInstructionSelector() const override {
430     return InstSelector.get();
431   }
432 
433   const LegalizerInfo *getLegalizerInfo() const override {
434     return Legalizer.get();
435   }
436 
437   const RegisterBankInfo *getRegBankInfo() const override {
438     return RegBankInfo.get();
439   }
440 
441   // Nothing implemented, just prevent crashes on use.
442   const SelectionDAGTargetInfo *getSelectionDAGInfo() const override {
443     return &TSInfo;
444   }
445 
446   const InstrItineraryData *getInstrItineraryData() const override {
447     return &InstrItins;
448   }
449 
450   void ParseSubtargetFeatures(StringRef CPU, StringRef FS);
451 
452   Generation getGeneration() const {
453     return (Generation)Gen;
454   }
455 
456   unsigned getWavefrontSizeLog2() const {
457     return Log2_32(WavefrontSize);
458   }
459 
460   /// Return the number of high bits known to be zero fror a frame index.
461   unsigned getKnownHighZeroBitsForFrameIndex() const {
462     return countLeadingZeros(MaxWaveScratchSize) + getWavefrontSizeLog2();
463   }
464 
465   int getLDSBankCount() const {
466     return LDSBankCount;
467   }
468 
469   unsigned getMaxPrivateElementSize() const {
470     return MaxPrivateElementSize;
471   }
472 
473   unsigned getConstantBusLimit(unsigned Opcode) const;
474 
475   bool hasIntClamp() const {
476     return HasIntClamp;
477   }
478 
479   bool hasFP64() const {
480     return FP64;
481   }
482 
483   bool hasMIMG_R128() const {
484     return MIMG_R128;
485   }
486 
487   bool hasHWFP64() const {
488     return FP64;
489   }
490 
491   bool hasFastFMAF32() const {
492     return FastFMAF32;
493   }
494 
495   bool hasHalfRate64Ops() const {
496     return HalfRate64Ops;
497   }
498 
499   bool hasAddr64() const {
500     return (getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS);
501   }
502 
503   // Return true if the target only has the reverse operand versions of VALU
504   // shift instructions (e.g. v_lshrrev_b32, and no v_lshr_b32).
505   bool hasOnlyRevVALUShifts() const {
506     return getGeneration() >= VOLCANIC_ISLANDS;
507   }
508 
509   bool hasFractBug() const {
510     return getGeneration() == SOUTHERN_ISLANDS;
511   }
512 
513   bool hasBFE() const {
514     return true;
515   }
516 
517   bool hasBFI() const {
518     return true;
519   }
520 
521   bool hasBFM() const {
522     return hasBFE();
523   }
524 
525   bool hasBCNT(unsigned Size) const {
526     return true;
527   }
528 
529   bool hasFFBL() const {
530     return true;
531   }
532 
533   bool hasFFBH() const {
534     return true;
535   }
536 
537   bool hasMed3_16() const {
538     return getGeneration() >= AMDGPUSubtarget::GFX9;
539   }
540 
541   bool hasMin3Max3_16() const {
542     return getGeneration() >= AMDGPUSubtarget::GFX9;
543   }
544 
545   bool hasFmaMixInsts() const {
546     return HasFmaMixInsts;
547   }
548 
549   bool hasCARRY() const {
550     return true;
551   }
552 
553   bool hasFMA() const {
554     return FMA;
555   }
556 
557   bool hasSwap() const {
558     return GFX9Insts;
559   }
560 
561   bool hasScalarPackInsts() const {
562     return GFX9Insts;
563   }
564 
565   bool hasScalarMulHiInsts() const {
566     return GFX9Insts;
567   }
568 
569   TrapHandlerAbi getTrapHandlerAbi() const {
570     return isAmdHsaOS() ? TrapHandlerAbiHsa : TrapHandlerAbiNone;
571   }
572 
573   /// True if the offset field of DS instructions works as expected. On SI, the
574   /// offset uses a 16-bit adder and does not always wrap properly.
575   bool hasUsableDSOffset() const {
576     return getGeneration() >= SEA_ISLANDS;
577   }
578 
579   bool unsafeDSOffsetFoldingEnabled() const {
580     return EnableUnsafeDSOffsetFolding;
581   }
582 
583   /// Condition output from div_scale is usable.
584   bool hasUsableDivScaleConditionOutput() const {
585     return getGeneration() != SOUTHERN_ISLANDS;
586   }
587 
588   /// Extra wait hazard is needed in some cases before
589   /// s_cbranch_vccnz/s_cbranch_vccz.
590   bool hasReadVCCZBug() const {
591     return getGeneration() <= SEA_ISLANDS;
592   }
593 
594   /// Writes to VCC_LO/VCC_HI update the VCCZ flag.
595   bool partialVCCWritesUpdateVCCZ() const {
596     return getGeneration() >= GFX10;
597   }
598 
599   /// A read of an SGPR by SMRD instruction requires 4 wait states when the SGPR
600   /// was written by a VALU instruction.
601   bool hasSMRDReadVALUDefHazard() const {
602     return getGeneration() == SOUTHERN_ISLANDS;
603   }
604 
605   /// A read of an SGPR by a VMEM instruction requires 5 wait states when the
606   /// SGPR was written by a VALU Instruction.
607   bool hasVMEMReadSGPRVALUDefHazard() const {
608     return getGeneration() >= VOLCANIC_ISLANDS;
609   }
610 
611   bool hasRFEHazards() const {
612     return getGeneration() >= VOLCANIC_ISLANDS;
613   }
614 
615   /// Number of hazard wait states for s_setreg_b32/s_setreg_imm32_b32.
616   unsigned getSetRegWaitStates() const {
617     return getGeneration() <= SEA_ISLANDS ? 1 : 2;
618   }
619 
620   bool dumpCode() const {
621     return DumpCode;
622   }
623 
624   /// Return the amount of LDS that can be used that will not restrict the
625   /// occupancy lower than WaveCount.
626   unsigned getMaxLocalMemSizeWithWaveCount(unsigned WaveCount,
627                                            const Function &) const;
628 
629   bool supportsMinMaxDenormModes() const {
630     return getGeneration() >= AMDGPUSubtarget::GFX9;
631   }
632 
633   /// \returns If target supports S_DENORM_MODE.
634   bool hasDenormModeInst() const {
635     return getGeneration() >= AMDGPUSubtarget::GFX10;
636   }
637 
638   bool useFlatForGlobal() const {
639     return FlatForGlobal;
640   }
641 
642   /// \returns If target supports ds_read/write_b128 and user enables generation
643   /// of ds_read/write_b128.
644   bool useDS128() const {
645     return CIInsts && EnableDS128;
646   }
647 
648   /// Have v_trunc_f64, v_ceil_f64, v_rndne_f64
649   bool haveRoundOpsF64() const {
650     return CIInsts;
651   }
652 
653   /// \returns If MUBUF instructions always perform range checking, even for
654   /// buffer resources used for private memory access.
655   bool privateMemoryResourceIsRangeChecked() const {
656     return getGeneration() < AMDGPUSubtarget::GFX9;
657   }
658 
659   /// \returns If target requires PRT Struct NULL support (zero result registers
660   /// for sparse texture support).
661   bool usePRTStrictNull() const {
662     return EnablePRTStrictNull;
663   }
664 
665   bool hasAutoWaitcntBeforeBarrier() const {
666     return AutoWaitcntBeforeBarrier;
667   }
668 
669   bool hasCodeObjectV3() const {
670     // FIXME: Need to add code object v3 support for mesa and pal.
671     return isAmdHsaOS() ? CodeObjectV3 : false;
672   }
673 
674   bool hasUnalignedBufferAccess() const {
675     return UnalignedBufferAccess;
676   }
677 
678   bool hasUnalignedScratchAccess() const {
679     return UnalignedScratchAccess;
680   }
681 
682   bool hasApertureRegs() const {
683     return HasApertureRegs;
684   }
685 
686   bool isTrapHandlerEnabled() const {
687     return TrapHandler;
688   }
689 
690   bool isXNACKEnabled() const {
691     return EnableXNACK;
692   }
693 
694   bool isCuModeEnabled() const {
695     return EnableCuMode;
696   }
697 
698   bool hasFlatAddressSpace() const {
699     return FlatAddressSpace;
700   }
701 
702   bool hasFlatScrRegister() const {
703     return hasFlatAddressSpace();
704   }
705 
706   bool hasFlatInstOffsets() const {
707     return FlatInstOffsets;
708   }
709 
710   bool hasFlatGlobalInsts() const {
711     return FlatGlobalInsts;
712   }
713 
714   bool hasFlatScratchInsts() const {
715     return FlatScratchInsts;
716   }
717 
718   bool hasScalarFlatScratchInsts() const {
719     return ScalarFlatScratchInsts;
720   }
721 
722   bool hasMultiDwordFlatScratchAddressing() const {
723     return getGeneration() >= GFX9;
724   }
725 
726   bool hasFlatSegmentOffsetBug() const {
727     return HasFlatSegmentOffsetBug;
728   }
729 
730   bool hasFlatLgkmVMemCountInOrder() const {
731     return getGeneration() > GFX9;
732   }
733 
734   bool hasD16LoadStore() const {
735     return getGeneration() >= GFX9;
736   }
737 
738   bool d16PreservesUnusedBits() const {
739     return hasD16LoadStore() && !isSRAMECCEnabled();
740   }
741 
742   bool hasD16Images() const {
743     return getGeneration() >= VOLCANIC_ISLANDS;
744   }
745 
746   /// Return if most LDS instructions have an m0 use that require m0 to be
747   /// iniitalized.
748   bool ldsRequiresM0Init() const {
749     return getGeneration() < GFX9;
750   }
751 
752   // True if the hardware rewinds and replays GWS operations if a wave is
753   // preempted.
754   //
755   // If this is false, a GWS operation requires testing if a nack set the
756   // MEM_VIOL bit, and repeating if so.
757   bool hasGWSAutoReplay() const {
758     return getGeneration() >= GFX9;
759   }
760 
761   /// \returns if target has ds_gws_sema_release_all instruction.
762   bool hasGWSSemaReleaseAll() const {
763     return CIInsts;
764   }
765 
766   bool hasAddNoCarry() const {
767     return AddNoCarryInsts;
768   }
769 
770   bool hasUnpackedD16VMem() const {
771     return HasUnpackedD16VMem;
772   }
773 
774   // Covers VS/PS/CS graphics shaders
775   bool isMesaGfxShader(const Function &F) const {
776     return isMesa3DOS() && AMDGPU::isShader(F.getCallingConv());
777   }
778 
779   bool hasMad64_32() const {
780     return getGeneration() >= SEA_ISLANDS;
781   }
782 
783   bool hasSDWAOmod() const {
784     return HasSDWAOmod;
785   }
786 
787   bool hasSDWAScalar() const {
788     return HasSDWAScalar;
789   }
790 
791   bool hasSDWASdst() const {
792     return HasSDWASdst;
793   }
794 
795   bool hasSDWAMac() const {
796     return HasSDWAMac;
797   }
798 
799   bool hasSDWAOutModsVOPC() const {
800     return HasSDWAOutModsVOPC;
801   }
802 
803   bool hasDLInsts() const {
804     return HasDLInsts;
805   }
806 
807   bool hasDot1Insts() const {
808     return HasDot1Insts;
809   }
810 
811   bool hasDot2Insts() const {
812     return HasDot2Insts;
813   }
814 
815   bool hasDot3Insts() const {
816     return HasDot3Insts;
817   }
818 
819   bool hasDot4Insts() const {
820     return HasDot4Insts;
821   }
822 
823   bool hasDot5Insts() const {
824     return HasDot5Insts;
825   }
826 
827   bool hasDot6Insts() const {
828     return HasDot6Insts;
829   }
830 
831   bool hasMAIInsts() const {
832     return HasMAIInsts;
833   }
834 
835   bool hasPkFmacF16Inst() const {
836     return HasPkFmacF16Inst;
837   }
838 
839   bool hasAtomicFaddInsts() const {
840     return HasAtomicFaddInsts;
841   }
842 
843   bool isSRAMECCEnabled() const {
844     return EnableSRAMECC;
845   }
846 
847   bool hasNoSdstCMPX() const {
848     return HasNoSdstCMPX;
849   }
850 
851   bool hasVscnt() const {
852     return HasVscnt;
853   }
854 
855   bool hasRegisterBanking() const {
856     return HasRegisterBanking;
857   }
858 
859   bool hasVOP3Literal() const {
860     return HasVOP3Literal;
861   }
862 
863   bool hasNoDataDepHazard() const {
864     return HasNoDataDepHazard;
865   }
866 
867   bool vmemWriteNeedsExpWaitcnt() const {
868     return getGeneration() < SEA_ISLANDS;
869   }
870 
871   // Scratch is allocated in 256 dword per wave blocks for the entire
872   // wavefront. When viewed from the perspecive of an arbitrary workitem, this
873   // is 4-byte aligned.
874   //
875   // Only 4-byte alignment is really needed to access anything. Transformations
876   // on the pointer value itself may rely on the alignment / known low bits of
877   // the pointer. Set this to something above the minimum to avoid needing
878   // dynamic realignment in common cases.
879   Align getStackAlignment() const { return Align(16); }
880 
881   bool enableMachineScheduler() const override {
882     return true;
883   }
884 
885   bool enableSubRegLiveness() const override {
886     return true;
887   }
888 
889   void setScalarizeGlobalBehavior(bool b) { ScalarizeGlobal = b; }
890   bool getScalarizeGlobalBehavior() const { return ScalarizeGlobal; }
891 
892   // static wrappers
893   static bool hasHalfRate64Ops(const TargetSubtargetInfo &STI);
894 
895   // XXX - Why is this here if it isn't in the default pass set?
896   bool enableEarlyIfConversion() const override {
897     return true;
898   }
899 
900   void overrideSchedPolicy(MachineSchedPolicy &Policy,
901                            unsigned NumRegionInstrs) const override;
902 
903   unsigned getMaxNumUserSGPRs() const {
904     return 16;
905   }
906 
907   bool hasSMemRealTime() const {
908     return HasSMemRealTime;
909   }
910 
911   bool hasMovrel() const {
912     return HasMovrel;
913   }
914 
915   bool hasVGPRIndexMode() const {
916     return HasVGPRIndexMode;
917   }
918 
919   bool useVGPRIndexMode() const;
920 
921   bool hasScalarCompareEq64() const {
922     return getGeneration() >= VOLCANIC_ISLANDS;
923   }
924 
925   bool hasScalarStores() const {
926     return HasScalarStores;
927   }
928 
929   bool hasScalarAtomics() const {
930     return HasScalarAtomics;
931   }
932 
933   bool hasLDSFPAtomics() const {
934     return GFX8Insts;
935   }
936 
937   bool hasDPP() const {
938     return HasDPP;
939   }
940 
941   bool hasDPPBroadcasts() const {
942     return HasDPP && getGeneration() < GFX10;
943   }
944 
945   bool hasDPPWavefrontShifts() const {
946     return HasDPP && getGeneration() < GFX10;
947   }
948 
949   bool hasDPP8() const {
950     return HasDPP8;
951   }
952 
953   bool hasR128A16() const {
954     return HasR128A16;
955   }
956 
957   bool hasGFX10A16() const {
958     return HasGFX10A16;
959   }
960 
961   bool hasOffset3fBug() const {
962     return HasOffset3fBug;
963   }
964 
965   bool hasNSAEncoding() const {
966     return HasNSAEncoding;
967   }
968 
969   bool hasMadF16() const;
970 
971   bool enableSIScheduler() const {
972     return EnableSIScheduler;
973   }
974 
975   bool loadStoreOptEnabled() const {
976     return EnableLoadStoreOpt;
977   }
978 
979   bool hasSGPRInitBug() const {
980     return SGPRInitBug;
981   }
982 
983   bool hasMFMAInlineLiteralBug() const {
984     return HasMFMAInlineLiteralBug;
985   }
986 
987   bool has12DWordStoreHazard() const {
988     return getGeneration() != AMDGPUSubtarget::SOUTHERN_ISLANDS;
989   }
990 
991   // \returns true if the subtarget supports DWORDX3 load/store instructions.
992   bool hasDwordx3LoadStores() const {
993     return CIInsts;
994   }
995 
996   bool hasSMovFedHazard() const {
997     return getGeneration() == AMDGPUSubtarget::GFX9;
998   }
999 
1000   bool hasReadM0MovRelInterpHazard() const {
1001     return getGeneration() == AMDGPUSubtarget::GFX9;
1002   }
1003 
1004   bool hasReadM0SendMsgHazard() const {
1005     return getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
1006            getGeneration() <= AMDGPUSubtarget::GFX9;
1007   }
1008 
1009   bool hasVcmpxPermlaneHazard() const {
1010     return HasVcmpxPermlaneHazard;
1011   }
1012 
1013   bool hasVMEMtoScalarWriteHazard() const {
1014     return HasVMEMtoScalarWriteHazard;
1015   }
1016 
1017   bool hasSMEMtoVectorWriteHazard() const {
1018     return HasSMEMtoVectorWriteHazard;
1019   }
1020 
1021   bool hasLDSMisalignedBug() const {
1022     return LDSMisalignedBug && !EnableCuMode;
1023   }
1024 
1025   bool hasInstFwdPrefetchBug() const {
1026     return HasInstFwdPrefetchBug;
1027   }
1028 
1029   bool hasVcmpxExecWARHazard() const {
1030     return HasVcmpxExecWARHazard;
1031   }
1032 
1033   bool hasLdsBranchVmemWARHazard() const {
1034     return HasLdsBranchVmemWARHazard;
1035   }
1036 
1037   bool hasNSAtoVMEMBug() const {
1038     return HasNSAtoVMEMBug;
1039   }
1040 
1041   /// Return the maximum number of waves per SIMD for kernels using \p SGPRs
1042   /// SGPRs
1043   unsigned getOccupancyWithNumSGPRs(unsigned SGPRs) const;
1044 
1045   /// Return the maximum number of waves per SIMD for kernels using \p VGPRs
1046   /// VGPRs
1047   unsigned getOccupancyWithNumVGPRs(unsigned VGPRs) const;
1048 
1049   /// Return occupancy for the given function. Used LDS and a number of
1050   /// registers if provided.
1051   /// Note, occupancy can be affected by the scratch allocation as well, but
1052   /// we do not have enough information to compute it.
1053   unsigned computeOccupancy(const MachineFunction &MF, unsigned LDSSize = 0,
1054                             unsigned NumSGPRs = 0, unsigned NumVGPRs = 0) const;
1055 
1056   /// \returns true if the flat_scratch register should be initialized with the
1057   /// pointer to the wave's scratch memory rather than a size and offset.
1058   bool flatScratchIsPointer() const {
1059     return getGeneration() >= AMDGPUSubtarget::GFX9;
1060   }
1061 
1062   /// \returns true if the machine has merged shaders in which s0-s7 are
1063   /// reserved by the hardware and user SGPRs start at s8
1064   bool hasMergedShaders() const {
1065     return getGeneration() >= GFX9;
1066   }
1067 
1068   /// \returns SGPR allocation granularity supported by the subtarget.
1069   unsigned getSGPRAllocGranule() const {
1070     return AMDGPU::IsaInfo::getSGPRAllocGranule(this);
1071   }
1072 
1073   /// \returns SGPR encoding granularity supported by the subtarget.
1074   unsigned getSGPREncodingGranule() const {
1075     return AMDGPU::IsaInfo::getSGPREncodingGranule(this);
1076   }
1077 
1078   /// \returns Total number of SGPRs supported by the subtarget.
1079   unsigned getTotalNumSGPRs() const {
1080     return AMDGPU::IsaInfo::getTotalNumSGPRs(this);
1081   }
1082 
1083   /// \returns Addressable number of SGPRs supported by the subtarget.
1084   unsigned getAddressableNumSGPRs() const {
1085     return AMDGPU::IsaInfo::getAddressableNumSGPRs(this);
1086   }
1087 
1088   /// \returns Minimum number of SGPRs that meets the given number of waves per
1089   /// execution unit requirement supported by the subtarget.
1090   unsigned getMinNumSGPRs(unsigned WavesPerEU) const {
1091     return AMDGPU::IsaInfo::getMinNumSGPRs(this, WavesPerEU);
1092   }
1093 
1094   /// \returns Maximum number of SGPRs that meets the given number of waves per
1095   /// execution unit requirement supported by the subtarget.
1096   unsigned getMaxNumSGPRs(unsigned WavesPerEU, bool Addressable) const {
1097     return AMDGPU::IsaInfo::getMaxNumSGPRs(this, WavesPerEU, Addressable);
1098   }
1099 
1100   /// \returns Reserved number of SGPRs for given function \p MF.
1101   unsigned getReservedNumSGPRs(const MachineFunction &MF) const;
1102 
1103   /// \returns Maximum number of SGPRs that meets number of waves per execution
1104   /// unit requirement for function \p MF, or number of SGPRs explicitly
1105   /// requested using "amdgpu-num-sgpr" attribute attached to function \p MF.
1106   ///
1107   /// \returns Value that meets number of waves per execution unit requirement
1108   /// if explicitly requested value cannot be converted to integer, violates
1109   /// subtarget's specifications, or does not meet number of waves per execution
1110   /// unit requirement.
1111   unsigned getMaxNumSGPRs(const MachineFunction &MF) const;
1112 
1113   /// \returns VGPR allocation granularity supported by the subtarget.
1114   unsigned getVGPRAllocGranule() const {
1115     return AMDGPU::IsaInfo::getVGPRAllocGranule(this);
1116   }
1117 
1118   /// \returns VGPR encoding granularity supported by the subtarget.
1119   unsigned getVGPREncodingGranule() const {
1120     return AMDGPU::IsaInfo::getVGPREncodingGranule(this);
1121   }
1122 
1123   /// \returns Total number of VGPRs supported by the subtarget.
1124   unsigned getTotalNumVGPRs() const {
1125     return AMDGPU::IsaInfo::getTotalNumVGPRs(this);
1126   }
1127 
1128   /// \returns Addressable number of VGPRs supported by the subtarget.
1129   unsigned getAddressableNumVGPRs() const {
1130     return AMDGPU::IsaInfo::getAddressableNumVGPRs(this);
1131   }
1132 
1133   /// \returns Minimum number of VGPRs that meets given number of waves per
1134   /// execution unit requirement supported by the subtarget.
1135   unsigned getMinNumVGPRs(unsigned WavesPerEU) const {
1136     return AMDGPU::IsaInfo::getMinNumVGPRs(this, WavesPerEU);
1137   }
1138 
1139   /// \returns Maximum number of VGPRs that meets given number of waves per
1140   /// execution unit requirement supported by the subtarget.
1141   unsigned getMaxNumVGPRs(unsigned WavesPerEU) const {
1142     return AMDGPU::IsaInfo::getMaxNumVGPRs(this, WavesPerEU);
1143   }
1144 
1145   /// \returns Maximum number of VGPRs that meets number of waves per execution
1146   /// unit requirement for function \p MF, or number of VGPRs explicitly
1147   /// requested using "amdgpu-num-vgpr" attribute attached to function \p MF.
1148   ///
1149   /// \returns Value that meets number of waves per execution unit requirement
1150   /// if explicitly requested value cannot be converted to integer, violates
1151   /// subtarget's specifications, or does not meet number of waves per execution
1152   /// unit requirement.
1153   unsigned getMaxNumVGPRs(const MachineFunction &MF) const;
1154 
1155   void getPostRAMutations(
1156       std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations)
1157       const override;
1158 
1159   bool isWave32() const {
1160     return WavefrontSize == 32;
1161   }
1162 
1163   const TargetRegisterClass *getBoolRC() const {
1164     return getRegisterInfo()->getBoolRC();
1165   }
1166 
1167   /// \returns Maximum number of work groups per compute unit supported by the
1168   /// subtarget and limited by given \p FlatWorkGroupSize.
1169   unsigned getMaxWorkGroupsPerCU(unsigned FlatWorkGroupSize) const override {
1170     return AMDGPU::IsaInfo::getMaxWorkGroupsPerCU(this, FlatWorkGroupSize);
1171   }
1172 
1173   /// \returns Minimum flat work group size supported by the subtarget.
1174   unsigned getMinFlatWorkGroupSize() const override {
1175     return AMDGPU::IsaInfo::getMinFlatWorkGroupSize(this);
1176   }
1177 
1178   /// \returns Maximum flat work group size supported by the subtarget.
1179   unsigned getMaxFlatWorkGroupSize() const override {
1180     return AMDGPU::IsaInfo::getMaxFlatWorkGroupSize(this);
1181   }
1182 
1183   /// \returns Number of waves per execution unit required to support the given
1184   /// \p FlatWorkGroupSize.
1185   unsigned
1186   getWavesPerEUForWorkGroup(unsigned FlatWorkGroupSize) const override {
1187     return AMDGPU::IsaInfo::getWavesPerEUForWorkGroup(this, FlatWorkGroupSize);
1188   }
1189 
1190   /// \returns Minimum number of waves per execution unit supported by the
1191   /// subtarget.
1192   unsigned getMinWavesPerEU() const override {
1193     return AMDGPU::IsaInfo::getMinWavesPerEU(this);
1194   }
1195 
1196   void adjustSchedDependency(SUnit *Src, SUnit *Dst, SDep &Dep) const override;
1197 };
1198 
1199 class R600Subtarget final : public R600GenSubtargetInfo,
1200                             public AMDGPUSubtarget {
1201 private:
1202   R600InstrInfo InstrInfo;
1203   R600FrameLowering FrameLowering;
1204   bool FMA;
1205   bool CaymanISA;
1206   bool CFALUBug;
1207   bool HasVertexCache;
1208   bool R600ALUInst;
1209   bool FP64;
1210   short TexVTXClauseSize;
1211   Generation Gen;
1212   R600TargetLowering TLInfo;
1213   InstrItineraryData InstrItins;
1214   SelectionDAGTargetInfo TSInfo;
1215 
1216 public:
1217   R600Subtarget(const Triple &TT, StringRef CPU, StringRef FS,
1218                 const TargetMachine &TM);
1219 
1220   const R600InstrInfo *getInstrInfo() const override { return &InstrInfo; }
1221 
1222   const R600FrameLowering *getFrameLowering() const override {
1223     return &FrameLowering;
1224   }
1225 
1226   const R600TargetLowering *getTargetLowering() const override {
1227     return &TLInfo;
1228   }
1229 
1230   const R600RegisterInfo *getRegisterInfo() const override {
1231     return &InstrInfo.getRegisterInfo();
1232   }
1233 
1234   const InstrItineraryData *getInstrItineraryData() const override {
1235     return &InstrItins;
1236   }
1237 
1238   // Nothing implemented, just prevent crashes on use.
1239   const SelectionDAGTargetInfo *getSelectionDAGInfo() const override {
1240     return &TSInfo;
1241   }
1242 
1243   void ParseSubtargetFeatures(StringRef CPU, StringRef FS);
1244 
1245   Generation getGeneration() const {
1246     return Gen;
1247   }
1248 
1249   Align getStackAlignment() const { return Align(4); }
1250 
1251   R600Subtarget &initializeSubtargetDependencies(const Triple &TT,
1252                                                  StringRef GPU, StringRef FS);
1253 
1254   bool hasBFE() const {
1255     return (getGeneration() >= EVERGREEN);
1256   }
1257 
1258   bool hasBFI() const {
1259     return (getGeneration() >= EVERGREEN);
1260   }
1261 
1262   bool hasBCNT(unsigned Size) const {
1263     if (Size == 32)
1264       return (getGeneration() >= EVERGREEN);
1265 
1266     return false;
1267   }
1268 
1269   bool hasBORROW() const {
1270     return (getGeneration() >= EVERGREEN);
1271   }
1272 
1273   bool hasCARRY() const {
1274     return (getGeneration() >= EVERGREEN);
1275   }
1276 
1277   bool hasCaymanISA() const {
1278     return CaymanISA;
1279   }
1280 
1281   bool hasFFBL() const {
1282     return (getGeneration() >= EVERGREEN);
1283   }
1284 
1285   bool hasFFBH() const {
1286     return (getGeneration() >= EVERGREEN);
1287   }
1288 
1289   bool hasFMA() const { return FMA; }
1290 
1291   bool hasCFAluBug() const { return CFALUBug; }
1292 
1293   bool hasVertexCache() const { return HasVertexCache; }
1294 
1295   short getTexVTXClauseSize() const { return TexVTXClauseSize; }
1296 
1297   bool enableMachineScheduler() const override {
1298     return true;
1299   }
1300 
1301   bool enableSubRegLiveness() const override {
1302     return true;
1303   }
1304 
1305   /// \returns Maximum number of work groups per compute unit supported by the
1306   /// subtarget and limited by given \p FlatWorkGroupSize.
1307   unsigned getMaxWorkGroupsPerCU(unsigned FlatWorkGroupSize) const override {
1308     return AMDGPU::IsaInfo::getMaxWorkGroupsPerCU(this, FlatWorkGroupSize);
1309   }
1310 
1311   /// \returns Minimum flat work group size supported by the subtarget.
1312   unsigned getMinFlatWorkGroupSize() const override {
1313     return AMDGPU::IsaInfo::getMinFlatWorkGroupSize(this);
1314   }
1315 
1316   /// \returns Maximum flat work group size supported by the subtarget.
1317   unsigned getMaxFlatWorkGroupSize() const override {
1318     return AMDGPU::IsaInfo::getMaxFlatWorkGroupSize(this);
1319   }
1320 
1321   /// \returns Number of waves per execution unit required to support the given
1322   /// \p FlatWorkGroupSize.
1323   unsigned
1324   getWavesPerEUForWorkGroup(unsigned FlatWorkGroupSize) const override {
1325     return AMDGPU::IsaInfo::getWavesPerEUForWorkGroup(this, FlatWorkGroupSize);
1326   }
1327 
1328   /// \returns Minimum number of waves per execution unit supported by the
1329   /// subtarget.
1330   unsigned getMinWavesPerEU() const override {
1331     return AMDGPU::IsaInfo::getMinWavesPerEU(this);
1332   }
1333 };
1334 
1335 } // end namespace llvm
1336 
1337 #endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUSUBTARGET_H
1338