1 //=====-- AMDGPUSubtarget.h - Define Subtarget for AMDGPU ------*- C++ -*-====//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //==-----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// AMDGPU specific subclass of TargetSubtarget.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUSUBTARGET_H
15 #define LLVM_LIB_TARGET_AMDGPU_AMDGPUSUBTARGET_H
16 
17 #include "AMDGPU.h"
18 #include "AMDGPUCallLowering.h"
19 #include "R600FrameLowering.h"
20 #include "R600ISelLowering.h"
21 #include "R600InstrInfo.h"
22 #include "SIFrameLowering.h"
23 #include "SIISelLowering.h"
24 #include "SIInstrInfo.h"
25 #include "Utils/AMDGPUBaseInfo.h"
26 #include "llvm/ADT/Triple.h"
27 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
28 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
29 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
30 #include "llvm/CodeGen/MachineFunction.h"
31 #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
32 #include "llvm/MC/MCInstrItineraries.h"
33 #include "llvm/Support/MathExtras.h"
34 #include <cassert>
35 #include <cstdint>
36 #include <memory>
37 #include <utility>
38 
39 #define GET_SUBTARGETINFO_HEADER
40 #include "AMDGPUGenSubtargetInfo.inc"
41 #define GET_SUBTARGETINFO_HEADER
42 #include "R600GenSubtargetInfo.inc"
43 
44 namespace llvm {
45 
46 class StringRef;
47 
48 class AMDGPUSubtarget {
49 public:
50   enum Generation {
51     R600 = 0,
52     R700 = 1,
53     EVERGREEN = 2,
54     NORTHERN_ISLANDS = 3,
55     SOUTHERN_ISLANDS = 4,
56     SEA_ISLANDS = 5,
57     VOLCANIC_ISLANDS = 6,
58     GFX9 = 7,
59     GFX10 = 8
60   };
61 
62 private:
63   Triple TargetTriple;
64 
65 protected:
66   bool Has16BitInsts;
67   bool HasMadMixInsts;
68   bool FP32Denormals;
69   bool FPExceptions;
70   bool HasSDWA;
71   bool HasVOP3PInsts;
72   bool HasMulI24;
73   bool HasMulU24;
74   bool HasInv2PiInlineImm;
75   bool HasFminFmaxLegacy;
76   bool EnablePromoteAlloca;
77   bool HasTrigReducedRange;
78   unsigned MaxWavesPerEU;
79   int LocalMemorySize;
80   unsigned WavefrontSize;
81 
82 public:
83   AMDGPUSubtarget(const Triple &TT);
84 
85   static const AMDGPUSubtarget &get(const MachineFunction &MF);
86   static const AMDGPUSubtarget &get(const TargetMachine &TM,
87                                     const Function &F);
88 
89   /// \returns Default range flat work group size for a calling convention.
90   std::pair<unsigned, unsigned> getDefaultFlatWorkGroupSize(CallingConv::ID CC) const;
91 
92   /// \returns Subtarget's default pair of minimum/maximum flat work group sizes
93   /// for function \p F, or minimum/maximum flat work group sizes explicitly
94   /// requested using "amdgpu-flat-work-group-size" attribute attached to
95   /// function \p F.
96   ///
97   /// \returns Subtarget's default values if explicitly requested values cannot
98   /// be converted to integer, or violate subtarget's specifications.
99   std::pair<unsigned, unsigned> getFlatWorkGroupSizes(const Function &F) const;
100 
101   /// \returns Subtarget's default pair of minimum/maximum number of waves per
102   /// execution unit for function \p F, or minimum/maximum number of waves per
103   /// execution unit explicitly requested using "amdgpu-waves-per-eu" attribute
104   /// attached to function \p F.
105   ///
106   /// \returns Subtarget's default values if explicitly requested values cannot
107   /// be converted to integer, violate subtarget's specifications, or are not
108   /// compatible with minimum/maximum number of waves limited by flat work group
109   /// size, register usage, and/or lds usage.
110   std::pair<unsigned, unsigned> getWavesPerEU(const Function &F) const;
111 
112   /// Return the amount of LDS that can be used that will not restrict the
113   /// occupancy lower than WaveCount.
114   unsigned getMaxLocalMemSizeWithWaveCount(unsigned WaveCount,
115                                            const Function &) const;
116 
117   /// Inverse of getMaxLocalMemWithWaveCount. Return the maximum wavecount if
118   /// the given LDS memory size is the only constraint.
119   unsigned getOccupancyWithLocalMemSize(uint32_t Bytes, const Function &) const;
120 
121   unsigned getOccupancyWithLocalMemSize(const MachineFunction &MF) const;
122 
123   bool isAmdHsaOS() const {
124     return TargetTriple.getOS() == Triple::AMDHSA;
125   }
126 
127   bool isAmdPalOS() const {
128     return TargetTriple.getOS() == Triple::AMDPAL;
129   }
130 
131   bool isMesa3DOS() const {
132     return TargetTriple.getOS() == Triple::Mesa3D;
133   }
134 
135   bool isMesaKernel(const Function &F) const {
136     return isMesa3DOS() && !AMDGPU::isShader(F.getCallingConv());
137   }
138 
139   bool isAmdHsaOrMesa(const Function &F) const {
140     return isAmdHsaOS() || isMesaKernel(F);
141   }
142 
143   bool has16BitInsts() const {
144     return Has16BitInsts;
145   }
146 
147   bool hasMadMixInsts() const {
148     return HasMadMixInsts;
149   }
150 
151   bool hasFP32Denormals(const Function &F) const {
152     // FIXME: This should not be a property of the subtarget. This should be a
153     // property with a default set by the calling convention which can be
154     // overridden by attributes. For now, use the subtarget feature as a
155     // placeholder attribute. The function arguments only purpose is to
156     // discourage use without a function context until this is removed.
157     return FP32Denormals;
158   }
159 
160   bool hasFPExceptions() const {
161     return FPExceptions;
162   }
163 
164   bool hasSDWA() const {
165     return HasSDWA;
166   }
167 
168   bool hasVOP3PInsts() const {
169     return HasVOP3PInsts;
170   }
171 
172   bool hasMulI24() const {
173     return HasMulI24;
174   }
175 
176   bool hasMulU24() const {
177     return HasMulU24;
178   }
179 
180   bool hasInv2PiInlineImm() const {
181     return HasInv2PiInlineImm;
182   }
183 
184   bool hasFminFmaxLegacy() const {
185     return HasFminFmaxLegacy;
186   }
187 
188   bool hasTrigReducedRange() const {
189     return HasTrigReducedRange;
190   }
191 
192   bool isPromoteAllocaEnabled() const {
193     return EnablePromoteAlloca;
194   }
195 
196   unsigned getWavefrontSize() const {
197     return WavefrontSize;
198   }
199 
200   int getLocalMemorySize() const {
201     return LocalMemorySize;
202   }
203 
204   Align getAlignmentForImplicitArgPtr() const {
205     return isAmdHsaOS() ? Align(8) : Align(4);
206   }
207 
208   /// Returns the offset in bytes from the start of the input buffer
209   ///        of the first explicit kernel argument.
210   unsigned getExplicitKernelArgOffset(const Function &F) const {
211     return isAmdHsaOrMesa(F) ? 0 : 36;
212   }
213 
214   /// \returns Maximum number of work groups per compute unit supported by the
215   /// subtarget and limited by given \p FlatWorkGroupSize.
216   virtual unsigned getMaxWorkGroupsPerCU(unsigned FlatWorkGroupSize) const = 0;
217 
218   /// \returns Minimum flat work group size supported by the subtarget.
219   virtual unsigned getMinFlatWorkGroupSize() const = 0;
220 
221   /// \returns Maximum flat work group size supported by the subtarget.
222   virtual unsigned getMaxFlatWorkGroupSize() const = 0;
223 
224   /// \returns Maximum number of waves per execution unit supported by the
225   /// subtarget and limited by given \p FlatWorkGroupSize.
226   virtual unsigned getMaxWavesPerEU(unsigned FlatWorkGroupSize) const  = 0;
227 
228   /// \returns Minimum number of waves per execution unit supported by the
229   /// subtarget.
230   virtual unsigned getMinWavesPerEU() const = 0;
231 
232   /// \returns Maximum number of waves per execution unit supported by the
233   /// subtarget without any kind of limitation.
234   unsigned getMaxWavesPerEU() const { return MaxWavesPerEU; }
235 
236   /// Creates value range metadata on an workitemid.* inrinsic call or load.
237   bool makeLIDRangeMetadata(Instruction *I) const;
238 
239   /// \returns Number of bytes of arguments that are passed to a shader or
240   /// kernel in addition to the explicit ones declared for the function.
241   unsigned getImplicitArgNumBytes(const Function &F) const {
242     if (isMesaKernel(F))
243       return 16;
244     return AMDGPU::getIntegerAttribute(F, "amdgpu-implicitarg-num-bytes", 0);
245   }
246   uint64_t getExplicitKernArgSize(const Function &F, Align &MaxAlign) const;
247   unsigned getKernArgSegmentSize(const Function &F, Align &MaxAlign) const;
248 
249   virtual ~AMDGPUSubtarget() {}
250 };
251 
252 class GCNSubtarget : public AMDGPUGenSubtargetInfo,
253                      public AMDGPUSubtarget {
254 
255   using AMDGPUSubtarget::getMaxWavesPerEU;
256 
257 public:
258   enum TrapHandlerAbi {
259     TrapHandlerAbiNone = 0,
260     TrapHandlerAbiHsa = 1
261   };
262 
263   enum TrapID {
264     TrapIDHardwareReserved = 0,
265     TrapIDHSADebugTrap = 1,
266     TrapIDLLVMTrap = 2,
267     TrapIDLLVMDebugTrap = 3,
268     TrapIDDebugBreakpoint = 7,
269     TrapIDDebugReserved8 = 8,
270     TrapIDDebugReservedFE = 0xfe,
271     TrapIDDebugReservedFF = 0xff
272   };
273 
274   enum TrapRegValues {
275     LLVMTrapHandlerRegValue = 1
276   };
277 
278 private:
279   /// GlobalISel related APIs.
280   std::unique_ptr<AMDGPUCallLowering> CallLoweringInfo;
281   std::unique_ptr<InstructionSelector> InstSelector;
282   std::unique_ptr<LegalizerInfo> Legalizer;
283   std::unique_ptr<RegisterBankInfo> RegBankInfo;
284 
285 protected:
286   // Basic subtarget description.
287   Triple TargetTriple;
288   unsigned Gen;
289   InstrItineraryData InstrItins;
290   int LDSBankCount;
291   unsigned MaxPrivateElementSize;
292 
293   // Possibly statically set by tablegen, but may want to be overridden.
294   bool FastFMAF32;
295   bool HalfRate64Ops;
296 
297   // Dynamially set bits that enable features.
298   bool FP64FP16Denormals;
299   bool FlatForGlobal;
300   bool AutoWaitcntBeforeBarrier;
301   bool CodeObjectV3;
302   bool UnalignedScratchAccess;
303   bool UnalignedBufferAccess;
304   bool HasApertureRegs;
305   bool EnableXNACK;
306   bool DoesNotSupportXNACK;
307   bool EnableCuMode;
308   bool TrapHandler;
309 
310   // Used as options.
311   bool EnableLoadStoreOpt;
312   bool EnableUnsafeDSOffsetFolding;
313   bool EnableSIScheduler;
314   bool EnableDS128;
315   bool EnablePRTStrictNull;
316   bool DumpCode;
317 
318   // Subtarget statically properties set by tablegen
319   bool FP64;
320   bool FMA;
321   bool MIMG_R128;
322   bool IsGCN;
323   bool GCN3Encoding;
324   bool CIInsts;
325   bool GFX8Insts;
326   bool GFX9Insts;
327   bool GFX10Insts;
328   bool GFX7GFX8GFX9Insts;
329   bool SGPRInitBug;
330   bool HasSMemRealTime;
331   bool HasIntClamp;
332   bool HasFmaMixInsts;
333   bool HasMovrel;
334   bool HasVGPRIndexMode;
335   bool HasScalarStores;
336   bool HasScalarAtomics;
337   bool HasSDWAOmod;
338   bool HasSDWAScalar;
339   bool HasSDWASdst;
340   bool HasSDWAMac;
341   bool HasSDWAOutModsVOPC;
342   bool HasDPP;
343   bool HasDPP8;
344   bool HasR128A16;
345   bool HasNSAEncoding;
346   bool HasDLInsts;
347   bool HasDot1Insts;
348   bool HasDot2Insts;
349   bool HasDot3Insts;
350   bool HasDot4Insts;
351   bool HasDot5Insts;
352   bool HasDot6Insts;
353   bool HasMAIInsts;
354   bool HasPkFmacF16Inst;
355   bool HasAtomicFaddInsts;
356   bool EnableSRAMECC;
357   bool DoesNotSupportSRAMECC;
358   bool HasNoSdstCMPX;
359   bool HasVscnt;
360   bool HasRegisterBanking;
361   bool HasVOP3Literal;
362   bool HasNoDataDepHazard;
363   bool FlatAddressSpace;
364   bool FlatInstOffsets;
365   bool FlatGlobalInsts;
366   bool FlatScratchInsts;
367   bool ScalarFlatScratchInsts;
368   bool AddNoCarryInsts;
369   bool HasUnpackedD16VMem;
370   bool R600ALUInst;
371   bool CaymanISA;
372   bool CFALUBug;
373   bool LDSMisalignedBug;
374   bool HasMFMAInlineLiteralBug;
375   bool HasVertexCache;
376   short TexVTXClauseSize;
377   bool ScalarizeGlobal;
378 
379   bool HasVcmpxPermlaneHazard;
380   bool HasVMEMtoScalarWriteHazard;
381   bool HasSMEMtoVectorWriteHazard;
382   bool HasInstFwdPrefetchBug;
383   bool HasVcmpxExecWARHazard;
384   bool HasLdsBranchVmemWARHazard;
385   bool HasNSAtoVMEMBug;
386   bool HasOffset3fBug;
387   bool HasFlatSegmentOffsetBug;
388 
389   // Dummy feature to use for assembler in tablegen.
390   bool FeatureDisable;
391 
392   SelectionDAGTargetInfo TSInfo;
393 private:
394   SIInstrInfo InstrInfo;
395   SITargetLowering TLInfo;
396   SIFrameLowering FrameLowering;
397 
398   // See COMPUTE_TMPRING_SIZE.WAVESIZE, 13-bit field in units of 256-dword.
399   static const unsigned MaxWaveScratchSize = (256 * 4) * ((1 << 13) - 1);
400 
401 public:
402   GCNSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
403                const GCNTargetMachine &TM);
404   ~GCNSubtarget() override;
405 
406   GCNSubtarget &initializeSubtargetDependencies(const Triple &TT,
407                                                    StringRef GPU, StringRef FS);
408 
409   const SIInstrInfo *getInstrInfo() const override {
410     return &InstrInfo;
411   }
412 
413   const SIFrameLowering *getFrameLowering() const override {
414     return &FrameLowering;
415   }
416 
417   const SITargetLowering *getTargetLowering() const override {
418     return &TLInfo;
419   }
420 
421   const SIRegisterInfo *getRegisterInfo() const override {
422     return &InstrInfo.getRegisterInfo();
423   }
424 
425   const CallLowering *getCallLowering() const override {
426     return CallLoweringInfo.get();
427   }
428 
429   InstructionSelector *getInstructionSelector() const override {
430     return InstSelector.get();
431   }
432 
433   const LegalizerInfo *getLegalizerInfo() const override {
434     return Legalizer.get();
435   }
436 
437   const RegisterBankInfo *getRegBankInfo() const override {
438     return RegBankInfo.get();
439   }
440 
441   // Nothing implemented, just prevent crashes on use.
442   const SelectionDAGTargetInfo *getSelectionDAGInfo() const override {
443     return &TSInfo;
444   }
445 
446   const InstrItineraryData *getInstrItineraryData() const override {
447     return &InstrItins;
448   }
449 
450   void ParseSubtargetFeatures(StringRef CPU, StringRef FS);
451 
452   Generation getGeneration() const {
453     return (Generation)Gen;
454   }
455 
456   unsigned getWavefrontSizeLog2() const {
457     return Log2_32(WavefrontSize);
458   }
459 
460   /// Return the number of high bits known to be zero fror a frame index.
461   unsigned getKnownHighZeroBitsForFrameIndex() const {
462     return countLeadingZeros(MaxWaveScratchSize) + getWavefrontSizeLog2();
463   }
464 
465   int getLDSBankCount() const {
466     return LDSBankCount;
467   }
468 
469   unsigned getMaxPrivateElementSize() const {
470     return MaxPrivateElementSize;
471   }
472 
473   unsigned getConstantBusLimit(unsigned Opcode) const;
474 
475   bool hasIntClamp() const {
476     return HasIntClamp;
477   }
478 
479   bool hasFP64() const {
480     return FP64;
481   }
482 
483   bool hasMIMG_R128() const {
484     return MIMG_R128;
485   }
486 
487   bool hasHWFP64() const {
488     return FP64;
489   }
490 
491   bool hasFastFMAF32() const {
492     return FastFMAF32;
493   }
494 
495   bool hasHalfRate64Ops() const {
496     return HalfRate64Ops;
497   }
498 
499   bool hasAddr64() const {
500     return (getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS);
501   }
502 
503   // Return true if the target only has the reverse operand versions of VALU
504   // shift instructions (e.g. v_lshrrev_b32, and no v_lshr_b32).
505   bool hasOnlyRevVALUShifts() const {
506     return getGeneration() >= VOLCANIC_ISLANDS;
507   }
508 
509   bool hasBFE() const {
510     return true;
511   }
512 
513   bool hasBFI() const {
514     return true;
515   }
516 
517   bool hasBFM() const {
518     return hasBFE();
519   }
520 
521   bool hasBCNT(unsigned Size) const {
522     return true;
523   }
524 
525   bool hasFFBL() const {
526     return true;
527   }
528 
529   bool hasFFBH() const {
530     return true;
531   }
532 
533   bool hasMed3_16() const {
534     return getGeneration() >= AMDGPUSubtarget::GFX9;
535   }
536 
537   bool hasMin3Max3_16() const {
538     return getGeneration() >= AMDGPUSubtarget::GFX9;
539   }
540 
541   bool hasFmaMixInsts() const {
542     return HasFmaMixInsts;
543   }
544 
545   bool hasCARRY() const {
546     return true;
547   }
548 
549   bool hasFMA() const {
550     return FMA;
551   }
552 
553   bool hasSwap() const {
554     return GFX9Insts;
555   }
556 
557   bool hasScalarPackInsts() const {
558     return GFX9Insts;
559   }
560 
561   bool hasScalarMulHiInsts() const {
562     return GFX9Insts;
563   }
564 
565   TrapHandlerAbi getTrapHandlerAbi() const {
566     return isAmdHsaOS() ? TrapHandlerAbiHsa : TrapHandlerAbiNone;
567   }
568 
569   /// True if the offset field of DS instructions works as expected. On SI, the
570   /// offset uses a 16-bit adder and does not always wrap properly.
571   bool hasUsableDSOffset() const {
572     return getGeneration() >= SEA_ISLANDS;
573   }
574 
575   bool unsafeDSOffsetFoldingEnabled() const {
576     return EnableUnsafeDSOffsetFolding;
577   }
578 
579   /// Condition output from div_scale is usable.
580   bool hasUsableDivScaleConditionOutput() const {
581     return getGeneration() != SOUTHERN_ISLANDS;
582   }
583 
584   /// Extra wait hazard is needed in some cases before
585   /// s_cbranch_vccnz/s_cbranch_vccz.
586   bool hasReadVCCZBug() const {
587     return getGeneration() <= SEA_ISLANDS;
588   }
589 
590   /// Writes to VCC_LO/VCC_HI update the VCCZ flag.
591   bool partialVCCWritesUpdateVCCZ() const {
592     return getGeneration() >= GFX10;
593   }
594 
595   /// A read of an SGPR by SMRD instruction requires 4 wait states when the SGPR
596   /// was written by a VALU instruction.
597   bool hasSMRDReadVALUDefHazard() const {
598     return getGeneration() == SOUTHERN_ISLANDS;
599   }
600 
601   /// A read of an SGPR by a VMEM instruction requires 5 wait states when the
602   /// SGPR was written by a VALU Instruction.
603   bool hasVMEMReadSGPRVALUDefHazard() const {
604     return getGeneration() >= VOLCANIC_ISLANDS;
605   }
606 
607   bool hasRFEHazards() const {
608     return getGeneration() >= VOLCANIC_ISLANDS;
609   }
610 
611   /// Number of hazard wait states for s_setreg_b32/s_setreg_imm32_b32.
612   unsigned getSetRegWaitStates() const {
613     return getGeneration() <= SEA_ISLANDS ? 1 : 2;
614   }
615 
616   bool dumpCode() const {
617     return DumpCode;
618   }
619 
620   /// Return the amount of LDS that can be used that will not restrict the
621   /// occupancy lower than WaveCount.
622   unsigned getMaxLocalMemSizeWithWaveCount(unsigned WaveCount,
623                                            const Function &) const;
624 
625   /// Alias for hasFP64FP16Denormals
626   bool hasFP16Denormals(const Function &F) const {
627     return FP64FP16Denormals;
628   }
629 
630   /// Alias for hasFP64FP16Denormals
631   bool hasFP64Denormals(const Function &F) const {
632     return FP64FP16Denormals;
633   }
634 
635   bool hasFP64FP16Denormals(const Function &F) const {
636     return FP64FP16Denormals;
637   }
638 
639   bool supportsMinMaxDenormModes() const {
640     return getGeneration() >= AMDGPUSubtarget::GFX9;
641   }
642 
643   /// \returns If target supports S_DENORM_MODE.
644   bool hasDenormModeInst() const {
645     return getGeneration() >= AMDGPUSubtarget::GFX10;
646   }
647 
648   bool useFlatForGlobal() const {
649     return FlatForGlobal;
650   }
651 
652   /// \returns If target supports ds_read/write_b128 and user enables generation
653   /// of ds_read/write_b128.
654   bool useDS128() const {
655     return CIInsts && EnableDS128;
656   }
657 
658   /// Have v_trunc_f64, v_ceil_f64, v_rndne_f64
659   bool haveRoundOpsF64() const {
660     return CIInsts;
661   }
662 
663   /// \returns If MUBUF instructions always perform range checking, even for
664   /// buffer resources used for private memory access.
665   bool privateMemoryResourceIsRangeChecked() const {
666     return getGeneration() < AMDGPUSubtarget::GFX9;
667   }
668 
669   /// \returns If target requires PRT Struct NULL support (zero result registers
670   /// for sparse texture support).
671   bool usePRTStrictNull() const {
672     return EnablePRTStrictNull;
673   }
674 
675   bool hasAutoWaitcntBeforeBarrier() const {
676     return AutoWaitcntBeforeBarrier;
677   }
678 
679   bool hasCodeObjectV3() const {
680     // FIXME: Need to add code object v3 support for mesa and pal.
681     return isAmdHsaOS() ? CodeObjectV3 : false;
682   }
683 
684   bool hasUnalignedBufferAccess() const {
685     return UnalignedBufferAccess;
686   }
687 
688   bool hasUnalignedScratchAccess() const {
689     return UnalignedScratchAccess;
690   }
691 
692   bool hasApertureRegs() const {
693     return HasApertureRegs;
694   }
695 
696   bool isTrapHandlerEnabled() const {
697     return TrapHandler;
698   }
699 
700   bool isXNACKEnabled() const {
701     return EnableXNACK;
702   }
703 
704   bool isCuModeEnabled() const {
705     return EnableCuMode;
706   }
707 
708   bool hasFlatAddressSpace() const {
709     return FlatAddressSpace;
710   }
711 
712   bool hasFlatScrRegister() const {
713     return hasFlatAddressSpace();
714   }
715 
716   bool hasFlatInstOffsets() const {
717     return FlatInstOffsets;
718   }
719 
720   bool hasFlatGlobalInsts() const {
721     return FlatGlobalInsts;
722   }
723 
724   bool hasFlatScratchInsts() const {
725     return FlatScratchInsts;
726   }
727 
728   bool hasScalarFlatScratchInsts() const {
729     return ScalarFlatScratchInsts;
730   }
731 
732   bool hasMultiDwordFlatScratchAddressing() const {
733     return getGeneration() >= GFX9;
734   }
735 
736   bool hasFlatSegmentOffsetBug() const {
737     return HasFlatSegmentOffsetBug;
738   }
739 
740   bool hasFlatLgkmVMemCountInOrder() const {
741     return getGeneration() > GFX9;
742   }
743 
744   bool hasD16LoadStore() const {
745     return getGeneration() >= GFX9;
746   }
747 
748   bool d16PreservesUnusedBits() const {
749     return hasD16LoadStore() && !isSRAMECCEnabled();
750   }
751 
752   bool hasD16Images() const {
753     return getGeneration() >= VOLCANIC_ISLANDS;
754   }
755 
756   /// Return if most LDS instructions have an m0 use that require m0 to be
757   /// iniitalized.
758   bool ldsRequiresM0Init() const {
759     return getGeneration() < GFX9;
760   }
761 
762   // True if the hardware rewinds and replays GWS operations if a wave is
763   // preempted.
764   //
765   // If this is false, a GWS operation requires testing if a nack set the
766   // MEM_VIOL bit, and repeating if so.
767   bool hasGWSAutoReplay() const {
768     return getGeneration() >= GFX9;
769   }
770 
771   /// \returns if target has ds_gws_sema_release_all instruction.
772   bool hasGWSSemaReleaseAll() const {
773     return CIInsts;
774   }
775 
776   bool hasAddNoCarry() const {
777     return AddNoCarryInsts;
778   }
779 
780   bool hasUnpackedD16VMem() const {
781     return HasUnpackedD16VMem;
782   }
783 
784   // Covers VS/PS/CS graphics shaders
785   bool isMesaGfxShader(const Function &F) const {
786     return isMesa3DOS() && AMDGPU::isShader(F.getCallingConv());
787   }
788 
789   bool hasMad64_32() const {
790     return getGeneration() >= SEA_ISLANDS;
791   }
792 
793   bool hasSDWAOmod() const {
794     return HasSDWAOmod;
795   }
796 
797   bool hasSDWAScalar() const {
798     return HasSDWAScalar;
799   }
800 
801   bool hasSDWASdst() const {
802     return HasSDWASdst;
803   }
804 
805   bool hasSDWAMac() const {
806     return HasSDWAMac;
807   }
808 
809   bool hasSDWAOutModsVOPC() const {
810     return HasSDWAOutModsVOPC;
811   }
812 
813   bool hasDLInsts() const {
814     return HasDLInsts;
815   }
816 
817   bool hasDot1Insts() const {
818     return HasDot1Insts;
819   }
820 
821   bool hasDot2Insts() const {
822     return HasDot2Insts;
823   }
824 
825   bool hasDot3Insts() const {
826     return HasDot3Insts;
827   }
828 
829   bool hasDot4Insts() const {
830     return HasDot4Insts;
831   }
832 
833   bool hasDot5Insts() const {
834     return HasDot5Insts;
835   }
836 
837   bool hasDot6Insts() const {
838     return HasDot6Insts;
839   }
840 
841   bool hasMAIInsts() const {
842     return HasMAIInsts;
843   }
844 
845   bool hasPkFmacF16Inst() const {
846     return HasPkFmacF16Inst;
847   }
848 
849   bool hasAtomicFaddInsts() const {
850     return HasAtomicFaddInsts;
851   }
852 
853   bool isSRAMECCEnabled() const {
854     return EnableSRAMECC;
855   }
856 
857   bool hasNoSdstCMPX() const {
858     return HasNoSdstCMPX;
859   }
860 
861   bool hasVscnt() const {
862     return HasVscnt;
863   }
864 
865   bool hasRegisterBanking() const {
866     return HasRegisterBanking;
867   }
868 
869   bool hasVOP3Literal() const {
870     return HasVOP3Literal;
871   }
872 
873   bool hasNoDataDepHazard() const {
874     return HasNoDataDepHazard;
875   }
876 
877   bool vmemWriteNeedsExpWaitcnt() const {
878     return getGeneration() < SEA_ISLANDS;
879   }
880 
881   // Scratch is allocated in 256 dword per wave blocks for the entire
882   // wavefront. When viewed from the perspecive of an arbitrary workitem, this
883   // is 4-byte aligned.
884   //
885   // Only 4-byte alignment is really needed to access anything. Transformations
886   // on the pointer value itself may rely on the alignment / known low bits of
887   // the pointer. Set this to something above the minimum to avoid needing
888   // dynamic realignment in common cases.
889   Align getStackAlignment() const { return Align(16); }
890 
891   bool enableMachineScheduler() const override {
892     return true;
893   }
894 
895   bool enableSubRegLiveness() const override {
896     return true;
897   }
898 
899   void setScalarizeGlobalBehavior(bool b) { ScalarizeGlobal = b; }
900   bool getScalarizeGlobalBehavior() const { return ScalarizeGlobal; }
901 
902   /// \returns Number of execution units per compute unit supported by the
903   /// subtarget.
904   unsigned getEUsPerCU() const {
905     return AMDGPU::IsaInfo::getEUsPerCU(this);
906   }
907 
908   /// \returns Maximum number of waves per compute unit supported by the
909   /// subtarget without any kind of limitation.
910   unsigned getMaxWavesPerCU() const {
911     return AMDGPU::IsaInfo::getMaxWavesPerCU(this);
912   }
913 
914   /// \returns Maximum number of waves per compute unit supported by the
915   /// subtarget and limited by given \p FlatWorkGroupSize.
916   unsigned getMaxWavesPerCU(unsigned FlatWorkGroupSize) const {
917     return AMDGPU::IsaInfo::getMaxWavesPerCU(this, FlatWorkGroupSize);
918   }
919 
920   /// \returns Number of waves per work group supported by the subtarget and
921   /// limited by given \p FlatWorkGroupSize.
922   unsigned getWavesPerWorkGroup(unsigned FlatWorkGroupSize) const {
923     return AMDGPU::IsaInfo::getWavesPerWorkGroup(this, FlatWorkGroupSize);
924   }
925 
926   // static wrappers
927   static bool hasHalfRate64Ops(const TargetSubtargetInfo &STI);
928 
929   // XXX - Why is this here if it isn't in the default pass set?
930   bool enableEarlyIfConversion() const override {
931     return true;
932   }
933 
934   void overrideSchedPolicy(MachineSchedPolicy &Policy,
935                            unsigned NumRegionInstrs) const override;
936 
937   unsigned getMaxNumUserSGPRs() const {
938     return 16;
939   }
940 
941   bool hasSMemRealTime() const {
942     return HasSMemRealTime;
943   }
944 
945   bool hasMovrel() const {
946     return HasMovrel;
947   }
948 
949   bool hasVGPRIndexMode() const {
950     return HasVGPRIndexMode;
951   }
952 
953   bool useVGPRIndexMode() const;
954 
955   bool hasScalarCompareEq64() const {
956     return getGeneration() >= VOLCANIC_ISLANDS;
957   }
958 
959   bool hasScalarStores() const {
960     return HasScalarStores;
961   }
962 
963   bool hasScalarAtomics() const {
964     return HasScalarAtomics;
965   }
966 
967   bool hasLDSFPAtomics() const {
968     return GFX8Insts;
969   }
970 
971   bool hasDPP() const {
972     return HasDPP;
973   }
974 
975   bool hasDPPBroadcasts() const {
976     return HasDPP && getGeneration() < GFX10;
977   }
978 
979   bool hasDPPWavefrontShifts() const {
980     return HasDPP && getGeneration() < GFX10;
981   }
982 
983   bool hasDPP8() const {
984     return HasDPP8;
985   }
986 
987   bool hasR128A16() const {
988     return HasR128A16;
989   }
990 
991   bool hasOffset3fBug() const {
992     return HasOffset3fBug;
993   }
994 
995   bool hasNSAEncoding() const {
996     return HasNSAEncoding;
997   }
998 
999   bool hasMadF16() const;
1000 
1001   bool enableSIScheduler() const {
1002     return EnableSIScheduler;
1003   }
1004 
1005   bool loadStoreOptEnabled() const {
1006     return EnableLoadStoreOpt;
1007   }
1008 
1009   bool hasSGPRInitBug() const {
1010     return SGPRInitBug;
1011   }
1012 
1013   bool hasMFMAInlineLiteralBug() const {
1014     return HasMFMAInlineLiteralBug;
1015   }
1016 
1017   bool has12DWordStoreHazard() const {
1018     return getGeneration() != AMDGPUSubtarget::SOUTHERN_ISLANDS;
1019   }
1020 
1021   // \returns true if the subtarget supports DWORDX3 load/store instructions.
1022   bool hasDwordx3LoadStores() const {
1023     return CIInsts;
1024   }
1025 
1026   bool hasSMovFedHazard() const {
1027     return getGeneration() == AMDGPUSubtarget::GFX9;
1028   }
1029 
1030   bool hasReadM0MovRelInterpHazard() const {
1031     return getGeneration() == AMDGPUSubtarget::GFX9;
1032   }
1033 
1034   bool hasReadM0SendMsgHazard() const {
1035     return getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
1036            getGeneration() <= AMDGPUSubtarget::GFX9;
1037   }
1038 
1039   bool hasVcmpxPermlaneHazard() const {
1040     return HasVcmpxPermlaneHazard;
1041   }
1042 
1043   bool hasVMEMtoScalarWriteHazard() const {
1044     return HasVMEMtoScalarWriteHazard;
1045   }
1046 
1047   bool hasSMEMtoVectorWriteHazard() const {
1048     return HasSMEMtoVectorWriteHazard;
1049   }
1050 
1051   bool hasLDSMisalignedBug() const {
1052     return LDSMisalignedBug && !EnableCuMode;
1053   }
1054 
1055   bool hasInstFwdPrefetchBug() const {
1056     return HasInstFwdPrefetchBug;
1057   }
1058 
1059   bool hasVcmpxExecWARHazard() const {
1060     return HasVcmpxExecWARHazard;
1061   }
1062 
1063   bool hasLdsBranchVmemWARHazard() const {
1064     return HasLdsBranchVmemWARHazard;
1065   }
1066 
1067   bool hasNSAtoVMEMBug() const {
1068     return HasNSAtoVMEMBug;
1069   }
1070 
1071   /// Return the maximum number of waves per SIMD for kernels using \p SGPRs
1072   /// SGPRs
1073   unsigned getOccupancyWithNumSGPRs(unsigned SGPRs) const;
1074 
1075   /// Return the maximum number of waves per SIMD for kernels using \p VGPRs
1076   /// VGPRs
1077   unsigned getOccupancyWithNumVGPRs(unsigned VGPRs) const;
1078 
1079   /// Return occupancy for the given function. Used LDS and a number of
1080   /// registers if provided.
1081   /// Note, occupancy can be affected by the scratch allocation as well, but
1082   /// we do not have enough information to compute it.
1083   unsigned computeOccupancy(const MachineFunction &MF, unsigned LDSSize = 0,
1084                             unsigned NumSGPRs = 0, unsigned NumVGPRs = 0) const;
1085 
1086   /// \returns true if the flat_scratch register should be initialized with the
1087   /// pointer to the wave's scratch memory rather than a size and offset.
1088   bool flatScratchIsPointer() const {
1089     return getGeneration() >= AMDGPUSubtarget::GFX9;
1090   }
1091 
1092   /// \returns true if the machine has merged shaders in which s0-s7 are
1093   /// reserved by the hardware and user SGPRs start at s8
1094   bool hasMergedShaders() const {
1095     return getGeneration() >= GFX9;
1096   }
1097 
1098   /// \returns SGPR allocation granularity supported by the subtarget.
1099   unsigned getSGPRAllocGranule() const {
1100     return AMDGPU::IsaInfo::getSGPRAllocGranule(this);
1101   }
1102 
1103   /// \returns SGPR encoding granularity supported by the subtarget.
1104   unsigned getSGPREncodingGranule() const {
1105     return AMDGPU::IsaInfo::getSGPREncodingGranule(this);
1106   }
1107 
1108   /// \returns Total number of SGPRs supported by the subtarget.
1109   unsigned getTotalNumSGPRs() const {
1110     return AMDGPU::IsaInfo::getTotalNumSGPRs(this);
1111   }
1112 
1113   /// \returns Addressable number of SGPRs supported by the subtarget.
1114   unsigned getAddressableNumSGPRs() const {
1115     return AMDGPU::IsaInfo::getAddressableNumSGPRs(this);
1116   }
1117 
1118   /// \returns Minimum number of SGPRs that meets the given number of waves per
1119   /// execution unit requirement supported by the subtarget.
1120   unsigned getMinNumSGPRs(unsigned WavesPerEU) const {
1121     return AMDGPU::IsaInfo::getMinNumSGPRs(this, WavesPerEU);
1122   }
1123 
1124   /// \returns Maximum number of SGPRs that meets the given number of waves per
1125   /// execution unit requirement supported by the subtarget.
1126   unsigned getMaxNumSGPRs(unsigned WavesPerEU, bool Addressable) const {
1127     return AMDGPU::IsaInfo::getMaxNumSGPRs(this, WavesPerEU, Addressable);
1128   }
1129 
1130   /// \returns Reserved number of SGPRs for given function \p MF.
1131   unsigned getReservedNumSGPRs(const MachineFunction &MF) const;
1132 
1133   /// \returns Maximum number of SGPRs that meets number of waves per execution
1134   /// unit requirement for function \p MF, or number of SGPRs explicitly
1135   /// requested using "amdgpu-num-sgpr" attribute attached to function \p MF.
1136   ///
1137   /// \returns Value that meets number of waves per execution unit requirement
1138   /// if explicitly requested value cannot be converted to integer, violates
1139   /// subtarget's specifications, or does not meet number of waves per execution
1140   /// unit requirement.
1141   unsigned getMaxNumSGPRs(const MachineFunction &MF) const;
1142 
1143   /// \returns VGPR allocation granularity supported by the subtarget.
1144   unsigned getVGPRAllocGranule() const {
1145     return AMDGPU::IsaInfo::getVGPRAllocGranule(this);
1146   }
1147 
1148   /// \returns VGPR encoding granularity supported by the subtarget.
1149   unsigned getVGPREncodingGranule() const {
1150     return AMDGPU::IsaInfo::getVGPREncodingGranule(this);
1151   }
1152 
1153   /// \returns Total number of VGPRs supported by the subtarget.
1154   unsigned getTotalNumVGPRs() const {
1155     return AMDGPU::IsaInfo::getTotalNumVGPRs(this);
1156   }
1157 
1158   /// \returns Addressable number of VGPRs supported by the subtarget.
1159   unsigned getAddressableNumVGPRs() const {
1160     return AMDGPU::IsaInfo::getAddressableNumVGPRs(this);
1161   }
1162 
1163   /// \returns Minimum number of VGPRs that meets given number of waves per
1164   /// execution unit requirement supported by the subtarget.
1165   unsigned getMinNumVGPRs(unsigned WavesPerEU) const {
1166     return AMDGPU::IsaInfo::getMinNumVGPRs(this, WavesPerEU);
1167   }
1168 
1169   /// \returns Maximum number of VGPRs that meets given number of waves per
1170   /// execution unit requirement supported by the subtarget.
1171   unsigned getMaxNumVGPRs(unsigned WavesPerEU) const {
1172     return AMDGPU::IsaInfo::getMaxNumVGPRs(this, WavesPerEU);
1173   }
1174 
1175   /// \returns Maximum number of VGPRs that meets number of waves per execution
1176   /// unit requirement for function \p MF, or number of VGPRs explicitly
1177   /// requested using "amdgpu-num-vgpr" attribute attached to function \p MF.
1178   ///
1179   /// \returns Value that meets number of waves per execution unit requirement
1180   /// if explicitly requested value cannot be converted to integer, violates
1181   /// subtarget's specifications, or does not meet number of waves per execution
1182   /// unit requirement.
1183   unsigned getMaxNumVGPRs(const MachineFunction &MF) const;
1184 
1185   void getPostRAMutations(
1186       std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations)
1187       const override;
1188 
1189   bool isWave32() const {
1190     return WavefrontSize == 32;
1191   }
1192 
1193   const TargetRegisterClass *getBoolRC() const {
1194     return getRegisterInfo()->getBoolRC();
1195   }
1196 
1197   /// \returns Maximum number of work groups per compute unit supported by the
1198   /// subtarget and limited by given \p FlatWorkGroupSize.
1199   unsigned getMaxWorkGroupsPerCU(unsigned FlatWorkGroupSize) const override {
1200     return AMDGPU::IsaInfo::getMaxWorkGroupsPerCU(this, FlatWorkGroupSize);
1201   }
1202 
1203   /// \returns Minimum flat work group size supported by the subtarget.
1204   unsigned getMinFlatWorkGroupSize() const override {
1205     return AMDGPU::IsaInfo::getMinFlatWorkGroupSize(this);
1206   }
1207 
1208   /// \returns Maximum flat work group size supported by the subtarget.
1209   unsigned getMaxFlatWorkGroupSize() const override {
1210     return AMDGPU::IsaInfo::getMaxFlatWorkGroupSize(this);
1211   }
1212 
1213   /// \returns Maximum number of waves per execution unit supported by the
1214   /// subtarget and limited by given \p FlatWorkGroupSize.
1215   unsigned getMaxWavesPerEU(unsigned FlatWorkGroupSize) const override {
1216     return AMDGPU::IsaInfo::getMaxWavesPerEU(this, FlatWorkGroupSize);
1217   }
1218 
1219   /// \returns Minimum number of waves per execution unit supported by the
1220   /// subtarget.
1221   unsigned getMinWavesPerEU() const override {
1222     return AMDGPU::IsaInfo::getMinWavesPerEU(this);
1223   }
1224 
1225   void adjustSchedDependency(SUnit *Src, SUnit *Dst, SDep &Dep) const override;
1226 };
1227 
1228 class R600Subtarget final : public R600GenSubtargetInfo,
1229                             public AMDGPUSubtarget {
1230 private:
1231   R600InstrInfo InstrInfo;
1232   R600FrameLowering FrameLowering;
1233   bool FMA;
1234   bool CaymanISA;
1235   bool CFALUBug;
1236   bool HasVertexCache;
1237   bool R600ALUInst;
1238   bool FP64;
1239   short TexVTXClauseSize;
1240   Generation Gen;
1241   R600TargetLowering TLInfo;
1242   InstrItineraryData InstrItins;
1243   SelectionDAGTargetInfo TSInfo;
1244 
1245 public:
1246   R600Subtarget(const Triple &TT, StringRef CPU, StringRef FS,
1247                 const TargetMachine &TM);
1248 
1249   const R600InstrInfo *getInstrInfo() const override { return &InstrInfo; }
1250 
1251   const R600FrameLowering *getFrameLowering() const override {
1252     return &FrameLowering;
1253   }
1254 
1255   const R600TargetLowering *getTargetLowering() const override {
1256     return &TLInfo;
1257   }
1258 
1259   const R600RegisterInfo *getRegisterInfo() const override {
1260     return &InstrInfo.getRegisterInfo();
1261   }
1262 
1263   const InstrItineraryData *getInstrItineraryData() const override {
1264     return &InstrItins;
1265   }
1266 
1267   // Nothing implemented, just prevent crashes on use.
1268   const SelectionDAGTargetInfo *getSelectionDAGInfo() const override {
1269     return &TSInfo;
1270   }
1271 
1272   void ParseSubtargetFeatures(StringRef CPU, StringRef FS);
1273 
1274   Generation getGeneration() const {
1275     return Gen;
1276   }
1277 
1278   Align getStackAlignment() const { return Align(4); }
1279 
1280   R600Subtarget &initializeSubtargetDependencies(const Triple &TT,
1281                                                  StringRef GPU, StringRef FS);
1282 
1283   bool hasBFE() const {
1284     return (getGeneration() >= EVERGREEN);
1285   }
1286 
1287   bool hasBFI() const {
1288     return (getGeneration() >= EVERGREEN);
1289   }
1290 
1291   bool hasBCNT(unsigned Size) const {
1292     if (Size == 32)
1293       return (getGeneration() >= EVERGREEN);
1294 
1295     return false;
1296   }
1297 
1298   bool hasBORROW() const {
1299     return (getGeneration() >= EVERGREEN);
1300   }
1301 
1302   bool hasCARRY() const {
1303     return (getGeneration() >= EVERGREEN);
1304   }
1305 
1306   bool hasCaymanISA() const {
1307     return CaymanISA;
1308   }
1309 
1310   bool hasFFBL() const {
1311     return (getGeneration() >= EVERGREEN);
1312   }
1313 
1314   bool hasFFBH() const {
1315     return (getGeneration() >= EVERGREEN);
1316   }
1317 
1318   bool hasFMA() const { return FMA; }
1319 
1320   bool hasCFAluBug() const { return CFALUBug; }
1321 
1322   bool hasVertexCache() const { return HasVertexCache; }
1323 
1324   short getTexVTXClauseSize() const { return TexVTXClauseSize; }
1325 
1326   bool enableMachineScheduler() const override {
1327     return true;
1328   }
1329 
1330   bool enableSubRegLiveness() const override {
1331     return true;
1332   }
1333 
1334   /// \returns Maximum number of work groups per compute unit supported by the
1335   /// subtarget and limited by given \p FlatWorkGroupSize.
1336   unsigned getMaxWorkGroupsPerCU(unsigned FlatWorkGroupSize) const override {
1337     return AMDGPU::IsaInfo::getMaxWorkGroupsPerCU(this, FlatWorkGroupSize);
1338   }
1339 
1340   /// \returns Minimum flat work group size supported by the subtarget.
1341   unsigned getMinFlatWorkGroupSize() const override {
1342     return AMDGPU::IsaInfo::getMinFlatWorkGroupSize(this);
1343   }
1344 
1345   /// \returns Maximum flat work group size supported by the subtarget.
1346   unsigned getMaxFlatWorkGroupSize() const override {
1347     return AMDGPU::IsaInfo::getMaxFlatWorkGroupSize(this);
1348   }
1349 
1350   /// \returns Maximum number of waves per execution unit supported by the
1351   /// subtarget and limited by given \p FlatWorkGroupSize.
1352   unsigned getMaxWavesPerEU(unsigned FlatWorkGroupSize) const override {
1353     return AMDGPU::IsaInfo::getMaxWavesPerEU(this, FlatWorkGroupSize);
1354   }
1355 
1356   /// \returns Minimum number of waves per execution unit supported by the
1357   /// subtarget.
1358   unsigned getMinWavesPerEU() const override {
1359     return AMDGPU::IsaInfo::getMinWavesPerEU(this);
1360   }
1361 };
1362 
1363 } // end namespace llvm
1364 
1365 #endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUSUBTARGET_H
1366