1 //===- SIInstrInfo.h - SI Instruction Info Interface ------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Interface definition for SIInstrInfo.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #ifndef LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
15 #define LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
16 
17 #include "AMDGPUMIRFormatter.h"
18 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
19 #include "SIRegisterInfo.h"
20 #include "Utils/AMDGPUBaseInfo.h"
21 #include "llvm/ADT/SetVector.h"
22 #include "llvm/CodeGen/TargetInstrInfo.h"
23 #include "llvm/CodeGen/TargetSchedule.h"
24 
25 #define GET_INSTRINFO_HEADER
26 #include "AMDGPUGenInstrInfo.inc"
27 
28 namespace llvm {
29 
30 class APInt;
31 class GCNSubtarget;
32 class LiveVariables;
33 class MachineDominatorTree;
34 class MachineRegisterInfo;
35 class RegScavenger;
36 class TargetRegisterClass;
37 class ScheduleHazardRecognizer;
38 
39 /// Mark the MMO of a uniform load if there are no potentially clobbering stores
40 /// on any path from the start of an entry function to this load.
41 static const MachineMemOperand::Flags MONoClobber =
42     MachineMemOperand::MOTargetFlag1;
43 
44 class SIInstrInfo final : public AMDGPUGenInstrInfo {
45 private:
46   const SIRegisterInfo RI;
47   const GCNSubtarget &ST;
48   TargetSchedModel SchedModel;
49   mutable std::unique_ptr<AMDGPUMIRFormatter> Formatter;
50 
51   // The inverse predicate should have the negative value.
52   enum BranchPredicate {
53     INVALID_BR = 0,
54     SCC_TRUE = 1,
55     SCC_FALSE = -1,
56     VCCNZ = 2,
57     VCCZ = -2,
58     EXECNZ = -3,
59     EXECZ = 3
60   };
61 
62   using SetVectorType = SmallSetVector<MachineInstr *, 32>;
63 
64   static unsigned getBranchOpcode(BranchPredicate Cond);
65   static BranchPredicate getBranchPredicate(unsigned Opcode);
66 
67 public:
68   unsigned buildExtractSubReg(MachineBasicBlock::iterator MI,
69                               MachineRegisterInfo &MRI,
70                               MachineOperand &SuperReg,
71                               const TargetRegisterClass *SuperRC,
72                               unsigned SubIdx,
73                               const TargetRegisterClass *SubRC) const;
74   MachineOperand buildExtractSubRegOrImm(MachineBasicBlock::iterator MI,
75                                          MachineRegisterInfo &MRI,
76                                          MachineOperand &SuperReg,
77                                          const TargetRegisterClass *SuperRC,
78                                          unsigned SubIdx,
79                                          const TargetRegisterClass *SubRC) const;
80 private:
81   void swapOperands(MachineInstr &Inst) const;
82 
83   std::pair<bool, MachineBasicBlock *>
84   moveScalarAddSub(SetVectorType &Worklist, MachineInstr &Inst,
85                    MachineDominatorTree *MDT = nullptr) const;
86 
87   void lowerSelect(SetVectorType &Worklist, MachineInstr &Inst,
88                    MachineDominatorTree *MDT = nullptr) const;
89 
90   void lowerScalarAbs(SetVectorType &Worklist,
91                       MachineInstr &Inst) const;
92 
93   void lowerScalarXnor(SetVectorType &Worklist,
94                        MachineInstr &Inst) const;
95 
96   void splitScalarNotBinop(SetVectorType &Worklist,
97                            MachineInstr &Inst,
98                            unsigned Opcode) const;
99 
100   void splitScalarBinOpN2(SetVectorType &Worklist,
101                           MachineInstr &Inst,
102                           unsigned Opcode) const;
103 
104   void splitScalar64BitUnaryOp(SetVectorType &Worklist,
105                                MachineInstr &Inst, unsigned Opcode,
106                                bool Swap = false) const;
107 
108   void splitScalar64BitAddSub(SetVectorType &Worklist, MachineInstr &Inst,
109                               MachineDominatorTree *MDT = nullptr) const;
110 
111   void splitScalar64BitBinaryOp(SetVectorType &Worklist, MachineInstr &Inst,
112                                 unsigned Opcode,
113                                 MachineDominatorTree *MDT = nullptr) const;
114 
115   void splitScalar64BitXnor(SetVectorType &Worklist, MachineInstr &Inst,
116                                 MachineDominatorTree *MDT = nullptr) const;
117 
118   void splitScalar64BitBCNT(SetVectorType &Worklist,
119                             MachineInstr &Inst) const;
120   void splitScalar64BitBFE(SetVectorType &Worklist,
121                            MachineInstr &Inst) const;
122   void movePackToVALU(SetVectorType &Worklist,
123                       MachineRegisterInfo &MRI,
124                       MachineInstr &Inst) const;
125 
126   void addUsersToMoveToVALUWorklist(Register Reg, MachineRegisterInfo &MRI,
127                                     SetVectorType &Worklist) const;
128 
129   void addSCCDefUsersToVALUWorklist(MachineOperand &Op,
130                                     MachineInstr &SCCDefInst,
131                                     SetVectorType &Worklist,
132                                     Register NewCond = Register()) const;
133   void addSCCDefsToVALUWorklist(MachineOperand &Op,
134                                 SetVectorType &Worklist) const;
135 
136   const TargetRegisterClass *
137   getDestEquivalentVGPRClass(const MachineInstr &Inst) const;
138 
139   bool checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
140                                     const MachineInstr &MIb) const;
141 
142   Register findUsedSGPR(const MachineInstr &MI, int OpIndices[3]) const;
143 
144 protected:
145   bool swapSourceModifiers(MachineInstr &MI,
146                            MachineOperand &Src0, unsigned Src0OpName,
147                            MachineOperand &Src1, unsigned Src1OpName) const;
148 
149   MachineInstr *commuteInstructionImpl(MachineInstr &MI, bool NewMI,
150                                        unsigned OpIdx0,
151                                        unsigned OpIdx1) const override;
152 
153 public:
154   enum TargetOperandFlags {
155     MO_MASK = 0xf,
156 
157     MO_NONE = 0,
158     // MO_GOTPCREL -> symbol@GOTPCREL -> R_AMDGPU_GOTPCREL.
159     MO_GOTPCREL = 1,
160     // MO_GOTPCREL32_LO -> symbol@gotpcrel32@lo -> R_AMDGPU_GOTPCREL32_LO.
161     MO_GOTPCREL32 = 2,
162     MO_GOTPCREL32_LO = 2,
163     // MO_GOTPCREL32_HI -> symbol@gotpcrel32@hi -> R_AMDGPU_GOTPCREL32_HI.
164     MO_GOTPCREL32_HI = 3,
165     // MO_REL32_LO -> symbol@rel32@lo -> R_AMDGPU_REL32_LO.
166     MO_REL32 = 4,
167     MO_REL32_LO = 4,
168     // MO_REL32_HI -> symbol@rel32@hi -> R_AMDGPU_REL32_HI.
169     MO_REL32_HI = 5,
170 
171     MO_FAR_BRANCH_OFFSET = 6,
172 
173     MO_ABS32_LO = 8,
174     MO_ABS32_HI = 9,
175   };
176 
177   explicit SIInstrInfo(const GCNSubtarget &ST);
178 
179   const SIRegisterInfo &getRegisterInfo() const {
180     return RI;
181   }
182 
183   const GCNSubtarget &getSubtarget() const {
184     return ST;
185   }
186 
187   bool isReallyTriviallyReMaterializable(const MachineInstr &MI,
188                                          AAResults *AA) const override;
189 
190   bool isIgnorableUse(const MachineOperand &MO) const override;
191 
192   bool areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2,
193                                int64_t &Offset1,
194                                int64_t &Offset2) const override;
195 
196   bool getMemOperandsWithOffsetWidth(
197       const MachineInstr &LdSt,
198       SmallVectorImpl<const MachineOperand *> &BaseOps, int64_t &Offset,
199       bool &OffsetIsScalable, unsigned &Width,
200       const TargetRegisterInfo *TRI) const final;
201 
202   bool shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
203                            ArrayRef<const MachineOperand *> BaseOps2,
204                            unsigned NumLoads, unsigned NumBytes) const override;
205 
206   bool shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1, int64_t Offset0,
207                                int64_t Offset1, unsigned NumLoads) const override;
208 
209   void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
210                    const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg,
211                    bool KillSrc) const override;
212 
213   void materializeImmediate(MachineBasicBlock &MBB,
214                             MachineBasicBlock::iterator MI,
215                             const DebugLoc &DL,
216                             unsigned DestReg,
217                             int64_t Value) const;
218 
219   const TargetRegisterClass *getPreferredSelectRegClass(
220                                unsigned Size) const;
221 
222   Register insertNE(MachineBasicBlock *MBB,
223                     MachineBasicBlock::iterator I, const DebugLoc &DL,
224                     Register SrcReg, int Value) const;
225 
226   Register insertEQ(MachineBasicBlock *MBB,
227                     MachineBasicBlock::iterator I, const DebugLoc &DL,
228                     Register SrcReg, int Value)  const;
229 
230   void storeRegToStackSlot(MachineBasicBlock &MBB,
231                            MachineBasicBlock::iterator MI, Register SrcReg,
232                            bool isKill, int FrameIndex,
233                            const TargetRegisterClass *RC,
234                            const TargetRegisterInfo *TRI) const override;
235 
236   void loadRegFromStackSlot(MachineBasicBlock &MBB,
237                             MachineBasicBlock::iterator MI, Register DestReg,
238                             int FrameIndex, const TargetRegisterClass *RC,
239                             const TargetRegisterInfo *TRI) const override;
240 
241   bool expandPostRAPseudo(MachineInstr &MI) const override;
242 
243   // Splits a V_MOV_B64_DPP_PSEUDO opcode into a pair of v_mov_b32_dpp
244   // instructions. Returns a pair of generated instructions.
245   // Can split either post-RA with physical registers or pre-RA with
246   // virtual registers. In latter case IR needs to be in SSA form and
247   // and a REG_SEQUENCE is produced to define original register.
248   std::pair<MachineInstr*, MachineInstr*>
249   expandMovDPP64(MachineInstr &MI) const;
250 
251   // Returns an opcode that can be used to move a value to a \p DstRC
252   // register.  If there is no hardware instruction that can store to \p
253   // DstRC, then AMDGPU::COPY is returned.
254   unsigned getMovOpcode(const TargetRegisterClass *DstRC) const;
255 
256   const MCInstrDesc &getIndirectRegWriteMovRelPseudo(unsigned VecSize,
257                                                      unsigned EltSize,
258                                                      bool IsSGPR) const;
259 
260   const MCInstrDesc &getIndirectGPRIDXPseudo(unsigned VecSize,
261                                              bool IsIndirectSrc) const;
262   LLVM_READONLY
263   int commuteOpcode(unsigned Opc) const;
264 
265   LLVM_READONLY
266   inline int commuteOpcode(const MachineInstr &MI) const {
267     return commuteOpcode(MI.getOpcode());
268   }
269 
270   bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1,
271                              unsigned &SrcOpIdx2) const override;
272 
273   bool findCommutedOpIndices(MCInstrDesc Desc, unsigned & SrcOpIdx0,
274    unsigned & SrcOpIdx1) const;
275 
276   bool isBranchOffsetInRange(unsigned BranchOpc,
277                              int64_t BrOffset) const override;
278 
279   MachineBasicBlock *getBranchDestBlock(const MachineInstr &MI) const override;
280 
281   void insertIndirectBranch(MachineBasicBlock &MBB,
282                             MachineBasicBlock &NewDestBB,
283                             MachineBasicBlock &RestoreBB, const DebugLoc &DL,
284                             int64_t BrOffset, RegScavenger *RS) const override;
285 
286   bool analyzeBranchImpl(MachineBasicBlock &MBB,
287                          MachineBasicBlock::iterator I,
288                          MachineBasicBlock *&TBB,
289                          MachineBasicBlock *&FBB,
290                          SmallVectorImpl<MachineOperand> &Cond,
291                          bool AllowModify) const;
292 
293   bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
294                      MachineBasicBlock *&FBB,
295                      SmallVectorImpl<MachineOperand> &Cond,
296                      bool AllowModify = false) const override;
297 
298   unsigned removeBranch(MachineBasicBlock &MBB,
299                         int *BytesRemoved = nullptr) const override;
300 
301   unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
302                         MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
303                         const DebugLoc &DL,
304                         int *BytesAdded = nullptr) const override;
305 
306   bool reverseBranchCondition(
307     SmallVectorImpl<MachineOperand> &Cond) const override;
308 
309   bool canInsertSelect(const MachineBasicBlock &MBB,
310                        ArrayRef<MachineOperand> Cond, Register DstReg,
311                        Register TrueReg, Register FalseReg, int &CondCycles,
312                        int &TrueCycles, int &FalseCycles) const override;
313 
314   void insertSelect(MachineBasicBlock &MBB,
315                     MachineBasicBlock::iterator I, const DebugLoc &DL,
316                     Register DstReg, ArrayRef<MachineOperand> Cond,
317                     Register TrueReg, Register FalseReg) const override;
318 
319   void insertVectorSelect(MachineBasicBlock &MBB,
320                           MachineBasicBlock::iterator I, const DebugLoc &DL,
321                           Register DstReg, ArrayRef<MachineOperand> Cond,
322                           Register TrueReg, Register FalseReg) const;
323 
324   bool analyzeCompare(const MachineInstr &MI, Register &SrcReg,
325                       Register &SrcReg2, int64_t &CmpMask,
326                       int64_t &CmpValue) const override;
327 
328   bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
329                             Register SrcReg2, int64_t CmpMask, int64_t CmpValue,
330                             const MachineRegisterInfo *MRI) const override;
331 
332   bool
333   areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
334                                   const MachineInstr &MIb) const override;
335 
336   static bool isFoldableCopy(const MachineInstr &MI);
337 
338   void removeModOperands(MachineInstr &MI) const;
339 
340   bool FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg,
341                      MachineRegisterInfo *MRI) const final;
342 
343   unsigned getMachineCSELookAheadLimit() const override { return 500; }
344 
345   MachineInstr *convertToThreeAddress(MachineInstr &MI, LiveVariables *LV,
346                                       LiveIntervals *LIS) const override;
347 
348   bool isSchedulingBoundary(const MachineInstr &MI,
349                             const MachineBasicBlock *MBB,
350                             const MachineFunction &MF) const override;
351 
352   static bool isSALU(const MachineInstr &MI) {
353     return MI.getDesc().TSFlags & SIInstrFlags::SALU;
354   }
355 
356   bool isSALU(uint16_t Opcode) const {
357     return get(Opcode).TSFlags & SIInstrFlags::SALU;
358   }
359 
360   static bool isVALU(const MachineInstr &MI) {
361     return MI.getDesc().TSFlags & SIInstrFlags::VALU;
362   }
363 
364   bool isVALU(uint16_t Opcode) const {
365     return get(Opcode).TSFlags & SIInstrFlags::VALU;
366   }
367 
368   static bool isVMEM(const MachineInstr &MI) {
369     return isMUBUF(MI) || isMTBUF(MI) || isMIMG(MI);
370   }
371 
372   bool isVMEM(uint16_t Opcode) const {
373     return isMUBUF(Opcode) || isMTBUF(Opcode) || isMIMG(Opcode);
374   }
375 
376   static bool isSOP1(const MachineInstr &MI) {
377     return MI.getDesc().TSFlags & SIInstrFlags::SOP1;
378   }
379 
380   bool isSOP1(uint16_t Opcode) const {
381     return get(Opcode).TSFlags & SIInstrFlags::SOP1;
382   }
383 
384   static bool isSOP2(const MachineInstr &MI) {
385     return MI.getDesc().TSFlags & SIInstrFlags::SOP2;
386   }
387 
388   bool isSOP2(uint16_t Opcode) const {
389     return get(Opcode).TSFlags & SIInstrFlags::SOP2;
390   }
391 
392   static bool isSOPC(const MachineInstr &MI) {
393     return MI.getDesc().TSFlags & SIInstrFlags::SOPC;
394   }
395 
396   bool isSOPC(uint16_t Opcode) const {
397     return get(Opcode).TSFlags & SIInstrFlags::SOPC;
398   }
399 
400   static bool isSOPK(const MachineInstr &MI) {
401     return MI.getDesc().TSFlags & SIInstrFlags::SOPK;
402   }
403 
404   bool isSOPK(uint16_t Opcode) const {
405     return get(Opcode).TSFlags & SIInstrFlags::SOPK;
406   }
407 
408   static bool isSOPP(const MachineInstr &MI) {
409     return MI.getDesc().TSFlags & SIInstrFlags::SOPP;
410   }
411 
412   bool isSOPP(uint16_t Opcode) const {
413     return get(Opcode).TSFlags & SIInstrFlags::SOPP;
414   }
415 
416   static bool isPacked(const MachineInstr &MI) {
417     return MI.getDesc().TSFlags & SIInstrFlags::IsPacked;
418   }
419 
420   bool isPacked(uint16_t Opcode) const {
421     return get(Opcode).TSFlags & SIInstrFlags::IsPacked;
422   }
423 
424   static bool isVOP1(const MachineInstr &MI) {
425     return MI.getDesc().TSFlags & SIInstrFlags::VOP1;
426   }
427 
428   bool isVOP1(uint16_t Opcode) const {
429     return get(Opcode).TSFlags & SIInstrFlags::VOP1;
430   }
431 
432   static bool isVOP2(const MachineInstr &MI) {
433     return MI.getDesc().TSFlags & SIInstrFlags::VOP2;
434   }
435 
436   bool isVOP2(uint16_t Opcode) const {
437     return get(Opcode).TSFlags & SIInstrFlags::VOP2;
438   }
439 
440   static bool isVOP3(const MachineInstr &MI) {
441     return MI.getDesc().TSFlags & SIInstrFlags::VOP3;
442   }
443 
444   bool isVOP3(uint16_t Opcode) const {
445     return get(Opcode).TSFlags & SIInstrFlags::VOP3;
446   }
447 
448   static bool isSDWA(const MachineInstr &MI) {
449     return MI.getDesc().TSFlags & SIInstrFlags::SDWA;
450   }
451 
452   bool isSDWA(uint16_t Opcode) const {
453     return get(Opcode).TSFlags & SIInstrFlags::SDWA;
454   }
455 
456   static bool isVOPC(const MachineInstr &MI) {
457     return MI.getDesc().TSFlags & SIInstrFlags::VOPC;
458   }
459 
460   bool isVOPC(uint16_t Opcode) const {
461     return get(Opcode).TSFlags & SIInstrFlags::VOPC;
462   }
463 
464   static bool isMUBUF(const MachineInstr &MI) {
465     return MI.getDesc().TSFlags & SIInstrFlags::MUBUF;
466   }
467 
468   bool isMUBUF(uint16_t Opcode) const {
469     return get(Opcode).TSFlags & SIInstrFlags::MUBUF;
470   }
471 
472   static bool isMTBUF(const MachineInstr &MI) {
473     return MI.getDesc().TSFlags & SIInstrFlags::MTBUF;
474   }
475 
476   bool isMTBUF(uint16_t Opcode) const {
477     return get(Opcode).TSFlags & SIInstrFlags::MTBUF;
478   }
479 
480   static bool isSMRD(const MachineInstr &MI) {
481     return MI.getDesc().TSFlags & SIInstrFlags::SMRD;
482   }
483 
484   bool isSMRD(uint16_t Opcode) const {
485     return get(Opcode).TSFlags & SIInstrFlags::SMRD;
486   }
487 
488   bool isBufferSMRD(const MachineInstr &MI) const;
489 
490   static bool isDS(const MachineInstr &MI) {
491     return MI.getDesc().TSFlags & SIInstrFlags::DS;
492   }
493 
494   bool isDS(uint16_t Opcode) const {
495     return get(Opcode).TSFlags & SIInstrFlags::DS;
496   }
497 
498   bool isAlwaysGDS(uint16_t Opcode) const;
499 
500   static bool isMIMG(const MachineInstr &MI) {
501     return MI.getDesc().TSFlags & SIInstrFlags::MIMG;
502   }
503 
504   bool isMIMG(uint16_t Opcode) const {
505     return get(Opcode).TSFlags & SIInstrFlags::MIMG;
506   }
507 
508   static bool isGather4(const MachineInstr &MI) {
509     return MI.getDesc().TSFlags & SIInstrFlags::Gather4;
510   }
511 
512   bool isGather4(uint16_t Opcode) const {
513     return get(Opcode).TSFlags & SIInstrFlags::Gather4;
514   }
515 
516   static bool isFLAT(const MachineInstr &MI) {
517     return MI.getDesc().TSFlags & SIInstrFlags::FLAT;
518   }
519 
520   // Is a FLAT encoded instruction which accesses a specific segment,
521   // i.e. global_* or scratch_*.
522   static bool isSegmentSpecificFLAT(const MachineInstr &MI) {
523     auto Flags = MI.getDesc().TSFlags;
524     return Flags & (SIInstrFlags::FlatGlobal | SIInstrFlags::FlatScratch);
525   }
526 
527   bool isSegmentSpecificFLAT(uint16_t Opcode) const {
528     auto Flags = get(Opcode).TSFlags;
529     return Flags & (SIInstrFlags::FlatGlobal | SIInstrFlags::FlatScratch);
530   }
531 
532   static bool isFLATGlobal(const MachineInstr &MI) {
533     return MI.getDesc().TSFlags & SIInstrFlags::FlatGlobal;
534   }
535 
536   bool isFLATGlobal(uint16_t Opcode) const {
537     return get(Opcode).TSFlags & SIInstrFlags::FlatGlobal;
538   }
539 
540   static bool isFLATScratch(const MachineInstr &MI) {
541     return MI.getDesc().TSFlags & SIInstrFlags::FlatScratch;
542   }
543 
544   bool isFLATScratch(uint16_t Opcode) const {
545     return get(Opcode).TSFlags & SIInstrFlags::FlatScratch;
546   }
547 
548   // Any FLAT encoded instruction, including global_* and scratch_*.
549   bool isFLAT(uint16_t Opcode) const {
550     return get(Opcode).TSFlags & SIInstrFlags::FLAT;
551   }
552 
553   static bool isEXP(const MachineInstr &MI) {
554     return MI.getDesc().TSFlags & SIInstrFlags::EXP;
555   }
556 
557   static bool isDualSourceBlendEXP(const MachineInstr &MI) {
558     if (!isEXP(MI))
559       return false;
560     unsigned Target = MI.getOperand(0).getImm();
561     return Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND0 ||
562            Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND1;
563   }
564 
565   bool isEXP(uint16_t Opcode) const {
566     return get(Opcode).TSFlags & SIInstrFlags::EXP;
567   }
568 
569   static bool isAtomicNoRet(const MachineInstr &MI) {
570     return MI.getDesc().TSFlags & SIInstrFlags::IsAtomicNoRet;
571   }
572 
573   bool isAtomicNoRet(uint16_t Opcode) const {
574     return get(Opcode).TSFlags & SIInstrFlags::IsAtomicNoRet;
575   }
576 
577   static bool isAtomicRet(const MachineInstr &MI) {
578     return MI.getDesc().TSFlags & SIInstrFlags::IsAtomicRet;
579   }
580 
581   bool isAtomicRet(uint16_t Opcode) const {
582     return get(Opcode).TSFlags & SIInstrFlags::IsAtomicRet;
583   }
584 
585   static bool isAtomic(const MachineInstr &MI) {
586     return MI.getDesc().TSFlags & (SIInstrFlags::IsAtomicRet |
587                                    SIInstrFlags::IsAtomicNoRet);
588   }
589 
590   bool isAtomic(uint16_t Opcode) const {
591     return get(Opcode).TSFlags & (SIInstrFlags::IsAtomicRet |
592                                   SIInstrFlags::IsAtomicNoRet);
593   }
594 
595   static bool isWQM(const MachineInstr &MI) {
596     return MI.getDesc().TSFlags & SIInstrFlags::WQM;
597   }
598 
599   bool isWQM(uint16_t Opcode) const {
600     return get(Opcode).TSFlags & SIInstrFlags::WQM;
601   }
602 
603   static bool isDisableWQM(const MachineInstr &MI) {
604     return MI.getDesc().TSFlags & SIInstrFlags::DisableWQM;
605   }
606 
607   bool isDisableWQM(uint16_t Opcode) const {
608     return get(Opcode).TSFlags & SIInstrFlags::DisableWQM;
609   }
610 
611   static bool isVGPRSpill(const MachineInstr &MI) {
612     return MI.getDesc().TSFlags & SIInstrFlags::VGPRSpill;
613   }
614 
615   bool isVGPRSpill(uint16_t Opcode) const {
616     return get(Opcode).TSFlags & SIInstrFlags::VGPRSpill;
617   }
618 
619   static bool isSGPRSpill(const MachineInstr &MI) {
620     return MI.getDesc().TSFlags & SIInstrFlags::SGPRSpill;
621   }
622 
623   bool isSGPRSpill(uint16_t Opcode) const {
624     return get(Opcode).TSFlags & SIInstrFlags::SGPRSpill;
625   }
626 
627   static bool isDPP(const MachineInstr &MI) {
628     return MI.getDesc().TSFlags & SIInstrFlags::DPP;
629   }
630 
631   bool isDPP(uint16_t Opcode) const {
632     return get(Opcode).TSFlags & SIInstrFlags::DPP;
633   }
634 
635   static bool isTRANS(const MachineInstr &MI) {
636     return MI.getDesc().TSFlags & SIInstrFlags::TRANS;
637   }
638 
639   bool isTRANS(uint16_t Opcode) const {
640     return get(Opcode).TSFlags & SIInstrFlags::TRANS;
641   }
642 
643   static bool isVOP3P(const MachineInstr &MI) {
644     return MI.getDesc().TSFlags & SIInstrFlags::VOP3P;
645   }
646 
647   bool isVOP3P(uint16_t Opcode) const {
648     return get(Opcode).TSFlags & SIInstrFlags::VOP3P;
649   }
650 
651   static bool isVINTRP(const MachineInstr &MI) {
652     return MI.getDesc().TSFlags & SIInstrFlags::VINTRP;
653   }
654 
655   bool isVINTRP(uint16_t Opcode) const {
656     return get(Opcode).TSFlags & SIInstrFlags::VINTRP;
657   }
658 
659   static bool isMAI(const MachineInstr &MI) {
660     return MI.getDesc().TSFlags & SIInstrFlags::IsMAI;
661   }
662 
663   bool isMAI(uint16_t Opcode) const {
664     return get(Opcode).TSFlags & SIInstrFlags::IsMAI;
665   }
666 
667   static bool isMFMA(const MachineInstr &MI) {
668     return isMAI(MI) && MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
669            MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64;
670   }
671 
672   static bool isDOT(const MachineInstr &MI) {
673     return MI.getDesc().TSFlags & SIInstrFlags::IsDOT;
674   }
675 
676   static bool isWMMA(const MachineInstr &MI) {
677     return MI.getDesc().TSFlags & SIInstrFlags::IsWMMA;
678   }
679 
680   bool isWMMA(uint16_t Opcode) const {
681     return get(Opcode).TSFlags & SIInstrFlags::IsWMMA;
682   }
683 
684   bool isDOT(uint16_t Opcode) const {
685     return get(Opcode).TSFlags & SIInstrFlags::IsDOT;
686   }
687 
688   static bool isLDSDIR(const MachineInstr &MI) {
689     return MI.getDesc().TSFlags & SIInstrFlags::LDSDIR;
690   }
691 
692   bool isLDSDIR(uint16_t Opcode) const {
693     return get(Opcode).TSFlags & SIInstrFlags::LDSDIR;
694   }
695 
696   static bool isVINTERP(const MachineInstr &MI) {
697     return MI.getDesc().TSFlags & SIInstrFlags::VINTERP;
698   }
699 
700   bool isVINTERP(uint16_t Opcode) const {
701     return get(Opcode).TSFlags & SIInstrFlags::VINTERP;
702   }
703 
704   static bool isScalarUnit(const MachineInstr &MI) {
705     return MI.getDesc().TSFlags & (SIInstrFlags::SALU | SIInstrFlags::SMRD);
706   }
707 
708   static bool usesVM_CNT(const MachineInstr &MI) {
709     return MI.getDesc().TSFlags & SIInstrFlags::VM_CNT;
710   }
711 
712   static bool usesLGKM_CNT(const MachineInstr &MI) {
713     return MI.getDesc().TSFlags & SIInstrFlags::LGKM_CNT;
714   }
715 
716   static bool sopkIsZext(const MachineInstr &MI) {
717     return MI.getDesc().TSFlags & SIInstrFlags::SOPK_ZEXT;
718   }
719 
720   bool sopkIsZext(uint16_t Opcode) const {
721     return get(Opcode).TSFlags & SIInstrFlags::SOPK_ZEXT;
722   }
723 
724   /// \returns true if this is an s_store_dword* instruction. This is more
725   /// specific than than isSMEM && mayStore.
726   static bool isScalarStore(const MachineInstr &MI) {
727     return MI.getDesc().TSFlags & SIInstrFlags::SCALAR_STORE;
728   }
729 
730   bool isScalarStore(uint16_t Opcode) const {
731     return get(Opcode).TSFlags & SIInstrFlags::SCALAR_STORE;
732   }
733 
734   static bool isFixedSize(const MachineInstr &MI) {
735     return MI.getDesc().TSFlags & SIInstrFlags::FIXED_SIZE;
736   }
737 
738   bool isFixedSize(uint16_t Opcode) const {
739     return get(Opcode).TSFlags & SIInstrFlags::FIXED_SIZE;
740   }
741 
742   static bool hasFPClamp(const MachineInstr &MI) {
743     return MI.getDesc().TSFlags & SIInstrFlags::FPClamp;
744   }
745 
746   bool hasFPClamp(uint16_t Opcode) const {
747     return get(Opcode).TSFlags & SIInstrFlags::FPClamp;
748   }
749 
750   static bool hasIntClamp(const MachineInstr &MI) {
751     return MI.getDesc().TSFlags & SIInstrFlags::IntClamp;
752   }
753 
754   uint64_t getClampMask(const MachineInstr &MI) const {
755     const uint64_t ClampFlags = SIInstrFlags::FPClamp |
756                                 SIInstrFlags::IntClamp |
757                                 SIInstrFlags::ClampLo |
758                                 SIInstrFlags::ClampHi;
759       return MI.getDesc().TSFlags & ClampFlags;
760   }
761 
762   static bool usesFPDPRounding(const MachineInstr &MI) {
763     return MI.getDesc().TSFlags & SIInstrFlags::FPDPRounding;
764   }
765 
766   bool usesFPDPRounding(uint16_t Opcode) const {
767     return get(Opcode).TSFlags & SIInstrFlags::FPDPRounding;
768   }
769 
770   static bool isFPAtomic(const MachineInstr &MI) {
771     return MI.getDesc().TSFlags & SIInstrFlags::FPAtomic;
772   }
773 
774   bool isFPAtomic(uint16_t Opcode) const {
775     return get(Opcode).TSFlags & SIInstrFlags::FPAtomic;
776   }
777 
778   bool isVGPRCopy(const MachineInstr &MI) const {
779     assert(MI.isCopy());
780     Register Dest = MI.getOperand(0).getReg();
781     const MachineFunction &MF = *MI.getParent()->getParent();
782     const MachineRegisterInfo &MRI = MF.getRegInfo();
783     return !RI.isSGPRReg(MRI, Dest);
784   }
785 
786   bool hasVGPRUses(const MachineInstr &MI) const {
787     const MachineFunction &MF = *MI.getParent()->getParent();
788     const MachineRegisterInfo &MRI = MF.getRegInfo();
789     return llvm::any_of(MI.explicit_uses(),
790                         [&MRI, this](const MachineOperand &MO) {
791       return MO.isReg() && RI.isVGPR(MRI, MO.getReg());});
792   }
793 
794   /// Return true if the instruction modifies the mode register.q
795   static bool modifiesModeRegister(const MachineInstr &MI);
796 
797   /// Whether we must prevent this instruction from executing with EXEC = 0.
798   bool hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const;
799 
800   /// Returns true if the instruction could potentially depend on the value of
801   /// exec. If false, exec dependencies may safely be ignored.
802   bool mayReadEXEC(const MachineRegisterInfo &MRI, const MachineInstr &MI) const;
803 
804   bool isInlineConstant(const APInt &Imm) const;
805 
806   bool isInlineConstant(const APFloat &Imm) const {
807     return isInlineConstant(Imm.bitcastToAPInt());
808   }
809 
810   bool isInlineConstant(const MachineOperand &MO, uint8_t OperandType) const;
811 
812   bool isInlineConstant(const MachineOperand &MO,
813                         const MCOperandInfo &OpInfo) const {
814     return isInlineConstant(MO, OpInfo.OperandType);
815   }
816 
817   /// \p returns true if \p UseMO is substituted with \p DefMO in \p MI it would
818   /// be an inline immediate.
819   bool isInlineConstant(const MachineInstr &MI,
820                         const MachineOperand &UseMO,
821                         const MachineOperand &DefMO) const {
822     assert(UseMO.getParent() == &MI);
823     int OpIdx = MI.getOperandNo(&UseMO);
824     if (!MI.getDesc().OpInfo || OpIdx >= MI.getDesc().NumOperands) {
825       return false;
826     }
827 
828     return isInlineConstant(DefMO, MI.getDesc().OpInfo[OpIdx]);
829   }
830 
831   /// \p returns true if the operand \p OpIdx in \p MI is a valid inline
832   /// immediate.
833   bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx) const {
834     const MachineOperand &MO = MI.getOperand(OpIdx);
835     return isInlineConstant(MO, MI.getDesc().OpInfo[OpIdx].OperandType);
836   }
837 
838   bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx,
839                         const MachineOperand &MO) const {
840     if (!MI.getDesc().OpInfo || OpIdx >= MI.getDesc().NumOperands)
841       return false;
842 
843     if (MI.isCopy()) {
844       unsigned Size = getOpSize(MI, OpIdx);
845       assert(Size == 8 || Size == 4);
846 
847       uint8_t OpType = (Size == 8) ?
848         AMDGPU::OPERAND_REG_IMM_INT64 : AMDGPU::OPERAND_REG_IMM_INT32;
849       return isInlineConstant(MO, OpType);
850     }
851 
852     return isInlineConstant(MO, MI.getDesc().OpInfo[OpIdx].OperandType);
853   }
854 
855   bool isInlineConstant(const MachineOperand &MO) const {
856     const MachineInstr *Parent = MO.getParent();
857     return isInlineConstant(*Parent, Parent->getOperandNo(&MO));
858   }
859 
860   bool isLiteralConstant(const MachineOperand &MO,
861                          const MCOperandInfo &OpInfo) const {
862     return MO.isImm() && !isInlineConstant(MO, OpInfo.OperandType);
863   }
864 
865   bool isLiteralConstant(const MachineInstr &MI, int OpIdx) const {
866     const MachineOperand &MO = MI.getOperand(OpIdx);
867     return MO.isImm() && !isInlineConstant(MI, OpIdx);
868   }
869 
870   // Returns true if this operand could potentially require a 32-bit literal
871   // operand, but not necessarily. A FrameIndex for example could resolve to an
872   // inline immediate value that will not require an additional 4-bytes; this
873   // assumes that it will.
874   bool isLiteralConstantLike(const MachineOperand &MO,
875                              const MCOperandInfo &OpInfo) const;
876 
877   bool isImmOperandLegal(const MachineInstr &MI, unsigned OpNo,
878                          const MachineOperand &MO) const;
879 
880   /// Return true if this 64-bit VALU instruction has a 32-bit encoding.
881   /// This function will return false if you pass it a 32-bit instruction.
882   bool hasVALU32BitEncoding(unsigned Opcode) const;
883 
884   /// Returns true if this operand uses the constant bus.
885   bool usesConstantBus(const MachineRegisterInfo &MRI,
886                        const MachineOperand &MO,
887                        const MCOperandInfo &OpInfo) const;
888 
889   /// Return true if this instruction has any modifiers.
890   ///  e.g. src[012]_mod, omod, clamp.
891   bool hasModifiers(unsigned Opcode) const;
892 
893   bool hasModifiersSet(const MachineInstr &MI,
894                        unsigned OpName) const;
895   bool hasAnyModifiersSet(const MachineInstr &MI) const;
896 
897   bool canShrink(const MachineInstr &MI,
898                  const MachineRegisterInfo &MRI) const;
899 
900   MachineInstr *buildShrunkInst(MachineInstr &MI,
901                                 unsigned NewOpcode) const;
902 
903   bool verifyInstruction(const MachineInstr &MI,
904                          StringRef &ErrInfo) const override;
905 
906   unsigned getVALUOp(const MachineInstr &MI) const;
907 
908   /// Return the correct register class for \p OpNo.  For target-specific
909   /// instructions, this will return the register class that has been defined
910   /// in tablegen.  For generic instructions, like REG_SEQUENCE it will return
911   /// the register class of its machine operand.
912   /// to infer the correct register class base on the other operands.
913   const TargetRegisterClass *getOpRegClass(const MachineInstr &MI,
914                                            unsigned OpNo) const;
915 
916   /// Return the size in bytes of the operand OpNo on the given
917   // instruction opcode.
918   unsigned getOpSize(uint16_t Opcode, unsigned OpNo) const {
919     const MCOperandInfo &OpInfo = get(Opcode).OpInfo[OpNo];
920 
921     if (OpInfo.RegClass == -1) {
922       // If this is an immediate operand, this must be a 32-bit literal.
923       assert(OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE);
924       return 4;
925     }
926 
927     return RI.getRegSizeInBits(*RI.getRegClass(OpInfo.RegClass)) / 8;
928   }
929 
930   /// This form should usually be preferred since it handles operands
931   /// with unknown register classes.
932   unsigned getOpSize(const MachineInstr &MI, unsigned OpNo) const {
933     const MachineOperand &MO = MI.getOperand(OpNo);
934     if (MO.isReg()) {
935       if (unsigned SubReg = MO.getSubReg()) {
936         return RI.getSubRegIdxSize(SubReg) / 8;
937       }
938     }
939     return RI.getRegSizeInBits(*getOpRegClass(MI, OpNo)) / 8;
940   }
941 
942   /// Legalize the \p OpIndex operand of this instruction by inserting
943   /// a MOV.  For example:
944   /// ADD_I32_e32 VGPR0, 15
945   /// to
946   /// MOV VGPR1, 15
947   /// ADD_I32_e32 VGPR0, VGPR1
948   ///
949   /// If the operand being legalized is a register, then a COPY will be used
950   /// instead of MOV.
951   void legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const;
952 
953   /// Check if \p MO is a legal operand if it was the \p OpIdx Operand
954   /// for \p MI.
955   bool isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
956                       const MachineOperand *MO = nullptr) const;
957 
958   /// Check if \p MO would be a valid operand for the given operand
959   /// definition \p OpInfo. Note this does not attempt to validate constant bus
960   /// restrictions (e.g. literal constant usage).
961   bool isLegalVSrcOperand(const MachineRegisterInfo &MRI,
962                           const MCOperandInfo &OpInfo,
963                           const MachineOperand &MO) const;
964 
965   /// Check if \p MO (a register operand) is a legal register for the
966   /// given operand description.
967   bool isLegalRegOperand(const MachineRegisterInfo &MRI,
968                          const MCOperandInfo &OpInfo,
969                          const MachineOperand &MO) const;
970 
971   /// Legalize operands in \p MI by either commuting it or inserting a
972   /// copy of src1.
973   void legalizeOperandsVOP2(MachineRegisterInfo &MRI, MachineInstr &MI) const;
974 
975   /// Fix operands in \p MI to satisfy constant bus requirements.
976   void legalizeOperandsVOP3(MachineRegisterInfo &MRI, MachineInstr &MI) const;
977 
978   /// Copy a value from a VGPR (\p SrcReg) to SGPR.  This function can only
979   /// be used when it is know that the value in SrcReg is same across all
980   /// threads in the wave.
981   /// \returns The SGPR register that \p SrcReg was copied to.
982   Register readlaneVGPRToSGPR(Register SrcReg, MachineInstr &UseMI,
983                               MachineRegisterInfo &MRI) const;
984 
985   void legalizeOperandsSMRD(MachineRegisterInfo &MRI, MachineInstr &MI) const;
986   void legalizeOperandsFLAT(MachineRegisterInfo &MRI, MachineInstr &MI) const;
987 
988   void legalizeGenericOperand(MachineBasicBlock &InsertMBB,
989                               MachineBasicBlock::iterator I,
990                               const TargetRegisterClass *DstRC,
991                               MachineOperand &Op, MachineRegisterInfo &MRI,
992                               const DebugLoc &DL) const;
993 
994   /// Legalize all operands in this instruction.  This function may create new
995   /// instructions and control-flow around \p MI.  If present, \p MDT is
996   /// updated.
997   /// \returns A new basic block that contains \p MI if new blocks were created.
998   MachineBasicBlock *
999   legalizeOperands(MachineInstr &MI, MachineDominatorTree *MDT = nullptr) const;
1000 
1001   /// Change SADDR form of a FLAT \p Inst to its VADDR form if saddr operand
1002   /// was moved to VGPR. \returns true if succeeded.
1003   bool moveFlatAddrToVGPR(MachineInstr &Inst) const;
1004 
1005   /// Replace this instruction's opcode with the equivalent VALU
1006   /// opcode.  This function will also move the users of \p MI to the
1007   /// VALU if necessary. If present, \p MDT is updated.
1008   MachineBasicBlock *moveToVALU(MachineInstr &MI,
1009                                 MachineDominatorTree *MDT = nullptr) const;
1010 
1011   void insertNoop(MachineBasicBlock &MBB,
1012                   MachineBasicBlock::iterator MI) const override;
1013 
1014   void insertNoops(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
1015                    unsigned Quantity) const override;
1016 
1017   void insertReturn(MachineBasicBlock &MBB) const;
1018   /// Return the number of wait states that result from executing this
1019   /// instruction.
1020   static unsigned getNumWaitStates(const MachineInstr &MI);
1021 
1022   /// Returns the operand named \p Op.  If \p MI does not have an
1023   /// operand named \c Op, this function returns nullptr.
1024   LLVM_READONLY
1025   MachineOperand *getNamedOperand(MachineInstr &MI, unsigned OperandName) const;
1026 
1027   LLVM_READONLY
1028   const MachineOperand *getNamedOperand(const MachineInstr &MI,
1029                                         unsigned OpName) const {
1030     return getNamedOperand(const_cast<MachineInstr &>(MI), OpName);
1031   }
1032 
1033   /// Get required immediate operand
1034   int64_t getNamedImmOperand(const MachineInstr &MI, unsigned OpName) const {
1035     int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpName);
1036     return MI.getOperand(Idx).getImm();
1037   }
1038 
1039   uint64_t getDefaultRsrcDataFormat() const;
1040   uint64_t getScratchRsrcWords23() const;
1041 
1042   bool isLowLatencyInstruction(const MachineInstr &MI) const;
1043   bool isHighLatencyDef(int Opc) const override;
1044 
1045   /// Return the descriptor of the target-specific machine instruction
1046   /// that corresponds to the specified pseudo or native opcode.
1047   const MCInstrDesc &getMCOpcodeFromPseudo(unsigned Opcode) const {
1048     return get(pseudoToMCOpcode(Opcode));
1049   }
1050 
1051   unsigned isStackAccess(const MachineInstr &MI, int &FrameIndex) const;
1052   unsigned isSGPRStackAccess(const MachineInstr &MI, int &FrameIndex) const;
1053 
1054   unsigned isLoadFromStackSlot(const MachineInstr &MI,
1055                                int &FrameIndex) const override;
1056   unsigned isStoreToStackSlot(const MachineInstr &MI,
1057                               int &FrameIndex) const override;
1058 
1059   unsigned getInstBundleSize(const MachineInstr &MI) const;
1060   unsigned getInstSizeInBytes(const MachineInstr &MI) const override;
1061 
1062   bool mayAccessFlatAddressSpace(const MachineInstr &MI) const;
1063 
1064   bool isNonUniformBranchInstr(MachineInstr &Instr) const;
1065 
1066   void convertNonUniformIfRegion(MachineBasicBlock *IfEntry,
1067                                  MachineBasicBlock *IfEnd) const;
1068 
1069   void convertNonUniformLoopRegion(MachineBasicBlock *LoopEntry,
1070                                    MachineBasicBlock *LoopEnd) const;
1071 
1072   std::pair<unsigned, unsigned>
1073   decomposeMachineOperandsTargetFlags(unsigned TF) const override;
1074 
1075   ArrayRef<std::pair<int, const char *>>
1076   getSerializableTargetIndices() const override;
1077 
1078   ArrayRef<std::pair<unsigned, const char *>>
1079   getSerializableDirectMachineOperandTargetFlags() const override;
1080 
1081   ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
1082   getSerializableMachineMemOperandTargetFlags() const override;
1083 
1084   ScheduleHazardRecognizer *
1085   CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
1086                                  const ScheduleDAG *DAG) const override;
1087 
1088   ScheduleHazardRecognizer *
1089   CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const override;
1090 
1091   ScheduleHazardRecognizer *
1092   CreateTargetMIHazardRecognizer(const InstrItineraryData *II,
1093                                  const ScheduleDAGMI *DAG) const override;
1094 
1095   bool isBasicBlockPrologue(const MachineInstr &MI) const override;
1096 
1097   MachineInstr *createPHIDestinationCopy(MachineBasicBlock &MBB,
1098                                          MachineBasicBlock::iterator InsPt,
1099                                          const DebugLoc &DL, Register Src,
1100                                          Register Dst) const override;
1101 
1102   MachineInstr *createPHISourceCopy(MachineBasicBlock &MBB,
1103                                     MachineBasicBlock::iterator InsPt,
1104                                     const DebugLoc &DL, Register Src,
1105                                     unsigned SrcSubReg,
1106                                     Register Dst) const override;
1107 
1108   bool isWave32() const;
1109 
1110   /// Return a partially built integer add instruction without carry.
1111   /// Caller must add source operands.
1112   /// For pre-GFX9 it will generate unused carry destination operand.
1113   /// TODO: After GFX9 it should return a no-carry operation.
1114   MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB,
1115                                     MachineBasicBlock::iterator I,
1116                                     const DebugLoc &DL,
1117                                     Register DestReg) const;
1118 
1119   MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB,
1120                                     MachineBasicBlock::iterator I,
1121                                     const DebugLoc &DL,
1122                                     Register DestReg,
1123                                     RegScavenger &RS) const;
1124 
1125   static bool isKillTerminator(unsigned Opcode);
1126   const MCInstrDesc &getKillTerminatorFromPseudo(unsigned Opcode) const;
1127 
1128   static bool isLegalMUBUFImmOffset(unsigned Imm) {
1129     return isUInt<12>(Imm);
1130   }
1131 
1132   /// Returns if \p Offset is legal for the subtarget as the offset to a FLAT
1133   /// encoded instruction. If \p Signed, this is for an instruction that
1134   /// interprets the offset as signed.
1135   bool isLegalFLATOffset(int64_t Offset, unsigned AddrSpace,
1136                          uint64_t FlatVariant) const;
1137 
1138   /// Split \p COffsetVal into {immediate offset field, remainder offset}
1139   /// values.
1140   std::pair<int64_t, int64_t> splitFlatOffset(int64_t COffsetVal,
1141                                               unsigned AddrSpace,
1142                                               uint64_t FlatVariant) const;
1143 
1144   /// \brief Return a target-specific opcode if Opcode is a pseudo instruction.
1145   /// Return -1 if the target-specific opcode for the pseudo instruction does
1146   /// not exist. If Opcode is not a pseudo instruction, this is identity.
1147   int pseudoToMCOpcode(int Opcode) const;
1148 
1149   /// \brief Check if this instruction should only be used by assembler.
1150   /// Return true if this opcode should not be used by codegen.
1151   bool isAsmOnlyOpcode(int MCOp) const;
1152 
1153   const TargetRegisterClass *getRegClass(const MCInstrDesc &TID, unsigned OpNum,
1154                                          const TargetRegisterInfo *TRI,
1155                                          const MachineFunction &MF)
1156     const override;
1157 
1158   void fixImplicitOperands(MachineInstr &MI) const;
1159 
1160   MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
1161                                       ArrayRef<unsigned> Ops,
1162                                       MachineBasicBlock::iterator InsertPt,
1163                                       int FrameIndex,
1164                                       LiveIntervals *LIS = nullptr,
1165                                       VirtRegMap *VRM = nullptr) const override;
1166 
1167   unsigned getInstrLatency(const InstrItineraryData *ItinData,
1168                            const MachineInstr &MI,
1169                            unsigned *PredCost = nullptr) const override;
1170 
1171   const MIRFormatter *getMIRFormatter() const override {
1172     if (!Formatter.get())
1173       Formatter = std::make_unique<AMDGPUMIRFormatter>();
1174     return Formatter.get();
1175   }
1176 
1177   static unsigned getDSShaderTypeValue(const MachineFunction &MF);
1178 
1179   const TargetSchedModel &getSchedModel() const { return SchedModel; }
1180 
1181   // Enforce operand's \p OpName even alignment if required by target.
1182   // This is used if an operand is a 32 bit register but needs to be aligned
1183   // regardless.
1184   void enforceOperandRCAlignment(MachineInstr &MI, unsigned OpName) const;
1185 };
1186 
1187 /// \brief Returns true if a reg:subreg pair P has a TRC class
1188 inline bool isOfRegClass(const TargetInstrInfo::RegSubRegPair &P,
1189                          const TargetRegisterClass &TRC,
1190                          MachineRegisterInfo &MRI) {
1191   auto *RC = MRI.getRegClass(P.Reg);
1192   if (!P.SubReg)
1193     return RC == &TRC;
1194   auto *TRI = MRI.getTargetRegisterInfo();
1195   return RC == TRI->getMatchingSuperRegClass(RC, &TRC, P.SubReg);
1196 }
1197 
1198 /// \brief Create RegSubRegPair from a register MachineOperand
1199 inline
1200 TargetInstrInfo::RegSubRegPair getRegSubRegPair(const MachineOperand &O) {
1201   assert(O.isReg());
1202   return TargetInstrInfo::RegSubRegPair(O.getReg(), O.getSubReg());
1203 }
1204 
1205 /// \brief Return the SubReg component from REG_SEQUENCE
1206 TargetInstrInfo::RegSubRegPair getRegSequenceSubReg(MachineInstr &MI,
1207                                                     unsigned SubReg);
1208 
1209 /// \brief Return the defining instruction for a given reg:subreg pair
1210 /// skipping copy like instructions and subreg-manipulation pseudos.
1211 /// Following another subreg of a reg:subreg isn't supported.
1212 MachineInstr *getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P,
1213                                MachineRegisterInfo &MRI);
1214 
1215 /// \brief Return false if EXEC is not changed between the def of \p VReg at \p
1216 /// DefMI and the use at \p UseMI. Should be run on SSA. Currently does not
1217 /// attempt to track between blocks.
1218 bool execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI,
1219                                 Register VReg,
1220                                 const MachineInstr &DefMI,
1221                                 const MachineInstr &UseMI);
1222 
1223 /// \brief Return false if EXEC is not changed between the def of \p VReg at \p
1224 /// DefMI and all its uses. Should be run on SSA. Currently does not attempt to
1225 /// track between blocks.
1226 bool execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI,
1227                                    Register VReg,
1228                                    const MachineInstr &DefMI);
1229 
1230 namespace AMDGPU {
1231 
1232   LLVM_READONLY
1233   int getVOPe64(uint16_t Opcode);
1234 
1235   LLVM_READONLY
1236   int getVOPe32(uint16_t Opcode);
1237 
1238   LLVM_READONLY
1239   int getSDWAOp(uint16_t Opcode);
1240 
1241   LLVM_READONLY
1242   int getDPPOp32(uint16_t Opcode);
1243 
1244   LLVM_READONLY
1245   int getDPPOp64(uint16_t Opcode);
1246 
1247   LLVM_READONLY
1248   int getBasicFromSDWAOp(uint16_t Opcode);
1249 
1250   LLVM_READONLY
1251   int getCommuteRev(uint16_t Opcode);
1252 
1253   LLVM_READONLY
1254   int getCommuteOrig(uint16_t Opcode);
1255 
1256   LLVM_READONLY
1257   int getAddr64Inst(uint16_t Opcode);
1258 
1259   /// Check if \p Opcode is an Addr64 opcode.
1260   ///
1261   /// \returns \p Opcode if it is an Addr64 opcode, otherwise -1.
1262   LLVM_READONLY
1263   int getIfAddr64Inst(uint16_t Opcode);
1264 
1265   LLVM_READONLY
1266   int getAtomicNoRetOp(uint16_t Opcode);
1267 
1268   LLVM_READONLY
1269   int getSOPKOp(uint16_t Opcode);
1270 
1271   /// \returns SADDR form of a FLAT Global instruction given an \p Opcode
1272   /// of a VADDR form.
1273   LLVM_READONLY
1274   int getGlobalSaddrOp(uint16_t Opcode);
1275 
1276   /// \returns VADDR form of a FLAT Global instruction given an \p Opcode
1277   /// of a SADDR form.
1278   LLVM_READONLY
1279   int getGlobalVaddrOp(uint16_t Opcode);
1280 
1281   LLVM_READONLY
1282   int getVCMPXNoSDstOp(uint16_t Opcode);
1283 
1284   /// \returns ST form with only immediate offset of a FLAT Scratch instruction
1285   /// given an \p Opcode of an SS (SADDR) form.
1286   LLVM_READONLY
1287   int getFlatScratchInstSTfromSS(uint16_t Opcode);
1288 
1289   /// \returns SV (VADDR) form of a FLAT Scratch instruction given an \p Opcode
1290   /// of an SVS (SADDR + VADDR) form.
1291   LLVM_READONLY
1292   int getFlatScratchInstSVfromSVS(uint16_t Opcode);
1293 
1294   /// \returns SS (SADDR) form of a FLAT Scratch instruction given an \p Opcode
1295   /// of an SV (VADDR) form.
1296   LLVM_READONLY
1297   int getFlatScratchInstSSfromSV(uint16_t Opcode);
1298 
1299   /// \returns SV (VADDR) form of a FLAT Scratch instruction given an \p Opcode
1300   /// of an SS (SADDR) form.
1301   LLVM_READONLY
1302   int getFlatScratchInstSVfromSS(uint16_t Opcode);
1303 
1304   /// \returns earlyclobber version of a MAC MFMA is exists.
1305   LLVM_READONLY
1306   int getMFMAEarlyClobberOp(uint16_t Opcode);
1307 
1308   /// \returns v_cmpx version of a v_cmp instruction.
1309   LLVM_READONLY
1310   int getVCMPXOpFromVCMP(uint16_t Opcode);
1311 
1312   const uint64_t RSRC_DATA_FORMAT = 0xf00000000000LL;
1313   const uint64_t RSRC_ELEMENT_SIZE_SHIFT = (32 + 19);
1314   const uint64_t RSRC_INDEX_STRIDE_SHIFT = (32 + 21);
1315   const uint64_t RSRC_TID_ENABLE = UINT64_C(1) << (32 + 23);
1316 
1317 } // end namespace AMDGPU
1318 
1319 namespace SI {
1320 namespace KernelInputOffsets {
1321 
1322 /// Offsets in bytes from the start of the input buffer
1323 enum Offsets {
1324   NGROUPS_X = 0,
1325   NGROUPS_Y = 4,
1326   NGROUPS_Z = 8,
1327   GLOBAL_SIZE_X = 12,
1328   GLOBAL_SIZE_Y = 16,
1329   GLOBAL_SIZE_Z = 20,
1330   LOCAL_SIZE_X = 24,
1331   LOCAL_SIZE_Y = 28,
1332   LOCAL_SIZE_Z = 32
1333 };
1334 
1335 } // end namespace KernelInputOffsets
1336 } // end namespace SI
1337 
1338 } // end namespace llvm
1339 
1340 #endif // LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
1341