1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/STLExtras.h"
19 #include "llvm/ADT/SmallBitVector.h"
20 #include "llvm/ADT/SmallString.h"
21 #include "llvm/ADT/StringSwitch.h"
22 #include "llvm/ADT/Twine.h"
23 #include "llvm/CodeGen/MachineValueType.h"
24 #include "llvm/MC/MCContext.h"
25 #include "llvm/MC/MCExpr.h"
26 #include "llvm/MC/MCInst.h"
27 #include "llvm/MC/MCInstrInfo.h"
28 #include "llvm/MC/MCParser/MCAsmLexer.h"
29 #include "llvm/MC/MCParser/MCAsmParser.h"
30 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
31 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
32 #include "llvm/MC/MCRegisterInfo.h"
33 #include "llvm/MC/MCStreamer.h"
34 #include "llvm/MC/MCSubtargetInfo.h"
35 #include "llvm/MC/MCSymbolELF.h"
36 #include "llvm/Support/Debug.h"
37 #include "llvm/Support/ELF.h"
38 #include "llvm/Support/SourceMgr.h"
39 #include "llvm/Support/TargetRegistry.h"
40 #include "llvm/Support/raw_ostream.h"
41 #include "llvm/Support/MathExtras.h"
42 
43 using namespace llvm;
44 using namespace llvm::AMDGPU;
45 
46 namespace {
47 
48 class AMDGPUAsmParser;
49 struct OptionalOperand;
50 
51 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
52 
53 //===----------------------------------------------------------------------===//
54 // Operand
55 //===----------------------------------------------------------------------===//
56 
57 class AMDGPUOperand : public MCParsedAsmOperand {
58   enum KindTy {
59     Token,
60     Immediate,
61     Register,
62     Expression
63   } Kind;
64 
65   SMLoc StartLoc, EndLoc;
66   const AMDGPUAsmParser *AsmParser;
67 
68 public:
69   AMDGPUOperand(enum KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
70     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
71 
72   typedef std::unique_ptr<AMDGPUOperand> Ptr;
73 
74   struct Modifiers {
75     bool Abs;
76     bool Neg;
77     bool Sext;
78 
79     bool hasFPModifiers() const { return Abs || Neg; }
80     bool hasIntModifiers() const { return Sext; }
81     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
82 
83     int64_t getFPModifiersOperand() const {
84       int64_t Operand = 0;
85       Operand |= Abs ? SISrcMods::ABS : 0;
86       Operand |= Neg ? SISrcMods::NEG : 0;
87       return Operand;
88     }
89 
90     int64_t getIntModifiersOperand() const {
91       int64_t Operand = 0;
92       Operand |= Sext ? SISrcMods::SEXT : 0;
93       return Operand;
94     }
95 
96     int64_t getModifiersOperand() const {
97       assert(!(hasFPModifiers() && hasIntModifiers())
98            && "fp and int modifiers should not be used simultaneously");
99       if (hasFPModifiers()) {
100         return getFPModifiersOperand();
101       } else if (hasIntModifiers()) {
102         return getIntModifiersOperand();
103       } else {
104         return 0;
105       }
106     }
107 
108     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
109   };
110 
111   enum ImmTy {
112     ImmTyNone,
113     ImmTyGDS,
114     ImmTyOffen,
115     ImmTyIdxen,
116     ImmTyAddr64,
117     ImmTyOffset,
118     ImmTyOffset0,
119     ImmTyOffset1,
120     ImmTyGLC,
121     ImmTySLC,
122     ImmTyTFE,
123     ImmTyClampSI,
124     ImmTyOModSI,
125     ImmTyDppCtrl,
126     ImmTyDppRowMask,
127     ImmTyDppBankMask,
128     ImmTyDppBoundCtrl,
129     ImmTySdwaDstSel,
130     ImmTySdwaSrc0Sel,
131     ImmTySdwaSrc1Sel,
132     ImmTySdwaDstUnused,
133     ImmTyDMask,
134     ImmTyUNorm,
135     ImmTyDA,
136     ImmTyR128,
137     ImmTyLWE,
138     ImmTyHwreg,
139     ImmTySendMsg,
140   };
141 
142   struct TokOp {
143     const char *Data;
144     unsigned Length;
145   };
146 
147   struct ImmOp {
148     int64_t Val;
149     ImmTy Type;
150     bool IsFPImm;
151     Modifiers Mods;
152   };
153 
154   struct RegOp {
155     unsigned RegNo;
156     bool IsForcedVOP3;
157     Modifiers Mods;
158   };
159 
160   union {
161     TokOp Tok;
162     ImmOp Imm;
163     RegOp Reg;
164     const MCExpr *Expr;
165   };
166 
167   bool isToken() const override {
168     if (Kind == Token)
169       return true;
170 
171     if (Kind != Expression || !Expr)
172       return false;
173 
174     // When parsing operands, we can't always tell if something was meant to be
175     // a token, like 'gds', or an expression that references a global variable.
176     // In this case, we assume the string is an expression, and if we need to
177     // interpret is a token, then we treat the symbol name as the token.
178     return isa<MCSymbolRefExpr>(Expr);
179   }
180 
181   bool isImm() const override {
182     return Kind == Immediate;
183   }
184 
185   bool isInlinableImm(MVT type) const;
186   bool isLiteralImm(MVT type) const;
187 
188   bool isRegKind() const {
189     return Kind == Register;
190   }
191 
192   bool isReg() const override {
193     return isRegKind() && !Reg.Mods.hasModifiers();
194   }
195 
196   bool isRegOrImmWithInputMods(MVT type) const {
197     return isRegKind() || isInlinableImm(type);
198   }
199 
200   bool isRegOrImmWithInt32InputMods() const {
201     return isRegOrImmWithInputMods(MVT::i32);
202   }
203 
204   bool isRegOrImmWithInt64InputMods() const {
205     return isRegOrImmWithInputMods(MVT::i64);
206   }
207 
208   bool isRegOrImmWithFP32InputMods() const {
209     return isRegOrImmWithInputMods(MVT::f32);
210   }
211 
212   bool isRegOrImmWithFP64InputMods() const {
213     return isRegOrImmWithInputMods(MVT::f64);
214   }
215 
216   bool isImmTy(ImmTy ImmT) const {
217     return isImm() && Imm.Type == ImmT;
218   }
219 
220   bool isImmModifier() const {
221     return isImm() && Imm.Type != ImmTyNone;
222   }
223 
224   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
225   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
226   bool isDMask() const { return isImmTy(ImmTyDMask); }
227   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
228   bool isDA() const { return isImmTy(ImmTyDA); }
229   bool isR128() const { return isImmTy(ImmTyUNorm); }
230   bool isLWE() const { return isImmTy(ImmTyLWE); }
231   bool isOffen() const { return isImmTy(ImmTyOffen); }
232   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
233   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
234   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
235   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
236   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
237   bool isGDS() const { return isImmTy(ImmTyGDS); }
238   bool isGLC() const { return isImmTy(ImmTyGLC); }
239   bool isSLC() const { return isImmTy(ImmTySLC); }
240   bool isTFE() const { return isImmTy(ImmTyTFE); }
241   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
242   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
243   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
244   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
245   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
246   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
247   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
248 
249   bool isMod() const {
250     return isClampSI() || isOModSI();
251   }
252 
253   bool isRegOrImm() const {
254     return isReg() || isImm();
255   }
256 
257   bool isRegClass(unsigned RCID) const;
258 
259   bool isSCSrcB32() const {
260     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::i32);
261   }
262 
263   bool isSCSrcB64() const {
264     return isRegClass(AMDGPU::SReg_64RegClassID) || isInlinableImm(MVT::i64);
265   }
266 
267   bool isSCSrcF32() const {
268     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::f32);
269   }
270 
271   bool isSCSrcF64() const {
272     return isRegClass(AMDGPU::SReg_64RegClassID) || isInlinableImm(MVT::f64);
273   }
274 
275   bool isSSrcB32() const {
276     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
277   }
278 
279   bool isSSrcB64() const {
280     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
281     // See isVSrc64().
282     return isSCSrcB64() || isLiteralImm(MVT::i64);
283   }
284 
285   bool isSSrcF32() const {
286     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
287   }
288 
289   bool isSSrcF64() const {
290     return isSCSrcB64() || isLiteralImm(MVT::f64);
291   }
292 
293   bool isVCSrcB32() const {
294     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::i32);
295   }
296 
297   bool isVCSrcB64() const {
298     return isRegClass(AMDGPU::VS_64RegClassID) || isInlinableImm(MVT::i64);
299   }
300 
301   bool isVCSrcF32() const {
302     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::f32);
303   }
304 
305   bool isVCSrcF64() const {
306     return isRegClass(AMDGPU::VS_64RegClassID) || isInlinableImm(MVT::f64);
307   }
308 
309   bool isVSrcB32() const {
310     return isVCSrcF32() || isLiteralImm(MVT::i32);
311   }
312 
313   bool isVSrcB64() const {
314     return isVCSrcF64() || isLiteralImm(MVT::i64);
315   }
316 
317   bool isVSrcF32() const {
318     return isVCSrcF32() || isLiteralImm(MVT::f32);
319   }
320 
321   bool isVSrcF64() const {
322     return isVCSrcF64() || isLiteralImm(MVT::f64);
323   }
324 
325   bool isKImmFP32() const {
326     return isLiteralImm(MVT::f32);
327   }
328 
329   bool isMem() const override {
330     return false;
331   }
332 
333   bool isExpr() const {
334     return Kind == Expression;
335   }
336 
337   bool isSoppBrTarget() const {
338     return isExpr() || isImm();
339   }
340 
341   bool isSWaitCnt() const;
342   bool isHwreg() const;
343   bool isSendMsg() const;
344   bool isSMRDOffset8() const;
345   bool isSMRDOffset20() const;
346   bool isSMRDLiteralOffset() const;
347   bool isDPPCtrl() const;
348   bool isGPRIdxMode() const;
349 
350   StringRef getExpressionAsToken() const {
351     assert(isExpr());
352     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
353     return S->getSymbol().getName();
354   }
355 
356 
357   StringRef getToken() const {
358     assert(isToken());
359 
360     if (Kind == Expression)
361       return getExpressionAsToken();
362 
363     return StringRef(Tok.Data, Tok.Length);
364   }
365 
366   int64_t getImm() const {
367     assert(isImm());
368     return Imm.Val;
369   }
370 
371   enum ImmTy getImmTy() const {
372     assert(isImm());
373     return Imm.Type;
374   }
375 
376   unsigned getReg() const override {
377     return Reg.RegNo;
378   }
379 
380   SMLoc getStartLoc() const override {
381     return StartLoc;
382   }
383 
384   SMLoc getEndLoc() const override {
385     return EndLoc;
386   }
387 
388   Modifiers getModifiers() const {
389     assert(isRegKind() || isImmTy(ImmTyNone));
390     return isRegKind() ? Reg.Mods : Imm.Mods;
391   }
392 
393   void setModifiers(Modifiers Mods) {
394     assert(isRegKind() || isImmTy(ImmTyNone));
395     if (isRegKind())
396       Reg.Mods = Mods;
397     else
398       Imm.Mods = Mods;
399   }
400 
401   bool hasModifiers() const {
402     return getModifiers().hasModifiers();
403   }
404 
405   bool hasFPModifiers() const {
406     return getModifiers().hasFPModifiers();
407   }
408 
409   bool hasIntModifiers() const {
410     return getModifiers().hasIntModifiers();
411   }
412 
413   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
414 
415   void addLiteralImmOperand(MCInst &Inst, int64_t Val) const;
416 
417   void addKImmFP32Operands(MCInst &Inst, unsigned N) const;
418 
419   void addRegOperands(MCInst &Inst, unsigned N) const;
420 
421   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
422     if (isRegKind())
423       addRegOperands(Inst, N);
424     else if (isExpr())
425       Inst.addOperand(MCOperand::createExpr(Expr));
426     else
427       addImmOperands(Inst, N);
428   }
429 
430   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
431     Modifiers Mods = getModifiers();
432     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
433     if (isRegKind()) {
434       addRegOperands(Inst, N);
435     } else {
436       addImmOperands(Inst, N, false);
437     }
438   }
439 
440   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
441     assert(!hasIntModifiers());
442     addRegOrImmWithInputModsOperands(Inst, N);
443   }
444 
445   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
446     assert(!hasFPModifiers());
447     addRegOrImmWithInputModsOperands(Inst, N);
448   }
449 
450   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
451     if (isImm())
452       addImmOperands(Inst, N);
453     else {
454       assert(isExpr());
455       Inst.addOperand(MCOperand::createExpr(Expr));
456     }
457   }
458 
459   void printImmTy(raw_ostream& OS, ImmTy Type) const {
460     switch (Type) {
461     case ImmTyNone: OS << "None"; break;
462     case ImmTyGDS: OS << "GDS"; break;
463     case ImmTyOffen: OS << "Offen"; break;
464     case ImmTyIdxen: OS << "Idxen"; break;
465     case ImmTyAddr64: OS << "Addr64"; break;
466     case ImmTyOffset: OS << "Offset"; break;
467     case ImmTyOffset0: OS << "Offset0"; break;
468     case ImmTyOffset1: OS << "Offset1"; break;
469     case ImmTyGLC: OS << "GLC"; break;
470     case ImmTySLC: OS << "SLC"; break;
471     case ImmTyTFE: OS << "TFE"; break;
472     case ImmTyClampSI: OS << "ClampSI"; break;
473     case ImmTyOModSI: OS << "OModSI"; break;
474     case ImmTyDppCtrl: OS << "DppCtrl"; break;
475     case ImmTyDppRowMask: OS << "DppRowMask"; break;
476     case ImmTyDppBankMask: OS << "DppBankMask"; break;
477     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
478     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
479     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
480     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
481     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
482     case ImmTyDMask: OS << "DMask"; break;
483     case ImmTyUNorm: OS << "UNorm"; break;
484     case ImmTyDA: OS << "DA"; break;
485     case ImmTyR128: OS << "R128"; break;
486     case ImmTyLWE: OS << "LWE"; break;
487     case ImmTyHwreg: OS << "Hwreg"; break;
488     case ImmTySendMsg: OS << "SendMsg"; break;
489     }
490   }
491 
492   void print(raw_ostream &OS) const override {
493     switch (Kind) {
494     case Register:
495       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
496       break;
497     case Immediate:
498       OS << '<' << getImm();
499       if (getImmTy() != ImmTyNone) {
500         OS << " type: "; printImmTy(OS, getImmTy());
501       }
502       OS << " mods: " << Imm.Mods << '>';
503       break;
504     case Token:
505       OS << '\'' << getToken() << '\'';
506       break;
507     case Expression:
508       OS << "<expr " << *Expr << '>';
509       break;
510     }
511   }
512 
513   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
514                                       int64_t Val, SMLoc Loc,
515                                       enum ImmTy Type = ImmTyNone,
516                                       bool IsFPImm = false) {
517     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
518     Op->Imm.Val = Val;
519     Op->Imm.IsFPImm = IsFPImm;
520     Op->Imm.Type = Type;
521     Op->Imm.Mods = {false, false, false};
522     Op->StartLoc = Loc;
523     Op->EndLoc = Loc;
524     return Op;
525   }
526 
527   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
528                                         StringRef Str, SMLoc Loc,
529                                         bool HasExplicitEncodingSize = true) {
530     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
531     Res->Tok.Data = Str.data();
532     Res->Tok.Length = Str.size();
533     Res->StartLoc = Loc;
534     Res->EndLoc = Loc;
535     return Res;
536   }
537 
538   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
539                                       unsigned RegNo, SMLoc S,
540                                       SMLoc E,
541                                       bool ForceVOP3) {
542     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
543     Op->Reg.RegNo = RegNo;
544     Op->Reg.Mods = {false, false, false};
545     Op->Reg.IsForcedVOP3 = ForceVOP3;
546     Op->StartLoc = S;
547     Op->EndLoc = E;
548     return Op;
549   }
550 
551   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
552                                        const class MCExpr *Expr, SMLoc S) {
553     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
554     Op->Expr = Expr;
555     Op->StartLoc = S;
556     Op->EndLoc = S;
557     return Op;
558   }
559 };
560 
561 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
562   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
563   return OS;
564 }
565 
566 //===----------------------------------------------------------------------===//
567 // AsmParser
568 //===----------------------------------------------------------------------===//
569 
570 class AMDGPUAsmParser : public MCTargetAsmParser {
571   const MCInstrInfo &MII;
572   MCAsmParser &Parser;
573 
574   unsigned ForcedEncodingSize;
575   bool ForcedDPP;
576   bool ForcedSDWA;
577 
578   /// @name Auto-generated Match Functions
579   /// {
580 
581 #define GET_ASSEMBLER_HEADER
582 #include "AMDGPUGenAsmMatcher.inc"
583 
584   /// }
585 
586 private:
587   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
588   bool ParseDirectiveHSACodeObjectVersion();
589   bool ParseDirectiveHSACodeObjectISA();
590   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
591   bool ParseDirectiveAMDKernelCodeT();
592   bool ParseSectionDirectiveHSAText();
593   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
594   bool ParseDirectiveAMDGPUHsaKernel();
595   bool ParseDirectiveAMDGPUHsaModuleGlobal();
596   bool ParseDirectiveAMDGPUHsaProgramGlobal();
597   bool ParseSectionDirectiveHSADataGlobalAgent();
598   bool ParseSectionDirectiveHSADataGlobalProgram();
599   bool ParseSectionDirectiveHSARodataReadonlyAgent();
600   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum);
601   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth);
602   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands, bool IsAtomic, bool IsAtomicReturn);
603 
604 public:
605   enum AMDGPUMatchResultTy {
606     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
607   };
608 
609   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
610                const MCInstrInfo &MII,
611                const MCTargetOptions &Options)
612       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser),
613         ForcedEncodingSize(0),
614         ForcedDPP(false),
615         ForcedSDWA(false) {
616     MCAsmParserExtension::Initialize(Parser);
617 
618     if (getSTI().getFeatureBits().none()) {
619       // Set default features.
620       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
621     }
622 
623     setAvailableFeatures(ComputeAvailableFeatures(getSTI().getFeatureBits()));
624 
625     {
626       // TODO: make those pre-defined variables read-only.
627       // Currently there is none suitable machinery in the core llvm-mc for this.
628       // MCSymbol::isRedefinable is intended for another purpose, and
629       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
630       AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
631       MCContext &Ctx = getContext();
632       MCSymbol *Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
633       Sym->setVariableValue(MCConstantExpr::create(Isa.Major, Ctx));
634       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
635       Sym->setVariableValue(MCConstantExpr::create(Isa.Minor, Ctx));
636       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
637       Sym->setVariableValue(MCConstantExpr::create(Isa.Stepping, Ctx));
638     }
639   }
640 
641   bool isSI() const {
642     return AMDGPU::isSI(getSTI());
643   }
644 
645   bool isCI() const {
646     return AMDGPU::isCI(getSTI());
647   }
648 
649   bool isVI() const {
650     return AMDGPU::isVI(getSTI());
651   }
652 
653   bool hasSGPR102_SGPR103() const {
654     return !isVI();
655   }
656 
657   AMDGPUTargetStreamer &getTargetStreamer() {
658     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
659     return static_cast<AMDGPUTargetStreamer &>(TS);
660   }
661 
662   const MCRegisterInfo *getMRI() const {
663     // We need this const_cast because for some reason getContext() is not const
664     // in MCAsmParser.
665     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
666   }
667 
668   const MCInstrInfo *getMII() const {
669     return &MII;
670   }
671 
672   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
673   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
674   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
675 
676   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
677   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
678   bool isForcedDPP() const { return ForcedDPP; }
679   bool isForcedSDWA() const { return ForcedSDWA; }
680 
681   std::unique_ptr<AMDGPUOperand> parseRegister();
682   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
683   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
684   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
685                                       unsigned Kind) override;
686   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
687                                OperandVector &Operands, MCStreamer &Out,
688                                uint64_t &ErrorInfo,
689                                bool MatchingInlineAsm) override;
690   bool ParseDirective(AsmToken DirectiveID) override;
691   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
692   StringRef parseMnemonicSuffix(StringRef Name);
693   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
694                         SMLoc NameLoc, OperandVector &Operands) override;
695   //bool ProcessInstruction(MCInst &Inst);
696 
697   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
698   OperandMatchResultTy parseIntWithPrefix(const char *Prefix,
699                                           OperandVector &Operands,
700                                           enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
701                                           bool (*ConvertResult)(int64_t&) = 0);
702   OperandMatchResultTy parseNamedBit(const char *Name, OperandVector &Operands,
703                                      enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
704   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix, StringRef &Value);
705 
706   OperandMatchResultTy parseImm(OperandVector &Operands);
707   OperandMatchResultTy parseRegOrImm(OperandVector &Operands);
708   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands);
709   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands);
710 
711   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
712   void cvtDS(MCInst &Inst, const OperandVector &Operands);
713 
714   bool parseCnt(int64_t &IntVal);
715   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
716   OperandMatchResultTy parseHwreg(OperandVector &Operands);
717 
718 private:
719   struct OperandInfoTy {
720     int64_t Id;
721     bool IsSymbolic;
722     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
723   };
724 
725   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
726   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
727 public:
728   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
729 
730   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
731   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
732 
733   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
734   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
735   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
736   AMDGPUOperand::Ptr defaultGLC() const;
737   AMDGPUOperand::Ptr defaultSLC() const;
738   AMDGPUOperand::Ptr defaultTFE() const;
739 
740   AMDGPUOperand::Ptr defaultDMask() const;
741   AMDGPUOperand::Ptr defaultUNorm() const;
742   AMDGPUOperand::Ptr defaultDA() const;
743   AMDGPUOperand::Ptr defaultR128() const;
744   AMDGPUOperand::Ptr defaultLWE() const;
745   AMDGPUOperand::Ptr defaultSMRDOffset8() const;
746   AMDGPUOperand::Ptr defaultSMRDOffset20() const;
747   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
748 
749   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
750 
751   void cvtId(MCInst &Inst, const OperandVector &Operands);
752   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
753   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
754 
755   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
756   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
757 
758   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
759   AMDGPUOperand::Ptr defaultRowMask() const;
760   AMDGPUOperand::Ptr defaultBankMask() const;
761   AMDGPUOperand::Ptr defaultBoundCtrl() const;
762   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
763 
764   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
765                                     AMDGPUOperand::ImmTy Type);
766   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
767   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
768   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
769   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
770   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
771                uint64_t BasicInstType);
772 };
773 
774 struct OptionalOperand {
775   const char *Name;
776   AMDGPUOperand::ImmTy Type;
777   bool IsBit;
778   bool (*ConvertResult)(int64_t&);
779 };
780 
781 }
782 
783 //===----------------------------------------------------------------------===//
784 // Operand
785 //===----------------------------------------------------------------------===//
786 
787 bool AMDGPUOperand::isInlinableImm(MVT type) const {
788   if (!isImmTy(ImmTyNone)) {
789     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
790     return false;
791   }
792   // TODO: We should avoid using host float here. It would be better to
793   // check the float bit values which is what a few other places do.
794   // We've had bot failures before due to weird NaN support on mips hosts.
795 
796   APInt Literal(64, Imm.Val);
797 
798   if (Imm.IsFPImm) { // We got fp literal token
799     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
800       return AMDGPU::isInlinableLiteral64(Imm.Val, AsmParser->isVI());
801     } else { // Expected 32-bit operand
802       bool lost;
803       APFloat FPLiteral(APFloat::IEEEdouble, Literal);
804       // Convert literal to single precision
805       APFloat::opStatus status = FPLiteral.convert(APFloat::IEEEsingle,
806                                                     APFloat::rmNearestTiesToEven,
807                                                     &lost);
808       // We allow precision lost but not overflow or underflow
809       if (status != APFloat::opOK &&
810           lost &&
811           ((status & APFloat::opOverflow)  != 0 ||
812             (status & APFloat::opUnderflow) != 0)) {
813         return false;
814       }
815       // Check if single precision literal is inlinable
816       return AMDGPU::isInlinableLiteral32(
817               static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
818               AsmParser->isVI());
819     }
820   } else { // We got int literal token
821     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
822       return AMDGPU::isInlinableLiteral64(Imm.Val, AsmParser->isVI());
823     } else { // Expected 32-bit operand
824       return AMDGPU::isInlinableLiteral32(
825             static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
826             AsmParser->isVI());
827     }
828   }
829   return false;
830 }
831 
832 bool AMDGPUOperand::isLiteralImm(MVT type) const {
833   // Check that this imediate can be added as literal
834   if (!isImmTy(ImmTyNone)) {
835     return false;
836   }
837 
838   APInt Literal(64, Imm.Val);
839 
840   if (Imm.IsFPImm) { // We got fp literal token
841     if (type == MVT::f64) { // Expected 64-bit fp operand
842       // We would set low 64-bits of literal to zeroes but we accept this literals
843       return true;
844     } else if (type == MVT::i64) { // Expected 64-bit int operand
845         // We don't allow fp literals in 64-bit integer instructions. It is
846         // unclear how we should encode them.
847       return false;
848     } else { // Expected 32-bit operand
849       bool lost;
850       APFloat FPLiteral(APFloat::IEEEdouble, Literal);
851       // Convert literal to single precision
852       APFloat::opStatus status = FPLiteral.convert(APFloat::IEEEsingle,
853                                                     APFloat::rmNearestTiesToEven,
854                                                     &lost);
855       // We allow precision lost but not overflow or underflow
856       if (status != APFloat::opOK &&
857           lost &&
858           ((status & APFloat::opOverflow)  != 0 ||
859             (status & APFloat::opUnderflow) != 0)) {
860         return false;
861       }
862       return true;
863     }
864   } else { // We got int literal token
865     APInt HiBits = Literal.getHiBits(32);
866     if (HiBits == 0xffffffff &&
867         (*Literal.getLoBits(32).getRawData() & 0x80000000) != 0) {
868       // If high 32 bits aren't zeroes then they all should be ones and 32nd
869       // bit should be set. So that this 64-bit literal is sign-extension of
870       // 32-bit value.
871       return true;
872     } else if (HiBits == 0) {
873       return true;
874     }
875   }
876   return false;
877 }
878 
879 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
880   return isReg() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
881 }
882 
883 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
884   int64_t Val = Imm.Val;
885   if (isImmTy(ImmTyNone) && ApplyModifiers && Imm.Mods.hasFPModifiers() && Imm.Mods.Neg) {
886     // Apply modifiers to immediate value. Only negate can get here
887     if (Imm.IsFPImm) {
888       APFloat F(BitsToDouble(Val));
889       F.changeSign();
890       Val = F.bitcastToAPInt().getZExtValue();
891     } else {
892       Val = -Val;
893     }
894   }
895 
896   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()), Inst.getNumOperands())) {
897     addLiteralImmOperand(Inst, Val);
898   } else {
899     Inst.addOperand(MCOperand::createImm(Val));
900   }
901 }
902 
903 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val) const {
904   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
905   auto OpNum = Inst.getNumOperands();
906   // Check that this operand accepts literals
907   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
908 
909   APInt Literal(64, Val);
910   auto OpSize = AMDGPU::getRegOperandSize(AsmParser->getMRI(), InstDesc, OpNum); // expected operand size
911 
912   if (Imm.IsFPImm) { // We got fp literal token
913     if (OpSize == 8) { // Expected 64-bit operand
914       // Check if literal is inlinable
915       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(), AsmParser->isVI())) {
916         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
917       } else if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
918         // For fp operands we check if low 32 bits are zeros
919         if (Literal.getLoBits(32) != 0) {
920           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
921                             "Can't encode literal as exact 64-bit"
922                             " floating-point operand. Low 32-bits will be"
923                             " set to zero");
924         }
925         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
926       } else {
927         // We don't allow fp literals in 64-bit integer instructions. It is
928         // unclear how we should encode them. This case should be checked earlier
929         // in predicate methods (isLiteralImm())
930         llvm_unreachable("fp literal in 64-bit integer instruction.");
931       }
932     } else { // Expected 32-bit operand
933       bool lost;
934       APFloat FPLiteral(APFloat::IEEEdouble, Literal);
935       // Convert literal to single precision
936       FPLiteral.convert(APFloat::IEEEsingle, APFloat::rmNearestTiesToEven, &lost);
937       // We allow precision lost but not overflow or underflow. This should be
938       // checked earlier in isLiteralImm()
939       Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
940     }
941   } else { // We got int literal token
942     if (OpSize == 8) { // Expected 64-bit operand
943       auto LiteralVal = Literal.getZExtValue();
944       if (AMDGPU::isInlinableLiteral64(LiteralVal, AsmParser->isVI())) {
945         Inst.addOperand(MCOperand::createImm(LiteralVal));
946         return;
947       }
948     } else { // Expected 32-bit operand
949       auto LiteralVal = static_cast<int32_t>(Literal.getLoBits(32).getZExtValue());
950       if (AMDGPU::isInlinableLiteral32(LiteralVal, AsmParser->isVI())) {
951         Inst.addOperand(MCOperand::createImm(LiteralVal));
952         return;
953       }
954     }
955     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(32).getZExtValue()));
956   }
957 }
958 
959 void AMDGPUOperand::addKImmFP32Operands(MCInst &Inst, unsigned N) const {
960   APInt Literal(64, Imm.Val);
961   if (Imm.IsFPImm) { // We got fp literal
962     bool lost;
963     APFloat FPLiteral(APFloat::IEEEdouble, Literal);
964     FPLiteral.convert(APFloat::IEEEsingle, APFloat::rmNearestTiesToEven, &lost);
965     Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
966   } else { // We got int literal token
967     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(32).getZExtValue()));
968   }
969 }
970 
971 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
972   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
973 }
974 
975 //===----------------------------------------------------------------------===//
976 // AsmParser
977 //===----------------------------------------------------------------------===//
978 
979 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
980   if (Is == IS_VGPR) {
981     switch (RegWidth) {
982       default: return -1;
983       case 1: return AMDGPU::VGPR_32RegClassID;
984       case 2: return AMDGPU::VReg_64RegClassID;
985       case 3: return AMDGPU::VReg_96RegClassID;
986       case 4: return AMDGPU::VReg_128RegClassID;
987       case 8: return AMDGPU::VReg_256RegClassID;
988       case 16: return AMDGPU::VReg_512RegClassID;
989     }
990   } else if (Is == IS_TTMP) {
991     switch (RegWidth) {
992       default: return -1;
993       case 1: return AMDGPU::TTMP_32RegClassID;
994       case 2: return AMDGPU::TTMP_64RegClassID;
995       case 4: return AMDGPU::TTMP_128RegClassID;
996     }
997   } else if (Is == IS_SGPR) {
998     switch (RegWidth) {
999       default: return -1;
1000       case 1: return AMDGPU::SGPR_32RegClassID;
1001       case 2: return AMDGPU::SGPR_64RegClassID;
1002       case 4: return AMDGPU::SGPR_128RegClassID;
1003       case 8: return AMDGPU::SReg_256RegClassID;
1004       case 16: return AMDGPU::SReg_512RegClassID;
1005     }
1006   }
1007   return -1;
1008 }
1009 
1010 static unsigned getSpecialRegForName(StringRef RegName) {
1011   return StringSwitch<unsigned>(RegName)
1012     .Case("exec", AMDGPU::EXEC)
1013     .Case("vcc", AMDGPU::VCC)
1014     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1015     .Case("m0", AMDGPU::M0)
1016     .Case("scc", AMDGPU::SCC)
1017     .Case("tba", AMDGPU::TBA)
1018     .Case("tma", AMDGPU::TMA)
1019     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1020     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1021     .Case("vcc_lo", AMDGPU::VCC_LO)
1022     .Case("vcc_hi", AMDGPU::VCC_HI)
1023     .Case("exec_lo", AMDGPU::EXEC_LO)
1024     .Case("exec_hi", AMDGPU::EXEC_HI)
1025     .Case("tma_lo", AMDGPU::TMA_LO)
1026     .Case("tma_hi", AMDGPU::TMA_HI)
1027     .Case("tba_lo", AMDGPU::TBA_LO)
1028     .Case("tba_hi", AMDGPU::TBA_HI)
1029     .Default(0);
1030 }
1031 
1032 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) {
1033   auto R = parseRegister();
1034   if (!R) return true;
1035   assert(R->isReg());
1036   RegNo = R->getReg();
1037   StartLoc = R->getStartLoc();
1038   EndLoc = R->getEndLoc();
1039   return false;
1040 }
1041 
1042 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum)
1043 {
1044   switch (RegKind) {
1045   case IS_SPECIAL:
1046     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) { Reg = AMDGPU::EXEC; RegWidth = 2; return true; }
1047     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) { Reg = AMDGPU::FLAT_SCR; RegWidth = 2; return true; }
1048     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) { Reg = AMDGPU::VCC; RegWidth = 2; return true; }
1049     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) { Reg = AMDGPU::TBA; RegWidth = 2; return true; }
1050     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) { Reg = AMDGPU::TMA; RegWidth = 2; return true; }
1051     return false;
1052   case IS_VGPR:
1053   case IS_SGPR:
1054   case IS_TTMP:
1055     if (Reg1 != Reg + RegWidth) { return false; }
1056     RegWidth++;
1057     return true;
1058   default:
1059     assert(false); return false;
1060   }
1061 }
1062 
1063 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth)
1064 {
1065   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1066   if (getLexer().is(AsmToken::Identifier)) {
1067     StringRef RegName = Parser.getTok().getString();
1068     if ((Reg = getSpecialRegForName(RegName))) {
1069       Parser.Lex();
1070       RegKind = IS_SPECIAL;
1071     } else {
1072       unsigned RegNumIndex = 0;
1073       if (RegName[0] == 'v') {
1074         RegNumIndex = 1;
1075         RegKind = IS_VGPR;
1076       } else if (RegName[0] == 's') {
1077         RegNumIndex = 1;
1078         RegKind = IS_SGPR;
1079       } else if (RegName.startswith("ttmp")) {
1080         RegNumIndex = strlen("ttmp");
1081         RegKind = IS_TTMP;
1082       } else {
1083         return false;
1084       }
1085       if (RegName.size() > RegNumIndex) {
1086         // Single 32-bit register: vXX.
1087         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1088           return false;
1089         Parser.Lex();
1090         RegWidth = 1;
1091       } else {
1092         // Range of registers: v[XX:YY]. ":YY" is optional.
1093         Parser.Lex();
1094         int64_t RegLo, RegHi;
1095         if (getLexer().isNot(AsmToken::LBrac))
1096           return false;
1097         Parser.Lex();
1098 
1099         if (getParser().parseAbsoluteExpression(RegLo))
1100           return false;
1101 
1102         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1103         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1104           return false;
1105         Parser.Lex();
1106 
1107         if (isRBrace) {
1108           RegHi = RegLo;
1109         } else {
1110           if (getParser().parseAbsoluteExpression(RegHi))
1111             return false;
1112 
1113           if (getLexer().isNot(AsmToken::RBrac))
1114             return false;
1115           Parser.Lex();
1116         }
1117         RegNum = (unsigned) RegLo;
1118         RegWidth = (RegHi - RegLo) + 1;
1119       }
1120     }
1121   } else if (getLexer().is(AsmToken::LBrac)) {
1122     // List of consecutive registers: [s0,s1,s2,s3]
1123     Parser.Lex();
1124     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth))
1125       return false;
1126     if (RegWidth != 1)
1127       return false;
1128     RegisterKind RegKind1;
1129     unsigned Reg1, RegNum1, RegWidth1;
1130     do {
1131       if (getLexer().is(AsmToken::Comma)) {
1132         Parser.Lex();
1133       } else if (getLexer().is(AsmToken::RBrac)) {
1134         Parser.Lex();
1135         break;
1136       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1)) {
1137         if (RegWidth1 != 1) {
1138           return false;
1139         }
1140         if (RegKind1 != RegKind) {
1141           return false;
1142         }
1143         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1144           return false;
1145         }
1146       } else {
1147         return false;
1148       }
1149     } while (true);
1150   } else {
1151     return false;
1152   }
1153   switch (RegKind) {
1154   case IS_SPECIAL:
1155     RegNum = 0;
1156     RegWidth = 1;
1157     break;
1158   case IS_VGPR:
1159   case IS_SGPR:
1160   case IS_TTMP:
1161   {
1162     unsigned Size = 1;
1163     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1164       // SGPR and TTMP registers must be are aligned. Max required alignment is 4 dwords.
1165       Size = std::min(RegWidth, 4u);
1166     }
1167     if (RegNum % Size != 0)
1168       return false;
1169     RegNum = RegNum / Size;
1170     int RCID = getRegClass(RegKind, RegWidth);
1171     if (RCID == -1)
1172       return false;
1173     const MCRegisterClass RC = TRI->getRegClass(RCID);
1174     if (RegNum >= RC.getNumRegs())
1175       return false;
1176     Reg = RC.getRegister(RegNum);
1177     break;
1178   }
1179 
1180   default:
1181     assert(false); return false;
1182   }
1183 
1184   if (!subtargetHasRegister(*TRI, Reg))
1185     return false;
1186   return true;
1187 }
1188 
1189 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1190   const auto &Tok = Parser.getTok();
1191   SMLoc StartLoc = Tok.getLoc();
1192   SMLoc EndLoc = Tok.getEndLoc();
1193   RegisterKind RegKind;
1194   unsigned Reg, RegNum, RegWidth;
1195 
1196   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth)) {
1197     return nullptr;
1198   }
1199   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1200 }
1201 
1202 AMDGPUAsmParser::OperandMatchResultTy
1203 AMDGPUAsmParser::parseImm(OperandVector &Operands) {
1204   // TODO: add syntactic sugar for 1/(2*PI)
1205   bool Minus = false;
1206   if (getLexer().getKind() == AsmToken::Minus) {
1207     Minus = true;
1208     Parser.Lex();
1209   }
1210 
1211   SMLoc S = Parser.getTok().getLoc();
1212   switch(getLexer().getKind()) {
1213   case AsmToken::Integer: {
1214     int64_t IntVal;
1215     if (getParser().parseAbsoluteExpression(IntVal))
1216       return MatchOperand_ParseFail;
1217     if (Minus)
1218       IntVal *= -1;
1219     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1220     return MatchOperand_Success;
1221   }
1222   case AsmToken::Real: {
1223     int64_t IntVal;
1224     if (getParser().parseAbsoluteExpression(IntVal))
1225       return MatchOperand_ParseFail;
1226 
1227     APFloat F(BitsToDouble(IntVal));
1228     if (Minus)
1229       F.changeSign();
1230     Operands.push_back(
1231         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1232                                  AMDGPUOperand::ImmTyNone, true));
1233     return MatchOperand_Success;
1234   }
1235   default:
1236     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1237   }
1238 }
1239 
1240 AMDGPUAsmParser::OperandMatchResultTy
1241 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands) {
1242   auto res = parseImm(Operands);
1243   if (res != MatchOperand_NoMatch) {
1244     return res;
1245   }
1246 
1247   if (auto R = parseRegister()) {
1248     assert(R->isReg());
1249     R->Reg.IsForcedVOP3 = isForcedVOP3();
1250     Operands.push_back(std::move(R));
1251     return MatchOperand_Success;
1252   }
1253   return MatchOperand_ParseFail;
1254 }
1255 
1256 AMDGPUAsmParser::OperandMatchResultTy
1257 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands) {
1258   // XXX: During parsing we can't determine if minus sign means
1259   // negate-modifier or negative immediate value.
1260   // By default we suppose it is modifier.
1261   bool Negate = false, Abs = false, Abs2 = false;
1262 
1263   if (getLexer().getKind()== AsmToken::Minus) {
1264     Parser.Lex();
1265     Negate = true;
1266   }
1267 
1268   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "abs") {
1269     Parser.Lex();
1270     Abs2 = true;
1271     if (getLexer().isNot(AsmToken::LParen)) {
1272       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1273       return MatchOperand_ParseFail;
1274     }
1275     Parser.Lex();
1276   }
1277 
1278   if (getLexer().getKind() == AsmToken::Pipe) {
1279     if (Abs2) {
1280       Error(Parser.getTok().getLoc(), "expected register or immediate");
1281       return MatchOperand_ParseFail;
1282     }
1283     Parser.Lex();
1284     Abs = true;
1285   }
1286 
1287   auto Res = parseRegOrImm(Operands);
1288   if (Res != MatchOperand_Success) {
1289     return Res;
1290   }
1291 
1292   AMDGPUOperand::Modifiers Mods = {false, false, false};
1293   if (Negate) {
1294     Mods.Neg = true;
1295   }
1296   if (Abs) {
1297     if (getLexer().getKind() != AsmToken::Pipe) {
1298       Error(Parser.getTok().getLoc(), "expected vertical bar");
1299       return MatchOperand_ParseFail;
1300     }
1301     Parser.Lex();
1302     Mods.Abs = true;
1303   }
1304   if (Abs2) {
1305     if (getLexer().isNot(AsmToken::RParen)) {
1306       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1307       return MatchOperand_ParseFail;
1308     }
1309     Parser.Lex();
1310     Mods.Abs = true;
1311   }
1312 
1313   if (Mods.hasFPModifiers()) {
1314     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1315     Op.setModifiers(Mods);
1316   }
1317   return MatchOperand_Success;
1318 }
1319 
1320 AMDGPUAsmParser::OperandMatchResultTy
1321 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands) {
1322   bool Sext = false;
1323 
1324   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "sext") {
1325     Parser.Lex();
1326     Sext = true;
1327     if (getLexer().isNot(AsmToken::LParen)) {
1328       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1329       return MatchOperand_ParseFail;
1330     }
1331     Parser.Lex();
1332   }
1333 
1334   auto Res = parseRegOrImm(Operands);
1335   if (Res != MatchOperand_Success) {
1336     return Res;
1337   }
1338 
1339   AMDGPUOperand::Modifiers Mods = {false, false, false};
1340   if (Sext) {
1341     if (getLexer().isNot(AsmToken::RParen)) {
1342       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1343       return MatchOperand_ParseFail;
1344     }
1345     Parser.Lex();
1346     Mods.Sext = true;
1347   }
1348 
1349   if (Mods.hasIntModifiers()) {
1350     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1351     Op.setModifiers(Mods);
1352   }
1353   return MatchOperand_Success;
1354 }
1355 
1356 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1357 
1358   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1359 
1360   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1361       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1362       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1363       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1364     return Match_InvalidOperand;
1365 
1366   if ((TSFlags & SIInstrFlags::VOP3) &&
1367       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1368       getForcedEncodingSize() != 64)
1369     return Match_PreferE32;
1370 
1371   if (Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa ||
1372       Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa) {
1373     // v_mac_f32/16 allow only dst_sel == DWORD;
1374     auto OpNum = AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
1375     const auto &Op = Inst.getOperand(OpNum);
1376     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
1377       return Match_InvalidOperand;
1378     }
1379   }
1380 
1381   return Match_Success;
1382 }
1383 
1384 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1385                                               OperandVector &Operands,
1386                                               MCStreamer &Out,
1387                                               uint64_t &ErrorInfo,
1388                                               bool MatchingInlineAsm) {
1389   // What asm variants we should check
1390   std::vector<unsigned> MatchedVariants;
1391   if (getForcedEncodingSize() == 32) {
1392     MatchedVariants = {AMDGPUAsmVariants::DEFAULT};
1393   } else if (isForcedVOP3()) {
1394     MatchedVariants = {AMDGPUAsmVariants::VOP3};
1395   } else if (isForcedSDWA()) {
1396     MatchedVariants = {AMDGPUAsmVariants::SDWA};
1397   } else if (isForcedDPP()) {
1398     MatchedVariants = {AMDGPUAsmVariants::DPP};
1399   } else {
1400     MatchedVariants = {AMDGPUAsmVariants::DEFAULT,
1401                        AMDGPUAsmVariants::VOP3,
1402                        AMDGPUAsmVariants::SDWA,
1403                        AMDGPUAsmVariants::DPP};
1404   }
1405 
1406   MCInst Inst;
1407   unsigned Result = Match_Success;
1408   for (auto Variant : MatchedVariants) {
1409     uint64_t EI;
1410     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
1411                                   Variant);
1412     // We order match statuses from least to most specific. We use most specific
1413     // status as resulting
1414     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
1415     if ((R == Match_Success) ||
1416         (R == Match_PreferE32) ||
1417         (R == Match_MissingFeature && Result != Match_PreferE32) ||
1418         (R == Match_InvalidOperand && Result != Match_MissingFeature
1419                                    && Result != Match_PreferE32) ||
1420         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
1421                                    && Result != Match_MissingFeature
1422                                    && Result != Match_PreferE32)) {
1423       Result = R;
1424       ErrorInfo = EI;
1425     }
1426     if (R == Match_Success)
1427       break;
1428   }
1429 
1430   switch (Result) {
1431   default: break;
1432   case Match_Success:
1433     Inst.setLoc(IDLoc);
1434     Out.EmitInstruction(Inst, getSTI());
1435     return false;
1436 
1437   case Match_MissingFeature:
1438     return Error(IDLoc, "instruction not supported on this GPU");
1439 
1440   case Match_MnemonicFail:
1441     return Error(IDLoc, "unrecognized instruction mnemonic");
1442 
1443   case Match_InvalidOperand: {
1444     SMLoc ErrorLoc = IDLoc;
1445     if (ErrorInfo != ~0ULL) {
1446       if (ErrorInfo >= Operands.size()) {
1447         return Error(IDLoc, "too few operands for instruction");
1448       }
1449       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
1450       if (ErrorLoc == SMLoc())
1451         ErrorLoc = IDLoc;
1452     }
1453     return Error(ErrorLoc, "invalid operand for instruction");
1454   }
1455 
1456   case Match_PreferE32:
1457     return Error(IDLoc, "internal error: instruction without _e64 suffix "
1458                         "should be encoded as e32");
1459   }
1460   llvm_unreachable("Implement any new match types added!");
1461 }
1462 
1463 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
1464                                                uint32_t &Minor) {
1465   if (getLexer().isNot(AsmToken::Integer))
1466     return TokError("invalid major version");
1467 
1468   Major = getLexer().getTok().getIntVal();
1469   Lex();
1470 
1471   if (getLexer().isNot(AsmToken::Comma))
1472     return TokError("minor version number required, comma expected");
1473   Lex();
1474 
1475   if (getLexer().isNot(AsmToken::Integer))
1476     return TokError("invalid minor version");
1477 
1478   Minor = getLexer().getTok().getIntVal();
1479   Lex();
1480 
1481   return false;
1482 }
1483 
1484 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
1485 
1486   uint32_t Major;
1487   uint32_t Minor;
1488 
1489   if (ParseDirectiveMajorMinor(Major, Minor))
1490     return true;
1491 
1492   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
1493   return false;
1494 }
1495 
1496 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
1497 
1498   uint32_t Major;
1499   uint32_t Minor;
1500   uint32_t Stepping;
1501   StringRef VendorName;
1502   StringRef ArchName;
1503 
1504   // If this directive has no arguments, then use the ISA version for the
1505   // targeted GPU.
1506   if (getLexer().is(AsmToken::EndOfStatement)) {
1507     AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
1508     getTargetStreamer().EmitDirectiveHSACodeObjectISA(Isa.Major, Isa.Minor,
1509                                                       Isa.Stepping,
1510                                                       "AMD", "AMDGPU");
1511     return false;
1512   }
1513 
1514 
1515   if (ParseDirectiveMajorMinor(Major, Minor))
1516     return true;
1517 
1518   if (getLexer().isNot(AsmToken::Comma))
1519     return TokError("stepping version number required, comma expected");
1520   Lex();
1521 
1522   if (getLexer().isNot(AsmToken::Integer))
1523     return TokError("invalid stepping version");
1524 
1525   Stepping = getLexer().getTok().getIntVal();
1526   Lex();
1527 
1528   if (getLexer().isNot(AsmToken::Comma))
1529     return TokError("vendor name required, comma expected");
1530   Lex();
1531 
1532   if (getLexer().isNot(AsmToken::String))
1533     return TokError("invalid vendor name");
1534 
1535   VendorName = getLexer().getTok().getStringContents();
1536   Lex();
1537 
1538   if (getLexer().isNot(AsmToken::Comma))
1539     return TokError("arch name required, comma expected");
1540   Lex();
1541 
1542   if (getLexer().isNot(AsmToken::String))
1543     return TokError("invalid arch name");
1544 
1545   ArchName = getLexer().getTok().getStringContents();
1546   Lex();
1547 
1548   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
1549                                                     VendorName, ArchName);
1550   return false;
1551 }
1552 
1553 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
1554                                                amd_kernel_code_t &Header) {
1555   SmallString<40> ErrStr;
1556   raw_svector_ostream Err(ErrStr);
1557   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
1558     return TokError(Err.str());
1559   }
1560   Lex();
1561   return false;
1562 }
1563 
1564 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
1565 
1566   amd_kernel_code_t Header;
1567   AMDGPU::initDefaultAMDKernelCodeT(Header, getSTI().getFeatureBits());
1568 
1569   while (true) {
1570 
1571     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
1572     // will set the current token to EndOfStatement.
1573     while(getLexer().is(AsmToken::EndOfStatement))
1574       Lex();
1575 
1576     if (getLexer().isNot(AsmToken::Identifier))
1577       return TokError("expected value identifier or .end_amd_kernel_code_t");
1578 
1579     StringRef ID = getLexer().getTok().getIdentifier();
1580     Lex();
1581 
1582     if (ID == ".end_amd_kernel_code_t")
1583       break;
1584 
1585     if (ParseAMDKernelCodeTValue(ID, Header))
1586       return true;
1587   }
1588 
1589   getTargetStreamer().EmitAMDKernelCodeT(Header);
1590 
1591   return false;
1592 }
1593 
1594 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
1595   getParser().getStreamer().SwitchSection(
1596       AMDGPU::getHSATextSection(getContext()));
1597   return false;
1598 }
1599 
1600 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
1601   if (getLexer().isNot(AsmToken::Identifier))
1602     return TokError("expected symbol name");
1603 
1604   StringRef KernelName = Parser.getTok().getString();
1605 
1606   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
1607                                            ELF::STT_AMDGPU_HSA_KERNEL);
1608   Lex();
1609   return false;
1610 }
1611 
1612 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
1613   if (getLexer().isNot(AsmToken::Identifier))
1614     return TokError("expected symbol name");
1615 
1616   StringRef GlobalName = Parser.getTok().getIdentifier();
1617 
1618   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
1619   Lex();
1620   return false;
1621 }
1622 
1623 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
1624   if (getLexer().isNot(AsmToken::Identifier))
1625     return TokError("expected symbol name");
1626 
1627   StringRef GlobalName = Parser.getTok().getIdentifier();
1628 
1629   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
1630   Lex();
1631   return false;
1632 }
1633 
1634 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
1635   getParser().getStreamer().SwitchSection(
1636       AMDGPU::getHSADataGlobalAgentSection(getContext()));
1637   return false;
1638 }
1639 
1640 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
1641   getParser().getStreamer().SwitchSection(
1642       AMDGPU::getHSADataGlobalProgramSection(getContext()));
1643   return false;
1644 }
1645 
1646 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
1647   getParser().getStreamer().SwitchSection(
1648       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
1649   return false;
1650 }
1651 
1652 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
1653   StringRef IDVal = DirectiveID.getString();
1654 
1655   if (IDVal == ".hsa_code_object_version")
1656     return ParseDirectiveHSACodeObjectVersion();
1657 
1658   if (IDVal == ".hsa_code_object_isa")
1659     return ParseDirectiveHSACodeObjectISA();
1660 
1661   if (IDVal == ".amd_kernel_code_t")
1662     return ParseDirectiveAMDKernelCodeT();
1663 
1664   if (IDVal == ".hsatext")
1665     return ParseSectionDirectiveHSAText();
1666 
1667   if (IDVal == ".amdgpu_hsa_kernel")
1668     return ParseDirectiveAMDGPUHsaKernel();
1669 
1670   if (IDVal == ".amdgpu_hsa_module_global")
1671     return ParseDirectiveAMDGPUHsaModuleGlobal();
1672 
1673   if (IDVal == ".amdgpu_hsa_program_global")
1674     return ParseDirectiveAMDGPUHsaProgramGlobal();
1675 
1676   if (IDVal == ".hsadata_global_agent")
1677     return ParseSectionDirectiveHSADataGlobalAgent();
1678 
1679   if (IDVal == ".hsadata_global_program")
1680     return ParseSectionDirectiveHSADataGlobalProgram();
1681 
1682   if (IDVal == ".hsarodata_readonly_agent")
1683     return ParseSectionDirectiveHSARodataReadonlyAgent();
1684 
1685   return true;
1686 }
1687 
1688 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
1689                                            unsigned RegNo) const {
1690   if (isCI())
1691     return true;
1692 
1693   if (isSI()) {
1694     // No flat_scr
1695     switch (RegNo) {
1696     case AMDGPU::FLAT_SCR:
1697     case AMDGPU::FLAT_SCR_LO:
1698     case AMDGPU::FLAT_SCR_HI:
1699       return false;
1700     default:
1701       return true;
1702     }
1703   }
1704 
1705   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
1706   // SI/CI have.
1707   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
1708        R.isValid(); ++R) {
1709     if (*R == RegNo)
1710       return false;
1711   }
1712 
1713   return true;
1714 }
1715 
1716 AMDGPUAsmParser::OperandMatchResultTy
1717 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
1718 
1719   // Try to parse with a custom parser
1720   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
1721 
1722   // If we successfully parsed the operand or if there as an error parsing,
1723   // we are done.
1724   //
1725   // If we are parsing after we reach EndOfStatement then this means we
1726   // are appending default values to the Operands list.  This is only done
1727   // by custom parser, so we shouldn't continue on to the generic parsing.
1728   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
1729       getLexer().is(AsmToken::EndOfStatement))
1730     return ResTy;
1731 
1732   ResTy = parseRegOrImm(Operands);
1733 
1734   if (ResTy == MatchOperand_Success)
1735     return ResTy;
1736 
1737   if (getLexer().getKind() == AsmToken::Identifier) {
1738     // If this identifier is a symbol, we want to create an expression for it.
1739     // It is a little difficult to distinguish between a symbol name, and
1740     // an instruction flag like 'gds'.  In order to do this, we parse
1741     // all tokens as expressions and then treate the symbol name as the token
1742     // string when we want to interpret the operand as a token.
1743     const auto &Tok = Parser.getTok();
1744     SMLoc S = Tok.getLoc();
1745     const MCExpr *Expr = nullptr;
1746     if (!Parser.parseExpression(Expr)) {
1747       Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
1748       return MatchOperand_Success;
1749     }
1750 
1751     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), Tok.getLoc()));
1752     Parser.Lex();
1753     return MatchOperand_Success;
1754   }
1755   return MatchOperand_NoMatch;
1756 }
1757 
1758 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
1759   // Clear any forced encodings from the previous instruction.
1760   setForcedEncodingSize(0);
1761   setForcedDPP(false);
1762   setForcedSDWA(false);
1763 
1764   if (Name.endswith("_e64")) {
1765     setForcedEncodingSize(64);
1766     return Name.substr(0, Name.size() - 4);
1767   } else if (Name.endswith("_e32")) {
1768     setForcedEncodingSize(32);
1769     return Name.substr(0, Name.size() - 4);
1770   } else if (Name.endswith("_dpp")) {
1771     setForcedDPP(true);
1772     return Name.substr(0, Name.size() - 4);
1773   } else if (Name.endswith("_sdwa")) {
1774     setForcedSDWA(true);
1775     return Name.substr(0, Name.size() - 5);
1776   }
1777   return Name;
1778 }
1779 
1780 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
1781                                        StringRef Name,
1782                                        SMLoc NameLoc, OperandVector &Operands) {
1783   // Add the instruction mnemonic
1784   Name = parseMnemonicSuffix(Name);
1785   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
1786 
1787   while (!getLexer().is(AsmToken::EndOfStatement)) {
1788     AMDGPUAsmParser::OperandMatchResultTy Res = parseOperand(Operands, Name);
1789 
1790     // Eat the comma or space if there is one.
1791     if (getLexer().is(AsmToken::Comma))
1792       Parser.Lex();
1793 
1794     switch (Res) {
1795       case MatchOperand_Success: break;
1796       case MatchOperand_ParseFail:
1797         Error(getLexer().getLoc(), "failed parsing operand.");
1798         while (!getLexer().is(AsmToken::EndOfStatement)) {
1799           Parser.Lex();
1800         }
1801         return true;
1802       case MatchOperand_NoMatch:
1803         Error(getLexer().getLoc(), "not a valid operand.");
1804         while (!getLexer().is(AsmToken::EndOfStatement)) {
1805           Parser.Lex();
1806         }
1807         return true;
1808     }
1809   }
1810 
1811   return false;
1812 }
1813 
1814 //===----------------------------------------------------------------------===//
1815 // Utility functions
1816 //===----------------------------------------------------------------------===//
1817 
1818 AMDGPUAsmParser::OperandMatchResultTy
1819 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
1820   switch(getLexer().getKind()) {
1821     default: return MatchOperand_NoMatch;
1822     case AsmToken::Identifier: {
1823       StringRef Name = Parser.getTok().getString();
1824       if (!Name.equals(Prefix)) {
1825         return MatchOperand_NoMatch;
1826       }
1827 
1828       Parser.Lex();
1829       if (getLexer().isNot(AsmToken::Colon))
1830         return MatchOperand_ParseFail;
1831 
1832       Parser.Lex();
1833       if (getLexer().isNot(AsmToken::Integer))
1834         return MatchOperand_ParseFail;
1835 
1836       if (getParser().parseAbsoluteExpression(Int))
1837         return MatchOperand_ParseFail;
1838       break;
1839     }
1840   }
1841   return MatchOperand_Success;
1842 }
1843 
1844 AMDGPUAsmParser::OperandMatchResultTy
1845 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
1846                                     enum AMDGPUOperand::ImmTy ImmTy,
1847                                     bool (*ConvertResult)(int64_t&)) {
1848 
1849   SMLoc S = Parser.getTok().getLoc();
1850   int64_t Value = 0;
1851 
1852   AMDGPUAsmParser::OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
1853   if (Res != MatchOperand_Success)
1854     return Res;
1855 
1856   if (ConvertResult && !ConvertResult(Value)) {
1857     return MatchOperand_ParseFail;
1858   }
1859 
1860   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
1861   return MatchOperand_Success;
1862 }
1863 
1864 AMDGPUAsmParser::OperandMatchResultTy
1865 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
1866                                enum AMDGPUOperand::ImmTy ImmTy) {
1867   int64_t Bit = 0;
1868   SMLoc S = Parser.getTok().getLoc();
1869 
1870   // We are at the end of the statement, and this is a default argument, so
1871   // use a default value.
1872   if (getLexer().isNot(AsmToken::EndOfStatement)) {
1873     switch(getLexer().getKind()) {
1874       case AsmToken::Identifier: {
1875         StringRef Tok = Parser.getTok().getString();
1876         if (Tok == Name) {
1877           Bit = 1;
1878           Parser.Lex();
1879         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
1880           Bit = 0;
1881           Parser.Lex();
1882         } else {
1883           return MatchOperand_NoMatch;
1884         }
1885         break;
1886       }
1887       default:
1888         return MatchOperand_NoMatch;
1889     }
1890   }
1891 
1892   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
1893   return MatchOperand_Success;
1894 }
1895 
1896 typedef std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
1897 
1898 void addOptionalImmOperand(MCInst& Inst, const OperandVector& Operands,
1899                            OptionalImmIndexMap& OptionalIdx,
1900                            enum AMDGPUOperand::ImmTy ImmT, int64_t Default = 0) {
1901   auto i = OptionalIdx.find(ImmT);
1902   if (i != OptionalIdx.end()) {
1903     unsigned Idx = i->second;
1904     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
1905   } else {
1906     Inst.addOperand(MCOperand::createImm(Default));
1907   }
1908 }
1909 
1910 AMDGPUAsmParser::OperandMatchResultTy
1911 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
1912   if (getLexer().isNot(AsmToken::Identifier)) {
1913     return MatchOperand_NoMatch;
1914   }
1915   StringRef Tok = Parser.getTok().getString();
1916   if (Tok != Prefix) {
1917     return MatchOperand_NoMatch;
1918   }
1919 
1920   Parser.Lex();
1921   if (getLexer().isNot(AsmToken::Colon)) {
1922     return MatchOperand_ParseFail;
1923   }
1924 
1925   Parser.Lex();
1926   if (getLexer().isNot(AsmToken::Identifier)) {
1927     return MatchOperand_ParseFail;
1928   }
1929 
1930   Value = Parser.getTok().getString();
1931   return MatchOperand_Success;
1932 }
1933 
1934 //===----------------------------------------------------------------------===//
1935 // ds
1936 //===----------------------------------------------------------------------===//
1937 
1938 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
1939                                     const OperandVector &Operands) {
1940 
1941   OptionalImmIndexMap OptionalIdx;
1942 
1943   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1944     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1945 
1946     // Add the register arguments
1947     if (Op.isReg()) {
1948       Op.addRegOperands(Inst, 1);
1949       continue;
1950     }
1951 
1952     // Handle optional arguments
1953     OptionalIdx[Op.getImmTy()] = i;
1954   }
1955 
1956   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
1957   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
1958   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1959 
1960   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1961 }
1962 
1963 void AMDGPUAsmParser::cvtDS(MCInst &Inst, const OperandVector &Operands) {
1964 
1965   std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
1966   bool GDSOnly = false;
1967 
1968   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1969     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1970 
1971     // Add the register arguments
1972     if (Op.isReg()) {
1973       Op.addRegOperands(Inst, 1);
1974       continue;
1975     }
1976 
1977     if (Op.isToken() && Op.getToken() == "gds") {
1978       GDSOnly = true;
1979       continue;
1980     }
1981 
1982     // Handle optional arguments
1983     OptionalIdx[Op.getImmTy()] = i;
1984   }
1985 
1986   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
1987   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1988 
1989   if (!GDSOnly) {
1990     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1991   }
1992   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1993 }
1994 
1995 
1996 //===----------------------------------------------------------------------===//
1997 // s_waitcnt
1998 //===----------------------------------------------------------------------===//
1999 
2000 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
2001   StringRef CntName = Parser.getTok().getString();
2002   int64_t CntVal;
2003 
2004   Parser.Lex();
2005   if (getLexer().isNot(AsmToken::LParen))
2006     return true;
2007 
2008   Parser.Lex();
2009   if (getLexer().isNot(AsmToken::Integer))
2010     return true;
2011 
2012   if (getParser().parseAbsoluteExpression(CntVal))
2013     return true;
2014 
2015   if (getLexer().isNot(AsmToken::RParen))
2016     return true;
2017 
2018   Parser.Lex();
2019   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
2020     Parser.Lex();
2021 
2022   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2023   if (CntName == "vmcnt")
2024     IntVal = encodeVmcnt(IV, IntVal, CntVal);
2025   else if (CntName == "expcnt")
2026     IntVal = encodeExpcnt(IV, IntVal, CntVal);
2027   else if (CntName == "lgkmcnt")
2028     IntVal = encodeLgkmcnt(IV, IntVal, CntVal);
2029   else
2030     return true;
2031 
2032   return false;
2033 }
2034 
2035 AMDGPUAsmParser::OperandMatchResultTy
2036 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
2037   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2038   int64_t Waitcnt = getWaitcntBitMask(IV);
2039   SMLoc S = Parser.getTok().getLoc();
2040 
2041   switch(getLexer().getKind()) {
2042     default: return MatchOperand_ParseFail;
2043     case AsmToken::Integer:
2044       // The operand can be an integer value.
2045       if (getParser().parseAbsoluteExpression(Waitcnt))
2046         return MatchOperand_ParseFail;
2047       break;
2048 
2049     case AsmToken::Identifier:
2050       do {
2051         if (parseCnt(Waitcnt))
2052           return MatchOperand_ParseFail;
2053       } while(getLexer().isNot(AsmToken::EndOfStatement));
2054       break;
2055   }
2056   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
2057   return MatchOperand_Success;
2058 }
2059 
2060 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width) {
2061   using namespace llvm::AMDGPU::Hwreg;
2062 
2063   if (Parser.getTok().getString() != "hwreg")
2064     return true;
2065   Parser.Lex();
2066 
2067   if (getLexer().isNot(AsmToken::LParen))
2068     return true;
2069   Parser.Lex();
2070 
2071   if (getLexer().is(AsmToken::Identifier)) {
2072     HwReg.IsSymbolic = true;
2073     HwReg.Id = ID_UNKNOWN_;
2074     const StringRef tok = Parser.getTok().getString();
2075     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
2076       if (tok == IdSymbolic[i]) {
2077         HwReg.Id = i;
2078         break;
2079       }
2080     }
2081     Parser.Lex();
2082   } else {
2083     HwReg.IsSymbolic = false;
2084     if (getLexer().isNot(AsmToken::Integer))
2085       return true;
2086     if (getParser().parseAbsoluteExpression(HwReg.Id))
2087       return true;
2088   }
2089 
2090   if (getLexer().is(AsmToken::RParen)) {
2091     Parser.Lex();
2092     return false;
2093   }
2094 
2095   // optional params
2096   if (getLexer().isNot(AsmToken::Comma))
2097     return true;
2098   Parser.Lex();
2099 
2100   if (getLexer().isNot(AsmToken::Integer))
2101     return true;
2102   if (getParser().parseAbsoluteExpression(Offset))
2103     return true;
2104 
2105   if (getLexer().isNot(AsmToken::Comma))
2106     return true;
2107   Parser.Lex();
2108 
2109   if (getLexer().isNot(AsmToken::Integer))
2110     return true;
2111   if (getParser().parseAbsoluteExpression(Width))
2112     return true;
2113 
2114   if (getLexer().isNot(AsmToken::RParen))
2115     return true;
2116   Parser.Lex();
2117 
2118   return false;
2119 }
2120 
2121 AMDGPUAsmParser::OperandMatchResultTy
2122 AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
2123   using namespace llvm::AMDGPU::Hwreg;
2124 
2125   int64_t Imm16Val = 0;
2126   SMLoc S = Parser.getTok().getLoc();
2127 
2128   switch(getLexer().getKind()) {
2129     default: return MatchOperand_NoMatch;
2130     case AsmToken::Integer:
2131       // The operand can be an integer value.
2132       if (getParser().parseAbsoluteExpression(Imm16Val))
2133         return MatchOperand_NoMatch;
2134       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2135         Error(S, "invalid immediate: only 16-bit values are legal");
2136         // Do not return error code, but create an imm operand anyway and proceed
2137         // to the next operand, if any. That avoids unneccessary error messages.
2138       }
2139       break;
2140 
2141     case AsmToken::Identifier: {
2142         OperandInfoTy HwReg(ID_UNKNOWN_);
2143         int64_t Offset = OFFSET_DEFAULT_;
2144         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
2145         if (parseHwregConstruct(HwReg, Offset, Width))
2146           return MatchOperand_ParseFail;
2147         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
2148           if (HwReg.IsSymbolic)
2149             Error(S, "invalid symbolic name of hardware register");
2150           else
2151             Error(S, "invalid code of hardware register: only 6-bit values are legal");
2152         }
2153         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
2154           Error(S, "invalid bit offset: only 5-bit values are legal");
2155         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
2156           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
2157         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
2158       }
2159       break;
2160   }
2161   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
2162   return MatchOperand_Success;
2163 }
2164 
2165 bool AMDGPUOperand::isSWaitCnt() const {
2166   return isImm();
2167 }
2168 
2169 bool AMDGPUOperand::isHwreg() const {
2170   return isImmTy(ImmTyHwreg);
2171 }
2172 
2173 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
2174   using namespace llvm::AMDGPU::SendMsg;
2175 
2176   if (Parser.getTok().getString() != "sendmsg")
2177     return true;
2178   Parser.Lex();
2179 
2180   if (getLexer().isNot(AsmToken::LParen))
2181     return true;
2182   Parser.Lex();
2183 
2184   if (getLexer().is(AsmToken::Identifier)) {
2185     Msg.IsSymbolic = true;
2186     Msg.Id = ID_UNKNOWN_;
2187     const std::string tok = Parser.getTok().getString();
2188     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
2189       switch(i) {
2190         default: continue; // Omit gaps.
2191         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
2192       }
2193       if (tok == IdSymbolic[i]) {
2194         Msg.Id = i;
2195         break;
2196       }
2197     }
2198     Parser.Lex();
2199   } else {
2200     Msg.IsSymbolic = false;
2201     if (getLexer().isNot(AsmToken::Integer))
2202       return true;
2203     if (getParser().parseAbsoluteExpression(Msg.Id))
2204       return true;
2205     if (getLexer().is(AsmToken::Integer))
2206       if (getParser().parseAbsoluteExpression(Msg.Id))
2207         Msg.Id = ID_UNKNOWN_;
2208   }
2209   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
2210     return false;
2211 
2212   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
2213     if (getLexer().isNot(AsmToken::RParen))
2214       return true;
2215     Parser.Lex();
2216     return false;
2217   }
2218 
2219   if (getLexer().isNot(AsmToken::Comma))
2220     return true;
2221   Parser.Lex();
2222 
2223   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
2224   Operation.Id = ID_UNKNOWN_;
2225   if (getLexer().is(AsmToken::Identifier)) {
2226     Operation.IsSymbolic = true;
2227     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
2228     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
2229     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
2230     const StringRef Tok = Parser.getTok().getString();
2231     for (int i = F; i < L; ++i) {
2232       if (Tok == S[i]) {
2233         Operation.Id = i;
2234         break;
2235       }
2236     }
2237     Parser.Lex();
2238   } else {
2239     Operation.IsSymbolic = false;
2240     if (getLexer().isNot(AsmToken::Integer))
2241       return true;
2242     if (getParser().parseAbsoluteExpression(Operation.Id))
2243       return true;
2244   }
2245 
2246   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2247     // Stream id is optional.
2248     if (getLexer().is(AsmToken::RParen)) {
2249       Parser.Lex();
2250       return false;
2251     }
2252 
2253     if (getLexer().isNot(AsmToken::Comma))
2254       return true;
2255     Parser.Lex();
2256 
2257     if (getLexer().isNot(AsmToken::Integer))
2258       return true;
2259     if (getParser().parseAbsoluteExpression(StreamId))
2260       return true;
2261   }
2262 
2263   if (getLexer().isNot(AsmToken::RParen))
2264     return true;
2265   Parser.Lex();
2266   return false;
2267 }
2268 
2269 AMDGPUAsmParser::OperandMatchResultTy
2270 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
2271   using namespace llvm::AMDGPU::SendMsg;
2272 
2273   int64_t Imm16Val = 0;
2274   SMLoc S = Parser.getTok().getLoc();
2275 
2276   switch(getLexer().getKind()) {
2277   default:
2278     return MatchOperand_NoMatch;
2279   case AsmToken::Integer:
2280     // The operand can be an integer value.
2281     if (getParser().parseAbsoluteExpression(Imm16Val))
2282       return MatchOperand_NoMatch;
2283     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2284       Error(S, "invalid immediate: only 16-bit values are legal");
2285       // Do not return error code, but create an imm operand anyway and proceed
2286       // to the next operand, if any. That avoids unneccessary error messages.
2287     }
2288     break;
2289   case AsmToken::Identifier: {
2290       OperandInfoTy Msg(ID_UNKNOWN_);
2291       OperandInfoTy Operation(OP_UNKNOWN_);
2292       int64_t StreamId = STREAM_ID_DEFAULT_;
2293       if (parseSendMsgConstruct(Msg, Operation, StreamId))
2294         return MatchOperand_ParseFail;
2295       do {
2296         // Validate and encode message ID.
2297         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
2298                 || Msg.Id == ID_SYSMSG)) {
2299           if (Msg.IsSymbolic)
2300             Error(S, "invalid/unsupported symbolic name of message");
2301           else
2302             Error(S, "invalid/unsupported code of message");
2303           break;
2304         }
2305         Imm16Val = (Msg.Id << ID_SHIFT_);
2306         // Validate and encode operation ID.
2307         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
2308           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
2309             if (Operation.IsSymbolic)
2310               Error(S, "invalid symbolic name of GS_OP");
2311             else
2312               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
2313             break;
2314           }
2315           if (Operation.Id == OP_GS_NOP
2316               && Msg.Id != ID_GS_DONE) {
2317             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
2318             break;
2319           }
2320           Imm16Val |= (Operation.Id << OP_SHIFT_);
2321         }
2322         if (Msg.Id == ID_SYSMSG) {
2323           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
2324             if (Operation.IsSymbolic)
2325               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
2326             else
2327               Error(S, "invalid/unsupported code of SYSMSG_OP");
2328             break;
2329           }
2330           Imm16Val |= (Operation.Id << OP_SHIFT_);
2331         }
2332         // Validate and encode stream ID.
2333         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2334           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
2335             Error(S, "invalid stream id: only 2-bit values are legal");
2336             break;
2337           }
2338           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
2339         }
2340       } while (0);
2341     }
2342     break;
2343   }
2344   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
2345   return MatchOperand_Success;
2346 }
2347 
2348 bool AMDGPUOperand::isSendMsg() const {
2349   return isImmTy(ImmTySendMsg);
2350 }
2351 
2352 //===----------------------------------------------------------------------===//
2353 // sopp branch targets
2354 //===----------------------------------------------------------------------===//
2355 
2356 AMDGPUAsmParser::OperandMatchResultTy
2357 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
2358   SMLoc S = Parser.getTok().getLoc();
2359 
2360   switch (getLexer().getKind()) {
2361     default: return MatchOperand_ParseFail;
2362     case AsmToken::Integer: {
2363       int64_t Imm;
2364       if (getParser().parseAbsoluteExpression(Imm))
2365         return MatchOperand_ParseFail;
2366       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
2367       return MatchOperand_Success;
2368     }
2369 
2370     case AsmToken::Identifier:
2371       Operands.push_back(AMDGPUOperand::CreateExpr(this,
2372           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
2373                                   Parser.getTok().getString()), getContext()), S));
2374       Parser.Lex();
2375       return MatchOperand_Success;
2376   }
2377 }
2378 
2379 //===----------------------------------------------------------------------===//
2380 // mubuf
2381 //===----------------------------------------------------------------------===//
2382 
2383 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
2384   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
2385 }
2386 
2387 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
2388   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
2389 }
2390 
2391 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
2392   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
2393 }
2394 
2395 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
2396                                const OperandVector &Operands,
2397                                bool IsAtomic, bool IsAtomicReturn) {
2398   OptionalImmIndexMap OptionalIdx;
2399   assert(IsAtomicReturn ? IsAtomic : true);
2400 
2401   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2402     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2403 
2404     // Add the register arguments
2405     if (Op.isReg()) {
2406       Op.addRegOperands(Inst, 1);
2407       continue;
2408     }
2409 
2410     // Handle the case where soffset is an immediate
2411     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
2412       Op.addImmOperands(Inst, 1);
2413       continue;
2414     }
2415 
2416     // Handle tokens like 'offen' which are sometimes hard-coded into the
2417     // asm string.  There are no MCInst operands for these.
2418     if (Op.isToken()) {
2419       continue;
2420     }
2421     assert(Op.isImm());
2422 
2423     // Handle optional arguments
2424     OptionalIdx[Op.getImmTy()] = i;
2425   }
2426 
2427   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
2428   if (IsAtomicReturn) {
2429     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
2430     Inst.insert(I, *I);
2431   }
2432 
2433   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2434   if (!IsAtomic) { // glc is hard-coded.
2435     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2436   }
2437   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2438   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2439 }
2440 
2441 //===----------------------------------------------------------------------===//
2442 // mimg
2443 //===----------------------------------------------------------------------===//
2444 
2445 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
2446   unsigned I = 1;
2447   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2448   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2449     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2450   }
2451 
2452   OptionalImmIndexMap OptionalIdx;
2453 
2454   for (unsigned E = Operands.size(); I != E; ++I) {
2455     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2456 
2457     // Add the register arguments
2458     if (Op.isRegOrImm()) {
2459       Op.addRegOrImmOperands(Inst, 1);
2460       continue;
2461     } else if (Op.isImmModifier()) {
2462       OptionalIdx[Op.getImmTy()] = I;
2463     } else {
2464       assert(false);
2465     }
2466   }
2467 
2468   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2469   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2470   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2471   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2472   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2473   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2474   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2475   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2476 }
2477 
2478 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
2479   unsigned I = 1;
2480   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2481   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2482     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2483   }
2484 
2485   // Add src, same as dst
2486   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
2487 
2488   OptionalImmIndexMap OptionalIdx;
2489 
2490   for (unsigned E = Operands.size(); I != E; ++I) {
2491     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2492 
2493     // Add the register arguments
2494     if (Op.isRegOrImm()) {
2495       Op.addRegOrImmOperands(Inst, 1);
2496       continue;
2497     } else if (Op.isImmModifier()) {
2498       OptionalIdx[Op.getImmTy()] = I;
2499     } else {
2500       assert(false);
2501     }
2502   }
2503 
2504   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2505   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2506   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2507   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2508   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2509   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2510   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2511   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2512 }
2513 
2514 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
2515   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
2516 }
2517 
2518 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
2519   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
2520 }
2521 
2522 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
2523   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
2524 }
2525 
2526 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
2527   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
2528 }
2529 
2530 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
2531   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
2532 }
2533 
2534 //===----------------------------------------------------------------------===//
2535 // smrd
2536 //===----------------------------------------------------------------------===//
2537 
2538 bool AMDGPUOperand::isSMRDOffset8() const {
2539   return isImm() && isUInt<8>(getImm());
2540 }
2541 
2542 bool AMDGPUOperand::isSMRDOffset20() const {
2543   return isImm() && isUInt<20>(getImm());
2544 }
2545 
2546 bool AMDGPUOperand::isSMRDLiteralOffset() const {
2547   // 32-bit literals are only supported on CI and we only want to use them
2548   // when the offset is > 8-bits.
2549   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
2550 }
2551 
2552 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
2553   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2554 }
2555 
2556 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
2557   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2558 }
2559 
2560 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
2561   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2562 }
2563 
2564 //===----------------------------------------------------------------------===//
2565 // vop3
2566 //===----------------------------------------------------------------------===//
2567 
2568 static bool ConvertOmodMul(int64_t &Mul) {
2569   if (Mul != 1 && Mul != 2 && Mul != 4)
2570     return false;
2571 
2572   Mul >>= 1;
2573   return true;
2574 }
2575 
2576 static bool ConvertOmodDiv(int64_t &Div) {
2577   if (Div == 1) {
2578     Div = 0;
2579     return true;
2580   }
2581 
2582   if (Div == 2) {
2583     Div = 3;
2584     return true;
2585   }
2586 
2587   return false;
2588 }
2589 
2590 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
2591   if (BoundCtrl == 0) {
2592     BoundCtrl = 1;
2593     return true;
2594   } else if (BoundCtrl == -1) {
2595     BoundCtrl = 0;
2596     return true;
2597   }
2598   return false;
2599 }
2600 
2601 // Note: the order in this table matches the order of operands in AsmString.
2602 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
2603   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
2604   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
2605   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
2606   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
2607   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
2608   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
2609   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
2610   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
2611   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
2612   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
2613   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
2614   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
2615   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
2616   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
2617   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
2618   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
2619   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
2620   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
2621   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
2622   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
2623   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
2624   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
2625   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
2626   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
2627 };
2628 
2629 AMDGPUAsmParser::OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
2630   OperandMatchResultTy res;
2631   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
2632     // try to parse any optional operand here
2633     if (Op.IsBit) {
2634       res = parseNamedBit(Op.Name, Operands, Op.Type);
2635     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
2636       res = parseOModOperand(Operands);
2637     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
2638                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
2639                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
2640       res = parseSDWASel(Operands, Op.Name, Op.Type);
2641     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
2642       res = parseSDWADstUnused(Operands);
2643     } else {
2644       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
2645     }
2646     if (res != MatchOperand_NoMatch) {
2647       return res;
2648     }
2649   }
2650   return MatchOperand_NoMatch;
2651 }
2652 
2653 AMDGPUAsmParser::OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands)
2654 {
2655   StringRef Name = Parser.getTok().getString();
2656   if (Name == "mul") {
2657     return parseIntWithPrefix("mul", Operands, AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
2658   } else if (Name == "div") {
2659     return parseIntWithPrefix("div", Operands, AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
2660   } else {
2661     return MatchOperand_NoMatch;
2662   }
2663 }
2664 
2665 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
2666   unsigned I = 1;
2667   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2668   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2669     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2670   }
2671   for (unsigned E = Operands.size(); I != E; ++I)
2672     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
2673 }
2674 
2675 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
2676   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
2677   if (TSFlags & SIInstrFlags::VOP3) {
2678     cvtVOP3(Inst, Operands);
2679   } else {
2680     cvtId(Inst, Operands);
2681   }
2682 }
2683 
2684 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
2685       // 1. This operand is input modifiers
2686   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
2687       // 2. This is not last operand
2688       && Desc.NumOperands > (OpNum + 1)
2689       // 3. Next operand is register class
2690       && Desc.OpInfo[OpNum + 1].RegClass != -1
2691       // 4. Next register is not tied to any other operand
2692       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
2693 }
2694 
2695 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
2696   OptionalImmIndexMap OptionalIdx;
2697   unsigned I = 1;
2698   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2699   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2700     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2701   }
2702 
2703   for (unsigned E = Operands.size(); I != E; ++I) {
2704     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2705     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
2706       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
2707     } else if (Op.isImm()) {
2708       OptionalIdx[Op.getImmTy()] = I;
2709     } else {
2710       assert(false);
2711     }
2712   }
2713 
2714   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
2715   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
2716 
2717   // special case v_mac_f32:
2718   // it has src2 register operand that is tied to dst operand
2719   // we don't allow modifiers for this operand in assembler so src2_modifiers
2720   // should be 0
2721   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_si ||
2722       Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_vi) {
2723     auto it = Inst.begin();
2724     std::advance(it, AMDGPU::getNamedOperandIdx(AMDGPU::V_MAC_F32_e64, AMDGPU::OpName::src2_modifiers));
2725     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
2726     ++it;
2727     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
2728   }
2729 }
2730 
2731 //===----------------------------------------------------------------------===//
2732 // dpp
2733 //===----------------------------------------------------------------------===//
2734 
2735 bool AMDGPUOperand::isDPPCtrl() const {
2736   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
2737   if (result) {
2738     int64_t Imm = getImm();
2739     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
2740            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
2741            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
2742            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
2743            (Imm == 0x130) ||
2744            (Imm == 0x134) ||
2745            (Imm == 0x138) ||
2746            (Imm == 0x13c) ||
2747            (Imm == 0x140) ||
2748            (Imm == 0x141) ||
2749            (Imm == 0x142) ||
2750            (Imm == 0x143);
2751   }
2752   return false;
2753 }
2754 
2755 bool AMDGPUOperand::isGPRIdxMode() const {
2756   return isImm() && isUInt<4>(getImm());
2757 }
2758 
2759 AMDGPUAsmParser::OperandMatchResultTy
2760 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
2761   SMLoc S = Parser.getTok().getLoc();
2762   StringRef Prefix;
2763   int64_t Int;
2764 
2765   if (getLexer().getKind() == AsmToken::Identifier) {
2766     Prefix = Parser.getTok().getString();
2767   } else {
2768     return MatchOperand_NoMatch;
2769   }
2770 
2771   if (Prefix == "row_mirror") {
2772     Int = 0x140;
2773     Parser.Lex();
2774   } else if (Prefix == "row_half_mirror") {
2775     Int = 0x141;
2776     Parser.Lex();
2777   } else {
2778     // Check to prevent parseDPPCtrlOps from eating invalid tokens
2779     if (Prefix != "quad_perm"
2780         && Prefix != "row_shl"
2781         && Prefix != "row_shr"
2782         && Prefix != "row_ror"
2783         && Prefix != "wave_shl"
2784         && Prefix != "wave_rol"
2785         && Prefix != "wave_shr"
2786         && Prefix != "wave_ror"
2787         && Prefix != "row_bcast") {
2788       return MatchOperand_NoMatch;
2789     }
2790 
2791     Parser.Lex();
2792     if (getLexer().isNot(AsmToken::Colon))
2793       return MatchOperand_ParseFail;
2794 
2795     if (Prefix == "quad_perm") {
2796       // quad_perm:[%d,%d,%d,%d]
2797       Parser.Lex();
2798       if (getLexer().isNot(AsmToken::LBrac))
2799         return MatchOperand_ParseFail;
2800       Parser.Lex();
2801 
2802       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
2803         return MatchOperand_ParseFail;
2804 
2805       for (int i = 0; i < 3; ++i) {
2806         if (getLexer().isNot(AsmToken::Comma))
2807           return MatchOperand_ParseFail;
2808         Parser.Lex();
2809 
2810         int64_t Temp;
2811         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
2812           return MatchOperand_ParseFail;
2813         const int shift = i*2 + 2;
2814         Int += (Temp << shift);
2815       }
2816 
2817       if (getLexer().isNot(AsmToken::RBrac))
2818         return MatchOperand_ParseFail;
2819       Parser.Lex();
2820 
2821     } else {
2822       // sel:%d
2823       Parser.Lex();
2824       if (getParser().parseAbsoluteExpression(Int))
2825         return MatchOperand_ParseFail;
2826 
2827       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
2828         Int |= 0x100;
2829       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
2830         Int |= 0x110;
2831       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
2832         Int |= 0x120;
2833       } else if (Prefix == "wave_shl" && 1 == Int) {
2834         Int = 0x130;
2835       } else if (Prefix == "wave_rol" && 1 == Int) {
2836         Int = 0x134;
2837       } else if (Prefix == "wave_shr" && 1 == Int) {
2838         Int = 0x138;
2839       } else if (Prefix == "wave_ror" && 1 == Int) {
2840         Int = 0x13C;
2841       } else if (Prefix == "row_bcast") {
2842         if (Int == 15) {
2843           Int = 0x142;
2844         } else if (Int == 31) {
2845           Int = 0x143;
2846         } else {
2847           return MatchOperand_ParseFail;
2848         }
2849       } else {
2850         return MatchOperand_ParseFail;
2851       }
2852     }
2853   }
2854 
2855   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
2856   return MatchOperand_Success;
2857 }
2858 
2859 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
2860   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
2861 }
2862 
2863 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
2864   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
2865 }
2866 
2867 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
2868   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
2869 }
2870 
2871 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
2872   OptionalImmIndexMap OptionalIdx;
2873 
2874   unsigned I = 1;
2875   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2876   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2877     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2878   }
2879 
2880   for (unsigned E = Operands.size(); I != E; ++I) {
2881     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2882     // Add the register arguments
2883     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
2884       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
2885     } else if (Op.isDPPCtrl()) {
2886       Op.addImmOperands(Inst, 1);
2887     } else if (Op.isImm()) {
2888       // Handle optional arguments
2889       OptionalIdx[Op.getImmTy()] = I;
2890     } else {
2891       llvm_unreachable("Invalid operand type");
2892     }
2893   }
2894 
2895   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
2896   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
2897   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
2898 
2899   // special case v_mac_f32:
2900   // it has src2 register operand that is tied to dst operand
2901   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_dpp) {
2902     auto it = Inst.begin();
2903     std::advance(it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
2904     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
2905   }
2906 }
2907 
2908 //===----------------------------------------------------------------------===//
2909 // sdwa
2910 //===----------------------------------------------------------------------===//
2911 
2912 AMDGPUAsmParser::OperandMatchResultTy
2913 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
2914                               AMDGPUOperand::ImmTy Type) {
2915   using namespace llvm::AMDGPU::SDWA;
2916 
2917   SMLoc S = Parser.getTok().getLoc();
2918   StringRef Value;
2919   AMDGPUAsmParser::OperandMatchResultTy res;
2920 
2921   res = parseStringWithPrefix(Prefix, Value);
2922   if (res != MatchOperand_Success) {
2923     return res;
2924   }
2925 
2926   int64_t Int;
2927   Int = StringSwitch<int64_t>(Value)
2928         .Case("BYTE_0", SdwaSel::BYTE_0)
2929         .Case("BYTE_1", SdwaSel::BYTE_1)
2930         .Case("BYTE_2", SdwaSel::BYTE_2)
2931         .Case("BYTE_3", SdwaSel::BYTE_3)
2932         .Case("WORD_0", SdwaSel::WORD_0)
2933         .Case("WORD_1", SdwaSel::WORD_1)
2934         .Case("DWORD", SdwaSel::DWORD)
2935         .Default(0xffffffff);
2936   Parser.Lex(); // eat last token
2937 
2938   if (Int == 0xffffffff) {
2939     return MatchOperand_ParseFail;
2940   }
2941 
2942   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
2943   return MatchOperand_Success;
2944 }
2945 
2946 AMDGPUAsmParser::OperandMatchResultTy
2947 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
2948   using namespace llvm::AMDGPU::SDWA;
2949 
2950   SMLoc S = Parser.getTok().getLoc();
2951   StringRef Value;
2952   AMDGPUAsmParser::OperandMatchResultTy res;
2953 
2954   res = parseStringWithPrefix("dst_unused", Value);
2955   if (res != MatchOperand_Success) {
2956     return res;
2957   }
2958 
2959   int64_t Int;
2960   Int = StringSwitch<int64_t>(Value)
2961         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
2962         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
2963         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
2964         .Default(0xffffffff);
2965   Parser.Lex(); // eat last token
2966 
2967   if (Int == 0xffffffff) {
2968     return MatchOperand_ParseFail;
2969   }
2970 
2971   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
2972   return MatchOperand_Success;
2973 }
2974 
2975 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
2976   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
2977 }
2978 
2979 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
2980   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
2981 }
2982 
2983 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
2984   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC);
2985 }
2986 
2987 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
2988                               uint64_t BasicInstType) {
2989   OptionalImmIndexMap OptionalIdx;
2990 
2991   unsigned I = 1;
2992   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2993   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2994     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2995   }
2996 
2997   for (unsigned E = Operands.size(); I != E; ++I) {
2998     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2999     // Add the register arguments
3000     if (BasicInstType == SIInstrFlags::VOPC &&
3001         Op.isReg() &&
3002         Op.Reg.RegNo == AMDGPU::VCC) {
3003       // VOPC sdwa use "vcc" token as dst. Skip it.
3004       continue;
3005     } else if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3006       Op.addRegOrImmWithInputModsOperands(Inst, 2);
3007     } else if (Op.isImm()) {
3008       // Handle optional arguments
3009       OptionalIdx[Op.getImmTy()] = I;
3010     } else {
3011       llvm_unreachable("Invalid operand type");
3012     }
3013   }
3014 
3015   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
3016 
3017   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa) {
3018     // V_NOP_sdwa has no optional sdwa arguments
3019     switch (BasicInstType) {
3020     case SIInstrFlags::VOP1: {
3021       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
3022       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
3023       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3024       break;
3025     }
3026     case SIInstrFlags::VOP2: {
3027       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
3028       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
3029       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3030       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, 6);
3031       break;
3032     }
3033     case SIInstrFlags::VOPC: {
3034       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3035       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, 6);
3036       break;
3037     }
3038     default:
3039       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
3040     }
3041   }
3042 
3043   // special case v_mac_f32:
3044   // it has src2 register operand that is tied to dst operand
3045   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa) {
3046     auto it = Inst.begin();
3047     std::advance(it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3048     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3049   }
3050 
3051 }
3052 
3053 /// Force static initialization.
3054 extern "C" void LLVMInitializeAMDGPUAsmParser() {
3055   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
3056   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
3057 }
3058 
3059 #define GET_REGISTER_MATCHER
3060 #define GET_MATCHER_IMPLEMENTATION
3061 #include "AMDGPUGenAsmMatcher.inc"
3062 
3063 
3064 // This fuction should be defined after auto-generated include so that we have
3065 // MatchClassKind enum defined
3066 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
3067                                                      unsigned Kind) {
3068   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
3069   // But MatchInstructionImpl() expects to meet token and fails to validate
3070   // operand. This method checks if we are given immediate operand but expect to
3071   // get corresponding token.
3072   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
3073   switch (Kind) {
3074   case MCK_addr64:
3075     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
3076   case MCK_gds:
3077     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
3078   case MCK_glc:
3079     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
3080   case MCK_idxen:
3081     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
3082   case MCK_offen:
3083     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
3084   case MCK_SSrcB32:
3085     // When operands have expression values, they will return true for isToken,
3086     // because it is not possible to distinguish between a token and an
3087     // expression at parse time. MatchInstructionImpl() will always try to
3088     // match an operand as a token, when isToken returns true, and when the
3089     // name of the expression is not a valid token, the match will fail,
3090     // so we need to handle it here.
3091     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
3092   case MCK_SSrcF32:
3093     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
3094   case MCK_SoppBrTarget:
3095     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
3096   default: return Match_InvalidOperand;
3097   }
3098 }
3099