1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/STLExtras.h"
19 #include "llvm/ADT/SmallBitVector.h"
20 #include "llvm/ADT/SmallString.h"
21 #include "llvm/ADT/StringSwitch.h"
22 #include "llvm/ADT/Twine.h"
23 #include "llvm/CodeGen/MachineValueType.h"
24 #include "llvm/MC/MCContext.h"
25 #include "llvm/MC/MCExpr.h"
26 #include "llvm/MC/MCInst.h"
27 #include "llvm/MC/MCInstrInfo.h"
28 #include "llvm/MC/MCParser/MCAsmLexer.h"
29 #include "llvm/MC/MCParser/MCAsmParser.h"
30 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
31 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
32 #include "llvm/MC/MCRegisterInfo.h"
33 #include "llvm/MC/MCStreamer.h"
34 #include "llvm/MC/MCSubtargetInfo.h"
35 #include "llvm/MC/MCSymbolELF.h"
36 #include "llvm/Support/Debug.h"
37 #include "llvm/Support/ELF.h"
38 #include "llvm/Support/SourceMgr.h"
39 #include "llvm/Support/TargetRegistry.h"
40 #include "llvm/Support/raw_ostream.h"
41 #include "llvm/Support/MathExtras.h"
42 
43 using namespace llvm;
44 using namespace llvm::AMDGPU;
45 
46 namespace {
47 
48 class AMDGPUAsmParser;
49 struct OptionalOperand;
50 
51 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
52 
53 //===----------------------------------------------------------------------===//
54 // Operand
55 //===----------------------------------------------------------------------===//
56 
57 class AMDGPUOperand : public MCParsedAsmOperand {
58   enum KindTy {
59     Token,
60     Immediate,
61     Register,
62     Expression
63   } Kind;
64 
65   SMLoc StartLoc, EndLoc;
66   const AMDGPUAsmParser *AsmParser;
67 
68 public:
69   AMDGPUOperand(enum KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
70     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
71 
72   typedef std::unique_ptr<AMDGPUOperand> Ptr;
73 
74   struct Modifiers {
75     bool Abs;
76     bool Neg;
77     bool Sext;
78 
79     bool hasFPModifiers() const { return Abs || Neg; }
80     bool hasIntModifiers() const { return Sext; }
81     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
82 
83     int64_t getFPModifiersOperand() const {
84       int64_t Operand = 0;
85       Operand |= Abs ? SISrcMods::ABS : 0;
86       Operand |= Neg ? SISrcMods::NEG : 0;
87       return Operand;
88     }
89 
90     int64_t getIntModifiersOperand() const {
91       int64_t Operand = 0;
92       Operand |= Sext ? SISrcMods::SEXT : 0;
93       return Operand;
94     }
95 
96     int64_t getModifiersOperand() const {
97       assert(!(hasFPModifiers() && hasIntModifiers())
98            && "fp and int modifiers should not be used simultaneously");
99       if (hasFPModifiers()) {
100         return getFPModifiersOperand();
101       } else if (hasIntModifiers()) {
102         return getIntModifiersOperand();
103       } else {
104         return 0;
105       }
106     }
107 
108     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
109   };
110 
111   enum ImmTy {
112     ImmTyNone,
113     ImmTyGDS,
114     ImmTyOffen,
115     ImmTyIdxen,
116     ImmTyAddr64,
117     ImmTyOffset,
118     ImmTyOffset0,
119     ImmTyOffset1,
120     ImmTyGLC,
121     ImmTySLC,
122     ImmTyTFE,
123     ImmTyClampSI,
124     ImmTyOModSI,
125     ImmTyDppCtrl,
126     ImmTyDppRowMask,
127     ImmTyDppBankMask,
128     ImmTyDppBoundCtrl,
129     ImmTySdwaDstSel,
130     ImmTySdwaSrc0Sel,
131     ImmTySdwaSrc1Sel,
132     ImmTySdwaDstUnused,
133     ImmTyDMask,
134     ImmTyUNorm,
135     ImmTyDA,
136     ImmTyR128,
137     ImmTyLWE,
138     ImmTyHwreg,
139     ImmTySendMsg,
140   };
141 
142   struct TokOp {
143     const char *Data;
144     unsigned Length;
145   };
146 
147   struct ImmOp {
148     int64_t Val;
149     ImmTy Type;
150     bool IsFPImm;
151     Modifiers Mods;
152   };
153 
154   struct RegOp {
155     unsigned RegNo;
156     bool IsForcedVOP3;
157     Modifiers Mods;
158   };
159 
160   union {
161     TokOp Tok;
162     ImmOp Imm;
163     RegOp Reg;
164     const MCExpr *Expr;
165   };
166 
167   bool isToken() const override {
168     if (Kind == Token)
169       return true;
170 
171     if (Kind != Expression || !Expr)
172       return false;
173 
174     // When parsing operands, we can't always tell if something was meant to be
175     // a token, like 'gds', or an expression that references a global variable.
176     // In this case, we assume the string is an expression, and if we need to
177     // interpret is a token, then we treat the symbol name as the token.
178     return isa<MCSymbolRefExpr>(Expr);
179   }
180 
181   bool isImm() const override {
182     return Kind == Immediate;
183   }
184 
185   bool isInlinableImm(MVT type) const;
186   bool isLiteralImm(MVT type) const;
187 
188   bool isRegKind() const {
189     return Kind == Register;
190   }
191 
192   bool isReg() const override {
193     return isRegKind() && !Reg.Mods.hasModifiers();
194   }
195 
196   bool isRegOrImmWithInputMods(MVT type) const {
197     return isRegKind() || isInlinableImm(type);
198   }
199 
200   bool isRegOrImmWithInt32InputMods() const {
201     return isRegOrImmWithInputMods(MVT::i32);
202   }
203 
204   bool isRegOrImmWithInt64InputMods() const {
205     return isRegOrImmWithInputMods(MVT::i64);
206   }
207 
208   bool isRegOrImmWithFP32InputMods() const {
209     return isRegOrImmWithInputMods(MVT::f32);
210   }
211 
212   bool isRegOrImmWithFP64InputMods() const {
213     return isRegOrImmWithInputMods(MVT::f64);
214   }
215 
216   bool isImmTy(ImmTy ImmT) const {
217     return isImm() && Imm.Type == ImmT;
218   }
219 
220   bool isImmModifier() const {
221     return isImm() && Imm.Type != ImmTyNone;
222   }
223 
224   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
225   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
226   bool isDMask() const { return isImmTy(ImmTyDMask); }
227   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
228   bool isDA() const { return isImmTy(ImmTyDA); }
229   bool isR128() const { return isImmTy(ImmTyUNorm); }
230   bool isLWE() const { return isImmTy(ImmTyLWE); }
231   bool isOffen() const { return isImmTy(ImmTyOffen); }
232   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
233   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
234   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
235   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
236   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
237   bool isGDS() const { return isImmTy(ImmTyGDS); }
238   bool isGLC() const { return isImmTy(ImmTyGLC); }
239   bool isSLC() const { return isImmTy(ImmTySLC); }
240   bool isTFE() const { return isImmTy(ImmTyTFE); }
241   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
242   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
243   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
244   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
245   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
246   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
247   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
248 
249   bool isMod() const {
250     return isClampSI() || isOModSI();
251   }
252 
253   bool isRegOrImm() const {
254     return isReg() || isImm();
255   }
256 
257   bool isRegClass(unsigned RCID) const;
258 
259   bool isSCSrcB32() const {
260     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::i32);
261   }
262 
263   bool isSCSrcB64() const {
264     return isRegClass(AMDGPU::SReg_64RegClassID) || isInlinableImm(MVT::i64);
265   }
266 
267   bool isSCSrcF32() const {
268     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::f32);
269   }
270 
271   bool isSCSrcF64() const {
272     return isRegClass(AMDGPU::SReg_64RegClassID) || isInlinableImm(MVT::f64);
273   }
274 
275   bool isSSrcB32() const {
276     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
277   }
278 
279   bool isSSrcB64() const {
280     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
281     // See isVSrc64().
282     return isSCSrcB64() || isLiteralImm(MVT::i64);
283   }
284 
285   bool isSSrcF32() const {
286     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
287   }
288 
289   bool isSSrcF64() const {
290     return isSCSrcB64() || isLiteralImm(MVT::f64);
291   }
292 
293   bool isVCSrcB32() const {
294     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::i32);
295   }
296 
297   bool isVCSrcB64() const {
298     return isRegClass(AMDGPU::VS_64RegClassID) || isInlinableImm(MVT::i64);
299   }
300 
301   bool isVCSrcF32() const {
302     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::f32);
303   }
304 
305   bool isVCSrcF64() const {
306     return isRegClass(AMDGPU::VS_64RegClassID) || isInlinableImm(MVT::f64);
307   }
308 
309   bool isVSrcB32() const {
310     return isVCSrcF32() || isLiteralImm(MVT::i32);
311   }
312 
313   bool isVSrcB64() const {
314     return isVCSrcF64() || isLiteralImm(MVT::i64);
315   }
316 
317   bool isVSrcF32() const {
318     return isVCSrcF32() || isLiteralImm(MVT::f32);
319   }
320 
321   bool isVSrcF64() const {
322     return isVCSrcF64() || isLiteralImm(MVT::f64);
323   }
324 
325   bool isKImmFP32() const {
326     return isLiteralImm(MVT::f32);
327   }
328 
329   bool isMem() const override {
330     return false;
331   }
332 
333   bool isExpr() const {
334     return Kind == Expression;
335   }
336 
337   bool isSoppBrTarget() const {
338     return isExpr() || isImm();
339   }
340 
341   bool isSWaitCnt() const;
342   bool isHwreg() const;
343   bool isSendMsg() const;
344   bool isSMRDOffset() const;
345   bool isSMRDLiteralOffset() const;
346   bool isDPPCtrl() const;
347 
348   StringRef getExpressionAsToken() const {
349     assert(isExpr());
350     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
351     return S->getSymbol().getName();
352   }
353 
354 
355   StringRef getToken() const {
356     assert(isToken());
357 
358     if (Kind == Expression)
359       return getExpressionAsToken();
360 
361     return StringRef(Tok.Data, Tok.Length);
362   }
363 
364   int64_t getImm() const {
365     assert(isImm());
366     return Imm.Val;
367   }
368 
369   enum ImmTy getImmTy() const {
370     assert(isImm());
371     return Imm.Type;
372   }
373 
374   unsigned getReg() const override {
375     return Reg.RegNo;
376   }
377 
378   SMLoc getStartLoc() const override {
379     return StartLoc;
380   }
381 
382   SMLoc getEndLoc() const override {
383     return EndLoc;
384   }
385 
386   Modifiers getModifiers() const {
387     assert(isRegKind() || isImmTy(ImmTyNone));
388     return isRegKind() ? Reg.Mods : Imm.Mods;
389   }
390 
391   void setModifiers(Modifiers Mods) {
392     assert(isRegKind() || isImmTy(ImmTyNone));
393     if (isRegKind())
394       Reg.Mods = Mods;
395     else
396       Imm.Mods = Mods;
397   }
398 
399   bool hasModifiers() const {
400     return getModifiers().hasModifiers();
401   }
402 
403   bool hasFPModifiers() const {
404     return getModifiers().hasFPModifiers();
405   }
406 
407   bool hasIntModifiers() const {
408     return getModifiers().hasIntModifiers();
409   }
410 
411   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
412 
413   void addLiteralImmOperand(MCInst &Inst, int64_t Val) const;
414 
415   void addKImmFP32Operands(MCInst &Inst, unsigned N) const;
416 
417   void addRegOperands(MCInst &Inst, unsigned N) const;
418 
419   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
420     if (isRegKind())
421       addRegOperands(Inst, N);
422     else if (isExpr())
423       Inst.addOperand(MCOperand::createExpr(Expr));
424     else
425       addImmOperands(Inst, N);
426   }
427 
428   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
429     Modifiers Mods = getModifiers();
430     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
431     if (isRegKind()) {
432       addRegOperands(Inst, N);
433     } else {
434       addImmOperands(Inst, N, false);
435     }
436   }
437 
438   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
439     assert(!hasIntModifiers());
440     addRegOrImmWithInputModsOperands(Inst, N);
441   }
442 
443   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
444     assert(!hasFPModifiers());
445     addRegOrImmWithInputModsOperands(Inst, N);
446   }
447 
448   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
449     if (isImm())
450       addImmOperands(Inst, N);
451     else {
452       assert(isExpr());
453       Inst.addOperand(MCOperand::createExpr(Expr));
454     }
455   }
456 
457   void printImmTy(raw_ostream& OS, ImmTy Type) const {
458     switch (Type) {
459     case ImmTyNone: OS << "None"; break;
460     case ImmTyGDS: OS << "GDS"; break;
461     case ImmTyOffen: OS << "Offen"; break;
462     case ImmTyIdxen: OS << "Idxen"; break;
463     case ImmTyAddr64: OS << "Addr64"; break;
464     case ImmTyOffset: OS << "Offset"; break;
465     case ImmTyOffset0: OS << "Offset0"; break;
466     case ImmTyOffset1: OS << "Offset1"; break;
467     case ImmTyGLC: OS << "GLC"; break;
468     case ImmTySLC: OS << "SLC"; break;
469     case ImmTyTFE: OS << "TFE"; break;
470     case ImmTyClampSI: OS << "ClampSI"; break;
471     case ImmTyOModSI: OS << "OModSI"; break;
472     case ImmTyDppCtrl: OS << "DppCtrl"; break;
473     case ImmTyDppRowMask: OS << "DppRowMask"; break;
474     case ImmTyDppBankMask: OS << "DppBankMask"; break;
475     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
476     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
477     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
478     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
479     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
480     case ImmTyDMask: OS << "DMask"; break;
481     case ImmTyUNorm: OS << "UNorm"; break;
482     case ImmTyDA: OS << "DA"; break;
483     case ImmTyR128: OS << "R128"; break;
484     case ImmTyLWE: OS << "LWE"; break;
485     case ImmTyHwreg: OS << "Hwreg"; break;
486     case ImmTySendMsg: OS << "SendMsg"; break;
487     }
488   }
489 
490   void print(raw_ostream &OS) const override {
491     switch (Kind) {
492     case Register:
493       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
494       break;
495     case Immediate:
496       OS << '<' << getImm();
497       if (getImmTy() != ImmTyNone) {
498         OS << " type: "; printImmTy(OS, getImmTy());
499       }
500       OS << " mods: " << Imm.Mods << '>';
501       break;
502     case Token:
503       OS << '\'' << getToken() << '\'';
504       break;
505     case Expression:
506       OS << "<expr " << *Expr << '>';
507       break;
508     }
509   }
510 
511   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
512                                       int64_t Val, SMLoc Loc,
513                                       enum ImmTy Type = ImmTyNone,
514                                       bool IsFPImm = false) {
515     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
516     Op->Imm.Val = Val;
517     Op->Imm.IsFPImm = IsFPImm;
518     Op->Imm.Type = Type;
519     Op->Imm.Mods = {false, false, false};
520     Op->StartLoc = Loc;
521     Op->EndLoc = Loc;
522     return Op;
523   }
524 
525   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
526                                         StringRef Str, SMLoc Loc,
527                                         bool HasExplicitEncodingSize = true) {
528     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
529     Res->Tok.Data = Str.data();
530     Res->Tok.Length = Str.size();
531     Res->StartLoc = Loc;
532     Res->EndLoc = Loc;
533     return Res;
534   }
535 
536   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
537                                       unsigned RegNo, SMLoc S,
538                                       SMLoc E,
539                                       bool ForceVOP3) {
540     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
541     Op->Reg.RegNo = RegNo;
542     Op->Reg.Mods = {false, false, false};
543     Op->Reg.IsForcedVOP3 = ForceVOP3;
544     Op->StartLoc = S;
545     Op->EndLoc = E;
546     return Op;
547   }
548 
549   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
550                                        const class MCExpr *Expr, SMLoc S) {
551     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
552     Op->Expr = Expr;
553     Op->StartLoc = S;
554     Op->EndLoc = S;
555     return Op;
556   }
557 };
558 
559 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
560   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
561   return OS;
562 }
563 
564 //===----------------------------------------------------------------------===//
565 // AsmParser
566 //===----------------------------------------------------------------------===//
567 
568 class AMDGPUAsmParser : public MCTargetAsmParser {
569   const MCInstrInfo &MII;
570   MCAsmParser &Parser;
571 
572   unsigned ForcedEncodingSize;
573   bool ForcedDPP;
574   bool ForcedSDWA;
575 
576   /// @name Auto-generated Match Functions
577   /// {
578 
579 #define GET_ASSEMBLER_HEADER
580 #include "AMDGPUGenAsmMatcher.inc"
581 
582   /// }
583 
584 private:
585   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
586   bool ParseDirectiveHSACodeObjectVersion();
587   bool ParseDirectiveHSACodeObjectISA();
588   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
589   bool ParseDirectiveAMDKernelCodeT();
590   bool ParseSectionDirectiveHSAText();
591   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
592   bool ParseDirectiveAMDGPUHsaKernel();
593   bool ParseDirectiveAMDGPUHsaModuleGlobal();
594   bool ParseDirectiveAMDGPUHsaProgramGlobal();
595   bool ParseSectionDirectiveHSADataGlobalAgent();
596   bool ParseSectionDirectiveHSADataGlobalProgram();
597   bool ParseSectionDirectiveHSARodataReadonlyAgent();
598   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum);
599   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth);
600   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands, bool IsAtomic, bool IsAtomicReturn);
601 
602 public:
603   enum AMDGPUMatchResultTy {
604     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
605   };
606 
607   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
608                const MCInstrInfo &MII,
609                const MCTargetOptions &Options)
610       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser),
611         ForcedEncodingSize(0),
612         ForcedDPP(false),
613         ForcedSDWA(false) {
614     MCAsmParserExtension::Initialize(Parser);
615 
616     if (getSTI().getFeatureBits().none()) {
617       // Set default features.
618       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
619     }
620 
621     setAvailableFeatures(ComputeAvailableFeatures(getSTI().getFeatureBits()));
622 
623     {
624       // TODO: make those pre-defined variables read-only.
625       // Currently there is none suitable machinery in the core llvm-mc for this.
626       // MCSymbol::isRedefinable is intended for another purpose, and
627       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
628       AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
629       MCContext &Ctx = getContext();
630       MCSymbol *Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
631       Sym->setVariableValue(MCConstantExpr::create(Isa.Major, Ctx));
632       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
633       Sym->setVariableValue(MCConstantExpr::create(Isa.Minor, Ctx));
634       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
635       Sym->setVariableValue(MCConstantExpr::create(Isa.Stepping, Ctx));
636     }
637   }
638 
639   bool isSI() const {
640     return AMDGPU::isSI(getSTI());
641   }
642 
643   bool isCI() const {
644     return AMDGPU::isCI(getSTI());
645   }
646 
647   bool isVI() const {
648     return AMDGPU::isVI(getSTI());
649   }
650 
651   bool hasSGPR102_SGPR103() const {
652     return !isVI();
653   }
654 
655   AMDGPUTargetStreamer &getTargetStreamer() {
656     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
657     return static_cast<AMDGPUTargetStreamer &>(TS);
658   }
659 
660   const MCRegisterInfo *getMRI() const {
661     // We need this const_cast because for some reason getContext() is not const
662     // in MCAsmParser.
663     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
664   }
665 
666   const MCInstrInfo *getMII() const {
667     return &MII;
668   }
669 
670   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
671   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
672   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
673 
674   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
675   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
676   bool isForcedDPP() const { return ForcedDPP; }
677   bool isForcedSDWA() const { return ForcedSDWA; }
678 
679   std::unique_ptr<AMDGPUOperand> parseRegister();
680   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
681   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
682   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
683                                       unsigned Kind) override;
684   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
685                                OperandVector &Operands, MCStreamer &Out,
686                                uint64_t &ErrorInfo,
687                                bool MatchingInlineAsm) override;
688   bool ParseDirective(AsmToken DirectiveID) override;
689   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
690   StringRef parseMnemonicSuffix(StringRef Name);
691   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
692                         SMLoc NameLoc, OperandVector &Operands) override;
693   //bool ProcessInstruction(MCInst &Inst);
694 
695   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
696   OperandMatchResultTy parseIntWithPrefix(const char *Prefix,
697                                           OperandVector &Operands,
698                                           enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
699                                           bool (*ConvertResult)(int64_t&) = 0);
700   OperandMatchResultTy parseNamedBit(const char *Name, OperandVector &Operands,
701                                      enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
702   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix, StringRef &Value);
703 
704   OperandMatchResultTy parseImm(OperandVector &Operands);
705   OperandMatchResultTy parseRegOrImm(OperandVector &Operands);
706   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands);
707   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands);
708 
709   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
710   void cvtDS(MCInst &Inst, const OperandVector &Operands);
711 
712   bool parseCnt(int64_t &IntVal);
713   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
714   OperandMatchResultTy parseHwreg(OperandVector &Operands);
715 
716 private:
717   struct OperandInfoTy {
718     int64_t Id;
719     bool IsSymbolic;
720     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
721   };
722 
723   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
724   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
725 public:
726   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
727 
728   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
729   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
730 
731   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
732   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
733   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
734   AMDGPUOperand::Ptr defaultGLC() const;
735   AMDGPUOperand::Ptr defaultSLC() const;
736   AMDGPUOperand::Ptr defaultTFE() const;
737 
738   AMDGPUOperand::Ptr defaultDMask() const;
739   AMDGPUOperand::Ptr defaultUNorm() const;
740   AMDGPUOperand::Ptr defaultDA() const;
741   AMDGPUOperand::Ptr defaultR128() const;
742   AMDGPUOperand::Ptr defaultLWE() const;
743   AMDGPUOperand::Ptr defaultSMRDOffset() const;
744   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
745 
746   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
747 
748   void cvtId(MCInst &Inst, const OperandVector &Operands);
749   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
750   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
751 
752   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
753   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
754 
755   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
756   AMDGPUOperand::Ptr defaultRowMask() const;
757   AMDGPUOperand::Ptr defaultBankMask() const;
758   AMDGPUOperand::Ptr defaultBoundCtrl() const;
759   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
760 
761   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
762                                     AMDGPUOperand::ImmTy Type);
763   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
764   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
765   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
766   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
767   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
768                uint64_t BasicInstType);
769 };
770 
771 struct OptionalOperand {
772   const char *Name;
773   AMDGPUOperand::ImmTy Type;
774   bool IsBit;
775   bool (*ConvertResult)(int64_t&);
776 };
777 
778 }
779 
780 //===----------------------------------------------------------------------===//
781 // Operand
782 //===----------------------------------------------------------------------===//
783 
784 bool AMDGPUOperand::isInlinableImm(MVT type) const {
785   if (!isImmTy(ImmTyNone)) {
786     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
787     return false;
788   }
789   // TODO: We should avoid using host float here. It would be better to
790   // check the float bit values which is what a few other places do.
791   // We've had bot failures before due to weird NaN support on mips hosts.
792 
793   APInt Literal(64, Imm.Val);
794 
795   if (Imm.IsFPImm) { // We got fp literal token
796     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
797       return AMDGPU::isInlinableLiteral64(Imm.Val, AsmParser->isVI());
798     } else { // Expected 32-bit operand
799       bool lost;
800       APFloat FPLiteral(APFloat::IEEEdouble, Literal);
801       // Convert literal to single precision
802       APFloat::opStatus status = FPLiteral.convert(APFloat::IEEEsingle,
803                                                     APFloat::rmNearestTiesToEven,
804                                                     &lost);
805       // We allow precision lost but not overflow or underflow
806       if (status != APFloat::opOK &&
807           lost &&
808           ((status & APFloat::opOverflow)  != 0 ||
809             (status & APFloat::opUnderflow) != 0)) {
810         return false;
811       }
812       // Check if single precision literal is inlinable
813       return AMDGPU::isInlinableLiteral32(
814               static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
815               AsmParser->isVI());
816     }
817   } else { // We got int literal token
818     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
819       return AMDGPU::isInlinableLiteral64(Imm.Val, AsmParser->isVI());
820     } else { // Expected 32-bit operand
821       return AMDGPU::isInlinableLiteral32(
822             static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
823             AsmParser->isVI());
824     }
825   }
826   return false;
827 }
828 
829 bool AMDGPUOperand::isLiteralImm(MVT type) const {
830   // Check that this imediate can be added as literal
831   if (!isImmTy(ImmTyNone)) {
832     return false;
833   }
834 
835   APInt Literal(64, Imm.Val);
836 
837   if (Imm.IsFPImm) { // We got fp literal token
838     if (type == MVT::f64) { // Expected 64-bit fp operand
839       // We would set low 64-bits of literal to zeroes but we accept this literals
840       return true;
841     } else if (type == MVT::i64) { // Expected 64-bit int operand
842         // We don't allow fp literals in 64-bit integer instructions. It is
843         // unclear how we should encode them.
844       return false;
845     } else { // Expected 32-bit operand
846       bool lost;
847       APFloat FPLiteral(APFloat::IEEEdouble, Literal);
848       // Convert literal to single precision
849       APFloat::opStatus status = FPLiteral.convert(APFloat::IEEEsingle,
850                                                     APFloat::rmNearestTiesToEven,
851                                                     &lost);
852       // We allow precision lost but not overflow or underflow
853       if (status != APFloat::opOK &&
854           lost &&
855           ((status & APFloat::opOverflow)  != 0 ||
856             (status & APFloat::opUnderflow) != 0)) {
857         return false;
858       }
859       return true;
860     }
861   } else { // We got int literal token
862     APInt HiBits = Literal.getHiBits(32);
863     if (HiBits == 0xffffffff &&
864         (*Literal.getLoBits(32).getRawData() & 0x80000000) != 0) {
865       // If high 32 bits aren't zeroes then they all should be ones and 32nd
866       // bit should be set. So that this 64-bit literal is sign-extension of
867       // 32-bit value.
868       return true;
869     } else if (HiBits == 0) {
870       return true;
871     }
872   }
873   return false;
874 }
875 
876 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
877   return isReg() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
878 }
879 
880 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
881   int64_t Val = Imm.Val;
882   if (isImmTy(ImmTyNone) && ApplyModifiers && Imm.Mods.hasFPModifiers() && Imm.Mods.Neg) {
883     // Apply modifiers to immediate value. Only negate can get here
884     if (Imm.IsFPImm) {
885       APFloat F(BitsToDouble(Val));
886       F.changeSign();
887       Val = F.bitcastToAPInt().getZExtValue();
888     } else {
889       Val = -Val;
890     }
891   }
892 
893   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()), Inst.getNumOperands())) {
894     addLiteralImmOperand(Inst, Val);
895   } else {
896     Inst.addOperand(MCOperand::createImm(Val));
897   }
898 }
899 
900 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val) const {
901   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
902   auto OpNum = Inst.getNumOperands();
903   // Check that this operand accepts literals
904   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
905 
906   APInt Literal(64, Val);
907   auto OpSize = AMDGPU::getRegOperandSize(AsmParser->getMRI(), InstDesc, OpNum); // expected operand size
908 
909   if (Imm.IsFPImm) { // We got fp literal token
910     if (OpSize == 8) { // Expected 64-bit operand
911       // Check if literal is inlinable
912       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(), AsmParser->isVI())) {
913         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
914       } else if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
915         // For fp operands we check if low 32 bits are zeros
916         if (Literal.getLoBits(32) != 0) {
917           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
918                             "Can't encode literal as exact 64-bit"
919                             " floating-point operand. Low 32-bits will be"
920                             " set to zero");
921         }
922         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
923       } else {
924         // We don't allow fp literals in 64-bit integer instructions. It is
925         // unclear how we should encode them. This case should be checked earlier
926         // in predicate methods (isLiteralImm())
927         llvm_unreachable("fp literal in 64-bit integer instruction.");
928       }
929     } else { // Expected 32-bit operand
930       bool lost;
931       APFloat FPLiteral(APFloat::IEEEdouble, Literal);
932       // Convert literal to single precision
933       FPLiteral.convert(APFloat::IEEEsingle, APFloat::rmNearestTiesToEven, &lost);
934       // We allow precision lost but not overflow or underflow. This should be
935       // checked earlier in isLiteralImm()
936       Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
937     }
938   } else { // We got int literal token
939     if (OpSize == 8) { // Expected 64-bit operand
940       auto LiteralVal = Literal.getZExtValue();
941       if (AMDGPU::isInlinableLiteral64(LiteralVal, AsmParser->isVI())) {
942         Inst.addOperand(MCOperand::createImm(LiteralVal));
943         return;
944       }
945     } else { // Expected 32-bit operand
946       auto LiteralVal = static_cast<int32_t>(Literal.getLoBits(32).getZExtValue());
947       if (AMDGPU::isInlinableLiteral32(LiteralVal, AsmParser->isVI())) {
948         Inst.addOperand(MCOperand::createImm(LiteralVal));
949         return;
950       }
951     }
952     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(32).getZExtValue()));
953   }
954 }
955 
956 void AMDGPUOperand::addKImmFP32Operands(MCInst &Inst, unsigned N) const {
957   APInt Literal(64, Imm.Val);
958   if (Imm.IsFPImm) { // We got fp literal
959     bool lost;
960     APFloat FPLiteral(APFloat::IEEEdouble, Literal);
961     FPLiteral.convert(APFloat::IEEEsingle, APFloat::rmNearestTiesToEven, &lost);
962     Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
963   } else { // We got int literal token
964     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(32).getZExtValue()));
965   }
966 }
967 
968 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
969   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
970 }
971 
972 //===----------------------------------------------------------------------===//
973 // AsmParser
974 //===----------------------------------------------------------------------===//
975 
976 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
977   if (Is == IS_VGPR) {
978     switch (RegWidth) {
979       default: return -1;
980       case 1: return AMDGPU::VGPR_32RegClassID;
981       case 2: return AMDGPU::VReg_64RegClassID;
982       case 3: return AMDGPU::VReg_96RegClassID;
983       case 4: return AMDGPU::VReg_128RegClassID;
984       case 8: return AMDGPU::VReg_256RegClassID;
985       case 16: return AMDGPU::VReg_512RegClassID;
986     }
987   } else if (Is == IS_TTMP) {
988     switch (RegWidth) {
989       default: return -1;
990       case 1: return AMDGPU::TTMP_32RegClassID;
991       case 2: return AMDGPU::TTMP_64RegClassID;
992       case 4: return AMDGPU::TTMP_128RegClassID;
993     }
994   } else if (Is == IS_SGPR) {
995     switch (RegWidth) {
996       default: return -1;
997       case 1: return AMDGPU::SGPR_32RegClassID;
998       case 2: return AMDGPU::SGPR_64RegClassID;
999       case 4: return AMDGPU::SGPR_128RegClassID;
1000       case 8: return AMDGPU::SReg_256RegClassID;
1001       case 16: return AMDGPU::SReg_512RegClassID;
1002     }
1003   }
1004   return -1;
1005 }
1006 
1007 static unsigned getSpecialRegForName(StringRef RegName) {
1008   return StringSwitch<unsigned>(RegName)
1009     .Case("exec", AMDGPU::EXEC)
1010     .Case("vcc", AMDGPU::VCC)
1011     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1012     .Case("m0", AMDGPU::M0)
1013     .Case("scc", AMDGPU::SCC)
1014     .Case("tba", AMDGPU::TBA)
1015     .Case("tma", AMDGPU::TMA)
1016     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1017     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1018     .Case("vcc_lo", AMDGPU::VCC_LO)
1019     .Case("vcc_hi", AMDGPU::VCC_HI)
1020     .Case("exec_lo", AMDGPU::EXEC_LO)
1021     .Case("exec_hi", AMDGPU::EXEC_HI)
1022     .Case("tma_lo", AMDGPU::TMA_LO)
1023     .Case("tma_hi", AMDGPU::TMA_HI)
1024     .Case("tba_lo", AMDGPU::TBA_LO)
1025     .Case("tba_hi", AMDGPU::TBA_HI)
1026     .Default(0);
1027 }
1028 
1029 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) {
1030   auto R = parseRegister();
1031   if (!R) return true;
1032   assert(R->isReg());
1033   RegNo = R->getReg();
1034   StartLoc = R->getStartLoc();
1035   EndLoc = R->getEndLoc();
1036   return false;
1037 }
1038 
1039 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum)
1040 {
1041   switch (RegKind) {
1042   case IS_SPECIAL:
1043     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) { Reg = AMDGPU::EXEC; RegWidth = 2; return true; }
1044     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) { Reg = AMDGPU::FLAT_SCR; RegWidth = 2; return true; }
1045     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) { Reg = AMDGPU::VCC; RegWidth = 2; return true; }
1046     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) { Reg = AMDGPU::TBA; RegWidth = 2; return true; }
1047     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) { Reg = AMDGPU::TMA; RegWidth = 2; return true; }
1048     return false;
1049   case IS_VGPR:
1050   case IS_SGPR:
1051   case IS_TTMP:
1052     if (Reg1 != Reg + RegWidth) { return false; }
1053     RegWidth++;
1054     return true;
1055   default:
1056     assert(false); return false;
1057   }
1058 }
1059 
1060 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth)
1061 {
1062   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1063   if (getLexer().is(AsmToken::Identifier)) {
1064     StringRef RegName = Parser.getTok().getString();
1065     if ((Reg = getSpecialRegForName(RegName))) {
1066       Parser.Lex();
1067       RegKind = IS_SPECIAL;
1068     } else {
1069       unsigned RegNumIndex = 0;
1070       if (RegName[0] == 'v') {
1071         RegNumIndex = 1;
1072         RegKind = IS_VGPR;
1073       } else if (RegName[0] == 's') {
1074         RegNumIndex = 1;
1075         RegKind = IS_SGPR;
1076       } else if (RegName.startswith("ttmp")) {
1077         RegNumIndex = strlen("ttmp");
1078         RegKind = IS_TTMP;
1079       } else {
1080         return false;
1081       }
1082       if (RegName.size() > RegNumIndex) {
1083         // Single 32-bit register: vXX.
1084         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1085           return false;
1086         Parser.Lex();
1087         RegWidth = 1;
1088       } else {
1089         // Range of registers: v[XX:YY]. ":YY" is optional.
1090         Parser.Lex();
1091         int64_t RegLo, RegHi;
1092         if (getLexer().isNot(AsmToken::LBrac))
1093           return false;
1094         Parser.Lex();
1095 
1096         if (getParser().parseAbsoluteExpression(RegLo))
1097           return false;
1098 
1099         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1100         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1101           return false;
1102         Parser.Lex();
1103 
1104         if (isRBrace) {
1105           RegHi = RegLo;
1106         } else {
1107           if (getParser().parseAbsoluteExpression(RegHi))
1108             return false;
1109 
1110           if (getLexer().isNot(AsmToken::RBrac))
1111             return false;
1112           Parser.Lex();
1113         }
1114         RegNum = (unsigned) RegLo;
1115         RegWidth = (RegHi - RegLo) + 1;
1116       }
1117     }
1118   } else if (getLexer().is(AsmToken::LBrac)) {
1119     // List of consecutive registers: [s0,s1,s2,s3]
1120     Parser.Lex();
1121     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth))
1122       return false;
1123     if (RegWidth != 1)
1124       return false;
1125     RegisterKind RegKind1;
1126     unsigned Reg1, RegNum1, RegWidth1;
1127     do {
1128       if (getLexer().is(AsmToken::Comma)) {
1129         Parser.Lex();
1130       } else if (getLexer().is(AsmToken::RBrac)) {
1131         Parser.Lex();
1132         break;
1133       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1)) {
1134         if (RegWidth1 != 1) {
1135           return false;
1136         }
1137         if (RegKind1 != RegKind) {
1138           return false;
1139         }
1140         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1141           return false;
1142         }
1143       } else {
1144         return false;
1145       }
1146     } while (true);
1147   } else {
1148     return false;
1149   }
1150   switch (RegKind) {
1151   case IS_SPECIAL:
1152     RegNum = 0;
1153     RegWidth = 1;
1154     break;
1155   case IS_VGPR:
1156   case IS_SGPR:
1157   case IS_TTMP:
1158   {
1159     unsigned Size = 1;
1160     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1161       // SGPR and TTMP registers must be are aligned. Max required alignment is 4 dwords.
1162       Size = std::min(RegWidth, 4u);
1163     }
1164     if (RegNum % Size != 0)
1165       return false;
1166     RegNum = RegNum / Size;
1167     int RCID = getRegClass(RegKind, RegWidth);
1168     if (RCID == -1)
1169       return false;
1170     const MCRegisterClass RC = TRI->getRegClass(RCID);
1171     if (RegNum >= RC.getNumRegs())
1172       return false;
1173     Reg = RC.getRegister(RegNum);
1174     break;
1175   }
1176 
1177   default:
1178     assert(false); return false;
1179   }
1180 
1181   if (!subtargetHasRegister(*TRI, Reg))
1182     return false;
1183   return true;
1184 }
1185 
1186 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1187   const auto &Tok = Parser.getTok();
1188   SMLoc StartLoc = Tok.getLoc();
1189   SMLoc EndLoc = Tok.getEndLoc();
1190   RegisterKind RegKind;
1191   unsigned Reg, RegNum, RegWidth;
1192 
1193   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth)) {
1194     return nullptr;
1195   }
1196   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1197 }
1198 
1199 AMDGPUAsmParser::OperandMatchResultTy
1200 AMDGPUAsmParser::parseImm(OperandVector &Operands) {
1201   // TODO: add syntactic sugar for 1/(2*PI)
1202   bool Minus = false;
1203   if (getLexer().getKind() == AsmToken::Minus) {
1204     Minus = true;
1205     Parser.Lex();
1206   }
1207 
1208   SMLoc S = Parser.getTok().getLoc();
1209   switch(getLexer().getKind()) {
1210   case AsmToken::Integer: {
1211     int64_t IntVal;
1212     if (getParser().parseAbsoluteExpression(IntVal))
1213       return MatchOperand_ParseFail;
1214     if (Minus)
1215       IntVal *= -1;
1216     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1217     return MatchOperand_Success;
1218   }
1219   case AsmToken::Real: {
1220     int64_t IntVal;
1221     if (getParser().parseAbsoluteExpression(IntVal))
1222       return MatchOperand_ParseFail;
1223 
1224     APFloat F(BitsToDouble(IntVal));
1225     if (Minus)
1226       F.changeSign();
1227     Operands.push_back(
1228         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1229                                  AMDGPUOperand::ImmTyNone, true));
1230     return MatchOperand_Success;
1231   }
1232   default:
1233     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1234   }
1235 }
1236 
1237 AMDGPUAsmParser::OperandMatchResultTy
1238 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands) {
1239   auto res = parseImm(Operands);
1240   if (res != MatchOperand_NoMatch) {
1241     return res;
1242   }
1243 
1244   if (auto R = parseRegister()) {
1245     assert(R->isReg());
1246     R->Reg.IsForcedVOP3 = isForcedVOP3();
1247     Operands.push_back(std::move(R));
1248     return MatchOperand_Success;
1249   }
1250   return MatchOperand_ParseFail;
1251 }
1252 
1253 AMDGPUAsmParser::OperandMatchResultTy
1254 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands) {
1255   // XXX: During parsing we can't determine if minus sign means
1256   // negate-modifier or negative immediate value.
1257   // By default we suppose it is modifier.
1258   bool Negate = false, Abs = false, Abs2 = false;
1259 
1260   if (getLexer().getKind()== AsmToken::Minus) {
1261     Parser.Lex();
1262     Negate = true;
1263   }
1264 
1265   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "abs") {
1266     Parser.Lex();
1267     Abs2 = true;
1268     if (getLexer().isNot(AsmToken::LParen)) {
1269       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1270       return MatchOperand_ParseFail;
1271     }
1272     Parser.Lex();
1273   }
1274 
1275   if (getLexer().getKind() == AsmToken::Pipe) {
1276     if (Abs2) {
1277       Error(Parser.getTok().getLoc(), "expected register or immediate");
1278       return MatchOperand_ParseFail;
1279     }
1280     Parser.Lex();
1281     Abs = true;
1282   }
1283 
1284   auto Res = parseRegOrImm(Operands);
1285   if (Res != MatchOperand_Success) {
1286     return Res;
1287   }
1288 
1289   AMDGPUOperand::Modifiers Mods = {false, false, false};
1290   if (Negate) {
1291     Mods.Neg = true;
1292   }
1293   if (Abs) {
1294     if (getLexer().getKind() != AsmToken::Pipe) {
1295       Error(Parser.getTok().getLoc(), "expected vertical bar");
1296       return MatchOperand_ParseFail;
1297     }
1298     Parser.Lex();
1299     Mods.Abs = true;
1300   }
1301   if (Abs2) {
1302     if (getLexer().isNot(AsmToken::RParen)) {
1303       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1304       return MatchOperand_ParseFail;
1305     }
1306     Parser.Lex();
1307     Mods.Abs = true;
1308   }
1309 
1310   if (Mods.hasFPModifiers()) {
1311     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1312     Op.setModifiers(Mods);
1313   }
1314   return MatchOperand_Success;
1315 }
1316 
1317 AMDGPUAsmParser::OperandMatchResultTy
1318 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands) {
1319   bool Sext = false;
1320 
1321   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "sext") {
1322     Parser.Lex();
1323     Sext = true;
1324     if (getLexer().isNot(AsmToken::LParen)) {
1325       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1326       return MatchOperand_ParseFail;
1327     }
1328     Parser.Lex();
1329   }
1330 
1331   auto Res = parseRegOrImm(Operands);
1332   if (Res != MatchOperand_Success) {
1333     return Res;
1334   }
1335 
1336   AMDGPUOperand::Modifiers Mods = {false, false, false};
1337   if (Sext) {
1338     if (getLexer().isNot(AsmToken::RParen)) {
1339       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1340       return MatchOperand_ParseFail;
1341     }
1342     Parser.Lex();
1343     Mods.Sext = true;
1344   }
1345 
1346   if (Mods.hasIntModifiers()) {
1347     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1348     Op.setModifiers(Mods);
1349   }
1350   return MatchOperand_Success;
1351 }
1352 
1353 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1354 
1355   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1356 
1357   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1358       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1359       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1360       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1361     return Match_InvalidOperand;
1362 
1363   if ((TSFlags & SIInstrFlags::VOP3) &&
1364       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1365       getForcedEncodingSize() != 64)
1366     return Match_PreferE32;
1367 
1368   if (Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa ||
1369       Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa) {
1370     // v_mac_f32/16 allow only dst_sel == DWORD;
1371     auto OpNum = AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
1372     const auto &Op = Inst.getOperand(OpNum);
1373     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
1374       return Match_InvalidOperand;
1375     }
1376   }
1377 
1378   return Match_Success;
1379 }
1380 
1381 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1382                                               OperandVector &Operands,
1383                                               MCStreamer &Out,
1384                                               uint64_t &ErrorInfo,
1385                                               bool MatchingInlineAsm) {
1386   // What asm variants we should check
1387   std::vector<unsigned> MatchedVariants;
1388   if (getForcedEncodingSize() == 32) {
1389     MatchedVariants = {AMDGPUAsmVariants::DEFAULT};
1390   } else if (isForcedVOP3()) {
1391     MatchedVariants = {AMDGPUAsmVariants::VOP3};
1392   } else if (isForcedSDWA()) {
1393     MatchedVariants = {AMDGPUAsmVariants::SDWA};
1394   } else if (isForcedDPP()) {
1395     MatchedVariants = {AMDGPUAsmVariants::DPP};
1396   } else {
1397     MatchedVariants = {AMDGPUAsmVariants::DEFAULT,
1398                        AMDGPUAsmVariants::VOP3,
1399                        AMDGPUAsmVariants::SDWA,
1400                        AMDGPUAsmVariants::DPP};
1401   }
1402 
1403   MCInst Inst;
1404   unsigned Result = Match_Success;
1405   for (auto Variant : MatchedVariants) {
1406     uint64_t EI;
1407     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
1408                                   Variant);
1409     // We order match statuses from least to most specific. We use most specific
1410     // status as resulting
1411     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
1412     if ((R == Match_Success) ||
1413         (R == Match_PreferE32) ||
1414         (R == Match_MissingFeature && Result != Match_PreferE32) ||
1415         (R == Match_InvalidOperand && Result != Match_MissingFeature
1416                                    && Result != Match_PreferE32) ||
1417         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
1418                                    && Result != Match_MissingFeature
1419                                    && Result != Match_PreferE32)) {
1420       Result = R;
1421       ErrorInfo = EI;
1422     }
1423     if (R == Match_Success)
1424       break;
1425   }
1426 
1427   switch (Result) {
1428   default: break;
1429   case Match_Success:
1430     Inst.setLoc(IDLoc);
1431     Out.EmitInstruction(Inst, getSTI());
1432     return false;
1433 
1434   case Match_MissingFeature:
1435     return Error(IDLoc, "instruction not supported on this GPU");
1436 
1437   case Match_MnemonicFail:
1438     return Error(IDLoc, "unrecognized instruction mnemonic");
1439 
1440   case Match_InvalidOperand: {
1441     SMLoc ErrorLoc = IDLoc;
1442     if (ErrorInfo != ~0ULL) {
1443       if (ErrorInfo >= Operands.size()) {
1444         return Error(IDLoc, "too few operands for instruction");
1445       }
1446       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
1447       if (ErrorLoc == SMLoc())
1448         ErrorLoc = IDLoc;
1449     }
1450     return Error(ErrorLoc, "invalid operand for instruction");
1451   }
1452 
1453   case Match_PreferE32:
1454     return Error(IDLoc, "internal error: instruction without _e64 suffix "
1455                         "should be encoded as e32");
1456   }
1457   llvm_unreachable("Implement any new match types added!");
1458 }
1459 
1460 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
1461                                                uint32_t &Minor) {
1462   if (getLexer().isNot(AsmToken::Integer))
1463     return TokError("invalid major version");
1464 
1465   Major = getLexer().getTok().getIntVal();
1466   Lex();
1467 
1468   if (getLexer().isNot(AsmToken::Comma))
1469     return TokError("minor version number required, comma expected");
1470   Lex();
1471 
1472   if (getLexer().isNot(AsmToken::Integer))
1473     return TokError("invalid minor version");
1474 
1475   Minor = getLexer().getTok().getIntVal();
1476   Lex();
1477 
1478   return false;
1479 }
1480 
1481 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
1482 
1483   uint32_t Major;
1484   uint32_t Minor;
1485 
1486   if (ParseDirectiveMajorMinor(Major, Minor))
1487     return true;
1488 
1489   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
1490   return false;
1491 }
1492 
1493 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
1494 
1495   uint32_t Major;
1496   uint32_t Minor;
1497   uint32_t Stepping;
1498   StringRef VendorName;
1499   StringRef ArchName;
1500 
1501   // If this directive has no arguments, then use the ISA version for the
1502   // targeted GPU.
1503   if (getLexer().is(AsmToken::EndOfStatement)) {
1504     AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
1505     getTargetStreamer().EmitDirectiveHSACodeObjectISA(Isa.Major, Isa.Minor,
1506                                                       Isa.Stepping,
1507                                                       "AMD", "AMDGPU");
1508     return false;
1509   }
1510 
1511 
1512   if (ParseDirectiveMajorMinor(Major, Minor))
1513     return true;
1514 
1515   if (getLexer().isNot(AsmToken::Comma))
1516     return TokError("stepping version number required, comma expected");
1517   Lex();
1518 
1519   if (getLexer().isNot(AsmToken::Integer))
1520     return TokError("invalid stepping version");
1521 
1522   Stepping = getLexer().getTok().getIntVal();
1523   Lex();
1524 
1525   if (getLexer().isNot(AsmToken::Comma))
1526     return TokError("vendor name required, comma expected");
1527   Lex();
1528 
1529   if (getLexer().isNot(AsmToken::String))
1530     return TokError("invalid vendor name");
1531 
1532   VendorName = getLexer().getTok().getStringContents();
1533   Lex();
1534 
1535   if (getLexer().isNot(AsmToken::Comma))
1536     return TokError("arch name required, comma expected");
1537   Lex();
1538 
1539   if (getLexer().isNot(AsmToken::String))
1540     return TokError("invalid arch name");
1541 
1542   ArchName = getLexer().getTok().getStringContents();
1543   Lex();
1544 
1545   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
1546                                                     VendorName, ArchName);
1547   return false;
1548 }
1549 
1550 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
1551                                                amd_kernel_code_t &Header) {
1552   SmallString<40> ErrStr;
1553   raw_svector_ostream Err(ErrStr);
1554   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
1555     return TokError(Err.str());
1556   }
1557   Lex();
1558   return false;
1559 }
1560 
1561 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
1562 
1563   amd_kernel_code_t Header;
1564   AMDGPU::initDefaultAMDKernelCodeT(Header, getSTI().getFeatureBits());
1565 
1566   while (true) {
1567 
1568     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
1569     // will set the current token to EndOfStatement.
1570     while(getLexer().is(AsmToken::EndOfStatement))
1571       Lex();
1572 
1573     if (getLexer().isNot(AsmToken::Identifier))
1574       return TokError("expected value identifier or .end_amd_kernel_code_t");
1575 
1576     StringRef ID = getLexer().getTok().getIdentifier();
1577     Lex();
1578 
1579     if (ID == ".end_amd_kernel_code_t")
1580       break;
1581 
1582     if (ParseAMDKernelCodeTValue(ID, Header))
1583       return true;
1584   }
1585 
1586   getTargetStreamer().EmitAMDKernelCodeT(Header);
1587 
1588   return false;
1589 }
1590 
1591 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
1592   getParser().getStreamer().SwitchSection(
1593       AMDGPU::getHSATextSection(getContext()));
1594   return false;
1595 }
1596 
1597 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
1598   if (getLexer().isNot(AsmToken::Identifier))
1599     return TokError("expected symbol name");
1600 
1601   StringRef KernelName = Parser.getTok().getString();
1602 
1603   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
1604                                            ELF::STT_AMDGPU_HSA_KERNEL);
1605   Lex();
1606   return false;
1607 }
1608 
1609 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
1610   if (getLexer().isNot(AsmToken::Identifier))
1611     return TokError("expected symbol name");
1612 
1613   StringRef GlobalName = Parser.getTok().getIdentifier();
1614 
1615   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
1616   Lex();
1617   return false;
1618 }
1619 
1620 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
1621   if (getLexer().isNot(AsmToken::Identifier))
1622     return TokError("expected symbol name");
1623 
1624   StringRef GlobalName = Parser.getTok().getIdentifier();
1625 
1626   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
1627   Lex();
1628   return false;
1629 }
1630 
1631 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
1632   getParser().getStreamer().SwitchSection(
1633       AMDGPU::getHSADataGlobalAgentSection(getContext()));
1634   return false;
1635 }
1636 
1637 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
1638   getParser().getStreamer().SwitchSection(
1639       AMDGPU::getHSADataGlobalProgramSection(getContext()));
1640   return false;
1641 }
1642 
1643 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
1644   getParser().getStreamer().SwitchSection(
1645       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
1646   return false;
1647 }
1648 
1649 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
1650   StringRef IDVal = DirectiveID.getString();
1651 
1652   if (IDVal == ".hsa_code_object_version")
1653     return ParseDirectiveHSACodeObjectVersion();
1654 
1655   if (IDVal == ".hsa_code_object_isa")
1656     return ParseDirectiveHSACodeObjectISA();
1657 
1658   if (IDVal == ".amd_kernel_code_t")
1659     return ParseDirectiveAMDKernelCodeT();
1660 
1661   if (IDVal == ".hsatext")
1662     return ParseSectionDirectiveHSAText();
1663 
1664   if (IDVal == ".amdgpu_hsa_kernel")
1665     return ParseDirectiveAMDGPUHsaKernel();
1666 
1667   if (IDVal == ".amdgpu_hsa_module_global")
1668     return ParseDirectiveAMDGPUHsaModuleGlobal();
1669 
1670   if (IDVal == ".amdgpu_hsa_program_global")
1671     return ParseDirectiveAMDGPUHsaProgramGlobal();
1672 
1673   if (IDVal == ".hsadata_global_agent")
1674     return ParseSectionDirectiveHSADataGlobalAgent();
1675 
1676   if (IDVal == ".hsadata_global_program")
1677     return ParseSectionDirectiveHSADataGlobalProgram();
1678 
1679   if (IDVal == ".hsarodata_readonly_agent")
1680     return ParseSectionDirectiveHSARodataReadonlyAgent();
1681 
1682   return true;
1683 }
1684 
1685 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
1686                                            unsigned RegNo) const {
1687   if (isCI())
1688     return true;
1689 
1690   if (isSI()) {
1691     // No flat_scr
1692     switch (RegNo) {
1693     case AMDGPU::FLAT_SCR:
1694     case AMDGPU::FLAT_SCR_LO:
1695     case AMDGPU::FLAT_SCR_HI:
1696       return false;
1697     default:
1698       return true;
1699     }
1700   }
1701 
1702   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
1703   // SI/CI have.
1704   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
1705        R.isValid(); ++R) {
1706     if (*R == RegNo)
1707       return false;
1708   }
1709 
1710   return true;
1711 }
1712 
1713 AMDGPUAsmParser::OperandMatchResultTy
1714 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
1715 
1716   // Try to parse with a custom parser
1717   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
1718 
1719   // If we successfully parsed the operand or if there as an error parsing,
1720   // we are done.
1721   //
1722   // If we are parsing after we reach EndOfStatement then this means we
1723   // are appending default values to the Operands list.  This is only done
1724   // by custom parser, so we shouldn't continue on to the generic parsing.
1725   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
1726       getLexer().is(AsmToken::EndOfStatement))
1727     return ResTy;
1728 
1729   ResTy = parseRegOrImm(Operands);
1730 
1731   if (ResTy == MatchOperand_Success)
1732     return ResTy;
1733 
1734   if (getLexer().getKind() == AsmToken::Identifier) {
1735     // If this identifier is a symbol, we want to create an expression for it.
1736     // It is a little difficult to distinguish between a symbol name, and
1737     // an instruction flag like 'gds'.  In order to do this, we parse
1738     // all tokens as expressions and then treate the symbol name as the token
1739     // string when we want to interpret the operand as a token.
1740     const auto &Tok = Parser.getTok();
1741     SMLoc S = Tok.getLoc();
1742     const MCExpr *Expr = nullptr;
1743     if (!Parser.parseExpression(Expr)) {
1744       Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
1745       return MatchOperand_Success;
1746     }
1747 
1748     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), Tok.getLoc()));
1749     Parser.Lex();
1750     return MatchOperand_Success;
1751   }
1752   return MatchOperand_NoMatch;
1753 }
1754 
1755 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
1756   // Clear any forced encodings from the previous instruction.
1757   setForcedEncodingSize(0);
1758   setForcedDPP(false);
1759   setForcedSDWA(false);
1760 
1761   if (Name.endswith("_e64")) {
1762     setForcedEncodingSize(64);
1763     return Name.substr(0, Name.size() - 4);
1764   } else if (Name.endswith("_e32")) {
1765     setForcedEncodingSize(32);
1766     return Name.substr(0, Name.size() - 4);
1767   } else if (Name.endswith("_dpp")) {
1768     setForcedDPP(true);
1769     return Name.substr(0, Name.size() - 4);
1770   } else if (Name.endswith("_sdwa")) {
1771     setForcedSDWA(true);
1772     return Name.substr(0, Name.size() - 5);
1773   }
1774   return Name;
1775 }
1776 
1777 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
1778                                        StringRef Name,
1779                                        SMLoc NameLoc, OperandVector &Operands) {
1780   // Add the instruction mnemonic
1781   Name = parseMnemonicSuffix(Name);
1782   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
1783 
1784   while (!getLexer().is(AsmToken::EndOfStatement)) {
1785     AMDGPUAsmParser::OperandMatchResultTy Res = parseOperand(Operands, Name);
1786 
1787     // Eat the comma or space if there is one.
1788     if (getLexer().is(AsmToken::Comma))
1789       Parser.Lex();
1790 
1791     switch (Res) {
1792       case MatchOperand_Success: break;
1793       case MatchOperand_ParseFail:
1794         Error(getLexer().getLoc(), "failed parsing operand.");
1795         while (!getLexer().is(AsmToken::EndOfStatement)) {
1796           Parser.Lex();
1797         }
1798         return true;
1799       case MatchOperand_NoMatch:
1800         Error(getLexer().getLoc(), "not a valid operand.");
1801         while (!getLexer().is(AsmToken::EndOfStatement)) {
1802           Parser.Lex();
1803         }
1804         return true;
1805     }
1806   }
1807 
1808   return false;
1809 }
1810 
1811 //===----------------------------------------------------------------------===//
1812 // Utility functions
1813 //===----------------------------------------------------------------------===//
1814 
1815 AMDGPUAsmParser::OperandMatchResultTy
1816 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
1817   switch(getLexer().getKind()) {
1818     default: return MatchOperand_NoMatch;
1819     case AsmToken::Identifier: {
1820       StringRef Name = Parser.getTok().getString();
1821       if (!Name.equals(Prefix)) {
1822         return MatchOperand_NoMatch;
1823       }
1824 
1825       Parser.Lex();
1826       if (getLexer().isNot(AsmToken::Colon))
1827         return MatchOperand_ParseFail;
1828 
1829       Parser.Lex();
1830       if (getLexer().isNot(AsmToken::Integer))
1831         return MatchOperand_ParseFail;
1832 
1833       if (getParser().parseAbsoluteExpression(Int))
1834         return MatchOperand_ParseFail;
1835       break;
1836     }
1837   }
1838   return MatchOperand_Success;
1839 }
1840 
1841 AMDGPUAsmParser::OperandMatchResultTy
1842 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
1843                                     enum AMDGPUOperand::ImmTy ImmTy,
1844                                     bool (*ConvertResult)(int64_t&)) {
1845 
1846   SMLoc S = Parser.getTok().getLoc();
1847   int64_t Value = 0;
1848 
1849   AMDGPUAsmParser::OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
1850   if (Res != MatchOperand_Success)
1851     return Res;
1852 
1853   if (ConvertResult && !ConvertResult(Value)) {
1854     return MatchOperand_ParseFail;
1855   }
1856 
1857   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
1858   return MatchOperand_Success;
1859 }
1860 
1861 AMDGPUAsmParser::OperandMatchResultTy
1862 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
1863                                enum AMDGPUOperand::ImmTy ImmTy) {
1864   int64_t Bit = 0;
1865   SMLoc S = Parser.getTok().getLoc();
1866 
1867   // We are at the end of the statement, and this is a default argument, so
1868   // use a default value.
1869   if (getLexer().isNot(AsmToken::EndOfStatement)) {
1870     switch(getLexer().getKind()) {
1871       case AsmToken::Identifier: {
1872         StringRef Tok = Parser.getTok().getString();
1873         if (Tok == Name) {
1874           Bit = 1;
1875           Parser.Lex();
1876         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
1877           Bit = 0;
1878           Parser.Lex();
1879         } else {
1880           return MatchOperand_NoMatch;
1881         }
1882         break;
1883       }
1884       default:
1885         return MatchOperand_NoMatch;
1886     }
1887   }
1888 
1889   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
1890   return MatchOperand_Success;
1891 }
1892 
1893 typedef std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
1894 
1895 void addOptionalImmOperand(MCInst& Inst, const OperandVector& Operands,
1896                            OptionalImmIndexMap& OptionalIdx,
1897                            enum AMDGPUOperand::ImmTy ImmT, int64_t Default = 0) {
1898   auto i = OptionalIdx.find(ImmT);
1899   if (i != OptionalIdx.end()) {
1900     unsigned Idx = i->second;
1901     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
1902   } else {
1903     Inst.addOperand(MCOperand::createImm(Default));
1904   }
1905 }
1906 
1907 AMDGPUAsmParser::OperandMatchResultTy
1908 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
1909   if (getLexer().isNot(AsmToken::Identifier)) {
1910     return MatchOperand_NoMatch;
1911   }
1912   StringRef Tok = Parser.getTok().getString();
1913   if (Tok != Prefix) {
1914     return MatchOperand_NoMatch;
1915   }
1916 
1917   Parser.Lex();
1918   if (getLexer().isNot(AsmToken::Colon)) {
1919     return MatchOperand_ParseFail;
1920   }
1921 
1922   Parser.Lex();
1923   if (getLexer().isNot(AsmToken::Identifier)) {
1924     return MatchOperand_ParseFail;
1925   }
1926 
1927   Value = Parser.getTok().getString();
1928   return MatchOperand_Success;
1929 }
1930 
1931 //===----------------------------------------------------------------------===//
1932 // ds
1933 //===----------------------------------------------------------------------===//
1934 
1935 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
1936                                     const OperandVector &Operands) {
1937 
1938   OptionalImmIndexMap OptionalIdx;
1939 
1940   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1941     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1942 
1943     // Add the register arguments
1944     if (Op.isReg()) {
1945       Op.addRegOperands(Inst, 1);
1946       continue;
1947     }
1948 
1949     // Handle optional arguments
1950     OptionalIdx[Op.getImmTy()] = i;
1951   }
1952 
1953   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
1954   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
1955   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1956 
1957   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1958 }
1959 
1960 void AMDGPUAsmParser::cvtDS(MCInst &Inst, const OperandVector &Operands) {
1961 
1962   std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
1963   bool GDSOnly = false;
1964 
1965   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1966     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1967 
1968     // Add the register arguments
1969     if (Op.isReg()) {
1970       Op.addRegOperands(Inst, 1);
1971       continue;
1972     }
1973 
1974     if (Op.isToken() && Op.getToken() == "gds") {
1975       GDSOnly = true;
1976       continue;
1977     }
1978 
1979     // Handle optional arguments
1980     OptionalIdx[Op.getImmTy()] = i;
1981   }
1982 
1983   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
1984   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1985 
1986   if (!GDSOnly) {
1987     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1988   }
1989   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1990 }
1991 
1992 
1993 //===----------------------------------------------------------------------===//
1994 // s_waitcnt
1995 //===----------------------------------------------------------------------===//
1996 
1997 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
1998   StringRef CntName = Parser.getTok().getString();
1999   int64_t CntVal;
2000 
2001   Parser.Lex();
2002   if (getLexer().isNot(AsmToken::LParen))
2003     return true;
2004 
2005   Parser.Lex();
2006   if (getLexer().isNot(AsmToken::Integer))
2007     return true;
2008 
2009   if (getParser().parseAbsoluteExpression(CntVal))
2010     return true;
2011 
2012   if (getLexer().isNot(AsmToken::RParen))
2013     return true;
2014 
2015   Parser.Lex();
2016   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
2017     Parser.Lex();
2018 
2019   int CntShift;
2020   int CntMask;
2021 
2022   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2023   if (CntName == "vmcnt") {
2024     CntMask = getVmcntMask(IV);
2025     CntShift = getVmcntShift(IV);
2026   } else if (CntName == "expcnt") {
2027     CntMask = getExpcntMask(IV);
2028     CntShift = getExpcntShift(IV);
2029   } else if (CntName == "lgkmcnt") {
2030     CntMask = getLgkmcntMask(IV);
2031     CntShift = getLgkmcntShift(IV);
2032   } else {
2033     return true;
2034   }
2035 
2036   IntVal &= ~(CntMask << CntShift);
2037   IntVal |= (CntVal << CntShift);
2038   return false;
2039 }
2040 
2041 AMDGPUAsmParser::OperandMatchResultTy
2042 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
2043   // Disable all counters by default.
2044   // vmcnt   [3:0]
2045   // expcnt  [6:4]
2046   // lgkmcnt [11:8]
2047   int64_t CntVal = 0xf7f;
2048   SMLoc S = Parser.getTok().getLoc();
2049 
2050   switch(getLexer().getKind()) {
2051     default: return MatchOperand_ParseFail;
2052     case AsmToken::Integer:
2053       // The operand can be an integer value.
2054       if (getParser().parseAbsoluteExpression(CntVal))
2055         return MatchOperand_ParseFail;
2056       break;
2057 
2058     case AsmToken::Identifier:
2059       do {
2060         if (parseCnt(CntVal))
2061           return MatchOperand_ParseFail;
2062       } while(getLexer().isNot(AsmToken::EndOfStatement));
2063       break;
2064   }
2065   Operands.push_back(AMDGPUOperand::CreateImm(this, CntVal, S));
2066   return MatchOperand_Success;
2067 }
2068 
2069 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width) {
2070   using namespace llvm::AMDGPU::Hwreg;
2071 
2072   if (Parser.getTok().getString() != "hwreg")
2073     return true;
2074   Parser.Lex();
2075 
2076   if (getLexer().isNot(AsmToken::LParen))
2077     return true;
2078   Parser.Lex();
2079 
2080   if (getLexer().is(AsmToken::Identifier)) {
2081     HwReg.IsSymbolic = true;
2082     HwReg.Id = ID_UNKNOWN_;
2083     const StringRef tok = Parser.getTok().getString();
2084     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
2085       if (tok == IdSymbolic[i]) {
2086         HwReg.Id = i;
2087         break;
2088       }
2089     }
2090     Parser.Lex();
2091   } else {
2092     HwReg.IsSymbolic = false;
2093     if (getLexer().isNot(AsmToken::Integer))
2094       return true;
2095     if (getParser().parseAbsoluteExpression(HwReg.Id))
2096       return true;
2097   }
2098 
2099   if (getLexer().is(AsmToken::RParen)) {
2100     Parser.Lex();
2101     return false;
2102   }
2103 
2104   // optional params
2105   if (getLexer().isNot(AsmToken::Comma))
2106     return true;
2107   Parser.Lex();
2108 
2109   if (getLexer().isNot(AsmToken::Integer))
2110     return true;
2111   if (getParser().parseAbsoluteExpression(Offset))
2112     return true;
2113 
2114   if (getLexer().isNot(AsmToken::Comma))
2115     return true;
2116   Parser.Lex();
2117 
2118   if (getLexer().isNot(AsmToken::Integer))
2119     return true;
2120   if (getParser().parseAbsoluteExpression(Width))
2121     return true;
2122 
2123   if (getLexer().isNot(AsmToken::RParen))
2124     return true;
2125   Parser.Lex();
2126 
2127   return false;
2128 }
2129 
2130 AMDGPUAsmParser::OperandMatchResultTy
2131 AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
2132   using namespace llvm::AMDGPU::Hwreg;
2133 
2134   int64_t Imm16Val = 0;
2135   SMLoc S = Parser.getTok().getLoc();
2136 
2137   switch(getLexer().getKind()) {
2138     default: return MatchOperand_NoMatch;
2139     case AsmToken::Integer:
2140       // The operand can be an integer value.
2141       if (getParser().parseAbsoluteExpression(Imm16Val))
2142         return MatchOperand_NoMatch;
2143       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2144         Error(S, "invalid immediate: only 16-bit values are legal");
2145         // Do not return error code, but create an imm operand anyway and proceed
2146         // to the next operand, if any. That avoids unneccessary error messages.
2147       }
2148       break;
2149 
2150     case AsmToken::Identifier: {
2151         OperandInfoTy HwReg(ID_UNKNOWN_);
2152         int64_t Offset = OFFSET_DEFAULT_;
2153         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
2154         if (parseHwregConstruct(HwReg, Offset, Width))
2155           return MatchOperand_ParseFail;
2156         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
2157           if (HwReg.IsSymbolic)
2158             Error(S, "invalid symbolic name of hardware register");
2159           else
2160             Error(S, "invalid code of hardware register: only 6-bit values are legal");
2161         }
2162         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
2163           Error(S, "invalid bit offset: only 5-bit values are legal");
2164         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
2165           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
2166         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
2167       }
2168       break;
2169   }
2170   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
2171   return MatchOperand_Success;
2172 }
2173 
2174 bool AMDGPUOperand::isSWaitCnt() const {
2175   return isImm();
2176 }
2177 
2178 bool AMDGPUOperand::isHwreg() const {
2179   return isImmTy(ImmTyHwreg);
2180 }
2181 
2182 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
2183   using namespace llvm::AMDGPU::SendMsg;
2184 
2185   if (Parser.getTok().getString() != "sendmsg")
2186     return true;
2187   Parser.Lex();
2188 
2189   if (getLexer().isNot(AsmToken::LParen))
2190     return true;
2191   Parser.Lex();
2192 
2193   if (getLexer().is(AsmToken::Identifier)) {
2194     Msg.IsSymbolic = true;
2195     Msg.Id = ID_UNKNOWN_;
2196     const std::string tok = Parser.getTok().getString();
2197     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
2198       switch(i) {
2199         default: continue; // Omit gaps.
2200         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
2201       }
2202       if (tok == IdSymbolic[i]) {
2203         Msg.Id = i;
2204         break;
2205       }
2206     }
2207     Parser.Lex();
2208   } else {
2209     Msg.IsSymbolic = false;
2210     if (getLexer().isNot(AsmToken::Integer))
2211       return true;
2212     if (getParser().parseAbsoluteExpression(Msg.Id))
2213       return true;
2214     if (getLexer().is(AsmToken::Integer))
2215       if (getParser().parseAbsoluteExpression(Msg.Id))
2216         Msg.Id = ID_UNKNOWN_;
2217   }
2218   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
2219     return false;
2220 
2221   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
2222     if (getLexer().isNot(AsmToken::RParen))
2223       return true;
2224     Parser.Lex();
2225     return false;
2226   }
2227 
2228   if (getLexer().isNot(AsmToken::Comma))
2229     return true;
2230   Parser.Lex();
2231 
2232   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
2233   Operation.Id = ID_UNKNOWN_;
2234   if (getLexer().is(AsmToken::Identifier)) {
2235     Operation.IsSymbolic = true;
2236     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
2237     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
2238     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
2239     const StringRef Tok = Parser.getTok().getString();
2240     for (int i = F; i < L; ++i) {
2241       if (Tok == S[i]) {
2242         Operation.Id = i;
2243         break;
2244       }
2245     }
2246     Parser.Lex();
2247   } else {
2248     Operation.IsSymbolic = false;
2249     if (getLexer().isNot(AsmToken::Integer))
2250       return true;
2251     if (getParser().parseAbsoluteExpression(Operation.Id))
2252       return true;
2253   }
2254 
2255   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2256     // Stream id is optional.
2257     if (getLexer().is(AsmToken::RParen)) {
2258       Parser.Lex();
2259       return false;
2260     }
2261 
2262     if (getLexer().isNot(AsmToken::Comma))
2263       return true;
2264     Parser.Lex();
2265 
2266     if (getLexer().isNot(AsmToken::Integer))
2267       return true;
2268     if (getParser().parseAbsoluteExpression(StreamId))
2269       return true;
2270   }
2271 
2272   if (getLexer().isNot(AsmToken::RParen))
2273     return true;
2274   Parser.Lex();
2275   return false;
2276 }
2277 
2278 AMDGPUAsmParser::OperandMatchResultTy
2279 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
2280   using namespace llvm::AMDGPU::SendMsg;
2281 
2282   int64_t Imm16Val = 0;
2283   SMLoc S = Parser.getTok().getLoc();
2284 
2285   switch(getLexer().getKind()) {
2286   default:
2287     return MatchOperand_NoMatch;
2288   case AsmToken::Integer:
2289     // The operand can be an integer value.
2290     if (getParser().parseAbsoluteExpression(Imm16Val))
2291       return MatchOperand_NoMatch;
2292     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2293       Error(S, "invalid immediate: only 16-bit values are legal");
2294       // Do not return error code, but create an imm operand anyway and proceed
2295       // to the next operand, if any. That avoids unneccessary error messages.
2296     }
2297     break;
2298   case AsmToken::Identifier: {
2299       OperandInfoTy Msg(ID_UNKNOWN_);
2300       OperandInfoTy Operation(OP_UNKNOWN_);
2301       int64_t StreamId = STREAM_ID_DEFAULT_;
2302       if (parseSendMsgConstruct(Msg, Operation, StreamId))
2303         return MatchOperand_ParseFail;
2304       do {
2305         // Validate and encode message ID.
2306         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
2307                 || Msg.Id == ID_SYSMSG)) {
2308           if (Msg.IsSymbolic)
2309             Error(S, "invalid/unsupported symbolic name of message");
2310           else
2311             Error(S, "invalid/unsupported code of message");
2312           break;
2313         }
2314         Imm16Val = (Msg.Id << ID_SHIFT_);
2315         // Validate and encode operation ID.
2316         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
2317           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
2318             if (Operation.IsSymbolic)
2319               Error(S, "invalid symbolic name of GS_OP");
2320             else
2321               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
2322             break;
2323           }
2324           if (Operation.Id == OP_GS_NOP
2325               && Msg.Id != ID_GS_DONE) {
2326             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
2327             break;
2328           }
2329           Imm16Val |= (Operation.Id << OP_SHIFT_);
2330         }
2331         if (Msg.Id == ID_SYSMSG) {
2332           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
2333             if (Operation.IsSymbolic)
2334               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
2335             else
2336               Error(S, "invalid/unsupported code of SYSMSG_OP");
2337             break;
2338           }
2339           Imm16Val |= (Operation.Id << OP_SHIFT_);
2340         }
2341         // Validate and encode stream ID.
2342         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2343           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
2344             Error(S, "invalid stream id: only 2-bit values are legal");
2345             break;
2346           }
2347           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
2348         }
2349       } while (0);
2350     }
2351     break;
2352   }
2353   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
2354   return MatchOperand_Success;
2355 }
2356 
2357 bool AMDGPUOperand::isSendMsg() const {
2358   return isImmTy(ImmTySendMsg);
2359 }
2360 
2361 //===----------------------------------------------------------------------===//
2362 // sopp branch targets
2363 //===----------------------------------------------------------------------===//
2364 
2365 AMDGPUAsmParser::OperandMatchResultTy
2366 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
2367   SMLoc S = Parser.getTok().getLoc();
2368 
2369   switch (getLexer().getKind()) {
2370     default: return MatchOperand_ParseFail;
2371     case AsmToken::Integer: {
2372       int64_t Imm;
2373       if (getParser().parseAbsoluteExpression(Imm))
2374         return MatchOperand_ParseFail;
2375       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
2376       return MatchOperand_Success;
2377     }
2378 
2379     case AsmToken::Identifier:
2380       Operands.push_back(AMDGPUOperand::CreateExpr(this,
2381           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
2382                                   Parser.getTok().getString()), getContext()), S));
2383       Parser.Lex();
2384       return MatchOperand_Success;
2385   }
2386 }
2387 
2388 //===----------------------------------------------------------------------===//
2389 // mubuf
2390 //===----------------------------------------------------------------------===//
2391 
2392 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
2393   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
2394 }
2395 
2396 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
2397   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
2398 }
2399 
2400 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
2401   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
2402 }
2403 
2404 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
2405                                const OperandVector &Operands,
2406                                bool IsAtomic, bool IsAtomicReturn) {
2407   OptionalImmIndexMap OptionalIdx;
2408   assert(IsAtomicReturn ? IsAtomic : true);
2409 
2410   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2411     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2412 
2413     // Add the register arguments
2414     if (Op.isReg()) {
2415       Op.addRegOperands(Inst, 1);
2416       continue;
2417     }
2418 
2419     // Handle the case where soffset is an immediate
2420     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
2421       Op.addImmOperands(Inst, 1);
2422       continue;
2423     }
2424 
2425     // Handle tokens like 'offen' which are sometimes hard-coded into the
2426     // asm string.  There are no MCInst operands for these.
2427     if (Op.isToken()) {
2428       continue;
2429     }
2430     assert(Op.isImm());
2431 
2432     // Handle optional arguments
2433     OptionalIdx[Op.getImmTy()] = i;
2434   }
2435 
2436   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
2437   if (IsAtomicReturn) {
2438     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
2439     Inst.insert(I, *I);
2440   }
2441 
2442   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2443   if (!IsAtomic) { // glc is hard-coded.
2444     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2445   }
2446   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2447   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2448 }
2449 
2450 //===----------------------------------------------------------------------===//
2451 // mimg
2452 //===----------------------------------------------------------------------===//
2453 
2454 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
2455   unsigned I = 1;
2456   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2457   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2458     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2459   }
2460 
2461   OptionalImmIndexMap OptionalIdx;
2462 
2463   for (unsigned E = Operands.size(); I != E; ++I) {
2464     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2465 
2466     // Add the register arguments
2467     if (Op.isRegOrImm()) {
2468       Op.addRegOrImmOperands(Inst, 1);
2469       continue;
2470     } else if (Op.isImmModifier()) {
2471       OptionalIdx[Op.getImmTy()] = I;
2472     } else {
2473       assert(false);
2474     }
2475   }
2476 
2477   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2478   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2479   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2480   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2481   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2482   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2483   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2484   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2485 }
2486 
2487 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
2488   unsigned I = 1;
2489   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2490   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2491     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2492   }
2493 
2494   // Add src, same as dst
2495   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
2496 
2497   OptionalImmIndexMap OptionalIdx;
2498 
2499   for (unsigned E = Operands.size(); I != E; ++I) {
2500     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2501 
2502     // Add the register arguments
2503     if (Op.isRegOrImm()) {
2504       Op.addRegOrImmOperands(Inst, 1);
2505       continue;
2506     } else if (Op.isImmModifier()) {
2507       OptionalIdx[Op.getImmTy()] = I;
2508     } else {
2509       assert(false);
2510     }
2511   }
2512 
2513   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2514   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2515   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2516   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2517   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2518   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2519   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2520   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2521 }
2522 
2523 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
2524   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
2525 }
2526 
2527 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
2528   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
2529 }
2530 
2531 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
2532   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
2533 }
2534 
2535 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
2536   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
2537 }
2538 
2539 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
2540   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
2541 }
2542 
2543 //===----------------------------------------------------------------------===//
2544 // smrd
2545 //===----------------------------------------------------------------------===//
2546 
2547 bool AMDGPUOperand::isSMRDOffset() const {
2548 
2549   // FIXME: Support 20-bit offsets on VI.  We need to to pass subtarget
2550   // information here.
2551   return isImm() && isUInt<8>(getImm());
2552 }
2553 
2554 bool AMDGPUOperand::isSMRDLiteralOffset() const {
2555   // 32-bit literals are only supported on CI and we only want to use them
2556   // when the offset is > 8-bits.
2557   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
2558 }
2559 
2560 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset() const {
2561   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2562 }
2563 
2564 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
2565   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2566 }
2567 
2568 //===----------------------------------------------------------------------===//
2569 // vop3
2570 //===----------------------------------------------------------------------===//
2571 
2572 static bool ConvertOmodMul(int64_t &Mul) {
2573   if (Mul != 1 && Mul != 2 && Mul != 4)
2574     return false;
2575 
2576   Mul >>= 1;
2577   return true;
2578 }
2579 
2580 static bool ConvertOmodDiv(int64_t &Div) {
2581   if (Div == 1) {
2582     Div = 0;
2583     return true;
2584   }
2585 
2586   if (Div == 2) {
2587     Div = 3;
2588     return true;
2589   }
2590 
2591   return false;
2592 }
2593 
2594 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
2595   if (BoundCtrl == 0) {
2596     BoundCtrl = 1;
2597     return true;
2598   } else if (BoundCtrl == -1) {
2599     BoundCtrl = 0;
2600     return true;
2601   }
2602   return false;
2603 }
2604 
2605 // Note: the order in this table matches the order of operands in AsmString.
2606 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
2607   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
2608   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
2609   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
2610   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
2611   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
2612   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
2613   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
2614   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
2615   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
2616   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
2617   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
2618   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
2619   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
2620   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
2621   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
2622   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
2623   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
2624   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
2625   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
2626   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
2627   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
2628   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
2629   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
2630   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
2631 };
2632 
2633 AMDGPUAsmParser::OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
2634   OperandMatchResultTy res;
2635   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
2636     // try to parse any optional operand here
2637     if (Op.IsBit) {
2638       res = parseNamedBit(Op.Name, Operands, Op.Type);
2639     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
2640       res = parseOModOperand(Operands);
2641     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
2642                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
2643                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
2644       res = parseSDWASel(Operands, Op.Name, Op.Type);
2645     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
2646       res = parseSDWADstUnused(Operands);
2647     } else {
2648       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
2649     }
2650     if (res != MatchOperand_NoMatch) {
2651       return res;
2652     }
2653   }
2654   return MatchOperand_NoMatch;
2655 }
2656 
2657 AMDGPUAsmParser::OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands)
2658 {
2659   StringRef Name = Parser.getTok().getString();
2660   if (Name == "mul") {
2661     return parseIntWithPrefix("mul", Operands, AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
2662   } else if (Name == "div") {
2663     return parseIntWithPrefix("div", Operands, AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
2664   } else {
2665     return MatchOperand_NoMatch;
2666   }
2667 }
2668 
2669 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
2670   unsigned I = 1;
2671   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2672   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2673     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2674   }
2675   for (unsigned E = Operands.size(); I != E; ++I)
2676     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
2677 }
2678 
2679 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
2680   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
2681   if (TSFlags & SIInstrFlags::VOP3) {
2682     cvtVOP3(Inst, Operands);
2683   } else {
2684     cvtId(Inst, Operands);
2685   }
2686 }
2687 
2688 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
2689       // 1. This operand is input modifiers
2690   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
2691       // 2. This is not last operand
2692       && Desc.NumOperands > (OpNum + 1)
2693       // 3. Next operand is register class
2694       && Desc.OpInfo[OpNum + 1].RegClass != -1
2695       // 4. Next register is not tied to any other operand
2696       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
2697 }
2698 
2699 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
2700   OptionalImmIndexMap OptionalIdx;
2701   unsigned I = 1;
2702   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2703   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2704     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2705   }
2706 
2707   for (unsigned E = Operands.size(); I != E; ++I) {
2708     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2709     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
2710       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
2711     } else if (Op.isImm()) {
2712       OptionalIdx[Op.getImmTy()] = I;
2713     } else {
2714       assert(false);
2715     }
2716   }
2717 
2718   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
2719   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
2720 
2721   // special case v_mac_f32:
2722   // it has src2 register operand that is tied to dst operand
2723   // we don't allow modifiers for this operand in assembler so src2_modifiers
2724   // should be 0
2725   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_si ||
2726       Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_vi) {
2727     auto it = Inst.begin();
2728     std::advance(it, AMDGPU::getNamedOperandIdx(AMDGPU::V_MAC_F32_e64, AMDGPU::OpName::src2_modifiers));
2729     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
2730     ++it;
2731     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
2732   }
2733 }
2734 
2735 //===----------------------------------------------------------------------===//
2736 // dpp
2737 //===----------------------------------------------------------------------===//
2738 
2739 bool AMDGPUOperand::isDPPCtrl() const {
2740   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
2741   if (result) {
2742     int64_t Imm = getImm();
2743     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
2744            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
2745            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
2746            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
2747            (Imm == 0x130) ||
2748            (Imm == 0x134) ||
2749            (Imm == 0x138) ||
2750            (Imm == 0x13c) ||
2751            (Imm == 0x140) ||
2752            (Imm == 0x141) ||
2753            (Imm == 0x142) ||
2754            (Imm == 0x143);
2755   }
2756   return false;
2757 }
2758 
2759 AMDGPUAsmParser::OperandMatchResultTy
2760 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
2761   SMLoc S = Parser.getTok().getLoc();
2762   StringRef Prefix;
2763   int64_t Int;
2764 
2765   if (getLexer().getKind() == AsmToken::Identifier) {
2766     Prefix = Parser.getTok().getString();
2767   } else {
2768     return MatchOperand_NoMatch;
2769   }
2770 
2771   if (Prefix == "row_mirror") {
2772     Int = 0x140;
2773     Parser.Lex();
2774   } else if (Prefix == "row_half_mirror") {
2775     Int = 0x141;
2776     Parser.Lex();
2777   } else {
2778     // Check to prevent parseDPPCtrlOps from eating invalid tokens
2779     if (Prefix != "quad_perm"
2780         && Prefix != "row_shl"
2781         && Prefix != "row_shr"
2782         && Prefix != "row_ror"
2783         && Prefix != "wave_shl"
2784         && Prefix != "wave_rol"
2785         && Prefix != "wave_shr"
2786         && Prefix != "wave_ror"
2787         && Prefix != "row_bcast") {
2788       return MatchOperand_NoMatch;
2789     }
2790 
2791     Parser.Lex();
2792     if (getLexer().isNot(AsmToken::Colon))
2793       return MatchOperand_ParseFail;
2794 
2795     if (Prefix == "quad_perm") {
2796       // quad_perm:[%d,%d,%d,%d]
2797       Parser.Lex();
2798       if (getLexer().isNot(AsmToken::LBrac))
2799         return MatchOperand_ParseFail;
2800       Parser.Lex();
2801 
2802       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
2803         return MatchOperand_ParseFail;
2804 
2805       for (int i = 0; i < 3; ++i) {
2806         if (getLexer().isNot(AsmToken::Comma))
2807           return MatchOperand_ParseFail;
2808         Parser.Lex();
2809 
2810         int64_t Temp;
2811         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
2812           return MatchOperand_ParseFail;
2813         const int shift = i*2 + 2;
2814         Int += (Temp << shift);
2815       }
2816 
2817       if (getLexer().isNot(AsmToken::RBrac))
2818         return MatchOperand_ParseFail;
2819       Parser.Lex();
2820 
2821     } else {
2822       // sel:%d
2823       Parser.Lex();
2824       if (getParser().parseAbsoluteExpression(Int))
2825         return MatchOperand_ParseFail;
2826 
2827       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
2828         Int |= 0x100;
2829       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
2830         Int |= 0x110;
2831       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
2832         Int |= 0x120;
2833       } else if (Prefix == "wave_shl" && 1 == Int) {
2834         Int = 0x130;
2835       } else if (Prefix == "wave_rol" && 1 == Int) {
2836         Int = 0x134;
2837       } else if (Prefix == "wave_shr" && 1 == Int) {
2838         Int = 0x138;
2839       } else if (Prefix == "wave_ror" && 1 == Int) {
2840         Int = 0x13C;
2841       } else if (Prefix == "row_bcast") {
2842         if (Int == 15) {
2843           Int = 0x142;
2844         } else if (Int == 31) {
2845           Int = 0x143;
2846         } else {
2847           return MatchOperand_ParseFail;
2848         }
2849       } else {
2850         return MatchOperand_ParseFail;
2851       }
2852     }
2853   }
2854 
2855   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
2856   return MatchOperand_Success;
2857 }
2858 
2859 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
2860   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
2861 }
2862 
2863 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
2864   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
2865 }
2866 
2867 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
2868   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
2869 }
2870 
2871 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
2872   OptionalImmIndexMap OptionalIdx;
2873 
2874   unsigned I = 1;
2875   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2876   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2877     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2878   }
2879 
2880   for (unsigned E = Operands.size(); I != E; ++I) {
2881     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2882     // Add the register arguments
2883     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
2884       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
2885     } else if (Op.isDPPCtrl()) {
2886       Op.addImmOperands(Inst, 1);
2887     } else if (Op.isImm()) {
2888       // Handle optional arguments
2889       OptionalIdx[Op.getImmTy()] = I;
2890     } else {
2891       llvm_unreachable("Invalid operand type");
2892     }
2893   }
2894 
2895   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
2896   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
2897   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
2898 
2899   // special case v_mac_f32:
2900   // it has src2 register operand that is tied to dst operand
2901   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_dpp) {
2902     auto it = Inst.begin();
2903     std::advance(it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
2904     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
2905   }
2906 }
2907 
2908 //===----------------------------------------------------------------------===//
2909 // sdwa
2910 //===----------------------------------------------------------------------===//
2911 
2912 AMDGPUAsmParser::OperandMatchResultTy
2913 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
2914                               AMDGPUOperand::ImmTy Type) {
2915   using namespace llvm::AMDGPU::SDWA;
2916 
2917   SMLoc S = Parser.getTok().getLoc();
2918   StringRef Value;
2919   AMDGPUAsmParser::OperandMatchResultTy res;
2920 
2921   res = parseStringWithPrefix(Prefix, Value);
2922   if (res != MatchOperand_Success) {
2923     return res;
2924   }
2925 
2926   int64_t Int;
2927   Int = StringSwitch<int64_t>(Value)
2928         .Case("BYTE_0", SdwaSel::BYTE_0)
2929         .Case("BYTE_1", SdwaSel::BYTE_1)
2930         .Case("BYTE_2", SdwaSel::BYTE_2)
2931         .Case("BYTE_3", SdwaSel::BYTE_3)
2932         .Case("WORD_0", SdwaSel::WORD_0)
2933         .Case("WORD_1", SdwaSel::WORD_1)
2934         .Case("DWORD", SdwaSel::DWORD)
2935         .Default(0xffffffff);
2936   Parser.Lex(); // eat last token
2937 
2938   if (Int == 0xffffffff) {
2939     return MatchOperand_ParseFail;
2940   }
2941 
2942   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
2943   return MatchOperand_Success;
2944 }
2945 
2946 AMDGPUAsmParser::OperandMatchResultTy
2947 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
2948   using namespace llvm::AMDGPU::SDWA;
2949 
2950   SMLoc S = Parser.getTok().getLoc();
2951   StringRef Value;
2952   AMDGPUAsmParser::OperandMatchResultTy res;
2953 
2954   res = parseStringWithPrefix("dst_unused", Value);
2955   if (res != MatchOperand_Success) {
2956     return res;
2957   }
2958 
2959   int64_t Int;
2960   Int = StringSwitch<int64_t>(Value)
2961         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
2962         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
2963         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
2964         .Default(0xffffffff);
2965   Parser.Lex(); // eat last token
2966 
2967   if (Int == 0xffffffff) {
2968     return MatchOperand_ParseFail;
2969   }
2970 
2971   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
2972   return MatchOperand_Success;
2973 }
2974 
2975 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
2976   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
2977 }
2978 
2979 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
2980   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
2981 }
2982 
2983 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
2984   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC);
2985 }
2986 
2987 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
2988                               uint64_t BasicInstType) {
2989   OptionalImmIndexMap OptionalIdx;
2990 
2991   unsigned I = 1;
2992   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2993   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2994     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2995   }
2996 
2997   for (unsigned E = Operands.size(); I != E; ++I) {
2998     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2999     // Add the register arguments
3000     if (BasicInstType == SIInstrFlags::VOPC &&
3001         Op.isReg() &&
3002         Op.Reg.RegNo == AMDGPU::VCC) {
3003       // VOPC sdwa use "vcc" token as dst. Skip it.
3004       continue;
3005     } else if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
3006       Op.addRegOrImmWithInputModsOperands(Inst, 2);
3007     } else if (Op.isImm()) {
3008       // Handle optional arguments
3009       OptionalIdx[Op.getImmTy()] = I;
3010     } else {
3011       llvm_unreachable("Invalid operand type");
3012     }
3013   }
3014 
3015   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
3016 
3017   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa) {
3018     // V_NOP_sdwa has no optional sdwa arguments
3019     switch (BasicInstType) {
3020     case SIInstrFlags::VOP1: {
3021       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
3022       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
3023       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3024       break;
3025     }
3026     case SIInstrFlags::VOP2: {
3027       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
3028       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
3029       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3030       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, 6);
3031       break;
3032     }
3033     case SIInstrFlags::VOPC: {
3034       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3035       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, 6);
3036       break;
3037     }
3038     default:
3039       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
3040     }
3041   }
3042 
3043   // special case v_mac_f32:
3044   // it has src2 register operand that is tied to dst operand
3045   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa) {
3046     auto it = Inst.begin();
3047     std::advance(it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3048     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3049   }
3050 
3051 }
3052 
3053 /// Force static initialization.
3054 extern "C" void LLVMInitializeAMDGPUAsmParser() {
3055   RegisterMCAsmParser<AMDGPUAsmParser> A(TheAMDGPUTarget);
3056   RegisterMCAsmParser<AMDGPUAsmParser> B(TheGCNTarget);
3057 }
3058 
3059 #define GET_REGISTER_MATCHER
3060 #define GET_MATCHER_IMPLEMENTATION
3061 #include "AMDGPUGenAsmMatcher.inc"
3062 
3063 
3064 // This fuction should be defined after auto-generated include so that we have
3065 // MatchClassKind enum defined
3066 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
3067                                                      unsigned Kind) {
3068   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
3069   // But MatchInstructionImpl() expects to meet token and fails to validate
3070   // operand. This method checks if we are given immediate operand but expect to
3071   // get corresponding token.
3072   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
3073   switch (Kind) {
3074   case MCK_addr64:
3075     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
3076   case MCK_gds:
3077     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
3078   case MCK_glc:
3079     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
3080   case MCK_idxen:
3081     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
3082   case MCK_offen:
3083     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
3084   case MCK_SSrcB32:
3085     // When operands have expression values, they will return true for isToken,
3086     // because it is not possible to distinguish between a token and an
3087     // expression at parse time. MatchInstructionImpl() will always try to
3088     // match an operand as a token, when isToken returns true, and when the
3089     // name of the expression is not a valid token, the match will fail,
3090     // so we need to handle it here.
3091     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
3092   case MCK_SSrcF32:
3093     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
3094   case MCK_SoppBrTarget:
3095     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
3096   default: return Match_InvalidOperand;
3097   }
3098 }
3099