1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/STLExtras.h"
19 #include "llvm/ADT/SmallBitVector.h"
20 #include "llvm/ADT/SmallString.h"
21 #include "llvm/ADT/StringSwitch.h"
22 #include "llvm/ADT/Twine.h"
23 #include "llvm/CodeGen/MachineValueType.h"
24 #include "llvm/MC/MCContext.h"
25 #include "llvm/MC/MCExpr.h"
26 #include "llvm/MC/MCInst.h"
27 #include "llvm/MC/MCInstrInfo.h"
28 #include "llvm/MC/MCParser/MCAsmLexer.h"
29 #include "llvm/MC/MCParser/MCAsmParser.h"
30 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
31 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
32 #include "llvm/MC/MCRegisterInfo.h"
33 #include "llvm/MC/MCStreamer.h"
34 #include "llvm/MC/MCSubtargetInfo.h"
35 #include "llvm/MC/MCSymbolELF.h"
36 #include "llvm/Support/Debug.h"
37 #include "llvm/Support/ELF.h"
38 #include "llvm/Support/SourceMgr.h"
39 #include "llvm/Support/TargetRegistry.h"
40 #include "llvm/Support/raw_ostream.h"
41 #include "llvm/Support/MathExtras.h"
42 
43 using namespace llvm;
44 using namespace llvm::AMDGPU;
45 
46 namespace {
47 
48 class AMDGPUAsmParser;
49 struct OptionalOperand;
50 
51 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
52 
53 //===----------------------------------------------------------------------===//
54 // Operand
55 //===----------------------------------------------------------------------===//
56 
57 class AMDGPUOperand : public MCParsedAsmOperand {
58   enum KindTy {
59     Token,
60     Immediate,
61     Register,
62     Expression
63   } Kind;
64 
65   SMLoc StartLoc, EndLoc;
66   const AMDGPUAsmParser *AsmParser;
67 
68 public:
69   AMDGPUOperand(enum KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
70     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
71 
72   typedef std::unique_ptr<AMDGPUOperand> Ptr;
73 
74   struct Modifiers {
75     bool Abs;
76     bool Neg;
77     bool Sext;
78 
79     bool hasFPModifiers() const { return Abs || Neg; }
80     bool hasIntModifiers() const { return Sext; }
81     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
82 
83     int64_t getFPModifiersOperand() const {
84       int64_t Operand = 0;
85       Operand |= Abs ? SISrcMods::ABS : 0;
86       Operand |= Neg ? SISrcMods::NEG : 0;
87       return Operand;
88     }
89 
90     int64_t getIntModifiersOperand() const {
91       int64_t Operand = 0;
92       Operand |= Sext ? SISrcMods::SEXT : 0;
93       return Operand;
94     }
95 
96     int64_t getModifiersOperand() const {
97       assert(!(hasFPModifiers() && hasIntModifiers())
98            && "fp and int modifiers should not be used simultaneously");
99       if (hasFPModifiers()) {
100         return getFPModifiersOperand();
101       } else if (hasIntModifiers()) {
102         return getIntModifiersOperand();
103       } else {
104         return 0;
105       }
106     }
107 
108     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
109   };
110 
111   enum ImmTy {
112     ImmTyNone,
113     ImmTyGDS,
114     ImmTyOffen,
115     ImmTyIdxen,
116     ImmTyAddr64,
117     ImmTyOffset,
118     ImmTyOffset0,
119     ImmTyOffset1,
120     ImmTyGLC,
121     ImmTySLC,
122     ImmTyTFE,
123     ImmTyClampSI,
124     ImmTyOModSI,
125     ImmTyDppCtrl,
126     ImmTyDppRowMask,
127     ImmTyDppBankMask,
128     ImmTyDppBoundCtrl,
129     ImmTySdwaDstSel,
130     ImmTySdwaSrc0Sel,
131     ImmTySdwaSrc1Sel,
132     ImmTySdwaDstUnused,
133     ImmTyDMask,
134     ImmTyUNorm,
135     ImmTyDA,
136     ImmTyR128,
137     ImmTyLWE,
138     ImmTyHwreg,
139     ImmTySendMsg,
140   };
141 
142   struct TokOp {
143     const char *Data;
144     unsigned Length;
145   };
146 
147   struct ImmOp {
148     int64_t Val;
149     ImmTy Type;
150     bool IsFPImm;
151     Modifiers Mods;
152   };
153 
154   struct RegOp {
155     unsigned RegNo;
156     bool IsForcedVOP3;
157     Modifiers Mods;
158   };
159 
160   union {
161     TokOp Tok;
162     ImmOp Imm;
163     RegOp Reg;
164     const MCExpr *Expr;
165   };
166 
167   bool isToken() const override {
168     if (Kind == Token)
169       return true;
170 
171     if (Kind != Expression || !Expr)
172       return false;
173 
174     // When parsing operands, we can't always tell if something was meant to be
175     // a token, like 'gds', or an expression that references a global variable.
176     // In this case, we assume the string is an expression, and if we need to
177     // interpret is a token, then we treat the symbol name as the token.
178     return isa<MCSymbolRefExpr>(Expr);
179   }
180 
181   bool isImm() const override {
182     return Kind == Immediate;
183   }
184 
185   bool isInlinableImm(MVT type) const;
186   bool isLiteralImm(MVT type) const;
187 
188   bool isRegKind() const {
189     return Kind == Register;
190   }
191 
192   bool isReg() const override {
193     return isRegKind() && !Reg.Mods.hasModifiers();
194   }
195 
196   bool isRegOrImmWithInputMods(MVT type) const {
197     return isRegKind() || isInlinableImm(type);
198   }
199 
200   bool isRegOrImmWithInt32InputMods() const {
201     return isRegOrImmWithInputMods(MVT::i32);
202   }
203 
204   bool isRegOrImmWithInt64InputMods() const {
205     return isRegOrImmWithInputMods(MVT::i64);
206   }
207 
208   bool isRegOrImmWithFP32InputMods() const {
209     return isRegOrImmWithInputMods(MVT::f32);
210   }
211 
212   bool isRegOrImmWithFP64InputMods() const {
213     return isRegOrImmWithInputMods(MVT::f64);
214   }
215 
216   bool isImmTy(ImmTy ImmT) const {
217     return isImm() && Imm.Type == ImmT;
218   }
219 
220   bool isImmModifier() const {
221     return isImm() && Imm.Type != ImmTyNone;
222   }
223 
224   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
225   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
226   bool isDMask() const { return isImmTy(ImmTyDMask); }
227   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
228   bool isDA() const { return isImmTy(ImmTyDA); }
229   bool isR128() const { return isImmTy(ImmTyUNorm); }
230   bool isLWE() const { return isImmTy(ImmTyLWE); }
231   bool isOffen() const { return isImmTy(ImmTyOffen); }
232   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
233   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
234   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
235   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
236   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
237   bool isGDS() const { return isImmTy(ImmTyGDS); }
238   bool isGLC() const { return isImmTy(ImmTyGLC); }
239   bool isSLC() const { return isImmTy(ImmTySLC); }
240   bool isTFE() const { return isImmTy(ImmTyTFE); }
241   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
242   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
243   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
244   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
245   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
246   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
247   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
248 
249   bool isMod() const {
250     return isClampSI() || isOModSI();
251   }
252 
253   bool isRegOrImm() const {
254     return isReg() || isImm();
255   }
256 
257   bool isRegClass(unsigned RCID) const;
258 
259   bool isSCSrcB32() const {
260     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::i32);
261   }
262 
263   bool isSCSrcB64() const {
264     return isRegClass(AMDGPU::SReg_64RegClassID) || isInlinableImm(MVT::i64);
265   }
266 
267   bool isSCSrcF32() const {
268     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::f32);
269   }
270 
271   bool isSCSrcF64() const {
272     return isRegClass(AMDGPU::SReg_64RegClassID) || isInlinableImm(MVT::f64);
273   }
274 
275   bool isSSrcB32() const {
276     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
277   }
278 
279   bool isSSrcB64() const {
280     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
281     // See isVSrc64().
282     return isSCSrcB64() || isLiteralImm(MVT::i64);
283   }
284 
285   bool isSSrcF32() const {
286     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
287   }
288 
289   bool isSSrcF64() const {
290     return isSCSrcB64() || isLiteralImm(MVT::f64);
291   }
292 
293   bool isVCSrcB32() const {
294     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::i32);
295   }
296 
297   bool isVCSrcB64() const {
298     return isRegClass(AMDGPU::VS_64RegClassID) || isInlinableImm(MVT::i64);
299   }
300 
301   bool isVCSrcF32() const {
302     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::f32);
303   }
304 
305   bool isVCSrcF64() const {
306     return isRegClass(AMDGPU::VS_64RegClassID) || isInlinableImm(MVT::f64);
307   }
308 
309   bool isVSrcB32() const {
310     return isVCSrcF32() || isLiteralImm(MVT::i32);
311   }
312 
313   bool isVSrcB64() const {
314     return isVCSrcF64() || isLiteralImm(MVT::i64);
315   }
316 
317   bool isVSrcF32() const {
318     return isVCSrcF32() || isLiteralImm(MVT::f32);
319   }
320 
321   bool isVSrcF64() const {
322     return isVCSrcF64() || isLiteralImm(MVT::f64);
323   }
324 
325   bool isKImmFP32() const {
326     return isLiteralImm(MVT::f32);
327   }
328 
329   bool isMem() const override {
330     return false;
331   }
332 
333   bool isExpr() const {
334     return Kind == Expression;
335   }
336 
337   bool isSoppBrTarget() const {
338     return isExpr() || isImm();
339   }
340 
341   bool isSWaitCnt() const;
342   bool isHwreg() const;
343   bool isSendMsg() const;
344   bool isSMRDOffset() const;
345   bool isSMRDLiteralOffset() const;
346   bool isDPPCtrl() const;
347 
348   StringRef getExpressionAsToken() const {
349     assert(isExpr());
350     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
351     return S->getSymbol().getName();
352   }
353 
354 
355   StringRef getToken() const {
356     assert(isToken());
357 
358     if (Kind == Expression)
359       return getExpressionAsToken();
360 
361     return StringRef(Tok.Data, Tok.Length);
362   }
363 
364   int64_t getImm() const {
365     assert(isImm());
366     return Imm.Val;
367   }
368 
369   enum ImmTy getImmTy() const {
370     assert(isImm());
371     return Imm.Type;
372   }
373 
374   unsigned getReg() const override {
375     return Reg.RegNo;
376   }
377 
378   SMLoc getStartLoc() const override {
379     return StartLoc;
380   }
381 
382   SMLoc getEndLoc() const override {
383     return EndLoc;
384   }
385 
386   Modifiers getModifiers() const {
387     assert(isRegKind() || isImmTy(ImmTyNone));
388     return isRegKind() ? Reg.Mods : Imm.Mods;
389   }
390 
391   void setModifiers(Modifiers Mods) {
392     assert(isRegKind() || isImmTy(ImmTyNone));
393     if (isRegKind())
394       Reg.Mods = Mods;
395     else
396       Imm.Mods = Mods;
397   }
398 
399   bool hasModifiers() const {
400     return getModifiers().hasModifiers();
401   }
402 
403   bool hasFPModifiers() const {
404     return getModifiers().hasFPModifiers();
405   }
406 
407   bool hasIntModifiers() const {
408     return getModifiers().hasIntModifiers();
409   }
410 
411   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
412 
413   void addLiteralImmOperand(MCInst &Inst, int64_t Val) const;
414 
415   void addKImmFP32Operands(MCInst &Inst, unsigned N) const;
416 
417   void addRegOperands(MCInst &Inst, unsigned N) const;
418 
419   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
420     if (isRegKind())
421       addRegOperands(Inst, N);
422     else if (isExpr())
423       Inst.addOperand(MCOperand::createExpr(Expr));
424     else
425       addImmOperands(Inst, N);
426   }
427 
428   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
429     Modifiers Mods = getModifiers();
430     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
431     if (isRegKind()) {
432       addRegOperands(Inst, N);
433     } else {
434       addImmOperands(Inst, N, false);
435     }
436   }
437 
438   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
439     assert(!hasIntModifiers());
440     addRegOrImmWithInputModsOperands(Inst, N);
441   }
442 
443   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
444     assert(!hasFPModifiers());
445     addRegOrImmWithInputModsOperands(Inst, N);
446   }
447 
448   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
449     if (isImm())
450       addImmOperands(Inst, N);
451     else {
452       assert(isExpr());
453       Inst.addOperand(MCOperand::createExpr(Expr));
454     }
455   }
456 
457   void printImmTy(raw_ostream& OS, ImmTy Type) const {
458     switch (Type) {
459     case ImmTyNone: OS << "None"; break;
460     case ImmTyGDS: OS << "GDS"; break;
461     case ImmTyOffen: OS << "Offen"; break;
462     case ImmTyIdxen: OS << "Idxen"; break;
463     case ImmTyAddr64: OS << "Addr64"; break;
464     case ImmTyOffset: OS << "Offset"; break;
465     case ImmTyOffset0: OS << "Offset0"; break;
466     case ImmTyOffset1: OS << "Offset1"; break;
467     case ImmTyGLC: OS << "GLC"; break;
468     case ImmTySLC: OS << "SLC"; break;
469     case ImmTyTFE: OS << "TFE"; break;
470     case ImmTyClampSI: OS << "ClampSI"; break;
471     case ImmTyOModSI: OS << "OModSI"; break;
472     case ImmTyDppCtrl: OS << "DppCtrl"; break;
473     case ImmTyDppRowMask: OS << "DppRowMask"; break;
474     case ImmTyDppBankMask: OS << "DppBankMask"; break;
475     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
476     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
477     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
478     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
479     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
480     case ImmTyDMask: OS << "DMask"; break;
481     case ImmTyUNorm: OS << "UNorm"; break;
482     case ImmTyDA: OS << "DA"; break;
483     case ImmTyR128: OS << "R128"; break;
484     case ImmTyLWE: OS << "LWE"; break;
485     case ImmTyHwreg: OS << "Hwreg"; break;
486     case ImmTySendMsg: OS << "SendMsg"; break;
487     }
488   }
489 
490   void print(raw_ostream &OS) const override {
491     switch (Kind) {
492     case Register:
493       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
494       break;
495     case Immediate:
496       OS << '<' << getImm();
497       if (getImmTy() != ImmTyNone) {
498         OS << " type: "; printImmTy(OS, getImmTy());
499       }
500       OS << " mods: " << Imm.Mods << '>';
501       break;
502     case Token:
503       OS << '\'' << getToken() << '\'';
504       break;
505     case Expression:
506       OS << "<expr " << *Expr << '>';
507       break;
508     }
509   }
510 
511   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
512                                       int64_t Val, SMLoc Loc,
513                                       enum ImmTy Type = ImmTyNone,
514                                       bool IsFPImm = false) {
515     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
516     Op->Imm.Val = Val;
517     Op->Imm.IsFPImm = IsFPImm;
518     Op->Imm.Type = Type;
519     Op->Imm.Mods = {false, false, false};
520     Op->StartLoc = Loc;
521     Op->EndLoc = Loc;
522     return Op;
523   }
524 
525   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
526                                         StringRef Str, SMLoc Loc,
527                                         bool HasExplicitEncodingSize = true) {
528     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
529     Res->Tok.Data = Str.data();
530     Res->Tok.Length = Str.size();
531     Res->StartLoc = Loc;
532     Res->EndLoc = Loc;
533     return Res;
534   }
535 
536   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
537                                       unsigned RegNo, SMLoc S,
538                                       SMLoc E,
539                                       bool ForceVOP3) {
540     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
541     Op->Reg.RegNo = RegNo;
542     Op->Reg.Mods = {false, false, false};
543     Op->Reg.IsForcedVOP3 = ForceVOP3;
544     Op->StartLoc = S;
545     Op->EndLoc = E;
546     return Op;
547   }
548 
549   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
550                                        const class MCExpr *Expr, SMLoc S) {
551     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
552     Op->Expr = Expr;
553     Op->StartLoc = S;
554     Op->EndLoc = S;
555     return Op;
556   }
557 };
558 
559 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
560   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
561   return OS;
562 }
563 
564 //===----------------------------------------------------------------------===//
565 // AsmParser
566 //===----------------------------------------------------------------------===//
567 
568 class AMDGPUAsmParser : public MCTargetAsmParser {
569   const MCInstrInfo &MII;
570   MCAsmParser &Parser;
571 
572   unsigned ForcedEncodingSize;
573   bool ForcedDPP;
574   bool ForcedSDWA;
575 
576   /// @name Auto-generated Match Functions
577   /// {
578 
579 #define GET_ASSEMBLER_HEADER
580 #include "AMDGPUGenAsmMatcher.inc"
581 
582   /// }
583 
584 private:
585   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
586   bool ParseDirectiveHSACodeObjectVersion();
587   bool ParseDirectiveHSACodeObjectISA();
588   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
589   bool ParseDirectiveAMDKernelCodeT();
590   bool ParseSectionDirectiveHSAText();
591   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
592   bool ParseDirectiveAMDGPUHsaKernel();
593   bool ParseDirectiveAMDGPUHsaModuleGlobal();
594   bool ParseDirectiveAMDGPUHsaProgramGlobal();
595   bool ParseSectionDirectiveHSADataGlobalAgent();
596   bool ParseSectionDirectiveHSADataGlobalProgram();
597   bool ParseSectionDirectiveHSARodataReadonlyAgent();
598   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum);
599   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth);
600   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands, bool IsAtomic, bool IsAtomicReturn);
601 
602 public:
603   enum AMDGPUMatchResultTy {
604     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
605   };
606 
607   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
608                const MCInstrInfo &MII,
609                const MCTargetOptions &Options)
610       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser),
611         ForcedEncodingSize(0),
612         ForcedDPP(false),
613         ForcedSDWA(false) {
614     MCAsmParserExtension::Initialize(Parser);
615 
616     if (getSTI().getFeatureBits().none()) {
617       // Set default features.
618       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
619     }
620 
621     setAvailableFeatures(ComputeAvailableFeatures(getSTI().getFeatureBits()));
622 
623     {
624       // TODO: make those pre-defined variables read-only.
625       // Currently there is none suitable machinery in the core llvm-mc for this.
626       // MCSymbol::isRedefinable is intended for another purpose, and
627       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
628       AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
629       MCContext &Ctx = getContext();
630       MCSymbol *Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
631       Sym->setVariableValue(MCConstantExpr::create(Isa.Major, Ctx));
632       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
633       Sym->setVariableValue(MCConstantExpr::create(Isa.Minor, Ctx));
634       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
635       Sym->setVariableValue(MCConstantExpr::create(Isa.Stepping, Ctx));
636     }
637   }
638 
639   bool isSI() const {
640     return AMDGPU::isSI(getSTI());
641   }
642 
643   bool isCI() const {
644     return AMDGPU::isCI(getSTI());
645   }
646 
647   bool isVI() const {
648     return AMDGPU::isVI(getSTI());
649   }
650 
651   bool hasSGPR102_SGPR103() const {
652     return !isVI();
653   }
654 
655   AMDGPUTargetStreamer &getTargetStreamer() {
656     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
657     return static_cast<AMDGPUTargetStreamer &>(TS);
658   }
659 
660   const MCRegisterInfo *getMRI() const {
661     // We need this const_cast because for some reason getContext() is not const
662     // in MCAsmParser.
663     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
664   }
665 
666   const MCInstrInfo *getMII() const {
667     return &MII;
668   }
669 
670   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
671   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
672   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
673 
674   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
675   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
676   bool isForcedDPP() const { return ForcedDPP; }
677   bool isForcedSDWA() const { return ForcedSDWA; }
678 
679   std::unique_ptr<AMDGPUOperand> parseRegister();
680   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
681   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
682   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
683                                       unsigned Kind) override;
684   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
685                                OperandVector &Operands, MCStreamer &Out,
686                                uint64_t &ErrorInfo,
687                                bool MatchingInlineAsm) override;
688   bool ParseDirective(AsmToken DirectiveID) override;
689   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
690   StringRef parseMnemonicSuffix(StringRef Name);
691   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
692                         SMLoc NameLoc, OperandVector &Operands) override;
693   //bool ProcessInstruction(MCInst &Inst);
694 
695   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
696   OperandMatchResultTy parseIntWithPrefix(const char *Prefix,
697                                           OperandVector &Operands,
698                                           enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
699                                           bool (*ConvertResult)(int64_t&) = 0);
700   OperandMatchResultTy parseNamedBit(const char *Name, OperandVector &Operands,
701                                      enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
702   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix, StringRef &Value);
703 
704   OperandMatchResultTy parseImm(OperandVector &Operands);
705   OperandMatchResultTy parseRegOrImm(OperandVector &Operands);
706   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands);
707   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands);
708 
709   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
710   void cvtDS(MCInst &Inst, const OperandVector &Operands);
711 
712   bool parseCnt(int64_t &IntVal);
713   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
714   OperandMatchResultTy parseHwreg(OperandVector &Operands);
715 
716 private:
717   struct OperandInfoTy {
718     int64_t Id;
719     bool IsSymbolic;
720     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
721   };
722 
723   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
724   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
725 public:
726   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
727 
728   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
729   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
730 
731   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
732   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
733   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
734   AMDGPUOperand::Ptr defaultGLC() const;
735   AMDGPUOperand::Ptr defaultSLC() const;
736   AMDGPUOperand::Ptr defaultTFE() const;
737 
738   AMDGPUOperand::Ptr defaultDMask() const;
739   AMDGPUOperand::Ptr defaultUNorm() const;
740   AMDGPUOperand::Ptr defaultDA() const;
741   AMDGPUOperand::Ptr defaultR128() const;
742   AMDGPUOperand::Ptr defaultLWE() const;
743   AMDGPUOperand::Ptr defaultSMRDOffset() const;
744   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
745 
746   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
747 
748   void cvtId(MCInst &Inst, const OperandVector &Operands);
749   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
750   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
751 
752   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
753   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
754 
755   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
756   AMDGPUOperand::Ptr defaultRowMask() const;
757   AMDGPUOperand::Ptr defaultBankMask() const;
758   AMDGPUOperand::Ptr defaultBoundCtrl() const;
759   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
760 
761   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
762                                     AMDGPUOperand::ImmTy Type);
763   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
764   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
765   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
766   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
767   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
768                uint64_t BasicInstType);
769 };
770 
771 struct OptionalOperand {
772   const char *Name;
773   AMDGPUOperand::ImmTy Type;
774   bool IsBit;
775   bool (*ConvertResult)(int64_t&);
776 };
777 
778 }
779 
780 //===----------------------------------------------------------------------===//
781 // Operand
782 //===----------------------------------------------------------------------===//
783 
784 bool AMDGPUOperand::isInlinableImm(MVT type) const {
785   if (!isImmTy(ImmTyNone)) {
786     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
787     return false;
788   }
789   // TODO: We should avoid using host float here. It would be better to
790   // check the float bit values which is what a few other places do.
791   // We've had bot failures before due to weird NaN support on mips hosts.
792 
793   APInt Literal(64, Imm.Val);
794 
795   if (Imm.IsFPImm) { // We got fp literal token
796     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
797       return AMDGPU::isInlinableLiteral64(Imm.Val, AsmParser->isVI());
798     } else { // Expected 32-bit operand
799       bool lost;
800       APFloat FPLiteral(APFloat::IEEEdouble, Literal);
801       // Convert literal to single precision
802       APFloat::opStatus status = FPLiteral.convert(APFloat::IEEEsingle,
803                                                     APFloat::rmNearestTiesToEven,
804                                                     &lost);
805       // We allow precision lost but not overflow or underflow
806       if (status != APFloat::opOK &&
807           lost &&
808           ((status & APFloat::opOverflow)  != 0 ||
809             (status & APFloat::opUnderflow) != 0)) {
810         return false;
811       }
812       // Check if single precision literal is inlinable
813       return AMDGPU::isInlinableLiteral32(
814               static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
815               AsmParser->isVI());
816     }
817   } else { // We got int literal token
818     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
819       return AMDGPU::isInlinableLiteral64(Imm.Val, AsmParser->isVI());
820     } else { // Expected 32-bit operand
821       return AMDGPU::isInlinableLiteral32(
822             static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
823             AsmParser->isVI());
824     }
825   }
826   return false;
827 }
828 
829 bool AMDGPUOperand::isLiteralImm(MVT type) const {
830   // Check that this imediate can be added as literal
831   if (!isImmTy(ImmTyNone)) {
832     return false;
833   }
834 
835   APInt Literal(64, Imm.Val);
836 
837   if (Imm.IsFPImm) { // We got fp literal token
838     if (type == MVT::f64) { // Expected 64-bit fp operand
839       // We would set low 64-bits of literal to zeroes but we accept this literals
840       return true;
841     } else if (type == MVT::i64) { // Expected 64-bit int operand
842         // We don't allow fp literals in 64-bit integer instructions. It is
843         // unclear how we should encode them.
844       return false;
845     } else { // Expected 32-bit operand
846       bool lost;
847       APFloat FPLiteral(APFloat::IEEEdouble, Literal);
848       // Convert literal to single precision
849       APFloat::opStatus status = FPLiteral.convert(APFloat::IEEEsingle,
850                                                     APFloat::rmNearestTiesToEven,
851                                                     &lost);
852       // We allow precision lost but not overflow or underflow
853       if (status != APFloat::opOK &&
854           lost &&
855           ((status & APFloat::opOverflow)  != 0 ||
856             (status & APFloat::opUnderflow) != 0)) {
857         return false;
858       }
859       return true;
860     }
861   } else { // We got int literal token
862     APInt HiBits = Literal.getHiBits(32);
863     if (HiBits == 0xffffffff &&
864         (*Literal.getLoBits(32).getRawData() & 0x80000000) != 0) {
865       // If high 32 bits aren't zeroes then they all should be ones and 32nd
866       // bit should be set. So that this 64-bit literal is sign-extension of
867       // 32-bit value.
868       return true;
869     } else if (HiBits == 0) {
870       return true;
871     }
872   }
873   return false;
874 }
875 
876 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
877   return isReg() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
878 }
879 
880 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
881   int64_t Val = Imm.Val;
882   if (isImmTy(ImmTyNone) && ApplyModifiers && Imm.Mods.hasFPModifiers() && Imm.Mods.Neg) {
883     // Apply modifiers to immediate value. Only negate can get here
884     if (Imm.IsFPImm) {
885       APFloat F(BitsToDouble(Val));
886       F.changeSign();
887       Val = F.bitcastToAPInt().getZExtValue();
888     } else {
889       Val = -Val;
890     }
891   }
892 
893   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()), Inst.getNumOperands())) {
894     addLiteralImmOperand(Inst, Val);
895   } else {
896     Inst.addOperand(MCOperand::createImm(Val));
897   }
898 }
899 
900 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val) const {
901   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
902   auto OpNum = Inst.getNumOperands();
903   // Check that this operand accepts literals
904   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
905 
906   APInt Literal(64, Val);
907   auto OpSize = AMDGPU::getRegOperandSize(AsmParser->getMRI(), InstDesc, OpNum); // expected operand size
908 
909   if (Imm.IsFPImm) { // We got fp literal token
910     if (OpSize == 8) { // Expected 64-bit operand
911       // Check if literal is inlinable
912       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(), AsmParser->isVI())) {
913         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
914       } else if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
915         // For fp operands we check if low 32 bits are zeros
916         if (Literal.getLoBits(32) != 0) {
917           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
918                             "Can't encode literal as exact 64-bit"
919                             " floating-point operand. Low 32-bits will be"
920                             " set to zero");
921         }
922         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
923       } else {
924         // We don't allow fp literals in 64-bit integer instructions. It is
925         // unclear how we should encode them. This case should be checked earlier
926         // in predicate methods (isLiteralImm())
927         llvm_unreachable("fp literal in 64-bit integer instruction.");
928       }
929     } else { // Expected 32-bit operand
930       bool lost;
931       APFloat FPLiteral(APFloat::IEEEdouble, Literal);
932       // Convert literal to single precision
933       FPLiteral.convert(APFloat::IEEEsingle, APFloat::rmNearestTiesToEven, &lost);
934       // We allow precision lost but not overflow or underflow. This should be
935       // checked earlier in isLiteralImm()
936       Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
937     }
938   } else { // We got int literal token
939     if (OpSize == 8) { // Expected 64-bit operand
940       auto LiteralVal = Literal.getZExtValue();
941       if (AMDGPU::isInlinableLiteral64(LiteralVal, AsmParser->isVI())) {
942         Inst.addOperand(MCOperand::createImm(LiteralVal));
943         return;
944       }
945     } else { // Expected 32-bit operand
946       auto LiteralVal = static_cast<int32_t>(Literal.getLoBits(32).getZExtValue());
947       if (AMDGPU::isInlinableLiteral32(LiteralVal, AsmParser->isVI())) {
948         Inst.addOperand(MCOperand::createImm(LiteralVal));
949         return;
950       }
951     }
952     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(32).getZExtValue()));
953   }
954 }
955 
956 void AMDGPUOperand::addKImmFP32Operands(MCInst &Inst, unsigned N) const {
957   APInt Literal(64, Imm.Val);
958   if (Imm.IsFPImm) { // We got fp literal
959     bool lost;
960     APFloat FPLiteral(APFloat::IEEEdouble, Literal);
961     FPLiteral.convert(APFloat::IEEEsingle, APFloat::rmNearestTiesToEven, &lost);
962     Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
963   } else { // We got int literal token
964     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(32).getZExtValue()));
965   }
966 }
967 
968 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
969   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
970 }
971 
972 //===----------------------------------------------------------------------===//
973 // AsmParser
974 //===----------------------------------------------------------------------===//
975 
976 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
977   if (Is == IS_VGPR) {
978     switch (RegWidth) {
979       default: return -1;
980       case 1: return AMDGPU::VGPR_32RegClassID;
981       case 2: return AMDGPU::VReg_64RegClassID;
982       case 3: return AMDGPU::VReg_96RegClassID;
983       case 4: return AMDGPU::VReg_128RegClassID;
984       case 8: return AMDGPU::VReg_256RegClassID;
985       case 16: return AMDGPU::VReg_512RegClassID;
986     }
987   } else if (Is == IS_TTMP) {
988     switch (RegWidth) {
989       default: return -1;
990       case 1: return AMDGPU::TTMP_32RegClassID;
991       case 2: return AMDGPU::TTMP_64RegClassID;
992       case 4: return AMDGPU::TTMP_128RegClassID;
993     }
994   } else if (Is == IS_SGPR) {
995     switch (RegWidth) {
996       default: return -1;
997       case 1: return AMDGPU::SGPR_32RegClassID;
998       case 2: return AMDGPU::SGPR_64RegClassID;
999       case 4: return AMDGPU::SGPR_128RegClassID;
1000       case 8: return AMDGPU::SReg_256RegClassID;
1001       case 16: return AMDGPU::SReg_512RegClassID;
1002     }
1003   }
1004   return -1;
1005 }
1006 
1007 static unsigned getSpecialRegForName(StringRef RegName) {
1008   return StringSwitch<unsigned>(RegName)
1009     .Case("exec", AMDGPU::EXEC)
1010     .Case("vcc", AMDGPU::VCC)
1011     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1012     .Case("m0", AMDGPU::M0)
1013     .Case("scc", AMDGPU::SCC)
1014     .Case("tba", AMDGPU::TBA)
1015     .Case("tma", AMDGPU::TMA)
1016     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1017     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1018     .Case("vcc_lo", AMDGPU::VCC_LO)
1019     .Case("vcc_hi", AMDGPU::VCC_HI)
1020     .Case("exec_lo", AMDGPU::EXEC_LO)
1021     .Case("exec_hi", AMDGPU::EXEC_HI)
1022     .Case("tma_lo", AMDGPU::TMA_LO)
1023     .Case("tma_hi", AMDGPU::TMA_HI)
1024     .Case("tba_lo", AMDGPU::TBA_LO)
1025     .Case("tba_hi", AMDGPU::TBA_HI)
1026     .Default(0);
1027 }
1028 
1029 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) {
1030   auto R = parseRegister();
1031   if (!R) return true;
1032   assert(R->isReg());
1033   RegNo = R->getReg();
1034   StartLoc = R->getStartLoc();
1035   EndLoc = R->getEndLoc();
1036   return false;
1037 }
1038 
1039 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum)
1040 {
1041   switch (RegKind) {
1042   case IS_SPECIAL:
1043     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) { Reg = AMDGPU::EXEC; RegWidth = 2; return true; }
1044     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) { Reg = AMDGPU::FLAT_SCR; RegWidth = 2; return true; }
1045     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) { Reg = AMDGPU::VCC; RegWidth = 2; return true; }
1046     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) { Reg = AMDGPU::TBA; RegWidth = 2; return true; }
1047     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) { Reg = AMDGPU::TMA; RegWidth = 2; return true; }
1048     return false;
1049   case IS_VGPR:
1050   case IS_SGPR:
1051   case IS_TTMP:
1052     if (Reg1 != Reg + RegWidth) { return false; }
1053     RegWidth++;
1054     return true;
1055   default:
1056     assert(false); return false;
1057   }
1058 }
1059 
1060 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth)
1061 {
1062   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1063   if (getLexer().is(AsmToken::Identifier)) {
1064     StringRef RegName = Parser.getTok().getString();
1065     if ((Reg = getSpecialRegForName(RegName))) {
1066       Parser.Lex();
1067       RegKind = IS_SPECIAL;
1068     } else {
1069       unsigned RegNumIndex = 0;
1070       if (RegName[0] == 'v') {
1071         RegNumIndex = 1;
1072         RegKind = IS_VGPR;
1073       } else if (RegName[0] == 's') {
1074         RegNumIndex = 1;
1075         RegKind = IS_SGPR;
1076       } else if (RegName.startswith("ttmp")) {
1077         RegNumIndex = strlen("ttmp");
1078         RegKind = IS_TTMP;
1079       } else {
1080         return false;
1081       }
1082       if (RegName.size() > RegNumIndex) {
1083         // Single 32-bit register: vXX.
1084         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1085           return false;
1086         Parser.Lex();
1087         RegWidth = 1;
1088       } else {
1089         // Range of registers: v[XX:YY]. ":YY" is optional.
1090         Parser.Lex();
1091         int64_t RegLo, RegHi;
1092         if (getLexer().isNot(AsmToken::LBrac))
1093           return false;
1094         Parser.Lex();
1095 
1096         if (getParser().parseAbsoluteExpression(RegLo))
1097           return false;
1098 
1099         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1100         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1101           return false;
1102         Parser.Lex();
1103 
1104         if (isRBrace) {
1105           RegHi = RegLo;
1106         } else {
1107           if (getParser().parseAbsoluteExpression(RegHi))
1108             return false;
1109 
1110           if (getLexer().isNot(AsmToken::RBrac))
1111             return false;
1112           Parser.Lex();
1113         }
1114         RegNum = (unsigned) RegLo;
1115         RegWidth = (RegHi - RegLo) + 1;
1116       }
1117     }
1118   } else if (getLexer().is(AsmToken::LBrac)) {
1119     // List of consecutive registers: [s0,s1,s2,s3]
1120     Parser.Lex();
1121     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth))
1122       return false;
1123     if (RegWidth != 1)
1124       return false;
1125     RegisterKind RegKind1;
1126     unsigned Reg1, RegNum1, RegWidth1;
1127     do {
1128       if (getLexer().is(AsmToken::Comma)) {
1129         Parser.Lex();
1130       } else if (getLexer().is(AsmToken::RBrac)) {
1131         Parser.Lex();
1132         break;
1133       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1)) {
1134         if (RegWidth1 != 1) {
1135           return false;
1136         }
1137         if (RegKind1 != RegKind) {
1138           return false;
1139         }
1140         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1141           return false;
1142         }
1143       } else {
1144         return false;
1145       }
1146     } while (true);
1147   } else {
1148     return false;
1149   }
1150   switch (RegKind) {
1151   case IS_SPECIAL:
1152     RegNum = 0;
1153     RegWidth = 1;
1154     break;
1155   case IS_VGPR:
1156   case IS_SGPR:
1157   case IS_TTMP:
1158   {
1159     unsigned Size = 1;
1160     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1161       // SGPR and TTMP registers must be are aligned. Max required alignment is 4 dwords.
1162       Size = std::min(RegWidth, 4u);
1163     }
1164     if (RegNum % Size != 0)
1165       return false;
1166     RegNum = RegNum / Size;
1167     int RCID = getRegClass(RegKind, RegWidth);
1168     if (RCID == -1)
1169       return false;
1170     const MCRegisterClass RC = TRI->getRegClass(RCID);
1171     if (RegNum >= RC.getNumRegs())
1172       return false;
1173     Reg = RC.getRegister(RegNum);
1174     break;
1175   }
1176 
1177   default:
1178     assert(false); return false;
1179   }
1180 
1181   if (!subtargetHasRegister(*TRI, Reg))
1182     return false;
1183   return true;
1184 }
1185 
1186 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1187   const auto &Tok = Parser.getTok();
1188   SMLoc StartLoc = Tok.getLoc();
1189   SMLoc EndLoc = Tok.getEndLoc();
1190   RegisterKind RegKind;
1191   unsigned Reg, RegNum, RegWidth;
1192 
1193   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth)) {
1194     return nullptr;
1195   }
1196   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1197 }
1198 
1199 AMDGPUAsmParser::OperandMatchResultTy
1200 AMDGPUAsmParser::parseImm(OperandVector &Operands) {
1201   // TODO: add syntactic sugar for 1/(2*PI)
1202   bool Minus = false;
1203   if (getLexer().getKind() == AsmToken::Minus) {
1204     Minus = true;
1205     Parser.Lex();
1206   }
1207 
1208   SMLoc S = Parser.getTok().getLoc();
1209   switch(getLexer().getKind()) {
1210   case AsmToken::Integer: {
1211     int64_t IntVal;
1212     if (getParser().parseAbsoluteExpression(IntVal))
1213       return MatchOperand_ParseFail;
1214     if (Minus)
1215       IntVal *= -1;
1216     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1217     return MatchOperand_Success;
1218   }
1219   case AsmToken::Real: {
1220     int64_t IntVal;
1221     if (getParser().parseAbsoluteExpression(IntVal))
1222       return MatchOperand_ParseFail;
1223 
1224     APFloat F(BitsToDouble(IntVal));
1225     if (Minus)
1226       F.changeSign();
1227     Operands.push_back(
1228         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1229                                  AMDGPUOperand::ImmTyNone, true));
1230     return MatchOperand_Success;
1231   }
1232   default:
1233     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1234   }
1235 }
1236 
1237 AMDGPUAsmParser::OperandMatchResultTy
1238 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands) {
1239   auto res = parseImm(Operands);
1240   if (res != MatchOperand_NoMatch) {
1241     return res;
1242   }
1243 
1244   if (auto R = parseRegister()) {
1245     assert(R->isReg());
1246     R->Reg.IsForcedVOP3 = isForcedVOP3();
1247     Operands.push_back(std::move(R));
1248     return MatchOperand_Success;
1249   }
1250   return MatchOperand_ParseFail;
1251 }
1252 
1253 AMDGPUAsmParser::OperandMatchResultTy
1254 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands) {
1255   // XXX: During parsing we can't determine if minus sign means
1256   // negate-modifier or negative immediate value.
1257   // By default we suppose it is modifier.
1258   bool Negate = false, Abs = false, Abs2 = false;
1259 
1260   if (getLexer().getKind()== AsmToken::Minus) {
1261     Parser.Lex();
1262     Negate = true;
1263   }
1264 
1265   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "abs") {
1266     Parser.Lex();
1267     Abs2 = true;
1268     if (getLexer().isNot(AsmToken::LParen)) {
1269       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1270       return MatchOperand_ParseFail;
1271     }
1272     Parser.Lex();
1273   }
1274 
1275   if (getLexer().getKind() == AsmToken::Pipe) {
1276     if (Abs2) {
1277       Error(Parser.getTok().getLoc(), "expected register or immediate");
1278       return MatchOperand_ParseFail;
1279     }
1280     Parser.Lex();
1281     Abs = true;
1282   }
1283 
1284   auto Res = parseRegOrImm(Operands);
1285   if (Res != MatchOperand_Success) {
1286     return Res;
1287   }
1288 
1289   AMDGPUOperand::Modifiers Mods = {false, false, false};
1290   if (Negate) {
1291     Mods.Neg = true;
1292   }
1293   if (Abs) {
1294     if (getLexer().getKind() != AsmToken::Pipe) {
1295       Error(Parser.getTok().getLoc(), "expected vertical bar");
1296       return MatchOperand_ParseFail;
1297     }
1298     Parser.Lex();
1299     Mods.Abs = true;
1300   }
1301   if (Abs2) {
1302     if (getLexer().isNot(AsmToken::RParen)) {
1303       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1304       return MatchOperand_ParseFail;
1305     }
1306     Parser.Lex();
1307     Mods.Abs = true;
1308   }
1309 
1310   if (Mods.hasFPModifiers()) {
1311     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1312     Op.setModifiers(Mods);
1313   }
1314   return MatchOperand_Success;
1315 }
1316 
1317 AMDGPUAsmParser::OperandMatchResultTy
1318 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands) {
1319   bool Sext = false;
1320 
1321   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "sext") {
1322     Parser.Lex();
1323     Sext = true;
1324     if (getLexer().isNot(AsmToken::LParen)) {
1325       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1326       return MatchOperand_ParseFail;
1327     }
1328     Parser.Lex();
1329   }
1330 
1331   auto Res = parseRegOrImm(Operands);
1332   if (Res != MatchOperand_Success) {
1333     return Res;
1334   }
1335 
1336   AMDGPUOperand::Modifiers Mods = {false, false, false};
1337   if (Sext) {
1338     if (getLexer().isNot(AsmToken::RParen)) {
1339       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1340       return MatchOperand_ParseFail;
1341     }
1342     Parser.Lex();
1343     Mods.Sext = true;
1344   }
1345 
1346   if (Mods.hasIntModifiers()) {
1347     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1348     Op.setModifiers(Mods);
1349   }
1350   return MatchOperand_Success;
1351 }
1352 
1353 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1354 
1355   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1356 
1357   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1358       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1359       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1360       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1361     return Match_InvalidOperand;
1362 
1363   if ((TSFlags & SIInstrFlags::VOP3) &&
1364       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1365       getForcedEncodingSize() != 64)
1366     return Match_PreferE32;
1367 
1368   if (Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa ||
1369       Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa) {
1370     // v_mac_f32/16 allow only dst_sel == DWORD;
1371     auto OpNum = AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
1372     const auto &Op = Inst.getOperand(OpNum);
1373     if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
1374       return Match_InvalidOperand;
1375     }
1376   }
1377 
1378   return Match_Success;
1379 }
1380 
1381 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1382                                               OperandVector &Operands,
1383                                               MCStreamer &Out,
1384                                               uint64_t &ErrorInfo,
1385                                               bool MatchingInlineAsm) {
1386   // What asm variants we should check
1387   std::vector<unsigned> MatchedVariants;
1388   if (getForcedEncodingSize() == 32) {
1389     MatchedVariants = {AMDGPUAsmVariants::DEFAULT};
1390   } else if (isForcedVOP3()) {
1391     MatchedVariants = {AMDGPUAsmVariants::VOP3};
1392   } else if (isForcedSDWA()) {
1393     MatchedVariants = {AMDGPUAsmVariants::SDWA};
1394   } else if (isForcedDPP()) {
1395     MatchedVariants = {AMDGPUAsmVariants::DPP};
1396   } else {
1397     MatchedVariants = {AMDGPUAsmVariants::DEFAULT,
1398                        AMDGPUAsmVariants::VOP3,
1399                        AMDGPUAsmVariants::SDWA,
1400                        AMDGPUAsmVariants::DPP};
1401   }
1402 
1403   MCInst Inst;
1404   unsigned Result = Match_Success;
1405   for (auto Variant : MatchedVariants) {
1406     uint64_t EI;
1407     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
1408                                   Variant);
1409     // We order match statuses from least to most specific. We use most specific
1410     // status as resulting
1411     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
1412     if ((R == Match_Success) ||
1413         (R == Match_PreferE32) ||
1414         (R == Match_MissingFeature && Result != Match_PreferE32) ||
1415         (R == Match_InvalidOperand && Result != Match_MissingFeature
1416                                    && Result != Match_PreferE32) ||
1417         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
1418                                    && Result != Match_MissingFeature
1419                                    && Result != Match_PreferE32)) {
1420       Result = R;
1421       ErrorInfo = EI;
1422     }
1423     if (R == Match_Success)
1424       break;
1425   }
1426 
1427   switch (Result) {
1428   default: break;
1429   case Match_Success:
1430     Inst.setLoc(IDLoc);
1431     Out.EmitInstruction(Inst, getSTI());
1432     return false;
1433 
1434   case Match_MissingFeature:
1435     return Error(IDLoc, "instruction not supported on this GPU");
1436 
1437   case Match_MnemonicFail:
1438     return Error(IDLoc, "unrecognized instruction mnemonic");
1439 
1440   case Match_InvalidOperand: {
1441     SMLoc ErrorLoc = IDLoc;
1442     if (ErrorInfo != ~0ULL) {
1443       if (ErrorInfo >= Operands.size()) {
1444         return Error(IDLoc, "too few operands for instruction");
1445       }
1446       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
1447       if (ErrorLoc == SMLoc())
1448         ErrorLoc = IDLoc;
1449     }
1450     return Error(ErrorLoc, "invalid operand for instruction");
1451   }
1452 
1453   case Match_PreferE32:
1454     return Error(IDLoc, "internal error: instruction without _e64 suffix "
1455                         "should be encoded as e32");
1456   }
1457   llvm_unreachable("Implement any new match types added!");
1458 }
1459 
1460 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
1461                                                uint32_t &Minor) {
1462   if (getLexer().isNot(AsmToken::Integer))
1463     return TokError("invalid major version");
1464 
1465   Major = getLexer().getTok().getIntVal();
1466   Lex();
1467 
1468   if (getLexer().isNot(AsmToken::Comma))
1469     return TokError("minor version number required, comma expected");
1470   Lex();
1471 
1472   if (getLexer().isNot(AsmToken::Integer))
1473     return TokError("invalid minor version");
1474 
1475   Minor = getLexer().getTok().getIntVal();
1476   Lex();
1477 
1478   return false;
1479 }
1480 
1481 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
1482 
1483   uint32_t Major;
1484   uint32_t Minor;
1485 
1486   if (ParseDirectiveMajorMinor(Major, Minor))
1487     return true;
1488 
1489   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
1490   return false;
1491 }
1492 
1493 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
1494 
1495   uint32_t Major;
1496   uint32_t Minor;
1497   uint32_t Stepping;
1498   StringRef VendorName;
1499   StringRef ArchName;
1500 
1501   // If this directive has no arguments, then use the ISA version for the
1502   // targeted GPU.
1503   if (getLexer().is(AsmToken::EndOfStatement)) {
1504     AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
1505     getTargetStreamer().EmitDirectiveHSACodeObjectISA(Isa.Major, Isa.Minor,
1506                                                       Isa.Stepping,
1507                                                       "AMD", "AMDGPU");
1508     return false;
1509   }
1510 
1511 
1512   if (ParseDirectiveMajorMinor(Major, Minor))
1513     return true;
1514 
1515   if (getLexer().isNot(AsmToken::Comma))
1516     return TokError("stepping version number required, comma expected");
1517   Lex();
1518 
1519   if (getLexer().isNot(AsmToken::Integer))
1520     return TokError("invalid stepping version");
1521 
1522   Stepping = getLexer().getTok().getIntVal();
1523   Lex();
1524 
1525   if (getLexer().isNot(AsmToken::Comma))
1526     return TokError("vendor name required, comma expected");
1527   Lex();
1528 
1529   if (getLexer().isNot(AsmToken::String))
1530     return TokError("invalid vendor name");
1531 
1532   VendorName = getLexer().getTok().getStringContents();
1533   Lex();
1534 
1535   if (getLexer().isNot(AsmToken::Comma))
1536     return TokError("arch name required, comma expected");
1537   Lex();
1538 
1539   if (getLexer().isNot(AsmToken::String))
1540     return TokError("invalid arch name");
1541 
1542   ArchName = getLexer().getTok().getStringContents();
1543   Lex();
1544 
1545   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
1546                                                     VendorName, ArchName);
1547   return false;
1548 }
1549 
1550 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
1551                                                amd_kernel_code_t &Header) {
1552   SmallString<40> ErrStr;
1553   raw_svector_ostream Err(ErrStr);
1554   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
1555     return TokError(Err.str());
1556   }
1557   Lex();
1558   return false;
1559 }
1560 
1561 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
1562 
1563   amd_kernel_code_t Header;
1564   AMDGPU::initDefaultAMDKernelCodeT(Header, getSTI().getFeatureBits());
1565 
1566   while (true) {
1567 
1568     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
1569     // will set the current token to EndOfStatement.
1570     while(getLexer().is(AsmToken::EndOfStatement))
1571       Lex();
1572 
1573     if (getLexer().isNot(AsmToken::Identifier))
1574       return TokError("expected value identifier or .end_amd_kernel_code_t");
1575 
1576     StringRef ID = getLexer().getTok().getIdentifier();
1577     Lex();
1578 
1579     if (ID == ".end_amd_kernel_code_t")
1580       break;
1581 
1582     if (ParseAMDKernelCodeTValue(ID, Header))
1583       return true;
1584   }
1585 
1586   getTargetStreamer().EmitAMDKernelCodeT(Header);
1587 
1588   return false;
1589 }
1590 
1591 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
1592   getParser().getStreamer().SwitchSection(
1593       AMDGPU::getHSATextSection(getContext()));
1594   return false;
1595 }
1596 
1597 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
1598   if (getLexer().isNot(AsmToken::Identifier))
1599     return TokError("expected symbol name");
1600 
1601   StringRef KernelName = Parser.getTok().getString();
1602 
1603   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
1604                                            ELF::STT_AMDGPU_HSA_KERNEL);
1605   Lex();
1606   return false;
1607 }
1608 
1609 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
1610   if (getLexer().isNot(AsmToken::Identifier))
1611     return TokError("expected symbol name");
1612 
1613   StringRef GlobalName = Parser.getTok().getIdentifier();
1614 
1615   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
1616   Lex();
1617   return false;
1618 }
1619 
1620 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
1621   if (getLexer().isNot(AsmToken::Identifier))
1622     return TokError("expected symbol name");
1623 
1624   StringRef GlobalName = Parser.getTok().getIdentifier();
1625 
1626   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
1627   Lex();
1628   return false;
1629 }
1630 
1631 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
1632   getParser().getStreamer().SwitchSection(
1633       AMDGPU::getHSADataGlobalAgentSection(getContext()));
1634   return false;
1635 }
1636 
1637 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
1638   getParser().getStreamer().SwitchSection(
1639       AMDGPU::getHSADataGlobalProgramSection(getContext()));
1640   return false;
1641 }
1642 
1643 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
1644   getParser().getStreamer().SwitchSection(
1645       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
1646   return false;
1647 }
1648 
1649 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
1650   StringRef IDVal = DirectiveID.getString();
1651 
1652   if (IDVal == ".hsa_code_object_version")
1653     return ParseDirectiveHSACodeObjectVersion();
1654 
1655   if (IDVal == ".hsa_code_object_isa")
1656     return ParseDirectiveHSACodeObjectISA();
1657 
1658   if (IDVal == ".amd_kernel_code_t")
1659     return ParseDirectiveAMDKernelCodeT();
1660 
1661   if (IDVal == ".hsatext")
1662     return ParseSectionDirectiveHSAText();
1663 
1664   if (IDVal == ".amdgpu_hsa_kernel")
1665     return ParseDirectiveAMDGPUHsaKernel();
1666 
1667   if (IDVal == ".amdgpu_hsa_module_global")
1668     return ParseDirectiveAMDGPUHsaModuleGlobal();
1669 
1670   if (IDVal == ".amdgpu_hsa_program_global")
1671     return ParseDirectiveAMDGPUHsaProgramGlobal();
1672 
1673   if (IDVal == ".hsadata_global_agent")
1674     return ParseSectionDirectiveHSADataGlobalAgent();
1675 
1676   if (IDVal == ".hsadata_global_program")
1677     return ParseSectionDirectiveHSADataGlobalProgram();
1678 
1679   if (IDVal == ".hsarodata_readonly_agent")
1680     return ParseSectionDirectiveHSARodataReadonlyAgent();
1681 
1682   return true;
1683 }
1684 
1685 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
1686                                            unsigned RegNo) const {
1687   if (isCI())
1688     return true;
1689 
1690   if (isSI()) {
1691     // No flat_scr
1692     switch (RegNo) {
1693     case AMDGPU::FLAT_SCR:
1694     case AMDGPU::FLAT_SCR_LO:
1695     case AMDGPU::FLAT_SCR_HI:
1696       return false;
1697     default:
1698       return true;
1699     }
1700   }
1701 
1702   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
1703   // SI/CI have.
1704   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
1705        R.isValid(); ++R) {
1706     if (*R == RegNo)
1707       return false;
1708   }
1709 
1710   return true;
1711 }
1712 
1713 AMDGPUAsmParser::OperandMatchResultTy
1714 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
1715 
1716   // Try to parse with a custom parser
1717   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
1718 
1719   // If we successfully parsed the operand or if there as an error parsing,
1720   // we are done.
1721   //
1722   // If we are parsing after we reach EndOfStatement then this means we
1723   // are appending default values to the Operands list.  This is only done
1724   // by custom parser, so we shouldn't continue on to the generic parsing.
1725   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
1726       getLexer().is(AsmToken::EndOfStatement))
1727     return ResTy;
1728 
1729   ResTy = parseRegOrImm(Operands);
1730 
1731   if (ResTy == MatchOperand_Success)
1732     return ResTy;
1733 
1734   if (getLexer().getKind() == AsmToken::Identifier) {
1735     // If this identifier is a symbol, we want to create an expression for it.
1736     // It is a little difficult to distinguish between a symbol name, and
1737     // an instruction flag like 'gds'.  In order to do this, we parse
1738     // all tokens as expressions and then treate the symbol name as the token
1739     // string when we want to interpret the operand as a token.
1740     const auto &Tok = Parser.getTok();
1741     SMLoc S = Tok.getLoc();
1742     const MCExpr *Expr = nullptr;
1743     if (!Parser.parseExpression(Expr)) {
1744       Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
1745       return MatchOperand_Success;
1746     }
1747 
1748     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), Tok.getLoc()));
1749     Parser.Lex();
1750     return MatchOperand_Success;
1751   }
1752   return MatchOperand_NoMatch;
1753 }
1754 
1755 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
1756   // Clear any forced encodings from the previous instruction.
1757   setForcedEncodingSize(0);
1758   setForcedDPP(false);
1759   setForcedSDWA(false);
1760 
1761   if (Name.endswith("_e64")) {
1762     setForcedEncodingSize(64);
1763     return Name.substr(0, Name.size() - 4);
1764   } else if (Name.endswith("_e32")) {
1765     setForcedEncodingSize(32);
1766     return Name.substr(0, Name.size() - 4);
1767   } else if (Name.endswith("_dpp")) {
1768     setForcedDPP(true);
1769     return Name.substr(0, Name.size() - 4);
1770   } else if (Name.endswith("_sdwa")) {
1771     setForcedSDWA(true);
1772     return Name.substr(0, Name.size() - 5);
1773   }
1774   return Name;
1775 }
1776 
1777 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
1778                                        StringRef Name,
1779                                        SMLoc NameLoc, OperandVector &Operands) {
1780   // Add the instruction mnemonic
1781   Name = parseMnemonicSuffix(Name);
1782   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
1783 
1784   while (!getLexer().is(AsmToken::EndOfStatement)) {
1785     AMDGPUAsmParser::OperandMatchResultTy Res = parseOperand(Operands, Name);
1786 
1787     // Eat the comma or space if there is one.
1788     if (getLexer().is(AsmToken::Comma))
1789       Parser.Lex();
1790 
1791     switch (Res) {
1792       case MatchOperand_Success: break;
1793       case MatchOperand_ParseFail:
1794         Error(getLexer().getLoc(), "failed parsing operand.");
1795         while (!getLexer().is(AsmToken::EndOfStatement)) {
1796           Parser.Lex();
1797         }
1798         return true;
1799       case MatchOperand_NoMatch:
1800         Error(getLexer().getLoc(), "not a valid operand.");
1801         while (!getLexer().is(AsmToken::EndOfStatement)) {
1802           Parser.Lex();
1803         }
1804         return true;
1805     }
1806   }
1807 
1808   return false;
1809 }
1810 
1811 //===----------------------------------------------------------------------===//
1812 // Utility functions
1813 //===----------------------------------------------------------------------===//
1814 
1815 AMDGPUAsmParser::OperandMatchResultTy
1816 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
1817   switch(getLexer().getKind()) {
1818     default: return MatchOperand_NoMatch;
1819     case AsmToken::Identifier: {
1820       StringRef Name = Parser.getTok().getString();
1821       if (!Name.equals(Prefix)) {
1822         return MatchOperand_NoMatch;
1823       }
1824 
1825       Parser.Lex();
1826       if (getLexer().isNot(AsmToken::Colon))
1827         return MatchOperand_ParseFail;
1828 
1829       Parser.Lex();
1830       if (getLexer().isNot(AsmToken::Integer))
1831         return MatchOperand_ParseFail;
1832 
1833       if (getParser().parseAbsoluteExpression(Int))
1834         return MatchOperand_ParseFail;
1835       break;
1836     }
1837   }
1838   return MatchOperand_Success;
1839 }
1840 
1841 AMDGPUAsmParser::OperandMatchResultTy
1842 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
1843                                     enum AMDGPUOperand::ImmTy ImmTy,
1844                                     bool (*ConvertResult)(int64_t&)) {
1845 
1846   SMLoc S = Parser.getTok().getLoc();
1847   int64_t Value = 0;
1848 
1849   AMDGPUAsmParser::OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
1850   if (Res != MatchOperand_Success)
1851     return Res;
1852 
1853   if (ConvertResult && !ConvertResult(Value)) {
1854     return MatchOperand_ParseFail;
1855   }
1856 
1857   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
1858   return MatchOperand_Success;
1859 }
1860 
1861 AMDGPUAsmParser::OperandMatchResultTy
1862 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
1863                                enum AMDGPUOperand::ImmTy ImmTy) {
1864   int64_t Bit = 0;
1865   SMLoc S = Parser.getTok().getLoc();
1866 
1867   // We are at the end of the statement, and this is a default argument, so
1868   // use a default value.
1869   if (getLexer().isNot(AsmToken::EndOfStatement)) {
1870     switch(getLexer().getKind()) {
1871       case AsmToken::Identifier: {
1872         StringRef Tok = Parser.getTok().getString();
1873         if (Tok == Name) {
1874           Bit = 1;
1875           Parser.Lex();
1876         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
1877           Bit = 0;
1878           Parser.Lex();
1879         } else {
1880           return MatchOperand_NoMatch;
1881         }
1882         break;
1883       }
1884       default:
1885         return MatchOperand_NoMatch;
1886     }
1887   }
1888 
1889   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
1890   return MatchOperand_Success;
1891 }
1892 
1893 typedef std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
1894 
1895 void addOptionalImmOperand(MCInst& Inst, const OperandVector& Operands,
1896                            OptionalImmIndexMap& OptionalIdx,
1897                            enum AMDGPUOperand::ImmTy ImmT, int64_t Default = 0) {
1898   auto i = OptionalIdx.find(ImmT);
1899   if (i != OptionalIdx.end()) {
1900     unsigned Idx = i->second;
1901     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
1902   } else {
1903     Inst.addOperand(MCOperand::createImm(Default));
1904   }
1905 }
1906 
1907 AMDGPUAsmParser::OperandMatchResultTy
1908 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
1909   if (getLexer().isNot(AsmToken::Identifier)) {
1910     return MatchOperand_NoMatch;
1911   }
1912   StringRef Tok = Parser.getTok().getString();
1913   if (Tok != Prefix) {
1914     return MatchOperand_NoMatch;
1915   }
1916 
1917   Parser.Lex();
1918   if (getLexer().isNot(AsmToken::Colon)) {
1919     return MatchOperand_ParseFail;
1920   }
1921 
1922   Parser.Lex();
1923   if (getLexer().isNot(AsmToken::Identifier)) {
1924     return MatchOperand_ParseFail;
1925   }
1926 
1927   Value = Parser.getTok().getString();
1928   return MatchOperand_Success;
1929 }
1930 
1931 //===----------------------------------------------------------------------===//
1932 // ds
1933 //===----------------------------------------------------------------------===//
1934 
1935 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
1936                                     const OperandVector &Operands) {
1937 
1938   OptionalImmIndexMap OptionalIdx;
1939 
1940   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1941     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1942 
1943     // Add the register arguments
1944     if (Op.isReg()) {
1945       Op.addRegOperands(Inst, 1);
1946       continue;
1947     }
1948 
1949     // Handle optional arguments
1950     OptionalIdx[Op.getImmTy()] = i;
1951   }
1952 
1953   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
1954   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
1955   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1956 
1957   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1958 }
1959 
1960 void AMDGPUAsmParser::cvtDS(MCInst &Inst, const OperandVector &Operands) {
1961 
1962   std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
1963   bool GDSOnly = false;
1964 
1965   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1966     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1967 
1968     // Add the register arguments
1969     if (Op.isReg()) {
1970       Op.addRegOperands(Inst, 1);
1971       continue;
1972     }
1973 
1974     if (Op.isToken() && Op.getToken() == "gds") {
1975       GDSOnly = true;
1976       continue;
1977     }
1978 
1979     // Handle optional arguments
1980     OptionalIdx[Op.getImmTy()] = i;
1981   }
1982 
1983   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
1984   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1985 
1986   if (!GDSOnly) {
1987     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1988   }
1989   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1990 }
1991 
1992 
1993 //===----------------------------------------------------------------------===//
1994 // s_waitcnt
1995 //===----------------------------------------------------------------------===//
1996 
1997 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
1998   StringRef CntName = Parser.getTok().getString();
1999   int64_t CntVal;
2000 
2001   Parser.Lex();
2002   if (getLexer().isNot(AsmToken::LParen))
2003     return true;
2004 
2005   Parser.Lex();
2006   if (getLexer().isNot(AsmToken::Integer))
2007     return true;
2008 
2009   if (getParser().parseAbsoluteExpression(CntVal))
2010     return true;
2011 
2012   if (getLexer().isNot(AsmToken::RParen))
2013     return true;
2014 
2015   Parser.Lex();
2016   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
2017     Parser.Lex();
2018 
2019   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2020   if (CntName == "vmcnt")
2021     IntVal = encodeVmcnt(IV, IntVal, CntVal);
2022   else if (CntName == "expcnt")
2023     IntVal = encodeExpcnt(IV, IntVal, CntVal);
2024   else if (CntName == "lgkmcnt")
2025     IntVal = encodeLgkmcnt(IV, IntVal, CntVal);
2026   else
2027     return true;
2028 
2029   return false;
2030 }
2031 
2032 AMDGPUAsmParser::OperandMatchResultTy
2033 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
2034   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2035   int64_t Waitcnt = getWaitcntBitMask(IV);
2036   SMLoc S = Parser.getTok().getLoc();
2037 
2038   switch(getLexer().getKind()) {
2039     default: return MatchOperand_ParseFail;
2040     case AsmToken::Integer:
2041       // The operand can be an integer value.
2042       if (getParser().parseAbsoluteExpression(Waitcnt))
2043         return MatchOperand_ParseFail;
2044       break;
2045 
2046     case AsmToken::Identifier:
2047       do {
2048         if (parseCnt(Waitcnt))
2049           return MatchOperand_ParseFail;
2050       } while(getLexer().isNot(AsmToken::EndOfStatement));
2051       break;
2052   }
2053   Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
2054   return MatchOperand_Success;
2055 }
2056 
2057 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width) {
2058   using namespace llvm::AMDGPU::Hwreg;
2059 
2060   if (Parser.getTok().getString() != "hwreg")
2061     return true;
2062   Parser.Lex();
2063 
2064   if (getLexer().isNot(AsmToken::LParen))
2065     return true;
2066   Parser.Lex();
2067 
2068   if (getLexer().is(AsmToken::Identifier)) {
2069     HwReg.IsSymbolic = true;
2070     HwReg.Id = ID_UNKNOWN_;
2071     const StringRef tok = Parser.getTok().getString();
2072     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
2073       if (tok == IdSymbolic[i]) {
2074         HwReg.Id = i;
2075         break;
2076       }
2077     }
2078     Parser.Lex();
2079   } else {
2080     HwReg.IsSymbolic = false;
2081     if (getLexer().isNot(AsmToken::Integer))
2082       return true;
2083     if (getParser().parseAbsoluteExpression(HwReg.Id))
2084       return true;
2085   }
2086 
2087   if (getLexer().is(AsmToken::RParen)) {
2088     Parser.Lex();
2089     return false;
2090   }
2091 
2092   // optional params
2093   if (getLexer().isNot(AsmToken::Comma))
2094     return true;
2095   Parser.Lex();
2096 
2097   if (getLexer().isNot(AsmToken::Integer))
2098     return true;
2099   if (getParser().parseAbsoluteExpression(Offset))
2100     return true;
2101 
2102   if (getLexer().isNot(AsmToken::Comma))
2103     return true;
2104   Parser.Lex();
2105 
2106   if (getLexer().isNot(AsmToken::Integer))
2107     return true;
2108   if (getParser().parseAbsoluteExpression(Width))
2109     return true;
2110 
2111   if (getLexer().isNot(AsmToken::RParen))
2112     return true;
2113   Parser.Lex();
2114 
2115   return false;
2116 }
2117 
2118 AMDGPUAsmParser::OperandMatchResultTy
2119 AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
2120   using namespace llvm::AMDGPU::Hwreg;
2121 
2122   int64_t Imm16Val = 0;
2123   SMLoc S = Parser.getTok().getLoc();
2124 
2125   switch(getLexer().getKind()) {
2126     default: return MatchOperand_NoMatch;
2127     case AsmToken::Integer:
2128       // The operand can be an integer value.
2129       if (getParser().parseAbsoluteExpression(Imm16Val))
2130         return MatchOperand_NoMatch;
2131       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2132         Error(S, "invalid immediate: only 16-bit values are legal");
2133         // Do not return error code, but create an imm operand anyway and proceed
2134         // to the next operand, if any. That avoids unneccessary error messages.
2135       }
2136       break;
2137 
2138     case AsmToken::Identifier: {
2139         OperandInfoTy HwReg(ID_UNKNOWN_);
2140         int64_t Offset = OFFSET_DEFAULT_;
2141         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
2142         if (parseHwregConstruct(HwReg, Offset, Width))
2143           return MatchOperand_ParseFail;
2144         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
2145           if (HwReg.IsSymbolic)
2146             Error(S, "invalid symbolic name of hardware register");
2147           else
2148             Error(S, "invalid code of hardware register: only 6-bit values are legal");
2149         }
2150         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
2151           Error(S, "invalid bit offset: only 5-bit values are legal");
2152         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
2153           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
2154         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
2155       }
2156       break;
2157   }
2158   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
2159   return MatchOperand_Success;
2160 }
2161 
2162 bool AMDGPUOperand::isSWaitCnt() const {
2163   return isImm();
2164 }
2165 
2166 bool AMDGPUOperand::isHwreg() const {
2167   return isImmTy(ImmTyHwreg);
2168 }
2169 
2170 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
2171   using namespace llvm::AMDGPU::SendMsg;
2172 
2173   if (Parser.getTok().getString() != "sendmsg")
2174     return true;
2175   Parser.Lex();
2176 
2177   if (getLexer().isNot(AsmToken::LParen))
2178     return true;
2179   Parser.Lex();
2180 
2181   if (getLexer().is(AsmToken::Identifier)) {
2182     Msg.IsSymbolic = true;
2183     Msg.Id = ID_UNKNOWN_;
2184     const std::string tok = Parser.getTok().getString();
2185     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
2186       switch(i) {
2187         default: continue; // Omit gaps.
2188         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
2189       }
2190       if (tok == IdSymbolic[i]) {
2191         Msg.Id = i;
2192         break;
2193       }
2194     }
2195     Parser.Lex();
2196   } else {
2197     Msg.IsSymbolic = false;
2198     if (getLexer().isNot(AsmToken::Integer))
2199       return true;
2200     if (getParser().parseAbsoluteExpression(Msg.Id))
2201       return true;
2202     if (getLexer().is(AsmToken::Integer))
2203       if (getParser().parseAbsoluteExpression(Msg.Id))
2204         Msg.Id = ID_UNKNOWN_;
2205   }
2206   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
2207     return false;
2208 
2209   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
2210     if (getLexer().isNot(AsmToken::RParen))
2211       return true;
2212     Parser.Lex();
2213     return false;
2214   }
2215 
2216   if (getLexer().isNot(AsmToken::Comma))
2217     return true;
2218   Parser.Lex();
2219 
2220   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
2221   Operation.Id = ID_UNKNOWN_;
2222   if (getLexer().is(AsmToken::Identifier)) {
2223     Operation.IsSymbolic = true;
2224     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
2225     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
2226     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
2227     const StringRef Tok = Parser.getTok().getString();
2228     for (int i = F; i < L; ++i) {
2229       if (Tok == S[i]) {
2230         Operation.Id = i;
2231         break;
2232       }
2233     }
2234     Parser.Lex();
2235   } else {
2236     Operation.IsSymbolic = false;
2237     if (getLexer().isNot(AsmToken::Integer))
2238       return true;
2239     if (getParser().parseAbsoluteExpression(Operation.Id))
2240       return true;
2241   }
2242 
2243   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2244     // Stream id is optional.
2245     if (getLexer().is(AsmToken::RParen)) {
2246       Parser.Lex();
2247       return false;
2248     }
2249 
2250     if (getLexer().isNot(AsmToken::Comma))
2251       return true;
2252     Parser.Lex();
2253 
2254     if (getLexer().isNot(AsmToken::Integer))
2255       return true;
2256     if (getParser().parseAbsoluteExpression(StreamId))
2257       return true;
2258   }
2259 
2260   if (getLexer().isNot(AsmToken::RParen))
2261     return true;
2262   Parser.Lex();
2263   return false;
2264 }
2265 
2266 AMDGPUAsmParser::OperandMatchResultTy
2267 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
2268   using namespace llvm::AMDGPU::SendMsg;
2269 
2270   int64_t Imm16Val = 0;
2271   SMLoc S = Parser.getTok().getLoc();
2272 
2273   switch(getLexer().getKind()) {
2274   default:
2275     return MatchOperand_NoMatch;
2276   case AsmToken::Integer:
2277     // The operand can be an integer value.
2278     if (getParser().parseAbsoluteExpression(Imm16Val))
2279       return MatchOperand_NoMatch;
2280     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2281       Error(S, "invalid immediate: only 16-bit values are legal");
2282       // Do not return error code, but create an imm operand anyway and proceed
2283       // to the next operand, if any. That avoids unneccessary error messages.
2284     }
2285     break;
2286   case AsmToken::Identifier: {
2287       OperandInfoTy Msg(ID_UNKNOWN_);
2288       OperandInfoTy Operation(OP_UNKNOWN_);
2289       int64_t StreamId = STREAM_ID_DEFAULT_;
2290       if (parseSendMsgConstruct(Msg, Operation, StreamId))
2291         return MatchOperand_ParseFail;
2292       do {
2293         // Validate and encode message ID.
2294         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
2295                 || Msg.Id == ID_SYSMSG)) {
2296           if (Msg.IsSymbolic)
2297             Error(S, "invalid/unsupported symbolic name of message");
2298           else
2299             Error(S, "invalid/unsupported code of message");
2300           break;
2301         }
2302         Imm16Val = (Msg.Id << ID_SHIFT_);
2303         // Validate and encode operation ID.
2304         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
2305           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
2306             if (Operation.IsSymbolic)
2307               Error(S, "invalid symbolic name of GS_OP");
2308             else
2309               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
2310             break;
2311           }
2312           if (Operation.Id == OP_GS_NOP
2313               && Msg.Id != ID_GS_DONE) {
2314             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
2315             break;
2316           }
2317           Imm16Val |= (Operation.Id << OP_SHIFT_);
2318         }
2319         if (Msg.Id == ID_SYSMSG) {
2320           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
2321             if (Operation.IsSymbolic)
2322               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
2323             else
2324               Error(S, "invalid/unsupported code of SYSMSG_OP");
2325             break;
2326           }
2327           Imm16Val |= (Operation.Id << OP_SHIFT_);
2328         }
2329         // Validate and encode stream ID.
2330         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2331           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
2332             Error(S, "invalid stream id: only 2-bit values are legal");
2333             break;
2334           }
2335           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
2336         }
2337       } while (0);
2338     }
2339     break;
2340   }
2341   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
2342   return MatchOperand_Success;
2343 }
2344 
2345 bool AMDGPUOperand::isSendMsg() const {
2346   return isImmTy(ImmTySendMsg);
2347 }
2348 
2349 //===----------------------------------------------------------------------===//
2350 // sopp branch targets
2351 //===----------------------------------------------------------------------===//
2352 
2353 AMDGPUAsmParser::OperandMatchResultTy
2354 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
2355   SMLoc S = Parser.getTok().getLoc();
2356 
2357   switch (getLexer().getKind()) {
2358     default: return MatchOperand_ParseFail;
2359     case AsmToken::Integer: {
2360       int64_t Imm;
2361       if (getParser().parseAbsoluteExpression(Imm))
2362         return MatchOperand_ParseFail;
2363       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
2364       return MatchOperand_Success;
2365     }
2366 
2367     case AsmToken::Identifier:
2368       Operands.push_back(AMDGPUOperand::CreateExpr(this,
2369           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
2370                                   Parser.getTok().getString()), getContext()), S));
2371       Parser.Lex();
2372       return MatchOperand_Success;
2373   }
2374 }
2375 
2376 //===----------------------------------------------------------------------===//
2377 // mubuf
2378 //===----------------------------------------------------------------------===//
2379 
2380 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
2381   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
2382 }
2383 
2384 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
2385   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
2386 }
2387 
2388 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
2389   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
2390 }
2391 
2392 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
2393                                const OperandVector &Operands,
2394                                bool IsAtomic, bool IsAtomicReturn) {
2395   OptionalImmIndexMap OptionalIdx;
2396   assert(IsAtomicReturn ? IsAtomic : true);
2397 
2398   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2399     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2400 
2401     // Add the register arguments
2402     if (Op.isReg()) {
2403       Op.addRegOperands(Inst, 1);
2404       continue;
2405     }
2406 
2407     // Handle the case where soffset is an immediate
2408     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
2409       Op.addImmOperands(Inst, 1);
2410       continue;
2411     }
2412 
2413     // Handle tokens like 'offen' which are sometimes hard-coded into the
2414     // asm string.  There are no MCInst operands for these.
2415     if (Op.isToken()) {
2416       continue;
2417     }
2418     assert(Op.isImm());
2419 
2420     // Handle optional arguments
2421     OptionalIdx[Op.getImmTy()] = i;
2422   }
2423 
2424   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
2425   if (IsAtomicReturn) {
2426     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
2427     Inst.insert(I, *I);
2428   }
2429 
2430   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2431   if (!IsAtomic) { // glc is hard-coded.
2432     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2433   }
2434   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2435   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2436 }
2437 
2438 //===----------------------------------------------------------------------===//
2439 // mimg
2440 //===----------------------------------------------------------------------===//
2441 
2442 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
2443   unsigned I = 1;
2444   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2445   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2446     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2447   }
2448 
2449   OptionalImmIndexMap OptionalIdx;
2450 
2451   for (unsigned E = Operands.size(); I != E; ++I) {
2452     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2453 
2454     // Add the register arguments
2455     if (Op.isRegOrImm()) {
2456       Op.addRegOrImmOperands(Inst, 1);
2457       continue;
2458     } else if (Op.isImmModifier()) {
2459       OptionalIdx[Op.getImmTy()] = I;
2460     } else {
2461       assert(false);
2462     }
2463   }
2464 
2465   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2466   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2467   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2468   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2469   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2470   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2471   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2472   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2473 }
2474 
2475 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
2476   unsigned I = 1;
2477   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2478   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2479     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2480   }
2481 
2482   // Add src, same as dst
2483   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
2484 
2485   OptionalImmIndexMap OptionalIdx;
2486 
2487   for (unsigned E = Operands.size(); I != E; ++I) {
2488     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2489 
2490     // Add the register arguments
2491     if (Op.isRegOrImm()) {
2492       Op.addRegOrImmOperands(Inst, 1);
2493       continue;
2494     } else if (Op.isImmModifier()) {
2495       OptionalIdx[Op.getImmTy()] = I;
2496     } else {
2497       assert(false);
2498     }
2499   }
2500 
2501   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2502   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2503   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2504   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2505   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2506   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2507   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2508   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2509 }
2510 
2511 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
2512   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
2513 }
2514 
2515 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
2516   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
2517 }
2518 
2519 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
2520   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
2521 }
2522 
2523 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
2524   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
2525 }
2526 
2527 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
2528   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
2529 }
2530 
2531 //===----------------------------------------------------------------------===//
2532 // smrd
2533 //===----------------------------------------------------------------------===//
2534 
2535 bool AMDGPUOperand::isSMRDOffset() const {
2536 
2537   // FIXME: Support 20-bit offsets on VI.  We need to to pass subtarget
2538   // information here.
2539   return isImm() && isUInt<8>(getImm());
2540 }
2541 
2542 bool AMDGPUOperand::isSMRDLiteralOffset() const {
2543   // 32-bit literals are only supported on CI and we only want to use them
2544   // when the offset is > 8-bits.
2545   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
2546 }
2547 
2548 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset() const {
2549   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2550 }
2551 
2552 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
2553   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2554 }
2555 
2556 //===----------------------------------------------------------------------===//
2557 // vop3
2558 //===----------------------------------------------------------------------===//
2559 
2560 static bool ConvertOmodMul(int64_t &Mul) {
2561   if (Mul != 1 && Mul != 2 && Mul != 4)
2562     return false;
2563 
2564   Mul >>= 1;
2565   return true;
2566 }
2567 
2568 static bool ConvertOmodDiv(int64_t &Div) {
2569   if (Div == 1) {
2570     Div = 0;
2571     return true;
2572   }
2573 
2574   if (Div == 2) {
2575     Div = 3;
2576     return true;
2577   }
2578 
2579   return false;
2580 }
2581 
2582 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
2583   if (BoundCtrl == 0) {
2584     BoundCtrl = 1;
2585     return true;
2586   } else if (BoundCtrl == -1) {
2587     BoundCtrl = 0;
2588     return true;
2589   }
2590   return false;
2591 }
2592 
2593 // Note: the order in this table matches the order of operands in AsmString.
2594 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
2595   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
2596   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
2597   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
2598   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
2599   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
2600   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
2601   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
2602   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
2603   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
2604   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
2605   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
2606   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
2607   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
2608   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
2609   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
2610   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
2611   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
2612   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
2613   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
2614   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
2615   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
2616   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
2617   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
2618   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
2619 };
2620 
2621 AMDGPUAsmParser::OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
2622   OperandMatchResultTy res;
2623   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
2624     // try to parse any optional operand here
2625     if (Op.IsBit) {
2626       res = parseNamedBit(Op.Name, Operands, Op.Type);
2627     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
2628       res = parseOModOperand(Operands);
2629     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
2630                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
2631                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
2632       res = parseSDWASel(Operands, Op.Name, Op.Type);
2633     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
2634       res = parseSDWADstUnused(Operands);
2635     } else {
2636       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
2637     }
2638     if (res != MatchOperand_NoMatch) {
2639       return res;
2640     }
2641   }
2642   return MatchOperand_NoMatch;
2643 }
2644 
2645 AMDGPUAsmParser::OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands)
2646 {
2647   StringRef Name = Parser.getTok().getString();
2648   if (Name == "mul") {
2649     return parseIntWithPrefix("mul", Operands, AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
2650   } else if (Name == "div") {
2651     return parseIntWithPrefix("div", Operands, AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
2652   } else {
2653     return MatchOperand_NoMatch;
2654   }
2655 }
2656 
2657 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
2658   unsigned I = 1;
2659   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2660   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2661     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2662   }
2663   for (unsigned E = Operands.size(); I != E; ++I)
2664     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
2665 }
2666 
2667 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
2668   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
2669   if (TSFlags & SIInstrFlags::VOP3) {
2670     cvtVOP3(Inst, Operands);
2671   } else {
2672     cvtId(Inst, Operands);
2673   }
2674 }
2675 
2676 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
2677       // 1. This operand is input modifiers
2678   return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
2679       // 2. This is not last operand
2680       && Desc.NumOperands > (OpNum + 1)
2681       // 3. Next operand is register class
2682       && Desc.OpInfo[OpNum + 1].RegClass != -1
2683       // 4. Next register is not tied to any other operand
2684       && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
2685 }
2686 
2687 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
2688   OptionalImmIndexMap OptionalIdx;
2689   unsigned I = 1;
2690   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2691   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2692     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2693   }
2694 
2695   for (unsigned E = Operands.size(); I != E; ++I) {
2696     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2697     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
2698       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
2699     } else if (Op.isImm()) {
2700       OptionalIdx[Op.getImmTy()] = I;
2701     } else {
2702       assert(false);
2703     }
2704   }
2705 
2706   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
2707   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
2708 
2709   // special case v_mac_f32:
2710   // it has src2 register operand that is tied to dst operand
2711   // we don't allow modifiers for this operand in assembler so src2_modifiers
2712   // should be 0
2713   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_si ||
2714       Inst.getOpcode() == AMDGPU::V_MAC_F32_e64_vi) {
2715     auto it = Inst.begin();
2716     std::advance(it, AMDGPU::getNamedOperandIdx(AMDGPU::V_MAC_F32_e64, AMDGPU::OpName::src2_modifiers));
2717     it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
2718     ++it;
2719     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
2720   }
2721 }
2722 
2723 //===----------------------------------------------------------------------===//
2724 // dpp
2725 //===----------------------------------------------------------------------===//
2726 
2727 bool AMDGPUOperand::isDPPCtrl() const {
2728   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
2729   if (result) {
2730     int64_t Imm = getImm();
2731     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
2732            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
2733            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
2734            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
2735            (Imm == 0x130) ||
2736            (Imm == 0x134) ||
2737            (Imm == 0x138) ||
2738            (Imm == 0x13c) ||
2739            (Imm == 0x140) ||
2740            (Imm == 0x141) ||
2741            (Imm == 0x142) ||
2742            (Imm == 0x143);
2743   }
2744   return false;
2745 }
2746 
2747 AMDGPUAsmParser::OperandMatchResultTy
2748 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
2749   SMLoc S = Parser.getTok().getLoc();
2750   StringRef Prefix;
2751   int64_t Int;
2752 
2753   if (getLexer().getKind() == AsmToken::Identifier) {
2754     Prefix = Parser.getTok().getString();
2755   } else {
2756     return MatchOperand_NoMatch;
2757   }
2758 
2759   if (Prefix == "row_mirror") {
2760     Int = 0x140;
2761     Parser.Lex();
2762   } else if (Prefix == "row_half_mirror") {
2763     Int = 0x141;
2764     Parser.Lex();
2765   } else {
2766     // Check to prevent parseDPPCtrlOps from eating invalid tokens
2767     if (Prefix != "quad_perm"
2768         && Prefix != "row_shl"
2769         && Prefix != "row_shr"
2770         && Prefix != "row_ror"
2771         && Prefix != "wave_shl"
2772         && Prefix != "wave_rol"
2773         && Prefix != "wave_shr"
2774         && Prefix != "wave_ror"
2775         && Prefix != "row_bcast") {
2776       return MatchOperand_NoMatch;
2777     }
2778 
2779     Parser.Lex();
2780     if (getLexer().isNot(AsmToken::Colon))
2781       return MatchOperand_ParseFail;
2782 
2783     if (Prefix == "quad_perm") {
2784       // quad_perm:[%d,%d,%d,%d]
2785       Parser.Lex();
2786       if (getLexer().isNot(AsmToken::LBrac))
2787         return MatchOperand_ParseFail;
2788       Parser.Lex();
2789 
2790       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
2791         return MatchOperand_ParseFail;
2792 
2793       for (int i = 0; i < 3; ++i) {
2794         if (getLexer().isNot(AsmToken::Comma))
2795           return MatchOperand_ParseFail;
2796         Parser.Lex();
2797 
2798         int64_t Temp;
2799         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
2800           return MatchOperand_ParseFail;
2801         const int shift = i*2 + 2;
2802         Int += (Temp << shift);
2803       }
2804 
2805       if (getLexer().isNot(AsmToken::RBrac))
2806         return MatchOperand_ParseFail;
2807       Parser.Lex();
2808 
2809     } else {
2810       // sel:%d
2811       Parser.Lex();
2812       if (getParser().parseAbsoluteExpression(Int))
2813         return MatchOperand_ParseFail;
2814 
2815       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
2816         Int |= 0x100;
2817       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
2818         Int |= 0x110;
2819       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
2820         Int |= 0x120;
2821       } else if (Prefix == "wave_shl" && 1 == Int) {
2822         Int = 0x130;
2823       } else if (Prefix == "wave_rol" && 1 == Int) {
2824         Int = 0x134;
2825       } else if (Prefix == "wave_shr" && 1 == Int) {
2826         Int = 0x138;
2827       } else if (Prefix == "wave_ror" && 1 == Int) {
2828         Int = 0x13C;
2829       } else if (Prefix == "row_bcast") {
2830         if (Int == 15) {
2831           Int = 0x142;
2832         } else if (Int == 31) {
2833           Int = 0x143;
2834         } else {
2835           return MatchOperand_ParseFail;
2836         }
2837       } else {
2838         return MatchOperand_ParseFail;
2839       }
2840     }
2841   }
2842 
2843   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
2844   return MatchOperand_Success;
2845 }
2846 
2847 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
2848   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
2849 }
2850 
2851 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
2852   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
2853 }
2854 
2855 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
2856   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
2857 }
2858 
2859 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
2860   OptionalImmIndexMap OptionalIdx;
2861 
2862   unsigned I = 1;
2863   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2864   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2865     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2866   }
2867 
2868   for (unsigned E = Operands.size(); I != E; ++I) {
2869     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2870     // Add the register arguments
2871     if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
2872       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
2873     } else if (Op.isDPPCtrl()) {
2874       Op.addImmOperands(Inst, 1);
2875     } else if (Op.isImm()) {
2876       // Handle optional arguments
2877       OptionalIdx[Op.getImmTy()] = I;
2878     } else {
2879       llvm_unreachable("Invalid operand type");
2880     }
2881   }
2882 
2883   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
2884   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
2885   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
2886 
2887   // special case v_mac_f32:
2888   // it has src2 register operand that is tied to dst operand
2889   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_dpp) {
2890     auto it = Inst.begin();
2891     std::advance(it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
2892     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
2893   }
2894 }
2895 
2896 //===----------------------------------------------------------------------===//
2897 // sdwa
2898 //===----------------------------------------------------------------------===//
2899 
2900 AMDGPUAsmParser::OperandMatchResultTy
2901 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
2902                               AMDGPUOperand::ImmTy Type) {
2903   using namespace llvm::AMDGPU::SDWA;
2904 
2905   SMLoc S = Parser.getTok().getLoc();
2906   StringRef Value;
2907   AMDGPUAsmParser::OperandMatchResultTy res;
2908 
2909   res = parseStringWithPrefix(Prefix, Value);
2910   if (res != MatchOperand_Success) {
2911     return res;
2912   }
2913 
2914   int64_t Int;
2915   Int = StringSwitch<int64_t>(Value)
2916         .Case("BYTE_0", SdwaSel::BYTE_0)
2917         .Case("BYTE_1", SdwaSel::BYTE_1)
2918         .Case("BYTE_2", SdwaSel::BYTE_2)
2919         .Case("BYTE_3", SdwaSel::BYTE_3)
2920         .Case("WORD_0", SdwaSel::WORD_0)
2921         .Case("WORD_1", SdwaSel::WORD_1)
2922         .Case("DWORD", SdwaSel::DWORD)
2923         .Default(0xffffffff);
2924   Parser.Lex(); // eat last token
2925 
2926   if (Int == 0xffffffff) {
2927     return MatchOperand_ParseFail;
2928   }
2929 
2930   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
2931   return MatchOperand_Success;
2932 }
2933 
2934 AMDGPUAsmParser::OperandMatchResultTy
2935 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
2936   using namespace llvm::AMDGPU::SDWA;
2937 
2938   SMLoc S = Parser.getTok().getLoc();
2939   StringRef Value;
2940   AMDGPUAsmParser::OperandMatchResultTy res;
2941 
2942   res = parseStringWithPrefix("dst_unused", Value);
2943   if (res != MatchOperand_Success) {
2944     return res;
2945   }
2946 
2947   int64_t Int;
2948   Int = StringSwitch<int64_t>(Value)
2949         .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
2950         .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
2951         .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
2952         .Default(0xffffffff);
2953   Parser.Lex(); // eat last token
2954 
2955   if (Int == 0xffffffff) {
2956     return MatchOperand_ParseFail;
2957   }
2958 
2959   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
2960   return MatchOperand_Success;
2961 }
2962 
2963 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
2964   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
2965 }
2966 
2967 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
2968   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
2969 }
2970 
2971 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
2972   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC);
2973 }
2974 
2975 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
2976                               uint64_t BasicInstType) {
2977   OptionalImmIndexMap OptionalIdx;
2978 
2979   unsigned I = 1;
2980   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2981   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2982     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2983   }
2984 
2985   for (unsigned E = Operands.size(); I != E; ++I) {
2986     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2987     // Add the register arguments
2988     if (BasicInstType == SIInstrFlags::VOPC &&
2989         Op.isReg() &&
2990         Op.Reg.RegNo == AMDGPU::VCC) {
2991       // VOPC sdwa use "vcc" token as dst. Skip it.
2992       continue;
2993     } else if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
2994       Op.addRegOrImmWithInputModsOperands(Inst, 2);
2995     } else if (Op.isImm()) {
2996       // Handle optional arguments
2997       OptionalIdx[Op.getImmTy()] = I;
2998     } else {
2999       llvm_unreachable("Invalid operand type");
3000     }
3001   }
3002 
3003   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
3004 
3005   if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa) {
3006     // V_NOP_sdwa has no optional sdwa arguments
3007     switch (BasicInstType) {
3008     case SIInstrFlags::VOP1: {
3009       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
3010       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
3011       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3012       break;
3013     }
3014     case SIInstrFlags::VOP2: {
3015       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
3016       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
3017       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3018       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, 6);
3019       break;
3020     }
3021     case SIInstrFlags::VOPC: {
3022       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
3023       addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, 6);
3024       break;
3025     }
3026     default:
3027       llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
3028     }
3029   }
3030 
3031   // special case v_mac_f32:
3032   // it has src2 register operand that is tied to dst operand
3033   if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa) {
3034     auto it = Inst.begin();
3035     std::advance(it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
3036     Inst.insert(it, Inst.getOperand(0)); // src2 = dst
3037   }
3038 
3039 }
3040 
3041 /// Force static initialization.
3042 extern "C" void LLVMInitializeAMDGPUAsmParser() {
3043   RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
3044   RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
3045 }
3046 
3047 #define GET_REGISTER_MATCHER
3048 #define GET_MATCHER_IMPLEMENTATION
3049 #include "AMDGPUGenAsmMatcher.inc"
3050 
3051 
3052 // This fuction should be defined after auto-generated include so that we have
3053 // MatchClassKind enum defined
3054 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
3055                                                      unsigned Kind) {
3056   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
3057   // But MatchInstructionImpl() expects to meet token and fails to validate
3058   // operand. This method checks if we are given immediate operand but expect to
3059   // get corresponding token.
3060   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
3061   switch (Kind) {
3062   case MCK_addr64:
3063     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
3064   case MCK_gds:
3065     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
3066   case MCK_glc:
3067     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
3068   case MCK_idxen:
3069     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
3070   case MCK_offen:
3071     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
3072   case MCK_SSrcB32:
3073     // When operands have expression values, they will return true for isToken,
3074     // because it is not possible to distinguish between a token and an
3075     // expression at parse time. MatchInstructionImpl() will always try to
3076     // match an operand as a token, when isToken returns true, and when the
3077     // name of the expression is not a valid token, the match will fail,
3078     // so we need to handle it here.
3079     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
3080   case MCK_SSrcF32:
3081     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
3082   case MCK_SoppBrTarget:
3083     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
3084   default: return Match_InvalidOperand;
3085   }
3086 }
3087