1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/STLExtras.h"
19 #include "llvm/ADT/SmallBitVector.h"
20 #include "llvm/ADT/SmallString.h"
21 #include "llvm/ADT/StringSwitch.h"
22 #include "llvm/ADT/Twine.h"
23 #include "llvm/MC/MCContext.h"
24 #include "llvm/MC/MCExpr.h"
25 #include "llvm/MC/MCInst.h"
26 #include "llvm/MC/MCInstrInfo.h"
27 #include "llvm/MC/MCParser/MCAsmLexer.h"
28 #include "llvm/MC/MCParser/MCAsmParser.h"
29 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
30 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
31 #include "llvm/MC/MCRegisterInfo.h"
32 #include "llvm/MC/MCStreamer.h"
33 #include "llvm/MC/MCSubtargetInfo.h"
34 #include "llvm/MC/MCSymbolELF.h"
35 #include "llvm/Support/Debug.h"
36 #include "llvm/Support/ELF.h"
37 #include "llvm/Support/SourceMgr.h"
38 #include "llvm/Support/TargetRegistry.h"
39 #include "llvm/Support/raw_ostream.h"
40 #include "llvm/Support/MathExtras.h"
41 
42 using namespace llvm;
43 
44 namespace {
45 
46 struct OptionalOperand;
47 
48 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
49 
50 class AMDGPUOperand : public MCParsedAsmOperand {
51   enum KindTy {
52     Token,
53     Immediate,
54     Register,
55     Expression
56   } Kind;
57 
58   SMLoc StartLoc, EndLoc;
59 
60 public:
61   AMDGPUOperand(enum KindTy K) : MCParsedAsmOperand(), Kind(K) {}
62 
63   typedef std::unique_ptr<AMDGPUOperand> Ptr;
64 
65   struct Modifiers {
66     bool Abs;
67     bool Neg;
68     bool Sext;
69 
70     bool hasFPModifiers() const { return Abs || Neg; }
71     bool hasIntModifiers() const { return Sext; }
72     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
73 
74     int64_t getFPModifiersOperand() const {
75       int64_t Operand = 0;
76       Operand |= Abs ? SISrcMods::ABS : 0;
77       Operand |= Neg ? SISrcMods::NEG : 0;
78       return Operand;
79     }
80 
81     int64_t getIntModifiersOperand() const {
82       int64_t Operand = 0;
83       Operand |= Sext ? SISrcMods::SEXT : 0;
84       return Operand;
85     }
86 
87     int64_t getModifiersOperand() const {
88       assert(!(hasFPModifiers() && hasIntModifiers())
89            && "fp and int modifiers should not be used simultaneously");
90       if (hasFPModifiers()) {
91         return getFPModifiersOperand();
92       } else if (hasIntModifiers()) {
93         return getIntModifiersOperand();
94       } else {
95         return 0;
96       }
97     }
98 
99     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
100   };
101 
102   enum ImmTy {
103     ImmTyNone,
104     ImmTyGDS,
105     ImmTyOffen,
106     ImmTyIdxen,
107     ImmTyAddr64,
108     ImmTyOffset,
109     ImmTyOffset0,
110     ImmTyOffset1,
111     ImmTyGLC,
112     ImmTySLC,
113     ImmTyTFE,
114     ImmTyClampSI,
115     ImmTyOModSI,
116     ImmTyDppCtrl,
117     ImmTyDppRowMask,
118     ImmTyDppBankMask,
119     ImmTyDppBoundCtrl,
120     ImmTySdwaDstSel,
121     ImmTySdwaSrc0Sel,
122     ImmTySdwaSrc1Sel,
123     ImmTySdwaDstUnused,
124     ImmTyDMask,
125     ImmTyUNorm,
126     ImmTyDA,
127     ImmTyR128,
128     ImmTyLWE,
129     ImmTyHwreg,
130     ImmTySendMsg,
131   };
132 
133   struct TokOp {
134     const char *Data;
135     unsigned Length;
136   };
137 
138   struct ImmOp {
139     bool IsFPImm;
140     ImmTy Type;
141     int64_t Val;
142     Modifiers Mods;
143   };
144 
145   struct RegOp {
146     unsigned RegNo;
147     Modifiers Mods;
148     const MCRegisterInfo *TRI;
149     const MCSubtargetInfo *STI;
150     bool IsForcedVOP3;
151   };
152 
153   union {
154     TokOp Tok;
155     ImmOp Imm;
156     RegOp Reg;
157     const MCExpr *Expr;
158   };
159 
160   bool isToken() const override {
161     if (Kind == Token)
162       return true;
163 
164     if (Kind != Expression || !Expr)
165       return false;
166 
167     // When parsing operands, we can't always tell if something was meant to be
168     // a token, like 'gds', or an expression that references a global variable.
169     // In this case, we assume the string is an expression, and if we need to
170     // interpret is a token, then we treat the symbol name as the token.
171     return isa<MCSymbolRefExpr>(Expr);
172   }
173 
174   bool isImm() const override {
175     return Kind == Immediate;
176   }
177 
178   bool isInlinableImm() const {
179     if (!isImmTy(ImmTyNone)) {
180       // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
181       return false;
182     }
183     // TODO: We should avoid using host float here. It would be better to
184     // check the float bit values which is what a few other places do.
185     // We've had bot failures before due to weird NaN support on mips hosts.
186     const float F = BitsToFloat(Imm.Val);
187     // TODO: Add 1/(2*pi) for VI
188     return (Imm.Val <= 64 && Imm.Val >= -16) ||
189            (F == 0.0 || F == 0.5 || F == -0.5 || F == 1.0 || F == -1.0 ||
190            F == 2.0 || F == -2.0 || F == 4.0 || F == -4.0);
191   }
192 
193   bool isRegKind() const {
194     return Kind == Register;
195   }
196 
197   bool isReg() const override {
198     return isRegKind() && !Reg.Mods.hasModifiers();
199   }
200 
201   bool isRegOrImmWithInputMods() const {
202     return isRegKind() || isInlinableImm();
203   }
204 
205   bool isImmTy(ImmTy ImmT) const {
206     return isImm() && Imm.Type == ImmT;
207   }
208 
209   bool isImmModifier() const {
210     return isImm() && Imm.Type != ImmTyNone;
211   }
212 
213   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
214   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
215   bool isDMask() const { return isImmTy(ImmTyDMask); }
216   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
217   bool isDA() const { return isImmTy(ImmTyDA); }
218   bool isR128() const { return isImmTy(ImmTyUNorm); }
219   bool isLWE() const { return isImmTy(ImmTyLWE); }
220   bool isOffen() const { return isImmTy(ImmTyOffen); }
221   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
222   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
223   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
224   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
225   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
226   bool isGDS() const { return isImmTy(ImmTyGDS); }
227   bool isGLC() const { return isImmTy(ImmTyGLC); }
228   bool isSLC() const { return isImmTy(ImmTySLC); }
229   bool isTFE() const { return isImmTy(ImmTyTFE); }
230   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
231   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
232   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
233   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
234   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
235   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
236   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
237 
238   bool isMod() const {
239     return isClampSI() || isOModSI();
240   }
241 
242   bool isRegOrImm() const {
243     return isReg() || isImm();
244   }
245 
246   bool isRegClass(unsigned RCID) const {
247     return isReg() && Reg.TRI->getRegClass(RCID).contains(getReg());
248   }
249 
250   bool isSCSrc32() const {
251     return isInlinableImm() || isRegClass(AMDGPU::SReg_32RegClassID);
252   }
253 
254   bool isSCSrc64() const {
255     return isInlinableImm() || isRegClass(AMDGPU::SReg_64RegClassID);
256   }
257 
258   bool isSSrc32() const {
259     return isImm() || isSCSrc32() || isExpr();
260   }
261 
262   bool isSSrc64() const {
263     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
264     // See isVSrc64().
265     return isImm() || isSCSrc64();
266   }
267 
268   bool isVCSrc32() const {
269     return isInlinableImm() || isRegClass(AMDGPU::VS_32RegClassID);
270   }
271 
272   bool isVCSrc64() const {
273     return isInlinableImm() || isRegClass(AMDGPU::VS_64RegClassID);
274   }
275 
276   bool isVSrc32() const {
277     return isImm() || isVCSrc32();
278   }
279 
280   bool isVSrc64() const {
281     // TODO: Check if the 64-bit value (coming from assembly source) can be
282     // narrowed to 32 bits (in the instruction stream). That require knowledge
283     // of instruction type (unsigned/signed, floating or "untyped"/B64),
284     // see [AMD GCN3 ISA 6.3.1].
285     // TODO: How 64-bit values are formed from 32-bit literals in _B64 insns?
286     return isImm() || isVCSrc64();
287   }
288 
289   bool isMem() const override {
290     return false;
291   }
292 
293   bool isExpr() const {
294     return Kind == Expression;
295   }
296 
297   bool isSoppBrTarget() const {
298     return isExpr() || isImm();
299   }
300 
301   bool isSWaitCnt() const;
302   bool isHwreg() const;
303   bool isSendMsg() const;
304   bool isSMRDOffset() const;
305   bool isSMRDLiteralOffset() const;
306   bool isDPPCtrl() const;
307 
308   StringRef getExpressionAsToken() const {
309     assert(isExpr());
310     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
311     return S->getSymbol().getName();
312   }
313 
314 
315   StringRef getToken() const {
316     assert(isToken());
317 
318     if (Kind == Expression)
319       return getExpressionAsToken();
320 
321     return StringRef(Tok.Data, Tok.Length);
322   }
323 
324   int64_t getImm() const {
325     assert(isImm());
326     return Imm.Val;
327   }
328 
329   enum ImmTy getImmTy() const {
330     assert(isImm());
331     return Imm.Type;
332   }
333 
334   unsigned getReg() const override {
335     return Reg.RegNo;
336   }
337 
338   SMLoc getStartLoc() const override {
339     return StartLoc;
340   }
341 
342   SMLoc getEndLoc() const override {
343     return EndLoc;
344   }
345 
346   Modifiers getModifiers() const {
347     assert(isRegKind() || isImmTy(ImmTyNone));
348     return isRegKind() ? Reg.Mods : Imm.Mods;
349   }
350 
351   void setModifiers(Modifiers Mods) {
352     assert(isRegKind() || isImmTy(ImmTyNone));
353     if (isRegKind())
354       Reg.Mods = Mods;
355     else
356       Imm.Mods = Mods;
357   }
358 
359   bool hasModifiers() const {
360     return getModifiers().hasModifiers();
361   }
362 
363   bool hasFPModifiers() const {
364     return getModifiers().hasFPModifiers();
365   }
366 
367   bool hasIntModifiers() const {
368     return getModifiers().hasIntModifiers();
369   }
370 
371   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const {
372     if (isImmTy(ImmTyNone) && ApplyModifiers && Imm.Mods.hasFPModifiers()) {
373       // Apply modifiers to immediate value
374       int64_t Val = Imm.Val;
375       bool Negate = Imm.Mods.Neg; // Only negate can get here
376       if (Imm.IsFPImm) {
377         APFloat F(BitsToFloat(Val));
378         if (Negate) {
379           F.changeSign();
380         }
381         Val = F.bitcastToAPInt().getZExtValue();
382       } else {
383         Val = Negate ? -Val : Val;
384       }
385       Inst.addOperand(MCOperand::createImm(Val));
386     } else {
387       Inst.addOperand(MCOperand::createImm(getImm()));
388     }
389   }
390 
391   void addRegOperands(MCInst &Inst, unsigned N) const {
392     Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), *Reg.STI)));
393   }
394 
395   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
396     if (isRegKind())
397       addRegOperands(Inst, N);
398     else if (isExpr())
399       Inst.addOperand(MCOperand::createExpr(Expr));
400     else
401       addImmOperands(Inst, N);
402   }
403 
404   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
405     Modifiers Mods = getModifiers();
406     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
407     if (isRegKind()) {
408       addRegOperands(Inst, N);
409     } else {
410       addImmOperands(Inst, N, false);
411     }
412   }
413 
414   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
415     assert(!hasIntModifiers());
416     addRegOrImmWithInputModsOperands(Inst, N);
417   }
418 
419   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
420     assert(!hasFPModifiers());
421     addRegOrImmWithInputModsOperands(Inst, N);
422   }
423 
424   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
425     if (isImm())
426       addImmOperands(Inst, N);
427     else {
428       assert(isExpr());
429       Inst.addOperand(MCOperand::createExpr(Expr));
430     }
431   }
432 
433   void printImmTy(raw_ostream& OS, ImmTy Type) const {
434     switch (Type) {
435     case ImmTyNone: OS << "None"; break;
436     case ImmTyGDS: OS << "GDS"; break;
437     case ImmTyOffen: OS << "Offen"; break;
438     case ImmTyIdxen: OS << "Idxen"; break;
439     case ImmTyAddr64: OS << "Addr64"; break;
440     case ImmTyOffset: OS << "Offset"; break;
441     case ImmTyOffset0: OS << "Offset0"; break;
442     case ImmTyOffset1: OS << "Offset1"; break;
443     case ImmTyGLC: OS << "GLC"; break;
444     case ImmTySLC: OS << "SLC"; break;
445     case ImmTyTFE: OS << "TFE"; break;
446     case ImmTyClampSI: OS << "ClampSI"; break;
447     case ImmTyOModSI: OS << "OModSI"; break;
448     case ImmTyDppCtrl: OS << "DppCtrl"; break;
449     case ImmTyDppRowMask: OS << "DppRowMask"; break;
450     case ImmTyDppBankMask: OS << "DppBankMask"; break;
451     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
452     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
453     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
454     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
455     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
456     case ImmTyDMask: OS << "DMask"; break;
457     case ImmTyUNorm: OS << "UNorm"; break;
458     case ImmTyDA: OS << "DA"; break;
459     case ImmTyR128: OS << "R128"; break;
460     case ImmTyLWE: OS << "LWE"; break;
461     case ImmTyHwreg: OS << "Hwreg"; break;
462     case ImmTySendMsg: OS << "SendMsg"; break;
463     }
464   }
465 
466   void print(raw_ostream &OS) const override {
467     switch (Kind) {
468     case Register:
469       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
470       break;
471     case Immediate:
472       OS << '<' << getImm();
473       if (getImmTy() != ImmTyNone) {
474         OS << " type: "; printImmTy(OS, getImmTy());
475       }
476       OS << " mods: " << Imm.Mods << '>';
477       break;
478     case Token:
479       OS << '\'' << getToken() << '\'';
480       break;
481     case Expression:
482       OS << "<expr " << *Expr << '>';
483       break;
484     }
485   }
486 
487   static AMDGPUOperand::Ptr CreateImm(int64_t Val, SMLoc Loc,
488                                       enum ImmTy Type = ImmTyNone,
489                                       bool IsFPImm = false) {
490     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate);
491     Op->Imm.Val = Val;
492     Op->Imm.IsFPImm = IsFPImm;
493     Op->Imm.Type = Type;
494     Op->Imm.Mods = {false, false, false};
495     Op->StartLoc = Loc;
496     Op->EndLoc = Loc;
497     return Op;
498   }
499 
500   static AMDGPUOperand::Ptr CreateToken(StringRef Str, SMLoc Loc,
501                                         bool HasExplicitEncodingSize = true) {
502     auto Res = llvm::make_unique<AMDGPUOperand>(Token);
503     Res->Tok.Data = Str.data();
504     Res->Tok.Length = Str.size();
505     Res->StartLoc = Loc;
506     Res->EndLoc = Loc;
507     return Res;
508   }
509 
510   static AMDGPUOperand::Ptr CreateReg(unsigned RegNo, SMLoc S,
511                                       SMLoc E,
512                                       const MCRegisterInfo *TRI,
513                                       const MCSubtargetInfo *STI,
514                                       bool ForceVOP3) {
515     auto Op = llvm::make_unique<AMDGPUOperand>(Register);
516     Op->Reg.RegNo = RegNo;
517     Op->Reg.TRI = TRI;
518     Op->Reg.STI = STI;
519     Op->Reg.Mods = {false, false, false};
520     Op->Reg.IsForcedVOP3 = ForceVOP3;
521     Op->StartLoc = S;
522     Op->EndLoc = E;
523     return Op;
524   }
525 
526   static AMDGPUOperand::Ptr CreateExpr(const class MCExpr *Expr, SMLoc S) {
527     auto Op = llvm::make_unique<AMDGPUOperand>(Expression);
528     Op->Expr = Expr;
529     Op->StartLoc = S;
530     Op->EndLoc = S;
531     return Op;
532   }
533 };
534 
535 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
536   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
537   return OS;
538 }
539 
540 class AMDGPUAsmParser : public MCTargetAsmParser {
541   const MCInstrInfo &MII;
542   MCAsmParser &Parser;
543 
544   unsigned ForcedEncodingSize;
545   bool ForcedDPP;
546   bool ForcedSDWA;
547 
548   bool isSI() const {
549     return AMDGPU::isSI(getSTI());
550   }
551 
552   bool isCI() const {
553     return AMDGPU::isCI(getSTI());
554   }
555 
556   bool isVI() const {
557     return AMDGPU::isVI(getSTI());
558   }
559 
560   bool hasSGPR102_SGPR103() const {
561     return !isVI();
562   }
563 
564   /// @name Auto-generated Match Functions
565   /// {
566 
567 #define GET_ASSEMBLER_HEADER
568 #include "AMDGPUGenAsmMatcher.inc"
569 
570   /// }
571 
572 private:
573   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
574   bool ParseDirectiveHSACodeObjectVersion();
575   bool ParseDirectiveHSACodeObjectISA();
576   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
577   bool ParseDirectiveAMDKernelCodeT();
578   bool ParseSectionDirectiveHSAText();
579   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
580   bool ParseDirectiveAMDGPUHsaKernel();
581   bool ParseDirectiveAMDGPUHsaModuleGlobal();
582   bool ParseDirectiveAMDGPUHsaProgramGlobal();
583   bool ParseSectionDirectiveHSADataGlobalAgent();
584   bool ParseSectionDirectiveHSADataGlobalProgram();
585   bool ParseSectionDirectiveHSARodataReadonlyAgent();
586   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum);
587   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth);
588   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands, bool IsAtomic, bool IsAtomicReturn);
589 
590 public:
591   enum AMDGPUMatchResultTy {
592     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
593   };
594 
595   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
596                const MCInstrInfo &MII,
597                const MCTargetOptions &Options)
598       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser),
599         ForcedEncodingSize(0),
600         ForcedDPP(false),
601         ForcedSDWA(false) {
602     MCAsmParserExtension::Initialize(Parser);
603 
604     if (getSTI().getFeatureBits().none()) {
605       // Set default features.
606       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
607     }
608 
609     setAvailableFeatures(ComputeAvailableFeatures(getSTI().getFeatureBits()));
610 
611     {
612       // TODO: make those pre-defined variables read-only.
613       // Currently there is none suitable machinery in the core llvm-mc for this.
614       // MCSymbol::isRedefinable is intended for another purpose, and
615       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
616       AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
617       MCContext &Ctx = getContext();
618       MCSymbol *Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
619       Sym->setVariableValue(MCConstantExpr::create(Isa.Major, Ctx));
620       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
621       Sym->setVariableValue(MCConstantExpr::create(Isa.Minor, Ctx));
622       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
623       Sym->setVariableValue(MCConstantExpr::create(Isa.Stepping, Ctx));
624     }
625   }
626 
627   AMDGPUTargetStreamer &getTargetStreamer() {
628     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
629     return static_cast<AMDGPUTargetStreamer &>(TS);
630   }
631 
632   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
633   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
634   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
635 
636   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
637   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
638   bool isForcedDPP() const { return ForcedDPP; }
639   bool isForcedSDWA() const { return ForcedSDWA; }
640 
641   std::unique_ptr<AMDGPUOperand> parseRegister();
642   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
643   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
644   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
645                                       unsigned Kind) override;
646   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
647                                OperandVector &Operands, MCStreamer &Out,
648                                uint64_t &ErrorInfo,
649                                bool MatchingInlineAsm) override;
650   bool ParseDirective(AsmToken DirectiveID) override;
651   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
652   StringRef parseMnemonicSuffix(StringRef Name);
653   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
654                         SMLoc NameLoc, OperandVector &Operands) override;
655 
656   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
657   OperandMatchResultTy parseIntWithPrefix(const char *Prefix,
658                                           OperandVector &Operands,
659                                           enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
660                                           bool (*ConvertResult)(int64_t&) = 0);
661   OperandMatchResultTy parseNamedBit(const char *Name, OperandVector &Operands,
662                                      enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
663   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix, StringRef &Value);
664 
665   OperandMatchResultTy parseImm(OperandVector &Operands);
666   OperandMatchResultTy parseRegOrImm(OperandVector &Operands);
667   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands);
668   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands);
669 
670   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
671   void cvtDS(MCInst &Inst, const OperandVector &Operands);
672 
673   bool parseCnt(int64_t &IntVal);
674   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
675   OperandMatchResultTy parseHwreg(OperandVector &Operands);
676 
677 private:
678   struct OperandInfoTy {
679     int64_t Id;
680     bool IsSymbolic;
681     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
682   };
683 
684   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
685   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
686 public:
687   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
688 
689   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
690   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
691 
692   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
693   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
694   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
695   AMDGPUOperand::Ptr defaultGLC() const;
696   AMDGPUOperand::Ptr defaultSLC() const;
697   AMDGPUOperand::Ptr defaultTFE() const;
698 
699   AMDGPUOperand::Ptr defaultDMask() const;
700   AMDGPUOperand::Ptr defaultUNorm() const;
701   AMDGPUOperand::Ptr defaultDA() const;
702   AMDGPUOperand::Ptr defaultR128() const;
703   AMDGPUOperand::Ptr defaultLWE() const;
704   AMDGPUOperand::Ptr defaultSMRDOffset() const;
705   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
706 
707   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
708 
709   void cvtId(MCInst &Inst, const OperandVector &Operands);
710   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
711   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
712 
713   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
714   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
715 
716   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
717   AMDGPUOperand::Ptr defaultRowMask() const;
718   AMDGPUOperand::Ptr defaultBankMask() const;
719   AMDGPUOperand::Ptr defaultBoundCtrl() const;
720   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
721 
722   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
723                                     AMDGPUOperand::ImmTy Type);
724   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
725   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
726   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
727   void cvtSDWA(MCInst &Inst, const OperandVector &Operands, bool IsVOP1);
728 };
729 
730 struct OptionalOperand {
731   const char *Name;
732   AMDGPUOperand::ImmTy Type;
733   bool IsBit;
734   bool (*ConvertResult)(int64_t&);
735 };
736 
737 }
738 
739 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
740   if (Is == IS_VGPR) {
741     switch (RegWidth) {
742       default: return -1;
743       case 1: return AMDGPU::VGPR_32RegClassID;
744       case 2: return AMDGPU::VReg_64RegClassID;
745       case 3: return AMDGPU::VReg_96RegClassID;
746       case 4: return AMDGPU::VReg_128RegClassID;
747       case 8: return AMDGPU::VReg_256RegClassID;
748       case 16: return AMDGPU::VReg_512RegClassID;
749     }
750   } else if (Is == IS_TTMP) {
751     switch (RegWidth) {
752       default: return -1;
753       case 1: return AMDGPU::TTMP_32RegClassID;
754       case 2: return AMDGPU::TTMP_64RegClassID;
755       case 4: return AMDGPU::TTMP_128RegClassID;
756     }
757   } else if (Is == IS_SGPR) {
758     switch (RegWidth) {
759       default: return -1;
760       case 1: return AMDGPU::SGPR_32RegClassID;
761       case 2: return AMDGPU::SGPR_64RegClassID;
762       case 4: return AMDGPU::SGPR_128RegClassID;
763       case 8: return AMDGPU::SReg_256RegClassID;
764       case 16: return AMDGPU::SReg_512RegClassID;
765     }
766   }
767   return -1;
768 }
769 
770 static unsigned getSpecialRegForName(StringRef RegName) {
771   return StringSwitch<unsigned>(RegName)
772     .Case("exec", AMDGPU::EXEC)
773     .Case("vcc", AMDGPU::VCC)
774     .Case("flat_scratch", AMDGPU::FLAT_SCR)
775     .Case("m0", AMDGPU::M0)
776     .Case("scc", AMDGPU::SCC)
777     .Case("tba", AMDGPU::TBA)
778     .Case("tma", AMDGPU::TMA)
779     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
780     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
781     .Case("vcc_lo", AMDGPU::VCC_LO)
782     .Case("vcc_hi", AMDGPU::VCC_HI)
783     .Case("exec_lo", AMDGPU::EXEC_LO)
784     .Case("exec_hi", AMDGPU::EXEC_HI)
785     .Case("tma_lo", AMDGPU::TMA_LO)
786     .Case("tma_hi", AMDGPU::TMA_HI)
787     .Case("tba_lo", AMDGPU::TBA_LO)
788     .Case("tba_hi", AMDGPU::TBA_HI)
789     .Default(0);
790 }
791 
792 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) {
793   auto R = parseRegister();
794   if (!R) return true;
795   assert(R->isReg());
796   RegNo = R->getReg();
797   StartLoc = R->getStartLoc();
798   EndLoc = R->getEndLoc();
799   return false;
800 }
801 
802 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum)
803 {
804   switch (RegKind) {
805   case IS_SPECIAL:
806     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) { Reg = AMDGPU::EXEC; RegWidth = 2; return true; }
807     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) { Reg = AMDGPU::FLAT_SCR; RegWidth = 2; return true; }
808     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) { Reg = AMDGPU::VCC; RegWidth = 2; return true; }
809     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) { Reg = AMDGPU::TBA; RegWidth = 2; return true; }
810     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) { Reg = AMDGPU::TMA; RegWidth = 2; return true; }
811     return false;
812   case IS_VGPR:
813   case IS_SGPR:
814   case IS_TTMP:
815     if (Reg1 != Reg + RegWidth) { return false; }
816     RegWidth++;
817     return true;
818   default:
819     assert(false); return false;
820   }
821 }
822 
823 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth)
824 {
825   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
826   if (getLexer().is(AsmToken::Identifier)) {
827     StringRef RegName = Parser.getTok().getString();
828     if ((Reg = getSpecialRegForName(RegName))) {
829       Parser.Lex();
830       RegKind = IS_SPECIAL;
831     } else {
832       unsigned RegNumIndex = 0;
833       if (RegName[0] == 'v') {
834         RegNumIndex = 1;
835         RegKind = IS_VGPR;
836       } else if (RegName[0] == 's') {
837         RegNumIndex = 1;
838         RegKind = IS_SGPR;
839       } else if (RegName.startswith("ttmp")) {
840         RegNumIndex = strlen("ttmp");
841         RegKind = IS_TTMP;
842       } else {
843         return false;
844       }
845       if (RegName.size() > RegNumIndex) {
846         // Single 32-bit register: vXX.
847         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
848           return false;
849         Parser.Lex();
850         RegWidth = 1;
851       } else {
852         // Range of registers: v[XX:YY]. ":YY" is optional.
853         Parser.Lex();
854         int64_t RegLo, RegHi;
855         if (getLexer().isNot(AsmToken::LBrac))
856           return false;
857         Parser.Lex();
858 
859         if (getParser().parseAbsoluteExpression(RegLo))
860           return false;
861 
862         const bool isRBrace = getLexer().is(AsmToken::RBrac);
863         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
864           return false;
865         Parser.Lex();
866 
867         if (isRBrace) {
868           RegHi = RegLo;
869         } else {
870           if (getParser().parseAbsoluteExpression(RegHi))
871             return false;
872 
873           if (getLexer().isNot(AsmToken::RBrac))
874             return false;
875           Parser.Lex();
876         }
877         RegNum = (unsigned) RegLo;
878         RegWidth = (RegHi - RegLo) + 1;
879       }
880     }
881   } else if (getLexer().is(AsmToken::LBrac)) {
882     // List of consecutive registers: [s0,s1,s2,s3]
883     Parser.Lex();
884     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth))
885       return false;
886     if (RegWidth != 1)
887       return false;
888     RegisterKind RegKind1;
889     unsigned Reg1, RegNum1, RegWidth1;
890     do {
891       if (getLexer().is(AsmToken::Comma)) {
892         Parser.Lex();
893       } else if (getLexer().is(AsmToken::RBrac)) {
894         Parser.Lex();
895         break;
896       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1)) {
897         if (RegWidth1 != 1) {
898           return false;
899         }
900         if (RegKind1 != RegKind) {
901           return false;
902         }
903         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
904           return false;
905         }
906       } else {
907         return false;
908       }
909     } while (true);
910   } else {
911     return false;
912   }
913   switch (RegKind) {
914   case IS_SPECIAL:
915     RegNum = 0;
916     RegWidth = 1;
917     break;
918   case IS_VGPR:
919   case IS_SGPR:
920   case IS_TTMP:
921   {
922     unsigned Size = 1;
923     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
924       // SGPR and TTMP registers must be are aligned. Max required alignment is 4 dwords.
925       Size = std::min(RegWidth, 4u);
926     }
927     if (RegNum % Size != 0)
928       return false;
929     RegNum = RegNum / Size;
930     int RCID = getRegClass(RegKind, RegWidth);
931     if (RCID == -1)
932       return false;
933     const MCRegisterClass RC = TRI->getRegClass(RCID);
934     if (RegNum >= RC.getNumRegs())
935       return false;
936     Reg = RC.getRegister(RegNum);
937     break;
938   }
939 
940   default:
941     assert(false); return false;
942   }
943 
944   if (!subtargetHasRegister(*TRI, Reg))
945     return false;
946   return true;
947 }
948 
949 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
950   const auto &Tok = Parser.getTok();
951   SMLoc StartLoc = Tok.getLoc();
952   SMLoc EndLoc = Tok.getEndLoc();
953   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
954 
955   RegisterKind RegKind;
956   unsigned Reg, RegNum, RegWidth;
957 
958   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth)) {
959     return nullptr;
960   }
961   return AMDGPUOperand::CreateReg(Reg, StartLoc, EndLoc,
962                                   TRI, &getSTI(), false);
963 }
964 
965 AMDGPUAsmParser::OperandMatchResultTy
966 AMDGPUAsmParser::parseImm(OperandVector &Operands) {
967   bool Minus = false;
968   if (getLexer().getKind() == AsmToken::Minus) {
969     Minus = true;
970     Parser.Lex();
971   }
972 
973   SMLoc S = Parser.getTok().getLoc();
974   switch(getLexer().getKind()) {
975   case AsmToken::Integer: {
976     int64_t IntVal;
977     if (getParser().parseAbsoluteExpression(IntVal))
978       return MatchOperand_ParseFail;
979     if (!isInt<32>(IntVal) && !isUInt<32>(IntVal)) {
980       Error(S, "invalid immediate: only 32-bit values are legal");
981       return MatchOperand_ParseFail;
982     }
983 
984     if (Minus)
985       IntVal *= -1;
986     Operands.push_back(AMDGPUOperand::CreateImm(IntVal, S));
987     return MatchOperand_Success;
988   }
989   case AsmToken::Real: {
990     // FIXME: We should emit an error if a double precisions floating-point
991     // value is used.  I'm not sure the best way to detect this.
992     int64_t IntVal;
993     if (getParser().parseAbsoluteExpression(IntVal))
994       return MatchOperand_ParseFail;
995 
996     APFloat F((float)BitsToDouble(IntVal));
997     if (Minus)
998       F.changeSign();
999     Operands.push_back(
1000         AMDGPUOperand::CreateImm(F.bitcastToAPInt().getZExtValue(), S,
1001                                  AMDGPUOperand::ImmTyNone, true));
1002     return MatchOperand_Success;
1003   }
1004   default:
1005     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1006   }
1007 }
1008 
1009 AMDGPUAsmParser::OperandMatchResultTy
1010 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands) {
1011   auto res = parseImm(Operands);
1012   if (res != MatchOperand_NoMatch) {
1013     return res;
1014   }
1015 
1016   if (auto R = parseRegister()) {
1017     assert(R->isReg());
1018     R->Reg.IsForcedVOP3 = isForcedVOP3();
1019     Operands.push_back(std::move(R));
1020     return MatchOperand_Success;
1021   }
1022   return MatchOperand_ParseFail;
1023 }
1024 
1025 AMDGPUAsmParser::OperandMatchResultTy
1026 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands) {
1027   // XXX: During parsing we can't determine if minus sign means
1028   // negate-modifier or negative immediate value.
1029   // By default we suppose it is modifier.
1030   bool Negate = false, Abs = false, Abs2 = false;
1031 
1032   if (getLexer().getKind()== AsmToken::Minus) {
1033     Parser.Lex();
1034     Negate = true;
1035   }
1036 
1037   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "abs") {
1038     Parser.Lex();
1039     Abs2 = true;
1040     if (getLexer().isNot(AsmToken::LParen)) {
1041       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1042       return MatchOperand_ParseFail;
1043     }
1044     Parser.Lex();
1045   }
1046 
1047   if (getLexer().getKind() == AsmToken::Pipe) {
1048     if (Abs2) {
1049       Error(Parser.getTok().getLoc(), "expected register or immediate");
1050       return MatchOperand_ParseFail;
1051     }
1052     Parser.Lex();
1053     Abs = true;
1054   }
1055 
1056   auto Res = parseRegOrImm(Operands);
1057   if (Res != MatchOperand_Success) {
1058     return Res;
1059   }
1060 
1061   AMDGPUOperand::Modifiers Mods = {false, false, false};
1062   if (Negate) {
1063     Mods.Neg = true;
1064   }
1065   if (Abs) {
1066     if (getLexer().getKind() != AsmToken::Pipe) {
1067       Error(Parser.getTok().getLoc(), "expected vertical bar");
1068       return MatchOperand_ParseFail;
1069     }
1070     Parser.Lex();
1071     Mods.Abs = true;
1072   }
1073   if (Abs2) {
1074     if (getLexer().isNot(AsmToken::RParen)) {
1075       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1076       return MatchOperand_ParseFail;
1077     }
1078     Parser.Lex();
1079     Mods.Abs = true;
1080   }
1081 
1082   if (Mods.hasFPModifiers()) {
1083     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1084     Op.setModifiers(Mods);
1085   }
1086   return MatchOperand_Success;
1087 }
1088 
1089 AMDGPUAsmParser::OperandMatchResultTy
1090 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands) {
1091   bool Sext = false;
1092 
1093   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "sext") {
1094     Parser.Lex();
1095     Sext = true;
1096     if (getLexer().isNot(AsmToken::LParen)) {
1097       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1098       return MatchOperand_ParseFail;
1099     }
1100     Parser.Lex();
1101   }
1102 
1103   auto Res = parseRegOrImm(Operands);
1104   if (Res != MatchOperand_Success) {
1105     return Res;
1106   }
1107 
1108   AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1109   if (Op.isImm() && Op.Imm.IsFPImm) {
1110     Error(Parser.getTok().getLoc(), "floating point operands not allowed with sext() modifier");
1111     return MatchOperand_ParseFail;
1112   }
1113 
1114   AMDGPUOperand::Modifiers Mods = {false, false, false};
1115   if (Sext) {
1116     if (getLexer().isNot(AsmToken::RParen)) {
1117       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1118       return MatchOperand_ParseFail;
1119     }
1120     Parser.Lex();
1121     Mods.Sext = true;
1122   }
1123 
1124   if (Mods.hasIntModifiers()) {
1125     Op.setModifiers(Mods);
1126   }
1127   return MatchOperand_Success;
1128 }
1129 
1130 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1131 
1132   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1133 
1134   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1135       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1136       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1137       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1138     return Match_InvalidOperand;
1139 
1140   if ((TSFlags & SIInstrFlags::VOP3) &&
1141       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1142       getForcedEncodingSize() != 64)
1143     return Match_PreferE32;
1144 
1145   return Match_Success;
1146 }
1147 
1148 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1149                                               OperandVector &Operands,
1150                                               MCStreamer &Out,
1151                                               uint64_t &ErrorInfo,
1152                                               bool MatchingInlineAsm) {
1153   MCInst Inst;
1154 
1155   switch (MatchInstructionImpl(Operands, Inst, ErrorInfo, MatchingInlineAsm)) {
1156     default: break;
1157     case Match_Success:
1158       Inst.setLoc(IDLoc);
1159       Out.EmitInstruction(Inst, getSTI());
1160       return false;
1161     case Match_MissingFeature:
1162       return Error(IDLoc, "instruction not supported on this GPU");
1163 
1164     case Match_MnemonicFail:
1165       return Error(IDLoc, "unrecognized instruction mnemonic");
1166 
1167     case Match_InvalidOperand: {
1168       SMLoc ErrorLoc = IDLoc;
1169       if (ErrorInfo != ~0ULL) {
1170         if (ErrorInfo >= Operands.size()) {
1171           return Error(IDLoc, "too few operands for instruction");
1172         }
1173         ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
1174         if (ErrorLoc == SMLoc())
1175           ErrorLoc = IDLoc;
1176       }
1177       return Error(ErrorLoc, "invalid operand for instruction");
1178     }
1179     case Match_PreferE32:
1180       return Error(IDLoc, "internal error: instruction without _e64 suffix "
1181                           "should be encoded as e32");
1182   }
1183   llvm_unreachable("Implement any new match types added!");
1184 }
1185 
1186 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
1187                                                uint32_t &Minor) {
1188   if (getLexer().isNot(AsmToken::Integer))
1189     return TokError("invalid major version");
1190 
1191   Major = getLexer().getTok().getIntVal();
1192   Lex();
1193 
1194   if (getLexer().isNot(AsmToken::Comma))
1195     return TokError("minor version number required, comma expected");
1196   Lex();
1197 
1198   if (getLexer().isNot(AsmToken::Integer))
1199     return TokError("invalid minor version");
1200 
1201   Minor = getLexer().getTok().getIntVal();
1202   Lex();
1203 
1204   return false;
1205 }
1206 
1207 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
1208 
1209   uint32_t Major;
1210   uint32_t Minor;
1211 
1212   if (ParseDirectiveMajorMinor(Major, Minor))
1213     return true;
1214 
1215   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
1216   return false;
1217 }
1218 
1219 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
1220 
1221   uint32_t Major;
1222   uint32_t Minor;
1223   uint32_t Stepping;
1224   StringRef VendorName;
1225   StringRef ArchName;
1226 
1227   // If this directive has no arguments, then use the ISA version for the
1228   // targeted GPU.
1229   if (getLexer().is(AsmToken::EndOfStatement)) {
1230     AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
1231     getTargetStreamer().EmitDirectiveHSACodeObjectISA(Isa.Major, Isa.Minor,
1232                                                       Isa.Stepping,
1233                                                       "AMD", "AMDGPU");
1234     return false;
1235   }
1236 
1237 
1238   if (ParseDirectiveMajorMinor(Major, Minor))
1239     return true;
1240 
1241   if (getLexer().isNot(AsmToken::Comma))
1242     return TokError("stepping version number required, comma expected");
1243   Lex();
1244 
1245   if (getLexer().isNot(AsmToken::Integer))
1246     return TokError("invalid stepping version");
1247 
1248   Stepping = getLexer().getTok().getIntVal();
1249   Lex();
1250 
1251   if (getLexer().isNot(AsmToken::Comma))
1252     return TokError("vendor name required, comma expected");
1253   Lex();
1254 
1255   if (getLexer().isNot(AsmToken::String))
1256     return TokError("invalid vendor name");
1257 
1258   VendorName = getLexer().getTok().getStringContents();
1259   Lex();
1260 
1261   if (getLexer().isNot(AsmToken::Comma))
1262     return TokError("arch name required, comma expected");
1263   Lex();
1264 
1265   if (getLexer().isNot(AsmToken::String))
1266     return TokError("invalid arch name");
1267 
1268   ArchName = getLexer().getTok().getStringContents();
1269   Lex();
1270 
1271   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
1272                                                     VendorName, ArchName);
1273   return false;
1274 }
1275 
1276 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
1277                                                amd_kernel_code_t &Header) {
1278   SmallString<40> ErrStr;
1279   raw_svector_ostream Err(ErrStr);
1280   if (!parseAmdKernelCodeField(ID, getLexer(), Header, Err)) {
1281     return TokError(Err.str());
1282   }
1283   Lex();
1284   return false;
1285 }
1286 
1287 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
1288 
1289   amd_kernel_code_t Header;
1290   AMDGPU::initDefaultAMDKernelCodeT(Header, getSTI().getFeatureBits());
1291 
1292   while (true) {
1293 
1294     if (getLexer().isNot(AsmToken::EndOfStatement))
1295       return TokError("amd_kernel_code_t values must begin on a new line");
1296 
1297     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
1298     // will set the current token to EndOfStatement.
1299     while(getLexer().is(AsmToken::EndOfStatement))
1300       Lex();
1301 
1302     if (getLexer().isNot(AsmToken::Identifier))
1303       return TokError("expected value identifier or .end_amd_kernel_code_t");
1304 
1305     StringRef ID = getLexer().getTok().getIdentifier();
1306     Lex();
1307 
1308     if (ID == ".end_amd_kernel_code_t")
1309       break;
1310 
1311     if (ParseAMDKernelCodeTValue(ID, Header))
1312       return true;
1313   }
1314 
1315   getTargetStreamer().EmitAMDKernelCodeT(Header);
1316 
1317   return false;
1318 }
1319 
1320 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
1321   getParser().getStreamer().SwitchSection(
1322       AMDGPU::getHSATextSection(getContext()));
1323   return false;
1324 }
1325 
1326 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
1327   if (getLexer().isNot(AsmToken::Identifier))
1328     return TokError("expected symbol name");
1329 
1330   StringRef KernelName = Parser.getTok().getString();
1331 
1332   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
1333                                            ELF::STT_AMDGPU_HSA_KERNEL);
1334   Lex();
1335   return false;
1336 }
1337 
1338 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
1339   if (getLexer().isNot(AsmToken::Identifier))
1340     return TokError("expected symbol name");
1341 
1342   StringRef GlobalName = Parser.getTok().getIdentifier();
1343 
1344   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
1345   Lex();
1346   return false;
1347 }
1348 
1349 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
1350   if (getLexer().isNot(AsmToken::Identifier))
1351     return TokError("expected symbol name");
1352 
1353   StringRef GlobalName = Parser.getTok().getIdentifier();
1354 
1355   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
1356   Lex();
1357   return false;
1358 }
1359 
1360 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
1361   getParser().getStreamer().SwitchSection(
1362       AMDGPU::getHSADataGlobalAgentSection(getContext()));
1363   return false;
1364 }
1365 
1366 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
1367   getParser().getStreamer().SwitchSection(
1368       AMDGPU::getHSADataGlobalProgramSection(getContext()));
1369   return false;
1370 }
1371 
1372 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
1373   getParser().getStreamer().SwitchSection(
1374       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
1375   return false;
1376 }
1377 
1378 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
1379   StringRef IDVal = DirectiveID.getString();
1380 
1381   if (IDVal == ".hsa_code_object_version")
1382     return ParseDirectiveHSACodeObjectVersion();
1383 
1384   if (IDVal == ".hsa_code_object_isa")
1385     return ParseDirectiveHSACodeObjectISA();
1386 
1387   if (IDVal == ".amd_kernel_code_t")
1388     return ParseDirectiveAMDKernelCodeT();
1389 
1390   if (IDVal == ".hsatext")
1391     return ParseSectionDirectiveHSAText();
1392 
1393   if (IDVal == ".amdgpu_hsa_kernel")
1394     return ParseDirectiveAMDGPUHsaKernel();
1395 
1396   if (IDVal == ".amdgpu_hsa_module_global")
1397     return ParseDirectiveAMDGPUHsaModuleGlobal();
1398 
1399   if (IDVal == ".amdgpu_hsa_program_global")
1400     return ParseDirectiveAMDGPUHsaProgramGlobal();
1401 
1402   if (IDVal == ".hsadata_global_agent")
1403     return ParseSectionDirectiveHSADataGlobalAgent();
1404 
1405   if (IDVal == ".hsadata_global_program")
1406     return ParseSectionDirectiveHSADataGlobalProgram();
1407 
1408   if (IDVal == ".hsarodata_readonly_agent")
1409     return ParseSectionDirectiveHSARodataReadonlyAgent();
1410 
1411   return true;
1412 }
1413 
1414 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
1415                                            unsigned RegNo) const {
1416   if (isCI())
1417     return true;
1418 
1419   if (isSI()) {
1420     // No flat_scr
1421     switch (RegNo) {
1422     case AMDGPU::FLAT_SCR:
1423     case AMDGPU::FLAT_SCR_LO:
1424     case AMDGPU::FLAT_SCR_HI:
1425       return false;
1426     default:
1427       return true;
1428     }
1429   }
1430 
1431   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
1432   // SI/CI have.
1433   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
1434        R.isValid(); ++R) {
1435     if (*R == RegNo)
1436       return false;
1437   }
1438 
1439   return true;
1440 }
1441 
1442 AMDGPUAsmParser::OperandMatchResultTy
1443 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
1444 
1445   // Try to parse with a custom parser
1446   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
1447 
1448   // If we successfully parsed the operand or if there as an error parsing,
1449   // we are done.
1450   //
1451   // If we are parsing after we reach EndOfStatement then this means we
1452   // are appending default values to the Operands list.  This is only done
1453   // by custom parser, so we shouldn't continue on to the generic parsing.
1454   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
1455       getLexer().is(AsmToken::EndOfStatement))
1456     return ResTy;
1457 
1458   ResTy = parseRegOrImm(Operands);
1459 
1460   if (ResTy == MatchOperand_Success)
1461     return ResTy;
1462 
1463   if (getLexer().getKind() == AsmToken::Identifier) {
1464     // If this identifier is a symbol, we want to create an expression for it.
1465     // It is a little difficult to distinguish between a symbol name, and
1466     // an instruction flag like 'gds'.  In order to do this, we parse
1467     // all tokens as expressions and then treate the symbol name as the token
1468     // string when we want to interpret the operand as a token.
1469     const auto &Tok = Parser.getTok();
1470     SMLoc S = Tok.getLoc();
1471     const MCExpr *Expr = nullptr;
1472     if (!Parser.parseExpression(Expr)) {
1473       Operands.push_back(AMDGPUOperand::CreateExpr(Expr, S));
1474       return MatchOperand_Success;
1475     }
1476 
1477     Operands.push_back(AMDGPUOperand::CreateToken(Tok.getString(), Tok.getLoc()));
1478     Parser.Lex();
1479     return MatchOperand_Success;
1480   }
1481   return MatchOperand_NoMatch;
1482 }
1483 
1484 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
1485   // Clear any forced encodings from the previous instruction.
1486   setForcedEncodingSize(0);
1487   setForcedDPP(false);
1488   setForcedSDWA(false);
1489 
1490   if (Name.endswith("_e64")) {
1491     setForcedEncodingSize(64);
1492     return Name.substr(0, Name.size() - 4);
1493   } else if (Name.endswith("_e32")) {
1494     setForcedEncodingSize(32);
1495     return Name.substr(0, Name.size() - 4);
1496   } else if (Name.endswith("_dpp")) {
1497     setForcedDPP(true);
1498     return Name.substr(0, Name.size() - 4);
1499   } else if (Name.endswith("_sdwa")) {
1500     setForcedSDWA(true);
1501     return Name.substr(0, Name.size() - 5);
1502   }
1503   return Name;
1504 }
1505 
1506 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
1507                                        StringRef Name,
1508                                        SMLoc NameLoc, OperandVector &Operands) {
1509   // Add the instruction mnemonic
1510   Name = parseMnemonicSuffix(Name);
1511   Operands.push_back(AMDGPUOperand::CreateToken(Name, NameLoc));
1512 
1513   while (!getLexer().is(AsmToken::EndOfStatement)) {
1514     AMDGPUAsmParser::OperandMatchResultTy Res = parseOperand(Operands, Name);
1515 
1516     // Eat the comma or space if there is one.
1517     if (getLexer().is(AsmToken::Comma))
1518       Parser.Lex();
1519 
1520     switch (Res) {
1521       case MatchOperand_Success: break;
1522       case MatchOperand_ParseFail:
1523         Error(getLexer().getLoc(), "failed parsing operand.");
1524         while (!getLexer().is(AsmToken::EndOfStatement)) {
1525           Parser.Lex();
1526         }
1527         return true;
1528       case MatchOperand_NoMatch:
1529         Error(getLexer().getLoc(), "not a valid operand.");
1530         while (!getLexer().is(AsmToken::EndOfStatement)) {
1531           Parser.Lex();
1532         }
1533         return true;
1534     }
1535   }
1536 
1537   return false;
1538 }
1539 
1540 //===----------------------------------------------------------------------===//
1541 // Utility functions
1542 //===----------------------------------------------------------------------===//
1543 
1544 AMDGPUAsmParser::OperandMatchResultTy
1545 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
1546   switch(getLexer().getKind()) {
1547     default: return MatchOperand_NoMatch;
1548     case AsmToken::Identifier: {
1549       StringRef Name = Parser.getTok().getString();
1550       if (!Name.equals(Prefix)) {
1551         return MatchOperand_NoMatch;
1552       }
1553 
1554       Parser.Lex();
1555       if (getLexer().isNot(AsmToken::Colon))
1556         return MatchOperand_ParseFail;
1557 
1558       Parser.Lex();
1559       if (getLexer().isNot(AsmToken::Integer))
1560         return MatchOperand_ParseFail;
1561 
1562       if (getParser().parseAbsoluteExpression(Int))
1563         return MatchOperand_ParseFail;
1564       break;
1565     }
1566   }
1567   return MatchOperand_Success;
1568 }
1569 
1570 AMDGPUAsmParser::OperandMatchResultTy
1571 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
1572                                     enum AMDGPUOperand::ImmTy ImmTy,
1573                                     bool (*ConvertResult)(int64_t&)) {
1574 
1575   SMLoc S = Parser.getTok().getLoc();
1576   int64_t Value = 0;
1577 
1578   AMDGPUAsmParser::OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
1579   if (Res != MatchOperand_Success)
1580     return Res;
1581 
1582   if (ConvertResult && !ConvertResult(Value)) {
1583     return MatchOperand_ParseFail;
1584   }
1585 
1586   Operands.push_back(AMDGPUOperand::CreateImm(Value, S, ImmTy));
1587   return MatchOperand_Success;
1588 }
1589 
1590 AMDGPUAsmParser::OperandMatchResultTy
1591 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
1592                                enum AMDGPUOperand::ImmTy ImmTy) {
1593   int64_t Bit = 0;
1594   SMLoc S = Parser.getTok().getLoc();
1595 
1596   // We are at the end of the statement, and this is a default argument, so
1597   // use a default value.
1598   if (getLexer().isNot(AsmToken::EndOfStatement)) {
1599     switch(getLexer().getKind()) {
1600       case AsmToken::Identifier: {
1601         StringRef Tok = Parser.getTok().getString();
1602         if (Tok == Name) {
1603           Bit = 1;
1604           Parser.Lex();
1605         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
1606           Bit = 0;
1607           Parser.Lex();
1608         } else {
1609           return MatchOperand_NoMatch;
1610         }
1611         break;
1612       }
1613       default:
1614         return MatchOperand_NoMatch;
1615     }
1616   }
1617 
1618   Operands.push_back(AMDGPUOperand::CreateImm(Bit, S, ImmTy));
1619   return MatchOperand_Success;
1620 }
1621 
1622 typedef std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
1623 
1624 void addOptionalImmOperand(MCInst& Inst, const OperandVector& Operands,
1625                            OptionalImmIndexMap& OptionalIdx,
1626                            enum AMDGPUOperand::ImmTy ImmT, int64_t Default = 0) {
1627   auto i = OptionalIdx.find(ImmT);
1628   if (i != OptionalIdx.end()) {
1629     unsigned Idx = i->second;
1630     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
1631   } else {
1632     Inst.addOperand(MCOperand::createImm(Default));
1633   }
1634 }
1635 
1636 AMDGPUAsmParser::OperandMatchResultTy
1637 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
1638   if (getLexer().isNot(AsmToken::Identifier)) {
1639     return MatchOperand_NoMatch;
1640   }
1641   StringRef Tok = Parser.getTok().getString();
1642   if (Tok != Prefix) {
1643     return MatchOperand_NoMatch;
1644   }
1645 
1646   Parser.Lex();
1647   if (getLexer().isNot(AsmToken::Colon)) {
1648     return MatchOperand_ParseFail;
1649   }
1650 
1651   Parser.Lex();
1652   if (getLexer().isNot(AsmToken::Identifier)) {
1653     return MatchOperand_ParseFail;
1654   }
1655 
1656   Value = Parser.getTok().getString();
1657   return MatchOperand_Success;
1658 }
1659 
1660 //===----------------------------------------------------------------------===//
1661 // ds
1662 //===----------------------------------------------------------------------===//
1663 
1664 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
1665                                     const OperandVector &Operands) {
1666 
1667   OptionalImmIndexMap OptionalIdx;
1668 
1669   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1670     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1671 
1672     // Add the register arguments
1673     if (Op.isReg()) {
1674       Op.addRegOperands(Inst, 1);
1675       continue;
1676     }
1677 
1678     // Handle optional arguments
1679     OptionalIdx[Op.getImmTy()] = i;
1680   }
1681 
1682   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
1683   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
1684   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1685 
1686   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1687 }
1688 
1689 void AMDGPUAsmParser::cvtDS(MCInst &Inst, const OperandVector &Operands) {
1690 
1691   std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
1692   bool GDSOnly = false;
1693 
1694   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1695     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1696 
1697     // Add the register arguments
1698     if (Op.isReg()) {
1699       Op.addRegOperands(Inst, 1);
1700       continue;
1701     }
1702 
1703     if (Op.isToken() && Op.getToken() == "gds") {
1704       GDSOnly = true;
1705       continue;
1706     }
1707 
1708     // Handle optional arguments
1709     OptionalIdx[Op.getImmTy()] = i;
1710   }
1711 
1712   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
1713   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1714 
1715   if (!GDSOnly) {
1716     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1717   }
1718   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1719 }
1720 
1721 
1722 //===----------------------------------------------------------------------===//
1723 // s_waitcnt
1724 //===----------------------------------------------------------------------===//
1725 
1726 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
1727   StringRef CntName = Parser.getTok().getString();
1728   int64_t CntVal;
1729 
1730   Parser.Lex();
1731   if (getLexer().isNot(AsmToken::LParen))
1732     return true;
1733 
1734   Parser.Lex();
1735   if (getLexer().isNot(AsmToken::Integer))
1736     return true;
1737 
1738   if (getParser().parseAbsoluteExpression(CntVal))
1739     return true;
1740 
1741   if (getLexer().isNot(AsmToken::RParen))
1742     return true;
1743 
1744   Parser.Lex();
1745   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
1746     Parser.Lex();
1747 
1748   int CntShift;
1749   int CntMask;
1750 
1751   if (CntName == "vmcnt") {
1752     CntMask = 0xf;
1753     CntShift = 0;
1754   } else if (CntName == "expcnt") {
1755     CntMask = 0x7;
1756     CntShift = 4;
1757   } else if (CntName == "lgkmcnt") {
1758     CntMask = 0xf;
1759     CntShift = 8;
1760   } else {
1761     return true;
1762   }
1763 
1764   IntVal &= ~(CntMask << CntShift);
1765   IntVal |= (CntVal << CntShift);
1766   return false;
1767 }
1768 
1769 AMDGPUAsmParser::OperandMatchResultTy
1770 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
1771   // Disable all counters by default.
1772   // vmcnt   [3:0]
1773   // expcnt  [6:4]
1774   // lgkmcnt [11:8]
1775   int64_t CntVal = 0xf7f;
1776   SMLoc S = Parser.getTok().getLoc();
1777 
1778   switch(getLexer().getKind()) {
1779     default: return MatchOperand_ParseFail;
1780     case AsmToken::Integer:
1781       // The operand can be an integer value.
1782       if (getParser().parseAbsoluteExpression(CntVal))
1783         return MatchOperand_ParseFail;
1784       break;
1785 
1786     case AsmToken::Identifier:
1787       do {
1788         if (parseCnt(CntVal))
1789           return MatchOperand_ParseFail;
1790       } while(getLexer().isNot(AsmToken::EndOfStatement));
1791       break;
1792   }
1793   Operands.push_back(AMDGPUOperand::CreateImm(CntVal, S));
1794   return MatchOperand_Success;
1795 }
1796 
1797 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width) {
1798   using namespace llvm::AMDGPU::Hwreg;
1799 
1800   if (Parser.getTok().getString() != "hwreg")
1801     return true;
1802   Parser.Lex();
1803 
1804   if (getLexer().isNot(AsmToken::LParen))
1805     return true;
1806   Parser.Lex();
1807 
1808   if (getLexer().is(AsmToken::Identifier)) {
1809     HwReg.IsSymbolic = true;
1810     HwReg.Id = ID_UNKNOWN_;
1811     const StringRef tok = Parser.getTok().getString();
1812     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
1813       if (tok == IdSymbolic[i]) {
1814         HwReg.Id = i;
1815         break;
1816       }
1817     }
1818     Parser.Lex();
1819   } else {
1820     HwReg.IsSymbolic = false;
1821     if (getLexer().isNot(AsmToken::Integer))
1822       return true;
1823     if (getParser().parseAbsoluteExpression(HwReg.Id))
1824       return true;
1825   }
1826 
1827   if (getLexer().is(AsmToken::RParen)) {
1828     Parser.Lex();
1829     return false;
1830   }
1831 
1832   // optional params
1833   if (getLexer().isNot(AsmToken::Comma))
1834     return true;
1835   Parser.Lex();
1836 
1837   if (getLexer().isNot(AsmToken::Integer))
1838     return true;
1839   if (getParser().parseAbsoluteExpression(Offset))
1840     return true;
1841 
1842   if (getLexer().isNot(AsmToken::Comma))
1843     return true;
1844   Parser.Lex();
1845 
1846   if (getLexer().isNot(AsmToken::Integer))
1847     return true;
1848   if (getParser().parseAbsoluteExpression(Width))
1849     return true;
1850 
1851   if (getLexer().isNot(AsmToken::RParen))
1852     return true;
1853   Parser.Lex();
1854 
1855   return false;
1856 }
1857 
1858 AMDGPUAsmParser::OperandMatchResultTy
1859 AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
1860   using namespace llvm::AMDGPU::Hwreg;
1861 
1862   int64_t Imm16Val = 0;
1863   SMLoc S = Parser.getTok().getLoc();
1864 
1865   switch(getLexer().getKind()) {
1866     default: return MatchOperand_NoMatch;
1867     case AsmToken::Integer:
1868       // The operand can be an integer value.
1869       if (getParser().parseAbsoluteExpression(Imm16Val))
1870         return MatchOperand_NoMatch;
1871       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
1872         Error(S, "invalid immediate: only 16-bit values are legal");
1873         // Do not return error code, but create an imm operand anyway and proceed
1874         // to the next operand, if any. That avoids unneccessary error messages.
1875       }
1876       break;
1877 
1878     case AsmToken::Identifier: {
1879         OperandInfoTy HwReg(ID_UNKNOWN_);
1880         int64_t Offset = OFFSET_DEFAULT_;
1881         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
1882         if (parseHwregConstruct(HwReg, Offset, Width))
1883           return MatchOperand_ParseFail;
1884         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
1885           if (HwReg.IsSymbolic)
1886             Error(S, "invalid symbolic name of hardware register");
1887           else
1888             Error(S, "invalid code of hardware register: only 6-bit values are legal");
1889         }
1890         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
1891           Error(S, "invalid bit offset: only 5-bit values are legal");
1892         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
1893           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
1894         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
1895       }
1896       break;
1897   }
1898   Operands.push_back(AMDGPUOperand::CreateImm(Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
1899   return MatchOperand_Success;
1900 }
1901 
1902 bool AMDGPUOperand::isSWaitCnt() const {
1903   return isImm();
1904 }
1905 
1906 bool AMDGPUOperand::isHwreg() const {
1907   return isImmTy(ImmTyHwreg);
1908 }
1909 
1910 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
1911   using namespace llvm::AMDGPU::SendMsg;
1912 
1913   if (Parser.getTok().getString() != "sendmsg")
1914     return true;
1915   Parser.Lex();
1916 
1917   if (getLexer().isNot(AsmToken::LParen))
1918     return true;
1919   Parser.Lex();
1920 
1921   if (getLexer().is(AsmToken::Identifier)) {
1922     Msg.IsSymbolic = true;
1923     Msg.Id = ID_UNKNOWN_;
1924     const std::string tok = Parser.getTok().getString();
1925     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
1926       switch(i) {
1927         default: continue; // Omit gaps.
1928         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
1929       }
1930       if (tok == IdSymbolic[i]) {
1931         Msg.Id = i;
1932         break;
1933       }
1934     }
1935     Parser.Lex();
1936   } else {
1937     Msg.IsSymbolic = false;
1938     if (getLexer().isNot(AsmToken::Integer))
1939       return true;
1940     if (getParser().parseAbsoluteExpression(Msg.Id))
1941       return true;
1942     if (getLexer().is(AsmToken::Integer))
1943       if (getParser().parseAbsoluteExpression(Msg.Id))
1944         Msg.Id = ID_UNKNOWN_;
1945   }
1946   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
1947     return false;
1948 
1949   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
1950     if (getLexer().isNot(AsmToken::RParen))
1951       return true;
1952     Parser.Lex();
1953     return false;
1954   }
1955 
1956   if (getLexer().isNot(AsmToken::Comma))
1957     return true;
1958   Parser.Lex();
1959 
1960   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
1961   Operation.Id = ID_UNKNOWN_;
1962   if (getLexer().is(AsmToken::Identifier)) {
1963     Operation.IsSymbolic = true;
1964     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
1965     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
1966     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
1967     const StringRef Tok = Parser.getTok().getString();
1968     for (int i = F; i < L; ++i) {
1969       if (Tok == S[i]) {
1970         Operation.Id = i;
1971         break;
1972       }
1973     }
1974     Parser.Lex();
1975   } else {
1976     Operation.IsSymbolic = false;
1977     if (getLexer().isNot(AsmToken::Integer))
1978       return true;
1979     if (getParser().parseAbsoluteExpression(Operation.Id))
1980       return true;
1981   }
1982 
1983   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
1984     // Stream id is optional.
1985     if (getLexer().is(AsmToken::RParen)) {
1986       Parser.Lex();
1987       return false;
1988     }
1989 
1990     if (getLexer().isNot(AsmToken::Comma))
1991       return true;
1992     Parser.Lex();
1993 
1994     if (getLexer().isNot(AsmToken::Integer))
1995       return true;
1996     if (getParser().parseAbsoluteExpression(StreamId))
1997       return true;
1998   }
1999 
2000   if (getLexer().isNot(AsmToken::RParen))
2001     return true;
2002   Parser.Lex();
2003   return false;
2004 }
2005 
2006 AMDGPUAsmParser::OperandMatchResultTy
2007 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
2008   using namespace llvm::AMDGPU::SendMsg;
2009 
2010   int64_t Imm16Val = 0;
2011   SMLoc S = Parser.getTok().getLoc();
2012 
2013   switch(getLexer().getKind()) {
2014   default:
2015     return MatchOperand_NoMatch;
2016   case AsmToken::Integer:
2017     // The operand can be an integer value.
2018     if (getParser().parseAbsoluteExpression(Imm16Val))
2019       return MatchOperand_NoMatch;
2020     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2021       Error(S, "invalid immediate: only 16-bit values are legal");
2022       // Do not return error code, but create an imm operand anyway and proceed
2023       // to the next operand, if any. That avoids unneccessary error messages.
2024     }
2025     break;
2026   case AsmToken::Identifier: {
2027       OperandInfoTy Msg(ID_UNKNOWN_);
2028       OperandInfoTy Operation(OP_UNKNOWN_);
2029       int64_t StreamId = STREAM_ID_DEFAULT_;
2030       if (parseSendMsgConstruct(Msg, Operation, StreamId))
2031         return MatchOperand_ParseFail;
2032       do {
2033         // Validate and encode message ID.
2034         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
2035                 || Msg.Id == ID_SYSMSG)) {
2036           if (Msg.IsSymbolic)
2037             Error(S, "invalid/unsupported symbolic name of message");
2038           else
2039             Error(S, "invalid/unsupported code of message");
2040           break;
2041         }
2042         Imm16Val = (Msg.Id << ID_SHIFT_);
2043         // Validate and encode operation ID.
2044         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
2045           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
2046             if (Operation.IsSymbolic)
2047               Error(S, "invalid symbolic name of GS_OP");
2048             else
2049               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
2050             break;
2051           }
2052           if (Operation.Id == OP_GS_NOP
2053               && Msg.Id != ID_GS_DONE) {
2054             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
2055             break;
2056           }
2057           Imm16Val |= (Operation.Id << OP_SHIFT_);
2058         }
2059         if (Msg.Id == ID_SYSMSG) {
2060           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
2061             if (Operation.IsSymbolic)
2062               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
2063             else
2064               Error(S, "invalid/unsupported code of SYSMSG_OP");
2065             break;
2066           }
2067           Imm16Val |= (Operation.Id << OP_SHIFT_);
2068         }
2069         // Validate and encode stream ID.
2070         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2071           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
2072             Error(S, "invalid stream id: only 2-bit values are legal");
2073             break;
2074           }
2075           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
2076         }
2077       } while (0);
2078     }
2079     break;
2080   }
2081   Operands.push_back(AMDGPUOperand::CreateImm(Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
2082   return MatchOperand_Success;
2083 }
2084 
2085 bool AMDGPUOperand::isSendMsg() const {
2086   return isImmTy(ImmTySendMsg);
2087 }
2088 
2089 //===----------------------------------------------------------------------===//
2090 // sopp branch targets
2091 //===----------------------------------------------------------------------===//
2092 
2093 AMDGPUAsmParser::OperandMatchResultTy
2094 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
2095   SMLoc S = Parser.getTok().getLoc();
2096 
2097   switch (getLexer().getKind()) {
2098     default: return MatchOperand_ParseFail;
2099     case AsmToken::Integer: {
2100       int64_t Imm;
2101       if (getParser().parseAbsoluteExpression(Imm))
2102         return MatchOperand_ParseFail;
2103       Operands.push_back(AMDGPUOperand::CreateImm(Imm, S));
2104       return MatchOperand_Success;
2105     }
2106 
2107     case AsmToken::Identifier:
2108       Operands.push_back(AMDGPUOperand::CreateExpr(
2109           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
2110                                   Parser.getTok().getString()), getContext()), S));
2111       Parser.Lex();
2112       return MatchOperand_Success;
2113   }
2114 }
2115 
2116 //===----------------------------------------------------------------------===//
2117 // mubuf
2118 //===----------------------------------------------------------------------===//
2119 
2120 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
2121   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyGLC);
2122 }
2123 
2124 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
2125   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTySLC);
2126 }
2127 
2128 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
2129   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyTFE);
2130 }
2131 
2132 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
2133                                const OperandVector &Operands,
2134                                bool IsAtomic, bool IsAtomicReturn) {
2135   OptionalImmIndexMap OptionalIdx;
2136   assert(IsAtomicReturn ? IsAtomic : true);
2137 
2138   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2139     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2140 
2141     // Add the register arguments
2142     if (Op.isReg()) {
2143       Op.addRegOperands(Inst, 1);
2144       continue;
2145     }
2146 
2147     // Handle the case where soffset is an immediate
2148     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
2149       Op.addImmOperands(Inst, 1);
2150       continue;
2151     }
2152 
2153     // Handle tokens like 'offen' which are sometimes hard-coded into the
2154     // asm string.  There are no MCInst operands for these.
2155     if (Op.isToken()) {
2156       continue;
2157     }
2158     assert(Op.isImm());
2159 
2160     // Handle optional arguments
2161     OptionalIdx[Op.getImmTy()] = i;
2162   }
2163 
2164   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
2165   if (IsAtomicReturn) {
2166     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
2167     Inst.insert(I, *I);
2168   }
2169 
2170   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2171   if (!IsAtomic) { // glc is hard-coded.
2172     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2173   }
2174   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2175   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2176 }
2177 
2178 //===----------------------------------------------------------------------===//
2179 // mimg
2180 //===----------------------------------------------------------------------===//
2181 
2182 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
2183   unsigned I = 1;
2184   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2185   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2186     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2187   }
2188 
2189   OptionalImmIndexMap OptionalIdx;
2190 
2191   for (unsigned E = Operands.size(); I != E; ++I) {
2192     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2193 
2194     // Add the register arguments
2195     if (Op.isRegOrImm()) {
2196       Op.addRegOrImmOperands(Inst, 1);
2197       continue;
2198     } else if (Op.isImmModifier()) {
2199       OptionalIdx[Op.getImmTy()] = I;
2200     } else {
2201       assert(false);
2202     }
2203   }
2204 
2205   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2206   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2207   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2208   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2209   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2210   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2211   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2212   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2213 }
2214 
2215 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
2216   unsigned I = 1;
2217   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2218   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2219     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2220   }
2221 
2222   // Add src, same as dst
2223   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
2224 
2225   OptionalImmIndexMap OptionalIdx;
2226 
2227   for (unsigned E = Operands.size(); I != E; ++I) {
2228     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2229 
2230     // Add the register arguments
2231     if (Op.isRegOrImm()) {
2232       Op.addRegOrImmOperands(Inst, 1);
2233       continue;
2234     } else if (Op.isImmModifier()) {
2235       OptionalIdx[Op.getImmTy()] = I;
2236     } else {
2237       assert(false);
2238     }
2239   }
2240 
2241   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2242   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2243   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2244   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2245   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2246   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2247   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2248   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2249 }
2250 
2251 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
2252   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyDMask);
2253 }
2254 
2255 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
2256   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
2257 }
2258 
2259 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
2260   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyDA);
2261 }
2262 
2263 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
2264   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyR128);
2265 }
2266 
2267 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
2268   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyLWE);
2269 }
2270 
2271 //===----------------------------------------------------------------------===//
2272 // smrd
2273 //===----------------------------------------------------------------------===//
2274 
2275 bool AMDGPUOperand::isSMRDOffset() const {
2276 
2277   // FIXME: Support 20-bit offsets on VI.  We need to to pass subtarget
2278   // information here.
2279   return isImm() && isUInt<8>(getImm());
2280 }
2281 
2282 bool AMDGPUOperand::isSMRDLiteralOffset() const {
2283   // 32-bit literals are only supported on CI and we only want to use them
2284   // when the offset is > 8-bits.
2285   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
2286 }
2287 
2288 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset() const {
2289   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2290 }
2291 
2292 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
2293   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2294 }
2295 
2296 //===----------------------------------------------------------------------===//
2297 // vop3
2298 //===----------------------------------------------------------------------===//
2299 
2300 static bool ConvertOmodMul(int64_t &Mul) {
2301   if (Mul != 1 && Mul != 2 && Mul != 4)
2302     return false;
2303 
2304   Mul >>= 1;
2305   return true;
2306 }
2307 
2308 static bool ConvertOmodDiv(int64_t &Div) {
2309   if (Div == 1) {
2310     Div = 0;
2311     return true;
2312   }
2313 
2314   if (Div == 2) {
2315     Div = 3;
2316     return true;
2317   }
2318 
2319   return false;
2320 }
2321 
2322 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
2323   if (BoundCtrl == 0) {
2324     BoundCtrl = 1;
2325     return true;
2326   } else if (BoundCtrl == -1) {
2327     BoundCtrl = 0;
2328     return true;
2329   }
2330   return false;
2331 }
2332 
2333 // Note: the order in this table matches the order of operands in AsmString.
2334 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
2335   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
2336   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
2337   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
2338   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
2339   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
2340   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
2341   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
2342   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
2343   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
2344   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
2345   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
2346   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
2347   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
2348   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
2349   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
2350   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
2351   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
2352   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
2353   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
2354   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
2355   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
2356   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
2357   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
2358   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
2359 };
2360 
2361 AMDGPUAsmParser::OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
2362   OperandMatchResultTy res;
2363   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
2364     // try to parse any optional operand here
2365     if (Op.IsBit) {
2366       res = parseNamedBit(Op.Name, Operands, Op.Type);
2367     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
2368       res = parseOModOperand(Operands);
2369     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
2370                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
2371                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
2372       res = parseSDWASel(Operands, Op.Name, Op.Type);
2373     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
2374       res = parseSDWADstUnused(Operands);
2375     } else {
2376       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
2377     }
2378     if (res != MatchOperand_NoMatch) {
2379       return res;
2380     }
2381   }
2382   return MatchOperand_NoMatch;
2383 }
2384 
2385 AMDGPUAsmParser::OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands)
2386 {
2387   StringRef Name = Parser.getTok().getString();
2388   if (Name == "mul") {
2389     return parseIntWithPrefix("mul", Operands, AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
2390   } else if (Name == "div") {
2391     return parseIntWithPrefix("div", Operands, AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
2392   } else {
2393     return MatchOperand_NoMatch;
2394   }
2395 }
2396 
2397 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
2398   unsigned I = 1;
2399   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2400   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2401     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2402   }
2403   for (unsigned E = Operands.size(); I != E; ++I)
2404     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
2405 }
2406 
2407 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
2408   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
2409   if (TSFlags & SIInstrFlags::VOP3) {
2410     cvtVOP3(Inst, Operands);
2411   } else {
2412     cvtId(Inst, Operands);
2413   }
2414 }
2415 
2416 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
2417   OptionalImmIndexMap OptionalIdx;
2418   unsigned I = 1;
2419   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2420   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2421     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2422   }
2423 
2424   for (unsigned E = Operands.size(); I != E; ++I) {
2425     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2426     if (Op.isRegOrImmWithInputMods()) {
2427       // only fp modifiers allowed in VOP3
2428       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
2429     } else if (Op.isImm()) {
2430       OptionalIdx[Op.getImmTy()] = I;
2431     } else {
2432       assert(false);
2433     }
2434   }
2435 
2436   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
2437   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
2438 }
2439 
2440 //===----------------------------------------------------------------------===//
2441 // dpp
2442 //===----------------------------------------------------------------------===//
2443 
2444 bool AMDGPUOperand::isDPPCtrl() const {
2445   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
2446   if (result) {
2447     int64_t Imm = getImm();
2448     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
2449            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
2450            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
2451            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
2452            (Imm == 0x130) ||
2453            (Imm == 0x134) ||
2454            (Imm == 0x138) ||
2455            (Imm == 0x13c) ||
2456            (Imm == 0x140) ||
2457            (Imm == 0x141) ||
2458            (Imm == 0x142) ||
2459            (Imm == 0x143);
2460   }
2461   return false;
2462 }
2463 
2464 AMDGPUAsmParser::OperandMatchResultTy
2465 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
2466   SMLoc S = Parser.getTok().getLoc();
2467   StringRef Prefix;
2468   int64_t Int;
2469 
2470   if (getLexer().getKind() == AsmToken::Identifier) {
2471     Prefix = Parser.getTok().getString();
2472   } else {
2473     return MatchOperand_NoMatch;
2474   }
2475 
2476   if (Prefix == "row_mirror") {
2477     Int = 0x140;
2478   } else if (Prefix == "row_half_mirror") {
2479     Int = 0x141;
2480   } else {
2481     // Check to prevent parseDPPCtrlOps from eating invalid tokens
2482     if (Prefix != "quad_perm"
2483         && Prefix != "row_shl"
2484         && Prefix != "row_shr"
2485         && Prefix != "row_ror"
2486         && Prefix != "wave_shl"
2487         && Prefix != "wave_rol"
2488         && Prefix != "wave_shr"
2489         && Prefix != "wave_ror"
2490         && Prefix != "row_bcast") {
2491       return MatchOperand_NoMatch;
2492     }
2493 
2494     Parser.Lex();
2495     if (getLexer().isNot(AsmToken::Colon))
2496       return MatchOperand_ParseFail;
2497 
2498     if (Prefix == "quad_perm") {
2499       // quad_perm:[%d,%d,%d,%d]
2500       Parser.Lex();
2501       if (getLexer().isNot(AsmToken::LBrac))
2502         return MatchOperand_ParseFail;
2503 
2504       Parser.Lex();
2505       if (getLexer().isNot(AsmToken::Integer))
2506         return MatchOperand_ParseFail;
2507       Int = getLexer().getTok().getIntVal();
2508 
2509       Parser.Lex();
2510       if (getLexer().isNot(AsmToken::Comma))
2511         return MatchOperand_ParseFail;
2512       Parser.Lex();
2513       if (getLexer().isNot(AsmToken::Integer))
2514         return MatchOperand_ParseFail;
2515       Int += (getLexer().getTok().getIntVal() << 2);
2516 
2517       Parser.Lex();
2518       if (getLexer().isNot(AsmToken::Comma))
2519         return MatchOperand_ParseFail;
2520       Parser.Lex();
2521       if (getLexer().isNot(AsmToken::Integer))
2522         return MatchOperand_ParseFail;
2523       Int += (getLexer().getTok().getIntVal() << 4);
2524 
2525       Parser.Lex();
2526       if (getLexer().isNot(AsmToken::Comma))
2527         return MatchOperand_ParseFail;
2528       Parser.Lex();
2529       if (getLexer().isNot(AsmToken::Integer))
2530         return MatchOperand_ParseFail;
2531       Int += (getLexer().getTok().getIntVal() << 6);
2532 
2533       Parser.Lex();
2534       if (getLexer().isNot(AsmToken::RBrac))
2535         return MatchOperand_ParseFail;
2536 
2537     } else {
2538       // sel:%d
2539       Parser.Lex();
2540       if (getLexer().isNot(AsmToken::Integer))
2541         return MatchOperand_ParseFail;
2542       Int = getLexer().getTok().getIntVal();
2543 
2544       if (Prefix == "row_shl") {
2545         Int |= 0x100;
2546       } else if (Prefix == "row_shr") {
2547         Int |= 0x110;
2548       } else if (Prefix == "row_ror") {
2549         Int |= 0x120;
2550       } else if (Prefix == "wave_shl") {
2551         Int = 0x130;
2552       } else if (Prefix == "wave_rol") {
2553         Int = 0x134;
2554       } else if (Prefix == "wave_shr") {
2555         Int = 0x138;
2556       } else if (Prefix == "wave_ror") {
2557         Int = 0x13C;
2558       } else if (Prefix == "row_bcast") {
2559         if (Int == 15) {
2560           Int = 0x142;
2561         } else if (Int == 31) {
2562           Int = 0x143;
2563         }
2564       } else {
2565         return MatchOperand_ParseFail;
2566       }
2567     }
2568   }
2569   Parser.Lex(); // eat last token
2570 
2571   Operands.push_back(AMDGPUOperand::CreateImm(Int, S,
2572                                               AMDGPUOperand::ImmTyDppCtrl));
2573   return MatchOperand_Success;
2574 }
2575 
2576 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
2577   return AMDGPUOperand::CreateImm(0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
2578 }
2579 
2580 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
2581   return AMDGPUOperand::CreateImm(0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
2582 }
2583 
2584 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
2585   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
2586 }
2587 
2588 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
2589   OptionalImmIndexMap OptionalIdx;
2590 
2591   unsigned I = 1;
2592   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2593   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2594     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2595   }
2596 
2597   for (unsigned E = Operands.size(); I != E; ++I) {
2598     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2599     // Add the register arguments
2600     if (Op.isRegOrImmWithInputMods()) {
2601       // Only float modifiers supported in DPP
2602       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
2603     } else if (Op.isDPPCtrl()) {
2604       Op.addImmOperands(Inst, 1);
2605     } else if (Op.isImm()) {
2606       // Handle optional arguments
2607       OptionalIdx[Op.getImmTy()] = I;
2608     } else {
2609       llvm_unreachable("Invalid operand type");
2610     }
2611   }
2612 
2613   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
2614   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
2615   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
2616 }
2617 
2618 //===----------------------------------------------------------------------===//
2619 // sdwa
2620 //===----------------------------------------------------------------------===//
2621 
2622 AMDGPUAsmParser::OperandMatchResultTy
2623 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
2624                               AMDGPUOperand::ImmTy Type) {
2625   SMLoc S = Parser.getTok().getLoc();
2626   StringRef Value;
2627   AMDGPUAsmParser::OperandMatchResultTy res;
2628 
2629   res = parseStringWithPrefix(Prefix, Value);
2630   if (res != MatchOperand_Success) {
2631     return res;
2632   }
2633 
2634   int64_t Int;
2635   Int = StringSwitch<int64_t>(Value)
2636         .Case("BYTE_0", 0)
2637         .Case("BYTE_1", 1)
2638         .Case("BYTE_2", 2)
2639         .Case("BYTE_3", 3)
2640         .Case("WORD_0", 4)
2641         .Case("WORD_1", 5)
2642         .Case("DWORD", 6)
2643         .Default(0xffffffff);
2644   Parser.Lex(); // eat last token
2645 
2646   if (Int == 0xffffffff) {
2647     return MatchOperand_ParseFail;
2648   }
2649 
2650   Operands.push_back(AMDGPUOperand::CreateImm(Int, S, Type));
2651   return MatchOperand_Success;
2652 }
2653 
2654 AMDGPUAsmParser::OperandMatchResultTy
2655 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
2656   SMLoc S = Parser.getTok().getLoc();
2657   StringRef Value;
2658   AMDGPUAsmParser::OperandMatchResultTy res;
2659 
2660   res = parseStringWithPrefix("dst_unused", Value);
2661   if (res != MatchOperand_Success) {
2662     return res;
2663   }
2664 
2665   int64_t Int;
2666   Int = StringSwitch<int64_t>(Value)
2667         .Case("UNUSED_PAD", 0)
2668         .Case("UNUSED_SEXT", 1)
2669         .Case("UNUSED_PRESERVE", 2)
2670         .Default(0xffffffff);
2671   Parser.Lex(); // eat last token
2672 
2673   if (Int == 0xffffffff) {
2674     return MatchOperand_ParseFail;
2675   }
2676 
2677   Operands.push_back(AMDGPUOperand::CreateImm(Int, S,
2678                                               AMDGPUOperand::ImmTySdwaDstUnused));
2679   return MatchOperand_Success;
2680 }
2681 
2682 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
2683   cvtSDWA(Inst, Operands, true);
2684 }
2685 
2686 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
2687   cvtSDWA(Inst, Operands, false);
2688 }
2689 
2690 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
2691                               bool IsVOP1) {
2692   OptionalImmIndexMap OptionalIdx;
2693 
2694   unsigned I = 1;
2695   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2696   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2697     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2698   }
2699 
2700   for (unsigned E = Operands.size(); I != E; ++I) {
2701     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2702     // Add the register arguments
2703     if (Op.isRegOrImmWithInputMods()) {
2704        Op.addRegOrImmWithInputModsOperands(Inst, 2);
2705     } else if (Op.isImm()) {
2706       // Handle optional arguments
2707       OptionalIdx[Op.getImmTy()] = I;
2708     } else {
2709       llvm_unreachable("Invalid operand type");
2710     }
2711   }
2712 
2713   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
2714 
2715   if (Inst.getOpcode() == AMDGPU::V_NOP_sdwa) {
2716     // V_NOP_sdwa has no optional sdwa arguments
2717     return;
2718   }
2719   if (IsVOP1) {
2720     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
2721     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
2722     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
2723   } else { // VOP2
2724     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
2725     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
2726     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
2727     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, 6);
2728   }
2729 }
2730 
2731 /// Force static initialization.
2732 extern "C" void LLVMInitializeAMDGPUAsmParser() {
2733   RegisterMCAsmParser<AMDGPUAsmParser> A(TheAMDGPUTarget);
2734   RegisterMCAsmParser<AMDGPUAsmParser> B(TheGCNTarget);
2735 }
2736 
2737 #define GET_REGISTER_MATCHER
2738 #define GET_MATCHER_IMPLEMENTATION
2739 #include "AMDGPUGenAsmMatcher.inc"
2740 
2741 
2742 // This fuction should be defined after auto-generated include so that we have
2743 // MatchClassKind enum defined
2744 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
2745                                                      unsigned Kind) {
2746   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
2747   // But MatchInstructionImpl() expects to meet token and fails to validate
2748   // operand. This method checks if we are given immediate operand but expect to
2749   // get corresponding token.
2750   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
2751   switch (Kind) {
2752   case MCK_addr64:
2753     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
2754   case MCK_gds:
2755     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
2756   case MCK_glc:
2757     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
2758   case MCK_idxen:
2759     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
2760   case MCK_offen:
2761     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
2762   case MCK_SSrc32:
2763     // When operands have expression values, they will return true for isToken,
2764     // because it is not possible to distinguish between a token and an
2765     // expression at parse time. MatchInstructionImpl() will always try to
2766     // match an operand as a token, when isToken returns true, and when the
2767     // name of the expression is not a valid token, the match will fail,
2768     // so we need to handle it here.
2769     return Operand.isSSrc32() ? Match_Success : Match_InvalidOperand;
2770   default: return Match_InvalidOperand;
2771   }
2772 }
2773