1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/STLExtras.h"
19 #include "llvm/ADT/SmallBitVector.h"
20 #include "llvm/ADT/SmallString.h"
21 #include "llvm/ADT/StringSwitch.h"
22 #include "llvm/ADT/Twine.h"
23 #include "llvm/MC/MCContext.h"
24 #include "llvm/MC/MCExpr.h"
25 #include "llvm/MC/MCInst.h"
26 #include "llvm/MC/MCInstrInfo.h"
27 #include "llvm/MC/MCParser/MCAsmLexer.h"
28 #include "llvm/MC/MCParser/MCAsmParser.h"
29 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
30 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
31 #include "llvm/MC/MCRegisterInfo.h"
32 #include "llvm/MC/MCStreamer.h"
33 #include "llvm/MC/MCSubtargetInfo.h"
34 #include "llvm/MC/MCSymbolELF.h"
35 #include "llvm/Support/Debug.h"
36 #include "llvm/Support/ELF.h"
37 #include "llvm/Support/SourceMgr.h"
38 #include "llvm/Support/TargetRegistry.h"
39 #include "llvm/Support/raw_ostream.h"
40 #include "llvm/Support/MathExtras.h"
41 
42 using namespace llvm;
43 
44 namespace {
45 
46 struct OptionalOperand;
47 
48 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
49 
50 class AMDGPUOperand : public MCParsedAsmOperand {
51   enum KindTy {
52     Token,
53     Immediate,
54     Register,
55     Expression
56   } Kind;
57 
58   SMLoc StartLoc, EndLoc;
59 
60 public:
61   AMDGPUOperand(enum KindTy K) : MCParsedAsmOperand(), Kind(K) {}
62 
63   typedef std::unique_ptr<AMDGPUOperand> Ptr;
64 
65   struct Modifiers {
66     bool Abs;
67     bool Neg;
68     bool Sext;
69 
70     bool hasFPModifiers() const { return Abs || Neg; }
71     bool hasIntModifiers() const { return Sext; }
72     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
73 
74     int64_t getFPModifiersOperand() const {
75       int64_t Operand = 0;
76       Operand |= Abs ? SISrcMods::ABS : 0;
77       Operand |= Neg ? SISrcMods::NEG : 0;
78       return Operand;
79     }
80 
81     int64_t getIntModifiersOperand() const {
82       int64_t Operand = 0;
83       Operand |= Sext ? SISrcMods::SEXT : 0;
84       return Operand;
85     }
86 
87     int64_t getModifiersOperand() const {
88       assert(!(hasFPModifiers() && hasIntModifiers())
89            && "fp and int modifiers should not be used simultaneously");
90       if (hasFPModifiers()) {
91         return getFPModifiersOperand();
92       } else if (hasIntModifiers()) {
93         return getIntModifiersOperand();
94       } else {
95         return 0;
96       }
97     }
98 
99     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
100   };
101 
102   enum ImmTy {
103     ImmTyNone,
104     ImmTyGDS,
105     ImmTyOffen,
106     ImmTyIdxen,
107     ImmTyAddr64,
108     ImmTyOffset,
109     ImmTyOffset0,
110     ImmTyOffset1,
111     ImmTyGLC,
112     ImmTySLC,
113     ImmTyTFE,
114     ImmTyClampSI,
115     ImmTyOModSI,
116     ImmTyDppCtrl,
117     ImmTyDppRowMask,
118     ImmTyDppBankMask,
119     ImmTyDppBoundCtrl,
120     ImmTySdwaDstSel,
121     ImmTySdwaSrc0Sel,
122     ImmTySdwaSrc1Sel,
123     ImmTySdwaDstUnused,
124     ImmTyDMask,
125     ImmTyUNorm,
126     ImmTyDA,
127     ImmTyR128,
128     ImmTyLWE,
129     ImmTyHwreg,
130     ImmTySendMsg,
131   };
132 
133   struct TokOp {
134     const char *Data;
135     unsigned Length;
136   };
137 
138   struct ImmOp {
139     bool IsFPImm;
140     ImmTy Type;
141     int64_t Val;
142     Modifiers Mods;
143   };
144 
145   struct RegOp {
146     unsigned RegNo;
147     Modifiers Mods;
148     const MCRegisterInfo *TRI;
149     const MCSubtargetInfo *STI;
150     bool IsForcedVOP3;
151   };
152 
153   union {
154     TokOp Tok;
155     ImmOp Imm;
156     RegOp Reg;
157     const MCExpr *Expr;
158   };
159 
160   bool isToken() const override {
161     if (Kind == Token)
162       return true;
163 
164     if (Kind != Expression || !Expr)
165       return false;
166 
167     // When parsing operands, we can't always tell if something was meant to be
168     // a token, like 'gds', or an expression that references a global variable.
169     // In this case, we assume the string is an expression, and if we need to
170     // interpret is a token, then we treat the symbol name as the token.
171     return isa<MCSymbolRefExpr>(Expr);
172   }
173 
174   bool isImm() const override {
175     return Kind == Immediate;
176   }
177 
178   bool isInlinableImm() const {
179     if (!isImmTy(ImmTyNone)) {
180       // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
181       return false;
182     }
183     // TODO: We should avoid using host float here. It would be better to
184     // check the float bit values which is what a few other places do.
185     // We've had bot failures before due to weird NaN support on mips hosts.
186     const float F = BitsToFloat(Imm.Val);
187     // TODO: Add 1/(2*pi) for VI
188     return (Imm.Val <= 64 && Imm.Val >= -16) ||
189            (F == 0.0 || F == 0.5 || F == -0.5 || F == 1.0 || F == -1.0 ||
190            F == 2.0 || F == -2.0 || F == 4.0 || F == -4.0);
191   }
192 
193   bool isRegKind() const {
194     return Kind == Register;
195   }
196 
197   bool isReg() const override {
198     return isRegKind() && !Reg.Mods.hasModifiers();
199   }
200 
201   bool isRegOrImmWithInputMods() const {
202     return isRegKind() || isInlinableImm();
203   }
204 
205   bool isImmTy(ImmTy ImmT) const {
206     return isImm() && Imm.Type == ImmT;
207   }
208 
209   bool isImmModifier() const {
210     return isImm() && Imm.Type != ImmTyNone;
211   }
212 
213   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
214   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
215   bool isDMask() const { return isImmTy(ImmTyDMask); }
216   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
217   bool isDA() const { return isImmTy(ImmTyDA); }
218   bool isR128() const { return isImmTy(ImmTyUNorm); }
219   bool isLWE() const { return isImmTy(ImmTyLWE); }
220   bool isOffen() const { return isImmTy(ImmTyOffen); }
221   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
222   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
223   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
224   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
225   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
226   bool isGDS() const { return isImmTy(ImmTyGDS); }
227   bool isGLC() const { return isImmTy(ImmTyGLC); }
228   bool isSLC() const { return isImmTy(ImmTySLC); }
229   bool isTFE() const { return isImmTy(ImmTyTFE); }
230   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
231   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
232   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
233   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
234   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
235   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
236   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
237 
238   bool isMod() const {
239     return isClampSI() || isOModSI();
240   }
241 
242   bool isRegOrImm() const {
243     return isReg() || isImm();
244   }
245 
246   bool isRegClass(unsigned RCID) const {
247     return isReg() && Reg.TRI->getRegClass(RCID).contains(getReg());
248   }
249 
250   bool isSCSrc32() const {
251     return isInlinableImm() || isRegClass(AMDGPU::SReg_32RegClassID);
252   }
253 
254   bool isSCSrc64() const {
255     return isInlinableImm() || isRegClass(AMDGPU::SReg_64RegClassID);
256   }
257 
258   bool isSSrc32() const {
259     return isImm() || isSCSrc32() || isExpr();
260   }
261 
262   bool isSSrc64() const {
263     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
264     // See isVSrc64().
265     return isImm() || isSCSrc64();
266   }
267 
268   bool isVCSrc32() const {
269     return isInlinableImm() || isRegClass(AMDGPU::VS_32RegClassID);
270   }
271 
272   bool isVCSrc64() const {
273     return isInlinableImm() || isRegClass(AMDGPU::VS_64RegClassID);
274   }
275 
276   bool isVSrc32() const {
277     return isImm() || isVCSrc32();
278   }
279 
280   bool isVSrc64() const {
281     // TODO: Check if the 64-bit value (coming from assembly source) can be
282     // narrowed to 32 bits (in the instruction stream). That require knowledge
283     // of instruction type (unsigned/signed, floating or "untyped"/B64),
284     // see [AMD GCN3 ISA 6.3.1].
285     // TODO: How 64-bit values are formed from 32-bit literals in _B64 insns?
286     return isImm() || isVCSrc64();
287   }
288 
289   bool isMem() const override {
290     return false;
291   }
292 
293   bool isExpr() const {
294     return Kind == Expression;
295   }
296 
297   bool isSoppBrTarget() const {
298     return isExpr() || isImm();
299   }
300 
301   bool isSWaitCnt() const;
302   bool isHwreg() const;
303   bool isSendMsg() const;
304   bool isSMRDOffset() const;
305   bool isSMRDLiteralOffset() const;
306   bool isDPPCtrl() const;
307 
308   StringRef getExpressionAsToken() const {
309     assert(isExpr());
310     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
311     return S->getSymbol().getName();
312   }
313 
314 
315   StringRef getToken() const {
316     assert(isToken());
317 
318     if (Kind == Expression)
319       return getExpressionAsToken();
320 
321     return StringRef(Tok.Data, Tok.Length);
322   }
323 
324   int64_t getImm() const {
325     assert(isImm());
326     return Imm.Val;
327   }
328 
329   enum ImmTy getImmTy() const {
330     assert(isImm());
331     return Imm.Type;
332   }
333 
334   unsigned getReg() const override {
335     return Reg.RegNo;
336   }
337 
338   SMLoc getStartLoc() const override {
339     return StartLoc;
340   }
341 
342   SMLoc getEndLoc() const override {
343     return EndLoc;
344   }
345 
346   Modifiers getModifiers() const {
347     assert(isRegKind() || isImmTy(ImmTyNone));
348     return isRegKind() ? Reg.Mods : Imm.Mods;
349   }
350 
351   void setModifiers(Modifiers Mods) {
352     assert(isRegKind() || isImmTy(ImmTyNone));
353     if (isRegKind())
354       Reg.Mods = Mods;
355     else
356       Imm.Mods = Mods;
357   }
358 
359   bool hasModifiers() const {
360     return getModifiers().hasModifiers();
361   }
362 
363   bool hasFPModifiers() const {
364     return getModifiers().hasFPModifiers();
365   }
366 
367   bool hasIntModifiers() const {
368     return getModifiers().hasIntModifiers();
369   }
370 
371   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const {
372     if (isImmTy(ImmTyNone) && ApplyModifiers && Imm.Mods.hasFPModifiers()) {
373       // Apply modifiers to immediate value
374       int64_t Val = Imm.Val;
375       bool Negate = Imm.Mods.Neg; // Only negate can get here
376       if (Imm.IsFPImm) {
377         APFloat F(BitsToFloat(Val));
378         if (Negate) {
379           F.changeSign();
380         }
381         Val = F.bitcastToAPInt().getZExtValue();
382       } else {
383         Val = Negate ? -Val : Val;
384       }
385       Inst.addOperand(MCOperand::createImm(Val));
386     } else {
387       Inst.addOperand(MCOperand::createImm(getImm()));
388     }
389   }
390 
391   void addRegOperands(MCInst &Inst, unsigned N) const {
392     Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), *Reg.STI)));
393   }
394 
395   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
396     if (isRegKind())
397       addRegOperands(Inst, N);
398     else if (isExpr())
399       Inst.addOperand(MCOperand::createExpr(Expr));
400     else
401       addImmOperands(Inst, N);
402   }
403 
404   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
405     Modifiers Mods = getModifiers();
406     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
407     if (isRegKind()) {
408       addRegOperands(Inst, N);
409     } else {
410       addImmOperands(Inst, N, false);
411     }
412   }
413 
414   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
415     assert(!hasIntModifiers());
416     addRegOrImmWithInputModsOperands(Inst, N);
417   }
418 
419   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
420     assert(!hasFPModifiers());
421     addRegOrImmWithInputModsOperands(Inst, N);
422   }
423 
424   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
425     if (isImm())
426       addImmOperands(Inst, N);
427     else {
428       assert(isExpr());
429       Inst.addOperand(MCOperand::createExpr(Expr));
430     }
431   }
432 
433   void printImmTy(raw_ostream& OS, ImmTy Type) const {
434     switch (Type) {
435     case ImmTyNone: OS << "None"; break;
436     case ImmTyGDS: OS << "GDS"; break;
437     case ImmTyOffen: OS << "Offen"; break;
438     case ImmTyIdxen: OS << "Idxen"; break;
439     case ImmTyAddr64: OS << "Addr64"; break;
440     case ImmTyOffset: OS << "Offset"; break;
441     case ImmTyOffset0: OS << "Offset0"; break;
442     case ImmTyOffset1: OS << "Offset1"; break;
443     case ImmTyGLC: OS << "GLC"; break;
444     case ImmTySLC: OS << "SLC"; break;
445     case ImmTyTFE: OS << "TFE"; break;
446     case ImmTyClampSI: OS << "ClampSI"; break;
447     case ImmTyOModSI: OS << "OModSI"; break;
448     case ImmTyDppCtrl: OS << "DppCtrl"; break;
449     case ImmTyDppRowMask: OS << "DppRowMask"; break;
450     case ImmTyDppBankMask: OS << "DppBankMask"; break;
451     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
452     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
453     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
454     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
455     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
456     case ImmTyDMask: OS << "DMask"; break;
457     case ImmTyUNorm: OS << "UNorm"; break;
458     case ImmTyDA: OS << "DA"; break;
459     case ImmTyR128: OS << "R128"; break;
460     case ImmTyLWE: OS << "LWE"; break;
461     case ImmTyHwreg: OS << "Hwreg"; break;
462     case ImmTySendMsg: OS << "SendMsg"; break;
463     }
464   }
465 
466   void print(raw_ostream &OS) const override {
467     switch (Kind) {
468     case Register:
469       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
470       break;
471     case Immediate:
472       OS << '<' << getImm();
473       if (getImmTy() != ImmTyNone) {
474         OS << " type: "; printImmTy(OS, getImmTy());
475       }
476       OS << " mods: " << Imm.Mods << '>';
477       break;
478     case Token:
479       OS << '\'' << getToken() << '\'';
480       break;
481     case Expression:
482       OS << "<expr " << *Expr << '>';
483       break;
484     }
485   }
486 
487   static AMDGPUOperand::Ptr CreateImm(int64_t Val, SMLoc Loc,
488                                       enum ImmTy Type = ImmTyNone,
489                                       bool IsFPImm = false) {
490     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate);
491     Op->Imm.Val = Val;
492     Op->Imm.IsFPImm = IsFPImm;
493     Op->Imm.Type = Type;
494     Op->Imm.Mods = {false, false, false};
495     Op->StartLoc = Loc;
496     Op->EndLoc = Loc;
497     return Op;
498   }
499 
500   static AMDGPUOperand::Ptr CreateToken(StringRef Str, SMLoc Loc,
501                                         bool HasExplicitEncodingSize = true) {
502     auto Res = llvm::make_unique<AMDGPUOperand>(Token);
503     Res->Tok.Data = Str.data();
504     Res->Tok.Length = Str.size();
505     Res->StartLoc = Loc;
506     Res->EndLoc = Loc;
507     return Res;
508   }
509 
510   static AMDGPUOperand::Ptr CreateReg(unsigned RegNo, SMLoc S,
511                                       SMLoc E,
512                                       const MCRegisterInfo *TRI,
513                                       const MCSubtargetInfo *STI,
514                                       bool ForceVOP3) {
515     auto Op = llvm::make_unique<AMDGPUOperand>(Register);
516     Op->Reg.RegNo = RegNo;
517     Op->Reg.TRI = TRI;
518     Op->Reg.STI = STI;
519     Op->Reg.Mods = {false, false, false};
520     Op->Reg.IsForcedVOP3 = ForceVOP3;
521     Op->StartLoc = S;
522     Op->EndLoc = E;
523     return Op;
524   }
525 
526   static AMDGPUOperand::Ptr CreateExpr(const class MCExpr *Expr, SMLoc S) {
527     auto Op = llvm::make_unique<AMDGPUOperand>(Expression);
528     Op->Expr = Expr;
529     Op->StartLoc = S;
530     Op->EndLoc = S;
531     return Op;
532   }
533 };
534 
535 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
536   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
537   return OS;
538 }
539 
540 class AMDGPUAsmParser : public MCTargetAsmParser {
541   const MCInstrInfo &MII;
542   MCAsmParser &Parser;
543 
544   unsigned ForcedEncodingSize;
545   bool ForcedDPP;
546   bool ForcedSDWA;
547 
548   bool isSI() const {
549     return AMDGPU::isSI(getSTI());
550   }
551 
552   bool isCI() const {
553     return AMDGPU::isCI(getSTI());
554   }
555 
556   bool isVI() const {
557     return AMDGPU::isVI(getSTI());
558   }
559 
560   bool hasSGPR102_SGPR103() const {
561     return !isVI();
562   }
563 
564   /// @name Auto-generated Match Functions
565   /// {
566 
567 #define GET_ASSEMBLER_HEADER
568 #include "AMDGPUGenAsmMatcher.inc"
569 
570   /// }
571 
572 private:
573   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
574   bool ParseDirectiveHSACodeObjectVersion();
575   bool ParseDirectiveHSACodeObjectISA();
576   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
577   bool ParseDirectiveAMDKernelCodeT();
578   bool ParseSectionDirectiveHSAText();
579   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
580   bool ParseDirectiveAMDGPUHsaKernel();
581   bool ParseDirectiveAMDGPUHsaModuleGlobal();
582   bool ParseDirectiveAMDGPUHsaProgramGlobal();
583   bool ParseSectionDirectiveHSADataGlobalAgent();
584   bool ParseSectionDirectiveHSADataGlobalProgram();
585   bool ParseSectionDirectiveHSARodataReadonlyAgent();
586   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum);
587   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth);
588   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands, bool IsAtomic, bool IsAtomicReturn);
589 
590 public:
591   enum AMDGPUMatchResultTy {
592     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
593   };
594 
595   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
596                const MCInstrInfo &MII,
597                const MCTargetOptions &Options)
598       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser),
599         ForcedEncodingSize(0),
600         ForcedDPP(false),
601         ForcedSDWA(false) {
602     MCAsmParserExtension::Initialize(Parser);
603 
604     if (getSTI().getFeatureBits().none()) {
605       // Set default features.
606       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
607     }
608 
609     setAvailableFeatures(ComputeAvailableFeatures(getSTI().getFeatureBits()));
610 
611     {
612       // TODO: make those pre-defined variables read-only.
613       // Currently there is none suitable machinery in the core llvm-mc for this.
614       // MCSymbol::isRedefinable is intended for another purpose, and
615       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
616       AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
617       MCContext &Ctx = getContext();
618       MCSymbol *Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
619       Sym->setVariableValue(MCConstantExpr::create(Isa.Major, Ctx));
620       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
621       Sym->setVariableValue(MCConstantExpr::create(Isa.Minor, Ctx));
622       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
623       Sym->setVariableValue(MCConstantExpr::create(Isa.Stepping, Ctx));
624     }
625   }
626 
627   AMDGPUTargetStreamer &getTargetStreamer() {
628     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
629     return static_cast<AMDGPUTargetStreamer &>(TS);
630   }
631 
632   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
633   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
634   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
635 
636   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
637   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
638   bool isForcedDPP() const { return ForcedDPP; }
639   bool isForcedSDWA() const { return ForcedSDWA; }
640 
641   std::unique_ptr<AMDGPUOperand> parseRegister();
642   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
643   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
644   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
645                                       unsigned Kind) override;
646   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
647                                OperandVector &Operands, MCStreamer &Out,
648                                uint64_t &ErrorInfo,
649                                bool MatchingInlineAsm) override;
650   bool ParseDirective(AsmToken DirectiveID) override;
651   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
652   StringRef parseMnemonicSuffix(StringRef Name);
653   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
654                         SMLoc NameLoc, OperandVector &Operands) override;
655 
656   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
657   OperandMatchResultTy parseIntWithPrefix(const char *Prefix,
658                                           OperandVector &Operands,
659                                           enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
660                                           bool (*ConvertResult)(int64_t&) = 0);
661   OperandMatchResultTy parseNamedBit(const char *Name, OperandVector &Operands,
662                                      enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
663   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix, StringRef &Value);
664 
665   OperandMatchResultTy parseImm(OperandVector &Operands);
666   OperandMatchResultTy parseRegOrImm(OperandVector &Operands);
667   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands);
668   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands);
669 
670   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
671   void cvtDS(MCInst &Inst, const OperandVector &Operands);
672 
673   bool parseCnt(int64_t &IntVal);
674   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
675   OperandMatchResultTy parseHwreg(OperandVector &Operands);
676 
677 private:
678   struct OperandInfoTy {
679     int64_t Id;
680     bool IsSymbolic;
681     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
682   };
683 
684   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
685   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
686 public:
687   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
688 
689   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
690   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
691 
692   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
693   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
694   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
695   AMDGPUOperand::Ptr defaultGLC() const;
696   AMDGPUOperand::Ptr defaultSLC() const;
697   AMDGPUOperand::Ptr defaultTFE() const;
698 
699   AMDGPUOperand::Ptr defaultDMask() const;
700   AMDGPUOperand::Ptr defaultUNorm() const;
701   AMDGPUOperand::Ptr defaultDA() const;
702   AMDGPUOperand::Ptr defaultR128() const;
703   AMDGPUOperand::Ptr defaultLWE() const;
704   AMDGPUOperand::Ptr defaultSMRDOffset() const;
705   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
706 
707   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
708 
709   void cvtId(MCInst &Inst, const OperandVector &Operands);
710   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
711   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
712 
713   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
714   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
715 
716   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
717   AMDGPUOperand::Ptr defaultRowMask() const;
718   AMDGPUOperand::Ptr defaultBankMask() const;
719   AMDGPUOperand::Ptr defaultBoundCtrl() const;
720   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
721 
722   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
723                                     AMDGPUOperand::ImmTy Type);
724   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
725   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
726   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
727   void cvtSDWA(MCInst &Inst, const OperandVector &Operands, bool IsVOP1);
728 };
729 
730 struct OptionalOperand {
731   const char *Name;
732   AMDGPUOperand::ImmTy Type;
733   bool IsBit;
734   bool (*ConvertResult)(int64_t&);
735 };
736 
737 }
738 
739 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
740   if (Is == IS_VGPR) {
741     switch (RegWidth) {
742       default: return -1;
743       case 1: return AMDGPU::VGPR_32RegClassID;
744       case 2: return AMDGPU::VReg_64RegClassID;
745       case 3: return AMDGPU::VReg_96RegClassID;
746       case 4: return AMDGPU::VReg_128RegClassID;
747       case 8: return AMDGPU::VReg_256RegClassID;
748       case 16: return AMDGPU::VReg_512RegClassID;
749     }
750   } else if (Is == IS_TTMP) {
751     switch (RegWidth) {
752       default: return -1;
753       case 1: return AMDGPU::TTMP_32RegClassID;
754       case 2: return AMDGPU::TTMP_64RegClassID;
755       case 4: return AMDGPU::TTMP_128RegClassID;
756     }
757   } else if (Is == IS_SGPR) {
758     switch (RegWidth) {
759       default: return -1;
760       case 1: return AMDGPU::SGPR_32RegClassID;
761       case 2: return AMDGPU::SGPR_64RegClassID;
762       case 4: return AMDGPU::SGPR_128RegClassID;
763       case 8: return AMDGPU::SReg_256RegClassID;
764       case 16: return AMDGPU::SReg_512RegClassID;
765     }
766   }
767   return -1;
768 }
769 
770 static unsigned getSpecialRegForName(StringRef RegName) {
771   return StringSwitch<unsigned>(RegName)
772     .Case("exec", AMDGPU::EXEC)
773     .Case("vcc", AMDGPU::VCC)
774     .Case("flat_scratch", AMDGPU::FLAT_SCR)
775     .Case("m0", AMDGPU::M0)
776     .Case("scc", AMDGPU::SCC)
777     .Case("tba", AMDGPU::TBA)
778     .Case("tma", AMDGPU::TMA)
779     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
780     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
781     .Case("vcc_lo", AMDGPU::VCC_LO)
782     .Case("vcc_hi", AMDGPU::VCC_HI)
783     .Case("exec_lo", AMDGPU::EXEC_LO)
784     .Case("exec_hi", AMDGPU::EXEC_HI)
785     .Case("tma_lo", AMDGPU::TMA_LO)
786     .Case("tma_hi", AMDGPU::TMA_HI)
787     .Case("tba_lo", AMDGPU::TBA_LO)
788     .Case("tba_hi", AMDGPU::TBA_HI)
789     .Default(0);
790 }
791 
792 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) {
793   auto R = parseRegister();
794   if (!R) return true;
795   assert(R->isReg());
796   RegNo = R->getReg();
797   StartLoc = R->getStartLoc();
798   EndLoc = R->getEndLoc();
799   return false;
800 }
801 
802 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum)
803 {
804   switch (RegKind) {
805   case IS_SPECIAL:
806     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) { Reg = AMDGPU::EXEC; RegWidth = 2; return true; }
807     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) { Reg = AMDGPU::FLAT_SCR; RegWidth = 2; return true; }
808     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) { Reg = AMDGPU::VCC; RegWidth = 2; return true; }
809     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) { Reg = AMDGPU::TBA; RegWidth = 2; return true; }
810     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) { Reg = AMDGPU::TMA; RegWidth = 2; return true; }
811     return false;
812   case IS_VGPR:
813   case IS_SGPR:
814   case IS_TTMP:
815     if (Reg1 != Reg + RegWidth) { return false; }
816     RegWidth++;
817     return true;
818   default:
819     assert(false); return false;
820   }
821 }
822 
823 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth)
824 {
825   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
826   if (getLexer().is(AsmToken::Identifier)) {
827     StringRef RegName = Parser.getTok().getString();
828     if ((Reg = getSpecialRegForName(RegName))) {
829       Parser.Lex();
830       RegKind = IS_SPECIAL;
831     } else {
832       unsigned RegNumIndex = 0;
833       if (RegName[0] == 'v') {
834         RegNumIndex = 1;
835         RegKind = IS_VGPR;
836       } else if (RegName[0] == 's') {
837         RegNumIndex = 1;
838         RegKind = IS_SGPR;
839       } else if (RegName.startswith("ttmp")) {
840         RegNumIndex = strlen("ttmp");
841         RegKind = IS_TTMP;
842       } else {
843         return false;
844       }
845       if (RegName.size() > RegNumIndex) {
846         // Single 32-bit register: vXX.
847         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
848           return false;
849         Parser.Lex();
850         RegWidth = 1;
851       } else {
852         // Range of registers: v[XX:YY]. ":YY" is optional.
853         Parser.Lex();
854         int64_t RegLo, RegHi;
855         if (getLexer().isNot(AsmToken::LBrac))
856           return false;
857         Parser.Lex();
858 
859         if (getParser().parseAbsoluteExpression(RegLo))
860           return false;
861 
862         const bool isRBrace = getLexer().is(AsmToken::RBrac);
863         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
864           return false;
865         Parser.Lex();
866 
867         if (isRBrace) {
868           RegHi = RegLo;
869         } else {
870           if (getParser().parseAbsoluteExpression(RegHi))
871             return false;
872 
873           if (getLexer().isNot(AsmToken::RBrac))
874             return false;
875           Parser.Lex();
876         }
877         RegNum = (unsigned) RegLo;
878         RegWidth = (RegHi - RegLo) + 1;
879       }
880     }
881   } else if (getLexer().is(AsmToken::LBrac)) {
882     // List of consecutive registers: [s0,s1,s2,s3]
883     Parser.Lex();
884     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth))
885       return false;
886     if (RegWidth != 1)
887       return false;
888     RegisterKind RegKind1;
889     unsigned Reg1, RegNum1, RegWidth1;
890     do {
891       if (getLexer().is(AsmToken::Comma)) {
892         Parser.Lex();
893       } else if (getLexer().is(AsmToken::RBrac)) {
894         Parser.Lex();
895         break;
896       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1)) {
897         if (RegWidth1 != 1) {
898           return false;
899         }
900         if (RegKind1 != RegKind) {
901           return false;
902         }
903         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
904           return false;
905         }
906       } else {
907         return false;
908       }
909     } while (true);
910   } else {
911     return false;
912   }
913   switch (RegKind) {
914   case IS_SPECIAL:
915     RegNum = 0;
916     RegWidth = 1;
917     break;
918   case IS_VGPR:
919   case IS_SGPR:
920   case IS_TTMP:
921   {
922     unsigned Size = 1;
923     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
924       // SGPR and TTMP registers must be are aligned. Max required alignment is 4 dwords.
925       Size = std::min(RegWidth, 4u);
926     }
927     if (RegNum % Size != 0)
928       return false;
929     RegNum = RegNum / Size;
930     int RCID = getRegClass(RegKind, RegWidth);
931     if (RCID == -1)
932       return false;
933     const MCRegisterClass RC = TRI->getRegClass(RCID);
934     if (RegNum >= RC.getNumRegs())
935       return false;
936     Reg = RC.getRegister(RegNum);
937     break;
938   }
939 
940   default:
941     assert(false); return false;
942   }
943 
944   if (!subtargetHasRegister(*TRI, Reg))
945     return false;
946   return true;
947 }
948 
949 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
950   const auto &Tok = Parser.getTok();
951   SMLoc StartLoc = Tok.getLoc();
952   SMLoc EndLoc = Tok.getEndLoc();
953   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
954 
955   RegisterKind RegKind;
956   unsigned Reg, RegNum, RegWidth;
957 
958   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth)) {
959     return nullptr;
960   }
961   return AMDGPUOperand::CreateReg(Reg, StartLoc, EndLoc,
962                                   TRI, &getSTI(), false);
963 }
964 
965 AMDGPUAsmParser::OperandMatchResultTy
966 AMDGPUAsmParser::parseImm(OperandVector &Operands) {
967   bool Minus = false;
968   if (getLexer().getKind() == AsmToken::Minus) {
969     Minus = true;
970     Parser.Lex();
971   }
972 
973   SMLoc S = Parser.getTok().getLoc();
974   switch(getLexer().getKind()) {
975   case AsmToken::Integer: {
976     int64_t IntVal;
977     if (getParser().parseAbsoluteExpression(IntVal))
978       return MatchOperand_ParseFail;
979     if (!isInt<32>(IntVal) && !isUInt<32>(IntVal)) {
980       Error(S, "invalid immediate: only 32-bit values are legal");
981       return MatchOperand_ParseFail;
982     }
983 
984     if (Minus)
985       IntVal *= -1;
986     Operands.push_back(AMDGPUOperand::CreateImm(IntVal, S));
987     return MatchOperand_Success;
988   }
989   case AsmToken::Real: {
990     // FIXME: We should emit an error if a double precisions floating-point
991     // value is used.  I'm not sure the best way to detect this.
992     int64_t IntVal;
993     if (getParser().parseAbsoluteExpression(IntVal))
994       return MatchOperand_ParseFail;
995 
996     APFloat F((float)BitsToDouble(IntVal));
997     if (Minus)
998       F.changeSign();
999     Operands.push_back(
1000         AMDGPUOperand::CreateImm(F.bitcastToAPInt().getZExtValue(), S,
1001                                  AMDGPUOperand::ImmTyNone, true));
1002     return MatchOperand_Success;
1003   }
1004   default:
1005     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1006   }
1007 }
1008 
1009 AMDGPUAsmParser::OperandMatchResultTy
1010 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands) {
1011   auto res = parseImm(Operands);
1012   if (res != MatchOperand_NoMatch) {
1013     return res;
1014   }
1015 
1016   if (auto R = parseRegister()) {
1017     assert(R->isReg());
1018     R->Reg.IsForcedVOP3 = isForcedVOP3();
1019     Operands.push_back(std::move(R));
1020     return MatchOperand_Success;
1021   }
1022   return MatchOperand_ParseFail;
1023 }
1024 
1025 AMDGPUAsmParser::OperandMatchResultTy
1026 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands) {
1027   // XXX: During parsing we can't determine if minus sign means
1028   // negate-modifier or negative immediate value.
1029   // By default we suppose it is modifier.
1030   bool Negate = false, Abs = false, Abs2 = false;
1031 
1032   if (getLexer().getKind()== AsmToken::Minus) {
1033     Parser.Lex();
1034     Negate = true;
1035   }
1036 
1037   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "abs") {
1038     Parser.Lex();
1039     Abs2 = true;
1040     if (getLexer().isNot(AsmToken::LParen)) {
1041       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1042       return MatchOperand_ParseFail;
1043     }
1044     Parser.Lex();
1045   }
1046 
1047   if (getLexer().getKind() == AsmToken::Pipe) {
1048     if (Abs2) {
1049       Error(Parser.getTok().getLoc(), "expected register or immediate");
1050       return MatchOperand_ParseFail;
1051     }
1052     Parser.Lex();
1053     Abs = true;
1054   }
1055 
1056   auto Res = parseRegOrImm(Operands);
1057   if (Res != MatchOperand_Success) {
1058     return Res;
1059   }
1060 
1061   AMDGPUOperand::Modifiers Mods = {false, false, false};
1062   if (Negate) {
1063     Mods.Neg = true;
1064   }
1065   if (Abs) {
1066     if (getLexer().getKind() != AsmToken::Pipe) {
1067       Error(Parser.getTok().getLoc(), "expected vertical bar");
1068       return MatchOperand_ParseFail;
1069     }
1070     Parser.Lex();
1071     Mods.Abs = true;
1072   }
1073   if (Abs2) {
1074     if (getLexer().isNot(AsmToken::RParen)) {
1075       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1076       return MatchOperand_ParseFail;
1077     }
1078     Parser.Lex();
1079     Mods.Abs = true;
1080   }
1081 
1082   if (Mods.hasFPModifiers()) {
1083     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1084     Op.setModifiers(Mods);
1085   }
1086   return MatchOperand_Success;
1087 }
1088 
1089 AMDGPUAsmParser::OperandMatchResultTy
1090 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands) {
1091   bool Sext = false;
1092 
1093   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "sext") {
1094     Parser.Lex();
1095     Sext = true;
1096     if (getLexer().isNot(AsmToken::LParen)) {
1097       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1098       return MatchOperand_ParseFail;
1099     }
1100     Parser.Lex();
1101   }
1102 
1103   auto Res = parseRegOrImm(Operands);
1104   if (Res != MatchOperand_Success) {
1105     return Res;
1106   }
1107 
1108   AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1109   if (Op.isImm() && Op.Imm.IsFPImm) {
1110     Error(Parser.getTok().getLoc(), "floating point operands not allowed with sext() modifier");
1111     return MatchOperand_ParseFail;
1112   }
1113 
1114   AMDGPUOperand::Modifiers Mods = {false, false, false};
1115   if (Sext) {
1116     if (getLexer().isNot(AsmToken::RParen)) {
1117       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1118       return MatchOperand_ParseFail;
1119     }
1120     Parser.Lex();
1121     Mods.Sext = true;
1122   }
1123 
1124   if (Mods.hasIntModifiers()) {
1125     Op.setModifiers(Mods);
1126   }
1127   return MatchOperand_Success;
1128 }
1129 
1130 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1131 
1132   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1133 
1134   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1135       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1136       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1137       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1138     return Match_InvalidOperand;
1139 
1140   if ((TSFlags & SIInstrFlags::VOP3) &&
1141       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1142       getForcedEncodingSize() != 64)
1143     return Match_PreferE32;
1144 
1145   return Match_Success;
1146 }
1147 
1148 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1149                                               OperandVector &Operands,
1150                                               MCStreamer &Out,
1151                                               uint64_t &ErrorInfo,
1152                                               bool MatchingInlineAsm) {
1153   MCInst Inst;
1154 
1155   switch (MatchInstructionImpl(Operands, Inst, ErrorInfo, MatchingInlineAsm)) {
1156     default: break;
1157     case Match_Success:
1158       Inst.setLoc(IDLoc);
1159       Out.EmitInstruction(Inst, getSTI());
1160       return false;
1161     case Match_MissingFeature:
1162       return Error(IDLoc, "instruction not supported on this GPU");
1163 
1164     case Match_MnemonicFail:
1165       return Error(IDLoc, "unrecognized instruction mnemonic");
1166 
1167     case Match_InvalidOperand: {
1168       SMLoc ErrorLoc = IDLoc;
1169       if (ErrorInfo != ~0ULL) {
1170         if (ErrorInfo >= Operands.size()) {
1171           return Error(IDLoc, "too few operands for instruction");
1172         }
1173         ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
1174         if (ErrorLoc == SMLoc())
1175           ErrorLoc = IDLoc;
1176       }
1177       return Error(ErrorLoc, "invalid operand for instruction");
1178     }
1179     case Match_PreferE32:
1180       return Error(IDLoc, "internal error: instruction without _e64 suffix "
1181                           "should be encoded as e32");
1182   }
1183   llvm_unreachable("Implement any new match types added!");
1184 }
1185 
1186 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
1187                                                uint32_t &Minor) {
1188   if (getLexer().isNot(AsmToken::Integer))
1189     return TokError("invalid major version");
1190 
1191   Major = getLexer().getTok().getIntVal();
1192   Lex();
1193 
1194   if (getLexer().isNot(AsmToken::Comma))
1195     return TokError("minor version number required, comma expected");
1196   Lex();
1197 
1198   if (getLexer().isNot(AsmToken::Integer))
1199     return TokError("invalid minor version");
1200 
1201   Minor = getLexer().getTok().getIntVal();
1202   Lex();
1203 
1204   return false;
1205 }
1206 
1207 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
1208 
1209   uint32_t Major;
1210   uint32_t Minor;
1211 
1212   if (ParseDirectiveMajorMinor(Major, Minor))
1213     return true;
1214 
1215   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
1216   return false;
1217 }
1218 
1219 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
1220 
1221   uint32_t Major;
1222   uint32_t Minor;
1223   uint32_t Stepping;
1224   StringRef VendorName;
1225   StringRef ArchName;
1226 
1227   // If this directive has no arguments, then use the ISA version for the
1228   // targeted GPU.
1229   if (getLexer().is(AsmToken::EndOfStatement)) {
1230     AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
1231     getTargetStreamer().EmitDirectiveHSACodeObjectISA(Isa.Major, Isa.Minor,
1232                                                       Isa.Stepping,
1233                                                       "AMD", "AMDGPU");
1234     return false;
1235   }
1236 
1237 
1238   if (ParseDirectiveMajorMinor(Major, Minor))
1239     return true;
1240 
1241   if (getLexer().isNot(AsmToken::Comma))
1242     return TokError("stepping version number required, comma expected");
1243   Lex();
1244 
1245   if (getLexer().isNot(AsmToken::Integer))
1246     return TokError("invalid stepping version");
1247 
1248   Stepping = getLexer().getTok().getIntVal();
1249   Lex();
1250 
1251   if (getLexer().isNot(AsmToken::Comma))
1252     return TokError("vendor name required, comma expected");
1253   Lex();
1254 
1255   if (getLexer().isNot(AsmToken::String))
1256     return TokError("invalid vendor name");
1257 
1258   VendorName = getLexer().getTok().getStringContents();
1259   Lex();
1260 
1261   if (getLexer().isNot(AsmToken::Comma))
1262     return TokError("arch name required, comma expected");
1263   Lex();
1264 
1265   if (getLexer().isNot(AsmToken::String))
1266     return TokError("invalid arch name");
1267 
1268   ArchName = getLexer().getTok().getStringContents();
1269   Lex();
1270 
1271   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
1272                                                     VendorName, ArchName);
1273   return false;
1274 }
1275 
1276 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
1277                                                amd_kernel_code_t &Header) {
1278   SmallString<40> ErrStr;
1279   raw_svector_ostream Err(ErrStr);
1280   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
1281     return TokError(Err.str());
1282   }
1283   Lex();
1284   return false;
1285 }
1286 
1287 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
1288 
1289   amd_kernel_code_t Header;
1290   AMDGPU::initDefaultAMDKernelCodeT(Header, getSTI().getFeatureBits());
1291 
1292   while (true) {
1293 
1294     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
1295     // will set the current token to EndOfStatement.
1296     while(getLexer().is(AsmToken::EndOfStatement))
1297       Lex();
1298 
1299     if (getLexer().isNot(AsmToken::Identifier))
1300       return TokError("expected value identifier or .end_amd_kernel_code_t");
1301 
1302     StringRef ID = getLexer().getTok().getIdentifier();
1303     Lex();
1304 
1305     if (ID == ".end_amd_kernel_code_t")
1306       break;
1307 
1308     if (ParseAMDKernelCodeTValue(ID, Header))
1309       return true;
1310   }
1311 
1312   getTargetStreamer().EmitAMDKernelCodeT(Header);
1313 
1314   return false;
1315 }
1316 
1317 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
1318   getParser().getStreamer().SwitchSection(
1319       AMDGPU::getHSATextSection(getContext()));
1320   return false;
1321 }
1322 
1323 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
1324   if (getLexer().isNot(AsmToken::Identifier))
1325     return TokError("expected symbol name");
1326 
1327   StringRef KernelName = Parser.getTok().getString();
1328 
1329   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
1330                                            ELF::STT_AMDGPU_HSA_KERNEL);
1331   Lex();
1332   return false;
1333 }
1334 
1335 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
1336   if (getLexer().isNot(AsmToken::Identifier))
1337     return TokError("expected symbol name");
1338 
1339   StringRef GlobalName = Parser.getTok().getIdentifier();
1340 
1341   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
1342   Lex();
1343   return false;
1344 }
1345 
1346 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
1347   if (getLexer().isNot(AsmToken::Identifier))
1348     return TokError("expected symbol name");
1349 
1350   StringRef GlobalName = Parser.getTok().getIdentifier();
1351 
1352   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
1353   Lex();
1354   return false;
1355 }
1356 
1357 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
1358   getParser().getStreamer().SwitchSection(
1359       AMDGPU::getHSADataGlobalAgentSection(getContext()));
1360   return false;
1361 }
1362 
1363 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
1364   getParser().getStreamer().SwitchSection(
1365       AMDGPU::getHSADataGlobalProgramSection(getContext()));
1366   return false;
1367 }
1368 
1369 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
1370   getParser().getStreamer().SwitchSection(
1371       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
1372   return false;
1373 }
1374 
1375 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
1376   StringRef IDVal = DirectiveID.getString();
1377 
1378   if (IDVal == ".hsa_code_object_version")
1379     return ParseDirectiveHSACodeObjectVersion();
1380 
1381   if (IDVal == ".hsa_code_object_isa")
1382     return ParseDirectiveHSACodeObjectISA();
1383 
1384   if (IDVal == ".amd_kernel_code_t")
1385     return ParseDirectiveAMDKernelCodeT();
1386 
1387   if (IDVal == ".hsatext")
1388     return ParseSectionDirectiveHSAText();
1389 
1390   if (IDVal == ".amdgpu_hsa_kernel")
1391     return ParseDirectiveAMDGPUHsaKernel();
1392 
1393   if (IDVal == ".amdgpu_hsa_module_global")
1394     return ParseDirectiveAMDGPUHsaModuleGlobal();
1395 
1396   if (IDVal == ".amdgpu_hsa_program_global")
1397     return ParseDirectiveAMDGPUHsaProgramGlobal();
1398 
1399   if (IDVal == ".hsadata_global_agent")
1400     return ParseSectionDirectiveHSADataGlobalAgent();
1401 
1402   if (IDVal == ".hsadata_global_program")
1403     return ParseSectionDirectiveHSADataGlobalProgram();
1404 
1405   if (IDVal == ".hsarodata_readonly_agent")
1406     return ParseSectionDirectiveHSARodataReadonlyAgent();
1407 
1408   return true;
1409 }
1410 
1411 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
1412                                            unsigned RegNo) const {
1413   if (isCI())
1414     return true;
1415 
1416   if (isSI()) {
1417     // No flat_scr
1418     switch (RegNo) {
1419     case AMDGPU::FLAT_SCR:
1420     case AMDGPU::FLAT_SCR_LO:
1421     case AMDGPU::FLAT_SCR_HI:
1422       return false;
1423     default:
1424       return true;
1425     }
1426   }
1427 
1428   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
1429   // SI/CI have.
1430   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
1431        R.isValid(); ++R) {
1432     if (*R == RegNo)
1433       return false;
1434   }
1435 
1436   return true;
1437 }
1438 
1439 AMDGPUAsmParser::OperandMatchResultTy
1440 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
1441 
1442   // Try to parse with a custom parser
1443   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
1444 
1445   // If we successfully parsed the operand or if there as an error parsing,
1446   // we are done.
1447   //
1448   // If we are parsing after we reach EndOfStatement then this means we
1449   // are appending default values to the Operands list.  This is only done
1450   // by custom parser, so we shouldn't continue on to the generic parsing.
1451   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
1452       getLexer().is(AsmToken::EndOfStatement))
1453     return ResTy;
1454 
1455   ResTy = parseRegOrImm(Operands);
1456 
1457   if (ResTy == MatchOperand_Success)
1458     return ResTy;
1459 
1460   if (getLexer().getKind() == AsmToken::Identifier) {
1461     // If this identifier is a symbol, we want to create an expression for it.
1462     // It is a little difficult to distinguish between a symbol name, and
1463     // an instruction flag like 'gds'.  In order to do this, we parse
1464     // all tokens as expressions and then treate the symbol name as the token
1465     // string when we want to interpret the operand as a token.
1466     const auto &Tok = Parser.getTok();
1467     SMLoc S = Tok.getLoc();
1468     const MCExpr *Expr = nullptr;
1469     if (!Parser.parseExpression(Expr)) {
1470       Operands.push_back(AMDGPUOperand::CreateExpr(Expr, S));
1471       return MatchOperand_Success;
1472     }
1473 
1474     Operands.push_back(AMDGPUOperand::CreateToken(Tok.getString(), Tok.getLoc()));
1475     Parser.Lex();
1476     return MatchOperand_Success;
1477   }
1478   return MatchOperand_NoMatch;
1479 }
1480 
1481 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
1482   // Clear any forced encodings from the previous instruction.
1483   setForcedEncodingSize(0);
1484   setForcedDPP(false);
1485   setForcedSDWA(false);
1486 
1487   if (Name.endswith("_e64")) {
1488     setForcedEncodingSize(64);
1489     return Name.substr(0, Name.size() - 4);
1490   } else if (Name.endswith("_e32")) {
1491     setForcedEncodingSize(32);
1492     return Name.substr(0, Name.size() - 4);
1493   } else if (Name.endswith("_dpp")) {
1494     setForcedDPP(true);
1495     return Name.substr(0, Name.size() - 4);
1496   } else if (Name.endswith("_sdwa")) {
1497     setForcedSDWA(true);
1498     return Name.substr(0, Name.size() - 5);
1499   }
1500   return Name;
1501 }
1502 
1503 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
1504                                        StringRef Name,
1505                                        SMLoc NameLoc, OperandVector &Operands) {
1506   // Add the instruction mnemonic
1507   Name = parseMnemonicSuffix(Name);
1508   Operands.push_back(AMDGPUOperand::CreateToken(Name, NameLoc));
1509 
1510   while (!getLexer().is(AsmToken::EndOfStatement)) {
1511     AMDGPUAsmParser::OperandMatchResultTy Res = parseOperand(Operands, Name);
1512 
1513     // Eat the comma or space if there is one.
1514     if (getLexer().is(AsmToken::Comma))
1515       Parser.Lex();
1516 
1517     switch (Res) {
1518       case MatchOperand_Success: break;
1519       case MatchOperand_ParseFail:
1520         Error(getLexer().getLoc(), "failed parsing operand.");
1521         while (!getLexer().is(AsmToken::EndOfStatement)) {
1522           Parser.Lex();
1523         }
1524         return true;
1525       case MatchOperand_NoMatch:
1526         Error(getLexer().getLoc(), "not a valid operand.");
1527         while (!getLexer().is(AsmToken::EndOfStatement)) {
1528           Parser.Lex();
1529         }
1530         return true;
1531     }
1532   }
1533 
1534   return false;
1535 }
1536 
1537 //===----------------------------------------------------------------------===//
1538 // Utility functions
1539 //===----------------------------------------------------------------------===//
1540 
1541 AMDGPUAsmParser::OperandMatchResultTy
1542 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
1543   switch(getLexer().getKind()) {
1544     default: return MatchOperand_NoMatch;
1545     case AsmToken::Identifier: {
1546       StringRef Name = Parser.getTok().getString();
1547       if (!Name.equals(Prefix)) {
1548         return MatchOperand_NoMatch;
1549       }
1550 
1551       Parser.Lex();
1552       if (getLexer().isNot(AsmToken::Colon))
1553         return MatchOperand_ParseFail;
1554 
1555       Parser.Lex();
1556       if (getLexer().isNot(AsmToken::Integer))
1557         return MatchOperand_ParseFail;
1558 
1559       if (getParser().parseAbsoluteExpression(Int))
1560         return MatchOperand_ParseFail;
1561       break;
1562     }
1563   }
1564   return MatchOperand_Success;
1565 }
1566 
1567 AMDGPUAsmParser::OperandMatchResultTy
1568 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
1569                                     enum AMDGPUOperand::ImmTy ImmTy,
1570                                     bool (*ConvertResult)(int64_t&)) {
1571 
1572   SMLoc S = Parser.getTok().getLoc();
1573   int64_t Value = 0;
1574 
1575   AMDGPUAsmParser::OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
1576   if (Res != MatchOperand_Success)
1577     return Res;
1578 
1579   if (ConvertResult && !ConvertResult(Value)) {
1580     return MatchOperand_ParseFail;
1581   }
1582 
1583   Operands.push_back(AMDGPUOperand::CreateImm(Value, S, ImmTy));
1584   return MatchOperand_Success;
1585 }
1586 
1587 AMDGPUAsmParser::OperandMatchResultTy
1588 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
1589                                enum AMDGPUOperand::ImmTy ImmTy) {
1590   int64_t Bit = 0;
1591   SMLoc S = Parser.getTok().getLoc();
1592 
1593   // We are at the end of the statement, and this is a default argument, so
1594   // use a default value.
1595   if (getLexer().isNot(AsmToken::EndOfStatement)) {
1596     switch(getLexer().getKind()) {
1597       case AsmToken::Identifier: {
1598         StringRef Tok = Parser.getTok().getString();
1599         if (Tok == Name) {
1600           Bit = 1;
1601           Parser.Lex();
1602         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
1603           Bit = 0;
1604           Parser.Lex();
1605         } else {
1606           return MatchOperand_NoMatch;
1607         }
1608         break;
1609       }
1610       default:
1611         return MatchOperand_NoMatch;
1612     }
1613   }
1614 
1615   Operands.push_back(AMDGPUOperand::CreateImm(Bit, S, ImmTy));
1616   return MatchOperand_Success;
1617 }
1618 
1619 typedef std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
1620 
1621 void addOptionalImmOperand(MCInst& Inst, const OperandVector& Operands,
1622                            OptionalImmIndexMap& OptionalIdx,
1623                            enum AMDGPUOperand::ImmTy ImmT, int64_t Default = 0) {
1624   auto i = OptionalIdx.find(ImmT);
1625   if (i != OptionalIdx.end()) {
1626     unsigned Idx = i->second;
1627     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
1628   } else {
1629     Inst.addOperand(MCOperand::createImm(Default));
1630   }
1631 }
1632 
1633 AMDGPUAsmParser::OperandMatchResultTy
1634 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
1635   if (getLexer().isNot(AsmToken::Identifier)) {
1636     return MatchOperand_NoMatch;
1637   }
1638   StringRef Tok = Parser.getTok().getString();
1639   if (Tok != Prefix) {
1640     return MatchOperand_NoMatch;
1641   }
1642 
1643   Parser.Lex();
1644   if (getLexer().isNot(AsmToken::Colon)) {
1645     return MatchOperand_ParseFail;
1646   }
1647 
1648   Parser.Lex();
1649   if (getLexer().isNot(AsmToken::Identifier)) {
1650     return MatchOperand_ParseFail;
1651   }
1652 
1653   Value = Parser.getTok().getString();
1654   return MatchOperand_Success;
1655 }
1656 
1657 //===----------------------------------------------------------------------===//
1658 // ds
1659 //===----------------------------------------------------------------------===//
1660 
1661 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
1662                                     const OperandVector &Operands) {
1663 
1664   OptionalImmIndexMap OptionalIdx;
1665 
1666   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1667     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1668 
1669     // Add the register arguments
1670     if (Op.isReg()) {
1671       Op.addRegOperands(Inst, 1);
1672       continue;
1673     }
1674 
1675     // Handle optional arguments
1676     OptionalIdx[Op.getImmTy()] = i;
1677   }
1678 
1679   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
1680   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
1681   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1682 
1683   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1684 }
1685 
1686 void AMDGPUAsmParser::cvtDS(MCInst &Inst, const OperandVector &Operands) {
1687 
1688   std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
1689   bool GDSOnly = false;
1690 
1691   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1692     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1693 
1694     // Add the register arguments
1695     if (Op.isReg()) {
1696       Op.addRegOperands(Inst, 1);
1697       continue;
1698     }
1699 
1700     if (Op.isToken() && Op.getToken() == "gds") {
1701       GDSOnly = true;
1702       continue;
1703     }
1704 
1705     // Handle optional arguments
1706     OptionalIdx[Op.getImmTy()] = i;
1707   }
1708 
1709   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
1710   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1711 
1712   if (!GDSOnly) {
1713     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1714   }
1715   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1716 }
1717 
1718 
1719 //===----------------------------------------------------------------------===//
1720 // s_waitcnt
1721 //===----------------------------------------------------------------------===//
1722 
1723 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
1724   StringRef CntName = Parser.getTok().getString();
1725   int64_t CntVal;
1726 
1727   Parser.Lex();
1728   if (getLexer().isNot(AsmToken::LParen))
1729     return true;
1730 
1731   Parser.Lex();
1732   if (getLexer().isNot(AsmToken::Integer))
1733     return true;
1734 
1735   if (getParser().parseAbsoluteExpression(CntVal))
1736     return true;
1737 
1738   if (getLexer().isNot(AsmToken::RParen))
1739     return true;
1740 
1741   Parser.Lex();
1742   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
1743     Parser.Lex();
1744 
1745   int CntShift;
1746   int CntMask;
1747 
1748   if (CntName == "vmcnt") {
1749     CntMask = 0xf;
1750     CntShift = 0;
1751   } else if (CntName == "expcnt") {
1752     CntMask = 0x7;
1753     CntShift = 4;
1754   } else if (CntName == "lgkmcnt") {
1755     CntMask = 0xf;
1756     CntShift = 8;
1757   } else {
1758     return true;
1759   }
1760 
1761   IntVal &= ~(CntMask << CntShift);
1762   IntVal |= (CntVal << CntShift);
1763   return false;
1764 }
1765 
1766 AMDGPUAsmParser::OperandMatchResultTy
1767 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
1768   // Disable all counters by default.
1769   // vmcnt   [3:0]
1770   // expcnt  [6:4]
1771   // lgkmcnt [11:8]
1772   int64_t CntVal = 0xf7f;
1773   SMLoc S = Parser.getTok().getLoc();
1774 
1775   switch(getLexer().getKind()) {
1776     default: return MatchOperand_ParseFail;
1777     case AsmToken::Integer:
1778       // The operand can be an integer value.
1779       if (getParser().parseAbsoluteExpression(CntVal))
1780         return MatchOperand_ParseFail;
1781       break;
1782 
1783     case AsmToken::Identifier:
1784       do {
1785         if (parseCnt(CntVal))
1786           return MatchOperand_ParseFail;
1787       } while(getLexer().isNot(AsmToken::EndOfStatement));
1788       break;
1789   }
1790   Operands.push_back(AMDGPUOperand::CreateImm(CntVal, S));
1791   return MatchOperand_Success;
1792 }
1793 
1794 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width) {
1795   using namespace llvm::AMDGPU::Hwreg;
1796 
1797   if (Parser.getTok().getString() != "hwreg")
1798     return true;
1799   Parser.Lex();
1800 
1801   if (getLexer().isNot(AsmToken::LParen))
1802     return true;
1803   Parser.Lex();
1804 
1805   if (getLexer().is(AsmToken::Identifier)) {
1806     HwReg.IsSymbolic = true;
1807     HwReg.Id = ID_UNKNOWN_;
1808     const StringRef tok = Parser.getTok().getString();
1809     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
1810       if (tok == IdSymbolic[i]) {
1811         HwReg.Id = i;
1812         break;
1813       }
1814     }
1815     Parser.Lex();
1816   } else {
1817     HwReg.IsSymbolic = false;
1818     if (getLexer().isNot(AsmToken::Integer))
1819       return true;
1820     if (getParser().parseAbsoluteExpression(HwReg.Id))
1821       return true;
1822   }
1823 
1824   if (getLexer().is(AsmToken::RParen)) {
1825     Parser.Lex();
1826     return false;
1827   }
1828 
1829   // optional params
1830   if (getLexer().isNot(AsmToken::Comma))
1831     return true;
1832   Parser.Lex();
1833 
1834   if (getLexer().isNot(AsmToken::Integer))
1835     return true;
1836   if (getParser().parseAbsoluteExpression(Offset))
1837     return true;
1838 
1839   if (getLexer().isNot(AsmToken::Comma))
1840     return true;
1841   Parser.Lex();
1842 
1843   if (getLexer().isNot(AsmToken::Integer))
1844     return true;
1845   if (getParser().parseAbsoluteExpression(Width))
1846     return true;
1847 
1848   if (getLexer().isNot(AsmToken::RParen))
1849     return true;
1850   Parser.Lex();
1851 
1852   return false;
1853 }
1854 
1855 AMDGPUAsmParser::OperandMatchResultTy
1856 AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
1857   using namespace llvm::AMDGPU::Hwreg;
1858 
1859   int64_t Imm16Val = 0;
1860   SMLoc S = Parser.getTok().getLoc();
1861 
1862   switch(getLexer().getKind()) {
1863     default: return MatchOperand_NoMatch;
1864     case AsmToken::Integer:
1865       // The operand can be an integer value.
1866       if (getParser().parseAbsoluteExpression(Imm16Val))
1867         return MatchOperand_NoMatch;
1868       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
1869         Error(S, "invalid immediate: only 16-bit values are legal");
1870         // Do not return error code, but create an imm operand anyway and proceed
1871         // to the next operand, if any. That avoids unneccessary error messages.
1872       }
1873       break;
1874 
1875     case AsmToken::Identifier: {
1876         OperandInfoTy HwReg(ID_UNKNOWN_);
1877         int64_t Offset = OFFSET_DEFAULT_;
1878         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
1879         if (parseHwregConstruct(HwReg, Offset, Width))
1880           return MatchOperand_ParseFail;
1881         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
1882           if (HwReg.IsSymbolic)
1883             Error(S, "invalid symbolic name of hardware register");
1884           else
1885             Error(S, "invalid code of hardware register: only 6-bit values are legal");
1886         }
1887         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
1888           Error(S, "invalid bit offset: only 5-bit values are legal");
1889         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
1890           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
1891         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
1892       }
1893       break;
1894   }
1895   Operands.push_back(AMDGPUOperand::CreateImm(Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
1896   return MatchOperand_Success;
1897 }
1898 
1899 bool AMDGPUOperand::isSWaitCnt() const {
1900   return isImm();
1901 }
1902 
1903 bool AMDGPUOperand::isHwreg() const {
1904   return isImmTy(ImmTyHwreg);
1905 }
1906 
1907 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
1908   using namespace llvm::AMDGPU::SendMsg;
1909 
1910   if (Parser.getTok().getString() != "sendmsg")
1911     return true;
1912   Parser.Lex();
1913 
1914   if (getLexer().isNot(AsmToken::LParen))
1915     return true;
1916   Parser.Lex();
1917 
1918   if (getLexer().is(AsmToken::Identifier)) {
1919     Msg.IsSymbolic = true;
1920     Msg.Id = ID_UNKNOWN_;
1921     const std::string tok = Parser.getTok().getString();
1922     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
1923       switch(i) {
1924         default: continue; // Omit gaps.
1925         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
1926       }
1927       if (tok == IdSymbolic[i]) {
1928         Msg.Id = i;
1929         break;
1930       }
1931     }
1932     Parser.Lex();
1933   } else {
1934     Msg.IsSymbolic = false;
1935     if (getLexer().isNot(AsmToken::Integer))
1936       return true;
1937     if (getParser().parseAbsoluteExpression(Msg.Id))
1938       return true;
1939     if (getLexer().is(AsmToken::Integer))
1940       if (getParser().parseAbsoluteExpression(Msg.Id))
1941         Msg.Id = ID_UNKNOWN_;
1942   }
1943   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
1944     return false;
1945 
1946   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
1947     if (getLexer().isNot(AsmToken::RParen))
1948       return true;
1949     Parser.Lex();
1950     return false;
1951   }
1952 
1953   if (getLexer().isNot(AsmToken::Comma))
1954     return true;
1955   Parser.Lex();
1956 
1957   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
1958   Operation.Id = ID_UNKNOWN_;
1959   if (getLexer().is(AsmToken::Identifier)) {
1960     Operation.IsSymbolic = true;
1961     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
1962     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
1963     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
1964     const StringRef Tok = Parser.getTok().getString();
1965     for (int i = F; i < L; ++i) {
1966       if (Tok == S[i]) {
1967         Operation.Id = i;
1968         break;
1969       }
1970     }
1971     Parser.Lex();
1972   } else {
1973     Operation.IsSymbolic = false;
1974     if (getLexer().isNot(AsmToken::Integer))
1975       return true;
1976     if (getParser().parseAbsoluteExpression(Operation.Id))
1977       return true;
1978   }
1979 
1980   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
1981     // Stream id is optional.
1982     if (getLexer().is(AsmToken::RParen)) {
1983       Parser.Lex();
1984       return false;
1985     }
1986 
1987     if (getLexer().isNot(AsmToken::Comma))
1988       return true;
1989     Parser.Lex();
1990 
1991     if (getLexer().isNot(AsmToken::Integer))
1992       return true;
1993     if (getParser().parseAbsoluteExpression(StreamId))
1994       return true;
1995   }
1996 
1997   if (getLexer().isNot(AsmToken::RParen))
1998     return true;
1999   Parser.Lex();
2000   return false;
2001 }
2002 
2003 AMDGPUAsmParser::OperandMatchResultTy
2004 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
2005   using namespace llvm::AMDGPU::SendMsg;
2006 
2007   int64_t Imm16Val = 0;
2008   SMLoc S = Parser.getTok().getLoc();
2009 
2010   switch(getLexer().getKind()) {
2011   default:
2012     return MatchOperand_NoMatch;
2013   case AsmToken::Integer:
2014     // The operand can be an integer value.
2015     if (getParser().parseAbsoluteExpression(Imm16Val))
2016       return MatchOperand_NoMatch;
2017     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2018       Error(S, "invalid immediate: only 16-bit values are legal");
2019       // Do not return error code, but create an imm operand anyway and proceed
2020       // to the next operand, if any. That avoids unneccessary error messages.
2021     }
2022     break;
2023   case AsmToken::Identifier: {
2024       OperandInfoTy Msg(ID_UNKNOWN_);
2025       OperandInfoTy Operation(OP_UNKNOWN_);
2026       int64_t StreamId = STREAM_ID_DEFAULT_;
2027       if (parseSendMsgConstruct(Msg, Operation, StreamId))
2028         return MatchOperand_ParseFail;
2029       do {
2030         // Validate and encode message ID.
2031         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
2032                 || Msg.Id == ID_SYSMSG)) {
2033           if (Msg.IsSymbolic)
2034             Error(S, "invalid/unsupported symbolic name of message");
2035           else
2036             Error(S, "invalid/unsupported code of message");
2037           break;
2038         }
2039         Imm16Val = (Msg.Id << ID_SHIFT_);
2040         // Validate and encode operation ID.
2041         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
2042           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
2043             if (Operation.IsSymbolic)
2044               Error(S, "invalid symbolic name of GS_OP");
2045             else
2046               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
2047             break;
2048           }
2049           if (Operation.Id == OP_GS_NOP
2050               && Msg.Id != ID_GS_DONE) {
2051             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
2052             break;
2053           }
2054           Imm16Val |= (Operation.Id << OP_SHIFT_);
2055         }
2056         if (Msg.Id == ID_SYSMSG) {
2057           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
2058             if (Operation.IsSymbolic)
2059               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
2060             else
2061               Error(S, "invalid/unsupported code of SYSMSG_OP");
2062             break;
2063           }
2064           Imm16Val |= (Operation.Id << OP_SHIFT_);
2065         }
2066         // Validate and encode stream ID.
2067         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2068           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
2069             Error(S, "invalid stream id: only 2-bit values are legal");
2070             break;
2071           }
2072           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
2073         }
2074       } while (0);
2075     }
2076     break;
2077   }
2078   Operands.push_back(AMDGPUOperand::CreateImm(Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
2079   return MatchOperand_Success;
2080 }
2081 
2082 bool AMDGPUOperand::isSendMsg() const {
2083   return isImmTy(ImmTySendMsg);
2084 }
2085 
2086 //===----------------------------------------------------------------------===//
2087 // sopp branch targets
2088 //===----------------------------------------------------------------------===//
2089 
2090 AMDGPUAsmParser::OperandMatchResultTy
2091 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
2092   SMLoc S = Parser.getTok().getLoc();
2093 
2094   switch (getLexer().getKind()) {
2095     default: return MatchOperand_ParseFail;
2096     case AsmToken::Integer: {
2097       int64_t Imm;
2098       if (getParser().parseAbsoluteExpression(Imm))
2099         return MatchOperand_ParseFail;
2100       Operands.push_back(AMDGPUOperand::CreateImm(Imm, S));
2101       return MatchOperand_Success;
2102     }
2103 
2104     case AsmToken::Identifier:
2105       Operands.push_back(AMDGPUOperand::CreateExpr(
2106           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
2107                                   Parser.getTok().getString()), getContext()), S));
2108       Parser.Lex();
2109       return MatchOperand_Success;
2110   }
2111 }
2112 
2113 //===----------------------------------------------------------------------===//
2114 // mubuf
2115 //===----------------------------------------------------------------------===//
2116 
2117 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
2118   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyGLC);
2119 }
2120 
2121 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
2122   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTySLC);
2123 }
2124 
2125 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
2126   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyTFE);
2127 }
2128 
2129 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
2130                                const OperandVector &Operands,
2131                                bool IsAtomic, bool IsAtomicReturn) {
2132   OptionalImmIndexMap OptionalIdx;
2133   assert(IsAtomicReturn ? IsAtomic : true);
2134 
2135   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2136     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2137 
2138     // Add the register arguments
2139     if (Op.isReg()) {
2140       Op.addRegOperands(Inst, 1);
2141       continue;
2142     }
2143 
2144     // Handle the case where soffset is an immediate
2145     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
2146       Op.addImmOperands(Inst, 1);
2147       continue;
2148     }
2149 
2150     // Handle tokens like 'offen' which are sometimes hard-coded into the
2151     // asm string.  There are no MCInst operands for these.
2152     if (Op.isToken()) {
2153       continue;
2154     }
2155     assert(Op.isImm());
2156 
2157     // Handle optional arguments
2158     OptionalIdx[Op.getImmTy()] = i;
2159   }
2160 
2161   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
2162   if (IsAtomicReturn) {
2163     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
2164     Inst.insert(I, *I);
2165   }
2166 
2167   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2168   if (!IsAtomic) { // glc is hard-coded.
2169     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2170   }
2171   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2172   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2173 }
2174 
2175 //===----------------------------------------------------------------------===//
2176 // mimg
2177 //===----------------------------------------------------------------------===//
2178 
2179 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
2180   unsigned I = 1;
2181   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2182   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2183     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2184   }
2185 
2186   OptionalImmIndexMap OptionalIdx;
2187 
2188   for (unsigned E = Operands.size(); I != E; ++I) {
2189     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2190 
2191     // Add the register arguments
2192     if (Op.isRegOrImm()) {
2193       Op.addRegOrImmOperands(Inst, 1);
2194       continue;
2195     } else if (Op.isImmModifier()) {
2196       OptionalIdx[Op.getImmTy()] = I;
2197     } else {
2198       assert(false);
2199     }
2200   }
2201 
2202   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2203   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2204   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2205   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2206   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2207   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2208   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2209   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2210 }
2211 
2212 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
2213   unsigned I = 1;
2214   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2215   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2216     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2217   }
2218 
2219   // Add src, same as dst
2220   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
2221 
2222   OptionalImmIndexMap OptionalIdx;
2223 
2224   for (unsigned E = Operands.size(); I != E; ++I) {
2225     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2226 
2227     // Add the register arguments
2228     if (Op.isRegOrImm()) {
2229       Op.addRegOrImmOperands(Inst, 1);
2230       continue;
2231     } else if (Op.isImmModifier()) {
2232       OptionalIdx[Op.getImmTy()] = I;
2233     } else {
2234       assert(false);
2235     }
2236   }
2237 
2238   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2239   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2240   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2241   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2242   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2243   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2244   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2245   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2246 }
2247 
2248 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
2249   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyDMask);
2250 }
2251 
2252 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
2253   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
2254 }
2255 
2256 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
2257   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyDA);
2258 }
2259 
2260 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
2261   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyR128);
2262 }
2263 
2264 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
2265   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyLWE);
2266 }
2267 
2268 //===----------------------------------------------------------------------===//
2269 // smrd
2270 //===----------------------------------------------------------------------===//
2271 
2272 bool AMDGPUOperand::isSMRDOffset() const {
2273 
2274   // FIXME: Support 20-bit offsets on VI.  We need to to pass subtarget
2275   // information here.
2276   return isImm() && isUInt<8>(getImm());
2277 }
2278 
2279 bool AMDGPUOperand::isSMRDLiteralOffset() const {
2280   // 32-bit literals are only supported on CI and we only want to use them
2281   // when the offset is > 8-bits.
2282   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
2283 }
2284 
2285 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset() const {
2286   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2287 }
2288 
2289 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
2290   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2291 }
2292 
2293 //===----------------------------------------------------------------------===//
2294 // vop3
2295 //===----------------------------------------------------------------------===//
2296 
2297 static bool ConvertOmodMul(int64_t &Mul) {
2298   if (Mul != 1 && Mul != 2 && Mul != 4)
2299     return false;
2300 
2301   Mul >>= 1;
2302   return true;
2303 }
2304 
2305 static bool ConvertOmodDiv(int64_t &Div) {
2306   if (Div == 1) {
2307     Div = 0;
2308     return true;
2309   }
2310 
2311   if (Div == 2) {
2312     Div = 3;
2313     return true;
2314   }
2315 
2316   return false;
2317 }
2318 
2319 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
2320   if (BoundCtrl == 0) {
2321     BoundCtrl = 1;
2322     return true;
2323   } else if (BoundCtrl == -1) {
2324     BoundCtrl = 0;
2325     return true;
2326   }
2327   return false;
2328 }
2329 
2330 // Note: the order in this table matches the order of operands in AsmString.
2331 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
2332   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
2333   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
2334   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
2335   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
2336   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
2337   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
2338   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
2339   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
2340   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
2341   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
2342   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
2343   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
2344   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
2345   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
2346   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
2347   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
2348   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
2349   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
2350   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
2351   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
2352   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
2353   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
2354   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
2355   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
2356 };
2357 
2358 AMDGPUAsmParser::OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
2359   OperandMatchResultTy res;
2360   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
2361     // try to parse any optional operand here
2362     if (Op.IsBit) {
2363       res = parseNamedBit(Op.Name, Operands, Op.Type);
2364     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
2365       res = parseOModOperand(Operands);
2366     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
2367                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
2368                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
2369       res = parseSDWASel(Operands, Op.Name, Op.Type);
2370     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
2371       res = parseSDWADstUnused(Operands);
2372     } else {
2373       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
2374     }
2375     if (res != MatchOperand_NoMatch) {
2376       return res;
2377     }
2378   }
2379   return MatchOperand_NoMatch;
2380 }
2381 
2382 AMDGPUAsmParser::OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands)
2383 {
2384   StringRef Name = Parser.getTok().getString();
2385   if (Name == "mul") {
2386     return parseIntWithPrefix("mul", Operands, AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
2387   } else if (Name == "div") {
2388     return parseIntWithPrefix("div", Operands, AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
2389   } else {
2390     return MatchOperand_NoMatch;
2391   }
2392 }
2393 
2394 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
2395   unsigned I = 1;
2396   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2397   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2398     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2399   }
2400   for (unsigned E = Operands.size(); I != E; ++I)
2401     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
2402 }
2403 
2404 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
2405   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
2406   if (TSFlags & SIInstrFlags::VOP3) {
2407     cvtVOP3(Inst, Operands);
2408   } else {
2409     cvtId(Inst, Operands);
2410   }
2411 }
2412 
2413 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
2414   OptionalImmIndexMap OptionalIdx;
2415   unsigned I = 1;
2416   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2417   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2418     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2419   }
2420 
2421   for (unsigned E = Operands.size(); I != E; ++I) {
2422     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2423     if (Op.isRegOrImmWithInputMods()) {
2424       // only fp modifiers allowed in VOP3
2425       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
2426     } else if (Op.isImm()) {
2427       OptionalIdx[Op.getImmTy()] = I;
2428     } else {
2429       assert(false);
2430     }
2431   }
2432 
2433   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
2434   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
2435 }
2436 
2437 //===----------------------------------------------------------------------===//
2438 // dpp
2439 //===----------------------------------------------------------------------===//
2440 
2441 bool AMDGPUOperand::isDPPCtrl() const {
2442   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
2443   if (result) {
2444     int64_t Imm = getImm();
2445     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
2446            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
2447            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
2448            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
2449            (Imm == 0x130) ||
2450            (Imm == 0x134) ||
2451            (Imm == 0x138) ||
2452            (Imm == 0x13c) ||
2453            (Imm == 0x140) ||
2454            (Imm == 0x141) ||
2455            (Imm == 0x142) ||
2456            (Imm == 0x143);
2457   }
2458   return false;
2459 }
2460 
2461 AMDGPUAsmParser::OperandMatchResultTy
2462 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
2463   SMLoc S = Parser.getTok().getLoc();
2464   StringRef Prefix;
2465   int64_t Int;
2466 
2467   if (getLexer().getKind() == AsmToken::Identifier) {
2468     Prefix = Parser.getTok().getString();
2469   } else {
2470     return MatchOperand_NoMatch;
2471   }
2472 
2473   if (Prefix == "row_mirror") {
2474     Int = 0x140;
2475   } else if (Prefix == "row_half_mirror") {
2476     Int = 0x141;
2477   } else {
2478     // Check to prevent parseDPPCtrlOps from eating invalid tokens
2479     if (Prefix != "quad_perm"
2480         && Prefix != "row_shl"
2481         && Prefix != "row_shr"
2482         && Prefix != "row_ror"
2483         && Prefix != "wave_shl"
2484         && Prefix != "wave_rol"
2485         && Prefix != "wave_shr"
2486         && Prefix != "wave_ror"
2487         && Prefix != "row_bcast") {
2488       return MatchOperand_NoMatch;
2489     }
2490 
2491     Parser.Lex();
2492     if (getLexer().isNot(AsmToken::Colon))
2493       return MatchOperand_ParseFail;
2494 
2495     if (Prefix == "quad_perm") {
2496       // quad_perm:[%d,%d,%d,%d]
2497       Parser.Lex();
2498       if (getLexer().isNot(AsmToken::LBrac))
2499         return MatchOperand_ParseFail;
2500 
2501       Parser.Lex();
2502       if (getLexer().isNot(AsmToken::Integer))
2503         return MatchOperand_ParseFail;
2504       Int = getLexer().getTok().getIntVal();
2505 
2506       Parser.Lex();
2507       if (getLexer().isNot(AsmToken::Comma))
2508         return MatchOperand_ParseFail;
2509       Parser.Lex();
2510       if (getLexer().isNot(AsmToken::Integer))
2511         return MatchOperand_ParseFail;
2512       Int += (getLexer().getTok().getIntVal() << 2);
2513 
2514       Parser.Lex();
2515       if (getLexer().isNot(AsmToken::Comma))
2516         return MatchOperand_ParseFail;
2517       Parser.Lex();
2518       if (getLexer().isNot(AsmToken::Integer))
2519         return MatchOperand_ParseFail;
2520       Int += (getLexer().getTok().getIntVal() << 4);
2521 
2522       Parser.Lex();
2523       if (getLexer().isNot(AsmToken::Comma))
2524         return MatchOperand_ParseFail;
2525       Parser.Lex();
2526       if (getLexer().isNot(AsmToken::Integer))
2527         return MatchOperand_ParseFail;
2528       Int += (getLexer().getTok().getIntVal() << 6);
2529 
2530       Parser.Lex();
2531       if (getLexer().isNot(AsmToken::RBrac))
2532         return MatchOperand_ParseFail;
2533 
2534     } else {
2535       // sel:%d
2536       Parser.Lex();
2537       if (getLexer().isNot(AsmToken::Integer))
2538         return MatchOperand_ParseFail;
2539       Int = getLexer().getTok().getIntVal();
2540 
2541       if (Prefix == "row_shl") {
2542         Int |= 0x100;
2543       } else if (Prefix == "row_shr") {
2544         Int |= 0x110;
2545       } else if (Prefix == "row_ror") {
2546         Int |= 0x120;
2547       } else if (Prefix == "wave_shl") {
2548         Int = 0x130;
2549       } else if (Prefix == "wave_rol") {
2550         Int = 0x134;
2551       } else if (Prefix == "wave_shr") {
2552         Int = 0x138;
2553       } else if (Prefix == "wave_ror") {
2554         Int = 0x13C;
2555       } else if (Prefix == "row_bcast") {
2556         if (Int == 15) {
2557           Int = 0x142;
2558         } else if (Int == 31) {
2559           Int = 0x143;
2560         }
2561       } else {
2562         return MatchOperand_ParseFail;
2563       }
2564     }
2565   }
2566   Parser.Lex(); // eat last token
2567 
2568   Operands.push_back(AMDGPUOperand::CreateImm(Int, S,
2569                                               AMDGPUOperand::ImmTyDppCtrl));
2570   return MatchOperand_Success;
2571 }
2572 
2573 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
2574   return AMDGPUOperand::CreateImm(0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
2575 }
2576 
2577 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
2578   return AMDGPUOperand::CreateImm(0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
2579 }
2580 
2581 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
2582   return AMDGPUOperand::CreateImm(0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
2583 }
2584 
2585 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
2586   OptionalImmIndexMap OptionalIdx;
2587 
2588   unsigned I = 1;
2589   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2590   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2591     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2592   }
2593 
2594   for (unsigned E = Operands.size(); I != E; ++I) {
2595     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2596     // Add the register arguments
2597     if (Op.isRegOrImmWithInputMods()) {
2598       // Only float modifiers supported in DPP
2599       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
2600     } else if (Op.isDPPCtrl()) {
2601       Op.addImmOperands(Inst, 1);
2602     } else if (Op.isImm()) {
2603       // Handle optional arguments
2604       OptionalIdx[Op.getImmTy()] = I;
2605     } else {
2606       llvm_unreachable("Invalid operand type");
2607     }
2608   }
2609 
2610   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
2611   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
2612   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
2613 }
2614 
2615 //===----------------------------------------------------------------------===//
2616 // sdwa
2617 //===----------------------------------------------------------------------===//
2618 
2619 AMDGPUAsmParser::OperandMatchResultTy
2620 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
2621                               AMDGPUOperand::ImmTy Type) {
2622   SMLoc S = Parser.getTok().getLoc();
2623   StringRef Value;
2624   AMDGPUAsmParser::OperandMatchResultTy res;
2625 
2626   res = parseStringWithPrefix(Prefix, Value);
2627   if (res != MatchOperand_Success) {
2628     return res;
2629   }
2630 
2631   int64_t Int;
2632   Int = StringSwitch<int64_t>(Value)
2633         .Case("BYTE_0", 0)
2634         .Case("BYTE_1", 1)
2635         .Case("BYTE_2", 2)
2636         .Case("BYTE_3", 3)
2637         .Case("WORD_0", 4)
2638         .Case("WORD_1", 5)
2639         .Case("DWORD", 6)
2640         .Default(0xffffffff);
2641   Parser.Lex(); // eat last token
2642 
2643   if (Int == 0xffffffff) {
2644     return MatchOperand_ParseFail;
2645   }
2646 
2647   Operands.push_back(AMDGPUOperand::CreateImm(Int, S, Type));
2648   return MatchOperand_Success;
2649 }
2650 
2651 AMDGPUAsmParser::OperandMatchResultTy
2652 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
2653   SMLoc S = Parser.getTok().getLoc();
2654   StringRef Value;
2655   AMDGPUAsmParser::OperandMatchResultTy res;
2656 
2657   res = parseStringWithPrefix("dst_unused", Value);
2658   if (res != MatchOperand_Success) {
2659     return res;
2660   }
2661 
2662   int64_t Int;
2663   Int = StringSwitch<int64_t>(Value)
2664         .Case("UNUSED_PAD", 0)
2665         .Case("UNUSED_SEXT", 1)
2666         .Case("UNUSED_PRESERVE", 2)
2667         .Default(0xffffffff);
2668   Parser.Lex(); // eat last token
2669 
2670   if (Int == 0xffffffff) {
2671     return MatchOperand_ParseFail;
2672   }
2673 
2674   Operands.push_back(AMDGPUOperand::CreateImm(Int, S,
2675                                               AMDGPUOperand::ImmTySdwaDstUnused));
2676   return MatchOperand_Success;
2677 }
2678 
2679 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
2680   cvtSDWA(Inst, Operands, true);
2681 }
2682 
2683 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
2684   cvtSDWA(Inst, Operands, false);
2685 }
2686 
2687 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
2688                               bool IsVOP1) {
2689   OptionalImmIndexMap OptionalIdx;
2690 
2691   unsigned I = 1;
2692   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2693   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2694     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2695   }
2696 
2697   for (unsigned E = Operands.size(); I != E; ++I) {
2698     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2699     // Add the register arguments
2700     if (Op.isRegOrImmWithInputMods()) {
2701        Op.addRegOrImmWithInputModsOperands(Inst, 2);
2702     } else if (Op.isImm()) {
2703       // Handle optional arguments
2704       OptionalIdx[Op.getImmTy()] = I;
2705     } else {
2706       llvm_unreachable("Invalid operand type");
2707     }
2708   }
2709 
2710   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
2711 
2712   if (Inst.getOpcode() == AMDGPU::V_NOP_sdwa) {
2713     // V_NOP_sdwa has no optional sdwa arguments
2714     return;
2715   }
2716   if (IsVOP1) {
2717     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
2718     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
2719     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
2720   } else { // VOP2
2721     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
2722     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
2723     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
2724     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, 6);
2725   }
2726 }
2727 
2728 /// Force static initialization.
2729 extern "C" void LLVMInitializeAMDGPUAsmParser() {
2730   RegisterMCAsmParser<AMDGPUAsmParser> A(TheAMDGPUTarget);
2731   RegisterMCAsmParser<AMDGPUAsmParser> B(TheGCNTarget);
2732 }
2733 
2734 #define GET_REGISTER_MATCHER
2735 #define GET_MATCHER_IMPLEMENTATION
2736 #include "AMDGPUGenAsmMatcher.inc"
2737 
2738 
2739 // This fuction should be defined after auto-generated include so that we have
2740 // MatchClassKind enum defined
2741 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
2742                                                      unsigned Kind) {
2743   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
2744   // But MatchInstructionImpl() expects to meet token and fails to validate
2745   // operand. This method checks if we are given immediate operand but expect to
2746   // get corresponding token.
2747   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
2748   switch (Kind) {
2749   case MCK_addr64:
2750     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
2751   case MCK_gds:
2752     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
2753   case MCK_glc:
2754     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
2755   case MCK_idxen:
2756     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
2757   case MCK_offen:
2758     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
2759   case MCK_SSrc32:
2760     // When operands have expression values, they will return true for isToken,
2761     // because it is not possible to distinguish between a token and an
2762     // expression at parse time. MatchInstructionImpl() will always try to
2763     // match an operand as a token, when isToken returns true, and when the
2764     // name of the expression is not a valid token, the match will fail,
2765     // so we need to handle it here.
2766     return Operand.isSSrc32() ? Match_Success : Match_InvalidOperand;
2767   default: return Match_InvalidOperand;
2768   }
2769 }
2770