1 //===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ---------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "AMDKernelCodeT.h"
11 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
13 #include "SIDefines.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "Utils/AMDKernelCodeTUtils.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "llvm/ADT/APFloat.h"
18 #include "llvm/ADT/STLExtras.h"
19 #include "llvm/ADT/SmallBitVector.h"
20 #include "llvm/ADT/SmallString.h"
21 #include "llvm/ADT/StringSwitch.h"
22 #include "llvm/ADT/Twine.h"
23 #include "llvm/CodeGen/MachineValueType.h"
24 #include "llvm/MC/MCContext.h"
25 #include "llvm/MC/MCExpr.h"
26 #include "llvm/MC/MCInst.h"
27 #include "llvm/MC/MCInstrInfo.h"
28 #include "llvm/MC/MCParser/MCAsmLexer.h"
29 #include "llvm/MC/MCParser/MCAsmParser.h"
30 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
31 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
32 #include "llvm/MC/MCRegisterInfo.h"
33 #include "llvm/MC/MCStreamer.h"
34 #include "llvm/MC/MCSubtargetInfo.h"
35 #include "llvm/MC/MCSymbolELF.h"
36 #include "llvm/Support/Debug.h"
37 #include "llvm/Support/ELF.h"
38 #include "llvm/Support/SourceMgr.h"
39 #include "llvm/Support/TargetRegistry.h"
40 #include "llvm/Support/raw_ostream.h"
41 #include "llvm/Support/MathExtras.h"
42 
43 using namespace llvm;
44 using namespace llvm::AMDGPU;
45 
46 namespace {
47 
48 class AMDGPUAsmParser;
49 struct OptionalOperand;
50 
51 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
52 
53 //===----------------------------------------------------------------------===//
54 // Operand
55 //===----------------------------------------------------------------------===//
56 
57 class AMDGPUOperand : public MCParsedAsmOperand {
58   enum KindTy {
59     Token,
60     Immediate,
61     Register,
62     Expression
63   } Kind;
64 
65   SMLoc StartLoc, EndLoc;
66   const AMDGPUAsmParser *AsmParser;
67 
68 public:
69   AMDGPUOperand(enum KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
70     : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
71 
72   typedef std::unique_ptr<AMDGPUOperand> Ptr;
73 
74   struct Modifiers {
75     bool Abs;
76     bool Neg;
77     bool Sext;
78 
79     bool hasFPModifiers() const { return Abs || Neg; }
80     bool hasIntModifiers() const { return Sext; }
81     bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
82 
83     int64_t getFPModifiersOperand() const {
84       int64_t Operand = 0;
85       Operand |= Abs ? SISrcMods::ABS : 0;
86       Operand |= Neg ? SISrcMods::NEG : 0;
87       return Operand;
88     }
89 
90     int64_t getIntModifiersOperand() const {
91       int64_t Operand = 0;
92       Operand |= Sext ? SISrcMods::SEXT : 0;
93       return Operand;
94     }
95 
96     int64_t getModifiersOperand() const {
97       assert(!(hasFPModifiers() && hasIntModifiers())
98            && "fp and int modifiers should not be used simultaneously");
99       if (hasFPModifiers()) {
100         return getFPModifiersOperand();
101       } else if (hasIntModifiers()) {
102         return getIntModifiersOperand();
103       } else {
104         return 0;
105       }
106     }
107 
108     friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
109   };
110 
111   enum ImmTy {
112     ImmTyNone,
113     ImmTyGDS,
114     ImmTyOffen,
115     ImmTyIdxen,
116     ImmTyAddr64,
117     ImmTyOffset,
118     ImmTyOffset0,
119     ImmTyOffset1,
120     ImmTyGLC,
121     ImmTySLC,
122     ImmTyTFE,
123     ImmTyClampSI,
124     ImmTyOModSI,
125     ImmTyDppCtrl,
126     ImmTyDppRowMask,
127     ImmTyDppBankMask,
128     ImmTyDppBoundCtrl,
129     ImmTySdwaDstSel,
130     ImmTySdwaSrc0Sel,
131     ImmTySdwaSrc1Sel,
132     ImmTySdwaDstUnused,
133     ImmTyDMask,
134     ImmTyUNorm,
135     ImmTyDA,
136     ImmTyR128,
137     ImmTyLWE,
138     ImmTyHwreg,
139     ImmTySendMsg,
140   };
141 
142   struct TokOp {
143     const char *Data;
144     unsigned Length;
145   };
146 
147   struct ImmOp {
148     int64_t Val;
149     ImmTy Type;
150     bool IsFPImm;
151     Modifiers Mods;
152   };
153 
154   struct RegOp {
155     unsigned RegNo;
156     bool IsForcedVOP3;
157     Modifiers Mods;
158   };
159 
160   union {
161     TokOp Tok;
162     ImmOp Imm;
163     RegOp Reg;
164     const MCExpr *Expr;
165   };
166 
167   bool isToken() const override {
168     if (Kind == Token)
169       return true;
170 
171     if (Kind != Expression || !Expr)
172       return false;
173 
174     // When parsing operands, we can't always tell if something was meant to be
175     // a token, like 'gds', or an expression that references a global variable.
176     // In this case, we assume the string is an expression, and if we need to
177     // interpret is a token, then we treat the symbol name as the token.
178     return isa<MCSymbolRefExpr>(Expr);
179   }
180 
181   bool isImm() const override {
182     return Kind == Immediate;
183   }
184 
185   bool isInlinableImm(MVT type) const;
186   bool isLiteralImm(MVT type) const;
187 
188   bool isRegKind() const {
189     return Kind == Register;
190   }
191 
192   bool isReg() const override {
193     return isRegKind() && !Reg.Mods.hasModifiers();
194   }
195 
196   bool isRegOrImmWithInputMods(MVT type) const {
197     return isRegKind() || isInlinableImm(type);
198   }
199 
200   bool isRegOrImmWithInt32InputMods() const {
201     return isRegOrImmWithInputMods(MVT::i32);
202   }
203 
204   bool isRegOrImmWithInt64InputMods() const {
205     return isRegOrImmWithInputMods(MVT::i64);
206   }
207 
208   bool isRegOrImmWithFP32InputMods() const {
209     return isRegOrImmWithInputMods(MVT::f32);
210   }
211 
212   bool isRegOrImmWithFP64InputMods() const {
213     return isRegOrImmWithInputMods(MVT::f64);
214   }
215 
216   bool isImmTy(ImmTy ImmT) const {
217     return isImm() && Imm.Type == ImmT;
218   }
219 
220   bool isImmModifier() const {
221     return isImm() && Imm.Type != ImmTyNone;
222   }
223 
224   bool isClampSI() const { return isImmTy(ImmTyClampSI); }
225   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
226   bool isDMask() const { return isImmTy(ImmTyDMask); }
227   bool isUNorm() const { return isImmTy(ImmTyUNorm); }
228   bool isDA() const { return isImmTy(ImmTyDA); }
229   bool isR128() const { return isImmTy(ImmTyUNorm); }
230   bool isLWE() const { return isImmTy(ImmTyLWE); }
231   bool isOffen() const { return isImmTy(ImmTyOffen); }
232   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
233   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
234   bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
235   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<16>(getImm()); }
236   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
237   bool isGDS() const { return isImmTy(ImmTyGDS); }
238   bool isGLC() const { return isImmTy(ImmTyGLC); }
239   bool isSLC() const { return isImmTy(ImmTySLC); }
240   bool isTFE() const { return isImmTy(ImmTyTFE); }
241   bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
242   bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
243   bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
244   bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
245   bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
246   bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
247   bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
248 
249   bool isMod() const {
250     return isClampSI() || isOModSI();
251   }
252 
253   bool isRegOrImm() const {
254     return isReg() || isImm();
255   }
256 
257   bool isRegClass(unsigned RCID) const;
258 
259   bool isSCSrcB32() const {
260     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::i32);
261   }
262 
263   bool isSCSrcB64() const {
264     return isRegClass(AMDGPU::SReg_64RegClassID) || isInlinableImm(MVT::i64);
265   }
266 
267   bool isSCSrcF32() const {
268     return isRegClass(AMDGPU::SReg_32RegClassID) || isInlinableImm(MVT::f32);
269   }
270 
271   bool isSCSrcF64() const {
272     return isRegClass(AMDGPU::SReg_64RegClassID) || isInlinableImm(MVT::f64);
273   }
274 
275   bool isSSrcB32() const {
276     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
277   }
278 
279   bool isSSrcB64() const {
280     // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
281     // See isVSrc64().
282     return isSCSrcB64() || isLiteralImm(MVT::i64);
283   }
284 
285   bool isSSrcF32() const {
286     return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
287   }
288 
289   bool isSSrcF64() const {
290     return isSCSrcB64() || isLiteralImm(MVT::f64);
291   }
292 
293   bool isVCSrcB32() const {
294     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::i32);
295   }
296 
297   bool isVCSrcB64() const {
298     return isRegClass(AMDGPU::VS_64RegClassID) || isInlinableImm(MVT::i64);
299   }
300 
301   bool isVCSrcF32() const {
302     return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(MVT::f32);
303   }
304 
305   bool isVCSrcF64() const {
306     return isRegClass(AMDGPU::VS_64RegClassID) || isInlinableImm(MVT::f64);
307   }
308 
309   bool isVSrcB32() const {
310     return isVCSrcF32() || isLiteralImm(MVT::i32);
311   }
312 
313   bool isVSrcB64() const {
314     return isVCSrcF64() || isLiteralImm(MVT::i64);
315   }
316 
317   bool isVSrcF32() const {
318     return isVCSrcF32() || isLiteralImm(MVT::f32);
319   }
320 
321   bool isVSrcF64() const {
322     return isVCSrcF64() || isLiteralImm(MVT::f64);
323   }
324 
325   bool isKImmFP32() const {
326     return isLiteralImm(MVT::f32);
327   }
328 
329   bool isMem() const override {
330     return false;
331   }
332 
333   bool isExpr() const {
334     return Kind == Expression;
335   }
336 
337   bool isSoppBrTarget() const {
338     return isExpr() || isImm();
339   }
340 
341   bool isSWaitCnt() const;
342   bool isHwreg() const;
343   bool isSendMsg() const;
344   bool isSMRDOffset() const;
345   bool isSMRDLiteralOffset() const;
346   bool isDPPCtrl() const;
347 
348   StringRef getExpressionAsToken() const {
349     assert(isExpr());
350     const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
351     return S->getSymbol().getName();
352   }
353 
354 
355   StringRef getToken() const {
356     assert(isToken());
357 
358     if (Kind == Expression)
359       return getExpressionAsToken();
360 
361     return StringRef(Tok.Data, Tok.Length);
362   }
363 
364   int64_t getImm() const {
365     assert(isImm());
366     return Imm.Val;
367   }
368 
369   enum ImmTy getImmTy() const {
370     assert(isImm());
371     return Imm.Type;
372   }
373 
374   unsigned getReg() const override {
375     return Reg.RegNo;
376   }
377 
378   SMLoc getStartLoc() const override {
379     return StartLoc;
380   }
381 
382   SMLoc getEndLoc() const override {
383     return EndLoc;
384   }
385 
386   Modifiers getModifiers() const {
387     assert(isRegKind() || isImmTy(ImmTyNone));
388     return isRegKind() ? Reg.Mods : Imm.Mods;
389   }
390 
391   void setModifiers(Modifiers Mods) {
392     assert(isRegKind() || isImmTy(ImmTyNone));
393     if (isRegKind())
394       Reg.Mods = Mods;
395     else
396       Imm.Mods = Mods;
397   }
398 
399   bool hasModifiers() const {
400     return getModifiers().hasModifiers();
401   }
402 
403   bool hasFPModifiers() const {
404     return getModifiers().hasFPModifiers();
405   }
406 
407   bool hasIntModifiers() const {
408     return getModifiers().hasIntModifiers();
409   }
410 
411   void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
412 
413   void addLiteralImmOperand(MCInst &Inst, int64_t Val) const;
414 
415   void addKImmFP32Operands(MCInst &Inst, unsigned N) const;
416 
417   void addRegOperands(MCInst &Inst, unsigned N) const;
418 
419   void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
420     if (isRegKind())
421       addRegOperands(Inst, N);
422     else if (isExpr())
423       Inst.addOperand(MCOperand::createExpr(Expr));
424     else
425       addImmOperands(Inst, N);
426   }
427 
428   void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
429     Modifiers Mods = getModifiers();
430     Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
431     if (isRegKind()) {
432       addRegOperands(Inst, N);
433     } else {
434       addImmOperands(Inst, N, false);
435     }
436   }
437 
438   void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
439     assert(!hasIntModifiers());
440     addRegOrImmWithInputModsOperands(Inst, N);
441   }
442 
443   void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
444     assert(!hasFPModifiers());
445     addRegOrImmWithInputModsOperands(Inst, N);
446   }
447 
448   void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
449     if (isImm())
450       addImmOperands(Inst, N);
451     else {
452       assert(isExpr());
453       Inst.addOperand(MCOperand::createExpr(Expr));
454     }
455   }
456 
457   void printImmTy(raw_ostream& OS, ImmTy Type) const {
458     switch (Type) {
459     case ImmTyNone: OS << "None"; break;
460     case ImmTyGDS: OS << "GDS"; break;
461     case ImmTyOffen: OS << "Offen"; break;
462     case ImmTyIdxen: OS << "Idxen"; break;
463     case ImmTyAddr64: OS << "Addr64"; break;
464     case ImmTyOffset: OS << "Offset"; break;
465     case ImmTyOffset0: OS << "Offset0"; break;
466     case ImmTyOffset1: OS << "Offset1"; break;
467     case ImmTyGLC: OS << "GLC"; break;
468     case ImmTySLC: OS << "SLC"; break;
469     case ImmTyTFE: OS << "TFE"; break;
470     case ImmTyClampSI: OS << "ClampSI"; break;
471     case ImmTyOModSI: OS << "OModSI"; break;
472     case ImmTyDppCtrl: OS << "DppCtrl"; break;
473     case ImmTyDppRowMask: OS << "DppRowMask"; break;
474     case ImmTyDppBankMask: OS << "DppBankMask"; break;
475     case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
476     case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
477     case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
478     case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
479     case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
480     case ImmTyDMask: OS << "DMask"; break;
481     case ImmTyUNorm: OS << "UNorm"; break;
482     case ImmTyDA: OS << "DA"; break;
483     case ImmTyR128: OS << "R128"; break;
484     case ImmTyLWE: OS << "LWE"; break;
485     case ImmTyHwreg: OS << "Hwreg"; break;
486     case ImmTySendMsg: OS << "SendMsg"; break;
487     }
488   }
489 
490   void print(raw_ostream &OS) const override {
491     switch (Kind) {
492     case Register:
493       OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
494       break;
495     case Immediate:
496       OS << '<' << getImm();
497       if (getImmTy() != ImmTyNone) {
498         OS << " type: "; printImmTy(OS, getImmTy());
499       }
500       OS << " mods: " << Imm.Mods << '>';
501       break;
502     case Token:
503       OS << '\'' << getToken() << '\'';
504       break;
505     case Expression:
506       OS << "<expr " << *Expr << '>';
507       break;
508     }
509   }
510 
511   static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
512                                       int64_t Val, SMLoc Loc,
513                                       enum ImmTy Type = ImmTyNone,
514                                       bool IsFPImm = false) {
515     auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
516     Op->Imm.Val = Val;
517     Op->Imm.IsFPImm = IsFPImm;
518     Op->Imm.Type = Type;
519     Op->Imm.Mods = {false, false, false};
520     Op->StartLoc = Loc;
521     Op->EndLoc = Loc;
522     return Op;
523   }
524 
525   static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
526                                         StringRef Str, SMLoc Loc,
527                                         bool HasExplicitEncodingSize = true) {
528     auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
529     Res->Tok.Data = Str.data();
530     Res->Tok.Length = Str.size();
531     Res->StartLoc = Loc;
532     Res->EndLoc = Loc;
533     return Res;
534   }
535 
536   static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
537                                       unsigned RegNo, SMLoc S,
538                                       SMLoc E,
539                                       bool ForceVOP3) {
540     auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
541     Op->Reg.RegNo = RegNo;
542     Op->Reg.Mods = {false, false, false};
543     Op->Reg.IsForcedVOP3 = ForceVOP3;
544     Op->StartLoc = S;
545     Op->EndLoc = E;
546     return Op;
547   }
548 
549   static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
550                                        const class MCExpr *Expr, SMLoc S) {
551     auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
552     Op->Expr = Expr;
553     Op->StartLoc = S;
554     Op->EndLoc = S;
555     return Op;
556   }
557 };
558 
559 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
560   OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
561   return OS;
562 }
563 
564 //===----------------------------------------------------------------------===//
565 // AsmParser
566 //===----------------------------------------------------------------------===//
567 
568 class AMDGPUAsmParser : public MCTargetAsmParser {
569   const MCInstrInfo &MII;
570   MCAsmParser &Parser;
571 
572   unsigned ForcedEncodingSize;
573   bool ForcedDPP;
574   bool ForcedSDWA;
575 
576   /// @name Auto-generated Match Functions
577   /// {
578 
579 #define GET_ASSEMBLER_HEADER
580 #include "AMDGPUGenAsmMatcher.inc"
581 
582   /// }
583 
584 private:
585   bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
586   bool ParseDirectiveHSACodeObjectVersion();
587   bool ParseDirectiveHSACodeObjectISA();
588   bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
589   bool ParseDirectiveAMDKernelCodeT();
590   bool ParseSectionDirectiveHSAText();
591   bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
592   bool ParseDirectiveAMDGPUHsaKernel();
593   bool ParseDirectiveAMDGPUHsaModuleGlobal();
594   bool ParseDirectiveAMDGPUHsaProgramGlobal();
595   bool ParseSectionDirectiveHSADataGlobalAgent();
596   bool ParseSectionDirectiveHSADataGlobalProgram();
597   bool ParseSectionDirectiveHSARodataReadonlyAgent();
598   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum);
599   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth);
600   void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands, bool IsAtomic, bool IsAtomicReturn);
601 
602 public:
603   enum AMDGPUMatchResultTy {
604     Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
605   };
606 
607   AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
608                const MCInstrInfo &MII,
609                const MCTargetOptions &Options)
610       : MCTargetAsmParser(Options, STI), MII(MII), Parser(_Parser),
611         ForcedEncodingSize(0),
612         ForcedDPP(false),
613         ForcedSDWA(false) {
614     MCAsmParserExtension::Initialize(Parser);
615 
616     if (getSTI().getFeatureBits().none()) {
617       // Set default features.
618       copySTI().ToggleFeature("SOUTHERN_ISLANDS");
619     }
620 
621     setAvailableFeatures(ComputeAvailableFeatures(getSTI().getFeatureBits()));
622 
623     {
624       // TODO: make those pre-defined variables read-only.
625       // Currently there is none suitable machinery in the core llvm-mc for this.
626       // MCSymbol::isRedefinable is intended for another purpose, and
627       // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
628       AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
629       MCContext &Ctx = getContext();
630       MCSymbol *Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
631       Sym->setVariableValue(MCConstantExpr::create(Isa.Major, Ctx));
632       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
633       Sym->setVariableValue(MCConstantExpr::create(Isa.Minor, Ctx));
634       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
635       Sym->setVariableValue(MCConstantExpr::create(Isa.Stepping, Ctx));
636     }
637   }
638 
639   bool isSI() const {
640     return AMDGPU::isSI(getSTI());
641   }
642 
643   bool isCI() const {
644     return AMDGPU::isCI(getSTI());
645   }
646 
647   bool isVI() const {
648     return AMDGPU::isVI(getSTI());
649   }
650 
651   bool hasSGPR102_SGPR103() const {
652     return !isVI();
653   }
654 
655   AMDGPUTargetStreamer &getTargetStreamer() {
656     MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
657     return static_cast<AMDGPUTargetStreamer &>(TS);
658   }
659 
660   const MCRegisterInfo *getMRI() const {
661     // We need this const_cast because for some reason getContext() is not const
662     // in MCAsmParser.
663     return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
664   }
665 
666   const MCInstrInfo *getMII() const {
667     return &MII;
668   }
669 
670   void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
671   void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
672   void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
673 
674   unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
675   bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
676   bool isForcedDPP() const { return ForcedDPP; }
677   bool isForcedSDWA() const { return ForcedSDWA; }
678 
679   std::unique_ptr<AMDGPUOperand> parseRegister();
680   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
681   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
682   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
683                                       unsigned Kind) override;
684   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
685                                OperandVector &Operands, MCStreamer &Out,
686                                uint64_t &ErrorInfo,
687                                bool MatchingInlineAsm) override;
688   bool ParseDirective(AsmToken DirectiveID) override;
689   OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
690   StringRef parseMnemonicSuffix(StringRef Name);
691   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
692                         SMLoc NameLoc, OperandVector &Operands) override;
693   //bool ProcessInstruction(MCInst &Inst);
694 
695   OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
696   OperandMatchResultTy parseIntWithPrefix(const char *Prefix,
697                                           OperandVector &Operands,
698                                           enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
699                                           bool (*ConvertResult)(int64_t&) = 0);
700   OperandMatchResultTy parseNamedBit(const char *Name, OperandVector &Operands,
701                                      enum AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
702   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix, StringRef &Value);
703 
704   OperandMatchResultTy parseImm(OperandVector &Operands);
705   OperandMatchResultTy parseRegOrImm(OperandVector &Operands);
706   OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands);
707   OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands);
708 
709   void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
710   void cvtDS(MCInst &Inst, const OperandVector &Operands);
711 
712   bool parseCnt(int64_t &IntVal);
713   OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
714   OperandMatchResultTy parseHwreg(OperandVector &Operands);
715 
716 private:
717   struct OperandInfoTy {
718     int64_t Id;
719     bool IsSymbolic;
720     OperandInfoTy(int64_t Id_) : Id(Id_), IsSymbolic(false) { }
721   };
722 
723   bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
724   bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
725 public:
726   OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
727 
728   OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
729   OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
730 
731   void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
732   void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
733   void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
734   AMDGPUOperand::Ptr defaultGLC() const;
735   AMDGPUOperand::Ptr defaultSLC() const;
736   AMDGPUOperand::Ptr defaultTFE() const;
737 
738   AMDGPUOperand::Ptr defaultDMask() const;
739   AMDGPUOperand::Ptr defaultUNorm() const;
740   AMDGPUOperand::Ptr defaultDA() const;
741   AMDGPUOperand::Ptr defaultR128() const;
742   AMDGPUOperand::Ptr defaultLWE() const;
743   AMDGPUOperand::Ptr defaultSMRDOffset() const;
744   AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
745 
746   OperandMatchResultTy parseOModOperand(OperandVector &Operands);
747 
748   void cvtId(MCInst &Inst, const OperandVector &Operands);
749   void cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands);
750   void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
751 
752   void cvtMIMG(MCInst &Inst, const OperandVector &Operands);
753   void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
754 
755   OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
756   AMDGPUOperand::Ptr defaultRowMask() const;
757   AMDGPUOperand::Ptr defaultBankMask() const;
758   AMDGPUOperand::Ptr defaultBoundCtrl() const;
759   void cvtDPP(MCInst &Inst, const OperandVector &Operands);
760 
761   OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
762                                     AMDGPUOperand::ImmTy Type);
763   OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
764   void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
765   void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
766   void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
767   void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
768                uint64_t BasicInstType);
769 };
770 
771 struct OptionalOperand {
772   const char *Name;
773   AMDGPUOperand::ImmTy Type;
774   bool IsBit;
775   bool (*ConvertResult)(int64_t&);
776 };
777 
778 }
779 
780 //===----------------------------------------------------------------------===//
781 // Operand
782 //===----------------------------------------------------------------------===//
783 
784 bool AMDGPUOperand::isInlinableImm(MVT type) const {
785   if (!isImmTy(ImmTyNone)) {
786     // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
787     return false;
788   }
789   // TODO: We should avoid using host float here. It would be better to
790   // check the float bit values which is what a few other places do.
791   // We've had bot failures before due to weird NaN support on mips hosts.
792 
793   APInt Literal(64, Imm.Val);
794 
795   if (Imm.IsFPImm) { // We got fp literal token
796     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
797       return AMDGPU::isInlinableLiteral64(Imm.Val, AsmParser->isVI());
798     } else { // Expected 32-bit operand
799       bool lost;
800       APFloat FPLiteral(APFloat::IEEEdouble, Literal);
801       // Convert literal to single precision
802       APFloat::opStatus status = FPLiteral.convert(APFloat::IEEEsingle,
803                                                     APFloat::rmNearestTiesToEven,
804                                                     &lost);
805       // We allow precision lost but not overflow or underflow
806       if (status != APFloat::opOK &&
807           lost &&
808           ((status & APFloat::opOverflow)  != 0 ||
809             (status & APFloat::opUnderflow) != 0)) {
810         return false;
811       }
812       // Check if single precision literal is inlinable
813       return AMDGPU::isInlinableLiteral32(
814               static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
815               AsmParser->isVI());
816     }
817   } else { // We got int literal token
818     if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
819       return AMDGPU::isInlinableLiteral64(Imm.Val, AsmParser->isVI());
820     } else { // Expected 32-bit operand
821       return AMDGPU::isInlinableLiteral32(
822             static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
823             AsmParser->isVI());
824     }
825   }
826   return false;
827 }
828 
829 bool AMDGPUOperand::isLiteralImm(MVT type) const {
830   // Check that this imediate can be added as literal
831   if (!isImmTy(ImmTyNone)) {
832     return false;
833   }
834 
835   APInt Literal(64, Imm.Val);
836 
837   if (Imm.IsFPImm) { // We got fp literal token
838     if (type == MVT::f64) { // Expected 64-bit fp operand
839       // We would set low 64-bits of literal to zeroes but we accept this literals
840       return true;
841     } else if (type == MVT::i64) { // Expected 64-bit int operand
842         // We don't allow fp literals in 64-bit integer instructions. It is
843         // unclear how we should encode them.
844       return false;
845     } else { // Expected 32-bit operand
846       bool lost;
847       APFloat FPLiteral(APFloat::IEEEdouble, Literal);
848       // Convert literal to single precision
849       APFloat::opStatus status = FPLiteral.convert(APFloat::IEEEsingle,
850                                                     APFloat::rmNearestTiesToEven,
851                                                     &lost);
852       // We allow precision lost but not overflow or underflow
853       if (status != APFloat::opOK &&
854           lost &&
855           ((status & APFloat::opOverflow)  != 0 ||
856             (status & APFloat::opUnderflow) != 0)) {
857         return false;
858       }
859       return true;
860     }
861   } else { // We got int literal token
862     APInt HiBits = Literal.getHiBits(32);
863     if (HiBits == 0xffffffff &&
864         (*Literal.getLoBits(32).getRawData() & 0x80000000) != 0) {
865       // If high 32 bits aren't zeroes then they all should be ones and 32nd
866       // bit should be set. So that this 64-bit literal is sign-extension of
867       // 32-bit value.
868       return true;
869     } else if (HiBits == 0) {
870       return true;
871     }
872   }
873   return false;
874 }
875 
876 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
877   return isReg() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
878 }
879 
880 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
881   int64_t Val = Imm.Val;
882   if (isImmTy(ImmTyNone) && ApplyModifiers && Imm.Mods.hasFPModifiers() && Imm.Mods.Neg) {
883     // Apply modifiers to immediate value. Only negate can get here
884     if (Imm.IsFPImm) {
885       APFloat F(BitsToDouble(Val));
886       F.changeSign();
887       Val = F.bitcastToAPInt().getZExtValue();
888     } else {
889       Val = -Val;
890     }
891   }
892 
893   if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()), Inst.getNumOperands())) {
894     addLiteralImmOperand(Inst, Val);
895   } else {
896     Inst.addOperand(MCOperand::createImm(Val));
897   }
898 }
899 
900 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val) const {
901   const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
902   auto OpNum = Inst.getNumOperands();
903   // Check that this operand accepts literals
904   assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
905 
906   APInt Literal(64, Val);
907   auto OpSize = AMDGPU::getRegOperandSize(AsmParser->getMRI(), InstDesc, OpNum); // expected operand size
908 
909   if (Imm.IsFPImm) { // We got fp literal token
910     if (OpSize == 8) { // Expected 64-bit operand
911       // Check if literal is inlinable
912       if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(), AsmParser->isVI())) {
913         Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
914       } else if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
915         // For fp operands we check if low 32 bits are zeros
916         if (Literal.getLoBits(32) != 0) {
917           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
918                             "Can't encode literal as exact 64-bit"
919                             " floating-point operand. Low 32-bits will be"
920                             " set to zero");
921         }
922         Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
923       } else {
924         // We don't allow fp literals in 64-bit integer instructions. It is
925         // unclear how we should encode them. This case should be checked earlier
926         // in predicate methods (isLiteralImm())
927         llvm_unreachable("fp literal in 64-bit integer instruction.");
928       }
929     } else { // Expected 32-bit operand
930       bool lost;
931       APFloat FPLiteral(APFloat::IEEEdouble, Literal);
932       // Convert literal to single precision
933       FPLiteral.convert(APFloat::IEEEsingle, APFloat::rmNearestTiesToEven, &lost);
934       // We allow precision lost but not overflow or underflow. This should be
935       // checked earlier in isLiteralImm()
936       Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
937     }
938   } else { // We got int literal token
939     if (OpSize == 8) { // Expected 64-bit operand
940       auto LiteralVal = Literal.getZExtValue();
941       if (AMDGPU::isInlinableLiteral64(LiteralVal, AsmParser->isVI())) {
942         Inst.addOperand(MCOperand::createImm(LiteralVal));
943         return;
944       }
945     } else { // Expected 32-bit operand
946       auto LiteralVal = static_cast<int32_t>(Literal.getLoBits(32).getZExtValue());
947       if (AMDGPU::isInlinableLiteral32(LiteralVal, AsmParser->isVI())) {
948         Inst.addOperand(MCOperand::createImm(LiteralVal));
949         return;
950       }
951     }
952     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(32).getZExtValue()));
953   }
954 }
955 
956 void AMDGPUOperand::addKImmFP32Operands(MCInst &Inst, unsigned N) const {
957   APInt Literal(64, Imm.Val);
958   if (Imm.IsFPImm) { // We got fp literal
959     bool lost;
960     APFloat FPLiteral(APFloat::IEEEdouble, Literal);
961     FPLiteral.convert(APFloat::IEEEsingle, APFloat::rmNearestTiesToEven, &lost);
962     Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
963   } else { // We got int literal token
964     Inst.addOperand(MCOperand::createImm(Literal.getLoBits(32).getZExtValue()));
965   }
966 }
967 
968 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
969   Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
970 }
971 
972 //===----------------------------------------------------------------------===//
973 // AsmParser
974 //===----------------------------------------------------------------------===//
975 
976 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
977   if (Is == IS_VGPR) {
978     switch (RegWidth) {
979       default: return -1;
980       case 1: return AMDGPU::VGPR_32RegClassID;
981       case 2: return AMDGPU::VReg_64RegClassID;
982       case 3: return AMDGPU::VReg_96RegClassID;
983       case 4: return AMDGPU::VReg_128RegClassID;
984       case 8: return AMDGPU::VReg_256RegClassID;
985       case 16: return AMDGPU::VReg_512RegClassID;
986     }
987   } else if (Is == IS_TTMP) {
988     switch (RegWidth) {
989       default: return -1;
990       case 1: return AMDGPU::TTMP_32RegClassID;
991       case 2: return AMDGPU::TTMP_64RegClassID;
992       case 4: return AMDGPU::TTMP_128RegClassID;
993     }
994   } else if (Is == IS_SGPR) {
995     switch (RegWidth) {
996       default: return -1;
997       case 1: return AMDGPU::SGPR_32RegClassID;
998       case 2: return AMDGPU::SGPR_64RegClassID;
999       case 4: return AMDGPU::SGPR_128RegClassID;
1000       case 8: return AMDGPU::SReg_256RegClassID;
1001       case 16: return AMDGPU::SReg_512RegClassID;
1002     }
1003   }
1004   return -1;
1005 }
1006 
1007 static unsigned getSpecialRegForName(StringRef RegName) {
1008   return StringSwitch<unsigned>(RegName)
1009     .Case("exec", AMDGPU::EXEC)
1010     .Case("vcc", AMDGPU::VCC)
1011     .Case("flat_scratch", AMDGPU::FLAT_SCR)
1012     .Case("m0", AMDGPU::M0)
1013     .Case("scc", AMDGPU::SCC)
1014     .Case("tba", AMDGPU::TBA)
1015     .Case("tma", AMDGPU::TMA)
1016     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1017     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1018     .Case("vcc_lo", AMDGPU::VCC_LO)
1019     .Case("vcc_hi", AMDGPU::VCC_HI)
1020     .Case("exec_lo", AMDGPU::EXEC_LO)
1021     .Case("exec_hi", AMDGPU::EXEC_HI)
1022     .Case("tma_lo", AMDGPU::TMA_LO)
1023     .Case("tma_hi", AMDGPU::TMA_HI)
1024     .Case("tba_lo", AMDGPU::TBA_LO)
1025     .Case("tba_hi", AMDGPU::TBA_HI)
1026     .Default(0);
1027 }
1028 
1029 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) {
1030   auto R = parseRegister();
1031   if (!R) return true;
1032   assert(R->isReg());
1033   RegNo = R->getReg();
1034   StartLoc = R->getStartLoc();
1035   EndLoc = R->getEndLoc();
1036   return false;
1037 }
1038 
1039 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth, RegisterKind RegKind, unsigned Reg1, unsigned RegNum)
1040 {
1041   switch (RegKind) {
1042   case IS_SPECIAL:
1043     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) { Reg = AMDGPU::EXEC; RegWidth = 2; return true; }
1044     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) { Reg = AMDGPU::FLAT_SCR; RegWidth = 2; return true; }
1045     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) { Reg = AMDGPU::VCC; RegWidth = 2; return true; }
1046     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) { Reg = AMDGPU::TBA; RegWidth = 2; return true; }
1047     if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) { Reg = AMDGPU::TMA; RegWidth = 2; return true; }
1048     return false;
1049   case IS_VGPR:
1050   case IS_SGPR:
1051   case IS_TTMP:
1052     if (Reg1 != Reg + RegWidth) { return false; }
1053     RegWidth++;
1054     return true;
1055   default:
1056     assert(false); return false;
1057   }
1058 }
1059 
1060 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg, unsigned& RegNum, unsigned& RegWidth)
1061 {
1062   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1063   if (getLexer().is(AsmToken::Identifier)) {
1064     StringRef RegName = Parser.getTok().getString();
1065     if ((Reg = getSpecialRegForName(RegName))) {
1066       Parser.Lex();
1067       RegKind = IS_SPECIAL;
1068     } else {
1069       unsigned RegNumIndex = 0;
1070       if (RegName[0] == 'v') {
1071         RegNumIndex = 1;
1072         RegKind = IS_VGPR;
1073       } else if (RegName[0] == 's') {
1074         RegNumIndex = 1;
1075         RegKind = IS_SGPR;
1076       } else if (RegName.startswith("ttmp")) {
1077         RegNumIndex = strlen("ttmp");
1078         RegKind = IS_TTMP;
1079       } else {
1080         return false;
1081       }
1082       if (RegName.size() > RegNumIndex) {
1083         // Single 32-bit register: vXX.
1084         if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1085           return false;
1086         Parser.Lex();
1087         RegWidth = 1;
1088       } else {
1089         // Range of registers: v[XX:YY]. ":YY" is optional.
1090         Parser.Lex();
1091         int64_t RegLo, RegHi;
1092         if (getLexer().isNot(AsmToken::LBrac))
1093           return false;
1094         Parser.Lex();
1095 
1096         if (getParser().parseAbsoluteExpression(RegLo))
1097           return false;
1098 
1099         const bool isRBrace = getLexer().is(AsmToken::RBrac);
1100         if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1101           return false;
1102         Parser.Lex();
1103 
1104         if (isRBrace) {
1105           RegHi = RegLo;
1106         } else {
1107           if (getParser().parseAbsoluteExpression(RegHi))
1108             return false;
1109 
1110           if (getLexer().isNot(AsmToken::RBrac))
1111             return false;
1112           Parser.Lex();
1113         }
1114         RegNum = (unsigned) RegLo;
1115         RegWidth = (RegHi - RegLo) + 1;
1116       }
1117     }
1118   } else if (getLexer().is(AsmToken::LBrac)) {
1119     // List of consecutive registers: [s0,s1,s2,s3]
1120     Parser.Lex();
1121     if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth))
1122       return false;
1123     if (RegWidth != 1)
1124       return false;
1125     RegisterKind RegKind1;
1126     unsigned Reg1, RegNum1, RegWidth1;
1127     do {
1128       if (getLexer().is(AsmToken::Comma)) {
1129         Parser.Lex();
1130       } else if (getLexer().is(AsmToken::RBrac)) {
1131         Parser.Lex();
1132         break;
1133       } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1)) {
1134         if (RegWidth1 != 1) {
1135           return false;
1136         }
1137         if (RegKind1 != RegKind) {
1138           return false;
1139         }
1140         if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1141           return false;
1142         }
1143       } else {
1144         return false;
1145       }
1146     } while (true);
1147   } else {
1148     return false;
1149   }
1150   switch (RegKind) {
1151   case IS_SPECIAL:
1152     RegNum = 0;
1153     RegWidth = 1;
1154     break;
1155   case IS_VGPR:
1156   case IS_SGPR:
1157   case IS_TTMP:
1158   {
1159     unsigned Size = 1;
1160     if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1161       // SGPR and TTMP registers must be are aligned. Max required alignment is 4 dwords.
1162       Size = std::min(RegWidth, 4u);
1163     }
1164     if (RegNum % Size != 0)
1165       return false;
1166     RegNum = RegNum / Size;
1167     int RCID = getRegClass(RegKind, RegWidth);
1168     if (RCID == -1)
1169       return false;
1170     const MCRegisterClass RC = TRI->getRegClass(RCID);
1171     if (RegNum >= RC.getNumRegs())
1172       return false;
1173     Reg = RC.getRegister(RegNum);
1174     break;
1175   }
1176 
1177   default:
1178     assert(false); return false;
1179   }
1180 
1181   if (!subtargetHasRegister(*TRI, Reg))
1182     return false;
1183   return true;
1184 }
1185 
1186 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
1187   const auto &Tok = Parser.getTok();
1188   SMLoc StartLoc = Tok.getLoc();
1189   SMLoc EndLoc = Tok.getEndLoc();
1190   RegisterKind RegKind;
1191   unsigned Reg, RegNum, RegWidth;
1192 
1193   if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth)) {
1194     return nullptr;
1195   }
1196   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc, false);
1197 }
1198 
1199 AMDGPUAsmParser::OperandMatchResultTy
1200 AMDGPUAsmParser::parseImm(OperandVector &Operands) {
1201   // TODO: add syntactic sugar for 1/(2*PI)
1202   bool Minus = false;
1203   if (getLexer().getKind() == AsmToken::Minus) {
1204     Minus = true;
1205     Parser.Lex();
1206   }
1207 
1208   SMLoc S = Parser.getTok().getLoc();
1209   switch(getLexer().getKind()) {
1210   case AsmToken::Integer: {
1211     int64_t IntVal;
1212     if (getParser().parseAbsoluteExpression(IntVal))
1213       return MatchOperand_ParseFail;
1214     if (Minus)
1215       IntVal *= -1;
1216     Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
1217     return MatchOperand_Success;
1218   }
1219   case AsmToken::Real: {
1220     int64_t IntVal;
1221     if (getParser().parseAbsoluteExpression(IntVal))
1222       return MatchOperand_ParseFail;
1223 
1224     APFloat F(BitsToDouble(IntVal));
1225     if (Minus)
1226       F.changeSign();
1227     Operands.push_back(
1228         AMDGPUOperand::CreateImm(this, F.bitcastToAPInt().getZExtValue(), S,
1229                                  AMDGPUOperand::ImmTyNone, true));
1230     return MatchOperand_Success;
1231   }
1232   default:
1233     return Minus ? MatchOperand_ParseFail : MatchOperand_NoMatch;
1234   }
1235 }
1236 
1237 AMDGPUAsmParser::OperandMatchResultTy
1238 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands) {
1239   auto res = parseImm(Operands);
1240   if (res != MatchOperand_NoMatch) {
1241     return res;
1242   }
1243 
1244   if (auto R = parseRegister()) {
1245     assert(R->isReg());
1246     R->Reg.IsForcedVOP3 = isForcedVOP3();
1247     Operands.push_back(std::move(R));
1248     return MatchOperand_Success;
1249   }
1250   return MatchOperand_ParseFail;
1251 }
1252 
1253 AMDGPUAsmParser::OperandMatchResultTy
1254 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands) {
1255   // XXX: During parsing we can't determine if minus sign means
1256   // negate-modifier or negative immediate value.
1257   // By default we suppose it is modifier.
1258   bool Negate = false, Abs = false, Abs2 = false;
1259 
1260   if (getLexer().getKind()== AsmToken::Minus) {
1261     Parser.Lex();
1262     Negate = true;
1263   }
1264 
1265   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "abs") {
1266     Parser.Lex();
1267     Abs2 = true;
1268     if (getLexer().isNot(AsmToken::LParen)) {
1269       Error(Parser.getTok().getLoc(), "expected left paren after abs");
1270       return MatchOperand_ParseFail;
1271     }
1272     Parser.Lex();
1273   }
1274 
1275   if (getLexer().getKind() == AsmToken::Pipe) {
1276     if (Abs2) {
1277       Error(Parser.getTok().getLoc(), "expected register or immediate");
1278       return MatchOperand_ParseFail;
1279     }
1280     Parser.Lex();
1281     Abs = true;
1282   }
1283 
1284   auto Res = parseRegOrImm(Operands);
1285   if (Res != MatchOperand_Success) {
1286     return Res;
1287   }
1288 
1289   AMDGPUOperand::Modifiers Mods = {false, false, false};
1290   if (Negate) {
1291     Mods.Neg = true;
1292   }
1293   if (Abs) {
1294     if (getLexer().getKind() != AsmToken::Pipe) {
1295       Error(Parser.getTok().getLoc(), "expected vertical bar");
1296       return MatchOperand_ParseFail;
1297     }
1298     Parser.Lex();
1299     Mods.Abs = true;
1300   }
1301   if (Abs2) {
1302     if (getLexer().isNot(AsmToken::RParen)) {
1303       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1304       return MatchOperand_ParseFail;
1305     }
1306     Parser.Lex();
1307     Mods.Abs = true;
1308   }
1309 
1310   if (Mods.hasFPModifiers()) {
1311     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1312     Op.setModifiers(Mods);
1313   }
1314   return MatchOperand_Success;
1315 }
1316 
1317 AMDGPUAsmParser::OperandMatchResultTy
1318 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands) {
1319   bool Sext = false;
1320 
1321   if (getLexer().getKind() == AsmToken::Identifier && Parser.getTok().getString() == "sext") {
1322     Parser.Lex();
1323     Sext = true;
1324     if (getLexer().isNot(AsmToken::LParen)) {
1325       Error(Parser.getTok().getLoc(), "expected left paren after sext");
1326       return MatchOperand_ParseFail;
1327     }
1328     Parser.Lex();
1329   }
1330 
1331   auto Res = parseRegOrImm(Operands);
1332   if (Res != MatchOperand_Success) {
1333     return Res;
1334   }
1335 
1336   AMDGPUOperand::Modifiers Mods = {false, false, false};
1337   if (Sext) {
1338     if (getLexer().isNot(AsmToken::RParen)) {
1339       Error(Parser.getTok().getLoc(), "expected closing parentheses");
1340       return MatchOperand_ParseFail;
1341     }
1342     Parser.Lex();
1343     Mods.Sext = true;
1344   }
1345 
1346   if (Mods.hasIntModifiers()) {
1347     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
1348     Op.setModifiers(Mods);
1349   }
1350   return MatchOperand_Success;
1351 }
1352 
1353 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
1354 
1355   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
1356 
1357   if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
1358       (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
1359       (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
1360       (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
1361     return Match_InvalidOperand;
1362 
1363   if ((TSFlags & SIInstrFlags::VOP3) &&
1364       (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
1365       getForcedEncodingSize() != 64)
1366     return Match_PreferE32;
1367 
1368   return Match_Success;
1369 }
1370 
1371 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1372                                               OperandVector &Operands,
1373                                               MCStreamer &Out,
1374                                               uint64_t &ErrorInfo,
1375                                               bool MatchingInlineAsm) {
1376   // What asm variants we should check
1377   std::vector<unsigned> MatchedVariants;
1378   if (getForcedEncodingSize() == 32) {
1379     MatchedVariants = {AMDGPUAsmVariants::DEFAULT};
1380   } else if (isForcedVOP3()) {
1381     MatchedVariants = {AMDGPUAsmVariants::VOP3};
1382   } else if (isForcedSDWA()) {
1383     MatchedVariants = {AMDGPUAsmVariants::SDWA};
1384   } else if (isForcedDPP()) {
1385     MatchedVariants = {AMDGPUAsmVariants::DPP};
1386   } else {
1387     MatchedVariants = {AMDGPUAsmVariants::DEFAULT,
1388                        AMDGPUAsmVariants::VOP3,
1389                        AMDGPUAsmVariants::SDWA,
1390                        AMDGPUAsmVariants::DPP};
1391   }
1392 
1393   MCInst Inst;
1394   unsigned Result = Match_Success;
1395   for (auto Variant : MatchedVariants) {
1396     uint64_t EI;
1397     auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
1398                                   Variant);
1399     // We order match statuses from least to most specific. We use most specific
1400     // status as resulting
1401     // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
1402     if ((R == Match_Success) ||
1403         (R == Match_PreferE32) ||
1404         (R == Match_MissingFeature && Result != Match_PreferE32) ||
1405         (R == Match_InvalidOperand && Result != Match_MissingFeature
1406                                    && Result != Match_PreferE32) ||
1407         (R == Match_MnemonicFail   && Result != Match_InvalidOperand
1408                                    && Result != Match_MissingFeature
1409                                    && Result != Match_PreferE32)) {
1410       Result = R;
1411       ErrorInfo = EI;
1412     }
1413     if (R == Match_Success)
1414       break;
1415   }
1416 
1417   switch (Result) {
1418   default: break;
1419   case Match_Success:
1420     Inst.setLoc(IDLoc);
1421     Out.EmitInstruction(Inst, getSTI());
1422     return false;
1423 
1424   case Match_MissingFeature:
1425     return Error(IDLoc, "instruction not supported on this GPU");
1426 
1427   case Match_MnemonicFail:
1428     return Error(IDLoc, "unrecognized instruction mnemonic");
1429 
1430   case Match_InvalidOperand: {
1431     SMLoc ErrorLoc = IDLoc;
1432     if (ErrorInfo != ~0ULL) {
1433       if (ErrorInfo >= Operands.size()) {
1434         return Error(IDLoc, "too few operands for instruction");
1435       }
1436       ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
1437       if (ErrorLoc == SMLoc())
1438         ErrorLoc = IDLoc;
1439     }
1440     return Error(ErrorLoc, "invalid operand for instruction");
1441   }
1442 
1443   case Match_PreferE32:
1444     return Error(IDLoc, "internal error: instruction without _e64 suffix "
1445                         "should be encoded as e32");
1446   }
1447   llvm_unreachable("Implement any new match types added!");
1448 }
1449 
1450 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
1451                                                uint32_t &Minor) {
1452   if (getLexer().isNot(AsmToken::Integer))
1453     return TokError("invalid major version");
1454 
1455   Major = getLexer().getTok().getIntVal();
1456   Lex();
1457 
1458   if (getLexer().isNot(AsmToken::Comma))
1459     return TokError("minor version number required, comma expected");
1460   Lex();
1461 
1462   if (getLexer().isNot(AsmToken::Integer))
1463     return TokError("invalid minor version");
1464 
1465   Minor = getLexer().getTok().getIntVal();
1466   Lex();
1467 
1468   return false;
1469 }
1470 
1471 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
1472 
1473   uint32_t Major;
1474   uint32_t Minor;
1475 
1476   if (ParseDirectiveMajorMinor(Major, Minor))
1477     return true;
1478 
1479   getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
1480   return false;
1481 }
1482 
1483 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
1484 
1485   uint32_t Major;
1486   uint32_t Minor;
1487   uint32_t Stepping;
1488   StringRef VendorName;
1489   StringRef ArchName;
1490 
1491   // If this directive has no arguments, then use the ISA version for the
1492   // targeted GPU.
1493   if (getLexer().is(AsmToken::EndOfStatement)) {
1494     AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(getSTI().getFeatureBits());
1495     getTargetStreamer().EmitDirectiveHSACodeObjectISA(Isa.Major, Isa.Minor,
1496                                                       Isa.Stepping,
1497                                                       "AMD", "AMDGPU");
1498     return false;
1499   }
1500 
1501 
1502   if (ParseDirectiveMajorMinor(Major, Minor))
1503     return true;
1504 
1505   if (getLexer().isNot(AsmToken::Comma))
1506     return TokError("stepping version number required, comma expected");
1507   Lex();
1508 
1509   if (getLexer().isNot(AsmToken::Integer))
1510     return TokError("invalid stepping version");
1511 
1512   Stepping = getLexer().getTok().getIntVal();
1513   Lex();
1514 
1515   if (getLexer().isNot(AsmToken::Comma))
1516     return TokError("vendor name required, comma expected");
1517   Lex();
1518 
1519   if (getLexer().isNot(AsmToken::String))
1520     return TokError("invalid vendor name");
1521 
1522   VendorName = getLexer().getTok().getStringContents();
1523   Lex();
1524 
1525   if (getLexer().isNot(AsmToken::Comma))
1526     return TokError("arch name required, comma expected");
1527   Lex();
1528 
1529   if (getLexer().isNot(AsmToken::String))
1530     return TokError("invalid arch name");
1531 
1532   ArchName = getLexer().getTok().getStringContents();
1533   Lex();
1534 
1535   getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
1536                                                     VendorName, ArchName);
1537   return false;
1538 }
1539 
1540 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
1541                                                amd_kernel_code_t &Header) {
1542   SmallString<40> ErrStr;
1543   raw_svector_ostream Err(ErrStr);
1544   if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
1545     return TokError(Err.str());
1546   }
1547   Lex();
1548   return false;
1549 }
1550 
1551 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
1552 
1553   amd_kernel_code_t Header;
1554   AMDGPU::initDefaultAMDKernelCodeT(Header, getSTI().getFeatureBits());
1555 
1556   while (true) {
1557 
1558     // Lex EndOfStatement.  This is in a while loop, because lexing a comment
1559     // will set the current token to EndOfStatement.
1560     while(getLexer().is(AsmToken::EndOfStatement))
1561       Lex();
1562 
1563     if (getLexer().isNot(AsmToken::Identifier))
1564       return TokError("expected value identifier or .end_amd_kernel_code_t");
1565 
1566     StringRef ID = getLexer().getTok().getIdentifier();
1567     Lex();
1568 
1569     if (ID == ".end_amd_kernel_code_t")
1570       break;
1571 
1572     if (ParseAMDKernelCodeTValue(ID, Header))
1573       return true;
1574   }
1575 
1576   getTargetStreamer().EmitAMDKernelCodeT(Header);
1577 
1578   return false;
1579 }
1580 
1581 bool AMDGPUAsmParser::ParseSectionDirectiveHSAText() {
1582   getParser().getStreamer().SwitchSection(
1583       AMDGPU::getHSATextSection(getContext()));
1584   return false;
1585 }
1586 
1587 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
1588   if (getLexer().isNot(AsmToken::Identifier))
1589     return TokError("expected symbol name");
1590 
1591   StringRef KernelName = Parser.getTok().getString();
1592 
1593   getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
1594                                            ELF::STT_AMDGPU_HSA_KERNEL);
1595   Lex();
1596   return false;
1597 }
1598 
1599 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaModuleGlobal() {
1600   if (getLexer().isNot(AsmToken::Identifier))
1601     return TokError("expected symbol name");
1602 
1603   StringRef GlobalName = Parser.getTok().getIdentifier();
1604 
1605   getTargetStreamer().EmitAMDGPUHsaModuleScopeGlobal(GlobalName);
1606   Lex();
1607   return false;
1608 }
1609 
1610 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaProgramGlobal() {
1611   if (getLexer().isNot(AsmToken::Identifier))
1612     return TokError("expected symbol name");
1613 
1614   StringRef GlobalName = Parser.getTok().getIdentifier();
1615 
1616   getTargetStreamer().EmitAMDGPUHsaProgramScopeGlobal(GlobalName);
1617   Lex();
1618   return false;
1619 }
1620 
1621 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalAgent() {
1622   getParser().getStreamer().SwitchSection(
1623       AMDGPU::getHSADataGlobalAgentSection(getContext()));
1624   return false;
1625 }
1626 
1627 bool AMDGPUAsmParser::ParseSectionDirectiveHSADataGlobalProgram() {
1628   getParser().getStreamer().SwitchSection(
1629       AMDGPU::getHSADataGlobalProgramSection(getContext()));
1630   return false;
1631 }
1632 
1633 bool AMDGPUAsmParser::ParseSectionDirectiveHSARodataReadonlyAgent() {
1634   getParser().getStreamer().SwitchSection(
1635       AMDGPU::getHSARodataReadonlyAgentSection(getContext()));
1636   return false;
1637 }
1638 
1639 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
1640   StringRef IDVal = DirectiveID.getString();
1641 
1642   if (IDVal == ".hsa_code_object_version")
1643     return ParseDirectiveHSACodeObjectVersion();
1644 
1645   if (IDVal == ".hsa_code_object_isa")
1646     return ParseDirectiveHSACodeObjectISA();
1647 
1648   if (IDVal == ".amd_kernel_code_t")
1649     return ParseDirectiveAMDKernelCodeT();
1650 
1651   if (IDVal == ".hsatext")
1652     return ParseSectionDirectiveHSAText();
1653 
1654   if (IDVal == ".amdgpu_hsa_kernel")
1655     return ParseDirectiveAMDGPUHsaKernel();
1656 
1657   if (IDVal == ".amdgpu_hsa_module_global")
1658     return ParseDirectiveAMDGPUHsaModuleGlobal();
1659 
1660   if (IDVal == ".amdgpu_hsa_program_global")
1661     return ParseDirectiveAMDGPUHsaProgramGlobal();
1662 
1663   if (IDVal == ".hsadata_global_agent")
1664     return ParseSectionDirectiveHSADataGlobalAgent();
1665 
1666   if (IDVal == ".hsadata_global_program")
1667     return ParseSectionDirectiveHSADataGlobalProgram();
1668 
1669   if (IDVal == ".hsarodata_readonly_agent")
1670     return ParseSectionDirectiveHSARodataReadonlyAgent();
1671 
1672   return true;
1673 }
1674 
1675 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
1676                                            unsigned RegNo) const {
1677   if (isCI())
1678     return true;
1679 
1680   if (isSI()) {
1681     // No flat_scr
1682     switch (RegNo) {
1683     case AMDGPU::FLAT_SCR:
1684     case AMDGPU::FLAT_SCR_LO:
1685     case AMDGPU::FLAT_SCR_HI:
1686       return false;
1687     default:
1688       return true;
1689     }
1690   }
1691 
1692   // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
1693   // SI/CI have.
1694   for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
1695        R.isValid(); ++R) {
1696     if (*R == RegNo)
1697       return false;
1698   }
1699 
1700   return true;
1701 }
1702 
1703 AMDGPUAsmParser::OperandMatchResultTy
1704 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
1705 
1706   // Try to parse with a custom parser
1707   OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
1708 
1709   // If we successfully parsed the operand or if there as an error parsing,
1710   // we are done.
1711   //
1712   // If we are parsing after we reach EndOfStatement then this means we
1713   // are appending default values to the Operands list.  This is only done
1714   // by custom parser, so we shouldn't continue on to the generic parsing.
1715   if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
1716       getLexer().is(AsmToken::EndOfStatement))
1717     return ResTy;
1718 
1719   ResTy = parseRegOrImm(Operands);
1720 
1721   if (ResTy == MatchOperand_Success)
1722     return ResTy;
1723 
1724   if (getLexer().getKind() == AsmToken::Identifier) {
1725     // If this identifier is a symbol, we want to create an expression for it.
1726     // It is a little difficult to distinguish between a symbol name, and
1727     // an instruction flag like 'gds'.  In order to do this, we parse
1728     // all tokens as expressions and then treate the symbol name as the token
1729     // string when we want to interpret the operand as a token.
1730     const auto &Tok = Parser.getTok();
1731     SMLoc S = Tok.getLoc();
1732     const MCExpr *Expr = nullptr;
1733     if (!Parser.parseExpression(Expr)) {
1734       Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
1735       return MatchOperand_Success;
1736     }
1737 
1738     Operands.push_back(AMDGPUOperand::CreateToken(this, Tok.getString(), Tok.getLoc()));
1739     Parser.Lex();
1740     return MatchOperand_Success;
1741   }
1742   return MatchOperand_NoMatch;
1743 }
1744 
1745 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
1746   // Clear any forced encodings from the previous instruction.
1747   setForcedEncodingSize(0);
1748   setForcedDPP(false);
1749   setForcedSDWA(false);
1750 
1751   if (Name.endswith("_e64")) {
1752     setForcedEncodingSize(64);
1753     return Name.substr(0, Name.size() - 4);
1754   } else if (Name.endswith("_e32")) {
1755     setForcedEncodingSize(32);
1756     return Name.substr(0, Name.size() - 4);
1757   } else if (Name.endswith("_dpp")) {
1758     setForcedDPP(true);
1759     return Name.substr(0, Name.size() - 4);
1760   } else if (Name.endswith("_sdwa")) {
1761     setForcedSDWA(true);
1762     return Name.substr(0, Name.size() - 5);
1763   }
1764   return Name;
1765 }
1766 
1767 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
1768                                        StringRef Name,
1769                                        SMLoc NameLoc, OperandVector &Operands) {
1770   // Add the instruction mnemonic
1771   Name = parseMnemonicSuffix(Name);
1772   Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
1773 
1774   while (!getLexer().is(AsmToken::EndOfStatement)) {
1775     AMDGPUAsmParser::OperandMatchResultTy Res = parseOperand(Operands, Name);
1776 
1777     // Eat the comma or space if there is one.
1778     if (getLexer().is(AsmToken::Comma))
1779       Parser.Lex();
1780 
1781     switch (Res) {
1782       case MatchOperand_Success: break;
1783       case MatchOperand_ParseFail:
1784         Error(getLexer().getLoc(), "failed parsing operand.");
1785         while (!getLexer().is(AsmToken::EndOfStatement)) {
1786           Parser.Lex();
1787         }
1788         return true;
1789       case MatchOperand_NoMatch:
1790         Error(getLexer().getLoc(), "not a valid operand.");
1791         while (!getLexer().is(AsmToken::EndOfStatement)) {
1792           Parser.Lex();
1793         }
1794         return true;
1795     }
1796   }
1797 
1798   return false;
1799 }
1800 
1801 //===----------------------------------------------------------------------===//
1802 // Utility functions
1803 //===----------------------------------------------------------------------===//
1804 
1805 AMDGPUAsmParser::OperandMatchResultTy
1806 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int) {
1807   switch(getLexer().getKind()) {
1808     default: return MatchOperand_NoMatch;
1809     case AsmToken::Identifier: {
1810       StringRef Name = Parser.getTok().getString();
1811       if (!Name.equals(Prefix)) {
1812         return MatchOperand_NoMatch;
1813       }
1814 
1815       Parser.Lex();
1816       if (getLexer().isNot(AsmToken::Colon))
1817         return MatchOperand_ParseFail;
1818 
1819       Parser.Lex();
1820       if (getLexer().isNot(AsmToken::Integer))
1821         return MatchOperand_ParseFail;
1822 
1823       if (getParser().parseAbsoluteExpression(Int))
1824         return MatchOperand_ParseFail;
1825       break;
1826     }
1827   }
1828   return MatchOperand_Success;
1829 }
1830 
1831 AMDGPUAsmParser::OperandMatchResultTy
1832 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
1833                                     enum AMDGPUOperand::ImmTy ImmTy,
1834                                     bool (*ConvertResult)(int64_t&)) {
1835 
1836   SMLoc S = Parser.getTok().getLoc();
1837   int64_t Value = 0;
1838 
1839   AMDGPUAsmParser::OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
1840   if (Res != MatchOperand_Success)
1841     return Res;
1842 
1843   if (ConvertResult && !ConvertResult(Value)) {
1844     return MatchOperand_ParseFail;
1845   }
1846 
1847   Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
1848   return MatchOperand_Success;
1849 }
1850 
1851 AMDGPUAsmParser::OperandMatchResultTy
1852 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
1853                                enum AMDGPUOperand::ImmTy ImmTy) {
1854   int64_t Bit = 0;
1855   SMLoc S = Parser.getTok().getLoc();
1856 
1857   // We are at the end of the statement, and this is a default argument, so
1858   // use a default value.
1859   if (getLexer().isNot(AsmToken::EndOfStatement)) {
1860     switch(getLexer().getKind()) {
1861       case AsmToken::Identifier: {
1862         StringRef Tok = Parser.getTok().getString();
1863         if (Tok == Name) {
1864           Bit = 1;
1865           Parser.Lex();
1866         } else if (Tok.startswith("no") && Tok.endswith(Name)) {
1867           Bit = 0;
1868           Parser.Lex();
1869         } else {
1870           return MatchOperand_NoMatch;
1871         }
1872         break;
1873       }
1874       default:
1875         return MatchOperand_NoMatch;
1876     }
1877   }
1878 
1879   Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
1880   return MatchOperand_Success;
1881 }
1882 
1883 typedef std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalImmIndexMap;
1884 
1885 void addOptionalImmOperand(MCInst& Inst, const OperandVector& Operands,
1886                            OptionalImmIndexMap& OptionalIdx,
1887                            enum AMDGPUOperand::ImmTy ImmT, int64_t Default = 0) {
1888   auto i = OptionalIdx.find(ImmT);
1889   if (i != OptionalIdx.end()) {
1890     unsigned Idx = i->second;
1891     ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
1892   } else {
1893     Inst.addOperand(MCOperand::createImm(Default));
1894   }
1895 }
1896 
1897 AMDGPUAsmParser::OperandMatchResultTy
1898 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
1899   if (getLexer().isNot(AsmToken::Identifier)) {
1900     return MatchOperand_NoMatch;
1901   }
1902   StringRef Tok = Parser.getTok().getString();
1903   if (Tok != Prefix) {
1904     return MatchOperand_NoMatch;
1905   }
1906 
1907   Parser.Lex();
1908   if (getLexer().isNot(AsmToken::Colon)) {
1909     return MatchOperand_ParseFail;
1910   }
1911 
1912   Parser.Lex();
1913   if (getLexer().isNot(AsmToken::Identifier)) {
1914     return MatchOperand_ParseFail;
1915   }
1916 
1917   Value = Parser.getTok().getString();
1918   return MatchOperand_Success;
1919 }
1920 
1921 //===----------------------------------------------------------------------===//
1922 // ds
1923 //===----------------------------------------------------------------------===//
1924 
1925 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
1926                                     const OperandVector &Operands) {
1927 
1928   OptionalImmIndexMap OptionalIdx;
1929 
1930   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1931     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1932 
1933     // Add the register arguments
1934     if (Op.isReg()) {
1935       Op.addRegOperands(Inst, 1);
1936       continue;
1937     }
1938 
1939     // Handle optional arguments
1940     OptionalIdx[Op.getImmTy()] = i;
1941   }
1942 
1943   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
1944   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
1945   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1946 
1947   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1948 }
1949 
1950 void AMDGPUAsmParser::cvtDS(MCInst &Inst, const OperandVector &Operands) {
1951 
1952   std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
1953   bool GDSOnly = false;
1954 
1955   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1956     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1957 
1958     // Add the register arguments
1959     if (Op.isReg()) {
1960       Op.addRegOperands(Inst, 1);
1961       continue;
1962     }
1963 
1964     if (Op.isToken() && Op.getToken() == "gds") {
1965       GDSOnly = true;
1966       continue;
1967     }
1968 
1969     // Handle optional arguments
1970     OptionalIdx[Op.getImmTy()] = i;
1971   }
1972 
1973   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
1974   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1975 
1976   if (!GDSOnly) {
1977     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
1978   }
1979   Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1980 }
1981 
1982 
1983 //===----------------------------------------------------------------------===//
1984 // s_waitcnt
1985 //===----------------------------------------------------------------------===//
1986 
1987 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
1988   StringRef CntName = Parser.getTok().getString();
1989   int64_t CntVal;
1990 
1991   Parser.Lex();
1992   if (getLexer().isNot(AsmToken::LParen))
1993     return true;
1994 
1995   Parser.Lex();
1996   if (getLexer().isNot(AsmToken::Integer))
1997     return true;
1998 
1999   if (getParser().parseAbsoluteExpression(CntVal))
2000     return true;
2001 
2002   if (getLexer().isNot(AsmToken::RParen))
2003     return true;
2004 
2005   Parser.Lex();
2006   if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
2007     Parser.Lex();
2008 
2009   int CntShift;
2010   int CntMask;
2011 
2012   IsaVersion IV = getIsaVersion(getSTI().getFeatureBits());
2013   if (CntName == "vmcnt") {
2014     CntMask = getVmcntMask(IV);
2015     CntShift = getVmcntShift(IV);
2016   } else if (CntName == "expcnt") {
2017     CntMask = getExpcntMask(IV);
2018     CntShift = getExpcntShift(IV);
2019   } else if (CntName == "lgkmcnt") {
2020     CntMask = getLgkmcntMask(IV);
2021     CntShift = getLgkmcntShift(IV);
2022   } else {
2023     return true;
2024   }
2025 
2026   IntVal &= ~(CntMask << CntShift);
2027   IntVal |= (CntVal << CntShift);
2028   return false;
2029 }
2030 
2031 AMDGPUAsmParser::OperandMatchResultTy
2032 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
2033   // Disable all counters by default.
2034   // vmcnt   [3:0]
2035   // expcnt  [6:4]
2036   // lgkmcnt [11:8]
2037   int64_t CntVal = 0xf7f;
2038   SMLoc S = Parser.getTok().getLoc();
2039 
2040   switch(getLexer().getKind()) {
2041     default: return MatchOperand_ParseFail;
2042     case AsmToken::Integer:
2043       // The operand can be an integer value.
2044       if (getParser().parseAbsoluteExpression(CntVal))
2045         return MatchOperand_ParseFail;
2046       break;
2047 
2048     case AsmToken::Identifier:
2049       do {
2050         if (parseCnt(CntVal))
2051           return MatchOperand_ParseFail;
2052       } while(getLexer().isNot(AsmToken::EndOfStatement));
2053       break;
2054   }
2055   Operands.push_back(AMDGPUOperand::CreateImm(this, CntVal, S));
2056   return MatchOperand_Success;
2057 }
2058 
2059 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width) {
2060   using namespace llvm::AMDGPU::Hwreg;
2061 
2062   if (Parser.getTok().getString() != "hwreg")
2063     return true;
2064   Parser.Lex();
2065 
2066   if (getLexer().isNot(AsmToken::LParen))
2067     return true;
2068   Parser.Lex();
2069 
2070   if (getLexer().is(AsmToken::Identifier)) {
2071     HwReg.IsSymbolic = true;
2072     HwReg.Id = ID_UNKNOWN_;
2073     const StringRef tok = Parser.getTok().getString();
2074     for (int i = ID_SYMBOLIC_FIRST_; i < ID_SYMBOLIC_LAST_; ++i) {
2075       if (tok == IdSymbolic[i]) {
2076         HwReg.Id = i;
2077         break;
2078       }
2079     }
2080     Parser.Lex();
2081   } else {
2082     HwReg.IsSymbolic = false;
2083     if (getLexer().isNot(AsmToken::Integer))
2084       return true;
2085     if (getParser().parseAbsoluteExpression(HwReg.Id))
2086       return true;
2087   }
2088 
2089   if (getLexer().is(AsmToken::RParen)) {
2090     Parser.Lex();
2091     return false;
2092   }
2093 
2094   // optional params
2095   if (getLexer().isNot(AsmToken::Comma))
2096     return true;
2097   Parser.Lex();
2098 
2099   if (getLexer().isNot(AsmToken::Integer))
2100     return true;
2101   if (getParser().parseAbsoluteExpression(Offset))
2102     return true;
2103 
2104   if (getLexer().isNot(AsmToken::Comma))
2105     return true;
2106   Parser.Lex();
2107 
2108   if (getLexer().isNot(AsmToken::Integer))
2109     return true;
2110   if (getParser().parseAbsoluteExpression(Width))
2111     return true;
2112 
2113   if (getLexer().isNot(AsmToken::RParen))
2114     return true;
2115   Parser.Lex();
2116 
2117   return false;
2118 }
2119 
2120 AMDGPUAsmParser::OperandMatchResultTy
2121 AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
2122   using namespace llvm::AMDGPU::Hwreg;
2123 
2124   int64_t Imm16Val = 0;
2125   SMLoc S = Parser.getTok().getLoc();
2126 
2127   switch(getLexer().getKind()) {
2128     default: return MatchOperand_NoMatch;
2129     case AsmToken::Integer:
2130       // The operand can be an integer value.
2131       if (getParser().parseAbsoluteExpression(Imm16Val))
2132         return MatchOperand_NoMatch;
2133       if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2134         Error(S, "invalid immediate: only 16-bit values are legal");
2135         // Do not return error code, but create an imm operand anyway and proceed
2136         // to the next operand, if any. That avoids unneccessary error messages.
2137       }
2138       break;
2139 
2140     case AsmToken::Identifier: {
2141         OperandInfoTy HwReg(ID_UNKNOWN_);
2142         int64_t Offset = OFFSET_DEFAULT_;
2143         int64_t Width = WIDTH_M1_DEFAULT_ + 1;
2144         if (parseHwregConstruct(HwReg, Offset, Width))
2145           return MatchOperand_ParseFail;
2146         if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
2147           if (HwReg.IsSymbolic)
2148             Error(S, "invalid symbolic name of hardware register");
2149           else
2150             Error(S, "invalid code of hardware register: only 6-bit values are legal");
2151         }
2152         if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
2153           Error(S, "invalid bit offset: only 5-bit values are legal");
2154         if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
2155           Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
2156         Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
2157       }
2158       break;
2159   }
2160   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
2161   return MatchOperand_Success;
2162 }
2163 
2164 bool AMDGPUOperand::isSWaitCnt() const {
2165   return isImm();
2166 }
2167 
2168 bool AMDGPUOperand::isHwreg() const {
2169   return isImmTy(ImmTyHwreg);
2170 }
2171 
2172 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
2173   using namespace llvm::AMDGPU::SendMsg;
2174 
2175   if (Parser.getTok().getString() != "sendmsg")
2176     return true;
2177   Parser.Lex();
2178 
2179   if (getLexer().isNot(AsmToken::LParen))
2180     return true;
2181   Parser.Lex();
2182 
2183   if (getLexer().is(AsmToken::Identifier)) {
2184     Msg.IsSymbolic = true;
2185     Msg.Id = ID_UNKNOWN_;
2186     const std::string tok = Parser.getTok().getString();
2187     for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
2188       switch(i) {
2189         default: continue; // Omit gaps.
2190         case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:  case ID_SYSMSG: break;
2191       }
2192       if (tok == IdSymbolic[i]) {
2193         Msg.Id = i;
2194         break;
2195       }
2196     }
2197     Parser.Lex();
2198   } else {
2199     Msg.IsSymbolic = false;
2200     if (getLexer().isNot(AsmToken::Integer))
2201       return true;
2202     if (getParser().parseAbsoluteExpression(Msg.Id))
2203       return true;
2204     if (getLexer().is(AsmToken::Integer))
2205       if (getParser().parseAbsoluteExpression(Msg.Id))
2206         Msg.Id = ID_UNKNOWN_;
2207   }
2208   if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
2209     return false;
2210 
2211   if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
2212     if (getLexer().isNot(AsmToken::RParen))
2213       return true;
2214     Parser.Lex();
2215     return false;
2216   }
2217 
2218   if (getLexer().isNot(AsmToken::Comma))
2219     return true;
2220   Parser.Lex();
2221 
2222   assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
2223   Operation.Id = ID_UNKNOWN_;
2224   if (getLexer().is(AsmToken::Identifier)) {
2225     Operation.IsSymbolic = true;
2226     const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
2227     const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
2228     const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
2229     const StringRef Tok = Parser.getTok().getString();
2230     for (int i = F; i < L; ++i) {
2231       if (Tok == S[i]) {
2232         Operation.Id = i;
2233         break;
2234       }
2235     }
2236     Parser.Lex();
2237   } else {
2238     Operation.IsSymbolic = false;
2239     if (getLexer().isNot(AsmToken::Integer))
2240       return true;
2241     if (getParser().parseAbsoluteExpression(Operation.Id))
2242       return true;
2243   }
2244 
2245   if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2246     // Stream id is optional.
2247     if (getLexer().is(AsmToken::RParen)) {
2248       Parser.Lex();
2249       return false;
2250     }
2251 
2252     if (getLexer().isNot(AsmToken::Comma))
2253       return true;
2254     Parser.Lex();
2255 
2256     if (getLexer().isNot(AsmToken::Integer))
2257       return true;
2258     if (getParser().parseAbsoluteExpression(StreamId))
2259       return true;
2260   }
2261 
2262   if (getLexer().isNot(AsmToken::RParen))
2263     return true;
2264   Parser.Lex();
2265   return false;
2266 }
2267 
2268 AMDGPUAsmParser::OperandMatchResultTy
2269 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
2270   using namespace llvm::AMDGPU::SendMsg;
2271 
2272   int64_t Imm16Val = 0;
2273   SMLoc S = Parser.getTok().getLoc();
2274 
2275   switch(getLexer().getKind()) {
2276   default:
2277     return MatchOperand_NoMatch;
2278   case AsmToken::Integer:
2279     // The operand can be an integer value.
2280     if (getParser().parseAbsoluteExpression(Imm16Val))
2281       return MatchOperand_NoMatch;
2282     if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
2283       Error(S, "invalid immediate: only 16-bit values are legal");
2284       // Do not return error code, but create an imm operand anyway and proceed
2285       // to the next operand, if any. That avoids unneccessary error messages.
2286     }
2287     break;
2288   case AsmToken::Identifier: {
2289       OperandInfoTy Msg(ID_UNKNOWN_);
2290       OperandInfoTy Operation(OP_UNKNOWN_);
2291       int64_t StreamId = STREAM_ID_DEFAULT_;
2292       if (parseSendMsgConstruct(Msg, Operation, StreamId))
2293         return MatchOperand_ParseFail;
2294       do {
2295         // Validate and encode message ID.
2296         if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
2297                 || Msg.Id == ID_SYSMSG)) {
2298           if (Msg.IsSymbolic)
2299             Error(S, "invalid/unsupported symbolic name of message");
2300           else
2301             Error(S, "invalid/unsupported code of message");
2302           break;
2303         }
2304         Imm16Val = (Msg.Id << ID_SHIFT_);
2305         // Validate and encode operation ID.
2306         if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
2307           if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
2308             if (Operation.IsSymbolic)
2309               Error(S, "invalid symbolic name of GS_OP");
2310             else
2311               Error(S, "invalid code of GS_OP: only 2-bit values are legal");
2312             break;
2313           }
2314           if (Operation.Id == OP_GS_NOP
2315               && Msg.Id != ID_GS_DONE) {
2316             Error(S, "invalid GS_OP: NOP is for GS_DONE only");
2317             break;
2318           }
2319           Imm16Val |= (Operation.Id << OP_SHIFT_);
2320         }
2321         if (Msg.Id == ID_SYSMSG) {
2322           if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
2323             if (Operation.IsSymbolic)
2324               Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
2325             else
2326               Error(S, "invalid/unsupported code of SYSMSG_OP");
2327             break;
2328           }
2329           Imm16Val |= (Operation.Id << OP_SHIFT_);
2330         }
2331         // Validate and encode stream ID.
2332         if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
2333           if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
2334             Error(S, "invalid stream id: only 2-bit values are legal");
2335             break;
2336           }
2337           Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
2338         }
2339       } while (0);
2340     }
2341     break;
2342   }
2343   Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
2344   return MatchOperand_Success;
2345 }
2346 
2347 bool AMDGPUOperand::isSendMsg() const {
2348   return isImmTy(ImmTySendMsg);
2349 }
2350 
2351 //===----------------------------------------------------------------------===//
2352 // sopp branch targets
2353 //===----------------------------------------------------------------------===//
2354 
2355 AMDGPUAsmParser::OperandMatchResultTy
2356 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
2357   SMLoc S = Parser.getTok().getLoc();
2358 
2359   switch (getLexer().getKind()) {
2360     default: return MatchOperand_ParseFail;
2361     case AsmToken::Integer: {
2362       int64_t Imm;
2363       if (getParser().parseAbsoluteExpression(Imm))
2364         return MatchOperand_ParseFail;
2365       Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
2366       return MatchOperand_Success;
2367     }
2368 
2369     case AsmToken::Identifier:
2370       Operands.push_back(AMDGPUOperand::CreateExpr(this,
2371           MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
2372                                   Parser.getTok().getString()), getContext()), S));
2373       Parser.Lex();
2374       return MatchOperand_Success;
2375   }
2376 }
2377 
2378 //===----------------------------------------------------------------------===//
2379 // mubuf
2380 //===----------------------------------------------------------------------===//
2381 
2382 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
2383   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
2384 }
2385 
2386 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
2387   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
2388 }
2389 
2390 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const {
2391   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE);
2392 }
2393 
2394 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
2395                                const OperandVector &Operands,
2396                                bool IsAtomic, bool IsAtomicReturn) {
2397   OptionalImmIndexMap OptionalIdx;
2398   assert(IsAtomicReturn ? IsAtomic : true);
2399 
2400   for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
2401     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
2402 
2403     // Add the register arguments
2404     if (Op.isReg()) {
2405       Op.addRegOperands(Inst, 1);
2406       continue;
2407     }
2408 
2409     // Handle the case where soffset is an immediate
2410     if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
2411       Op.addImmOperands(Inst, 1);
2412       continue;
2413     }
2414 
2415     // Handle tokens like 'offen' which are sometimes hard-coded into the
2416     // asm string.  There are no MCInst operands for these.
2417     if (Op.isToken()) {
2418       continue;
2419     }
2420     assert(Op.isImm());
2421 
2422     // Handle optional arguments
2423     OptionalIdx[Op.getImmTy()] = i;
2424   }
2425 
2426   // Copy $vdata_in operand and insert as $vdata for MUBUF_Atomic RTN insns.
2427   if (IsAtomicReturn) {
2428     MCInst::iterator I = Inst.begin(); // $vdata_in is always at the beginning.
2429     Inst.insert(I, *I);
2430   }
2431 
2432   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
2433   if (!IsAtomic) { // glc is hard-coded.
2434     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2435   }
2436   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2437   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2438 }
2439 
2440 //===----------------------------------------------------------------------===//
2441 // mimg
2442 //===----------------------------------------------------------------------===//
2443 
2444 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands) {
2445   unsigned I = 1;
2446   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2447   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2448     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2449   }
2450 
2451   OptionalImmIndexMap OptionalIdx;
2452 
2453   for (unsigned E = Operands.size(); I != E; ++I) {
2454     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2455 
2456     // Add the register arguments
2457     if (Op.isRegOrImm()) {
2458       Op.addRegOrImmOperands(Inst, 1);
2459       continue;
2460     } else if (Op.isImmModifier()) {
2461       OptionalIdx[Op.getImmTy()] = I;
2462     } else {
2463       assert(false);
2464     }
2465   }
2466 
2467   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2468   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2469   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2470   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2471   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2472   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2473   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2474   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2475 }
2476 
2477 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
2478   unsigned I = 1;
2479   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2480   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2481     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2482   }
2483 
2484   // Add src, same as dst
2485   ((AMDGPUOperand &)*Operands[I]).addRegOperands(Inst, 1);
2486 
2487   OptionalImmIndexMap OptionalIdx;
2488 
2489   for (unsigned E = Operands.size(); I != E; ++I) {
2490     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2491 
2492     // Add the register arguments
2493     if (Op.isRegOrImm()) {
2494       Op.addRegOrImmOperands(Inst, 1);
2495       continue;
2496     } else if (Op.isImmModifier()) {
2497       OptionalIdx[Op.getImmTy()] = I;
2498     } else {
2499       assert(false);
2500     }
2501   }
2502 
2503   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
2504   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
2505   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
2506   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
2507   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128);
2508   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
2509   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
2510   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
2511 }
2512 
2513 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const {
2514   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask);
2515 }
2516 
2517 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const {
2518   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm);
2519 }
2520 
2521 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const {
2522   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA);
2523 }
2524 
2525 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const {
2526   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128);
2527 }
2528 
2529 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const {
2530   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE);
2531 }
2532 
2533 //===----------------------------------------------------------------------===//
2534 // smrd
2535 //===----------------------------------------------------------------------===//
2536 
2537 bool AMDGPUOperand::isSMRDOffset() const {
2538 
2539   // FIXME: Support 20-bit offsets on VI.  We need to to pass subtarget
2540   // information here.
2541   return isImm() && isUInt<8>(getImm());
2542 }
2543 
2544 bool AMDGPUOperand::isSMRDLiteralOffset() const {
2545   // 32-bit literals are only supported on CI and we only want to use them
2546   // when the offset is > 8-bits.
2547   return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
2548 }
2549 
2550 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset() const {
2551   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2552 }
2553 
2554 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
2555   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
2556 }
2557 
2558 //===----------------------------------------------------------------------===//
2559 // vop3
2560 //===----------------------------------------------------------------------===//
2561 
2562 static bool ConvertOmodMul(int64_t &Mul) {
2563   if (Mul != 1 && Mul != 2 && Mul != 4)
2564     return false;
2565 
2566   Mul >>= 1;
2567   return true;
2568 }
2569 
2570 static bool ConvertOmodDiv(int64_t &Div) {
2571   if (Div == 1) {
2572     Div = 0;
2573     return true;
2574   }
2575 
2576   if (Div == 2) {
2577     Div = 3;
2578     return true;
2579   }
2580 
2581   return false;
2582 }
2583 
2584 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
2585   if (BoundCtrl == 0) {
2586     BoundCtrl = 1;
2587     return true;
2588   } else if (BoundCtrl == -1) {
2589     BoundCtrl = 0;
2590     return true;
2591   }
2592   return false;
2593 }
2594 
2595 // Note: the order in this table matches the order of operands in AsmString.
2596 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
2597   {"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
2598   {"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
2599   {"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
2600   {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
2601   {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
2602   {"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
2603   {"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
2604   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
2605   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
2606   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
2607   {"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
2608   {"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
2609   {"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
2610   {"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
2611   {"r128",    AMDGPUOperand::ImmTyR128,  true, nullptr},
2612   {"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
2613   {"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
2614   {"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
2615   {"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
2616   {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
2617   {"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
2618   {"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
2619   {"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
2620   {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
2621 };
2622 
2623 AMDGPUAsmParser::OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
2624   OperandMatchResultTy res;
2625   for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
2626     // try to parse any optional operand here
2627     if (Op.IsBit) {
2628       res = parseNamedBit(Op.Name, Operands, Op.Type);
2629     } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
2630       res = parseOModOperand(Operands);
2631     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
2632                Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
2633                Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
2634       res = parseSDWASel(Operands, Op.Name, Op.Type);
2635     } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
2636       res = parseSDWADstUnused(Operands);
2637     } else {
2638       res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
2639     }
2640     if (res != MatchOperand_NoMatch) {
2641       return res;
2642     }
2643   }
2644   return MatchOperand_NoMatch;
2645 }
2646 
2647 AMDGPUAsmParser::OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands)
2648 {
2649   StringRef Name = Parser.getTok().getString();
2650   if (Name == "mul") {
2651     return parseIntWithPrefix("mul", Operands, AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
2652   } else if (Name == "div") {
2653     return parseIntWithPrefix("div", Operands, AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
2654   } else {
2655     return MatchOperand_NoMatch;
2656   }
2657 }
2658 
2659 void AMDGPUAsmParser::cvtId(MCInst &Inst, const OperandVector &Operands) {
2660   unsigned I = 1;
2661   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2662   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2663     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2664   }
2665   for (unsigned E = Operands.size(); I != E; ++I)
2666     ((AMDGPUOperand &)*Operands[I]).addRegOrImmOperands(Inst, 1);
2667 }
2668 
2669 void AMDGPUAsmParser::cvtVOP3_2_mod(MCInst &Inst, const OperandVector &Operands) {
2670   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
2671   if (TSFlags & SIInstrFlags::VOP3) {
2672     cvtVOP3(Inst, Operands);
2673   } else {
2674     cvtId(Inst, Operands);
2675   }
2676 }
2677 
2678 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
2679   OptionalImmIndexMap OptionalIdx;
2680   unsigned I = 1;
2681   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2682   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2683     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2684   }
2685 
2686   for (unsigned E = Operands.size(); I != E; ++I) {
2687     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2688     if (Desc.OpInfo[Inst.getNumOperands()].OperandType == AMDGPU::OPERAND_INPUT_MODS) {
2689       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
2690     } else if (Op.isImm()) {
2691       OptionalIdx[Op.getImmTy()] = I;
2692     } else {
2693       assert(false);
2694     }
2695   }
2696 
2697   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
2698   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
2699 }
2700 
2701 //===----------------------------------------------------------------------===//
2702 // dpp
2703 //===----------------------------------------------------------------------===//
2704 
2705 bool AMDGPUOperand::isDPPCtrl() const {
2706   bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
2707   if (result) {
2708     int64_t Imm = getImm();
2709     return ((Imm >= 0x000) && (Imm <= 0x0ff)) ||
2710            ((Imm >= 0x101) && (Imm <= 0x10f)) ||
2711            ((Imm >= 0x111) && (Imm <= 0x11f)) ||
2712            ((Imm >= 0x121) && (Imm <= 0x12f)) ||
2713            (Imm == 0x130) ||
2714            (Imm == 0x134) ||
2715            (Imm == 0x138) ||
2716            (Imm == 0x13c) ||
2717            (Imm == 0x140) ||
2718            (Imm == 0x141) ||
2719            (Imm == 0x142) ||
2720            (Imm == 0x143);
2721   }
2722   return false;
2723 }
2724 
2725 AMDGPUAsmParser::OperandMatchResultTy
2726 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
2727   SMLoc S = Parser.getTok().getLoc();
2728   StringRef Prefix;
2729   int64_t Int;
2730 
2731   if (getLexer().getKind() == AsmToken::Identifier) {
2732     Prefix = Parser.getTok().getString();
2733   } else {
2734     return MatchOperand_NoMatch;
2735   }
2736 
2737   if (Prefix == "row_mirror") {
2738     Int = 0x140;
2739     Parser.Lex();
2740   } else if (Prefix == "row_half_mirror") {
2741     Int = 0x141;
2742     Parser.Lex();
2743   } else {
2744     // Check to prevent parseDPPCtrlOps from eating invalid tokens
2745     if (Prefix != "quad_perm"
2746         && Prefix != "row_shl"
2747         && Prefix != "row_shr"
2748         && Prefix != "row_ror"
2749         && Prefix != "wave_shl"
2750         && Prefix != "wave_rol"
2751         && Prefix != "wave_shr"
2752         && Prefix != "wave_ror"
2753         && Prefix != "row_bcast") {
2754       return MatchOperand_NoMatch;
2755     }
2756 
2757     Parser.Lex();
2758     if (getLexer().isNot(AsmToken::Colon))
2759       return MatchOperand_ParseFail;
2760 
2761     if (Prefix == "quad_perm") {
2762       // quad_perm:[%d,%d,%d,%d]
2763       Parser.Lex();
2764       if (getLexer().isNot(AsmToken::LBrac))
2765         return MatchOperand_ParseFail;
2766       Parser.Lex();
2767 
2768       if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
2769         return MatchOperand_ParseFail;
2770 
2771       for (int i = 0; i < 3; ++i) {
2772         if (getLexer().isNot(AsmToken::Comma))
2773           return MatchOperand_ParseFail;
2774         Parser.Lex();
2775 
2776         int64_t Temp;
2777         if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
2778           return MatchOperand_ParseFail;
2779         const int shift = i*2 + 2;
2780         Int += (Temp << shift);
2781       }
2782 
2783       if (getLexer().isNot(AsmToken::RBrac))
2784         return MatchOperand_ParseFail;
2785       Parser.Lex();
2786 
2787     } else {
2788       // sel:%d
2789       Parser.Lex();
2790       if (getParser().parseAbsoluteExpression(Int))
2791         return MatchOperand_ParseFail;
2792 
2793       if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
2794         Int |= 0x100;
2795       } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
2796         Int |= 0x110;
2797       } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
2798         Int |= 0x120;
2799       } else if (Prefix == "wave_shl" && 1 == Int) {
2800         Int = 0x130;
2801       } else if (Prefix == "wave_rol" && 1 == Int) {
2802         Int = 0x134;
2803       } else if (Prefix == "wave_shr" && 1 == Int) {
2804         Int = 0x138;
2805       } else if (Prefix == "wave_ror" && 1 == Int) {
2806         Int = 0x13C;
2807       } else if (Prefix == "row_bcast") {
2808         if (Int == 15) {
2809           Int = 0x142;
2810         } else if (Int == 31) {
2811           Int = 0x143;
2812         } else {
2813           return MatchOperand_ParseFail;
2814         }
2815       } else {
2816         return MatchOperand_ParseFail;
2817       }
2818     }
2819   }
2820 
2821   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
2822   return MatchOperand_Success;
2823 }
2824 
2825 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
2826   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
2827 }
2828 
2829 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
2830   return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
2831 }
2832 
2833 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
2834   return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
2835 }
2836 
2837 void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands) {
2838   OptionalImmIndexMap OptionalIdx;
2839 
2840   unsigned I = 1;
2841   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2842   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2843     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2844   }
2845 
2846   for (unsigned E = Operands.size(); I != E; ++I) {
2847     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2848     // Add the register arguments
2849     if (Desc.OpInfo[Inst.getNumOperands()].OperandType == AMDGPU::OPERAND_INPUT_MODS) {
2850       Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
2851     } else if (Op.isDPPCtrl()) {
2852       Op.addImmOperands(Inst, 1);
2853     } else if (Op.isImm()) {
2854       // Handle optional arguments
2855       OptionalIdx[Op.getImmTy()] = I;
2856     } else {
2857       llvm_unreachable("Invalid operand type");
2858     }
2859   }
2860 
2861   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
2862   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
2863   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
2864 }
2865 
2866 //===----------------------------------------------------------------------===//
2867 // sdwa
2868 //===----------------------------------------------------------------------===//
2869 
2870 AMDGPUAsmParser::OperandMatchResultTy
2871 AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
2872                               AMDGPUOperand::ImmTy Type) {
2873   SMLoc S = Parser.getTok().getLoc();
2874   StringRef Value;
2875   AMDGPUAsmParser::OperandMatchResultTy res;
2876 
2877   res = parseStringWithPrefix(Prefix, Value);
2878   if (res != MatchOperand_Success) {
2879     return res;
2880   }
2881 
2882   int64_t Int;
2883   Int = StringSwitch<int64_t>(Value)
2884         .Case("BYTE_0", 0)
2885         .Case("BYTE_1", 1)
2886         .Case("BYTE_2", 2)
2887         .Case("BYTE_3", 3)
2888         .Case("WORD_0", 4)
2889         .Case("WORD_1", 5)
2890         .Case("DWORD", 6)
2891         .Default(0xffffffff);
2892   Parser.Lex(); // eat last token
2893 
2894   if (Int == 0xffffffff) {
2895     return MatchOperand_ParseFail;
2896   }
2897 
2898   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
2899   return MatchOperand_Success;
2900 }
2901 
2902 AMDGPUAsmParser::OperandMatchResultTy
2903 AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
2904   SMLoc S = Parser.getTok().getLoc();
2905   StringRef Value;
2906   AMDGPUAsmParser::OperandMatchResultTy res;
2907 
2908   res = parseStringWithPrefix("dst_unused", Value);
2909   if (res != MatchOperand_Success) {
2910     return res;
2911   }
2912 
2913   int64_t Int;
2914   Int = StringSwitch<int64_t>(Value)
2915         .Case("UNUSED_PAD", 0)
2916         .Case("UNUSED_SEXT", 1)
2917         .Case("UNUSED_PRESERVE", 2)
2918         .Default(0xffffffff);
2919   Parser.Lex(); // eat last token
2920 
2921   if (Int == 0xffffffff) {
2922     return MatchOperand_ParseFail;
2923   }
2924 
2925   Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
2926   return MatchOperand_Success;
2927 }
2928 
2929 void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
2930   cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
2931 }
2932 
2933 void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
2934   cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
2935 }
2936 
2937 void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
2938   cvtSDWA(Inst, Operands, SIInstrFlags::VOPC);
2939 }
2940 
2941 void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
2942                               uint64_t BasicInstType) {
2943   OptionalImmIndexMap OptionalIdx;
2944 
2945   unsigned I = 1;
2946   const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2947   for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
2948     ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
2949   }
2950 
2951   for (unsigned E = Operands.size(); I != E; ++I) {
2952     AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
2953     // Add the register arguments
2954     if (BasicInstType == SIInstrFlags::VOPC &&
2955         Op.isReg() &&
2956         Op.Reg.RegNo == AMDGPU::VCC) {
2957       // VOPC sdwa use "vcc" token as dst. Skip it.
2958       continue;
2959     } else if (Desc.OpInfo[Inst.getNumOperands()].OperandType == AMDGPU::OPERAND_INPUT_MODS) {
2960       Op.addRegOrImmWithInputModsOperands(Inst, 2);
2961     } else if (Op.isImm()) {
2962       // Handle optional arguments
2963       OptionalIdx[Op.getImmTy()] = I;
2964     } else {
2965       llvm_unreachable("Invalid operand type");
2966     }
2967   }
2968 
2969   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
2970 
2971   if (Inst.getOpcode() == AMDGPU::V_NOP_sdwa) {
2972     // V_NOP_sdwa has no optional sdwa arguments
2973     return;
2974   }
2975   switch (BasicInstType) {
2976   case SIInstrFlags::VOP1: {
2977     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
2978     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
2979     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
2980     break;
2981   }
2982   case SIInstrFlags::VOP2: {
2983     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, 6);
2984     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, 2);
2985     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
2986     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, 6);
2987     break;
2988   }
2989   case SIInstrFlags::VOPC: {
2990     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, 6);
2991     addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, 6);
2992     break;
2993   }
2994   default:
2995     llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed");
2996   }
2997 }
2998 
2999 /// Force static initialization.
3000 extern "C" void LLVMInitializeAMDGPUAsmParser() {
3001   RegisterMCAsmParser<AMDGPUAsmParser> A(TheAMDGPUTarget);
3002   RegisterMCAsmParser<AMDGPUAsmParser> B(TheGCNTarget);
3003 }
3004 
3005 #define GET_REGISTER_MATCHER
3006 #define GET_MATCHER_IMPLEMENTATION
3007 #include "AMDGPUGenAsmMatcher.inc"
3008 
3009 
3010 // This fuction should be defined after auto-generated include so that we have
3011 // MatchClassKind enum defined
3012 unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
3013                                                      unsigned Kind) {
3014   // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
3015   // But MatchInstructionImpl() expects to meet token and fails to validate
3016   // operand. This method checks if we are given immediate operand but expect to
3017   // get corresponding token.
3018   AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
3019   switch (Kind) {
3020   case MCK_addr64:
3021     return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
3022   case MCK_gds:
3023     return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
3024   case MCK_glc:
3025     return Operand.isGLC() ? Match_Success : Match_InvalidOperand;
3026   case MCK_idxen:
3027     return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
3028   case MCK_offen:
3029     return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
3030   case MCK_SSrcB32:
3031     // When operands have expression values, they will return true for isToken,
3032     // because it is not possible to distinguish between a token and an
3033     // expression at parse time. MatchInstructionImpl() will always try to
3034     // match an operand as a token, when isToken returns true, and when the
3035     // name of the expression is not a valid token, the match will fail,
3036     // so we need to handle it here.
3037     return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
3038   case MCK_SSrcF32:
3039     return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
3040   case MCK_SoppBrTarget:
3041     return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
3042   default: return Match_InvalidOperand;
3043   }
3044 }
3045